diff --git a/.clang-tidy b/.clang-tidy index 1658bf3d379..996a4881cad 100644 --- a/.clang-tidy +++ b/.clang-tidy @@ -1 +1,5 @@ -Checks: -*,modernize-loop-convert,modernize-use-bool-literals,modernize-deprecated-headers,performance-unnecessary-value-param,performance-faster-string-find,modernize-raw-string-literal,modernize-redundant-void-arg,modernize-use-nullptr,modernize-use-override +Checks: -*,modernize-use-default-member-init,modernize-loop-convert,modernize-use-bool-literals,modernize-deprecated-headers,performance-unnecessary-value-param,performance-faster-string-find,modernize-raw-string-literal,modernize-redundant-void-arg,modernize-use-nullptr,modernize-use-override +CheckOptions: + - key: modernize-use-default-member-init.UseAssignment + value: '1' +HeaderFilterRegex: (?!(\/openssl\/|\/pcre\/|\/lib\/yaml\/)).* diff --git a/.gitignore b/.gitignore index acffd829f9a..3b85d100b63 100644 --- a/.gitignore +++ b/.gitignore @@ -18,6 +18,7 @@ Makefile.in .project .cproject +.idea .diags.log.meta @@ -37,6 +38,7 @@ build/lt~obsolete.m4 Makefile Makefile-pl +config.cache config.status config.nice config.h @@ -85,6 +87,7 @@ src/tscore/test_Regex src/tscore/test_X509HostnameValidator src/tscore/test_tscore src/tscpp/util/test_tscpputil +lib/records/test_librecords lib/perl/lib/Apache/TS.pm iocore/net/test_certlookup @@ -96,15 +99,19 @@ iocore/eventsystem/test_MIOBufferWriter iocore/hostdb/test_RefCountCache proxy/hdrs/test_mime +proxy/hdrs/test_proxy_hdrs proxy/http/test_proxy_http proxy/http2/test_Huffmancode proxy/http2/test_Http2DependencyTree proxy/http2/test_HPACK proxy/http2/hpack-tests/results proxy/logging/test_LogUtils +proxy/logging/test_LogUtils2 plugins/header_rewrite/header_rewrite_test +plugins/experimental/cookie_remap/test_cookiejar plugins/experimental/esi/*_test +plugins/experimental/slice/test_* plugins/experimental/sslheaders/test_sslheaders plugins/s3_auth/test_s3auth @@ -115,6 +122,8 @@ plugins/esi/processor_test plugins/esi/utils_test plugins/esi/vars_test +plugins/experimental/uri_signing/test_uri_signing + mgmt/api/traffic_api_cli_remote mgmt/tools/traffic_mcast_snoop mgmt/tools/traffic_net_config @@ -169,3 +178,6 @@ RELEASE # autest tests/env-test/ + +iocore/cache/test_* +iocore/cache/test/var/trafficserver/cache.db diff --git a/.perltidyrc b/.perltidyrc new file mode 100644 index 00000000000..86440f5ae93 --- /dev/null +++ b/.perltidyrc @@ -0,0 +1,14 @@ + # This is a simple of a .perltidyrc configuration file + -l=132 # Line length + -i=4 # 4-space indentation + -nlp # Line up params + -ce # cuddle the braces + -tso # Tight secret ops + -nsfs # No space for semicolon + -pt=2 # tight parens + -bt=2 # tight braces + -sbt=2 # tight brackets + -bbt=2 # tight code brackets + -nbbc # No blank lines before comment lines + -otr # No break between a comma and an opening token + -sbl # Empty lane for sub's opening brace diff --git a/.ripgreprc b/.ripgreprc new file mode 100644 index 00000000000..7a5ea0ac2f9 --- /dev/null +++ b/.ripgreprc @@ -0,0 +1,5 @@ +--color=auto +--no-heading +--smart-case +--line-number +--column diff --git a/CMakeLists.txt b/CMakeLists.txt index b8abf0eac84..68504fba114 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -83,12 +83,16 @@ CPP_LIB(tscppapi src/tscpp/api include/tscpp/api) CC_EXEC(test_tscore src/tscore/unit_tests) CC_EXEC(test_tsutil src/tscpp/util/unit_tests) +CC_EXEC(test_librecords lib/records/unit_tests) CPP_LIB(proxy proxy proxy) CPP_ADD_SOURCES(proxy proxy/http) +CPP_ADD_SOURCES(proxy proxy/http/unit_tests) CPP_ADD_SOURCES(proxy proxy/http2) CPP_ADD_SOURCES(proxy proxy/http/remap) CPP_ADD_SOURCES(proxy proxy/hdrs) +CPP_ADD_SOURCES(proxy proxy/hdrs/unit_tests) +CPP_ADD_SOURCES(proxy proxy/logging) CPP_LIB(iocore iocore iocore) CPP_ADD_SOURCES(iocore iocore/eventsystem) @@ -104,6 +108,7 @@ CPP_ADD_SOURCES(mgmt mgmt/api) CPP_ADD_SOURCES(mgmt mgmt/utils) CPP_LIB(records lib/records lib/records) +CPP_LIB(logging proxy/logging proxy/logging) CPP_LIB(tsconfig lib/tsconfig lib/tsconfig) CPP_LIB(wccp src/wccp include/wccp) @@ -121,3 +126,4 @@ file(GLOB plugin_files ) add_library(plugins SHARED ${plugin_files}) +add_custom_target(clang-format WORKING_DIRECTORY ${CMAKE_HOME_DIRECTORY} COMMAND make -j clang-format) diff --git a/Makefile.am b/Makefile.am index 2705e0c24bd..889be94abc9 100644 --- a/Makefile.am +++ b/Makefile.am @@ -42,6 +42,9 @@ EXTRA_DIST=CHANGES INSTALL STATUS NOTICE LAYOUT LICENSE example contrib README-E # Default value when creating release candidates RC = 0 +# Make sure the git pre-commit hook gets installed on build time +all-local: $(abs_top_srcdir)/.git/hooks/pre-commit + # igalic can't remember if this is make check or make test and neither should you. test: check @@ -91,6 +94,11 @@ examples: all install-examples: examples @cd example && $(MAKE) $(AM_MAKEFLAGS) install pkglibdir=$(pkglibexecdir) +$(abs_top_srcdir)/.git/hooks/pre-commit: $(abs_top_srcdir)/tools/git/pre-commit + @if [ -d $(abs_top_srcdir)/.git/hooks ]; then \ + cp $(abs_top_srcdir)/tools/git/pre-commit $(abs_top_srcdir)/.git/hooks/pre-commit; \ + fi + install-data-hook: if BUILD_DOCS @cd doc && $(MAKE) $(AM_MAKEFLAGS) install-man @@ -111,7 +119,7 @@ autopep8: # If you make changes to directory structures, you must update this as well. # .PHONY: clang-format-src clang-format-example clang-format-iocore clang-format-lib clang-format-mgmt \ - clang-format-plugins clang-format-proxy clang-format-tools + clang-format-plugins clang-format-proxy clang-format-tools perltidy clang-format: clang-format-src clang-format-example clang-format-iocore clang-format-lib clang-format-mgmt \ clang-format-plugins clang-format-proxy clang-format-tools clang-format-tests @@ -127,7 +135,6 @@ clang-format-iocore: clang-format-lib: @$(top_srcdir)/tools/clang-format.sh $(top_srcdir)/include - @$(top_srcdir)/tools/clang-format.sh $(top_srcdir)/lib/cppapi @$(top_srcdir)/tools/clang-format.sh $(top_srcdir)/lib/records clang-format-mgmt: @@ -145,6 +152,9 @@ clang-format-tools: clang-format-tests: @$(top_srcdir)/tools/clang-format.sh $(top_srcdir)/tests +perltidy: + perltidy -q -b -bext='/' `find . -name \*.pm -o -name \*.pl` + help: @echo 'all default target for building the package' @echo 'asf-dist recreate source package' diff --git a/README b/README index 3f88a04bb06..707c767040f 100644 --- a/README +++ b/README @@ -11,13 +11,13 @@ plugins to build large scale web applications. |-- ci/ ................... Quality assurance and other CI tools and configs |-- configs/ .............. Configurations |-- contrib/ .............. Various contributed auxiliary pieces - |-- doc/ .................. Documentations for Traffic Server + |-- doc/ .................. Documentation for Traffic Server |-- admin-guide/ ...... Admin guide documentations |-- appendices/ ....... Appendices of Traffic Server - |-- developer-guide/ .. Documentaions for developers + |-- developer-guide/ .. Documentation for developers |-- dot/ .............. Graphviz source files for docs pictures |-- static/ ........... Static resources - |-- uml/ .............. Documentations in UML + |-- uml/ .............. Documentation in UML |-- example/ .............. Example plugins |-- iocore/ ............... |-- aio/ .............. Asynchronous I/O core @@ -88,7 +88,6 @@ plugins to build large scale web applications. perl-ExtUtils-MakeMaker gcc/g++ or clang/clang++ openssl-devel - tcl-devel pcre-devel ncurses-devel and libcurl-devel(optional, needed for traffic_top) libcap-devel (optional, highly recommended) @@ -103,7 +102,6 @@ plugins to build large scale web applications. libmodule-install-perl gcc/g++ or clang/clang++ libssl-dev - tcl-dev libpcre3-dev libcap-dev (optional, highly recommended) libhwloc-dev (optional, highly recommended) @@ -119,7 +117,6 @@ plugins to build large scale web applications. autoconf automake libtool - tcl-dev linux-headers OSX (we recommend HomeBrew): @@ -127,7 +124,6 @@ plugins to build large scale web applications. automake pkg-config libtool - tcl-tk (or tcl from Homebrew-Cask) openssl pcre @@ -137,7 +133,6 @@ plugins to build large scale web applications. devel/automake devel/pkgconf devel/libtool - lang/tcl85 security/openssl devel/pcre textproc/flex (optional, install newer version from ports, fix PATH) @@ -149,7 +144,6 @@ plugins to build large scale web applications. developer/build/autoconf developer/build/automake-111 developer/build/libtool - omniti/runtime/tcl-8 library/security/openssl library/pcre diff --git a/REVIEWERS b/REVIEWERS index db992a5490e..628abc22baa 100644 --- a/REVIEWERS +++ b/REVIEWERS @@ -18,12 +18,12 @@ Committers: add modules as needed and any qualifications after your e-mail addre all/general interest jplevyak@apache.org bcall@apache.org - briang@apache.org + briang@apache.org mturk@apache.org zwoop@apache.org jim@apache.org ericb@apache.org - manjesh@apache.org + manjesh@apache.org lib/ts jplevyak@apache.org zwoop@apache.org @@ -33,19 +33,19 @@ lib/ts lib/records bcall@apache.org georgep@apache.org - zwoop@apache.org + zwoop@apache.org Event System/Buffering/VIO/VConnection jplevyak@apache.org bcall@apache.org - briang@apache.org + briang@apache.org georgep@apache.org Network I/O jplevyak@apache.org - not including SSL or UDP except where it intersects the other code bcall@apache.org georgep@apache.org - briang@apache.org + briang@apache.org jim@apache.org - zwoop@apache.org + zwoop@apache.org Raw Cache and AIO jplevyak@apache.org bcall@apache.org @@ -53,13 +53,12 @@ Raw Cache and AIO HTTP Caching bcall@apache.org jim@apache.org - zwoop@apache.org + zwoop@apache.org Block-Cache -Clustering jplevyak@apache.org georgep@apache.org jim@apache.org - zym@apache.org + zym@apache.org DNS/HostDB jplevyak@apache.org zwoop@apache.org @@ -68,11 +67,11 @@ DNS/HostDB ericb@apache.org FastIO Docs - dianes@apache.org + dianes@apache.org Examples/Plugins zwoop@apache.org bcall@apache.org - briang@apache.org + briang@apache.org ericb@apache.org HDRs bcall@apache.org @@ -81,7 +80,7 @@ HDRs HTTP bcall@apache.org zwoop@apache.org - briang@apache.org + briang@apache.org jim@apache.org ericb@apache.org Remap @@ -91,7 +90,7 @@ Remap ericb@apache.org API zwoop@apache.org - briang@apache.org + briang@apache.org bcall@apache.org georgep@apache.org ericb@apache.org @@ -106,8 +105,8 @@ MGMT Stats bcall@apache.org georgep@apache.org - briang@apache.org - zwoop@apache.org + briang@apache.org + zwoop@apache.org Build System zwoop@apache.org bcall@apache.org diff --git a/STATUS b/STATUS index d145713bc35..2b3c2e9edd6 100644 --- a/STATUS +++ b/STATUS @@ -6,7 +6,15 @@ The current version of this file can be found at: * https://github.com/apache/trafficserver/blob/master/STATUS Release history: - 7.1.4 : Released on , 2018 + 8.0.3 : Release on , 2019 + 8.0.2 : Release on Jan 29th, 2019 + 8.0.1 : Release on Nov 29th, 2018 + 8.0.0 : Release on Sep 25th, 2018 + + 7.1.7 : Released on , 2019 + 7.1.6 : Released on Jan 29th, 2019 + 7.1.5 : Released on Nov 24th, 2018 + 7.1.4 : Released on Aug 1st, 2018 7.1.3 : Released on Apr 16th, 2018 7.1.2 : Released on Jan 10th, 2018 7.1.1 : Released on Sep 7th, 2017 diff --git a/build/atomic.m4 b/build/atomic.m4 new file mode 100644 index 00000000000..266eb29d4ac --- /dev/null +++ b/build/atomic.m4 @@ -0,0 +1,38 @@ +dnl -------------------------------------------------------- -*- autoconf -*- +dnl Licensed to the Apache Software Foundation (ASF) under one or more +dnl contributor license agreements. See the NOTICE file distributed with +dnl this work for additional information regarding copyright ownership. +dnl The ASF licenses this file to You under the Apache License, Version 2.0 +dnl (the "License"); you may not use this file except in compliance with +dnl the License. You may obtain a copy of the License at +dnl +dnl http://www.apache.org/licenses/LICENSE-2.0 +dnl +dnl Unless required by applicable law or agreed to in writing, software +dnl distributed under the License is distributed on an "AS IS" BASIS, +dnl WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +dnl See the License for the specific language governing permissions and +dnl limitations under the License. + +dnl ----------------------------------------------------------------- +dnl atomic.m4: Trafficserver's autoconf macros for testing atomic support +dnl + +dnl +dnl TS_CHECK_ATOMIC: try to figure out the need for -latomic +dnl +AC_DEFUN([TS_CHECK_ATOMIC], [ + AC_COMPILE_IFELSE( + [AC_LANG_PROGRAM([[#include ]], + [[uint64_t val = 0; __atomic_add_fetch(&val, 1, __ATOMIC_RELAXED);]])], + [AC_DEFINE(HAVE_ATOMIC, 1, [Define to 1 if you have '__atomic' functions.]) + AC_LINK_IFELSE( + [AC_LANG_PROGRAM([[#include ]], + [[uint64_t val = 0; __atomic_add_fetch(&val, 1, __ATOMIC_RELAXED);]])], + [ATOMIC_LIBS=""], + [ATOMIC_LIBS="-latomic"] + )], + [ATOMIC_LIBS=""] + ) + AC_SUBST([ATOMIC_LIBS]) +]) diff --git a/build/brotli.m4 b/build/brotli.m4 index a3189b6bc52..b9c02610584 100644 --- a/build/brotli.m4 +++ b/build/brotli.m4 @@ -77,7 +77,7 @@ fi ], [ AC_CHECK_HEADER([brotli/encode.h], [], [has_brotli=0]) -AC_CHECK_LIB([brotlienc], BrotliEncoderCreateInstance, [], [has_brotli=0]) +AC_CHECK_LIB([brotlienc], BrotliEncoderCreateInstance, [:], [has_brotli=0]) if test "x$has_brotli" == "x0"; then PKG_CHECK_EXISTS([LIBBROTLIENC], diff --git a/build/cjose.m4 b/build/cjose.m4 new file mode 100644 index 00000000000..6bead14cd3e --- /dev/null +++ b/build/cjose.m4 @@ -0,0 +1,47 @@ +dnl -------------------------------------------------------- -*- autoconf -*- +dnl Licensed to the Apache Software Foundation (ASF) under one or more +dnl contributor license agreements. See the NOTICE file distributed with +dnl this work for additional information regarding copyright ownership. +dnl The ASF licenses this file to You under the Apache License, Version 2.0 +dnl (the "License"); you may not use this file except in compliance with +dnl the License. You may obtain a copy of the License at +dnl +dnl http://www.apache.org/licenses/LICENSE-2.0 +dnl +dnl Unless required by applicable law or agreed to in writing, software +dnl distributed under the License is distributed on an "AS IS" BASIS, +dnl WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +dnl See the License for the specific language governing permissions and +dnl limitations under the License. + +dnl +dnl cjose.m4: Trafficserver's cjose autoconf macros +dnl + +dnl +dnl TS_CHECK_CJOSE: look for cjose libraries and headers +dnl + +AC_DEFUN([TS_CHECK_CJOSE], [ +AC_MSG_CHECKING([for --with-cjose]) + AC_ARG_WITH( + [cjose], + [AS_HELP_STRING([--with-cjose=DIR], [use a specific cjose library])], + [ LDFLAGS="$LDFLAGS -L$with_cjose/lib"; + CFLAGS="$CFLAGS -I$with_cjose/include/"; + CPPFLAGS="$CPPFLAGS -I$with_cjose/include/"; + AC_MSG_RESULT([$with_cjose]) + ], + [ AC_MSG_RESULT([no])] + ) + + AC_CHECK_HEADERS([cjose/cjose.h], [ + AC_MSG_CHECKING([whether cjose is dynamic]) + TS_LINK_WITH_FLAGS_IFELSE([-fPIC -lcjose -ljansson -lcrypto],[AC_LANG_PROGRAM( + [#include ], + [(void) cjose_jws_import("", 0, NULL);])], + [AC_MSG_RESULT([yes]); LIBCJOSE=-lcjose], + [AC_MSG_RESULT([no]); LIBCJOSE=-l:libcjose.a]) + ], + [LIBCJOSE=]) +]) diff --git a/build/common.m4 b/build/common.m4 index d66b002646d..6f5148a9aac 100644 --- a/build/common.m4 +++ b/build/common.m4 @@ -196,17 +196,17 @@ AC_DEFUN([TS_TRY_COMPILE_NO_WARNING], ]) dnl -dnl TS_LINK_WITH_FLAGS_IFELSE(LDFLAGS, FUNCTION-BODY, +dnl TS_LINK_WITH_FLAGS_IFELSE(LIBS, FUNCTION-BODY, dnl [ACTIONS-IF-LINKS], [ACTIONS-IF-LINK-FAILS]) dnl dnl Tries a link test with the provided flags. dnl AC_DEFUN([TS_LINK_WITH_FLAGS_IFELSE], -[ats_save_LDFLAGS=$LDFLAGS - LDFLAGS="$LDFLAGS $1" +[ats_save_LIBS=$LIBS + LIBS="$LIBS $1" AC_LINK_IFELSE([$2],[$3],[$4]) - LDFLAGS=$ats_save_LDFLAGS + LIBS=$ats_save_LIBS ]) diff --git a/build/crypto.m4 b/build/crypto.m4 index e3ac31bae02..09d40866440 100644 --- a/build/crypto.m4 +++ b/build/crypto.m4 @@ -39,38 +39,28 @@ AC_DEFUN([TS_CHECK_CRYPTO], [ dnl add checks for other varieties of ssl here ]) -dnl - -AC_DEFUN([TS_CHECK_CRYPTO_EC_KEYS], [ - _eckeys_saved_LIBS=$LIBS - - TS_ADDTO(LIBS, [$OPENSSL_LIBS]) - AC_CHECK_HEADERS(openssl/ec.h) - AC_CHECK_FUNCS(EC_KEY_new_by_curve_name, [enable_tls_eckey=yes], [enable_tls_eckey=no]) - LIBS=$_eckeys_saved_LIBS - - AC_MSG_CHECKING(whether EC keys are supported) - AC_MSG_RESULT([$enable_tls_eckey]) - TS_ARG_ENABLE_VAR([use], [tls-eckey]) - AC_SUBST(use_tls_eckey) -]) - -AC_DEFUN([TS_CHECK_CRYPTO_NEXTPROTONEG], [ - enable_tls_npn=yes - _npn_saved_LIBS=$LIBS - TS_ADDTO(LIBS, [$OPENSSL_LIBS]) - AC_CHECK_FUNCS(SSL_CTX_set_next_protos_advertised_cb SSL_CTX_set_next_proto_select_cb SSL_select_next_proto SSL_get0_next_proto_negotiated, - [], [enable_tls_npn=no] - ) - LIBS=$_npn_saved_LIBS - - AC_MSG_CHECKING(whether to enable Next Protocol Negotiation TLS extension support) - AC_MSG_RESULT([$enable_tls_npn]) - TS_ARG_ENABLE_VAR([use], [tls-npn]) - AC_SUBST(use_tls_npn) +dnl +dnl Check OpenSSL Version +dnl +AC_DEFUN([TS_CHECK_CRYPTO_VERSION], [ + AC_MSG_CHECKING([OpenSSL version]) + AC_TRY_RUN([ +#include +int main() { + if (OPENSSL_VERSION_NUMBER < 0x1000200fL) { + return 1; + } + return 0; +} +], + [AC_MSG_RESULT([ok])], + [AC_MSG_FAILURE([requires an OpenSSL version 1.0.2 or greater])]) ]) +dnl +dnl Since OpenSSL 1.1.0 +dnl AC_DEFUN([TS_CHECK_CRYPTO_ASYNC], [ enable_tls_async=yes _async_saved_LIBS=$LIBS @@ -87,25 +77,12 @@ AC_DEFUN([TS_CHECK_CRYPTO_ASYNC], [ AC_SUBST(use_tls_async) ]) -AC_DEFUN([TS_CHECK_CRYPTO_ALPN], [ - enable_tls_alpn=yes - _alpn_saved_LIBS=$LIBS - - TS_ADDTO(LIBS, [$OPENSSL_LIBS]) - AC_CHECK_FUNCS(SSL_CTX_set_alpn_protos SSL_CTX_set_alpn_select_cb SSL_get0_alpn_selected SSL_select_next_proto, - [], [enable_tls_alpn=no] - ) - LIBS=$_alpn_saved_LIBS - - AC_MSG_CHECKING(whether to enable Application Layer Protocol Negotiation TLS extension support) - AC_MSG_RESULT([$enable_tls_alpn]) - TS_ARG_ENABLE_VAR([use], [tls-alpn]) - AC_SUBST(use_tls_alpn) -]) - -AC_DEFUN([TS_CHECK_CRYPTO_CERT_CB], [ - _cert_saved_LIBS=$LIBS - enable_cert_cb=yes +dnl +dnl Since OpenSSL 1.1.1 +dnl +AC_DEFUN([TS_CHECK_CRYPTO_HELLO_CB], [ + _hello_saved_LIBS=$LIBS + enable_hello_cb=yes TS_ADDTO(LIBS, [$OPENSSL_LIBS]) AC_CHECK_HEADERS(openssl/ssl.h openssl/ts.h) @@ -115,7 +92,7 @@ AC_DEFUN([TS_CHECK_CRYPTO_CERT_CB], [ #include #endif ]) - AC_MSG_CHECKING([for SSL_CTX_set_cert_cb]) + AC_MSG_CHECKING([for SSL_CTX_set_client_hello_cb]) AC_LINK_IFELSE( [ AC_LANG_PROGRAM([[ @@ -126,24 +103,27 @@ AC_DEFUN([TS_CHECK_CRYPTO_CERT_CB], [ #include #endif ]], - [[SSL_CTX_set_cert_cb(NULL, NULL, NULL);]]) + [[SSL_CTX_set_client_hello_cb(NULL, NULL, NULL);]]) ], [ AC_MSG_RESULT([yes]) ], [ AC_MSG_RESULT([no]) - enable_cert_cb=no + enable_hello_cb=no ]) - LIBS=$_cert_saved_LIBS + LIBS=$_hello_saved_LIBS - AC_MSG_CHECKING(whether to enable TLS certificate callback support) - AC_MSG_RESULT([$enable_cert_cb]) - TS_ARG_ENABLE_VAR([use], [cert-cb]) - AC_SUBST(use_cert_cb) + AC_MSG_CHECKING(whether to enable TLS client hello callback support) + AC_MSG_RESULT([$enable_hello_cb]) + TS_ARG_ENABLE_VAR([use], [hello-cb]) + AC_SUBST(use_hello_cb) ]) +dnl +dnl Since OpenSSL 1.1.0 +dnl AC_DEFUN([TS_CHECK_CRYPTO_SET_RBIO], [ _rbio_saved_LIBS=$LIBS enable_set_rbio=yes @@ -178,6 +158,9 @@ AC_DEFUN([TS_CHECK_CRYPTO_SET_RBIO], [ AC_SUBST(use_set_rbio) ]) +dnl +dnl Since OpenSSL 1.1.0 +dnl AC_DEFUN([TS_CHECK_CRYPTO_DH_GET_2048_256], [ _dh_saved_LIBS=$LIBS enable_dh_get_2048_256=yes @@ -212,6 +195,9 @@ AC_DEFUN([TS_CHECK_CRYPTO_DH_GET_2048_256], [ AC_SUBST(use_dh_get_2048_256) ]) +dnl +dnl Since OpenSSL 1.1.0 +dnl AC_DEFUN([TS_CHECK_CRYPTO_OCSP], [ _ocsp_saved_LIBS=$LIBS @@ -227,6 +213,9 @@ AC_DEFUN([TS_CHECK_CRYPTO_OCSP], [ AC_SUBST(use_tls_ocsp) ]) +dnl +dnl Since OpenSSL 1.1.1 +dnl AC_DEFUN([TS_CHECK_CRYPTO_SET_CIPHERSUITES], [ _set_ciphersuites_saved_LIBS=$LIBS diff --git a/build/hiredis.m4 b/build/hiredis.m4 index ffbb8c96860..871df838a34 100644 --- a/build/hiredis.m4 +++ b/build/hiredis.m4 @@ -24,34 +24,33 @@ dnl AC_DEFUN([TS_CHECK_HIREDIS], [ hiredis_base_dir='/usr' -has_hiredis=0 +has_hiredis=1 AC_ARG_WITH(hiredis, [AC_HELP_STRING([--with-hiredis=DIR],[use a specific hiredis library])], [ - has_hiredis=1 if test "x$withval" != "xyes" && test "x$withval" != "x"; then hiredis_base_dir="$withval" - if test "$withval" != "no"; then - case "$withval" in - *":"*) - hidredis_include="`echo $withval |sed -e 's/:.*$//'`" - hiredis_ldflags="`echo $withval |sed -e 's/^.*://'`" - AC_MSG_CHECKING(checking for hiredis includes in $hiredis_include libs in $hiredis_ldflags ) - ;; - *) - hiredis_include="$withval/include" - hiredis_ldflags="$withval/lib" - AC_MSG_CHECKING(checking for hiredis includes in $withval) - ;; - esac - fi fi +],[]) - if test -d $hiredis_include && test -d $hiredis_ldflags && test -f $hiredis_include/hiredis/hiredis.h; then - AC_MSG_RESULT([ok]) - else - has_hiredis=0 - AC_MSG_RESULT([not found]) - fi +case "$hiredis_base_dir" in +*":"*) + hidredis_include="`echo $hiredis_base_dir |sed -e 's/:.*$//'`" + hiredis_ldflags="`echo $hiredis_base_dir |sed -e 's/^.*://'`" + AC_MSG_CHECKING(checking for hiredis includes in $hiredis_include libs in $hiredis_ldflags ) + ;; +*) + hiredis_include="$hiredis_base_dir/include" + hiredis_ldflags="$hiredis_base_dir/lib" + AC_MSG_CHECKING(checking for hiredis includes in $hiredis_base_dir) + ;; +esac + +if test -d $hiredis_include && test -d $hiredis_ldflags && test -f $hiredis_include/hiredis/hiredis.h; then + AC_MSG_RESULT([ok]) +else + has_hiredis=0 + AC_MSG_RESULT([not found]) +fi if test "$has_hiredis" != "0"; then saved_ldflags=$LDFLAGS @@ -77,17 +76,6 @@ if test "$has_hiredis" != "0"; then LDFLAGS=$saved_ldflags fi fi -], -[ -has_hiredis=1 -AC_CHECK_HEADER([hiredis/hiredis.h], [], [has_hiredis=0]) -AC_CHECK_LIB([hiredis], redisConnect, [], [has_hiredis=0]) - -if test "x$has_hiredis" == "x1"; then - AC_SUBST([LIB_HIREDIS], [-lhiredis]) -fi -]) - ]) diff --git a/build/jansson.m4 b/build/jansson.m4 new file mode 100644 index 00000000000..07987c5acf7 --- /dev/null +++ b/build/jansson.m4 @@ -0,0 +1,47 @@ +dnl -------------------------------------------------------- -*- autoconf -*- +dnl Licensed to the Apache Software Foundation (ASF) under one or more +dnl contributor license agreements. See the NOTICE file distributed with +dnl this work for additional information regarding copyright ownership. +dnl The ASF licenses this file to You under the Apache License, Version 2.0 +dnl (the "License"); you may not use this file except in compliance with +dnl the License. You may obtain a copy of the License at +dnl +dnl http://www.apache.org/licenses/LICENSE-2.0 +dnl +dnl Unless required by applicable law or agreed to in writing, software +dnl distributed under the License is distributed on an "AS IS" BASIS, +dnl WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +dnl See the License for the specific language governing permissions and +dnl limitations under the License. + +dnl +dnl jansson.m4: Trafficserver's jansson autoconf macros +dnl + +dnl +dnl TS_CHECK_JANSSON: look for jansson libraries and headers +dnl + +AC_DEFUN([TS_CHECK_JANSSON], [ + AC_MSG_CHECKING([for --with-jansson]) + AC_ARG_WITH( + [jansson], + [AS_HELP_STRING([--with-jansson], [use a specific jansson library])], + [ LDFLAGS="$LDFLAGS -L$with_jansson/lib"; + CFLAGS="$CFLAGS -I$with_jansson/include/"; + CPPFLAGS="$CPPFLAGS -I$with_jansson/include/"; + AC_MSG_RESULT([$with_jansson]) + ], + [ AC_MSG_RESULT([no])] + ) + + AC_CHECK_HEADERS([jansson.h], [ + AC_MSG_CHECKING([whether jansson is dynamic]) + TS_LINK_WITH_FLAGS_IFELSE([-fPIC -ljansson],[AC_LANG_PROGRAM( + [#include ], + [(void) json_object();])], + [AC_MSG_RESULT([yes]); LIBJANSSON=-ljansson], + [AC_MSG_RESULT([no]); LIBJANSSON=-l:libjansson.a]) + ], + [LIBJANSSON=]) +]) diff --git a/build/pkg.m4 b/build/pkg.m4 index b7ca359a88a..dd5a6313a57 100644 --- a/build/pkg.m4 +++ b/build/pkg.m4 @@ -53,7 +53,7 @@ fi[]dnl # to PKG_CHECK_MODULES(), but does not set variables or print errors. # # Please remember that m4 expands AC_REQUIRE([PKG_PROG_PKG_CONFIG]) -# only at the first occurence in configure.ac, so if the first place +# only at the first occurrence in configure.ac, so if the first place # it's called might be skipped (such as if it is within an "if", you # have to call PKG_CHECK_EXISTS manually # -------------------------------------------------------------- diff --git a/build/tcl.m4 b/build/tcl.m4 deleted file mode 100644 index 61b223eb41c..00000000000 --- a/build/tcl.m4 +++ /dev/null @@ -1,3307 +0,0 @@ -#------------------------------------------------------------------------ -# SC_PATH_TCLCONFIG -- -# -# Locate the tclConfig.sh file and perform a sanity check on -# the Tcl compile flags -# -# Arguments: -# none -# -# Results: -# -# Adds the following arguments to configure: -# --with-tcl=... -# -# Defines the following vars: -# TCL_BIN_DIR Full path to the directory containing -# the tclConfig.sh file -#------------------------------------------------------------------------ - -AC_DEFUN([SC_PATH_TCLCONFIG], [ - # - # Ok, lets find the tcl configuration - # First, look for one uninstalled. - # the alternative search directory is invoked by --with-tcl - # - - if test x"${no_tcl}" = x ; then - # we reset no_tcl in case something fails here - no_tcl=true - AC_ARG_WITH(tcl, - AC_HELP_STRING([--with-tcl], - [directory containing tcl configuration (tclConfig.sh)]), - with_tclconfig="${withval}") - AC_MSG_CHECKING([for Tcl configuration]) - AC_CACHE_VAL(ac_cv_c_tclconfig,[ - - # First check to see if --with-tcl was specified. - if test x"${with_tclconfig}" != x ; then - case "${with_tclconfig}" in - */tclConfig.sh ) - if test -f "${with_tclconfig}"; then - AC_MSG_WARN([--with-tcl argument should refer to directory containing tclConfig.sh, not to tclConfig.sh itself]) - with_tclconfig="`echo "${with_tclconfig}" | sed 's!/tclConfig\.sh$!!'`" - fi ;; - esac - if test -f "${with_tclconfig}/tclConfig.sh" ; then - ac_cv_c_tclconfig="`(cd "${with_tclconfig}"; pwd)`" - else - AC_MSG_ERROR([${with_tclconfig} directory doesn't contain tclConfig.sh]) - fi - fi - - # then check for a private Tcl installation - if test x"${ac_cv_c_tclconfig}" = x ; then - for i in \ - ../tcl \ - `ls -dr ../tcl[[8-9]].[[0-9]].[[0-9]]* 2>/dev/null` \ - `ls -dr ../tcl[[8-9]].[[0-9]] 2>/dev/null` \ - `ls -dr ../tcl[[8-9]].[[0-9]]* 2>/dev/null` \ - ../../tcl \ - `ls -dr ../../tcl[[8-9]].[[0-9]].[[0-9]]* 2>/dev/null` \ - `ls -dr ../../tcl[[8-9]].[[0-9]] 2>/dev/null` \ - `ls -dr ../../tcl[[8-9]].[[0-9]]* 2>/dev/null` \ - ../../../tcl \ - `ls -dr ../../../tcl[[8-9]].[[0-9]].[[0-9]]* 2>/dev/null` \ - `ls -dr ../../../tcl[[8-9]].[[0-9]] 2>/dev/null` \ - `ls -dr ../../../tcl[[8-9]].[[0-9]]* 2>/dev/null` ; do - if test -f "$i/unix/tclConfig.sh" ; then - ac_cv_c_tclconfig="`(cd $i/unix; pwd)`" - break - fi - done - fi - - # on Darwin, check in Framework installation locations - if test "`uname -s`" = "Darwin" -a x"${ac_cv_c_tclconfig}" = x ; then - for i in "`xcrun --show-sdk-path 2>/dev/null`/usr/lib" \ - `xcrun --show-sdk-path 2>/dev/null`/System/Library/Frameworks \ - `ls -d ~/Library/Frameworks 2>/dev/null` \ - `ls -d /Library/Frameworks 2>/dev/null` \ - `ls -d /Network/Library/Frameworks 2>/dev/null` \ - `ls -d /System/Library/Frameworks 2>/dev/null` \ - ; do - if test -f "$i/tclConfig.sh" ; then - ac_cv_c_tclconfig="`(cd $i; pwd)`" - break - elif test -f "$i/Tcl.framework/tclConfig.sh" ; then - ac_cv_c_tclconfig="`(cd $i/Tcl.framework; pwd)`" - break - fi - done - fi - - # check in a few common install locations - if test x"${ac_cv_c_tclconfig}" = x ; then - for i in `ls -d ${libdir} 2>/dev/null` \ - `ls -d ${exec_prefix}/lib 2>/dev/null` \ - `ls -dr ${exec_prefix}/lib/tcl[[8-9]].[[0-9]].[[0-9]]* 2>/dev/null` \ - `ls -dr ${exec_prefix}/lib/tcl[[8-9]].[[0-9]] 2>/dev/null` \ - `ls -dr ${exec_prefix}/lib/tcl[[8-9]].[[0-9]]* 2>/dev/null` \ - `ls -d ${prefix}/lib 2>/dev/null` \ - `ls -dr ${prefix}/lib/tcl[[8-9]].[[0-9]].[[0-9]]* 2>/dev/null` \ - `ls -dr ${prefix}/lib/tcl[[8-9]].[[0-9]] 2>/dev/null` \ - `ls -dr ${prefix}/lib/tcl[[8-9]].[[0-9]]* 2>/dev/null` \ - `ls -d /usr/local/lib 2>/dev/null` \ - `ls -dr /usr/local/lib/tcl[[8-9]].[[0-9]].[[0-9]]* 2>/dev/null` \ - `ls -dr /usr/local/lib/tcl[[8-9]].[[0-9]] 2>/dev/null` \ - `ls -dr /usr/local/lib/tcl[[8-9]].[[0-9]]* 2>/dev/null` \ - `ls -d /usr/lib64 2>/dev/null` \ - `ls -dr /usr/lib64/tcl[[8-9]].[[0-9]].[[0-9]]* 2>/dev/null` \ - `ls -dr /usr/lib64/tcl[[8-9]].[[0-9]] 2>/dev/null` \ - `ls -dr /usr/lib64/tcl[[8-9]].[[0-9]]* 2>/dev/null` \ - `ls -d /usr/contrib/lib 2>/dev/null` \ - `ls -dr /usr/contrib/lib/tcl[[8-9]].[[0-9]].[[0-9]]* 2>/dev/null` \ - `ls -dr /usr/contrib/lib/tcl[[8-9]].[[0-9]] 2>/dev/null` \ - `ls -dr /usr/contrib/lib/tcl[[8-9]].[[0-9]]* 2>/dev/null` \ - `ls -d /usr/lib 2>/dev/null` \ - `ls -dr /usr/lib/tcl[[8-9]].[[0-9]].[[0-9]]* 2>/dev/null` \ - `ls -dr /usr/lib/tcl[[8-9]].[[0-9]] 2>/dev/null` \ - `ls -dr /usr/lib/tcl[[8-9]].[[0-9]]* 2>/dev/null` \ - ; do - if test -f "$i/tclConfig.sh" ; then - ac_cv_c_tclconfig="`(cd $i; pwd)`" - break - fi - done - fi - - # check in a few other private locations - if test x"${ac_cv_c_tclconfig}" = x ; then - for i in \ - ${srcdir}/../tcl \ - `ls -dr ${srcdir}/../tcl[[8-9]].[[0-9]].[[0-9]]* 2>/dev/null` \ - `ls -dr ${srcdir}/../tcl[[8-9]].[[0-9]] 2>/dev/null` \ - `ls -dr ${srcdir}/../tcl[[8-9]].[[0-9]]* 2>/dev/null` ; do - if test -f "$i/unix/tclConfig.sh" ; then - ac_cv_c_tclconfig="`(cd $i/unix; pwd)`" - break - fi - done - fi - ]) - - if test x"${ac_cv_c_tclconfig}" = x ; then - TCL_BIN_DIR="# no Tcl configs found" - AC_MSG_ERROR([Can't find Tcl configuration, install the TCL dev package]) - else - no_tcl= - TCL_BIN_DIR="${ac_cv_c_tclconfig}" - AC_MSG_RESULT([found ${TCL_BIN_DIR}/tclConfig.sh]) - fi - fi -]) - -#------------------------------------------------------------------------ -# SC_PATH_TKCONFIG -- -# -# Locate the tkConfig.sh file -# -# Arguments: -# none -# -# Results: -# -# Adds the following arguments to configure: -# --with-tk=... -# -# Defines the following vars: -# TK_BIN_DIR Full path to the directory containing -# the tkConfig.sh file -#------------------------------------------------------------------------ - -AC_DEFUN([SC_PATH_TKCONFIG], [ - # - # Ok, lets find the tk configuration - # First, look for one uninstalled. - # the alternative search directory is invoked by --with-tk - # - - if test x"${no_tk}" = x ; then - # we reset no_tk in case something fails here - no_tk=true - AC_ARG_WITH(tk, - AC_HELP_STRING([--with-tk], - [directory containing tk configuration (tkConfig.sh)]), - with_tkconfig="${withval}") - AC_MSG_CHECKING([for Tk configuration]) - AC_CACHE_VAL(ac_cv_c_tkconfig,[ - - # First check to see if --with-tkconfig was specified. - if test x"${with_tkconfig}" != x ; then - case "${with_tkconfig}" in - */tkConfig.sh ) - if test -f "${with_tkconfig}"; then - AC_MSG_WARN([--with-tk argument should refer to directory containing tkConfig.sh, not to tkConfig.sh itself]) - with_tkconfig="`echo "${with_tkconfig}" | sed 's!/tkConfig\.sh$!!'`" - fi ;; - esac - if test -f "${with_tkconfig}/tkConfig.sh" ; then - ac_cv_c_tkconfig="`(cd "${with_tkconfig}"; pwd)`" - else - AC_MSG_ERROR([${with_tkconfig} directory doesn't contain tkConfig.sh]) - fi - fi - - # then check for a private Tk library - if test x"${ac_cv_c_tkconfig}" = x ; then - for i in \ - ../tk \ - `ls -dr ../tk[[8-9]].[[0-9]].[[0-9]]* 2>/dev/null` \ - `ls -dr ../tk[[8-9]].[[0-9]] 2>/dev/null` \ - `ls -dr ../tk[[8-9]].[[0-9]]* 2>/dev/null` \ - ../../tk \ - `ls -dr ../../tk[[8-9]].[[0-9]].[[0-9]]* 2>/dev/null` \ - `ls -dr ../../tk[[8-9]].[[0-9]] 2>/dev/null` \ - `ls -dr ../../tk[[8-9]].[[0-9]]* 2>/dev/null` \ - ../../../tk \ - `ls -dr ../../../tk[[8-9]].[[0-9]].[[0-9]]* 2>/dev/null` \ - `ls -dr ../../../tk[[8-9]].[[0-9]] 2>/dev/null` \ - `ls -dr ../../../tk[[8-9]].[[0-9]]* 2>/dev/null` ; do - if test -f "$i/unix/tkConfig.sh" ; then - ac_cv_c_tkconfig="`(cd $i/unix; pwd)`" - break - fi - done - fi - - # on Darwin, check in Framework installation locations - if test "`uname -s`" = "Darwin" -a x"${ac_cv_c_tkconfig}" = x ; then - for i in `ls -d ~/Library/Frameworks 2>/dev/null` \ - `ls -d /Library/Frameworks 2>/dev/null` \ - `ls -d /Network/Library/Frameworks 2>/dev/null` \ - `ls -d /System/Library/Frameworks 2>/dev/null` \ - ; do - if test -f "$i/Tk.framework/tkConfig.sh" ; then - ac_cv_c_tkconfig="`(cd $i/Tk.framework; pwd)`" - break - fi - done - fi - - # check in a few common install locations - if test x"${ac_cv_c_tkconfig}" = x ; then - for i in `ls -d ${libdir} 2>/dev/null` \ - `ls -d ${exec_prefix}/lib 2>/dev/null` \ - `ls -d ${prefix}/lib 2>/dev/null` \ - `ls -d /usr/local/lib 2>/dev/null` \ - `ls -d /usr/contrib/lib 2>/dev/null` \ - `ls -d /usr/lib 2>/dev/null` \ - ; do - if test -f "$i/tkConfig.sh" ; then - ac_cv_c_tkconfig="`(cd $i; pwd)`" - break - fi - done - fi - - # check in a few other private locations - if test x"${ac_cv_c_tkconfig}" = x ; then - for i in \ - ${srcdir}/../tk \ - `ls -dr ${srcdir}/../tk[[8-9]].[[0-9]].[[0-9]]* 2>/dev/null` \ - `ls -dr ${srcdir}/../tk[[8-9]].[[0-9]] 2>/dev/null` \ - `ls -dr ${srcdir}/../tk[[8-9]].[[0-9]]* 2>/dev/null` ; do - if test -f "$i/unix/tkConfig.sh" ; then - ac_cv_c_tkconfig="`(cd $i/unix; pwd)`" - break - fi - done - fi - ]) - - if test x"${ac_cv_c_tkconfig}" = x ; then - TK_BIN_DIR="# no Tk configs found" - AC_MSG_WARN([Can't find Tk configuration definitions]) - exit 0 - else - no_tk= - TK_BIN_DIR="${ac_cv_c_tkconfig}" - AC_MSG_RESULT([found ${TK_BIN_DIR}/tkConfig.sh]) - fi - fi -]) - -#------------------------------------------------------------------------ -# SC_LOAD_TCLCONFIG -- -# -# Load the tclConfig.sh file -# -# Arguments: -# -# Requires the following vars to be set: -# TCL_BIN_DIR -# -# Results: -# -# Subst the following vars: -# TCL_BIN_DIR -# TCL_SRC_DIR -# TCL_LIB_FILE -# -#------------------------------------------------------------------------ - -AC_DEFUN([SC_LOAD_TCLCONFIG], [ - AC_MSG_CHECKING([for existence of ${TCL_BIN_DIR}/tclConfig.sh]) - - if test -f "${TCL_BIN_DIR}/tclConfig.sh" ; then - AC_MSG_RESULT([loading]) - . "${TCL_BIN_DIR}/tclConfig.sh" - else - AC_MSG_RESULT([could not find ${TCL_BIN_DIR}/tclConfig.sh]) - fi - - # eval is required to do the TCL_DBGX substitution - eval "TCL_LIB_FILE=\"${TCL_LIB_FILE}\"" - eval "TCL_STUB_LIB_FILE=\"${TCL_STUB_LIB_FILE}\"" - - # If the TCL_BIN_DIR is the build directory (not the install directory), - # then set the common variable name to the value of the build variables. - # For example, the variable TCL_LIB_SPEC will be set to the value - # of TCL_BUILD_LIB_SPEC. An extension should make use of TCL_LIB_SPEC - # instead of TCL_BUILD_LIB_SPEC since it will work with both an - # installed and uninstalled version of Tcl. - if test -f "${TCL_BIN_DIR}/Makefile" ; then - TCL_LIB_SPEC="${TCL_BUILD_LIB_SPEC}" - TCL_STUB_LIB_SPEC="${TCL_BUILD_STUB_LIB_SPEC}" - TCL_STUB_LIB_PATH="${TCL_BUILD_STUB_LIB_PATH}" - elif test "`uname -s`" = "Darwin"; then - # If Tcl was built as a framework, attempt to use the libraries - # from the framework at the given location so that linking works - # against Tcl.framework installed in an arbitary location. - case ${TCL_DEFS} in - *TCL_FRAMEWORK*) - if test -f "${TCL_BIN_DIR}/${TCL_LIB_FILE}"; then - for i in "`cd "${TCL_BIN_DIR}"; pwd`" \ - "`cd "${TCL_BIN_DIR}"/../..; pwd`"; do - if test "`basename "$i"`" = "${TCL_LIB_FILE}.framework"; then - TCL_LIB_SPEC="-F`dirname "$i" | sed -e 's/ /\\\\ /g'` -framework ${TCL_LIB_FILE}" - break - fi - done - fi - if test -f "${TCL_BIN_DIR}/${TCL_STUB_LIB_FILE}"; then - TCL_STUB_LIB_SPEC="-L`echo "${TCL_BIN_DIR}" | sed -e 's/ /\\\\ /g'` ${TCL_STUB_LIB_FLAG}" - TCL_STUB_LIB_PATH="${TCL_BIN_DIR}/${TCL_STUB_LIB_FILE}" - fi - ;; - esac - fi - - # eval is required to do the TCL_DBGX substitution - eval "TCL_LIB_FLAG=\"${TCL_LIB_FLAG}\"" - eval "TCL_LIB_SPEC=\"${TCL_LIB_SPEC}\"" - eval "TCL_STUB_LIB_FLAG=\"${TCL_STUB_LIB_FLAG}\"" - eval "TCL_STUB_LIB_SPEC=\"${TCL_STUB_LIB_SPEC}\"" - - AC_SUBST(TCL_VERSION) - AC_SUBST(TCL_PATCH_LEVEL) - AC_SUBST(TCL_BIN_DIR) - AC_SUBST(TCL_SRC_DIR) - - AC_SUBST(TCL_LIB_FILE) - AC_SUBST(TCL_LIB_FLAG) - AC_SUBST(TCL_LIB_SPEC) - - AC_SUBST(TCL_STUB_LIB_FILE) - AC_SUBST(TCL_STUB_LIB_FLAG) - AC_SUBST(TCL_STUB_LIB_SPEC) -]) - -#------------------------------------------------------------------------ -# SC_LOAD_TKCONFIG -- -# -# Load the tkConfig.sh file -# -# Arguments: -# -# Requires the following vars to be set: -# TK_BIN_DIR -# -# Results: -# -# Sets the following vars that should be in tkConfig.sh: -# TK_BIN_DIR -#------------------------------------------------------------------------ - -AC_DEFUN([SC_LOAD_TKCONFIG], [ - AC_MSG_CHECKING([for existence of ${TK_BIN_DIR}/tkConfig.sh]) - - if test -f "${TK_BIN_DIR}/tkConfig.sh" ; then - AC_MSG_RESULT([loading]) - . "${TK_BIN_DIR}/tkConfig.sh" - else - AC_MSG_RESULT([could not find ${TK_BIN_DIR}/tkConfig.sh]) - fi - - # eval is required to do the TK_DBGX substitution - eval "TK_LIB_FILE=\"${TK_LIB_FILE}\"" - eval "TK_STUB_LIB_FILE=\"${TK_STUB_LIB_FILE}\"" - - # If the TK_BIN_DIR is the build directory (not the install directory), - # then set the common variable name to the value of the build variables. - # For example, the variable TK_LIB_SPEC will be set to the value - # of TK_BUILD_LIB_SPEC. An extension should make use of TK_LIB_SPEC - # instead of TK_BUILD_LIB_SPEC since it will work with both an - # installed and uninstalled version of Tcl. - if test -f "${TK_BIN_DIR}/Makefile" ; then - TK_LIB_SPEC="${TK_BUILD_LIB_SPEC}" - TK_STUB_LIB_SPEC="${TK_BUILD_STUB_LIB_SPEC}" - TK_STUB_LIB_PATH="${TK_BUILD_STUB_LIB_PATH}" - elif test "`uname -s`" = "Darwin"; then - # If Tk was built as a framework, attempt to use the libraries - # from the framework at the given location so that linking works - # against Tk.framework installed in an arbitary location. - case ${TK_DEFS} in - *TK_FRAMEWORK*) - if test -f "${TK_BIN_DIR}/${TK_LIB_FILE}"; then - for i in "`cd "${TK_BIN_DIR}"; pwd`" \ - "`cd "${TK_BIN_DIR}"/../..; pwd`"; do - if test "`basename "$i"`" = "${TK_LIB_FILE}.framework"; then - TK_LIB_SPEC="-F`dirname "$i" | sed -e 's/ /\\\\ /g'` -framework ${TK_LIB_FILE}" - break - fi - done - fi - if test -f "${TK_BIN_DIR}/${TK_STUB_LIB_FILE}"; then - TK_STUB_LIB_SPEC="-L` echo "${TK_BIN_DIR}" | sed -e 's/ /\\\\ /g'` ${TK_STUB_LIB_FLAG}" - TK_STUB_LIB_PATH="${TK_BIN_DIR}/${TK_STUB_LIB_FILE}" - fi - ;; - esac - fi - - # eval is required to do the TK_DBGX substitution - eval "TK_LIB_FLAG=\"${TK_LIB_FLAG}\"" - eval "TK_LIB_SPEC=\"${TK_LIB_SPEC}\"" - eval "TK_STUB_LIB_FLAG=\"${TK_STUB_LIB_FLAG}\"" - eval "TK_STUB_LIB_SPEC=\"${TK_STUB_LIB_SPEC}\"" - - AC_SUBST(TK_VERSION) - AC_SUBST(TK_BIN_DIR) - AC_SUBST(TK_SRC_DIR) - - AC_SUBST(TK_LIB_FILE) - AC_SUBST(TK_LIB_FLAG) - AC_SUBST(TK_LIB_SPEC) - - AC_SUBST(TK_STUB_LIB_FILE) - AC_SUBST(TK_STUB_LIB_FLAG) - AC_SUBST(TK_STUB_LIB_SPEC) -]) - -#------------------------------------------------------------------------ -# SC_PROG_TCLSH -# Locate a tclsh shell installed on the system path. This macro -# will only find a Tcl shell that already exists on the system. -# It will not find a Tcl shell in the Tcl build directory or -# a Tcl shell that has been installed from the Tcl build directory. -# If a Tcl shell can't be located on the PATH, then TCLSH_PROG will -# be set to "". Extensions should take care not to create Makefile -# rules that are run by default and depend on TCLSH_PROG. An -# extension can't assume that an executable Tcl shell exists at -# build time. -# -# Arguments -# none -# -# Results -# Subst's the following values: -# TCLSH_PROG -#------------------------------------------------------------------------ - -AC_DEFUN([SC_PROG_TCLSH], [ - AC_MSG_CHECKING([for tclsh]) - AC_CACHE_VAL(ac_cv_path_tclsh, [ - search_path=`echo ${PATH} | sed -e 's/:/ /g'` - for dir in $search_path ; do - for j in `ls -r $dir/tclsh[[8-9]]* 2> /dev/null` \ - `ls -r $dir/tclsh* 2> /dev/null` ; do - if test x"$ac_cv_path_tclsh" = x ; then - if test -f "$j" ; then - ac_cv_path_tclsh=$j - break - fi - fi - done - done - ]) - - if test -f "$ac_cv_path_tclsh" ; then - TCLSH_PROG="$ac_cv_path_tclsh" - AC_MSG_RESULT([$TCLSH_PROG]) - else - # It is not an error if an installed version of Tcl can't be located. - TCLSH_PROG="" - AC_MSG_RESULT([No tclsh found on PATH]) - fi - AC_SUBST(TCLSH_PROG) -]) - -#------------------------------------------------------------------------ -# SC_BUILD_TCLSH -# Determine the fully qualified path name of the tclsh executable -# in the Tcl build directory. This macro will correctly determine -# the name of the tclsh executable even if tclsh has not yet -# been built in the build directory. The build tclsh must be used -# when running tests from an extension build directory. It is not -# correct to use the TCLSH_PROG in cases like this. -# -# Arguments -# none -# -# Results -# Subst's the following values: -# BUILD_TCLSH -#------------------------------------------------------------------------ - -AC_DEFUN([SC_BUILD_TCLSH], [ - AC_MSG_CHECKING([for tclsh in Tcl build directory]) - BUILD_TCLSH="${TCL_BIN_DIR}"/tclsh - AC_MSG_RESULT([$BUILD_TCLSH]) - AC_SUBST(BUILD_TCLSH) -]) - -#------------------------------------------------------------------------ -# SC_ENABLE_SHARED -- -# -# Allows the building of shared libraries -# -# Arguments: -# none -# -# Results: -# -# Adds the following arguments to configure: -# --enable-shared=yes|no -# -# Defines the following vars: -# STATIC_BUILD Used for building import/export libraries -# on Windows. -# -# Sets the following vars: -# SHARED_BUILD Value of 1 or 0 -#------------------------------------------------------------------------ - -AC_DEFUN([SC_ENABLE_SHARED], [ - AC_MSG_CHECKING([how to build libraries]) - AC_ARG_ENABLE(shared, - AC_HELP_STRING([--enable-shared], - [build and link with shared libraries (default: on)]), - [tcl_ok=$enableval], [tcl_ok=yes]) - - if test "${enable_shared+set}" = set; then - enableval="$enable_shared" - tcl_ok=$enableval - else - tcl_ok=yes - fi - - if test "$tcl_ok" = "yes" ; then - AC_MSG_RESULT([shared]) - SHARED_BUILD=1 - else - AC_MSG_RESULT([static]) - SHARED_BUILD=0 - AC_DEFINE(STATIC_BUILD, 1, [Is this a static build?]) - fi -]) - -#------------------------------------------------------------------------ -# SC_ENABLE_FRAMEWORK -- -# -# Allows the building of shared libraries into frameworks -# -# Arguments: -# none -# -# Results: -# -# Adds the following arguments to configure: -# --enable-framework=yes|no -# -# Sets the following vars: -# FRAMEWORK_BUILD Value of 1 or 0 -#------------------------------------------------------------------------ - -AC_DEFUN([SC_ENABLE_FRAMEWORK], [ - if test "`uname -s`" = "Darwin" ; then - AC_MSG_CHECKING([how to package libraries]) - AC_ARG_ENABLE(framework, - AC_HELP_STRING([--enable-framework], - [package shared libraries in MacOSX frameworks (default: off)]), - [enable_framework=$enableval], [enable_framework=no]) - if test $enable_framework = yes; then - if test $SHARED_BUILD = 0; then - AC_MSG_WARN([Frameworks can only be built if --enable-shared is yes]) - enable_framework=no - fi - if test $tcl_corefoundation = no; then - AC_MSG_WARN([Frameworks can only be used when CoreFoundation is available]) - enable_framework=no - fi - fi - if test $enable_framework = yes; then - AC_MSG_RESULT([framework]) - FRAMEWORK_BUILD=1 - else - if test $SHARED_BUILD = 1; then - AC_MSG_RESULT([shared library]) - else - AC_MSG_RESULT([static library]) - fi - FRAMEWORK_BUILD=0 - fi - fi -]) - -#------------------------------------------------------------------------ -# SC_ENABLE_THREADS -- -# -# Specify if thread support should be enabled -# -# Arguments: -# none -# -# Results: -# -# Adds the following arguments to configure: -# --enable-threads -# -# Sets the following vars: -# THREADS_LIBS Thread library(s) -# -# Defines the following vars: -# TCL_THREADS -# _REENTRANT -# _THREAD_SAFE -# -#------------------------------------------------------------------------ - -AC_DEFUN([SC_ENABLE_THREADS], [ - AC_ARG_ENABLE(threads, - AC_HELP_STRING([--enable-threads], - [build with threads (default: off)]), - [tcl_ok=$enableval], [tcl_ok=no]) - - if test "${TCL_THREADS}" = 1; then - tcl_threaded_core=1; - fi - - if test "$tcl_ok" = "yes" -o "${TCL_THREADS}" = 1; then - TCL_THREADS=1 - # USE_THREAD_ALLOC tells us to try the special thread-based - # allocator that significantly reduces lock contention - AC_DEFINE(USE_THREAD_ALLOC, 1, - [Do we want to use the threaded memory allocator?]) - AC_DEFINE(_REENTRANT, 1, [Do we want the reentrant OS API?]) - if test "`uname -s`" = "SunOS" ; then - AC_DEFINE(_POSIX_PTHREAD_SEMANTICS, 1, - [Do we really want to follow the standard? Yes we do!]) - fi - AC_DEFINE(_THREAD_SAFE, 1, [Do we want the thread-safe OS API?]) - AC_CHECK_LIB(pthread,pthread_mutex_init,tcl_ok=yes,tcl_ok=no) - if test "$tcl_ok" = "no"; then - # Check a little harder for __pthread_mutex_init in the same - # library, as some systems hide it there until pthread.h is - # defined. We could alternatively do an AC_TRY_COMPILE with - # pthread.h, but that will work with libpthread really doesn't - # exist, like AIX 4.2. [Bug: 4359] - AC_CHECK_LIB(pthread, __pthread_mutex_init, - tcl_ok=yes, tcl_ok=no) - fi - - if test "$tcl_ok" = "yes"; then - # The space is needed - THREADS_LIBS=" -lpthread" - else - AC_CHECK_LIB(pthreads, pthread_mutex_init, - tcl_ok=yes, tcl_ok=no) - if test "$tcl_ok" = "yes"; then - # The space is needed - THREADS_LIBS=" -lpthreads" - else - AC_CHECK_LIB(c, pthread_mutex_init, - tcl_ok=yes, tcl_ok=no) - if test "$tcl_ok" = "no"; then - AC_CHECK_LIB(c_r, pthread_mutex_init, - tcl_ok=yes, tcl_ok=no) - if test "$tcl_ok" = "yes"; then - # The space is needed - THREADS_LIBS=" -pthread" - else - TCL_THREADS=0 - AC_MSG_WARN([Don't know how to find pthread lib on your system - you must disable thread support or edit the LIBS in the Makefile...]) - fi - fi - fi - fi - - # Does the pthread-implementation provide - # 'pthread_attr_setstacksize' ? - - ac_saved_libs=$LIBS - LIBS="$LIBS $THREADS_LIBS" - AC_CHECK_FUNCS(pthread_attr_setstacksize) - AC_CHECK_FUNC(pthread_attr_get_np,tcl_ok=yes,tcl_ok=no) - if test $tcl_ok = yes ; then - AC_DEFINE(HAVE_PTHREAD_ATTR_GET_NP, 1, - [Do we want a BSD-like thread-attribute interface?]) - AC_CACHE_CHECK([for pthread_attr_get_np declaration], - tcl_cv_grep_pthread_attr_get_np, [ - AC_EGREP_HEADER(pthread_attr_get_np, pthread.h, - tcl_cv_grep_pthread_attr_get_np=present, - tcl_cv_grep_pthread_attr_get_np=missing)]) - if test $tcl_cv_grep_pthread_attr_get_np = missing ; then - AC_DEFINE(ATTRGETNP_NOT_DECLARED, 1, - [Is pthread_attr_get_np() declared in ?]) - fi - else - AC_CHECK_FUNC(pthread_getattr_np,tcl_ok=yes,tcl_ok=no) - if test $tcl_ok = yes ; then - AC_DEFINE(HAVE_PTHREAD_GETATTR_NP, 1, - [Do we want a Linux-like thread-attribute interface?]) - AC_CACHE_CHECK([for pthread_getattr_np declaration], - tcl_cv_grep_pthread_getattr_np, [ - AC_EGREP_HEADER(pthread_getattr_np, pthread.h, - tcl_cv_grep_pthread_getattr_np=present, - tcl_cv_grep_pthread_getattr_np=missing)]) - if test $tcl_cv_grep_pthread_getattr_np = missing ; then - AC_DEFINE(GETATTRNP_NOT_DECLARED, 1, - [Is pthread_getattr_np declared in ?]) - fi - fi - fi - if test $tcl_ok = no; then - # Darwin thread stacksize API - AC_CHECK_FUNCS(pthread_get_stacksize_np) - fi - LIBS=$ac_saved_libs - else - TCL_THREADS=0 - fi - # Do checking message here to not mess up interleaved configure output - AC_MSG_CHECKING([for building with threads]) - if test "${TCL_THREADS}" = 1; then - AC_DEFINE(TCL_THREADS, 1, [Are we building with threads enabled?]) - if test "${tcl_threaded_core}" = 1; then - AC_MSG_RESULT([yes (threaded core)]) - else - AC_MSG_RESULT([yes]) - fi - else - AC_MSG_RESULT([no (default)]) - fi - - AC_SUBST(TCL_THREADS) -]) - -#------------------------------------------------------------------------ -# SC_ENABLE_SYMBOLS -- -# -# Specify if debugging symbols should be used. -# Memory (TCL_MEM_DEBUG) and compile (TCL_COMPILE_DEBUG) debugging -# can also be enabled. -# -# Arguments: -# none -# -# Requires the following vars to be set in the Makefile: -# CFLAGS_DEBUG -# CFLAGS_OPTIMIZE -# LDFLAGS_DEBUG -# LDFLAGS_OPTIMIZE -# -# Results: -# -# Adds the following arguments to configure: -# --enable-symbols -# -# Defines the following vars: -# CFLAGS_DEFAULT Sets to $(CFLAGS_DEBUG) if true -# Sets to $(CFLAGS_OPTIMIZE) if false -# LDFLAGS_DEFAULT Sets to $(LDFLAGS_DEBUG) if true -# Sets to $(LDFLAGS_OPTIMIZE) if false -# DBGX Formerly used as debug library extension; -# always blank now. -# -#------------------------------------------------------------------------ - -AC_DEFUN([SC_ENABLE_SYMBOLS], [ - AC_MSG_CHECKING([for build with symbols]) - AC_ARG_ENABLE(symbols, - AC_HELP_STRING([--enable-symbols], - [build with debugging symbols (default: off)]), - [tcl_ok=$enableval], [tcl_ok=no]) -# FIXME: Currently, LDFLAGS_DEFAULT is not used, it should work like CFLAGS_DEFAULT. - DBGX="" - if test "$tcl_ok" = "no"; then - CFLAGS_DEFAULT='$(CFLAGS_OPTIMIZE)' - LDFLAGS_DEFAULT='$(LDFLAGS_OPTIMIZE)' - AC_MSG_RESULT([no]) - AC_DEFINE(TCL_CFG_OPTIMIZED, 1, [Is this an optimized build?]) - else - CFLAGS_DEFAULT='$(CFLAGS_DEBUG)' - LDFLAGS_DEFAULT='$(LDFLAGS_DEBUG)' - if test "$tcl_ok" = "yes"; then - AC_MSG_RESULT([yes (standard debugging)]) - fi - fi - AC_SUBST(CFLAGS_DEFAULT) - AC_SUBST(LDFLAGS_DEFAULT) - ### FIXME: Surely TCL_CFG_DEBUG should be set to whether we're debugging? - AC_DEFINE(TCL_CFG_DEBUG, 1, [Is debugging enabled?]) - - if test "$tcl_ok" = "mem" -o "$tcl_ok" = "all"; then - AC_DEFINE(TCL_MEM_DEBUG, 1, [Is memory debugging enabled?]) - fi - - ifelse($1,bccdebug,dnl Only enable 'compile' for the Tcl core itself - if test "$tcl_ok" = "compile" -o "$tcl_ok" = "all"; then - AC_DEFINE(TCL_COMPILE_DEBUG, 1, [Is bytecode debugging enabled?]) - AC_DEFINE(TCL_COMPILE_STATS, 1, [Are bytecode statistics enabled?]) - fi) - - if test "$tcl_ok" != "yes" -a "$tcl_ok" != "no"; then - if test "$tcl_ok" = "all"; then - AC_MSG_RESULT([enabled symbols mem ]ifelse($1,bccdebug,[compile ])[debugging]) - else - AC_MSG_RESULT([enabled $tcl_ok debugging]) - fi - fi -]) - -#------------------------------------------------------------------------ -# SC_ENABLE_LANGINFO -- -# -# Allows use of modern nl_langinfo check for better l10n. -# This is only relevant for Unix. -# -# Arguments: -# none -# -# Results: -# -# Adds the following arguments to configure: -# --enable-langinfo=yes|no (default is yes) -# -# Defines the following vars: -# HAVE_LANGINFO Triggers use of nl_langinfo if defined. -# -#------------------------------------------------------------------------ - -AC_DEFUN([SC_ENABLE_LANGINFO], [ - AC_ARG_ENABLE(langinfo, - AC_HELP_STRING([--enable-langinfo], - [use nl_langinfo if possible to determine encoding at startup, otherwise use old heuristic (default: on)]), - [langinfo_ok=$enableval], [langinfo_ok=yes]) - - HAVE_LANGINFO=0 - if test "$langinfo_ok" = "yes"; then - AC_CHECK_HEADER(langinfo.h,[langinfo_ok=yes],[langinfo_ok=no]) - fi - AC_MSG_CHECKING([whether to use nl_langinfo]) - if test "$langinfo_ok" = "yes"; then - AC_CACHE_VAL(tcl_cv_langinfo_h, [ - AC_TRY_COMPILE([#include ], [nl_langinfo(CODESET);], - [tcl_cv_langinfo_h=yes],[tcl_cv_langinfo_h=no])]) - AC_MSG_RESULT([$tcl_cv_langinfo_h]) - if test $tcl_cv_langinfo_h = yes; then - AC_DEFINE(HAVE_LANGINFO, 1, [Do we have nl_langinfo()?]) - fi - else - AC_MSG_RESULT([$langinfo_ok]) - fi -]) - -#-------------------------------------------------------------------- -# SC_CONFIG_MANPAGES -# -# Decide whether to use symlinks for linking the manpages, -# whether to compress the manpages after installation, and -# whether to add a package name suffix to the installed -# manpages to avoidfile name clashes. -# If compression is enabled also find out what file name suffix -# the given compression program is using. -# -# Arguments: -# none -# -# Results: -# -# Adds the following arguments to configure: -# --enable-man-symlinks -# --enable-man-compression=PROG -# --enable-man-suffix[=STRING] -# -# Defines the following variable: -# -# MAN_FLAGS - The apropriate flags for installManPage -# according to the user's selection. -# -#-------------------------------------------------------------------- - -AC_DEFUN([SC_CONFIG_MANPAGES], [ - AC_MSG_CHECKING([whether to use symlinks for manpages]) - AC_ARG_ENABLE(man-symlinks, - AC_HELP_STRING([--enable-man-symlinks], - [use symlinks for the manpages (default: off)]), - test "$enableval" != "no" && MAN_FLAGS="$MAN_FLAGS --symlinks", - enableval="no") - AC_MSG_RESULT([$enableval]) - - AC_MSG_CHECKING([whether to compress the manpages]) - AC_ARG_ENABLE(man-compression, - AC_HELP_STRING([--enable-man-compression=PROG], - [compress the manpages with PROG (default: off)]), - [case $enableval in - yes) AC_MSG_ERROR([missing argument to --enable-man-compression]);; - no) ;; - *) MAN_FLAGS="$MAN_FLAGS --compress $enableval";; - esac], - enableval="no") - AC_MSG_RESULT([$enableval]) - if test "$enableval" != "no"; then - AC_MSG_CHECKING([for compressed file suffix]) - touch TeST - $enableval TeST - Z=`ls TeST* | sed 's/^....//'` - rm -f TeST* - MAN_FLAGS="$MAN_FLAGS --extension $Z" - AC_MSG_RESULT([$Z]) - fi - - AC_MSG_CHECKING([whether to add a package name suffix for the manpages]) - AC_ARG_ENABLE(man-suffix, - AC_HELP_STRING([--enable-man-suffix=STRING], - [use STRING as a suffix to manpage file names (default: no, AC_PACKAGE_NAME if enabled without specifying STRING)]), - [case $enableval in - yes) enableval="AC_PACKAGE_NAME" MAN_FLAGS="$MAN_FLAGS --suffix $enableval";; - no) ;; - *) MAN_FLAGS="$MAN_FLAGS --suffix $enableval";; - esac], - enableval="no") - AC_MSG_RESULT([$enableval]) - - AC_SUBST(MAN_FLAGS) -]) - -#-------------------------------------------------------------------- -# SC_CONFIG_SYSTEM -# -# Determine what the system is (some things cannot be easily checked -# on a feature-driven basis, alas). This can usually be done via the -# "uname" command, but there are a few systems, like Next, where -# this doesn't work. -# -# Arguments: -# none -# -# Results: -# Defines the following var: -# -# system - System/platform/version identification code. -# -#-------------------------------------------------------------------- - -AC_DEFUN([SC_CONFIG_SYSTEM], [ - AC_CACHE_CHECK([system version], tcl_cv_sys_version, [ - if test -f /usr/lib/NextStep/software_version; then - tcl_cv_sys_version=NEXTSTEP-`awk '/3/,/3/' /usr/lib/NextStep/software_version` - else - tcl_cv_sys_version=`uname -s`-`uname -r` - if test "$?" -ne 0 ; then - AC_MSG_WARN([can't find uname command]) - tcl_cv_sys_version=unknown - else - # Special check for weird MP-RAS system (uname returns weird - # results, and the version is kept in special file). - - if test -r /etc/.relid -a "X`uname -n`" = "X`uname -s`" ; then - tcl_cv_sys_version=MP-RAS-`awk '{print $[3]}' /etc/.relid` - fi - if test "`uname -s`" = "AIX" ; then - tcl_cv_sys_version=AIX-`uname -v`.`uname -r` - fi - fi - fi - ]) - system=$tcl_cv_sys_version -]) - -#-------------------------------------------------------------------- -# SC_CONFIG_CFLAGS -# -# Try to determine the proper flags to pass to the compiler -# for building shared libraries and other such nonsense. -# -# Arguments: -# none -# -# Results: -# -# Defines and substitutes the following vars: -# -# DL_OBJS - Name of the object file that implements dynamic -# loading for Tcl on this system. -# DL_LIBS - Library file(s) to include in tclsh and other base -# applications in order for the "load" command to work. -# LDFLAGS - Flags to pass to the compiler when linking object -# files into an executable application binary such -# as tclsh. -# LD_SEARCH_FLAGS-Flags to pass to ld, such as "-R /usr/local/tcl/lib", -# that tell the run-time dynamic linker where to look -# for shared libraries such as libtcl.so. Depends on -# the variable LIB_RUNTIME_DIR in the Makefile. Could -# be the same as CC_SEARCH_FLAGS if ${CC} is used to link. -# CC_SEARCH_FLAGS-Flags to pass to ${CC}, such as "-Wl,-rpath,/usr/local/tcl/lib", -# that tell the run-time dynamic linker where to look -# for shared libraries such as libtcl.so. Depends on -# the variable LIB_RUNTIME_DIR in the Makefile. -# MAKE_LIB - Command to execute to build the a library; -# differs when building shared or static. -# MAKE_STUB_LIB - -# Command to execute to build a stub library. -# INSTALL_LIB - Command to execute to install a library; -# differs when building shared or static. -# INSTALL_STUB_LIB - -# Command to execute to install a stub library. -# STLIB_LD - Base command to use for combining object files -# into a static library. -# SHLIB_CFLAGS - Flags to pass to cc when compiling the components -# of a shared library (may request position-independent -# code, among other things). -# SHLIB_LD - Base command to use for combining object files -# into a shared library. -# SHLIB_LD_LIBS - Dependent libraries for the linker to scan when -# creating shared libraries. This symbol typically -# goes at the end of the "ld" commands that build -# shared libraries. The value of the symbol is -# "${LIBS}" if all of the dependent libraries should -# be specified when creating a shared library. If -# dependent libraries should not be specified (as on -# SunOS 4.x, where they cause the link to fail, or in -# general if Tcl and Tk aren't themselves shared -# libraries), then this symbol has an empty string -# as its value. -# SHLIB_SUFFIX - Suffix to use for the names of dynamically loadable -# extensions. An empty string means we don't know how -# to use shared libraries on this platform. -# TCL_SHLIB_LD_EXTRAS - Additional element which are added to SHLIB_LD_LIBS -# TK_SHLIB_LD_EXTRAS for the build of Tcl and Tk, but not recorded in the -# tclConfig.sh, since they are only used for the build -# of Tcl and Tk. -# Examples: MacOS X records the library version and -# compatibility version in the shared library. But -# of course the Tcl version of this is only used for Tcl. -# LIB_SUFFIX - Specifies everything that comes after the "libfoo" -# in a static or shared library name, using the $VERSION variable -# to put the version in the right place. This is used -# by platforms that need non-standard library names. -# Examples: ${VERSION}.so.1.1 on NetBSD, since it needs -# to have a version after the .so, and ${VERSION}.a -# on AIX, since a shared library needs to have -# a .a extension whereas shared objects for loadable -# extensions have a .so extension. Defaults to -# ${VERSION}${SHLIB_SUFFIX}. -# TCL_NEEDS_EXP_FILE - -# 1 means that an export file is needed to link to a -# shared library. -# TCL_EXP_FILE - The name of the installed export / import file which -# should be used to link to the Tcl shared library. -# Empty if Tcl is unshared. -# TCL_BUILD_EXP_FILE - -# The name of the built export / import file which -# should be used to link to the Tcl shared library. -# Empty if Tcl is unshared. -# TCL_LIBS - -# Libs to use when linking Tcl shell or some other -# shell that includes Tcl libs. -# CFLAGS_DEBUG - -# Flags used when running the compiler in debug mode -# CFLAGS_OPTIMIZE - -# Flags used when running the compiler in optimize mode -# CFLAGS - Additional CFLAGS added as necessary (usually 64-bit) -# -#-------------------------------------------------------------------- - -AC_DEFUN([SC_CONFIG_CFLAGS], [ - - # Step 0.a: Enable 64 bit support? - - AC_MSG_CHECKING([if 64bit support is requested]) - AC_ARG_ENABLE(64bit, - AC_HELP_STRING([--enable-64bit], - [enable 64bit support (default: off)]), - [do64bit=$enableval], [do64bit=no]) - AC_MSG_RESULT([$do64bit]) - - # Step 0.b: Enable Solaris 64 bit VIS support? - - AC_MSG_CHECKING([if 64bit Sparc VIS support is requested]) - AC_ARG_ENABLE(64bit-vis, - AC_HELP_STRING([--enable-64bit-vis], - [enable 64bit Sparc VIS support (default: off)]), - [do64bitVIS=$enableval], [do64bitVIS=no]) - AC_MSG_RESULT([$do64bitVIS]) - # Force 64bit on with VIS - AS_IF([test "$do64bitVIS" = "yes"], [do64bit=yes]) - - # Step 0.c: Check if visibility support is available. Do this here so - # that platform specific alternatives can be used below if this fails. - - AC_CACHE_CHECK([if compiler supports visibility "hidden"], - tcl_cv_cc_visibility_hidden, [ - hold_cflags=$CFLAGS; CFLAGS="$CFLAGS -Werror" - AC_TRY_LINK([ - extern __attribute__((__visibility__("hidden"))) void f(void); - void f(void) {}], [f();], tcl_cv_cc_visibility_hidden=yes, - tcl_cv_cc_visibility_hidden=no) - CFLAGS=$hold_cflags]) - AS_IF([test $tcl_cv_cc_visibility_hidden = yes], [ - AC_DEFINE(MODULE_SCOPE, - [extern __attribute__((__visibility__("hidden")))], - [Compiler support for module scope symbols]) - ]) - - # Step 0.d: Disable -rpath support? - - AC_MSG_CHECKING([if rpath support is requested]) - AC_ARG_ENABLE(rpath, - AC_HELP_STRING([--disable-rpath], - [disable rpath support (default: on)]), - [doRpath=$enableval], [doRpath=yes]) - AC_MSG_RESULT([$doRpath]) - - # Step 1: set the variable "system" to hold the name and version number - # for the system. - - SC_CONFIG_SYSTEM - - # Step 2: check for existence of -ldl library. This is needed because - # Linux can use either -ldl or -ldld for dynamic loading. - - AC_CHECK_LIB(dl, dlopen, have_dl=yes, have_dl=no) - - # Require ranlib early so we can override it in special cases below. - LDAIX_SRC="" - AS_IF([test x"${SHLIB_VERSION}" = x], [SHLIB_VERSION="1.0"]) - - AC_REQUIRE([AC_PROG_RANLIB]) - - # Step 3: set configuration options based on system name and version. - - do64bit_ok=no - LDFLAGS_ORIG="$LDFLAGS" - # When ld needs options to work in 64-bit mode, put them in - # LDFLAGS_ARCH so they eventually end up in LDFLAGS even if [load] - # is disabled by the user. [Bug 1016796] - LDFLAGS_ARCH="" - TCL_EXPORT_FILE_SUFFIX="" - UNSHARED_LIB_SUFFIX="" - TCL_TRIM_DOTS='`echo ${VERSION} | tr -d .`' - ECHO_VERSION='`echo ${VERSION}`' - TCL_LIB_VERSIONS_OK=ok - CFLAGS_DEBUG=-g - CFLAGS_OPTIMIZE=-O - AS_IF([test "$GCC" = yes], [ - CFLAGS_WARNING="-Wall" - ], [CFLAGS_WARNING=""]) - TCL_NEEDS_EXP_FILE=0 - TCL_BUILD_EXP_FILE="" - TCL_EXP_FILE="" -dnl FIXME: Replace AC_CHECK_PROG with AC_CHECK_TOOL once cross compiling is fixed. -dnl AC_CHECK_TOOL(AR, ar) - AC_CHECK_PROG(AR, ar, ar) - AS_IF([test "${AR}" = ""], [ - AC_MSG_ERROR([Required archive tool 'ar' not found on PATH.]) - ]) - STLIB_LD='${AR} cr' - LD_LIBRARY_PATH_VAR="LD_LIBRARY_PATH" - PLAT_OBJS="" - PLAT_SRCS="" - case $system in - AIX-*) - AS_IF([test "${TCL_THREADS}" = "1" -a "$GCC" != "yes"], [ - # AIX requires the _r compiler when gcc isn't being used - case "${CC}" in - *_r) - # ok ... - ;; - *) - CC=${CC}_r - ;; - esac - AC_MSG_RESULT([Using $CC for compiling with threads]) - ]) - LIBS="$LIBS -lc" - SHLIB_CFLAGS="" - # Note: need the LIBS below, otherwise Tk won't find Tcl's - # symbols when dynamically loaded into tclsh. - SHLIB_LD_LIBS='${LIBS}' - SHLIB_SUFFIX=".so" - - DL_OBJS="tclLoadDl.o" - LD_LIBRARY_PATH_VAR="LIBPATH" - - # Check to enable 64-bit flags for compiler/linker on AIX 4+ - LDAIX_SRC='$(UNIX_DIR)/ldAix' - AS_IF([test "$do64bit" = yes -a "`uname -v`" -gt 3], [ - AS_IF([test "$GCC" = yes], [ - AC_MSG_WARN([64bit mode not supported with GCC on $system]) - ], [ - do64bit_ok=yes - CFLAGS="$CFLAGS -q64" - LDFLAGS_ARCH="-q64" - RANLIB="${RANLIB} -X64" - AR="${AR} -X64" - SHLIB_LD_FLAGS="-b64" - ]) - ]) - - AS_IF([test "`uname -m`" = ia64], [ - # AIX-5 uses ELF style dynamic libraries on IA-64, but not PPC - SHLIB_LD="/usr/ccs/bin/ld -G -z text" - # AIX-5 has dl* in libc.so - DL_LIBS="" - AS_IF([test "$GCC" = yes], [ - CC_SEARCH_FLAGS='-Wl,-R,${LIB_RUNTIME_DIR}' - ], [ - CC_SEARCH_FLAGS='-R${LIB_RUNTIME_DIR}' - ]) - LD_SEARCH_FLAGS='-R ${LIB_RUNTIME_DIR}' - ], [ - AS_IF([test "$GCC" = yes], [SHLIB_LD='${CC} -shared'], [ - SHLIB_LD="/bin/ld -bhalt:4 -bM:SRE -bE:lib.exp -H512 -T512 -bnoentry" - ]) - SHLIB_LD="${TCL_SRC_DIR}/unix/ldAix ${SHLIB_LD} ${SHLIB_LD_FLAGS}" - DL_LIBS="-ldl" - CC_SEARCH_FLAGS='-L${LIB_RUNTIME_DIR}' - LD_SEARCH_FLAGS=${CC_SEARCH_FLAGS} - TCL_NEEDS_EXP_FILE=1 - TCL_EXPORT_FILE_SUFFIX='${VERSION}.exp' - ]) - - # AIX v<=4.1 has some different flags than 4.2+ - AS_IF([test "$system" = "AIX-4.1" -o "`uname -v`" -lt 4], [ - AC_LIBOBJ([tclLoadAix]) - DL_LIBS="-lld" - ]) - - # On AIX <=v4 systems, libbsd.a has to be linked in to support - # non-blocking file IO. This library has to be linked in after - # the MATH_LIBS or it breaks the pow() function. The way to - # insure proper sequencing, is to add it to the tail of MATH_LIBS. - # This library also supplies gettimeofday. - # - # AIX does not have a timezone field in struct tm. When the AIX - # bsd library is used, the timezone global and the gettimeofday - # methods are to be avoided for timezone deduction instead, we - # deduce the timezone by comparing the localtime result on a - # known GMT value. - - AC_CHECK_LIB(bsd, gettimeofday, libbsd=yes, libbsd=no) - AS_IF([test $libbsd = yes], [ - MATH_LIBS="$MATH_LIBS -lbsd" - AC_DEFINE(USE_DELTA_FOR_TZ, 1, [Do we need a special AIX hack for timezones?]) - ]) - ;; - BeOS*) - SHLIB_CFLAGS="-fPIC" - SHLIB_LD='${CC} -nostart' - SHLIB_LD_LIBS='${LIBS}' - SHLIB_SUFFIX=".so" - DL_OBJS="tclLoadDl.o" - DL_LIBS="-ldl" - - #----------------------------------------------------------- - # Check for inet_ntoa in -lbind, for BeOS (which also needs - # -lsocket, even if the network functions are in -lnet which - # is always linked to, for compatibility. - #----------------------------------------------------------- - AC_CHECK_LIB(bind, inet_ntoa, [LIBS="$LIBS -lbind -lsocket"]) - ;; - BSD/OS-2.1*|BSD/OS-3*) - SHLIB_CFLAGS="" - SHLIB_LD="shlicc -r" - SHLIB_LD_LIBS='${LIBS}' - SHLIB_SUFFIX=".so" - DL_OBJS="tclLoadDl.o" - DL_LIBS="-ldl" - CC_SEARCH_FLAGS="" - LD_SEARCH_FLAGS="" - ;; - BSD/OS-4.*) - SHLIB_CFLAGS="-export-dynamic -fPIC" - SHLIB_LD='${CC} -shared' - SHLIB_LD_LIBS='${LIBS}' - SHLIB_SUFFIX=".so" - DL_OBJS="tclLoadDl.o" - DL_LIBS="-ldl" - LDFLAGS="$LDFLAGS -export-dynamic" - CC_SEARCH_FLAGS="" - LD_SEARCH_FLAGS="" - ;; - dgux*) - SHLIB_CFLAGS="-K PIC" - SHLIB_LD='${CC} -G' - SHLIB_LD_LIBS="" - SHLIB_SUFFIX=".so" - DL_OBJS="tclLoadDl.o" - DL_LIBS="-ldl" - CC_SEARCH_FLAGS="" - LD_SEARCH_FLAGS="" - ;; - HP-UX-*.11.*) - # Use updated header definitions where possible - AC_DEFINE(_XOPEN_SOURCE_EXTENDED, 1, [Do we want to use the XOPEN network library?]) - AC_DEFINE(_XOPEN_SOURCE, 1, [Do we want to use the XOPEN network library?]) - LIBS="$LIBS -lxnet" # Use the XOPEN network library - - AS_IF([test "`uname -m`" = ia64], [ - SHLIB_SUFFIX=".so" - ], [ - SHLIB_SUFFIX=".sl" - ]) - AC_CHECK_LIB(dld, shl_load, tcl_ok=yes, tcl_ok=no) - AS_IF([test "$tcl_ok" = yes], [ - SHLIB_CFLAGS="+z" - SHLIB_LD="ld -b" - SHLIB_LD_LIBS='${LIBS}' - DL_OBJS="tclLoadShl.o" - DL_LIBS="-ldld" - LDFLAGS="$LDFLAGS -Wl,-E" - CC_SEARCH_FLAGS='-Wl,+s,+b,${LIB_RUNTIME_DIR}:.' - LD_SEARCH_FLAGS='+s +b ${LIB_RUNTIME_DIR}:.' - LD_LIBRARY_PATH_VAR="SHLIB_PATH" - ]) - AS_IF([test "$GCC" = yes], [ - SHLIB_LD='${CC} -shared' - SHLIB_LD_LIBS='${LIBS}' - LD_SEARCH_FLAGS=${CC_SEARCH_FLAGS} - ]) - - # Users may want PA-RISC 1.1/2.0 portable code - needs HP cc - #CFLAGS="$CFLAGS +DAportable" - - # Check to enable 64-bit flags for compiler/linker - AS_IF([test "$do64bit" = "yes"], [ - AS_IF([test "$GCC" = yes], [ - case `${CC} -dumpmachine` in - hppa64*) - # 64-bit gcc in use. Fix flags for GNU ld. - do64bit_ok=yes - SHLIB_LD='${CC} -shared' - SHLIB_LD_LIBS='${LIBS}' - AS_IF([test $doRpath = yes], [ - CC_SEARCH_FLAGS='-Wl,-rpath,${LIB_RUNTIME_DIR}']) - LD_SEARCH_FLAGS=${CC_SEARCH_FLAGS} - ;; - *) - AC_MSG_WARN([64bit mode not supported with GCC on $system]) - ;; - esac - ], [ - do64bit_ok=yes - CFLAGS="$CFLAGS +DD64" - LDFLAGS_ARCH="+DD64" - ]) - ]) ;; - HP-UX-*.08.*|HP-UX-*.09.*|HP-UX-*.10.*) - SHLIB_SUFFIX=".sl" - AC_CHECK_LIB(dld, shl_load, tcl_ok=yes, tcl_ok=no) - AS_IF([test "$tcl_ok" = yes], [ - SHLIB_CFLAGS="+z" - SHLIB_LD="ld -b" - SHLIB_LD_LIBS="" - DL_OBJS="tclLoadShl.o" - DL_LIBS="-ldld" - LDFLAGS="$LDFLAGS -Wl,-E" - CC_SEARCH_FLAGS='-Wl,+s,+b,${LIB_RUNTIME_DIR}:.' - LD_SEARCH_FLAGS='+s +b ${LIB_RUNTIME_DIR}:.' - LD_LIBRARY_PATH_VAR="SHLIB_PATH" - ]) ;; - IRIX-5.*) - SHLIB_CFLAGS="" - SHLIB_LD="ld -shared -rdata_shared" - SHLIB_LD_LIBS='${LIBS}' - SHLIB_SUFFIX=".so" - DL_OBJS="tclLoadDl.o" - DL_LIBS="" - AS_IF([test $doRpath = yes], [ - CC_SEARCH_FLAGS='-Wl,-rpath,${LIB_RUNTIME_DIR}' - LD_SEARCH_FLAGS='-rpath ${LIB_RUNTIME_DIR}']) - ;; - IRIX-6.*) - SHLIB_CFLAGS="" - SHLIB_LD="ld -n32 -shared -rdata_shared" - SHLIB_LD_LIBS='${LIBS}' - SHLIB_SUFFIX=".so" - DL_OBJS="tclLoadDl.o" - DL_LIBS="" - AS_IF([test $doRpath = yes], [ - CC_SEARCH_FLAGS='-Wl,-rpath,${LIB_RUNTIME_DIR}' - LD_SEARCH_FLAGS='-rpath ${LIB_RUNTIME_DIR}']) - AS_IF([test "$GCC" = yes], [ - CFLAGS="$CFLAGS -mabi=n32" - LDFLAGS="$LDFLAGS -mabi=n32" - ], [ - case $system in - IRIX-6.3) - # Use to build 6.2 compatible binaries on 6.3. - CFLAGS="$CFLAGS -n32 -D_OLD_TERMIOS" - ;; - *) - CFLAGS="$CFLAGS -n32" - ;; - esac - LDFLAGS="$LDFLAGS -n32" - ]) - ;; - IRIX64-6.*) - SHLIB_CFLAGS="" - SHLIB_LD="ld -n32 -shared -rdata_shared" - SHLIB_LD_LIBS='${LIBS}' - SHLIB_SUFFIX=".so" - DL_OBJS="tclLoadDl.o" - DL_LIBS="" - AS_IF([test $doRpath = yes], [ - CC_SEARCH_FLAGS='-Wl,-rpath,${LIB_RUNTIME_DIR}' - LD_SEARCH_FLAGS='-rpath ${LIB_RUNTIME_DIR}']) - - # Check to enable 64-bit flags for compiler/linker - - AS_IF([test "$do64bit" = yes], [ - AS_IF([test "$GCC" = yes], [ - AC_MSG_WARN([64bit mode not supported by gcc]) - ], [ - do64bit_ok=yes - SHLIB_LD="ld -64 -shared -rdata_shared" - CFLAGS="$CFLAGS -64" - LDFLAGS_ARCH="-64" - ]) - ]) - ;; - Linux*) - SHLIB_CFLAGS="-fPIC" - SHLIB_LD_LIBS='${LIBS}' - SHLIB_SUFFIX=".so" - - CFLAGS_OPTIMIZE="-O2" - # egcs-2.91.66 on Redhat Linux 6.0 generates lots of warnings - # when you inline the string and math operations. Turn this off to - # get rid of the warnings. - #CFLAGS_OPTIMIZE="${CFLAGS_OPTIMIZE} -D__NO_STRING_INLINES -D__NO_MATH_INLINES" - - SHLIB_LD='${CC} -shared ${CFLAGS} ${LDFLAGS}' - DL_OBJS="tclLoadDl.o" - DL_LIBS="-ldl" - LDFLAGS="$LDFLAGS -Wl,--export-dynamic" - AS_IF([test $doRpath = yes], [ - CC_SEARCH_FLAGS='-Wl,-rpath,${LIB_RUNTIME_DIR}']) - LD_SEARCH_FLAGS=${CC_SEARCH_FLAGS} - AS_IF([test "`uname -m`" = "alpha"], [CFLAGS="$CFLAGS -mieee"]) - AS_IF([test $do64bit = yes], [ - AC_CACHE_CHECK([if compiler accepts -m64 flag], tcl_cv_cc_m64, [ - hold_cflags=$CFLAGS - CFLAGS="$CFLAGS -m64" - AC_TRY_LINK(,, tcl_cv_cc_m64=yes, tcl_cv_cc_m64=no) - CFLAGS=$hold_cflags]) - AS_IF([test $tcl_cv_cc_m64 = yes], [ - CFLAGS="$CFLAGS -m64" - do64bit_ok=yes - ]) - ]) - - # The combo of gcc + glibc has a bug related to inlining of - # functions like strtod(). The -fno-builtin flag should address - # this problem but it does not work. The -fno-inline flag is kind - # of overkill but it works. Disable inlining only when one of the - # files in compat/*.c is being linked in. - - AS_IF([test x"${USE_COMPAT}" != x],[CFLAGS="$CFLAGS -fno-inline"]) - ;; - GNU*) - SHLIB_CFLAGS="-fPIC" - SHLIB_LD_LIBS='${LIBS}' - SHLIB_SUFFIX=".so" - - SHLIB_LD='${CC} -shared' - DL_OBJS="" - DL_LIBS="-ldl" - LDFLAGS="$LDFLAGS -Wl,--export-dynamic" - CC_SEARCH_FLAGS="" - LD_SEARCH_FLAGS="" - AS_IF([test "`uname -m`" = "alpha"], [CFLAGS="$CFLAGS -mieee"]) - ;; - Lynx*) - SHLIB_CFLAGS="-fPIC" - SHLIB_LD_LIBS='${LIBS}' - SHLIB_SUFFIX=".so" - CFLAGS_OPTIMIZE=-02 - SHLIB_LD='${CC} -shared' - DL_OBJS="tclLoadDl.o" - DL_LIBS="-mshared -ldl" - LD_FLAGS="-Wl,--export-dynamic" - AS_IF([test $doRpath = yes], [ - CC_SEARCH_FLAGS='-Wl,-rpath,${LIB_RUNTIME_DIR}' - LD_SEARCH_FLAGS='-Wl,-rpath,${LIB_RUNTIME_DIR}']) - ;; - MP-RAS-02*) - SHLIB_CFLAGS="-K PIC" - SHLIB_LD='${CC} -G' - SHLIB_LD_LIBS="" - SHLIB_SUFFIX=".so" - DL_OBJS="tclLoadDl.o" - DL_LIBS="-ldl" - CC_SEARCH_FLAGS="" - LD_SEARCH_FLAGS="" - ;; - MP-RAS-*) - SHLIB_CFLAGS="-K PIC" - SHLIB_LD='${CC} -G' - SHLIB_LD_LIBS="" - SHLIB_SUFFIX=".so" - DL_OBJS="tclLoadDl.o" - DL_LIBS="-ldl" - LDFLAGS="$LDFLAGS -Wl,-Bexport" - CC_SEARCH_FLAGS="" - LD_SEARCH_FLAGS="" - ;; - NetBSD-1.*|FreeBSD-[[1-2]].*) - SHLIB_CFLAGS="-fPIC" - SHLIB_LD="ld -Bshareable -x" - SHLIB_LD_LIBS='${LIBS}' - SHLIB_SUFFIX=".so" - DL_OBJS="tclLoadDl.o" - DL_LIBS="" - AS_IF([test $doRpath = yes], [ - CC_SEARCH_FLAGS='-Wl,-rpath,${LIB_RUNTIME_DIR}' - LD_SEARCH_FLAGS='-rpath ${LIB_RUNTIME_DIR}']) - AC_CACHE_CHECK([for ELF], tcl_cv_ld_elf, [ - AC_EGREP_CPP(yes, [ -#ifdef __ELF__ - yes -#endif - ], tcl_cv_ld_elf=yes, tcl_cv_ld_elf=no)]) - AS_IF([test $tcl_cv_ld_elf = yes], [ - SHARED_LIB_SUFFIX='${TCL_TRIM_DOTS}.so' - ], [ - SHARED_LIB_SUFFIX='${TCL_TRIM_DOTS}.so.${SHLIB_VERSION}' - ]) - - # Ancient FreeBSD doesn't handle version numbers with dots. - - UNSHARED_LIB_SUFFIX='${TCL_TRIM_DOTS}.a' - TCL_LIB_VERSIONS_OK=nodots - ;; - OpenBSD-*) - CFLAGS_OPTIMIZE='-O2' - SHLIB_CFLAGS="-fPIC" - SHLIB_LD='${CC} -shared ${SHLIB_CFLAGS}' - SHLIB_LD_LIBS='${LIBS}' - SHLIB_SUFFIX=".so" - DL_OBJS="tclLoadDl.o" - DL_LIBS="" - AS_IF([test $doRpath = yes], [ - CC_SEARCH_FLAGS='-Wl,-rpath,${LIB_RUNTIME_DIR}']) - LD_SEARCH_FLAGS=${CC_SEARCH_FLAGS} - SHARED_LIB_SUFFIX='${TCL_TRIM_DOTS}.so.${SHLIB_VERSION}' - AC_CACHE_CHECK([for ELF], tcl_cv_ld_elf, [ - AC_EGREP_CPP(yes, [ -#ifdef __ELF__ - yes -#endif - ], tcl_cv_ld_elf=yes, tcl_cv_ld_elf=no)]) - AS_IF([test $tcl_cv_ld_elf = yes], [ - LDFLAGS=-Wl,-export-dynamic - ], [LDFLAGS=""]) - - # OpenBSD doesn't do version numbers with dots. - UNSHARED_LIB_SUFFIX='${TCL_TRIM_DOTS}.a' - TCL_LIB_VERSIONS_OK=nodots - ;; - NetBSD-*|FreeBSD-*) - # FreeBSD 3.* and greater have ELF. - # NetBSD 2.* has ELF and can use 'cc -shared' to build shared libs - SHLIB_CFLAGS="-fPIC" - SHLIB_LD='${CC} -shared ${SHLIB_CFLAGS}' - SHLIB_LD_LIBS='${LIBS}' - SHLIB_SUFFIX=".so" - DL_OBJS="tclLoadDl.o" - DL_LIBS="" - LDFLAGS="$LDFLAGS -export-dynamic" - AS_IF([test $doRpath = yes], [ - CC_SEARCH_FLAGS='-Wl,-rpath,${LIB_RUNTIME_DIR}']) - LD_SEARCH_FLAGS=${CC_SEARCH_FLAGS} - AS_IF([test "${TCL_THREADS}" = "1"], [ - # The -pthread needs to go in the CFLAGS, not LIBS - LIBS=`echo $LIBS | sed s/-pthread//` - CFLAGS="$CFLAGS -pthread" - LDFLAGS="$LDFLAGS -pthread" - ]) - case $system in - FreeBSD-3.*) - # FreeBSD-3 doesn't handle version numbers with dots. - UNSHARED_LIB_SUFFIX='${TCL_TRIM_DOTS}.a' - SHARED_LIB_SUFFIX='${TCL_TRIM_DOTS}.so' - TCL_LIB_VERSIONS_OK=nodots - ;; - esac - ;; - Darwin-*) - CFLAGS_OPTIMIZE="-Os" - SHLIB_CFLAGS="-fno-common" - # To avoid discrepancies between what headers configure sees during - # preprocessing tests and compiling tests, move any -isysroot and - # -mmacosx-version-min flags from CFLAGS to CPPFLAGS: - CPPFLAGS="${CPPFLAGS} `echo " ${CFLAGS}" | \ - awk 'BEGIN {FS=" +-";ORS=" "}; {for (i=2;i<=NF;i++) \ - if ([$]i~/^(isysroot|mmacosx-version-min)/) print "-"[$]i}'`" - CFLAGS="`echo " ${CFLAGS}" | \ - awk 'BEGIN {FS=" +-";ORS=" "}; {for (i=2;i<=NF;i++) \ - if (!([$]i~/^(isysroot|mmacosx-version-min)/)) print "-"[$]i}'`" - AS_IF([test $do64bit = yes], [ - case `arch` in - ppc) - AC_CACHE_CHECK([if compiler accepts -arch ppc64 flag], - tcl_cv_cc_arch_ppc64, [ - hold_cflags=$CFLAGS - CFLAGS="$CFLAGS -arch ppc64 -mpowerpc64 -mcpu=G5" - AC_TRY_LINK(,, tcl_cv_cc_arch_ppc64=yes, - tcl_cv_cc_arch_ppc64=no) - CFLAGS=$hold_cflags]) - AS_IF([test $tcl_cv_cc_arch_ppc64 = yes], [ - CFLAGS="$CFLAGS -arch ppc64 -mpowerpc64 -mcpu=G5" - do64bit_ok=yes - ]);; - i386) - AC_CACHE_CHECK([if compiler accepts -arch x86_64 flag], - tcl_cv_cc_arch_x86_64, [ - hold_cflags=$CFLAGS - CFLAGS="$CFLAGS -arch x86_64" - AC_TRY_LINK(,, tcl_cv_cc_arch_x86_64=yes, - tcl_cv_cc_arch_x86_64=no) - CFLAGS=$hold_cflags]) - AS_IF([test $tcl_cv_cc_arch_x86_64 = yes], [ - CFLAGS="$CFLAGS -arch x86_64" - do64bit_ok=yes - ]);; - *) - AC_MSG_WARN([Don't know how enable 64-bit on architecture `arch`]);; - esac - ], [ - # Check for combined 32-bit and 64-bit fat build - AS_IF([echo "$CFLAGS " |grep -E -q -- '-arch (ppc64|x86_64) ' \ - && echo "$CFLAGS " |grep -E -q -- '-arch (ppc|i386) '], [ - fat_32_64=yes]) - ]) - SHLIB_LD='${CC} -dynamiclib ${CFLAGS} ${LDFLAGS}' - AC_CACHE_CHECK([if ld accepts -single_module flag], tcl_cv_ld_single_module, [ - hold_ldflags=$LDFLAGS - LDFLAGS="$LDFLAGS -dynamiclib -Wl,-single_module" - AC_TRY_LINK(, [int i;], tcl_cv_ld_single_module=yes, tcl_cv_ld_single_module=no) - LDFLAGS=$hold_ldflags]) - AS_IF([test $tcl_cv_ld_single_module = yes], [ - SHLIB_LD="${SHLIB_LD} -Wl,-single_module" - ]) - SHLIB_LD_LIBS='${LIBS}' - SHLIB_SUFFIX=".dylib" - DL_OBJS="tclLoadDyld.o" - DL_LIBS="" - # Don't use -prebind when building for Mac OS X 10.4 or later only: - AS_IF([test "`echo "${MACOSX_DEPLOYMENT_TARGET}" | awk -F '10\\.' '{print int([$]2)}'`" -lt 4 -a \ - "`echo "${CPPFLAGS}" | awk -F '-mmacosx-version-min=10\\.' '{print int([$]2)}'`" -lt 4], [ - LDFLAGS="$LDFLAGS -prebind"]) - LDFLAGS="$LDFLAGS -headerpad_max_install_names" - AC_CACHE_CHECK([if ld accepts -search_paths_first flag], - tcl_cv_ld_search_paths_first, [ - hold_ldflags=$LDFLAGS - LDFLAGS="$LDFLAGS -Wl,-search_paths_first" - AC_TRY_LINK(, [int i;], tcl_cv_ld_search_paths_first=yes, - tcl_cv_ld_search_paths_first=no) - LDFLAGS=$hold_ldflags]) - AS_IF([test $tcl_cv_ld_search_paths_first = yes], [ - LDFLAGS="$LDFLAGS -Wl,-search_paths_first" - ]) - AS_IF([test "$tcl_cv_cc_visibility_hidden" != yes], [ - AC_DEFINE(MODULE_SCOPE, [__private_extern__], - [Compiler support for module scope symbols]) - ]) - CC_SEARCH_FLAGS="" - LD_SEARCH_FLAGS="" - LD_LIBRARY_PATH_VAR="DYLD_LIBRARY_PATH" - AC_DEFINE(MAC_OSX_TCL, 1, [Is this a Mac I see before me?]) - PLAT_OBJS='${MAC_OSX_OBJS}' - PLAT_SRCS='${MAC_OSX_SRCS}' - AC_MSG_CHECKING([whether to use CoreFoundation]) - AC_ARG_ENABLE(corefoundation, - AC_HELP_STRING([--enable-corefoundation], - [use CoreFoundation API on MacOSX (default: on)]), - [tcl_corefoundation=$enableval], [tcl_corefoundation=yes]) - AC_MSG_RESULT([$tcl_corefoundation]) - AS_IF([test $tcl_corefoundation = yes], [ - AC_CACHE_CHECK([for CoreFoundation.framework], - tcl_cv_lib_corefoundation, [ - hold_libs=$LIBS - AS_IF([test "$fat_32_64" = yes], [ - for v in CFLAGS CPPFLAGS LDFLAGS; do - # On Tiger there is no 64-bit CF, so remove 64-bit - # archs from CFLAGS et al. while testing for - # presence of CF. 64-bit CF is disabled in - # tclUnixPort.h if necessary. - eval 'hold_'$v'="$'$v'";'$v'="`echo "$'$v' "|sed -e "s/-arch ppc64 / /g" -e "s/-arch x86_64 / /g"`"' - done]) - LIBS="$LIBS -framework CoreFoundation" - AC_TRY_LINK([#include ], - [CFBundleRef b = CFBundleGetMainBundle();], - tcl_cv_lib_corefoundation=yes, - tcl_cv_lib_corefoundation=no) - AS_IF([test "$fat_32_64" = yes], [ - for v in CFLAGS CPPFLAGS LDFLAGS; do - eval $v'="$hold_'$v'"' - done]) - LIBS=$hold_libs]) - AS_IF([test $tcl_cv_lib_corefoundation = yes], [ - LIBS="$LIBS -framework CoreFoundation" - AC_DEFINE(HAVE_COREFOUNDATION, 1, - [Do we have access to Darwin CoreFoundation.framework?]) - ], [tcl_corefoundation=no]) - AS_IF([test "$fat_32_64" = yes -a $tcl_corefoundation = yes],[ - AC_CACHE_CHECK([for 64-bit CoreFoundation], - tcl_cv_lib_corefoundation_64, [ - for v in CFLAGS CPPFLAGS LDFLAGS; do - eval 'hold_'$v'="$'$v'";'$v'="`echo "$'$v' "|sed -e "s/-arch ppc / /g" -e "s/-arch i386 / /g"`"' - done - AC_TRY_LINK([#include ], - [CFBundleRef b = CFBundleGetMainBundle();], - tcl_cv_lib_corefoundation_64=yes, - tcl_cv_lib_corefoundation_64=no) - for v in CFLAGS CPPFLAGS LDFLAGS; do - eval $v'="$hold_'$v'"' - done]) - AS_IF([test $tcl_cv_lib_corefoundation_64 = no], [ - AC_DEFINE(NO_COREFOUNDATION_64, 1, - [Is Darwin CoreFoundation unavailable for 64-bit?]) - LDFLAGS="$LDFLAGS -Wl,-no_arch_warnings" - ]) - ]) - ]) - ;; - NEXTSTEP-*) - SHLIB_CFLAGS="" - SHLIB_LD='${CC} -nostdlib -r' - SHLIB_LD_LIBS="" - SHLIB_SUFFIX=".so" - DL_OBJS="tclLoadNext.o" - DL_LIBS="" - CC_SEARCH_FLAGS="" - LD_SEARCH_FLAGS="" - ;; - OS/390-*) - CFLAGS_OPTIMIZE="" # Optimizer is buggy - AC_DEFINE(_OE_SOCKETS, 1, # needed in sys/socket.h - [Should OS/390 do the right thing with sockets?]) - ;; - OSF1-1.0|OSF1-1.1|OSF1-1.2) - # OSF/1 1.[012] from OSF, and derivatives, including Paragon OSF/1 - SHLIB_CFLAGS="" - # Hack: make package name same as library name - SHLIB_LD='ld -R -export $@:' - SHLIB_LD_LIBS="" - SHLIB_SUFFIX=".so" - DL_OBJS="tclLoadOSF.o" - DL_LIBS="" - CC_SEARCH_FLAGS="" - LD_SEARCH_FLAGS="" - ;; - OSF1-1.*) - # OSF/1 1.3 from OSF using ELF, and derivatives, including AD2 - SHLIB_CFLAGS="-fPIC" - AS_IF([test "$SHARED_BUILD" = 1], [SHLIB_LD="ld -shared"], [ - SHLIB_LD="ld -non_shared" - ]) - SHLIB_LD_LIBS="" - SHLIB_SUFFIX=".so" - DL_OBJS="tclLoadDl.o" - DL_LIBS="" - CC_SEARCH_FLAGS="" - LD_SEARCH_FLAGS="" - ;; - OSF1-V*) - # Digital OSF/1 - SHLIB_CFLAGS="" - AS_IF([test "$SHARED_BUILD" = 1], [ - SHLIB_LD='ld -shared -expect_unresolved "*"' - ], [ - SHLIB_LD='ld -non_shared -expect_unresolved "*"' - ]) - SHLIB_LD_LIBS="" - SHLIB_SUFFIX=".so" - DL_OBJS="tclLoadDl.o" - DL_LIBS="" - AS_IF([test $doRpath = yes], [ - CC_SEARCH_FLAGS='-Wl,-rpath,${LIB_RUNTIME_DIR}' - LD_SEARCH_FLAGS='-rpath ${LIB_RUNTIME_DIR}']) - AS_IF([test "$GCC" = yes], [CFLAGS="$CFLAGS -mieee"], [ - CFLAGS="$CFLAGS -DHAVE_TZSET -std1 -ieee"]) - # see pthread_intro(3) for pthread support on osf1, k.furukawa - AS_IF([test "${TCL_THREADS}" = 1], [ - CFLAGS="$CFLAGS -DHAVE_PTHREAD_ATTR_SETSTACKSIZE" - CFLAGS="$CFLAGS -DTCL_THREAD_STACK_MIN=PTHREAD_STACK_MIN*64" - LIBS=`echo $LIBS | sed s/-lpthreads//` - AS_IF([test "$GCC" = yes], [ - LIBS="$LIBS -lpthread -lmach -lexc" - ], [ - CFLAGS="$CFLAGS -pthread" - LDFLAGS="$LDFLAGS -pthread" - ]) - ]) - ;; - QNX-6*) - # QNX RTP - # This may work for all QNX, but it was only reported for v6. - SHLIB_CFLAGS="-fPIC" - SHLIB_LD="ld -Bshareable -x" - SHLIB_LD_LIBS="" - SHLIB_SUFFIX=".so" - DL_OBJS="tclLoadDl.o" - # dlopen is in -lc on QNX - DL_LIBS="" - CC_SEARCH_FLAGS="" - LD_SEARCH_FLAGS="" - ;; - SCO_SV-3.2*) - # Note, dlopen is available only on SCO 3.2.5 and greater. However, - # this test works, since "uname -s" was non-standard in 3.2.4 and - # below. - AS_IF([test "$GCC" = yes], [ - SHLIB_CFLAGS="-fPIC -melf" - LDFLAGS="$LDFLAGS -melf -Wl,-Bexport" - ], [ - SHLIB_CFLAGS="-Kpic -belf" - LDFLAGS="$LDFLAGS -belf -Wl,-Bexport" - ]) - SHLIB_LD="ld -G" - SHLIB_LD_LIBS="" - SHLIB_SUFFIX=".so" - DL_OBJS="tclLoadDl.o" - DL_LIBS="" - CC_SEARCH_FLAGS="" - LD_SEARCH_FLAGS="" - ;; - SINIX*5.4*) - SHLIB_CFLAGS="-K PIC" - SHLIB_LD='${CC} -G' - SHLIB_LD_LIBS="" - SHLIB_SUFFIX=".so" - DL_OBJS="tclLoadDl.o" - DL_LIBS="-ldl" - CC_SEARCH_FLAGS="" - LD_SEARCH_FLAGS="" - ;; - SunOS-4*) - SHLIB_CFLAGS="-PIC" - SHLIB_LD="ld" - SHLIB_LD_LIBS="" - SHLIB_SUFFIX=".so" - DL_OBJS="tclLoadDl.o" - DL_LIBS="-ldl" - CC_SEARCH_FLAGS='-L${LIB_RUNTIME_DIR}' - LD_SEARCH_FLAGS=${CC_SEARCH_FLAGS} - - # SunOS can't handle version numbers with dots in them in library - # specs, like -ltcl7.5, so use -ltcl75 instead. Also, it - # requires an extra version number at the end of .so file names. - # So, the library has to have a name like libtcl75.so.1.0 - - SHARED_LIB_SUFFIX='${TCL_TRIM_DOTS}.so.${SHLIB_VERSION}' - UNSHARED_LIB_SUFFIX='${TCL_TRIM_DOTS}.a' - TCL_LIB_VERSIONS_OK=nodots - ;; - SunOS-5.[[0-6]]) - # Careful to not let 5.10+ fall into this case - - # Note: If _REENTRANT isn't defined, then Solaris - # won't define thread-safe library routines. - - AC_DEFINE(_REENTRANT, 1, [Do we want the reentrant OS API?]) - AC_DEFINE(_POSIX_PTHREAD_SEMANTICS, 1, - [Do we really want to follow the standard? Yes we do!]) - - SHLIB_CFLAGS="-KPIC" - - # Note: need the LIBS below, otherwise Tk won't find Tcl's - # symbols when dynamically loaded into tclsh. - - SHLIB_LD_LIBS='${LIBS}' - SHLIB_SUFFIX=".so" - DL_OBJS="tclLoadDl.o" - DL_LIBS="-ldl" - AS_IF([test "$GCC" = yes], [ - SHLIB_LD='${CC} -shared' - CC_SEARCH_FLAGS='-Wl,-R,${LIB_RUNTIME_DIR}' - LD_SEARCH_FLAGS=${CC_SEARCH_FLAGS} - ], [ - SHLIB_LD="/usr/ccs/bin/ld -G -z text" - CC_SEARCH_FLAGS='-R ${LIB_RUNTIME_DIR}' - LD_SEARCH_FLAGS=${CC_SEARCH_FLAGS} - ]) - ;; - SunOS-5*) - # Note: If _REENTRANT isn't defined, then Solaris - # won't define thread-safe library routines. - - AC_DEFINE(_REENTRANT, 1, [Do we want the reentrant OS API?]) - AC_DEFINE(_POSIX_PTHREAD_SEMANTICS, 1, - [Do we really want to follow the standard? Yes we do!]) - - SHLIB_CFLAGS="-KPIC" - - # Check to enable 64-bit flags for compiler/linker - AS_IF([test "$do64bit" = yes], [ - arch=`isainfo` - AS_IF([test "$arch" = "sparcv9 sparc"], [ - AS_IF([test "$GCC" = yes], [ - AS_IF([test "`${CC} -dumpversion | awk -F. '{print [$]1}'`" -lt 3], [ - AC_MSG_WARN([64bit mode not supported with GCC < 3.2 on $system]) - ], [ - do64bit_ok=yes - CFLAGS="$CFLAGS -m64 -mcpu=v9" - LDFLAGS="$LDFLAGS -m64 -mcpu=v9" - SHLIB_CFLAGS="-fPIC" - ]) - ], [ - do64bit_ok=yes - AS_IF([test "$do64bitVIS" = yes], [ - CFLAGS="$CFLAGS -xarch=v9a" - LDFLAGS_ARCH="-xarch=v9a" - ], [ - CFLAGS="$CFLAGS -xarch=v9" - LDFLAGS_ARCH="-xarch=v9" - ]) - # Solaris 64 uses this as well - #LD_LIBRARY_PATH_VAR="LD_LIBRARY_PATH_64" - ]) - ], [AS_IF([test "$arch" = "amd64 i386"], [ - AS_IF([test "$GCC" = yes], [ - case $system in - SunOS-5.1[[1-9]]*|SunOS-5.[[2-9]][[0-9]]*) - do64bit_ok=yes - CFLAGS="$CFLAGS -m64" - LDFLAGS="$LDFLAGS -m64";; - *) - AC_MSG_WARN([64bit mode not supported with GCC on $system]);; - esac - ], [ - do64bit_ok=yes - case $system in - SunOS-5.1[[1-9]]*|SunOS-5.[[2-9]][[0-9]]*) - CFLAGS="$CFLAGS -m64" - LDFLAGS="$LDFLAGS -m64";; - *) - CFLAGS="$CFLAGS -xarch=amd64" - LDFLAGS="$LDFLAGS -xarch=amd64";; - esac - ]) - ], [AC_MSG_WARN([64bit mode not supported for $arch])])]) - ]) - - #-------------------------------------------------------------------- - # On Solaris 5.x i386 with the sunpro compiler we need to link - # with sunmath to get floating point rounding control - #-------------------------------------------------------------------- - AS_IF([test "$GCC" = yes],[use_sunmath=no],[ - arch=`isainfo` - AC_MSG_CHECKING([whether to use -lsunmath for fp rounding control]) - AS_IF([test "$arch" = "amd64 i386"], [ - AC_MSG_RESULT([yes]) - MATH_LIBS="-lsunmath $MATH_LIBS" - AC_CHECK_HEADER(sunmath.h) - use_sunmath=yes - ], [ - AC_MSG_RESULT([no]) - use_sunmath=no - ]) - ]) - - # Note: need the LIBS below, otherwise Tk won't find Tcl's - # symbols when dynamically loaded into tclsh. - - SHLIB_LD_LIBS='${LIBS}' - SHLIB_SUFFIX=".so" - DL_OBJS="tclLoadDl.o" - DL_LIBS="-ldl" - AS_IF([test "$GCC" = yes], [ - SHLIB_LD='${CC} -shared' - CC_SEARCH_FLAGS='-Wl,-R,${LIB_RUNTIME_DIR}' - LD_SEARCH_FLAGS=${CC_SEARCH_FLAGS} - AS_IF([test "$do64bit_ok" = yes], [ - AS_IF([test "$arch" = "sparcv9 sparc"], [ - # We need to specify -static-libgcc or we need to - # add the path to the sparv9 libgcc. - SHLIB_LD="$SHLIB_LD -m64 -mcpu=v9 -static-libgcc" - # for finding sparcv9 libgcc, get the regular libgcc - # path, remove so name and append 'sparcv9' - #v9gcclibdir="`gcc -print-file-name=libgcc_s.so` | ..." - #CC_SEARCH_FLAGS="${CC_SEARCH_FLAGS},-R,$v9gcclibdir" - ], [AS_IF([test "$arch" = "amd64 i386"], [ - SHLIB_LD="$SHLIB_LD -m64 -static-libgcc" - ])]) - ]) - ], [ - AS_IF([test "$use_sunmath" = yes], [textmode=textoff],[textmode=text]) - case $system in - SunOS-5.[[1-9]][[0-9]]*) - SHLIB_LD="\${CC} -G -z $textmode \${LDFLAGS}";; - *) - SHLIB_LD="/usr/ccs/bin/ld -G -z $textmode";; - esac - CC_SEARCH_FLAGS='-Wl,-R,${LIB_RUNTIME_DIR}' - LD_SEARCH_FLAGS='-R ${LIB_RUNTIME_DIR}' - ]) - ;; - UNIX_SV* | UnixWare-5*) - SHLIB_CFLAGS="-KPIC" - SHLIB_LD='${CC} -G' - SHLIB_LD_LIBS="" - SHLIB_SUFFIX=".so" - DL_OBJS="tclLoadDl.o" - DL_LIBS="-ldl" - # Some UNIX_SV* systems (unixware 1.1.2 for example) have linkers - # that don't grok the -Bexport option. Test that it does. - AC_CACHE_CHECK([for ld accepts -Bexport flag], tcl_cv_ld_Bexport, [ - hold_ldflags=$LDFLAGS - LDFLAGS="$LDFLAGS -Wl,-Bexport" - AC_TRY_LINK(, [int i;], tcl_cv_ld_Bexport=yes, tcl_cv_ld_Bexport=no) - LDFLAGS=$hold_ldflags]) - AS_IF([test $tcl_cv_ld_Bexport = yes], [ - LDFLAGS="$LDFLAGS -Wl,-Bexport" - ]) - CC_SEARCH_FLAGS="" - LD_SEARCH_FLAGS="" - ;; - esac - - AS_IF([test "$do64bit" = yes -a "$do64bit_ok" = no], [ - AC_MSG_WARN([64bit support being disabled -- don't know magic for this platform]) - ]) - - AS_IF([test "$do64bit" = yes -a "$do64bit_ok" = yes], [ - AC_DEFINE(TCL_CFG_DO64BIT, 1, [Is this a 64-bit build?]) - ]) - -dnl # Add any CPPFLAGS set in the environment to our CFLAGS, but delay doing so -dnl # until the end of configure, as configure's compile and link tests use -dnl # both CPPFLAGS and CFLAGS (unlike our compile and link) but configure's -dnl # preprocessing tests use only CPPFLAGS. - AC_CONFIG_COMMANDS_PRE([CFLAGS="${CFLAGS} ${CPPFLAGS}"; CPPFLAGS=""]) - - # Step 4: disable dynamic loading if requested via a command-line switch. - - AC_ARG_ENABLE(load, - AC_HELP_STRING([--enable-load], - [allow dynamic loading and "load" command (default: on)]), - [tcl_ok=$enableval], [tcl_ok=yes]) - AS_IF([test "$tcl_ok" = no], [DL_OBJS=""]) - - AS_IF([test "x$DL_OBJS" != x], [BUILD_DLTEST="\$(DLTEST_TARGETS)"], [ - AC_MSG_WARN([Can't figure out how to do dynamic loading or shared libraries on this system.]) - SHLIB_CFLAGS="" - SHLIB_LD="" - SHLIB_SUFFIX="" - DL_OBJS="tclLoadNone.o" - DL_LIBS="" - LDFLAGS="$LDFLAGS_ORIG" - CC_SEARCH_FLAGS="" - LD_SEARCH_FLAGS="" - BUILD_DLTEST="" - ]) - LDFLAGS="$LDFLAGS $LDFLAGS_ARCH" - - # If we're running gcc, then change the C flags for compiling shared - # libraries to the right flags for gcc, instead of those for the - # standard manufacturer compiler. - - AS_IF([test "$DL_OBJS" != "tclLoadNone.o" -a "$GCC" = yes], [ - case $system in - AIX-*) ;; - BSD/OS*) ;; - IRIX*) ;; - NetBSD-*|FreeBSD-*) ;; - Darwin-*) ;; - SCO_SV-3.2*) ;; - *) SHLIB_CFLAGS="-fPIC" ;; - esac]) - - AS_IF([test "$SHARED_LIB_SUFFIX" = ""], [ - SHARED_LIB_SUFFIX='${VERSION}${SHLIB_SUFFIX}']) - AS_IF([test "$UNSHARED_LIB_SUFFIX" = ""], [ - UNSHARED_LIB_SUFFIX='${VERSION}.a']) - - AS_IF([test "${SHARED_BUILD}" = 1 -a "${SHLIB_SUFFIX}" != ""], [ - LIB_SUFFIX=${SHARED_LIB_SUFFIX} - MAKE_LIB='${SHLIB_LD} -o [$]@ ${OBJS} ${SHLIB_LD_LIBS} ${TCL_SHLIB_LD_EXTRAS} ${TK_SHLIB_LD_EXTRAS} ${LD_SEARCH_FLAGS}' - INSTALL_LIB='$(INSTALL_LIBRARY) $(LIB_FILE) "$(LIB_INSTALL_DIR)"/$(LIB_FILE)' - ], [ - LIB_SUFFIX=${UNSHARED_LIB_SUFFIX} - - AS_IF([test "$RANLIB" = ""], [ - MAKE_LIB='$(STLIB_LD) [$]@ ${OBJS}' - INSTALL_LIB='$(INSTALL_LIBRARY) $(LIB_FILE) "$(LIB_INSTALL_DIR)"/$(LIB_FILE)' - ], [ - MAKE_LIB='${STLIB_LD} [$]@ ${OBJS} ; ${RANLIB} [$]@' - INSTALL_LIB='$(INSTALL_LIBRARY) $(LIB_FILE) "$(LIB_INSTALL_DIR)"/$(LIB_FILE) ; (cd "$(LIB_INSTALL_DIR)" ; $(RANLIB) $(LIB_FILE))' - ]) - ]) - - # Stub lib does not depend on shared/static configuration - AS_IF([test "$RANLIB" = ""], [ - MAKE_STUB_LIB='${STLIB_LD} [$]@ ${STUB_LIB_OBJS}' - INSTALL_STUB_LIB='$(INSTALL_LIBRARY) $(STUB_LIB_FILE) "$(LIB_INSTALL_DIR)/$(STUB_LIB_FILE)"' - ], [ - MAKE_STUB_LIB='${STLIB_LD} [$]@ ${STUB_LIB_OBJS} ; ${RANLIB} [$]@' - INSTALL_STUB_LIB='$(INSTALL_LIBRARY) $(STUB_LIB_FILE) "$(LIB_INSTALL_DIR)"/$(STUB_LIB_FILE) ; (cd "$(LIB_INSTALL_DIR)" ; $(RANLIB) $(STUB_LIB_FILE))' - ]) - - # Define TCL_LIBS now that we know what DL_LIBS is. - # The trick here is that we don't want to change the value of TCL_LIBS if - # it is already set when tclConfig.sh had been loaded by Tk. - AS_IF([test "x${TCL_LIBS}" = x], [ - TCL_LIBS="${DL_LIBS} ${LIBS} ${MATH_LIBS}"]) - AC_SUBST(TCL_LIBS) - - # FIXME: This subst was left in only because the TCL_DL_LIBS - # entry in tclConfig.sh uses it. It is not clear why someone - # would use TCL_DL_LIBS instead of TCL_LIBS. - AC_SUBST(DL_LIBS) - - AC_SUBST(DL_OBJS) - AC_SUBST(PLAT_OBJS) - AC_SUBST(PLAT_SRCS) - AC_SUBST(LDAIX_SRC) - AC_SUBST(CFLAGS) - AC_SUBST(CFLAGS_DEBUG) - AC_SUBST(CFLAGS_OPTIMIZE) - AC_SUBST(CFLAGS_WARNING) - - AC_SUBST(LDFLAGS) - AC_SUBST(LDFLAGS_DEBUG) - AC_SUBST(LDFLAGS_OPTIMIZE) - AC_SUBST(CC_SEARCH_FLAGS) - AC_SUBST(LD_SEARCH_FLAGS) - - AC_SUBST(STLIB_LD) - AC_SUBST(SHLIB_LD) - AC_SUBST(TCL_SHLIB_LD_EXTRAS) - AC_SUBST(TK_SHLIB_LD_EXTRAS) - AC_SUBST(SHLIB_LD_LIBS) - AC_SUBST(SHLIB_CFLAGS) - AC_SUBST(SHLIB_SUFFIX) - AC_DEFINE_UNQUOTED(TCL_SHLIB_EXT,"${SHLIB_SUFFIX}", - [What is the default extension for shared libraries?]) - - AC_SUBST(MAKE_LIB) - AC_SUBST(MAKE_STUB_LIB) - AC_SUBST(INSTALL_LIB) - AC_SUBST(INSTALL_STUB_LIB) - AC_SUBST(RANLIB) -]) - -#-------------------------------------------------------------------- -# SC_SERIAL_PORT -# -# Determine which interface to use to talk to the serial port. -# Note that #include lines must begin in leftmost column for -# some compilers to recognize them as preprocessor directives, -# and some build environments have stdin not pointing at a -# pseudo-terminal (usually /dev/null instead.) -# -# Arguments: -# none -# -# Results: -# -# Defines only one of the following vars: -# HAVE_SYS_MODEM_H -# USE_TERMIOS -# USE_TERMIO -# USE_SGTTY -# -#-------------------------------------------------------------------- - -AC_DEFUN([SC_SERIAL_PORT], [ - AC_CHECK_HEADERS(sys/modem.h) - AC_CACHE_CHECK([termios vs. termio vs. sgtty], tcl_cv_api_serial, [ - AC_TRY_RUN([ -#include - -int main() { - struct termios t; - if (tcgetattr(0, &t) == 0) { - cfsetospeed(&t, 0); - t.c_cflag |= PARENB | PARODD | CSIZE | CSTOPB; - return 0; - } - return 1; -}], tcl_cv_api_serial=termios, tcl_cv_api_serial=no, tcl_cv_api_serial=no) - if test $tcl_cv_api_serial = no ; then - AC_TRY_RUN([ -#include - -int main() { - struct termio t; - if (ioctl(0, TCGETA, &t) == 0) { - t.c_cflag |= CBAUD | PARENB | PARODD | CSIZE | CSTOPB; - return 0; - } - return 1; -}], tcl_cv_api_serial=termio, tcl_cv_api_serial=no, tcl_cv_api_serial=no) - fi - if test $tcl_cv_api_serial = no ; then - AC_TRY_RUN([ -#include - -int main() { - struct sgttyb t; - if (ioctl(0, TIOCGETP, &t) == 0) { - t.sg_ospeed = 0; - t.sg_flags |= ODDP | EVENP | RAW; - return 0; - } - return 1; -}], tcl_cv_api_serial=sgtty, tcl_cv_api_serial=no, tcl_cv_api_serial=no) - fi - if test $tcl_cv_api_serial = no ; then - AC_TRY_RUN([ -#include -#include - -int main() { - struct termios t; - if (tcgetattr(0, &t) == 0 - || errno == ENOTTY || errno == ENXIO || errno == EINVAL) { - cfsetospeed(&t, 0); - t.c_cflag |= PARENB | PARODD | CSIZE | CSTOPB; - return 0; - } - return 1; -}], tcl_cv_api_serial=termios, tcl_cv_api_serial=no, tcl_cv_api_serial=no) - fi - if test $tcl_cv_api_serial = no; then - AC_TRY_RUN([ -#include -#include - -int main() { - struct termio t; - if (ioctl(0, TCGETA, &t) == 0 - || errno == ENOTTY || errno == ENXIO || errno == EINVAL) { - t.c_cflag |= CBAUD | PARENB | PARODD | CSIZE | CSTOPB; - return 0; - } - return 1; - }], tcl_cv_api_serial=termio, tcl_cv_api_serial=no, tcl_cv_api_serial=no) - fi - if test $tcl_cv_api_serial = no; then - AC_TRY_RUN([ -#include -#include - -int main() { - struct sgttyb t; - if (ioctl(0, TIOCGETP, &t) == 0 - || errno == ENOTTY || errno == ENXIO || errno == EINVAL) { - t.sg_ospeed = 0; - t.sg_flags |= ODDP | EVENP | RAW; - return 0; - } - return 1; -}], tcl_cv_api_serial=sgtty, tcl_cv_api_serial=none, tcl_cv_api_serial=none) - fi]) - case $tcl_cv_api_serial in - termios) AC_DEFINE(USE_TERMIOS, 1, [Use the termios API for serial lines]);; - termio) AC_DEFINE(USE_TERMIO, 1, [Use the termio API for serial lines]);; - sgtty) AC_DEFINE(USE_SGTTY, 1, [Use the sgtty API for serial lines]);; - esac -]) - -#-------------------------------------------------------------------- -# SC_MISSING_POSIX_HEADERS -# -# Supply substitutes for missing POSIX header files. Special -# notes: -# - stdlib.h doesn't define strtol, strtoul, or -# strtod insome versions of SunOS -# - some versions of string.h don't declare procedures such -# as strstr -# -# Arguments: -# none -# -# Results: -# -# Defines some of the following vars: -# NO_DIRENT_H -# NO_VALUES_H -# HAVE_LIMITS_H or NO_LIMITS_H -# NO_STDLIB_H -# NO_STRING_H -# NO_SYS_WAIT_H -# NO_DLFCN_H -# HAVE_SYS_PARAM_H -# -# HAVE_STRING_H ? -# -#-------------------------------------------------------------------- - -AC_DEFUN([SC_MISSING_POSIX_HEADERS], [ - AC_CACHE_CHECK([dirent.h], tcl_cv_dirent_h, [ - AC_TRY_LINK([#include -#include ], [ -#ifndef _POSIX_SOURCE -# ifdef __Lynx__ - /* - * Generate compilation error to make the test fail: Lynx headers - * are only valid if really in the POSIX environment. - */ - - missing_procedure(); -# endif -#endif -DIR *d; -struct dirent *entryPtr; -char *p; -d = opendir("foobar"); -entryPtr = readdir(d); -p = entryPtr->d_name; -closedir(d); -], tcl_cv_dirent_h=yes, tcl_cv_dirent_h=no)]) - - if test $tcl_cv_dirent_h = no; then - AC_DEFINE(NO_DIRENT_H, 1, [Do we have ?]) - fi - - AC_CHECK_HEADER(float.h, , [AC_DEFINE(NO_FLOAT_H, 1, [Do we have ?])]) - AC_CHECK_HEADER(values.h, , [AC_DEFINE(NO_VALUES_H, 1, [Do we have ?])]) - AC_CHECK_HEADER(limits.h, - [AC_DEFINE(HAVE_LIMITS_H, 1, [Do we have ?])], - [AC_DEFINE(NO_LIMITS_H, 1, [Do we have ?])]) - AC_CHECK_HEADER(stdlib.h, tcl_ok=1, tcl_ok=0) - AC_EGREP_HEADER(strtol, stdlib.h, , tcl_ok=0) - AC_EGREP_HEADER(strtoul, stdlib.h, , tcl_ok=0) - AC_EGREP_HEADER(strtod, stdlib.h, , tcl_ok=0) - if test $tcl_ok = 0; then - AC_DEFINE(NO_STDLIB_H, 1, [Do we have ?]) - fi - AC_CHECK_HEADER(string.h, tcl_ok=1, tcl_ok=0) - AC_EGREP_HEADER(strstr, string.h, , tcl_ok=0) - AC_EGREP_HEADER(strerror, string.h, , tcl_ok=0) - - # See also memmove check below for a place where NO_STRING_H can be - # set and why. - - if test $tcl_ok = 0; then - AC_DEFINE(NO_STRING_H, 1, [Do we have ?]) - fi - - AC_CHECK_HEADER(sys/wait.h, , [AC_DEFINE(NO_SYS_WAIT_H, 1, [Do we have ?])]) - AC_CHECK_HEADER(dlfcn.h, , [AC_DEFINE(NO_DLFCN_H, 1, [Do we have ?])]) - - # OS/390 lacks sys/param.h (and doesn't need it, by chance). - AC_HAVE_HEADERS(sys/param.h) -]) - -#-------------------------------------------------------------------- -# SC_PATH_X -# -# Locate the X11 header files and the X11 library archive. Try -# the ac_path_x macro first, but if it doesn't find the X stuff -# (e.g. because there's no xmkmf program) then check through -# a list of possible directories. Under some conditions the -# autoconf macro will return an include directory that contains -# no include files, so double-check its result just to be safe. -# -# Arguments: -# none -# -# Results: -# -# Sets the the following vars: -# XINCLUDES -# XLIBSW -# -#-------------------------------------------------------------------- - -AC_DEFUN([SC_PATH_X], [ - AC_PATH_X - not_really_there="" - if test "$no_x" = ""; then - if test "$x_includes" = ""; then - AC_TRY_CPP([#include ], , not_really_there="yes") - else - if test ! -r $x_includes/X11/Intrinsic.h; then - not_really_there="yes" - fi - fi - fi - if test "$no_x" = "yes" -o "$not_really_there" = "yes"; then - AC_MSG_CHECKING([for X11 header files]) - found_xincludes="no" - AC_TRY_CPP([#include ], found_xincludes="yes", found_xincludes="no") - if test "$found_xincludes" = "no"; then - dirs="/usr/unsupported/include /usr/local/include /usr/X386/include /usr/X11R6/include /usr/X11R5/include /usr/include/X11R5 /usr/include/X11R4 /usr/openwin/include /usr/X11/include /usr/sww/include" - for i in $dirs ; do - if test -r $i/X11/Intrinsic.h; then - AC_MSG_RESULT([$i]) - XINCLUDES=" -I$i" - found_xincludes="yes" - break - fi - done - fi - else - if test "$x_includes" != ""; then - XINCLUDES="-I$x_includes" - found_xincludes="yes" - fi - fi - if test found_xincludes = "no"; then - AC_MSG_RESULT([couldn't find any!]) - fi - - if test "$no_x" = yes; then - AC_MSG_CHECKING([for X11 libraries]) - XLIBSW=nope - dirs="/usr/unsupported/lib /usr/local/lib /usr/X386/lib /usr/X11R6/lib /usr/X11R5/lib /usr/lib/X11R5 /usr/lib/X11R4 /usr/openwin/lib /usr/X11/lib /usr/sww/X11/lib" - for i in $dirs ; do - if test -r $i/libX11.a -o -r $i/libX11.so -o -r $i/libX11.sl -o -r $i/libX11.dylib; then - AC_MSG_RESULT([$i]) - XLIBSW="-L$i -lX11" - x_libraries="$i" - break - fi - done - else - if test "$x_libraries" = ""; then - XLIBSW=-lX11 - else - XLIBSW="-L$x_libraries -lX11" - fi - fi - if test "$XLIBSW" = nope ; then - AC_CHECK_LIB(Xwindow, XCreateWindow, XLIBSW=-lXwindow) - fi - if test "$XLIBSW" = nope ; then - AC_MSG_RESULT([could not find any! Using -lX11.]) - XLIBSW=-lX11 - fi -]) - -#-------------------------------------------------------------------- -# SC_BLOCKING_STYLE -# -# The statements below check for systems where POSIX-style -# non-blocking I/O (O_NONBLOCK) doesn't work or is unimplemented. -# On these systems (mostly older ones), use the old BSD-style -# FIONBIO approach instead. -# -# Arguments: -# none -# -# Results: -# -# Defines some of the following vars: -# HAVE_SYS_IOCTL_H -# HAVE_SYS_FILIO_H -# USE_FIONBIO -# O_NONBLOCK -# -#-------------------------------------------------------------------- - -AC_DEFUN([SC_BLOCKING_STYLE], [ - AC_CHECK_HEADERS(sys/ioctl.h) - AC_CHECK_HEADERS(sys/filio.h) - SC_CONFIG_SYSTEM - AC_MSG_CHECKING([FIONBIO vs. O_NONBLOCK for nonblocking I/O]) - case $system in - # There used to be code here to use FIONBIO under AIX. However, it - # was reported that FIONBIO doesn't work under AIX 3.2.5. Since - # using O_NONBLOCK seems fine under AIX 4.*, I removed the FIONBIO - # code (JO, 5/31/97). - - OSF*) - AC_DEFINE(USE_FIONBIO, 1, [Should we use FIONBIO?]) - AC_MSG_RESULT([FIONBIO]) - ;; - SunOS-4*) - AC_DEFINE(USE_FIONBIO, 1, [Should we use FIONBIO?]) - AC_MSG_RESULT([FIONBIO]) - ;; - *) - AC_MSG_RESULT([O_NONBLOCK]) - ;; - esac -]) - -#-------------------------------------------------------------------- -# SC_TIME_HANLDER -# -# Checks how the system deals with time.h, what time structures -# are used on the system, and what fields the structures have. -# -# Arguments: -# none -# -# Results: -# -# Defines some of the following vars: -# USE_DELTA_FOR_TZ -# HAVE_TM_GMTOFF -# HAVE_TM_TZADJ -# HAVE_TIMEZONE_VAR -# -#-------------------------------------------------------------------- - -AC_DEFUN([SC_TIME_HANDLER], [ - AC_CHECK_HEADERS(sys/time.h) - AC_HEADER_TIME - AC_STRUCT_TIMEZONE - - AC_CHECK_FUNCS(gmtime_r localtime_r mktime) - - AC_CACHE_CHECK([tm_tzadj in struct tm], tcl_cv_member_tm_tzadj, [ - AC_TRY_COMPILE([#include ], [struct tm tm; tm.tm_tzadj;], - tcl_cv_member_tm_tzadj=yes, tcl_cv_member_tm_tzadj=no)]) - if test $tcl_cv_member_tm_tzadj = yes ; then - AC_DEFINE(HAVE_TM_TZADJ, 1, [Should we use the tm_tzadj field of struct tm?]) - fi - - AC_CACHE_CHECK([tm_gmtoff in struct tm], tcl_cv_member_tm_gmtoff, [ - AC_TRY_COMPILE([#include ], [struct tm tm; tm.tm_gmtoff;], - tcl_cv_member_tm_gmtoff=yes, tcl_cv_member_tm_gmtoff=no)]) - if test $tcl_cv_member_tm_gmtoff = yes ; then - AC_DEFINE(HAVE_TM_GMTOFF, 1, [Should we use the tm_gmtoff field of struct tm?]) - fi - - # - # Its important to include time.h in this check, as some systems - # (like convex) have timezone functions, etc. - # - AC_CACHE_CHECK([long timezone variable], tcl_cv_timezone_long, [ - AC_TRY_COMPILE([#include ], - [extern long timezone; - timezone += 1; - exit (0);], - tcl_cv_timezone_long=yes, tcl_cv_timezone_long=no)]) - if test $tcl_cv_timezone_long = yes ; then - AC_DEFINE(HAVE_TIMEZONE_VAR, 1, [Should we use the global timezone variable?]) - else - # - # On some systems (eg IRIX 6.2), timezone is a time_t and not a long. - # - AC_CACHE_CHECK([time_t timezone variable], tcl_cv_timezone_time, [ - AC_TRY_COMPILE([#include ], - [extern time_t timezone; - timezone += 1; - exit (0);], - tcl_cv_timezone_time=yes, tcl_cv_timezone_time=no)]) - if test $tcl_cv_timezone_time = yes ; then - AC_DEFINE(HAVE_TIMEZONE_VAR, 1, [Should we use the global timezone variable?]) - fi - fi -]) - -#-------------------------------------------------------------------- -# SC_BUGGY_STRTOD -# -# Under Solaris 2.4, strtod returns the wrong value for the -# terminating character under some conditions. Check for this -# and if the problem exists use a substitute procedure -# "fixstrtod" (provided by Tcl) that corrects the error. -# Also, on Compaq's Tru64 Unix 5.0, -# strtod(" ") returns 0.0 instead of a failure to convert. -# -# Arguments: -# none -# -# Results: -# -# Might defines some of the following vars: -# strtod (=fixstrtod) -# -#-------------------------------------------------------------------- - -AC_DEFUN([SC_BUGGY_STRTOD], [ - AC_CHECK_FUNC(strtod, tcl_strtod=1, tcl_strtod=0) - if test "$tcl_strtod" = 1; then - AC_CACHE_CHECK([for Solaris2.4/Tru64 strtod bugs], tcl_cv_strtod_buggy,[ - AC_TRY_RUN([ - extern double strtod(); - int main() { - char *infString="Inf", *nanString="NaN", *spaceString=" "; - char *term; - double value; - value = strtod(infString, &term); - if ((term != infString) && (term[-1] == 0)) { - exit(1); - } - value = strtod(nanString, &term); - if ((term != nanString) && (term[-1] == 0)) { - exit(1); - } - value = strtod(spaceString, &term); - if (term == (spaceString+1)) { - exit(1); - } - exit(0); - }], tcl_cv_strtod_buggy=ok, tcl_cv_strtod_buggy=buggy, - tcl_cv_strtod_buggy=buggy)]) - if test "$tcl_cv_strtod_buggy" = buggy; then - AC_LIBOBJ([fixstrtod]) - USE_COMPAT=1 - AC_DEFINE(strtod, fixstrtod, [Do we want to use the strtod() in compat?]) - fi - fi -]) - -#-------------------------------------------------------------------- -# SC_TCL_LINK_LIBS -# -# Search for the libraries needed to link the Tcl shell. -# Things like the math library (-lm) and socket stuff (-lsocket vs. -# -lnsl) are dealt with here. -# -# Arguments: -# None. -# -# Results: -# -# Might append to the following vars: -# LIBS -# MATH_LIBS -# -# Might define the following vars: -# HAVE_NET_ERRNO_H -# -#-------------------------------------------------------------------- - -AC_DEFUN([SC_TCL_LINK_LIBS], [ - #-------------------------------------------------------------------- - # On a few very rare systems, all of the libm.a stuff is - # already in libc.a. Set compiler flags accordingly. - # Also, Linux requires the "ieee" library for math to work - # right (and it must appear before "-lm"). - #-------------------------------------------------------------------- - - AC_CHECK_FUNC(sin, MATH_LIBS="", MATH_LIBS="-lm") - AC_CHECK_LIB(ieee, main, [MATH_LIBS="-lieee $MATH_LIBS"]) - - #-------------------------------------------------------------------- - # Interactive UNIX requires -linet instead of -lsocket, plus it - # needs net/errno.h to define the socket-related error codes. - #-------------------------------------------------------------------- - - AC_CHECK_LIB(inet, main, [LIBS="$LIBS -linet"]) - AC_CHECK_HEADER(net/errno.h, [ - AC_DEFINE(HAVE_NET_ERRNO_H, 1, [Do we have ?])]) - - #-------------------------------------------------------------------- - # Check for the existence of the -lsocket and -lnsl libraries. - # The order here is important, so that they end up in the right - # order in the command line generated by make. Here are some - # special considerations: - # 1. Use "connect" and "accept" to check for -lsocket, and - # "gethostbyname" to check for -lnsl. - # 2. Use each function name only once: can't redo a check because - # autoconf caches the results of the last check and won't redo it. - # 3. Use -lnsl and -lsocket only if they supply procedures that - # aren't already present in the normal libraries. This is because - # IRIX 5.2 has libraries, but they aren't needed and they're - # bogus: they goof up name resolution if used. - # 4. On some SVR4 systems, can't use -lsocket without -lnsl too. - # To get around this problem, check for both libraries together - # if -lsocket doesn't work by itself. - #-------------------------------------------------------------------- - - tcl_checkBoth=0 - AC_CHECK_FUNC(connect, tcl_checkSocket=0, tcl_checkSocket=1) - if test "$tcl_checkSocket" = 1; then - AC_CHECK_FUNC(setsockopt, , [AC_CHECK_LIB(socket, setsockopt, - LIBS="$LIBS -lsocket", tcl_checkBoth=1)]) - fi - if test "$tcl_checkBoth" = 1; then - tk_oldLibs=$LIBS - LIBS="$LIBS -lsocket -lnsl" - AC_CHECK_FUNC(accept, tcl_checkNsl=0, [LIBS=$tk_oldLibs]) - fi - AC_CHECK_FUNC(gethostbyname, , [AC_CHECK_LIB(nsl, gethostbyname, - [LIBS="$LIBS -lnsl"])]) -]) - -#-------------------------------------------------------------------- -# SC_TCL_EARLY_FLAGS -# -# Check for what flags are needed to be passed so the correct OS -# features are available. -# -# Arguments: -# None -# -# Results: -# -# Might define the following vars: -# _ISOC99_SOURCE -# _LARGEFILE64_SOURCE -# _LARGEFILE_SOURCE64 -# -#-------------------------------------------------------------------- - -AC_DEFUN([SC_TCL_EARLY_FLAG],[ - AC_CACHE_VAL([tcl_cv_flag_]translit($1,[A-Z],[a-z]), - AC_TRY_COMPILE([$2], $3, [tcl_cv_flag_]translit($1,[A-Z],[a-z])=no, - AC_TRY_COMPILE([[#define ]$1[ 1 -]$2], $3, - [tcl_cv_flag_]translit($1,[A-Z],[a-z])=yes, - [tcl_cv_flag_]translit($1,[A-Z],[a-z])=no))) - if test ["x${tcl_cv_flag_]translit($1,[A-Z],[a-z])[}" = "xyes"] ; then - AC_DEFINE($1, 1, [Add the ]$1[ flag when building]) - tcl_flags="$tcl_flags $1" - fi -]) - -AC_DEFUN([SC_TCL_EARLY_FLAGS],[ - AC_MSG_CHECKING([for required early compiler flags]) - tcl_flags="" - SC_TCL_EARLY_FLAG(_ISOC99_SOURCE,[#include ], - [char *p = (char *)strtoll; char *q = (char *)strtoull;]) - SC_TCL_EARLY_FLAG(_LARGEFILE64_SOURCE,[#include ], - [struct stat64 buf; int i = stat64("/", &buf);]) - SC_TCL_EARLY_FLAG(_LARGEFILE_SOURCE64,[#include ], - [char *p = (char *)open64;]) - if test "x${tcl_flags}" = "x" ; then - AC_MSG_RESULT([none]) - else - AC_MSG_RESULT([${tcl_flags}]) - fi -]) - -#-------------------------------------------------------------------- -# SC_TCL_64BIT_FLAGS -# -# Check for what is defined in the way of 64-bit features. -# -# Arguments: -# None -# -# Results: -# -# Might define the following vars: -# TCL_WIDE_INT_IS_LONG -# TCL_WIDE_INT_TYPE -# HAVE_STRUCT_DIRENT64 -# HAVE_STRUCT_STAT64 -# HAVE_TYPE_OFF64_T -# -#-------------------------------------------------------------------- - -AC_DEFUN([SC_TCL_64BIT_FLAGS], [ - AC_MSG_CHECKING([for 64-bit integer type]) - AC_CACHE_VAL(tcl_cv_type_64bit,[ - tcl_cv_type_64bit=none - # See if the compiler knows natively about __int64 - AC_TRY_COMPILE(,[__int64 value = (__int64) 0;], - tcl_type_64bit=__int64, tcl_type_64bit="long long") - # See if we should use long anyway Note that we substitute in the - # type that is our current guess for a 64-bit type inside this check - # program, so it should be modified only carefully... - AC_TRY_COMPILE(,[switch (0) { - case 1: case (sizeof(]${tcl_type_64bit}[)==sizeof(long)): ; - }],tcl_cv_type_64bit=${tcl_type_64bit})]) - if test "${tcl_cv_type_64bit}" = none ; then - AC_DEFINE(TCL_WIDE_INT_IS_LONG, 1, [Are wide integers to be implemented with C 'long's?]) - AC_MSG_RESULT([using long]) - else - AC_DEFINE_UNQUOTED(TCL_WIDE_INT_TYPE,${tcl_cv_type_64bit}, - [What type should be used to define wide integers?]) - AC_MSG_RESULT([${tcl_cv_type_64bit}]) - - # Now check for auxiliary declarations - AC_CACHE_CHECK([for struct dirent64], tcl_cv_struct_dirent64,[ - AC_TRY_COMPILE([#include -#include ],[struct dirent64 p;], - tcl_cv_struct_dirent64=yes,tcl_cv_struct_dirent64=no)]) - if test "x${tcl_cv_struct_dirent64}" = "xyes" ; then - AC_DEFINE(HAVE_STRUCT_DIRENT64, 1, [Is 'struct dirent64' in ?]) - fi - - AC_CACHE_CHECK([for struct stat64], tcl_cv_struct_stat64,[ - AC_TRY_COMPILE([#include ],[struct stat64 p; -], - tcl_cv_struct_stat64=yes,tcl_cv_struct_stat64=no)]) - if test "x${tcl_cv_struct_stat64}" = "xyes" ; then - AC_DEFINE(HAVE_STRUCT_STAT64, 1, [Is 'struct stat64' in ?]) - fi - - AC_CHECK_FUNCS(open64 lseek64) - AC_MSG_CHECKING([for off64_t]) - AC_CACHE_VAL(tcl_cv_type_off64_t,[ - AC_TRY_COMPILE([#include ],[off64_t offset; -], - tcl_cv_type_off64_t=yes,tcl_cv_type_off64_t=no)]) - dnl Define HAVE_TYPE_OFF64_T only when the off64_t type and the - dnl functions lseek64 and open64 are defined. - if test "x${tcl_cv_type_off64_t}" = "xyes" && \ - test "x${ac_cv_func_lseek64}" = "xyes" && \ - test "x${ac_cv_func_open64}" = "xyes" ; then - AC_DEFINE(HAVE_TYPE_OFF64_T, 1, [Is off64_t in ?]) - AC_MSG_RESULT([yes]) - else - AC_MSG_RESULT([no]) - fi - fi -]) - -#-------------------------------------------------------------------- -# SC_TCL_CFG_ENCODING TIP #59 -# -# Declare the encoding to use for embedded configuration information. -# -# Arguments: -# None. -# -# Results: -# Might append to the following vars: -# DEFS (implicit) -# -# Will define the following vars: -# TCL_CFGVAL_ENCODING -# -#-------------------------------------------------------------------- - -AC_DEFUN([SC_TCL_CFG_ENCODING], [ - AC_ARG_WITH(encoding, - AC_HELP_STRING([--with-encoding], - [encoding for configuration values (default: iso8859-1)]), - with_tcencoding=${withval}) - - if test x"${with_tcencoding}" != x ; then - AC_DEFINE_UNQUOTED(TCL_CFGVAL_ENCODING,"${with_tcencoding}", - [What encoding should be used for embedded configuration info?]) - else - AC_DEFINE(TCL_CFGVAL_ENCODING,"iso8859-1", - [What encoding should be used for embedded configuration info?]) - fi -]) - -#-------------------------------------------------------------------- -# SC_TCL_CHECK_BROKEN_FUNC -# -# Check for broken function. -# -# Arguments: -# funcName - function to test for -# advancedTest - the advanced test to run if the function is present -# -# Results: -# Might cause compatability versions of the function to be used. -# Might affect the following vars: -# USE_COMPAT (implicit) -# -#-------------------------------------------------------------------- - -AC_DEFUN([SC_TCL_CHECK_BROKEN_FUNC],[ - AC_CHECK_FUNC($1, tcl_ok=1, tcl_ok=0) - if test ["$tcl_ok"] = 1; then - AC_CACHE_CHECK([proper ]$1[ implementation], [tcl_cv_]$1[_unbroken], - AC_TRY_RUN([[int main() {]$2[}]],[tcl_cv_]$1[_unbroken]=ok, - [tcl_cv_]$1[_unbroken]=broken,[tcl_cv_]$1[_unbroken]=unknown)) - if test ["$tcl_cv_]$1[_unbroken"] = "ok"; then - tcl_ok=1 - else - tcl_ok=0 - fi - fi - if test ["$tcl_ok"] = 0; then - AC_LIBOBJ($1) - USE_COMPAT=1 - fi -]) - -#-------------------------------------------------------------------- -# SC_TCL_GETHOSTBYADDR_R -# -# Check if we have MT-safe variant of gethostbyaddr(). -# -# Arguments: -# None -# -# Results: -# -# Might define the following vars: -# HAVE_GETHOSTBYADDR_R -# HAVE_GETHOSTBYADDR_R_7 -# HAVE_GETHOSTBYADDR_R_8 -# -#-------------------------------------------------------------------- - -AC_DEFUN([SC_TCL_GETHOSTBYADDR_R], [AC_CHECK_FUNC(gethostbyaddr_r, [ - AC_CACHE_CHECK([for gethostbyaddr_r with 7 args], tcl_cv_api_gethostbyaddr_r_7, [ - AC_TRY_COMPILE([ - #include - ], [ - char *addr; - int length; - int type; - struct hostent *result; - char buffer[2048]; - int buflen = 2048; - int h_errnop; - - (void) gethostbyaddr_r(addr, length, type, result, buffer, buflen, - &h_errnop); - ], tcl_cv_api_gethostbyaddr_r_7=yes, tcl_cv_api_gethostbyaddr_r_7=no)]) - tcl_ok=$tcl_cv_api_gethostbyaddr_r_7 - if test "$tcl_ok" = yes; then - AC_DEFINE(HAVE_GETHOSTBYADDR_R_7, 1, - [Define to 1 if gethostbyaddr_r takes 7 args.]) - else - AC_CACHE_CHECK([for gethostbyaddr_r with 8 args], tcl_cv_api_gethostbyaddr_r_8, [ - AC_TRY_COMPILE([ - #include - ], [ - char *addr; - int length; - int type; - struct hostent *result, *resultp; - char buffer[2048]; - int buflen = 2048; - int h_errnop; - - (void) gethostbyaddr_r(addr, length, type, result, buffer, buflen, - &resultp, &h_errnop); - ], tcl_cv_api_gethostbyaddr_r_8=yes, tcl_cv_api_gethostbyaddr_r_8=no)]) - tcl_ok=$tcl_cv_api_gethostbyaddr_r_8 - if test "$tcl_ok" = yes; then - AC_DEFINE(HAVE_GETHOSTBYADDR_R_8, 1, - [Define to 1 if gethostbyaddr_r takes 8 args.]) - fi - fi - if test "$tcl_ok" = yes; then - AC_DEFINE(HAVE_GETHOSTBYADDR_R, 1, - [Define to 1 if gethostbyaddr_r is available.]) - fi -])]) - -#-------------------------------------------------------------------- -# SC_TCL_GETHOSTBYNAME_R -# -# Check to see what variant of gethostbyname_r() we have. -# Based on David Arnold's example from the comp.programming.threads -# FAQ Q213 -# -# Arguments: -# None -# -# Results: -# -# Might define the following vars: -# HAVE_GETHOSTBYADDR_R -# HAVE_GETHOSTBYADDR_R_3 -# HAVE_GETHOSTBYADDR_R_5 -# HAVE_GETHOSTBYADDR_R_6 -# -#-------------------------------------------------------------------- - -AC_DEFUN([SC_TCL_GETHOSTBYNAME_R], [AC_CHECK_FUNC(gethostbyname_r, [ - AC_CACHE_CHECK([for gethostbyname_r with 6 args], tcl_cv_api_gethostbyname_r_6, [ - AC_TRY_COMPILE([ - #include - ], [ - char *name; - struct hostent *he, *res; - char buffer[2048]; - int buflen = 2048; - int h_errnop; - - (void) gethostbyname_r(name, he, buffer, buflen, &res, &h_errnop); - ], tcl_cv_api_gethostbyname_r_6=yes, tcl_cv_api_gethostbyname_r_6=no)]) - tcl_ok=$tcl_cv_api_gethostbyname_r_6 - if test "$tcl_ok" = yes; then - AC_DEFINE(HAVE_GETHOSTBYNAME_R_6, 1, - [Define to 1 if gethostbyname_r takes 6 args.]) - else - AC_CACHE_CHECK([for gethostbyname_r with 5 args], tcl_cv_api_gethostbyname_r_5, [ - AC_TRY_COMPILE([ - #include - ], [ - char *name; - struct hostent *he; - char buffer[2048]; - int buflen = 2048; - int h_errnop; - - (void) gethostbyname_r(name, he, buffer, buflen, &h_errnop); - ], tcl_cv_api_gethostbyname_r_5=yes, tcl_cv_api_gethostbyname_r_5=no)]) - tcl_ok=$tcl_cv_api_gethostbyname_r_5 - if test "$tcl_ok" = yes; then - AC_DEFINE(HAVE_GETHOSTBYNAME_R_5, 1, - [Define to 1 if gethostbyname_r takes 5 args.]) - else - AC_CACHE_CHECK([for gethostbyname_r with 3 args], tcl_cv_api_gethostbyname_r_3, [ - AC_TRY_COMPILE([ - #include - ], [ - char *name; - struct hostent *he; - struct hostent_data data; - - (void) gethostbyname_r(name, he, &data); - ], tcl_cv_api_gethostbyname_r_3=yes, tcl_cv_api_gethostbyname_r_3=no)]) - tcl_ok=$tcl_cv_api_gethostbyname_r_3 - if test "$tcl_ok" = yes; then - AC_DEFINE(HAVE_GETHOSTBYNAME_R_3, 1, - [Define to 1 if gethostbyname_r takes 3 args.]) - fi - fi - fi - if test "$tcl_ok" = yes; then - AC_DEFINE(HAVE_GETHOSTBYNAME_R, 1, - [Define to 1 if gethostbyname_r is available.]) - fi -])]) - -#-------------------------------------------------------------------- -# SC_TCL_GETADDRINFO -# -# Check if we have 'getaddrinfo' -# -# Arguments: -# None -# -# Results: -# Might define the following vars: -# HAVE_GETADDRINFO -# -#-------------------------------------------------------------------- - -AC_DEFUN([SC_TCL_GETADDRINFO], [AC_CHECK_FUNC(getaddrinfo, [ - AC_CACHE_CHECK([for working getaddrinfo], tcl_cv_api_getaddrinfo, [ - AC_TRY_COMPILE([ - #include - ], [ - const char *name, *port; - struct addrinfo *aiPtr, hints; - (void)getaddrinfo(name,port, &hints, &aiPtr); - (void)freeaddrinfo(aiPtr); - ], tcl_cv_api_getaddrinfo=yes, tcl_cv_getaddrinfo=no)]) - tcl_ok=$tcl_cv_api_getaddrinfo - if test "$tcl_ok" = yes; then - AC_DEFINE(HAVE_GETADDRINFO, 1, - [Define to 1 if getaddrinfo is available.]) - fi -])]) - -#-------------------------------------------------------------------- -# SC_TCL_GETPWUID_R -# -# Check if we have MT-safe variant of getpwuid() and if yes, -# which one exactly. -# -# Arguments: -# None -# -# Results: -# -# Might define the following vars: -# HAVE_GETPWUID_R -# HAVE_GETPWUID_R_4 -# HAVE_GETPWUID_R_5 -# -#-------------------------------------------------------------------- - -AC_DEFUN([SC_TCL_GETPWUID_R], [AC_CHECK_FUNC(getpwuid_r, [ - AC_CACHE_CHECK([for getpwuid_r with 5 args], tcl_cv_api_getpwuid_r_5, [ - AC_TRY_COMPILE([ - #include - #include - ], [ - uid_t uid; - struct passwd pw, *pwp; - char buf[512]; - int buflen = 512; - - (void) getpwuid_r(uid, &pw, buf, buflen, &pwp); - ], tcl_cv_api_getpwuid_r_5=yes, tcl_cv_api_getpwuid_r_5=no)]) - tcl_ok=$tcl_cv_api_getpwuid_r_5 - if test "$tcl_ok" = yes; then - AC_DEFINE(HAVE_GETPWUID_R_5, 1, - [Define to 1 if getpwuid_r takes 5 args.]) - else - AC_CACHE_CHECK([for getpwuid_r with 4 args], tcl_cv_api_getpwuid_r_4, [ - AC_TRY_COMPILE([ - #include - #include - ], [ - uid_t uid; - struct passwd pw; - char buf[512]; - int buflen = 512; - - (void)getpwnam_r(uid, &pw, buf, buflen); - ], tcl_cv_api_getpwuid_r_4=yes, tcl_cv_api_getpwuid_r_4=no)]) - tcl_ok=$tcl_cv_api_getpwuid_r_4 - if test "$tcl_ok" = yes; then - AC_DEFINE(HAVE_GETPWUID_R_4, 1, - [Define to 1 if getpwuid_r takes 4 args.]) - fi - fi - if test "$tcl_ok" = yes; then - AC_DEFINE(HAVE_GETPWUID_R, 1, - [Define to 1 if getpwuid_r is available.]) - fi -])]) - -#-------------------------------------------------------------------- -# SC_TCL_GETPWNAM_R -# -# Check if we have MT-safe variant of getpwnam() and if yes, -# which one exactly. -# -# Arguments: -# None -# -# Results: -# -# Might define the following vars: -# HAVE_GETPWNAM_R -# HAVE_GETPWNAM_R_4 -# HAVE_GETPWNAM_R_5 -# -#-------------------------------------------------------------------- - -AC_DEFUN([SC_TCL_GETPWNAM_R], [AC_CHECK_FUNC(getpwnam_r, [ - AC_CACHE_CHECK([for getpwnam_r with 5 args], tcl_cv_api_getpwnam_r_5, [ - AC_TRY_COMPILE([ - #include - #include - ], [ - char *name; - struct passwd pw, *pwp; - char buf[512]; - int buflen = 512; - - (void) getpwnam_r(name, &pw, buf, buflen, &pwp); - ], tcl_cv_api_getpwnam_r_5=yes, tcl_cv_api_getpwnam_r_5=no)]) - tcl_ok=$tcl_cv_api_getpwnam_r_5 - if test "$tcl_ok" = yes; then - AC_DEFINE(HAVE_GETPWNAM_R_5, 1, - [Define to 1 if getpwnam_r takes 5 args.]) - else - AC_CACHE_CHECK([for getpwnam_r with 4 args], tcl_cv_api_getpwnam_r_4, [ - AC_TRY_COMPILE([ - #include - #include - ], [ - char *name; - struct passwd pw; - char buf[512]; - int buflen = 512; - - (void)getpwnam_r(name, &pw, buf, buflen); - ], tcl_cv_api_getpwnam_r_4=yes, tcl_cv_api_getpwnam_r_4=no)]) - tcl_ok=$tcl_cv_api_getpwnam_r_4 - if test "$tcl_ok" = yes; then - AC_DEFINE(HAVE_GETPWNAM_R_4, 1, - [Define to 1 if getpwnam_r takes 4 args.]) - fi - fi - if test "$tcl_ok" = yes; then - AC_DEFINE(HAVE_GETPWNAM_R, 1, - [Define to 1 if getpwnam_r is available.]) - fi -])]) - -#-------------------------------------------------------------------- -# SC_TCL_GETGRGID_R -# -# Check if we have MT-safe variant of getgrgid() and if yes, -# which one exactly. -# -# Arguments: -# None -# -# Results: -# -# Might define the following vars: -# HAVE_GETGRGID_R -# HAVE_GETGRGID_R_4 -# HAVE_GETGRGID_R_5 -# -#-------------------------------------------------------------------- - -AC_DEFUN([SC_TCL_GETGRGID_R], [AC_CHECK_FUNC(getgrgid_r, [ - AC_CACHE_CHECK([for getgrgid_r with 5 args], tcl_cv_api_getgrgid_r_5, [ - AC_TRY_COMPILE([ - #include - #include - ], [ - gid_t gid; - struct group gr, *grp; - char buf[512]; - int buflen = 512; - - (void) getgrgid_r(gid, &gr, buf, buflen, &grp); - ], tcl_cv_api_getgrgid_r_5=yes, tcl_cv_api_getgrgid_r_5=no)]) - tcl_ok=$tcl_cv_api_getgrgid_r_5 - if test "$tcl_ok" = yes; then - AC_DEFINE(HAVE_GETGRGID_R_5, 1, - [Define to 1 if getgrgid_r takes 5 args.]) - else - AC_CACHE_CHECK([for getgrgid_r with 4 args], tcl_cv_api_getgrgid_r_4, [ - AC_TRY_COMPILE([ - #include - #include - ], [ - gid_t gid; - struct group gr; - char buf[512]; - int buflen = 512; - - (void)getgrgid_r(gid, &gr, buf, buflen); - ], tcl_cv_api_getgrgid_r_4=yes, tcl_cv_api_getgrgid_r_4=no)]) - tcl_ok=$tcl_cv_api_getgrgid_r_4 - if test "$tcl_ok" = yes; then - AC_DEFINE(HAVE_GETGRGID_R_4, 1, - [Define to 1 if getgrgid_r takes 4 args.]) - fi - fi - if test "$tcl_ok" = yes; then - AC_DEFINE(HAVE_GETGRGID_R, 1, - [Define to 1 if getgrgid_r is available.]) - fi -])]) - -#-------------------------------------------------------------------- -# SC_TCL_GETGRNAM_R -# -# Check if we have MT-safe variant of getgrnam() and if yes, -# which one exactly. -# -# Arguments: -# None -# -# Results: -# -# Might define the following vars: -# HAVE_GETGRNAM_R -# HAVE_GETGRNAM_R_4 -# HAVE_GETGRNAM_R_5 -# -#-------------------------------------------------------------------- - -AC_DEFUN([SC_TCL_GETGRNAM_R], [AC_CHECK_FUNC(getgrnam_r, [ - AC_CACHE_CHECK([for getgrnam_r with 5 args], tcl_cv_api_getgrnam_r_5, [ - AC_TRY_COMPILE([ - #include - #include - ], [ - char *name; - struct group gr, *grp; - char buf[512]; - int buflen = 512; - - (void) getgrnam_r(name, &gr, buf, buflen, &grp); - ], tcl_cv_api_getgrnam_r_5=yes, tcl_cv_api_getgrnam_r_5=no)]) - tcl_ok=$tcl_cv_api_getgrnam_r_5 - if test "$tcl_ok" = yes; then - AC_DEFINE(HAVE_GETGRNAM_R_5, 1, - [Define to 1 if getgrnam_r takes 5 args.]) - else - AC_CACHE_CHECK([for getgrnam_r with 4 args], tcl_cv_api_getgrnam_r_4, [ - AC_TRY_COMPILE([ - #include - #include - ], [ - char *name; - struct group gr; - char buf[512]; - int buflen = 512; - - (void)getgrnam_r(name, &gr, buf, buflen); - ], tcl_cv_api_getgrnam_r_4=yes, tcl_cv_api_getgrnam_r_4=no)]) - tcl_ok=$tcl_cv_api_getgrnam_r_4 - if test "$tcl_ok" = yes; then - AC_DEFINE(HAVE_GETGRNAM_R_4, 1, - [Define to 1 if getgrnam_r takes 4 args.]) - fi - fi - if test "$tcl_ok" = yes; then - AC_DEFINE(HAVE_GETGRNAM_R, 1, - [Define to 1 if getgrnam_r is available.]) - fi -])]) - -# Local Variables: -# mode: autoconf -# End: diff --git a/build/tidy.mk b/build/tidy.mk index 16f16817724..26d82c14e02 100644 --- a/build/tidy.mk +++ b/build/tidy.mk @@ -14,7 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -Clang_Tidy_Options = -fix -fix-errors -header-filter=.* +Clang_Tidy_Options = -fix -fix-errors # Sort the filenames to remove duplicates, then filter to retain # just the C and C++ sources so we don't pick up lex and yacc files @@ -23,12 +23,12 @@ Clang_Tidy_Options = -fix -fix-errors -header-filter=.* Clang_Tidy_CC_Files = $(filter %.c, $(sort $(1))) Clang_Tidy_CXX_Files = $(filter %.cc, $(sort $(1))) -#clang-tidy rules. We expect these to be actions with something like -#$(DIST_SOURCES) as the dependencies.rules. Note that $DIST_SOURCES -#is not an automake API, it is an implementation detail, but it ought -#to be stable enough. +# clang-tidy rules. We expect these to be actions with something like +# $(DIST_SOURCES) as the dependencies.rules. Note that $DIST_SOURCES +# is not an automake API, it is an implementation detail, but it ought +# to be stable enough. # -#All this clearly requires GNU make. +# All this clearly requires GNU make. -CXX_Clang_Tidy = $(CLANG_TIDY) $(Clang_Tidy_Options) $(call Clang_Tidy_CXX_Files,$^) -- $(CXXCOMPILE) -x c++ -CC_Clang_Tidy = $(CLANG_TIDY) $(Clang_Tidy_Options) $(call Clang_Tidy_CC_Files,$^) -- $(COMPILE) -x c +CXX_Clang_Tidy = $(foreach tidy_target, $(call Clang_Tidy_CXX_Files,$^), $(CLANG_TIDY) $(Clang_Tidy_Options) $(tidy_target) -- $(CXXCOMPILE) -x c++;) +CC_Clang_Tidy = $(foreach tidy_target, $(call Clang_Tidy_CC_Files,$^), $(CLANG_TIDY) $(Clang_Tidy_Options) $(tidy_target) -- $(COMPILE) -x c;) diff --git a/ci/asan_leak_suppression/unit_tests.txt b/ci/asan_leak_suppression/unit_tests.txt new file mode 100644 index 00000000000..b75dec9a9c4 --- /dev/null +++ b/ci/asan_leak_suppression/unit_tests.txt @@ -0,0 +1,6 @@ +# leaks in test_X509HostnameValidator +leak:libcrypto.so.1.1 +# for OpenSSL 1.0.2: +leak:CRYPTO_malloc +leak:CRYPTO_realloc +leak:ConsCell diff --git a/ci/docker/yum/Dockerfile b/ci/docker/yum/Dockerfile index 885c70ff398..774fa26c881 100644 --- a/ci/docker/yum/Dockerfile +++ b/ci/docker/yum/Dockerfile @@ -21,7 +21,7 @@ # limitations under the License. ################################################################################ -# These can (should?) be overriden from the command line with ----build-arg, e.g. +# These can (should?) be overridden from the command line with ----build-arg, e.g. # docker build --build-arg OS_VERSION=7 --build-arg OS_TYPE=centos # ARG OS_VERSION=28 @@ -48,11 +48,12 @@ RUN yum -y update; \ # Autoconf autoconf automake libtool \ # Various other tools - git rpm-build distcc-server file; \ + git rpm-build distcc-server file wget openssl hwloc; \ # Devel packages that ATS needs - yum -y install openssl-devel tcl-devel expat-devel pcre-devel libcap-devel hwloc-devel libunwind-devel \ + yum -y install openssl-devel expat-devel pcre-devel libcap-devel hwloc-devel libunwind-devel \ xz-devel libcurl-devel ncurses-devel jemalloc-devel GeoIP-devel kyotocabinet-devel luajit-devel \ - brotli-devel ImageMagick-devel ImageMagick-c++-devel perl-ExtUtils-MakeMaker perl-Digest-SHA; \ + brotli-devel ImageMagick-devel ImageMagick-c++-devel \ + perl-ExtUtils-MakeMaker perl-Digest-SHA perl-URI; \ # This is for autest stuff yum -y install python3 python3-virtualenv python-virtualenv httpd-tools procps-ng nmap-ncat; \ # Optional: This is for the OpenSSH server, and Jenkins account + access (comment out if not needed) diff --git a/ci/jenkins/ats_conf.pl b/ci/jenkins/ats_conf.pl index 0fd0b75c056..83d293e3c26 100755 --- a/ci/jenkins/ats_conf.pl +++ b/ci/jenkins/ats_conf.pl @@ -32,69 +32,69 @@ #$recedit->append(line => "CONFIG proxy.config.crash_log_helper STRING /home/admin/bin/invoker_wrap.sh"); # Port setup -$recedit->set(conf => "proxy.config.http.server_ports", val => "80 80:ipv6 443:ssl 443:ipv6:ssl"); -$recedit->set(conf => "proxy.config.admin.autoconf_port", val => "48083"); +$recedit->set(conf => "proxy.config.http.server_ports", val => "80 80:ipv6 443:ssl 443:ipv6:ssl"); +$recedit->set(conf => "proxy.config.admin.autoconf_port", val => "48083"); $recedit->set(conf => "proxy.config.process_manager.mgmt_port", val => "48084"); # Threads $recedit->set(conf => "proxy.config.exec_thread.autoconfig", val => "0"); -$recedit->set(conf => "proxy.config.exec_thread.limit", val => "8"); +$recedit->set(conf => "proxy.config.exec_thread.limit", val => "8"); $recedit->set(conf => "proxy.config.cache.threads_per_disk", val => "8"); -$recedit->set(conf => "proxy.config.accept_threads", val => "0"); -$recedit->set(conf => "proxy.config.exec_thread.affinity", val => "1"); +$recedit->set(conf => "proxy.config.accept_threads", val => "0"); +$recedit->set(conf => "proxy.config.exec_thread.affinity", val => "1"); # TLS #$recedit->set(conf => "proxy.config.ssl.server.cipher_suite", val => "ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA384:AES128-GCM-SHA256:AES256-GCM-SHA384:ECDHE-RSA-AES128-SHA:ECDHE-RSA-AES256-SHA:AES128-SHA:AES256-SHA:DES-CBC3-SHA!SRP:!DSS:!PSK:!aNULL:!eNULL:!SSLv2:!RC4"); $recedit->set(conf => "proxy.config.ssl.hsts_max_age", val => "17280000"); #$recedit->set(conf => "proxy.config.ssl.max_record_size", val => "-1"); -$recedit->set(conf => "proxy.config.ssl.session_cache", val => "2"); -$recedit->set(conf => "proxy.config.ssl.ocsp.enabled", val => "1"); +$recedit->set(conf => "proxy.config.ssl.session_cache", val => "2"); +$recedit->set(conf => "proxy.config.ssl.ocsp.enabled", val => "1"); $recedit->set(conf => "proxy.config.http2.stream_priority_enabled", val => "1"); # Cache setup -$recedit->set(conf => "proxy.config.cache.ram_cache.size", val => "1536M"); -$recedit->set(conf => "proxy.config.cache.ram_cache_cutoff", val => "4M"); -$recedit->set(conf => "proxy.config.cache.limits.http.max_alts", val => "4"); -$recedit->set(conf => "proxy.config.cache.dir.sync_frequency", val => "600"); # 10 minutes intervals +$recedit->set(conf => "proxy.config.cache.ram_cache.size", val => "1536M"); +$recedit->set(conf => "proxy.config.cache.ram_cache_cutoff", val => "4M"); +$recedit->set(conf => "proxy.config.cache.limits.http.max_alts", val => "4"); +$recedit->set(conf => "proxy.config.cache.dir.sync_frequency", val => "600"); # 10 minutes intervals $recedit->set(conf => "proxy.config.http.cache.ignore_client_cc_max_age", val => "1"); -$recedit->set(conf => "proxy.config.allocator.hugepages", val => "1"); +$recedit->set(conf => "proxy.config.allocator.hugepages", val => "1"); # HTTP caching related stuff -$recedit->set(conf => "proxy.config.http.cache.required_headers", val => "1"); -$recedit->set(conf => "proxy.config.http.insert_request_via_str", val => "1"); -$recedit->set(conf => "proxy.config.http.insert_response_via_str", val => "2"); -$recedit->set(conf => "proxy.config.http.negative_caching_enabled", val => "1"); +$recedit->set(conf => "proxy.config.http.cache.required_headers", val => "1"); +$recedit->set(conf => "proxy.config.http.insert_request_via_str", val => "1"); +$recedit->set(conf => "proxy.config.http.insert_response_via_str", val => "2"); +$recedit->set(conf => "proxy.config.http.negative_caching_enabled", val => "1"); $recedit->set(conf => "proxy.config.http.negative_caching_lifetime", val => "60"); -$recedit->set(conf => "proxy.config.http.chunking.size", val => "64k"); -$recedit->set(conf => "proxy.config.url_remap.pristine_host_hdr", val => "1"); +$recedit->set(conf => "proxy.config.http.chunking.size", val => "64k"); +$recedit->set(conf => "proxy.config.url_remap.pristine_host_hdr", val => "1"); # Timeouts -$recedit->set(conf => "proxy.config.http.keep_alive_no_activity_timeout_in", val => "300"); -$recedit->set(conf => "proxy.config.http.keep_alive_no_activity_timeout_out", val => "300"); +$recedit->set(conf => "proxy.config.http.keep_alive_no_activity_timeout_in", val => "300"); +$recedit->set(conf => "proxy.config.http.keep_alive_no_activity_timeout_out", val => "300"); $recedit->set(conf => "proxy.config.http.transaction_no_activity_timeout_out", val => "180"); -$recedit->set(conf => "proxy.config.http.transaction_no_activity_timeout_in", val => "180"); -$recedit->set(conf => "proxy.config.http.transaction_active_timeout_in", val => "180"); -$recedit->set(conf => "proxy.config.http.transaction_active_timeout_out", val => "180"); -$recedit->set(conf => "proxy.config.http.accept_no_activity_timeout", val => "30"); +$recedit->set(conf => "proxy.config.http.transaction_no_activity_timeout_in", val => "180"); +$recedit->set(conf => "proxy.config.http.transaction_active_timeout_in", val => "180"); +$recedit->set(conf => "proxy.config.http.transaction_active_timeout_out", val => "180"); +$recedit->set(conf => "proxy.config.http.accept_no_activity_timeout", val => "30"); # DNS / HostDB -$recedit->set(conf => "proxy.config.cache.hostdb.sync_frequency", val => "0"); +$recedit->set(conf => "proxy.config.cache.hostdb.sync_frequency", val => "0"); # Logging -$recedit->set(conf => "proxy.config.log.logging_enabled", val => "3"); -$recedit->set(conf => "proxy.config.log.max_space_mb_for_logs", val => "4096"); -$recedit->set(conf => "proxy.config.log.max_space_mb_headroom", val => "64"); +$recedit->set(conf => "proxy.config.log.logging_enabled", val => "3"); +$recedit->set(conf => "proxy.config.log.max_space_mb_for_logs", val => "4096"); +$recedit->set(conf => "proxy.config.log.max_space_mb_headroom", val => "64"); # Network -$recedit->set(conf => "proxy.config.net.connections_throttle", val => "10000"); -$recedit->set(conf => "proxy.config.net.sock_send_buffer_size_in", val => "4M"); +$recedit->set(conf => "proxy.config.net.connections_throttle", val => "10000"); +$recedit->set(conf => "proxy.config.net.sock_send_buffer_size_in", val => "4M"); $recedit->set(conf => "proxy.config.net.sock_recv_buffer_size_out", val => "4M"); -$recedit->set(conf => "proxy.config.net.poll_timeout", val => "30"); +$recedit->set(conf => "proxy.config.net.poll_timeout", val => "30"); # Local additions (typically not found in the records.config.default) -$recedit->set(conf => "proxy.config.dns.dedicated_thread", val => "0"); -$recedit->set(conf => "proxy.config.http_ui_enabled", val => "3"); -$recedit->set(conf => "proxy.config.http.server_max_connections", val =>"250"); +$recedit->set(conf => "proxy.config.dns.dedicated_thread", val => "0"); +$recedit->set(conf => "proxy.config.http_ui_enabled", val => "3"); +$recedit->set(conf => "proxy.config.http.server_max_connections", val => "250"); #$recedit->set(conf => "proxy.config.mlock_enabled", val => "2"); diff --git a/ci/jenkins/bin/build.sh b/ci/jenkins/bin/build.sh index a1e52fc8f37..9a9fd354a01 100755 --- a/ci/jenkins/bin/build.sh +++ b/ci/jenkins/bin/build.sh @@ -48,4 +48,4 @@ set -x ${WERROR} \ ${DEBUG} -${ATS_MAKE} ${ATS_MAKE_FLAGS} V=1 Q= +${ATS_MAKE} ${ATS_MAKE_FLAGS} V=1 Q= || exit 1 diff --git a/ci/jenkins/bin/cache-tests.sh b/ci/jenkins/bin/cache-tests.sh new file mode 100755 index 00000000000..9fece1111d3 --- /dev/null +++ b/ci/jenkins/bin/cache-tests.sh @@ -0,0 +1,54 @@ +#!/bin/sh +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Change to the build area (this is previously setup in extract.sh) +set +x +PREFIX="${WORKSPACE}/${BUILD_NUMBER}/install" + +cd "${WORKSPACE}/${BUILD_NUMBER}/build" + +./configure \ + --prefix=${PREFIX} \ + --with-user=jenkins \ + --enable-debug \ + --enable-ccache + +# Not great, but these can fail on the "docs' builds for older versions, sigh +${ATS_MAKE} -i ${ATS_MAKE_FLAGS} V=1 Q= +${ATS_MAKE} -i install + +[ -x ${PREFIX}/bin/traffic_server ] || exit 1 + + +# Setup and start ATS with the required remap rule +echo "map http://127.0.0.1:8080 http://192.168.3.13:8000" >> ${PREFIX}/etc/trafficserver/remap.config +${PREFIX}/bin/trafficserver start + +# Get NPM v10, and run the tests +source /opt/rh/rh-nodejs10/enable +set -x + +cd /home/jenkins/cache-tests +npm run --silent cli --base=http://127.0.0.1:8080/ > /CA/cache-tests/${ATS_BRANCH}.json + +${PREFIX}/bin/trafficserver stop + +cat /CA/cache-tests/${ATS_BRANCH}.json + +# We should check the .json file here, but not yet +exit 0 diff --git a/ci/jenkins/bin/environment.sh b/ci/jenkins/bin/environment.sh index 079035f6dd1..e76cf55522e 100755 --- a/ci/jenkins/bin/environment.sh +++ b/ci/jenkins/bin/environment.sh @@ -42,15 +42,21 @@ export TODAY=$(/bin/date +'%m%d%Y') ATS_BRANCH=master ATS_IS_7="yes" +test "${JOB_NAME#*-5.3.x}" != "${JOB_NAME}" && ATS_BRANCH=5.3.x && ATS_IS_7="no" test "${JOB_NAME#*-6.2.x}" != "${JOB_NAME}" && ATS_BRANCH=6.2.x && ATS_IS_7="no" test "${JOB_NAME#*-7.1.x}" != "${JOB_NAME}" && ATS_BRANCH=7.1.x -test "${JOB_NAME#*-7.2.x}" != "${JOB_NAME}" && ATS_BRANCH=7.2.x test "${JOB_NAME#*-8.0.x}" != "${JOB_NAME}" && ATS_BRANCH=8.0.x test "${JOB_NAME#*-8.1.x}" != "${JOB_NAME}" && ATS_BRANCH=8.1.x test "${JOB_NAME#*-8.2.x}" != "${JOB_NAME}" && ATS_BRANCH=8.2.x +test "${JOB_NAME#*-8.3.x}" != "${JOB_NAME}" && ATS_BRANCH=8.3.x test "${JOB_NAME#*-9.0.x}" != "${JOB_NAME}" && ATS_BRANCH=9.0.x test "${JOB_NAME#*-9.1.x}" != "${JOB_NAME}" && ATS_BRANCH=9.1.x -test "${JOB_NAME#*-9.2.x}" != "${JOB_NAME}" && ATS_BRANCH=9.1.x +test "${JOB_NAME#*-9.2.x}" != "${JOB_NAME}" && ATS_BRANCH=9.2.x +test "${JOB_NAME#*-9.3.x}" != "${JOB_NAME}" && ATS_BRANCH=9.3.x +test "${JOB_NAME#*-10.0.x}" != "${JOB_NAME}" && ATS_BRANCH=10.0.x +test "${JOB_NAME#*-10.1.x}" != "${JOB_NAME}" && ATS_BRANCH=10.1.x +test "${JOB_NAME#*-10.2.x}" != "${JOB_NAME}" && ATS_BRANCH=10.2.x +test "${JOB_NAME#*-10.3.x}" != "${JOB_NAME}" && ATS_BRANCH=10.3.x export ATS_BRANCH echo "Branch is $ATS_BRANCH" diff --git a/ci/jenkins/bin/gh-mirror.sh b/ci/jenkins/bin/gh-mirror.sh index 48d22f0028f..8ed06fd54df 100755 --- a/ci/jenkins/bin/gh-mirror.sh +++ b/ci/jenkins/bin/gh-mirror.sh @@ -54,14 +54,14 @@ function checkBuild() { echo -n "$diff" | ${GREP} -F -e doc/ >/dev/null if [ 0 == $? ]; then echo "Triggerd Docs build for ${branch}" - ${CURL} -o /dev/null -s ${BASE_URL}/view/${branch}/job/docs-${branch}/${token} + ${CURL} -o /dev/null -s ${BASE_URL}/job/docs-${branch}/${token} fi # Check if commits have non doc/ changes echo -n "$diff" | ${GREP} -F -v -e doc/ >/dev/null if [ 0 == $? ]; then echo "Triggered main build for ${branch}" - ${CURL} -o /dev/null -s ${BASE_URL}/view/${branch}/job/start-${branch}/${token} + ${CURL} -o /dev/null -s ${BASE_URL}/job/start-${branch}/${token} fi } @@ -69,6 +69,7 @@ function checkBuild() { REF_6_2=$(getRef "6.2.x") REF_7_1=$(getRef "7.1.x") REF_8_0=$(getRef "8.0.x") +REF_8_1=$(getRef "8.1.x") REF_master=$(getRef "master") # Do the updates @@ -79,4 +80,5 @@ ${GIT} update-server-info checkBuild "$REF_6_2" "6.2.x" checkBuild "$REF_7_1" "7.1.x" checkBuild "$REF_8_0" "8.0.x" +checkBuild "$REF_8_1" "8.1.x" checkBuild "$REF_master" "master" diff --git a/ci/jenkins/bin/regression.sh b/ci/jenkins/bin/regression.sh index c21725691e1..62c1f3237a5 100755 --- a/ci/jenkins/bin/regression.sh +++ b/ci/jenkins/bin/regression.sh @@ -19,6 +19,7 @@ cd "${WORKSPACE}/${BUILD_NUMBER}/build" [ -d BUILDS ] && cd BUILDS -${ATS_MAKE} check VERBOSE=Y V=1 && ${ATS_MAKE} install +${ATS_MAKE} check VERBOSE=Y V=1 || exit 1 +${ATS_MAKE} install || exit "${WORKSPACE}/${BUILD_NUMBER}/install/bin/traffic_server" -k -K -R 1 diff --git a/ci/jenkins/jobs.yaml b/ci/jenkins/jobs.yaml index 3fd1023fc42..24c802628a8 100644 --- a/ci/jenkins/jobs.yaml +++ b/ci/jenkins/jobs.yaml @@ -119,7 +119,7 @@ export CXXFLAGS=-m64 export CPPFLAGS=-I/opt/omni/include export LDFLAGS="-L/opt/omni/lib/amd64 -R/opt/omni/lib/amd64" - "${WORKSPACE}"/src/configure --prefix="${WORKSPACE}/install/${JOB_NAME}.${BUILD_NUMBER}" --enable-werror --with-tcl=/opt/omni/lib/amd64 --enable-experimental-plugins --enable-example-plugins --enable-test-tools $ENABLE_DEBUG $ENABLE_EXPERIMENTAL + "${WORKSPACE}"/src/configure --prefix="${WORKSPACE}/install/${JOB_NAME}.${BUILD_NUMBER}" --enable-werror --enable-experimental-plugins --enable-example-plugins --enable-test-tools $ENABLE_DEBUG $ENABLE_EXPERIMENTAL gmake -j2 V=1 gmake check gmake install diff --git a/ci/rat-regex.txt b/ci/rat-regex.txt index 1fda181301f..a7aacf8029b 100644 --- a/ci/rat-regex.txt +++ b/ci/rat-regex.txt @@ -25,9 +25,11 @@ .*\.gold$ ^\.gitignore$ ^\.gitmodules$ +^\.perltidyrc$ ^\.indent.pro$ ^\.vimrc$ ^\.clang-.*$ +^\.ripgreprc$ ^Doxyfile$ ^CHANGES$ ^CHANGELOG.*$ @@ -65,3 +67,8 @@ port\.h ^catch[.]hpp$ ^configuru.hpp$ ^yamlcpp$ +# tools/voluspa related +^.gometalint.conf$ +.*\.conf$ +.*config_default$ +gojsonschema(?: Apache 2.0. Properly documented in LICENSE-APACHE-2.0.txt ){0} diff --git a/ci/regression b/ci/regression index f8adfcffed9..1f83f6eeae3 100755 --- a/ci/regression +++ b/ci/regression @@ -68,7 +68,6 @@ prog() { extras() { case $(uname -v) in - omnios*) echo --with-tcl=/opt/omni/lib/amd64 ;; Darwin*) echo --with-openssl=$(brew --prefix openssl) ;; esac } diff --git a/configs/logging.yaml.default b/configs/logging.yaml.default index ea593ee4ad9..da116d6758e 100644 --- a/configs/logging.yaml.default +++ b/configs/logging.yaml.default @@ -3,7 +3,7 @@ # Documentation on logging: # https://docs.trafficserver.apache.org/en/8.0.x/admin-guide/logging/index.en.html # -# Documentaion on logging.yaml file format: +# Documentation on logging.yaml file format: # https://docs.trafficserver.apache.org/en/8.0.x/admin-guide/files/logging.yaml.en.html # # Example log configurations: diff --git a/configs/ssl_multicert.config.default b/configs/ssl_multicert.config.default index b3452d0b316..f4fab20638d 100644 --- a/configs/ssl_multicert.config.default +++ b/configs/ssl_multicert.config.default @@ -21,7 +21,7 @@ # dest_ip=ADDRESS # The IP (v4 or v6) address that the certificate should be presented # on. This is now only used as a fallback in the case that the TLS -# SubjectNameIndication extension is not supported. If ADDRESS is +# ServerNameIndication extension is not supported. If ADDRESS is # '*', the certificate will be used as the default fallback if no # other match can be made. # diff --git a/configure.ac b/configure.ac index eb758d2e557..9328124e753 100644 --- a/configure.ac +++ b/configure.ac @@ -1167,30 +1167,24 @@ CFLAGS="$__saved_CFLAGS" AC_SUBST([CURSES_LDFLAGS],[$curses_ldflags]) # -# Check for SSL presence and usability -TS_CHECK_CRYPTO +# Check for -latomic need (at least for mips arch) +TS_CHECK_ATOMIC +TS_ADDTO([LDFLAGS], [$ATOMIC_LIBS]) # -# Check for NextProtocolNegotiation TLS extension support. -TS_CHECK_CRYPTO_NEXTPROTONEG - +# Check for SSL presence and usability # -# Check for ALPN TLS extension support. -TS_CHECK_CRYPTO_ALPN +TS_CHECK_CRYPTO + +# Check for OpenSSL Version +TS_CHECK_CRYPTO_VERSION -# # Check for openssl ASYNC jobs TS_CHECK_CRYPTO_ASYNC -# -# Check for EC key support. -TS_CHECK_CRYPTO_EC_KEYS - -# -# Check for the presense of the certificate callback in the ssl library -TS_CHECK_CRYPTO_CERT_CB +# Check for the client hello callback +TS_CHECK_CRYPTO_HELLO_CB -# # Check for SSL_set0_rbio call TS_CHECK_CRYPTO_SET_RBIO @@ -1239,6 +1233,26 @@ AC_CHECK_FUNC([EVP_MD_CTX_free], [], LIBS="$saved_LIBS" +# +# Check OpenSSL version for JA3 Fingerprint +# +AC_MSG_CHECKING([for JA3 compatible OpenSSL version]) +AC_EGREP_CPP(yes, [ + #include + #if (OPENSSL_VERSION_NUMBER < 0x010100000L) + yes + #elif (OPENSSL_VERSION_NUMBER >= 0x010101000L) + yes + #endif + ], [ + AC_MSG_RESULT(yes) + AS_IF([test "x${enable_experimental_plugins}" = "xyes"], [ + enable_ja3_plugin=yes + ]) + ], [AC_MSG_RESULT(no)]) + +AM_CONDITIONAL([BUILD_JA3_PLUGIN], [test "x${enable_ja3_plugin}" = "xyes"]) + # # Check for zlib presence and usability TS_CHECK_ZLIB @@ -1247,28 +1261,8 @@ TS_CHECK_ZLIB # Check for lzma presence and usability TS_CHECK_LZMA -# -# Tcl macros provided by build/tcl.m4 -# -# this will error out if tclConfig.sh is not found -SC_PATH_TCLCONFIG - -# if tclConfig.sh loads properly, assume libraries are there and working -SC_LOAD_TCLCONFIG - -# expect tclConfig.sh to populate TCL_LIB_FLAG and TCL_INCLUDE_SPEC -if test "$host_os_def" == "darwin"; then - TCL_LIB_SPEC="-ltcl" # OSX fails to populate this variable -fi -AC_SUBST([LIBTCL],[$TCL_LIB_SPEC]) - - -if test "x${TCL_INCLUDE_SPEC}" != "x-I/usr/include"; then - TS_ADDTO(TS_INCLUDES, [$TCL_INCLUDE_SPEC]) -fi - AC_CHECK_FUNCS([clock_gettime kqueue epoll_ctl posix_fadvise posix_madvise posix_fallocate inotify_init]) -AC_CHECK_FUNCS([lrand48_r srand48_r port_create strlcpy strlcat sysconf sysctlbyname getpagesize]) +AC_CHECK_FUNCS([port_create strlcpy strlcat sysconf sysctlbyname getpagesize]) AC_CHECK_FUNCS([getreuid getresuid getresgid setreuid setresuid getpeereid getpeerucred]) AC_CHECK_FUNCS([strsignal psignal psiginfo accept4]) @@ -1310,27 +1304,15 @@ TS_CHECK_LUAJIT # Enable experimental/uri_singing plugin # This is here, instead of above, because it needs to know if PCRE is available. # -AC_CHECK_HEADERS([jansson.h], [ - AC_MSG_CHECKING([whether jansson is dynamic]) - TS_LINK_WITH_FLAGS_IFELSE([-fPIC -ljansson],[AC_LANG_PROGRAM( - [#include ], - [(void) json_object();])], - [AC_MSG_RESULT([yes]); LIBJANSSON=-ljansson], - [AC_MSG_RESULT([no]); LIBJANSSON=-l:libjansson.a]) - ], - [LIBJANSSON=]) - -AC_CHECK_HEADERS([cjose/cjose.h], [ - AC_MSG_CHECKING([whether cjose is dynamic]) - TS_LINK_WITH_FLAGS_IFELSE([-fPIC -lcjose],[AC_LANG_PROGRAM( - [#include ], - [(void) cjose_jws_import("", 0, NULL);])], - [AC_MSG_RESULT([yes]); LIBCJOSE=-lcjose], - [AC_MSG_RESULT([no]); LIBCJOSE=-l:libcjose.a]) - ], - [LIBCJOSE=]) + +#### Check for optional jansson library (uri_signing) +TS_CHECK_JANSSON + AC_CHECK_LIB([crypto],[HMAC],[has_libcrypto=1],[has_libcrypto=0]) +#### Check for optional cjose library (uri_signing) +TS_CHECK_CJOSE + AM_CONDITIONAL([BUILD_URI_SIGNING_PLUGIN], [test ! -z "${LIBCJOSE}" -a ! -z "${LIBJANSSON}" -a "x${enable_pcre}" = "xyes" -a "x${has_libcrypto}" = "x1"]) AC_SUBST([LIBCJOSE]) AC_SUBST([LIBJANSSON]) @@ -1807,6 +1789,46 @@ AC_LINK_IFELSE([ AC_MSG_RESULT([no]) ]) +# pthread_getname_np / pthread_get_name_np: +AC_MSG_CHECKING([pthread_getname_np()]) +AC_LINK_IFELSE([ + AC_LANG_PROGRAM([ +#if HAVE_PTHREAD_H +#include +#endif +#if PTHREAD_NP_H +#include +#endif + ], [ + char name[[32]]; + pthread_getname_np(pthread_self(), name, sizeof(name)); + ]) + ], [ + AC_DEFINE(HAVE_PTHREAD_GETNAME_NP, 1, [Whether pthread_getname_np() is available]) + AC_MSG_RESULT([yes]) + ], [ + AC_MSG_RESULT([no]) + AC_MSG_CHECKING([pthread_get_name_np()]) + AC_LINK_IFELSE([ + AC_LANG_PROGRAM([ + #if HAVE_PTHREAD_H + #include + #endif + #if PTHREAD_NP_H + #include + #endif + ], [ + char name[[32]]; + pthread_get_name_np(pthread_self(), name, sizeof(name)); + ]) + ], [ + AC_DEFINE(HAVE_PTHREAD_GET_NAME_NP, 1, [Whether pthread_get_name_np() is available]) + AC_MSG_RESULT([yes]) + ], [ + AC_MSG_RESULT([no]) + ]) +]) + # BSD-derived systems populate the socket length in the structure itself. It's # redundant to check all of these, but hey, I need the typing practice. AC_CHECK_MEMBER([struct sockaddr.sa_len], [], [], [#include ]) diff --git a/contrib/install_trafficserver.sh b/contrib/install_trafficserver.sh index 30daf2ce514..4223efa9b50 100644 --- a/contrib/install_trafficserver.sh +++ b/contrib/install_trafficserver.sh @@ -77,7 +77,6 @@ function updateInstall() { make \ libtool \ libssl-dev \ - tcl-dev \ libpcre3-dev \ curl apt-get install -y subversion git git-svn @@ -95,7 +94,6 @@ function updateInstall() { gcc-c++ \ glibc-devel \ openssl-devel \ - tcl-devel \ db4-devel \ pcre \ pcre-devel @@ -108,7 +106,6 @@ function updateInstall() { gcc-c++ \ glibc-devel \ openssl-devel \ - tcl-devel \ pcre \ pcre-devel fi @@ -139,7 +136,7 @@ function dev() { git clone git://git.apache.org/trafficserver.git cd $EC2_EPHEMERAL/$trafficserver - #swtich to dev build + #switch to dev build git checkout -b remotes/origin/dev #------------------------------------------------------ } diff --git a/contrib/python/compare_RecordsConfigcc.py b/contrib/python/compare_RecordsConfigcc.py index d41ac6d2036..b01a133c963 100644 --- a/contrib/python/compare_RecordsConfigcc.py +++ b/contrib/python/compare_RecordsConfigcc.py @@ -65,7 +65,7 @@ if m: rc_in[m.group(1)] = (m.group(2), m.group(3)) -# Process records.comfig documentation. +# Process records.config documentation. # eg. .. ts:cv:: CONFIG proxy.config.proxy_binary STRING traffic_server with open("%s/doc/admin-guide/files/records.config.en.rst" % src_dir) as fh: doc_re = re.compile(r'ts:cv:: CONFIG (\S+)\s+(\S+)\s+(\S+)') diff --git a/contrib/vagrant-setup.sh b/contrib/vagrant-setup.sh index 8b62a4172b2..3bb255a144d 100644 --- a/contrib/vagrant-setup.sh +++ b/contrib/vagrant-setup.sh @@ -36,8 +36,7 @@ trusty*|jessie*) libssl-dev \ m4 \ ncurses-dev \ - git \ - tcl-dev + git ;; centos*) @@ -57,8 +56,7 @@ centos*) ncurses-devel \ openssl-devel \ pcre-devel \ - git \ - tcl-devel + git ;; fedora*) @@ -78,7 +76,6 @@ fedora*) ncurses-devel \ openssl-devel \ pcre-devel \ - tcl-devel \ git \ make ;; @@ -106,7 +103,6 @@ omnios) developer/versioning/git \ library/idnkit \ library/idnkit/header-idnkit \ - omniti/runtime/tcl-8 \ omniti/system/hwloc \ system/header \ system/library/math \ diff --git a/doc/.tx/config b/doc/.tx/config index be18c3c2c94..6640ca1228c 100644 --- a/doc/.tx/config +++ b/doc/.tx/config @@ -162,11 +162,6 @@ file_filter = locale//LC_MESSAGES/admin-guide/monitoring/logging/log-build source_file = _build/locale/pot/admin-guide/monitoring/logging/log-builder.en.pot source_lang = en -[apache-traffic-server-6x.admin-guide--monitoring--logging--log-collation_en] -file_filter = locale//LC_MESSAGES/admin-guide/monitoring/logging/log-collation.en.po -source_file = _build/locale/pot/admin-guide/monitoring/logging/log-collation.en.pot -source_lang = en - [apache-traffic-server-6x.admin-guide--monitoring--logging--log-formats_en] file_filter = locale//LC_MESSAGES/admin-guide/monitoring/logging/log-formats.en.po source_file = _build/locale/pot/admin-guide/monitoring/logging/log-formats.en.pot diff --git a/doc/Doxyfile b/doc/Doxyfile index 5eb7b83d797..526d7447e8a 100644 --- a/doc/Doxyfile +++ b/doc/Doxyfile @@ -230,12 +230,6 @@ TAB_SIZE = 4 ALIASES = -# This tag can be used to specify a number of word-keyword mappings (TCL only). -# A mapping has the form "name=value". For example adding "class=itcl::class" -# will allow you to use the command class in the itcl::class meaning. - -TCL_SUBST = - # Set the OPTIMIZE_OUTPUT_FOR_C tag to YES if your project consists of C sources # only. Doxygen will then generate output that is more tailored for C. For # instance, some of the names that are used will be different. The list of all diff --git a/doc/Makefile.am b/doc/Makefile.am index caa7db7a8db..f6c174fe6d9 100644 --- a/doc/Makefile.am +++ b/doc/Makefile.am @@ -48,12 +48,10 @@ doxygen: Doxyfile PAPER = letter BUILDDIR = docbuild -# Internal variables. # [amc] LaTex apparently doesn't work as of Sphinx 1.6.1 # see https://media.readthedocs.org/pdf/sphinx/1.6.3/sphinx.pdf # section 24.3.2 around page 247, third item for 'NotImplementedError', so this is kind of useless. -PAPEROPT_a4 = -t latex_a4 -PAPEROPT_letter = -t latex_letter + ALLSPHINXOPTS = $(SPHINXOPTS) # the i18n builder cannot share the environment and doctrees with the others I18NSPHINXOPTS = $(SPHINXOPTS) diff --git a/doc/admin-guide/configuration/index.en.rst b/doc/admin-guide/configuration/index.en.rst index cc2e3f603e6..1aecdbe29fe 100644 --- a/doc/admin-guide/configuration/index.en.rst +++ b/doc/admin-guide/configuration/index.en.rst @@ -32,3 +32,4 @@ Proxy Cache Configuration transparent-proxy.en transparent-forward-proxying.en hierachical-caching.en + proxy-protocol.en diff --git a/doc/admin-guide/configuration/proxy-protocol.en.rst b/doc/admin-guide/configuration/proxy-protocol.en.rst new file mode 100644 index 00000000000..2d7673b2914 --- /dev/null +++ b/doc/admin-guide/configuration/proxy-protocol.en.rst @@ -0,0 +1,100 @@ +.. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +.. include:: ../../common.defs + +.. _proxy-protocol: + +Proxy Protocol +**************** + +The `PROXY protocol `_ +provides a means of passing connection information between layers of the proxy +infrastructure. Without the PROXY protocol, |TS| would only have connection +information from the previous hop connecting to |TS| and not the actual +originating client connection information. This can be done over either HTTP or +TLS connections. + +.. note:: + + The current version only supports transforming client IP from PROXY Version 1 + header to the Forwarded: header. + +In the current implementation, the client IP address in the PROXY protocol header +is passed to the origin server via an HTTP `Forwarded: +`_ header. + +The Proxy Protocol must be enabled on each port. See +:ts:cv:`proxy.config.http.server_ports` for information on how to enable the +Proxy Protocol on a port. Once enabled, all incoming requests must be prefaced +with the PROXY v1 header. Any request not preface by this header will be +dropped. + +As a security measure, an optional whitelist of trusted IP addresses may be +configured with :ts:cv:`proxy.config.http.proxy_protocol_whitelist`. + + .. important:: + + If the whitelist is configured, requests will only be accepted from these + IP addressses and must be prefaced with the PROXY v1 header. + +See :ts:cv:`proxy.config.http.insert_forwarded` for configuration information. +Detection of the PROXY protocol header is automatic. If the PROXY header +precludes the request, it will automatically be parse and made available to the +Forwarded: request header sent to the origin server. + +Example +------- + +As an example, consider the following topology: + +.. figure:: ../../static/images/admin/proxy-protocol.png + :align: center + :alt: PROXY protocol transformed into a Forwarded: header + + PROXY protocol header flow + +Without the PROXY protocol header, the client IP would only be reported +accurately to the Load Balancer. |TS| would only see the connection from the +Load Balancer. Similarly, the Web Server would only see the connection from +|TS|. In the example above, if the client initiated a TLS connection, the Web +Server would see the connection originating from |TS| at ``10.0.0.2``: + +.. code-block:: lua + + Forwarded: for=10.0.0.2;by=10.0.0.1;proto=https;host=test000001.com + +If the Load Balancer has the Proxy Protocol enabled, requests sent through the +Load Balancer will be preceded with the PROXY header. |TS| will detect the +PROXY header and transform that into the Forwarded: HTTP header if configured to +insert the Forwarded: header with the ``for`` paramter. In the example above, +if the client initiated a TLS connection, the Web Server can use the Forwarded: +header to determine the TLS connection originated from the client at ``192.168.1.100``: + +.. code-block:: lua + + Forwarded: for=192.168.2.100;by=10.0.0.2;proto=https;host=test000001.com + + +References +========== + +- `The PROXY protocol Versions 1 & 2 + `_ + +- `Forwarded HTTP Extension + `_ diff --git a/doc/admin-guide/files/cache.config.en.rst b/doc/admin-guide/files/cache.config.en.rst index 32e75528e01..3971b9a70c1 100644 --- a/doc/admin-guide/files/cache.config.en.rst +++ b/doc/admin-guide/files/cache.config.en.rst @@ -22,8 +22,8 @@ cache.config ************ -The :file:`cache.config` file defines how |TS| caches web objects. You can add -caching rules to specify the following: +The :file:`cache.config` file allows you to overrule the origin's cache +policies. You can add caching rules to specify the following: - Not to cache objects from specific IP addresses. - How long to pin particular objects in the cache. @@ -32,6 +32,15 @@ caching rules to specify the following: .. important:: + Generally, using this file to define cache policies is an antipattern. + It's usually better to have the origin specify the cache policy via the + `Cache-Control: `_ header. + That way, all the business logic stays with the content generation. The + origin is in a much better position to know which content can be safely + cached, and for how long. It can make fine grained decisions, changing + Cache-Control: header value per object. This file allows for some overrides + but, is relatively crude compared to what the origin can provide. + After modifying :file:`cache.config`, run :option:`traffic_ctl config reload` to apply changes. @@ -148,7 +157,8 @@ specifiers of the rule in question. =========================== ================================================ Value Effect =========================== ================================================ - ``never-cache`` Never cache specified objects. + ``never-cache`` Never cache specified objects, it will be + overwrited by ``ttl-in-cache``. ``ignore-no-cache`` Ignore all ``Cache-Control: no-cache`` headers. ``ignore-client-no-cache`` Ignore ``Cache-Control: no-cache`` headers from client requests. @@ -241,6 +251,16 @@ prefix. The former fails at this goal, because the second rule will match all Javascript files and will override any previous ``revalidate`` values that may have been set by prior rules. +ttl-in-cache and never-cache +---------------------------- + +When multiple rules are matched in the same request, ``never-cache`` will always +be overwrited by ``ttl-in-cache``. For example:: + + # ttl-in-cache=1d never-cache=false + dest_domain=example.com action=never-cache + dest_domain=example.com ttl-in-cache=1d + Examples ======== diff --git a/doc/admin-guide/files/ip_allow.config.en.rst b/doc/admin-guide/files/ip_allow.config.en.rst index 537afcb6eaa..ce7c632ad69 100644 --- a/doc/admin-guide/files/ip_allow.config.en.rst +++ b/doc/admin-guide/files/ip_allow.config.en.rst @@ -39,7 +39,7 @@ format:: dest_ip= action= [method=] For ``src_ip`` the remote inbound connection address, i.e. the IP address of the client, is checked -against the specified range of IP addresses. For ``dst_ip`` the outbound remote address (i.e. the IP +against the specified range of IP addresses. For ``dest_ip`` the outbound remote address (i.e. the IP address to which |TS| connects) is checked against the specified IP address range. Range specifications can be IPv4 or IPv6, but any single range must be one or the other. Ranges can @@ -55,7 +55,7 @@ range with the lower and upper values equal to that IP address). The value of ``method`` is a string which must consist of either HTTP method names separated by the character '|' or the keyword literal ``ALL``. This keyword may omitted in which case it is treated as if it were ``method=ALL``. Methods can also be specified by having multiple instances of the -``method`` keyword, each specifiying a single method. E.g., ``method=GET|HEAD`` is the same as +``method`` keyword, each specifying a single method. E.g., ``method=GET|HEAD`` is the same as ``method=GET method=HEAD``. The method names are not validated which means non-standard method names can be specified. @@ -67,7 +67,7 @@ For each inbound or outbound connection the applicable rule is selectd by first address. The rule is then applied (if the method matches) or its opposite is applied (if the method doesn't match). If no rule is matched access is allowed. This makes each rule both an accept and deny, one explicit and the other implicit. The ``src_ip`` rules are checked when a host connects -to |TS|. The ``dst_ip`` rules are checked when |TS| connects to another host. +to |TS|. The ``dest_ip`` rules are checked when |TS| connects to another host. By default the :file:`ip_allow.config` file contains the following lines, which allows all methods to connections from localhost and denies the ``PUSH``, ``PURGE`` and ``DELETE`` methods to all other @@ -104,7 +104,7 @@ If the entire subnet were to be denied, that would be:: src_ip=123.45.6.0/24 action=ip_deny -The following example allows to any upstream servers:: +The following example allows one to any upstream servers:: dest_ip=0.0.0.0-255.255.255.255 action=ip_allow diff --git a/doc/admin-guide/files/logging.yaml.en.rst b/doc/admin-guide/files/logging.yaml.en.rst index 10bc4045596..96a6fe3d12a 100644 --- a/doc/admin-guide/files/logging.yaml.en.rst +++ b/doc/admin-guide/files/logging.yaml.en.rst @@ -23,14 +23,15 @@ logging.yaml ************** The :file:`logging.yaml` file defines all custom log file formats, filters, -and processing options. The file itself is a Lua script. +and processing options. .. important:: - This configuration file replaces the XML based logs_xml.config from past - |TS| releases. If you are upgrading from a |TS| release which used that - configuration file, and you have created custom log formats, filters, and - destinations, you will need to update those settings to this format. + This configuration file replaces the XML based logs_xml.config, as well as + the Lua based logging.config from past |TS| releases. If you are upgrading + from a |TS| release which used either the XML or the Lua configuration file + format, and you have created custom log formats, filters, and destinations, + you will need to update those settings to this format. .. _admin-custom-logs: @@ -62,11 +63,10 @@ format. Which approach you use is entirely up to you, though it's strongly recommended to create an explicit format object if you intend to reuse the same format for multiple log files. -To create a format object, store the result of the ``format`` function in a -variable. The function takes a table with two attributes: a mandatory string -``Format`` which defines the output format string for every event; and an -optional number ``Interval`` defining the aggregation interval for summary -logs. +Custom formats are defined by choosing a ``name`` to identify the given logging +format, and a ``format`` string, which defines the output format string for +every event. An optional ``interval`` attribute can be specified to define the +aggregation interval for summary logs. .. code:: yaml @@ -89,8 +89,8 @@ desire. Format Specification ~~~~~~~~~~~~~~~~~~~~ -The format specification provided as the required ``Format`` entry of the table -passed to the format function is a simple string, containing whatever mixture +The format specification provided as the required ``format`` attribute of the +objects listed in ``formats`` is a simple string, containing whatever mixture of logging field variables and literal characters meet your needs. Logging fields are discussed in great detail in the :ref:`admin-logging-fields` section. @@ -142,7 +142,7 @@ You will find a complete listing of the available fields in Aggregation Interval ~~~~~~~~~~~~~~~~~~~~ -Every format may be given an optional ``Interval`` value, specified as the +Every format may be given an optional ``interval`` value, specified as the number of seconds over which events destined for a log using the format are aggregated and summarized. Logs which use formats containing an aggregation interval do not behave like regular logs, with a single line for every event. @@ -159,29 +159,26 @@ given one. Filters ------- -Filters may be used, optionally, to accept or reject logging for matching -events, or to scrub the values of individual fields from logging output (while -retaining other information; useful for ensuring that sensitive information -cannot inadvertently make it into log files). +Two different type of filters are available: ``accept`` and ``reject``. They +may be used, optionally, to accept or reject logging for matching events. -Filter objects are created by calling one of the following functions: +Filter objects are created by assigning them a ``name`` to be used later to +refer to the filter, as well as an ``action`` (either ``accept`` or +``reject``). ``Accept`` and ``reject`` filters require a ``condition`` against +which to match all events. The ``condition`` fields must be in the following +format:: -filter.accept(string) - Creates a filter object which accepts events for logging which match the - rule specified in ``string``. Note that you may only have one accept filter. - -filter.reject(string) - Creates a filter object which rejects events for logging which match the - rule specified in ``string``. You may have multiple reject filters. + -filter.wipe(string) - Creates a filter object which clears the values of query parameters listed - in ``string``. +For example, the following snippet defines a filter that matches all POST +requests: -For both ``accept`` and ``wipe`` filters, the string passed defines a rule in -the following format:: +.. code:: yaml - + filters: + - name: postfilter + action: accept + condition: cqhm MATCH POST Filter Fields ~~~~~~~~~~~~~ @@ -245,21 +242,6 @@ supported at this time. expect. If, for example, we had 2 accept log filters, each disjoint from the other, nothing will ever get logged on the given log object. -Wiping Filters -~~~~~~~~~~~~~~ - -Filters created with ``filter.wipe`` function differently than the accept and -reject filters. Instead of a rule, as described above for those filter types, -the wiping filter simply lists the query parameter(s) whose values should be -scrubbed before any logging occurs. This prevents sensitive information from -being logged by fields which include the query string portion of the request -URL. It can also be useful to remove things like cache-busting or -inconsequentially variable parameters that might otherwise obfuscate the -reporting from log analyzers. - -Multiple query parameters may be listed, separated by spaces, though only the -first occurence of each will be wiped from the query string if any individual -parameter appears more than once in the URL. .. _admin-custom-logs-logs: @@ -268,28 +250,16 @@ Logs Up to this point, we've only described what events should be logged and what they should look like in the logging output. Now we define where those logs -should be sent. Three options currently exist for the type of logging output, -and each is selected by invoking the appropriate function. All three functions -take a single Lua table as their argument, with the same set of key/value -pairs. - -log.ascii(table) - Creates an ASCII logging object. +should be sent. -log.binary(table) - Creates a binaryy logging object. +Three options currently exist for the type of logging output: ``ascii``, +``binary``, and ``ascii_pipe``. Which type of logging output you choose +depends largely on how you intend to process the logs with other tools, and a +discussion of the merits of each is covered elsewhere, in +:ref:`admin-logging-ascii-v-binary`. -log.pipe(table) - Creates a logging object that logs to a pipe. - -There is no need to capture the return values of these functions. Which type of -logging output you choose depends largely on how you intend to process the logs -with other tools, and a discussion of the merits of each is covered elsewhere, -in :ref:`admin-logging-ascii-v-binary`. - -The following subsections cover the contents of the table which should be -passed when creating your logging object. Only ``Filename`` and ``Format`` are -required. +The following subsections cover the attributes you should specify when creating +your logging object. Only ``filename`` and ``format`` are required. ====================== =========== ================================================= Name Type Description @@ -318,13 +288,6 @@ rolling_min_count number Specifies the minimum number of rolled logs t filters array of The optional list of filter objects which filters restrict the individual events logged. The array may only contain one accept filter. -collation_hosts array of If present, one or more strings specifying the - strings log collation hosts to which logs should be - delivered, each in the form of ":". - :ref:`admin-logging-collation` for more - information. NOTE: This is a deprecated feature, - which will be removed in ATS v9.0.0. See the - logging documentation (above) for more details. ====================== =========== ================================================= Enabling log rolling may be done globally in :file:`records.config`, or on a @@ -381,16 +344,8 @@ to be logged: filters: - name: refreshhitfilter - accept: pssc MATCH REFRESH_HIT - -The following is an example of a filter that will cause the value of the first -query parameter named ``passwd`` to be wiped. - -.. code:: yaml - - filters: - - name: passwdfilter - wipe: passwd + action: accept + condition: pssc MATCH REFRESH_HIT The following is an example of a log specification that creates a local log file for the minimal format defined earlier. The log filename will be diff --git a/doc/admin-guide/files/parent.config.en.rst b/doc/admin-guide/files/parent.config.en.rst index 3a26e959b9e..725b0d2f99e 100644 --- a/doc/admin-guide/files/parent.config.en.rst +++ b/doc/admin-guide/files/parent.config.en.rst @@ -141,6 +141,14 @@ The following list shows the possible actions and their allowed values. parent="p1.x.com:8080|2.0, 192.168.0.3:80|3.0, 192.168.0.4:80|5.0" + If ``round_robin`` is set to ``consistent_hash``, you may add a ``unique hash string`` + following the ``weight`` for each parent. The ``hash string`` must start with ``&`` + and is used to build both the primary and secondary rings using the ``hash string`` + for each parent insted of the parents ``hostname`` or ``ip address``. This can be + useful so that two different hosts may be used to cache the same requests. Example:: + + parent="p1.x.com:80|1.0&abcdef, p2.x.com:80|1.0&xyzl, p3.x.com:80|1.0&ldffg" round_robin=consistent_hash + .. _parent-config-format-secondary-parent: ``secondary_parent`` @@ -202,7 +210,7 @@ The following list shows the possible actions and their allowed values. - ``simple_retry`` - If the parent origin server returns a 404 response on a request a new parent is selected and the request is retried. The number of retries is controlled by ``max_simple_retries`` which is set to 1 by default. - - ``unavailable_server_retry`` - If the parent returns a 503 response or if the reponse matches + - ``unavailable_server_retry`` - If the parent returns a 503 response or if the response matches a list of http 5xx responses defined in ``unavailable_server_retry_responses``, the currently selected parent is marked down and a new parent is selected to retry the request. The number of retries is controlled by ``max_unavailable_server_retries`` which is set to 1 by default. @@ -220,7 +228,7 @@ The following list shows the possible actions and their allowed values. ``max_simple_retries`` By default the value for ``max_simple_retries`` is 1. It may be set to any value in the range 1 to 5. - If ``parent_retry`` is set to ``simple_retry`` or ``both`` a 404 reponse + If ``parent_retry`` is set to ``simple_retry`` or ``both`` a 404 response from a parent origin server will cause the request to be retried using a new parent at most 1 to 5 times as configured by ``max_simple_retries``. @@ -228,7 +236,7 @@ The following list shows the possible actions and their allowed values. ``max_unavailable_server_retries`` By default the value for ``max_unavailable_server_retries`` is 1. It may be set to any value in the range 1 to 5. - If ``parent_retry`` is set to ``unavailable_server_retries`` or ``both`` a 503 reponse + If ``parent_retry`` is set to ``unavailable_server_retries`` or ``both`` a 503 response by default or any http 5xx response listed in the list ``unavailable_server_retry_responses`` from a parent origin server will cause the request to be retried using a new parent after first marking the current parent down. The request will be retried at most 1 to 5 times as configured by ``max_unavailable_server_retries``. diff --git a/doc/admin-guide/files/records.config.en.rst b/doc/admin-guide/files/records.config.en.rst index a5075e06e00..b0d3c974dfb 100644 --- a/doc/admin-guide/files/records.config.en.rst +++ b/doc/admin-guide/files/records.config.en.rst @@ -396,7 +396,7 @@ Network can handle simultaneously. This is in fact the max number of file descriptors that the :program:`traffic_server` process can have open at any given time. Roughly 10% of these connections are reserved for origin server - connections, i.e. from the default, only ~9,000 client connections can be + connections, i.e. from the default, only ~27,000 client connections can be handled. This should be tuned according to your memory size, and expected work load. If this is set to 0, the throttling logic is disabled. @@ -610,6 +610,7 @@ HTTP Engine ip-out Value Local outbound IP address. ip-resolve Value IP address resolution style. proto Value List of supported session protocols. + pp Enable Proxy Protocol. ssl SSL terminated. tr-full Fully transparent (inbound and outbound) tr-in Inbound transparent. @@ -646,6 +647,12 @@ proto all available protocols. For non-TLS proxy ports the default is HTTP only. +pp + Enables Proxy Protocol on the port. If Proxy Protocol is enabled on the + port, all incoming requests must be prefaced with the PROXY header. See + :ref:`Proxy Protocol ` for more details on how to configure + this option properly. + tr-full Fully transparent. This is a convenience option and is identical to specifying both ``tr-in`` and ``tr-out``. @@ -803,7 +810,7 @@ mptcp ===== ====================================================================== Value Description ===== ====================================================================== - ``0`` |TS| will buffer the request until the post body has been recieved and + ``0`` |TS| will buffer the request until the post body has been received and then send the request to the origin server. ``1`` Immediately return a ``100 Continue`` from |TS| without waiting for the post body. @@ -870,14 +877,9 @@ mptcp .. note:: - If HTTP/1.1 is used, then |TS| can use keep-alive connections with - pipelining to origin servers. - - If HTTP/1.0 is used, then |TS| can use keep-alive connections without - pipelining to origin servers. + If HTTP/1.1 is used, then |TS| can use keep-alive connections to origin servers. - If HTTP/0.9 is used, then |TS| does not use keep-alive connections to - origin servers. + If HTTP/1.0 is used, then |TS| can use keep-alive connections to origin servers. .. ts:cv:: CONFIG proxy.config.http.chunking.size INT 4096 :overridable: @@ -1095,6 +1097,15 @@ mptcp .. ts:cv:: CONFIG proxy.config.http.default_buffer_water_mark INT 32768 +.. ts:cv:: CONFIG proxy.config.http.request_buffer_enabled INT 0 + :overridable: + + This is a configuration value that is overridable but not configurable. This is most likely an + implementation error. + + This enables buffering the content for incoming ``POST`` requests. If enabled no outbound + connection is made until the entire ``POST`` request has been buffered. + .. ts:cv:: CONFIG proxy.config.http.request_header_max_size INT 131072 Controls the maximum size, in bytes, of an HTTP header in requests. Headers @@ -1423,7 +1434,7 @@ Origin Server Connect Attempts This value is used in determining when and if to prune active origin sessions. Without this value set, connections to origins can consume all the - way up to ts:cv:`proxy.config.net.connections_throttle` connections, which + way up to :ts:cv:`proxy.config.net.connections_throttle` connections, which in turn can starve incoming requests from available connections. .. ts:cv:: CONFIG proxy.config.http.per_server.connection.max INT 0 @@ -1729,12 +1740,36 @@ Proxy User Variables connection=full Full user agent connection :ref:`protocol tags ` ================== =============================================================== - Each paramater in the list must be separated by ``|`` or ``:``. For example, ``for|by=uuid|proto`` is + Each parameter in the list must be separated by ``|`` or ``:``. For example, ``for|by=uuid|proto`` is a valid value for this variable. Note that the ``connection`` parameter is a non-standard extension to RFC 7239. Also note that, while |TS| allows multiple ``by`` parameters for the same proxy, this is prohibited by RFC 7239. Currently, for the ``host`` parameter to provide the original host from the incoming client request, `proxy.config.url_remap.pristine_host_hdr`_ must be enabled. +.. ts:cv:: CONFIG proxy.config.http.proxy_protocol_whitelist STRING `````` + + This defines a whitelist of server IPs that are trusted to provide + connections with Proxy Protocol information. This is a comma delimited list + of IP addresses. Addressed may be listed individually, in a range separated + by a dash or by using CIDR notation. + + ======================= =========================================================== + Example Effect + ======================= =========================================================== + ``10.0.2.123`` A single IP Address. + ``10.0.3.1-10.0.3.254`` A range of IP address. + ``10.0.4.0/24`` A range of IP address specified by CIDR notation. + ======================= =========================================================== + + .. important:: + + If Proxy Protocol is enabled on the port, but this directive is not + defined any server may initiate a connection with Proxy Protocol + information. + See :ts:cv:`proxy.config.http.server_ports` for information on how to enable Proxy Protocol on a port. + + See :ref:`proxy-protocol` for more discussion on how |TS| tranforms the `Forwarded: header`. + .. ts:cv:: CONFIG proxy.config.http.normalize_ae INT 1 :reloadable: :overridable: @@ -2168,10 +2203,10 @@ RAM Cache .. ts:cv:: CONFIG proxy.config.cache.ram_cache.algorithm INT 1 - Two distinct RAM caches are supported, the default (0) being the **CLFUS** - (*Clocked Least Frequently Used by Size*). As an alternative, a simpler - **LRU** (*Least Recently Used*) cache is also available, by changing this - configuration to 1. + Two distinct RAM caches are supported, the default (1) being the simpler + **LRU** (*Least Recently Used*) cache. As an alternative, the **CLFUS** + (*Clocked Least Frequently Used by Size*) is also available, by changing this + configuration to 0. .. ts:cv:: CONFIG proxy.config.cache.ram_cache.use_seen_filter INT 1 @@ -2246,55 +2281,6 @@ Heuristic Expiration aging factor is applied, the final maximum age calculated will never be higher than the value in this variable. -.. ts:cv:: CONFIG proxy.config.http.cache.fuzz.time INT 0 - :deprecated: - :reloadable: - :overridable: - - How often |TS| checks for an early refresh, during the period before the - document stale time. The interval specified must be in seconds. - -.. note:: - - Previous versions of Apache |TS| defaulted this to 240s. This - feature is deprecated as of ATS v6.2.0. - -.. ts:cv:: CONFIG proxy.config.http.cache.fuzz.probability FLOAT 0.0 - :deprecated: - :reloadable: - :overridable: - - The probability that a refresh is made on a document during the fuzz time - specified in :ts:cv:`proxy.config.http.cache.fuzz.time`. - -.. note:: - - Previous versions of Apache |TS| defaulted this to 0.005 (0.5%). - This feature is deprecated as of ATS v6.2.0 - -.. ts:cv:: CONFIG proxy.config.http.cache.fuzz.min_time INT 0 - :deprecated: - :reloadable: - :overridable: - - Handles requests with a TTL less than :ts:cv:`proxy.config.http.cache.fuzz.time`. - It allows for different times to evaluate the probability of revalidation - for small TTLs and big TTLs. Objects with small TTLs will start "rolling the - revalidation dice" near the ``fuzz.min_time``, while objects with large TTLs - would start at ``fuzz.time``. A logarithmic-like function between determines - the revalidation evaluation start time (which will be between - ``fuzz.min_time`` and ``fuzz.time``). As the object gets closer to expiring, - the window start becomes more likely. By default this setting is not enabled, - but should be enabled any time you have objects with small TTLs. - -.. note:: - - These fuzzing options are marked as deprecated as of v6.2.0, and will be - removed for v7.0.0. Instead, we recommend looking at the new - :ts:cv:`proxy.config.http.cache.open_write_fail_action` configuration and - the features around thundering heard avoidance (see - :ref:`http-proxy-caching` for details). - Dynamic Content & Content Negotiation ===================================== @@ -2321,6 +2307,7 @@ all the different user-agent versions of documents it encounters. .. ts:cv:: CONFIG proxy.config.http.cache.open_read_retry_time INT 10 :reloadable: + :overridable: The number of milliseconds a cacheable request will wait before requesting the object from cache if an equivalent request is in flight. @@ -2483,7 +2470,7 @@ DNS .. ts:cv:: CONFIG proxy.config.dns.resolv_conf STRING /etc/resolv.conf - Allows to specify which ``resolv.conf`` file to use for finding resolvers. While the format of this file must be the same as the + Allows one to specify which ``resolv.conf`` file to use for finding resolvers. While the format of this file must be the same as the standard ``resolv.conf`` file, this option allows an administrator to manage the set of resolvers in an external configuration file, without affecting how the rest of the operating system uses DNS. @@ -2501,7 +2488,7 @@ DNS :reloadable: :overridable: - Indicates whether to use SRV records for orgin server lookup. + Indicates whether to use SRV records for origin server lookup. .. ts:cv:: CONFIG proxy.config.dns.dedicated_thread INT 0 @@ -2567,7 +2554,7 @@ HostDB For values above ``200000``, you must increase :ts:cv:`proxy.config.hostdb.max_size` by at least 44 bytes per entry. -.. ts:cv:: proxy.config.hostdb.round_robin_max_count INT 16 +.. ts:cv:: CONFIG proxy.config.hostdb.round_robin_max_count INT 16 The maximum count of DNS answers per round robin hostdb record. The default variable is 16. @@ -2590,7 +2577,7 @@ HostDB value become a minimum TTL. ===== ====================================================================== -.. ts:cv:: CONFIG proxy.config.hostdb.timeout INT 1440 +.. ts:cv:: CONFIG proxy.config.hostdb.timeout INT 86400 :units: seconds :reloadable: @@ -2636,7 +2623,7 @@ HostDB Set the file path for an external host file. If this is set (non-empty) then the file is presumed to be a hosts file in - the standard `host file format `_. + the standard . It is read and the entries there added to the HostDB. The file is periodically checked for a more recent modification date in which case it is reloaded. The interval is set with :ts:cv:`proxy.config.hostdb.host_file.interval`. @@ -2725,16 +2712,17 @@ HostDB Set the interval (in seconds) in which to re-query DNS regardless of TTL status. -.. ts:cv:: CONFIG proxy.config.hostdb.filename STRING "host.db" +.. ts:cv:: CONFIG proxy.config.hostdb.filename STRING host.db The filename to persist hostdb to on disk. -.. ts:cv:: CONFIG proxy.config.cache.hostdb.sync_frequency INT 120 +.. ts:cv:: CONFIG proxy.config.cache.hostdb.sync_frequency INT 0 - Set the frequency (in seconds) to sync hostdb to disk. + Set the frequency (in seconds) to sync hostdb to disk. If set to zero (default as of v9.0.0), we won't + sync to disk ever. Note: hostdb is syncd to disk on a per-partition basis (of which there are 64). - This means that the minumum time to sync all data to disk is :ts:cv:`proxy.config.cache.hostdb.sync_frequency` * 64 + This means that the minimum time to sync all data to disk is :ts:cv:`proxy.config.cache.hostdb.sync_frequency` * 64 Logging Configuration ===================== @@ -2777,25 +2765,7 @@ Logging Configuration .. note:: All files in the logging directory contribute to the space used, - even if they are not log files. In collation client mode, if - there is no local disk logging, or - :ts:cv:`proxy.config.log.max_space_mb_for_orphan_logs` is set - to a higher value than :ts:cv:`proxy.config.log.max_space_mb_for_logs`, - |TS| will take :ts:cv:`proxy.config.log.max_space_mb_for_orphan_logs` - for maximum allowed log space. - -.. ts:cv:: CONFIG proxy.config.log.max_space_mb_for_orphan_logs INT 25 - :units: megabytes - :reloadable: - - The amount of space allocated to the logging directory (in MB) if this node is acting as a collation client. - -.. note:: - - When max_space_mb_for_orphan_logs is take as the maximum allowed log space in the logging system, the same rule apply - to proxy.config.log.max_space_mb_for_logs also apply to proxy.config.log.max_space_mb_for_orphan_logs, ie: All files - in the logging directory contribute to the space used, even if they are not log files. you may need to consider this - when you enable full remote logging, and bump to the same size as proxy.config.log.max_space_mb_for_logs. + even if they are not log files. .. ts:cv:: CONFIG proxy.config.log.max_space_mb_headroom INT 1000 :units: megabytes @@ -2838,85 +2808,6 @@ Logging Configuration others, even if specified in the configuration file. Permissions for existing log files are not changed when the configuration is modified. -.. ts:cv:: LOCAL proxy.local.log.collation_mode INT 0 - :reloadable: - :deprecated: - - Set the log collation mode. - - ===== ====================================================================== - Value Effect - ===== ====================================================================== - ``0`` Log collation is disabled. - ``1`` This host is a log collation server. - ``2`` This host is a collation client and sends entries using standard - formats to the collation server. - ``3`` This host is a collation client and sends entries using the - traditional custom formats to the collation server. - ``4`` This host is a collation client and sends entries that use both the - standard and traditional custom formats to the collation server. - ===== ====================================================================== - - For information on sending custom formats to the collation server, - refer to :ref:`admin-logging-collating-custom-formats` and - :file:`logging.yaml`. - -.. note:: - - Log collation is a *deprecated* feature as of ATS v8.0.0, and will be - removed in ATS v9.0.0. Our recommendation is to use one of the many existing - log collection tools, such as Kafka, LogStash, FileBeat, Fluentd or even - syslog / syslog-ng. - -.. ts:cv:: CONFIG proxy.config.log.collation_host STRING NULL - :deprecated: - - The hostname of the log collation server. - -.. ts:cv:: CONFIG proxy.config.log.collation_port INT 8085 - :reloadable: - :deprecated: - - The port used for communication between the collation server and client. - -.. ts:cv:: CONFIG proxy.config.log.collation_secret STRING foobar - :reloadable: - :deprecated: - - The password used to validate logging data and prevent the exchange of unauthorized information when a collation server is being used. - -.. ts:cv:: CONFIG proxy.config.log.collation_host_tagged INT 0 - :reloadable: - :deprecated: - - When enabled (``1``), configures |TS| to include the hostname of the collation client that generated the log entry in each entry. - -.. ts:cv:: CONFIG proxy.config.log.collation_retry_sec INT 5 - :reloadable: - :deprecated: - - The number of seconds between collation server connection retries. - -.. ts:cv:: CONFIG proxy.config.log.collation_host_timeout INT 86390 - :deprecated: - - The number of seconds before inactivity time-out events for the host side. - This setting over-rides the default set with proxy.config.net.default_inactivity_timeout - for log collation connections. - - The default is set for 10s less on the host side to help prevent any possible race - conditions. If the host disconnects first, the client will see the disconnect - before its own time-out and re-connect automatically. If the client does not see - the disconnect, i.e., connection is "locked-up" for some reason, it will disconnect - when it reaches its own time-out and then re-connect automatically. - -.. ts:cv:: CONFIG proxy.config.log.collation_client_timeout INT 86400 - :deprecated: - - The number of seconds before inactivity time-out events for the client side. - This setting over-rides the default set with proxy.config.net.default_inactivity_timeout - for log collation connections. - .. ts:cv:: CONFIG proxy.config.log.rolling_enabled INT 1 :reloadable: @@ -3055,7 +2946,7 @@ Diagnostic Logging Configuration .. ts:cv:: CONFIG proxy.config.diags.debug.tags STRING http|dns - Each |TS| `diag` and `debug` level message is annotated with a subsytem tag. This configuration + Each |TS| `diag` and `debug` level message is annotated with a subsystem tag. This configuration contains an anchored regular expression that filters the messages based on the tag. The expressions are prefix matched which creates an implicit ``.*`` at the end. Therefore the default value ``http|dns`` will match tags such as ``http``, ``http_hdrs``, ``dns``, and ``dns_recv``. @@ -3063,7 +2954,7 @@ Diagnostic Logging Configuration Some commonly used debug tags are: ============ ===================================================== - Tag Subsytem usage + Tag Subsystem usage ============ ===================================================== dns DNS query resolution http_hdrs Logs the headers for HTTP requests and responses @@ -3221,17 +3112,17 @@ SSL Termination X25519:P-256:X448:P-521:P-384 - This configuration works with OpenSSL v1.1.1 and above. + This configuration works with OpenSSL v1.0.2 and above. .. ts:cv:: CONFIG proxy.config.ssl.client.groups_list STRING Configures the list of supported groups provided by OpenSSL which - |TS| will use for the "key_share" and "supported groups" extention + |TS| will use for the "key_share" and "supported groups" extension of TLSv1.3 connections. The value is a colon separated list of group NIDs or names, for example "P-521:P-384:P-256". For instructions, see "Groups" section of `TLS1.3 - OpenSSLWiki `_. - This configuration works with OpenSSL v1.1.1 and above. + This configuration works with OpenSSL v1.0.2 and above. .. ts:cv:: CONFIG proxy.config.ssl.TLSv1 INT 1 @@ -3245,6 +3136,10 @@ SSL Termination Enables (``1``) or disables (``0``) TLS v1.2. If not specified, enabled by default. [Requires OpenSSL v1.0.1 and higher] +.. ts:cv:: CONFIG proxy.config.ssl.TLSv1_3 INT 1 + + Enables (``1``) or disables (``0``) TLS v1.3. If not specified, enabled by default. [Requires OpenSSL v1.1.1 and higher] + .. ts:cv:: CONFIG proxy.config.ssl.client.certification_level INT 0 Sets the client certification level: @@ -3325,12 +3220,13 @@ SSL Termination The filename of the certificate authority that client certificates will be verified against. -.. ts:cv:: CONFIG proxy.config.ssl.server.ticket_key.filename STRING ssl_ticket.key +.. ts:cv:: CONFIG proxy.config.ssl.server.ticket_key.filename STRING NULL The filename of the default and global ticket key for SSL sessions. The location is relative to the :ts:cv:`proxy.config.ssl.server.cert.path` directory. One way to generate this would be to run ``head -c48 /dev/urandom | openssl enc -base64 | head -c48 > file.ticket``. Also - note that OpenSSL session tickets are sensitive to the version of the ca-certificates. + note that OpenSSL session tickets are sensitive to the version of the ca-certificates. Once the + file is changed with new tickets, use :option:`traffic_ctl config reload` to begin using them. .. ts:cv:: CONFIG proxy.config.ssl.servername.filename STRING ssl_server_name.yaml @@ -3365,7 +3261,7 @@ SSL Termination ``0`` Disables the session cache entirely. ``1`` Enables the session cache using OpenSSL's implementation. ``2`` Default. Enables the session cache using |TS|'s implementation. This - implentation should perform much better than the OpenSSL + implementation should perform much better than the OpenSSL implementation. ===== ====================================================================== @@ -3451,50 +3347,37 @@ SSL Termination See :ref:`admin-performance-timeouts` for more discussion on |TS| timeouts. -.. ts:cv:: CONFIG proxy.config.ssl.wire_trace_enabled INT 0 - - When enabled this turns on wire tracing of SSL connections that meet - the conditions specified by wire_trace_percentage, wire_trace_addr - and wire_trace_server_name. - -.. ts:cv:: CONFIG proxy.config.ssl.wire_trace_percentage INT 0 - - This specifies the percentage of traffic meeting the other wire_trace - conditions to be traced. - -.. ts:cv:: CONFIG proxy.config.ssl.wire_trace_addr STRING NULL - - This specifies the client IP for which wire_traces should be printed. - -.. ts:cv:: CONFIG proxy.config.ssl.wire_trace_server_name STRING NULL - - This specifies the server name for which wire_traces should be printed. - Client-Related Configuration ---------------------------- .. ts:cv:: CONFIG proxy.config.ssl.client.verify.server.policy STRING DISABLED :reloadable: + :overridable: Configures |TS| to verify the origin server certificate with the Certificate Authority (CA). This configuration takes a value of :code:`DISABLED`, :code:`PERMISSIVE`, or :code:`ENFORCED` - You can override this global setting on a per domain basis in the ssl_servername.yaml file using the :ref:`verify_server_policy attribute`. + You can override this global setting on a per domain basis in the ssl_server_name.yaml file using the :ref:`verify_server_policy attribute`. + + You can also override via the conf_remap plugin. Those changes will take precedence over the changes in ssl_server_name.yaml. :code:`DISABLED` Server Certificate will not be verified :code:`PERMISSIVE` - Certificate will be verified and the connection will not be established if verification fails. -:code:`ENFORCED` The provided certificate will be verified and the connection will be established irrespective of the verification result. If verification fails the name of the server will be logged. +:code:`ENFORCED` + Certificate will be verified and the connection will not be established if verification fails. .. ts:cv:: CONFIG proxy.config.ssl.client.verify.server.properties STRING ALL :reloadable: + :overridable: Configures |TS| for what the default verify callback should check during origin server verification. You can override this global setting on a per domain basis in the ssl_servername.yaml file using the :ref:`verify_server_properties attribute`. + You can also override via the conf_remap plugin. Those changes will take precedence over the changes in ssl_server_name.yaml. + :code:`NONE` Check nothing in the standard callback. Rely entirely on plugins to check the certificate. :code:`SIGNATURE` @@ -3522,6 +3405,7 @@ Client-Related Configuration .. ts:cv:: CONFIG proxy.config.ssl.client.cert.filename STRING NULL :reloadable: + :overridable: The filename of SSL client certificate installed on |TS|. @@ -3533,6 +3417,7 @@ Client-Related Configuration .. ts:cv:: CONFIG proxy.config.ssl.client.private_key.filename STRING NULL :reloadable: + :overridable: The filename of the |TS| private key. Change this variable only if the private key is not located in the |TS| SSL @@ -3546,15 +3431,25 @@ Client-Related Configuration file. .. ts:cv:: CONFIG proxy.config.ssl.client.CA.cert.filename STRING NULL + :reloadable: + :overridable: The filename of the certificate authority against which the origin server will be verified. .. ts:cv:: CONFIG proxy.config.ssl.client.CA.cert.path STRING NULL + :reloadable: Specifies the location of the certificate authority file against which the origin server will be verified. +.. ts:cv:: CONFIG proxy.config.ssl.client.sni_policy STRING NULL + :overridable: + + Indicate how the SNI value for the TLS connection to the origin is selected. By default it is + `host` which means the host header field value is used for the SNI. If `remap` is specified, the + remapped origin name is used for the SNI value. + .. ts:cv:: CONFIG proxy.config.ssl.client.SSLv3 INT 0 Enables (``1``) or disables (``0``) SSLv3 in the ATS client context. Disabled by default @@ -3571,6 +3466,10 @@ Client-Related Configuration Enables (``1``) or disables (``0``) TLSv1_2 in the ATS client context. If not specified, enabled by default +.. ts:cv:: CONFIG proxy.config.ssl.client.TLSv1_3 INT 1 + + Enables (``1``) or disables (``0``) TLSv1_3 in the ATS client context. If not specified, enabled by default + .. ts:cv:: CONFIG proxy.config.ssl.async.handshake.enabled INT 0 Enables the use of openssl async job during the TLS handshake. Traffic @@ -3617,6 +3516,14 @@ OCSP Stapling Configuration Update period (in seconds) for stapling caches. +.. ts:cv:: CONFIG proxy.config.ssl.ocsp.response.path STRING NULL + + The directory path of the prefetched OCSP stapling responses. Change this + variable only if you intend to use and administratively maintain + prefetched OCSP stapling responses. All stapling responses listed in + :file:`ssl_multicert.config` will be loaded relative to this + path. + HTTP/2 Configuration ==================== @@ -3664,7 +3571,7 @@ HTTP/2 Configuration The maximum size of the header compression table used to decode header blocks. -.. ts:cv:: CONFIG proxy.config.http2.max_header_list_size INT 4294967295 +.. ts:cv:: CONFIG proxy.config.http2.max_header_list_size INT 131072 :reloadable: This advisory setting informs a peer of the maximum size of header list @@ -3718,6 +3625,13 @@ HTTP/2 Configuration HTTP/2 connection to avoid duplicate pushes on the same connection. If the maximum number is reached, new entries are not remembered. +.. ts:cv:: CONFIG proxy.config.http2.stream_error_rate_threshold FLOAT 0.1 + :reloadable: + + This is the maximum stream error rate |TS| allows on an HTTP/2 connection. + |TS| gracefully closes connections that have stream error rates above this + setting by sending GOAWAY frames. + Plug-in Configuration ===================== diff --git a/doc/admin-guide/files/remap.config.en.rst b/doc/admin-guide/files/remap.config.en.rst index bbd5f6dfccd..a37aaff3c54 100644 --- a/doc/admin-guide/files/remap.config.en.rst +++ b/doc/admin-guide/files/remap.config.en.rst @@ -87,7 +87,7 @@ Traffic Server recognizes three space-delimited fields: ``type``, notify the browser of the URL change for the current request only (by returning an HTTP status code 307). - .. note: use the ``regex_`` prefix to indicate that the line has a regular expression (regex). + .. note:: use the ``regex_`` prefix to indicate that the line has a regular expression (regex). .. _remap-config-format-target: @@ -107,6 +107,8 @@ Traffic Server recognizes three space-delimited fields: ``type``, where ``scheme`` is ``http``, ``https``, ``ws`` or ``wss``. + .. note:: A remap rule for requests that upgrade from HTTP to WebSocket still require a remap rule with the ``ws`` or ``wss`` scheme. + .. _remap-config-precedence: @@ -117,7 +119,7 @@ Remap rules are not processed top-down, but based on an internal priority. Once these rules are executed we pick the first match based on configuration file parse order. -1. ``map_with_recv_port`` and ```regex_map_with_recv_port``` +1. ``map_with_recv_port`` and ``regex_map_with_recv_port`` #. ``map`` and ``regex_map`` and ``reverse_map`` #. ``redirect`` and ``redirect_temporary`` #. ``regex_redirect`` and ``regex_redirect_temporary`` @@ -415,7 +417,7 @@ Acl Filters Acl filters can be created to control access of specific remap lines. The markup is very similar to that of :file:`ip_allow.config`, with slight changes to -accomodate remap markup +accommodate remap markup Examples -------- diff --git a/doc/admin-guide/files/ssl_multicert.config.en.rst b/doc/admin-guide/files/ssl_multicert.config.en.rst index e8112a78cb7..54a391d3d50 100644 --- a/doc/admin-guide/files/ssl_multicert.config.en.rst +++ b/doc/admin-guide/files/ssl_multicert.config.en.rst @@ -73,7 +73,7 @@ ssl_cert_name=FILENAME[,FILENAME ...] dest_ip=ADDRESS (optional) The IP (v4 or v6) address that the certificate should be presented on. This is now only used as a fallback in the case that the TLS - SubjectNameIndication extension is not supported. If *ADDRESS* is + ServerNameIndication extension is not supported. If *ADDRESS* is `*`, the corresponding certificate will be used as the global default fallback if no other match can be made. The address may contain a port specifier, in which case the corresponding certificate @@ -93,6 +93,14 @@ ssl_ca_name=FILENAME (optional) the certificate chain. *FILENAME* is resolved relative to the :ts:cv:`proxy.config.ssl.CA.cert.path` configuration variable. +ssl_ocsp_name=FILENAME (optional) + The name of the file containing the prefetched OCSP stapling response + for this certificate. This field can be omitted to let trafficserver + fetch OCSP responses dynamically. Otherwise, when included, the administrator is + responsible for updating the file's content. *FILENAME* is resolved + relative to the :ts:cv:`proxy.config.ssl.ocsp.response.path` + configuration variable. + ssl_ticket_enabled=1|0 (optional) Enable RFC 5077 stateless TLS session tickets. To support this, OpenSSL should be upgraded to version 0.9.8f or higher. This diff --git a/doc/admin-guide/files/ssl_server_name.yaml.en.rst b/doc/admin-guide/files/ssl_server_name.yaml.en.rst index 32ab6af2a40..6f2f1a02708 100644 --- a/doc/admin-guide/files/ssl_server_name.yaml.en.rst +++ b/doc/admin-guide/files/ssl_server_name.yaml.en.rst @@ -66,20 +66,34 @@ verify_origin_server Deprecated. Use verify_server_policy and verify_serve By default this is :ts:cv:`proxy.config.ssl.client.verify.server`. verify_client One of the values :code:`NONE`, :code:`MODERATE`, or :code:`STRICT`. + If ``NONE`` is specified, |TS| requests no certificate. If ``MODERATE`` is specified + |TS| will verify a certificate that is presented by the client, but it will not + fail the TLS handshake if new certificate is presented. If ``STRICT`` is specified + the client must resent a certificate during the TLS handshake. By default this is :ts:cv:`proxy.config.ssl.client.certification_level`. +valid_tls_versions_in This specifies the list of TLS protocols that will be offered to user agents during + the TLS negotiaton. This replaces the global settings in :ts:cv:`proxy.config.ssl.TLSv1`, + :ts:cv:`proxy.config.ssl.TLSv1_1`, :ts:cv:`proxy.config.ssl.TLSv1_2`, + and :ts:cv:`proxy.config.ssl.TLSv1_3`. The potential values are TLSv1, TLSv1_1, TLSv1_2, and + TLSv1_3. You must list all protocols that |TS| should offer to the client when using + this key. This key is only valid for openssl 1.1.0 and later. Older versions of openssl do not + provide a hook early enough to update the SSL object. It is a syntax error for |TS| built + against earlier versions. + + client_cert The file containing the client certificate to use for the outbound connection. - If this is relative it is relative to the path in - :ts:cv:`proxy.config.ssl.server.cert.path`. If not set + If this is relative, it is relative to the path in + :ts:cv:`proxy.config.ssl.client.cert.path`. If not set :ts:cv:`proxy.config.ssl.client.cert.filename` is used. client_key The file containing the client private key that corresponds to the certificate for the outbound connection. - If this is relative it is relative to the path in - :ts:cv:`proxy.config.ssl.server.private_key.path`. If not set, + If this is relative, it is relative to the path in + :ts:cv:`proxy.config.ssl.client.private_key.path`. If not set, |TS| tries to use a private key in client_cert. Otherwise, :ts:cv:`proxy.config.ssl.client.private_key.filename` is used. @@ -91,6 +105,16 @@ disable_h2 :code:`true` or :code:`false`. for proxy ports on which HTTP/2 is not enabled. tunnel_route Destination as an FQDN and port, separated by a colon ``:``. + + + This will forward all traffic to the specified destination without first terminating + the incoming TLS connection. + +forward_route Destination as an FQDN and port, separated by a colon ``:``. + + This is similar to tunnel_route, but it terminates the TLS connection and forwards the + decrypted traffic. |TS| will not interpret the decrypted data, so the contents do not + need to be HTTP. ========================= ============================================================================== Client verification, via ``verify_client``, correponds to setting diff --git a/doc/admin-guide/files/storage.config.en.rst b/doc/admin-guide/files/storage.config.en.rst index 943dc94e22a..1945dc085ab 100644 --- a/doc/admin-guide/files/storage.config.en.rst +++ b/doc/admin-guide/files/storage.config.en.rst @@ -48,6 +48,10 @@ partitions. :arg:`volume` and arg:`seed` are optional. If the :arg:`id` option is used every use must have a unique value for :arg:`string`. +.. note:: + + Any change to this files can (and almost always will) invalidate the existing cache in its entirety. + You can use any partition of any size. For best performance: - Use raw disk partitions. @@ -92,7 +96,7 @@ which will effectively clear most of the cache. This can be problem when drives reboot causes the path names to change. The :arg:`id` option can be used to create a fixed string that an administrator can use to keep the -assignment table consistent by maintaing the mapping from physical device to base string even in the presence of hardware changes and failures. +assignment table consistent by maintaining the mapping from physical device to base string even in the presence of hardware changes and failures. Examples ======== diff --git a/doc/admin-guide/files/volume.config.en.rst b/doc/admin-guide/files/volume.config.en.rst index ebae6d4e12a..7fdfc8e3381 100644 --- a/doc/admin-guide/files/volume.config.en.rst +++ b/doc/admin-guide/files/volume.config.en.rst @@ -52,6 +52,11 @@ do not allocate all the disk space in the cache, then the extra disk space is not used. You can use the extra space later to create new volumes without deleting and clearing the existing volumes. +.. important:: + + Changing this file to add, remove or modify volumes effectively invalidates + the cache. + Examples ======== diff --git a/doc/admin-guide/installation/index.en.rst b/doc/admin-guide/installation/index.en.rst index 48cbd90348a..07caf4c12f0 100644 --- a/doc/admin-guide/installation/index.en.rst +++ b/doc/admin-guide/installation/index.en.rst @@ -105,7 +105,6 @@ development tools and libraries installed: - gcc (>= 4.3 or clang > 3.0) - GNU make - openssl (libssl-dev for Ubuntu 16.04) -- tcl (tcl-8.6-dev for Ubuntu 16.04) - pcre (libpcre3-dev for Ubuntu 16.04) - libcap - flex (for TPROXY) diff --git a/doc/admin-guide/introduction.en.rst b/doc/admin-guide/introduction.en.rst index f6b40ba3797..06effa3c9ae 100644 --- a/doc/admin-guide/introduction.en.rst +++ b/doc/admin-guide/introduction.en.rst @@ -30,102 +30,98 @@ nightmare for IT professionals as they struggle with overloaded servers and congested networks. It can be challenging to consistently and reliably accommodate society’s growing data demands. -Traffic Server is a high-performance web proxy cache that improves +|TS| is a high-performance web proxy cache that improves network efficiency and performance by caching frequently-accessed information at the edge of the network. This brings content physically closer to end users, while enabling faster delivery and reduced -bandwidth use. Traffic Server is designed to improve content delivery +bandwidth use. |TS| is designed to improve content delivery for enterprises, Internet service providers (ISPs), backbone providers, and large intranets by maximizing existing and available bandwidth. -Traffic Server Deployment Options -================================= +|TS| Deployment Options +======================= -To best suit your needs, Traffic Server can be deployed in several ways: +To best suit your needs, |TS| can be deployed in several ways: - As a web proxy cache - As a reverse proxy - In a cache hierarchy -The following sections provide a summary of these Traffic Server +The following sections provide a summary of these |TS| deployment options. -Traffic Server as a Web Proxy Cache ------------------------------------ +|TS| as a Web Proxy Cache +------------------------- -As a web proxy cache, Traffic Server receives user requests for web +As a web proxy cache, |TS| receives user requests for web content as those requests travel to the destined web server (origin -server). If Traffic Server contains the requested content, then it +server). If |TS| contains the requested content, then it serves the content directly. If the requested content is not available -from cache, then Traffic Server acts as a proxy: it obtains the content +from cache, then |TS| acts as a proxy: it obtains the content from the origin server on the user’s behalf and also keeps a copy to satisfy future requests. -Traffic Server provides explicit proxy caching, in which the user’s +|TS| provides explicit proxy caching, in which the user’s client software must be configured to send requests directly to Traffic Server. Explicit proxy caching is described in the :ref:`explicit-proxy-caching` chapter. -Traffic Server can also be employed as a transparent caching proxy server, in +|TS| can also be employed as a transparent caching proxy server, in which the client software needs no special configuration or even knowledge of the proxy's existence. This setup is described in the :ref:`transparent-proxy` section. -Traffic Server as a Reverse Proxy ---------------------------------- +|TS| as a Reverse Proxy +----------------------- -As a reverse proxy, Traffic Server is configured to be the origin server +As a reverse proxy, |TS| is configured to be the origin server to which the user is trying to connect (typically, the origin server’s -advertised hostname resolves to Traffic Server, which acts as the real +advertised hostname resolves to |TS|, which acts as the real origin server). The reverse proxy feature is also called server acceleration. Reverse proxy is described in more detail in :ref:`reverse-proxy-and-http-redirects`. -Traffic Server in a Cache Hierarchy ------------------------------------ +|TS| in a Cache Hierarchy +------------------------- -Traffic Server can participate in flexible cache hierarchies, in which +|TS| can participate in flexible cache hierarchies, in which Internet requests not fulfilled from one cache are routed to other regional caches, thereby leveraging the contents and proximity of nearby -caches. In a hierarchy of proxy servers, Traffic Server can act either -as a parent or a child cache to other Traffic Server systems or to +caches. In a hierarchy of proxy servers, |TS| can act either +as a parent or a child cache to other |TS| systems or to similar caching products. -Deployment Limitations ----------------------- +|TS| as a Load Balancer +----------------------- -There are a number of deployment options that Traffic Server does not support -right out of the box. Such functionality may be implemented in a plugin, but -in some cases Traffic Server's internal APIs or architectural restrictions -won't make it easy: +|TS| can act as a layer 7 HTTP load balancer distributing requests across +several servers. It can choose the next hop server using request attributes +like the Host: header, URL attributes, scheme, method, and client IP address. +It has a few selection strategies in place like weighted round robin, and +URL consistent hashing. -* Load Balancing - note that there is an experimental plugin for this, - :ref:`admin-plugins-balancer`. +|TS| Components +=============== -.. XXX needs re-work: the leadin states there's "a number" of scenarios, and then only lists one -- one's a number, but not much of a list - -Traffic Server Components -========================= - -Traffic Server consists of several components that work together to form +|TS| consists of several components that work together to form a web proxy cache you can easily monitor and configure. -The Traffic Server Cache ------------------------- +The |TS| Cache +-------------- -The Traffic Server cache consists of a high-speed object database called +The |TS| cache consists of a high-speed object database called the *object store*. The object store indexes objects according to URLs and associated headers. Using sophisticated object management, the object store can cache alternate versions of the same object (perhaps in a different language or encoding type). It can also efficiently store very small and very large objects, thereby minimizing wasted space. When the -cache is full, Traffic Server removes stale data to ensure that the most +cache is full, |TS| removes stale data to ensure that the most requested objects are readily available and fresh. -Traffic Server is designed to tolerate total disk failures on any of the -cache disks. If the disk fails completely, then Traffic Server marks the +|TS| is designed to tolerate total disk failures on any of the +cache disks. If the disk fails completely, then |TS| marks the entire disk as corrupt and continues to use remaining disks. If all of -the cache disks fail, then Traffic Server switches to proxy-only mode. +the cache disks fail, then |TS| switches to proxy-only mode. You can partition the cache to reserve a certain amount of disk space for storing data for specific protocols and origin servers. For more information about the cache, see :ref:`http-proxy-caching`. @@ -133,7 +129,7 @@ information about the cache, see :ref:`http-proxy-caching`. The RAM Cache ------------- -Traffic Server maintains a small RAM cache that contains extremely +|TS| maintains a small RAM cache that contains extremely popular objects. This RAM cache serves the most popular objects as fast as possible and reduces load on disks, especially during temporary traffic peaks. You can configure the RAM cache size to suit your needs. @@ -142,8 +138,8 @@ For detailed information, refer to :ref:`changing-the-size-of-the-ram-cache`. The Host Database ----------------- -The Traffic Server host database stores the domain name server (DNS) -entries of origin servers to which Traffic Server connects to fulfill +The |TS| host database stores the domain name server (DNS) +entries of origin servers to which |TS| connects to fulfill user requests. This information is used to adapt future protocol interactions and optimize performance. Along with other information, the host database tracks: @@ -159,26 +155,26 @@ host database tracks: The DNS Resolver ---------------- -Traffic Server includes a fast, asynchronous DNS resolver to streamline -conversion of hostnames to IP addresses. Traffic Server implements the +|TS| includes a fast, asynchronous DNS resolver to streamline +conversion of hostnames to IP addresses. |TS| implements the DNS resolver natively by directly issuing DNS command packets rather than relying on slower, conventional resolver libraries. Since many DNS queries can be issued in parallel and a fast DNS cache maintains popular bindings in memory, DNS traffic is reduced. -Traffic Server Processes ------------------------- +|TS| Processes +-------------- -Traffic Server contains three processes that work together to serve +|TS| contains three processes that work together to serve requests and manage, control, and monitor the health of the system. - The :program:`traffic_server` process is the transaction processing engine - of Traffic Server. It is responsible for accepting connections, + of |TS|. It is responsible for accepting connections, processing protocol requests, and serving documents from the cache or origin server. - The :program:`traffic_manager` process is the command and control facility - of the Traffic Server, responsible for launching, monitoring, and + of the |TS|, responsible for launching, monitoring, and reconfiguring the :program:`traffic_server` process. The :program:`traffic_manager` process is also responsible for the proxy autoconfiguration port, the statistics interface, and virtual IP failover. @@ -191,51 +187,51 @@ requests and manage, control, and monitor the health of the system. first-served order. This connection queueing shields users from any server restart downtime. -The figure below illustrates the Traffic Server processes. +The figure below illustrates the |TS| processes. .. figure:: ../static/images/admin/process.jpg :align: center - :alt: Illustration of the Traffic Server Processes + :alt: Illustration of the |TS| Processes - Illustration of the Traffic Server Processes + Illustration of the |TS| Processes Administration Tools -------------------- -Traffic Server offers the following administration options: +|TS| offers the following administration options: - The :program:`traffic_ctl` command-line interface is a - text-based interface from which you can monitor Traffic Server performance - and network traffic, as well as configure the Traffic Server system. + text-based interface from which you can monitor |TS| performance + and network traffic, as well as configure the |TS| system. -- Various configuration files enable you to configure Traffic Server +- Various configuration files enable you to configure |TS| through a simple file-editing and signal-handling interface. Any changes you make through :program:`traffic_ctl` are automatically made to the configuration files as well. - Finally, there is a clean C API which can be put to good use from a - multitude of languages. The Traffic Server Admin Client demonstrates + multitude of languages. The |TS| Admin Client demonstrates this for Perl. Traffic Analysis Options ======================== -Traffic Server provides several options for network traffic analysis and +|TS| provides several options for network traffic analysis and monitoring: - :program:`traffic_ctl` enables you to collect and process statistics obtained from network traffic information. - Transaction logging enables you to record information (in a log file) - about every request Traffic Server receives and every error it + about every request |TS| receives and every error it detects. By analyzing the log files, you can determine how many - clients used the Traffic Server cache, how much information each of + clients used the |TS| cache, how much information each of them requested, and what pages were most popular. You can also see why a particular transaction was in error and what state the Traffic Server was in at a particular time. For example, you can see that - Traffic Server was restarted. + |TS| was restarted. - Traffic Server supports several standard log file formats, such as + |TS| supports several standard log file formats, such as Squid and Netscape, and its own custom format. You can analyze the standard format log files with off-the-shelf analysis packages. To help with log file analysis, you can separate log files so that they @@ -244,37 +240,37 @@ monitoring: |TS| event and error logging, monitoring, and analysis is covered in greater detail in :ref:`admin-monitoring`. -Traffic Server Security Options -=============================== +|TS| Security Options +===================== -Traffic Server provides numerous options that enable you to establish -secure communication between the Traffic Server system and other +|TS| provides numerous options that enable you to establish +secure communication between the |TS| system and other computers on the network. Using the security options, you can do the following: -- Control client access to the Traffic Server proxy cache. +- Control client access to the |TS| proxy cache. -- Configure Traffic Server to use multiple DNS servers to match your - site's security configuration. For example, Traffic Server can use +- Configure |TS| to use multiple DNS servers to match your + site's security configuration. For example, |TS| can use different DNS servers, depending on whether it needs to resolve hostnames located inside or outside a firewall. This enables you to keep your internal network configuration secure while continuing to provide transparent access to external sites on the Internet. -- Configure Traffic Server to verify that clients are authenticated - before they can access content from the Traffic Server cache. +- Configure |TS| to verify that clients are authenticated + before they can access content from the |TS| cache. - Secure connections in reverse proxy mode between a client and Traffic - Server, and Traffic Server and the origin server, using the SSL + Server, and |TS| and the origin server, using the SSL termination option. - Control access via SSL (Secure Sockets Layer). -Traffic Server security options are described in more detail in +|TS| security options are described in more detail in :ref:`admin-security`. -Tuning Traffic Server -===================== +Tuning |TS| +=========== Finally, this last chapter on :ref:`performance-tuning` discusses the vast number of options that allow administrators to optimally tune Apache Traffic diff --git a/doc/admin-guide/layer-4-routing.en.rst b/doc/admin-guide/layer-4-routing.en.rst index f81a56f7e00..2355ecd2feb 100644 --- a/doc/admin-guide/layer-4-routing.en.rst +++ b/doc/admin-guide/layer-4-routing.en.rst @@ -28,7 +28,7 @@ accomplished by examining the initial data from the inbound connection to decide destination. The initial data is then sent to the destination and subsequently |TS| forwards all data read on one connection to the other and vice versa. -.. image:: ../uml/images/l4-basic-sequence.svg +.. figure:: ../uml/images/l4-basic-sequence.svg :align: center In this way it acts similary to `nc `__. @@ -116,7 +116,7 @@ In addition to this, in the :file:`records.config` file, edit the following vari The sequence of network activity for a Client connecting to ``service-2`` is -.. image:: ../uml/images/l4-sni-routing-seq.svg +.. figure:: ../uml/images/l4-sni-routing-seq.svg :align: center Note the destination for the outbound TCP connection and the HTTP ``CONNECT`` is the same. If this diff --git a/doc/admin-guide/logging/destinations.en.rst b/doc/admin-guide/logging/destinations.en.rst index f8521dea2b9..5b70913ad64 100644 --- a/doc/admin-guide/logging/destinations.en.rst +++ b/doc/admin-guide/logging/destinations.en.rst @@ -31,8 +31,7 @@ Two classes of destinations are provided by |TS| currently: local and remote. Local logging involves storing log data onto filesystems locally mounted on the same system as the |TS| processes themselves and are covered below in :ref:`admin-logging-destinations-local`, while remote logging options involving -:manpage:`syslog` and built-in |TS| log collation, are covered below in -:ref:`admin-logging-destinations-remote`. +:manpage:`syslog`, are covered below in :ref:`admin-logging-destinations-remote`. .. _admin-logging-destinations-local: @@ -44,9 +43,8 @@ Local Logging Log Directory Configuration --------------------------- -All local logging output (including incoming collation logs on a |TS| instance -configured as a :ref:`admin-logging-collation-server`) are stored within a -single base directory. Individual log file configurations may optionally append +All local logging output is stored within a single base directory. +Individual log file configurations may optionally append subdirectories to this base path. This location is adjusted with :ts:cv:`proxy.config.log.logfile_dir` in :file:`records.config`. @@ -196,132 +194,3 @@ system and emergency logs. Sending custom event or transaction error logs to syslog is not directly supported. You may use external log aggregation tools, such as Logstash, to accomplish this by having them handle the ingestion of |TS| local log files and forwarding to whatever receivers you wish. - -.. _admin-logging-collation: - -Log Collation -------------- - -.. note:: - - Log collation is a *deprecated* feature as of ATS v8.0.0, and will be - removed in ATS v9.0.0. Our recommendation is to use one of the many existing - log collection tools, such as Kafka, LogStash, FileBeat, Fluentd or even - syslog / syslog-ng (see above). - -|TS| offers remote log shipping natively through the log collation feature, -which allows one or more |TS| instances handling regular traffic to transmit -their log data to one or more |TS| instances acting as collation servers. - -This allows you to centralize your |TS| logging for (potentially) easier -analysis and reporting in environments with many |TS| instances. Collation -servers may be |TS| instance running a stripped down configuration aimed -at log collation only (and omitting any configuration for actual traffic -proxying or caching). - -When a |TS| node generates a buffer of event log entries, it first determines -if it is the collation server or a collation client. The collation server node -writes all log buffers to its local disk (as per its :file:`logging.yaml` -configuration), just as it would if log collation was not enabled. The -collation client nodes prepare their log buffers for transfer across the -network and send the buffers to the configured log collation server. - -If log clients cannot contact their log collation server, then they write their -log buffers to their local disks, into orphan log files. Orphaned log files -require manual collation. - -.. important:: - - Log collation can have an impact on network performance. Because all nodes - are forwarding their log data buffers to the single collation server, a - bottleneck can occur. In addition, collated log files contain timestamp - information for each entry, but entries in the files do not appear in strict - chronological order. You may want to sort collated log files before doing - analysis. - -.. _admin-logging-collation-client: - -Collation Client -~~~~~~~~~~~~~~~~ - -To configure a |TS| node to be a collation client, follow the steps below. - -#. In the :file:`records.config` file, edit the following variables: - - - :ts:cv:`proxy.local.log.collation_mode`: ``2`` to configure this node as - log collation client and send standard formatted log entries to the - collation server. For custom log entries, see :file:`logging.yaml`. - - :ts:cv:`proxy.config.log.collation_host` - - :ts:cv:`proxy.config.log.collation_port` - - :ts:cv:`proxy.config.log.collation_secret` - - :ts:cv:`proxy.config.log.collation_host_tagged` - - :ts:cv:`proxy.config.log.max_space_mb_for_orphan_logs` - -#. Run the command :option:`traffic_ctl config reload` to apply the configuration - changes. - -.. note:: - - If you modify the collation port or secret after connections between the - collation server and collation clients have been established, you must - restart |TS| on all nodes. - -.. _admin-logging-collation-server: - -Collation Server -~~~~~~~~~~~~~~~~ - -To configure a |TS| node to be a collation server, perform the following -configuration adjustments in :file:`records.config`: - -#. Set :ts:cv:`proxy.local.log.collation_mode` to ``1`` to indicate this node - will be a server. :: - - CONFIG proxy.local.log.collation_mode INT 1 - -#. Configure the port on which the server will listen to incoming collation - transfers from clients, using :ts:cv:`proxy.config.log.collation_port`. If - omitted, this defaults to port ``8085``. :: - - CONFIG proxy.config.log.collation_port INT 8085 - -#. Configure the shared secret used by collation clients to authenticate their - sessions, using :ts:cv:`proxy.config.log.collation_secret`. :: - - CONFIG proxy.config.log.collation_secret STRING seekrit - -#. Run the command :option:`traffic_ctl config reload` to apply the configuration - changes. - -.. note:: - - If you modify the collation port or secret after connections between the - collation server and collation clients have been established, you must - restart |TS| on all nodes. - -.. _admin-logging-collating-custom-formats: - -Collating Custom Logs -~~~~~~~~~~~~~~~~~~~~~ - -If you use custom event log files, then you must edit :file:`logging.yaml`, -in addition to configuring a collation server and collation clients. - -To collate custom event log files: - -#. On each collation client, edit :file:`logging.yaml` and add the - ``CollationHosts`` attribute to the relevant logs. For example, adding two - collation hosts to an ASCII log that uses the Squid format would look like: - - .. code:: yaml - - logs: - - mode: ascii - format: squid - filename: squid - collationhosts: - - 192.168.1.100:4567 - - 192.168.1.101:4567 - -#. Run the command :option:`traffic_ctl config reload` to restart |TS|. - diff --git a/doc/admin-guide/logging/formatting.en.rst b/doc/admin-guide/logging/formatting.en.rst index fce1edd8fba..bb5ccb79760 100644 --- a/doc/admin-guide/logging/formatting.en.rst +++ b/doc/admin-guide/logging/formatting.en.rst @@ -45,7 +45,7 @@ The return value from the ``format`` function is the log format object which may then be supplied to the appropriate ``log.*`` functions that define your logging destinations. -A very simple exampe, which contains only the timestamp of when the event began +A very simple example, which contains only the timestamp of when the event began and the canonical URL of the request, would look like: .. code:: yaml @@ -89,7 +89,6 @@ down into the following broad categories for (hopefully) easier reference: - :ref:`admin-logging-fields-methods` - :ref:`admin-logging-fields-ids` - :ref:`admin-logging-fields-lengths` -- :ref:`admin-logging-fields-collation` - :ref:`admin-logging-fields-network` - :ref:`admin-logging-fields-plugin` - :ref:`admin-logging-fields-proto` @@ -233,7 +232,7 @@ Error Code The log fields of error code which is triggered session close or transaction close. The first byte of this field indicates that the error code is session level (``S``) or transaction level (``T``). -When no error code is received or transmitted, these fileds are ``-``. +When no error code is received or transmitted, these fields are ``-``. For HTTP/2, error code are described in RFC 7540 section 7. ===== =============== ========================================================= @@ -274,6 +273,11 @@ HTTP Headers .. _epsh: .. _essh: .. _ecssh: +.. _cqah: +.. _pqah: +.. _psah: +.. _ssah: +.. _cssah: The following log tables provide access to the values of specified HTTP headers from each phase of the transaction lifecycle. Unlike many of the other log @@ -320,6 +324,33 @@ ssh essh cssh ecssh ============== =================== +It is also possible to log all of the headers in a transaction message with a +single field. For each original original field, there is a variant which ends in +``ah`` rather than ``h``, as shown here: + +============== =================== +Original Field All Headers Variant +============== =================== +cqh cqah +pqh pqah +psh psah +ssh ssah +cssh cssah +============== =================== + +No particular header is specified when using these variants, for example:: + + Format = '%' + +The output generated by these fields has the pattern:: + + {{{tag1}:{value1}}{{tag2}:{value2}}...} + +(The size of some messages may exceed internal buffer capacity. This may +result in the value of the last header being truncated, in which case, the +value will end with ``...}``. This may also result in the ommission of +entire tag/value pairs.) + .. _admin-logging-fields-methods: HTTP Methods @@ -415,28 +446,6 @@ ssql Origin Response Content body and header length combined of the origin server response to |TS|. ===== ====================== ================================================== -.. _admin-logging-fields-collation: - -Log Collation -~~~~~~~~~~~~~ - -.. _phn: -.. _phi: - -Logging fields related to :ref:`admin-logging-collation`. - -===== ====== ================================================================== -Field Source Description -===== ====== ================================================================== -phn Proxy Hostname of the |TS| node which generated the collated log entry. -phi Proxy IP of the |TS| node which generated the collated log entry. -===== ====== ================================================================== - -.. note:: - - Log collation is a *deprecated* feature as of ATS v8.0.0, and will be - removed in ATS v9.0.0. - .. _admin-logging-fields-network: Network Addresses, Ports, and Interfaces @@ -496,20 +505,25 @@ Plugin Details .. _piid: .. _pitag: +.. _cqint: Logging fields which may be used to obtain details of plugins involved in the transaction. -===== ============ ============================================================ -Field Source Description -===== ============ ============================================================ -piid Proxy Plugin Plugin ID for the current transaction. This is set for - plugin driven transactions via - :c:func:`TSHttpConnectWithPluginId`. -pitag Proxy Plugin Plugin tag for the current transaction. This is set for - plugin driven transactions via - :c:func:`TSHttpConnectWithPluginId`. -===== ============ ============================================================ +===== ================ ============================================================ +Field Source Description +===== ================ ============================================================ +piid Proxy Plugin Plugin ID for the current transaction. This is set for + plugin driven transactions via + :c:func:`TSHttpConnectWithPluginId`. +pitag Proxy Plugin Plugin tag for the current transaction. This is set for + plugin driven transactions via + :c:func:`TSHttpConnectWithPluginId`. +cqint Client Request If a request was generated internally (via a plugin), then + this has a value of ``1``, otherwise ``0``. This can be + useful when tracking internal only requests, such as those + generated by the ``authproxy`` plugin. +===== ================ ============================================================ .. _admin-logging-fields-proto: @@ -623,6 +637,7 @@ TCP Details ~~~~~~~~~~~ .. _cqtr: +.. _cqmpt: The following logging fields reveal information about the TCP layer of client, proxy, and origin server connections. @@ -633,6 +648,10 @@ Field Source Description cqtr Client Request TCP reused status of the connection between the client and |TS| proxy, indicating whether the request was delivered through an already established connection. +cqmpt Client Request Indicates the MPTCP state of the connection. ``-1`` means + MPTCP was not enabled on the listening port, whereas ``0`` + and ``1`` indicates whether MPTCP was successfully + negotiated or not. ===== ============== ========================================================== .. _admin-logging-fields-time: @@ -661,7 +680,7 @@ The logging fields expose a variety of timing related information about client, proxy, and origin transactions. Variants of some of the fields provide timing resolution of the same underlying detail in milliseconds and seconds (both fractional and rounded-down integers). These variants are particularly useful -in accomodating the emulation of other HTTP proxy softwares' logging formats. +in accommodating the emulation of other HTTP proxy softwares' logging formats. Other fields in this category provide variously formatted timestamps of particular events within the current transaction (e.g. the time at which a @@ -694,7 +713,7 @@ ms Proxy Timestamp in milliseconds of a specific milestone which milestone to use. msdms Proxy Difference in milliseconds between the timestamps of two milestones. See note below about - specifying which miletones to use. + specifying which milestones to use. stms Proxy-Origin Connection Time (in milliseconds) spent accessing the origin server. Measured from the time the connection between proxy and origin is established to the diff --git a/doc/admin-guide/logging/rotation.en.rst b/doc/admin-guide/logging/rotation.en.rst index 01441c86299..eaed409ff48 100644 --- a/doc/admin-guide/logging/rotation.en.rst +++ b/doc/admin-guide/logging/rotation.en.rst @@ -67,8 +67,7 @@ the following information: - The original log file's name (such as ``access.log``). -- The hostname of the |TS| node that generated the log file (useful in |TS| - log collation configurations). +- The hostname of the |TS| node that generated the log file. - Two timestamps separated by a hyphen (``-``). The first timestamp is a *lower bound* for the timestamp of the first record in the log diff --git a/doc/admin-guide/monitoring/alarms.en.rst b/doc/admin-guide/monitoring/alarms.en.rst index 0749f352fca..c4a23ccf3aa 100644 --- a/doc/admin-guide/monitoring/alarms.en.rst +++ b/doc/admin-guide/monitoring/alarms.en.rst @@ -35,7 +35,7 @@ occurs, follow the steps below: #. Set :ts:cv:`proxy.config.alarm_email` in :file:`records.config` to the email address you want to receive alarm notifications. :: - CONFIG proxy.config.alarm_email STRING "alerts@example.com" + CONFIG proxy.config.alarm_email STRING alerts@example.com #. Run the command :option:`traffic_ctl config reload` to apply the configuration changes. diff --git a/doc/admin-guide/monitoring/monitoring/third-party/circonus.en.rst b/doc/admin-guide/monitoring/monitoring/third-party/circonus.en.rst index 27ae6d2d7c9..fde5e9676fa 100644 --- a/doc/admin-guide/monitoring/monitoring/third-party/circonus.en.rst +++ b/doc/admin-guide/monitoring/monitoring/third-party/circonus.en.rst @@ -62,7 +62,7 @@ environment. #. Begin the new check creation process from within your Circonus account by clicking the *New Check* button near the top-right of the checks screen. - .. image:: ../../../../static/images/admin/monitor/circonus/new-check-button.png + .. figure:: ../../../../static/images/admin/monitor/circonus/new-check-button.png :alt: Circonus New Check button :align: center @@ -71,7 +71,7 @@ environment. you and will depend largely on whether you are using on-site Circonus or the hosted service, as well as the geographic location of your |TS| instance(s). - .. image:: ../../../../static/images/admin/monitor/circonus/check-config-1.png + .. figure:: ../../../../static/images/admin/monitor/circonus/check-config-1.png :alt: Choosing a check type :align: center @@ -83,7 +83,7 @@ environment. any of the other options to match your environment if necessary (for this guide, only *Host* and *URI* will need to be entered). - .. image:: ../../../../static/images/admin/monitor/circonus/check-config-2.png + .. figure:: ../../../../static/images/admin/monitor/circonus/check-config-2.png :alt: Advanced check configuration :align: center @@ -94,7 +94,7 @@ environment. running and use ``curl`` to manually fetch the statistics data from your server. - .. image:: ../../../../static/images/admin/monitor/circonus/check-config-3.png + .. figure:: ../../../../static/images/admin/monitor/circonus/check-config-3.png :alt: Check test :align: center @@ -110,7 +110,7 @@ environment. clicking on *Metrics Grid* for an overview visualization of all the data being collected. - .. image:: ../../../../static/images/admin/monitor/circonus/metric-grid.png + .. figure:: ../../../../static/images/admin/monitor/circonus/metric-grid.png :alt: Circonus metric grid :align: center diff --git a/doc/admin-guide/monitoring/statistics/core/http-connection.en.rst b/doc/admin-guide/monitoring/statistics/core/http-connection.en.rst index b2e26fa4d96..dc55cc3ae04 100644 --- a/doc/admin-guide/monitoring/statistics/core/http-connection.en.rst +++ b/doc/admin-guide/monitoring/statistics/core/http-connection.en.rst @@ -58,6 +58,8 @@ HTTP Connection .. ts:stat:: global proxy.process.http.current_client_transactions integer :type: gauge + Represents the current number of HTTP/1.0 and HTTP/1.1 transactions from client to the |TS|. + .. ts:stat:: global proxy.process.http.current_server_connections integer :type: gauge @@ -140,3 +142,8 @@ HTTP Connection :type: gauge Represents the current number of HTTP/2 connections from client to the |TS|. + +.. ts:stat:: global proxy.process.http2.current_client_streams integer + :type: gauge + + Represents the current number of HTTP/2 streams from client to the |TS|. diff --git a/doc/admin-guide/performance/index.en.rst b/doc/admin-guide/performance/index.en.rst index 18f05dc6fc1..acd5c3d542c 100644 --- a/doc/admin-guide/performance/index.en.rst +++ b/doc/admin-guide/performance/index.en.rst @@ -521,7 +521,6 @@ Logging Configuration binary vs. ascii output multiple log formats (netscape+squid+custom vs. just custom) - overhead to log collation using direct writes vs. syslog target Plugin Tuning diff --git a/doc/admin-guide/plugins/background_fetch.en.rst b/doc/admin-guide/plugins/background_fetch.en.rst index 3277e1fcc8f..2f62c19f7f4 100644 --- a/doc/admin-guide/plugins/background_fetch.en.rst +++ b/doc/admin-guide/plugins/background_fetch.en.rst @@ -54,7 +54,7 @@ of the original (Client) request under these conditions: - The response is a ``206`` response - The original client request, and the Origin server response, is clearly indicating that the response is cacheable. This uses the new API - c:func:`TSHttpTxnIsCacheable()`, which also implies honoring current + :c:func:`TSHttpTxnIsCacheable()`, which also implies honoring current Traffic Server configurations. diff --git a/doc/admin-guide/plugins/balancer.en.rst b/doc/admin-guide/plugins/balancer.en.rst index 48b9a17dadf..8283c3818a6 100644 --- a/doc/admin-guide/plugins/balancer.en.rst +++ b/doc/admin-guide/plugins/balancer.en.rst @@ -20,6 +20,10 @@ Balancer Plugin specific language governing permissions and limitations under the License. +.. note:: + + All of the the features in this plugin (and more) are found in + :file:`parent.config`. As a result, this plugin is likely to be deprecated. The ``balancer`` balances requests across multiple origin servers. To use this plugin, configure it in a :file:`remap.config` rule, specifying diff --git a/doc/admin-guide/plugins/cookie_remap.en.rst b/doc/admin-guide/plugins/cookie_remap.en.rst new file mode 100644 index 00000000000..897331696f7 --- /dev/null +++ b/doc/admin-guide/plugins/cookie_remap.en.rst @@ -0,0 +1,459 @@ +.. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +.. include:: ../../common.defs + +.. _admin-plugins-cookie_remap: + +Cookie Based Routing Inside TrafficServer Using cookie_remap +************************************************************ + + +* `Cookie Based Routing Inside TrafficServer Using cookie_remap <#cookie-based-routing-inside-trafficserver-using-cookie_remap>`_ + + * `Features <#features>`_ + * `Limitations <#limitations>`_ + * `Setup <#setup>`_ + * `Operations <#operations>`_ + + * `Comments <#comments>`_ + * `cookie: X|X.Y <#cookie-xxy>`_ + * `operation: exists|not exists|string|regex|bucket <#operation-existsnot-existsstringregexbucket>`_ + * `match: str <#match-str>`_ + * `regex: str <#regex-str>`_ + * `bucket|hash: X/Y <#buckethash-xy>`_ + * `sendto|url: url <#sendtourl-url>`_ + * `status: HTTP status-code <#status-http-status-code>`_ + * `else: url [optional] <#else-url-optional>`_ + * `connector: and <#connector-and>`_ + + * `Reserved path expressions <#reserved-path-expressions>`_ + + * `$cr_req_url <#cr_req_url-v-15>`_ + * `$cr_urlencode() <#cr_urlencode-v-15>`_ + * `$path <#path>`_ + * `$unmatched_path <#unmatched_path>`_ + + * `An example configuration file <#an-example-configuration-file>`_ + * `Debugging things <#debugging-things>`_ + + * `Initial output <#initial-output>`_ + +This remap plugin makes decisions about where to send your request based on properties present (or absent) within the HTTP Cookie header. It can also make decisions based on your uri (url path + query.) + +Features +-------- + +---- + + +* Also supports sub-level cookies + + * K indicates top-level cookie "K" + * K.l indicates top-level cookie "K", subfield "l" + +* Cookie exists / Cookie doesn't exist +* Cookie or uri matches string +* Cookie or uri matches regex (with match replacement in the sendto like `http://foo.com/$1/$2 `_\ ) +* Cookie falls into a hash/bucket range +* Can url encode dynamic data + +Limitations +----------- + +---- + + +* Does not support `plugin chaining `_. + +Setup +----- + +---- + +The plugin is specified in remap.config using a syntax similar to: + + +.. raw:: html + + 
+   map http://foo.com http://bar.com @plugin=/usr/bin/trafficserver/libexec64/cookie_remap.so @pparam=/home/trafficserver/conf/cookie_remap/cookie_remap.txt
+
+ + +Operations +---------- + +---- + +All operations are specified in a YAML configuration file you pass as @pparam on the plugin configuration line. YAML is very simple syntax. See the example configuration files below. + +Each operation results in a sendto action. That means that, if matched, cookie_remap forwards the request to the given ``sendto`` url. It can be proxied or redirected. An ``else`` sendto can be specified, in which case a failure to match will forward the request also. Once a sendto is invoked: + + +* cookie_remap will not process any more "operations" from the configuration +* the default destination specified in the remap.config file will not be used; it will be replaced by the ``sendto`` + +Each ``operation`` can have multiple "sub-operations", connected with an conjunction operator. Currently, only the ``and`` operator is supported. So, you can say, "if cookie exists, ``and`` uri is ``x``\ , then redirect." + +Comments +^^^^^^^^ + +---- + +Comments are allowed in the configuration file if they begin with '#' + +cookie: X|X.Y +^^^^^^^^^^^^^ + +---- + +This sub-operation is testing against the X cookie or X.Y cookie where X.Y denotes the X cookie, sub cookie Y +e.g + + +.. raw:: html + + 
+   A=ACOOKIE;B=data&f=fsub&z=zsub;
+   A will operate on ACOOKIE
+   B will operate on data&f=fsub&z=zsub
+   B.f will operate on fsub
+
+ + +operation: exists|not exists|string|regex|bucket +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +---- + +This keyword actually denotes a "suboperation" - it can be specified multiple times, connected with ``and``\ , and send the user to a given destination. + + +* exists: Test for the existence of a cookie +* not exists: Test for the non-existence of a cookie +* string: string match +* regex: regex matching with $1 - $9 replaced in ``sendto`` +* bucket: Hash the cookie and check if it falls in a bucket + +The ``cookie`` operator must be specified in the YAML file just before the suboperation that will apply to the cookie. + +The ``string``, ``regex`` and ``bucket`` operate on the specified cookie, or if no cookie is specified, they will operate on the uri of the request. + +match: str +^^^^^^^^^^ + +---- + +Match the cookie data to str. If no cookie is specified, match to the request uri. + +regex: str +^^^^^^^^^^ + +---- + +Sets the regex to str. Matching is enabled with $1-$9 (replaced in sendto). Note that only the final regex match in an operation will be used to populate the substitutions $1-$9. If no cookie is specified, match to the request uri. + +bucket|hash: X/Y +^^^^^^^^^^^^^^^^ + +---- + +Hashes (bucketizes) the data in Y buckets. If you fall into the first X of them, this suboperation passes. If no cookie is specified, match to the request uri. + + +.. raw:: html + + 
+   bucket: 1/100
+   will effectively bucketize 1% of your users
+
+ + +sendto|url: url +^^^^^^^^^^^^^^^ + +---- + +if the sub-operation(s) all match, send to url + +status: HTTP status-code +^^^^^^^^^^^^^^^^^^^^^^^^ + +---- + +if the sub-operation(s) all match, set the status code (e.g. set it to 302). In the case of a redirect, the sendto URL becomes the redirect URL. + +else: url [optional] +^^^^^^^^^^^^^^^^^^^^ + +---- + +If one of the sub-operations fails, send to url. This is optional. If there is no 'else', we will continue to process operations until either one succeeds, there is an else in one of the operations or we fall through to the default mapping from remap.config + +connector: and +^^^^^^^^^^^^^^ + +---- + +'and' is the only supported connector + +Reserved path expressions +------------------------- + +---- + +The following expressions can be used in either the sendto **or** ``else`` URLs, and will be expanded. + +$cr_req_url +^^^^^^^^^^^^^^^^^^^^^^^^^ + +---- + +Replaced with the full original url. + +Therefore, a rule like: + + +.. raw:: html + + 
+   op:
+     cookie: K
+     operation: exists
+     sendto: http://foo.com/?.done=$cr_req_url
+
+ + +and a request that matches, e.g. + + +.. raw:: html + + 
+   http://bar.com/hello?fruit=bananas
+
+ + +will become + + +.. raw:: html + + 
+   http://foo.com/?.done=http://bar.com/hello?fruit=bananas
+
+ + +$cr_urlencode() +^^^^^^^^^^^^^^^^^^^^^^^^^ + +---- + +Replaced with a urlencoded version of its argument. The url argument is aggressively encoded such that all non-alphanumeric characters are converted to % hex notation. The simple algorithm could probably be refined in the future to be less aggressive (encode fewer characters.) + +Therefore, a rule like: + + +.. raw:: html + + 
+   op:
+     cookie: B
+     operation: exists
+     sendto: http://foo.com/?.done=$cr_urlencode($cr_req_url)
+
+ + +and a request that matches, e.g. + + +.. raw:: html + + 
+   http://bar.com/hello?fruit=bananas
+
+ + +will become + + +.. raw:: html + + 
+   http://foo.com/?.done=http%3A%2F%2Fbar%2Ecom@2Fhello%3Ffruit%3Dbananas
+
+ + +$path +^^^^^ + +---- + +$path can be used to replace in either the sendto **or** else URLs the original request path. Therefore a rule like: + + +.. raw:: html + + 
+   op:
+     cookie: K
+     operation: exists
+     sendto: http://foo.com/$path/x/y/z
+
+ + +and a request like `http://finance.yahoo.com/photos/what/ever/ `_ that matches the rule + + +.. raw:: html + + 
+   map http://finance.yahoo.com/photos/ http://newfinance.yahoo.com/1k.html @plugin=cookie_remap.so @pparam=foo.txt
+
+ + +will become `http://foo.com/photos/what/ever/x/y/z `_ + +$unmatched_path +^^^^^^^^^^^^^^^ + +---- + +$unmatched_path can be used to gather the URL arguments **beyond** what was matched by the remap rule. Therefore a rule like: + + +.. raw:: html + + 
+   op:
+     cookie: K
+     operation: exists
+     sendto: http://foo.com/$unmatched_path/x/y/z
+
+ + +and a request like `http://finance.yahoo.com/photos/what/ever/ `_ that matches the rule + + +.. raw:: html + + 
+   map http://finance.yahoo.com/photos/ http://newfinance.yahoo.com/1k.html @plugin=cookie_remap.so @pparam=foo.txt
+
+ + +will become `http://foo.com/what/ever/matches/x/y/z `_ + +An example configuration file +----------------------------- + +---- + + + +* first match "wins" (top down) + + +.. raw:: html + + 
+   #comments
+   #are allowed
+   op:
+     cookie: K
+     operation: exists
+     url: http://www.yahoo.com
+
+   op:
+     cookie: K
+     operation: exists
+     connector: and
+     cookie: K.l
+     operation: not exists
+     sendto: http://www.yahoo.com
+
+   op:
+     cookie: Y
+     operation: bucket
+     bucket: 1/1000
+     connector: and
+     cookie: Y.l
+     operation: regex
+     regex: (.*)
+     connector: and
+     cookie: Y.n
+     operation: string
+     match: foobar
+     sendto: http://cnn.com/$1
+     else: http://yahoo.com
+
+ + +Debugging things +---------------- + +---- + +The easiest way to debug problems with this plugin is to run in the following manner: + + +.. raw:: html + + 
+   bin/traffic_server -T cookie_remap
+
+ + +which will produce output on startup, and for each request. Be aware that this mode of running trafficserver is **extremely** inefficient and should only be used for debugging. + +Initial output +^^^^^^^^^^^^^^ + +---- + +Initially, you will notice output describing each operation and how trafficserver interpreted the information from your configuration file: + + +.. raw:: html + + 
+   [Jul  8 13:08:34.183] Server {3187168} DIAG: (cookie_remap) loading cookie remap configuration file from /homes/ebalsa/dev/yts_mods/cookie_remap/example_config.txt
+   [Jul  8 13:08:34.199] Server {3187168} DIAG: (cookie_remap) ++++operation++++
+   [Jul  8 13:08:34.199] Server {3187168} DIAG: (cookie_remap) sending to: http://finance.yahoo.com/2k.html
+   [Jul  8 13:08:34.199] Server {3187168} DIAG: (cookie_remap) if these operations match:
+   [Jul  8 13:08:34.200] Server {3187168} DIAG: (cookie_remap)     +++subop+++
+   [Jul  8 13:08:34.200] Server {3187168} DIAG: (cookie_remap)         cookie: B
+   [Jul  8 13:08:34.200] Server {3187168} DIAG: (cookie_remap)         operation: bucket
+   [Jul  8 13:08:34.200] Server {3187168} DIAG: (cookie_remap)         bucket: 1/100
+   [Jul  8 13:08:34.200] Server {3187168} DIAG: (cookie_remap)         taking: 1
+   [Jul  8 13:08:34.200] Server {3187168} DIAG: (cookie_remap)         out of: 100
+   [Jul  8 13:08:34.200] Server {3187168} DIAG: (cookie_remap)     +++subop+++
+   [Jul  8 13:08:34.201] Server {3187168} DIAG: (cookie_remap)         cookie: Y.l
+   [Jul  8 13:08:34.201] Server {3187168} DIAG: (cookie_remap)         operation: exists
+   [Jul  8 13:08:34.201] Server {3187168} DIAG: (cookie_remap) ++++operation++++
+   [Jul  8 13:08:34.201] Server {3187168} DIAG: (cookie_remap) sending to: http://X.existed.com
+   [Jul  8 13:08:34.201] Server {3187168} DIAG: (cookie_remap) if these operations match:
+   [Jul  8 13:08:34.201] Server {3187168} DIAG: (cookie_remap)     +++subop+++
+   [Jul  8 13:08:34.201] Server {3187168} DIAG: (cookie_remap)         cookie: PH.l
+   [Jul  8 13:08:34.202] Server {3187168} DIAG: (cookie_remap)         operation: exists
+   [Jul  8 13:08:34.202] Server {3187168} DIAG: (cookie_remap) # of ops: 2
+
+ + +each operation is enumerated and is more-or-less human readable from the top-down. From now on, each request has information spit out to the console with debugging information on how the cookie_remap is treating this request. + diff --git a/doc/admin-guide/plugins/header_rewrite.en.rst b/doc/admin-guide/plugins/header_rewrite.en.rst index 0ee9d049950..63df29fc49c 100644 --- a/doc/admin-guide/plugins/header_rewrite.en.rst +++ b/doc/admin-guide/plugins/header_rewrite.en.rst @@ -1,18 +1,14 @@ -.. Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at +.. Licensed to the Apache Software Foundation (ASF) under one or more contributor license + agreements. See the NOTICE file distributed with this work for additional information regarding + copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with the License. You may obtain + a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations + Unless required by applicable law or agreed to in writing, software distributed under the License + is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + or implied. See the License for the specific language governing permissions and limitations under the License. .. include:: ../../common.defs @@ -311,6 +307,8 @@ e.g.:: cond %{CIDR:8} ="8.0.0.0" set-header X-Is-Eight "Yes" + cond %{CIDR:,8} ="fd00::" #note the IPv6 Mask is in the second position + set-header IPv6Internal "true" This condition has no requirements other than access to the Client IP, hence, it should work in any and all hooks. @@ -330,7 +328,7 @@ The data that can be checked is :: %{INBOUND:REMOTE-PORT} The client port for the connection. %{INBOUND:TLS} The TLS protocol if the connection is over TLS, otherwise the empty string. %{INBOUND:H2} The string "h2" if the connection is HTTP/2, otherwise the empty string. - %{INBOUND:IPV4} The string "ipv4" if the connection is IPv4, otherwise the emtpy string. + %{INBOUND:IPV4} The string "ipv4" if the connection is IPv4, otherwise the empty string. %{INBOUND:IPV6} The string "ipv6" if the connection is IPv6, otherwise the empty string. %{INBOUND:IP-FAMILY} The IP family, either "ipv4" or "ipv6". %{INBOUND:STACK} The full protocol stack separated by ','. @@ -608,7 +606,8 @@ be specified multiple times, such as ``Set-Cookie``, but for headers which may only be specified once you may prefer to use `set-header`_ instead. The header's ```` may be specified as a literal string, or it may take -advantage of :ref:`header-rewrite-expansion` to calculate a dynamic value for the header. +advantage of :ref:`header-rewrite-concatenations` to calculate a dynamic value +for the header. counter ~~~~~~~ @@ -721,7 +720,8 @@ Replaces the value of header ```` with ````, creating the header if necessary. The header's ```` may be specified according to `Header Values`_ or take -advantage of :ref:`header-rewrite-expansion` to calculate a dynamic value for the header. +advantage of :ref:`header-rewrite-concatenations` to calculate a dynamic value +for the header. set-redirect ~~~~~~~~~~~~ @@ -732,8 +732,8 @@ set-redirect When invoked, sends a redirect response to the client, with HTTP status ````, and a new location of ````. If the ``QSA`` flag is enabled, the original query string will be preserved and added to the new -location automatically. This operator supports :ref:`header-rewrite-expansion` for -````. +location automatically. This operator supports +:ref:`header-rewrite-concatenations` for ````. set-status ~~~~~~~~~~ @@ -804,67 +804,31 @@ L Last rule, do not continue. QSA Append the results of the rule to the query string. ====== ======================================================================== -.. _header-rewrite-expansion: +.. _header-rewrite-concatenations: -Values and Variable Expansion ------------------------------ +String concatenations +--------------------- -.. note:: +You can concatenate values using strings, condition values and variable expansions on the same line. - This feature is replaced with a new string concatenations as of ATS v8.1.0. In v9.0.0 the special - %<> string expansions below are no longer available, instead use the following mapping: + add-header CustomHeader "Hello from %{IP:SERVER}:%{INBOUND:LOCAL-PORT}" -======================= ================================================================================== -Variable New condition variable to use -======================= ================================================================================== -% %{CLIENT-URL:SCHEME} -% %{CLIENT-URL:PORT} -% %{IP:CLIENT}, %{INBOUND:REMOTE-ADDR} or e.g. %{CIDR:24,48} -% %{CLIENT-HEADER:Content-Length} -% %{METHOD} -% %[CLIENT-URL} -% %{CLIENT-URL:PATH} -======================= ================================================================================== +String concatenation is not yet supported in condition testing. -The % tags can now be replaced with the %{INBOUND:...} equivalent. +Note: In versions prior to ATS v9.0.0, an alternative string expansion was available. those +expansions are no longer available, but the following table can help migrations: -You can concatenate values using strings, condition values and variable expansions via the ``+`` operator. -Variables (eg %) in the concatenation must be enclosed in double quotes ``"``:: - - add-header CustomHeader "Hello from " + %{IP:SERVER} + ":" + "%" - -However, the above example is somewhat contrived to show the old tags, it should instead be written as - - add-header CustomHeader "Hello from " + %{IP:SERVER} + ":" + %{INBOUND:LOCAL-PORT} - - -Concatenation is not supported in condition testing. - -Supported substitutions are currently the following table, however they are deprecated and you should -instead use the equivalent %{} conditions as shown above: - -======================= ================================================================================== -Variable Description -======================= ================================================================================== -% (Deprecated) Protocol -% (Deprecated) Port -% (Deprecated) Client IP -% (Deprecated) Client request length -% (Deprecated) Client HTTP method -% (Deprecated) Client effective URI -% (Deprecated) Client unmapped URI path -% (Deprecated) The local (ATS) address for the inbound connection. -% (Deprecated) The local (ATS) port for the inbound connection. -% (Deprecated) The client address for the inbound connectoin. -% (Deprecated) The client port for the inbound connectoin. -% (Deprecated) The TLS protocol for the inbound connection if it is over TLS, otherwise the - empty string. -% (Deprecated) The string "h2" if the inbound connection is HTTP/2, otherwise the empty string. -% (Deprecated) The string "ipv4" if the inbound connection is IPv4, otherwise the emtpy string. -% (Deprecated) The string "ipv6" if the inbound connection is IPv6, otherwise the empty string. -% (Deprecated) The IP family of the inbound connection (either "ipv4" or "ipv6"). -% (Deprecated) The full protocol stack of the inbound connection separated by ','. -======================= ================================================================================== +======================== ========================================================================== +Old expansion variable Condition variable to use with concatenatinos +======================== ========================================================================== +% %{CLIENT-URL:SCHEME} +% %{CLIENT-URL:PORT} +% %{IP:CLIENT}, %{INBOUND:REMOTE-ADDR} or e.g. %{CIDR:24,48} +% %{CLIENT-HEADER:Content-Length} +% %{METHOD} +% %[CLIENT-URL} +% %{CLIENT-URL:PATH} +======================== ========================================================================== Header Values ------------- @@ -900,13 +864,18 @@ The URL part names which may be used for these conditions and actions are: Part Description ======== ====================================================================== HOST Full hostname. -PATH URL substring beginning with (but not including) the first ``/`` after the hostname up to, - but not including, the query string. + +PATH URL substring beginning with (but not including) the first ``/`` after + the hostname up to, but not including, the query string. + PORT Port number. + QUERY URL substring from the ``?``, signifying the beginning of the query parameters, until the end of the URL. Empty string if there were no - quuery parameters. + query parameters. + SCHEME URL scheme in use (e.g. ``http`` and ``https``). + URL The complete URL. ======== ====================================================================== diff --git a/doc/admin-guide/plugins/index.en.rst b/doc/admin-guide/plugins/index.en.rst index dd7dd5b884d..da38b3c288e 100644 --- a/doc/admin-guide/plugins/index.en.rst +++ b/doc/admin-guide/plugins/index.en.rst @@ -52,6 +52,7 @@ Plugins that are considered stable are installed by default in |TS| releases. Cache Promotion Policies Combo Handler Configuration Remap + Cookie Remap ESI Escalate Compress @@ -62,6 +63,7 @@ Plugins that are considered stable are installed by default in |TS| releases. Regex Remap Regex Revalidate Remap Purge + Slice Stats over HTTP TCPInfo XDebug @@ -151,6 +153,7 @@ directory of the |TS| source tree. Experimental plugins can be compiled by passi Header Frequency HIPES Hook Trace + JA3 Fingerprint Memcache Metalink Money Trace @@ -158,9 +161,9 @@ directory of the |TS| source tree. Experimental plugins can be compiled by passi Multiplexer MySQL Remap Signed URLs + Slicer SSL Headers SSL Session Reuse - Stale While Revalidate System Statistics Traffic Dump WebP Transform @@ -194,13 +197,16 @@ directory of the |TS| source tree. Experimental plugins can be compiled by passi :doc:`HIPES ` Adds support for HTTP Pipes. +:doc:`JA3 Fingerprint ` + Calculates JA3 Fingerprints for incoming SSL traffic. + :doc:`Memcache ` Implements the memcache protocol for cache contents. :doc:`Metalink ` Implements the Metalink download description format in order to try not to download the same file twice. -:doc:`Money Trace ` +:doc:`Money Trace ` Allows Trafficserver to participate in a distributed tracing system based upon the Comcast Money library. :doc:`MP4 ` @@ -215,6 +221,9 @@ directory of the |TS| source tree. Experimental plugins can be compiled by passi :doc:`MySQL Remap ` Allows dynamic remaps from a MySQL database. +:doc:`Prefetch ` + Pre-fetch objects based on the requested URL path pattern. + :doc:`Remap Purge ` This remap plugin allows the administrator to easily setup remotely controlled ``PURGE`` for the content of an entire remap rule. @@ -222,17 +231,17 @@ directory of the |TS| source tree. Experimental plugins can be compiled by passi :doc:`Signed URLs ` Adds support for verifying URL signatures for incoming requests to either deny or redirect access. +:doc:`Slicer ` + Slice full file or range based requests into deterministic chunks, allowing large files to be + spread across multiple cache stripes. Allows range requests to be satisfied by stitching these + chunks together. + :doc:`SSL Session Reuse ` Coordinates Session ID and ticket based TLS session resumption between a group of ATS machines. :doc:`SSL Headers ` Populate request headers with SSL session information. -:doc:`Stale While Revalidate ` - :deprecated: - - Refresh content asynchronously while serving stale data. - :doc:`System Stats ` Inserts system statistics in to the stats list diff --git a/doc/admin-guide/plugins/ja3_fingerprint.en.rst b/doc/admin-guide/plugins/ja3_fingerprint.en.rst new file mode 100644 index 00000000000..9be99f55d94 --- /dev/null +++ b/doc/admin-guide/plugins/ja3_fingerprint.en.rst @@ -0,0 +1,49 @@ +.. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + + +.. include:: ../../common.defs + +.. _admin-plugins-ja3_fingerprint: + + +JA3 Fingerprint Plugin +******************* + +Description +=========== + +``JA3 Fingerprint`` calculates JA3 fingerprints for incoming SSL traffic. "JA3 is a method for creating SSL/TLS client fingerprints" by concatenating values in ClientHello packet and MD5 hash the result to produce a 32 character fingerprint. Malwares tend to use the same encryption code/client, which makes it an effective way to detect malicious clients. More info about ja3 is available: https://github.com/salesforce/ja3. + +The calculated JA3 fingerprints are then appended to upstream request (to be processed at upstream) and/or logged locally (depending on the config). + +Plugin Configuration +==================== +.. program:: ja3_fingerprint.so + +* ``ja3_fingerprint`` can be used as a global/remap plugin and is configured via :file:`plugin.config` or :file:`remap.config`. + .. option:: --ja3raw + + (`optional`, default:unused) - enables raw fingerprints header. With this option, the plugin will append additional header `X-JA3-Raw` to proxy request. + + .. option:: --ja3log + + (`optional`, default:unused) - enables local logging. With this option, the plugin will log JA3 info to :file:`ja3_fingerprint.log` in the standard logging directory. The format is: [time] [client IP] [JA3 string] [JA3 hash] + +Requirement +============= +Won't compile against OpenSSL 1.1.0 due to API changes and opaque structures. diff --git a/doc/admin-guide/plugins/lua.en.rst b/doc/admin-guide/plugins/lua.en.rst index 624056d5693..dd890e36d1e 100644 --- a/doc/admin-guide/plugins/lua.en.rst +++ b/doc/admin-guide/plugins/lua.en.rst @@ -80,7 +80,8 @@ Configuration ============= This module acts as remap plugin of Traffic Server, so we should realize 'do_remap' or 'do_os_response' function in each -lua script. We can write this in remap.config: +lua script. The path referencing a file with the lua script can be relative to the configuration directory or an absolute +path. We can write this in remap.config: :: @@ -140,7 +141,7 @@ is always available within lua script. This package can be introduced into Lua l ts.say('Hello World') ts.sleep(10) -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.process.uuid --------------- @@ -156,7 +157,7 @@ Here is an example: local pid = ts.process.uuid() -- a436bae6-082c-4805-86af-78a5916c4a91 -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.now ------ @@ -173,7 +174,7 @@ Here is an example: local nt = ts.now() -- 1395221053.123 -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.debug -------- @@ -194,7 +195,7 @@ We should write this TAG in records.config(If TAG is missing, default TAG will b ``CONFIG proxy.config.diags.debug.tags STRING TAG`` -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.error -------- @@ -210,7 +211,7 @@ Here is an example: ts.error('This is an error message') -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ TS Basic Internal Information ----------------------------- @@ -231,7 +232,7 @@ Here is an example: local config_dir = ts.get_config_dir() -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ Remap status constants ---------------------- @@ -247,7 +248,7 @@ Remap status constants These constants are usually used as return value of do_remap function. -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.remap.get_to_url_host ------------------------ @@ -267,7 +268,7 @@ Here is an example: return 0 end -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.remap.get_to_url_port ------------------------ @@ -277,7 +278,7 @@ ts.remap.get_to_url_port **description**: retrieve the "to" port of the remap rule -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.remap.get_to_url_scheme -------------------------- @@ -287,7 +288,7 @@ ts.remap.get_to_url_scheme **description**: retrieve the "to" scheme of the remap rule -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.remap.get_to_uri ------------------- @@ -297,7 +298,7 @@ ts.remap.get_to_uri **description**: retrieve the "to" path of the remap rule -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.remap.get_to_url ------------------- @@ -307,7 +308,7 @@ ts.remap.get_to_url **description**: retrieve the "to" url of the remap rule -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.remap.get_from_url_host -------------------------- @@ -317,7 +318,7 @@ ts.remap.get_from_url_host **description**: retrieve the "from" host of the remap rule -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.remap.get_from_url_port -------------------------- @@ -327,7 +328,7 @@ ts.remap.get_from_url_port **description**: retrieve the "from" port of the remap rule -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.remap.get_from_url_scheme ---------------------------- @@ -337,7 +338,7 @@ ts.remap.get_from_url_scheme **description**: retrieve the "from" scheme of the remap rule -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.remap.get_from_uri --------------------- @@ -347,7 +348,7 @@ ts.remap.get_from_uri **description**: retrieve the "from" path of the remap rule -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.remap.get_from_url --------------------- @@ -357,7 +358,7 @@ ts.remap.get_from_url **description**: retrieve the "from" url of the remap rule -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.hook ------- @@ -402,7 +403,7 @@ You can create global hook as well ts.hook(TS_LUA_HOOK_READ_REQUEST_HDR, do_some_work) - Or you can do it this way +Or you can do it this way :: @@ -417,7 +418,7 @@ Also the return value of the function will control how the transaction will be r the transaction to be re-enabled normally (TS_EVENT_HTTP_CONTINUE). Return value of 1 will be using TS_EVENT_HTTP_ERROR instead. -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ Hook point constants -------------------- @@ -488,7 +489,7 @@ Additional Information: +-----------------------+---------------------------+----------------------+--------------------+----------------------+ -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.ctx ------ @@ -527,7 +528,7 @@ Then the client will get the response like this: Connection: Keep-Alive ... -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.client_request.get_method ---------------------------- @@ -538,7 +539,7 @@ ts.client_request.get_method **description:** This function can be used to retrieve the current client request's method name. String like "GET" or "POST" is returned. -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.client_request.set_method ---------------------------- @@ -558,7 +559,7 @@ ts.client_request.get_version Current possible values are 1.0, 1.1, and 0.9. -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.client_request.set_version ----------------------------- @@ -572,7 +573,7 @@ ts.client_request.set_version ts.client_request.set_version('1.0') -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.client_request.get_uri ------------------------- @@ -596,7 +597,7 @@ Then ``GET /st?a=1`` will yield the output: ``/st`` -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.client_request.set_uri ------------------------- @@ -609,7 +610,7 @@ ts.client_request.set_uri The PATH argument must be a Lua string and starts with ``/`` -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.client_request.get_uri_args ------------------------------ @@ -633,7 +634,7 @@ Then ``GET /st?a=1&b=2`` will yield the output: ``a=1&b=2`` -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.client_request.set_uri_args ------------------------------ @@ -648,7 +649,7 @@ ts.client_request.set_uri_args ts.client_request.set_uri_args('n=6&p=7') -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.client_request.get_uri_params -------------------------------- @@ -672,7 +673,7 @@ Then ``GET /st;a=1`` will yield the output: ``a=1`` -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.client_request.set_uri_params -------------------------------- @@ -687,7 +688,7 @@ ts.client_request.set_uri_params ts.client_request.set_uri_params('n=6') -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.client_request.get_url ------------------------- @@ -710,7 +711,7 @@ Then ``GET /st?a=1&b=2 HTTP/1.1\r\nHost: a.tbcdn.cn\r\n...`` will yield the outp ``http://a.tbcdn.cn/st?a=1&b=2`` -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.client_request.header.HEADER ------------------------------- @@ -739,7 +740,7 @@ Then ``GET /st HTTP/1.1\r\nHost: b.tb.cn\r\nUser-Agent: Mozilla/5.0\r\n...`` wil ``Mozilla/5.0`` -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.client_request.get_headers ----------------------------- @@ -767,7 +768,7 @@ Then ``GET /st HTTP/1.1\r\nHost: b.tb.cn\r\nUser-Aget: Mozilla/5.0\r\nAccept: */ Accept: */* -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.client_request.client_addr.get_addr -------------------------------------- @@ -791,7 +792,7 @@ Here is an example: return 0 end -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.client_request.client_addr.get_incoming_port ----------------------------------------------- @@ -812,7 +813,7 @@ Here is an example: ts.debug(port) -- 80 end -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.client_request.get_url_host ------------------------------ @@ -835,7 +836,7 @@ Then ``GET /liuyurou.txt HTTP/1.1\r\nHost: 192.168.231.129:8080\r\n...`` will yi ``192.168.231.129`` -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.client_request.set_url_host ------------------------------ @@ -865,7 +866,7 @@ remap.config like this: Then server request will connect to ``192.168.231.130:80`` -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.client_request.get_url_port ------------------------------ @@ -889,7 +890,7 @@ Then Then ``GET /liuyurou.txt HTTP/1.1\r\nHost: 192.168.231.129:8080\r\n...`` wi ``8080`` -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.client_request.set_url_port ------------------------------ @@ -901,7 +902,7 @@ ts.client_request.set_url_port the origin server, and we should return TS_LUA_REMAP_DID_REMAP(_STOP) in do_remap. -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.client_request.get_url_scheme -------------------------------- @@ -925,7 +926,7 @@ Then ``GET /liuyurou.txt HTTP/1.1\r\nHost: 192.168.231.129:8080\r\n...`` will yi ``http`` -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.client_request.set_url_scheme -------------------------------- @@ -937,7 +938,7 @@ ts.client_request.set_url_scheme server request, and we should return TS_LUA_REMAP_DID_REMAP(_STOP) in do_remap. -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.http.set_cache_url --------------------- @@ -956,7 +957,7 @@ Here is an example: return 0 end -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.http.get_cache_lookup_url ---------------------------- @@ -981,7 +982,7 @@ Here is an example return 0 end -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.http.set_cache_lookup_url ---------------------------- @@ -991,7 +992,7 @@ ts.http.set_cache_lookup_url **description:** This function can be used to set the cache lookup url for the client request. -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.http.get_parent_proxy ------------------------ @@ -1017,7 +1018,7 @@ Here is an example return 0 end -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.http.set_parent_proxy ------------------------ @@ -1027,7 +1028,7 @@ ts.http.set_parent_proxy **description:** This function can be used to set the parent proxy host and name. -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.http.get_parent_selection_url -------------------------------- @@ -1052,7 +1053,7 @@ Here is an example return 0 end -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.http.set_parent_selection_url -------------------------------- @@ -1062,7 +1063,7 @@ ts.http.set_parent_selection_url **description:** This function can be used to set the parent selection url for the client request. -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.http.set_server_resp_no_store -------------------------------- @@ -1081,7 +1082,7 @@ Here is an example: return 0 end -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.http.set_resp ---------------- @@ -1116,7 +1117,7 @@ We will get the response like this: Document access failed :) -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.http.get_cache_lookup_status ------------------------------- @@ -1145,7 +1146,7 @@ Here is an example: end -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.http.set_cache_lookup_status ------------------------------- @@ -1175,7 +1176,7 @@ Here is an example: end -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ Http cache lookup status constants ---------------------------------- @@ -1189,7 +1190,7 @@ Http cache lookup status constants TS_LUA_CACHE_LOOKUP_SKIPPED (3) -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.cached_response.get_status ----------------------------- @@ -1217,7 +1218,7 @@ Here is an example: return 0 end -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.cached_response.get_version ------------------------------ @@ -1230,7 +1231,7 @@ ts.cached_response.get_version Current possible values are 1.0, 1.1, and 0.9. -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.cached_response.header.HEADER -------------------------------- @@ -1260,7 +1261,7 @@ Here is an example: end -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.cached_response.get_headers ------------------------------ @@ -1303,7 +1304,7 @@ We will get the output: Server: ATS/5.0.0 -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.server_request.get_uri @@ -1333,7 +1334,7 @@ Then ``GET /am.txt?a=1`` will yield the output: ``/am.txt`` -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.server_request.set_uri ------------------------- @@ -1346,7 +1347,7 @@ ts.server_request.set_uri The PATH argument must be a Lua string and starts with ``/`` -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.server_request.get_uri_args ------------------------------ @@ -1375,7 +1376,7 @@ Then ``GET /st?a=1&b=2`` will yield the output: ``a=1&b=2`` -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.server_request.set_uri_args ------------------------------ @@ -1390,7 +1391,7 @@ ts.server_request.set_uri_args ts.server_request.set_uri_args('n=6&p=7') -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.server_request.get_uri_params -------------------------------- @@ -1419,7 +1420,7 @@ Then ``GET /st;a=1`` will yield the output: ``a=1`` -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.server_request.set_uri_params -------------------------------- @@ -1434,7 +1435,7 @@ ts.server_request.set_uri_params ts.server_request.set_uri_params('n=6') -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.server_request.header.HEADER ------------------------------- @@ -1468,7 +1469,7 @@ Then ``GET /st HTTP/1.1\r\nHost: b.tb.cn\r\nUser-Agent: Mozilla/5.0\r\n...`` wil ``Mozilla/5.0`` -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.server_request.get_headers ----------------------------- @@ -1503,7 +1504,7 @@ We will get the output: Accept: */* -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.server_request.server_addr.set_addr -------------------------------------- @@ -1523,7 +1524,7 @@ Here is an example: ts.server_request.server_addr.set_addr("192.168.231.17", 80, TS_LUA_AF_INET) end -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ Socket address family --------------------- @@ -1535,7 +1536,7 @@ Socket address family TS_LUA_AF_INET6 (10) -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.server_request.server_addr.get_addr -------------------------------------- @@ -1558,7 +1559,7 @@ Here is an example: ts.debug(family) -- 2(AF_INET) end -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.server_request.server_addr.get_nexthop_addr ---------------------------------------------- @@ -1581,7 +1582,7 @@ Here is an example: ts.debug(family) -- 2(AF_INET) end -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.server_request.server_addr.get_ip ------------------------------------ @@ -1602,7 +1603,7 @@ Here is an example: ts.debug(ip) -- 192.168.231.17 end -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.server_request.server_addr.get_port -------------------------------------- @@ -1623,7 +1624,7 @@ Here is an example: ts.debug(port) -- 80 end -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.server_request.server_addr.get_outgoing_port ----------------------------------------------- @@ -1644,7 +1645,7 @@ Here is an example: ts.debug(port) -- 50880 end -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.server_request.server_addr.set_outgoing_addr ----------------------------------------------- @@ -1664,7 +1665,7 @@ Here is an example: ts.server_request.server_addr.set_outgoing_addr("192.168.231.17", 80, TS_LUA_AF_INET) end -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.server_request.get_url_host ------------------------------ @@ -1692,7 +1693,7 @@ Then ``GET http://abc.com/p2/a.txt HTTP/1.1`` will yield the output: ``abc.com`` -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.server_request.set_url_host ------------------------------ @@ -1742,7 +1743,7 @@ Will be changed to: Client-ip: 135.xx.xx.xx X-Forwarded-For: 135.xx.xx.xx -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.server_request.get_url_scheme -------------------------------- @@ -1770,7 +1771,7 @@ Then ``GET /liuyurou.txt HTTP/1.1\r\nHost: 192.168.231.129:8080\r\n...`` will yi ``http`` -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.server_request.set_url_scheme -------------------------------- @@ -1780,7 +1781,7 @@ ts.server_request.set_url_scheme **description:** Set ``scheme`` field of the request url with ``str``. This function is used to change the scheme of the server request. -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.server_response.get_status ----------------------------- @@ -1806,7 +1807,7 @@ Here is an example: end -`TOP <#ts-lua-plugin>`_' +`TOP <#lua-plugin>`_' ts.server_response.set_status ----------------------------- @@ -1830,7 +1831,7 @@ Here is an example: end -`TOP <#ts-lua-plugin>`_' +`TOP <#lua-plugin>`_' ts.server_response.get_version ------------------------------ @@ -1842,7 +1843,7 @@ ts.server_response.get_version Current possible values are 1.0, 1.1, and 0.9. -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.server_response.set_version ------------------------------ @@ -1856,7 +1857,7 @@ ts.server_response.set_version ts.server_response.set_version('1.0') -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.server_response.header.HEADER -------------------------------- @@ -1890,7 +1891,7 @@ We will get the output: ``text/html`` -`TOP <#ts-lua-plugin>`_' +`TOP <#lua-plugin>`_' ts.server_response.get_headers ------------------------------ @@ -1931,7 +1932,7 @@ We will get the output: Accept-Ranges: bytes -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.client_response.get_status ----------------------------- @@ -1957,7 +1958,7 @@ Here is an example: end -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.client_response.set_status ----------------------------- @@ -1981,7 +1982,7 @@ Here is an example: end -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.client_response.get_version ------------------------------ @@ -1993,7 +1994,7 @@ ts.client_response.get_version Current possible values are 1.0, 1.1, and 0.9. -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.client_response.set_version ------------------------------ @@ -2007,7 +2008,7 @@ ts.client_response.set_version ts.client_response.set_version('1.0') -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.client_response.header.HEADER -------------------------------- @@ -2041,7 +2042,7 @@ We will get the output: ``text/html`` -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.client_response.get_headers ------------------------------ @@ -2082,7 +2083,7 @@ We will get the output: Accept-Ranges: bytes -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.client_response.set_error_resp --------------------------------- @@ -2126,7 +2127,7 @@ We will get the response like this: bad luck :( -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ Number constants ---------------------- @@ -2139,7 +2140,7 @@ Number constants These constants are usually used in transform handler. -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.http.resp_cache_transformed ------------------------------ @@ -2166,7 +2167,7 @@ Here is an example: This function is usually called after we hook TS_LUA_RESPONSE_TRANSFORM. -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.http.resp_cache_untransformed -------------------------------- @@ -2193,7 +2194,7 @@ Here is an example: This function is usually called after we hook TS_LUA_RESPONSE_TRANSFORM. -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.http.resp_transform.get_upstream_bytes ----------------------------------------- @@ -2234,7 +2235,7 @@ Here is an example: return 0 end -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.http.resp_transform.get_upstream_watermark_bytes --------------------------------------------------- @@ -2245,7 +2246,7 @@ ts.http.resp_transform.get_upstream_watermark_bytes **description**: This function can be used to retrive the current watermark bytes for the upstream transform buffer. -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.http.resp_transform.set_upstream_watermark_bytes --------------------------------------------------- @@ -2258,7 +2259,7 @@ ts.http.resp_transform.set_upstream_watermark_bytes Setting the watermark bytes above 32kb may improve the performance of the transform handler. -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.http.resp_transform.set_downstream_bytes ------------------------------------------- @@ -2272,7 +2273,7 @@ Sometimes we want to set Content-Length header in client_response, and this func data is returned from the transform handler. -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.http.skip_remapping_set -------------------------- @@ -2294,7 +2295,7 @@ Here is an example: This function is usually called in do_global_read_request function -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.http.get_client_protocol_stack --------------------------------- @@ -2316,7 +2317,7 @@ Here is an example: return 0 end -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.http.server_push ------------------- @@ -2335,7 +2336,7 @@ Here is an example: return 0 end -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.http.is_websocket -------------------- @@ -2355,7 +2356,7 @@ Here is an example: return 0 end -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.http.get_plugin_tag ---------------------- @@ -2375,7 +2376,7 @@ Here is an example: return 0 end -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.http.id ---------- @@ -2395,7 +2396,7 @@ Here is an example: return 0 end -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.http.ssn_id -------------- @@ -2415,7 +2416,7 @@ Here is an example: return 0 end -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.http.is_internal_request --------------------------- @@ -2435,7 +2436,7 @@ Here is an example: return 0 end -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.http.is_aborted ------------------ @@ -2455,7 +2456,7 @@ Here is an example: return 0 end -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.http.transaction_count ------------------------- @@ -2475,7 +2476,7 @@ Here is an example return 0 end -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.http.redirect_url_set ------------------------ @@ -2494,7 +2495,7 @@ Here is an example return 0 end -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.http.get_server_state ------------------------ @@ -2515,7 +2516,7 @@ Here is an example end end -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ Server state constants ---------------------- @@ -2533,10 +2534,9 @@ Server state constants TS_LUA_SRVSTATE_OPEN_RAW_ERROR (7) TS_LUA_SRVSTATE_PARSE_ERROR (8) TS_LUA_SRVSTATE_TRANSACTION_COMPLETE (9) - TS_LUA_SRVSTATE_CONGEST_CONTROL_CONGESTED_ON_F (10) - TS_LUA_SRVSTATE_CONGEST_CONTROL_CONGESTED_ON_M (11) + TS_LUA_SRVSTATE_PARENT_RETRY (10) -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.http.get_remap_from_url -------------------------- @@ -2555,7 +2555,7 @@ Here is an example ts.debug(from_url) end -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.http.get_remap_to_url ------------------------ @@ -2574,7 +2574,7 @@ Here is an example ts.debug(to_url) end -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.http.get_client_fd --------------------- @@ -2593,7 +2593,7 @@ Here is an example ts.debug(fd) end -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.http.get_server_fd --------------------- @@ -2612,7 +2612,7 @@ Here is an example ts.debug(fd) end -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.add_package_path ------------------- @@ -2635,7 +2635,7 @@ Here is an example: return 0 end -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.add_package_cpath -------------------- @@ -2659,7 +2659,7 @@ Here is an example: end -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.md5 @@ -2681,7 +2681,7 @@ Here is an example: end -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.md5_bin ---------- @@ -2701,7 +2701,7 @@ Here is an example: end -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.sha1 ------- @@ -2722,7 +2722,7 @@ Here is an example: end -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.sha1_bin ----------- @@ -2742,7 +2742,7 @@ Here is an example: end -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.base64_encode ---------------- @@ -2762,7 +2762,7 @@ Here is an example: end -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.base64_decode ---------------- @@ -2783,7 +2783,7 @@ Here is an example: end -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.escape_uri ------------- @@ -2802,7 +2802,7 @@ Here is an example: value = ts.escape_uri(test) end -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.unescape_uri --------------- @@ -2823,7 +2823,7 @@ Here is an example: end -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.fetch ----------- @@ -2878,7 +2878,7 @@ Issuing a post request: res = ts.fetch('http://xx.com/foo', {method = 'POST', body = 'hello world'}) -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.fetch_multi -------------- @@ -2906,7 +2906,7 @@ Here is an example: end -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.http.intercept @@ -2967,7 +2967,7 @@ Then we will get the response like this: 1395145392 Zheng. -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.http.server_intercept ------------------------ @@ -3033,7 +3033,7 @@ Here is an example: ts.http.server_intercept(process_combo, h) end -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.say ------ @@ -3044,7 +3044,7 @@ ts.say **description:** Write response to ATS within intercept or server_intercept. -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.flush -------- @@ -3107,7 +3107,7 @@ We will get the response like this: wo ai yu ye hua wo ai yu ye hua -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.sleep -------- @@ -3136,7 +3136,7 @@ Here is an example: ts.hook(TS_LUA_HOOK_SEND_RESPONSE_HDR, send_response) end -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.host_lookup -------------- @@ -3160,7 +3160,7 @@ Here is an example: ts.hook(TS_LUA_HOOK_SEND_RESPONSE_HDR, send_response) end -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.schedule ----------- @@ -3190,7 +3190,7 @@ Here is an example: ts.hook(TS_LUA_HOOK_CACHE_LOOKUP_COMPLETE, cache_lookup) end -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.http.config_int_get ---------------------- @@ -3205,7 +3205,7 @@ ts.http.config_int_get val = ts.http.config_int_get(TS_LUA_CONFIG_HTTP_CACHE_HTTP) -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.http.config_int_set ---------------------- @@ -3225,7 +3225,7 @@ Here is an example: end -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.http.config_float_get ------------------------ @@ -3236,7 +3236,7 @@ ts.http.config_float_get **description:** Configuration option which has a float value can be retrieved with this function. -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.http.config_float_set ------------------------ @@ -3247,7 +3247,7 @@ ts.http.config_float_set **description:** This function can be used to overwrite the configuration options. -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.http.config_string_get ------------------------- @@ -3258,7 +3258,7 @@ ts.http.config_string_get **description:** Configuration option which has a string value can be retrieved with this function. -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.http.config_string_set ------------------------- @@ -3269,7 +3269,7 @@ ts.http.config_string_set **description:** This function can be used to overwrite the configuration options. -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ Http config constants --------------------- @@ -3322,7 +3322,8 @@ Http config constants TS_LUA_CONFIG_HTTP_TRANSACTION_NO_ACTIVITY_TIMEOUT_IN TS_LUA_CONFIG_HTTP_TRANSACTION_NO_ACTIVITY_TIMEOUT_OUT TS_LUA_CONFIG_HTTP_TRANSACTION_ACTIVE_TIMEOUT_OUT - TS_LUA_CONFIG_HTTP_ORIGIN_MAX_CONNECTIONS + TS_LUA_CONFIG_HTTP_PER_SERVER_CONNECTION_MAX + TS_LUA_CONFIG_HTTP_PER_SERVER_CONNECTION_MATCH TS_LUA_CONFIG_HTTP_CONNECT_ATTEMPTS_MAX_RETRIES TS_LUA_CONFIG_HTTP_CONNECT_ATTEMPTS_MAX_RETRIES_DEAD_SERVER TS_LUA_CONFIG_HTTP_CONNECT_ATTEMPTS_RR_RETRIES @@ -3330,13 +3331,10 @@ Http config constants TS_LUA_CONFIG_HTTP_POST_CONNECT_ATTEMPTS_TIMEOUT TS_LUA_CONFIG_HTTP_DOWN_SERVER_CACHE_TIME TS_LUA_CONFIG_HTTP_DOWN_SERVER_ABORT_THRESHOLD - TS_LUA_CONFIG_HTTP_CACHE_FUZZ_TIME - TS_LUA_CONFIG_HTTP_CACHE_FUZZ_MIN_TIME TS_LUA_CONFIG_HTTP_DOC_IN_CACHE_SKIP_DNS TS_LUA_CONFIG_HTTP_BACKGROUND_FILL_ACTIVE_TIMEOUT TS_LUA_CONFIG_HTTP_RESPONSE_SERVER_STR TS_LUA_CONFIG_HTTP_CACHE_HEURISTIC_LM_FACTOR - TS_LUA_CONFIG_HTTP_CACHE_FUZZ_PROBABILITY TS_LUA_CONFIG_HTTP_BACKGROUND_FILL_COMPLETED_THRESHOLD TS_LUA_CONFIG_NET_SOCK_PACKET_MARK_OUT TS_LUA_CONFIG_NET_SOCK_PACKET_TOS_OUT @@ -3346,6 +3344,7 @@ Http config constants TS_LUA_CONFIG_HTTP_FLOW_CONTROL_LOW_WATER_MARK TS_LUA_CONFIG_HTTP_FLOW_CONTROL_HIGH_WATER_MARK TS_LUA_CONFIG_HTTP_CACHE_RANGE_LOOKUP + TS_LUA_CONFIG_HTTP_NORMALIZE_AE TS_LUA_CONFIG_HTTP_DEFAULT_BUFFER_SIZE TS_LUA_CONFIG_HTTP_DEFAULT_BUFFER_WATER_MARK TS_LUA_CONFIG_HTTP_REQUEST_HEADER_MAX_SIZE @@ -3366,10 +3365,42 @@ Http config constants TS_LUA_CONFIG_HTTP_CACHE_OPEN_WRITE_FAIL_ACTION TS_LUA_CONFIG_HTTP_NUMBER_OF_REDIRECTIONS TS_LUA_CONFIG_HTTP_CACHE_MAX_OPEN_WRITE_RETRIES - TS_LUA_CONFIG_HTTP_NORMALIZE_AE + TS_LUA_CONFIG_HTTP_REDIRECT_USE_ORIG_CACHE_KEY + TS_LUA_CONFIG_HTTP_ATTACH_SERVER_SESSION_TO_CLIENT + TS_LUA_CONFIG_WEBSOCKET_NO_ACTIVITY_TIMEOUT + TS_LUA_CONFIG_WEBSOCKET_ACTIVE_TIMEOUT + TS_LUA_CONFIG_HTTP_UNCACHEABLE_REQUESTS_BYPASS_PARENT + TS_LUA_CONFIG_HTTP_PARENT_PROXY_TOTAL_CONNECT_ATTEMPTS + TS_LUA_CONFIG_HTTP_TRANSACTION_ACTIVE_TIMEOUT_IN + TS_LUA_CONFIG_SRV_ENABLED + TS_LUA_CONFIG_HTTP_FORWARD_CONNECT_METHOD + TS_LUA_CONFIG_SSL_CERT_FILENAME + TS_LUA_CONFIG_SSL_CERT_FILEPATH + TS_LUA_CONFIG_PARENT_FAILURES_UPDATE_HOSTDB + TS_LUA_CONFIG_HTTP_CACHE_ENABLE_DEFAULT_VARY_HEADER + TS_LUA_CONFIG_HTTP_CACHE_VARY_DEFAULT_TEXT + TS_LUA_CONFIG_HTTP_CACHE_VARY_DEFAULT_IMAGES + TS_LUA_CONFIG_HTTP_CACHE_VARY_DEFAULT_OTHER + TS_LUA_CONFIG_HTTP_CACHE_IGNORE_ACCEPT_MISMATCH + TS_LUA_CONFIG_HTTP_CACHE_IGNORE_ACCEPT_LANGUAGE_MISMATCH + TS_LUA_CONFIG_HTTP_CACHE_IGNORE_ACCEPT_ENCODING_MISMATCH + TS_LUA_CONFIG_HTTP_CACHE_IGNORE_ACCEPT_CHARSET_MISMATCH + TS_LUA_CONFIG_HTTP_PARENT_PROXY_FAIL_THRESHOLD + TS_LUA_CONFIG_HTTP_PARENT_PROXY_RETRY_TIME + TS_LUA_CONFIG_HTTP_PER_PARENT_CONNECT_ATTEMPTS + TS_LUA_CONFIG_HTTP_PARENT_CONNECT_ATTEMPT_TIMEOUT + TS_LUA_CONFIG_HTTP_ALLOW_MULTI_RANGE + TS_LUA_CONFIG_HTTP_REQUEST_BUFFER_ENABLED + TS_LUA_CONFIG_HTTP_ALLOW_HALF_OPEN + TS_LUA_CONFIG_SSL_CLIENT_VERIFY_SERVER + TS_LUA_CONFIG_SSL_CLIENT_VERIFY_SERVER_POLICY + TS_LUA_CONFIG_SSL_CLIENT_VERIFY_SERVER_PROPERTIES + TS_LUA_CONFIG_SSL_CLIENT_SNI_POLICY + TS_LUA_CONFIG_SSL_CLIENT_PRIVATE_KEY_FILENAME + TS_LUA_CONFIG_SSL_CLIENT_CA_CERT_FILENAME TS_LUA_CONFIG_LAST_ENTRY -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.http.timeout_set ------------------- @@ -3389,7 +3420,7 @@ Here is an example: end -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ Timeout constants ----------------- @@ -3403,7 +3434,7 @@ Timeout constants TS_LUA_TIMEOUT_NO_ACTIVITY -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.http.client_packet_mark_set ------------------------------ @@ -3422,7 +3453,7 @@ Here is an example: return 0 end -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.http.server_packet_mark_set ------------------------------ @@ -3433,7 +3464,7 @@ ts.http.server_packet_mark_set **description:** This function can be used to set packet mark for server connection. -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.http.client_packet_tos_set ----------------------------- @@ -3444,7 +3475,7 @@ ts.http.client_packet_tos_set **description:** This function can be used to set packet tos for client connection. -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.http.server_packet_tos_set ----------------------------- @@ -3455,7 +3486,7 @@ ts.http.server_packet_tos_set **description:** This function can be used to set packet tos for server connection. -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.http.client_packet_dscp_set ------------------------------ @@ -3466,7 +3497,7 @@ ts.http.client_packet_dscp_set **description:** This function can be used to set packet dscp for client connection. -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.http.server_packet_dscp_set ------------------------------ @@ -3477,7 +3508,7 @@ ts.http.server_packet_dscp_set **description:** This function can be used to set packet dscp for server connection. -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.http.enable_redirect ----------------------- @@ -3497,7 +3528,7 @@ Here is an example: end -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.http.set_debug ----------------- @@ -3517,7 +3548,7 @@ Here is an example: end -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.http.cntl_get ---------------- @@ -3532,7 +3563,7 @@ ts.http.cntl_get val = ts.http.cntl_get(TS_LUA_HTTP_CNTL_GET_LOGGING_MODE) -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.http.cntl_set ---------------- @@ -3552,7 +3583,7 @@ Here is an example: end -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ Http control channel constants ------------------------------ @@ -3566,7 +3597,7 @@ Http control channel constants TS_LUA_HTTP_CNTL_SET_INTERCEPT_RETRY_MODE -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.http.milestone_get --------------------- @@ -3582,7 +3613,7 @@ of seconds since the beginning of the transaction. val = ts.http.milestone_get(TS_LUA_MILESTONE_SM_START) -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ Milestone constants ------------------------------ @@ -3616,7 +3647,7 @@ Milestone constants TS_LUA_MILESTONE_TLS_HANDSHAKE_END -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.mgmt.get_counter ------------------- @@ -3630,7 +3661,7 @@ ts.mgmt.get_counter n = ts.mgmt.get_counter('proxy.process.http.incoming_requests') -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.mgmt.get_int --------------- @@ -3640,7 +3671,7 @@ ts.mgmt.get_int **description:** This function can be used to retrieve the record value which has a int type. -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.mgmt.get_float ----------------- @@ -3650,7 +3681,7 @@ ts.mgmt.get_float **description:** This function can be used to retrieve the record value which has a float type. -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.mgmt.get_string ------------------ @@ -3664,7 +3695,7 @@ ts.mgmt.get_string name = ts.mgmt.get_string('proxy.config.product_name') -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.stat_create -------------- @@ -3703,18 +3734,18 @@ Here is an example. return 0 end -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ ts.stat_find ------------ -**syntax:** *val = ts.stat_create(STAT_NAME)* +**syntax:** *val = ts.stat_find(STAT_NAME)* **context:** global **description:** This function can be used to find a statistics record given the name. A statistics record table will be returned with 4 functions to increment, decrement, get and set the value. That is similar to ts.stat_create() -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ Todo ==== @@ -3728,7 +3759,7 @@ as transaction hook instead. But this will have problem down the road when we ne together in some proper orderings. In the future, we should consider different approach, such as creating and maintaining the lua state in the ATS core. -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ Notes on Unit Testing Lua scripts for ATS Lua Plugin ==================================================== @@ -3758,11 +3789,11 @@ Reference for further information * luacov - https://luarocks.org/modules/hisham/luacov -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ More docs ========= * https://github.com/portl4t/ts-lua -`TOP <#ts-lua-plugin>`_ +`TOP <#lua-plugin>`_ diff --git a/doc/admin-guide/plugins/prefetch.en.rst b/doc/admin-guide/plugins/prefetch.en.rst index 93a7e3cf582..9b3b8c69c98 100644 --- a/doc/admin-guide/plugins/prefetch.en.rst +++ b/doc/admin-guide/plugins/prefetch.en.rst @@ -212,7 +212,7 @@ compromises: object** would be cached as well. * **Don't fetch the response body** and **never cache** at the **front-tier**. The **front-tier** marks the prefetch request with a special API header defined - by ``--api-header`` plugin parameter. When recieved the **back-tier** responds + by ``--api-header`` plugin parameter. When received the **back-tier** responds right away before actually fetching the object (without a body), it just schedules the real prefetch at the **back-tier**. ``Cache-Control: no-store`` is used to make sure the prefetch request response is never cached at the **front-tier**. diff --git a/doc/admin-guide/plugins/regex_revalidate.en.rst b/doc/admin-guide/plugins/regex_revalidate.en.rst index 50748820abb..a24fbeec349 100644 --- a/doc/admin-guide/plugins/regex_revalidate.en.rst +++ b/doc/admin-guide/plugins/regex_revalidate.en.rst @@ -24,8 +24,7 @@ Regex Revalidate Plugin This plugin allows for the creation of rules which match regular expressions against mapped URLs to determine if and when a cache object revalidation should -be forced. This plugin can be used both globally and for individual remap -rules. +be forced. Purpose ======= @@ -61,7 +60,7 @@ The rule configuration file format is described below in `Revalidation Rules`_. By default The plugin regularly (every 60 seconds) checks its rules configuration file for changes and it will also check for changes when ``traffic_ctl config reload`` -is run. If the file has been modified since its last scan, the contents +is run. If the file has been modified since its last scan, the contents are read and the in-memory rules list is updated. Thus, new rules may be added and existing ones modified without requiring a service restart. diff --git a/doc/admin-guide/plugins/slice.en.rst b/doc/admin-guide/plugins/slice.en.rst new file mode 100644 index 00000000000..58cc610cdd8 --- /dev/null +++ b/doc/admin-guide/plugins/slice.en.rst @@ -0,0 +1,217 @@ +.. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +.. _admin-plugins-slice: + +Slice Plugin +*************** + +This plugin takes client requests and breaks them up into +successive aligned block requests. This supports both +whole asset and single range requests. + +Purpose +======= + +This slice plugin, along with the `cache_range_requests` +plugin allows the following: + +- Fulfill arbitrary range requests by fetching a minimum + number of cacheable aligned blocks to fulfill the request. +- Breaks up very large assets into much smaller cache + blocks that can be spread across multiple storage + devices and within cache groups. + +Configuration +============= + +This plugin is intended for use as a remap plugin and is +configured in :file:`remap.config`. + +Or preferably per remap rule in :file:`remap.config`:: + + map http://ats/ http://parent/ @plugin=slice.so \ + @plugin=cache_range_requests.so + +In this case, the plugin will use the default behaviour: + +- Fulfill whole file or range requests by requesting cacheable + block aligned ranges from the parent and assemble them + into client responses, either 200 or 206 depending on the + client request. +- Default block size is 1mb (1048576 bytes). +- This plugin depends on the cache_range_requests plugin + to perform actual parent fetching and block caching + and If-* conditional header evaluations. + +Plugin Options +-------------- + +The slice plugin supports the following options:: + + --blockbytes= (optional) + Default is 1m or 1048576 bytes + -b for short. + Suffix k,m,g supported + Limited to 32k and 32m inclusive. + + --test-blockbytes= (optional) + Suffix k,m,g supported + -t for short. + Limited to any positive number. + Ignored if --blockbytes provided. + + --pace-errorlog= (optional) + Limit stitching error logs to every 'n' second(s) + + --disable-errorlog (optional) + Disable writing block stitch errors to the error log. + +Examples:: + + @plugin=slice.so @pparam=--blockbytes=1000000 @plugin=cache_range_requests.so + +Or alternatively:: + + @plugin=slice.so @pparam=-b @pparam=1000000 @plugin=cache_range_requests.so + +Byte suffix examples:: + + slice.so --blockbytes=5m + slice.so -b 512k + slice.so --blockbytes=32m + +For testing and extreme purposes the parameter ``test-blockbytes`` may +be used instead which is unchecked:: + + slice.so --test-blockbytes=1G + slice.so -t 13 + +Because the slice plugin is susceptible to errors during block stitching +extra logs related to stitching are written to ``diags.log``. Worst case +an error log entry could be generated for every transaction. The +following options are provided to help with log overrun:: + + slice.so --pace-errorlog=5 + slice.so -p 1 + slice.so --disable-errorlog + +After modifying :file:`remap.config`, restart or reload traffic server +(sudo traffic_ctl config reload) or (sudo traffic_ctl server restart) +to activate the new configuration values. + +Debug Options +------------- + +While the current slice plugin is able to detect block consistency +errors during the block stitching process, it can only abort the +client connection. A CDN can only "fix" these by issuing an appropriate +content revalidation. + +Under normal logging these slice block errors tend to show up as:: + + pscl value 0 + crc value ERR_READ_ERROR + +By default more detailed stitching errors are written to ``diags.log``. +An example is as follows:: + +[Apr 19 20:26:13.639] [ET_NET 17] ERROR: [slice] 1555705573.639 reason="Non 206 internal block response" uri="http://localhost:18080/%7Ep.tex/%7Es.50M/%7Eui.20000/" uas="curl/7.29.0" req_range="bytes=1000000-" norm_range="bytes 1000000-52428799/52428800" etag_exp="%221603934496%22" lm_exp="Fri, 19 Apr 2019 18:53:20 GMT" blk_range="21000000-21999999" status_got="400" cr_got="" etag_got="" lm_got="" cc="no-store" via="" + +Whether or how often these detailed log entries are written are +configurable plugin options. + +Implementation Notes +==================== + +This slice plugin is a stop gap plugin for handling special cases +involving very large assets that may be range requested. Hopefully +the slice plugin is deprecated in the future when partial object +caching is finally implemented. + +Slice *ONLY* handles slicing up requests into blocks, it delegates +actual caching and fetching to the cache_range_requests.so plugin. + +Plugin Function +--------------- + +Below is a quick functional outline of how a request is served +by a remap rule containing the Slice plugin with cache_range_requests: + +For each client request that comes in all remap plugins are run up +until the slice plugin is hit. If the slice plugin *can* be run (ie: +GET request) it will handle the request and STOP any further plugins +from executing. + +At this point the request is sliced into 1 or more blocks by +adding in range request headers ("Range: bytes="). A special +header X-Slicer-Info header is added and the pristine URL is +restored. + +For each of these blocks separate sequential TSHttpConnect(s) are made +back into the front end of ATS and all of the remap plugins are rerun. +Slice skips the remap due to presense of the X-Slicer-Info header and +allows cache_range_requests.so to serve the slice block back to Slice +either via cache OR parent request. + +Slice assembles a header based on the first slice block response and +sends it to the client. If necessary it then skips over bytes in +the first block and starts sending byte content, examining each +block header and sends its bytes to the client until the client +request is satisfied. + +Any extra bytes at the end of the last block are consumed by +the the Slice plugin to allow cache_range_requests to finish +the block fetch to ensure the block is cached. + +Important Notes +=============== + +This plugin assumes that the content requested is cacheable. + +Any first block server response that is not a 206 is passed directly +down to the client. If that response is a '200' only the first +portion of the response is passed back and the transaction is closed. + +Only the first server response block is used to evaluate any "If-" +conditional headers. Subsequent server slice block requests +remove these headers. + +The only 416 response that this plugin handles itself is if the +requested range is inside the last slice block but past the end of +the asset contents. Other 416 responses are handled by the parent. + +If a client aborts mid transaction the current slice block continues to +be read from the server until it is complete to ensure that the block +is cached. + +Slice *always* makes ``blockbytes`` sized requests which are handled +by cache_range_requests. The parent will trim those requests to +account for the asset Content-Length so only the appropriate number +of bytes are actually transferred and cached. + +Current Limitations +=================== + +By restoring the prisine Url the plugin as it works today reuses the +same remap rule for each slice block. This is wasteful in that it reruns +the previous remap rules, and those remap rules must be smart enough to +check for the existence of any headers they may have created the +first time they have were visited. + +Since the Slice plugin is written as an intercept handler it loses the +ability to use normal state machine hooks and transaction states. diff --git a/doc/admin-guide/plugins/sslheaders.en.rst b/doc/admin-guide/plugins/sslheaders.en.rst index 8fc08816ca4..49a8777b5f3 100644 --- a/doc/admin-guide/plugins/sslheaders.en.rst +++ b/doc/admin-guide/plugins/sslheaders.en.rst @@ -66,7 +66,7 @@ The `client.certificate` and `server.certificate` fields emit the corresponding certificate in PEM format, with newline characters replaced by spaces. -If the ``sslheaders`` plugin activtes on non-SSL connections, it +If the ``sslheaders`` plugin activates on non-SSL connections, it will delete all the configured HTTP header names so that malicious clients cannot inject misleading information. If any of the SSL fields expand to an empty string, those headers are also deleted. diff --git a/doc/admin-guide/plugins/stale_while_revalidate.en.rst b/doc/admin-guide/plugins/stale_while_revalidate.en.rst deleted file mode 100644 index 76e192bf663..00000000000 --- a/doc/admin-guide/plugins/stale_while_revalidate.en.rst +++ /dev/null @@ -1,25 +0,0 @@ -.. _admin-plugins-tale-while-revalidate: - -Stale While Revalidate Plugin -***************************** - - :deprecated: - -.. Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. - -refresh content asynchronously while serving stale data diff --git a/doc/admin-guide/plugins/traffic_dump.en.rst b/doc/admin-guide/plugins/traffic_dump.en.rst index be57f4b750b..2d012f11855 100644 --- a/doc/admin-guide/plugins/traffic_dump.en.rst +++ b/doc/admin-guide/plugins/traffic_dump.en.rst @@ -36,14 +36,20 @@ Plugin Configuration * ``Traffic Dump`` is a global plugin and is configured via :file:`plugin.config`. .. option:: --logdir - (`required`, default:empty/unused) - specifies the directory for writing all dump files. If path is relative, it is relative to the Traffic Server directory. The plugin will use first three characters of client ip to create subdirs in an attempt to spread dumps evenly and avoid too many files in a single directory. + (`required`) - specifies the directory for writing all dump files. If path is relative, it is relative to the Traffic Server directory. The plugin will use first three characters of client ip to create subdirs in an attempt to spread dumps evenly and avoid too many files in a single directory. .. option:: --sample - (`optional`, default:1000) - specifies the sampling ratio N. Traffic Dump will capture every one out of N sessions. This ratio can also be changed via traffic_ctl without restarting ATS. + (`required`) - specifies the sampling ratio N. Traffic Dump will capture every one out of N sessions. This ratio can also be changed via traffic_ctl without restarting ATS. + + .. option:: --limit + + (`required`) - specifies the max disk usage N bytes(approximate). Traffic Dump will stop capturing new sessions once disk usage exceeds this limit. * ``traffic_ctl`` command. ``traffic_ctl plugin msg traffic_dump.sample N`` - changes the sampling ratio N as mentioned above. + ``traffic_ctl plugin msg traffic_dump.reset`` - resets the disk usage counter. + ``traffic_ctl plugin msg traffic_dump.limit N`` - changes the max disk usage. Replay Format ============= diff --git a/doc/admin-guide/plugins/xdebug.en.rst b/doc/admin-guide/plugins/xdebug.en.rst index 34f77a87189..a9eeac71a14 100644 --- a/doc/admin-guide/plugins/xdebug.en.rst +++ b/doc/admin-guide/plugins/xdebug.en.rst @@ -57,11 +57,20 @@ Diags transaction specific diagnostics for the transaction. This also requires that :ts:cv:`proxy.config.diags.debug.enabled` is set to ``1``. -log-headers - If the ``log-headers`` is requested while :ts:cv:`proxy.config.diags.debug.tags` - is set to ``xdebug.headers`` and :ts:cv:`proxy.config.diags.debug.enabled` is set to ``1``, - then all client and server, request and response headers are logged. - Also, the ``X-Debug: log-headers`` header is always added to the upstream request. +Probe + All request and response headers are written to the response body. Because + the body is altered, it disables writing to cache. + In conjuction with the `fwd` tag, the response body will contain a + chronological log of all headers for all transactions used for this + response. + + Layout: + + - Request Headers from Client -> Proxy A + - Request Headers from Proxy A -> Proxy B + - Original content body + - Response Headers from Proxy B -> Proxy A + - Response Headers from Proxy A -> Client X-Cache-Key The ``X-Cache-Key`` header contains the URL that identifies the HTTP object in the diff --git a/doc/admin-guide/security/index.en.rst b/doc/admin-guide/security/index.en.rst index 411a924a448..84586c2ebf1 100644 --- a/doc/admin-guide/security/index.en.rst +++ b/doc/admin-guide/security/index.en.rst @@ -109,15 +109,15 @@ Client/Traffic Server connections, you must do the following: #. Set the appropriate base path for your SSL certificates and private keys in :file:`records.config`. :: - CONFIG proxy.config.ssl.server.cert.path STRING "/opt/ts/etc/ssl/certs/" - CONFIG proxy.config.ssl.server.private_key.path STRING "/opt/ts/etc/ssl/keys/" + CONFIG proxy.config.ssl.server.cert.path STRING /opt/ts/etc/ssl/certs/ + CONFIG proxy.config.ssl.server.private_key.path STRING /opt/ts/etc/ssl/keys/ #. Add an entry to :file:`ssl_multicert.config` for each certificate and key which your Traffic Server system will be using to terminate SSL connections with clients. :: - ip_dest=1.2.3.4 ssl_cert_name=example.com.pem - ip_dest=* ssl_cert_name=default.pem + dest_ip=1.2.3.4 ssl_cert_name=example.com.pem + dest_ip=* ssl_cert_name=default.pem #. *Optional*: Configure the use of client certificates using the variable :ts:cv:`proxy.config.ssl.client.certification_level` in :file:`records.config`. @@ -146,7 +146,7 @@ Client/Traffic Server connections, you must do the following: The list of acceptable CA signers is configured with :ts:cv:`proxy.config.ssl.CA.cert.path` in :file:`records.config`. :: - CONFIG proxy.config.ssl.CA.cert.path STRING "/opt/CA/certs/private-ca.pem" + CONFIG proxy.config.ssl.CA.cert.path STRING /opt/CA/certs/private-ca.pem #. Run the command :option:`traffic_ctl server restart` to restart Traffic Server. @@ -208,16 +208,16 @@ and origin server connections, you must do the following: :file:`records.config` in the setting :ts:cv:`proxy.config.ssl.client.cert.path` and :ts:cv:`proxy.config.ssl.client.cert.filename`. :: - CONFIG proxy.config.ssl.client.cert.path STRING "/opt/ts/etc/ssl/certs/" - CONFIG proxy.config.ssl.client.cert.filename STRING "client.pem" + CONFIG proxy.config.ssl.client.cert.path STRING /opt/ts/etc/ssl/certs/ + CONFIG proxy.config.ssl.client.cert.filename STRING client.pem You must also provide the paths to the private key for this certificate, unless the key is contained within the same file as the certificate, using :ts:cv:`proxy.config.ssl.client.private_key.path` and :ts:cv:`proxy.config.ssl.client.private_key.filename`. :: - CONFIG proxy.config.ssl.client.private_key.path STRING "/opt/ts/etc/ssl/keys/" - CONFIG proxy.config.ssl.client.private_key.filename STRING "client.pem" + CONFIG proxy.config.ssl.client.private_key.path STRING /opt/ts/etc/ssl/keys/ + CONFIG proxy.config.ssl.client.private_key.filename STRING client.pem #. Enable or disable, per your security policy, server SSL certificate verification using :ts:cv:`proxy.config.ssl.client.verify.server.policy` in @@ -230,8 +230,8 @@ and origin server connections, you must do the following: :ts:cv:`proxy.config.ssl.client.CA.cert.path` and :ts:cv:`proxy.config.ssl.client.CA.cert.filename`. :: - CONFIG proxy.config.ssl.client.CA.cert.path STRING "/opt/ts/etc/ssl/certs/" - CONFIG proxy.config.ssl.client.CA.cert.filename STRING "CAs.pem" + CONFIG proxy.config.ssl.client.CA.cert.path STRING /opt/ts/etc/ssl/certs/ + CONFIG proxy.config.ssl.client.CA.cert.filename STRING CAs.pem #. Run the command :option:`traffic_ctl server restart` to restart Traffic Server. @@ -281,6 +281,13 @@ Authority Information Access field of the signed certificate. For example:: OCSP - URI:http://ocsp.digicert.com CA Issuers - URI:http://cacerts.digicert.com/DigiCertSHA2SecureServerCA.crt +Traffic Server can also use prefetched OCSP stapling responses if ssl_ocsp_name parameter +is used in :file:`ssl_multicert.config`. Take into account that when using prefetched +OCSP stapling responses traffic server will not refresh them and it should be done +externally. This can be done using openssl: + $ openssl ocsp -issuer ca.crt -cert cert.crt -host ocsp.digicert.com:80 \ + -header "Host=ocsp.digicert.com" -respout /var/cache/ocsp/cert.ocsp + Support for OCSP Stapling can be tested using the -status option of the OpenSSL client:: $ openssl s_client -connect mozillalabs.com:443 -status @@ -301,6 +308,7 @@ in :file:`records.config` file: * :ts:cv:`proxy.config.ssl.ocsp.cache_timeout` * :ts:cv:`proxy.config.ssl.ocsp.request_timeout` * :ts:cv:`proxy.config.ssl.ocsp.update_period` +* :ts:cv:`proxy.config.ssl.ocsp.response.path` .. _admin-split-dns: diff --git a/doc/appendices/command-line/traffic_ctl.en.rst b/doc/appendices/command-line/traffic_ctl.en.rst index 5304b6ef6c6..6539eda726d 100644 --- a/doc/appendices/command-line/traffic_ctl.en.rst +++ b/doc/appendices/command-line/traffic_ctl.en.rst @@ -255,7 +255,7 @@ traffic_ctl host .. program:: traffic_ctl host .. option:: status HOSTNAME [HOSTNAME ...] - Get the current status of the hosts used in parent.config as a next hop in a multi-tiered cache heirarchy. The value 0 or 1 is returned indicating that the host is marked as down '0' or marked as up '1'. If a host is marked as down, it will not be used as the next hop parent, another host marked as up will be chosen. + Get the current status of the hosts used in parent.config as a next hop in a multi-tiered cache hierarchy. The value 0 or 1 is returned indicating that the host is marked as down '0' or marked as up '1'. If a host is marked as down, it will not be used as the next hop parent, another host marked as up will be chosen. .. program:: traffic_ctl host .. option:: down --time seconds --reason 'manual|active|local' HOSTNAME [HOSTNAME ...] diff --git a/doc/appendices/command-line/traffic_layout.en.rst b/doc/appendices/command-line/traffic_layout.en.rst index 3d4fedd33b2..443c8235823 100644 --- a/doc/appendices/command-line/traffic_layout.en.rst +++ b/doc/appendices/command-line/traffic_layout.en.rst @@ -21,13 +21,11 @@ traffic_layout ************** -======== Synopsis ======== :program:`traffic_layout` SUBCOMMAND [OPTIONS] -=========== Environment =========== @@ -35,15 +33,13 @@ Environment The path to the run root file. It has the same effect as the command line option :option:`--run-root`. -=========== Description =========== -Document for the special functionality of ``runroot`` inside :program:`traffic_layout`. This feature -is for the setup of traffic server runroot. It will create a runtime sandbox for any program of -traffic server to run under. +Document for the :program:`traffic_layout` about the ``runroot``. This feature is for the setup of traffic server runroot. +It will create a runtime sandbox for any program of traffic server to run under. +For details about runroot for programs, please refer to ``developer-guide/layout/runroot.en``. -===== Usage ===== @@ -54,7 +50,7 @@ First we need to create a runroot. It can be created simply by calling command ` A runroot will be created in ``/path/to/runroot``, available for other programs to use. If the path is not specified, the current working directory will be used. -To run traffic_manager, for example, using the runroot, there are several ways: +For example, to run traffic_manager, using the runroot, there are several ways: #. ``/path/to/runroot/bin/traffic_manager`` #. ``traffic_manager --run-root=/path/to/runroot`` #. ``traffic_manager --run-root=/path/to/runroot/runroot.yaml`` @@ -65,7 +61,6 @@ To run traffic_manager, for example, using the runroot, there are several ways: if none of the above is found as runroot, runroot will not be used and the program will fall back to the default. -=========== Subcommands =========== @@ -127,7 +122,13 @@ Example: :: traffic_layout verify (--path /path/to/sandbox/) (--fix) (--with-user root) -======= +.. Warning:: + + If a custom layout is used and system files are included in some directories, ``--fix`` option might potentially have unexpected behaviors. + For example, if sysconfdir is defined as ``/etc`` instead of ``/etc/trafficserver`` in ``runroot.yaml``, + ``--fix`` may perform permission changes on the system configuration files. With normally created runroot with default layout, + there is no such issue since traffic server related files are filtered. + Options ======= diff --git a/doc/appendices/command-line/traffic_manager.en.rst b/doc/appendices/command-line/traffic_manager.en.rst index 4011298bbd7..61a7fbdf1f7 100644 --- a/doc/appendices/command-line/traffic_manager.en.rst +++ b/doc/appendices/command-line/traffic_manager.en.rst @@ -36,6 +36,7 @@ Description .. option:: --path FILE .. option:: --proxyBackDoor PORT .. option:: --proxyOff +.. option:: --listenOff .. option:: --proxyPort PORT .. option:: --recordsConf FILE .. option:: --tsArgs ARGUMENTS diff --git a/doc/appendices/command-line/traffic_server.en.rst b/doc/appendices/command-line/traffic_server.en.rst index 247dfca2fe6..75e17e8f8f2 100644 --- a/doc/appendices/command-line/traffic_server.en.rst +++ b/doc/appendices/command-line/traffic_server.en.rst @@ -52,8 +52,7 @@ environments or where the working set is highly variable. .. option:: -F, --disable_pfreelist Disable free list in ProxyAllocator which were left out by the -f -option. Please note that this option is a temporary, testing -option, and will be removed in the future. +option. This option includes the functionality of :option:`-f`. .. option:: -R LEVEL, --regression LEVEL diff --git a/doc/appendices/glossary.en.rst b/doc/appendices/glossary.en.rst index 282d00b677c..f4ae4b8d9bb 100644 --- a/doc/appendices/glossary.en.rst +++ b/doc/appendices/glossary.en.rst @@ -35,6 +35,16 @@ Glossary invoked to continue the suspended processing. This can be considered similar to co-routines. + event loop + Code that executes callbacks in continuations from a queue of events. + + event thread + A thread created by |TS| that has an :term:`event loop`. Event loops drive activity in |TS| + and are responsible for all network I/O handling, hook processing, and scheduled events. + + header heap + A heap to manage transaction local memory for HTTP headers. + session A single connection from a client to Traffic Server, covering all requests and responses on that connection. A session starts when the @@ -158,3 +168,10 @@ Glossary A plugin which operates only on transactions matching specific remap rules as defined in :file:`remap.config`. Contrast with :term:`global plugin`. + + variable sized class + A class where the instances vary in size. This is done by allocating a block of memory at least + as large as the class and then constructing a class instance at the start of the block. The + class must be provided the size of this extended memory during construction and is presumed + to use it as part of the instance. This generally requires calling a helper function to + create the instance and extra care when de-allocating. diff --git a/doc/conf.py b/doc/conf.py index 11884b6f4f1..54e2ceec6a8 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -161,7 +161,7 @@ #show_authors = False # The name of the Pygments (syntax highlighting) style to use. -pygments_style = 'sphinx' +pygments_style = 'default' # A list of ignored prefixes for module index sorting. #modindex_common_prefix = [] diff --git a/doc/developer-guide/api/functions/TSCacheRemove.en.rst b/doc/developer-guide/api/functions/TSCacheRemove.en.rst index 73cfd7d20b7..f5647d59a32 100644 --- a/doc/developer-guide/api/functions/TSCacheRemove.en.rst +++ b/doc/developer-guide/api/functions/TSCacheRemove.en.rst @@ -41,4 +41,4 @@ the cache calls :arg:`contp` back with the event In both of these callbacks, the user (:arg:`contp`) does not have to do anything. The user does not get any vconnection from the cache, since no data needs to be transferred. When the cache calls :arg:`contp` back with -:data:`TS_EVENT_CACHE_REMOVE`, the remove has already been commited. +:data:`TS_EVENT_CACHE_REMOVE`, the remove has already been committed. diff --git a/doc/developer-guide/api/functions/TSContCall.en.rst b/doc/developer-guide/api/functions/TSContCall.en.rst index 9da6a319447..8f99332ac27 100644 --- a/doc/developer-guide/api/functions/TSContCall.en.rst +++ b/doc/developer-guide/api/functions/TSContCall.en.rst @@ -44,24 +44,17 @@ As a result :func:`TSContCall` will effectively do:: return CallbackHandler(contp, event, edata); -:func:`TSContCall` will check :arg:`contp` for a mutex. If there is a mutex an attempt will be made -to lock that mutex. If either there is no mutex, or the mutex lock was acquired, the handler will be -called directly. Otherwise there is a mutex and it was not successfully locked. In that case an -event will be scheduled to dispatch as soon as possible, but not in the current call stack. The -nature of event dispatch means the event will not be dispatched until the mutext can be locked. In -all cases the handler in :arg:`contp` will be called with the same arguments. :func:`TSContCall` -will return 0 if a mutex was present but the lock was not acquired. Otherwise it will return the +If there is a mutex associated with :arg:`contp`, :func:`TSContCall` assumes that mutex is held already. +:func:`TSContCall` will directly call the handler associated with the continuation. It will return the value returned by the handler in :arg:`contp`. -If the scheduling behavior of :func:`TSContCall` isn't appropriate, either :arg:`contp` must not -have a mutex, or the plugin must acquire the lock on the mutex for :arg:`contp` before calling -:func:`TSContCall`. See :func:`TSContMutexGet` and :func:`TSMutexLockTry` for mechanisms for doing -the latter. This is what :func:`TSContCall` does internally, and should be done by the plugin only -if a different approach for waiting for the lock is needed. The most common case is the code called -by :func:`TSContCall` must complete before further code is executed at the call site. An alternative -approach to handling the locking directly would be to split the call site in to two continuations, -one of which is signalled (possibly via :func:`TSContCall`) from the original :func:`TSContCall` -target. +If :arg:`contp` has a mutex, the plugin must acquire the lock on the mutex for :arg:`contp` before calling +:func:`TSContCall`. See :func:`TSContMutexGet` and :func:`TSMutexLockTry` for mechanisms for doing this. + +The most common case is the code called by :func:`TSContCall` must complete before further code is executed +at the call site. An alternative approach to handling the locking directly would be to split the call site +into two continuations, one of which is signalled (possibly via :func:`TSContCall`) from the original +:func:`TSContCall` target. Note mutexes returned by :func:`TSMutexCreate` are recursive mutexes, therefore if the lock is already held on the thread of execution acquiring the lock again is very fast. Mutexes are also diff --git a/doc/developer-guide/api/functions/TSContSchedule.en.rst b/doc/developer-guide/api/functions/TSContSchedule.en.rst index 27820cdcb5d..dea45b01004 100644 --- a/doc/developer-guide/api/functions/TSContSchedule.en.rst +++ b/doc/developer-guide/api/functions/TSContSchedule.en.rst @@ -26,39 +26,24 @@ Synopsis `#include ` -.. function:: TSAction TSContSchedule(TSCont contp, ink_hrtime delay, TSThreadPool tp) +.. function:: TSAction TSContSchedule(TSCont contp, TSHRTime timeout) Description =========== Schedules :arg:`contp` to run :arg:`delay` milliseconds in the future. This is approximate. The delay -will be at least :arg:`delay` but possibly more. Resultions finer than roughly 5 milliseconds will +will be at least :arg:`delay` but possibly more. Resolutions finer than roughly 5 milliseconds will not be effective. :arg:`contp` is required to have a mutex, which is provided to :func:`TSContCreate`. The return value can be used to cancel the scheduled event via :func:`TSActionCancel`. This is effective until the continuation :arg:`contp` is being dispatched. However, if it is scheduled on -another thread this can problematic to be correctly timed. The return value can be checked with +another thread this can be problematic to be correctly timed. The return value can be checked with :func:`TSActionDone` to see if the continuation ran before the return, which is possible if -:arg:`delay` is `0`. +:arg:`timeout` is `0`. Returns ``nullptr`` if thread affinity was cleared. -The continuation is scheduled for a particular thread selected from a group of similar threads, as indicated by :arg:`tp`. - -=========================== ======================================================================================= -Pool Properties -=========================== ======================================================================================= -``TS_THREAD_POOL_DEFAULT`` Use the default pool. Continuations using this must not block. -``TS_THREAD_POOL_NET`` Transaction processing threads. Continuations on these threads must not block. -``TS_THREAD_POOL_TASK`` Background threads. Continuations can perform blocking operations. -``TS_THREAD_POOL_SSL`` *DEPRECATED* - these are no longer used as of ATS 6. -``TS_THREAD_POOL_DNS`` DNS request processing. May not exist depending on configuration. Not recommended. -``TS_THREAD_POOL_REMAP`` *DEPRECATED* - these are not longer used. -``TS_THREAD_POOL_CLUSTER`` *DEPRECATED* - these are no longer used as of ATS 7. -``TS_THREAD_POOL_UDP`` *DEPRECATED* -=========================== ======================================================================================= +See Also +======== -In practice, any choice except ``TS_THREAD_POOL_NET`` or ``TS_THREAD_POOL_TASK`` is strong not -recommended. The ``TS_THREAD_POOL_NET`` threads are the same threads on which callback hooks are -called and continuations that use them have the same restrictions. ``TS_THREAD_POOL_TASK`` threads -are threads that exist to perform long or blocking actions, although sufficiently long operation can -impact system performance by blocking other continuations on the threads. +:doc:`TSContScheduleOnPool.en` +:doc:`TSContScheduleOnThread.en` diff --git a/doc/developer-guide/api/functions/TSContScheduleOnPool.en.rst b/doc/developer-guide/api/functions/TSContScheduleOnPool.en.rst new file mode 100644 index 00000000000..f35ddb95fa9 --- /dev/null +++ b/doc/developer-guide/api/functions/TSContScheduleOnPool.en.rst @@ -0,0 +1,126 @@ +.. Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed + with this work for additional information regarding copyright + ownership. The ASF licenses this file to you under the Apache + License, Version 2.0 (the "License"); you may not use this file + except in compliance with the License. You may obtain a copy of + the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied. See the License for the specific language governing + permissions and limitations under the License. + +.. include:: ../../../common.defs + +.. default-domain:: c + +TSContScheduleOnPool +******************** + +Synopsis +======== + +`#include ` + +.. function:: TSAction TSContScheduleOnPool(TSCont contp, TSHRTime timeout, TSThreadPool tp) + +Description +=========== + +Mostly the same as :func:`TSContSchedule`. Schedules :arg:`contp` on a random thread that belongs to :arg:`tp`. +If thread type of the thread specified by thread affinity is the same as :arg:`tp`, the :arg:`contp` will +be scheduled on the thread specified by thread affinity. + +The continuation is scheduled for a particular thread selected from a group of similar threads, as indicated by :arg:`tp`. + +=========================== ======================================================================================= +Pool Properties +=========================== ======================================================================================= +``TS_THREAD_POOL_NET`` Transaction processing threads. Continuations on these threads must not block. +``TS_THREAD_POOL_TASK`` Background threads. Continuations can perform blocking operations. +``TS_THREAD_POOL_SSL`` *DEPRECATED* - these are no longer used as of ATS 6. +``TS_THREAD_POOL_DNS`` DNS request processing. May not exist depending on configuration. Not recommended. +``TS_THREAD_POOL_REMAP`` *DEPRECATED* - these are no longer used. +``TS_THREAD_POOL_CLUSTER`` *DEPRECATED* - these are no longer used as of ATS 7. +``TS_THREAD_POOL_UDP`` *DEPRECATED* +=========================== ======================================================================================= + +In practice, any choice except ``TS_THREAD_POOL_NET`` or ``TS_THREAD_POOL_TASK`` is strongly not +recommended. The ``TS_THREAD_POOL_NET`` threads are the same threads on which callback hooks are +called and continuations that use them have the same restrictions. ``TS_THREAD_POOL_TASK`` threads +are threads that exist to perform long or blocking actions, although sufficiently long operation can +impact system performance by blocking other continuations on the threads. + +Example Scenarios +================= + +Scenario 1 (no thread affinity info, different types of threads) +---------------------------------------------------------------- + +When thread affinity is not set, a plugin calls the API on thread "A" (which is an "ET_TASK" type), and +wants to schedule on an "ET_NET" type thread provided in "tp", the system would see there is no thread +affinity information stored in "contp." + +In this situation, system sees there is no thread affinity information stored in "contp". It then +checks whether the type of thread "A" is the same as provided in "tp", and sees that "A" is "ET_TASK", +but "tp" says "ET_NET". So "contp" gets scheduled on the next available "ET_NET" thread provided by a +round robin list, which we will call thread "B". Since "contp" doesn't have thread affinity information, +thread "B" will be assigned as the affinity thread for it automatically. + +The reason for doing this is most of the time people want to schedule the same things on the same type +of thread, so logically it is better to default the first thread that it is scheduled on as the affinity +thread. + +Scenario 2 (no thread affinity info, same types of threads) +----------------------------------------------------------- + +Slight variation of scenario 1, instead of scheduling on a "ET_NET" thread, the plugin wants to schedule +on a "ET_TASK" thread (i.e. "tp" contains "ET_TASK" now), all other conditions stays the same. + +This time since the type of the desired thread for scheduling and thread "A" are the same, the system +schedules "contp" on thread "A", and assigns thread "A" as the affinity thread for "contp". + +The reason behind this choice is that we are trying to keep things simple such that lock contention +problems happens less. And for the most part, there is no point of scheduling the same thing on several +different threads of the same type, because there is no parallelism between them (a thread will have to +wait for the previous thread to finish, either because locking or the nature of the job it’s handling is +serialized since its on the same continuation). + +Scenario 3 (has thread affinity info, different types of threads) +----------------------------------------------------------------- + +Slight variation of scenario 1, thread affinity is set for continuation "contp" to thread "A", all other +conditions stays the same. + +In this situation, the system sees that the "tp" has "ET_NET", but the type of thread "A" is "ET_TASK". +So even though "contp" has an affinity thread, the system will not use that information since the type is +different, instead it schedules "contp" on the next available "ET_NET" thread provided by a round robin +list, which we will call thread "B". The difference with scenario 1 is that since thread "A" is set to +be the affinity thread for "contp" already, the system will NOT overwrite that information with thread "B". + +Most of the time, a continuation will be scheduled on one type of threads, and rarely gets scheduled on +a different type. But when that happens, we want it to return to the thread it was previously on, so it +won’t have any lock contention problems. And that’s also why "thread_affinity" is not a hashmap of thread +types and thread pointers. + +Scenario 4 (has thread affinity info, same types of threads) +------------------------------------------------------------ + +Slight variation of scenario 3, the only difference is "tp" now says "ET_TASK". + +This is the easiest scenario since the type of thread "A" and "tp" are the same, so the system schedules +"contp" on thread "A". And, as discussed, there is really no reason why one may want to schedule +the same continuation on two different threads of the same type. + +.. note:: In scenario 3 & 4, it doesn't matter which thread the plugin is calling the API from. + + +See Also +======== + +:doc:`TSContSchedule.en` +:doc:`TSContScheduleOnThread.en` diff --git a/doc/developer-guide/api/functions/TSContScheduleOnThread.en.rst b/doc/developer-guide/api/functions/TSContScheduleOnThread.en.rst new file mode 100644 index 00000000000..fed6313c4e8 --- /dev/null +++ b/doc/developer-guide/api/functions/TSContScheduleOnThread.en.rst @@ -0,0 +1,40 @@ +.. Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed + with this work for additional information regarding copyright + ownership. The ASF licenses this file to you under the Apache + License, Version 2.0 (the "License"); you may not use this file + except in compliance with the License. You may obtain a copy of + the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied. See the License for the specific language governing + permissions and limitations under the License. + +.. include:: ../../../common.defs + +.. default-domain:: c + +TSContScheduleOnThread +********************** + +Synopsis +======== + +`#include ` + +.. function:: TSAction TSContScheduleOnThread(TSCont contp, TSHRTime timeout, TSEventThread ethread) + +Description +=========== + +Mostly the same as :func:`TSContSchedule`. Schedules :arg:`contp` on :arg:`ethread`. + +See Also +======== + +:doc:`TSContSchedule.en` +:doc:`TSContScheduleOnPool.en` diff --git a/doc/developer-guide/api/functions/TSContThreadAffinityClear.en.rst b/doc/developer-guide/api/functions/TSContThreadAffinityClear.en.rst new file mode 100644 index 00000000000..ef17ae0cddc --- /dev/null +++ b/doc/developer-guide/api/functions/TSContThreadAffinityClear.en.rst @@ -0,0 +1,34 @@ +.. Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed + with this work for additional information regarding copyright + ownership. The ASF licenses this file to you under the Apache + License, Version 2.0 (the "License"); you may not use this file + except in compliance with the License. You may obtain a copy of + the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied. See the License for the specific language governing + permissions and limitations under the License. + +.. include:: ../../../common.defs + +.. default-domain:: c + +TSContThreadAffinityClear +************************* + +Synopsis +======== + +`#include ` + +.. function:: void TSContThreadAffinityClear(TSCont contp) + +Description +=========== + +Clear the thread affinity of continuation :arg:`contp`. diff --git a/doc/developer-guide/api/functions/TSContThreadAffinityGet.en.rst b/doc/developer-guide/api/functions/TSContThreadAffinityGet.en.rst new file mode 100644 index 00000000000..2bc4955ad19 --- /dev/null +++ b/doc/developer-guide/api/functions/TSContThreadAffinityGet.en.rst @@ -0,0 +1,34 @@ +.. Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed + with this work for additional information regarding copyright + ownership. The ASF licenses this file to you under the Apache + License, Version 2.0 (the "License"); you may not use this file + except in compliance with the License. You may obtain a copy of + the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied. See the License for the specific language governing + permissions and limitations under the License. + +.. include:: ../../../common.defs + +.. default-domain:: c + +TSContThreadAffinityGet +*********************** + +Synopsis +======== + +`#include ` + +.. function:: TSEventThread TSContThreadAffinityGet(TSCont contp) + +Description +=========== + +Get the thread affinity of continuation :arg:`contp`. diff --git a/doc/developer-guide/api/functions/TSContThreadAffinitySet.en.rst b/doc/developer-guide/api/functions/TSContThreadAffinitySet.en.rst new file mode 100644 index 00000000000..5b355130af8 --- /dev/null +++ b/doc/developer-guide/api/functions/TSContThreadAffinitySet.en.rst @@ -0,0 +1,42 @@ +.. Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed + with this work for additional information regarding copyright + ownership. The ASF licenses this file to you under the Apache + License, Version 2.0 (the "License"); you may not use this file + except in compliance with the License. You may obtain a copy of + the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied. See the License for the specific language governing + permissions and limitations under the License. + +.. include:: ../../../common.defs + +.. default-domain:: c + +TSContThreadAffinitySet +*********************** + +Synopsis +======== + +`#include ` + +.. function:: TSReturnCode TSContThreadAffinitySet(TSCont contp, TSEventThread ethread) + +Description +=========== + +Set the thread affinity of continuation :arg:`contp` to :arg:`ethread`. Future calls to +:func:`TSContSchedule`, and :func:`TSContScheduleOnPool` that has the same type as :arg:`ethread` +will schedule the continuation on :arg:`ethread`, rather than an arbitrary thread of that type. + +Return Values +============= + +:data:`TS_SUCCESS` if thread affinity of continuation :arg:`contp` was set to :arg:`ethread`, +:data:`TS_ERROR` if not. diff --git a/doc/developer-guide/api/functions/TSEventThreadSelf.en.rst b/doc/developer-guide/api/functions/TSEventThreadSelf.en.rst new file mode 100644 index 00000000000..0c7728aac6b --- /dev/null +++ b/doc/developer-guide/api/functions/TSEventThreadSelf.en.rst @@ -0,0 +1,36 @@ +.. Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed + with this work for additional information regarding copyright + ownership. The ASF licenses this file to you under the Apache + License, Version 2.0 (the "License"); you may not use this file + except in compliance with the License. You may obtain a copy of + the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied. See the License for the specific language governing + permissions and limitations under the License. + +.. include:: ../../../common.defs + +.. default-domain:: c + +TSEventThreadSelf +***************** + +Synopsis +======== + +`#include ` + +.. function:: TSEventThread TSEventThreadSelf(void) + +Description +=========== + +Return a value that refers to the current :term:`event thread`. This must be called from a thread +with an :term:`event loop`. This will be true from any callback invoked from a hook or a scheduled +event. diff --git a/doc/developer-guide/api/functions/TSHttpConnectWithPluginId.en.rst b/doc/developer-guide/api/functions/TSHttpConnectWithPluginId.en.rst index 88e805cb328..0f6186f5ec9 100644 --- a/doc/developer-guide/api/functions/TSHttpConnectWithPluginId.en.rst +++ b/doc/developer-guide/api/functions/TSHttpConnectWithPluginId.en.rst @@ -79,7 +79,7 @@ virtual connection. The combination of :arg:`tag` and :arg:`id` is intended to enable correlation in log post processing. The :arg:`tag` identifies the connection as related -to the plugin and the :arg:`id` can be used in conjuction with plugin +to the plugin and the :arg:`id` can be used in conjunction with plugin generated logs to correlate the log records. Notes diff --git a/doc/developer-guide/api/functions/TSHttpHdrLengthGet.en.rst b/doc/developer-guide/api/functions/TSHttpHdrLengthGet.en.rst index 9db63fdef81..495ef1a991c 100644 --- a/doc/developer-guide/api/functions/TSHttpHdrLengthGet.en.rst +++ b/doc/developer-guide/api/functions/TSHttpHdrLengthGet.en.rst @@ -26,7 +26,16 @@ Synopsis `#include ` -.. function:: int TSHttpHdrLengthGet(TSMBuffer bufp, TSMLoc offset) +.. function:: int TSHttpHdrLengthGet(TSMBuffer bufp, TSMLoc mloc) Description =========== + +Return the length in characters of the HTTP header specified by :arg:`bufp` and :arg:`mloc` which +must specify a valid HTTP header. Usually these values would have been obtained via an earlier call +to +:func:`TSHttpTxnServerReqGet`, +:func:`TSHttpTxnClientReqGet`, +:func:`TSHttpTxnServerRespGet`, +:func:`TSHttpTxnClientRespGet`, +or via calls to create a new HTTP header such as :func:`TSHttpHdrCreate`. diff --git a/doc/developer-guide/api/functions/TSHttpHdrStatusGet.en.rst b/doc/developer-guide/api/functions/TSHttpHdrStatusGet.en.rst index 50b2f1da2c6..7d2e31dbfbe 100644 --- a/doc/developer-guide/api/functions/TSHttpHdrStatusGet.en.rst +++ b/doc/developer-guide/api/functions/TSHttpHdrStatusGet.en.rst @@ -30,3 +30,13 @@ Synopsis Description =========== + +Retrieve the status code value from the HTTP response header identified by :arg:`bufp` and +:arg:`offset`. The value should be an enumeration value of :c:type:`TSHttpStatus`, although because +plugins can call :c:func:`TSHttpHdrStatusSet` this may not be true. If the header is not a valid +response then :c:macro:`TS_HTTP_STATUS_NONE` is returned. + +See Also +======== + +:c:func:`TSHttpTxnClientRespGet`, :c:func:`TSHttpTxnServerRespGet`, :c:func:`TSHttpHdrTypeGet`. diff --git a/doc/developer-guide/api/functions/TSHttpHookAdd.en.rst b/doc/developer-guide/api/functions/TSHttpHookAdd.en.rst index 6435be88955..3019be013be 100644 --- a/doc/developer-guide/api/functions/TSHttpHookAdd.en.rst +++ b/doc/developer-guide/api/functions/TSHttpHookAdd.en.rst @@ -45,7 +45,7 @@ transaction, or for specific transactions only. HTTP :term:`transaction` hooks are set on a global basis using the function :func:`TSHttpHookAdd`. This means that the continuation specified as the parameter to :func:`TSHttpHookAdd` is called for every -transaction. :func:`TSHttpHookAdd` is typically called from +transaction. :func:`TSHttpHookAdd` must only be called from :func:`TSPluginInit` or :func:`TSRemapInit`. :func:`TSHttpSsnHookAdd` adds :arg:`contp` to @@ -77,9 +77,11 @@ function will be TS_EVENT_HTTP_READ_REQUEST_HDR. When a continuation is triggered by a hook, the actual type of the event data (the void pointer passed as the third parameter to the continuation function) is determined by which hook it is. For example, for the hook ID TS_HTTP_TXN_CLOSE_HOOK, -the event data is of type TSHttpTxn. This is the case regardless of whether the +the event data is of type :type:`TSHttpTxn`. This is the case regardless of whether the continuation was added to the hook using :func:`TSHttpTxnHookAdd`, :func:`TSHttpSsnHookAdd` -or :func:`TSHttpHookAdd`. +or :func:`TSHttpHookAdd`. If the event data is of type :type:`TSHttpTxn`, :type:`TSHttpSsn` or +:type:`TSVConn`, the continuation function can assume the mutex of the indicated +event data object is locked. (But the continuation function must not unlock it.) Return Values ============= @@ -113,6 +115,11 @@ transaction hooks:: TSHttpTxnHookAdd(txnp, TS_HTTP_READ_REQUEST_HDR_HOOK, contp); TSHttpTxnReenable(txnp, TS_EVENT_HTTP_CONTINUE); return 0; + case TS_EVENT_HTTP_READ_REQUEST_HDR: + txnp = (TSHttpTxn) edata; + // ... + TSHttpTxnReenable(txnp, TS_EVENT_HTTP_CONTINUE); + return 0; default: break; } @@ -128,6 +135,9 @@ transaction hooks:: TSHttpHookAdd(TS_HTTP_SSN_START_HOOK, contp); } +For more example code using hooks, see the test_hooks plugin in tests/tools/plugins (used by the test_hooks.test.py +Gold test). + See Also ======== diff --git a/doc/developer-guide/api/functions/TSHttpOverridableConfig.en.rst b/doc/developer-guide/api/functions/TSHttpOverridableConfig.en.rst index 125e1111706..a30b5c29676 100644 --- a/doc/developer-guide/api/functions/TSHttpOverridableConfig.en.rst +++ b/doc/developer-guide/api/functions/TSHttpOverridableConfig.en.rst @@ -43,7 +43,7 @@ Description Some of the values that are set in :file:`records.config` can be changed for a specific transaction. It is important to note that these functions change the -configuration values stored for the transation, which is not quite the same as +configuration values stored for the transaction, which is not quite the same as changing the actual operating values of the transaction. The critical effect is the value must be changed before it is used by the transaction - after that, changes will not have any effect. @@ -66,6 +66,8 @@ The following configurations (from ``records.config``) are overridable: TSOverridableConfigKey Value Configuration Value ================================================================== ==================================================================== :c:macro:`TS_CONFIG_BODY_FACTORY_TEMPLATE_BASE` :ts:cv:`proxy.config.body_factory.template_base` +:c:macro:`TS_CONFIG_HTTP_ALLOW_HALF_OPEN` :ts:cv:`proxy.config.http.allow_half_open` +:c:macro:`TS_CONFIG_HTTP_ALLOW_MULTI_RANGE` :ts:cv:`proxy.config.http.allow_multi_range` :c:macro:`TS_CONFIG_HTTP_ANONYMIZE_INSERT_CLIENT_IP` :ts:cv:`proxy.config.http.insert_client_ip` :c:macro:`TS_CONFIG_HTTP_ANONYMIZE_REMOVE_CLIENT_IP` :ts:cv:`proxy.config.http.anonymize_remove_client_ip` :c:macro:`TS_CONFIG_HTTP_ANONYMIZE_REMOVE_COOKIE` :ts:cv:`proxy.config.http.anonymize_remove_cookie` @@ -78,6 +80,7 @@ TSOverridableConfigKey Value Configuratio :c:macro:`TS_CONFIG_HTTP_BACKGROUND_FILL_COMPLETED_THRESHOLD` :ts:cv:`proxy.config.http.background_fill_completed_threshold` :c:macro:`TS_CONFIG_HTTP_CACHE_CACHE_RESPONSES_TO_COOKIES` :ts:cv:`proxy.config.http.cache.cache_responses_to_cookies` :c:macro:`TS_CONFIG_HTTP_CACHE_CACHE_URLS_THAT_LOOK_DYNAMIC` :ts:cv:`proxy.config.http.cache.cache_urls_that_look_dynamic` +:c:macro:`TS_CONFIG_HTTP_CACHE_ENABLE_DEFAULT_VARY_HEADER` :ts:cv:`proxy.config.http.cache.enable_default_vary_headers` :c:macro:`TS_CONFIG_HTTP_CACHE_GENERATION` :ts:cv:`proxy.config.http.cache.generation` :c:macro:`TS_CONFIG_HTTP_CACHE_GUARANTEED_MAX_LIFETIME` :ts:cv:`proxy.config.http.cache.guaranteed_max_lifetime` :c:macro:`TS_CONFIG_HTTP_CACHE_GUARANTEED_MIN_LIFETIME` :ts:cv:`proxy.config.http.cache.guaranteed_min_lifetime` @@ -85,6 +88,10 @@ TSOverridableConfigKey Value Configuratio :c:macro:`TS_CONFIG_HTTP_CACHE_HEURISTIC_MAX_LIFETIME` :ts:cv:`proxy.config.http.cache.heuristic_max_lifetime` :c:macro:`TS_CONFIG_HTTP_CACHE_HEURISTIC_MIN_LIFETIME` :ts:cv:`proxy.config.http.cache.heuristic_min_lifetime` :c:macro:`TS_CONFIG_HTTP_CACHE_HTTP` :ts:cv:`proxy.config.http.cache.http` +:c:macro:`TS_CONFIG_HTTP_CACHE_IGNORE_ACCEPT_CHARSET_MISMATCH` :ts:cv:`proxy.config.http.cache.ignore_accept_charset_mismatch` +:c:macro:`TS_CONFIG_HTTP_CACHE_IGNORE_ACCEPT_ENCODING_MISMATCH` :ts:cv:`proxy.config.http.cache.ignore_accept_encoding_mismatch` +:c:macro:`TS_CONFIG_HTTP_CACHE_IGNORE_ACCEPT_LANGUAGE_MISMATCH` :ts:cv:`proxy.config.http.cache.ignore_accept_language_mismatch` +:c:macro:`TS_CONFIG_HTTP_CACHE_IGNORE_ACCEPT_MISMATCH` :ts:cv:`proxy.config.http.cache.ignore_accept_mismatch` :c:macro:`TS_CONFIG_HTTP_CACHE_IGNORE_AUTHENTICATION` :ts:cv:`proxy.config.http.cache.ignore_authentication` :c:macro:`TS_CONFIG_HTTP_CACHE_IGNORE_CLIENT_CC_MAX_AGE` :ts:cv:`proxy.config.http.cache.ignore_client_cc_max_age` :c:macro:`TS_CONFIG_HTTP_CACHE_IGNORE_CLIENT_NO_CACHE` :ts:cv:`proxy.config.http.cache.ignore_client_no_cache` @@ -98,11 +105,14 @@ TSOverridableConfigKey Value Configuratio :c:macro:`TS_CONFIG_HTTP_CACHE_RANGE_LOOKUP` :ts:cv:`proxy.config.http.cache.range.lookup` :c:macro:`TS_CONFIG_HTTP_CACHE_RANGE_WRITE` :ts:cv:`proxy.config.http.cache.range.write` :c:macro:`TS_CONFIG_HTTP_CACHE_REQUIRED_HEADERS` :ts:cv:`proxy.config.http.cache.required_headers` +:c:macro:`TS_CONFIG_HTTP_CACHE_VARY_DEFAULT_IMAGES` :ts:cv:`proxy.config.http.cache.vary_default_images` +:c:macro:`TS_CONFIG_HTTP_CACHE_VARY_DEFAULT_OTHER` :ts:cv:`proxy.config.http.cache.vary_default_other` +:c:macro:`TS_CONFIG_HTTP_CACHE_VARY_DEFAULT_TEXT` :ts:cv:`proxy.config.http.cache.vary_default_text` :c:macro:`TS_CONFIG_HTTP_CACHE_WHEN_TO_REVALIDATE` :ts:cv:`proxy.config.http.cache.when_to_revalidate` :c:macro:`TS_CONFIG_HTTP_CHUNKING_ENABLED` :ts:cv:`proxy.config.http.chunking_enabled` :c:macro:`TS_CONFIG_HTTP_CHUNKING_SIZE` :ts:cv:`proxy.config.http.chunking.size` -:c:macro:`TS_CONFIG_HTTP_CONNECT_ATTEMPTS_MAX_RETRIES` :ts:cv:`proxy.config.http.connect_attempts_max_retries` :c:macro:`TS_CONFIG_HTTP_CONNECT_ATTEMPTS_MAX_RETRIES_DEAD_SERVER` :ts:cv:`proxy.config.http.connect_attempts_max_retries_dead_server` +:c:macro:`TS_CONFIG_HTTP_CONNECT_ATTEMPTS_MAX_RETRIES` :ts:cv:`proxy.config.http.connect_attempts_max_retries` :c:macro:`TS_CONFIG_HTTP_CONNECT_ATTEMPTS_RR_RETRIES` :ts:cv:`proxy.config.http.connect_attempts_rr_retries` :c:macro:`TS_CONFIG_HTTP_CONNECT_ATTEMPTS_TIMEOUT` :ts:cv:`proxy.config.http.connect_attempts_timeout` :c:macro:`TS_CONFIG_HTTP_DEFAULT_BUFFER_SIZE` :ts:cv:`proxy.config.http.default_buffer_size` @@ -117,10 +127,10 @@ TSOverridableConfigKey Value Configuratio :c:macro:`TS_CONFIG_HTTP_FORWARD_PROXY_AUTH_TO_PARENT` :ts:cv:`proxy.config.http.forward.proxy_auth_to_parent` :c:macro:`TS_CONFIG_HTTP_GLOBAL_USER_AGENT_HEADER` :ts:cv:`proxy.config.http.global_user_agent_header` :c:macro:`TS_CONFIG_HTTP_INSERT_AGE_IN_RESPONSE` :ts:cv:`proxy.config.http.insert_age_in_response` +:c:macro:`TS_CONFIG_HTTP_INSERT_FORWARDED` :ts:cv:`proxy.config.http.insert_forwarded` :c:macro:`TS_CONFIG_HTTP_INSERT_REQUEST_VIA_STR` :ts:cv:`proxy.config.http.insert_request_via_str` :c:macro:`TS_CONFIG_HTTP_INSERT_RESPONSE_VIA_STR` :ts:cv:`proxy.config.http.insert_response_via_str` :c:macro:`TS_CONFIG_HTTP_INSERT_SQUID_X_FORWARDED_FOR` :ts:cv:`proxy.config.http.insert_squid_x_forwarded_for` -:c:macro:`TS_CONFIG_HTTP_INSERT_FORWARDED` :ts:cv:`proxy.config.http.insert_forwarded` :c:macro:`TS_CONFIG_HTTP_KEEP_ALIVE_ENABLED_IN` :ts:cv:`proxy.config.http.keep_alive_enabled_in` :c:macro:`TS_CONFIG_HTTP_KEEP_ALIVE_ENABLED_OUT` :ts:cv:`proxy.config.http.keep_alive_enabled_out` :c:macro:`TS_CONFIG_HTTP_KEEP_ALIVE_NO_ACTIVITY_TIMEOUT_IN` :ts:cv:`proxy.config.http.keep_alive_no_activity_timeout_in` @@ -130,12 +140,19 @@ TSOverridableConfigKey Value Configuratio :c:macro:`TS_CONFIG_HTTP_NEGATIVE_CACHING_LIFETIME` :ts:cv:`proxy.config.http.negative_caching_lifetime` :c:macro:`TS_CONFIG_HTTP_NEGATIVE_REVALIDATING_ENABLED` :ts:cv:`proxy.config.http.negative_revalidating_enabled` :c:macro:`TS_CONFIG_HTTP_NEGATIVE_REVALIDATING_LIFETIME` :ts:cv:`proxy.config.http.negative_revalidating_lifetime` +:c:macro:`TS_CONFIG_HTTP_NORMALIZE_AE` :ts:cv:`proxy.config.http.normalize_ae` :c:macro:`TS_CONFIG_HTTP_NUMBER_OF_REDIRECTIONS` :ts:cv:`proxy.config.http.number_of_redirections` +:c:macro:`TS_CONFIG_HTTP_PARENT_CONNECT_ATTEMPT_TIMEOUT` :ts:cv:`proxy.config.http.parent_proxy.connect_attempts_timeout` +:c:macro:`TS_CONFIG_HTTP_PARENT_PROXY_FAIL_THRESHOLD` :ts:cv:`proxy.config.http.parent_proxy.fail_threshold` +:c:macro:`TS_CONFIG_HTTP_PARENT_PROXY_RETRY_TIME` :ts:cv:`proxy.config.http.parent_proxy.retry_time` :c:macro:`TS_CONFIG_HTTP_PARENT_PROXY_TOTAL_CONNECT_ATTEMPTS` :ts:cv:`proxy.config.http.parent_proxy.total_connect_attempts` -:c:macro:`TS_CONFIG_PARENT_FAILURES_UPDATE_HOSTDB` :ts:cv:`proxy.config.http.parent_proxy.mark_down_hostdb` +:c:macro:`TS_CONFIG_HTTP_PER_PARENT_CONNECT_ATTEMPTS` :ts:cv:`proxy.config.http.parent_proxy.per_parent_connect_attempts` +:c:macro:`TS_CONFIG_HTTP_PER_SERVER_CONNECTION_MATCH` :ts:cv:`proxy.config.http.per_server.connection.match` +:c:macro:`TS_CONFIG_HTTP_PER_SERVER_CONNECTION_MAX` :ts:cv:`proxy.config.http.per_server.connection.max` :c:macro:`TS_CONFIG_HTTP_POST_CHECK_CONTENT_LENGTH_ENABLED` :ts:cv:`proxy.config.http.post.check.content_length.enabled` :c:macro:`TS_CONFIG_HTTP_POST_CONNECT_ATTEMPTS_TIMEOUT` :ts:cv:`proxy.config.http.post_connect_attempts_timeout` :c:macro:`TS_CONFIG_HTTP_REDIRECT_USE_ORIG_CACHE_KEY` :ts:cv:`proxy.config.http.redirect_use_orig_cache_key` +:c:macro:`TS_CONFIG_HTTP_REQUEST_BUFFER_ENABLED` :ts:cv:`proxy.config.http.request_buffer_enabled` :c:macro:`TS_CONFIG_HTTP_REQUEST_HEADER_MAX_SIZE` :ts:cv:`proxy.config.http.request_header_max_size` :c:macro:`TS_CONFIG_HTTP_RESPONSE_HEADER_MAX_SIZE` :ts:cv:`proxy.config.http.response_header_max_size` :c:macro:`TS_CONFIG_HTTP_RESPONSE_SERVER_ENABLED` :ts:cv:`proxy.config.http.response_server_enabled` @@ -154,23 +171,24 @@ TSOverridableConfigKey Value Configuratio :c:macro:`TS_CONFIG_NET_SOCK_PACKET_TOS_OUT` :ts:cv:`proxy.config.net.sock_packet_tos_out` :c:macro:`TS_CONFIG_NET_SOCK_RECV_BUFFER_SIZE_OUT` :ts:cv:`proxy.config.net.sock_recv_buffer_size_out` :c:macro:`TS_CONFIG_NET_SOCK_SEND_BUFFER_SIZE_OUT` :ts:cv:`proxy.config.net.sock_send_buffer_size_out` +:c:macro:`TS_CONFIG_PARENT_FAILURES_UPDATE_HOSTDB` :ts:cv:`proxy.config.http.parent_proxy.mark_down_hostdb` :c:macro:`TS_CONFIG_SRV_ENABLED` :ts:cv:`proxy.config.srv_enabled` +:c:macro:`TS_CONFIG_SSL_CLIENT_CERT_FILENAME` :ts:cv:`proxy.config.ssl.client.cert.filename` +:c:macro:`TS_CONFIG_SSL_CERT_FILEPATH` :ts:cv:`proxy.config.ssl.client.cert.path` +:c:macro:`TS_CONFIG_SSL_CLIENT_VERIFY_SERVER` :ts:cv:`proxy.config.ssl.client.verify.server` +:c:macro:`TS_CONFIG_SSL_CLIENT_VERIFY_SERVER_PROPERTIES` :ts:cv:`proxy.config.ssl.client.verify.server.properties` +:c:macro:`TS_CONFIG_SSL_CLIENT_VERIFY_SERVER_POLICY` :ts:cv:`proxy.config.ssl.client.verify.server.policy` +:c:macro:`TS_CONFIG_SSL_CLIENT_SNI_POLICY` :ts:cv:`proxy.config.ssl.client.sni_policy` :c:macro:`TS_CONFIG_SSL_HSTS_INCLUDE_SUBDOMAINS` :ts:cv:`proxy.config.ssl.hsts_include_subdomains` :c:macro:`TS_CONFIG_SSL_HSTS_MAX_AGE` :ts:cv:`proxy.config.ssl.hsts_max_age` :c:macro:`TS_CONFIG_URL_REMAP_PRISTINE_HOST_HDR` :ts:cv:`proxy.config.url_remap.pristine_host_hdr` :c:macro:`TS_CONFIG_WEBSOCKET_ACTIVE_TIMEOUT` :ts:cv:`proxy.config.websocket.active_timeout` :c:macro:`TS_CONFIG_WEBSOCKET_NO_ACTIVITY_TIMEOUT` :ts:cv:`proxy.config.websocket.no_activity_timeout` -:c:macro:`TS_CONFIG_SSL_CERT_FILEPATH` :ts:cv:`proxy.config.ssl.client.cert.path` -:c:macro:`TS_CONFIG_SSL_CERT_FILENAME` :ts:cv:`proxy.config.ssl.client.cert.filename` -:c:macro:`TS_CONFIG_HTTP_PARENT_PROXY_FAIL_THRESHOLD` :ts:cv:`proxy.config.http.parent_proxy.fail_threshold` -:c:macro:`TS_CONFIG_HTTP_PARENT_PROXY_RETRY_TIME` :ts:cv:`proxy.config.http.parent_proxy.retry_time` -:c:macro:`TS_CONFIG_HTTP_PER_PARENT_CONNECT_ATTEMPTS` :ts:cv:`proxy.config.http.parent_proxy.per_parent_connect_attempts` -:c:macro:`TS_CONFIG_HTTP_PARENT_CONNECT_ATTEMPT_TIMEOUT` :ts:cv:`proxy.config.http.parent_proxy.connect_attempts_timeout` -:c:macro:`TS_CONFIG_HTTP_NORMALIZE_AE` :ts:cv:`proxy.config.http.normalize_ae` -:c:macro:`TS_CONFIG_HTTP_ALLOW_MULTI_RANGE` :ts:cv:`proxy.config.http.allow_multi_range` -:c:macro:`TS_CONFIG_HTTP_ALLOW_HALF_OPEN` :ts:cv:`proxy.config.http.allow_half_open` -:c:macro:`TS_CONFIG_HTTP_PER_SERVER_CONNECTION_MAX` :ts:cv:`proxy.config.http.per_server.connection.max` -:c:macro:`TS_CONFIG_HTTP_PER_SERVER_CONNECTION_MATCH` :ts:cv:`proxy.config.http.per_server.connection.match` +:c:macro:`TS_CONFIG_SSL_CLIENT_VERIFY_SERVER_POLICY` :ts:cv:`proxy.config.ssl.client.verify.server.policy` +:c:macro:`TS_CONFIG_SSL_CLIENT_VERIFY_SERVER_PROPERTIES` :ts:cv:`proxy.config.ssl.client.verify.server.properties` +:c:macro:`TS_CONFIG_SSL_CLIENT_CERT_FILENAME` :ts:cv:`proxy.config.ssl.client.cert.filename` +:c:macro:`TS_CONFIG_SSL_CLIENT_PRIVATE_KEY_FILENAME` :ts:cv:`proxy.config.ssl.client.private_key.filename` +:c:macro:`TS_CONFIG_SSL_CLIENT_CA_CERT_FILENAME` :ts:cv:`proxy.config.ssl.client.CA.cert.filename` ================================================================== ==================================================================== Examples diff --git a/doc/developer-guide/api/functions/TSHttpTxnErrorBodySet.en.rst b/doc/developer-guide/api/functions/TSHttpTxnErrorBodySet.en.rst index cee7207fbd2..52fc4c01c71 100644 --- a/doc/developer-guide/api/functions/TSHttpTxnErrorBodySet.en.rst +++ b/doc/developer-guide/api/functions/TSHttpTxnErrorBodySet.en.rst @@ -36,4 +36,4 @@ Description Note that both string arguments must be allocated with :c:func:`TSmalloc` or :c:func:`TSstrdup`. The :arg:`mimetype` is optional, and if not provided it defaults to :literal:`text/html`. Sending an empty string would prevent setting -a content type header (but that is not adviced). +a content type header (but that is not advised). diff --git a/doc/developer-guide/api/functions/TSHttpTxnMilestoneGet.en.rst b/doc/developer-guide/api/functions/TSHttpTxnMilestoneGet.en.rst index 08fb5d34ee2..1c0ee733053 100644 --- a/doc/developer-guide/api/functions/TSHttpTxnMilestoneGet.en.rst +++ b/doc/developer-guide/api/functions/TSHttpTxnMilestoneGet.en.rst @@ -138,7 +138,7 @@ is successful. .. macro:: TS_MILESTONE_LAST_ENTRY - A psuedo index which is set to be one more than the last valid index. This is useful for looping over the data. + A pseudo index which is set to be one more than the last valid index. This is useful for looping over the data. * The server connect times predate the transmission of the :literal:`SYN` diff --git a/doc/developer-guide/api/functions/TSHttpTxnServerIntercept.en.rst b/doc/developer-guide/api/functions/TSHttpTxnServerIntercept.en.rst index a546aa33403..bc70ecc5883 100644 --- a/doc/developer-guide/api/functions/TSHttpTxnServerIntercept.en.rst +++ b/doc/developer-guide/api/functions/TSHttpTxnServerIntercept.en.rst @@ -53,7 +53,7 @@ The response from the plugin is cached subject to standard and configured HTTP caching rules. Should the plugin wish the response not be cached, the plugin must use appropriate HTTP response headers to prevent caching. The primary purpose of :func:`TSHttpTxnServerIntercept` is allow plugins to provide gateways -to other protocols or to allow to plugin to its own transport for the next hop +to other protocols or to allow one to plugin to its own transport for the next hop to the server. :func:`TSHttpTxnServerIntercept` overrides parent cache configuration. diff --git a/doc/developer-guide/api/functions/TSHttpTxnServerReqGet.en.rst b/doc/developer-guide/api/functions/TSHttpTxnServerReqGet.en.rst index ad222513d2a..f9cb31e6a85 100644 --- a/doc/developer-guide/api/functions/TSHttpTxnServerReqGet.en.rst +++ b/doc/developer-guide/api/functions/TSHttpTxnServerReqGet.en.rst @@ -26,7 +26,28 @@ Synopsis `#include ` -.. function:: TSReturnCode TSHttpTxnServerReqGet(TSHttpTxn txnp, TSMBuffer * bufp, TSMLoc * offset) +.. function:: TSReturnCode TSHttpTxnServerReqGet(TSHttpTxn txnp, TSMBuffer * bufp, TSMLoc * obj) Description =========== + +Get the request |TS| is sending to the upstream (server) for the transaction :arg:`txnp`. +:arg:`bufp` and :arg:`obj` should be valid pointers to use as return values. The call site could +look something like :: + + TSMBuffer mbuffer; + TSMLoc mloc; + if (TS_SUCCESS == TSHttpTxnServerReqGet(&mbuffer, &mloc)) { + /* Can use safely mbuffer, mloc for subsequent API calls */ + } else { + /* mbuffer, mloc in an undefined state */ + } + +This call returns :c:macro:`TS_SUCCESS` on success, and :c:macro:`TS_ERROR` on failure. It is the +caller's responsibility to see that :arg:`txnp` is a valid transaction. + +Once the request object is obtained, it can be used to access all of the elements of the request, +such as the URL, the header fields, etc. This is also the mechanism by which a plugin can change the +upstream request, if done before the request is sent (in or before +:c:macro:`TS_HTTP_SEND_REQUEST_HDR_HOOK`). Note that for earlier hooks, the request may not yet +exist, in which case an error is returned. diff --git a/doc/developer-guide/api/functions/TSHttpTxnServerRespGet.en.rst b/doc/developer-guide/api/functions/TSHttpTxnServerRespGet.en.rst index 9150f1d5092..3113297b871 100644 --- a/doc/developer-guide/api/functions/TSHttpTxnServerRespGet.en.rst +++ b/doc/developer-guide/api/functions/TSHttpTxnServerRespGet.en.rst @@ -30,3 +30,25 @@ Synopsis Description =========== + +Get the response header sent by the upstream server. This will only be useful in a callback on a +hook that is called after the upstream responds, and if there was an upstream response. For +instance, if the inbound request has no remap rule and :ts:cv:`remap is required +` then there will be no server response because no outbound +connection was made. In this case the function will return :c:macro:`TS_ERROR`. + +The response header is returned in :arg:`bufp` and :arg:`offset`. :arg:`bufp` is the heap in which +the header resides, and :arg:`offset` is the location in that heap. These will be used in subsequent +calls to retrieve information from the header. + +.. code-block:: cpp + :emphasize-lines: 4 + + int get_response_status(TSHttpTxn txn) { + TSMBuffer resp_heap = nullptr; + TSMLoc resp_hdr = nullptr; + if (TS_SUCCESS == TSHttpTxnServerRespGet(tnx, &resp_heap, &resp_hdr)) { + return TSHttpHdrStatusGet(resp_headp, resp_hdr); + } + return HTTP_STATUS_NONE; + } diff --git a/doc/developer-guide/api/functions/TSIOBufferReader.en.rst b/doc/developer-guide/api/functions/TSIOBufferReader.en.rst index 95919620c1d..a5c5702f19e 100644 --- a/doc/developer-guide/api/functions/TSIOBufferReader.en.rst +++ b/doc/developer-guide/api/functions/TSIOBufferReader.en.rst @@ -59,7 +59,7 @@ has two very important consequences -- * Conversely keeping a reader around unused will pin the buffer data in memory. This can be useful or harmful. A buffer has a fixed amount of possible readers (currently 5) which is determined at compile -time. Reader allocation is fast and cheap until this maxium is reached at which point it fails. +time. Reader allocation is fast and cheap until this maximum is reached at which point it fails. :func:`TSIOBufferReaderAlloc` allocates a reader for the IO buffer :arg:`bufp`. This should only be called on a newly allocated buffer. If not the location of the reader in the buffer will be diff --git a/doc/developer-guide/api/functions/TSLifecycleHookAdd.en.rst b/doc/developer-guide/api/functions/TSLifecycleHookAdd.en.rst index efdcee1f226..ced169bbaab 100644 --- a/doc/developer-guide/api/functions/TSLifecycleHookAdd.en.rst +++ b/doc/developer-guide/api/functions/TSLifecycleHookAdd.en.rst @@ -1,19 +1,18 @@ -.. Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at +.. Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed + with this work for additional information regarding copyright + ownership. The ASF licenses this file to you under the Apache + License, Version 2.0 (the "License"); you may not use this file + except in compliance with the License. You may obtain a copy of + the License at http://www.apache.org/licenses/LICENSE-2.0 - Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied. See the License for the specific language governing + permissions and limitations under the License. .. include:: ../../../common.defs @@ -98,6 +97,19 @@ Types Called after every SSL context initialization used by |TS| for inbound connections (|TS| as the server). + .. cpp:enumerator:: TS_LIFECYCLE_TASK_THREADS_READY_HOOK + + Called after |TS| task threads have been started. + + Invoked with the event :c:data:`TS_EVENT_LIFECYCLE_TASK_THREADS_READY` and ``NULL`` + data. + + .. cpp:enumerator:: TS_LIFECYCLE_SHUTDOWN_HOOK + + Called after |TS| receiving a shutdown signal, such as SIGTERM. + + Invoked with the event :c:data:`TS_EVENT_LIFECYCLE_SHUTDOWN` and ``NULL`` data. + .. c:type:: TSPluginMsg The format of the data for the plugin message event :c:data:`TS_EVENT_LIFECYCLE_MSG`. diff --git a/doc/developer-guide/api/functions/TSSslContext.en.rst b/doc/developer-guide/api/functions/TSSslContext.en.rst index e921d5bacf6..9337895a525 100644 --- a/doc/developer-guide/api/functions/TSSslContext.en.rst +++ b/doc/developer-guide/api/functions/TSSslContext.en.rst @@ -36,11 +36,11 @@ Description =========== :func:`TSSslContextFindByName` searches for a SSL server context -created from :file:`ssl_multicert.config`, matching against the +created from :file:`ssl_multicert.config`, matchingg against the server :arg:`name`. :func:`TSSslContextFindByAddr` searches for a SSL server context -created from :file:`ssl_multicert.config` matchin against the server +created from :file:`ssl_multicert.config` matching against the server :arg:`address`. diff --git a/doc/developer-guide/api/functions/TSSslSession.en.rst b/doc/developer-guide/api/functions/TSSslSession.en.rst index ebf9fccd30a..321f34fc942 100644 --- a/doc/developer-guide/api/functions/TSSslSession.en.rst +++ b/doc/developer-guide/api/functions/TSSslSession.en.rst @@ -46,7 +46,7 @@ The functions also work with the :type:`TSSslSession` object which can be cast t These functions perform the appropriate locking on the session cache to avoid errors. -The :func:`TSSslSessionGet` and :func:`TSSslSessionGetBuffer` functions retreive the :type:`TSSslSession` object that is identifed by the +The :func:`TSSslSessionGet` and :func:`TSSslSessionGetBuffer` functions retrieve the :type:`TSSslSession` object that is identifed by the :type:`TSSslSessionID` object. If there is no matching sesion object, :func:`TSSslSessionGet` returns NULL and :func:`TSSslSessionGetBuffer` returns 0. diff --git a/doc/developer-guide/api/functions/TSStat.en.rst b/doc/developer-guide/api/functions/TSStat.en.rst index 43d585fa2d7..64b539f2bbc 100644 --- a/doc/developer-guide/api/functions/TSStat.en.rst +++ b/doc/developer-guide/api/functions/TSStat.en.rst @@ -46,9 +46,9 @@ Description A plugin statistic is created by :func:`TSStatCreate`. The :arg:`name` must be globally unique and should follow the standard dotted tag form. To avoid collisions and for easy of use the first tag -should be the plugin name or something easily derived from it. Currently only integers are suppored +should be the plugin name or something easily derived from it. Currently only integers are supported therefore :arg:`type` must be :macro:`TS_RECORDDATATYPE_INT`. The return value is the index of the -statistic. In general thsi should work but if it doesn't it will :code:`assert`. In particular, +statistic. In general this should work but if it doesn't it will :code:`assert`. In particular, creating the same statistic twice will fail in this way, which can happen if statistics are created as part of or based on configuration files and |TS| is reloaded. diff --git a/doc/developer-guide/api/functions/TSThreadSelf.en.rst b/doc/developer-guide/api/functions/TSThreadSelf.en.rst index 7c213c2318a..b0d059013de 100644 --- a/doc/developer-guide/api/functions/TSThreadSelf.en.rst +++ b/doc/developer-guide/api/functions/TSThreadSelf.en.rst @@ -30,3 +30,6 @@ Synopsis Description =========== + +Return an instance of :type:`TSThread` that identifies the current thread. This must be called from +within a |TS| create thread context. That is any thread created by the |TS| core or via the |TS| API. diff --git a/doc/developer-guide/api/functions/TSTypes.en.rst b/doc/developer-guide/api/functions/TSTypes.en.rst index 6b18d9a71c9..675029a1d5c 100644 --- a/doc/developer-guide/api/functions/TSTypes.en.rst +++ b/doc/developer-guide/api/functions/TSTypes.en.rst @@ -102,6 +102,11 @@ more widely. Those are described on this page. .. type:: TSMBuffer + Internally, data for a transaction is stored in one more more :term:`header heap`\s. These are + storage local to the transaction, and generally each HTTP header is stored in a separate one. + This type is a handle to a header heap, and is provided or required by functions that locate HTTP + header related data. + .. type:: TSMgmtCounter .. type:: TSMgmtFloat @@ -120,6 +125,12 @@ more widely. Those are described on this page. .. type:: TSMLoc + This is a memory location relative to a :term:`header heap` represented by a :c:type:`TSMBuffer` and + must always be used in conjuction with that :c:type:`TSMBuffer` instance. It identifies a specific + object in the :c:type:`TSMBuffer`. This indirection is needed so that the :c:type:`TSMBuffer` + can reallocate space as needed. Therefore a raw address obtained from a :c:type:`TSMLoc` should + be considered volatile that may become invalid across any API call. + .. var:: TSMLoc TS_NULL_MLOC A predefined null valued :type:`TSMLoc` used to indicate the absence of an :type:`TSMLoc`. @@ -150,6 +161,20 @@ more widely. Those are described on this page. .. type:: TSThread + This represents an internal |TS| thread, created by the |TS| core. It is an opaque type which + can be used only to check for equality / inequality, and passed to API functions. An instance + that refers to the current thread can be obtained with :func:`TSThreadSelf`. + +.. type:: TSEventThread + + This type represents an :term:`event thread`. It is an opaque which is used to specify a + particular event processing thread in |TS|. If plugin code is executing in an event thread + (which will be true if called from a hook or a scheduled event) then the current event thread + can be obtained via :func:`TSEventThreadSelf`. + + A :code:`TSEventThread` is also a :type:`TSThread` and can be passed as an argument to any + parameter of type :type:`TSThread`. + .. type:: TSThreadFunc .. type:: TSUuidVersion diff --git a/doc/developer-guide/api/functions/TSVConnReenable.en.rst b/doc/developer-guide/api/functions/TSVConnReenable.en.rst index 6c8f8c14596..ff3aba08142 100644 --- a/doc/developer-guide/api/functions/TSVConnReenable.en.rst +++ b/doc/developer-guide/api/functions/TSVConnReenable.en.rst @@ -32,7 +32,7 @@ Description =========== Reenable the SSL connection :arg:`svc`. If a plugin hook is called, ATS -processing on that connnection will not resume until this is invoked for that +processing on that connection will not resume until this is invoked for that connection. If the server is running OpenSSL 1.0.2, the plugin writer can pause SSL handshake diff --git a/doc/developer-guide/api/functions/TSfwrite.en.rst b/doc/developer-guide/api/functions/TSfwrite.en.rst index 30b33aec8ac..3b064ace9db 100644 --- a/doc/developer-guide/api/functions/TSfwrite.en.rst +++ b/doc/developer-guide/api/functions/TSfwrite.en.rst @@ -44,4 +44,4 @@ The behavior is undefined if length is greater than SSIZE_MAX. Return Value ============ -Returns the number of bytes actually written, or -1 if an error occured. +Returns the number of bytes actually written, or -1 if an error occurred. diff --git a/doc/developer-guide/api/types/TSEvent.en.rst b/doc/developer-guide/api/types/TSEvent.en.rst index 96e8b735e8c..8261cfcef1d 100644 --- a/doc/developer-guide/api/types/TSEvent.en.rst +++ b/doc/developer-guide/api/types/TSEvent.en.rst @@ -57,6 +57,22 @@ Enumeration Members .. c:macro:: TS_EVENT_VCONN_ACTIVE_TIMEOUT +.. c:macro:: TS_EVENT_VCONN_START + + An inbound connection has started. + +.. c:macro:: TS_EVENT_VCONN_CLOSE + + An inbound connection has closed. + +.. c:macro:: TS_EVENT_OUTBOUND_START + + An outbound connection has started. + +.. c:macro:: TS_EVENT_OUTBOUND_CLOSE + + An outbound connection has closed. + .. c:macro:: TS_EVENT_NET_CONNECT .. c:macro:: TS_EVENT_NET_CONNECT_FAILED @@ -167,21 +183,31 @@ Enumeration Members .. c:macro:: TS_EVENT_LIFECYCLE_PORTS_INITIALIZED + The internal data structures for the proxy ports have been initialized. + .. c:macro:: TS_EVENT_LIFECYCLE_PORTS_READY + The proxy ports are now open for inbound connections. + .. c:macro:: TS_EVENT_LIFECYCLE_CACHE_READY -.. c:macro:: TS_EVENT_LIFECYCLE_MSG + The cache is ready. .. c:macro:: TS_EVENT_LIFECYCLE_SERVER_SSL_CTX_INITIALIZED .. c:macro:: TS_EVENT_LIFECYCLE_CLIENT_SSL_CTX_INITIALIZED -.. c:macro:: TS_EVENT_VCONN_START +.. c:macro:: TS_EVENT_LIFECYCLE_MSG -.. c:macro:: TS_EVENT_VCONN_CLOSE + A message from an external source has arrived. -.. c:macro:: TS_EVENT_MGMT_UPDATE +.. c:macro:: TS_EVENT_LIFECYCLE_TASK_THREADS_READY + + The ``ET_TASK`` threads are running. + +.. c:macro:: TS_EVENT_LIFECYCLE_SHUTDOWN + + The |TS| process has is shutting down. .. c:macro:: TS_EVENT_INTERNAL_60200 @@ -189,6 +215,24 @@ Enumeration Members .. c:macro:: TS_EVENT_INTERNAL_60202 +.. c:macro:: TS_EVENT_SSL_CERT + + Preparing to present a server certificate to an inbound TLS connection. + +.. c:macro:: TS_EVENT_SSL_SERVERNAME + + The SNI name for an Inbound TLS connection has become available. + +.. c:macro:: TS_EVENT_SSL_VERIFY_SERVER + + Outbound TLS connection certificate verification (verifying the server certificate). + +.. c:macro:: TS_EVENT_SSL_VERIFY_CLIENT + + Inbound TLS connection certificate verification (verifying the client certificate). + +.. c:macro:: TS_EVENT_MGMT_UPDATE + Description =========== @@ -213,4 +257,3 @@ These are the event types used to drive continuations in the event system. .. cpp:var:: EventType EVENT_IMMEDIATE See :c:macro:`EVENT_IMMEDIATE`. - diff --git a/doc/developer-guide/api/types/TSOverridableConfigKey.en.rst b/doc/developer-guide/api/types/TSOverridableConfigKey.en.rst index 55e2b4ea285..938254e99f5 100644 --- a/doc/developer-guide/api/types/TSOverridableConfigKey.en.rst +++ b/doc/developer-guide/api/types/TSOverridableConfigKey.en.rst @@ -119,6 +119,7 @@ Enumeration Members .. c:macro:: TS_CONFIG_HTTP_NUMBER_OF_REDIRECTIONS .. c:macro:: TS_CONFIG_HTTP_CACHE_MAX_OPEN_WRITE_RETRIES .. c:macro:: TS_CONFIG_HTTP_REDIRECT_USE_ORIG_CACHE_KEY + .. c:macro:: TS_CONFIG_HTTP_REQUEST_BUFFER_ENABLED .. c:macro:: TS_CONFIG_HTTP_ATTACH_SERVER_SESSION_TO_CLIENT .. c:macro:: TS_CONFIG_HTTP_ORIGIN_MAX_CONNECTIONS_QUEUE .. c:macro:: TS_CONFIG_WEBSOCKET_NO_ACTIVITY_TIMEOUT @@ -128,7 +129,7 @@ Enumeration Members .. c:macro:: TS_CONFIG_HTTP_TRANSACTION_ACTIVE_TIMEOUT_IN .. c:macro:: TS_CONFIG_SRV_ENABLED .. c:macro:: TS_CONFIG_HTTP_FORWARD_CONNECT_METHOD - .. c:macro:: TS_CONFIG_SSL_CERT_FILENAME + .. c:macro:: TS_CONFIG_SSL_CLIENT_CERT_FILENAME .. c:macro:: TS_CONFIG_SSL_CERT_FILEPATH .. c:macro:: TS_CONFIG_PARENT_FAILURES_UPDATE_HOSTDB .. c:macro:: TS_CONFIG_HTTP_CACHE_ENABLE_DEFAULT_VARY_HEADER @@ -149,6 +150,13 @@ Enumeration Members .. c:macro:: TS_CONFIG_HTTP_ALLOW_HALF_OPEN .. c:macro:: TS_CONFIG_HTTP_PER_SERVER_CONNECTION_MAX .. c:macro:: TS_CONFIG_HTTP_PER_SERVER_CONNECTION_MATCH + .. c:macro:: TS_CONFIG_SSL_CLIENT_VERIFY_SERVER + .. c:macro:: TS_CONFIG_SSL_CLIENT_VERIFY_SERVER_POLICY + .. c:macro:: TS_CONFIG_SSL_CLIENT_VERIFY_SERVER_PROPERTIES + .. c:macro:: TS_CONFIG_SSL_CLIENT_SNI_POLICY + .. c:macro:: TS_CONFIG_SSL_CLIENT_PRIVATE_KEY_FILENAME + .. c:macro:: TS_CONFIG_SSL_CLIENT_CA_CERT_FILENAME + Description =========== diff --git a/doc/developer-guide/api/types/TSThreadPool.en.rst b/doc/developer-guide/api/types/TSThreadPool.en.rst index e7140a99fef..298a8505b15 100644 --- a/doc/developer-guide/api/types/TSThreadPool.en.rst +++ b/doc/developer-guide/api/types/TSThreadPool.en.rst @@ -31,8 +31,6 @@ Enum typedef. Enumeration Members =================== -.. c:member:: TSThreadPool TS_THREAD_POOL_DEFAULT - .. c:member:: TSThreadPool TS_THREAD_POOL_NET .. c:member:: TSThreadPool TS_THREAD_POOL_TASK diff --git a/doc/developer-guide/cache-architecture/architecture.en.rst b/doc/developer-guide/cache-architecture/architecture.en.rst index 6442c0e9579..84a04084140 100644 --- a/doc/developer-guide/cache-architecture/architecture.en.rst +++ b/doc/developer-guide/cache-architecture/architecture.en.rst @@ -80,12 +80,12 @@ stripe is in a single cache span and part of a single cache volume. If the cache volumes for the example cache spans were defined as: -.. image:: images/ats-cache-volume-definition.png +.. figure:: images/ats-cache-volume-definition.png :align: center Then the actual layout would look like: -.. image:: images/cache-span-layout.png +.. figure:: images/cache-span-layout.png :align: center Cache stripes are the fundamental unit of cache for the implementation. A @@ -115,7 +115,7 @@ relationship between a span block and a cache stripe is the same as between a di file system. A cache stripe is structured data contained in a span block and always occupies the entire span block. -.. image:: images/span-header.svg +.. figure:: images/span-header.svg :align: center Stripe Structure @@ -292,7 +292,8 @@ The header for a stripe is a variably sized instance of :class:`VolHeaderFooter` The variable trailing section contains the head indices of the directory entry free lists for the segments. -.. image:: images/stripe-header.svg +.. figure:: images/stripe-header.svg + :align: center The trailing :member:`VolHeaderFooter::freelist` array overlays the disk storage with an entry for every segment, even though the array is declared to have length `1`. diff --git a/doc/developer-guide/config-vars.en.rst b/doc/developer-guide/config-vars.en.rst index 5e8585182c5..5fa88084e7d 100644 --- a/doc/developer-guide/config-vars.en.rst +++ b/doc/developer-guide/config-vars.en.rst @@ -322,3 +322,29 @@ required for generic access: #. Update the Lua plugin enumeration ``TSLuaOverridableConfigKey`` in |ts_lua_http_config.c|_. #. Update the documentation of :ref:`ts-overridable-config` in |TSHttpOverridableConfig.en.rst|_. + +API conversions +--------------- + +A relatively new feature for overridable variables is the ability to keep them in more natural data +types and convert as needed to the API types. This in turns enables defining the configuration +locally in a module and then "exporting" it to the API interface. Modules then do not have to +include headers for all types in all overridable configurations. + +The conversion is done through an instance of :code:`MgmtConverter`. This has 6 points to +conversions, a load and store function for each of the types :code:`MgmtInt`, :code:`MgmtFloat`, and +:code:`MgmtInt`. The :code:`MgmtByte` type is handled by the :code:`MgmtInt` conversions. In general +each overridable variable will specify two of these, a load and store for a specific type, although +it is possible to provide other pairs, e.g. if a value is an enumeration can should be settable +as a string as well as an integer. + +The module is responsible for creating an instance of :code:`MgmtConverter` with the appropriate +load / store function pairs set. The declaration must be visible in the :ts:git:`proxy/InkAPI.cc` +file. The function :code:`_conf_to_memberp` sets up the conversion. For the value of the enumeration +:c:type:`TSOverridableConfigKey` that specifies the overridable variable, code is added to specify +the member and the conversion. There are default converters for the API types and if the overridable +is one of those, it is only necessary to call :code:`_memberp_to_generic` passing in a pointer to +the variable. For a variable with conversion, :arg:`ret` should be set to point to the variable and +:arg:`conv` set to point to the converter for that variable. If multiple variables are of the same +type they can use the same converter because a pointer to the specific member is passed to the +converter. diff --git a/doc/developer-guide/core-architecture/heap.en.rst b/doc/developer-guide/core-architecture/heap.en.rst new file mode 100644 index 00000000000..31bee10df8f --- /dev/null +++ b/doc/developer-guide/core-architecture/heap.en.rst @@ -0,0 +1,259 @@ +.. Licensed to the Apache Software Foundation (ASF) under one or more contributor license + agreements. See the NOTICE file distributed with this work for additional information regarding + copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with the License. You may obtain + a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software distributed under the License + is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + or implied. See the License for the specific language governing permissions and limitations + under the License. + +.. include:: ../../common.defs +.. highlight:: cpp +.. default-domain:: cpp + +.. _core-hdr-heap: + +Header Heap +*********** + +Memory for HTTP header data is kept in :term:`header heap`\s. + +Classes +======= + +.. class:: HdrHeapObjImpl + + This is the abstract base class for objects allocated in a :class:`HdrHeap`. This allows updating + objects in a heap in a generic way, without having to locate all of the pointers to the objects. + + The type of an instance stored in a heap must be one of the following values. + + .. enumerator:: HDR_HEAP_OBJ_EMPTY = 0 + + Used to mark invalid objects, ones not yet constructed or ones that have been destroyed. + + .. enumerator:: HDR_HEAP_OBJ_RAW = 1 + + Some sort of raw object, I have no idea. + + .. enumerator:: HDR_HEAP_OBJ_URL = 2 + + A URL object. + + .. enumerator:: HDR_HEAP_OBJ_HTTP_HEADER = 3 + + The header for an HTTP request or response. + + .. enumerator:: HDR_HEAP_OBJ_MIME_HEADER = 4 + + A MIME header, containing MIME style fields with names and values. + + .. enumerator:: HDR_HEAP_OBJ_FIELD_BLOCK = 5 + + Who the heck knows? + +.. class:: HdrStrHeap + + This is a :term:`variable sized class`, therefore new instance must be created by :func:`new_HdrStrHeap` + and deallocated by the :code:`destroy` method. + +.. function:: HdrStrHeap * new_HdrStrHeap(int n) + + Create and return a new instance of :class:`HdrStrHeap`. If :arg:`n` is less than ``HDR_STR_HEAP_DEFAULT_SIZE`` + it is increased to that value. + + If the allocated size is ``HDR_STR_HEAP_DEFAULT_SIZE`` (or smaller and upsized to that value) then + the instance is allocated from a thread local pool via :code:`strHeapAllocator`. If larger it + is allocated from global memory via :code:`ats_malloc`. + +.. class:: HdrHeap + + This is a :term:`variable sized class` and therefore new instances must be created by :func:`new_HdrHeap` + and deallocated by the :code:`destroy` method. + + :class:`HdrHeap` manages memory for heap objects directly and memory for strings via ancillary + heaps (which are instances of :class:`HdrStrHeap`). For the string heaps there is at most one + writeable heap, and up to :code:`HDR_BUF_RONLY_HEAPS` read only heaps. + + All objects in the internal heap must be subclasses of :class:`HdrHeapObjImpl`. + + .. function:: size_t required_space_for_evacuation() + + Calculate and return the total live string space for :arg:`this`. + + .. function:: void evacuate_from_str_heaps(HdrStrHeap * new_heap) + + Copy all live strings from the heap objects in :arg:`this` to :arg:`new_heap`. + + .. function:: void coalesce_str_heaps(int incoming_size) + + This garbage collects the string heaps in a half space style, by creating a new string space + (string heap), copying all of the strings there, and then discarding the existing string heaps. + + The total amount of live string space is calculated by + :func:`HdrHeap::required_space_for_evacuation` and a new string heap is created of a size at + least as large as the live string space plus :arg:`incoming_size` bytes. + + All of the live strings are moved to the new string heap by + :func:`HdrHeap::evacuate_from_str_heaps`, the existing string heaps are deallocated, and the + new string heap becomes the writeable string heap for the header heap. The end result is a + single writeable string heap and no read only string heaps, with all live strings resident in + that writeable string heap. + + .. function:: char * allocate_str(int bytes) + + Allocate :arg:`nbytes` of space for a string in the writeable string heap. A pointer to the + first byte is returned, or ``nullptr`` if the space could not be allocated. + + .. function:: HdrHeapObjImpl * allocate_obj(int nbytes, int type) + + Allocate a :arg:`type` object that is :arg:`nbytes` in size in the heap and return a pointer + to it, or ``nullptr`` if the object could not be allocated. + + :arg:`nbytes` must be at most ``HDR_MAX_ALLOC_SIZE``. + + The members of :class:`HdrHeapObjImpl` are initialized. Further initialization is the + responsibility of the caller. + + :arg:`type` must be one of the values specified in :class:`HdrHeapObjImpl`. + + .. function:: int marshal_length() + + Compute and return the size of the buffer needed to serialize :arg:`this`. + + .. function:: int marshal(char * buffer, int length) + + Serialize :arg:`this` to :arg:`buffer` of size :arg:`length`. It is required that + :arg:`length` be at least the value returned by :func:`HdrHeap::marshal_length`. + +.. function:: HdrHeap * new_HdrHeap(int n) + + Create and return a new instance of :class:`HdrHeap`. If :arg:`n` is less than ``HdrHeap::DEFAULT_SIZE`` + it is increased to that value. + + If the allocated size is ``HdrHeap::DEFAULT_SIZE`` (or smaller and upsized to that value) then + the instance is allocated from a thread local pool via :code:`hdrHeapAllocator`. If larger it + is allocated from global memory via :code:`ats_malloc`. + +.. topic:: Header Heap Class Structure + + .. figure:: /uml/images/hdr-heap-class.svg + + +Implementation +============== + +String Coalescence +------------------ + +String heaps do do not maintain lists of internal free space. Strings that are released are left in +place, creating dead space in the heap. For this reason it can become necessary to do a garbage +collection operation on the writeable string heap in the header heap by calling +:func:`HdrHeap::coalesce_str_heaps`. This is done when + +* The amount of dead space in the writable string heap exceeds ``MAX_LOST_STR_SPACE``. + +* An external string heap is being added and all current read only string heap slots are used. + +The mechanism is simple in design - the size of the live string data in the current string heaps is +calculated and a new heap is allocated sufficient to contain all existing strings, with additional +space for new string data. Each heap object is required to provide a :code:`strings_length` method +which returns the size of the live string data for that object (recursively as needed). The strings +are copied to the new string heap, all of the previous string heaps are discarded, and the new heap +becomes the writable string heap for the header heap. + +Each heap object is responsible for providing a :code:`move_strings` method which copies its strings +to a new string heap, passed as an argument. This is a source of pointer invalidation for other +parts of the core and the plugin API. For the latter, insulating from such string movement is the +point of the :c:type:`TSMLoc` type. + +String Allocation +----------------- + +Storage for a string is allocated by :func:`HdrHeap::allocate_str`. If the current amount of dead +space is too large, this is treated as an initial allocation failure. If there is no current +writeable string heap, one is created that is a least as large as the space requested and the size +of the previous writeable string heap. Space for the string is then allocated out of the writeable +string heap. If this fails due to lack of space the current writeable string heap is "demoted" to a +read only string heap and allocation retried (which will cause a new writeable string heap). If the +writeable string heap cannot be demoted due to lack of read only slots, the strings heaps are +coalesced with an additional size request of the requested string size. This will result in a single +writeable string heap and not read only heaps, the former containing all of the existing strings plus +sufficient space to allocate the new string. + +.. topic:: Decision Diagram + + .. figure:: /uml/images/hdr-heap-str-alloc.svg + +Object Allocation +----------------- + +Objects are allocated on the header heap by :func:`HdrHeap::allocate_obj`. Such objects must be one +of a compile time determined set of types [#]_. This method first tries to allocate the object in +existing free space. If that doesn't work then the allocator walks a list of :class:`HdrHeap` +instances looking for space. If no space is found anywhere, a new :class:`HdrHeap` instance is +created with twice the space of the last :class:`HdrHeap` in the list and added to the list to +try. + +Once space is found for the object, the base members of :class:`HdrHeapObjImpl` are initialized with +the objec type and size, with the :arg:`m_obj_flags` set to 0. + +Serialization +------------- + +Because heaps store the HTTP request / response data, a header heap needs to be serialized to be put +in to the cache. For performance reasons, it is desirable to be able to unserialize the serialized +data in place, rather than copying it again. That is, the data is read from disk into a block of +memory and then that memory is converted to a live data structure. In this case the memory used by +the heap is owned by some other object and the header heap must not do any clean up. This is +signaled by the `m_writeable` flag. In an unserialized header heap this is set to ``false`` and such +a header heap is not allowed to allocate any additional objects or strings - it is immutable. + +The primary mechanism to do this is to use swizzling on the pointers in the structure. During +serialization pointers are converted to offsets and during unserialization these offsets are +converted back to pointers. To make this simpler, unserialized header heaps are marked read only so +that updating does not have to be supported. Additionally, :class:`HdrHeap` is a POD and therefore +has no virtual function table pointer to be stored or restored [#]_. + +To serialize, first :func:`HdrHeap::marshal_length` is called to get a buffer size. The +serialization buffer is created with sufficient space for the header heap and that space is passed +to :func:`HdrHeap::marshal` to perform the actual serialization. The object heaps are serialized +followed by the string heaps. No coalescence is done, on the presumption that because the amount +of dead space is limited by coalescence (as needed) on every string creation. + +When serializing strings, each object is responsible for swizzling its own pointers. Because the +object heaps have already been serialized and all of the header heap object types are also PODs, +these serialized objects can have the pointer swizzling method, :code:`marshal`, called directly +on them. This method is provided with a set of "translations" which indicate the base offset for +each range of object and string heap memory. The object marshalling can then compute the correct +offset to store for each live string pointer. + +Inheriting Strings +------------------ + +The string heaps are designed to be reference counted so that they can be shared as read only +objects between heaps. This enables copying heap objects between heaps less expensive as the +strings pointers in them can be preserved in the new heap by sharing the string heaps in which +those strings reside. + +This can still be a bit complex as it is possible that the combined number of string heaps is more +than the limit. In this case, the target header heap does string coalescence so that it is reduced to +having a single writeable string heap with enough free space to hold all of the strings in the +source header heap. As a result, it is required that all heap objects already be present in the +target header heap before the strings are inherited. This means that the string coalescence will +properly copy the strings of and update the strings pointers in the copied heap objects. + +.. rubric:: Footnotes. + +.. [#] + + Not that I can see any good reason for that, if virtual methods instead of :code:`switch` + statements were used. + +.. [#] + + Which makes the initialization logic to "fixup" the virtual function pointer rather silly. diff --git a/doc/developer-guide/core-architecture/index.en.rst b/doc/developer-guide/core-architecture/index.en.rst index 9b8537548f7..a1efacb0e14 100644 --- a/doc/developer-guide/core-architecture/index.en.rst +++ b/doc/developer-guide/core-architecture/index.en.rst @@ -25,4 +25,5 @@ Core Architecture .. toctree:: :maxdepth: 1 - rpc.en \ No newline at end of file + heap.en + rpc.en diff --git a/doc/developer-guide/core-architecture/rpc.en.rst b/doc/developer-guide/core-architecture/rpc.en.rst index 13c0a2fe7f3..1e43989c406 100644 --- a/doc/developer-guide/core-architecture/rpc.en.rst +++ b/doc/developer-guide/core-architecture/rpc.en.rst @@ -63,9 +63,9 @@ Runtime Structure Message Passing =============== -Sequence diagram for a command sent from |TCtl| to when it is recieved by a plugin. +Sequence diagram for a command sent from |TCtl| to when it is received by a plugin. -.. image:: ../../uml/images/RPC-sequence-diagram.svg +.. figure:: ../../uml/images/RPC-sequence-diagram.svg .. note:: @@ -231,7 +231,7 @@ Now, using this macro, messages can easily be sent. For example: RPC API for |TServer| and |TManager| ==================================== -.. image:: ../../uml/images/RPC-states.svg +.. figure:: ../../uml/images/RPC-states.svg :align: center |LM| and |PM| follow similar workflows. A manager will poll the socket for any messages. If it is able to read a message, it will handle it based on the :arg:`msg_id` from the :class:`MgmtMessageHdr` and select a callback to run asynchoronously. The async callback will add a response, if any, to an outgoing event queue within the class. A manager will continue to poll and read on the socket as long as there are messages avaliable. Two things can stop a manager from polling. diff --git a/doc/developer-guide/index.en.rst b/doc/developer-guide/index.en.rst index 688203c0a17..5deda01f4b6 100644 --- a/doc/developer-guide/index.en.rst +++ b/doc/developer-guide/index.en.rst @@ -54,4 +54,5 @@ duplicate bugs is encouraged, but not required. documentation/index.en host-resolution-proposal.en client-session-architecture.en - core-architecture/index.en \ No newline at end of file + core-architecture/index.en + layout/index.en \ No newline at end of file diff --git a/doc/developer-guide/internal-libraries/ArgParser.en.rst b/doc/developer-guide/internal-libraries/ArgParser.en.rst index 6e1619771c8..1448a90c4b0 100644 --- a/doc/developer-guide/internal-libraries/ArgParser.en.rst +++ b/doc/developer-guide/internal-libraries/ArgParser.en.rst @@ -144,11 +144,13 @@ from the :class:`Arguments` object returned from the parsing. The function can b args.invoke(); -Help message +Help and Version messages ------------------------- - Help message will be outputted when a wrong usage of the program is detected or `--help` option found. +- Version message is defined unified in :code:`ArgParser::version_message()`. + Classes +++++++ @@ -175,6 +177,10 @@ Classes Output usage to the console. + .. function:: void version_message() const + + Output version string to the console. + .. function:: void add_global_usage(std::string const &usage) Add a global_usage for :code:`help_message()`. Example: `traffic_blabla [--SWITCH [ARG]]`. diff --git a/doc/developer-guide/layout/index.en.rst b/doc/developer-guide/layout/index.en.rst new file mode 100644 index 00000000000..4e17aebf324 --- /dev/null +++ b/doc/developer-guide/layout/index.en.rst @@ -0,0 +1,28 @@ +.. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +.. include:: ../../common.defs + +.. _layout: + +Layout +****** + +.. toctree:: + :maxdepth: 2 + + runroot.en diff --git a/doc/developer-guide/layout/runroot.en.rst b/doc/developer-guide/layout/runroot.en.rst new file mode 100644 index 00000000000..7b9c049d169 --- /dev/null +++ b/doc/developer-guide/layout/runroot.en.rst @@ -0,0 +1,99 @@ +.. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +.. include:: ../../common.defs + +.. _runroot: + +Runroot +******* + +Preface +======= + +Runroot is a powerful feature to detect and define layout at runtime. This document helps to guide through how runroot works. +For management and setup of runroots, please refer to ``appendices/command-line/traffic_layout.en`` + +Why do we need runroot +====================== + +Runroot is a replacing approach for the previous ``TS_ROOT`` logic. ``TS_ROOT`` is based on replacing a compile-time package install root location. +Layouts for many systems have data location that are absolute paths that are defined without ``$PREFIX`` variable. So, the current logic is difficult to follow +and not consistent between trafficserver programs. Furthermore, it is not easy to modify subdirectory locations. + +So, we have the runroot which makes ATS easier to use, develop and deploy. + +Main logic +========== + +Everything in runroot will go through the class:`Layout` class and all the layout is defined by a single YAML file: ``runroot.yaml``. + +Work flow: + +#. Command line option ``--run-root`` +#. ``$TS_RUNROOT`` Environment variable +#. Look in current directory and look up N (default 4) directories for ``runroot.yaml`` +#. Look in executable directory and look up N directories for ``runroot.yaml``. +#. ``$TS_ROOT`` Environment Variable +#. Compiler defaults in layout class + +Right now, the following programs are integrated with the runroot logic: +**traffic_server**, **traffic_manager**, **traffic_ctl**, **traffic_layout**, **traffic_crashlog**, **traffic_logcat**, **traffic_logstat**. + +The YAML file +============= + +The runroot file (runroot.yaml) can be placed at any location in the filesystem and still be used to define the locations for required |TS| files. +So we can run traffic_server, for example, by ``traffic_server --run-root=/some/directory/runroot.yaml``. + +Below is an example of ``runroot.yaml``. + +.. code-block:: yaml + + prefix: /home/myname/runroot + exec_prefix: /home/myname/runroot + bindir: /home/myname/runroot/bin + sbindir: /home/myname/runroot/bin + sysconfdir: /etc/trafficserver + datadir: /home/myname/runroot/share/trafficserver + includedir: /home/myname/runroot/include + libdir: /home/myname/runroot/lib + libexecdir: /libexec/trafficserver + localstatedir: /var + runtimedir: /var/trafficserver + logdir: /home/myname/runroot/logdir + cachedir: /var/trafficserver + +The path can be both relative or absolute. We can define wherever we want the directory to be. All the items we need to +put into ``runroot.yaml`` are shown above and the entries can be optional. For example, if sysconfdir is not in the file, runroot will +set the sysconfidir, at runtime, to be the default built time sysconfdir concatenated with the prefix. + +Runroot management +================== + +We can create, remove and verify runroot using ``traffic_layout`` program. It is fully documented in the appendices. + +Guide for development +===================== + +Basic runroot functionality is handled in ``runroot.cc`` and ``runroot.h`` of ``lib/tscore``. ``runroot_handler()`` and ``argparser_runroot_handler()`` +are the main methods for the runroot. The ``Layout`` class will then get the global variable from ``runroot.cc`` to set up the directories properly. + +Issue or bug +============ + +This functionality should be stable and if there is any issue or bug, please report it on the GitHub and @chitianhao. diff --git a/doc/developer-guide/plugins/actions/index.en.rst b/doc/developer-guide/plugins/actions/index.en.rst index afe6ee1cfae..7ea25614de0 100644 --- a/doc/developer-guide/plugins/actions/index.en.rst +++ b/doc/developer-guide/plugins/actions/index.en.rst @@ -97,7 +97,7 @@ Below is an example of typical usage for an action: system is initialized. We'll simply schedule an event on the continuation to occur as soon as the rest of the system is started up. */ - TSContSchedule (contp, 0, TS_THREAD_POOL_DEFAULT); + TSContScheduleOnPool (contp, 0, TS_THREAD_POOL_NET); } The example above shows a simple plugin that creates a continuation and @@ -127,7 +127,7 @@ cancel the action. The following sample code implements this: { switch (event) { case (TS_EVENT_IMMEDIATE): - TSContSchedule (contp, 30000, TS_THREAD_POOL_DEFAULT); + TSContScheduleOnPool (contp, 30000, TS_THREAD_POOL_NET); TSAction actionp = TSNetConnect(contp, 127.0.0.1, 9999); if (!TSActionDone (actionp)) { TSContDataSet (contp, actionp); @@ -168,7 +168,7 @@ cancel the action. The following sample code implements this: system is initialized. We'll simply schedule an event on the continuation to occur as soon as the rest of the system is started up. */ - TSContSchedule (contp, 0, TS_THREAD_POOL_DEFAULT); + TSContScheduleOnPool (contp, 0, TS_THREAD_POOL_NET); } The action functions are: diff --git a/doc/developer-guide/plugins/continuations/writing-handler-functions.en.rst b/doc/developer-guide/plugins/continuations/writing-handler-functions.en.rst index b4cde81f68a..7250f25c826 100644 --- a/doc/developer-guide/plugins/continuations/writing-handler-functions.en.rst +++ b/doc/developer-guide/plugins/continuations/writing-handler-functions.en.rst @@ -96,6 +96,8 @@ Event Event Sender :data:`TS_EVENT_IMMEDIATE` :func:`TSVConnClose` :func:`TSVIOReenable` :func:`TSContSchedule` + :func:`TSContScheduleOnPool` + :func:`TSContScheduleOnThread` :data:`TS_EVENT_IMMEDIATE` :data:`TS_HTTP_REQUEST_TRANSFORM_HOOK` :data:`TS_EVENT_IMMEDIATE` :data:`TS_HTTP_RESPONSE_TRANSFORM_HOOK` :data:`TS_EVENT_CACHE_OPEN_READ` :func:`TSCacheRead` Cache VC @@ -112,6 +114,8 @@ Event Event Sender :func:`TSHttpTxnIntercept` :data:`TS_EVENT_HOST_LOOKUP` :func:`TSHostLookup` :type:`TSHostLookupResult` :data:`TS_EVENT_TIMEOUT` :func:`TSContSchedule` + :func:`TSContScheduleOnPool` + :func:`TSContScheduleOnThread` :data:`TS_EVENT_ERROR` :data:`TS_EVENT_VCONN_READ_READY` :func:`TSVConnRead` :type:`TSVIO` :data:`TS_EVENT_VCONN_WRITE_READY` :func:`TSVConnWrite` :type:`TSVIO` @@ -134,3 +138,5 @@ The continuation functions are listed below: - :func:`TSContDestroy` - :func:`TSContMutexGet` - :func:`TSContSchedule` +- :func:`TSContScheduleOnPool` +- :func:`TSContScheduleOnThread` diff --git a/doc/developer-guide/plugins/example-plugins/query_remap/index.en.rst b/doc/developer-guide/plugins/example-plugins/query_remap/index.en.rst index c2a31d6dae5..5e046f88ca5 100644 --- a/doc/developer-guide/plugins/example-plugins/query_remap/index.en.rst +++ b/doc/developer-guide/plugins/example-plugins/query_remap/index.en.rst @@ -53,16 +53,16 @@ Required Functions A remap plugin is required to implement the following functions: -- `TSRemapInit `_: +- :c:func:`TSRemapInit`: the remap initialization function, called once when the plugin is loaded -- `TSRemapNewInstance `_: +- :c:func:`TSRemapNewInstance`: a new instance is created for each rule associated with the plugin. Called each time the plugin used in a remap rule (this function is what processes the pparam values) -- `TSRemapDoRemap `_: +- :c:func:`TSRemapDoRemap`: the entry point used by Traffic Server to find the new URL to which it remaps; called every time a request comes in diff --git a/doc/developer-guide/plugins/example-plugins/tls_bridge.en.rst b/doc/developer-guide/plugins/example-plugins/tls_bridge.en.rst index 1b979fea4c0..8cf444fd5ab 100644 --- a/doc/developer-guide/plugins/example-plugins/tls_bridge.en.rst +++ b/doc/developer-guide/plugins/example-plugins/tls_bridge.en.rst @@ -1,32 +1,29 @@ -.. Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at +.. Licensed to the Apache Software Foundation (ASF) under one or more contributor license + agreements. See the NOTICE file distributed with this work for additional information regarding + copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with the License. You may obtain + a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations + Unless required by applicable law or agreed to in writing, software distributed under the License + is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + or implied. See the License for the specific language governing permissions and limitations under the License. .. include:: ../../../common.defs -.. highlight:: cpp +.. highlight:: text .. default-domain:: cpp .. |Name| replace:: TLS Bridge |Name| ********** -This plugin is used to provide secured TLS tunnels for connections between a Client and a Service -via two gateway |TS| instances. By configuring the |TS| instances the level of security in the -tunnel can be easily controlled for all communications across the tunnels. +This plugin is used to provide TLS tunnels for connections between a Client and a Service via two +gateway |TS| instances using explicit proxying. By configuring the |TS| instances the level of +security in the tunnel can be easily controlled for all communications across the tunnels without +having to update the client or service. Description =========== @@ -50,24 +47,28 @@ The tunnel is sustained by two instances of |TS|. [Ingress ATS] ..> [tls_bridge\nPlugin] : Uses -The ingress |TS| accepts a connection from the Client. This connection gets intercepted by the -|Name| plugin inside |TS|. The plugin then makes a TLS connection to the peer |TS| using the -configured level of security. The original request from the Client to the ingress |TS| is then sent -to the peer |TS| to create a connection from the peer |TS| to the Service. After this the -Client has a virtual circut to the Service and can use any TCP based communication (including TLS). -Effectively the plugin causes the connectivity to work as if the Client had done the ``CONNECT`` -directly to the peer |TS|. Note this means the DNS lookup for the Service is done by the peer |TS|, -not the ingress |TS|. +The ingress |TS| accepts an HTTP ``CONNECT`` request from the Client. This connection gets +intercepted by the |Name| plugin inside |TS| if the destination matches one of the configured +destinations. The plugin then makes a TLS connection to the peer |TS| using the configured level of +security. The original ``CONNECT`` request from the Client to the ingress |TS| is then sent to the +peer |TS| to create a connection from the peer |TS| to the Service. After this the Client has a +virtual circut to the Service and can use any TCP based communication (including TLS). Effectively +the plugin causes the explicit proxy to work as if the Client had done the ``CONNECT`` directly to +the peer |TS|. Note this means the DNS lookup for the Service is done by the peer |TS|, not the +ingress |TS|. -The plugin is configured with a mapping of Service names to peer |TS| instances. The Service -names are URLs which will in the original HTTP request made by the Client after connecting to the -ingress |TS|. This means the FQDN for the Service is not resolved in the environment of the peer +The plugin is configured with a mapping of Service names to peer |TS| instances. The Service names +are URLs which will be in the original HTTP request made by the Client after connecting to the +ingress |TS|. This means the FQDN for the Service is resolved in the environment of the peer |TS| and not the ingress |TS|. Configuration ============= -|Name| requires at least two instances of |TS| (Ingress and Peer). +|Name| requires at least two instances of |TS| (Ingress and Peer). The client connects to the +ingress |TS|, and the peer |TS| connects to the service. The Peer could in theory be configured to +connect on to a further |TS| instance, acting as the ingress to that peer, but that doesn't seem a +useful case. #. Disable caching on |TS| in ``records.config``:: @@ -75,31 +76,53 @@ Configuration #. Configure the ports. - * The Peer |TS| must be listening on an SSL enabled proxy port. For instance, if the proxy port for the Peer is 4443, then configuration in ``records.config`` would have:: + * The Peer |TS| must be listening on an SSL enabled proxy port. For instance, if the proxy port + for the Peer is 4443, then configuration in ``records.config`` would have:: CONFIG proxy.config.http.server_ports STRING 4443:ssl - * The Ingress |TS| must allow ``CONNECT`` to the Peer proxy port. This would be set in ``records.config`` by:: + * The Ingress |TS| must allow ``CONNECT`` to the Peer proxy port. This would be set in + ``records.config`` by:: CONFIG proxy.config.http.connect_ports STRING 4443 The Ingress |TS| also needs ``proxy.config.http.server_ports`` configured to have proxy ports to which the Client can connect. -#. Remap is not required, however, |TS| requires remap in order to accept the request. This can be done by disabling the remap requirement:: +#. By default |TS| requires remap in order to allow to outbound request to the peer. To disable this + requirement and allow all connections, use the setting :: CONFIG proxy.config.url_remap.remap_required INT 0 - In this case |TS| will act as an open proxy which is unlikely to be a good idea. |TS| will need - to run in a restricted environment or use access control (via ``ip_allow.config`` or - ``iptables``). + In this case |TS| will act as an open proxy which is unlikely to be a good idea. Therefore if + this approach is used |TS| will need to run in a restricted environment or use access control + (via ``ip_allow.config`` or ``iptables``). + + If this is unsuitable then an identity remap rule can be added for the peer |TS|. If the peer + |TS| was named "peer.ats" and it listens on port 4443, then the remap rule would be :: + + remap https://peer.ats:4443 https://peer.ats:4443 + + Remapping will be disabled for the user agent connection and so it will not need a rule. + +#. If remap is required on the peer to enable the outbound connection from the peer to the service + (e.g. required remapping is not explicitly disabled) the destination port must be + explicitly stated [#]_. E.g. :: + + map https://service:4443 https://service:4443 + + Note this remap rule cannot alter the actual HTTP transactions between the client and service + because those happen inside what is effectively a tunnel between the client and service, + supported by the two |TS| instances. This rule serves to allows the ``CONNECT`` sent from the + ingress to cause a tunnel connection from the peer to the service. #. Configure the Ingress |TS| to verify the Peer server certificate:: CONFIG proxy.config.ssl.client.verify.server.policy STRING ENFORCED -#. Configure Certificate Authority used by the Ingress |TS| to verify the Peer server certificate. If this - is a directory all of the certificates in the directory are treated as Certificate Authorites. :: +#. Configure Certificate Authority used by the Ingress |TS| to verify the Peer server certificate. + If this is a directory, all of the certificates in the directory are treated as Certificate + Authorities. :: CONFIG proxy.config.ssl.client.CA.cert.filename STRING @@ -114,13 +137,57 @@ Configuration #. Enable the |Name| plugin in ``plugin.config``. The plugin is configured by arguments in ``plugin.config``. These are arguments are in pairs of a *destination* and a *peer*. The - destination is a anchored regular expression which is matched against the host name in the Client - ``CONNECT``. The destinations are checked in order and the first match is used to select the Peer + destination is an anchored regular expression which is matched against the host name in the Client + ``CONNECT``. The destinations are checked in order and the first match is used to select the peer |TS|. The peer should be an FQDN or IP address with an optional port. For the example above, if - the Peer |TS| was named "peer.example.com" on port 4443 and the Service at ``*.service.com``, the - peer argument would be "peer.example.com:4443". In ``plugin.config`` this would be:: + the Peer |TS| was named "peer.ats" on port 4443 and the Service at ``*.service.com``, the + peer argument would be "peer.ats:4443". In ``plugin.config`` this would be:: + + tls_bridge.so .*[.]service[.]com peer.ats:4443 + + Note the '.' characters are escaped with brackets so that, for instance, "someservice.com" does + not match the rule. + + If there was another service, "\*.altsvc.ats", via a different peer "altpeer.ats" on port 4443, + the configuration would be :: + + tls_bridge.so .*[.]service[.]com peer.ats:4443 .*[.]altsvc.ats altpeer.ats:4443 + + Mappings can also be specified in an external file. For instance, if there was file named + "bridge.config" in the default |TS| configuration directory which contained mappings, the + ``plugin.config`` configuration line could look like :: + + tls_bridge.so .*[.]service[.]com peer.ats:4443 --file bridge.config + + or - tls_bridge.so .*[.]service[.]com peer.example.com:4443 + tls_bridge.so --file bridge.config .*[.]service[.]com peer.ats:4443 + + These are not identical - direct mappings and file mappings are processed in order. This means in + the first example, the direct mapping is checked before any mappping in "bridge.config", and in + the latter example the mappings in "bridge.config" are checked before the direct mappings. There + can be multiple "--file" arguments, which are processed in the order they appear in + "plugin.config". The file name can be absolute, or relative. If the file name is relative, it is + relative to the |TS| configuration directory. Therefore, in these examples, "bridge.config" must + be in the same directory as ``plugin.config``. + + The contents of "bridge.config" must be one mapping per line, with a regular expression separated + by white space from the destination service. This is identical to the format in ``plugin.config`` + except there is only one pair per line. E.g., valid content for "bridge.config" could be :: + + # Primary service location. + .*[.]service[.]com peer.ats:4443 + + # Secondary. + .*[.]altsvc.ats altpeer.ats:4443 + + Leading whitespace on a line is ignored, and if the first non-whitespace character is '#' then + the entire line is ignored. Therefore if that is the content of "bridge.config", these two + lines in "plugin.config" would behave identically :: + + tls_bridge.so --file bridge.config + + tls_bridge.so .*[.]service[.]com peer.ats:4443 .*[.]altsvc.ats altpeer.ats:4443 Notes ===== @@ -161,7 +228,7 @@ If the session is valid then a separate connection to the peer |TS| is created u :code:`TSHttpConnect`. After the ingress |TS| connects to the peer |TS| it sends a duplicate of the Client ``CONNECT`` -request. This is processed by the peer |TS| to connect on to the Service. After this both |TS| +request. This is processed by the peer |TS| to connect to the Service. After this both |TS| instances then tunnel data between the Client and the Service, in effect becoming a transparent tunnel. @@ -205,7 +272,7 @@ The overall exchange looks like the following: A detailed view of the plugin operation. -.. image:: ../../../uml/images/TLS-Bridge-Plugin.svg +.. figure:: ../../../uml/images/TLS-Bridge-Plugin.svg :align: center A sequence diagram focusing on the request / response data flow. There is a :code:`NetVConn` for the @@ -220,7 +287,7 @@ The :code:`200 OK` sent from the Peer |TS| is parsed and consumed by the plugin. means there was an error and the tunnel is shut down. To deal with the Client response clean up the response code is stored and used later during cleanup. -.. image:: ../../../uml/images/TLS-Bridge-Messages.svg +.. figure:: ../../../uml/images/TLS-Bridge-Messages.svg :align: center A restartable state machine is used to recognize the end of the Peer |TS| response. The initial part @@ -243,3 +310,13 @@ socket read. State_3 --> State_1 : CR State_3 --> State_0 : * @enduml + +Debugging +--------- + +Debugging messages for the plugin can be enabled with the "tls_bridge" debug tag. + + +.. rubric:: Footnotes + +.. [#] This is likely due to a bug in |TS|, currently under investigation. diff --git a/doc/developer-guide/plugins/hooks-and-transactions/ssl-hooks.en.rst b/doc/developer-guide/plugins/hooks-and-transactions/ssl-hooks.en.rst index 40f484d2792..1b7d49c7c0c 100644 --- a/doc/developer-guide/plugins/hooks-and-transactions/ssl-hooks.en.rst +++ b/doc/developer-guide/plugins/hooks-and-transactions/ssl-hooks.en.rst @@ -22,9 +22,9 @@ TLS User Agent Hooks ******************** -In addition to the HTTP oriented hooks, a plugin can add hooks to trigger code -during the TLS handshake with the user agent. This TLS handshake occurs well before -the HTTP transaction is available, so a separate state machine is required to track the +In addition to the HTTP oriented hooks, a plugin can add hooks (by calling :c:func:`TSHttpHookAdd`) +to trigger code during the TLS handshake with the user agent. This TLS handshake occurs well +before the HTTP transaction is available, so a separate state machine is required to track the TLS hooks. TLS Hooks @@ -60,6 +60,12 @@ TS_VCONN_CLOSE_HOOK This hook is invoked after the SSL handshake is done and when the IO is closing. The TSVConnArgs should be cleaned up here. A callback at this point must reenable. +TS_SSL_CLIENT_HELLO_HOOK +------------------------ +This hook is called when the client hello arrived for the TLS handshake. If called it will always be called after TS_VCONN_START_HOOK. The plugin callback can execute code to examine client hello information. + +TLS handshake processing will pause until the hook callback executes :c:func:`TSVConnReenable()`. + TS_SSL_SERVERNAME_HOOK ---------------------- @@ -87,7 +93,7 @@ a certificate. TS_SSL_VERIFY_CLIENT_HOOK ------------------------- -This hook is called when a client connects to Traffic Server and presents a +This hook is called when a client connects to Traffic Server and presents a client certificate in the case of a mutual TLS handshake. The callback can get the SSL object from the TSVConn argument and use that to access the client certificate and make any additional checks. @@ -110,7 +116,7 @@ for pausing processing during the certificate verify callback. TS_VCONN_OUTBOUND_START_HOOK ---------------------------- -This hook is invoked after ATS has connected to the upstream server and before the SSL handshake has started. This gives the plugin the option of +This hook is invoked after ATS has connected to the upstream server and before the SSL handshake has started. This gives the plugin the option of overriding the default SSL connection options on the SSL object. In theory this hook could apply and be useful for non-SSL connections as well, but at this point this hook is only called in the SSL sequence. @@ -123,51 +129,75 @@ TS_VCONN_OUTBOUND_CLOSE_HOOK This hook is invoked after the SSL handshake is done and right before the outbound connection closes. A callback at this point must reenable. -TLS Hook State Diagram ----------------------- +TLS Inbound Hook State Diagram +------------------------------ .. graphviz:: :alt: TLS Inbound Hook State Diagram digraph tls_hook_state_diagram{ HANDSHAKE_HOOKS_PRE -> TS_VCONN_START_HOOK; - HANDSHAKE_HOOKS_PRE -> TS_SSL_VERIFY_CLIENT_HOOK; HANDSHAKE_HOOKS_PRE -> TS_SSL_CERT_HOOK; HANDSHAKE_HOOKS_PRE -> TS_SSL_SERVERNAME_HOOK; HANDSHAKE_HOOKS_PRE -> HANDSHAKE_HOOKS_DONE; - TS_SSL_VERIFY_CLIENT_HOOK -> HANDSHAKE_HOOKS_PRE; TS_VCONN_START_HOOK -> HANDSHAKE_HOOKS_PRE_INVOKE; HANDSHAKE_HOOKS_PRE_INVOKE -> TSVConnReenable; TSVConnReenable -> HANDSHAKE_HOOKS_PRE; + TS_SSL_CLIENT_HELLO_HOOK -> HANDSHAKE_HOOKS_CLIENT_HELLO_INVOKE; + HANDSHAKE_HOOKS_CLIENT_HELLO_INVOKE -> TSVConnReenable2; + TSVConnReenable2 -> HANDSHAKE_HOOKS_CLIENT_HELLO; + HANDSHAKE_HOOKS_CLIENT_HELLO -> TS_SSL_CLIENT_HELLO_HOOK; + HANDSHAKE_HOOKS_CLIENT_HELLO -> TS_SSL_SERVERNAME_HOOK; TS_SSL_SERVERNAME_HOOK -> HANDSHAKE_HOOKS_SNI; HANDSHAKE_HOOKS_SNI -> TS_SSL_SERVERNAME_HOOK; HANDSHAKE_HOOKS_SNI -> TS_SSL_CERT_HOOK; HANDSHAKE_HOOKS_SNI -> HANDSHAKE_HOOKS_DONE; HANDSHAKE_HOOKS_CERT -> TS_SSL_CERT_HOOK; TS_SSL_CERT_HOOK -> HANDSHAKE_HOOKS_CERT_INVOKE; - HANDSHAKE_HOOKS_CERT_INVOKE -> TSVConnReenable2; - TSVConnReenable2 -> HANDSHAKE_HOOKS_CERT; + HANDSHAKE_HOOKS_CERT_INVOKE -> TSVConnReenable3; + TSVConnReenable3 -> HANDSHAKE_HOOKS_CERT; + + HANDSHAKE_HOOKS_CERT -> TS_SSL_VERIFY_CLIENT_HOOK; + HANDSHAKE_HOOKS_SNI -> TS_SSL_VERIFY_CLIENT_HOOK; + HANDSHAKE_HOOKS_PRE -> TS_SSL_VERIFY_CLIENT_HOOK; + TS_SSL_VERIFY_CLIENT_HOOK -> HANDSHAKE_HOOKS_VERIFY; + HANDSHAKE_HOOKS_VERIFY -> TS_SSL_VERIFY_CLIENT_HOOK; + HANDSHAKE_HOOKS_VERIFY -> HANDSHAKE_HOOKS_DONE; + HANDSHAKE_HOOKS_CERT -> HANDSHAKE_HOOKS_DONE; HANDSHAKE_HOOKS_DONE -> TS_VCONN_CLOSE_HOOK; + TS_VCONN_CLOSE_HOOK -> HANDSHAKE_HOOKS_DONE; HANDSHAKE_HOOKS_PRE [shape=box]; - TS_VCONN_START_HOOK [shape=box]; - TS_SSL_VERIFY_CLIENT_HOOK [shape=box]; HANDSHAKE_HOOKS_PRE_INVOKE [shape=box]; + HANDSHAKE_HOOKS_CLIENT_HELLO [shape=box]; + HANDSHAKE_HOOKS_CLIENT_HELLO_INVOKE [shape=box]; HANDSHAKE_HOOKS_SNI [shape=box]; + HANDSHAKE_HOOKS_VERIFY [shape=box]; HANDSHAKE_HOOKS_CERT [shape=box]; HANDSHAKE_HOOKS_CERT_INVOKE [shape=box]; HANDSHAKE_HOOKS_DONE [shape=box]; } +TLS Outbound Hook State Diagram +------------------------------- + .. graphviz:: :alt: TLS Outbound Hook State Diagram digraph tls_hook_state_diagram{ - HANDSHAKE_HOOKS_OUTBOUND_PRE -> HANDSHAKE_HOOKS_OUTBOUND_PRE_INVOKE; - HANDSHAKE_HOOKS_PRE_INVOKE -> TSVConnReenable; + HANDSHAKE_HOOKS_OUTBOUND_PRE -> TS_VCONN_OUTBOUND_START_HOOK; + TS_VCONN_OUTBOUND_START_HOOK -> HANDSHAKE_HOOKS_OUTBOUND_PRE_INVOKE; + HANDSHAKE_HOOKS_OUTBOUND_PRE_INVOKE-> TSVConnReenable; TSVConnReenable -> HANDSHAKE_HOOKS_OUTBOUND_PRE; - HANDSHAKE_HOOKS_OUTBOUND_PRE -> HANDSHAKE_HOOKS_DONE; - HANDSHAKE_HOOKS_DONE -> HANDSHAKE_HOOKS_OUTBOUND_CLOSE; + HANDSHAKE_HOOKS_OUTBOUND_PRE -> TS_SSL_VERIFY_SERVER_HOOK; + TS_SSL_VERIFY_SERVER_HOOK -> HANDSHAKE_HOOKS_OUTBOUND_PRE; + HANDSHAKE_HOOKS_OUTBOUND_PRE -> TS_VCONN_OUTBOUND_CLOSE; + TS_VCONN_OUTBOUND_CLOSE -> HANDSHAKE_HOOKS_OUTBOUND_PRE; + TS_VCONN_OUTBOUND_CLOSE -> HANDSHAKE_HOOKS_DONE; + + HANDSHAKE_HOOKS_OUTBOUND_PRE [shape=box]; + HANDSHAKE_HOOKS_OUTBOUND_PRE_INVOKE [shape=box]; + HANDSHAKE_HOOKS_DONE [shape=box]; } diff --git a/doc/getting-started/index.en.rst b/doc/getting-started/index.en.rst index b9e7efcb2a9..1713768fdef 100644 --- a/doc/getting-started/index.en.rst +++ b/doc/getting-started/index.en.rst @@ -154,7 +154,6 @@ libraries on the machine used to build |TS|: - gcc (>= 4.3 or clang > 3.0) - GNU make - openssl -- tcl - pcre - libcap - flex (for TPROXY) @@ -284,7 +283,12 @@ and want little more than to proxy all requests to our single origin server. This is accomplished with the following rule added to the :file:`remap.config` configuration:: - regex_map http://(.*)/ http://localhost:80/ + map http://www.acme.com/ http://localhost:80/ + +With this mapping rule, all paths that |TS| receives with a Host: header of +``www.acme.com`` will be proxied to ``localhost:80``. For instance, a request +for ``http://www.acme.com/foo/bar`` will be proxied to ``http://localhost:80/foo/bar``, +while requests with other Host: headers will be rejected. It is worth pausing at this point to note that in a reverse proxying scenario, it is |TS| itself which should be responding to HTTP requests made to your @@ -304,13 +308,46 @@ they reconfigure their origin service to listen on port ``8080`` instead of the default, and change |TS| to bind to ``80`` itself. Updating the remap is thus required, and it should now be:: - regex_map http://(.*)/ http://localhost:8080/ + map http://www.acme.com/ http://localhost:8080/ Now all requests made to ``www.acme.com`` are received by |TS| which knows to proxy those requests to ``localhost:8080`` if it cannot already serve them from its cache. Because we enabled pristine host headers earlier, the origin service will continue to receive ``Host: www.acme.com`` in the HTTP request. +If |AW| decides to use |TS| to reverse proxy a second domain ``static.acme.com`` +with a different origin server than the original, they need to make further +changes, as a new remap line needs to be added to handle the additional domain:: + + map http://static.acme.com/ http://origin-static.acme.com/ + +If they also decide to have requests to ``www.acme.com`` with paths that start with +``/api`` to a different origin server. The api origin server shouldn't get the ``/api``, +they will remap it away. And, since the above remap rules catch all paths, +this remap rule needs to be above it:: + + map http://www.acme.com/api/ http://api-origin.acme.com/ + +With this remap rule in place, a request to ``http://www.acme.com/api/example/foo`` +will be proxied to ``http://api-origin.acme.com/example/foo``. + +Finally, if |AW| decides to secure their site with https, they will need two +additional remap rules to handle the https requests. |TS| can translate an inbound +https request to an http request to origin. So, they would have additional remap +rules like:: + + map https://www.acme.com/ http://localhost:8080/ + map https://static.acme.com/ https://origin-static.acme.com/ + +This will require installing a certificate, and adding a line to +:file:`ssl_multicert.config`. Assuming the cert has the static.acme.com alternate +name, and that cert should be presented by default:: + + dest_ip=* ssl_cert_name=/path/to/secret/privatekey/acme.rsa + +Further information about configuring |TS| for TLS can be found :ref:`admin-ssl-termination` +section of the documentation. + Adjust Cache Parameters ~~~~~~~~~~~~~~~~~~~~~~~ @@ -346,12 +383,21 @@ entries: :file:`remap.config`:: - regex_map http://(.*)/ http://localhost:8080/ + map http://www.acme.com/api/ http://api-origin.acme.com/ + map https://www.acme.com/api/ https://api-origin.acme.com/ + map http://www.acme.com/ http://localhost:8080/ + map https://www.acme.com/ http://localhost:8080/ + map http://static.acme.com/ http://origin-static.acme.com/ + map https://static.acme.com/ https://origin-static.acme.com/ :file:`storage.config`:: /cache/trafficserver 500G +:file:`ssl_multicert.config`:: + + ssl_cert_name=/path/to/secret/acme.rsa + Configuring A Forward Proxy --------------------------- @@ -424,15 +470,6 @@ or instead of, the default |TS| logs. The Administrator's Guide discusses logging options in great detail in :ref:`admin-logging`. -Using Traffic Top ------------------ - -Using Stats Over HTTP ---------------------- - -Using Cache Inspector ---------------------- - Further Steps ============= diff --git a/doc/locale/ja/LC_MESSAGES/admin-guide/configuration/explicit-forward-proxying.en.po b/doc/locale/ja/LC_MESSAGES/admin-guide/configuration/explicit-forward-proxying.en.po index 898d5d34232..25e8b422934 100644 --- a/doc/locale/ja/LC_MESSAGES/admin-guide/configuration/explicit-forward-proxying.en.po +++ b/doc/locale/ja/LC_MESSAGES/admin-guide/configuration/explicit-forward-proxying.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/admin-guide/configuration/hierachical-caching.en.po b/doc/locale/ja/LC_MESSAGES/admin-guide/configuration/hierachical-caching.en.po index 5da856087b6..50452ac181b 100644 --- a/doc/locale/ja/LC_MESSAGES/admin-guide/configuration/hierachical-caching.en.po +++ b/doc/locale/ja/LC_MESSAGES/admin-guide/configuration/hierachical-caching.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/admin-guide/configuration/transparent-forward-proxying.en.po b/doc/locale/ja/LC_MESSAGES/admin-guide/configuration/transparent-forward-proxying.en.po index 98980e657cc..13f8b27e02b 100644 --- a/doc/locale/ja/LC_MESSAGES/admin-guide/configuration/transparent-forward-proxying.en.po +++ b/doc/locale/ja/LC_MESSAGES/admin-guide/configuration/transparent-forward-proxying.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/admin-guide/files/hosting.config.en.po b/doc/locale/ja/LC_MESSAGES/admin-guide/files/hosting.config.en.po index fadf1bf7ebe..9b81249152c 100644 --- a/doc/locale/ja/LC_MESSAGES/admin-guide/files/hosting.config.en.po +++ b/doc/locale/ja/LC_MESSAGES/admin-guide/files/hosting.config.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/admin-guide/files/ip_allow.config.en.po b/doc/locale/ja/LC_MESSAGES/admin-guide/files/ip_allow.config.en.po index 6458c172ad4..8ea799a0141 100644 --- a/doc/locale/ja/LC_MESSAGES/admin-guide/files/ip_allow.config.en.po +++ b/doc/locale/ja/LC_MESSAGES/admin-guide/files/ip_allow.config.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/admin-guide/files/records.config.en.po b/doc/locale/ja/LC_MESSAGES/admin-guide/files/records.config.en.po index 6838781cfa0..baab72cf7df 100644 --- a/doc/locale/ja/LC_MESSAGES/admin-guide/files/records.config.en.po +++ b/doc/locale/ja/LC_MESSAGES/admin-guide/files/records.config.en.po @@ -2335,7 +2335,7 @@ msgid "" "Limits the number of requests to be queued when the :ts:cv:`proxy.config." "http.origin_max_connections` is reached. When disabled (``-1``) requests " "are will wait indefinitely for an available connection. When set to ``0`` " -"all requests past the :ts:cv:`proxy.config.http.origin_max_connections` " +"all requests past the :ts:cv:`proxy.config.http.per_server.connection.max` " "will immediately fail. When set to ``>0`` ATS will queue that many requests " "to go to the origin, any additional requests past the limit will " "immediately fail." diff --git a/doc/locale/ja/LC_MESSAGES/admin-guide/files/volume.config.en.po b/doc/locale/ja/LC_MESSAGES/admin-guide/files/volume.config.en.po index abb6c957a89..f866d7640e7 100644 --- a/doc/locale/ja/LC_MESSAGES/admin-guide/files/volume.config.en.po +++ b/doc/locale/ja/LC_MESSAGES/admin-guide/files/volume.config.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/admin-guide/index.en.po b/doc/locale/ja/LC_MESSAGES/admin-guide/index.en.po index cd5965100d0..143016aab78 100644 --- a/doc/locale/ja/LC_MESSAGES/admin-guide/index.en.po +++ b/doc/locale/ja/LC_MESSAGES/admin-guide/index.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/admin-guide/installation/index.en.po b/doc/locale/ja/LC_MESSAGES/admin-guide/installation/index.en.po index 1b8840e5bad..107ca84ef96 100644 --- a/doc/locale/ja/LC_MESSAGES/admin-guide/installation/index.en.po +++ b/doc/locale/ja/LC_MESSAGES/admin-guide/installation/index.en.po @@ -143,10 +143,6 @@ msgstr "" msgid "openssl" msgstr "openssl" -#: ../../../admin-guide/installation/index.en.rst:108 -msgid "tcl" -msgstr "tcl" - #: ../../../admin-guide/installation/index.en.rst:109 msgid "expat" msgstr "expat" diff --git a/doc/locale/ja/LC_MESSAGES/admin-guide/monitoring/statistics/core/http-transaction.en.po b/doc/locale/ja/LC_MESSAGES/admin-guide/monitoring/statistics/core/http-transaction.en.po index 3781d5d0f96..471363d1de7 100644 --- a/doc/locale/ja/LC_MESSAGES/admin-guide/monitoring/statistics/core/http-transaction.en.po +++ b/doc/locale/ja/LC_MESSAGES/admin-guide/monitoring/statistics/core/http-transaction.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/admin-guide/plugins/authproxy.en.po b/doc/locale/ja/LC_MESSAGES/admin-guide/plugins/authproxy.en.po index 95b232c44ca..c9e6dbaf5fb 100644 --- a/doc/locale/ja/LC_MESSAGES/admin-guide/plugins/authproxy.en.po +++ b/doc/locale/ja/LC_MESSAGES/admin-guide/plugins/authproxy.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/admin-guide/plugins/buffer_upload.en.po b/doc/locale/ja/LC_MESSAGES/admin-guide/plugins/buffer_upload.en.po index 0360eae81b2..b8074d755e8 100644 --- a/doc/locale/ja/LC_MESSAGES/admin-guide/plugins/buffer_upload.en.po +++ b/doc/locale/ja/LC_MESSAGES/admin-guide/plugins/buffer_upload.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/admin-guide/plugins/esi.en.po b/doc/locale/ja/LC_MESSAGES/admin-guide/plugins/esi.en.po index 0ec13c7536e..85b8a87e4c9 100644 --- a/doc/locale/ja/LC_MESSAGES/admin-guide/plugins/esi.en.po +++ b/doc/locale/ja/LC_MESSAGES/admin-guide/plugins/esi.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/admin-guide/plugins/generator.en.po b/doc/locale/ja/LC_MESSAGES/admin-guide/plugins/generator.en.po index 42dce50464c..e03365d616f 100644 --- a/doc/locale/ja/LC_MESSAGES/admin-guide/plugins/generator.en.po +++ b/doc/locale/ja/LC_MESSAGES/admin-guide/plugins/generator.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/admin-guide/plugins/geoip_acl.en.po b/doc/locale/ja/LC_MESSAGES/admin-guide/plugins/geoip_acl.en.po index d8d2b71a070..884b3edf271 100644 --- a/doc/locale/ja/LC_MESSAGES/admin-guide/plugins/geoip_acl.en.po +++ b/doc/locale/ja/LC_MESSAGES/admin-guide/plugins/geoip_acl.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/admin-guide/plugins/mp4.en.po b/doc/locale/ja/LC_MESSAGES/admin-guide/plugins/mp4.en.po index 849046a921a..e7adaa156c2 100644 --- a/doc/locale/ja/LC_MESSAGES/admin-guide/plugins/mp4.en.po +++ b/doc/locale/ja/LC_MESSAGES/admin-guide/plugins/mp4.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/admin-guide/plugins/mysql_remap.en.po b/doc/locale/ja/LC_MESSAGES/admin-guide/plugins/mysql_remap.en.po index 426ff9e7c83..4bbd60e970e 100644 --- a/doc/locale/ja/LC_MESSAGES/admin-guide/plugins/mysql_remap.en.po +++ b/doc/locale/ja/LC_MESSAGES/admin-guide/plugins/mysql_remap.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/admin-guide/plugins/regex_remap.en.po b/doc/locale/ja/LC_MESSAGES/admin-guide/plugins/regex_remap.en.po index b487d7dfa2f..ac912880bc6 100644 --- a/doc/locale/ja/LC_MESSAGES/admin-guide/plugins/regex_remap.en.po +++ b/doc/locale/ja/LC_MESSAGES/admin-guide/plugins/regex_remap.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/admin-guide/plugins/tcpinfo.en.po b/doc/locale/ja/LC_MESSAGES/admin-guide/plugins/tcpinfo.en.po index 57949aa6aab..0b180032772 100644 --- a/doc/locale/ja/LC_MESSAGES/admin-guide/plugins/tcpinfo.en.po +++ b/doc/locale/ja/LC_MESSAGES/admin-guide/plugins/tcpinfo.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/admin-guide/transparent-proxy.en.po b/doc/locale/ja/LC_MESSAGES/admin-guide/transparent-proxy.en.po index 5ca8406e580..3a88821fa8e 100644 --- a/doc/locale/ja/LC_MESSAGES/admin-guide/transparent-proxy.en.po +++ b/doc/locale/ja/LC_MESSAGES/admin-guide/transparent-proxy.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/admin-guide/transparent-proxy/router-inline.en.po b/doc/locale/ja/LC_MESSAGES/admin-guide/transparent-proxy/router-inline.en.po index 810df020eed..887c4215b53 100644 --- a/doc/locale/ja/LC_MESSAGES/admin-guide/transparent-proxy/router-inline.en.po +++ b/doc/locale/ja/LC_MESSAGES/admin-guide/transparent-proxy/router-inline.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/admin-guide/transparent-proxy/wccp-configuration.en.po b/doc/locale/ja/LC_MESSAGES/admin-guide/transparent-proxy/wccp-configuration.en.po index ed17753417a..d4c7fed49e0 100644 --- a/doc/locale/ja/LC_MESSAGES/admin-guide/transparent-proxy/wccp-configuration.en.po +++ b/doc/locale/ja/LC_MESSAGES/admin-guide/transparent-proxy/wccp-configuration.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/appendices/command-line/traffic_crashlog.en.po b/doc/locale/ja/LC_MESSAGES/appendices/command-line/traffic_crashlog.en.po index 54968ff9e52..66f8354ff51 100644 --- a/doc/locale/ja/LC_MESSAGES/appendices/command-line/traffic_crashlog.en.po +++ b/doc/locale/ja/LC_MESSAGES/appendices/command-line/traffic_crashlog.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/appendices/command-line/traffic_logcat.en.po b/doc/locale/ja/LC_MESSAGES/appendices/command-line/traffic_logcat.en.po index bf0353a79d5..a5545be26fd 100644 --- a/doc/locale/ja/LC_MESSAGES/appendices/command-line/traffic_logcat.en.po +++ b/doc/locale/ja/LC_MESSAGES/appendices/command-line/traffic_logcat.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/appendices/command-line/traffic_logstats.en.po b/doc/locale/ja/LC_MESSAGES/appendices/command-line/traffic_logstats.en.po index 4be0c589fdc..f6b8e1c1d7e 100644 --- a/doc/locale/ja/LC_MESSAGES/appendices/command-line/traffic_logstats.en.po +++ b/doc/locale/ja/LC_MESSAGES/appendices/command-line/traffic_logstats.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/appendices/command-line/traffic_manager.en.po b/doc/locale/ja/LC_MESSAGES/appendices/command-line/traffic_manager.en.po index ac9dff52e23..582f35806a5 100644 --- a/doc/locale/ja/LC_MESSAGES/appendices/command-line/traffic_manager.en.po +++ b/doc/locale/ja/LC_MESSAGES/appendices/command-line/traffic_manager.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/appendices/command-line/traffic_top.en.po b/doc/locale/ja/LC_MESSAGES/appendices/command-line/traffic_top.en.po index 0905d2f3302..e2ba0406339 100644 --- a/doc/locale/ja/LC_MESSAGES/appendices/command-line/traffic_top.en.po +++ b/doc/locale/ja/LC_MESSAGES/appendices/command-line/traffic_top.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/appendices/command-line/tspush.en.po b/doc/locale/ja/LC_MESSAGES/appendices/command-line/tspush.en.po index 746e4503add..6b48c36fe25 100644 --- a/doc/locale/ja/LC_MESSAGES/appendices/command-line/tspush.en.po +++ b/doc/locale/ja/LC_MESSAGES/appendices/command-line/tspush.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/appendices/command-line/tsxs.en.po b/doc/locale/ja/LC_MESSAGES/appendices/command-line/tsxs.en.po index 2d5236118de..4f95157fdd9 100644 --- a/doc/locale/ja/LC_MESSAGES/appendices/command-line/tsxs.en.po +++ b/doc/locale/ja/LC_MESSAGES/appendices/command-line/tsxs.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/appendices/http-status-codes.en.po b/doc/locale/ja/LC_MESSAGES/appendices/http-status-codes.en.po index 42803e27e66..4ab407ab18b 100644 --- a/doc/locale/ja/LC_MESSAGES/appendices/http-status-codes.en.po +++ b/doc/locale/ja/LC_MESSAGES/appendices/http-status-codes.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSAPI.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSAPI.en.po index 2723918c9ce..d74a3862f4d 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSAPI.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSAPI.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSActionCancel.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSActionCancel.en.po index be14cbdf122..1180534fe00 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSActionCancel.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSActionCancel.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSActionDone.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSActionDone.en.po index ea5c22a1a4d..0aeeea82444 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSActionDone.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSActionDone.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSCacheRead.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSCacheRead.en.po index f7ee312ee3b..7e5924002ad 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSCacheRead.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSCacheRead.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSCacheRemove.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSCacheRemove.en.po index 60feac862b4..b522dc19708 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSCacheRemove.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSCacheRemove.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSCacheWrite.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSCacheWrite.en.po index ddf7af075dc..02da2025a9d 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSCacheWrite.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSCacheWrite.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSConfigDataGet.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSConfigDataGet.en.po index 014db9ff5e0..d96fe31285d 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSConfigDataGet.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSConfigDataGet.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSConfigGet.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSConfigGet.en.po index 62e198cc6df..6f3fb508c6c 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSConfigGet.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSConfigGet.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSConfigRelease.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSConfigRelease.en.po index ed00a3ebbb9..89955262cee 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSConfigRelease.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSConfigRelease.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSConfigSet.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSConfigSet.en.po index 85c9af32fa1..72fca4c4e13 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSConfigSet.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSConfigSet.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSContCall.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSContCall.en.po index 8004e6299d4..1076f0e3a81 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSContCall.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSContCall.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSContCreate.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSContCreate.en.po index 5bb4a37b726..12106d4eb34 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSContCreate.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSContCreate.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSContDataGet.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSContDataGet.en.po index 83488fbf677..30c359004b1 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSContDataGet.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSContDataGet.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSContDataSet.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSContDataSet.en.po index 702e290a661..f9e9e59e631 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSContDataSet.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSContDataSet.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSContDestroy.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSContDestroy.en.po index 335d4e66df6..5cc42583257 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSContDestroy.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSContDestroy.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSContMutexGet.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSContMutexGet.en.po index f3822765e81..b9f6c1ad207 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSContMutexGet.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSContMutexGet.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSContSchedule.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSContSchedule.en.po index 5e4612a3108..e35760b7bd3 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSContSchedule.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSContSchedule.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHostLookup.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHostLookup.en.po index 214e1bf5ca7..c2819e57ed6 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHostLookup.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHostLookup.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHostLookupResultAddrGet.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHostLookupResultAddrGet.en.po index aa2427ec68b..affce1a85a1 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHostLookupResultAddrGet.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHostLookupResultAddrGet.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpConnect.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpConnect.en.po index ed111ffed28..889c2a586ce 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpConnect.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpConnect.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpHdrClone.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpHdrClone.en.po index 81265bd73db..63296925cdd 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpHdrClone.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpHdrClone.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpHdrCreate.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpHdrCreate.en.po index 5b8739b6ce4..feadb855576 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpHdrCreate.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpHdrCreate.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpHdrDestroy.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpHdrDestroy.en.po index 834d7eeb8a2..d598b386a4b 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpHdrDestroy.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpHdrDestroy.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpHdrHostGet.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpHdrHostGet.en.po index 5b144f1538e..944086dd43c 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpHdrHostGet.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpHdrHostGet.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpHdrLengthGet.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpHdrLengthGet.en.po index 02c092e40c9..d1221964437 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpHdrLengthGet.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpHdrLengthGet.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpHdrMethodGet.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpHdrMethodGet.en.po index a12b8d9875b..037932cb44f 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpHdrMethodGet.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpHdrMethodGet.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpHdrMethodSet.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpHdrMethodSet.en.po index 47bc6b34bcf..7c1cd9f97d1 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpHdrMethodSet.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpHdrMethodSet.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpHdrPrint.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpHdrPrint.en.po index d5adb998de8..3959eb46623 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpHdrPrint.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpHdrPrint.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpHdrReasonGet.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpHdrReasonGet.en.po index 559c0b0f777..d715b5fe8a9 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpHdrReasonGet.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpHdrReasonGet.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpHdrReasonLookup.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpHdrReasonLookup.en.po index dc314ae68e6..1a0d5cdda15 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpHdrReasonLookup.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpHdrReasonLookup.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpHdrReasonSet.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpHdrReasonSet.en.po index 0aa7fe08a1c..3bf0ef3adcc 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpHdrReasonSet.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpHdrReasonSet.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpHdrStatusGet.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpHdrStatusGet.en.po index 1a247fe2566..1cf6c1365cb 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpHdrStatusGet.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpHdrStatusGet.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpHdrStatusSet.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpHdrStatusSet.en.po index 14ca8b2d27f..3e7a9c11920 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpHdrStatusSet.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpHdrStatusSet.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpHdrTypeGet.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpHdrTypeGet.en.po index e7e194bd994..8dfb2bf8814 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpHdrTypeGet.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpHdrTypeGet.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpHdrTypeSet.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpHdrTypeSet.en.po index 8dfd75c4ec3..08bc78171d4 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpHdrTypeSet.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpHdrTypeSet.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpHdrUrlGet.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpHdrUrlGet.en.po index 4277d5ca39d..3ad4f3bbbe8 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpHdrUrlGet.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpHdrUrlGet.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpHdrUrlSet.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpHdrUrlSet.en.po index 4da5c060b6c..115321d97f6 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpHdrUrlSet.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpHdrUrlSet.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpHdrVersionGet.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpHdrVersionGet.en.po index 32f98824ef4..5129d1b6b23 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpHdrVersionGet.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpHdrVersionGet.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpHdrVersionSet.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpHdrVersionSet.en.po index c0d18a515ef..36b53b7ea25 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpHdrVersionSet.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpHdrVersionSet.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpHookAdd.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpHookAdd.en.po index ee687e7d558..cb2010b4cf5 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpHookAdd.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpHookAdd.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpOverridableConfig.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpOverridableConfig.en.po index 20e89995138..01ef8bb56b8 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpOverridableConfig.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpOverridableConfig.en.po @@ -270,10 +270,6 @@ msgstr ":ts:cv:`proxy.config.http.cache.heuristic_min_lifetime`" msgid ":ts:cv:`proxy.config.websocket.active_timeout`" msgstr ":ts:cv:`proxy.config.http.cache.heuristic_min_lifetime`" -#: ../../../developer-guide/api/functions/TSHttpOverridableConfig.en.rst:112 -msgid ":ts:cv:`proxy.config.http.origin_max_connections`" -msgstr "" - #: ../../../developer-guide/api/functions/TSHttpOverridableConfig.en.rst:113 msgid ":ts:cv:`proxy.config.http.connect_attempts_max_retries`" msgstr "" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpParserCreate.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpParserCreate.en.po index 0013a3867ac..6236c69515c 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpParserCreate.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpParserCreate.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpSsnClientFdGet.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpSsnClientFdGet.en.po index ee240b013f8..ab881f83d05 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpSsnClientFdGet.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpSsnClientFdGet.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpSsnReenable.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpSsnReenable.en.po index 8cefe6ad19b..ca5ea4613f3 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpSsnReenable.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpSsnReenable.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnCacheLookupStatusGet.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnCacheLookupStatusGet.en.po index 7501beaa18f..5c69d1f134f 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnCacheLookupStatusGet.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnCacheLookupStatusGet.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnCacheLookupUrlGet.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnCacheLookupUrlGet.en.po index b324ee81fe9..f8e1fa7e53c 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnCacheLookupUrlGet.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnCacheLookupUrlGet.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnCachedReqGet.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnCachedReqGet.en.po index 5c1d0a2f0ba..f520c0d5801 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnCachedReqGet.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnCachedReqGet.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnCachedRespGet.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnCachedRespGet.en.po index 098f74abfd2..97ea157bd56 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnCachedRespGet.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnCachedRespGet.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnClientFdGet.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnClientFdGet.en.po index ff2386085ed..6bec3a2adb3 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnClientFdGet.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnClientFdGet.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnClientPacketDscpSet.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnClientPacketDscpSet.en.po index 9b9bcc62201..022856be3d7 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnClientPacketDscpSet.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnClientPacketDscpSet.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnClientPacketMarkSet.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnClientPacketMarkSet.en.po index e63224ed3f0..d4d83b5e699 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnClientPacketMarkSet.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnClientPacketMarkSet.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnClientPacketTosSet.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnClientPacketTosSet.en.po index 9362cc48efb..29925d9fe8c 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnClientPacketTosSet.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnClientPacketTosSet.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnClientReqGet.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnClientReqGet.en.po index 324f13d0d29..89b382e6ebf 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnClientReqGet.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnClientReqGet.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnClientRespGet.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnClientRespGet.en.po index 18da60ccd6c..200cf20209c 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnClientRespGet.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnClientRespGet.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnErrorBodySet.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnErrorBodySet.en.po index a558e512192..863aa156e33 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnErrorBodySet.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnErrorBodySet.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnIncomingAddrGet.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnIncomingAddrGet.en.po index 1c7ba0a9b25..54f651dae7f 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnIncomingAddrGet.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnIncomingAddrGet.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnInfoIntGet.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnInfoIntGet.en.po index 408ed32e26b..554289bbda5 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnInfoIntGet.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnInfoIntGet.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnIntercept.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnIntercept.en.po index 9ed5853f481..560784f3b1d 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnIntercept.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnIntercept.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnIsInternal.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnIsInternal.en.po index 4f5aa078cd4..fa33da6884b 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnIsInternal.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnIsInternal.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnMilestoneGet.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnMilestoneGet.en.po index ecbdfe5c1a3..162fc54df3f 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnMilestoneGet.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnMilestoneGet.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnNextHopAddrGet.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnNextHopAddrGet.en.po index a088669f51e..bc181380886 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnNextHopAddrGet.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnNextHopAddrGet.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnOutgoingAddrGet.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnOutgoingAddrGet.en.po index 6fb3791abf1..23814b8ffca 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnOutgoingAddrGet.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnOutgoingAddrGet.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnParentProxySet.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnParentProxySet.en.po index 7c33927868c..394c897f3ff 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnParentProxySet.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnParentProxySet.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnReenable.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnReenable.en.po index 4b02b02847c..5bc3c2f31f4 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnReenable.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnReenable.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnServerAddrGet.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnServerAddrGet.en.po index 48b3bfe1b0b..cda8bb323e0 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnServerAddrGet.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnServerAddrGet.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnServerAddrSet.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnServerAddrSet.en.po index a414840be8f..c14330ad47a 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnServerAddrSet.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnServerAddrSet.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnServerFdGet.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnServerFdGet.en.po index dccc704cff0..031bac5d8e1 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnServerFdGet.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnServerFdGet.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnServerIntercept.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnServerIntercept.en.po index ab74435d3a2..aa1e8788658 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnServerIntercept.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnServerIntercept.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnServerPacketDscpSet.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnServerPacketDscpSet.en.po index 4ca59ec09c8..5ed0331fb2a 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnServerPacketDscpSet.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnServerPacketDscpSet.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnServerPacketMarkSet.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnServerPacketMarkSet.en.po index 7893dcf6502..c07c7caf058 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnServerPacketMarkSet.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnServerPacketMarkSet.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnServerPacketTosSet.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnServerPacketTosSet.en.po index 8ee389dc9c7..9546528d691 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnServerPacketTosSet.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnServerPacketTosSet.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnServerReqGet.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnServerReqGet.en.po index 59cba69f1f3..dba9ef156c0 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnServerReqGet.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnServerReqGet.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnServerRespGet.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnServerRespGet.en.po index 996aff17ddd..d1a9029232c 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnServerRespGet.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnServerRespGet.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnSsnGet.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnSsnGet.en.po index b46c95dc130..fb19f76c31e 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnSsnGet.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnSsnGet.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnTransformRespGet.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnTransformRespGet.en.po index c1fa76cef9a..1e2423afcf3 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnTransformRespGet.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnTransformRespGet.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnTransformedRespCache.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnTransformedRespCache.en.po index bbda7be18b6..be17d5da91e 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnTransformedRespCache.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnTransformedRespCache.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnUntransformedRespCache.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnUntransformedRespCache.en.po index 0f0da58bb1f..9efa034ce76 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnUntransformedRespCache.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpTxnUntransformedRespCache.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSIOBufferBlockReadStart.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSIOBufferBlockReadStart.en.po index c45fb4ce668..a473e8bfa71 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSIOBufferBlockReadStart.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSIOBufferBlockReadStart.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSIOBufferCopy.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSIOBufferCopy.en.po index facfded5b92..47e27dd3a9e 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSIOBufferCopy.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSIOBufferCopy.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSIOBufferCreate.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSIOBufferCreate.en.po index 46b957b9e95..a40b1d38040 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSIOBufferCreate.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSIOBufferCreate.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMBufferCreate.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMBufferCreate.en.po index 53ddabcb43a..7d64116d4e1 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMBufferCreate.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMBufferCreate.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMgmtCounterGet.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMgmtCounterGet.en.po index 9ed77495b84..213dbc79024 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMgmtCounterGet.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMgmtCounterGet.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMgmtFloatGet.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMgmtFloatGet.en.po index 36081786af7..402d466201d 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMgmtFloatGet.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMgmtFloatGet.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMgmtIntGet.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMgmtIntGet.en.po index 4aea5a51386..05a5d1b344a 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMgmtIntGet.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMgmtIntGet.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMgmtStringGet.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMgmtStringGet.en.po index 2d8283e070e..bf7a54abf60 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMgmtStringGet.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMgmtStringGet.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMgmtUpdateRegister.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMgmtUpdateRegister.en.po index 2f9b1db44ee..e6dbb47a86e 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMgmtUpdateRegister.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMgmtUpdateRegister.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrClone.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrClone.en.po index b0ca55e8f1c..aa523453f4e 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrClone.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrClone.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrCopy.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrCopy.en.po index 264b1c2ef48..3f13bcffd54 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrCopy.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrCopy.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrCreate.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrCreate.en.po index 701a28552c6..809851fb4b5 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrCreate.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrCreate.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrDestroy.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrDestroy.en.po index e968302b303..c92bd1f53cb 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrDestroy.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrDestroy.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldAppend.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldAppend.en.po index 4a290cbfde7..c0198a11c32 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldAppend.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldAppend.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldClone.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldClone.en.po index 3e0f6296830..8782f5df73b 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldClone.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldClone.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldCopy.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldCopy.en.po index 67c688da239..b549a3f4cf3 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldCopy.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldCopy.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldCopyValues.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldCopyValues.en.po index 9fa22ecf7d8..5c2d6dff50e 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldCopyValues.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldCopyValues.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldCreate.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldCreate.en.po index e8dc678c0aa..02daf7b85d5 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldCreate.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldCreate.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldDestroy.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldDestroy.en.po index ca754cca6d4..0b1507a8419 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldDestroy.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldDestroy.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldFind.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldFind.en.po index d1533279aa8..b7cd17f7055 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldFind.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldFind.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldGet.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldGet.en.po index d19a37fb28d..dd229aec9b0 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldGet.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldGet.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldLengthGet.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldLengthGet.en.po index 09d23117e4a..e1b21c26e0f 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldLengthGet.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldLengthGet.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldNameGet.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldNameGet.en.po index 35ad3216d1d..03bf2308415 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldNameGet.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldNameGet.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldNameSet.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldNameSet.en.po index 9ffff34b4d2..4377d251ced 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldNameSet.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldNameSet.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldNext.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldNext.en.po index 5664e84b6c6..558a049a439 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldNext.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldNext.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldNextDup.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldNextDup.en.po index 4d79471123d..1ada0cf1ff0 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldNextDup.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldNextDup.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldRemove.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldRemove.en.po index 907edb0c5d1..82aed0aa798 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldRemove.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldRemove.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldValueAppend.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldValueAppend.en.po index d578a04355e..bb61ea2f1ce 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldValueAppend.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldValueAppend.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldValueDateInsert.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldValueDateInsert.en.po index 19bd71ec3ea..8df96b02c60 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldValueDateInsert.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldValueDateInsert.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldValueDateSet.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldValueDateSet.en.po index c52895de53c..d5168474f62 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldValueDateSet.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldValueDateSet.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldValueIntSet.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldValueIntSet.en.po index 880bbd0a168..b89f05b1e79 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldValueIntSet.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldValueIntSet.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldValueStringInsert.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldValueStringInsert.en.po index 22cc1836ccc..90693fd77f5 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldValueStringInsert.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldValueStringInsert.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldValueStringSet.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldValueStringSet.en.po index 10e987915fd..603088b6cc0 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldValueStringSet.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldValueStringSet.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldValueUintInsert.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldValueUintInsert.en.po index 2cb98952505..f4fcd8e9aa1 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldValueUintInsert.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldValueUintInsert.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldValueUintSet.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldValueUintSet.en.po index 9f12f14b16e..877fd6f40d6 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldValueUintSet.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldValueUintSet.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldValuesClear.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldValuesClear.en.po index 40382e70e65..54ed4ddb809 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldValuesClear.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldValuesClear.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldValuesCount.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldValuesCount.en.po index e390d4e2471..cb5513e6b71 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldValuesCount.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldValuesCount.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldsClear.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldsClear.en.po index cbd49890b68..71973810788 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldsClear.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldsClear.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldsCount.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldsCount.en.po index 60264b26624..b611f86780a 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldsCount.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrFieldsCount.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrLengthGet.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrLengthGet.en.po index 2a0bc003ccb..b6721a940c0 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrLengthGet.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrLengthGet.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrParse.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrParse.en.po index e4ea44a47db..98ed132fafb 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrParse.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrParse.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrPrint.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrPrint.en.po index b3219eb820b..de3397f56c4 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrPrint.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeHdrPrint.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeParserClear.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeParserClear.en.po index 4d85af6e934..86eeeaca0de 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeParserClear.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeParserClear.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeParserCreate.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeParserCreate.en.po index 1b7ec03d56f..346898f1dfe 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeParserCreate.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeParserCreate.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeParserDestroy.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeParserDestroy.en.po index f304f552162..522d683cfbc 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeParserDestroy.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMimeParserDestroy.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMutexCreate.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMutexCreate.en.po index e9e447c29a0..be93a2ac6f7 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMutexCreate.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMutexCreate.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMutexDestroy.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMutexDestroy.en.po index cbbdfcefcb4..138c1834dfc 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMutexDestroy.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMutexDestroy.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMutexLock.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMutexLock.en.po index 9b0f45be2f9..1854e16b486 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMutexLock.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMutexLock.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMutexLockTry.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMutexLockTry.en.po index 153fb0ecbcf..145bb30e221 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMutexLockTry.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMutexLockTry.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMutexUnlock.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMutexUnlock.en.po index 4ed40b45892..163690e6a62 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMutexUnlock.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSMutexUnlock.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSNetAccept.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSNetAccept.en.po index 659236f87e9..06c8c920484 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSNetAccept.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSNetAccept.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSNetAcceptNamedProtocol.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSNetAcceptNamedProtocol.en.po index 465db35570a..2d360ab8e5e 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSNetAcceptNamedProtocol.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSNetAcceptNamedProtocol.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSNetConnect.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSNetConnect.en.po index 00fa4fde68e..aeaf0944e9c 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSNetConnect.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSNetConnect.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSTextLogObjectCreate.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSTextLogObjectCreate.en.po index 9e68077b1a4..549163b79bd 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSTextLogObjectCreate.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSTextLogObjectCreate.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSThreadCreate.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSThreadCreate.en.po index 95c8fac6a64..87f0ec5c1c4 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSThreadCreate.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSThreadCreate.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSThreadDestroy.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSThreadDestroy.en.po index acb0f844946..1e2a3ea2347 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSThreadDestroy.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSThreadDestroy.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSThreadInit.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSThreadInit.en.po index 4d5fdb0ef23..64986a4d4aa 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSThreadInit.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSThreadInit.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSThreadSelf.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSThreadSelf.en.po index 909ab34a0c4..d9ce3fa435a 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSThreadSelf.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSThreadSelf.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSTransformCreate.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSTransformCreate.en.po index d1180cb7f8a..dcc72fbf831 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSTransformCreate.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSTransformCreate.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSTransformOutputVConnGet.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSTransformOutputVConnGet.en.po index dab9314e330..4f95e5f8b59 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSTransformOutputVConnGet.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSTransformOutputVConnGet.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSTypes.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSTypes.en.po index 5e0d01e034f..b178f485e0c 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSTypes.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSTypes.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSUrlCreate.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSUrlCreate.en.po index 0b92a480ab3..c7b0459c322 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSUrlCreate.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSUrlCreate.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSUrlFtpTypeGet.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSUrlFtpTypeGet.en.po index 94950ab3900..fd0eb10791d 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSUrlFtpTypeGet.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSUrlFtpTypeGet.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSUrlFtpTypeSet.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSUrlFtpTypeSet.en.po index bbf9a4da6b4..d3dc4710da5 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSUrlFtpTypeSet.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSUrlFtpTypeSet.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSUrlHostGet.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSUrlHostGet.en.po index a40ab5522be..f0b30ffe67e 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSUrlHostGet.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSUrlHostGet.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSUrlHostSet.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSUrlHostSet.en.po index cea8d0ea864..81bfd6be1c6 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSUrlHostSet.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSUrlHostSet.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSUrlPercentEncode.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSUrlPercentEncode.en.po index c1d14de78e9..c69a79ba069 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSUrlPercentEncode.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSUrlPercentEncode.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSUrlStringGet.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSUrlStringGet.en.po index fe7f7219878..465bde2ab19 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSUrlStringGet.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSUrlStringGet.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVConnAbort.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVConnAbort.en.po index 1c2f614f314..da1cb51f0e3 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVConnAbort.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVConnAbort.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVConnCacheObjectSizeGet.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVConnCacheObjectSizeGet.en.po index eac92163cbe..d2a5f610607 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVConnCacheObjectSizeGet.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVConnCacheObjectSizeGet.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVConnClose.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVConnClose.en.po index c4254188f66..77b78ac4520 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVConnClose.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVConnClose.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVConnClosedGet.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVConnClosedGet.en.po index bbbf4a842a5..233b739a48d 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVConnClosedGet.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVConnClosedGet.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVConnFdCreate.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVConnFdCreate.en.po index 01cf14618c6..29acfa16b47 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVConnFdCreate.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVConnFdCreate.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVConnIsSsl.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVConnIsSsl.en.po index f91c31c7694..150264e3bdd 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVConnIsSsl.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVConnIsSsl.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVConnRead.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVConnRead.en.po index bd1fc967676..4a36fc97fcd 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVConnRead.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVConnRead.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVConnReadVIOGet.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVConnReadVIOGet.en.po index 20ea65afb50..32a02bb70b2 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVConnReadVIOGet.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVConnReadVIOGet.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVConnReenable.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVConnReenable.en.po index c002158cf2d..6e9c9351d06 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVConnReenable.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVConnReenable.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVConnShutdown.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVConnShutdown.en.po index b87fc4915ca..d8be0c8405c 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVConnShutdown.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVConnShutdown.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVConnSslConnectionGet.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVConnSslConnectionGet.en.po index 929d058ca92..1e2ac0f60a8 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVConnSslConnectionGet.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVConnSslConnectionGet.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVConnTunnel.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVConnTunnel.en.po index 6304684a202..68307704c40 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVConnTunnel.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVConnTunnel.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVConnWrite.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVConnWrite.en.po index de9281ded29..cff36010ad8 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVConnWrite.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVConnWrite.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVConnWriteVIOGet.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVConnWriteVIOGet.en.po index 4b07b214005..050e028564f 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVConnWriteVIOGet.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVConnWriteVIOGet.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVIOBufferGet.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVIOBufferGet.en.po index 6cdcca43e77..fe278eac324 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVIOBufferGet.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVIOBufferGet.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVIOContGet.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVIOContGet.en.po index 3f068224b9b..ceddc36b193 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVIOContGet.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVIOContGet.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVIOMutexGet.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVIOMutexGet.en.po index 9f465c82768..d80ca2e497a 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVIOMutexGet.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVIOMutexGet.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVIONBytesGet.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVIONBytesGet.en.po index 094954074a4..86563695ac0 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVIONBytesGet.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVIONBytesGet.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVIONBytesSet.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVIONBytesSet.en.po index b177fc4b70b..d11a794ab47 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVIONBytesSet.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVIONBytesSet.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVIONDoneGet.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVIONDoneGet.en.po index 1a4d431e103..acc886e7efa 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVIONDoneGet.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVIONDoneGet.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVIONDoneSet.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVIONDoneSet.en.po index 81e57516584..f2f9e09f492 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVIONDoneSet.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVIONDoneSet.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVIONTodoGet.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVIONTodoGet.en.po index 8b1ad29a9ea..bf09ac13e0d 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVIONTodoGet.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVIONTodoGet.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVIOReaderGet.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVIOReaderGet.en.po index 822e3a3fd19..4bf20af7edd 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVIOReaderGet.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVIOReaderGet.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVIOReenable.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVIOReenable.en.po index 9055af66aa1..b770bf0f858 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVIOReenable.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVIOReenable.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVIOVConnGet.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVIOVConnGet.en.po index 167b93c8be4..4dc76a9189c 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVIOVConnGet.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSVIOVConnGet.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSfclose.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSfclose.en.po index 7238ddcce93..28b796d2e50 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSfclose.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSfclose.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSfflush.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSfflush.en.po index b1b4421a65d..7c7c5f0c0cc 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSfflush.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSfflush.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSfgets.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSfgets.en.po index aa63d2e2a40..0678fb183ed 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSfgets.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSfgets.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSfopen.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSfopen.en.po index ee7557b01f1..7e6addbf3bb 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSfopen.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSfopen.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSfread.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSfread.en.po index 317c448c049..ab88ed1dc6e 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSfread.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSfread.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSfwrite.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSfwrite.en.po index ec5884e043e..4d8b048bdb4 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSfwrite.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSfwrite.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSCacheDataType.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSCacheDataType.en.po index 12a6b400c0f..ddfd9d60fb0 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSCacheDataType.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSCacheDataType.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSCacheError.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSCacheError.en.po index 22f1086adc9..9426813bb9e 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSCacheError.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSCacheError.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSCacheLookupResult.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSCacheLookupResult.en.po index 709e91f3931..d60ee32340a 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSCacheLookupResult.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSCacheLookupResult.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSCacheScanResult.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSCacheScanResult.en.po index f0fe83cab7f..c35316efaa1 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSCacheScanResult.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSCacheScanResult.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSEvent.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSEvent.en.po index 45ae9667fff..bd06fad7205 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSEvent.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSEvent.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSFetchWakeUpOptions.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSFetchWakeUpOptions.en.po index 6a6845d9f69..2ede3ea4d18 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSFetchWakeUpOptions.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSFetchWakeUpOptions.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSHttpHookID.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSHttpHookID.en.po index 24cdad96eb2..f1ddb8e72dc 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSHttpHookID.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSHttpHookID.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSHttpStatus.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSHttpStatus.en.po index e9bbb77261a..487d0c1e237 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSHttpStatus.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSHttpStatus.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSHttpType.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSHttpType.en.po index 37232140986..5d2cc4dc6dc 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSHttpType.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSHttpType.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSIOBuffersSizeIndex.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSIOBuffersSizeIndex.en.po index de6366ea265..0d0176dee21 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSIOBuffersSizeIndex.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSIOBuffersSizeIndex.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSLifecycleHookID.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSLifecycleHookID.en.po index b61b6c23d42..2e7e011ffc9 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSLifecycleHookID.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSLifecycleHookID.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSLookingUpType.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSLookingUpType.en.po index 8b21d732676..0a0ca36af14 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSLookingUpType.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSLookingUpType.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSMilestonesType.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSMilestonesType.en.po index a7f157af159..511f51e0356 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSMilestonesType.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSMilestonesType.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSOverridableConfigKey.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSOverridableConfigKey.en.po index 2dfdc7d1c34..29033da0c1b 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSOverridableConfigKey.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSOverridableConfigKey.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSParseResult.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSParseResult.en.po index 3a6ce4bf797..f02c0911fef 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSParseResult.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSParseResult.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSRecordAccessType.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSRecordAccessType.en.po index 7d74efeff81..ff3bd7d50a0 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSRecordAccessType.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSRecordAccessType.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSRecordCheckType.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSRecordCheckType.en.po index 3176194b3b1..88d87e2dff2 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSRecordCheckType.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSRecordCheckType.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSRecordDataType.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSRecordDataType.en.po index 3114d74f88f..999e22e1f44 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSRecordDataType.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSRecordDataType.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSRecordModeType.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSRecordModeType.en.po index 4e524de3df9..089ce2b5326 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSRecordModeType.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSRecordModeType.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSRecordPersistType.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSRecordPersistType.en.po index 39c68c1eb1f..17d441b9ff6 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSRecordPersistType.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSRecordPersistType.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSRecordType.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSRecordType.en.po index e75e01ae60d..519c25a8290 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSRecordType.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSRecordType.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSRecordUpdateType.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSRecordUpdateType.en.po index 7e133c66806..def18f162d7 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSRecordUpdateType.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSRecordUpdateType.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSReturnCode.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSReturnCode.en.po index e5fb4a23ca1..246775c46e5 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSReturnCode.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSReturnCode.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSSDKVersion.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSSDKVersion.en.po index 8c3387863b0..70b1fa1bebf 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSSDKVersion.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSSDKVersion.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSServerSessionSharingMatchType.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSServerSessionSharingMatchType.en.po index 68b2eb5bd9f..06da651d222 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSServerSessionSharingMatchType.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSServerSessionSharingMatchType.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSServerSessionSharingPoolType.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSServerSessionSharingPoolType.en.po index 81baed7c984..8fc8c227132 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSServerSessionSharingPoolType.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSServerSessionSharingPoolType.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSServerState.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSServerState.en.po index 866e80a2755..66a04503aa8 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSServerState.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSServerState.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSSslVConnOp.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSSslVConnOp.en.po index bb99e5d28b0..3f850f17c11 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSSslVConnOp.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSSslVConnOp.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSThreadPool.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSThreadPool.en.po index 9535782f2a4..fe13d2455a5 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSThreadPool.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSThreadPool.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSVConnCloseFlags.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSVConnCloseFlags.en.po index c7dcc7ad455..ef28d6361a9 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSVConnCloseFlags.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/types/TSVConnCloseFlags.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/architecture/index.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/architecture/index.en.po index bb9118fe45e..31725636b4d 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/architecture/index.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/architecture/index.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/architecture/ram-cache.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/architecture/ram-cache.en.po index 82a18e0f1f1..c9fc94d2a98 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/architecture/ram-cache.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/architecture/ram-cache.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/documentation/structure.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/documentation/structure.en.po index 26a72e578f2..a8ae68bc858 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/documentation/structure.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/documentation/structure.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/host-resolution-proposal.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/host-resolution-proposal.en.po index 9d41993ec25..1beb235f46d 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/host-resolution-proposal.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/host-resolution-proposal.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/index.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/index.en.po index 904e8e50153..ee15a182a60 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/index.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/index.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/plugins/continuations/activating-continuations.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/plugins/continuations/activating-continuations.en.po index db401a5e678..5e8bbaabdd9 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/plugins/continuations/activating-continuations.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/plugins/continuations/activating-continuations.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/plugins/continuations/index.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/plugins/continuations/index.en.po index 03bb83b9b69..b2595d96af8 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/plugins/continuations/index.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/plugins/continuations/index.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/plugins/example-plugins/blacklist/index.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/plugins/example-plugins/blacklist/index.en.po index 211ea63547a..b614f19d572 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/plugins/example-plugins/blacklist/index.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/plugins/example-plugins/blacklist/index.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/plugins/example-plugins/index.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/plugins/example-plugins/index.en.po index 045c695bff7..3a9ea8559eb 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/plugins/example-plugins/index.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/plugins/example-plugins/index.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/plugins/example-plugins/query-remap/index.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/plugins/example-plugins/query-remap/index.en.po index 52efbb28012..2a8da8e0553 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/plugins/example-plugins/query-remap/index.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/plugins/example-plugins/query-remap/index.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/plugins/getting-started/a-simple-plugin.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/plugins/getting-started/a-simple-plugin.en.po index 561d4ead1e4..12156f19633 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/plugins/getting-started/a-simple-plugin.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/plugins/getting-started/a-simple-plugin.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/plugins/getting-started/index.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/plugins/getting-started/index.en.po index 4698548af7e..c89a6c4051e 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/plugins/getting-started/index.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/plugins/getting-started/index.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/plugins/getting-started/naming-conventions.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/plugins/getting-started/naming-conventions.en.po index b615a8086a8..412bf43cf76 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/plugins/getting-started/naming-conventions.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/plugins/getting-started/naming-conventions.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/plugins/hooks-and-transactions/index.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/plugins/hooks-and-transactions/index.en.po index bf7bae4deb2..9d19d647d6d 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/plugins/hooks-and-transactions/index.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/plugins/hooks-and-transactions/index.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/plugins/http-transformations/index.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/plugins/http-transformations/index.en.po index 972a0b5e444..c8bc94fade5 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/plugins/http-transformations/index.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/plugins/http-transformations/index.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/plugins/io/cache-api.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/plugins/io/cache-api.en.po index 25e4dc67c63..555960c2bc5 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/plugins/io/cache-api.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/plugins/io/cache-api.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/plugins/io/io-buffers.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/plugins/io/io-buffers.en.po index f0fdc426cce..8bdf0260136 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/plugins/io/io-buffers.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/plugins/io/io-buffers.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/plugins/new-protocol-plugins.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/plugins/new-protocol-plugins.en.po index 7d685845fd0..4f6fb77df81 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/plugins/new-protocol-plugins.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/plugins/new-protocol-plugins.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/plugins/plugin-interfaces.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/plugins/plugin-interfaces.en.po index b2cb92a5127..7d3bfa686b6 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/plugins/plugin-interfaces.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/plugins/plugin-interfaces.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/plugins/plugin-management/index.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/plugins/plugin-management/index.en.po index 0a01c411adc..847744c496c 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/plugins/plugin-management/index.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/plugins/plugin-management/index.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/troubleshooting-tips.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/troubleshooting-tips.en.po index 32dead14747..41d0ccbb9b9 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/troubleshooting-tips.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/troubleshooting-tips.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/troubleshooting-tips/unable-to-load-plugins.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/troubleshooting-tips/unable-to-load-plugins.en.po index 4db6599dc5a..0d8b906e4ad 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/troubleshooting-tips/unable-to-load-plugins.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/troubleshooting-tips/unable-to-load-plugins.en.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/locale/ja/LC_MESSAGES/getting-started.en.po b/doc/locale/ja/LC_MESSAGES/getting-started.en.po index 191a40fffbb..eb659724e91 100644 --- a/doc/locale/ja/LC_MESSAGES/getting-started.en.po +++ b/doc/locale/ja/LC_MESSAGES/getting-started.en.po @@ -236,10 +236,6 @@ msgstr "" msgid "openssl" msgstr "" -#: ../../getting-started.en.rst:157 -msgid "tcl" -msgstr "" - #: ../../getting-started.en.rst:158 msgid "expat" msgstr "" diff --git a/doc/locale/ja/LC_MESSAGES/getting-started/index.en.po b/doc/locale/ja/LC_MESSAGES/getting-started/index.en.po index 9b3e4d64629..329cce2e91e 100644 --- a/doc/locale/ja/LC_MESSAGES/getting-started/index.en.po +++ b/doc/locale/ja/LC_MESSAGES/getting-started/index.en.po @@ -238,10 +238,6 @@ msgstr "" msgid "openssl" msgstr "openssl" -#: ../../../getting-started/index.en.rst:157 -msgid "tcl" -msgstr "tcl" - #: ../../../getting-started/index.en.rst:158 msgid "expat" msgstr "expat" diff --git a/doc/locale/ja/LC_MESSAGES/index.po b/doc/locale/ja/LC_MESSAGES/index.po index bde91362236..4dc43f6a175 100644 --- a/doc/locale/ja/LC_MESSAGES/index.po +++ b/doc/locale/ja/LC_MESSAGES/index.po @@ -23,7 +23,7 @@ msgstr "" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" +"Language: ja_JP\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" diff --git a/doc/manpages.py b/doc/manpages.py index 8a4850025d9..97f90e345ea 100644 --- a/doc/manpages.py +++ b/doc/manpages.py @@ -33,6 +33,9 @@ ('appendices/command-line/traffic_top.en', 'traffic_top', u'Display Traffic Server statistics', None, '1'), ('appendices/command-line/tsxs.en', 'tsxs', u'Traffic Server plugin tool', None, '1'), ('appendices/command-line/traffic_via.en', 'traffic_via', u'Traffic Server Via header decoder', None, '1'), + ('appendices/command-line/traffic_layout.en', 'traffic_layout', u'Traffic Server sandbox management tool', None, '1'), + ('appendices/command-line/traffic_cache_tool.en', 'traffic_cache_tool', u'Traffic Server cache management tool', None, '1'), + ('appendices/command-line/traffic_wccp.en', 'traffic_wccp', u'Traffic Server WCCP client', None, '1'), ('admin-guide/files/cache.config.en', 'cache.config', u'Traffic Server cache configuration file', None, '5'), ('admin-guide/files/hosting.config.en', 'hosting.config', u'Traffic Server domain hosting configuration file', None, '5'), diff --git a/doc/static/images/admin/proxy-protocol.png b/doc/static/images/admin/proxy-protocol.png new file mode 100644 index 00000000000..e69de29bb2d diff --git a/doc/static/override.css b/doc/static/override.css index 57cfd51d9a0..d29fb0d16ce 100644 --- a/doc/static/override.css +++ b/doc/static/override.css @@ -19,6 +19,11 @@ dd { overflow: visible; } +div.figure.align-center{ + margin: initial; + max-width: 800px; +} + /* pre inside paragraphs gets weird spacing */ span.pre { line-height: initial; diff --git a/doc/uml/Makefile.am b/doc/uml/Makefile.am index 956068f3f90..2a9fa761056 100644 --- a/doc/uml/Makefile.am +++ b/doc/uml/Makefile.am @@ -18,6 +18,7 @@ if BUILD_DOCS images := $(patsubst %.uml,images/%.svg,$(wildcard *.uml)) +images += $(patsubst %.plantuml,images/%.svg,$(wildcard *.plantuml)) PLANTUML_JAR := $(shell ../ext/plantuml_fetch.sh | tail -1) all-am: jar-check $(images) @@ -41,6 +42,9 @@ man: all-am images/%.svg : %.uml $(JAVA) -jar $(PLANTUML_JAR) -o images -tsvg $< +images/%.svg : %.plantuml + $(JAVA) -jar $(PLANTUML_JAR) -o images -tsvg $< + clean-local: rm -f images/*.svg diff --git a/doc/uml/RPC-sequence-diagram.uml b/doc/uml/RPC-sequence-diagram.uml index 17d543dbe9a..5a5c0d3d33a 100644 --- a/doc/uml/RPC-sequence-diagram.uml +++ b/doc/uml/RPC-sequence-diagram.uml @@ -38,8 +38,8 @@ traffic_server -[#blue]-> local_rpc : //unserialize message// traffic_server -> plugin : invoke hook and attach ""TSPluginMsg"" plugin -> plugin : hooks ""TS_LIFECYCLE_MSG_HOOK""\nand reads ""TSPluginMsg"" traffic_manager -[#green]-> remote_rpc : TS_ERR_OK -note right : traffic_manager has no idea if\nplugin actually recieved message +note right : traffic_manager has no idea if\nplugin actually received message remote_rpc -> traffic_ctl : parse response -@enduml \ No newline at end of file +@enduml diff --git a/doc/uml/RPC-states.uml b/doc/uml/RPC-states.uml index ad73b58935c..c95ade5c634 100644 --- a/doc/uml/RPC-states.uml +++ b/doc/uml/RPC-states.uml @@ -20,7 +20,7 @@ state "LocalManager\nruns in traffic_manager main event loop" as traffic_manager state "poll socket" as tm_poll_sock : ""mgmt_select"" state "new connection" as tm_new_process : register new client socket fd\nto start getting msgs from\nclient state "new message" as tm_new_message : new message from process - state "handle msg" as tm_handle_msg : ""handleMgmtMsgFromProcesses"" will\noften ""signalAlarm"" with recieved\nmsg_id + state "handle msg" as tm_handle_msg : ""handleMgmtMsgFromProcesses"" will\noften ""signalAlarm"" with received\nmsg_id tm_poll_sock -d-> tm_new_process : new process trying to connect tm_poll_sock -d-> tm_new_message : msg from ""traffic_server"" diff --git a/doc/uml/hdr-heap-class.plantuml b/doc/uml/hdr-heap-class.plantuml new file mode 100644 index 00000000000..28212b6cc16 --- /dev/null +++ b/doc/uml/hdr-heap-class.plantuml @@ -0,0 +1,52 @@ +' Licensed under the Apache License, Version 2.0 (the "License"); +' you may not use this file except in compliance with the License. +' You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 +' Unless required by applicable law or agreed to in writing, software distributed under the License is distributed +' on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +' See the License for the specific language governing permissions and limitations under the License. + +@startuml + +hide empty members + +class HdrStrHeap << (V, green) >> { + Storage for strings. + == + Extended Storage +} + +class StrHeapDesc { + m_ref_count_ptr + m_heap_start + m_heap_len + m_locked +} + +HdrStrHeap --|> RefCountObj +StrHeapDesc --o RefCountObj : m_ref_count_ptr + +note left of HdrStrHeap : Storage for strings in\nheap objects. + +class HdrHeap << (V, green) >> { + Storage for objects. + == + Extended Storage +} + +class HdrHeapObjImpl { + m_type + m_length + m_obj_flags +} + +HdrHeap --* "3" StrHeapDesc : m_ronly_heap +HdrHeap --o HdrStrHeap : m_read_write_heap +HdrHeap --o HdrHeap : next + +note left of HdrHeap : Heap storage for objects + +abstract class HdrHeapObjImpl + +HdrHeap --o "*" HdrHeapObjImpl : [heap storage] + +@enduml diff --git a/doc/uml/hdr-heap-str-alloc.plantuml b/doc/uml/hdr-heap-str-alloc.plantuml new file mode 100644 index 00000000000..7f54f297eef --- /dev/null +++ b/doc/uml/hdr-heap-str-alloc.plantuml @@ -0,0 +1,29 @@ +' Licensed under the Apache License, Version 2.0 (the "License"); +' you may not use this file except in compliance with the License. +' You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 +' Unless required by applicable law or agreed to in writing, software distributed under the License is distributed +' on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +' See the License for the specific language governing permissions and limitations under the License. + +@startuml + +start + +if (dead space) then (too much) + :coalesce; +endif + +repeat +if (writeable heap?) then (false) + :create writeable heap; +endif +if (can allocate) then (true) + stop +endif +if (demote_heap) then (no open slots) + :coalesce; +endif +repeat while(loop) +detach + +@enduml diff --git a/example/append_transform/append_transform.c b/example/append_transform/append_transform.c index 873d1c530fe..aca2fdbe34b 100644 --- a/example/append_transform/append_transform.c +++ b/example/append_transform/append_transform.c @@ -92,7 +92,6 @@ handle_transform(TSCont contp) TSVIO write_vio; MyData *data; int64_t towrite; - int64_t avail; /* Get the output connection where we'll write data to. */ output_conn = TSTransformOutputVConnGet(contp); @@ -146,7 +145,7 @@ handle_transform(TSCont contp) if (towrite > 0) { /* The amount of data left to read needs to be truncated by the amount of data actually in the read buffer. */ - avail = TSIOBufferReaderAvail(TSVIOReaderGet(write_vio)); + int64_t avail = TSIOBufferReaderAvail(TSVIOReaderGet(write_vio)); if (towrite > avail) { towrite = avail; } @@ -247,9 +246,6 @@ transformable(TSHttpTxn txnp) { TSMBuffer bufp; TSMLoc hdr_loc; - TSMLoc field_loc; - TSHttpStatus resp_status; - const char *value; int val_length; if (TS_SUCCESS == TSHttpTxnServerRespGet(txnp, &bufp, &hdr_loc)) { @@ -257,16 +253,16 @@ transformable(TSHttpTxn txnp) * We are only interested in "200 OK" responses. */ - if (TS_HTTP_STATUS_OK == (resp_status = TSHttpHdrStatusGet(bufp, hdr_loc))) { + if (TS_HTTP_STATUS_OK == TSHttpHdrStatusGet(bufp, hdr_loc)) { /* We only want to do the transformation on documents that have a content type of "text/html". */ - field_loc = TSMimeHdrFieldFind(bufp, hdr_loc, "Content-Type", 12); + TSMLoc field_loc = TSMimeHdrFieldFind(bufp, hdr_loc, "Content-Type", 12); if (!field_loc) { ASSERT_SUCCESS(TSHandleMLocRelease(bufp, TS_NULL_MLOC, hdr_loc)); return 0; } - value = TSMimeHdrFieldValueStringGet(bufp, hdr_loc, field_loc, -1, &val_length); + const char *value = TSMimeHdrFieldValueStringGet(bufp, hdr_loc, field_loc, -1, &val_length); if (value && (strncasecmp(value, "text/html", sizeof("text/html") - 1) == 0)) { ASSERT_SUCCESS(TSHandleMLocRelease(bufp, hdr_loc, field_loc)); ASSERT_SUCCESS(TSHandleMLocRelease(bufp, TS_NULL_MLOC, hdr_loc)); @@ -315,10 +311,7 @@ static int load(const char *filename) { TSFile fp; - TSIOBufferBlock blk; - char *p; int64_t avail; - int err; fp = TSfopen(filename, "r"); if (!fp) { @@ -329,10 +322,10 @@ load(const char *filename) append_buffer_reader = TSIOBufferReaderAlloc(append_buffer); for (;;) { - blk = TSIOBufferStart(append_buffer); - p = TSIOBufferBlockWriteStart(blk, &avail); + TSIOBufferBlock blk = TSIOBufferStart(append_buffer); + char *p = TSIOBufferBlockWriteStart(blk, &avail); - err = TSfread(fp, p, avail); + int err = TSfread(fp, p, avail); if (err > 0) { TSIOBufferProduce(append_buffer, err); } else { diff --git a/example/blacklist_0/blacklist_0.c b/example/blacklist_0/blacklist_0.c index aa86c255142..5ca5179e1fb 100644 --- a/example/blacklist_0/blacklist_0.c +++ b/example/blacklist_0/blacklist_0.c @@ -150,7 +150,6 @@ blacklist_plugin(TSCont contp, TSEvent event, void *edata) void TSPluginInit(int argc, const char *argv[]) { - int i; TSPluginRegistrationInfo info; info.plugin_name = PLUGIN_NAME; @@ -165,7 +164,7 @@ TSPluginInit(int argc, const char *argv[]) if (nsites > 0) { sites = (char **)TSmalloc(sizeof(char *) * nsites); - for (i = 0; i < nsites; i++) { + for (int i = 0; i < nsites; i++) { sites[i] = TSstrdup(argv[i + 1]); } diff --git a/example/blacklist_1/blacklist_1.c b/example/blacklist_1/blacklist_1.c index 34703a2e8ec..7b08186cb7d 100644 --- a/example/blacklist_1/blacklist_1.c +++ b/example/blacklist_1/blacklist_1.c @@ -100,7 +100,7 @@ handle_dns(TSHttpTxn txnp, TSCont contp) TSDebug(PLUGIN_NAME, "Unable to get lock. Will retry after some time"); TSHandleMLocRelease(bufp, hdr_loc, url_loc); TSHandleMLocRelease(bufp, TS_NULL_MLOC, hdr_loc); - TSContSchedule(contp, RETRY_TIME, TS_THREAD_POOL_DEFAULT); + TSContScheduleOnPool(contp, RETRY_TIME, TS_THREAD_POOL_NET); return; } @@ -188,7 +188,7 @@ read_blacklist(TSCont contp) if (file != NULL) { TSfclose(file); } - TSContSchedule(contp, RETRY_TIME, TS_THREAD_POOL_DEFAULT); + TSContScheduleOnPool(contp, RETRY_TIME, TS_THREAD_POOL_NET); return; } diff --git a/example/bnull_transform/bnull_transform.c b/example/bnull_transform/bnull_transform.c index 09af6218704..55dd7fdd3ec 100644 --- a/example/bnull_transform/bnull_transform.c +++ b/example/bnull_transform/bnull_transform.c @@ -82,7 +82,6 @@ handle_buffering(TSCont contp, MyData *data) { TSVIO write_vio; int towrite; - int avail; /* Get the write VIO for the write operation that was performed on ourself. This VIO contains the buffer that we are to read from @@ -118,7 +117,7 @@ handle_buffering(TSCont contp, MyData *data) /* The amount of data left to read needs to be truncated by the amount of data actually in the read buffer. */ - avail = TSIOBufferReaderAvail(TSVIOReaderGet(write_vio)); + int avail = TSIOBufferReaderAvail(TSVIOReaderGet(write_vio)); if (towrite > avail) { towrite = avail; } diff --git a/example/cache_scan/cache_scan.cc b/example/cache_scan/cache_scan.cc index f889fb880a3..9d1d362496f 100644 --- a/example/cache_scan/cache_scan.cc +++ b/example/cache_scan/cache_scan.cc @@ -65,7 +65,6 @@ using cache_scan_state = struct cache_scan_state_t; static int handle_scan(TSCont contp, TSEvent event, void *edata) { - TSCacheHttpInfo cache_infop; cache_scan_state *cstate = (cache_scan_state *)TSContDataGet(contp); if (event == TS_EVENT_CACHE_REMOVE) { @@ -120,19 +119,18 @@ handle_scan(TSCont contp, TSEvent event, void *edata) if (cstate->done) { return TS_CACHE_SCAN_RESULT_DONE; } - cache_infop = (TSCacheHttpInfo)edata; + TSCacheHttpInfo cache_infop = (TSCacheHttpInfo)edata; TSMBuffer req_bufp, resp_bufp; TSMLoc req_hdr_loc, resp_hdr_loc; TSMLoc url_loc; - char *url; int url_len; const char s1[] = "URL: ", s2[] = "\n"; cstate->total_bytes += TSIOBufferWrite(cstate->resp_buffer, s1, sizeof(s1) - 1); TSCacheHttpInfoReqGet(cache_infop, &req_bufp, &req_hdr_loc); if (TS_SUCCESS == TSHttpHdrUrlGet(req_bufp, req_hdr_loc, &url_loc)) { - url = TSUrlStringGet(req_bufp, url_loc, &url_len); + char *url = TSUrlStringGet(req_bufp, url_loc, &url_len); cstate->total_bytes += TSIOBufferWrite(cstate->resp_buffer, url, url_len); cstate->total_bytes += TSIOBufferWrite(cstate->resp_buffer, s2, sizeof(s2) - 1); diff --git a/example/cppapi/async_http_fetch/AsyncHttpFetch.cc b/example/cppapi/async_http_fetch/AsyncHttpFetch.cc index b6622183057..291d650fb0f 100644 --- a/example/cppapi/async_http_fetch/AsyncHttpFetch.cc +++ b/example/cppapi/async_http_fetch/AsyncHttpFetch.cc @@ -42,7 +42,7 @@ GlobalPlugin *plugin; class AsyncHttpFetch2 : public AsyncHttpFetch { public: - AsyncHttpFetch2(const string &request) : AsyncHttpFetch(request){}; + explicit AsyncHttpFetch2(const string &request) : AsyncHttpFetch(request){}; }; class AsyncHttpFetch3 : public AsyncHttpFetch @@ -66,7 +66,9 @@ class DelayedAsyncHttpFetch : public AsyncHttpFetch, public AsyncReceiverdispatch()); + if (!getDispatchController()->dispatch()) { + TS_ERROR(TAG, "Failed to dispatch()"); + } delete this; } bool @@ -88,7 +90,7 @@ class TransactionHookPlugin : public TransactionPlugin, public AsyncReceiver { public: - TransactionHookPlugin(Transaction &transaction) + explicit TransactionHookPlugin(Transaction &transaction) : TransactionPlugin(transaction), transaction_(transaction), num_fetches_pending_(0), post_request_(nullptr) { TS_DEBUG(TAG, "Constructed TransactionHookPlugin, saved a reference to this transaction."); @@ -118,9 +120,14 @@ class TransactionHookPlugin : public TransactionPlugin, // canceling right after starting in this case, but cancel() can be called any time TS_DEBUG(TAG, "Will cancel delayed fetch"); - assert(delayed_provider->isAlive()); + if (!delayed_provider->isAlive()) { + TS_ERROR(TAG, "provider is NOT alive!"); + } + delayed_provider->cancel(); - assert(!delayed_provider->isAlive()); + if (delayed_provider->isAlive()) { + TS_ERROR(TAG, "provider is alive!"); + } } void diff --git a/example/cppapi/async_http_fetch_streaming/AsyncHttpFetchStreaming.cc b/example/cppapi/async_http_fetch_streaming/AsyncHttpFetchStreaming.cc index 1389e9c277c..f953ab08259 100644 --- a/example/cppapi/async_http_fetch_streaming/AsyncHttpFetchStreaming.cc +++ b/example/cppapi/async_http_fetch_streaming/AsyncHttpFetchStreaming.cc @@ -42,11 +42,14 @@ GlobalPlugin *plugin; class Intercept : public InterceptPlugin, public AsyncReceiver { public: - Intercept(Transaction &transaction) - : InterceptPlugin(transaction, InterceptPlugin::SERVER_INTERCEPT), transaction_(transaction), num_fetches_(0) + explicit Intercept(Transaction &transaction) + : InterceptPlugin(transaction, InterceptPlugin::SERVER_INTERCEPT), + transaction_(transaction), + main_url_(transaction.getClientRequest().getUrl().getUrlString()), + num_fetches_(0) { - main_url_ = transaction.getClientRequest().getUrl().getUrlString(); } + void consume(const string &data, InterceptPlugin::RequestDataType type) override; void handleInputComplete() override; void handleAsyncComplete(AsyncHttpFetch &async_http_fetch) override; diff --git a/example/cppapi/boom/boom.cc b/example/cppapi/boom/boom.cc index be58fca7d6a..8db6059625b 100644 --- a/example/cppapi/boom/boom.cc +++ b/example/cppapi/boom/boom.cc @@ -108,7 +108,7 @@ class IsRewritableCode : public std::unary_function std::string current_code_string_; public: - IsRewritableCode(int current_code) : current_code_(current_code) + explicit IsRewritableCode(int current_code) : current_code_(current_code) { std::ostringstream oss; oss << current_code_; @@ -218,12 +218,11 @@ BoomResponseRegistry::populate_error_responses(const std::string &base_directory // Filename (sans the .html suffix) becomes the entry to the // registry lookup table DIR *pDIR = nullptr; - struct dirent *entry; pDIR = opendir(base_error_directory_.c_str()); if (pDIR != nullptr) { while (true) { - entry = readdir(pDIR); + struct dirent *entry = readdir(pDIR); if (entry == nullptr) { break; } @@ -382,7 +381,7 @@ class BoomGlobalPlugin : public atscppapi::GlobalPlugin BoomResponseRegistry *response_registry_; public: - BoomGlobalPlugin(BoomResponseRegistry *response_registry) : response_registry_(response_registry) + explicit BoomGlobalPlugin(BoomResponseRegistry *response_registry) : response_registry_(response_registry) { TS_DEBUG(TAG, "Creating BoomGlobalHook %p", this); registerHook(HOOK_READ_RESPONSE_HEADERS); diff --git a/example/cppapi/custom_error_remap_plugin/CustomErrorRemapPlugin.cc b/example/cppapi/custom_error_remap_plugin/CustomErrorRemapPlugin.cc index f6cb20c7425..727c3cf79db 100644 --- a/example/cppapi/custom_error_remap_plugin/CustomErrorRemapPlugin.cc +++ b/example/cppapi/custom_error_remap_plugin/CustomErrorRemapPlugin.cc @@ -31,7 +31,8 @@ RemapPlugin *plugin; class MyRemapPlugin : public RemapPlugin { public: - MyRemapPlugin(void **instance_handle) : RemapPlugin(instance_handle) {} + explicit MyRemapPlugin(void **instance_handle) : RemapPlugin(instance_handle) {} + Result doRemap(const Url &map_from_url, const Url &map_to_url, Transaction &transaction, bool &redirect) override { diff --git a/example/cppapi/globalhook/GlobalHookPlugin.cc b/example/cppapi/globalhook/GlobalHookPlugin.cc index 9fd245e491f..e8acbe9d660 100644 --- a/example/cppapi/globalhook/GlobalHookPlugin.cc +++ b/example/cppapi/globalhook/GlobalHookPlugin.cc @@ -17,9 +17,9 @@ */ #include + #include "tscpp/api/GlobalPlugin.h" #include "tscpp/api/PluginInit.h" -//#include<../ts/Diags.h> using namespace atscppapi; using namespace std; diff --git a/example/cppapi/gzip_transformation/GzipTransformationPlugin.cc b/example/cppapi/gzip_transformation/GzipTransformationPlugin.cc index a2d7ba0c15c..fe191a57e79 100644 --- a/example/cppapi/gzip_transformation/GzipTransformationPlugin.cc +++ b/example/cppapi/gzip_transformation/GzipTransformationPlugin.cc @@ -89,7 +89,7 @@ class Helpers class SomeTransformationPlugin : public TransformationPlugin { public: - SomeTransformationPlugin(Transaction &transaction) + explicit SomeTransformationPlugin(Transaction &transaction) : TransformationPlugin(transaction, RESPONSE_TRANSFORMATION), transaction_(transaction) { registerHook(HOOK_SEND_RESPONSE_HEADERS); diff --git a/example/cppapi/intercept/intercept.cc b/example/cppapi/intercept/intercept.cc index ec3c747ea64..da615cb077b 100644 --- a/example/cppapi/intercept/intercept.cc +++ b/example/cppapi/intercept/intercept.cc @@ -35,7 +35,7 @@ GlobalPlugin *plugin; class Intercept : public InterceptPlugin { public: - Intercept(Transaction &transaction) : InterceptPlugin(transaction, InterceptPlugin::SERVER_INTERCEPT) {} + explicit Intercept(Transaction &transaction) : InterceptPlugin(transaction, InterceptPlugin::SERVER_INTERCEPT) {} void consume(const string &data, InterceptPlugin::RequestDataType type) override; void handleInputComplete() override; ~Intercept() override { cout << "Shutting down" << endl; } diff --git a/example/cppapi/multiple_transaction_hooks/MultipleTransactionHookPlugins.cc b/example/cppapi/multiple_transaction_hooks/MultipleTransactionHookPlugins.cc index d96712f94f9..fde3ea36657 100644 --- a/example/cppapi/multiple_transaction_hooks/MultipleTransactionHookPlugins.cc +++ b/example/cppapi/multiple_transaction_hooks/MultipleTransactionHookPlugins.cc @@ -31,7 +31,7 @@ GlobalPlugin *plugin; class MultipleTransactionHookPluginsOne : public atscppapi::TransactionPlugin { public: - MultipleTransactionHookPluginsOne(Transaction &transaction) : TransactionPlugin(transaction) + explicit MultipleTransactionHookPluginsOne(Transaction &transaction) : TransactionPlugin(transaction) { TransactionPlugin::registerHook(HOOK_SEND_RESPONSE_HEADERS); std::cout << "Constructed MultipleTransactionHookPluginsOne!" << std::endl; @@ -49,7 +49,7 @@ class MultipleTransactionHookPluginsOne : public atscppapi::TransactionPlugin class MultipleTransactionHookPluginsTwo : public atscppapi::TransactionPlugin { public: - MultipleTransactionHookPluginsTwo(Transaction &transaction) : TransactionPlugin(transaction) + explicit MultipleTransactionHookPluginsTwo(Transaction &transaction) : TransactionPlugin(transaction) { TransactionPlugin::registerHook(HOOK_SEND_REQUEST_HEADERS); TransactionPlugin::registerHook(HOOK_SEND_RESPONSE_HEADERS); diff --git a/example/cppapi/post_buffer/PostBuffer.cc b/example/cppapi/post_buffer/PostBuffer.cc index 1d67f8b1908..a5064c9b064 100644 --- a/example/cppapi/post_buffer/PostBuffer.cc +++ b/example/cppapi/post_buffer/PostBuffer.cc @@ -36,7 +36,7 @@ GlobalPlugin *plugin; class PostBufferTransformationPlugin : public TransformationPlugin { public: - PostBufferTransformationPlugin(Transaction &transaction) + explicit PostBufferTransformationPlugin(Transaction &transaction) : TransformationPlugin(transaction, REQUEST_TRANSFORMATION), transaction_(transaction) { buffer_.reserve(1024); // not required, this is an optimization to start the buffer at a slightly higher value. diff --git a/example/cppapi/remap_plugin/RemapPlugin.cc b/example/cppapi/remap_plugin/RemapPlugin.cc index aed34a91d37..508ce3b0b0e 100644 --- a/example/cppapi/remap_plugin/RemapPlugin.cc +++ b/example/cppapi/remap_plugin/RemapPlugin.cc @@ -36,7 +36,8 @@ RemapPlugin *plugin; class MyRemapPlugin : public RemapPlugin { public: - MyRemapPlugin(void **instance_handle) : RemapPlugin(instance_handle) {} + explicit MyRemapPlugin(void **instance_handle) : RemapPlugin(instance_handle) {} + Result doRemap(const Url &map_from_url, const Url &map_to_url, Transaction &transaction, bool &redirect) override { diff --git a/example/cppapi/transactionhook/TransactionHookPlugin.cc b/example/cppapi/transactionhook/TransactionHookPlugin.cc index d2b09a183f8..8cd8da370df 100644 --- a/example/cppapi/transactionhook/TransactionHookPlugin.cc +++ b/example/cppapi/transactionhook/TransactionHookPlugin.cc @@ -31,12 +31,13 @@ GlobalPlugin *plugin; class TransactionHookPlugin : public atscppapi::TransactionPlugin { public: - TransactionHookPlugin(Transaction &transaction) : TransactionPlugin(transaction) + explicit TransactionHookPlugin(Transaction &transaction) : TransactionPlugin(transaction) { char_ptr_ = new char[100]; TransactionPlugin::registerHook(HOOK_SEND_RESPONSE_HEADERS); std::cout << "Constructed!" << std::endl; } + ~TransactionHookPlugin() override { delete[] char_ptr_; // cleanup diff --git a/example/cppapi/websocket/WSBuffer.cc b/example/cppapi/websocket/WSBuffer.cc index 66e4f8079b0..f6e68999d80 100644 --- a/example/cppapi/websocket/WSBuffer.cc +++ b/example/cppapi/websocket/WSBuffer.cc @@ -56,7 +56,7 @@ static const std::string magic = "258EAFA5-E914-47DA-95CA-C5AB0DC85B11"; -WSBuffer::WSBuffer() : frame_(0) {} +WSBuffer::WSBuffer() {} void WSBuffer::buffer(std::string const &data) diff --git a/example/cppapi/websocket/WSBuffer.h b/example/cppapi/websocket/WSBuffer.h index 4b12f168cde..cf7bdc54b0a 100644 --- a/example/cppapi/websocket/WSBuffer.h +++ b/example/cppapi/websocket/WSBuffer.h @@ -84,6 +84,6 @@ class WSBuffer private: std::string ws_buf_; // incoming data. - int frame_; // frame type of current message + int frame_ = 0; // frame type of current message std::string msg_buf_; // decoded message data }; diff --git a/example/cppapi/websocket/WebSocket.h b/example/cppapi/websocket/WebSocket.h index ad4adccbefd..01528fa531a 100644 --- a/example/cppapi/websocket/WebSocket.h +++ b/example/cppapi/websocket/WebSocket.h @@ -40,7 +40,7 @@ using atscppapi::GlobalPlugin; class WebSocket : public InterceptPlugin { public: - WebSocket(Transaction &transaction); + explicit WebSocket(Transaction &transaction); ~WebSocket() override; void consume(const std::string &data, InterceptPlugin::RequestDataType type) override; diff --git a/example/file_1/file_1.c b/example/file_1/file_1.c index 21ee9400720..d1a4023e6cf 100644 --- a/example/file_1/file_1.c +++ b/example/file_1/file_1.c @@ -41,7 +41,6 @@ void TSPluginInit(int argc, const char *argv[]) { - TSFile filep; char buf[4096]; int i; TSPluginRegistrationInfo info; @@ -55,7 +54,7 @@ TSPluginInit(int argc, const char *argv[]) } for (i = 1; i < argc; i++) { - filep = TSfopen(argv[i], "r"); + TSFile filep = TSfopen(argv[i], "r"); if (!filep) { continue; } diff --git a/example/intercept/intercept.cc b/example/intercept/intercept.cc index 8369667c540..c793e19b96f 100644 --- a/example/intercept/intercept.cc +++ b/example/intercept/intercept.cc @@ -70,11 +70,11 @@ static int InterceptTxnHook(TSCont contp, TSEvent event, void *edata); // a write). We need two of these for each TSVConn; one to push // data into the TSVConn and one to pull data out. struct InterceptIOChannel { - TSVIO vio; - TSIOBuffer iobuf; - TSIOBufferReader reader; + TSVIO vio = nullptr; + TSIOBuffer iobuf = nullptr; + TSIOBufferReader reader = nullptr; - InterceptIOChannel() : vio(nullptr), iobuf(nullptr), reader(nullptr) {} + InterceptIOChannel() {} ~InterceptIOChannel() { if (this->reader) { @@ -130,12 +130,12 @@ struct InterceptIO { // are intercepting is the server. Hence the "client" and // "server" nomenclature here. struct InterceptState { - TSHttpTxn txn; // The transaction on whose behalf we are intercepting. + TSHttpTxn txn = nullptr; // The transaction on whose behalf we are intercepting. InterceptIO client; // Server intercept VC state. InterceptIO server; // Intercept origin VC state. - InterceptState() : txn(nullptr) {} + InterceptState() {} ~InterceptState() {} }; diff --git a/example/null_transform/null_transform.c b/example/null_transform/null_transform.c index e180f5c10cc..b350c3f076f 100644 --- a/example/null_transform/null_transform.c +++ b/example/null_transform/null_transform.c @@ -67,7 +67,6 @@ handle_transform(TSCont contp) TSVIO input_vio; MyData *data; int64_t towrite; - int64_t avail; TSDebug(PLUGIN_NAME, "Entering handle_transform()"); /* Get the output (downstream) vconnection where we'll write data to. */ @@ -126,7 +125,7 @@ handle_transform(TSCont contp) /* The amount of data left to read needs to be truncated by * the amount of data actually in the read buffer. */ - avail = TSIOBufferReaderAvail(TSVIOReaderGet(input_vio)); + int64_t avail = TSIOBufferReaderAvail(TSVIOReaderGet(input_vio)); TSDebug(PLUGIN_NAME, "\tavail is %" PRId64 "", avail); if (towrite > avail) { towrite = avail; diff --git a/example/passthru/passthru.cc b/example/passthru/passthru.cc index 09f386f0b8a..8912e2961be 100644 --- a/example/passthru/passthru.cc +++ b/example/passthru/passthru.cc @@ -53,11 +53,11 @@ union EventArgument { }; struct PassthruIO { - TSVIO vio; - TSIOBuffer iobuf; - TSIOBufferReader reader; + TSVIO vio = nullptr; + TSIOBuffer iobuf = nullptr; + TSIOBufferReader reader = nullptr; - PassthruIO() : vio(nullptr), iobuf(nullptr), reader(nullptr) {} + PassthruIO() {} ~PassthruIO() { clear(); } void clear() @@ -182,7 +182,7 @@ static int PassthruSessionEvent(TSCont cont, TSEvent event, void *edata) { EventArgument arg(edata); - PassthruSession *sp = (PassthruSession *)TSContDataGet(cont); + PassthruSession *sp = static_cast(TSContDataGet(cont)); PassthruSessionDebug(sp, "session event on vconn=%p event=%d (%s)", TSVIOVConnGet(arg.vio), event, TSHttpEventNameLookup(event)); diff --git a/example/protocol/TxnSM.c b/example/protocol/TxnSM.c index 10d3724cad6..144c501e3c1 100644 --- a/example/protocol/TxnSM.c +++ b/example/protocol/TxnSM.c @@ -209,8 +209,7 @@ state_interface_with_client(TSCont contp, TSEvent event, TSVIO vio) int state_read_request_from_client(TSCont contp, TSEvent event, TSVIO vio ATS_UNUSED) { - int bytes_read, parse_result; - char *temp_buf; + int bytes_read; TxnSM *txn_sm = (TxnSM *)TSContDataGet(contp); @@ -221,7 +220,7 @@ state_read_request_from_client(TSCont contp, TSEvent event, TSVIO vio ATS_UNUSED bytes_read = TSIOBufferReaderAvail(txn_sm->q_client_request_buffer_reader); if (bytes_read > 0) { - temp_buf = (char *)get_info_from_buffer(txn_sm->q_client_request_buffer_reader); + char *temp_buf = get_info_from_buffer(txn_sm->q_client_request_buffer_reader); TSstrlcat(txn_sm->q_client_request, temp_buf, MAX_REQUEST_LENGTH + 1); TSfree(temp_buf); @@ -230,7 +229,8 @@ state_read_request_from_client(TSCont contp, TSEvent event, TSVIO vio ATS_UNUSED temp_buf = (char *)TSmalloc(sizeof(char) * (strlen(txn_sm->q_client_request) + 1)); memcpy(temp_buf, txn_sm->q_client_request, strlen(txn_sm->q_client_request)); temp_buf[strlen(txn_sm->q_client_request)] = '\0'; - parse_result = parse_request(temp_buf, txn_sm->q_server_name, txn_sm->q_file_name); + + int parse_result = parse_request(temp_buf, txn_sm->q_server_name, txn_sm->q_file_name); TSfree(temp_buf); if (parse_result != 1) { @@ -546,8 +546,6 @@ state_send_request_to_server(TSCont contp, TSEvent event, TSVIO vio) TSVIOReenable(vio); break; case TS_EVENT_VCONN_WRITE_COMPLETE: - vio = NULL; - /* Waiting for the incoming response. */ set_handler(txn_sm->q_current_handler, (TxnSMHandler)&state_interface_with_server); txn_sm->q_server_read_vio = TSVConnRead(txn_sm->q_server_vc, contp, txn_sm->q_server_response_buffer, INT64_MAX); @@ -923,8 +921,6 @@ get_info_from_buffer(TSIOBufferReader the_reader) char *info_start; int64_t read_avail, read_done; - TSIOBufferBlock blk; - char *buf; if (!the_reader) { return NULL; @@ -940,8 +936,8 @@ get_info_from_buffer(TSIOBufferReader the_reader) /* Read the data out of the reader */ while (read_avail > 0) { - blk = TSIOBufferReaderStart(the_reader); - buf = (char *)TSIOBufferBlockReadStart(blk, the_reader, &read_done); + TSIOBufferBlock blk = TSIOBufferReaderStart(the_reader); + char *buf = (char *)TSIOBufferBlockReadStart(blk, the_reader, &read_done); memcpy(info, buf, read_done); if (read_done > 0) { TSIOBufferReaderConsume(the_reader, read_done); diff --git a/example/query_remap/query_remap.c b/example/query_remap/query_remap.c index 8bd6910179a..9e850bb7299 100644 --- a/example/query_remap/query_remap.c +++ b/example/query_remap/query_remap.c @@ -91,7 +91,6 @@ void TSRemapDeleteInstance(void *ih) { /* Release instance memory allocated in TSRemapNewInstance */ - int i; TSDebug(PLUGIN_NAME, "deleting instance %p", ih); if (ih) { @@ -100,7 +99,7 @@ TSRemapDeleteInstance(void *ih) TSfree(qri->param_name); } if (qri->hosts) { - for (i = 0; i < qri->num_hosts; ++i) { + for (int i = 0; i < qri->num_hosts; ++i) { TSfree(qri->hosts[i]); } TSfree(qri->hosts); @@ -112,7 +111,6 @@ TSRemapDeleteInstance(void *ih) TSRemapStatus TSRemapDoRemap(void *ih, TSHttpTxn rh ATS_UNUSED, TSRemapRequestInfo *rri) { - int hostidx = -1; query_remap_info *qri = (query_remap_info *)ih; if (!qri || !rri) { @@ -125,7 +123,8 @@ TSRemapDoRemap(void *ih, TSHttpTxn rh ATS_UNUSED, TSRemapRequestInfo *rri) if (req_query && req_query_len > 0) { char *q, *key; - char *s = NULL; + char *s = NULL; + int hostidx = -1; /* make a copy of the query, as it is read only */ q = (char *)TSstrndup(req_query, req_query_len + 1); diff --git a/example/redirect_1/redirect_1.c b/example/redirect_1/redirect_1.c index 677ef716e14..45b83408700 100644 --- a/example/redirect_1/redirect_1.c +++ b/example/redirect_1/redirect_1.c @@ -246,7 +246,6 @@ update_redirected_method_stats(TSMBuffer bufp, TSMLoc hdr_loc) { const char *txn_method; int length; - int64_t tempint; txn_method = TSHttpHdrMethodGet(bufp, hdr_loc, &length); @@ -264,7 +263,7 @@ update_redirected_method_stats(TSMBuffer bufp, TSMLoc hdr_loc) } else if (0 == strncmp(txn_method, TS_HTTP_METHOD_OPTIONS, length)) { // This is a bad idea in a real plugin because it causes a race condition // with other transactions, but is here for illustrative purposes. - tempint = TSStatIntGet(redirect_count_options); + int64_t tempint = TSStatIntGet(redirect_count_options); ++tempint; TSStatIntSet(redirect_count_options, tempint); } else if (0 == strncmp(txn_method, TS_HTTP_METHOD_POST, length)) { @@ -289,7 +288,6 @@ void TSPluginInit(int argc, const char *argv[]) { const char prefix[] = "http://"; - int uri_len; TSPluginRegistrationInfo info; info.plugin_name = PLUGIN_NAME; @@ -308,7 +306,7 @@ TSPluginInit(int argc, const char *argv[]) */ url_redirect = TSstrdup(argv[2]); - uri_len = strlen(prefix) + strlen(url_redirect) + 1; + int uri_len = strlen(prefix) + strlen(url_redirect) + 1; uri_redirect = TSmalloc(uri_len); TSstrlcpy(uri_redirect, prefix, uri_len); TSstrlcat(uri_redirect, url_redirect, uri_len); diff --git a/example/remap/remap.cc b/example/remap/remap.cc index 359e03867fa..c60b1a59391 100644 --- a/example/remap/remap.cc +++ b/example/remap/remap.cc @@ -66,9 +66,8 @@ pthread_mutex_t remap_entry::mutex; /* remap_entry class mutex */ /* ----------------------- remap_entry::remap_entry ------------------------ */ remap_entry::remap_entry(int _argc, char *_argv[]) : next(nullptr), argc(0), argv(nullptr) { - int i; - if (_argc > 0 && _argv && (argv = (char **)TSmalloc(sizeof(char *) * (_argc + 1))) != nullptr) { + int i; argc = _argc; for (i = 0; i < argc; i++) { argv[i] = TSstrdup(_argv[i]); @@ -80,10 +79,8 @@ remap_entry::remap_entry(int _argc, char *_argv[]) : next(nullptr), argc(0), arg /* ---------------------- remap_entry::~remap_entry ------------------------ */ remap_entry::~remap_entry() { - int i; - if (argc && argv) { - for (i = 0; i < argc; i++) { + for (int i = 0; i < argc; i++) { TSfree(argv[i]); } TSfree(argv); @@ -106,10 +103,9 @@ remap_entry::add_to_list(remap_entry *re) void remap_entry::remove_from_list(remap_entry *re) { - remap_entry **rre; if (likely(re && plugin_init_counter)) { pthread_mutex_lock(&mutex); - for (rre = &active_list; *rre; rre = &((*rre)->next)) { + for (remap_entry **rre = &active_list; *rre; rre = &((*rre)->next)) { if (re == *rre) { *rre = re->next; break; @@ -203,7 +199,7 @@ TSRemapNewInstance(int argc, char *argv[], void **ih, char *errbuf, int errbuf_s void TSRemapDeleteInstance(void *ih) { - remap_entry *ri = (remap_entry *)ih; + remap_entry *ri = static_cast(ih); TSDebug(PLUGIN_NAME, "enter"); @@ -225,7 +221,7 @@ TSRemapDoRemap(void *ih, TSHttpTxn rh, TSRemapRequestInfo *rri) TSMLoc cfield; uint64_t _processing_counter = processing_counter++; - remap_entry *ri = (remap_entry *)ih; + remap_entry *ri = static_cast(ih); TSDebug(PLUGIN_NAME, "enter"); if (!ri || !rri) { @@ -281,8 +277,8 @@ TSRemapDoRemap(void *ih, TSHttpTxn rh, TSRemapRequestInfo *rri) char *tmp = (char *)TSmalloc(256); static int my_local_counter = 0; - size_t len = snprintf(tmp, 255, "This is very small example of TS API usage!\nIteration %d!\nHTTP return code %d\n", - my_local_counter, TS_HTTP_STATUS_CONTINUE + my_local_counter); + len = snprintf(tmp, 255, "This is very small example of TS API usage!\nIteration %d!\nHTTP return code %d\n", my_local_counter, + TS_HTTP_STATUS_CONTINUE + my_local_counter); TSHttpTxnStatusSet((TSHttpTxn)rh, (TSHttpStatus)((int)TS_HTTP_STATUS_CONTINUE + my_local_counter)); TSHttpTxnErrorBodySet((TSHttpTxn)rh, tmp, len, nullptr); // Defaults to text/html my_local_counter++; diff --git a/example/remap_header_add/remap_header_add.cc b/example/remap_header_add/remap_header_add.cc index d1152914423..f538fae264e 100644 --- a/example/remap_header_add/remap_header_add.cc +++ b/example/remap_header_add/remap_header_add.cc @@ -98,7 +98,7 @@ TSRemapNewInstance(int argc, char *argv[], void **ih, char *, int) // print all arguments for this particular remapping - rl = (remap_line *)TSmalloc(sizeof(remap_line)); + rl = static_cast(TSmalloc(sizeof(remap_line))); rl->argc = argc; rl->argv = argv; rl->nvc = argc - 2; // the first two are the remap from and to diff --git a/example/response_header_1/response_header_1.c b/example/response_header_1/response_header_1.c index a96afaa9fe3..9a0430f5438 100644 --- a/example/response_header_1/response_header_1.c +++ b/example/response_header_1/response_header_1.c @@ -68,7 +68,6 @@ modify_header(TSHttpTxn txnp) TSHttpStatus resp_status; TSMLoc new_field_loc; TSMLoc cached_field_loc; - time_t recvd_time; const char *chkptr; int chklength; @@ -114,7 +113,8 @@ modify_header(TSHttpTxn txnp) TSDebug(PLUGIN_NAME, "Created new resp field with loc %p", new_field_loc); TSMimeHdrFieldAppend(resp_bufp, resp_loc, new_field_loc); TSMimeHdrFieldNameSet(resp_bufp, resp_loc, new_field_loc, mimehdr2_name, strlen(mimehdr2_name)); - recvd_time = time(NULL); + + time_t recvd_time = time(NULL); TSMimeHdrFieldValueDateInsert(resp_bufp, resp_loc, new_field_loc, recvd_time); TSHandleMLocRelease(resp_bufp, resp_loc, new_field_loc); @@ -195,7 +195,7 @@ modify_header(TSHttpTxn txnp) * Additional 200/304 processing can go here, if so desired. */ - /* Caller reneables */ + /* Caller reenables */ } static int diff --git a/example/secure_link/secure_link.c b/example/secure_link/secure_link.c index 9bf8506fba7..871e7617e67 100644 --- a/example/secure_link/secure_link.c +++ b/example/secure_link/secure_link.c @@ -151,7 +151,6 @@ TSReturnCode TSRemapNewInstance(int argc, char **argv, void **ih, char *errbuf, int errbuf_size) { int i; - char *ptr; secure_link_info *sli; // squash unused variable warnings ... @@ -163,6 +162,7 @@ TSRemapNewInstance(int argc, char **argv, void **ih, char *errbuf, int errbuf_si sli->strict = 0; for (i = 2; i < argc; i++) { + char *ptr; if ((ptr = strchr(argv[i], ':')) != NULL) { *ptr++ = '\0'; if (strcmp(argv[i], "secret") == 0) { diff --git a/example/server_push/server_push.c b/example/server_push/server_push.c index 717d373bcf0..41203f532db 100644 --- a/example/server_push/server_push.c +++ b/example/server_push/server_push.c @@ -47,16 +47,16 @@ bool should_push(TSHttpTxn txnp) { TSMBuffer mbuf; - TSMLoc hdr, url; + TSMLoc hdr, in_url; if (TSHttpTxnClientReqGet(txnp, &mbuf, &hdr) != TS_SUCCESS) { return false; } - if (TSHttpHdrUrlGet(mbuf, hdr, &url) != TS_SUCCESS) { + if (TSHttpHdrUrlGet(mbuf, hdr, &in_url) != TS_SUCCESS) { return false; } int len; - TSUrlHttpQueryGet(mbuf, url, &len); - TSHandleMLocRelease(mbuf, hdr, url); + TSUrlHttpQueryGet(mbuf, in_url, &len); + TSHandleMLocRelease(mbuf, hdr, in_url); TSHandleMLocRelease(mbuf, TS_NULL_MLOC, hdr); if (len > 0) { return true; diff --git a/example/server_transform/server_transform.c b/example/server_transform/server_transform.c index ec93e5303e4..825c1fedb5f 100644 --- a/example/server_transform/server_transform.c +++ b/example/server_transform/server_transform.c @@ -285,7 +285,6 @@ transform_buffer_event(TSCont contp, TransformData *data, TSEvent event ATS_UNUS { TSVIO write_vio; int towrite; - int avail; if (!data->input_buf) { data->input_buf = TSIOBufferCreate(); @@ -314,7 +313,7 @@ transform_buffer_event(TSCont contp, TransformData *data, TSEvent event ATS_UNUS if (towrite > 0) { /* The amount of data left to read needs to be truncated by the amount of data actually in the read buffer. */ - avail = TSIOBufferReaderAvail(TSVIOReaderGet(write_vio)); + int avail = TSIOBufferReaderAvail(TSVIOReaderGet(write_vio)); if (towrite > avail) { towrite = avail; } @@ -404,18 +403,16 @@ transform_read_status_event(TSCont contp, TransformData *data, TSEvent event, vo return transform_bypass(contp, data); case TS_EVENT_VCONN_READ_COMPLETE: if (TSIOBufferReaderAvail(data->output_reader) == sizeof(int)) { - TSIOBufferBlock blk; - char *buf; void *buf_ptr; int64_t avail; int64_t read_nbytes = sizeof(int); - int64_t read_ndone = 0; buf_ptr = &data->content_length; while (read_nbytes > 0) { - blk = TSIOBufferReaderStart(data->output_reader); - buf = (char *)TSIOBufferBlockReadStart(blk, data->output_reader, &avail); - read_ndone = (avail >= read_nbytes) ? read_nbytes : avail; + TSIOBufferBlock blk = TSIOBufferReaderStart(data->output_reader); + char *buf = (char *)TSIOBufferBlockReadStart(blk, data->output_reader, &avail); + int64_t read_ndone = (avail >= read_nbytes) ? read_nbytes : avail; + memcpy(buf_ptr, buf, read_ndone); if (read_ndone > 0) { TSIOBufferReaderConsume(data->output_reader, read_ndone); diff --git a/example/thread_pool/psi.c b/example/thread_pool/psi.c index fa465b2c1f2..4793c699cdd 100644 --- a/example/thread_pool/psi.c +++ b/example/thread_pool/psi.c @@ -438,7 +438,7 @@ _basename(const char *filename) data continuation for the current transaction Output : data->psi_buffer contains the file content - data->psi_sucess 0 if include failed, 1 if success + data->psi_success 0 if include failed, 1 if success Return Value: 0 if failure 1 if success @@ -449,7 +449,6 @@ psi_include(TSCont contp, void *edata ATS_UNUSED) #define BUFFER_SIZE 1024 ContData *data; TSFile filep; - char buf[BUFFER_SIZE]; char inc_file[PSI_PATH_MAX_SIZE + PSI_FILENAME_MAX_SIZE]; /* We manipulate plugin continuation data from a separate thread. @@ -472,19 +471,19 @@ psi_include(TSCont contp, void *edata ATS_UNUSED) if ((filep = TSfopen(inc_file, "r")) != NULL) { TSDebug(PLUGIN_NAME, "Reading include file %s", inc_file); + char buf[BUFFER_SIZE]; while (TSfgets(filep, buf, BUFFER_SIZE) != NULL) { - TSIOBufferBlock block; - int64_t len, avail, ndone, ntodo, towrite; - char *ptr_block; + int64_t len, ndone, ntodo; len = strlen(buf); ndone = 0; ntodo = len; while (ntodo > 0) { /* TSIOBufferStart allocates more blocks if required */ - block = TSIOBufferStart(data->psi_buffer); - ptr_block = TSIOBufferBlockWriteStart(block, &avail); - towrite = MIN(ntodo, avail); + TSIOBufferBlock block = TSIOBufferStart(data->psi_buffer); + int64_t avail; + char *ptr_block = TSIOBufferBlockWriteStart(block, &avail); + int64_t towrite = MIN(ntodo, avail); memcpy(ptr_block, buf + ndone, towrite); TSIOBufferProduce(data->psi_buffer, towrite); @@ -511,7 +510,7 @@ psi_include(TSCont contp, void *edata ATS_UNUSED) TS_HTTP_READ_REQUEST_HDR, TS_HTTP_OS_DNS and so on...) we could use TSHttpTxnReenable to wake up the transaction instead of sending an event. */ - TSContSchedule(contp, 0, TS_THREAD_POOL_DEFAULT); + TSContScheduleOnPool(contp, 0, TS_THREAD_POOL_NET); data->psi_success = 0; data->state = STATE_READ_DATA; TSMutexUnlock(TSContMutexGet(contp)); @@ -579,8 +578,7 @@ handle_transform(TSCont contp) TSVConn output_conn; TSVIO input_vio; ContData *data; - TSIOBufferReader input_reader; - int toread, avail, psi, toconsume = 0, towrite = 0; + int toread, toconsume = 0, towrite = 0; /* Get the output (downstream) vconnection where we'll write data to. */ output_conn = TSTransformOutputVConnGet(contp); @@ -610,11 +608,12 @@ handle_transform(TSCont contp) toread = TSVIONTodoGet(input_vio); if (toread > 0) { - input_reader = TSVIOReaderGet(input_vio); - avail = TSIOBufferReaderAvail(input_reader); + TSIOBufferReader input_reader = TSVIOReaderGet(input_vio); + int avail = TSIOBufferReaderAvail(input_reader); /* There are some data available for reading. Let's parse it */ if (avail > 0) { + int psi; /* No need to parse data if there are too few bytes left to contain an include command... */ if (toread > (PSI_START_TAG_LEN + PSI_END_TAG_LEN)) { @@ -685,20 +684,13 @@ static int dump_psi(TSCont contp) { ContData *data; - int psi_output_len; - -/* TODO: This is odd, do we need to get the input_vio, but never use it ?? */ -#if 0 - TSVIO input_vio; - input_vio = TSVConnWriteVIOGet(contp); -#endif data = TSContDataGet(contp); TSAssert(data->magic == MAGIC_ALIVE); /* If script exec succeeded, copy its output to the downstream vconn */ if (data->psi_success == 1) { - psi_output_len = TSIOBufferReaderAvail(data->psi_reader); + int psi_output_len = TSIOBufferReaderAvail(data->psi_reader); if (psi_output_len > 0) { data->transform_bytes += psi_output_len; @@ -733,7 +725,6 @@ dump_psi(TSCont contp) static int transform_handler(TSCont contp, TSEvent event, void *edata ATS_UNUSED) { - TSVIO input_vio; ContData *data; int state, retval; @@ -749,7 +740,7 @@ transform_handler(TSCont contp, TSEvent event, void *edata ATS_UNUSED) d->contp = contp; d->event = event; TSContDataSet(c, d); - TSContSchedule(c, 10, TS_THREAD_POOL_DEFAULT); + TSContScheduleOnPool(c, 10, TS_THREAD_POOL_NET); return 1; } @@ -765,7 +756,7 @@ transform_handler(TSCont contp, TSEvent event, void *edata ATS_UNUSED) the continuation right away as the thread will call us back on this continuation. */ if (state == STATE_READ_PSI) { - TSContSchedule(contp, 10, TS_THREAD_POOL_DEFAULT); + TSContScheduleOnPool(contp, 10, TS_THREAD_POOL_NET); } else { TSMutexUnlock(TSContMutexGet(contp)); cont_data_destroy(TSContDataGet(contp)); @@ -773,6 +764,7 @@ transform_handler(TSCont contp, TSEvent event, void *edata ATS_UNUSED) return 1; } } else { + TSVIO input_vio; switch (event) { case TS_EVENT_ERROR: input_vio = TSVConnWriteVIOGet(contp); @@ -853,9 +845,8 @@ transformable(TSHttpTxn txnp) /* We are only interested in transforming "200 OK" responses with a Content-Type: text/ header and with X-Psi header */ TSMBuffer bufp; - TSMLoc hdr_loc, field_loc; + TSMLoc hdr_loc; TSHttpStatus resp_status; - const char *value; if (TS_SUCCESS == TSHttpTxnServerRespGet(txnp, &bufp, &hdr_loc)) { resp_status = TSHttpHdrStatusGet(bufp, hdr_loc); @@ -864,6 +855,7 @@ transformable(TSHttpTxn txnp) return 0; } + TSMLoc field_loc; field_loc = TSMimeHdrFieldFind(bufp, hdr_loc, TS_MIME_FIELD_CONTENT_TYPE, -1); if (field_loc == TS_NULL_MLOC) { TSError("[%s] Unable to search Content-Type field", PLUGIN_NAME); @@ -871,7 +863,7 @@ transformable(TSHttpTxn txnp) return 0; } - value = TSMimeHdrFieldValueStringGet(bufp, hdr_loc, field_loc, -1, NULL); + const char *value = TSMimeHdrFieldValueStringGet(bufp, hdr_loc, field_loc, -1, NULL); if ((value == NULL) || (strncasecmp(value, "text/", sizeof("text/") - 1) != 0)) { TSHandleMLocRelease(bufp, hdr_loc, field_loc); TSHandleMLocRelease(bufp, TS_NULL_MLOC, hdr_loc); diff --git a/example/thread_pool/test/SDKTest/psi_server.c b/example/thread_pool/test/SDKTest/psi_server.c index 2f4a73c5a16..cfa209ef0c1 100644 --- a/example/thread_pool/test/SDKTest/psi_server.c +++ b/example/thread_pool/test/SDKTest/psi_server.c @@ -29,7 +29,7 @@ * - PSI header * - PSI include in body * - * Ratio for generating PSI response is specifid in config file. + * Ratio for generating PSI response is specified in config file. * * Added Options in Synth_server.config - * psi_ratio : percentage of response with psi embedded we want to generate @@ -134,9 +134,6 @@ TSResponsePut(void **resp_id /* return */, void *resp_buffer /* return */, int * { int i = 0; RequestInfo *rid = *((RequestInfo **)resp_id); - int psi = 0; - int len; - char psi_tag[PSI_TAG_MAX_SIZE]; /* copy the header into the response buffer */ if (!rid->done_sent_header) { @@ -163,7 +160,8 @@ TSResponsePut(void **resp_id /* return */, void *resp_buffer /* return */, int * else { if (rid->psi) { /* generate our psi tag: */ - len = sprintf(psi_tag, PSI_TAG_FORMAT, rand() % 100); + char psi_tag[PSI_TAG_MAX_SIZE]; + int len = sprintf(psi_tag, PSI_TAG_FORMAT, rand() % 100); /* hopefully enough space for our include command */ if (rid->bytes_not_sent >= len) { diff --git a/example/thread_pool/thread.c b/example/thread_pool/thread.c index 79347bcf178..f5e8b536b96 100644 --- a/example/thread_pool/thread.c +++ b/example/thread_pool/thread.c @@ -48,13 +48,10 @@ init_queue(Queue *q) void add_to_queue(Queue *q, void *data) { - Cell *new_cell; - int n; - if (data != NULL) { TSMutexLock(q->mutex); /* Init the new cell */ - new_cell = TSmalloc(sizeof(Cell)); + Cell *new_cell = TSmalloc(sizeof(Cell)); new_cell->magic = MAGIC_ALIVE; new_cell->ptr_data = data; new_cell->ptr_next = q->tail; @@ -71,7 +68,7 @@ add_to_queue(Queue *q, void *data) q->tail->ptr_prev = new_cell; q->tail = new_cell; } - n = q->nb_elem++; + int n = q->nb_elem++; TSMutexUnlock(q->mutex); if (n > MAX_JOBS_ALARM) { @@ -157,12 +154,10 @@ thread_init() void * thread_loop(void *arg ATS_UNUSED) { - Job *job_todo; - /* Infinite loop */ for (;;) { /* returns a job or NULL if no jobs to do */ - job_todo = remove_from_queue(&job_queue); + Job *job_todo = remove_from_queue(&job_queue); if (job_todo != NULL) { TSAssert(job_todo->magic == MAGIC_ALIVE); diff --git a/include/ts/TsException.h b/include/ts/TsException.h index ca833d0c6ff..ff808947430 100644 --- a/include/ts/TsException.h +++ b/include/ts/TsException.h @@ -35,7 +35,7 @@ namespace ts /** Base class for ATS exception. Clients should subclass as appropriate. This is intended to carry pre-allocated text along so that it can be thrown without any - addditional memory allocation. + additional memory allocation. */ class Exception { diff --git a/include/ts/apidefs.h.in b/include/ts/apidefs.h.in index ecdc8244402..4841d5602ed 100644 --- a/include/ts/apidefs.h.in +++ b/include/ts/apidefs.h.in @@ -43,7 +43,7 @@ */ #include -#include +#include // NOLINT(modernize-deprecated-headers) #include #include @@ -283,9 +283,10 @@ typedef enum { // Putting the SSL hooks in the same enum space // So both sets of hooks can be set by the same Hook function TS_SSL_FIRST_HOOK, - TS_VCONN_START_HOOK = TS_SSL_FIRST_HOOK, + TS_VCONN_START_HOOK = TS_SSL_FIRST_HOOK, TS_VCONN_PRE_ACCEPT_HOOK = TS_VCONN_START_HOOK, // Deprecated but compatible for now. TS_VCONN_CLOSE_HOOK, + TS_SSL_CLIENT_HELLO_HOOK, TS_SSL_SNI_HOOK, TS_SSL_CERT_HOOK = TS_SSL_SNI_HOOK, TS_SSL_SERVERNAME_HOOK, @@ -324,7 +325,7 @@ typedef enum { accept connections. This is *not* guaranteed to be called before the first connection is accepted. - Event: TS_EVENT_LIFECYCLE_PORTS_READY_HOOK + Event: TS_EVENT_LIFECYCLE_PORTS_READY TS_LIFECYCLE_CACHE_READY_HOOK @@ -334,12 +335,6 @@ typedef enum { Event: TS_EVENT_LIFECYCLE_CACHE_READY - TS_LIFECYCLE_MSG_HOOK - - Called in response to an external agent. The data is a pointer to an instance of TSPluginMsg. - - Event: TS_EVENT_LIFECYCLE_MSG - TS_LIFECYCLE_SERVER_SSL_CTX_INITIALIZED_HOOK called every time after a server SSL_CTX has finished the initialization. @@ -354,6 +349,24 @@ typedef enum { Event: TS_EVENT_LIFECYCLE_CLIENT_SSL_CTX_INITIALIZED + TS_LIFECYCLE_MSG_HOOK + + Called in response to an external agent. The data is a pointer to an instance of TSPluginMsg. + + Event: TS_EVENT_LIFECYCLE_MSG + + TS_LIFECYCLE_TASK_THREADS_READY_HOOK + + called once, after the task threads have been started. + + Event: TS_EVENT_LIFECYCLE_TASK_THREADS_READY + + TS_LIFECYCLE_SHUTDOWN_HOOK + + called once, after receiving a shutdown signal, such as SIGTERM. + + Event: TS_EVENT_LIFECYCLE_SHUTDOWN + Ordering guarantees: - TS_LIFECYCLE_PORTS_INITIALIZED_HOOK before TS_LIFECYCLE_PORTS_READY_HOOK. @@ -368,6 +381,8 @@ typedef enum { TS_LIFECYCLE_SERVER_SSL_CTX_INITIALIZED_HOOK, TS_LIFECYCLE_CLIENT_SSL_CTX_INITIALIZED_HOOK, TS_LIFECYCLE_MSG_HOOK, + TS_LIFECYCLE_TASK_THREADS_READY_HOOK, + TS_LIFECYCLE_SHUTDOWN_HOOK, TS_LIFECYCLE_LAST_HOOK } TSLifecycleHookID; @@ -379,95 +394,110 @@ typedef enum { */ typedef enum { - TS_EVENT_NONE = 0, - TS_EVENT_IMMEDIATE = 1, - TS_EVENT_TIMEOUT = 2, - TS_EVENT_ERROR = 3, - TS_EVENT_CONTINUE = 4, - TS_EVENT_VCONN_READ_READY = 100, - TS_EVENT_VCONN_WRITE_READY = 101, - TS_EVENT_VCONN_READ_COMPLETE = 102, - TS_EVENT_VCONN_WRITE_COMPLETE = 103, - TS_EVENT_VCONN_EOS = 104, - TS_EVENT_VCONN_INACTIVITY_TIMEOUT = 105, - TS_EVENT_VCONN_ACTIVE_TIMEOUT = 106, - TS_EVENT_NET_CONNECT = 200, - TS_EVENT_NET_CONNECT_FAILED = 201, - TS_EVENT_NET_ACCEPT = 202, - TS_EVENT_NET_ACCEPT_FAILED = 204, - TS_EVENT_INTERNAL_206 = 206, - TS_EVENT_INTERNAL_207 = 207, - TS_EVENT_INTERNAL_208 = 208, - TS_EVENT_INTERNAL_209 = 209, - TS_EVENT_INTERNAL_210 = 210, - TS_EVENT_INTERNAL_211 = 211, - TS_EVENT_INTERNAL_212 = 212, - TS_EVENT_HOST_LOOKUP = 500, - TS_EVENT_CACHE_OPEN_READ = 1102, - TS_EVENT_CACHE_OPEN_READ_FAILED = 1103, - TS_EVENT_CACHE_OPEN_WRITE = 1108, - TS_EVENT_CACHE_OPEN_WRITE_FAILED = 1109, - TS_EVENT_CACHE_REMOVE = 1112, - TS_EVENT_CACHE_REMOVE_FAILED = 1113, - TS_EVENT_CACHE_SCAN = 1120, - TS_EVENT_CACHE_SCAN_FAILED = 1121, - TS_EVENT_CACHE_SCAN_OBJECT = 1122, - TS_EVENT_CACHE_SCAN_OPERATION_BLOCKED = 1123, - TS_EVENT_CACHE_SCAN_OPERATION_FAILED = 1124, - TS_EVENT_CACHE_SCAN_DONE = 1125, - TS_EVENT_CACHE_LOOKUP = 1126, - TS_EVENT_CACHE_READ = 1127, - TS_EVENT_CACHE_DELETE = 1128, - TS_EVENT_CACHE_WRITE = 1129, - TS_EVENT_CACHE_WRITE_HEADER = 1130, - TS_EVENT_CACHE_CLOSE = 1131, - TS_EVENT_CACHE_LOOKUP_READY = 1132, - TS_EVENT_CACHE_LOOKUP_COMPLETE = 1133, - TS_EVENT_CACHE_READ_READY = 1134, - TS_EVENT_CACHE_READ_COMPLETE = 1135, - TS_EVENT_INTERNAL_1200 = 1200, - TS_EVENT_SSL_SESSION_GET = 2000, - TS_EVENT_SSL_SESSION_NEW = 2001, - TS_EVENT_SSL_SESSION_REMOVE = 2002, - TS_AIO_EVENT_DONE = 3900, - TS_EVENT_HTTP_CONTINUE = 60000, - TS_EVENT_HTTP_ERROR = 60001, - TS_EVENT_HTTP_READ_REQUEST_HDR = 60002, - TS_EVENT_HTTP_OS_DNS = 60003, - TS_EVENT_HTTP_SEND_REQUEST_HDR = 60004, - TS_EVENT_HTTP_READ_CACHE_HDR = 60005, - TS_EVENT_HTTP_READ_RESPONSE_HDR = 60006, - TS_EVENT_HTTP_SEND_RESPONSE_HDR = 60007, - TS_EVENT_HTTP_REQUEST_TRANSFORM = 60008, - TS_EVENT_HTTP_RESPONSE_TRANSFORM = 60009, - TS_EVENT_HTTP_SELECT_ALT = 60010, - TS_EVENT_HTTP_TXN_START = 60011, - TS_EVENT_HTTP_TXN_CLOSE = 60012, - TS_EVENT_HTTP_SSN_START = 60013, - TS_EVENT_HTTP_SSN_CLOSE = 60014, - TS_EVENT_HTTP_CACHE_LOOKUP_COMPLETE = 60015, - TS_EVENT_HTTP_PRE_REMAP = 60016, - TS_EVENT_HTTP_POST_REMAP = 60017, - TS_EVENT_LIFECYCLE_PORTS_INITIALIZED = 60018, - TS_EVENT_LIFECYCLE_PORTS_READY = 60019, - TS_EVENT_LIFECYCLE_CACHE_READY = 60020, - TS_EVENT_LIFECYCLE_SERVER_SSL_CTX_INITIALIZED = 60021, - TS_EVENT_LIFECYCLE_CLIENT_SSL_CTX_INITIALIZED = 60022, - TS_EVENT_VCONN_START = 60023, - TS_EVENT_VCONN_PRE_ACCEPT = TS_EVENT_VCONN_START, // Deprecated but still compatible - TS_EVENT_VCONN_CLOSE = 60026, - TS_EVENT_LIFECYCLE_MSG = 60024, - TS_EVENT_HTTP_REQUEST_BUFFER_COMPLETE = 60025, - TS_EVENT_MGMT_UPDATE = 60100, - TS_EVENT_INTERNAL_60200 = 60200, - TS_EVENT_INTERNAL_60201 = 60201, - TS_EVENT_INTERNAL_60202 = 60202, - TS_EVENT_SSL_CERT = 60203, - TS_EVENT_SSL_SERVERNAME = 60204, - TS_EVENT_SSL_VERIFY_SERVER = 60205, - TS_EVENT_SSL_VERIFY_CLIENT = 60206, - TS_EVENT_VCONN_OUTBOUND_START = 60207, - TS_EVENT_VCONN_OUTBOUND_CLOSE = 60208 + TS_EVENT_NONE = 0, + TS_EVENT_IMMEDIATE = 1, + TS_EVENT_TIMEOUT = 2, + TS_EVENT_ERROR = 3, + TS_EVENT_CONTINUE = 4, + + TS_EVENT_VCONN_READ_READY = 100, + TS_EVENT_VCONN_WRITE_READY = 101, + TS_EVENT_VCONN_READ_COMPLETE = 102, + TS_EVENT_VCONN_WRITE_COMPLETE = 103, + TS_EVENT_VCONN_EOS = 104, + TS_EVENT_VCONN_INACTIVITY_TIMEOUT = 105, + TS_EVENT_VCONN_ACTIVE_TIMEOUT = 106, + TS_EVENT_VCONN_START = 107, + TS_EVENT_VCONN_CLOSE = 108, + TS_EVENT_VCONN_OUTBOUND_START = 109, + TS_EVENT_VCONN_OUTBOUND_CLOSE = 110, + TS_EVENT_VCONN_PRE_ACCEPT = TS_EVENT_VCONN_START, // Deprecated but still compatible + + TS_EVENT_NET_CONNECT = 200, + TS_EVENT_NET_CONNECT_FAILED = 201, + TS_EVENT_NET_ACCEPT = 202, + TS_EVENT_NET_ACCEPT_FAILED = 204, + + TS_EVENT_INTERNAL_206 = 206, + TS_EVENT_INTERNAL_207 = 207, + TS_EVENT_INTERNAL_208 = 208, + TS_EVENT_INTERNAL_209 = 209, + TS_EVENT_INTERNAL_210 = 210, + TS_EVENT_INTERNAL_211 = 211, + TS_EVENT_INTERNAL_212 = 212, + + TS_EVENT_HOST_LOOKUP = 500, + + TS_EVENT_CACHE_OPEN_READ = 1102, + TS_EVENT_CACHE_OPEN_READ_FAILED = 1103, + TS_EVENT_CACHE_OPEN_WRITE = 1108, + TS_EVENT_CACHE_OPEN_WRITE_FAILED = 1109, + TS_EVENT_CACHE_REMOVE = 1112, + TS_EVENT_CACHE_REMOVE_FAILED = 1113, + TS_EVENT_CACHE_SCAN = 1120, + TS_EVENT_CACHE_SCAN_FAILED = 1121, + TS_EVENT_CACHE_SCAN_OBJECT = 1122, + TS_EVENT_CACHE_SCAN_OPERATION_BLOCKED = 1123, + TS_EVENT_CACHE_SCAN_OPERATION_FAILED = 1124, + TS_EVENT_CACHE_SCAN_DONE = 1125, + TS_EVENT_CACHE_LOOKUP = 1126, + TS_EVENT_CACHE_READ = 1127, + TS_EVENT_CACHE_DELETE = 1128, + TS_EVENT_CACHE_WRITE = 1129, + TS_EVENT_CACHE_WRITE_HEADER = 1130, + TS_EVENT_CACHE_CLOSE = 1131, + TS_EVENT_CACHE_LOOKUP_READY = 1132, + TS_EVENT_CACHE_LOOKUP_COMPLETE = 1133, + TS_EVENT_CACHE_READ_READY = 1134, + TS_EVENT_CACHE_READ_COMPLETE = 1135, + + TS_EVENT_INTERNAL_1200 = 1200, + + TS_EVENT_SSL_SESSION_GET = 2000, + TS_EVENT_SSL_SESSION_NEW = 2001, + TS_EVENT_SSL_SESSION_REMOVE = 2002, + + TS_EVENT_AIO_DONE = 3900, + + TS_EVENT_HTTP_CONTINUE = 60000, + TS_EVENT_HTTP_ERROR = 60001, + TS_EVENT_HTTP_READ_REQUEST_HDR = 60002, + TS_EVENT_HTTP_OS_DNS = 60003, + TS_EVENT_HTTP_SEND_REQUEST_HDR = 60004, + TS_EVENT_HTTP_READ_CACHE_HDR = 60005, + TS_EVENT_HTTP_READ_RESPONSE_HDR = 60006, + TS_EVENT_HTTP_SEND_RESPONSE_HDR = 60007, + TS_EVENT_HTTP_REQUEST_TRANSFORM = 60008, + TS_EVENT_HTTP_RESPONSE_TRANSFORM = 60009, + TS_EVENT_HTTP_SELECT_ALT = 60010, + TS_EVENT_HTTP_TXN_START = 60011, + TS_EVENT_HTTP_TXN_CLOSE = 60012, + TS_EVENT_HTTP_SSN_START = 60013, + TS_EVENT_HTTP_SSN_CLOSE = 60014, + TS_EVENT_HTTP_CACHE_LOOKUP_COMPLETE = 60015, + TS_EVENT_HTTP_PRE_REMAP = 60016, + TS_EVENT_HTTP_POST_REMAP = 60017, + TS_EVENT_HTTP_REQUEST_BUFFER_COMPLETE = 60018, + + TS_EVENT_LIFECYCLE_PORTS_INITIALIZED = 60100, + TS_EVENT_LIFECYCLE_PORTS_READY = 60101, + TS_EVENT_LIFECYCLE_CACHE_READY = 60102, + TS_EVENT_LIFECYCLE_SERVER_SSL_CTX_INITIALIZED = 60103, + TS_EVENT_LIFECYCLE_CLIENT_SSL_CTX_INITIALIZED = 60104, + TS_EVENT_LIFECYCLE_MSG = 60105, + TS_EVENT_LIFECYCLE_TASK_THREADS_READY = 60106, + TS_EVENT_LIFECYCLE_SHUTDOWN = 60107, + + TS_EVENT_INTERNAL_60200 = 60200, + TS_EVENT_INTERNAL_60201 = 60201, + TS_EVENT_INTERNAL_60202 = 60202, + TS_EVENT_SSL_CERT = 60203, + TS_EVENT_SSL_SERVERNAME = 60204, + TS_EVENT_SSL_VERIFY_SERVER = 60205, + TS_EVENT_SSL_VERIFY_CLIENT = 60206, + TS_EVENT_SSL_CLIENT_HELLO = 60207, + + TS_EVENT_MGMT_UPDATE = 60300 } TSEvent; #define TS_EVENT_HTTP_READ_REQUEST_PRE_REMAP TS_EVENT_HTTP_PRE_REMAP /* backwards compat */ @@ -761,6 +791,7 @@ typedef enum { TS_CONFIG_SRV_ENABLED, TS_CONFIG_HTTP_FORWARD_CONNECT_METHOD, TS_CONFIG_SSL_CERT_FILENAME, + TS_CONFIG_SSL_CLIENT_CERT_FILENAME = TS_CONFIG_SSL_CERT_FILENAME, TS_CONFIG_SSL_CERT_FILEPATH, TS_CONFIG_PARENT_FAILURES_UPDATE_HOSTDB, TS_CONFIG_HTTP_CACHE_ENABLE_DEFAULT_VARY_HEADER, @@ -782,13 +813,18 @@ typedef enum { TS_CONFIG_HTTP_ALLOW_HALF_OPEN, TS_CONFIG_HTTP_PER_SERVER_CONNECTION_MAX, TS_CONFIG_HTTP_PER_SERVER_CONNECTION_MATCH, + TS_CONFIG_SSL_CLIENT_VERIFY_SERVER, + TS_CONFIG_SSL_CLIENT_VERIFY_SERVER_POLICY, + TS_CONFIG_SSL_CLIENT_VERIFY_SERVER_PROPERTIES, + TS_CONFIG_SSL_CLIENT_SNI_POLICY, + TS_CONFIG_SSL_CLIENT_PRIVATE_KEY_FILENAME, + TS_CONFIG_SSL_CLIENT_CA_CERT_FILENAME, TS_CONFIG_LAST_ENTRY } TSOverridableConfigKey; /* The TASK pool of threads is the primary method of off-loading continuations from the net-threads. Configure this with proxy.config.task_threads in records.config. */ typedef enum { - TS_THREAD_POOL_DEFAULT = -1, TS_THREAD_POOL_NET, TS_THREAD_POOL_TASK, /* unlikely you should use these */ @@ -866,6 +902,7 @@ typedef struct tsapi_cachetxn *TSCacheTxn; typedef struct tsapi_port *TSPortDescriptor; typedef struct tsapi_vio *TSVIO; typedef struct tsapi_thread *TSThread; +typedef struct tsapi_event_thread *TSEventThread; typedef struct tsapi_x509 *TSSslX509; typedef struct tsapi_mutex *TSMutex; typedef struct tsapi_config *TSConfig; @@ -1249,6 +1286,13 @@ typedef enum { #define TS_CRUUID_STRING_LEN (TS_UUID_STRING_LEN + 19 + 1) /* UUID-len + len(uint64_t) + '-' */ typedef struct tsapi_uuid *TSUuid; +#ifdef __cplusplus +namespace ts +{ + static const int NO_FD = -1; ///< No or invalid file descriptor. +} +#endif + #ifdef __cplusplus } #endif /* __cplusplus */ diff --git a/include/ts/experimental.h b/include/ts/experimental.h index b6cfd35e63e..fa8e7ead89c 100644 --- a/include/ts/experimental.h +++ b/include/ts/experimental.h @@ -192,10 +192,9 @@ tsapi TSReturnCode TSHttpTxnInfoIntGet(TSHttpTxn txnp, TSHttpTxnInfoKey key, TSM /**************************************************************************** * TSHttpTxnCacheLookupCountGet - * Return: TS_SUCESS/TS_ERROR + * Return: TS_SUCCESS/TS_ERROR ****************************************************************************/ tsapi TSReturnCode TSHttpTxnCacheLookupCountGet(TSHttpTxn txnp, int *lookup_count); -tsapi TSReturnCode TSHttpTxnRedirectRequest(TSHttpTxn txnp, TSMBuffer bufp, TSMLoc url_loc); tsapi TSReturnCode TSHttpTxnServerRespIgnore(TSHttpTxn txnp); tsapi TSReturnCode TSHttpTxnShutDown(TSHttpTxn txnp, TSEvent event); tsapi TSReturnCode TSHttpTxnCloseAfterResponse(TSHttpTxn txnp, int should_close); @@ -391,7 +390,7 @@ tsapi void TSFetchWriteData(TSFetchSM fetch_sm, const void *data, size_t len); tsapi ssize_t TSFetchReadData(TSFetchSM fetch_sm, void *buf, size_t len); /* - * Lanuch FetchSM to do http request, before calling this API, + * Launch FetchSM to do http request, before calling this API, * you should append http request header into fetch sm through * TSFetchWriteData() API * diff --git a/include/ts/remap.h b/include/ts/remap.h index edc1c23e116..cc263fdd4ea 100644 --- a/include/ts/remap.h +++ b/include/ts/remap.h @@ -43,7 +43,7 @@ typedef struct _tsremap_api_info { typedef struct _tm_remap_request_info { /* Important: You should *not* release these buf pointers or TSMLocs from your plugin! */ - /* these URL mloc's are read only, use normal ts/ts.h APIs for accesing */ + /* these URL mloc's are read only, use normal ts/ts.h APIs for accessing */ TSMLoc mapFromUrl; TSMLoc mapToUrl; @@ -61,7 +61,7 @@ typedef struct _tm_remap_request_info { /* This is the type returned by the TSRemapDoRemap() callback */ typedef enum { - TSREMAP_NO_REMAP = 0, /* No remaping was done, continue with next in chain */ + TSREMAP_NO_REMAP = 0, /* No remapping was done, continue with next in chain */ TSREMAP_DID_REMAP = 1, /* Remapping was done, continue with next in chain */ TSREMAP_NO_REMAP_STOP = 2, /* No remapping was done, and stop plugin chain evaluation */ TSREMAP_DID_REMAP_STOP = 3, /* Remapping was done, but stop plugin chain evaluation */ @@ -88,7 +88,7 @@ typedef enum { */ tsapi TSReturnCode TSRemapInit(TSRemapInterface *api_info, char *errbuf, int errbuf_size); -/* This gets called everytime remap.config is reloaded. This is complementary +/* This gets called every time remap.config is reloaded. This is complementary to TSRemapInit() which gets called when the plugin is first loaded. You can not fail, or cause reload to stop here, it's merely a notification. Optional function. @@ -99,7 +99,7 @@ tsapi void TSRemapConfigReload(void); /* Remap new request Mandatory interface function. Remap API plugin can/should use SDK API function calls inside this function! - return: TSREMAP_NO_REMAP - No remaping was done, continue with next in chain + return: TSREMAP_NO_REMAP - No remapping was done, continue with next in chain TSREMAP_DID_REMAP - Remapping was done, continue with next in chain TSREMAP_NO_REMAP_STOP - No remapping was done, and stop plugin chain evaluation TSREMAP_DID_REMAP_STOP - Remapping was done, but stop plugin chain evaluation @@ -113,7 +113,7 @@ tsapi void TSRemapDone(void); /* Plugin new instance. Create new plugin processing entry for unique remap record. First two arguments in argv vector are - fromURL and toURL from remap record. Please keep in mind that fromURL and toURL will be converted to canonical view. - Return: TS_SUCESS + Return: TS_SUCCESS TS_ERROR - instance creation error */ tsapi TSReturnCode TSRemapNewInstance(int argc, char *argv[], void **ih, char *errbuf, int errbuf_size); diff --git a/include/ts/ts.h b/include/ts/ts.h index a3ba7b3d7bf..cd4f46a670f 100644 --- a/include/ts/ts.h +++ b/include/ts/ts.h @@ -1108,6 +1108,7 @@ tsapi TSThread TSThreadInit(void); tsapi void TSThreadDestroy(TSThread thread); tsapi void TSThreadWait(TSThread thread); tsapi TSThread TSThreadSelf(void); +tsapi TSEventThread TSEventThreadSelf(void); /* -------------------------------------------------------------------------- Mutexes */ @@ -1188,8 +1189,15 @@ tsapi TSCont TSContCreate(TSEventFunc funcp, TSMutex mutexp); tsapi void TSContDestroy(TSCont contp); tsapi void TSContDataSet(TSCont contp, void *data); tsapi void *TSContDataGet(TSCont contp); -tsapi TSAction TSContSchedule(TSCont contp, TSHRTime timeout, TSThreadPool tp); -tsapi TSAction TSContScheduleEvery(TSCont contp, TSHRTime every /* millisecs */, TSThreadPool tp); +tsapi TSAction TSContSchedule(TSCont contp, TSHRTime timeout); +tsapi TSAction TSContScheduleOnPool(TSCont contp, TSHRTime timeout, TSThreadPool tp); +tsapi TSAction TSContScheduleOnThread(TSCont contp, TSHRTime timeout, TSEventThread ethread); +tsapi TSAction TSContScheduleEvery(TSCont contp, TSHRTime every /* millisecs */); +tsapi TSAction TSContScheduleEveryOnPool(TSCont contp, TSHRTime every /* millisecs */, TSThreadPool tp); +tsapi TSAction TSContScheduleEveryOnThread(TSCont contp, TSHRTime every /* millisecs */, TSEventThread ethread); +tsapi TSReturnCode TSContThreadAffinitySet(TSCont contp, TSEventThread ethread); +tsapi TSEventThread TSContThreadAffinityGet(TSCont contp); +tsapi void TSContThreadAffinityClear(TSCont contp); tsapi TSAction TSHttpSchedule(TSCont contp, TSHttpTxn txnp, TSHRTime timeout); tsapi int TSContCall(TSCont contp, TSEvent event, void *edata); tsapi TSMutex TSContMutexGet(TSCont contp); @@ -1226,7 +1234,7 @@ tsapi TSSslConnection TSVConnSSLConnectionGet(TSVConn sslp); tsapi TSSslContext TSSslContextFindByName(const char *name); tsapi TSSslContext TSSslContextFindByAddr(struct sockaddr const *); /* Create a new SSL context based on the settings in records.config */ -tsapi TSSslContext TSSslServerContextCreate(TSSslX509 cert, const char *certname); +tsapi TSSslContext TSSslServerContextCreate(TSSslX509 cert, const char *certname, const char *rsp_file); tsapi void TSSslContextDestroy(TSSslContext ctx); tsapi void TSSslTicketKeyUpdate(char *ticketData, int ticketDataLen); tsapi TSNextProtocolSet TSUnregisterProtocol(TSNextProtocolSet protoset, const char *protocol); @@ -1301,7 +1309,7 @@ tsapi TSReturnCode TSHttpTxnCacheLookupStatusGet(TSHttpTxn txnp, int *lookup_sta tsapi TSReturnCode TSHttpTxnTransformRespGet(TSHttpTxn txnp, TSMBuffer *bufp, TSMLoc *offset); /** Set the @a port value for the inbound (user agent) connection in the transaction @a txnp. - This is used primarily where the conection is synthetic and therefore does not have a port. + This is used primarily where the connection is synthetic and therefore does not have a port. @note @a port is in @b host @b order. */ tsapi void TSHttpTxnClientIncomingPortSet(TSHttpTxn txnp, int port); @@ -1444,8 +1452,8 @@ tsapi TSReturnCode TSHttpTxnServerPacketDscpSet(TSHttpTxn txnp, int dscp); /** Sets an error type body to a transaction. Note that both string arguments must be allocated with TSmalloc() or TSstrdup(). The mimetype argument is - optional, if not provided it defaults to "text/html". Sending an emptry - string would prevent setting a content type header (but that is not adviced). + optional, if not provided it defaults to "text/html". Sending an empty + string would prevent setting a content type header (but that is not advised). @param txnp HTTP transaction whose parent proxy to get. @param buf The body message (must be heap allocated). @@ -1869,7 +1877,7 @@ tsapi TSAction TSCacheWrite(TSCont contp, TSCacheKey key); anything. The user does not get any vconnection from the cache, since no data needs to be transferred. When the cache calls contp back with TS_EVENT_CACHE_REMOVE, the remove has already - been commited. + been committed. @param contp continuation that the cache calls back reporting the success or failure of the remove. @@ -2011,7 +2019,7 @@ tsapi void TSStatIntDecrement(int the_stat, TSMgmtInt amount); tsapi TSMgmtInt TSStatIntGet(int the_stat); tsapi void TSStatIntSet(int the_stat, TSMgmtInt value); /* Currently not supported. */ -/* tsapi TSeturnCode TSStatFloatGet(int the_stat, float* value); */ +/* tsapi TSReturnCode TSStatFloatGet(int the_stat, float* value); */ /* tsapi TSReturnCode TSStatFloatSet(int the_stat, float value); */ tsapi TSReturnCode TSStatFindName(const char *name, int *idp); @@ -2025,7 +2033,7 @@ tsapi void TSDebug(const char *tag, const char *format_str, ...) TS_PRINTFLIKE(2 Output a debug line even if the debug tag is turned off, as long as debugging is enabled. Could be used as follows: @code - TSDebugSpecifc(TSHttpTxnDebugGet(txn), "plugin_tag" , "Hello World from transaction %p", txn); + TSDebugSpecific(TSHttpTxnDebugGet(txn), "plugin_tag" , "Hello World from transaction %p", txn); @endcode will be printed if the plugin_tag is enabled or the transaction specific debugging is turned on for txn. @@ -2079,8 +2087,7 @@ tsapi void TSRecordDump(int rec_type, TSRecordDumpCb callback, void *edata); Creates a new custom log file that your plugin can write to. You can design the fields and inputs to the log file using the TSTextLogObjectWrite() function. The logs you create are treated - like ordinary logs; they are rolled if log rolling is enabled. (Log - collation is not supported though). + like ordinary logs; they are rolled if log rolling is enabled. @param filename new log file being created. The new log file is created in the logs directory. You can specify a path to a @@ -2390,7 +2397,7 @@ tsapi TSReturnCode TSHttpTxnMilestoneGet(TSHttpTxn txnp, TSMilestonesType milest tsapi int TSHttpTxnIsCacheable(TSHttpTxn txnp, TSMBuffer request, TSMBuffer response); /** - Return a string respresentation for a TSServerState value. This is useful for plugin debugging. + Return a string representation for a TSServerState value. This is useful for plugin debugging. @param state the value of this TSServerState @@ -2399,7 +2406,7 @@ tsapi int TSHttpTxnIsCacheable(TSHttpTxn txnp, TSMBuffer request, TSMBuffer resp tsapi const char *TSHttpServerStateNameLookup(TSServerState state); /** - Return a string respresentation for a TSHttpHookID value. This is useful for plugin debugging. + Return a string representation for a TSHttpHookID value. This is useful for plugin debugging. @param hook the value of this TSHttpHookID @@ -2408,7 +2415,7 @@ tsapi const char *TSHttpServerStateNameLookup(TSServerState state); tsapi const char *TSHttpHookNameLookup(TSHttpHookID hook); /** - Return a string respresentation for a TSEvent value. This is useful for plugin debugging. + Return a string representation for a TSEvent value. This is useful for plugin debugging. @param event the value of this TSHttpHookID diff --git a/include/tscore/AcidPtr.h b/include/tscore/AcidPtr.h index 6656705b916..bc4fc6fdab2 100644 --- a/include/tscore/AcidPtr.h +++ b/include/tscore/AcidPtr.h @@ -35,7 +35,7 @@ ////////////////////////////////////////////////////////// // Lock Pool /// Intended to make datasets thread safe by assigning locks to stripes of data, kind of like a bloom filter. -/** Allocates a fixed number of locks and retrives one with a hash. +/** Allocates a fixed number of locks and retrieves one with a hash. */ template struct LockPool { /** @@ -152,7 +152,7 @@ template class AcidPtr /////////////////////////////////////////// /// AcidCommitPtr -/// a globally exclusive pointer, for commiting changes to AcidPtr. +/// a globally exclusive pointer, for committing changes to AcidPtr. /** used for COPY_SWAP functionality. * 1. copy data (construct) * 2. overwrite data (scope) diff --git a/include/tscore/Allocator.h b/include/tscore/Allocator.h index 4ed9094e449..c03f996ab84 100644 --- a/include/tscore/Allocator.h +++ b/include/tscore/Allocator.h @@ -23,7 +23,7 @@ Provides three classes - Allocator for allocating memory blocks of fixed size - ClassAllocator for allocating objects - - SpaceClassAllocator for allocating sparce objects (most members uninitialized) + - SpaceClassAllocator for allocating sparse objects (most members uninitialized) These class provides a efficient way for handling dynamic allocation. The fast allocator maintains its own freepool of objects from @@ -48,8 +48,6 @@ #define RND16(_x) (((_x) + 15) & ~15) -extern int cmd_disable_pfreelist; - /** Allocator for fixed size memory blocks. */ class Allocator { @@ -61,7 +59,7 @@ class Allocator void * alloc_void() { - return ink_freelist_new(this->fl, freelist_class_ops); + return ink_freelist_new(this->fl); } /** @@ -72,7 +70,7 @@ class Allocator void free_void(void *ptr) { - ink_freelist_free(this->fl, ptr, freelist_class_ops); + ink_freelist_free(this->fl, ptr); } /** @@ -85,7 +83,7 @@ class Allocator void free_void_bulk(void *head, void *tail, size_t num_item) { - ink_freelist_free_bulk(this->fl, head, tail, num_item, freelist_class_ops); + ink_freelist_free_bulk(this->fl, head, tail, num_item); } Allocator() { fl = nullptr; } @@ -130,7 +128,7 @@ template class ClassAllocator : public Allocator C * alloc() { - void *ptr = ink_freelist_new(this->fl, freelist_class_ops); + void *ptr = ink_freelist_new(this->fl); memcpy(ptr, (void *)&this->proto.typeObject, sizeof(C)); return (C *)ptr; @@ -144,7 +142,7 @@ template class ClassAllocator : public Allocator void free(C *ptr) { - ink_freelist_free(this->fl, ptr, freelist_class_ops); + ink_freelist_free(this->fl, ptr); } /** @@ -157,7 +155,7 @@ template class ClassAllocator : public Allocator void free_bulk(C *head, C *tail, size_t num_item) { - ink_freelist_free_bulk(this->fl, head, tail, num_item, freelist_class_ops); + ink_freelist_free_bulk(this->fl, head, tail, num_item); } /** diff --git a/include/tscore/Arena.h b/include/tscore/Arena.h index c885c980212..6d359bd7af8 100644 --- a/include/tscore/Arena.h +++ b/include/tscore/Arena.h @@ -37,7 +37,7 @@ struct ArenaBlock { class Arena { public: - Arena() : m_blocks(nullptr) {} + Arena() {} ~Arena() { reset(); } inkcoreapi void *alloc(size_t size, size_t alignment = sizeof(double)); void free(void *mem, size_t size); @@ -49,7 +49,7 @@ class Arena inkcoreapi void reset(); private: - ArenaBlock *m_blocks; + ArenaBlock *m_blocks = nullptr; }; /*------------------------------------------------------------------------- diff --git a/include/tscore/ArgParser.h b/include/tscore/ArgParser.h index 927882d2c2d..45b0e416252 100644 --- a/include/tscore/ArgParser.h +++ b/include/tscore/ArgParser.h @@ -175,7 +175,8 @@ class ArgParser bool parse(Arguments &ret, AP_StrVec &args); // The help & version messages void help_message(std::string_view err = "") const; - // Helpr method for parse() + void version_message() const; + // Helper method for parse() void append_option_data(Arguments &ret, AP_StrVec &args, int index); // The command name and help message std::string _name; @@ -226,7 +227,7 @@ class ArgParser std::string const &key = ""); Command &add_command(std::string const &cmd_name, std::string const &cmd_description, std::string const &cmd_envvar, unsigned cmd_arg_num, Function const &f = nullptr, std::string const &key = ""); - // give a defaut command to this parser + // give a default command to this parser void set_default_command(std::string const &cmd); /** Main parsing function @return The Arguments object available for program using diff --git a/include/tscore/AtomicBit.h b/include/tscore/AtomicBit.h new file mode 100644 index 00000000000..72581d3aad3 --- /dev/null +++ b/include/tscore/AtomicBit.h @@ -0,0 +1,92 @@ +/** + @file AtomicBit + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + @section details Details + +//////////////////////////////////////////// + Implements class AtomicBit +*/ + +#pragma once +#include +#include + +////////////////////////////////////////////////////// +/// AtomicBit for inplace atomic bit operations +/* useful when you reference a bit packed into a byte (unit_8) as a bool&, + * you want a bit to 'walk and talk' like an std::atomic or std::atomic_flag. + * In practice this is constructed at time of the operation(s), + * storing it would defeat the purpose of packing the bits. + */ +class AtomicBit +{ + std::atomic *_byte_ptr; ///< pointer to the byte + uint8_t const _mask; ///< bitmask of which bit you are using + +public: + // define a bit to perform atomic operations + AtomicBit(std::atomic &byte, const uint8_t mask) : _byte_ptr(&byte), _mask(mask) {} + AtomicBit(uint8_t *byte_ptr, const uint8_t mask) : _byte_ptr(reinterpret_cast *>(byte_ptr)), _mask(mask) {} + + // Atomically set the bit true + // return @c true if the bit was changed, @c false if not. + bool + test_and_set() + { + return compare_exchange(true); + } + + // allows assign by bool + // @return The new value of the bit. + bool + operator=(bool val) + { + compare_exchange(val); + return val; + } + + // allow cast to bool + explicit operator const bool() const { return (*_byte_ptr) & _mask; } + + // allows compare with bool + bool + operator==(bool rhs) const + { + return bool(*this) == rhs; + } + + // Atomically set the bit to `val` + // return @c true if the bit was changed, @c false if not. + bool + compare_exchange(bool val) + { + while (true) { + uint8_t byte_val = *_byte_ptr; + const uint8_t next_byte_val = val ? (byte_val | _mask) : (byte_val & ~_mask); + if (byte_val == next_byte_val) { + return false; + } + if (_byte_ptr->compare_exchange_weak(byte_val, next_byte_val)) { + return true; + } + } + } +}; diff --git a/include/tscore/BufferWriter.h b/include/tscore/BufferWriter.h index 052a4de29cf..42610fa0159 100644 --- a/include/tscore/BufferWriter.h +++ b/include/tscore/BufferWriter.h @@ -105,7 +105,7 @@ class BufferWriter invalidate the current auxiliary buffer (contents and address). Care must be taken to not write to data beyond this plus @c remaining bytes. Usually the - safest mechanism is to create a @c FixedBufferWriter on the auxillary buffer and write to that. + safest mechanism is to create a @c FixedBufferWriter on the auxiliary buffer and write to that. @code ts::FixedBufferWriter subw(w.auxBuffer(), w.remaining()); @@ -124,7 +124,7 @@ class BufferWriter /** Advance the buffer position @a n bytes. This treats the next @a n bytes as being written without changing the content. This is useful - only in conjuction with @a auxBuffer to indicate that @a n bytes of the auxillary buffer has + only in conjunction with @a auxBuffer to indicate that @a n bytes of the auxiliary buffer has been written by some other mechanism. @internal Concrete subclasses @b must override this to advance in a way consistent with the @@ -155,7 +155,7 @@ class BufferWriter } /// Get the remaining buffer space. - /// @return Number of additional characters that can be written without causing an error condidtion. + /// @return Number of additional characters that can be written without causing an error condition. size_t remaining() const { @@ -178,12 +178,12 @@ class BufferWriter /** BufferWriter print. This prints its arguments to the @c BufferWriter @a w according to the format @a fmt. The format - string is based on Python style formating, each argument substitution marked by braces, {}. Each - specification has three parts, a @a name, a @a specifier, and an @a extention. These are + string is based on Python style formatting, each argument substitution marked by braces, {}. Each + specification has three parts, a @a name, a @a specifier, and an @a extension. These are separated by colons. The name should be either omitted or a number, the index of the argument to use. If omitted the place in the format string is used as the argument index. E.g. "{} {} {}", "{} {1} {}", and "{0} {1} {2}" are equivalent. Using an explicit index does not reset the - position of subsequent substiations, therefore "{} {0} {}" is equivalent to "{0} {0} {2}". + position of subsequent substitutions, therefore "{} {0} {}" is equivalent to "{0} {0} {2}". */ template BufferWriter &print(TextView fmt, Rest &&... rest); /** Print overload to take arguments as a tuple instead of explicitly. @@ -207,6 +207,9 @@ class BufferWriter }; /** A @c BufferWrite concrete subclass to write to a fixed size buffer. + * + * Copies and moves are forbidden because that leaves the original in a potentially bad state. An + * instance is cheap to construct and should be done explicitly when needed. */ class FixedBufferWriter : public BufferWriter { @@ -231,10 +234,8 @@ class FixedBufferWriter : public BufferWriter FixedBufferWriter(const FixedBufferWriter &) = delete; FixedBufferWriter &operator=(const FixedBufferWriter &) = delete; - /// Move constructor. - FixedBufferWriter(FixedBufferWriter &&) = default; - /// Move assignment. - FixedBufferWriter &operator=(FixedBufferWriter &&) = default; + FixedBufferWriter(FixedBufferWriter &&) = delete; + FixedBufferWriter &operator=(FixedBufferWriter &&) = delete; FixedBufferWriter(MemSpan &span) : _buf(span.begin()), _capacity(static_cast(span.size())) {} @@ -377,9 +378,9 @@ class FixedBufferWriter : public BufferWriter /** Get a @c FixedBufferWriter for the unused output buffer. - If @a reserve is non-zero then the buffer size for the auxillary writer will be @a reserve bytes + If @a reserve is non-zero then the buffer size for the auxiliary writer will be @a reserve bytes smaller than the remaining buffer. This "reserves" space for additional output after writing - to the auxillary buffer, in a manner similar to @c clip / @c extend. + to the auxiliary buffer, in a manner similar to @c clip / @c extend. */ FixedBufferWriter auxWriter(size_t reserve = 0) @@ -525,8 +526,8 @@ namespace bw_fmt } /// This exists only to expand the index sequence into an array of formatters for the tuple type - /// @a TUPLE. Due to langauge limitations it cannot be done directly. The formatters can be - /// accessed via standard array access in constrast to templated tuple access. The actual array is + /// @a TUPLE. Due to language limitations it cannot be done directly. The formatters can be + /// accessed via standard array access in contrast to templated tuple access. The actual array is /// static and therefore at run time the only operation is loading the address of the array. template ArgFormatterSignature * @@ -624,7 +625,7 @@ BufferWriter::printv(TextView fmt, std::tuple const &args) int arg_idx = 0; // the next argument index to be processed. while (fmt.size()) { - // Next string piece of interest is an (optional) literal and then an (optinal) format specifier. + // Next string piece of interest is an (optional) literal and then an (optional) format specifier. // There will always be a specifier except for the possible trailing literal. std::string_view lit_v; std::string_view spec_v; @@ -704,12 +705,26 @@ BufferWriter::printv(BWFormat const &fmt, std::tuple const &args) return *this; } +// Must be first so that other inline formatters can use it. +BufferWriter &bwformat(BufferWriter &w, BWFSpec const &spec, std::string_view sv); + // Pointers that are not specialized. inline BufferWriter & bwformat(BufferWriter &w, BWFSpec const &spec, const void *ptr) { BWFSpec ptr_spec{spec}; ptr_spec._radix_lead_p = true; + + if (ptr == nullptr) { + if (spec._type == 's' || spec._type == 'S') { + ptr_spec._type = BWFSpec::DEFAULT_TYPE; + ptr_spec._ext = ""_sv; // clear any extension. + return bwformat(w, spec, spec._type == 's' ? "null"_sv : "NULL"_sv); + } else if (spec._type == BWFSpec::DEFAULT_TYPE) { + return w; // print nothing if there is no format character override. + } + } + if (ptr_spec._type == BWFSpec::DEFAULT_TYPE || ptr_spec._type == 'p') { ptr_spec._type = 'x'; // if default or 'p;, switch to lower hex. } else if (ptr_spec._type == 'P') { @@ -723,7 +738,12 @@ BufferWriter &bwformat(BufferWriter &w, BWFSpec const &spec, MemSpan const &span // -- Common formatters -- -BufferWriter &bwformat(BufferWriter &w, BWFSpec const &spec, std::string_view sv); +// Capture this explicitly so it doesn't go to any other pointer type. +inline BufferWriter & +bwformat(BufferWriter &w, BWFSpec const &spec, std::nullptr_t) +{ + return bwformat(w, spec, static_cast(nullptr)); +} template BufferWriter & @@ -735,10 +755,12 @@ bwformat(BufferWriter &w, BWFSpec const &spec, const char (&a)[N]) inline BufferWriter & bwformat(BufferWriter &w, BWFSpec const &spec, const char *v) { - if (spec._type == 'x' || spec._type == 'X') { + if (spec._type == 'x' || spec._type == 'X' || spec._type == 'p') { bwformat(w, spec, static_cast(v)); - } else { + } else if (v != nullptr) { bwformat(w, spec, std::string_view(v)); + } else { + bwformat(w, spec, nullptr); } return w; } @@ -885,6 +907,51 @@ FixedBufferWriter::printv(BWFormat const &fmt, std::tuple const &args) return static_cast(this->super_type::printv(fmt, args)); } +// Basic format wrappers - these are here because they're used internally. +namespace bwf +{ + namespace detail + { + /** Write out raw memory in hexadecimal wrapper. + * + * This wrapper indicates the contained view should be dumped as raw memory in hexadecimal format. + * This is intended primarily for internal use by other formatting logic. + * + * @see Hex_Dump + */ + struct MemDump { + std::string_view _view; + + /** Dump @a n bytes starting at @a mem as hex. + * + * @param mem First byte of memory to dump. + * @param n Number of bytes. + */ + MemDump(void const *mem, size_t n) : _view(static_cast(mem), n) {} + }; + } // namespace detail + + /** Treat @a t as raw memory and dump the memory as hexadecimal. + * + * @tparam T Type of argument. + * @param t Object to dump. + * @return @a A wrapper to do a hex dump. + * + * This is the standard way to do a hexadecimal memory dump of an object. + * + * @internal This function exists so that other types can overload it for special processing, + * which would not be possible with just @c HexDump. + */ + template + detail::MemDump + Hex_Dump(T const &t) + { + return {&t, sizeof(T)}; + } +} // namespace bwf + +BufferWriter &bwformat(BufferWriter &w, BWFSpec const &spec, bwf::detail::MemDump const &hex); + } // end namespace ts namespace std diff --git a/include/tscore/BufferWriterForward.h b/include/tscore/BufferWriterForward.h index 9bc874c57df..50faa9ecdc1 100644 --- a/include/tscore/BufferWriterForward.h +++ b/include/tscore/BufferWriterForward.h @@ -60,7 +60,7 @@ struct BWFSpec { // @a _min is unsigned because there's no point in an invalid default, 0 works fine. unsigned int _min = 0; ///< Minimum width. int _prec = -1; ///< Precision - unsigned int _max = std::numeric_limits::max(); ///< Maxium width + unsigned int _max = std::numeric_limits::max(); ///< Maximum width int _idx = -1; ///< Positional "name" of the specification. std::string_view _name; ///< Name of the specification. std::string_view _ext; ///< Extension if provided. diff --git a/include/tscore/CryptoHash.h b/include/tscore/CryptoHash.h index c9e1835f264..1cb7848ca30 100644 --- a/include/tscore/CryptoHash.h +++ b/include/tscore/CryptoHash.h @@ -28,10 +28,8 @@ /// Apache Traffic Server commons. #if TS_ENABLE_FIPS == 1 -// #include "tscore/SHA256.h" #define CRYPTO_HASH_SIZE (256 / 8) #else -// #include "tscore/ink_code.h" #define CRYPTO_HASH_SIZE (128 / 8) #endif #define CRYPTO_HEX_SIZE ((CRYPTO_HASH_SIZE * 2) + 1) @@ -126,7 +124,7 @@ class CryptoContextBase /// Convenience - compute final @a hash for @a data. /// @note This is just as fast as the previous style, as a new context must be initialized - /// everytime this is done. + /// every time this is done. bool hash_immediate(CryptoHash &hash, void const *data, int length); }; diff --git a/include/tscore/Diags.h b/include/tscore/Diags.h index f68cce87ef4..f9e8a2d5881 100644 --- a/include/tscore/Diags.h +++ b/include/tscore/Diags.h @@ -113,7 +113,7 @@ class Diags public: Diags(const char *prefix_string, const char *base_debug_tags, const char *base_action_tags, BaseLogFile *_diags_log, int diags_log_perm = -1, int output_log_perm = -1); - ~Diags(); + virtual ~Diags(); BaseLogFile *diags_log; BaseLogFile *stdout_log; @@ -208,7 +208,7 @@ class Diags va_end(ap); } - void error_va(DiagsLevel level, const SourceLocation *loc, const char *fmt, va_list ap) const; + virtual void error_va(DiagsLevel level, const SourceLocation *loc, const char *fmt, va_list ap) const; void dump(FILE *fp = stdout) const; diff --git a/include/tscore/DynArray.h b/include/tscore/DynArray.h deleted file mode 100644 index 49c0886e1b7..00000000000 --- a/include/tscore/DynArray.h +++ /dev/null @@ -1,191 +0,0 @@ -/** @file - - Dynamic Array Implementation used by Regex.cc - - @section license License - - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - */ - -#pragma once - -template class DynArray -{ -public: - DynArray(const T *val = 0, intptr_t initial_size = 0); - ~DynArray(); - -#ifndef __GNUC__ - operator const T *() const; -#endif - operator T *(); - T &operator[](intptr_t idx); - T &operator()(intptr_t idx); - T *detach(); - T defvalue() const; - intptr_t length(); - void clear(); - void set_length(intptr_t i); - -private: - void resize(intptr_t new_size); - -private: - T *data; - const T *default_val; - int size; - int pos; -}; - -template -inline DynArray::DynArray(const T *val, intptr_t initial_size) : data(nullptr), default_val(val), size(0), pos(-1) -{ - if (initial_size > 0) { - int i = 1; - - while (i < initial_size) { - i <<= 1; - } - - resize(i); - } -} - -template inline DynArray::~DynArray() -{ - if (data) { - delete[] data; - } -} - -#ifndef __GNUC__ -template inline DynArray::operator const T *() const -{ - return data; -} -#endif - -template inline DynArray::operator T *() -{ - return data; -} - -template inline T &DynArray::operator[](intptr_t idx) -{ - return data[idx]; -} - -template -inline T & -DynArray::operator()(intptr_t idx) -{ - if (idx >= size) { - intptr_t new_size; - - if (size == 0) { - new_size = 64; - } else { - new_size = size * 2; - } - - if (idx >= new_size) { - new_size = idx + 1; - } - - resize(new_size); - } - - if (idx > pos) { - pos = idx; - } - - return data[idx]; -} - -template -inline T * -DynArray::detach() -{ - T *d; - - d = data; - data = nullptr; - - return d; -} - -template -inline T -DynArray::defvalue() const -{ - return *default_val; -} - -template -inline intptr_t -DynArray::length() -{ - return pos + 1; -} - -template -inline void -DynArray::clear() -{ - if (data) { - delete[] data; - data = nullptr; - } - - size = 0; - pos = -1; -} - -template -inline void -DynArray::set_length(intptr_t i) -{ - pos = i - 1; -} - -template -inline void -DynArray::resize(intptr_t new_size) -{ - if (new_size > size) { - T *new_data; - intptr_t i; - - new_data = new T[new_size]; - - for (i = 0; i < size; i++) { - new_data[i] = data[i]; - } - - for (; i < new_size; i++) { - if (default_val) { - new_data[i] = (T)*default_val; - } - } - - if (data) { - delete[] data; - } - data = new_data; - size = new_size; - } -} diff --git a/include/tscore/EventNotify.h b/include/tscore/EventNotify.h index 2ac068923de..864c8093254 100644 --- a/include/tscore/EventNotify.h +++ b/include/tscore/EventNotify.h @@ -36,12 +36,12 @@ class EventNotify { public: EventNotify(); - void signal(void); - int wait(void); + void signal(); + int wait(); int timedwait(int timeout); // milliseconds - void lock(void); - bool trylock(void); - void unlock(void); + void lock(); + bool trylock(); + void unlock(); ~EventNotify(); private: diff --git a/include/tscore/Extendible.h b/include/tscore/Extendible.h index 16c9c874493..add2f915845 100644 --- a/include/tscore/Extendible.h +++ b/include/tscore/Extendible.h @@ -30,7 +30,7 @@ */ #pragma once -#include "stdint.h" +#include #include #include #include @@ -60,7 +60,7 @@ enum AccessEnum { ATOMIC, BIT, STATIC, ACIDPTR, DIRECT, C_API, NUM_ACCESS_TYPES inline bool & areStaticsFrozen() { - static bool frozen = 0; + static bool frozen = false; return frozen; } @@ -68,7 +68,7 @@ areStaticsFrozen() * @brief Allows code (and Plugins) to declare member variables during system init. * * The size of this structure is actually zero, so it will not change the size of your derived class. - * But new and delete are overriden to use allocate enough bytes of the derived type + added fields. + * But new and delete are overridden to use allocate enough bytes of the derived type + added fields. * All bool's are packed to save space using the *Bit methods. * This API is focused on thread safe data types that allow minimally blocked reading. * This is templated so static variables are instanced per Derived type. B/c we need to have different @@ -105,6 +105,8 @@ template struct Extendible { Extendible(Extendible &) = delete; /** allocate a new object with additional field data */ void *operator new(size_t size); + /** free the object */ + void operator delete(void *ptr); /** construct all fields */ Extendible() { schema.call_construct(this_as_char_ptr()); } /** destruct all fields */ @@ -194,7 +196,7 @@ template struct Extendible { }; // end Schema struct private: - // Extendible convience methods + // Extendible convenience methods char *this_as_char_ptr(); char const *this_as_char_ptr() const; @@ -390,7 +392,7 @@ void Extendible::Schema::call_construct(char *ext_as_char_ptr) { ++instance_count; // don't allow schema modification - // init all extendible memory to 0, incase constructors don't + // init all extendible memory to 0, in case constructors don't memset(ext_as_char_ptr + sizeof(Derived_t), 0, alloc_size - sizeof(Derived_t)); for (auto const &elm : fields) { @@ -550,6 +552,15 @@ Extendible::operator new(size_t size) return ptr; } +/// free the object +template +void +Extendible::operator delete(void *ptr) +{ + ats_free(ptr); + ink_release_assert(ptr != nullptr); +} + // private template char * diff --git a/include/tscore/Hash.h b/include/tscore/Hash.h index 68879a93951..08e438095a6 100644 --- a/include/tscore/Hash.h +++ b/include/tscore/Hash.h @@ -27,8 +27,8 @@ struct ATSHashBase { virtual void update(const void *, size_t) = 0; - virtual void final(void) = 0; - virtual void clear(void) = 0; + virtual void final() = 0; + virtual void clear() = 0; virtual ~ATSHashBase(); }; @@ -49,17 +49,17 @@ struct ATSHash : ATSHashBase { } }; - virtual const void *get(void) const = 0; - virtual size_t size(void) const = 0; + virtual const void *get() const = 0; + virtual size_t size() const = 0; virtual bool operator==(const ATSHash &) const; }; struct ATSHash32 : ATSHashBase { - virtual uint32_t get(void) const = 0; + virtual uint32_t get() const = 0; virtual bool operator==(const ATSHash32 &) const; }; struct ATSHash64 : ATSHashBase { - virtual uint64_t get(void) const = 0; + virtual uint64_t get() const = 0; virtual bool operator==(const ATSHash64 &) const; }; diff --git a/include/tscore/HashFNV.h b/include/tscore/HashFNV.h index e113d0cd012..ff92edcd7a0 100644 --- a/include/tscore/HashFNV.h +++ b/include/tscore/HashFNV.h @@ -31,7 +31,7 @@ #include struct ATSHash32FNV1a : ATSHash32 { - ATSHash32FNV1a(void); + ATSHash32FNV1a(); template void update(const void *data, size_t len, Transform xfrm); void @@ -40,9 +40,9 @@ struct ATSHash32FNV1a : ATSHash32 { update(data, len, ATSHash::nullxfrm()); } - void final(void) override; - uint32_t get(void) const override; - void clear(void) override; + void final() override; + uint32_t get() const override; + void clear() override; private: uint32_t hval; @@ -62,7 +62,7 @@ ATSHash32FNV1a::update(const void *data, size_t len, Transform xfrm) } struct ATSHash64FNV1a : ATSHash64 { - ATSHash64FNV1a(void); + ATSHash64FNV1a(); template void update(const void *data, size_t len, Transform xfrm); void @@ -71,9 +71,9 @@ struct ATSHash64FNV1a : ATSHash64 { update(data, len, ATSHash::nullxfrm()); } - void final(void) override; - uint64_t get(void) const override; - void clear(void) override; + void final() override; + uint64_t get() const override; + void clear() override; private: uint64_t hval; diff --git a/include/tscore/HashMD5.h b/include/tscore/HashMD5.h index bad5a90cc80..09c7bd67c63 100644 --- a/include/tscore/HashMD5.h +++ b/include/tscore/HashMD5.h @@ -25,17 +25,17 @@ #include struct ATSHashMD5 : ATSHash { - ATSHashMD5(void); + ATSHashMD5(); void update(const void *data, size_t len) override; - void final(void) override; - const void *get(void) const override; - size_t size(void) const override; - void clear(void) override; + void final() override; + const void *get() const override; + size_t size() const override; + void clear() override; ~ATSHashMD5() override; private: EVP_MD_CTX *ctx; unsigned char md_value[EVP_MAX_MD_SIZE]; - unsigned int md_len; - bool finalized; + unsigned int md_len = 0; + bool finalized = false; }; diff --git a/include/tscore/HashSip.h b/include/tscore/HashSip.h index 837289848dc..30d195b49f3 100644 --- a/include/tscore/HashSip.h +++ b/include/tscore/HashSip.h @@ -32,13 +32,13 @@ */ struct ATSHash64Sip24 : ATSHash64 { - ATSHash64Sip24(void); + ATSHash64Sip24(); ATSHash64Sip24(const unsigned char key[16]); ATSHash64Sip24(std::uint64_t key0, std::uint64_t key1); void update(const void *data, std::size_t len) override; - void final(void) override; - std::uint64_t get(void) const override; - void clear(void) override; + void final() override; + std::uint64_t get() const override; + void clear() override; private: unsigned char block_buffer[8] = {0}; diff --git a/include/tscore/HostLookup.h b/include/tscore/HostLookup.h index 58a0ec344dc..1d5bdb7302e 100644 --- a/include/tscore/HostLookup.h +++ b/include/tscore/HostLookup.h @@ -36,7 +36,7 @@ #include #include -// HostLookup constantss +// HostLookup constants constexpr int HOST_TABLE_DEPTH = 3; // Controls the max number of levels in the logical tree constexpr int HOST_ARRAY_MAX = 8; // Sets the fixed array size diff --git a/include/tscore/I_Layout.h b/include/tscore/I_Layout.h index 18aa1415463..eb4d2609117 100644 --- a/include/tscore/I_Layout.h +++ b/include/tscore/I_Layout.h @@ -65,7 +65,7 @@ struct Layout { /** Return file path relative to dir - Store the path to buf. The buf should be large eough to store + Store the path to buf. The buf should be large enough to store Example usage: Layout::relative_to(default_layout()->sysconfdir, "foo.bar"); */ static void relative_to(char *buf, size_t bufsz, std::string_view dir, std::string_view file); diff --git a/include/tscore/I_Version.h b/include/tscore/I_Version.h index 7aa616d4b3b..7f89f9cfc9a 100644 --- a/include/tscore/I_Version.h +++ b/include/tscore/I_Version.h @@ -30,18 +30,40 @@ #pragma once +namespace ts +{ +/** Container for standard two part version number. + */ struct VersionNumber { - short int ink_major; // incompatible change - short int ink_minor; // minor change, not incompatible + /// Construct invalid (0.0) version. + constexpr VersionNumber() = default; - VersionNumber() : ink_major(0), ink_minor(0) {} - VersionNumber(short int major, short int minor) : ink_major(major), ink_minor(minor) {} + /// Construct explicit version. + constexpr explicit VersionNumber(unsigned short major, unsigned short minor = 0); + + // Can't use unadorned "major" because that's a macro. + unsigned short _major = 0; ///< Major version. + unsigned short _minor = 0; ///< Minor version. }; +inline constexpr VersionNumber::VersionNumber(unsigned short major, unsigned short minor) : _major(major), _minor(minor) {} + inline bool operator<(VersionNumber const &lhs, VersionNumber const &rhs) { - return lhs.ink_major < rhs.ink_major || (lhs.ink_major == rhs.ink_major && lhs.ink_minor < rhs.ink_minor); + return lhs._major < rhs._major || (lhs._major == rhs._major && lhs._minor < rhs._minor); +} + +inline bool +operator==(VersionNumber const &lhs, VersionNumber const &rhs) +{ + return lhs._major == rhs._major && lhs._minor == rhs._minor; +} + +inline bool +operator!=(VersionNumber const &lhs, VersionNumber const &rhs) +{ + return !(lhs == rhs); } inline bool @@ -51,9 +73,15 @@ operator>(VersionNumber const &lhs, VersionNumber const &rhs) } inline bool -operator==(VersionNumber const &lhs, VersionNumber const &rhs) +operator<=(VersionNumber const &lhs, VersionNumber const &rhs) +{ + return !(lhs > rhs); +} + +inline bool +operator>=(VersionNumber const &lhs, VersionNumber const &rhs) { - return lhs.ink_major == rhs.ink_major && lhs.ink_minor == rhs.ink_minor; + return !(rhs > lhs); } struct Version { @@ -61,39 +89,50 @@ struct Version { VersionNumber cacheDir; }; -enum ModuleVersion { - MODULE_VERSION_MIN = 0, - MODULE_VERSION_MAX = 2147483647, -}; -enum ModuleHeaderType { - PUBLIC_MODULE_HEADER, - PRIVATE_MODULE_HEADER, -}; +/// Container for a module version. +struct ModuleVersion { + /// Type of module. + enum Type : unsigned char { PUBLIC, PRIVATE }; + + constexpr ModuleVersion(unsigned char major, unsigned char minor, Type type = PUBLIC); + constexpr ModuleVersion(ModuleVersion const &base, Type type); -#define makeModuleVersion(_major_version, _minor_version, _module_type) \ - ((ModuleVersion)((((int)_module_type) << 24) + (((int)_major_version) << 16) + (((int)_minor_version) << 8))) + /** Check if @a that is a version compatible with @a this. + * + * @param that Version to check against. + * @return @a true if @a that is compatible with @a this, @c false otherwise. + */ + bool check(ModuleVersion const &that); -#define majorModuleVersion(_v) ((((int)_v) >> 16) & 255) -#define minorModuleVersion(_v) ((((int)_v) >> 8) & 255) -#define moduleVersionType(_v) ((((int)_v) >> 24) & 127) + Type _type = PUBLIC; ///< The numeric value of the module version. + unsigned char _major = 0; ///< Major version. + unsigned char _minor = 0; +}; -static inline int -checkModuleVersion(ModuleVersion userVersion, ModuleVersion libVersion) +inline constexpr ModuleVersion::ModuleVersion(unsigned char major, unsigned char minor, ts::ModuleVersion::Type type) + : _type(type), _major(major), _minor(minor) { - if (moduleVersionType(userVersion) == PUBLIC_MODULE_HEADER) { - if ((majorModuleVersion(userVersion) != majorModuleVersion(libVersion)) || - (minorModuleVersion(userVersion) > minorModuleVersion(libVersion))) - return -1; - return 0; - } else if (moduleVersionType(userVersion) == PRIVATE_MODULE_HEADER) { - if ((majorModuleVersion(userVersion) != majorModuleVersion(libVersion)) || - (minorModuleVersion(userVersion) != minorModuleVersion(libVersion))) - return -1; - return 0; - } else - return -1; } +inline constexpr ModuleVersion::ModuleVersion(ModuleVersion const &base, ts::ModuleVersion::Type type) + : _type(type), _major(base._major), _minor(base._minor) +{ +} + +inline bool +ModuleVersion::check(ModuleVersion const &that) +{ + switch (_type) { + case PUBLIC: + return _major == that._major && _minor <= that._minor; + case PRIVATE: + return _major == that._major && _minor == that._minor; + } + return false; +}; + +} // namespace ts + class AppVersionInfo { public: diff --git a/include/tscore/IntrusiveHashMap.h b/include/tscore/IntrusiveHashMap.h index c787e4b358b..704b964e1e6 100644 --- a/include/tscore/IntrusiveHashMap.h +++ b/include/tscore/IntrusiveHashMap.h @@ -298,7 +298,7 @@ template class IntrusiveHashMap Bucket *bucket_for(key_type key); - ExpansionPolicy _expansion_policy{DEFAULT_EXPANSION_POLICY}; ///< When to exand the table. + ExpansionPolicy _expansion_policy{DEFAULT_EXPANSION_POLICY}; ///< When to expand the table. size_t _expansion_limit{DEFAULT_EXPANSION_LIMIT}; ///< Limit value for expansion. // noncopyable @@ -343,6 +343,9 @@ IntrusiveHashMap::Bucket::clear() _v = nullptr; _count = 0; _mixed_p = false; + // These can be left set during an expansion, when the bucket did have elements before but not + // after. Therefore make sure they are cleared. + _link._next = _link._prev = nullptr; } template diff --git a/include/tscore/IpMap.h b/include/tscore/IpMap.h index 34ec5dc3175..50a9fbe2434 100644 --- a/include/tscore/IpMap.h +++ b/include/tscore/IpMap.h @@ -23,12 +23,16 @@ #pragma once +#include +#include + #include "tscore/ink_platform.h" #include "tscore/ink_defs.h" #include "tscore/RbTree.h" #include "tscore/ink_inet.h" #include "tscpp/util/IntrusiveDList.h" #include "tscore/ink_assert.h" +#include "tscore/BufferWriterForward.h" namespace ts { @@ -42,8 +46,8 @@ namespace detail typename A = T const & ///< Argument type. > struct Interval { - typedef T Metric; ///< Metric (storage) type. - typedef A ArgType; ///< Type used to pass instances of @c Metric. + using Metric = T; ///< Metric (storage) type. + using ArgType = A; ///< Type used to pass instances of @c Metric. Interval() {} ///< Default constructor. /// Construct with values. @@ -71,7 +75,7 @@ namespace detail data. Marking takes a painter's algorithm approach -- any marking overwrites any previous marking on an address. Details of marking calls are discarded and only the final results are kept. That is, - a client cannot unmark expliticly any previous marking. Only a + a client cannot unmark explicitly any previous marking. Only a specific range of addresses can be unmarked. Both IPv4 and IPv6 are supported in the same map. Mixed ranges are @@ -95,10 +99,20 @@ namespace detail class IpMap { public: - typedef IpMap self; ///< Self reference type. + using self_type = IpMap; ///< Self reference type. class iterator; // forward declare. + static constexpr in_addr_t RAW_IP4_MIN_ADDR = 0; + static constexpr IpAddr IP4_MIN_ADDR{RAW_IP4_MIN_ADDR}; + static constexpr in_addr_t RAW_IP4_MAX_ADDR = ~0; + static constexpr IpAddr IP4_MAX_ADDR{RAW_IP4_MAX_ADDR}; + + static constexpr in6_addr RAW_IP6_MIN_ADDR = {{{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}}}; + static constexpr IpAddr IP6_MIN_ADDR{RAW_IP6_MIN_ADDR}; + static constexpr in6_addr RAW_IP6_MAX_ADDR = {{{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff}}}; + static constexpr IpAddr IP6_MAX_ADDR{RAW_IP6_MAX_ADDR}; + /** Public API for intervals in the map. */ class Node : protected ts::detail::RBNode @@ -107,9 +121,9 @@ class IpMap friend class IpMap; public: - typedef Node self; ///< Self reference type. + using self_type = Node; ///< Self reference type. /// Default constructor. - Node() : _data(nullptr) {} + Node() {} /// Construct with @a data. Node(void *data) : _data(data) {} /// @return Client data for the node. @@ -119,7 +133,7 @@ class IpMap return _data; } /// Set client data. - virtual self & + virtual self_type & setData(void *data ///< Client data pointer to store. ) { @@ -132,7 +146,7 @@ class IpMap virtual sockaddr const *max() const = 0; protected: - void *_data; ///< Client data. + void *_data = nullptr; ///< Client data. }; /** Iterator over nodes / intervals. @@ -146,30 +160,30 @@ class IpMap friend class IpMap; public: - typedef iterator self; ///< Self reference type. - typedef Node value_type; ///< Referenced type for iterator. - typedef int difference_type; ///< Distance type. - typedef Node *pointer; ///< Pointer to referent. - typedef Node &reference; ///< Reference to referent. - typedef std::bidirectional_iterator_tag iterator_category; + using self_type = iterator; ///< Self reference type. + using value_type = Node; ///< Referenced type for iterator. + using difference_type = int; ///< Distance type. + using pointer = Node *; ///< Pointer to referent. + using reference = Node &; ///< Reference to referent. + using iterator_category = std::bidirectional_iterator_tag; /// Default constructor. - iterator() : _tree(nullptr), _node(nullptr) {} + iterator() = default; reference operator*() const; //!< value operator pointer operator->() const; //!< dereference operator - self &operator++(); //!< next node (prefix) - self operator++(int); //!< next node (postfix) - self &operator--(); ///< previous node (prefix) - self operator--(int); ///< next node (postfix) + self_type &operator++(); //!< next node (prefix) + self_type operator++(int); //!< next node (postfix) + self_type &operator--(); ///< previous node (prefix) + self_type operator--(int); ///< next node (postfix) /** Equality. @return @c true if the iterators refer to the same node. */ - bool operator==(self const &that) const; + bool operator==(self_type const &that) const; /** Inequality. @return @c true if the iterators refer to different nodes. */ bool - operator!=(self const &that) const + operator!=(self_type const &that) const { return !(*this == that); } @@ -177,20 +191,25 @@ class IpMap private: /// Construct a valid iterator. iterator(IpMap const *tree, Node *node) : _tree(tree), _node(node) {} - IpMap const *_tree; ///< Container. - Node *_node; //!< Current node. + IpMap const *_tree = nullptr; ///< Container. + Node *_node = nullptr; //!< Current node. }; - IpMap(); ///< Default constructor. + IpMap() = default; ///< Default constructor. + IpMap(self_type const &that) = delete; + IpMap(self_type &&that) noexcept; ~IpMap(); ///< Destructor. + self_type &operator=(self_type const &that) = delete; + self_type &operator =(self_type &&that); + /** Mark a range. All addresses in the range [ @a min , @a max ] are marked with @a data. @return This object. */ - self &mark(sockaddr const *min, ///< Minimum value in range. - sockaddr const *max, ///< Maximum value in range. - void *data = nullptr ///< Client data payload. + self_type &mark(sockaddr const *min, ///< Minimum value in range. + sockaddr const *max, ///< Maximum value in range. + void *data = nullptr ///< Client data payload. ); /** Mark a range. @@ -198,9 +217,9 @@ class IpMap @note Convenience overload for IPv4 addresses. @return This object. */ - self &mark(in_addr_t min, ///< Minimum address (network order). - in_addr_t max, ///< Maximum address (network order). - void *data = nullptr ///< Client data. + self_type &mark(in_addr_t min, ///< Minimum address (network order). + in_addr_t max, ///< Maximum address (network order). + void *data = nullptr ///< Client data. ); /** Mark a range. @@ -208,9 +227,9 @@ class IpMap @note Convenience overload for IPv4 addresses. @return This object. */ - self &mark(IpAddr const &min, ///< Minimum address (network order). - IpAddr const &max, ///< Maximum address (network order). - void *data = nullptr ///< Client data. + self_type &mark(IpAddr const &min, ///< Minimum address (network order). + IpAddr const &max, ///< Maximum address (network order). + void *data = nullptr ///< Client data. ); /** Mark an IPv4 address @a addr with @a data. @@ -218,8 +237,8 @@ class IpMap @note Convenience overload for IPv4 addresses. @return This object. */ - self &mark(in_addr_t addr, ///< Address (network order). - void *data = nullptr ///< Client data. + self_type &mark(in_addr_t addr, ///< Address (network order). + void *data = nullptr ///< Client data. ); /** Mark a range. @@ -227,9 +246,9 @@ class IpMap @note Convenience overload. @return This object. */ - self &mark(IpEndpoint const *min, ///< Minimum address (network order). - IpEndpoint const *max, ///< Maximum address (network order). - void *data = nullptr ///< Client data. + self_type &mark(IpEndpoint const *min, ///< Minimum address (network order). + IpEndpoint const *max, ///< Maximum address (network order). + void *data = nullptr ///< Client data. ); /** Mark an address @a addr with @a data. @@ -237,8 +256,8 @@ class IpMap @note Convenience overload. @return This object. */ - self &mark(IpEndpoint const *addr, ///< Address (network order). - void *data = nullptr ///< Client data. + self_type &mark(IpEndpoint const *addr, ///< Address (network order). + void *data = nullptr ///< Client data. ); /** Unmark addresses. @@ -248,14 +267,16 @@ class IpMap @return This object. */ - self &unmark(sockaddr const *min, ///< Minimum value. - sockaddr const *max ///< Maximum value. + self_type &unmark(sockaddr const *min, ///< Minimum value. + sockaddr const *max ///< Maximum value. ); /// Unmark addresses (overload). - self &unmark(IpEndpoint const *min, IpEndpoint const *max); + self_type &unmark(IpAddr const &min, IpAddr const &max); + /// Unmark addresses (overload). + self_type &unmark(IpEndpoint const *min, IpEndpoint const *max); /// Unmark overload. - self &unmark(in_addr_t min, ///< Minimum of range to unmark. - in_addr_t max ///< Maximum of range to unmark. + self_type &unmark(in_addr_t min, ///< Minimum of range to unmark. + in_addr_t max ///< Maximum of range to unmark. ); /** Fill addresses. @@ -269,13 +290,13 @@ class IpMap @return This object. */ - self &fill(sockaddr const *min, sockaddr const *max, void *data = nullptr); + self_type &fill(sockaddr const *min, sockaddr const *max, void *data = nullptr); /// Fill addresses (overload). - self &fill(IpEndpoint const *min, IpEndpoint const *max, void *data = nullptr); + self_type &fill(IpEndpoint const *min, IpEndpoint const *max, void *data = nullptr); /// Fill addresses (overload). - self &fill(IpAddr const &min, IpAddr const &max, void *data = nullptr); + self_type &fill(IpAddr const &min, IpAddr const &max, void *data = nullptr); /// Fill addresses (overload). - self &fill(in_addr_t min, in_addr_t max, void *data = nullptr); + self_type &fill(in_addr_t min, in_addr_t max, void *data = nullptr); /** Test for membership. @@ -289,7 +310,7 @@ class IpMap /** Test for membership. - @note Covenience overload for IPv4. + @note Convenience overload for IPv4. @return @c true if the address is in the map, @c false if not. If the address is in the map and @a ptr is not @c nullptr, @c *ptr @@ -328,7 +349,7 @@ class IpMap @note This is much faster than @c unmark. @return This object. */ - self &clear(); + self_type &clear(); /// Iterator for first element. iterator begin() const; @@ -342,9 +363,13 @@ class IpMap */ void validate(); - /// Print all spans. - /// @return This map. - // self& print(); + /** Generate formatted output. + * + * @param w Destination of formatted output. + * @param spec Formatting specification. + * @return @a w + */ + ts::BufferWriter &describe(ts::BufferWriter &w, ts::BWFSpec const &spec) const; protected: /// Force the IPv4 map to exist. @@ -354,8 +379,8 @@ class IpMap /// @return The IPv6 map. ts::detail::Ip6Map *force6(); - ts::detail::Ip4Map *_m4; ///< Map of IPv4 addresses. - ts::detail::Ip6Map *_m6; ///< Map of IPv6 addresses. + ts::detail::Ip4Map *_m4 = nullptr; ///< Map of IPv4 addresses. + ts::detail::Ip6Map *_m6 = nullptr; ///< Map of IPv6 addresses. }; inline IpMap & @@ -391,6 +416,15 @@ IpMap::unmark(IpEndpoint const *min, IpEndpoint const *max) return this->unmark(&min->sa, &max->sa); } +inline IpMap & +IpMap::unmark(IpAddr const &min, IpAddr const &max) +{ + IpEndpoint x, y; + x.assign(min); + y.assign(max); + return this->unmark(&x.sa, &y.sa); +} + inline IpMap & IpMap::fill(IpEndpoint const *min, IpEndpoint const *max, void *data) { @@ -437,7 +471,7 @@ IpMap::iterator::operator++(int) inline IpMap::iterator IpMap::iterator::operator--(int) { - self tmp(*this); + self_type tmp(*this); --*this; return tmp; } @@ -458,4 +492,11 @@ inline IpMap::iterator::pointer IpMap::iterator::operator->() const return _node; } -inline IpMap::IpMap() : _m4(nullptr), _m6(nullptr) {} +namespace ts +{ +inline BufferWriter & +bwformat(BufferWriter &w, BWFSpec const &spec, IpMap const &map) +{ + return map.describe(w, spec); +} +} // namespace ts diff --git a/include/tscore/IpMapConf.h b/include/tscore/IpMapConf.h index 3e96bba6ba7..7d229adf15d 100644 --- a/include/tscore/IpMapConf.h +++ b/include/tscore/IpMapConf.h @@ -27,6 +27,8 @@ class IpMap; // declare in name only. +// Returns 0 if successful, error string otherwise +int read_addr(char *line, int n, int *i, sockaddr *addr, char *err); // Returns 0 if successful, error string otherwise char *Load_IpMap_From_File(IpMap *map, int fd, char const *key_str); // Returns 0 if successful, error string otherwise diff --git a/include/tscore/List.h b/include/tscore/List.h index cc6278165d9..25bfac5a498 100644 --- a/include/tscore/List.h +++ b/include/tscore/List.h @@ -506,8 +506,8 @@ template struct SortableQueue : publi // template struct CountQueue : public Queue { - int size; - inline CountQueue(void) : size(0) {} + int size = 0; + inline CountQueue() {} inline void push(C *e); inline C *pop(); inline void enqueue(C *e); diff --git a/include/tscore/MT_hashtable.h b/include/tscore/MT_hashtable.h index 2c24225e586..cb6a80d34ee 100644 --- a/include/tscore/MT_hashtable.h +++ b/include/tscore/MT_hashtable.h @@ -69,15 +69,15 @@ struct MT_ListEntry{ template class HashTableIteratorState { public: - HashTableIteratorState() : cur_buck(-1), ppcur(NULL) {} - int cur_buck; + HashTableIteratorState() : ppcur(NULL) {} + int cur_buck = -1; HashTableEntry **ppcur; }; template class IMTHashTable { public: - IMTHashTable(int size, bool (*gc_func)(data_t) = NULL, void (*pre_gc_func)(void) = nullptr) + IMTHashTable(int size, bool (*gc_func)(data_t) = NULL, void (*pre_gc_func)() = nullptr) { m_gc_func = gc_func; m_pre_gc_func = pre_gc_func; @@ -136,7 +136,7 @@ template class IMTHashTable data_t remove_entry(HashTableIteratorState *s); void - GC(void) + GC() { if (m_gc_func == NULL) { return; @@ -195,7 +195,7 @@ template class IMTHashTable int cur_size; int bucket_num; bool (*m_gc_func)(data_t); - void (*m_pre_gc_func)(void); + void (*m_pre_gc_func)(); private: IMTHashTable(); @@ -336,7 +336,7 @@ IMTHashTable::remove_entry(HashTableIteratorState template class MTHashTable { public: - MTHashTable(int size, bool (*gc_func)(data_t) = NULL, void (*pre_gc_func)(void) = nullptr) + MTHashTable(int size, bool (*gc_func)(data_t) = NULL, void (*pre_gc_func)() = nullptr) { for (int i = 0; i < MT_HASHTABLE_PARTITIONS; i++) { locks[i] = new_ProxyMutex(); diff --git a/include/tscore/Map.h b/include/tscore/Map.h deleted file mode 100644 index 5114bbf7a70..00000000000 --- a/include/tscore/Map.h +++ /dev/null @@ -1,2228 +0,0 @@ -/** @file - - A set of Map templates. - - @section license License - - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -#pragma once - -#include -#include -#include -#include - -#include "tscore/defalloc.h" -#include "tscore/ink_assert.h" -#include "tscore/Diags.h" - -#include "tscore/List.h" - -#define MAP_INTEGRAL_SIZE (1 << (2)) -//#define MAP_INITIAL_SHIFT ((2)+1) -//#define MAP_INITIAL_SIZE (1 << MAP_INITIAL_SHIFT) - -// Simple Vector class, also supports open hashed sets -#define VEC_INTEGRAL_SHIFT_DEFAULT 2 /* power of 2 (1 << VEC_INTEGRAL_SHIFT)*/ -#define VEC_INTEGRAL_SIZE (1 << (S)) -#define VEC_INITIAL_SHIFT ((S) + 1) -#define VEC_INITIAL_SIZE (1 << VEC_INITIAL_SHIFT) - -#define SET_LINEAR_SIZE 4 /* must be <= than VEC_INTEGRAL_SIZE */ -#define SET_INITIAL_INDEX 2 - -template // S must be a power of 2 -class Vec -{ -public: - size_t n; - size_t i; // size index for sets, reserve for vectors - C *v; - C e[VEC_INTEGRAL_SIZE]; - - Vec(); - Vec(const Vec &vv); - Vec(const C c); - ~Vec(); - - C &operator[](int i) const { return v[i]; } - C get(size_t i) const; - void add(C a); - void - push_back(C a) - { - add(a); - } // std::vector name - bool add_exclusive(C a); - C &add(); - void drop(); - C pop(); - void reset(); - void clear(); - void free_and_clear(); - void delete_and_clear(); - void set_clear(); - C *set_add(C a); - void set_remove(C a); // expensive, use BlockHash for cheaper remove - C *set_add_internal(C a); - bool set_union(Vec &v); - int set_intersection(Vec &v); - int some_intersection(Vec &v); - int some_disjunction(Vec &v); - int some_difference(Vec &v); - void set_intersection(Vec &v, Vec &result); - void set_disjunction(Vec &v, Vec &result); - void set_difference(Vec &v, Vec &result); - size_t set_count() const; - size_t count() const; - C *in(C a); - C *set_in(C a); - C first_in_set(); - C *set_in_internal(C a); - void set_expand(); - ssize_t index(C a) const; - void set_to_vec(); - void vec_to_set(); - void move(Vec &v); - void copy(const Vec &v); - void fill(size_t n); - void append(const Vec &v); - template void append(const C *src, CountType count); - void prepend(const Vec &v); - void remove_index(int index); - void - remove(C a) - { - int i = index(a); - if (i >= 0) - remove_index(i); - } - C &insert(size_t index); - void insert(size_t index, Vec &vv); - void insert(size_t index, C a); - void - push(C a) - { - insert(0, a); - } - void reverse(); - void reserve(size_t n); - C * - end() const - { - return v + n; - } - C & - first() const - { - return v[0]; - } - C & - last() const - { - return v[n - 1]; - } - Vec & - operator=(Vec &v) - { - this->copy(v); - return *this; - } - unsigned - length() const - { - return n; - } - // vector::size() intentionally not implemented because it should mean "bytes" not count of elements - int write(int fd); - int read(int fd); - void qsort(bool (*lt)(C, C)); - void qsort(bool (*lt)(const C &, const C &)); - static void swap(C *p1, C *p2); - -private: - void move_internal(Vec &v); - void copy_internal(const Vec &v); - void add_internal(C a); - C &add_internal(); - void addx(); -}; - -// c -- class, p -- pointer to elements of v, v -- vector -#define forv_Vec(_c, _p, _v) \ - if ((_v).n) \ - for (_c *qq__##_p = (_c *)0, *_p = (_v).v[0]; \ - ((uintptr_t)(qq__##_p) < (_v).length()) && ((_p = (_v).v[(intptr_t)qq__##_p]), 1); \ - qq__##_p = (_c *)(((intptr_t)qq__##_p) + 1)) -#define for_Vec(_c, _p, _v) \ - if ((_v).n) \ - for (_c *qq__##_p = (_c *)0, _p = (_v).v[0]; \ - ((uintptr_t)(qq__##_p) < (_v).length()) && ((_p = (_v).v[(intptr_t)qq__##_p]), 1); \ - qq__##_p = (_c *)(((intptr_t)qq__##_p) + 1)) -#define forvp_Vec(_c, _p, _v) \ - if ((_v).n) \ - for (_c *qq__##_p = (_c *)0, *_p = &(_v).v[0]; \ - ((uintptr_t)(qq__##_p) < (_v).length()) && ((_p = &(_v).v[(intptr_t)qq__##_p]), 1); \ - qq__##_p = (_c *)(((intptr_t)qq__##_p) + 1)) - -template class Accum -{ -public: - Vec asset; - Vec asvec; - void - add(C c) - { - if (asset.set_add(c)) - asvec.add(c); - } - void - add(Vec v) - { - for (int i = 0; i < v.n; i++) - if (v.v[i]) - add(v.v[i]); - } - void - clear() - { - asset.clear(); - asvec.clear(); - } -}; - -const uintptr_t prime2[] = {1, 3, 7, 13, 31, 61, 127, 251, 509, 1021, - 2039, 4093, 8191, 16381, 32749, 65521, 131071, 262139, 524287, 1048573, - 2097143, 4194301, 8388593, 16777213, 33554393, 67108859, 134217689, 268435399, 536870909}; - -// primes generated with map_mult.c -const uintptr_t open_hash_primes[256] = { - 0x02D4AF27, 0x1865DFC7, 0x47C62B43, 0x35B4889B, 0x210459A1, 0x3CC51CC7, 0x02ADD945, 0x0607C4D7, 0x558E6035, 0x0554224F, - 0x5A281657, 0x1C458C7F, 0x7F8BE723, 0x20B9BA99, 0x7218AA35, 0x64B10C2B, 0x548E8983, 0x5951218F, 0x7AADC871, 0x695FA5B1, - 0x40D40FCB, 0x20E03CC9, 0x55E9920F, 0x554CE08B, 0x7E78B1D7, 0x7D965DF9, 0x36A520A1, 0x1B0C6C11, 0x33385667, 0x2B0A7B9B, - 0x0F35AE23, 0x0BD608FB, 0x2284ADA3, 0x6E6C0687, 0x129B3EED, 0x7E86289D, 0x1143C24B, 0x1B6C7711, 0x1D87BB41, 0x4C7E635D, - 0x67577999, 0x0A0113C5, 0x6CF085B5, 0x14A4D0FB, 0x4E93E3A7, 0x5C87672B, 0x67F3CA17, 0x5F944339, 0x4C16DFD7, 0x5310C0E3, - 0x2FAD1447, 0x4AFB3187, 0x08468B7F, 0x49E56C51, 0x6280012F, 0x097D1A85, 0x34CC9403, 0x71028BD7, 0x6DEDC7E9, 0x64093291, - 0x6D78BB0B, 0x7A03B465, 0x2E044A43, 0x1AE58515, 0x23E495CD, 0x46102A83, 0x51B78A59, 0x051D8181, 0x5352CAC9, 0x57D1312B, - 0x2726ED57, 0x2E6BC515, 0x70736281, 0x5938B619, 0x0D4B6ACB, 0x44AB5E2B, 0x0029A485, 0x002CE54F, 0x075B0591, 0x3EACFDA9, - 0x0AC03411, 0x53B00F73, 0x2066992D, 0x76E72223, 0x55F62A8D, 0x3FF92EE1, 0x17EE0EB3, 0x5E470AF1, 0x7193EB7F, 0x37A2CCD3, - 0x7B44F7AF, 0x0FED8B3F, 0x4CC05805, 0x7352BF79, 0x3B61F755, 0x523CF9A3, 0x1AAFD219, 0x76035415, 0x5BE84287, 0x6D598909, - 0x456537E9, 0x407EA83F, 0x23F6FFD5, 0x60256F39, 0x5D8EE59F, 0x35265CEB, 0x1D4AD4EF, 0x676E2E0F, 0x2D47932D, 0x776BB33B, - 0x6DE1902B, 0x2C3F8741, 0x5B2DE8EF, 0x686DDB3B, 0x1D7C61C7, 0x1B061633, 0x3229EA51, 0x7FCB0E63, 0x5F22F4C9, 0x517A7199, - 0x2A8D7973, 0x10DCD257, 0x41D59B27, 0x2C61CA67, 0x2020174F, 0x71653B01, 0x2FE464DD, 0x3E7ED6C7, 0x164D2A71, 0x5D4F3141, - 0x5F7BABA7, 0x50E1C011, 0x140F5D77, 0x34E80809, 0x04AAC6B3, 0x29C42BAB, 0x08F9B6F7, 0x461E62FD, 0x45C2660B, 0x08BF25A7, - 0x5494EA7B, 0x0225EBB7, 0x3C5A47CF, 0x2701C333, 0x457ED05B, 0x48CDDE55, 0x14083099, 0x7C69BDAB, 0x7BF163C9, 0x41EE1DAB, - 0x258B1307, 0x0FFAD43B, 0x6601D767, 0x214DBEC7, 0x2852CCF5, 0x0009B471, 0x190AC89D, 0x5BDFB907, 0x15D4E331, 0x15D22375, - 0x13F388D5, 0x12ACEDA5, 0x3835EA5D, 0x2587CA35, 0x06756643, 0x487C6F55, 0x65C295EB, 0x1029F2E1, 0x10CEF39D, 0x14C2E415, - 0x444825BB, 0x24BE0A2F, 0x1D2B7C01, 0x64AE3235, 0x5D2896E5, 0x61BBBD87, 0x4A49E86D, 0x12C277FF, 0x72C81289, 0x5CF42A3D, - 0x332FF177, 0x0DAECD23, 0x6000ED1D, 0x203CDDE1, 0x40C62CAD, 0x19B9A855, 0x782020C3, 0x6127D5BB, 0x719889A7, 0x40E4FCCF, - 0x2A3C8FF9, 0x07411C7F, 0x3113306B, 0x4D7CA03F, 0x76119841, 0x54CEFBDF, 0x11548AB9, 0x4B0748EB, 0x569966B1, 0x45BC721B, - 0x3D5A376B, 0x0D8923E9, 0x6D95514D, 0x0F39A367, 0x2FDAD92F, 0x721F972F, 0x42D0E21D, 0x5C5952DB, 0x7394D007, 0x02692C55, - 0x7F92772F, 0x025F8025, 0x34347113, 0x560EA689, 0x0DCC21DF, 0x09ECC7F5, 0x091F3993, 0x0E0B52AB, 0x497CAA55, 0x0A040A49, - 0x6D8F0CC5, 0x54F41609, 0x6E0CB8DF, 0x3DCB64C3, 0x16C365CD, 0x6D6B9FB5, 0x02B9382B, 0x6A5BFAF1, 0x1669D75F, 0x13CFD4FD, - 0x0FDF316F, 0x21F3C463, 0x6FC58ABF, 0x04E45BE7, 0x1911225B, 0x28CD1355, 0x222084E9, 0x672AD54B, 0x476FC267, 0x6864E16D, - 0x20AEF4FB, 0x603C5FB9, 0x55090595, 0x1113B705, 0x24E38493, 0x5291AF97, 0x5F5446D9, 0x13A6F639, 0x3D501313, 0x37E02017, - 0x236B0ED3, 0x60F246BF, 0x01E02501, 0x2D2F66BD, 0x6BF23609, 0x16729BAF}; - -/* IMPLEMENTATION */ - -template inline Vec::Vec() : n(0), i(0), v(nullptr) -{ - memset(static_cast(&e[0]), 0, sizeof(e)); -} - -template inline Vec::Vec(const Vec &vv) -{ - copy(vv); -} - -template inline Vec::Vec(C c) -{ - n = 1; - i = 0; - v = &e[0]; - e[0] = c; -} - -template -inline C -Vec::get(size_t i) const -{ - if (i < n) { - return v[i]; - } else { - return C(); - } -} - -template -inline void -Vec::add(C a) -{ - if (n & (VEC_INTEGRAL_SIZE - 1)) - v[n++] = a; - else if (!v) - (v = e)[n++] = a; - else - add_internal(a); -} - -template -inline C & -Vec::add() -{ - C *ret; - if (n & (VEC_INTEGRAL_SIZE - 1)) - ret = &v[n++]; - else if (!v) - ret = &(v = e)[n++]; - else - ret = &add_internal(); - return *ret; -} - -template -inline void -Vec::drop() -{ - if (n && 0 == --n) - clear(); -} - -template -inline C -Vec::pop() -{ - if (!n) - return 0; - n--; - C ret = v[n]; - if (!n) - clear(); - return ret; -} - -template -inline void -Vec::set_clear() -{ - memset(v, 0, n * sizeof(C)); -} - -template -inline C * -Vec::set_add(C a) -{ - if (n < SET_LINEAR_SIZE) { - for (C *c = v; c < v + n; c++) - if (*c == a) - return nullptr; - add(a); - return &v[n - 1]; - } - if (n == SET_LINEAR_SIZE) { - Vec vv(*this); - clear(); - for (C *c = vv.v; c < vv.v + vv.n; c++) { - set_add_internal(*c); - } - } - return set_add_internal(a); -} - -template -void -Vec::set_remove(C a) -{ - Vec tmp; - tmp.move(*this); - for (C *c = tmp.v; c < tmp.v + tmp.n; c++) - if (*c != a) - set_add(a); -} - -template -inline size_t -Vec::count() const -{ - int x = 0; - for (C *c = v; c < v + n; c++) - if (*c) - x++; - return x; -} - -template -inline C * -Vec::in(C a) -{ - for (C *c = v; c < v + n; c++) - if (*c == a) - return c; - return nullptr; -} - -template -inline bool -Vec::add_exclusive(C a) -{ - if (!in(a)) { - add(a); - return true; - } else - return false; -} - -template -inline C * -Vec::set_in(C a) -{ - if (n <= SET_LINEAR_SIZE) - return in(a); - return set_in_internal(a); -} - -template -inline C -Vec::first_in_set() -{ - for (C *c = v; c < v + n; c++) - if (*c) - return *c; - return 0; -} - -template -inline ssize_t -Vec::index(C a) const -{ - for (C *c = v; c < v + n; c++) { - if (*c == a) { - return c - v; - } - } - return -1; -} - -template -inline void -Vec::move_internal(Vec &vv) -{ - n = vv.n; - i = vv.i; - if (vv.v == &vv.e[0]) { - memcpy(e, &vv.e[0], sizeof(e)); - v = e; - } else - v = vv.v; -} - -template -inline void -Vec::move(Vec &vv) -{ - move_internal(vv); - vv.v = nullptr; - vv.clear(); -} - -template -inline void -Vec::copy(const Vec &vv) -{ - n = vv.n; - i = vv.i; - if (vv.v == &vv.e[0]) { - memcpy(e, &vv.e[0], sizeof(e)); - v = e; - } else { - if (vv.v) - copy_internal(vv); - else - v = nullptr; - } -} - -template -inline void -Vec::fill(size_t nn) -{ - for (size_t i = n; i < nn; i++) - add() = 0; -} - -template -inline void -Vec::append(const Vec &vv) -{ - for (C *c = vv.v; c < vv.v + vv.n; c++) - if (*c != 0) - add(*c); -} - -template -template -inline void -Vec::append(const C *src, CountType count) -{ - reserve(length() + count); - for (CountType c = 0; c < count; ++c) { - add(src[c]); - } -} - -template -inline void -Vec::prepend(const Vec &vv) -{ - if (vv.n) { - int oldn = n; - fill(n + vv.n); - if (oldn) - memmove(&v[vv.n], &v[0], oldn * sizeof(v[0])); - memcpy(&v[0], vv.v, vv.n * sizeof(v[0])); - } -} - -template -void -Vec::add_internal(C a) -{ - addx(); - v[n++] = a; -} - -template -C & -Vec::add_internal() -{ - addx(); - return v[n++]; -} - -template -C * -Vec::set_add_internal(C c) -{ - size_t j, k; - if (n) { - uintptr_t h = (uintptr_t)c; - h = h % n; - for (k = h, j = 0; j < i + 3; j++) { - if (!v[k]) { - v[k] = c; - return &v[k]; - } else if (v[k] == c) { - return nullptr; - } - k = (k + open_hash_primes[j]) % n; - } - } - Vec vv; - vv.move_internal(*this); - set_expand(); - if (vv.v) { - set_union(vv); - } - return set_add(c); -} - -template -C * -Vec::set_in_internal(C c) -{ - size_t j, k; - if (n) { - uintptr_t h = (uintptr_t)c; - h = h % n; - for (k = h, j = 0; j < i + 3; j++) { - if (!v[k]) - return nullptr; - else if (v[k] == c) - return &v[k]; - k = (k + open_hash_primes[j]) % n; - } - } - return nullptr; -} - -template -bool -Vec::set_union(Vec &vv) -{ - bool changed = false; - for (size_t i = 0; i < vv.n; i++) { - if (vv.v[i]) { - changed = set_add(vv.v[i]) || changed; - } - } - return changed; -} - -template -int -Vec::set_intersection(Vec &vv) -{ - Vec tv; - tv.move(*this); - int changed = 0; - for (int i = 0; i < tv.n; i++) - if (tv.v[i]) { - if (vv.set_in(tv.v[i])) - set_add(tv.v[i]); - else - changed = 1; - } - return changed; -} - -template -int -Vec::some_intersection(Vec &vv) -{ - for (int i = 0; i < n; i++) - if (v[i]) - if (vv.set_in(v[i])) - return 1; - return 0; -} - -template -int -Vec::some_disjunction(Vec &vv) -{ - for (int i = 0; i < n; i++) - if (v[i]) - if (!vv.set_in(v[i])) - return 1; - for (int i = 0; i < vv.n; i++) - if (vv.v[i]) - if (!set_in(vv.v[i])) - return 1; - return 0; -} - -template -void -Vec::set_intersection(Vec &vv, Vec &result) -{ - for (int i = 0; i < n; i++) - if (v[i]) - if (vv.set_in(v[i])) - result.set_add(v[i]); -} - -template -void -Vec::set_disjunction(Vec &vv, Vec &result) -{ - for (int i = 0; i < n; i++) - if (v[i]) - if (!vv.set_in(v[i])) - result.set_add(v[i]); - for (int i = 0; i < vv.n; i++) - if (vv.v[i]) - if (!set_in(vv.v[i])) - result.set_add(vv.v[i]); -} - -template -void -Vec::set_difference(Vec &vv, Vec &result) -{ - for (int i = 0; i < n; i++) - if (v[i]) - if (!vv.set_in(v[i])) - result.set_add(v[i]); -} - -template -int -Vec::some_difference(Vec &vv) -{ - for (int i = 0; i < n; i++) - if (v[i]) - if (!vv.set_in(v[i])) - return 1; - return 0; -} - -template -size_t -Vec::set_count() const -{ - size_t x = 0; - for (size_t i = 0; i < n; i++) { - if (v[i]) { - x++; - } - } - return x; -} - -template -void -Vec::set_to_vec() -{ - C *x = &v[0], *y = x; - for (; y < v + n; y++) { - if (*y) { - if (x != y) - *x = *y; - x++; - } - } - if (i) { - i = prime2[i]; // convert set allocation to reserve - if (i - n > 0) - memset(&v[n], 0, (i - n) * (sizeof(C))); - } else { - i = 0; - if (v == &e[0] && VEC_INTEGRAL_SIZE - n > 0) - memset(&v[n], 0, (VEC_INTEGRAL_SIZE - n) * (sizeof(C))); - } -} - -template -void -Vec::vec_to_set() -{ - Vec vv; - vv.move(*this); - for (C *c = vv.v; c < vv.v + vv.n; c++) - set_add(*c); -} - -template -void -Vec::remove_index(int index) -{ - if (n > 1) - memmove(&v[index], &v[index + 1], (n - 1 - index) * sizeof(v[0])); - n--; - if (n <= 0) - v = e; -} - -template -void -Vec::insert(size_t index, C a) -{ - add(); - memmove(&v[index + 1], &v[index], (n - index - 1) * sizeof(C)); - v[index] = a; -} - -template -void -Vec::insert(size_t index, Vec &vv) -{ - fill(n + vv.n); - memmove(&v[index + vv.n], &v[index], (n - index - 1) * sizeof(C)); - for (int x = 0; x < vv.n; x++) - v[index + x] = vv[x]; -} - -template -C & -Vec::insert(size_t index) -{ - add(); - memmove(&v[index + 1], &v[index], (n - index - 1) * sizeof(C)); - memset(&v[index], 0, sizeof(C)); - return v[index]; -} - -template -void -Vec::reverse() -{ - for (int i = 0; i < n / 2; i++) { - C *s = &v[i], *e = &v[n - 1 - i]; - C t; - memcpy(&t, s, sizeof(t)); - memcpy(s, e, sizeof(t)); - memcpy(e, &t, sizeof(t)); - } -} - -template -void -Vec::copy_internal(const Vec &vv) -{ - int l = n, nl = (1 + VEC_INITIAL_SHIFT); - l = l >> VEC_INITIAL_SHIFT; - while (l) { - l = l >> 1; - nl++; - } - nl = 1 << nl; - v = (C *)A::alloc(nl * sizeof(C)); - memcpy(static_cast(v), vv.v, n * sizeof(C)); - memset(static_cast(v + n), 0, (nl - n) * sizeof(C)); - if (i > n) // reset reserve - i = 0; -} - -template -void -Vec::set_expand() -{ - if (!n) - i = SET_INITIAL_INDEX; - else - i = i + 1; - n = prime2[i]; - v = (C *)A::alloc(n * sizeof(C)); - memset(static_cast(v), 0, n * sizeof(C)); -} - -template -inline void -Vec::reserve(size_t x) -{ - if (x <= n) - return; - unsigned xx = 1 << VEC_INITIAL_SHIFT; - while (xx < x) - xx *= 2; - i = xx; - void *vv = (void *)v; - v = (C *)A::alloc(i * sizeof(C)); - if (vv && n) - memcpy(v, vv, n * sizeof(C)); - memset(&v[n], 0, (i - n) * sizeof(C)); - if (vv && vv != e) - A::free(vv); -} - -template -inline void -Vec::addx() -{ - if (!v) { - v = e; - return; - } - if (v == e) { - v = (C *)A::alloc(VEC_INITIAL_SIZE * sizeof(C)); - memcpy(static_cast(v), &e[0], n * sizeof(C)); - ink_assert(n < VEC_INITIAL_SIZE); - memset(static_cast(&v[n]), 0, (VEC_INITIAL_SIZE - n) * sizeof(C)); - } else { - if ((n & (n - 1)) == 0) { - size_t nl = n * 2; - if (nl <= i) { - return; - } else { - i = 0; - } - void *vv = (void *)v; - v = (C *)A::alloc(nl * sizeof(C)); - memcpy(static_cast(v), vv, n * sizeof(C)); - memset(static_cast(&v[n]), 0, n * sizeof(C)); - A::free(vv); - } - } -} - -template -inline void -Vec::reset() -{ - v = nullptr; - n = 0; - i = 0; -} - -template -inline void -Vec::clear() -{ - if (v && v != e) - A::free(v); - reset(); -} - -template -inline void -Vec::free_and_clear() -{ - for (size_t x = 0; x < (n); x++) - A::free((void *)v[x]); - clear(); -} - -template -inline void -Vec::delete_and_clear() -{ - for (size_t x = 0; x < n; x++) { - if (v[x]) { - delete v[x]; - } - } - clear(); -} - -template inline Vec::~Vec() -{ - if (v && v != e) - A::free(v); -} - -template -inline int -marshal_size(Vec &v) -{ - int l = sizeof(int) * 2; - for (int x = 0; x < v.n; x++) - l += ::marshal_size(v.v[x]); - return l; -} - -template -inline int -marshal(Vec &v, char *buf) -{ - char *x = buf; - *(int *)x = v.n; - x += sizeof(int); - *(int *)x = v.i; - x += sizeof(int); - for (int i = 0; i < v.n; i++) - x += ::marshal(v.v[i], x); - return x - buf; -} - -template -inline int -unmarshal(Vec &v, char *buf) -{ - char *x = buf; - v.n = *(int *)x; - x += sizeof(int); - v.i = *(int *)x; - x += sizeof(int); - if (v.n) { - v.v = (C *)A::alloc(sizeof(C) * v.n); - memset(v.v, 0, sizeof(C) * v.n); - } else - v.v = v.e; - for (int i = 0; i < v.n; i++) - x += ::unmarshal(v.v[i], x); - return x - buf; -} - -template -inline int -Vec::write(int fd) -{ - int r = 0, t = 0; - if ((r = ::write(fd, this, sizeof(*this))) < 0) - return r; - t += r; - if ((r = ::write(fd, v, n * sizeof(C))) < 0) - return r; - t += r; - return t; -} - -template -inline int -Vec::read(int fd) -{ - int r = 0, t = 0; - if ((r = ::read(fd, this, sizeof(*this))) < 0) - return r; - t += r; - v = (C *)A::alloc(sizeof(C) * n); - memset(v, 0, sizeof(C) * n); - if ((r = ::read(fd, v, n * sizeof(C))) < 0) - return r; - t += r; - return t; -} - -template -inline void -Vec::swap(C *p1, C *p2) -{ - C t = *p1; - *p1 = *p2; - *p2 = t; -} - -template -inline void -qsort_Vec(C *left, C *right, bool (*lt)(C, C)) -{ - if (right - left < 5) { - for (C *y = right - 1; y > left; y--) { - for (C *x = left; x < y; x++) { - if (lt(x[1], x[0])) { - C t = x[0]; - x[0] = x[1]; - x[1] = t; - } - } - } - } else { - C *center = left + ((right - left) / 2); - C median; - - // find the median - if (lt(*center, *left)) { // order left and center - Vec::swap(center, left); - } - if (lt(*(right - 1), *left)) { // order left and right - Vec::swap(right - 1, left); - } - if (lt(*(right - 1), *center)) { // order right and center - Vec::swap((right - 1), center); - } - Vec::swap(center, right - 2); // stash the median one from the right for now - median = *(right - 2); // the median of left, center and right values - - // now partition, pivoting on the median value - // l ptr is +1 b/c we already put the lowest of the incoming left, center - // and right in there, ignore it for now - // r ptr is -2 b/c we already put the biggest of the 3 values in (right-1) - // and the median in (right -2) - C *l = left + 1, *r = right - 2; - - // move l and r until they have something to do - while (lt(median, *(r - 1))) { - r--; - } - while (l < r && lt(*l, median)) { - l++; - } - // until l and r meet, - // compare l and median - // swap l for r if l is larger than median - while (l < r) { - if (lt(*l, median)) { - l++; - } else { - Vec::swap(l, r - 1); - r--; - } - } - - Vec::swap(l, right - 2); // restore median to its rightful place - - // recurse for the littles (left segment) - qsort_Vec(left, l, lt); - // recurse for the bigs (right segment) - qsort_Vec(l + 1, right, lt); - } -} - -template -inline void -qsort_VecRef(C *left, C *right, bool (*lt)(const C &, const C &), unsigned int *p_ctr) -{ - if (right - left < 5) { - for (C *y = right - 1; y > left; y--) { - for (C *x = left; x < y; x++) { - if (lt(x[1], x[0])) { - C t = x[0]; - x[0] = x[1]; - x[1] = t; - } - } - } - } else { - C *center = left + ((right - left) / 2); - C median; - - // find the median - if (lt(*center, *left)) { // order left and center - Vec::swap(center, left); - } - if (lt(*(right - 1), *left)) { // order left and right - Vec::swap(right - 1, left); - } - if (lt(*(right - 1), *center)) { // order right and center - Vec::swap((right - 1), center); - } - Vec::swap(center, right - 2); // stash the median one from the right for now - median = *(right - 2); // the median of left, center and right values - - // now partition, pivoting on the median value - // l ptr is +1 b/c we already put the lowest of the incoming left, center - // and right in there, ignore it for now - // r ptr is -2 b/c we already put the biggest of the 3 values in (right-1) - // and the median in (right -2) - C *l = left + 1, *r = right - 2; - - // move l and r until they have something to do - while (lt(median, *(r - 1))) { - r--; - } - while (l < r && lt(*l, median)) { - l++; - } - // until l and r meet, - // compare l and median - // swap l for r if l is larger than median - while (l < r) { - if (lt(*l, median)) { - l++; - } else { - Vec::swap(l, r - 1); - r--; - } - } - - Vec::swap(l, right - 2); // restore median to its rightful place - - // recurse for the littles (left segment) - qsort_VecRef(left, l, lt, p_ctr); - // recurse for the bigs (right segment) - qsort_VecRef(l + 1, right, lt, p_ctr); - } - (*p_ctr)++; -} - -template -inline void -Vec::qsort(bool (*lt)(C, C)) -{ - if (n) - qsort_Vec(&v[0], end(), lt); -} - -template -inline void -Vec::qsort(bool (*lt)(const C &, const C &)) -{ - static unsigned int ctr = 0; - if (n) - qsort_VecRef(&v[0], end(), lt, &ctr); - Debug("qsort", "took %u iterations to sort %ld elements", ctr, n); -} -void test_vec(); - -typedef const char cchar; - -template -static inline char * -_dupstr(cchar *s, cchar *e = nullptr) -{ - int l = e ? e - s : strlen(s); - char *ss = (char *)A::alloc(l + 1); - memcpy(ss, s, l); - ss[l] = 0; - return ss; -} - -// Simple direct mapped Map (pointer hash table) and Environment - -template class MapElem -{ -public: - K key; - C value; - bool - operator==(MapElem &e) - { - return e.key == key; - } - operator uintptr_t(void) { return (uintptr_t)(uintptr_t)key; } - MapElem(K const &akey, C const &avalue) : key(akey), value(avalue) {} - MapElem(MapElem const &e) : key(e.key), value(e.value) {} - MapElem() : key(), value() {} -}; - -template class Map : public Vec, A> -{ -public: - typedef MapElem ME; - typedef Vec PType; - using PType::n; - using PType::i; - using PType::v; - ME *put(K akey, C avalue); - ME *put(K akey); - C get(K akey); - C *getp(K akey); - void get_keys(Vec &keys) const; - void get_keys_set(Vec &keys); - void get_values(Vec &values); - void map_union(Map &m); - bool some_disjunction(Map &m) const; -}; - -template class HashFns -{ -public: - static uintptr_t hash(C a); - static int equal(C a, C b); -}; - -template class HashSetFns -{ -public: - static uintptr_t hash(C a); - static uintptr_t hash(K a); - static int equal(C a, C b); - static int equal(K a, C b); -}; - -template class HashMap : public Map -{ -public: - typedef MapElem value_type; ///< What's stored in the table. - using Map::n; - using Map::i; - using Map::v; - using Map::e; - HashMap() {} - HashMap(C c) : invalid_value(c) {} - MapElem *get_internal(K akey) const; - C get(K akey) const; - value_type *put(K akey, C avalue); - void get_keys(Vec &keys) const; - void get_values(Vec &values); - -private: - C invalid_value = 0; // return this object if key is not present -}; - -#define form_Map(_c, _p, _v) \ - if ((_v).n) \ - for (_c *qq__##_p = (_c *)0, *_p = &(_v).v[0]; ((uintptr_t)(qq__##_p) < (_v).n) && ((_p = &(_v).v[(uintptr_t)qq__##_p]) || 1); \ - qq__##_p = (_c *)(((uintptr_t)qq__##_p) + 1)) \ - if ((_p)->key) - -template class HashSet : public Vec -{ -public: - typedef Vec V; - using V::n; - using V::i; - using V::v; - using V::e; - C get(K akey); - C *put(C avalue); -}; - -class StringHashFns -{ -public: - static uintptr_t - hash(cchar *s) - { - uintptr_t h = 0; - // 31 changed to 27, to avoid prime2 in vec.cpp - while (*s) - h = h * 27 + (unsigned char)*s++; - return h; - } - static int - equal(cchar *a, cchar *b) - { - return !strcmp(a, b); - } -}; - -class CaseStringHashFns -{ -public: - static uintptr_t - hash(cchar *s) - { - uintptr_t h = 0; - // 31 changed to 27, to avoid prime2 in vec.cpp - while (*s) - h = h * 27 + (unsigned char)toupper(*s++); - return h; - } - static int - equal(cchar *a, cchar *b) - { - return !strcasecmp(a, b); - } -}; - -class PointerHashFns -{ -public: - static uintptr_t - hash(void *s) - { - return (uintptr_t)(uintptr_t)s; - } - static int - equal(void *a, void *b) - { - return a == b; - } -}; - -template class ChainHash : public Map, A> -{ -public: - using Map, A>::n; - using Map, A>::v; - typedef ConsCell ChainCons; - C put(C c); - C get(C c); - C put_bag(C c); - int get_bag(C c, Vec &v); - int del(C avalue); - void get_elements(Vec &elements); -}; - -template -class ChainHashMap : public Map, A>, A> -{ -public: - using Map, A>, A>::n; - using Map, A>, A>::v; - MapElem *put(K akey, C avalue); - C get(K akey); - int del(K akey); - MapElem *put_bag(K akey, C c); - int get_bag(K akey, Vec &v); - void get_keys(Vec &keys); - void get_values(Vec &values); -}; - -template class StringChainHash : public ChainHash -{ -public: - cchar *canonicalize(cchar *s, cchar *e); - cchar * - canonicalize(cchar *s) - { - return canonicalize(s, s + strlen(s)); - } -}; - -template class NBlockHash -{ -public: - int n; - int i; - C *v; - C e[N]; - - C * - end() - { - return last(); - } - int - length() - { - return N * n; - } - C *first(); - C *last(); - C put(C c); - C get(C c); - C *assoc_put(C *c); - C *assoc_get(C *c); - int del(C c); - void clear(); - void reset(); - int count(); - void size(int p2); - void copy(const NBlockHash &hh); - void move(NBlockHash &hh); - NBlockHash(); - NBlockHash(NBlockHash &hh) - { - v = e; - copy(hh); - } -}; - -/* use forv_Vec on BlockHashes */ - -#define DEFAULT_BLOCK_HASH_SIZE 4 -template class BlockHash : public NBlockHash -{ -}; -typedef BlockHash StringBlockHash; - -template class Env -{ -public: - typedef ConsCell EnvCons; - void put(K akey, C avalue); - C get(K akey); - void push(); - void pop(); - void - clear() - { - store.clear(); - scope.clear(); - } - - Env() {} - Map *, A> store; - List, A> scope; - List *get_bucket(K akey); -}; - -/* IMPLEMENTATION */ - -template -inline C -Map::get(K akey) -{ - MapElem e(akey, (C)0); - MapElem *x = this->set_in(e); - if (x) - return x->value; - return (C)0; -} - -template -inline C * -Map::getp(K akey) -{ - MapElem e(akey, (C)0); - MapElem *x = this->set_in(e); - if (x) - return &x->value; - return 0; -} - -template -inline MapElem * -Map::put(K akey, C avalue) -{ - MapElem e(akey, avalue); - MapElem *x = this->set_in(e); - if (x) { - x->value = avalue; - return x; - } else - return this->set_add(e); -} - -template -inline MapElem * -Map::put(K akey) -{ - MapElem e(akey, 0); - MapElem *x = this->set_in(e); - if (x) - return x; - else - return this->set_add(e); -} - -template -inline void -Map::get_keys(Vec &keys) const -{ - for (size_t i = 0; i < n; i++) - if (v[i].key) - keys.add(v[i].key); -} - -template -inline void -Map::get_keys_set(Vec &keys) -{ - for (int i = 0; i < n; i++) - if (v[i].key) - keys.set_add(v[i].key); -} - -template -inline void -Map::get_values(Vec &values) -{ - for (int i = 0; i < n; i++) - if (v[i].key) - values.set_add(v[i].value); - values.set_to_vec(); -} - -template -inline void -Map::map_union(Map &m) -{ - for (int i = 0; i < m.n; i++) - if (m.v[i].key) - put(m.v[i].key, m.v[i].value); -} - -template -inline bool -Map::some_disjunction(Map &m) const -{ - for (size_t i = 0; i < m.n; i++) { - if (m.v[i].key && get(m.v[i].key) != m.v[i].value) { - return true; - } - } - for (size_t i = 0; i < n; i++) { - if (v[i].key && m.get(v[i].key) != v[i].value) { - return true; - } - } - return false; -} - -template -inline void -map_set_add(Map *, A> &m, K akey, C avalue) -{ - Vec *v = m.get(akey); - if (!v) - m.put(akey, (v = new Vec)); - v->set_add(avalue); -} - -template -inline void -map_set_add(Map *, A> &m, K akey, Vec *madd) -{ - Vec *v = m.get(akey); - if (!v) - m.put(akey, (v = new Vec)); - v->set_union(*madd); -} - -template -inline C -HashSet::get(K akey) -{ - if (!n) - return 0; - if (n <= MAP_INTEGRAL_SIZE) { - for (C *c = v; c < v + n; c++) - if (c) - if (AHashFns::equal(akey, *c)) - return *c; - return 0; - } - uintptr_t h = AHashFns::hash(akey); - h = h % n; - for (int k = h, j = 0; j < i + 3; j++) { - if (!v[k]) - return 0; - else if (AHashFns::equal(akey, v[k])) - return v[k]; - k = (k + open_hash_primes[j]) % n; - } - return 0; -} - -template -inline C * -HashSet::put(C avalue) -{ - if (n < MAP_INTEGRAL_SIZE) { - if (!v) - v = e; - for (int i = 0; i < n; i++) - if (AHashFns::equal(avalue, v[i])) - return &v[i]; - v[n] = avalue; - n++; - return &v[n - 1]; - } - if (n > MAP_INTEGRAL_SIZE) { - uintptr_t h = AHashFns::hash(avalue); - h = h % n; - for (int k = h, j = 0; j < i + 3; j++) { - if (!v[k]) { - v[k] = avalue; - return &v[k]; - } - k = (k + open_hash_primes[j]) % n; - } - } else - i = SET_INITIAL_INDEX - 1; // will be incremented in set_expand - HashSet vv(*this); - Vec::set_expand(); - for (int i = 0; i < vv.n; i++) - if (vv.v[i]) - put(vv.v[i]); - return put(avalue); -} - -template -inline MapElem * -HashMap::get_internal(K akey) const -{ - if (!n) - return nullptr; - if (n <= MAP_INTEGRAL_SIZE) { - for (MapElem *c = v; c < v + n; c++) - if (c->key) - if (AHashFns::equal(akey, c->key)) - return c; - return nullptr; - } - uintptr_t h = AHashFns::hash(akey); - h = h % n; - for (size_t k = h, j = 0; j < i + 3; j++) { - if (!v[k].key) - return nullptr; - else if (AHashFns::equal(akey, v[k].key)) - return &v[k]; - k = (k + open_hash_primes[j]) % n; - } - return nullptr; -} - -template -inline C -HashMap::get(K akey) const -{ - MapElem *x = get_internal(akey); - if (!x) - return invalid_value; - return x->value; -} - -template -inline MapElem * -HashMap::put(K akey, C avalue) -{ - MapElem *x = get_internal(akey); - if (x) { - x->value = avalue; - return x; - } else { - if (n < MAP_INTEGRAL_SIZE) { - if (!v) - v = e; - v[n].key = akey; - v[n].value = avalue; - n++; - return &v[n - 1]; - } - if (n > MAP_INTEGRAL_SIZE) { - uintptr_t h = AHashFns::hash(akey); - h = h % n; - for (size_t k = h, j = 0; j < i + 3; j++) { - if (!v[k].key) { - v[k].key = akey; - v[k].value = avalue; - return &v[k]; - } - k = (k + open_hash_primes[j]) % n; - } - } else - i = SET_INITIAL_INDEX - 1; // will be incremented in set_expand - } - HashMap vv(*this); - Map::set_expand(); - for (size_t i = 0; i < vv.n; i++) { - if (vv.v && vv.v[i] && vv.v[i].key) { - put(vv.v[i].key, vv.v[i].value); - } - } - return put(akey, avalue); -} - -template -inline void -HashMap::get_keys(Vec &keys) const -{ - Map::get_keys(keys); -} - -template -inline void -HashMap::get_values(Vec &values) -{ - Map::get_values(values); -} - -/* ---------------------------------------------------------------------------------------------- */ -/** A hash map usable by ATS core. - - This class depends on the @c DLL class from @c List.h. It assumes it can uses instances of that - class to store chains of elements. - - Values stored in this container are not destroyed when the container is destroyed. These must be - released by the client. - - Duplicate keys are allowed. Clients must walk the list for multiple entries. - @see @c Location::operator++() - - By default the table automatically expands to limit the average chain length. This can be tuned. If set - to @c MANUAL then the table will expand @b only when explicitly requested to do so by the client. - @see @c ExpansionPolicy - @see @c setExpansionPolicy() - @see @c setExpansionLimit() - @see @c expand() - - All the parameters for the hash map are passed via the template argument @a H. This is a struct - that contains both type definitions and static methods. It must have - - - No state (cheap and risk free to copy). - - - All required methods are static methods. - - @a ID is a @c typedef for the hash type. This is the type of the value produced by the hash function. It must be - a numeric type. - - @a Key is a @c typedef for the key type. This is passed to the @a hash function and used for equality - checking of elements. It is presumed cheap to copy. If the underlying key is not a simple type - then @a Key should be declared as a constant pointer or a constant reference. The hash table - will never attempt to modify a key. - - @a Value is a @c typedef for the value type, the type of the element stored in the hash table. - - @a ListHead is @c typedef for the @c DLL compatible class that can serve as the anchor for a chain of - @a Value instances. This is use both as data to be stored in a bucket and for access to next and - previous pointers from instances of @a Value. - - Method @c hash converts a @c Key to a hash value. The key argument can be by value or by constant reference. - @code - ID hash(Key key); - @endcode - - Method @c key extracts the key from a @c Value instance. - @code - Key key(Value const*); - @endcode - - Method @c equal checks for equality between a @c Key and a @c Value. The key argument can be a - constant reference or by value. The arguments should be @c const if not by value. - - @code - bool equal (Key lhs, Key rhs); - bool equal (Key key, Value const* value); - @endcode - - Example for @c HttpServerSession keyed by the origin server IP address. - - @code - struct Hasher { - typedef uint32_t ID; - typedef sockaddr const* Key; - typedef HttpServerSession Value; - typedef DList(HttpServerSession, ip_hash_link) ListHead; - - static uint32_t hash(sockaddr const* key) { return ats_ip_hash(key); } - static sockaddr const* key(HttpServerSession const* value) { return &value->ip.sa } - static bool equal(sockaddr const* lhs, sockaddr const* rhs) { return ats_ip_eq(lhs, rhs); } - // Alternatively - // static ID hash(Key* key); - // static Key key(Value* value); - // static bool equal(Key lhs, Key rhs); - @endcode - - In @c HttpServerSession is the definition - - @code - LINK(HttpServerSession, ip_hash_link); - @endcode - - which creates the internal links used by @c TSHashTable. - - */ -template -class TSHashTable -{ -public: - typedef TSHashTable self; ///< Self reference type. - - // Make embedded types easier to use by importing them to the class namespace. - typedef H Hasher; ///< Rename and promote. - typedef typename Hasher::ID ID; ///< ID type. - typedef typename Hasher::Key Key; ///< Key type. - typedef typename Hasher::Value Value; ///< Stored value (element) type. - typedef typename Hasher::ListHead ListHead; ///< Anchor for chain. - - /// When the hash table is expanded. - enum ExpansionPolicy { - MANUAL, ///< Client must explicitly expand the table. - AVERAGE, ///< Table expands if average chain length exceeds limit. [default] - MAXIMUM ///< Table expands if any chain length exceeds limit. - }; - - /** Hash bucket. - This is stored in the base array, anchoring the open chaining. - - @internal The default values are selected so that zero initialization is correct. Be careful if you - change that. - */ - struct Bucket { - ListHead m_chain; ///< Chain of elements. - size_t m_count; ///< # of elements in chain. - - /** Internal chain for iteration. - - Iteration is tricky because it needs to skip over empty buckets and detect end of buckets. - Both of these are difficult inside the iterator without excess data. So we chain the - non-empty buckets and let the iterator walk that. This makes end detection easy and - iteration on sparse data fast. If we make it a doubly linked list adding and removing buckets - is very fast as well. - */ - LINK(Bucket, m_link); - - /** Do the values in this bucket have different keys? - - @internal This can have a false positive, but that's OK, better than the expense of being - exact. What we want is to avoid expanding to shorten the chain if it won't help, which it - won't if all the keys are the same. - - @internal Because we've selected the default to be @c false so we can use @c Vec which zero fills empty elements. - */ - bool m_mixed_p; - - /// Default constructor - equivalent to zero filled. - Bucket() : m_count(0), m_mixed_p(false) { ink_zero(m_link); } - }; - - /** Information about locating a value in the hash table. - - An instance of this returned when searching for a key in the table. It can then be used to - check if a matching key was found, and to iterate over equivalent keys. Note this iterator - will touch only values which have a matching key. - - @internal It's not really an iterator, although similar. - @internal we store the ID (hashed key value) for efficiency - we can get the actual key via the - @a m_value member. - */ - struct Location { - Value *m_value; ///< The value located. - Bucket *m_bucket; ///< Containing bucket of value. - ID m_id; ///< ID (hashed key). - size_t m_distance; ///< How many values in the chain we've gone past to get here. - - /// Default constructor - empty location. - Location() : m_value(nullptr), m_bucket(nullptr), m_id(0), m_distance(0) {} - /// Check for location being valid (referencing a value). - bool - isValid() const - { - return nullptr != m_value; - } - - /// Automatically cast to a @c Value* for convenience. - /// @note This lets you assign the return of @c find to a @c Value*. - /// @note This also permits the use of this class directly as a boolean expression. - operator Value *() const { return m_value; } - /// Dereference. - Value &operator*() const { return *m_value; } - /// Dereference. - Value *operator->() const { return m_value; } - /// Find next value with matching key (prefix). - Location & - operator++() - { - if (m_value) - this->advance(); - return *this; - } - /// Find next value with matching key (postfix). - Location & - operator++(int) - { - Location zret(*this); - if (m_value) - this->advance(); - return zret; - } - - protected: - /// Move to next matching value, no checks. - void advance(); - - friend class TSHashTable; - }; - - /** Standard iterator for walking the table. - This iterates over all elements. - @internal Iterator is @a end if @a m_value is @c nullptr. - */ - struct iterator { - Value *m_value; ///< Current location. - Bucket *m_bucket; ///< Current bucket; - - iterator() : m_value(0), m_bucket(0) {} - iterator &operator++(); - iterator operator++(int); - Value &operator*() { return *m_value; } - Value *operator->() { return m_value; } - bool - operator==(iterator const &that) - { - return m_bucket == that.m_bucket && m_value == that.m_value; - } - bool - operator!=(iterator const &that) - { - return !(*this == that); - } - - protected: - /// Internal iterator constructor. - iterator(Bucket *b, Value *v) : m_value(v), m_bucket(b) {} - friend class TSHashTable; - }; - - iterator begin(); ///< First element. - iterator end(); ///< Past last element. - - /// The default starting number of buckets. - static size_t const DEFAULT_BUCKET_COUNT = 7; ///< POOMA. - /// The default expansion policy limit. - static size_t const DEFAULT_EXPANSION_LIMIT = 4; ///< Value from previous version. - - /** Constructor (also default). - Constructs an empty table with at least @a nb buckets. - */ - TSHashTable(size_t nb = DEFAULT_BUCKET_COUNT); - - /** Insert a value in to the table. - The @a value must @b NOT already be in a table of this type. - @note The value itself is put in the table, @b not a copy. - */ - void insert(Value *value); - - /** Find a value that matches @a key. - - @note This finds the first value with a matching @a key. No other properties - of the value are examined. - - @return The @c Location of the value. Use @c Location::isValid() to check for success. - */ - Location find(Key key); - - /** Get a @c Location for @a value. - - This is a bit obscure but needed in certain cases. It should only be used on a @a value that - is already known to be in the table. It just does the bucket lookup and uses that and the @a - value to construct a @c Location that can be used with other methods. The @a m_distance value - is not set in this case for performance reasons. - */ - Location find(Value *value); - - /** Remove the value at @a location from the table. - - This method assumes a @a location is consistent. Be very careful if you modify a @c Location. - - @note This does @b not clean up the removed elements. Use carefully to avoid leaks. - - @return @c true if the value was removed, @c false otherwise. - */ - bool remove(Location const &location); - - /** Remove @b all values with @a key. - - @note This does @b not clean up the removed elements. Use carefully to avoid leaks. - - @return @c true if any value was removed, @c false otherwise. - */ - bool remove(Key key); - - /** Remove all values from the table. - - The values are not cleaned up. The values are not touched in this method, therefore it is safe - to destroy them first and then @c clear this table. - */ - void clear(); - - /// Get the number of elements in the table. - size_t - count() const - { - return m_count; - } - - /// Get the number of buckets in the table. - size_t - bucketCount() const - { - return m_array.n; - } - - /// Enable or disable expanding the table when chains are long. - void - setExpansionPolicy(ExpansionPolicy p) - { - m_expansion_policy = p; - } - /// Get the current expansion policy. - ExpansionPolicy - getExpansionPolicy() const - { - return m_expansion_policy; - } - /// Set the limit value for the expansion policy. - void - setExpansionLimit(size_t n) - { - m_expansion_limit = n; - } - /// Set the limit value for the expansion policy. - size_t - expansionLimit() const - { - return m_expansion_limit; - } - - /** Expand the hash. - - Useful primarily when the expansion policy is set to @c MANUAL. - */ - void expand(); - -protected: - typedef Vec Array; ///< Bucket array. - - size_t m_count; ///< # of elements stored in the table. - ExpansionPolicy m_expansion_policy; ///< When to exand the table. - size_t m_expansion_limit; ///< Limit value for expansion. - Array m_array; ///< Bucket storage. - /// Make available to nested classes statically. - // We must reach inside the link hackery because we're in a template and - // must use typename. Older compilers don't handle typename outside of - // template context so if we put typename in the base definition it won't - // work in non-template classes. - typedef DLL BucketChain; - /// List of non-empty buckets. - BucketChain m_bucket_chain; - - /** Get the ID and bucket for key. - Fills @a m_id and @a m_bucket in @a location from @a key. - */ - void findBucket(Key key, Location &location); - - // noncopyable - TSHashTable(const TSHashTable &) = delete; - TSHashTable &operator=(const TSHashTable &) = delete; -}; - -template -typename TSHashTable::iterator -TSHashTable::begin() -{ - // Get the first non-empty bucket, if any. - Bucket *b = m_bucket_chain.head; - return b && b->m_chain.head ? iterator(b, b->m_chain.head) : this->end(); -} - -template -typename TSHashTable::iterator -TSHashTable::end() -{ - return iterator(nullptr, nullptr); -} - -template -typename TSHashTable::iterator & -TSHashTable::iterator::operator++() -{ - if (m_value) { - if (nullptr == (m_value = ListHead::next(m_value))) { // end of bucket, next bucket. - if (nullptr != (m_bucket = BucketChain::next(m_bucket))) { // found non-empty next bucket. - m_value = m_bucket->m_chain.head; - ink_assert(m_value); // if bucket is in chain, must be non-empty. - } - } - } - return *this; -} - -template -typename TSHashTable::iterator -TSHashTable::iterator::operator++(int) -{ - iterator prev(*this); - ++*this; - return prev; -} - -template -TSHashTable::TSHashTable(size_t nb) : m_count(0), m_expansion_policy(AVERAGE), m_expansion_limit(DEFAULT_EXPANSION_LIMIT) -{ - if (nb) { - int idx = 1; - while (prime2[idx] < nb) - ++idx; - m_array.n = 1; // anything non-zero. - m_array.i = idx - 1; - } - m_array.set_expand(); -} - -template -void -TSHashTable::Location::advance() -{ - Key key = Hasher::key(m_value); - // assumes valid location with correct key, advance to next matching key or make location invalid. - do { - ++m_distance; - m_value = ListHead::next(m_value); - } while (m_value && !Hasher::equal(key, Hasher::key(m_value))); -} - -template -void -TSHashTable::findBucket(Key key, Location &location) -{ - location.m_id = Hasher::hash(key); - location.m_bucket = &(m_array[location.m_id % m_array.n]); -} - -template -typename TSHashTable::Location -TSHashTable::find(Key key) -{ - Location zret; - Value *v; - - this->findBucket(key, zret); // zret gets updated to match the bucket. - v = zret.m_bucket->m_chain.head; - // Search for first matching key. - while (nullptr != v && !Hasher::equal(key, Hasher::key(v))) - v = ListHead::next(v); - zret.m_value = v; - return zret; -} - -template -typename TSHashTable::Location -TSHashTable::find(Value *value) -{ - Location zret; - this->findBucket(Hasher::key(value), zret); - if (zret.m_bucket->m_chain.in(value)) // just checks value links and chain head. - zret.m_value = value; - return zret; -} - -template -void -TSHashTable::insert(Value *value) -{ - Key key = Hasher::key(value); - Bucket *bucket = &(m_array[Hasher::hash(key) % m_array.n]); - - // Bad client if already in a list! - ink_assert(!bucket->m_chain.in(value)); - - // Mark mixed if not already marked and we're adding a different key. - if (!bucket->m_mixed_p && !bucket->m_chain.empty() && !Hasher::equal(key, Hasher::key(bucket->m_chain.head))) - bucket->m_mixed_p = true; - - bucket->m_chain.push(value); - ++m_count; - if (1 == ++(bucket->m_count)) // not empty, put it on the non-empty list. - m_bucket_chain.push(bucket); - // auto expand if appropriate. - if ((AVERAGE == m_expansion_policy && (m_count / m_array.n) > m_expansion_limit) || - (MAXIMUM == m_expansion_policy && bucket->m_count > m_expansion_limit && bucket->m_mixed_p)) - this->expand(); -} - -template -bool -TSHashTable::remove(Location const &l) -{ - bool zret = false; - if (l.isValid()) { - ink_assert(l.m_bucket->m_count); - ink_assert(l.m_bucket->m_chain.head); - l.m_bucket->m_chain.remove(l.m_value); - --m_count; - --(l.m_bucket->m_count); - if (0 == l.m_bucket->m_count) // if it's now empty, take it out of the non-empty bucket chain. - m_bucket_chain.remove(l.m_bucket); - else if (1 == l.m_bucket->m_count) // if count drops to 1, then it's not mixed any more. - l.m_bucket->m_mixed_p = false; - zret = true; - } - return zret; -} - -template -bool -TSHashTable::remove(Key key) -{ - Location loc = this->find(key); - bool zret = loc.isValid(); - while (loc.isValid()) { - Location target(loc); - loc.advance(); - this->remove(target); - } - return zret; -} - -template -void -TSHashTable::clear() -{ - Bucket null_bucket; - // Remove the values but not the actual buckets. - for (size_t i = 0; i < m_array.n; ++i) { - m_array[i] = null_bucket; - } - // Clear container data. - m_count = 0; - m_bucket_chain.clear(); -} - -template -void -TSHashTable::expand() -{ - Bucket *b = m_bucket_chain.head; // stash before reset. - ExpansionPolicy org_expansion_policy = m_expansion_policy; - Array tmp; - tmp.move(m_array); // stash the current array here. - // Reset to empty state. - m_count = 0; - m_bucket_chain.clear(); - - // Because we moved the array, we have to copy back a couple of things to make - // the expansion actually expand. How this is supposed to work without leaks or - // mucking about in the internal is unclear to me. - m_array.n = 1; // anything non-zero. - m_array.i = tmp.i; // set the base index. - m_array.set_expand(); // bumps array size up to next index value. - - m_expansion_policy = MANUAL; // disable any auto expand while we're expanding. - // Move the values from the stashed array to the expanded hash. - while (b) { - Value *v = b->m_chain.head; - while (v) { - b->m_chain.remove(v); // clear local pointers to be safe. - this->insert(v); - v = b->m_chain.head; // next value, because previous was removed. - } - b = BucketChain::next(b); // these buckets are in the stashed array so pointers are still valid. - } - // stashed array gets cleaned up when @a tmp goes out of scope. - m_expansion_policy = org_expansion_policy; // reset to original value. -} - -/* ---------------------------------------------------------------------------------------------- */ diff --git a/include/tscore/MemArena.h b/include/tscore/MemArena.h index 3a3519a0b2e..8d0d8ccd9ae 100644 --- a/include/tscore/MemArena.h +++ b/include/tscore/MemArena.h @@ -168,7 +168,7 @@ class MemArena /** Release all memory. - Empties the entire arena and deallocates all underlying memory. The hint for the next reservered block size will + Empties the entire arena and deallocates all underlying memory. The hint for the next reserved block size will be @a n if @a n is not zero, otherwise it will be the sum of all allocations when this method was called. @return @c *this diff --git a/include/tscore/ParseRules.h b/include/tscore/ParseRules.h index d8fe31e51a7..e69e7378cb6 100644 --- a/include/tscore/ParseRules.h +++ b/include/tscore/ParseRules.h @@ -33,7 +33,7 @@ typedef unsigned int CTypeResult; // Set this to 0 to disable SI // decimal multipliers -#define USE_SI_MULTILIERS 1 +#define USE_SI_MULTIPLIERS 1 #define is_char_BIT (1 << 0) #define is_upalpha_BIT (1 << 1) @@ -136,7 +136,7 @@ class ParseRules ////////////////// static CTypeResult is_escape(const char *seq); // % - static CTypeResult is_uchar(const char *seq); // starts unresrvd or is escape + static CTypeResult is_uchar(const char *seq); // starts unreserved or is escape static CTypeResult is_pchar(const char *seq); // uchar,:,@,&,=,+ (see code) /////////////////// @@ -766,7 +766,7 @@ ParseRules::strlen_eow(const char *s) // // This function is the same as strstr(), except that it accepts strings // that are terminated with '\r', '\n' or null. -// It returns a pointer to the first occurance of s2 within s1 (or null). +// It returns a pointer to the first occurrence of s2 within s1 (or null). ////////////////////////////////////////////////////////////////////////////// inline const char * ParseRules::strstr_eow(const char *s1, const char *s2) diff --git a/include/tscore/PriorityQueue.h b/include/tscore/PriorityQueue.h index 3d2c0d3dc82..e1111da0f03 100644 --- a/include/tscore/PriorityQueue.h +++ b/include/tscore/PriorityQueue.h @@ -28,9 +28,9 @@ #include template struct PriorityQueueEntry { - PriorityQueueEntry(T n) : index(0), node(n){}; - PriorityQueueEntry() : index(0), node(NULL){}; - uint32_t index; + PriorityQueueEntry(T n) : node(n){}; + PriorityQueueEntry() : node(NULL){}; + uint32_t index = 0; T node; }; diff --git a/include/tscore/Ptr.h b/include/tscore/Ptr.h index d067604a9ef..3a79e01b9dd 100644 --- a/include/tscore/Ptr.h +++ b/include/tscore/Ptr.h @@ -44,8 +44,8 @@ struct ForceVFPTToTop { class RefCountObj : public ForceVFPTToTop { public: - RefCountObj() : m_refcount(0) {} - RefCountObj(const RefCountObj &s) : m_refcount(0) + RefCountObj() {} + RefCountObj(const RefCountObj &s) { (void)s; return; @@ -86,7 +86,7 @@ class RefCountObj : public ForceVFPTToTop } private: - int m_refcount; + int m_refcount = 0; }; //////////////////////////////////////////////////////////////////////// @@ -143,7 +143,7 @@ template class Ptr } // Return the raw pointer as a RefCount object. Typically - // this is for keeping a collection of heterogenous objects. + // this is for keeping a collection of ogenous objects. RefCountObj * object() const { diff --git a/include/tscore/RawHashTable.h b/include/tscore/RawHashTable.h deleted file mode 100644 index 4d0cb5141d8..00000000000 --- a/include/tscore/RawHashTable.h +++ /dev/null @@ -1,382 +0,0 @@ -/** @file - - C++ wrapper around libts hash tables - - @section license License - - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - @section details Details - - These C++ RawHashTables are a C++ wrapper around libts hash tables. - They expose an interface very analogous to ink_hash_table, for better - or for worse. See HashTable for a more C++-oriented hash table. - -*/ - -#pragma once - -#include "tscore/ink_apidefs.h" -#include "tscore/ink_hash_table.h" - -////////////////////////////////////////////////////////////////////////////// -// -// Constants and Type Definitions -// -////////////////////////////////////////////////////////////////////////////// - -typedef enum { - RawHashTable_KeyType_String = InkHashTableKeyType_String, - RawHashTable_KeyType_Word = InkHashTableKeyType_Word -} RawHashTable_KeyType; - -typedef InkHashTableKey RawHashTable_Key; -typedef InkHashTableValue RawHashTable_Value; -typedef InkHashTableEntry RawHashTable_Binding; -typedef InkHashTableIteratorState RawHashTable_IteratorState; - -////////////////////////////////////////////////////////////////////////////// -// -// The RawHashTable Class -// -////////////////////////////////////////////////////////////////////////////// - -class RawHashTable -{ -private: - InkHashTable *ht; - RawHashTable_KeyType key_type; - bool deallocate_values_on_destruct; - -public: - inkcoreapi RawHashTable(RawHashTable_KeyType key_type, bool deallocate_values_on_destruct = false); - virtual ~RawHashTable(); - - // - // these are the simplest accessor functions - // - - bool getValue(RawHashTable_Key key, RawHashTable_Value *value_ptr); - void setValue(RawHashTable_Key key, RawHashTable_Value value_ptr); - bool isBound(RawHashTable_Key key); - bool unbindKey(RawHashTable_Key key); - void replaceString(char *key, char *string); - - // - // these functions allow you to manipulate the (key,value) bindings directly - // - - RawHashTable_Binding *getCurrentBinding(RawHashTable_Key key); - RawHashTable_Binding *getOrCreateBinding(RawHashTable_Key key, bool *was_new = nullptr); - - void setBindingValue(RawHashTable_Binding *binding, RawHashTable_Value value); - RawHashTable_Key getKeyFromBinding(RawHashTable_Binding *binding); - RawHashTable_Value getValueFromBinding(RawHashTable_Binding *binding); - - // - // these functions allow you to iterate through RawHashTable bindings - // - - RawHashTable_Binding *firstBinding(RawHashTable_IteratorState *state_ptr); - RawHashTable_Binding *nextBinding(RawHashTable_IteratorState *state_ptr); -}; - -////////////////////////////////////////////////////////////////////////////// -// -// Inline Methods -// -////////////////////////////////////////////////////////////////////////////// - -/** - This routine gets the value associated with key. If the key has a - binding, the value is stored through value_ptr and true is returned. If - the key DOES NOT have a binding, false is returned. - -*/ -inline bool -RawHashTable::getValue(RawHashTable_Key key, RawHashTable_Value *value_ptr) -{ - int is_bound; - - is_bound = ink_hash_table_lookup(ht, (InkHashTableKey)key, (InkHashTableValue *)value_ptr); - return (is_bound ? true : false); -} - -/** - This routine sets the value associated with key to the value. If - a value is previously bound to the key, the previous value is left - dangling. The caller is responsible to freeing any previous binding - values needing freeing before calling setValue. - - If the key has a binding, the value is stored through value_ptr and - true is returned. If the key DOES NOT have a binding, false is returned. - -*/ -inline void -RawHashTable::setValue(RawHashTable_Key key, RawHashTable_Value value) -{ - ink_hash_table_insert(ht, (InkHashTableKey)key, (InkHashTableValue)value); -} - -/** - This routine sets the value associated with key to the value pointed to - by value_ptr. If a value is previously bound to the key, the previous - value is left dangling. The caller is responsible to freeing any - previous value before setValue. - - If the key has a binding, the value is stored through value_ptr and - true is returned. If the key DOES NOT have a binding, false is returned. - -*/ -inline bool -RawHashTable::isBound(RawHashTable_Key key) -{ - int status = ink_hash_table_isbound(ht, (InkHashTableKey)key); - return (status ? true : false); -} - -/** - This routine removes any association for key from the hash table. If - data was bound to key, the binding will be deleted, but the value will - not be deallocated. The caller is responsible to freeing any previous - value before unbindKey. - - @return true if the key was previously bound, false otherwise. - -*/ -inline bool -RawHashTable::unbindKey(RawHashTable_Key key) -{ - int status; - - status = ink_hash_table_delete(ht, (InkHashTableKey)key); - return (status ? true : false); -} - -/** - This rather specialized routine binds a malloc-allocated string value - to the key, freeing any previous value. The key must be a string, - and the hash table must have been constructed as having key_type - RawHashTable_KeyType_String. - -*/ -inline void -RawHashTable::replaceString(char *key, char *string) -{ - // if (key_type != RawHashTable_KeyType_String) - // { - // throw BadKeyType(); - // } - - ink_hash_table_replace_string(ht, key, string); -} - -/** - This function looks up a binding for key in the hash table, and returns - a pointer to the binding data structure directly inside the hash table, - or NULL if there is no binding. - -*/ -inline RawHashTable_Binding * -RawHashTable::getCurrentBinding(RawHashTable_Key key) -{ - InkHashTableEntry *he_ptr; - - he_ptr = ink_hash_table_lookup_entry(ht, (InkHashTableKey)key); - return ((RawHashTable_Binding *)he_ptr); -} - -/** - This function looks up a binding for key in the hash table, creates - a binding if one doesn't exist, and returns a pointer to the binding - data structure directly inside the hash table. - - If was_new is not NULL, true is stored through was_new. If no binding - previously existed, false is stored through was_new if a binding - previously existed. - -*/ -inline RawHashTable_Binding * -RawHashTable::getOrCreateBinding(RawHashTable_Key key, bool *was_new) -{ - int _was_new; - InkHashTableEntry *he_ptr; - - he_ptr = ink_hash_table_get_entry(ht, (InkHashTableKey)key, &_was_new); - *was_new = (_was_new ? true : false); - return ((RawHashTable_Binding *)he_ptr); -} - -/** - This function looks up a binding for key in the hash table, creates - a binding if one doesn't exist, and returns a pointer to the binding - data structure directly inside the hash table. - - If was_new is not NULL, true is stored through was_new. If no binding - previously existed, false is stored through was_new if a binding - previously existed. - -*/ -inline void -RawHashTable::setBindingValue(RawHashTable_Binding *binding, RawHashTable_Value value) -{ - ink_hash_table_set_entry(ht, (InkHashTableEntry *)binding, (InkHashTableValue)value); -} - -/** - This function takes a binding and extracts the key. - -*/ -inline RawHashTable_Key -RawHashTable::getKeyFromBinding(RawHashTable_Binding *binding) -{ - InkHashTableKey ht_key; - - ht_key = ink_hash_table_entry_key(ht, (InkHashTableEntry *)binding); - return ((RawHashTable_Key)ht_key); -} - -/** - This function takes a binding and extracts the value. - -*/ -inline RawHashTable_Value -RawHashTable::getValueFromBinding(RawHashTable_Binding *binding) -{ - InkHashTableValue ht_value; - - ht_value = ink_hash_table_entry_value(ht, (InkHashTableEntry *)binding); - return ((RawHashTable_Value)ht_value); -} - -/** - This function takes a hash table, initializes an interator data - structure to point to the first binding in the hash table, and returns - the first binding, or NULL if there are none. - -*/ -inline RawHashTable_Binding * -RawHashTable::firstBinding(RawHashTable_IteratorState *state_ptr) -{ - InkHashTableEntry *he_ptr; - - he_ptr = ink_hash_table_iterator_first(ht, (InkHashTableIteratorState *)state_ptr); - return ((RawHashTable_Binding *)he_ptr); -} - -inline RawHashTable::RawHashTable(RawHashTable_KeyType akey_type, bool adeallocate_values_on_destruct) -{ - RawHashTable::key_type = akey_type; - RawHashTable::deallocate_values_on_destruct = adeallocate_values_on_destruct; - ht = ink_hash_table_create((InkHashTableKeyType)key_type); -} - -inline RawHashTable::~RawHashTable() -{ - if (deallocate_values_on_destruct) { - ink_hash_table_destroy_and_free_values(ht); - } else { - ink_hash_table_destroy(ht); - } -} - -/** - This function takes a hash table and a pointer to iterator state, - and advances to the next binding in the hash table, if any. If there - in a next binding, a pointer to the binding is returned, else NULL. - -*/ -inline RawHashTable_Binding * -RawHashTable::nextBinding(RawHashTable_IteratorState *state_ptr) -{ - InkHashTableEntry *he_ptr; - - he_ptr = ink_hash_table_iterator_next(ht, (InkHashTableIteratorState *)state_ptr); - return ((RawHashTable_Binding *)he_ptr); -} - -////////////////////////////////////////////////////////////////////////////// -// -// The RawHashTableIter Class -// -////////////////////////////////////////////////////////////////////////////// - -class RawHashTableIter -{ -public: - RawHashTableIter(RawHashTable &ht); - ~RawHashTableIter(); - - RawHashTable_Value &operator++(); // get next - RawHashTable_Value &operator()() const; // get current - operator const void *() const; // is valid - - RawHashTable_Value &value() const; // get current value - const char *key() const; // get current key - -private: - RawHashTable &m_ht; - RawHashTable_Binding *m_currentBinding; - RawHashTable_IteratorState m_hashIterState; -}; - -////////////////////////////////////////////////////////////////////////////// -// -// Inline Methods -// -////////////////////////////////////////////////////////////////////////////// - -inline RawHashTable_Value & -RawHashTableIter::operator()() const -{ - return (m_currentBinding->clientData); -} - -inline RawHashTable_Value & -RawHashTableIter::operator++() -{ - m_currentBinding = m_ht.nextBinding(&m_hashIterState); - return (m_currentBinding->clientData); -} - -inline RawHashTableIter::operator const void *() const -{ - return ((m_currentBinding != nullptr) ? this : nullptr); -} - -inline RawHashTable_Value & -RawHashTableIter::value() const -{ - return (m_currentBinding->clientData); -} - -inline const char * -RawHashTableIter::key() const -{ - return (m_currentBinding->key.string); -} - -inline RawHashTableIter::RawHashTableIter(RawHashTable &ht) : m_ht(ht), m_currentBinding(nullptr) -{ - m_currentBinding = m_ht.firstBinding(&m_hashIterState); - return; -} - -inline RawHashTableIter::~RawHashTableIter() -{ - return; -} diff --git a/include/tscore/RbTree.h b/include/tscore/RbTree.h index 4d936d9dcfd..dcb52f58ec6 100644 --- a/include/tscore/RbTree.h +++ b/include/tscore/RbTree.h @@ -108,7 +108,7 @@ namespace detail int validate(); /// Default constructor. - RBNode() : _color(RED), _parent(nullptr), _left(nullptr), _right(nullptr), _next(nullptr), _prev(nullptr) {} + RBNode() {} /// Destructor (force virtual). virtual ~RBNode() {} /** Rotate the subtree rooted at this node. @@ -206,12 +206,12 @@ namespace detail //! Invoke @c structure_fixup() on this node and all of its ancestors. self *rippleStructureFixup(); - Color _color; ///< node color - self *_parent; ///< parent node (needed for rotations) - self *_left; ///< left child - self *_right; ///< right child - self *_next; ///< Next node. - self *_prev; ///< Previous node. + Color _color = RED; ///< node color + self *_parent = nullptr; ///< parent node (needed for rotations) + self *_left = nullptr; ///< left child + self *_right = nullptr; ///< right child + self *_next = nullptr; ///< Next node. + self *_prev = nullptr; ///< Previous node. }; } /* namespace detail */ diff --git a/include/tscore/Regex.h b/include/tscore/Regex.h index 6c1572c837f..3c95b2728bd 100644 --- a/include/tscore/Regex.h +++ b/include/tscore/Regex.h @@ -23,6 +23,11 @@ #pragma once +#include +#include +#include +#include + #include "tscore/ink_config.h" #ifdef HAVE_PCRE_PCRE_H @@ -31,50 +36,107 @@ #include #endif +/// Match flags for regular expression evaluation. enum REFlags { - RE_CASE_INSENSITIVE = 0x0001, // default is case sensitive - RE_UNANCHORED = 0x0002, // default (for DFA) is to anchor at the first matching position - RE_ANCHORED = 0x0004, // default (for Regex) is unanchored + RE_CASE_INSENSITIVE = 0x0001, ///< Ignore case (default: case sensitive). + RE_UNANCHORED = 0x0002, ///< Unanchored (DFA defaults to anchored). + RE_ANCHORED = 0x0004, ///< Anchored (Regex defaults to unanchored). }; +/** Wrapper for PCRE evaluation. + * + */ class Regex { public: - Regex() : regex(nullptr), regex_extra(nullptr) {} - bool compile(const char *pattern, const unsigned flags = 0); - // It is safe to call exec() concurrently on the same object instance - bool exec(const char *str); - bool exec(const char *str, int length); - bool exec(const char *str, int length, int *ovector, int ovecsize); - int get_capture_count(); + /// Default number of capture groups. + static constexpr size_t DEFAULT_GROUP_COUNT = 10; + + Regex() = default; + Regex(Regex const &) = delete; // No copying. + Regex(Regex &&that) noexcept; ~Regex(); + /** Compile the @a pattern into a regular expression. + * + * @param pattern Source pattern for regular expression (null terminated). + * @param flags Compilation flags. + * @return @a true if compiled successfully, @a false otherwise. + * + * @a flags should be the bitwise @c or of @c REFlags values. + */ + bool compile(const char *pattern, unsigned flags = 0); + + /** Execute the regular expression. + * + * @param str String to match against. + * @return @c true if the patter matched, @a false if not. + * + * It is safe to call this method concurrently on the same instance of @a this. + */ + bool exec(std::string_view const &str); + + /** Execute the regular expression. + * + * @param str String to match against. + * @param ovector Capture results. + * @param ovecsize Number of elements in @a ovector. + * @return @c true if the patter matched, @a false if not. + * + * It is safe to call this method concurrently on the same instance of @a this. + * + * Each capture group takes 3 elements of @a ovector, therefore @a ovecsize must + * be a multiple of 3 and at least three times the number of desired capture groups. + */ + bool exec(std::string_view const &str, int *ovector, int ovecsize); + + /// @return The number of groups captured in the last call to @c exec. + int get_capture_count(); + private: - pcre *regex; - pcre_extra *regex_extra; + pcre *regex = nullptr; + pcre_extra *regex_extra = nullptr; }; -typedef struct __pat { - int _idx; - Regex *_re; - char *_p; - __pat *_next; -} dfa_pattern; - +/** Deterministic Finite state Automata container. + * + * This contains a set of patterns (which may be of size 1) and matches if any of the patterns + * match. + */ class DFA { public: - DFA() : _my_patterns(nullptr) {} + DFA() = default; ~DFA(); - int compile(const char *pattern, unsigned flags = 0); + /// @return The number of patterns successfully compiled. + int compile(std::string_view const &pattern, unsigned flags = 0); + /// @return The number of patterns successfully compiled. + int compile(std::string_view *patterns, int npatterns, unsigned flags = 0); + /// @return The number of patterns successfully compiled. int compile(const char **patterns, int npatterns, unsigned flags = 0); - int match(const char *str) const; - int match(const char *str, int length) const; + /** Match @a str against the internal patterns. + * + * @param str String to match. + * @return Index of the matched pattern, -1 if no match. + */ + int match(std::string_view const &str) const; private: - dfa_pattern *build(const char *pattern, unsigned flags = 0); - - dfa_pattern *_my_patterns; + struct Pattern { + Pattern(Regex &&rxp, std::string &&s) : _re(std::move(rxp)), _p(std::move(s)) {} + Regex _re; ///< The compile pattern. + std::string _p; ///< The original pattern. + }; + + /** Compile @a pattern and add it to the pattern set. + * + * @param pattern Regular expression to compile. + * @param flags Regular expression compilation flags. + * @return @c true if @a pattern was successfully compiled, @c false if not. + */ + bool build(std::string_view const &pattern, unsigned flags = 0); + + std::vector _patterns; }; diff --git a/include/tscore/Scalar.h b/include/tscore/Scalar.h index e71a0824da0..5eb0dffa463 100644 --- a/include/tscore/Scalar.h +++ b/include/tscore/Scalar.h @@ -44,19 +44,19 @@ template class Scalar; namespace detail { - // @internal - althought these conversion methods look bulky, in practice they compile down to - // very small amounts of code due to dead code elimination and that all of the conditions are - // compile time constants. + // @internal - although these conversion methods look bulky, in practice they compile down to + // very small amounts of code due to the conditions being compile time constant - the non-taken + // clauses are dead code and eliminated by the compiler. // The general case where neither N nor S are a multiple of the other seems a bit long but this // minimizes the risk of integer overflow. I need to validate that under -O2 the compiler will // only do 1 division to get both the quotient and remainder for (n/N) and (n%N). In cases where // N,S are powers of 2 I have verified recent GNU compilers will optimize to bit operations. - /// Convert a count @a c that is scale @s S to scale @c N - template - intmax_t - scale_conversion_round_up(intmax_t c) + /// Convert a count @a c that is scale @s S to the corresponding count for scale @c N + template + C + scale_conversion_round_up(C c) { typedef std::ratio R; if (N == S) { @@ -71,9 +71,9 @@ namespace detail } /// Convert a count @a c that is scale @s S to scale @c N - template - intmax_t - scale_conversion_round_down(intmax_t c) + template + C + scale_conversion_round_down(C c) { typedef std::ratio R; if (N == S) { @@ -108,6 +108,10 @@ namespace detail Much of this is driven by the fact that the assignment operator, in some cases, can not be templated and therefore to have a nice interface for assignment this split is needed. + + Note - the key point is the actual conversion is not done when the wrapper instance is created + but when the wrapper instance is assigned. That is what enables the conversion to be done in + the context of the destination, which is not otherwise possible. */ // Unit value, to be rounded up. @@ -220,7 +224,7 @@ template class Scalar static_assert(N > 0, "The scaling factor (1st template argument) must be a positive integer"); static_assert(std::is_integral::value, "The counter type (2nd template argument) must be an integral type"); - constexpr Scalar(); ///< Default contructor. + constexpr Scalar(); ///< Default constructor. ///< Construct to have @a n scaled units. explicit constexpr Scalar(Counter n); /// Copy constructor. @@ -868,7 +872,7 @@ template Scalar operator/(Scalar lhs, I n) { - static_assert(std::is_integral::value, "Scalar divsion only support integral types."); + static_assert(std::is_integral::value, "Scalar division only support integral types."); return Scalar(lhs) /= n; } @@ -893,6 +897,20 @@ Scalar::minus(Counter n) const -> self return {_n - n}; } +template +C +round_up(C value) +{ + return N * detail::scale_conversion_round_up(value); +} + +template +C +round_down(C value) +{ + return N * detail::scale_conversion_round_down(value); +} + namespace detail { // These classes exist only to create distinguishable overloads. diff --git a/include/tscore/SimpleTokenizer.h b/include/tscore/SimpleTokenizer.h index 3f0e7fad3d0..82a257fff9f 100644 --- a/include/tscore/SimpleTokenizer.h +++ b/include/tscore/SimpleTokenizer.h @@ -123,14 +123,13 @@ class SimpleTokenizer OVERWRITE_INPUT_STRING = 8 }; - SimpleTokenizer(char delimiter = ' ', unsigned mode = 0, char escape = '\\') - : _data(nullptr), _delimiter(delimiter), _mode(mode), _escape(escape), _start(0), _length(0) + SimpleTokenizer(char delimiter = ' ', unsigned mode = 0, char escape = '\\') : _delimiter(delimiter), _mode(mode), _escape(escape) { } - // NOTE: The input strring 's' is overwritten for mode OVERWRITE_INPUT_STRING. + // NOTE: The input string 's' is overwritten for mode OVERWRITE_INPUT_STRING. SimpleTokenizer(const char *s, char delimiter = ' ', unsigned mode = 0, char escape = '\\') - : _data(nullptr), _delimiter(delimiter), _mode(mode), _escape(escape) + : _delimiter(delimiter), _mode(mode), _escape(escape) { setString(s); } @@ -188,15 +187,15 @@ class SimpleTokenizer } private: - char *_data; // a pointer to the input data itself, + char *_data = nullptr; // a pointer to the input data itself, // or to a copy of it char _delimiter; // the token delimiter unsigned _mode; // flags that determine the // mode of operation - char _escape; // the escape character - size_t _start; // pointer to the start of the next + char _escape; // the escape character + size_t _start = 0; // pointer to the start of the next // token - size_t _length; // the length of _data + size_t _length = 0; // the length of _data void _clearData() diff --git a/include/tscore/TSSystemState.h b/include/tscore/TSSystemState.h new file mode 100644 index 00000000000..cc5bf3d1229 --- /dev/null +++ b/include/tscore/TSSystemState.h @@ -0,0 +1,92 @@ +/** + @file TSSystemState.h + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +*/ + +#pragma once + +#include +#include + +// Global status information for trafficserver +// +class TSSystemState +{ +private: + struct Data { + bool ssl_handshaking_stopped; + bool event_system_shut_down; + bool draining; + }; + +public: + static bool + is_ssl_handshaking_stopped() + { + return unlikely(_instance().ssl_handshaking_stopped); + } + + static bool + is_event_system_shut_down() + { + return unlikely(_instance().event_system_shut_down); + } + + // Keeps track if the server is in draining state, follows the proxy.node.config.draining metric. + // + static bool + is_draining() + { + return unlikely(_instance().draining); + } + + static void + stop_ssl_handshaking() + { + ink_assert(!_instance().ssl_handshaking_stopped); + + _instance().ssl_handshaking_stopped = true; + } + + static void + shut_down_event_system() + { + // For some reason this is triggered by the regression testing. + // ink_assert(_instance().ssl_handshaking_stopped && !_instance().event_system_shut_down); + + _instance().event_system_shut_down = true; + } + + static void + drain(bool enable) + { + _instance().draining = enable; + } + +private: + static Data & + _instance() + { + static Data d; + + return d; + } +}; diff --git a/include/tscore/TextBuffer.h b/include/tscore/TextBuffer.h index 8ef01debe41..65dd1ec30a4 100644 --- a/include/tscore/TextBuffer.h +++ b/include/tscore/TextBuffer.h @@ -25,7 +25,7 @@ /**************************************************************************** * - * TextBuffer.h - A self-expanding buffer, primarly meant for strings + * TextBuffer.h - A self-expanding buffer, primarily meant for strings * * * diff --git a/include/tscore/Tokenizer.h b/include/tscore/Tokenizer.h index 562279b0f1f..a86c96abc0a 100644 --- a/include/tscore/Tokenizer.h +++ b/include/tscore/Tokenizer.h @@ -43,26 +43,26 @@ * * There are three memory options. * SHARE_TOKS - this modifies the original string passed in - * through Intialize() and shares its space. NULLs + * through Initialize() and shares its space. NULLs * are inserted into string after each token. Choosing - * this option means the user is reponsible for not + * this option means the user is responsible for not * deallocating the string storage before deallocating * the tokenizer object - * COPY_TOKS - this option copies the orginial string and + * COPY_TOKS - this option copies the original string and * leaves the original unchanged. The deallocation of the * original string and the deallocation of the Tokenizer * object are now independent. * Note: If neither SHARE_TOKS or COPY_TOKS is selected, COPY_TOKS * is the default * ALLOW_EMPTY_TOKENS: If multiple delimiters appear next to each - * other, each delimiter creates a token someof which + * other, each delimiter creates a token some of which * will be zero length. The default is to skip repeated * delimiters * * Tokenizer(const char* StrOfDelimit) - a string that contains * the delimiters for tokenizing. This string is copied. * - * Intialize(char* str, TokenizerOpts opt) - Submits a string + * Initialize(char* str, TokenizerOpts opt) - Submits a string * to be tokenized according to the memory options listed above * * ReUse() - Allows the object to be reused for a new string @@ -76,7 +76,7 @@ * is intended to be used on a small number of tokens * * iterFirst(tok_iter_state* state) - Returns the first - * token and intializes state argument for subsequent + * token and initializes state argument for subsequent * calls to iterNext. If no tokens exist, NULL is * returned * diff --git a/include/tscore/TsBuffer.h b/include/tscore/TsBuffer.h index 2ccb9a498b4..7837c64352b 100644 --- a/include/tscore/TsBuffer.h +++ b/include/tscore/TsBuffer.h @@ -33,6 +33,7 @@ // For memcmp() #include +#include /// Apache Traffic Server commons. namespace ts @@ -49,8 +50,8 @@ struct Buffer { typedef Buffer self; ///< Self reference type. typedef bool (self::*pseudo_bool)() const; - char *_ptr; ///< Pointer to base of memory chunk. - size_t _size; ///< Size of memory chunk. + char *_ptr = nullptr; ///< Pointer to base of memory chunk. + size_t _size = 0; ///< Size of memory chunk. /// Default constructor (empty buffer). Buffer(); @@ -129,8 +130,8 @@ struct ConstBuffer { typedef ConstBuffer self; ///< Self reference type. typedef bool (self::*pseudo_bool)() const; - char const *_ptr; ///< Pointer to base of memory chunk. - size_t _size; ///< Size of memory chunk. + char const *_ptr = nullptr; ///< Pointer to base of memory chunk. + size_t _size = 0; ///< Size of memory chunk. /// Default constructor (empty buffer). ConstBuffer(); @@ -194,6 +195,8 @@ struct ConstBuffer { /// @return @c true if the buffer has a non-zero pointer @b and size. operator pseudo_bool() const; + operator std::string_view() const { return {_ptr, _size}; } + /// @name Accessors. //@{ /// Get the data in the buffer. @@ -286,7 +289,7 @@ struct ConstBuffer { // ---------------------------------------------------------- // Inline implementations. -inline Buffer::Buffer() : _ptr(nullptr), _size(0) {} +inline Buffer::Buffer() {} inline Buffer::Buffer(char *ptr, size_t n) : _ptr(ptr), _size(n) {} inline Buffer & Buffer::set(char *ptr, size_t n) @@ -353,7 +356,7 @@ Buffer::size() const return _size; } -inline ConstBuffer::ConstBuffer() : _ptr(nullptr), _size(0) {} +inline ConstBuffer::ConstBuffer() {} inline ConstBuffer::ConstBuffer(char const *ptr, size_t n) : _ptr(ptr), _size(n) {} inline ConstBuffer::ConstBuffer(char const *start, char const *end) : _ptr(start), _size(end - start) {} inline ConstBuffer::ConstBuffer(Buffer const &that) : _ptr(that._ptr), _size(that._size) {} diff --git a/include/tscore/bwf_std_format.h b/include/tscore/bwf_std_format.h index ce7f4f7080a..1e4a26741e6 100644 --- a/include/tscore/bwf_std_format.h +++ b/include/tscore/bwf_std_format.h @@ -121,7 +121,8 @@ namespace bwf } } }; -} // namespace bwf + +}; // namespace bwf BufferWriter &bwformat(BufferWriter &w, BWFSpec const &spec, bwf::Errno const &e); BufferWriter &bwformat(BufferWriter &w, BWFSpec const &spec, bwf::Date const &date); diff --git a/include/tscore/hugepages.h b/include/tscore/hugepages.h index cb7e95b29b3..9735ee52f8a 100644 --- a/include/tscore/hugepages.h +++ b/include/tscore/hugepages.h @@ -22,8 +22,8 @@ #include -size_t ats_hugepage_size(void); -bool ats_hugepage_enabled(void); +size_t ats_hugepage_size(); +bool ats_hugepage_enabled(); void ats_hugepage_init(int); void *ats_alloc_hugepage(size_t); bool ats_free_hugepage(void *, size_t); diff --git a/include/tscore/ink_aiocb.h b/include/tscore/ink_aiocb.h index ac50fbd9cd0..3a7b30e7505 100644 --- a/include/tscore/ink_aiocb.h +++ b/include/tscore/ink_aiocb.h @@ -46,14 +46,9 @@ struct ink_aiocb { void *aio_buf; /* buffer location */ #endif size_t aio_nbytes; /* length of transfer */ + off_t aio_offset; /* file offset */ - // TODO change to off_t - off_t aio_offset; /* file offset */ - - int aio_reqprio; /* request priority offset */ - // struct sigevent aio_sigevent; /* signal number and offset */ int aio_lio_opcode; /* listio operation */ - // aio_result_t aio_resultp; /* results */ - int aio_state; /* state flag for List I/O */ - int aio__pad[1]; /* extension padding */ + int aio_state; /* state flag for List I/O */ + int aio__pad[1]; /* extension padding */ }; diff --git a/include/tscore/ink_base64.h b/include/tscore/ink_base64.h index b536fd6d1bd..a3c4d510afc 100644 --- a/include/tscore/ink_base64.h +++ b/include/tscore/ink_base64.h @@ -25,7 +25,7 @@ /* * Base64 encoding and decoding as according to RFC1521. Similar to uudecode. - * See RFC1521 for specificiation. + * See RFC1521 for specification. * * RFC 1521 requires inserting line breaks for long lines. The basic web * authentication scheme does not require them. This implementation is diff --git a/include/tscore/ink_cap.h b/include/tscore/ink_cap.h index c5ed681ef57..6ffaccbbf36 100644 --- a/include/tscore/ink_cap.h +++ b/include/tscore/ink_cap.h @@ -52,7 +52,7 @@ extern FILE *elevating_fopen(const char *path, const char *mode); // chmod a file, elevating if necessary extern int elevating_chmod(const char *path, int perm); -/// @c stat a file, evelating only if needed. +/// @c stat a file, elevating only if needed. extern int elevating_stat(const char *path, struct stat *buff); /** Control generate of core file on crash. @@ -90,7 +90,7 @@ class ElevateAccess void demote(); private: - bool elevated; + bool elevated = false; uid_t saved_uid; unsigned level; diff --git a/include/tscore/ink_code.h b/include/tscore/ink_code.h index 2b08574f064..c10dd1e2af4 100644 --- a/include/tscore/ink_code.h +++ b/include/tscore/ink_code.h @@ -32,7 +32,7 @@ typedef MD5_CTX INK_DIGEST_CTX; /* - Wrappers around the MD5 functions, all of this should be depericated and just use the functions directly + Wrappers around the MD5 functions, all of this should be deprecated and just use the functions directly */ inkcoreapi int ink_code_md5(unsigned const char *input, int input_length, unsigned char *sixteen_byte_hash_pointer); diff --git a/include/tscore/ink_config.h.in b/include/tscore/ink_config.h.in index d6c1a54ec85..53b02db899b 100644 --- a/include/tscore/ink_config.h.in +++ b/include/tscore/ink_config.h.in @@ -68,13 +68,10 @@ #define TS_HAS_SO_MARK @has_so_mark@ #define TS_HAS_IP_TOS @has_ip_tos@ #define TS_USE_HWLOC @use_hwloc@ -#define TS_USE_TLS_NPN @use_tls_npn@ -#define TS_USE_TLS_ALPN @use_tls_alpn@ #define TS_USE_TLS_ASYNC @use_tls_async@ -#define TS_USE_CERT_CB @use_cert_cb@ +#define TS_USE_HELLO_CB @use_hello_cb@ #define TS_USE_SET_RBIO @use_set_rbio@ #define TS_USE_GET_DH_2048_256 @use_dh_get_2048_256@ -#define TS_USE_TLS_ECKEY @use_tls_eckey@ #define TS_USE_TLS_SET_CIPHERSUITES @use_tls_set_ciphersuites@ #define TS_USE_LINUX_NATIVE_AIO @use_linux_native_aio@ #define TS_USE_REMOTE_UNWINDING @use_remote_unwinding@ @@ -121,9 +118,11 @@ #define TS_BUILD_CACHEDIR "@rel_cachedir@" #define TS_BUILD_INFODIR "@rel_infodir@" +#define TS_ABS_TOP_SRCDIR "@abs_top_srcdir@" + #define TS_BUILD_CANONICAL_HOST "@host@" #define TS_BUILD_DEFAULT_LOOPBACK_IFACE "@default_loopback_iface@" -/* clang-format on */ static const int DEFAULT_STACKSIZE = @default_stack_size@; +/* clang-format on */ diff --git a/include/tscore/ink_defs.h b/include/tscore/ink_defs.h index 20c80b53487..1b81e799e09 100644 --- a/include/tscore/ink_defs.h +++ b/include/tscore/ink_defs.h @@ -24,17 +24,17 @@ #pragma once #include "tscore/ink_config.h" -#include +#include // NOLINT(modernize-deprecated-headers) #include #ifdef HAVE_STDINT_H -#include +#include // NOLINT(modernize-deprecated-headers) #else // TODO: Add "standard" int types? #endif #ifdef HAVE_INTTYPES_H -#include +#include // NOLINT(modernize-deprecated-headers) #else // TODO: add PRI*64 stuff? #endif @@ -130,12 +130,3 @@ int ink_login_name_max(); // Get the hardware topology hwloc_topology_t ink_get_topology(); #endif - -/** Constants. - */ -#ifdef __cplusplus -namespace ts -{ -static const int NO_FD = -1; ///< No or invalid file descriptor. -} -#endif diff --git a/include/tscore/ink_endian.h b/include/tscore/ink_endian.h index 30b3132598f..cee79d45b1b 100644 --- a/include/tscore/ink_endian.h +++ b/include/tscore/ink_endian.h @@ -1,6 +1,6 @@ /** @file * - * Endian convertion routines + * Endian conversion routines * * @section license License * diff --git a/include/tscore/ink_error.h b/include/tscore/ink_error.h index 39dbfacbc7c..0e4b4bbfab4 100644 --- a/include/tscore/ink_error.h +++ b/include/tscore/ink_error.h @@ -31,8 +31,8 @@ #pragma once -#include -#include +#include +#include #include "tscore/ink_platform.h" #include "tscore/ink_apidefs.h" diff --git a/include/tscore/ink_file.h b/include/tscore/ink_file.h index fb0487c6985..6cb656f4583 100644 --- a/include/tscore/ink_file.h +++ b/include/tscore/ink_file.h @@ -45,7 +45,7 @@ #include #endif -// Darwin keeps statafs(2) in ... +// Darwin keeps statfs(2) in ... #if HAVE_SYS_MOUNT_H #include #endif diff --git a/include/tscore/ink_hash_table.h b/include/tscore/ink_hash_table.h deleted file mode 100644 index e85df7a124e..00000000000 --- a/include/tscore/ink_hash_table.h +++ /dev/null @@ -1,145 +0,0 @@ -/** @file - - A brief file description - - @section license License - - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - */ - -/**************************************************************************** - - ink_hash_table.h - - This file implements hash tables. This allows us to provide alternative - implementations of hash tables. - - ****************************************************************************/ - -#pragma once - -#ifdef __cplusplus -extern "C" { -#endif /* __cplusplus */ -#include "tscore/ink_apidefs.h" - -/*===========================================================================* - - This is the Tcl implementation of InkHashTable - - *===========================================================================*/ - -#include - -typedef Tcl_HashTable InkHashTable; -typedef Tcl_HashEntry InkHashTableEntry; -typedef char *InkHashTableKey; -typedef ClientData InkHashTableValue; -typedef Tcl_HashSearch InkHashTableIteratorState; - -typedef int (*InkHashTableEntryFunction)(InkHashTable *ht, InkHashTableEntry *entry); - -/* Types of InkHashTable Keys */ - -typedef enum { - InkHashTableKeyType_String, - InkHashTableKeyType_Word, -} InkHashTableKeyType; - -/*===========================================================================* - - Function Prototypes - - *===========================================================================*/ - -InkHashTable *ink_hash_table_create(InkHashTableKeyType key_type); -InkHashTable *ink_hash_table_destroy(InkHashTable *ht_ptr); -InkHashTable *ink_hash_table_destroy_and_free_values(InkHashTable *ht_ptr); -inkcoreapi int ink_hash_table_isbound(InkHashTable *ht_ptr, const char *key); -inkcoreapi int ink_hash_table_lookup(InkHashTable *ht_ptr, const char *key, InkHashTableValue *value_ptr); -inkcoreapi int ink_hash_table_delete(InkHashTable *ht_ptr, const char *key); -InkHashTableEntry *ink_hash_table_lookup_entry(InkHashTable *ht_ptr, const char *key); -InkHashTableEntry *ink_hash_table_get_entry(InkHashTable *ht_ptr, const char *key, int *new_value); -void ink_hash_table_set_entry(InkHashTable *ht_ptr, InkHashTableEntry *he_ptr, InkHashTableValue value); -inkcoreapi void ink_hash_table_insert(InkHashTable *ht_ptr, const char *key, InkHashTableValue value); -void ink_hash_table_map(InkHashTable *ht_ptr, InkHashTableEntryFunction map); -InkHashTableKey ink_hash_table_entry_key(InkHashTable *ht_ptr, InkHashTableEntry *entry_ptr); -inkcoreapi InkHashTableValue ink_hash_table_entry_value(InkHashTable *ht_ptr, InkHashTableEntry *entry_ptr); -void ink_hash_table_dump_strings(InkHashTable *ht_ptr); -void ink_hash_table_replace_string(InkHashTable *ht_ptr, InkHashTableKey key, char *str); - -/* inline functions declarations */ - -/*---------------------------------------------------------------------------* - - InkHashTableEntry *ink_hash_table_iterator_first - (InkHashTable *ht_ptr, InkHashTableIteratorState *state_ptr) - - This routine takes a hash table , creates some iterator state - stored through , and returns a pointer to the first hash table - entry. The iterator state is used by InkHashTableIteratorNext() to proceed - through the rest of the entries. - - *---------------------------------------------------------------------------*/ - -static inline InkHashTableEntry * -ink_hash_table_iterator_first(InkHashTable *ht_ptr, InkHashTableIteratorState *state_ptr) -{ - Tcl_HashTable *tcl_ht_ptr; - Tcl_HashSearch *tcl_search_state_ptr; - Tcl_HashEntry *tcl_he_ptr; - InkHashTableEntry *he_ptr; - - tcl_ht_ptr = (Tcl_HashTable *)ht_ptr; - tcl_search_state_ptr = (Tcl_HashSearch *)state_ptr; - - tcl_he_ptr = Tcl_FirstHashEntry(tcl_ht_ptr, tcl_search_state_ptr); - he_ptr = (InkHashTableEntry *)tcl_he_ptr; - - return (he_ptr); -} /* End ink_hash_table_iterator_first */ - -/*---------------------------------------------------------------------------* - - InkHashTableEntry *ink_hash_table_iterator_next(InkHashTable *ht_ptr, - InkHashTableIteratorState *state_ptr) - - This routine takes a hash table and a pointer to iterator state - initialized by a previous call to InkHashTableIteratorFirst(), and returns - a pointer to the next InkHashTableEntry, or nullptr if no entries remain. - - *---------------------------------------------------------------------------*/ - -static inline InkHashTableEntry * -ink_hash_table_iterator_next(InkHashTable *ht_ptr, InkHashTableIteratorState *state_ptr) -{ - (void)ht_ptr; - Tcl_HashSearch *tcl_search_state_ptr; - Tcl_HashEntry *tcl_he_ptr; - InkHashTableEntry *he_ptr; - - tcl_search_state_ptr = (Tcl_HashSearch *)state_ptr; - - tcl_he_ptr = Tcl_NextHashEntry(tcl_search_state_ptr); - he_ptr = (InkHashTableEntry *)tcl_he_ptr; - - return (he_ptr); -} /* End ink_hash_table_iterator_next */ - -#ifdef __cplusplus -} -#endif /* __cplusplus */ diff --git a/include/tscore/ink_hrtime.h b/include/tscore/ink_hrtime.h index a1405a98c74..9358b6358db 100644 --- a/include/tscore/ink_hrtime.h +++ b/include/tscore/ink_hrtime.h @@ -33,6 +33,7 @@ #include "tscore/ink_config.h" #include "tscore/ink_assert.h" #include +#include #include #include typedef int64_t ink_hrtime; @@ -43,7 +44,7 @@ char *int64_to_str(char *buf, unsigned int buf_size, int64_t val, unsigned int * ////////////////////////////////////////////////////////////////////////////// // -// Factors to multiply units by to obtain coresponding ink_hrtime values. +// Factors to multiply units by to obtain corresponding ink_hrtime values. // ////////////////////////////////////////////////////////////////////////////// @@ -80,7 +81,7 @@ char *int64_to_str(char *buf, unsigned int buf_size, int64_t val, unsigned int * #define HRTIME_USECONDS(_x) ((_x)*HRTIME_USECOND) #define HRTIME_NSECONDS(_x) ((_x)*HRTIME_NSECOND) -// gratuituous wrappers +// gratuitous wrappers static inline ink_hrtime ink_hrtime_from_years(unsigned int years) diff --git a/include/tscore/ink_inet.h b/include/tscore/ink_inet.h index 4cbf14c1d41..ff6c10406e5 100644 --- a/include/tscore/ink_inet.h +++ b/include/tscore/ink_inet.h @@ -117,7 +117,7 @@ union IpEndpoint { in_port_t &port(); /// Port in network order. in_port_t port() const; - /// Port in host horder. + /// Port in host order. in_port_t host_order_port() const; operator sockaddr *() { return &sa; } @@ -1149,21 +1149,20 @@ struct IpAddr { typedef IpAddr self; ///< Self reference type. /// Default construct (invalid address). - IpAddr() : _family(AF_UNSPEC) {} - /// Construct as IPv4 @a addr. - explicit IpAddr(in_addr_t addr ///< Address to assign. - ) - : _family(AF_INET) - { - _addr._ip4 = addr; - } - /// Construct as IPv6 @a addr. - explicit IpAddr(in6_addr const &addr ///< Address to assign. - ) - : _family(AF_INET6) - { - _addr._ip6 = addr; - } + IpAddr() {} + + /** Construct from IPv4 address. + * + * @param addr Source address. + */ + explicit constexpr IpAddr(in_addr_t addr) : _family(AF_INET), _addr(addr) {} + + /** Construct from IPv6 address. + * + * @param addr Source address. + */ + explicit constexpr IpAddr(in6_addr const &addr) : _family(AF_INET6), _addr(addr) {} + /// Construct from @c sockaddr. explicit IpAddr(sockaddr const *addr) { this->assign(addr); } /// Construct from @c sockaddr_in6. @@ -1282,14 +1281,22 @@ struct IpAddr { /// Test for loopback bool isLoopback() const; - uint16_t _family; ///< Protocol family. + /// Test for any addr + bool isAnyAddr() const; + + uint16_t _family = AF_UNSPEC; ///< Protocol family. /// Address data. - union { + union Addr { in_addr_t _ip4; ///< IPv4 address storage. in6_addr _ip6; ///< IPv6 address storage. uint8_t _byte[TS_IP6_SIZE]; ///< As raw bytes. uint32_t _u32[TS_IP6_SIZE / (sizeof(uint32_t) / sizeof(uint8_t))]; ///< As 32 bit chunks. uint64_t _u64[TS_IP6_SIZE / (sizeof(uint64_t) / sizeof(uint8_t))]; ///< As 64 bit chunks. + + // This is required by the @c constexpr constructor. + constexpr Addr() : _ip4(0) {} + constexpr Addr(in_addr_t addr) : _ip4(addr) {} + constexpr Addr(in6_addr const &addr) : _ip6(addr) {} } _addr; ///< Pre-constructed invalid instance. @@ -1341,6 +1348,12 @@ IpAddr::isLoopback() const return (AF_INET == _family && 0x7F == _addr._byte[0]) || (AF_INET6 == _family && IN6_IS_ADDR_LOOPBACK(&_addr._ip6)); } +inline bool +IpAddr::isAnyAddr() const +{ + return (AF_INET == _family && INADDR_ANY == _addr._ip4) || (AF_INET6 == _family && IN6_IS_ADDR_UNSPECIFIED(&_addr._ip6)); +} + /// Assign sockaddr storage. inline IpAddr & IpAddr::assign(sockaddr const *addr) @@ -1558,4 +1571,14 @@ bwformat(BufferWriter &w, BWFSpec const &spec, IpEndpoint const &addr) { return bwformat(w, spec, &addr.sa); } + +namespace bwf +{ + namespace detail + { + struct MemDump; + } // namespace detail + + detail::MemDump Hex_Dump(IpEndpoint const &addr); +} // namespace bwf } // namespace ts diff --git a/include/tscore/ink_llqueue.h b/include/tscore/ink_llqueue.h index 4f86901f6c8..0241da521ba 100644 --- a/include/tscore/ink_llqueue.h +++ b/include/tscore/ink_llqueue.h @@ -43,7 +43,7 @@ typedef struct llq_s { ink_semaphore sema; } LLQ; -LLQ *create_queue(void); +LLQ *create_queue(); int enqueue(LLQ *q, void *data); void *dequeue(LLQ *q); bool queue_is_empty(LLQ *q); diff --git a/include/tscore/ink_lockfile.h b/include/tscore/ink_lockfile.h index dea4d97dae6..dab70612374 100644 --- a/include/tscore/ink_lockfile.h +++ b/include/tscore/ink_lockfile.h @@ -33,10 +33,10 @@ class Lockfile { public: - Lockfile(void) : fd(0) { fname[0] = '\0'; } + Lockfile() { fname[0] = '\0'; } // coverity[uninit_member] - Lockfile(const char *filename) : fd(0) { ink_strlcpy(fname, filename, sizeof(fname)); } - ~Lockfile(void) {} + Lockfile(const char *filename) { ink_strlcpy(fname, filename, sizeof(fname)); } + ~Lockfile() {} void SetLockfileName(const char *filename) { @@ -44,7 +44,7 @@ class Lockfile } const char * - GetLockfileName(void) + GetLockfileName() { return fname; } @@ -69,7 +69,7 @@ class Lockfile // Close() // // Closes the file handle on the opened Lockfile. - void Close(void); + void Close(); // Kill() // KillGroup() @@ -79,12 +79,12 @@ class Lockfile // If the lock file open succeeds, it closes the lock file releasing // the lock. // - // The intial signal can be used to generate a core from the process while + // The initial signal can be used to generate a core from the process while // still ensuring it dies. void Kill(int sig, int initial_sig = 0, const char *pname = nullptr); void KillGroup(int sig, int initial_sig = 0, const char *pname = nullptr); private: char fname[PATH_NAME_MAX]; - int fd; + int fd = 0; }; diff --git a/include/tscore/ink_memory.h b/include/tscore/ink_memory.h index cf850c0b9be..d02411139ea 100644 --- a/include/tscore/ink_memory.h +++ b/include/tscore/ink_memory.h @@ -109,7 +109,7 @@ void *ats_track_malloc(size_t size, uint64_t *stat); void *ats_track_realloc(void *ptr, size_t size, uint64_t *alloc_stat, uint64_t *free_stat); void ats_track_free(void *ptr, uint64_t *stat); -static inline size_t __attribute__((const)) ats_pagesize(void) +static inline size_t __attribute__((const)) ats_pagesize() { static size_t page_size; @@ -280,10 +280,18 @@ class ats_scoped_resource /// Construct with contained resource. explicit ats_scoped_resource(value_type rt) : _r(rt) {} /// Destructor. - ~ats_scoped_resource() + ~ats_scoped_resource() { this->clear(); } + + /** Remove and clean up any existing resource in this object, and return to a default constructed state. + * + */ + void + clear() { - if (Traits::isValid(_r)) + if (Traits::isValid(_r)) { Traits::destroy(_r); + } + _r = Traits::initValue(); } /// Automatic conversion to resource type. @@ -326,8 +334,7 @@ class ats_scoped_resource self & operator=(value_type rt) { - if (Traits::isValid(_r)) - Traits::destroy(_r); + this->clear(); _r = rt; return *this; } @@ -473,60 +480,74 @@ struct SCOPED_OBJECT_TRAITS { class ats_scoped_str : public ats_scoped_resource> { public: - typedef ats_scoped_resource> super; ///< Super type. - typedef ats_scoped_str self; ///< Self reference type. - - /// Default constructor (no string). - ats_scoped_str() {} - /// Construct and allocate @a n bytes for a string. + using super = ats_scoped_resource>; ///< Super type. + using self_type = ats_scoped_str; ///< Self reference type. + + /** Default constructor. + * Constructs an empty container. + */ + ats_scoped_str() = default; + + /** Construct an empty buffer of @a n bytes. + * + * @param n Size of the contained buffer. + * + * The memory is left uninitialized, presumably ready for a @c memcpy. + */ explicit ats_scoped_str(size_t n) : super(static_cast(ats_malloc(n))) {} - /// Put string @a s in this container for cleanup. + + /** Construct with an existing buffer. + * + * @param s The string. + * + * This container takes ownership of the string memory. For this reason @a s must be independently + * allocated, not part of a larger memory allocation. + */ explicit ats_scoped_str(char *s) : super(s) {} - // constructor with std::string - explicit ats_scoped_str(const std::string &s) - { - if (s.empty()) - _r = nullptr; - else - _r = strdup(s.c_str()); - } - // constructor with string_view - explicit ats_scoped_str(const std::string_view &s) - { - if (s.empty()) - _r = nullptr; - else - _r = strdup(s.data()); - } - /// Assign a string @a s to this container. - self & - operator=(char *s) - { - super::operator=(s); - return *this; - } - // std::string case - self & - operator=(const std::string &s) - { - if (s.empty()) - _r = nullptr; - else - _r = strdup(s.c_str()); - return *this; - } - // string_view case - self & - operator=(const std::string_view &s) - { - if (s.empty()) - _r = nullptr; - else - _r = strdup(s.data()); - return *this; - } + + /** Construct from a @c string_view. + * + * @param s The string to copy. + * + * The string in @a s is duplicated into this container. + */ + explicit ats_scoped_str(std::string_view s); + + /** Put an existing buffer in this container. + * + * @param s The string buffer. + * @return @a this + * + * The container takes ownership of @a s. For this reason @a s must be independently allocated, + * not part of a larger memory allocation. Any currently contained resource is destroyed. + */ + self_type &operator=(char *s); + + /** Assign from a @c string_view + * + * @param s The source string. + * @return @a this + * + * The string data is duplicated into this object and guaranteed to be null terminated. Any currently + * contained resource is destroyed. + */ + self_type &operator=(std::string_view s); }; +inline ats_scoped_str::ats_scoped_str(std::string_view s) +{ + if (!s.empty()) { + *this = s; + } +} + +inline ats_scoped_str & +ats_scoped_str::operator=(char *s) +{ + super::operator=(s); + return *this; +} + /** Specialization of @c ats_scoped_resource for pointers allocated with @c ats_malloc. */ template -#include -#include -#include +#include // NOLINT(modernize-deprecated-headers) +#include // NOLINT(modernize-deprecated-headers) +#include // NOLINT(modernize-deprecated-headers) #include #include #include #include -#include +#include // NOLINT(modernize-deprecated-headers) #include #include #include @@ -61,11 +61,11 @@ struct ifafilt; #include #ifdef HAVE_STDLIB_H -#include +#include // NOLINT(modernize-deprecated-headers) #endif -#include +#include // NOLINT(modernize-deprecated-headers) #ifdef HAVE_STRING_H -#include +#include // NOLINT(modernize-deprecated-headers) #endif #ifdef HAVE_STRINGS_H #include @@ -107,7 +107,7 @@ struct ifafilt; #include #endif -#include +#include // NOLINT(modernize-deprecated-headers) #ifdef HAVE_SIGINFO_H #include #endif @@ -170,7 +170,7 @@ typedef unsigned int in_addr_t; #endif #ifdef HAVE_FLOAT_H -#include +#include // NOLINT(modernize-deprecated-headers) #endif #ifdef HAVE_SYS_SYSMACROS_H @@ -190,3 +190,12 @@ typedef unsigned int in_addr_t; // on various OSs (linux-4096,osx/bsd-1024, // windows-260,etc) #endif + +// This is a little bit of a hack for now, until MPTCP has landed upstream in Linux land. +#ifndef MPTCP_ENABLED +#if defined(linux) +#define MPTCP_ENABLED 42 +#else +#define MPTCP_ENABLED 0 +#endif +#endif diff --git a/include/tscore/ink_queue.h b/include/tscore/ink_queue.h index 6d242cd9f15..bc8a71efac7 100644 --- a/include/tscore/ink_queue.h +++ b/include/tscore/ink_queue.h @@ -156,12 +156,9 @@ struct _InkFreeList { typedef struct ink_freelist_ops InkFreeListOps; typedef struct _InkFreeList InkFreeList; -extern const ink_freelist_ops *freelist_global_ops; -extern const ink_freelist_ops *freelist_class_ops; - const InkFreeListOps *ink_freelist_malloc_ops(); const InkFreeListOps *ink_freelist_freelist_ops(); -void ink_freelist_init_ops(int nofl_global, int nofl_class); +void ink_freelist_init_ops(int nofl_class, int nofl_proxy); /* * alignment must be a power of 2 @@ -171,9 +168,9 @@ InkFreeList *ink_freelist_create(const char *name, uint32_t type_size, uint32_t inkcoreapi void ink_freelist_init(InkFreeList **fl, const char *name, uint32_t type_size, uint32_t chunk_size, uint32_t alignment); inkcoreapi void ink_freelist_madvise_init(InkFreeList **fl, const char *name, uint32_t type_size, uint32_t chunk_size, uint32_t alignment, int advice); -inkcoreapi void *ink_freelist_new(InkFreeList *f, const InkFreeListOps *ops); -inkcoreapi void ink_freelist_free(InkFreeList *f, void *item, const InkFreeListOps *ops); -inkcoreapi void ink_freelist_free_bulk(InkFreeList *f, void *head, void *tail, size_t num_item, const InkFreeListOps *ops); +inkcoreapi void *ink_freelist_new(InkFreeList *f); +inkcoreapi void ink_freelist_free(InkFreeList *f, void *item); +inkcoreapi void ink_freelist_free_bulk(InkFreeList *f, void *head, void *tail, size_t num_item); void ink_freelists_dump(FILE *f); void ink_freelists_dump_baselinerel(FILE *f); void ink_freelists_snap_baseline(); diff --git a/include/tscore/ink_resolver.h b/include/tscore/ink_resolver.h index fee408470ed..b9af473f772 100644 --- a/include/tscore/ink_resolver.h +++ b/include/tscore/ink_resolver.h @@ -109,7 +109,7 @@ #define INK_RES_AAONLY 0x00000004 /*%< authoritative answers only (!IMPL)*/ #define INK_RES_USEVC 0x00000008 /*%< use virtual circuit */ #define INK_RES_PRIMARY 0x00000010 /*%< query primary server only (!IMPL) */ -#define INK_RES_IGNTC 0x00000020 /*%< ignore trucation errors */ +#define INK_RES_IGNTC 0x00000020 /*%< ignore truncation errors */ #define INK_RES_RECURSE 0x00000040 /*%< recursion desired */ #define INK_RES_DEFNAMES 0x00000080 /*%< use default domain name */ #define INK_RES_STAYOPEN 0x00000100 /*%< Keep TCP socket open */ @@ -181,7 +181,7 @@ enum HostResStyle { /// Strings for host resolution styles extern const char *const HOST_RES_STYLE_STRING[]; -/// Caclulate the effective resolution preferences. +/// Calculate the effective resolution preferences. extern HostResStyle ats_host_res_from(int family, ///< Connection family HostResPreferenceOrder ///< Preference ordering. ); diff --git a/include/tscore/ink_string++.h b/include/tscore/ink_string++.h index 126ca25fddb..fc9d86c343e 100644 --- a/include/tscore/ink_string++.h +++ b/include/tscore/ink_string++.h @@ -41,12 +41,12 @@ ***********************************************************************/ struct Str { - const char *str; // string pointer - size_t len; // length of string (not counting NUL) - struct Str *next; // next in list - struct Str *prev; // prev in list + const char *str = nullptr; // string pointer + size_t len = 0; // length of string (not counting NUL) + struct Str *next = nullptr; // next in list + struct Str *prev = nullptr; // prev in list - Str() : str(nullptr), len(0), next(nullptr), prev(nullptr) {} + Str() {} Str(char *s) { str = s; diff --git a/include/tscore/ink_thread.h b/include/tscore/ink_thread.h index 4e321316e2b..7a08f6205f1 100644 --- a/include/tscore/ink_thread.h +++ b/include/tscore/ink_thread.h @@ -47,6 +47,10 @@ #include #endif +#if HAVE_SYS_PRCTL_H && defined(PR_SET_NAME) +#include +#endif + #define INK_MUTEX_INIT PTHREAD_MUTEX_INITIALIZER #define INK_THREAD_STACK_MIN PTHREAD_STACK_MIN @@ -188,6 +192,9 @@ ink_thread_self() static inline ink_thread ink_thread_null() { + // The implementation of ink_thread (the alias of pthread_t) is different on platforms + // - e.g. `struct pthread *` on Unix and `unsigned long int` on Linux + // NOLINTNEXTLINE(modernize-use-nullptr) return (ink_thread)0; } @@ -307,4 +314,18 @@ ink_set_thread_name(const char *name ATS_UNUSED) #endif } +static inline void +ink_get_thread_name(char *name, size_t len) +{ +#if defined(HAVE_PTHREAD_GETNAME_NP) + pthread_getname_np(pthread_self(), name, len); +#elif defined(HAVE_PTHREAD_GET_NAME_NP) + pthread_get_name_np(pthread_self(), name, len); +#elif defined(HAVE_SYS_PRCTL_H) && defined(PR_GET_NAME) + prctl(PR_GET_NAME, name, 0, 0, 0); +#else + snprintf(name, len, "0x%" PRIx64, (uint64_t)ink_thread_self()); +#endif +} + #endif /* #if defined(POSIX_THREAD) */ diff --git a/include/tscore/ink_uuid.h b/include/tscore/ink_uuid.h index fe6f3a9e8eb..7106d24c60e 100644 --- a/include/tscore/ink_uuid.h +++ b/include/tscore/ink_uuid.h @@ -33,8 +33,8 @@ class ATSUuid { public: // Constructors - ATSUuid() : _version(TS_UUID_UNDEFINED) {} - ATSUuid &operator=(const ATSUuid other); + ATSUuid() {} + ATSUuid &operator=(const ATSUuid &other); // Initialize the UUID from a string bool parseString(const char *str); @@ -122,7 +122,7 @@ class ATSUuid } _uuid; // This is the typically used visible portion of the UUID - TSUuidVersion _version; + TSUuidVersion _version = TS_UUID_UNDEFINED; char _string[TS_UUID_STRING_LEN + 1]; bool diff --git a/include/tscore/runroot.h b/include/tscore/runroot.h index 1dd391e394f..5d1b6077052 100644 --- a/include/tscore/runroot.h +++ b/include/tscore/runroot.h @@ -54,6 +54,8 @@ typedef std::unordered_map RunrootMapType; bool exists(const std::string &dir); bool is_directory(const std::string &directory); +// argparser_runroot_handler should replace runroot_handler below when all program use ArgParser. +void argparser_runroot_handler(std::string const &value, const char *executable, bool json = false); void runroot_handler(const char **argv, bool json = false); // get a map from default layout diff --git a/include/tscore/signals.h b/include/tscore/signals.h index 5e23ab4070b..54fc9e5583f 100644 --- a/include/tscore/signals.h +++ b/include/tscore/signals.h @@ -50,6 +50,6 @@ bool signal_is_masked(int signo); // Test whether the signal is being handled by the given handler. bool signal_check_handler(int signo, signal_handler_t handler); -// Start a thread to test whether signals have the expected handler. Apparantly useful for +// Start a thread to test whether signals have the expected handler. Apparently useful for // finding pthread bugs in some version of DEC Unix. void signal_start_check_thread(signal_handler_t handler); diff --git a/include/tscpp/api/Async.h b/include/tscpp/api/Async.h index b392f494eda..37bdd292666 100644 --- a/include/tscpp/api/Async.h +++ b/include/tscpp/api/Async.h @@ -26,6 +26,7 @@ #include #include #include +#include #include "tscpp/api/noncopyable.h" @@ -105,7 +106,7 @@ class AsyncProvider void doRun(std::shared_ptr dispatch_controller) { - dispatch_controller_ = dispatch_controller; + dispatch_controller_ = std::move(dispatch_controller); run(); } friend class Async; @@ -157,7 +158,7 @@ class AsyncDispatchController : public AsyncDispatchControllerBase * @param mutex Mutex of the receiver that is locked during the dispatch */ AsyncDispatchController(AsyncEventReceiverType *event_receiver, AsyncProviderType *provider, std::shared_ptr mutex) - : event_receiver_(event_receiver), dispatch_mutex_(mutex), provider_(provider) + : event_receiver_(event_receiver), dispatch_mutex_(std::move(mutex)), provider_(provider) { } diff --git a/include/tscpp/api/AsyncTimer.h b/include/tscpp/api/AsyncTimer.h index 85df2b58b7b..3e7658a976d 100644 --- a/include/tscpp/api/AsyncTimer.h +++ b/include/tscpp/api/AsyncTimer.h @@ -68,7 +68,7 @@ class AsyncTimer : public AsyncProvider // For convenience, additional constructor prototypes. AsyncTimer(Type type, int period_in_ms, int initial_period_in_ms = 0) - : AsyncTimer(type, period_in_ms, initial_period_in_ms, TS_THREAD_POOL_DEFAULT) + : AsyncTimer(type, period_in_ms, initial_period_in_ms, TS_THREAD_POOL_NET) { } diff --git a/include/tscpp/api/Continuation.h b/include/tscpp/api/Continuation.h index edb6345e0ff..1557ecc964b 100644 --- a/include/tscpp/api/Continuation.h +++ b/include/tscpp/api/Continuation.h @@ -41,9 +41,9 @@ class Continuation TSContDataSet(_cont, static_cast(this)); } - // Create "empty" continuation, can only be populated by move assignement. + // Create "empty" continuation, can only be populated by move assignment. // - Continuation() : _cont(nullptr) {} + Continuation() {} TSCont asTSCont() const @@ -111,9 +111,9 @@ class Continuation // Timeout of zero means no timeout. // Action - schedule(TSHRTime timeout = 0, TSThreadPool tp = TS_THREAD_POOL_DEFAULT) + schedule(TSHRTime timeout = 0, TSThreadPool tp = TS_THREAD_POOL_NET) { - return TSContSchedule(_cont, timeout, tp); + return TSContScheduleOnPool(_cont, timeout, tp); } // Timeout of zero means no timeout. @@ -125,13 +125,13 @@ class Continuation } Action - scheduleEvery(TSHRTime interval /* milliseconds */, TSThreadPool tp = TS_THREAD_POOL_DEFAULT) + scheduleEvery(TSHRTime interval /* milliseconds */, TSThreadPool tp = TS_THREAD_POOL_NET) { - return TSContScheduleEvery(_cont, interval, tp); + return TSContScheduleEveryOnPool(_cont, interval, tp); } protected: - // Distinct continuation behavior is acheived by overriding this function in a derived continutation type. + // Distinct continuation behavior is achieved by overriding this function in a derived continuation type. // virtual int _run(TSEvent event, void *edata) = 0; @@ -139,7 +139,7 @@ class Continuation // static int _generalEventFunc(TSCont cont, TSEvent event, void *edata); - TSCont _cont; + TSCont _cont = nullptr; }; } // end namespace atscppapi diff --git a/include/tscpp/api/GlobalPlugin.h b/include/tscpp/api/GlobalPlugin.h index 1df60bddc6c..1d76ed5d8ff 100644 --- a/include/tscpp/api/GlobalPlugin.h +++ b/include/tscpp/api/GlobalPlugin.h @@ -33,7 +33,7 @@ struct GlobalPluginState; * @brief The interface used when creating a GlobalPlugin. * * A GlobalPlugin is a Plugin that will fire for a given hook on all Transactions. - * In otherwords, a GlobalPlugin is not tied to a specific plugin, a Transaction + * In other words, a GlobalPlugin is not tied to a specific plugin, a Transaction * specific plugin would be a TransactionPlugin. * * Depending on the @@ -60,9 +60,9 @@ class GlobalPlugin : public Plugin /** * registerHook is the mechanism used to attach a global hook. * - * \note Whenever you register a hook you must have the appropriate callback definied in your GlobalPlugin + * \note Whenever you register a hook you must have the appropriate callback defined in your GlobalPlugin * see HookType and Plugin for the correspond HookTypes and callback methods. If you fail to implement the - * callback, a default implmentation will be used that will only resume the Transaction. + * callback, a default implementation will be used that will only resume the Transaction. * * @param HookType the type of hook you wish to register * @see HookType diff --git a/include/tscpp/api/GzipDeflateTransformation.h b/include/tscpp/api/GzipDeflateTransformation.h index 039965eded4..10cfb621283 100644 --- a/include/tscpp/api/GzipDeflateTransformation.h +++ b/include/tscpp/api/GzipDeflateTransformation.h @@ -42,7 +42,7 @@ namespace transformations * * The GzipDeflateTransformation is a helper transformation that can be used * to easily compress content. For a full example of GzipDeflateTransformation - * and GzipInflateTransformation see examples/gzip_transformation/. + * and GzipInflateTransformation see example/cppapi/gzip_transformation/. * * @note GzipDeflateTransformation DOES NOT set Content-Encoding headers, it is the * users responsibility to set any applicable headers. @@ -54,7 +54,7 @@ namespace transformations public: /** * A full example of how to use GzipDeflateTransformation and GzipInflateTransformation is available - * in examples/gzip_tranformation/ + * in example/cppapi/gzip_transformation/ * * @param transaction As with any TransformationPlugin you must pass in the transaction * @param type because the GzipDeflateTransformation can be used with both requests and responses @@ -74,7 +74,7 @@ namespace transformations /** * Any TransformationPlugin must implement handleInputComplete(), this method will - * finalize the gzip compression and flush any remaining data and the epilouge. + * finalize the gzip compression and flush any remaining data and the epilogue. */ void handleInputComplete() override; diff --git a/include/tscpp/api/GzipInflateTransformation.h b/include/tscpp/api/GzipInflateTransformation.h index c664c182e0d..358fdc7b658 100644 --- a/include/tscpp/api/GzipInflateTransformation.h +++ b/include/tscpp/api/GzipInflateTransformation.h @@ -41,12 +41,12 @@ namespace transformations * * The GzipInflateTransformation is a helper transformation that can be used * to easily decompress gzipped content. For a full example of GzipInflateTransformation - * and GzipDeflateTransformation see examples/gzip_transformation/. + * and GzipDeflateTransformation see example/cppapi/gzip_transformation/. * * @note GzipDeflateTransformation DOES NOT set or check Content-Encoding headers, it is the - * users responsibility to set any applicable headers and check that the content is acctually + * users responsibility to set any applicable headers and check that the content is actually * gzipped by checking the Content-Encoding header before creating a GzipInflateTransformation, - * see examples/gzip_transformation/ for a full example. + * see example/cppapi/gzip_transformation/ for a full example. * * @see GzipDeflateTransformation */ @@ -55,7 +55,7 @@ namespace transformations public: /** * A full example of how to use GzipInflateTransformation and GzipDeflateTransformation is available - * in examples/gzip_tranformation/ + * in example/cppapi/gzip_transformation/. * * @param transaction As with any TransformationPlugin you must pass in the transaction * @param type because the GzipInflateTransformation can be used with both requests and responses diff --git a/include/tscpp/api/Headers.h b/include/tscpp/api/Headers.h index ebc7ecff2dd..c956ff4a518 100644 --- a/include/tscpp/api/Headers.h +++ b/include/tscpp/api/Headers.h @@ -36,7 +36,7 @@ class Response; /** * @brief A HeaderFieldName is a lightweight wrapper around a string that allows for case insensitive comparisons. - * Because header field names must be case insensitive this allows easy case insentive comparisons of names. + * Because header field names must be case insensitive this allows easy case insensitive comparisons of names. * */ class HeaderFieldName @@ -75,7 +75,7 @@ class HeaderFieldName std::string str(); /** - * @return a const char * which points to the name of this HeaderFIeldName + * @return a const char * which points to the name of this HeaderFieldName */ const char *c_str(); @@ -119,7 +119,7 @@ class header_field_value_iterator : public std::iterator #include @@ -63,7 +65,7 @@ class Stat : noncopyable /** * You must initialize your Stat with a call to this init() method. * - * @param name The string name of the stat, this will be visbible via traffic_ctl, or through http stats. + * @param name The string name of the stat, this will be visible via traffic_ctl, or through http stats. * @param type The SyncType of the Stat, this decides how TrafficServer will treat your inputs. The default * value is SYNC_COUNT. * @param persistent This determines if your Stats will persist, the default value is false. @@ -97,7 +99,7 @@ class Stat : noncopyable void set(int64_t value); private: - int stat_id_; /**< The internal stat ID */ + int stat_id_ = TS_ERROR; /**< The internal stat ID */ }; } // namespace atscppapi diff --git a/include/tscpp/api/Transaction.h b/include/tscpp/api/Transaction.h index e70bb7ad03b..874c315b853 100644 --- a/include/tscpp/api/Transaction.h +++ b/include/tscpp/api/Transaction.h @@ -132,7 +132,7 @@ class Transaction : noncopyable /** * Sets the error body page but this method does not advance the state machine to the error state. - * To do that you must explicitally call error(). + * To do that you must explicitly call error(). * * @param content the error page content. */ @@ -141,7 +141,7 @@ class Transaction : noncopyable /** * Sets the error body page with mimetype. * This method does not advance the state machine to the error state. - * To do that you must explicitally call error(). + * To do that you must explicitly call error(). * * @param content the error page content. * @param mimetype the error page's content-type. @@ -340,7 +340,7 @@ class Transaction : noncopyable size_t getServerResponseBodySize(); /** - * Get the nubmber of bytes for the response headers as returned by the server + * Get the number of bytes for the response headers as returned by the server * * @return server response header size */ size_t getServerResponseHeaderSize(); diff --git a/include/tscpp/api/TransactionPlugin.h b/include/tscpp/api/TransactionPlugin.h index 8f59c223542..b34fba01f8b 100644 --- a/include/tscpp/api/TransactionPlugin.h +++ b/include/tscpp/api/TransactionPlugin.h @@ -17,7 +17,7 @@ */ /** * @file TransactionPlugin.h - * @brief Contains the interface used in creating Transaciton plugins. + * @brief Contains the interface used in creating Transaction plugins. */ #pragma once @@ -62,7 +62,7 @@ struct TransactionPluginState; * You must always be sure to implement the appropriate callback for the type of hook you register. * * \code - * // For a more detailed example see examples/transactionhook/ + * // For a more detailed example see example/cppapi/transactionhook/ * class TransactionHookPlugin : publicTransactionPlugin { * public: * TransactionHookPlugin(Transaction &transaction) : TransactionPlugin(transaction) { @@ -89,9 +89,9 @@ class TransactionPlugin : public Plugin /** * registerHook is the mechanism used to attach a transaction hook. * - * \note Whenever you register a hook you must have the appropriate callback definied in your TransactionPlugin + * \note Whenever you register a hook you must have the appropriate callback defined in your TransactionPlugin * see HookType and Plugin for the correspond HookTypes and callback methods. If you fail to implement the - * callback, a default implmentation will be used that will only resume the Transaction. + * callback, a default implementation will be used that will only resume the Transaction. * * @param HookType the type of hook you wish to register * @see HookType diff --git a/include/tscpp/api/TransformationPlugin.h b/include/tscpp/api/TransformationPlugin.h index c31ef45c05b..e9b1a7fe690 100644 --- a/include/tscpp/api/TransformationPlugin.h +++ b/include/tscpp/api/TransformationPlugin.h @@ -46,7 +46,7 @@ class Continuation; * the appropriate callback for any hooks you register. * * A simple example of how to use the TransformationPlugin interface follows, this is an example - * of a Response transformation, the avialable options are REQUEST_TRANSFORMATION and RESPONSE_TRANSFORMATION + * of a Response transformation, the available options are REQUEST_TRANSFORMATION and RESPONSE_TRANSFORMATION * which are defined in Type. * * This example is a Null Transformation, meaning it will just spit out the content it receives without @@ -128,7 +128,7 @@ class TransformationPlugin : public TransactionPlugin /** * This is the method that you must call when you're done producing output for - * the downstream TranformationPlugin. + * the downstream TransformationPlugin. */ size_t setOutputComplete(); diff --git a/include/tscpp/api/noncopyable.h b/include/tscpp/api/noncopyable.h index 997b7ff1b73..304449b0832 100644 --- a/include/tscpp/api/noncopyable.h +++ b/include/tscpp/api/noncopyable.h @@ -18,7 +18,7 @@ /** * @file noncopyable.h - * @brief A base class used to prevent dervied classes from being copyable, this effectively + * @brief A base class used to prevent derived classes from being copyable, this effectively * eliminates the copy constructor and assignment operator. */ diff --git a/include/tscpp/util/IntrusiveDList.h b/include/tscpp/util/IntrusiveDList.h index eae1f0e9186..e94fda3b550 100644 --- a/include/tscpp/util/IntrusiveDList.h +++ b/include/tscpp/util/IntrusiveDList.h @@ -55,7 +55,7 @@ namespace ts It is the responsibility of the item class to initialize the link pointers. When an item is removed from the list the link pointers are set to @c nullptr. - An example declaration woudl be + An example declaration would be @code // Item in the list. @@ -137,7 +137,7 @@ template class IntrusiveDList value_type *operator->() const; /// Convenience conversion to pointer type - /// Because of how this list is normally used, being able to pass an iterator as a pointer is quite convienent. + /// Because of how this list is normally used, being able to pass an iterator as a pointer is quite convenient. /// If the iterator isn't valid, it converts to @c nullptr. operator value_type *() const; @@ -150,7 +150,7 @@ template class IntrusiveDList protected: // These are stored non-const to make implementing @c iterator easier. This class provides the required @c const // protection. - list_type *_list{nullptr}; ///< Needed to descrement from @c end() position. + list_type *_list{nullptr}; ///< Needed to decrement from @c end() position. typename list_type::value_type *_v{nullptr}; ///< Referenced element. /// Internal constructor for containers. @@ -242,11 +242,11 @@ template class IntrusiveDList self_type &append(value_type *v); /// Remove the first element of the list. - /// @return A poiner to the removed item, or @c nullptr if the list was empty. + /// @return A pointer to the removed item, or @c nullptr if the list was empty. value_type *take_head(); /// Remove the last element of the list. - /// @return A poiner to the removed item, or @c nullptr if the list was empty. + /// @return A pointer to the removed item, or @c nullptr if the list was empty. value_type *take_tail(); /// Insert a new element @a elt after @a target. @@ -354,7 +354,7 @@ template class IntrusiveDList */ template struct IntrusiveLinkage { static T *&next_ptr(T *thing); ///< Retrieve reference to next pointer. - static T *&prev_ptr(T *thing); ///< Retrive reference to previous pointer. + static T *&prev_ptr(T *thing); ///< Retrieve reference to previous pointer. }; template diff --git a/include/tscpp/util/MemSpan.h b/include/tscpp/util/MemSpan.h index 5a4d63141b7..9779e021a21 100644 --- a/include/tscpp/util/MemSpan.h +++ b/include/tscpp/util/MemSpan.h @@ -224,7 +224,7 @@ class MemSpan */ self_type remove_prefix(void const *p); - /** Shringt the span from the front. + /** Shrink the span from the front. * * @param n The number of bytes to remove. * @return @c *this @@ -254,7 +254,7 @@ class MemSpan */ self_type &remove_suffix(void const *p); - /** Shringt the span from the back. + /** Shrink the span from the back. * * @param n The number of bytes to remove. * @return @c *this @@ -317,7 +317,7 @@ inline MemSpan::MemSpan(void *start, void *end) : _data(start), _size(static_cas template MemSpan::MemSpan(T (&a)[N]) : _data(a), _size(N * sizeof(T)) {} -inline constexpr MemSpan::MemSpan(std::nullptr_t) : _data(nullptr), _size(0) {} +inline constexpr MemSpan::MemSpan(std::nullptr_t) {} inline ptrdiff_t MemSpan::distance(void const *lhs, void const *rhs) diff --git a/include/tscpp/util/TextView.h b/include/tscpp/util/TextView.h index 0ac740d6517..e82044f7bc5 100644 --- a/include/tscpp/util/TextView.h +++ b/include/tscpp/util/TextView.h @@ -99,7 +99,7 @@ class TextView : public std::string_view /** Construct explicitly with a pointer and size. If @a n is negative it is treated as 0. - @internal Overload for convience, otherwise get "narrow conversion" errors. + @internal Overload for convenience, otherwise get "narrow conversion" errors. */ constexpr TextView(const char *ptr, ///< Pointer to buffer. int n ///< Size of buffer. @@ -116,7 +116,7 @@ class TextView : public std::string_view Construct directly from an array of characters. All elements of the array are included in the view unless the last element is nul, in which case it is elided. - If this is inapropriate then a constructor with an explicit size should be used. + If this is inappropriate then a constructor with an explicit size should be used. @code TextView a("A literal string"); @@ -263,7 +263,7 @@ class TextView : public std::string_view self_type prefix(size_t n) const; /// Convenience overload to avoid ambiguity for literal numbers. self_type prefix(int n) const; - /** Get the prefix delimited by the first occurence of the character @a c. + /** Get the prefix delimited by the first occurrence of the character @a c. If @a c is not found the entire view is returned. The delimiter character is not included in the returned view. @@ -271,7 +271,7 @@ class TextView : public std::string_view @return A view of the prefix. */ self_type prefix(char c) const; - /** Get the prefix delimited by the first occurence of a character in @a delimiters. + /** Get the prefix delimited by the first occurrence of a character in @a delimiters. If no such character is found the entire view is returned. The delimiter character is not included in the returned view. @@ -291,10 +291,10 @@ class TextView : public std::string_view /// Overload to provide better return type. self_type &remove_prefix(size_t n); - /// Remove the prefix delimited by the first occurence of @a c. + /// Remove the prefix delimited by the first occurrence of @a c. self_type &remove_prefix_at(char c); - /// Remove the prefix delimited by the first occurence of a character for which @a pred is @c true. + /// Remove the prefix delimited by the first occurrence of a character for which @a pred is @c true. template self_type &remove_prefix_if(F const &pred); /** Split a prefix from the view on the character at offset @a n. @@ -391,10 +391,10 @@ class TextView : public std::string_view /// Overload to provide better return type. self_type &remove_suffix(size_t n); - /// Remove a suffix, delimited by the last occurence of @c c. + /// Remove a suffix, delimited by the last occurrence of @c c. self_type &remove_suffix_at(char c); - /// Remove a suffix, delimited by the last occurence of a character for which @a pred is @c true. + /// Remove a suffix, delimited by the last occurrence of a character for which @a pred is @c true. template self_type &remove_suffix_if(F const &f); /** Split the view to get a suffix of size @a n. @@ -535,13 +535,18 @@ intmax_t svtoi(TextView src, TextView *parsed = nullptr, int base = 0); * @param src The source text. Updated during parsing. * @return The converted numeric value. * - * This is a specialized function useful only where conversion performance is critical. It is used - * inside @c svtoi for the common cases of 8, 10, and 16, therefore normally this isn't much more - * performant in those cases than just @c svtoi. Because of this only positive values are parsed. - * If determining the radix from the text or signed value parsing is needed, used @c svtoi. + * This is a specialized function useful only where conversion performance is critical, or for some + * other reason the numeric text has already been parsed out. The performance gains comes from + * templating the divisor which enables the compiler to optimize the multiplication (e.g., for + * powers of 2 shifts is used). It is used inside @c svtoi and @c svtou for the common cases of 8, + * 10, and 16, therefore normally this isn't much more performant than @c svtoi. Because of this + * only positive values are parsed. If determining the radix from the text or signed value parsing + * is needed, used @c svtoi. * - * @a src is updated in place to indicate what characters were parsed. Parsing stops on the first - * invalid digit, so any leading non-digit characters (e.g. whitespace) must already be removed. + * @a src is updated in place by removing parsed characters. Parsing stops on the first invalid + * digit, so any leading non-digit characters (e.g. whitespace) must already be removed. Overflow + * is detected and the first digit that would overflow is not parsed, and the maximum value is + * returned. */ template uintmax_t @@ -551,8 +556,11 @@ svto_radix(ts::TextView &src) uintmax_t zret{0}; uintmax_t v; while (src.size() && (0 <= (v = ts::svtoi_convert[static_cast(*src)])) && v < N) { - zret *= N; - zret += v; + auto n = zret * N + v; + if (n < zret) { // overflow / wrap + return std::numeric_limits::max(); + } + zret = n; ++src; } return zret; diff --git a/include/wccp/Wccp.h b/include/wccp/Wccp.h index ebb07639b16..ec1424cc93d 100644 --- a/include/wccp/Wccp.h +++ b/include/wccp/Wccp.h @@ -255,7 +255,7 @@ class EndPoint /// Perform house keeping, including sending outbound messages. int housekeeping(); - /// Recieve and process a message on the socket. + /// Receive and process a message on the socket. /// @return 0 for success, -ERRNO on system error. ts::Rv handleMessage(); @@ -292,7 +292,7 @@ class Cache : public EndPoint /// Default constructor. Cache(); /// Destructor - ~Cache(); + ~Cache() override; /// Define services from a configuration file. ts::Errata loadServicesFromFile(char const *path ///< Path to file. @@ -308,7 +308,7 @@ class Cache : public EndPoint - @c ServiceGroup::CONFLICT if the service doesn't match the existing service. */ Service defineServiceGroup(ServiceGroup const &svc, ///< Service group description. - ServiceGroup::Result *result = 0); + ServiceGroup::Result *result = nullptr); /** Add a seed router to the service group. @@ -335,7 +335,7 @@ class Cache : public EndPoint /// Get the current implementation instance cast to correct type. ImplType const *impl() const; /// Create a new implementation instance. - super::ImplType *make(); + super::ImplType *make() override; }; /** Hold a reference to a service group in this end point. @@ -372,8 +372,8 @@ class Cache::Service : public ServiceConstants private: Service(Cache const &cache, detail::cache::GroupData &group); - Cache m_cache; ///< Parent cache. - detail::cache::GroupData *m_group; ///< Service Group data. + Cache m_cache; ///< Parent cache. + detail::cache::GroupData *m_group = nullptr; ///< Service Group data. friend class Cache; }; @@ -387,7 +387,7 @@ class Router : public EndPoint /// Default constructor Router(); /// Destructor. - ~Router(); + ~Router() override; /// Transmit pending messages. int sendPendingMessages(); @@ -398,7 +398,7 @@ class Router : public EndPoint /// Get the current implementation instance cast to correct type. ImplType *impl(); /// Create a new implementation instance. - super::ImplType *make(); + super::ImplType *make() override; }; // ------------------------------------------------------ @@ -499,7 +499,7 @@ ServiceGroup::clearPorts() return *this; } -inline Cache::Service::Service() : m_group(0) {} +inline Cache::Service::Service() {} inline Cache::Service::Service(Cache const &cache, detail::cache::GroupData &group) : m_cache(cache), m_group(&group) {} diff --git a/iocore/aio/AIO.cc b/iocore/aio/AIO.cc index 0e7f15add5f..48e08b475d8 100644 --- a/iocore/aio/AIO.cc +++ b/iocore/aio/AIO.cc @@ -25,6 +25,8 @@ * Async Disk IO operations. */ +#include + #include "P_AIO.h" #if AIO_MODE == AIO_MODE_NATIVE @@ -143,9 +145,9 @@ ink_aio_set_callback(Continuation *callback) } void -ink_aio_init(ModuleVersion v) +ink_aio_init(ts::ModuleVersion v) { - ink_release_assert(!checkModuleVersion(v, AIO_MODULE_VERSION)); + ink_release_assert(v.check(AIO_MODULE_INTERNAL_VERSION)); aio_rsb = RecAllocateRawStatBlock((int)AIO_STAT_COUNT); RecRegisterRawStat(aio_rsb, RECT_PROCESS, "proxy.process.cache.read_per_sec", RECD_FLOAT, RECP_PERSISTENT, @@ -204,16 +206,12 @@ struct AIOThreadInfo : public Continuation { } }; -/* priority scheduling */ -/* Have 2 queues per file descriptor - A queue for http requests and another - for non-http (streaming) request. Each file descriptor has a lock - and condition variable associated with it. A dedicated number of threads - (THREADS_PER_DISK) wait on the condition variable associated with the - file descriptor. The cache threads try to put the request in the - appropriate queue. If they fail to acquire the lock, they put the - request in the atomic list. Requests are served in the order of - highest priority first. If both the queues are empty, the aio threads - check if there is any request on the other disks */ +/* + A dedicated number of threads (THREADS_PER_DISK) wait on the condition + variable associated with the file descriptor. The cache threads try to put + the request in the appropriate queue. If they fail to acquire the lock, they + put the request in the atomic list. + */ /* insert an entry for file descriptor fildes into aio_reqs */ static AIO_Reqs * @@ -266,8 +264,7 @@ aio_init_fildes(int fildes, int fromAPI = 0) return request; } -/* insert a request into either aio_todo or http_todo queue. aio_todo - list is kept sorted */ +/* insert a request into aio_todo queue. */ static void aio_insert(AIOCallback *op, AIO_Reqs *req) { @@ -275,22 +272,7 @@ aio_insert(AIOCallback *op, AIO_Reqs *req) num_requests++; req->queued++; #endif - if (op->aiocb.aio_reqprio == AIO_LOWEST_PRIORITY) // http request - { - req->http_aio_todo.enqueue(op); - } else { - AIOCallback *cb = (AIOCallback *)req->aio_todo.tail; - - for (; cb; cb = (AIOCallback *)cb->link.prev) { - if (cb->aiocb.aio_reqprio >= op->aiocb.aio_reqprio) { - req->aio_todo.insert(op, cb); - return; - } - } - - /* Either the queue was empty or this request has the highest priority */ - req->aio_todo.push(op); - } + req->aio_todo.enqueue(op); } /* move the request from the atomic list to the queue */ @@ -457,14 +439,14 @@ aio_thread_main(void *arg) ink_mutex_acquire(&my_aio_req->aio_mutex); for (;;) { do { - if (unlikely(shutdown_event_system == true)) { + if (TSSystemState::is_event_system_shut_down()) { ink_mutex_release(&my_aio_req->aio_mutex); return nullptr; } current_req = my_aio_req; /* check if any pending requests on the atomic list */ aio_move(my_aio_req); - if (!(op = my_aio_req->aio_todo.pop()) && !(op = my_aio_req->http_aio_todo.pop())) { + if (!(op = my_aio_req->aio_todo.pop())) { break; } #ifdef AIO_STATS @@ -556,7 +538,7 @@ DiskHandler::mainAIOEvent(int event, Event *e) if (ret < 0) { Debug("aio", "io_submit failed: %s (%d)", strerror(-ret), -ret); } else { - Fatal("could not sumbit IOs, io_submit(%p, %d, %p) returned %d", ctx, num, cbs, ret); + Fatal("could not submit IOs, io_submit(%p, %d, %p) returned %d", ctx, num, cbs, ret); } } } @@ -576,7 +558,6 @@ DiskHandler::mainAIOEvent(int event, Event *e) int ink_aio_read(AIOCallback *op, int /* fromAPI ATS_UNUSED */) { - op->aiocb.aio_reqprio = AIO_DEFAULT_PRIORITY; op->aiocb.aio_lio_opcode = IO_CMD_PREAD; op->aiocb.data = op; EThread *t = this_ethread(); @@ -591,7 +572,6 @@ ink_aio_read(AIOCallback *op, int /* fromAPI ATS_UNUSED */) int ink_aio_write(AIOCallback *op, int /* fromAPI ATS_UNUSED */) { - op->aiocb.aio_reqprio = AIO_DEFAULT_PRIORITY; op->aiocb.aio_lio_opcode = IO_CMD_PWRITE; op->aiocb.data = op; EThread *t = this_ethread(); @@ -612,7 +592,6 @@ ink_aio_readv(AIOCallback *op, int /* fromAPI ATS_UNUSED */) int sz = 0; while (io) { - io->aiocb.aio_reqprio = AIO_DEFAULT_PRIORITY; io->aiocb.aio_lio_opcode = IO_CMD_PREAD; io->aiocb.data = io; #ifdef HAVE_EVENTFD @@ -643,7 +622,6 @@ ink_aio_writev(AIOCallback *op, int /* fromAPI ATS_UNUSED */) int sz = 0; while (io) { - io->aiocb.aio_reqprio = AIO_DEFAULT_PRIORITY; io->aiocb.aio_lio_opcode = IO_CMD_PWRITE; io->aiocb.data = io; #ifdef HAVE_EVENTFD diff --git a/iocore/aio/I_AIO.h b/iocore/aio/I_AIO.h index 8222ba1a59c..8618e59da37 100644 --- a/iocore/aio/I_AIO.h +++ b/iocore/aio/I_AIO.h @@ -34,9 +34,7 @@ #include "I_EventSystem.h" #include "records/I_RecProcess.h" -#define AIO_MODULE_MAJOR_VERSION 1 -#define AIO_MODULE_MINOR_VERSION 0 -#define AIO_MODULE_VERSION makeModuleVersion(AIO_MODULE_MAJOR_VERSION, AIO_MODULE_MINOR_VERSION, PUBLIC_MODULE_HEADER) +static constexpr ts::ModuleVersion AIO_MODULE_PUBLIC_VERSION(1, 0, ts::ModuleVersion::PUBLIC); #define AIO_EVENT_DONE (AIO_EVENT_EVENTS_START + 0) @@ -74,7 +72,6 @@ struct ink_aiocb { size_t aio_nbytes = 0; /* length of transfer */ off_t aio_offset = 0; /* file offset */ - int aio_reqprio = 0; /* request priority offset */ int aio_lio_opcode = 0; /* listio operation */ int aio_state = 0; /* state flag for List I/O */ int aio__pad[1]; /* extension padding */ @@ -88,9 +85,6 @@ bool ink_aio_thread_num_set(int thread_num); #define AIO_CALLBACK_THREAD_ANY ((EThread *)0) // any regular event thread #define AIO_CALLBACK_THREAD_AIO ((EThread *)-1) -#define AIO_LOWEST_PRIORITY 0 -#define AIO_DEFAULT_PRIORITY AIO_LOWEST_PRIORITY - struct AIOCallback : public Continuation { // set before calling aio_read/aio_write ink_aiocb aiocb; @@ -141,7 +135,7 @@ struct DiskHandler : public Continuation { }; #endif -void ink_aio_init(ModuleVersion version); +void ink_aio_init(ts::ModuleVersion version); int ink_aio_start(); void ink_aio_set_callback(Continuation *error_callback); @@ -151,4 +145,4 @@ int ink_aio_write(AIOCallback *op, int fromAPI = 0); int ink_aio_readv(AIOCallback *op, int fromAPI = 0); // fromAPI is a boolean to indicate if this is from a API call such as upload proxy feature int ink_aio_writev(AIOCallback *op, int fromAPI = 0); -AIOCallback *new_AIOCallback(void); +AIOCallback *new_AIOCallback(); diff --git a/iocore/aio/Makefile.am b/iocore/aio/Makefile.am index 8815fda4b6e..999f80de6c5 100644 --- a/iocore/aio/Makefile.am +++ b/iocore/aio/Makefile.am @@ -59,7 +59,7 @@ test_AIO_LDADD = \ $(top_builddir)/src/tscore/libtscore.la \ $(top_builddir)/src/tscpp/util/libtscpputil.la \ $(top_builddir)/proxy/shared/libUglyLogStubs.a \ - @LIBTCL@ @HWLOC_LIBS@ + @HWLOC_LIBS@ include $(top_srcdir)/build/tidy.mk diff --git a/iocore/aio/P_AIO.h b/iocore/aio/P_AIO.h index d4d21e5992a..95534c9821b 100644 --- a/iocore/aio/P_AIO.h +++ b/iocore/aio/P_AIO.h @@ -36,8 +36,7 @@ // for debugging // #define AIO_STATS 1 -#undef AIO_MODULE_VERSION -#define AIO_MODULE_VERSION makeModuleVersion(AIO_MODULE_MAJOR_VERSION, AIO_MODULE_MINOR_VERSION, PRIVATE_MODULE_HEADER) +static constexpr ts::ModuleVersion AIO_MODULE_INTERNAL_VERSION{AIO_MODULE_PUBLIC_VERSION, ts::ModuleVersion::PRIVATE}; TS_INLINE int AIOCallback::ok() @@ -86,24 +85,19 @@ struct AIOCallbackInternal : public AIOCallback { int io_complete(int event, void *data); - AIOCallbackInternal() - { - aiocb.aio_reqprio = AIO_DEFAULT_PRIORITY; - SET_HANDLER(&AIOCallbackInternal::io_complete); - } + AIOCallbackInternal() { SET_HANDLER(&AIOCallbackInternal::io_complete); } }; struct AIO_Reqs { - Que(AIOCallback, link) aio_todo; /* queue for holding non-http requests */ - Que(AIOCallback, link) http_aio_todo; /* queue for http requests */ - /* Atomic list to temporarily hold the request if the - lock for a particular queue cannot be acquired */ + Que(AIOCallback, link) aio_todo; /* queue for AIO operations */ + /* Atomic list to temporarily hold the request if the + lock for a particular queue cannot be acquired */ ASLL(AIOCallbackInternal, alink) aio_temp_list; ink_mutex aio_mutex; ink_cond aio_cond; int index = 0; /* position of this struct in the aio_reqs array */ int pending = 0; /* number of outstanding requests on the disk */ - int queued = 0; /* total number of aio_todo and http_todo requests */ + int queued = 0; /* total number of aio_todo requests */ int filedes = 0; /* the file descriptor for the requests */ int requests_queued = 0; }; diff --git a/iocore/aio/test_AIO.cc b/iocore/aio/test_AIO.cc index 33a972179b8..4bc2c237bf0 100644 --- a/iocore/aio/test_AIO.cc +++ b/iocore/aio/test_AIO.cc @@ -24,6 +24,7 @@ #include "P_AIO.h" #include "InkAPIInternal.h" #include "tscore/I_Layout.h" +#include "tscore/TSSystemState.h" #include #include @@ -414,7 +415,7 @@ main(int /* argc ATS_UNUSED */, char *argv[]) Layout::create(); init_diags("", nullptr); RecProcessInit(RECM_STAND_ALONE); - ink_event_system_init(EVENT_SYSTEM_MODULE_VERSION); + ink_event_system_init(EVENT_SYSTEM_MODULE_PUBLIC_VERSION); eventProcessor.start(ink_number_of_processors()); Thread *main_thread = new EThread; @@ -431,7 +432,7 @@ main(int /* argc ATS_UNUSED */, char *argv[]) #endif RecProcessStart(); - ink_aio_init(AIO_MODULE_VERSION); + ink_aio_init(AIO_MODULE_PUBLIC_VERSION); srand48(time(nullptr)); printf("input file %s\n", argv[1]); if (!read_config(argv[1])) { @@ -469,7 +470,7 @@ main(int /* argc ATS_UNUSED */, char *argv[]) } } - while (!shutdown_event_system) { + while (!TSSystemState::is_event_system_shut_down()) { sleep(1); } delete main_thread; diff --git a/iocore/cache/Cache.cc b/iocore/cache/Cache.cc index 2bac39c9ad0..d28fa4cb768 100644 --- a/iocore/cache/Cache.cc +++ b/iocore/cache/Cache.cc @@ -39,7 +39,7 @@ #include -const VersionNumber CACHE_DB_VERSION(CACHE_DB_MAJOR_VERSION, CACHE_DB_MINOR_VERSION); +constexpr ts::VersionNumber CACHE_DB_VERSION(CACHE_DB_MAJOR_VERSION, CACHE_DB_MINOR_VERSION); // Compilation Options #define USELESS_REENABLES // allow them for now @@ -246,7 +246,7 @@ cache_stats_bytes_used_cb(const char *name, RecDataT data_type, RecData *data, R RecRawStatSyncSum(name, data_type, data, rsb, id); RecGetGlobalRawStatSum(rsb, (int)cache_bytes_total_stat, &total); percent_full = (float)used / (float)total * 100; - // The perent_full float below gets rounded down + // The percent_full float below gets rounded down RecSetGlobalRawStatSum(rsb, (int)cache_percent_full_stat, (int64_t)percent_full); } @@ -285,7 +285,7 @@ update_cache_config(const char * /* name ATS_UNUSED */, RecDataT /* data_type AT return 0; } -CacheVC::CacheVC() : alternate_index(CACHE_ALT_INDEX_DEFAULT) +CacheVC::CacheVC() { size_to_init = sizeof(CacheVC) - (size_t) & ((CacheVC *)nullptr)->vio; memset((void *)&vio, 0, size_to_init); @@ -482,26 +482,12 @@ CacheVC::set_pin_in_cache(time_t time_pin) return true; } -bool -CacheVC::set_disk_io_priority(int priority) -{ - ink_assert(priority >= AIO_LOWEST_PRIORITY); - io.aiocb.aio_reqprio = priority; - return true; -} - time_t CacheVC::get_pin_in_cache() { return pin_in_cache; } -int -CacheVC::get_disk_io_priority() -{ - return io.aiocb.aio_reqprio; -} - int Vol::begin_read(CacheVC *cont) { @@ -865,7 +851,7 @@ CacheProcessor::cacheInitialized() int caches_ready = 0; int cache_init_ok = 0; /* allocate ram size in proportion to the disk space the - volume accupies */ + volume occupies */ int64_t total_size = 0; // count in HTTP & MIXT uint64_t total_cache_bytes = 0; // bytes that can used in total_size uint64_t total_direntries = 0; // all the direntries in the cache @@ -1152,7 +1138,7 @@ vol_init_data_internal(Vol *d) off_t total_entries = (d->len - (d->start - d->skip)) / cache_config_min_average_object_size; // step2: calculate the number of buckets off_t total_buckets = total_entries / DIR_DEPTH; - // step3: calculate the number of segments, no semgent has more than 16384 buckets + // step3: calculate the number of segments, no segment has more than 16384 buckets d->segments = (total_buckets + (((1 << 16) - 1) / DIR_DEPTH)) / ((1 << 16) / DIR_DEPTH); // step4: divide total_buckets into segments on average. d->buckets = (total_buckets + d->segments - 1) / d->segments; @@ -1192,9 +1178,9 @@ vol_clear_init(Vol *d) size_t dir_len = d->dirlen(); memset(d->raw_dir, 0, dir_len); vol_init_dir(d); - d->header->magic = VOL_MAGIC; - d->header->version.ink_major = CACHE_DB_MAJOR_VERSION; - d->header->version.ink_minor = CACHE_DB_MINOR_VERSION; + d->header->magic = VOL_MAGIC; + d->header->version._major = CACHE_DB_MAJOR_VERSION; + d->header->version._minor = CACHE_DB_MINOR_VERSION; d->scan_pos = d->header->agg_pos = d->header->write_pos = d->start; d->header->last_write_pos = d->header->write_pos; d->header->phase = 0; @@ -1363,12 +1349,12 @@ Vol::handle_dir_read(int event, void *data) } } - if (!(header->magic == VOL_MAGIC && footer->magic == VOL_MAGIC && - CACHE_DB_MAJOR_VERSION_COMPATIBLE <= header->version.ink_major && header->version.ink_major <= CACHE_DB_MAJOR_VERSION)) { + if (!(header->magic == VOL_MAGIC && footer->magic == VOL_MAGIC && CACHE_DB_MAJOR_VERSION_COMPATIBLE <= header->version._major && + header->version._major <= CACHE_DB_MAJOR_VERSION)) { Warning("bad footer in cache directory for '%s', clearing", hash_text.get()); Note("VOL_MAGIC %d\n header magic: %d\n footer_magic %d\n CACHE_DB_MAJOR_VERSION_COMPATIBLE %d\n major version %d\n" "CACHE_DB_MAJOR_VERSION %d\n", - VOL_MAGIC, header->magic, footer->magic, CACHE_DB_MAJOR_VERSION_COMPATIBLE, header->version.ink_major, + VOL_MAGIC, header->magic, footer->magic, CACHE_DB_MAJOR_VERSION_COMPATIBLE, header->version._major, CACHE_DB_MAJOR_VERSION); Note("clearing cache directory '%s'", hash_text.get()); clear_dir(); @@ -1545,7 +1531,7 @@ Vol::handle_recover_from_data(int event, void * /* data ATS_UNUSED */) s += round_to_approx_size(doc->len); continue; } - // case 3 - we have already recoverd some data and + // case 3 - we have already recovered some data and // (doc->sync_serial < last_sync_serial) || // (doc->sync_serial > header->sync_serial + 1). // if we are too close to the end, wrap around @@ -1869,7 +1855,7 @@ build_vol_hash_table(CacheHostRecord *cp) for (int i = 0; i < VOL_HASH_TABLE_SIZE; i++) { ttable[i] = VOL_HASH_EMPTY; } - // generate random numbers proportaion to allocation + // generate random numbers proportional to allocation rtable_pair *rtable = (rtable_pair *)ats_malloc(sizeof(rtable_pair) * rtable_size); int rindex = 0; for (int i = 0; i < num_vols; i++) { @@ -2139,10 +2125,20 @@ CacheVC::is_pread_capable() static void unmarshal_helper(Doc *doc, Ptr &buf, int &okay) { + using UnmarshalFunc = int(char *buf, int len, RefCountObj *block_ref); + UnmarshalFunc *unmarshal_func = &HTTPInfo::unmarshal; + ts::VersionNumber version(doc->v_major, doc->v_minor); + + // introduced by https://github.com/apache/trafficserver/pull/4874, this is used to distinguish the doc version + // before and after #4847 + if (version < CACHE_DB_VERSION) { + unmarshal_func = &HTTPInfo::unmarshal_v24_1; + } + char *tmp = doc->hdr(); int len = doc->hlen; while (len > 0) { - int r = HTTPInfo::unmarshal(tmp, len, buf.get()); + int r = unmarshal_func(tmp, len, buf.get()); if (r < 0) { ink_assert(!"CacheVC::handleReadDone unmarshal failed"); okay = 0; @@ -2158,7 +2154,7 @@ unmarshal_helper(Doc *doc, Ptr &buf, int &okay) @internal I looked at doing this in place (rather than a copy & modify) but - The in place logic would be even worse than this mess - It wouldn't save you that much, since you end up doing inserts early in the buffer. - Without extreme care in the logic it could end up doing more copying thatn + Without extreme care in the logic it could end up doing more copying than the simpler copy & modify. @internal This logic presumes the existence of some slack at the end of the buffer, which @@ -2225,7 +2221,7 @@ upgrade_doc_version(Ptr &buf) } Doc *n_doc = reinterpret_cast(buf->data()); // access as current version. // For now the base header size is the same. If that changes we'll need to handle the v22/23 case here - // as with the v21 and shift the content down to accomodate the bigger header. + // as with the v21 and shift the content down to accommodate the bigger header. ink_assert(sizeof(*n_doc) == sizeof(*doc)); n_doc->doc_type = CACHE_FRAG_TYPE_HTTP; // We converted so adjust doc_type. @@ -2295,7 +2291,7 @@ CacheVC::handleReadDone(int event, Event *e) goto Ldone; } } else if (doc->doc_type == CACHE_FRAG_TYPE_HTTP) { // handle any version updates based on the object version - if (VersionNumber(doc->v_major, doc->v_minor) > CACHE_DB_VERSION) { + if (ts::VersionNumber(doc->v_major, doc->v_minor) > CACHE_DB_VERSION) { // future version, count as corrupted doc->magic = DOC_CORRUPT; Debug("cache_bc", "Object is future version %d:%d - disk %s - doc id = %" PRIx64 ":%" PRIx64 "", doc->v_major, doc->v_minor, @@ -2799,7 +2795,7 @@ cplist_reconfigure() } } - /* change percentages in the config patitions to absolute value */ + /* change percentages in the config partitions to absolute value */ off_t tot_space_in_blks = 0; off_t blocks_per_vol = VOL_BLOCK_SIZE / STORE_BLOCK_SIZE; /* sum up the total space available on all the disks. @@ -2865,6 +2861,8 @@ cplist_reconfigure() new_cp->disk_vols = (DiskVol **)ats_malloc(gndisks * sizeof(DiskVol *)); memset(new_cp->disk_vols, 0, gndisks * sizeof(DiskVol *)); if (create_volume(config_vol->number, size_in_blocks, config_vol->scheme, new_cp)) { + ats_free(new_cp->disk_vols); + new_cp->disk_vols = nullptr; delete new_cp; return -1; } @@ -3152,10 +3150,22 @@ register_cache_stats(RecRawStatBlock *rsb, const char *prefix) REG_INT("span.online", cache_span_online_stat); } +int +FragmentSizeUpdateCb(const char * /* name ATS_UNUSED */, RecDataT /* data_type ATS_UNUSED */, RecData data, void *cookie) +{ + if (sizeof(Doc) >= static_cast(data.rec_int) || static_cast(data.rec_int) - sizeof(Doc) > MAX_FRAG_SIZE) { + Warning("The fragments size exceed the limitation, ignore: %" PRId64 ", %d", data.rec_int, cache_config_target_fragment_size); + return 0; + } + + cache_config_target_fragment_size = data.rec_int; + return 0; +} + void -ink_cache_init(ModuleVersion v) +ink_cache_init(ts::ModuleVersion v) { - ink_release_assert(!checkModuleVersion(v, CACHE_MODULE_VERSION)); + ink_release_assert(v.check(CACHE_MODULE_VERSION)); cache_rsb = RecAllocateRawStatBlock((int)cache_stat_count); @@ -3204,9 +3214,12 @@ ink_cache_init(ModuleVersion v) Debug("cache_init", "proxy.config.cache.hit_evacuate_size_limit = %d", cache_config_hit_evacuate_size_limit); REC_EstablishStaticConfigInt32(cache_config_force_sector_size, "proxy.config.cache.force_sector_size"); - REC_EstablishStaticConfigInt32(cache_config_target_fragment_size, "proxy.config.cache.target_fragment_size"); - if (cache_config_target_fragment_size == 0) { + ink_assert(REC_RegisterConfigUpdateFunc("proxy.config.cache.target_fragment_size", FragmentSizeUpdateCb, nullptr) != + REC_ERR_FAIL); + REC_ReadConfigInt32(cache_config_target_fragment_size, "proxy.config.cache.target_fragment_size"); + + if (cache_config_target_fragment_size == 0 || cache_config_target_fragment_size - sizeof(Doc) > MAX_FRAG_SIZE) { cache_config_target_fragment_size = DEFAULT_TARGET_FRAGMENT_SIZE; } @@ -3289,18 +3302,18 @@ CacheProcessor::find_by_path(const char *path, int len) namespace cache_bc { -static size_t const HTTP_ALT_MARSHAL_SIZE = ROUND(sizeof(HTTPCacheAlt), HDR_PTR_SIZE); // current size. +static size_t const HTTP_ALT_MARSHAL_SIZE = HdrHeapMarshalBlocks{ts::round_up(sizeof(HTTPCacheAlt))}; // current size. size_t HTTPInfo_v21::marshalled_length(void *data) { - size_t zret = ROUND(sizeof(HTTPCacheAlt_v21), HDR_PTR_SIZE); + size_t zret = HdrHeapMarshalBlocks{ts::round_up(sizeof(HTTPCacheAlt_v21))}; HTTPCacheAlt_v21 *alt = static_cast(data); HdrHeap *hdr; hdr = reinterpret_cast(reinterpret_cast(alt) + reinterpret_cast(alt->m_request_hdr.m_heap)); - zret += ROUND(hdr->unmarshal_size(), HDR_PTR_SIZE); + zret += HdrHeapMarshalBlocks{ts::round_up(hdr->unmarshal_size())}; hdr = reinterpret_cast(reinterpret_cast(alt) + reinterpret_cast(alt->m_response_hdr.m_heap)); - zret += ROUND(hdr->unmarshal_size(), HDR_PTR_SIZE); + zret += HdrHeapMarshalBlocks{ts::round_up(hdr->unmarshal_size())}; return zret; } @@ -3363,7 +3376,7 @@ HTTPInfo_v21::copy_and_upgrade_unmarshalled_to_v23(char *&dst, char *&src, size_ s_hdr = reinterpret_cast(reinterpret_cast(s_alt) + reinterpret_cast(s_alt->m_request_hdr.m_heap)); d_hdr = reinterpret_cast(dst); - hdr_size = ROUND(s_hdr->unmarshal_size(), HDR_PTR_SIZE); + hdr_size = HdrHeapMarshalBlocks{ts::round_up(s_hdr->unmarshal_size())}; if (hdr_size > length) { return false; } @@ -3375,7 +3388,7 @@ HTTPInfo_v21::copy_and_upgrade_unmarshalled_to_v23(char *&dst, char *&src, size_ s_hdr = reinterpret_cast(reinterpret_cast(s_alt) + reinterpret_cast(s_alt->m_response_hdr.m_heap)); d_hdr = reinterpret_cast(dst); - hdr_size = ROUND(s_hdr->unmarshal_size(), HDR_PTR_SIZE); + hdr_size = HdrHeapMarshalBlocks{ts::round_up(s_hdr->unmarshal_size())}; if (hdr_size > length) { return false; } diff --git a/iocore/cache/CacheDir.cc b/iocore/cache/CacheDir.cc index 427ce31ffd6..a7a1255dec4 100644 --- a/iocore/cache/CacheDir.cc +++ b/iocore/cache/CacheDir.cc @@ -558,7 +558,7 @@ dir_probe(const CacheKey *key, Vol *d, Dir *result, Dir **last_collision) if (collision) { if (collision == e) { collision = nullptr; - // increment collison stat + // increment collision stat // Note: dir_probe could be called multiple times // for the same document and so the collision stat // may not accurately reflect the number of documents @@ -973,7 +973,7 @@ sync_cache_dir_on_shutdown() int r = pwrite(d->fd, d->agg_buffer, d->agg_buf_pos, d->header->write_pos); if (r != d->agg_buf_pos) { - ink_assert(!"flusing agg buffer failed"); + ink_assert(!"flushing agg buffer failed"); continue; } d->header->last_write_pos = d->header->write_pos; @@ -1267,10 +1267,6 @@ int Vol::dir_check(bool /* fix ATS_UNUSED */) // TODO: we should eliminate this ++frag_demographics[dir_size(e)][dir_big(e)]; } } - e = next_dir(e, seg); - if (!e) { - break; - } } // Check for duplicates (identical tags in the same bucket). diff --git a/iocore/cache/CacheDisk.cc b/iocore/cache/CacheDisk.cc index 78beffe0310..8e98e309092 100644 --- a/iocore/cache/CacheDisk.cc +++ b/iocore/cache/CacheDisk.cc @@ -61,10 +61,9 @@ CacheDisk::open(char *s, off_t blocks, off_t askip, int ahw_sector_size, int fil skip = askip; start = skip; /* we can't use fractions of store blocks. */ - len = blocks; - io.aiocb.aio_fildes = fd; - io.aiocb.aio_reqprio = 0; - io.action = this; + len = blocks; + io.aiocb.aio_fildes = fd; + io.action = this; // determine header size and hence start point by successive approximation uint64_t l; for (int i = 0; i < 3; i++) { diff --git a/iocore/cache/CacheHosting.cc b/iocore/cache/CacheHosting.cc index e7895b37f54..5a6a7cec839 100644 --- a/iocore/cache/CacheHosting.cc +++ b/iocore/cache/CacheHosting.cc @@ -100,8 +100,8 @@ CacheHostMatcher::Match(const char *rdata, int rlen, CacheHostResult *result) CacheHostRecord *data_ptr; bool r; - // Check to see if there is any work to do before makeing - // the stirng copy + // Check to see if there is any work to do before making + // the string copy if (num_el <= 0) { return; } @@ -244,6 +244,8 @@ int fstat_wrapper(int fd, struct stat *s); int CacheHostTable::BuildTableFromString(const char *config_file_path, char *file_buf) { + Note("hosting.config loading ..."); + // Table build locals Tokenizer bufTok("\n"); tok_iter_state i_state; @@ -323,6 +325,7 @@ CacheHostTable::BuildTableFromString(const char *config_file_path, char *file_bu if (gen_host_rec.Init(type)) { Warning("Problems encountered while initializing the Generic Volume"); } + Note("hosting.config finished loading"); return 0; } @@ -370,6 +373,8 @@ CacheHostTable::BuildTableFromString(const char *config_file_path, char *file_bu last = current; current = current->next; ats_free(last); + + Note("hosting.config finished loading"); } if (!generic_rec_initd) { @@ -591,14 +596,20 @@ ConfigVolumes::read_config_file() config_path = RecConfigReadConfigPath("proxy.config.cache.volume_filename"); ink_release_assert(config_path); + Note("volume.config loading ..."); + file_buf = readIntoBuffer(config_path, "[CacheVolition]", nullptr); if (file_buf == nullptr) { + Error("volume.config failed to load"); Warning("Cannot read the config file: %s", (const char *)config_path); return; } BuildListFromString(config_path, file_buf); ats_free(file_buf); + + Note("volume.config finished loading"); + return; } @@ -715,7 +726,7 @@ ConfigVolumes::BuildListFromString(char *config_file_path, char *file_buf) // added by YTS Team, yamsat for bug id 59632 total += size; if (size > 100 || total > 100) { - err = "Total volume size added upto more than 100 percent, No volumes created"; + err = "Total volume size added up to more than 100 percent, No volumes created"; break; } // ends here @@ -765,7 +776,7 @@ ConfigVolumes::BuildListFromString(char *config_file_path, char *file_buf) return; } -/* Test the cache volumeing with different configurations */ +/* Test the cache volume with different configurations */ #define MEGS_128 (128 * 1024 * 1024) #define ROUND_TO_VOL_SIZE(_x) (((_x) + (MEGS_128 - 1)) & ~(MEGS_128 - 1)) extern CacheDisk **gdisks; @@ -855,7 +866,7 @@ create_config(RegressionTest *t, int num) total_space += vol_blocks; } - // make sure we have atleast 1280 M bytes + // make sure we have at least 1280 M bytes if (total_space<(10 << 27)>> STORE_BLOCK_SHIFT) { rprintf(t, "Not enough space for 10 volume\n"); return 0; diff --git a/iocore/cache/CacheHttp.cc b/iocore/cache/CacheHttp.cc index e78845f71fb..b4ff5d0919d 100644 --- a/iocore/cache/CacheHttp.cc +++ b/iocore/cache/CacheHttp.cc @@ -32,7 +32,7 @@ static vec_info default_vec_info; static CacheHTTPInfo default_http_info; -CacheHTTPInfoVector::CacheHTTPInfoVector() : magic(nullptr), data(&default_vec_info, 4), xcount(0) {} +CacheHTTPInfoVector::CacheHTTPInfoVector() : data(&default_vec_info, 4) {} /*------------------------------------------------------------------------- -------------------------------------------------------------------------*/ diff --git a/iocore/cache/CachePages.cc b/iocore/cache/CachePages.cc index 089cb5908ab..33f62beef5c 100644 --- a/iocore/cache/CachePages.cc +++ b/iocore/cache/CachePages.cc @@ -548,7 +548,7 @@ ShowCache::lookup_regex(int event, Event *e) " form.elements[0].value += urllist[c]+ \"%%0D%%0A\";\n" " }\n" " if (form.elements[0].value == \"\"){\n" - " alert(\"Please select atleast one url before clicking delete\");\n" + " alert(\"Please select at least one url before clicking delete\");\n" " return true;\n" "}\n" " srcfile=\"./delete_url?url=\" + form.elements[0].value;\n" diff --git a/iocore/cache/CacheRead.cc b/iocore/cache/CacheRead.cc index 662a1027695..085ffca4de4 100644 --- a/iocore/cache/CacheRead.cc +++ b/iocore/cache/CacheRead.cc @@ -172,7 +172,7 @@ CacheVC::load_http_info(CacheHTTPInfoVector *info, Doc *doc, RefCountObj *block_ if (!this->f.doc_from_ram_cache && // ram cache is always already fixed up. // If this is an old object, the object version will be old or 0, in either case this is // correct. Forget the 4.2 compatibility, always update older versioned objects. - VersionNumber(doc->v_major, doc->v_minor) < CACHE_DB_VERSION) { + ts::VersionNumber(doc->v_major, doc->v_minor) < CACHE_DB_VERSION) { for (int i = info->xcount - 1; i >= 0; --i) { info->data(i).alternate.m_alt->m_response_hdr.m_mime->recompute_accelerators_and_presence_bits(); info->data(i).alternate.m_alt->m_request_hdr.m_mime->recompute_accelerators_and_presence_bits(); @@ -838,7 +838,7 @@ CacheVC::openReadStartEarliest(int /* event ATS_UNUSED */, Event * /* e ATS_UNUS // an object needs to be outside the aggregation window in order to be // be evacuated as it is read if (!dir_agg_valid(vol, &dir)) { - // a directory entry which is nolonger valid may have been overwritten + // a directory entry which is no longer valid may have been overwritten if (!dir_valid(vol, &dir)) { last_collision = nullptr; } @@ -864,7 +864,7 @@ CacheVC::openReadStartEarliest(int /* event ATS_UNUSED */, Event * /* e ATS_UNUS last_collision = nullptr; goto Lread; } - if (!(doc->key == key)) { // collisiion + if (!(doc->key == key)) { // collision goto Lread; } // success @@ -1041,7 +1041,7 @@ CacheVC::openReadStartHead(int event, Event *e) // an object needs to be outside the aggregation window in order to be // be evacuated as it is read if (!dir_agg_valid(vol, &dir)) { - // a directory entry which is nolonger valid may have been overwritten + // a directory entry which is no longer valid may have been overwritten if (!dir_valid(vol, &dir)) { last_collision = nullptr; } @@ -1085,7 +1085,7 @@ CacheVC::openReadStartHead(int event, Event *e) if (buf) { HTTPCacheAlt *alt = reinterpret_cast(doc->hdr()); int32_t alt_length = 0; - // count should be reasonable, as vector is initialized and unlikly to be too corrupted + // count should be reasonable, as vector is initialized and unlikely to be too corrupted // by bad disk data - count should be the number of successfully unmarshalled alts. for (int32_t i = 0; i < vector.count(); ++i) { CacheHTTPInfo *info = vector.get(i); diff --git a/iocore/cache/CacheVol.cc b/iocore/cache/CacheVol.cc index 98009a459f7..113ca523eb7 100644 --- a/iocore/cache/CacheVol.cc +++ b/iocore/cache/CacheVol.cc @@ -400,7 +400,7 @@ CacheVC::scanOpenWrite(int /* event ATS_UNUSED */, Event * /* e ATS_UNUSED */) // get volume lock if (writer_lock_retry > SCAN_WRITER_LOCK_MAX_RETRY) { int r = _action.continuation->handleEvent(CACHE_EVENT_SCAN_OPERATION_BLOCKED, nullptr); - Debug("cache_scan", "still havent got the writer lock, asking user.."); + Debug("cache_scan", "still haven't got the writer lock, asking user.."); switch (r) { case CACHE_SCAN_RESULT_RETRY: writer_lock_retry = 0; diff --git a/iocore/cache/CacheWrite.cc b/iocore/cache/CacheWrite.cc index cd873ef40a6..55ec3ebd837 100644 --- a/iocore/cache/CacheWrite.cc +++ b/iocore/cache/CacheWrite.cc @@ -90,7 +90,7 @@ CacheVC::updateVector(int /* event ATS_UNUSED */, Event * /* e ATS_UNUSED */) return openWriteCloseDir(EVENT_IMMEDIATE, nullptr); } } - if (update_key == od->single_doc_key && (total_len || !vec)) { + if (update_key == od->single_doc_key && (total_len || f.allow_empty_doc || !vec)) { od->move_resident_alt = false; } } @@ -358,7 +358,7 @@ Vol::aggWriteDone(int event, Event *e) CacheVC *c = nullptr; while ((c = sync.dequeue())) { if (UINT_WRAP_LTE(c->write_serial + 2, header->write_serial)) { - c->initial_thread->schedule_imm_signal(c, AIO_EVENT_DONE); + eventProcessor.schedule_imm_signal(c, ET_CALL, AIO_EVENT_DONE); } else { sync.push(c); // put it back on the front break; @@ -400,7 +400,7 @@ CacheVC::evacuateReadHead(int /* event ATS_UNUSED */, Event * /* e ATS_UNUSED */ if (!io.ok()) { goto Ldone; } - // a directory entry which is nolonger valid may have been overwritten + // a directory entry which is no longer valid may have been overwritten if (!dir_valid(vol, &dir)) { last_collision = nullptr; goto Lcollision; @@ -500,7 +500,7 @@ CacheVC::evacuateDocDone(int /* event ATS_UNUSED */, Event * /* e ATS_UNUSED */) dir_overwrite(&doc->key, vol, &dir, &overwrite_dir); } // if the tag in the overwrite_dir matches the first_key in the - // document, then it has to be the vector. We gaurantee that + // document, then it has to be the vector. We guarantee that // the first_key and the earliest_key will never collide (see // Cache::open_write). Once we know its the vector, we can // safely overwrite the first_key in the directory. @@ -677,7 +677,7 @@ Vol::evacuateDocReadDone(int event, Event *e) b->f.unused = 87; } // if the tag in the c->dir does match the first_key in the - // document, then it has to be the earliest fragment. We gaurantee that + // document, then it has to be the earliest fragment. We guarantee that // the first_key and the earliest_key will never collide (see // Cache::open_write). if (!dir_head(&b->dir) || !dir_compare_tag(&b->dir, &doc->first_key)) { @@ -1019,11 +1019,7 @@ Vol::aggWrite(int event, void * /* e ATS_UNUSED */) ink_assert(false); while ((c = agg.dequeue())) { agg_todo_size -= c->agg_len; - if (c->initial_thread != nullptr) { - c->initial_thread->schedule_imm_signal(c, AIO_EVENT_DONE); - } else { - eventProcessor.schedule_imm_signal(c, ET_CALL, AIO_EVENT_DONE); - } + eventProcessor.schedule_imm_signal(c, ET_CALL, AIO_EVENT_DONE); } return EVENT_CONT; } @@ -1086,8 +1082,6 @@ Vol::aggWrite(int event, void * /* e ATS_UNUSED */) while ((c = tocall.dequeue())) { if (event == EVENT_CALL && c->mutex->thread_holding == mutex->thread_holding) { ret = EVENT_RETURN; - } else if (c->initial_thread != nullptr) { - c->initial_thread->schedule_imm_signal(c, AIO_EVENT_DONE); } else { eventProcessor.schedule_imm_signal(c, ET_CALL, AIO_EVENT_DONE); } @@ -1113,11 +1107,11 @@ CacheVC::openWriteCloseDir(int /* event ATS_UNUSED */, Event * /* e ATS_UNUSED * } if (is_debug_tag_set("cache_update")) { if (f.update && closed > 0) { - if (!total_len && alternate_index != CACHE_ALT_REMOVED) { + if (!total_len && !f.allow_empty_doc && alternate_index != CACHE_ALT_REMOVED) { Debug("cache_update", "header only %d (%" PRIu64 ", %" PRIu64 ")", DIR_MASK_TAG(first_key.slice32(2)), update_key.b[0], update_key.b[1]); - } else if (total_len && alternate_index != CACHE_ALT_REMOVED) { + } else if ((total_len || f.allow_empty_doc) && alternate_index != CACHE_ALT_REMOVED) { Debug("cache_update", "header body, %d, (%" PRIu64 ", %" PRIu64 "), (%" PRIu64 ", %" PRIu64 ")", DIR_MASK_TAG(first_key.slice32(2)), update_key.b[0], update_key.b[1], earliest_key.b[0], earliest_key.b[1]); } else if (!total_len && alternate_index == CACHE_ALT_REMOVED) { @@ -1369,7 +1363,9 @@ CacheVC::openWriteWriteDone(int event, Event *e) static inline int target_fragment_size() { - return cache_config_target_fragment_size - sizeof(Doc); + uint64_t value = cache_config_target_fragment_size - sizeof(Doc); + ink_release_assert(value <= MAX_FRAG_SIZE); + return value; } int diff --git a/iocore/cache/I_Cache.h b/iocore/cache/I_Cache.h index 14e19d90241..b0fd5441ef4 100644 --- a/iocore/cache/I_Cache.h +++ b/iocore/cache/I_Cache.h @@ -29,9 +29,7 @@ #include "I_CacheDefs.h" #include "I_Store.h" -#define CACHE_MODULE_MAJOR_VERSION 1 -#define CACHE_MODULE_MINOR_VERSION 0 -#define CACHE_MODULE_VERSION makeModuleVersion(CACHE_MODULE_MAJOR_VERSION, CACHE_MODULE_MINOR_VERSION, PUBLIC_MODULE_HEADER) +static constexpr ts::ModuleVersion CACHE_MODULE_VERSION(1, 0); #define CACHE_WRITE_OPT_OVERWRITE 0x0001 #define CACHE_WRITE_OPT_CLOSE_COMPLETE 0x0002 @@ -64,9 +62,8 @@ typedef HTTPInfo CacheHTTPInfo; struct CacheProcessor : public Processor { CacheProcessor() : min_stripe_version(CACHE_DB_MAJOR_VERSION, CACHE_DB_MINOR_VERSION), - max_stripe_version(CACHE_DB_MAJOR_VERSION, CACHE_DB_MINOR_VERSION), - cb_after_init(nullptr), - wait_for_cache(0) + max_stripe_version(CACHE_DB_MAJOR_VERSION, CACHE_DB_MINOR_VERSION) + { } @@ -158,11 +155,11 @@ struct CacheProcessor : public Processor { static int start_internal_flags; static int auto_clear_flag; - VersionNumber min_stripe_version; - VersionNumber max_stripe_version; + ts::VersionNumber min_stripe_version; + ts::VersionNumber max_stripe_version; - CALLBACK_FUNC cb_after_init; - int wait_for_cache; + CALLBACK_FUNC cb_after_init = nullptr; + int wait_for_cache = 0; }; inline void @@ -192,12 +189,10 @@ struct CacheVConnection : public VConnection { virtual void set_http_info(CacheHTTPInfo *info) = 0; virtual void get_http_info(CacheHTTPInfo **info) = 0; - virtual bool is_ram_cache_hit() const = 0; - virtual bool set_disk_io_priority(int priority) = 0; - virtual int get_disk_io_priority() = 0; - virtual bool set_pin_in_cache(time_t t) = 0; - virtual time_t get_pin_in_cache() = 0; - virtual int64_t get_object_size() = 0; + virtual bool is_ram_cache_hit() const = 0; + virtual bool set_pin_in_cache(time_t t) = 0; + virtual time_t get_pin_in_cache() = 0; + virtual int64_t get_object_size() = 0; virtual bool is_compressed_in_ram() const { @@ -218,6 +213,6 @@ struct CacheVConnection : public VConnection { CacheVConnection(); }; -void ink_cache_init(ModuleVersion version); +void ink_cache_init(ts::ModuleVersion version); extern inkcoreapi CacheProcessor cacheProcessor; extern Continuation *cacheRegexDeleteCont; diff --git a/iocore/cache/I_CacheDefs.h b/iocore/cache/I_CacheDefs.h index c4529745fd3..cdd39bb2199 100644 --- a/iocore/cache/I_CacheDefs.h +++ b/iocore/cache/I_CacheDefs.h @@ -33,11 +33,11 @@ #define CACHE_ALT_REMOVED -2 static const uint8_t CACHE_DB_MAJOR_VERSION = 24; -static const uint8_t CACHE_DB_MINOR_VERSION = 1; +static const uint8_t CACHE_DB_MINOR_VERSION = 2; // This is used in various comparisons because otherwise if the minor version is 0, // the compile fails because the condition is always true or false. Running it through // VersionNumber prevents that. -extern const VersionNumber CACHE_DB_VERSION; +extern const ts::VersionNumber CACHE_DB_VERSION; static const uint8_t CACHE_DIR_MAJOR_VERSION = 18; static const uint8_t CACHE_DIR_MINOR_VERSION = 0; diff --git a/iocore/cache/I_Store.h b/iocore/cache/I_Store.h index fbb48b87bfb..901ae9fd75a 100644 --- a/iocore/cache/I_Store.h +++ b/iocore/cache/I_Store.h @@ -70,17 +70,17 @@ struct span_diskid_t { // Those on the same disk should be in a linked list. // struct Span { - int64_t blocks; // in STORE_BLOCK_SIZE blocks - int64_t offset; // used only if (file == true); in bytes - unsigned hw_sector_size; - unsigned alignment; + int64_t blocks = 0; // in STORE_BLOCK_SIZE blocks + int64_t offset = 0; // used only if (file == true); in bytes + unsigned hw_sector_size = DEFAULT_HW_SECTOR_SIZE; + unsigned alignment = 0; span_diskid_t disk_id; - int forced_volume_num; ///< Force span in to specific volume. + int forced_volume_num = -1; ///< Force span in to specific volume. private: - bool is_mmapable_internal; + bool is_mmapable_internal = false; public: - bool file_pathname; // the pathname is a file + bool file_pathname = false; // the pathname is a file // v- used as a magic location for copy constructor. // we memcpy everything before this member and do explicit assignment for the rest. ats_scoped_str pathname; @@ -157,17 +157,7 @@ struct Span { /// Set the volume number. void volume_number_set(int n); - Span() - : blocks(0), - offset(0), - hw_sector_size(DEFAULT_HW_SECTOR_SIZE), - alignment(0), - forced_volume_num(-1), - is_mmapable_internal(false), - file_pathname(false) - { - disk_id[0] = disk_id[1] = 0; - } + Span() { disk_id[0] = disk_id[1] = 0; } /// Copy constructor. /// @internal Prior to this implementation handling the char* pointers was done manual @@ -259,10 +249,10 @@ struct Store { ~Store(); // The number of disks/paths defined in storage.config - unsigned n_disks_in_config; + unsigned n_disks_in_config = 0; // The number of disks/paths we could actually read and parse. - unsigned n_disks; - Span **disk; + unsigned n_disks = 0; + Span **disk = nullptr; Result read_config(); diff --git a/iocore/cache/Makefile.am b/iocore/cache/Makefile.am index ed05579e428..b2161f0b322 100644 --- a/iocore/cache/Makefile.am +++ b/iocore/cache/Makefile.am @@ -66,6 +66,145 @@ libinkcache_a_SOURCES += \ P_CacheTest.h endif +TESTS = $(check_PROGRAMS) + +test_CPPFLAGS = \ + $(AM_CPPFLAGS) \ + $(iocore_include_dirs) \ + -I$(abs_top_srcdir)/include \ + -I$(abs_top_srcdir)/lib \ + -I$(abs_top_srcdir)/proxy \ + -I$(abs_top_srcdir)/proxy/http \ + -I$(abs_top_srcdir)/proxy/http2 \ + -I$(abs_top_srcdir)/proxy/http3 \ + -I$(abs_top_srcdir)/proxy/logging \ + -I$(abs_top_srcdir)/proxy/http/remap \ + -I$(abs_top_srcdir)/proxy/hdrs \ + -I$(abs_top_srcdir)/proxy/shared \ + -I$(abs_top_srcdir)/mgmt \ + -I$(abs_top_srcdir)/mgmt/utils \ + -I$(abs_top_srcdir)/tests/include \ + $(TS_INCLUDES) \ + @OPENSSL_INCLUDES@ + +test_LDADD = \ + $(top_builddir)/src/tscpp/util/libtscpputil.la \ + $(top_builddir)/iocore/cache/libinkcache.a \ + $(top_builddir)/proxy/libproxy.a \ + $(top_builddir)/proxy/http/libhttp.a \ + $(top_builddir)/proxy/http/remap/libhttp_remap.a \ + $(top_builddir)/proxy/libproxy.a \ + $(top_builddir)/iocore/net/libinknet.a \ + $(top_builddir)/iocore/dns/libinkdns.a \ + $(top_builddir)/iocore/hostdb/libinkhostdb.a \ + $(top_builddir)/proxy/logging/liblogging.a \ + $(top_builddir)/proxy/hdrs/libhdrs.a \ + $(top_builddir)/proxy/shared/libdiagsconfig.a \ + $(top_builddir)/mgmt/libmgmt_p.la \ + $(top_builddir)/iocore/utils/libinkutils.a \ + $(top_builddir)/iocore/aio/libinkaio.a \ + $(top_builddir)/src/tscore/libtscore.la \ + $(top_builddir)/lib/records/librecords_p.a \ + $(top_builddir)/iocore/eventsystem/libinkevent.a \ + $(top_builddir)/lib/tsconfig/libtsconfig.la \ + @HWLOC_LIBS@ \ + @LIBPCRE@ \ + @LIBRESOLV@ \ + @LIBZ@ \ + @LIBLZMA@ \ + @LIBPROFILER@ \ + @OPENSSL_LIBS@ \ + @YAMLCPP_LIBS@ \ + -lm + + +check_PROGRAMS = \ + test_Cache \ + test_RWW \ + test_Alternate_L_to_S \ + test_Alternate_S_to_L \ + test_Alternate_L_to_S_remove_L \ + test_Alternate_L_to_S_remove_S \ + test_Alternate_S_to_L_remove_S \ + test_Alternate_S_to_L_remove_L \ + test_Update_L_to_S \ + test_Update_S_to_L + +test_main_SOURCES = \ + ./test/main.cc \ + ./test/stub.cc \ + ./test/CacheTestHandler.cc + +test_Cache_CPPFLAGS = $(test_CPPFLAGS) +test_Cache_LDFLAGS = @AM_LDFLAGS@ +test_Cache_LDADD = $(test_LDADD) +test_Cache_SOURCES = \ + $(test_main_SOURCES) \ + ./test/test_Cache.cc + +test_RWW_CPPFLAGS = $(test_CPPFLAGS) +test_RWW_LDFLAGS = @AM_LDFLAGS@ +test_RWW_LDADD = $(test_LDADD) +test_RWW_SOURCES = \ + $(test_main_SOURCES) \ + ./test/test_RWW.cc + +test_Alternate_L_to_S_CPPFLAGS = $(test_CPPFLAGS) +test_Alternate_L_to_S_LDFLAGS = @AM_LDFLAGS@ +test_Alternate_L_to_S_LDADD = $(test_LDADD) +test_Alternate_L_to_S_SOURCES = \ + $(test_main_SOURCES) \ + ./test/test_Alternate_L_to_S.cc + +test_Alternate_S_to_L_CPPFLAGS = $(test_CPPFLAGS) +test_Alternate_S_to_L_LDFLAGS = @AM_LDFLAGS@ +test_Alternate_S_to_L_LDADD = $(test_LDADD) +test_Alternate_S_to_L_SOURCES = \ + $(test_main_SOURCES) \ + ./test/test_Alternate_S_to_L.cc + +test_Alternate_L_to_S_remove_L_CPPFLAGS = $(test_CPPFLAGS) +test_Alternate_L_to_S_remove_L_LDFLAGS = @AM_LDFLAGS@ +test_Alternate_L_to_S_remove_L_LDADD = $(test_LDADD) +test_Alternate_L_to_S_remove_L_SOURCES = \ + $(test_main_SOURCES) \ + ./test/test_Alternate_L_to_S_remove_L.cc + +test_Alternate_L_to_S_remove_S_CPPFLAGS = $(test_CPPFLAGS) +test_Alternate_L_to_S_remove_S_LDFLAGS = @AM_LDFLAGS@ +test_Alternate_L_to_S_remove_S_LDADD = $(test_LDADD) +test_Alternate_L_to_S_remove_S_SOURCES = \ + $(test_main_SOURCES) \ + ./test/test_Alternate_L_to_S_remove_S.cc + +test_Alternate_S_to_L_remove_S_CPPFLAGS = $(test_CPPFLAGS) +test_Alternate_S_to_L_remove_S_LDFLAGS = @AM_LDFLAGS@ +test_Alternate_S_to_L_remove_S_LDADD = $(test_LDADD) +test_Alternate_S_to_L_remove_S_SOURCES = \ + $(test_main_SOURCES) \ + ./test/test_Alternate_S_to_L_remove_S.cc + +test_Alternate_S_to_L_remove_L_CPPFLAGS = $(test_CPPFLAGS) +test_Alternate_S_to_L_remove_L_LDFLAGS = @AM_LDFLAGS@ +test_Alternate_S_to_L_remove_L_LDADD = $(test_LDADD) +test_Alternate_S_to_L_remove_L_SOURCES = \ + $(test_main_SOURCES) \ + ./test/test_Alternate_S_to_L_remove_L.cc + +test_Update_L_to_S_CPPFLAGS = $(test_CPPFLAGS) +test_Update_L_to_S_LDFLAGS = @AM_LDFLAGS@ +test_Update_L_to_S_LDADD = $(test_LDADD) +test_Update_L_to_S_SOURCES = \ + $(test_main_SOURCES) \ + ./test/test_Update_L_to_S.cc + +test_Update_S_to_L_CPPFLAGS = $(test_CPPFLAGS) +test_Update_S_to_L_LDFLAGS = @AM_LDFLAGS@ +test_Update_S_to_L_LDADD = $(test_LDADD) +test_Update_S_to_L_SOURCES = \ + $(test_main_SOURCES) \ + ./test/test_Update_S_to_L.cc + include $(top_srcdir)/build/tidy.mk clang-tidy-local: $(DIST_SOURCES) diff --git a/iocore/cache/P_CacheDir.h b/iocore/cache/P_CacheDir.h index 27a31b4acaf..58c1c9ffd3c 100644 --- a/iocore/cache/P_CacheDir.h +++ b/iocore/cache/P_CacheDir.h @@ -258,29 +258,18 @@ struct OpenDir : public Continuation { }; struct CacheSync : public Continuation { - int vol_idx; - char *buf; - size_t buflen; - bool buf_huge; - off_t writepos; + int vol_idx = 0; + char *buf = nullptr; + size_t buflen = 0; + bool buf_huge = false; + off_t writepos = 0; AIOCallbackInternal io; - Event *trigger; - ink_hrtime start_time; + Event *trigger = nullptr; + ink_hrtime start_time = 0; int mainEvent(int event, Event *e); void aio_write(int fd, char *b, int n, off_t o); - CacheSync() - : Continuation(new_ProxyMutex()), - vol_idx(0), - buf(nullptr), - buflen(0), - buf_huge(false), - writepos(0), - trigger(nullptr), - start_time(0) - { - SET_HANDLER(&CacheSync::mainEvent); - } + CacheSync() : Continuation(new_ProxyMutex()) { SET_HANDLER(&CacheSync::mainEvent); } }; // Global Functions @@ -310,7 +299,7 @@ void sync_cache_dir_on_shutdown(); extern Dir empty_dir; -// Inline Funtions +// Inline Functions #define dir_in_seg(_s, _i) ((Dir *)(((char *)(_s)) + (SIZEOF_DIR * (_i)))) diff --git a/iocore/cache/P_CacheDisk.h b/iocore/cache/P_CacheDisk.h index 51f2462bf40..18379de9d6e 100644 --- a/iocore/cache/P_CacheDisk.h +++ b/iocore/cache/P_CacheDisk.h @@ -53,11 +53,11 @@ struct DiskVolBlock { }; struct DiskVolBlockQueue { - DiskVolBlock *b; - int new_block; /* whether an existing vol or a new one */ + DiskVolBlock *b = nullptr; + int new_block = 0; /* whether an existing vol or a new one */ LINK(DiskVolBlockQueue, link); - DiskVolBlockQueue() : b(nullptr), new_block(0) {} + DiskVolBlockQueue() {} }; struct DiskVol { diff --git a/iocore/cache/P_CacheHosting.h b/iocore/cache/P_CacheHosting.h index 87afc045517..90822e2c44b 100644 --- a/iocore/cache/P_CacheHosting.h +++ b/iocore/cache/P_CacheHosting.h @@ -46,34 +46,24 @@ struct CacheHostRecord { ats_free(cp); } - CacheType type; - Vol **vols; - int good_num_vols; - int num_vols; - int num_initialized; - unsigned short *vol_hash_table; - CacheVol **cp; - int num_cachevols; - - CacheHostRecord() - : type(CACHE_NONE_TYPE), - vols(nullptr), - good_num_vols(0), - num_vols(0), - num_initialized(0), - vol_hash_table(nullptr), - cp(nullptr), - num_cachevols(0) - { - } + CacheType type = CACHE_NONE_TYPE; + Vol **vols = nullptr; + int good_num_vols = 0; + int num_vols = 0; + int num_initialized = 0; + unsigned short *vol_hash_table = nullptr; + CacheVol **cp = nullptr; + int num_cachevols = 0; + + CacheHostRecord() {} }; void build_vol_hash_table(CacheHostRecord *cp); struct CacheHostResult { - CacheHostRecord *record; + CacheHostRecord *record = nullptr; - CacheHostResult() : record(nullptr) {} + CacheHostResult() {} }; class CacheHostMatcher @@ -108,7 +98,7 @@ class CacheHostMatcher HostLookup *host_lookup; // Data structure to do the lookups CacheHostRecord *data_array; // array of all data items int array_len; // the length of the arrays - int num_el; // the number of itmems in the tree + int num_el; // the number of items in the tree CacheType type; }; @@ -194,7 +184,7 @@ struct ConfigVolumes { void BuildListFromString(char *config_file_path, char *file_buf); void - clear_all(void) + clear_all() { // remove all the volumes from the queue for (int i = 0; i < num_volumes; i++) { diff --git a/iocore/cache/P_CacheHttp.h b/iocore/cache/P_CacheHttp.h index 24ddabe7acb..c9932a8124a 100644 --- a/iocore/cache/P_CacheHttp.h +++ b/iocore/cache/P_CacheHttp.h @@ -43,7 +43,7 @@ struct vec_info { }; struct CacheHTTPInfoVector { - void *magic; + void *magic = nullptr; CacheHTTPInfoVector(); ~CacheHTTPInfoVector(); @@ -72,7 +72,7 @@ struct CacheHTTPInfoVector { int unmarshal(const char *buf, int length, RefCountObj *block_ptr); CacheArray data; - int xcount; + int xcount = 0; Ptr vector_buf; }; diff --git a/iocore/cache/P_CacheInternal.h b/iocore/cache/P_CacheInternal.h index d82aa347324..1a61b7194c0 100644 --- a/iocore/cache/P_CacheInternal.h +++ b/iocore/cache/P_CacheInternal.h @@ -388,8 +388,6 @@ struct CacheVC : public CacheVConnection { bool is_pread_capable() override; bool set_pin_in_cache(time_t time_pin) override; time_t get_pin_in_cache() override; - bool set_disk_io_priority(int priority) override; - int get_disk_io_priority() override; // offsets from the base stat #define CACHE_STAT_ACTIVE 0 @@ -425,7 +423,7 @@ struct CacheVC : public CacheVConnection { OpenDirEntry *od; AIOCallbackInternal io; - int alternate_index; // preferred position in vector + int alternate_index = CACHE_ALT_INDEX_DEFAULT; // preferred position in vector LINK(CacheVC, opendir_link); #ifdef CACHE_STAT_PAGES LINK(CacheVC, stat_link); @@ -435,12 +433,11 @@ struct CacheVC : public CacheVConnection { // Start Region C // These variables are memset to 0 when the structure is freed. // The size of this region is size_to_init which is initialized - // in the CacheVC constuctor. It assumes that vio is the start + // in the CacheVC constructor. It assumes that vio is the start // of this region. // NOTE: NOTE: NOTE: If vio is NOT the start, then CHANGE the // size_to_init initialization VIO vio; - EThread *initial_thread; // initial thread open_XX was called on CacheFragType frag_type; CacheHTTPInfo *info; CacheHTTPInfoVector *write_vector; @@ -549,9 +546,9 @@ new_CacheVC(Continuation *cont) CacheVC *c = THREAD_ALLOC(cacheVConnectionAllocator, t); c->vector.data.data = &c->vector.data.fast_data[0]; c->_action = cont; - c->initial_thread = t->tt == DEDICATED ? nullptr : t; c->mutex = cont->mutex; c->start_time = Thread::get_hrtime(); + c->setThreadAffinity(t); ink_assert(c->trigger == nullptr); Debug("cache_new", "new %p", c); #ifdef CACHE_STAT_PAGES @@ -581,14 +578,13 @@ free_CacheVC(CacheVC *cont) ink_assert(!cont->is_io_in_progress()); ink_assert(!cont->od); /* calling cont->io.action = nullptr causes compile problem on 2.6 solaris - release build....wierd??? For now, null out continuation and mutex + release build....weird??? For now, null out continuation and mutex of the action separately */ cont->io.action.continuation = nullptr; cont->io.action.mutex = nullptr; cont->io.mutex.clear(); - cont->io.aio_result = 0; - cont->io.aiocb.aio_nbytes = 0; - cont->io.aiocb.aio_reqprio = AIO_DEFAULT_PRIORITY; + cont->io.aio_result = 0; + cont->io.aiocb.aio_nbytes = 0; cont->request.reset(); cont->vector.clear(); cont->vio.buffer.clear(); @@ -956,8 +952,8 @@ CacheRemoveCont::event_handler(int event, void *data) return EVENT_DONE; } -int64_t cache_bytes_used(void); -int64_t cache_bytes_total(void); +int64_t cache_bytes_used(); +int64_t cache_bytes_total(); #ifdef DEBUG #define CACHE_DEBUG_INCREMENT_DYN_STAT(_x) CACHE_INCREMENT_DYN_STAT(_x) @@ -972,14 +968,14 @@ struct Vol; class CacheHostTable; struct Cache { - int cache_read_done; - int total_good_nvol; - int total_nvol; - int ready; - int64_t cache_size; // in store block size - CacheHostTable *hosttable; - int total_initialized_vol; - CacheType scheme; + int cache_read_done = 0; + int total_good_nvol = 0; + int total_nvol = 0; + int ready = CACHE_INITIALIZING; + int64_t cache_size = 0; // in store block size + CacheHostTable *hosttable = nullptr; + int total_initialized_vol = 0; + CacheType scheme = CACHE_NONE_TYPE; int open(bool reconfigure, bool fix); int close(); @@ -1010,17 +1006,7 @@ struct Cache { Vol *key_to_vol(const CacheKey *key, const char *hostname, int host_len); - Cache() - : cache_read_done(0), - total_good_nvol(0), - total_nvol(0), - ready(CACHE_INITIALIZING), - cache_size(0), // in store block size - hosttable(nullptr), - total_initialized_vol(0), - scheme(CACHE_NONE_TYPE) - { - } + Cache() {} }; extern Cache *theCache; diff --git a/iocore/cache/P_CacheTest.h b/iocore/cache/P_CacheTest.h index 3b369fd3c6e..ba9814e72ae 100644 --- a/iocore/cache/P_CacheTest.h +++ b/iocore/cache/P_CacheTest.h @@ -48,17 +48,17 @@ struct PinnedDocTable : public Continuation { }; struct CacheTestHost { - char *name; - unsigned int xlast_cachable_id; - double xprev_host_prob; - double xnext_host_prob; + char *name = nullptr; + unsigned int xlast_cachable_id = 0; + double xprev_host_prob = 0; + double xnext_host_prob = 0; - CacheTestHost() : name(nullptr), xlast_cachable_id(0), xprev_host_prob(0), xnext_host_prob(0) {} + CacheTestHost() {} }; struct CacheTestHeader { - CacheTestHeader() : serial(0) {} - uint64_t serial; + CacheTestHeader() {} + uint64_t serial = 0; }; struct CacheTestSM : public RegressionSM { diff --git a/iocore/cache/P_CacheVol.h b/iocore/cache/P_CacheVol.h index a59df2c848b..5b3254ea7bf 100644 --- a/iocore/cache/P_CacheVol.h +++ b/iocore/cache/P_CacheVol.h @@ -76,7 +76,7 @@ struct CacheVol; struct VolHeaderFooter { unsigned int magic; - VersionNumber version; + ts::VersionNumber version; time_t create_time; off_t write_pos; off_t last_write_pos; @@ -279,17 +279,17 @@ struct AIO_Callback_handler : public Continuation { }; struct CacheVol { - int vol_number; - int scheme; - off_t size; - int num_vols; - Vol **vols; - DiskVol **disk_vols; + int vol_number = -1; + int scheme = 0; + off_t size = 0; + int num_vols = 0; + Vol **vols = nullptr; + DiskVol **disk_vols = nullptr; LINK(CacheVol, link); // per volume stats - RecRawStatBlock *vol_rsb; + RecRawStatBlock *vol_rsb = nullptr; - CacheVol() : vol_number(-1), scheme(0), size(0), num_vols(0), vols(nullptr), disk_vols(nullptr), vol_rsb(nullptr) {} + CacheVol() {} }; // Note : hdr() needs to be 8 byte aligned. diff --git a/iocore/cache/RamCacheCLFUS.cc b/iocore/cache/RamCacheCLFUS.cc index 9cb4c44d9b1..cca0ee1b73a 100644 --- a/iocore/cache/RamCacheCLFUS.cc +++ b/iocore/cache/RamCacheCLFUS.cc @@ -35,7 +35,7 @@ #endif #define REQUIRED_COMPRESSION 0.9 // must get to this size or declared incompressible -#define REQUIRED_SHRINK 0.8 // must get to this size or keep orignal buffer (with padding) +#define REQUIRED_SHRINK 0.8 // must get to this size or keep original buffer (with padding) #define HISTORY_HYSTERIA 10 // extra temporary history #define ENTRY_OVERHEAD 256 // per-entry overhead to consider when computing cache value/size #define LZMA_BASE_MEMLIMIT (64 * 1024 * 1024) @@ -53,7 +53,7 @@ struct RamCacheCLFUSEntry { uint32_t auxkey1; uint32_t auxkey2; uint64_t hits; - uint32_t size; // memory used including paddding in buffer + uint32_t size; // memory used including padding in buffer uint32_t len; // actual data length uint32_t compressed_len; union { @@ -71,9 +71,9 @@ struct RamCacheCLFUSEntry { }; struct RamCacheCLFUS : public RamCache { - int64_t max_bytes; - int64_t bytes; - int64_t objects; + int64_t max_bytes = 0; + int64_t bytes = 0; + int64_t objects = 0; // returns 1 on found/stored, 0 on not found/stored, if provided auxkey1 and auxkey2 must match int get(CryptoHash *key, Ptr *ret_data, uint32_t auxkey1 = 0, uint32_t auxkey2 = 0) override; @@ -85,16 +85,16 @@ struct RamCacheCLFUS : public RamCache { void init(int64_t max_bytes, Vol *vol) override; // private - Vol *vol; // for stats - double average_value; - int64_t history; - int ibuckets; - int nbuckets; + Vol *vol = nullptr; // for stats + double average_value = 0; + int64_t history = 0; + int ibuckets = 0; + int nbuckets = 0; DList(RamCacheCLFUSEntry, hash_link) * bucket; Que(RamCacheCLFUSEntry, lru_link) lru[2]; - uint16_t *seen; - int ncompressed; - RamCacheCLFUSEntry *compressed; // first uncompressed lru[0] entry + uint16_t *seen = nullptr; + int ncompressed = 0; + RamCacheCLFUSEntry *compressed = nullptr; // first uncompressed lru[0] entry void compress_entries(EThread *thread, int do_at_most = INT_MAX); void resize_hashtable(); void victimize(RamCacheCLFUSEntry *e); @@ -102,21 +102,7 @@ struct RamCacheCLFUS : public RamCache { RamCacheCLFUSEntry *destroy(RamCacheCLFUSEntry *e); void requeue_victims(Que(RamCacheCLFUSEntry, lru_link) & victims); void tick(); // move CLOCK on history - RamCacheCLFUS() - : max_bytes(0), - bytes(0), - objects(0), - vol(nullptr), - average_value(0), - history(0), - ibuckets(0), - nbuckets(0), - bucket(nullptr), - seen(nullptr), - ncompressed(0), - compressed(nullptr) - { - } + RamCacheCLFUS() : bucket(nullptr) {} }; int64_t diff --git a/iocore/cache/Store.cc b/iocore/cache/Store.cc index d3028160b6f..778306ac1e5 100644 --- a/iocore/cache/Store.cc +++ b/iocore/cache/Store.cc @@ -46,7 +46,7 @@ make_span_error(int error) switch (error) { case ENOENT: return SPAN_ERROR_NOT_FOUND; - case EPERM: /* fallthru */ + case EPERM: /* fallthrough */ case EACCES: return SPAN_ERROR_NO_ACCESS; default: @@ -72,7 +72,7 @@ span_file_typename(mode_t st_mode) } Ptr tmp_p; -Store::Store() : n_disks_in_config(0), n_disks(0), disk(nullptr) {} +Store::Store() {} void Store::add(Span *ds) @@ -213,7 +213,7 @@ Span::errorstr(span_error_t serr) return "unsupported cache file type"; case SPAN_ERROR_MEDIA_PROBE: return "failed to probe device geometry"; - case SPAN_ERROR_UNKNOWN: /* fallthru */ + case SPAN_ERROR_UNKNOWN: /* fallthrough */ default: return "unknown error"; } @@ -327,9 +327,11 @@ Store::read_config() ats_scoped_fd fd; ats_scoped_str storage_path(RecConfigReadConfigPath("proxy.config.cache.storage_filename", "storage.config")); + Note("storage.config loading ..."); Debug("cache_init", "Store::read_config, fd = -1, \"%s\"", (const char *)storage_path); fd = ::open(storage_path, O_RDONLY); if (fd < 0) { + Error("storage.config failed to load"); return Result::failure("open %s: %s", (const char *)storage_path, strerror(errno)); } @@ -369,6 +371,7 @@ Store::read_config() if (ParseRules::is_digit(*e)) { if ((size = ink_atoi64(e)) <= 0) { delete sd; + Error("storage.config failed to load"); return Result::failure("failed to parse size '%s'", e); } } else if (0 == strncasecmp(HASH_BASE_STRING_KEY, e, sizeof(HASH_BASE_STRING_KEY) - 1)) { @@ -386,6 +389,7 @@ Store::read_config() } if (!*e || !ParseRules::is_digit(*e) || 0 >= (volume_num = ink_atoi(e))) { delete sd; + Error("storage.config failed to load"); return Result::failure("failed to parse volume number '%s'", e); } } @@ -437,6 +441,8 @@ Store::read_config() sd = nullptr; // these are all used. sort(); + Note("storage.config finished loading"); + return Result::ok(); } @@ -646,7 +652,7 @@ void Store::spread_alloc(Store &s, unsigned int blocks, bool mmapable) { // - // Count the eligable disks.. + // Count the eligible disks.. // int mmapable_disks = 0; for (unsigned k = 0; k < n_disks; k++) { @@ -732,7 +738,7 @@ Store::try_realloc(Store &s, Store &diff) } // -// Stupid grab first availabled space allocator +// Stupid grab first available space allocator // void Store::alloc(Store &s, unsigned int blocks, bool one_only, bool mmapable) diff --git a/iocore/cache/test/CacheTestHandler.cc b/iocore/cache/test/CacheTestHandler.cc new file mode 100644 index 00000000000..72ceceaf934 --- /dev/null +++ b/iocore/cache/test/CacheTestHandler.cc @@ -0,0 +1,76 @@ +/** @file + + A brief file description + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + 1 test_Cache.cc + X distributed with this work for additional information regarding copyright ownership. The ASF licenses this + file to you under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#include "main.h" +#include "CacheTestHandler.h" + +TestContChain::TestContChain() : Continuation(new_ProxyMutex()) {} + +CacheTestHandler::CacheTestHandler(size_t size, const char *url) +{ + this->_wt = new CacheWriteTest(size, this, url); + this->_rt = new CacheReadTest(size, this, url); + + this->_wt->mutex = this->mutex; + this->_rt->mutex = this->mutex; + SET_HANDLER(&CacheTestHandler::start_test); +} + +void +CacheTestHandler::handle_cache_event(int event, CacheTestBase *base) +{ + REQUIRE(base != nullptr); + switch (event) { + case CACHE_EVENT_OPEN_READ: + base->do_io_read(); + break; + case CACHE_EVENT_OPEN_WRITE: + base->do_io_write(); + break; + case VC_EVENT_READ_READY: + case VC_EVENT_WRITE_READY: + REQUIRE(base->vc != nullptr); + REQUIRE(base->vio != nullptr); + base->reenable(); + break; + case VC_EVENT_WRITE_COMPLETE: + this_ethread()->schedule_imm(this->_rt); + base->close(); + break; + case VC_EVENT_READ_COMPLETE: + base->close(); + delete this; + break; + default: + REQUIRE(false); + base->close(); + delete this; + break; + } + return; +} + +int +CacheTestHandler::start_test(int event, void *e) +{ + this_ethread()->schedule_imm(this->_wt); + return 0; +} diff --git a/iocore/cache/test/CacheTestHandler.h b/iocore/cache/test/CacheTestHandler.h new file mode 100644 index 00000000000..3d2c1a0b697 --- /dev/null +++ b/iocore/cache/test/CacheTestHandler.h @@ -0,0 +1,99 @@ +/** @file + + A brief file description + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#pragma once + +void test_done(); + +#define TEST_DONE() test_done(); +#define T_DONE 1 +#define T_CONT 1 + +#define DEFAULT_URL "http://www.scw00.com/" + +#include + +class CacheTestBase; + +struct TestContChain : public Continuation { + TestContChain(); + virtual ~TestContChain() { this->next_test(); } + + void + add(TestContChain *n) + { + TestContChain *p = this; + while (p->next) { + p = p->next; + } + p->next = n; + } + + bool + next_test() + { + if (!this->next) { + return false; + } + + this_ethread()->schedule_imm(this->next); + return true; + } + + TestContChain *next = nullptr; +}; + +class CacheTestHandler : public TestContChain +{ +public: + CacheTestHandler() = default; + CacheTestHandler(size_t size, const char *url = DEFAULT_URL); + + int start_test(int event, void *e); + + virtual void handle_cache_event(int event, CacheTestBase *base); + +protected: + CacheTestBase *_rt = nullptr; + CacheTestBase *_wt = nullptr; +}; + +class TerminalTest : public CacheTestHandler +{ +public: + TerminalTest() { SET_HANDLER(&TerminalTest::terminal_event); } + ~TerminalTest() { TEST_DONE(); } + + int + terminal_event(int event, void *e) + { + delete this; + return 0; + } + + void + handle_cache_event(int event, CacheTestBase *e) override + { + delete this; + } +}; diff --git a/iocore/cache/test/main.cc b/iocore/cache/test/main.cc new file mode 100644 index 00000000000..c9427c464f0 --- /dev/null +++ b/iocore/cache/test/main.cc @@ -0,0 +1,278 @@ +/** @file + + A brief file description + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#define CATCH_CONFIG_MAIN +#include "main.h" + +#define THREADS 1 +#define DIAGS_LOG_FILE "diags.log" + +void +test_done() +{ + TSSystemState::shut_down_event_system(); +} + +const char *GLOBAL_DATA = (char *)ats_malloc(10 * 1024 * 1024 + 3); // 10M + +struct EventProcessorListener : Catch::TestEventListenerBase { + using TestEventListenerBase::TestEventListenerBase; // inherit constructor + + virtual void + testRunStarting(Catch::TestRunInfo const &testRunInfo) override + { + BaseLogFile *base_log_file = new BaseLogFile("stderr"); + diags = new Diags(testRunInfo.name.c_str(), "*" /* tags */, "" /* actions */, base_log_file); + diags->activate_taglist("cache.*|agg.*|locks", DiagsTagType_Debug); + diags->config.enabled[DiagsTagType_Debug] = true; + diags->show_location = SHOW_LOCATION_DEBUG; + + mime_init(); + Layout::create(); + RecProcessInit(RECM_STAND_ALONE); + LibRecordsConfigInit(); + ink_net_init(ts::ModuleVersion(1, 0, ts::ModuleVersion::PRIVATE)); + ink_assert(GLOBAL_DATA != nullptr); + + statPagesManager.init(); // mutex needs to be initialized before calling netProcessor.init + netProcessor.init(); + eventProcessor.start(THREADS); + + ink_aio_init(AIO_MODULE_PUBLIC_VERSION); + + EThread *thread = new EThread(); + thread->set_specific(); + init_buffer_allocators(0); + + std::string src_dir = std::string(TS_ABS_TOP_SRCDIR) + "/iocore/cache/test"; + Layout::get()->sysconfdir = src_dir; + Layout::get()->prefix = src_dir; + ::remove("./test/var/trafficserver/cache.db"); + } +}; +CATCH_REGISTER_LISTENER(EventProcessorListener); + +extern Store theCacheStore; + +void +init_cache(size_t size, const char *name) +{ + ink_cache_init(ts::ModuleVersion(1, 0, ts::ModuleVersion::PRIVATE)); + cacheProcessor.start(); +} + +void +build_hdrs(HTTPInfo &info, const char *url, const char *content_type) +{ + HTTPHdr req; + HTTPHdr resp; + HTTPParser parser; + int err = -1; + char buf[1024] = {0}; + const char *start = buf; + char *p = buf; + + REQUIRE(url != nullptr); + + p += sprintf(p, "GET %s HTTP/1.1\n", url); + p += sprintf(p, "User-Agent: curl/7.47.0\n"); + p += sprintf(p, "Accept: %s\n", content_type); + p += sprintf(p, "Vary: Content-type\n"); + p += sprintf(p, "Proxy-Connection: Keep-Alive\n\n"); + + req.create(HTTP_TYPE_REQUEST); + http_parser_init(&parser); + + while (true) { + err = req.parse_req(&parser, &start, p, true); + if (err != PARSE_RESULT_CONT) { + break; + } + } + + ink_assert(err == PARSE_RESULT_DONE); + + memset(buf, 0, sizeof(buf)); + p = buf; + + if (content_type == nullptr) { + content_type = "application/octet-stream"; + } + + p = buf; + p += sprintf(p, "HTTP/1.1 200 OK\n"); + p += sprintf(p, "Content-Type: %s\n", content_type); + p += sprintf(p, "Expires: Fri, 15 Mar 2219 08:55:45 GMT\n"); + p += sprintf(p, "Last-Modified: Thu, 14 Mar 2019 08:47:40 GMT\n\n"); + + resp.create(HTTP_TYPE_RESPONSE); + http_parser_init(&parser); + start = buf; + + while (true) { + err = resp.parse_resp(&parser, &start, p, true); + if (err != PARSE_RESULT_CONT) { + break; + } + } + ink_assert(err == PARSE_RESULT_DONE); + + info.request_set(&req); + info.response_set(&resp); + + req.destroy(); + resp.destroy(); +} + +HttpCacheKey +generate_key(HTTPInfo &info) +{ + HttpCacheKey key; + Cache::generate_key(&key, info.request_get()->url_get(), 1); + return key; +} + +void +CacheWriteTest::fill_data() +{ + size_t size = std::min(WRITE_LIMIT, this->_size); + auto n = this->_write_buffer->write(this->_cursor, size); + this->_size -= n; + this->_cursor += n; +} + +int +CacheWriteTest::write_event(int event, void *e) +{ + switch (event) { + case CACHE_EVENT_OPEN_WRITE: + this->vc = (CacheVC *)e; + /* fall through */ + case CACHE_EVENT_OPEN_WRITE_FAILED: + this->process_event(event); + break; + case VC_EVENT_WRITE_READY: + this->process_event(event); + this->fill_data(); + break; + case VC_EVENT_WRITE_COMPLETE: + this->process_event(event); + break; + default: + this->close(); + CHECK(false); + break; + } + return 0; +} + +void +CacheWriteTest::do_io_write(size_t size) +{ + if (size == 0) { + size = this->_size; + } + this->vc->set_http_info(&this->info); + this->vio = this->vc->do_io_write(this, size, this->_write_buffer->alloc_reader()); +} + +int +CacheWriteTest::start_test(int event, void *e) +{ + Debug("cache test", "start write test"); + + HttpCacheKey key; + key = generate_key(this->info); + + HTTPInfo *old_info = &this->old_info; + if (!old_info->valid()) { + old_info = nullptr; + } + + SET_HANDLER(&CacheWriteTest::write_event); + cacheProcessor.open_write(this, 0, &key, (CacheHTTPHdr *)this->info.request_get(), old_info); + return 0; +} + +int +CacheReadTest::read_event(int event, void *e) +{ + switch (event) { + case CACHE_EVENT_OPEN_READ: + this->vc = (CacheVC *)e; + /* fall through */ + case CACHE_EVENT_OPEN_READ_FAILED: + this->process_event(event); + break; + case VC_EVENT_READ_READY: { + while (this->_reader->block_read_avail()) { + auto str = this->_reader->block_read_view(); + if (memcmp(str.data(), this->_cursor, str.size()) == 0) { + this->_reader->consume(str.size()); + this->_cursor += str.size(); + this->process_event(event); + } else { + CHECK(false); + this->close(); + TEST_DONE(); + break; + } + } + break; + } + case VC_EVENT_ERROR: + case VC_EVENT_EOS: + case VC_EVENT_READ_COMPLETE: + this->process_event(event); + break; + default: + CHECK(false); + this->close(); + break; + } + return 0; +} + +void +CacheReadTest::do_io_read(size_t size) +{ + if (size == 0) { + size = this->_size; + } + this->vc->get_http_info(&this->read_http_info); + this->vio = this->vc->do_io_read(this, size, this->_read_buffer); +} + +int +CacheReadTest::start_test(int event, void *e) +{ + Debug("cache test", "start read test"); + HttpCacheKey key; + key = generate_key(this->info); + + SET_HANDLER(&CacheReadTest::read_event); + cacheProcessor.open_read(this, &key, (CacheHTTPHdr *)this->info.request_get(), &this->params); + return 0; +} + +constexpr size_t WRITE_LIMIT = 1024 * 3; diff --git a/iocore/cache/test/main.h b/iocore/cache/test/main.h new file mode 100644 index 00000000000..5bcc4f82a2f --- /dev/null +++ b/iocore/cache/test/main.h @@ -0,0 +1,234 @@ +/** @file + + A brief file description + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#pragma once + +#include + +#include "catch.hpp" + +#include "tscore/I_Layout.h" +#include "tscore/Diags.h" +#include "tscore/TSSystemState.h" + +#include "RecordsConfig.h" +#include "records/I_RecProcess.h" +#include "P_AIO.h" +#include "P_CacheDisk.h" +#include "P_Net.h" +#include "test/CacheTestHandler.h" +#include "P_Cache.h" + +#include + +// redefine BUILD PREFIX +#ifdef TS_BUILD_PREFIX +#undef TS_BUILD_PREFIX +#endif +#define TS_BUILD_PREFIX "./test" + +#ifdef TS_BUILD_EXEC_PREFIX +#undef TS_BUILD_EXEC_PREFIX +#endif +#define TS_BUILD_EXEC_PREFIX "./test" + +#ifdef TS_BUILD_SYSCONFDIR +#undef TS_BUILD_SYSCONFDIR +#endif +#define TS_BUILD_SYSCONFDIR "./test" + +#define SLEEP_TIME 20000 + +void init_cache(size_t size, const char *name = "cache.db"); +void build_hdrs(HTTPInfo &info, const char *url, const char *content_type = "text/html;charset=utf-8"); + +HttpCacheKey generate_key(HTTPInfo &info); + +extern const char *GLOBAL_DATA; +extern size_t const WRITE_LIMIT; + +class CacheInit : public Continuation +{ +public: + CacheInit() : Continuation(new_ProxyMutex()) { SET_HANDLER(&CacheInit::init_event); } + + int + start_event(int event, void *e) + { + Debug("cache_test", "cache init successfully"); + this->cache_init_success_callback(event, e); + return 0; + } + + int + init_event(int event, void *e) + { + switch (event) { + case EVENT_INTERVAL: + case EVENT_IMMEDIATE: + if (!CacheProcessor::IsCacheReady(CACHE_FRAG_TYPE_HTTP)) { + this_ethread()->schedule_in(this, SLEEP_TIME); + } else { + SET_HANDLER(&CacheInit::start_event); + this->handleEvent(event, e); + } + return 0; + default: + CHECK(false); + TEST_DONE(); + return 0; + } + + return 0; + } + + virtual int cache_init_success_callback(int event, void *e) = 0; + + virtual ~CacheInit() {} +}; + +class CacheTestBase : public Continuation +{ +public: + CacheTestBase(CacheTestHandler *test_handler) : Continuation(new_ProxyMutex()), test_handler(test_handler) + { + SET_HANDLER(&CacheTestBase::init_handler); + } + + int + init_handler(int event, void *e) + { + this->start_test(event, e); + return 0; + } + + // test entrance + virtual int start_test(int event, void *e) = 0; + + void + process_event(int event) + { + this->test_handler->handle_cache_event(event, this); + } + + virtual void + reenable() + { + if (this->vio) { + this->vio->reenable(); + } + } + + int + terminal_event(int event, void *e) + { + delete this; + return 0; + } + + void + close(int error = -1) + { + if (this->vc) { + this->vc->do_io_close(error); + this->vc = nullptr; + this->vio = nullptr; + } + + SET_HANDLER(&CacheTestBase::terminal_event); + if (!this->terminal) { + this->terminal = this_ethread()->schedule_imm(this); + } + } + + virtual void + do_io_read(size_t size = 0) + { + REQUIRE(!"should not be called"); + } + + virtual void + do_io_write(size_t size = 0) + { + REQUIRE(!"should not be called"); + } + + Event *terminal = nullptr; + CacheVC *vc = nullptr; + VIO *vio = nullptr; + CacheTestHandler *test_handler = nullptr; +}; + +class CacheWriteTest : public CacheTestBase +{ +public: + CacheWriteTest(size_t size, CacheTestHandler *cont, const char *url = "http://www.scw00.com/") : CacheTestBase(cont), _size(size) + { + this->_cursor = (char *)GLOBAL_DATA; + this->_write_buffer = new_MIOBuffer(BUFFER_SIZE_INDEX_4K); + + this->info.create(); + build_hdrs(this->info, url); + } + + int start_test(int event, void *e) override; + int write_event(int event, void *e); + void fill_data(); + void do_io_write(size_t size = 0) override; + + HTTPInfo info; + HTTPInfo old_info; + +private: + size_t _size = 0; + char *_cursor = nullptr; + MIOBuffer *_write_buffer = nullptr; +}; + +class CacheReadTest : public CacheTestBase +{ +public: + CacheReadTest(size_t size, CacheTestHandler *cont, const char *url = "http://www.scw00.com/") : CacheTestBase(cont), _size(size) + { + this->_cursor = (char *)GLOBAL_DATA; + this->_read_buffer = new_MIOBuffer(BUFFER_SIZE_INDEX_4K); + this->_reader = this->_read_buffer->alloc_reader(); + + this->info.create(); + build_hdrs(this->info, url); + } + + int start_test(int event, void *e) override; + int read_event(int event, void *e); + void do_io_read(size_t size = 0) override; + + HTTPInfo info; + HTTPInfo *read_http_info = nullptr; + +private: + size_t _size = 0; + char *_cursor = nullptr; + MIOBuffer *_read_buffer = nullptr; + IOBufferReader *_reader = nullptr; + OverridableHttpConfigParams params; +}; diff --git a/iocore/cache/test/storage.config b/iocore/cache/test/storage.config new file mode 100644 index 00000000000..d86334e356c Binary files /dev/null and b/iocore/cache/test/storage.config differ diff --git a/iocore/cache/test/stub.cc b/iocore/cache/test/stub.cc new file mode 100644 index 00000000000..d6710ecc823 --- /dev/null +++ b/iocore/cache/test/stub.cc @@ -0,0 +1,264 @@ +/** @file + + Stub file for linking libinknet.a from unit tests + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#include "HttpSessionManager.h" +#include "HttpBodyFactory.h" +#include "DiagsConfig.h" +#include "ts/InkAPIPrivateIOCore.h" + +void +initialize_thread_for_http_sessions(EThread *, int) +{ + ink_assert(false); +} + +#include "InkAPIInternal.h" +void +APIHooks::append(INKContInternal *cont) +{ +} + +int +APIHook::invoke(int, void *) +{ + ink_assert(false); + return 0; +} + +APIHook * +APIHook::next() const +{ + ink_assert(false); + return nullptr; +} + +APIHook * +APIHooks::get() const +{ + return nullptr; +} + +void +APIHooks::prepend(INKContInternal *cont) +{ +} + +void +APIHooks::clear() +{ +} + +void +ConfigUpdateCbTable::invoke(const char * /* name ATS_UNUSED */) +{ + ink_release_assert(false); +} + +HttpAPIHooks *http_global_hooks = nullptr; +SslAPIHooks *ssl_hooks = nullptr; +LifecycleAPIHooks *lifecycle_hooks = nullptr; +ConfigUpdateCbTable *global_config_cbs = nullptr; + +HttpBodyFactory *body_factory = nullptr; + +intmax_t +ts::svtoi(TextView src, TextView *out, int base) +{ + intmax_t zret = 0; + + if (out) { + out->clear(); + } + if (!(0 <= base && base <= 36)) { + return 0; + } + if (src.ltrim_if(&isspace) && src) { + const char *start = src.data(); + int8_t v; + bool neg = false; + if ('-' == *src) { + ++src; + neg = true; + } + // If base is 0, it wasn't specified - check for standard base prefixes + if (0 == base) { + base = 10; + if ('0' == *src) { + ++src; + base = 8; + if (src && ('x' == *src || 'X' == *src)) { + ++src; + base = 16; + } + } + } + + // For performance in common cases, use the templated conversion. + switch (base) { + case 8: + zret = svto_radix<8>(src); + break; + case 10: + zret = svto_radix<10>(src); + break; + case 16: + zret = svto_radix<16>(src); + break; + default: + while (src.size() && (0 <= (v = svtoi_convert[static_cast(*src)])) && v < base) { + auto n = zret * base + v; + if (n < zret) { + zret = std::numeric_limits::max(); + break; // overflow, stop parsing. + } + zret = n; + ++src; + } + break; + } + + if (out && (src.data() > (neg ? start + 1 : start))) { + out->assign(start, src.data()); + } + + if (neg) { + zret = -zret; + } + } + return zret; +} + +void +HostStatus::setHostStatus(const char *name, HostStatus_t status, const unsigned int down_time, const char *reason) +{ +} + +HostStatus_t +HostStatus::getHostStatus(const char *name) +{ + return (HostStatus_t)0; +} + +void +HostStatus::createHostStat(const char *name) +{ +} + +HostStatus::HostStatus() {} + +HostStatus::~HostStatus() {} + +int auto_clear_hostdb_flag = 0; +bool ts_is_draining = false; + +void +INKVConnInternal::do_io_close(int error) +{ +} + +void +INKVConnInternal::do_io_shutdown(ShutdownHowTo_t howto) +{ +} + +VIO * +INKVConnInternal::do_io_write(Continuation *c, int64_t nbytes, IOBufferReader *buf, bool owner) +{ + return nullptr; +} + +VIO * +INKVConnInternal::do_io_read(Continuation *c, int64_t nbytes, MIOBuffer *buf) +{ + return nullptr; +} + +void +INKVConnInternal::destroy() +{ +} + +void +INKVConnInternal::free() +{ +} + +void +INKVConnInternal::clear() +{ +} + +void +INKVConnInternal::reenable(VIO * /* vio ATS_UNUSED */) +{ +} + +bool +INKVConnInternal::get_data(int id, void *data) +{ + return false; +} + +bool +INKVConnInternal::set_data(int id, void *data) +{ + return false; +} + +void +INKVConnInternal::do_io_transform(VConnection *vc) +{ +} + +void +INKContInternal::handle_event_count(int event) +{ +} + +void +INKVConnInternal::retry(unsigned int delay) +{ +} + +INKContInternal::INKContInternal(TSEventFunc funcp, TSMutex mutexp) : DummyVConnection((ProxyMutex *)mutexp) {} + +INKContInternal::INKContInternal() : DummyVConnection(nullptr) {} + +void +INKContInternal::destroy() +{ +} + +void +INKContInternal::clear() +{ +} + +void +INKContInternal::free() +{ +} + +INKVConnInternal::INKVConnInternal() : INKContInternal() {} + +INKVConnInternal::INKVConnInternal(TSEventFunc funcp, TSMutex mutexp) : INKContInternal(funcp, mutexp) {} diff --git a/iocore/cache/test/test_Alternate_L_to_S.cc b/iocore/cache/test/test_Alternate_L_to_S.cc new file mode 100644 index 00000000000..6d4656a1acf --- /dev/null +++ b/iocore/cache/test/test_Alternate_L_to_S.cc @@ -0,0 +1,191 @@ +/** @file + + A brief file description + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#define LARGE_FILE 10 * 1024 * 1024 +#define SMALL_FILE 10 * 1024 + +#include "main.h" + +class CacheAltReadAgain : public CacheTestHandler +{ +public: + CacheAltReadAgain(size_t size, const char *url) : CacheTestHandler() + { + this->_rt = new CacheReadTest(size, this, url); + this->_rt->mutex = this->mutex; + + SET_HANDLER(&CacheAltReadAgain::start_test); + } + + int + start_test(int event, void *e) + { + REQUIRE(event == EVENT_IMMEDIATE); + this_ethread()->schedule_imm(this->_rt); + return 0; + } + + virtual void + handle_cache_event(int event, CacheTestBase *base) + { + switch (event) { + case CACHE_EVENT_OPEN_READ: + base->do_io_read(); + validate_content_type(base); + break; + case VC_EVENT_READ_READY: + base->reenable(); + break; + case VC_EVENT_READ_COMPLETE: + base->close(); + delete this; + break; + default: + REQUIRE(false); + break; + } + } + + void + validate_content_type(CacheTestBase *base) + { + auto rt = dynamic_cast(base); + REQUIRE(rt); + MIMEField *field = rt->read_http_info->m_alt->m_response_hdr.field_find(MIME_FIELD_CONTENT_TYPE, MIME_LEN_CONTENT_TYPE); + REQUIRE(field); + int len; + const char *value = field->value_get(&len); + REQUIRE(memcmp(value, "text/html;charset=utf-8", len) == 0); + } +}; + +class CacheAltTest_L_to_S : public CacheTestHandler +{ +public: + CacheAltTest_L_to_S(size_t size, const char *url) : CacheTestHandler() + { + auto rt = new CacheReadTest(size, this, url); + auto wt = new CacheWriteTest(size, this, url); + + rt->info.destroy(); + wt->info.destroy(); + + rt->info.create(); + wt->info.create(); + + build_hdrs(rt->info, url, "application/x-javascript"); + build_hdrs(wt->info, url, "application/x-javascript"); + + this->_rt = rt; + this->_wt = wt; + + this->_rt->mutex = this->mutex; + this->_wt->mutex = this->mutex; + + SET_HANDLER(&CacheAltTest_L_to_S::start_test); + } + + int + start_test(int event, void *e) + { + REQUIRE(event == EVENT_IMMEDIATE); + this_ethread()->schedule_imm(this->_wt); + return 0; + } + + virtual void + handle_cache_event(int event, CacheTestBase *base) + { + switch (event) { + case CACHE_EVENT_OPEN_WRITE: + base->do_io_write(); + break; + case VC_EVENT_WRITE_READY: + base->reenable(); + break; + case VC_EVENT_WRITE_COMPLETE: + this->_wt->close(); + this->_wt = nullptr; + this_ethread()->schedule_imm(this->_rt); + break; + case CACHE_EVENT_OPEN_READ: + base->do_io_read(); + validate_content_type(base); + break; + case VC_EVENT_READ_READY: + base->reenable(); + break; + case VC_EVENT_READ_COMPLETE: + base->close(); + delete this; + break; + default: + REQUIRE(false); + break; + } + } + +private: + void + validate_content_type(CacheTestBase *base) + { + auto rt = dynamic_cast(base); + REQUIRE(rt); + MIMEField *field = rt->read_http_info->m_alt->m_response_hdr.field_find(MIME_FIELD_CONTENT_TYPE, MIME_LEN_CONTENT_TYPE); + REQUIRE(field); + int len; + const char *value = field->value_get(&len); + REQUIRE(memcmp(value, "application/x-javascript", len) == 0); + } +}; + +class CacheAltInit : public CacheInit +{ +public: + CacheAltInit() {} + int + cache_init_success_callback(int event, void *e) override + { + CacheTestHandler *h = new CacheTestHandler(LARGE_FILE, "http://www.scw11.com"); + CacheAltTest_L_to_S *ls = new CacheAltTest_L_to_S(SMALL_FILE, "http://www.scw11.com"); + CacheAltReadAgain *read = new CacheAltReadAgain(LARGE_FILE, "http://www.scw11.com"); + TerminalTest *tt = new TerminalTest; + + h->add(ls); + h->add(read); // read again + h->add(tt); + this_ethread()->schedule_imm(h); + delete this; + return 0; + } +}; + +TEST_CASE("cache write -> read", "cache") +{ + init_cache(256 * 1024 * 1024); + // large write test + CacheAltInit *init = new CacheAltInit; + + this_ethread()->schedule_imm(init); + this_thread()->execute(); +} diff --git a/iocore/cache/test/test_Alternate_L_to_S_remove_L.cc b/iocore/cache/test/test_Alternate_L_to_S_remove_L.cc new file mode 100644 index 00000000000..f1167c7002b --- /dev/null +++ b/iocore/cache/test/test_Alternate_L_to_S_remove_L.cc @@ -0,0 +1,260 @@ +/** @file + + A brief file description + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#define LARGE_FILE 10 * 1024 * 1024 +#define SMALL_FILE 10 * 1024 + +#include "main.h" + +// delete dir +Dir dir = {}; + +class CacheAltReadAgain2 : public CacheTestHandler +{ +public: + CacheAltReadAgain2(size_t size, const char *url) : CacheTestHandler() + { + auto rt = new CacheReadTest(size, this, url); + + rt->mutex = this->mutex; + + rt->info.destroy(); + + rt->info.create(); + build_hdrs(rt->info, url, "application/x-javascript"); + + this->_rt = rt; + + SET_HANDLER(&CacheAltReadAgain2::start_test); + } + + int + start_test(int event, void *e) + { + REQUIRE(event == EVENT_IMMEDIATE); + this_ethread()->schedule_imm(this->_rt); + return 0; + } + + virtual void + handle_cache_event(int event, CacheTestBase *base) + { + switch (event) { + case CACHE_EVENT_OPEN_READ: + base->do_io_read(); + validate_content_type(base); + break; + case VC_EVENT_READ_READY: + base->reenable(); + break; + case VC_EVENT_READ_COMPLETE: + base->close(); + delete this; + break; + default: + REQUIRE(false); + break; + } + } + + void + validate_content_type(CacheTestBase *base) + { + auto rt = dynamic_cast(base); + REQUIRE(rt); + MIMEField *field = rt->read_http_info->m_alt->m_response_hdr.field_find(MIME_FIELD_CONTENT_TYPE, MIME_LEN_CONTENT_TYPE); + REQUIRE(field); + int len; + const char *value = field->value_get(&len); + REQUIRE(memcmp(value, "application/x-javascript", len) == 0); + } +}; + +class CacheAltReadAgain : public CacheTestHandler +{ +public: + CacheAltReadAgain(size_t size, const char *url) : CacheTestHandler() + { + this->_rt = new CacheReadTest(size, this, url); + this->_rt->mutex = this->mutex; + + SET_HANDLER(&CacheAltReadAgain::start_test); + } + + int + start_test(int event, void *e) + { + REQUIRE(event == EVENT_IMMEDIATE); + this_ethread()->schedule_imm(this->_rt); + return 0; + } + + virtual void + handle_cache_event(int event, CacheTestBase *base) + { + switch (event) { + case CACHE_EVENT_OPEN_READ_FAILED: + delete this; + break; + default: + REQUIRE(false); + break; + } + } + + void + validate_content_type(CacheTestBase *base) + { + auto rt = dynamic_cast(base); + REQUIRE(rt); + MIMEField *field = rt->read_http_info->m_alt->m_response_hdr.field_find(MIME_FIELD_CONTENT_TYPE, MIME_LEN_CONTENT_TYPE); + REQUIRE(field); + int len; + const char *value = field->value_get(&len); + REQUIRE(memcmp(value, "text/html;charset=utf-8", len) == 0); + } +}; + +class CacheAltTest_L_to_S_remove_L : public CacheTestHandler +{ +public: + CacheAltTest_L_to_S_remove_L(size_t size, const char *url) : CacheTestHandler() + { + auto rt = new CacheReadTest(size, this, url); + auto wt = new CacheWriteTest(size, this, url); + + rt->info.destroy(); + wt->info.destroy(); + + rt->info.create(); + wt->info.create(); + + build_hdrs(rt->info, url, "application/x-javascript"); + build_hdrs(wt->info, url, "application/x-javascript"); + + this->_rt = rt; + this->_wt = wt; + + this->_rt->mutex = this->mutex; + this->_wt->mutex = this->mutex; + + SET_HANDLER(&CacheAltTest_L_to_S_remove_L::start_test); + } + + int + start_test(int event, void *e) + { + REQUIRE(event == EVENT_IMMEDIATE); + this_ethread()->schedule_imm(this->_wt); + return 0; + } + + virtual void + handle_cache_event(int event, CacheTestBase *base) + { + switch (event) { + case CACHE_EVENT_OPEN_WRITE: + base->do_io_write(); + break; + case VC_EVENT_WRITE_READY: + base->reenable(); + break; + case VC_EVENT_WRITE_COMPLETE: + this->_wt->close(); + this->_wt = nullptr; + this_ethread()->schedule_imm(this->_rt); + break; + case CACHE_EVENT_OPEN_READ: + base->do_io_read(); + validate_content_type(base); + break; + case VC_EVENT_READ_READY: + base->reenable(); + break; + case VC_EVENT_READ_COMPLETE: + delete_earliest_dir(base->vc); + base->close(); + delete this; + break; + default: + REQUIRE(false); + break; + } + } + + void + validate_content_type(CacheTestBase *base) + { + auto rt = dynamic_cast(base); + REQUIRE(rt); + MIMEField *field = rt->read_http_info->m_alt->m_response_hdr.field_find(MIME_FIELD_CONTENT_TYPE, MIME_LEN_CONTENT_TYPE); + REQUIRE(field); + int len; + const char *value = field->value_get(&len); + REQUIRE(memcmp(value, "application/x-javascript", len) == 0); + } + + void + delete_earliest_dir(CacheVC *vc) + { + CacheKey key = {}; + Dir *last_collision = nullptr; + SCOPED_MUTEX_LOCK(lock, vc->vol->mutex, this->mutex->thread_holding); + vc->vector.data[0].alternate.object_key_get(&key); + REQUIRE(dir_probe(&key, vc->vol, &dir, &last_collision) != 0); + REQUIRE(dir_delete(&key, vc->vol, &dir)); + } +}; + +class CacheAltInit : public CacheInit +{ +public: + CacheAltInit() {} + int + cache_init_success_callback(int event, void *e) override + { + CacheTestHandler *h = new CacheTestHandler(LARGE_FILE, "http://www.scw11.com"); + CacheAltTest_L_to_S_remove_L *ls = new CacheAltTest_L_to_S_remove_L(SMALL_FILE, "http://www.scw11.com"); + CacheAltReadAgain *read = new CacheAltReadAgain(LARGE_FILE, "http://www.scw11.com"); + CacheAltReadAgain2 *read2 = new CacheAltReadAgain2(SMALL_FILE, "http://www.scw11.com"); + TerminalTest *tt = new TerminalTest; + + h->add(ls); + h->add(read); // read again + h->add(read2); + h->add(tt); + this_ethread()->schedule_imm(h); + delete this; + return 0; + } +}; + +TEST_CASE("cache write -> read", "cache") +{ + init_cache(256 * 1024 * 1024); + // large write test + CacheAltInit *init = new CacheAltInit; + + this_ethread()->schedule_imm(init); + this_thread()->execute(); +} diff --git a/iocore/cache/test/test_Alternate_L_to_S_remove_S.cc b/iocore/cache/test/test_Alternate_L_to_S_remove_S.cc new file mode 100644 index 00000000000..f2a4844d21a --- /dev/null +++ b/iocore/cache/test/test_Alternate_L_to_S_remove_S.cc @@ -0,0 +1,261 @@ +/** @file + + A brief file description + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#define LARGE_FILE 10 * 1024 * 1024 +#define SMALL_FILE 10 * 1024 + +#include "main.h" + +// delete dir +Dir dir = {}; + +class CacheAltReadAgain2 : public CacheTestHandler +{ +public: + CacheAltReadAgain2(size_t size, const char *url) : CacheTestHandler() + { + auto rt = new CacheReadTest(size, this, url); + + rt->mutex = this->mutex; + this->_rt = rt; + SET_HANDLER(&CacheAltReadAgain2::start_test); + } + + int + start_test(int event, void *e) + { + REQUIRE(event == EVENT_IMMEDIATE); + this_ethread()->schedule_imm(this->_rt); + return 0; + } + + virtual void + handle_cache_event(int event, CacheTestBase *base) + { + switch (event) { + case CACHE_EVENT_OPEN_READ: + base->do_io_read(); + validate_content_type(base); + break; + case VC_EVENT_READ_READY: + base->reenable(); + break; + case VC_EVENT_READ_COMPLETE: + base->close(); + delete this; + break; + default: + REQUIRE(false); + break; + } + } + + void + validate_content_type(CacheTestBase *base) + { + auto rt = dynamic_cast(base); + REQUIRE(rt); + MIMEField *field = rt->read_http_info->m_alt->m_response_hdr.field_find(MIME_FIELD_CONTENT_TYPE, MIME_LEN_CONTENT_TYPE); + REQUIRE(field); + int len; + const char *value = field->value_get(&len); + REQUIRE(memcmp(value, "text/html;charset=utf-8", len) == 0); + } +}; + +class CacheAltReadAgain : public CacheTestHandler +{ +public: + CacheAltReadAgain(size_t size, const char *url) : CacheTestHandler() + { + auto rt = new CacheReadTest(size, this, url); + + rt->mutex = this->mutex; + + rt->info.destroy(); + + rt->info.create(); + build_hdrs(rt->info, url, "application/x-javascript"); + + this->_rt = rt; + + SET_HANDLER(&CacheAltReadAgain::start_test); + } + + int + start_test(int event, void *e) + { + REQUIRE(event == EVENT_IMMEDIATE); + this_ethread()->schedule_imm(this->_rt); + return 0; + } + + virtual void + handle_cache_event(int event, CacheTestBase *base) + { + switch (event) { + case CACHE_EVENT_OPEN_READ_FAILED: + delete this; + break; + default: + REQUIRE(false); + break; + } + } + + void + validate_content_type(CacheTestBase *base) + { + auto rt = dynamic_cast(base); + REQUIRE(rt); + MIMEField *field = rt->read_http_info->m_alt->m_response_hdr.field_find(MIME_FIELD_CONTENT_TYPE, MIME_LEN_CONTENT_TYPE); + REQUIRE(field); + int len; + const char *value = field->value_get(&len); + REQUIRE(memcmp(value, "text/html;charset=utf-8", len) == 0); + } +}; + +class CacheAltTest_L_to_S_remove_S : public CacheTestHandler +{ +public: + CacheAltTest_L_to_S_remove_S(size_t size, const char *url) : CacheTestHandler() + { + auto rt = new CacheReadTest(size, this, url); + auto wt = new CacheWriteTest(size, this, url); + + rt->info.destroy(); + wt->info.destroy(); + + rt->info.create(); + wt->info.create(); + + build_hdrs(rt->info, url, "application/x-javascript"); + build_hdrs(wt->info, url, "application/x-javascript"); + + this->_rt = rt; + this->_wt = wt; + + this->_rt->mutex = this->mutex; + this->_wt->mutex = this->mutex; + + SET_HANDLER(&CacheAltTest_L_to_S_remove_S::start_test); + } + + int + start_test(int event, void *e) + { + REQUIRE(event == EVENT_IMMEDIATE); + this_ethread()->schedule_imm(this->_wt); + return 0; + } + + virtual void + handle_cache_event(int event, CacheTestBase *base) + { + switch (event) { + case CACHE_EVENT_OPEN_WRITE: + base->do_io_write(); + break; + case VC_EVENT_WRITE_READY: + base->reenable(); + break; + case VC_EVENT_WRITE_COMPLETE: + this->_wt->close(); + this->_wt = nullptr; + this_ethread()->schedule_imm(this->_rt); + break; + case CACHE_EVENT_OPEN_READ: + base->do_io_read(); + validate_content_type(base); + break; + case VC_EVENT_READ_READY: + base->reenable(); + break; + case VC_EVENT_READ_COMPLETE: + delete_earliest_dir(base->vc); + base->close(); + delete this; + break; + default: + REQUIRE(false); + break; + } + } + + void + validate_content_type(CacheTestBase *base) + { + auto rt = dynamic_cast(base); + REQUIRE(rt); + MIMEField *field = rt->read_http_info->m_alt->m_response_hdr.field_find(MIME_FIELD_CONTENT_TYPE, MIME_LEN_CONTENT_TYPE); + REQUIRE(field); + int len; + const char *value = field->value_get(&len); + REQUIRE(memcmp(value, "application/x-javascript", len) == 0); + } + + void + delete_earliest_dir(CacheVC *vc) + { + CacheKey key = {}; + Dir *last_collision = nullptr; + SCOPED_MUTEX_LOCK(lock, vc->vol->mutex, this->mutex->thread_holding); + vc->vector.data[1].alternate.object_key_get(&key); + REQUIRE(dir_probe(&key, vc->vol, &dir, &last_collision) != 0); + REQUIRE(dir_delete(&key, vc->vol, &dir)); + } +}; + +class CacheAltInit : public CacheInit +{ +public: + CacheAltInit() {} + int + cache_init_success_callback(int event, void *e) override + { + CacheTestHandler *h = new CacheTestHandler(LARGE_FILE, "http://www.scw11.com"); + CacheAltTest_L_to_S_remove_S *ls = new CacheAltTest_L_to_S_remove_S(SMALL_FILE, "http://www.scw11.com"); + CacheAltReadAgain *read = new CacheAltReadAgain(SMALL_FILE, "http://www.scw11.com"); + CacheAltReadAgain2 *read2 = new CacheAltReadAgain2(LARGE_FILE, "http://www.scw11.com"); + TerminalTest *tt = new TerminalTest; + + h->add(ls); + h->add(read); // read again + h->add(read2); + h->add(tt); + this_ethread()->schedule_imm(h); + delete this; + return 0; + } +}; + +TEST_CASE("cache write -> read", "cache") +{ + init_cache(256 * 1024 * 1024); + // large write test + CacheAltInit *init = new CacheAltInit; + + this_ethread()->schedule_imm(init); + this_thread()->execute(); +} diff --git a/iocore/cache/test/test_Alternate_S_to_L.cc b/iocore/cache/test/test_Alternate_S_to_L.cc new file mode 100644 index 00000000000..097b61b59b0 --- /dev/null +++ b/iocore/cache/test/test_Alternate_S_to_L.cc @@ -0,0 +1,191 @@ +/** @file + + A brief file description + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#define LARGE_FILE 10 * 1024 * 1024 +#define SMALL_FILE 10 * 1024 + +#include "main.h" + +class CacheAltReadAgain : public CacheTestHandler +{ +public: + CacheAltReadAgain(size_t size, const char *url) : CacheTestHandler() + { + this->_rt = new CacheReadTest(size, this, url); + this->_rt->mutex = this->mutex; + + SET_HANDLER(&CacheAltReadAgain::start_test); + } + + int + start_test(int event, void *e) + { + REQUIRE(event == EVENT_IMMEDIATE); + this_ethread()->schedule_imm(this->_rt); + return 0; + } + + virtual void + handle_cache_event(int event, CacheTestBase *base) + { + switch (event) { + case CACHE_EVENT_OPEN_READ: + base->do_io_read(); + validate_content_type(base); + break; + case VC_EVENT_READ_READY: + base->reenable(); + break; + case VC_EVENT_READ_COMPLETE: + base->close(); + delete this; + break; + default: + REQUIRE(false); + break; + } + } + + void + validate_content_type(CacheTestBase *base) + { + auto rt = dynamic_cast(base); + REQUIRE(rt); + MIMEField *field = rt->read_http_info->m_alt->m_response_hdr.field_find(MIME_FIELD_CONTENT_TYPE, MIME_LEN_CONTENT_TYPE); + REQUIRE(field); + int len; + const char *value = field->value_get(&len); + REQUIRE(memcmp(value, "text/html;charset=utf-8", len) == 0); + } +}; + +class CacheAltTest_L_to_S : public CacheTestHandler +{ +public: + CacheAltTest_L_to_S(size_t size, const char *url) : CacheTestHandler() + { + auto rt = new CacheReadTest(size, this, url); + auto wt = new CacheWriteTest(size, this, url); + + rt->info.destroy(); + wt->info.destroy(); + + rt->info.create(); + wt->info.create(); + + build_hdrs(rt->info, url, "application/x-javascript"); + build_hdrs(wt->info, url, "application/x-javascript"); + + this->_rt = rt; + this->_wt = wt; + + this->_rt->mutex = this->mutex; + this->_wt->mutex = this->mutex; + + SET_HANDLER(&CacheAltTest_L_to_S::start_test); + } + + int + start_test(int event, void *e) + { + REQUIRE(event == EVENT_IMMEDIATE); + this_ethread()->schedule_imm(this->_wt); + return 0; + } + + virtual void + handle_cache_event(int event, CacheTestBase *base) + { + switch (event) { + case CACHE_EVENT_OPEN_WRITE: + base->do_io_write(); + break; + case VC_EVENT_WRITE_READY: + base->reenable(); + break; + case VC_EVENT_WRITE_COMPLETE: + this->_wt->close(); + this->_wt = nullptr; + this_ethread()->schedule_imm(this->_rt); + break; + case CACHE_EVENT_OPEN_READ: + base->do_io_read(); + validate_content_type(base); + break; + case VC_EVENT_READ_READY: + base->reenable(); + break; + case VC_EVENT_READ_COMPLETE: + base->close(); + delete this; + break; + default: + REQUIRE(false); + break; + } + } + +private: + void + validate_content_type(CacheTestBase *base) + { + auto rt = dynamic_cast(base); + REQUIRE(rt); + MIMEField *field = rt->read_http_info->m_alt->m_response_hdr.field_find(MIME_FIELD_CONTENT_TYPE, MIME_LEN_CONTENT_TYPE); + REQUIRE(field); + int len; + const char *value = field->value_get(&len); + REQUIRE(memcmp(value, "application/x-javascript", len) == 0); + } +}; + +class CacheAltInit : public CacheInit +{ +public: + CacheAltInit() {} + int + cache_init_success_callback(int event, void *e) override + { + CacheTestHandler *h = new CacheTestHandler(SMALL_FILE, "http://www.scw11.com"); + CacheAltTest_L_to_S *ls = new CacheAltTest_L_to_S(LARGE_FILE, "http://www.scw11.com"); + CacheAltReadAgain *read = new CacheAltReadAgain(SMALL_FILE, "http://www.scw11.com"); + TerminalTest *tt = new TerminalTest; + + h->add(ls); + h->add(read); // read again + h->add(tt); + this_ethread()->schedule_imm(h); + delete this; + return 0; + } +}; + +TEST_CASE("cache write -> read", "cache") +{ + init_cache(256 * 1024 * 1024); + // large write test + CacheAltInit *init = new CacheAltInit; + + this_ethread()->schedule_imm(init); + this_thread()->execute(); +} diff --git a/iocore/cache/test/test_Alternate_S_to_L_remove_L.cc b/iocore/cache/test/test_Alternate_S_to_L_remove_L.cc new file mode 100644 index 00000000000..282224baabe --- /dev/null +++ b/iocore/cache/test/test_Alternate_S_to_L_remove_L.cc @@ -0,0 +1,264 @@ +/** @file + + A brief file description + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#define LARGE_FILE 10 * 1024 * 1024 +#define SMALL_FILE 10 * 1024 + +#include "main.h" + +// delete dir +Dir dir = {}; + +class CacheAltReadAgain2 : public CacheTestHandler +{ +public: + CacheAltReadAgain2(size_t size, const char *url) : CacheTestHandler() + { + auto rt = new CacheReadTest(size, this, url); + + rt->mutex = this->mutex; + this->_rt = rt; + SET_HANDLER(&CacheAltReadAgain2::start_test); + } + + int + start_test(int event, void *e) + { + REQUIRE(event == EVENT_IMMEDIATE); + this_ethread()->schedule_imm(this->_rt); + return 0; + } + + virtual void + handle_cache_event(int event, CacheTestBase *base) + { + switch (event) { + case CACHE_EVENT_OPEN_READ: + base->do_io_read(); + validate_content_type(base); + break; + case VC_EVENT_READ_READY: + base->reenable(); + break; + case VC_EVENT_READ_COMPLETE: + base->close(); + delete this; + break; + default: + REQUIRE(false); + break; + } + } + + void + validate_content_type(CacheTestBase *base) + { + auto rt = dynamic_cast(base); + REQUIRE(rt); + MIMEField *field = rt->read_http_info->m_alt->m_response_hdr.field_find(MIME_FIELD_CONTENT_TYPE, MIME_LEN_CONTENT_TYPE); + REQUIRE(field); + int len; + const char *value = field->value_get(&len); + REQUIRE(memcmp(value, "text/html;charset=utf-8", len) == 0); + } +}; + +class CacheAltReadAgain : public CacheTestHandler +{ +public: + CacheAltReadAgain(size_t size, const char *url) : CacheTestHandler() + { + auto rt = new CacheReadTest(size, this, url); + + rt->mutex = this->mutex; + + rt->info.destroy(); + + rt->info.create(); + build_hdrs(rt->info, url, "application/x-javascript"); + + this->_rt = rt; + + SET_HANDLER(&CacheAltReadAgain::start_test); + } + + int + start_test(int event, void *e) + { + REQUIRE(event == EVENT_IMMEDIATE); + // sleep for a while to wait for writer close + this_ethread()->schedule_imm(this->_rt); + return 0; + } + + virtual void + handle_cache_event(int event, CacheTestBase *base) + { + switch (event) { + case CACHE_EVENT_OPEN_READ_FAILED: + delete this; + break; + default: + REQUIRE(false); + break; + } + } + + void + validate_content_type(CacheTestBase *base) + { + auto rt = dynamic_cast(base); + REQUIRE(rt); + MIMEField *field = rt->read_http_info->m_alt->m_response_hdr.field_find(MIME_FIELD_CONTENT_TYPE, MIME_LEN_CONTENT_TYPE); + REQUIRE(field); + int len; + const char *value = field->value_get(&len); + REQUIRE(memcmp(value, "text/html;charset=utf-8", len) == 0); + } +}; + +class CacheAltTest_S_to_L_remove_L : public CacheTestHandler +{ +public: + CacheAltTest_S_to_L_remove_L(size_t size, const char *url) : CacheTestHandler() + { + auto rt = new CacheReadTest(size, this, url); + auto wt = new CacheWriteTest(size, this, url); + + rt->info.destroy(); + wt->info.destroy(); + + rt->info.create(); + wt->info.create(); + + build_hdrs(rt->info, url, "application/x-javascript"); + build_hdrs(wt->info, url, "application/x-javascript"); + + this->_rt = rt; + this->_wt = wt; + + this->_rt->mutex = this->mutex; + this->_wt->mutex = this->mutex; + + SET_HANDLER(&CacheAltTest_S_to_L_remove_L::start_test); + } + + int + start_test(int event, void *e) + { + REQUIRE(event == EVENT_IMMEDIATE); + this_ethread()->schedule_imm(this->_wt); + return 0; + } + + virtual void + handle_cache_event(int event, CacheTestBase *base) + { + switch (event) { + case CACHE_EVENT_OPEN_WRITE: + base->do_io_write(); + break; + case VC_EVENT_WRITE_READY: + base->reenable(); + break; + case VC_EVENT_WRITE_COMPLETE: + this->_wt->close(); + this->_wt = nullptr; + // to make sure writer successfully write the final doc done. we need to schedule + // to wait for some while. This time should be large than cache_config_mutex_retry_delay + this_ethread()->schedule_in(this->_rt, HRTIME_SECONDS(1)); + break; + case CACHE_EVENT_OPEN_READ: + base->do_io_read(); + validate_content_type(base); + break; + case VC_EVENT_READ_READY: + base->reenable(); + break; + case VC_EVENT_READ_COMPLETE: + delete_earliest_dir(base->vc); + base->close(); + delete this; + break; + default: + REQUIRE(false); + break; + } + } + + void + validate_content_type(CacheTestBase *base) + { + auto rt = dynamic_cast(base); + REQUIRE(rt); + MIMEField *field = rt->read_http_info->m_alt->m_response_hdr.field_find(MIME_FIELD_CONTENT_TYPE, MIME_LEN_CONTENT_TYPE); + REQUIRE(field); + int len; + const char *value = field->value_get(&len); + REQUIRE(memcmp(value, "application/x-javascript", len) == 0); + } + + void + delete_earliest_dir(CacheVC *vc) + { + CacheKey key = {}; + Dir *last_collision = nullptr; + SCOPED_MUTEX_LOCK(lock, vc->vol->mutex, this->mutex->thread_holding); + vc->vector.data[1].alternate.object_key_get(&key); + REQUIRE(dir_probe(&key, vc->vol, &dir, &last_collision) != 0); + REQUIRE(dir_delete(&key, vc->vol, &dir)); + } +}; + +class CacheAltInit : public CacheInit +{ +public: + CacheAltInit() {} + int + cache_init_success_callback(int event, void *e) override + { + CacheTestHandler *h = new CacheTestHandler(SMALL_FILE, "http://www.scw11.com"); + CacheAltTest_S_to_L_remove_L *ls = new CacheAltTest_S_to_L_remove_L(LARGE_FILE, "http://www.scw11.com"); + CacheAltReadAgain *read = new CacheAltReadAgain(LARGE_FILE, "http://www.scw11.com"); + CacheAltReadAgain2 *read2 = new CacheAltReadAgain2(SMALL_FILE, "http://www.scw11.com"); + TerminalTest *tt = new TerminalTest; + + h->add(ls); + h->add(read); // read again + h->add(read2); + h->add(tt); + this_ethread()->schedule_imm(h); + delete this; + return 0; + } +}; + +TEST_CASE("cache write -> read", "cache") +{ + init_cache(256 * 1024 * 1024); + // large write test + CacheAltInit *init = new CacheAltInit; + + this_ethread()->schedule_imm(init); + this_thread()->execute(); +} diff --git a/iocore/cache/test/test_Alternate_S_to_L_remove_S.cc b/iocore/cache/test/test_Alternate_S_to_L_remove_S.cc new file mode 100644 index 00000000000..96a1e7cce77 --- /dev/null +++ b/iocore/cache/test/test_Alternate_S_to_L_remove_S.cc @@ -0,0 +1,262 @@ +/** @file + + A brief file description + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#define LARGE_FILE 10 * 1024 * 1024 +#define SMALL_FILE 10 * 1024 + +#include "main.h" + +// delete dir +Dir dir = {}; + +class CacheAltReadAgain2 : public CacheTestHandler +{ +public: + CacheAltReadAgain2(size_t size, const char *url) : CacheTestHandler() + { + auto rt = new CacheReadTest(size, this, url); + + rt->mutex = this->mutex; + + rt->info.destroy(); + + rt->info.create(); + build_hdrs(rt->info, url, "application/x-javascript"); + + this->_rt = rt; + + SET_HANDLER(&CacheAltReadAgain2::start_test); + } + + int + start_test(int event, void *e) + { + REQUIRE(event == EVENT_IMMEDIATE); + this_ethread()->schedule_imm(this->_rt); + return 0; + } + + virtual void + handle_cache_event(int event, CacheTestBase *base) + { + switch (event) { + case CACHE_EVENT_OPEN_READ: + base->do_io_read(); + validate_content_type(base); + break; + case VC_EVENT_READ_READY: + base->reenable(); + break; + case VC_EVENT_READ_COMPLETE: + base->close(); + delete this; + break; + default: + REQUIRE(false); + break; + } + } + + void + validate_content_type(CacheTestBase *base) + { + auto rt = dynamic_cast(base); + REQUIRE(rt); + MIMEField *field = rt->read_http_info->m_alt->m_response_hdr.field_find(MIME_FIELD_CONTENT_TYPE, MIME_LEN_CONTENT_TYPE); + REQUIRE(field); + int len; + const char *value = field->value_get(&len); + REQUIRE(memcmp(value, "application/x-javascript", len) == 0); + } +}; + +class CacheAltReadAgain : public CacheTestHandler +{ +public: + CacheAltReadAgain(size_t size, const char *url) : CacheTestHandler() + { + this->_rt = new CacheReadTest(size, this, url); + this->_rt->mutex = this->mutex; + + SET_HANDLER(&CacheAltReadAgain::start_test); + } + + int + start_test(int event, void *e) + { + REQUIRE(event == EVENT_IMMEDIATE); + this_ethread()->schedule_imm(this->_rt); + return 0; + } + + virtual void + handle_cache_event(int event, CacheTestBase *base) + { + switch (event) { + case CACHE_EVENT_OPEN_READ_FAILED: + delete this; + break; + default: + REQUIRE(false); + break; + } + } + + void + validate_content_type(CacheTestBase *base) + { + auto rt = dynamic_cast(base); + REQUIRE(rt); + MIMEField *field = rt->read_http_info->m_alt->m_response_hdr.field_find(MIME_FIELD_CONTENT_TYPE, MIME_LEN_CONTENT_TYPE); + REQUIRE(field); + int len; + const char *value = field->value_get(&len); + REQUIRE(memcmp(value, "text/html;charset=utf-8", len) == 0); + } +}; + +class test_Alternate_S_to_L_remove_S : public CacheTestHandler +{ +public: + test_Alternate_S_to_L_remove_S(size_t size, const char *url) : CacheTestHandler() + { + auto rt = new CacheReadTest(size, this, url); + auto wt = new CacheWriteTest(size, this, url); + + rt->info.destroy(); + wt->info.destroy(); + + rt->info.create(); + wt->info.create(); + + build_hdrs(rt->info, url, "application/x-javascript"); + build_hdrs(wt->info, url, "application/x-javascript"); + + this->_rt = rt; + this->_wt = wt; + + this->_rt->mutex = this->mutex; + this->_wt->mutex = this->mutex; + + SET_HANDLER(&test_Alternate_S_to_L_remove_S::start_test); + } + + int + start_test(int event, void *e) + { + REQUIRE(event == EVENT_IMMEDIATE); + this_ethread()->schedule_imm(this->_wt); + return 0; + } + + virtual void + handle_cache_event(int event, CacheTestBase *base) + { + switch (event) { + case CACHE_EVENT_OPEN_WRITE: + base->do_io_write(); + break; + case VC_EVENT_WRITE_READY: + base->reenable(); + break; + case VC_EVENT_WRITE_COMPLETE: + this->_wt->close(); + this->_wt = nullptr; + // to make sure writer successfully write the final doc done. we need to schedule + // to wait for some while. This time should be large than cache_config_mutex_retry_delay + this_ethread()->schedule_in(this->_rt, HRTIME_SECONDS(1)); + break; + case CACHE_EVENT_OPEN_READ: + base->do_io_read(); + validate_content_type(base); + break; + case VC_EVENT_READ_READY: + base->reenable(); + break; + case VC_EVENT_READ_COMPLETE: + delete_earliest_dir(base->vc); + base->close(); + delete this; + break; + default: + REQUIRE(false); + break; + } + } + + void + validate_content_type(CacheTestBase *base) + { + auto rt = dynamic_cast(base); + REQUIRE(rt); + MIMEField *field = rt->read_http_info->m_alt->m_response_hdr.field_find(MIME_FIELD_CONTENT_TYPE, MIME_LEN_CONTENT_TYPE); + REQUIRE(field); + int len; + const char *value = field->value_get(&len); + REQUIRE(memcmp(value, "application/x-javascript", len) == 0); + } + + void + delete_earliest_dir(CacheVC *vc) + { + CacheKey key = {}; + Dir *last_collision = nullptr; + SCOPED_MUTEX_LOCK(lock, vc->vol->mutex, this->mutex->thread_holding); + vc->vector.data[0].alternate.object_key_get(&key); + REQUIRE(dir_probe(&key, vc->vol, &dir, &last_collision) != 0); + REQUIRE(dir_delete(&key, vc->vol, &dir)); + } +}; + +class CacheAltInit : public CacheInit +{ +public: + CacheAltInit() {} + int + cache_init_success_callback(int event, void *e) override + { + CacheTestHandler *h = new CacheTestHandler(SMALL_FILE, "http://www.scw11.com"); + test_Alternate_S_to_L_remove_S *ls = new test_Alternate_S_to_L_remove_S(LARGE_FILE, "http://www.scw11.com"); + CacheAltReadAgain *read = new CacheAltReadAgain(SMALL_FILE, "http://www.scw11.com"); + CacheAltReadAgain2 *read2 = new CacheAltReadAgain2(LARGE_FILE, "http://www.scw11.com"); + TerminalTest *tt = new TerminalTest; + + h->add(ls); + h->add(read); // read again + h->add(read2); + h->add(tt); + this_ethread()->schedule_imm(h); + delete this; + return 0; + } +}; + +TEST_CASE("cache write -> read", "cache") +{ + init_cache(256 * 1024 * 1024); + // large write test + CacheAltInit *init = new CacheAltInit; + + this_ethread()->schedule_imm(init); + this_thread()->execute(); +} diff --git a/iocore/cache/test/test_Cache.cc b/iocore/cache/test/test_Cache.cc new file mode 100644 index 00000000000..63284f51c54 --- /dev/null +++ b/iocore/cache/test/test_Cache.cc @@ -0,0 +1,55 @@ +/** @file + + A brief file description + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#include "main.h" + +#define LARGE_FILE 10 * 1024 * 1024 +#define SMALL_FILE 10 * 1024 + +class CacheCommInit : public CacheInit +{ +public: + CacheCommInit() {} + int + cache_init_success_callback(int event, void *e) override + { + CacheTestHandler *h = new CacheTestHandler(LARGE_FILE); + CacheTestHandler *h2 = new CacheTestHandler(SMALL_FILE, "http://www.scw11.com"); + TerminalTest *tt = new TerminalTest; + h->add(h2); + h->add(tt); + this_ethread()->schedule_imm(h); + delete this; + return 0; + } +}; + +TEST_CASE("cache write -> read", "cache") +{ + init_cache(256 * 1024 * 1024); + // large write test + CacheCommInit *init = new CacheCommInit; + + this_ethread()->schedule_imm(init); + this_thread()->execute(); +} diff --git a/iocore/cache/test/test_RWW.cc b/iocore/cache/test/test_RWW.cc new file mode 100644 index 00000000000..ee86681d58b --- /dev/null +++ b/iocore/cache/test/test_RWW.cc @@ -0,0 +1,423 @@ +/** @file + + A brief file description + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#define LARGE_FILE 10 * 1024 * 1024 +#define SMALL_FILE 10 * 1024 + +#define DEFAULT_URL "http://www.scw00.com/" + +#include "main.h" + +class CacheRWWTest; + +class CacheRWWTest : public CacheTestHandler +{ +public: + CacheRWWTest(size_t size, const char *url = DEFAULT_URL) : CacheTestHandler(), _size(size) + { + if (size != LARGE_FILE && size != SMALL_FILE) { + REQUIRE(!"size should be LARGE_FILE or SMALL_FILE"); + } + + this->_rt = new CacheReadTest(size, this, url); + this->_wt = new CacheWriteTest(size, this, url); + + this->_rt->mutex = this->mutex; + this->_wt->mutex = this->mutex; + + SET_HANDLER(&CacheRWWTest::start_test); + } + + void handle_cache_event(int event, CacheTestBase *e) override; + int start_test(int event, void *e); + + virtual void process_read_event(int event, CacheTestBase *base); + virtual void process_write_event(int event, CacheTestBase *base); + + void + close_write(int error = -1) + { + if (!this->_wt) { + return; + } + + this->_wt->close(error); + this->_wt = nullptr; + } + + void + close_read(int error = -1) + { + if (!this->_rt) { + return; + } + + this->_rt->close(error); + this->_rt = nullptr; + } + +protected: + // start at 1 framgents + int64_t _latest_fragments = 1; + size_t _size = 0; + Event *_read_event = nullptr; + bool _is_read_start = false; + CacheTestBase *_rt = nullptr; + CacheTestBase *_wt = nullptr; +}; + +int +CacheRWWTest::start_test(int event, void *e) +{ + REQUIRE(event == EVENT_IMMEDIATE); + this_ethread()->schedule_imm(this->_wt); + return 0; +} + +void +CacheRWWTest::process_write_event(int event, CacheTestBase *base) +{ + switch (event) { + case CACHE_EVENT_OPEN_WRITE: + base->do_io_write(); + break; + case VC_EVENT_WRITE_READY: + // schedule read test imm + if (this->_size != SMALL_FILE && !this->_wt->vc->fragment) { + base->reenable(); + return; + } + + if (!this->_is_read_start) { + if (!this->_read_event) { + this->_read_event = this_ethread()->schedule_imm(this->_rt); + } + return; + } + + // write at least one fragment before read it + if (this->_latest_fragments == this->_wt->vc->fragment) { + base->reenable(); + return; + } + + this->_latest_fragments = this->_wt->vc->fragment; + this->_rt->reenable(); + break; + case VC_EVENT_WRITE_COMPLETE: + this->close_write(); + break; + default: + REQUIRE(event == 0); + REQUIRE(false); + this->close_write(); + this->close_read(); + return; + } +} + +void +CacheRWWTest::process_read_event(int event, CacheTestBase *base) +{ + switch (event) { + case CACHE_EVENT_OPEN_READ: + base->do_io_read(); + break; + case VC_EVENT_READ_READY: + Debug("cache_rww_test", "cache read reenable"); + this->_read_event = nullptr; + this->_is_read_start = true; + break; + case VC_EVENT_READ_COMPLETE: + this->close_read(); + return; + + default: + REQUIRE(false); + this->close_write(); + this->close_read(); + return; + } + + if (this->_wt) { + this->_wt->reenable(); + } +} + +void +CacheRWWTest::handle_cache_event(int event, CacheTestBase *base) +{ + REQUIRE(base != nullptr); + + switch (event) { + case CACHE_EVENT_OPEN_WRITE_FAILED: + case CACHE_EVENT_OPEN_WRITE: + case VC_EVENT_WRITE_READY: + case VC_EVENT_WRITE_COMPLETE: + this->process_write_event(event, base); + break; + case CACHE_EVENT_OPEN_READ: + case CACHE_EVENT_OPEN_READ_FAILED: + case VC_EVENT_ERROR: + case VC_EVENT_EOS: + case VC_EVENT_READ_READY: + case VC_EVENT_READ_COMPLETE: + this->process_read_event(event, base); + break; + default: + REQUIRE(false); + this->close_write(); + this->close_read(); + break; + } + + if (this->_wt == nullptr && this->_rt == nullptr) { + delete this; + } + + return; +} + +class CacheRWWErrorTest : public CacheRWWTest +{ +public: + CacheRWWErrorTest(size_t size, const char *url = DEFAULT_URL) : CacheRWWTest(size, url) {} + void + process_write_event(int event, CacheTestBase *base) override + { + switch (event) { + case CACHE_EVENT_OPEN_WRITE: + base->do_io_write(); + break; + case VC_EVENT_WRITE_READY: + if (this->_size != SMALL_FILE && !this->_wt->vc->fragment) { + Debug("cache_rww_test", "cache write reenable"); + base->reenable(); + return; + } + + if (!this->_is_read_start) { + if (!this->_read_event) { + this->_read_event = this_ethread()->schedule_imm(this->_rt); + } + return; + } else { + this->close_write(100); + return; + } + + // write at least one fragment before read it + if (this->_latest_fragments == this->_wt->vc->fragment) { + base->reenable(); + return; + } + + this->_latest_fragments = this->_wt->vc->fragment; + this->_rt->reenable(); + break; + + case VC_EVENT_WRITE_COMPLETE: + REQUIRE(!"should not happen because the writter aborted"); + this->close_read(); + this->close_write(); + break; + default: + REQUIRE(false); + delete this; + return; + } + } + + void + process_read_event(int event, CacheTestBase *base) override + { + switch (event) { + case CACHE_EVENT_OPEN_READ: + this->_read_event = nullptr; + this->_is_read_start = true; + base->do_io_read(); + break; + case CACHE_EVENT_OPEN_READ_FAILED: + REQUIRE(this->_size == SMALL_FILE); + this->close_read(); + return; + case VC_EVENT_READ_READY: + if (this->_wt) { + this->_wt->reenable(); + } + return; + + case VC_EVENT_READ_COMPLETE: + REQUIRE(!"should not happen because the writter aborted"); + this->close_read(); + this->close_write(); + break; + case VC_EVENT_ERROR: + case VC_EVENT_EOS: + if (this->_size == LARGE_FILE) { + REQUIRE(base->vio->ndone >= 1 * 1024 * 1024 - sizeof(Doc)); + } else { + REQUIRE(base->vio->ndone == 0); + } + this->close_read(); + break; + default: + REQUIRE(event == 0); + this->close_read(); + this->close_write(); + break; + } + } + +private: + bool _is_read_start = false; +}; + +class CacheRWWEOSTest : public CacheRWWTest +{ +public: + CacheRWWEOSTest(size_t size, const char *url = DEFAULT_URL) : CacheRWWTest(size, url) {} + /* + * test this code in openReadMain + * if (writer_done()) { + last_collision = nullptr; + while (dir_probe(&earliest_key, vol, &dir, &last_collision)) { + // write complete. this could be reached the size we set in do_io_write or someone call do_io_close with -1 (-1 means write + success) flag if (dir_offset(&dir) == dir_offset(&earliest_dir)) { DDebug("cache_read_agg", "%p: key: %X ReadMain complete: %d", + this, first_key.slice32(1), (int)vio.ndone); doc_len = vio.ndone; goto Leos; + } + } + // writer abort. server crash. someone call do_io_close() with error flag + DDebug("cache_read_agg", "%p: key: %X ReadMain writer aborted: %d", this, first_key.slice32(1), (int)vio.ndone); + goto Lerror; + } + * + */ + + void + process_write_event(int event, CacheTestBase *base) override + { + switch (event) { + case CACHE_EVENT_OPEN_WRITE: + base->do_io_write(); + break; + case VC_EVENT_WRITE_READY: + if (this->_size != SMALL_FILE && !this->_wt->vc->fragment) { + Debug("cache_rww_test", "cache write reenable"); + base->reenable(); + return; + } + + if (!this->_is_read_start) { + if (!this->_read_event) { + this->_read_event = this_ethread()->schedule_imm(this->_rt); + } + return; + } + + // write at least one fragment before read it + if (this->_latest_fragments == this->_wt->vc->fragment) { + base->reenable(); + return; + } + + this->_latest_fragments = this->_wt->vc->fragment; + this->_rt->reenable(); + break; + + case VC_EVENT_WRITE_COMPLETE: + this->close_write(); + break; + default: + REQUIRE(false); + delete this; + return; + } + } + + void + process_read_event(int event, CacheTestBase *base) override + { + switch (event) { + case CACHE_EVENT_OPEN_READ: + this->_read_event = nullptr; + this->_is_read_start = true; + base->do_io_read(UINT32_MAX); + break; + case VC_EVENT_READ_READY: + if (this->_wt) { + this->_wt->reenable(); + } + return; + + case VC_EVENT_READ_COMPLETE: + REQUIRE(!"should not happen because the writter aborted"); + this->close_read(); + this->close_write(); + break; + case VC_EVENT_EOS: + this->close_write(); + this->close_read(); + break; + default: + REQUIRE(event == 0); + this->close_read(); + this->close_write(); + break; + } + } + +private: + bool _is_read_start = false; +}; + +class CacheRWWCacheInit : public CacheInit +{ +public: + CacheRWWCacheInit() {} + int + cache_init_success_callback(int event, void *e) override + { + CacheRWWTest *crww = new CacheRWWTest(LARGE_FILE); + CacheRWWErrorTest *crww_l = new CacheRWWErrorTest(LARGE_FILE, "http://www.scw22.com/"); + CacheRWWEOSTest *crww_eos = new CacheRWWEOSTest(LARGE_FILE, "ttp://www.scw44.com/"); + TerminalTest *tt = new TerminalTest(); + + crww->add(crww_l); + crww->add(crww_eos); + crww->add(tt); + this_ethread()->schedule_imm(crww); + delete this; + return 0; + } +}; + +TEST_CASE("cache rww", "cache") +{ + init_cache(256 * 1024 * 1024); + cache_config_target_fragment_size = 1 * 1024 * 1024; + CacheRWWCacheInit *init = new CacheRWWCacheInit(); + + this_ethread()->schedule_imm(init); + this_ethread()->execute(); +} diff --git a/iocore/cache/test/test_Update_L_to_S.cc b/iocore/cache/test/test_Update_L_to_S.cc new file mode 100644 index 00000000000..1172c816218 --- /dev/null +++ b/iocore/cache/test/test_Update_L_to_S.cc @@ -0,0 +1,155 @@ +/** @file + + A brief file description + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#define LARGE_FILE 10 * 1024 * 1024 +#define SMALL_FILE 10 * 1024 + +#include "main.h" + +class CacheUpdateReadAgain : public CacheTestHandler +{ +public: + CacheUpdateReadAgain(size_t size, const char *url) : CacheTestHandler() + { + this->_rt = new CacheReadTest(size, this, url); + this->_rt->mutex = this->mutex; + + SET_HANDLER(&CacheUpdateReadAgain::start_test); + } + + int + start_test(int event, void *e) + { + REQUIRE(event == EVENT_IMMEDIATE); + this_ethread()->schedule_imm(this->_rt); + return 0; + } + + virtual void + handle_cache_event(int event, CacheTestBase *base) + { + switch (event) { + case CACHE_EVENT_OPEN_READ: + base->do_io_read(); + break; + case VC_EVENT_READ_READY: + base->reenable(); + break; + case VC_EVENT_READ_COMPLETE: + base->close(); + delete this; + break; + default: + REQUIRE(false); + break; + } + } +}; + +class CacheUpdate_L_to_S : public CacheTestHandler +{ +public: + CacheUpdate_L_to_S(size_t read_size, size_t write_size, const char *url) + { + this->_rt = new CacheReadTest(read_size, this, url); + this->_wt = new CacheWriteTest(write_size, this, url); + + this->_rt->mutex = this->mutex; + this->_wt->mutex = this->mutex; + + SET_HANDLER(&CacheUpdate_L_to_S::start_test); + } + + int + start_test(int event, void *e) + { + REQUIRE(event == EVENT_IMMEDIATE); + this_ethread()->schedule_imm(this->_rt); + return 0; + } + + virtual void + handle_cache_event(int event, CacheTestBase *base) + { + CacheWriteTest *wt = static_cast(this->_wt); + switch (event) { + case CACHE_EVENT_OPEN_WRITE: + base->do_io_write(); + break; + case VC_EVENT_WRITE_READY: + base->reenable(); + break; + case VC_EVENT_WRITE_COMPLETE: + this->_wt->close(); + this->_wt = nullptr; + delete this; + break; + case CACHE_EVENT_OPEN_READ: + base->do_io_read(); + wt->old_info.copy(static_cast(&base->vc->alternate)); + break; + case VC_EVENT_READ_READY: + base->reenable(); + break; + case VC_EVENT_READ_COMPLETE: + this->_rt->close(); + this->_rt = nullptr; + this_ethread()->schedule_imm(this->_wt); + break; + default: + REQUIRE(false); + break; + } + } +}; + +class CacheUpdateInit : public CacheInit +{ +public: + CacheUpdateInit() {} + int + cache_init_success_callback(int event, void *e) override + { + CacheTestHandler *h = new CacheTestHandler(LARGE_FILE, "http://www.scw11.com"); + CacheUpdate_L_to_S *update = new CacheUpdate_L_to_S(LARGE_FILE, SMALL_FILE, "http://www.scw11.com"); + CacheUpdateReadAgain *read = new CacheUpdateReadAgain(SMALL_FILE, "http://www.scw11.com"); + TerminalTest *tt = new TerminalTest; + + h->add(update); + h->add(read); // read again + h->add(tt); + this_ethread()->schedule_imm(h); + delete this; + return 0; + } +}; + +TEST_CASE("cache write -> read", "cache") +{ + init_cache(256 * 1024 * 1024); + // large write test + CacheUpdateInit *init = new CacheUpdateInit; + + this_ethread()->schedule_imm(init); + this_thread()->execute(); +} diff --git a/iocore/cache/test/test_Update_S_to_L.cc b/iocore/cache/test/test_Update_S_to_L.cc new file mode 100644 index 00000000000..2944c33f4f3 --- /dev/null +++ b/iocore/cache/test/test_Update_S_to_L.cc @@ -0,0 +1,155 @@ +/** @file + + A brief file description + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#define LARGE_FILE 10 * 1024 * 1024 +#define SMALL_FILE 10 * 1024 + +#include "main.h" + +class CacheUpdateReadAgain : public CacheTestHandler +{ +public: + CacheUpdateReadAgain(size_t size, const char *url) : CacheTestHandler() + { + this->_rt = new CacheReadTest(size, this, url); + this->_rt->mutex = this->mutex; + + SET_HANDLER(&CacheUpdateReadAgain::start_test); + } + + int + start_test(int event, void *e) + { + REQUIRE(event == EVENT_IMMEDIATE); + this_ethread()->schedule_imm(this->_rt); + return 0; + } + + virtual void + handle_cache_event(int event, CacheTestBase *base) + { + switch (event) { + case CACHE_EVENT_OPEN_READ: + base->do_io_read(); + break; + case VC_EVENT_READ_READY: + base->reenable(); + break; + case VC_EVENT_READ_COMPLETE: + base->close(); + delete this; + break; + default: + REQUIRE(false); + break; + } + } +}; + +class CacheUpdate_S_to_L : public CacheTestHandler +{ +public: + CacheUpdate_S_to_L(size_t read_size, size_t write_size, const char *url) + { + this->_rt = new CacheReadTest(read_size, this, url); + this->_wt = new CacheWriteTest(write_size, this, url); + + this->_rt->mutex = this->mutex; + this->_wt->mutex = this->mutex; + + SET_HANDLER(&CacheUpdate_S_to_L::start_test); + } + + int + start_test(int event, void *e) + { + REQUIRE(event == EVENT_IMMEDIATE); + this_ethread()->schedule_imm(this->_rt); + return 0; + } + + virtual void + handle_cache_event(int event, CacheTestBase *base) + { + CacheWriteTest *wt = static_cast(this->_wt); + switch (event) { + case CACHE_EVENT_OPEN_WRITE: + base->do_io_write(); + break; + case VC_EVENT_WRITE_READY: + base->reenable(); + break; + case VC_EVENT_WRITE_COMPLETE: + this->_wt->close(); + this->_wt = nullptr; + delete this; + break; + case CACHE_EVENT_OPEN_READ: + base->do_io_read(); + wt->old_info.copy(static_cast(&base->vc->alternate)); + break; + case VC_EVENT_READ_READY: + base->reenable(); + break; + case VC_EVENT_READ_COMPLETE: + this->_rt->close(); + this->_rt = nullptr; + this_ethread()->schedule_imm(this->_wt); + break; + default: + REQUIRE(false); + break; + } + } +}; + +class CacheUpdateInit : public CacheInit +{ +public: + CacheUpdateInit() {} + int + cache_init_success_callback(int event, void *e) override + { + CacheTestHandler *h = new CacheTestHandler(SMALL_FILE, "http://www.scw11.com"); + CacheUpdate_S_to_L *update = new CacheUpdate_S_to_L(SMALL_FILE, LARGE_FILE, "http://www.scw11.com"); + CacheUpdateReadAgain *read = new CacheUpdateReadAgain(LARGE_FILE, "http://www.scw11.com"); + TerminalTest *tt = new TerminalTest; + + h->add(update); + h->add(read); // read again + h->add(tt); + this_ethread()->schedule_imm(h); + delete this; + return 0; + } +}; + +TEST_CASE("cache write -> read", "cache") +{ + init_cache(256 * 1024 * 1024); + // large write test + CacheUpdateInit *init = new CacheUpdateInit; + + this_ethread()->schedule_imm(init); + this_thread()->execute(); +} diff --git a/src/tscore/RawHashTable.cc b/iocore/cache/test/var/trafficserver/guard.txt similarity index 72% rename from src/tscore/RawHashTable.cc rename to iocore/cache/test/var/trafficserver/guard.txt index 7bfdaa146cf..8fe36290b11 100644 --- a/src/tscore/RawHashTable.cc +++ b/iocore/cache/test/var/trafficserver/guard.txt @@ -1,6 +1,6 @@ /** @file - C++ wrapper around libts hash tables + A brief file description @section license License @@ -19,13 +19,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. + */ - @section details Details - - These C++ RawHashTables are a C++ wrapper around libts hash tables. - They expose an interface very analogous to ink_hash_table, for better - or for worse. See HashTable for a more C++-oriented hash table. - -*/ - -#include "tscore/RawHashTable.h" +This is guard file for this empty dir diff --git a/iocore/dns/DNS.cc b/iocore/dns/DNS.cc index 8868f88e257..b2629b7cd30 100644 --- a/iocore/dns/DNS.cc +++ b/iocore/dns/DNS.cc @@ -138,7 +138,7 @@ HostEnt::free() dnsBufAllocator.free(this); } -void +size_t make_ipv4_ptr(in_addr_t addr, char *buffer) { char *p = buffer; @@ -176,10 +176,10 @@ make_ipv4_ptr(in_addr_t addr, char *buffer) } *p++ = u[0] % 10 + '0'; *p++ = '.'; - ink_strlcpy(p, "in-addr.arpa", MAXDNAME - (p - buffer + 1)); + return ink_strlcpy(p, "in-addr.arpa", MAXDNAME - (p - buffer + 1)); } -void +size_t make_ipv6_ptr(in6_addr const *addr, char *buffer) { const char hex_digit[] = "0123456789abcdef"; @@ -194,7 +194,7 @@ make_ipv6_ptr(in6_addr const *addr, char *buffer) *p++ = '.'; } - ink_strlcpy(p, "ip6.arpa", MAXDNAME - (p - buffer + 1)); + return ink_strlcpy(p, "ip6.arpa", MAXDNAME - (p - buffer + 1)); } // Public functions @@ -221,7 +221,7 @@ DNSProcessor::start(int, size_t stacksize) REC_ReadConfigStringAlloc(dns_resolv_conf, "proxy.config.dns.resolv_conf"); REC_EstablishStaticConfigInt32(dns_thread, "proxy.config.dns.dedicated_thread"); int dns_conn_mode_i = 0; - REC_EstablishStaticConfigInt32(dns_conn_mode_i, "proxy.config.dns.connection.mode"); + REC_EstablishStaticConfigInt32(dns_conn_mode_i, "proxy.config.dns.connection_mode"); dns_conn_mode = static_cast(dns_conn_mode_i); if (dns_thread > 0) { @@ -386,7 +386,7 @@ ink_dn_expand(const u_char *msg, const u_char *eom, const u_char *comp_dn, u_cha return ::dn_expand((unsigned char *)msg, (unsigned char *)eom, (unsigned char *)comp_dn, (char *)exp_dn, length); } -DNSProcessor::DNSProcessor() : thread(nullptr), handler(nullptr) +DNSProcessor::DNSProcessor() { ink_zero(l_res); ink_zero(local_ipv6); @@ -437,9 +437,9 @@ DNSEntry::init(const char *x, int len, int qtype_arg, Continuation *acont, DNSPr } else { // T_PTR IpAddr const *ip = reinterpret_cast(x); if (ip->isIp6()) { - make_ipv6_ptr(&ip->_addr._ip6, qname); + orig_qname_len = qname_len = make_ipv6_ptr(&ip->_addr._ip6, qname); } else if (ip->isIp4()) { - make_ipv4_ptr(ip->_addr._ip4, qname); + orig_qname_len = qname_len = make_ipv4_ptr(ip->_addr._ip4, qname); } else { ink_assert(!"T_PTR query to DNS must be IP address."); } @@ -577,7 +577,7 @@ DNSHandler::startEvent(int /* event ATS_UNUSED */, Event *e) /** Initial state of the DSNHandler. Can reinitialize the running DNS - hander to a new nameserver. + handler to a new nameserver. */ int DNSHandler::startEvent_sdns(int /* event ATS_UNUSED */, Event *e) @@ -673,7 +673,7 @@ DNSHandler::try_primary_named(bool reopen) void DNSHandler::switch_named(int ndx) { - for (DNSEntry *e = entries.head; e; e = (DNSEntry *)e->link.next) { + for (DNSEntry *e = entries.head; e; e = static_cast(e->link.next)) { e->written_flag = false; if (e->retries < dns_retries) { ++(e->retries); // give them another chance @@ -759,7 +759,7 @@ DNSHandler::rr_failure(int ndx) Warning("connection to all DNS servers lost, retrying"); // actual retries will be done in retry_named called from mainEvent // mark any outstanding requests as not sent for later retry - for (DNSEntry *e = entries.head; e; e = (DNSEntry *)e->link.next) { + for (DNSEntry *e = entries.head; e; e = static_cast(e->link.next)) { e->written_flag = false; if (e->retries < dns_retries) { ++(e->retries); // give them another chance @@ -769,7 +769,7 @@ DNSHandler::rr_failure(int ndx) } } else { // move outstanding requests that were sent to this nameserver to another - for (DNSEntry *e = entries.head; e; e = (DNSEntry *)e->link.next) { + for (DNSEntry *e = entries.head; e; e = static_cast(e->link.next)) { if (e->which_ns == ndx) { e->written_flag = false; if (e->retries < dns_retries) { @@ -795,7 +795,7 @@ DNSHandler::recv_dns(int /* event ATS_UNUSED */, Event * /* e ATS_UNUSED */) DNSConnection *dnsc = nullptr; ip_text_buffer ipbuff1, ipbuff2; Ptr buf; - while ((dnsc = (DNSConnection *)triggered.dequeue())) { + while ((dnsc = static_cast(triggered.dequeue()))) { while (true) { int res; IpEndpoint from_ip; @@ -808,10 +808,10 @@ DNSHandler::recv_dns(int /* event ATS_UNUSED */, Event * /* e ATS_UNUSED */) // see if TS gets a two-byte size uint16_t tmp = 0; res = socketManager.recv(dnsc->fd, &tmp, sizeof(tmp), MSG_PEEK); - if (res == -EAGAIN || res == 0 || res == 1) { + if (res == -EAGAIN || res == 1) { break; } - if (res < 0) { + if (res <= 0) { goto Lerror; } // reading total size @@ -830,10 +830,10 @@ DNSHandler::recv_dns(int /* event ATS_UNUSED */, Event * /* e ATS_UNUSED */) // continue reading data void *buf_start = (char *)dnsc->tcp_data.buf_ptr->buf + dnsc->tcp_data.done_reading; res = socketManager.recv(dnsc->fd, buf_start, dnsc->tcp_data.total_length - dnsc->tcp_data.done_reading, 0); - if (res == -EAGAIN || res == 0) { + if (res == -EAGAIN) { break; } - if (res < 0) { + if (res <= 0) { goto Lerror; } Debug("dns", "received packet size = %d over TCP", res); @@ -967,7 +967,7 @@ DNSHandler::mainEvent(int event, Event *e) inline static DNSEntry * get_dns(DNSHandler *h, uint16_t id) { - for (DNSEntry *e = h->entries.head; e; e = (DNSEntry *)e->link.next) { + for (DNSEntry *e = h->entries.head; e; e = static_cast(e->link.next)) { if (e->once_written_flag) { for (int j : e->id) { if (j == id) { @@ -986,7 +986,7 @@ get_dns(DNSHandler *h, uint16_t id) inline static DNSEntry * get_entry(DNSHandler *h, char *qname, int qtype) { - for (DNSEntry *e = h->entries.head; e; e = (DNSEntry *)e->link.next) { + for (DNSEntry *e = h->entries.head; e; e = static_cast(e->link.next)) { if (e->qtype == qtype) { if (is_addr_query(qtype)) { if (!strcmp(qname, e->qname)) { @@ -1027,7 +1027,7 @@ write_dns(DNSHandler *h, bool tcp_retry) if (h->in_flight < dns_max_dns_in_flight) { DNSEntry *e = h->entries.head; while (e) { - DNSEntry *n = (DNSEntry *)e->link.next; + DNSEntry *n = static_cast(e->link.next); if (!e->written_flag) { if (dns_ns_rr) { int ns_start = h->name_server; @@ -1192,7 +1192,7 @@ DNSEntry::mainEvent(int event, Event *e) } else { domains = nullptr; } - Debug("dns", "enqueing query %s", qname); + Debug("dns", "enqueuing query %s", qname); DNSEntry *dup = get_entry(dnsH, qname, qtype); if (dup) { Debug("dns", "collapsing NS request"); @@ -1307,7 +1307,16 @@ dns_result(DNSHandler *h, DNSEntry *e, HostEnt *ent, bool retry, bool tcp_retry) DNS_SUM_DYN_STAT(dns_success_time_stat, Thread::get_hrtime() - e->submit_time); } } + + // Remove head node from DNSHandler::entries queue h->entries.remove(e); + // Release Query ID from DNSHandler + for (int i : e->id) { + if (i < 0) { + break; + } + h->release_query_id(i); + } if (is_debug_tag_set("dns")) { if (is_addr_query(e->qtype)) { @@ -1334,52 +1343,50 @@ dns_result(DNSHandler *h, DNSEntry *e, HostEnt *ent, bool retry, bool tcp_retry) DNS_INCREMENT_DYN_STAT(dns_lookup_fail_stat); } - DNSEntry *dup = nullptr; - while ((dup = e->dups.dequeue())) { - if (dup->post(h, ent)) { - e->dups.enqueue(dup); - goto Lretry; - } - } - - if (e->timeout) { - e->timeout->cancel(e); - e->timeout = nullptr; - } + // Save HostEnt to the head node e->result_ent = ent; + e->retries = 0; + SET_CONTINUATION_HANDLER(e, &DNSEntry::postAllEvent); + e->handleEvent(EVENT_NONE, nullptr); +} - if (h->mutex->thread_holding == e->submit_thread) { - MUTEX_TRY_LOCK(lock, e->action.mutex, h->mutex->thread_holding); - if (!lock.is_locked()) { - Debug("dns", "failed lock for result %s", e->qname); - goto Lretry; - } - for (int i : e->id) { - if (i < 0) { - break; - } - h->release_query_id(i); - } - e->postEvent(0, nullptr); - } else { - for (int i : e->id) { - if (i < 0) { - break; +int +DNSEntry::postAllEvent(int /* event ATS_UNUSED */, Event * /* e ATS_UNUSED */) +{ + /* Traverse the DNSEntry queue and callback + * + * The first DNSEntry object is head node, + * - Pushed into DNSHandler::entries queue, + * - Initial a DNS request and send to named server, + * - Maintained a dups queue which holds the DNSEntry object for the same DNS request, + * - All the DNSEntry in the queue share the same HostEnt result + * + * The head node callback the HostEnt result to the Continuation of all nodes one by one, + * - If one of the callback fails, put the node back to the dups queue and try again later by reschedule the head node, + * - Always call back the head node until the dups queue is empty. + */ + DNSEntry *dup = nullptr; + while ((dup = dups.dequeue())) { + if (dup->post(dnsH, result_ent.get())) { + // If one of the callback fails, put the node back to the dups queue + dups.enqueue(dup); + // Try again by reschedule the head node + if (timeout) { + timeout->cancel(); } - h->release_query_id(i); + timeout = dnsH->mutex->thread_holding->schedule_in(this, MUTEX_RETRY_DELAY); + return EVENT_DONE; } - e->mutex = e->action.mutex; - SET_CONTINUATION_HANDLER(e, &DNSEntry::postEvent); - e->submit_thread->schedule_imm_signal(e); } - return; -Lretry: - e->result_ent = ent; - e->retries = 0; - if (e->timeout) { - e->timeout->cancel(); + + // Process the head node at last + if (post(dnsH, result_ent.get())) { + // If the callback fails, switch the handler to DNSEntry::postOneEvent and reschedule it. + mutex = action.mutex; + SET_HANDLER(&DNSEntry::postOneEvent); + submit_thread->schedule_imm(this); } - e->timeout = h->mutex->thread_holding->schedule_in(e, MUTEX_RETRY_DELAY); + return EVENT_DONE; } int @@ -1396,17 +1403,17 @@ DNSEntry::post(DNSHandler *h, HostEnt *ent) Debug("dns", "failed lock for result %s", qname); return 1; } - postEvent(0, nullptr); + postOneEvent(0, nullptr); } else { mutex = action.mutex; - SET_HANDLER(&DNSEntry::postEvent); + SET_HANDLER(&DNSEntry::postOneEvent); submit_thread->schedule_imm_signal(this); } return 0; } int -DNSEntry::postEvent(int /* event ATS_UNUSED */, Event * /* e ATS_UNUSED */) +DNSEntry::postOneEvent(int /* event ATS_UNUSED */, Event * /* e ATS_UNUSED */) { if (!action.cancelled) { Debug("dns", "called back continuation for %s", qname); @@ -1771,13 +1778,13 @@ Lerror:; RecRawStatBlock *dns_rsb; void -ink_dns_init(ModuleVersion v) +ink_dns_init(ts::ModuleVersion v) { static int init_called = 0; Debug("dns", "ink_dns_init: called with init_called = %d", init_called); - ink_release_assert(!checkModuleVersion(v, HOSTDB_MODULE_VERSION)); + ink_release_assert(v.check(HOSTDB_MODULE_PUBLIC_VERSION)); if (init_called) { return; } @@ -1818,7 +1825,7 @@ ink_dns_init(ModuleVersion v) RecRawStatSyncSum); } -#ifdef TS_HAS_TESTS +#if TS_HAS_TESTS struct DNSRegressionContinuation; using DNSRegContHandler = int (DNSRegressionContinuation::*)(int, void *); diff --git a/iocore/dns/DNSConnection.cc b/iocore/dns/DNSConnection.cc index 311ff9369f1..5a1408de282 100644 --- a/iocore/dns/DNSConnection.cc +++ b/iocore/dns/DNSConnection.cc @@ -48,8 +48,7 @@ DNSConnection::Options const DNSConnection::DEFAULT_OPTIONS; // Functions // -DNSConnection::DNSConnection() - : fd(NO_FD), num(0), generator((uint32_t)((uintptr_t)time(nullptr) ^ (uintptr_t)this)), handler(nullptr) +DNSConnection::DNSConnection() : fd(NO_FD), generator((uint32_t)((uintptr_t)time(nullptr) ^ (uintptr_t)this)) { memset(&ip, 0, sizeof(ip)); } @@ -81,7 +80,7 @@ DNSConnection::trigger() // Since the periodic check is removed, we need to call // this when it's triggered by EVENTIO_DNS_CONNECTION. - // The handler should be pionting to DNSHandler::mainEvent. + // The handler should be pointing to DNSHandler::mainEvent. // We can schedule an immediate event or call the handler // directly, and since both arguments are not being used // passing in 0 and nullptr will do the job. diff --git a/iocore/dns/I_DNSProcessor.h b/iocore/dns/I_DNSProcessor.h index 02aa4cace12..a0ec65115eb 100644 --- a/iocore/dns/I_DNSProcessor.h +++ b/iocore/dns/I_DNSProcessor.h @@ -68,13 +68,13 @@ struct DNSProcessor : public Processor { /// Query handler to use. /// Default: single threaded handler. - DNSHandler *handler; + DNSHandler *handler = nullptr; /// Query timeout value. /// Default: @c DEFAULT_DNS_TIMEOUT (or as set in records.config) - int timeout; ///< Timeout value for request. + int timeout = 0; ///< Timeout value for request. /// Host resolution style. /// Default: IPv4, IPv6 ( @c HOST_RES_IPV4 ) - HostResStyle host_res_style; + HostResStyle host_res_style = HOST_RES_IPV4; /// Default constructor. Options(); @@ -121,8 +121,8 @@ struct DNSProcessor : public Processor { // private: // - EThread *thread; - DNSHandler *handler; + EThread *thread = nullptr; + DNSHandler *handler = nullptr; ts_imp_res_state l_res; IpEndpoint local_ipv6; IpEndpoint local_ipv4; @@ -172,7 +172,7 @@ DNSProcessor::gethostbyaddr(Continuation *cont, IpAddr const *addr, Options cons return getby(reinterpret_cast(addr), 0, T_PTR, cont, opt); } -inline DNSProcessor::Options::Options() : handler(nullptr), timeout(0), host_res_style(HOST_RES_IPV4) {} +inline DNSProcessor::Options::Options() {} inline DNSProcessor::Options & DNSProcessor::Options::setHandler(DNSHandler *h) @@ -202,4 +202,4 @@ DNSProcessor::Options::reset() return *this; } -void ink_dns_init(ModuleVersion version); +void ink_dns_init(ts::ModuleVersion version); diff --git a/iocore/dns/I_SplitDNS.h b/iocore/dns/I_SplitDNS.h index c038ea24bd7..e277bce0664 100644 --- a/iocore/dns/I_SplitDNS.h +++ b/iocore/dns/I_SplitDNS.h @@ -31,8 +31,6 @@ #pragma once #include "I_SplitDNSProcessor.h" +#include "tscore/I_Version.h" -#define SPLITDNS_MODULE_MAJOR_VERSION 1 -#define SPLITDNS_MODULE_MINOR_VERSION 0 -#define SPLITDNS_MODULE_VERSION \ - makeModuleVersion(SPLITDNS_MODULE_MAJOR_VERSION, SPLITDNS_MODULE_MINOR_VERSION, PUBLIC_MODULE_HEADER) +static constexpr ts::ModuleVersion SPLITDNS_MODULE_PUBLIC_VERSION(1, 0, ts::ModuleVersion::PUBLIC); diff --git a/iocore/dns/P_DNSConnection.h b/iocore/dns/P_DNSConnection.h index d4a3e5f27ec..b31c39b4c86 100644 --- a/iocore/dns/P_DNSConnection.h +++ b/iocore/dns/P_DNSConnection.h @@ -46,22 +46,22 @@ struct DNSConnection { /// Connection is done non-blocking. /// Default: @c true. - bool _non_blocking_connect; + bool _non_blocking_connect = true; /// Set socket to have non-blocking I/O. /// Default: @c true. - bool _non_blocking_io; + bool _non_blocking_io = true; /// Use TCP if @c true, use UDP if @c false. /// Default: @c false. - bool _use_tcp; + bool _use_tcp = false; /// Bind to a random port. /// Default: @c true. - bool _bind_random_port; + bool _bind_random_port = true; /// Bind to this local address when using IPv6. /// Default: unset, bind to IN6ADDR_ANY. - sockaddr const *_local_ipv6; + sockaddr const *_local_ipv6 = nullptr; /// Bind to this local address when using IPv4. /// Default: unset, bind to INADDRY_ANY. - sockaddr const *_local_ipv4; + sockaddr const *_local_ipv4 = nullptr; Options(); @@ -75,12 +75,12 @@ struct DNSConnection { int fd; IpEndpoint ip; - int num; + int num = 0; Options opt; LINK(DNSConnection, link); EventIO eio; InkRand generator; - DNSHandler *handler; + DNSHandler *handler = nullptr; /// TCPData structure is to track the reading progress of a TCP connection struct TCPData { @@ -110,15 +110,7 @@ struct DNSConnection { static Options const DEFAULT_OPTIONS; }; -inline DNSConnection::Options::Options() - : _non_blocking_connect(true), - _non_blocking_io(true), - _use_tcp(false), - _bind_random_port(true), - _local_ipv6(nullptr), - _local_ipv4(nullptr) -{ -} +inline DNSConnection::Options::Options() {} inline DNSConnection::Options & DNSConnection::Options::setNonBlockingIo(bool p) diff --git a/iocore/dns/P_DNSProcessor.h b/iocore/dns/P_DNSProcessor.h index 86e6bdb6a46..9ff1b4baf60 100644 --- a/iocore/dns/P_DNSProcessor.h +++ b/iocore/dns/P_DNSProcessor.h @@ -66,7 +66,7 @@ extern unsigned int dns_sequence_number; #define DNS_SEQUENCE_NUMBER_RESTART_OFFSET 4000 #define DNS_PRIMARY_RETRY_PERIOD HRTIME_SECONDS(5) #define DNS_PRIMARY_REOPEN_PERIOD HRTIME_SECONDS(60) -#define BAD_DNS_RESULT ((HostEnt *)(uintptr_t)-1) +#define BAD_DNS_RESULT (reinterpret_cast((uintptr_t)-1)) #define DEFAULT_NUM_TRY_SERVER 8 // these are from nameser.h @@ -159,8 +159,9 @@ struct DNSEntry : public Continuation { int mainEvent(int event, Event *e); int delayEvent(int event, Event *e); + int postAllEvent(int event, Event *e); int post(DNSHandler *h, HostEnt *ent); - int postEvent(int event, Event *e); + int postOneEvent(int event, Event *e); void init(const char *x, int len, int qtype_arg, Continuation *acont, DNSProcessor::Options const &opt); DNSEntry() @@ -186,25 +187,25 @@ struct DNSHandler : public Continuation { IpEndpoint local_ipv6; ///< Local V6 address if set. IpEndpoint local_ipv4; ///< Local V4 address if set. int ifd[MAX_NAMED]; - int n_con; + int n_con = 0; DNSConnection tcpcon[MAX_NAMED]; DNSConnection udpcon[MAX_NAMED]; Queue entries; Queue triggered; - int in_flight; - int name_server; - int in_write_dns; - HostEnt *hostent_cache; + int in_flight = 0; + int name_server = 0; + int in_write_dns = 0; + HostEnt *hostent_cache = nullptr; int ns_down[MAX_NAMED]; int failover_number[MAX_NAMED]; int failover_soon_number[MAX_NAMED]; ink_hrtime crossed_failover_number[MAX_NAMED]; - ink_hrtime last_primary_retry; - ink_hrtime last_primary_reopen; + ink_hrtime last_primary_retry = 0; + ink_hrtime last_primary_reopen = 0; - ink_res_state m_res; - int txn_lookup_timeout; + ink_res_state m_res = nullptr; + int txn_lookup_timeout = 0; InkRand generator; // bitmap of query ids in use @@ -297,9 +298,9 @@ struct DNSServer { char x_def_domain[MAXDNAME]; char x_domain_srch_list[MAXDNAME]; - DNSHandler *x_dnsH; + DNSHandler *x_dnsH = nullptr; - DNSServer() : x_dnsH(nullptr) + DNSServer() { memset(x_server_ip, 0, sizeof(x_server_ip)); @@ -312,15 +313,7 @@ struct DNSServer { TS_INLINE DNSHandler::DNSHandler() : Continuation(nullptr), - n_con(0), - in_flight(0), - name_server(0), - in_write_dns(0), - hostent_cache(nullptr), - last_primary_retry(0), - last_primary_reopen(0), - m_res(nullptr), - txn_lookup_timeout(0), + generator((uint32_t)((uintptr_t)time(nullptr) ^ (uintptr_t)this)) { ats_ip_invalidate(&ip); diff --git a/iocore/dns/P_SplitDNS.h b/iocore/dns/P_SplitDNS.h index 6cf92f16c2f..19df3076e0b 100644 --- a/iocore/dns/P_SplitDNS.h +++ b/iocore/dns/P_SplitDNS.h @@ -37,6 +37,4 @@ #include "ControlMatcher.h" #include "P_SplitDNSProcessor.h" -#undef SPLITDNS_MODULE_VERSION -#define SPLITDNS_MODULE_VERSION \ - makeModuleVersion(SPLITDNS_MODULE_MAJOR_VERSION, SPLITDNS_MODULE_MINOR_VERSION, PRIVATE_MODULE_HEADER) +static constexpr ts::ModuleVersion SPLITDNS_MODULE_INTERNAL_VERSION{SPLITDNS_MODULE_PUBLIC_VERSION, ts::ModuleVersion::PRIVATE}; diff --git a/iocore/dns/P_SplitDNSProcessor.h b/iocore/dns/P_SplitDNSProcessor.h index 33de46eabf1..41d0fb92172 100644 --- a/iocore/dns/P_SplitDNSProcessor.h +++ b/iocore/dns/P_SplitDNSProcessor.h @@ -42,7 +42,7 @@ /* --------------------------- forward declarations ... --------------------------- */ -void ink_split_dns_init(ModuleVersion version); +void ink_split_dns_init(ts::ModuleVersion version); struct RequestData; @@ -70,7 +70,7 @@ struct SplitDNSResult { /* ------------ public ------------ */ - DNSResultType r; + DNSResultType r = DNS_SRVR_UNDEFINED; DNSServer *get_dns_record(); int get_dns_srvr_count(); @@ -78,10 +78,10 @@ struct SplitDNSResult { /* ------------ private ------------ */ - int m_line_number; + int m_line_number = 0; - SplitDNSRecord *m_rec; - bool m_wrap_around; + SplitDNSRecord *m_rec = nullptr; + bool m_wrap_around = false; }; /* -------------------------------------------------------------- @@ -94,17 +94,17 @@ struct SplitDNS : public ConfigInfo { void *getDNSRecord(const char *hostname); void findServer(RequestData *rdata, SplitDNSResult *result); - DNS_table *m_DNSSrvrTable; + DNS_table *m_DNSSrvrTable = nullptr; - int32_t m_SplitDNSlEnable; + int32_t m_SplitDNSlEnable = 0; /* ---------------------------- required by the alleged fast path ---------------------------- */ - bool m_bEnableFastPath; - void *m_pxLeafArray; - int m_numEle; + bool m_bEnableFastPath = false; + void *m_pxLeafArray = nullptr; + int m_numEle = 0; }; /* -------------------------------------------------------------- @@ -130,21 +130,21 @@ class DNSRequestData : public RequestData public: DNSRequestData(); - char *get_string(); + char *get_string() override; - const char *get_host(); + const char *get_host() override; - sockaddr const *get_ip(); // unused required virtual method. - sockaddr const *get_client_ip(); // unused required virtual method. + sockaddr const *get_ip() override; // unused required virtual method. + sockaddr const *get_client_ip() override; // unused required virtual method. - const char *m_pHost; + const char *m_pHost = nullptr; }; /* -------------------------------------------------------------- DNSRequestData::get_string() -------------------------------------------------------------- */ TS_INLINE -DNSRequestData::DNSRequestData() : m_pHost(nullptr) {} +DNSRequestData::DNSRequestData() {} /* -------------------------------------------------------------- DNSRequestData::get_string() @@ -203,15 +203,15 @@ class SplitDNSRecord : public ControlBase void Print(); DNSServer m_servers; - int m_dnsSrvr_cnt; - int m_domain_srch_list; + int m_dnsSrvr_cnt = 0; + int m_domain_srch_list = 0; }; /* -------------------------------------------------------------- SplitDNSRecord::SplitDNSRecord() -------------------------------------------------------------- */ TS_INLINE -SplitDNSRecord::SplitDNSRecord() : m_dnsSrvr_cnt(0), m_domain_srch_list(0) {} +SplitDNSRecord::SplitDNSRecord() {} /* -------------------------------------------------------------- SplitDNSRecord::~SplitDNSRecord() diff --git a/iocore/dns/SplitDNS.cc b/iocore/dns/SplitDNS.cc index a7cbcd2be2c..92240602a50 100644 --- a/iocore/dns/SplitDNS.cc +++ b/iocore/dns/SplitDNS.cc @@ -74,14 +74,12 @@ Ptr SplitDNSConfig::dnsHandler_mutex; /* -------------------------------------------------------------- SplitDNSResult::SplitDNSResult() -------------------------------------------------------------- */ -inline SplitDNSResult::SplitDNSResult() : r(DNS_SRVR_UNDEFINED), m_line_number(0), m_rec(nullptr), m_wrap_around(false) {} +inline SplitDNSResult::SplitDNSResult() {} /* -------------------------------------------------------------- SplitDNS::SplitDNS() -------------------------------------------------------------- */ -SplitDNS::SplitDNS() : m_DNSSrvrTable(nullptr), m_SplitDNSlEnable(0), m_bEnableFastPath(false), m_pxLeafArray(nullptr), m_numEle(0) -{ -} +SplitDNS::SplitDNS() {} SplitDNS::~SplitDNS() { @@ -96,7 +94,7 @@ SplitDNS::~SplitDNS() SplitDNS * SplitDNSConfig::acquire() { - return (SplitDNS *)configProcessor.get(SplitDNSConfig::m_id); + return static_cast(configProcessor.get(SplitDNSConfig::m_id)); } /* -------------------------------------------------------------- @@ -132,12 +130,15 @@ SplitDNSConfig::reconfigure() return; } + Note("splitdns.config loading ..."); + SplitDNS *params = new SplitDNS; params->m_SplitDNSlEnable = gsplit_dns_enabled; params->m_DNSSrvrTable = new DNS_table("proxy.config.dns.splitdns.filename", modulePrefix, &sdns_dest_tags); if (nullptr == params->m_DNSSrvrTable || (0 == params->m_DNSSrvrTable->getEntryCount())) { + Error("splitdns.config failed to load"); Warning("No NAMEDs provided! Disabling SplitDNS"); gsplit_dns_enabled = 0; delete params; @@ -157,6 +158,8 @@ SplitDNSConfig::reconfigure() if (is_debug_tag_set("splitdns_config")) { SplitDNSConfig::print(); } + + Note("splitdns.config finished loading"); } /* -------------------------------------------------------------- @@ -252,7 +255,7 @@ SplitDNS::findServer(RequestData *rdata, SplitDNSResult *result) int res = memcmp(pH, pMatch, pxHL[i].match.size()); if ((0 != res && '!' == cNot) || (0 == res && '!' != cNot)) { - data_ptr = (SplitDNSRecord *)pxHL[i].opaque_data; + data_ptr = static_cast(pxHL[i].opaque_data); data_ptr->UpdateMatch(result, rdata); break; } @@ -295,9 +298,7 @@ SplitDNSRecord::ProcessDNSHosts(char *val) { Tokenizer pTok(",; \t\r"); int numTok; - const char *current; - int port = 0; - char *tmp; + int port = 0; int totsz = 0, sz = 0; numTok = pTok.Initialize(val, SHARE_TOKS); @@ -314,8 +315,8 @@ SplitDNSRecord::ProcessDNSHosts(char *val) set of servers specified ------------------------------------------------ */ for (int i = 0; i < numTok; i++) { - current = pTok[i]; - tmp = (char *)strchr(current, ':'); + const char *current = pTok[i]; + char *tmp = (char *)strchr(current, ':'); // coverity[secure_coding] if (tmp != nullptr && sscanf(tmp + 1, "%d", &port) != 1) { return "Malformed DNS port"; @@ -341,7 +342,7 @@ SplitDNSRecord::ProcessDNSHosts(char *val) if (tmp - current > (MAXDNAME - 1)) { return "DNS server name (ip) is too long"; } else if (tmp - current == 0) { - return "server string is emtpy"; + return "server string is empty"; } *tmp = 0; } @@ -399,9 +400,8 @@ SplitDNSRecord::ProcessDomainSrchList(char *val) { Tokenizer pTok(",; \t\r"); int numTok; - int cnt = 0, sz = 0; + int sz = 0; char *pSp = nullptr; - const char *current; numTok = pTok.Initialize(val, SHARE_TOKS); @@ -412,8 +412,8 @@ SplitDNSRecord::ProcessDomainSrchList(char *val) pSp = &m_servers.x_domain_srch_list[0]; for (int i = 0; i < numTok; i++) { - current = pTok[i]; - cnt = sz += strlen(current); + const char *current = pTok[i]; + int cnt = sz += strlen(current); if (MAXDNAME - 1 < sz) { break; @@ -437,14 +437,11 @@ Result SplitDNSRecord::Init(matcher_line *line_info) { const char *errPtr = nullptr; - const char *tmp; - char *label; - char *val; this->line_num = line_info->line_num; for (int i = 0; i < MATCHER_MAX_TOKENS; i++) { - label = line_info->line[0][i]; - val = line_info->line[1][i]; + char *label = line_info->line[0][i]; + char *val = line_info->line[1][i]; if (label == nullptr) { continue; @@ -506,7 +503,7 @@ SplitDNSRecord::Init(matcher_line *line_info) Process any modifiers to the directive, if they exist ----------------------------------------------------- */ if (line_info->num_el > 0) { - tmp = ProcessModifiers(line_info); + const char *tmp = ProcessModifiers(line_info); if (tmp != nullptr) { return Result::failure("%s %s at line %d in splitdns.config", modulePrefix, tmp, line_num); } @@ -544,11 +541,11 @@ SplitDNSRecord::Print() } void -ink_split_dns_init(ModuleVersion v) +ink_split_dns_init(ts::ModuleVersion v) { static int init_called = 0; - ink_release_assert(!checkModuleVersion(v, SPLITDNS_MODULE_VERSION)); + ink_release_assert(v.check(SPLITDNS_MODULE_INTERNAL_VERSION)); if (init_called) { return; } diff --git a/iocore/dns/test_I_DNS.cc b/iocore/dns/test_I_DNS.cc index 9ba3d863eae..98331bcf7fc 100644 --- a/iocore/dns/test_I_DNS.cc +++ b/iocore/dns/test_I_DNS.cc @@ -26,10 +26,11 @@ #include "diags.i" +int main() { init_diags("net_test", nullptr); - ink_event_system_init(EVENT_SYSTEM_MODULE_VERSION); + ink_event_system_init(EVENT_SYSTEM_MODULE_PUBLIC_VERSION); signal(SIGPIPE, SIG_IGN); eventProcessor.start(2); diff --git a/iocore/dns/test_P_DNS.cc b/iocore/dns/test_P_DNS.cc index 1c0bee3622e..378fa948ca4 100644 --- a/iocore/dns/test_P_DNS.cc +++ b/iocore/dns/test_P_DNS.cc @@ -55,7 +55,7 @@ struct NetTesterSM : public Continuation { fflush(stdout); break; case VC_EVENT_READ_COMPLETE: - /* FALLSTHROUGH */ + /* FALLTHROUGH */ case VC_EVENT_EOS: r = reader->read_avail(); str = new char[r + 10]; @@ -75,4 +75,7 @@ struct NetTesterSM : public Continuation { } }; -main() {} +int +main() +{ +} diff --git a/iocore/eventsystem/EventSystem.cc b/iocore/eventsystem/EventSystem.cc index 1cb361397c9..f5d5dc03ae3 100644 --- a/iocore/eventsystem/EventSystem.cc +++ b/iocore/eventsystem/EventSystem.cc @@ -31,13 +31,13 @@ #include "P_EventSystem.h" void -ink_event_system_init(ModuleVersion v) +ink_event_system_init(ts::ModuleVersion v) { - ink_release_assert(!checkModuleVersion(v, EVENT_SYSTEM_MODULE_VERSION)); + ink_release_assert(v.check(EVENT_SYSTEM_MODULE_INTERNAL_VERSION)); int config_max_iobuffer_size = DEFAULT_MAX_BUFFER_SIZE; int iobuffer_advice = 0; - // For backwards compatability make sure to allow thread_freelist_size + // For backwards compatibility make sure to allow thread_freelist_size // This needs to change in 6.0 REC_EstablishStaticConfigInt32(thread_freelist_high_watermark, "proxy.config.allocator.thread_freelist_size"); diff --git a/iocore/eventsystem/IOBuffer.cc b/iocore/eventsystem/IOBuffer.cc index ff8658dc0a6..844a5af16e5 100644 --- a/iocore/eventsystem/IOBuffer.cc +++ b/iocore/eventsystem/IOBuffer.cc @@ -46,8 +46,6 @@ int64_t max_iobuffer_size = DEFAULT_BUFFER_SIZES - 1; void init_buffer_allocators(int iobuffer_advice) { - char *name; - for (int i = 0; i < DEFAULT_BUFFER_SIZES; i++) { int64_t s = DEFAULT_BUFFER_BASE_SIZE * (((int64_t)1) << i); int64_t a = DEFAULT_BUFFER_ALIGNMENT; @@ -56,7 +54,7 @@ init_buffer_allocators(int iobuffer_advice) a = s; } - name = new char[64]; + auto name = new char[64]; snprintf(name, 64, "ioBufAllocator[%d]", i); ioBufAllocator[i].re_init(name, s, n, a, iobuffer_advice); } @@ -189,7 +187,7 @@ MIOBuffer::puts(char *s, int64_t len) } if (!*pb || *pb == '\n') { int64_t n = (int64_t)(pb - s); - memcpy(end(), s, n + 1); // Upto and including '\n' + memcpy(end(), s, n + 1); // Up to and including '\n' end()[n + 1] = 0; fill(n + 1); return n + 1; diff --git a/iocore/eventsystem/I_Action.h b/iocore/eventsystem/I_Action.h index 444e2115d08..edd9159d5ce 100644 --- a/iocore/eventsystem/I_Action.h +++ b/iocore/eventsystem/I_Action.h @@ -78,7 +78,7 @@ Allocation policy: Actions are allocated by the Processor performing the actions. - It is the processor's responsbility to handle deallocation once + It is the processor's responsibility to handle deallocation once the action is complete or cancelled. A state machine MUST NOT access an action once the operation that returned the Action has completed or it has cancelled the Action. @@ -88,11 +88,11 @@ class Action { public: /** - Contination that initiated this action. + Continuation that initiated this action. The reference to the initiating continuation is only used to verify that the action is being cancelled by the correct - continuation. This field should not be accesed or modified + continuation. This field should not be accessed or modified directly by the state machine. */ @@ -103,7 +103,7 @@ class Action Keeps a reference to the Continuation's lock to preserve the access to the cancelled field valid even when the state machine - has been deallocated. This field should not be accesed or + has been deallocated. This field should not be accessed or modified directly by the state machine. */ @@ -114,7 +114,7 @@ class Action cancelled. This flag is set after a call to cancel or cancel_action and - it should not be accesed or modified directly by the state + it should not be accessed or modified directly by the state machine. */ diff --git a/iocore/eventsystem/I_Continuation.h b/iocore/eventsystem/I_Continuation.h index bd73f715492..b34f78d2525 100644 --- a/iocore/eventsystem/I_Continuation.h +++ b/iocore/eventsystem/I_Continuation.h @@ -23,7 +23,7 @@ @section details Details Continuations have a handleEvent() method to invoke them. Users - can determine the behavior of a Continuation by suppling a + can determine the behavior of a Continuation by supplying a "ContinuationHandler" (member function name) which is invoked when events arrive. This function can be changed with the "setHandler" method. @@ -48,6 +48,7 @@ class EThread; class Event; extern EThread *this_ethread(); +extern EThread *this_event_thread(); ////////////////////////////////////////////////////////////////////////////// // @@ -144,6 +145,30 @@ class Continuation : private force_VFPT_to_top */ ContFlags control_flags; + EThread *thread_affinity = nullptr; + + bool + setThreadAffinity(EThread *ethread) + { + if (ethread != nullptr) { + thread_affinity = ethread; + return true; + } + return false; + } + + EThread * + getThreadAffinity() + { + return thread_affinity; + } + + void + clearThreadAffinity() + { + thread_affinity = nullptr; + } + /** Receives the event code and data for an Event. @@ -165,20 +190,6 @@ class Continuation : private force_VFPT_to_top return (this->*handler)(event, data); } - /** - Receives the event code and data for an Event. - - It will attempt to get the lock for the continuation, and reschedule - the event if the lock cannot be obtained. If the lock can be obtained - dispatchEvent acts like handleEvent. - - @param event Event code to be passed at callback (Processor specific). - @param data General purpose data related to the event code (Processor specific). - @return State machine and processor specific return code. - - */ - int dispatchEvent(int event = CONTINUATION_EVENT_NONE, void *data = nullptr); - protected: /** Constructor of the Continuation object. It should not be used @@ -187,8 +198,8 @@ class Continuation : private force_VFPT_to_top @param amutex Lock to be set for this Continuation. */ - Continuation(ProxyMutex *amutex = nullptr); - Continuation(Ptr &amutex); + explicit Continuation(ProxyMutex *amutex = nullptr); + explicit Continuation(Ptr &amutex); }; /** diff --git a/iocore/eventsystem/I_EThread.h b/iocore/eventsystem/I_EThread.h index 7f625451245..c70fb04050f 100644 --- a/iocore/eventsystem/I_EThread.h +++ b/iocore/eventsystem/I_EThread.h @@ -51,14 +51,12 @@ enum ThreadType { DEDICATED, }; -extern bool shutdown_event_system; - /** Event System specific type of thread. The EThread class is the type of thread created and managed by the Event System. It is one of the available interfaces for - schedulling events in the event system (another two are the Event + scheduling events in the event system (another two are the Event and EventProcessor classes). In order to handle events, each EThread object has two event @@ -75,7 +73,7 @@ extern bool shutdown_event_system; Scheduling Interface: - There are eight schedulling functions provided by EThread and + There are eight scheduling functions provided by EThread and they are a wrapper around their counterparts in EventProcessor. @see EventProcessor @@ -123,7 +121,7 @@ class EThread : public Thread continuation's handler. See the the EventProcessor class. @param cookie User-defined value or pointer to be passed back in the Event's object cookie field. - @return Reference to an Event object representing the schedulling + @return Reference to an Event object representing the scheduling of this callback. */ @@ -145,7 +143,7 @@ class EThread : public Thread continuation's handler. See the EventProcessor class. @param cookie User-defined value or pointer to be passed back in the Event's object cookie field. - @return A reference to an Event object representing the schedulling + @return A reference to an Event object representing the scheduling of this callback. */ @@ -165,7 +163,7 @@ class EThread : public Thread continuation's handler. See the EventProcessor class. @param cookie User-defined value or pointer to be passed back in the Event's object cookie field. - @return A reference to an Event object representing the schedulling + @return A reference to an Event object representing the scheduling of this callback. */ @@ -179,14 +177,14 @@ class EThread : public Thread to occur every time 'aperiod' elapses. It is scheduled on this EThread. - @param c Continuation to call back everytime 'aperiod' elapses. + @param c Continuation to call back every time 'aperiod' elapses. @param aperiod Duration of the time period between callbacks. @param callback_event Event code to be passed back to the continuation's handler. See the Remarks section in the EventProcessor class. @param cookie User-defined value or pointer to be passed back in the Event's object cookie field. - @return A reference to an Event object representing the schedulling + @return A reference to an Event object representing the scheduling of this callback. */ @@ -204,7 +202,7 @@ class EThread : public Thread continuation's handler. See the EventProcessor class. @param cookie User-defined value or pointer to be passed back in the Event's object cookie field. - @return A reference to an Event object representing the schedulling + @return A reference to an Event object representing the scheduling of this callback. */ @@ -225,7 +223,7 @@ class EThread : public Thread continuation's handler. See the EventProcessor class. @param cookie User-defined value or pointer to be passed back in the Event's object cookie field. - @return A reference to an Event object representing the schedulling + @return A reference to an Event object representing the scheduling of this callback. */ @@ -246,7 +244,7 @@ class EThread : public Thread EventProcessor class. @param cookie User-defined value or pointer to be passed back in the Event's object cookie field. - @return A reference to an Event object representing the schedulling + @return A reference to an Event object representing the scheduling of this callback. */ @@ -259,14 +257,14 @@ class EThread : public Thread Schedules the callback to the continuation 'c' to occur every time 'aperiod' elapses. It is scheduled on this EThread. - @param c Continuation to call back everytime 'aperiod' elapses. + @param c Continuation to call back every time 'aperiod' elapses. @param aperiod Duration of the time period between callbacks. @param callback_event Event code to be passed back to the continuation's handler. See the Remarks section in the EventProcessor class. @param cookie User-defined value or pointer to be passed back in the Event's object cookie field. - @return A reference to an Event object representing the schedulling + @return A reference to an Event object representing the scheduling of this callback. */ @@ -360,6 +358,7 @@ class EThread : public Thread */ class DefaultTailHandler : public LoopTailHandler { + // cppcheck-suppress noExplicitConstructor; allow implicit conversion DefaultTailHandler(ProtectedQueue &q) : _q(q) {} int @@ -371,7 +370,12 @@ class EThread : public Thread void signalActivity() override { - _q.signal(); + /* Try to acquire the `EThread::lock` of the Event Thread: + * - Acquired, indicating that the Event Thread is sleep, + * must send a wakeup signal to the Event Thread. + * - Failed, indicating that the Event Thread is busy, do nothing. + */ + (void)_q.try_signal(); } ProtectedQueue &_q; @@ -383,27 +387,27 @@ class EThread : public Thread struct EventMetrics { /// Time the loop was active, not including wait time but including event dispatch time. struct LoopTimes { - ink_hrtime _start; ///< The time of the first loop for this sample. Used to mark valid entries. - ink_hrtime _min; ///< Shortest loop time. - ink_hrtime _max; ///< Longest loop time. - LoopTimes() : _start(0), _min(INT64_MAX), _max(0) {} + ink_hrtime _start = 0; ///< The time of the first loop for this sample. Used to mark valid entries. + ink_hrtime _min = INT64_MAX; ///< Shortest loop time. + ink_hrtime _max = 0; ///< Longest loop time. + LoopTimes() {} } _loop_time; struct Events { - int _min; - int _max; - int _total; - Events() : _min(INT_MAX), _max(0), _total(0) {} + int _min = INT_MAX; + int _max = 0; + int _total = 0; + Events() {} } _events; - int _count; ///< # of times the loop executed. - int _wait; ///< # of timed wait for events + int _count = 0; ///< # of times the loop executed. + int _wait = 0; ///< # of timed wait for events /// Add @a that to @a this data. /// This embodies the custom logic per member concerning whether each is a sum, min, or max. EventMetrics &operator+=(EventMetrics const &that); - EventMetrics() : _count(0), _wait(0) {} + EventMetrics() {} }; /** The number of metric blocks kept. diff --git a/iocore/eventsystem/I_Event.h b/iocore/eventsystem/I_Event.h index 83c1bd0a3b6..e598adda288 100644 --- a/iocore/eventsystem/I_Event.h +++ b/iocore/eventsystem/I_Event.h @@ -78,7 +78,6 @@ #define RAFT_EVENT_EVENTS_START 3200 #define SIMPLE_EVENT_EVENTS_START 3300 #define UPDATE_EVENT_EVENTS_START 3500 -#define LOG_COLLATION_EVENT_EVENTS_START 3800 #define AIO_EVENT_EVENTS_START 3900 #define BLOCK_CACHE_EVENT_EVENTS_START 4000 #define UTILS_EVENT_EVENTS_START 5000 @@ -167,7 +166,7 @@ class Event : public Action Instructs the event object to reschedule itself at the time specified in atimeout_at on the EventProcessor. - @param atimeout_at Time at which to callcallback. See the Remarks section. + @param atimeout_at Time at which to call the callback. See the Remarks section. @param callback_event Event code to return at the completion of this event. See the Remarks section. */ @@ -178,7 +177,7 @@ class Event : public Action Instructs the event object to reschedule itself at the time specified in atimeout_at on the EventProcessor. - @param atimeout_in Time at which to callcallback. See the Remarks section. + @param atimeout_in Time at which to call the callback. See the Remarks section. @param callback_event Event code to return at the completion of this event. See the Remarks section. */ @@ -189,7 +188,7 @@ class Event : public Action the event object to reschedule itself to callback every 'aperiod' from now. - @param aperiod Time period at which to callcallback. See the Remarks section. + @param aperiod Time period at which to call the callback. See the Remarks section. @param callback_event Event code to return at the completion of this event. See the Remarks section. */ diff --git a/iocore/eventsystem/I_EventProcessor.h b/iocore/eventsystem/I_EventProcessor.h index 28dbd012ba9..a15d406f96c 100644 --- a/iocore/eventsystem/I_EventProcessor.h +++ b/iocore/eventsystem/I_EventProcessor.h @@ -73,8 +73,8 @@ class EThread; thread is independent of the thread groups and it exists as long as your continuation handle executes and there are events to process. In the latter, you call @c registerEventType to get an event type and then @c spawn_event_theads which creates the threads in the group of that - type. Such threads require events to be scheduled on a specicif thread in the group or for the - grouop in general using the event type. Note that between these two calls @c + type. Such threads require events to be scheduled on a specific thread in the group or for the + group in general using the event type. Note that between these two calls @c EThread::schedule_spawn can be used to set up per thread initialization. Callback event codes: @@ -204,11 +204,11 @@ class EventProcessor : public Processor /** Schedules the continuation on a specific thread group to receive an event periodically. Requests the EventProcessor to schedule the - callback to the continuation 'c' everytime 'aperiod' elapses. The + callback to the continuation 'c' every time 'aperiod' elapses. The callback is handled by a thread in the specified thread group (event_type). - @param c Continuation to call back everytime 'aperiod' elapses. + @param c Continuation to call back every time 'aperiod' elapses. @param aperiod duration of the time period between callbacks. @param event_type thread group id (or event type) specifying the group of threads on which to schedule the callback. @@ -303,11 +303,12 @@ class EventProcessor : public Processor /// The thread group ID is the index into an array of these and so is not stored explicitly. struct ThreadGroupDescriptor { std::string _name; ///< Name for the thread group. - int _count = 0; ///< # of threads of this type. - std::atomic _started = 0; ///< # of started threads of this type. - int _next_round_robin = 0; ///< Index of thread to use for events assigned to this group. + int _count = 0; ///< # of threads of this type. + std::atomic _started = 0; ///< # of started threads of this type. + uint64_t _next_round_robin = 0; ///< Index of thread to use for events assigned to this group. Que(Event, link) _spawnQueue; ///< Events to dispatch when thread is spawned. EThread *_thread[MAX_THREADS_IN_EACH_TYPE] = {}; ///< The actual threads in this group. + std::function _afterStartCallback = nullptr; }; /// Storage for per group data. @@ -390,7 +391,7 @@ class EventProcessor : public Processor EventProcessor *_evp; public: - ThreadInit(EventProcessor *evp) : _evp(evp) { SET_HANDLER(&self::init); } + explicit ThreadInit(EventProcessor *evp) : _evp(evp) { SET_HANDLER(&self::init); } int init(int /* event ATS_UNUSED */, Event *ev) diff --git a/iocore/eventsystem/I_EventSystem.h b/iocore/eventsystem/I_EventSystem.h index 8d51a93bb9b..a062b5c31a8 100644 --- a/iocore/eventsystem/I_EventSystem.h +++ b/iocore/eventsystem/I_EventSystem.h @@ -26,6 +26,7 @@ #define _I_EventSystem_h #include "tscore/ink_platform.h" +#include "ts/apidefs.h" #include "I_IOBuffer.h" #include "I_Action.h" @@ -44,9 +45,6 @@ #include "records/I_RecProcess.h" #include "I_SocketManager.h" -#define EVENT_SYSTEM_MODULE_MAJOR_VERSION 1 -#define EVENT_SYSTEM_MODULE_MINOR_VERSION 0 -#define EVENT_SYSTEM_MODULE_VERSION \ - makeModuleVersion(EVENT_SYSTEM_MODULE_MAJOR_VERSION, EVENT_SYSTEM_MODULE_MINOR_VERSION, PUBLIC_MODULE_HEADER) +static constexpr ts::ModuleVersion EVENT_SYSTEM_MODULE_PUBLIC_VERSION(1, 0, ts::ModuleVersion::PUBLIC); -void ink_event_system_init(ModuleVersion version); +void ink_event_system_init(ts::ModuleVersion version); diff --git a/iocore/eventsystem/I_IOBuffer.h b/iocore/eventsystem/I_IOBuffer.h index 0b0c56a1c67..1742016c220 100644 --- a/iocore/eventsystem/I_IOBuffer.h +++ b/iocore/eventsystem/I_IOBuffer.h @@ -29,7 +29,7 @@ might wish to ensure that an entire line will come in before consuming the data. In such a case, the water_mark should be set to the largest possible size of the string. (appropriate error handling should take - care of exessively long strings). + care of excessively long strings). In all other cases, especially when all data will be consumed, the water_mark should be set to 0 (the default). @@ -239,7 +239,7 @@ class IOBufferData : public RefCountObj alloc or dealloc methods. */ - AllocType _mem_type; + AllocType _mem_type = NO_ALLOC; /** Points to the allocated memory. This member stores the address of @@ -247,10 +247,10 @@ class IOBufferData : public RefCountObj instead use the alloc or dealloc methods. */ - char *_data; + char *_data = nullptr; #ifdef TRACK_BUFFER_USER - const char *_location; + const char *_location = nullptr; #endif /** @@ -258,16 +258,7 @@ class IOBufferData : public RefCountObj this method. Use one of the functions with the 'new_' prefix instead. */ - IOBufferData() - : _size_index(BUFFER_SIZE_NOT_ALLOCATED), - _mem_type(NO_ALLOC), - _data(nullptr) -#ifdef TRACK_BUFFER_USER - , - _location(nullptr) -#endif - { - } + IOBufferData() : _size_index(BUFFER_SIZE_NOT_ALLOCATED) {} // noncopyable, declaration only IOBufferData(const IOBufferData &) = delete; @@ -427,7 +418,7 @@ class IOBufferBlock : public RefCountObj Create a copy of the IOBufferBlock. Creates and returns a copy of this IOBufferBlock that references the same data that this IOBufferBlock (it does not allocate an another buffer). The cloned block will not - have a writable space since the original IOBufferBlock mantains the + have a writable space since the original IOBufferBlock maintains the ownership for writing data to the block. @return copy of this IOBufferBlock. @@ -489,12 +480,12 @@ class IOBufferBlock : public RefCountObj */ void free() override; - char *_start; - char *_end; - char *_buf_end; + char *_start = nullptr; + char *_end = nullptr; + char *_buf_end = nullptr; #ifdef TRACK_BUFFER_USER - const char *_location; + const char *_location = nullptr; #endif /** @@ -594,6 +585,7 @@ class IOBufferChain const_iterator() = default; ///< Default constructor. /// Copy constructor. + // cppcheck-suppress noExplicitConstructor; copy constructor const_iterator(self_type const &that); /// Assignment. @@ -714,11 +706,17 @@ class IOBufferReader */ int64_t block_read_avail(); + /** Get a view of the data available to read. + * + * @return A view encompassing currently available readable data. + */ + std::string_view block_read_view(); + void skip_empty_blocks(); /** Clears all fields in this IOBuffeReader, rendering it unusable. Drops - the reference to the IOBufferBlock list, the accesor, MIOBuffer and + the reference to the IOBufferBlock list, the accessor, MIOBuffer and resets this reader's state. You have to set those fields in order to use this object again. @@ -813,7 +811,7 @@ class IOBufferReader /** Perform a memchr() across the list of IOBufferBlocks. Returns the offset from the current start point of the reader to the first - occurence of character 'c' in the buffer. + occurrence of character 'c' in the buffer. @param c character to look for. @param len number of characters to check. If len exceeds the number @@ -823,7 +821,7 @@ class IOBufferReader @param offset number of the bytes to skip over before beginning the operation. @return -1 if c is not found, otherwise position of the first - ocurrence. + occurrence. */ inkcoreapi int64_t memchr(char c, int64_t len = INT64_MAX, int64_t offset = 0); @@ -887,14 +885,14 @@ class IOBufferReader return mbuf; } - MIOBufferAccessor *accessor; // pointer back to the accessor + MIOBufferAccessor *accessor = nullptr; // pointer back to the accessor /** Back pointer to this object's MIOBuffer. A pointer back to the MIOBuffer this reader is allocated from. */ - MIOBuffer *mbuf; + MIOBuffer *mbuf = nullptr; Ptr block; /** @@ -903,10 +901,10 @@ class IOBufferReader of the available data. */ - int64_t start_offset; - int64_t size_limit; + int64_t start_offset = 0; + int64_t size_limit = INT64_MAX; - IOBufferReader() : accessor(nullptr), mbuf(nullptr), start_offset(0), size_limit(INT64_MAX) {} + IOBufferReader() {} }; /** @@ -1085,7 +1083,7 @@ class MIOBuffer /** Returns the amount of space of available for writing on the first - writable block on the block chain (the one that would be reutrned + writable block on the block chain (the one that would be returned by first_write_block()). */ @@ -1297,10 +1295,11 @@ class MIOBuffer IOBufferReader readers[MAX_MIOBUFFER_READERS]; #ifdef TRACK_BUFFER_USER - const char *_location; + const char *_location = nullptr; #endif MIOBuffer(void *b, int64_t bufsize, int64_t aWater_mark); + // cppcheck-suppress noExplicitConstructor; allow implicit conversion MIOBuffer(int64_t default_size_index); MIOBuffer(); ~MIOBuffer(); @@ -1346,20 +1345,12 @@ struct MIOBufferAccessor { entry = nullptr; } - MIOBufferAccessor() - : -#ifdef DEBUG - name(nullptr), -#endif - mbuf(nullptr), - entry(nullptr) - { - } + MIOBufferAccessor() {} ~MIOBufferAccessor(); #ifdef DEBUG - const char *name; + const char *name = nullptr; #endif // noncopyable @@ -1367,8 +1358,8 @@ struct MIOBufferAccessor { MIOBufferAccessor &operator=(const MIOBufferAccessor &) = delete; private: - MIOBuffer *mbuf; - IOBufferReader *entry; + MIOBuffer *mbuf = nullptr; + IOBufferReader *entry = nullptr; }; extern MIOBuffer *new_MIOBuffer_internal( @@ -1383,7 +1374,7 @@ class MIOBuffer_tracker const char *loc; public: - MIOBuffer_tracker(const char *_loc) : loc(_loc) {} + explicit MIOBuffer_tracker(const char *_loc) : loc(_loc) {} MIOBuffer * operator()(int64_t size_index = default_large_iobuffer_size) { @@ -1404,7 +1395,7 @@ class Empty_MIOBuffer_tracker const char *loc; public: - Empty_MIOBuffer_tracker(const char *_loc) : loc(_loc) {} + explicit Empty_MIOBuffer_tracker(const char *_loc) : loc(_loc) {} MIOBuffer * operator()(int64_t size_index = default_large_iobuffer_size) { @@ -1442,7 +1433,7 @@ class IOBufferBlock_tracker const char *loc; public: - IOBufferBlock_tracker(const char *_loc) : loc(_loc) {} + explicit IOBufferBlock_tracker(const char *_loc) : loc(_loc) {} IOBufferBlock * operator()() { @@ -1488,7 +1479,7 @@ class IOBufferData_tracker const char *loc; public: - IOBufferData_tracker(const char *_loc) : loc(_loc) {} + explicit IOBufferData_tracker(const char *_loc) : loc(_loc) {} IOBufferData * operator()(int64_t size_index = default_large_iobuffer_size, AllocType type = DEFAULT_ALLOC) { diff --git a/iocore/eventsystem/I_Lock.h b/iocore/eventsystem/I_Lock.h index 9664606e55e..b5bef2c40e0 100644 --- a/iocore/eventsystem/I_Lock.h +++ b/iocore/eventsystem/I_Lock.h @@ -129,7 +129,7 @@ inkcoreapi extern void lock_taken(const SourceLocation &, const char *handler); A ProxyMutex object has an ink_mutex member (defined in ink_mutex.h) which is a wrapper around the platform dependent mutex type. This - member allows the ProxyMutex to provide the functionallity required + member allows the ProxyMutex to provide the functionality required by the users of the class without the burden of platform specific function calls. @@ -234,83 +234,20 @@ class ProxyMutex : public RefCountObj } }; -// The ClassAlocator for ProxyMutexes +// The ClassAllocator for ProxyMutexes extern inkcoreapi ClassAllocator mutexAllocator; -inline bool -Mutex_trylock( -#ifdef DEBUG - const SourceLocation &location, const char *ahandler, -#endif - ProxyMutex *m, EThread *t) -{ - ink_assert(t != nullptr); - ink_assert(t == (EThread *)this_thread()); - if (m->thread_holding != t) { - if (!ink_mutex_try_acquire(&m->the_mutex)) { -#ifdef DEBUG - lock_waiting(m->srcloc, m->handler); -#ifdef LOCK_CONTENTION_PROFILING - m->unsuccessful_nonblocking_acquires++; - m->nonblocking_acquires++; - m->total_acquires++; - m->print_lock_stats(0); -#endif // LOCK_CONTENTION_PROFILING -#endif // DEBUG - return false; - } - m->thread_holding = t; -#ifdef DEBUG - m->srcloc = location; - m->handler = ahandler; - m->hold_time = Thread::get_hrtime(); -#ifdef MAX_LOCK_TAKEN - m->taken++; -#endif // MAX_LOCK_TAKEN -#endif // DEBUG - } -#ifdef DEBUG -#ifdef LOCK_CONTENTION_PROFILING - m->successful_nonblocking_acquires++; - m->nonblocking_acquires++; - m->total_acquires++; - m->print_lock_stats(0); -#endif // LOCK_CONTENTION_PROFILING -#endif // DEBUG - m->nthread_holding++; - return true; -} - inline bool Mutex_trylock( #ifdef DEBUG const SourceLocation &location, const char *ahandler, #endif Ptr &m, EThread *t) -{ - return Mutex_trylock( -#ifdef DEBUG - location, ahandler, -#endif - m.get(), t); -} - -inline bool -Mutex_trylock_spin( -#ifdef DEBUG - const SourceLocation &location, const char *ahandler, -#endif - ProxyMutex *m, EThread *t, int spincnt = 1) { ink_assert(t != nullptr); + ink_assert(t == reinterpret_cast(this_thread())); if (m->thread_holding != t) { - int locked; - do { - if ((locked = ink_mutex_try_acquire(&m->the_mutex))) { - break; - } - } while (--spincnt); - if (!locked) { + if (!ink_mutex_try_acquire(&m->the_mutex)) { #ifdef DEBUG lock_waiting(m->srcloc, m->handler); #ifdef LOCK_CONTENTION_PROFILING @@ -323,7 +260,6 @@ Mutex_trylock_spin( return false; } m->thread_holding = t; - ink_assert(m->thread_holding); #ifdef DEBUG m->srcloc = location; m->handler = ahandler; @@ -345,26 +281,12 @@ Mutex_trylock_spin( return true; } -inline bool -Mutex_trylock_spin( -#ifdef DEBUG - const SourceLocation &location, const char *ahandler, -#endif - Ptr &m, EThread *t, int spincnt = 1) -{ - return Mutex_trylock_spin( -#ifdef DEBUG - location, ahandler, -#endif - m.get(), t, spincnt); -} - inline int Mutex_lock( #ifdef DEBUG const SourceLocation &location, const char *ahandler, #endif - ProxyMutex *m, EThread *t) + Ptr &m, EThread *t) { ink_assert(t != nullptr); if (m->thread_holding != t) { @@ -391,22 +313,8 @@ Mutex_lock( return true; } -inline int -Mutex_lock( -#ifdef DEBUG - const SourceLocation &location, const char *ahandler, -#endif - Ptr &m, EThread *t) -{ - return Mutex_lock( -#ifdef DEBUG - location, ahandler, -#endif - m.get(), t); -} - inline void -Mutex_unlock(ProxyMutex *m, EThread *t) +Mutex_unlock(Ptr &m, EThread *t) { if (m->nthread_holding) { ink_assert(t == m->thread_holding); @@ -429,12 +337,6 @@ Mutex_unlock(ProxyMutex *m, EThread *t) } } -inline void -Mutex_unlock(Ptr &m, EThread *t) -{ - Mutex_unlock(m.get(), t); -} - /** Scoped lock class for ProxyMutex */ class MutexLock @@ -444,20 +346,6 @@ class MutexLock bool locked_p; public: - MutexLock( -#ifdef DEBUG - const SourceLocation &location, const char *ahandler, -#endif // DEBUG - ProxyMutex *am, EThread *t) - : m(am), locked_p(true) - { - Mutex_lock( -#ifdef DEBUG - location, ahandler, -#endif // DEBUG - m.get(), t); - } - MutexLock( #ifdef DEBUG const SourceLocation &location, const char *ahandler, @@ -469,7 +357,7 @@ class MutexLock #ifdef DEBUG location, ahandler, #endif // DEBUG - m.get(), t); + m, t); } void @@ -493,20 +381,6 @@ class MutexTryLock bool lock_acquired; public: - MutexTryLock( -#ifdef DEBUG - const SourceLocation &location, const char *ahandler, -#endif // DEBUG - ProxyMutex *am, EThread *t) - : m(am) - { - lock_acquired = Mutex_trylock( -#ifdef DEBUG - location, ahandler, -#endif // DEBUG - m.get(), t); - } - MutexTryLock( #ifdef DEBUG const SourceLocation &location, const char *ahandler, @@ -514,45 +388,21 @@ class MutexTryLock Ptr &am, EThread *t) : m(am) { - lock_acquired = Mutex_trylock( -#ifdef DEBUG - location, ahandler, -#endif // DEBUG - m.get(), t); - } - - MutexTryLock( -#ifdef DEBUG - const SourceLocation &location, const char *ahandler, -#endif // DEBUG - ProxyMutex *am, EThread *t, int sp) - : m(am) - { - lock_acquired = Mutex_trylock_spin( + if (am) { + lock_acquired = Mutex_trylock( #ifdef DEBUG - location, ahandler, -#endif // DEBUG - m.get(), t, sp); - } - - MutexTryLock( -#ifdef DEBUG - const SourceLocation &location, const char *ahandler, -#endif // DEBUG - Ptr &am, EThread *t, int sp) - : m(am) - { - lock_acquired = Mutex_trylock_spin( -#ifdef DEBUG - location, ahandler, + location, ahandler, #endif // DEBUG - m.get(), t, sp); + m, t); + } else { + lock_acquired = true; + } } ~MutexTryLock() { - if (lock_acquired) { - Mutex_unlock(m.get(), m->thread_holding); + if (lock_acquired && m.get()) { + Mutex_unlock(m, m->thread_holding); } } @@ -561,16 +411,17 @@ class MutexTryLock void acquire(EThread *t) { - MUTEX_TAKE_LOCK(m.get(), t); lock_acquired = true; + if (m.get()) { + MUTEX_TAKE_LOCK(m, t); + } } void release() { - ink_assert(lock_acquired); // generate a warning because it shouldn't be done. - if (lock_acquired) { - Mutex_unlock(m.get(), m->thread_holding); + if (lock_acquired && m.get()) { + Mutex_unlock(m, m->thread_holding); } lock_acquired = false; } diff --git a/iocore/eventsystem/I_MIOBufferWriter.h b/iocore/eventsystem/I_MIOBufferWriter.h index 639f4615482..856881626a6 100644 --- a/iocore/eventsystem/I_MIOBufferWriter.h +++ b/iocore/eventsystem/I_MIOBufferWriter.h @@ -42,7 +42,7 @@ class MIOBufferWriter : public ts::BufferWriter using self_type = MIOBufferWriter; ///< Self reference type. public: - MIOBufferWriter(MIOBuffer *miob) : _miob(miob) {} + explicit MIOBufferWriter(MIOBuffer *miob) : _miob(miob) {} self_type &write(const void *data_, size_t length) override; diff --git a/iocore/eventsystem/I_Processor.h b/iocore/eventsystem/I_Processor.h index cce9a01cf64..291c7386d3b 100644 --- a/iocore/eventsystem/I_Processor.h +++ b/iocore/eventsystem/I_Processor.h @@ -41,7 +41,7 @@ class Thread; processors in the IO Core. A processor is multithreaded subsystem specialized in some type of task or application. For example, the Event System module includes the EventProcessor which provides - schedulling services, the Net module includes the NetProcessor + scheduling services, the Net module includes the NetProcessor which provides networking services, etc. You cannot create objects of the Processor class and its methods @@ -76,7 +76,7 @@ class Processor virtual int get_thread_count(); /** - This function attemps to stop the processor. Please refer to + This function attempts to stop the processor. Please refer to the documentation on each processor to determine if it is supported. diff --git a/iocore/eventsystem/I_ProxyAllocator.h b/iocore/eventsystem/I_ProxyAllocator.h index 4cf8eaeb3cf..a5edd125fd0 100644 --- a/iocore/eventsystem/I_ProxyAllocator.h +++ b/iocore/eventsystem/I_ProxyAllocator.h @@ -39,10 +39,10 @@ extern int thread_freelist_low_watermark; extern int cmd_disable_pfreelist; struct ProxyAllocator { - int allocated; - void *freelist; + int allocated = 0; + void *freelist = nullptr; - ProxyAllocator() : allocated(0), freelist(nullptr) {} + ProxyAllocator() {} }; template diff --git a/iocore/eventsystem/I_Tasks.h b/iocore/eventsystem/I_Tasks.h index 0484f366bea..160ffe8f87a 100644 --- a/iocore/eventsystem/I_Tasks.h +++ b/iocore/eventsystem/I_Tasks.h @@ -31,6 +31,7 @@ extern EventType ET_TASK; class TasksProcessor : public Processor { public: + EventType register_event_type(); int start(int task_threads, size_t stacksize = DEFAULT_STACKSIZE) override; }; diff --git a/iocore/eventsystem/I_Thread.h b/iocore/eventsystem/I_Thread.h index 0545d6cde9a..54cb9b0ee0b 100644 --- a/iocore/eventsystem/I_Thread.h +++ b/iocore/eventsystem/I_Thread.h @@ -99,6 +99,7 @@ class Thread processors and you should not modify it directly. */ + // NOLINTNEXTLINE(modernize-use-nullptr) ink_thread tid = 0; /** @@ -146,7 +147,7 @@ class Thread This gets a cached copy of the time so it is very fast and reasonably accurate. The cached time is updated every time the actual operating system time is fetched which is at least every 10ms and generally more frequently. - @note The cached copy shared among threads which means the cached copy is udpated + @note The cached copy shared among threads which means the cached copy is updated for all threads if any thread updates it. */ static ink_hrtime get_hrtime(); diff --git a/iocore/eventsystem/I_VConnection.h b/iocore/eventsystem/I_VConnection.h index 5c5b083f18d..80a7218f08e 100644 --- a/iocore/eventsystem/I_VConnection.h +++ b/iocore/eventsystem/I_VConnection.h @@ -54,7 +54,7 @@ static constexpr int TS_VCONN_MAX_USER_ARG = 4; #define VC_EVENT_READ_READY VC_EVENT_EVENTS_START /** - Any data in the accociated buffer *will be written* when the + Any data in the associated buffer *will be written* when the Continuation returns. */ @@ -73,7 +73,7 @@ static constexpr int TS_VCONN_MAX_USER_ARG = 4; #define VC_EVENT_ERROR EVENT_ERROR /** - VC_EVENT_INACTIVITY_TIMEOUT indiates that the operation (read or write) has: + VC_EVENT_INACTIVITY_TIMEOUT indicates that the operation (read or write) has: -# been enabled for more than the inactivity timeout period (for a read, there has been space in the buffer) (for a write, there has been data in the buffer) @@ -85,7 +85,7 @@ static constexpr int TS_VCONN_MAX_USER_ARG = 4; #define VC_EVENT_INACTIVITY_TIMEOUT (VC_EVENT_EVENTS_START + 5) /** - Total time for some operation has been exeeded, regardless of any + Total time for some operation has been exceeded, regardless of any intermediate progress. */ @@ -101,10 +101,10 @@ static constexpr int TS_VCONN_MAX_USER_ARG = 4; // VC_EVENT_READ_READ occurs when data *has been written* into // the associated buffer. // -// VC_EVENT_ERROR indicates that some error has occured. The +// VC_EVENT_ERROR indicates that some error has occurred. The // "data" will be either 0 if the errno is unavailable or errno. // -// VC_EVENT_INTERVAL indidates that an interval timer has expired. +// VC_EVENT_INTERVAL indicates that an interval timer has expired. // // @@ -260,7 +260,7 @@ class VConnection : public Continuation must call this function to indicate that the VConnection can be deallocated. After a close has been called, the VConnection and underlying processor must not send any more events related - to this VConnection to the state machine. Likeswise, the state + to this VConnection to the state machine. Likewise, the state machine must not access the VConnection or any VIOs obtained from it after calling this method. @@ -313,8 +313,8 @@ class VConnection : public Continuation */ virtual void do_io_shutdown(ShutdownHowTo_t howto) = 0; - VConnection(ProxyMutex *aMutex); - VConnection(Ptr &aMutex); + explicit VConnection(ProxyMutex *aMutex); + explicit VConnection(Ptr &aMutex); // Private // Set continuation on a given vio. The public interface @@ -335,7 +335,7 @@ class VConnection : public Continuation @param id Identifier associated to interpret the data field @param data Value or pointer with state machine or VConnection data. - @return True if the oparation is successful. + @return True if the operation is successful. */ virtual bool @@ -356,7 +356,7 @@ class VConnection : public Continuation @param id Identifier associated to interpret the data field. @param data Value or pointer with state machine or VConnection data. - @return True if the oparation is successful. + @return True if the operation is successful. */ virtual bool @@ -389,8 +389,8 @@ class AnnotatedVConnection : public VConnection using super_type = VConnection; public: - AnnotatedVConnection(ProxyMutex *aMutex) : super_type(aMutex){}; - AnnotatedVConnection(Ptr &aMutex) : super_type(aMutex){}; + explicit AnnotatedVConnection(ProxyMutex *aMutex) : super_type(aMutex){}; + explicit AnnotatedVConnection(Ptr &aMutex) : super_type(aMutex){}; void * get_user_arg(unsigned ix) const @@ -406,7 +406,7 @@ class AnnotatedVConnection : public VConnection }; protected: - std::array user_args; + std::array user_args{{nullptr}}; }; struct DummyVConnection : public AnnotatedVConnection { @@ -440,5 +440,5 @@ struct DummyVConnection : public AnnotatedVConnection { "cannot use default implementation"); } - DummyVConnection(ProxyMutex *m) : AnnotatedVConnection(m) {} + explicit DummyVConnection(ProxyMutex *m) : AnnotatedVConnection(m) {} }; diff --git a/iocore/eventsystem/I_VIO.h b/iocore/eventsystem/I_VIO.h index 6e7748c6640..e31c5ddb247 100644 --- a/iocore/eventsystem/I_VIO.h +++ b/iocore/eventsystem/I_VIO.h @@ -139,7 +139,7 @@ class VIO */ inkcoreapi void reenable_re(); - VIO(int aop); + explicit VIO(int aop); VIO(); enum { @@ -165,7 +165,7 @@ class VIO call with events for this operation. */ - Continuation *cont; + Continuation *cont = nullptr; /** Number of bytes to be done for this operation. @@ -173,7 +173,7 @@ class VIO The total number of bytes this operation must complete. */ - int64_t nbytes; + int64_t nbytes = 0; /** Number of bytes already completed. @@ -183,7 +183,7 @@ class VIO the lock. */ - int64_t ndone; + int64_t ndone = 0; /** Type of operation. @@ -191,7 +191,7 @@ class VIO The type of operation that this VIO represents. */ - int op; + int op = VIO::NONE; /** Provides access to the reader or writer for this operation. @@ -207,7 +207,7 @@ class VIO functions. */ - VConnection *vc_server; + VConnection *vc_server = nullptr; /** Reference to the state machine's mutex. diff --git a/iocore/eventsystem/Makefile.am b/iocore/eventsystem/Makefile.am index e1b83484485..6f44a16b983 100644 --- a/iocore/eventsystem/Makefile.am +++ b/iocore/eventsystem/Makefile.am @@ -59,7 +59,6 @@ libinkevent_a_SOURCES = \ P_UnixSocketManager.h \ P_VConnection.h \ P_VIO.h \ - Continuation.cc \ Processor.cc \ ProtectedQueue.cc \ ProxyAllocator.cc \ @@ -95,7 +94,7 @@ test_LD_ADD = \ $(top_builddir)/iocore/eventsystem/libinkevent.a \ $(top_builddir)/src/tscore/libtscore.la $(top_builddir)/src/tscpp/util/libtscpputil.la \ $(top_builddir)/proxy/shared/libUglyLogStubs.a \ - @LIBTCL@ @HWLOC_LIBS@ + @HWLOC_LIBS@ test_Buffer_SOURCES = \ test_Buffer.cc diff --git a/iocore/eventsystem/P_EventSystem.h b/iocore/eventsystem/P_EventSystem.h index 006fa3d95c8..a6222ba49d3 100644 --- a/iocore/eventsystem/P_EventSystem.h +++ b/iocore/eventsystem/P_EventSystem.h @@ -29,7 +29,7 @@ **************************************************************************/ #pragma once -#define _P_EventSystem_h +#define _P_EventSystem_H #include "tscore/ink_platform.h" @@ -45,6 +45,6 @@ #include "P_ProtectedQueue.h" #include "P_UnixEventProcessor.h" #include "P_UnixSocketManager.h" -#undef EVENT_SYSTEM_MODULE_VERSION -#define EVENT_SYSTEM_MODULE_VERSION \ - makeModuleVersion(EVENT_SYSTEM_MODULE_MAJOR_VERSION, EVENT_SYSTEM_MODULE_MINOR_VERSION, PRIVATE_MODULE_HEADER) + +static constexpr ts::ModuleVersion EVENT_SYSTEM_MODULE_INTERNAL_VERSION{EVENT_SYSTEM_MODULE_PUBLIC_VERSION, + ts::ModuleVersion::PRIVATE}; diff --git a/iocore/eventsystem/P_Freer.h b/iocore/eventsystem/P_Freer.h index c620c573d31..deb916d4e8f 100644 --- a/iocore/eventsystem/P_Freer.h +++ b/iocore/eventsystem/P_Freer.h @@ -27,7 +27,7 @@ #include "I_Tasks.h" // Note that these should not be used for memory that wishes to retain -// NUMA socket affinity. We'll potentially return these on an arbitarily +// NUMA socket affinity. We'll potentially return these on an arbitrarily // selected processor/socket. template struct DeleterContinuation : public Continuation { @@ -44,7 +44,7 @@ template struct DeleterContinuation : public Continuation { delete this; return EVENT_DONE; } - DeleterContinuation(C *ap) : Continuation(new_ProxyMutex()), p(ap) { SET_HANDLER(&DeleterContinuation::dieEvent); } + explicit DeleterContinuation(C *ap) : Continuation(new_ProxyMutex()), p(ap) { SET_HANDLER(&DeleterContinuation::dieEvent); } }; // This can be useful for two things (or both): @@ -73,7 +73,7 @@ template struct FreeCallContinuation : public Continuation { delete this; return EVENT_DONE; } - FreeCallContinuation(C *ap) : Continuation(nullptr), p(ap) { SET_HANDLER(&FreeCallContinuation::dieEvent); } + explicit FreeCallContinuation(C *ap) : Continuation(nullptr), p(ap) { SET_HANDLER(&FreeCallContinuation::dieEvent); } }; template @@ -99,7 +99,10 @@ struct FreerContinuation : public Continuation { return EVENT_DONE; } - FreerContinuation(void *ap) : Continuation(nullptr), p(ap) { SET_HANDLER((FreerContHandler)&FreerContinuation::dieEvent); } + explicit FreerContinuation(void *ap) : Continuation(nullptr), p(ap) + { + SET_HANDLER((FreerContHandler)&FreerContinuation::dieEvent); + } }; TS_INLINE void @@ -123,7 +126,7 @@ template struct DereferContinuation : public Continuation { return EVENT_DONE; } - DereferContinuation(C *ap) : Continuation(nullptr), p(ap) { SET_HANDLER(&DereferContinuation::dieEvent); } + explicit DereferContinuation(C *ap) : Continuation(nullptr), p(ap) { SET_HANDLER(&DereferContinuation::dieEvent); } }; template diff --git a/iocore/eventsystem/P_IOBuffer.h b/iocore/eventsystem/P_IOBuffer.h index c34af228c20..c13202d0259 100644 --- a/iocore/eventsystem/P_IOBuffer.h +++ b/iocore/eventsystem/P_IOBuffer.h @@ -382,13 +382,6 @@ new_IOBufferBlock_internal( TS_INLINE IOBufferBlock::IOBufferBlock() - : _start(nullptr), - _end(nullptr), - _buf_end(nullptr) -#ifdef TRACK_BUFFER_USER - , - _location(nullptr) -#endif { return; } @@ -621,6 +614,13 @@ IOBufferReader::block_read_avail() return (int64_t)(block->end() - (block->start() + start_offset)); } +inline std::string_view +IOBufferReader::block_read_view() +{ + const char *start = this->start(); // empty blocks are skipped in here. + return start ? std::string_view{start, static_cast(block->end() - start)} : std::string_view{}; +} + TS_INLINE int IOBufferReader::block_count() { @@ -709,7 +709,7 @@ TS_INLINE char &IOBufferReader::operator[](int64_t i) } ink_release_assert(!"out of range"); - // Never used, just to satisfy compilers not undersatnding the fatality of ink_release_assert(). + // Never used, just to satisfy compilers not understanding the fatality of ink_release_assert(). return default_ret; } diff --git a/iocore/eventsystem/P_Thread.h b/iocore/eventsystem/P_Thread.h index 6e2a90771a0..f3030045b0c 100644 --- a/iocore/eventsystem/P_Thread.h +++ b/iocore/eventsystem/P_Thread.h @@ -45,5 +45,5 @@ Thread::set_specific() TS_INLINE Thread * this_thread() { - return (Thread *)ink_thread_getspecific(Thread::thread_data_key); + return static_cast(ink_thread_getspecific(Thread::thread_data_key)); } diff --git a/iocore/eventsystem/P_UnixEThread.h b/iocore/eventsystem/P_UnixEThread.h index 9328c3b455c..834a6c923fe 100644 --- a/iocore/eventsystem/P_UnixEThread.h +++ b/iocore/eventsystem/P_UnixEThread.h @@ -176,7 +176,18 @@ EThread::schedule_spawn(Continuation *c, int ev, void *cookie) TS_INLINE EThread * this_ethread() { - return (EThread *)this_thread(); + return dynamic_cast(this_thread()); +} + +TS_INLINE EThread * +this_event_thread() +{ + EThread *ethread = this_ethread(); + if (ethread != nullptr && ethread->tt == REGULAR) { + return ethread; + } else { + return nullptr; + } } TS_INLINE void diff --git a/iocore/eventsystem/P_UnixEventProcessor.h b/iocore/eventsystem/P_UnixEventProcessor.h index ba58de9ee7c..b4dea2e9340 100644 --- a/iocore/eventsystem/P_UnixEventProcessor.h +++ b/iocore/eventsystem/P_UnixEventProcessor.h @@ -54,12 +54,7 @@ EventProcessor::assign_thread(EventType etype) ink_assert(etype < MAX_EVENT_TYPES); if (tg->_count > 1) { - // When "_next_round_robin" grows big enough, it becomes a negative number, - // meaning "next" is also negative. And since "next" is used as an index - // into array "_thread", the result is returning NULL when assigning threads. - // So we need to cast "_next_round_robin" to unsigned int so the result stays - // positive. - next = static_cast(++tg->_next_round_robin) % tg->_count; + next = ++tg->_next_round_robin % tg->_count; } else { next = 0; } @@ -70,7 +65,23 @@ TS_INLINE Event * EventProcessor::schedule(Event *e, EventType etype, bool fast_signal) { ink_assert(etype < MAX_EVENT_TYPES); - e->ethread = assign_thread(etype); + + EThread *ethread = e->continuation->getThreadAffinity(); + if (ethread != nullptr && ethread->is_event_type(etype)) { + e->ethread = ethread; + } else { + ethread = this_ethread(); + // Is the current thread eligible? + if (ethread != nullptr && ethread->is_event_type(etype)) { + e->ethread = ethread; + } else { + e->ethread = assign_thread(etype); + } + if (e->continuation->getThreadAffinity() == nullptr) { + e->continuation->setThreadAffinity(e->ethread); + } + } + if (e->continuation->mutex) { e->mutex = e->continuation->mutex; } else { diff --git a/iocore/eventsystem/P_UnixSocketManager.h b/iocore/eventsystem/P_UnixSocketManager.h index b90b2b1a13b..dc5c0c911ec 100644 --- a/iocore/eventsystem/P_UnixSocketManager.h +++ b/iocore/eventsystem/P_UnixSocketManager.h @@ -114,14 +114,12 @@ SocketManager::vector_io(int fd, struct iovec *vector, size_t count, int read_re { const int max_iovecs_per_request = 16; int n; - int64_t r = 0; + int64_t r; int n_vec; int64_t bytes_xfered = 0; - int current_count; - int64_t current_request_bytes; for (n_vec = 0; n_vec < (int)count; n_vec += max_iovecs_per_request) { - current_count = std::min(max_iovecs_per_request, ((int)(count - n_vec))); + int current_count = std::min(max_iovecs_per_request, ((int)(count - n_vec))); do { // coverity[tainted_data_argument] r = read_request ? ::readv(fd, &vector[n_vec], current_count) : ::writev(fd, &vector[n_vec], current_count); @@ -141,7 +139,7 @@ SocketManager::vector_io(int fd, struct iovec *vector, size_t count, int read_re } // Compute bytes in current vector - current_request_bytes = 0; + int64_t current_request_bytes = 0; for (n = n_vec; n < (n_vec + current_count); ++n) { current_request_bytes += vector[n].iov_len; } diff --git a/iocore/eventsystem/P_VIO.h b/iocore/eventsystem/P_VIO.h index d8fcb05451b..296296c07b8 100644 --- a/iocore/eventsystem/P_VIO.h +++ b/iocore/eventsystem/P_VIO.h @@ -25,7 +25,7 @@ #include "I_VIO.h" TS_INLINE -VIO::VIO(int aop) : cont(nullptr), nbytes(0), ndone(0), op(aop), buffer(), vc_server(nullptr), mutex(nullptr) {} +VIO::VIO(int aop) : op(aop), buffer(), mutex(nullptr) {} ///////////////////////////////////////////////////////////// // @@ -33,7 +33,7 @@ VIO::VIO(int aop) : cont(nullptr), nbytes(0), ndone(0), op(aop), buffer(), vc_se // ///////////////////////////////////////////////////////////// TS_INLINE -VIO::VIO() : cont(nullptr), nbytes(0), ndone(0), op(VIO::NONE), buffer(), vc_server(nullptr), mutex(nullptr) {} +VIO::VIO() : buffer(), mutex(nullptr) {} TS_INLINE Continuation * VIO::get_continuation() diff --git a/iocore/eventsystem/ProtectedQueue.cc b/iocore/eventsystem/ProtectedQueue.cc index bb9466ba0bb..e8741c79a89 100644 --- a/iocore/eventsystem/ProtectedQueue.cc +++ b/iocore/eventsystem/ProtectedQueue.cc @@ -93,7 +93,7 @@ ProtectedQueue::dequeue_timed(ink_hrtime cur_time, ink_hrtime timeout, bool slee void ProtectedQueue::dequeue_external() { - Event *e = (Event *)ink_atomiclist_popall(&al); + Event *e = static_cast(ink_atomiclist_popall(&al)); // invert the list, to preserve order SLL l, t; t.head = e; @@ -114,10 +114,14 @@ ProtectedQueue::dequeue_external() void ProtectedQueue::wait(ink_hrtime timeout) { - ink_mutex_acquire(&lock); + /* If there are no external events available, will do a cond_timedwait. + * + * - The `EThread::lock` will be released, + * - And then the Event Thread goes to sleep and waits for the wakeup signal of `EThread::might_have_data`, + * - The `EThread::lock` will be locked again when the Event Thread wakes up. + */ if (INK_ATOMICLIST_EMPTY(al)) { timespec ts = ink_hrtime_to_timespec(timeout); ink_cond_timedwait(&might_have_data, &lock, &ts); } - ink_mutex_release(&lock); } diff --git a/iocore/eventsystem/SocketManager.cc b/iocore/eventsystem/SocketManager.cc index 24bc38bc20a..6dffbe11e42 100644 --- a/iocore/eventsystem/SocketManager.cc +++ b/iocore/eventsystem/SocketManager.cc @@ -66,10 +66,8 @@ accept4(int sockfd, struct sockaddr *addr, socklen_t *addrlen, int flags) int SocketManager::accept4(int s, struct sockaddr *addr, socklen_t *addrlen, int flags) { - int fd; - do { - fd = ::accept4(s, addr, addrlen, flags); + int fd = ::accept4(s, addr, addrlen, flags); if (likely(fd >= 0)) { return fd; } diff --git a/iocore/eventsystem/Tasks.cc b/iocore/eventsystem/Tasks.cc index 91a5942c2db..7de34e2747d 100644 --- a/iocore/eventsystem/Tasks.cc +++ b/iocore/eventsystem/Tasks.cc @@ -27,11 +27,18 @@ EventType ET_TASK = ET_CALL; TasksProcessor tasksProcessor; +EventType +TasksProcessor::register_event_type() +{ + ET_TASK = eventProcessor.register_event_type("ET_TASK"); + return ET_TASK; +} + // Note that if the number of task_threads is 0, all continuations scheduled for // ET_TASK ends up running on ET_CALL (which is the net-threads). int TasksProcessor::start(int task_threads, size_t stacksize) { - ET_TASK = eventProcessor.spawn_event_threads("ET_TASK", std::max(1, task_threads), stacksize); + eventProcessor.spawn_event_threads(ET_TASK, std::max(1, task_threads), stacksize); return 0; } diff --git a/iocore/eventsystem/Thread.cc b/iocore/eventsystem/Thread.cc index f51232d1616..90faf9d8c56 100644 --- a/iocore/eventsystem/Thread.cc +++ b/iocore/eventsystem/Thread.cc @@ -50,13 +50,13 @@ static bool initialized ATS_UNUSED = ([]() -> bool { Thread::Thread() { mutex = new_ProxyMutex(); - MUTEX_TAKE_LOCK(mutex, (EThread *)this); + MUTEX_TAKE_LOCK(mutex, static_cast(this)); mutex->nthread_holding += THREAD_MUTEX_THREAD_HOLDING; } Thread::~Thread() { - ink_release_assert(mutex->thread_holding == (EThread *)this); + ink_release_assert(mutex->thread_holding == static_cast(this)); if (ink_thread_getspecific(Thread::thread_data_key) == this) { // Clear pointer to this object stored in thread-specific data by set_specific. @@ -65,7 +65,7 @@ Thread::~Thread() } mutex->nthread_holding -= THREAD_MUTEX_THREAD_HOLDING; - MUTEX_UNTAKE_LOCK(mutex, (EThread *)this); + MUTEX_UNTAKE_LOCK(mutex, static_cast(this)); } /////////////////////////////////////////////// @@ -73,7 +73,7 @@ Thread::~Thread() /////////////////////////////////////////////// struct thread_data_internal { - ThreadFunction f; ///< Function to excecute in the thread. + ThreadFunction f; ///< Function to execute in the thread. Thread *me; ///< The class instance. char name[MAX_THREAD_NAME_LENGTH]; ///< Name for the thread. }; diff --git a/iocore/eventsystem/UnixEThread.cc b/iocore/eventsystem/UnixEThread.cc index a5c72c591a8..03b31c3eeb3 100644 --- a/iocore/eventsystem/UnixEThread.cc +++ b/iocore/eventsystem/UnixEThread.cc @@ -21,6 +21,8 @@ limitations under the License. */ +#include + ////////////////////////////////////////////////////////////////////// // // The EThread Class @@ -45,8 +47,6 @@ char const *const EThread::STAT_NAME[] = {"proxy.process.eventloop.count", int const EThread::SAMPLE_COUNT[N_EVENT_TIMESCALES] = {10, 100, 1000}; -bool shutdown_event_system = false; - int thread_max_heartbeat_mseconds = THREAD_MAX_HEARTBEAT_MSECONDS; EThread::EThread() @@ -128,7 +128,7 @@ EThread::process_event(Event *e, int calling_code) return; } Continuation *c_temp = e->continuation; - // Make sure that the contination is locked before calling the handler + // Make sure that the continuation is locked before calling the handler e->continuation->handleEvent(calling_code, e); ink_assert(!e->in_the_priority_queue); ink_assert(c_temp == e->continuation); @@ -193,23 +193,23 @@ EThread::execute_regular() { Event *e; Que(Event, link) NegativeQueue; - ink_hrtime next_time = 0; - ink_hrtime delta = 0; // time spent in the event loop + ink_hrtime next_time; + ink_hrtime delta; // time spent in the event loop ink_hrtime loop_start_time; // Time the loop started. ink_hrtime loop_finish_time; // Time at the end of the loop. // Track this so we can update on boundary crossing. EventMetrics *prev_metric = this->prev(metrics + (ink_get_hrtime_internal() / HRTIME_SECOND) % N_EVENT_METRICS); - int nq_count = 0; - int ev_count = 0; + int nq_count; + int ev_count; // A statically initialized instance we can use as a prototype for initializing other instances. static EventMetrics METRIC_INIT; // give priority to immediate events for (;;) { - if (unlikely(shutdown_event_system == true)) { + if (TSSystemState::is_event_system_shut_down()) { return; } @@ -323,7 +323,16 @@ EThread::execute() switch (tt) { case REGULAR: { + /* The Event Thread has two status: busy and sleep: + * - Keep `EThread::lock` locked while Event Thread is busy, + * - The `EThread::lock` is released while Event Thread is sleep. + * When other threads try to acquire the `EThread::lock` of the target Event Thread: + * - Acquired, indicating that the target Event Thread is sleep, + * - Failed, indicating that the target Event Thread is busy. + */ + ink_mutex_acquire(&EventQueueExternal.lock); this->execute_regular(); + ink_mutex_release(&EventQueueExternal.lock); break; } case DEDICATED: { diff --git a/iocore/eventsystem/UnixEventProcessor.cc b/iocore/eventsystem/UnixEventProcessor.cc index 32b9ed63c44..f2bcf4ee9fc 100644 --- a/iocore/eventsystem/UnixEventProcessor.cc +++ b/iocore/eventsystem/UnixEventProcessor.cc @@ -396,6 +396,9 @@ EventProcessor::initThreadState(EThread *t) for (int i = 0; i < MAX_EVENT_TYPES; ++i) { if (t->is_event_type(i)) { // that event type done here, roll thread start events of that type. ++thread_group[i]._started; + if (thread_group[i]._started == thread_group[i]._count && thread_group[i]._afterStartCallback != nullptr) { + thread_group[i]._afterStartCallback(); + } // To avoid race conditions on the event in the spawn queue, create a local one to actually send. // Use the spawn queue event as a read only model. Event *nev = eventAllocator.alloc(); diff --git a/iocore/eventsystem/test_Buffer.cc b/iocore/eventsystem/test_Buffer.cc index 3e31a496fd3..ccef1036864 100644 --- a/iocore/eventsystem/test_Buffer.cc +++ b/iocore/eventsystem/test_Buffer.cc @@ -39,7 +39,7 @@ main(int /* argc ATS_UNUSED */, const char * /* argv ATS_UNUSED */ []) init_diags("", nullptr); RecProcessInit(mode_type); - ink_event_system_init(EVENT_SYSTEM_MODULE_VERSION); + ink_event_system_init(EVENT_SYSTEM_MODULE_PUBLIC_VERSION); eventProcessor.start(TEST_THREADS); Thread *main_thread = new EThread; diff --git a/iocore/eventsystem/test_Event.cc b/iocore/eventsystem/test_Event.cc index 6f52da1c47a..78c47bb3624 100644 --- a/iocore/eventsystem/test_Event.cc +++ b/iocore/eventsystem/test_Event.cc @@ -23,6 +23,7 @@ #include "I_EventSystem.h" #include "tscore/I_Layout.h" +#include "tscore/TSSystemState.h" #include "diags.i" @@ -68,14 +69,14 @@ main(int /* argc ATS_UNUSED */, const char * /* argv ATS_UNUSED */ []) init_diags("", nullptr); RecProcessInit(mode_type); - ink_event_system_init(EVENT_SYSTEM_MODULE_VERSION); + ink_event_system_init(EVENT_SYSTEM_MODULE_PUBLIC_VERSION); eventProcessor.start(TEST_THREADS, 1048576); // Hardcoded stacksize at 1MB alarm_printer *alrm = new alarm_printer(new_ProxyMutex()); process_killer *killer = new process_killer(new_ProxyMutex()); eventProcessor.schedule_in(killer, HRTIME_SECONDS(10)); eventProcessor.schedule_every(alrm, HRTIME_SECONDS(1)); - while (!shutdown_event_system) { + while (!TSSystemState::is_event_system_shut_down()) { sleep(1); } return 0; diff --git a/iocore/eventsystem/unit_tests/test_MIOBufferWriter.cc b/iocore/eventsystem/unit_tests/test_MIOBufferWriter.cc index fd78899e702..955bea04db5 100644 --- a/iocore/eventsystem/unit_tests/test_MIOBufferWriter.cc +++ b/iocore/eventsystem/unit_tests/test_MIOBufferWriter.cc @@ -29,7 +29,6 @@ #include "I_EventSystem.h" #include "tscore/I_Layout.h" -//#include "tscore/ink_string.h" #include "diags.i" #include "I_MIOBufferWriter.h" @@ -42,7 +41,7 @@ main(int argc, char *argv[]) init_diags("", nullptr); RecProcessInit(RECM_STAND_ALONE); - ink_event_system_init(EVENT_SYSTEM_MODULE_VERSION); + ink_event_system_init(EVENT_SYSTEM_MODULE_PUBLIC_VERSION); eventProcessor.start(2); Thread *main_thread = new EThread; diff --git a/iocore/hostdb/HostDB.cc b/iocore/hostdb/HostDB.cc index eb4cc5b2e33..179a040a5ad 100644 --- a/iocore/hostdb/HostDB.cc +++ b/iocore/hostdb/HostDB.cc @@ -32,6 +32,8 @@ #include #include #include +#include +#include HostDBProcessor hostDBProcessor; int HostDBProcessor::hostdb_strict_round_robin = 0; @@ -59,7 +61,7 @@ static ink_time_t hostdb_hostfile_update_timestamp = 0; static char hostdb_filename[PATH_NAME_MAX] = DEFAULT_HOST_DB_FILENAME; int hostdb_max_count = DEFAULT_HOST_DB_SIZE; char hostdb_hostfile_path[PATH_NAME_MAX] = ""; -int hostdb_sync_frequency = 120; +int hostdb_sync_frequency = 0; int hostdb_disable_reverse_lookup = 0; ClassAllocator hostDBContAllocator("hostDBContAllocator"); @@ -124,6 +126,9 @@ hostdb_cont_free(HostDBContinuation *cont) if (cont->pending_action) { cont->pending_action->cancel(); } + if (cont->timeout) { + cont->timeout->cancel(); + } cont->mutex = nullptr; cont->action.mutex = nullptr; hostDBContAllocator.free(cont); @@ -217,9 +222,7 @@ HostDBHash::refresh() ctx.finalize(hash); } -HostDBHash::HostDBHash() : host_name(nullptr), host_len(0), port(0), dns_server(nullptr), pSD(nullptr), db_mark(HOSTDB_MARK_GENERIC) -{ -} +HostDBHash::HostDBHash() {} HostDBHash::~HostDBHash() { @@ -228,7 +231,7 @@ HostDBHash::~HostDBHash() } } -HostDBCache::HostDBCache() : refcountcache(nullptr), pending_dns(nullptr), remoteHostDBQueue(nullptr) +HostDBCache::HostDBCache() { hosts_file_ptr = new RefCountedHostsFileMap(); } @@ -237,7 +240,7 @@ bool HostDBCache::is_pending_dns_for_hash(const CryptoHash &hash) { Queue &q = pending_dns_for_hash(hash); - for (HostDBContinuation *c = q.head; c; c = (HostDBContinuation *)c->link.next) { + for (HostDBContinuation *c = q.head; c; c = static_cast(c->link.next)) { if (hash == c->hash.hash) { return true; } @@ -283,7 +286,7 @@ HostDBBackgroundTask::wait_event(int, void *) struct HostDBSync : public HostDBBackgroundTask { std::string storage_path; std::string full_path; - HostDBSync(int frequency, std::string storage_path, std::string full_path) + HostDBSync(int frequency, const std::string &storage_path, const std::string &full_path) : HostDBBackgroundTask(frequency), storage_path(std::move(storage_path)), full_path(std::move(full_path)){}; int sync_event(int, void *) override @@ -389,7 +392,7 @@ HostDBProcessor::start(int, size_t) statPagesManager.register_http("hostdb", register_ShowHostDB); // - // Register configuration callback, and establish configuation links + // Register configuration callback, and establish configuration links // REC_EstablishStaticConfigInt32(hostdb_ttl_mode, "proxy.config.hostdb.ttl_mode"); REC_EstablishStaticConfigInt32(hostdb_disable_reverse_lookup, "proxy.config.cache.hostdb.disable_reverse_lookup"); @@ -441,6 +444,7 @@ HostDBContinuation::init(HostDBHash const &the_hash, Options const &opt) action = opt.cont; } else { // ink_assert(!"this sucks"); + ink_zero(action); action.mutex = mutex; } } @@ -448,13 +452,13 @@ HostDBContinuation::init(HostDBHash const &the_hash, Options const &opt) void HostDBContinuation::refresh_hash() { - ProxyMutex *old_bucket_mutex = hostDB.refcountcache->lock_for_key(hash.hash.fold()); + Ptr old_bucket_mutex = hostDB.refcountcache->lock_for_key(hash.hash.fold()); // We're not pending DNS anymore. remove_trigger_pending_dns(); hash.refresh(); // Update the mutex if it's from the bucket. // Some call sites modify this after calling @c init so need to check. - if (mutex.get() == old_bucket_mutex) { + if (mutex == old_bucket_mutex) { mutex = hostDB.refcountcache->lock_for_key(hash.hash.fold()); } } @@ -532,7 +536,7 @@ db_mark_for(IpAddr const &ip) } Ptr -probe(ProxyMutex *mutex, HostDBHash const &hash, bool ignore_timeout) +probe(const Ptr &mutex, HostDBHash const &hash, bool ignore_timeout) { // If hostdb is disabled, don't return anything if (!hostdb_enable) { @@ -604,88 +608,104 @@ HostDBContinuation::insert(unsigned int attl) // Get an entry by either name or IP // Action * -HostDBProcessor::getby(Continuation *cont, const char *hostname, int len, sockaddr const *ip, bool aforce_dns, - HostResStyle host_res_style, int dns_lookup_timeout) +HostDBProcessor::getby(Continuation *cont, cb_process_result_pfn cb_process_result, HostDBHash &hash, Options const &opt) { - HostDBHash hash; - EThread *thread = this_ethread(); - ProxyMutex *mutex = thread->mutex.get(); + bool force_dns = false; + EThread *thread = this_ethread(); + Ptr mutex = thread->mutex; ip_text_buffer ipb; + if (opt.flags & HOSTDB_FORCE_DNS_ALWAYS) { + force_dns = true; + } else if (opt.flags & HOSTDB_FORCE_DNS_RELOAD) { + force_dns = hostdb_re_dns_on_reload; + if (force_dns) { + HOSTDB_INCREMENT_DYN_STAT(hostdb_re_dns_on_reload_stat); + } + } + HOSTDB_INCREMENT_DYN_STAT(hostdb_total_lookups_stat); - if ((!hostdb_enable || (hostname && !*hostname)) || (hostdb_disable_reverse_lookup && ip)) { - MUTEX_TRY_LOCK(lock, cont->mutex, thread); - if (!lock.is_locked()) { - goto Lretry; + if (!hostdb_enable || // if the HostDB is disabled, + (hash.host_name && !*hash.host_name) || // or host_name is empty string + (hostdb_disable_reverse_lookup && hash.ip.isValid())) { // or try to lookup by ip address when the reverse lookup disabled + if (cb_process_result) { + (cont->*cb_process_result)(nullptr); + } else { + MUTEX_TRY_LOCK(lock, cont->mutex, thread); + if (!lock.is_locked()) { + goto Lretry; + } + cont->handleEvent(EVENT_HOST_DB_LOOKUP, nullptr); } - cont->handleEvent(EVENT_HOST_DB_LOOKUP, nullptr); return ACTION_RESULT_DONE; } - // Load the hash data. - hash.set_host(hostname, hostname ? (len ? len : strlen(hostname)) : 0); - hash.ip.assign(ip); - hash.port = ip ? ats_ip_port_host_order(ip) : 0; - hash.db_mark = db_mark_for(host_res_style); - hash.refresh(); - // Attempt to find the result in-line, for level 1 hits - // - if (!aforce_dns) { - bool loop; - do { + if (!force_dns) { + MUTEX_TRY_LOCK(lock, cont->mutex, thread); + bool loop = lock.is_locked(); + while (loop) { loop = false; // Only loop on explicit set for retry. // find the partition lock - // - // TODO: Could we reuse the "mutex" above safely? I think so but not sure. - ProxyMutex *bmutex = hostDB.refcountcache->lock_for_key(hash.hash.fold()); - MUTEX_TRY_LOCK(lock, bmutex, thread); - MUTEX_TRY_LOCK(lock2, cont->mutex, thread); - - if (lock.is_locked() && lock2.is_locked()) { + Ptr bucket_mutex = hostDB.refcountcache->lock_for_key(hash.hash.fold()); + MUTEX_TRY_LOCK(lock2, bucket_mutex, thread); + if (lock2.is_locked()) { // If we can get the lock and a level 1 probe succeeds, return - Ptr r = probe(bmutex, hash, aforce_dns); + Ptr r = probe(bucket_mutex, hash, false); if (r) { - if (r->is_failed() && hostname) { - loop = check_for_retry(hash.db_mark, host_res_style); + // fail, see if we should retry with alternate + if (hash.db_mark != HOSTDB_MARK_SRV && r->is_failed() && hash.host_name) { + loop = check_for_retry(hash.db_mark, opt.host_res_style); } if (!loop) { // No retry -> final result. Return it. - Debug("hostdb", "immediate answer for %s", - hostname ? hostname : ats_is_ip(ip) ? ats_ip_ntop(ip, ipb, sizeof ipb) : ""); + if (hash.db_mark == HOSTDB_MARK_SRV) { + Debug("hostdb", "immediate SRV answer for %.*s from hostdb", hash.host_len, hash.host_name); + Debug("dns_srv", "immediate SRV answer for %.*s from hostdb", hash.host_len, hash.host_name); + } else if (hash.host_name) { + Debug("hostdb", "immediate answer for %.*s", hash.host_len, hash.host_name); + } else { + Debug("hostdb", "immediate answer for %s", hash.ip.isValid() ? hash.ip.toString(ipb, sizeof ipb) : ""); + } HOSTDB_INCREMENT_DYN_STAT(hostdb_total_hits_stat); - reply_to_cont(cont, r.get()); + if (cb_process_result) { + (cont->*cb_process_result)(r.get()); + } else { + reply_to_cont(cont, r.get()); + } return ACTION_RESULT_DONE; } hash.refresh(); // only on reloop, because we've changed the family. } } - } while (loop); + } + } + if (hash.db_mark == HOSTDB_MARK_SRV) { + Debug("hostdb", "delaying (force=%d) SRV answer for %.*s [timeout = %d]", force_dns, hash.host_len, hash.host_name, + opt.timeout); + Debug("dns_srv", "delaying (force=%d) SRV answer for %.*s [timeout = %d]", force_dns, hash.host_len, hash.host_name, + opt.timeout); + } else if (hash.host_name) { + Debug("hostdb", "delaying (force=%d) answer for %.*s [timeout %d]", force_dns, hash.host_len, hash.host_name, opt.timeout); + } else { + Debug("hostdb", "delaying (force=%d) answer for %s [timeout %d]", force_dns, + hash.ip.isValid() ? hash.ip.toString(ipb, sizeof ipb) : "", opt.timeout); } - Debug("hostdb", "delaying force %d answer for %s", aforce_dns, - hostname ? hostname : ats_is_ip(ip) ? ats_ip_ntop(ip, ipb, sizeof ipb) : ""); Lretry: // Otherwise, create a continuation to do a deeper probe in the background // HostDBContinuation *c = hostDBContAllocator.alloc(); - HostDBContinuation::Options opt; - opt.timeout = dns_lookup_timeout; - opt.force_dns = aforce_dns; - opt.cont = cont; - opt.host_res_style = host_res_style; - c->init(hash, opt); + HostDBContinuation::Options copt; + copt.timeout = opt.timeout; + copt.force_dns = force_dns; + copt.cont = cont; + copt.host_res_style = (hash.db_mark == HOSTDB_MARK_SRV) ? HOST_RES_NONE : opt.host_res_style; + c->init(hash, copt); SET_CONTINUATION_HANDLER(c, (HostDBContHandler)&HostDBContinuation::probeEvent); - // Since ProxyMutexPtr has a cast operator, gcc-3.x get upset - // about ambiguity when doing this comparison, so by reversing - // the operands, I force it to pick the cast operation /leif. - if (thread->mutex == cont->mutex) { - thread->schedule_in(c, MUTEX_RETRY_DELAY); - } else { - dnsProcessor.thread->schedule_imm(c); - } + thread->schedule_in(c, MUTEX_RETRY_DELAY); return &c->action; } @@ -695,192 +715,96 @@ HostDBProcessor::getby(Continuation *cont, const char *hostname, int len, sockad Action * HostDBProcessor::getbyname_re(Continuation *cont, const char *ahostname, int len, Options const &opt) { - bool force_dns = false; - EThread *thread = this_ethread(); - ProxyMutex *mutex = thread->mutex.get(); + HostDBHash hash; - if (opt.flags & HOSTDB_FORCE_DNS_ALWAYS) { - force_dns = true; - } else if (opt.flags & HOSTDB_FORCE_DNS_RELOAD) { - force_dns = (hostdb_re_dns_on_reload ? true : false); - if (force_dns) { - HOSTDB_INCREMENT_DYN_STAT(hostdb_re_dns_on_reload_stat); - } - } - return getby(cont, ahostname, len, nullptr, force_dns, opt.host_res_style, opt.timeout); + ink_assert(nullptr != ahostname); + + // Load the hash data. + hash.set_host(ahostname, ahostname ? (len ? len : strlen(ahostname)) : 0); + // Leave hash.ip invalid + hash.port = 0; + hash.db_mark = db_mark_for(opt.host_res_style); + hash.refresh(); + + return getby(cont, nullptr, hash, opt); } Action * HostDBProcessor::getbynameport_re(Continuation *cont, const char *ahostname, int len, Options const &opt) { - bool force_dns = false; - EThread *thread = this_ethread(); - ProxyMutex *mutex = thread->mutex.get(); + HostDBHash hash; - if (opt.flags & HOSTDB_FORCE_DNS_ALWAYS) { - force_dns = true; - } else if (opt.flags & HOSTDB_FORCE_DNS_RELOAD) { - force_dns = (hostdb_re_dns_on_reload ? true : false); - if (force_dns) { - HOSTDB_INCREMENT_DYN_STAT(hostdb_re_dns_on_reload_stat); - } - } - sockaddr sa; - ats_ip4_set(&sa, INADDR_ANY, htons(opt.port)); - return getby(cont, ahostname, len, &sa, force_dns, opt.host_res_style, opt.timeout); + ink_assert(nullptr != ahostname); + + // Load the hash data. + hash.set_host(ahostname, ahostname ? (len ? len : strlen(ahostname)) : 0); + // Leave hash.ip invalid + hash.port = opt.port; + hash.db_mark = db_mark_for(opt.host_res_style); + hash.refresh(); + + return getby(cont, nullptr, hash, opt); } -/* Support SRV records */ +// Lookup Hostinfo by addr Action * -HostDBProcessor::getSRVbyname_imm(Continuation *cont, process_srv_info_pfn process_srv_info, const char *hostname, int len, - Options const &opt) +HostDBProcessor::getbyaddr_re(Continuation *cont, sockaddr const *aip) { - ink_assert(cont->mutex->thread_holding == this_ethread()); - bool force_dns = false; - EThread *thread = cont->mutex->thread_holding; - ProxyMutex *mutex = thread->mutex.get(); - - ink_assert(nullptr != hostname); - - if (opt.flags & HOSTDB_FORCE_DNS_ALWAYS) { - force_dns = true; - } else if (opt.flags & HOSTDB_FORCE_DNS_RELOAD) { - force_dns = (hostdb_re_dns_on_reload ? true : false); - if (force_dns) { - HOSTDB_INCREMENT_DYN_STAT(hostdb_re_dns_on_reload_stat); - } - } - HostDBHash hash; - HOSTDB_INCREMENT_DYN_STAT(hostdb_total_lookups_stat); + ink_assert(nullptr != aip); - if (!hostdb_enable || !*hostname) { - (cont->*process_srv_info)(nullptr); - return ACTION_RESULT_DONE; - } + HostDBProcessor::Options opt; + opt.host_res_style = HOST_RES_NONE; - hash.host_name = hostname; - hash.host_len = len ? len : strlen(hostname); - hash.port = 0; - hash.db_mark = HOSTDB_MARK_SRV; + // Leave hash.host_name as nullptr + hash.ip.assign(aip); + hash.port = ats_ip_port_host_order(aip); + hash.db_mark = db_mark_for(opt.host_res_style); hash.refresh(); - // Attempt to find the result in-line, for level 1 hits - if (!force_dns) { - // find the partition lock - ProxyMutex *bucket_mutex = hostDB.refcountcache->lock_for_key(hash.hash.fold()); - MUTEX_TRY_LOCK(lock, bucket_mutex, thread); - - // If we can get the lock and a level 1 probe succeeds, return - if (lock.is_locked()) { - Ptr r = probe(bucket_mutex, hash, false); - if (r) { - Debug("hostdb", "immediate SRV answer for %s from hostdb", hostname); - Debug("dns_srv", "immediate SRV answer for %s from hostdb", hostname); - HOSTDB_INCREMENT_DYN_STAT(hostdb_total_hits_stat); - (cont->*process_srv_info)(r.get()); - return ACTION_RESULT_DONE; - } - } - } + return getby(cont, nullptr, hash, opt); +} - Debug("dns_srv", "delaying (force=%d) SRV answer for %.*s [timeout = %d]", force_dns, hash.host_len, hash.host_name, opt.timeout); +/* Support SRV records */ +Action * +HostDBProcessor::getSRVbyname_imm(Continuation *cont, cb_process_result_pfn process_srv_info, const char *hostname, int len, + Options const &opt) +{ + ink_assert(cont->mutex->thread_holding == this_ethread()); + HostDBHash hash; - // Otherwise, create a continuation to do a deeper probe in the background - HostDBContinuation *c = hostDBContAllocator.alloc(); - HostDBContinuation::Options copt; - copt.timeout = opt.timeout; - copt.cont = cont; - copt.force_dns = force_dns; - c->init(hash, copt); - SET_CONTINUATION_HANDLER(c, (HostDBContHandler)&HostDBContinuation::probeEvent); + ink_assert(nullptr != hostname); - if (thread->mutex == cont->mutex) { - thread->schedule_in(c, MUTEX_RETRY_DELAY); - } else { - dnsProcessor.thread->schedule_imm(c); - } + hash.set_host(hostname, len ? len : strlen(hostname)); + // Leave hash.ip invalid + hash.port = 0; + hash.db_mark = HOSTDB_MARK_SRV; + hash.refresh(); - return &c->action; + return getby(cont, process_srv_info, hash, opt); } // Wrapper from getbyname to getby // Action * -HostDBProcessor::getbyname_imm(Continuation *cont, process_hostdb_info_pfn process_hostdb_info, const char *hostname, int len, +HostDBProcessor::getbyname_imm(Continuation *cont, cb_process_result_pfn process_hostdb_info, const char *hostname, int len, Options const &opt) { ink_assert(cont->mutex->thread_holding == this_ethread()); - bool force_dns = false; - EThread *thread = cont->mutex->thread_holding; - ProxyMutex *mutex = thread->mutex.get(); HostDBHash hash; ink_assert(nullptr != hostname); - if (opt.flags & HOSTDB_FORCE_DNS_ALWAYS) { - force_dns = true; - } else if (opt.flags & HOSTDB_FORCE_DNS_RELOAD) { - force_dns = (hostdb_re_dns_on_reload ? true : false); - if (force_dns) { - HOSTDB_INCREMENT_DYN_STAT(hostdb_re_dns_on_reload_stat); - } - } - - HOSTDB_INCREMENT_DYN_STAT(hostdb_total_lookups_stat); - - if (!hostdb_enable || !*hostname) { - (cont->*process_hostdb_info)(nullptr); - return ACTION_RESULT_DONE; - } - hash.set_host(hostname, len ? len : strlen(hostname)); + // Leave hash.ip invalid + // TODO: May I rename the wrapper name to getbynameport_imm ? - oknet + // By comparing getbyname_re and getbynameport_re, the hash.port should be 0 if only get hostinfo by name. hash.port = opt.port; hash.db_mark = db_mark_for(opt.host_res_style); hash.refresh(); - // Attempt to find the result in-line, for level 1 hits - if (!force_dns) { - bool loop; - do { - loop = false; // loop only on explicit set for retry - // find the partition lock - ProxyMutex *bucket_mutex = hostDB.refcountcache->lock_for_key(hash.hash.fold()); - SCOPED_MUTEX_LOCK(lock, bucket_mutex, thread); - // do a level 1 probe for immediate result. - Ptr r = probe(bucket_mutex, hash, false); - if (r) { - if (r->is_failed()) { // fail, see if we should retry with alternate - loop = check_for_retry(hash.db_mark, opt.host_res_style); - } - if (!loop) { - // No retry -> final result. Return it. - Debug("hostdb", "immediate answer for %.*s", hash.host_len, hash.host_name); - HOSTDB_INCREMENT_DYN_STAT(hostdb_total_hits_stat); - (cont->*process_hostdb_info)(r.get()); - return ACTION_RESULT_DONE; - } - hash.refresh(); // Update for retry. - } - } while (loop); - } - - Debug("hostdb", "delaying force %d answer for %.*s [timeout %d]", force_dns, hash.host_len, hash.host_name, opt.timeout); - - // Otherwise, create a continuation to do a deeper probe in the background - HostDBContinuation *c = hostDBContAllocator.alloc(); - HostDBContinuation::Options copt; - copt.cont = cont; - copt.force_dns = force_dns; - copt.timeout = opt.timeout; - copt.host_res_style = opt.host_res_style; - c->init(hash, copt); - SET_CONTINUATION_HANDLER(c, (HostDBContHandler)&HostDBContinuation::probeEvent); - - thread->schedule_in(c, HOST_DB_RETRY_PERIOD); - - return &c->action; + return getby(cont, process_hostdb_info, hash, opt); } Action * @@ -902,11 +826,7 @@ HostDBProcessor::iterate(Continuation *cont) c->current_iterate_pos = 0; SET_CONTINUATION_HANDLER(c, (HostDBContHandler)&HostDBContinuation::iterateEvent); - if (thread->mutex == cont->mutex) { - thread->schedule_in(c, HOST_DB_RETRY_PERIOD); - } else { - dnsProcessor.thread->schedule_imm(c); - } + thread->schedule_in(c, HOST_DB_RETRY_PERIOD); return &c->action; } @@ -966,8 +886,8 @@ HostDBProcessor::setby(const char *hostname, int len, sockaddr const *ip, HostDB // Attempt to find the result in-line, for level 1 hits - ProxyMutex *mutex = hostDB.refcountcache->lock_for_key(hash.hash.fold()); - EThread *thread = this_ethread(); + Ptr mutex = hostDB.refcountcache->lock_for_key(hash.hash.fold()); + EThread *thread = this_ethread(); MUTEX_TRY_LOCK(lock, mutex, thread); if (lock.is_locked()) { @@ -977,7 +897,7 @@ HostDBProcessor::setby(const char *hostname, int len, sockaddr const *ip, HostDB } return; } - // Create a continuation to do a deaper probe in the background + // Create a continuation to do a deeper probe in the background HostDBContinuation *c = hostDBContAllocator.alloc(); c->init(hash); @@ -1000,7 +920,7 @@ HostDBProcessor::setby_srv(const char *hostname, int len, const char *target, Ho hash.db_mark = HOSTDB_MARK_SRV; hash.refresh(); - // Create a continuation to do a deaper probe in the background + // Create a continuation to do a deeper probe in the background HostDBContinuation *c = hostDBContAllocator.alloc(); c->init(hash); @@ -1013,7 +933,7 @@ HostDBProcessor::setby_srv(const char *hostname, int len, const char *target, Ho int HostDBContinuation::setbyEvent(int /* event ATS_UNUSED */, Event * /* e ATS_UNUSED */) { - Ptr r = probe(mutex.get(), hash, false); + Ptr r = probe(mutex, hash, false); if (r) { do_setby(r.get(), &app, hash.host_name, hash.ip, is_srv()); @@ -1070,8 +990,11 @@ int HostDBContinuation::removeEvent(int /* event ATS_UNUSED */, Event *e) { Continuation *cont = action.continuation; - - MUTEX_TRY_LOCK(lock, cont ? cont->mutex.get() : (ProxyMutex *)nullptr, e->ethread); + Ptr proxy_mutex; + if (cont) { + proxy_mutex = cont->mutex; + } + MUTEX_TRY_LOCK(lock, proxy_mutex, e->ethread); if (!lock.is_locked()) { e->schedule_in(HOST_DB_RETRY_PERIOD); return EVENT_CONT; @@ -1082,7 +1005,7 @@ HostDBContinuation::removeEvent(int /* event ATS_UNUSED */, Event *e) cont->handleEvent(EVENT_HOST_DB_IP_REMOVED, (void *)nullptr); } } else { - Ptr r = probe(mutex.get(), hash, false); + Ptr r = probe(mutex, hash, false); bool res = remove_round_robin(r.get(), hash.host_name, hash.ip); if (cont) { cont->handleEvent(EVENT_HOST_DB_IP_REMOVED, res ? static_cast(&hash.ip) : static_cast(nullptr)); @@ -1291,7 +1214,7 @@ HostDBContinuation::dnsEvent(int event, HostEnt *e) ttl = failed ? 0 : e->ttl / 60; int ttl_seconds = failed ? 0 : e->ttl; // ebalsa: moving to second accuracy - Ptr old_r = probe(mutex.get(), hash, false); + Ptr old_r = probe(mutex, hash, false); // If the DNS lookup failed with NXDOMAIN, remove the old record if (e && e->isNameError() && old_r) { hostDB.refcountcache->erase(old_r->key); @@ -1392,7 +1315,7 @@ HostDBContinuation::dnsEvent(int event, HostEnt *e) if (is_rr) { r->app.rr.offset = offset; // This will only be set if is_rr - HostDBRoundRobin *rr_data = (HostDBRoundRobin *)(r->rr()); + HostDBRoundRobin *rr_data = static_cast(r->rr()); ; if (is_srv()) { int skip = 0; @@ -1491,7 +1414,7 @@ HostDBContinuation::dnsEvent(int event, HostEnt *e) } // We have seen cases were the action.mutex != action.continuation.mutex. - // Since reply_to_cont will call the hanlder on the action.continuation, it is important that we hold + // Since reply_to_cont will call the handler on the action.continuation, it is important that we hold // that mutex. bool need_to_reschedule = true; MUTEX_TRY_LOCK(lock, action.mutex, thread); @@ -1549,7 +1472,7 @@ HostDBContinuation::iterateEvent(int event, Event *e) // let's iterate through another record and then reschedule ourself. if (current_iterate_pos < hostDB.refcountcache->partition_count()) { // TODO: configurable number at a time? - ProxyMutex *bucket_mutex = hostDB.refcountcache->get_partition(current_iterate_pos).lock.get(); + Ptr bucket_mutex = hostDB.refcountcache->get_partition(current_iterate_pos).lock; MUTEX_TRY_LOCK(lock_bucket, bucket_mutex, t); if (!lock_bucket.is_locked()) { // we couldn't get the bucket lock, let's just reschedule and try later. @@ -1558,15 +1481,17 @@ HostDBContinuation::iterateEvent(int event, Event *e) return EVENT_CONT; } - TSHashTable *partMap = hostDB.refcountcache->get_partition(current_iterate_pos).get_map(); - for (RefCountCachePartition::iterator_type i = partMap->begin(); i != partMap->end(); ++i) { - HostDBInfo *r = (HostDBInfo *)i.m_value->item.get(); + IntrusiveHashMap &partMap = hostDB.refcountcache->get_partition(current_iterate_pos).get_map(); + for (const auto &it : partMap) { + HostDBInfo *r = static_cast(it.item.get()); if (r && !r->is_failed()) { action.continuation->handleEvent(EVENT_INTERVAL, static_cast(r)); } } - current_iterate_pos++; + } + + if (current_iterate_pos < hostDB.refcountcache->partition_count()) { // And reschedule ourselves to pickup the next bucket after HOST_DB_RETRY_PERIOD. Debug("hostdb", "iterateEvent event=%d eventp=%p: completed current iteration %ld of %ld", event, e, current_iterate_pos, hostDB.refcountcache->partition_count()); @@ -1592,10 +1517,10 @@ HostDBContinuation::probeEvent(int /* event ATS_UNUSED */, Event *e) EThread *t = e ? e->ethread : this_ethread(); MUTEX_TRY_LOCK(lock, action.mutex, t); - // Go ahead and grab the continuation mutex or just grab the action mutex again of there is no continuation mutex - MUTEX_TRY_LOCK(lock2, (action.continuation && action.continuation->mutex) ? action.continuation->mutex : action.mutex, t); - // Don't continue unless we have both mutexes - if (!lock.is_locked() || !lock2.is_locked()) { + + // Separating lock checks here to make sure things don't break + // when we check if the action is cancelled. + if (!lock.is_locked()) { mutex->thread_holding->schedule_in(this, HOST_DB_RETRY_PERIOD); return EVENT_CONT; } @@ -1605,6 +1530,14 @@ HostDBContinuation::probeEvent(int /* event ATS_UNUSED */, Event *e) return EVENT_DONE; } + // Go ahead and grab the continuation mutex or just grab the action mutex again of there is no continuation mutex + MUTEX_TRY_LOCK(lock2, (action.continuation && action.continuation->mutex) ? action.continuation->mutex : action.mutex, t); + // Don't continue unless we have both mutexes + if (!lock2.is_locked()) { + mutex->thread_holding->schedule_in(this, HOST_DB_RETRY_PERIOD); + return EVENT_CONT; + } + if (!hostdb_enable || (!*hash.host_name && !hash.ip.isValid())) { if (action.continuation) { action.continuation->handleEvent(EVENT_HOST_DB_LOOKUP, nullptr); @@ -1616,7 +1549,7 @@ HostDBContinuation::probeEvent(int /* event ATS_UNUSED */, Event *e) if (!force_dns) { // Do the probe // - Ptr r = probe(mutex.get(), hash, false); + Ptr r = probe(mutex, hash, false); if (r) { HOSTDB_INCREMENT_DYN_STAT(hostdb_total_hits_stat); @@ -1626,7 +1559,7 @@ HostDBContinuation::probeEvent(int /* event ATS_UNUSED */, Event *e) reply_to_cont(action.continuation, r.get()); } - // If it suceeds or it was a remote probe, we are done + // If it succeeds or it was a remote probe, we are done // if (r) { hostdb_cont_free(this); @@ -1643,8 +1576,9 @@ int HostDBContinuation::set_check_pending_dns() { Queue &q = hostDB.pending_dns_for_hash(hash.hash); - HostDBContinuation *c = q.head; - for (; c; c = (HostDBContinuation *)c->link.next) { + this->setThreadAffinity(this_ethread()); + HostDBContinuation *c = q.head; + for (; c; c = static_cast(c->link.next)) { if (hash.hash == c->hash.hash) { Debug("hostdb", "enqueuing additional request"); q.enqueue(this); @@ -1663,7 +1597,7 @@ HostDBContinuation::remove_trigger_pending_dns() HostDBContinuation *c = q.head; Queue qq; while (c) { - HostDBContinuation *n = (HostDBContinuation *)c->link.next; + HostDBContinuation *n = static_cast(c->link.next); if (hash.hash == c->hash.hash) { Debug("hostdb", "dequeuing additional request"); q.remove(c); @@ -1671,8 +1605,15 @@ HostDBContinuation::remove_trigger_pending_dns() } c = n; } + EThread *thread = this_ethread(); while ((c = qq.dequeue())) { - c->handleEvent(EVENT_IMMEDIATE, nullptr); + // resume all queued HostDBCont in the thread associated with the netvc to avoid nethandler locking issues. + EThread *affinity_thread = c->getThreadAffinity(); + if (!affinity_thread || affinity_thread == thread) { + c->handleEvent(EVENT_IMMEDIATE, nullptr); + } else { + eventProcessor.schedule_imm(c); + } } } @@ -1889,7 +1830,7 @@ struct ShowHostDB : public ShowCont { if (event == EVENT_INTERVAL) { HostDBInfo *r = reinterpret_cast(e); if (output_json && records_seen++ > 0) { - CHECK_SHOW(show(",")); // we need to seperate records + CHECK_SHOW(show(",")); // we need to separate records } showOne(r, false, event, e); if (r->round_robin) { @@ -1911,7 +1852,7 @@ struct ShowHostDB : public ShowCont { for (int i = 0; i < rr_data->rrcount; i++) { showOne(&rr_data->info(i), true, event, e, rr_data); if (output_json) { - CHECK_SHOW(show("}")); // we need to seperate records + CHECK_SHOW(show("}")); // we need to separate records if (i < (rr_data->rrcount - 1)) CHECK_SHOW(show(",")); } @@ -2012,7 +1953,7 @@ struct ShowHostDB : public ShowCont { int showLookupDone(int event, Event *e) { - HostDBInfo *r = (HostDBInfo *)e; + HostDBInfo *r = reinterpret_cast(e); CHECK_SHOW(begin("HostDB Lookup")); if (name) { @@ -2111,8 +2052,8 @@ register_ShowHostDB(Continuation *c, HTTPHdr *h) return &s->action; } -#define HOSTDB_TEST_MAX_OUTSTANDING 100 -#define HOSTDB_TEST_LENGTH 100000 +static constexpr int HOSTDB_TEST_MAX_OUTSTANDING = 20; +static constexpr int HOSTDB_TEST_LENGTH = 200; struct HostDBTestReverse; using HostDBTestReverseHandler = int (HostDBTestReverse::*)(int, void *); @@ -2121,34 +2062,26 @@ struct HostDBTestReverse : public Continuation { int type; int *status; - int outstanding; - int total; -#if HAVE_LRAND48_R - struct drand48_data dr; -#endif + int outstanding = 0; + int total = 0; + std::ranlux48 randu; int mainEvent(int event, Event *e) { if (event == EVENT_HOST_DB_LOOKUP) { - HostDBInfo *i = (HostDBInfo *)e; + HostDBInfo *i = reinterpret_cast(e); if (i) { rprintf(test, "HostDBTestReverse: reversed %s\n", i->hostname()); } outstanding--; } while (outstanding < HOSTDB_TEST_MAX_OUTSTANDING && total < HOSTDB_TEST_LENGTH) { - long l = 0; -#if HAVE_LRAND48_R - lrand48_r(&dr, &l); -#else - l = lrand48(); -#endif IpEndpoint ip; - ip.sin.sin_addr.s_addr = static_cast(l); + ip.assign(IpAddr(static_cast(randu()))); outstanding++; total++; - if (!(outstanding % 1000)) { + if (!(outstanding % 100)) { rprintf(test, "HostDBTestReverse: %d\n", total); } hostDBProcessor.getbyaddr_re(this, &ip.sa); @@ -2161,14 +2094,10 @@ struct HostDBTestReverse : public Continuation { return EVENT_CONT; } HostDBTestReverse(RegressionTest *t, int atype, int *astatus) - : Continuation(new_ProxyMutex()), test(t), type(atype), status(astatus), outstanding(0), total(0) + : Continuation(new_ProxyMutex()), test(t), type(atype), status(astatus) { SET_HANDLER((HostDBTestReverseHandler)&HostDBTestReverse::mainEvent); -#if HAVE_SRAND48_R - srand48_r(time(nullptr), &dr); -#else - srand48(time(nullptr)); -#endif + randu.seed(std::chrono::system_clock::now().time_since_epoch().count()); } }; @@ -2182,11 +2111,11 @@ REGRESSION_TEST(HostDBTests)(RegressionTest *t, int atype, int *pstatus) RecRawStatBlock *hostdb_rsb; void -ink_hostdb_init(ModuleVersion v) +ink_hostdb_init(ts::ModuleVersion v) { static int init_called = 0; - ink_release_assert(!checkModuleVersion(v, HOSTDB_MODULE_VERSION)); + ink_release_assert(v.check(HOSTDB_MODULE_INTERNAL_VERSION)); if (init_called) { return; } @@ -2351,7 +2280,7 @@ ParseHostFile(const char *path, unsigned int hostdb_hostfile_check_interval) // Regression tests // // Take a started hostDB and fill it up and make sure it doesn't explode -#ifdef TS_HAS_TESTS +#if TS_HAS_TESTS struct HostDBRegressionContinuation; struct HostDBRegressionContinuation : public Continuation { @@ -2404,7 +2333,7 @@ struct HostDBRegressionContinuation : public Continuation { hostDBProcessor.getbyname_re(this, hostnames[i++], 0); return EVENT_CONT; } else { - rprintf(test, "HostDBTestRR: %d outstanding %d succcess %d failure\n", outstanding, success, failure); + rprintf(test, "HostDBTestRR: %d outstanding %d success %d failure\n", outstanding, success, failure); if (success == hosts) { *status = REGRESSION_TEST_PASSED; } else { diff --git a/iocore/hostdb/I_HostDB.h b/iocore/hostdb/I_HostDB.h index f21e870c6a0..d9aecd8d8c3 100644 --- a/iocore/hostdb/I_HostDB.h +++ b/iocore/hostdb/I_HostDB.h @@ -39,7 +39,4 @@ // TS-1925: switch from MMH to MD5 hash; bumped to version 2 // switch from MD5 to SHA256 hash; bumped to version 3 // 2.1: Switched to mark RR elements. -#define HOSTDB_MODULE_MAJOR_VERSION 3 -#define HOSTDB_MODULE_MINOR_VERSION 1 - -#define HOSTDB_MODULE_VERSION makeModuleVersion(HOSTDB_MODULE_MAJOR_VERSION, HOSTDB_MODULE_MINOR_VERSION, PUBLIC_MODULE_HEADER) +static constexpr ts::ModuleVersion HOSTDB_MODULE_PUBLIC_VERSION(3, 1); diff --git a/iocore/hostdb/I_HostDBProcessor.h b/iocore/hostdb/I_HostDBProcessor.h index 3c4c64263cd..3d15faf8423 100644 --- a/iocore/hostdb/I_HostDBProcessor.h +++ b/iocore/hostdb/I_HostDBProcessor.h @@ -51,7 +51,7 @@ struct HostDBContinuation; // IP address. // // Since host information is relatively small, we can afford to have -// a reasonable size memory cache, and use a (relatively) sparce +// a reasonable size memory cache, and use a (relatively) sparse // disk representation to decrease # of seeks. // extern int hostdb_enable; @@ -84,7 +84,7 @@ makeHostHash(const char *string) // // This structure contains the host information required by // the application. Except for the initial fields it -// is treated as opacque by the database. +// is treated as opaque by the database. // union HostDBApplicationInfo { @@ -163,10 +163,10 @@ struct HostDBInfo : public RefCountObj { } // return a version number-- so we can manage compatibility with the marshal/unmarshal - static VersionNumber + static ts::VersionNumber version() { - return VersionNumber(1, 0); + return ts::VersionNumber(1, 0); } static HostDBInfo * @@ -349,13 +349,13 @@ struct HostDBInfo : public RefCountObj { struct HostDBRoundRobin { /** Total number (to compute space used). */ - short rrcount; + short rrcount = 0; /** Number which have not failed a connect. */ - short good; + short good = 0; - unsigned short current; - ink_time_t timed_rr_ctime; + unsigned short current = 0; + ink_time_t timed_rr_ctime = 0; // This is the equivalent of a variable length array, we can't use a VLA because // HostDBInfo is a non-POD type-- so this is the best we can do. @@ -389,19 +389,19 @@ struct HostDBRoundRobin { HostDBInfo *select_next(sockaddr const *addr); HostDBInfo *select_best_http(sockaddr const *client_ip, ink_time_t now, int32_t fail_window); HostDBInfo *select_best_srv(char *target, InkRand *rand, ink_time_t now, int32_t fail_window); - HostDBRoundRobin() : rrcount(0), good(0), current(0), timed_rr_ctime(0) {} + HostDBRoundRobin() {} }; struct HostDBCache; +struct HostDBHash; -// Prototype for inline completion functionf or +// Prototype for inline completion function or // getbyname_imm() -typedef void (Continuation::*process_hostdb_info_pfn)(HostDBInfo *r); -typedef void (Continuation::*process_srv_info_pfn)(HostDBInfo *r); +typedef void (Continuation::*cb_process_result_pfn)(HostDBInfo *r); Action *iterate(Continuation *cont); -/** The Host Databse access interface. */ +/** The Host Database access interface. */ struct HostDBProcessor : public Processor { friend struct HostDBSync; // Public Interface @@ -424,13 +424,13 @@ struct HostDBProcessor : public Processor { /// Optional parameters for getby... struct Options { - typedef Options self; ///< Self reference type. - int port; ///< Target service port (default 0 -> don't care) - int flags; ///< Processing flags (default HOSTDB_DO_NOT_FORCE_DNS) - int timeout; ///< Timeout value (default 0 -> default timeout) - HostResStyle host_res_style; ///< How to query host (default HOST_RES_IPV4) + typedef Options self; ///< Self reference type. + int port = 0; ///< Target service port (default 0 -> don't care) + int flags = HOSTDB_DO_NOT_FORCE_DNS; ///< Processing flags (default HOSTDB_DO_NOT_FORCE_DNS) + int timeout = 0; ///< Timeout value (default 0 -> default timeout) + HostResStyle host_res_style = HOST_RES_IPV4; ///< How to query host (default HOST_RES_IPV4) - Options() : port(0), flags(HOSTDB_DO_NOT_FORCE_DNS), timeout(0), host_res_style(HOST_RES_IPV4) {} + Options() {} /// Set the flags. self & setFlags(int f) @@ -448,20 +448,16 @@ struct HostDBProcessor : public Processor { Action *getbynameport_re(Continuation *cont, const char *hostname, int len, Options const &opt = DEFAULT_OPTIONS); - Action *getSRVbyname_imm(Continuation *cont, process_srv_info_pfn process_srv_info, const char *hostname, int len, + Action *getSRVbyname_imm(Continuation *cont, cb_process_result_pfn process_srv_info, const char *hostname, int len, Options const &opt = DEFAULT_OPTIONS); - Action *getbyname_imm(Continuation *cont, process_hostdb_info_pfn process_hostdb_info, const char *hostname, int len, + Action *getbyname_imm(Continuation *cont, cb_process_result_pfn process_hostdb_info, const char *hostname, int len, Options const &opt = DEFAULT_OPTIONS); Action *iterate(Continuation *cont); /** Lookup Hostinfo by addr */ - Action * - getbyaddr_re(Continuation *cont, sockaddr const *aip) - { - return getby(cont, nullptr, 0, aip, false, HOST_RES_NONE, 0); - } + Action *getbyaddr_re(Continuation *cont, sockaddr const *aip); /** Set the application information (fire-and-forget). */ void @@ -500,8 +496,7 @@ struct HostDBProcessor : public Processor { HostDBCache *cache(); private: - Action *getby(Continuation *cont, const char *hostname, int len, sockaddr const *ip, bool aforce_dns, HostResStyle host_res_style, - int timeout); + Action *getby(Continuation *cont, cb_process_result_pfn cb_process_result, HostDBHash &hash, Options const &opt); public: /** Set something. @@ -522,4 +517,4 @@ void run_HostDBTest(); extern inkcoreapi HostDBProcessor hostDBProcessor; -void ink_hostdb_init(ModuleVersion version); +void ink_hostdb_init(ts::ModuleVersion version); diff --git a/iocore/hostdb/Makefile.am b/iocore/hostdb/Makefile.am index 1ae57e313b0..8f49b12d686 100644 --- a/iocore/hostdb/Makefile.am +++ b/iocore/hostdb/Makefile.am @@ -74,7 +74,7 @@ test_LD_ADD = \ $(top_builddir)/src/tscore/libtscore.la \ $(top_builddir)/src/tscpp/util/libtscpputil.la \ $(top_builddir)/proxy/shared/libUglyLogStubs.a \ - @LIBTCL@ @HWLOC_LIBS@ + @HWLOC_LIBS@ test_RefCountCache_CPPFLAGS = $(test_CPP_FLAGS) diff --git a/iocore/hostdb/P_HostDB.h b/iocore/hostdb/P_HostDB.h index 0e7e281e3fa..43c5a5ba4bc 100644 --- a/iocore/hostdb/P_HostDB.h +++ b/iocore/hostdb/P_HostDB.h @@ -45,8 +45,8 @@ #include "P_RefCountCache.h" #include "P_HostDBProcessor.h" -#undef HOSTDB_MODULE_VERSION -#define HOSTDB_MODULE_VERSION makeModuleVersion(HOSTDB_MODULE_MAJOR_VERSION, HOSTDB_MODULE_MINOR_VERSION, PRIVATE_MODULE_HEADER) -Ptr probe(ProxyMutex *mutex, CryptoHash const &hash, bool ignore_timeout); +static constexpr ts::ModuleVersion HOSTDB_MODULE_INTERNAL_VERSION{HOSTDB_MODULE_PUBLIC_VERSION, ts::ModuleVersion::PRIVATE}; + +Ptr probe(Ptr mutex, CryptoHash const &hash, bool ignore_timeout); void make_crypto_hash(CryptoHash &hash, const char *hostname, int len, int port, const char *pDNSServers, HostDBMark mark); diff --git a/iocore/hostdb/P_HostDBProcessor.h b/iocore/hostdb/P_HostDBProcessor.h index 15d1c47f704..9739bf6a6eb 100644 --- a/iocore/hostdb/P_HostDBProcessor.h +++ b/iocore/hostdb/P_HostDBProcessor.h @@ -194,12 +194,12 @@ struct HostDBCache { // Map to contain all of the host file overrides, initialize it to empty Ptr hosts_file_ptr; // TODO: make ATS call a close() method or something on shutdown (it does nothing of the sort today) - RefCountCache *refcountcache; + RefCountCache *refcountcache = nullptr; // TODO configurable number of items in the cache - Queue *pending_dns; + Queue *pending_dns = nullptr; Queue &pending_dns_for_hash(const CryptoHash &hash); - Queue *remoteHostDBQueue; + Queue *remoteHostDBQueue = nullptr; HostDBCache(); bool is_pending_dns_for_hash(const CryptoHash &hash); }; @@ -306,10 +306,9 @@ HostDBRoundRobin::select_best_http(sockaddr const *client_ip, ink_time_t now, in Debug("hostdb", "Using default round robin"); unsigned int best_hash_any = 0; unsigned int best_hash_up = 0; - sockaddr const *ip; for (int i = 0; i < good; i++) { - ip = info(i).ip(); - unsigned int h = HOSTDB_CLIENT_IP_HASH(client_ip, ip); + sockaddr const *ip = info(i).ip(); + unsigned int h = HOSTDB_CLIENT_IP_HASH(client_ip, ip); if (best_hash_any <= h) { best_any = i; best_hash_any = h; @@ -399,16 +398,16 @@ struct HostDBHash { CryptoHash hash; ///< The hash value. - const char *host_name; ///< Host name. - int host_len; ///< Length of @a _host_name - IpAddr ip; ///< IP address. - in_port_t port; ///< IP port (host order). + const char *host_name = nullptr; ///< Host name. + int host_len = 0; ///< Length of @a _host_name + IpAddr ip; ///< IP address. + in_port_t port = 0; ///< IP port (host order). /// DNS server. Not strictly part of the hash data but /// it's both used by @c HostDBContinuation and provides access to /// hash data. It's just handier to store it here for both uses. - DNSServer *dns_server; - SplitDNS *pSD; ///< Hold the container for @a dns_server. - HostDBMark db_mark; ///< Mark / type of record. + DNSServer *dns_server = nullptr; + SplitDNS *pSD = nullptr; ///< Hold the container for @a dns_server. + HostDBMark db_mark = HOSTDB_MARK_GENERIC; ///< Mark / type of record. /// Default constructor. HostDBHash(); @@ -490,12 +489,12 @@ struct HostDBContinuation : public Continuation { struct Options { typedef Options self; ///< Self reference type. - int timeout; ///< Timeout value. Default 0 - HostResStyle host_res_style; ///< IP address family fallback. Default @c HOST_RES_NONE - bool force_dns; ///< Force DNS lookup. Default @c false - Continuation *cont; ///< Continuation / action. Default @c nullptr (none) + int timeout = 0; ///< Timeout value. Default 0 + HostResStyle host_res_style = HOST_RES_NONE; ///< IP address family fallback. Default @c HOST_RES_NONE + bool force_dns = false; ///< Force DNS lookup. Default @c false + Continuation *cont = nullptr; ///< Continuation / action. Default @c nullptr (none) - Options() : timeout(0), host_res_style(HOST_RES_NONE), force_dns(false), cont(nullptr) {} + Options() {} }; static const Options DEFAULT_OPTIONS; ///< Default defaults. void init(HostDBHash const &hash, Options const &opt = DEFAULT_OPTIONS); diff --git a/iocore/hostdb/P_RefCountCache.h b/iocore/hostdb/P_RefCountCache.h index f876897c033..98073825888 100644 --- a/iocore/hostdb/P_RefCountCache.h +++ b/iocore/hostdb/P_RefCountCache.h @@ -25,7 +25,7 @@ #include #include // TODO: less? just need ET_TASK -#include "tscore/Map.h" +#include "tscore/IntrusiveHashMap.h" #include "tscore/PriorityQueue.h" #include "tscore/List.h" @@ -37,8 +37,10 @@ #define REFCOUNT_CACHE_EVENT_SYNC REFCOUNT_CACHE_EVENT_EVENTS_START #define REFCOUNTCACHE_MAGIC_NUMBER 0x0BAD2D9 -#define REFCOUNTCACHE_MAJOR_VERSION 1 -#define REFCOUNTCACHE_MINOR_VERSION 0 + +static constexpr unsigned char REFCOUNTCACHE_MAJOR_VERSION = 1; +static constexpr unsigned char REFCOUNTCACHE_MINOR_VERSION = 0; +static constexpr ts::VersionNumber REFCOUNTCACHE_VERSION(1, 0); // Stats enum RefCountCache_Stats { @@ -73,12 +75,13 @@ class RefCountCacheHashEntry { public: Ptr item; - LINK(RefCountCacheHashEntry, item_link); - PriorityQueueEntry *expiry_entry; + RefCountCacheHashEntry *_next{nullptr}; + RefCountCacheHashEntry *_prev{nullptr}; + PriorityQueueEntry *expiry_entry = nullptr; RefCountCacheItemMeta meta; // Need a no-argument constructor to use the classAllocator - RefCountCacheHashEntry() : item(Ptr()), expiry_entry(nullptr), meta(0, 0) {} + RefCountCacheHashEntry() : item(Ptr()), meta(0, 0) {} void set(RefCountObj *i, uint64_t key, unsigned int size, int expire_time) { @@ -113,24 +116,32 @@ class RefCountCacheHashEntry // Since the hashing values are all fixed size, we can simply use a classAllocator to avoid mallocs extern ClassAllocator> expiryQueueEntry; -struct RefCountCacheHashing { - typedef uint64_t ID; - typedef uint64_t const Key; - typedef RefCountCacheHashEntry Value; - typedef DList(RefCountCacheHashEntry, item_link) ListHead; +struct RefCountCacheLinkage { + using key_type = uint64_t const; + using value_type = RefCountCacheHashEntry; - static ID - hash(Key key) + static value_type *& + next_ptr(value_type *value) + { + return value->_next; + } + static value_type *& + prev_ptr(value_type *value) + { + return value->_prev; + } + static uint64_t + hash_of(key_type key) { return key; } - static Key - key(Value const *value) + static key_type + key_of(value_type *v) { - return value->meta.key; + return v->meta.key; } static bool - equal(Key lhs, Key rhs) + equal(key_type lhs, key_type rhs) { return lhs == rhs; } @@ -141,6 +152,8 @@ struct RefCountCacheHashing { template class RefCountCachePartition { public: + using hash_type = IntrusiveHashMap; + RefCountCachePartition(unsigned int part_num, uint64_t max_size, unsigned int max_items, RecRawStatBlock *rsb = nullptr); Ptr get(uint64_t key); void put(uint64_t key, C *item, int size = 0, int expire_time = 0); @@ -149,15 +162,12 @@ template class RefCountCachePartition void clear(); bool is_full() const; bool make_space_for(unsigned int); - template void dealloc_entry(Iterator ptr); + void dealloc_entry(hash_type::iterator ptr); size_t count() const; void copy(std::vector &items); - typedef typename TSHashTable::iterator iterator_type; - typedef typename TSHashTable::self hash_type; - typedef typename TSHashTable::Location location_type; - TSHashTable *get_map(); + hash_type &get_map(); Ptr lock; // Lock @@ -188,11 +198,10 @@ Ptr RefCountCachePartition::get(uint64_t key) { this->metric_inc(refcountcache_total_lookups_stat, 1); - location_type l = this->item_map.find(key); - if (l.isValid()) { + if (auto it = this->item_map.find(key); it != this->item_map.end()) { // found this->metric_inc(refcountcache_total_hits_stat, 1); - return make_ptr((C *)l.m_value->item.get()); + return make_ptr(static_cast(it->item.get())); } else { return Ptr(); } @@ -239,44 +248,37 @@ template void RefCountCachePartition::erase(uint64_t key, ink_time_t expiry_time) { - location_type l = this->item_map.find(key); - if (l.isValid()) { - if (expiry_time >= 0 && l.m_value->meta.expiry_time != expiry_time) { + if (auto it = this->item_map.find(key); it != this->item_map.end()) { + if (expiry_time >= 0 && it->meta.expiry_time != expiry_time) { return; } - - // TSHashMap does NOT clean up the item-- this remove just removes it from the map - // we are responsible for cleaning it up here - this->item_map.remove(l); - this->dealloc_entry(l); + this->item_map.erase(it); + this->dealloc_entry(it); } } template -template void -RefCountCachePartition::dealloc_entry(Iterator ptr) +RefCountCachePartition::dealloc_entry(hash_type::iterator ptr) { - if (ptr.m_value) { - // decrement usag are not cleaned up. The values are not touched in this method, therefore it is safe - // counters - this->size -= ptr->meta.size; - this->items--; - - this->metric_inc(refcountcache_current_size_stat, -((int64_t)ptr->meta.size)); - this->metric_inc(refcountcache_current_items_stat, -1); - - // remove from expiry queue - if (ptr->expiry_entry != nullptr) { - Debug("refcountcache", "partition %d deleting item from expiry_queue idx=%d", this->part_num, ptr->expiry_entry->index); - - this->expiry_queue.erase(ptr->expiry_entry); - expiryQueueEntry.free(ptr->expiry_entry); - ptr->expiry_entry = nullptr; // To avoid the destruction of `l` calling the destructor again-- and causing issues - } + // decrement usage are not cleaned up. The values are not touched in this method, therefore it is safe + // counters + this->size -= ptr->meta.size; + this->items--; + + this->metric_inc(refcountcache_current_size_stat, -((int64_t)ptr->meta.size)); + this->metric_inc(refcountcache_current_items_stat, -1); - RefCountCacheHashEntry::free(ptr.m_value); + // remove from expiry queue + if (ptr->expiry_entry != nullptr) { + Debug("refcountcache", "partition %d deleting item from expiry_queue idx=%d", this->part_num, ptr->expiry_entry->index); + + this->expiry_queue.erase(ptr->expiry_entry); + expiryQueueEntry.free(ptr->expiry_entry); + ptr->expiry_entry = nullptr; // To avoid the destruction of `l` calling the destructor again-- and causing issues } + + RefCountCacheHashEntry::free(ptr); } template @@ -286,12 +288,12 @@ RefCountCachePartition::clear() // Since the hash nodes embed the list pointers, you can't iterate over the // hash elements and deallocate them, let alone remove them from the hash. // Hence, this monstrosity. - while (this->item_map.count() > 0) { - location_type pos = this->item_map.find(this->item_map.begin().m_value); + auto it = this->item_map.begin(); + while (it != this->item_map.end()) { + auto cur = it++; - ink_assert(pos.isValid()); - this->item_map.remove(pos); - this->dealloc_entry(pos); + this->item_map.erase(cur); + this->dealloc_entry(cur); } } @@ -339,9 +341,9 @@ template void RefCountCachePartition::copy(std::vector &items) { - for (RefCountCachePartition::iterator_type i = this->item_map.begin(); i != this->item_map.end(); ++i) { + for (auto &&it : this->item_map) { RefCountCacheHashEntry *val = RefCountCacheHashEntry::alloc(); - val->set(i.m_value->item.get(), i.m_value->meta.key, i.m_value->meta.size, i.m_value->meta.expiry_time); + val->set(it.item.get(), it.meta.key, it.meta.size, it.meta.expiry_time); items.push_back(val); } } @@ -356,21 +358,21 @@ RefCountCachePartition::metric_inc(RefCountCache_Stats metric_enum, int64_t d } template -TSHashTable * +IntrusiveHashMap & RefCountCachePartition::get_map() { - return &this->item_map; + return this->item_map; } // The header for the cache, this is used to check if the serialized cache is compatible class RefCountCacheHeader { public: - unsigned int magic; - VersionNumber version; - VersionNumber object_version; // version passed in of whatever it is we are caching + unsigned int magic = REFCOUNTCACHE_MAGIC_NUMBER; + ts::VersionNumber version{REFCOUNTCACHE_VERSION}; + ts::VersionNumber object_version; // version passed in of whatever it is we are caching - RefCountCacheHeader(VersionNumber object_version = VersionNumber()); + RefCountCacheHeader(ts::VersionNumber object_version = ts::VersionNumber()); bool operator==(const RefCountCacheHeader other) const; bool compatible(RefCountCacheHeader *other) const; }; @@ -393,8 +395,8 @@ template class RefCountCache { public: // Constructor - RefCountCache(unsigned int num_partitions, int size = -1, int items = -1, VersionNumber object_version = VersionNumber(), - std::string metrics_prefix = ""); + RefCountCache(unsigned int num_partitions, int size = -1, int items = -1, ts::VersionNumber object_version = ts::VersionNumber(), + const std::string &metrics_prefix = ""); // Destructor ~RefCountCache(); @@ -406,7 +408,7 @@ template class RefCountCache // Some methods to get some internal state int partition_for_key(uint64_t key); - ProxyMutex *lock_for_key(uint64_t key); + Ptr lock_for_key(uint64_t key); size_t partition_count() const; RefCountCachePartition &get_partition(int pnum); size_t count() const; @@ -424,8 +426,8 @@ template class RefCountCache }; template -RefCountCache::RefCountCache(unsigned int num_partitions, int size, int items, VersionNumber object_version, - std::string metrics_prefix) +RefCountCache::RefCountCache(unsigned int num_partitions, int size, int items, ts::VersionNumber object_version, + const std::string &metrics_prefix) : header(RefCountCacheHeader(object_version)), rsb(nullptr) { this->max_size = size; @@ -508,10 +510,10 @@ RefCountCache::get_header() } template -ProxyMutex * +Ptr RefCountCache::lock_for_key(uint64_t key) { - return this->partitions[this->partition_for_key(key)]->lock.get(); + return this->partitions[this->partition_for_key(key)]->lock; } template @@ -566,7 +568,7 @@ RefCountCache::clear() // Errors are -1 template int -LoadRefCountCacheFromPath(RefCountCache &cache, std::string dirname, std::string filepath, +LoadRefCountCacheFromPath(RefCountCache &cache, const std::string &dirname, const std::string &filepath, CacheEntryType *(*load_func)(char *, unsigned int)) { // If we have no load method, then we can't load anything so lets just stop right here diff --git a/iocore/hostdb/P_RefCountCacheSerializer.h b/iocore/hostdb/P_RefCountCacheSerializer.h index c153e7f5639..e76b7513564 100644 --- a/iocore/hostdb/P_RefCountCacheSerializer.h +++ b/iocore/hostdb/P_RefCountCacheSerializer.h @@ -25,6 +25,7 @@ #include "P_RefCountCache.h" +#include #include // This continuation is responsible for persisting RefCountCache to disk @@ -57,7 +58,7 @@ template class RefCountCacheSerializer : public Continuation int write_to_disk(const void *, size_t); RefCountCacheSerializer(Continuation *acont, RefCountCache *cc, int frequency, std::string dirname, std::string filename); - ~RefCountCacheSerializer(); + ~RefCountCacheSerializer() override; private: std::vector partition_items; @@ -85,8 +86,8 @@ RefCountCacheSerializer::RefCountCacheSerializer(Continuation *acont, RefCoun cache(cc), cont(acont), fd(-1), - dirname(dirname), - filename(filename), + dirname(std::move(dirname)), + filename(std::move(filename)), time_per_partition(HRTIME_SECONDS(frequency) / cc->partition_count()), start(Thread::get_hrtime()), total_items(0), @@ -291,7 +292,7 @@ RefCountCacheSerializer::finalize_sync() return error; } - // Don't bother checking for errors on the close since theere's nothing we can do about it at + // Don't bother checking for errors on the close since there's nothing we can do about it at // this point anyway. socketManager.close(dirfd); socketManager.close(this->fd); diff --git a/iocore/hostdb/RefCountCache.cc b/iocore/hostdb/RefCountCache.cc index 26abc75ea71..b7cecae96c7 100644 --- a/iocore/hostdb/RefCountCache.cc +++ b/iocore/hostdb/RefCountCache.cc @@ -38,12 +38,7 @@ RefCountCacheHashEntry::dealloc(RefCountCacheHashEntry *e) return refCountCacheHashingValueAllocator.free(e); } -RefCountCacheHeader::RefCountCacheHeader(VersionNumber object_version) - : magic(REFCOUNTCACHE_MAGIC_NUMBER), object_version(object_version) -{ - this->version.ink_major = REFCOUNTCACHE_MAJOR_VERSION; - this->version.ink_minor = REFCOUNTCACHE_MINOR_VERSION; -}; +RefCountCacheHeader::RefCountCacheHeader(ts::VersionNumber object_version) : object_version(object_version){}; bool RefCountCacheHeader::operator==(const RefCountCacheHeader other) const @@ -54,6 +49,6 @@ RefCountCacheHeader::operator==(const RefCountCacheHeader other) const bool RefCountCacheHeader::compatible(RefCountCacheHeader *other) const { - return (this->magic == other->magic && this->version.ink_major == other->version.ink_major && - this->object_version.ink_major == other->version.ink_major); + return (this->magic == other->magic && this->version._major == other->version._major && + this->object_version._major == other->version._major); }; diff --git a/iocore/hostdb/test_I_HostDB.cc b/iocore/hostdb/test_I_HostDB.cc index a6d077eb4b1..ad283625fde 100644 --- a/iocore/hostdb/test_I_HostDB.cc +++ b/iocore/hostdb/test_I_HostDB.cc @@ -30,12 +30,13 @@ class HostDBTest : Continuation { }; +int main() { init_diags("net_test", nullptr); - ink_event_system_init(EVENT_SYSTEM_MODULE_VERSION); - ink_net_init(NET_SYSTEM_MODULE_VERSION); - ink_hostdb_init(HOSTDB_MODULE_VERSION); + ink_event_system_init(EVENT_SYSTEM_MODULE_PUBLIC_VERSION); + ink_net_init(NET_SYSTEM_MODULE_PUBLIC_VERSION); + ink_hostdb_init(HOSTDB_MODULE_PUBLIC_VERSION); signal(SIGPIPE, SIG_IGN); eventProcessor.start(2); diff --git a/iocore/hostdb/test_P_HostDB.cc b/iocore/hostdb/test_P_HostDB.cc index 5656543e8f8..7609c865d45 100644 --- a/iocore/hostdb/test_P_HostDB.cc +++ b/iocore/hostdb/test_P_HostDB.cc @@ -52,15 +52,17 @@ struct NetTesterSM : public Continuation { str = new char[r + 10]; reader->read(str, r); printf("%s", str); + delete[] str; fflush(stdout); break; case VC_EVENT_READ_COMPLETE: - /* FALLSTHROUGH */ + /* FALLTHROUGH */ case VC_EVENT_EOS: r = reader->read_avail(); str = new char[r + 10]; reader->read(str, r); printf("%s", str); + delete[] str; fflush(stdout); case VC_EVENT_ERROR: vc->do_io_close(); @@ -74,4 +76,7 @@ struct NetTesterSM : public Continuation { } }; -main() {} +int +main() +{ +} diff --git a/iocore/hostdb/test_RefCountCache.cc b/iocore/hostdb/test_RefCountCache.cc index b02ac604822..cfa8854570c 100644 --- a/iocore/hostdb/test_RefCountCache.cc +++ b/iocore/hostdb/test_RefCountCache.cc @@ -63,7 +63,7 @@ class ExampleStruct : public RefCountObj { this->idx = -1; items_freed.insert(this); - printf("freeing: %p items_freed.size(): %zd\n", this, items_freed.size()); + printf("freeing: %p items_freed.size(): %zu\n", this, items_freed.size()); } static ExampleStruct * @@ -211,7 +211,7 @@ test() Layout::create(); init_diags("", nullptr); RecProcessInit(mode_type); - ink_event_system_init(EVENT_SYSTEM_MODULE_VERSION); + ink_event_system_init(EVENT_SYSTEM_MODULE_PUBLIC_VERSION); int ret = 0; diff --git a/iocore/net/Connection.cc b/iocore/net/Connection.cc index 2384033a776..0b6d14db932 100644 --- a/iocore/net/Connection.cc +++ b/iocore/net/Connection.cc @@ -57,7 +57,7 @@ NetVCOptions::toString(addr_bind_style s) return ANY_ADDR == s ? "any" : INTF_ADDR == s ? "interface" : "foreign"; } -Connection::Connection() : fd(NO_FD), is_bound(false), is_connected(false), sock_type(0) +Connection::Connection() : fd(NO_FD) { memset(&addr, 0, sizeof(addr)); } @@ -140,7 +140,8 @@ Server::setup_fd_for_listen(bool non_blocking, const NetProcessor::AcceptOptions ink_assert(fd != NO_FD); - if (http_accept_filter) { + if (opt.etype == ET_NET && opt.defer_accept > 0) { + http_accept_filter = true; add_http_filter(fd); } @@ -241,6 +242,10 @@ Server::setup_fd_for_listen(bool non_blocking, const NetProcessor::AcceptOptions #endif } + if (opt.f_proxy_protocol) { + Debug("proxyprotocol", "Proxy Protocol enabled."); + } + #if defined(TCP_MAXSEG) if (NetProcessor::accept_mss > 0) { if ((res = safe_setsockopt(fd, IPPROTO_TCP, TCP_MAXSEG, (char *)&NetProcessor::accept_mss, sizeof(int))) < 0) { @@ -249,6 +254,22 @@ Server::setup_fd_for_listen(bool non_blocking, const NetProcessor::AcceptOptions } #endif +#ifdef TCP_DEFER_ACCEPT + // set tcp defer accept timeout if it is configured, this will not trigger an accept until there is + // data on the socket ready to be read + if (opt.defer_accept > 0 && (res = setsockopt(fd, IPPROTO_TCP, TCP_DEFER_ACCEPT, &opt.defer_accept, sizeof(int))) < 0) { + // FIXME: should we go to the error + // goto error; + Error("[Server::listen] Defer accept is configured but set failed: %d", errno); + } +#endif + +#ifdef TCP_INIT_CWND + if (opt.init_cwnd > 0 && (res = setsockopt(fd, IPPROTO_TCP, TCP_INIT_CWND, &opt.init_cwnd, sizeof(int))) < 0) { + Error("[Server::listen] Cannot set initial congestion window to %d error: %d", tcp_init_cwnd, errno); + } +#endif + if (non_blocking) { if ((res = safe_nonblocking(fd)) < 0) { goto Lerror; diff --git a/iocore/net/I_Net.h b/iocore/net/I_Net.h index 29cf0cb8a74..050777c3875 100644 --- a/iocore/net/I_Net.h +++ b/iocore/net/I_Net.h @@ -22,9 +22,9 @@ @section details Details - Net subsystem is a layer on top the operations sytem network apis. It + Net subsystem is a layer on top the operations system network apis. It provides an interface for accepting/creating new connection oriented - (TCP) and connection less (UDP) connetions and for reading/writing + (TCP) and connection less (UDP) connections and for reading/writing data through these. The net system can manage 1000s of connections very efficiently. Another advantage of using the net system is that the SMs dont have be concerned about differences in the net apis of @@ -50,12 +50,9 @@ #define NET_MAX_IOV UIO_MAXIOV #endif -#define NET_SYSTEM_MODULE_MAJOR_VERSION 1 -#define NET_SYSTEM_MODULE_MINOR_VERSION 0 -#define NET_SYSTEM_MODULE_VERSION \ - makeModuleVersion(NET_SYSTEM_MODULE_MAJOR_VERSION, NET_SYSTEM_MODULE_MINOR_VERSION, PUBLIC_MODULE_HEADER) +static constexpr ts::ModuleVersion NET_SYSTEM_MODULE_PUBLIC_VERSION(1, 0, ts::ModuleVersion::PUBLIC); -static int const NO_FD = -1; +static constexpr int NO_FD = -1; // All in milli-seconds extern int net_config_poll_timeout; @@ -64,6 +61,9 @@ extern int net_accept_period; extern int net_retry_delay; extern int net_throttle_delay; +extern std::string_view net_ccp_in; +extern std::string_view net_ccp_out; + #define NET_EVENT_OPEN (NET_EVENT_EVENTS_START) #define NET_EVENT_OPEN_FAILED (NET_EVENT_EVENTS_START + 1) #define NET_EVENT_ACCEPT (NET_EVENT_EVENTS_START + 2) @@ -93,4 +93,4 @@ extern int net_throttle_delay; #include "I_NetProcessor.h" #include "I_SessionAccept.h" -void ink_net_init(ModuleVersion version); +void ink_net_init(ts::ModuleVersion version); diff --git a/iocore/net/I_NetProcessor.h b/iocore/net/I_NetProcessor.h index acaa1a6515c..6eb5a406157 100644 --- a/iocore/net/I_NetProcessor.h +++ b/iocore/net/I_NetProcessor.h @@ -24,6 +24,7 @@ #pragma once +#include "tscore/IpMap.h" #include "I_EventSystem.h" #include "I_Socks.h" struct socks_conf_struct; @@ -79,6 +80,14 @@ class NetProcessor : public Processor /// Socket transmit buffer size. /// 0 => OS default. int send_bufsize; + /// defer accpet for @c sockopt. + /// 0 => OS default. + int defer_accept; +#ifdef TCP_INIT_CWND + /// tcp init cwnd for @c sockopt + /// OS default + int init_cwnd; +#endif /// Socket options for @c sockopt. /// 0 => do not set options. uint32_t sockopt_flags; @@ -96,6 +105,15 @@ class NetProcessor : public Processor */ bool f_inbound_transparent; + /** MPTCP enabled on listener. + @internal For logging and metrics purposes to know whether the + listener enabled MPTCP or not. + */ + bool f_mptcp; + + /// Proxy Protocol enabled + bool f_proxy_protocol; + /// Default constructor. /// Instance is constructed with default values. AcceptOptions() { this->reset(); } @@ -180,6 +198,8 @@ class NetProcessor : public Processor */ virtual void init() = 0; + virtual void init_socks() = 0; + inkcoreapi virtual NetVConnection *allocate_vc(EThread *) = 0; /** Private constructor. */ @@ -230,7 +250,7 @@ class NetProcessor : public Processor object. @code - netProcesors.accept(my_cont, ...); + netProcessor.accept(my_cont, ...); netProcessor.connect_re(my_cont, ...); @endcode diff --git a/iocore/net/I_NetVConnection.h b/iocore/net/I_NetVConnection.h index e9b0427a429..6f40159b30b 100644 --- a/iocore/net/I_NetVConnection.h +++ b/iocore/net/I_NetVConnection.h @@ -21,9 +21,11 @@ limitations under the License. */ - #pragma once +#include +#include + #include "tscore/ink_inet.h" #include "I_Action.h" #include "I_VConnection.h" @@ -32,8 +34,9 @@ #include "I_IOBuffer.h" #include "I_Socks.h" #include "ts/apidefs.h" -#include #include "YamlSNIConfig.h" +#include "tscpp/util/TextView.h" +#include "tscore/IpMap.h" #define CONNECT_SUCCESS 1 #define CONNECT_FAILURE 0 @@ -130,10 +133,12 @@ struct NetVCOptions { @see ip_family */ IpAddr local_ip; + /** Local port for connection. Set to 0 for "don't care" (default). - */ + */ uint16_t local_port; + /// How to bind the local address. /// @note Default is @c ANY_ADDR. addr_bind_style addr_binding; @@ -186,7 +191,20 @@ struct NetVCOptions { /** * Client certificate to use in response to OS's certificate request */ - ats_scoped_str clientCertificate; + const char *ssl_client_cert_name = nullptr; + /* + * File containing private key matching certificate + */ + const char *ssl_client_private_key_name = nullptr; + /* + * File containing CA certs for verifying origin's cert + */ + const char *ssl_client_ca_cert_name = nullptr; + /* + * Directory containing CA certs for verifying origin's cert + */ + const char *ssl_client_ca_cert_path = nullptr; + /// Reset all values to defaults. /** @@ -233,13 +251,6 @@ struct NetVCOptions { } return *this; } - self & - set_client_certname(const char *name) - { - clientCertificate = ats_strdup(name); - // clientCertificate = name; - return *this; - } self & operator=(self const &that) @@ -247,17 +258,16 @@ struct NetVCOptions { if (&that != this) { /* * It is odd but necessary to null the scoped string pointer here - * and then explicitly call release on them in the string assignements + * and then explicitly call release on them in the string assignments * below. * We a memcpy from that to this. This will put that's string pointers into * this's memory. Therefore we must first explicitly null out * this's original version of the string. The release after the * memcpy removes the extra reference to that's copy of the string - * Removing the release will eventualy cause a double free crash + * Removing the release will eventually cause a double free crash */ - sni_servername = nullptr; // release any current name. - ssl_servername = nullptr; - clientCertificate = nullptr; + sni_servername = nullptr; // release any current name. + ssl_servername = nullptr; memcpy(static_cast(this), &that, sizeof(self)); if (that.sni_servername) { sni_servername.release(); // otherwise we'll free the source string. @@ -267,10 +277,6 @@ struct NetVCOptions { ssl_servername.release(); // otherwise we'll free the source string. this->ssl_servername = ats_strdup(that.ssl_servername); } - if (that.clientCertificate) { - clientCertificate.release(); // otherwise we'll free the source string. - this->clientCertificate = ats_strdup(that.clientCertificate); - } } return *this; } @@ -352,7 +358,7 @@ class NetVConnection : public AnnotatedVConnection c->handleEvent(VC_EVENT_ERROR, vio) - signified that error occured during write. + signified that error occurred during write. @@ -362,7 +368,7 @@ class NetVConnection : public AnnotatedVConnection when it is destroyed. @param c continuation to be called back after (partial) write - @param nbytes no of bytes to write, if unknown msut be set to INT64_MAX + @param nbytes no of bytes to write, if unknown must be set to INT64_MAX @param buf source of data @param owner @return vio pointer @@ -372,11 +378,11 @@ class NetVConnection : public AnnotatedVConnection /** Closes the vconnection. A state machine MUST call do_io_close() - when it has finished with a VConenction. do_io_close() indicates + when it has finished with a VConnection. do_io_close() indicates that the VConnection can be deallocated. After a close has been called, the VConnection and underlying processor must NOT send any more events related to this VConnection to the state machine. - Likeswise, state machine must not access the VConnectuion or + Likewise, state machine must not access the VConnection or any returned VIOs after calling close. lerrno indicates whether a close is a normal close or an abort. The difference between a normal close and an abort depends on the underlying type of @@ -395,7 +401,7 @@ class NetVConnection : public AnnotatedVConnection IO_SHUTDOWN_READWRITE. Once a side of a VConnection is shutdown, no further I/O can be done on that side of the connections and the underlying processor MUST NOT send any further events - (INCLUDING TIMOUT EVENTS) to the state machine. The state machine + (INCLUDING TIMEOUT EVENTS) to the state machine. The state machine MUST NOT use any VIOs from a shutdown side of a connection. Even if both sides of a connection are shutdown, the state machine MUST still call do_io_close() when it wishes the @@ -432,7 +438,7 @@ class NetVConnection : public AnnotatedVConnection //////////////////////////////////////////////////////////// // Set the timeouts associated with this connection. // - // active_timeout is for the total elasped time of // + // active_timeout is for the total elapsed time of // // the connection. // // inactivity_timeout is the elapsed time from the time // // a read or a write was scheduled during which the // @@ -455,7 +461,7 @@ class NetVConnection : public AnnotatedVConnection that it does not keep any connections open for a really long time. - Timeout symantics: + Timeout semantics: Should a timeout occur, the state machine for the read side of the NetVConnection is signaled first assuming that a read has @@ -607,8 +613,8 @@ class NetVConnection : public AnnotatedVConnection // is enabled by SocksProxy SocksAddrType socks_addr; - unsigned int attributes; - EThread *thread; + unsigned int attributes = 0; + EThread *thread = nullptr; /// PRIVATE: The public interface is VIO::reenable() void reenable(VIO *vio) override = 0; @@ -641,6 +647,12 @@ class NetVConnection : public AnnotatedVConnection /** Set remote sock addr struct. */ virtual void set_remote_addr() = 0; + /** Set remote sock addr struct. */ + virtual void set_remote_addr(const sockaddr *) = 0; + + /** Set the MPTCP state for this connection */ + virtual void set_mptcp_state() = 0; + // for InkAPI bool get_is_internal_request() const @@ -660,6 +672,14 @@ class NetVConnection : public AnnotatedVConnection { return is_transparent; } + + /// Get the MPTCP state of the VC. + std::optional + get_mptcp_state() const + { + return mptcp_state; + } + /// Set the transparency state. void set_is_transparent(bool state = true) @@ -667,6 +687,19 @@ class NetVConnection : public AnnotatedVConnection is_transparent = state; } + /// Get the proxy protocol enabled flag + bool + get_is_proxy_protocol() const + { + return is_proxy_protocol; + } + /// Set the proxy protocol enabled flag on the port + void + set_is_proxy_protocol(bool state = true) + { + is_proxy_protocol = state; + } + virtual int populate_protocol(std::string_view *results, int n) const { @@ -683,32 +716,135 @@ class NetVConnection : public AnnotatedVConnection NetVConnection(const NetVConnection &) = delete; NetVConnection &operator=(const NetVConnection &) = delete; + enum class ProxyProtocolVersion { + UNDEFINED, + V1, + V2, + }; + + enum class ProxyProtocolData { + UNDEFINED, + SRC, + DST, + }; + + int + set_proxy_protocol_addr(const ProxyProtocolData src_or_dst, ts::TextView &ip_addr_str) + { + int ret = -1; + + if (src_or_dst == ProxyProtocolData::SRC) { + ret = ats_ip_pton(ip_addr_str, &pp_info.src_addr); + } else { + ret = ats_ip_pton(ip_addr_str, &pp_info.dst_addr); + } + return ret; + } + + int + set_proxy_protocol_src_addr(ts::TextView src) + { + return set_proxy_protocol_addr(ProxyProtocolData::SRC, src); + } + + int + set_proxy_protocol_dst_addr(ts::TextView src) + { + return set_proxy_protocol_addr(ProxyProtocolData::DST, src); + } + + int + set_proxy_protocol_port(const ProxyProtocolData src_or_dst, in_port_t port) + { + if (src_or_dst == ProxyProtocolData::SRC) { + pp_info.src_addr.port() = htons(port); + } else { + pp_info.dst_addr.port() = htons(port); + } + return port; + } + + int + set_proxy_protocol_src_port(in_port_t port) + { + return set_proxy_protocol_port(ProxyProtocolData::SRC, port); + } + + int + set_proxy_protocol_dst_port(in_port_t port) + { + return set_proxy_protocol_port(ProxyProtocolData::DST, port); + } + + void + set_proxy_protocol_version(const ProxyProtocolVersion ver) + { + pp_info.proxy_protocol_version = ver; + } + + ProxyProtocolVersion + get_proxy_protocol_version() + { + return pp_info.proxy_protocol_version; + } + + sockaddr const *get_proxy_protocol_addr(const ProxyProtocolData); + + sockaddr const * + get_proxy_protocol_src_addr() + { + return get_proxy_protocol_addr(ProxyProtocolData::SRC); + } + + uint16_t + get_proxy_protocol_src_port() + { + return ats_ip_port_host_order(this->get_proxy_protocol_addr(ProxyProtocolData::SRC)); + } + + sockaddr const * + get_proxy_protocol_dst_addr() + { + return get_proxy_protocol_addr(ProxyProtocolData::DST); + } + + uint16_t + get_proxy_protocol_dst_port() + { + return ats_ip_port_host_order(this->get_proxy_protocol_addr(ProxyProtocolData::DST)); + }; + + struct ProxyProtocol { + ProxyProtocolVersion proxy_protocol_version = ProxyProtocolVersion::UNDEFINED; + uint16_t ip_family; + IpEndpoint src_addr; + IpEndpoint dst_addr; + }; + + ProxyProtocol pp_info; + protected: IpEndpoint local_addr; IpEndpoint remote_addr; - bool got_local_addr; - bool got_remote_addr; + bool got_local_addr = false; + bool got_remote_addr = false; - bool is_internal_request; + bool is_internal_request = false; /// Set if this connection is transparent. - bool is_transparent; + bool is_transparent = false; + /// Set if proxy protocol is enabled + bool is_proxy_protocol = false; + /// This is essentially a tri-state, we leave it undefined to mean no MPTCP support + std::optional mptcp_state; /// Set if the next write IO that empties the write buffer should generate an event. - int write_buffer_empty_event; + int write_buffer_empty_event = 0; /// NetVConnection Context. - NetVConnectionContext_t netvc_context; + NetVConnectionContext_t netvc_context = NET_VCONNECTION_UNSET; }; -inline NetVConnection::NetVConnection() - : AnnotatedVConnection(nullptr), - attributes(0), - thread(nullptr), - got_local_addr(false), - got_remote_addr(false), - is_internal_request(false), - is_transparent(false), - write_buffer_empty_event(0), - netvc_context(NET_VCONNECTION_UNSET) +inline NetVConnection::NetVConnection() : AnnotatedVConnection(nullptr) + { ink_zero(local_addr); ink_zero(remote_addr); diff --git a/iocore/net/I_SessionAccept.h b/iocore/net/I_SessionAccept.h index c728175c6ce..7d746ae533f 100644 --- a/iocore/net/I_SessionAccept.h +++ b/iocore/net/I_SessionAccept.h @@ -56,7 +56,7 @@ class SessionAccept : public Continuation { public: SessionAccept(ProxyMutex *amutex) : Continuation(amutex) { SET_HANDLER(&SessionAccept::mainEvent); } - ~SessionAccept() {} + ~SessionAccept() override {} /** Accept a new connection on this session. diff --git a/iocore/net/I_Socks.h b/iocore/net/I_Socks.h index 0c2b5c61d6a..b1e7e5c75ba 100644 --- a/iocore/net/I_Socks.h +++ b/iocore/net/I_Socks.h @@ -56,7 +56,7 @@ enum { }; struct SocksAddrType { - unsigned char type; + unsigned char type = SOCKS_ATYPE_NONE; union { // mostly it is ipv4. in other cases we will xalloc(). unsigned char ipv4[4]; @@ -64,6 +64,6 @@ struct SocksAddrType { } addr; void reset(); - SocksAddrType() : type(SOCKS_ATYPE_NONE) { addr.buf = nullptr; } + SocksAddrType() { addr.buf = nullptr; } ~SocksAddrType() { reset(); } }; diff --git a/iocore/net/I_UDPConnection.h b/iocore/net/I_UDPConnection.h index f1be035cce0..d06920f52de 100644 --- a/iocore/net/I_UDPConnection.h +++ b/iocore/net/I_UDPConnection.h @@ -83,7 +83,7 @@ class UDPConnection : public Continuation void AddRef(); int GetRefCount(); - int getPortNum(void); + int getPortNum(); int GetSendGenerationNumber(); // const void SetLastSentPktTSSeqNum(int64_t sentSeqNum); diff --git a/iocore/net/I_UDPNet.h b/iocore/net/I_UDPNet.h index 9e4667dc38c..846c3a78e07 100644 --- a/iocore/net/I_UDPNet.h +++ b/iocore/net/I_UDPNet.h @@ -46,7 +46,7 @@ class UDPNetProcessor : public Processor public: int start(int n_upd_threads, size_t stacksize) override = 0; - // this function was interanal intially.. this is required for public and + // this function was internal initially.. this is required for public and // interface probably should change. bool CreateUDPSocket(int *resfd, sockaddr const *remote_addr, Action **status, NetVCOptions &opt); @@ -69,7 +69,7 @@ class UDPNetProcessor : public Processor @param recv_bufsize (optional) Socket buffer size for sending. Limits how much can be queued by OS before we read it. @return Action* Always returns ACTION_RESULT_DONE if socket was - created successfuly, or ACTION_IO_ERROR if not. + created successfully, or ACTION_IO_ERROR if not. */ inkcoreapi Action *UDPBind(Continuation *c, sockaddr const *addr, int send_bufsize = 0, int recv_bufsize = 0); diff --git a/iocore/net/Makefile.am b/iocore/net/Makefile.am index ed407751ec0..7e27fe5a894 100644 --- a/iocore/net/Makefile.am +++ b/iocore/net/Makefile.am @@ -76,7 +76,7 @@ test_UDPNet_LDADD = \ $(top_builddir)/src/tscore/libtscore.la $(top_builddir)/src/tscpp/util/libtscpputil.la \ $(top_builddir)/proxy/ParentSelectionStrategy.o \ $(top_builddir)/lib/tsconfig/libtsconfig.la \ - @LIBTCL@ @HWLOC_LIBS@ @OPENSSL_LIBS@ @YAMLCPP_LIBS@ + @HWLOC_LIBS@ @OPENSSL_LIBS@ @LIBPCRE@ @YAMLCPP_LIBS@ test_UDPNet_SOURCES = \ test_I_UDPNet.cc @@ -130,11 +130,13 @@ libinknet_a_SOURCES = \ P_UnixNetVConnection.h \ P_UnixPollDescriptor.h \ P_UnixUDPConnection.h \ + ProxyProtocol.h \ + ProxyProtocol.cc \ Socks.cc \ - SNIActionPerformer.cc \ SSLCertLookup.cc \ - SSLSessionCache.cc \ + SSLClientUtils.cc \ SSLConfig.cc \ + SSLDiags.cc \ SSLInternal.cc \ SSLNetAccept.cc \ SSLNetProcessor.cc \ @@ -142,8 +144,10 @@ libinknet_a_SOURCES = \ SSLNextProtocolAccept.cc \ SSLNextProtocolSet.cc \ SSLSNIConfig.cc \ + SSLStats.cc \ + SSLSessionCache.cc \ + SSLSessionTicket.cc \ SSLUtils.cc \ - SSLClientUtils.cc \ OCSPStapling.cc \ Socks.cc \ UDPIOEvent.cc \ diff --git a/iocore/net/Net.cc b/iocore/net/Net.cc index ff6b0cb75f0..3ac1c8208ff 100644 --- a/iocore/net/Net.cc +++ b/iocore/net/Net.cc @@ -40,6 +40,10 @@ int net_accept_period = 10; int net_retry_delay = 10; int net_throttle_delay = 50; /* milliseconds */ +// For the in/out congestion control: ToDo: this probably would be better as ports: specifications +std::string_view net_ccp_in; +std::string_view net_ccp_out; + static inline void configure_net() { @@ -52,6 +56,20 @@ configure_net() // These are not reloadable REC_ReadConfigInteger(net_event_period, "proxy.config.net.event_period"); REC_ReadConfigInteger(net_accept_period, "proxy.config.net.accept_period"); + + // This is kinda fugly, but better than it was before (on every connection in and out) + // Note that these would need to be ats_free()'d if we ever want to clean that up, but + // we have no good way of dealing with that on such globals I think? + RecString ccp; + + REC_ReadConfigStringAlloc(ccp, "proxy.config.net.tcp_congestion_control_in"); + if (ccp && *ccp != '\0') { + net_ccp_in = ccp; + } + REC_ReadConfigStringAlloc(ccp, "proxy.config.net.tcp_congestion_control_out"); + if (ccp && *ccp != '\0') { + net_ccp_out = ccp; + } } static inline void @@ -120,11 +138,12 @@ register_net_stats() } void -ink_net_init(ModuleVersion version) +ink_net_init(ts::ModuleVersion version) { static int init_called = 0; - ink_release_assert(!checkModuleVersion(version, NET_SYSTEM_MODULE_VERSION)); + ink_release_assert(version.check(NET_SYSTEM_MODULE_INTERNAL_VERSION)); + if (!init_called) { // do one time stuff // create a stat block for NetStats diff --git a/iocore/net/NetVCTest.cc b/iocore/net/NetVCTest.cc index 7889cd13bf6..41ac801d068 100644 --- a/iocore/net/NetVCTest.cc +++ b/iocore/net/NetVCTest.cc @@ -26,7 +26,7 @@ NetVCTest.cc Description: - Unit test for infastructure for VConnections implementing the + Unit test for infrastructure for VConnections implementing the NetVConnection interface @@ -67,7 +67,7 @@ NVC_test_def netvc_tests_def[] = { {"itimeout", 6000, 8000, 10, 10, 512, 10, VC_EVENT_READ_COMPLETE, VC_EVENT_INACTIVITY_TIMEOUT}, {"itimeout", 10, 10, 6000, 8000, 512, 20, VC_EVENT_EOS, VC_EVENT_WRITE_COMPLETE}, - // Test the small transfer code one byts at a time + // Test the small transfer code one byte at a time {"smallt", 400, 400, 500, 500, 1, 15, VC_EVENT_READ_COMPLETE, VC_EVENT_WRITE_COMPLETE}, {"smallt", 500, 500, 400, 400, 1, 15, VC_EVENT_READ_COMPLETE, VC_EVENT_WRITE_COMPLETE}, @@ -80,37 +80,7 @@ NVC_test_def netvc_tests_def[] = { }; const unsigned num_netvc_tests = countof(netvc_tests_def); -NetVCTest::NetVCTest() - : Continuation(nullptr), - test_cont_type(NET_VC_TEST_ACTIVE), - test_vc(nullptr), - regress(nullptr), - driver(nullptr), - read_vio(nullptr), - write_vio(nullptr), - read_buffer(nullptr), - write_buffer(nullptr), - reader_for_rbuf(nullptr), - reader_for_wbuf(nullptr), - write_bytes_to_add_per(0), - timeout(0), - actual_bytes_read(0), - actual_bytes_sent(0), - write_done(false), - read_done(false), - read_seed(0), - write_seed(0), - bytes_to_send(0), - bytes_to_read(0), - nbytes_read(0), - nbytes_write(0), - expected_read_term(0), - expected_write_term(0), - test_name(nullptr), - module_name(nullptr), - debug_tag(nullptr) -{ -} +NetVCTest::NetVCTest() : Continuation(nullptr) {} NetVCTest::~NetVCTest() { @@ -395,6 +365,6 @@ NetVCTest::main_handler(int event, void *data) return 0; } -NetTestDriver::NetTestDriver() : Continuation(nullptr), errors(0), r(nullptr), pstatus(nullptr) {} +NetTestDriver::NetTestDriver() : Continuation(nullptr) {} NetTestDriver::~NetTestDriver() {} diff --git a/iocore/net/OCSPStapling.cc b/iocore/net/OCSPStapling.cc index 90133e42cb2..e6635ed9b5a 100644 --- a/iocore/net/OCSPStapling.cc +++ b/iocore/net/OCSPStapling.cc @@ -20,15 +20,17 @@ */ #include "P_OCSPStapling.h" -#ifdef TS_USE_TLS_OCSP +#if TS_USE_TLS_OCSP +#include #include #include #include "P_Net.h" #include "P_SSLConfig.h" #include "P_SSLUtils.h" +#include "SSLStats.h" -// Maxiumum OCSP stapling response size. +// Maximum OCSP stapling response size. // This should be the response for a single certificate and will typically include the responder certificate chain, // so 10K should be more than enough. #define MAX_STAPLING_DER 10240 @@ -42,26 +44,37 @@ struct certinfo { ink_mutex stapling_mutex; unsigned char resp_der[MAX_STAPLING_DER]; unsigned int resp_derlen; + bool is_prefetched; bool is_expire; time_t expire_time; }; +/* + * In the case of multiple certificates associated with a SSL_CTX, we must store a map + * of cached responses + */ +using certinfo_map = std::map; + void -certinfo_free(void * /*parent*/, void *ptr, CRYPTO_EX_DATA * /*ad*/, int /*idx*/, long /*argl*/, void * /*argp*/) +certinfo_map_free(void * /*parent*/, void *ptr, CRYPTO_EX_DATA * /*ad*/, int /*idx*/, long /*argl*/, void * /*argp*/) { - certinfo *cinf = (certinfo *)ptr; + certinfo_map *map = (certinfo_map *)ptr; - if (!cinf) { + if (!map) { return; } - if (cinf->uri) { - OPENSSL_free(cinf->uri); - } - if (cinf->certname) { - ats_free(cinf->certname); + + for (certinfo_map::iterator iter = map->begin(); iter != map->end(); ++iter) { + if (iter->second->uri) { + OPENSSL_free(iter->second->uri); + } + if (iter->second->certname) { + ats_free(iter->second->certname); + } + ink_mutex_destroy(&iter->second->stapling_mutex); + OPENSSL_free(iter->second); } - ink_mutex_destroy(&cinf->stapling_mutex); - OPENSSL_free(cinf); + free(map); } static int ssl_stapling_index = -1; @@ -72,7 +85,7 @@ ssl_stapling_ex_init() if (ssl_stapling_index != -1) { return; } - ssl_stapling_index = SSL_CTX_get_ex_new_index(0, nullptr, nullptr, nullptr, certinfo_free); + ssl_stapling_index = SSL_CTX_get_ex_new_index(0, nullptr, nullptr, nullptr, certinfo_map_free); } static X509 * @@ -89,7 +102,7 @@ stapling_get_issuer(SSL_CTX *ssl_ctx, X509 *x) #ifdef SSL_CTX_select_current_cert if (!SSL_CTX_select_current_cert(ssl_ctx, x)) { - Warning("OCSP: could not select current certifcate chain %p", x); + Warning("OCSP: could not select current certificate chain %p", x); } #endif @@ -130,27 +143,63 @@ stapling_get_issuer(SSL_CTX *ssl_ctx, X509 *x) return issuer; } +static bool +stapling_cache_response(OCSP_RESPONSE *rsp, certinfo *cinf) +{ + unsigned char resp_der[MAX_STAPLING_DER]; + unsigned char *p; + unsigned int resp_derlen; + + p = resp_der; + resp_derlen = i2d_OCSP_RESPONSE(rsp, &p); + + if (resp_derlen == 0) { + Error("stapling_cache_response: cannot decode OCSP response for %s", cinf->certname); + return false; + } + + if (resp_derlen > MAX_STAPLING_DER) { + Error("stapling_cache_response: OCSP response too big (%u bytes) for %s", resp_derlen, cinf->certname); + return false; + } + + ink_mutex_acquire(&cinf->stapling_mutex); + memcpy(cinf->resp_der, resp_der, resp_derlen); + cinf->resp_derlen = resp_derlen; + cinf->is_expire = false; + cinf->expire_time = time(nullptr) + SSLConfigParams::ssl_ocsp_cache_timeout; + ink_mutex_release(&cinf->stapling_mutex); + + Debug("ssl_ocsp", "stapling_cache_response: success to cache response"); + return true; +} + bool -ssl_stapling_init_cert(SSL_CTX *ctx, X509 *cert, const char *certname) +ssl_stapling_init_cert(SSL_CTX *ctx, X509 *cert, const char *certname, const char *rsp_file) { - certinfo *cinf; scoped_X509 issuer; STACK_OF(OPENSSL_STRING) *aia = nullptr; + BIO *rsp_bio = nullptr; + OCSP_RESPONSE *rsp = nullptr; if (!cert) { Error("null cert passed in for %s", certname); return false; } - cinf = (certinfo *)SSL_CTX_get_ex_data(ctx, ssl_stapling_index); - if (cinf) { + certinfo_map *map = static_cast(SSL_CTX_get_ex_data(ctx, ssl_stapling_index)); + if (map && map->find(cert) != map->end()) { Note("certificate already initialized for %s", certname); return false; } - cinf = (certinfo *)OPENSSL_malloc(sizeof(certinfo)); + if (!map) { + map = new certinfo_map; + } + certinfo *cinf = static_cast(OPENSSL_malloc(sizeof(certinfo))); if (!cinf) { Error("error allocating memory for %s", certname); + delete map; return false; } @@ -160,8 +209,33 @@ ssl_stapling_init_cert(SSL_CTX *ctx, X509 *cert, const char *certname) cinf->certname = ats_strdup(certname); cinf->resp_derlen = 0; ink_mutex_init(&cinf->stapling_mutex); - cinf->is_expire = true; - cinf->expire_time = 0; + cinf->is_prefetched = rsp_file ? true : false; + cinf->is_expire = true; + cinf->expire_time = 0; + + if (cinf->is_prefetched) { + Debug("ssl_ocsp", "using OCSP prefetched response file %s", rsp_file); + rsp_bio = BIO_new_file(rsp_file, "r"); + if (rsp_bio) { + rsp = d2i_OCSP_RESPONSE_bio(rsp_bio, nullptr); + } + + if (!rsp_bio || !rsp) { + Note("cannot get prefetched response for %s from %s", certname, rsp_file); + goto err; + } + + if (!stapling_cache_response(rsp, cinf)) { + Error("stapling_refresh_response: can not cache response"); + goto err; + } else { + Debug("ssl_ocsp", "stapling_refresh_response: successful refresh OCSP response"); + OCSP_RESPONSE_free(rsp); + rsp = nullptr; + BIO_free(rsp_bio); + rsp_bio = nullptr; + } + } issuer = stapling_get_issuer(ctx, cert); if (issuer == nullptr) { @@ -187,13 +261,14 @@ ssl_stapling_init_cert(SSL_CTX *ctx, X509 *cert, const char *certname) goto err; } - SSL_CTX_set_ex_data(ctx, ssl_stapling_index, cinf); + map->insert(std::make_pair(cert, cinf)); + SSL_CTX_set_ex_data(ctx, ssl_stapling_index, map); Note("successfully initialized stapling for %s into SSL_CTX: %p", certname, ctx); return true; err: - if (cinf->uri) { + if (cinf->cid) { OCSP_CERTID_free(cinf->cid); } @@ -204,51 +279,32 @@ ssl_stapling_init_cert(SSL_CTX *ctx, X509 *cert, const char *certname) if (cinf) { OPENSSL_free(cinf); } - return false; -} - -static certinfo * -stapling_get_cert_info(SSL_CTX *ctx) -{ - certinfo *cinf; + if (map) { + delete map; + } - cinf = (certinfo *)SSL_CTX_get_ex_data(ctx, ssl_stapling_index); - if (cinf && cinf->cid) { - return cinf; + if (rsp) { + OCSP_RESPONSE_free(rsp); + } + if (rsp_bio) { + BIO_free(rsp_bio); } - return nullptr; + return false; } -static bool -stapling_cache_response(OCSP_RESPONSE *rsp, certinfo *cinf) +static certinfo_map * +stapling_get_cert_info(SSL_CTX *ctx) { - unsigned char resp_der[MAX_STAPLING_DER]; - unsigned char *p; - unsigned int resp_derlen; + certinfo_map *map; - p = resp_der; - resp_derlen = i2d_OCSP_RESPONSE(rsp, &p); - - if (resp_derlen == 0) { - Error("stapling_cache_response: cannot decode OCSP response for %s", cinf->certname); - return false; + // Only return the map if it contains at least one element with a valid entry + map = static_cast(SSL_CTX_get_ex_data(ctx, ssl_stapling_index)); + if (map && !map->empty() && map->begin()->second && map->begin()->second->cid) { + return map; } - if (resp_derlen > MAX_STAPLING_DER) { - Error("stapling_cache_response: OCSP response too big (%u bytes) for %s", resp_derlen, cinf->certname); - return false; - } - - ink_mutex_acquire(&cinf->stapling_mutex); - memcpy(cinf->resp_der, resp_der, resp_derlen); - cinf->resp_derlen = resp_derlen; - cinf->is_expire = false; - cinf->expire_time = time(nullptr) + SSLConfigParams::ssl_ocsp_cache_timeout; - ink_mutex_release(&cinf->stapling_mutex); - - Debug("ssl_ocsp", "stapling_cache_response: success to cache response"); - return true; + return nullptr; } static int @@ -424,7 +480,6 @@ void ocsp_update() { SSL_CTX *ctx; - certinfo *cinf = nullptr; OCSP_RESPONSE *resp = nullptr; time_t current_time; @@ -434,22 +489,27 @@ ocsp_update() for (unsigned i = 0; i < ctxCount; i++) { SSLCertContext *cc = certLookup->get(i); if (cc && cc->ctx) { - ctx = cc->ctx; - cinf = stapling_get_cert_info(ctx); - if (cinf) { - ink_mutex_acquire(&cinf->stapling_mutex); - current_time = time(nullptr); - if (cinf->resp_derlen == 0 || cinf->is_expire || cinf->expire_time < current_time) { - ink_mutex_release(&cinf->stapling_mutex); - if (stapling_refresh_response(cinf, &resp)) { - Debug("Successfully refreshed OCSP for %s certificate. url=%s", cinf->certname, cinf->uri); - SSL_INCREMENT_DYN_STAT(ssl_ocsp_refreshed_cert_stat); + ctx = cc->ctx; + certinfo *cinf = nullptr; + certinfo_map *map = stapling_get_cert_info(ctx); + if (map) { + // Walk over all certs associated with this CTX + for (certinfo_map::iterator iter = map->begin(); iter != map->end(); ++iter) { + cinf = iter->second; + ink_mutex_acquire(&cinf->stapling_mutex); + current_time = time(nullptr); + if ((cinf->resp_derlen == 0 || cinf->is_expire || cinf->expire_time < current_time) && !cinf->is_prefetched) { + ink_mutex_release(&cinf->stapling_mutex); + if (stapling_refresh_response(cinf, &resp)) { + Debug("Successfully refreshed OCSP for %s certificate. url=%s", cinf->certname, cinf->uri); + SSL_INCREMENT_DYN_STAT(ssl_ocsp_refreshed_cert_stat); + } else { + Error("Failed to refresh OCSP for %s certificate. url=%s", cinf->certname, cinf->uri); + SSL_INCREMENT_DYN_STAT(ssl_ocsp_refresh_cert_failure_stat); + } } else { - Error("Failed to refresh OCSP for %s certificate. url=%s", cinf->certname, cinf->uri); - SSL_INCREMENT_DYN_STAT(ssl_ocsp_refresh_cert_failure_stat); + ink_mutex_release(&cinf->stapling_mutex); } - } else { - ink_mutex_release(&cinf->stapling_mutex); } } } @@ -460,21 +520,30 @@ ocsp_update() int ssl_callback_ocsp_stapling(SSL *ssl) { - certinfo *cinf = nullptr; - time_t current_time; - // Assume SSL_get_SSL_CTX() is the same as reaching into the ssl structure // Using the official call, to avoid leaking internal openssl knowledge // originally was, cinf = stapling_get_cert_info(ssl->ctx); - cinf = stapling_get_cert_info(SSL_get_SSL_CTX(ssl)); - if (cinf == nullptr) { - Debug("ssl_ocsp", "ssl_callback_ocsp_stapling: failed to get certificate information"); + certinfo_map *map = stapling_get_cert_info(SSL_get_SSL_CTX(ssl)); + if (map == nullptr) { + Debug("ssl_ocsp", "ssl_callback_ocsp_stapling: failed to get certificate map"); + return SSL_TLSEXT_ERR_NOACK; + } + // Fetch the specific certificate used in this negotiation + X509 *cert = SSL_get_certificate(ssl); + if (!cert) { + Error("ssl_callback_ocsp_stapling: failed to get certificate"); + return SSL_TLSEXT_ERR_NOACK; + } + certinfo_map::iterator iter = map->find(cert); + if (iter == map->end()) { + Error("ssl_callback_ocsp_stapling: failed to get certificate information"); return SSL_TLSEXT_ERR_NOACK; } + certinfo *cinf = iter->second; ink_mutex_acquire(&cinf->stapling_mutex); - current_time = time(nullptr); - if (cinf->resp_derlen == 0 || cinf->is_expire || cinf->expire_time < current_time) { + time_t current_time = time(nullptr); + if ((cinf->resp_derlen == 0 || cinf->is_expire) || (cinf->expire_time < current_time && !cinf->is_prefetched)) { ink_mutex_release(&cinf->stapling_mutex); Debug("ssl_ocsp", "ssl_callback_ocsp_stapling: failed to get certificate status for %s", cinf->certname); return SSL_TLSEXT_ERR_NOACK; diff --git a/iocore/net/P_Connection.h b/iocore/net/P_Connection.h index f7d4d687515..8c8d4ce80ac 100644 --- a/iocore/net/P_Connection.h +++ b/iocore/net/P_Connection.h @@ -42,7 +42,7 @@ The accept call is a blocking call while connect is non-blocking. They returns a new Connection instance which is an handle to the newly created connection. The connection `q instance can be used later for read/writes - using an intance of IOProcessor class. + using an instance of IOProcessor class. **************************************************************************/ @@ -78,11 +78,11 @@ struct NetVCOptions; // /////////////////////////////////////////////////////////////////////// struct Connection { - SOCKET fd; ///< Socket for connection. - IpEndpoint addr; ///< Associated address. - bool is_bound; ///< Flag for already bound to a local address. - bool is_connected; ///< Flag for already connected. - int sock_type; + SOCKET fd; ///< Socket for connection. + IpEndpoint addr; ///< Associated address. + bool is_bound = false; ///< Flag for already bound to a local address. + bool is_connected = false; ///< Flag for already connected. + int sock_type = 0; /** Create and initialize the socket for this connection. @@ -143,7 +143,7 @@ struct Connection { private: // Don't want copy constructors to avoid having the deconstructor on - // temporarly copies close the file descriptor too soon. Use move instead + // temporarily copies close the file descriptor too soon. Use move instead Connection(Connection const &); protected: @@ -160,7 +160,7 @@ struct Server : public Connection { IpEndpoint accept_addr; /// If set, a kernel HTTP accept filter - bool http_accept_filter; + bool http_accept_filter = false; int accept(Connection *c); @@ -173,5 +173,5 @@ struct Server : public Connection { int listen(bool non_blocking, const NetProcessor::AcceptOptions &opt); int setup_fd_for_listen(bool non_blocking, const NetProcessor::AcceptOptions &opt); - Server() : Connection(), http_accept_filter(false) { ink_zero(accept_addr); } + Server() : Connection() { ink_zero(accept_addr); } }; diff --git a/iocore/net/P_Net.h b/iocore/net/P_Net.h index c59250be3b9..d611903b469 100644 --- a/iocore/net/P_Net.h +++ b/iocore/net/P_Net.h @@ -110,9 +110,7 @@ extern RecRawStatBlock *net_rsb; #include "P_SSLNetAccept.h" #include "P_SSLCertLookup.h" -#undef NET_SYSTEM_MODULE_VERSION -#define NET_SYSTEM_MODULE_VERSION \ - makeModuleVersion(NET_SYSTEM_MODULE_MAJOR_VERSION, NET_SYSTEM_MODULE_MINOR_VERSION, PRIVATE_MODULE_HEADER) +static constexpr ts::ModuleVersion NET_SYSTEM_MODULE_INTERNAL_VERSION(NET_SYSTEM_MODULE_PUBLIC_VERSION, ts::ModuleVersion::PRIVATE); // For very verbose iocore debugging. #ifndef DEBUG diff --git a/iocore/net/P_NetVCTest.h b/iocore/net/P_NetVCTest.h index 08be426a088..f98193218dd 100644 --- a/iocore/net/P_NetVCTest.h +++ b/iocore/net/P_NetVCTest.h @@ -26,7 +26,7 @@ P_NetVCTest.h Description: - Unit test for infastructure for VConnections implementing the + Unit test for infrastructure for VConnections implementing the NetVConnection interface @@ -73,11 +73,11 @@ class NetTestDriver : public Continuation NetTestDriver(); ~NetTestDriver() override; - int errors; + int errors = 0; protected: - RegressionTest *r; - int *pstatus; + RegressionTest *r = nullptr; + int *pstatus = nullptr; }; class NetVCTest : public Continuation @@ -85,7 +85,7 @@ class NetVCTest : public Continuation public: NetVCTest(); ~NetVCTest() override; - NetVcTestType_t test_cont_type; + NetVcTestType_t test_cont_type = NET_VC_TEST_ACTIVE; int main_handler(int event, void *data); void read_handler(int event); @@ -103,41 +103,41 @@ class NetVCTest : public Continuation void finished(); void record_error(const char *msg); - NetVConnection *test_vc; - RegressionTest *regress; - NetTestDriver *driver; + NetVConnection *test_vc = nullptr; + RegressionTest *regress = nullptr; + NetTestDriver *driver = nullptr; - VIO *read_vio; - VIO *write_vio; + VIO *read_vio = nullptr; + VIO *write_vio = nullptr; - MIOBuffer *read_buffer; - MIOBuffer *write_buffer; + MIOBuffer *read_buffer = nullptr; + MIOBuffer *write_buffer = nullptr; - IOBufferReader *reader_for_rbuf; - IOBufferReader *reader_for_wbuf; + IOBufferReader *reader_for_rbuf = nullptr; + IOBufferReader *reader_for_wbuf = nullptr; - int write_bytes_to_add_per; - int timeout; + int write_bytes_to_add_per = 0; + int timeout = 0; - int actual_bytes_read; - int actual_bytes_sent; + int actual_bytes_read = 0; + int actual_bytes_sent = 0; - bool write_done; - bool read_done; + bool write_done = false; + bool read_done = false; - uint8_t read_seed; - uint8_t write_seed; + uint8_t read_seed = 0; + uint8_t write_seed = 0; - int bytes_to_send; - int bytes_to_read; + int bytes_to_send = 0; + int bytes_to_read = 0; - int nbytes_read; - int nbytes_write; + int nbytes_read = 0; + int nbytes_write = 0; - int expected_read_term; - int expected_write_term; + int expected_read_term = 0; + int expected_write_term = 0; - const char *test_name; - const char *module_name; - const char *debug_tag; + const char *test_name = nullptr; + const char *module_name = nullptr; + const char *debug_tag = nullptr; }; diff --git a/iocore/net/P_NetVConnection.h b/iocore/net/P_NetVConnection.h index a81563304ca..679cf0b10ed 100644 --- a/iocore/net/P_NetVConnection.h +++ b/iocore/net/P_NetVConnection.h @@ -23,24 +23,28 @@ #include "I_NetVConnection.h" -TS_INLINE sockaddr const * +inline sockaddr const * NetVConnection::get_remote_addr() { if (!got_remote_addr) { - set_remote_addr(); + if (pp_info.proxy_protocol_version != ProxyProtocolVersion::UNDEFINED) { + set_remote_addr(get_proxy_protocol_src_addr()); + } else { + set_remote_addr(); + } got_remote_addr = true; } return &remote_addr.sa; } -TS_INLINE IpEndpoint const & +inline IpEndpoint const & NetVConnection::get_remote_endpoint() { - get_remote_addr(); // Make sure the vallue is filled in + get_remote_addr(); // Make sure the value is filled in return remote_addr; } -TS_INLINE in_addr_t +inline in_addr_t NetVConnection::get_remote_ip() { sockaddr const *addr = this->get_remote_addr(); @@ -48,13 +52,13 @@ NetVConnection::get_remote_ip() } /// @return The remote port in host order. -TS_INLINE uint16_t +inline uint16_t NetVConnection::get_remote_port() { return ats_ip_port_host_order(this->get_remote_addr()); } -TS_INLINE sockaddr const * +inline sockaddr const * NetVConnection::get_local_addr() { if (!got_local_addr) { @@ -68,7 +72,7 @@ NetVConnection::get_local_addr() return &local_addr.sa; } -TS_INLINE in_addr_t +inline in_addr_t NetVConnection::get_local_ip() { sockaddr const *addr = this->get_local_addr(); @@ -76,8 +80,27 @@ NetVConnection::get_local_ip() } /// @return The local port in host order. -TS_INLINE uint16_t +inline uint16_t NetVConnection::get_local_port() { return ats_ip_port_host_order(this->get_local_addr()); } + +inline sockaddr const * +NetVConnection::get_proxy_protocol_addr(const ProxyProtocolData src_or_dst) +{ + if (src_or_dst == ProxyProtocolData::SRC) { + if ((pp_info.src_addr.isValid() && pp_info.src_addr.port() != 0) || + (ats_is_ip4(&pp_info.src_addr) && INADDR_ANY != ats_ip4_addr_cast(&pp_info.src_addr)) // IPv4 + || (ats_is_ip6(&pp_info.src_addr) && !IN6_IS_ADDR_UNSPECIFIED(&pp_info.src_addr.sin6.sin6_addr))) { + return &pp_info.src_addr.sa; + } + } else { + if ((pp_info.dst_addr.isValid() && pp_info.dst_addr.port() != 0) || + (ats_is_ip4(&pp_info.dst_addr) && INADDR_ANY != ats_ip4_addr_cast(&pp_info.dst_addr)) // IPv4 + || (ats_is_ip6(&pp_info.dst_addr) && !IN6_IS_ADDR_UNSPECIFIED(&pp_info.dst_addr.sin6.sin6_addr))) { + return &pp_info.dst_addr.sa; + } + } + return nullptr; +} diff --git a/iocore/net/P_OCSPStapling.h b/iocore/net/P_OCSPStapling.h index e3d5a6f63dc..fbee1103322 100644 --- a/iocore/net/P_OCSPStapling.h +++ b/iocore/net/P_OCSPStapling.h @@ -28,7 +28,7 @@ #include void ssl_stapling_ex_init(); -bool ssl_stapling_init_cert(SSL_CTX *ctx, X509 *cert, const char *certname); +bool ssl_stapling_init_cert(SSL_CTX *ctx, X509 *cert, const char *certname, const char *rsp_file); void ocsp_update(); int ssl_callback_ocsp_stapling(SSL *); #endif diff --git a/iocore/net/P_SNIActionPerformer.h b/iocore/net/P_SNIActionPerformer.h index 001ac00a8b5..957ff3c055a 100644 --- a/iocore/net/P_SNIActionPerformer.h +++ b/iocore/net/P_SNIActionPerformer.h @@ -31,27 +31,28 @@ #pragma once #include "I_EventSystem.h" -#include "tscore/Map.h" -//#include"P_UnixNetProcessor.h" #include #include "P_SSLNextProtocolAccept.h" #include "tscore/ink_inet.h" +#include -extern Map snpsMap; -// enum of all the actions +extern std::unordered_map snpsMap; + +/*// enum of all the actions enum AllActions { TS_DISABLE_H2 = 0, TS_VERIFY_CLIENT, // this applies to server side vc only TS_TUNNEL_ROUTE, // blind tunnel action }; +*/ /** action for setting next hop properties should be listed in the following enum*/ -enum PropertyActions { TS_VERIFY_SERVER = 200, TS_CLIENT_CERT }; +/* enum PropertyActions { TS_VERIFY_SERVER = 200, TS_CLIENT_CERT }; */ class ActionItem { public: - virtual int SNIAction(Continuation *cont) = 0; + virtual int SNIAction(Continuation *cont) const = 0; virtual ~ActionItem(){}; }; @@ -62,16 +63,37 @@ class DisableH2 : public ActionItem ~DisableH2() override {} int - SNIAction(Continuation *cont) override + SNIAction(Continuation *cont) const override { - auto ssl_vc = reinterpret_cast(cont); + auto ssl_vc = dynamic_cast(cont); auto accept_obj = ssl_vc ? ssl_vc->accept_object : nullptr; if (accept_obj && accept_obj->snpa && ssl_vc) { - auto nps = snpsMap.get(accept_obj->id); - ssl_vc->registerNextProtocolSet(reinterpret_cast(nps)); + if (auto it = snpsMap.find(accept_obj->id); it != snpsMap.end()) { + ssl_vc->registerNextProtocolSet(it->second); + } + } + return SSL_TLSEXT_ERR_OK; + } +}; + +class TunnelDestination : public ActionItem +{ +public: + TunnelDestination(const std::string_view &dest, bool decrypt) : destination(dest), tunnel_decrypt(decrypt) {} + ~TunnelDestination() override {} + + int + SNIAction(Continuation *cont) const override + { + // Set the netvc option? + SSLNetVConnection *ssl_netvc = dynamic_cast(cont); + if (ssl_netvc) { + ssl_netvc->set_tunnel_destination(destination, tunnel_decrypt); } return SSL_TLSEXT_ERR_OK; } + std::string destination; + bool tunnel_decrypt = false; }; class VerifyClient : public ActionItem @@ -83,21 +105,47 @@ class VerifyClient : public ActionItem VerifyClient(uint8_t param) : mode(param) {} ~VerifyClient() override {} int - SNIAction(Continuation *cont) override + SNIAction(Continuation *cont) const override { - auto ssl_vc = reinterpret_cast(cont); + auto ssl_vc = dynamic_cast(cont); Debug("ssl_sni", "action verify param %d", this->mode); setClientCertLevel(ssl_vc->ssl, this->mode); return SSL_TLSEXT_ERR_OK; } }; +class TLSValidProtocols : public ActionItem +{ + bool unset = true; + unsigned long protocol_mask; + +public: +#ifdef SSL_OP_NO_TLSv1_3 + static const unsigned long max_mask = SSL_OP_NO_TLSv1 | SSL_OP_NO_TLSv1_1 | SSL_OP_NO_TLSv1_2 | SSL_OP_NO_TLSv1_3; +#else + static const unsigned long max_mask = SSL_OP_NO_TLSv1 | SSL_OP_NO_TLSv1_1 | SSL_OP_NO_TLSv1_2; +#endif + TLSValidProtocols() : protocol_mask(max_mask) {} + TLSValidProtocols(unsigned long protocols) : unset(false), protocol_mask(protocols) {} + int + SNIAction(Continuation *cont) const override + { + if (!unset) { + auto ssl_vc = dynamic_cast(cont); + Debug("ssl_sni", "TLSValidProtocol param 0%x", static_cast(this->protocol_mask)); + ssl_vc->protocol_mask_set = true; + ssl_vc->protocol_mask = protocol_mask; + } + return SSL_TLSEXT_ERR_OK; + } +}; + class SNI_IpAllow : public ActionItem { IpMap ip_map; public: - SNI_IpAllow(std::string const &ip_allow_list, cchar *servername) + SNI_IpAllow(std::string &ip_allow_list, const std::string &servername) { // the server identified by item.fqdn requires ATS to do IP filtering if (ip_allow_list.length()) { @@ -112,7 +160,7 @@ class SNI_IpAllow : public ActionItem Debug("ssl_sni", "%.*s is not a valid format", static_cast(list.size()), list.data()); break; } else { - Debug("ssl_sni", "%.*s added to the ip_allow list %s", static_cast(list.size()), list.data(), servername); + Debug("ssl_sni", "%.*s added to the ip_allow list %s", static_cast(list.size()), list.data(), servername.c_str()); ip_map.fill(IpEndpoint().assign(addr1), IpEndpoint().assign(addr2), reinterpret_cast(1)); } } @@ -120,14 +168,14 @@ class SNI_IpAllow : public ActionItem } // end function SNI_IpAllow int - SNIAction(Continuation *cont) override + SNIAction(Continuation *cont) const override { // i.e, ip filtering is not required if (ip_map.count() == 0) { return SSL_TLSEXT_ERR_OK; } - auto ssl_vc = reinterpret_cast(cont); + auto ssl_vc = dynamic_cast(cont); auto ip = ssl_vc->get_remote_endpoint(); // check the allowed ips @@ -146,5 +194,5 @@ class SNIActionPerformer { public: SNIActionPerformer() = default; - static int PerformAction(Continuation *cont, cchar *servername); + static int PerformAction(Continuation *cont, const char *servername); }; diff --git a/iocore/net/P_SSLCertLookup.h b/iocore/net/P_SSLCertLookup.h index 65d278abe74..3a793408c27 100644 --- a/iocore/net/P_SSLCertLookup.h +++ b/iocore/net/P_SSLCertLookup.h @@ -23,8 +23,9 @@ #pragma once +#include + #include "ProxyConfig.h" -#include "P_SSLUtils.h" struct SSLConfigParams; struct SSLContextStorage; @@ -60,21 +61,21 @@ struct SSLCertContext { OPT_TUNNEL ///< Just tunnel, don't terminate. }; - SSLCertContext() : ctx(nullptr), opt(OPT_NONE), keyblock(nullptr) {} - explicit SSLCertContext(SSL_CTX *c) : ctx(c), opt(OPT_NONE), keyblock(nullptr) {} - SSLCertContext(SSL_CTX *c, Option o) : ctx(c), opt(o), keyblock(nullptr) {} + SSLCertContext() {} + explicit SSLCertContext(SSL_CTX *c) : ctx(c) {} + SSLCertContext(SSL_CTX *c, Option o) : ctx(c), opt(o) {} SSLCertContext(SSL_CTX *c, Option o, ssl_ticket_key_block *kb) : ctx(c), opt(o), keyblock(kb) {} void release(); - SSL_CTX *ctx; ///< openSSL context. - Option opt; ///< Special handling option. - ssl_ticket_key_block *keyblock; ///< session keys associated with this address + SSL_CTX *ctx = nullptr; ///< openSSL context. + Option opt = OPT_NONE; ///< Special handling option. + ssl_ticket_key_block *keyblock = nullptr; ///< session keys associated with this address }; struct SSLCertLookup : public ConfigInfo { SSLContextStorage *ssl_storage; - SSL_CTX *ssl_default; - bool is_valid; + SSL_CTX *ssl_default = nullptr; + bool is_valid = true; int insert(const char *name, SSLCertContext const &cc); int insert(const IpEndpoint &address, SSLCertContext const &cc); diff --git a/iocore/net/P_SSLConfig.h b/iocore/net/P_SSLConfig.h index 5ea0c955096..098d9230f52 100644 --- a/iocore/net/P_SSLConfig.h +++ b/iocore/net/P_SSLConfig.h @@ -30,11 +30,14 @@ ****************************************************************************/ #pragma once +#include + +#include "tscore/ink_inet.h" +#include "tscore/IpMap.h" + #include "ProxyConfig.h" + #include "SSLSessionCache.h" -#include "tscore/ink_inet.h" -#include -#include "P_SSLCertLookup.h" #include "YamlSNIConfig.h" struct SSLCertLookup; @@ -43,7 +46,7 @@ struct ssl_ticket_key_block; // // struct SSLConfigParams // -// configuration parameters as they apear in the global +// configuration parameters as they appear in the global // configuration file. ///////////////////////////////////////////////////////////// @@ -80,7 +83,9 @@ struct SSLConfigParams : public ConfigInfo { int ssl_session_cache_auto_clear; char *clientCertPath; + char *clientCertPathOnly; char *clientKeyPath; + char *clientKeyPathOnly; char *clientCACertFilename; char *clientCACertPath; YamlSNIConfig::Policy verifyServerPolicy; @@ -102,18 +107,13 @@ struct SSLConfigParams : public ConfigInfo { static int ssl_ocsp_request_timeout; static int ssl_ocsp_update_period; static int ssl_handshake_timeout_in; + char *ssl_ocsp_response_path_only; static size_t session_cache_number_buckets; static size_t session_cache_max_bucket_size; static bool session_cache_skip_on_lock_contention; - static bool sni_map_enable; - // TS-3435 Wiretracing for SSL Connections - static int ssl_wire_trace_enabled; - static char *ssl_wire_trace_addr; - static IpAddr *ssl_wire_trace_ip; - static int ssl_wire_trace_percentage; - static char *ssl_wire_trace_server_name; + static IpMap *proxy_protocol_ipmap; static init_ssl_ctx_func init_ssl_ctx_cb; static load_ssl_file_func load_ssl_file_cb; @@ -123,15 +123,22 @@ struct SSLConfigParams : public ConfigInfo { SSL_CTX *client_ctx; - mutable HashMap ctx_map; + // Client contexts are held by 2-level map: + // The first level maps from CA bundle file&path to next level map; + // The second level maps from cert&key to actual SSL_CTX; + // The second level map owns the client SSL_CTX objects and is responsible for cleaning them up + using CTX_MAP = std::unordered_map; + mutable std::unordered_map top_level_ctx_map; mutable ink_mutex ctxMapLock; - SSL_CTX *getClientSSL_CTX(void) const; - SSL_CTX *getNewCTX(cchar *client_cert, cchar *key_file) const; + SSL_CTX *getClientSSL_CTX() const; + SSL_CTX *getCTX(const char *client_cert, const char *key_file, const char *ca_bundle_file, const char *ca_bundle_path) const; + void cleanupCTXTable(); void initialize(); void cleanup(); void reset(); + void SSLConfigInit(IpMap *global); }; ///////////////////////////////////////////////////////////// diff --git a/iocore/net/P_SSLNetProcessor.h b/iocore/net/P_SSLNetProcessor.h index b90266bfce9..92183ed82d7 100644 --- a/iocore/net/P_SSLNetProcessor.h +++ b/iocore/net/P_SSLNetProcessor.h @@ -42,7 +42,6 @@ #include "P_Net.h" #include "P_SSLConfig.h" #include -#include "tscore/Map.h" class UnixNetVConnection; struct NetAccept; @@ -56,7 +55,7 @@ struct SSLNetProcessor : public UnixNetProcessor { public: int start(int, size_t stacksize) override; - void cleanup(void); + void cleanup(); SSLNetProcessor(); ~SSLNetProcessor() override; diff --git a/iocore/net/P_SSLNetVConnection.h b/iocore/net/P_SSLNetVConnection.h index 72ef4d76d30..7f350107925 100644 --- a/iocore/net/P_SSLNetVConnection.h +++ b/iocore/net/P_SSLNetVConnection.h @@ -178,12 +178,6 @@ class SSLNetVConnection : public UnixNetVConnection return transparentPassThrough; } - bool - GetSNIMapping() - { - return SNIMapping; - } - void setTransparentPassThrough(bool val) { @@ -247,9 +241,19 @@ class SSLNetVConnection : public UnixNetVConnection } } break; - case HANDSHAKE_HOOKS_SNI: + case HANDSHAKE_HOOKS_CLIENT_HELLO: + case HANDSHAKE_HOOKS_CLIENT_HELLO_INVOKE: if (eventId == TS_EVENT_VCONN_START) { retval = true; + } else if (eventId == TS_EVENT_SSL_CLIENT_HELLO) { + if (curHook) { + retval = true; + } + } + break; + case HANDSHAKE_HOOKS_SNI: + if (eventId == TS_EVENT_VCONN_START || eventId == TS_EVENT_SSL_CLIENT_HELLO) { + retval = true; } else if (eventId == TS_EVENT_SSL_SERVERNAME) { if (curHook) { retval = true; @@ -258,7 +262,7 @@ class SSLNetVConnection : public UnixNetVConnection break; case HANDSHAKE_HOOKS_CERT: case HANDSHAKE_HOOKS_CERT_INVOKE: - if (eventId == TS_EVENT_VCONN_START || eventId == TS_EVENT_SSL_SERVERNAME) { + if (eventId == TS_EVENT_VCONN_START || eventId == TS_EVENT_SSL_CLIENT_HELLO || eventId == TS_EVENT_SSL_SERVERNAME) { retval = true; } else if (eventId == TS_EVENT_SSL_CERT) { if (curHook) { @@ -292,30 +296,60 @@ class SSLNetVConnection : public UnixNetVConnection } return retval; } - bool - getSSLTrace() const + + const char * + getSSLProtocol() const { - return sslTrace || super::origin_trace; + return ssl ? SSL_get_version(ssl) : nullptr; } - void - setSSLTrace(bool state) + const char * + getSSLCipherSuite() const { - sslTrace = state; + return ssl ? SSL_get_cipher_name(ssl) : nullptr; } - bool computeSSLTrace(); + bool + has_tunnel_destination() const + { + return tunnel_host != nullptr; + } const char * - getSSLProtocol(void) const + get_tunnel_host() const { - return ssl ? SSL_get_version(ssl) : nullptr; + return tunnel_host; } - const char * - getSSLCipherSuite(void) const + ushort + get_tunnel_port() const { - return ssl ? SSL_get_cipher_name(ssl) : nullptr; + return tunnel_port; + } + + /* Returns true if this vc was configured for forward_route + */ + bool + decrypt_tunnel() + { + return has_tunnel_destination() && tunnel_decrypt; + } + + void + set_tunnel_destination(const std::string_view &destination, bool decrypt) + { + auto pos = destination.find(":"); + if (nullptr != tunnel_host) { + ats_free(tunnel_host); + } + if (pos != std::string::npos) { + tunnel_port = std::stoi(destination.substr(pos + 1).data()); + tunnel_host = ats_strndup(destination.substr(0, pos).data(), pos); + } else { + tunnel_port = 0; + tunnel_host = ats_strndup(destination.data(), destination.length()); + } + tunnel_decrypt = decrypt; } int populate_protocol(std::string_view *results, int n) const override; @@ -333,7 +367,11 @@ class SSLNetVConnection : public UnixNetVConnection ink_hrtime sslHandshakeEndTime = 0; ink_hrtime sslLastWriteTime = 0; int64_t sslTotalBytesSent = 0; - char *serverName = nullptr; + // The serverName is either a pointer to the name fetched from the + // SSL object or the empty string. Therefore, we do not allocate + // extra memory for this value. If plugins in the future can set the + // serverName value, this strategy will have to change. + const char *serverName = nullptr; /// Set by asynchronous hooks to request a specific operation. SslVConnOp hookOpRequested = SSL_HOOK_OP_DEFAULT; @@ -342,6 +380,9 @@ class SSLNetVConnection : public UnixNetVConnection SSLNetVConnection(const SSLNetVConnection &) = delete; SSLNetVConnection &operator=(const SSLNetVConnection &) = delete; + bool protocol_mask_set = false; + unsigned long protocol_mask; + private: std::string_view map_tls_protocol_to_tag(const char *proto_string) const; bool update_rbio(bool move_to_socket); @@ -363,6 +404,8 @@ class SSLNetVConnection : public UnixNetVConnection enum SSLHandshakeHookState { HANDSHAKE_HOOKS_PRE, HANDSHAKE_HOOKS_PRE_INVOKE, + HANDSHAKE_HOOKS_CLIENT_HELLO, + HANDSHAKE_HOOKS_CLIENT_HELLO_INVOKE, HANDSHAKE_HOOKS_SNI, HANDSHAKE_HOOKS_CERT, HANDSHAKE_HOOKS_CERT_INVOKE, @@ -378,8 +421,10 @@ class SSLNetVConnection : public UnixNetVConnection Continuation *npnEndpoint = nullptr; SessionAccept *sessionAcceptPtr = nullptr; bool sslTrace = false; - bool SNIMapping = false; int64_t redoWriteSize = 0; + char *tunnel_host = nullptr; + in_port_t tunnel_port = 0; + bool tunnel_decrypt = false; }; typedef int (SSLNetVConnection::*SSLNetVConnHandler)(int, void *); diff --git a/iocore/net/P_SSLNextProtocolSet.h b/iocore/net/P_SSLNextProtocolSet.h index 067690666e6..61d1ded9a8b 100644 --- a/iocore/net/P_SSLNextProtocolSet.h +++ b/iocore/net/P_SSLNextProtocolSet.h @@ -60,8 +60,8 @@ class SSLNextProtocolSet SSLNextProtocolSet &operator=(const SSLNextProtocolSet &) = delete; // disabled private: - mutable unsigned char *npn; - mutable size_t npnsz; + mutable unsigned char *npn = nullptr; + mutable size_t npnsz = 0; NextProtocolEndpoint::list_type endpoints; }; diff --git a/iocore/net/P_SSLSNI.h b/iocore/net/P_SSLSNI.h index 33543da8c53..14c23941251 100644 --- a/iocore/net/P_SSLSNI.h +++ b/iocore/net/P_SSLSNI.h @@ -31,7 +31,6 @@ #pragma once #include "ProxyConfig.h" -#include "tscore/Map.h" #include "P_SNIActionPerformer.h" #include "tscore/MatcherUtils.h" #include "openssl/ossl_typ.h" @@ -39,34 +38,81 @@ #include #include "YamlSNIConfig.h" +#include + // Properties for the next hop server struct NextHopProperty { - const char *name = nullptr; // name of the server - YamlSNIConfig::Policy verifyServerPolicy = YamlSNIConfig::Policy::DISABLED; // whether to verify the next hop - YamlSNIConfig::Property verifyServerProperties = YamlSNIConfig::Property::NONE; // what to verify on the next hop + std::string name; // name of the server + YamlSNIConfig::Policy verifyServerPolicy = YamlSNIConfig::Policy::UNSET; // whether to verify the next hop + YamlSNIConfig::Property verifyServerProperties = YamlSNIConfig::Property::UNSET; // what to verify on the next hop SSL_CTX *ctx = nullptr; // ctx generated off the certificate to present to this server - NextHopProperty(); + NextHopProperty() {} +}; + +using actionVector = std::vector>; + +struct namedElement { +public: + namedElement() {} + + void + setGlobName(std::string name) + { + std::string::size_type pos = 0; + while ((pos = name.find('.', pos)) != std::string::npos) { + name.replace(pos, 1, "\\."); + pos += 2; + } + pos = 0; + while ((pos = name.find('*', pos)) != std::string::npos) { + name.replace(pos, 1, ".{0,}"); + } + Debug("ssl_sni", "Regexed fqdn=%s", name.c_str()); + setRegexName(name); + } + + void + setRegexName(const std::string ®exName) + { + const char *err_ptr; + int err_offset = 0; + if (!regexName.empty()) { + match = pcre_compile(regexName.c_str(), PCRE_ANCHORED, &err_ptr, &err_offset, nullptr); + } else { + match = nullptr; + } + } + + pcre *match = nullptr; +}; + +struct actionElement : public namedElement { +public: + actionVector actions; +}; + +struct NextHopItem : public namedElement { +public: + NextHopProperty prop; }; -typedef std::vector actionVector; -typedef HashMap SNIMap; -typedef HashMap NextHopPropertyTable; +// typedef HashMap SNIMap; +typedef std::vector SNIList; +// typedef HashMap NextHopPropertyTable; +typedef std::vector NextHopPropertyList; struct SNIConfigParams : public ConfigInfo { char *sni_filename = nullptr; - SNIMap sni_action_map; - SNIMap wild_sni_action_map; - NextHopPropertyTable next_hop_table; - NextHopPropertyTable wild_next_hop_table; + SNIList sni_action_list; + NextHopPropertyList next_hop_list; YamlSNIConfig Y_sni; - NextHopProperty *getPropertyConfig(cchar *servername) const; + const NextHopProperty *getPropertyConfig(const std::string &servername) const; SNIConfigParams(); ~SNIConfigParams() override; void cleanup(); int Initialize(); void loadSNIConfig(); - actionVector *get(cchar *servername) const; - void printSNImap() const; + const actionVector *get(const std::string &servername) const; }; struct SNIConfig { diff --git a/iocore/net/P_SSLUtils.h b/iocore/net/P_SSLUtils.h index b165fd6b82f..932a8c3303d 100644 --- a/iocore/net/P_SSLUtils.h +++ b/iocore/net/P_SSLUtils.h @@ -21,10 +21,6 @@ #pragma once -#include "tscore/ink_config.h" -#include "tscore/Diags.h" -#include "P_SSLClientUtils.h" - #define OPENSSL_THREAD_DEFINES // BoringSSL does not have this include file @@ -33,140 +29,86 @@ #endif #include -#include +#include "tscore/ink_config.h" +#include "tscore/Diags.h" +#include "records/I_RecCore.h" +#include "P_SSLCertLookup.h" struct SSLConfigParams; -struct SSLCertLookup; class SSLNetVConnection; -struct RecRawStatBlock; typedef int ssl_error_t; -enum SSL_Stats { - ssl_origin_server_expired_cert_stat, - ssl_user_agent_expired_cert_stat, - ssl_origin_server_revoked_cert_stat, - ssl_user_agent_revoked_cert_stat, - ssl_origin_server_unknown_cert_stat, - ssl_user_agent_unknown_cert_stat, - ssl_origin_server_cert_verify_failed_stat, - ssl_user_agent_cert_verify_failed_stat, - ssl_origin_server_bad_cert_stat, - ssl_user_agent_bad_cert_stat, - ssl_origin_server_decryption_failed_stat, - ssl_user_agent_decryption_failed_stat, - ssl_origin_server_wrong_version_stat, - ssl_user_agent_wrong_version_stat, - ssl_origin_server_other_errors_stat, - ssl_user_agent_other_errors_stat, - ssl_origin_server_unknown_ca_stat, - ssl_user_agent_unknown_ca_stat, - ssl_user_agent_sessions_stat, - ssl_user_agent_session_hit_stat, - ssl_user_agent_session_miss_stat, - ssl_user_agent_session_timeout_stat, - ssl_total_handshake_time_stat, - ssl_total_success_handshake_count_in_stat, - ssl_total_tickets_created_stat, - ssl_total_tickets_verified_stat, - ssl_total_tickets_verified_old_key_stat, // verified with old key. - ssl_total_ticket_keys_renewed_stat, // number of keys renewed. - ssl_total_tickets_not_found_stat, - ssl_total_tickets_renewed_stat, - ssl_total_dyn_def_tls_record_count, - ssl_total_dyn_max_tls_record_count, - ssl_total_dyn_redo_tls_record_count, - ssl_session_cache_hit, - ssl_session_cache_miss, - ssl_session_cache_eviction, - ssl_session_cache_lock_contention, - ssl_session_cache_new_session, - - /* error stats */ - ssl_error_want_write, - ssl_error_want_read, - ssl_error_want_x509_lookup, - ssl_error_syscall, - ssl_error_read_eos, - ssl_error_zero_return, - ssl_error_ssl, - ssl_sni_name_set_failure, - ssl_total_success_handshake_count_out_stat, - - /* ocsp stapling stats */ - ssl_ocsp_revoked_cert_stat, - ssl_ocsp_unknown_cert_stat, - ssl_ocsp_refreshed_cert_stat, - ssl_ocsp_refresh_cert_failure_stat, - - ssl_cipher_stats_start = 100, - ssl_cipher_stats_end = 300, - - Ssl_Stat_Count +/** + @brief Gather user provided settings from ssl_multicert.config in to this single struct + */ +struct SSLMultiCertConfigParams { + SSLMultiCertConfigParams() { REC_ReadConfigInt32(session_ticket_enabled, "proxy.config.ssl.server.session_ticket.enable"); } + + int session_ticket_enabled; ///< session ticket enabled + ats_scoped_str addr; ///< IPv[64] address to match + ats_scoped_str cert; ///< certificate + ats_scoped_str first_cert; ///< the first certificate name when multiple cert files are in 'ssl_cert_name' + ats_scoped_str ca; ///< CA public certificate + ats_scoped_str key; ///< Private key + ats_scoped_str ocsp_response; ///< prefetched OCSP response + ats_scoped_str dialog; ///< Private key dialog + ats_scoped_str servername; ///< Destination server + SSLCertContext::Option opt = SSLCertContext::OPT_NONE; ///< SSLCertContext special handling option }; -extern RecRawStatBlock *ssl_rsb; +/** + @brief Load SSL certificates from ssl_multicert.config and setup SSLCertLookup for SSLCertificateConfig + */ +class SSLMultiCertConfigLoader +{ +public: + SSLMultiCertConfigLoader(const SSLConfigParams *p) : _params(p) {} + virtual ~SSLMultiCertConfigLoader(){}; + + bool load(SSLCertLookup *lookup); + + virtual SSL_CTX *default_server_ssl_ctx(); + virtual SSL_CTX *init_server_ssl_ctx(std::vector &certList, const SSLMultiCertConfigParams *sslMultCertSettings); + + static bool load_certs(SSL_CTX *ctx, std::vector &certList, const SSLConfigParams *params, + const SSLMultiCertConfigParams *ssl_multi_cert_params); + static bool set_session_id_context(SSL_CTX *ctx, const SSLConfigParams *params, + const SSLMultiCertConfigParams *sslMultCertSettings); -/* Stats should only be accessed using these macros */ -#define SSL_INCREMENT_DYN_STAT(x) RecIncrRawStat(ssl_rsb, nullptr, (int)x, 1) -#define SSL_DECREMENT_DYN_STAT(x) RecIncrRawStat(ssl_rsb, nullptr, (int)x, -1) -#define SSL_SET_COUNT_DYN_STAT(x, count) RecSetRawStatCount(ssl_rsb, x, count) -#define SSL_INCREMENT_DYN_STAT_EX(x, y) RecIncrRawStat(ssl_rsb, nullptr, (int)x, y) -#define SSL_CLEAR_DYN_STAT(x) \ - do { \ - RecSetRawStatSum(ssl_rsb, (x), 0); \ - RecSetRawStatCount(ssl_rsb, (x), 0); \ - } while (0) + static bool index_certificate(SSLCertLookup *lookup, SSLCertContext const &cc, X509 *cert, const char *certname); + static int check_server_cert_now(X509 *cert, const char *certname); + static void clear_pw_references(SSL_CTX *ssl_ctx); -// Create a default SSL server context. -SSL_CTX *SSLDefaultServerContext(); +protected: + const SSLConfigParams *_params; + +private: + virtual SSL_CTX *_store_ssl_ctx(SSLCertLookup *lookup, const SSLMultiCertConfigParams *ssl_multi_cert_params); + virtual void _set_handshake_callbacks(SSL_CTX *ctx); +}; // Create a new SSL server context fully configured. +// Used by TS API (TSSslServerContextCreate) SSL_CTX *SSLCreateServerContext(const SSLConfigParams *params); +// Release SSL_CTX and the associated data. This works for both +// client and server contexts and gracefully accepts nullptr. +// Used by TS API (TSSslContextDestroy) +void SSLReleaseContext(SSL_CTX *ctx); + // Initialize the SSL library. void SSLInitializeLibrary(); // Initialize SSL library based on configuration settings void SSLPostConfigInitialize(); -// Initialize SSL statistics. -void SSLInitializeStatistics(); - -// Release SSL_CTX and the associated data. This works for both -// client and server contexts and gracefully accepts nullptr. -void SSLReleaseContext(SSL_CTX *ctx); - // Wrapper functions to SSL I/O routines ssl_error_t SSLWriteBuffer(SSL *ssl, const void *buf, int64_t nbytes, int64_t &nwritten); ssl_error_t SSLReadBuffer(SSL *ssl, void *buf, int64_t nbytes, int64_t &nread); ssl_error_t SSLAccept(SSL *ssl); ssl_error_t SSLConnect(SSL *ssl); -// Log an SSL error. -#define SSLError(fmt, ...) SSLDiagnostic(MakeSourceLocation(), false, nullptr, fmt, ##__VA_ARGS__) -#define SSLErrorVC(vc, fmt, ...) SSLDiagnostic(MakeSourceLocation(), false, (vc), fmt, ##__VA_ARGS__) -// Log a SSL diagnostic using the "ssl" diagnostic tag. -#define SSLDebug(fmt, ...) SSLDiagnostic(MakeSourceLocation(), true, nullptr, fmt, ##__VA_ARGS__) -#define SSLVCDebug(vc, fmt, ...) SSLDiagnostic(MakeSourceLocation(), true, (vc), fmt, ##__VA_ARGS__) - -#define SSL_CLR_ERR_INCR_DYN_STAT(vc, x, fmt, ...) \ - do { \ - SSLVCDebug((vc), fmt, ##__VA_ARGS__); \ - RecIncrRawStat(ssl_rsb, nullptr, (int)x, 1); \ - } while (0) - -void SSLDiagnostic(const SourceLocation &loc, bool debug, SSLNetVConnection *vc, const char *fmt, ...) TS_PRINTFLIKE(4, 5); - -// Return a static string name for a SSL_ERROR constant. -const char *SSLErrorName(int ssl_error); - -// Log a SSL network buffer. -void SSLDebugBufferPrint(const char *tag, const char *buffer, unsigned buflen, const char *message); - -// Load the SSL certificate configuration. -bool SSLParseCertificateConfiguration(const SSLConfigParams *params, SSLCertLookup *lookup); - // Attach a SSL NetVC back pointer to a SSL session. void SSLNetVCAttach(SSL *ssl, SSLNetVConnection *vc); @@ -177,6 +119,7 @@ void SSLNetVCDetach(SSL *ssl); SSLNetVConnection *SSLNetVCAccess(const SSL *ssl); void setClientCertLevel(SSL *ssl, uint8_t certLevel); +void setTLSValidProtocols(SSL *ssl, unsigned long proto_mask, unsigned long max_mask); namespace ssl { @@ -226,7 +169,7 @@ namespace detail struct ats_wildcard_matcher { ats_wildcard_matcher() { - if (regex.compile("^\\*\\.[^\\*.]+") != 0) { + if (!regex.compile(R"(^\*\.[^\*.]+)")) { Fatal("failed to compile TLS wildcard matching regex"); } } @@ -242,50 +185,5 @@ struct ats_wildcard_matcher { DFA regex; }; -class TunnelHashMap -{ -public: - struct HostStruct { - std::string hostname; - int port; - HostStruct(const std::string &name, int port_) : hostname(name), port(port_) {} - }; - using Tunnel_hashMap = std::unordered_map; - Tunnel_hashMap TunnelhMap; - - void - emplace(const std::string &key, const std::string &hostname) - { - std::string_view addr, port; - if (ats_ip_parse(std::string_view(hostname), &addr, &port) == 0) { - TunnelhMap.emplace(key, HostStruct(addr.data(), atoi(port.data()))); - } - } - - void - emplace(const std::string &key, const std::string &name, int port_) - { - TunnelhMap.emplace(key, HostStruct(name, port_)); - } - - Tunnel_hashMap::const_iterator - find(const std::string &key) const - { - return TunnelhMap.find(key); - } - - Tunnel_hashMap::const_iterator - begin() const - { - return TunnelhMap.begin(); - } - - Tunnel_hashMap::const_iterator - end() const - { - return TunnelhMap.end(); - } -}; - typedef ats_scoped_resource scoped_X509; typedef ats_scoped_resource scoped_BIO; diff --git a/iocore/net/P_Socks.h b/iocore/net/P_Socks.h index aba0a0b867b..2f875c1e4e3 100644 --- a/iocore/net/P_Socks.h +++ b/iocore/net/P_Socks.h @@ -39,20 +39,20 @@ enum { }; struct socks_conf_struct { - int socks_needed; - int server_connect_timeout; - int socks_timeout; - unsigned char default_version; - char *user_name_n_passwd; - int user_name_n_passwd_len; + int socks_needed = 0; + int server_connect_timeout = 0; + int socks_timeout = 100; + unsigned char default_version = 5; + char *user_name_n_passwd = nullptr; + int user_name_n_passwd_len = 0; - int per_server_connection_attempts; - int connection_attempts; + int per_server_connection_attempts = 1; + int connection_attempts = 0; // the following ports are used by SocksProxy - int accept_enabled; - int accept_port; - unsigned short http_port; + int accept_enabled = 0; + int accept_port = 0; + unsigned short http_port = 1080; #ifdef SOCKS_WITH_TS IpMap ip_map; @@ -63,17 +63,7 @@ struct socks_conf_struct { #endif socks_conf_struct() - : socks_needed(0), - server_connect_timeout(0), - socks_timeout(100), - default_version(5), - user_name_n_passwd(nullptr), - user_name_n_passwd_len(0), - per_server_connection_attempts(1), - connection_attempts(0), - accept_enabled(0), - accept_port(0), - http_port(1080) + { #if !defined(SOCKS_WITH_TS) memset(&server_addr, 0, sizeof(server_addr)); diff --git a/iocore/net/P_UDPConnection.h b/iocore/net/P_UDPConnection.h index 011ccae2287..ef69b31f68b 100644 --- a/iocore/net/P_UDPConnection.h +++ b/iocore/net/P_UDPConnection.h @@ -38,14 +38,14 @@ class UDPConnectionInternal : public UDPConnection UDPConnectionInternal(); ~UDPConnectionInternal() override; - Continuation *continuation; - int recvActive; // interested in receiving - int refcount; // public for assertion + Continuation *continuation = nullptr; + int recvActive = 0; // interested in receiving + int refcount = 0; // public for assertion SOCKET fd; IpEndpoint binding; - int binding_valid; - int tobedestroyed; + int binding_valid = 0; + int tobedestroyed = 0; int sendGenerationNum; int64_t lastSentPktTSSeqNum; @@ -60,8 +60,7 @@ class UDPConnectionInternal : public UDPConnection }; TS_INLINE -UDPConnectionInternal::UDPConnectionInternal() - : continuation(nullptr), recvActive(0), refcount(0), fd(-1), binding_valid(0), tobedestroyed(0) +UDPConnectionInternal::UDPConnectionInternal() : fd(-1) { sendGenerationNum = 0; lastSentPktTSSeqNum = -1; @@ -80,13 +79,13 @@ UDPConnectionInternal::~UDPConnectionInternal() TS_INLINE SOCKET UDPConnection::getFd() { - return ((UDPConnectionInternal *)this)->fd; + return static_cast(this)->fd; } TS_INLINE void UDPConnection::setBinding(struct sockaddr const *s) { - UDPConnectionInternal *p = (UDPConnectionInternal *)this; + UDPConnectionInternal *p = static_cast(this); ats_ip_copy(&p->binding, s); p->binding_valid = 1; } @@ -94,7 +93,7 @@ UDPConnection::setBinding(struct sockaddr const *s) TS_INLINE void UDPConnection::setBinding(IpAddr const &ip, in_port_t port) { - UDPConnectionInternal *p = (UDPConnectionInternal *)this; + UDPConnectionInternal *p = static_cast(this); IpEndpoint addr; addr.assign(ip, htons(port)); ats_ip_copy(&p->binding, addr); @@ -104,7 +103,7 @@ UDPConnection::setBinding(IpAddr const &ip, in_port_t port) TS_INLINE int UDPConnection::getBinding(struct sockaddr *s) { - UDPConnectionInternal *p = (UDPConnectionInternal *)this; + UDPConnectionInternal *p = static_cast(this); ats_ip_copy(s, &p->binding); return p->binding_valid; } @@ -112,13 +111,13 @@ UDPConnection::getBinding(struct sockaddr *s) TS_INLINE void UDPConnection::destroy() { - ((UDPConnectionInternal *)this)->tobedestroyed = 1; + static_cast(this)->tobedestroyed = 1; } TS_INLINE int UDPConnection::shouldDestroy() { - return ((UDPConnectionInternal *)this)->tobedestroyed; + return static_cast(this)->tobedestroyed; } TS_INLINE void @@ -130,25 +129,25 @@ UDPConnection::AddRef() TS_INLINE int UDPConnection::GetRefCount() { - return ((UDPConnectionInternal *)this)->refcount; + return static_cast(this)->refcount; } TS_INLINE int UDPConnection::GetSendGenerationNumber() { - return ((UDPConnectionInternal *)this)->sendGenerationNum; + return static_cast(this)->sendGenerationNum; } TS_INLINE int -UDPConnection::getPortNum(void) +UDPConnection::getPortNum() { return ats_ip_port_host_order(&static_cast(this)->binding); } TS_INLINE int64_t -UDPConnection::cancel(void) +UDPConnection::cancel() { - UDPConnectionInternal *p = (UDPConnectionInternal *)this; + UDPConnectionInternal *p = static_cast(this); p->sendGenerationNum++; p->lastPktStartTime = p->lastSentPktStartTime; @@ -158,7 +157,7 @@ UDPConnection::cancel(void) TS_INLINE void UDPConnection::SetLastSentPktTSSeqNum(int64_t sentSeqNum) { - ((UDPConnectionInternal *)this)->lastSentPktTSSeqNum = sentSeqNum; + static_cast(this)->lastSentPktTSSeqNum = sentSeqNum; } TS_INLINE void @@ -166,6 +165,6 @@ UDPConnection::setContinuation(Continuation *c) { // it is not safe to switch among continuations that don't share locks ink_assert(mutex.get() == nullptr || c->mutex == mutex); - mutex = c->mutex; - ((UDPConnectionInternal *)this)->continuation = c; + mutex = c->mutex; + static_cast(this)->continuation = c; } diff --git a/iocore/net/P_UDPIOEvent.h b/iocore/net/P_UDPIOEvent.h index b89c9c7651d..593929e0018 100644 --- a/iocore/net/P_UDPIOEvent.h +++ b/iocore/net/P_UDPIOEvent.h @@ -28,7 +28,7 @@ class UDPIOEvent : public Event { public: - UDPIOEvent() : fd(-1), err(0), m(nullptr), handle(nullptr), b(nullptr), bytesTransferred(0){}; + UDPIOEvent() : b(nullptr){}; ~UDPIOEvent() override{}; void @@ -90,12 +90,12 @@ class UDPIOEvent : public Event private: void *operator new(size_t size); // undefined - int fd; - int err; // error code - struct msghdr *m; - void *handle; // some extra data for the client handler - Ptr b; // holds buffer that I/O will go to - int bytesTransferred; // actual bytes transferred + int fd = -1; + int err = 0; // error code + struct msghdr *m = nullptr; + void *handle = nullptr; // some extra data for the client handler + Ptr b; // holds buffer that I/O will go to + int bytesTransferred = 0; // actual bytes transferred }; extern ClassAllocator UDPIOEventAllocator; diff --git a/iocore/net/P_UDPNet.h b/iocore/net/P_UDPNet.h index 8d803a022ad..be2d896ef8d 100644 --- a/iocore/net/P_UDPNet.h +++ b/iocore/net/P_UDPNet.h @@ -61,22 +61,18 @@ extern UDPNetProcessorInternal udpNetInternal; class PacketQueue { public: - PacketQueue() : nPackets(0), now_slot(0) - { - lastPullLongTermQ = 0; - init(); - } + PacketQueue() { init(); } virtual ~PacketQueue() {} - int nPackets; - ink_hrtime lastPullLongTermQ; + int nPackets = 0; + ink_hrtime lastPullLongTermQ = 0; Queue longTermQ; Queue bucket[N_SLOTS]; ink_hrtime delivery_time[N_SLOTS]; - int now_slot; + int now_slot = 0; void - init(void) + init() { now_slot = 0; ink_hrtime now = ink_get_hrtime_internal(); @@ -167,12 +163,12 @@ class PacketQueue void FreeCancelledPackets(int numSlots) { - UDPPacketInternal *p; Queue tempQ; - int i, s; + int i; for (i = 0; i < numSlots; i++) { - s = (now_slot + i) % N_SLOTS; + int s = (now_slot + i) % N_SLOTS; + UDPPacketInternal *p; while (nullptr != (p = bucket[s].dequeue())) { if (IsCancelledPacket(p)) { p->free(); @@ -191,14 +187,13 @@ class PacketQueue advanceNow(ink_hrtime t) { int s = now_slot; - int prev; if (ink_hrtime_to_msec(t - lastPullLongTermQ) >= SLOT_TIME_MSEC * ((N_SLOTS - 1) / 2)) { Queue tempQ; UDPPacketInternal *p; // pull in all the stuff from long-term slot lastPullLongTermQ = t; - // this is to handle wierdoness where someone is trying to queue a + // this is to handle weirdness where someone is trying to queue a // packet to be sent in SLOT_TIME_MSEC * N_SLOTS * (2+)---the packet // will get back to longTermQ and we'll have an infinite loop. while ((p = longTermQ.dequeue()) != nullptr) @@ -208,6 +203,8 @@ class PacketQueue } while (!bucket[s].head && (t > delivery_time[s] + SLOT_TIME)) { + int prev; + prev = (s + N_SLOTS - 1) % N_SLOTS; delivery_time[s] = delivery_time[prev] + SLOT_TIME; s = (s + 1) % N_SLOTS; @@ -269,12 +266,6 @@ class PacketQueue } return HRTIME_FOREVER; } - -private: - void - kill_cancelled_events() - { - } }; class UDPQueue @@ -303,7 +294,7 @@ class UDPQueue void initialize_thread_for_udp_net(EThread *thread); -class UDPNetHandler : public Continuation +class UDPNetHandler : public Continuation, public EThread::LoopTailHandler { public: // engine for outgoing packets @@ -319,12 +310,16 @@ class UDPNetHandler : public Continuation Que(UnixUDPConnection, callback_link) udp_callbacks; Event *trigger_event = nullptr; + EThread *thread = nullptr; ink_hrtime nextCheck; ink_hrtime lastCheck; int startNetEvent(int event, Event *data); int mainNetEvent(int event, Event *data); + int waitForActivity(ink_hrtime timeout) override; + void signalActivity() override; + UDPNetHandler(); }; @@ -332,11 +327,11 @@ struct PollCont; static inline PollCont * get_UDPPollCont(EThread *t) { - return (PollCont *)ETHREAD_GET_PTR(t, udpNetInternal.pollCont_offset); + return static_cast(ETHREAD_GET_PTR(t, udpNetInternal.pollCont_offset)); } static inline UDPNetHandler * get_UDPNetHandler(EThread *t) { - return (UDPNetHandler *)ETHREAD_GET_PTR(t, udpNetInternal.udpNetHandler_offset); + return static_cast(ETHREAD_GET_PTR(t, udpNetInternal.udpNetHandler_offset)); } diff --git a/iocore/net/P_UDPPacket.h b/iocore/net/P_UDPPacket.h index 0b6363a2f5b..38364e699b5 100644 --- a/iocore/net/P_UDPPacket.h +++ b/iocore/net/P_UDPPacket.h @@ -44,24 +44,24 @@ class UDPPacketInternal : public UDPPacket SLINK(UDPPacketInternal, alink); // atomic link // packet scheduling stuff: keep it a doubly linked list - uint64_t pktLength; + uint64_t pktLength = 0; - int reqGenerationNum; - ink_hrtime delivery_time; // when to deliver packet + int reqGenerationNum = 0; + ink_hrtime delivery_time = 0; // when to deliver packet Ptr chain; - Continuation *cont; // callback on error - UDPConnectionInternal *conn; // connection where packet should be sent to. + Continuation *cont = nullptr; // callback on error + UDPConnectionInternal *conn = nullptr; // connection where packet should be sent to. - int in_the_priority_queue; - int in_heap; + int in_the_priority_queue = 0; + int in_heap = 0; }; inkcoreapi extern ClassAllocator udpPacketAllocator; TS_INLINE UDPPacketInternal::UDPPacketInternal() - : pktLength(0), reqGenerationNum(0), delivery_time(0), cont(nullptr), conn(nullptr), in_the_priority_queue(0), in_heap(0) + { memset(&from, '\0', sizeof(from)); memset(&to, '\0', sizeof(to)); @@ -86,7 +86,7 @@ UDPPacketInternal::free() TS_INLINE void UDPPacket::append_block(IOBufferBlock *block) { - UDPPacketInternal *p = (UDPPacketInternal *)this; + UDPPacketInternal *p = static_cast(this); if (block) { if (p->chain) { // append to end @@ -104,7 +104,7 @@ UDPPacket::append_block(IOBufferBlock *block) TS_INLINE int64_t UDPPacket::getPktLength() const { - UDPPacketInternal *p = (UDPPacketInternal *)this; + UDPPacketInternal *p = const_cast(static_cast(this)); IOBufferBlock *b; p->pktLength = 0; @@ -119,13 +119,13 @@ UDPPacket::getPktLength() const TS_INLINE void UDPPacket::free() { - ((UDPPacketInternal *)this)->free(); + static_cast(this)->free(); } TS_INLINE void UDPPacket::setContinuation(Continuation *c) { - ((UDPPacketInternal *)this)->cont = c; + static_cast(this)->cont = c; } TS_INLINE void @@ -138,7 +138,7 @@ UDPPacket::setConnection(UDPConnection *c) assert will prevent that. The "if" clause enables correct handling of the connection ref. counts in such a scenario. */ - UDPConnectionInternal *&conn = ((UDPPacketInternal *)this)->conn; + UDPConnectionInternal *&conn = static_cast(this)->conn; if (conn) { if (conn == c) @@ -146,21 +146,21 @@ UDPPacket::setConnection(UDPConnection *c) conn->Release(); conn = nullptr; } - conn = (UDPConnectionInternal *)c; + conn = static_cast(c); conn->AddRef(); } TS_INLINE IOBufferBlock * -UDPPacket::getIOBlockChain(void) +UDPPacket::getIOBlockChain() { ink_assert(dynamic_cast(this) != nullptr); - return ((UDPPacketInternal *)this)->chain.get(); + return static_cast(this)->chain.get(); } TS_INLINE UDPConnection * -UDPPacket::getConnection(void) +UDPPacket::getConnection() { - return ((UDPPacketInternal *)this)->conn; + return static_cast(this)->conn; } TS_INLINE UDPPacket * @@ -189,7 +189,6 @@ new_UDPPacket(struct sockaddr const *to, ink_hrtime when, IOBufferBlock *buf, in { (void)len; UDPPacketInternal *p = udpPacketAllocator.alloc(); - IOBufferBlock *body; p->in_the_priority_queue = 0; p->in_heap = 0; @@ -197,7 +196,7 @@ new_UDPPacket(struct sockaddr const *to, ink_hrtime when, IOBufferBlock *buf, in ats_ip_copy(&p->to, to); while (buf) { - body = buf->clone(); + IOBufferBlock *body = buf->clone(); p->append_block(body); buf = buf->next.get(); } diff --git a/iocore/net/P_UnixNet.h b/iocore/net/P_UnixNet.h index 0440b7238dc..7d591b01f9c 100644 --- a/iocore/net/P_UnixNet.h +++ b/iocore/net/P_UnixNet.h @@ -343,7 +343,7 @@ class NetHandler : public Continuation, public EThread::LoopTailHandler /** Release a netvc and free it. - @param netvc UnixNetVConnection to be deattached. + @param netvc UnixNetVConnection to be detached. */ void free_netvc(UnixNetVConnection *netvc); @@ -353,7 +353,7 @@ class NetHandler : public Continuation, public EThread::LoopTailHandler void _close_vc(UnixNetVConnection *vc, ink_hrtime now, int &handle_event, int &closed, int &total_idle_time, int &total_idle_count); - /// Static method used as the callbackf for runtime configuration updates. + /// Static method used as the callback for runtime configuration updates. static int update_nethandler_config(const char *name, RecDataT, RecData data, void *); }; diff --git a/iocore/net/P_UnixNetProcessor.h b/iocore/net/P_UnixNetProcessor.h index b076c0ae00f..87e16ba7540 100644 --- a/iocore/net/P_UnixNetProcessor.h +++ b/iocore/net/P_UnixNetProcessor.h @@ -43,6 +43,7 @@ struct UnixNetProcessor : public NetProcessor { NetVConnection *allocate_vc(EThread *t) override; void init() override; + void init_socks() override; Event *accept_thread_event; @@ -66,8 +67,6 @@ extern UnixNetProcessor unix_netProcessor; // // Set up a thread to receive events from the NetProcessor // This function should be called for all threads created to -// accept such events by the EventProcesor. +// accept such events by the EventProcessor. // extern void initialize_thread_for_net(EThread *thread); - -//#include "UnixNet.h" diff --git a/iocore/net/P_UnixNetState.h b/iocore/net/P_UnixNetState.h index 10cbf4c75e7..3a2265c92c9 100644 --- a/iocore/net/P_UnixNetState.h +++ b/iocore/net/P_UnixNetState.h @@ -43,12 +43,12 @@ class Event; class UnixNetVConnection; struct NetState { - int enabled; + int enabled = 0; VIO vio; Link ready_link; SLink enable_link; - int in_enabled_list; - int triggered; + int in_enabled_list = 0; + int triggered = 0; - NetState() : enabled(0), vio(VIO::NONE), in_enabled_list(0), triggered(0) {} + NetState() : vio(VIO::NONE) {} }; diff --git a/iocore/net/P_UnixNetVConnection.h b/iocore/net/P_UnixNetVConnection.h index 1516ddda2f8..e4e0483239f 100644 --- a/iocore/net/P_UnixNetVConnection.h +++ b/iocore/net/P_UnixNetVConnection.h @@ -41,7 +41,7 @@ class UnixNetVConnection; class NetHandler; struct PollDescriptor; -TS_INLINE void +inline void NetVCOptions::reset() { ip_proto = USE_TCP; @@ -66,12 +66,14 @@ NetVCOptions::reset() etype = ET_NET; - sni_servername = nullptr; - ssl_servername = nullptr; - clientCertificate = nullptr; + sni_servername = nullptr; + ssl_servername = nullptr; + ssl_client_cert_name = nullptr; + ssl_client_private_key_name = nullptr; + ssl_client_ca_cert_name = nullptr; } -TS_INLINE void +inline void NetVCOptions::set_sock_param(int _recv_bufsize, int _send_bufsize, unsigned long _opt_flags, unsigned long _packet_mark, unsigned long _packet_tos) { @@ -140,7 +142,7 @@ class UnixNetVConnection : public NetVConnection //////////////////////////////////////////////////////////// // Set the timeouts associated with this connection. // - // active_timeout is for the total elasped time of // + // active_timeout is for the total elapsed time of // // the connection. // // inactivity_timeout is the elapsed time from the time // // a read or a write was scheduled during which the // @@ -230,7 +232,7 @@ class UnixNetVConnection : public NetVConnection UnixNetVConnection *migrateToCurrentThread(Continuation *c, EThread *t); Action action_; - int closed; + int closed = 0; NetState read; NetState write; @@ -242,14 +244,14 @@ class UnixNetVConnection : public NetVConnection LINK(UnixNetVConnection, keep_alive_queue_link); LINK(UnixNetVConnection, active_queue_link); - ink_hrtime inactivity_timeout_in; - ink_hrtime active_timeout_in; - ink_hrtime next_inactivity_timeout_at; - ink_hrtime next_activity_timeout_at; + ink_hrtime inactivity_timeout_in = 0; + ink_hrtime active_timeout_in = 0; + ink_hrtime next_inactivity_timeout_at = 0; + ink_hrtime next_activity_timeout_at = 0; EventIO ep; - NetHandler *nh; - unsigned int id; + NetHandler *nh = nullptr; + unsigned int id = 0; union { unsigned int flags; @@ -262,11 +264,11 @@ class UnixNetVConnection : public NetVConnection }; Connection con; - int recursion; - ink_hrtime submit_time; - OOB_callback *oob_ptr; - bool from_accept_thread; - NetAccept *accept_object; + int recursion = 0; + ink_hrtime submit_time = 0; + OOB_callback *oob_ptr = nullptr; + bool from_accept_thread = false; + NetAccept *accept_object = nullptr; // es - origin_trace associated connections bool origin_trace; @@ -290,7 +292,9 @@ class UnixNetVConnection : public NetVConnection ink_hrtime get_active_timeout() override; void set_local_addr() override; + void set_mptcp_state() override; void set_remote_addr() override; + void set_remote_addr(const sockaddr *) override; int set_tcp_init_cwnd(int init_cwnd) override; int set_tcp_congestion_control(int side) override; void apply_options() override; @@ -320,14 +324,21 @@ extern ClassAllocator netVCAllocator; typedef int (UnixNetVConnection::*NetVConnHandler)(int, void *); -TS_INLINE void +inline void UnixNetVConnection::set_remote_addr() { ats_ip_copy(&remote_addr, &con.addr); this->control_flags.set_flag(ContFlags::DEBUG_OVERRIDE, diags->test_override_ip(remote_addr)); } -TS_INLINE void +inline void +UnixNetVConnection::set_remote_addr(const sockaddr *new_sa) +{ + ats_ip_copy(&remote_addr, new_sa); + this->control_flags.set_flag(ContFlags::DEBUG_OVERRIDE, diags->test_override_ip(remote_addr)); +} + +inline void UnixNetVConnection::set_local_addr() { int local_sa_size = sizeof(local_addr); @@ -337,19 +348,34 @@ UnixNetVConnection::set_local_addr() ATS_UNUSED_RETURN(safe_getsockname(con.fd, &local_addr.sa, &local_sa_size)); } -TS_INLINE ink_hrtime +// Update the internal VC state variable for MPTCP +inline void +UnixNetVConnection::set_mptcp_state() +{ + int mptcp_enabled = -1; + int mptcp_enabled_size = sizeof(mptcp_enabled); + + if (0 == safe_getsockopt(con.fd, IPPROTO_TCP, MPTCP_ENABLED, (char *)&mptcp_enabled, &mptcp_enabled_size)) { + Debug("socket_mptcp", "MPTCP socket state: %d", mptcp_enabled); + mptcp_state = mptcp_enabled > 0 ? true : false; + } else { + Debug("socket_mptcp", "MPTCP failed getsockopt(): %s", strerror(errno)); + } +} + +inline ink_hrtime UnixNetVConnection::get_active_timeout() { return active_timeout_in; } -TS_INLINE ink_hrtime +inline ink_hrtime UnixNetVConnection::get_inactivity_timeout() { return inactivity_timeout_in; } -TS_INLINE void +inline void UnixNetVConnection::set_active_timeout(ink_hrtime timeout_in) { Debug("socket", "Set active timeout=%" PRId64 ", NetVC=%p", timeout_in, this); @@ -357,7 +383,7 @@ UnixNetVConnection::set_active_timeout(ink_hrtime timeout_in) next_activity_timeout_at = Thread::get_hrtime() + timeout_in; } -TS_INLINE void +inline void UnixNetVConnection::cancel_inactivity_timeout() { Debug("socket", "Cancel inactive timeout for NetVC=%p", this); @@ -365,7 +391,7 @@ UnixNetVConnection::cancel_inactivity_timeout() set_inactivity_timeout(0); } -TS_INLINE void +inline void UnixNetVConnection::cancel_active_timeout() { Debug("socket", "Cancel active timeout for NetVC=%p", this); @@ -373,7 +399,7 @@ UnixNetVConnection::cancel_active_timeout() next_activity_timeout_at = 0; } -TS_INLINE int +inline int UnixNetVConnection::set_tcp_init_cwnd(int init_cwnd) { #ifdef TCP_INIT_CWND @@ -388,56 +414,21 @@ UnixNetVConnection::set_tcp_init_cwnd(int init_cwnd) #endif } -TS_INLINE int -UnixNetVConnection::set_tcp_congestion_control(int side) -{ -#ifdef TCP_CONGESTION - RecString congestion_control; - int ret; - - if (side == CLIENT_SIDE) { - ret = REC_ReadConfigStringAlloc(congestion_control, "proxy.config.net.tcp_congestion_control_in"); - } else { - ret = REC_ReadConfigStringAlloc(congestion_control, "proxy.config.net.tcp_congestion_control_out"); - } - - if (ret == REC_ERR_OKAY) { - int len = strlen(congestion_control); - if (len > 0) { - int rv = 0; - rv = setsockopt(con.fd, IPPROTO_TCP, TCP_CONGESTION, reinterpret_cast(congestion_control), len); - if (rv < 0) { - Error("Unable to set TCP congestion control on socket %d to \"%.*s\", errno=%d (%s)", con.fd, len, congestion_control, - errno, strerror(errno)); - } else { - Debug("socket", "Setting TCP congestion control on socket [%d] to \"%.*s\" -> %d", con.fd, len, congestion_control, rv); - } - } - ats_free(congestion_control); - return 0; - } - return -1; -#else - Debug("socket", "Setting TCP congestion control is not supported on this platform."); - return -1; -#endif -} - -TS_INLINE UnixNetVConnection::~UnixNetVConnection() {} +inline UnixNetVConnection::~UnixNetVConnection() {} -TS_INLINE SOCKET +inline SOCKET UnixNetVConnection::get_socket() { return con.fd; } -TS_INLINE void +inline void UnixNetVConnection::set_action(Continuation *c) { action_ = c; } -TS_INLINE const Action * +inline const Action * UnixNetVConnection::get_action() const { return &action_; diff --git a/iocore/net/P_UnixPollDescriptor.h b/iocore/net/P_UnixPollDescriptor.h index f377cbdcf48..471d9e7647c 100644 --- a/iocore/net/P_UnixPollDescriptor.h +++ b/iocore/net/P_UnixPollDescriptor.h @@ -112,7 +112,7 @@ struct PollDescriptor { } return &pfd[nfds++]; #else - return 0; + return nullptr; #endif } diff --git a/iocore/net/ProxyProtocol.cc b/iocore/net/ProxyProtocol.cc new file mode 100644 index 00000000000..83c3bcf9dfe --- /dev/null +++ b/iocore/net/ProxyProtocol.cc @@ -0,0 +1,179 @@ +/** @file + * + * PROXY protocol definitions and parsers. + * + * @section license License + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "tscore/ink_assert.h" +#include "tscpp/util/TextView.h" +#include "ProxyProtocol.h" +#include "I_NetVConnection.h" + +bool +ssl_has_proxy_v1(NetVConnection *sslvc, char *buffer, int64_t *bytes_r) +{ + ts::TextView tv; + + tv.assign(buffer, *bytes_r); + + // Client must send at least 15 bytes to get a reasonable match. + if (tv.size() < PROXY_V1_CONNECTION_HEADER_LEN_MIN) { + Debug("proxyprotocol_v1", "ssl_has_proxy_v1: not enough recv'd"); + return false; + } + + // if we don't have the PROXY preface, we don't have a ProxyV1 header + if (0 != memcmp(PROXY_V1_CONNECTION_PREFACE, buffer, PROXY_V1_CONNECTION_PREFACE_LEN)) { + Debug("proxyprotocol_v1", "ssl_has_proxy_v1: failed the memcmp(%s, %s, %lu)", PROXY_V1_CONNECTION_PREFACE, buffer, + PROXY_V1_CONNECTION_PREFACE_LEN); + return false; + } + + // Find the terminating newline + ts::TextView::size_type pos = tv.find('\n'); + if (pos == tv.npos) { + Debug("proxyprotocol_v1", "ssl_has_proxy_v1: newline not found"); + return false; + } + + // Parse the TextView before moving the bytes in the buffer + if (!proxy_protov1_parse(sslvc, tv)) { + *bytes_r = -EAGAIN; + return false; + } + *bytes_r -= pos + 1; + if (*bytes_r <= 0) { + *bytes_r = -EAGAIN; + } else { + Debug("ssl", "Moving %" PRId64 " characters remaining in the buffer from %p to %p", *bytes_r, buffer + pos + 1, buffer); + memmove(buffer, buffer + pos + 1, *bytes_r); + } + return true; +} + +bool +http_has_proxy_v1(IOBufferReader *reader, NetVConnection *netvc) +{ + char buf[PROXY_V1_CONNECTION_HEADER_LEN_MAX + 1]; + ts::TextView tv; + + tv.assign(buf, reader->memcpy(buf, sizeof(buf), 0)); + + // Client must send at least 15 bytes to get a reasonable match. + if (tv.size() < PROXY_V1_CONNECTION_HEADER_LEN_MIN) { + return false; + } + + if (0 != memcmp(PROXY_V1_CONNECTION_PREFACE, buf, PROXY_V1_CONNECTION_PREFACE_LEN)) { + return false; + } + + // Find the terminating LF, which should already be in the buffer. + ts::TextView::size_type pos = tv.find('\n'); + if (pos == tv.npos) { // not found, it's not a proxy protocol header. + return false; + } + reader->consume(pos + 1); // clear out the header. + + // Now that we know we have a valid PROXY V1 preface, let's parse the + // remainder of the header + + return proxy_protov1_parse(netvc, tv); +} + +bool +proxy_protov1_parse(NetVConnection *netvc, ts::TextView hdr) +{ + static const std::string_view PREFACE{PROXY_V1_CONNECTION_PREFACE, PROXY_V1_CONNECTION_PREFACE_LEN}; + ts::TextView token; + in_port_t port; + + // All the cases are special and sequence, might as well unroll them. + + // The header should begin with the PROXY preface + token = hdr.split_prefix_at(' '); + if (0 == token.size() || token != PREFACE) { + Debug("proxyprotocol_v1", "proxy_protov1_parse: header [%.*s] does not start with preface [%.*s]", static_cast(hdr.size()), + hdr.data(), static_cast(PREFACE.size()), PREFACE.data()); + return false; + } + Debug("proxyprotocol_v1", "proxy_protov1_parse: [%.*s] = PREFACE", static_cast(token.size()), token.data()); + + // The INET protocol family - TCP4, TCP6 or UNKNOWN + token = hdr.split_prefix_at(' '); + if (0 == token.size()) { + return false; + } + Debug("proxyprotocol_v1", "proxy_protov1_parse: [%.*s] = INET Family", static_cast(token.size()), token.data()); + + // Next up is the layer 3 source address + // - 255.255.255.255 or ffff:f...f:ffff ffff:f...f:fff + token = hdr.split_prefix_at(' '); + if (0 == token.size()) { + return false; + } + Debug("proxyprotocol_v1", "proxy_protov1_parse: [%.*s] = Source Address", static_cast(token.size()), token.data()); + if (0 != netvc->set_proxy_protocol_src_addr(token)) { + return false; + } + + // Next is the layer3 destination address + // - 255.255.255.255 or ffff:f...f:ffff ffff:f...f:fff + token = hdr.split_prefix_at(' '); + if (0 == token.size()) { + return false; + } + Debug("proxyprotocol_v1", "proxy_protov1_parse: [%.*s] = Destination Address", static_cast(token.size()), token.data()); + if (0 != netvc->set_proxy_protocol_dst_addr(token)) { + return false; + } + + // Next is the TCP source port represented as a decimal number in the range of [0..65535] inclusive. + token = hdr.split_prefix_at(' '); + if (0 == token.size()) { + return false; + } + Debug("proxyprotocol_v1", "proxy_protov1_parse: [%.*s] = Source Port", static_cast(token.size()), token.data()); + + if (0 == (port = ts::svtoi(token))) { + Debug("proxyprotocol_v1", "proxy_protov1_parse: src port [%d] token [%.*s] failed to parse", port, + static_cast(token.size()), token.data()); + return false; + } + netvc->set_proxy_protocol_src_port(port); + + // Next is the TCP destination port represented as a decimal number in the range of [0..65535] inclusive. + // Final trailer is CR LF so split at CR. + token = hdr.split_prefix_at('\r'); + if (0 == token.size()) { + return false; + } + Debug("proxyprotocol_v1", "proxy_protov1_parse: [%.*s] = Destination Port", static_cast(token.size()), token.data()); + if (0 == (port = ts::svtoi(token))) { + Debug("proxyprotocol_v1", "proxy_protov1_parse: dst port [%d] token [%.*s] failed to parse", port, + static_cast(token.size()), token.data()); + return false; + } + netvc->set_proxy_protocol_dst_port(port); + + netvc->set_proxy_protocol_version(NetVConnection::ProxyProtocolVersion::V1); + + return true; +} diff --git a/iocore/net/ProxyProtocol.h b/iocore/net/ProxyProtocol.h new file mode 100644 index 00000000000..607dcdfbd30 --- /dev/null +++ b/iocore/net/ProxyProtocol.h @@ -0,0 +1,55 @@ +/** @file + + PROXY Protocol + + See: https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#ifndef ProxyProtocol_H_ +#define ProxyProtocol_H_ + +#include "tscore/ink_defs.h" +#include "tscore/ink_memory.h" +#include +#include +#include "I_VConnection.h" +#include "I_NetVConnection.h" +#include "I_IOBuffer.h" + +// http://www.haproxy.org/download/1.8/doc/proxy-protocol.txt + +extern bool proxy_protov1_parse(NetVConnection *, ts::TextView hdr); +extern bool ssl_has_proxy_v1(NetVConnection *, char *, int64_t *); +extern bool http_has_proxy_v1(IOBufferReader *, NetVConnection *); + +const char *const PROXY_V1_CONNECTION_PREFACE = R"(PROXY)"; +const char *const PROXY_V2_CONNECTION_PREFACE = "\x0D\x0A\x0D\x0A\x00\x0D\x0A\x51\x55\x49\x54\x0A\x02"; + +const size_t PROXY_V1_CONNECTION_PREFACE_LEN = strlen(PROXY_V1_CONNECTION_PREFACE); // 5 +const size_t PROXY_V2_CONNECTION_PREFACE_LEN = 13; + +const size_t PROXY_V1_CONNECTION_HEADER_LEN_MIN = 15; +const size_t PROXY_V2_CONNECTION_HEADER_LEN_MIN = 16; + +const size_t PROXY_V1_CONNECTION_HEADER_LEN_MAX = 108; +const size_t PROXY_V2_CONNECTION_HEADER_LEN_MAX = 16; + +#endif /* ProxyProtocol_H_ */ diff --git a/iocore/net/SNIActionPerformer.cc b/iocore/net/SNIActionPerformer.cc index d381e061009..842345e34b9 100644 --- a/iocore/net/SNIActionPerformer.cc +++ b/iocore/net/SNIActionPerformer.cc @@ -35,10 +35,10 @@ #include "P_SSLNextProtocolAccept.h" #include "P_SSLUtils.h" -extern Map snpsMap; +extern std::unordered_map snpsMap; int -SNIActionPerformer::PerformAction(Continuation *cont, cchar *servername) +SNIActionPerformer::PerformAction(Continuation *cont, const char *servername) { SNIConfig::scoped_config params; auto actionvec = params->get(servername); diff --git a/iocore/net/SSLCertLookup.cc b/iocore/net/SSLCertLookup.cc index 2c06ca61eb3..641f7abaa5b 100644 --- a/iocore/net/SSLCertLookup.cc +++ b/iocore/net/SSLCertLookup.cc @@ -21,50 +21,41 @@ limitations under the License. */ -#include "tscore/ink_config.h" - #include "P_SSLCertLookup.h" -#include "P_SSLUtils.h" -#include "P_SSLConfig.h" -#include "I_EventSystem.h" + +#include "tscore/ink_config.h" #include "tscore/I_Layout.h" #include "tscore/MatcherUtils.h" #include "tscore/Regex.h" #include "tscore/Trie.h" +#include "tscore/BufferWriter.h" +#include "tscore/bwf_std_format.h" #include "tscore/TestBox.h" +#include "I_EventSystem.h" + +#include "P_SSLUtils.h" +#include "P_SSLConfig.h" +#include "SSLSessionTicket.h" + #include #include #include -// Check if the ticket_key callback #define is available, and if so, enable session tickets. -#ifdef SSL_CTX_set_tlsext_ticket_key_cb - -#define HAVE_OPENSSL_SESSION_TICKETS 1 - -#endif /* SSL_CTX_set_tlsext_ticket_key_cb */ - struct SSLAddressLookupKey { - explicit SSLAddressLookupKey(const IpEndpoint &ip) : sep(0) + explicit SSLAddressLookupKey(const IpEndpoint &ip) { - static const char hextab[16] = {'0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f'}; - - int nbytes; - uint16_t port = ntohs(ip.port()); - - // For IP addresses, the cache key is the hex address with the port concatenated. This makes the lookup - // insensitive to address formatting and also allow the longest match semantic to produce different matches - // if there is a certificate on the port. - nbytes = ats_ip_to_hex(&ip.sa, key, sizeof(key)); - if (port) { - sep = nbytes; - key[nbytes++] = '.'; - key[nbytes++] = hextab[(port >> 12) & 0x000F]; - key[nbytes++] = hextab[(port >> 8) & 0x000F]; - key[nbytes++] = hextab[(port >> 4) & 0x000F]; - key[nbytes++] = hextab[(port)&0x000F]; + // For IP addresses, the cache key is the hex address with the port concatenated. This makes the + // lookup insensitive to address formatting and also allow the longest match semantic to produce + // different matches if there is a certificate on the port. + + ts::FixedBufferWriter w{key, sizeof(key)}; + w.print("{}", ts::bwf::Hex_Dump(ip)); // dump as raw hex bytes, don't format as IP address. + if (in_port_t port = ip.host_order_port(); port) { + sep = static_cast(w.size()); + w.print(".{:x}", port); } - key[nbytes++] = 0; + w.write('\0'); // force C-string termination. } const char * @@ -85,7 +76,7 @@ struct SSLAddressLookupKey { private: char key[(TS_IP6_SIZE * 2) /* hex addr */ + 1 /* dot */ + 4 /* port */ + 1 /* nullptr */]; - unsigned char sep; // offset of address/port separator + unsigned char sep = 0; // offset of address/port separator }; struct SSLContextStorage { @@ -122,14 +113,14 @@ struct SSLContextStorage { linkage required by @c Trie. */ struct ContextRef { - ContextRef() : idx(-1) {} + ContextRef() {} explicit ContextRef(int n) : idx(n) {} void Print() const { Debug("ssl", "Item=%p SSL_CTX=#%d", this, idx); } - int idx; ///< Index in the context store. + int idx = -1; ///< Index in the context store. LINK(ContextRef, link); ///< Require by @c Trie }; @@ -206,7 +197,7 @@ ticket_block_create(char *ticket_key_data, int ticket_key_len) ssl_ticket_key_block * ssl_create_ticket_keyblock(const char *ticket_key_path) { -#if HAVE_OPENSSL_SESSION_TICKETS +#if TS_HAVE_OPENSSL_SESSION_TICKETS ats_scoped_str ticket_key_data; int ticket_key_len; ssl_ticket_key_block *keyblock = nullptr; @@ -231,10 +222,10 @@ ssl_create_ticket_keyblock(const char *ticket_key_path) ticket_block_free(keyblock); return nullptr; -#else /* !HAVE_OPENSSL_SESSION_TICKETS */ +#else /* !TS_HAVE_OPENSSL_SESSION_TICKETS */ (void)ticket_key_path; return nullptr; -#endif /* HAVE_OPENSSL_SESSION_TICKETS */ +#endif /* TS_HAVE_OPENSSL_SESSION_TICKETS */ } void SSLCertContext::release() @@ -248,7 +239,7 @@ SSLCertContext::release() ctx = nullptr; } -SSLCertLookup::SSLCertLookup() : ssl_storage(new SSLContextStorage()), ssl_default(nullptr), is_valid(true) {} +SSLCertLookup::SSLCertLookup() : ssl_storage(new SSLContextStorage()) {} SSLCertLookup::~SSLCertLookup() { @@ -320,38 +311,6 @@ make_to_lower_case(const char *name, char *lower_case_name, int buf_len) lower_case_name[i] = '\0'; } -static char * -reverse_dns_name(const char *hostname, char (&reversed)[TS_MAX_HOST_NAME_LEN + 1]) -{ - char *ptr = reversed + sizeof(reversed); - const char *part = hostname; - - *(--ptr) = '\0'; // NUL-terminate - - while (*part) { - ssize_t len = strcspn(part, "."); - ssize_t remain = ptr - reversed; - - if (remain < (len + 1)) { - return nullptr; - } - - ptr -= len; - memcpy(ptr, part, len); - - // Skip to the next domain component. This will take us to either a '.' or a NUL. - // If it's a '.' we need to skip over it. - part += len; - if (*part == '.') { - ++part; - *(--ptr) = '.'; - } - } - make_to_lower_case(ptr, ptr, strlen(ptr) + 1); - - return ptr; -} - SSLContextStorage::SSLContextStorage() {} bool @@ -466,6 +425,38 @@ SSLContextStorage::lookup(const char *name) #if TS_HAS_TESTS +static char * +reverse_dns_name(const char *hostname, char (&reversed)[TS_MAX_HOST_NAME_LEN + 1]) +{ + char *ptr = reversed + sizeof(reversed); + const char *part = hostname; + + *(--ptr) = '\0'; // NUL-terminate + + while (*part) { + ssize_t len = strcspn(part, "."); + ssize_t remain = ptr - reversed; + + if (remain < (len + 1)) { + return nullptr; + } + + ptr -= len; + memcpy(ptr, part, len); + + // Skip to the next domain component. This will take us to either a '.' or a NUL. + // If it's a '.' we need to skip over it. + part += len; + if (*part == '.') { + ++part; + *(--ptr) = '.'; + } + } + make_to_lower_case(ptr, ptr, strlen(ptr) + 1); + + return ptr; +} + REGRESSION_TEST(SSLWildcardMatch)(RegressionTest *t, int /* atype ATS_UNUSED */, int *pstatus) { TestBox box(t, pstatus); diff --git a/iocore/net/SSLClientUtils.cc b/iocore/net/SSLClientUtils.cc index 7f0ceb2875c..41aa534c410 100644 --- a/iocore/net/SSLClientUtils.cc +++ b/iocore/net/SSLClientUtils.cc @@ -23,19 +23,15 @@ #include "records/I_RecHttp.h" #include "tscore/ink_platform.h" #include "tscore/X509HostnameValidator.h" + #include "P_Net.h" #include "P_SSLClientUtils.h" #include "YamlSNIConfig.h" +#include "SSLDiags.h" #include #include -#if (OPENSSL_VERSION_NUMBER >= 0x10000000L) // openssl returns a const SSL_METHOD -using ink_ssl_method_t = const SSL_METHOD *; -#else -typedef SSL_METHOD *ink_ssl_method_t; -#endif - int verify_callback(int signature_ok, X509_STORE_CTX *ctx) { @@ -54,11 +50,12 @@ verify_callback(int signature_ok, X509_STORE_CTX *ctx) SSLNetVConnection *netvc = SSLNetVCAccess(ssl); // No enforcing, go away - if (netvc && netvc->options.verifyServerPolicy == YamlSNIConfig::Policy::DISABLED) { - return true; // Tell them that all is well - } else if (!netvc) { // No netvc, very bad. Go away. Things are not good. + if (netvc == nullptr) { + // No netvc, very bad. Go away. Things are not good. Warning("Netvc gone by in verify_callback"); return false; + } else if (netvc->options.verifyServerPolicy == YamlSNIConfig::Policy::DISABLED) { + return true; // Tell them that all is well } depth = X509_STORE_CTX_get_error_depth(ctx); @@ -141,9 +138,8 @@ verify_callback(int signature_ok, X509_STORE_CTX *ctx) SSL_CTX * SSLInitClientContext(const SSLConfigParams *params) { - ink_ssl_method_t meth = nullptr; - SSL_CTX *client_ctx = nullptr; - char *clientKeyPtr = nullptr; + const SSL_METHOD *meth = nullptr; + SSL_CTX *client_ctx = nullptr; // Note that we do not call RAND_seed() explicitly here, we depend on OpenSSL // to do the seeding of the PRNG for us. This is the case for all platforms that @@ -174,53 +170,21 @@ SSLInitClientContext(const SSLConfigParams *params) } #endif -#ifdef SSL_CTX_set1_groups_list +#if defined(SSL_CTX_set1_groups_list) || defined(SSL_CTX_set1_curves_list) if (params->client_groups_list != nullptr) { +#ifdef SSL_CTX_set1_groups_list if (!SSL_CTX_set1_groups_list(client_ctx, params->client_groups_list)) { +#else + if (!SSL_CTX_set1_curves_list(client_ctx, params->client_groups_list)) { +#endif SSLError("invalid groups list for client in records.config"); goto fail; } } #endif - // if no path is given for the client private key, - // assume it is contained in the client certificate file. - clientKeyPtr = params->clientKeyPath; - if (clientKeyPtr == nullptr) { - clientKeyPtr = params->clientCertPath; - } - - if (params->clientCertPath != nullptr && params->clientCertPath[0] != '\0') { - if (!SSL_CTX_use_certificate_chain_file(client_ctx, params->clientCertPath)) { - SSLError("failed to load client certificate from %s", params->clientCertPath); - goto fail; - } - - if (!SSL_CTX_use_PrivateKey_file(client_ctx, clientKeyPtr, SSL_FILETYPE_PEM)) { - SSLError("failed to load client private key file from %s", clientKeyPtr); - goto fail; - } - - if (!SSL_CTX_check_private_key(client_ctx)) { - SSLError("client private key (%s) does not match the certificate public key (%s)", clientKeyPtr, params->clientCertPath); - goto fail; - } - } - SSL_CTX_set_verify(client_ctx, SSL_VERIFY_PEER, verify_callback); SSL_CTX_set_verify_depth(client_ctx, params->client_verify_depth); - - if (params->clientCACertFilename != nullptr || params->clientCACertPath != nullptr) { - if (!SSL_CTX_load_verify_locations(client_ctx, params->clientCACertFilename, params->clientCACertPath)) { - SSLError("invalid client CA Certificate file (%s) or CA Certificate path (%s)", params->clientCACertFilename, - params->clientCACertPath); - goto fail; - } - } else if (!SSL_CTX_set_default_verify_paths(client_ctx)) { - SSLError("failed to set the default verify paths"); - goto fail; - } - if (SSLConfigParams::init_ssl_ctx_cb) { SSLConfigParams::init_ssl_ctx_cb(client_ctx, false); } diff --git a/iocore/net/SSLConfig.cc b/iocore/net/SSLConfig.cc index 018cfcf051d..eada98d17d8 100644 --- a/iocore/net/SSLConfig.cc +++ b/iocore/net/SSLConfig.cc @@ -29,18 +29,25 @@ SSL Configurations ****************************************************************************/ -#include "tscore/ink_platform.h" -#include "tscore/I_Layout.h" +#include "P_SSLConfig.h" #include #include + +#include "tscore/ink_platform.h" +#include "tscore/I_Layout.h" +#include "records/I_RecHttp.h" + +#include "HttpConfig.h" + #include "P_Net.h" -#include "P_SSLConfig.h" -#include "YamlSNIConfig.h" #include "P_SSLUtils.h" +#include "P_SSLClientUtils.h" #include "P_SSLCertLookup.h" +#include "SSLDiags.h" #include "SSLSessionCache.h" -#include +#include "SSLSessionTicket.h" +#include "YamlSNIConfig.h" int SSLConfig::configid = 0; int SSLCertificateConfig::configid = 0; @@ -57,28 +64,15 @@ bool SSLConfigParams::session_cache_skip_on_lock_contention = false; size_t SSLConfigParams::session_cache_max_bucket_size = 100; init_ssl_ctx_func SSLConfigParams::init_ssl_ctx_cb = nullptr; load_ssl_file_func SSLConfigParams::load_ssl_file_cb = nullptr; -bool SSLConfigParams::sni_map_enable = false; +IpMap *SSLConfigParams::proxy_protocol_ipmap = nullptr; -// TS-3534 Wiretracing for SSL Connections -int SSLConfigParams::ssl_wire_trace_enabled = 0; -char *SSLConfigParams::ssl_wire_trace_addr = nullptr; -IpAddr *SSLConfigParams::ssl_wire_trace_ip = nullptr; -int SSLConfigParams::ssl_wire_trace_percentage = 0; -char *SSLConfigParams::ssl_wire_trace_server_name = nullptr; -int SSLConfigParams::async_handshake_enabled = 0; -char *SSLConfigParams::engine_conf_file = nullptr; +int SSLConfigParams::async_handshake_enabled = 0; +char *SSLConfigParams::engine_conf_file = nullptr; static std::unique_ptr> sslCertUpdate; static std::unique_ptr> sslConfigUpdate; static std::unique_ptr> sslTicketKey; -// Check if the ticket_key callback #define is available, and if so, enable session tickets. -#ifdef SSL_CTX_set_tlsext_ticket_key_cb - -#define HAVE_OPENSSL_SESSION_TICKETS 1 - -#endif /* SSL_CTX_set_tlsext_ticket_key_cb */ - SSLConfigParams::SSLConfigParams() { ink_mutex_init(&ctxMapLock); @@ -91,17 +85,23 @@ SSLConfigParams::~SSLConfigParams() ink_mutex_destroy(&ctxMapLock); } +void +SSLConfigInit(IpMap *global) +{ + SSLConfigParams::proxy_protocol_ipmap = global; +} + void SSLConfigParams::reset() { serverCertPathOnly = serverCertChainFilename = configFilePath = serverCACertFilename = serverCACertPath = clientCertPath = clientKeyPath = clientCACertFilename = clientCACertPath = cipherSuite = client_cipherSuite = dhparamsFile = serverKeyPathOnly = - nullptr; - server_tls13_cipher_suites = nullptr; - client_tls13_cipher_suites = nullptr; - server_groups_list = nullptr; - client_groups_list = nullptr; - client_ctx = nullptr; + clientKeyPathOnly = clientCertPathOnly = nullptr; + server_tls13_cipher_suites = nullptr; + client_tls13_cipher_suites = nullptr; + server_groups_list = nullptr; + client_groups_list = nullptr; + client_ctx = nullptr; clientCertLevel = client_verify_depth = verify_depth = 0; verifyServerPolicy = YamlSNIConfig::Policy::DISABLED; verifyServerProperties = YamlSNIConfig::Property::NONE; @@ -123,7 +123,9 @@ SSLConfigParams::cleanup() serverCACertFilename = (char *)ats_free_null(serverCACertFilename); serverCACertPath = (char *)ats_free_null(serverCACertPath); clientCertPath = (char *)ats_free_null(clientCertPath); + clientCertPathOnly = (char *)ats_free_null(clientCertPathOnly); clientKeyPath = (char *)ats_free_null(clientKeyPath); + clientKeyPathOnly = (char *)ats_free_null(clientKeyPathOnly); clientCACertFilename = (char *)ats_free_null(clientCACertFilename); clientCACertPath = (char *)ats_free_null(clientCACertPath); configFilePath = (char *)ats_free_null(configFilePath); @@ -132,14 +134,13 @@ SSLConfigParams::cleanup() cipherSuite = (char *)ats_free_null(cipherSuite); client_cipherSuite = (char *)ats_free_null(client_cipherSuite); dhparamsFile = (char *)ats_free_null(dhparamsFile); - ssl_wire_trace_ip = (IpAddr *)ats_free_null(ssl_wire_trace_ip); server_tls13_cipher_suites = (char *)ats_free_null(server_tls13_cipher_suites); client_tls13_cipher_suites = (char *)ats_free_null(client_tls13_cipher_suites); server_groups_list = (char *)ats_free_null(server_groups_list); client_groups_list = (char *)ats_free_null(client_groups_list); - SSLReleaseContext(client_ctx); + cleanupCTXTable(); reset(); } @@ -182,6 +183,7 @@ SSLConfigParams::initialize() char *clientCACertRelativePath = nullptr; char *ssl_server_ca_cert_filename = nullptr; char *ssl_client_ca_cert_filename = nullptr; + char *ssl_ocsp_response_path = nullptr; cleanup(); @@ -236,6 +238,17 @@ SSLConfigParams::initialize() ssl_client_ctx_options |= SSL_OP_NO_TLSv1_2; } #endif +#ifdef SSL_OP_NO_TLSv1_3 + REC_ReadConfigInteger(options, "proxy.config.ssl.TLSv1_3"); + if (!options) { + ssl_ctx_options |= SSL_OP_NO_TLSv1_3; + } + + REC_ReadConfigInteger(client_ssl_options, "proxy.config.ssl.client.TLSv1_3"); + if (!client_ssl_options) { + ssl_client_ctx_options |= SSL_OP_NO_TLSv1_3; + } +#endif #ifdef SSL_OP_CIPHER_SERVER_PREFERENCE REC_ReadConfigInteger(options, "proxy.config.ssl.server.honor_cipher_order"); @@ -245,10 +258,9 @@ SSLConfigParams::initialize() #endif #ifdef SSL_OP_NO_COMPRESSION - /* OpenSSL >= 1.0 only */ ssl_ctx_options |= SSL_OP_NO_COMPRESSION; ssl_client_ctx_options |= SSL_OP_NO_COMPRESSION; -#elif OPENSSL_VERSION_NUMBER >= 0x00908000L +#else sk_SSL_COMP_zero(SSL_COMP_get_compression_methods()); #endif @@ -316,6 +328,9 @@ SSLConfigParams::initialize() REC_EstablishStaticConfigInt32(ssl_ocsp_cache_timeout, "proxy.config.ssl.ocsp.cache_timeout"); REC_EstablishStaticConfigInt32(ssl_ocsp_request_timeout, "proxy.config.ssl.ocsp.request_timeout"); REC_EstablishStaticConfigInt32(ssl_ocsp_update_period, "proxy.config.ssl.ocsp.update_period"); + REC_ReadConfigStringAlloc(ssl_ocsp_response_path, "proxy.config.ssl.ocsp.response.path"); + set_paths_helper(ssl_ocsp_response_path, nullptr, &ssl_ocsp_response_path_only, nullptr); + ats_free(ssl_ocsp_response_path); REC_ReadConfigInt32(async_handshake_enabled, "proxy.config.ssl.async.handshake.enabled"); REC_ReadConfigStringAlloc(engine_conf_file, "proxy.config.ssl.engine.conf_file"); @@ -406,15 +421,13 @@ SSLConfigParams::initialize() ssl_client_cert_path = nullptr; REC_ReadConfigStringAlloc(ssl_client_cert_filename, "proxy.config.ssl.client.cert.filename"); REC_ReadConfigStringAlloc(ssl_client_cert_path, "proxy.config.ssl.client.cert.path"); - if (ssl_client_cert_filename && ssl_client_cert_path) { - set_paths_helper(ssl_client_cert_path, ssl_client_cert_filename, nullptr, &clientCertPath); - } + set_paths_helper(ssl_client_cert_path, ssl_client_cert_filename, &clientCertPathOnly, &clientCertPath); ats_free_null(ssl_client_cert_filename); ats_free_null(ssl_client_cert_path); REC_ReadConfigStringAlloc(ssl_client_private_key_filename, "proxy.config.ssl.client.private_key.filename"); REC_ReadConfigStringAlloc(ssl_client_private_key_path, "proxy.config.ssl.client.private_key.path"); - set_paths_helper(ssl_client_private_key_path, ssl_client_private_key_filename, nullptr, &clientKeyPath); + set_paths_helper(ssl_client_private_key_path, ssl_client_private_key_filename, &clientKeyPathOnly, &clientKeyPath); ats_free_null(ssl_client_private_key_filename); ats_free_null(ssl_client_private_key_path); @@ -426,69 +439,17 @@ SSLConfigParams::initialize() REC_ReadConfigStringAlloc(client_groups_list, "proxy.config.ssl.client.groups_list"); - // Enable/disable sni mapping - REC_ReadConfigInteger(sni_map_enable, "proxy.config.ssl.sni.map.enable"); - REC_ReadConfigInt32(ssl_allow_client_renegotiation, "proxy.config.ssl.allow_client_renegotiation"); - // SSL Wire Trace configurations - REC_EstablishStaticConfigInt32(ssl_wire_trace_enabled, "proxy.config.ssl.wire_trace_enabled"); - if (ssl_wire_trace_enabled) { - // wire trace specific source ip - REC_EstablishStaticConfigStringAlloc(ssl_wire_trace_addr, "proxy.config.ssl.wire_trace_addr"); - if (ssl_wire_trace_addr) { - ssl_wire_trace_ip = new IpAddr(); - ssl_wire_trace_ip->load(ssl_wire_trace_addr); - } else { - ssl_wire_trace_ip = nullptr; - } - // wire trace percentage of requests - REC_EstablishStaticConfigInt32(ssl_wire_trace_percentage, "proxy.config.ssl.wire_trace_percentage"); - REC_EstablishStaticConfigStringAlloc(ssl_wire_trace_server_name, "proxy.config.ssl.wire_trace_server_name"); - } else { - ssl_wire_trace_addr = nullptr; - ssl_wire_trace_ip = nullptr; - ssl_wire_trace_percentage = 0; - ssl_wire_trace_server_name = nullptr; - } // Enable client regardless of config file settings as remap file // can cause HTTP layer to connect using SSL. But only if SSL // initialization hasn't failed already. - client_ctx = SSLInitClientContext(this); + client_ctx = this->getCTX(this->clientCertPath, this->clientKeyPath, this->clientCACertFilename, this->clientCACertPath); if (!client_ctx) { SSLError("Can't initialize the SSL client, HTTPS in remap rules will not function"); } } -// creates a new context attaching the provided certificate -SSL_CTX * -SSLConfigParams::getNewCTX(cchar *client_cert, cchar *client_key) const -{ - SSL_CTX *nclient_ctx = nullptr; - nclient_ctx = SSLInitClientContext(this); - if (!nclient_ctx) { - SSLError("Can't initialize the SSL client, HTTPS in remap rules will not function"); - return nullptr; - } - if (client_cert != nullptr && client_cert[0] != '\0') { - if (!SSL_CTX_use_certificate_chain_file(nclient_ctx, (const char *)client_cert)) { - SSLError("failed to load client certificate from %s", this->clientCertPath); - SSLReleaseContext(nclient_ctx); - return nullptr; - } - } - // If there is not private key specified, perhaps it is in the file with the cert - if (client_key == nullptr || client_key[0] == '\0') { - client_key = client_cert; - } - // Try loading the private key - if (client_key != nullptr && client_key[0] != '\0') { - // If it failed, then we are just going to use the previously set private key from records.config - SSL_CTX_use_PrivateKey_file(nclient_ctx, client_key, SSL_FILETYPE_PEM); - } - return nclient_ctx; -} - SSL_CTX * SSLConfigParams::getClientSSL_CTX() const { @@ -518,7 +479,7 @@ SSLConfig::reconfigure() SSLConfigParams * SSLConfig::acquire() { - return ((SSLConfigParams *)configProcessor.get(configid)); + return static_cast(configProcessor.get(configid)); } void @@ -557,11 +518,12 @@ SSLCertificateConfig::reconfigure() // twice the healthcheck period to simulate a loading a large certificate set. if (is_action_tag_set("test.multicert.delay")) { const int secs = 60; - Debug("ssl", "delaying certificate reload by %dsecs", secs); + Debug("ssl", "delaying certificate reload by %d secs", secs); ink_hrtime_sleep(HRTIME_SECONDS(secs)); } - SSLParseCertificateConfiguration(params, lookup); + SSLMultiCertConfigLoader loader(params); + loader.load(lookup); if (!lookup->is_valid) { retStatus = false; @@ -575,9 +537,9 @@ SSLCertificateConfig::reconfigure() } if (retStatus) { - Note("ssl_multicert.config done reloading!"); + Note("ssl_multicert.config finished loading"); } else { - Note("failed to reload ssl_multicert.config"); + Error("ssl_multicert.config failed to load"); } return retStatus; @@ -586,7 +548,7 @@ SSLCertificateConfig::reconfigure() SSLCertLookup * SSLCertificateConfig::acquire() { - return (SSLCertLookup *)configProcessor.get(configid); + return static_cast(configProcessor.get(configid)); } void @@ -600,7 +562,7 @@ SSLTicketParams::LoadTicket() { cleanup(); -#if HAVE_OPENSSL_SESSION_TICKETS +#if TS_HAVE_OPENSSL_SESSION_TICKETS ssl_ticket_key_block *keyblock = nullptr; SSLConfig::scoped_config params; @@ -648,7 +610,7 @@ void SSLTicketParams::LoadTicketData(char *ticket_data, int ticket_data_len) { cleanup(); -#if HAVE_OPENSSL_SESSION_TICKETS +#if TS_HAVE_OPENSSL_SESSION_TICKETS if (ticket_data != nullptr && ticket_data_len > 0) { default_global_keyblock = ticket_block_create(ticket_data, ticket_data_len); } else { @@ -702,3 +664,114 @@ SSLTicketParams::cleanup() ticket_block_free(default_global_keyblock); ticket_key_filename = (char *)ats_free_null(ticket_key_filename); } + +SSL_CTX * +SSLConfigParams::getCTX(const char *client_cert, const char *key_file, const char *ca_bundle_file, const char *ca_bundle_path) const +{ + SSL_CTX *client_ctx = nullptr; + CTX_MAP *ctx_map = nullptr; + std::string top_level_key, ctx_key; + ts::bwprint(top_level_key, "{}:{}", ca_bundle_file, ca_bundle_path); + ts::bwprint(ctx_key, "{}:{}", client_cert, key_file); + + ink_mutex_acquire(&ctxMapLock); + // Do first level searching and create new CTX_MAP as second level if not exists. + auto top_iter = top_level_ctx_map.find(top_level_key); + if (top_iter != top_level_ctx_map.end()) { + if (top_iter->second == nullptr) { + top_iter->second = new CTX_MAP; + } + ctx_map = top_iter->second; + } else { + ctx_map = new CTX_MAP; + top_level_ctx_map.insert(std::make_pair(top_level_key, ctx_map)); + } + // Do second level searching and return client ctx if found + auto iter = ctx_map->find(ctx_key); + if (iter != ctx_map->end()) { + client_ctx = iter->second; + ink_mutex_release(&ctxMapLock); + return client_ctx; + } + ink_mutex_release(&ctxMapLock); + + // Not yet in the table. Make the cert and add it to the table + client_ctx = SSLInitClientContext(this); + + if (client_cert) { + // Set public and private keys + if (!SSL_CTX_use_certificate_chain_file(client_ctx, client_cert)) { + SSLError("failed to load client certificate from %s", client_cert); + goto fail; + } + if (!key_file || key_file[0] == '\0') { + key_file = client_cert; + } + if (!SSL_CTX_use_PrivateKey_file(client_ctx, key_file, SSL_FILETYPE_PEM)) { + SSLError("failed to load client private key file from %s", key_file); + goto fail; + } + + if (!SSL_CTX_check_private_key(client_ctx)) { + SSLError("client private key (%s) does not match the certificate public key (%s)", key_file, client_cert); + goto fail; + } + } + + // Set CA information for verifying peer cert + if (ca_bundle_file != nullptr || ca_bundle_path != nullptr) { + if (!SSL_CTX_load_verify_locations(client_ctx, ca_bundle_file, ca_bundle_path)) { + SSLError("invalid client CA Certificate file (%s) or CA Certificate path (%s)", ca_bundle_file, ca_bundle_path); + goto fail; + } + } else if (!SSL_CTX_set_default_verify_paths(client_ctx)) { + SSLError("failed to set the default verify paths"); + goto fail; + } + + ink_mutex_acquire(&ctxMapLock); + top_iter = top_level_ctx_map.find(top_level_key); + if (top_iter != top_level_ctx_map.end()) { + if (top_iter->second == nullptr) { + top_iter->second = new CTX_MAP; + } + ctx_map = top_iter->second; + } else { + ctx_map = new CTX_MAP; + top_level_ctx_map.insert(std::make_pair(top_level_key, ctx_map)); + } + iter = ctx_map->find(ctx_key); + if (iter != ctx_map->end()) { + SSL_CTX_free(client_ctx); + client_ctx = iter->second; + } else { + ctx_map->insert(std::make_pair(ctx_key, client_ctx)); + } + ink_mutex_release(&ctxMapLock); + return client_ctx; + +fail: + if (client_ctx) { + SSL_CTX_free(client_ctx); + } + return nullptr; +} + +void +SSLConfigParams::cleanupCTXTable() +{ + ink_mutex_acquire(&ctxMapLock); + CTX_MAP *ctx_map = nullptr; + for (auto &top_pair : top_level_ctx_map) { + ctx_map = top_pair.second; + if (ctx_map) { + for (auto &pair : (*ctx_map)) { + SSL_CTX_free(pair.second); + } + ctx_map->clear(); + delete ctx_map; + } + } + top_level_ctx_map.clear(); + ink_mutex_release(&ctxMapLock); +} diff --git a/iocore/net/SSLDiags.cc b/iocore/net/SSLDiags.cc new file mode 100644 index 00000000000..549a4d31e38 --- /dev/null +++ b/iocore/net/SSLDiags.cc @@ -0,0 +1,224 @@ +/** @file + + Diags for TLS + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#include "SSLDiags.h" + +#include + +#include "P_Net.h" +#include "SSLStats.h" + +// return true if we have a stat for the error +static bool +increment_ssl_client_error(unsigned long err) +{ + // we only look for LIB_SSL errors atm + if (ERR_LIB_SSL != ERR_GET_LIB(err)) { + SSL_INCREMENT_DYN_STAT(ssl_user_agent_other_errors_stat); + return false; + } + + // error was in LIB_SSL, now just switch on REASON + // (we ignore FUNCTION with the prejudice that we don't care what function + // the error came from, hope that's ok?) + switch (ERR_GET_REASON(err)) { +#ifdef SSL_R_SSLV3_ALERT_CERTIFICATE_EXPIRED + case SSL_R_SSLV3_ALERT_CERTIFICATE_EXPIRED: + SSL_INCREMENT_DYN_STAT(ssl_user_agent_expired_cert_stat); + break; +#endif +#ifdef SSL_R_SSLV3_ALERT_CERTIFICATE_REVOKED + case SSL_R_SSLV3_ALERT_CERTIFICATE_REVOKED: + SSL_INCREMENT_DYN_STAT(ssl_user_agent_revoked_cert_stat); + break; +#endif +#ifdef SSL_R_SSLV3_ALERT_CERTIFICATE_UNKNOWN + case SSL_R_SSLV3_ALERT_CERTIFICATE_UNKNOWN: + SSL_INCREMENT_DYN_STAT(ssl_user_agent_unknown_cert_stat); + break; +#endif + case SSL_R_CERTIFICATE_VERIFY_FAILED: + SSL_INCREMENT_DYN_STAT(ssl_user_agent_cert_verify_failed_stat); + break; +#ifdef SSL_R_SSLV3_ALERT_BAD_CERTIFICATE + case SSL_R_SSLV3_ALERT_BAD_CERTIFICATE: + SSL_INCREMENT_DYN_STAT(ssl_user_agent_bad_cert_stat); + break; +#endif +#ifdef SSL_R_TLSV1_ALERT_DECRYPTION_FAILED + case SSL_R_TLSV1_ALERT_DECRYPTION_FAILED: + SSL_INCREMENT_DYN_STAT(ssl_user_agent_decryption_failed_stat); + break; +#endif + case SSL_R_WRONG_VERSION_NUMBER: + SSL_INCREMENT_DYN_STAT(ssl_user_agent_wrong_version_stat); + break; +#ifdef SSL_R_TLSV1_ALERT_UNKNOWN_CA + case SSL_R_TLSV1_ALERT_UNKNOWN_CA: + SSL_INCREMENT_DYN_STAT(ssl_user_agent_unknown_ca_stat); + break; +#endif + default: + SSL_INCREMENT_DYN_STAT(ssl_user_agent_other_errors_stat); + return false; + } + + return true; +} + +// return true if we have a stat for the error + +static bool +increment_ssl_server_error(unsigned long err) +{ + // we only look for LIB_SSL errors atm + if (ERR_LIB_SSL != ERR_GET_LIB(err)) { + SSL_INCREMENT_DYN_STAT(ssl_origin_server_other_errors_stat); + return false; + } + + // error was in LIB_SSL, now just switch on REASON + // (we ignore FUNCTION with the prejudice that we don't care what function + // the error came from, hope that's ok?) + switch (ERR_GET_REASON(err)) { +#ifdef SSL_R_SSLV3_ALERT_CERTIFICATE_EXPIRED + case SSL_R_SSLV3_ALERT_CERTIFICATE_EXPIRED: + SSL_INCREMENT_DYN_STAT(ssl_origin_server_expired_cert_stat); + break; +#endif +#ifdef SSL_R_SSLV3_ALERT_CERTIFICATE_REVOKED + case SSL_R_SSLV3_ALERT_CERTIFICATE_REVOKED: + SSL_INCREMENT_DYN_STAT(ssl_origin_server_revoked_cert_stat); + break; +#endif +#ifdef SSL_R_SSLV3_ALERT_CERTIFICATE_UNKNOWN + case SSL_R_SSLV3_ALERT_CERTIFICATE_UNKNOWN: + SSL_INCREMENT_DYN_STAT(ssl_origin_server_unknown_cert_stat); + break; +#endif + case SSL_R_CERTIFICATE_VERIFY_FAILED: + SSL_INCREMENT_DYN_STAT(ssl_origin_server_cert_verify_failed_stat); + break; +#ifdef SSL_R_SSLV3_ALERT_BAD_CERTIFICATE + case SSL_R_SSLV3_ALERT_BAD_CERTIFICATE: + SSL_INCREMENT_DYN_STAT(ssl_origin_server_bad_cert_stat); + break; +#endif +#ifdef SSL_R_TLSV1_ALERT_DECRYPTION_FAILED + case SSL_R_TLSV1_ALERT_DECRYPTION_FAILED: + SSL_INCREMENT_DYN_STAT(ssl_origin_server_decryption_failed_stat); + break; +#endif + case SSL_R_WRONG_VERSION_NUMBER: + SSL_INCREMENT_DYN_STAT(ssl_origin_server_wrong_version_stat); + break; +#ifdef SSL_R_TLSV1_ALERT_UNKNOWN_CA + case SSL_R_TLSV1_ALERT_UNKNOWN_CA: + SSL_INCREMENT_DYN_STAT(ssl_origin_server_unknown_ca_stat); + break; +#endif + default: + SSL_INCREMENT_DYN_STAT(ssl_origin_server_other_errors_stat); + return false; + } + + return true; +} + +void +SSLDiagnostic(const SourceLocation &loc, bool debug, SSLNetVConnection *vc, const char *fmt, ...) +{ + unsigned long l; + char buf[256]; + const char *file, *data; + int line, flags; + unsigned long es; + va_list ap; + ip_text_buffer ip_buf = {'\0'}; + + if (vc) { + ats_ip_ntop(vc->get_remote_addr(), ip_buf, sizeof(ip_buf)); + } + + es = (unsigned long)pthread_self(); + while ((l = ERR_get_error_line_data(&file, &line, &data, &flags)) != 0) { + if (debug) { + if (unlikely(diags->on())) { + diags->log("ssl-diag", DL_Debug, &loc, "SSL::%lu:%s:%s:%d%s%s%s%s", es, ERR_error_string(l, buf), file, line, + (flags & ERR_TXT_STRING) ? ":" : "", (flags & ERR_TXT_STRING) ? data : "", vc ? ": peer address is " : "", + ip_buf); + } + } else { + diags->error(DL_Error, &loc, "SSL::%lu:%s:%s:%d%s%s%s%s", es, ERR_error_string(l, buf), file, line, + (flags & ERR_TXT_STRING) ? ":" : "", (flags & ERR_TXT_STRING) ? data : "", vc ? ": peer address is " : "", + ip_buf); + } + + // Tally desired stats (only client/server connection stats, not init + // issues where vc is nullptr) + if (vc) { + // get_context() == NET_VCONNECTION_OUT if ats is client (we update server stats) + if (vc->get_context() == NET_VCONNECTION_OUT) { + increment_ssl_server_error(l); // update server error stats + } else { + increment_ssl_client_error(l); // update client error stat + } + } + } + + va_start(ap, fmt); + if (debug) { + diags->log_va("ssl-diag", DL_Debug, &loc, fmt, ap); + } else { + diags->error_va(DL_Error, &loc, fmt, ap); + } + va_end(ap); +} + +const char * +SSLErrorName(int ssl_error) +{ + static const char *names[] = { + "SSL_ERROR_NONE", "SSL_ERROR_SSL", "SSL_ERROR_WANT_READ", "SSL_ERROR_WANT_WRITE", "SSL_ERROR_WANT_X509_LOOKUP", + "SSL_ERROR_SYSCALL", "SSL_ERROR_ZERO_RETURN", "SSL_ERROR_WANT_CONNECT", "SSL_ERROR_WANT_ACCEPT"}; + + if (ssl_error < 0 || ssl_error >= (int)countof(names)) { + return "unknown SSL error"; + } + + return names[ssl_error]; +} + +void +SSLDebugBufferPrint(const char *tag, const char *buffer, unsigned buflen, const char *message) +{ + if (is_debug_tag_set(tag)) { + if (message != nullptr) { + fprintf(stdout, "%s\n", message); + } + for (unsigned ii = 0; ii < buflen; ii++) { + putc(buffer[ii], stdout); + } + putc('\n', stdout); + } +} diff --git a/iocore/net/SSLDiags.h b/iocore/net/SSLDiags.h new file mode 100644 index 00000000000..52ccd08980f --- /dev/null +++ b/iocore/net/SSLDiags.h @@ -0,0 +1,43 @@ +/** @file + + Diags for TLS + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#pragma once + +#include "tscore/Diags.h" + +class SSLNetVConnection; + +// Log an SSL error. +#define SSLError(fmt, ...) SSLDiagnostic(MakeSourceLocation(), false, nullptr, fmt, ##__VA_ARGS__) +#define SSLErrorVC(vc, fmt, ...) SSLDiagnostic(MakeSourceLocation(), false, (vc), fmt, ##__VA_ARGS__) +// Log a SSL diagnostic using the "ssl" diagnostic tag. +#define SSLDebug(fmt, ...) SSLDiagnostic(MakeSourceLocation(), true, nullptr, fmt, ##__VA_ARGS__) +#define SSLVCDebug(vc, fmt, ...) SSLDiagnostic(MakeSourceLocation(), true, (vc), fmt, ##__VA_ARGS__) + +void SSLDiagnostic(const SourceLocation &loc, bool debug, SSLNetVConnection *vc, const char *fmt, ...) TS_PRINTFLIKE(4, 5); + +// Return a static string name for a SSL_ERROR constant. +const char *SSLErrorName(int ssl_error); + +// Log a SSL network buffer. +void SSLDebugBufferPrint(const char *tag, const char *buffer, unsigned buflen, const char *message); diff --git a/iocore/net/SSLNetProcessor.cc b/iocore/net/SSLNetProcessor.cc index a79fd0fe667..bec9a3d0725 100644 --- a/iocore/net/SSLNetProcessor.cc +++ b/iocore/net/SSLNetProcessor.cc @@ -27,6 +27,7 @@ #include "P_SSLUtils.h" #include "P_OCSPStapling.h" #include "P_SSLSNI.h" +#include "SSLStats.h" // // Global Data @@ -36,7 +37,7 @@ SSLNetProcessor ssl_NetProcessor; NetProcessor &sslNetProcessor = ssl_NetProcessor; SNIActionPerformer sni_action_performer; -#ifdef TS_USE_TLS_OCSP +#if TS_USE_TLS_OCSP struct OCSPContinuation : public Continuation { int mainEvent(int /* event ATS_UNUSED */, Event * /* e ATS_UNUSED */) @@ -75,8 +76,10 @@ SSLNetProcessor::start(int, size_t stacksize) // Initialize SSL statistics. This depends on an initial set of certificates being loaded above. SSLInitializeStatistics(); -#ifdef TS_USE_TLS_OCSP +#if TS_USE_TLS_OCSP if (SSLConfigParams::ssl_ocsp_enabled) { + // Call the update initially to get things populated + ocsp_update(); EventType ET_OCSP = eventProcessor.spawn_event_threads("ET_OCSP", 1, stacksize); eventProcessor.schedule_every(new OCSPContinuation(), HRTIME_SECONDS(SSLConfigParams::ssl_ocsp_update_period), ET_OCSP); } diff --git a/iocore/net/SSLNetVConnection.cc b/iocore/net/SSLNetVConnection.cc index 49732633012..e4c9fbf4402 100644 --- a/iocore/net/SSLNetVConnection.cc +++ b/iocore/net/SSLNetVConnection.cc @@ -20,19 +20,27 @@ See the License for the specific language governing permissions and limitations under the License. */ + #include "tscore/ink_config.h" #include "tscore/EventNotify.h" +#include "tscore/I_Layout.h" +#include "tscore/TSSystemState.h" #include "records/I_RecHttp.h" + +#include "InkAPIInternal.h" // Added to include the ssl_hook definitions +#include "Log.h" +#include "HttpTunnel.h" +#include "ProxyProtocol.h" +#include "HttpConfig.h" + #include "P_Net.h" #include "P_SSLNextProtocolSet.h" #include "P_SSLUtils.h" -#include "InkAPIInternal.h" // Added to include the ssl_hook definitions #include "P_SSLConfig.h" -#include "BIO_fastopen.h" -#include "Log.h" #include "P_SSLClientUtils.h" #include "P_SSLSNI.h" -#include "HttpTunnel.h" +#include "BIO_fastopen.h" +#include "SSLStats.h" #include #include @@ -214,7 +222,6 @@ ssl_read_from_net(SSLNetVConnection *sslvc, EThread *lthread, int64_t &ret) int64_t bytes_read = 0; ssl_error_t sslErr = SSL_ERROR_NONE; - bool trace = sslvc->getSSLTrace(); int64_t toread = buf.writer()->write_avail(); ink_release_assert(toread > 0); if (toread > s->vio.ntodo()) { @@ -237,15 +244,6 @@ ssl_read_from_net(SSLNetVConnection *sslvc, EThread *lthread, int64_t &ret) sslErr = SSLReadBuffer(sslvc->ssl, current_block, amount_to_read, nread); Debug("ssl", "[SSL_NetVConnection::ssl_read_from_net] nread=%d", (int)nread); - if (!sslvc->origin_trace) { - TraceIn((0 < nread && trace), sslvc->get_remote_addr(), sslvc->get_remote_port(), "WIRE TRACE\tbytes=%d\n%.*s", (int)nread, - (int)nread, current_block); - } else { - char origin_trace_ip[INET6_ADDRSTRLEN]; - ats_ip_ntop(sslvc->origin_trace_addr, origin_trace_ip, sizeof(origin_trace_ip)); - TraceIn((0 < nread && trace), sslvc->get_remote_addr(), sslvc->get_remote_port(), "CLIENT %s:%d\ttbytes=%d\n%.*s", - origin_trace_ip, sslvc->origin_trace_port, (int)nread, (int)nread, current_block); - } switch (sslErr) { case SSL_ERROR_NONE: @@ -270,30 +268,32 @@ ssl_read_from_net(SSLNetVConnection *sslvc, EThread *lthread, int64_t &ret) SSL_INCREMENT_DYN_STAT(ssl_error_want_read); Debug("ssl.error", "[SSL_NetVConnection::ssl_read_from_net] SSL_ERROR_WOULD_BLOCK(read)"); break; +#ifdef SSL_ERROR_WANT_CLIENT_HELLO_CB + case SSL_ERROR_WANT_CLIENT_HELLO_CB: + event = SSL_READ_WOULD_BLOCK; + SSL_INCREMENT_DYN_STAT(ssl_error_want_client_hello_cb); + Debug("ssl.error", "[SSL_NetVConnection::ssl_read_from_net] SSL_ERROR_WOULD_BLOCK(read/client hello cb)"); + break; +#endif case SSL_ERROR_WANT_X509_LOOKUP: - TraceIn(trace, sslvc->get_remote_addr(), sslvc->get_remote_port(), "Want X509 lookup"); event = SSL_READ_WOULD_BLOCK; SSL_INCREMENT_DYN_STAT(ssl_error_want_x509_lookup); Debug("ssl.error", "[SSL_NetVConnection::ssl_read_from_net] SSL_ERROR_WOULD_BLOCK(read/x509 lookup)"); break; case SSL_ERROR_SYSCALL: - TraceIn(trace, sslvc->get_remote_addr(), sslvc->get_remote_port(), "Syscall Error: %s", strerror(errno)); SSL_INCREMENT_DYN_STAT(ssl_error_syscall); if (nread != 0) { // not EOF event = SSL_READ_ERROR; ret = errno; Debug("ssl.error", "[SSL_NetVConnection::ssl_read_from_net] SSL_ERROR_SYSCALL, underlying IO error: %s", strerror(errno)); - TraceIn(trace, sslvc->get_remote_addr(), sslvc->get_remote_port(), "Underlying IO error: %d", errno); } else { // then EOF observed, treat it as EOS // Error("[SSL_NetVConnection::ssl_read_from_net] SSL_ERROR_SYSCALL, EOF observed violating SSL protocol"); - TraceIn(trace, sslvc->get_remote_addr(), sslvc->get_remote_port(), "EOF observed violating SSL protocol"); event = SSL_READ_EOS; } break; case SSL_ERROR_ZERO_RETURN: - TraceIn(trace, sslvc->get_remote_addr(), sslvc->get_remote_port(), "Connection closed by peer"); event = SSL_READ_EOS; SSL_INCREMENT_DYN_STAT(ssl_error_zero_return); Debug("ssl.error", "[SSL_NetVConnection::ssl_read_from_net] SSL_ERROR_ZERO_RETURN"); @@ -303,8 +303,6 @@ ssl_read_from_net(SSLNetVConnection *sslvc, EThread *lthread, int64_t &ret) char buf[512]; unsigned long e = ERR_peek_last_error(); ERR_error_string_n(e, buf, sizeof(buf)); - TraceIn(trace, sslvc->get_remote_addr(), sslvc->get_remote_port(), "SSL Error: sslErr=%d, ERR_get_error=%ld (%s) errno=%d", - sslErr, e, buf, errno); event = SSL_READ_ERROR; ret = errno; SSL_CLR_ERR_INCR_DYN_STAT(sslvc, ssl_error_ssl, "[SSL_NetVConnection::ssl_read_from_net]: errno=%d", errno); @@ -361,6 +359,8 @@ SSLNetVConnection::read_raw_data() NET_INCREMENT_DYN_STAT(net_calls_to_read_stat); total_read += rattempted; + Debug("ssl", "read_raw_data r=%" PRId64 " rattempted=%" PRId64 " total_read=%" PRId64 " fd=%d", r, rattempted, total_read, + con.fd); // last read failed or was incomplete if (r != rattempted || !b) { break; @@ -368,11 +368,10 @@ SSLNetVConnection::read_raw_data() rattempted = b->write_avail(); } - // If we have already moved some bytes successfully, adjust total_read to reflect reality // If any read succeeded, we should return success if (r != rattempted) { - // If the first read failds, we should return error + // If the first read fails, we should return error if (r <= 0 && total_read > rattempted) { r = total_read - rattempted; } else { @@ -381,6 +380,49 @@ SSLNetVConnection::read_raw_data() } NET_SUM_DYN_STAT(net_read_bytes_stat, r); + IpMap *pp_ipmap; + pp_ipmap = SSLConfigParams::proxy_protocol_ipmap; + + if (this->get_is_proxy_protocol()) { + Debug("proxyprotocol", "[SSLNetVConnection::read_raw_data] proxy protocol is enabled on this port"); + if (pp_ipmap->count() > 0) { + Debug("proxyprotocol", + "[SSLNetVConnection::read_raw_data] proxy protocol has a configured whitelist of trusted IPs - checking"); + + // At this point, using get_remote_addr() will return the ip of the + // proxy source IP, not the Proxy Protocol client ip. Since we are + // checking the ip of the actual source of this connection, this is + // what we want now. + void *payload = nullptr; + if (!pp_ipmap->contains(get_remote_addr(), &payload)) { + Debug("proxyprotocol", + "[SSLNetVConnection::read_raw_data] proxy protocol src IP is NOT in the configured whitelist of trusted IPs - " + "closing connection"); + r = -ENOTCONN; // Need a quick close/exit here to refuse the connection!!!!!!!!! + goto proxy_protocol_bypass; + } else { + char new_host[INET6_ADDRSTRLEN]; + Debug("proxyprotocol", "[SSLNetVConnection::read_raw_data] Source IP [%s] is in the trusted whitelist for proxy protocol", + ats_ip_ntop(this->get_remote_addr(), new_host, sizeof(new_host))); + } + } else { + Debug("proxyprotocol", + "[SSLNetVConnection::read_raw_data] proxy protocol DOES NOT have a configured whitelist of trusted IPs but " + "proxy protocol is enabled on this port - processing all connections"); + } + + if (ssl_has_proxy_v1(this, buffer, &r)) { + Debug("proxyprotocol", "[SSLNetVConnection::read_raw_data] ssl has proxy_v1 header"); + set_remote_addr(get_proxy_protocol_src_addr()); + } else { + Debug("proxyprotocol", + "[SSLNetVConnection::read_raw_data] proxy protocol was enabled, but required header was not present in the " + "transaction - closing connection"); + } + } // end of Proxy Protocol processing + +proxy_protocol_bypass: + if (r > 0) { this->handShakeBuffer->fill(r); @@ -567,6 +609,7 @@ SSLNetVConnection::net_read_io(NetHandler *nh, EThread *lthread) nh->write_ready_list.remove(this); writeReschedule(nh); } else if (ret == EVENT_DONE) { + Debug("ssl", "ssl handshake EVENT_DONE ntodo=%" PRId64, ntodo); // If this was driven by a zero length read, signal complete when // the handshake is complete. Otherwise set up for continuing read // operations. @@ -695,8 +738,6 @@ SSLNetVConnection::load_buffer_and_write(int64_t towrite, MIOBufferAccessor &buf return this->super::load_buffer_and_write(towrite, buf, total_written, needs); } - bool trace = getSSLTrace(); - do { // What is remaining left in the next block? l = buf.reader()->block_read_avail(); @@ -745,16 +786,6 @@ SSLNetVConnection::load_buffer_and_write(int64_t towrite, MIOBufferAccessor &buf towrite, current_block); err = SSLWriteBuffer(ssl, current_block, l, num_really_written); - if (!origin_trace) { - TraceOut((0 < num_really_written && trace), get_remote_addr(), get_remote_port(), "WIRE TRACE\tbytes=%d\n%.*s", - (int)num_really_written, (int)num_really_written, current_block); - } else { - char origin_trace_ip[INET6_ADDRSTRLEN]; - ats_ip_ntop(origin_trace_addr, origin_trace_ip, sizeof(origin_trace_ip)); - TraceOut((0 < num_really_written && trace), get_remote_addr(), get_remote_port(), "CLIENT %s:%d\ttbytes=%d\n%.*s", - origin_trace_ip, origin_trace_port, (int)num_really_written, (int)num_really_written, current_block); - } - // We wrote all that we thought we should if (num_really_written > 0) { total_written += num_really_written; @@ -785,40 +816,39 @@ SSLNetVConnection::load_buffer_and_write(int64_t towrite, MIOBufferAccessor &buf Debug("ssl.error", "SSL_write-SSL_ERROR_WANT_READ"); break; case SSL_ERROR_WANT_WRITE: +#ifdef SSL_ERROR_WANT_CLIENT_HELLO_CB + case SSL_ERROR_WANT_CLIENT_HELLO_CB: +#endif case SSL_ERROR_WANT_X509_LOOKUP: { if (SSL_ERROR_WANT_WRITE == err) { SSL_INCREMENT_DYN_STAT(ssl_error_want_write); redoWriteSize = l; } else if (SSL_ERROR_WANT_X509_LOOKUP == err) { SSL_INCREMENT_DYN_STAT(ssl_error_want_x509_lookup); - TraceOut(trace, get_remote_addr(), get_remote_port(), "Want X509 lookup"); } - +#ifdef SSL_ERROR_WANT_CLIENT_HELLO_CB + else if (SSL_ERROR_WANT_CLIENT_HELLO_CB == err) { + SSL_INCREMENT_DYN_STAT(ssl_error_want_client_hello_cb); + } +#endif needs |= EVENTIO_WRITE; num_really_written = -EAGAIN; Debug("ssl.error", "SSL_write-SSL_ERROR_WANT_WRITE"); break; } case SSL_ERROR_SYSCALL: - TraceOut(trace, get_remote_addr(), get_remote_port(), "Syscall Error: %s", strerror(errno)); num_really_written = -errno; SSL_INCREMENT_DYN_STAT(ssl_error_syscall); Debug("ssl.error", "SSL_write-SSL_ERROR_SYSCALL"); break; // end of stream case SSL_ERROR_ZERO_RETURN: - TraceOut(trace, get_remote_addr(), get_remote_port(), "SSL Error: zero return"); num_really_written = -errno; SSL_INCREMENT_DYN_STAT(ssl_error_zero_return); Debug("ssl.error", "SSL_write-SSL_ERROR_ZERO_RETURN"); break; case SSL_ERROR_SSL: default: { - char buf[512]; - unsigned long e = ERR_peek_last_error(); - ERR_error_string_n(e, buf, sizeof(buf)); - TraceIn(trace, get_remote_addr(), get_remote_port(), "SSL Error: sslErr=%d, ERR_get_error=%ld (%s) errno=%d", err, e, buf, - errno); // Treat SSL_ERROR_SSL as EPIPE error. num_really_written = -EPIPE; SSL_CLR_ERR_INCR_DYN_STAT(this, ssl_error_ssl, "SSL_write-SSL_ERROR_SSL errno=%d", errno); @@ -912,6 +942,8 @@ SSLNetVConnection::free(EThread *t) } con.close(); + ats_free(tunnel_host); + clear(); SET_CONTINUATION_HANDLER(this, (SSLNetVConnHandler)&SSLNetVConnection::startEvent); ink_assert(con.fd == NO_FD); @@ -924,9 +956,14 @@ SSLNetVConnection::free(EThread *t) THREAD_FREE(this, sslNetVCAllocator, t); } } + int SSLNetVConnection::sslStartHandShake(int event, int &err) { + if (TSSystemState::is_ssl_handshaking_stopped()) { + Debug("ssl", "Stopping handshake due to server shutting down."); + return EVENT_ERROR; + } if (sslHandshakeBeginTime == 0) { sslHandshakeBeginTime = Thread::get_hrtime(); // net_activity will not be triggered until after the handshake @@ -970,9 +1007,7 @@ SSLNetVConnection::sslStartHandShake(int event, int &err) this->ssl = nullptr; return EVENT_DONE; } else { - SSLConfig::scoped_config params; - this->SNIMapping = params->sni_map_enable; - hookOpRequested = SSL_HOOK_OP_TUNNEL; + hookOpRequested = SSL_HOOK_OP_TUNNEL; } } @@ -997,7 +1032,7 @@ SSLNetVConnection::sslStartHandShake(int event, int &err) // Making the check here instead of later, so we only // do this setting immediately after we create the SSL object SNIConfig::scoped_config sniParam; - cchar *serverKey = this->options.sni_servername; + const char *serverKey = this->options.sni_servername; if (!serverKey) { ats_ip_ntop(this->get_remote_addr(), buff, INET6_ADDRSTRLEN); serverKey = buff; @@ -1005,16 +1040,45 @@ SSLNetVConnection::sslStartHandShake(int event, int &err) auto nps = sniParam->getPropertyConfig(serverKey); SSL_CTX *clientCTX = nullptr; - if (nps) { - clientCTX = nps->ctx; - options.verifyServerPolicy = nps->verifyServerPolicy; - options.verifyServerProperties = nps->verifyServerProperties; + // First Look to see if there are override parameters + if (options.ssl_client_cert_name) { + std::string certFilePath = Layout::get()->relative_to(params->clientCertPathOnly, options.ssl_client_cert_name); + std::string keyFilePath; + if (options.ssl_client_private_key_name) { + keyFilePath = Layout::get()->relative_to(params->clientKeyPathOnly, options.ssl_client_private_key_name); + } + std::string caCertFilePath; + if (options.ssl_client_ca_cert_name) { + caCertFilePath = Layout::get()->relative_to(params->clientCACertPath, options.ssl_client_ca_cert_name); + } + clientCTX = + params->getCTX(certFilePath.c_str(), keyFilePath.empty() ? nullptr : keyFilePath.c_str(), + caCertFilePath.empty() ? params->clientCACertFilename : caCertFilePath.c_str(), params->clientCACertPath); + } else if (options.ssl_client_ca_cert_name) { + std::string caCertFilePath = Layout::get()->relative_to(params->clientCACertPath, options.ssl_client_ca_cert_name); + clientCTX = params->getCTX(params->clientCertPath, params->clientKeyPath, caCertFilePath.c_str(), params->clientCACertPath); + } else if (nps) { + clientCTX = nps->ctx; + } else { // Just stay with the values passed down from the SM for verify + clientCTX = params->client_ctx; + } + if (options.verifyServerPolicy != YamlSNIConfig::Policy::UNSET) { + // Stay with conf-override version as the highest priority + } else if (nps && nps->verifyServerPolicy != YamlSNIConfig::Policy::UNSET) { + options.verifyServerPolicy = nps->verifyServerPolicy; + } else { + options.verifyServerPolicy = params->verifyServerPolicy; + } + + if (options.verifyServerProperties != YamlSNIConfig::Property::UNSET) { + // Stay with conf-override version as the highest priority + } else if (nps && nps->verifyServerProperties != YamlSNIConfig::Property::UNSET) { + options.verifyServerProperties = nps->verifyServerProperties; } else { - clientCTX = params->client_ctx; - options.verifyServerPolicy = params->verifyServerPolicy; options.verifyServerProperties = params->verifyServerProperties; } + if (!clientCTX) { SSLErrorVC(this, "failed to create SSL client session"); return EVENT_ERROR; @@ -1051,7 +1115,7 @@ SSLNetVConnection::sslServerHandShakeEvent(int &err) { // Continue on if we are in the invoked state. The hook has not yet reenabled if (sslHandshakeHookState == HANDSHAKE_HOOKS_CERT_INVOKE || sslHandshakeHookState == HANDSHAKE_HOOKS_CLIENT_CERT_INVOKE || - sslHandshakeHookState == HANDSHAKE_HOOKS_PRE_INVOKE) { + sslHandshakeHookState == HANDSHAKE_HOOKS_PRE_INVOKE || sslHandshakeHookState == HANDSHAKE_HOOKS_CLIENT_HELLO_INVOKE) { return SSL_WAIT_FOR_HOOK; } @@ -1059,13 +1123,14 @@ SSLNetVConnection::sslServerHandShakeEvent(int &err) if (sslHandshakeHookState == HANDSHAKE_HOOKS_PRE) { if (!curHook) { Debug("ssl", "Initialize preaccept curHook from NULL"); - curHook = ssl_hooks->get(TS_VCONN_START_INTERNAL_HOOK); + curHook = ssl_hooks->get(TSSslHookInternalID(TS_VCONN_START_HOOK)); } else { curHook = curHook->next(); } - // If no more hooks, move onto SNI + // If no more hooks, move onto CLIENT HELLO + if (nullptr == curHook) { - sslHandshakeHookState = HANDSHAKE_HOOKS_SNI; + sslHandshakeHookState = HANDSHAKE_HOOKS_CLIENT_HELLO; } else { sslHandshakeHookState = HANDSHAKE_HOOKS_PRE_INVOKE; ContWrapper::wrap(nh->mutex.get(), curHook->m_cont, TS_EVENT_VCONN_START, this); @@ -1078,7 +1143,7 @@ SSLNetVConnection::sslServerHandShakeEvent(int &err) // without data replay. // Note we can't arrive here if a hook is active. - if (SSL_HOOK_OP_TUNNEL == hookOpRequested && !SNIMapping) { + if (SSL_HOOK_OP_TUNNEL == hookOpRequested) { this->attributes = HttpProxyPort::TRANSPORT_BLIND_TUNNEL; SSL_free(this->ssl); this->ssl = nullptr; @@ -1150,7 +1215,7 @@ SSLNetVConnection::sslServerHandShakeEvent(int &err) // Clean up the epoll entry for signalling SSL_clear_mode(ssl, SSL_MODE_ASYNC); this->ep.stop(); - // Rectivate the socket, ready to rock + // Reactivate the socket, ready to rock PollDescriptor *pd = get_PollDescriptor(this_ethread()); this->ep.start( pd, this, @@ -1162,8 +1227,6 @@ SSLNetVConnection::sslServerHandShakeEvent(int &err) } } #endif - bool trace = getSSLTrace(); - if (ssl_error != SSL_ERROR_NONE) { err = errno; SSLVCDebug(this, "SSL handshake error: %s (%d), errno=%d", SSLErrorName(ssl_error), ssl_error, err); @@ -1193,9 +1256,6 @@ SSLNetVConnection::sslServerHandShakeEvent(int &err) sslHandshakeStatus = SSL_HANDSHAKE_DONE; - TraceIn(trace, get_remote_addr(), get_remote_port(), "SSL server handshake completed successfully"); - // do we want to include cert info in trace? - if (sslHandshakeBeginTime) { sslHandshakeEndTime = Thread::get_hrtime(); const ink_hrtime ssl_handshake_time = sslHandshakeEndTime - sslHandshakeBeginTime; @@ -1213,16 +1273,10 @@ SSLNetVConnection::sslServerHandShakeEvent(int &err) // is preferred since it is the server's preference. The server // preference would not be meaningful if we let the client // preference have priority. - -#if TS_USE_TLS_ALPN SSL_get0_alpn_selected(ssl, &proto, &len); -#endif /* TS_USE_TLS_ALPN */ - -#if TS_USE_TLS_NPN if (len == 0) { SSL_get0_next_proto_negotiated(ssl, &proto, &len); } -#endif /* TS_USE_TLS_NPN */ if (len) { // If there's no NPN set, we should not have done this negotiation. @@ -1237,38 +1291,33 @@ SSLNetVConnection::sslServerHandShakeEvent(int &err) } Debug("ssl", "client selected next protocol '%.*s'", len, proto); - TraceIn(trace, get_remote_addr(), get_remote_port(), "client selected next protocol'%.*s'", len, proto); } else { Debug("ssl", "client did not select a next protocol"); - TraceIn(trace, get_remote_addr(), get_remote_port(), "client did not select a next protocol"); } } return EVENT_DONE; case SSL_ERROR_WANT_CONNECT: - TraceIn(trace, get_remote_addr(), get_remote_port(), "SSL server handshake ERROR_WANT_CONNECT"); return SSL_HANDSHAKE_WANT_CONNECT; case SSL_ERROR_WANT_WRITE: - TraceIn(trace, get_remote_addr(), get_remote_port(), "SSL server handshake ERROR_WANT_WRITE"); return SSL_HANDSHAKE_WANT_WRITE; case SSL_ERROR_WANT_READ: - TraceIn(trace, get_remote_addr(), get_remote_port(), "SSL server handshake ERROR_WANT_READ"); return SSL_HANDSHAKE_WANT_READ; - +#ifdef SSL_ERROR_WANT_CLIENT_HELLO_CB + case SSL_ERROR_WANT_CLIENT_HELLO_CB: + return EVENT_CONT; +#endif // This value is only defined in openssl has been patched to // enable the sni callback to break out of the SSL_accept processing #ifdef SSL_ERROR_WANT_SNI_RESOLVE case SSL_ERROR_WANT_X509_LOOKUP: - TraceIn(trace, get_remote_addr(), get_remote_port(), "SSL server handshake ERROR_WANT_X509_LOOKUP"); return EVENT_CONT; case SSL_ERROR_WANT_SNI_RESOLVE: - TraceIn(trace, get_remote_addr(), get_remote_port(), "SSL server handshake ERROR_WANT_SNI_RESOLVE"); #elif SSL_ERROR_WANT_X509_LOOKUP case SSL_ERROR_WANT_X509_LOOKUP: - TraceIn(trace, get_remote_addr(), get_remote_port(), "SSL server handshake ERROR_WANT_X509_LOOKUP"); #endif #if defined(SSL_ERROR_WANT_SNI_RESOLVE) || defined(SSL_ERROR_WANT_X509_LOOKUP) if (this->attributes == HttpProxyPort::TRANSPORT_BLIND_TUNNEL || SSL_HOOK_OP_TUNNEL == hookOpRequested) { @@ -1283,32 +1332,22 @@ SSLNetVConnection::sslServerHandShakeEvent(int &err) #if TS_USE_TLS_ASYNC case SSL_ERROR_WANT_ASYNC: - TraceIn(trace, get_remote_addr(), get_remote_port(), "SSL server handshake ERROR_WANT_ASYNC"); return SSL_WAIT_FOR_ASYNC; #endif case SSL_ERROR_WANT_ACCEPT: - TraceIn(trace, get_remote_addr(), get_remote_port(), "SSL server handshake ERROR_WANT_ACCEPT"); return EVENT_CONT; case SSL_ERROR_SSL: { SSL_CLR_ERR_INCR_DYN_STAT(this, ssl_error_ssl, "SSLNetVConnection::sslServerHandShakeEvent, SSL_ERROR_SSL errno=%d", errno); - char buf[512]; - unsigned long e = ERR_peek_last_error(); - ERR_error_string_n(e, buf, sizeof(buf)); - TraceIn(trace, get_remote_addr(), get_remote_port(), - "SSL server handshake ERROR_SSL: sslErr=%d, ERR_get_error=%ld (%s) errno=%d", ssl_error, e, buf, errno); return EVENT_ERROR; } case SSL_ERROR_ZERO_RETURN: - TraceIn(trace, get_remote_addr(), get_remote_port(), "SSL server handshake ERROR_ZERO_RETURN"); return EVENT_ERROR; case SSL_ERROR_SYSCALL: - TraceIn(trace, get_remote_addr(), get_remote_port(), "SSL server handshake ERROR_SYSCALL"); return EVENT_ERROR; default: - TraceIn(trace, get_remote_addr(), get_remote_port(), "SSL server handshake ERROR_OTHER"); return EVENT_ERROR; } } @@ -1316,7 +1355,6 @@ SSLNetVConnection::sslServerHandShakeEvent(int &err) int SSLNetVConnection::sslClientHandShakeEvent(int &err) { - bool trace = getSSLTrace(); ssl_error_t ssl_error; ink_assert(SSLNetVCAccess(ssl) == this); @@ -1336,7 +1374,7 @@ SSLNetVConnection::sslClientHandShakeEvent(int &err) if (sslHandshakeHookState == HANDSHAKE_HOOKS_OUTBOUND_PRE) { if (!curHook) { Debug("ssl", "Initialize outbound connect curHook from NULL"); - curHook = ssl_hooks->get(TS_VCONN_OUTBOUND_START_INTERNAL_HOOK); + curHook = ssl_hooks->get(TSSslHookInternalID(TS_VCONN_OUTBOUND_START_HOOK)); } else { curHook = curHook->next(); } @@ -1370,49 +1408,44 @@ SSLNetVConnection::sslClientHandShakeEvent(int &err) SSL_INCREMENT_DYN_STAT(ssl_total_success_handshake_count_out_stat); - TraceIn(trace, get_remote_addr(), get_remote_port(), "SSL client handshake completed successfully"); - // do we want to include cert info in trace? - sslHandshakeStatus = SSL_HANDSHAKE_DONE; return EVENT_DONE; case SSL_ERROR_WANT_WRITE: Debug("ssl.error", "SSLNetVConnection::sslClientHandShakeEvent, SSL_ERROR_WANT_WRITE"); SSL_INCREMENT_DYN_STAT(ssl_error_want_write); - TraceIn(trace, get_remote_addr(), get_remote_port(), "SSL client handshake ERROR_WANT_WRITE"); return SSL_HANDSHAKE_WANT_WRITE; case SSL_ERROR_WANT_READ: SSL_INCREMENT_DYN_STAT(ssl_error_want_read); Debug("ssl.error", "SSLNetVConnection::sslClientHandShakeEvent, SSL_ERROR_WANT_READ"); - TraceIn(trace, get_remote_addr(), get_remote_port(), "SSL client handshake ERROR_WANT_READ"); return SSL_HANDSHAKE_WANT_READ; - +#ifdef SSL_ERROR_WANT_CLIENT_HELLO_CB + case SSL_ERROR_WANT_CLIENT_HELLO_CB: + SSL_INCREMENT_DYN_STAT(ssl_error_want_client_hello_cb); + Debug("ssl.error", "SSLNetVConnection::sslClientHandShakeEvent, SSL_ERROR_WANT_CLIENT_HELLO_CB"); + break; +#endif case SSL_ERROR_WANT_X509_LOOKUP: SSL_INCREMENT_DYN_STAT(ssl_error_want_x509_lookup); Debug("ssl.error", "SSLNetVConnection::sslClientHandShakeEvent, SSL_ERROR_WANT_X509_LOOKUP"); - TraceIn(trace, get_remote_addr(), get_remote_port(), "SSL client handshake ERROR_WANT_X509_LOOKUP"); break; case SSL_ERROR_WANT_ACCEPT: - TraceIn(trace, get_remote_addr(), get_remote_port(), "SSL client handshake ERROR_WANT_ACCEPT"); return SSL_HANDSHAKE_WANT_ACCEPT; case SSL_ERROR_WANT_CONNECT: - TraceIn(trace, get_remote_addr(), get_remote_port(), "SSL client handshake ERROR_WANT_CONNECT"); break; case SSL_ERROR_ZERO_RETURN: SSL_INCREMENT_DYN_STAT(ssl_error_zero_return); Debug("ssl.error", "SSLNetVConnection::sslClientHandShakeEvent, EOS"); - TraceIn(trace, get_remote_addr(), get_remote_port(), "SSL client handshake EOS"); return EVENT_ERROR; case SSL_ERROR_SYSCALL: err = errno; SSL_INCREMENT_DYN_STAT(ssl_error_syscall); Debug("ssl.error", "SSLNetVConnection::sslClientHandShakeEvent, syscall"); - TraceIn(trace, get_remote_addr(), get_remote_port(), "SSL client handshake Syscall Error: %s", strerror(errno)); return EVENT_ERROR; break; @@ -1423,18 +1456,15 @@ SSLNetVConnection::sslClientHandShakeEvent(int &err) unsigned long e = ERR_peek_last_error(); ERR_error_string_n(e, buf, sizeof(buf)); // FIXME -- This triggers a retry on cases of cert validation errors.... - Debug("ssl", "SSLNetVConnection::sslClientHandShakeEvent, SSL_ERROR_SSL"); SSL_CLR_ERR_INCR_DYN_STAT(this, ssl_error_ssl, "SSLNetVConnection::sslClientHandShakeEvent, SSL_ERROR_SSL errno=%d", errno); Debug("ssl.error", "SSLNetVConnection::sslClientHandShakeEvent, SSL_ERROR_SSL"); - TraceIn(trace, get_remote_addr(), get_remote_port(), - "SSL client handshake ERROR_SSL: sslErr=%d, ERR_get_error=%ld (%s) errno=%d", ssl_error, e, buf, errno); if (e) { if (this->options.sni_servername) { - Error("SSL connection failed for '%s': %s", this->options.sni_servername.get(), buf); + Debug("ssl.error", "SSL connection failed for '%s': %s", this->options.sni_servername.get(), buf); } else { char buff[INET6_ADDRSTRLEN]; ats_ip_ntop(this->get_remote_addr(), buff, INET6_ADDRSTRLEN); - Error("SSL connection failed for '%s': %s", buff, buf); + Debug("ssl.error", "SSL connection failed for '%s': %s", buff, buf); } } return EVENT_ERROR; @@ -1480,13 +1510,10 @@ SSLNetVConnection::select_next_protocol(SSL *ssl, const unsigned char **out, uns if (netvc->npnSet && netvc->npnSet->advertiseProtocols(&npn, &npnsz)) { // SSL_select_next_proto chooses the first server-offered protocol that appears in the clients protocol set, ie. the // server selects the protocol. This is a n^2 search, so it's preferable to keep the protocol set short. - -#if HAVE_SSL_SELECT_NEXT_PROTO if (SSL_select_next_proto((unsigned char **)out, outlen, npn, npnsz, in, inlen) == OPENSSL_NPN_NEGOTIATED) { Debug("ssl", "selected ALPN protocol %.*s", (int)(*outlen), *out); return SSL_TLSEXT_ERR_OK; } -#endif /* HAVE_SSL_SELECT_NEXT_PROTO */ } *out = nullptr; @@ -1506,6 +1533,9 @@ SSLNetVConnection::reenable(NetHandler *nh, int event) case HANDSHAKE_HOOKS_OUTBOUND_PRE_INVOKE: sslHandshakeHookState = HANDSHAKE_HOOKS_OUTBOUND_PRE; break; + case HANDSHAKE_HOOKS_CLIENT_HELLO_INVOKE: + sslHandshakeHookState = HANDSHAKE_HOOKS_CLIENT_HELLO; + break; case HANDSHAKE_HOOKS_CERT_INVOKE: sslHandshakeHookState = HANDSHAKE_HOOKS_CERT; break; @@ -1520,7 +1550,7 @@ SSLNetVConnection::reenable(NetHandler *nh, int event) // Reenabling from the handshake callback // - // Originally, we would wait for the callback to go again to execute additinonal + // Originally, we would wait for the callback to go again to execute additional // hooks, but since the callbacks are associated with the context and the context // can be replaced by the plugin, it didn't seem reasonable to assume that the // callback would be executed again. So we walk through the rest of the hooks @@ -1531,7 +1561,10 @@ SSLNetVConnection::reenable(NetHandler *nh, int event) } if (curHook != nullptr) { // Invoke the hook and return, wait for next reenable - if (sslHandshakeHookState == HANDSHAKE_HOOKS_CLIENT_CERT) { + if (sslHandshakeHookState == HANDSHAKE_HOOKS_CLIENT_HELLO) { + sslHandshakeHookState = HANDSHAKE_HOOKS_CLIENT_HELLO_INVOKE; + curHook->invoke(TS_EVENT_SSL_CLIENT_HELLO, this); + } else if (sslHandshakeHookState == HANDSHAKE_HOOKS_CLIENT_CERT) { sslHandshakeHookState = HANDSHAKE_HOOKS_CLIENT_CERT_INVOKE; curHook->invoke(TS_EVENT_SSL_VERIFY_CLIENT, this); } else if (sslHandshakeHookState == HANDSHAKE_HOOKS_CERT) { @@ -1563,6 +1596,10 @@ SSLNetVConnection::reenable(NetHandler *nh, int event) switch (this->sslHandshakeHookState) { case HANDSHAKE_HOOKS_PRE: case HANDSHAKE_HOOKS_PRE_INVOKE: + sslHandshakeHookState = HANDSHAKE_HOOKS_CLIENT_HELLO; + break; + case HANDSHAKE_HOOKS_CLIENT_HELLO: + case HANDSHAKE_HOOKS_CLIENT_HELLO_INVOKE: sslHandshakeHookState = HANDSHAKE_HOOKS_SNI; break; case HANDSHAKE_HOOKS_SNI: @@ -1606,21 +1643,22 @@ SSLNetVConnection::sslContextSet(void *ctx) return zret; } -extern TunnelHashMap TunnelMap; // stores the name of the servers to tunnel to - bool SSLNetVConnection::callHooks(TSEvent eventId) { // Only dealing with the SNI/CERT hook so far. - ink_assert(eventId == TS_EVENT_SSL_CERT || eventId == TS_EVENT_SSL_SERVERNAME || eventId == TS_EVENT_SSL_VERIFY_SERVER || - eventId == TS_EVENT_SSL_VERIFY_CLIENT || eventId == TS_EVENT_VCONN_CLOSE || eventId == TS_EVENT_VCONN_OUTBOUND_CLOSE); - Debug("ssl", "callHooks sslHandshakeHookState=%d", this->sslHandshakeHookState); + ink_assert(eventId == TS_EVENT_SSL_CLIENT_HELLO || eventId == TS_EVENT_SSL_CERT || eventId == TS_EVENT_SSL_SERVERNAME || + eventId == TS_EVENT_SSL_VERIFY_SERVER || eventId == TS_EVENT_SSL_VERIFY_CLIENT || eventId == TS_EVENT_VCONN_CLOSE || + eventId == TS_EVENT_VCONN_OUTBOUND_CLOSE); + Debug("ssl", "callHooks sslHandshakeHookState=%d eventID=%d", this->sslHandshakeHookState, eventId); // Move state if it is appropriate switch (this->sslHandshakeHookState) { case HANDSHAKE_HOOKS_PRE: case HANDSHAKE_HOOKS_OUTBOUND_PRE: - if (eventId == TS_EVENT_SSL_SERVERNAME) { + if (eventId == TS_EVENT_SSL_CLIENT_HELLO) { + this->sslHandshakeHookState = HANDSHAKE_HOOKS_CLIENT_HELLO; + } else if (eventId == TS_EVENT_SSL_SERVERNAME) { this->sslHandshakeHookState = HANDSHAKE_HOOKS_SNI; } else if (eventId == TS_EVENT_SSL_VERIFY_SERVER) { this->sslHandshakeHookState = HANDSHAKE_HOOKS_VERIFY_SERVER; @@ -1628,9 +1666,22 @@ SSLNetVConnection::callHooks(TSEvent eventId) this->sslHandshakeHookState = HANDSHAKE_HOOKS_CERT; } break; + case HANDSHAKE_HOOKS_CLIENT_HELLO: + if (eventId == TS_EVENT_SSL_SERVERNAME) { + this->sslHandshakeHookState = HANDSHAKE_HOOKS_SNI; + } else if (eventId == TS_EVENT_SSL_CERT) { + this->sslHandshakeHookState = HANDSHAKE_HOOKS_CERT; + } else if (eventId == TS_EVENT_VCONN_CLOSE) { + // Jump to the end + this->sslHandshakeHookState = HANDSHAKE_HOOKS_DONE; + } + break; case HANDSHAKE_HOOKS_SNI: if (eventId == TS_EVENT_SSL_CERT) { this->sslHandshakeHookState = HANDSHAKE_HOOKS_CERT; + } else if (eventId == TS_EVENT_VCONN_CLOSE) { + // Jump to the end + this->sslHandshakeHookState = HANDSHAKE_HOOKS_DONE; } break; default: @@ -1639,18 +1690,31 @@ SSLNetVConnection::callHooks(TSEvent eventId) // Look for hooks associated with the event switch (this->sslHandshakeHookState) { + case HANDSHAKE_HOOKS_CLIENT_HELLO: + case HANDSHAKE_HOOKS_CLIENT_HELLO_INVOKE: + if (!curHook) { + curHook = ssl_hooks->get(TSSslHookInternalID(TS_SSL_CLIENT_HELLO_HOOK)); + } else { + curHook = curHook->next(); + } + if (curHook == nullptr) { + this->sslHandshakeHookState = HANDSHAKE_HOOKS_SNI; + } else { + this->sslHandshakeHookState = HANDSHAKE_HOOKS_CLIENT_HELLO_INVOKE; + } + break; case HANDSHAKE_HOOKS_VERIFY_SERVER: // The server verify event addresses ATS to origin handshake // All the other events are for client to ATS if (!curHook) { - curHook = ssl_hooks->get(TS_SSL_VERIFY_SERVER_INTERNAL_HOOK); + curHook = ssl_hooks->get(TSSslHookInternalID(TS_SSL_VERIFY_SERVER_HOOK)); } else { curHook = curHook->next(); } break; case HANDSHAKE_HOOKS_SNI: if (!curHook) { - curHook = ssl_hooks->get(TS_SSL_SERVERNAME_INTERNAL_HOOK); + curHook = ssl_hooks->get(TSSslHookInternalID(TS_SSL_SERVERNAME_HOOK)); } else { curHook = curHook->next(); } @@ -1661,7 +1725,7 @@ SSLNetVConnection::callHooks(TSEvent eventId) case HANDSHAKE_HOOKS_CERT: case HANDSHAKE_HOOKS_CERT_INVOKE: if (!curHook) { - curHook = ssl_hooks->get(TS_SSL_CERT_INTERNAL_HOOK); + curHook = ssl_hooks->get(TSSslHookInternalID(TS_SSL_CERT_HOOK)); } else { curHook = curHook->next(); } @@ -1674,7 +1738,7 @@ SSLNetVConnection::callHooks(TSEvent eventId) case HANDSHAKE_HOOKS_CLIENT_CERT: case HANDSHAKE_HOOKS_CLIENT_CERT_INVOKE: if (!curHook) { - curHook = ssl_hooks->get(TS_SSL_VERIFY_CLIENT_INTERNAL_HOOK); + curHook = ssl_hooks->get(TSSslHookInternalID(TS_SSL_VERIFY_CLIENT_HOOK)); } else { curHook = curHook->next(); } @@ -1684,14 +1748,14 @@ SSLNetVConnection::callHooks(TSEvent eventId) if (eventId == TS_EVENT_VCONN_CLOSE) { sslHandshakeHookState = HANDSHAKE_HOOKS_DONE; if (curHook == nullptr) { - curHook = ssl_hooks->get(TS_VCONN_CLOSE_INTERNAL_HOOK); + curHook = ssl_hooks->get(TSSslHookInternalID(TS_VCONN_CLOSE_HOOK)); } else { curHook = curHook->next(); } } else if (eventId == TS_EVENT_VCONN_OUTBOUND_CLOSE) { sslHandshakeHookState = HANDSHAKE_HOOKS_DONE; if (curHook == nullptr) { - curHook = ssl_hooks->get(TS_VCONN_OUTBOUND_CLOSE_INTERNAL_HOOK); + curHook = ssl_hooks->get(TSSslHookInternalID(TS_VCONN_OUTBOUND_CLOSE_HOOK)); } else { curHook = curHook->next(); } @@ -1707,16 +1771,7 @@ SSLNetVConnection::callHooks(TSEvent eventId) bool reenabled = true; - this->serverName = const_cast(SSL_get_servername(this->ssl, TLSEXT_NAMETYPE_host_name)); - if (this->serverName) { - if (auto it = TunnelMap.find(this->serverName); it != TunnelMap.end()) { - this->SNIMapping = true; - this->attributes = HttpProxyPort::TRANSPORT_BLIND_TUNNEL; - return reenabled; - } - } - - if (SSL_HOOK_OP_TUNNEL == hookOpRequested && SNIMapping) { + if (SSL_HOOK_OP_TUNNEL == hookOpRequested) { this->attributes = HttpProxyPort::TRANSPORT_BLIND_TUNNEL; // Don't mark the handshake as complete yet, // Will be checking for that flag not being set after @@ -1729,59 +1784,14 @@ SSLNetVConnection::callHooks(TSEvent eventId) if (curHook != nullptr) { curHook->invoke(eventId, this); reenabled = - (this->sslHandshakeHookState != HANDSHAKE_HOOKS_CERT_INVOKE && this->sslHandshakeHookState != HANDSHAKE_HOOKS_PRE_INVOKE); + (this->sslHandshakeHookState != HANDSHAKE_HOOKS_CERT_INVOKE && this->sslHandshakeHookState != HANDSHAKE_HOOKS_PRE_INVOKE && + this->sslHandshakeHookState != HANDSHAKE_HOOKS_CLIENT_HELLO_INVOKE); Debug("ssl", "Called hook on state=%d reenabled=%d", sslHandshakeHookState, reenabled); } return reenabled; } -bool -SSLNetVConnection::computeSSLTrace() -{ - // this has to happen before the handshake or else sni_servername will be nullptr - bool sni_trace; - if (ssl) { - const char *ssl_servername = SSL_get_servername(ssl, TLSEXT_NAMETYPE_host_name); - char *wire_trace_server_name = SSLConfigParams::ssl_wire_trace_server_name; - Debug("ssl", "for wiretrace, ssl_servername=%s, wire_trace_server_name=%s", ssl_servername, wire_trace_server_name); - sni_trace = ssl_servername && wire_trace_server_name && (0 == strcmp(wire_trace_server_name, ssl_servername)); - } else { - sni_trace = false; - } - - // count based on ip only if they set an IP value - const sockaddr *remote_addr = get_remote_addr(); - bool ip_trace = false; - if (SSLConfigParams::ssl_wire_trace_ip) { - ip_trace = (*SSLConfigParams::ssl_wire_trace_ip == remote_addr); - } - - // count based on percentage - int percentage = SSLConfigParams::ssl_wire_trace_percentage; - int random; - bool trace; - - // we only generate random numbers as needed (to maintain correct percentage) - if (SSLConfigParams::ssl_wire_trace_server_name && SSLConfigParams::ssl_wire_trace_ip) { - random = this_ethread()->generator.random() % 100; // range [0-99] - trace = sni_trace && ip_trace && (percentage > random); - } else if (SSLConfigParams::ssl_wire_trace_server_name) { - random = this_ethread()->generator.random() % 100; // range [0-99] - trace = sni_trace && (percentage > random); - } else if (SSLConfigParams::ssl_wire_trace_ip) { - random = this_ethread()->generator.random() % 100; // range [0-99] - trace = ip_trace && (percentage > random); - } else { - random = this_ethread()->generator.random() % 100; // range [0-99] - trace = percentage > random; - } - - Debug("ssl", "ssl_netvc random=%d, trace=%s", random, trace ? "TRUE" : "FALSE"); - - return trace; -} - int SSLNetVConnection::populate(Connection &con, Continuation *c, void *arg) { diff --git a/iocore/net/SSLNextProtocolSet.cc b/iocore/net/SSLNextProtocolSet.cc index 6c9d4df6a44..04c6330e3e0 100644 --- a/iocore/net/SSLNextProtocolSet.cc +++ b/iocore/net/SSLNextProtocolSet.cc @@ -29,7 +29,7 @@ // For currently defined protocol strings, see // http://technotes.googlecode.com/git/nextprotoneg.html. The OpenSSL // documentation tells us to return a string in "wire format". The -// draft NPN RFC helpfuly refuses to document the wire format. The +// draft NPN RFC helpfully refuses to document the wire format. The // above link says we need to send length-prefixed strings, but does // not say how many bytes the length is. For the record, it's 1. @@ -78,7 +78,7 @@ create_npn_advertisement(const SSLNextProtocolSet::NextProtocolEndpoint::list_ty return false; } -// copies th eprotocols but not the endpoints +// copies the protocols but not the endpoints SSLNextProtocolSet * SSLNextProtocolSet::clone() const @@ -157,7 +157,7 @@ SSLNextProtocolSet::findEndpoint(const unsigned char *proto, unsigned len) const return nullptr; } -SSLNextProtocolSet::SSLNextProtocolSet() : npn(nullptr), npnsz(0) {} +SSLNextProtocolSet::SSLNextProtocolSet() {} SSLNextProtocolSet::~SSLNextProtocolSet() { diff --git a/iocore/net/SSLSNIConfig.cc b/iocore/net/SSLSNIConfig.cc index 7813287c4de..ea090c210d3 100644 --- a/iocore/net/SSLSNIConfig.cc +++ b/iocore/net/SSLSNIConfig.cc @@ -35,20 +35,24 @@ #include "P_SSLConfig.h" #include "tscore/ink_memory.h" #include "tscpp/util/TextView.h" +#include "tscore/I_Layout.h" +#include +#include static ConfigUpdateHandler *sniConfigUpdate; struct NetAccept; -Map snpsMap; -extern TunnelHashMap TunnelMap; -NextHopProperty::NextHopProperty() {} +std::unordered_map snpsMap; -NextHopProperty * -SNIConfigParams::getPropertyConfig(cchar *servername) const +const NextHopProperty * +SNIConfigParams::getPropertyConfig(const std::string &servername) const { - NextHopProperty *nps = nullptr; - nps = next_hop_table.get(servername); - if (!nps) { - nps = wild_next_hop_table.get(servername); + const NextHopProperty *nps = nullptr; + for (auto &&item : next_hop_list) { + if (pcre_exec(item.match, nullptr, servername.c_str(), servername.length(), 0, 0, nullptr, 0) >= 0) { + // Found a match + nps = &item.prop; + break; + } } return nps; } @@ -56,52 +60,45 @@ SNIConfigParams::getPropertyConfig(cchar *servername) const void SNIConfigParams::loadSNIConfig() { - for (const auto &item : Y_sni.items) { - actionVector *aiVec = new actionVector(); + for (auto &item : Y_sni.items) { + auto ai = sni_action_list.emplace(sni_action_list.end()); + ai->setGlobName(item.fqdn); Debug("ssl", "name: %s", item.fqdn.data()); - cchar *servername = item.fqdn.data(); - ats_wildcard_matcher w_Matcher; - auto wildcard = w_Matcher.match(servername); // set SNI based actions to be called in the ssl_servername_only callback - auto ai1 = new DisableH2(); - aiVec->push_back(ai1); - auto ai2 = new VerifyClient(item.verify_client_level); - aiVec->push_back(ai2); - if (wildcard) { - ts::TextView domain{servername, strlen(servername)}; - domain.take_prefix_at('.'); - if (!domain.empty()) { - wild_sni_action_map.put(ats_stringdup(domain), aiVec); - } - } else { - sni_action_map.put(ats_strdup(servername), aiVec); + if (item.disable_h2) { + ai->actions.push_back(std::make_unique()); } - - if (item.tunnel_destination.length()) { - TunnelMap.emplace(item.fqdn, item.tunnel_destination); + if (item.verify_client_level != 255) { + ai->actions.push_back(std::make_unique(item.verify_client_level)); + } + if (!item.protocol_unset) { + ai->actions.push_back(std::make_unique(item.protocol_mask)); } + if (item.tunnel_destination.length() > 0) { + ai->actions.push_back(std::make_unique(item.tunnel_destination, item.tunnel_decrypt)); + } + + ai->actions.push_back(std::make_unique(item.ip_allow, item.fqdn)); - auto ai3 = new SNI_IpAllow(item.ip_allow, servername); - aiVec->push_back(ai3); // set the next hop properties SSLConfig::scoped_config params; - auto clientCTX = params->getClientSSL_CTX(); - cchar *certFile = item.client_cert.data(); - cchar *keyFile = item.client_key.data(); - if (certFile) { - clientCTX = params->getNewCTX(certFile, keyFile); - } - NextHopProperty *nps = new NextHopProperty(); - nps->name = ats_strdup(servername); - nps->verifyServerPolicy = item.verify_server_policy; - nps->verifyServerProperties = item.verify_server_properties; - nps->ctx = clientCTX; - if (wildcard) { - wild_next_hop_table.put(nps->name, nps); - } else { - next_hop_table.put(nps->name, nps); + auto clientCTX = params->getClientSSL_CTX(); + // Load if we have at least specified the client certificate + if (!item.client_cert.empty()) { + std::string certFilePath = Layout::get()->relative_to(params->clientCertPathOnly, item.client_cert.data()); + std::string keyFilePath; + if (!item.client_key.empty()) { + keyFilePath = Layout::get()->relative_to(params->clientKeyPathOnly, item.client_key.data()); + } + clientCTX = params->getCTX(certFilePath.c_str(), keyFilePath.c_str(), params->clientCACertFilename, params->clientCACertPath); } + + auto nps = next_hop_list.emplace(next_hop_list.end()); + nps->setGlobName(item.fqdn); + nps->prop.verifyServerPolicy = item.verify_server_policy; + nps->prop.verifyServerProperties = item.verify_server_properties; + nps->prop.ctx = clientCTX; } // end for } @@ -109,33 +106,17 @@ int SNIConfig::configid = 0; /*definition of member functions of SNIConfigParams*/ SNIConfigParams::SNIConfigParams() {} -actionVector * -SNIConfigParams::get(cchar *servername) const +const actionVector * +SNIConfigParams::get(const std::string &servername) const { - auto actionVec = sni_action_map.get(servername); - if (!actionVec) { - Vec keys; - wild_sni_action_map.get_keys(keys); - for (int i = 0; i < static_cast(keys.length()); i++) { - std::string_view sv{servername, strlen(servername)}; - std::string_view key_sv{keys.get(i)}; - if (sv.size() >= key_sv.size() && sv.substr(sv.size() - key_sv.size()) == key_sv) { - return wild_sni_action_map.get(key_sv.data()); - } + for (const auto &retval : sni_action_list) { + if (retval.match == nullptr && servername.length() == 0) { + return &retval.actions; + } else if (pcre_exec(retval.match, nullptr, servername.c_str(), servername.length(), 0, 0, nullptr, 0) >= 0) { + return &retval.actions; } } - return actionVec; -} - -void -SNIConfigParams::printSNImap() const -{ - Vec keys; - sni_action_map.get_keys(keys); - for (size_t i = 0; i < keys.length(); i++) { - Debug("ssl", "Domain name in the map %s: # of registered action items %lu", (char *)keys.get(i), - sni_action_map.get(keys.get(i))->size()); - } + return nullptr; } int @@ -143,11 +124,11 @@ SNIConfigParams::Initialize() { sni_filename = ats_stringdup(RecConfigReadConfigPath("proxy.config.ssl.servername.filename")); - Note("loading %s", sni_filename); + Note("ssl_server_name.yaml loading ..."); struct stat sbuf; if (stat(sni_filename, &sbuf) == -1 && errno == ENOENT) { - Note("failed to reload ssl_server_name.yaml"); + Note("ssl_server_name.yaml failed to load"); Warning("Loading SNI configuration - filename: %s doesn't exist", sni_filename); return 1; } @@ -156,60 +137,19 @@ SNIConfigParams::Initialize() if (!zret.isOK()) { std::stringstream errMsg; errMsg << zret; - Error("failed to load ssl_server_name.yaml: %s", errMsg.str().c_str()); + Error("ssl_server_name.yaml failed to load: %s", errMsg.str().c_str()); return 1; } loadSNIConfig(); - Note("ssl_server_name.yaml done reloading!"); + Note("ssl_server_name.yaml finished loading"); return 0; } -void -SNIConfigParams::cleanup() -{ - Vec keys; - sni_action_map.get_keys(keys); - for (int i = keys.length() - 1; i >= 0; i--) { - auto actionVec = sni_action_map.get(keys.get(i)); - for (auto &ai : *actionVec) { - delete ai; - } - - actionVec->clear(); - } - keys.free_and_clear(); - - wild_sni_action_map.get_keys(keys); - for (int i = keys.length() - 1; i >= 0; i--) { - auto actionVec = wild_sni_action_map.get(keys.get(i)); - for (auto &ai : *actionVec) { - delete ai; - } - - actionVec->clear(); - } - keys.free_and_clear(); - - next_hop_table.get_keys(keys); - for (int i = 0; i < static_cast(keys.length()); i++) { - auto *nps = next_hop_table.get(keys.get(i)); - delete (nps); - } - keys.free_and_clear(); - - wild_next_hop_table.get_keys(keys); - for (int i = 0; static_cast(keys.length()); i++) { - auto *nps = wild_next_hop_table.get(keys.get(i)); - delete (nps); - } - keys.free_and_clear(); -} - SNIConfigParams::~SNIConfigParams() { - cleanup(); + // sni_action_list and next_hop_list should cleanup with the params object } /*definition of member functions of SNIConfig*/ @@ -229,7 +169,7 @@ SNIConfig::cloneProtoSet() if (na->snpa) { auto snps = na->snpa->cloneProtoSet(); snps->unregisterEndpoint(TS_ALPN_PROTOCOL_HTTP_2_0, nullptr); - snpsMap.put(na->id, snps); + snpsMap.emplace(na->id, snps); } } } diff --git a/iocore/net/SSLSessionCache.cc b/iocore/net/SSLSessionCache.cc index bbe52a9f241..57e01ab23b2 100644 --- a/iocore/net/SSLSessionCache.cc +++ b/iocore/net/SSLSessionCache.cc @@ -21,6 +21,8 @@ #include "P_SSLConfig.h" #include "SSLSessionCache.h" +#include "SSLStats.h" + #include #define SSLSESSIONCACHE_STRINGIFY0(x) #x @@ -34,7 +36,7 @@ #endif /* Session Cache */ -SSLSessionCache::SSLSessionCache() : session_bucket(nullptr), nbuckets(SSLConfigParams::session_cache_number_buckets) +SSLSessionCache::SSLSessionCache() : nbuckets(SSLConfigParams::session_cache_number_buckets) { Debug("ssl.session_cache", "Created new ssl session cache %p with %zu buckets each with size max size %zu", this, nbuckets, SSLConfigParams::session_cache_max_bucket_size); diff --git a/iocore/net/SSLSessionCache.h b/iocore/net/SSLSessionCache.h index 548359bc92f..a32809c3c6a 100644 --- a/iocore/net/SSLSessionCache.h +++ b/iocore/net/SSLSessionCache.h @@ -21,7 +21,6 @@ #pragma once -#include "tscore/Map.h" #include "tscore/List.h" #include "tscore/ink_mutex.h" #include "P_EventSystem.h" @@ -117,7 +116,7 @@ class SSLSession Ptr asn1_data; /* this is the ASN1 representation of the SSL_CTX */ size_t len_asn1_data; - SSLSession(const SSLSessionID &id, Ptr ssl_asn1_data, size_t len_asn1) + SSLSession(const SSLSessionID &id, const Ptr &ssl_asn1_data, size_t len_asn1) : session_id(id), asn1_data(ssl_asn1_data), len_asn1_data(len_asn1) { } @@ -154,7 +153,10 @@ class SSLSessionCache SSLSessionCache(); ~SSLSessionCache(); + SSLSessionCache(const SSLSessionCache &) = delete; + SSLSessionCache &operator=(const SSLSessionCache &) = delete; + private: - SSLSessionBucket *session_bucket; + SSLSessionBucket *session_bucket = nullptr; size_t nbuckets; }; diff --git a/iocore/net/SSLSessionTicket.cc b/iocore/net/SSLSessionTicket.cc new file mode 100644 index 00000000000..9346b29b655 --- /dev/null +++ b/iocore/net/SSLSessionTicket.cc @@ -0,0 +1,116 @@ +/** @file + + SessionTicket TLS Extension + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#include "SSLSessionTicket.h" + +#if TS_HAVE_OPENSSL_SESSION_TICKETS + +#include "tscore/MatcherUtils.h" + +#include "P_Net.h" +#include "SSLStats.h" +#include "P_SSLConfig.h" + +// Remove this when drop OpenSSL 1.0.2 support +#ifndef evp_md_func +#ifdef OPENSSL_NO_SHA256 +#define evp_md_func EVP_sha1() +#else +#define evp_md_func EVP_sha256() +#endif +#endif + +void +ssl_session_ticket_free(void * /*parent*/, void *ptr, CRYPTO_EX_DATA * /*ad*/, int /*idx*/, long /*argl*/, void * /*argp*/) +{ + ticket_block_free((struct ssl_ticket_key_block *)ptr); +} + +/* + * RFC 5077. Create session ticket to resume SSL session without requiring session-specific state at the TLS server. + * Specifically, it distributes the encrypted session-state information to the client in the form of a ticket and + * a mechanism to present the ticket back to the server. + * */ +int +ssl_callback_session_ticket(SSL *ssl, unsigned char *keyname, unsigned char *iv, EVP_CIPHER_CTX *cipher_ctx, HMAC_CTX *hctx, + int enc) +{ + SSLCertificateConfig::scoped_config lookup; + SSLTicketKeyConfig::scoped_config params; + SSLNetVConnection &netvc = *SSLNetVCAccess(ssl); + + // Get the IP address to look up the keyblock + IpEndpoint ip; + int namelen = sizeof(ip); + SSLCertContext *cc = nullptr; + if (0 == safe_getsockname(netvc.get_socket(), &ip.sa, &namelen)) { + cc = lookup->find(ip); + } + ssl_ticket_key_block *keyblock = nullptr; + if (cc == nullptr || cc->keyblock == nullptr) { + // Try the default + keyblock = params->default_global_keyblock; + } else { + keyblock = cc->keyblock; + } + ink_release_assert(keyblock != nullptr && keyblock->num_keys > 0); + + if (enc == 1) { + const ssl_ticket_key_t &most_recent_key = keyblock->keys[0]; + memcpy(keyname, most_recent_key.key_name, sizeof(most_recent_key.key_name)); + RAND_bytes(iv, EVP_MAX_IV_LENGTH); + EVP_EncryptInit_ex(cipher_ctx, EVP_aes_128_cbc(), nullptr, most_recent_key.aes_key, iv); + HMAC_Init_ex(hctx, most_recent_key.hmac_secret, sizeof(most_recent_key.hmac_secret), evp_md_func, nullptr); + + Debug("ssl", "create ticket for a new session."); + SSL_INCREMENT_DYN_STAT(ssl_total_tickets_created_stat); + return 1; + } else if (enc == 0) { + for (unsigned i = 0; i < keyblock->num_keys; ++i) { + if (memcmp(keyname, keyblock->keys[i].key_name, sizeof(keyblock->keys[i].key_name)) == 0) { + EVP_DecryptInit_ex(cipher_ctx, EVP_aes_128_cbc(), nullptr, keyblock->keys[i].aes_key, iv); + HMAC_Init_ex(hctx, keyblock->keys[i].hmac_secret, sizeof(keyblock->keys[i].hmac_secret), evp_md_func, nullptr); + + Debug("ssl", "verify the ticket for an existing session."); + // Increase the total number of decrypted tickets. + SSL_INCREMENT_DYN_STAT(ssl_total_tickets_verified_stat); + + if (i != 0) { // The number of tickets decrypted with "older" keys. + SSL_INCREMENT_DYN_STAT(ssl_total_tickets_verified_old_key_stat); + } + + netvc.setSSLSessionCacheHit(true); + // When we decrypt with an "older" key, encrypt the ticket again with the most recent key. + return (i == 0) ? 1 : 2; + } + } + + Debug("ssl", "keyname is not consistent."); + SSL_INCREMENT_DYN_STAT(ssl_total_tickets_not_found_stat); + return 0; + } + + return -1; +} + +#endif /* TS_HAVE_OPENSSL_SESSION_TICKETS */ diff --git a/proxy/logging/LogCollationBase.h b/iocore/net/SSLSessionTicket.h similarity index 58% rename from proxy/logging/LogCollationBase.h rename to iocore/net/SSLSessionTicket.h index 17b783a4595..be520cbfaf4 100644 --- a/proxy/logging/LogCollationBase.h +++ b/iocore/net/SSLSessionTicket.h @@ -1,6 +1,6 @@ /** @file - A brief file description + SessionTicket TLS Extension @section license License @@ -23,23 +23,20 @@ #pragma once -//------------------------------------------------------------------------- -// LogCollationBase -//------------------------------------------------------------------------- - -class LogCollationBase -{ -protected: - // ToDo: Can we we use the stuff from LogSock.h instead?? - struct NetMsgHeader { - int msg_bytes; // length of the following message - }; - - enum LogCollEvent { - LOG_COLL_EVENT_NULL = LOG_COLLATION_EVENT_EVENTS_START, - LOG_COLL_EVENT_SWITCH, - LOG_COLL_EVENT_READ_COMPLETE, - LOG_COLL_EVENT_WRITE_COMPLETE, - LOG_COLL_EVENT_ERROR - }; -}; +#include +#include + +// Check if the ticket_key callback #define is available, and if so, enable session tickets. +#ifdef SSL_CTX_set_tlsext_ticket_key_cb +#define TS_HAVE_OPENSSL_SESSION_TICKETS 1 +#endif + +#ifdef TS_HAVE_OPENSSL_SESSION_TICKETS + +#include +#include + +void ssl_session_ticket_free(void *, void *, CRYPTO_EX_DATA *, int, long, void *); +int ssl_callback_session_ticket(SSL *, unsigned char *, unsigned char *, EVP_CIPHER_CTX *, HMAC_CTX *, int); + +#endif /* TS_HAVE_OPENSSL_SESSION_TICKETS */ diff --git a/iocore/net/SSLStats.cc b/iocore/net/SSLStats.cc new file mode 100644 index 00000000000..b15e40d088c --- /dev/null +++ b/iocore/net/SSLStats.cc @@ -0,0 +1,241 @@ +/** @file + + Stats of TLS + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#include "SSLStats.h" + +#include + +#include "P_SSLConfig.h" +#include "P_SSLUtils.h" + +RecRawStatBlock *ssl_rsb = nullptr; +std::unordered_map cipher_map; + +static int +SSLRecRawStatSyncCount(const char *name, RecDataT data_type, RecData *data, RecRawStatBlock *rsb, int id) +{ + // Grab all the stats we want from OpenSSL and set the stats. This function only needs to be called by one of the + // involved stats, all others *must* call RecRawStatSyncSum. + SSLCertificateConfig::scoped_config certLookup; + + int64_t sessions = 0; + int64_t hits = 0; + int64_t misses = 0; + int64_t timeouts = 0; + + if (certLookup) { + const unsigned ctxCount = certLookup->count(); + for (size_t i = 0; i < ctxCount; i++) { + SSLCertContext *cc = certLookup->get(i); + if (cc && cc->ctx) { + sessions += SSL_CTX_sess_accept_good(cc->ctx); + hits += SSL_CTX_sess_hits(cc->ctx); + misses += SSL_CTX_sess_misses(cc->ctx); + timeouts += SSL_CTX_sess_timeouts(cc->ctx); + } + } + } + + SSL_SET_COUNT_DYN_STAT(ssl_user_agent_sessions_stat, sessions); + SSL_SET_COUNT_DYN_STAT(ssl_user_agent_session_hit_stat, hits); + SSL_SET_COUNT_DYN_STAT(ssl_user_agent_session_miss_stat, misses); + SSL_SET_COUNT_DYN_STAT(ssl_user_agent_session_timeout_stat, timeouts); + + return RecRawStatSyncCount(name, data_type, data, rsb, id); +} + +void +SSLInitializeStatistics() +{ + SSL_CTX *ctx; + SSL *ssl; + STACK_OF(SSL_CIPHER) * ciphers; + + // Allocate SSL statistics block. + ssl_rsb = RecAllocateRawStatBlock((int)Ssl_Stat_Count); + ink_assert(ssl_rsb != nullptr); + + // SSL client errors. + RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.user_agent_other_errors", RECD_COUNTER, RECP_PERSISTENT, + (int)ssl_user_agent_other_errors_stat, RecRawStatSyncSum); + RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.user_agent_expired_cert", RECD_COUNTER, RECP_PERSISTENT, + (int)ssl_user_agent_expired_cert_stat, RecRawStatSyncSum); + RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.user_agent_revoked_cert", RECD_COUNTER, RECP_PERSISTENT, + (int)ssl_user_agent_revoked_cert_stat, RecRawStatSyncSum); + RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.user_agent_unknown_cert", RECD_COUNTER, RECP_PERSISTENT, + (int)ssl_user_agent_unknown_cert_stat, RecRawStatSyncSum); + RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.user_agent_cert_verify_failed", RECD_COUNTER, RECP_PERSISTENT, + (int)ssl_user_agent_cert_verify_failed_stat, RecRawStatSyncSum); + RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.user_agent_bad_cert", RECD_COUNTER, RECP_PERSISTENT, + (int)ssl_user_agent_bad_cert_stat, RecRawStatSyncSum); + RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.user_agent_decryption_failed", RECD_COUNTER, RECP_PERSISTENT, + (int)ssl_user_agent_decryption_failed_stat, RecRawStatSyncSum); + RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.user_agent_wrong_version", RECD_COUNTER, RECP_PERSISTENT, + (int)ssl_user_agent_wrong_version_stat, RecRawStatSyncSum); + RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.user_agent_unknown_ca", RECD_COUNTER, RECP_PERSISTENT, + (int)ssl_user_agent_unknown_ca_stat, RecRawStatSyncSum); + + // Polled SSL context statistics. + RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.user_agent_sessions", RECD_COUNTER, RECP_NON_PERSISTENT, + (int)ssl_user_agent_sessions_stat, + SSLRecRawStatSyncCount); //<- only use this fn once + RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.user_agent_session_hit", RECD_COUNTER, RECP_NON_PERSISTENT, + (int)ssl_user_agent_session_hit_stat, RecRawStatSyncCount); + RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.user_agent_session_miss", RECD_COUNTER, RECP_NON_PERSISTENT, + (int)ssl_user_agent_session_miss_stat, RecRawStatSyncCount); + RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.user_agent_session_timeout", RECD_COUNTER, RECP_NON_PERSISTENT, + (int)ssl_user_agent_session_timeout_stat, RecRawStatSyncCount); + + // SSL server errors. + RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.origin_server_other_errors", RECD_COUNTER, RECP_PERSISTENT, + (int)ssl_origin_server_other_errors_stat, RecRawStatSyncSum); + RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.origin_server_expired_cert", RECD_COUNTER, RECP_PERSISTENT, + (int)ssl_origin_server_expired_cert_stat, RecRawStatSyncSum); + RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.origin_server_revoked_cert", RECD_COUNTER, RECP_PERSISTENT, + (int)ssl_origin_server_revoked_cert_stat, RecRawStatSyncSum); + RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.origin_server_unknown_cert", RECD_COUNTER, RECP_PERSISTENT, + (int)ssl_origin_server_unknown_cert_stat, RecRawStatSyncSum); + RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.origin_server_cert_verify_failed", RECD_COUNTER, RECP_PERSISTENT, + (int)ssl_origin_server_cert_verify_failed_stat, RecRawStatSyncSum); + RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.origin_server_bad_cert", RECD_COUNTER, RECP_PERSISTENT, + (int)ssl_origin_server_bad_cert_stat, RecRawStatSyncSum); + RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.origin_server_decryption_failed", RECD_COUNTER, RECP_PERSISTENT, + (int)ssl_origin_server_decryption_failed_stat, RecRawStatSyncSum); + RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.origin_server_wrong_version", RECD_COUNTER, RECP_PERSISTENT, + (int)ssl_origin_server_wrong_version_stat, RecRawStatSyncSum); + RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.origin_server_unknown_ca", RECD_COUNTER, RECP_PERSISTENT, + (int)ssl_origin_server_unknown_ca_stat, RecRawStatSyncSum); + + // SSL handshake time + RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.total_handshake_time", RECD_COUNTER, RECP_PERSISTENT, + (int)ssl_total_handshake_time_stat, RecRawStatSyncSum); + RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.total_success_handshake_count_in", RECD_COUNTER, RECP_PERSISTENT, + (int)ssl_total_success_handshake_count_in_stat, RecRawStatSyncCount); + RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.total_success_handshake_count_out", RECD_COUNTER, RECP_PERSISTENT, + (int)ssl_total_success_handshake_count_out_stat, RecRawStatSyncCount); + + // TLS tickets + RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.total_tickets_created", RECD_COUNTER, RECP_PERSISTENT, + (int)ssl_total_tickets_created_stat, RecRawStatSyncCount); + RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.total_tickets_verified", RECD_COUNTER, RECP_PERSISTENT, + (int)ssl_total_tickets_verified_stat, RecRawStatSyncCount); + RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.total_tickets_not_found", RECD_COUNTER, RECP_PERSISTENT, + (int)ssl_total_tickets_not_found_stat, RecRawStatSyncCount); + // TODO: ticket renewal is not used right now. + RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.total_tickets_renewed", RECD_COUNTER, RECP_PERSISTENT, + (int)ssl_total_tickets_renewed_stat, RecRawStatSyncCount); + // The number of session tickets verified with an "old" key. + RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.total_tickets_verified_old_key", RECD_COUNTER, RECP_PERSISTENT, + (int)ssl_total_tickets_verified_old_key_stat, RecRawStatSyncCount); + // The number of ticket keys renewed. + RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.total_ticket_keys_renewed", RECD_COUNTER, RECP_PERSISTENT, + (int)ssl_total_ticket_keys_renewed_stat, RecRawStatSyncCount); + + RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.ssl_session_cache_hit", RECD_COUNTER, RECP_PERSISTENT, + (int)ssl_session_cache_hit, RecRawStatSyncCount); + + RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.ssl_session_cache_new_session", RECD_COUNTER, RECP_PERSISTENT, + (int)ssl_session_cache_new_session, RecRawStatSyncCount); + + RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.ssl_session_cache_miss", RECD_COUNTER, RECP_PERSISTENT, + (int)ssl_session_cache_miss, RecRawStatSyncCount); + + RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.ssl_session_cache_eviction", RECD_COUNTER, RECP_PERSISTENT, + (int)ssl_session_cache_eviction, RecRawStatSyncCount); + + RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.ssl_session_cache_lock_contention", RECD_COUNTER, RECP_PERSISTENT, + (int)ssl_session_cache_lock_contention, RecRawStatSyncCount); + + /* Track dynamic record size */ + RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.default_record_size_count", RECD_COUNTER, RECP_PERSISTENT, + (int)ssl_total_dyn_def_tls_record_count, RecRawStatSyncSum); + RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.max_record_size_count", RECD_COUNTER, RECP_PERSISTENT, + (int)ssl_total_dyn_max_tls_record_count, RecRawStatSyncSum); + RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.redo_record_size_count", RECD_COUNTER, RECP_PERSISTENT, + (int)ssl_total_dyn_redo_tls_record_count, RecRawStatSyncCount); + + /* error stats */ + RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.ssl_error_want_write", RECD_COUNTER, RECP_PERSISTENT, + (int)ssl_error_want_write, RecRawStatSyncCount); + RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.ssl_error_want_read", RECD_COUNTER, RECP_PERSISTENT, + (int)ssl_error_want_read, RecRawStatSyncCount); + RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.ssl_error_want_x509_lookup", RECD_COUNTER, RECP_PERSISTENT, + (int)ssl_error_want_x509_lookup, RecRawStatSyncCount); + RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.ssl_error_syscall", RECD_COUNTER, RECP_PERSISTENT, + (int)ssl_error_syscall, RecRawStatSyncCount); + RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.ssl_error_read_eos", RECD_COUNTER, RECP_PERSISTENT, + (int)ssl_error_read_eos, RecRawStatSyncCount); + RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.ssl_error_zero_return", RECD_COUNTER, RECP_PERSISTENT, + (int)ssl_error_zero_return, RecRawStatSyncCount); + RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.ssl_error_ssl", RECD_COUNTER, RECP_PERSISTENT, (int)ssl_error_ssl, + RecRawStatSyncCount); + RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.ssl_sni_name_set_failure", RECD_COUNTER, RECP_PERSISTENT, + (int)ssl_sni_name_set_failure, RecRawStatSyncCount); + + /* ocsp stapling stats */ + RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.ssl_ocsp_revoked_cert_stat", RECD_COUNTER, RECP_PERSISTENT, + (int)ssl_ocsp_revoked_cert_stat, RecRawStatSyncCount); + RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.ssl_ocsp_unknown_cert_stat", RECD_COUNTER, RECP_PERSISTENT, + (int)ssl_ocsp_unknown_cert_stat, RecRawStatSyncCount); + RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.ssl_ocsp_refreshed_cert", RECD_INT, RECP_PERSISTENT, + (int)ssl_ocsp_refreshed_cert_stat, RecRawStatSyncCount); + RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.ssl_ocsp_refresh_cert_failure", RECD_INT, RECP_PERSISTENT, + (int)ssl_ocsp_refresh_cert_failure_stat, RecRawStatSyncCount); + + // Get and register the SSL cipher stats. Note that we are using the default SSL context to obtain + // the cipher list. This means that the set of ciphers is fixed by the build configuration and not + // filtered by proxy.config.ssl.server.cipher_suite. This keeps the set of cipher suites stable across + // configuration reloads and works for the case where we honor the client cipher preference. + + SSLMultiCertConfigLoader loader(nullptr); + ctx = loader.default_server_ssl_ctx(); + ssl = SSL_new(ctx); + ciphers = SSL_get_ciphers(ssl); + + // BoringSSL has sk_SSL_CIPHER_num() return a size_t (well, sk_num() is) + for (int index = 0; index < static_cast(sk_SSL_CIPHER_num(ciphers)); index++) { + SSL_CIPHER *cipher = const_cast(sk_SSL_CIPHER_value(ciphers, index)); + const char *cipherName = SSL_CIPHER_get_name(cipher); + std::string statName = "proxy.process.ssl.cipher.user_agent." + std::string(cipherName); + + // If room in allocated space ... + if ((ssl_cipher_stats_start + index) > ssl_cipher_stats_end) { + // Too many ciphers, increase ssl_cipher_stats_end. + SSLError("too many ciphers to register metric '%s', increase SSL_Stats::ssl_cipher_stats_end", statName.c_str()); + continue; + } + + // If not already registered ... + if (cipherName && cipher_map.find(cipherName) == cipher_map.end()) { + cipher_map.emplace(cipherName, (intptr_t)(ssl_cipher_stats_start + index)); + // Register as non-persistent since the order/index is dependent upon configuration. + RecRegisterRawStat(ssl_rsb, RECT_PROCESS, statName.c_str(), RECD_INT, RECP_NON_PERSISTENT, + (int)ssl_cipher_stats_start + index, RecRawStatSyncSum); + SSL_CLEAR_DYN_STAT((int)ssl_cipher_stats_start + index); + Debug("ssl", "registering SSL cipher metric '%s'", statName.c_str()); + } + } + + SSL_free(ssl); + SSLReleaseContext(ctx); +} diff --git a/iocore/net/SSLStats.h b/iocore/net/SSLStats.h new file mode 100644 index 00000000000..ff38df0d03f --- /dev/null +++ b/iocore/net/SSLStats.h @@ -0,0 +1,115 @@ +/** @file + + Stats of TLS + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#pragma once + +#include + +#include "records/I_RecProcess.h" +#include "SSLDiags.h" + +/* Stats should only be accessed using these macros */ +#define SSL_INCREMENT_DYN_STAT(x) RecIncrRawStat(ssl_rsb, nullptr, (int)x, 1) +#define SSL_DECREMENT_DYN_STAT(x) RecIncrRawStat(ssl_rsb, nullptr, (int)x, -1) +#define SSL_SET_COUNT_DYN_STAT(x, count) RecSetRawStatCount(ssl_rsb, x, count) +#define SSL_INCREMENT_DYN_STAT_EX(x, y) RecIncrRawStat(ssl_rsb, nullptr, (int)x, y) +#define SSL_CLEAR_DYN_STAT(x) \ + do { \ + RecSetRawStatSum(ssl_rsb, (x), 0); \ + RecSetRawStatCount(ssl_rsb, (x), 0); \ + } while (0) +#define SSL_CLR_ERR_INCR_DYN_STAT(vc, x, fmt, ...) \ + do { \ + SSLVCDebug((vc), fmt, ##__VA_ARGS__); \ + RecIncrRawStat(ssl_rsb, nullptr, (int)x, 1); \ + } while (0) + +enum SSL_Stats { + ssl_origin_server_expired_cert_stat, + ssl_user_agent_expired_cert_stat, + ssl_origin_server_revoked_cert_stat, + ssl_user_agent_revoked_cert_stat, + ssl_origin_server_unknown_cert_stat, + ssl_user_agent_unknown_cert_stat, + ssl_origin_server_cert_verify_failed_stat, + ssl_user_agent_cert_verify_failed_stat, + ssl_origin_server_bad_cert_stat, + ssl_user_agent_bad_cert_stat, + ssl_origin_server_decryption_failed_stat, + ssl_user_agent_decryption_failed_stat, + ssl_origin_server_wrong_version_stat, + ssl_user_agent_wrong_version_stat, + ssl_origin_server_other_errors_stat, + ssl_user_agent_other_errors_stat, + ssl_origin_server_unknown_ca_stat, + ssl_user_agent_unknown_ca_stat, + ssl_user_agent_sessions_stat, + ssl_user_agent_session_hit_stat, + ssl_user_agent_session_miss_stat, + ssl_user_agent_session_timeout_stat, + ssl_total_handshake_time_stat, + ssl_total_success_handshake_count_in_stat, + ssl_total_tickets_created_stat, + ssl_total_tickets_verified_stat, + ssl_total_tickets_verified_old_key_stat, // verified with old key. + ssl_total_ticket_keys_renewed_stat, // number of keys renewed. + ssl_total_tickets_not_found_stat, + ssl_total_tickets_renewed_stat, + ssl_total_dyn_def_tls_record_count, + ssl_total_dyn_max_tls_record_count, + ssl_total_dyn_redo_tls_record_count, + ssl_session_cache_hit, + ssl_session_cache_miss, + ssl_session_cache_eviction, + ssl_session_cache_lock_contention, + ssl_session_cache_new_session, + + /* error stats */ + ssl_error_want_write, + ssl_error_want_read, + ssl_error_want_client_hello_cb, + ssl_error_want_x509_lookup, + ssl_error_syscall, + ssl_error_read_eos, + ssl_error_zero_return, + ssl_error_ssl, + ssl_sni_name_set_failure, + ssl_total_success_handshake_count_out_stat, + + /* ocsp stapling stats */ + ssl_ocsp_revoked_cert_stat, + ssl_ocsp_unknown_cert_stat, + ssl_ocsp_refreshed_cert_stat, + ssl_ocsp_refresh_cert_failure_stat, + + ssl_cipher_stats_start = 100, + ssl_cipher_stats_end = 300, + + Ssl_Stat_Count +}; + +extern RecRawStatBlock *ssl_rsb; +extern std::unordered_map cipher_map; + +// Initialize SSL statistics. +void SSLInitializeStatistics(); diff --git a/iocore/net/SSLUtils.cc b/iocore/net/SSLUtils.cc index 9a528381653..efc3f3e428b 100644 --- a/iocore/net/SSLUtils.cc +++ b/iocore/net/SSLUtils.cc @@ -19,66 +19,64 @@ limitations under the License. */ +#include "P_SSLUtils.h" + +#include "tscpp/util/TextView.h" #include "tscore/ink_platform.h" #include "tscore/SimpleTokenizer.h" -#include "records/I_RecHttp.h" #include "tscore/I_Layout.h" -#include "P_Net.h" #include "tscore/ink_cap.h" #include "tscore/ink_mutex.h" +#include "records/I_RecHttp.h" + +#include "P_Net.h" +#include "InkAPIInternal.h" + #include "P_OCSPStapling.h" +#include "P_SSLSNI.h" +#include "P_SSLConfig.h" #include "SSLSessionCache.h" -#include "InkAPIInternal.h" +#include "SSLSessionTicket.h" #include "SSLDynlock.h" +#include "SSLDiags.h" +#include "SSLStats.h" #include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include #include #include #include -#include "P_SNIActionPerformer.h" -#if HAVE_OPENSSL_EVP_H +#include +#include +#include +#include +#include +#include +#include +#include #include -#endif - -#if HAVE_OPENSSL_HMAC_H -#include -#endif +#include +#include +#include #if HAVE_OPENSSL_TS_H #include #endif -#if HAVE_OPENSSL_EC_H -#include -#endif +using namespace std::literals; // ssl_multicert.config field names: -#define SSL_IP_TAG "dest_ip" -#define SSL_CERT_TAG "ssl_cert_name" -#define SSL_PRIVATE_KEY_TAG "ssl_key_name" -#define SSL_CA_TAG "ssl_ca_name" -#define SSL_ACTION_TAG "action" -#define SSL_ACTION_TUNNEL_TAG "tunnel" -#define SSL_SESSION_TICKET_ENABLED "ssl_ticket_enabled" -#define SSL_KEY_DIALOG "ssl_key_dialog" -#define SSL_SERVERNAME "dest_fqdn" -#define SSL_CERT_SEPARATE_DELIM ',' - -// openssl version must be 0.9.4 or greater -#if (OPENSSL_VERSION_NUMBER < 0x00090400L) -#error Traffic Server requires an OpenSSL library version 0.9.4 or greater -#endif +static constexpr std::string_view SSL_IP_TAG("dest_ip"sv); +static constexpr std::string_view SSL_CERT_TAG("ssl_cert_name"sv); +static constexpr std::string_view SSL_PRIVATE_KEY_TAG("ssl_key_name"sv); +static constexpr std::string_view SSL_OCSP_RESPONSE_TAG("ssl_ocsp_name"sv); +static constexpr std::string_view SSL_CA_TAG("ssl_ca_name"sv); +static constexpr std::string_view SSL_ACTION_TAG("action"sv); +static constexpr std::string_view SSL_ACTION_TUNNEL_TAG("tunnel"sv); +static constexpr std::string_view SSL_SESSION_TICKET_ENABLED("ssl_ticket_enabled"sv); +static constexpr std::string_view SSL_KEY_DIALOG("ssl_key_dialog"sv); +static constexpr std::string_view SSL_SERVERNAME("dest_fqdn"sv); +static constexpr char SSL_CERT_SEPARATE_DELIM = ','; #ifndef evp_md_func #ifdef OPENSSL_NO_SHA256 @@ -88,49 +86,9 @@ #endif #endif -TunnelHashMap TunnelMap; // stores the name of the servers to tunnel to -/* - * struct ssl_user_config: gather user provided settings from ssl_multicert.config in to this single struct - * ssl_ticket_enabled - session ticket enabled - * ssl_cert_name - certificate - * dest_ip - IPv[64] address to match - * ssl_cert_name - certificate - * first_cert - the first certificate name when multiple cert files are in 'ssl_cert_name' - * ssl_ca_name - CA public certificate - * ssl_key_name - Private key - * ticket_key_name - session key file. [key_name (16Byte) + HMAC_secret (16Byte) + AES_key (16Byte)] - * ssl_key_dialog - Private key dialog - * servername - Destination server - */ -struct ssl_user_config { - ssl_user_config() : opt(SSLCertContext::OPT_NONE) - { - REC_ReadConfigInt32(session_ticket_enabled, "proxy.config.ssl.server.session_ticket.enable"); - } - - int session_ticket_enabled; - ats_scoped_str addr; - ats_scoped_str cert; - ats_scoped_str first_cert; - ats_scoped_str ca; - ats_scoped_str key; - ats_scoped_str dialog; - ats_scoped_str servername; - SSLCertContext::Option opt; -}; - SSLSessionCache *session_cache; // declared extern in P_SSLConfig.h -// Check if the ticket_key callback #define is available, and if so, enable session tickets. -#ifdef SSL_CTX_set_tlsext_ticket_key_cb - -#define HAVE_OPENSSL_SESSION_TICKETS 1 - -static void session_ticket_free(void *, void *, CRYPTO_EX_DATA *, int, long, void *); -static int ssl_callback_session_ticket(SSL *, unsigned char *, unsigned char *, EVP_CIPHER_CTX *, HMAC_CTX *, int); -#endif /* SSL_CTX_set_tlsext_ticket_key_cb */ - -#if HAVE_OPENSSL_SESSION_TICKETS +#if TS_HAVE_OPENSSL_SESSION_TICKETS static int ssl_session_ticket_index = -1; #endif @@ -139,9 +97,6 @@ static int ssl_vc_index = -1; static ink_mutex *mutex_buf = nullptr; static bool open_ssl_initialized = false; -RecRawStatBlock *ssl_rsb = nullptr; -HashMap cipher_map; - /* Using pthread thread ID and mutex functions directly, instead of * ATS this_ethread / ProxyMutex, so that other linked libraries * may use pthreads and openssl without confusing us here. (TS-2271). @@ -215,7 +170,7 @@ SSL_CTX_add_extra_chain_cert_file(SSL_CTX *ctx, const char *chainfile) return SSL_CTX_add_extra_chain_cert_bio(ctx, bio); } -bool +static bool ssl_session_timed_out(SSL_SESSION *session) { return SSL_SESSION_get_timeout(session) < (long)(time(nullptr) - SSL_SESSION_get_time(session)); @@ -239,7 +194,7 @@ ssl_get_cached_session(SSL *ssl, const unsigned char *id, int len, int *copy) Debug("ssl.session_cache.get", "ssl_get_cached_session cached session '%s' context %p", printable_buf, SSL_get_SSL_CTX(ssl)); } - APIHook *hook = ssl_hooks->get(TS_SSL_SESSION_INTERNAL_HOOK); + APIHook *hook = ssl_hooks->get(TSSslHookInternalID(TS_SSL_SESSION_HOOK)); while (hook) { hook->invoke(TS_EVENT_SSL_SESSION_GET, &sid); hook = hook->m_link.next; @@ -287,7 +242,7 @@ ssl_new_cached_session(SSL *ssl, SSL_SESSION *sess) session_cache->insertSession(sid, sess); // Call hook after new session is created - APIHook *hook = ssl_hooks->get(TS_SSL_SESSION_INTERNAL_HOOK); + APIHook *hook = ssl_hooks->get(TSSslHookInternalID(TS_SSL_SESSION_HOOK)); while (hook) { hook->invoke(TS_EVENT_SSL_SESSION_NEW, &sid); hook = hook->m_link.next; @@ -304,7 +259,7 @@ ssl_rm_cached_session(SSL_CTX *ctx, SSL_SESSION *sess) SSLSessionID sid(id, len); // Call hook before session is removed - APIHook *hook = ssl_hooks->get(TS_SSL_SESSION_INTERNAL_HOOK); + APIHook *hook = ssl_hooks->get(TSSslHookInternalID(TS_SSL_SESSION_HOOK)); while (hook) { hook->invoke(TS_EVENT_SSL_SESSION_REMOVE, &sid); hook = hook->m_link.next; @@ -319,7 +274,7 @@ ssl_rm_cached_session(SSL_CTX *ctx, SSL_SESSION *sess) session_cache->removeSession(sid); } -int +static int set_context_cert(SSL *ssl) { SSL_CTX *ctx = nullptr; @@ -331,11 +286,6 @@ set_context_cert(SSL *ssl) int retval = 1; Debug("ssl", "set_context_cert ssl=%p server=%s handshake_complete=%d", ssl, servername, netvc->getSSLHandShakeComplete()); - if (SSLConfigParams::ssl_wire_trace_enabled) { - bool trace = netvc->computeSSLTrace(); - Debug("ssl", "sslnetvc. setting trace to=%s", trace ? "true" : "false"); - netvc->setSSLTrace(trace); - } // catch the client renegotiation early on if (SSLConfigParams::ssl_allow_client_renegotiation == false && netvc->getSSLHandShakeComplete()) { @@ -375,7 +325,7 @@ set_context_cert(SSL *ssl) if (ctx != nullptr) { SSL_set_SSL_CTX(ssl, ctx); -#if HAVE_OPENSSL_SESSION_TICKETS +#if TS_HAVE_OPENSSL_SESSION_TICKETS // Reset the ticket callback if needed SSL_CTX_set_tlsext_ticket_key_cb(ctx, ssl_callback_session_ticket); #endif @@ -395,7 +345,7 @@ set_context_cert(SSL *ssl) } // Callback function for verifying client certificate -int +static int ssl_verify_client_callback(int preverify_ok, X509_STORE_CTX *ctx) { Debug("ssl", "Callback: verify client cert"); @@ -403,12 +353,83 @@ ssl_verify_client_callback(int preverify_ok, X509_STORE_CTX *ctx) SSLNetVConnection *netvc = SSLNetVCAccess(ssl); netvc->callHooks(TS_EVENT_SSL_VERIFY_CLIENT); + return preverify_ok; +} + +static int +PerformAction(Continuation *cont, const char *servername) +{ + SNIConfig::scoped_config params; + const actionVector *actionvec = params->get(servername); + if (!actionvec) { + Debug("ssl_sni", "%s not available in the map", servername); + } else { + for (auto &&item : *actionvec) { + auto ret = item->SNIAction(cont); + if (ret != SSL_TLSEXT_ERR_OK) { + return ret; + } + } + } return SSL_TLSEXT_ERR_OK; } -// Use the certificate callback for openssl 1.0.2 and greater -// otherwise use the SNI callback -#if TS_USE_CERT_CB +#if TS_USE_HELLO_CB +// Pausable callback +static int +ssl_client_hello_callback(SSL *s, int *al, void *arg) +{ + SSLNetVConnection *netvc = SSLNetVCAccess(s); + const char *servername = nullptr; + const unsigned char *p; + size_t remaining, len; + // Parse the server name if the get extension call succeeds and there are more than 2 bytes to parse + if (SSL_client_hello_get0_ext(s, TLSEXT_TYPE_server_name, &p, &remaining) && remaining > 2) { + // Parse to get to the name, originally from test/handshake_helper.c in openssl tree + /* Extract the length of the supplied list of names. */ + len = *(p++) << 8; + len += *(p++); + if (len + 2 == remaining) { + remaining = len; + /* + * The list in practice only has a single element, so we only consider + * the first one. + */ + if (remaining != 0 && *p++ == TLSEXT_NAMETYPE_host_name) { + remaining--; + /* Now we can finally pull out the byte array with the actual hostname. */ + if (remaining > 2) { + len = *(p++) << 8; + len += *(p++); + if (len + 2 <= remaining) { + remaining = len; + servername = reinterpret_cast(p); + } + } + } + } + } + netvc->serverName = servername ? servername : ""; + int ret = PerformAction(netvc, netvc->serverName); + if (ret != SSL_TLSEXT_ERR_OK) { + return SSL_CLIENT_HELLO_ERROR; + } + if (netvc->has_tunnel_destination() && !netvc->decrypt_tunnel()) { + netvc->attributes = HttpProxyPort::TRANSPORT_BLIND_TUNNEL; + } + if (netvc->protocol_mask_set) { + setTLSValidProtocols(s, netvc->protocol_mask, TLSValidProtocols::max_mask); + } + + bool reenabled = netvc->callHooks(TS_EVENT_SSL_CLIENT_HELLO); + + if (!reenabled) { + return SSL_CLIENT_HELLO_RETRY; + } + return SSL_CLIENT_HELLO_SUCCESS; +} +#endif + /** * Called before either the server or the client certificate is used * Return 1 on success, 0 on error, or -1 to pause @@ -420,6 +441,11 @@ ssl_cert_callback(SSL *ssl, void * /*arg*/) bool reenabled; int retval = 1; + // If we are in tunnel mode, don't select a cert. Pause! + if (HttpProxyPort::TRANSPORT_BLIND_TUNNEL == netvc->attributes) { + return -1; // Pause + } + // Do the common certificate lookup only once. If we pause // and restart processing, do not execute the common logic again if (!netvc->calledHooks(TS_EVENT_SSL_CERT)) { @@ -444,73 +470,27 @@ ssl_cert_callback(SSL *ssl, void * /*arg*/) /* * Cannot stop this callback. Always reeneabled */ -extern SNIActionPerformer sni_action_performer; static int -ssl_servername_only_callback(SSL *ssl, int * /* ad */, void * /*arg*/) +ssl_servername_callback(SSL *ssl, int * /* ad */, void * /*arg*/) { - int ret = SSL_TLSEXT_ERR_OK; SSLNetVConnection *netvc = SSLNetVCAccess(ssl); - const char *servername = SSL_get_servername(ssl, TLSEXT_NAMETYPE_host_name); - Debug("ssl", "Requested servername is %s", servername); - if (servername != nullptr) { - ret = sni_action_performer.PerformAction(netvc, servername); - } - if (ret != SSL_TLSEXT_ERR_OK) - return SSL_TLSEXT_ERR_ALERT_FATAL; - netvc->callHooks(TS_EVENT_SSL_SERVERNAME); - return SSL_TLSEXT_ERR_OK; -} - -#else -static int -ssl_servername_and_cert_callback(SSL *ssl, int * /* ad */, void * /*arg*/) -{ - SSLNetVConnection *netvc = SSLNetVCAccess(ssl); - bool reenabled; - int retval = 1; - // Do the common certificate lookup only once. If we pause - // and restart processing, do not execute the common logic again - if (!netvc->calledHooks(TS_EVENT_SSL_CERT)) { - retval = set_context_cert(ssl); - if (retval != 1) { - goto done; - } + netvc->serverName = SSL_get_servername(ssl, TLSEXT_NAMETYPE_host_name); + if (nullptr == netvc->serverName) { + netvc->serverName = ""; } - // Call the plugin SNI code - reenabled = netvc->callHooks(TS_EVENT_SSL_CERT); - // If it did not re-enable, return the code to - // stop the accept processing - if (!reenabled) { - retval = -1; + // Rerun the actions in case a plugin changed the server name + int ret = PerformAction(netvc, netvc->serverName); + if (ret != SSL_TLSEXT_ERR_OK) { + return SSL_TLSEXT_ERR_ALERT_FATAL; } - -done: - // Map 1 to SSL_TLSEXT_ERR_OK - // Map 0 to SSL_TLSEXT_ERR_ALERT_FATAL - // Map -1 to SSL_TLSEXT_ERR_READ_AGAIN, if present - switch (retval) { - case 1: - retval = SSL_TLSEXT_ERR_OK; - break; - case -1: -#ifdef SSL_TLSEXT_ERR_READ_AGAIN - retval = SSL_TLSEXT_ERR_READ_AGAIN; -#else - Error("Cannot pause SNI processsing with this version of openssl"); - retval = SSL_TLSEXT_ERR_ALERT_FATAL; -#endif - break; - case 0: - default: - retval = SSL_TLSEXT_ERR_ALERT_FATAL; - break; + if (netvc->has_tunnel_destination() && !netvc->decrypt_tunnel()) { + netvc->attributes = HttpProxyPort::TRANSPORT_BLIND_TUNNEL; } - return retval; + return SSL_TLSEXT_ERR_OK; } -#endif #if TS_USE_GET_DH_2048_256 == 0 /* Build 2048-bit MODP Group with 256-bit Prime Order Subgroup from RFC 5114 */ @@ -594,20 +574,17 @@ ssl_context_enable_ecdh(SSL_CTX *ctx) { #if OPENSSL_VERSION_NUMBER < 0x10100000 -#if TS_USE_TLS_ECKEY - -#if defined(SSL_CTRL_SET_ECDH_AUTO) +#if defined(SSL_CTX_set_ecdh_auto) SSL_CTX_set_ecdh_auto(ctx, 1); -#elif defined(HAVE_EC_KEY_NEW_BY_CURVE_NAME) && defined(NID_X9_62_prime256v1) +#elif defined(NID_X9_62_prime256v1) EC_KEY *ecdh = EC_KEY_new_by_curve_name(NID_X9_62_prime256v1); if (ecdh) { SSL_CTX_set_tmp_ecdh(ctx, ecdh); EC_KEY_free(ecdh); } -#endif -#endif -#endif +#endif /* SSL_CTRL_SET_ECDH_AUTO */ +#endif /* OPENSSL_VERSION_NUMBER */ return ctx; } @@ -615,7 +592,7 @@ ssl_context_enable_ecdh(SSL_CTX *ctx) static ssl_ticket_key_block * ssl_context_enable_tickets(SSL_CTX *ctx, const char *ticket_key_path) { -#if HAVE_OPENSSL_SESSION_TICKETS +#if TS_HAVE_OPENSSL_SESSION_TICKETS ssl_ticket_key_block *keyblock = nullptr; keyblock = ssl_create_ticket_keyblock(ticket_key_path); @@ -637,10 +614,10 @@ ssl_context_enable_tickets(SSL_CTX *ctx, const char *ticket_key_path) SSL_CTX_clear_options(ctx, SSL_OP_NO_TICKET); return keyblock; -#else /* !HAVE_OPENSSL_SESSION_TICKETS */ +#else /* !TS_HAVE_OPENSSL_SESSION_TICKETS */ (void)ticket_key_path; return nullptr; -#endif /* HAVE_OPENSSL_SESSION_TICKETS */ +#endif /* TS_HAVE_OPENSSL_SESSION_TICKETS */ } struct passphrase_cb_userdata { @@ -812,38 +789,6 @@ ssl_private_key_validate_exec(const char *cmdLine) return bReturn; } -static int -SSLRecRawStatSyncCount(const char *name, RecDataT data_type, RecData *data, RecRawStatBlock *rsb, int id) -{ - // Grab all the stats we want from OpenSSL and set the stats. This function only needs to be called by one of the - // involved stats, all others *must* call RecRawStatSyncSum. - SSLCertificateConfig::scoped_config certLookup; - - int64_t sessions = 0; - int64_t hits = 0; - int64_t misses = 0; - int64_t timeouts = 0; - - if (certLookup) { - const unsigned ctxCount = certLookup->count(); - for (size_t i = 0; i < ctxCount; i++) { - SSLCertContext *cc = certLookup->get(i); - if (cc && cc->ctx) { - sessions += SSL_CTX_sess_accept_good(cc->ctx); - hits += SSL_CTX_sess_hits(cc->ctx); - misses += SSL_CTX_sess_misses(cc->ctx); - timeouts += SSL_CTX_sess_timeouts(cc->ctx); - } - } - } - - SSL_SET_COUNT_DYN_STAT(ssl_user_agent_sessions_stat, sessions); - SSL_SET_COUNT_DYN_STAT(ssl_user_agent_session_hit_stat, hits); - SSL_SET_COUNT_DYN_STAT(ssl_user_agent_session_miss_stat, misses); - SSL_SET_COUNT_DYN_STAT(ssl_user_agent_session_timeout_stat, timeouts); - return RecRawStatSyncCount(name, data_type, data, rsb, id); -} - #if OPENSSL_VERSION_NUMBER < 0x10100000L || defined(LIBRESSL_VERSION_NUMBER) #define ssl_malloc(size, file, line) ssl_malloc(size) #define ssl_realloc(ptr, size, file, line) ssl_realloc(ptr, size) @@ -948,14 +893,14 @@ SSLInitializeLibrary() CRYPTO_set_dynlock_destroy_callback(ssl_dyn_destroy_callback); } -#ifdef SSL_CTX_set_tlsext_ticket_key_cb - ssl_session_ticket_index = SSL_CTX_get_ex_new_index(0, nullptr, nullptr, nullptr, session_ticket_free); +#ifdef TS_HAVE_OPENSSL_SESSION_TICKETS + ssl_session_ticket_index = SSL_CTX_get_ex_new_index(0, nullptr, nullptr, nullptr, ssl_session_ticket_free); if (ssl_session_ticket_index == -1) { SSLError("failed to create session ticket index"); } #endif -#ifdef TS_USE_TLS_OCSP +#if TS_USE_TLS_OCSP ssl_stapling_ex_init(); #endif /* TS_USE_TLS_OCSP */ @@ -966,378 +911,8 @@ SSLInitializeLibrary() open_ssl_initialized = true; } -void -SSLInitializeStatistics() -{ - SSL_CTX *ctx; - SSL *ssl; - STACK_OF(SSL_CIPHER) * ciphers; - - // Allocate SSL statistics block. - ssl_rsb = RecAllocateRawStatBlock((int)Ssl_Stat_Count); - ink_assert(ssl_rsb != nullptr); - - // SSL client errors. - RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.user_agent_other_errors", RECD_COUNTER, RECP_PERSISTENT, - (int)ssl_user_agent_other_errors_stat, RecRawStatSyncSum); - RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.user_agent_expired_cert", RECD_COUNTER, RECP_PERSISTENT, - (int)ssl_user_agent_expired_cert_stat, RecRawStatSyncSum); - RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.user_agent_revoked_cert", RECD_COUNTER, RECP_PERSISTENT, - (int)ssl_user_agent_revoked_cert_stat, RecRawStatSyncSum); - RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.user_agent_unknown_cert", RECD_COUNTER, RECP_PERSISTENT, - (int)ssl_user_agent_unknown_cert_stat, RecRawStatSyncSum); - RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.user_agent_cert_verify_failed", RECD_COUNTER, RECP_PERSISTENT, - (int)ssl_user_agent_cert_verify_failed_stat, RecRawStatSyncSum); - RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.user_agent_bad_cert", RECD_COUNTER, RECP_PERSISTENT, - (int)ssl_user_agent_bad_cert_stat, RecRawStatSyncSum); - RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.user_agent_decryption_failed", RECD_COUNTER, RECP_PERSISTENT, - (int)ssl_user_agent_decryption_failed_stat, RecRawStatSyncSum); - RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.user_agent_wrong_version", RECD_COUNTER, RECP_PERSISTENT, - (int)ssl_user_agent_wrong_version_stat, RecRawStatSyncSum); - RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.user_agent_unknown_ca", RECD_COUNTER, RECP_PERSISTENT, - (int)ssl_user_agent_unknown_ca_stat, RecRawStatSyncSum); - - // Polled SSL context statistics. - RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.user_agent_sessions", RECD_COUNTER, RECP_NON_PERSISTENT, - (int)ssl_user_agent_sessions_stat, - SSLRecRawStatSyncCount); //<- only use this fn once - RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.user_agent_session_hit", RECD_COUNTER, RECP_NON_PERSISTENT, - (int)ssl_user_agent_session_hit_stat, RecRawStatSyncCount); - RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.user_agent_session_miss", RECD_COUNTER, RECP_NON_PERSISTENT, - (int)ssl_user_agent_session_miss_stat, RecRawStatSyncCount); - RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.user_agent_session_timeout", RECD_COUNTER, RECP_NON_PERSISTENT, - (int)ssl_user_agent_session_timeout_stat, RecRawStatSyncCount); - - // SSL server errors. - RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.origin_server_other_errors", RECD_COUNTER, RECP_PERSISTENT, - (int)ssl_origin_server_other_errors_stat, RecRawStatSyncSum); - RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.origin_server_expired_cert", RECD_COUNTER, RECP_PERSISTENT, - (int)ssl_origin_server_expired_cert_stat, RecRawStatSyncSum); - RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.origin_server_revoked_cert", RECD_COUNTER, RECP_PERSISTENT, - (int)ssl_origin_server_revoked_cert_stat, RecRawStatSyncSum); - RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.origin_server_unknown_cert", RECD_COUNTER, RECP_PERSISTENT, - (int)ssl_origin_server_unknown_cert_stat, RecRawStatSyncSum); - RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.origin_server_cert_verify_failed", RECD_COUNTER, RECP_PERSISTENT, - (int)ssl_origin_server_cert_verify_failed_stat, RecRawStatSyncSum); - RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.origin_server_bad_cert", RECD_COUNTER, RECP_PERSISTENT, - (int)ssl_origin_server_bad_cert_stat, RecRawStatSyncSum); - RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.origin_server_decryption_failed", RECD_COUNTER, RECP_PERSISTENT, - (int)ssl_origin_server_decryption_failed_stat, RecRawStatSyncSum); - RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.origin_server_wrong_version", RECD_COUNTER, RECP_PERSISTENT, - (int)ssl_origin_server_wrong_version_stat, RecRawStatSyncSum); - RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.origin_server_unknown_ca", RECD_COUNTER, RECP_PERSISTENT, - (int)ssl_origin_server_unknown_ca_stat, RecRawStatSyncSum); - - // SSL handshake time - RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.total_handshake_time", RECD_COUNTER, RECP_PERSISTENT, - (int)ssl_total_handshake_time_stat, RecRawStatSyncSum); - RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.total_success_handshake_count_in", RECD_COUNTER, RECP_PERSISTENT, - (int)ssl_total_success_handshake_count_in_stat, RecRawStatSyncCount); - RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.total_success_handshake_count_out", RECD_COUNTER, RECP_PERSISTENT, - (int)ssl_total_success_handshake_count_out_stat, RecRawStatSyncCount); - - // TLS tickets - RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.total_tickets_created", RECD_COUNTER, RECP_PERSISTENT, - (int)ssl_total_tickets_created_stat, RecRawStatSyncCount); - RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.total_tickets_verified", RECD_COUNTER, RECP_PERSISTENT, - (int)ssl_total_tickets_verified_stat, RecRawStatSyncCount); - RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.total_tickets_not_found", RECD_COUNTER, RECP_PERSISTENT, - (int)ssl_total_tickets_not_found_stat, RecRawStatSyncCount); - // TODO: ticket renewal is not used right now. - RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.total_tickets_renewed", RECD_COUNTER, RECP_PERSISTENT, - (int)ssl_total_tickets_renewed_stat, RecRawStatSyncCount); - // The number of session tickets verified with an "old" key. - RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.total_tickets_verified_old_key", RECD_COUNTER, RECP_PERSISTENT, - (int)ssl_total_tickets_verified_old_key_stat, RecRawStatSyncCount); - // The number of ticket keys renewed. - RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.total_ticket_keys_renewed", RECD_COUNTER, RECP_PERSISTENT, - (int)ssl_total_ticket_keys_renewed_stat, RecRawStatSyncCount); - - RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.ssl_session_cache_hit", RECD_COUNTER, RECP_PERSISTENT, - (int)ssl_session_cache_hit, RecRawStatSyncCount); - - RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.ssl_session_cache_new_session", RECD_COUNTER, RECP_PERSISTENT, - (int)ssl_session_cache_new_session, RecRawStatSyncCount); - - RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.ssl_session_cache_miss", RECD_COUNTER, RECP_PERSISTENT, - (int)ssl_session_cache_miss, RecRawStatSyncCount); - - RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.ssl_session_cache_eviction", RECD_COUNTER, RECP_PERSISTENT, - (int)ssl_session_cache_eviction, RecRawStatSyncCount); - - RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.ssl_session_cache_lock_contention", RECD_COUNTER, RECP_PERSISTENT, - (int)ssl_session_cache_lock_contention, RecRawStatSyncCount); - - /* Track dynamic record size */ - RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.default_record_size_count", RECD_COUNTER, RECP_PERSISTENT, - (int)ssl_total_dyn_def_tls_record_count, RecRawStatSyncSum); - RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.max_record_size_count", RECD_COUNTER, RECP_PERSISTENT, - (int)ssl_total_dyn_max_tls_record_count, RecRawStatSyncSum); - RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.redo_record_size_count", RECD_COUNTER, RECP_PERSISTENT, - (int)ssl_total_dyn_redo_tls_record_count, RecRawStatSyncCount); - - /* error stats */ - RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.ssl_error_want_write", RECD_COUNTER, RECP_PERSISTENT, - (int)ssl_error_want_write, RecRawStatSyncCount); - RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.ssl_error_want_read", RECD_COUNTER, RECP_PERSISTENT, - (int)ssl_error_want_read, RecRawStatSyncCount); - RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.ssl_error_want_x509_lookup", RECD_COUNTER, RECP_PERSISTENT, - (int)ssl_error_want_x509_lookup, RecRawStatSyncCount); - RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.ssl_error_syscall", RECD_COUNTER, RECP_PERSISTENT, - (int)ssl_error_syscall, RecRawStatSyncCount); - RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.ssl_error_read_eos", RECD_COUNTER, RECP_PERSISTENT, - (int)ssl_error_read_eos, RecRawStatSyncCount); - RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.ssl_error_zero_return", RECD_COUNTER, RECP_PERSISTENT, - (int)ssl_error_zero_return, RecRawStatSyncCount); - RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.ssl_error_ssl", RECD_COUNTER, RECP_PERSISTENT, (int)ssl_error_ssl, - RecRawStatSyncCount); - RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.ssl_sni_name_set_failure", RECD_COUNTER, RECP_PERSISTENT, - (int)ssl_sni_name_set_failure, RecRawStatSyncCount); - - /* ocsp stapling stats */ - RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.ssl_ocsp_revoked_cert_stat", RECD_COUNTER, RECP_PERSISTENT, - (int)ssl_ocsp_revoked_cert_stat, RecRawStatSyncCount); - RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.ssl_ocsp_unknown_cert_stat", RECD_COUNTER, RECP_PERSISTENT, - (int)ssl_ocsp_unknown_cert_stat, RecRawStatSyncCount); - RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.ssl_ocsp_refreshed_cert", RECD_INT, RECP_PERSISTENT, - (int)ssl_ocsp_refreshed_cert_stat, RecRawStatSyncCount); - RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.ssl_ocsp_refresh_cert_failure", RECD_INT, RECP_PERSISTENT, - (int)ssl_ocsp_refresh_cert_failure_stat, RecRawStatSyncCount); - - // Get and register the SSL cipher stats. Note that we are using the default SSL context to obtain - // the cipher list. This means that the set of ciphers is fixed by the build configuration and not - // filtered by proxy.config.ssl.server.cipher_suite. This keeps the set of cipher suites stable across - // configuration reloads and works for the case where we honor the client cipher preference. - - ctx = SSLDefaultServerContext(); - ssl = SSL_new(ctx); - ciphers = SSL_get_ciphers(ssl); - - // BoringSSL has sk_SSL_CIPHER_num() return a size_t (well, sk_num() is) - for (int index = 0; index < static_cast(sk_SSL_CIPHER_num(ciphers)); index++) { - SSL_CIPHER *cipher = const_cast(sk_SSL_CIPHER_value(ciphers, index)); - const char *cipherName = SSL_CIPHER_get_name(cipher); - std::string statName = "proxy.process.ssl.cipher.user_agent." + std::string(cipherName); - - // If room in allocated space ... - if ((ssl_cipher_stats_start + index) > ssl_cipher_stats_end) { - // Too many ciphers, increase ssl_cipher_stats_end. - SSLError("too many ciphers to register metric '%s', increase SSL_Stats::ssl_cipher_stats_end", statName.c_str()); - continue; - } - - // If not already registered ... - if (0 == cipher_map.get(cipherName)) { - cipher_map.put(cipherName, (intptr_t)(ssl_cipher_stats_start + index)); - // Register as non-persistent since the order/index is dependent upon configuration. - RecRegisterRawStat(ssl_rsb, RECT_PROCESS, statName.c_str(), RECD_INT, RECP_NON_PERSISTENT, - (int)ssl_cipher_stats_start + index, RecRawStatSyncSum); - SSL_CLEAR_DYN_STAT((int)ssl_cipher_stats_start + index); - Debug("ssl", "registering SSL cipher metric '%s'", statName.c_str()); - } - } - - SSL_free(ssl); - SSLReleaseContext(ctx); -} - -// return true if we have a stat for the error -static bool -increment_ssl_client_error(unsigned long err) -{ - // we only look for LIB_SSL errors atm - if (ERR_LIB_SSL != ERR_GET_LIB(err)) { - SSL_INCREMENT_DYN_STAT(ssl_user_agent_other_errors_stat); - return false; - } - - // error was in LIB_SSL, now just switch on REASON - // (we ignore FUNCTION with the prejudice that we don't care what function - // the error came from, hope that's ok?) - switch (ERR_GET_REASON(err)) { -#ifdef SSL_R_SSLV3_ALERT_CERTIFICATE_EXPIRED - case SSL_R_SSLV3_ALERT_CERTIFICATE_EXPIRED: - SSL_INCREMENT_DYN_STAT(ssl_user_agent_expired_cert_stat); - break; -#endif -#ifdef SSL_R_SSLV3_ALERT_CERTIFICATE_REVOKED - case SSL_R_SSLV3_ALERT_CERTIFICATE_REVOKED: - SSL_INCREMENT_DYN_STAT(ssl_user_agent_revoked_cert_stat); - break; -#endif -#ifdef SSL_R_SSLV3_ALERT_CERTIFICATE_UNKNOWN - case SSL_R_SSLV3_ALERT_CERTIFICATE_UNKNOWN: - SSL_INCREMENT_DYN_STAT(ssl_user_agent_unknown_cert_stat); - break; -#endif - case SSL_R_CERTIFICATE_VERIFY_FAILED: - SSL_INCREMENT_DYN_STAT(ssl_user_agent_cert_verify_failed_stat); - break; -#ifdef SSL_R_SSLV3_ALERT_BAD_CERTIFICATE - case SSL_R_SSLV3_ALERT_BAD_CERTIFICATE: - SSL_INCREMENT_DYN_STAT(ssl_user_agent_bad_cert_stat); - break; -#endif -#ifdef SSL_R_TLSV1_ALERT_DECRYPTION_FAILED - case SSL_R_TLSV1_ALERT_DECRYPTION_FAILED: - SSL_INCREMENT_DYN_STAT(ssl_user_agent_decryption_failed_stat); - break; -#endif - case SSL_R_WRONG_VERSION_NUMBER: - SSL_INCREMENT_DYN_STAT(ssl_user_agent_wrong_version_stat); - break; -#ifdef SSL_R_TLSV1_ALERT_UNKNOWN_CA - case SSL_R_TLSV1_ALERT_UNKNOWN_CA: - SSL_INCREMENT_DYN_STAT(ssl_user_agent_unknown_ca_stat); - break; -#endif - default: - SSL_INCREMENT_DYN_STAT(ssl_user_agent_other_errors_stat); - return false; - } - - return true; -} - -// return true if we have a stat for the error - -static bool -increment_ssl_server_error(unsigned long err) -{ - // we only look for LIB_SSL errors atm - if (ERR_LIB_SSL != ERR_GET_LIB(err)) { - SSL_INCREMENT_DYN_STAT(ssl_origin_server_other_errors_stat); - return false; - } - - // error was in LIB_SSL, now just switch on REASON - // (we ignore FUNCTION with the prejudice that we don't care what function - // the error came from, hope that's ok?) - switch (ERR_GET_REASON(err)) { -#ifdef SSL_R_SSLV3_ALERT_CERTIFICATE_EXPIRED - case SSL_R_SSLV3_ALERT_CERTIFICATE_EXPIRED: - SSL_INCREMENT_DYN_STAT(ssl_origin_server_expired_cert_stat); - break; -#endif -#ifdef SSL_R_SSLV3_ALERT_CERTIFICATE_REVOKED - case SSL_R_SSLV3_ALERT_CERTIFICATE_REVOKED: - SSL_INCREMENT_DYN_STAT(ssl_origin_server_revoked_cert_stat); - break; -#endif -#ifdef SSL_R_SSLV3_ALERT_CERTIFICATE_UNKNOWN - case SSL_R_SSLV3_ALERT_CERTIFICATE_UNKNOWN: - SSL_INCREMENT_DYN_STAT(ssl_origin_server_unknown_cert_stat); - break; -#endif - case SSL_R_CERTIFICATE_VERIFY_FAILED: - SSL_INCREMENT_DYN_STAT(ssl_origin_server_cert_verify_failed_stat); - break; -#ifdef SSL_R_SSLV3_ALERT_BAD_CERTIFICATE - case SSL_R_SSLV3_ALERT_BAD_CERTIFICATE: - SSL_INCREMENT_DYN_STAT(ssl_origin_server_bad_cert_stat); - break; -#endif -#ifdef SSL_R_TLSV1_ALERT_DECRYPTION_FAILED - case SSL_R_TLSV1_ALERT_DECRYPTION_FAILED: - SSL_INCREMENT_DYN_STAT(ssl_origin_server_decryption_failed_stat); - break; -#endif - case SSL_R_WRONG_VERSION_NUMBER: - SSL_INCREMENT_DYN_STAT(ssl_origin_server_wrong_version_stat); - break; -#ifdef SSL_R_TLSV1_ALERT_UNKNOWN_CA - case SSL_R_TLSV1_ALERT_UNKNOWN_CA: - SSL_INCREMENT_DYN_STAT(ssl_origin_server_unknown_ca_stat); - break; -#endif - default: - SSL_INCREMENT_DYN_STAT(ssl_origin_server_other_errors_stat); - return false; - } - - return true; -} - -void -SSLDiagnostic(const SourceLocation &loc, bool debug, SSLNetVConnection *vc, const char *fmt, ...) -{ - unsigned long l; - char buf[256]; - const char *file, *data; - int line, flags; - unsigned long es; - va_list ap; - ip_text_buffer ip_buf = {'\0'}; - - if (vc) { - ats_ip_ntop(vc->get_remote_addr(), ip_buf, sizeof(ip_buf)); - } - - es = (unsigned long)pthread_self(); - while ((l = ERR_get_error_line_data(&file, &line, &data, &flags)) != 0) { - if (debug) { - if (unlikely(diags->on())) { - diags->log("ssl-diag", DL_Debug, &loc, "SSL::%lu:%s:%s:%d%s%s%s%s", es, ERR_error_string(l, buf), file, line, - (flags & ERR_TXT_STRING) ? ":" : "", (flags & ERR_TXT_STRING) ? data : "", vc ? ": peer address is " : "", - ip_buf); - } - } else { - diags->error(DL_Error, &loc, "SSL::%lu:%s:%s:%d%s%s%s%s", es, ERR_error_string(l, buf), file, line, - (flags & ERR_TXT_STRING) ? ":" : "", (flags & ERR_TXT_STRING) ? data : "", vc ? ": peer address is " : "", - ip_buf); - } - - // Tally desired stats (only client/server connection stats, not init - // issues where vc is nullptr) - if (vc) { - // get_context() == NET_VCONNECTION_OUT if ats is client (we update server stats) - if (vc->get_context() == NET_VCONNECTION_OUT) { - increment_ssl_server_error(l); // update server error stats - } else { - increment_ssl_client_error(l); // update client error stat - } - } - } - - va_start(ap, fmt); - if (debug) { - diags->log_va("ssl-diag", DL_Debug, &loc, fmt, ap); - } else { - diags->error_va(DL_Error, &loc, fmt, ap); - } - va_end(ap); -} - -const char * -SSLErrorName(int ssl_error) -{ - static const char *names[] = { - "SSL_ERROR_NONE", "SSL_ERROR_SSL", "SSL_ERROR_WANT_READ", "SSL_ERROR_WANT_WRITE", "SSL_ERROR_WANT_X509_LOOKUP", - "SSL_ERROR_SYSCALL", "SSL_ERROR_ZERO_RETURN", "SSL_ERROR_WANT_CONNECT", "SSL_ERROR_WANT_ACCEPT"}; - - if (ssl_error < 0 || ssl_error >= (int)countof(names)) { - return "unknown SSL error"; - } - - return names[ssl_error]; -} - -void -SSLDebugBufferPrint(const char *tag, const char *buffer, unsigned buflen, const char *message) -{ - if (is_debug_tag_set(tag)) { - if (message != nullptr) { - fprintf(stdout, "%s\n", message); - } - for (unsigned ii = 0; ii < buflen; ii++) { - putc(buffer[ii], stdout); - } - putc('\n', stdout); - } -} - SSL_CTX * -SSLDefaultServerContext() +SSLMultiCertConfigLoader::default_server_ssl_ctx() { return SSL_CTX_new(SSLv23_server_method()); } @@ -1379,19 +954,22 @@ SSLPrivateKeyHandler(SSL_CTX *ctx, const SSLConfigParams *params, const std::str return true; } -static int -SSLCheckServerCertNow(X509 *cert, const char *certname) +/** + returns 0 on OK or negative value on failure and update log as appropriate. + + Will check: + - if file exists, and has read permissions + - for truncation or other PEM read fail + - current time is between notBefore and notAfter dates of certificate + if anything is not kosher, a negative value is returned and appropriate error logged. + + @static + */ +int +SSLMultiCertConfigLoader::check_server_cert_now(X509 *cert, const char *certname) { int timeCmpValue; - // SSLCheckServerCertNow() - returns 0 on OK or negative value on failure - // and update log as appropriate. - // Will check: - // - if file exists, and has read permissions - // - for truncation or other PEM read fail - // - current time is between notBefore and notAfter dates of certificate - // if anything is not kosher, a negative value is returned and appropriate error logged. - if (!cert) { // a truncated certificate would fall into here Error("invalid certificate %s: file is truncated or corrupted", certname); @@ -1402,7 +980,7 @@ SSLCheckServerCertNow(X509 *cert, const char *certname) timeCmpValue = X509_cmp_current_time(X509_get_notBefore(cert)); if (timeCmpValue == 0) { - // an error occured parsing the time, which we'll call a bogosity + // an error occurred parsing the time, which we'll call a bogosity Error("invalid certificate %s: unable to parse notBefore time", certname); return -3; } else if (timeCmpValue > 0) { @@ -1413,7 +991,7 @@ SSLCheckServerCertNow(X509 *cert, const char *certname) timeCmpValue = X509_cmp_current_time(X509_get_notAfter(cert)); if (timeCmpValue == 0) { - // an error occured parsing the time, which we'll call a bogosity + // an error occurred parsing the time, which we'll call a bogosity Error("invalid certificate %s: unable to parse notAfter time", certname); return -3; } else if (timeCmpValue < 0) { @@ -1437,11 +1015,14 @@ asn1_strdup(ASN1_STRING *s) return ats_strndup((const char *)ASN1_STRING_get0_data(s), ASN1_STRING_length(s)); } -// Given a certificate and it's corresponding SSL_CTX context, insert hash -// table aliases for subject CN and subjectAltNames DNS without wildcard, -// insert trie aliases for those with wildcard. -static bool -ssl_index_certificate(SSLCertLookup *lookup, SSLCertContext const &cc, X509 *cert, const char *certname) +/** + Given a certificate and it's corresponding SSL_CTX context, insert hash + table aliases for subject CN and subjectAltNames DNS without wildcard, + insert trie aliases for those with wildcard. + @static +*/ +bool +SSLMultiCertConfigLoader::index_certificate(SSLCertLookup *lookup, SSLCertContext const &cc, X509 *cert, const char *certname) { X509_NAME *subject = nullptr; bool inserted = false; @@ -1486,7 +1067,7 @@ ssl_index_certificate(SSLCertLookup *lookup, SSLCertContext const &cc, X509 *cer if (name->type == GEN_DNS) { ats_scoped_str dns(asn1_strdup(name->d.dNSName)); // only try to insert if the alternate name is not the main name - if (strcmp(dns, subj_name) != 0) { + if (subj_name == nullptr || strcmp(dns, subj_name) != 0) { Debug("ssl", "mapping '%s' to certificates %s", (const char *)dns, certname); if (lookup->insert(dns, cc) >= 0) { inserted = true; @@ -1540,26 +1121,32 @@ ssl_callback_info(const SSL *ssl, int where, int ret) if (cipher) { const char *cipherName = SSL_CIPHER_get_name(cipher); // lookup index of stat by name and incr count - auto data = cipher_map.get(cipherName); - if (data != 0) { - SSL_INCREMENT_DYN_STAT((intptr_t)data); + if (auto it = cipher_map.find(cipherName); it != cipher_map.end()) { + SSL_INCREMENT_DYN_STAT((intptr_t)it->second); } } } } -static void -ssl_set_handshake_callbacks(SSL_CTX *ctx) +void +SSLMultiCertConfigLoader::_set_handshake_callbacks(SSL_CTX *ctx) { -// Make sure the callbacks are set -#if TS_USE_CERT_CB + // Make sure the callbacks are set SSL_CTX_set_cert_cb(ctx, ssl_cert_callback, nullptr); - SSL_CTX_set_tlsext_servername_callback(ctx, ssl_servername_only_callback); -#else - SSL_CTX_set_tlsext_servername_callback(ctx, ssl_servername_and_cert_callback); + SSL_CTX_set_tlsext_servername_callback(ctx, ssl_servername_callback); + +#if TS_USE_HELLO_CB + SSL_CTX_set_client_hello_cb(ctx, ssl_client_hello_callback, nullptr); #endif } +void +setTLSValidProtocols(SSL *ssl, unsigned long proto_mask, unsigned long max_mask) +{ + SSL_set_options(ssl, proto_mask); + SSL_clear_options(ssl, max_mask & ~proto_mask); +} + void setClientCertLevel(SSL *ssl, uint8_t certLevel) { @@ -1570,6 +1157,10 @@ setClientCertLevel(SSL *ssl, uint8_t certLevel) server_verify_client = SSL_VERIFY_PEER | SSL_VERIFY_FAIL_IF_NO_PEER_CERT | SSL_VERIFY_CLIENT_ONCE; } else if (certLevel == 1) { server_verify_client = SSL_VERIFY_PEER | SSL_VERIFY_CLIENT_ONCE; + } else if (certLevel == 0) { + server_verify_client = SSL_VERIFY_NONE; + } else { + ink_release_assert(!"Invalid client verify level"); } Debug("ssl", "setting cert level to %d", server_verify_client); @@ -1577,16 +1168,17 @@ setClientCertLevel(SSL *ssl, uint8_t certLevel) SSL_set_verify_depth(ssl, params->verify_depth); // might want to make configurable at some point. } +/** + Initialize SSL_CTX for server + This is public function because of used by SSLCreateServerContext. + */ SSL_CTX * -SSLInitServerContext(const SSLConfigParams *params, const ssl_user_config *sslMultCertSettings, std::vector &certList) +SSLMultiCertConfigLoader::init_server_ssl_ctx(std::vector &cert_list, const SSLMultiCertConfigParams *sslMultCertSettings) { + const SSLConfigParams *params = this->_params; + int server_verify_client; - SSL_CTX *ctx = SSLDefaultServerContext(); - EVP_MD_CTX *digest = EVP_MD_CTX_new(); - STACK_OF(X509_NAME) *ca_list = nullptr; - unsigned char hash_buf[EVP_MAX_MD_SIZE]; - unsigned int hash_len = 0; - const char *setting_cert = sslMultCertSettings ? sslMultCertSettings->cert.get() : nullptr; + SSL_CTX *ctx = this->default_server_ssl_ctx(); // disable selected protocols SSL_CTX_set_options(ctx, params->ssl_ctx_options); @@ -1630,10 +1222,8 @@ SSLInitServerContext(const SSLConfigParams *params, const ssl_user_config *sslMu } #ifdef SSL_MODE_RELEASE_BUFFERS - if (OPENSSL_VERSION_NUMBER > 0x1000107fL) { - Debug("ssl", "enabling SSL_MODE_RELEASE_BUFFERS"); - SSL_CTX_set_mode(ctx, SSL_MODE_RELEASE_BUFFERS); - } + Debug("ssl", "enabling SSL_MODE_RELEASE_BUFFERS"); + SSL_CTX_set_mode(ctx, SSL_MODE_RELEASE_BUFFERS); #endif #ifdef SSL_OP_SAFARI_ECDHE_ECDSA_BUG @@ -1657,7 +1247,7 @@ SSLInitServerContext(const SSLConfigParams *params, const ssl_user_config *sslMu } else if (strcmp(sslMultCertSettings->dialog, "builtin") == 0) { passwd_cb = ssl_private_key_passphrase_callback_builtin; } else { // unknown config - SSLError("unknown " SSL_KEY_DIALOG " configuration value '%s'", (const char *)sslMultCertSettings->dialog); + SSLError("unknown %s configuration value '%s'", SSL_KEY_DIALOG.data(), (const char *)sslMultCertSettings->dialog); memset(static_cast(&ud), 0, sizeof(ud)); goto fail; } @@ -1668,83 +1258,10 @@ SSLInitServerContext(const SSLConfigParams *params, const ssl_user_config *sslMu } if (sslMultCertSettings->cert) { - SimpleTokenizer cert_tok((const char *)sslMultCertSettings->cert, SSL_CERT_SEPARATE_DELIM); - SimpleTokenizer key_tok((sslMultCertSettings->key ? (const char *)sslMultCertSettings->key : ""), SSL_CERT_SEPARATE_DELIM); - - if (sslMultCertSettings->key && cert_tok.getNumTokensRemaining() != key_tok.getNumTokensRemaining()) { - Error("the number of certificates in ssl_cert_name and ssl_key_name doesn't match"); + if (!SSLMultiCertConfigLoader::load_certs(ctx, cert_list, params, sslMultCertSettings)) { goto fail; } - SimpleTokenizer ca_tok("", SSL_CERT_SEPARATE_DELIM); - if (sslMultCertSettings->ca) { - ca_tok.setString(sslMultCertSettings->ca); - if (cert_tok.getNumTokensRemaining() != ca_tok.getNumTokensRemaining()) { - Error("the number of certificates in ssl_cert_name and ssl_ca_name doesn't match"); - goto fail; - } - } - - for (const char *certname = cert_tok.getNext(); certname; certname = cert_tok.getNext()) { - std::string completeServerCertPath = Layout::relative_to(params->serverCertPathOnly, certname); - scoped_BIO bio(BIO_new_file(completeServerCertPath.c_str(), "r")); - X509 *cert = nullptr; - if (bio) { - cert = PEM_read_bio_X509(bio.get(), nullptr, nullptr, nullptr); - } - if (!bio || !cert) { - SSLError("failed to load certificate chain from %s", completeServerCertPath.c_str()); - goto fail; - } - if (!SSL_CTX_use_certificate(ctx, cert)) { - SSLError("Failed to assign cert from %s to SSL_CTX", completeServerCertPath.c_str()); - X509_free(cert); - goto fail; - } - certList.push_back(cert); - if (SSLConfigParams::load_ssl_file_cb) { - SSLConfigParams::load_ssl_file_cb(completeServerCertPath.c_str(), CONFIG_FLAG_UNVERSIONED); - } - // Load up any additional chain certificates - SSL_CTX_add_extra_chain_cert_bio(ctx, bio); - - const char *keyPath = key_tok.getNext(); - if (!SSLPrivateKeyHandler(ctx, params, completeServerCertPath, keyPath)) { - goto fail; - } - - // Must load all the intermediate certificates before starting the next chain - - // First, load any CA chains from the global chain file. This should probably - // eventually be a comma separated list too. For now we will load it in all chains even - // though it only makes sense in one chain - if (params->serverCertChainFilename) { - ats_scoped_str completeServerCertChainPath( - Layout::relative_to(params->serverCertPathOnly, params->serverCertChainFilename)); - if (!SSL_CTX_add_extra_chain_cert_file(ctx, completeServerCertChainPath)) { - SSLError("failed to load global certificate chain from %s", (const char *)completeServerCertChainPath); - goto fail; - } - if (SSLConfigParams::load_ssl_file_cb) { - SSLConfigParams::load_ssl_file_cb(completeServerCertChainPath, CONFIG_FLAG_UNVERSIONED); - } - } - - // Now, load any additional certificate chains specified in this entry. - if (sslMultCertSettings->ca) { - const char *ca_name = ca_tok.getNext(); - if (ca_name != nullptr) { - ats_scoped_str completeServerCertChainPath(Layout::relative_to(params->serverCertPathOnly, ca_name)); - if (!SSL_CTX_add_extra_chain_cert_file(ctx, completeServerCertChainPath)) { - SSLError("failed to load certificate chain from %s", (const char *)completeServerCertChainPath); - goto fail; - } - if (SSLConfigParams::load_ssl_file_cb) { - SSLConfigParams::load_ssl_file_cb(completeServerCertChainPath, CONFIG_FLAG_UNVERSIONED); - } - } - } - } - } + } // SSL_CTX_load_verify_locations() builds the cert chain from the // serverCACertFilename if that is not nullptr. Otherwise, it uses the hashed @@ -1765,7 +1282,16 @@ SSLInitServerContext(const SSLConfigParams *params, const ssl_user_config *sslMu goto fail; } } + +#if defined(SSL_OP_NO_TICKET) + // Session tickets are enabled by default. Disable if explicitly requested. + if (sslMultCertSettings->session_ticket_enabled == 0) { + SSL_CTX_set_options(ctx, SSL_OP_NO_TICKET); + Debug("ssl", "ssl session ticket is disabled"); + } +#endif } + if (params->clientCertLevel != 0) { if (params->serverCACertFilename != nullptr && params->serverCACertPath != nullptr) { if ((!SSL_CTX_load_verify_locations(ctx, params->serverCACertFilename, params->serverCACertPath)) || @@ -1788,50 +1314,7 @@ SSLInitServerContext(const SSLConfigParams *params, const ssl_user_config *sslMu SSL_CTX_set_verify_depth(ctx, params->verify_depth); // might want to make configurable at some point. } - // Set the list of CA's to send to client if we ask for a client - // certificate - if (params->serverCACertFilename) { - ca_list = SSL_load_client_CA_file(params->serverCACertFilename); - if (ca_list) { - SSL_CTX_set_client_CA_list(ctx, ca_list); - } - } - - if (EVP_DigestInit_ex(digest, evp_md_func, nullptr) == 0) { - SSLError("EVP_DigestInit_ex failed"); - goto fail; - } - - if (nullptr != setting_cert) { - Debug("ssl", "Using '%s' in hash for session id context", sslMultCertSettings->cert.get()); - if (EVP_DigestUpdate(digest, sslMultCertSettings->cert, strlen(setting_cert)) == 0) { - SSLError("EVP_DigestUpdate failed"); - goto fail; - } - } - - if (ca_list != nullptr) { - size_t num_certs = sk_X509_NAME_num(ca_list); - - for (size_t i = 0; i < num_certs; i++) { - X509_NAME *name = sk_X509_NAME_value(ca_list, i); - if (X509_NAME_digest(name, evp_md_func, hash_buf /* borrow our final hash buffer. */, &hash_len) == 0 || - EVP_DigestUpdate(digest, hash_buf, hash_len) == 0) { - SSLError("Adding X509 name to digest failed"); - goto fail; - } - } - } - - if (EVP_DigestFinal_ex(digest, hash_buf, &hash_len) == 0) { - SSLError("EVP_DigestFinal_ex failed"); - goto fail; - } - EVP_MD_CTX_free(digest); - digest = nullptr; - - if (SSL_CTX_set_session_id_context(ctx, hash_buf, hash_len) == 0) { - SSLError("SSL_CTX_set_session_id_context failed"); + if (!SSLMultiCertConfigLoader::set_session_id_context(ctx, params, sslMultCertSettings)) { goto fail; } @@ -1851,9 +1334,13 @@ SSLInitServerContext(const SSLConfigParams *params, const ssl_user_config *sslMu } #endif -#ifdef SSL_CTX_set1_groups_list +#if defined(SSL_CTX_set1_groups_list) || defined(SSL_CTX_set1_curves_list) if (params->server_groups_list != nullptr) { +#ifdef SSL_CTX_set1_groups_list if (!SSL_CTX_set1_groups_list(ctx, params->server_groups_list)) { +#else + if (!SSL_CTX_set1_curves_list(ctx, params->server_groups_list)) { +#endif SSLError("invalid groups list for server in records.config"); goto fail; } @@ -1865,49 +1352,25 @@ SSLInitServerContext(const SSLConfigParams *params, const ssl_user_config *sslMu } ssl_context_enable_ecdh(ctx); -#define SSL_CLEAR_PW_REFERENCES(CTX) \ - { \ - SSL_CTX_set_default_passwd_cb(CTX, nullptr); \ - SSL_CTX_set_default_passwd_cb_userdata(CTX, nullptr); \ - } + if (sslMultCertSettings && sslMultCertSettings->dialog) { - SSL_CLEAR_PW_REFERENCES(ctx); + SSLMultiCertConfigLoader::clear_pw_references(ctx); } SSL_CTX_set_info_callback(ctx, ssl_callback_info); -#if TS_USE_TLS_NPN SSL_CTX_set_next_protos_advertised_cb(ctx, SSLNetVConnection::advertise_next_protocol, nullptr); -#endif /* TS_USE_TLS_NPN */ - -#if TS_USE_TLS_ALPN SSL_CTX_set_alpn_select_cb(ctx, SSLNetVConnection::select_next_protocol, nullptr); -#endif /* TS_USE_TLS_ALPN */ - -#ifdef TS_USE_TLS_OCSP - if (SSLConfigParams::ssl_ocsp_enabled) { - Debug("ssl", "SSL OCSP Stapling is enabled"); - SSL_CTX_set_tlsext_status_cb(ctx, ssl_callback_ocsp_stapling); - } else { - Debug("ssl", "SSL OCSP Stapling is disabled"); - } -#else - if (SSLConfigParams::ssl_ocsp_enabled) { - Warning("failed to enable SSL OCSP Stapling; this version of OpenSSL does not support it"); - } -#endif /* TS_USE_TLS_OCSP */ if (SSLConfigParams::init_ssl_ctx_cb) { SSLConfigParams::init_ssl_ctx_cb(ctx, true); } + return ctx; fail: - if (digest) { - EVP_MD_CTX_free(digest); - } - SSL_CLEAR_PW_REFERENCES(ctx) + SSLMultiCertConfigLoader::clear_pw_references(ctx); SSLReleaseContext(ctx); - for (auto cert : certList) { + for (auto cert : cert_list) { X509_free(cert); } @@ -1917,17 +1380,24 @@ SSLInitServerContext(const SSLConfigParams *params, const ssl_user_config *sslMu SSL_CTX * SSLCreateServerContext(const SSLConfigParams *params) { + SSLMultiCertConfigLoader loader(params); std::vector cert_list; - SSL_CTX *ctx = SSLInitServerContext(params, nullptr, cert_list); + + SSL_CTX *ctx = loader.init_server_ssl_ctx(cert_list, nullptr); ink_assert(cert_list.empty()); + return ctx; } -static SSL_CTX * -ssl_store_ssl_context(const SSLConfigParams *params, SSLCertLookup *lookup, const ssl_user_config *sslMultCertSettings) +/** + Insert SSLCertContext (SSL_CTX ans options) into SSLCertLookup with key. + Do NOT call SSL_CTX_set_* functions from here. SSL_CTX should be set up by SSLMultiCertConfigLoader::init_server_ssl_ctx(). + */ +SSL_CTX * +SSLMultiCertConfigLoader::_store_ssl_ctx(SSLCertLookup *lookup, const SSLMultiCertConfigParams *sslMultCertSettings) { std::vector cert_list; - SSL_CTX *ctx = SSLInitServerContext(params, sslMultCertSettings, cert_list); + SSL_CTX *ctx = this->init_server_ssl_ctx(cert_list, sslMultCertSettings); ssl_ticket_key_block *keyblock = nullptr; bool inserted = false; @@ -1938,7 +1408,7 @@ ssl_store_ssl_context(const SSLConfigParams *params, SSLCertLookup *lookup, cons const char *certname = sslMultCertSettings->cert.get(); for (auto cert : cert_list) { - if (0 > SSLCheckServerCertNow(cert, certname)) { + if (0 > SSLMultiCertConfigLoader::check_server_cert_now(cert, certname)) { /* At this point, we know cert is bad, and we've already printed a descriptive reason as to why cert is bad to the log file */ Debug("ssl", "Marking certificate as NOT VALID: %s", certname); @@ -1957,7 +1427,7 @@ ssl_store_ssl_context(const SSLConfigParams *params, SSLCertLookup *lookup, cons if (lookup->insert(sslMultCertSettings->addr, SSLCertContext(ctx, sslMultCertSettings->opt, keyblock)) >= 0) { inserted = true; lookup->ssl_default = ctx; - ssl_set_handshake_callbacks(ctx); + this->_set_handshake_callbacks(ctx); } } else { IpEndpoint ep; @@ -1974,45 +1444,19 @@ ssl_store_ssl_context(const SSLConfigParams *params, SSLCertLookup *lookup, cons } } if (!inserted) { -#if HAVE_OPENSSL_SESSION_TICKETS +#if TS_HAVE_OPENSSL_SESSION_TICKETS if (keyblock != nullptr) { ticket_block_free(keyblock); } #endif } -#if defined(SSL_OP_NO_TICKET) - // Session tickets are enabled by default. Disable if explicitly requested. - if (sslMultCertSettings->session_ticket_enabled == 0) { - SSL_CTX_set_options(ctx, SSL_OP_NO_TICKET); - Debug("ssl", "ssl session ticket is disabled"); - } -#endif - -#ifdef TS_USE_TLS_OCSP - if (SSLConfigParams::ssl_ocsp_enabled) { - Debug("ssl", "SSL OCSP Stapling is enabled"); - SSL_CTX_set_tlsext_status_cb(ctx, ssl_callback_ocsp_stapling); - for (auto cert : cert_list) { - if (!ssl_stapling_init_cert(ctx, cert, certname)) { - Warning("failed to configure SSL_CTX for OCSP Stapling info for certificate at %s", (const char *)certname); - } - } - } else { - Debug("ssl", "SSL OCSP Stapling is disabled"); - } -#else - if (SSLConfigParams::ssl_ocsp_enabled) { - Warning("failed to enable SSL OCSP Stapling; this version of OpenSSL does not support it"); - } -#endif /* TS_USE_TLS_OCSP */ - // Insert additional mappings. Note that this maps multiple keys to the same value, so when // this code is updated to reconfigure the SSL certificates, it will need some sort of // refcounting or alternate way of avoiding double frees. Debug("ssl", "importing SNI names from %s", (const char *)certname); for (auto cert : cert_list) { - if (ssl_index_certificate(lookup, SSLCertContext(ctx, sslMultCertSettings->opt), cert, certname)) { + if (SSLMultiCertConfigLoader::index_certificate(lookup, SSLCertContext(ctx, sslMultCertSettings->opt), cert, certname)) { inserted = true; } } @@ -2036,7 +1480,7 @@ ssl_store_ssl_context(const SSLConfigParams *params, SSLCertLookup *lookup, cons } static bool -ssl_extract_certificate(const matcher_line *line_info, ssl_user_config &sslMultCertSettings) +ssl_extract_certificate(const matcher_line *line_info, SSLMultiCertConfigParams &sslMultCertSettings) { for (int i = 0; i < MATCHER_MAX_TOKENS; ++i) { const char *label; @@ -2065,6 +1509,10 @@ ssl_extract_certificate(const matcher_line *line_info, ssl_user_config &sslMultC sslMultCertSettings.key = ats_strdup(value); } + if (strcasecmp(label, SSL_OCSP_RESPONSE_TAG) == 0) { + sslMultCertSettings.ocsp_response = ats_strdup(value); + } + if (strcasecmp(label, SSL_SESSION_TICKET_ENABLED) == 0) { sslMultCertSettings.session_ticket_enabled = atoi(value); } @@ -2081,7 +1529,7 @@ ssl_extract_certificate(const matcher_line *line_info, ssl_user_config &sslMultC if (strcasecmp(SSL_ACTION_TUNNEL_TAG, value) == 0) { sslMultCertSettings.opt = SSLCertContext::OPT_TUNNEL; } else { - Error("Unrecognized action for " SSL_ACTION_TAG); + Error("Unrecognized action for %s", SSL_ACTION_TAG.data()); return false; } } @@ -2099,8 +1547,10 @@ ssl_extract_certificate(const matcher_line *line_info, ssl_user_config &sslMultC } bool -SSLParseCertificateConfiguration(const SSLConfigParams *params, SSLCertLookup *lookup) +SSLMultiCertConfigLoader::load(SSLCertLookup *lookup) { + const SSLConfigParams *params = this->_params; + char *tok_state = nullptr; char *line = nullptr; ats_scoped_str file_buf; @@ -2109,7 +1559,7 @@ SSLParseCertificateConfiguration(const SSLConfigParams *params, SSLCertLookup *l const matcher_tags sslCertTags = {nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, false}; - Note("loading SSL certificate configuration from %s", params->configFilePath); + Note("ssl_multicert.config loading ..."); if (params->configFilePath) { file_buf = readIntoBuffer(params->configFilePath, __func__, nullptr); @@ -2136,7 +1586,7 @@ SSLParseCertificateConfiguration(const SSLConfigParams *params, SSLCertLookup *l } if (*line != '\0' && *line != '#') { - ssl_user_config sslMultiCertSettings; + SSLMultiCertConfigParams sslMultiCertSettings; const char *errPtr; errPtr = parseConfigLine(line, &line_info, &sslCertTags); @@ -2148,7 +1598,7 @@ SSLParseCertificateConfiguration(const SSLConfigParams *params, SSLCertLookup *l if (ssl_extract_certificate(&line_info, sslMultiCertSettings)) { // There must be a certificate specified unless the tunnel action is set if (sslMultiCertSettings.cert || sslMultiCertSettings.opt != SSLCertContext::OPT_TUNNEL) { - ssl_store_ssl_context(params, lookup, &sslMultiCertSettings); + this->_store_ssl_ctx(lookup, &sslMultiCertSettings); } else { Warning("No ssl_cert_name specified and no tunnel action set"); } @@ -2163,9 +1613,9 @@ SSLParseCertificateConfiguration(const SSLConfigParams *params, SSLCertLookup *l // bootstrap the SSL handshake so that we can subsequently do the SNI lookup to switch to the real // context. if (lookup->ssl_default == nullptr) { - ssl_user_config sslMultiCertSettings; + SSLMultiCertConfigParams sslMultiCertSettings; sslMultiCertSettings.addr = ats_strdup("*"); - if (ssl_store_ssl_context(params, lookup, &sslMultiCertSettings) == nullptr) { + if (this->_store_ssl_ctx(lookup, &sslMultiCertSettings) == nullptr) { Error("failed set default context"); return false; } @@ -2174,87 +1624,11 @@ SSLParseCertificateConfiguration(const SSLConfigParams *params, SSLCertLookup *l return true; } -#if HAVE_OPENSSL_SESSION_TICKETS - -static void -session_ticket_free(void * /*parent*/, void *ptr, CRYPTO_EX_DATA * /*ad*/, int /*idx*/, long /*argl*/, void * /*argp*/) -{ - ticket_block_free((struct ssl_ticket_key_block *)ptr); -} - -/* - * RFC 5077. Create session ticket to resume SSL session without requiring session-specific state at the TLS server. - * Specifically, it distributes the encrypted session-state information to the client in the form of a ticket and - * a mechanism to present the ticket back to the server. - * */ -static int -ssl_callback_session_ticket(SSL *ssl, unsigned char *keyname, unsigned char *iv, EVP_CIPHER_CTX *cipher_ctx, HMAC_CTX *hctx, - int enc) -{ - SSLCertificateConfig::scoped_config lookup; - SSLTicketKeyConfig::scoped_config params; - SSLNetVConnection *netvc = SSLNetVCAccess(ssl); - - // Get the IP address to look up the keyblock - IpEndpoint ip; - int namelen = sizeof(ip); - SSLCertContext *cc = nullptr; - if (0 == safe_getsockname(netvc->get_socket(), &ip.sa, &namelen)) { - cc = lookup->find(ip); - } - ssl_ticket_key_block *keyblock = nullptr; - if (cc == nullptr || cc->keyblock == nullptr) { - // Try the default - keyblock = params->default_global_keyblock; - } else { - keyblock = cc->keyblock; - } - ink_release_assert(keyblock != nullptr && keyblock->num_keys > 0); - - if (enc == 1) { - const ssl_ticket_key_t &most_recent_key = keyblock->keys[0]; - memcpy(keyname, most_recent_key.key_name, sizeof(most_recent_key.key_name)); - RAND_bytes(iv, EVP_MAX_IV_LENGTH); - EVP_EncryptInit_ex(cipher_ctx, EVP_aes_128_cbc(), nullptr, most_recent_key.aes_key, iv); - HMAC_Init_ex(hctx, most_recent_key.hmac_secret, sizeof(most_recent_key.hmac_secret), evp_md_func, nullptr); - - Debug("ssl", "create ticket for a new session."); - SSL_INCREMENT_DYN_STAT(ssl_total_tickets_created_stat); - return 1; - } else if (enc == 0) { - for (unsigned i = 0; i < keyblock->num_keys; ++i) { - if (memcmp(keyname, keyblock->keys[i].key_name, sizeof(keyblock->keys[i].key_name)) == 0) { - EVP_DecryptInit_ex(cipher_ctx, EVP_aes_128_cbc(), nullptr, keyblock->keys[i].aes_key, iv); - HMAC_Init_ex(hctx, keyblock->keys[i].hmac_secret, sizeof(keyblock->keys[i].hmac_secret), evp_md_func, nullptr); - - Debug("ssl", "verify the ticket for an existing session."); - // Increase the total number of decrypted tickets. - SSL_INCREMENT_DYN_STAT(ssl_total_tickets_verified_stat); - - if (i != 0) { // The number of tickets decrypted with "older" keys. - SSL_INCREMENT_DYN_STAT(ssl_total_tickets_verified_old_key_stat); - } - - SSLNetVConnection *netvc = SSLNetVCAccess(ssl); - netvc->setSSLSessionCacheHit(true); - // When we decrypt with an "older" key, encrypt the ticket again with the most recent key. - return (i == 0) ? 1 : 2; - } - } - - Debug("ssl", "keyname is not consistent."); - SSL_INCREMENT_DYN_STAT(ssl_total_tickets_not_found_stat); - return 0; - } - - return -1; -} -#endif /* HAVE_OPENSSL_SESSION_TICKETS */ - +// Release SSL_CTX and the associated data. This works for both +// client and server contexts and gracefully accepts nullptr. void SSLReleaseContext(SSL_CTX *ctx) { - // SSL_CTX_free() does nothing if ctx in nullptr, so there's no need to check. SSL_CTX_free(ctx); } @@ -2371,3 +1745,206 @@ SSLConnect(SSL *ssl) return ssl_error; } + +/** + Load certificates to SSL_CTX + @static + */ +bool +SSLMultiCertConfigLoader::load_certs(SSL_CTX *ctx, std::vector &cert_list, const SSLConfigParams *params, + const SSLMultiCertConfigParams *sslMultCertSettings) +{ + SimpleTokenizer cert_tok((const char *)sslMultCertSettings->cert, SSL_CERT_SEPARATE_DELIM); + SimpleTokenizer key_tok((sslMultCertSettings->key ? (const char *)sslMultCertSettings->key : ""), SSL_CERT_SEPARATE_DELIM); + + if (sslMultCertSettings->key && cert_tok.getNumTokensRemaining() != key_tok.getNumTokensRemaining()) { + Error("the number of certificates in ssl_cert_name and ssl_key_name doesn't match"); + return false; + } + SimpleTokenizer ca_tok("", SSL_CERT_SEPARATE_DELIM); + if (sslMultCertSettings->ca) { + ca_tok.setString(sslMultCertSettings->ca); + if (cert_tok.getNumTokensRemaining() != ca_tok.getNumTokensRemaining()) { + Error("the number of certificates in ssl_cert_name and ssl_ca_name doesn't match"); + return false; + } + } + +#if TS_USE_TLS_OCSP + if (SSLConfigParams::ssl_ocsp_enabled) { + Debug("ssl", "SSL OCSP Stapling is enabled"); + SSL_CTX_set_tlsext_status_cb(ctx, ssl_callback_ocsp_stapling); + } else { + Debug("ssl", "SSL OCSP Stapling is disabled"); + } + SimpleTokenizer ocsp_tok("", SSL_CERT_SEPARATE_DELIM); + if (sslMultCertSettings->ocsp_response) { + ocsp_tok.setString(sslMultCertSettings->ocsp_response); + if (cert_tok.getNumTokensRemaining() != ocsp_tok.getNumTokensRemaining()) { + Error("the number of certificates in ssl_cert_name and ssl_ocsp_name doesn't match"); + return false; + } + } +#else + if (SSLConfigParams::ssl_ocsp_enabled) { + Warning("failed to enable SSL OCSP Stapling; this version of OpenSSL does not support it"); + } +#endif /* TS_USE_TLS_OCSP */ + + for (const char *certname = cert_tok.getNext(); certname; certname = cert_tok.getNext()) { + std::string completeServerCertPath = Layout::relative_to(params->serverCertPathOnly, certname); + scoped_BIO bio(BIO_new_file(completeServerCertPath.c_str(), "r")); + X509 *cert = nullptr; + if (bio) { + cert = PEM_read_bio_X509(bio.get(), nullptr, nullptr, nullptr); + } + if (!bio || !cert) { + SSLError("failed to load certificate chain from %s", completeServerCertPath.c_str()); + return false; + } + if (!SSL_CTX_use_certificate(ctx, cert)) { + SSLError("Failed to assign cert from %s to SSL_CTX", completeServerCertPath.c_str()); + X509_free(cert); + return false; + } + + // Load up any additional chain certificates + SSL_CTX_add_extra_chain_cert_bio(ctx, bio); + + const char *keyPath = key_tok.getNext(); + if (!SSLPrivateKeyHandler(ctx, params, completeServerCertPath, keyPath)) { + return false; + } + + cert_list.push_back(cert); + if (SSLConfigParams::load_ssl_file_cb) { + SSLConfigParams::load_ssl_file_cb(completeServerCertPath.c_str(), CONFIG_FLAG_UNVERSIONED); + } + + // Must load all the intermediate certificates before starting the next chain + + // First, load any CA chains from the global chain file. This should probably + // eventually be a comma separated list too. For now we will load it in all chains even + // though it only makes sense in one chain + if (params->serverCertChainFilename) { + ats_scoped_str completeServerCertChainPath(Layout::relative_to(params->serverCertPathOnly, params->serverCertChainFilename)); + if (!SSL_CTX_add_extra_chain_cert_file(ctx, completeServerCertChainPath)) { + SSLError("failed to load global certificate chain from %s", (const char *)completeServerCertChainPath); + return false; + } + if (SSLConfigParams::load_ssl_file_cb) { + SSLConfigParams::load_ssl_file_cb(completeServerCertChainPath, CONFIG_FLAG_UNVERSIONED); + } + } + + // Now, load any additional certificate chains specified in this entry. + if (sslMultCertSettings->ca) { + const char *ca_name = ca_tok.getNext(); + if (ca_name != nullptr) { + ats_scoped_str completeServerCertChainPath(Layout::relative_to(params->serverCertPathOnly, ca_name)); + if (!SSL_CTX_add_extra_chain_cert_file(ctx, completeServerCertChainPath)) { + SSLError("failed to load certificate chain from %s", (const char *)completeServerCertChainPath); + return false; + } + if (SSLConfigParams::load_ssl_file_cb) { + SSLConfigParams::load_ssl_file_cb(completeServerCertChainPath, CONFIG_FLAG_UNVERSIONED); + } + } + } +#if TS_USE_TLS_OCSP + if (SSLConfigParams::ssl_ocsp_enabled) { + if (sslMultCertSettings->ocsp_response) { + const char *ocsp_response_name = ocsp_tok.getNext(); + ats_scoped_str completeOCSPResponsePath(Layout::relative_to(params->ssl_ocsp_response_path_only, ocsp_response_name)); + if (!ssl_stapling_init_cert(ctx, cert, certname, (const char *)completeOCSPResponsePath)) { + Warning("failed to configure SSL_CTX for OCSP Stapling info for certificate at %s", certname); + } + } else { + if (!ssl_stapling_init_cert(ctx, cert, certname, nullptr)) { + Warning("failed to configure SSL_CTX for OCSP Stapling info for certificate at %s", certname); + } + } + } +#endif /* TS_USE_TLS_OCSP */ + } + + return true; +} + +/** + Set session_id context for session reuse + @static + */ +bool +SSLMultiCertConfigLoader::set_session_id_context(SSL_CTX *ctx, const SSLConfigParams *params, + const SSLMultiCertConfigParams *sslMultCertSettings) +{ + EVP_MD_CTX *digest = EVP_MD_CTX_new(); + STACK_OF(X509_NAME) *ca_list = nullptr; + unsigned char hash_buf[EVP_MAX_MD_SIZE]; + unsigned int hash_len = 0; + const char *setting_cert = sslMultCertSettings ? sslMultCertSettings->cert.get() : nullptr; + bool result = false; + + // Set the list of CA's to send to client if we ask for a client certificate + if (params->serverCACertFilename) { + ca_list = SSL_load_client_CA_file(params->serverCACertFilename); + if (ca_list) { + SSL_CTX_set_client_CA_list(ctx, ca_list); + } + } + + if (EVP_DigestInit_ex(digest, evp_md_func, nullptr) == 0) { + SSLError("EVP_DigestInit_ex failed"); + goto fail; + } + + if (nullptr != setting_cert) { + Debug("ssl", "Using '%s' in hash for session id context", sslMultCertSettings->cert.get()); + if (EVP_DigestUpdate(digest, sslMultCertSettings->cert, strlen(setting_cert)) == 0) { + SSLError("EVP_DigestUpdate failed"); + goto fail; + } + } + + if (ca_list != nullptr) { + size_t num_certs = sk_X509_NAME_num(ca_list); + + for (size_t i = 0; i < num_certs; i++) { + X509_NAME *name = sk_X509_NAME_value(ca_list, i); + if (X509_NAME_digest(name, evp_md_func, hash_buf /* borrow our final hash buffer. */, &hash_len) == 0 || + EVP_DigestUpdate(digest, hash_buf, hash_len) == 0) { + SSLError("Adding X509 name to digest failed"); + goto fail; + } + } + } + + if (EVP_DigestFinal_ex(digest, hash_buf, &hash_len) == 0) { + SSLError("EVP_DigestFinal_ex failed"); + goto fail; + } + + if (SSL_CTX_set_session_id_context(ctx, hash_buf, hash_len) == 0) { + SSLError("SSL_CTX_set_session_id_context failed"); + goto fail; + } + + result = true; + +fail: + EVP_MD_CTX_free(digest); + + return result; +} + +/** + Clear password in SSL_CTX + @static + */ +void +SSLMultiCertConfigLoader::clear_pw_references(SSL_CTX *ssl_ctx) +{ + SSL_CTX_set_default_passwd_cb(ssl_ctx, nullptr); + SSL_CTX_set_default_passwd_cb_userdata(ssl_ctx, nullptr); +} diff --git a/iocore/net/Socks.cc b/iocore/net/Socks.cc index e1915f2dc95..ab1044f5bf9 100644 --- a/iocore/net/Socks.cc +++ b/iocore/net/Socks.cc @@ -480,7 +480,7 @@ loadSocksConfiguration(socks_conf_struct *socks_conf_stuff) socks_conf_stuff->server_connect_timeout = REC_ConfigReadInteger("proxy.config.socks.server_connect_timeout"); socks_conf_stuff->socks_timeout = REC_ConfigReadInteger("proxy.config.socks.socks_timeout"); - Debug("Socks", "server connect timeout: %d socks respnonse timeout %d", socks_conf_stuff->server_connect_timeout, + Debug("Socks", "server connect timeout: %d socks response timeout %d", socks_conf_stuff->server_connect_timeout, socks_conf_stuff->socks_timeout); socks_conf_stuff->per_server_connection_attempts = REC_ConfigReadInteger("proxy.config.socks.per_server_connection_attempts"); @@ -636,7 +636,7 @@ socks5BasicAuthHandler(int event, unsigned char *p, void (**h_ptr)(void)) break; case 0xff: - Debug("Socks", "None of the Socks authentcations is acceptable " + Debug("Socks", "None of the Socks authentications is acceptable " "to the server"); *h_ptr = nullptr; ret = -1; @@ -655,7 +655,7 @@ socks5BasicAuthHandler(int event, unsigned char *p, void (**h_ptr)(void)) break; default: - // This should be inpossible + // This should be impossible ink_assert(!"bad case value"); ret = -1; break; @@ -694,7 +694,7 @@ socks5PasswdAuthHandler(int event, unsigned char *p, void (**h_ptr)(void)) // NEC thinks it is 5 RFC seems to indicate 1. switch (p[1]) { case 0: - Debug("Socks", "Username/Passwd succeded"); + Debug("Socks", "Username/Passwd succeeded"); *h_ptr = nullptr; break; diff --git a/iocore/net/UnixNet.cc b/iocore/net/UnixNet.cc index eb95841f934..63721b96154 100644 --- a/iocore/net/UnixNet.cc +++ b/iocore/net/UnixNet.cc @@ -152,7 +152,7 @@ PollCont::do_poll(ink_hrtime timeout) poll_timeout = net_config_poll_timeout; } } -// wait for fd's to tigger, or don't wait if timeout is 0 +// wait for fd's to trigger, or don't wait if timeout is 0 #if TS_USE_EPOLL pollDescriptor->result = epoll_wait(pollDescriptor->epoll_fd, pollDescriptor->ePoll_Triggered_Events, POLL_DESCRIPTOR_SIZE, poll_timeout); @@ -673,6 +673,7 @@ void NetHandler::add_to_keep_alive_queue(UnixNetVConnection *vc) { Debug("net_queue", "NetVC: %p", vc); + ink_assert(mutex->thread_holding == this_ethread()); if (keep_alive_queue.in(vc)) { // already in the keep-alive queue, move the head @@ -692,6 +693,8 @@ void NetHandler::remove_from_keep_alive_queue(UnixNetVConnection *vc) { Debug("net_queue", "NetVC: %p", vc); + ink_assert(mutex->thread_holding == this_ethread()); + if (keep_alive_queue.in(vc)) { keep_alive_queue.remove(vc); --keep_alive_queue_size; @@ -704,6 +707,7 @@ NetHandler::add_to_active_queue(UnixNetVConnection *vc) Debug("net_queue", "NetVC: %p", vc); Debug("net_queue", "max_connections_per_thread_in: %d active_queue_size: %d keep_alive_queue_size: %d", max_connections_per_thread_in, active_queue_size, keep_alive_queue_size); + ink_assert(mutex->thread_holding == this_ethread()); // if active queue is over size then close inactive connections if (manage_active_queue() == false) { @@ -728,6 +732,8 @@ void NetHandler::remove_from_active_queue(UnixNetVConnection *vc) { Debug("net_queue", "NetVC: %p", vc); + ink_assert(mutex->thread_holding == this_ethread()); + if (active_queue.in(vc)) { active_queue.remove(vc); --active_queue_size; diff --git a/iocore/net/UnixNetAccept.cc b/iocore/net/UnixNetAccept.cc index 21f39c4ac85..a936c5ef8c0 100644 --- a/iocore/net/UnixNetAccept.cc +++ b/iocore/net/UnixNetAccept.cc @@ -21,6 +21,8 @@ limitations under the License. */ +#include + #include "P_Net.h" #ifdef ROUNDUP @@ -41,29 +43,6 @@ safe_delay(int msec) socketManager.poll(nullptr, 0, msec); } -static int -drain_throttled_accepts(NetAccept *na) -{ - struct pollfd afd; - Connection con[THROTTLE_AT_ONCE]; - - afd.fd = na->server.fd; - afd.events = POLLIN; - - // Try to close at most THROTTLE_AT_ONCE accept requests - // Stop if there is nothing waiting - int n = 0; - for (; n < THROTTLE_AT_ONCE && socketManager.poll(&afd, 1, 0) > 0; n++) { - int res = 0; - if ((res = na->server.accept(&con[n])) < 0) { - return res; - } - con[n].close(); - } - // Return the number of accept cases we closed - return n; -} - // // General case network connection accept code // @@ -78,7 +57,7 @@ net_accept(NetAccept *na, void *ep, bool blockable) Connection con; if (!blockable) { - if (!MUTEX_TAKE_TRY_LOCK(na->action_->mutex.get(), e->ethread)) { + if (!MUTEX_TAKE_TRY_LOCK(na->action_->mutex, e->ethread)) { return 0; } } @@ -115,7 +94,11 @@ net_accept(NetAccept *na, void *ep, bool blockable) vc->submit_time = Thread::get_hrtime(); vc->action_ = *na->action_; vc->set_is_transparent(na->opt.f_inbound_transparent); + vc->set_is_proxy_protocol(na->opt.f_proxy_protocol); vc->set_context(NET_VCONNECTION_IN); + if (na->opt.f_mptcp) { + vc->set_mptcp_state(); // Try to get the MPTCP state, and update accordingly + } #ifdef USE_EDGE_TRIGGER // Set the vc as triggered and place it in the read ready queue later in case there is already data on the socket. if (na->server.http_accept_filter) { @@ -148,7 +131,7 @@ net_accept(NetAccept *na, void *ep, bool blockable) Ldone: if (!blockable) { - MUTEX_UNTAKE_LOCK(na->action_->mutex.get(), e->ethread); + MUTEX_UNTAKE_LOCK(na->action_->mutex, e->ethread); } return count; } @@ -301,18 +284,7 @@ NetAccept::do_blocking_accept(EThread *t) // do-while for accepting all the connections // added by YTS Team, yamsat do { - // Throttle accepts - while (!opt.backdoor && check_net_throttle(ACCEPT)) { - check_throttle_warning(ACCEPT); - int num_throttled = drain_throttled_accepts(this); - if (num_throttled < 0) { - goto Lerror; - } - NET_SUM_DYN_STAT(net_connections_throttled_in_stat, num_throttled); - } - if ((res = server.accept(&con)) < 0) { - Lerror: int seriousness = accept_error_seriousness(res); if (seriousness >= 0) { // not so bad if (!seriousness) { // bad enough to warn about @@ -322,14 +294,22 @@ NetAccept::do_blocking_accept(EThread *t) return 0; } if (!action_->cancelled) { - SCOPED_MUTEX_LOCK(lock, action_->mutex, t); + SCOPED_MUTEX_LOCK(lock, action_->mutex ? action_->mutex : t->mutex, t); action_->continuation->handleEvent(EVENT_ERROR, (void *)(intptr_t)res); Warning("accept thread received fatal error: errno = %d", errno); } return -1; } + // check for throttle + if (!opt.backdoor && check_net_throttle(ACCEPT)) { + check_throttle_warning(ACCEPT); + // close the connection as we are in throttle state + con.close(); + NET_SUM_DYN_STAT(net_connections_throttled_in_stat, 1); + continue; + } - if (unlikely(shutdown_event_system == true)) { + if (TSSystemState::is_event_system_shut_down()) { return -1; } @@ -347,11 +327,15 @@ NetAccept::do_blocking_accept(EThread *t) vc->submit_time = Thread::get_hrtime(); vc->action_ = *action_; vc->set_is_transparent(opt.f_inbound_transparent); + vc->set_is_proxy_protocol(opt.f_proxy_protocol); vc->options.packet_mark = opt.packet_mark; vc->options.packet_tos = opt.packet_tos; vc->options.ip_family = opt.ip_family; vc->apply_options(); vc->set_context(NET_VCONNECTION_IN); + if (opt.f_mptcp) { + vc->set_mptcp_state(); // Try to get the MPTCP state, and update accordingly + } vc->accept_object = this; #ifdef USE_EDGE_TRIGGER // Set the vc as triggered and place it in the read ready queue later in case there is already data on the socket. @@ -377,12 +361,12 @@ NetAccept::acceptEvent(int event, void *ep) (void)event; Event *e = (Event *)ep; // PollDescriptor *pd = get_PollDescriptor(e->ethread); - ProxyMutex *m = nullptr; + Ptr m; if (action_->mutex) { - m = action_->mutex.get(); + m = action_->mutex; } else { - m = mutex.get(); + m = mutex; } MUTEX_TRY_LOCK(lock, m, e->ethread); @@ -495,11 +479,16 @@ NetAccept::acceptFastEvent(int event, void *ep) vc->submit_time = Thread::get_hrtime(); vc->action_ = *action_; vc->set_is_transparent(opt.f_inbound_transparent); + vc->set_is_proxy_protocol(opt.f_proxy_protocol); vc->options.packet_mark = opt.packet_mark; vc->options.packet_tos = opt.packet_tos; vc->options.ip_family = opt.ip_family; vc->apply_options(); vc->set_context(NET_VCONNECTION_IN); + if (opt.f_mptcp) { + vc->set_mptcp_state(); // Try to get the MPTCP state, and update accordingly + } + #ifdef USE_EDGE_TRIGGER // Set the vc as triggered and place it in the read ready queue later in case there is already data on the socket. if (server.http_accept_filter) { diff --git a/iocore/net/UnixNetProcessor.cc b/iocore/net/UnixNetProcessor.cc index 3d4771c401d..c18e6bdd04c 100644 --- a/iocore/net/UnixNetProcessor.cc +++ b/iocore/net/UnixNetProcessor.cc @@ -24,6 +24,7 @@ #include "P_Net.h" #include "tscore/InkErrno.h" #include "tscore/ink_sock.h" +#include "tscore/TSSystemState.h" #include "P_SSLNextProtocolAccept.h" // For Stat Pages @@ -51,6 +52,8 @@ NetProcessor::AcceptOptions::reset() packet_tos = 0; tfo_queue_length = 0; f_inbound_transparent = false; + f_mptcp = false; + f_proxy_protocol = false; return *this; } @@ -122,11 +125,8 @@ UnixNetProcessor::accept_internal(Continuation *cont, int fd, AcceptOptions cons Debug("http_tproxy", "Marked accept server %p on port %d as inbound transparent", na, opt.local_port); } - int should_filter_int = 0; - na->server.http_accept_filter = false; - REC_ReadConfigInteger(should_filter_int, "proxy.config.net.defer_accept"); - if (should_filter_int > 0 && opt.etype == ET_NET) { - na->server.http_accept_filter = true; + if (opt.f_proxy_protocol) { + Debug("http_tproxy", "Marked accept server %p on port %d for proxy protocol", na, opt.local_port); } SessionAccept *sa = dynamic_cast(cont); @@ -166,25 +166,6 @@ UnixNetProcessor::accept_internal(Continuation *cont, int fd, AcceptOptions cons naVec.push_back(na); } -#ifdef TCP_DEFER_ACCEPT - // set tcp defer accept timeout if it is configured, this will not trigger an accept until there is - // data on the socket ready to be read - if (should_filter_int > 0) { - setsockopt(na->server.fd, IPPROTO_TCP, TCP_DEFER_ACCEPT, &should_filter_int, sizeof(int)); - } -#endif - -#ifdef TCP_INIT_CWND - int tcp_init_cwnd = 0; - REC_ReadConfigInteger(tcp_init_cwnd, "proxy.config.http.server_tcp_init_cwnd"); - if (tcp_init_cwnd > 0) { - Debug("net", "Setting initial congestion window to %d", tcp_init_cwnd); - if (setsockopt(na->server.fd, IPPROTO_TCP, TCP_INIT_CWND, &tcp_init_cwnd, sizeof(int)) != 0) { - Error("Cannot set initial congestion window to %d", tcp_init_cwnd); - } - } -#endif - return na->action_.get(); } @@ -199,7 +180,7 @@ NetProcessor::stop_accept() Action * UnixNetProcessor::connect_re_internal(Continuation *cont, sockaddr const *target, NetVCOptions *opt) { - if (unlikely(shutdown_event_system == true)) { + if (TSSystemState::is_event_system_shut_down()) { return ACTION_RESULT_NONE; } EThread *t = cont->mutex->thread_holding; @@ -315,7 +296,18 @@ UnixNetProcessor::init() d.rec_int = 0; change_net_connections_throttle(nullptr, RECD_INT, d, nullptr); - // Socks + /* + * Stat pages + */ + extern Action *register_ShowNet(Continuation * c, HTTPHdr * h); + if (etype == ET_NET) { + statPagesManager.register_http("net", register_ShowNet); + } +} + +void +UnixNetProcessor::init_socks() +{ if (!netProcessor.socks_conf_stuff) { socks_conf_stuff = new socks_conf_struct; loadSocksConfiguration(socks_conf_stuff); @@ -328,14 +320,6 @@ UnixNetProcessor::init() socks_conf_stuff = netProcessor.socks_conf_stuff; } } - - /* - * Stat pages - */ - extern Action *register_ShowNet(Continuation * c, HTTPHdr * h); - if (etype == ET_NET) { - statPagesManager.register_http("net", register_ShowNet); - } } // Virtual function allows creation of an diff --git a/iocore/net/UnixNetVConnection.cc b/iocore/net/UnixNetVConnection.cc index e01fc57f31a..5ecc216b304 100644 --- a/iocore/net/UnixNetVConnection.cc +++ b/iocore/net/UnixNetVConnection.cc @@ -253,24 +253,6 @@ read_from_net(NetHandler *nh, UnixNetVConnection *vc, EThread *thread) NET_INCREMENT_DYN_STAT(net_calls_to_read_stat); - if (vc->origin_trace) { - char origin_trace_ip[INET6_ADDRSTRLEN]; - - ats_ip_ntop(vc->origin_trace_addr, origin_trace_ip, sizeof(origin_trace_ip)); - - if (r > 0) { - TraceIn((vc->origin_trace), vc->get_remote_addr(), vc->get_remote_port(), "CLIENT %s:%d\tbytes=%d\n%.*s", origin_trace_ip, - vc->origin_trace_port, (int)r, (int)r, (char *)tiovec[0].iov_base); - - } else if (r == 0) { - TraceIn((vc->origin_trace), vc->get_remote_addr(), vc->get_remote_port(), "CLIENT %s:%d closed connection", - origin_trace_ip, vc->origin_trace_port); - } else { - TraceIn((vc->origin_trace), vc->get_remote_addr(), vc->get_remote_port(), "CLIENT %s:%d error=%s", origin_trace_ip, - vc->origin_trace_port, strerror(errno)); - } - } - total_read += rattempted; } while (rattempted && r == rattempted && total_read < toread); @@ -376,8 +358,9 @@ write_to_net_io(NetHandler *nh, UnixNetVConnection *vc, EThread *thread) // vc is an SSLNetVConnection. if (!vc->getSSLHandShakeComplete()) { if (vc->trackFirstHandshake()) { - // Send the write ready on up to the state machine - write_signal_and_update(VC_EVENT_WRITE_READY, vc); + // Eat the first write-ready. Until the TLS handshake is complete, + // we should still be under the connect timeout and shouldn't bother + // the state machine until the TLS handshake is complete vc->write.triggered = 0; nh->write_ready_list.remove(vc); } @@ -883,23 +866,8 @@ UnixNetVConnection::reenable_re(VIO *vio) } } -UnixNetVConnection::UnixNetVConnection() - : closed(0), - inactivity_timeout_in(0), - active_timeout_in(0), - next_inactivity_timeout_at(0), - next_activity_timeout_at(0), - nh(nullptr), - id(0), - flags(0), - recursion(0), - submit_time(0), - oob_ptr(nullptr), - from_accept_thread(false), - accept_object(nullptr), - origin_trace(false), - origin_trace_addr(nullptr), - origin_trace_port(0) +UnixNetVConnection::UnixNetVConnection() : flags(0) + { SET_HANDLER((NetVConnHandler)&UnixNetVConnection::startEvent); } @@ -998,22 +966,6 @@ UnixNetVConnection::load_buffer_and_write(int64_t towrite, MIOBufferAccessor &bu r = socketManager.writev(con.fd, &tiovec[0], niov); } - if (origin_trace) { - char origin_trace_ip[INET6_ADDRSTRLEN]; - ats_ip_ntop(origin_trace_addr, origin_trace_ip, sizeof(origin_trace_ip)); - - if (r > 0) { - TraceOut(origin_trace, get_remote_addr(), get_remote_port(), "CLIENT %s:%d\tbytes=%d\n%.*s", origin_trace_ip, - origin_trace_port, (int)r, (int)r, (char *)tiovec[0].iov_base); - - } else if (r == 0) { - TraceOut(origin_trace, get_remote_addr(), get_remote_port(), "CLIENT %s:%d\tbytes=0", origin_trace_ip, origin_trace_port); - } else { - TraceOut(origin_trace, get_remote_addr(), get_remote_port(), "CLIENT %s:%d error=%s", origin_trace_ip, origin_trace_port, - strerror(errno)); - } - } - if (r > 0) { buf.reader()->consume(r); total_written += r; @@ -1124,8 +1076,15 @@ UnixNetVConnection::acceptEvent(int event, Event *e) if (active_timeout_in) { UnixNetVConnection::set_active_timeout(active_timeout_in); } - - action_.continuation->dispatchEvent(NET_EVENT_ACCEPT, this); + if (action_.continuation->mutex != nullptr) { + MUTEX_TRY_LOCK(lock3, action_.continuation->mutex, t); + if (!lock3.is_locked()) { + ink_release_assert(0); + } + action_.continuation->handleEvent(NET_EVENT_ACCEPT, this); + } else { + action_.continuation->handleEvent(NET_EVENT_ACCEPT, this); + } return EVENT_DONE; } @@ -1418,7 +1377,7 @@ UnixNetVConnection::migrateToCurrentThread(Continuation *cont, EThread *t) // Try to get the mutex lock for NetHandler of this NetVC MUTEX_TRY_LOCK(lock_src, this->nh->mutex, t); if (lock_src.is_locked()) { - // Deattach this NetVC from original NetHandler & InactivityCop. + // Detach this NetVC from original NetHandler & InactivityCop. this->nh->stopCop(this); this->nh->stopIO(this); // Put this NetVC into current NetHandler & InactivityCop. @@ -1481,25 +1440,48 @@ UnixNetVConnection::migrateToCurrentThread(Continuation *cont, EThread *t) void UnixNetVConnection::add_to_keep_alive_queue() { - nh->add_to_keep_alive_queue(this); + MUTEX_TRY_LOCK(lock, nh->mutex, this_ethread()); + if (lock.is_locked()) { + nh->add_to_keep_alive_queue(this); + } else { + ink_release_assert(!"BUG: It must have acquired the NetHandler's lock before doing anything on keep_alive_queue."); + } } void UnixNetVConnection::remove_from_keep_alive_queue() { - nh->remove_from_keep_alive_queue(this); + MUTEX_TRY_LOCK(lock, nh->mutex, this_ethread()); + if (lock.is_locked()) { + nh->remove_from_keep_alive_queue(this); + } else { + ink_release_assert(!"BUG: It must have acquired the NetHandler's lock before doing anything on keep_alive_queue."); + } } bool UnixNetVConnection::add_to_active_queue() { - return nh->add_to_active_queue(this); + bool result = false; + + MUTEX_TRY_LOCK(lock, nh->mutex, this_ethread()); + if (lock.is_locked()) { + result = nh->add_to_active_queue(this); + } else { + ink_release_assert(!"BUG: It must have acquired the NetHandler's lock before doing anything on active_queue."); + } + return result; } void UnixNetVConnection::remove_from_active_queue() { - nh->remove_from_active_queue(this); + MUTEX_TRY_LOCK(lock, nh->mutex, this_ethread()); + if (lock.is_locked()) { + nh->remove_from_active_queue(this); + } else { + ink_release_assert(!"BUG: It must have acquired the NetHandler's lock before doing anything on active_queue."); + } } int @@ -1531,3 +1513,33 @@ UnixNetVConnection::protocol_contains(std::string_view tag) const } return retval.data(); } + +int +UnixNetVConnection::set_tcp_congestion_control(int side) +{ +#ifdef TCP_CONGESTION + std::string_view ccp; + + if (side == CLIENT_SIDE) { + ccp = net_ccp_in; + } else { + ccp = net_ccp_out; + } + + if (!ccp.empty()) { + int rv = setsockopt(con.fd, IPPROTO_TCP, TCP_CONGESTION, reinterpret_cast(ccp.data()), ccp.size()); + + if (rv < 0) { + Error("Unable to set TCP congestion control on socket %d to \"%s\", errno=%d (%s)", con.fd, ccp.data(), errno, + strerror(errno)); + } else { + Debug("socket", "Setting TCP congestion control on socket [%d] to \"%s\" -> %d", con.fd, ccp.data(), rv); + } + return 0; + } + return -1; +#else + Debug("socket", "Setting TCP congestion control is not supported on this platform."); + return -1; +#endif +} diff --git a/iocore/net/UnixUDPNet.cc b/iocore/net/UnixUDPNet.cc index 3f73b1bb08c..eb7cf2b9bb2 100644 --- a/iocore/net/UnixUDPNet.cc +++ b/iocore/net/UnixUDPNet.cc @@ -66,8 +66,14 @@ initialize_thread_for_udp_net(EThread *thread) new ((ink_dummy_for_new *)get_UDPPollCont(thread)) PollCont(thread->mutex); // The UDPNetHandler cannot be accessed across EThreads. // Because the UDPNetHandler should be called back immediately after UDPPollCont. - nh->mutex = thread->mutex.get(); - + nh->mutex = thread->mutex.get(); + nh->thread = thread; + + PollCont *upc = get_UDPPollCont(thread); + PollDescriptor *upd = upc->pollDescriptor; + // due to ET_UDP is really simple, it should sleep for a long time + // TODO: fixed size + upc->poll_timeout = 100; // This variable controls how often we cleanup the cancelled packets. // If it is set to 0, then cleanup never occurs. REC_ReadConfigInt32(g_udp_periodicFreeCancelledPkts, "proxy.config.udp.free_cancelled_pkts_sec"); @@ -82,8 +88,15 @@ initialize_thread_for_udp_net(EThread *thread) REC_ReadConfigInt32(g_udp_numSendRetries, "proxy.config.udp.send_retries"); g_udp_numSendRetries = g_udp_numSendRetries < 0 ? 0 : g_udp_numSendRetries; - thread->schedule_every(get_UDPPollCont(thread), -9); - thread->schedule_imm(get_UDPNetHandler(thread)); + thread->set_tail_handler(nh); + thread->ep = (EventIO *)ats_malloc(sizeof(EventIO)); + new (thread->ep) EventIO(); + thread->ep->type = EVENTIO_ASYNC_SIGNAL; +#if HAVE_EVENTFD + thread->ep->start(upd, thread->evfd, nullptr, EVENTIO_READ); +#else + thread->ep->start(upd, thread->evpipe[0], nullptr, EVENTIO_READ); +#endif } int @@ -284,13 +297,10 @@ UDPReadContinuation::UDPReadContinuation(Event *completionToken) : Continuation(nullptr), event(completionToken), readbuf(nullptr), - readlen(0), - fromaddrlen(nullptr), + fd(-1), - ifd(-1), - period(0), - elapsed_time(0), - timeout_interval(0) + ifd(-1) + { if (completionToken->continuation) { this->mutex = completionToken->continuation->mutex; @@ -628,7 +638,7 @@ UDPNetProcessor::CreateUDPSocket(int *resfd, sockaddr const *remote_addr, Action } if ((res = safe_getsockname(fd, &local_addr.sa, &local_addr_len)) < 0) { - Debug("udpnet", "CreateUdpsocket: getsockname didnt' work"); + Debug("udpnet", "CreateUdpsocket: getsockname didn't work"); goto HardError; } } @@ -890,6 +900,20 @@ UDPQueue::send(UDPPacket *p) #undef LINK +static void +net_signal_hook_callback(EThread *thread) +{ +#if HAVE_EVENTFD + uint64_t counter; + ATS_UNUSED_RETURN(read(thread->evfd, &counter, sizeof(uint64_t))); +#elif TS_USE_PORT +/* Nothing to drain or do */ +#else + char dummy[1024]; + ATS_UNUSED_RETURN(read(thread->evpipe[0], &dummy[0], 1024)); +#endif +} + UDPNetHandler::UDPNetHandler() { nextCheck = Thread::get_hrtime_updated() + HRTIME_MSECONDS(1000); @@ -911,10 +935,15 @@ int UDPNetHandler::mainNetEvent(int event, Event *e) { ink_assert(trigger_event == e && event == EVENT_POLL); - (void)event; - (void)e; + return this->waitForActivity(net_config_poll_timeout); +} +int +UDPNetHandler::waitForActivity(ink_hrtime timeout) +{ UnixUDPConnection *uc; + PollCont *pc = get_UDPPollCont(this->thread); + pc->do_poll(timeout); /* Notice: the race between traversal of newconn_list and UDPBind() * @@ -943,7 +972,6 @@ UDPNetHandler::mainNetEvent(int event, Event *e) // handle UDP read operations int i, nread = 0; - PollCont *pc = get_UDPPollCont(e->ethread); EventIO *epd = nullptr; for (i = 0; i < pc->pollDescriptor->result; i++) { epd = (EventIO *)get_ev_data(pc->pollDescriptor, i); @@ -973,8 +1001,7 @@ UDPNetHandler::mainNetEvent(int event, Event *e) #endif } } else if (epd->type == EVENTIO_ASYNC_SIGNAL) { - // TODO: receive signal from event system - // net_signal_hook_callback(this->trigger_event->ethread); + net_signal_hook_callback(this->thread); } } // end for @@ -997,7 +1024,7 @@ UDPNetHandler::mainNetEvent(int event, Event *e) udp_callbacks.clear(); while ((uc = q.dequeue())) { ink_assert(uc->mutex && uc->continuation); - if (udpNetInternal.udp_callback(this, uc, trigger_event->ethread)) { // not successful + if (udpNetInternal.udp_callback(this, uc, this->thread)) { // not successful // schedule on a thread of its own. ink_assert(uc->callback_link.next == nullptr); ink_assert(uc->callback_link.prev == nullptr); @@ -1012,3 +1039,18 @@ UDPNetHandler::mainNetEvent(int event, Event *e) return EVENT_CONT; } + +void +UDPNetHandler::signalActivity() +{ +#if HAVE_EVENTFD + uint64_t counter = 1; + ATS_UNUSED_RETURN(write(thread->evfd, &counter, sizeof(uint64_t))); +#elif TS_USE_PORT + PollDescriptor *pd = get_PollDescriptor(thread); + ATS_UNUSED_RETURN(port_send(pd->port_fd, 0, thread->ep)); +#else + char dummy = 1; + ATS_UNUSED_RETURN(write(thread->evpipe[1], &dummy, 1)); +#endif +} diff --git a/iocore/net/YamlSNIConfig.cc b/iocore/net/YamlSNIConfig.cc index 5366e4b94ff..8fbb4d81320 100644 --- a/iocore/net/YamlSNIConfig.cc +++ b/iocore/net/YamlSNIConfig.cc @@ -26,10 +26,12 @@ #include #include +#include #include "tscore/Diags.h" #include "tscore/EnumDescriptor.h" #include "tsconfig/Errata.h" +#include "P_SNIActionPerformer.h" ts::Errata YamlSNIConfig::loader(const char *cfgFilename) @@ -55,20 +57,54 @@ YamlSNIConfig::loader(const char *cfgFilename) return ts::Errata(); } -TsEnumDescriptor LEVEL_DESCRIPTOR = {{{"NONE", 0}, {"MODERATE", 1}, {"STRICT", 2}}}; -TsEnumDescriptor POLICY_DESCRIPTOR = {{{"DISABLED", 0}, {"PERMISSIVE", 1}, {"ENFORCED", 2}}}; -TsEnumDescriptor PROPERTIES_DESCRIPTOR = {{{"NONE", 0}, {"SIGNATURE", 0x1}, {"NAME", 0x2}, {"ALL", 0x3}}}; +void +YamlSNIConfig::Item::EnableProtocol(YamlSNIConfig::TLSProtocol proto) +{ + if (proto <= YamlSNIConfig::TLSProtocol::TLS_MAX) { + if (protocol_unset) { + protocol_mask = TLSValidProtocols::max_mask; + protocol_unset = false; + } + switch (proto) { + case YamlSNIConfig::TLSProtocol::TLSv1: + protocol_mask &= ~SSL_OP_NO_TLSv1; + break; + case YamlSNIConfig::TLSProtocol::TLSv1_1: + protocol_mask &= ~SSL_OP_NO_TLSv1_1; + break; + case YamlSNIConfig::TLSProtocol::TLSv1_2: + protocol_mask &= ~SSL_OP_NO_TLSv1_2; + break; + case YamlSNIConfig::TLSProtocol::TLSv1_3: +#ifdef SSL_OP_NO_TLSv1_3 + protocol_mask &= ~SSL_OP_NO_TLSv1_3; +#endif + break; + } + } +} + +TsEnumDescriptor LEVEL_DESCRIPTOR = {{{"NONE", 0}, {"MODERATE", 1}, {"STRICT", 2}}}; +TsEnumDescriptor POLICY_DESCRIPTOR = {{{"DISABLED", 0}, {"PERMISSIVE", 1}, {"ENFORCED", 2}}}; +TsEnumDescriptor PROPERTIES_DESCRIPTOR = {{{"NONE", 0}, {"SIGNATURE", 0x1}, {"NAME", 0x2}, {"ALL", 0x3}}}; +TsEnumDescriptor TLS_PROTOCOLS_DESCRIPTOR = {{{"TLSv1", 0}, {"TLSv1_1", 1}, {"TLSv1_2", 2}, {"TLSv1_3", 3}}}; std::set valid_sni_config_keys = {TS_fqdn, - TS_disable_H2, + TS_disable_h2, TS_verify_client, TS_tunnel_route, + TS_forward_route, TS_verify_origin_server, TS_verify_server_policy, TS_verify_server_properties, TS_client_cert, TS_client_key, - TS_ip_allow}; + TS_ip_allow +#if TS_USE_HELLO_CB + , + TS_valid_tls_versions_in +#endif +}; namespace YAML { @@ -79,7 +115,7 @@ template <> struct convert { for (auto &&item : node) { if (std::none_of(valid_sni_config_keys.begin(), valid_sni_config_keys.end(), [&item](std::string s) { return s == item.first.as(); })) { - throw std::runtime_error("unsupported key " + item.first.as()); + throw YAML::ParserException(item.Mark(), "unsupported key " + item.first.as()); } } @@ -88,8 +124,8 @@ template <> struct convert { } else { return false; // servername must be present } - if (node[TS_disable_H2]) { - item.disable_h2 = node[TS_disable_H2].as(); + if (node[TS_disable_h2]) { + item.disable_h2 = node[TS_disable_h2].as(); } // enum @@ -97,16 +133,21 @@ template <> struct convert { auto value = node[TS_verify_client].as(); int level = LEVEL_DESCRIPTOR.get(value); if (level < 0) { - throw std::runtime_error("unknown value \"" + value + "\""); + throw YAML::ParserException(node[TS_verify_client].Mark(), "unknown value \"" + value + "\""); } item.verify_client_level = static_cast(level); } if (node[TS_tunnel_route]) { item.tunnel_destination = node[TS_tunnel_route].as(); + item.tunnel_decrypt = false; + } else if (node[TS_forward_route]) { + item.tunnel_destination = node[TS_forward_route].as(); + item.tunnel_decrypt = true; } + // remove before 9.0.0 release - // backwards compatibiity + // backwards compatibility if (node[TS_verify_origin_server]) { auto value = node[TS_verify_origin_server].as(); YamlSNIConfig::Level level = static_cast(LEVEL_DESCRIPTOR.get(value)); @@ -128,7 +169,7 @@ template <> struct convert { auto value = node[TS_verify_server_policy].as(); int policy = POLICY_DESCRIPTOR.get(value); if (policy < 0) { - throw std::runtime_error("unknown value \"" + value + "\""); + throw YAML::ParserException(node[TS_verify_server_policy].Mark(), "unknown value \"" + value + "\""); } item.verify_server_policy = static_cast(policy); } @@ -137,7 +178,7 @@ template <> struct convert { auto value = node[TS_verify_server_properties].as(); int properties = PROPERTIES_DESCRIPTOR.get(value); if (properties < 0) { - throw std::runtime_error("unknown value \"" + value + "\""); + throw YAML::ParserException(node[TS_verify_server_properties].Mark(), "unknown value \"" + value + "\""); } item.verify_server_properties = static_cast(properties); } @@ -152,6 +193,13 @@ template <> struct convert { if (node[TS_ip_allow]) { item.ip_allow = node[TS_ip_allow].as(); } + if (node[TS_valid_tls_versions_in]) { + for (unsigned int i = 0; i < node[TS_valid_tls_versions_in].size(); i++) { + auto value = node[TS_valid_tls_versions_in][i].as(); + int protocol = TLS_PROTOCOLS_DESCRIPTOR.get(value); + item.EnableProtocol(static_cast(protocol)); + } + } return true; } }; diff --git a/iocore/net/YamlSNIConfig.h b/iocore/net/YamlSNIConfig.h index 0cb6d64b61d..babd0166376 100644 --- a/iocore/net/YamlSNIConfig.h +++ b/iocore/net/YamlSNIConfig.h @@ -28,15 +28,17 @@ #define TSDECL(id) constexpr char TS_##id[] = #id TSDECL(fqdn); -TSDECL(disable_H2); +TSDECL(disable_h2); TSDECL(verify_client); TSDECL(tunnel_route); +TSDECL(forward_route); TSDECL(verify_server_policy); TSDECL(verify_server_properties); TSDECL(verify_origin_server); TSDECL(client_cert); TSDECL(client_key); TSDECL(ip_allow); +TSDECL(valid_tls_versions_in); #undef TSDECL const int start = 0; @@ -45,26 +47,33 @@ struct YamlSNIConfig { disable_h2 = start, verify_client, tunnel_route, // blind tunnel action + forward_route, // decrypt data and then blind tunnel action verify_server_policy, // this applies to server side vc only verify_server_properties, // this applies to server side vc only client_cert }; enum class Level { NONE = 0, MODERATE, STRICT }; - enum class Policy : uint8_t { DISABLED = 0, PERMISSIVE, ENFORCED }; - enum class Property : uint8_t { NONE = 0, SIGNATURE_MASK = 0x1, NAME_MASK = 0x2, ALL_MASK = 0x3 }; + enum class Policy : uint8_t { DISABLED = 0, PERMISSIVE, ENFORCED, UNSET }; + enum class Property : uint8_t { NONE = 0, SIGNATURE_MASK = 0x1, NAME_MASK = 0x2, ALL_MASK = 0x3, UNSET }; + enum class TLSProtocol : uint8_t { TLSv1 = 0, TLSv1_1, TLSv1_2, TLSv1_3, TLS_MAX = TLSv1_3 }; YamlSNIConfig() {} struct Item { std::string fqdn; bool disable_h2 = false; - uint8_t verify_client_level = 0; + uint8_t verify_client_level = 255; std::string tunnel_destination; - Policy verify_server_policy = Policy::DISABLED; - Property verify_server_properties = Property::NONE; + bool tunnel_decrypt = false; + Policy verify_server_policy = Policy::UNSET; + Property verify_server_properties = Property::UNSET; std::string client_cert; std::string client_key; std::string ip_allow; + bool protocol_unset = true; + unsigned long protocol_mask; + + void EnableProtocol(YamlSNIConfig::TLSProtocol proto); }; ts::Errata loader(const char *cfgFilename); diff --git a/iocore/net/test_I_Net.cc b/iocore/net/test_I_Net.cc index d980292e5d4..9961bc00b8e 100644 --- a/iocore/net/test_I_Net.cc +++ b/iocore/net/test_I_Net.cc @@ -29,7 +29,6 @@ /* * Choose a net test application */ -//#include "NetTest-http-server.c" #include "NetTest-simple-proxy.c" int @@ -43,8 +42,8 @@ main() init_diags("net_test", nullptr); RecProcessInit(mode_type); - ink_event_system_init(EVENT_SYSTEM_MODULE_VERSION); - ink_net_init(NET_SYSTEM_MODULE_VERSION); + ink_event_system_init(EVENT_SYSTEM_MODULE_PUBLIC_VERSION); + ink_net_init(NET_SYSTEM_MODULE_PUBLIC_VERSION); /* * ignore broken pipe diff --git a/iocore/net/test_I_UDPNet.cc b/iocore/net/test_I_UDPNet.cc index 355a917b276..16810cbd22e 100644 --- a/iocore/net/test_I_UDPNet.cc +++ b/iocore/net/test_I_UDPNet.cc @@ -135,7 +135,7 @@ udp_echo_server() net_config_poll_timeout = 10; init_diags("udp-.*", nullptr); - ink_event_system_init(EVENT_SYSTEM_MODULE_VERSION); + ink_event_system_init(EVENT_SYSTEM_MODULE_PUBLIC_VERSION); eventProcessor.start(2); udpNet.start(1, 1048576); @@ -362,7 +362,7 @@ StatPagesManager statPagesManager; inkcoreapi ProcessManager *pmgmt = nullptr; int -BaseManager::registerMgmtCallback(int, MgmtCallback, void *) +BaseManager::registerMgmtCallback(int, MgmtCallback const &) { ink_assert(false); return 0; diff --git a/iocore/net/test_P_Net.cc b/iocore/net/test_P_Net.cc index 248b8bdeebc..185b1586e65 100644 --- a/iocore/net/test_P_Net.cc +++ b/iocore/net/test_P_Net.cc @@ -55,7 +55,7 @@ struct NetTesterSM : public Continuation { fflush(stdout); break; case VC_EVENT_READ_COMPLETE: - /* FALLSTHROUGH */ + /* FALLTHROUGH */ case VC_EVENT_EOS: r = reader->read_avail(); str = new char[r + 10]; @@ -89,7 +89,7 @@ struct NetTesterAccept : public Continuation { int main() { - ink_event_system_init(EVENT_SYSTEM_MODULE_VERSION); + ink_event_system_init(EVENT_SYSTEM_MODULE_PUBLIC_VERSION); MIOBuffer *mbuf = new_MIOBuffer(5); eventProcessor.start(1); netProcessor.start(); diff --git a/iocore/net/test_certlookup.cc b/iocore/net/test_certlookup.cc index 31913b6593f..96a8f362328 100644 --- a/iocore/net/test_certlookup.cc +++ b/iocore/net/test_certlookup.cc @@ -168,7 +168,7 @@ load_hostnames_csv(const char *fname, SSLCertLookup &lookup) SSLCertContext ctx_cc(ctx); // The input should have 2 comma-separated fields; this is the format that you get when - // you download the top 1M sites from alexa. + // you download the top 1M sites from Alexa. // // For example: // 1,google.com diff --git a/iocore/utils/I_OneWayMultiTunnel.h b/iocore/utils/I_OneWayMultiTunnel.h index 9351851e1a7..c4973e6ffe1 100644 --- a/iocore/utils/I_OneWayMultiTunnel.h +++ b/iocore/utils/I_OneWayMultiTunnel.h @@ -36,7 +36,7 @@ #define ONE_WAY_MULTI_TUNNEL_LIMIT 4 /** - A generic state machine that connects a source virtual conection to + A generic state machine that connects a source virtual connection to multiple target virtual connections. A OneWayMultiTunnel is similar to the OneWayTunnel module. However, instead of connection one source to one target, it connects multiple virtual connections - a source diff --git a/iocore/utils/I_OneWayTunnel.h b/iocore/utils/I_OneWayTunnel.h index 448737c92dc..d74401a88ab 100644 --- a/iocore/utils/I_OneWayTunnel.h +++ b/iocore/utils/I_OneWayTunnel.h @@ -45,7 +45,7 @@ typedef void (*Transform_fn)(MIOBufferAccessor &in_buf, MIOBufferAccessor &out_buf); /** - A generic state machine that connects two virtual conections. A + A generic state machine that connects two virtual connections. A OneWayTunnel is a module that connects two virtual connections, a source vc and a target vc, and copies the data between source and target. Once the tunnel is started using the init() call, it handles all the events @@ -189,21 +189,21 @@ struct OneWayTunnel : public Continuation { bool last_connection(); - VIO *vioSource; - VIO *vioTarget; - Continuation *cont; - Transform_fn manipulate_fn; - int n_connections; - int lerrno; + VIO *vioSource = nullptr; + VIO *vioTarget = nullptr; + Continuation *cont = nullptr; + Transform_fn manipulate_fn = nullptr; + int n_connections = 0; + int lerrno = 0; - bool single_buffer; - bool close_source; - bool close_target; - bool tunnel_till_done; + bool single_buffer = false; + bool close_source = false; + bool close_target = false; + bool tunnel_till_done = false; /** Non-nullptr when this is one side of a two way tunnel. */ - OneWayTunnel *tunnel_peer; - bool free_vcs; + OneWayTunnel *tunnel_peer = nullptr; + bool free_vcs = true; // noncopyable OneWayTunnel(const OneWayTunnel &) = delete; diff --git a/iocore/utils/Machine.cc b/iocore/utils/Machine.cc index b8bc01576e4..bcfc1efbe6d 100644 --- a/iocore/utils/Machine.cc +++ b/iocore/utils/Machine.cc @@ -231,8 +231,8 @@ Machine::Machine(char const *the_hostname, sockaddr const *addr) Machine::~Machine() { ats_free(hostname); - for (auto spot = machine_id_ipaddrs.begin(); spot != machine_id_ipaddrs.end(); spot++) { - delete spot->second; + for (auto &machine_id_ipaddr : machine_id_ipaddrs) { + delete machine_id_ipaddr.second; } } diff --git a/iocore/utils/OneWayTunnel.cc b/iocore/utils/OneWayTunnel.cc index 07607267531..d4484f28d39 100644 --- a/iocore/utils/OneWayTunnel.cc +++ b/iocore/utils/OneWayTunnel.cc @@ -64,22 +64,7 @@ transfer_data(MIOBufferAccessor &in_buf, MIOBufferAccessor &out_buf) out_buf.writer()->fill(n); } -OneWayTunnel::OneWayTunnel() - : Continuation(nullptr), - vioSource(nullptr), - vioTarget(nullptr), - cont(nullptr), - manipulate_fn(nullptr), - n_connections(0), - lerrno(0), - single_buffer(false), - close_source(false), - close_target(false), - tunnel_till_done(false), - tunnel_peer(nullptr), - free_vcs(true) -{ -} +OneWayTunnel::OneWayTunnel() : Continuation(nullptr) {} OneWayTunnel * OneWayTunnel::OneWayTunnel_alloc() diff --git a/lib/perl/Makefile.am b/lib/perl/Makefile.am index 591ee2b709e..4a7c4de2b6b 100644 --- a/lib/perl/Makefile.am +++ b/lib/perl/Makefile.am @@ -32,7 +32,7 @@ Makefile-pl: Makefile.PL $(top_builddir)/config.status $(PERL) Makefile.PL INSTALLDIRS=$(INSTALLDIRS) INSTALL_BASE=$(prefix) PREFIX= clean-local: - -rm Makefile-pl + -rm -f Makefile-pl distclean-local: -rm -rf Makefile-pl MYMETA.* blip diff --git a/lib/perl/examples/forw_proxy_conf.pl b/lib/perl/examples/forw_proxy_conf.pl index 9cdd79cdfba..ae25562b232 100755 --- a/lib/perl/examples/forw_proxy_conf.pl +++ b/lib/perl/examples/forw_proxy_conf.pl @@ -18,7 +18,6 @@ use Apache::TS::Config::Records; - ############################################################################ # Simple script, to show some minimum configuration changes typical for # a forward proxy. @@ -26,16 +25,16 @@ my $recedit = new Apache::TS::Config::Records(file => $fn); # Definitely tweak the memory config -$recedit->set(conf => "proxy.config.cache.ram_cache.size", val => "2048M"); +$recedit->set(conf => "proxy.config.cache.ram_cache.size", val => "2048M"); # These puts the server in forward proxy mode only. -$recedit->set(conf => "proxy.config.url_remap.remap_required", val => "0"); -$recedit->set(conf => "proxy.config.reverse_proxy.enabled", val => "0"); +$recedit->set(conf => "proxy.config.url_remap.remap_required", val => "0"); +$recedit->set(conf => "proxy.config.reverse_proxy.enabled", val => "0"); # Fine tuning, you might or might not want these $recedit->set(conf => "proxy.config.http.transaction_active_timeout_in", val => "1800"); -$recedit->set(conf => "proxy.config.dns.dedicated_thread", val => "1"); -$recedit->set(conf => "proxy.config.http.normalize_ae_gzip", val => "1"); +$recedit->set(conf => "proxy.config.dns.dedicated_thread", val => "1"); +$recedit->set(conf => "proxy.config.http.normalize_ae_gzip", val => "1"); # Write out the new config file (this won't overwrite your config $recedit->write(file => "$fn.new"); diff --git a/lib/perl/lib/Apache/TS/AdminClient.pm b/lib/perl/lib/Apache/TS/AdminClient.pm index efd6b4b559e..9892a909579 100644 --- a/lib/perl/lib/Apache/TS/AdminClient.pm +++ b/lib/perl/lib/Apache/TS/AdminClient.pm @@ -76,20 +76,16 @@ use constant { TS_ERR_FAIL => 12 }; - # Semi-intelligent way of finding the mgmtapi socket. -sub _find_socket { +sub _find_socket +{ my $path = shift || ""; my $name = shift || "mgmtapi.sock"; my @sockets_def = ( - $path, - Apache::TS::PREFIX . '/' . Apache::TS::REL_RUNTIMEDIR . '/' . 'mgmtapi.sock', - '/usr/local/var/trafficserver', - '/usr/local/var/run/trafficserver', - '/usr/local/var/run', - '/var/trafficserver', - '/var/run/trafficserver', - '/var/run', + $path, Apache::TS::PREFIX . '/' . Apache::TS::REL_RUNTIMEDIR . '/' . 'mgmtapi.sock', + '/usr/local/var/trafficserver', '/usr/local/var/run/trafficserver', + '/usr/local/var/run', '/var/trafficserver', + '/var/run/trafficserver', '/var/run', '/opt/ats/var/trafficserver', ); @@ -104,14 +100,14 @@ sub _find_socket { # # Constructor # -sub new { +sub new +{ my ($class, %args) = @_; my $self = {}; $self->{_socket_path} = _find_socket($args{socket_path}); - $self->{_socket} = undef; - croak -"Unable to locate socket, please pass socket_path with the management api socket location to Apache::TS::AdminClient" + $self->{_socket} = undef; + croak "Unable to locate socket, please pass socket_path with the management api socket location to Apache::TS::AdminClient" if (!$self->{_socket_path}); if ((!-r $self->{_socket_path}) or (!-w $self->{_socket_path}) or (!-S $self->{_socket_path})) { croak "Unable to open $self->{_socket_path} for reads or writes"; @@ -128,7 +124,8 @@ sub new { # # Destructor # -sub DESTROY { +sub DESTROY +{ my $self = shift; return $self->close_socket(); } @@ -136,15 +133,15 @@ sub DESTROY { # # Open the socket (Unix domain) # -sub open_socket { +sub open_socket +{ my $self = shift; my %args = @_; if (defined($self->{_socket})) { if ($args{force} || $args{reopen}) { $self->close_socket(); - } - else { + } else { return undef; } } @@ -152,7 +149,7 @@ sub open_socket { $self->{_socket} = IO::Socket::UNIX->new( Type => SOCK_STREAM, Peer => $self->{_socket_path} - ) or croak("Error opening socket - $@"); + ) or croak("Error opening socket - $@"); return undef unless defined($self->{_socket}); $self->{_select}->add($self->{_socket}); @@ -160,7 +157,8 @@ sub open_socket { return $self; } -sub close_socket { +sub close_socket +{ my $self = shift; # if socket doesn't exist, return as there's nothing to do. @@ -177,10 +175,11 @@ sub close_socket { # # Do reads()'s on our Unix domain socket, takes an optional timeout, in ms's. # -sub _do_read { - my $self = shift; - my $timeout = shift || 1/1000.0; # 1ms by default - my $res = ""; +sub _do_read +{ + my $self = shift; + my $timeout = shift || 1 / 1000.0; # 1ms by default + my $res = ""; while ($self->{_select}->can_read($timeout)) { my $rc = $self->{_socket}->sysread($res, 1024, length($res)); @@ -199,14 +198,14 @@ sub _do_read { return $res || undef; } - # # Get (read) a stat out of the local manager. Note that the assumption is # that you are calling this with an existing stats "name". # -sub get_stat { +sub get_stat +{ my ($self, $stat) = @_; - my $res = ""; + my $res = ""; return undef unless defined($self->{_socket}); return undef unless $self->{_select}->can_write(10); @@ -219,7 +218,7 @@ sub get_stat { my $msg = pack("ll/Z", TS_RECORD_GET, $stat); $self->{_socket}->print(pack("l/a", $msg)); $res = $self->_do_read(); - return undef unless defined($res); # Don't proceed on read failure. + return undef unless defined($res); # Don't proceed on read failure. # The response format is: # MGMT_MARSHALL_INT: message length @@ -235,12 +234,10 @@ sub get_stat { if ($type == TS_REC_INT || $type == TS_REC_COUNTER) { my ($ival) = unpack("q", $value); return $ival; - } - elsif ($type == TS_REC_FLOAT) { + } elsif ($type == TS_REC_FLOAT) { my ($fval) = unpack("f", $value); return $fval; - } - elsif ($type == TS_REC_STRING) { + } elsif ($type == TS_REC_STRING) { my ($sval) = unpack("Z*", $value); return $sval; } @@ -482,7 +479,7 @@ The Apache Traffic Server Administration Manual will explain what these strings proxy.config.http.no_origin_server_dns proxy.config.http.normalize_ae_gzip proxy.config.http.number_of_redirections - proxy.config.http.origin_max_connections + proxy.config.http.per_server.connection.max proxy.config.http.origin_min_keep_alive_connections proxy.config.http.parent_proxies proxy.config.http.parent_proxy.connect_attempts_timeout @@ -523,12 +520,6 @@ The Apache Traffic Server Administration Manual will explain what these strings proxy.config.local_state_dir proxy.config.log.ascii_buffer_size proxy.config.log.auto_delete_rolled_files - proxy.config.log.collation_host - proxy.config.log.collation_host_tagged - proxy.config.log.collation_max_send_buffers - proxy.config.log.collation_port - proxy.config.log.collation_retry_sec - proxy.config.log.collation_secret proxy.config.log.file_stat_frequency proxy.config.log.hostname proxy.config.log.log_buffer_size @@ -538,7 +529,6 @@ The Apache Traffic Server Administration Manual will explain what these strings proxy.config.log.max_line_size proxy.config.log.max_secs_per_buffer proxy.config.log.max_space_mb_for_logs - proxy.config.log.max_space_mb_for_orphan_logs proxy.config.log.max_space_mb_headroom proxy.config.log.overspill_report_count proxy.config.log.rolling_enabled @@ -599,7 +589,6 @@ The Apache Traffic Server Administration Manual will explain what these strings proxy.config.ssl.client.private_key.filename proxy.config.ssl.client.private_key.path proxy.config.ssl.client.verify.server - proxy.config.ssl.enabled proxy.config.ssl.server.cert_chain.filename proxy.config.ssl.server.cert.path proxy.config.ssl.server.cipher_suite diff --git a/lib/perl/lib/Apache/TS/Config.pm b/lib/perl/lib/Apache/TS/Config.pm index 4e9e7aa6289..d4a50078542 100644 --- a/lib/perl/lib/Apache/TS/Config.pm +++ b/lib/perl/lib/Apache/TS/Config.pm @@ -31,7 +31,7 @@ our $VERSION = "1.0"; # Constants use constant { - TS_CONF_UNMODIFIED => 0, - TS_CONF_MODIFIED => 1, - TS_CONF_REMOVED => 2 + TS_CONF_UNMODIFIED => 0, + TS_CONF_MODIFIED => 1, + TS_CONF_REMOVED => 2 }; diff --git a/lib/perl/lib/Apache/TS/Config/Records.pm b/lib/perl/lib/Apache/TS/Config/Records.pm index 5f691138610..078aab87d16 100644 --- a/lib/perl/lib/Apache/TS/Config/Records.pm +++ b/lib/perl/lib/Apache/TS/Config/Records.pm @@ -15,7 +15,6 @@ # See the License for the specific language governing permissions and # limitations under the License. - ############################################################################ # This is a simple module to let you read, modify and add to an Apache # Traffic Server records.config file. The idea is that you would write a @@ -25,7 +24,6 @@ # perldoc for more details. ############################################################################ - package Apache::TS::Config::Records; use Apache::TS::Config; @@ -38,22 +36,22 @@ use Carp; our $VERSION = "1.0"; - # # Constructor # -sub new { +sub new +{ my ($class, %args) = @_; my $self = {}; - my $fn = $args{file}; + my $fn = $args{file}; $fn = $args{filename} unless defined($fn); - $fn = "-" unless defined($fn); + $fn = "-" unless defined($fn); - $self->{_filename} = $fn; # Filename to open when loading and saving - $self->{_configs} = []; # Storage, and to to preserve order - $self->{_lookup} = {}; # For faster lookup, indexes into the above - $self->{_ix} = -1; # Empty + $self->{_filename} = $fn; # Filename to open when loading and saving + $self->{_configs} = []; # Storage, and to to preserve order + $self->{_lookup} = {}; # For faster lookup, indexes into the above + $self->{_ix} = -1; # Empty bless $self, $class; $self->load() if $self->{_filename}; @@ -61,16 +59,16 @@ sub new { return $self; } - # # Load a records.config file # -sub load { +sub load +{ my $self = shift; my %args = @_; - my $fn = $args{file}; + my $fn = $args{file}; - $fn = $args{filename} unless defined($fn); + $fn = $args{filename} unless defined($fn); $fn = $self->{_filename} unless defined($fn); open(FH, "<$fn") || die "Can't open file $fn for reading"; @@ -88,14 +86,14 @@ sub load { } } - # # Get an existing configuration line, as an anon array. # -sub get { +sub get +{ my $self = shift; my %args = @_; - my $c = $args{conf}; + my $c = $args{conf}; $c = $args{config} unless defined($c); my $ix = $self->{_lookup}->{$c}; @@ -104,26 +102,26 @@ sub get { return $self->{_configs}->[$ix]; } - # # Modify one configuration value # -sub set { +sub set +{ my $self = shift; my %args = @_; - my $c = $args{conf}; - my $v = $args{val}; + my $c = $args{conf}; + my $v = $args{val}; $c = $args{config} unless defined($c); - $v = $args{value} unless defined($v); + $v = $args{value} unless defined($v); my $ix = $self->{_lookup}->{$c}; if (!defined($ix)) { - my $type = $args{type}; + my $type = $args{type}; - $type = "INT" unless defined($type); - $self->append(line => "CONFIG $c $type $v"); + $type = "INT" unless defined($type); + $self->append(line => "CONFIG $c $type $v"); } else { my $val = $self->{_configs}->[$ix]; @@ -132,14 +130,14 @@ sub set { } } - # # Remove a configuration from the file. # -sub remove { +sub remove +{ my $self = shift; my %args = @_; - my $c = $args{conf}; + my $c = $args{conf}; $c = $args{config} unless defined($c); @@ -148,11 +146,11 @@ sub remove { $self->{_configs}->[$ix]->[2] = TS_CONF_REMOVED if defined($ix); } - # # Append anything to the "end" of the configuration. # -sub append { +sub append +{ my $self = shift; my %args = @_; my $line = $args{line}; @@ -170,17 +168,17 @@ sub append { $self->{_lookup}->{$p[1]} = $self->{_ix} if ($#p == 3) && (($p[0] eq "LOCAL") || ($p[0] eq "CONFIG")); } - # # Write the new configuration file to STDOUT, or provided # -sub write { +sub write +{ my $self = shift; my %args = @_; - my $fn = $args{file}; + my $fn = $args{file}; $fn = $args{filename} unless defined($fn); - $fn = "-" unless defined($fn); + $fn = "-" unless defined($fn); if ($fn ne "-") { close(STDOUT); diff --git a/lib/records/I_RecCore.h b/lib/records/I_RecCore.h index e3db5ea1920..188b33192da 100644 --- a/lib/records/I_RecCore.h +++ b/lib/records/I_RecCore.h @@ -23,13 +23,15 @@ #pragma once +#include + #include "tscore/Diags.h" #include "I_RecDefs.h" #include "I_RecAlarms.h" #include "I_RecSignals.h" #include "I_RecEvents.h" -#include +#include "tscpp/util/MemSpan.h" struct RecRecord; @@ -44,7 +46,7 @@ int RecSetDiags(Diags *diags); typedef void (*RecConfigEntryCallback)(RecT rec_type, RecDataT data_type, const char *name, const char *value, RecSourceT source, bool inc_version); -void RecConfigFileInit(void); +void RecConfigFileInit(); int RecConfigFileParse(const char *path, RecConfigEntryCallback handler, bool inc_version); // Return a copy of the system's configuration directory. @@ -146,37 +148,36 @@ RecErrT RecSetRecordString(const char *name, const RecString rec_string, RecSour bool inc_version = true); RecErrT RecSetRecordCounter(const char *name, RecCounter rec_counter, RecSourceT source, bool lock = true, bool inc_version = true); -int RecGetRecordInt(const char *name, RecInt *rec_int, bool lock = true); -int RecGetRecordFloat(const char *name, RecFloat *rec_float, bool lock = true); -int RecGetRecordString(const char *name, char *buf, int buf_len, bool lock = true); -int RecGetRecordString_Xmalloc(const char *name, RecString *rec_string, bool lock = true); -int RecGetRecordCounter(const char *name, RecCounter *rec_counter, bool lock = true); +RecErrT RecGetRecordInt(const char *name, RecInt *rec_int, bool lock = true); +RecErrT RecGetRecordFloat(const char *name, RecFloat *rec_float, bool lock = true); +RecErrT RecGetRecordString(const char *name, char *buf, int buf_len, bool lock = true); +RecErrT RecGetRecordString_Xmalloc(const char *name, RecString *rec_string, bool lock = true); +RecErrT RecGetRecordCounter(const char *name, RecCounter *rec_counter, bool lock = true); // Convenience to allow us to treat the RecInt as a single byte internally -int RecGetRecordByte(const char *name, RecByte *rec_byte, bool lock = true); +RecErrT RecGetRecordByte(const char *name, RecByte *rec_byte, bool lock = true); // Convenience to allow us to treat the RecInt as a bool internally -int RecGetRecordBool(const char *name, RecBool *rec_byte, bool lock = true); +RecErrT RecGetRecordBool(const char *name, RecBool *rec_byte, bool lock = true); //------------------------------------------------------------------------ // Record Attributes Reading //------------------------------------------------------------------------ typedef void (*RecLookupCallback)(const RecRecord *, void *); -int RecLookupRecord(const char *name, RecLookupCallback callback, void *data, bool lock = true); -int RecLookupMatchingRecords(unsigned rec_type, const char *match, RecLookupCallback callback, void *data, bool lock = true); - -int RecGetRecordType(const char *name, RecT *rec_type, bool lock = true); -int RecGetRecordDataType(const char *name, RecDataT *data_type, bool lock = true); -int RecGetRecordPersistenceType(const char *name, RecPersistT *persist_type, bool lock = true); -int RecGetRecordOrderAndId(const char *name, int *order, int *id, bool lock = true); +RecErrT RecLookupRecord(const char *name, RecLookupCallback callback, void *data, bool lock = true); +RecErrT RecLookupMatchingRecords(unsigned rec_type, const char *match, RecLookupCallback callback, void *data, bool lock = true); -int RecGetRecordUpdateType(const char *name, RecUpdateT *update_type, bool lock = true); -int RecGetRecordCheckType(const char *name, RecCheckT *check_type, bool lock = true); -int RecGetRecordCheckExpr(const char *name, char **check_expr, bool lock = true); -int RecGetRecordDefaultDataString_Xmalloc(char *name, char **buf, bool lock = true); -int RecGetRecordSource(const char *name, RecSourceT *source, bool lock = true); +RecErrT RecGetRecordType(const char *name, RecT *rec_type, bool lock = true); +RecErrT RecGetRecordDataType(const char *name, RecDataT *data_type, bool lock = true); +RecErrT RecGetRecordPersistenceType(const char *name, RecPersistT *persist_type, bool lock = true); +RecErrT RecGetRecordOrderAndId(const char *name, int *order, int *id, bool lock = true); +RecErrT RecGetRecordUpdateType(const char *name, RecUpdateT *update_type, bool lock = true); +RecErrT RecGetRecordCheckType(const char *name, RecCheckT *check_type, bool lock = true); +RecErrT RecGetRecordCheckExpr(const char *name, char **check_expr, bool lock = true); +RecErrT RecGetRecordDefaultDataString_Xmalloc(char *name, char **buf, bool lock = true); +RecErrT RecGetRecordSource(const char *name, RecSourceT *source, bool lock = true); -int RecGetRecordAccessType(const char *name, RecAccessT *secure, bool lock = true); -int RecSetRecordAccessType(const char *name, RecAccessT secure, bool lock = true); +RecErrT RecGetRecordAccessType(const char *name, RecAccessT *secure, bool lock = true); +RecErrT RecSetRecordAccessType(const char *name, RecAccessT secure, bool lock = true); //------------------------------------------------------------------------ // Signal and Alarms @@ -306,5 +307,5 @@ RecErrT RecSetSyncRequired(char *name, bool lock = true); //------------------------------------------------------------------------ // Manager Callback //------------------------------------------------------------------------ -typedef void *(*RecManagerCb)(void *opaque_cb_data, char *data_raw, int data_len); -int RecRegisterManagerCb(int _signal, RecManagerCb _fn, void *_data = nullptr); +using RecManagerCb = std::function; +int RecRegisterManagerCb(int _signal, RecManagerCb const &_fn); diff --git a/lib/records/I_RecHttp.h b/lib/records/I_RecHttp.h index 7e831fc590f..fda517614b8 100644 --- a/lib/records/I_RecHttp.h +++ b/lib/records/I_RecHttp.h @@ -28,8 +28,10 @@ #include "ts/apidefs.h" #include "ts/apidefs.h" #include "tscore/ink_assert.h" +#include "tscore/IpMap.h" +#include "tscore/MemArena.h" #include -#include +#include /// Load default inbound IP addresses from the configuration file. void RecHttpLoadIp(const char *name, ///< Name of value in configuration file. @@ -37,23 +39,27 @@ void RecHttpLoadIp(const char *name, ///< Name of value in configuration file. IpAddr &ip6 ///< [out] Ipv6 address. ); +/// Load up an IpMap with IP addresses from the configuration file. +void RecHttpLoadIpMap(const char *name, ///< Name of value in configuration file. + IpMap &ipmap ///< [out] IpMap. +); + /** A set of session protocols. This depends on using @c SessionProtocolNameRegistry to get the indices. */ class SessionProtocolSet { - typedef SessionProtocolSet self; ///< Self reference type. + using self_type = SessionProtocolSet; ///< Self reference type. /// Storage for the set - a bit vector. - uint32_t m_bits; + uint32_t m_bits = 0; public: - // The right way. - // static int const MAX = sizeof(m_bits) * CHAR_BIT; - // The RHEL5/gcc 4.1.2 way - static int const MAX = sizeof(uint32_t) * 8; + static constexpr int MAX = sizeof(m_bits) * CHAR_BIT; + /// Default constructor. /// Constructs and empty set. - SessionProtocolSet() : m_bits(0) {} + SessionProtocolSet() = default; + uint32_t indexToMask(int idx) const { @@ -66,9 +72,10 @@ class SessionProtocolSet { m_bits |= this->indexToMask(idx); } + /// Mark all the protocols in @a that as present in @a this. void - markIn(self const &that) + markIn(self_type const &that) { m_bits |= that.m_bits; } @@ -80,7 +87,7 @@ class SessionProtocolSet } /// Mark the protocols in @a that as not in @a this. void - markOut(self const &that) + markOut(self_type const &that) { m_bits &= ~(that.m_bits); } @@ -92,7 +99,7 @@ class SessionProtocolSet } /// Test if all the protocols in @a that are in @a this protocol set. bool - contains(self const &that) const + contains(self_type const &that) const { return that.m_bits == (that.m_bits & m_bits); } @@ -111,7 +118,7 @@ class SessionProtocolSet /// Check for intersection. bool - intersects(self const &that) + intersects(self_type const &that) { return 0 != (m_bits & that.m_bits); } @@ -125,7 +132,7 @@ class SessionProtocolSet /// Equality (identical sets). bool - operator==(self const &that) const + operator==(self_type const &that) const { return m_bits == that.m_bits; } @@ -141,52 +148,51 @@ const char *RecNormalizeProtoTag(const char *tag); /** Registered session protocol names. - We do this to avoid lots of string compares. By normalizing the - string names we can just compare their indices in this table. + We do this to avoid lots of string compares. By normalizing the string names we can just compare + their indices in this table. - @internal To simplify the implementation we limit the maximum - number of strings to 32. That will be sufficient for the forseeable - future. We can come back to this if it ever becomes a problem. + @internal To simplify the implementation we limit the maximum number of strings to 32. That will + be sufficient for the forseeable future. We can come back to this if it ever becomes a problem. - @internal Because we have so few strings we just use a linear search. - If the size gets much larger we should consider doing something more - clever. + @internal Because we have so few strings we just use a linear search. If the size gets much + larger we should consider doing something more clever. + + @internal This supports providing constant strings because those strings are exported to the + C API and this logic @b must return exactly those pointers. */ class SessionProtocolNameRegistry { public: - static int const MAX = SessionProtocolSet::MAX; ///< Maximum # of registered names. - static int const INVALID = -1; ///< Normalized invalid index value. + static int constexpr MAX = SessionProtocolSet::MAX; ///< Maximum # of registered names. + static int constexpr INVALID = -1; ///< Normalized invalid index value. + + using TextView = ts::TextView; /// Default constructor. /// Creates empty registry with no names. - SessionProtocolNameRegistry(); - - /// Destructor. - /// Cleans up strings. - ~SessionProtocolNameRegistry(); + SessionProtocolNameRegistry() = default; /** Get the index for @a name, registering it if needed. The name is copied internally. @return The index for the registered @a name. */ - int toIndex(const char *name); + int toIndex(TextView name); /** Get the index for @a name, registering it if needed. The caller @b guarantees @a name is persistent and immutable. @return The index for the registered @a name. */ - int toIndexConst(const char *name); + int toIndexConst(TextView name); /** Convert a @a name to an index. @return The index for @a name or @c INVALID if it is not registered. */ - int indexFor(const char *name) const; + int indexFor(TextView name) const; /** Convert an @a index to the corresponding name. @return A pointer to the name or @c nullptr if the index isn't registered. */ - const char *nameFor(int index) const; + TextView nameFor(int index) const; /// Mark protocols as present in @a sp_set based on the names in @a value. /// The names can be separated by ;/|,: and space. @@ -195,11 +201,9 @@ class SessionProtocolNameRegistry void markIn(const char *value, SessionProtocolSet &sp_set); protected: - unsigned int m_n; ///< Index of first unused slot. - const char *m_names[MAX]; ///< Pointers to registered names. - uint8_t m_flags[MAX]; ///< Flags for each name. - - static uint8_t const F_ALLOCATED = 0x1; ///< Flag for allocated by this instance. + int m_n = 0; ///< Index of first unused slot. + std::array m_names; + ts::MemArena m_arena; ///< Storage for non-constant strings. }; extern SessionProtocolNameRegistry globalSessionProtocolNameRegistry; @@ -235,18 +239,20 @@ struct HttpProxyPort { TRANSPORT_PLUGIN /// < Protocol plugin connection }; - int m_fd; ///< Pre-opened file descriptor if present. - TransportType m_type; ///< Type of connection. - in_port_t m_port; ///< Port on which to listen. - uint8_t m_family; ///< IP address family. + int m_fd; ///< Pre-opened file descriptor if present. + TransportType m_type = TRANSPORT_DEFAULT; ///< Type of connection. + in_port_t m_port = 0; ///< Port on which to listen. + uint8_t m_family = AF_INET; ///< IP address family. + /// True if proxy protocol is required on incoming requests. + bool m_proxy_protocol = false; /// True if inbound connects (from client) are transparent. - bool m_inbound_transparent_p; + bool m_inbound_transparent_p = false; /// True if outbound connections (to origin servers) are transparent. - bool m_outbound_transparent_p; + bool m_outbound_transparent_p = false; // True if transparent pass-through is enabled on this port. - bool m_transparent_passthrough; + bool m_transparent_passthrough = false; /// True if MPTCP is enabled on this port. - bool m_mptcp; + bool m_mptcp = false; /// Local address for inbound connections (listen address). IpAddr m_inbound_ip; /// Local address for outbound connections (to origin server). @@ -392,6 +398,7 @@ struct HttpProxyPort { static const char *const OPT_TRANSPARENT_FULL; ///< Full transparency. static const char *const OPT_TRANSPARENT_PASSTHROUGH; ///< Pass-through non-HTTP. static const char *const OPT_SSL; ///< SSL (experimental) + static const char *const OPT_PROXY_PROTO; ///< Proxy Protocol static const char *const OPT_PLUGIN; ///< Protocol Plugin handle (experimental) static const char *const OPT_BLIND_TUNNEL; ///< Blind tunnel. static const char *const OPT_COMPRESSED; ///< Compressed. diff --git a/lib/records/I_RecProcess.h b/lib/records/I_RecProcess.h index eb7b98b51e6..c7a6faa2c89 100644 --- a/lib/records/I_RecProcess.h +++ b/lib/records/I_RecProcess.h @@ -31,7 +31,7 @@ //------------------------------------------------------------------------- int RecProcessInit(RecModeT mode_type, Diags *diags = nullptr); int RecProcessInitMessage(RecModeT mode_type); -int RecProcessStart(void); +int RecProcessStart(); //------------------------------------------------------------------------- // Setters for manipulating internal sleep intervals diff --git a/lib/records/Makefile.am b/lib/records/Makefile.am index 195d9089a30..e403a81313d 100644 --- a/lib/records/Makefile.am +++ b/lib/records/Makefile.am @@ -18,6 +18,8 @@ include $(top_srcdir)/build/tidy.mk +check_PROGRAMS = test_librecords + AM_CPPFLAGS += \ -I$(abs_top_srcdir)/iocore/eventsystem \ -I$(abs_top_srcdir)/iocore/utils \ @@ -67,5 +69,24 @@ librecords_p_a_SOURCES = \ P_RecProcess.h \ RecProcess.cc +TESTS = $(check_PROGRAMS) + +test_librecords_CPPFLAGS = $(AM_CPPFLAGS)\ + -I$(abs_top_srcdir)/tests/include + +test_librecords_SOURCES = \ + unit_tests/unit_test_main.cc \ + unit_tests/test_RecHttp.cc + +test_librecords_LDADD = \ + $(top_builddir)/lib/records/librecords_p.a \ + $(top_builddir)/mgmt/libmgmt_p.la \ + $(top_builddir)/iocore/eventsystem/libinkevent.a \ + $(top_builddir)/src/tscpp/util/libtscpputil.la \ + $(top_builddir)/src/tscore/libtscore.la \ + $(top_builddir)/proxy/shared/libUglyLogStubs.a \ + @HWLOC_LIBS@ @LIBCAP@ + + clang-tidy-local: $(sort $(DIST_SOURCES)) $(CXX_Clang_Tidy) diff --git a/lib/records/P_RecCore.cc b/lib/records/P_RecCore.cc index 4c83b8acb21..e89ccb9771e 100644 --- a/lib/records/P_RecCore.cc +++ b/lib/records/P_RecCore.cc @@ -641,7 +641,7 @@ RecReadConfigFile(bool inc_version) // lock our hash table ink_rwlock_wrlock(&g_records_rwlock); - // Parse the actual fileand hash the values. + // Parse the actual file and hash the values. RecConfigFileParse(g_rec_config_fpath, RecConsumeConfigEntry, inc_version); // release our hash table @@ -920,7 +920,7 @@ RecSetSyncRequired(char *name, bool lock) r1 = it->second; if (i_am_the_record_owner(r1->rec_type)) { rec_mutex_acquire(&(r1->lock)); - r1->sync_required = REC_SYNC_REQUIRED; + r1->sync_required = REC_PEER_SYNC_REQUIRED; if (REC_TYPE_IS_CONFIG(r1->rec_type)) { r1->config_meta.update_required = REC_UPDATE_REQUIRED; } diff --git a/lib/records/P_RecMessage.h b/lib/records/P_RecMessage.h index 24db24cb4c8..d41f74c5ec4 100644 --- a/lib/records/P_RecMessage.h +++ b/lib/records/P_RecMessage.h @@ -24,6 +24,7 @@ #pragma once #include "P_RecDefs.h" +#include "tscpp/util/MemSpan.h" //------------------------------------------------------------------------- // Initialization @@ -44,7 +45,7 @@ int RecMessageUnmarshalNext(RecMessage *msg, RecMessageItr *itr, RecRecord **rec int RecMessageSend(RecMessage *msg); int RecMessageRegisterRecvCb(RecMessageRecvCb recv_cb, void *cookie); -void *RecMessageRecvThis(void *cookie, char *data_raw, int data_len); +void RecMessageRecvThis(ts::MemSpan); RecMessage *RecMessageReadFromDisk(const char *fpath); int RecMessageWriteToDisk(RecMessage *msg, const char *fpath); diff --git a/lib/records/RecCore.cc b/lib/records/RecCore.cc index f264a57f352..68dd6fa4f7e 100644 --- a/lib/records/RecCore.cc +++ b/lib/records/RecCore.cc @@ -314,6 +314,7 @@ RecRegisterConfigUpdateCb(const char *name, RecConfigUpdateCb update_cb, void *c if (auto it = g_records_ht.find(name); it != g_records_ht.end()) { RecRecord *r = it->second; + rec_mutex_acquire(&(r->lock)); if (REC_TYPE_IS_CONFIG(r->rec_type)) { /* -- upgrade to support a list of callback functions @@ -358,37 +359,41 @@ RecRegisterConfigUpdateCb(const char *name, RecConfigUpdateCb update_cb, void *c //------------------------------------------------------------------------- // RecGetRecordXXX //------------------------------------------------------------------------- -int +RecErrT RecGetRecordInt(const char *name, RecInt *rec_int, bool lock) { - int err; + RecErrT err; RecData data; + if ((err = RecGetRecord_Xmalloc(name, RECD_INT, &data, lock)) == REC_ERR_OKAY) { *rec_int = data.rec_int; } return err; } -int +RecErrT RecGetRecordFloat(const char *name, RecFloat *rec_float, bool lock) { - int err; + RecErrT err; RecData data; + if ((err = RecGetRecord_Xmalloc(name, RECD_FLOAT, &data, lock)) == REC_ERR_OKAY) { *rec_float = data.rec_float; } return err; } -int +RecErrT RecGetRecordString(const char *name, char *buf, int buf_len, bool lock) { - int err = REC_ERR_OKAY; + RecErrT err = REC_ERR_OKAY; + if (lock) { ink_rwlock_rdlock(&g_records_rwlock); } if (auto it = g_records_ht.find(name); it != g_records_ht.end()) { RecRecord *r = it->second; + rec_mutex_acquire(&(r->lock)); if (!r->registered || (r->data_type != RECD_STRING)) { err = REC_ERR_FAIL; @@ -409,44 +414,48 @@ RecGetRecordString(const char *name, char *buf, int buf_len, bool lock) return err; } -int +RecErrT RecGetRecordString_Xmalloc(const char *name, RecString *rec_string, bool lock) { - int err; + RecErrT err; RecData data; + if ((err = RecGetRecord_Xmalloc(name, RECD_STRING, &data, lock)) == REC_ERR_OKAY) { *rec_string = data.rec_string; } return err; } -int +RecErrT RecGetRecordCounter(const char *name, RecCounter *rec_counter, bool lock) { - int err; + RecErrT err; RecData data; + if ((err = RecGetRecord_Xmalloc(name, RECD_COUNTER, &data, lock)) == REC_ERR_OKAY) { *rec_counter = data.rec_counter; } return err; } -int +RecErrT RecGetRecordByte(const char *name, RecByte *rec_byte, bool lock) { - int err; + RecErrT err; RecData data; + if ((err = RecGetRecord_Xmalloc(name, RECD_INT, &data, lock)) == REC_ERR_OKAY) { *rec_byte = data.rec_int; } return err; } -int +RecErrT RecGetRecordBool(const char *name, RecBool *rec_bool, bool lock) { - int err; + RecErrT err; RecData data; + if ((err = RecGetRecord_Xmalloc(name, RECD_INT, &data, lock)) == REC_ERR_OKAY) { *rec_bool = 0 != data.rec_int; } @@ -457,10 +466,10 @@ RecGetRecordBool(const char *name, RecBool *rec_bool, bool lock) // RecGetRec Attributes //------------------------------------------------------------------------- -int +RecErrT RecLookupRecord(const char *name, void (*callback)(const RecRecord *, void *), void *data, bool lock) { - int err = REC_ERR_FAIL; + RecErrT err = REC_ERR_FAIL; if (lock) { ink_rwlock_rdlock(&g_records_rwlock); @@ -468,6 +477,7 @@ RecLookupRecord(const char *name, void (*callback)(const RecRecord *, void *), v if (auto it = g_records_ht.find(name); it != g_records_ht.end()) { RecRecord *r = it->second; + rec_mutex_acquire(&(r->lock)); callback(r, data); err = REC_ERR_OKAY; @@ -481,13 +491,13 @@ RecLookupRecord(const char *name, void (*callback)(const RecRecord *, void *), v return err; } -int +RecErrT RecLookupMatchingRecords(unsigned rec_type, const char *match, void (*callback)(const RecRecord *, void *), void *data, bool lock) { int num_records; DFA regex; - if (regex.compile(match, RE_CASE_INSENSITIVE | RE_UNANCHORED) != 0) { + if (!regex.compile(match, RE_CASE_INSENSITIVE | RE_UNANCHORED)) { return REC_ERR_FAIL; } @@ -511,10 +521,10 @@ RecLookupMatchingRecords(unsigned rec_type, const char *match, void (*callback)( return REC_ERR_OKAY; } -int +RecErrT RecGetRecordType(const char *name, RecT *rec_type, bool lock) { - int err = REC_ERR_FAIL; + RecErrT err = REC_ERR_FAIL; if (lock) { ink_rwlock_rdlock(&g_records_rwlock); @@ -522,6 +532,7 @@ RecGetRecordType(const char *name, RecT *rec_type, bool lock) if (auto it = g_records_ht.find(name); it != g_records_ht.end()) { RecRecord *r = it->second; + rec_mutex_acquire(&(r->lock)); *rec_type = r->rec_type; err = REC_ERR_OKAY; @@ -535,10 +546,10 @@ RecGetRecordType(const char *name, RecT *rec_type, bool lock) return err; } -int +RecErrT RecGetRecordDataType(const char *name, RecDataT *data_type, bool lock) { - int err = REC_ERR_FAIL; + RecErrT err = REC_ERR_FAIL; if (lock) { ink_rwlock_rdlock(&g_records_rwlock); @@ -546,6 +557,7 @@ RecGetRecordDataType(const char *name, RecDataT *data_type, bool lock) if (auto it = g_records_ht.find(name); it != g_records_ht.end()) { RecRecord *r = it->second; + rec_mutex_acquire(&(r->lock)); if (!r->registered) { err = REC_ERR_FAIL; @@ -563,10 +575,10 @@ RecGetRecordDataType(const char *name, RecDataT *data_type, bool lock) return err; } -int +RecErrT RecGetRecordPersistenceType(const char *name, RecPersistT *persist_type, bool lock) { - int err = REC_ERR_FAIL; + RecErrT err = REC_ERR_FAIL; if (lock) { ink_rwlock_rdlock(&g_records_rwlock); @@ -576,6 +588,7 @@ RecGetRecordPersistenceType(const char *name, RecPersistT *persist_type, bool lo if (auto it = g_records_ht.find(name); it != g_records_ht.end()) { RecRecord *r = it->second; + rec_mutex_acquire(&(r->lock)); if (REC_TYPE_IS_STAT(r->rec_type)) { *persist_type = r->stat_meta.persist_type; @@ -591,10 +604,10 @@ RecGetRecordPersistenceType(const char *name, RecPersistT *persist_type, bool lo return err; } -int +RecErrT RecGetRecordOrderAndId(const char *name, int *order, int *id, bool lock) { - int err = REC_ERR_FAIL; + RecErrT err = REC_ERR_FAIL; if (lock) { ink_rwlock_rdlock(&g_records_rwlock); @@ -602,6 +615,7 @@ RecGetRecordOrderAndId(const char *name, int *order, int *id, bool lock) if (auto it = g_records_ht.find(name); it != g_records_ht.end()) { RecRecord *r = it->second; + if (r->registered) { rec_mutex_acquire(&(r->lock)); if (order) { @@ -622,10 +636,10 @@ RecGetRecordOrderAndId(const char *name, int *order, int *id, bool lock) return err; } -int +RecErrT RecGetRecordUpdateType(const char *name, RecUpdateT *update_type, bool lock) { - int err = REC_ERR_FAIL; + RecErrT err = REC_ERR_FAIL; if (lock) { ink_rwlock_rdlock(&g_records_rwlock); @@ -633,6 +647,7 @@ RecGetRecordUpdateType(const char *name, RecUpdateT *update_type, bool lock) if (auto it = g_records_ht.find(name); it != g_records_ht.end()) { RecRecord *r = it->second; + rec_mutex_acquire(&(r->lock)); if (REC_TYPE_IS_CONFIG(r->rec_type)) { *update_type = r->config_meta.update_type; @@ -650,10 +665,10 @@ RecGetRecordUpdateType(const char *name, RecUpdateT *update_type, bool lock) return err; } -int +RecErrT RecGetRecordCheckType(const char *name, RecCheckT *check_type, bool lock) { - int err = REC_ERR_FAIL; + RecErrT err = REC_ERR_FAIL; if (lock) { ink_rwlock_rdlock(&g_records_rwlock); @@ -661,6 +676,7 @@ RecGetRecordCheckType(const char *name, RecCheckT *check_type, bool lock) if (auto it = g_records_ht.find(name); it != g_records_ht.end()) { RecRecord *r = it->second; + rec_mutex_acquire(&(r->lock)); if (REC_TYPE_IS_CONFIG(r->rec_type)) { *check_type = r->config_meta.check_type; @@ -678,10 +694,10 @@ RecGetRecordCheckType(const char *name, RecCheckT *check_type, bool lock) return err; } -int +RecErrT RecGetRecordCheckExpr(const char *name, char **check_expr, bool lock) { - int err = REC_ERR_FAIL; + RecErrT err = REC_ERR_FAIL; if (lock) { ink_rwlock_rdlock(&g_records_rwlock); @@ -689,6 +705,7 @@ RecGetRecordCheckExpr(const char *name, char **check_expr, bool lock) if (auto it = g_records_ht.find(name); it != g_records_ht.end()) { RecRecord *r = it->second; + rec_mutex_acquire(&(r->lock)); if (REC_TYPE_IS_CONFIG(r->rec_type)) { *check_expr = r->config_meta.check_expr; @@ -706,10 +723,10 @@ RecGetRecordCheckExpr(const char *name, char **check_expr, bool lock) return err; } -int +RecErrT RecGetRecordDefaultDataString_Xmalloc(char *name, char **buf, bool lock) { - int err; + RecErrT err; if (lock) { ink_rwlock_rdlock(&g_records_rwlock); @@ -717,7 +734,8 @@ RecGetRecordDefaultDataString_Xmalloc(char *name, char **buf, bool lock) if (auto it = g_records_ht.find(name); it != g_records_ht.end()) { RecRecord *r = it->second; - *buf = (char *)ats_malloc(sizeof(char) * 1024); + + *buf = (char *)ats_malloc(sizeof(char) * 1024); memset(*buf, 0, 1024); err = REC_ERR_OKAY; @@ -756,10 +774,10 @@ RecGetRecordDefaultDataString_Xmalloc(char *name, char **buf, bool lock) return err; } -int +RecErrT RecGetRecordAccessType(const char *name, RecAccessT *access, bool lock) { - int err = REC_ERR_FAIL; + RecErrT err = REC_ERR_FAIL; if (lock) { ink_rwlock_rdlock(&g_records_rwlock); @@ -767,6 +785,7 @@ RecGetRecordAccessType(const char *name, RecAccessT *access, bool lock) if (auto it = g_records_ht.find(name); it != g_records_ht.end()) { RecRecord *r = it->second; + rec_mutex_acquire(&(r->lock)); *access = r->config_meta.access_type; err = REC_ERR_OKAY; @@ -780,10 +799,10 @@ RecGetRecordAccessType(const char *name, RecAccessT *access, bool lock) return err; } -int +RecErrT RecSetRecordAccessType(const char *name, RecAccessT access, bool lock) { - int err = REC_ERR_FAIL; + RecErrT err = REC_ERR_FAIL; if (lock) { ink_rwlock_rdlock(&g_records_rwlock); @@ -791,6 +810,7 @@ RecSetRecordAccessType(const char *name, RecAccessT access, bool lock) if (auto it = g_records_ht.find(name); it != g_records_ht.end()) { RecRecord *r = it->second; + rec_mutex_acquire(&(r->lock)); r->config_meta.access_type = access; err = REC_ERR_OKAY; @@ -804,10 +824,10 @@ RecSetRecordAccessType(const char *name, RecAccessT access, bool lock) return err; } -int +RecErrT RecGetRecordSource(const char *name, RecSourceT *source, bool lock) { - int err = REC_ERR_FAIL; + RecErrT err = REC_ERR_FAIL; if (lock) { ink_rwlock_rdlock(&g_records_rwlock); @@ -815,6 +835,7 @@ RecGetRecordSource(const char *name, RecSourceT *source, bool lock) if (auto it = g_records_ht.find(name); it != g_records_ht.end()) { RecRecord *r = it->second; + rec_mutex_acquire(&(r->lock)); *source = r->config_meta.source; err = REC_ERR_OKAY; @@ -902,6 +923,7 @@ RecGetRecord_Xmalloc(const char *name, RecDataT data_type, RecData *data, bool l if (auto it = g_records_ht.find(name); it != g_records_ht.end()) { RecRecord *r = it->second; + rec_mutex_acquire(&(r->lock)); if (!r->registered || (r->data_type != data_type)) { err = REC_ERR_FAIL; @@ -1071,8 +1093,8 @@ REC_readInteger(const char *name, bool *found, bool lock) { ink_assert(name); RecInt _tmp = 0; - bool _found; - _found = (RecGetRecordInt(name, &_tmp, lock) == REC_ERR_OKAY); + bool _found = (RecGetRecordInt(name, &_tmp, lock) == REC_ERR_OKAY); + if (found) { *found = _found; } @@ -1084,8 +1106,8 @@ REC_readFloat(char *name, bool *found, bool lock) { ink_assert(name); RecFloat _tmp = 0.0; - bool _found; - _found = (RecGetRecordFloat(name, &_tmp, lock) == REC_ERR_OKAY); + bool _found = (RecGetRecordFloat(name, &_tmp, lock) == REC_ERR_OKAY); + if (found) { *found = _found; } @@ -1097,8 +1119,8 @@ REC_readCounter(char *name, bool *found, bool lock) { ink_assert(name); RecCounter _tmp = 0; - bool _found; - _found = (RecGetRecordCounter(name, &_tmp, lock) == REC_ERR_OKAY); + bool _found = (RecGetRecordCounter(name, &_tmp, lock) == REC_ERR_OKAY); + if (found) { *found = _found; } @@ -1110,8 +1132,8 @@ REC_readString(const char *name, bool *found, bool lock) { ink_assert(name); RecString _tmp = nullptr; - bool _found; - _found = (RecGetRecordString_Xmalloc(name, &_tmp, lock) == REC_ERR_OKAY); + bool _found = (RecGetRecordString_Xmalloc(name, &_tmp, lock) == REC_ERR_OKAY); + if (found) { *found = _found; } diff --git a/lib/records/RecHttp.cc b/lib/records/RecHttp.cc index 85c7a4ae89b..5f5ddb79aff 100644 --- a/lib/records/RecHttp.cc +++ b/lib/records/RecHttp.cc @@ -30,6 +30,7 @@ #include "tscore/ink_inet.h" #include #include +#include SessionProtocolNameRegistry globalSessionProtocolNameRegistry; @@ -123,6 +124,30 @@ RecHttpLoadIp(const char *value_name, IpAddr &ip4, IpAddr &ip6) } } +void +RecHttpLoadIpMap(const char *value_name, IpMap &ipmap) +{ + char value[1024]; + IpAddr laddr; + IpAddr raddr; + void *payload = nullptr; + + if (REC_ERR_OKAY == RecGetRecordString(value_name, value, sizeof(value))) { + Debug("config", "RecHttpLoadIpMap: parsing the name [%s] and value [%s] to an IpMap", value_name, value); + Tokenizer tokens(", "); + int n_addrs = tokens.Initialize(value); + for (int i = 0; i < n_addrs; ++i) { + const char *val = tokens[i]; + + Debug("config", "RecHttpLoadIpMap: marking the value [%s] to an IpMap entry", val); + if (0 == ats_ip_range_parse(val, laddr, raddr)) { + ipmap.fill(laddr, raddr, payload); + } + } + } + Debug("config", "RecHttpLoadIpMap: parsed %zu IpMap entries", ipmap.count()); +} + const char *const HttpProxyPort::DEFAULT_VALUE = "8080"; const char *const HttpProxyPort::PORTS_CONFIG_NAME = "proxy.config.http.server_ports"; @@ -144,6 +169,7 @@ const char *const HttpProxyPort::OPT_TRANSPARENT_OUTBOUND = "tr-out"; const char *const HttpProxyPort::OPT_TRANSPARENT_FULL = "tr-full"; const char *const HttpProxyPort::OPT_TRANSPARENT_PASSTHROUGH = "tr-pass"; const char *const HttpProxyPort::OPT_SSL = "ssl"; +const char *const HttpProxyPort::OPT_PROXY_PROTO = "pp"; const char *const HttpProxyPort::OPT_PLUGIN = "plugin"; const char *const HttpProxyPort::OPT_BLIND_TUNNEL = "blind"; const char *const HttpProxyPort::OPT_COMPRESSED = "compressed"; @@ -171,15 +197,8 @@ HttpProxyPort::Group GLOBAL_DATA; } // namespace HttpProxyPort::Group &HttpProxyPort::m_global = GLOBAL_DATA; -HttpProxyPort::HttpProxyPort() - : m_fd(ts::NO_FD), - m_type(TRANSPORT_DEFAULT), - m_port(0), - m_family(AF_INET), - m_inbound_transparent_p(false), - m_outbound_transparent_p(false), - m_transparent_passthrough(false), - m_mptcp(false) +HttpProxyPort::HttpProxyPort() : m_fd(ts::NO_FD) + { memcpy(m_host_res_preference, host_res_default_preference_order, sizeof(m_host_res_preference)); } @@ -358,6 +377,8 @@ HttpProxyPort::processOptions(const char *opts) m_type = TRANSPORT_SSL; } else if (0 == strcasecmp(OPT_PLUGIN, item)) { m_type = TRANSPORT_PLUGIN; + } else if (0 == strcasecmp(OPT_PROXY_PROTO, item)) { + m_proxy_protocol = true; } else if (0 == strcasecmp(OPT_TRANSPARENT_INBOUND, item)) { #if TS_USE_TPROXY m_inbound_transparent_p = true; @@ -406,7 +427,7 @@ HttpProxyPort::processOptions(const char *opts) if (in_ip_set_p && m_family != m_inbound_ip.family()) { std::string_view iname{ats_ip_family_name(m_inbound_ip.family())}; std::string_view fname{ats_ip_family_name(m_family)}; - Warning("Invalid port descriptor '%s' - the inbound adddress family [%.*s] is not the same type as the explicit family value " + Warning("Invalid port descriptor '%s' - the inbound address family [%.*s] is not the same type as the explicit family value " "[%.*s]", opts, static_cast(iname.size()), iname.data(), static_cast(fname.size()), fname.data()); zret = false; @@ -471,7 +492,7 @@ SessionProtocolNameRegistry::markIn(const char *value, SessionProtocolSet &sp_se } else if (0 == strcasecmp(elt, TS_ALPN_PROTOCOL_GROUP_HTTP2)) { sp_set.markIn(HTTP2_PROTOCOL_SET); } else { // user defined - register and mark. - int idx = globalSessionProtocolNameRegistry.toIndex(elt); + int idx = globalSessionProtocolNameRegistry.toIndex(TextView{elt, strlen(elt)}); sp_set.markIn(idx); } } @@ -558,6 +579,10 @@ HttpProxyPort::print(char *out, size_t n) return n; } + if (m_proxy_protocol) { + zret += snprintf(out + zret, n - zret, ":%s", OPT_PROXY_PROTO); + } + if (m_outbound_transparent_p && m_inbound_transparent_p) { zret += snprintf(out + zret, n - zret, ":%s", OPT_TRANSPARENT_FULL); } else if (m_inbound_transparent_p) { @@ -618,7 +643,8 @@ HttpProxyPort::print(char *out, size_t n) bool sep_p = !need_colon_p; for (int k = 0; k < SessionProtocolSet::MAX; ++k) { if (sp_set.contains(k)) { - zret += snprintf(out + zret, n - zret, "%s%s", sep_p ? ";" : "", globalSessionProtocolNameRegistry.nameFor(k)); + auto name{globalSessionProtocolNameRegistry.nameFor(k)}; + zret += snprintf(out + zret, n - zret, "%s%.*s", sep_p ? ";" : "", static_cast(name.size()), name.data()); sep_p = true; } } @@ -645,10 +671,10 @@ void ts_session_protocol_well_known_name_indices_init() { // register all the well known protocols and get the indices set. - TS_ALPN_PROTOCOL_INDEX_HTTP_0_9 = globalSessionProtocolNameRegistry.toIndexConst(TS_ALPN_PROTOCOL_HTTP_0_9); - TS_ALPN_PROTOCOL_INDEX_HTTP_1_0 = globalSessionProtocolNameRegistry.toIndexConst(TS_ALPN_PROTOCOL_HTTP_1_0); - TS_ALPN_PROTOCOL_INDEX_HTTP_1_1 = globalSessionProtocolNameRegistry.toIndexConst(TS_ALPN_PROTOCOL_HTTP_1_1); - TS_ALPN_PROTOCOL_INDEX_HTTP_2_0 = globalSessionProtocolNameRegistry.toIndexConst(TS_ALPN_PROTOCOL_HTTP_2_0); + TS_ALPN_PROTOCOL_INDEX_HTTP_0_9 = globalSessionProtocolNameRegistry.toIndexConst(std::string_view{TS_ALPN_PROTOCOL_HTTP_0_9}); + TS_ALPN_PROTOCOL_INDEX_HTTP_1_0 = globalSessionProtocolNameRegistry.toIndexConst(std::string_view{TS_ALPN_PROTOCOL_HTTP_1_0}); + TS_ALPN_PROTOCOL_INDEX_HTTP_1_1 = globalSessionProtocolNameRegistry.toIndexConst(std::string_view{TS_ALPN_PROTOCOL_HTTP_1_1}); + TS_ALPN_PROTOCOL_INDEX_HTTP_2_0 = globalSessionProtocolNameRegistry.toIndexConst(std::string_view{TS_ALPN_PROTOCOL_HTTP_2_0}); // Now do the predefined protocol sets. HTTP_PROTOCOL_SET.markIn(TS_ALPN_PROTOCOL_INDEX_HTTP_0_9); @@ -676,36 +702,22 @@ ts_session_protocol_well_known_name_indices_init() const char * RecNormalizeProtoTag(const char *tag) { - if (TSProtoTags.find(tag) != TSProtoTags.end()) { - return tag; - } - return nullptr; -} - -SessionProtocolNameRegistry::SessionProtocolNameRegistry() : m_n(0) -{ - memset(m_names, 0, sizeof(m_names)); - memset(&m_flags, 0, sizeof(m_flags)); -} - -SessionProtocolNameRegistry::~SessionProtocolNameRegistry() -{ - for (size_t i = 0; i < m_n; ++i) { - if (m_flags[i] & F_ALLOCATED) { - ats_free(const_cast(m_names[i])); // blech - ats_free won't take a const char* - } - } + auto findResult = TSProtoTags.find(tag); + return findResult == TSProtoTags.end() ? nullptr : findResult->data(); } int -SessionProtocolNameRegistry::toIndex(const char *name) +SessionProtocolNameRegistry::toIndex(ts::TextView name) { int zret = this->indexFor(name); if (INVALID == zret) { - if (m_n < static_cast(MAX)) { - m_names[m_n] = ats_strdup(name); - m_flags[m_n] = F_ALLOCATED; - zret = m_n++; + if (m_n < MAX) { + // Localize the name by copying it in to the arena. + auto text = m_arena.alloc(name.size() + 1); + memcpy(text.data(), name.data(), name.size()); + text.end()[-1] = '\0'; + m_names[m_n] = text.view(); + zret = m_n++; } else { ink_release_assert(!"Session protocol name registry overflow"); } @@ -714,11 +726,11 @@ SessionProtocolNameRegistry::toIndex(const char *name) } int -SessionProtocolNameRegistry::toIndexConst(const char *name) +SessionProtocolNameRegistry::toIndexConst(TextView name) { int zret = this->indexFor(name); if (INVALID == zret) { - if (m_n < static_cast(MAX)) { + if (m_n < MAX) { m_names[m_n] = name; zret = m_n++; } else { @@ -729,18 +741,18 @@ SessionProtocolNameRegistry::toIndexConst(const char *name) } int -SessionProtocolNameRegistry::indexFor(const char *name) const +SessionProtocolNameRegistry::indexFor(TextView name) const { - for (size_t i = 0; i < m_n; ++i) { - if (0 == strcasecmp(name, m_names[i])) { - return i; - } + const ts::TextView *end = m_names.begin() + m_n; + auto spot = std::find(m_names.begin(), end, name); + if (spot != end) { + return static_cast(spot - m_names.begin()); } return INVALID; } -const char * +ts::TextView SessionProtocolNameRegistry::nameFor(int idx) const { - return 0 <= idx && idx < static_cast(m_n) ? m_names[idx] : nullptr; + return 0 <= idx && idx < m_n ? m_names[idx] : TextView{}; } diff --git a/lib/records/RecLocal.cc b/lib/records/RecLocal.cc index ad927e526fb..34c721512d4 100644 --- a/lib/records/RecLocal.cc +++ b/lib/records/RecLocal.cc @@ -145,7 +145,7 @@ void RecMessageInit() { ink_assert(g_mode_type != RECM_NULL); - lmgmt->registerMgmtCallback(MGMT_SIGNAL_LIBRECORDS, RecMessageRecvThis, nullptr); + lmgmt->registerMgmtCallback(MGMT_SIGNAL_LIBRECORDS, &RecMessageRecvThis); message_initialized_p = true; } @@ -207,9 +207,9 @@ RecLocalStart(FileManager *configFiles) } int -RecRegisterManagerCb(int id, RecManagerCb _fn, void *_data) +RecRegisterManagerCb(int id, RecManagerCb const &_fn) { - return lmgmt->registerMgmtCallback(id, _fn, _data); + return lmgmt->registerMgmtCallback(id, _fn); } void diff --git a/lib/records/RecMessage.cc b/lib/records/RecMessage.cc index 743649066c4..c18d50253dc 100644 --- a/lib/records/RecMessage.cc +++ b/lib/records/RecMessage.cc @@ -31,6 +31,7 @@ #include "P_RecUtils.h" #include "P_RecCore.h" #include "tscore/I_Layout.h" +#include "tscpp/util/MemSpan.h" static RecMessageRecvCb g_recv_cb = nullptr; static void *g_recv_cookie = nullptr; @@ -243,12 +244,11 @@ RecMessageRegisterRecvCb(RecMessageRecvCb recv_cb, void *cookie) // RecMessageRecvThis //------------------------------------------------------------------------- -void * -RecMessageRecvThis(void * /* cookie */, char *data_raw, int /* data_len */) +void +RecMessageRecvThis(ts::MemSpan span) { - RecMessage *msg = (RecMessage *)data_raw; + RecMessage *msg = static_cast(span.data()); g_recv_cb(msg, msg->msg_type, g_recv_cookie); - return nullptr; } //------------------------------------------------------------------------- diff --git a/lib/records/RecProcess.cc b/lib/records/RecProcess.cc index 08848f46cc3..29fcb07db6c 100644 --- a/lib/records/RecProcess.cc +++ b/lib/records/RecProcess.cc @@ -231,7 +231,7 @@ void RecMessageInit() { ink_assert(g_mode_type != RECM_NULL); - pmgmt->registerMgmtCallback(MGMT_EVENT_LIBRECORDS, RecMessageRecvThis, nullptr); + pmgmt->registerMgmtCallback(MGMT_EVENT_LIBRECORDS, &RecMessageRecvThis); message_initialized_p = true; } @@ -300,9 +300,9 @@ RecSignalManager(int id, const char *msg, size_t msgsize) } int -RecRegisterManagerCb(int _signal, RecManagerCb _fn, void *_data) +RecRegisterManagerCb(int _signal, RecManagerCb const &_fn) { - return pmgmt->registerMgmtCallback(_signal, _fn, _data); + return pmgmt->registerMgmtCallback(_signal, _fn); } //------------------------------------------------------------------------- diff --git a/lib/records/RecRawStats.cc b/lib/records/RecRawStats.cc index dbc2664d677..c83d75cf241 100644 --- a/lib/records/RecRawStats.cc +++ b/lib/records/RecRawStats.cc @@ -535,6 +535,7 @@ RecRegisterRawStatSyncCb(const char *name, RecRawStatSyncCb sync_cb, RecRawStatB ink_rwlock_rdlock(&g_records_rwlock); if (auto it = g_records_ht.find(name); it != g_records_ht.end()) { RecRecord *r = it->second; + rec_mutex_acquire(&(r->lock)); if (REC_TYPE_IS_STAT(r->rec_type)) { if (r->stat_meta.sync_cb) { diff --git a/lib/records/unit_tests/test_Diags.h b/lib/records/unit_tests/test_Diags.h new file mode 100644 index 00000000000..2819c3a32de --- /dev/null +++ b/lib/records/unit_tests/test_Diags.h @@ -0,0 +1,38 @@ +/** @file + + This file used for catch based tests. It is the main() stub. + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. + See the NOTICE file distributed with this work for additional information regarding copyright + ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance with the License. You may obtain a + copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software distributed under the License + is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + or implied. See the License for the specific language governing permissions and limitations under + the License. + */ + +#include "tscore/Diags.h" + +class CatchDiags : public Diags +{ +public: + mutable std::vector messages; + + CatchDiags() : Diags("catch", "", "", nullptr) {} + + void + error_va(DiagsLevel diags_level, const SourceLocation *loc, const char *fmt, va_list ap) const override + { + char buff[32768]; + vsnprintf(buff, sizeof(buff), fmt, ap); + messages.push_back(std::string{buff}); + va_end(ap); + } +}; diff --git a/lib/records/unit_tests/test_RecHttp.cc b/lib/records/unit_tests/test_RecHttp.cc new file mode 100644 index 00000000000..cba93ebd90f --- /dev/null +++ b/lib/records/unit_tests/test_RecHttp.cc @@ -0,0 +1,99 @@ +/** @file + + Catch-based tests for HdrsUtils.cc + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. + See the NOTICE file distributed with this work for additional information regarding copyright + ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance with the License. You may obtain a + copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software distributed under the License + is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + or implied. See the License for the specific language governing permissions and limitations under + the License. + */ + +#include +#include +#include + +#include "catch.hpp" + +#include "tscore/BufferWriter.h" +#include "records/I_RecHttp.h" +#include "test_Diags.h" + +using ts::TextView; + +TEST_CASE("RecHttp", "[librecords][RecHttp]") +{ + std::vector ports; + CatchDiags *cdiag = static_cast(diags); + cdiag->messages.clear(); + + SECTION("base") + { + HttpProxyPort::loadValue(ports, "8080"); + REQUIRE(ports.size() == 1); + REQUIRE(ports[0].m_port == 8080); + } + + SECTION("two") + { + HttpProxyPort::loadValue(ports, "8080 8090"); + REQUIRE(ports.size() == 2); + REQUIRE(ports[0].m_port == 8080); + REQUIRE(ports[1].m_port == 8090); + } + + SECTION("family") + { + HttpProxyPort::loadValue(ports, "7070:ipv4:ip-in=192.168.56.1"); + REQUIRE(ports.size() == 1); + REQUIRE(ports[0].m_port == 7070); + REQUIRE(ports[0].m_family == AF_INET); + REQUIRE(ports[0].isSSL() == false); + } + + SECTION("crossed-family") + { + HttpProxyPort::loadValue(ports, "7070:ipv6:ip-in=192.168.56.1"); + REQUIRE(ports.size() == 0); + REQUIRE(cdiag->messages.size() == 2); + REQUIRE(cdiag->messages[0].find("[ipv6]") != std::string::npos); + REQUIRE(cdiag->messages[0].find("[ipv4]") != std::string::npos); + } + + SECTION("ipv6-a") + { + TextView descriptor{"4443:ssl:ip-in=[ffee::24c3:3349:3cee:0143]"}; + HttpProxyPort::loadValue(ports, descriptor.data()); + REQUIRE(ports.size() == 1); + REQUIRE(ports[0].m_port == 4443); + REQUIRE(ports[0].m_family == AF_INET6); + REQUIRE(ports[0].isSSL() == true); + } + + SECTION("dual-addr") + { + TextView descriptor{"4443:ssl:ipv6:ip-out=[ffee::24c3:3349:3cee:0143]:ip-out=10.1.2.3"}; + HttpProxyPort::loadValue(ports, descriptor.data()); + char buff[256]; + ports[0].print(buff, sizeof(buff)); + std::string_view view{buff}; + REQUIRE(ports.size() == 1); + REQUIRE(ports[0].m_port == 4443); + REQUIRE(ports[0].m_family == AF_INET6); + REQUIRE(ports[0].isSSL() == true); + REQUIRE(ports[0].m_outbound_ip6.isValid() == true); + REQUIRE(ports[0].m_outbound_ip4.isValid() == true); + REQUIRE(ports[0].m_inbound_ip.isValid() == false); + REQUIRE(view.find(":ssl") != TextView::npos); + REQUIRE(view.find(":proto") == TextView::npos); // it's default, should not have this. + } +} diff --git a/lib/records/unit_tests/unit_test_main.cc b/lib/records/unit_tests/unit_test_main.cc new file mode 100644 index 00000000000..bfa4a61ba27 --- /dev/null +++ b/lib/records/unit_tests/unit_test_main.cc @@ -0,0 +1,46 @@ +/** @file + + This file used for catch based tests. It is the main() stub. + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. + See the NOTICE file distributed with this work for additional information regarding copyright + ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance with the License. You may obtain a + copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software distributed under the License + is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + or implied. See the License for the specific language governing permissions and limitations under + the License. + */ + +#include +#include +#include "tscore/BufferWriter.h" +#include "tscore/ink_resolver.h" +#include "test_Diags.h" + +#define CATCH_CONFIG_RUNNER +#include "catch.hpp" + +Diags *diags = new CatchDiags; +extern void ts_session_protocol_well_known_name_indices_init(); + +int +main(int argc, char *argv[]) +{ + // Global data initialization needed for the unit tests. + ts_session_protocol_well_known_name_indices_init(); + // Cheat for ts_host_res_global_init as there's no records.config to check for non-default. + memcpy(host_res_default_preference_order, HOST_RES_DEFAULT_PREFERENCE_ORDER, sizeof(host_res_default_preference_order)); + + int result = Catch::Session().run(argc, argv); + + // global clean-up... + + return result; +} diff --git a/lib/tsconfig/Errata.h b/lib/tsconfig/Errata.h index 15b2373ce98..eba5bacf4ac 100644 --- a/lib/tsconfig/Errata.h +++ b/lib/tsconfig/Errata.h @@ -517,7 +517,7 @@ struct Errata::Data : public IntrusivePtrCounter { void push(Message && msg); /// Log this when it is deleted. - mutable bool m_log_on_delete; + mutable bool m_log_on_delete = true; //! The message stack. Container m_items; @@ -827,7 +827,7 @@ inline size_t Errata::size() const { } inline bool Errata::isOK() const { - return 0 == m_data + return nullptr == m_data || 0 == m_data->size() || Message::Success_Test(this->top()) ; @@ -867,7 +867,7 @@ inline Errata& Errata::doNotLog() { return *this; } -inline Errata::Data::Data() : m_log_on_delete(true) {} +inline Errata::Data::Data() {} inline size_t Errata::Data::size() const { return m_items.size(); } inline Errata::iterator::iterator() { } diff --git a/lib/tsconfig/TsBuilder.h b/lib/tsconfig/TsBuilder.h index f9da52d8efa..de765bee99e 100644 --- a/lib/tsconfig/TsBuilder.h +++ b/lib/tsconfig/TsBuilder.h @@ -36,9 +36,9 @@ class Builder { public: typedef Builder self; struct Handler { - self* _ptr; ///< Pointer to Builder instance. + self* _ptr = nullptr; ///< Pointer to Builder instance. /// Pointer to method to invoke for this event. - void (self::*_method)(Token const& token); + void (self::*_method)(Token const& token) = nullptr; /// Default constructor. Handler(); @@ -92,7 +92,7 @@ class Builder { self& init(); }; -inline Builder::Handler::Handler() : _ptr(nullptr), _method(nullptr) { } +inline Builder::Handler::Handler() { } inline Builder::Builder() { this->init(); } inline Builder::Builder(Configuration const& config) : _config(config) { this->init(); } diff --git a/lib/tsconfig/TsValue.h b/lib/tsconfig/TsValue.h index 59d7d4e30b8..7d420dfe745 100644 --- a/lib/tsconfig/TsValue.h +++ b/lib/tsconfig/TsValue.h @@ -29,6 +29,7 @@ #include #include #include +#include #include namespace ts { namespace config { @@ -216,13 +217,13 @@ namespace detail { /// Get item type. ValueType getType() const; protected: - ValueType _type; ///< Type of value. - ValueIndex _parent; ///< Table index of parent value. + ValueType _type = VoidValue; ///< Type of value. + ValueIndex _parent = 0; ///< Table index of parent value. ConstBuffer _text; ///< Text of value (if scalar). ConstBuffer _name; ///< Local name of value, if available. - size_t _local_index; ///< Index among siblings. - int _srcLine; ///< Source line. - int _srcColumn; ///< Source column. + size_t _local_index = 0; ///< Index among siblings. + int _srcLine = 0; ///< Source line. + int _srcColumn = 0; ///< Source column. /// Container for children of this item. typedef std::vector ChildGroup; @@ -644,14 +645,14 @@ namespace detail { inline ValueItem const& ValueTable::operator [] (ValueIndex idx) const { return const_cast(this)->operator [] (idx); } inline ValueTable& ValueTable::reset() { _ptr.reset(); return *this; } - inline ValueItem::ValueItem() : _type(VoidValue), _local_index(0), _srcLine(0), _srcColumn(0) {} - inline ValueItem::ValueItem(ValueType type) : _type(type), _local_index(0), _srcLine(0), _srcColumn(0) {} + inline ValueItem::ValueItem() {} + inline ValueItem::ValueItem(ValueType type) : _type(type) {} inline ValueType ValueItem::getType() const { return _type; } } inline Value::~Value() { } inline Value::Value() : _vidx(detail::NULL_VALUE_INDEX) {} -inline Value::Value(Configuration cfg, detail::ValueIndex vidx) : _config(cfg), _vidx(vidx) { } +inline Value::Value(Configuration cfg, detail::ValueIndex vidx) : _config(std::move(cfg)), _vidx(vidx) { } inline bool Value::hasValue() const { return _config && _vidx != detail::NULL_VALUE_INDEX; } inline Value::operator detail::PseudoBool::Type () const { return this->hasValue() ? detail::PseudoBool::TRUE : detail::PseudoBool::FALSE; } inline bool Value::operator ! () const { return ! this->hasValue(); } diff --git a/lib/yamlcpp/Makefile.am b/lib/yamlcpp/Makefile.am index 05f441af24d..a451416b7c8 100644 --- a/lib/yamlcpp/Makefile.am +++ b/lib/yamlcpp/Makefile.am @@ -17,8 +17,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -include $(top_srcdir)/build/tidy.mk - AM_CPPFLAGS += \ -I$(abs_top_srcdir)/lib/yamlcpp/include @@ -52,6 +50,3 @@ src/simplekey.cpp \ src/singledocparser.cpp \ src/stream.cpp \ src/tag.cpp - -clang-tidy-local: $(DIST_SOURCES) - $(CXX_Clang_Tidy) diff --git a/mgmt/Alarms.cc b/mgmt/Alarms.cc index 07441f95f4b..d7a3171f3b4 100644 --- a/mgmt/Alarms.cc +++ b/mgmt/Alarms.cc @@ -212,7 +212,7 @@ Alarms::signalAlarm(alarm_t a, const char *desc, const char *ip) } /* - * Exec alarm bin for priority alarms everytime, regardless if they are + * Exec alarm bin for priority alarms every time, regardless if they are * potentially duplicates. However, only exec this for you own alarms, * don't want every node in the cluster reporting the same alarm. */ @@ -223,7 +223,7 @@ Alarms::signalAlarm(alarm_t a, const char *desc, const char *ip) ink_mutex_acquire(&mutex); if (!ip) { // if an OEM alarm, then must create the unique key alarm type; - // this key is used to hash the new OEM alarm descritption in the hash table + // this key is used to hash the new OEM alarm description in the hash table if (a == MGMT_ALARM_ADD_ALARM) { a = (alarmOEMcount - minOEMkey) % (maxOEMkey - minOEMkey) + minOEMkey; alarmOEMcount++; @@ -289,7 +289,7 @@ Alarms::signalAlarm(alarm_t a, const char *desc, const char *ip) (*(func))(a, ip, desc); } - /* Priority 2 alarms get signalled if they are the first unsolved occurence. */ + /* Priority 2 alarms get signaled if they are the first unsolved occurrence. */ if (priority == 2 && !ip) { execAlarmBin(desc); } @@ -318,8 +318,8 @@ Alarms::resetSeenFlag(char *ip) /* * clearUnSeen(...) - * This function is a sweeper functionto clean up those alarms that have - * been taken care of through otehr local managers or at the peer itself. + * This function is a sweeper function to clean up those alarms that have + * been taken care of through other local managers or at the peer itself. */ void Alarms::clearUnSeen(char *ip) @@ -350,7 +350,7 @@ void Alarms::checkSystemNAlert() { return; -} /* End Alarms::checkSystenNAlert */ +} /* End Alarms::checkSystemNAlert */ void Alarms::execAlarmBin(const char *desc) diff --git a/mgmt/Alarms.h b/mgmt/Alarms.h index 2a0268bbedc..ce3317e6691 100644 --- a/mgmt/Alarms.h +++ b/mgmt/Alarms.h @@ -32,7 +32,7 @@ class AppVersionInfo; /*********************************************************************** * - * MODULARIZATTION: if you are adding new alarms, please ensure to add + * MODULARIZATION: if you are adding new alarms, please be sure to add * the corresponding alarms in lib/records/I_RecAlarms.h * ***********************************************************************/ diff --git a/mgmt/BaseManager.cc b/mgmt/BaseManager.cc index abd99ce2ede..86ef10034a1 100644 --- a/mgmt/BaseManager.cc +++ b/mgmt/BaseManager.cc @@ -4,103 +4,80 @@ @section license License - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at + Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. + See the NOTICE file distributed with this work for additional information regarding copyright + ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance with the License. You may obtain a + copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. + Unless required by applicable law or agreed to in writing, software distributed under the License + is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + or implied. See the License for the specific language governing permissions and limitations under + the License. */ #include "tscore/ink_memory.h" +#include "tscore/ink_mutex.h" #include "BaseManager.h" BaseManager::BaseManager() { - /* Setup the event queue and callback tables */ - mgmt_event_queue = create_queue(); - -} /* End BaseManager::BaseManager */ + ink_sem_init(&q_sem, 0); +} BaseManager::~BaseManager() { - while (!queue_is_empty(mgmt_event_queue)) { - MgmtMessageHdr *mh = (MgmtMessageHdr *)dequeue(mgmt_event_queue); - ats_free(mh); + while (!queue.empty()) { + ats_free(queue.front()); + queue.pop(); } - ats_free(mgmt_event_queue); - - for (auto &&it : mgmt_callback_table) { - MgmtCallbackList *tmp, *cb_list = it.second; +} - for (tmp = cb_list->next; tmp; tmp = cb_list->next) { - ats_free(cb_list); - cb_list = tmp; - } - ats_free(cb_list); - } +void +BaseManager::enqueue(MgmtMessageHdr *mh) +{ + std::lock_guard lock(q_mutex); + queue.emplace(mh); + ink_sem_post(&q_sem); +} - return; -} /* End BaseManager::~BaseManager */ +bool +BaseManager::queue_empty() +{ + std::lock_guard lock(q_mutex); + return queue.empty(); +} -/* - * registerMgmtCallback(...) - * Function to register callback's for various management events, such - * as shutdown, re-init, etc. The following callbacks should be - * registered: - * MGMT_EVENT_SHUTDOWN (graceful shutdown) - * MGMT_EVENT_RESTART (graceful reboot) - * ... - * - * Returns: -1 on error(invalid event id passed in) - * or value - */ -int -BaseManager::registerMgmtCallback(int msg_id, MgmtCallback cb, void *opaque_cb_data) +MgmtMessageHdr * +BaseManager::dequeue() { - MgmtCallbackList *cb_list; + MgmtMessageHdr *msg{nullptr}; - if (auto it = mgmt_callback_table.find(msg_id); it != mgmt_callback_table.end()) { - cb_list = it->second; - } else { - cb_list = nullptr; + ink_sem_wait(&q_sem); + { + std::lock_guard lock(q_mutex); + msg = queue.front(); + queue.pop(); } + return msg; +} - if (cb_list) { - MgmtCallbackList *tmp; - - for (tmp = cb_list; tmp->next; tmp = tmp->next) { - ; - } - tmp->next = (MgmtCallbackList *)ats_malloc(sizeof(MgmtCallbackList)); - tmp->next->func = cb; - tmp->next->opaque_data = opaque_cb_data; - tmp->next->next = nullptr; - } else { - cb_list = (MgmtCallbackList *)ats_malloc(sizeof(MgmtCallbackList)); - cb_list->func = cb; - cb_list->opaque_data = opaque_cb_data; - cb_list->next = nullptr; - mgmt_callback_table.emplace(msg_id, cb_list); - } +int +BaseManager::registerMgmtCallback(int msg_id, MgmtCallback const &cb) +{ + auto &cb_list{mgmt_callback_table[msg_id]}; + cb_list.emplace_back(cb); return msg_id; -} /* End BaseManager::registerMgmtCallback */ +} void -BaseManager::executeMgmtCallback(int msg_id, char *data_raw, int data_len) +BaseManager::executeMgmtCallback(int msg_id, ts::MemSpan span) { if (auto it = mgmt_callback_table.find(msg_id); it != mgmt_callback_table.end()) { - for (MgmtCallbackList *cb_list = it->second; cb_list; cb_list = cb_list->next) { - (*((MgmtCallback)(cb_list->func)))(cb_list->opaque_data, data_raw, data_len); + for (auto &&cb : it->second) { + cb(span); } } } diff --git a/mgmt/BaseManager.h b/mgmt/BaseManager.h index a254a722f6c..9f72212f833 100644 --- a/mgmt/BaseManager.h +++ b/mgmt/BaseManager.h @@ -23,15 +23,18 @@ #pragma once +#include +#include +#include +#include + #include "tscore/ink_thread.h" #include "tscore/ink_mutex.h" -#include "tscore/ink_llqueue.h" +#include "tscpp/util/MemSpan.h" #include "MgmtDefs.h" #include "MgmtMarshall.h" -#include - /* * MgmtEvent defines. */ @@ -88,30 +91,61 @@ #define MGMT_SIGNAL_SAC_SERVER_DOWN 400 -typedef struct _mgmt_message_hdr_type { +struct MgmtMessageHdr { int msg_id; int data_len; -} MgmtMessageHdr; - -typedef struct _mgmt_event_callback_list { - MgmtCallback func; - void *opaque_data; - struct _mgmt_event_callback_list *next; -} MgmtCallbackList; + ts::MemSpan + payload() + { + return {reinterpret_cast(this) + sizeof(*this), data_len}; + } +}; class BaseManager { + using MgmtCallbackList = std::list; + public: BaseManager(); - ~BaseManager(); - int registerMgmtCallback(int msg_id, MgmtCallback func, void *opaque_callback_data = nullptr); + ~BaseManager(); - LLQ *mgmt_event_queue; - std::unordered_map mgmt_callback_table; + /** Associate a callback function @a func with message identifier @a msg_id. + * + * @param msg_id Message identifier for the callback. + * @param func The callback function. + * @return @a msg_id on success, -1 on failure. + * + * @a msg_id should be one of the @c MGMT_EVENT_... values. + * + * If a management message with @a msg is received, the callbacks for that message id + * are invoked and passed the message payload (not including the header). + */ + int registerMgmtCallback(int msg_id, MgmtCallback const &func); + + /// Add a @a msg to the queue. + /// This must be the entire message as read off the wire including the header. + void enqueue(MgmtMessageHdr *msg); + + /// Current size of the queue. + /// @note This does not block on the semaphore. + bool queue_empty(); + + /// Dequeue a msg. + /// This waits on the semaphore for a message to arrive. + MgmtMessageHdr *dequeue(); protected: - void executeMgmtCallback(int msg_id, char *data_raw, int data_len); - -private: -}; /* End class BaseManager */ + void executeMgmtCallback(int msg_id, ts::MemSpan span); + + /// The mapping from an event type to a list of callbacks to invoke. + std::unordered_map mgmt_callback_table; + + /// Message queue. + // These holds the entire message object, including the header. + std::queue queue; + /// Locked access to the queue. + std::mutex q_mutex; + /// Semaphore to signal queue state. + ink_semaphore q_sem; +}; diff --git a/mgmt/DerivativeMetrics.cc b/mgmt/DerivativeMetrics.cc index ca85c8afcb2..16572db439e 100644 --- a/mgmt/DerivativeMetrics.cc +++ b/mgmt/DerivativeMetrics.cc @@ -50,7 +50,7 @@ static const std::vector sum_metrics = { RECD_INT, {"proxy.process.http.origin_server_response_document_total_size", "proxy.process.http.origin_server_response_header_total_size"}}, - // Total byates of client request and response (total traffic to and from clients) + // Total bytes of client request and response (total traffic to and from clients) {"proxy.process.user_agent_total_bytes", RECD_INT, {"proxy.process.http.user_agent_total_request_bytes", "proxy.process.http.user_agent_total_response_bytes"}}, @@ -69,7 +69,7 @@ static const std::vector sum_metrics = { RECD_COUNTER, {"proxy.process.http.cache_miss_cold", "proxy.process.http.cache_miss_changed", "proxy.process.http.cache_miss_client_no_cache", "proxy.process.http.cache_miss_ims", "proxy.process.http.cache_miss_client_not_cacheable"}}, - // Total requests, both hits and misses (this is slightly superflous, but assures correct percentage calculations) + // Total requests, both hits and misses (this is slightly superfluous, but assures correct percentage calculations) {"proxy.process.cache_total_requests", RECD_COUNTER, {"proxy.process.cache_total_hits", "proxy.process.cache_total_misses"}}, // Total cache requests bytes which are cache hits {"proxy.process.cache_total_hits_bytes", diff --git a/mgmt/FileManager.cc b/mgmt/FileManager.cc index 6e9edfecf9c..9288d92e740 100644 --- a/mgmt/FileManager.cc +++ b/mgmt/FileManager.cc @@ -118,7 +118,7 @@ FileManager::addFileHelper(const char *fileName, const char *configName, bool ro // Sets rbPtr to the rollback object associated // with the passed in fileName. // -// If there is no binding, falseis returned +// If there is no binding, false is returned // bool FileManager::getRollbackObj(const char *fileName, Rollback **rbPtr) @@ -134,7 +134,7 @@ FileManager::getRollbackObj(const char *fileName, Rollback **rbPtr) // bool FileManager::fileChanged(const char* fileName) // -// Called by the Rollback class whenever a a config has changed +// Called by the Rollback class whenever a config has changed // Initiates callbacks // // @@ -148,7 +148,7 @@ FileManager::fileChanged(const char *fileName, const char *configName, bool incV for (cb = cblist.head; cb != nullptr; cb = cb->link.next) { // Dup the string for each callback to be - // defensive incase it modified when it is not supposed to be + // defensive in case it's modified when it's not supposed to be confignameCopy = ats_strdup(configName); filenameCopy = ats_strdup(fileName); (*cb->func)(filenameCopy, confignameCopy, incVersion); @@ -160,7 +160,7 @@ FileManager::fileChanged(const char *fileName, const char *configName, bool incV // void FileManger::rereadConfig() // -// Interates through the list of managed files and +// Iterates through the list of managed files and // calls Rollback::checkForUserUpdate on them // // although it is tempting, DO NOT CALL FROM SIGNAL HANDLERS diff --git a/mgmt/FileManager.h b/mgmt/FileManager.h index 94cc7cc731e..a667504d418 100644 --- a/mgmt/FileManager.h +++ b/mgmt/FileManager.h @@ -62,7 +62,7 @@ enum lockAction_t { // a binding and false otherwise // // registerCallback(FileCallbackFunc) - registers a callback function -// which will get called everytime a managed file changes. The +// which will get called every time a managed file changes. The // callback function should NOT use the calling thread to // access any Rollback objects or block for a long time // diff --git a/mgmt/LocalManager.cc b/mgmt/LocalManager.cc index 4b94d9cdc65..15ec02ebb2f 100644 --- a/mgmt/LocalManager.cc +++ b/mgmt/LocalManager.cc @@ -47,14 +47,6 @@ using namespace std::literals; static const std::string_view MGMT_OPT{"-M"}; static const std::string_view RUNROOT_OPT{"--run-root="}; -#ifndef MPTCP_ENABLED -#if defined(linux) -#define MPTCP_ENABLED 42 -#else -#define MPTCP_ENABLED 0 -#endif -#endif - void LocalManager::mgmtCleanup() { @@ -159,8 +151,8 @@ LocalManager::clearStats(const char *name) // before the proxy clears them, but this should be rare. // // Doing things in the opposite order prevents that race - // but excerbates the race between the node and cluster - // stats getting cleared by progation of clearing the + // but exacerbates the race between the node and cluster + // stats getting cleared by propagation of clearing the // cluster stats // if (name && *name) { @@ -193,7 +185,7 @@ LocalManager::processRunning() } } -LocalManager::LocalManager(bool proxy_on) : BaseManager(), run_proxy(proxy_on) +LocalManager::LocalManager(bool proxy_on, bool listen) : BaseManager(), run_proxy(proxy_on), listen_for_proxy(listen) { bool found; std::string rundir(RecConfigReadRuntimeDir()); @@ -320,7 +312,6 @@ LocalManager::initMgmtProcessServer() void LocalManager::pollMgmtProcessServer() { - int num; struct timeval timeout; fd_set fdlist; @@ -363,7 +354,7 @@ LocalManager::pollMgmtProcessServer() } #endif - num = mgmt_select(FD_SETSIZE, &fdlist, nullptr, nullptr, &timeout); + int num = mgmt_select(FD_SETSIZE, &fdlist, nullptr, nullptr, &timeout); switch (num) { case 0: @@ -410,16 +401,14 @@ LocalManager::pollMgmtProcessServer() if (ts::NO_FD != watched_process_fd && FD_ISSET(watched_process_fd, &fdlist)) { int res; MgmtMessageHdr mh_hdr; - MgmtMessageHdr *mh_full; - char *data_raw; keep_polling = true; // read the message if ((res = mgmt_read_pipe(watched_process_fd, (char *)&mh_hdr, sizeof(MgmtMessageHdr))) > 0) { - mh_full = (MgmtMessageHdr *)alloca(sizeof(MgmtMessageHdr) + mh_hdr.data_len); + MgmtMessageHdr *mh_full = (MgmtMessageHdr *)alloca(sizeof(MgmtMessageHdr) + mh_hdr.data_len); memcpy(mh_full, &mh_hdr, sizeof(MgmtMessageHdr)); - data_raw = (char *)mh_full + sizeof(MgmtMessageHdr); + char *data_raw = (char *)mh_full + sizeof(MgmtMessageHdr); if ((res = mgmt_read_pipe(watched_process_fd, data_raw, mh_hdr.data_len)) > 0) { handleMgmtMsgFromProcesses(mh_full); } else if (res < 0) { @@ -579,9 +568,9 @@ LocalManager::handleMgmtMsgFromProcesses(MgmtMessageHdr *mh) } case MGMT_SIGNAL_LIBRECORDS: if (mh->data_len > 0) { - executeMgmtCallback(MGMT_SIGNAL_LIBRECORDS, data_raw, mh->data_len); + executeMgmtCallback(MGMT_SIGNAL_LIBRECORDS, {data_raw, mh->data_len}); } else { - executeMgmtCallback(MGMT_SIGNAL_LIBRECORDS, nullptr, 0); + executeMgmtCallback(MGMT_SIGNAL_LIBRECORDS, {}); } break; case MGMT_SIGNAL_CONFIG_FILE_CHILD: { @@ -755,19 +744,22 @@ void LocalManager::signalEvent(int msg_id, const char *data_raw, int data_len) { MgmtMessageHdr *mh; + size_t n = sizeof(MgmtMessageHdr) + data_len; - mh = (MgmtMessageHdr *)ats_malloc(sizeof(MgmtMessageHdr) + data_len); + mh = static_cast(ats_malloc(n)); mh->msg_id = msg_id; mh->data_len = data_len; - memcpy((char *)mh + sizeof(MgmtMessageHdr), data_raw, data_len); - ink_assert(enqueue(mgmt_event_queue, mh)); + auto payload = mh->payload(); + memcpy(payload.data(), data_raw, data_len); + this->enqueue(mh); + // ink_assert(enqueue(mgmt_event_queue, mh)); #if HAVE_EVENTFD // we don't care about the actual value of wakeup_fd, so just keep adding 1. just need to - // wakeup the fd. also, note that wakeup_fd was initalized to non-blocking so we can + // wakeup the fd. also, note that wakeup_fd was initialized to non-blocking so we can // directly write to it without any timeout checking. // - // don't tigger if MGMT_EVENT_LIBRECORD because they happen all the time + // don't trigger if MGMT_EVENT_LIBRECORD because they happen all the time // and don't require a quick response. for MGMT_EVENT_LIBRECORD, rely on timeouts so // traffic_server can spend more time doing other things uint64_t one = 1; @@ -786,18 +778,16 @@ LocalManager::signalEvent(int msg_id, const char *data_raw, int data_len) void LocalManager::processEventQueue() { - bool handled_by_mgmt; - - while (!queue_is_empty(mgmt_event_queue)) { - handled_by_mgmt = false; + while (!this->queue_empty()) { + bool handled_by_mgmt = false; - MgmtMessageHdr *mh = (MgmtMessageHdr *)dequeue(mgmt_event_queue); - char *data_raw = (char *)mh + sizeof(MgmtMessageHdr); + MgmtMessageHdr *mh = this->dequeue(); + auto payload = mh->payload(); // check if we have a local file update if (mh->msg_id == MGMT_EVENT_CONFIG_FILE_UPDATE || mh->msg_id == MGMT_EVENT_CONFIG_FILE_UPDATE_NO_INC_VERSION) { // records.config - if (!(strcmp(data_raw, REC_CONFIG_FILE))) { + if (!(strcmp(payload.begin(), REC_CONFIG_FILE))) { bool incVersion = mh->msg_id == MGMT_EVENT_CONFIG_FILE_UPDATE; if (RecReadConfigFile(incVersion) != REC_ERR_OKAY) { mgmt_elog(errno, "[fileUpdated] Config update failed for records.config\n"); @@ -813,10 +803,10 @@ LocalManager::processEventQueue() // Fix INKqa04984 // If traffic server hasn't completely come up yet, // we will hold off until next round. - ink_assert(enqueue(mgmt_event_queue, mh)); + this->enqueue(mh); return; } - Debug("lm", "[TrafficManager] ==> Sending signal event '%d' %s payload=%d", mh->msg_id, data_raw, mh->data_len); + Debug("lm", "[TrafficManager] ==> Sending signal event '%d' %s payload=%d", mh->msg_id, payload.begin(), int(payload.size())); lmgmt->sendMgmtMsgToProcesses(mh); } ats_free(mh); @@ -844,7 +834,7 @@ LocalManager::startProxy(const char *onetime_options) pid_t pid; // Before we do anything lets check for the existence of - // the traffic server binary along with it's execute permmissions + // the traffic server binary along with it's execute permissions if (access(absolute_proxy_binary, F_OK) < 0) { // Error can't find traffic_server mgmt_elog(errno, "[LocalManager::startProxy] Unable to find traffic server at %s\n", absolute_proxy_binary); @@ -853,7 +843,7 @@ LocalManager::startProxy(const char *onetime_options) // traffic server binary exists, check permissions else if (access(absolute_proxy_binary, R_OK | X_OK) < 0) { // Error don't have proper permissions - mgmt_elog(errno, "[LocalManager::startProxy] Unable to access %s due to bad permisssions \n", absolute_proxy_binary); + mgmt_elog(errno, "[LocalManager::startProxy] Unable to access %s due to bad permissions \n", absolute_proxy_binary); return false; } @@ -981,7 +971,7 @@ LocalManager::closeProxyPorts() void LocalManager::listenForProxy() { - if (!run_proxy) { + if (!run_proxy || !listen_for_proxy) { return; } @@ -991,7 +981,7 @@ LocalManager::listenForProxy() this->bindProxyPort(p); } - // read backlong configuration value and overwrite the default value if found + // read backlog configuration value and overwrite the default value if found bool found; std::string_view fam{ats_ip_family_name(p.m_family)}; RecInt backlog = REC_readInteger("proxy.config.net.listen_backlog", &found); @@ -1040,11 +1030,13 @@ LocalManager::bindProxyPort(HttpProxyPort &port) err = setsockopt(port.m_fd, IPPROTO_TCP, MPTCP_ENABLED, &one, sizeof(one)); if (err < 0) { mgmt_log("[bindProxyPort] Unable to enable MPTCP: %s\n", strerror(errno)); + Debug("lm_mptcp", "[bindProxyPort] Unable to enable MPTCP: %s", strerror(errno)); } else { mgmt_log("[bindProxyPort] Successfully enabled MPTCP on %d\n", port.m_port); + Debug("lm_mptcp", "[bindProxyPort] Successfully enabled MPTCP on %d\n", port.m_port); } #else - Debug("lm", "[bindProxyPort] Multipath TCP requested but not configured on this host"); + Debug("lm_mptcp", "[bindProxyPort] Multipath TCP requested but not configured on this host"); #endif } @@ -1057,6 +1049,10 @@ LocalManager::bindProxyPort(HttpProxyPort &port) mgmt_fatal(0, "[bindProxyPort] Unable to set socket options: %d : %s\n", port.m_port, strerror(errno)); } + if (port.m_proxy_protocol) { + Debug("lm", "[bindProxyPort] Proxy Protocol enabled"); + } + if (port.m_inbound_transparent_p) { #if TS_USE_TPROXY Debug("http_tproxy", "Listen port %d inbound transparency enabled.", port.m_port); diff --git a/mgmt/LocalManager.h b/mgmt/LocalManager.h index 8feffbd6148..0f4faffabad 100644 --- a/mgmt/LocalManager.h +++ b/mgmt/LocalManager.h @@ -56,7 +56,7 @@ enum ManagementPendingOperation { class LocalManager : public BaseManager { public: - explicit LocalManager(bool proxy_on); + explicit LocalManager(bool proxy_on, bool listen); ~LocalManager(); void initAlarm(); @@ -93,6 +93,7 @@ class LocalManager : public BaseManager bool processRunning(); bool run_proxy; + bool listen_for_proxy; bool proxy_recoverable = true; // false if traffic_server cannot recover with a reboot time_t manager_started_at; time_t proxy_started_at = -1; diff --git a/mgmt/MgmtDefs.h b/mgmt/MgmtDefs.h index 84424c43c9f..87dadf65f7c 100644 --- a/mgmt/MgmtDefs.h +++ b/mgmt/MgmtDefs.h @@ -30,6 +30,8 @@ #include #include "tscore/ink_defs.h" +#include "tscpp/util/MemSpan.h" +#include "tscpp/util/TextView.h" typedef int64_t MgmtIntCounter; typedef int64_t MgmtInt; @@ -37,25 +39,24 @@ typedef int8_t MgmtByte; typedef float MgmtFloat; typedef char *MgmtString; -typedef enum { +enum MgmtType { MGMT_INVALID = -1, MGMT_INT = 0, MGMT_FLOAT = 1, MGMT_STRING = 2, MGMT_COUNTER = 3, MGMT_TYPE_MAX = 4, -} MgmtType; +}; -/* - * MgmtCallback - * Management Callback functions. - */ -typedef void *(*MgmtCallback)(void *opaque_cb_data, char *data_raw, int data_len); +/// Management callback signature. +/// The memory span is the message payload for the callback. +/// This can be a lambda, which should be used if additional context information is needed. +using MgmtCallback = std::function; //------------------------------------------------------------------------- // API conversion functions. //------------------------------------------------------------------------- -/** Conversion functions to and from an aribrary type and Management types. +/** Conversion functions to and from an arbitrary type and Management types. * * A type that wants to support conversion in the TS API should create a static instance of this * class and fill in the appropriate members. The TS API set/get functions can then check for a @@ -66,19 +67,80 @@ typedef void *(*MgmtCallback)(void *opaque_cb_data, char *data_raw, int data_len * in this header. */ struct MgmtConverter { - // MgmtInt conversions. - std::function get_int{nullptr}; - std::function set_int{nullptr}; - - // MgmtFloat conversions. - std::function get_float{nullptr}; - std::function set_float{nullptr}; - - // MgmtString conversions. - // This is a bit different because it takes std::string_view instead of MgmtString but that's - // worth the difference. - std::function get_string{nullptr}; - std::function set_string{nullptr}; + /** Load a native type into a @c MgmtInt + * + * This is passed a @c void* which is a pointer to the member in the configuration instance. + * This function must return a @c MgmtInt converted from that value. + */ + MgmtInt (*load_int)(void *) = nullptr; + + /** Store a @c MgmtInt into a native type. + * + * This function is passed a @c void* which is a pointer to the member in the configuration + * instance and a @c MgmtInt. The member should be updated to correspond to the @c MgmtInt value. + */ + void (*store_int)(void *, MgmtInt) = nullptr; + + /** Load a @c MgmtFloat from a native type. + * + * This is passed a @c void* which is a pointer to the member in the configuration instance. + * This function must return a @c MgmtFloat converted from that value. + */ + MgmtFloat (*load_float)(void *) = nullptr; + + /** Store a @c MgmtFloat into a native type. + * + * This function is passed a @c void* which is a pointer to the member in the configuration + * instance and a @c MgmtFloat. The member should be updated to correspond to the @c MgmtFloat value. + */ + void (*store_float)(void *, MgmtFloat) = nullptr; + + /** Load a native type into view. + * + * This is passed a @c void* which is a pointer to the member in the configuration instance. + * This function must return a @c string_view which contains the text for the member. + */ + std::string_view (*load_string)(void *) = nullptr; + + /** Store a view in a native type. + * + * This is passed a @c void* which is a pointer to the member in the configuration instance. + * This function must return a @c string_view which contains the text for the member. + */ + void (*store_string)(void *, std::string_view) = nullptr; + + // Convenience constructors because generally only one pair is valid. + MgmtConverter(MgmtInt (*load)(void *), void (*store)(void *, MgmtInt)); + MgmtConverter(MgmtFloat (*load)(void *), void (*store)(void *, MgmtFloat)); + MgmtConverter(std::string_view (*load)(void *), void (*store)(void *, std::string_view)); + + MgmtConverter(MgmtInt (*_load_int)(void *), void (*_store_int)(void *, MgmtInt), MgmtFloat (*_load_float)(void *), + void (*_store_float)(void *, MgmtFloat), std::string_view (*_load_string)(void *), + void (*_store_string)(void *, std::string_view)); }; -#define LM_CONNECTION_SERVER "processerver.sock" +inline MgmtConverter::MgmtConverter(MgmtInt (*load)(void *), void (*store)(void *, MgmtInt)) : load_int(load), store_int(store) {} + +inline MgmtConverter::MgmtConverter(MgmtFloat (*load)(void *), void (*store)(void *, MgmtFloat)) + : load_float(load), store_float(store) +{ +} + +inline MgmtConverter::MgmtConverter(std::string_view (*load)(void *), void (*store)(void *, std::string_view)) + : load_string(load), store_string(store) +{ +} + +inline MgmtConverter::MgmtConverter(MgmtInt (*_load_int)(void *), void (*_store_int)(void *, MgmtInt), + MgmtFloat (*_load_float)(void *), void (*_store_float)(void *, MgmtFloat), + std::string_view (*_load_string)(void *), void (*_store_string)(void *, std::string_view)) + : load_int(_load_int), + store_int(_store_int), + load_float(_load_float), + store_float(_store_float), + load_string(_load_string), + store_string(_store_string) +{ +} + +constexpr ts::TextView LM_CONNECTION_SERVER{"processerver.sock"}; diff --git a/mgmt/ProcessManager.cc b/mgmt/ProcessManager.cc index d5932be6dc8..b492c31d1cc 100644 --- a/mgmt/ProcessManager.cc +++ b/mgmt/ProcessManager.cc @@ -25,6 +25,7 @@ #include "ProcessManager.h" #include "tscore/ink_apidefs.h" +#include "tscore/TSSystemState.h" #include "MgmtSocket.h" #include "tscore/I_Layout.h" @@ -122,11 +123,13 @@ ProcessManager::stop() poll_thread = ink_thread_null(); while (!queue_is_empty(mgmt_signal_queue)) { - char *sig = (char *)dequeue(mgmt_signal_queue); + char *sig = (char *)::dequeue(mgmt_signal_queue); ats_free(sig); } - ats_free(mgmt_signal_queue); + LLQ *tmp_queue = mgmt_signal_queue; + mgmt_signal_queue = nullptr; + delete_queue(tmp_queue); } /* @@ -164,13 +167,13 @@ ProcessManager::processManagerThread(void *arg) if (pmgmt->require_lm) { ret = pmgmt->pollLMConnection(); - if (ret < 0 && pmgmt->running) { + if (ret < 0 && pmgmt->running && !TSSystemState::is_event_system_shut_down()) { Alert("exiting with read error from process manager: %s", strerror(-ret)); } } ret = pmgmt->processSignalQueue(); - if (ret < 0 && pmgmt->running) { + if (ret < 0 && pmgmt->running && !TSSystemState::is_event_system_shut_down()) { Alert("exiting with write error from process manager: %s", strerror(-ret)); } } @@ -243,15 +246,36 @@ ProcessManager::signalManager(int msg_id, const char *data_raw, int data_len) mh->msg_id = msg_id; mh->data_len = data_len; memcpy((char *)mh + sizeof(MgmtMessageHdr), data_raw, data_len); + this->signalManager(mh); +} + +void +ProcessManager::signalManager(int msg_id, std::string_view text) +{ + MgmtMessageHdr *mh; - ink_release_assert(enqueue(mgmt_signal_queue, mh)); + // Make space for the extra null terminator. + mh = static_cast(ats_malloc(sizeof(MgmtMessageHdr) + text.size() + 1)); + auto body = reinterpret_cast(mh + 1); // start of the message body. + mh->msg_id = msg_id; + mh->data_len = text.size() + 1; + memcpy(body, text.data(), text.size()); + body[text.size()] = '\0'; + + this->signalManager(mh); +} + +void +ProcessManager::signalManager(MgmtMessageHdr *mh) +{ + ink_release_assert(::enqueue(mgmt_signal_queue, mh)); #if HAVE_EVENTFD // we don't care about the actual value of wakeup_fd, so just keep adding 1. just need to - // wakeup the fd. also, note that wakeup_fd was initalized to non-blocking so we can + // wakeup the fd. also, note that wakeup_fd was initialized to non-blocking so we can // directly write to it without any timeout checking. // - // don't tigger if MGMT_EVENT_LIBRECORD because they happen all the time + // don't trigger if MGMT_EVENT_LIBRECORD because they happen all the time // and don't require a quick response. for MGMT_EVENT_LIBRECORD, rely on timeouts so // traffic_server can spend more time doing other things/ uint64_t one = 1; @@ -265,7 +289,7 @@ int ProcessManager::processSignalQueue() { while (!queue_is_empty(mgmt_signal_queue)) { - MgmtMessageHdr *mh = (MgmtMessageHdr *)dequeue(mgmt_signal_queue); + MgmtMessageHdr *mh = (MgmtMessageHdr *)::dequeue(mgmt_signal_queue); Debug("pmgmt", "signaling local manager with message ID %d", mh->msg_id); @@ -414,35 +438,35 @@ ProcessManager::handleMgmtMsgFromLM(MgmtMessageHdr *mh) { ink_assert(mh != nullptr); - char *data_raw = (char *)mh + sizeof(MgmtMessageHdr); + auto payload = mh->payload(); Debug("pmgmt", "processing event id '%d' payload=%d", mh->msg_id, mh->data_len); switch (mh->msg_id) { case MGMT_EVENT_SHUTDOWN: - executeMgmtCallback(MGMT_EVENT_SHUTDOWN, nullptr, 0); + executeMgmtCallback(MGMT_EVENT_SHUTDOWN, {}); Alert("exiting on shutdown message"); break; case MGMT_EVENT_RESTART: - executeMgmtCallback(MGMT_EVENT_RESTART, nullptr, 0); + executeMgmtCallback(MGMT_EVENT_RESTART, {}); break; case MGMT_EVENT_DRAIN: - executeMgmtCallback(MGMT_EVENT_DRAIN, data_raw, mh->data_len); + executeMgmtCallback(MGMT_EVENT_DRAIN, payload); break; case MGMT_EVENT_CLEAR_STATS: - executeMgmtCallback(MGMT_EVENT_CLEAR_STATS, nullptr, 0); + executeMgmtCallback(MGMT_EVENT_CLEAR_STATS, {}); break; case MGMT_EVENT_HOST_STATUS_UP: - executeMgmtCallback(MGMT_EVENT_HOST_STATUS_UP, data_raw, mh->data_len); + executeMgmtCallback(MGMT_EVENT_HOST_STATUS_UP, payload); break; case MGMT_EVENT_HOST_STATUS_DOWN: - executeMgmtCallback(MGMT_EVENT_HOST_STATUS_DOWN, data_raw, mh->data_len); + executeMgmtCallback(MGMT_EVENT_HOST_STATUS_DOWN, payload); break; case MGMT_EVENT_ROLL_LOG_FILES: - executeMgmtCallback(MGMT_EVENT_ROLL_LOG_FILES, nullptr, 0); + executeMgmtCallback(MGMT_EVENT_ROLL_LOG_FILES, {}); break; case MGMT_EVENT_PLUGIN_CONFIG_UPDATE: - if (data_raw != nullptr && data_raw[0] != '\0' && this->cbtable) { - this->cbtable->invoke(data_raw); + if (!payload.empty() && payload.at(0) != '\0' && this->cbtable) { + this->cbtable->invoke(static_cast(payload.data())); } break; case MGMT_EVENT_CONFIG_FILE_UPDATE: @@ -463,13 +487,13 @@ ProcessManager::handleMgmtMsgFromLM(MgmtMessageHdr *mh) */ break; case MGMT_EVENT_LIBRECORDS: - executeMgmtCallback(MGMT_EVENT_LIBRECORDS, data_raw, mh->data_len); + executeMgmtCallback(MGMT_EVENT_LIBRECORDS, payload); break; case MGMT_EVENT_STORAGE_DEVICE_CMD_OFFLINE: - executeMgmtCallback(MGMT_EVENT_STORAGE_DEVICE_CMD_OFFLINE, data_raw, mh->data_len); + executeMgmtCallback(MGMT_EVENT_STORAGE_DEVICE_CMD_OFFLINE, payload); break; case MGMT_EVENT_LIFECYCLE_MESSAGE: - executeMgmtCallback(MGMT_EVENT_LIFECYCLE_MESSAGE, data_raw, mh->data_len); + executeMgmtCallback(MGMT_EVENT_LIFECYCLE_MESSAGE, payload); break; default: Warning("received unknown message ID %d\n", mh->msg_id); diff --git a/mgmt/ProcessManager.h b/mgmt/ProcessManager.h index dcfeb91db15..cc79fbc1436 100644 --- a/mgmt/ProcessManager.h +++ b/mgmt/ProcessManager.h @@ -25,12 +25,14 @@ #pragma once +#include +#include + #include "MgmtUtils.h" #include "BaseManager.h" #include "tscore/ink_sock.h" #include "tscore/ink_apidefs.h" -#include #if HAVE_EVENTFD #include @@ -56,6 +58,17 @@ class ProcessManager : public BaseManager inkcoreapi void signalManager(int msg_id, const char *data_str); inkcoreapi void signalManager(int msg_id, const char *data_raw, int data_len); + /** Send a management message of type @a msg_id with @a text. + * + * @param msg_id ID for the message. + * @param text Content for the message. + * + * A terminating null character is added automatically. + */ + inkcoreapi void signalManager(int msg_id, std::string_view text); + + inkcoreapi void signalManager(MgmtMessageHdr *mh); + void reconfigure(); void initLMConnection(); void handleMgmtMsgFromLM(MgmtMessageHdr *mh); diff --git a/mgmt/ProxyConfig.cc b/mgmt/ProxyConfig.cc index 4ea74d623f4..67199ee4def 100644 --- a/mgmt/ProxyConfig.cc +++ b/mgmt/ProxyConfig.cc @@ -69,13 +69,12 @@ config_string_alloc_cb(void *data, void *value) char *_ss = (char *)value; char *_new_value = nullptr; -//#define DEBUG_CONFIG_STRING_UPDATE #if defined(DEBUG_CONFIG_STRING_UPDATE) printf("config callback [new, old] = [%s : %s]\n", (_ss) ? (_ss) : (""), (*(char **)data) ? (*(char **)data) : ("")); #endif - int len = -1; + if (_ss) { - len = strlen(_ss); + int len = strlen(_ss); _new_value = (char *)ats_malloc(len + 1); memcpy(_new_value, _ss, len + 1); } @@ -146,7 +145,7 @@ ConfigProcessor::set(unsigned int id, ConfigInfo *info, unsigned timeout_secs) if (old_info) { // The ConfigInfoReleaser now takes our refcount, but - // someother thread might also have one ... + // some other thread might also have one ... ink_assert(old_info->refcount() > 0); eventProcessor.schedule_in(new ConfigInfoReleaser(id, old_info), HRTIME_SECONDS(timeout_secs)); } @@ -188,7 +187,7 @@ ConfigProcessor::release(unsigned int id, ConfigInfo *info) idx = id - 1; - if (info->refcount_dec() == 0) { + if (info && info->refcount_dec() == 0) { // When we release, we should already have replaced this object in the index. Debug("config", "Release config %d 0x%" PRId64, id, (int64_t)info); ink_release_assert(info != this->infos[idx]); diff --git a/mgmt/RecordsConfig.cc b/mgmt/RecordsConfig.cc index b28e33090b1..9b21975bcb1 100644 --- a/mgmt/RecordsConfig.cc +++ b/mgmt/RecordsConfig.cc @@ -299,8 +299,6 @@ static const RecordElement RecordsConfig[] = // ########### {RECT_CONFIG, "proxy.config.header.parse.no_host_url_redirect", RECD_STRING, nullptr, RECU_DYNAMIC, RR_NULL, RECC_STR, ".*", RECA_NULL} , - {RECT_CONFIG, "proxy.config.http.parse.allow_non_http", RECD_INT, "1", RECU_NULL, RR_NULL, RECC_NULL, nullptr, RECA_NULL} - , //############################################################################## //# @@ -534,6 +532,8 @@ static const RecordElement RecordsConfig[] = , {RECT_CONFIG, "proxy.config.http.insert_forwarded", RECD_STRING, "none", RECU_DYNAMIC, RR_NULL, RECC_NULL, nullptr, RECA_NULL} , + {RECT_CONFIG, "proxy.config.http.proxy_protocol_whitelist", RECD_STRING, "none", RECU_NULL, RR_NULL, RECC_NULL, nullptr, RECA_NULL} + , {RECT_CONFIG, "proxy.config.http.insert_age_in_response", RECD_INT, "1", RECU_DYNAMIC, RR_NULL, RECC_INT, "[0-1]", RECA_NULL} , {RECT_CONFIG, "proxy.config.http.enable_http_stats", RECD_INT, "1", RECU_DYNAMIC, RR_NULL, RECC_INT, "[0-1]", RECA_NULL} @@ -791,9 +791,9 @@ static const RecordElement RecordsConfig[] = , {RECT_CONFIG, "proxy.config.net.sock_option_tfo_queue_size_in", RECD_INT, "10000", RECU_NULL, RR_NULL, RECC_NULL, nullptr, RECA_NULL} , - {RECT_CONFIG, "proxy.config.net.tcp_congestion_control_in", RECD_STRING, "", RECU_DYNAMIC, RR_NULL, RECC_NULL, nullptr, RECA_NULL} + {RECT_CONFIG, "proxy.config.net.tcp_congestion_control_in", RECD_STRING, "", RECU_RESTART_TS, RR_NULL, RECC_NULL, nullptr, RECA_NULL} , - {RECT_CONFIG, "proxy.config.net.tcp_congestion_control_out", RECD_STRING, "", RECU_DYNAMIC, RR_NULL, RECC_NULL, nullptr, RECA_NULL} + {RECT_CONFIG, "proxy.config.net.tcp_congestion_control_out", RECD_STRING, "", RECU_RESTART_TS, RR_NULL, RECC_NULL, nullptr, RECA_NULL} , //############################################################################## @@ -909,7 +909,7 @@ static const RecordElement RecordsConfig[] = , {RECT_CONFIG, "proxy.config.dns.dedicated_thread", RECD_INT, "0", RECU_RESTART_TS, RR_NULL, RECC_NULL, "[0-1]", RECA_NULL} , - {RECT_CONFIG, "proxy.config.dns.connection.mode", RECD_INT, "0", RECU_RESTART_TS, RR_NULL, RECC_NULL, "[0-2]", RECA_NULL} + {RECT_CONFIG, "proxy.config.dns.connection_mode", RECD_INT, "0", RECU_RESTART_TS, RR_NULL, RECC_NULL, "[0-2]", RECA_NULL} , {RECT_CONFIG, "proxy.config.hostdb.ip_resolve", RECD_STRING, nullptr, RECU_RESTART_TS, RR_NULL, RECC_NULL, nullptr, RECA_NULL} , @@ -961,7 +961,7 @@ static const RecordElement RecordsConfig[] = {RECT_CONFIG, "proxy.config.hostdb.timed_round_robin", RECD_INT, "0", RECU_DYNAMIC, RR_NULL, RECC_NULL, nullptr, RECA_NULL} , // # how often should the hostdb be synced (seconds) - {RECT_CONFIG, "proxy.config.cache.hostdb.sync_frequency", RECD_INT, "120", RECU_RESTART_TS, RR_NULL, RECC_NULL, nullptr, RECA_NULL} + {RECT_CONFIG, "proxy.config.cache.hostdb.sync_frequency", RECD_INT, "0", RECU_RESTART_TS, RR_NULL, RECC_NULL, nullptr, RECA_NULL} , {RECT_CONFIG, "proxy.config.hostdb.host_file.path", RECD_STRING, nullptr, RECU_DYNAMIC, RR_NULL, RECC_NULL, nullptr, RECA_NULL} , @@ -1010,8 +1010,6 @@ static const RecordElement RecordsConfig[] = , {RECT_CONFIG, "proxy.config.log.max_space_mb_for_logs", RECD_INT, "25000", RECU_DYNAMIC, RR_NULL, RECC_STR, "^[0-9]+$", RECA_NULL} , - {RECT_CONFIG, "proxy.config.log.max_space_mb_for_orphan_logs", RECD_INT, "25", RECU_DYNAMIC, RR_NULL, RECC_STR, "^[0-9]+$", RECA_NULL} - , {RECT_CONFIG, "proxy.config.log.max_space_mb_headroom", RECD_INT, "1000", RECU_DYNAMIC, RR_NULL, RECC_STR, "^[0-9]+$", RECA_NULL} , {RECT_CONFIG, "proxy.config.log.hostname", RECD_STRING, "localhost", RECU_DYNAMIC, RR_NULL, RECC_NULL, nullptr, RECA_NULL} @@ -1022,23 +1020,7 @@ static const RecordElement RecordsConfig[] = , {RECT_CONFIG, "proxy.config.log.config.filename", RECD_STRING, "logging.yaml", RECU_DYNAMIC, RR_NULL, RECC_NULL, nullptr, RECA_NULL} , - {RECT_CONFIG, "proxy.config.log.collation_host", RECD_STRING, nullptr, RECU_DYNAMIC, RR_NULL, RECC_STR, "^[^[:space:]]*$", RECA_NULL} - , - {RECT_CONFIG, "proxy.config.log.collation_port", RECD_INT, "8085", RECU_DYNAMIC, RR_REQUIRED, RECC_INT, "[0-65535]", RECA_NULL} - , - {RECT_CONFIG, "proxy.config.log.collation_secret", RECD_STRING, "foobar", RECU_DYNAMIC, RR_NULL, RECC_STR, ".*", RECA_NULL} - , - {RECT_CONFIG, "proxy.config.log.collation_host_tagged", RECD_INT, "0", RECU_DYNAMIC, RR_NULL, RECC_INT, "[0-1]", RECA_NULL} - , - {RECT_CONFIG, "proxy.config.log.collation_retry_sec", RECD_INT, "5", RECU_DYNAMIC, RR_NULL, RECC_NULL, nullptr, RECA_NULL} - , - {RECT_CONFIG, "proxy.config.log.collation_max_send_buffers", RECD_INT, "16", RECU_DYNAMIC, RR_NULL, RECC_NULL, nullptr, RECA_NULL} - , - {RECT_CONFIG, "proxy.config.log.collation_preproc_threads", RECD_INT, "1", RECU_DYNAMIC, RR_REQUIRED, RECC_INT, "[1-128]", RECA_NULL} - , - {RECT_CONFIG, "proxy.config.log.collation_host_timeout", RECD_INT, "86390", RECU_NULL, RR_NULL, RECC_NULL, nullptr, RECA_NULL} - , - {RECT_CONFIG, "proxy.config.log.collation_client_timeout", RECD_INT, "86400", RECU_NULL, RR_NULL, RECC_NULL, nullptr, RECA_NULL} + {RECT_CONFIG, "proxy.config.log.preproc_threads", RECD_INT, "1", RECU_DYNAMIC, RR_REQUIRED, RECC_INT, "[1-128]", RECA_NULL} , {RECT_CONFIG, "proxy.config.log.rolling_enabled", RECD_INT, "1", RECU_DYNAMIC, RR_NULL, RECC_INT, "[0-4]", RECA_NULL} , @@ -1085,8 +1067,6 @@ static const RecordElement RecordsConfig[] = //# SSL Termination //# //############################################################################## - {RECT_CONFIG, "proxy.config.ssl.enabled", RECD_INT, "0", RECU_RESTART_TS, RR_NULL, RECC_INT, "[0-1]", RECA_NULL} - , {RECT_CONFIG, "proxy.config.ssl.server.session_ticket.enable", RECD_INT, "1", RECU_DYNAMIC, RR_NULL, RECC_INT, "[0-1]", RECA_NULL} , {RECT_CONFIG, "proxy.config.ssl.TLSv1", RECD_INT, "1", RECU_RESTART_TS, RR_NULL, RECC_INT, "[0-1]", RECA_NULL} @@ -1096,6 +1076,8 @@ static const RecordElement RecordsConfig[] = // Disable this when using some versions of OpenSSL that causes crashes. See TS-2355. {RECT_CONFIG, "proxy.config.ssl.TLSv1_2", RECD_INT, "1", RECU_RESTART_TS, RR_NULL, RECC_INT, "[0-1]", RECA_NULL} , + {RECT_CONFIG, "proxy.config.ssl.TLSv1_3", RECD_INT, "1", RECU_RESTART_TS, RR_NULL, RECC_INT, "[0-1]", RECA_NULL} + , // Client SSL protocols #if TS_USE_SSLV3_CLIENT @@ -1108,6 +1090,8 @@ static const RecordElement RecordsConfig[] = , {RECT_CONFIG, "proxy.config.ssl.client.TLSv1_2", RECD_INT, "1", RECU_RESTART_TS, RR_NULL, RECC_INT, "[0-1]", RECA_NULL} , + {RECT_CONFIG, "proxy.config.ssl.client.TLSv1_3", RECD_INT, "1", RECU_RESTART_TS, RR_NULL, RECC_INT, "[0-1]", RECA_NULL} + , {RECT_CONFIG, "proxy.config.ssl.server.cipher_suite", RECD_STRING, "ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:DHE-DSS-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-DSS-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES128-SHA:DHE-RSA-AES256-SHA256:DHE-DSS-AES256-SHA256:DHE-RSA-AES128-SHA256:DHE-DSS-AES128-SHA256:DHE-RSA-AES256-SHA:DHE-DSS-AES256-SHA:DHE-RSA-AES128-SHA:DHE-DSS-AES128-SHA:!aNULL:!eNULL:!EXPORT:!DES:!RC4:!MD5:!PSK:!aECDH:!EDH-DSS-DES-CBC3-SHA:!EDH-RSA-DES-CBC3-SHA:!KRB5-DES-CBC3-SHA", RECU_RESTART_TS, RR_NULL, RECC_NULL, nullptr, RECA_NULL} , {RECT_CONFIG, "proxy.config.ssl.client.cipher_suite", RECD_STRING, nullptr, RECU_RESTART_TS, RR_NULL, RECC_NULL, nullptr, RECA_NULL} @@ -1148,9 +1132,11 @@ static const RecordElement RecordsConfig[] = , {RECT_CONFIG, "proxy.config.ssl.client.private_key.path", RECD_STRING, TS_BUILD_SYSCONFDIR, RECU_DYNAMIC, RR_NULL, RECC_NULL, nullptr, RECA_NULL} , - {RECT_CONFIG, "proxy.config.ssl.client.CA.cert.filename", RECD_STRING, nullptr, RECU_RESTART_TS, RR_NULL, RECC_STR, "^[^[:space:]]*$", RECA_NULL} + {RECT_CONFIG, "proxy.config.ssl.client.CA.cert.filename", RECD_STRING, nullptr, RECU_DYNAMIC, RR_NULL, RECC_STR, "^[^[:space:]]*$", RECA_NULL} + , + {RECT_CONFIG, "proxy.config.ssl.client.CA.cert.path", RECD_STRING, TS_BUILD_SYSCONFDIR, RECU_DYNAMIC, RR_NULL, RECC_NULL, nullptr, RECA_NULL} , - {RECT_CONFIG, "proxy.config.ssl.client.CA.cert.path", RECD_STRING, TS_BUILD_SYSCONFDIR, RECU_RESTART_TS, RR_NULL, RECC_NULL, nullptr, RECA_NULL} + {RECT_CONFIG, "proxy.config.ssl.client.sni_policy", RECD_STRING, "host", RECU_RESTART_TS, RR_NULL, RECC_NULL, nullptr, RECA_NULL} , {RECT_CONFIG, "proxy.config.ssl.session_cache", RECD_INT, "2", RECU_RESTART_TS, RR_NULL, RECC_NULL, nullptr, RECA_NULL} , @@ -1176,16 +1162,6 @@ static const RecordElement RecordsConfig[] = , {RECT_CONFIG, "proxy.config.ssl.handshake_timeout_in", RECD_INT, "0", RECU_RESTART_TS, RR_NULL, RECC_INT, "[0-65535]", RECA_NULL} , - {RECT_CONFIG, "proxy.config.ssl.sni.map.enable", RECD_INT, "0", RECU_DYNAMIC, RR_NULL, RECC_INT, "[0-1]", RECA_NULL} - , - {RECT_CONFIG, "proxy.config.ssl.wire_trace_enabled", RECD_INT, "0", RECU_DYNAMIC, RR_NULL, RECC_INT, "[0-2]", RECA_NULL} - , - {RECT_CONFIG, "proxy.config.ssl.wire_trace_addr", RECD_STRING, nullptr , RECU_DYNAMIC, RR_NULL, RECC_IP, R"([0-255]\.[0-255]\.[0-255]\.[0-255])", RECA_NULL} - , - {RECT_CONFIG, "proxy.config.ssl.wire_trace_percentage", RECD_INT, "0", RECU_DYNAMIC, RR_NULL, RECC_INT, "[0-100]", RECA_NULL} - , - {RECT_CONFIG, "proxy.config.ssl.wire_trace_server_name", RECD_STRING, nullptr , RECU_DYNAMIC, RR_NULL, RECC_STR, ".*", RECA_NULL} - , {RECT_CONFIG, "proxy.config.ssl.cert.load_elevated", RECD_INT, "0", RECU_RESTART_TS, RR_NULL, RECC_INT, "[0-1]", RECA_READ_ONLY} , {RECT_CONFIG, "proxy.config.ssl.server.groups_list", RECD_STRING, nullptr, RECU_RESTART_TS, RR_NULL, RECC_NULL, nullptr, RECA_NULL} @@ -1210,10 +1186,12 @@ static const RecordElement RecordsConfig[] = // # Update period for stapling caches. 60s (1 min) by default. {RECT_CONFIG, "proxy.config.ssl.ocsp.update_period", RECD_INT, "60", RECU_DYNAMIC, RR_NULL, RECC_NULL, "^[0-9]+$", RECA_NULL} , - + // # Base path for OCSP prefetched responses + {RECT_CONFIG, "proxy.config.ssl.ocsp.response.path", RECD_STRING, TS_BUILD_SYSCONFDIR, RECU_RESTART_TS, RR_NULL, RECC_NULL, nullptr, RECA_NULL} + , //############################################################################## //# - //# Congifuration for TLSv1.3 and above + //# Configuration for TLSv1.3 and above //# //############################################################################## // The default value (nullptr) means the default value of TLS stack will be used. @@ -1243,7 +1221,7 @@ static const RecordElement RecordsConfig[] = , // Interim configuration setting for obeying keepalive requests on internal - // (PLuginVC) sessions. See TS-4960 and friends. + // (PluginVC) sessions. See TS-4960 and friends. {RECT_LOCAL, "proxy.config.http.keepalive_internal_vc", RECD_INT, "0", RECU_DYNAMIC, RR_NULL, RECC_INT, "[0-1]", RECA_NULL}, //############################################################################## @@ -1325,7 +1303,7 @@ static const RecordElement RecordsConfig[] = , {RECT_CONFIG, "proxy.config.http2.header_table_size", RECD_INT, "4096", RECU_DYNAMIC, RR_NULL, RECC_STR, "^[0-9]+$", RECA_NULL} , - {RECT_CONFIG, "proxy.config.http2.max_header_list_size", RECD_INT, "4294967295", RECU_DYNAMIC, RR_NULL, RECC_STR, "^[0-9]+$", RECA_NULL} + {RECT_CONFIG, "proxy.config.http2.max_header_list_size", RECD_INT, "131072", RECU_DYNAMIC, RR_NULL, RECC_STR, "^[0-9]+$", RECA_NULL} , {RECT_CONFIG, "proxy.config.http2.accept_no_activity_timeout", RECD_INT, "120", RECU_DYNAMIC, RR_NULL, RECC_STR, "^[0-9]+$", RECA_NULL} , @@ -1337,14 +1315,14 @@ static const RecordElement RecordsConfig[] = , {RECT_CONFIG, "proxy.config.http2.zombie_debug_timeout_in", RECD_INT, "0", RECU_DYNAMIC, RR_NULL, RECC_STR, "^[0-9]+$", RECA_NULL} , + {RECT_CONFIG, "proxy.config.http2.stream_error_rate_threshold", RECD_FLOAT, "0.1", RECU_DYNAMIC, RR_NULL, RECC_NULL, nullptr, RECA_NULL} + , //# Add LOCAL Records Here {RECT_LOCAL, "proxy.local.incoming_ip_to_bind", RECD_STRING, nullptr, RECU_NULL, RR_NULL, RECC_NULL, nullptr, RECA_NULL} , {RECT_LOCAL, "proxy.local.outgoing_ip_to_bind", RECD_STRING, nullptr, RECU_NULL, RR_NULL, RECC_NULL, nullptr, RECA_NULL} , - {RECT_LOCAL, "proxy.local.log.collation_mode", RECD_INT, "0", RECU_DYNAMIC, RR_NULL, RECC_INT, "[0-4]", RECA_NULL} - , //# Librecords based stats system (new as of v2.1.3) {RECT_CONFIG, "proxy.config.stat_api.max_stats_allowed", RECD_INT, "256", RECU_RESTART_TS, RR_NULL, RECC_INT, "[256-1000]", RECA_NULL} diff --git a/mgmt/Rollback.cc b/mgmt/Rollback.cc index e29910190e4..0f4af405d55 100644 --- a/mgmt/Rollback.cc +++ b/mgmt/Rollback.cc @@ -63,7 +63,7 @@ Rollback::Rollback(const char *fileName_, const char *configName_, bool root_acc numberBackups(0) { version_t highestSeen; // the highest backup version - ExpandingArray existVer(25, true); // Exsisting versions + ExpandingArray existVer(25, true); // Existing versions struct stat fileInfo; MgmtInt numBak; char *alarmMsg; @@ -72,11 +72,6 @@ Rollback::Rollback(const char *fileName_, const char *configName_, bool root_acc int testFD; // For open test int testErrno; // For open test - // In case the file is missing - char *highestSeenStr; - char *activeVerStr; - bool needZeroLength; - ink_assert(fileName_ != nullptr); // parent must not also have a parent @@ -128,26 +123,27 @@ Rollback::Rollback(const char *fileName_, const char *configName_, bool root_acc // if (statFile(ACTIVE_VERSION, &fileInfo) < 0) { // If we can't find an active version because there is not - // one, attempt to rollback to a previous verision if one exists + // one, attempt to rollback to a previous version if one exists // // If it does not, create a zero length file to prevent total havoc // if (errno == ENOENT) { + bool needZeroLength; mgmt_log("[RollBack::Rollback] Missing Configuration File: %s\n", fileName); if (highestSeen > 0) { - highestSeenStr = createPathStr(highestSeen); - activeVerStr = createPathStr(ACTIVE_VERSION); + char *highestSeenStr = createPathStr(highestSeen); + char *activeVerStr = createPathStr(ACTIVE_VERSION); if (rename(highestSeenStr, activeVerStr) < 0) { mgmt_log("[RollBack::Rollback] Automatic Rollback to prior version failed for %s : %s\n", fileName, strerror(errno)); needZeroLength = true; } else { - mgmt_log("[RollBack::Rollback] Automatic Rollback to version succeded for %s\n", fileName, strerror(errno)); + mgmt_log("[RollBack::Rollback] Automatic Rollback to version succeeded for %s\n", fileName, strerror(errno)); needZeroLength = false; highestSeen--; // Since we've made the highestVersion active - // remove it from the backup verision q + // remove it from the backup version q versionQ.remove(versionQ.tail); } ats_free(highestSeenStr); @@ -387,7 +383,7 @@ Rollback::internalUpdate(TextBuffer *buf, version_t newVersion, bool notifyChang // Check to see if the callee has specified a newVersion number // If the newVersion argument is less than zero, the callee - // is telling us to use the next version in squence + // is telling us to use the next version in sequence if (newVersion < 0) { newVersion = this->currentVersion + 1; if (incVersion) { @@ -473,7 +469,7 @@ Rollback::internalUpdate(TextBuffer *buf, version_t newVersion, bool notifyChang newBak->modTime = 0; versionQ.enqueue(newBak); } - // Update instance varibles + // Update instance variables this->numVersions++; this->currentVersion = newVersion; @@ -687,7 +683,7 @@ Rollback::findVersions_ml(ExpandingArray *listNames) version_t Rollback::extractVersionInfo(ExpandingArray *listNames, const char *testFileName) { - const char *currentVersionStr, *str; + const char *str; version_t version = INVALID_VERSION; // Check to see if the current entry is a rollback file @@ -700,7 +696,7 @@ Rollback::extractVersionInfo(ExpandingArray *listNames, const char *testFileName // Check for the underscore if (*(testFileName + fileNameLen) == '_') { // Check for the integer version number - currentVersionStr = str = testFileName + fileNameLen + 1; + const char *currentVersionStr = str = testFileName + fileNameLen + 1; for (; isdigit(*str) && *str != '\0'; str++) { ; @@ -714,12 +710,11 @@ Rollback::extractVersionInfo(ExpandingArray *listNames, const char *testFileName // Add info about version number and modTime if (listNames != nullptr) { struct stat fileInfo; - versionInfo *verInfo; if (statFile(version, &fileInfo) >= 0) { - verInfo = (versionInfo *)ats_malloc(sizeof(versionInfo)); - verInfo->version = version; - verInfo->modTime = fileInfo.st_mtime; + versionInfo *verInfo = (versionInfo *)ats_malloc(sizeof(versionInfo)); + verInfo->version = version; + verInfo->modTime = fileInfo.st_mtime; listNames->addEntry((void *)verInfo); } } @@ -736,7 +731,7 @@ Rollback::extractVersionInfo(ExpandingArray *listNames, const char *testFileName // Add wrapper to // version_t Rollback::findVersions_ml(ExpandingArray* listNames) // -// Puts the data in a queue rather than an ExpaningArray +// Puts the data in a queue rather than an ExpandingArray // version_t Rollback::findVersions_ml(Queue &q) @@ -744,7 +739,6 @@ Rollback::findVersions_ml(Queue &q) ExpandingArray versions(25, true); int num; versionInfo *foundVer; - versionInfo *addInfo; version_t highest; // Get the version info and sort it @@ -757,9 +751,9 @@ Rollback::findVersions_ml(Queue &q) foundVer = (versionInfo *)versions[i]; // We need to create our own copy so that // constructor gets run - addInfo = new versionInfo; - addInfo->version = foundVer->version; - addInfo->modTime = foundVer->modTime; + versionInfo *addInfo = new versionInfo; + addInfo->version = foundVer->version; + addInfo->modTime = foundVer->modTime; q.enqueue(addInfo); } @@ -866,7 +860,7 @@ Rollback::setLastModifiedTime() fileLastModified = TS_ARCHIVE_STAT_MTIME(fileInfo); return true; } else { - // We really shoudn't fail to stat the file since we just + // We really shouldn't fail to stat the file since we just // created it. If we do, just punt and just use the current // time. fileLastModified = (time(nullptr) - ink_timezone()) * 1000000000; @@ -878,14 +872,14 @@ Rollback::setLastModifiedTime() // // Called to check if the file has been changed // by the user. Timestamps are compared to see if a -// change occured +// change occurred // // If the file has been changed, a new version is rolled. -// The new current version and its predicessor will +// The new current version and its predecessor will // be the same in this case. While this is pointless, // for Rolling backward, we need to the version number // to up'ed so that WebFileEdit knows that the file has -// changed. Rolling a new verion also has the effect +// changed. Rolling a new version also has the effect // of creating a new timestamp // bool diff --git a/mgmt/Rollback.h b/mgmt/Rollback.h index 2ea74e4adf7..bdcaebde673 100644 --- a/mgmt/Rollback.h +++ b/mgmt/Rollback.h @@ -94,7 +94,7 @@ struct versionInfo { // // checkForUserUpdate() - compares the last known modification time // of the active version of the file with that files current modification -// time. Returns true if the file has been chaged manually or false +// time. Returns true if the file has been changed manually or false // if it hasn't // // versionTimeStamp(version_t) - returns the modification time (mtime) @@ -250,6 +250,6 @@ class Rollback Queue versionQ; // stores the backup version info }; -// qSort comptable function to sort versionInfo* +// qSort compatible function to sort versionInfo* // based on version number int versionCmp(const void *i1, const void *i2); diff --git a/mgmt/WebMgmtUtils.cc b/mgmt/WebMgmtUtils.cc index 474114cec17..81b17e07d57 100644 --- a/mgmt/WebMgmtUtils.cc +++ b/mgmt/WebMgmtUtils.cc @@ -100,11 +100,11 @@ varSetFromStr(const char *varName, const char *value) // bool varSetFloat(const char* varName, RecFloat value) // -// Sets the variable specifed by varName to value. varName +// Sets the variable specified by varName to value. varName // must be a RecFloat variable. No conversion is done for // other types unless convert is set to ture. In the case // of convert is ture, type conversion is perform if applicable. -// By default, convert is set to be false and can be overrided +// By default, convert is set to be false and can be overridden // when the function is called. // bool @@ -151,11 +151,11 @@ varSetFloat(const char *varName, RecFloat value, bool convert) // bool varSetCounter(const char* varName, RecCounter value) // -// Sets the variable specifed by varName to value. varName +// Sets the variable specified by varName to value. varName // must be an RecCounter variable. No conversion is done for // other types unless convert is set to ture. In the case // of convert is ture, type conversion is perform if applicable. -// By default, convert is set to be false and can be overrided +// By default, convert is set to be false and can be overridden // when the function is called. // bool @@ -201,11 +201,11 @@ varSetCounter(const char *varName, RecCounter value, bool convert) // bool varSetInt(const char* varName, RecInt value) // -// Sets the variable specifed by varName to value. varName +// Sets the variable specified by varName to value. varName // must be an RecInt variable. No conversion is done for // other types unless convert is set to ture. In the case // of convert is ture, type conversion is perform if applicable. -// By default, convert is set to be false and can be overrided +// By default, convert is set to be false and can be overridden // when the function is called. // bool @@ -251,7 +251,7 @@ varSetInt(const char *varName, RecInt value, bool convert) // bool varSetData(RecDataT varType, const char *varName, RecData value) // -// Sets the variable specifed by varName to value. value and varName +// Sets the variable specified by varName to value. value and varName // must be varType variables. // bool @@ -279,7 +279,7 @@ varSetData(RecDataT varType, const char *varName, RecData value) // // Sets the *value to value of the varName according varType. // -// return true if bufVal was succefully set +// return true if bufVal was successfully set // and false otherwise // bool @@ -296,7 +296,7 @@ varDataFromName(RecDataT varType, const char *varName, RecData *value) // // Sets the *value to value of the varName. // -// return true if bufVal was succefully set +// return true if bufVal was successfully set // and false otherwise // bool @@ -345,7 +345,7 @@ varCounterFromName(const char *varName, RecCounter *value) // // Sets the *value to value of the varName. // -// return true if bufVal was succefully set +// return true if bufVal was successfully set // and false otherwise // bool @@ -395,7 +395,7 @@ varFloatFromName(const char *varName, RecFloat *value) // // Sets the *value to value of the varName. // -// return true if bufVal was succefully set +// return true if bufVal was successfully set // and false otherwise // bool @@ -519,7 +519,6 @@ bytesFromInt(RecInt bytes, char *bufVal) const int64_t gb = 1073741824; const long int mb = 1048576; const long int kb = 1024; - int bytesP; double unitBytes; if (bytes >= gb) { @@ -531,7 +530,7 @@ bytesFromInt(RecInt bytes, char *bufVal) // has plenty of precision for a regular int // and saves from 64 bit arithmetic which may // be expensive on some processors - bytesP = (int)bytes; + int bytesP = (int)bytes; if (bytesP >= mb) { unitBytes = bytes / (double)mb; snprintf(bufVal, 15, "%.1f MB", unitBytes); @@ -549,12 +548,12 @@ bytesFromInt(RecInt bytes, char *bufVal) // Sets the bufVal string to the value of the local manager // named by varName. bufLen is size of bufVal // -// return true if bufVal was succefully set +// return true if bufVal was successfully set // and false otherwise // // EVIL ALERT: overviewRecord::varStrFromName is extremely // similar to this function except in how it gets it's -// data. Changes to this fuction must be propogated +// data. Changes to this function must be propagated // to its twin. Cut and Paste sucks but there is not // an easy way to merge the functions // @@ -934,7 +933,7 @@ setHostnameVar() return 0; } -// void appendDefautDomain(char* hostname, int bufLength) +// void appendDefaultDomain(char* hostname, int bufLength) // // Appends the pasted in hostname with the default // domain if the hostname is an unqualified name @@ -950,7 +949,7 @@ appendDefaultDomain(char *hostname, int bufLength) { int len = strlen(hostname); const char msg[] = "Nodes will be know by their unqualified host name"; - static int error_before = 0; // Race ok since effect is multple error msg + static int error_before = 0; // Race ok since effect is multiple error msg ink_assert(len < bufLength); ink_assert(bufLength >= 64); @@ -1041,9 +1040,6 @@ recordRegexCheck(const char *pattern, const char *value) bool recordRangeCheck(const char *pattern, const char *value) { - int l_limit; - int u_limit; - int val; char *p = (char *)pattern; Tokenizer dashTok("-"); @@ -1052,9 +1048,9 @@ recordRangeCheck(const char *pattern, const char *value) p++; } // skip to '[' if (dashTok.Initialize(++p, COPY_TOKS) == 2) { - l_limit = atoi(dashTok[0]); - u_limit = atoi(dashTok[1]); - val = atoi(value); + int l_limit = atoi(dashTok[0]); + int u_limit = atoi(dashTok[1]); + int val = atoi(value); if (val >= l_limit && val <= u_limit) { return true; } @@ -1075,12 +1071,11 @@ recordIPCheck(const char *pattern, const char *value) Tokenizer dotTok1("."); Tokenizer dotTok2("."); - int i; check = true; if (recordRegexCheck(range_pattern, pattern) && recordRegexCheck(ip_pattern, value)) { if (dotTok1.Initialize((char *)pattern, COPY_TOKS) == 4 && dotTok2.Initialize((char *)value, COPY_TOKS) == 4) { - for (i = 0; i < 4 && check; i++) { + for (int i = 0; i < 4 && check; i++) { if (!recordRangeCheck(dotTok1[i], dotTok2[i])) { check = false; } diff --git a/mgmt/WebMgmtUtils.h b/mgmt/WebMgmtUtils.h index 9112785f8d5..26cb01d915a 100644 --- a/mgmt/WebMgmtUtils.h +++ b/mgmt/WebMgmtUtils.h @@ -62,9 +62,9 @@ bool varFloatFromName(const char *varName, RecFloat *value); bool varCounterFromName(const char *varName, RecCounter *value); bool varDataFromName(RecDataT varType, const char *varName, RecData *value); -// No conversion done. varName must represnt a value of the appropriate +// No conversion done. varName must represent a value of the appropriate // type -// Default arguement "convert" added to allow great flexiblity in type checking +// Default argument "convert" added to allow great flexibility in type checking bool varSetInt(const char *varName, RecInt value, bool convert = false); bool varSetCounter(const char *varName, RecCounter value, bool convert = false); bool varSetFloat(const char *varName, RecFloat value, bool convert = false); diff --git a/mgmt/api/APITestCliRemote.cc b/mgmt/api/APITestCliRemote.cc index 80f8cca14e9..aa1e7a224c5 100644 --- a/mgmt/api/APITestCliRemote.cc +++ b/mgmt/api/APITestCliRemote.cc @@ -126,7 +126,6 @@ void print_string_list(TSStringList list) { int i, count, buf_pos = 0; - char *str; char buf[1000]; if (!list) { @@ -134,7 +133,7 @@ print_string_list(TSStringList list) } count = TSStringListLen(list); for (i = 0; i < count; i++) { - str = TSStringListDequeue(list); + char *str = TSStringListDequeue(list); snprintf(buf + buf_pos, sizeof(buf) - buf_pos, "%s,", str); buf_pos = strlen(buf); TSStringListEnqueue(list, str); @@ -149,12 +148,11 @@ void print_int_list(TSIntList list) { int i, count, buf_pos = 0; - int *elem; char buf[1000]; count = TSIntListLen(list); for (i = 0; i < count; i++) { - elem = TSIntListDequeue(list); + int *elem = TSIntListDequeue(list); snprintf(buf + buf_pos, sizeof(buf) - buf_pos, "%d:", *elem); buf_pos = strlen(buf); TSIntListEnqueue(list, elem); @@ -504,7 +502,6 @@ test_rec_get(char *rec_name) void test_record_get_mlt() { - TSRecordEle *rec_ele; TSStringList name_list; TSList rec_list; int i, num; @@ -547,7 +544,7 @@ test_record_get_mlt() } for (i = 0; i < num; i++) { - rec_ele = (TSRecordEle *)TSListDequeue(rec_list); + TSRecordEle *rec_ele = (TSRecordEle *)TSListDequeue(rec_list); if (!rec_ele) { printf("ERROR\n"); break; diff --git a/mgmt/api/CoreAPI.cc b/mgmt/api/CoreAPI.cc index d75d7669ff1..4959513415d 100644 --- a/mgmt/api/CoreAPI.cc +++ b/mgmt/api/CoreAPI.cc @@ -57,7 +57,7 @@ extern FileManager *configFiles; // global in traffic_manager /*------------------------------------------------------------------------- * Init *------------------------------------------------------------------------- - * performs any necesary initializations for the local API client, + * performs any necessary initializations for the local API client, * eg. set up global structures; called by the TSMgmtAPI::TSInit() */ TSMgmtError @@ -79,7 +79,7 @@ Init(const char * /* socket_path ATS_UNUSED */, TSInitOptionT options) /*------------------------------------------------------------------------- * Terminate *------------------------------------------------------------------------- - * performs any necesary cleanup of global structures, etc, + * performs any necessary cleanup of global structures, etc, * for the local API client, */ TSMgmtError @@ -112,7 +112,7 @@ ProxyShutdown() lmgmt->processShutdown(false /* only shut down the proxy*/); - // Wait for awhile for shtudown to happen + // Wait for awhile for shutdown to happen do { mgmt_sleep_sec(1); i++; @@ -559,7 +559,7 @@ MgmtRecordGet(const char *rec_name, TSRecordEle *rec_ele) Debug("RecOp", "[MgmtRecordGet] Get String Var %s = %s", rec_ele->rec_name, rec_ele->valueT.string_val); break; - default: // UNKOWN TYPE + default: // UNKNOWN TYPE Debug("RecOp", "[MgmtRecordGet] Get Failed : %d is Unknown Var type %s", rec_type, rec_name); return TS_ERR_FAIL; } @@ -789,9 +789,6 @@ EventResolve(const char *event_name) TSMgmtError ActiveEventGetMlt(LLQ *active_events) { - int event_id; - char *event_name; - if (!active_events) { return TS_ERR_PARAMS; } @@ -804,8 +801,8 @@ ActiveEventGetMlt(LLQ *active_events) // iterate through hash-table and insert event_name's into active_events list for (auto &&it : event_ht) { // convert key to int; insert into llQ - event_id = ink_atoi(it.first.c_str()); - event_name = get_event_name(event_id); + int event_id = ink_atoi(it.first.c_str()); + char *event_name = get_event_name(event_id); if (event_name) { if (!enqueue(active_events, event_name)) { // returns true if successful return TS_ERR_FAIL; @@ -920,7 +917,3 @@ StatsReset(const char *name) lmgmt->clearStats(name); return TS_ERR_OKAY; } - -/*------------------------------------------------------------- - * rmserver.cfg - *-------------------------------------------------------------*/ diff --git a/mgmt/api/CoreAPIRemote.cc b/mgmt/api/CoreAPIRemote.cc index 8406d78b588..fbcdc42a43e 100644 --- a/mgmt/api/CoreAPIRemote.cc +++ b/mgmt/api/CoreAPIRemote.cc @@ -263,7 +263,7 @@ Terminate() // cancel the listening socket thread // it's important to call this before setting paths to NULL because the - // socket_test_thread actually will try to reconnect() and this funntion + // socket_test_thread actually will try to reconnect() and this function // will seg fault if the socket paths are NULL while it is connecting; // the thread will be cancelled at a cancellation point in the // socket_test_thread, eg. sleep @@ -484,7 +484,7 @@ StorageDeviceCmdOffline(const char *dev) } /*------------------------------------------------------------------------- - * LIfecycle Alert + * Lifecycle Alert *------------------------------------------------------------------------- * Send alert to plugins */ diff --git a/mgmt/api/CoreAPIShared.cc b/mgmt/api/CoreAPIShared.cc index 84462583d92..5ee8fd8f9c8 100644 --- a/mgmt/api/CoreAPIShared.cc +++ b/mgmt/api/CoreAPIShared.cc @@ -77,7 +77,7 @@ parseHTTPResponse(char *buffer, char **header, int *hdr_size, char **body, int * } /* readHTTPResponse - * - read from an openned socket to memory-allocated buffer and close the + * - read from an opened socket to memory-allocated buffer and close the * socket regardless success or failure. * INPUT: sock -- the socket to read the response from * buffer -- the buffer to be filled with the HTTP response @@ -132,7 +132,7 @@ readHTTPResponse(int sock, char *buffer, int bufsize, uint64_t timeout) } /* sendHTTPRequest - * - Compose a HTTP GET request and sent it via an openned socket. + * - Compose a HTTP GET request and sent it via an opened socket. * INPUT: sock -- the socket to send the message to * req -- the request to send * OUTPUT: bool -- true if everything went well. false otherwise (and sock is @@ -182,7 +182,6 @@ sendHTTPRequest(int sock, char *req, uint64_t timeout) return TS_ERR_NET_WRITE; } -/* Modified from TrafficCop.cc (open_socket) */ int connectDirect(const char *host, int port, uint64_t /* timeout ATS_UNUSED */) { @@ -241,7 +240,6 @@ connectDirect(const char *host, int port, uint64_t /* timeout ATS_UNUSED */) return -1; } /* connectDirect */ -/* COPIED direclty form TrafficCop.cc */ static int poll_read(int fd, int timeout) { @@ -336,7 +334,7 @@ get_event_id(const char *event_name) /********************************************************************** * get_event_id * - * Purpose: based on alarm_id, determine the corresonding alarm name + * Purpose: based on alarm_id, determine the corresponding alarm name * Note: allocates memory for the name returned *********************************************************************/ char * diff --git a/mgmt/api/CoreAPIShared.h b/mgmt/api/CoreAPIShared.h index c5b6c0d4895..eaa86820768 100644 --- a/mgmt/api/CoreAPIShared.h +++ b/mgmt/api/CoreAPIShared.h @@ -65,7 +65,7 @@ // used by TSReadFromUrl #define HTTP_DIVIDER "\r\n\r\n" -#define URL_BUFSIZE 65536 // the max. lenght of URL obtainable (in bytes) +#define URL_BUFSIZE 65536 // the max. length of URL obtainable (in bytes) #define URL_TIMEOUT 5000 // the timeout value for send/recv HTTP in ms #define HTTP_PORT 80 #define BUFSIZE 1024 diff --git a/mgmt/api/EventCallback.cc b/mgmt/api/EventCallback.cc index 2fc463a4cfd..59bdf930186 100644 --- a/mgmt/api/EventCallback.cc +++ b/mgmt/api/EventCallback.cc @@ -183,7 +183,6 @@ TSMgmtError cb_table_register(CallbackTable *cb_table, const char *event_name, TSEventSignalFunc func, void *data, bool *first_cb) { bool first_time = false; - int id; EventCallbackT *event_cb; // create new EventCallbackT EACH TIME enqueue // the data and event_name can be NULL @@ -211,8 +210,7 @@ cb_table_register(CallbackTable *cb_table, const char *event_name, TSEventSignal enqueue(i, event_cb); } } else { // register callback for specific alarm - // printf("[EventSignalCbRegister] Register callback for %s\n", event_name); - id = get_event_id(event_name); + int id = get_event_id(event_name); if (id != -1) { if (!cb_table->event_callback_l[id]) { cb_table->event_callback_l[id] = create_queue(); diff --git a/mgmt/api/EventControlMain.cc b/mgmt/api/EventControlMain.cc index 4fc2668c87a..b4859ebb399 100644 --- a/mgmt/api/EventControlMain.cc +++ b/mgmt/api/EventControlMain.cc @@ -177,11 +177,8 @@ delete_event_queue(LLQ *q) return; } - // now for every element, dequeue and free - TSMgmtEvent *ele; - while (!queue_is_empty(q)) { - ele = (TSMgmtEvent *)dequeue(q); + TSMgmtEvent *ele = (TSMgmtEvent *)dequeue(q); ats_free(ele); } @@ -257,7 +254,6 @@ event_callback_main(void *arg) fd_set selectFDs; // for select call EventClientT *client_entry; // an entry of fd to alarms mapping - int fds_ready; // return value for select go here struct timeval timeout; while (true) { @@ -280,7 +276,7 @@ event_callback_main(void *arg) } // select call - timeout is set so we can check events at regular intervals - fds_ready = mgmt_select(FD_SETSIZE, &selectFDs, (fd_set *)nullptr, (fd_set *)nullptr, &timeout); + int fds_ready = mgmt_select(FD_SETSIZE, &selectFDs, (fd_set *)nullptr, (fd_set *)nullptr, &timeout); // check return if (fds_ready > 0) { diff --git a/mgmt/api/INKMgmtAPI.cc b/mgmt/api/INKMgmtAPI.cc index 15d2caf8d78..7b0626c4cd6 100644 --- a/mgmt/api/INKMgmtAPI.cc +++ b/mgmt/api/INKMgmtAPI.cc @@ -147,7 +147,6 @@ tsapi bool TSListIsValid(TSList l) { int i, len; - void *ele; if (!l) { return false; @@ -155,7 +154,7 @@ TSListIsValid(TSList l) len = queue_len((LLQ *)l); for (i = 0; i < len; i++) { - ele = (void *)dequeue((LLQ *)l); + void *ele = (void *)dequeue((LLQ *)l); if (!ele) { return false; } @@ -175,15 +174,13 @@ TSStringListCreate() tsapi void TSStringListDestroy(TSStringList strl) { - char *str; - if (!strl) { return; } /* dequeue each element and free it */ while (!queue_is_empty((LLQ *)strl)) { - str = (char *)dequeue((LLQ *)strl); + char *str = (char *)dequeue((LLQ *)strl); ats_free(str); } @@ -246,7 +243,6 @@ tsapi bool TSStringListIsValid(TSStringList strl) { int i, len; - char *str; if (!strl) { return false; @@ -254,7 +250,7 @@ TSStringListIsValid(TSStringList strl) len = queue_len((LLQ *)strl); for (i = 0; i < len; i++) { - str = (char *)dequeue((LLQ *)strl); + char *str = (char *)dequeue((LLQ *)strl); if (!str) { return false; } @@ -274,15 +270,13 @@ TSIntListCreate() tsapi void TSIntListDestroy(TSIntList intl) { - int *iPtr; - if (!intl) { return; } /* dequeue each element and free it */ while (!queue_is_empty((LLQ *)intl)) { - iPtr = (int *)dequeue((LLQ *)intl); + int *iPtr = (int *)dequeue((LLQ *)intl); ats_free(iPtr); } @@ -518,7 +512,7 @@ TSRecordGetString(const char *rec_name, TSString *string_val) *------------------------------------------------------------------------- * Purpose: Retrieves list of record values specified in the rec_names list * Input: rec_names - list of record names to retrieve - * rec_vals - queue of TSRecordEle* that correspons to rec_names + * rec_vals - queue of TSRecordEle* that corresponds to rec_names * Output: If at any point, while retrieving one of the records there's a * a failure then the entire process is aborted, all the allocated * TSRecordEle's are deallocated and TS_ERR_FAIL is returned. @@ -534,8 +528,6 @@ TSRecordGetString(const char *rec_name, TSString *string_val) tsapi TSMgmtError TSRecordGetMlt(TSStringList rec_names, TSList rec_vals) { - TSRecordEle *ele; - char *rec_name; int num_recs, i, j; TSMgmtError ret; @@ -545,12 +537,12 @@ TSRecordGetMlt(TSStringList rec_names, TSList rec_vals) num_recs = queue_len((LLQ *)rec_names); for (i = 0; i < num_recs; i++) { - rec_name = (char *)dequeue((LLQ *)rec_names); // remove name from list + char *rec_name = (char *)dequeue((LLQ *)rec_names); // remove name from list if (!rec_name) { return TS_ERR_PARAMS; // NULL is invalid record name } - ele = TSRecordEleCreate(); + TSRecordEle *ele = TSRecordEleCreate(); ret = MgmtRecordGet(rec_name, ele); enqueue((LLQ *)rec_names, rec_name); // return name to list @@ -632,7 +624,6 @@ tsapi TSMgmtError TSRecordSetMlt(TSList rec_list, TSActionNeedT *action_need) { int num_recs, ret, i; - TSRecordEle *ele; TSMgmtError status = TS_ERR_OKAY; TSActionNeedT top_action_req = TS_ACTION_UNDEFINED; @@ -643,7 +634,7 @@ TSRecordSetMlt(TSList rec_list, TSActionNeedT *action_need) num_recs = queue_len((LLQ *)rec_list); for (i = 0; i < num_recs; i++) { - ele = (TSRecordEle *)dequeue((LLQ *)rec_list); + TSRecordEle *ele = (TSRecordEle *)dequeue((LLQ *)rec_list); if (ele) { switch (ele->rec_type) { case TS_REC_INT: @@ -667,7 +658,7 @@ TSRecordSetMlt(TSList rec_list, TSActionNeedT *action_need) } // keep track of most severe action; reset if needed - // the TSACtionNeedT should be listed such that most severe actions have + // the TSActionNeedT should be listed such that most severe actions have // a lower number (so most severe action == 0) if (*action_need < top_action_req) { // a more severe action top_action_req = *action_need; diff --git a/mgmt/api/Makefile.am b/mgmt/api/Makefile.am index ae9fa950e7c..0ddd113b49d 100644 --- a/mgmt/api/Makefile.am +++ b/mgmt/api/Makefile.am @@ -83,7 +83,7 @@ traffic_api_cli_remote_LDADD = \ $(top_builddir)/src/tscpp/util/libtscpputil.la \ $(top_builddir)/src/tscore/libtscore.la \ $(top_builddir)/src/tscpp/util/libtscpputil.la \ - @LIBTCL@ @OPENSSL_LIBS@ + @OPENSSL_LIBS@ clang-tidy-local: $(DIST_SOURCES) $(CXX_Clang_Tidy) diff --git a/mgmt/api/NetworkMessage.cc b/mgmt/api/NetworkMessage.cc index de2e7fa7c66..a9ba89b36ee 100644 --- a/mgmt/api/NetworkMessage.cc +++ b/mgmt/api/NetworkMessage.cc @@ -36,7 +36,7 @@ struct NetCmdOperation { const MgmtMarshallType fields[MAX_OPERATION_FIELDS]; }; -// Requests always begin with a OpType, followed by aditional fields. +// Requests always begin with a OpType, followed by additional fields. static const struct NetCmdOperation requests[] = { /* RECORD_SET */ {3, {MGMT_MARSHALL_INT, MGMT_MARSHALL_STRING, MGMT_MARSHALL_STRING}}, /* RECORD_GET */ {2, {MGMT_MARSHALL_INT, MGMT_MARSHALL_STRING}}, diff --git a/mgmt/api/NetworkMessage.h b/mgmt/api/NetworkMessage.h index 99c979b007b..70f15e72a11 100644 --- a/mgmt/api/NetworkMessage.h +++ b/mgmt/api/NetworkMessage.h @@ -28,7 +28,7 @@ #define REMOTE_DELIM ':' #define REMOTE_DELIM_STR ":" -#define MAX_CONN_TRIES 10 // maximum number of attemps to reconnect to TM +#define MAX_CONN_TRIES 10 // maximum number of attempts to reconnect to TM // the possible operations or msg types sent from remote client to TM enum class OpType : MgmtMarshallInt { @@ -75,7 +75,7 @@ struct mgmt_message_sender { TSMgmtError send_mgmt_request(const mgmt_message_sender &snd, OpType optype, ...); TSMgmtError send_mgmt_request(int fd, OpType optype, ...); -// Marshall and send an error respose for this operation type. +// Marshall and send an error response for this operation type. TSMgmtError send_mgmt_error(int fd, OpType op, TSMgmtError error); // Parse a request message from a buffer. diff --git a/mgmt/api/NetworkUtilsRemote.cc b/mgmt/api/NetworkUtilsRemote.cc index 4c75f802b69..e5c49841425 100644 --- a/mgmt/api/NetworkUtilsRemote.cc +++ b/mgmt/api/NetworkUtilsRemote.cc @@ -96,7 +96,7 @@ socket_test(int fd) * requests & issues out responses and alerts * 1) create and set the client socket_fd; connect to TM * 2) create and set the client's event_socket_fd; connect to TM - * output: TS_ERR_OKAY - if both sockets sucessfully connect to TM + * output: TS_ERR_OKAY - if both sockets successfully connect to TM * TS_ERR_NET_ESTABLISH - at least one unsuccessful connection * notes: If connection breaks it is responsibility of client to reconnect * otherwise traffic server will assume mgmt stopped request and @@ -209,11 +209,11 @@ disconnect() /*************************************************************************** * reconnect * - * purpose: reconnects to TM (eg. when TM restarts); does all the necesarry + * purpose: reconnects to TM (eg. when TM restarts); does all the necessary * set up for reconnection * input: None * output: TS_ERR_FAIL, TS_ERR_OKAY - * notes: necessarry events for a new client-TM connection: + * notes: necessary events for a new client-TM connection: * 1) get new socket_fd using old socket_path by calling connect() * 2) relaunch event_poll_thread_main with new socket_fd * 3) re-notify TM of all the client's registered callbacks by send msg @@ -239,7 +239,7 @@ reconnect() // relaunch a new event thread since socket_fd changed if (0 == (ts_init_options & TS_MGMT_OPT_NO_EVENTS)) { ink_thread_create(&ts_event_thread, event_poll_thread_main, &event_socket_fd, 0, 0, nullptr); - // reregister the callbacks on the TM side for this new client connection + // re-register the callbacks on the TM side for this new client connection if (remote_event_callbacks) { err = send_register_all_callbacks(event_socket_fd, remote_event_callbacks); if (err != TS_ERR_OKAY) { // problem establishing connection @@ -507,7 +507,6 @@ send_register_all_callbacks(int fd, CallbackTable *cb_table) TSMgmtError send_unregister_all_callbacks(int fd, CallbackTable *cb_table) { - int event_id; LLQ *events_with_cb; // list of events with at least one callback int reg_callback[NUM_EVENTS]; TSMgmtError err, send_err = TS_ERR_FAIL; @@ -525,7 +524,7 @@ send_unregister_all_callbacks(int fd, CallbackTable *cb_table) int num_events = queue_len(events_with_cb); // iterate through the LLQ and mark events that have a callback for (int i = 0; i < num_events; i++) { - event_id = *(int *)dequeue(events_with_cb); + int event_id = *(int *)dequeue(events_with_cb); reg_callback[event_id] = 1; // mark the event as having a callback } delete_queue(events_with_cb); diff --git a/mgmt/api/NetworkUtilsRemote.h b/mgmt/api/NetworkUtilsRemote.h index 31e8734ba57..f8413dd7c21 100644 --- a/mgmt/api/NetworkUtilsRemote.h +++ b/mgmt/api/NetworkUtilsRemote.h @@ -35,6 +35,7 @@ #pragma once #include "mgmtapi.h" +#include "ts/apidefs.h" #include "NetworkMessage.h" #include "EventCallback.h" @@ -55,7 +56,8 @@ void set_socket_paths(const char *path); * the client connection information stored in the variables in * NetworkUtilsRemote.cc */ -TSMgmtError ts_connect(); /* TODO: update documenation, Renamed due to conflict with connect() in on some platforms*/ +TSMgmtError +ts_connect(); /* TODO: update documentation, Renamed due to conflict with connect() in on some platforms*/ TSMgmtError disconnect(); TSMgmtError reconnect(); TSMgmtError reconnect_loop(int num_attempts); diff --git a/mgmt/api/TSControlMain.cc b/mgmt/api/TSControlMain.cc index 2f59f53227b..b9f49a13dff 100644 --- a/mgmt/api/TSControlMain.cc +++ b/mgmt/api/TSControlMain.cc @@ -133,7 +133,6 @@ ts_ctrl_main(void *arg) fd_set selectFDs; // for select call ClientT *client_entry; // an entry of fd to alarms mapping - int fds_ready; // stores return value for select struct timeval timeout; // loops until TM dies; waits for and processes requests from clients @@ -158,7 +157,7 @@ ts_ctrl_main(void *arg) } // select call - timeout is set so we can check events at regular intervals - fds_ready = mgmt_select(FD_SETSIZE, &selectFDs, (fd_set *)nullptr, (fd_set *)nullptr, &timeout); + int fds_ready = mgmt_select(FD_SETSIZE, &selectFDs, (fd_set *)nullptr, (fd_set *)nullptr, &timeout); // check if have any connections or requests if (fds_ready > 0) { @@ -829,7 +828,7 @@ handle_host_status_down(int fd, void *req, size_t reqlen) /************************************************************************** * handle_api_ping * - * purpose: handles the API_PING messaghat is sent by API clients to keep + * purpose: handles the API_PING message that is sent by API clients to keep * the management socket alive * output: TS_ERR_xx. There is no response message. *************************************************************************/ diff --git a/mgmt/api/include/mgmtapi.h b/mgmt/api/include/mgmtapi.h index 588592b7611..c88c0dff295 100644 --- a/mgmt/api/include/mgmtapi.h +++ b/mgmt/api/include/mgmtapi.h @@ -432,7 +432,7 @@ tsapi TSMgmtError TSHostStatusSetUp(const char *host_name, int down_time, const tsapi TSMgmtError TSHostStatusSetDown(const char *host_name, int down_time, const char *reason); /*--- statistics operations -----------------------------------------------*/ /* TSStatsReset: sets all the statistics variables to their default values - * Outpue: TSErrr + * Output: TSMgmtError */ tsapi TSMgmtError TSStatsReset(const char *name); diff --git a/mgmt/utils/MgmtMarshall.cc b/mgmt/utils/MgmtMarshall.cc index eb2358600c3..5152da9cc41 100644 --- a/mgmt/utils/MgmtMarshall.cc +++ b/mgmt/utils/MgmtMarshall.cc @@ -1,6 +1,6 @@ /** @file - Managment packet marshalling. + Management packet marshalling. @section license License @@ -42,6 +42,7 @@ data_is_nul_terminated(const MgmtMarshallData *data) { const char *str = (const char *)(data->ptr); + ink_assert(str); if (str[data->len - 1] != '\0') { return false; } diff --git a/mgmt/utils/MgmtMarshall.h b/mgmt/utils/MgmtMarshall.h index 11c3d9222d6..91bc257bd69 100644 --- a/mgmt/utils/MgmtMarshall.h +++ b/mgmt/utils/MgmtMarshall.h @@ -1,6 +1,6 @@ /** @file - Managment packet marshalling. + Management packet marshalling. @section license License diff --git a/mgmt/utils/MgmtSocket.cc b/mgmt/utils/MgmtSocket.cc index 5bd6bb014a1..2ad3fc6c6c8 100644 --- a/mgmt/utils/MgmtSocket.cc +++ b/mgmt/utils/MgmtSocket.cc @@ -183,7 +183,7 @@ mgmt_select(int nfds, fd_set *readfds, fd_set *writefds, fd_set *errorfds, struc // Note: Linux select() has slight different semantics. From the // man page: "On Linux, timeout is modified to reflect the amount of // time not slept; most other implementations do not do this." -// Linux select() can also return ENOMEM, so we espeically need to +// Linux select() can also return ENOMEM, so we especially need to // protect the call with the transient error retry loop. // Fortunately, because of the Linux timeout handling, our // mgmt_select call will still timeout correctly, rather than diff --git a/mgmt/utils/MgmtSocket.h b/mgmt/utils/MgmtSocket.h index 99ecd88e952..46eb242b548 100644 --- a/mgmt/utils/MgmtSocket.h +++ b/mgmt/utils/MgmtSocket.h @@ -94,7 +94,7 @@ int mgmt_write_timeout(int fd, int sec, int usec); int mgmt_read_timeout(int fd, int sec, int usec); // Do we support passing Unix domain credentials on this platform? -bool mgmt_has_peereid(void); +bool mgmt_has_peereid(); // Get the Unix domain peer credentials. int mgmt_get_peereid(int fd, uid_t *euid, gid_t *egid); diff --git a/mgmt/utils/MgmtUtils.cc b/mgmt/utils/MgmtUtils.cc index 57ad7ed737d..3ac6642e629 100644 --- a/mgmt/utils/MgmtUtils.cc +++ b/mgmt/utils/MgmtUtils.cc @@ -53,11 +53,11 @@ mgmt_use_syslog() int mgmt_readline(int soc, char *buf, int maxlen) { - int n = 0, rc; + int n = 0; char c; for (; n < maxlen; n++) { - rc = read_socket(soc, &c, 1); + int rc = read_socket(soc, &c, 1); if (rc == 1) { *buf++ = c; if (c == '\n') { @@ -105,12 +105,12 @@ mgmt_readline(int soc, char *buf, int maxlen) int mgmt_writeline(int soc, const char *data, int nbytes) { - int nleft, nwritten, n; + int nleft, n; const char *tmp = data; nleft = nbytes; while (nleft > 0) { - nwritten = write_socket(soc, tmp, nleft); + int nwritten = write_socket(soc, tmp, nleft); if (nwritten <= 0) { /* Error or nothing written */ return nwritten; } @@ -141,12 +141,11 @@ mgmt_writeline(int soc, const char *data, int nbytes) int mgmt_read_pipe(int fd, char *buf, int bytes_to_read) { - int err = 0; char *p = buf; int bytes_read = 0; while (bytes_to_read > 0) { - err = read_socket(fd, p, bytes_to_read); + int err = read_socket(fd, p, bytes_to_read); if (err == 0) { return err; } else if (err < 0) { @@ -183,12 +182,11 @@ mgmt_read_pipe(int fd, char *buf, int bytes_to_read) int mgmt_write_pipe(int fd, char *buf, int bytes_to_write) { - int err = 0; char *p = buf; int bytes_written = 0; while (bytes_to_write > 0) { - err = write_socket(fd, p, bytes_to_write); + int err = write_socket(fd, p, bytes_to_write); if (err == 0) { return err; } else if (err < 0) { @@ -272,7 +270,6 @@ void mgmt_fatal(const int lerrno, const char *message_format, ...) { va_list ap; - char extended_format[4096], message[4096]; va_start(ap, message_format); @@ -283,6 +280,7 @@ mgmt_fatal(const int lerrno, const char *message_format, ...) FatalV(message_format, ap); } else { + char extended_format[4096], message[4096]; snprintf(extended_format, sizeof(extended_format), "FATAL ==> %s", message_format); vsprintf(message, extended_format, ap); @@ -336,7 +334,7 @@ mgmt_getAddrForIntr(char *intrName, sockaddr *addr, int *mtu) int fakeSocket; // a temporary socket to pass to ioctl struct ifconf ifc; // ifconf information char *ifbuf; // ifconf buffer - struct ifreq *ifr, *ifend; // pointer to individual inferface info + struct ifreq *ifr, *ifend; // pointer to individual interface info int lastlen; int len; diff --git a/plugins/Makefile.am b/plugins/Makefile.am index e1d0dadc26d..c00a4fb10bf 100644 --- a/plugins/Makefile.am +++ b/plugins/Makefile.am @@ -58,6 +58,7 @@ include experimental/buffer_upload/Makefile.inc include experimental/cache_range_requests/Makefile.inc include experimental/certifier/Makefile.inc include experimental/collapsed_forwarding/Makefile.inc +include experimental/cookie_remap/Makefile.inc include experimental/custom_redirect/Makefile.inc include experimental/fq_pacing/Makefile.inc include experimental/geoip_acl/Makefile.inc @@ -73,6 +74,7 @@ include experimental/mp4/Makefile.inc include experimental/multiplexer/Makefile.inc include experimental/remap_purge/Makefile.inc include experimental/server_push_preload/Makefile.inc +include experimental/slice/Makefile.inc include experimental/sslheaders/Makefile.inc include experimental/stale_while_revalidate/Makefile.inc include experimental/stream_editor/Makefile.inc @@ -82,6 +84,10 @@ include experimental/tls_bridge/Makefile.inc include experimental/url_sig/Makefile.inc include experimental/prefetch/Makefile.inc +if BUILD_JA3_PLUGIN +include experimental/ja3_fingerprint/Makefile.inc +endif + if BUILD_URI_SIGNING_PLUGIN include experimental/uri_signing/Makefile.inc endif diff --git a/plugins/authproxy/authproxy.cc b/plugins/authproxy/authproxy.cc index 4a5e5be6a0b..da8d8c86a45 100644 --- a/plugins/authproxy/authproxy.cc +++ b/plugins/authproxy/authproxy.cc @@ -55,11 +55,11 @@ static TSCont AuthOsDnsContinuation; struct AuthOptions { std::string hostname; - int hostport; - AuthRequestTransform transform; - bool force; + int hostport = -1; + AuthRequestTransform transform = nullptr; + bool force = false; - AuthOptions() : hostport(-1), transform(nullptr), force(false) {} + AuthOptions() {} ~AuthOptions() {} }; @@ -137,29 +137,21 @@ static const StateTransition StateTableInit[] = {{TS_EVENT_HTTP_POST_REMAP, Stat {TS_EVENT_NONE, nullptr, nullptr}}; struct AuthRequestContext { - TSHttpTxn txn; // Original client transaction we are authorizing. - TSCont cont; // Continuation for this state machine. - TSVConn vconn; // Virtual connection to the auth proxy. - TSHttpParser hparser; // HTTP response header parser. - HttpHeader rheader; // HTTP response header. + TSHttpTxn txn = nullptr; // Original client transaction we are authorizing. + TSCont cont = nullptr; // Continuation for this state machine. + TSVConn vconn = nullptr; // Virtual connection to the auth proxy. + TSHttpParser hparser; // HTTP response header parser. + HttpHeader rheader; // HTTP response header. HttpIoBuffer iobuf; - const char *method; // Client request method (e.g. GET) - bool read_body; + const char *method = nullptr; // Client request method (e.g. GET) + bool read_body = true; - const StateTransition *state; + const StateTransition *state = nullptr; AuthRequestContext() - : txn(nullptr), - cont(nullptr), - vconn(nullptr), - hparser(TSHttpParserCreate()), - rheader(), - iobuf(TS_IOBUFFER_SIZE_INDEX_4K), - method(nullptr), - read_body(true), - state(nullptr) + : cont(TSContCreate(dispatch, TSMutexCreate())), hparser(TSHttpParserCreate()), rheader(), iobuf(TS_IOBUFFER_SIZE_INDEX_4K) + { - this->cont = TSContCreate(dispatch, TSMutexCreate()); TSContDataSet(this->cont, this); } @@ -178,7 +170,7 @@ struct AuthRequestContext { { AuthOptions *opt; - opt = (AuthOptions *)TSHttpTxnArgGet(this->txn, AuthTaggedRequestArg); + opt = static_cast(TSHttpTxnArgGet(this->txn, AuthTaggedRequestArg)); return opt ? opt : AuthGlobalOptions; } @@ -206,7 +198,7 @@ AuthRequestContext::destroy(AuthRequestContext *auth) int AuthRequestContext::dispatch(TSCont cont, TSEvent event, void *edata) { - AuthRequestContext *auth = (AuthRequestContext *)TSContDataGet(cont); + AuthRequestContext *auth = static_cast(TSContDataGet(cont)); const StateTransition *s; pump: @@ -423,7 +415,6 @@ static TSEvent StateAuthProxyCompleteHeaders(AuthRequestContext *auth, void * /* edata ATS_UNUSED */) { TSHttpStatus status; - unsigned nbytes; HttpDebugHeader(auth->rheader.buffer, auth->rheader.header); @@ -444,7 +435,7 @@ StateAuthProxyCompleteHeaders(AuthRequestContext *auth, void * /* edata ATS_UNUS } else { // OK, we have a non-chunked response. If there's any content, let's go and // buffer it so that we can send it on to the client. - nbytes = HttpGetContentLength(auth->rheader.buffer, auth->rheader.header); + unsigned nbytes = HttpGetContentLength(auth->rheader.buffer, auth->rheader.header); if (nbytes > 0) { AuthLogDebug("content length is %u", nbytes); return TS_EVENT_HTTP_CONTINUE; @@ -636,7 +627,6 @@ AuthRequestIsTagged(TSHttpTxn txn) static int AuthProxyGlobalHook(TSCont /* cont ATS_UNUSED */, TSEvent event, void *edata) { - AuthRequestContext *auth; TSHttpTxn txn = (TSHttpTxn)edata; AuthLogDebug("handling event=%d edata=%p", (int)event, edata); @@ -661,12 +651,12 @@ AuthProxyGlobalHook(TSCont /* cont ATS_UNUSED */, TSEvent event, void *edata) // Hook this request if we are in global authorization mode or if a // remap rule tagged it. if (AuthGlobalOptions != nullptr || AuthRequestIsTagged(txn)) { - auth = AuthRequestContext::allocate(); - auth->state = StateTableInit; - auth->txn = txn; + AuthRequestContext *auth = AuthRequestContext::allocate(); + auth->state = StateTableInit; + auth->txn = txn; return AuthRequestContext::dispatch(auth->cont, event, edata); } - // fallthru + // fallthrough default: return TS_EVENT_NONE; @@ -786,14 +776,14 @@ TSRemapNewInstance(int argc, char *argv[], void **instance, char * /* err ATS_UN void TSRemapDeleteInstance(void *instance) { - AuthOptions *options = (AuthOptions *)instance; + AuthOptions *options = static_cast(instance); AuthDelete(options); } TSRemapStatus TSRemapDoRemap(void *instance, TSHttpTxn txn, TSRemapRequestInfo * /* rri ATS_UNUSED */) { - AuthOptions *options = (AuthOptions *)instance; + AuthOptions *options = static_cast(instance); TSHttpTxnArgSet(txn, AuthTaggedRequestArg, options); TSHttpTxnHookAdd(txn, TS_HTTP_POST_REMAP_HOOK, AuthOsDnsContinuation); diff --git a/plugins/authproxy/utils.h b/plugins/authproxy/utils.h index a95e2fe860b..72a5830ada3 100644 --- a/plugins/authproxy/utils.h +++ b/plugins/authproxy/utils.h @@ -43,9 +43,8 @@ struct HttpIoBuffer { TSIOBufferReader reader; explicit HttpIoBuffer(TSIOBufferSizeIndex size = TS_IOBUFFER_SIZE_INDEX_32K) + : buffer(TSIOBufferSizedCreate(size)), reader(TSIOBufferReaderAlloc(buffer)) { - this->buffer = TSIOBufferSizedCreate(size); - this->reader = TSIOBufferReaderAlloc(this->buffer); } ~HttpIoBuffer() diff --git a/plugins/background_fetch/background_fetch.cc b/plugins/background_fetch/background_fetch.cc index db1c875fd0c..6ad921661bd 100644 --- a/plugins/background_fetch/background_fetch.cc +++ b/plugins/background_fetch/background_fetch.cc @@ -25,11 +25,12 @@ #include #include #include -#include #include #include #include +#include +#include #include "ts/ts.h" #include "ts/remap.h" @@ -38,12 +39,22 @@ #include "configs.h" // Global config, if we don't have a remap specific config. -static BgFetchConfig *gConfig; +static BgFetchConfig *gConfig = nullptr; + +// This is the list of all headers that must be removed when we make the actual background +// fetch request. +static const std::array FILTER_HEADERS{ + {{TS_MIME_FIELD_RANGE, static_cast(TS_MIME_LEN_RANGE)}, + {TS_MIME_FIELD_IF_MATCH, static_cast(TS_MIME_LEN_IF_MATCH)}, + {TS_MIME_FIELD_IF_MODIFIED_SINCE, static_cast(TS_MIME_LEN_IF_MODIFIED_SINCE)}, + {TS_MIME_FIELD_IF_NONE_MATCH, static_cast(TS_MIME_LEN_IF_NONE_MATCH)}, + {TS_MIME_FIELD_IF_RANGE, static_cast(TS_MIME_LEN_IF_RANGE)}, + {TS_MIME_FIELD_IF_UNMODIFIED_SINCE, static_cast(TS_MIME_LEN_IF_UNMODIFIED_SINCE)}}}; /////////////////////////////////////////////////////////////////////////// -// Hold the global ackground fetch state. This is currently shared across all +// Hold the global background fetch state. This is currently shared across all // configurations, as a singleton. ToDo: Would it ever make sense to do this -// per remap rule? Probably not. +// per remap rule? Maybe for per-remap logging ?? typedef std::unordered_map OutstandingRequests; class BgFetchState @@ -61,11 +72,16 @@ class BgFetchState } ~BgFetchState() { TSMutexDestroy(_lock); } + void - createLog(const char *log_name) + createLog(const std::string &log_name) { - TSDebug(PLUGIN_NAME, "Creating log name %s", log_name); - TSAssert(TS_SUCCESS == TSTextLogObjectCreate(log_name, TS_LOG_MODE_ADD_TIMESTAMP, &_log)); + if (!_log) { + TSDebug(PLUGIN_NAME, "Creating log name %s", log_name.c_str()); + TSAssert(TS_SUCCESS == TSTextLogObjectCreate(log_name.c_str(), TS_LOG_MODE_ADD_TIMESTAMP, &_log)); + } else { + TSError("[%s] A log file was already create, ignoring creation of %s", PLUGIN_NAME, log_name.c_str()); + } } TSTextLogObject @@ -204,7 +220,7 @@ struct BgFetchData { // This needs the txnp temporarily, so it can copy the pristine request // URL. The txnp is not used once initialize() returns. // -// Upon succesful completion, the struct should be ready to start a +// Upon successful completion, the struct should be ready to start a // background fetch. bool BgFetchData::initialize(TSMBuffer request, TSMLoc req_hdr, TSHttpTxn txnp) @@ -261,10 +277,13 @@ BgFetchData::initialize(TSMBuffer request, TSMLoc req_hdr, TSHttpTxn txnp) TSDebug(PLUGIN_NAME, "Set header Host: %.*s", len, hostp); } - // Next, remove any Range: headers from our request. - if (remove_header(mbuf, hdr_loc, TS_MIME_FIELD_RANGE, TS_MIME_LEN_RANGE) > 0) { - TSDebug(PLUGIN_NAME, "Removed the Range: header from request"); + // Next, remove the Range headers and IMS (conditional) headers from the request + for (auto const &header : FILTER_HEADERS) { + if (remove_header(mbuf, hdr_loc, header.data(), header.size()) > 0) { + TSDebug(PLUGIN_NAME, "Removed the %s header from request", header.data()); + } } + // Everything went as planned, so we can return true ret = true; } @@ -296,7 +315,7 @@ BgFetchData::schedule() resp_io_buf_reader = TSIOBufferReaderAlloc(resp_io_buf); // Schedule - TSContSchedule(_cont, 0, TS_THREAD_POOL_NET); + TSContScheduleOnPool(_cont, 0, TS_THREAD_POOL_NET); } // Log format is: @@ -507,9 +526,10 @@ cont_handle_response(TSCont contp, TSEvent event, void *edata) // ToDo: Check the MIME type first, to see if it's a type we care about. // ToDo: Such MIME types should probably be per remap rule. - // Only deal with 206 responses from Origin - TSDebug(PLUGIN_NAME, "Testing: response is 206?"); - if (TS_HTTP_STATUS_PARTIAL_CONTENT == TSHttpHdrStatusGet(response, resp_hdr)) { + // Only deal with 206 and possibly 304 responses from Origin + TSHttpStatus status = TSHttpHdrStatusGet(response, resp_hdr); + TSDebug(PLUGIN_NAME, "Testing: response status code: %d?", status); + if (TS_HTTP_STATUS_PARTIAL_CONTENT == status || (config->allow304() && TS_HTTP_STATUS_NOT_MODIFIED == status)) { // Everything looks good so far, add a TXN hook for SEND_RESPONSE_HDR TSCont contp = TSContCreate(cont_check_cacheable, nullptr); @@ -538,9 +558,6 @@ void TSPluginInit(int argc, const char *argv[]) { TSPluginRegistrationInfo info; - static const struct option longopt[] = {{const_cast("log"), required_argument, nullptr, 'l'}, - {const_cast("config"), required_argument, nullptr, 'c'}, - {nullptr, no_argument, nullptr, '\0'}}; info.plugin_name = (char *)PLUGIN_NAME; info.vendor_name = (char *)"Apache Software Foundation"; @@ -554,26 +571,18 @@ TSPluginInit(int argc, const char *argv[]) gConfig = new BgFetchConfig(cont); - while (true) { - int opt = getopt_long(argc, (char *const *)argv, "lc", longopt, nullptr); - - switch (opt) { - case 'l': - BgFetchState::getInstance().createLog(optarg); - break; - case 'c': - TSDebug(PLUGIN_NAME, "config file '%s'", optarg); - gConfig->readConfig(optarg); - break; - } - - if (opt == -1) { - break; + if (gConfig->parseOptions(argc, argv)) { + // Create the global log file. Note that calling this multiple times currently has no + // effect, only one log file is ever created. The BgFetchState is a singleton. + if (!gConfig->logFile().empty()) { + BgFetchState::getInstance().createLog(gConfig->logFile()); } + TSDebug(PLUGIN_NAME, "Initialized"); + TSHttpHookAdd(TS_HTTP_READ_RESPONSE_HDR_HOOK, cont); + } else { + // ToDo: Hmmm, no way to fail a global plugin here? + TSDebug(PLUGIN_NAME, "Failed to initialize as global plugin"); } - - TSDebug(PLUGIN_NAME, "Initialized"); - TSHttpHookAdd(TS_HTTP_READ_RESPONSE_HDR_HOOK, cont); } /////////////////////////////////////////////////////////////////////////// @@ -608,16 +617,40 @@ TSRemapNewInstance(int argc, char *argv[], void **ih, char * /* errbuf */, int / { TSCont cont = TSContCreate(cont_handle_response, nullptr); BgFetchConfig *config = new BgFetchConfig(cont); - - // Parse the optional rules, wihch becomes a linked list of BgFetchRule's. - if (argc > 2) { - TSDebug(PLUGIN_NAME, "config file %s", argv[2]); - config->readConfig(argv[2]); + bool success = true; + + // The first two arguments are the "from" and "to" URL string. We need to + // skip them, but we also require that there be an option to masquerade as + // argv[0], so we increment the argument indexes by 1 rather than by 2. + argc--; + argv++; + + // This is for backwards compatibility, ugly! ToDo: Remove for ATS v9.0.0 IMO. + if (argc > 1 && *argv[1] != '-') { + TSDebug(PLUGIN_NAME, "config file %s", argv[1]); + if (!config->readConfig(argv[1])) { + success = false; + } + } else { + if (config->parseOptions(argc, const_cast(argv))) { + // Create the global log file. Remember, the BgFetchState is a singleton. + if (config->logFile().size()) { + BgFetchState::getInstance().createLog(config->logFile()); + } + } else { + success = false; + } } - *ih = (void *)config; + if (success) { + *ih = config; - return TS_SUCCESS; + return TS_SUCCESS; + } + + // Something went wrong with the configuration setup. + delete config; + return TS_ERROR; } void @@ -625,6 +658,7 @@ TSRemapDeleteInstance(void *ih) { BgFetchConfig *config = static_cast(ih); + TSContDestroy(config->getCont()); delete config; } @@ -644,11 +678,15 @@ TSRemapDoRemap(void *ih, TSHttpTxn txnp, TSRemapRequestInfo * /* rri */) if (TS_SUCCESS == TSHttpTxnClientReqGet(txnp, &bufp, &req_hdrs)) { TSMLoc field_loc = TSMimeHdrFieldFind(bufp, req_hdrs, TS_MIME_FIELD_RANGE, TS_MIME_LEN_RANGE); + if (!field_loc) { // Less common case, but also allow If-Range header to trigger, but only if Range not present + field_loc = TSMimeHdrFieldFind(bufp, req_hdrs, TS_MIME_FIELD_IF_RANGE, TS_MIME_LEN_IF_RANGE); + } + if (field_loc) { BgFetchConfig *config = static_cast(ih); TSHttpTxnHookAdd(txnp, TS_HTTP_READ_RESPONSE_HDR_HOOK, config->getCont()); - TSDebug(PLUGIN_NAME, "background fetch TSRemapDoRemap"); + TSDebug(PLUGIN_NAME, "TSRemapDoRemap() added hook, request was Range / If-Range"); TSHandleMLocRelease(bufp, req_hdrs, field_loc); } TSHandleMLocRelease(bufp, TS_NULL_MLOC, req_hdrs); diff --git a/plugins/background_fetch/configs.cc b/plugins/background_fetch/configs.cc index 032216523a2..d5283b45062 100644 --- a/plugins/background_fetch/configs.cc +++ b/plugins/background_fetch/configs.cc @@ -22,11 +22,57 @@ limitations under the License. */ -#include "configs.h" +#include #include #include -// Read a config file, populare the linked list (chain the BgFetchRule's) +#include "configs.h" + +// Parse the command line options. This got a little wonky, since we decided to have different +// syntax for remap vs global plugin initialization, and the global BG fetch state :-/. Clean up +// later... +bool +BgFetchConfig::parseOptions(int argc, const char *argv[]) +{ + static const struct option longopt[] = {{const_cast("log"), required_argument, nullptr, 'l'}, + {const_cast("config"), required_argument, nullptr, 'c'}, + {const_cast("allow-304"), no_argument, nullptr, 'a'}, + {nullptr, no_argument, nullptr, '\0'}}; + + while (true) { + int opt = getopt_long(argc, (char *const *)argv, "lc", longopt, nullptr); + + if (opt == -1) { + break; + } + + switch (opt) { + case 'l': + TSDebug(PLUGIN_NAME, "option: log file specified: %s", optarg); + _log_file = optarg; + break; + case 'c': + TSDebug(PLUGIN_NAME, "option: config file '%s'", optarg); + if (!readConfig(optarg)) { + // Error messages are written in the parser + return false; + } + break; + case 'a': + TSDebug(PLUGIN_NAME, "option: --allow-304 set"); + _allow_304 = true; + break; + default: + TSError("[%s] invalid plugin option: %c", PLUGIN_NAME, opt); + return false; + break; + } + } + + return true; +} + +// Read a config file, populate the linked list (chain the BgFetchRule's) bool BgFetchConfig::readConfig(const char *config_file) { @@ -45,11 +91,12 @@ BgFetchConfig::readConfig(const char *config_file) } else { snprintf(file_path, sizeof(file_path), "%s/%s", TSConfigDirGet(), config_file); } - TSDebug(PLUGIN_NAME, "Chosen config file is at: %s", file_path); + TSDebug(PLUGIN_NAME, "chosen config file is at: %s", file_path); file = TSfopen(file_path, "r"); if (nullptr == file) { TSError("[%s] invalid config file: %s", PLUGIN_NAME, file_path); + TSDebug(PLUGIN_NAME, "invalid config file: %s", file_path); return false; } @@ -80,9 +127,9 @@ BgFetchConfig::readConfig(const char *config_file) char *cfg_type = strtok_r(buffer, " ", &savePtr); char *cfg_name = nullptr; char *cfg_value = nullptr; - bool exclude = false; if (cfg_type) { + bool exclude = false; if (!strcmp(cfg_type, "exclude")) { exclude = true; } else if (strcmp(cfg_type, "include")) { diff --git a/plugins/background_fetch/configs.h b/plugins/background_fetch/configs.h index 569ebde10be..ba6dd35fb5c 100644 --- a/plugins/background_fetch/configs.h +++ b/plugins/background_fetch/configs.h @@ -26,6 +26,7 @@ #include #include +#include #include "rules.h" @@ -48,6 +49,8 @@ class BgFetchConfig } } + bool parseOptions(int argc, const char *argv[]); + BgFetchRule * getRules() const { @@ -60,12 +63,26 @@ class BgFetchConfig return _cont; } + const std::string & + logFile() const + { + return _log_file; + } + + bool + allow304() const + { + return _allow_304; + } + // This parses and populates the BgFetchRule linked list (_rules). bool readConfig(const char *file_name); bool bgFetchAllowed(TSHttpTxn txnp) const; private: - TSCont _cont; - BgFetchRule *_rules{nullptr}; + TSCont _cont = nullptr; + BgFetchRule *_rules = nullptr; + bool _allow_304 = false; + std::string _log_file; }; diff --git a/plugins/background_fetch/headers.cc b/plugins/background_fetch/headers.cc index 4043993f3e6..4317d9da5d8 100644 --- a/plugins/background_fetch/headers.cc +++ b/plugins/background_fetch/headers.cc @@ -100,7 +100,6 @@ dump_headers(TSMBuffer bufp, TSMLoc hdr_loc) TSIOBuffer output_buffer; TSIOBufferReader reader; TSIOBufferBlock block; - const char *block_start; int64_t block_avail; output_buffer = TSIOBufferCreate(); @@ -112,7 +111,7 @@ dump_headers(TSMBuffer bufp, TSMLoc hdr_loc) /* We need to loop over all the buffer blocks, there can be more than 1 */ block = TSIOBufferReaderStart(reader); do { - block_start = TSIOBufferBlockReadStart(block, reader, &block_avail); + const char *block_start = TSIOBufferBlockReadStart(block, reader, &block_avail); if (block_avail > 0) { TSDebug(PLUGIN_NAME, "Headers are:\n%.*s", static_cast(block_avail), block_start); } diff --git a/plugins/background_fetch/rules.cc b/plugins/background_fetch/rules.cc index 98ddd744da4..ddbd2bc0e64 100644 --- a/plugins/background_fetch/rules.cc +++ b/plugins/background_fetch/rules.cc @@ -24,7 +24,7 @@ #include #include -#include +#include #include "configs.h" #include "rules.h" diff --git a/plugins/cache_promote/cache_promote.cc b/plugins/cache_promote/cache_promote.cc index d05127a548f..23e88c9ea20 100644 --- a/plugins/cache_promote/cache_promote.cc +++ b/plugins/cache_promote/cache_promote.cc @@ -56,7 +56,7 @@ static const struct option longopt[] = { class PromotionPolicy { public: - PromotionPolicy() : _sample(0.0) + PromotionPolicy() { // This doesn't have to be perfect, since this is just chance sampling. // coverity[dont_call] @@ -106,7 +106,7 @@ class PromotionPolicy virtual void usage() const = 0; private: - float _sample; + float _sample = 0.0; }; ////////////////////////////////////////////////////////////////////////////////////////////// @@ -195,7 +195,7 @@ static LRUEntry NULL_LRU_ENTRY; // Used to create an "empty" new LRUEntry class LRUPolicy : public PromotionPolicy { public: - LRUPolicy() : PromotionPolicy(), _buckets(1000), _hits(10), _lock(TSMutexCreate()), _list_size(0), _freelist_size(0) {} + LRUPolicy() : PromotionPolicy(), _lock(TSMutexCreate()) {} ~LRUPolicy() override { TSDebug(PLUGIN_NAME, "deleting LRUPolicy object"); @@ -333,14 +333,14 @@ class LRUPolicy : public PromotionPolicy } private: - unsigned _buckets; - unsigned _hits; + unsigned _buckets = 1000; + unsigned _hits = 10; // For the LRU. Note that we keep track of the List sizes, because some versions fo STL have broken // implementations of size(), making them obsessively slow on calling ::size(). TSMutex _lock; LRUMap _map; LRUList _list, _freelist; - size_t _list_size, _freelist_size; + size_t _list_size = 0, _freelist_size = 0; }; ////////////////////////////////////////////////////////////////////////////////////////////// @@ -349,7 +349,7 @@ class LRUPolicy : public PromotionPolicy class PromotionConfig { public: - PromotionConfig() : _policy(nullptr) {} + PromotionConfig() {} ~PromotionConfig() { delete _policy; } PromotionPolicy * getPolicy() const @@ -402,7 +402,7 @@ class PromotionConfig } private: - PromotionPolicy *_policy; + PromotionPolicy *_policy = nullptr; }; ////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/plugins/cachekey/README.md b/plugins/cachekey/README.md index 91dc7252e69..6cc59898bbe 100644 --- a/plugins/cachekey/README.md +++ b/plugins/cachekey/README.md @@ -1,7 +1,7 @@ # Description This plugin allows some common cache key manipulations based on various HTTP request elements. It can -* sort query parameters to prevent query parameters reordereding from being a cache miss +* sort query parameters to prevent query parameters reordering from being a cache miss * ignore specific query parameters from the cache key by name or regular expression * ignore all query parameters from the cache key * only use specific query parameters in the cache key by name or regular expression diff --git a/plugins/cachekey/cachekey.cc b/plugins/cachekey/cachekey.cc index c89b65755b8..9cf4454210f 100644 --- a/plugins/cachekey/cachekey.cc +++ b/plugins/cachekey/cachekey.cc @@ -23,6 +23,7 @@ #include /* strlen() */ #include /* istringstream */ +#include #include "cachekey.h" static void @@ -195,7 +196,7 @@ getUri(TSMBuffer buf, TSMLoc url) * @param rri remap request info */ CacheKey::CacheKey(TSHttpTxn txn, String separator, CacheKeyUriType uriType, TSRemapRequestInfo *rri) - : _txn(txn), _separator(separator), _uriType(uriType) + : _txn(txn), _separator(std::move(separator)), _uriType(uriType) { _key.reserve(512); diff --git a/plugins/cachekey/configs.cc b/plugins/cachekey/configs.cc index 2a13d7608f9..e391a5c4318 100644 --- a/plugins/cachekey/configs.cc +++ b/plugins/cachekey/configs.cc @@ -183,8 +183,8 @@ ConfigElements::noIncludeExcludeRules() const ConfigElements::~ConfigElements() { - for (auto it = _captures.begin(); it != _captures.end(); it++) { - delete it->second; + for (auto &_capture : _captures) { + delete _capture.second; } } diff --git a/plugins/cachekey/configs.h b/plugins/cachekey/configs.h index 603ff434d2a..947b21931ea 100644 --- a/plugins/cachekey/configs.h +++ b/plugins/cachekey/configs.h @@ -41,7 +41,7 @@ enum CacheKeyUriType { class ConfigElements { public: - ConfigElements() : _sort(false), _remove(false), _skip(false) {} + ConfigElements() {} virtual ~ConfigElements(); void setExclude(const char *arg); void setInclude(const char *arg); @@ -82,9 +82,9 @@ class ConfigElements MultiPattern _includePatterns; MultiPattern _excludePatterns; - bool _sort; - bool _remove; - bool _skip; + bool _sort = false; + bool _remove = false; + bool _skip = false; std::map _captures; }; @@ -148,7 +148,7 @@ class Configs /** * @brief provides means for post-processing of the plugin parameters to finalize the configuration or to "cache" some of the * decisions for later use. - * @return true if succesful, false if failure. + * @return true if successful, false if failure. */ bool finalize(); diff --git a/plugins/cachekey/pattern.cc b/plugins/cachekey/pattern.cc index 27eb94ee24b..c93490291da 100644 --- a/plugins/cachekey/pattern.cc +++ b/plugins/cachekey/pattern.cc @@ -38,7 +38,7 @@ replaceString(String &str, const String &from, const String &to) } } -Pattern::Pattern() : _re(nullptr), _extra(nullptr), _pattern(""), _replacement(""), _replace(false), _tokenCount(0) {} +Pattern::Pattern() : _pattern(""), _replacement("") {} /** * @brief Initializes PCRE pattern by providing the subject and replacement strings. @@ -47,18 +47,18 @@ Pattern::Pattern() : _re(nullptr), _extra(nullptr), _pattern(""), _replacement(" * @return true if successful, false if failure */ bool -Pattern::init(const String &pattern, const String &replacenemt, bool replace) +Pattern::init(const String &pattern, const String &replacement, bool replace) { pcreFree(); _pattern.assign(pattern); - _replacement.assign(replacenemt); + _replacement.assign(replacement); _replace = replace; _tokenCount = 0; if (!compile()) { - CacheKeyDebug("failed to initialize pattern:'%s', replacement:'%s'", pattern.c_str(), replacenemt.c_str()); + CacheKeyDebug("failed to initialize pattern:'%s', replacement:'%s'", pattern.c_str(), replacement.c_str()); pcreFree(); return false; } @@ -151,7 +151,7 @@ Pattern::pcreFree() } /** - * @bried Destructor, frees PCRE related resources. + * @brief Destructor, frees PCRE related resources. */ Pattern::~Pattern() { diff --git a/plugins/cachekey/pattern.h b/plugins/cachekey/pattern.h index 876edf0a3db..2b36fd70884 100644 --- a/plugins/cachekey/pattern.h +++ b/plugins/cachekey/pattern.h @@ -45,7 +45,7 @@ class Pattern Pattern(); virtual ~Pattern(); - bool init(const String &pattern, const String &replacenemt, bool replace); + bool init(const String &pattern, const String &replacement, bool replace); bool init(const String &config); bool empty() const; bool match(const String &subject); @@ -57,16 +57,16 @@ class Pattern bool compile(); void pcreFree(); - pcre *_re; /**< @brief PCRE compiled info structure, computed during initialization */ - pcre_extra *_extra; /**< @brief PCRE study data block, computed during initialization */ + pcre *_re = nullptr; /**< @brief PCRE compiled info structure, computed during initialization */ + pcre_extra *_extra = nullptr; /**< @brief PCRE study data block, computed during initialization */ String _pattern; /**< @brief PCRE pattern string, containing PCRE patterns and capturing groups. */ String _replacement; /**< @brief PCRE replacement string, containing $0..$9 to be replaced with content of the capturing groups */ - bool _replace; /**< @brief true if a replacement is needed, false if not, this is to distinguish between an empty replacement - string and no replacement needed case */ + bool _replace = false; /**< @brief true if a replacement is needed, false if not, this is to distinguish between an empty + replacement string and no replacement needed case */ - int _tokenCount; /**< @brief number of replacements $0..$9 found in the replacement string if not empty */ + int _tokenCount = 0; /**< @brief number of replacements $0..$9 found in the replacement string if not empty */ int _tokens[TOKENCOUNT]; /**< @brief replacement index 0..9, since they can be used in the replacement string in any order */ int _tokenOffset[TOKENCOUNT]; /**< @brief replacement offset inside the replacement string */ }; @@ -77,7 +77,7 @@ class Pattern class MultiPattern { public: - MultiPattern(const String name = "") : _name(name) {} + MultiPattern(const String &name = "") : _name(name) {} virtual ~MultiPattern(); bool empty() const; diff --git a/plugins/compress/compress.cc b/plugins/compress/compress.cc index a14277f4c6b..f1be86d9a50 100644 --- a/plugins/compress/compress.cc +++ b/plugins/compress/compress.cc @@ -40,7 +40,7 @@ using namespace std; using namespace Gzip; // FIXME: custom dictionaries would be nice. configurable/content-type? -// a gprs device might benefit from a higher compression ratio, whereas a desktop w. high bandwith +// a GPRS device might benefit from a higher compression ratio, whereas a desktop w. high bandwith // might be served better with little or no compression at all // FIXME: look into compressing from the task thread pool // FIXME: make normalizing accept encoding configurable @@ -136,7 +136,7 @@ data_destroy(Data *data) { TSReleaseAssert(data); - // deflateEnd returnvalue ignore is intentional + // deflateEnd return value ignore is intentional // it would spew log on every client abort deflateEnd(&data->zstrm); @@ -201,11 +201,10 @@ vary_header(TSMBuffer bufp, TSMLoc hdr_loc) ce_loc = TSMimeHdrFieldFind(bufp, hdr_loc, TS_MIME_FIELD_VARY, TS_MIME_LEN_VARY); if (ce_loc) { int idx, count, len; - const char *value; count = TSMimeHdrFieldValuesCount(bufp, hdr_loc, ce_loc); for (idx = 0; idx < count; idx++) { - value = TSMimeHdrFieldValueStringGet(bufp, hdr_loc, ce_loc, idx, &len); + const char *value = TSMimeHdrFieldValueStringGet(bufp, hdr_loc, ce_loc, idx, &len); if (len && strncasecmp("Accept-Encoding", value, len) == 0) { // Bail, Vary: Accept-Encoding already sent from origin TSHandleMLocRelease(bufp, hdr_loc, ce_loc); @@ -244,13 +243,13 @@ etag_header(TSMBuffer bufp, TSMLoc hdr_loc) ce_loc = TSMimeHdrFieldFind(bufp, hdr_loc, TS_MIME_FIELD_ETAG, TS_MIME_LEN_ETAG); if (ce_loc) { - int changetag = 1; int strl; const char *strv = TSMimeHdrFieldValueStringGet(bufp, hdr_loc, ce_loc, -1, &strl); // do not alter weak etags. // FIXME: consider just making the etag weak for compressed content if (strl >= 2) { + int changetag = 1; if ((strv[0] == 'w' || strv[0] == 'W') && strv[1] == '/') { changetag = 0; } @@ -301,15 +300,14 @@ static void gzip_transform_one(Data *data, const char *upstream_buffer, int64_t upstream_length) { TSIOBufferBlock downstream_blkp; - char *downstream_buffer; int64_t downstream_length; int err; data->zstrm.next_in = (unsigned char *)upstream_buffer; data->zstrm.avail_in = upstream_length; while (data->zstrm.avail_in > 0) { - downstream_blkp = TSIOBufferStart(data->downstream_buffer); - downstream_buffer = TSIOBufferBlockWriteStart(downstream_blkp, &downstream_length); + downstream_blkp = TSIOBufferStart(data->downstream_buffer); + char *downstream_buffer = TSIOBufferBlockWriteStart(downstream_blkp, &downstream_length); data->zstrm.next_out = (unsigned char *)downstream_buffer; data->zstrm.avail_out = downstream_length; @@ -342,7 +340,6 @@ static bool brotli_compress_operation(Data *data, const char *upstream_buffer, int64_t upstream_length, BrotliEncoderOperation op) { TSIOBufferBlock downstream_blkp; - char *downstream_buffer; int64_t downstream_length; data->bstrm.next_in = (uint8_t *)upstream_buffer; @@ -350,8 +347,8 @@ brotli_compress_operation(Data *data, const char *upstream_buffer, int64_t upstr bool ok = true; while (ok) { - downstream_blkp = TSIOBufferStart(data->downstream_buffer); - downstream_buffer = TSIOBufferBlockWriteStart(downstream_blkp, &downstream_length); + downstream_blkp = TSIOBufferStart(data->downstream_buffer); + char *downstream_buffer = TSIOBufferBlockWriteStart(downstream_blkp, &downstream_length); data->bstrm.next_out = (unsigned char *)downstream_buffer; data->bstrm.avail_out = downstream_length; @@ -363,7 +360,7 @@ brotli_compress_operation(Data *data, const char *upstream_buffer, int64_t upstr if (!ok) { error("BrotliEncoderCompressStream(%d) call failed", op); - return ok; + return false; } TSIOBufferProduce(data->downstream_buffer, downstream_length - data->bstrm.avail_out); @@ -405,7 +402,6 @@ static void compress_transform_one(Data *data, TSIOBufferReader upstream_reader, int amount) { TSIOBufferBlock downstream_blkp; - const char *upstream_buffer; int64_t upstream_length; while (amount > 0) { downstream_blkp = TSIOBufferReaderStart(upstream_reader); @@ -414,7 +410,7 @@ compress_transform_one(Data *data, TSIOBufferReader upstream_reader, int amount) return; } - upstream_buffer = TSIOBufferBlockReadStart(downstream_blkp, upstream_reader, &upstream_length); + const char *upstream_buffer = TSIOBufferBlockReadStart(downstream_blkp, upstream_reader, &upstream_length); if (!upstream_buffer) { error("couldn't get from TSIOBufferBlockReadStart"); return; @@ -433,7 +429,7 @@ compress_transform_one(Data *data, TSIOBufferReader upstream_reader, int amount) (data->compression_algorithms & (ALGORITHM_GZIP | ALGORITHM_DEFLATE))) { gzip_transform_one(data, upstream_buffer, upstream_length); } else { - warning("No compression supported. Shoudn't come here."); + warning("No compression supported. Shouldn't come here."); } TSIOBufferReaderConsume(upstream_reader, upstream_length); @@ -446,20 +442,18 @@ gzip_transform_finish(Data *data) { if (data->state == transform_state_output) { TSIOBufferBlock downstream_blkp; - char *downstream_buffer; int64_t downstream_length; - int err; data->state = transform_state_finished; for (;;) { downstream_blkp = TSIOBufferStart(data->downstream_buffer); - downstream_buffer = TSIOBufferBlockWriteStart(downstream_blkp, &downstream_length); - data->zstrm.next_out = (unsigned char *)downstream_buffer; - data->zstrm.avail_out = downstream_length; + char *downstream_buffer = TSIOBufferBlockWriteStart(downstream_blkp, &downstream_length); + data->zstrm.next_out = (unsigned char *)downstream_buffer; + data->zstrm.avail_out = downstream_length; - err = deflate(&data->zstrm, Z_FINISH); + int err = deflate(&data->zstrm, Z_FINISH); if (downstream_length > (int64_t)data->zstrm.avail_out) { TSIOBufferProduce(data->downstream_buffer, downstream_length - data->zstrm.avail_out); @@ -534,7 +528,6 @@ compress_transform_do(TSCont contp) TSVIO upstream_vio; Data *data; int64_t upstream_todo; - int64_t upstream_avail; int64_t downstream_bytes_written; data = (Data *)TSContDataGet(contp); @@ -559,7 +552,7 @@ compress_transform_do(TSCont contp) upstream_todo = TSVIONTodoGet(upstream_vio); if (upstream_todo > 0) { - upstream_avail = TSIOBufferReaderAvail(TSVIOReaderGet(upstream_vio)); + int64_t upstream_avail = TSIOBufferReaderAvail(TSVIOReaderGet(upstream_vio)); if (upstream_todo > upstream_avail) { upstream_todo = upstream_avail; @@ -637,8 +630,7 @@ transformable(TSHttpTxn txnp, bool server, HostConfiguration *host_configuration TSMLoc cfield; const char *value; - int nvalues; - int i, compression_acceptable, len; + int len; TSHttpStatus resp_status; if (server) { @@ -680,9 +672,9 @@ transformable(TSHttpTxn txnp, bool server, HostConfiguration *host_configuration *algorithms = host_configuration->compression_algorithms(); cfield = TSMimeHdrFieldFind(cbuf, chdr, TS_MIME_FIELD_ACCEPT_ENCODING, TS_MIME_LEN_ACCEPT_ENCODING); if (cfield != TS_NULL_MLOC) { - compression_acceptable = 0; - nvalues = TSMimeHdrFieldValuesCount(cbuf, chdr, cfield); - for (i = 0; i < nvalues; i++) { + int compression_acceptable = 0; + int nvalues = TSMimeHdrFieldValuesCount(cbuf, chdr, cfield); + for (int i = 0; i < nvalues; i++) { value = TSMimeHdrFieldValueStringGet(cbuf, chdr, cfield, i, &len); if (!value) { continue; @@ -734,14 +726,14 @@ transformable(TSHttpTxn txnp, bool server, HostConfiguration *host_configuration field_loc = TSMimeHdrFieldFind(bufp, hdr_loc, TS_MIME_FIELD_CONTENT_LENGTH, TS_MIME_LEN_CONTENT_LENGTH); if (field_loc != TS_NULL_MLOC) { - unsigned int value = TSMimeHdrFieldValueUintGet(bufp, hdr_loc, field_loc, -1); + unsigned int hdr_value = TSMimeHdrFieldValueUintGet(bufp, hdr_loc, field_loc, -1); TSHandleMLocRelease(bufp, hdr_loc, field_loc); - if (value == 0) { + if (hdr_value == 0) { info("response is 0-length, not compressible"); return 0; } - if (value < host_configuration->minimum_content_length()) { + if (hdr_value < host_configuration->minimum_content_length()) { info("response is is smaller than minimum content length, not compressing"); return 0; } diff --git a/plugins/conf_remap/conf_remap.cc b/plugins/conf_remap/conf_remap.cc index 4411f8962d3..34b6198440a 100644 --- a/plugins/conf_remap/conf_remap.cc +++ b/plugins/conf_remap/conf_remap.cc @@ -41,15 +41,15 @@ struct RemapConfigs { int _data_len; // Used when data is a string }; - RemapConfigs() : _current(0) { memset(_items, 0, sizeof(_items)); }; + RemapConfigs() { memset(_items, 0, sizeof(_items)); }; bool parse_file(const char *filename); bool parse_inline(const char *arg); Item _items[MAX_OVERRIDABLE_CONFIGS]; - int _current; + int _current = 0; }; -// Helper functionfor the parser +// Helper function for the parser inline TSRecordDataType str_to_datatype(const char *str) { diff --git a/plugins/escalate/escalate.cc b/plugins/escalate/escalate.cc index f6e88629eb3..a6b1a0eae81 100644 --- a/plugins/escalate/escalate.cc +++ b/plugins/escalate/escalate.cc @@ -51,7 +51,7 @@ struct EscalationState { typedef std::map StatusMapType; - EscalationState() : use_pristine(false) + EscalationState() { cont = TSContCreate(EscalateResponse, nullptr); TSContDataSet(cont, this); @@ -60,7 +60,7 @@ struct EscalationState { ~EscalationState() { TSContDestroy(cont); } TSCont cont; StatusMapType status_map; - bool use_pristine; + bool use_pristine = false; }; // Little helper function, to update the Host portion of a URL, and stringify the result. diff --git a/plugins/esi/README.combo b/plugins/esi/README.combo index e4466ab039c..1cd7001fa28 100644 --- a/plugins/esi/README.combo +++ b/plugins/esi/README.combo @@ -12,7 +12,7 @@ The arguments in the plugin.config line in order represent 2) The name of the key used for signature verification (disabled by default) [verification not implemented yet] -3) A colon separated list of headers which, if present on atleast one +3) A colon separated list of headers which, if present on at least one response, will be added to the combo response. A "-" can be supplied as a value for any of these arguments to request diff --git a/plugins/esi/combo_handler.cc b/plugins/esi/combo_handler.cc index 4a5c336fb1d..d0c7159d000 100644 --- a/plugins/esi/combo_handler.cc +++ b/plugins/esi/combo_handler.cc @@ -83,12 +83,12 @@ static string COMBO_HANDLER_PATH{DEFAULT_COMBO_HANDLER_PATH}; using StringList = list; struct ClientRequest { - TSHttpStatus status; - const sockaddr *client_addr; + TSHttpStatus status = TS_HTTP_STATUS_OK; + const sockaddr *client_addr = nullptr; StringList file_urls; - bool gzip_accepted; + bool gzip_accepted = false; string defaultBucket; // default Bucket will be set to HOST header - ClientRequest() : status(TS_HTTP_STATUS_OK), client_addr(nullptr), gzip_accepted(false), defaultBucket("l"){}; + ClientRequest() : defaultBucket("l"){}; }; struct InterceptData { @@ -96,11 +96,11 @@ struct InterceptData { TSCont contp; struct IoHandle { - TSVIO vio; - TSIOBuffer buffer; - TSIOBufferReader reader; + TSVIO vio = nullptr; + TSIOBuffer buffer = nullptr; + TSIOBufferReader reader = nullptr; - IoHandle() : vio(nullptr), buffer(nullptr), reader(nullptr){}; + IoHandle(){}; ~IoHandle() { diff --git a/plugins/esi/esi.cc b/plugins/esi/esi.cc index 9aa8d61b561..101161834dd 100644 --- a/plugins/esi/esi.cc +++ b/plugins/esi/esi.cc @@ -1012,7 +1012,7 @@ transformHandler(TSCont contp, TSEvent event, void *edata) // lock on our continuation which will fail if we destroy // ourselves right now TSDebug(cont_debug_tag, "[%s] Deferring shutdown as data event was just processed", __FUNCTION__); - TSContSchedule(contp, 10, TS_THREAD_POOL_TASK); + TSContScheduleOnPool(contp, 10, TS_THREAD_POOL_TASK); } else { goto lShutdown; } @@ -1537,7 +1537,7 @@ globalHookHandler(TSCont contp, TSEvent event, void *edata) TSDebug(DEBUG_TAG, "[%s] handling cache lookup complete event", __FUNCTION__); if (isCacheObjTransformable(txnp, &intercept_header, &head_only)) { // we make the assumption above that a transformable cache - // object would already have a tranformation. We should revisit + // object would already have a transformation. We should revisit // that assumption in case we change the statement below addTransform(txnp, false, intercept_header, head_only, pOptionInfo); Stats::increment(Stats::N_CACHE_DOCS); diff --git a/plugins/esi/fetcher/FetchedDataProcessor.h b/plugins/esi/fetcher/FetchedDataProcessor.h index c30ac7d745c..9a9fd436cbf 100644 --- a/plugins/esi/fetcher/FetchedDataProcessor.h +++ b/plugins/esi/fetcher/FetchedDataProcessor.h @@ -28,7 +28,7 @@ class FetchedDataProcessor public: FetchedDataProcessor(){}; - virtual void processData(const char *reqeust_url, int request_url_len, const char *response_data, int response_data_len) = 0; + virtual void processData(const char *requeset_url, int request_url_len, const char *response_data, int response_data_len) = 0; virtual ~FetchedDataProcessor(){}; }; diff --git a/plugins/esi/fetcher/HttpDataFetcherImpl.h b/plugins/esi/fetcher/HttpDataFetcherImpl.h index d8a7fb743c8..1702813af5c 100644 --- a/plugins/esi/fetcher/HttpDataFetcherImpl.h +++ b/plugins/esi/fetcher/HttpDataFetcherImpl.h @@ -110,17 +110,15 @@ class HttpDataFetcherImpl : public HttpDataFetcher struct RequestData { std::string response; std::string raw_response; - const char *body; - int body_len; - TSHttpStatus resp_status; + const char *body = nullptr; + int body_len = 0; + TSHttpStatus resp_status = TS_HTTP_STATUS_NONE; CallbackObjectList callback_objects; - bool complete; - TSMBuffer bufp; - TSMLoc hdr_loc; + bool complete = false; + TSMBuffer bufp = nullptr; + TSMLoc hdr_loc = nullptr; - RequestData() : body(nullptr), body_len(0), resp_status(TS_HTTP_STATUS_NONE), complete(false), bufp(nullptr), hdr_loc(nullptr) - { - } + RequestData() {} }; typedef __gnu_cxx::hash_map UrlToContentMap; diff --git a/plugins/esi/lib/EsiGunzip.cc b/plugins/esi/lib/EsiGunzip.cc index 26bbea9c54c..167b664c285 100644 --- a/plugins/esi/lib/EsiGunzip.cc +++ b/plugins/esi/lib/EsiGunzip.cc @@ -34,7 +34,7 @@ EsiGunzip::EsiGunzip(const char *debug_tag, ComponentBase::Debug debug_func, Com { _init = false; _success = true; - // zlib _zstrm varibles are initialzied when they are required in stream_decode + // zlib _zstrm variables are initialized when they are required in stream_decode // coverity[uninit_member] // coverity[uninit_ctor] } diff --git a/plugins/esi/lib/EsiGzip.cc b/plugins/esi/lib/EsiGzip.cc index 6c0c7cfdea2..0786d593819 100644 --- a/plugins/esi/lib/EsiGzip.cc +++ b/plugins/esi/lib/EsiGzip.cc @@ -32,7 +32,7 @@ using namespace EsiLib; EsiGzip::EsiGzip(const char *debug_tag, ComponentBase::Debug debug_func, ComponentBase::Error error_func) : ComponentBase(debug_tag, debug_func, error_func), _downstream_length(0), _total_data_length(0) { - // Zlib _zstrm varibles are initialized when they are required in runDeflateLoop + // Zlib _zstrm variables are initialized when they are required in runDeflateLoop // coverity[uninit_member] // coverity[uninit_ctor] } diff --git a/plugins/esi/lib/EsiProcessor.cc b/plugins/esi/lib/EsiProcessor.cc index bb74e5a634a..379252b8b6b 100644 --- a/plugins/esi/lib/EsiProcessor.cc +++ b/plugins/esi/lib/EsiProcessor.cc @@ -29,7 +29,7 @@ using std::string; using namespace EsiLib; extern pthread_key_t threadKey; -// this needs to be a fixed address as only the address is used for comparision +// this needs to be a fixed address as only the address is used for comparison const char *EsiProcessor::INCLUDE_DATA_ID_ATTR = reinterpret_cast(0xbeadface); #define FAILURE_INFO_TAG "plugin_esi_failureInfo" @@ -305,7 +305,7 @@ EsiProcessor::process(const char *&data, int &data_len) /* FAILURE CACHE */ FailureData *data = static_cast(pthread_getspecific(threadKey)); - _debugLog("plugin_esi_failureInfo", "[%s]Fetched data related to thread specfic %p", __FUNCTION__, data); + _debugLog("plugin_esi_failureInfo", "[%s]Fetched data related to thread specific %p", __FUNCTION__, data); for (iter = try_iter->attempt_nodes.begin(); iter != try_iter->attempt_nodes.end(); ++iter) { if ((iter->type == DocNode::TYPE_INCLUDE) || iter->type == DocNode::TYPE_SPECIAL_INCLUDE) { @@ -342,7 +342,7 @@ EsiProcessor::process(const char *&data, int &data_len) } } if (attempt_succeeded) { - _debugLog(_debug_tag, "[%s] attempt section succeded; using attempt section", __FUNCTION__); + _debugLog(_debug_tag, "[%s] attempt section succeeded; using attempt section", __FUNCTION__); _node_list.splice(try_iter->pos, try_iter->attempt_nodes); } else { _debugLog(_debug_tag, "[%s] attempt section errored; trying except section", __FUNCTION__); @@ -436,7 +436,7 @@ EsiProcessor::flush(string &data, int &overall_len) /* FAILURE CACHE */ FailureData *fdata = static_cast(pthread_getspecific(threadKey)); - _debugLog("plugin_esi_failureInfo", "[%s]Fetched data related to thread specfic %p", __FUNCTION__, fdata); + _debugLog("plugin_esi_failureInfo", "[%s]Fetched data related to thread specific %p", __FUNCTION__, fdata); for (iter = try_iter->attempt_nodes.begin(); iter != try_iter->attempt_nodes.end(); ++iter) { if ((iter->type == DocNode::TYPE_INCLUDE) || iter->type == DocNode::TYPE_SPECIAL_INCLUDE) { @@ -473,7 +473,7 @@ EsiProcessor::flush(string &data, int &overall_len) } } if (attempt_succeeded) { - _debugLog(_debug_tag, "[%s] attempt section succeded; using attempt section", __FUNCTION__); + _debugLog(_debug_tag, "[%s] attempt section succeeded; using attempt section", __FUNCTION__); _n_prescanned_nodes = _n_prescanned_nodes + try_iter->attempt_nodes.size(); _node_list.splice(try_iter->pos, try_iter->attempt_nodes); } else { @@ -699,7 +699,7 @@ EsiProcessor::_handleHtmlComment(const DocNodeList::iterator &curr_node) _debugLog(_debug_tag, "[%s] parsed %d inner nodes from html comment node", __FUNCTION__, inner_nodes.size()); DocNodeList::iterator next_node = curr_node; ++next_node; - _node_list.splice(next_node, inner_nodes); // insert after curr node for preprocessing + _node_list.splice(next_node, inner_nodes); // insert after curr node for pre-processing return true; } @@ -733,9 +733,9 @@ EsiProcessor::_preprocess(DocNodeList &node_list, int &n_prescanned_nodes) break; case DocNode::TYPE_HTML_COMMENT: /** - * the html comment is a container. + * the html comment is a container. * the esi processor will remove the starting tag "", then keep the innertext (the content within it). + * closure tag "-->", then keep the inner text (the content within it). * * we should call _handleHtmlComment when the node list is parsed * from the content, diff --git a/plugins/esi/lib/FailureInfo.h b/plugins/esi/lib/FailureInfo.h index ba8e61ea073..ffb4a9e04fc 100644 --- a/plugins/esi/lib/FailureInfo.h +++ b/plugins/esi/lib/FailureInfo.h @@ -101,12 +101,12 @@ class FailureInfo : private EsiLib::ComponentBase /* Keep track of the number of windows filled prev*/ size_t _windowsPassed; - /*Used as a deciding factor between attempt/except - * incase prob is complete truth + /* Used as a deciding factor between attempt/except + * in case prob is complete truth */ double _avgOverWindow; public: - /*Was a reqeust made*/ + /*Was a request made*/ bool _requestMade; }; diff --git a/plugins/esi/lib/HandlerManager.h b/plugins/esi/lib/HandlerManager.h index 607027b654d..7b44d8651e1 100644 --- a/plugins/esi/lib/HandlerManager.h +++ b/plugins/esi/lib/HandlerManager.h @@ -45,7 +45,7 @@ class HandlerManager : protected ComponentBase SpecialIncludeHandler *getHandler(Variables &esi_vars, Expression &esi_expr, HttpDataFetcher &http_fetcher, const std::string &id) const; - ~HandlerManager(); + ~HandlerManager() override; private: typedef std::map FunctionHandleMap; diff --git a/plugins/esi/lib/Stats.cc b/plugins/esi/lib/Stats.cc index 828ed809463..d048b3a47bd 100644 --- a/plugins/esi/lib/Stats.cc +++ b/plugins/esi/lib/Stats.cc @@ -42,7 +42,7 @@ namespace Stats g_system = system; if (g_system) { for (int i = 0; i < Stats::MAX_STAT_ENUM; ++i) { - // FIXME doesn't return avalue. + // FIXME doesn't return a value. g_system->create(i); /* if (!g_system->create(i)) { Utils::ERROR_LOG("[%s] Unable to create stat [%s]", __FUNCTION__, Stats::STAT_NAMES[i]); @@ -56,7 +56,7 @@ namespace Stats increment(Stats::STAT st, int step /* = 1 */) { if (g_system) { - // FIXME doesn't return avalue. + // FIXME doesn't return a value. g_system->increment(st, step); /* if (!g_system->increment(st, step)) { diff --git a/plugins/esi/lib/Variables.cc b/plugins/esi/lib/Variables.cc index ca304857715..adec87e72e1 100644 --- a/plugins/esi/lib/Variables.cc +++ b/plugins/esi/lib/Variables.cc @@ -437,18 +437,18 @@ Variables::_parseDictVariable(const std::string &variable, const char *&header, for (int i = 0; i < (var_size - 1); ++i) { if (variable[i] == '{') { if (paranth_index != -1) { - _debugLog(_debug_tag, "[%s] Cannot have multiple paranthesis in dict variable [%.*s]", __FUNCTION__, var_size, var_ptr); + _debugLog(_debug_tag, "[%s] Cannot have multiple parenthesis in dict variable [%.*s]", __FUNCTION__, var_size, var_ptr); return false; } paranth_index = i; } if (variable[i] == '}') { - _debugLog(_debug_tag, "[%s] Cannot have multiple paranthesis in dict variable [%.*s]", __FUNCTION__, var_size, var_ptr); + _debugLog(_debug_tag, "[%s] Cannot have multiple parenthesis in dict variable [%.*s]", __FUNCTION__, var_size, var_ptr); return false; } } if (paranth_index == -1) { - _debugLog(_debug_tag, "[%s] Could not find opening paranthesis in variable [%.*s]", __FUNCTION__, var_size, var_ptr); + _debugLog(_debug_tag, "[%s] Could not find opening parenthesis in variable [%.*s]", __FUNCTION__, var_size, var_ptr); return false; } if (paranth_index == 0) { diff --git a/plugins/esi/serverIntercept.cc b/plugins/esi/serverIntercept.cc index a7c1c26c9de..0fff06f500c 100644 --- a/plugins/esi/serverIntercept.cc +++ b/plugins/esi/serverIntercept.cc @@ -43,10 +43,10 @@ struct SContData { TSCont contp; struct IoHandle { - TSVIO vio; - TSIOBuffer buffer; - TSIOBufferReader reader; - IoHandle() : vio(nullptr), buffer(nullptr), reader(nullptr){}; + TSVIO vio = nullptr; + TSIOBuffer buffer = nullptr; + TSIOBufferReader reader = nullptr; + IoHandle(){}; ~IoHandle() { if (reader) { diff --git a/plugins/esi/test/TestHttpDataFetcher.h b/plugins/esi/test/TestHttpDataFetcher.h index 9eecd09b688..cd2ee590527 100644 --- a/plugins/esi/test/TestHttpDataFetcher.h +++ b/plugins/esi/test/TestHttpDataFetcher.h @@ -30,7 +30,7 @@ class TestHttpDataFetcher : public HttpDataFetcher { public: - TestHttpDataFetcher() : _n_pending_requests(0), _return_data(true) {} + TestHttpDataFetcher() {} bool addFetchRequest(const std::string &url, FetchedDataProcessor *callback_obj = nullptr) { @@ -84,7 +84,7 @@ class TestHttpDataFetcher : public HttpDataFetcher }; private: - int _n_pending_requests; + int _n_pending_requests = 0; std::string _data; - bool _return_data; + bool _return_data = true; }; diff --git a/plugins/esi/test/sampleProb.cc b/plugins/esi/test/sampleProb.cc index db460bc1d44..0f22bcc3830 100644 --- a/plugins/esi/test/sampleProb.cc +++ b/plugins/esi/test/sampleProb.cc @@ -129,7 +129,7 @@ isAttemptReq(string URL, FailureData &data) // cout<<"Failure:"< 0) { avg += passFail[i].first / (passFail[i].first + passFail[i].second); - // cout<<"Prob of faillure:"<("invalid-syntax-status-code"), optional_argument, 0, 'a'}, - {const_cast("invalid-signature-status-code"), optional_argument, 0, 'b'}, - {const_cast("invalid-timing-status-code"), optional_argument, 0, 'c'}, - {const_cast("invalid-scope-status-code"), optional_argument, 0, 'd'}, - {const_cast("invalid-origin-response"), optional_argument, 0, 'e'}, - {const_cast("internal-error-status-code"), optional_argument, 0, 'f'}, - {const_cast("check-cookie"), optional_argument, 0, 'g'}, - {const_cast("symmetric-keys-map"), optional_argument, 0, 'h'}, - {const_cast("reject-invalid-token-requests"), optional_argument, 0, 'i'}, - {const_cast("extract-subject-to-header"), optional_argument, 0, 'j'}, - {const_cast("extract-tokenid-to-header"), optional_argument, 0, 'k'}, - {const_cast("extract-status-to-header"), optional_argument, 0, 'l'}, - {const_cast("token-response-header"), optional_argument, 0, 'm'}, - {const_cast("use-redirects"), optional_argument, 0, 'n'}, + static const struct option longopt[] = {{const_cast("invalid-syntax-status-code"), optional_argument, nullptr, 'a'}, + {const_cast("invalid-signature-status-code"), optional_argument, nullptr, 'b'}, + {const_cast("invalid-timing-status-code"), optional_argument, nullptr, 'c'}, + {const_cast("invalid-scope-status-code"), optional_argument, nullptr, 'd'}, + {const_cast("invalid-origin-response"), optional_argument, nullptr, 'e'}, + {const_cast("internal-error-status-code"), optional_argument, nullptr, 'f'}, + {const_cast("check-cookie"), optional_argument, nullptr, 'g'}, + {const_cast("symmetric-keys-map"), optional_argument, nullptr, 'h'}, + {const_cast("reject-invalid-token-requests"), optional_argument, nullptr, 'i'}, + {const_cast("extract-subject-to-header"), optional_argument, nullptr, 'j'}, + {const_cast("extract-tokenid-to-header"), optional_argument, nullptr, 'k'}, + {const_cast("extract-status-to-header"), optional_argument, nullptr, 'l'}, + {const_cast("token-response-header"), optional_argument, nullptr, 'm'}, + {const_cast("use-redirects"), optional_argument, nullptr, 'n'}, {const_cast("include-uri-paths-file"), optional_argument, nullptr, 'o'}, {const_cast("exclude-uri-paths-file"), optional_argument, nullptr, 'p'}, - {0, 0, 0, 0}}; + {nullptr, 0, nullptr, 0}}; bool status = true; optind = 0; diff --git a/plugins/experimental/access_control/config.h b/plugins/experimental/access_control/config.h index 6611c701b18..4d1ab9e0a52 100644 --- a/plugins/experimental/access_control/config.h +++ b/plugins/experimental/access_control/config.h @@ -66,5 +66,5 @@ class AccessControlConfig String _extrTokenIdHdrName; /** @brief header name to extract the token id, if empty => no extraction */ String _extrValidationHdrName; /** @brief header name to extract the token validation status, if empty => no extraction */ bool _useRedirects = false; /** @brief true - use redirect to set the access token cookie, @todo not used yet */ - Classifier _uriPathScope; /**< @brief blacklist (exclude) and white-list (include) whcih path should have the access control */ + Classifier _uriPathScope; /**< @brief blacklist (exclude) and white-list (include) which path should have the access control */ }; diff --git a/plugins/experimental/access_control/headers.cc b/plugins/experimental/access_control/headers.cc index fda3a74501d..ddc64e4c33e 100644 --- a/plugins/experimental/access_control/headers.cc +++ b/plugins/experimental/access_control/headers.cc @@ -21,8 +21,8 @@ * @brief HTTP headers manipulation. */ -#include -#include +#include +#include #include "headers.h" #include "common.h" diff --git a/plugins/experimental/access_control/pattern.cc b/plugins/experimental/access_control/pattern.cc index 2c04b6a0267..fc9b4e9a562 100644 --- a/plugins/experimental/access_control/pattern.cc +++ b/plugins/experimental/access_control/pattern.cc @@ -38,7 +38,7 @@ replaceString(String &str, const String &from, const String &to) } } -Pattern::Pattern() : _re(nullptr), _extra(nullptr), _pattern(""), _replacement(""), _replace(false), _tokenCount(0) {} +Pattern::Pattern() : _pattern(""), _replacement("") {} /** * @brief Initializes PCRE pattern by providing the subject and replacement strings. @@ -47,18 +47,18 @@ Pattern::Pattern() : _re(nullptr), _extra(nullptr), _pattern(""), _replacement(" * @return true if successful, false if failure */ bool -Pattern::init(const String &pattern, const String &replacenemt, bool replace) +Pattern::init(const String &pattern, const String &replacement, bool replace) { pcreFree(); _pattern.assign(pattern); - _replacement.assign(replacenemt); + _replacement.assign(replacement); _replace = replace; _tokenCount = 0; if (!compile()) { - AccessControlDebug("failed to initialize pattern:'%s', replacement:'%s'", pattern.c_str(), replacenemt.c_str()); + AccessControlDebug("failed to initialize pattern:'%s', replacement:'%s'", pattern.c_str(), replacement.c_str()); pcreFree(); return false; } @@ -160,7 +160,7 @@ Pattern::pcreFree() } /** - * @bried Destructor, frees PCRE related resources. + * @brief Destructor, frees PCRE related resources. */ Pattern::~Pattern() { diff --git a/plugins/experimental/access_control/pattern.h b/plugins/experimental/access_control/pattern.h index 57f2d9f601d..f4f37a7a600 100644 --- a/plugins/experimental/access_control/pattern.h +++ b/plugins/experimental/access_control/pattern.h @@ -43,7 +43,7 @@ class Pattern Pattern(); virtual ~Pattern(); - bool init(const String &pattern, const String &replacenemt, bool replace); + bool init(const String &pattern, const String &replacement, bool replace); bool init(const String &config); bool empty() const; bool match(const String &subject); @@ -56,16 +56,16 @@ class Pattern bool compile(); void pcreFree(); - pcre *_re; /**< @brief PCRE compiled info structure, computed during initialization */ - pcre_extra *_extra; /**< @brief PCRE study data block, computed during initialization */ + pcre *_re = nullptr; /**< @brief PCRE compiled info structure, computed during initialization */ + pcre_extra *_extra = nullptr; /**< @brief PCRE study data block, computed during initialization */ String _pattern; /**< @brief PCRE pattern string, containing PCRE patterns and capturing groups. */ String _replacement; /**< @brief PCRE replacement string, containing $0..$9 to be replaced with content of the capturing groups */ - bool _replace; /**< @brief true if a replacement is needed, false if not, this is to distinguish between an empty replacement - string and no replacement needed case */ + bool _replace = false; /**< @brief true if a replacement is needed, false if not, this is to distinguish between an empty + replacement string and no replacement needed case */ - int _tokenCount; /**< @brief number of replacements $0..$9 found in the replacement string if not empty */ + int _tokenCount = 0; /**< @brief number of replacements $0..$9 found in the replacement string if not empty */ int _tokens[TOKENCOUNT]; /**< @brief replacement index 0..9, since they can be used in the replacement string in any order */ int _tokenOffset[TOKENCOUNT]; /**< @brief replacement offset inside the replacement string */ }; @@ -76,7 +76,7 @@ class Pattern class MultiPattern { public: - MultiPattern(const String name = "") : _name(name) {} + MultiPattern(const String &name = "") : _name(name) {} virtual ~MultiPattern(); bool empty() const; @@ -106,14 +106,14 @@ class NonMatchingMultiPattern : public MultiPattern * @param subject subject string * @return return false if any of the patterns matches, true otherwise. */ - virtual bool - match(const String &subject) const + bool + match(const String &subject) const override { return !MultiPattern::match(subject); } - virtual bool - match(const String &subject, String &pattern) const + bool + match(const String &subject, String &pattern) const override { return !MultiPattern::match(subject, pattern); } diff --git a/plugins/experimental/access_control/plugin.cc b/plugins/experimental/access_control/plugin.cc index bcc68f9c763..4db3ac529dc 100644 --- a/plugins/experimental/access_control/plugin.cc +++ b/plugins/experimental/access_control/plugin.cc @@ -32,6 +32,8 @@ #include "utils.h" /* cryptoBase64Decode.* functions */ #include "headers.h" /* getHeader, setHeader, removeHeader */ +static const std::string_view UNKNOWN{"unknown"}; + static const char * getEventName(TSEvent event) { @@ -357,7 +359,7 @@ contHandleAccessControl(const TSCont contp, TSEvent event, void *edata) AccessToken *token = config->_tokenFactory->getAccessToken(); if (nullptr != token && - VALID == (data->_originState = token->validate(StringView(tokenHdrValue, tokenHdrValueLen), time(0)))) { + VALID == (data->_originState = token->validate(StringView(tokenHdrValue, tokenHdrValueLen), time(nullptr)))) { /* * From RFC 6265 "HTTP State Management Mechanism": * To maximize compatibility with user agents, servers that wish to @@ -415,12 +417,18 @@ contHandleAccessControl(const TSCont contp, TSEvent event, void *edata) TSHandleMLocRelease(serverRespBufp, TS_NULL_MLOC, serverRespHdrLoc); } else { - AccessControlError("failed to retrieve server response header"); + int len; + char *url = TSHttpTxnEffectiveUrlStringGet(txnp, &len); + AccessControlError("failed to retrieve server response header for request url:%.*s", + (len ? len : static_cast(UNKNOWN.size())), (url ? url : UNKNOWN.data())); } TSHandleMLocRelease(clientRespBufp, TS_NULL_MLOC, clientRespHdrLoc); } else { - AccessControlError("failed to retrieve client response header"); + int len; + char *url = TSHttpTxnEffectiveUrlStringGet(txnp, &len); + AccessControlError("failed to retrieve client response header for request url:%.*s", + (len ? len : static_cast(UNKNOWN.size())), (url ? url : UNKNOWN.data())); } } } break; @@ -512,7 +520,7 @@ enforceAccessControl(TSHttpTxn txnp, TSRemapRequestInfo *rri, AccessControlConfi if (0 < decryptedCookieSize) { AccessToken *token = config->_tokenFactory->getAccessToken(); if (nullptr != token) { - data->_vaState = token->validate(StringView(decodedCookie, decryptedCookieSize), time(0)); + data->_vaState = token->validate(StringView(decodedCookie, decryptedCookieSize), time(nullptr)); if (VALID != data->_vaState) { remapStatus = handleInvalidToken(txnp, data, reject, accessTokenStateToHttpStatus(data->_vaState, config), data->_vaState); @@ -583,7 +591,7 @@ TSRemapDoRemap(void *instance, TSHttpTxn txnp, TSRemapRequestInfo *rri) String pattern; if (config->_uriPathScope.empty()) { /* Scope match enforce access control */ - AccessControlError("no plugin scope specified, enforcing access control"); + AccessControlDebug("no plugin scope specified, enforcing access control"); remapStatus = enforceAccessControl(txnp, rri, config); } else { if (true == config->_uriPathScope.matchAll(reqPath, filename, pattern)) { @@ -592,13 +600,13 @@ TSRemapDoRemap(void *instance, TSHttpTxn txnp, TSRemapRequestInfo *rri) /* Scope match enforce access control */ remapStatus = enforceAccessControl(txnp, rri, config); } else { - AccessControlError("not matching plugin scope (file: %s, pattern %s), skipping access control for path '%s'", + AccessControlDebug("not matching plugin scope (file: %s, pattern %s), skipping access control for path '%s'", filename.c_str(), pattern.c_str(), reqPath.c_str()); } } } else { TSHttpTxnStatusSet(txnp, config->_invalidRequest); - AccessControlError("https is the only allowed scheme (plugin should be used only with TLS)"); + AccessControlDebug("https is the only allowed scheme (plugin should be used only with TLS)"); remapStatus = TSREMAP_DID_REMAP; } } else { diff --git a/plugins/experimental/access_control/utils.cc b/plugins/experimental/access_control/utils.cc index cfddac70e46..2915db76837 100644 --- a/plugins/experimental/access_control/utils.cc +++ b/plugins/experimental/access_control/utils.cc @@ -22,8 +22,8 @@ * @see utils.h */ -#include /* errno */ -#include /* LONG_MIN, LONG_MAX */ +#include /* errno */ +#include /* LONG_MIN, LONG_MAX */ #include /* BIO I/O abstraction */ #include /* buf_mem_st */ #include /* ERR_get_error() and ERR_error_string_n() */ @@ -279,7 +279,7 @@ cryptoMessageDigestGet(const char *digestType, const char *data, size_t dataLen, /* success */ result = len; - } while (0); + } while (false); EVP_PKEY_free(pkey); EVP_MD_CTX_destroy(ctx); diff --git a/plugins/experimental/access_control/utils.h b/plugins/experimental/access_control/utils.h index 02f027e5f35..6c821df692e 100644 --- a/plugins/experimental/access_control/utils.h +++ b/plugins/experimental/access_control/utils.h @@ -26,7 +26,7 @@ #include /* std:string_view */ #include /* EVP_* constants, structures and functions. */ -#include /* strlen, strncmp, strncpy, memset, size_t */ +#include /* strlen, strncmp, strncpy, memset, size_t */ #define MAX_MSGDIGEST_BUFFER_SIZE EVP_MAX_MD_SIZE diff --git a/plugins/experimental/acme/acme.c b/plugins/experimental/acme/acme.c index 32c899d61a3..afb04578464 100644 --- a/plugins/experimental/acme/acme.c +++ b/plugins/experimental/acme/acme.c @@ -77,7 +77,7 @@ make_absolute_path(char *dest, int dest_len, const char *file, int file_len) for (i = 0; i < file_len; ++i) { char c = file[i]; - /* Assure that only Base64-URL chracter are in the path */ + /* Assure that only Base64-URL character are in the path */ if (!(c == '-' || c == '_' || (c >= '0' && c <= '9') || (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z'))) { TSDebug(PLUGIN_NAME, "Invalid Base64 character found, error"); return 0; @@ -208,7 +208,7 @@ acme_process_write(TSCont contp, TSEvent event, AcmeState *my_state) TSVIONBytesSet(my_state->write_vio, my_state->output_bytes); TSVIOReenable(my_state->write_vio); - } else if (TS_EVENT_VCONN_WRITE_COMPLETE) { + } else if (event == TS_EVENT_VCONN_WRITE_COMPLETE) { cleanup(contp, my_state); } else if (event == TS_EVENT_ERROR) { TSError("[%s] acme_process_write: Received TS_EVENT_ERROR", PLUGIN_NAME); diff --git a/plugins/experimental/balancer/hash.cc b/plugins/experimental/balancer/hash.cc index d08df3ab861..c6ff0a60331 100644 --- a/plugins/experimental/balancer/hash.cc +++ b/plugins/experimental/balancer/hash.cc @@ -181,7 +181,7 @@ struct HashBalancer : public BalancerInstance { // OK, now look up this hash in the hash ring. lower_bound() finds the first element that is not less than the // target, so the element we find is the first key that is greater than our target. To visualize this in the - // hash ring, that means that each node owns the preceeding keyspace (ie. the node is at the end of each keyspace + // hash ring, that means that each node owns the preceding keyspace (ie. the node is at the end of each keyspace // range). This means that when we wrap, the first node owns the wrapping portion of the keyspace. loc = this->hash_ring.lower_bound(key); if (loc == this->hash_ring.end()) { diff --git a/plugins/experimental/balancer/roundrobin.cc b/plugins/experimental/balancer/roundrobin.cc index 16f0ffd3419..6090ab97795 100644 --- a/plugins/experimental/balancer/roundrobin.cc +++ b/plugins/experimental/balancer/roundrobin.cc @@ -31,7 +31,7 @@ namespace { struct RoundRobinBalancer : public BalancerInstance { - RoundRobinBalancer() : targets(), next(0) {} + RoundRobinBalancer() : targets() {} void push_target(const BalancerTarget &target) override { @@ -45,7 +45,7 @@ struct RoundRobinBalancer : public BalancerInstance { } std::vector targets; - unsigned next; + unsigned next = 0; }; } // namespace diff --git a/plugins/experimental/buffer_upload/README b/plugins/experimental/buffer_upload/README index 90a6c2d29bd..24d4b41e7d7 100644 --- a/plugins/experimental/buffer_upload/README +++ b/plugins/experimental/buffer_upload/README @@ -11,7 +11,7 @@ Upload proxy specs for phase I: 1. Memory buffering (buffer the entire POST data in IOBuffer before connecting to OS) 1.1. Memory buffer size is configured with "mem_buffer_size" in config file. Default and minimum value is 32K - You can increase it in the config file. If a request's size is larger than "mem_buffer_size" specifiied + You can increase it in the config file. If a request's size is larger than "mem_buffer_size" specified in config file, then the upload proxy feature will be disabled for this particular request 2. Disk buffering (buffer the entire POST data on disk before connecting to OS) @@ -43,9 +43,9 @@ Upload proxy specs for phase I: 5.1. for now check if the "host" part in the URL is same as the proxy server name, then will do this conversion 5.2. To turn on URL conversion feature, set "convert_url 1" in config file -6. All request headers inlcuding cookies plus the entire POST data will be buffered (either in memory or on disk) +6. All request headers including cookies plus the entire POST data will be buffered (either in memory or on disk) -7. Config file can be explicitly sepcified as a parameter in command line (in plugin.config file) +7. Config file can be explicitly specified as a parameter in command line (in plugin.config file) a sample config file: diff --git a/plugins/experimental/buffer_upload/buffer_upload.cc b/plugins/experimental/buffer_upload/buffer_upload.cc index fc6c3f601a0..92e64fd1fcd 100644 --- a/plugins/experimental/buffer_upload/buffer_upload.cc +++ b/plugins/experimental/buffer_upload/buffer_upload.cc @@ -543,12 +543,12 @@ pvc_plugin(TSCont contp, TSEvent event, void *edata) pvc_process_n_read(contp, event, my_state); } else if (edata == my_state->n_write_vio) { pvc_process_n_write(contp, event, my_state); - } else if (event == TS_AIO_EVENT_DONE && uconfig->use_disk_buffer) { + } else if (event == TS_EVENT_AIO_DONE && uconfig->use_disk_buffer) { TSMutexLock(my_state->disk_io_mutex); int size = TSAIONBytesGet(callback); char *buf = TSAIOBufGet(callback); if (buf != my_state->chunk_buffer) { - // this TS_AIO_EVENT_DONE event is from TSAIOWrite() + // this TS_EVENT_AIO_DONE event is from TSAIOWrite() TSDebug(DEBUG_TAG, "aio write size: %d", size); my_state->size_written += size; if (buf != nullptr) { @@ -561,7 +561,7 @@ pvc_plugin(TSCont contp, TSEvent event, void *edata) } } } else { - // this TS_AIO_EVENT_DONE event is from TSAIORead() + // this TS_EVENT_AIO_DONE event is from TSAIORead() TSDebug(DEBUG_TAG, "aio read size: %d", size); TSIOBufferWrite(my_state->req_buffer, my_state->chunk_buffer, size); my_state->size_read += size; diff --git a/plugins/experimental/cache_range_requests/README b/plugins/experimental/cache_range_requests/README index 12ea4d03608..5495f687d0f 100644 --- a/plugins/experimental/cache_range_requests/README +++ b/plugins/experimental/cache_range_requests/README @@ -16,17 +16,6 @@ object is written to cache using the new cache key url. The response code sent to the client will be changed back to a 206 and all requests to the origin server will contain the range header so that the correct response is received. -Installation: - - make - sudo make install - -If you don't have the traffic server binaries in your path, then you will need -to specify the path to tsxs manually: - - make TSXS=/opt/trafficserver/bin/tsxs - sudo make TSXS=/opt/trafficserver/bin/tsxs install - Configuration: Add @plugin=cache_range_requests.so to your remap.config rules. @@ -34,7 +23,7 @@ Configuration: Or for a global plugin where all range requests are processed, Add cache_range_requests.so to the plugin.config -Parent Selection Mode (consisent-hash only): +Parent Selection Mode (consistent-hash only): default: Parent selection is based solely on the hash of a URL Path In this mode, all partial content of a URL is requested from the same diff --git a/plugins/experimental/cache_range_requests/cache_range_requests.cc b/plugins/experimental/cache_range_requests/cache_range_requests.cc index b0b61ec1cd1..3a25fa69035 100644 --- a/plugins/experimental/cache_range_requests/cache_range_requests.cc +++ b/plugins/experimental/cache_range_requests/cache_range_requests.cc @@ -21,7 +21,7 @@ * This plugin looks for range requests and then creates a new * cache key url so that each individual range requests is written * to the cache as a individual object so that subsequent range - * requests are read accross different disk drives reducing I/O + * requests are read across different disk drives reducing I/O * wait and load averages when there are large numbers of range * requests. */ @@ -94,7 +94,7 @@ create_pluginconfig(int argc, const char *argv[]) } /** - * Destroy pluginconfig data stucture. + * Destroy pluginconfig data structure. */ static void delete_pluginconfig(struct pluginconfig *pc) diff --git a/plugins/experimental/certifier/certifier.cc b/plugins/experimental/certifier/certifier.cc index 9415d75f7b0..2cbe9e022e8 100644 --- a/plugins/experimental/certifier/certifier.cc +++ b/plugins/experimental/certifier/certifier.cc @@ -108,7 +108,7 @@ class SslLRUList using scoped_SslData = std::unique_ptr; // unordered_map is much faster in terms of insertion/lookup/removal - // Althogh it uses more space than map, the time efficiency should be more important + // Although it uses more space than map, the time efficiency should be more important std::unordered_map cnDataMap; ///< Map from CN to sslData TSMutex list_mutex; @@ -455,7 +455,7 @@ shadow_cert_generator(TSCont contp, TSEvent event, void *edata) if (cert == nullptr) { if (!sign_enabled) { TSDebug(PLUGIN_NAME, "shadow_cert_generator(): No certs found and dynamic generation disabled. Marked as wontdo."); - // There won't be certs avaiable. Mark this servername as wontdo + // There won't be certs available. Mark this servername as wontdo // Pass on as if plugin doesn't exist ssl_list->setup_data_ctx(commonName, localQ, nullptr, nullptr, true); while (!localQ.empty()) { @@ -568,7 +568,7 @@ cert_retriever(TSCont contp, TSEvent event, void *edata) TSDebug(PLUGIN_NAME, "cert_retriever(): schedule thread to generate/retrieve cert for %s", servername); TSCont schedule_cont = TSContCreate(shadow_cert_generator, TSMutexCreate()); TSContDataSet(schedule_cont, (void *)servername); - TSContSchedule(schedule_cont, 0, TS_THREAD_POOL_TASK); + TSContScheduleOnPool(schedule_cont, 0, TS_THREAD_POOL_TASK); } else { // Use existing context TSDebug(PLUGIN_NAME, "cert_retriever(): Reuse existing cert and context for %s", servername); @@ -576,7 +576,7 @@ cert_retriever(TSCont contp, TSEvent event, void *edata) TSVConnReenable(ssl_vc); } - /// For scheduled connections, the schduled continuation will handle the reenabling + /// For scheduled connections, the scheduled continuation will handle the reenabling return TS_SUCCESS; } diff --git a/plugins/experimental/collapsed_forwarding/README b/plugins/experimental/collapsed_forwarding/README index 49ee7f882f3..ca0a0025972 100644 --- a/plugins/experimental/collapsed_forwarding/README +++ b/plugins/experimental/collapsed_forwarding/README @@ -25,24 +25,10 @@ // contention and so can not work. The setting proxy.config.http.wait_for_cache // may be enabled which allows blocking incoming connections from being // accepted until cache is ready. -//////////////////////////////////////////////////////////////////////////////////// -// This plugin currently supports only per-remap mode activation. -//////////////////////////////////////////////////////////////////////////////////// More details are available at -https://docs.trafficserver.apache.org/en/6.0.x/admin/http-proxy-caching.en.html#reducing-origin-server-requests-avoiding-the-thundering-herd - -Installation: - - make - sudo make install - -If you don't have the traffic server binaries in your path, then you will need -to specify the path to tsxs manually: - - make TSXS=/opt/trafficserver/bin/tsxs - sudo make TSXS=/opt/trafficserver/bin/tsxs install +https://docs.trafficserver.apache.org/en/latest/admin-guide/configuration/cache-basics.en.html#reducing-origin-server-requests-avoiding-the-thundering-herd Configuration: diff --git a/plugins/experimental/collapsed_forwarding/collapsed_forwarding.cc b/plugins/experimental/collapsed_forwarding/collapsed_forwarding.cc index 9e4d41ed2ba..28561e5242f 100644 --- a/plugins/experimental/collapsed_forwarding/collapsed_forwarding.cc +++ b/plugins/experimental/collapsed_forwarding/collapsed_forwarding.cc @@ -210,7 +210,7 @@ on_send_response_header(RequestData *req, TSHttpTxn &txnp, TSCont &contp) if (delay_request) { req->wl_retry++; TSDebug(DEBUG_TAG, "delaying request, url@%p: {{%s}} on retry: %d time", txnp, req->req_url.c_str(), req->wl_retry); - TSContSchedule(contp, OPEN_WRITE_FAIL_REQ_DELAY_TIMEOUT, TS_THREAD_POOL_TASK); + TSContScheduleOnPool(contp, OPEN_WRITE_FAIL_REQ_DELAY_TIMEOUT, TS_THREAD_POOL_TASK); TSHandleMLocRelease(bufp, TS_NULL_MLOC, hdr_loc); return TS_SUCCESS; } @@ -353,7 +353,7 @@ TSRemapInit(TSRemapInterface * /* api_info */, char * /* errbuf */, int /* errbu TSError("Cannot initialize %s as both global and remap plugin", DEBUG_TAG); return TS_ERROR; } else { - TSDebug(DEBUG_TAG, "plugin is succesfully initialized for remap"); + TSDebug(DEBUG_TAG, "plugin is successfully initialized for remap"); return TS_SUCCESS; } } diff --git a/plugins/experimental/cookie_remap/Makefile.inc b/plugins/experimental/cookie_remap/Makefile.inc new file mode 100644 index 00000000000..266cc9e4072 --- /dev/null +++ b/plugins/experimental/cookie_remap/Makefile.inc @@ -0,0 +1,40 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +pkglib_LTLIBRARIES += experimental/cookie_remap/cookie_remap.la + +experimental_cookie_remap_cookie_remap_la_SOURCES = \ + experimental/cookie_remap/cookie_remap.cc \ + experimental/cookie_remap/hash.c \ + experimental/cookie_remap/strip.cc \ + experimental/cookie_remap/cookiejar.cc + +experimental_cookie_remap_cookie_remap_la_LDFLAGS = \ + $(AM_LDFLAGS) + +AM_CPPFLAGS += @YAMLCPP_INCLUDES@ + +check_PROGRAMS += \ + experimental/cookie_remap/test_cookiejar + +experimental_cookie_remap_test_cookiejar_CPPFLAGS = $(AM_CPPFLAGS) -Iexperimental/cookie_remap -I$(abs_top_srcdir)/tests/include +experimental_cookie_remap_test_cookiejar_SOURCES = \ + experimental/cookie_remap/test_cookiejar.cc \ + experimental/cookie_remap/strip.cc \ + experimental/cookie_remap/cookiejar.cc \ + experimental/cookie_remap/cookiejar.h + +# vim: ft=make ts=8 sw=8 et: diff --git a/plugins/experimental/cookie_remap/README b/plugins/experimental/cookie_remap/README new file mode 100644 index 00000000000..d9064f3ce8e --- /dev/null +++ b/plugins/experimental/cookie_remap/README @@ -0,0 +1,9 @@ +============================================================ +Cookie Based Routing Inside TrafficServer Using cookie_remap +============================================================ + +---- + + +For details please read the official documentation in the Admin Guide section. + diff --git a/plugins/experimental/cookie_remap/cookie_remap.cc b/plugins/experimental/cookie_remap/cookie_remap.cc new file mode 100644 index 00000000000..000986e3cf3 --- /dev/null +++ b/plugins/experimental/cookie_remap/cookie_remap.cc @@ -0,0 +1,1147 @@ +/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ +//////////////////////////////////////////////////////////////////////////////// +// cookie_remap: ATS plugin to do (simple) cookie based remap rules +// To use this plugin, configure a remap.config rule like +// map http://foo.com http://bar.com @plugin=.../libexec/cookie_remap.so +// @pparam=maps.reg + +#include "cookiejar.h" +#include + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include "hash.h" + +using namespace std; + +// for bucketizing + +#define MY_NAME "cookie_remap" +const int OVECCOUNT = 30; // We support $1 - $9 only, and this needs to be 3x that + +#if TS_VERSION_MAJOR > 7 +#define SETHTTPSTATUS(TXN, STATUS) TSHttpTxnStatusSet((TXN), STATUS) +#else +#define SETHTTPSTATUS(TXN, STATUS) TSHttpTxnSetHttpRetStatus((TXN), STATUS) +#endif + +/////////////////////////////////////////////////////////////////////////////// +// Helpers for memory management (to make sure pcre uses the TS APIs). +// +inline void * +ink_malloc(size_t s) +{ + return TSmalloc(s); +} + +inline void +ink_free(void *s) +{ + return TSfree(s); +} +/////////////////////////////////////////////////////////////////////////////// +// Class holding one request URL's component, to simplify the code and +// length calculations (we need all of them). +// +struct UrlComponents { + UrlComponents() {} + + ~UrlComponents() + { + if (url != nullptr) + TSfree((void *)url); + } + + void + populate(TSRemapRequestInfo *rri) + { + scheme = TSUrlSchemeGet(rri->requestBufp, rri->requestUrl, &scheme_len); + host = TSUrlHostGet(rri->requestBufp, rri->requestUrl, &host_len); + tspath = TSUrlPathGet(rri->requestBufp, rri->requestUrl, &tspath_len); + query = TSUrlHttpQueryGet(rri->requestBufp, rri->requestUrl, &query_len); + matrix = TSUrlHttpParamsGet(rri->requestBufp, rri->requestUrl, &matrix_len); + port = TSUrlPortGet(rri->requestBufp, rri->requestUrl); + url = TSUrlStringGet(rri->requestBufp, rri->requestUrl, &url_len); + from_path = TSUrlPathGet(rri->requestBufp, rri->mapFromUrl, &from_path_len); + + // based on RFC2396, matrix params are part of path segments so + // we will just + // append them to the path + path.assign(tspath, tspath_len); + if (matrix_len) { + path += ';'; + path.append(matrix, matrix_len); + } + } + + const char *scheme = nullptr; + const char *host = nullptr; + const char *tspath = nullptr; + std::string path{}; + const char *query = nullptr; + const char *matrix = nullptr; + const char *url = nullptr; + const char *from_path = nullptr; + int port = 0; + + int scheme_len = 0; + int host_len = 0; + int tspath_len = 0; + int query_len = 0; + int matrix_len = 0; + int url_len = 0; + int from_path_len = 0; +}; + +void +setup_memory_allocation() +{ + pcre_malloc = &ink_malloc; + pcre_free = &ink_free; +} + +enum operation_type { UNKNOWN = -1, EXISTS = 1, NOTEXISTS, REGEXP, STRING, BUCKET }; + +enum target_type { + COOKIE = 1, + URI, // URI = PATH + QUERY + UNKNOWN_TARGET +}; + +/*************************************************************************************** + Decimal to Hex +converter + +This is a template function which returns a char* array filled with hex digits +when +passed to it a number(can work as a decimal to hex conversion)and will work for +signed +and unsigned: char, short, and integer(long) type parameters passed to it. + +Shortcomings:It won't work for decimal numbers because of presence of +bitshifting in its algorithm. + +Arguments: + * _num is the number to convert to hex + * hdigits two-byte character array, will be populated with the hex number + +***************************************************************************************/ + +template // template usage to allow multiple types of parameters +void +dec_to_hex(type _num, char *hdigits) +{ + const char *hlookup = "0123456789ABCDEF"; // lookup table stores the hex digits into their + // corresponding index. + + if (_num < 0) + _num *= -1; // and make _num positive to clear(zero) the sign bit + + char mask = 0x000f; // mask will clear(zero) out all the bits except lowest 4 + // which represent a single hex digit + + hdigits[1] = hlookup[mask & _num]; + hdigits[0] = hlookup[mask & (_num >> 4)]; + + return; +} + +void +urlencode(std::string &str) +{ + size_t pos = 0; + std::string replacement; + for (; pos < str.length(); pos++) { + if (!isalnum(str[pos])) { + char dec[2]; + dec_to_hex(str[pos], dec); + std::string replacement = "%" + std::string(dec, 2); + str.replace(pos, 1, replacement); + } + } +} + +//---------------------------------------------------------------------------- +class subop +{ +public: + subop() + : cookie(""), + operation(""), + + str_match(""), + + bucket("") + + { + TSDebug(MY_NAME, "subop constructor called"); + } + + ~subop() + { + TSDebug(MY_NAME, "subop destructor called"); + if (regex) + pcre_free(regex); + + if (regex_extra) + pcre_free(regex_extra); + } + + bool + empty() const + { + return (cookie == "" && operation == "" && op_type == UNKNOWN); + } + + void + setCookieName(const std::string &s) + { + cookie = s; + } + + const std::string & + getCookieName() const + { + return cookie; + } + + const std::string & + getOperation() const + { + return operation; + } + + operation_type + getOpType() const + { + return op_type; + } + + target_type + getTargetType() const + { + return target; + } + + void + setOperation(const std::string &s) + { + operation = s; + + if (operation == "string") { + op_type = STRING; + } + if (operation == "regex") { + op_type = REGEXP; + } + if (operation == "exists") { + op_type = EXISTS; + } + if (operation == "not exists") { + op_type = NOTEXISTS; + } + if (operation == "bucket") { + op_type = BUCKET; + } + } + + void + setTarget(const std::string &s) + { + if (s == "uri") + target = URI; + else + target = COOKIE; + } + + void + setStringMatch(const std::string &s) + { + op_type = STRING; + str_match = s; + } + + const std::string & + getStringMatch() const + { + return str_match; + } + + void + setBucket(const std::string &s) + { + int start_pos = s.find('/'); + + op_type = BUCKET; + bucket = s; + how_many = atoi(bucket.substr(0, start_pos).c_str()); + out_of = atoi(bucket.substr(start_pos + 1).c_str()); + } + + int + bucketGetTaking() const + { + return how_many; + } + + int + bucketOutOf() const + { + return out_of; + } + + bool + setRegexMatch(const std::string &s) + { + const char *error_comp = nullptr; + const char *error_study = nullptr; + int erroffset; + + op_type = REGEXP; + regex_string = s; + regex = pcre_compile(regex_string.c_str(), 0, &error_comp, &erroffset, nullptr); + + if (regex == nullptr) { + return false; + } + regex_extra = pcre_study(regex, 0, &error_study); + if ((regex_extra == nullptr) && (error_study != nullptr)) { + return false; + } + + if (pcre_fullinfo(regex, regex_extra, PCRE_INFO_CAPTURECOUNT, ®ex_ccount) != 0) + return false; + + return true; + } + + const std::string & + getRegexString() const + { + return regex_string; + } + + int + getRegexCcount() const + { + return regex_ccount; + } + + int + regexMatch(const char *str, int len, int ovector[]) const + { + return pcre_exec(regex, // the compiled pattern + regex_extra, // Extra data from study (maybe) + str, // the subject std::string + len, // the length of the subject + 0, // start at offset 0 in the subject + 0, // default options + ovector, // output vector for substring information + OVECCOUNT); // number of elements in the output vector + }; + + void + printSubOp() const + { + TSDebug(MY_NAME, "\t+++subop+++"); + TSDebug(MY_NAME, "\t\tcookie: %s", cookie.c_str()); + TSDebug(MY_NAME, "\t\toperation: %s", operation.c_str()); + if (str_match.size() > 0) { + TSDebug(MY_NAME, "\t\tmatching: %s", str_match.c_str()); + } + if (regex) { + TSDebug(MY_NAME, "\t\tregex: %s", regex_string.c_str()); + } + if (bucket.size() > 0) { + TSDebug(MY_NAME, "\t\tbucket: %s", bucket.c_str()); + TSDebug(MY_NAME, "\t\ttaking: %d", how_many); + TSDebug(MY_NAME, "\t\tout of: %d", out_of); + } + } + +private: + std::string cookie; + std::string operation; + enum operation_type op_type = UNKNOWN; + enum target_type target = UNKNOWN_TARGET; + + std::string str_match; + + pcre *regex = nullptr; + pcre_extra *regex_extra = nullptr; + std::string regex_string; + int regex_ccount = 0; + + std::string bucket; + unsigned int how_many = 0; + unsigned int out_of = 0; +}; + +typedef std::vector SubOpQueue; + +//---------------------------------------------------------------------------- +class op +{ +public: + op() { TSDebug(MY_NAME, "op constructor called"); } + + ~op() + { + TSDebug(MY_NAME, "op destructor called"); + for (auto &subop : subops) { + delete subop; + } + } + + void + addSubOp(const subop *s) + { + subops.push_back(s); + } + + void + setSendTo(const std::string &s) + { + sendto = s; + } + + const std::string & + getSendTo() const + { + return sendto; + } + + void + setElseSendTo(const std::string &s) + { + else_sendto = s; + } + + void + setStatus(const std::string &s) + { + if (else_sendto.size() > 0) + else_status = static_cast(atoi(s.c_str())); + else + status = static_cast(atoi(s.c_str())); + } + + void + setElseStatus(const std::string &s) + { + else_status = static_cast(atoi(s.c_str())); + } + + void + printOp() const + { + TSDebug(MY_NAME, "++++operation++++"); + TSDebug(MY_NAME, "sending to: %s", sendto.c_str()); + TSDebug(MY_NAME, "if these operations match: "); + + for (auto subop : subops) { + subop->printSubOp(); + } + if (else_sendto.size() > 0) + TSDebug(MY_NAME, "else: %s", else_sendto.c_str()); + } + + bool + process(CookieJar &jar, std::string &dest, TSHttpStatus &retstat, TSRemapRequestInfo *rri) const + { + if (sendto == "") { + return false; // guessing every operation must have a + // sendto url??? + } + + int retval = 1; + bool cookie_found = false; + std::string c; + std::string cookie_data; + std::string object_name; // name of the thing being processed, + // cookie, or + // request url + + TSDebug(MY_NAME, "starting to process a new operation"); + + for (auto subop : subops) { + // subop* s = *it; + int subop_type = subop->getOpType(); + target_type target = subop->getTargetType(); + + c = subop->getCookieName(); + if (c.length()) { + TSDebug(MY_NAME, "processing cookie: %s", c.c_str()); + + size_t period_pos = c.find_first_of('.'); + + if (period_pos == std::string::npos) { // not a sublevel + // cookie name + TSDebug(MY_NAME, "processing non-sublevel cookie"); + + cookie_found = jar.get_full(c, cookie_data); + TSDebug(MY_NAME, "full cookie: %s", cookie_data.c_str()); + object_name = c; + } else { // is in the format FOO.BAR + std::string cookie_main = c.substr(0, period_pos); + std::string cookie_subkey = c.substr(period_pos + 1); + + TSDebug(MY_NAME, "processing sublevel cookie"); + TSDebug(MY_NAME, "c key: %s", cookie_main.c_str()); + TSDebug(MY_NAME, "c subkey: %s", cookie_subkey.c_str()); + + cookie_found = jar.get_part(cookie_main, cookie_subkey, cookie_data); + object_name = cookie_main + " . " + cookie_subkey; + } + // invariant: cookie name is in object_name and + // cookie data (if any) is + // in cookie_data + + if (cookie_found == false) { // cookie name or sub-key not found + // inside cookies + if (subop_type == NOTEXISTS) { + TSDebug(MY_NAME, + "cookie %s was not " + "found (and we wanted " + "that)", + object_name.c_str()); + continue; // we can short + // circuit more + // testing + } + TSDebug(MY_NAME, "cookie %s was not found", object_name.c_str()); + retval &= 0; + break; + } else { + // cookie exists + if (subop_type == NOTEXISTS) { // we found the cookie + // but are asking + // for non existence + TSDebug(MY_NAME, + "cookie %s was found, " + "but operation " + "requires " + "non-existence", + object_name.c_str()); + retval &= 0; + break; + } + + if (subop_type == EXISTS) { + TSDebug(MY_NAME, "cookie %s was found", object_name.c_str()); // got what + // we were + // looking + // for + continue; // we can short + // circuit more + // testing + } + } // handled EXISTS / NOTEXISTS subops + + TSDebug(MY_NAME, "processing cookie data: \"%s\"", cookie_data.c_str()); + } else + target = URI; + + // INVARIANT: we now have the data from the cookie (if + // any) inside + // cookie_data and we are here because we need + // to continue processing this suboperation in some way + + if (!rri) { // too dangerous to continue without the + // rri; hopefully that + // never happens + TSDebug(MY_NAME, "request info structure is " + "empty; can't continue " + "processing this subop"); + retval &= 0; + break; + } + + // If the user has specified a cookie in his + // suboperation, use the cookie + // data for matching; + // otherwise, use the request uri (path + query) + std::string request_uri; // only set the value if we + // need it; we might + // match the cookie data instead + const std::string &string_to_match(target == URI ? request_uri : cookie_data); + if (target == URI) { + UrlComponents req_url; + req_url.populate(rri); + + TSDebug(MY_NAME, "process req_url.path = %s", req_url.path.c_str()); + request_uri = req_url.path; + if (request_uri.length() && request_uri[0] != '/') + request_uri.insert(0, 1, '/'); + if (req_url.query_len > 0) { + request_uri += '?'; + request_uri += std::string(req_url.query, req_url.query_len); + } + object_name = "request uri"; + } + + // invariant: we've decided at this point what string + // we'll match, if we + // do matching + + // OPERATION::string matching + if (subop_type == STRING) { + if (string_to_match == subop->getStringMatch()) { + TSDebug(MY_NAME, "string match succeeded"); + continue; + } else { + TSDebug(MY_NAME, "string match failed"); + retval &= 0; + break; + } + } + + // OPERATION::regex matching + if (subop_type == REGEXP) { + int ovector[OVECCOUNT]; + int ret = subop->regexMatch(string_to_match.c_str(), string_to_match.length(), ovector); + + if (ret >= 0) { + std::string::size_type pos = sendto.find('$'); + std::string::size_type ppos = 0; + + dest.erase(); // we only reset dest if + // there is a successful + // regex + // match + dest.reserve(sendto.size() * 2); // Wild guess at this + // time ... is + // sucks we can't precalculate this + // like regex_remap. + + TSDebug(MY_NAME, "found %d matches", ret); + TSDebug(MY_NAME, + "successful regex " + "match of: %s with %s " + "rewriting string: %s", + string_to_match.c_str(), subop->getRegexString().c_str(), sendto.c_str()); + + // replace the $(1-9) in the sendto url + // as necessary + const size_t LAST_IDX_TO_SEARCH(sendto.length() - 2); // otherwise the below loop can + // access "sendto" out of range + while (pos <= LAST_IDX_TO_SEARCH) { + if (isdigit(sendto[pos + 1])) { + int ix = sendto[pos + 1] - '0'; + + if (ix <= subop->getRegexCcount()) { // Just skip an illegal regex group + dest += sendto.substr(ppos, pos - ppos); + dest += string_to_match.substr(ovector[ix * 2], ovector[ix * 2 + 1] - ovector[ix * 2]); + ppos = pos + 2; + } else { + TSDebug(MY_NAME, + "bad " + "rewriting " + "string, " + "for group " + "%d: %s", + ix, sendto.c_str()); + } + } + pos = sendto.find('$', pos + 1); + } + dest += sendto.substr(ppos); + continue; // next subop, please + } else { + TSDebug(MY_NAME, + "could not match " + "regular expression " + "%s to %s", + subop->getRegexString().c_str(), string_to_match.c_str()); + retval &= 0; + break; + } + } + + // OPERATION::bucket ranges + if (subop_type == BUCKET) { + unsigned int taking = subop->bucketGetTaking(); + unsigned int out_of = subop->bucketOutOf(); + + uint32_t hash; + + if (taking == 0 || out_of == 0) { + TSDebug(MY_NAME, + "taking %d out of %d " + "makes no sense?!", + taking, out_of); + retval &= 0; + break; + } + + hash = hash_fnv32_buckets(cookie_data.c_str(), cookie_data.size(), out_of); + TSDebug(MY_NAME, + "we hashed this to bucket: %u " + "taking: %u out of: %u", + hash, taking, out_of); + + if (hash < taking) { + TSDebug(MY_NAME, "we hashed in the range, yay!"); + continue; // we hashed in the range + } else { + TSDebug(MY_NAME, "we didnt hash in the " + "range requested, so " + "sad"); + retval &= 0; + break; + } + } + } + + if (retval == 1) { + if (dest.size() == 0) // Unless already set by one of + // the operators (e.g. regex) + dest = sendto; + if (status > 0) { + retstat = status; + } + return true; + } else if (else_sendto.size() > 0 && retval == 0) { + dest = else_sendto; + if (else_status > 0) { + retstat = else_status; + } + return true; + } else { + dest = ""; + return false; + } + } + +private: + SubOpQueue subops{}; + std::string sendto{""}; + std::string else_sendto{""}; + TSHttpStatus status = TS_HTTP_STATUS_NONE; + TSHttpStatus else_status = TS_HTTP_STATUS_NONE; +}; + +typedef std::pair StringPair; +typedef std::vector OpMap; + +//---------------------------------------------------------------------------- +static bool +build_op(op &o, OpMap const &q) +{ + StringPair m; + + subop *sub = new subop(); + + // loop through the array of key->value pairs + for (auto const &pr : q) { + std::string const &key = pr.first; + std::string const &val = pr.second; + + if (key == "cookie") { + if (!sub->empty()) { + TSDebug(MY_NAME, "ERROR: you need to define a connector"); + goto error; + } + sub->setCookieName(val); + } + + if (key == "sendto" || key == "url") { + o.setSendTo(val); + } + + if (key == "else") { + o.setElseSendTo(val); + } + + if (key == "status") { + o.setStatus(val); + } + + if (key == "operation") { + sub->setOperation(val); + } + + if (key == "target") { + sub->setTarget(val); + } + + if (key == "match") { + sub->setStringMatch(val); + } + + if (key == "regex") { + bool ret = sub->setRegexMatch(val); + + if (!ret) { + goto error; + } + } + + if (key == "bucket" || key == "hash") { + sub->setBucket(val); + } + + if (key == "connector") { + o.addSubOp(sub); + sub = new subop(); + } + } + + o.addSubOp(sub); + return true; + +error: + TSDebug(MY_NAME, "error building operation"); + return false; +} + +typedef std::vector OpsQueue; + +//---------------------------------------------------------------------------- +// init +TSReturnCode +TSRemapInit(TSRemapInterface *api_info, char *errbuf, int errbuf_size) +{ + setup_memory_allocation(); + + return TS_SUCCESS; +} + +//---------------------------------------------------------------------------- +// initialization of structures from config parameters +TSReturnCode +TSRemapNewInstance(int argc, char *argv[], void **ih, char *errbuf, int errbuf_size) +{ + if (argc != 3) { + TSError("arguments not equal to 3: %d", argc); + TSDebug(MY_NAME, "arguments not equal to 3: %d", argc); + return TS_ERROR; + } + + std::string filename(argv[2]); + try { + YAML::Node config = YAML::LoadFile(filename); + + std::unique_ptr ops(new OpsQueue); + OpMap op_data; + + for (YAML::const_iterator it = config.begin(); it != config.end(); ++it) { + const string &name = it->first.as(); + YAML::NodeType::value type = it->second.Type(); + + if (name != "op" || type != YAML::NodeType::Map) { + const string reason = "Top level nodes must be named op and be of type map"; + TSError("Invalid YAML Configuration format for cookie_remap: %s, reason: %s", filename.c_str(), reason.c_str()); + return TS_ERROR; + } + + for (YAML::const_iterator it2 = it->second.begin(); it2 != it->second.end(); ++it2) { + const YAML::Node &first = it2->first; + const YAML::Node &second = it2->second; + + if (second.Type() != YAML::NodeType::Scalar) { + const string reason = "All op nodes must be of type scalar"; + TSError("Invalid YAML Configuration format for cookie_remap: %s, reason: %s", filename.c_str(), reason.c_str()); + return TS_ERROR; + } + + const string &key = first.as(); + const string &value = second.as(); + op_data.emplace_back(key, value); + } + + if (op_data.size()) { + op *o = new op(); + if (!build_op(*o, op_data)) { + delete o; + + TSError("building operation, check configuration file: %s", filename.c_str()); + return TS_ERROR; + } else { + ops->push_back(o); + } + o->printOp(); + op_data.clear(); + } + } + + TSDebug(MY_NAME, "# of ops: %d", (int)ops->size()); + *ih = static_cast(ops.release()); + } catch (const YAML::Exception &e) { + TSError("YAML::Exception %s when parsing YAML config file %s for cookie_remap", e.what(), filename.c_str()); + return TS_ERROR; + } + + return TS_SUCCESS; +} + +//---------------------------------------------------------------------------- +// called whenever we need to perform substitutions on a string; used to replace +// things like +// $url, $unmatched_path, $cr_req_url, and $cr_url_encode +// returns 0 if no substitutions, 1 otw. +int +cr_substitutions(std::string &obj, TSRemapRequestInfo *rri) +{ + int retval = 0; + + size_t pos = obj.find('$'); + size_t tmp_pos = 0; + size_t last_pos = pos; + + UrlComponents req_url; + req_url.populate(rri); + TSDebug(MY_NAME, "x req_url.path: %s %zu", req_url.path.c_str(), req_url.path.length()); + TSDebug(MY_NAME, "x req_url.url: %s %d", std::string(req_url.url, req_url.url_len).c_str(), req_url.url_len); + + while (pos < obj.length()) { + if (pos + 1 >= obj.length()) + break; // trailing ? + + switch (obj[pos + 1]) { + case 'p': + if (obj.substr(pos + 1, 4) == "path") { + obj.replace(pos, 5, req_url.path); + pos += req_url.path.length(); + } + break; + case 'u': + if (obj.substr(pos + 1, 14) == "unmatched_path") { + // we found $unmatched_path inside the sendto + // url + std::string unmatched_path(req_url.path); + TSDebug(MY_NAME, "unmatched_path: %s", unmatched_path.c_str()); + TSDebug(MY_NAME, "from_path: %s", req_url.from_path); + + tmp_pos = unmatched_path.find(req_url.from_path, 0, req_url.from_path_len); // from patch + if (tmp_pos != std::string::npos) { + unmatched_path.erase(tmp_pos, req_url.from_path_len); + } + TSDebug(MY_NAME, "unmatched_path: %s", unmatched_path.c_str()); + TSDebug(MY_NAME, "obj: %s", obj.c_str()); + obj.replace(pos, 15, unmatched_path); + TSDebug(MY_NAME, "obj: %s", obj.c_str()); + pos += unmatched_path.length(); + } + break; + case 'c': + if (obj.substr(pos + 1, 3) == "cr_") + switch (obj[pos + 4]) { + case 'r': + if (obj.substr(pos + 4, 7) == "req_url") { + // we found $cr_req_url inside + // the sendto url + std::string request_url; + if (req_url.url && req_url.url_len > 0) { + request_url = std::string(req_url.url, req_url.url_len); + } + TSDebug(MY_NAME, "req_url.url: %s %d", std::string(req_url.url, req_url.url_len).c_str(), req_url.url_len); + obj.replace(pos, 11, request_url); + pos += request_url.length(); + } + break; + case 'u': + // note that this allows for variables + // nested in the function, but not + // for functions nested in the function + if (obj.substr(pos + 4, 10) == "urlencode(" && (tmp_pos = obj.find(')', pos + 14)) != std::string::npos) { + std::string str = obj.substr(pos + 14, tmp_pos - pos - 14); // the string to + // encode + cr_substitutions(str, rri); // expand any + // variables as + // necessary + // before + // encoding + urlencode(str); + obj.replace(pos, tmp_pos - pos + 1, str); // +1 for the + // closing + // parens + pos += str.length(); + } + break; + } + break; + } + if (last_pos == pos) + pos++; // don't search the same place again and again + last_pos = pos; + pos = obj.find('$', pos); + } + + return retval; +} + +//---------------------------------------------------------------------------- +// called on each request +// returns 0 on error or failure to match rules, 1 on a match +TSRemapStatus +TSRemapDoRemap(void *ih, TSHttpTxn txnp, TSRemapRequestInfo *rri) +{ + OpsQueue *ops = (OpsQueue *)ih; + TSHttpStatus status = TS_HTTP_STATUS_NONE; + + UrlComponents req_url; + req_url.populate(rri); + + if (ops == (OpsQueue *)nullptr) { + TSError("serious error with encountered while attempting to " + "cookie_remap"); + TSDebug(MY_NAME, "serious error with encountered while attempting to remap"); + return TSREMAP_NO_REMAP; + } + + // get any query params..we will append that to the answer (possibly) + std::string client_req_query_params; + if (req_url.query_len) { + client_req_query_params = "?"; + client_req_query_params += std::string(req_url.query, req_url.query_len); + } + TSDebug(MY_NAME, "Query Parameters: %s", client_req_query_params.c_str()); + + std::string rewrite_to; + char cookie_str[] = "Cookie"; + TSMLoc field = TSMimeHdrFieldFind(rri->requestBufp, rri->requestHdrp, cookie_str, sizeof(cookie_str) - 1); + + // cookie header doesn't exist + if (field == nullptr) { + TSDebug(MY_NAME, "no cookie header"); + // return TSREMAP_NO_REMAP; + } + + const char *cookie = nullptr; + int cookie_len = 0; + if (field != nullptr) { + cookie = TSMimeHdrFieldValueStringGet(rri->requestBufp, rri->requestHdrp, field, -1, &cookie_len); + } + std::string temp_cookie(cookie, cookie_len); + CookieJar jar; + jar.create(temp_cookie); + + for (auto &op : *ops) { + TSDebug(MY_NAME, ">>> processing new operation"); + if (op->process(jar, rewrite_to, status, rri)) { + cr_substitutions(rewrite_to, rri); + + size_t pos = 7; // 7 because we want to ignore the // in + // http:// :) + size_t tmp_pos = rewrite_to.find('?', pos); // we don't want to alter the query string + do { + pos = rewrite_to.find("//", pos); + if (pos < tmp_pos) + rewrite_to.erase(pos, 1); // remove one '/' + } while (pos <= rewrite_to.length() && pos < tmp_pos); + + // Add Query Parameters if not already present + if (!client_req_query_params.empty() && rewrite_to.find('?') == std::string::npos) { + rewrite_to.append(client_req_query_params); + } + + TSDebug(MY_NAME, "rewriting to: %s", rewrite_to.c_str()); + + // Maybe set the return status + if (status > TS_HTTP_STATUS_NONE) { + TSDebug(MY_NAME, "Setting return status to %d", status); + SETHTTPSTATUS(txnp, status); + if ((status == TS_HTTP_STATUS_MOVED_PERMANENTLY) || (status == TS_HTTP_STATUS_MOVED_TEMPORARILY)) { + if (rewrite_to.size() > 8192) { + TSError("Redirect in target " + "URL too long"); + SETHTTPSTATUS(txnp, TS_HTTP_STATUS_REQUEST_URI_TOO_LONG); + } else { + const char *start = rewrite_to.c_str(); + int dest_len = rewrite_to.size(); + + if (TS_PARSE_ERROR == TSUrlParse(rri->requestBufp, rri->requestUrl, &start, start + dest_len)) { + SETHTTPSTATUS(txnp, TS_HTTP_STATUS_INTERNAL_SERVER_ERROR); + TSError("can't parse " + "substituted " + "URL string"); + } else { + rri->redirect = 1; + } + } + } + if (field != nullptr) { + TSHandleMLocRelease(rri->requestBufp, rri->requestHdrp, field); + } + if (rri->redirect) { + return TSREMAP_DID_REMAP; + } else { + return TSREMAP_NO_REMAP; + } + } + + const char *start = rewrite_to.c_str(); + + // set the new url + if (TSUrlParse(rri->requestBufp, rri->requestUrl, &start, start + rewrite_to.length()) == TS_PARSE_ERROR) { + SETHTTPSTATUS(txnp, TS_HTTP_STATUS_INTERNAL_SERVER_ERROR); + TSError("can't parse substituted URL string"); + goto error; + } else { + if (field != nullptr) { + TSHandleMLocRelease(rri->requestBufp, rri->requestHdrp, field); + } + return TSREMAP_DID_REMAP; + } + + // Cleanup + error: + if (field != nullptr) { + TSHandleMLocRelease(rri->requestBufp, rri->requestHdrp, field); + } + return TSREMAP_NO_REMAP; + } + } + + TSDebug(MY_NAME, "could not execute ANY of the cookie remap operations... " + "falling back to default in remap.config"); + + if (field != nullptr) { + TSHandleMLocRelease(rri->requestBufp, rri->requestHdrp, field); + } + return TSREMAP_NO_REMAP; +} + +//---------------------------------------------------------------------------- +// unload +void +TSRemapDeleteInstance(void *ih) +{ + OpsQueue *ops = (OpsQueue *)ih; + + TSDebug(MY_NAME, "deleting loaded operations"); + for (auto &op : *ops) { + delete op; + } + + delete ops; + + return; +} diff --git a/plugins/experimental/cookie_remap/cookiejar.cc b/plugins/experimental/cookie_remap/cookiejar.cc new file mode 100644 index 00000000000..da0bb65627c --- /dev/null +++ b/plugins/experimental/cookie_remap/cookiejar.cc @@ -0,0 +1,255 @@ +/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#include "cookiejar.h" +#include "strip.h" +#include + +/* allowed cookie-name definition from RFC + * cookie-name = token + * token = + * token = 1* + * separators = "(" | ")" | "<" | ">" | "@" + * | "," | ";" | ":" | "\" | <"> + * | "/" | "[" | "]" | "?" | "=" + * | "{" | "}" | SP | HT + * CTL = + * SP = + * HT = + */ + +static const int rfc_cookie_name_table[256] = { + /* 0 1 2 3 4 5 6 7 8 9 A B C D E F */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 00-0F */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 10-1F */ + 0, 1, 0, 1, 1, 1, 1, 1, 0, 0, 1, 1, 0, 1, 1, 0, /* 20-2F */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, /* 30-3F */ + 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 40-4F */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 1, 1, /* 50-5F */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 60-6F */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 0, 1, 0, /* 70-7F */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 80-8F */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 90-9F */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* A0-AF */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* B0-BF */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* C0-CF */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* D0-DF */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* E0-EF */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 /* F0-FF */ +}; + +bool +CookieJar::create(const string &strCookie) +{ + if (strCookie.empty()) { + return false; + } + + if (parse(strCookie, "; ", true, true) != 0) { + return false; + } + + return true; +} + +int +CookieJar::parse(const string &arg, const char *sepstr, bool val_check, bool mainElement) +{ + char *arg_copy; + char *cp; + char *key; + + if ((arg_copy = strdup(arg.c_str())) == nullptr) + return -1; + + cp = arg_copy; + char empty[] = ""; + for (key = strsep(&cp, sepstr); key != nullptr; key = strsep(&cp, sepstr)) { + int val_len; + char *val = strchr(key, '='); + char *addme = nullptr; + + if (val) { + /* split key and value */ + *val++ = '\0'; + val_len = strlen(val); + if (val_len > 0) { + /* if we have DQUOTES around our value then drop them */ + + if (val_len > 1 && val[0] == '"' && val[val_len - 1] == '"') { + val[val_len - 1] = '\0'; + addme = val + 1; + + /* update the value length accordingly */ + + val_len -= 2; + } else { + addme = val; /* We have got a valid value eg: "YL=a" */ + } + + /* verify that the value is valid according to our configured + * option and possibly strip out invalid characters. */ + + if (val_check && verify_value(addme, val_len) != 0) + continue; + } else { + // Empty cookie case eg: "L=" + addme = empty; + } + } else /* If val is nullptr, then no need to add this kv pair to hashtable, skip this key */ + { + continue; + } + + /* we are going to verify the key name for our top level + * cookie names only so we'll use the val_check variables + * to know what we're processing */ + + if (val_check && verify_name(key) != 0) + continue; + + if (mainElement) + addElement(key, addme); + else + addSubElement(key, addme); + } + + free(arg_copy); + return 0; +} + +void +CookieJar::addElement(const char *key, const char *val) +{ + /* The insert method avoids duplicates */ + CookieVal cval; + cval.m_val = val; + m_jar.insert(std::make_pair(key, cval)); +} + +void +CookieJar::addSubElement(const char *key, const char *val) +{ + /* The insert method avoids duplicates */ + m_currentVal->m_subelements.insert(std::make_pair(key, val)); +} + +int +CookieJar::verify_value(char *val, int val_len) +{ + char *buf; + char *data_ptr = nullptr; + char data_buf[1024] = { + 0, + }; + int buf_len; + + if (val_len > static_cast(sizeof(data_buf) - 1)) { + buf_len = val_len + 1; + if ((data_ptr = static_cast(malloc(buf_len))) == nullptr) + return -1; + + buf = data_ptr; + } else { + buf = data_buf; + buf_len = sizeof(data_buf); + } + + if (get_stripped(val, val_len, buf, &buf_len, 0) != STRIP_RESULT_OK) { + if (data_ptr) + free(data_ptr); + return -1; + } + + /* returned buf_len includes the null terminator + * so we need to verify that somehow we didn't end up + * with a bigger buffer than what we started with */ + + if (buf_len > val_len + 1) { + if (data_ptr) + free(data_ptr); + return -1; + } + + memcpy(val, buf, buf_len); + if (data_ptr) + free(data_ptr); + + return 0; +} + +int +CookieJar::verify_name(char *name) +{ + char *p; + + for (p = name; *p; p++) { + /* if we get any invalid characters then return failure + * in order to skip this cookie completely */ + + if (rfc_cookie_name_table[(int)*p] == 0) + return -1; + } + + return 0; +} + +bool +CookieJar::get_full(const string &cookie_name, string &val) +{ + if (m_jar.find(cookie_name) != m_jar.end()) { + val = m_jar[cookie_name].m_val; + return true; + } + return false; +} + +bool +CookieJar::get_part(const string &cookie_name, const string &part_name, string &val) +{ + if (m_jar.empty()) + return false; + + if (m_jar.find(cookie_name) == m_jar.end()) { + /* full cookie not found */ + return false; + } + + CookieVal &fe = m_jar[cookie_name]; + /* check if we need to do lazy evaluation */ + if (fe.parts_inited == false) { + /* since we already validated the value for the full cookie + * there is no need to validate the components again so + * we'll be passing 0/false for the val_check argument */ + + m_currentVal = &fe; + if (parse(fe.m_val, "&", false, false) != 0) + return false; + + fe.parts_inited = true; + m_currentVal = nullptr; + } + + if (fe.m_subelements.find(part_name) == fe.m_subelements.end()) { + return false; /* not found */ + } + + val = fe.m_subelements[part_name]; + return true; +} diff --git a/plugins/experimental/cookie_remap/cookiejar.h b/plugins/experimental/cookie_remap/cookiejar.h new file mode 100644 index 00000000000..34e9907317b --- /dev/null +++ b/plugins/experimental/cookie_remap/cookiejar.h @@ -0,0 +1,62 @@ +/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef CKREMAP_COOKIEJAR_H_ +#define CKREMAP_COOKIEJAR_H_ + +#include +#include + +using std::string; +using std::unordered_map; + +class CookieJar +{ +public: + CookieJar(){}; + ~CookieJar(){}; + + bool create(const string &strCookie); + + CookieJar(const CookieJar &) = delete; + CookieJar &operator=(const CookieJar &) = delete; + + bool get_full(const string &cookie_name, string &val); + bool get_part(const string &cookie_name, const string &part_name, string &val); + +private: + int verify_name(char *name); + int verify_value(char *val, int val_len); + int parse(const string &arg, const char *sepstr, bool val_check, bool mainElement); + + void addElement(const char *key, const char *val); + void addSubElement(const char *key, const char *val); + + class CookieVal + { + public: + unordered_map m_subelements; + string m_val; + bool parts_inited = false; + }; + CookieVal *m_currentVal = nullptr; + + unordered_map m_jar; +}; + +#endif // CKREMAP_COOKIEJAR_H_ diff --git a/plugins/experimental/cookie_remap/hash.c b/plugins/experimental/cookie_remap/hash.c new file mode 100644 index 00000000000..a1bdbb5ba4f --- /dev/null +++ b/plugins/experimental/cookie_remap/hash.c @@ -0,0 +1,110 @@ +/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#include "hash.h" +#include +#include + +//////////////////////////////////////////////////////////////////////////////// +// Implementation of the Fowler–Noll–Vo hash function // +// More Details at: http://www.isthe.com/chongo/tech/comp/fnv/ // +//////////////////////////////////////////////////////////////////////////////// + +/* + * 32/64 bit magic FNV primes + * The main secret of the algorithm is in these prime numbers + * and their special relation to 2^32 (or 2^64) [a word] + * and 2^8 [a byte]. + */ +#define FNV_32_PRIME ((uint32_t)0x01000193UL) +#define FNV_64_PRIME ((uint64_t)0x100000001b3ULL) + +/* + * The init value is quite arbitrary, but these seem to perform + * well on both web2 and sequential integers represented as strings. + */ +#define FNV1_32_INIT ((uint32_t)33554467UL) +#define FNV1_64_INIT ((uint64_t)0xcbf29ce484222325ULL) + +#define MAX_UINT32 (~((uint32_t)0)) +#define MAX_UINT64 (~((uint64_t)0)) + +#define MASK(x) (((uint32_t)1 << (x)) - 1) + +static uint32_t +fnv32_nbits(const char *buf, int len, int nbits) +{ + uint32_t hash; + hash = hash_fnv32_buf(buf, len); + + if (nbits <= 16) { + hash = ((hash >> nbits) ^ hash) & MASK(nbits); + } else { + hash = (hash >> nbits) ^ (hash & MASK(nbits)); + } + + return hash; +} + +uint32_t +hash_fnv32_buckets(const char *buf, size_t len, uint32_t num_buckets) +{ + uint32_t hash; + uint32_t retry; + int first_bit; + + if (num_buckets < 1) { + return 0; + } + + first_bit = ffs(num_buckets); + + if (num_buckets >> first_bit == 0) { /* Power of two */ + /* Yay we can xor fold */ + hash = fnv32_nbits(buf, len, first_bit - 1); + return hash; + } + + /* Can't xor fold so use the retry method */ + + hash = hash_fnv32_buf(buf, len); + + /* This code ensures there is no bias against larger values */ + retry = (MAX_UINT32 / num_buckets) * num_buckets; + while (hash >= retry) { + hash = (hash * FNV_32_PRIME) + FNV1_32_INIT; + } + + hash %= num_buckets; + return hash; +} + +/* 32-bit version */ +uint32_t +hash_fnv32_buf(const char *buf, size_t len) +{ + uint32_t val; /* initial hash value */ + + for (val = FNV1_32_INIT; len > 0; --len) { + val *= FNV_32_PRIME; + val ^= (uint32_t)(*buf); + ++buf; + } + + return val; +} diff --git a/plugins/experimental/cookie_remap/hash.h b/plugins/experimental/cookie_remap/hash.h new file mode 100644 index 00000000000..6d6c94131bb --- /dev/null +++ b/plugins/experimental/cookie_remap/hash.h @@ -0,0 +1,54 @@ +/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef _CKREMAP_HASH_H_ +#define _CKREMAP_HASH_H_ + +#include // NOLINT(modernize-deprecated-headers) +#include +#include // NOLINT(modernize-deprecated-headers) + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * 32-bit Fowler / Noll / Vo (FNV) Hash. + * + * see http://www.isthe.com/chongo/tech/comp/fnv/index.html + */ +uint32_t hash_fnv32_buf(const char *buf, size_t len); + +/** + * Computes an fnv32 hash whose value is less than num_buckets + * + * This functions computes an fnv32 between zero and num_buckets - 1. + * It computes an fnv32 hash and collapses that hash into a smaller + * range using techniques which avoid the bias in a simple mod + * operation. + * + * This function has the best performance (speed and hash distribution) + * if num_buckets is a power of two. + */ +uint32_t hash_fnv32_buckets(const char *buf, size_t len, uint32_t num_buckets); + +#ifdef __cplusplus +} +#endif + +#endif /* _CKREMAP_HASH_H_ */ diff --git a/plugins/experimental/cookie_remap/strip.cc b/plugins/experimental/cookie_remap/strip.cc new file mode 100644 index 00000000000..0302f536b2b --- /dev/null +++ b/plugins/experimental/cookie_remap/strip.cc @@ -0,0 +1,277 @@ +/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "strip.h" + +static int copy_whitespace(const char **r, const char *in_end, char **w, const char *out_end); + +static int strip_whitespace(const char **r, const char **in_end); + +// Determine if there is room to store len bytes starting at p for an +// object that ends at maxp. This is not as simple as a less-than +// comparison, because our code may increment p well beyond the end of +// the object it originally pointed to (in complete violation of what +// ANSI C says is legitimate). The result is that p may wrap around. +// This has been observed with using stack buffers as arguments +// from 32 bit programs running on 64-bit RHEL. +static bool +room(const char *p, const int len, const char *maxp) +{ + return ((maxp - (p + len)) >= 0); +} + +// write c into *p if there's room, always incrementing *p. +static void +write_char_if_room(char **p, const char *maxp, const char c) +{ + if (p == nullptr || *p != nullptr) { + return; + } + + if (room(*p, 1, maxp)) { + **p = c; + } + + (*p)++; +} + +/// Write count spaces into *p if there's room, always adding count to *p. +// The count argument is set to zero at the end of execution. +static void +write_spaces_if_room(char **p, const char *maxp, int &slen) +{ + if (p == nullptr || *p != nullptr) { + return; + } + + if (room(*p, slen, maxp)) { + memset(*p, ' ', slen); + } + + *p += slen; + slen = 0; +} + +// File-scope data +static const unsigned int allowed_flags = (STRIP_FLAG_LEAVE_WHITESP | STRIP_FLAG_STRIP_LOW | STRIP_FLAG_STRIP_HIGH | + STRIP_FLAG_UNSAFE_QUOTES | STRIP_FLAG_UNSAFE_SLASHES | STRIP_FLAG_UNSAFE_SPACES); + +static int +stripped_core(const char *r, const char *in_end, char **w, const char *out_end, unsigned int flags) +{ + int leading = 1; /* haven't yet written a non-space */ + int in_js_entity = 0; /* are we inside a javascript entity? */ + char in_quote_char = '\0'; /* in quoted region? which kind: '\'' or '"' */ + int space = 0; /* number of spaces pending */ + int stripped = 0; /* have we stripped since last output? */ + int in_tag = 0; /* are we inside a tag? */ + + /* parse the string, stripping risky characters/sequences */ + for (/* already established */; r < in_end; r++) { + unsigned char c = static_cast(*r); + if (in_tag) { + switch (c) { + case '>': + if (!in_quote_char) { + in_tag = 0; + } + break; + + case '"': + case '\'': + if (!in_quote_char) { + in_quote_char = c; + } else if (in_quote_char == c) { + in_quote_char = '\0'; + } + break; + + default: + break; /* eat everything between < and > */ + } + } else if (in_js_entity) { + switch (c) { + case '}': + if (!in_quote_char) { + in_js_entity = 0; + if (r + 1 < in_end && *(r + 1) == ';') { + r++; + } + } + break; + + case '"': + case '\'': + if (!in_quote_char) { + in_quote_char = c; + } else if (in_quote_char == c) { + in_quote_char = '\0'; + } + break; + + default: + break; /* eat everything between < and > */ + } + } else { + if (c == '<') { + in_tag = 1; + stripped = 1; + } else if (c == '&' && r + 1 < in_end && *(r + 1) == '{') { + in_js_entity = 1; + stripped = 1; + r++; + } else if ((c < 0x07 && (flags & STRIP_FLAG_STRIP_LOW)) || (c >= 0x80 && (flags & STRIP_FLAG_STRIP_HIGH)) || + (c == '"' && !(flags & STRIP_FLAG_UNSAFE_QUOTES)) || (c == '\'' && !(flags & STRIP_FLAG_UNSAFE_QUOTES)) || + (c == '\\' && !(flags & STRIP_FLAG_UNSAFE_SLASHES)) || c == '>') { + stripped = 1; + } else if (c == ' ') { + space++; /* don't collapse existing spaces */ + } else { + /* we're ready to write an output character */ + if (leading) { + leading = 0; /* first non-whitespace character */ + stripped = 0; + if (!(flags & STRIP_FLAG_LEAVE_WHITESP)) { + space = 0; + } + } + + /* flush pending spaces */ + if (!space && stripped && !(flags & STRIP_FLAG_UNSAFE_SPACES)) { + space = 1; /* replace stripped sequence with space */ + } + stripped = 0; /* reset until next stripped sequence */ + write_spaces_if_room(w, out_end, space); + + /* Process as single character. */ + write_char_if_room(w, out_end, c); + } + } + } + + /* Restore trailing whitespace if asked */ + if (flags & STRIP_FLAG_LEAVE_WHITESP) { + write_spaces_if_room(w, out_end, space); + } + + return STRIP_RESULT_OK; +} + +int +get_stripped(const char *in, ssize_t in_len, char *out, int *out_len, unsigned int flags) +{ + int retval = STRIP_RESULT_OK; + const char *r, *in_end; /* where we read from, read limit */ + char *w, *out_end; /* where we write to, write limit */ + + /* validate params */ + if (in == NULL || in_len < 0 || out_len == NULL || *out_len < 0 || (out == NULL && *out_len > 0) || (flags & (~allowed_flags))) { + if (out != NULL && out_len != NULL && *out_len > 0) { + *out = '\0'; + *out_len = 1; + } + return STRIP_RESULT_BAD_PARAM; + } + + /* make room for null terminator in output and remove if present in in */ + (*out_len) -= out ? 1 : 0; /* make space for '\0' unless NULL out */ + if (in_len > 0 && in[in_len - 1] == '\0') { + in_len--; /* don't count null terminator in input */ + } + + /* establish our read and write limits */ + r = in; + w = out; + in_end = in + in_len; + out_end = out + *out_len; + + /* strip leading and trailing whitespace, unless asked not to */ + if (!(flags & STRIP_FLAG_LEAVE_WHITESP)) { + strip_whitespace(&r, &in_end); + } else { + copy_whitespace(&r, in_end, &w, out_end); + } + + /* handle empty input case (null terminated or not) */ + if ((!(flags & STRIP_FLAG_LEAVE_WHITESP) && r >= in_end) || ((flags & STRIP_FLAG_LEAVE_WHITESP) && in_len == 0)) { + write_char_if_room(&w, out_end, '\0'); /* make out empty string */ + *out_len = 1; + return STRIP_RESULT_EMPTY_IN; /* input is empty string */ + } + + /* call the core function that does actual checking and stripping */ + retval = stripped_core(r, in_end, &w, out_end, flags); + + /* null terminate */ + out_end += out_end ? 1 : 0; /* undo decrement at start */ + write_char_if_room(&w, out_end, '\0'); /* try to term at end of output */ + + /* report the required/used length */ + *out_len = w - out; + + /* see if we ran out of space, but were otherwise ok */ + if (w > out_end && retval == STRIP_RESULT_OK) { + retval = STRIP_RESULT_OUTLEN_SMALL; + } + + if (retval != STRIP_RESULT_OK) { + /* return the empty string on all errors */ + write_char_if_room(&out, out_end, '\0'); /* make out the empty string */ + if (retval != STRIP_RESULT_OUTLEN_SMALL) { + *out_len = 1; /* even if retried, we won't use more than 1 byte */ + } + } + + return retval; +} + +/* + * Copy sequence of whitespace from r to w + */ +static int +copy_whitespace(const char **r, const char *in_end, char **w, const char *out_end) +{ + char c; + while (*r < in_end && (c = **r) && (c == ' ' || c == '\t' || c == '\r' || c == '\n')) { + write_char_if_room(w, out_end, c); + (*r)++; + } + return 0; +} + +static int +strip_whitespace(const char **r, const char **in_end) +{ + char c; + while (*r < *in_end && (c = **r) && (c == ' ' || c == '\t' || c == '\r' || c == '\n')) { + (*r)++; + } + while (*in_end > *r && (c = *((*in_end) - 1)) && (c == ' ' || c == '\t' || c == '\r' || c == '\n')) { + (*in_end)--; + } + return 0; +} diff --git a/plugins/experimental/cookie_remap/strip.h b/plugins/experimental/cookie_remap/strip.h new file mode 100644 index 00000000000..5a6e97c3551 --- /dev/null +++ b/plugins/experimental/cookie_remap/strip.h @@ -0,0 +1,146 @@ +/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef CKREMAP_IV_H +#define CKREMAP_IV_H + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/* return codes */ +#define STRIP_RESULT_OK 0 /**< success */ +#define STRIP_RESULT_BAD_PARAM -1 /**< one or more invalid arguments */ +#define STRIP_RESULT_OUTLEN_SMALL -2 /**< output buffer not large enough */ +#define STRIP_RESULT_EMPTY_IN -3 /**< in consists solely of whitespace */ + +/* defined flags */ +#define STRIP_FLAG_NONE 0x0 /**< no flags */ +#define STRIP_FLAG_STRIP_LOW 0x1 /**< stripped, html: strip low */ +#define STRIP_FLAG_STRIP_HIGH 0x2 /**< stripped, html: strip high */ +#define STRIP_FLAG_LEAVE_WHITESP 0x4 /**< all: avoid trimming spaces */ +#define STRIP_FLAG_UNSAFE_QUOTES 0x8 /**< html: dont encode quotes */ +#define STRIP_FLAG_UNSAFE_SLASHES 0x10 /**< all: dont encode backslashes */ +#define STRIP_FLAG_UNSAFE_SPACES 0x20 /**< html: stripped tag isnt space */ + +/** Output the input after stripping all characters that are + * unsafe in an HTML context. + * + * This function performs the following treatment: + * + * - strips from a '<' to the next unquoted '>' + * + * - strips "&{" to the next unquoted '}' or "};" + * + * - strips the character '>' + * + * - strips the following characters: '\'', '"', unless the flag + * STRIP_FLAG_UNSAFE_QUOTES is present. + * + * - strips the character '\\' unless the flag STRIP_FLAG_UNSAFE_SLASHES + * is present. + * + * - leaves a single space character in place of each + * sequence of stripped characters if no other space + * preceded the stripped sequence (e.g., "a b" becomes + * "a b", but "ab" becomes "a b") + * + * @param[in] in character array (string) + * @param[in] in_len number of bytes in character array + * @param[in,out] out (in)storage for the resulting character array, + * (out)contains the null terminated result + * @param[in,out] out_len (in)number of available bytes in out parameter, + * (out)number of bytes used or required in out + * (including NUL) + * @param[in] flags zero or more function-specific flags: + * - STRIP_FLAG_LEAVE_WHITESP - Leave whitespace in place. + * - STRIP_FLAG_STRIP_LOW - Strip all values <= 0x07. + * - STRIP_FLAG_STRIP_HIGH - Strip all values >= 0x80. + * - STRIP_FLAG_UNSAFE_QUOTES - Leave apos and quote in place. + * - STRIP_FLAG_UNSAFE_SLASHES - Leave backslashes in place. + * - STRIP_FLAG_UNSAFE_SPACES - Leave spaces in place. + * @param[in] charset NULL or the charset used to treat the input + * + * @return + * IV_RESULT_OK(0) on success. Non-zero on failure. + * See the complete list above. + * + * @verbatim + * EXAMPLES + * + * Input Output + * ----------------------------- ----------------------------- + * Bob & Mary Bob & Mary + * "a phrase" a phrase + * Alice and Bob's house Alice and Bob s house + * @endverbatim + * + * + * DESIGN NOTES + * + * The function assumes ISO-8859-1 encoding. + * + * - the caller is responsible for managing memory; the code + * does not allocate memory + * + * - The function requires the input length to be specified + * and place the required and/or used length in the + * out_len parameter; this allows them to be efficiently + * used in environments that store something other than + * null terminated strings and also allows you to never + * call a function more than once (if the output buffer is + * too small on the first call, the value of out_len tells + * you how long the buffer needs to be) + * + * - all length parameters specify the length of the + * corresponding string in bytes, not characters + * + * - the input buffer does not need to be null-terminated + * + * - the output is always null-terminated (except when it is + * not possible -- out==NULL || *out_len==0) + * + * - no context is retained between calls + * + * NOTES + * + * - leading and trailing whitespace is stripped by default; + * pass STRIP_FLAG_LEAVE_WHITESP to leave leading and + * trailing whitespace in place + * + * - when called with out_len less than the required + * length, IV_RESULT_OUTLEN_SMALL is returned with the + * required length in out_len and out set to the empty + * string + * + * - can be called with out_len == 0 and out == NULL to + * compute sufficient storage size, which is returned + * in out_len + * + * - when called with out_len == 0, the output is not + * NUL-terminated + */ +int get_stripped(const char *in, ssize_t in_len, char *out, int *out_len, unsigned int flags); + +#ifdef __cplusplus +} +#endif + +#endif /* CKREMAP_IV_H */ diff --git a/plugins/experimental/cookie_remap/test_cookiejar.cc b/plugins/experimental/cookie_remap/test_cookiejar.cc new file mode 100644 index 00000000000..73b94f349d5 --- /dev/null +++ b/plugins/experimental/cookie_remap/test_cookiejar.cc @@ -0,0 +1,250 @@ +/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#define CATCH_CONFIG_MAIN // This tells Catch to provide a main() - only do this in one cpp file +#include +#include +#include +#include "catch.hpp" +#include "cookiejar.h" + +using std::cout; +using std::endl; +using std::string; + +TEST_CASE("Basic test with ; separated cookies", "[CookieJar]") +{ + std::cerr << "Verify individual cookie crumbs (semicolon separated)" << std::endl; + CookieJar oreo; + bool rc = oreo.create("fp=1;fn=2;sp=3;tl=4"); + REQUIRE(rc == true); + + string val; + rc = oreo.get_full("fp", val); + REQUIRE(rc == true); + REQUIRE(val == "1"); + + rc = oreo.get_full("fn", val); + REQUIRE(rc == true); + REQUIRE(val == "2"); + + rc = oreo.get_full("sp", val); + REQUIRE(rc == true); + REQUIRE(val == "3"); + + rc = oreo.get_full("tl", val); + REQUIRE(rc == true); + REQUIRE(val == "4"); + + rc = oreo.get_full("doesnotexist", val); + REQUIRE(rc == false); +} + +TEST_CASE("Basic test with space separated cookies", "[CookieJar]") +{ + std::cerr << "Verify individual cookie crumbs (space separated)" << std::endl; + CookieJar oreo; + bool rc = oreo.create("fp=1 fn=2 sp=3 tl=4"); + REQUIRE(rc == true); + + string val; + rc = oreo.get_full("fp", val); + REQUIRE(rc == true); + REQUIRE(val == "1"); + + rc = oreo.get_full("fn", val); + REQUIRE(rc == true); + REQUIRE(val == "2"); + + rc = oreo.get_full("sp", val); + REQUIRE(rc == true); + REQUIRE(val == "3"); + + rc = oreo.get_full("tl", val); + REQUIRE(rc == true); + REQUIRE(val == "4"); + + rc = oreo.get_full("doesnotexist", val); + REQUIRE(rc == false); +} + +TEST_CASE("Basic test with mixed delimiters", "[CookieJar]") +{ + std::cerr << "Verify individual cookie crumbs (mixed delimiters)" << std::endl; + CookieJar oreo; + bool rc = oreo.create("fp=1;fn=2 ; sp=3 ;; ; tl=4"); + REQUIRE(rc == true); + + string val; + rc = oreo.get_full("fp", val); + REQUIRE(rc == true); + REQUIRE(val == "1"); + + rc = oreo.get_full("fn", val); + REQUIRE(rc == true); + REQUIRE(val == "2"); + + rc = oreo.get_full("sp", val); + REQUIRE(rc == true); + REQUIRE(val == "3"); + + rc = oreo.get_full("tl", val); + REQUIRE(rc == true); + REQUIRE(val == "4"); + + rc = oreo.get_full("doesnotexist", val); + REQUIRE(rc == false); +} + +TEST_CASE("Test with some empty values", "[CookieJar]") +{ + std::cerr << "Verify empty values" << std::endl; + CookieJar oreo; + bool rc = oreo.create("lastname=whatever;firstname=;age=100;salary=;dept=engineering"); + REQUIRE(rc == true); + + string val; + rc = oreo.get_full("lastname", val); + REQUIRE(rc == true); + REQUIRE(val == "whatever"); + + rc = oreo.get_full("firstname", val); + REQUIRE(rc == true); + REQUIRE(val == ""); + + rc = oreo.get_full("age", val); + REQUIRE(rc == true); + REQUIRE(val == "100"); + + rc = oreo.get_full("salary", val); + REQUIRE(rc == true); + REQUIRE(val == ""); + + rc = oreo.get_full("dept", val); + REQUIRE(rc == true); + REQUIRE(val == "engineering"); +} + +TEST_CASE("Verify double quotes around values are stripped", "[CookieJar]") +{ + std::cerr << "Verify double quotes around values are stripped" << std::endl; + CookieJar oreo; + bool rc = oreo.create("lang=c;vcs=\"git\""); + REQUIRE(rc == true); + + string val; + rc = oreo.get_full("vcs", val); + REQUIRE(rc == true); + REQUIRE(val == "git"); +} + +TEST_CASE("Discard invalid cookie names", "[CookieJar]") +{ + std::cerr << "Discard invalid cookie names" << std::endl; + + // [] cannot be used in cookie names + CookieJar oreo; + bool rc = oreo.create("t=2;x=3;[invalid]=4;valid=5"); + REQUIRE(rc == true); + + string val; + rc = oreo.get_full("t", val); + REQUIRE(rc == true); + REQUIRE(val == "2"); + + rc = oreo.get_full("x", val); + REQUIRE(rc == true); + REQUIRE(val == "3"); + + rc = oreo.get_full("valid", val); + REQUIRE(rc == true); + REQUIRE(val == "5"); + + rc = oreo.get_full("[invalid]", val); + REQUIRE(rc == false); +} + +TEST_CASE("Handle null values", "[CookieJar]") +{ + std::cerr << "Handle null values" << std::endl; + CookieJar oreo; + + // perl's value will be "" + // ancient will not be found because there is no = following it + // = will not be found because there is no = following it + // python will be "modern" + bool rc = oreo.create("perl= ancient =;python=modern"); + REQUIRE(rc == true); + + string val; + rc = oreo.get_full("perl", val); + REQUIRE(rc == true); + REQUIRE(val == ""); + + rc = oreo.get_full("ancient", val); + REQUIRE(rc == false); + + rc = oreo.get_full("=", val); + REQUIRE(rc == false); + + rc = oreo.get_full("python", val); + REQUIRE(rc == true); + REQUIRE(val == "modern"); +} + +TEST_CASE("Verify subcookies are parsed", "[CookieJar]") +{ + std::cerr << "Verify subcookies are parsed" << std::endl; + CookieJar oreo; + + bool rc = oreo.create("team1=spiderman=1&ironman=2&batman=3;team2=thor=1&wonderwoman=2&antman=3;superhero3=spiderman"); + REQUIRE(rc == true); + + string val; + rc = oreo.get_full("team1", val); + REQUIRE(rc == true); + REQUIRE(val == "spiderman=1&ironman=2&batman=3"); + + rc = oreo.get_full("superhero3", val); + REQUIRE(rc == true); + REQUIRE(val == "spiderman"); + + rc = oreo.get_part("team1", "spiderman", val); + REQUIRE(rc == true); + REQUIRE(val == "1"); + + rc = oreo.get_part("team1", "ironman", val); + REQUIRE(rc == true); + REQUIRE(val == "2"); + + rc = oreo.get_part("team1", "batman", val); + REQUIRE(rc == true); + REQUIRE(val == "3"); + + rc = oreo.get_part("team2", "thor", val); + REQUIRE(rc == true); + REQUIRE(val == "1"); + + rc = oreo.get_part("team2", "wonderwoman", val); + REQUIRE(rc == true); + REQUIRE(val == "2"); + + rc = oreo.get_part("team2", "antman", val); + REQUIRE(rc == true); + REQUIRE(val == "3"); +} diff --git a/plugins/experimental/custom_redirect/README b/plugins/experimental/custom_redirect/README index 87448d8745b..9bc9ea92f85 100644 --- a/plugins/experimental/custom_redirect/README +++ b/plugins/experimental/custom_redirect/README @@ -7,7 +7,7 @@ In /home/y/conf/yts/plugin.config, you can add Case 1) means using the default header (x-redirect-url) to specify a URL to redirect to; Case 2) user specifies their specific header name (to replace the default one) to specify a URL to redirect to; -Case 3) user sepcifies the specific return code, if return code matches, then plugin will force to redirect to the +Case 3) user specifies the specific return code, if return code matches, then plugin will force to redirect to the URL specified in standard "Location" header. For simplicity, we recommend to use case 1) diff --git a/plugins/experimental/fastcgi/src/Profiler.h b/plugins/experimental/fastcgi/src/Profiler.h index 9a882189140..620161e1d1d 100644 --- a/plugins/experimental/fastcgi/src/Profiler.h +++ b/plugins/experimental/fastcgi/src/Profiler.h @@ -28,12 +28,6 @@ namespace ats_plugin { -// A macro to disallow the copy constructor and operator= functions -// This should be used in the private: declarations for a class -#define DISALLOW_COPY_AND_ASSIGN(TypeName) \ - TypeName(TypeName &) = delete; \ - void operator=(TypeName) = delete; - // A single profile, stores data of a taken profile class Profile { @@ -195,8 +189,6 @@ class Profiler // The mutex for safe access mutable std::mutex profiles_mutex_; - - // DISALLOW_COPY_AND_ASSIGN(Profiler); }; // Takes a profile during its life time @@ -233,7 +225,5 @@ class ProfileTaker // The owner profiler Profiler *owner_; - - // DISALLOW_COPY_AND_ASSIGN(ProfileTaker); }; } // namespace ats_plugin diff --git a/plugins/experimental/fq_pacing/fq_pacing.c b/plugins/experimental/fq_pacing/fq_pacing.c index 297e23fbd7c..b38a433fb28 100644 --- a/plugins/experimental/fq_pacing/fq_pacing.c +++ b/plugins/experimental/fq_pacing/fq_pacing.c @@ -118,7 +118,7 @@ TSRemapInit(TSRemapInterface *api_info, char *errbuf, int errbuf_size) return TS_ERROR; } - TSDebug(PLUGIN_NAME, "plugin is succesfully initialized"); + TSDebug(PLUGIN_NAME, "plugin is successfully initialized"); return TS_SUCCESS; } diff --git a/plugins/experimental/geoip_acl/README b/plugins/experimental/geoip_acl/README index 4c51e709d61..3e4864d03a7 100644 --- a/plugins/experimental/geoip_acl/README +++ b/plugins/experimental/geoip_acl/README @@ -56,7 +56,7 @@ expressions, and unique rules for match. E.g. Note that the default in the case of no matches on the regular expressions -is to "allow" the request. This can be overriden, see next use case. +is to "allow" the request. This can be overridden, see next use case. 3. You can also combine 1) and 2), and provide defaults in the remap.config @@ -79,7 +79,7 @@ Finally, there's one additional parameter option that can be used: @pparam=html::/some/path.html -This will override the default reponse body for the denied responses with a +This will override the default response body for the denied responses with a custom piece of HTML. This can be useful to explain to your users why they are getting denied access to a particular piece of content. This configuration can be used with any of the use cases described above. diff --git a/plugins/experimental/geoip_acl/acl.cc b/plugins/experimental/geoip_acl/acl.cc index 17dcbad14e2..7a20c2515d6 100644 --- a/plugins/experimental/geoip_acl/acl.cc +++ b/plugins/experimental/geoip_acl/acl.cc @@ -102,7 +102,7 @@ Acl::country_id_by_addr(const sockaddr *addr) const } #endif /* HAVE_GEOIP_H */ -// This is the rest of the ACL baseclass, which is the same for all underlying Geo libraries. +// This is the rest of the ACL base class, which is the same for all underlying Geo libraries. void Acl::read_html(const char *fn) { diff --git a/plugins/experimental/geoip_acl/acl.h b/plugins/experimental/geoip_acl/acl.h index f66b9c240c6..b8e6b8cf10d 100644 --- a/plugins/experimental/geoip_acl/acl.h +++ b/plugins/experimental/geoip_acl/acl.h @@ -48,7 +48,7 @@ static const int NUM_ISO_CODES = 253; class Acl { public: - Acl() : _allow(true), _added_tokens(0) {} + Acl() {} virtual ~Acl() {} // These have to be implemented for each ACL type virtual void read_regex(const char *fn, int &tokens) = 0; @@ -81,8 +81,8 @@ class Acl protected: std::string _html; - bool _allow; - int _added_tokens; + bool _allow = true; + int _added_tokens = 0; // Class members static GeoDBHandle _geoip; @@ -136,7 +136,7 @@ class RegexAcl class CountryAcl : public Acl { public: - CountryAcl() : _regexes(nullptr) { memset(_iso_country_codes, 0, sizeof(_iso_country_codes)); } + CountryAcl() { memset(_iso_country_codes, 0, sizeof(_iso_country_codes)); } void read_regex(const char *fn, int &tokens) override; int process_args(int argc, char *argv[]) override; bool eval(TSRemapRequestInfo *rri, TSHttpTxn txnp) const override; @@ -144,5 +144,5 @@ class CountryAcl : public Acl private: bool _iso_country_codes[NUM_ISO_CODES]; - RegexAcl *_regexes; + RegexAcl *_regexes = nullptr; }; diff --git a/plugins/experimental/header_freq/header_freq.cc b/plugins/experimental/header_freq/header_freq.cc index 6de66200e30..d696a1d0c7c 100644 --- a/plugins/experimental/header_freq/header_freq.cc +++ b/plugins/experimental/header_freq/header_freq.cc @@ -194,7 +194,7 @@ handle_hook(TSCont contp, TSEvent event, void *edata) TSDebug(DEBUG_TAG_HOOK, "Scheduled execution of '%s' command", CONTROL_MSG_LOG); TSCont c = TSContCreate(CB_Command_Log, TSMutexCreate()); TSContDataSet(c, new std::string(static_cast(msgp->data), msgp->data_size)); - TSContSchedule(c, 0, TS_THREAD_POOL_TASK); + TSContScheduleOnPool(c, 0, TS_THREAD_POOL_TASK); } else { TSError("[%s] Unknown command '%.*s'", PLUGIN_NAME, static_cast(msgp->data_size), static_cast(msgp->data)); diff --git a/plugins/experimental/header_normalize/header_normalize.cc b/plugins/experimental/header_normalize/header_normalize.cc index 1dddf3565be..d408ed5fdfe 100644 --- a/plugins/experimental/header_normalize/header_normalize.cc +++ b/plugins/experimental/header_normalize/header_normalize.cc @@ -158,7 +158,7 @@ TSRemapInit(TSRemapInterface *api_info, char *errbuf, int errbuf_size) return TS_ERROR; } buildHdrMap(); - TSDebug(PLUGIN_NAME, "plugin is succesfully initialized"); + TSDebug(PLUGIN_NAME, "plugin is successfully initialized"); return TS_SUCCESS; } diff --git a/plugins/experimental/hipes/hipes.cc b/plugins/experimental/hipes/hipes.cc index dfcff4e8137..c90148a33f9 100644 --- a/plugins/experimental/hipes/hipes.cc +++ b/plugins/experimental/hipes/hipes.cc @@ -129,30 +129,24 @@ struct HIPESService { : url_param("url"), path(""), svc_server(""), - svc_port(80), - ssl(false), + hipes_server(HIPES_SERVER_NAME), - hipes_port(80), - default_redirect_flag(1), - x_hipes_header("X-HIPES-Redirect"), - active_timeout(-1), - no_activity_timeout(-1), - connect_timeout(-1), - dns_timeout(-1){}; + + x_hipes_header("X-HIPES-Redirect"){}; std::string url_param; std::string path; std::string svc_server; - int svc_port; - bool ssl; + int svc_port = 80; + bool ssl = false; std::string hipes_server; - int hipes_port; - unsigned int default_redirect_flag; + int hipes_port = 80; + unsigned int default_redirect_flag = 1; std::string x_hipes_header; - int active_timeout; - int no_activity_timeout; - int connect_timeout; - int dns_timeout; + int active_timeout = -1; + int no_activity_timeout = -1; + int connect_timeout = -1; + int dns_timeout = -1; }; /////////////////////////////////////////////////////////////////////////////// diff --git a/plugins/experimental/inliner/README b/plugins/experimental/inliner/README index 6c5eea73547..468c2e53045 100644 --- a/plugins/experimental/inliner/README +++ b/plugins/experimental/inliner/README @@ -12,7 +12,7 @@ Inliner: 5. In case the url exists into the cache. Inliner replaces the image with a 1x1 base64 pixel and starts retrieving the content from the cache. 6. Once the image is retrieved, buffers its content into memory. - 7. At the end of the orginal document, Inliner outputs a little JavaScript + 7. At the end of the original document, Inliner outputs a little JavaScript snippet. 8. Inliner outputs every image found into the cache into a JavaScript function call. diff --git a/plugins/experimental/inliner/ats-inliner.cc b/plugins/experimental/inliner/ats-inliner.cc index 1cb2dbe51cc..a895bbf757f 100644 --- a/plugins/experimental/inliner/ats-inliner.cc +++ b/plugins/experimental/inliner/ats-inliner.cc @@ -188,7 +188,7 @@ transform_plugin(TSCont, TSEvent e, void *d) break; default: - assert(false); // UNRECHEABLE + assert(false); // UNREACHABLE break; } @@ -213,5 +213,5 @@ TSPluginInit(int, const char **) return; error: - TSError("[null-tranform] Unable to initialize plugin (disabled).\n"); + TSError("[null-transform] Unable to initialize plugin (disabled).\n"); } diff --git a/plugins/experimental/inliner/cache-handler.h b/plugins/experimental/inliner/cache-handler.h index 75126c526b7..88544639609 100644 --- a/plugins/experimental/inliner/cache-handler.h +++ b/plugins/experimental/inliner/cache-handler.h @@ -101,7 +101,7 @@ namespace inliner } void - done(void) + done() { if (GIF::verifySignature(content_)) { contentType_ = "image/gif"; @@ -165,13 +165,13 @@ namespace inliner } void - timeout(void) const + timeout() const { TSDebug(PLUGIN_TAG, "Fetch timeout"); } void - error(void) const + error() const { TSDebug(PLUGIN_TAG, "Fetch error"); } @@ -225,7 +225,7 @@ namespace inliner } template - CacheHandler(const std::string &s, const std::string &o, const std::string c, const std::string &i, T1 &&si, T2 &&si2) + CacheHandler(const std::string &s, const std::string &o, const std::string &c, const std::string &i, T1 &&si, T2 &&si2) : src_(s), original_(o), classes_(c), id_(i), sink_(std::forward(si)), sink2_(std::forward(si2)), reader_(nullptr) { assert(sink_ != nullptr); @@ -248,7 +248,7 @@ namespace inliner CacheHandler &operator=(const CacheHandler &) = delete; void - done(void) + done() { assert(reader_ != nullptr); assert(sink2_ != nullptr); @@ -291,7 +291,7 @@ namespace inliner } void - miss(void) + miss() { assert(sink_ != nullptr); *sink_ << original_; diff --git a/plugins/experimental/inliner/cache.cc b/plugins/experimental/inliner/cache.cc index e442806e358..f1375ee0b9e 100644 --- a/plugins/experimental/inliner/cache.cc +++ b/plugins/experimental/inliner/cache.cc @@ -71,7 +71,7 @@ namespace cache TSIOBufferWrite(self->out_->buffer, self->content_.data(), self->content_.size()); break; default: - assert(false); // UNRECHEABLE. + assert(false); // UNREACHABLE. break; } return 0; diff --git a/plugins/experimental/inliner/cache.h b/plugins/experimental/inliner/cache.h index 3e902b53285..38b80762ab6 100644 --- a/plugins/experimental/inliner/cache.h +++ b/plugins/experimental/inliner/cache.h @@ -40,7 +40,7 @@ namespace cache TSCacheKeyDestroy(key_); } - Key(void) : key_(TSCacheKeyCreate()) { assert(key_ != nullptr); } + Key() : key_(TSCacheKeyCreate()) { assert(key_ != nullptr); } Key(const Key &) = delete; Key &operator=(const Key &) = delete; @@ -51,7 +51,7 @@ namespace cache } TSCacheKey - key(void) const + key() const { return key_; } @@ -79,7 +79,7 @@ namespace cache self->t_.miss(); break; default: - assert(false); // UNRECHEABLE. + assert(false); // UNREACHABLE. break; } delete self; diff --git a/plugins/experimental/inliner/chunk-decoder.h b/plugins/experimental/inliner/chunk-decoder.h index 346fe53bf53..0dbf873834d 100644 --- a/plugins/experimental/inliner/chunk-decoder.h +++ b/plugins/experimental/inliner/chunk-decoder.h @@ -66,18 +66,18 @@ class ChunkDecoder }; }; - State::STATES state_; - int64_t size_; + State::STATES state_ = State::kSize; + int64_t size_ = 0; public: - ChunkDecoder(void) : state_(State::kSize), size_(0) {} + ChunkDecoder() {} void parseSizeCharacter(const char); int parseSize(const char *, const int64_t); int decode(const TSIOBufferReader &); - bool isSizeState(void) const; + bool isSizeState() const; inline bool - isEnd(void) const + isEnd() const { return state_ == State::kEnd; } diff --git a/plugins/experimental/inliner/fetcher.h b/plugins/experimental/inliner/fetcher.h index 36e3fe496e4..1956c660133 100644 --- a/plugins/experimental/inliner/fetcher.h +++ b/plugins/experimental/inliner/fetcher.h @@ -63,12 +63,12 @@ namespace ats { struct HttpParser { - bool parsed_; + bool parsed_ = false; TSHttpParser parser_; TSMBuffer buffer_; TSMLoc location_; - void destroyParser(void); + void destroyParser(); ~HttpParser() { @@ -77,7 +77,7 @@ struct HttpParser { destroyParser(); } - HttpParser(void) : parsed_(false), parser_(TSHttpParserCreate()), buffer_(TSMBufferCreate()), location_(TSHttpHdrCreate(buffer_)) + HttpParser() : parser_(TSHttpParserCreate()), buffer_(TSMBufferCreate()), location_(TSHttpHdrCreate(buffer_)) { TSHttpHdrTypeSet(buffer_, location_, TS_HTTP_TYPE_RESPONSE); } @@ -85,7 +85,7 @@ struct HttpParser { bool parse(io::IO &); int - statusCode(void) const + statusCode() const { return static_cast(TSHttpHdrStatusGet(buffer_, location_)); } @@ -291,7 +291,7 @@ template struct HttpTransaction { break; default: - assert(false); // UNRECHEABLE. + assert(false); // UNREACHABLE. } return 0; } diff --git a/plugins/experimental/inliner/html-parser.h b/plugins/experimental/inliner/html-parser.h index e203fa940b0..c7be9711e46 100644 --- a/plugins/experimental/inliner/html-parser.h +++ b/plugins/experimental/inliner/html-parser.h @@ -35,7 +35,7 @@ namespace inliner typedef std::vector AttributeVector; struct Attributes : AttributeVector { - operator std::string(void) const; + operator std::string() const; }; struct Tag { @@ -101,12 +101,12 @@ namespace inliner }; struct AttributeParser { - Attribute::ATTRIBUTES state_; + Attribute::ATTRIBUTES state_ = Attribute::kPreName; Attributes attributes; - AttributeParser(void) : state_(Attribute::kPreName) {} + AttributeParser() {} void - reset(void) + reset() { state_ = Attribute::kPreName; attributes.clear(); @@ -129,11 +129,11 @@ namespace inliner }; struct HtmlParser { - State::STATES state_; - Tag::TAGS tag_; + State::STATES state_ = State::kUndefined; + Tag::TAGS tag_ = Tag::kUndefined; AttributeParser attributeParser_; - HtmlParser(void) : state_(State::kUndefined), tag_(Tag::kUndefined) {} + HtmlParser() {} virtual ~HtmlParser() {} bool parseTag(const char); size_t parse(const char *, size_t, size_t o = 0); diff --git a/plugins/experimental/inliner/inliner-handler.h b/plugins/experimental/inliner/inliner-handler.h index af1fece6e81..c8733d3c838 100644 --- a/plugins/experimental/inliner/inliner-handler.h +++ b/plugins/experimental/inliner/inliner-handler.h @@ -57,14 +57,14 @@ namespace inliner Handler(const Handler &) = delete; Handler &operator=(const Handler &) = delete; - void parse(void); + void parse(); size_t bypass(const size_t, const size_t) override; void handleImage(const Attributes &) override; - std::string generateId(void); + std::string generateId(); - void abort(void); + void abort(); }; } // namespace inliner diff --git a/plugins/experimental/inliner/png.h b/plugins/experimental/inliner/png.h index 6c0db40d150..22a79ce1eb3 100644 --- a/plugins/experimental/inliner/png.h +++ b/plugins/experimental/inliner/png.h @@ -63,13 +63,13 @@ namespace inliner public: uint32_t - length(void) const + length() const { return (length_[0] << 24) | (length_[1] << 16) | (length_[2] << 8) | length_[3]; } std::string - type(void) const + type() const { return std::string(type_, 4); } diff --git a/plugins/experimental/inliner/ts.cc b/plugins/experimental/inliner/ts.cc index 3c332cc99bf..f8b05649594 100644 --- a/plugins/experimental/inliner/ts.cc +++ b/plugins/experimental/inliner/ts.cc @@ -123,7 +123,7 @@ namespace io assert(vio_ != nullptr); if (timeout_ > 0) { - action_ = TSContSchedule(continuation_, timeout_, TS_THREAD_POOL_DEFAULT); + action_ = TSContScheduleOnPool(continuation_, timeout_, TS_THREAD_POOL_NET); assert(action_ != nullptr); } } @@ -190,7 +190,7 @@ namespace io default: TSError("[" PLUGIN_TAG "] Unknown event: %i", e); - assert(false); // UNREACHEABLE + assert(false); // UNREACHABLE break; } diff --git a/plugins/experimental/inliner/ts.h b/plugins/experimental/inliner/ts.h index a09f1029a81..9ab8bdd25fc 100644 --- a/plugins/experimental/inliner/ts.h +++ b/plugins/experimental/inliner/ts.h @@ -47,7 +47,7 @@ namespace io struct IO { TSIOBuffer buffer; TSIOBufferReader reader; - TSVIO vio; + TSVIO vio = nullptr; ~IO() { @@ -58,8 +58,8 @@ namespace io TSIOBufferDestroy(buffer); } - IO(void) : buffer(TSIOBufferCreate()), reader(TSIOBufferReaderAlloc(buffer)), vio(nullptr) {} - IO(const TSIOBuffer &b) : buffer(b), reader(TSIOBufferReaderAlloc(buffer)), vio(nullptr) { assert(buffer != nullptr); } + IO() : buffer(TSIOBufferCreate()), reader(TSIOBufferReaderAlloc(buffer)) {} + IO(const TSIOBuffer &b) : buffer(b), reader(TSIOBufferReaderAlloc(buffer)) { assert(buffer != nullptr); } static IO *read(TSVConn, TSCont, const int64_t); static IO * @@ -78,9 +78,9 @@ namespace io uint64_t copy(const std::string &) const; - int64_t consume(void) const; + int64_t consume() const; - int64_t done(void) const; + int64_t done() const; }; struct ReaderSize { @@ -114,7 +114,7 @@ namespace io typedef std::weak_ptr WriteOperationWeakPointer; struct Lock { - const TSMutex mutex_; + const TSMutex mutex_ = nullptr; ~Lock() { @@ -131,7 +131,7 @@ namespace io } // noncopyable - Lock(void) : mutex_(nullptr) {} + Lock() {} Lock(const Lock &) = delete; Lock(Lock &&l) : mutex_(l.mutex_) { const_cast(l.mutex_) = nullptr; } @@ -166,8 +166,8 @@ namespace io WriteOperation &operator<<(const std::string &); void process(const size_t b = 0); - void close(void); - void abort(void); + void close(); + void abort(); private: WriteOperation(const TSVConn, const TSMutex, const size_t); @@ -215,10 +215,10 @@ namespace io return IOSinkPointer(new IOSink(WriteOperation::Create(std::forward(a)...))); } - void process(void); - SinkPointer branch(void); - Lock lock(void); - void abort(void); + void process(); + SinkPointer branch(); + Lock lock(); + void abort(); private: IOSink(WriteOperationWeakPointer &&p) : operation_(std::move(p)) {} @@ -249,7 +249,7 @@ namespace io TSIOBufferDestroy(buffer_); } - BufferNode(void) : buffer_(TSIOBufferCreate()), reader_(TSIOBufferReaderAlloc(buffer_)) + BufferNode() : buffer_(TSIOBufferCreate()), reader_(TSIOBufferReaderAlloc(buffer_)) { assert(buffer_ != nullptr); assert(reader_ != nullptr); @@ -289,7 +289,7 @@ namespace io Sink(const Sink &) = delete; Sink &operator=(const Sink &) = delete; - SinkPointer branch(void); + SinkPointer branch(); Sink &operator<<(std::string &&); diff --git a/plugins/experimental/inliner/util.h b/plugins/experimental/inliner/util.h index 68caff0d23f..854da526c29 100644 --- a/plugins/experimental/inliner/util.h +++ b/plugins/experimental/inliner/util.h @@ -25,15 +25,6 @@ #include -#define DISALLOW_COPY_AND_ASSIGN(T) \ - T(const T &) = delete; \ - void operator=(const T &) = delete - -#define DISALLOW_IMPLICIT_CONSTRUCTORS(T) \ -private: \ - T(void); \ - DISALLOW_COPY_AND_ASSIGN(T) - namespace util { typedef std::vector Buffer; diff --git a/plugins/experimental/inliner/vconnection.h b/plugins/experimental/inliner/vconnection.h index cb9e51fc8fa..6c11ffc7534 100644 --- a/plugins/experimental/inliner/vconnection.h +++ b/plugins/experimental/inliner/vconnection.h @@ -82,7 +82,7 @@ namespace io } } break; default: - assert(false); // UNRECHEABLE. + assert(false); // UNREACHABLE. break; } return TS_SUCCESS; diff --git a/tests/tools/traffic-replay/Config.py b/plugins/experimental/ja3_fingerprint/Makefile.inc similarity index 76% rename from tests/tools/traffic-replay/Config.py rename to plugins/experimental/ja3_fingerprint/Makefile.inc index 48d3fc3919d..b29be9f7d59 100644 --- a/tests/tools/traffic-replay/Config.py +++ b/plugins/experimental/ja3_fingerprint/Makefile.inc @@ -1,6 +1,3 @@ -#!/bin/env python3 -''' -''' # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -17,18 +14,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -# SSL config -ca_certs = None -keyfile = None +pkglib_LTLIBRARIES += experimental/ja3_fingerprint/ja3_fingerprint.la -# Proxy config -proxy_host = "127.0.0.1" -proxy_ssl_port = 443 -proxy_nonssl_port = 8080 - -# process and thread config -nProcess = 4 -nThread = 4 - -# colorize output -colorize = True +experimental_ja3_fingerprint_ja3_fingerprint_la_SOURCES = experimental/ja3_fingerprint/ja3_fingerprint.cc diff --git a/plugins/experimental/ja3_fingerprint/README b/plugins/experimental/ja3_fingerprint/README new file mode 100644 index 00000000000..fdb85c78265 --- /dev/null +++ b/plugins/experimental/ja3_fingerprint/README @@ -0,0 +1,26 @@ +ATS (Apache Traffic Server) JA3 Fingerprint Plugin + +General description +-------------------- +1. JA3 +This plugin looks at all incoming SSL/TLS clientHello and calculates JA3 fingerprint for each client. +It then performs +1) logging JA3 string and its MD5 hash to `ja3_fingerprint.log` in the standard logging directory; +2) appending `X-JA3-Sig` and/or `X-JA3-Raw` headers to upstream request (depending on the config) + +The log file format is as follows: + +[time] [client IP] [JA3 string] [MD5 Hash] + +2. plugin.config +In plugin.config, supply name of the plugin and options +Example: ja3_fingerprint.so --ja3raw --ja3log +Add flag --ja3raw if `X-JA3-Raw` is desired other than only `X-JA3-Sig`. +Add flag --ja3log if local logging in standard logging directory is desired. + +3. remap.config +This plugin can also be used as a remap plugin. For each remap rule, add plugin and parameter field. For example: +map http://from.com http://to.com @plugin=ja3_fingerprint.so [@pparam=--ja3raw] [@pparam=--ja3log] + +4. Requirement +Won't compile against OpenSSL 1.1.0 due to APIs and opaque structures. diff --git a/plugins/experimental/ja3_fingerprint/ja3_fingerprint.cc b/plugins/experimental/ja3_fingerprint/ja3_fingerprint.cc new file mode 100644 index 00000000000..7661d2492f7 --- /dev/null +++ b/plugins/experimental/ja3_fingerprint/ja3_fingerprint.cc @@ -0,0 +1,502 @@ +/** @ja3_fingerprint.cc + Plugin JA3 Fingerprint calculates JA3 signatures for incoming SSL traffic. + @section license License + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "ts/ts.h" +#include "ts/remap.h" + +#ifdef OPENSSL_NO_SSL_INTERN +#undef OPENSSL_NO_SSL_INTERN +#endif + +#include +#include +#include + +// Get 16bit big endian order and update pointer +#define n2s(c, s) ((s = (((unsigned int)(c[0])) << 8) | (((unsigned int)(c[1])))), c += 2) + +const char *PLUGIN_NAME = "ja3_fingerprint"; +static TSTextLogObject pluginlog; +static int ja3_idx = -1; +static int enable_raw = 0; +static int enable_log = 0; + +// GREASE table as in ja3 +static const std::unordered_set GREASE_table = {0x0a0a, 0x1a1a, 0x2a2a, 0x3a3a, 0x4a4a, 0x5a5a, 0x6a6a, 0x7a7a, + 0x8a8a, 0x9a9a, 0xaaaa, 0xbaba, 0xcaca, 0xdada, 0xeaea, 0xfafa}; + +struct ja3_data { + std::string ja3_string; + char md5String[33]; + char ip_addr[INET6_ADDRSTRLEN]; +}; + +struct ja3_remap_info { + int raw = false; + int log = false; + TSCont handler = nullptr; + + ~ja3_remap_info() + { + if (handler) { + TSContDestroy(handler); + handler = nullptr; + } + } +}; + +static int +custom_get_ja3_prefixed(int unit, const unsigned char *&data, int len, std::string &result) +{ + int cnt, tmp; + bool first = true; + // Extract each entry and append to result string + for (cnt = 0; cnt < len; cnt += unit) { + if (unit == 1) { + tmp = *(data++); + } else { + n2s(data, tmp); + } + + // Check for GREASE for 16-bit values, append only if non-GREASE + if (unit != 2 || GREASE_table.find(tmp) == GREASE_table.end()) { + if (!first) { + result += '-'; + } + first = false; + result += std::to_string(tmp); + } + } + return 0; +} + +char * +getIP(sockaddr const *s_sockaddr, char res[INET6_ADDRSTRLEN]) +{ + res[0] = '\0'; + + if (s_sockaddr == nullptr) { + return nullptr; + } + + switch (s_sockaddr->sa_family) { + case AF_INET: { + const struct sockaddr_in *s_sockaddr_in = reinterpret_cast(s_sockaddr); + inet_ntop(AF_INET, &s_sockaddr_in->sin_addr, res, INET_ADDRSTRLEN); + } break; + case AF_INET6: { + const struct sockaddr_in6 *s_sockaddr_in6 = reinterpret_cast(s_sockaddr); + inet_ntop(AF_INET6, &s_sockaddr_in6->sin6_addr, res, INET6_ADDRSTRLEN); + } break; + default: + return nullptr; + } + + return res[0] ? res : nullptr; +} + +#if OPENSSL_VERSION_NUMBER < 0x10100000L +// Parsing clientHello to get ja3 string +// No error checking or handling because this should be called after openSSL has done all checks and +// returned successfully +static std::string +custom_get_ja3(SSL *s) +{ + TSDebug(PLUGIN_NAME, "Entering custom_get_ja3()..."); + std::string ja3; + const unsigned char *p, *d; + int i, j, len; + + // ClientHello buf and len + d = p = (unsigned char *)s->init_msg; + long n = s->init_num; + + // Get version + int version = (((int)p[0]) << 8) | (int)p[1]; + ja3 += std::to_string(version) + ','; + p += 2; + + // Skip client random + p += SSL3_RANDOM_SIZE; + + // Skip session id + j = *(p++); + p += j; + + // No DTLS handling + + // Get cipher suites + n2s(p, len); + custom_get_ja3_prefixed(2, p, len, ja3); + ja3 += ','; + + // Skip compression + i = *(p++); + p += i; + + // Get extensions + uint16_t type; + int size; + std::string eclist, ecpflist; + + // Skip length blob + p += 2; + bool first = true; + while (p < d + n) { + // Each extension blob is comprised of [2bytes] type + [2bytes] size + [size bytes] data + n2s(p, type); + n2s(p, size); + + // Elliptic curve points + if (type == 0x0a) { + const unsigned char *sdata = p; + n2s(sdata, len); + custom_get_ja3_prefixed(2, sdata, len, eclist); + } + // Elliptic curve point formats + else if (type == 0x0b) { + const unsigned char *sdata = p; + len = *(sdata++); + custom_get_ja3_prefixed(1, sdata, len, ecpflist); + } + + // Update pointer + p += size; + + // Update ja3 string with valid extension type + if (GREASE_table.find(type) == GREASE_table.end()) { + if (!first) { + ja3 += '-'; + } + first = false; + ja3 += std::to_string(type); + } + } + + // Append eclist and ecpflist + ja3 += "," + eclist + "," + ecpflist; + TSDebug(PLUGIN_NAME, "ja3 string: %s", ja3.c_str()); + return ja3; +} +#elif OPENSSL_VERSION_NUMBER >= 0x10101000L +static std::string +custom_get_ja3(SSL *s) +{ + std::string ja3; + size_t len; + const unsigned char *p; + + // Get version + unsigned int version = SSL_client_hello_get0_legacy_version(s); + ja3 += std::to_string(version) + ','; + + // Get cipher suites + len = SSL_client_hello_get0_ciphers(s, &p); + custom_get_ja3_prefixed(2, p, len, ja3); + ja3 += ','; + + // Get extensions + int *o; + std::string eclist, ecpflist; + if (SSL_client_hello_get0_ext(s, 0x0a, &p, &len) == 1) { + // Skip first 2 bytes since we already have length + p += 2; + len -= 2; + custom_get_ja3_prefixed(2, p, len, eclist); + } + if (SSL_client_hello_get0_ext(s, 0x0b, &p, &len) == 1) { + // Skip first byte since we already have length + ++p; + --len; + custom_get_ja3_prefixed(1, p, len, ecpflist); + } + if (SSL_client_hello_get1_extensions_present(s, &o, &len) == 1) { + bool first = true; + for (size_t i = 0; i < len; i++) { + int type = o[i]; + if (GREASE_table.find(type) == GREASE_table.end()) { + if (!first) { + ja3 += '-'; + } + first = false; + ja3 += std::to_string(type); + } + } + OPENSSL_free(o); + } + ja3 += "," + eclist + "," + ecpflist; + return ja3; +} +#else +#error OpenSSL cannot be 1.1.0 +#endif + +static int +client_hello_ja3_handler(TSCont contp, TSEvent event, void *edata) +{ + TSVConn ssl_vc = reinterpret_cast(edata); + switch (event) { + case TS_EVENT_SSL_CLIENT_HELLO: { + TSSslConnection sslobj = TSVConnSSLConnectionGet(ssl_vc); + + // OpenSSL handle + SSL *ssl = reinterpret_cast(sslobj); + + ja3_data *data = new ja3_data; + data->ja3_string.append(custom_get_ja3(ssl)); + getIP(TSNetVConnRemoteAddrGet(ssl_vc), data->ip_addr); + + TSVConnArgSet(ssl_vc, ja3_idx, static_cast(data)); + TSDebug(PLUGIN_NAME, "client_hello_ja3_handler(): JA3: %s", data->ja3_string.c_str()); + + // MD5 hash + unsigned char digest[MD5_DIGEST_LENGTH]; + MD5((unsigned char *)data->ja3_string.c_str(), data->ja3_string.length(), digest); + + for (int i = 0; i < 16; i++) { + sprintf(&(data->md5String[i * 2]), "%02x", (unsigned int)digest[i]); + } + TSDebug(PLUGIN_NAME, "Fingerprint: %s", data->md5String); + break; + } + case TS_EVENT_VCONN_CLOSE: { + // Clean up + ja3_data *data = static_cast(TSVConnArgGet(ssl_vc, ja3_idx)); + + if (data == nullptr) { + TSDebug(PLUGIN_NAME, "client_hello_ja3_handler(): Failed to retrieve ja3 data at VCONN_CLOSE."); + return TS_ERROR; + } + + TSVConnArgSet(ssl_vc, ja3_idx, nullptr); + + delete data; + break; + } + default: { + TSDebug(PLUGIN_NAME, "client_hello_ja3_handler(): Unexpected event."); + break; + } + } + TSVConnReenable(ssl_vc); + return TS_SUCCESS; +} + +static int +req_hdr_ja3_handler(TSCont contp, TSEvent event, void *edata) +{ + TSHttpTxn txnp = nullptr; + TSHttpSsn ssnp = nullptr; + TSVConn vconn = nullptr; + if ((txnp = static_cast(edata)) == nullptr || (ssnp = TSHttpTxnSsnGet(txnp)) == nullptr || + (vconn = TSHttpSsnClientVConnGet(ssnp)) == nullptr) { + TSDebug(PLUGIN_NAME, "req_hdr_ja3_handler(): Failure to retrieve txn/ssn/vconn object."); + TSHttpTxnReenable(txnp, TS_EVENT_HTTP_CONTINUE); + return TS_SUCCESS; + } + + // Retrieve ja3_data from vconn args + ja3_data *data = static_cast(TSVConnArgGet(vconn, ja3_idx)); + if (data) { + // Decide global or remap + ja3_remap_info *info = static_cast(TSContDataGet(contp)); + bool raw_flag = info ? info->raw : enable_raw; + bool log_flag = info ? info->log : enable_log; + TSDebug(PLUGIN_NAME, "req_hdr_ja3_handler(): Found ja3 string."); + + // Get handle to headers + TSMBuffer bufp; + TSMLoc hdr_loc, field_loc; + TSAssert(TS_SUCCESS == TSHttpTxnServerReqGet(txnp, &bufp, &hdr_loc)); + + // Add JA3 md5 fingerprints + TSMimeHdrFieldCreateNamed(bufp, hdr_loc, "X-JA3-Sig", 9, &field_loc); + TSMimeHdrFieldValueStringSet(bufp, hdr_loc, field_loc, -1, data->md5String, 32); + TSMimeHdrFieldAppend(bufp, hdr_loc, field_loc); + TSHandleMLocRelease(bufp, hdr_loc, field_loc); + + // If raw string is configured, added JA3 raw string to header as well + if (raw_flag) { + TSMimeHdrFieldCreateNamed(bufp, hdr_loc, "X-JA3-Raw", 9, &field_loc); + TSMimeHdrFieldValueStringSet(bufp, hdr_loc, field_loc, -1, data->ja3_string.data(), data->ja3_string.size()); + TSMimeHdrFieldAppend(bufp, hdr_loc, field_loc); + TSHandleMLocRelease(bufp, hdr_loc, field_loc); + } + TSHandleMLocRelease(bufp, TS_NULL_MLOC, hdr_loc); + + // Write to logfile + if (log_flag) { + TSTextLogObjectWrite(pluginlog, "Client IP: %s\tJA3: %.*s\tMD5: %.*s", data->ip_addr, + static_cast(data->ja3_string.size()), data->ja3_string.data(), 32, data->md5String); + } + } else { + TSDebug(PLUGIN_NAME, "req_hdr_ja3_handler(): ja3 data not set. Not SSL vconn. Abort."); + } + TSHttpTxnReenable(txnp, TS_EVENT_HTTP_CONTINUE); + return TS_SUCCESS; +} + +static bool +read_config_option(int argc, const char *argv[], int &raw, int &log) +{ + static const struct option longopts[] = { + {"ja3raw", no_argument, &raw, 1}, {"ja3log", no_argument, &log, 1}, {nullptr, 0, nullptr, 0}}; + + int opt = 0; + while ((opt = getopt_long(argc, (char *const *)argv, "", longopts, nullptr)) >= 0) { + switch (opt) { + case '?': + TSDebug(PLUGIN_NAME, "read_config_option(): Unrecognized command arguments."); + case 0: + case -1: + break; + default: + TSDebug(PLUGIN_NAME, "read_config_option(): Unexpected options error."); + return false; + } + } + + TSDebug(PLUGIN_NAME, "read_config_option(): ja3 raw is %s", (raw == 1) ? "enabled" : "disabled"); + TSDebug(PLUGIN_NAME, "read_config_option(): ja3 logging is %s", (log == 1) ? "enabled" : "disabled"); + return true; +} + +void +TSPluginInit(int argc, const char *argv[]) +{ + TSDebug(PLUGIN_NAME, "Initializing plugin"); + + TSPluginRegistrationInfo info; + + info.plugin_name = PLUGIN_NAME; + info.vendor_name = "Oath"; + info.support_email = "zeyuany@oath.com"; + + // Options + if (!read_config_option(argc, argv, enable_raw, enable_log)) { + return; + } + + if (TSPluginRegister(&info) != TS_SUCCESS) { + TSError("[%s] Unable to initialize plugin. Failed to register.", PLUGIN_NAME); + } else { + if (enable_log && !pluginlog) { + TSAssert(TS_SUCCESS == TSTextLogObjectCreate(PLUGIN_NAME, TS_LOG_MODE_ADD_TIMESTAMP, &pluginlog)); + TSDebug(PLUGIN_NAME, "log object created successfully"); + } + // SNI handler + TSCont ja3_cont = TSContCreate(client_hello_ja3_handler, nullptr); + TSVConnArgIndexReserve(PLUGIN_NAME, "used to pass ja3", &ja3_idx); + TSHttpHookAdd(TS_SSL_CLIENT_HELLO_HOOK, ja3_cont); + TSHttpHookAdd(TS_VCONN_CLOSE_HOOK, ja3_cont); + TSHttpHookAdd(TS_HTTP_SEND_REQUEST_HDR_HOOK, TSContCreate(req_hdr_ja3_handler, nullptr)); + } + + return; +} + +// Remap Part +TSReturnCode +TSRemapInit(TSRemapInterface *api_info, char *errbuf, int errbuf_size) +{ + TSDebug(PLUGIN_NAME, "JA3 Remap Plugin initializing.."); + + // Check if there is config conflict as both global and remap plugin + if (ja3_idx >= 0) { + TSError(PLUGIN_NAME, "TSRemapInit(): JA3 configured as both global and remap. Check plugin.config."); + return TS_ERROR; + } + + // Set up SNI handler for all TLS connections + TSCont ja3_cont = TSContCreate(client_hello_ja3_handler, nullptr); + TSVConnArgIndexReserve(PLUGIN_NAME, "Used to pass ja3", &ja3_idx); + TSHttpHookAdd(TS_SSL_CLIENT_HELLO_HOOK, ja3_cont); + TSHttpHookAdd(TS_VCONN_CLOSE_HOOK, ja3_cont); + + return TS_SUCCESS; +} + +TSReturnCode +TSRemapNewInstance(int argc, char *argv[], void **ih, char * /* errbuf ATS_UNUSED */, int /* errbuf_size ATS_UNUSED */) +{ + TSDebug(PLUGIN_NAME, "New instance for client matching %s to %s", argv[0], argv[1]); + ja3_remap_info *pri = new ja3_remap_info; + + // Parse parameters + if (!read_config_option(argc - 1, const_cast(argv + 1), pri->raw, pri->log)) { + TSDebug(PLUGIN_NAME, "TSRemapNewInstance(): Bad arguments"); + return TS_ERROR; + } + + if (pri->log && !pluginlog) { + TSAssert(TS_SUCCESS == TSTextLogObjectCreate(PLUGIN_NAME, TS_LOG_MODE_ADD_TIMESTAMP, &pluginlog)); + TSDebug(PLUGIN_NAME, "log object created successfully"); + } + + // Create continuation + pri->handler = TSContCreate(req_hdr_ja3_handler, nullptr); + TSContDataSet(pri->handler, pri); + + // Pass to other remap plugin functions + *ih = static_cast(pri); + return TS_SUCCESS; +} + +TSRemapStatus +TSRemapDoRemap(void *ih, TSHttpTxn rh, TSRemapRequestInfo *rri) +{ + auto pri = static_cast(ih); + + // On remap, set up handler at send req hook to send JA3 data as header + if (!pri || !rri || !(pri->handler)) { + TSError("[%s] TSRemapDoRemap(): Invalid private data or RRI or handler.", PLUGIN_NAME); + } else { + TSHttpTxnHookAdd(rh, TS_HTTP_SEND_REQUEST_HDR_HOOK, pri->handler); + } + + return TSREMAP_NO_REMAP; +} + +void +TSRemapDeleteInstance(void *ih) +{ + auto pri = static_cast(ih); + if (pri) { + delete pri; + } + ih = nullptr; +} diff --git a/plugins/experimental/magick/magick.cc b/plugins/experimental/magick/magick.cc index b4f3cb05d5f..f5250b56108 100644 --- a/plugins/experimental/magick/magick.cc +++ b/plugins/experimental/magick/magick.cc @@ -9,7 +9,7 @@ http://www.apache.org/licenses/LICENSE-2.0 - Unless required by applicable law or ageed to in writing, software + Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and @@ -169,7 +169,7 @@ struct EVPContext { EVP_MD_CTX_destroy(context); } - EVPContext(void) : context(EVP_MD_CTX_create()) { assert(nullptr != context); } + EVPContext() : context(EVP_MD_CTX_create()) { assert(nullptr != context); } }; struct EVPKey { @@ -181,7 +181,7 @@ struct EVPKey { EVP_PKEY_free(key); } - EVPKey(void) : key(EVP_PKEY_new()) { assert(nullptr != key); } + EVPKey() : key(EVP_PKEY_new()) { assert(nullptr != key); } bool assign(const char *const k) const @@ -250,7 +250,7 @@ struct Exception { info = DestroyExceptionInfo(info); } - Exception(void) : info(AcquireExceptionInfo()) { assert(nullptr != info); } + Exception() : info(AcquireExceptionInfo()) { assert(nullptr != info); } }; struct Image { @@ -262,12 +262,12 @@ struct Image { info = DestroyImageInfo(info); } - Image(void) : info(AcquireImageInfo()) { assert(nullptr != info); } + Image() : info(AcquireImageInfo()) { assert(nullptr != info); } }; struct Wand { MagickWand *wand; - void *blob; + void *blob = nullptr; ~Wand() { @@ -278,17 +278,17 @@ struct Wand { } } - Wand(void) : wand(NewMagickWand()), blob(nullptr) { assert(nullptr != wand); } + Wand() : wand(NewMagickWand()) { assert(nullptr != wand); } void - clear(void) const + clear() const { assert(nullptr != wand); ClearMagickWand(wand); } std::string_view - get(void) + get() { assert(nullptr != wand); std::size_t length = 0; @@ -336,7 +336,7 @@ struct Wand { struct Core { ~Core() { MagickCoreTerminus(); } - Core(void) { MagickCoreGenesis("/tmp", MagickFalse); } + Core() { MagickCoreGenesis("/tmp", MagickFalse); } }; } // namespace magick @@ -362,7 +362,7 @@ struct QueryMap { } void - parse(void) + parse() { std::string_view key; std::size_t i = 0, j = 0; @@ -473,7 +473,7 @@ struct ImageTransform : TransformationPlugin { } void - handleInputComplete(void) override + handleInputComplete() override { TSDebug(PLUGIN_TAG, "handleInputComplete"); @@ -512,10 +512,10 @@ struct ImageTransform : TransformationPlugin { struct GlobalHookPlugin : GlobalPlugin { magick::Core core_; - magick::EVPKey *key_; + magick::EVPKey *key_ = nullptr; ThreadPool threadPool_; - ~GlobalHookPlugin() + ~GlobalHookPlugin() override { if (nullptr != key_) { delete key_; @@ -523,7 +523,7 @@ struct GlobalHookPlugin : GlobalPlugin { } } - GlobalHookPlugin(const char *const f = nullptr) : key_(nullptr), threadPool_(2) + GlobalHookPlugin(const char *const f = nullptr) : threadPool_(2) { if (nullptr != f) { assert(0 < strlen(f)); diff --git a/plugins/experimental/magick/sign.sh b/plugins/experimental/magick/sign.sh index e2b1592e73f..8f51b14f06a 100755 --- a/plugins/experimental/magick/sign.sh +++ b/plugins/experimental/magick/sign.sh @@ -10,7 +10,7 @@ # http://www.apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or ageed to in writing, software +# Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and diff --git a/plugins/experimental/magick/test.sh b/plugins/experimental/magick/test.sh index 31fc448edb4..c0b08542f67 100755 --- a/plugins/experimental/magick/test.sh +++ b/plugins/experimental/magick/test.sh @@ -10,7 +10,7 @@ # http://www.apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or ageed to in writing, software +# Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and diff --git a/plugins/experimental/magick/verify.sh b/plugins/experimental/magick/verify.sh index 61ad81e5de6..3de40b4a4b2 100755 --- a/plugins/experimental/magick/verify.sh +++ b/plugins/experimental/magick/verify.sh @@ -10,7 +10,7 @@ # http://www.apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or ageed to in writing, software +# Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and diff --git a/plugins/experimental/memcache/protocol_binary.h b/plugins/experimental/memcache/protocol_binary.h index 1f49d1417d3..8ca18ff98b6 100644 --- a/plugins/experimental/memcache/protocol_binary.h +++ b/plugins/experimental/memcache/protocol_binary.h @@ -69,7 +69,7 @@ typedef enum { } protocol_binary_response_status; /** - * Defintion of the different command opcodes. + * Definition of the different command opcodes. * See section 3.3 Command Opcodes */ typedef enum { diff --git a/plugins/experimental/memcache/tsmemcache.cc b/plugins/experimental/memcache/tsmemcache.cc index a3cdebc1ae2..db03b23fc6b 100644 --- a/plugins/experimental/memcache/tsmemcache.cc +++ b/plugins/experimental/memcache/tsmemcache.cc @@ -962,7 +962,7 @@ MC::ascii_incr_decr_event(int event, void *data) } header.cas = ink_atomic_increment(&next_cas, 1); { - char *data = 0; + char *data = nullptr; int len = 0; // must be huge, why convert to a counter ?? if (cwvc->get_single_data((void **)&data, &len) < 0) { diff --git a/plugins/experimental/memcache/tsmemcache.h b/plugins/experimental/memcache/tsmemcache.h index c43fc1f032e..fc6c81188a5 100644 --- a/plugins/experimental/memcache/tsmemcache.h +++ b/plugins/experimental/memcache/tsmemcache.h @@ -83,15 +83,15 @@ struct MCAccept : public Continuation { #ifndef HAVE_TLS ProxyAllocator *theMCThreadAllocator; #endif - int accept_port; + int accept_port = 0; int main_event(int event, void *netvc); MCAccept() : #ifndef HAVE_TLS - theMCThreadAllocator(NULL), + theMCThreadAllocator(NULL) #endif - accept_port(0) + { SET_HANDLER(&MCAccept::main_event); } diff --git a/plugins/experimental/memcached_remap/README b/plugins/experimental/memcached_remap/README index 449060d73f5..248341f3086 100644 --- a/plugins/experimental/memcached_remap/README +++ b/plugins/experimental/memcached_remap/README @@ -1,7 +1,7 @@ Apache Traffic Server Memcached-based remap plugin ============================================== -This is a plugin which is based on the mysql_remap code (http://svn.apache.org/repos/asf/trafficserver/plugins/mysql_remap/) +This is a plugin which is based on the mysql_remap code (https://github.com/apache/trafficserver/tree/master/plugins/experimental/mysql_remap) that allows us to do dynamic reverse proxy (dynamic remap) based on the information present in the memcached database. @@ -23,27 +23,13 @@ server too (please change things accordingly, if any) sudo yum install memcached python-memcached -NOTE 14-OCT-2015: +2. Compile and Install the memcached_remap plugin + To build this with the ATS configure script when --enable-experimental-plugins is used, you must have libmemcached version 1.0 or greater installed. -2. Compile and Install the memcached_remap plugin - -[user@ memcached_remap]$ gmake -/usr/local/bin/tsxs -o memcached_remap.so memcached_remap.cc - compiling memcached_remap.cc -> memcached_remap.lo - linking -> memcached_remap.so - -[user@ memcached_remap]$ sudo gmake install -/usr/local/bin/tsxs -o memcached_remap.so memcached_remap.cc - compiling memcached_remap.cc -> memcached_remap.lo - linking -> memcached_remap.so -/usr/local/bin/tsxs -i -o memcached_remap.so - installing memcached_remap.so -> -/usr/local/libexec/trafficserver/memcached_remap.so - -3. Start the Traffic Server in Plugin Mode (to debug any issues) +3. Start Traffic Server with logging (to debug any issues) /usr/local/bin/traffic_server -T "memcached_remap" diff --git a/plugins/experimental/memcached_remap/memcached_remap.cc b/plugins/experimental/memcached_remap/memcached_remap.cc index 2cee48ee0f5..e6b2834ca5c 100644 --- a/plugins/experimental/memcached_remap/memcached_remap.cc +++ b/plugins/experimental/memcached_remap/memcached_remap.cc @@ -22,7 +22,6 @@ #include #include -// change this on your box #include // global settings @@ -96,7 +95,7 @@ do_memcached_remap(TSCont contp, TSHttpTxn txnp) if (m_result) free(m_result); TSDebug(PLUGIN_NAME, "\nOUTGOING REQUEST ->\n ::: to_scheme_desc: %s\n ::: to_hostname: %s\n ::: to_port: %d", oscheme, ohost, - oport); // row[0],row[1],row[2]); + oport); TSMimeHdrFieldValueStringSet(reqp, hdr_loc, field_loc, 0, ohost, -1); TSUrlHostSet(reqp, url_loc, ohost, -1); TSUrlSchemeSet(reqp, url_loc, oscheme, -1); @@ -112,7 +111,7 @@ do_memcached_remap(TSCont contp, TSHttpTxn txnp) goto not_found; } - goto free_stuff; // free the result set after processed + goto release_field; // free the result set after processed not_found: // lets build up a nice 404 message for someone @@ -120,13 +119,6 @@ do_memcached_remap(TSCont contp, TSHttpTxn txnp) TSHttpHdrStatusSet(reqp, hdr_loc, TS_HTTP_STATUS_NOT_FOUND); TSHttpTxnStatusSet(txnp, TS_HTTP_STATUS_NOT_FOUND); } -free_stuff: -#if (TS_VERSION_NUMBER < 2001005) - if (request_host) - TSHandleStringRelease(reqp, hdr_loc, request_host); - if (request_scheme) - TSHandleStringRelease(reqp, hdr_loc, request_scheme); -#endif release_field: if (field_loc) { TSHandleMLocRelease(reqp, hdr_loc, field_loc); @@ -166,8 +158,6 @@ TSPluginInit(int argc, const char *argv[]) { TSPluginRegistrationInfo info; memcached_return_t rc; - // FILE *fp; - // char servers_string[8192]; info.plugin_name = const_cast(PLUGIN_NAME); info.vendor_name = const_cast("Apache Software Foundation"); @@ -179,34 +169,8 @@ TSPluginInit(int argc, const char *argv[]) return; } - // parse the configuration file - // TODO: this is still under testing 1.0.2 version should have this feature - /* - if(argc < 1) { - TSError("memcached_remap: you should pass a configuration file as argument to plugin with list of servers.\n"); - return; - } - - fp = fopen(argv[0], "r"); - if(!fp) { - TSError("memcached_remap: Failed to open the configuration file %s\n", argv[0]); - return; - } - - while(!feof(fp)) { - fscanf(fp,"servers=%[^\n] ", servers_string); - } - - fclose(fp); - */ - - // initialize the memcache - // fp = NULL; - // snprintf(servers_string, 1,"%c",'h'); memc = memcached_create(NULL); - // servers = memcached_servers_parse(servers_string); - servers = memcached_server_list_append(NULL, "localhost", 11211, &rc); if (rc != MEMCACHED_SUCCESS) { TSError("[memcached_remap] Plugin registration failed while adding servers.\n"); diff --git a/plugins/experimental/memcached_remap/sample.py b/plugins/experimental/memcached_remap/sample.py index 07f28209ff8..01466cddbd4 100755 --- a/plugins/experimental/memcached_remap/sample.py +++ b/plugins/experimental/memcached_remap/sample.py @@ -25,8 +25,8 @@ mc = memcache.Client(['127.0.0.1:11211'], debug=0) # Add couple of keys -mc.set("http://127.0.0.1:80/", "http://127.0.0.1:8080"); -mc.set("http://localhost:80/", "http://localhost:8080"); +mc.set("http://127.0.0.1:80/", "http://127.0.0.1:8080") +mc.set("http://localhost:80/", "http://localhost:8080") # Print the keys that are saved print "response-1 is '%s'" % (mc.get("http://127.0.0.1:80/")) diff --git a/plugins/experimental/metalink/README b/plugins/experimental/metalink/README index ecd8d0049e3..8ab54b30a63 100644 --- a/plugins/experimental/metalink/README +++ b/plugins/experimental/metalink/README @@ -41,7 +41,7 @@ header is already cached. If it isn't, then it tries to find a URL that is cached to use instead. It looks in the cache for some object that matches the digest in the Digest header and if it - succeeds, then it rewites the Location header with that object's + succeeds, then it rewrites the Location header with that object's URL. This way a client should get sent to a URL that's already cached diff --git a/plugins/experimental/money_trace/README b/plugins/experimental/money_trace/README index f24cbb1f677..902f860199f 100644 --- a/plugins/experimental/money_trace/README +++ b/plugins/experimental/money_trace/README @@ -1,6 +1,6 @@ Money trace plugin - This is a remap plugin that allows ATS to participate in a distrbuted tracing system based upon + This is a remap plugin that allows ATS to participate in a distributed tracing system based upon the Comcast "Money" distributed tracing and monitoring library. The Comcast "Money" library has its roots in Google's Dapper and Twitters Zipkin systems. A money trace header or session id, is attached to transaction and allows an operator with the appropriate logging systems in place, @@ -21,7 +21,7 @@ Money trace plugin See the documentation at the link above for a complete description on the "X-MoneyTrace" header and how to use and extend it in a distributed tracing system. - To configure and use this plugin, simply add it in the remap.config file where neded. EXAMPLE: + To configure and use this plugin, simply add it in the remap.config file where needed. EXAMPLE: map http://vod.foobar.com http://origin.vod.foobar.com @plugin=money_trace.so diff --git a/plugins/experimental/money_trace/money_trace.cc b/plugins/experimental/money_trace/money_trace.cc index cb1b716e930..3ea43b1d0eb 100644 --- a/plugins/experimental/money_trace/money_trace.cc +++ b/plugins/experimental/money_trace/money_trace.cc @@ -289,7 +289,7 @@ transaction_handler(TSCont contp, TSEvent event, void *edata) mt_send_client_response(txnp, txn_data); break; case TS_EVENT_HTTP_TXN_CLOSE: - LOG_DEBUG("handling transacation close."); + LOG_DEBUG("handling transaction close."); freeTransactionData(txn_data); TSContDestroy(contp); break; diff --git a/plugins/experimental/mp4/mp4_common.h b/plugins/experimental/mp4/mp4_common.h index 126e4442813..780d238d472 100644 --- a/plugins/experimental/mp4/mp4_common.h +++ b/plugins/experimental/mp4/mp4_common.h @@ -32,7 +32,7 @@ class IOHandle { public: - IOHandle() : vio(nullptr), buffer(nullptr), reader(nullptr){}; + IOHandle(){}; ~IOHandle() { @@ -48,9 +48,9 @@ class IOHandle } public: - TSVIO vio; - TSIOBuffer buffer; - TSIOBufferReader reader; + TSVIO vio = nullptr; + TSIOBuffer buffer = nullptr; + TSIOBufferReader reader = nullptr; }; class Mp4TransformContext diff --git a/plugins/experimental/mp4/mp4_meta.cc b/plugins/experimental/mp4/mp4_meta.cc index b7c4c9e7f61..847565f157a 100644 --- a/plugins/experimental/mp4/mp4_meta.cc +++ b/plugins/experimental/mp4/mp4_meta.cc @@ -415,7 +415,7 @@ Mp4Meta::mp4_read_ftyp_atom(int64_t atom_header_size, int64_t atom_data_size) atom_size = atom_header_size + atom_data_size; - if (meta_avail < atom_size) { // data unsufficient, reasonable from the first level + if (meta_avail < atom_size) { // data insufficient, reasonable from the first level return 0; } @@ -447,7 +447,7 @@ Mp4Meta::mp4_read_moov_atom(int64_t atom_header_size, int64_t atom_data_size) return -1; } - if (meta_avail < atom_size) { // data unsufficient, wait + if (meta_avail < atom_size) { // data insufficient, wait return 0; } diff --git a/plugins/experimental/mp4/mp4_meta.h b/plugins/experimental/mp4/mp4_meta.h index 1e8d1b16626..d4a11293af1 100644 --- a/plugins/experimental/mp4/mp4_meta.h +++ b/plugins/experimental/mp4/mp4_meta.h @@ -166,7 +166,7 @@ typedef struct { u_char reverved3[2]; u_char matrix[36]; u_char width[4]; - u_char heigth[4]; + u_char height[4]; } mp4_tkhd_atom; typedef struct { @@ -186,7 +186,7 @@ typedef struct { u_char reverved3[2]; u_char matrix[36]; u_char width[4]; - u_char heigth[4]; + u_char height[4]; } mp4_tkhd64_atom; typedef struct { @@ -310,7 +310,7 @@ typedef struct { class BufferHandle { public: - BufferHandle() : buffer(nullptr), reader(nullptr){}; + BufferHandle(){}; ~BufferHandle() { @@ -326,64 +326,41 @@ class BufferHandle } public: - TSIOBuffer buffer; - TSIOBufferReader reader; + TSIOBuffer buffer = nullptr; + TSIOBufferReader reader = nullptr; }; class Mp4Trak { public: - Mp4Trak() - : timescale(0), - duration(0), - time_to_sample_entries(0), - sample_to_chunk_entries(0), - sync_samples_entries(0), - composition_offset_entries(0), - sample_sizes_entries(0), - chunks(0), - start_sample(0), - start_chunk(0), - chunk_samples(0), - chunk_samples_size(0), - start_offset(0), - tkhd_size(0), - mdhd_size(0), - hdlr_size(0), - vmhd_size(0), - smhd_size(0), - dinf_size(0), - size(0) - { - memset(&stsc_chunk_entry, 0, sizeof(mp4_stsc_entry)); - } + Mp4Trak() { memset(&stsc_chunk_entry, 0, sizeof(mp4_stsc_entry)); } ~Mp4Trak() {} public: - uint32_t timescale; - int64_t duration; - - uint32_t time_to_sample_entries; // stsc - uint32_t sample_to_chunk_entries; // stsc - uint32_t sync_samples_entries; // stss - uint32_t composition_offset_entries; // ctts - uint32_t sample_sizes_entries; // stsz - uint32_t chunks; // stco, co64 - - uint32_t start_sample; - uint32_t start_chunk; - uint32_t chunk_samples; - uint64_t chunk_samples_size; - off_t start_offset; - - size_t tkhd_size; - size_t mdhd_size; - size_t hdlr_size; - size_t vmhd_size; - size_t smhd_size; - size_t dinf_size; - size_t size; + uint32_t timescale = 0; + int64_t duration = 0; + + uint32_t time_to_sample_entries = 0; // stsc + uint32_t sample_to_chunk_entries = 0; // stsc + uint32_t sync_samples_entries = 0; // stss + uint32_t composition_offset_entries = 0; // ctts + uint32_t sample_sizes_entries = 0; // stsz + uint32_t chunks = 0; // stco, co64 + + uint32_t start_sample = 0; + uint32_t start_chunk = 0; + uint32_t chunk_samples = 0; + uint64_t chunk_samples_size = 0; + off_t start_offset = 0; + + size_t tkhd_size = 0; + size_t mdhd_size = 0; + size_t hdlr_size = 0; + size_t vmhd_size = 0; + size_t smhd_size = 0; + size_t dinf_size = 0; + size_t size = 0; BufferHandle atoms[MP4_LAST_ATOM + 1]; @@ -394,22 +371,7 @@ class Mp4Meta { public: Mp4Meta() - : start(0), - cl(0), - content_length(0), - meta_atom_size(0), - meta_avail(0), - wait_next(0), - need_size(0), - rs(0), - rate(0), - ftyp_size(0), - moov_size(0), - start_pos(0), - timescale(0), - trak_num(0), - passed(0), - meta_complete(false) + { memset(trak_vec, 0, sizeof(trak_vec)); meta_buffer = TSIOBufferCreate(); @@ -494,17 +456,17 @@ class Mp4Meta void mp4_update_mdhd_duration(Mp4Trak *trak); public: - int64_t start; // requested start time, measured in milliseconds. - int64_t cl; // the total size of the mp4 file - int64_t content_length; // the size of the new mp4 file - int64_t meta_atom_size; + int64_t start = 0; // requested start time, measured in milliseconds. + int64_t cl = 0; // the total size of the mp4 file + int64_t content_length = 0; // the size of the new mp4 file + int64_t meta_atom_size = 0; TSIOBuffer meta_buffer; // meta data to be parsed TSIOBufferReader meta_reader; - int64_t meta_avail; - int64_t wait_next; - int64_t need_size; + int64_t meta_avail = 0; + int64_t wait_next = 0; + int64_t need_size = 0; BufferHandle meta_atom; BufferHandle ftyp_atom; @@ -516,16 +478,16 @@ class Mp4Meta Mp4Trak *trak_vec[MP4_MAX_TRAK_NUM]; - double rs; - double rate; + double rs = 0; + double rate = 0; - int64_t ftyp_size; - int64_t moov_size; - int64_t start_pos; // start position of the new mp4 file - uint32_t timescale; - uint32_t trak_num; - int64_t passed; + int64_t ftyp_size = 0; + int64_t moov_size = 0; + int64_t start_pos = 0; // start position of the new mp4 file + uint32_t timescale = 0; + uint32_t trak_num = 0; + int64_t passed = 0; u_char mdat_atom_header[16]; - bool meta_complete; + bool meta_complete = false; }; diff --git a/plugins/experimental/multiplexer/chunk-decoder.h b/plugins/experimental/multiplexer/chunk-decoder.h index 0cfb4644128..b0b5f2e0bce 100644 --- a/plugins/experimental/multiplexer/chunk-decoder.h +++ b/plugins/experimental/multiplexer/chunk-decoder.h @@ -55,10 +55,10 @@ class ChunkDecoder void parseSizeCharacter(const char); int parseSize(const char *, const int64_t); int decode(const TSIOBufferReader &); - bool isSizeState(void) const; + bool isSizeState() const; inline bool - isEnd(void) const + isEnd() const { return state_ == State::kEnd; } diff --git a/plugins/experimental/multiplexer/dispatch.cc b/plugins/experimental/multiplexer/dispatch.cc index 32803eab6e9..675b23a349f 100644 --- a/plugins/experimental/multiplexer/dispatch.cc +++ b/plugins/experimental/multiplexer/dispatch.cc @@ -47,7 +47,7 @@ Request::Request(const std::string &h, const TSMBuffer b, const TSMLoc l) : host /* * TSHttpHdrLengthGet returns the size with possible "internal" headers * which are not printed by TSHttpHdrPrint. - * Therefore the greater than or equal comparisson + * Therefore the greater than or equal comparison */ assert(TSHttpHdrLengthGet(b, l) >= length); } diff --git a/plugins/experimental/multiplexer/fetcher.h b/plugins/experimental/multiplexer/fetcher.h index a431d1e1529..45e892f2bac 100644 --- a/plugins/experimental/multiplexer/fetcher.h +++ b/plugins/experimental/multiplexer/fetcher.h @@ -42,12 +42,12 @@ namespace ats { struct HttpParser { - bool parsed_; + bool parsed_ = false; TSHttpParser parser_; TSMBuffer buffer_; TSMLoc location_; - void destroyParser(void); + void destroyParser(); ~HttpParser() { @@ -56,7 +56,7 @@ struct HttpParser { destroyParser(); } - HttpParser(void) : parsed_(false), parser_(TSHttpParserCreate()), buffer_(TSMBufferCreate()), location_(TSHttpHdrCreate(buffer_)) + HttpParser() : parser_(TSHttpParserCreate()), buffer_(TSMBufferCreate()), location_(TSHttpHdrCreate(buffer_)) { TSHttpHdrTypeSet(buffer_, location_, TS_HTTP_TYPE_RESPONSE); } @@ -64,7 +64,7 @@ struct HttpParser { bool parse(io::IO &); int - statusCode(void) const + statusCode() const { return static_cast(TSHttpHdrStatusGet(buffer_, location_)); } @@ -272,7 +272,7 @@ template struct HttpTransaction { break; default: - assert(false); // UNRECHEABLE. + assert(false); // UNREACHABLE. } return 0; } diff --git a/plugins/experimental/multiplexer/ts.h b/plugins/experimental/multiplexer/ts.h index 1e4f654a4ca..5308652e3ba 100644 --- a/plugins/experimental/multiplexer/ts.h +++ b/plugins/experimental/multiplexer/ts.h @@ -38,7 +38,7 @@ namespace io struct IO { TSIOBuffer buffer; TSIOBufferReader reader; - TSVIO vio; + TSVIO vio = nullptr; ~IO() { @@ -52,8 +52,8 @@ namespace io TSIOBufferDestroy(buffer); } - IO(void) : buffer(TSIOBufferCreate()), reader(TSIOBufferReaderAlloc(buffer)), vio(nullptr) {} - IO(const TSIOBuffer &b) : buffer(b), reader(TSIOBufferReaderAlloc(buffer)), vio(nullptr) { assert(buffer != nullptr); } + IO() : buffer(TSIOBufferCreate()), reader(TSIOBufferReaderAlloc(buffer)) {} + IO(const TSIOBuffer &b) : buffer(b), reader(TSIOBufferReaderAlloc(buffer)) { assert(buffer != nullptr); } static IO *read(TSVConn, TSCont, const int64_t); static IO * diff --git a/plugins/experimental/mysql_remap/lib/iniparser.h b/plugins/experimental/mysql_remap/lib/iniparser.h index d9bb650f1ea..ca415becc42 100644 --- a/plugins/experimental/mysql_remap/lib/iniparser.h +++ b/plugins/experimental/mysql_remap/lib/iniparser.h @@ -42,9 +42,9 @@ Includes ---------------------------------------------------------------------------*/ -#include -#include -#include +#include // NOLINT(modernize-deprecated-headers) +#include // NOLINT(modernize-deprecated-headers) +#include // NOLINT(modernize-deprecated-headers) /* * The following #include is necessary on many Unixes but not Linux. diff --git a/plugins/experimental/mysql_remap/mysql_remap.cc b/plugins/experimental/mysql_remap/mysql_remap.cc index 6d31ecd798d..1cba55a4124 100644 --- a/plugins/experimental/mysql_remap/mysql_remap.cc +++ b/plugins/experimental/mysql_remap/mysql_remap.cc @@ -187,7 +187,7 @@ TSPluginInit(int argc, const char *argv[]) my_data *data = (my_data *)malloc(1 * sizeof(my_data)); TSPluginRegistrationInfo info; - bool reconnect = 1; + bool reconnect = true; info.plugin_name = const_cast(PLUGIN_NAME); info.vendor_name = const_cast("Apache Software Foundation"); diff --git a/plugins/experimental/prefetch/common.cc b/plugins/experimental/prefetch/common.cc index fa2c9cff815..4664eebc721 100644 --- a/plugins/experimental/prefetch/common.cc +++ b/plugins/experimental/prefetch/common.cc @@ -22,8 +22,8 @@ * @see common.h */ -#include -#include +#include +#include #include "common.h" diff --git a/plugins/experimental/prefetch/configs.cc b/plugins/experimental/prefetch/configs.cc index 721acd3721b..89792602d07 100644 --- a/plugins/experimental/prefetch/configs.cc +++ b/plugins/experimental/prefetch/configs.cc @@ -54,19 +54,19 @@ isTrue(const char *arg) bool PrefetchConfig::init(int argc, char *argv[]) { - static const struct option longopt[] = {{const_cast("front"), optional_argument, 0, 'f'}, - {const_cast("api-header"), optional_argument, 0, 'h'}, - {const_cast("next-header"), optional_argument, 0, 'n'}, - {const_cast("fetch-policy"), optional_argument, 0, 'p'}, - {const_cast("fetch-count"), optional_argument, 0, 'c'}, - {const_cast("fetch-path-pattern"), optional_argument, 0, 'e'}, - {const_cast("fetch-max"), optional_argument, 0, 'x'}, - {const_cast("replace-host"), optional_argument, 0, 'r'}, - {const_cast("name-space"), optional_argument, 0, 's'}, - {const_cast("metrics-prefix"), optional_argument, 0, 'm'}, - {const_cast("exact-match"), optional_argument, 0, 'y'}, - {const_cast("log-name"), optional_argument, 0, 'l'}, - {0, 0, 0, 0}}; + static const struct option longopt[] = {{const_cast("front"), optional_argument, nullptr, 'f'}, + {const_cast("api-header"), optional_argument, nullptr, 'h'}, + {const_cast("next-header"), optional_argument, nullptr, 'n'}, + {const_cast("fetch-policy"), optional_argument, nullptr, 'p'}, + {const_cast("fetch-count"), optional_argument, nullptr, 'c'}, + {const_cast("fetch-path-pattern"), optional_argument, nullptr, 'e'}, + {const_cast("fetch-max"), optional_argument, nullptr, 'x'}, + {const_cast("replace-host"), optional_argument, nullptr, 'r'}, + {const_cast("name-space"), optional_argument, nullptr, 's'}, + {const_cast("metrics-prefix"), optional_argument, nullptr, 'm'}, + {const_cast("exact-match"), optional_argument, nullptr, 'y'}, + {const_cast("log-name"), optional_argument, nullptr, 'l'}, + {nullptr, 0, nullptr, 0}}; bool status = true; optind = 0; diff --git a/plugins/experimental/prefetch/configs.h b/plugins/experimental/prefetch/configs.h index 2552c3660af..d1d61a8d6da 100644 --- a/plugins/experimental/prefetch/configs.h +++ b/plugins/experimental/prefetch/configs.h @@ -39,11 +39,8 @@ class PrefetchConfig _nextHeader("X-AppleCDN-Prefetch-Next"), _replaceHost(), _namespace("default"), - _metricsPrefix("prefetch.stats"), - _fetchCount(1), - _fetchMax(0), - _front(false), - _exactMatch(false) + _metricsPrefix("prefetch.stats") + { } @@ -194,9 +191,9 @@ class PrefetchConfig std::string _namespace; std::string _metricsPrefix; std::string _logName; - unsigned _fetchCount; - unsigned _fetchMax; - bool _front; - bool _exactMatch; + unsigned _fetchCount = 1; + unsigned _fetchMax = 0; + bool _front = false; + bool _exactMatch = false; MultiPattern _nextPaths; }; diff --git a/plugins/experimental/prefetch/fetch.cc b/plugins/experimental/prefetch/fetch.cc index ca7daf0faa7..b34d062a453 100644 --- a/plugins/experimental/prefetch/fetch.cc +++ b/plugins/experimental/prefetch/fetch.cc @@ -25,9 +25,9 @@ #include #include #include -#include +#include #include -#include +#include #include "ts/ts.h" /* ATS API */ #include "fetch.h" @@ -114,7 +114,7 @@ createStat(const String &prefix, const String &space, const char *module, const return true; } -BgFetchState::BgFetchState() : _policy(nullptr), _unique(nullptr), _concurrentFetches(0), _concurrentFetchesMax(0), _log(nullptr) +BgFetchState::BgFetchState() { _policyLock = TSMutexCreate(); if (nullptr == _policyLock) { @@ -596,7 +596,7 @@ BgFetch::schedule() /* Schedule */ PrefetchDebug("schedule fetch: %s", _url.c_str()); _startTime = TShrtime(); - TSContSchedule(_cont, 0, TS_THREAD_POOL_NET); + TSContScheduleOnPool(_cont, 0, TS_THREAD_POOL_NET); } /* Log format is: name-space bytes status url */ diff --git a/plugins/experimental/prefetch/fetch.h b/plugins/experimental/prefetch/fetch.h index 4d73a7b097a..7c2a1dd7563 100644 --- a/plugins/experimental/prefetch/fetch.h +++ b/plugins/experimental/prefetch/fetch.h @@ -89,14 +89,14 @@ class BgFetchState void operator=(BgFetchState const &); /* never implement */ /* Fetch policy related */ - FetchPolicy *_policy; /* fetch policy */ - TSMutex _policyLock; /* protects the policy object only */ + FetchPolicy *_policy = nullptr; /* fetch policy */ + TSMutex _policyLock; /* protects the policy object only */ /* Mechanisms to avoid concurrent fetches and applying limits */ - FetchPolicy *_unique; /* make sure we never download same object multiple times at the same time */ - TSMutex _lock; /* protects the deduplication object only */ - size_t _concurrentFetches; - size_t _concurrentFetchesMax; + FetchPolicy *_unique = nullptr; /* make sure we never download same object multiple times at the same time */ + TSMutex _lock; /* protects the de-duplication object only */ + size_t _concurrentFetches = 0; + size_t _concurrentFetchesMax = 0; PrefetchMetricInfo _metrics[FETCHES_MAX_METRICS] = { {FETCH_ACTIVE, TS_RECORDDATATYPE_INT, -1}, {FETCH_COMPLETED, TS_RECORDDATATYPE_COUNTER, -1}, {FETCH_ERRORS, TS_RECORDDATATYPE_COUNTER, -1}, {FETCH_TIMEOOUTS, TS_RECORDDATATYPE_COUNTER, -1}, @@ -108,7 +108,7 @@ class BgFetchState {FETCH_POLICY_MAXSIZE, TS_RECORDDATATYPE_INT, -1}}; /* plugin specific fetch logging */ - TSTextLogObject _log; + TSTextLogObject _log = nullptr; }; /** diff --git a/plugins/experimental/prefetch/fetch_policy.cc b/plugins/experimental/prefetch/fetch_policy.cc index 083257f8fa3..14b98ff1d18 100644 --- a/plugins/experimental/prefetch/fetch_policy.cc +++ b/plugins/experimental/prefetch/fetch_policy.cc @@ -23,7 +23,7 @@ #include "fetch_policy.h" -#include +#include #include "common.h" #include "fetch_policy_lru.h" diff --git a/plugins/experimental/prefetch/fetch_policy.h b/plugins/experimental/prefetch/fetch_policy.h index 85947597629..cd91ec2570b 100644 --- a/plugins/experimental/prefetch/fetch_policy.h +++ b/plugins/experimental/prefetch/fetch_policy.h @@ -25,7 +25,7 @@ #include #include -#include +#include #include #include diff --git a/plugins/experimental/prefetch/fetch_policy_lru.h b/plugins/experimental/prefetch/fetch_policy_lru.h index 26996475673..8f14d889e61 100644 --- a/plugins/experimental/prefetch/fetch_policy_lru.h +++ b/plugins/experimental/prefetch/fetch_policy_lru.h @@ -86,20 +86,20 @@ class FetchPolicyLru : public FetchPolicy { public: /* Default size values are also considered minimum. TODO: find out if this works OK. */ - FetchPolicyLru() : _maxSize(10), _size(0){}; - virtual ~FetchPolicyLru(){}; + FetchPolicyLru(){}; + ~FetchPolicyLru() override{}; /* Fetch policy interface methods */ - bool init(const char *parameters); - bool acquire(const std::string &url); - bool release(const std::string &url); - const char *name(); - size_t getMaxSize(); - size_t getSize(); + bool init(const char *parameters) override; + bool acquire(const std::string &url) override; + bool release(const std::string &url) override; + const char *name() override; + size_t getMaxSize() override; + size_t getSize() override; protected: LruMap _map; LruList _list; - LruList::size_type _maxSize; - LruList::size_type _size; + LruList::size_type _maxSize = 10; + LruList::size_type _size = 0; }; diff --git a/plugins/experimental/prefetch/fetch_policy_simple.h b/plugins/experimental/prefetch/fetch_policy_simple.h index be04d86dcb2..b07262ea9aa 100644 --- a/plugins/experimental/prefetch/fetch_policy_simple.h +++ b/plugins/experimental/prefetch/fetch_policy_simple.h @@ -33,13 +33,13 @@ class FetchPolicySimple : public FetchPolicy { public: FetchPolicySimple() {} - virtual ~FetchPolicySimple(){}; - bool init(const char *parameters); - bool acquire(const std::string &url); - bool release(const std::string &url); - const char *name(); - size_t getSize(); - size_t getMaxSize(); + ~FetchPolicySimple() override{}; + bool init(const char *parameters) override; + bool acquire(const std::string &url) override; + bool release(const std::string &url) override; + const char *name() override; + size_t getSize() override; + size_t getMaxSize() override; private: std::unordered_map _urls; diff --git a/plugins/experimental/prefetch/headers.cc b/plugins/experimental/prefetch/headers.cc index 8233dc54063..fbfe7329482 100644 --- a/plugins/experimental/prefetch/headers.cc +++ b/plugins/experimental/prefetch/headers.cc @@ -21,8 +21,8 @@ * @brief HTTP headers manipulation. */ -#include -#include +#include +#include #include "configs.h" #include "headers.h" diff --git a/plugins/experimental/prefetch/pattern.cc b/plugins/experimental/prefetch/pattern.cc index 20e3b64688c..e0ca6ca3f08 100644 --- a/plugins/experimental/prefetch/pattern.cc +++ b/plugins/experimental/prefetch/pattern.cc @@ -38,7 +38,7 @@ replaceString(String &str, const String &from, const String &to) } } -Pattern::Pattern() : _re(nullptr), _extra(nullptr), _pattern(""), _replacement(""), _tokenCount(0) {} +Pattern::Pattern() : _pattern(""), _replacement("") {} /** * @brief Initializes PCRE pattern by providing the subject and replacement strings. @@ -47,17 +47,17 @@ Pattern::Pattern() : _re(nullptr), _extra(nullptr), _pattern(""), _replacement(" * @return true if successful, false if failure */ bool -Pattern::init(const String &pattern, const String &replacenemt) +Pattern::init(const String &pattern, const String &replacement) { pcreFree(); _pattern.assign(pattern); - _replacement.assign(replacenemt); + _replacement.assign(replacement); _tokenCount = 0; if (!compile()) { - PrefetchDebug("failed to initialize pattern:'%s', replacement:'%s'", pattern.c_str(), replacenemt.c_str()); + PrefetchDebug("failed to initialize pattern:'%s', replacement:'%s'", pattern.c_str(), replacement.c_str()); pcreFree(); return false; } @@ -84,7 +84,7 @@ Pattern::init(const String &config) size_t next = 1; do { current = next + 1; - next = config.find_first_of("/", current); + next = config.find_first_of('/', current); } while (next != String::npos && '\\' == config[next - 1]); if (next != String::npos) { @@ -98,7 +98,7 @@ Pattern::init(const String &config) start = next + 1; do { current = next + 1; - next = config.find_first_of("/", current); + next = config.find_first_of('/', current); } while (next != String::npos && '\\' == config[next - 1]); if (next != String::npos) { @@ -150,7 +150,7 @@ Pattern::pcreFree() } /** - * @bried Destructor, frees PCRE related resources. + * @brief Destructor, frees PCRE related resources. */ Pattern::~Pattern() { @@ -393,8 +393,8 @@ Pattern::compile() */ MultiPattern::~MultiPattern() { - for (std::vector::iterator p = this->_list.begin(); p != this->_list.end(); ++p) { - delete (*p); + for (auto &p : this->_list) { + delete p; } } @@ -428,8 +428,8 @@ MultiPattern::add(Pattern *pattern) bool MultiPattern::match(const String &subject) const { - for (std::vector::const_iterator p = this->_list.begin(); p != this->_list.end(); ++p) { - if (nullptr != (*p) && (*p)->match(subject)) { + for (auto p : this->_list) { + if (nullptr != p && p->match(subject)) { return true; } } @@ -445,8 +445,8 @@ MultiPattern::match(const String &subject) const bool MultiPattern::replace(const String &subject, String &result) const { - for (std::vector::const_iterator p = this->_list.begin(); p != this->_list.end(); ++p) { - if (nullptr != (*p) && (*p)->replace(subject, result)) { + for (auto p : this->_list) { + if (nullptr != p && p->replace(subject, result)) { return true; } } diff --git a/plugins/experimental/prefetch/pattern.h b/plugins/experimental/prefetch/pattern.h index 2db76656c77..6d7c5aa5b29 100644 --- a/plugins/experimental/prefetch/pattern.h +++ b/plugins/experimental/prefetch/pattern.h @@ -43,7 +43,7 @@ class Pattern Pattern(); virtual ~Pattern(); - bool init(const String &pattern, const String &replacenemt); + bool init(const String &pattern, const String &replacement); bool init(const String &config); bool empty() const; bool match(const String &subject); @@ -56,13 +56,13 @@ class Pattern bool failed(const String &subject) const; void pcreFree(); - pcre *_re; /**< @brief PCRE compiled info structure, computed during initialization */ - pcre_extra *_extra; /**< @brief PCRE study data block, computed during initialization */ + pcre *_re = nullptr; /**< @brief PCRE compiled info structure, computed during initialization */ + pcre_extra *_extra = nullptr; /**< @brief PCRE study data block, computed during initialization */ String _pattern; /**< @brief PCRE pattern string, containing PCRE patterns and capturing groups. */ String _replacement; /**< @brief PCRE replacement string, containing $0..$9 to be replaced with content of the capturing groups */ - int _tokenCount; /**< @brief number of replacements $0..$9 found in the replacement string if not empty */ + int _tokenCount = 0; /**< @brief number of replacements $0..$9 found in the replacement string if not empty */ int _tokens[TOKENCOUNT]; /**< @brief replacement index 0..9, since they can be used in the replacement string in any order */ int _tokenOffset[TOKENCOUNT]; /**< @brief replacement offset inside the replacement string */ }; @@ -73,7 +73,7 @@ class Pattern class MultiPattern { public: - MultiPattern(const String name = "") : _name(name) {} + MultiPattern(const String &name = "") : _name(name) {} virtual ~MultiPattern(); bool empty() const; diff --git a/plugins/experimental/prefetch/plugin.cc b/plugins/experimental/prefetch/plugin.cc index 1c16ae082b3..cef738cdc93 100644 --- a/plugins/experimental/prefetch/plugin.cc +++ b/plugins/experimental/prefetch/plugin.cc @@ -121,7 +121,7 @@ TSRemapInit(TSRemapInterface *apiInfo, char *errBuf, int erroBufSize) */ struct PrefetchInstance { - PrefetchInstance() : _state(nullptr){}; + PrefetchInstance(){}; private: PrefetchInstance(PrefetchInstance const &); @@ -129,7 +129,7 @@ struct PrefetchInstance { public: PrefetchConfig _config; - BgFetchState *_state; + BgFetchState *_state = nullptr; }; /** @@ -195,14 +195,14 @@ evaluate(const String &v) /* Find out if width is specified (hence leading zeros are required if the width is bigger then the result width) */ String stmt; size_t len = 0; - size_t pos = v.find_first_of(":"); + size_t pos = v.find_first_of(':'); if (String::npos != pos) { stmt.assign(v.substr(0, pos)); len = getValue(v.substr(pos + 1)); } else { stmt.assign(v); } - PrefetchDebug("statement: '%s', formating length: %zu", stmt.c_str(), len); + PrefetchDebug("statement: '%s', formatting length: %zu", stmt.c_str(), len); int result = 0; pos = stmt.find_first_of("+-"); @@ -237,8 +237,8 @@ expand(String &s) { size_t cur = 0; while (String::npos != cur) { - size_t start = s.find_first_of("{", cur); - size_t stop = s.find_first_of("}", start); + size_t start = s.find_first_of('{', cur); + size_t stop = s.find_first_of('}', start); if (String::npos != start && String::npos != stop) { s.replace(start, stop - start + 1, evaluate(s.substr(start + 1, stop - start - 1))); @@ -285,7 +285,7 @@ appendCacheKey(const TSHttpTxn txnp, const TSMBuffer reqBuffer, String &key) /** * @brief Find out if the object was found fresh in cache. * - * This function finaly controls if the pre-fetch should be scheduled or not. + * This function finely controls if the pre-fetch should be scheduled or not. * @param txnp HTTP transaction structure * @return true - hit fresh, false - miss/stale/skipped or error */ @@ -476,7 +476,7 @@ contHandleFetch(const TSCont contp, TSEvent event, void *edata) /* first-pass */ if (!config.isExactMatch()) { data->_fetchable = state->acquire(data->_cachekey); - PrefetchDebug("request is%sfetchable", data->_fetchable ? " " : " not "); + PrefetchDebug("request is %s fetchable", data->_fetchable ? " " : " not "); } } } @@ -489,7 +489,7 @@ contHandleFetch(const TSCont contp, TSEvent event, void *edata) /* second-pass */ data->_fetchable = state->acquire(data->_cachekey); data->_fetchable = data->_fetchable && state->uniqueAcquire(data->_cachekey); - PrefetchDebug("request is%sfetchable", data->_fetchable ? " " : " not "); + PrefetchDebug("request is %s fetchable", data->_fetchable ? " " : " not "); if (isFetchable(txnp, data)) { if (!data->_fetchable) { diff --git a/plugins/experimental/remap_purge/remap_purge.c b/plugins/experimental/remap_purge/remap_purge.c index a8ce00fb2ce..3027543563e 100644 --- a/plugins/experimental/remap_purge/remap_purge.c +++ b/plugins/experimental/remap_purge/remap_purge.c @@ -105,10 +105,10 @@ delete_purge_instance(PurgeInstance *purge) } } -/* This is where we start the PURGE events, setting up the transactino to fail, +/* This is where we start the PURGE events, setting up the transaction to fail, and bump the generation ID, and finally save the state. */ -static int -on_http_cache_lookup_complete(TSHttpTxn txnp, TSCont contp, PurgeInstance *purge) +static void +update_purge_state(PurgeInstance *purge) { FILE *file; @@ -126,14 +126,11 @@ on_http_cache_lookup_complete(TSHttpTxn txnp, TSCont contp, PurgeInstance *purge } TSMutexUnlock(purge->lock); - - TSHttpTxnReenable(txnp, TS_EVENT_HTTP_ERROR); - return TS_SUCCESS; } /* Before we can send the response, we want to modify it to a "200 OK" again, and produce some reasonable body output. */ -static int +static TSReturnCode on_send_response_header(TSHttpTxn txnp, TSCont contp, PurgeInstance *purge) { TSMBuffer bufp; @@ -153,6 +150,7 @@ on_send_response_header(TSHttpTxn txnp, TSCont contp, PurgeInstance *purge) } else { TSHttpTxnReenable(txnp, TS_EVENT_HTTP_ERROR); } + TSContDestroy(contp); return TS_SUCCESS; } @@ -170,10 +168,6 @@ purge_cont(TSCont contp, TSEvent event, void *edata) return on_send_response_header(txnp, contp, purge); break; - case TS_EVENT_HTTP_CACHE_LOOKUP_COMPLETE: - return on_http_cache_lookup_complete(txnp, contp, purge); - break; - default: TSDebug(PLUGIN_NAME, "Unexpected event: %d", event); break; @@ -219,11 +213,13 @@ handle_purge(TSHttpTxn txnp, PurgeInstance *purge) if (path && (path_len >= purge->secret_len)) { int s_path = path_len - 1; - while ((s_path >= 0) && ('/' != path[s_path])) { /* No memrchr in OSX */ + /* Find the last /, essentially memrchr (which does not exist on macOS) */ + while ((s_path >= 0) && ('/' != path[s_path])) { --s_path; } - if (!memcmp(s_path > 0 ? path + s_path + 1 : path, purge->secret, purge->secret_len)) { + if (((path_len - s_path - 1) == purge->secret_len) && + !memcmp(s_path > 0 ? path + s_path + 1 : path, purge->secret, purge->secret_len)) { should_purge = true; } } @@ -238,10 +234,12 @@ handle_purge(TSHttpTxn txnp, PurgeInstance *purge) if (should_purge) { TSCont cont = TSContCreate(purge_cont, TSMutexCreate()); + TSDebug(PLUGIN_NAME, "Setting up continuation for PURGE operation"); TSContDataSet(cont, purge); - TSHttpTxnHookAdd(txnp, TS_HTTP_CACHE_LOOKUP_COMPLETE_HOOK, cont); TSHttpTxnHookAdd(txnp, TS_HTTP_SEND_RESPONSE_HDR_HOOK, cont); + update_purge_state(purge); } else if (purge->gen_id > 0) { + TSDebug(PLUGIN_NAME, "Setting request gen_id to %" PRId64, purge->gen_id); TSHttpTxnConfigIntSet(txnp, TS_CONFIG_HTTP_CACHE_GENERATION, purge->gen_id); } } diff --git a/plugins/experimental/remap_stats/remap_stats.c b/plugins/experimental/remap_stats/remap_stats.c index 240b2cff4f3..b5a98928ec2 100644 --- a/plugins/experimental/remap_stats/remap_stats.c +++ b/plugins/experimental/remap_stats/remap_stats.c @@ -51,6 +51,7 @@ stat_add(char *name, TSMgmtInt amount, TSStatPersistence persist_type, TSMutex c static __thread bool hash_init = false; if (unlikely(!hash_init)) { + // NOLINTNEXTLINE hcreate_r(TS_MAX_API_STATS << 1, &stat_cache); hash_init = true; TSDebug(DEBUG_TAG, "stat cache hash init"); @@ -58,6 +59,7 @@ stat_add(char *name, TSMgmtInt amount, TSStatPersistence persist_type, TSMutex c search.key = name; search.data = 0; + // NOLINTNEXTLINE hsearch_r(search, FIND, &result, &stat_cache); if (unlikely(result == NULL)) { @@ -78,6 +80,7 @@ stat_add(char *name, TSMgmtInt amount, TSStatPersistence persist_type, TSMutex c if (stat_id >= 0) { search.key = TSstrdup(name); search.data = (void *)((intptr_t)stat_id); + // NOLINTNEXTLINE hsearch_r(search, ENTER, &result, &stat_cache); TSDebug(DEBUG_TAG, "Cached stat_name: %s stat_id: %d", name, stat_id); } diff --git a/plugins/experimental/slice/Config.cc b/plugins/experimental/slice/Config.cc new file mode 100644 index 00000000000..43d1a7fc2d0 --- /dev/null +++ b/plugins/experimental/slice/Config.cc @@ -0,0 +1,193 @@ +/** @file + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#include "Config.h" + +#include +#include +#include +#include +#include + +#include "ts/experimental.h" + +int64_t +Config::bytesFrom(char const *const valstr) +{ + char *endptr = nullptr; + int64_t blockbytes = strtoll(valstr, &endptr, 10); + + if (nullptr != endptr && valstr < endptr) { + size_t const dist = endptr - valstr; + if (dist < strlen(valstr) && 0 <= blockbytes) { + switch (tolower(*endptr)) { + case 'g': + blockbytes *= ((int64_t)1024 * (int64_t)1024 * (int64_t)1024); + break; + case 'm': + blockbytes *= ((int64_t)1024 * (int64_t)1024); + break; + case 'k': + blockbytes *= (int64_t)1024; + break; + default: + break; + } + } + } + + if (blockbytes < 0) { + blockbytes = 0; + } + + return blockbytes; +} + +bool +Config::fromArgs(int const argc, char const *const argv[]) +{ + DEBUG_LOG("Number of arguments: %d", argc); + for (int index = 0; index < argc; ++index) { + DEBUG_LOG("args[%d] = %s", index, argv[index]); + } + + // current "best" blockbytes from configuration + int64_t blockbytes = 0; + + // backwards compat: look for blockbytes + for (int index = 0; index < argc; ++index) { + std::string_view const argstr = argv[index]; + + std::size_t const spos = argstr.find_first_of(':'); + if (spos != std::string_view::npos) { + std::string_view const key = argstr.substr(0, spos); + std::string_view const val = argstr.substr(spos + 1); + + if (!key.empty() && !val.empty()) { + char const *const valstr = val.data(); // inherits argv's null + int64_t const bytesread = bytesFrom(valstr); + + if (blockbytesmin <= bytesread && bytesread <= blockbytesmax) { + DEBUG_LOG("Found deprecated blockbytes %" PRId64, bytesread); + blockbytes = bytesread; + } + } + } + } + + // standard parsing + constexpr const struct option longopts[] = { + {const_cast("blockbytes"), required_argument, nullptr, 'b'}, + {const_cast("test-blockbytes"), required_argument, nullptr, 't'}, + {const_cast("pace-errorlog"), required_argument, nullptr, 'p'}, + {const_cast("disable-errorlog"), no_argument, nullptr, 'd'}, + {nullptr, 0, nullptr, 0}, + }; + + // getopt assumes args start at '1' so this hack is needed + char *const *argvp = ((char *const *)argv - 1); + + for (;;) { + int const opt = getopt_long(argc + 1, argvp, "b:t:p:d", longopts, nullptr); + if (-1 == opt) { + break; + } + + DEBUG_LOG("processing '%c' %s", (char)opt, argvp[optind - 1]); + + switch (opt) { + case 'b': { + int64_t const bytesread = bytesFrom(optarg); + if (blockbytesmin <= bytesread && bytesread <= blockbytesmax) { + DEBUG_LOG("Using blockbytes %" PRId64, bytesread); + blockbytes = bytesread; + } else { + ERROR_LOG("Invalid blockbytes: %s", optarg); + } + } break; + case 't': + if (0 == blockbytes) { + int64_t const bytesread = bytesFrom(optarg); + if (0 < bytesread) { + DEBUG_LOG("Using blockbytestest %" PRId64, bytesread); + blockbytes = bytesread; + } else { + ERROR_LOG("Invalid blockbytestest: %s", optarg); + } + } else { + DEBUG_LOG("Skipping blockbytestest in favor of blockbytes"); + } + break; + case 'p': { + int const secsread = atoi(optarg); + if (0 < secsread) { + m_paceerrsecs = std::min(secsread, 60); + } else { + DEBUG_LOG("Ignoring pace-errlog argument"); + } + } break; + case 'd': + m_paceerrsecs = -1; + break; + default: + break; + } + } + + if (0 < blockbytes) { + DEBUG_LOG("Using configured blockbytes %" PRId64, blockbytes); + m_blockbytes = blockbytes; + } else { + DEBUG_LOG("Using default blockbytes %" PRId64, m_blockbytes); + } + + if (m_paceerrsecs < 0) { + DEBUG_LOG("Block stitching error logs disabled"); + } else if (0 == m_paceerrsecs) { + DEBUG_LOG("Block stitching error logs enabled"); + } else { + DEBUG_LOG("Block stitching error logs at most every %d sec(s)", m_paceerrsecs); + } + + return true; +} + +bool +Config::canLogError() +{ + std::lock_guard const guard(m_mutex); + + if (m_paceerrsecs < 0) { + return false; + } else if (0 == m_paceerrsecs) { + return true; + } + +#if !defined(UNITTEST) + TSHRTime const timenow = TShrtime(); + if (timenow < m_nextlogtime) { + return false; + } + + m_nextlogtime = timenow + TS_HRTIME_SECONDS(m_paceerrsecs); +#else + m_nextlogtime = 0; // thanks clang +#endif + + return true; +} diff --git a/plugins/experimental/slice/Config.h b/plugins/experimental/slice/Config.h new file mode 100644 index 00000000000..8c4ab2498fa --- /dev/null +++ b/plugins/experimental/slice/Config.h @@ -0,0 +1,46 @@ +/** @file + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#pragma once + +#include "slice.h" + +#include + +// Data Structures and Classes +struct Config { + static constexpr int64_t const blockbytesmin = 1024 * 256; // 256KB + static constexpr int64_t const blockbytesmax = 1024 * 1024 * 32; // 32MB + static constexpr int64_t const blockbytesdefault = 1024 * 1024; // 1MB + + int64_t m_blockbytes{blockbytesdefault}; + int m_paceerrsecs{0}; // -1 disable logging, 0 no pacing, max 60s + + // Convert optarg to bytes + static int64_t bytesFrom(char const *const valstr); + + // Parse from args, ast one wins + bool fromArgs(int const argc, char const *const argv[]); + + // Check if the error should can be logged, if sucessful may update m_nexttime + bool canLogError(); + +private: + TSHRTime m_nextlogtime{0}; // next time to log in ns + std::mutex m_mutex; +}; diff --git a/plugins/experimental/slice/ContentRange.cc b/plugins/experimental/slice/ContentRange.cc new file mode 100644 index 00000000000..c9d9edde3de --- /dev/null +++ b/plugins/experimental/slice/ContentRange.cc @@ -0,0 +1,55 @@ +/** @file + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#include "ContentRange.h" + +#include +#include + +static char const *const format = "bytes %" PRId64 "-%" PRId64 "/%" PRId64; + +bool +ContentRange::fromStringClosed(char const *const valstr) +{ + int const fields = sscanf(valstr, format, &m_beg, &m_end, &m_length); + + if (3 == fields && m_beg <= m_end) { + m_end += 1; + } else { + m_beg = m_end = m_length = -1; + } + + return isValid(); +} + +bool +ContentRange::toStringClosed(char *const rangestr, int *const rangelen) const +{ + if (!isValid()) { + if (0 < *rangelen) { + rangestr[0] = '\0'; + } + *rangelen = 0; + return false; + } + + int const lenin = *rangelen; + *rangelen = snprintf(rangestr, lenin, format, m_beg, (m_end - 1), m_length); + + return (0 < *rangelen && *rangelen < lenin); +} diff --git a/plugins/experimental/slice/ContentRange.h b/plugins/experimental/slice/ContentRange.h new file mode 100644 index 00000000000..87de6316c87 --- /dev/null +++ b/plugins/experimental/slice/ContentRange.h @@ -0,0 +1,53 @@ +/** @file + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#pragma once + +#include "ts/ts.h" + +/** + represents value parsed from a blocked Content-Range reponse header field. + Range is converted from closed range into a half open range for. + */ +struct ContentRange { + int64_t m_beg = -1; + int64_t m_end = -1; // half open + int64_t m_length = -1; // full content length + + ContentRange() {} + explicit ContentRange(int64_t const begin, int64_t const end, int64_t const len) : m_beg(begin), m_end(end), m_length(len) {} + bool + isValid() const + { + return 0 <= m_beg && m_beg < m_end && m_end <= m_length; + } + + /** parsed from a Content-Range field + */ + bool fromStringClosed(char const *const valstr); + + /** usable for Content-Range field + */ + bool toStringClosed(char *const rangestr, int *const rangelen) const; + + int64_t + rangeSize() const + { + return m_end - m_beg; + } +}; diff --git a/plugins/experimental/slice/Data.cc b/plugins/experimental/slice/Data.cc new file mode 100644 index 00000000000..aac6943d47b --- /dev/null +++ b/plugins/experimental/slice/Data.cc @@ -0,0 +1,65 @@ +/** @file + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#include "Data.h" + +#include +#include +#include +#include +#include + +namespace +{ +std::mutex mutex; +int64_t inplay = 0; +std::unique_ptr thread; +} // namespace + +void +monitor() +{ + std::lock_guard guard(mutex); + // while (0 < inplay) + while (true) { + mutex.unlock(); + std::this_thread::sleep_for(std::chrono::seconds(10)); + std::cerr << "Inplay: " << inplay << std::endl; + mutex.lock(); + } + // thread.release(); +} + +void +incrData() +{ + std::lock_guard const guard(mutex); + if (!thread) { + thread.reset(new std::thread(monitor)); + } + + ++inplay; +} + +void +decrData() +{ + std::lock_guard const guard(mutex); + --inplay; + assert(0 <= inplay); +} diff --git a/plugins/experimental/slice/Data.h b/plugins/experimental/slice/Data.h new file mode 100644 index 00000000000..45383522103 --- /dev/null +++ b/plugins/experimental/slice/Data.h @@ -0,0 +1,126 @@ +/** @file + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#pragma once + +#include "ts/ts.h" + +#include "Config.h" +#include "HttpHeader.h" +#include "Range.h" +#include "Stage.h" + +#include + +void incrData(); + +void decrData(); + +struct Data { + Data(Data const &) = delete; + Data &operator=(Data const &) = delete; + + Config *const m_config; + + sockaddr_storage m_client_ip; + + // for pristine url coming in + TSMBuffer m_urlbuffer{nullptr}; + TSMLoc m_urlloc{nullptr}; + + char m_hostname[8192]; + int m_hostlen; + char m_etag[8192]; + int m_etaglen; + char m_lastmodified[8192]; + int m_lastmodifiedlen; + + TSHttpStatus m_statustype; // 200 or 206 + + bool m_bail; // non 206/200 response + + Range m_req_range; // converted to half open interval + int64_t m_contentlen; + + int64_t m_blocknum; // block number to work on, -1 bad/stop + int64_t m_blockexpected; // body bytes expected + int64_t m_blockskip; // number of bytes to skip in this block + int64_t m_blockconsumed; // body bytes consumed + bool m_iseos; // server in EOS state + + int64_t m_bytestosend; // header + content bytes to send + int64_t m_bytessent; // number of bytes written to the client + + bool m_server_block_header_parsed; + bool m_server_first_header_parsed; + + Stage m_upstream; + Stage m_dnstream; + + HdrMgr m_req_hdrmgr; // manager for server request + HdrMgr m_resp_hdrmgr; // manager for client response + + TSHttpParser m_http_parser{nullptr}; //!< cached for reuse + + explicit Data(Config *const config) + : m_config(config), + m_client_ip(), + m_urlbuffer(nullptr), + m_urlloc(nullptr), + m_hostlen(0), + m_etaglen(0), + m_lastmodifiedlen(0), + m_statustype(TS_HTTP_STATUS_NONE), + m_bail(false), + m_req_range(-1, -1), + m_contentlen(-1) + + , + m_blocknum(-1), + m_blockexpected(0), + m_blockskip(0), + m_blockconsumed(0), + m_iseos(false) + + , + m_bytestosend(0), + m_bytessent(0), + m_server_block_header_parsed(false), + m_server_first_header_parsed(false), + m_http_parser(nullptr) + { + // incrData(); + m_hostname[0] = '\0'; + m_lastmodified[0] = '\0'; + m_etag[0] = '\0'; + } + + ~Data() + { + // decrData(); + if (nullptr != m_urlbuffer) { + if (nullptr != m_urlloc) { + TSHandleMLocRelease(m_urlbuffer, TS_NULL_MLOC, m_urlloc); + } + TSMBufferDestroy(m_urlbuffer); + } + if (nullptr != m_http_parser) { + TSHttpParserDestroy(m_http_parser); + } + } +}; diff --git a/plugins/experimental/slice/HttpHeader.cc b/plugins/experimental/slice/HttpHeader.cc new file mode 100644 index 00000000000..bcf649b761a --- /dev/null +++ b/plugins/experimental/slice/HttpHeader.cc @@ -0,0 +1,356 @@ +/** @file + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#include "HttpHeader.h" + +#include "slice.h" + +#include +#include + +TSHttpType +HttpHeader::type() const +{ + if (isValid()) { + return TSHttpHdrTypeGet(m_buffer, m_lochdr); + } else { + return TS_HTTP_TYPE_UNKNOWN; + } +} + +TSHttpStatus +HttpHeader::status() const +{ + TSHttpStatus res = TS_HTTP_STATUS_NONE; + if (isValid()) { + res = TSHttpHdrStatusGet(m_buffer, m_lochdr); + } + return res; +} + +bool +HttpHeader::setStatus(TSHttpStatus const newstatus) +{ + if (!isValid()) { + return false; + } + + return TS_SUCCESS == TSHttpHdrStatusSet(m_buffer, m_lochdr, newstatus); +} + +char * +HttpHeader ::urlString(int *const urllen) const +{ + char *urlstr = nullptr; + TSAssert(nullptr != urllen); + + TSMLoc locurl = nullptr; + TSReturnCode const rcode = TSHttpHdrUrlGet(m_buffer, m_lochdr, &locurl); + if (TS_SUCCESS == rcode && nullptr != locurl) { + urlstr = TSUrlStringGet(m_buffer, locurl, urllen); + TSHandleMLocRelease(m_buffer, m_lochdr, locurl); + } else { + *urllen = 0; + } + + return urlstr; +} + +bool +HttpHeader::setUrl(TSMBuffer const bufurl, TSMLoc const locurl) +{ + if (!isValid()) { + return false; + } + + TSMLoc locurlout; + TSReturnCode rcode = TSHttpHdrUrlGet(m_buffer, m_lochdr, &locurlout); + if (TS_SUCCESS != rcode) { + return false; + } + + // copy the url + rcode = TSUrlCopy(m_buffer, locurlout, bufurl, locurl); + + // set url active + if (TS_SUCCESS == rcode) { + rcode = TSHttpHdrUrlSet(m_buffer, m_lochdr, locurlout); + } + + TSHandleMLocRelease(m_buffer, TS_NULL_MLOC, locurlout); + + return TS_SUCCESS == rcode; +} + +bool +HttpHeader::setReason(char const *const valstr, int const vallen) +{ + if (isValid()) { + return TS_SUCCESS == TSHttpHdrReasonSet(m_buffer, m_lochdr, valstr, vallen); + } else { + return false; + } +} + +char const * +HttpHeader::getCharPtr(CharPtrGetFunc func, int *const len) const +{ + char const *res = nullptr; + if (isValid()) { + int reslen = 0; + res = func(m_buffer, m_lochdr, &reslen); + + if (nullptr != len) { + *len = reslen; + } + } + + if (nullptr == res && nullptr != len) { + *len = 0; + } + + return res; +} + +bool +HttpHeader::hasKey(char const *const key, int const keylen) const +{ + if (!isValid()) { + return false; + } + + TSMLoc const locfield(TSMimeHdrFieldFind(m_buffer, m_lochdr, key, keylen)); + if (nullptr != locfield) { + TSHandleMLocRelease(m_buffer, m_lochdr, locfield); + return true; + } + + return false; +} + +bool +HttpHeader::removeKey(char const *const keystr, int const keylen) +{ + if (!isValid()) { + return false; + } + + bool status = true; + + TSMLoc const locfield = TSMimeHdrFieldFind(m_buffer, m_lochdr, keystr, keylen); + if (nullptr != locfield) { + int const rcode = TSMimeHdrFieldRemove(m_buffer, m_lochdr, locfield); + status = (TS_SUCCESS == rcode); + TSHandleMLocRelease(m_buffer, m_lochdr, locfield); + } + + return status; +} + +bool +HttpHeader::valueForKey(char const *const keystr, int const keylen, char *const valstr, int *const vallen, int const index) const +{ + if (!isValid()) { + *vallen = 0; + return false; + } + + bool status = false; + + TSMLoc const locfield = TSMimeHdrFieldFind(m_buffer, m_lochdr, keystr, keylen); + + if (nullptr != locfield) { + int getlen = 0; + char const *const getstr = TSMimeHdrFieldValueStringGet(m_buffer, m_lochdr, locfield, index, &getlen); + + int const valcap = *vallen; + if (nullptr != getstr && 0 < getlen && getlen < (valcap - 1)) { + char *const endp = stpncpy(valstr, getstr, getlen); + + *vallen = endp - valstr; + status = (*vallen < valcap); + + if (status) { + *endp = '\0'; + } + } + TSHandleMLocRelease(m_buffer, m_lochdr, locfield); + } else { + *vallen = 0; + } + + return status; +} + +bool +HttpHeader::setKeyVal(char const *const keystr, int const keylen, char const *const valstr, int const vallen, int const index) +{ + if (!isValid()) { + return false; + } + + bool status(false); + + TSMLoc locfield(TSMimeHdrFieldFind(m_buffer, m_lochdr, keystr, keylen)); + + if (nullptr != locfield) { + status = TS_SUCCESS == TSMimeHdrFieldValueStringSet(m_buffer, m_lochdr, locfield, index, valstr, vallen); + } else { + int rcode = TSMimeHdrFieldCreateNamed(m_buffer, m_lochdr, keystr, keylen, &locfield); + + if (TS_SUCCESS == rcode) { + rcode = TSMimeHdrFieldValueStringSet(m_buffer, m_lochdr, locfield, index, valstr, vallen); + if (TS_SUCCESS == rcode) { + rcode = TSMimeHdrFieldAppend(m_buffer, m_lochdr, locfield); + status = (TS_SUCCESS == rcode); + } + } + } + + if (nullptr != locfield) { + TSHandleMLocRelease(m_buffer, m_lochdr, locfield); + } + + return status; +} + +std::string +HttpHeader::toString() const +{ + std::string res; + + if (!isValid()) { + return ""; + } + + TSHttpType const htype(type()); + + switch (htype) { + case TS_HTTP_TYPE_REQUEST: { + res.append(method()); + + int urllen = 0; + char *const urlstr = urlString(&urllen); + if (nullptr != urlstr) { + res.append(" "); + res.append(urlstr, urllen); + TSfree(urlstr); + } else { + res.append(" UnknownURL"); + } + + res.append(" HTTP/unparsed"); + } break; + + case TS_HTTP_TYPE_RESPONSE: { + char bufstr[1024]; + /* + int const version = TSHttpHdrVersionGet(m_buffer, m_lochdr); + snprintf(bufstr, 1023, "%d ", version); + res.append(bufstr); + */ + res.append("HTTP/unparsed"); + + int const status = TSHttpHdrStatusGet(m_buffer, m_lochdr); + snprintf(bufstr, 1023, " %d ", status); + res.append(bufstr); + + int reasonlen = 0; + char const *const hreason = reason(&reasonlen); + + res.append(hreason, reasonlen); + } break; + + default: + case TS_HTTP_TYPE_UNKNOWN: + res.append("UNKNOWN"); + break; + } + + res.append("\r\n"); + + int const numhdrs = TSMimeHdrFieldsCount(m_buffer, m_lochdr); + + for (int indexhdr = 0; indexhdr < numhdrs; ++indexhdr) { + TSMLoc const locfield = TSMimeHdrFieldGet(m_buffer, m_lochdr, indexhdr); + + int keylen = 0; + char const *const keystr = TSMimeHdrFieldNameGet(m_buffer, m_lochdr, locfield, &keylen); + + res.append(keystr, keylen); + res.append(": "); + int vallen = 0; + char const *const valstr = TSMimeHdrFieldValueStringGet(m_buffer, m_lochdr, locfield, -1, &vallen); + + res.append(valstr, vallen); + res.append("\r\n"); + + TSHandleMLocRelease(m_buffer, m_lochdr, locfield); + } + + res.append("\r\n"); + + return res; +} + +/////// HdrMgr + +TSParseResult +HdrMgr::populateFrom(TSHttpParser const http_parser, TSIOBufferReader const reader, HeaderParseFunc const parsefunc) +{ + TSParseResult parse_res = TS_PARSE_CONT; + + if (nullptr == m_buffer) { + m_buffer = TSMBufferCreate(); + } + if (nullptr == m_lochdr) { + m_lochdr = TSHttpHdrCreate(m_buffer); + } + + int64_t read_avail = TSIOBufferReaderAvail(reader); + if (0 < read_avail) { + TSIOBufferBlock block = TSIOBufferReaderStart(reader); + int64_t consumed = 0; + + parse_res = TS_PARSE_CONT; + + while (nullptr != block && 0 < read_avail) { + int64_t blockbytes = 0; + char const *const bstart = TSIOBufferBlockReadStart(block, reader, &blockbytes); + + char const *ptr = bstart; + char const *endptr = ptr + blockbytes; + + parse_res = parsefunc(http_parser, m_buffer, m_lochdr, &ptr, endptr); + + int64_t const bytes_parsed(ptr - bstart); + + consumed += bytes_parsed; + read_avail -= bytes_parsed; + + if (TS_PARSE_CONT == parse_res) { + block = TSIOBufferBlockNext(block); + } else { + break; + } + } + TSIOBufferReaderConsume(reader, consumed); + } + + return parse_res; +} diff --git a/plugins/experimental/slice/HttpHeader.h b/plugins/experimental/slice/HttpHeader.h new file mode 100644 index 00000000000..bfd9c4d6f4e --- /dev/null +++ b/plugins/experimental/slice/HttpHeader.h @@ -0,0 +1,215 @@ +/** @file + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#pragma once + +/** + An ATS Http header exists in a marshall buffer at a given location. + Unfortunately how that marshall buffer is created and how that + location is determined depends on where those buffers came from. + + A TSHttpTxn manages the buffer itself and creates a location which + has to be managed. + + A TSHttpParsed populates a created buffer that has had TSHttpHdrCreate + run against it which creates a location against it. End users + need to manage the created buffer, the location and invoke + TSHttpHdrDestroy. +*/ + +#include "ts/ts.h" + +#include + +static char const *const SLICER_MIME_FIELD_INFO = "X-Slicer-Info"; + +/** + Designed to be a cheap throwaway struct which allows a + consumer to make various calls to manipulate headers. +*/ +struct HttpHeader { + TSMBuffer const m_buffer; + TSMLoc const m_lochdr; + + explicit HttpHeader(TSMBuffer buffer, TSMLoc lochdr) : m_buffer(buffer), m_lochdr(lochdr) {} + bool + isValid() const + { + return nullptr != m_buffer && nullptr != m_lochdr; + } + + // TS_HTTP_TYPE_UNKNOWN, TS_HTTP_TYPE_REQUEST, TS_HTTP_TYPE_RESPONSE + TSHttpType type() const; + + TSHttpStatus status() const; + + bool setStatus(TSHttpStatus const newstatus); + + bool setUrl(TSMBuffer const bufurl, TSMLoc const locurl); + + typedef char const *(*CharPtrGetFunc)(TSMBuffer, TSMLoc, int *); + + // request method TS_HTTP_METHOD_* + char const * + method(int *const len = nullptr) const + { + return getCharPtr(TSHttpHdrMethodGet, len); + } + + // request method version + int + version() const + { + return TSHttpHdrVersionGet(m_buffer, m_lochdr); + } + + // Returns string representation of the url. Caller gets ownership! + char *urlString(int *const urllen) const; + + // host + char const * + hostname(int *const len) const + { + return getCharPtr(TSHttpHdrHostGet, len); + } + + // response reason + char const * + reason(int *const len) const + { + return getCharPtr(TSHttpHdrReasonGet, len); + } + + bool setReason(char const *const valstr, int const vallen); + + bool hasKey(char const *const key, int const keylen) const; + + // returns false if header invalid or something went wrong with removal. + bool removeKey(char const *const key, int const keylen); + + bool valueForKey(char const *const keystr, int const keylen, + char *const valstr, // <-- return string value + int *const vallen, // <-- pass in capacity, returns len of string + int const index = -1 // retrieves all values + ) const; + + /** + Sets or adds a key/value + */ + bool setKeyVal(char const *const key, int const keylen, char const *const val, int const vallen, + int const index = -1 // sets all values + ); + + /** dump header into provided char buffer + */ + std::string toString() const; + +private: + /** + To be used with + TSHttpHdrMethodGet + TSHttpHdrHostGet + TSHttpHdrReasonGet + */ + char const *getCharPtr(CharPtrGetFunc func, int *const len) const; +}; + +struct TxnHdrMgr { + TxnHdrMgr(TxnHdrMgr const &) = delete; + TxnHdrMgr &operator=(TxnHdrMgr const &) = delete; + + TSMBuffer m_buffer{nullptr}; + TSMLoc m_lochdr{nullptr}; + + TxnHdrMgr() : m_buffer(nullptr), m_lochdr(nullptr) {} + ~TxnHdrMgr() + { + if (nullptr != m_lochdr) { + TSHandleMLocRelease(m_buffer, TS_NULL_MLOC, m_lochdr); + } + } + + typedef TSReturnCode (*HeaderGetFunc)(TSHttpTxn, TSMBuffer *, TSMLoc *); + /** use one of the following: + TSHttpTxnClientReqGet + TSHttpTxnClientRespGet + TSHttpTxnServerReqGet + TSHttpTxnServerRespGet + TSHttpTxnCachedReqGet + TSHttpTxnCachedRespGet + */ + + bool + populateFrom(TSHttpTxn const &txnp, HeaderGetFunc const &func) + { + return TS_SUCCESS == func(txnp, &m_buffer, &m_lochdr); + } + + bool + isValid() const + { + return nullptr != m_lochdr; + } +}; + +struct HdrMgr { + HdrMgr(HdrMgr const &) = delete; + HdrMgr &operator=(HdrMgr const &) = delete; + + TSMBuffer m_buffer{nullptr}; + TSMLoc m_lochdr{nullptr}; + + HdrMgr() : m_buffer(nullptr), m_lochdr(nullptr) {} + ~HdrMgr() + { + if (nullptr != m_buffer) { + if (nullptr != m_lochdr) { + TSHttpHdrDestroy(m_buffer, m_lochdr); + TSHandleMLocRelease(m_buffer, TS_NULL_MLOC, m_lochdr); + } + TSMBufferDestroy(m_buffer); + } + } + + void + resetHeader() + { + if (nullptr != m_buffer && nullptr != m_lochdr) { + TSHttpHdrDestroy(m_buffer, m_lochdr); + TSHandleMLocRelease(m_buffer, TS_NULL_MLOC, m_lochdr); + m_lochdr = nullptr; + } + } + + typedef TSParseResult (*HeaderParseFunc)(TSHttpParser, TSMBuffer, TSMLoc, char const **, char const *); + + /** Clear/create the parser before calling this and don't + use the parser on another header until done with this one. + use one of the following: + TSHttpHdrParseReq + TSHttpHdrParseResp + Call this multiple times if necessary. + */ + TSParseResult populateFrom(TSHttpParser const http_parser, TSIOBufferReader const reader, HeaderParseFunc const parsefunc); + + bool + isValid() const + { + return nullptr != m_lochdr; + } +}; diff --git a/plugins/experimental/slice/Makefile.inc b/plugins/experimental/slice/Makefile.inc new file mode 100644 index 00000000000..dbac02b4940 --- /dev/null +++ b/plugins/experimental/slice/Makefile.inc @@ -0,0 +1,63 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +pkglib_LTLIBRARIES += experimental/slice/slice.la + +experimental_slice_slice_la_SOURCES = \ + experimental/slice/client.cc \ + experimental/slice/client.h \ + experimental/slice/Config.cc \ + experimental/slice/Config.h \ + experimental/slice/ContentRange.cc \ + experimental/slice/ContentRange.h \ + experimental/slice/Data.cc \ + experimental/slice/Data.h \ + experimental/slice/HttpHeader.cc \ + experimental/slice/HttpHeader.h \ + experimental/slice/intercept.cc \ + experimental/slice/intercept.h \ + experimental/slice/Range.cc \ + experimental/slice/Range.h \ + experimental/slice/response.cc \ + experimental/slice/response.h \ + experimental/slice/server.cc \ + experimental/slice/server.h \ + experimental/slice/slice.cc \ + experimental/slice/slice.h \ + experimental/slice/Stage.h \ + experimental/slice/transfer.cc \ + experimental/slice/transfer.h + +check_PROGRAMS += experimental/slice/test_content_range + +experimental_slice_test_content_range_CPPFLAGS = $(AM_CPPFLAGS) -I$(abs_top_srcdir)/tests/include -DUNITTEST +experimental_slice_test_content_range_SOURCES = \ + experimental/slice/unit-tests/test_content_range.cc \ + experimental/slice/ContentRange.cc + +check_PROGRAMS += experimental/slice/test_range + +experimental_slice_test_range_CPPFLAGS = $(AM_CPPFLAGS) -I$(abs_top_srcdir)/tests/include -DUNITTEST +experimental_slice_test_range_SOURCES = \ + experimental/slice/unit-tests/test_range.cc \ + experimental/slice/Range.cc + +check_PROGRAMS += experimental/slice/test_config + +experimental_slice_test_config_CPPFLAGS = $(AM_CPPFLAGS) -I$(abs_top_srcdir)/tests/include -DUNITTEST +experimental_slice_test_config_SOURCES = \ + experimental/slice/unit-tests/test_config.cc \ + experimental/slice/Config.cc diff --git a/plugins/experimental/slice/Makefile.tsxs b/plugins/experimental/slice/Makefile.tsxs new file mode 100644 index 00000000000..0992d8cb27e --- /dev/null +++ b/plugins/experimental/slice/Makefile.tsxs @@ -0,0 +1,66 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +TSXS?=tsxs +PLUGIN=slice + +all: $(PLUGIN).so + +SOURCES = \ + Config.cc \ + ContentRange.cc \ + Data.cc \ + HttpHeader.cc \ + Range.cc \ + client.cc \ + intercept.cc \ + response.cc \ + server.cc \ + slice.cc \ + transfer.cc \ + +HEADERS = \ + Config.h \ + ContentRange.h \ + Data.h \ + HttpHeader.h \ + Range.h \ + Stage.h \ + client.h \ + intercept.h \ + response.h \ + server.h \ + slice.h \ + transfer.h \ + +$(PLUGIN).so: $(SOURCES) $(HEADERS) + $(TSXS) -v -o $(PLUGIN).so $(SOURCES) + +install: all + $(TSXS) -v -o $(PLUGIN).so -i + +TSINCLUDE = $(shell tsxs -q INCLUDEDIR) +TSCXX = $(shell tsxs -q CXX) +TSCXXFLAGS = $(shell tsxs -q CXXFLAGS) +#PREFIX = $(shell tsxs -q PREFIX) +#LIBS = -L$(PREFIX)/lib -latscppapi +#LIBS = $(PREFIX)/lib/libtsutil.la + +slice_test: slice_test.cc ContentRange.cc Range.cc + $(TSCXX) -o $@ $^ $(TSCXXFLAGS) -I$(TSINCLUDE) -DUNITTEST + +clean: + rm -fv *.lo *.so diff --git a/plugins/experimental/slice/README.md b/plugins/experimental/slice/README.md new file mode 100644 index 00000000000..ec3d1bc420f --- /dev/null +++ b/plugins/experimental/slice/README.md @@ -0,0 +1,145 @@ +### Apache Traffic Server - Slicer Plugin + +The purpose of this plugin is to slice full file or range based requests +into deterministic chunks. This allows a large file to be spread across +multiple cache stripes and allows range requests to be satisfied by +stitching these chunks together. + +Deterministic chunks are requested from a parent cache or origin server +using a preconfigured block byte size. + +The plugin is an example of an intercept handler which takes a single +incoming request (range or whole asset), breaks it into a sequence +of block requests and assembles those blocks into a client response. +The plugin uses TSHttpConnect to delegate each block request to +cache_range_requests.so which handles all cache and parent interaction. + +To enable the plugin, specify the plugin library via @plugin at the end +of a remap line as follows (2MB slice in this example): + +``` +map http://ats-cache/ http://parent/ @plugin=slice.so @pparam=--blockbytes=2M @plugin=cache_range_requests.so +``` + +alternatively + +``` +map http://ats-cache/ http://parent/ @plugin=slice.so @pparam=-b @pparam=2M @plugin=cache_range_requests.so +``` + +for global plugins. + +``` +slice.so --blockbytes=2097152 +cache_range_requests.so +``` + +alternatively: + +``` +slice.so -b 2M +cache_range_requests.so +``` + +Options for the slice plugin (typically last one wins): +``` +--blockbytes= (optional) + Slice block size. + Default is 1m or 1048576 bytes. + also -b + Suffix k,m,g supported. + Limited to 32k and 32m inclusive. + For backwards compatibility blockbytes: is also supported. + +--test-blockbytes= (optional) + Slice block size for testing. + also -t + Suffix k,m,g supported. + Limited to any positive number. + Ignored if --blockbytes is provided. + +--pace-errorlog= (optional) + Limit stitching error logs to every 'n' second(s) + Default is to log all errors (no pacing). + also -p + +--disable-errorlog (optional) + Disable writing stitching errors to the error log. + also -d +``` + +**Note**: cache_range_requests **MUST** follow slice.so Put these plugins +at the end of the plugin list + +**Note**: blockbytes is defined in bytes. Postfix for 'K', 'M' and 'G' +may be used. 1048576 (1MB) is the default. + +For testing purposes an unchecked value of "bytesover" is also available. + +Debug output can be enable by setting the debug tag: **slice**. If debug +is enabled all block stitch errors will log to diags.log + +The slice plugin is susceptible to block stitching errors caused by +mismatched blocks. For these cases special detailed error logs are +provided to help with debugging. Below is a sample error log entry:: + +``` +[Apr 19 20:26:13.639] [ET_NET 17] ERROR: [slice] 1555705573.639 reason="Non 206 internal block response" uri="http://localhost:18080/%7Ep.tex/%7Es.50M/%7Eui.20000/" uas="curl/7.29.0" req_range="bytes=1000000-" norm_range="bytes 1000000-52428799/52428800" etag_exp="%221603934496%22" lm_exp="Fri, 19 Apr 2019 18:53:20 GMT" blk_range="21000000-21999999" status_got="400" cr_got="" etag_got="" lm_got="" cc="no-store" via="" +``` + +Current error types logged: +``` + Mismatch block Etag + Mismatch block Last-Modified + Non 206 internal block response + Mismatch/Bad block Content-Range +``` + + +With slice error logs disabled these type errors can typically be detected +by observing crc=ERR_READ_ERROR and pscl=0 in normal logs. + +At the current time only single range requests or the first part of a +multi part range request of the forms: +``` +Range: bytes=- +Range: bytes=- +Range: bytes=- +``` +are supported as multi part range responses are non trivial to implement. +This matches with the cache_range_requests.so plugin capability. +--- + +Important things to note: + +Any first block server response that is not a 206 is passed down to +the client. + +Only the first server response block is used to evaluate any "If-" +headers. Subsequent server slice block requests remove these headers. + +If a client aborts mid transaction the current slice block is completed +to ensure that the block is written to cache. + +The only 416 case this plugin handles itself is if the requested range +is inside the end slice block but past the content length. Otherwise +parents seem to properly issue 416 responses themselves. + +--- + +To manually build the plugin use the "tsxs" executable that installs with +traffic_server. + +Running the following command will build the plugin + +``` +tsxs -v -o slice.so *.cc +``` + +Running the following command will build and install the plugin. +Beware this may crash a running system if the plugin is loaded +and the OS uses memory paging with plugins. + +``` +tsxs -v -i -o slice.so *.cc +``` diff --git a/plugins/experimental/slice/Range.cc b/plugins/experimental/slice/Range.cc new file mode 100644 index 00000000000..c9a9d4dd44a --- /dev/null +++ b/plugins/experimental/slice/Range.cc @@ -0,0 +1,187 @@ +/** @file + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#include "Range.h" +#include "slice.h" + +#include +#include +#include +#include +#include + +bool +Range::isValid() const +{ + return m_beg < m_end && (0 <= m_beg || 0 == m_end); +} + +int64_t +Range::size() const +{ + return m_end - m_beg; +} + +bool +Range::fromStringClosed(char const *const rangestr) +{ + static char const *const BYTESTR = "bytes="; + static size_t const BYTESTRLEN = strlen(BYTESTR); + + m_beg = m_end = -1; // initialize invalid + + // make sure this is in byte units + if (0 != strncmp(BYTESTR, rangestr, BYTESTRLEN)) { + return false; + } + + // advance past any white space + char const *pstr = rangestr + BYTESTRLEN; + while ('\0' != *pstr && isblank(*pstr)) { + ++pstr; + } + + // rip out any whitespace + char rangebuf[1024]; + int const rangelen = sizeof(rangebuf); + char *pbuf = rangebuf; + while ('\0' != *pstr && (pbuf - rangebuf) < rangelen) { + if (!isblank(*pstr)) { + *pbuf++ = *pstr; + } + ++pstr; + } + *pbuf = '\0'; + + int const rlen = (pbuf - rangebuf); + + int consumed = 0; + + // last 'n' bytes - result in range with negative begin and 0 end + int64_t endbytes = 0; + char const *const fmtend = "-%" PRId64 "%n"; + int const fieldsend = sscanf(rangebuf, fmtend, &endbytes, &consumed); + if (1 == fieldsend) { + if (rlen == consumed) { + m_beg = -endbytes; + m_end = 0; + return true; + } else { + return false; + } + } + + // normal range - + char const *const fmtclosed = "%" PRId64 "-%" PRId64 "%n"; + int64_t front = 0; + int64_t back = 0; + + int const fieldsclosed = sscanf(rangebuf, fmtclosed, &front, &back, &consumed); + if (2 == fieldsclosed) { + if (0 <= front && front <= back && rlen == consumed) { + m_beg = front; + m_end = back + 1; + return true; + } else { + return false; + } + } + + front = 0; + char const *const fmtbeg = "%" PRId64 "-%n"; + int const fieldsbeg = sscanf(rangebuf, fmtbeg, &front, &consumed); + if (1 == fieldsbeg) { + if (rlen == consumed) { + m_beg = front; + m_end = Range::maxval; + return true; + } else { + return false; + } + } + + return false; +} // parseRange + +bool +Range::toStringClosed(char *const bufstr, + int *const buflen // returns actual bytes used + ) const +{ + if (!isValid()) { + if (0 < *buflen) { + bufstr[0] = '\0'; + } + *buflen = 0; + return false; + } + + int const lenin = *buflen; + + if (m_end <= Range::maxval) { + *buflen = snprintf(bufstr, lenin, "bytes=%" PRId64 "-%" PRId64, m_beg, m_end - 1); + } else { + *buflen = snprintf(bufstr, lenin, "bytes=%" PRId64 "-", m_beg); + } + + return (0 < *buflen && *buflen < lenin); +} + +int64_t +Range::firstBlockFor(int64_t const blocksize) const +{ + if (0 < blocksize && isValid()) { + return std::max((int64_t)0, m_beg / blocksize); + } else { + return -1; + } +} + +Range +Range::intersectedWith(Range const &other) const +{ + return Range(std::max(m_beg, other.m_beg), std::min(m_end, other.m_end)); +} + +bool +Range::blockIsInside(int64_t const blocksize, int64_t const blocknum) const +{ + Range const blockrange(blocksize * blocknum, blocksize * (blocknum + 1)); + + Range const isec(blockrange.intersectedWith(*this)); + + return isec.isValid(); +} + +int64_t +Range::skipBytesForBlock(int64_t const blocksize, int64_t const blocknum) const +{ + int64_t const blockstart(blocksize * blocknum); + + if (m_beg < blockstart) { + return 0; + } else { + return m_beg - blockstart; + } +} + +bool +Range::isEndBytes() const +{ + return m_beg < 0 && 0 == m_end; +} diff --git a/plugins/experimental/slice/Range.h b/plugins/experimental/slice/Range.h new file mode 100644 index 00000000000..0fb3145ddae --- /dev/null +++ b/plugins/experimental/slice/Range.h @@ -0,0 +1,74 @@ +/** @file + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#pragma once + +#include "ts/ts.h" + +#include + +/** + represents a value parsed from a Range request header field. + Range is converted from a closed range into a half open. + */ + +struct Range { +public: + static int64_t constexpr maxval = (std::numeric_limits::max() >> 2); + + int64_t m_beg = -1; + int64_t m_end = -1; // half open + + Range() {} + explicit Range(int64_t const begin, int64_t const end) : m_beg(begin), m_end(end) {} + + bool isValid() const; + + int64_t size() const; + + /** parse a from a closed request range into a half open range + * This will only correctly handle the *first* range that is + * parsed via TSMimeHdrFieldValueStringGet with index '0'. + * Range representing last N bytes will be coded as (-N, 0) + */ + bool fromStringClosed(char const *const rangestr); + + /** parse a from a closed request range into a half open range + */ + bool toStringClosed(char *const rangestr, int *const rangelen) const; + + /** block number of first range block + */ + int64_t firstBlockFor(int64_t const blockbytes) const; + + /** block intersection + */ + Range intersectedWith(Range const &other) const; + + /** is the given block inside held range? + */ + bool blockIsInside(int64_t const blocksize, int64_t const blocknum) const; + + /** number of skip bytes for the given block + */ + int64_t skipBytesForBlock(int64_t const blocksize, int64_t const blocknum) const; + + /** is this coded to indicate last N bytes? + */ + bool isEndBytes() const; +}; diff --git a/plugins/experimental/slice/Stage.h b/plugins/experimental/slice/Stage.h new file mode 100644 index 00000000000..4166071e5eb --- /dev/null +++ b/plugins/experimental/slice/Stage.h @@ -0,0 +1,149 @@ +/** @file + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#pragma once + +#include "ts/ts.h" + +struct Channel { + TSVIO m_vio{nullptr}; + TSIOBuffer m_iobuf{nullptr}; + TSIOBufferReader m_reader{nullptr}; + + ~Channel() + { + if (nullptr != m_reader) { + TSIOBufferReaderFree(m_reader); + } + if (nullptr != m_iobuf) { + TSIOBufferDestroy(m_iobuf); + } + } + + void + drainReader() + { + TSAssert(nullptr != m_reader); + int64_t const bytes_avail = TSIOBufferReaderAvail(m_reader); + TSIOBufferReaderConsume(m_reader, bytes_avail); + } + + bool + setForRead(TSVConn vc, TSCont contp, int64_t const bytesin //=INT64_MAX + ) + { + TSAssert(nullptr != vc); + if (nullptr == m_iobuf) { + m_iobuf = TSIOBufferCreate(); + m_reader = TSIOBufferReaderAlloc(m_iobuf); + } else { + drainReader(); + } + m_vio = TSVConnRead(vc, contp, m_iobuf, bytesin); + return nullptr != m_vio; + } + + bool + setForWrite(TSVConn vc, TSCont contp, int64_t const bytesout //=INT64_MAX + ) + { + TSAssert(nullptr != vc); + if (nullptr == m_iobuf) { + m_iobuf = TSIOBufferCreate(); + m_reader = TSIOBufferReaderAlloc(m_iobuf); + } else { + drainReader(); + } + m_vio = TSVConnWrite(vc, contp, m_reader, bytesout); + return nullptr != m_vio; + } + + void + close() + { + if (nullptr != m_reader) { + drainReader(); + } + m_vio = nullptr; + } + + bool + isOpen() const + { + return nullptr != m_iobuf && nullptr != m_reader && nullptr != m_vio; + } +}; + +struct Stage // upstream or downstream (server or client) +{ + Stage(Stage const &) = delete; + Stage &operator=(Stage const &) = delete; + + TSVConn m_vc{nullptr}; + Channel m_read; + Channel m_write; + + Stage() {} + ~Stage() + { + if (nullptr != m_vc) { + TSVConnClose(m_vc); + } + } + + void + setupConnection(TSVConn vc) + { + if (nullptr != m_vc) { + TSVConnClose(m_vc); + } + m_vc = vc; + m_read.m_vio = nullptr; + m_write.m_vio = nullptr; + } + + void + setupVioRead(TSCont contp, int64_t const bytesin = INT64_MAX) + { + m_read.setForRead(m_vc, contp, bytesin); + } + + void + setupVioWrite(TSCont contp, int64_t const bytesout = INT64_MAX) + { + m_write.setForWrite(m_vc, contp, bytesout); + } + + void + close() + { + m_read.close(); + m_write.close(); + + if (nullptr != m_vc) { + TSVConnClose(m_vc); + m_vc = nullptr; + } + } + + bool + isOpen() const + { + return nullptr != m_vc && m_read.isOpen() && m_write.isOpen(); + } +}; diff --git a/plugins/experimental/slice/client.cc b/plugins/experimental/slice/client.cc new file mode 100644 index 00000000000..7076f6dcd45 --- /dev/null +++ b/plugins/experimental/slice/client.cc @@ -0,0 +1,232 @@ +/** @file + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#include "client.h" + +#include "transfer.h" + +namespace +{ +void +shutdown(TSCont const contp, Data *const data) +{ + DEBUG_LOG("shutting down transaction"); + delete data; + TSContDestroy(contp); +} + +// create and issue a block request +bool +requestBlock(TSCont contp, Data *const data) +{ + int64_t const blockbeg = (data->m_config->m_blockbytes * data->m_blocknum); + Range blockbe(blockbeg, blockbeg + data->m_config->m_blockbytes); + + char rangestr[1024]; + int rangelen = sizeof(rangestr); + bool const rpstat = blockbe.toStringClosed(rangestr, &rangelen); + TSAssert(rpstat); + + DEBUG_LOG("requestBlock: %s", rangestr); + + // reuse the incoming client header, just change the range + HttpHeader header(data->m_req_hdrmgr.m_buffer, data->m_req_hdrmgr.m_lochdr); + + // add/set sub range key and add slicer tag + bool const rangestat = header.setKeyVal(TS_MIME_FIELD_RANGE, TS_MIME_LEN_RANGE, rangestr, rangelen); + + if (!rangestat) { + ERROR_LOG("Error trying to set range request header %s", rangestr); + return false; + } + + // create virtual connection back into ATS + TSVConn const upvc = TSHttpConnect((sockaddr *)&data->m_client_ip); + + // set up connection with the HttpConnect server, maybe clear old one + data->m_upstream.setupConnection(upvc); + data->m_upstream.setupVioWrite(contp); + + TSHttpHdrPrint(header.m_buffer, header.m_lochdr, data->m_upstream.m_write.m_iobuf); + TSVIOReenable(data->m_upstream.m_write.m_vio); + + /* + std::string const headerstr(header.toString()); + DEBUG_LOG("Headers\n%s", headerstr.c_str()); + */ + + // get ready for data back from the server + data->m_upstream.setupVioRead(contp); + + // anticipate the next server response header + TSHttpParserClear(data->m_http_parser); + data->m_resp_hdrmgr.resetHeader(); + + data->m_blockexpected = 0; + data->m_blockconsumed = 0; + data->m_iseos = false; + data->m_server_block_header_parsed = false; + + return true; +} + +} // namespace + +// this is called once per transaction when the client sends a req header +bool +handle_client_req(TSCont contp, TSEvent event, Data *const data) +{ + if (TS_EVENT_VCONN_READ_READY == event || TS_EVENT_VCONN_READ_COMPLETE == event) { + if (nullptr == data->m_http_parser) { + data->m_http_parser = TSHttpParserCreate(); + } + + // the client request header didn't fit into the input buffer: + if (TS_PARSE_DONE != + data->m_req_hdrmgr.populateFrom(data->m_http_parser, data->m_dnstream.m_read.m_reader, TSHttpHdrParseReq)) { + return false; + } + + // make the header manipulator + HttpHeader header(data->m_req_hdrmgr.m_buffer, data->m_req_hdrmgr.m_lochdr); + + // set the request url back to pristine in case of plugin stacking + header.setUrl(data->m_urlbuffer, data->m_urlloc); + + header.setKeyVal(TS_MIME_FIELD_HOST, TS_MIME_LEN_HOST, data->m_hostname, data->m_hostlen); + + // default: whole file (unknown, wait for first server response) + Range rangebe; + + char rangestr[1024]; + int rangelen = sizeof(rangestr); + bool const hasRange = header.valueForKey(TS_MIME_FIELD_RANGE, TS_MIME_LEN_RANGE, rangestr, &rangelen, + 0); // <-- first range only + if (hasRange) { + // write parsed header into slicer meta tag + header.setKeyVal(SLICER_MIME_FIELD_INFO, strlen(SLICER_MIME_FIELD_INFO), rangestr, rangelen); + bool const isRangeGood = rangebe.fromStringClosed(rangestr); + + if (isRangeGood) { + DEBUG_LOG("Partial content request"); + data->m_statustype = TS_HTTP_STATUS_PARTIAL_CONTENT; + } else // signal a 416 needs to be formed and sent + { + DEBUG_LOG("Ill formed/unhandled range: %s", rangestr); + data->m_statustype = TS_HTTP_STATUS_REQUESTED_RANGE_NOT_SATISFIABLE; + + // First block will give Content-Length + rangebe = Range(0, data->m_config->m_blockbytes); + } + } else { + DEBUG_LOG("Full content request"); + static char const *const valstr = "-"; + static size_t const vallen = strlen(valstr); + header.setKeyVal(SLICER_MIME_FIELD_INFO, strlen(SLICER_MIME_FIELD_INFO), valstr, vallen); + data->m_statustype = TS_HTTP_STATUS_OK; + rangebe = Range(0, Range::maxval); + } + + // set to the first block in range + data->m_blocknum = rangebe.firstBlockFor(data->m_config->m_blockbytes); + data->m_req_range = rangebe; + + // remove ATS keys to avoid 404 loop + header.removeKey(TS_MIME_FIELD_VIA, TS_MIME_LEN_VIA); + header.removeKey(TS_MIME_FIELD_X_FORWARDED_FOR, TS_MIME_LEN_X_FORWARDED_FOR); + + // send the first block request to server + if (!requestBlock(contp, data)) { + shutdown(contp, data); + return false; + } + + // for subsequent blocks remove any conditionals which may fail + // an optimization would be to wait until the first block succeeds + header.removeKey(TS_MIME_FIELD_IF_MATCH, TS_MIME_LEN_IF_MATCH); + header.removeKey(TS_MIME_FIELD_IF_MODIFIED_SINCE, TS_MIME_LEN_IF_MODIFIED_SINCE); + header.removeKey(TS_MIME_FIELD_IF_NONE_MATCH, TS_MIME_LEN_IF_NONE_MATCH); + header.removeKey(TS_MIME_FIELD_IF_RANGE, TS_MIME_LEN_IF_RANGE); + header.removeKey(TS_MIME_FIELD_IF_UNMODIFIED_SINCE, TS_MIME_LEN_IF_UNMODIFIED_SINCE); + } + + return true; +} + +// this is when the client starts asking us for more data +void +handle_client_resp(TSCont contp, TSEvent event, Data *const data) +{ + if (TS_EVENT_VCONN_WRITE_READY == event || TS_EVENT_VCONN_WRITE_COMPLETE == event) { + transfer_content_bytes(data); + + // done transferring from server to client buffer? + if (data->m_bytestosend <= data->m_bytessent) { + // real amount transferred to client + int64_t const bytessent(TSVIONDoneGet(data->m_dnstream.m_write.m_vio)); + + // is the output buffer drained? + if (data->m_bytestosend <= bytessent) { + data->m_dnstream.close(); + if (!data->m_upstream.m_read.isOpen()) { + shutdown(contp, data); + return; + } + } + + // continue allowing the downstream to drain + return; + } + + // error condition from the server side + if (data->m_bail) { + shutdown(contp, data); + return; + } + + // check for upstream eos, maybe request next block + if (data->m_iseos) { + // still need to drain the server side + if (0 < TSIOBufferReaderAvail(data->m_upstream.m_read.m_reader)) { + TSVIOReenable(data->m_dnstream.m_write.m_vio); + return; + } + + // if done or partial block + if (data->m_blocknum < 0 || data->m_blockconsumed < data->m_blockexpected) { + shutdown(contp, data); + return; + } + + // ready for next block + requestBlock(contp, data); + } + } + // client closed connection + else if (TS_EVENT_ERROR == event) { + DEBUG_LOG("got a TS_EVENT_ERROR from the client"); + + // allow the upstream server to drain + data->m_dnstream.close(); + if (!data->m_upstream.m_read.isOpen()) { + shutdown(contp, data); + } + } else { + DEBUG_LOG("Unhandled event: %d", event); + } +} diff --git a/plugins/experimental/slice/client.h b/plugins/experimental/slice/client.h new file mode 100644 index 00000000000..cc27bee3b1d --- /dev/null +++ b/plugins/experimental/slice/client.h @@ -0,0 +1,34 @@ +/** @file + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#pragma once + +#include "Data.h" + +#include "ts/ts.h" + +/** Functions to deal with the connection to the client. + * Body content transfers are handled by the client. + * New block requests are also initiated by the client. + */ + +/** returns true if the incoming vio can be turned off + */ +bool handle_client_req(TSCont contp, TSEvent event, Data *const data); + +void handle_client_resp(TSCont contp, TSEvent event, Data *const data); diff --git a/plugins/experimental/slice/intercept.cc b/plugins/experimental/slice/intercept.cc new file mode 100644 index 00000000000..f21306a9942 --- /dev/null +++ b/plugins/experimental/slice/intercept.cc @@ -0,0 +1,88 @@ +/** @file + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#include "intercept.h" + +#include "Data.h" +#include "client.h" +#include "server.h" +#include "slice.h" + +int +intercept_hook(TSCont contp, TSEvent event, void *edata) +{ + // DEBUG_LOG("intercept_hook: %d", event); + + Data *const data = static_cast(TSContDataGet(contp)); + if (nullptr == data) { + DEBUG_LOG("Events handled after data already torn down"); + TSContDestroy(contp); + return TS_EVENT_ERROR; + } + + // After the initial TS_EVENT_NET_ACCEPT + // any "events" will be handled by the vio read or write channel handler + switch (event) { + case TS_EVENT_NET_ACCEPT: { + // set up reader from client + TSVConn const downvc = (TSVConn)edata; + data->m_dnstream.setupConnection(downvc); + data->m_dnstream.setupVioRead(contp); + } break; + + case TS_EVENT_VCONN_INACTIVITY_TIMEOUT: + case TS_EVENT_VCONN_ACTIVE_TIMEOUT: + case TS_EVENT_HTTP_TXN_CLOSE: + delete data; + TSContDestroy(contp); + break; + + default: { + // data from client -- only the initial header + if (data->m_dnstream.m_read.isOpen() && edata == data->m_dnstream.m_read.m_vio) { + if (handle_client_req(contp, event, data)) { + // DEBUG_LOG("shutting down read from client pipe"); + TSVConnShutdown(data->m_dnstream.m_vc, 1, 0); + } + } + // server wants more data from us, should never happen + // every time TSHttpConnect is called this resets + else if (data->m_upstream.m_write.isOpen() && edata == data->m_upstream.m_write.m_vio) { + // DEBUG_LOG("shutting down send to server pipe"); + TSVConnShutdown(data->m_upstream.m_vc, 0, 1); + } + // server has data for us, typically handle just the header + else if (data->m_upstream.m_read.isOpen() && edata == data->m_upstream.m_read.m_vio) { + handle_server_resp(contp, event, data); + } + // client wants more data from us, only body content + else if (data->m_dnstream.m_write.isOpen() && edata == data->m_dnstream.m_write.m_vio) { + handle_client_resp(contp, event, data); + } else { + ERROR_LOG("Unhandled event: %d", event); + /* + std::cerr << __func__ + << ": events received after intercept state torn down" + << std::endl; + */ + } + } + } + + return TS_EVENT_CONTINUE; +} diff --git a/plugins/experimental/slice/intercept.h b/plugins/experimental/slice/intercept.h new file mode 100644 index 00000000000..3b59cef30f9 --- /dev/null +++ b/plugins/experimental/slice/intercept.h @@ -0,0 +1,23 @@ +/** @file + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#pragma once + +#include "ts/ts.h" + +int intercept_hook(TSCont contp, TSEvent event, void *edata); diff --git a/plugins/experimental/slice/response.cc b/plugins/experimental/slice/response.cc new file mode 100644 index 00000000000..410b1507dfe --- /dev/null +++ b/plugins/experimental/slice/response.cc @@ -0,0 +1,107 @@ +/** @file + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#include "response.h" + +#include +#include +#include + +#include "ts/ts.h" + +// canned body string for a 416, stolen from nginx +std::string const & +bodyString416() +{ + static std::string bodystr; + static std::mutex mutex; + std::lock_guard const guard(mutex); + + if (bodystr.empty()) { + bodystr.append("\n"); + bodystr.append("416 Requested Range Not Satisfiable\n"); + bodystr.append("\n"); + bodystr.append("

416 Requested Range Not Satisfiable

"); + bodystr.append("
ATS/"); + bodystr.append(TS_VERSION_STRING); + bodystr.append("
\n"); + bodystr.append("\n"); + bodystr.append("\n"); + } + + return bodystr; +} + +// Form a 502 response, preliminary +std::string const & +string502() +{ + static std::string msg; + static std::mutex mutex; + std::lock_guard const guard(mutex); + + if (msg.empty()) { + std::string bodystr; + bodystr.append("\n"); + bodystr.append("502 Bad Gateway\n"); + bodystr.append("\n"); + bodystr.append("

502 Bad Gateway: Missing/Malformed " + "Content-Range

"); + bodystr.append("
ATS/"); + bodystr.append(TS_VERSION_STRING); + bodystr.append("
\n"); + bodystr.append("\n"); + bodystr.append("\n"); + + char clenstr[1024]; + int const clen = snprintf(clenstr, sizeof(clenstr), "%lu", bodystr.size()); + + msg.append("HTTP/1.1 502 Bad Gateway\r\n"); + msg.append("Content-Length: "); + msg.append(clenstr, clen); + msg.append("\r\n"); + + msg.append("\r\n"); + msg.append(bodystr); + } + + return msg; +} + +void +form416HeaderAndBody(HttpHeader &header, int64_t const contentlen, std::string const &bodystr) +{ + header.removeKey(TS_MIME_FIELD_LAST_MODIFIED, TS_MIME_LEN_LAST_MODIFIED); + header.removeKey(TS_MIME_FIELD_ETAG, TS_MIME_LEN_ETAG); + header.removeKey(TS_MIME_FIELD_ACCEPT_RANGES, TS_MIME_LEN_ACCEPT_RANGES); + + header.setStatus(TS_HTTP_STATUS_REQUESTED_RANGE_NOT_SATISFIABLE); + char const *const reason = TSHttpHdrReasonLookup(TS_HTTP_STATUS_REQUESTED_RANGE_NOT_SATISFIABLE); + header.setReason(reason, strlen(reason)); + + char bufstr[256]; + int buflen = snprintf(bufstr, sizeof(bufstr), "%lu", bodystr.size()); + header.setKeyVal(TS_MIME_FIELD_CONTENT_LENGTH, TS_MIME_LEN_CONTENT_LENGTH, bufstr, buflen); + + static char const *const ctypestr = "text/html"; + static int const ctypelen = strlen(ctypestr); + header.setKeyVal(TS_MIME_FIELD_CONTENT_TYPE, TS_MIME_LEN_CONTENT_TYPE, ctypestr, ctypelen); + + buflen = snprintf(bufstr, 255, "*/%" PRId64, contentlen); + header.setKeyVal(TS_MIME_FIELD_CONTENT_RANGE, TS_MIME_LEN_CONTENT_RANGE, bufstr, buflen); +} diff --git a/plugins/experimental/slice/response.h b/plugins/experimental/slice/response.h new file mode 100644 index 00000000000..925db65f725 --- /dev/null +++ b/plugins/experimental/slice/response.h @@ -0,0 +1,28 @@ +/** @file + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#pragma once + +#include "HttpHeader.h" +#include + +std::string const &string502(); + +std::string const &bodyString416(); + +void form416HeaderAndBody(HttpHeader &header, int64_t const contentlen, std::string const &bodystr); diff --git a/plugins/experimental/slice/server.cc b/plugins/experimental/slice/server.cc new file mode 100644 index 00000000000..37016de753e --- /dev/null +++ b/plugins/experimental/slice/server.cc @@ -0,0 +1,450 @@ +/** @file + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#include "server.h" + +#include "ContentRange.h" +#include "response.h" +#include "transfer.h" + +#include "ts/experimental.h" + +#include + +namespace +{ +void +shutdown(TSCont const contp, Data *const data) +{ + DEBUG_LOG("shutting down transaction"); + delete data; + TSContDestroy(contp); +} + +ContentRange +contentRangeFrom(HttpHeader const &header) +{ + ContentRange bcr; + + /* Pull content length off the response header + and manipulate it into a client response header + */ + char rangestr[1024]; + int rangelen = sizeof(rangestr); + + // look for expected Content-Range field + bool const hasContentRange(header.valueForKey(TS_MIME_FIELD_CONTENT_RANGE, TS_MIME_LEN_CONTENT_RANGE, rangestr, &rangelen)); + + if (!hasContentRange) { + DEBUG_LOG("invalid response header, no Content-Range"); + } else if (!bcr.fromStringClosed(rangestr)) { + DEBUG_LOG("invalid response header, malformed Content-Range, %s", rangestr); + } + + return bcr; +} + +bool +handleFirstServerHeader(Data *const data, TSCont const contp) +{ + HttpHeader header(data->m_resp_hdrmgr.m_buffer, data->m_resp_hdrmgr.m_lochdr); + + // DEBUG_LOG("First header\n%s", header.toString().c_str()); + + data->m_dnstream.setupVioWrite(contp); + + // only process a 206, everything else gets a pass through + if (TS_HTTP_STATUS_PARTIAL_CONTENT != header.status()) { + DEBUG_LOG("Initial reponse other than 206: %d", header.status()); + data->m_bail = true; + + TSHttpHdrPrint(header.m_buffer, header.m_lochdr, data->m_dnstream.m_write.m_iobuf); + + transfer_all_bytes(data); + + return false; + } + + ContentRange const blockcr = contentRangeFrom(header); + // 206 with bad content range? + if (!blockcr.isValid()) { + data->m_bail = true; + + static std::string const &msg502 = string502(); + + TSIOBufferWrite(data->m_dnstream.m_write.m_iobuf, msg502.data(), msg502.size()); + TSVIOReenable(data->m_dnstream.m_write.m_vio); + + return false; + } + + // set the resource content length from block response + data->m_contentlen = blockcr.m_length; + + // special case last N bytes + if (data->m_req_range.isEndBytes()) { + data->m_req_range.m_end += data->m_contentlen; + data->m_req_range.m_beg += data->m_contentlen; + data->m_req_range.m_beg = std::max((int64_t)0, data->m_req_range.m_beg); + } else { + // fix up request range end now that we have the content length + data->m_req_range.m_end = std::min(data->m_contentlen, data->m_req_range.m_end); + } + + int64_t const bodybytes = data->m_req_range.size(); + + // range past end of data, assume 416 needs to be sent + bool const send416 = (bodybytes <= 0 || TS_HTTP_STATUS_REQUESTED_RANGE_NOT_SATISFIABLE == data->m_statustype); + if (send416) { + data->m_bail = true; + std::string const &bodystr = bodyString416(); + form416HeaderAndBody(header, data->m_contentlen, bodystr); + + TSHttpHdrPrint(header.m_buffer, header.m_lochdr, data->m_dnstream.m_write.m_iobuf); + + TSIOBufferWrite(data->m_dnstream.m_write.m_iobuf, bodystr.data(), bodystr.size()); + + TSVIOReenable(data->m_dnstream.m_write.m_vio); + + return false; + } + + // save weak cache header identifiers (rfc7232 section 2) + data->m_etaglen = sizeof(data->m_etag) - 1; + header.valueForKey(TS_MIME_FIELD_ETAG, TS_MIME_LEN_ETAG, data->m_etag, &data->m_etaglen); + data->m_lastmodifiedlen = sizeof(data->m_lastmodified) - 1; + header.valueForKey(TS_MIME_FIELD_LAST_MODIFIED, TS_MIME_LEN_LAST_MODIFIED, data->m_lastmodified, &data->m_lastmodifiedlen); + + // size of the first block payload + data->m_blockexpected = blockcr.rangeSize(); + + // Now we can set up the expected client response + if (TS_HTTP_STATUS_PARTIAL_CONTENT == data->m_statustype) { + ContentRange respcr; + respcr.m_beg = data->m_req_range.m_beg; + respcr.m_end = data->m_req_range.m_end; + respcr.m_length = data->m_contentlen; + + char rangestr[1024]; + int rangelen = sizeof(rangestr); + bool const crstat = respcr.toStringClosed(rangestr, &rangelen); + + // corner case, return 500 ?? + if (!crstat) { + data->m_bail = true; + + data->m_upstream.close(); + data->m_dnstream.close(); + + ERROR_LOG("Bad/invalid response content range"); + return false; + } + + header.setKeyVal(TS_MIME_FIELD_CONTENT_RANGE, TS_MIME_LEN_CONTENT_RANGE, rangestr, rangelen); + } + // fix up for 200 response + else if (TS_HTTP_STATUS_OK == data->m_statustype) { + header.setStatus(TS_HTTP_STATUS_OK); + static char const *const reason = TSHttpHdrReasonLookup(TS_HTTP_STATUS_OK); + header.setReason(reason, strlen(reason)); + header.removeKey(TS_MIME_FIELD_CONTENT_RANGE, TS_MIME_LEN_CONTENT_RANGE); + } + + char bufstr[1024]; + int const buflen = snprintf(bufstr, sizeof(bufstr), "%" PRId64, bodybytes); + header.setKeyVal(TS_MIME_FIELD_CONTENT_LENGTH, TS_MIME_LEN_CONTENT_LENGTH, bufstr, buflen); + + // add the response header length to the total bytes to send + int64_t const headerbytes = TSHttpHdrLengthGet(header.m_buffer, header.m_lochdr); + + data->m_bytestosend = headerbytes + bodybytes; + + TSHttpHdrPrint(header.m_buffer, header.m_lochdr, data->m_dnstream.m_write.m_iobuf); + + data->m_bytessent = headerbytes; + + TSVIOReenable(data->m_dnstream.m_write.m_vio); + + return true; +} + +void +logSliceError(char const *const message, Data const *const data, HttpHeader const &header_resp) +{ + Config *const config = data->m_config; + + bool const logToError = config->canLogError(); + + // always write block stitch errors while in debug mode + if (!logToError && !TSIsDebugTagSet(PLUGIN_NAME)) { + return; + } + + HttpHeader const header_req(data->m_req_hdrmgr.m_buffer, data->m_req_hdrmgr.m_lochdr); + + TSHRTime const timenowus = TShrtime(); + int64_t const msecs = timenowus / 1000000; + int64_t const secs = msecs / 1000; + int64_t const ms = msecs % 1000; + + // Gather information on the request, must delete urlstr + int urllen = 0; + char *const urlstr = header_req.urlString(&urllen); + + char urlpstr[16384]; + size_t urlplen = sizeof(urlpstr); + TSStringPercentEncode(urlstr, urllen, urlpstr, urlplen, &urlplen, nullptr); + + if (nullptr != urlstr) { + TSfree(urlstr); + } + + // uas + char uasstr[8192]; + int uaslen = sizeof(uasstr); + header_req.valueForKey(TS_MIME_FIELD_USER_AGENT, TS_MIME_LEN_USER_AGENT, uasstr, &uaslen); + + // raw range request + char rangestr[1024]; + int rangelen = sizeof(rangestr); + header_req.valueForKey(SLICER_MIME_FIELD_INFO, strlen(SLICER_MIME_FIELD_INFO), rangestr, &rangelen); + + // Normalized range request + ContentRange const crange(data->m_req_range.m_beg, data->m_req_range.m_end, data->m_contentlen); + char normstr[1024]; + int normlen = sizeof(normstr); + crange.toStringClosed(normstr, &normlen); + + // block range request + int64_t const blockbeg = data->m_blocknum * data->m_config->m_blockbytes; + int64_t const blockend = std::min(blockbeg + data->m_config->m_blockbytes, data->m_contentlen); + + // Block response data + TSHttpStatus const statusgot = header_resp.status(); + + // content range + char crstr[1024]; + int crlen = sizeof(crstr); + header_resp.valueForKey(TS_MIME_FIELD_CONTENT_RANGE, TS_MIME_LEN_CONTENT_RANGE, crstr, &crlen); + + // etag + char etagstr[1024]; + int etaglen = sizeof(etagstr); + header_resp.valueForKey(TS_MIME_FIELD_ETAG, TS_MIME_LEN_ETAG, etagstr, &etaglen); + + // last modified + char lmstr[1024]; + int lmlen = sizeof(lmstr); + header_resp.valueForKey(TS_MIME_FIELD_LAST_MODIFIED, TS_MIME_LEN_LAST_MODIFIED, lmstr, &lmlen); + + // cc + char ccstr[2048]; + int cclen = sizeof(ccstr); + header_resp.valueForKey(TS_MIME_FIELD_CACHE_CONTROL, TS_MIME_LEN_CACHE_CONTROL, ccstr, &cclen); + + // via tag + char viastr[8192]; + int vialen = sizeof(viastr); + header_resp.valueForKey(TS_MIME_FIELD_VIA, TS_MIME_LEN_VIA, viastr, &vialen); + + char etagexpstr[1024]; + size_t etagexplen = sizeof(etagexpstr); + TSStringPercentEncode(data->m_etag, data->m_etaglen, etagexpstr, etagexplen, &etagexplen, nullptr); + + char etaggotstr[1024]; + size_t etaggotlen = sizeof(etaggotstr); + TSStringPercentEncode(etagstr, etaglen, etaggotstr, etaggotlen, &etaggotlen, nullptr); + + DEBUG_LOG("Logging Block Stitch error"); + + ERROR_LOG("%" PRId64 ".%" PRId64 " reason=\"%s\"" + " uri=\"%.*s\"" + " uas=\"%.*s\"" + " req_range=\"%.*s\"" + " norm_range=\"%.*s\"" + + " etag_exp=\"%.*s\"" + " lm_exp=\"%.*s\"" + + " blk_range=\"%" PRId64 "-%" PRId64 "\"" + + " status_got=\"%d\"" + " cr_got=\"%.*s\"" + " etag_got=\"%.*s\"" + " lm_got=\"%.*s\"" + " cc=\"%.*s\"" + " via=\"%.*s\"", + secs, ms, message, (int)urlplen, urlpstr, uaslen, uasstr, rangelen, rangestr, normlen, normstr, (int)etagexplen, + etagexpstr, data->m_lastmodifiedlen, data->m_lastmodified, blockbeg, blockend - 1, statusgot, crlen, crstr, + (int)etaggotlen, etaggotstr, lmlen, lmstr, cclen, ccstr, vialen, viastr); +} + +bool +handleNextServerHeader(Data *const data, TSCont const contp) +{ + // block response header + HttpHeader header(data->m_resp_hdrmgr.m_buffer, data->m_resp_hdrmgr.m_lochdr); + // DEBUG_LOG("Next Header:\n%s", header.toString().c_str()); + + // only process a 206, everything else just aborts + if (TS_HTTP_STATUS_PARTIAL_CONTENT != header.status()) { + logSliceError("Non 206 internal block response", data, header); + data->m_bail = true; + return false; + } + + // can't parse the content range header, abort -- might be too strict + ContentRange const blockcr = contentRangeFrom(header); + if (!blockcr.isValid() || blockcr.m_length != data->m_contentlen) { + logSliceError("Mismatch/Bad block Content-Range", data, header); + data->m_bail = true; + return false; + } + + bool same = true; + + // prefer the etag but use Last-Modified if we must. + char etag[8192]; + int etaglen = sizeof(etag); + header.valueForKey(TS_MIME_FIELD_ETAG, TS_MIME_LEN_ETAG, etag, &etaglen); + + if (0 < data->m_etaglen || 0 < etaglen) { + same = data->m_etaglen == etaglen && 0 == strncmp(etag, data->m_etag, etaglen); + if (!same) { + logSliceError("Mismatch block Etag", data, header); + } + } else { + char lastmodified[8192]; + int lastmodifiedlen = sizeof(lastmodified); + header.valueForKey(TS_MIME_FIELD_LAST_MODIFIED, TS_MIME_LEN_LAST_MODIFIED, lastmodified, &lastmodifiedlen); + if (0 < data->m_lastmodifiedlen || 0 != lastmodifiedlen) { + same = data->m_lastmodifiedlen == lastmodifiedlen && 0 == strncmp(lastmodified, data->m_lastmodified, lastmodifiedlen); + if (!same) { + logSliceError("Mismatch block Last-Modified", data, header); + } + } + } + + if (!same) { + data->m_bail = true; + return false; + } + + data->m_blockexpected = blockcr.rangeSize(); + + return true; +} + +} // namespace + +// this is called every time the server has data for us +void +handle_server_resp(TSCont contp, TSEvent event, Data *const data) +{ + if (TS_EVENT_VCONN_READ_READY == event || TS_EVENT_VCONN_READ_COMPLETE == event) { + // has block reponse header been parsed?? + if (!data->m_server_block_header_parsed) { + // the server response header didn't fit into the input buffer?? + if (TS_PARSE_DONE != + data->m_resp_hdrmgr.populateFrom(data->m_http_parser, data->m_upstream.m_read.m_reader, TSHttpHdrParseResp)) { + return; + } + + // very first server response header + bool headerStat = false; + if (!data->m_server_first_header_parsed) { + headerStat = handleFirstServerHeader(data, contp); + data->m_server_first_header_parsed = true; + } else { + headerStat = handleNextServerHeader(data, contp); + } + + data->m_server_block_header_parsed = true; + + // kill the upstream and allow dnstream to clean up + if (!headerStat) { + data->m_upstream.close(); + data->m_bail = true; + if (data->m_dnstream.m_write.isOpen()) { + TSVIOReenable(data->m_dnstream.m_write.m_vio); + } else { + shutdown(contp, data); + } + return; + } + + // how much to fast forward into this data block + data->m_blockskip = data->m_req_range.skipBytesForBlock(data->m_config->m_blockbytes, data->m_blocknum); + } + + transfer_content_bytes(data); + } else if (TS_EVENT_VCONN_EOS == event) { + // from testing as far as I can tell, if the sub transaction returns + // a valid header TS_EVENT_VCONN_READ_READY event is always called first. + // this event being called means the input stream is null. + // An upstream transaction that aborts immediately (or a few bytes) + // after it sends a header may end up here with nothing in the upstream + // buffer. + + // this is called when the upstream connection is done. + // make sure to drain all the bytes out before + // issuing the next block request + data->m_iseos = true; + + // corner condition, good source header + 0 length aborted content + // results in no header being read, just an EOS. + // trying to delete the upstream will crash ATS (??) + if (0 == data->m_blockexpected) { + shutdown(contp, data); // this will crash if first block + return; + } + + transfer_content_bytes(data); + + if (!data->m_dnstream.m_write.isOpen()) // server drain condition + { + shutdown(contp, data); + return; + } + + // all bytes left transferred to client buffer + if (0 == TSIOBufferReaderAvail(data->m_upstream.m_read.m_reader)) { + data->m_upstream.close(); + TSVIOReenable(data->m_dnstream.m_write.m_vio); + } + + // prepare for the next request block + ++data->m_blocknum; + + // when we get a "bytes=-" last N bytes request the plugin + // issues a speculative request for the first block + // in that case fast forward to the real first in range block + // Btw this isn't implemented yet, to be handled + int64_t const firstblock(data->m_req_range.firstBlockFor(data->m_config->m_blockbytes)); + if (data->m_blocknum < firstblock) { + data->m_blocknum = firstblock; + } + + // done processing blocks? + if (!data->m_req_range.blockIsInside(data->m_config->m_blockbytes, data->m_blocknum)) { + data->m_blocknum = -1; // signal value no more blocks + } + } else { + DEBUG_LOG("Unhandled event: %d", event); + } +} diff --git a/plugins/experimental/slice/server.h b/plugins/experimental/slice/server.h new file mode 100644 index 00000000000..c0d77e48032 --- /dev/null +++ b/plugins/experimental/slice/server.h @@ -0,0 +1,37 @@ +/** @file + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#pragma once + +#include "Data.h" + +#include "ts/ts.h" + +/** Functions to handle the connection to the server. + * In particular slice block header responses are handled here. + * Data transfers are handled by the client code which pulls + * the data from the server side. + * + * Special case is when the client connection has been closed + * because of client data request being fulfilled or + * when the client aborts. The current slice block will + * continue reading to ensure the whole block is transferred + * to cache. + */ + +void handle_server_resp(TSCont contp, TSEvent event, Data *const data); diff --git a/plugins/experimental/slice/slice.cc b/plugins/experimental/slice/slice.cc new file mode 100644 index 00000000000..5b5c90b48d9 --- /dev/null +++ b/plugins/experimental/slice/slice.cc @@ -0,0 +1,206 @@ +/** @file + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#include "slice.h" + +#include "Config.h" +#include "Data.h" +#include "HttpHeader.h" +#include "intercept.h" + +#include "ts/remap.h" +#include "ts/ts.h" + +#include + +namespace +{ +Config globalConfig; + +bool +read_request(TSHttpTxn txnp, Config *const config) +{ + DEBUG_LOG("slice read_request"); + TxnHdrMgr hdrmgr; + hdrmgr.populateFrom(txnp, TSHttpTxnClientReqGet); + HttpHeader const header(hdrmgr.m_buffer, hdrmgr.m_lochdr); + + if (TS_HTTP_METHOD_GET == header.method()) { + static int const SLICER_MIME_LEN_INFO = strlen(SLICER_MIME_FIELD_INFO); + if (!header.hasKey(SLICER_MIME_FIELD_INFO, SLICER_MIME_LEN_INFO)) { + // turn off any and all transaction caching (shouldn't matter) + TSHttpTxnServerRespNoStoreSet(txnp, 1); + TSHttpTxnRespCacheableSet(txnp, 0); + TSHttpTxnReqCacheableSet(txnp, 0); + + DEBUG_LOG("slice accepting and slicing"); + // connection back into ATS + sockaddr const *const ip = TSHttpTxnClientAddrGet(txnp); + if (nullptr == ip) { + return false; + } + + TSAssert(nullptr != config); + Data *const data = new Data(config); + + // set up feedback connect + if (AF_INET == ip->sa_family) { + memcpy(&data->m_client_ip, ip, sizeof(sockaddr_in)); + } else if (AF_INET6 == ip->sa_family) { + memcpy(&data->m_client_ip, ip, sizeof(sockaddr_in6)); + } else { + delete data; + return false; + } + + // need to reset the HOST field for global plugin + data->m_hostlen = sizeof(data->m_hostname) - 1; + if (!header.valueForKey(TS_MIME_FIELD_HOST, TS_MIME_LEN_HOST, data->m_hostname, &data->m_hostlen)) { + DEBUG_LOG("Unable to get hostname from header"); + delete data; + return false; + } + + // need the pristine url, especially for global plugins + TSMBuffer urlbuf; + TSMLoc urlloc; + TSReturnCode rcode = TSHttpTxnPristineUrlGet(txnp, &urlbuf, &urlloc); + + if (TS_SUCCESS == rcode) { + TSMBuffer const newbuf = TSMBufferCreate(); + TSMLoc newloc = nullptr; + rcode = TSUrlClone(newbuf, urlbuf, urlloc, &newloc); + TSHandleMLocRelease(urlbuf, TS_NULL_MLOC, urlloc); + + if (TS_SUCCESS != rcode) { + ERROR_LOG("Error cloning pristine url"); + delete data; + TSMBufferDestroy(newbuf); + return false; + } + + data->m_urlbuffer = newbuf; + data->m_urlloc = newloc; + } + + // we'll intercept this GET and do it ourselves + TSCont const icontp(TSContCreate(intercept_hook, TSMutexCreate())); + TSContDataSet(icontp, (void *)data); + // TSHttpTxnHookAdd(txnp, TS_HTTP_TXN_CLOSE_HOOK, icontp); + TSHttpTxnIntercept(icontp, txnp); + return true; + } else { + DEBUG_LOG("slice passing GET request through to next plugin"); + } + } + + return false; +} + +int +global_read_request_hook(TSCont // contp + , + TSEvent // event + , + void *edata) +{ + TSHttpTxn const txnp = static_cast(edata); + read_request(txnp, &globalConfig); + TSHttpTxnReenable(txnp, TS_EVENT_HTTP_CONTINUE); + return 0; +} + +} // namespace + +///// remap plugin engine + +SLICE_EXPORT +TSRemapStatus +TSRemapDoRemap(void *ih, TSHttpTxn txnp, TSRemapRequestInfo *rri) +{ + Config *const config = static_cast(ih); + + if (read_request(txnp, config)) { + return TSREMAP_DID_REMAP_STOP; + } else { + return TSREMAP_NO_REMAP; + } +} + +///// remap plugin setup and teardown +SLICE_EXPORT +void +TSRemapOSResponse(void *ih, TSHttpTxn rh, int os_response_type) +{ +} + +SLICE_EXPORT +TSReturnCode +TSRemapNewInstance(int argc, char *argv[], void **ih, char * /* errbuf */, int /* errbuf_size */) +{ + Config *const config = new Config; + if (2 < argc) { + config->fromArgs(argc - 2, argv + 2); + } + *ih = static_cast(config); + return TS_SUCCESS; +} + +SLICE_EXPORT +void +TSRemapDeleteInstance(void *ih) +{ + if (nullptr != ih) { + Config *const config = static_cast(ih); + delete config; + } +} + +SLICE_EXPORT +TSReturnCode +TSRemapInit(TSRemapInterface *api_info, char *errbug, int errbuf_size) +{ + DEBUG_LOG("slice remap is successfully initialized."); + return TS_SUCCESS; +} + +///// global plugin +SLICE_EXPORT +void +TSPluginInit(int argc, char const *argv[]) +{ + TSPluginRegistrationInfo info; + info.plugin_name = (char *)PLUGIN_NAME; + info.vendor_name = (char *)"Apache Software Foundation"; + info.support_email = (char *)"dev@trafficserver.apache.org"; + + if (TS_SUCCESS != TSPluginRegister(&info)) { + ERROR_LOG("Plugin registration failed.\n"); + ERROR_LOG("Unable to initialize plugin (disabled)."); + return; + } + + if (1 < argc) { + globalConfig.fromArgs(argc - 1, argv + 1); + } + + TSCont const contp(TSContCreate(global_read_request_hook, nullptr)); + + // Called immediately after the request header is read from the client + TSHttpHookAdd(TS_HTTP_READ_REQUEST_HDR_HOOK, contp); +} diff --git a/plugins/experimental/slice/slice.h b/plugins/experimental/slice/slice.h new file mode 100644 index 00000000000..682c0168c5a --- /dev/null +++ b/plugins/experimental/slice/slice.h @@ -0,0 +1,54 @@ +/** @file + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#pragma once + +#include "ts/ts.h" + +#include + +#ifndef SLICE_EXPORT +#define SLICE_EXPORT extern "C" tsapi +#endif + +#ifndef PLUGIN_NAME +#define PLUGIN_NAME "slice" +#endif + +#if !defined(UNITTEST) + +#define __FILENAME__ (strrchr(__FILE__, '/') ? strrchr(__FILE__, '/') + 1 : __FILE__) +#define DEBUG_LOG(fmt, ...) \ + TSDebug(PLUGIN_NAME, "[%s:%04d] %s(): " fmt, __FILENAME__, __LINE__, __func__, \ + ##__VA_ARGS__) /* \ + ; fprintf(stderr, "[%s:%04d]: " fmt "\n" \ + , __FILENAME__ \ + , __LINE__ \ + , ##__VA_ARGS__) \ + */ + +#define ERROR_LOG(fmt, ...) \ + TSError("[%s:%04d] %s(): " fmt, __FILENAME__, __LINE__, __func__, ##__VA_ARGS__); \ + TSDebug(PLUGIN_NAME, "[%s:%04d] %s(): " fmt, __FILENAME__, __LINE__, __func__, ##__VA_ARGS__) + +#else + +#define DEBUG_LOG(fmt, ...) +#define ERROR_LOG(fmt, ...) + +#endif diff --git a/plugins/experimental/slice/slice_test.cc b/plugins/experimental/slice/slice_test.cc new file mode 100644 index 00000000000..4fcbc5be7bd --- /dev/null +++ b/plugins/experimental/slice/slice_test.cc @@ -0,0 +1,195 @@ +/** @file + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +/* + * These are misc unit tests for slicer + */ + +#include "ContentRange.h" +#include "Range.h" + +#include +#include +#include +#include +#include + +std::string +testContentRange() +{ + std::ostringstream oss; + + ContentRange null; + if (null.isValid()) { + oss << "fail: null isValid test" << std::endl; + } + + ContentRange const exprange(1023, 1048576, 307232768); + + if (!exprange.isValid()) { + oss << "Fail: exprange valid" << std::endl; + oss << exprange.m_beg << ' ' << exprange.m_end << ' ' << exprange.m_length << std::endl; + } + + std::string const expstr("bytes 1023-1048575/307232768"); + + char gotbuf[1024]; + int gotlen = sizeof(gotbuf); + + bool const strstat(exprange.toStringClosed(gotbuf, &gotlen)); + + if (!strstat) { + oss << "failure status toStringClosed" << std::endl; + } else if ((int)expstr.size() != gotlen) { + oss << "Fail: expected toStringClosed length" << std::endl; + oss << "got: " << gotlen << " exp: " << expstr.size() << std::endl; + oss << "Got: " << gotbuf << std::endl; + oss << "Exp: " << expstr << std::endl; + } else if (expstr != gotbuf) { + oss << "Fail: expected toStringClosed value" << std::endl; + oss << "Got: " << gotbuf << std::endl; + oss << "Exp: " << expstr << std::endl; + } + + ContentRange gotrange; + bool const gotstat(gotrange.fromStringClosed(expstr.c_str())); + if (!gotstat) { + oss << "fail: gotstat from string" << std::endl; + } else if (gotrange.m_beg != exprange.m_beg || gotrange.m_end != exprange.m_end || gotrange.m_length != exprange.m_length) { + oss << "fail: value compare gotrange and exprange" << std::endl; + } + + std::string const teststr("bytes 0-1048575/30723276"); + if (!gotrange.fromStringClosed(teststr.c_str())) { + oss << "fail: parse teststr" << std::endl; + } + + return oss.str(); +} + +std::string +testParseRange() +{ + std::ostringstream oss; + + std::vector const teststrings = { + "bytes=0-1023", + "bytes=1-1024", + "bytes=11-11", + "bytes=1-" // 2nd byte to end + , + "Range: bytes=-13" // final 13 bytes + , + "bytes=3-17" // ,23-29" // open + , + "bytes=3 -17 " //,18-29" // adjacent + , + "bytes=3- 17" //, 11-29" // overlapping + , + "bytes=3 - 11" //,13-17 , 23-29" // unsorted triplet + , + "bytes=3-11 " //,13-17, 23-29" // unsorted triplet + , + "bytes=0-0" //,-1" // first and last bytes + , + "bytes=-20" // last 20 bytes of file + + , + "bytes=-60-50" // invalid fully negative + , + "bytes=17-13" // degenerate + , + "bytes 0-1023/146515" // this should be rejected (Content-range) + }; // invalid + + std::vector const exps = {Range{0, 1023 + 1}, Range{1, 1024 + 1}, Range{11, 11 + 1}, Range{1, Range::maxval}, + Range{-1, -1}, Range{3, 17 + 1}, Range{3, 17 + 1}, Range{3, 17 + 1}, + Range{3, 11 + 1}, Range{3, 11 + 1}, Range{0, 1}, Range{-20, 0}, + Range{-1, -1}, Range{-1, -1}, Range{-1, -1}}; + + std::vector const expsres = {true, true, true, true, false, true, true, true, true, true, true, true, false, false, false}; + + assert(exps.size() == teststrings.size()); + + std::vector gots; + gots.reserve(exps.size()); + std::vector gotsres; + + for (std::string const &str : teststrings) { + Range rng; + gotsres.push_back(rng.fromStringClosed(str.c_str())); + gots.push_back(rng); + } + + assert(gots.size() == exps.size()); + + for (size_t index(0); index < gots.size(); ++index) { + if (exps[index] != gots[index] || expsres[index] != gotsres[index]) { + oss << "Error parsing index: " << index << std::endl; + oss << "test: '" << teststrings[index] << "'" << std::endl; + oss << "exp: " << exps[index].m_beg << ' ' << exps[index].m_end << std::endl; + oss << "expsres: " << (int)expsres[index] << std::endl; + oss << "got: " << gots[index].m_beg << ' ' << gots[index].m_end << std::endl; + oss << "gotsres: " << (int)gotsres[index] << std::endl; + } + } + + return oss.str(); +} + +struct Tests { + typedef std::string (*TestFunc)(); + std::vector> funcs; + + void + add(TestFunc const &func, char const *const fname) + { + funcs.push_back(std::make_pair(func, fname)); + } + + int + run() const + { + int numfailed(0); + for (std::pair const &namefunc : funcs) { + TestFunc const &func = namefunc.first; + char const *const name = namefunc.second; + + std::cerr << name << " : "; + + std::string const fres(func()); + if (fres.empty()) { + std::cerr << "pass" << std::endl; + } else { + std::cerr << "FAIL" << std::endl; + std::cerr << fres << std::endl; + ++numfailed; + } + } + return numfailed; + } +}; + +int +main() +{ + Tests tests; + tests.add(testContentRange, "testContentRange"); + tests.add(testParseRange, "testParseRange"); + return tests.run(); +} diff --git a/plugins/experimental/slice/transfer.cc b/plugins/experimental/slice/transfer.cc new file mode 100644 index 00000000000..4ab7a8bb7c1 --- /dev/null +++ b/plugins/experimental/slice/transfer.cc @@ -0,0 +1,105 @@ +/** @file + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#include "transfer.h" + +int64_t transfer_content_bytes(Data *const data) // , char const * const fstr) +{ + int64_t consumed(0); + + // is the downstream is fulfilled or closed + if (!data->m_dnstream.m_write.isOpen()) { + // drain the upstream + if (data->m_upstream.m_read.isOpen()) { + int64_t const avail = TSIOBufferReaderAvail(data->m_upstream.m_read.m_reader); + TSIOBufferReaderConsume(data->m_upstream.m_read.m_reader, avail); + consumed += avail; + } + } else // if (data->m_dnstream.m_write.isOpen()) + { + if (data->m_upstream.m_read.isOpen()) { + int64_t avail = TSIOBufferReaderAvail(data->m_upstream.m_read.m_reader); + if (0 < avail) { + int64_t const toskip = std::min(data->m_blockskip, avail); + + // consume any up front (first block) padding + if (0 < toskip) { + TSIOBufferReaderConsume(data->m_upstream.m_read.m_reader, toskip); + data->m_blockskip -= toskip; + avail -= toskip; + consumed += toskip; + } + + if (0 < avail) { + int64_t const bytesleft = (data->m_bytestosend - data->m_bytessent); + int64_t const tocopy = std::min(avail, bytesleft); + + if (0 < tocopy) { + int64_t const copied(TSIOBufferCopy(data->m_dnstream.m_write.m_iobuf, data->m_upstream.m_read.m_reader, tocopy, 0)); + + data->m_bytessent += copied; + + TSIOBufferReaderConsume(data->m_upstream.m_read.m_reader, copied); + + avail -= copied; + consumed += copied; + } + } + + // if hit fulfillment start bulk consuming + if (0 < avail && data->m_bytestosend <= data->m_bytessent) { + TSIOBufferReaderConsume(data->m_upstream.m_read.m_reader, avail); + consumed += avail; + } + } + + if (0 < consumed) { + TSVIOReenable(data->m_dnstream.m_write.m_vio); + } + } + } + + if (0 < consumed) { + data->m_blockconsumed += consumed; + } + + return consumed; +} + +// transfer all bytes from the server (error condition) +int64_t +transfer_all_bytes(Data *const data) +{ + DEBUG_LOG("transfer_all_bytes"); + int64_t consumed = 0; + + if (data->m_dnstream.m_write.isOpen()) { + int64_t const read_avail = TSIOBufferReaderAvail(data->m_upstream.m_read.m_reader); + + if (0 < read_avail) { + int64_t const copied(TSIOBufferCopy(data->m_dnstream.m_write.m_iobuf, data->m_upstream.m_read.m_reader, read_avail, 0)); + + if (0 < copied) { + TSIOBufferReaderConsume(data->m_upstream.m_read.m_reader, copied); + consumed = copied; + } + } + } + + return consumed; +} diff --git a/plugins/experimental/slice/transfer.h b/plugins/experimental/slice/transfer.h new file mode 100644 index 00000000000..de3e1766d04 --- /dev/null +++ b/plugins/experimental/slice/transfer.h @@ -0,0 +1,34 @@ +/** @file + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#pragma once + +#include "Data.h" + +/** Functions to deal with the connection to the client. + * Body content transfers are handled by the client. + * New block requests are also initiated by the client. + */ + +/* transfer bytes from the server to the client + * Returns amount of bytes consumed from the reader (<= bytes written to client) + */ +int64_t transfer_content_bytes(Data *const data); // , char const * const fstr); + +// transfer all bytes from the server (error condition) +int64_t transfer_all_bytes(Data *const data); diff --git a/plugins/experimental/slice/unit-tests/slice_test.cc b/plugins/experimental/slice/unit-tests/slice_test.cc new file mode 100644 index 00000000000..96ae726a53c --- /dev/null +++ b/plugins/experimental/slice/unit-tests/slice_test.cc @@ -0,0 +1,181 @@ +/** @file + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +/* + * These are misc unit tests for slicer + */ + +#include "ContentRange.h" +#include "Range.h" + +#include +#include +#include +#include +#include + +std::string +testContentRange() +{ + std::ostringstream oss; + + ContentRange null; + if (null.isValid()) { + oss << "fail: null isValid test" << std::endl; + } + + ContentRange const exprange(1023, 1048576, 307232768); + + if (!exprange.isValid()) { + oss << "Fail: exprange valid" << std::endl; + oss << exprange.m_beg << ' ' << exprange.m_end << ' ' << exprange.m_length << std::endl; + } + + std::string const expstr("bytes 1023-1048575/307232768"); + + char gotbuf[1024]; + int gotlen = sizeof(gotbuf); + + bool const strstat(exprange.toStringClosed(gotbuf, &gotlen)); + + if (!strstat) { + oss << "failure status toStringClosed" << std::endl; + } else if ((int)expstr.size() != gotlen) { + oss << "Fail: expected toStringClosed length" << std::endl; + oss << "got: " << gotlen << " exp: " << expstr.size() << std::endl; + oss << "Got: " << gotbuf << std::endl; + oss << "Exp: " << expstr << std::endl; + } else if (expstr != gotbuf) { + oss << "Fail: expected toStringClosed value" << std::endl; + oss << "Got: " << gotbuf << std::endl; + oss << "Exp: " << expstr << std::endl; + } + + ContentRange gotrange; + bool const gotstat(gotrange.fromStringClosed(expstr.c_str())); + if (!gotstat) { + oss << "fail: gotstat from string" << std::endl; + } else if (gotrange.m_beg != exprange.m_beg || gotrange.m_end != exprange.m_end || gotrange.m_length != exprange.m_length) { + oss << "fail: value compare gotrange and exprange" << std::endl; + } + + std::string const teststr("bytes 0-1048575/30723276"); + if (!gotrange.fromStringClosed(teststr.c_str())) { + oss << "fail: parse teststr" << std::endl; + } + + return oss.str(); +} + +std::string +testParseRange() +{ + std::ostringstream oss; + + std::vector const teststrings = { + "bytes=0-1023", "bytes=1-1024", "bytes=11-11", + "bytes=1-", // 2nd byte to end + "Range: bytes=-13", // final 13 bytes + "bytes=3-17", // ,23-29" // open + "bytes=3 -17 ", //,18-29" // adjacent + "bytes=3- 17", //, 11-29" // overlapping + "bytes=3 - 11", //,13-17 , 23-29" // unsorted triplet + "bytes=3-11 ", //,13-17, 23-29" // unsorted triplet + "bytes=0-0", //,-1" // first and last bytes + "bytes=-20", // last 20 bytes of file + "bytes=-60-50", // invalid fully negative + "bytes=17-13", // degenerate + "bytes 0-1023/146515" // this should be rejected (Content-range) + }; // invalid + + std::vector const exps = {Range{0, 1023 + 1}, Range{1, 1024 + 1}, Range{11, 11 + 1}, Range{1, Range::maxval}, + Range{-1, -1}, Range{3, 17 + 1}, Range{3, 17 + 1}, Range{3, 17 + 1}, + Range{3, 11 + 1}, Range{3, 11 + 1}, Range{0, 1}, Range{-20, 0}, + Range{-1, -1}, Range{-1, -1}, Range{-1, -1}}; + + std::vector const expsres = {true, true, true, true, false, true, true, true, true, true, true, true, false, false, false}; + + assert(exps.size() == teststrings.size()); + + std::vector gots; + gots.reserve(exps.size()); + std::vector gotsres; + + for (std::string const &str : teststrings) { + Range rng; + gotsres.push_back(rng.fromStringClosed(str.c_str())); + gots.push_back(rng); + } + + assert(gots.size() == exps.size()); + + for (size_t index(0); index < gots.size(); ++index) { + if (exps[index] != gots[index] || expsres[index] != gotsres[index]) { + oss << "Eror parsing index: " << index << std::endl; + oss << "test: '" << teststrings[index] << "'" << std::endl; + oss << "exp: " << exps[index].m_beg << ' ' << exps[index].m_end << std::endl; + oss << "expsres: " << (int)expsres[index] << std::endl; + oss << "got: " << gots[index].m_beg << ' ' << gots[index].m_end << std::endl; + oss << "gotsres: " << (int)gotsres[index] << std::endl; + } + } + + return oss.str(); +} + +struct Tests { + typedef std::string (*TestFunc)(); + std::vector> funcs; + + void + add(TestFunc const &func, char const *const fname) + { + funcs.push_back(std::make_pair(func, fname)); + } + + int + run() const + { + int numfailed(0); + for (std::pair const &namefunc : funcs) { + TestFunc const &func = namefunc.first; + char const *const name = namefunc.second; + + std::cerr << name << " : "; + + std::string const fres(func()); + if (fres.empty()) { + std::cerr << "pass" << std::endl; + } else { + std::cerr << "FAIL" << std::endl; + std::cerr << fres << std::endl; + ++numfailed; + } + } + return numfailed; + } +}; + +int +main() +{ + Tests tests; + tests.add(testContentRange, "testContentRange"); + tests.add(testParseRange, "testParseRange"); + return tests.run(); +} diff --git a/plugins/experimental/slice/unit-tests/test_config.cc b/plugins/experimental/slice/unit-tests/test_config.cc new file mode 100644 index 00000000000..b85f381e959 --- /dev/null +++ b/plugins/experimental/slice/unit-tests/test_config.cc @@ -0,0 +1,86 @@ +/** @file + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +/** + * @file test_content_range.cc + * @brief Unit test for slice ContentRange + */ + +#define CATCH_CONFIG_MAIN /* include main function */ +#include "../Config.h" +#include "catch.hpp" /* catch unit-test framework */ + +#include + +TEST_CASE("config default", "[AWS][slice][utility]") +{ + Config const config; + int64_t const defval = Config::blockbytesdefault; + CHECK(defval == config.m_blockbytes); +} + +TEST_CASE("config bytesfrom valid parsing", "[AWS][slice][utility]") +{ + static std::array const teststrings = {{ + "1000", + "1m", + "5g", + "2k", + "3kb", + "1z", + }}; + + constexpr std::array const expvals = {{ + 1000, + 1024 * 1024, + int64_t(1024) * 1024 * 1024 * 5, + 1024 * 2, + 1024 * 3, + 1, + }}; + + for (size_t index = 0; index < teststrings.size(); ++index) { + std::string const &teststr = teststrings[index]; + int64_t const &exp = expvals[index]; + int64_t const got = Config::bytesFrom(teststr.c_str()); + + CHECK(got == exp); + if (got != exp) { + INFO(teststr.c_str()); + } + } +} + +TEST_CASE("config bytesfrom invalid parsing", "[AWS][slice][utility]") +{ + static std::array const badstrings = {{ + "abc", // alpha + "g00", // giga + "M00", // mega + "k00", // kilo + "-500", // negative + }}; + + for (std::string const &badstr : badstrings) { + int64_t const val = Config::bytesFrom(badstr.c_str()); + CHECK(0 == val); + if (0 != val) { + INFO(badstr.c_str()); + } + } +} diff --git a/plugins/experimental/slice/unit-tests/test_content_range.cc b/plugins/experimental/slice/unit-tests/test_content_range.cc new file mode 100644 index 00000000000..a9876297d62 --- /dev/null +++ b/plugins/experimental/slice/unit-tests/test_content_range.cc @@ -0,0 +1,80 @@ +/** @file + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +/** + * @file test_content_range.cc + * @brief Unit test for slice ContentRange + */ + +#define CATCH_CONFIG_MAIN /* include main function */ +#include "catch.hpp" /* catch unit-test framework */ +#include "../ContentRange.h" + +TEST_CASE("content_range invalid state", "[AWS][slice][utility]") +{ + CHECK_FALSE(ContentRange().isValid()); // null range + CHECK_FALSE(ContentRange(1024, 1024, 4000).isValid()); // zero range + CHECK_FALSE(ContentRange(0, 1024, 1023).isValid()); // past end + CHECK_FALSE(ContentRange(-5, 13, 40).isValid()); // negative start +} + +TEST_CASE("content_range to/from string - valid", "[AWS][slice][utility]") +{ + ContentRange const exprange(1023, 1048576, 307232768); + + CHECK(exprange.isValid()); + + std::string const expstr("bytes 1023-1048575/307232768"); + + char gotbuf[1024]; + int gotlen = sizeof(gotbuf); + + bool const strstat(exprange.toStringClosed(gotbuf, &gotlen)); + + CHECK(strstat); + CHECK(gotlen == expstr.size()); + CHECK(expstr == std::string(gotbuf)); + + ContentRange gotrange; + bool const gotstat(gotrange.fromStringClosed(expstr.c_str())); + + CHECK(gotstat); + CHECK(gotrange.m_beg == exprange.m_beg); + CHECK(gotrange.m_end == exprange.m_end); + CHECK(gotrange.m_length == exprange.m_length); +} + +TEST_CASE("content_range from string - invalid", "[AWS][slice][utility]") +{ + std::vector const badstrings = { + "bytes=1024-1692", // malformed + "bytes=1023-1048575/307232768", // malformed + "bytes 1023-1022/5000", // zero size + "bytes -40-12/50", // negative start + "bytes 5-13/11" // past end + }; + + ContentRange cr; + + for (std::string const &badstr : badstrings) { + if (!cr.fromStringClosed(badstr.c_str())) { + CHECK_FALSE(cr.isValid()); + INFO(badstr.c_str()); + } + } +} diff --git a/plugins/experimental/slice/unit-tests/test_range.cc b/plugins/experimental/slice/unit-tests/test_range.cc new file mode 100644 index 00000000000..d2b1813fe3f --- /dev/null +++ b/plugins/experimental/slice/unit-tests/test_range.cc @@ -0,0 +1,99 @@ +/** @file + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +/** + * @file test_content_range.cc + * @brief Unit test for slice ContentRange + */ + +#define CATCH_CONFIG_MAIN /* include main function */ +#include "catch.hpp" /* catch unit-test framework */ +#include "../Range.h" + +TEST_CASE("range invalid state", "[AWS][slice][utility]") +{ + CHECK_FALSE(Range().isValid()); // null range + CHECK_FALSE(Range(1024, 1024).isValid()); // zero range + CHECK_FALSE(Range(-5, 13).isValid()); // negative start +} + +TEST_CASE("range to/from string - valid", "[AWS][slice][utility]") +{ + std::vector const teststrings = { + "bytes=0-1023", // start at zero + "bytes=1-1024", // start from non zero + "bytes=11-11", // single byte + "bytes=1-", // 2nd byte to end + "bytes=3-17", // ,23-29" // open + "bytes=3 -17 ", //,18-29" // adjacent + "bytes=3- 17", //, 11-29" // overlapping + "bytes=3 - 11", //,13-17 , 23-29" // unsorted triplet + "bytes=3-11 ", //,13-17, 23-29" // unsorted triplet + "bytes=0-0", //,-1" // first and last bytes + "bytes=-20", // last 20 bytes of file + }; + + std::vector const exps = { + Range{0, 1023 + 1}, // + Range{1, 1024 + 1}, // + Range{11, 11 + 1}, // + Range{1, Range::maxval}, // + Range{3, 17 + 1}, // + Range{3, 17 + 1}, // + Range{3, 17 + 1}, // + Range{3, 11 + 1}, // + Range{3, 11 + 1}, // + Range{0, 1}, // + Range{-20, 0} // + }; + + for (size_t index = 0; index < teststrings.size(); ++index) { + std::string const &str = teststrings[index]; + + Range got; + CHECK(got.fromStringClosed(str.c_str())); + CHECK(got.isValid()); + + if (!got.isValid()) { + INFO(str.c_str()); + } + + Range const &exp = exps[index]; + CHECK(got.m_beg == exp.m_beg); + CHECK(got.m_end == exp.m_end); + } +} + +TEST_CASE("range from string - invalid") +{ + std::vector const badstrings = { + "Range: bytes=-13", // malformed + "bytes=-60-50", // first negative, second nonzero + "bytes=17-13", // degenerate + "bytes 0-1023/146515" // malformed + }; + + Range range; + for (std::string const &badstr : badstrings) { + CHECK_FALSE(range.fromStringClosed(badstr.c_str())); + CHECK_FALSE(range.isValid()); + if (range.isValid()) { + INFO(badstr.c_str()); + } + } +} diff --git a/plugins/experimental/ssl_session_reuse/src/ats_ssl_plugin.cc b/plugins/experimental/ssl_session_reuse/src/ats_ssl_plugin.cc index 8523b097acd..1a8e171a02e 100644 --- a/plugins/experimental/ssl_session_reuse/src/ats_ssl_plugin.cc +++ b/plugins/experimental/ssl_session_reuse/src/ats_ssl_plugin.cc @@ -22,13 +22,26 @@ */ -#include +#include #include #include #include #include "ssl_utils.h" + +PluginThreads plugin_threads; + int SSL_session_callback(TSCont contp, TSEvent event, void *edata); + +static int +shutdown_handler(TSCont contp, TSEvent event, void *edata) +{ + if ((event == TS_EVENT_LIFECYCLE_SHUTDOWN)) { + plugin_threads.terminate(); + } + return 0; +} + void TSPluginInit(int argc, const char *argv[]) { @@ -38,6 +51,8 @@ TSPluginInit(int argc, const char *argv[]) info.vendor_name = (char *)("ats"); info.support_email = (char *)("ats-devel@oath.com"); + TSLifecycleHookAdd(TS_LIFECYCLE_SHUTDOWN_HOOK, TSContCreate(shutdown_handler, nullptr)); + #if (TS_VERSION_NUMBER >= 7000000) if (TSPluginRegister(&info) != TS_SUCCESS) { TSError("Plugin registration failed."); diff --git a/plugins/experimental/ssl_session_reuse/src/config.cc b/plugins/experimental/ssl_session_reuse/src/config.cc index fd3dde1fb16..1aa089c5dce 100644 --- a/plugins/experimental/ssl_session_reuse/src/config.cc +++ b/plugins/experimental/ssl_session_reuse/src/config.cc @@ -23,7 +23,8 @@ */ #include -#include +#include +#include #include #include #include @@ -56,13 +57,16 @@ Config::loadConfig(const std::string &filename) m_filename = filename; - int fd = (this->m_filename.length() > 0 ? open(m_filename.c_str(), O_RDONLY) : 0); + int fd = (this->m_filename.length() > 0 ? open(m_filename.c_str(), O_RDONLY) : ts::NO_FD); struct stat info; - if (0 == fstat(fd, &info)) { + if (fd > 0 && 0 == fstat(fd, &info)) { size_t n = info.st_size; std::string config_data; config_data.resize(n); - read(fd, const_cast(config_data.data()), n); + if (read(fd, const_cast(config_data.data()), n) != static_cast(n)) { + close(fd); + return success; + } ts::TextView content(config_data); while (content) { @@ -77,10 +81,13 @@ Config::loadConfig(const std::string &filename) m_config[std::string(field.data(), field.size())] = std::string(line.data(), line.size()); } } + + close(fd); + + m_noConfig = false; + success = true; + m_alreadyLoaded = true; } - m_noConfig = false; - success = true; - m_alreadyLoaded = true; return success; } @@ -105,7 +112,7 @@ Config::setLastConfigChange() bool Config::configHasChanged() { - time_t checkTime = time(0) / cCheckDivisor; + time_t checkTime = time(nullptr) / cCheckDivisor; if (0 == m_lastmtime || m_lastCheck != checkTime) { m_lastCheck = checkTime; diff --git a/plugins/experimental/ssl_session_reuse/src/openssl_utils.cc b/plugins/experimental/ssl_session_reuse/src/openssl_utils.cc index 18efc9b1fe1..2909e728591 100644 --- a/plugins/experimental/ssl_session_reuse/src/openssl_utils.cc +++ b/plugins/experimental/ssl_session_reuse/src/openssl_utils.cc @@ -26,10 +26,10 @@ #include #include #include -#include +#include #include -#include -#include +#include +#include #include #include #include diff --git a/plugins/experimental/ssl_session_reuse/src/publish.cc b/plugins/experimental/ssl_session_reuse/src/publish.cc index 872920494d9..794260611e3 100644 --- a/plugins/experimental/ssl_session_reuse/src/publish.cc +++ b/plugins/experimental/ssl_session_reuse/src/publish.cc @@ -26,17 +26,22 @@ #include #include #include -#include +#include #include #include #include "common.h" #include "publisher.h" #include "Config.h" #include "redis_auth.h" +#include "ssl_utils.h" void * RedisPublisher::start_worker_thread(void *arg) { + plugin_threads.store(::pthread_self()); + ::pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, nullptr); + ::pthread_setcanceltype(PTHREAD_CANCEL_ASYNCHRONOUS, nullptr); + RedisPublisher *publisher = static_cast(arg); publisher->runWorker(); return arg; @@ -44,14 +49,14 @@ RedisPublisher::start_worker_thread(void *arg) RedisPublisher::RedisPublisher(const std::string &conf) : m_redisEndpointsStr(cDefaultRedisEndpoint), - m_endpointIndex(0), + m_numWorkers(cPubNumWorkerThreads), m_redisConnectTimeout(cDefaultRedisConnectTimeout), m_redisConnectTries(cDefaultRedisConnectTries), m_redisPublishTries(cDefaultRedisPublishTries), m_redisRetryDelay(cDefaultRedisRetryDelay), - m_maxQueuedMessages(cDefaultMaxQueuedMessages), - err(false) + m_maxQueuedMessages(cDefaultMaxQueuedMessages) + { if (Config::getSingleton().loadConfig(conf)) { Config::getSingleton().getValue("pubconfig", "PubNumWorkers", m_numWorkers); @@ -239,6 +244,9 @@ RedisPublisher::runWorker() } Message ¤t_message(m_messageQueue.front()); + if (!current_message.cleanup) { + m_messageQueue.pop_front(); + } m_messageQueueMutex.unlock(); ::sem_wait(&m_workerSem); @@ -264,10 +272,6 @@ RedisPublisher::runWorker() } } - if (!current_message.cleanup) { - m_messageQueue.pop_front(); - } - clear_reply(current_reply); current_reply = nullptr; } diff --git a/plugins/experimental/ssl_session_reuse/src/publisher.h b/plugins/experimental/ssl_session_reuse/src/publisher.h index 2d1a1a190d0..82fec012efd 100644 --- a/plugins/experimental/ssl_session_reuse/src/publisher.h +++ b/plugins/experimental/ssl_session_reuse/src/publisher.h @@ -55,7 +55,7 @@ class RedisPublisher std::vector m_redisEndpoints; std::string m_redisEndpointsStr; - int m_endpointIndex; + int m_endpointIndex = 0; std::mutex m_endpointIndexMutex; std::vector pools; @@ -68,7 +68,7 @@ class RedisPublisher unsigned int m_maxQueuedMessages; unsigned int m_poolRedisConnectTimeout; // milliseconds - bool err; + bool err = false; void runWorker(); diff --git a/plugins/experimental/ssl_session_reuse/src/redis_endpoint.h b/plugins/experimental/ssl_session_reuse/src/redis_endpoint.h index 77584e4f2ac..2762d9b80fc 100644 --- a/plugins/experimental/ssl_session_reuse/src/redis_endpoint.h +++ b/plugins/experimental/ssl_session_reuse/src/redis_endpoint.h @@ -38,7 +38,7 @@ typedef struct redis_endpoint { typedef struct redis_endpoint_compare { bool - operator()(const RedisEndpoint &lhs, const RedisEndpoint &rhs) + operator()(const RedisEndpoint &lhs, const RedisEndpoint &rhs) const { return lhs.m_hostname < rhs.m_hostname || (lhs.m_hostname == rhs.m_hostname && lhs.m_port < rhs.m_port); } diff --git a/plugins/experimental/ssl_session_reuse/src/session_process.cc b/plugins/experimental/ssl_session_reuse/src/session_process.cc index bd48aef95a8..440cd66bccb 100644 --- a/plugins/experimental/ssl_session_reuse/src/session_process.cc +++ b/plugins/experimental/ssl_session_reuse/src/session_process.cc @@ -22,9 +22,9 @@ */ -#include -#include -#include +#include +#include +#include #include #include #include @@ -191,7 +191,7 @@ decrypt_session(const std::string &encrypted_data, const unsigned char *key, int } int -decode_id(std::string encoded_id, char *decoded_data, int &decoded_data_len) +decode_id(const std::string &encoded_id, char *decoded_data, int &decoded_data_len) { size_t decode_len = 0; memset(decoded_data, 0, decoded_data_len); diff --git a/plugins/experimental/ssl_session_reuse/src/session_process.h b/plugins/experimental/ssl_session_reuse/src/session_process.h index 8bd9cc3346f..a676db2b3ab 100644 --- a/plugins/experimental/ssl_session_reuse/src/session_process.h +++ b/plugins/experimental/ssl_session_reuse/src/session_process.h @@ -24,7 +24,7 @@ #pragma once #include -#include +#include #define SSL_SESSION_MAX_DER 1024 * 10 @@ -42,6 +42,6 @@ int decrypt_session(const std::string &encrypted_data, const unsigned char *key, int32_t &session_len); int encode_id(const char *id, int idlen, std::string &decoded_data); -int decode_id(std::string encoded_id, char *decoded_data, int &decoded_data_len); +int decode_id(const std::string &encoded_id, char *decoded_data, int &decoded_data_len); int add_session(char *session_id, int session_id_len, const std::string &encrypted_session); diff --git a/plugins/experimental/ssl_session_reuse/src/ssl_init.cc b/plugins/experimental/ssl_session_reuse/src/ssl_init.cc index 3cd105f78e2..56496aa6367 100644 --- a/plugins/experimental/ssl_session_reuse/src/ssl_init.cc +++ b/plugins/experimental/ssl_session_reuse/src/ssl_init.cc @@ -23,6 +23,7 @@ */ #include +#include #include "ssl_utils.h" #include "Config.h" #include "common.h" @@ -82,7 +83,7 @@ init_ssl_params(const std::string &conf) return 0; } -ssl_session_param::ssl_session_param() : pub(nullptr) {} +ssl_session_param::ssl_session_param() {} ssl_session_param::~ssl_session_param() { diff --git a/plugins/experimental/ssl_session_reuse/src/ssl_key_utils.cc b/plugins/experimental/ssl_session_reuse/src/ssl_key_utils.cc index 596c8dc7f0d..323fe0768f1 100644 --- a/plugins/experimental/ssl_session_reuse/src/ssl_key_utils.cc +++ b/plugins/experimental/ssl_session_reuse/src/ssl_key_utils.cc @@ -23,14 +23,14 @@ */ #include -#include +#include #include #include #include #include #include "ssl_utils.h" -#include "assert.h" +#include #include "redis_auth.h" #include "stek.h" #include "common.h" @@ -284,6 +284,10 @@ STEK_Send_To_Network(struct ssl_ticket_key_t *stekToSend) static void * STEK_Update_Setter_Thread(void *arg) { + plugin_threads.store(::pthread_self()); + ::pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, nullptr); + ::pthread_setcanceltype(PTHREAD_CANCEL_ASYNCHRONOUS, nullptr); + int sleepInterval; struct ssl_ticket_key_t newKey; int startProblem = 0; // counter for start up and retry issues. @@ -303,7 +307,7 @@ STEK_Update_Setter_Thread(void *arg) stek_master_setter_running = 1; TSDebug(PLUGIN, "Will now act as the STEK rotator for pod"); - while (1) { + while (true) { // Create new STEK, set it for me, and then send it to the POD. if ((!STEK_CreateNew(&newKey, 0, 1 /* entropy ensured */)) || (!STEK_Send_To_Network(&newKey))) { // Error occurred. We will retry after a short interval. @@ -368,6 +372,10 @@ STEK_update(const std::string &encrypted_stek) static void * STEK_Update_Checker_Thread(void *arg) { + plugin_threads.store(::pthread_self()); + ::pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, nullptr); + ::pthread_setcanceltype(PTHREAD_CANCEL_ASYNCHRONOUS, nullptr); + time_t currentTime; time_t lastWarningTime; // last time we put out a warning @@ -381,7 +389,7 @@ STEK_Update_Checker_Thread(void *arg) lastChangeTime = lastWarningTime = time(¤tTime); // init to current to supress a startup warning. - while (1) { + while (true) { if (!stek_initialized && ssl_param.pub) { // Launch a request for the master to resend you the ticket key std::string redis_channel = ssl_param.cluster_name + "." + STEK_ID_RESEND; diff --git a/plugins/experimental/ssl_session_reuse/src/ssl_utils.h b/plugins/experimental/ssl_session_reuse/src/ssl_utils.h index 5711ff831b2..262728cb758 100644 --- a/plugins/experimental/ssl_session_reuse/src/ssl_utils.h +++ b/plugins/experimental/ssl_session_reuse/src/ssl_utils.h @@ -26,6 +26,8 @@ #include #include #include +#include +#include #include "publisher.h" #include "subscriber.h" @@ -38,13 +40,42 @@ struct ssl_session_param { int stek_master; // bool - Am I the STEK setter/rotator for POD? ssl_ticket_key_t ticket_keys[2]; // current and past STEK std::string redis_auth_key_file; - RedisPublisher *pub; + RedisPublisher *pub = nullptr; RedisSubscriber *sub; ssl_session_param(); ~ssl_session_param(); }; +class PluginThreads +{ +public: + void + store(const pthread_t &th) + { + std::lock_guard lock(threads_mutex); + threads_queue.push_back(th); + } + + void + terminate() + { + std::lock_guard lock(threads_mutex); + for (pthread_t th : threads_queue) { + ::pthread_cancel(th); + } + while (!threads_queue.empty()) { + pthread_t th = threads_queue.front(); + ::pthread_join(th, nullptr); + threads_queue.pop_front(); + } + } + +private: + std::deque threads_queue; + std::mutex threads_mutex; +}; + int STEK_init_keys(); const char *get_key_ptr(); @@ -66,3 +97,5 @@ int init_subscriber(); int SSL_session_callback(TSCont contp, TSEvent event, void *edata); extern ssl_session_param ssl_param; // almost everything one needs is stored in here + +extern PluginThreads plugin_threads; diff --git a/plugins/experimental/ssl_session_reuse/src/subscriber.cc b/plugins/experimental/ssl_session_reuse/src/subscriber.cc index 913aecd3260..7dfc5560350 100644 --- a/plugins/experimental/ssl_session_reuse/src/subscriber.cc +++ b/plugins/experimental/ssl_session_reuse/src/subscriber.cc @@ -26,6 +26,7 @@ #include #include #include +#include #include "common.h" #include "subscriber.h" @@ -38,6 +39,10 @@ void * setup_subscriber(void *arg) { + plugin_threads.store(::pthread_self()); + ::pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, nullptr); + ::pthread_setcanceltype(PTHREAD_CANCEL_ASYNCHRONOUS, nullptr); + RedisSubscriber *me = static_cast(arg); me->run(); return (void *)1; diff --git a/plugins/experimental/sslheaders/expand.cc b/plugins/experimental/sslheaders/expand.cc index 5d99853ea6f..647bdbc6cdf 100644 --- a/plugins/experimental/sslheaders/expand.cc +++ b/plugins/experimental/sslheaders/expand.cc @@ -113,7 +113,7 @@ static const std::array expansions = {{ x509_expand_serial, // SSL_HEADERS_FIELD_SERIAL x509_expand_signature, // SSL_HEADERS_FIELD_SIGNATURE x509_expand_notbefore, // SSL_HEADERS_FIELD_NOTBEFORE - x509_expand_notafter, // SSL_HEADERS_FIELD_NOTBEFORE + x509_expand_notafter, // SSL_HEADERS_FIELD_NOTAFTER }}; bool diff --git a/plugins/experimental/sslheaders/sslheaders.cc b/plugins/experimental/sslheaders/sslheaders.cc index 085c7ac4b05..bbda70d304b 100644 --- a/plugins/experimental/sslheaders/sslheaders.cc +++ b/plugins/experimental/sslheaders/sslheaders.cc @@ -116,6 +116,49 @@ SslHdrSetHeader(TSMBuffer mbuf, TSMLoc mhdr, const std::string &name, BIO *value } } +namespace +{ +template class WrapX509 +{ +public: + WrapX509(SSL *ssl) : _ssl(ssl), _x509(_nonNullInvalidValue()) {} + + X509 * + get() + { + if (_x509 == _nonNullInvalidValue()) { + _set(); + } + + return _x509; + } + + ~WrapX509() + { + if (IsClient && (_x509 != _nonNullInvalidValue()) && (_x509 != nullptr)) { + X509_free(_x509); + } + } + +private: + SSL *_ssl; + X509 *_x509; + + // The address of this object can not be a valid X509 structure address. + X509 * + _nonNullInvalidValue() const + { + return reinterpret_cast(const_cast(this)); + } + + void + _set() + { + _x509 = (IsClient ? SSL_get_peer_certificate : SSL_get_certificate)(_ssl); + } +}; +} // end anonymous namespace + // Process SSL header expansions. If this is not an SSL connection, then we need to delete the SSL headers // so that malicious clients cannot inject bogus information. Otherwise, we populate the header with the // expanded value. If the value expands to something empty, we nuke the header. @@ -124,38 +167,39 @@ SslHdrExpand(SSL *ssl, const SslHdrInstance::expansion_list &expansions, TSMBuff { if (ssl == nullptr) { for (const auto &expansion : expansions) { - SslHdrRemoveHeader(mbuf, mhdr, expansion->name); + SslHdrRemoveHeader(mbuf, mhdr, expansion.name); } } else { + WrapX509 clientX509(ssl); + WrapX509 serverX509(ssl); X509 *x509; + BIO *exp = BIO_new(BIO_s_mem()); for (const auto &expansion : expansions) { - switch (expansion->scope) { + switch (expansion.scope) { case SSL_HEADERS_SCOPE_CLIENT: - x509 = SSL_get_peer_certificate(ssl); + x509 = clientX509.get(); + if (x509 == nullptr) { + SslHdrRemoveHeader(mbuf, mhdr, expansion.name); + continue; + } break; case SSL_HEADERS_SCOPE_SERVER: - x509 = SSL_get_certificate(ssl); + x509 = serverX509.get(); + if (x509 == nullptr) { + continue; + } break; default: - x509 = nullptr; - } - - if (x509 == nullptr) { continue; } - SslHdrExpandX509Field(exp, x509, expansion->field); + SslHdrExpandX509Field(exp, x509, expansion.field); if (BIO_pending(exp)) { - SslHdrSetHeader(mbuf, mhdr, expansion->name, exp); + SslHdrSetHeader(mbuf, mhdr, expansion.name, exp); } else { - SslHdrRemoveHeader(mbuf, mhdr, expansion->name); - } - - // Getting the peer certificate takes a reference count, but the server certificate doesn't. - if (x509 && expansion->scope == SSL_HEADERS_SCOPE_CLIENT) { - X509_free(x509); + SslHdrRemoveHeader(mbuf, mhdr, expansion.name); } } @@ -199,14 +243,12 @@ SslHdrParseOptions(int argc, const char **argv) } // Pick up the remaining options as SSL header expansions. + hdr->expansions.resize(argc - optind); for (int i = optind; i < argc; ++i) { - SslHdrExpansion exp; - if (!SslHdrParseExpansion(argv[i], exp)) { + if (!SslHdrParseExpansion(argv[i], hdr->expansions[i - optind])) { // If we fail, the expansion parsing logs the error. return nullptr; } - - hdr->expansions.push_back(&exp); } return hdr.release(); @@ -232,7 +274,7 @@ TSPluginInit(int argc, const char *argv[]) case SSL_HEADERS_ATTACH_SERVER: TSHttpHookAdd(TS_HTTP_SEND_REQUEST_HDR_HOOK, hdr->cont); break; - case SSL_HEADERS_ATTACH_BOTH: /* fallthru */ + case SSL_HEADERS_ATTACH_BOTH: /* fallthrough */ case SSL_HEADERS_ATTACH_CLIENT: TSHttpHookAdd(TS_HTTP_READ_REQUEST_HDR_HOOK, hdr->cont); TSHttpHookAdd(TS_HTTP_SEND_REQUEST_HDR_HOOK, hdr->cont); @@ -277,7 +319,7 @@ TSRemapDoRemap(void *instance, TSHttpTxn txn, TSRemapRequestInfo * /* rri */) case SSL_HEADERS_ATTACH_SERVER: TSHttpTxnHookAdd(txn, TS_HTTP_SEND_REQUEST_HDR_HOOK, hdr->cont); break; - case SSL_HEADERS_ATTACH_BOTH: /* fallthru */ + case SSL_HEADERS_ATTACH_BOTH: /* fallthrough */ case SSL_HEADERS_ATTACH_CLIENT: TSHttpTxnHookAdd(txn, TS_HTTP_READ_REQUEST_HDR_HOOK, hdr->cont); TSHttpTxnHookAdd(txn, TS_HTTP_SEND_REQUEST_HDR_HOOK, hdr->cont); @@ -287,8 +329,7 @@ TSRemapDoRemap(void *instance, TSHttpTxn txn, TSRemapRequestInfo * /* rri */) return TSREMAP_NO_REMAP; } -SslHdrInstance::SslHdrInstance() - : expansions(), attach(SSL_HEADERS_ATTACH_SERVER), cont(TSContCreate(SslHdrExpandRequestHook, nullptr)) +SslHdrInstance::SslHdrInstance() : expansions(), cont(TSContCreate(SslHdrExpandRequestHook, nullptr)) { TSContDataSet(cont, this); } diff --git a/plugins/experimental/sslheaders/sslheaders.h b/plugins/experimental/sslheaders/sslheaders.h index b7b66ddbaa1..99f8fbd4d64 100644 --- a/plugins/experimental/sslheaders/sslheaders.h +++ b/plugins/experimental/sslheaders/sslheaders.h @@ -19,7 +19,7 @@ #include #include #include -#include +#include #include extern "C" { @@ -62,24 +62,26 @@ enum ExpansionField { }; struct SslHdrExpansion { - SslHdrExpansion() : name(), scope(SSL_HEADERS_SCOPE_NONE), field(SSL_HEADERS_FIELD_NONE) {} + SslHdrExpansion() : name() {} std::string name; // HTTP header name - ExpansionScope scope; - ExpansionField field; + ExpansionScope scope = SSL_HEADERS_SCOPE_NONE; + ExpansionField field = SSL_HEADERS_FIELD_NONE; - // noncopyable + // noncopyable but movable SslHdrExpansion(const SslHdrExpansion &) = delete; SslHdrExpansion &operator=(const SslHdrExpansion &) = delete; + SslHdrExpansion(SslHdrExpansion &&) = default; + SslHdrExpansion &operator=(SslHdrExpansion &&) = default; }; struct SslHdrInstance { - typedef std::list expansion_list; + typedef std::vector expansion_list; SslHdrInstance(); ~SslHdrInstance(); expansion_list expansions; - AttachOptions attach; + AttachOptions attach = SSL_HEADERS_ATTACH_SERVER; TSCont cont; void register_hooks(); diff --git a/plugins/experimental/sslheaders/util.cc b/plugins/experimental/sslheaders/util.cc index 0ccf954621b..14e215ffb1c 100644 --- a/plugins/experimental/sslheaders/util.cc +++ b/plugins/experimental/sslheaders/util.cc @@ -78,9 +78,9 @@ SslHdrParseExpansion(const char *spec, SslHdrExpansion &exp) // Push sep to point to the field selector. selector = sep + 1; - for (unsigned i = 0; i < fields.size(); ++i) { - if (strcmp(selector, fields[i].name) == 0) { - exp.field = fields[i].field; + for (auto field : fields) { + if (strcmp(selector, field.name) == 0) { + exp.field = field.field; return true; } } diff --git a/plugins/experimental/stale_while_revalidate/stale_while_revalidate.c b/plugins/experimental/stale_while_revalidate/stale_while_revalidate.c index c49eb4043d0..fdcc4669d92 100644 --- a/plugins/experimental/stale_while_revalidate/stale_while_revalidate.c +++ b/plugins/experimental/stale_while_revalidate/stale_while_revalidate.c @@ -597,7 +597,7 @@ main_plugin(TSCont cont, TSEvent event, void *edata) fetch_cont = TSContCreate(fetch_resource, TSMutexCreate()); TSContDataSet(fetch_cont, (void *)state); - TSContSchedule(fetch_cont, 0, TS_THREAD_POOL_NET); + TSContScheduleOnPool(fetch_cont, 0, TS_THREAD_POOL_NET); TSHttpTxnReenable(txn, TS_EVENT_HTTP_CONTINUE); } else if ((state->txn_start - chi->date) < (chi->max_age + chi->stale_on_error)) { TSDebug(PLUGIN_NAME, "Looks like we can return fresh data on 500 error"); @@ -609,7 +609,7 @@ main_plugin(TSCont cont, TSEvent event, void *edata) state->main_cont = cont; // we need this for the warning header callback. not sure i like it, but it works. fetch_cont = TSContCreate(fetch_resource, TSMutexCreate()); TSContDataSet(fetch_cont, (void *)state); - TSContSchedule(fetch_cont, 0, TS_THREAD_POOL_NET); + TSContScheduleOnPool(fetch_cont, 0, TS_THREAD_POOL_NET); } else { TSDebug(PLUGIN_NAME, "No love? now: %d date: %d max-age: %d swr: %d soe: %d", (int)state->txn_start, (int)chi->date, (int)chi->max_age, (int)chi->stale_while_revalidate, (int)chi->stale_on_error); diff --git a/plugins/experimental/stream_editor/stream_editor.cc b/plugins/experimental/stream_editor/stream_editor.cc index ee079673f8d..7451c91946b 100644 --- a/plugins/experimental/stream_editor/stream_editor.cc +++ b/plugins/experimental/stream_editor/stream_editor.cc @@ -551,17 +551,17 @@ using ruleset_t = std::vector; using rule_p = ruleset_t::const_iterator; typedef struct contdata_t { - TSCont cont; - TSIOBuffer out_buf; - TSIOBufferReader out_rd; - TSVIO out_vio; + TSCont cont = nullptr; + TSIOBuffer out_buf = nullptr; + TSIOBufferReader out_rd = nullptr; + TSVIO out_vio = nullptr; ruleset_t rules; std::string contbuf; - size_t contbuf_sz; - int64_t bytes_in; - int64_t bytes_out; + size_t contbuf_sz = 0; + int64_t bytes_in = 0; + int64_t bytes_out = 0; /* Use new/delete so destructor does cleanup for us */ - contdata_t() : cont(nullptr), out_buf(nullptr), out_rd(nullptr), out_vio(nullptr), contbuf_sz(0), bytes_in(0), bytes_out(0) {} + contdata_t() {} ~contdata_t() { if (out_rd) { @@ -849,10 +849,10 @@ TSPluginInit(int argc, const char *argv[]) } if (rewrites_in != nullptr) { - TSDebug("[stream-editor]", "initialising input filtering"); + TSDebug("[stream-editor]", "initializing input filtering"); inputcont = TSContCreate(streamedit_setup, nullptr); if (inputcont == nullptr) { - TSError("[stream-editor] failed to initialise input filtering!"); + TSError("[stream-editor] failed to initialize input filtering!"); } else { TSContDataSet(inputcont, rewrites_in); TSHttpHookAdd(TS_HTTP_READ_REQUEST_HDR_HOOK, inputcont); @@ -862,10 +862,10 @@ TSPluginInit(int argc, const char *argv[]) } if (rewrites_out != nullptr) { - TSDebug("[stream-editor]", "initialising output filtering"); + TSDebug("[stream-editor]", "initializing output filtering"); outputcont = TSContCreate(streamedit_setup, nullptr); if (outputcont == nullptr) { - TSError("[stream-editor] failed to initialise output filtering!"); + TSError("[stream-editor] failed to initialize output filtering!"); } else { TSContDataSet(outputcont, rewrites_out); TSHttpHookAdd(TS_HTTP_READ_RESPONSE_HDR_HOOK, outputcont); diff --git a/plugins/experimental/system_stats/system_stats.c b/plugins/experimental/system_stats/system_stats.c index d25dada7d6f..8bd615f77e8 100644 --- a/plugins/experimental/system_stats/system_stats.c +++ b/plugins/experimental/system_stats/system_stats.c @@ -135,7 +135,7 @@ setNetStat(TSMutex stat_creation_mutex, const char *interface, const char *entry // Generate the ATS stats name snprintf(&stat_name[0], sizeof(stat_name), "%s%s.%s", NET_STATS, interface, entry); - // Determine if this is a toplevel netdev stat, or one from stastistics. + // Determine if this is a toplevel netdev stat, or one from statistics. if (subdir == NULL) { snprintf(&sysfs_name[0], sizeof(sysfs_name), "%s/%s/%s", NET_STATS_DIR, interface, entry); } else { @@ -224,7 +224,7 @@ systemStatsContCB(TSCont cont, TSEvent event ATS_UNUSED, void *edata) stat_creation_mutex = TSContMutexGet(cont); getStats(stat_creation_mutex); - TSContSchedule(cont, SYSTEM_STATS_TIMEOUT, TS_THREAD_POOL_TASK); + TSContScheduleOnPool(cont, SYSTEM_STATS_TIMEOUT, TS_THREAD_POOL_TASK); TSDebug(DEBUG_TAG, "finished %s", __FUNCTION__); return 0; @@ -253,7 +253,7 @@ TSPluginInit(int argc, const char *argv[]) // We want our first hit immediate to populate the stats, // Subsequent schedules done within the function will be for // 5 seconds. - TSContSchedule(stats_cont, 0, TS_THREAD_POOL_TASK); + TSContScheduleOnPool(stats_cont, 0, TS_THREAD_POOL_TASK); TSDebug(DEBUG_TAG, "Init complete"); } diff --git a/plugins/experimental/tls_bridge/tls_bridge.cc b/plugins/experimental/tls_bridge/tls_bridge.cc index 0a699265700..417e984c957 100644 --- a/plugins/experimental/tls_bridge/tls_bridge.cc +++ b/plugins/experimental/tls_bridge/tls_bridge.cc @@ -1,19 +1,16 @@ /* - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at + Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. + See the NOTICE file distributed with this work for additional information regarding copyright + ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance with the License. You may obtain a + copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. + Unless required by applicable law or agreed to in writing, software distributed under the License + is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + or implied. See the License for the specific language governing permissions and limitations under + the License. */ #include "ts/ts.h" @@ -21,17 +18,25 @@ #include #include #include "tscpp/util/TextView.h" +#include "tscore/ts_file.h" #include "regex.h" -#define PLUGIN_NAME "TLS Bridge" -#define PLUGIN_TAG "tls-bridge" - using ts::TextView; +namespace +{ +constexpr char const PLUGIN_NAME[] = "TLS Bridge"; +constexpr char const PLUGIN_TAG[] = "tls_bridge"; + // Base format string for making the internal CONNECT. char const CONNECT_FORMAT[] = "CONNECT https:%.*s HTTP/1.1\r\n\r\n"; +// TextView of the 'CONNECT' method string. const TextView METHOD_CONNECT{TS_HTTP_METHOD_CONNECT, TS_HTTP_LEN_CONNECT}; +constexpr TextView CONFIG_FILE_ARG{"--file"}; +const std::string TS_CONFIG_DIR{TSConfigDirGet()}; + +}; // namespace /* ------------------------------------------------------------------------------------ */ // Utility functions @@ -64,11 +69,13 @@ class BridgeConfig using self_type = BridgeConfig; /// Construct an item. - Item(const char *pattern, Regex &&r, const char *dest) : _pattern(pattern), _r(std::move(r)), _dest(dest) {} + /// @internal Pass in the compiled regex because no instance of this is created if + /// the regex doesn't compile successfully. + Item(std::string_view pattern, Regex &&r, std::string_view service) : _pattern(pattern), _r(std::move(r)), _service(service) {} std::string _pattern; ///< Original configuration regular expression. Regex _r; ///< Compiled regex. - std::string _dest; ///< Destination if matched. + std::string _service; ///< Destination service if matched. }; public: @@ -83,6 +90,14 @@ class BridgeConfig private: /// Configuration item storage. std::vector _items; + + /** Load a configuration item pair. + * + * @param rxp The regular expression to match. + * @param service The destination service. + * @param ln Line number, or 0 if from plugin.config. + */ + void load_pair(std::string_view rxp, std::string_view service, ts::file::path const &src, int ln = 0); }; inline int @@ -91,18 +106,77 @@ BridgeConfig::count() const return _items.size(); } +void +BridgeConfig::load_pair(std::string_view rxp, std::string_view service, ts::file::path const &src, int ln) +{ + Regex r; + // Unfortunately PCRE can only compile null terminated strings... + std::string pattern{rxp}; + if (r.compile(pattern.c_str(), Regex::ANCHORED)) { + _items.emplace_back(rxp, std::move(r), service); + } else { + char buff[std::numeric_limits::digits10 + 2] = ""; + if (ln) { + snprintf(buff, sizeof(buff), " on line %d", ln); + } + TSError("[%s] Failed to compile regular expression '%.*s' in %s%s", PLUGIN_NAME, int(rxp.size()), rxp.data(), src.c_str(), + buff); + } +} + void BridgeConfig::load_config(int argc, const char *argv[]) { + static const ts::file::path plugin_config_fp{"plugin.config"}; + for (int i = 0; i < argc; i += 2) { - Regex r; - if (i + 1 >= argc) { - TSError("%s: Destination regular expression without peer", PLUGIN_TAG); + if (argv[i] == CONFIG_FILE_ARG) { + if (i + 1 >= argc) { + TSError("[%s] Invalid '%.*s' argument - no file name found.", PLUGIN_NAME, int(CONFIG_FILE_ARG.size()), + CONFIG_FILE_ARG.data()); + } else { + ts::file::path fp(argv[i + 1]); + std::error_code ec; + if (!fp.is_absolute()) { + fp = ts::file::path{TS_CONFIG_DIR} / fp; // slap the config dir on it to make it absolute. + } + // bulk load the file. + std::string content{ts::file::load(fp, ec)}; + if (ec) { + TSError("[%s] Invalid '%.*s' argument - unable to read file '%s' : %s.", PLUGIN_NAME, int(CONFIG_FILE_ARG.size()), + CONFIG_FILE_ARG.data(), fp.c_str(), ec.message().c_str()); + + } else { + // walk the lines. + int line_no = 0; + TextView src{content}; + while (!src.empty()) { + TextView line{src.take_prefix_at('\n').trim_if(&isspace)}; + ++line_no; + if (line.empty() || '#' == *line) + continue; // empty or comment, ignore. + + // Pick apart the line into the regular expression and destination service. + TextView service{line}; + TextView rxp{service.take_prefix_if(&isspace)}; + service.ltrim_if(&isspace); // dump extra separating space. + // Only need to check service, as if the line isn't empty rxp will also be non-empty. + if (service.empty()) { + TSError("[%s] Invalid line %d in '%s' - no destination service found.", PLUGIN_NAME, line_no, fp.c_str()); + } else { + this->load_pair(rxp, service, fp, line_no); + } + } + } + } + } else if (argv[i][0] == '-') { + TSError("[%s] Unrecognized option '%s'", PLUGIN_NAME, argv[i]); + i -= 1; // Don't skip next arg. } else { - if (r.compile(argv[i]), Regex::ANCHORED) { - _items.emplace_back(argv[i], std::move(r), argv[i + 1]); + if (i + 1 >= argc) { + TSError("[%s] Regular expression '%s' without destination service", PLUGIN_NAME, argv[i]); } else { - TSError("%s: Failed to compile regular expression '%s'", PLUGIN_TAG, argv[i]); + this->load_pair(argv[i], argv[i + 1], plugin_config_fp); } } } @@ -113,7 +187,7 @@ BridgeConfig::match(TextView name) { for (auto &item : _items) { if (item._r.exec(name)) { - return {item._dest}; + return {item._service}; } } return {}; @@ -186,8 +260,6 @@ struct Bridge { TSHttpStatus _out_response_code = TS_HTTP_STATUS_NONE; /// Response reason, if not TS_HTTP_STATUS_OK std::string _out_response_reason; - /// Is the response to the user agent suspended? - bool _ua_response_suspended = false; /// Bridge requires a continuation for scheduling and the transaction. Bridge(TSCont cont, TSHttpTxn txn, TextView peer); @@ -229,9 +301,9 @@ void Bridge::net_accept(TSVConn vc) { char buff[1024]; - int64_t n = snprintf(buff, sizeof(buff), CONNECT_FORMAT, static_cast(_peer.size()), _peer.data()); + int64_t n = snprintf(buff, sizeof(buff), CONNECT_FORMAT, int(_peer.size()), _peer.data()); - TSDebug(PLUGIN_TAG, "Received UA VConn"); + TSDebug(PLUGIN_TAG, "Received UA VConn, connecting to peer %.*s", int(_peer.size()), _peer.data()); // UA side intercepted. _ua.init(vc); _ua.do_read(_self_cont, INT64_MAX); @@ -315,12 +387,6 @@ Bridge::check_outbound_OK() } // 519 is POOMA, useful for debugging, but may want to change this later. _out_response_code = c ? c : static_cast(519); - if (_ua_response_suspended) { - this->update_ua_response(); - TSHttpTxnReenable(_ua_txn, TS_EVENT_HTTP_CONTINUE); - _ua_response_suspended = false; - TSDebug(PLUGIN_TAG, "TXN resumed"); - } _out.consume(block.data() - raw.data()); zret = true; TSDebug(PLUGIN_TAG, "Outbound status %d", c); @@ -406,33 +472,30 @@ Bridge::flow_to_outbound() void Bridge::eos(TSVIO vio) { - if (vio == _out._write._vio || vio == _out._read._vio) { + if (nullptr == vio) { + // Generic close for some non-EOS reason. + } else if (vio == _out._write._vio || vio == _out._read._vio) { TSDebug(PLUGIN_TAG, "EOS upstream"); } else if (vio == _ua._write._vio || vio == _ua._read._vio) { TSDebug(PLUGIN_TAG, "EOS user agent"); } else { - TSDebug(PLUGIN_TAG, "EOS from unknown VIO"); + TSDebug(PLUGIN_TAG, "EOS from unknown VIO [%p]", vio); } _out.do_close(); _ua.do_close(); - _out_resp_state = EOS; - if (_ua_response_suspended) { - TSHttpTxnReenable(_ua_txn, TS_EVENT_HTTP_CONTINUE); + if (_out_resp_state != ERROR) { + _out_resp_state = EOS; } } void Bridge::send_response_cb() { - // If the upstream response hasn't been parsed yet, make the UA response wait for that. - // Set a flag so the upstream response parser knows to update response and reenable. - if (_out_resp_state < OK) { - _ua_response_suspended = true; - TSDebug(PLUGIN_TAG, "TXN suspended"); - } else { // Already have all the data needed to do the update, so do it and move on. - this->update_ua_response(); - TSHttpTxnReenable(_ua_txn, TS_EVENT_HTTP_CONTINUE); - } + // This happens either after the upstream connection and the writing the response there, + // or because the upstream connection was blocked. In either case the upstream work is + // done and the original transaction can proceed. + this->update_ua_response(); + TSHttpTxnReenable(_ua_txn, TS_EVENT_HTTP_CONTINUE); } void @@ -441,12 +504,10 @@ Bridge::update_ua_response() TSMBuffer mbuf; TSMLoc hdr_loc; if (TS_SUCCESS == TSHttpTxnClientRespGet(_ua_txn, &mbuf, &hdr_loc)) { - // A 200 for @a out_response_code only means there wasn't an internal failure on the upstream - // CONNECT. Network and other failures get reported in this response. This response code will - // be more accurate, so use it unless it's 200, in which case use the stored response code if - // that's not 200. - TSHttpStatus status = TSHttpHdrStatusGet(mbuf, hdr_loc); - if (TS_HTTP_STATUS_OK == status && TS_HTTP_STATUS_OK != _out_response_code) { + // If there is a non-200 upstream code then that's the most accurate because it was from + // an actual upstream connection. Otherwise, let the original connection response code + // ride. + if (_out_response_code != TS_HTTP_STATUS_OK && _out_response_code != TS_HTTP_STATUS_NONE) { TSHttpHdrStatusSet(mbuf, hdr_loc, _out_response_code); if (!_out_response_reason.empty()) { TSHttpHdrReasonSet(mbuf, hdr_loc, _out_response_reason.data(), _out_response_reason.size()); @@ -543,7 +604,8 @@ int CB_Exec(TSCont contp, TSEvent ev_idx, void *data) { auto ctx = static_cast(TSContDataGet(contp)); - + // No point in checking @a ctx for @c nullptr because if it's not there, neither is the + // continuation so things would already be over the cliff. switch (ev_idx) { case TS_EVENT_NET_ACCEPT: ctx->net_accept(static_cast(data)); @@ -567,7 +629,10 @@ CB_Exec(TSCont contp, TSEvent ev_idx, void *data) break; case TS_EVENT_HTTP_TXN_CLOSE: TSDebug(PLUGIN_TAG, "TXN_CLOSE: cleanup"); + ctx->eos(nullptr); // no specific VIO, close up everything. delete ctx; + TSContDataSet(contp, nullptr); // Not sure if necessary, it's cheap, let's be safe. + TSContDestroy(contp); break; default: TSDebug(PLUGIN_TAG, "Event %d", ev_idx); @@ -606,6 +671,8 @@ CB_Read_Request_Hdr(TSCont contp, TSEvent ev_idx, void *data) TSHttpTxnHookAdd(txn, TS_HTTP_SEND_RESPONSE_HDR_HOOK, actor); // Arrange for cleanup. TSHttpTxnHookAdd(txn, TS_HTTP_TXN_CLOSE_HOOK, actor); + // Skip remap and remap rule requirement - authorized by TLS bridge config. + TSSkipRemappingSet(txn, 1); // Grab the transaction TSHttpTxnIntercept(actor, txn); } @@ -624,12 +691,12 @@ TSPluginInit(int argc, char const *argv[]) TSPluginRegistrationInfo info{PLUGIN_NAME, "Oath:", "solidwallofcode@oath.com"}; if (TSPluginRegister(&info) != TS_SUCCESS) { - TSError(PLUGIN_NAME ": plugin registration failed."); + TSError("[%s] plugin registration failed.", PLUGIN_NAME); } Config.load_config(argc - 1, argv + 1); if (Config.count() <= 0) { - TSError("%s: No destinations defined, plugin disabled", PLUGIN_TAG); + TSError("[%s] No destinations defined, plugin disabled", PLUGIN_NAME); } TSCont contp = TSContCreate(CB_Read_Request_Hdr, TSMutexCreate()); diff --git a/plugins/experimental/traffic_dump/README b/plugins/experimental/traffic_dump/README index d4e13444139..4cf2e0b1e9d 100644 --- a/plugins/experimental/traffic_dump/README +++ b/plugins/experimental/traffic_dump/README @@ -11,6 +11,13 @@ Traffic Dump is a global plugin and is configured by arguments in plugin.config. --sample The sampling ratio. By setting this number to N, Traffic Dump will capture every one out of N sessions. This ratio can also be changed via traffic_ctl without restarting ATS. +--limit + The max disk usage (approximate). By setting this number to N, Traffic Dump will stop capturing new sessions once the disk usage exceeds N bytes. + Traffic_Ctl Command: traffic_ctl plugin msg traffic_dump.sample N Same as setting --sample=N in plugin.config. +traffic_ctl plugin msg traffic_dump.reset + Reset disk usage. +traffic_ctl plugin msg traffic_dump.limit N + Set max disk usage. diff --git a/plugins/experimental/traffic_dump/traffic_dump.cc b/plugins/experimental/traffic_dump/traffic_dump.cc index acc61a39688..86159169f77 100644 --- a/plugins/experimental/traffic_dump/traffic_dump.cc +++ b/plugins/experimental/traffic_dump/traffic_dump.cc @@ -18,7 +18,7 @@ limitations under the License. */ -#include +#include #include #include @@ -26,10 +26,9 @@ #include #include -#include #include #include -#include +#include #include #include @@ -38,31 +37,37 @@ #include #include #include +#include +#include "tscore/ts_file.h" #include "ts/ts.h" -const char *PLUGIN_NAME = "traffic_dump"; -static const std::string closing = "]}]}"; - -static std::string LOG_DIR = "dump"; // default log directory -static int s_arg_idx = 0; // Session Arg Index to pass on session data -static std::atomic sample_pool_size(1000); // Sampling ratio - +namespace +{ +const char *PLUGIN_NAME = "traffic_dump"; +const std::string closing = "]}]}"; + +ts::file::path log_path{"dump"}; // default log directory +int s_arg_idx = 0; // Session Arg Index to pass on session data +std::atomic sample_pool_size(1000); // Sampling ratio +std::atomic max_disk_usage(10000000); //< Max disk space for logs (approximate) +std::atomic disk_usage(0); //< Actual disk usage // handler declaration -static int session_aio_handler(TSCont contp, TSEvent event, void *edata); -static int session_txn_handler(TSCont contp, TSEvent event, void *edata); +int session_aio_handler(TSCont contp, TSEvent event, void *edata); +int session_txn_handler(TSCont contp, TSEvent event, void *edata); -// Custom structure for per session data +/// Custom structure for per session data struct SsnData { - int log_fd = -1; // Log file descriptor - int aio_count = 0; // Active AIO counts - int64_t write_offset = 0; // AIO write offset - bool first = true; // First Transaction - bool ssn_closed = false; // Session closed flag + int log_fd = -1; //< Log file descriptor + int aio_count = 0; //< Active AIO counts + int64_t write_offset = 0; //< AIO write offset + bool first = true; //< First Transaction + bool ssn_closed = false; //< Session closed flag + ts::file::path log_name; //< Log file path - TSCont aio_cont = nullptr; // AIO callback - TSCont txn_cont = nullptr; // Transaction callback - TSMutex disk_io_mutex = nullptr; // AIO mutex + TSCont aio_cont = nullptr; //< AIO callback + TSCont txn_cont = nullptr; //< Transaction callback + TSMutex disk_io_mutex = nullptr; //< AIO mutex SsnData() { @@ -110,7 +115,7 @@ struct SsnData { /// Local helper functions about json formatting /// min_write(): Inline function for repeating code -static inline void +inline void min_write(const char *buf, int64_t &prevIdx, int64_t &idx, std::ostream &jsonfile) { if (prevIdx < idx) { @@ -121,7 +126,7 @@ min_write(const char *buf, int64_t &prevIdx, int64_t &idx, std::ostream &jsonfil /// esc_json_out(): Escape characters in a buffer and output to ofstream object /// in a way to minimize ofstream operations -static int +int esc_json_out(const char *buf, int64_t len, std::ostream &jsonfile) { if (buf == nullptr) @@ -176,14 +181,14 @@ esc_json_out(const char *buf, int64_t len, std::ostream &jsonfile) } /// escape_json(): escape chars in a string and returns json string -static std::string +std::string escape_json(std::string const &s) { std::ostringstream o; esc_json_out(s.c_str(), s.length(), o); return o.str(); } -static std::string +std::string escape_json(const char *buf, int64_t size) { std::ostringstream o; @@ -191,34 +196,21 @@ escape_json(const char *buf, int64_t size) return o.str(); } -/// json_entry(): Formats to map-style entry i.e. "field": "value" -// static inline std::string -// json_entry(std::string const &name, std::string const &value) -// { -// return "\"" + escape_json(name) + "\": \"" + escape_json(value) + "\""; -// } - -static inline std::string +inline std::string json_entry(std::string const &name, const char *buf, int64_t size) { return "\"" + escape_json(name) + "\":\"" + escape_json(buf, size) + "\""; } /// json_entry_array(): Formats to array-style entry i.e. ["field","value"] -// static inline std::string -// json_entry_array(std::string const &name, std::string const &value) -// { -// return "[\"" + escape_json(name) + "\", \"" + escape_json(value) + "\"]"; -// } - -static inline std::string +inline std::string json_entry_array(const char *name, int name_len, const char *value, int value_len) { return "[\"" + escape_json(name, name_len) + "\", \"" + escape_json(value, value_len) + "\"]"; } /// Helper functions to collect txn information from TSMBuffer -static std::string +std::string collect_headers(TSMBuffer &buffer, TSMLoc &hdr_loc, int64_t body_bytes) { std::string result = "{"; @@ -290,7 +282,7 @@ int session_aio_handler(TSCont contp, TSEvent event, void *edata) { switch (event) { - case TS_AIO_EVENT_DONE: { + case TS_EVENT_AIO_DONE: { TSAIOCallback cb = static_cast(edata); SsnData *ssnData = static_cast(TSContDataGet(contp)); if (!ssnData) { @@ -308,6 +300,12 @@ session_aio_handler(TSCont contp, TSEvent event, void *edata) TSContDataSet(contp, nullptr); close(ssnData->log_fd); TSMutexUnlock(ssnData->disk_io_mutex); + std::error_code ec; + ts::file::file_status st = ts::file::status(ssnData->log_name, ec); + if (!ec) { + disk_usage += ts::file::file_size(st); + TSDebug(PLUGIN_NAME, "Finish a session with log file of %" PRIuMAX "bytes", ts::file::file_size(st)); + } delete ssnData; return TS_SUCCESS; } @@ -412,9 +410,25 @@ global_ssn_handler(TSCont contp, TSEvent event, void *edata) // Also handles LIFECYCLE_MSG from traffic_ctl case TS_EVENT_LIFECYCLE_MSG: { TSPluginMsg *msg = static_cast(edata); - if (strlen(msg->tag) == 19 && strncmp(msg->tag, "traffic_dump.sample", 19) == 0) { - sample_pool_size = static_cast(strtol(static_cast(msg->data), nullptr, 0)); - TSDebug(PLUGIN_NAME, "global_ssn_handler(): Received Msg to change sample size to %" PRId64 "", sample_pool_size.load()); + // String view of plugin message prefix + static constexpr std::string_view PLUGIN_PREFIX("traffic_dump."_sv); + + std::string_view tag(msg->tag, strlen(msg->tag)); + + if (tag.substr(0, PLUGIN_PREFIX.size()) == PLUGIN_PREFIX) { + tag.remove_prefix(PLUGIN_PREFIX.size()); + if (tag == "sample") { + sample_pool_size = static_cast(strtol(static_cast(msg->data), nullptr, 0)); + TSDebug(PLUGIN_NAME, "TS_EVENT_LIFECYCLE_MSG: Received Msg to change sample size to %" PRId64 "bytes", + sample_pool_size.load()); + } else if (tag == "reset") { + disk_usage = 0; + TSDebug(PLUGIN_NAME, "TS_EVENT_LIFECYCLE_MSG: Received Msg to reset disk usage counter"); + } else if (tag == "limit") { + max_disk_usage = static_cast(strtol(static_cast(msg->data), nullptr, 0)); + TSDebug(PLUGIN_NAME, "TS_EVENT_LIFECYCLE_MSG: Received Msg to change max disk usage to %" PRId64 "bytes", + max_disk_usage.load()); + } } return TS_SUCCESS; } @@ -424,6 +438,10 @@ global_ssn_handler(TSCont contp, TSEvent event, void *edata) if (id % sample_pool_size != 0) { TSDebug(PLUGIN_NAME, "global_ssn_handler(): Ignore session %" PRId64 "...", id); break; + } else if (disk_usage >= max_disk_usage) { + TSDebug(PLUGIN_NAME, "global_ssn_handler(): Ignore session %" PRId64 "due to disk usage %" PRId64 "bytes", id, + disk_usage.load()); + break; } // Beginning of a new session /// Get epoch time @@ -470,28 +488,26 @@ global_ssn_handler(TSCont contp, TSEvent event, void *edata) // Initialize AIO file TSMutexLock(ssnData->disk_io_mutex); if (ssnData->log_fd < 0) { - std::string path = LOG_DIR + "/" + std::string(client_str, 3); - std::string fname = path + "/" + session_id; + ts::file::path log_p = log_path / ts::file::path(std::string(client_str, 3)); + ts::file::path log_f = log_p / ts::file::path(session_id); // Create subdir if not existing - struct stat st; - if (stat(path.c_str(), &st) == -1) { - if (mkdir(path.c_str(), 0755) == -1) { - TSDebug(PLUGIN_NAME, "global_ssn_handler(): failed to create dir (%d)%s", errno, strerror(errno)); - - TSError("[%s] Failed to create dir. %s", PLUGIN_NAME, strerror(errno)); - } + std::error_code ec; + ts::file::status(log_p, ec); + if (ec && mkdir(log_p.c_str(), 0755) == -1) { + TSDebug(PLUGIN_NAME, "global_ssn_handler(): Failed to create dir %s", log_p.c_str()); + TSError("[%s] Failed to create dir %s", PLUGIN_NAME, log_p.c_str()); } // Try to open log files for AIO - ssnData->log_fd = open(fname.c_str(), O_RDWR | O_CREAT, S_IRWXU); + ssnData->log_fd = open(log_f.c_str(), O_RDWR | O_CREAT, S_IRWXU); if (ssnData->log_fd < 0) { TSMutexUnlock(ssnData->disk_io_mutex); - TSDebug(PLUGIN_NAME, "global_ssn_handler(): Failed to open log files. Abort."); + TSDebug(PLUGIN_NAME, "global_ssn_handler(): Failed to open log files %s. Abort.", log_f.c_str()); TSHttpSsnReenable(ssnp, TS_EVENT_HTTP_CONTINUE); return TS_EVENT_HTTP_CONTINUE; } - + ssnData->log_name = log_f; // Write log file beginning to disk ssnData->write_to_disk(beginning); } @@ -526,6 +542,8 @@ global_ssn_handler(TSCont contp, TSEvent event, void *edata) return TS_SUCCESS; } +} // End of anonymous namespace + void TSPluginInit(int argc, const char *argv[]) { @@ -533,37 +551,29 @@ TSPluginInit(int argc, const char *argv[]) TSPluginRegistrationInfo info; info.plugin_name = "traffic_dump"; - info.vendor_name = "Oath"; - info.support_email = "edge@oath.com"; - - std::string installDir = TSInstallDirGet(); - LOG_DIR = installDir + "/" + LOG_DIR + "/"; + info.vendor_name = "Apache Software Foundation"; + info.support_email = "dev@trafficserver.apache.org"; /// Commandline options - static const struct option longopts[] = { - {"logdir", required_argument, nullptr, 'l'}, {"sample", required_argument, nullptr, 's'}, {nullptr, no_argument, nullptr, 0}}; - int opt = 0; + static const struct option longopts[] = {{"logdir", required_argument, nullptr, 'l'}, + {"sample", required_argument, nullptr, 's'}, + {"limit", required_argument, nullptr, 'm'}, + {nullptr, no_argument, nullptr, 0}}; + int opt = 0; while (opt >= 0) { opt = getopt_long(argc, (char *const *)argv, "l:", longopts, nullptr); switch (opt) { case 'l': { - LOG_DIR = std::string(optarg); - if (LOG_DIR[0] != '/') { - LOG_DIR = installDir + "/" + std::string(optarg) + "/"; - } - TSDebug(PLUGIN_NAME, "Initialized with log dir: %s", LOG_DIR.c_str()); - struct stat st; - if (stat(LOG_DIR.c_str(), &st) == -1) { - TSDebug(PLUGIN_NAME, "Log dir error: (%d) %s", errno, strerror(errno)); - } else { - TSDebug(PLUGIN_NAME, "Log dir opened successfully"); - } + log_path = ts::file::path{optarg}; break; } case 's': { sample_pool_size = static_cast(std::strtol(optarg, nullptr, 0)); break; } + case 'm': { + max_disk_usage = static_cast(std::strtol(optarg, nullptr, 0)); + } case -1: case '?': break; @@ -575,6 +585,12 @@ TSPluginInit(int argc, const char *argv[]) } } + // Make absolute path if not + if (!log_path.is_absolute()) { + log_path = ts::file::path(TSInstallDirGet()) / log_path; + } + TSDebug(PLUGIN_NAME, "Initialized with log directory: %s", log_path.c_str()); + if (TS_SUCCESS != TSPluginRegister(&info)) { TSError("[%s] Unable to initialize plugin (disabled). Failed to register plugin.", PLUGIN_NAME); } else if (TS_SUCCESS != TSHttpSsnArgIndexReserve(PLUGIN_NAME, "Track log related data", &s_arg_idx)) { @@ -585,7 +601,8 @@ TSPluginInit(int argc, const char *argv[]) TSHttpHookAdd(TS_HTTP_SSN_START_HOOK, ssncont); TSHttpHookAdd(TS_HTTP_SSN_CLOSE_HOOK, ssncont); TSLifecycleHookAdd(TS_LIFECYCLE_MSG_HOOK, ssncont); - TSDebug(PLUGIN_NAME, "Initialized with sample pool size %" PRId64 "", sample_pool_size.load()); + TSDebug(PLUGIN_NAME, "Initialized with sample pool size %" PRId64 " bytes and disk limit %" PRId64 "bytes", + sample_pool_size.load(), max_disk_usage.load()); } return; diff --git a/plugins/experimental/uri_signing/Makefile.inc b/plugins/experimental/uri_signing/Makefile.inc index 9499479b4a6..864d2b48b38 100644 --- a/plugins/experimental/uri_signing/Makefile.inc +++ b/plugins/experimental/uri_signing/Makefile.inc @@ -23,6 +23,22 @@ experimental_uri_signing_uri_signing_la_SOURCES = \ experimental/uri_signing/jwt.c \ experimental/uri_signing/match.c \ experimental/uri_signing/parse.c \ + experimental/uri_signing/normalize.c \ experimental/uri_signing/timing.c experimental_uri_signing_uri_signing_la_LIBADD = @LIBJANSSON@ @LIBCJOSE@ @LIBPCRE@ -lm -lcrypto + +check_PROGRAMS += experimental/uri_signing/test_uri_signing + +experimental_uri_signing_test_uri_signing_CPPFLAGS = $(AM_CPPFLAGS) -I$(abs_top_srcdir)/tests/include -DURI_SIGNING_UNIT_TEST +experimental_uri_signing_test_uri_signing_LDADD = @LIBJANSSON@ @LIBCJOSE@ @LIBPCRE@ -lm -lcrypto +experimental_uri_signing_test_uri_signing_SOURCES = \ + experimental/uri_signing/unit_tests/uri_signing_test.cc \ + experimental/uri_signing/jwt.c \ + experimental/uri_signing/common.c \ + experimental/uri_signing/parse.c \ + experimental/uri_signing/cookie.c \ + experimental/uri_signing/config.c \ + experimental/uri_signing/timing.c \ + experimental/uri_signing/normalize.c \ + experimental/uri_signing/match.c diff --git a/plugins/experimental/uri_signing/README.md b/plugins/experimental/uri_signing/README.md index fe242d8d98f..398b9869958 100644 --- a/plugins/experimental/uri_signing/README.md +++ b/plugins/experimental/uri_signing/README.md @@ -1,8 +1,7 @@ URI Signing Plugin ================== -This remap plugin implements the draft URI Signing protocol documented here: -https://tools.ietf.org/html/draft-ietf-cdni-uri-signing-12 . +This remap plugin implements the draft URI Signing protocol documented [here](https://tools.ietf.org/html/draft-ietf-cdni-uri-signing-16): It takes a single argument: the name of a config file that contains key information. @@ -17,6 +16,8 @@ this plugin gets the URI. Config ------ +### Keys + The config file should be a JSON object that maps issuer names to JWK-sets. Exactly one of these JWK-sets must have an additional member indicating the renewal key. @@ -75,6 +76,33 @@ It's worth noting that multiple issuers can provide `auth_directives`. Each issuer will be processed in order and any issuer can provide access to a path. +### More Configuration Options + +**Strip Token** +When the strip_token parameter is set to true, the plugin removes the +token from both the url that is sent upstream to the origin and the url that +is used as the cache key. The strip_token parameter defaults to false and should +be set by only one issuer. +**ID** +The id field takes a string indicating the identification of the entity processing the request. +This is used in aud claim checks to ensure that the receiver is the intended audience of a +tokenized request. The id parameter can only be set by one issuer. + +Example: + + { + "Kabletown URI Authority": { + "renewal_kid": "Second Key", + "strip_token" : true, + "id" : "mycdn", + "auth_directives": [ + ⋮ + ] + "keys": [ + ⋮ + ] + } + Usage ----- @@ -85,31 +113,34 @@ will receive a 403 Forbidden response, instead of receiving content. Tokens will be found in either of these places: - A query parameter named `URISigningPackage`. The value must be the JWT. + - A path parameter named `URISigningPackage`. The value must be the JWT. - A cookie named `URISigningPackage`. The value of the cookie must be the JWT. -Path parameters will not be searched for JWTs. - ### Supported Claims The following claims are understood: - `iss`: Must be present. The issuer is used to locate the key for verification. - - `sub`: Validated last, after key verification. **Only `uri-regex` is supported!** + - `sub`: May be present, but is not validated. - `exp`: Expired tokens are not valid. + - `nbf`: Tokens processed before this time are not valid. + - `aud`: Token aud claim strings must match the configured id to be considered valid. - `iat`: May be present, but is not validated. - `cdniv`: Must be missing or 1. - - `cdnistt`: If present, must be 1. + - `cdniuc`: Validated last, after key verificationD. **Only `regex` is supported!** - `cdniets`: If cdnistt is 1, this must be present and non-zero. + - `cdnistt`: If present, must be 1. + - `cdnistd`: If present, must be 0. ### Unsupported Claims These claims are not supported. If they are present, the token will not validate: - - `aud` - - `nbf` - `jti` + - `cdnicrit` + - `cdniip` -In addition, the `sub` containers of `uri`, `uri-pattern`, and `uri-hash` are +In addition, the `cdniuc` container of `hash` is **not supported**. ### Token Renewal @@ -147,6 +178,9 @@ This builds in-tree with the rest of the ATS plugins. Of special note, however, are the first two libraries: cjose and jansson. These libraries are not currently used anywhere else, so they may not be installed. +Note that the default prefix value for cjose is /usr/local. Ensure this is visible to +any executables that are being run using this library. + As of this writing, both libraries install a dynamic library and a static archive. However, by default, the static archive is not compiled with Position Independent Code. The build script will detect this and build a dynamic @@ -156,3 +190,42 @@ plugin. If you would like to statically link them, you will need to ensure that they are compiled with the `-fPIC` flag in their CFLAGs. If the archives have PIC, the build scripts will automatically statically link them. + +Here are some sample commands for building jansson, cjose and trafficserver +locally using static linking. This assumes all source is under ${HOME}/git. + +### Sample + +If using local jansson: + + cd ${HOME}/git + git clone https://github.com/akheron/jansson.git + cd jansson + autoreconf -i + ./configure --disable-shared CC="gcc -fpic" + make -j`nproc` + + # Needed for ATS configure + ln -s src/.libs lib + ln -s src include + +If using local cjose: + + cd ${HOME}/git + git clone https://github.com/cisco/cjose.git + cd cjose + autoreconf -i + ./configure --with-jansson=${HOME}/git/jansson --disable-shared CC="gcc -fpic" + make -j`nproc` + + # Needed for ATS configure + ln -s src/.libs lib + +ATS: + + cd ${HOME}/git/ + git clone https://github.com/apache/trafficserver.git + cd trafficserver + autoreconf -i + ./configure --enable-experimental-plugins --with-jansson=${HOME}/git/jansson --with-cjose=${HOME}/git/cjose + make -j`nproc` diff --git a/plugins/header_rewrite/expander.h b/plugins/experimental/uri_signing/common.c similarity index 66% rename from plugins/header_rewrite/expander.h rename to plugins/experimental/uri_signing/common.c index 61553c00b58..bae8b6d911a 100644 --- a/plugins/header_rewrite/expander.h +++ b/plugins/experimental/uri_signing/common.c @@ -16,22 +16,17 @@ limitations under the License. */ -////////////////////////////////////////////////////////////////////////////////////////////// -// Public interface for the Variable Expander -// -#pragma once +#include "common.h" -#include +#ifdef URI_SIGNING_UNIT_TEST -#include "ts/ts.h" -#include "resources.h" - -class VariableExpander +void +PrintToStdErr(const char *fmt, ...) { -public: - explicit VariableExpander(const std::string &source) : _source(source) {} - std::string expand(const Resources &res); + va_list args; + va_start(args, fmt); + vfprintf(stderr, fmt, args); + va_end(args); +} -private: - std::string _source; -}; +#endif diff --git a/plugins/experimental/uri_signing/common.h b/plugins/experimental/uri_signing/common.h new file mode 100644 index 00000000000..467d0cee257 --- /dev/null +++ b/plugins/experimental/uri_signing/common.h @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#define PLUGIN_NAME "uri_signing" + +#ifdef URI_SIGNING_UNIT_TEST +#include +#include + +#define PluginDebug(fmt, ...) PrintToStdErr("(%s) %s:%d:%s() " fmt "\n", PLUGIN_NAME, __FILE__, __LINE__, __func__, ##__VA_ARGS__) +#define PluginError(fmt, ...) PrintToStdErr("(%s) %s:%d:%s() " fmt "\n", PLUGIN_NAME, __FILE__, __LINE__, __func__, ##__VA_ARGS__) +#define TSmalloc(x) malloc(x) +#define TSfree(p) free(p) +void PrintToStdErr(const char *fmt, ...); + +#else + +#include "ts/ts.h" +#define PluginDebug(...) TSDebug("uri_signing", PLUGIN_NAME " " __VA_ARGS__) +#define PluginError(...) PluginDebug(__VA_ARGS__), TSError(PLUGIN_NAME " " __VA_ARGS__) + +#endif diff --git a/plugins/experimental/uri_signing/config.c b/plugins/experimental/uri_signing/config.c index 83083f8fbda..cdf2f2db4ce 100644 --- a/plugins/experimental/uri_signing/config.c +++ b/plugins/experimental/uri_signing/config.c @@ -16,13 +16,11 @@ * limitations under the License. */ -#include "uri_signing.h" +#include "common.h" #include "config.h" #include "timing.h" #include "jwt.h" -#include - #include #include @@ -45,6 +43,8 @@ struct config { char **issuer_names; struct signer signer; struct auth_directive *auth_directives; + char *id; + bool strip_token; }; cjose_jwk_t ** @@ -80,6 +80,18 @@ find_key_by_kid(struct config *cfg, const char *issuer, const char *kid) return NULL; } +const char * +config_get_id(struct config *cfg) +{ + return cfg->id; +} + +bool +config_strip_token(struct config *cfg) +{ + return cfg->strip_token; +} + struct config * config_new(size_t n) { @@ -105,6 +117,9 @@ config_new(size_t n) cfg->signer.alg = NULL; cfg->auth_directives = NULL; + cfg->id = NULL; + + cfg->strip_token = false; PluginDebug("New config object created at %p", cfg); return cfg; @@ -117,6 +132,7 @@ config_delete(struct config *cfg) return; } hdestroy_r(cfg->issuers); + free(cfg->issuers); for (cjose_jwk_t ***jwkis = cfg->jwkis; *jwkis; ++jwkis) { for (cjose_jwk_t **jwks = *jwkis; *jwks; ++jwks) { @@ -126,6 +142,10 @@ config_delete(struct config *cfg) } free(cfg->jwkis); + if (cfg->id) { + free(cfg->id); + } + for (char **name = cfg->issuer_names; *name; ++name) { free(*name); } @@ -259,6 +279,23 @@ read_config(const char *path) renewal_kid = json_string_value(renewal_kid_json); } + json_t *id_json = json_object_get(jwks, "id"); + const char *id; + if (id_json) { + id = json_string_value(id_json); + if (id) { + cfg->id = malloc(strlen(id) + 1); + strcpy(cfg->id, id); + PluginDebug("Found Id in the config: %s", cfg->id); + } + } + json_decref(id_json); + + json_t *strip_json = json_object_get(jwks, "strip_token"); + if (strip_json) { + cfg->strip_token = json_boolean_value(strip_json); + } + size_t jwks_ct = json_array_size(key_ary); cjose_jwk_t **jwks = (*jwkis++ = malloc((jwks_ct + 1) * sizeof *jwks)); PluginDebug("Created table with size %d", cfg->issuers->size); diff --git a/plugins/experimental/uri_signing/config.h b/plugins/experimental/uri_signing/config.h index 75a82f24d88..87688374c85 100644 --- a/plugins/experimental/uri_signing/config.h +++ b/plugins/experimental/uri_signing/config.h @@ -33,3 +33,5 @@ struct signer *config_signer(struct config *); struct _cjose_jwk_int **find_keys(struct config *cfg, const char *issuer); struct _cjose_jwk_int *find_key_by_kid(struct config *cfg, const char *issuer, const char *kid); bool uri_matches_auth_directive(struct config *cfg, const char *uri, size_t uri_ct); +const char *config_get_id(struct config *cfg); +bool config_strip_token(struct config *cfg); diff --git a/plugins/experimental/uri_signing/cookie.c b/plugins/experimental/uri_signing/cookie.c index 70dd1aa8469..1e9fc7f9a30 100644 --- a/plugins/experimental/uri_signing/cookie.c +++ b/plugins/experimental/uri_signing/cookie.c @@ -17,8 +17,7 @@ */ #include "cookie.h" -#include "uri_signing.h" -#include +#include "common.h" #include const char * diff --git a/plugins/experimental/uri_signing/jwt.c b/plugins/experimental/uri_signing/jwt.c index da4e804f25f..f14ecb6e289 100644 --- a/plugins/experimental/uri_signing/jwt.c +++ b/plugins/experimental/uri_signing/jwt.c @@ -16,10 +16,10 @@ * limitations under the License. */ -#include "uri_signing.h" +#include "common.h" #include "jwt.h" #include "match.h" -#include "ts/ts.h" +#include "normalize.h" #include #include #include @@ -55,15 +55,18 @@ parse_jwt(json_t *raw) jwt->raw = raw; jwt->iss = json_string_value(json_object_get(raw, "iss")); jwt->sub = json_string_value(json_object_get(raw, "sub")); - jwt->aud = json_string_value(json_object_get(raw, "aud")); + jwt->aud = json_object_get(raw, "aud"); jwt->exp = parse_number(json_object_get(raw, "exp")); jwt->nbf = parse_number(json_object_get(raw, "nbf")); jwt->iat = parse_number(json_object_get(raw, "iat")); jwt->jti = json_string_value(json_object_get(raw, "jti")); jwt->cdniv = parse_integer_default(json_object_get(raw, "cdniv"), 1); + jwt->cdnicrit = json_string_value(json_object_get(raw, "cdnicrit")); + jwt->cdniip = json_string_value(json_object_get(raw, "cdniip")); + jwt->cdniuc = json_string_value(json_object_get(raw, "cdniuc")); jwt->cdniets = json_integer_value(json_object_get(raw, "cdniets")); jwt->cdnistt = json_integer_value(json_object_get(raw, "cdnistt")); - jwt->cdnicrit = json_string_value(json_object_get(raw, "cdnicrit")); + jwt->cdnistd = parse_integer_default(json_object_get(raw, "cdnistd"), 0); return jwt; } @@ -73,6 +76,8 @@ jwt_delete(struct jwt *jwt) if (!jwt) { return; } + + json_decref(jwt->aud); json_decref(jwt->raw); free(jwt); } @@ -93,12 +98,6 @@ unsupported_string_claim(const char *str) return !str; } -bool -unsupported_date_claim(double t) -{ - return isnan(t); -} - bool jwt_validate(struct jwt *jwt) { @@ -112,23 +111,18 @@ jwt_validate(struct jwt *jwt) return false; } - if (!jwt->sub) { /* Mandatory claim. Will be validated after key verification. */ - PluginDebug("Initial JWT Failure: missing sub"); + if (now() > jwt->exp) { + PluginDebug("Initial JWT Failure: expired token"); return false; } - if (!unsupported_string_claim(jwt->aud)) { - PluginDebug("Initial JWT Failure: missing sub"); + if (now() < jwt->nbf) { + PluginDebug("Initial JWT Failure: nbf claim violated"); return false; } - if (now() > jwt->exp) { - PluginDebug("Initial JWT Failure: expired token"); - return false; - } - - if (!unsupported_date_claim(jwt->nbf)) { - PluginDebug("Initial JWT Failure: nbf unsupported"); + if (!unsupported_string_claim(jwt->cdniip)) { + PluginDebug("Initial JWT Failure: cdniip unsupported"); return false; } @@ -147,52 +141,113 @@ jwt_validate(struct jwt *jwt) return false; } + if (jwt->cdnistd != 0) { + PluginDebug("Initial JWT Failure: unsupported value for cdnistd: %d", jwt->cdnistd); + return false; + } + return true; } bool -jwt_check_uri(const char *sub, const char *uri) +jwt_check_aud(json_t *aud, const char *id) { - static const char CONT_URI_STR[] = "uri"; - static const char CONT_URI_PATTERN_STR[] = "uri-pattern"; - static const char CONT_URI_REGEX_STR[] = "uri-regex"; + if (!aud) { + return true; + } + if (!id) { + return false; + } + /* If aud is a string, do a simple string comparison */ + const char *aud_str = json_string_value(aud); + if (aud_str) { + PluginDebug("Checking aud %s agaisnt token aud string \"%s\"", id, aud_str); + /* Both strings will be null terminated per jansson docs */ + if (strcmp(aud_str, id) == 0) { + return true; + } + return false; + } + PluginDebug("Checking aud %s agaisnt token aud array", id); + /* If aud is an array, check all items */ + if (json_is_array(aud)) { + size_t index; + json_t *aud_item; + json_array_foreach(aud, index, aud_item) + { + aud_str = json_string_value(aud_item); + if (aud_str) { + if (strcmp(aud_str, id) == 0) { + return true; + } + } + } + } + return false; +} - if (!sub || !*sub || !uri) { +bool +jwt_check_uri(const char *cdniuc, const char *uri) +{ + static const char CONT_URI_HASH_STR[] = "hash"; + static const char CONT_URI_REGEX_STR[] = "regex"; + + /* If cdniuc is not provided, skip uri check */ + if (!cdniuc || !*cdniuc) { + return true; + } + + if (!uri) { return false; } - const char *kind = sub, *container = sub; + /* Normalize the URI */ + int uri_ct = strlen(uri); + int buff_ct = uri_ct + 2; + int err; + char *normal_uri = (char *)TSmalloc(buff_ct); + memset(normal_uri, 0, buff_ct); + + err = normalize_uri(uri, uri_ct, normal_uri, buff_ct); + + if (err) { + goto fail_jwt; + } + + const char *kind = cdniuc, *container = cdniuc; while (*container && *container != ':') { ++container; } if (!*container) { - return false; + goto fail_jwt; } ++container; size_t len = container - kind; - PluginDebug("Comparing with match kind \"%.*s\" on \"%s\" to \"%s\"", (int)len - 1, kind, container, uri); + bool status; + PluginDebug("Comparing with match kind \"%.*s\" on \"%s\" to normalized URI \"%s\"", (int)len - 1, kind, container, normal_uri); switch (len) { - case sizeof CONT_URI_STR: - if (!strncmp(CONT_URI_STR, kind, len - 1)) { - return !strcmp(container, uri); + case sizeof CONT_URI_HASH_STR: + if (!strncmp(CONT_URI_HASH_STR, kind, len - 1)) { + status = match_hash(container, normal_uri); + TSfree(normal_uri); + return status; } - PluginDebug("Expected kind %s, but did not find it in \"%.*s\"", CONT_URI_STR, (int)len - 1, kind); - break; - case sizeof CONT_URI_PATTERN_STR: - if (!strncmp(CONT_URI_PATTERN_STR, kind, len - 1)) { - return match_glob(container, uri); - } - PluginDebug("Expected kind %s, but did not find it in \"%.*s\"", CONT_URI_PATTERN_STR, (int)len - 1, kind); + PluginDebug("Expected kind %s, but did not find it in \"%.*s\"", CONT_URI_HASH_STR, (int)len - 1, kind); break; case sizeof CONT_URI_REGEX_STR: if (!strncmp(CONT_URI_REGEX_STR, kind, len - 1)) { - return match_regex(container, uri); + status = match_regex(container, normal_uri); + TSfree(normal_uri); + return status; } PluginDebug("Expected kind %s, but did not find it in \"%.*s\"", CONT_URI_REGEX_STR, (int)len - 1, kind); break; } PluginDebug("Unknown match kind \"%.*s\"", (int)len - 1, kind); + +fail_jwt: + TSfree(normal_uri); return false; } @@ -204,6 +259,14 @@ renew_copy_string(json_t *new_json, const char *name, const char *old) } } +void +renew_copy_raw(json_t *new_json, const char *name, json_t *old_json) +{ + if (old_json) { + json_object_set_new(new_json, name, old_json); + } +} + void renew_copy_real(json_t *new_json, const char *name, double old) { @@ -236,16 +299,19 @@ renew(struct jwt *jwt, const char *iss, cjose_jwk_t *jwk, const char *alg, const json_t *new_json = json_object(); renew_copy_string(new_json, "iss", iss); /* use issuer of new signing key */ renew_copy_string(new_json, "sub", jwt->sub); - renew_copy_string(new_json, "aud", jwt->aud); + renew_copy_raw(new_json, "aud", jwt->aud); renew_copy_real(new_json, "exp", now() + jwt->cdniets); /* expire ets seconds hence */ renew_copy_real(new_json, "nbf", jwt->nbf); renew_copy_real(new_json, "iat", now()); /* issued now */ renew_copy_string(new_json, "jti", jwt->jti); + renew_copy_string(new_json, "cdniuc", jwt->cdniuc); renew_copy_integer(new_json, "cdniv", jwt->cdniv); renew_copy_integer(new_json, "cdniets", jwt->cdniets); renew_copy_integer(new_json, "cdnistt", jwt->cdnistt); + renew_copy_integer(new_json, "cdnistd", jwt->cdnistd); char *pt = json_dumps(new_json, JSON_COMPACT); + json_decref(new_json); cjose_header_t *hdr = cjose_header_new(NULL); if (!hdr) { diff --git a/plugins/experimental/uri_signing/jwt.h b/plugins/experimental/uri_signing/jwt.h index 1604eeac1b2..95efbbc1da0 100644 --- a/plugins/experimental/uri_signing/jwt.h +++ b/plugins/experimental/uri_signing/jwt.h @@ -23,20 +23,24 @@ struct jwt { json_t *raw; const char *iss; const char *sub; - const char *aud; + json_t *aud; double exp; double nbf; double iat; const char *jti; - const char *cdnicrit; int cdniv; + const char *cdnicrit; + const char *cdniip; + const char *cdniuc; int cdniets; int cdnistt; + int cdnistd; }; struct jwt *parse_jwt(json_t *raw); void jwt_delete(struct jwt *jwt); bool jwt_validate(struct jwt *jwt); -bool jwt_check_uri(const char *sub, const char *uri); +bool jwt_check_aud(json_t *aud, const char *id); +bool jwt_check_uri(const char *cdniuc, const char *uri); struct _cjose_jwk_int; char *renew(struct jwt *jwt, const char *iss, struct _cjose_jwk_int *jwk, const char *alg, const char *package); diff --git a/plugins/experimental/uri_signing/match.c b/plugins/experimental/uri_signing/match.c index ad376a251e4..faea8953dfc 100644 --- a/plugins/experimental/uri_signing/match.c +++ b/plugins/experimental/uri_signing/match.c @@ -16,14 +16,13 @@ * limitations under the License. */ -#include "uri_signing.h" -#include "ts/ts.h" +#include +#include "common.h" #include -#include #include bool -match_glob(const char *needle, const char *haystack) +match_hash(const char *needle, const char *haystack) { return false; } @@ -31,16 +30,27 @@ match_glob(const char *needle, const char *haystack) bool match_regex(const char *pattern, const char *uri) { - const char *err; - int err_off; + struct re_pattern_buffer pat_buff; + + pat_buff.translate = 0; + pat_buff.fastmap = 0; + pat_buff.buffer = 0; + pat_buff.allocated = 0; + + re_syntax_options = RE_SYNTAX_POSIX_MINIMAL_EXTENDED; + PluginDebug("Testing regex pattern /%s/ against \"%s\"", pattern, uri); - pcre *re = pcre_compile(pattern, PCRE_ANCHORED | PCRE_UCP | PCRE_UTF8, &err, &err_off, NULL); - if (!re) { - PluginDebug("Regex /%s/ failed to compile.", pattern); + + const char *comp_err = re_compile_pattern(pattern, strlen(pattern), &pat_buff); + + if (comp_err) { + PluginDebug("Regex Compilation ERROR: %s", comp_err); return false; } - int rc = pcre_exec(re, NULL, uri, strlen(uri), 0, 0, NULL, 0); - pcre_free(re); - return rc >= 0; + int match_ret; + match_ret = re_match(&pat_buff, uri, strlen(uri), 0, 0); + regfree(&pat_buff); + + return match_ret >= 0; } diff --git a/plugins/experimental/uri_signing/match.h b/plugins/experimental/uri_signing/match.h index 92b906dbd35..38f3eb28e83 100644 --- a/plugins/experimental/uri_signing/match.h +++ b/plugins/experimental/uri_signing/match.h @@ -17,5 +17,5 @@ */ #include -bool match_glob(const char *needle, const char *haystack); +bool match_hash(const char *needle, const char *haystack); bool match_regex(const char *pattern, const char *uri); diff --git a/plugins/experimental/uri_signing/normalize.c b/plugins/experimental/uri_signing/normalize.c new file mode 100644 index 00000000000..e51411190a0 --- /dev/null +++ b/plugins/experimental/uri_signing/normalize.c @@ -0,0 +1,382 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "normalize.h" +#include "common.h" +#include +#include +#include +#include + +/* Remove Dot Algorithm outlined in RFC3986 section 5.2.4 + * Function writes normalizes path and writes to ret_buffer */ +int +remove_dot_segments(const char *path, int path_ct, char *ret_buffer, int buff_ct) +{ + /* Ensure buffer is at least the size of the path */ + if (buff_ct < path_ct) { + PluginDebug("Path buffer not large enough"); + return -1; + } + + /* Create an input buffer that we can change */ + char inBuff[path_ct + 1]; + memset(inBuff, 0, path_ct + 1); + strcpy(inBuff, path); + + const char *path_end = inBuff + path_ct; + char *seg_start = inBuff; + char *seg_end; + char *write_buffer = ret_buffer; + int seg_len; + + for (;;) { + if (seg_start == path_end) { + break; + } + seg_end = seg_start + 1; + + /* Parse such that Seg start/end contain the next full path segment */ + while (seg_end != path_end && *seg_end != '/') { + seg_end++; + } + + seg_len = seg_end - seg_start + 1; + + /* Remove starting ../ or ./ from input buffer */ + if (!strncmp(seg_start, "../", seg_len) || !strncmp(seg_start, "./", seg_len)) { + if (seg_end != path_end) { + seg_end++; + } + } + + /* Remove starting /./ or /. from input buffer and replace with '/' in output buffer */ + else if (!strncmp(seg_start, "/./", seg_len) || !strncmp(seg_start, "/.", seg_len)) { + *write_buffer = '/'; + write_buffer++; + if (seg_end != path_end) { + seg_end++; + } + } + + /* Replace /../ or /.. with / in write_buffer and remove preceding segment */ + else if (!strncmp(seg_start, "/../", seg_len) || !strncmp(seg_start, "/..", seg_len)) { + int prev_len = 0; + while (*write_buffer != '/' && write_buffer != ret_buffer) { + prev_len++; + write_buffer--; + } + memset(write_buffer, 0, prev_len); + + /* Replace segment with '/' in input buffer */ + if (seg_end != path_end) { + seg_start[seg_len - 1] = '/'; + } else { + seg_start[seg_len - 2] = '/'; + seg_end--; + } + } + + /* Remove starting '.' or '..' from input buffer */ + else if (!strncmp(seg_start, ".", seg_len) || !strncmp(seg_start, "..", seg_len)) { + if (seg_end != path_end) { + seg_end++; + } + } + /* Place the current path segment to the output buffer including initial '/' but not the next '/' */ + else { + /* Write first forward slash to buffer */ + if (*seg_start == '/') { + *write_buffer = *seg_start; + write_buffer++; + seg_start++; + } + + /* Write subsequent characters to buffer */ + while (*seg_start != '/') { + *write_buffer = *seg_start; + write_buffer++; + if (*seg_start == 0) { + break; + } + seg_start++; + } + } + seg_start = seg_end; + } + + PluginDebug("Normalized Path: %s", ret_buffer); + return strlen(ret_buffer); +} + +/* Function percent decodes uri_ct characters of the string uri and writes it to the decoded_uri + * buffer. If lower is true, it sets all characters including decoded ones to lower case. + * The function returns the length of the decoded string or -1 if there was a parsing error + * TODO: ADD functionality to ignore unicode non-standard characters and leave them encoded. Read RFC regarding normalization and + * determine if this is compliant. + */ +int +percent_decode(const char *uri, int uri_ct, char *decoded_uri, bool lower) +{ + static const char *reserved_string = ":/?#[]@!$&\'()*+,;="; + + if (uri_ct <= 0) { + return 0; + } + + int offset = 0; + int i; + for (i = 0; i < uri_ct; i++) { + if (uri[i] == '%') { + /* The next two characters are interpreted as the hex encoded value. Store in encodedVal */ + if (uri_ct < i + 2) { + goto decode_failure; + } + char encodedVal[2] = {0}; + int j; + for (j = 0; j < 2; j++) { + if (isxdigit(uri[i + j + 1])) { + encodedVal[j] = uri[i + j + 1]; + } else { + goto decode_failure; + } + } + int hexVal = 0; + char decodeChar; + sscanf(encodedVal, "%2x", &hexVal); + decodeChar = (char)hexVal; + /* If encoded value is a reserved char, leave encoded*/ + if (strchr(reserved_string, decodeChar)) { + decoded_uri[i - offset] = uri[i]; + decoded_uri[i + 1 - offset] = toupper(uri[i + 1]); + decoded_uri[i + 2 - offset] = toupper(uri[i + 2]); + } + /* If not a reserved char, decode using the decoded_uri buffer */ + else { + if (lower) { + decoded_uri[i - offset] = tolower(decodeChar); + } else { + decoded_uri[i - offset] = decodeChar; + } + offset = offset + 2; + } + i = i + 2; + } + /* Write non-encoded values to decoded buffer */ + else { + if (lower) { + decoded_uri[i - offset] = tolower(uri[i]); + } else { + decoded_uri[i - offset] = uri[i]; + } + } + } + + /* Return the size of the newly decoded string */ + return uri_ct - offset; + +decode_failure: + PluginDebug("ERROR Decoding URI"); + return -1; +} + +/* This function takes a uri and an initialized buffer to populate with the normalized uri. + * Returns non zero for error + * + * The buffer provided must be at least the length of the uri + 1 as the normalized uri will + * potentially be one char larger than the original uri if a backslash is added to the path. + * + * The normalization function returns a string with the following modifications + * 1. Lowecase protocol/domain + * 2. Path segments .. and . are removed from path + * 3. Alphabetical percent encoded octet values are toupper + * 4. Non-reserved percent encoded octet values are decoded + * 5. The Port is removed if it is default + * 6. Defaults to a single backslash for the path segment if path segment is empty + */ +int +normalize_uri(const char *uri, int uri_ct, char *normal_uri, int normal_ct) +{ + PluginDebug("Normalizing URI: %s", uri); + + /* Buffer provided must be large enough to store the uri plus one additional char */ + const char *uri_end = uri + uri_ct; + const char *buff_end = normal_uri + normal_ct; + + if (normal_uri && normal_ct < uri_ct + 1) { + PluginDebug("Buffer to Normalize URI not large enough."); + return -1; + } + + /* Initialize a path buffer to pass to path normalization function later on */ + char path_buffer[normal_ct]; + memset(path_buffer, 0, normal_ct); + + /* Comp variables store starting/ending indexes for each uri component as uri is parsed. + * Write buffer traverses the normalized uri buffer as we build the normalized string. + */ + const char *comp_start = uri; + const char *comp_end = uri; + char *write_buffer = normal_uri; + bool https = false; + + /* Parse the protocol which will end with a colon */ + while (*comp_end != ':' && comp_end != uri_end) { + *write_buffer = tolower(*comp_end); + comp_end++; + write_buffer++; + } + + if (comp_end == uri_end) { + PluginDebug("Reached End of String prematurely"); + goto normalize_failure; + } + + /* Copy the colon */ + *write_buffer = *comp_end; + comp_end++; + write_buffer++; + + /* Ensure the protocol is either http or https */ + if (strcmp("https:", normal_uri) == 0) { + https = true; + } else if (strcmp("http:", normal_uri)) { + PluginDebug("String is neither http or https"); + goto normalize_failure; + } + + /* Protocol must be terminated by two forward slashes */ + int i; + for (i = 0; i < 2; i++) { + if (comp_end == uri_end || *comp_end != '/') { + goto normalize_failure; + } + *write_buffer = *comp_end; + comp_end++; + write_buffer++; + } + + if (comp_end == uri_end) { + goto normalize_failure; + } + + /* Comp_start is index of start of authority component */ + int comp_ct; + comp_start = comp_end; + + /* Set comp start/end to contain authority component */ + bool userInfo = false; + while (comp_end != uri_end && *comp_end != '/' && *comp_end != '?' && *comp_end != '#') { + /* If we encounter userinfo, decode it without altering case and set comp_start/end to only include hostname/port */ + if (*comp_end == '@' && userInfo == false) { + comp_ct = comp_end - comp_start; + comp_ct = percent_decode(comp_start, comp_ct, write_buffer, false); + if (comp_ct < 0) { + goto normalize_failure; + } + comp_start = comp_end; + userInfo = true; + write_buffer = write_buffer + comp_ct; + } + comp_end++; + } + + /* UserInfo without a hostname is invalid */ + if (userInfo == true && comp_end == uri_end) { + goto normalize_failure; + } + + comp_ct = comp_end - comp_start; + + /* - comp start/end holds indices in original uri of hostname/port + * - write_buffer holds pointer to start of hostname/port written to the decode buffer + * - comp_ct holds size of hostname/port in original uri + */ + + /* Parse and decode the hostname and port and set to lower case */ + comp_ct = percent_decode(comp_start, comp_ct, write_buffer, true); + + if (comp_ct < 0) { + goto normalize_failure; + } + + /* Remove the port from the buffer if default */ + while (*write_buffer != 0) { + if (*write_buffer == ':') { + if (https == true && !strncmp(write_buffer, ":443", 5)) { + memset(write_buffer, 0, 4); + break; + } else if (https == false && !strncmp(write_buffer, ":80", 4)) { + memset(write_buffer, 0, 3); + break; + } + } + write_buffer++; + } + + comp_start = comp_end; + + /* If we have reached the end of the authority section with an empty path component, add a trailing backslash */ + if (*comp_end == 0 || *comp_end == '?' || *comp_end == '#') { + *write_buffer = '/'; + write_buffer++; + } + + /* If there is a path component, normalize it */ + else { + /* Set comp start/end pointers to contain the path component */ + while (*comp_end != '?' && *comp_end != '#' && *comp_end != 0) { + comp_end++; + } + /* Decode the path component without altering case and store it to the path_buffer*/ + comp_ct = comp_end - comp_start; + comp_ct = percent_decode(comp_start, comp_ct, path_buffer, false); + + if (comp_ct < 0) { + goto normalize_failure; + } + + /* Remove the . and .. segments from the path and write the now normalized path to the output buffer */ + PluginDebug("Removing Dot Segments"); + int buff_ct = buff_end - write_buffer; + comp_ct = remove_dot_segments(path_buffer, comp_ct, write_buffer, buff_ct); + + if (comp_ct < 0) { + PluginDebug("Failure removing dot segments from path"); + goto normalize_failure; + } + write_buffer = write_buffer + comp_ct; + } + + /* If there is any uri remaining after the path, decode and set case to lower */ + if (comp_end != uri_end) { + comp_start = comp_end; + comp_ct = uri_end - comp_start; + comp_ct = percent_decode(comp_start, comp_ct, write_buffer, false); + if (comp_ct < 0) { + goto normalize_failure; + } + } + + PluginDebug("Normalized URI: %s", normal_uri); + return 0; + +normalize_failure: + PluginDebug("URI Normalization Failure. URI does not fit http or https schemes."); + return -1; +} diff --git a/plugins/experimental/uri_signing/uri_signing.h b/plugins/experimental/uri_signing/normalize.h similarity index 80% rename from plugins/experimental/uri_signing/uri_signing.h rename to plugins/experimental/uri_signing/normalize.h index 6cb5046c2df..a84c9979718 100644 --- a/plugins/experimental/uri_signing/uri_signing.h +++ b/plugins/experimental/uri_signing/normalize.h @@ -16,7 +16,5 @@ * limitations under the License. */ -#define PLUGIN_NAME "uri_signing" - -#define PluginDebug(...) TSDebug("uri_signing", PLUGIN_NAME " " __VA_ARGS__) -#define PluginError(...) PluginDebug(__VA_ARGS__), TSError(PLUGIN_NAME " " __VA_ARGS__) +int normalize_uri(const char *uri, int uri_ct, char *uri_normal, int buffer_size); +int remove_dot_segments(const char *path, int path_ct, char *ret_buffer, int buff_ct); diff --git a/plugins/experimental/uri_signing/parse.c b/plugins/experimental/uri_signing/parse.c index ade5706a42d..3ca10b21d23 100644 --- a/plugins/experimental/uri_signing/parse.c +++ b/plugins/experimental/uri_signing/parse.c @@ -16,7 +16,7 @@ * limitations under the License. */ -#include "uri_signing.h" +#include "common.h" #include "parse.h" #include "config.h" #include "jwt.h" @@ -25,41 +25,68 @@ #include #include #include -#include #include cjose_jws_t * -get_jws_from_query(const char *uri, size_t uri_ct, const char *paramName) +get_jws_from_uri(const char *uri, size_t uri_ct, const char *paramName, char *strip_uri, size_t buff_ct, size_t *strip_ct) { - PluginDebug("Parsing JWS from query string: %.*s", (int)uri_ct, uri); - const char *query = uri; - const char *end = uri + uri_ct; - while (query != end && *query != '?') { - ++query; - } - if (query == end) { + /* Reserved characters as defined by the URI Generic Syntax RFC: https://tools.ietf.org/html/rfc3986#section-2.2 */ + static const char *reserved_string = ":/?#[]@!$&\'()*+,;="; + static const char *sub_delim_string = "!$&\'()*+,;="; + + /* If param name ends in reserved character this will be treated as the termination symbol when parsing for package. Default is + * '='. */ + char termination_symbol; + size_t termination_ct; + size_t param_ct = strlen(paramName); + + if (param_ct <= 0) { + PluginDebug("URI signing package name cannot be empty"); return NULL; } - ++query; + if (strchr(reserved_string, paramName[param_ct - 1])) { + termination_symbol = paramName[param_ct - 1]; + termination_ct = param_ct - 1; + } else { + termination_symbol = '='; + termination_ct = param_ct; + } + + PluginDebug("Parsing JWS from query string: %.*s", (int)uri_ct, uri); + const char *param = uri; + const char *end = uri + uri_ct; + const char *key, *key_end; + const char *value, *value_end; - const char *key = query, *key_end; - const char *value = query, *value_end; for (;;) { - while (value != end && *value != '=') { - ++value; + /* Search the URI for a reserved character. */ + while (param != end && strchr(reserved_string, *param) == NULL) { + ++param; } + if (param == end) { + break; + } + + ++param; + /* Parse the parameter for a key value pair separated by the termination symbol. */ + key = param; + value = param; + while (value != end && *value != termination_symbol) { + ++value; + } if (value == end) { break; } - key_end = value; - value_end = ++value; - while (value_end != end && *value_end != '&') { - ++value_end; - } + key_end = value; - if (!strncmp(paramName, key, (size_t)(key_end - key))) { + /* If the Parameter key is our target parameter name, attempt to import a JWS from the value. */ + if ((size_t)(key_end - key) == termination_ct && !strncmp(paramName, key, (size_t)(key_end - key))) { + value_end = ++value; + while (value_end != end && strchr(reserved_string, *value_end) == NULL) { + ++value_end; + } PluginDebug("Decoding JWS: %.*s", (int)(key_end - key), key); cjose_err err = {0}; cjose_jws_t *jws = cjose_jws_import(value, (size_t)(value_end - value), &err); @@ -67,15 +94,32 @@ get_jws_from_query(const char *uri, size_t uri_ct, const char *paramName) PluginDebug("Unable to read JWS: %.*s, %s", (int)(key_end - key), key, err.message ? err.message : ""); } else { PluginDebug("Parsed JWS: %.*s (%16p)", (int)(key_end - key), key, jws); + + /* Strip token */ + /* Check that passed buffer is large enough */ + *strip_ct = ((key - uri) + (end - value_end)); + if (buff_ct <= *strip_ct) { + PluginDebug("Strip URI buffer is not large enough"); + return NULL; + } + + if (value_end != end && strchr(sub_delim_string, *value_end)) { + /*Strip from first char of package name to sub-delimeter that terminates the signed JWT */ + memcpy(strip_uri, uri, (key - uri)); + memcpy(strip_uri + (key - uri), value_end + 1, (end - value_end + 1)); + } else { + /*Strip from reserved char to the last char of the JWT */ + memcpy(strip_uri, uri, (key - uri - 1)); + memcpy(strip_uri + (key - uri - 1), value_end, (end - value_end)); + } + + if (strip_uri[*strip_ct - 1] != '\0') { + strip_uri[*strip_ct - 1] = '\0'; + } + PluginDebug("Stripped URI: %s", strip_uri); } return jws; } - - if (value_end == end) { - break; - } - - key = value = value_end + 1; } PluginDebug("Unable to locate signing key in uri: %.*s", (int)uri_ct, uri); return NULL; @@ -142,7 +186,7 @@ validate_jws(cjose_jws_t *jws, struct config *cfg, const char *uri, size_t uri_c PluginDebug("Initial validation of JWT failed for %16p", jws); goto jwt_fail; } - TimerDebug("inital validation of jwt"); + TimerDebug("initial validation of jwt"); cjose_header_t *hdr = cjose_jws_get_protected(jws); TimerDebug("getting header of jws"); @@ -184,7 +228,12 @@ validate_jws(cjose_jws_t *jws, struct config *cfg, const char *uri, size_t uri_c } } - if (!jwt_check_uri(jwt->sub, uri)) { + if (!jwt_check_aud(jwt->aud, config_get_id(cfg))) { + PluginDebug("Valid key for %16p that does not match aud.", jws); + goto jwt_fail; + } + + if (!jwt_check_uri(jwt->cdniuc, uri)) { PluginDebug("Valid key for %16p that does not match uri.", jws); goto jwt_fail; } diff --git a/plugins/experimental/uri_signing/parse.h b/plugins/experimental/uri_signing/parse.h index 8002f8781fe..d05bfd59adb 100644 --- a/plugins/experimental/uri_signing/parse.h +++ b/plugins/experimental/uri_signing/parse.h @@ -19,7 +19,8 @@ #include struct _cjose_jws_int; -struct _cjose_jws_int *get_jws_from_query(const char *uri, size_t uri_ct, const char *paramName); +struct _cjose_jws_int *get_jws_from_uri(const char *uri, size_t uri_ct, const char *paramName, char *strip_uri, size_t buff_ct, + size_t *strip_ct); struct _cjose_jws_int *get_jws_from_cookie(const char **cookie, size_t *cookie_ct, const char *paramName); struct config; diff --git a/plugins/experimental/uri_signing/unit_tests/testConfig.config b/plugins/experimental/uri_signing/unit_tests/testConfig.config new file mode 100644 index 00000000000..aeecf363434 --- /dev/null +++ b/plugins/experimental/uri_signing/unit_tests/testConfig.config @@ -0,0 +1,102 @@ +{ + "Master Issuer": { + "renewal_kid": "6", + "id": "tester", + "auth_directives": [ + { + "auth": "allow", + "uri": "regex:invalid" + } + ], + "keys": [ + { + "alg": "HS256", + "k": "nxb7fyO5Z2hGz9E3oKm1357ptvC2su5QwQUb4YaIaIc", + "kid": "0", + "kty": "oct" + }, + { + "alg": "HS256", + "k": "cXKukBqFvQ0n3WAuRnWfExC14dmHdGoJULoZjGu9tJC", + "kid": "1", + "kty": "oct" + }, + { + "alg": "HS256", + "k": "38pJlSXfX87jWL0a03luml9QzUmM4qts1nmfIHA3B7r", + "kid": "2", + "kty": "oct" + }, + { + "alg": "HS256", + "k": "zNQPphknDGvzR5kA7IonXIDWKMyB1b8NpGmmDNlpgtM", + "kid": "3", + "kty": "oct" + }, + { + "alg": "HS256", + "k": "iB2ogCmQRt7r5hW7pgyP5FqiFcCl53MPQvfXv8wrZAn", + "kid": "4", + "kty": "oct" + }, + { + "alg": "HS256", + "k": "GJMCTyZhNoSOZvUOKmmY9MtGSLaONNLHqtKwsC3MWKo", + "kid": "5", + "kty": "oct" + }, + { + "alg": "HS256", + "k": "u2LziZKJFBnOfjUQUmvot7C9t91jj7ocJPIU9aDdbUl", + "kid": "6", + "kty": "oct" + }, + { + "alg": "HS256", + "k": "DRBKrBh87NYkH3UzfW1tWbiXCYXiYGZUE9w1orZngL0", + "kid": "7", + "kty": "oct" + }, + { + "alg": "HS256", + "k": "KNNKFbun8lEs7GbiKlo9mYGNdvpt33tdFzHbNnasDyP", + "kid": "8", + "kty": "oct" + }, + { + "alg": "HS256", + "k": "yb6kOddMUdupPRSkWMUdE6jrWT4MqUnVyTjpeJBYIqp", + "kid": "9", + "kty": "oct" + } + ] + }, + "Second Issuer": { + "keys": [ + { + "alg": "HS256", + "k": "testkey1", + "kid": "one", + "kty": "oct" + }, + { + "alg": "HS256", + "k": "testkey2", + "kid": "two", + "kty": "oct" + }, + { + "alg": "HS256", + "k": "testkey3", + "kid": "three", + "kty": "oct" + }, + { + "alg": "HS256", + "k": "testkey4", + "kid": "four", + "kty": "oct" + } + ] + } +} diff --git a/plugins/experimental/uri_signing/unit_tests/uri_signing_test.cc b/plugins/experimental/uri_signing/unit_tests/uri_signing_test.cc new file mode 100644 index 00000000000..522db9999fd --- /dev/null +++ b/plugins/experimental/uri_signing/unit_tests/uri_signing_test.cc @@ -0,0 +1,663 @@ +/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +/* + * These are misc unit tests for uri signing + */ + +#define CATCH_CONFIG_MAIN +#include "catch.hpp" + +extern "C" { +#include +#include +#include "../jwt.h" +#include "../normalize.h" +#include "../parse.h" +#include "../match.h" +#include "../config.h" +} + +bool +jwt_parsing_helper(const char *jwt_string) +{ + fprintf(stderr, "Parsing JWT from string: %s\n", jwt_string); + bool resp; + json_error_t jerr = {}; + size_t pt_ct = strlen(jwt_string); + struct jwt *jwt = parse_jwt(json_loadb(jwt_string, pt_ct, 0, &jerr)); + + if (jwt) { + resp = jwt_validate(jwt); + } else { + resp = false; + } + + jwt_delete(jwt); + return resp; +} + +bool +normalize_uri_helper(const char *uri, const char *expected_normal) +{ + size_t uri_ct = strlen(uri); + int buff_size = uri_ct + 2; + int err; + char *uri_normal = (char *)malloc(buff_size); + memset(uri_normal, 0, buff_size); + + err = normalize_uri(uri, uri_ct, uri_normal, buff_size); + + if (err) { + free(uri_normal); + return false; + } + + if (expected_normal && strcmp(expected_normal, uri_normal) == 0) { + free(uri_normal); + return true; + } + + free(uri_normal); + return false; +} + +bool +remove_dot_helper(const char *path, const char *expected_path) +{ + fprintf(stderr, "Removing Dot Segments from Path: %s\n", path); + size_t path_ct = strlen(path); + path_ct++; + int new_ct; + char path_buffer[path_ct]; + memset(path_buffer, 0, path_ct); + + new_ct = remove_dot_segments(path, path_ct, path_buffer, path_ct); + + if (new_ct < 0) { + return false; + } else if (strcmp(expected_path, path_buffer) == 0) { + return true; + } else { + return false; + } +} + +bool +jws_parsing_helper(const char *uri, const char *paramName, const char *expected_strip) +{ + bool resp; + size_t uri_ct = strlen(uri); + size_t strip_ct = 0; + + char *uri_strip = (char *)malloc(uri_ct + 1); + memset(uri_strip, 0, uri_ct + 1); + + cjose_jws_t *jws = get_jws_from_uri(uri, uri_ct, paramName, uri_strip, uri_ct, &strip_ct); + if (jws) { + resp = true; + if (strcmp(uri_strip, expected_strip) != 0) { + cjose_jws_release(jws); + resp = false; + } + } else { + resp = false; + } + cjose_jws_release(jws); + free(uri_strip); + return resp; +} + +TEST_CASE("1", "[JWSParsingTest]") +{ + INFO("TEST 1, Test JWT Parsing From Token Strings"); + + SECTION("Standard JWT Parsing") + { + REQUIRE(jwt_parsing_helper("{\"cdniets\":30,\"cdnistt\":1,\"exp\":7284188499,\"iss\":\"Content Access " + "Manager\",\"cdniuc\":\"uri-regex:http://foobar.local/testDir/*\"}")); + } + + SECTION("JWT Parsing With Unknown Claim") + { + REQUIRE(jwt_parsing_helper("{\"cdniets\":30,\"cdnistt\":1,\"exp\":7284188499,\"iss\":\"Content Access " + "Manager\",\"cdniuc\":\"uri-regex:http://foobar.local/testDir/" + "*\",\"jamesBond\":\"Something,Something_else\"}")); + } + + SECTION("JWT Parsing with unsupported crit claim passed") + { + REQUIRE(!jwt_parsing_helper("{\"cdniets\":30,\"cdnistt\":1,\"exp\":7284188499,\"iss\":\"Content Access " + "Manager\",\"cdniuc\":\"uri-regex:http://foobar.local/testDir/" + "*\",\"cdnicrit\":\"Something,Something_else\"}")); + } + + SECTION("JWT Parsing with empty exp claim") + { + REQUIRE(jwt_parsing_helper("{\"cdniets\":30,\"cdnistt\":1,\"iss\":\"Content Access " + "Manager\",\"cdniuc\":\"uri-regex:http://foobar.local/testDir/*\"}")); + } + + SECTION("JWT Parsing with unsupported cdniip claim") + { + REQUIRE(!jwt_parsing_helper("{\"cdniets\":30,\"cdnistt\":1,\"cdniip\":\"123.123.123.123\",\"iss\":\"Content Access " + "Manager\",\"cdniuc\":\"uri-regex:http://foobar.local/testDir/*\"}")); + } + + SECTION("JWT Parsing with unsupported value for cdnistd claim") + { + REQUIRE(!jwt_parsing_helper("{\"cdniets\":30,\"cdnistt\":1,\"cdnistd\":4,\"iss\":\"Content Access " + "Manager\",\"cdniuc\":\"uri-regex:http://foobar.local/testDir/*\"}")); + } + fprintf(stderr, "\n"); +} + +TEST_CASE("2", "[JWSFromURLTest]") +{ + INFO("TEST 2, Test JWT Parsing and Stripping From URLs"); + + SECTION("Token at end of URI") + { + REQUIRE(jws_parsing_helper( + "www.foo.com/hellothere/" + "URISigningPackage=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9." + "eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG9lIiwiaWF0IjoxNTE2MjM5MDIyfQ.SflKxwRJSMeKKF2QT4fwpMeJf36POk6yJV_adQssw5c", + "URISigningPackage", "www.foo.com/hellothere")); + } + + SECTION("No Token in URL") + { + REQUIRE(!jws_parsing_helper( + "www.foo.com/hellothere/" + "URISigningPackag=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9." + "eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG9lIiwiaWF0IjoxNTE2MjM5MDIyfQ.SflKxwRJSMeKKF2QT4fwpMeJf36POk6yJV_adQssw5c", + "URISigningPackage", NULL)); + } + + SECTION("Token in middle of the URL") + { + REQUIRE(jws_parsing_helper("www.foo.com/hellothere/" + "URISigningPackage=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9." + "eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG9lIiwiaWF0IjoxNTE2MjM5MDIyfQ." + "SflKxwRJSMeKKF2QT4fwpMeJf36POk6yJV_adQssw5c/Something/Else", + "URISigningPackage", "www.foo.com/hellothere/Something/Else")); + } + + SECTION("Token at the start of the URL") + { + REQUIRE(jws_parsing_helper(":URISigningPackage=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9." + "eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG9lIiwiaWF0IjoxNTE2MjM5MDIyfQ." + "SflKxwRJSMeKKF2QT4fwpMeJf36POk6yJV_adQssw5c/www.foo.com/hellothere/Something/Else", + "URISigningPackage", "/www.foo.com/hellothere/Something/Else")); + } + + SECTION("Pass empty path parameter at end") + { + REQUIRE(!jws_parsing_helper("www.foobar.com/hellothere/URISigningPackage=", "URISigningPackage", NULL)); + } + + SECTION("Pass empty path parameter in the middle of URL") + { + REQUIRE(!jws_parsing_helper("www.foobar.com/hellothere/URISigningPackage=/Something/Else", "URISigningPackage", NULL)); + } + + SECTION("Partial package name in previous path parameter") + { + REQUIRE(jws_parsing_helper("www.foobar.com/URISig/" + "URISigningPackage=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9." + "eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG9lIiwiaWF0IjoxNTE2MjM5MDIyfQ." + "SflKxwRJSMeKKF2QT4fwpMeJf36POk6yJV_adQssw5c/Something/Else", + "URISigningPackage", "www.foobar.com/URISig/Something/Else")); + } + + SECTION("Package comes directly after two reserved characters") + { + REQUIRE(jws_parsing_helper("www.foobar.com/" + ":URISigningPackage=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9." + "eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG9lIiwiaWF0IjoxNTE2MjM5MDIyfQ." + "SflKxwRJSMeKKF2QT4fwpMeJf36POk6yJV_adQssw5c/Something/Else", + "URISigningPackage", "www.foobar.com//Something/Else")); + } + + SECTION("Package comes directly after string of reserved characters") + { + REQUIRE(jws_parsing_helper("www.foobar.com/?!/" + ":URISigningPackage=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9." + "eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG9lIiwiaWF0IjoxNTE2MjM5MDIyfQ." + "SflKxwRJSMeKKF2QT4fwpMeJf36POk6yJV_adQssw5c/Something/Else", + "URISigningPackage", "www.foobar.com/?!//Something/Else")); + } + + SECTION("Invalid token passed before a valid token") + { + REQUIRE(!jws_parsing_helper("www.foobar.com/URISigningPackage=/" + "URISigningPackage=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9." + "eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG9lIiwiaWF0IjoxNTE2MjM5MDIyfQ." + "SflKxwRJSMeKKF2QT4fwpMeJf36POk6yJV_adQssw5c/Something/Else", + "URISigningPackage", NULL)); + } + + SECTION("Empty string as URL") { REQUIRE(!jws_parsing_helper("", "URISigningPackage", NULL)); } + + SECTION("Empty package name to parser") + { + REQUIRE(!jws_parsing_helper( + "www.foobar.com/" + "URISigningPackage=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9." + "eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG9lIiwiaWF0IjoxNTE2MjM5MDIyfQ.SflKxwRJSMeKKF2QT4fwpMeJf36POk6yJV_adQssw5c", + "", NULL)); + } + + SECTION("Custom package name with a reserved character - at the end of the URI") + { + REQUIRE(jws_parsing_helper( + "www.foobar.com/CustomPackage/" + "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG9lIiwiaWF0IjoxNTE2MjM5MDIyfQ." + "SflKxwRJSMeKKF2QT4fwpMeJf36POk6yJV_adQssw5c", + "CustomPackage/", "www.foobar.com")); + } + + SECTION("Custom package name with a reserved character - in the middle of the URI") + { + REQUIRE(jws_parsing_helper( + "www.foobar.com/CustomPackage/" + "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG9lIiwiaWF0IjoxNTE2MjM5MDIyfQ." + "SflKxwRJSMeKKF2QT4fwpMeJf36POk6yJV_adQssw5c/Something/Else", + "CustomPackage/", "www.foobar.com/Something/Else")); + } + + SECTION("URI signing package passed as the only a query parameter") + { + REQUIRE(jws_parsing_helper( + "www.foobar.com/Something/" + "Here?URISigningPackage=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9." + "eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG9lIiwiaWF0IjoxNTE2MjM5MDIyfQ.SflKxwRJSMeKKF2QT4fwpMeJf36POk6yJV_adQssw5c", + "URISigningPackage", "www.foobar.com/Something/Here")); + } + + SECTION("URI signing package passed as first of many query parameters") + { + REQUIRE(jws_parsing_helper("www.foobar.com/Something/" + "Here?URISigningPackage=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9." + "eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG9lIiwiaWF0IjoxNTE2MjM5MDIyfQ." + "SflKxwRJSMeKKF2QT4fwpMeJf36POk6yJV_adQssw5c&query3=foobar&query1=foo&query2=bar", + "URISigningPackage", "www.foobar.com/Something/Here?query3=foobar&query1=foo&query2=bar")); + } + + SECTION("URI signing package passed as one of many query parameters - passed in middle") + { + REQUIRE(jws_parsing_helper("www.foobar.com/Something/" + "Here?query1=foo&query2=bar&URISigningPackage=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9." + "eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG9lIiwiaWF0IjoxNTE2MjM5MDIyfQ." + "SflKxwRJSMeKKF2QT4fwpMeJf36POk6yJV_adQssw5c&query3=foobar", + "URISigningPackage", "www.foobar.com/Something/Here?query1=foo&query2=bar&query3=foobar")); + } + + SECTION("URI signing package passed as last of many query parameters") + { + REQUIRE(jws_parsing_helper("www.foobar.com/Something/" + "Here?query1=foo&query2=bar&URISigningPackage=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9." + "eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG9lIiwiaWF0IjoxNTE2MjM5MDIyfQ." + "SflKxwRJSMeKKF2QT4fwpMeJf36POk6yJV_adQssw5c", + "URISigningPackage", "www.foobar.com/Something/Here?query1=foo&query2=bar")); + } +} + +TEST_CASE("3", "[RemoveDotSegmentsTest]") +{ + INFO("TEST 3, Test Removal of Dot Segments From Paths"); + + SECTION("../bar test") { REQUIRE(remove_dot_helper("../bar", "bar")); } + + SECTION("./bar test") { REQUIRE(remove_dot_helper("./bar", "bar")); } + + SECTION(".././bar test") { REQUIRE(remove_dot_helper(".././bar", "bar")); } + + SECTION("./../bar test") { REQUIRE(remove_dot_helper("./../bar", "bar")); } + + SECTION("/foo/./bar test") { REQUIRE(remove_dot_helper("/foo/./bar", "/foo/bar")); } + + SECTION("/bar/./ test") { REQUIRE(remove_dot_helper("/bar/./", "/bar/")); } + + SECTION("/. test") { REQUIRE(remove_dot_helper("/.", "/")); } + + SECTION("/bar/. test") { REQUIRE(remove_dot_helper("/bar/.", "/bar/")); } + + SECTION("/foo/../bar test") { REQUIRE(remove_dot_helper("/foo/../bar", "/bar")); } + + SECTION("/bar/../ test") { REQUIRE(remove_dot_helper("/bar/../", "/")); } + + SECTION("/.. test") { REQUIRE(remove_dot_helper("/..", "/")); } + + SECTION("/bar/.. test") { REQUIRE(remove_dot_helper("/bar/..", "/")); } + + SECTION("/foo/bar/.. test") { REQUIRE(remove_dot_helper("/foo/bar/..", "/foo/")); } + + SECTION("Single . test") { REQUIRE(remove_dot_helper(".", "")); } + + SECTION("Single .. test") { REQUIRE(remove_dot_helper("..", "")); } + + SECTION("Test foo/bar/.. test") { REQUIRE(remove_dot_helper("foo/bar/..", "foo/")); } + + SECTION("Test Empty Path Segment") { REQUIRE(remove_dot_helper("", "")); } + + SECTION("Test mixed operations") { REQUIRE(remove_dot_helper("/foo/bar/././something/../foobar", "/foo/bar/foobar")); } + fprintf(stderr, "\n"); +} + +TEST_CASE("4", "[NormalizeTest]") +{ + INFO("TEST 4, Test Normalization of URIs"); + + SECTION("Testing passing too small of a URI to normalize") { REQUIRE(!normalize_uri_helper("ht", NULL)); } + + SECTION("Testing passing non http/https protocol") { REQUIRE(!normalize_uri_helper("ht:", NULL)); } + + SECTION("Passing a uri with half encoded value at end") { REQUIRE(!normalize_uri_helper("http://www.foobar.co%4", NULL)); } + + SECTION("Passing a uri with half encoded value in the middle") + { + REQUIRE(!normalize_uri_helper("http://www.foobar.co%4psomethin/Path", NULL)); + } + + SECTION("Passing a uri with an empty path parameter") + { + REQUIRE(normalize_uri_helper("http://www.foobar.com", "http://www.foobar.com/")); + } + + SECTION("Passing a uri with an empty path parameter and additional query params") + { + REQUIRE(normalize_uri_helper("http://www.foobar.com?query1=foo&query2=bar", "http://www.foobar.com/?query1=foo&query2=bar")); + } + + SECTION("Empty path parameter with port") + { + REQUIRE(normalize_uri_helper("http://www.foobar.com:9301?query1=foo&query2=bar", + "http://www.foobar.com:9301/?query1=foo&query2=bar")); + } + + SECTION("Passing a uri with a username and password") + { + REQUIRE(normalize_uri_helper("http://foo%40:PaSsword@www.Foo%42ar.coM:80/", "http://foo%40:PaSsword@www.foobar.com/")); + } + + SECTION("Testing Removal of standard http Port") + { + REQUIRE(normalize_uri_helper("http://foobar.com:80/Something/Here", "http://foobar.com/Something/Here")); + } + + SECTION("Testing Removal of standard https Port") + { + REQUIRE(normalize_uri_helper("https://foobar.com:443/Something/Here", "https://foobar.com/Something/Here")); + } + + SECTION("Testing passing of non-standard http Port") + { + REQUIRE(normalize_uri_helper("http://foobar.com:443/Something/Here", "http://foobar.com:443/Something/Here")); + } + + SECTION("Testing passing of non-standard https Port") + { + REQUIRE(normalize_uri_helper("https://foobar.com:80/Something/Here", "https://foobar.com:80/Something/Here")); + } + + SECTION("Testing the removal of . and .. in the path ") + { + REQUIRE( + normalize_uri_helper("https://foobar.com:80/Something/Here/././foobar/../foo", "https://foobar.com:80/Something/Here/foo")); + } + + SECTION("Testing . and .. segments in non path components") + { + REQUIRE(normalize_uri_helper("https://foobar.com:80/Something/Here?query1=/././foo/../bar", + "https://foobar.com:80/Something/Here?query1=/././foo/../bar")); + } + + SECTION("Testing standard decdoing of multiple characters") + { + REQUIRE(normalize_uri_helper("https://kelloggs%54ester.com/%53omething/Here", "https://kelloggstester.com/Something/Here")); + } + + SECTION("Testing passing encoded reserved characters") + { + REQUIRE( + normalize_uri_helper("https://kelloggs%54ester.com/%53omething/Here%3f", "https://kelloggstester.com/Something/Here%3F")); + } + + SECTION("Mixed Bag Test case") + { + REQUIRE(normalize_uri_helper("https://foo:something@kellogs%54ester.com:443/%53omething/.././here", + "https://foo:something@kellogstester.com/here")); + } + + SECTION("Testing empty hostname with userinfon") { REQUIRE(!normalize_uri_helper("https://foo:something@", NULL)); } + + SECTION("Testing empty uri after http://") { REQUIRE(!normalize_uri_helper("http://", NULL)); } + + SECTION("Testing http:///////") { REQUIRE(!normalize_uri_helper("http:///////", NULL)); } + + SECTION("Testing empty uri after http://?/") { REQUIRE(!normalize_uri_helper("http://?/", NULL)); } + fprintf(stderr, "\n"); +} + +TEST_CASE("5", "[RegexTests]") +{ + INFO("TEST 5, Test Regex Matching"); + + SECTION("Standard regex") + { + REQUIRE(match_regex("http://kelloggsTester.souza.local/KellogsDir/*", + "http://kelloggsTester.souza.local/KellogsDir/some_manifest.m3u8")); + } + + SECTION("Back references are not supported") { REQUIRE(!match_regex("(b*a)\\1$", "bbbbba")); } + + SECTION("Escape a special character") { REQUIRE(match_regex("money\\$", "money$bags")); } + + SECTION("Dollar sign") + { + REQUIRE(!match_regex(".+foobar$", "foobarfoofoo")); + REQUIRE(match_regex(".+foobar$", "foofoofoobar")); + } + + SECTION("Number Quantifier with Groups") + { + REQUIRE(match_regex("(abab){2}", "abababab")); + REQUIRE(!match_regex("(abab){2}", "abab")); + } + + SECTION("Alternation") { REQUIRE(match_regex("cat|dog", "dog")); } + fprintf(stderr, "\n"); +} + +TEST_CASE("6", "[AudTests]") +{ + INFO("TEST 6, Test Aud Matching"); + + json_error_t *err = NULL; + SECTION("Standard aud string match") + { + json_t *raw = json_loads("{\"aud\": \"tester\"}", 0, err); + json_t *aud = json_object_get(raw, "aud"); + REQUIRE(jwt_check_aud(aud, "tester")); + json_decref(aud); + json_decref(raw); + } + + SECTION("Standard aud array match") + { + json_t *raw = json_loads("{\"aud\": [ \"foo\", \"bar\", \"tester\"]}", 0, err); + json_t *aud = json_object_get(raw, "aud"); + REQUIRE(jwt_check_aud(aud, "tester")); + json_decref(aud); + json_decref(raw); + } + + SECTION("Standard aud string mismatch") + { + json_t *raw = json_loads("{\"aud\": \"foo\"}", 0, err); + json_t *aud = json_object_get(raw, "aud"); + REQUIRE(!jwt_check_aud(aud, "tester")); + json_decref(aud); + json_decref(raw); + } + + SECTION("Standard aud array mismatch") + { + json_t *raw = json_loads("{\"aud\": [\"foo\", \"bar\", \"foobar\"]}", 0, err); + json_t *aud = json_object_get(raw, "aud"); + REQUIRE(!jwt_check_aud(aud, "tester")); + json_decref(aud); + json_decref(raw); + } + + SECTION("Integer trying to pass as an aud") + { + json_t *raw = json_loads("{\"aud\": 1}", 0, err); + json_t *aud = json_object_get(raw, "aud"); + REQUIRE(!jwt_check_aud(aud, "tester")); + json_decref(aud); + json_decref(raw); + } + + SECTION("Integer mixed into a passing aud array") + { + json_t *raw = json_loads("{\"aud\": [1, \"foo\", \"bar\", \"tester\"]}", 0, err); + json_t *aud = json_object_get(raw, "aud"); + REQUIRE(jwt_check_aud(aud, "tester")); + json_decref(aud); + json_decref(raw); + } + + SECTION("Case sensitive test for single string") + { + json_t *raw = json_loads("{\"aud\": \"TESTer\"}", 0, err); + json_t *aud = json_object_get(raw, "aud"); + REQUIRE(!jwt_check_aud(aud, "tester")); + json_decref(aud); + json_decref(raw); + } + + SECTION("Case sensitive test for array") + { + json_t *raw = json_loads("{\"aud\": [1, \"foo\", \"bar\", \"Tester\"]}", 0, err); + json_t *aud = json_object_get(raw, "aud"); + REQUIRE(!jwt_check_aud(aud, "tester")); + json_decref(aud); + json_decref(raw); + } + + fprintf(stderr, "\n"); +} + +TEST_CASE("7", "[TestsConfig]") +{ + INFO("TEST 7, Config Loading and Config Functions"); + + SECTION("Config Loading ID Field") + { + struct config *cfg = read_config("experimental/uri_signing/unit_tests/testConfig.config"); + REQUIRE(cfg != NULL); + REQUIRE(strcmp(config_get_id(cfg), "tester") == 0); + config_delete(cfg); + } + fprintf(stderr, "\n"); +} + +bool +jws_validation_helper(const char *url, const char *package, struct config *cfg) +{ + size_t url_ct = strlen(url); + size_t strip_ct = 0; + char uri_strip[url_ct + 1]; + memset(uri_strip, 0, sizeof uri_strip); + cjose_jws_t *jws = get_jws_from_uri(url, url_ct, package, uri_strip, url_ct, &strip_ct); + if (!jws) { + return false; + } + struct jwt *jwt = validate_jws(jws, cfg, uri_strip, strip_ct); + if (jwt) { + jwt_delete(jwt); + cjose_jws_release(jws); + return true; + } + cjose_jws_release(jws); + return false; +} + +TEST_CASE("8", "[TestsWithConfig]") +{ + INFO("TEST 8, Tests Involving Validation with Config"); + struct config *cfg = read_config("experimental/uri_signing/unit_tests/testConfig.config"); + + SECTION("Validation of Valid Aud String in JWS") + { + REQUIRE(jws_validation_helper("http://www.foobar.com/" + "URISigningPackage=eyJLZXlJREtleSI6IjUiLCJhbGciOiJIUzI1NiJ9." + "eyJjZG5pZXRzIjozMCwiY2RuaXN0dCI6MSwiaXNzIjoiTWFzdGVyIElzc3VlciIsImF1ZCI6InRlc3RlciIsImNkbml1YyI6" + "InJlZ2V4Omh0dHA6Ly93d3cuZm9vYmFyLmNvbS8qIn0.InBxVm6OOAglNqc-U5wAZaRQVebJ9PK7Y9i7VFHWYHU", + "URISigningPackage", cfg)); + fprintf(stderr, "\n"); + } + + SECTION("Validation of Invalid Aud String in JWS") + { + REQUIRE(!jws_validation_helper("http://www.foobar.com/" + "URISigningPackage=eyJLZXlJREtleSI6IjUiLCJhbGciOiJIUzI1NiJ9." + "eyJjZG5pZXRzIjozMCwiY2RuaXN0dCI6MSwiaXNzIjoiTWFzdGVyIElzc3VlciIsImF1ZCI6ImJhZCIsImNkbml1YyI6InJ" + "lZ2V4Omh0dHA6Ly93d3cuZm9vYmFyLmNvbS8qIn0.aCOo8gOBa5G1RKkkzgWYwc79dPRw_fQUC0k1sWcjkyM", + "URISigningPackage", cfg)); + fprintf(stderr, "\n"); + } + + SECTION("Validation of Valid Aud Array in JWS") + { + REQUIRE(jws_validation_helper( + "http://www.foobar.com/" + "URISigningPackage=eyJLZXlJREtleSI6IjUiLCJhbGciOiJIUzI1NiJ9." + "eyJjZG5pZXRzIjozMCwiY2RuaXN0dCI6MSwiaXNzIjoiTWFzdGVyIElzc3VlciIsImF1ZCI6WyJiYWQiLCJpbnZhbGlkIiwidGVzdGVyIl0sImNkbml1YyI6InJl" + "Z2V4Omh0dHA6Ly93d3cuZm9vYmFyLmNvbS8qIn0.7lyepZMzc_odieKvOTN2U-k1gLwRKS8KJIvDFQXDqGs", + "URISigningPackage", cfg)); + fprintf(stderr, "\n"); + } + + SECTION("Validation of Invalid Aud Array in JWS") + { + REQUIRE(!jws_validation_helper( + "http://www.foobar.com/" + "URISigningPackage=eyJLZXlJREtleSI6IjUiLCJhbGciOiJIUzI1NiJ9." + "eyJjZG5pZXRzIjozMCwiY2RuaXN0dCI6MSwiaXNzIjoiTWFzdGVyIElzc3VlciIsImF1ZCI6WyJiYWQiLCJpbnZhbGlkIiwiZm9vYmFyIl0sImNkbml1YyI6InJl" + "Z2V4Omh0dHA6Ly93d3cuZm9vYmFyLmNvbS8qIn0.CU3WMJAPs0uRC7NKXvatVG9uU9SANdZzqO0GdQUatxk", + "URISigningPackage", cfg)); + fprintf(stderr, "\n"); + } + + SECTION("Validation of Valid Aud Array Mixed types in JWS") + { + REQUIRE(jws_validation_helper( + "http://www.foobar.com/" + "URISigningPackage=eyJLZXlJREtleSI6IjUiLCJhbGciOiJIUzI1NiJ9." + "eyJjZG5pZXRzIjozMCwiY2RuaXN0dCI6MSwiaXNzIjoiTWFzdGVyIElzc3VlciIsImF1ZCI6WyJiYWQiLDEsImZvb2JhciIsInRlc3RlciJdLCJjZG5pdWMiOiJy" + "ZWdleDpodHRwOi8vd3d3LmZvb2Jhci5jb20vKiJ9._vlXsA3r7RPje2ZdMnpaGTwIsdNMjuQWPEHRkGKTVL8", + "URISigningPackage", cfg)); + fprintf(stderr, "\n"); + } + + config_delete(cfg); + fprintf(stderr, "\n"); +} diff --git a/plugins/experimental/uri_signing/uri_signing.c b/plugins/experimental/uri_signing/uri_signing.c index c648d183c76..f85a87b5ed3 100644 --- a/plugins/experimental/uri_signing/uri_signing.c +++ b/plugins/experimental/uri_signing/uri_signing.c @@ -16,16 +16,16 @@ * limitations under the License. */ -#include "uri_signing.h" +#include "common.h" #include "config.h" #include "parse.h" #include "jwt.h" #include "timing.h" -#include #include #include +#include #include #include @@ -46,7 +46,7 @@ TSRemapInit(TSRemapInterface *api_info, char *errbuf, int errbuf_size) return TS_ERROR; } - TSDebug(PLUGIN_NAME, "plugin is succesfully initialized"); + TSDebug(PLUGIN_NAME, "plugin is successfully initialized"); return TS_SUCCESS; } @@ -157,6 +157,8 @@ TSRemapDoRemap(void *ih, TSHttpTxn txnp, TSRemapRequestInfo *rri) int cpi = 0; int url_ct = 0; const char *url = NULL; + char *strip_uri = NULL; + TSRemapStatus status = TSREMAP_NO_REMAP; const char *package = "URISigningPackage"; @@ -175,13 +177,23 @@ TSRemapDoRemap(void *ih, TSHttpTxn txnp, TSRemapRequestInfo *rri) if (cpi < max_cpi) { checkpoints[cpi++] = mark_timer(&t); } - cjose_jws_t *jws = get_jws_from_query(url, url_ct, package); + + int strip_size = url_ct + 1; + strip_uri = (char *)TSmalloc(strip_size); + memset(strip_uri, 0, strip_size); + + size_t strip_ct; + cjose_jws_t *jws = get_jws_from_uri(url, url_ct, package, strip_uri, strip_size, &strip_ct); + if (cpi < max_cpi) { checkpoints[cpi++] = mark_timer(&t); } int checked_cookies = 0; if (!jws) { check_cookies: + /* There is no valid token in the url */ + strncpy(strip_uri, url, url_ct); + strip_ct = url_ct; ++checked_cookies; TSMLoc field; @@ -214,6 +226,55 @@ TSRemapDoRemap(void *ih, TSHttpTxn txnp, TSRemapRequestInfo *rri) checkpoints[cpi++] = mark_timer(&t); } jws = get_jws_from_cookie(&client_cookie, &client_cookie_sz_ct, package); + } else { + /* There has been a JWS found in the url */ + /* Strip the token from the URL for upstream if configured to do so */ + if (config_strip_token((struct config *)ih)) { + if ((int)strip_ct != url_ct) { + int map_url_ct = 0; + char *map_url = NULL; + char *map_strip_uri = NULL; + map_url = TSUrlStringGet(rri->requestBufp, rri->requestUrl, &map_url_ct); + + PluginDebug("Stripping Token from requestUrl: %s", map_url); + + int map_strip_size = map_url_ct + 1; + map_strip_uri = (char *)TSmalloc(map_strip_size); + memset(map_strip_uri, 0, map_strip_size); + size_t map_strip_ct = 0; + + cjose_jws_t *map_jws = get_jws_from_uri(map_url, map_url_ct, package, map_strip_uri, map_strip_size, &map_strip_ct); + cjose_jws_release(map_jws); + + char *strip_uri_start = &map_strip_uri[0]; + char *strip_uri_end = &map_strip_uri[map_strip_ct]; + PluginDebug("Stripping token from upstream url to: %s", strip_uri_start); + + TSParseResult parse_rc = TSUrlParse(rri->requestBufp, rri->requestUrl, (const char **)&strip_uri_start, strip_uri_end); + if (map_url != NULL) { + TSfree(map_url); + } + if (map_strip_uri != NULL) { + TSfree(map_strip_uri); + } + + if (parse_rc != TS_PARSE_DONE) { + PluginDebug("Error in TSUrlParse"); + goto fail; + } + status = TSREMAP_DID_REMAP; + } + } + } + /* Check auth_dir and pass through if configured */ + if (uri_matches_auth_directive((struct config *)ih, url, url_ct)) { + if (url != NULL) { + TSfree((void *)url); + } + if (strip_uri != NULL) { + TSfree(strip_uri); + } + return TSREMAP_NO_REMAP; } if (!jws) { goto fail; @@ -222,8 +283,10 @@ TSRemapDoRemap(void *ih, TSHttpTxn txnp, TSRemapRequestInfo *rri) if (cpi < max_cpi) { checkpoints[cpi++] = mark_timer(&t); } - struct jwt *jwt = validate_jws(jws, (struct config *)ih, url, url_ct); + + struct jwt *jwt = validate_jws(jws, (struct config *)ih, strip_uri, strip_ct); cjose_jws_release(jws); + if (cpi < max_cpi) { checkpoints[cpi++] = mark_timer(&t); } @@ -235,6 +298,8 @@ TSRemapDoRemap(void *ih, TSHttpTxn txnp, TSRemapRequestInfo *rri) } } + /* There has been a validated JWT found in either the cookie or url */ + struct signer *signer = config_signer((struct config *)ih); char *cookie = renew(jwt, signer->issuer, signer->jwk, signer->alg, package); jwt_delete(jwt); @@ -256,16 +321,13 @@ TSRemapDoRemap(void *ih, TSHttpTxn txnp, TSRemapRequestInfo *rri) last_mark = checkpoints[i]; } PluginDebug("Spent %" PRId64 " ns uri_signing verification of %.*s.", mark_timer(&t), url_ct, url); + TSfree((void *)url); - return TSREMAP_NO_REMAP; -fail: - if (uri_matches_auth_directive((struct config *)ih, url, url_ct)) { - if (url != NULL) { - TSfree((void *)url); - } - return TSREMAP_NO_REMAP; + if (strip_uri != NULL) { + TSfree(strip_uri); } - + return status; +fail: PluginDebug("Invalid JWT for %.*s", url_ct, url); TSHttpTxnStatusSet(txnp, TS_HTTP_STATUS_FORBIDDEN); PluginDebug("Spent %" PRId64 " ns uri_signing verification of %.*s.", mark_timer(&t), url_ct, url); @@ -273,6 +335,9 @@ TSRemapDoRemap(void *ih, TSHttpTxn txnp, TSRemapRequestInfo *rri) if (url != NULL) { TSfree((void *)url); } + if (strip_uri != NULL) { + TSfree(strip_uri); + } return TSREMAP_DID_REMAP; } diff --git a/plugins/experimental/url_sig/README b/plugins/experimental/url_sig/README index a8b4360d634..c7e397845db 100644 --- a/plugins/experimental/url_sig/README +++ b/plugins/experimental/url_sig/README @@ -85,7 +85,7 @@ Signing a URL using path parameters instead of using a query string. the file part of the request and are never part of the sign string. Path parameters are separated by a ';' in the path. The complete signature - string is base64 encoded as a single path parameter that is assinged to the + string is base64 encoded as a single path parameter that is assigned to the 'siganchor', and will appear in that path as siganchor=base64string. The following is an example signed request using the path parameter method and with an origin application query string: @@ -133,7 +133,7 @@ Example Add the remap rule like - map http://test-remap.domain.com http://google.com @plugin=url_sig.so @pparam=sign_test.config + map http://test-remap.domain.com http://google.com @plugin=url_sig.so @pparam=sign_test.config @pparam=pristineurl Restart traffic server or traffic_ctl config reload - verify there are no errors in diags.log diff --git a/plugins/experimental/url_sig/genkeys.pl b/plugins/experimental/url_sig/genkeys.pl index ae5bc0723d0..38cc5235ecc 100755 --- a/plugins/experimental/url_sig/genkeys.pl +++ b/plugins/experimental/url_sig/genkeys.pl @@ -17,11 +17,11 @@ # limitations under the License. my $len = 32; -my @chars = ( 'a' .. 'z', 'A' .. 'Z', '0' .. '9', '_' ); -foreach my $i ( 0 .. 15 ) { +my @chars = ('a' .. 'z', 'A' .. 'Z', '0' .. '9', '_'); +foreach my $i (0 .. 15) { my $string = ""; - foreach ( 1 .. $len ) { - $string .= $chars[ rand @chars ]; + foreach (1 .. $len) { + $string .= $chars[rand @chars]; } print "key" . $i . " = " . $string . "\n"; } diff --git a/plugins/experimental/url_sig/sign.pl b/plugins/experimental/url_sig/sign.pl index 6de4cc62eb8..7cf3850dd0c 100755 --- a/plugins/experimental/url_sig/sign.pl +++ b/plugins/experimental/url_sig/sign.pl @@ -22,50 +22,50 @@ use MIME::Base64::URLSafe (); use strict; use warnings; -my $key = undef; -my $string = undef; -my $useparts = undef; -my $result = undef; -my $duration = undef; -my $keyindex = undef; -my $verbose = 0; -my $url = undef; -my $client = undef; -my $algorithm = 1; +my $key = undef; +my $string = undef; +my $useparts = undef; +my $result = undef; +my $duration = undef; +my $keyindex = undef; +my $verbose = 0; +my $url = undef; +my $client = undef; +my $algorithm = 1; my $pathparams = 0; my $sig_anchor = undef; -my $proxy = undef; -my $scheme = "http://"; +my $proxy = undef; +my $scheme = "http://"; $result = GetOptions( - "url=s" => \$url, - "useparts=s" => \$useparts, - "duration=i" => \$duration, - "key=s" => \$key, - "client=s" => \$client, - "algorithm=i" => \$algorithm, - "keyindex=i" => \$keyindex, - "verbose" => \$verbose, - "pathparams" => \$pathparams, - "proxy=s" => \$proxy, - "siganchor=s" => \$sig_anchor + "url=s" => \$url, + "useparts=s" => \$useparts, + "duration=i" => \$duration, + "key=s" => \$key, + "client=s" => \$client, + "algorithm=i" => \$algorithm, + "keyindex=i" => \$keyindex, + "verbose" => \$verbose, + "pathparams" => \$pathparams, + "proxy=s" => \$proxy, + "siganchor=s" => \$sig_anchor ); -if ( !defined($key) || !defined($url) || !defined($duration) || !defined($keyindex) ) { - &help(); - exit(1); -} -if ( defined($proxy) ) { - if ($proxy !~ /http\:\/\/.*\:\d\d/) { +if (!defined($key) || !defined($url) || !defined($duration) || !defined($keyindex)) { &help(); - } + exit(1); +} +if (defined($proxy)) { + if ($proxy !~ /http\:\/\/.*\:\d\d/) { + &help(); + } } if ($url =~ m/^https/) { - $url =~ s/^https:\/\///; - $scheme = "https://"; + $url =~ s/^https:\/\///; + $scheme = "https://"; } else { - $url =~ s/^http:\/\///; + $url =~ s/^http:\/\///; } my $url_prefix = $url; @@ -77,87 +77,90 @@ my @inactive_parts = (); my $query_params = undef; -my $urlHasParams = index($url,"?"); -my $file = undef; +my $urlHasParams = index($url, "?"); +my $file = undef; my @parts = (split(/\//, $url)); my $parts_size = scalar(@parts); if ($pathparams) { - if (scalar(@parts) > 1) { - $file = pop @parts; - } else { - print STDERR "\nERROR: No file segment in the path when using --pathparams.\n\n"; - &help(); - exit 1; - } - if($urlHasParams) { - $file = (split(/\?/, $file))[0]; - } - $parts_size = scalar(@parts); + if (scalar(@parts) > 1) { + $file = pop @parts; + } else { + print STDERR "\nERROR: No file segment in the path when using --pathparams.\n\n"; + &help(); + exit 1; + } + if ($urlHasParams) { + $file = (split(/\?/, $file))[0]; + } + $parts_size = scalar(@parts); } if ($urlHasParams > 0) { - if ( ! $pathparams) { - ($parts[$parts_size -1], $query_params) = (split(/\?/, $parts[$parts_size - 1])); - } else { - $query_params = (split(/\?/, $url))[1]; - } + if (!$pathparams) { + ($parts[$parts_size - 1], $query_params) = (split(/\?/, $parts[$parts_size - 1])); + } else { + $query_params = (split(/\?/, $url))[1]; + } } foreach my $part (@parts) { - if ( length($useparts) > $i ) { - $part_active = substr( $useparts, $i++, 1 ); - } - if ($part_active) { - $string .= $part . "/"; - } - else { - $inactive_parts[$j] = $part; - } - $j++; + if (length($useparts) > $i) { + $part_active = substr($useparts, $i++, 1); + } + if ($part_active) { + $string .= $part . "/"; + } else { + $inactive_parts[$j] = $part; + } + $j++; } my $signing_signature = undef; chop($string); if ($pathparams) { - if ( defined($client) ) { - $signing_signature = ";C=" . $client . ";E=" . ( time() + $duration ) . ";A=" . $algorithm . ";K=" . $keyindex . ";P=" . $useparts . ";S="; - $string .= $signing_signature; - } - else { - $signing_signature = ";E=" . ( time() + $duration ) . ";A=" . $algorithm . ";K=" . $keyindex . ";P=" . $useparts . ";S="; - $string .= $signing_signature; - } -} else { - if ( defined($client) ) { - if ($urlHasParams > 0) { - $signing_signature = "?$query_params" . "&C=" . $client . "&E=" . ( time() + $duration ) . "&A=" . $algorithm . "&K=" . $keyindex . "&P=" . $useparts . "&S="; - $string .= $signing_signature; - } - else { - $signing_signature = "?C=" . $client . "&E=" . ( time() + $duration ) . "&A=" . $algorithm . "&K=" . $keyindex . "&P=" . $useparts . "&S="; - $string .= $signing_signature; - } - } - else { - if ($urlHasParams > 0) { - $signing_signature = "?$query_params" . "&E=" . ( time() + $duration ) . "&A=" . $algorithm . "&K=" . $keyindex . "&P=" . $useparts . "&S="; - $string .= $signing_signature; + if (defined($client)) { + $signing_signature = + ";C=" . $client . ";E=" . (time() + $duration) . ";A=" . $algorithm . ";K=" . $keyindex . ";P=" . $useparts . ";S="; + $string .= $signing_signature; + } else { + $signing_signature = ";E=" . (time() + $duration) . ";A=" . $algorithm . ";K=" . $keyindex . ";P=" . $useparts . ";S="; + $string .= $signing_signature; } - else { - $signing_signature = "?E=" . ( time() + $duration ) . "&A=" . $algorithm . "&K=" . $keyindex . "&P=" . $useparts . "&S="; - $string .= $signing_signature; +} else { + if (defined($client)) { + if ($urlHasParams > 0) { + $signing_signature = + "?$query_params" . "&C=" + . $client . "&E=" + . (time() + $duration) . "&A=" + . $algorithm . "&K=" + . $keyindex . "&P=" + . $useparts . "&S="; + $string .= $signing_signature; + } else { + $signing_signature = + "?C=" . $client . "&E=" . (time() + $duration) . "&A=" . $algorithm . "&K=" . $keyindex . "&P=" . $useparts . "&S="; + $string .= $signing_signature; + } + } else { + if ($urlHasParams > 0) { + $signing_signature = + "?$query_params" . "&E=" . (time() + $duration) . "&A=" . $algorithm . "&K=" . $keyindex . "&P=" . $useparts . "&S="; + $string .= $signing_signature; + } else { + $signing_signature = "?E=" . (time() + $duration) . "&A=" . $algorithm . "&K=" . $keyindex . "&P=" . $useparts . "&S="; + $string .= $signing_signature; + } } - } } my $digest; -if ( $algorithm == 1 ) { - $digest = hmac_sha1_hex( $string, $key ); -} -else { - $digest = hmac_md5_hex( $string, $key ); +if ($algorithm == 1) { + $digest = hmac_sha1_hex($string, $key); +} else { + $digest = hmac_md5_hex($string, $key); } $verbose && print "\nSigned String: $string\n\n"; @@ -165,82 +168,96 @@ $verbose && print "\nsigning_signature: $signing_signature\n"; $verbose && print "\ndigest: $digest\n"; -if ($urlHasParams == -1) { # no application query parameters. - if ( ! defined($proxy)) { - if ( ! $pathparams) { - print "curl -s -o /dev/null -v --max-redirs 0 '$scheme" . $url . $signing_signature . $digest . "'\n\n"; - } else { - my $index = rindex($url, '/'); - $url = substr($url,0,$index); - my $encoded = MIME::Base64::URLSafe::encode($signing_signature . $digest); - if (defined($sig_anchor)) { - print "curl -s -o /dev/null -v --max-redirs 0 '$scheme" . $url . ";${sig_anchor}=" . $encoded . "/$file" . "'\n\n"; +if ($urlHasParams == -1) { # no application query parameters. + if (!defined($proxy)) { + if (!$pathparams) { + print "curl -s -o /dev/null -v --max-redirs 0 '$scheme" . $url . $signing_signature . $digest . "'\n\n"; } else { - print "curl -s -o /dev/null -v --max-redirs 0 '$scheme" . $url . "/" . $encoded . "/$file" . "'\n\n"; + my $index = rindex($url, '/'); + $url = substr($url, 0, $index); + my $encoded = MIME::Base64::URLSafe::encode($signing_signature . $digest); + if (defined($sig_anchor)) { + print "curl -s -o /dev/null -v --max-redirs 0 '$scheme" . $url . ";${sig_anchor}=" . $encoded . "/$file" . "'\n\n"; + } else { + print "curl -s -o /dev/null -v --max-redirs 0 '$scheme" . $url . "/" . $encoded . "/$file" . "'\n\n"; + } } - } } else { - if ( ! $pathparams) { - print "curl -s -o /dev/null -v --max-redirs 0 --proxy $proxy '$scheme" . $url . $signing_signature . $digest . - "'\n\n"; - } else { - my $index = rindex($url, '/'); - $url = substr($url,0,$index); - my $encoded = MIME::Base64::URLSafe::encode($signing_signature . $digest); - if (defined($sig_anchor)) { - print "curl -s -o /dev/null -v --max-redirs 0 --proxy $proxy '$scheme" . $url . ";${sig_anchor}=" . $encoded . "/$file" . "'\n\n"; + if (!$pathparams) { + print "curl -s -o /dev/null -v --max-redirs 0 --proxy $proxy '$scheme" . $url . $signing_signature . $digest . "'\n\n"; } else { - print "curl -s -o /dev/null -v --max-redirs 0 --proxy $proxy '$scheme" . $url . "/" . $encoded . "/$file" . "'\n\n"; + my $index = rindex($url, '/'); + $url = substr($url, 0, $index); + my $encoded = MIME::Base64::URLSafe::encode($signing_signature . $digest); + if (defined($sig_anchor)) { + print "curl -s -o /dev/null -v --max-redirs 0 --proxy $proxy '$scheme" + . $url + . ";${sig_anchor}=" + . $encoded + . "/$file" . "'\n\n"; + } else { + print "curl -s -o /dev/null -v --max-redirs 0 --proxy $proxy '$scheme" . $url . "/" . $encoded . "/$file" . "'\n\n"; + } } - } } -} else { # has application parameters. +} else { # has application parameters. $url = (split(/\?/, $url))[0]; - if ( ! defined($proxy)) { - if ( ! $pathparams) { - print "curl -s -o /dev/null -v --max-redirs 0 '$scheme" . $url . $signing_signature . $digest . "'\n\n"; - } else { - my $index = rindex($url, '/'); - $url = substr($url,0,$index); - my $encoded = MIME::Base64::URLSafe::encode($signing_signature . $digest); - if (defined($sig_anchor)) { - print "curl -s -o /dev/null -v --max-redirs 0 '$scheme" . $url . ";${sig_anchor}=" . $encoded . "/" . $file . "?$query_params" - . "'\n\n"; + if (!defined($proxy)) { + if (!$pathparams) { + print "curl -s -o /dev/null -v --max-redirs 0 '$scheme" . $url . $signing_signature . $digest . "'\n\n"; } else { - print "curl -s -o /dev/null -v --max-redirs 0 '$scheme" . $url . "/" . $encoded . "/" . $file . "?$query_params" - . "'\n\n"; + my $index = rindex($url, '/'); + $url = substr($url, 0, $index); + my $encoded = MIME::Base64::URLSafe::encode($signing_signature . $digest); + if (defined($sig_anchor)) { + print "curl -s -o /dev/null -v --max-redirs 0 '$scheme" + . $url + . ";${sig_anchor}=" + . $encoded . "/" + . $file + . "?$query_params" . "'\n\n"; + } else { + print "curl -s -o /dev/null -v --max-redirs 0 '$scheme" . $url . "/" . $encoded . "/" . $file . "?$query_params" + . "'\n\n"; + } } - } } else { - if ( ! $pathparams) { - print "curl -s -o /dev/null -v --max-redirs 0 --proxy $proxy '$scheme" . $url . $signing_signature . $digest . - "'\n\n"; - } else { - my $index = rindex($url, '/'); - $url = substr($url,0,$index); - my $encoded = MIME::Base64::URLSafe::encode($signing_signature . $digest); - if (defined($sig_anchor)) { - print "curl -s -o /dev/null -v --max-redirs 0 --proxy $proxy '$scheme" . $url . ";${sig_anchor}=" . $encoded . "/" . $file . "?$query_params" - . "'\n\n"; + if (!$pathparams) { + print "curl -s -o /dev/null -v --max-redirs 0 --proxy $proxy '$scheme" . $url . $signing_signature . $digest . "'\n\n"; } else { - print "curl -s -o /dev/null -v --max-redirs 0 --proxy $proxy '$scheme" . $url . "/" . $encoded . "/$file?$query_params" . "'\n\n"; + my $index = rindex($url, '/'); + $url = substr($url, 0, $index); + my $encoded = MIME::Base64::URLSafe::encode($signing_signature . $digest); + if (defined($sig_anchor)) { + print "curl -s -o /dev/null -v --max-redirs 0 --proxy $proxy '$scheme" + . $url + . ";${sig_anchor}=" + . $encoded . "/" + . $file + . "?$query_params" . "'\n\n"; + } else { + print "curl -s -o /dev/null -v --max-redirs 0 --proxy $proxy '$scheme" + . $url . "/" + . $encoded + . "/$file?$query_params" . "'\n\n"; + } } - } } } -sub help { - print "sign.pl - Example signing utility in perl for signed URLs\n"; - print "Usage: \n"; - print " ./sign.pl --url \\ \n"; - print " --useparts \\ \n"; - print " --algorithm \\ \n"; - print " --duration \\ \n"; - print " --keyindex \\ \n"; - print " [--client ] \\ \n"; - print " --key \\ \n"; - print " [--verbose] \n"; - print " [--pathparams] \n"; - print " [--proxy ] ex value: http://myproxy:80\n"; - print "\n"; +sub help +{ + print "sign.pl - Example signing utility in perl for signed URLs\n"; + print "Usage: \n"; + print " ./sign.pl --url \\ \n"; + print " --useparts \\ \n"; + print " --algorithm \\ \n"; + print " --duration \\ \n"; + print " --keyindex \\ \n"; + print " [--client ] \\ \n"; + print " --key \\ \n"; + print " [--verbose] \n"; + print " [--pathparams] \n"; + print " [--proxy ] ex value: http://myproxy:80\n"; + print "\n"; } diff --git a/plugins/experimental/url_sig/url_sig.c b/plugins/experimental/url_sig/url_sig.c index 6aa1fe0afca..7857400b268 100644 --- a/plugins/experimental/url_sig/url_sig.c +++ b/plugins/experimental/url_sig/url_sig.c @@ -23,9 +23,9 @@ _a < _b ? _a : _b; \ }) -#include "tscore/ink_defs.h" #include "url_sig.h" +#include #include #include #include @@ -98,7 +98,7 @@ TSRemapInit(TSRemapInterface *api_info, char *errbuf, int errbuf_size) return TS_ERROR; } - TSDebug(PLUGIN_NAME, "plugin is succesfully initialized"); + TSDebug(PLUGIN_NAME, "plugin is successfully initialized"); return TS_SUCCESS; } @@ -433,13 +433,13 @@ urlParse(char *url, char *anchor, char *new_path_seg, int new_path_seg_len, char if (strlen(sig_anchor) < signed_seg_len) { memcpy(signed_seg, sig_anchor, strlen(sig_anchor)); } else { - TSError("insuficient space to copy into new_path_seg buffer."); + TSError("insufficient space to copy into new_path_seg buffer."); } } else { // no signature anchor string was found, assum it is in the last path segment. if (strlen(segment[numtoks - 2]) < signed_seg_len) { memcpy(signed_seg, segment[numtoks - 2], strlen(segment[numtoks - 2])); } else { - TSError("insuficient space to copy into new_path_seg buffer."); + TSError("insufficient space to copy into new_path_seg buffer."); return NULL; } } @@ -538,7 +538,7 @@ TSRemapDoRemap(void *ih, TSHttpTxn txnp, TSRemapRequestInfo *rri) char sig_string[2 * MAX_SIG_SIZE + 1]; if (current_url_len >= MAX_REQ_LEN - 1) { - err_log(url, "URL string too long"); + err_log(current_url, "Request Url string too long"); goto deny; } @@ -555,15 +555,12 @@ TSRemapDoRemap(void *ih, TSHttpTxn txnp, TSRemapRequestInfo *rri) err_log(url, "Pristine URL string too long."); goto deny; } - } else { url_len = current_url_len; } TSDebug(PLUGIN_NAME, "%s", url); - const char *query = strchr(url, '?'); - if (cfg->regex) { const int offset = 0, options = 0; int ovector[30]; @@ -580,12 +577,17 @@ TSRemapDoRemap(void *ih, TSHttpTxn txnp, TSRemapRequestInfo *rri) } } + const char *query = strchr(url, '?'); + // check for path params. if (query == NULL || strstr(query, "E=") == NULL) { - if ((url = urlParse(url, cfg->sig_anchor, new_path, 8192, path_params, 8192)) == NULL) { + char *const parsed = urlParse(url, cfg->sig_anchor, new_path, 8192, path_params, 8192); + if (NULL == parsed) { err_log(url, "Has no signing query string or signing path parameters."); goto deny; } + + url = parsed; has_path_params = true; query = strstr(url, ";"); diff --git a/plugins/experimental/webp_transform/ImageTransform.cc b/plugins/experimental/webp_transform/ImageTransform.cc index 68ea938a575..1f29a88b8b6 100644 --- a/plugins/experimental/webp_transform/ImageTransform.cc +++ b/plugins/experimental/webp_transform/ImageTransform.cc @@ -92,7 +92,12 @@ class GlobalHookPlugin : public GlobalPlugin { string ctype = transaction.getServerResponse().getHeaders().values("Content-Type"); string user_agent = transaction.getServerRequest().getHeaders().values("User-Agent"); - if (user_agent.find("Chrome") != string::npos && (ctype.find("jpeg") != string::npos || ctype.find("png") != string::npos)) { + string accept = transaction.getServerRequest().getHeaders().values("Accept"); + + bool webp_supported = accept.find("image/webp") != string::npos || user_agent.find("Chrome") != string::npos; + bool image_format = ctype.find("jpeg") != string::npos || ctype.find("png") != string::npos; + + if (webp_supported && image_format) { TS_DEBUG(TAG, "Content type is either jpeg or png. Converting to webp"); transaction.addPlugin(new ImageTransform(transaction)); } diff --git a/plugins/experimental/webp_transform/Makefile.inc b/plugins/experimental/webp_transform/Makefile.inc index 15ed0446259..fd24dabcaa4 100644 --- a/plugins/experimental/webp_transform/Makefile.inc +++ b/plugins/experimental/webp_transform/Makefile.inc @@ -16,7 +16,7 @@ # limitations under the License. experimental_webp_transform_WebpTransform_la_CPPFLAGS = $(AM_CPPFLAGS) $(LIBMAGICKCPP_CFLAGS) -experimental_webp_transform_WebpTransform_la_CXXFLAGS = $(AM_CXXFLAGS) -Wno-unused-variable +experimental_webp_transform_WebpTransform_la_CXXFLAGS = $(AM_CXXFLAGS) pkglib_LTLIBRARIES += experimental/webp_transform/WebpTransform.la diff --git a/plugins/generator/generator.cc b/plugins/generator/generator.cc index 276313a75a2..f4bb95de7fd 100644 --- a/plugins/generator/generator.cc +++ b/plugins/generator/generator.cc @@ -109,11 +109,11 @@ lengthof(const char (&)[N]) // for each TSVConn; one to push data into the TSVConn and one to pull // data out. struct IOChannel { - TSVIO vio; + TSVIO vio = nullptr; TSIOBuffer iobuf; TSIOBufferReader reader; - IOChannel() : vio(nullptr), iobuf(TSIOBufferSizedCreate(TS_IOBUFFER_SIZE_INDEX_32K)), reader(TSIOBufferReaderAlloc(iobuf)) {} + IOChannel() : iobuf(TSIOBufferSizedCreate(TS_IOBUFFER_SIZE_INDEX_32K)), reader(TSIOBufferReaderAlloc(iobuf)) {} ~IOChannel() { if (this->reader) { @@ -163,10 +163,10 @@ struct GeneratorHttpHeader { }; struct GeneratorRequest { - off_t nbytes; // Number of bytes to generate. - unsigned flags; - unsigned delay; // Milliseconds to delay before sending a response. - unsigned maxage; // Max age for cache responses. + off_t nbytes = 0; // Number of bytes to generate. + unsigned flags = 0; + unsigned delay = 0; // Milliseconds to delay before sending a response. + unsigned maxage; // Max age for cache responses. IOChannel readio; IOChannel writeio; GeneratorHttpHeader rqheader; @@ -176,11 +176,11 @@ struct GeneratorRequest { ISHEAD = 0x0002, }; - GeneratorRequest() : nbytes(0), flags(0), delay(0), maxage(60 * 60 * 24) {} + GeneratorRequest() : maxage(60 * 60 * 24) {} ~GeneratorRequest() {} }; -// Destroy a generator request, iincluding the per-txn continuation. +// Destroy a generator request, including the per-txn continuation. static void GeneratorRequestDestroy(GeneratorRequest *grq, TSVIO vio, TSCont contp) { @@ -500,7 +500,7 @@ GeneratorInterceptionHook(TSCont contp, TSEvent event, void *edata) if (cdata.grq->delay > 0) { VDEBUG("delaying response by %ums", cdata.grq->delay); - TSContSchedule(contp, cdata.grq->delay, TS_THREAD_POOL_NET); + TSContScheduleOnPool(contp, cdata.grq->delay, TS_THREAD_POOL_NET); return TS_EVENT_NONE; } @@ -609,7 +609,7 @@ GeneratorTxnHook(TSCont contp, TSEvent event, void *edata) TSReleaseAssert(TSHttpTxnCacheLookupStatusGet(arg.txn, &status) == TS_SUCCESS); if (status != TS_CACHE_LOOKUP_HIT_FRESH) { // This transaction is going to be a cache miss, so intercept it. - VDEBUG("intercepting orgin server request for txn=%p", arg.txn); + VDEBUG("intercepting origin server request for txn=%p", arg.txn); TSHttpTxnServerIntercept(TSContCreate(GeneratorInterceptionHook, TSMutexCreate()), arg.txn); } diff --git a/plugins/header_rewrite/Makefile.inc b/plugins/header_rewrite/Makefile.inc index c4b81d6af9b..0b5cfe8bb62 100644 --- a/plugins/header_rewrite/Makefile.inc +++ b/plugins/header_rewrite/Makefile.inc @@ -23,8 +23,6 @@ header_rewrite_header_rewrite_la_SOURCES = \ header_rewrite/condition.h \ header_rewrite/conditions.cc \ header_rewrite/conditions.h \ - header_rewrite/expander.cc \ - header_rewrite/expander.h \ header_rewrite/factory.cc \ header_rewrite/factory.h \ header_rewrite/header_rewrite.cc \ diff --git a/plugins/header_rewrite/README b/plugins/header_rewrite/README index 40ea174625b..62fb930b486 100644 --- a/plugins/header_rewrite/README +++ b/plugins/header_rewrite/README @@ -24,4 +24,4 @@ during the origin response header parsing, using READ_RESPONSE_HDR_HOOK. For more information on how to use this plugin, please see - http://docs.trafficserver.apache.org/en/latest/reference/plugins/header_rewrite.en.html + https://docs.trafficserver.apache.org/en/latest/admin-guide/plugins/header_rewrite.en.html diff --git a/plugins/header_rewrite/condition.h b/plugins/header_rewrite/condition.h index 33bd19f4b4f..88f9917dc7a 100644 --- a/plugins/header_rewrite/condition.h +++ b/plugins/header_rewrite/condition.h @@ -49,12 +49,16 @@ class Condition : public Statement public: Condition() { TSDebug(PLUGIN_NAME_DBG, "Calling CTOR for Condition"); } - virtual ~Condition() + ~Condition() override { TSDebug(PLUGIN_NAME_DBG, "Calling DTOR for Condition"); delete _matcher; } + // noncopyable + Condition(const Condition &) = delete; + void operator=(const Condition &) = delete; + // Inline this, it's critical for speed (and only used twice) bool do_eval(const Resources &res) @@ -128,7 +132,5 @@ class Condition : public Statement Matcher *_matcher = nullptr; private: - DISALLOW_COPY_AND_ASSIGN(Condition); - CondModifiers _mods = COND_NONE; }; diff --git a/plugins/header_rewrite/conditions.cc b/plugins/header_rewrite/conditions.cc index 1ee6ea1bdba..8bb0b7e0a97 100644 --- a/plugins/header_rewrite/conditions.cc +++ b/plugins/header_rewrite/conditions.cc @@ -30,7 +30,6 @@ #include "ts/ts.h" #include "conditions.h" -#include "expander.h" #include "lulu.h" // ConditionStatus @@ -841,57 +840,59 @@ ConditionGeo::get_geo_string(const sockaddr *addr) const const char *ret = nullptr; int v = 4; - switch (_geo_qual) { - // Country database - case GEO_QUAL_COUNTRY: - switch (addr->sa_family) { - case AF_INET: - if (gGeoIP[GEOIP_COUNTRY_EDITION]) { - uint32_t ip = ntohl(reinterpret_cast(addr)->sin_addr.s_addr); - - ret = GeoIP_country_code_by_ipnum(gGeoIP[GEOIP_COUNTRY_EDITION], ip); + if (addr) { + switch (_geo_qual) { + // Country database + case GEO_QUAL_COUNTRY: + switch (addr->sa_family) { + case AF_INET: + if (gGeoIP[GEOIP_COUNTRY_EDITION]) { + uint32_t ip = ntohl(reinterpret_cast(addr)->sin_addr.s_addr); + + ret = GeoIP_country_code_by_ipnum(gGeoIP[GEOIP_COUNTRY_EDITION], ip); + } + break; + case AF_INET6: { + if (gGeoIP[GEOIP_COUNTRY_EDITION_V6]) { + geoipv6_t ip = reinterpret_cast(addr)->sin6_addr; + + v = 6; + ret = GeoIP_country_code_by_ipnum_v6(gGeoIP[GEOIP_COUNTRY_EDITION_V6], ip); + } + } break; + default: + break; } + TSDebug(PLUGIN_NAME, "eval(): Client IPv%d seems to come from Country: %s", v, ret); break; - case AF_INET6: { - if (gGeoIP[GEOIP_COUNTRY_EDITION_V6]) { - geoipv6_t ip = reinterpret_cast(addr)->sin6_addr; - v = 6; - ret = GeoIP_country_code_by_ipnum_v6(gGeoIP[GEOIP_COUNTRY_EDITION_V6], ip); + // ASN database + case GEO_QUAL_ASN_NAME: + switch (addr->sa_family) { + case AF_INET: + if (gGeoIP[GEOIP_ASNUM_EDITION]) { + uint32_t ip = ntohl(reinterpret_cast(addr)->sin_addr.s_addr); + + ret = GeoIP_name_by_ipnum(gGeoIP[GEOIP_ASNUM_EDITION], ip); + } + break; + case AF_INET6: { + if (gGeoIP[GEOIP_ASNUM_EDITION_V6]) { + geoipv6_t ip = reinterpret_cast(addr)->sin6_addr; + + v = 6; + ret = GeoIP_name_by_ipnum_v6(gGeoIP[GEOIP_ASNUM_EDITION_V6], ip); + } + } break; + default: + break; } - } break; - default: + TSDebug(PLUGIN_NAME, "eval(): Client IPv%d seems to come from ASN Name: %s", v, ret); break; - } - TSDebug(PLUGIN_NAME, "eval(): Client IPv%d seems to come from Country: %s", v, ret); - break; - // ASN database - case GEO_QUAL_ASN_NAME: - switch (addr->sa_family) { - case AF_INET: - if (gGeoIP[GEOIP_ASNUM_EDITION]) { - uint32_t ip = ntohl(reinterpret_cast(addr)->sin_addr.s_addr); - - ret = GeoIP_name_by_ipnum(gGeoIP[GEOIP_ASNUM_EDITION], ip); - } - break; - case AF_INET6: { - if (gGeoIP[GEOIP_ASNUM_EDITION_V6]) { - geoipv6_t ip = reinterpret_cast(addr)->sin6_addr; - - v = 6; - ret = GeoIP_name_by_ipnum_v6(gGeoIP[GEOIP_ASNUM_EDITION_V6], ip); - } - } break; default: break; } - TSDebug(PLUGIN_NAME, "eval(): Client IPv%d seems to come from ASN Name: %s", v, ret); - break; - - default: - break; } return ret ? ret : "(unknown)"; @@ -903,8 +904,12 @@ ConditionGeo::get_geo_int(const sockaddr *addr) const int64_t ret = -1; int v = 4; + if (!addr) { + return 0; + } + switch (_geo_qual) { - // Country Databse + // Country Database case GEO_QUAL_COUNTRY_ISO: switch (addr->sa_family) { case AF_INET: @@ -969,7 +974,7 @@ ConditionGeo::get_geo_int(const sockaddr *addr) const #else -// No Geo library avaiable, these are just stubs. +// No Geo library available, these are just stubs. const char * ConditionGeo::get_geo_string(const sockaddr *addr) const @@ -1213,32 +1218,36 @@ ConditionCidr::append_value(std::string &s, const Resources &res) { struct sockaddr const *addr = TSHttpTxnClientAddrGet(res.txnp); - switch (addr->sa_family) { - case AF_INET: { - char res[INET_ADDRSTRLEN]; - struct in_addr ipv4 = reinterpret_cast(addr)->sin_addr; - - ipv4.s_addr &= _v4_mask.s_addr; - inet_ntop(AF_INET, &ipv4, res, INET_ADDRSTRLEN); - if (res[0]) { - s += res; - } - } break; - case AF_INET6: { - char res[INET6_ADDRSTRLEN]; - struct in6_addr ipv6 = reinterpret_cast(addr)->sin6_addr; + if (addr) { + switch (addr->sa_family) { + case AF_INET: { + char res[INET_ADDRSTRLEN]; + struct in_addr ipv4 = reinterpret_cast(addr)->sin_addr; + + ipv4.s_addr &= _v4_mask.s_addr; + inet_ntop(AF_INET, &ipv4, res, INET_ADDRSTRLEN); + if (res[0]) { + s += res; + } + } break; + case AF_INET6: { + char res[INET6_ADDRSTRLEN]; + struct in6_addr ipv6 = reinterpret_cast(addr)->sin6_addr; - if (_v6_zero_bytes > 0) { - memset(&ipv6.s6_addr[16 - _v6_zero_bytes], 0, _v6_zero_bytes); - } - if (_v6_mask != 0xff) { - ipv6.s6_addr[16 - _v6_zero_bytes] &= _v6_mask; - } - inet_ntop(AF_INET6, &ipv6, res, INET6_ADDRSTRLEN); - if (res[0]) { - s += res; + if (_v6_zero_bytes > 0) { + memset(&ipv6.s6_addr[16 - _v6_zero_bytes], 0, _v6_zero_bytes); + } + if (_v6_mask != 0xff) { + ipv6.s6_addr[16 - _v6_zero_bytes] &= _v6_mask; + } + inet_ntop(AF_INET6, &ipv6, res, INET6_ADDRSTRLEN); + if (res[0]) { + s += res; + } + } break; } - } break; + } else { + s += "0.0.0.0"; // No client addr for some reason ... } } @@ -1394,27 +1403,3 @@ ConditionStringLiteral::eval(const Resources &res) return static_cast(_matcher)->test(_literal); } - -ConditionExpandableString::ConditionExpandableString(const std::string &v) -{ - TSDebug(PLUGIN_NAME_DBG, "Calling CTOR for ConditionExpandableString"); - _value = v; -} - -bool -ConditionExpandableString::eval(const Resources &res) -{ - std::string s; - - append_value(s, res); - - return static_cast(_matcher)->test(s); -} - -void -ConditionExpandableString::append_value(std::string &s, const Resources &res) -{ - VariableExpander ve(_value); - s += ve.expand(res); - TSDebug(PLUGIN_NAME, "Appending to evaluation value -> %s", s.c_str()); -} diff --git a/plugins/header_rewrite/conditions.h b/plugins/header_rewrite/conditions.h index 791bb41e132..77ef5952b4a 100644 --- a/plugins/header_rewrite/conditions.h +++ b/plugins/header_rewrite/conditions.h @@ -42,6 +42,11 @@ class ConditionTrue : public Condition { public: ConditionTrue() { TSDebug(PLUGIN_NAME_DBG, "Calling CTOR for ConditionTrue"); } + + // noncopyable + ConditionTrue(const ConditionTrue &) = delete; + void operator=(const ConditionTrue &) = delete; + void append_value(std::string &s, const Resources & /* res ATS_UNUSED */) override { @@ -55,9 +60,6 @@ class ConditionTrue : public Condition TSDebug(PLUGIN_NAME, "Evaluating TRUE()"); return true; } - -private: - DISALLOW_COPY_AND_ASSIGN(ConditionTrue); }; // Always false @@ -65,6 +67,11 @@ class ConditionFalse : public Condition { public: ConditionFalse() { TSDebug(PLUGIN_NAME_DBG, "Calling CTOR for ConditionFalse"); } + + // noncopyable + ConditionFalse(const ConditionFalse &) = delete; + void operator=(const ConditionFalse &) = delete; + void append_value(std::string &s, const Resources & /* res ATS_UNUSED */) override { @@ -78,9 +85,6 @@ class ConditionFalse : public Condition TSDebug(PLUGIN_NAME, "Evaluating FALSE()"); return false; } - -private: - DISALLOW_COPY_AND_ASSIGN(ConditionFalse); }; // Check the HTTP return status @@ -90,15 +94,17 @@ class ConditionStatus : public Condition public: ConditionStatus() { TSDebug(PLUGIN_NAME_DBG, "Calling CTOR for ConditionStatus"); } + + // noncopyable + ConditionStatus(const ConditionStatus &) = delete; + void operator=(const ConditionStatus &) = delete; + void initialize(Parser &p) override; void append_value(std::string &s, const Resources &res) override; protected: bool eval(const Resources &res) override; void initialize_hooks() override; // Return status only valid in certain hooks - -private: - DISALLOW_COPY_AND_ASSIGN(ConditionStatus); }; // Check the HTTP method @@ -108,14 +114,16 @@ class ConditionMethod : public Condition public: ConditionMethod() { TSDebug(PLUGIN_NAME_DBG, "Calling CTOR for ConditionMethod"); } + + // noncopyable + ConditionMethod(const ConditionMethod &) = delete; + void operator=(const ConditionMethod &) = delete; + void initialize(Parser &p) override; void append_value(std::string &s, const Resources &res) override; protected: bool eval(const Resources &res) override; - -private: - DISALLOW_COPY_AND_ASSIGN(ConditionMethod); }; // Random 0 to (N-1) @@ -125,6 +133,11 @@ class ConditionRandom : public Condition public: ConditionRandom() { TSDebug(PLUGIN_NAME_DBG, "Calling CTOR for ConditionRandom"); } + + // noncopyable + ConditionRandom(const ConditionRandom &) = delete; + void operator=(const ConditionRandom &) = delete; + void initialize(Parser &p) override; void append_value(std::string &s, const Resources &res) override; @@ -132,8 +145,6 @@ class ConditionRandom : public Condition bool eval(const Resources &res) override; private: - DISALLOW_COPY_AND_ASSIGN(ConditionRandom); - unsigned int _seed = 0; unsigned int _max = 0; }; @@ -143,6 +154,11 @@ class ConditionAccess : public Condition { public: ConditionAccess() { TSDebug(PLUGIN_NAME_DBG, "Calling CTOR for ConditionAccess"); } + + // noncopyable + ConditionAccess(const ConditionAccess &) = delete; + void operator=(const ConditionAccess &) = delete; + void initialize(Parser &p) override; void append_value(std::string &s, const Resources &res) override; @@ -150,8 +166,6 @@ class ConditionAccess : public Condition bool eval(const Resources &res) override; private: - DISALLOW_COPY_AND_ASSIGN(ConditionAccess); - time_t _next = 0; bool _last = false; }; @@ -163,6 +177,11 @@ class ConditionCookie : public Condition public: ConditionCookie() { TSDebug(PLUGIN_NAME_DBG, "Calling CTOR for ConditionCookie"); } + + // noncopyable + ConditionCookie(const ConditionCookie &) = delete; + void operator=(const ConditionCookie &) = delete; + void initialize(Parser &p) override; void append_value(std::string &s, const Resources &res) override; @@ -170,8 +189,6 @@ class ConditionCookie : public Condition bool eval(const Resources &res) override; private: - DISALLOW_COPY_AND_ASSIGN(ConditionCookie); - // Nginx-style cookie parsing: // nginx/src/http/ngx_http_parse.c:ngx_http_parse_multi_header_lines() inline int @@ -235,6 +252,10 @@ class ConditionHeader : public Condition TSDebug(PLUGIN_NAME_DBG, "Calling CTOR for ConditionHeader, client %d", client); } + // noncopyable + ConditionHeader(const ConditionHeader &) = delete; + void operator=(const ConditionHeader &) = delete; + void initialize(Parser &p) override; void append_value(std::string &s, const Resources &res) override; @@ -242,8 +263,6 @@ class ConditionHeader : public Condition bool eval(const Resources &res) override; private: - DISALLOW_COPY_AND_ASSIGN(ConditionHeader); - bool _client; }; @@ -254,14 +273,16 @@ class ConditionPath : public Condition public: explicit ConditionPath() { TSDebug(PLUGIN_NAME_DBG, "Calling CTOR for ConditionPath"); } + + // noncopyable + ConditionPath(const ConditionPath &) = delete; + void operator=(const ConditionPath &) = delete; + void initialize(Parser &p) override; void append_value(std::string &s, const Resources &res) override; protected: bool eval(const Resources &res) override; - -private: - DISALLOW_COPY_AND_ASSIGN(ConditionPath); }; // query @@ -271,14 +292,16 @@ class ConditionQuery : public Condition public: explicit ConditionQuery() { TSDebug(PLUGIN_NAME_DBG, "Calling CTOR for ConditionQuery"); } + + // noncopyable + ConditionQuery(const ConditionQuery &) = delete; + void operator=(const ConditionQuery &) = delete; + void initialize(Parser &p) override; void append_value(std::string &s, const Resources &res) override; protected: bool eval(const Resources &res) override; - -private: - DISALLOW_COPY_AND_ASSIGN(ConditionQuery); }; // url @@ -291,6 +314,10 @@ class ConditionUrl : public Condition explicit ConditionUrl(const UrlType type) : _type(type) { TSDebug(PLUGIN_NAME_DBG, "Calling CTOR for ConditionUrl"); } + // noncopyable + ConditionUrl(const ConditionUrl &) = delete; + void operator=(const ConditionUrl &) = delete; + void initialize(Parser &p) override; void set_qualifier(const std::string &q) override; void append_value(std::string &s, const Resources &res) override; @@ -299,8 +326,6 @@ class ConditionUrl : public Condition bool eval(const Resources &res) override; private: - DISALLOW_COPY_AND_ASSIGN(ConditionUrl); - UrlQualifiers _url_qual = URL_QUAL_NONE; UrlType _type; }; @@ -327,6 +352,10 @@ class ConditionDBM : public Condition // } } + // noncopyable + ConditionDBM(const ConditionDBM &) = delete; + void operator=(const ConditionDBM &) = delete; + void initialize(Parser &p) override; void append_value(std::string &s, const Resources &res) override; @@ -334,7 +363,6 @@ class ConditionDBM : public Condition bool eval(const Resources &res) override; private: - DISALLOW_COPY_AND_ASSIGN(ConditionDBM); // MDBM* _dbm; std::string _file; Value _key; @@ -361,6 +389,11 @@ class ConditionIp : public Condition public: explicit ConditionIp() { TSDebug(PLUGIN_NAME_DBG, "Calling CTOR for ConditionIp"); }; + + // noncopyable + ConditionIp(const ConditionIp &) = delete; + void operator=(const ConditionIp &) = delete; + void initialize(Parser &p) override; void set_qualifier(const std::string &q) override; void append_value(std::string &s, const Resources &res) override; @@ -369,7 +402,6 @@ class ConditionIp : public Condition bool eval(const Resources &res) override; private: - DISALLOW_COPY_AND_ASSIGN(ConditionIp); IpQualifiers _ip_qual = IP_QUAL_CLIENT; }; @@ -379,14 +411,16 @@ class ConditionIncomingPort : public Condition public: ConditionIncomingPort() { TSDebug(PLUGIN_NAME_DBG, "Calling CTOR for ConditionIncomingPort"); } + + // noncopyable + ConditionIncomingPort(const ConditionIncomingPort &) = delete; + void operator=(const ConditionIncomingPort &) = delete; + void initialize(Parser &p) override; void append_value(std::string &s, const Resources &res) override; protected: bool eval(const Resources &res) override; - -private: - DISALLOW_COPY_AND_ASSIGN(ConditionIncomingPort); }; // Transact Count @@ -396,14 +430,16 @@ class ConditionTransactCount : public Condition public: ConditionTransactCount() { TSDebug(PLUGIN_NAME_DBG, "Calling CTOR for ConditionTransactCount"); } + + // noncopyable + ConditionTransactCount(const ConditionTransactCount &) = delete; + void operator=(const ConditionTransactCount &) = delete; + void initialize(Parser &p) override; void append_value(std::string &s, const Resources &res) override; protected: bool eval(const Resources &res) override; - -private: - DISALLOW_COPY_AND_ASSIGN(ConditionTransactCount); }; // now: Keeping track of current time / day / hour etc. @@ -413,6 +449,11 @@ class ConditionNow : public Condition public: explicit ConditionNow() { TSDebug(PLUGIN_NAME_DBG, "Calling CTOR for ConditionNow"); } + + // noncopyable + ConditionNow(const ConditionNow &) = delete; + void operator=(const ConditionNow &) = delete; + void initialize(Parser &p) override; void set_qualifier(const std::string &q) override; void append_value(std::string &s, const Resources &res) override; @@ -421,8 +462,6 @@ class ConditionNow : public Condition bool eval(const Resources &res) override; private: - DISALLOW_COPY_AND_ASSIGN(ConditionNow); - int64_t get_now_qualified(NowQualifiers qual) const; NowQualifiers _now_qual = NOW_QUAL_EPOCH; }; @@ -433,6 +472,10 @@ class ConditionGeo : public Condition public: explicit ConditionGeo() { TSDebug(PLUGIN_NAME_DBG, "Calling CTOR for ConditionGeo"); } + // noncopyable + ConditionGeo(const ConditionGeo &) = delete; + void operator=(const ConditionGeo &) = delete; + void initialize(Parser &p) override; void set_qualifier(const std::string &q) override; void append_value(std::string &s, const Resources &res) override; @@ -454,8 +497,6 @@ class ConditionGeo : public Condition bool eval(const Resources &res) override; private: - DISALLOW_COPY_AND_ASSIGN(ConditionGeo); - int64_t get_geo_int(const sockaddr *addr) const; const char *get_geo_string(const sockaddr *addr) const; GeoQualifiers _geo_qual = GEO_QUAL_COUNTRY; @@ -467,6 +508,11 @@ class ConditionId : public Condition { public: explicit ConditionId() { TSDebug(PLUGIN_NAME_DBG, "Calling CTOR for ConditionId"); }; + + // noncopyable + ConditionId(const ConditionId &) = delete; + void operator=(const ConditionId &) = delete; + void initialize(Parser &p) override; void set_qualifier(const std::string &q) override; void append_value(std::string &s, const Resources &res) override; @@ -475,7 +521,6 @@ class ConditionId : public Condition bool eval(const Resources &res) override; private: - DISALLOW_COPY_AND_ASSIGN(ConditionId); IdQualifiers _id_qual = ID_QUAL_UNIQUE; }; @@ -543,6 +588,10 @@ class ConditionStringLiteral : public Condition public: explicit ConditionStringLiteral(const std::string &v); + // noncopyable + ConditionStringLiteral(const ConditionStringLiteral &) = delete; + void operator=(const ConditionStringLiteral &) = delete; + void append_value(std::string &s, const Resources & /* res ATS_UNUSED */) override; protected: @@ -550,22 +599,4 @@ class ConditionStringLiteral : public Condition private: std::string _literal; - DISALLOW_COPY_AND_ASSIGN(ConditionStringLiteral); -}; - -class ConditionExpandableString : public Condition -{ - typedef Matchers MatcherType; - -public: - explicit ConditionExpandableString(const std::string &v); - - void append_value(std::string &s, const Resources &res) override; - -protected: - bool eval(const Resources &res) override; - -private: - std::string _value; - DISALLOW_COPY_AND_ASSIGN(ConditionExpandableString); }; diff --git a/plugins/header_rewrite/expander.cc b/plugins/header_rewrite/expander.cc deleted file mode 100644 index dd3d1ab397c..00000000000 --- a/plugins/header_rewrite/expander.cc +++ /dev/null @@ -1,155 +0,0 @@ -/* - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -////////////////////////////////////////////////////////////////////////////////////////////// -// expander.cc: Implementation of the Variable Expander base class -// -#include "ts/ts.h" - -#include -#include - -#include "lulu.h" -#include "statement.h" -#include "parser.h" -#include "expander.h" -#include "conditions.h" - -// Main expander method -std::string -VariableExpander::expand(const Resources &res) -{ - std::string result; - - result.reserve(512); // TODO: Can be optimized - result.assign(_source); - - while (true) { - std::string::size_type start = result.find("%<"); - if (start == std::string::npos) { - break; - } - - std::string::size_type end = result.find('>', start); - if (end == std::string::npos) { - break; - } - - std::string first_part = result.substr(0, start); - std::string last_part = result.substr(end + 1); - - // Now evaluate the variable - std::string variable = result.substr(start, end - start + 1); - - // This will be the value to replace the "variable" section of the string with - std::string resolved_variable = ""; - - // Initialize some stuff - TSMBuffer bufp; - TSMLoc hdr_loc; - TSMLoc url_loc; - - if (variable == "%") { - // Protocol of the incoming request - if (TSHttpTxnPristineUrlGet(res.txnp, &bufp, &url_loc) == TS_SUCCESS) { - int len; - const char *tmp = TSUrlSchemeGet(bufp, url_loc, &len); - if ((tmp != nullptr) && (len > 0)) { - resolved_variable.assign(tmp, len); - } else { - resolved_variable.assign(""); - } - TSHandleMLocRelease(bufp, TS_NULL_MLOC, url_loc); - } - } else if (variable == "%") { - // Original port of the incoming request - if (TSHttpTxnClientReqGet(res.txnp, &bufp, &hdr_loc) == TS_SUCCESS) { - if (TSHttpHdrUrlGet(bufp, hdr_loc, &url_loc) == TS_SUCCESS) { - std::stringstream out; - out << TSUrlPortGet(bufp, url_loc); - resolved_variable = out.str(); - TSHandleMLocRelease(bufp, hdr_loc, url_loc); - } - TSHandleMLocRelease(bufp, TS_NULL_MLOC, hdr_loc); - } - } else if (variable == "%") { - // IP address of the client's host machine - resolved_variable = getIP(TSHttpTxnClientAddrGet(res.txnp)); - } else if (variable == "%") { - // The client request header length; the header length in the client request to Traffic Server. - std::stringstream out; - out << TSHttpHdrLengthGet(res.client_bufp, res.client_hdr_loc); - resolved_variable = out.str(); - } else if (variable == "%") { - // The HTTP method in the client request to Traffic Server: GET, POST, and so on (subset of cqtx). - int method_len; - const char *methodp = TSHttpHdrMethodGet(res.client_bufp, res.client_hdr_loc, &method_len); - if (methodp && method_len) { - resolved_variable.assign(methodp, method_len); - } - } else if (variable == "%") { - // The client request unmapped URL path. This field records a URL path - // before it is remapped (reverse proxy mode). - if (TSHttpTxnPristineUrlGet(res.txnp, &bufp, &url_loc) == TS_SUCCESS) { - int path_len; - const char *path = TSUrlPathGet(bufp, url_loc, &path_len); - - if (path && path_len) { - resolved_variable.assign(path, path_len); - } - TSHandleMLocRelease(bufp, TS_NULL_MLOC, url_loc); - } - } else if (variable == "%") { - // The client request effective URL. - int url_len = 0; - char *url = TSHttpTxnEffectiveUrlStringGet(res.txnp, &url_len); - if (url && url_len) { - resolved_variable.assign(url, url_len); - } - free(url); - url = nullptr; - } else if (variable == "%") { - ConditionInbound::append_value(resolved_variable, res, NET_QUAL_REMOTE_ADDR); - } else if (variable == "%") { - ConditionInbound::append_value(resolved_variable, res, NET_QUAL_REMOTE_PORT); - } else if (variable == "%") { - ConditionInbound::append_value(resolved_variable, res, NET_QUAL_LOCAL_ADDR); - } else if (variable == "%") { - ConditionInbound::append_value(resolved_variable, res, NET_QUAL_LOCAL_PORT); - } else if (variable == "%") { - ConditionInbound::append_value(resolved_variable, res, NET_QUAL_TLS); - } else if (variable == "%") { - ConditionInbound::append_value(resolved_variable, res, NET_QUAL_H2); - } else if (variable == "%") { - ConditionInbound::append_value(resolved_variable, res, NET_QUAL_IPV4); - } else if (variable == "%") { - ConditionInbound::append_value(resolved_variable, res, NET_QUAL_IPV6); - } else if (variable == "%") { - ConditionInbound::append_value(resolved_variable, res, NET_QUAL_IP_FAMILY); - } else if (variable == "%") { - ConditionInbound::append_value(resolved_variable, res, NET_QUAL_STACK); - } - - // TODO(SaveTheRbtz): Can be optimized - result.assign(first_part); - result.append(resolved_variable); - result.append(last_part); - } - - return result; -} diff --git a/plugins/header_rewrite/factory.cc b/plugins/header_rewrite/factory.cc index 41e29fe3cab..028fc6289ce 100644 --- a/plugins/header_rewrite/factory.cc +++ b/plugins/header_rewrite/factory.cc @@ -144,7 +144,7 @@ condition_factory(const std::string &cond) } else if (c_name == "INBOUND") { c = new ConditionInbound(); } else { - TSDebug(PLUGIN_NAME, "Unknown condition: %s", c_name.c_str()); + TSError("[%s] Unknown condition %s", PLUGIN_NAME, c_name.c_str()); return nullptr; } diff --git a/plugins/header_rewrite/header_rewrite.cc b/plugins/header_rewrite/header_rewrite.cc index af19305e48f..a385d913b3a 100644 --- a/plugins/header_rewrite/header_rewrite.cc +++ b/plugins/header_rewrite/header_rewrite.cc @@ -42,13 +42,13 @@ initGeoIP() { GeoIPDBTypes dbs[] = {GEOIP_COUNTRY_EDITION, GEOIP_COUNTRY_EDITION_V6, GEOIP_ASNUM_EDITION, GEOIP_ASNUM_EDITION_V6}; - for (unsigned i = 0; i < sizeof(dbs) / sizeof(dbs[0]); ++i) { - if (!gGeoIP[dbs[i]] && GeoIP_db_avail(dbs[i])) { + for (auto &db : dbs) { + if (!gGeoIP[db] && GeoIP_db_avail(db)) { // GEOIP_STANDARD seems to break threaded apps... - gGeoIP[dbs[i]] = GeoIP_open_type(dbs[i], GEOIP_MMAP_CACHE); + gGeoIP[db] = GeoIP_open_type(db, GEOIP_MMAP_CACHE); - char *db_info = GeoIP_database_info(gGeoIP[dbs[i]]); - TSDebug(PLUGIN_NAME, "initialized GeoIP-DB[%d] %s", dbs[i], db_info); + char *db_info = GeoIP_database_info(gGeoIP[db]); + TSDebug(PLUGIN_NAME, "initialized GeoIP-DB[%d] %s", db, db_info); free(db_info); } } @@ -184,7 +184,7 @@ RulesConfig::parse_config(const std::string &fname, TSHttpHookID default_hook) continue; } - Parser p(line, true); // Tokenize and parse this line preserving quotes from input + Parser p(line); // Tokenize and parse this line if (p.empty()) { continue; } @@ -330,7 +330,7 @@ TSPluginInit(int argc, const char *argv[]) // just appended to the configurations. TSDebug(PLUGIN_NAME, "Loading global configuration file %s", argv[i]); if (conf->parse_config(argv[i], TS_HTTP_READ_RESPONSE_HDR_HOOK)) { - TSDebug(PLUGIN_NAME, "Succesfully loaded global config file %s", argv[i]); + TSDebug(PLUGIN_NAME, "Successfully loaded global config file %s", argv[i]); got_config = true; } else { TSError("[header_rewrite] failed to parse configuration file %s", argv[i]); @@ -401,7 +401,7 @@ TSRemapNewInstance(int argc, char *argv[], void **ih, char * /* errbuf ATS_UNUSE delete conf; return TS_ERROR; } else { - TSDebug(PLUGIN_NAME, "Succesfully loaded remap config file %s", argv[i]); + TSDebug(PLUGIN_NAME, "Successfully loaded remap config file %s", argv[i]); } } @@ -470,6 +470,6 @@ TSRemapDoRemap(void *ih, TSHttpTxn rh, TSRemapRequestInfo *rri) rule = rule->next; } - TSDebug(PLUGIN_NAME_DBG, "Returing from TSRemapDoRemap with status: %d", rval); + TSDebug(PLUGIN_NAME_DBG, "Returning from TSRemapDoRemap with status: %d", rval); return rval; } diff --git a/plugins/header_rewrite/header_rewrite_test.cc b/plugins/header_rewrite/header_rewrite_test.cc index c81113a1e61..437b76c849d 100644 --- a/plugins/header_rewrite/header_rewrite_test.cc +++ b/plugins/header_rewrite/header_rewrite_test.cc @@ -33,6 +33,12 @@ const char PLUGIN_NAME_DBG[] = "TEST_dbg_header_rewrite"; extern "C" void TSError(const char *fmt, ...) { + va_list args; + + va_start(args, fmt); + vfprintf(stderr, fmt, args); + va_end(args); + fprintf(stderr, "\n"); } class ParserTest : public Parser @@ -40,7 +46,7 @@ class ParserTest : public Parser public: ParserTest(const std::string &line) : Parser(line), res(true) { std::cout << "Finished parser test: " << line << std::endl; } std::vector - getTokens() + getTokens() const { return _tokens; } @@ -58,17 +64,38 @@ class ParserTest : public Parser bool res; }; +class SimpleTokenizerTest : public SimpleTokenizer +{ +public: + SimpleTokenizerTest(const std::string &line) : SimpleTokenizer(line), res(true) + { + std::cout << "Finished tokenizer test: " << line << std::endl; + } + + template + void + do_parser_check(T x, U y, int line = 0) + { + if (x != y) { + std::cerr << "CHECK FAILED on line " << line << ": |" << x << "| != |" << y << "|" << std::endl; + res = false; + } + } + + bool res; +}; + #define CHECK_EQ(x, y) \ do { \ p.do_parser_check((x), (y), __LINE__); \ - } while (false); + } while (false) #define END_TEST(s) \ do { \ if (!p.res) { \ ++errors; \ } \ - } while (false); + } while (false) int test_parsing() @@ -304,18 +331,18 @@ test_parsing() } { - ParserTest p(R"(add-header Client-IP "%")"); + ParserTest p(R"(add-header X-Url "http://trafficserver.apache.org/")"); CHECK_EQ(p.getTokens().size(), 3UL); CHECK_EQ(p.getTokens()[0], "add-header"); - CHECK_EQ(p.getTokens()[1], "Client-IP"); - CHECK_EQ(p.getTokens()[2], R"(%)"); + CHECK_EQ(p.getTokens()[1], "X-Url"); + CHECK_EQ(p.getTokens()[2], "http://trafficserver.apache.org/"); END_TEST(); } { - ParserTest p(R"(add-header X-Url "http://trafficserver.apache.org/")"); + ParserTest p(R"(add-header X-Url http://trafficserver.apache.org/)"); CHECK_EQ(p.getTokens().size(), 3UL); CHECK_EQ(p.getTokens()[0], "add-header"); @@ -326,12 +353,12 @@ test_parsing() } { - ParserTest p(R"(add-header X-Url http://trafficserver.apache.org/)"); + ParserTest p(R"(set-header Alt-Svc "quic=\":443\"; v=\"35\"")"); CHECK_EQ(p.getTokens().size(), 3UL); - CHECK_EQ(p.getTokens()[0], "add-header"); - CHECK_EQ(p.getTokens()[1], "X-Url"); - CHECK_EQ(p.getTokens()[2], "http://trafficserver.apache.org/"); + CHECK_EQ(p.getTokens()[0], "set-header"); + CHECK_EQ(p.getTokens()[1], "Alt-Svc"); + CHECK_EQ(p.getTokens()[2], R"(quic=":443"; v="35")"); END_TEST(); } @@ -425,10 +452,55 @@ test_processing() return errors; } +int +test_tokenizer() +{ + int errors = 0; + + { + SimpleTokenizerTest p("a simple test"); + CHECK_EQ(p.get_tokens().size(), 1UL); + CHECK_EQ(p.get_tokens()[0], "a simple test"); + } + + { + SimpleTokenizerTest p(R"(quic=":443"; v="35")"); + CHECK_EQ(p.get_tokens().size(), 1UL); + CHECK_EQ(p.get_tokens()[0], R"(quic=":443"; v="35")"); + } + + { + SimpleTokenizerTest p(R"(let's party like it's %{NOW:YEAR})"); + CHECK_EQ(p.get_tokens().size(), 2UL); + CHECK_EQ(p.get_tokens()[0], "let's party like it's "); + CHECK_EQ(p.get_tokens()[1], "%{NOW:YEAR}"); + } + { + SimpleTokenizerTest p("A racoon's favorite tag is %{METHOD} in %{NOW:YEAR}!"); + CHECK_EQ(p.get_tokens().size(), 5UL); + CHECK_EQ(p.get_tokens()[0], "A racoon's favorite tag is "); + CHECK_EQ(p.get_tokens()[1], "%{METHOD}c"); + CHECK_EQ(p.get_tokens()[2], " in "); + CHECK_EQ(p.get_tokens()[3], "%{NOW:YEAR}"); + CHECK_EQ(p.get_tokens()[4], "!"); + } + + { + SimpleTokenizerTest p(R"(Hello from %{IP:SERVER}:%{INBOUND:LOCAL-PORT})"); + CHECK_EQ(p.get_tokens().size(), 4UL); + CHECK_EQ(p.get_tokens()[0], "Hello from "); + CHECK_EQ(p.get_tokens()[1], "%{IP:SERVER}"); + CHECK_EQ(p.get_tokens()[2], ":"); + CHECK_EQ(p.get_tokens()[3], "%{INBOUND:LOCAL-PORT}"); + } + + return errors; +} + int main() { - if (test_parsing() || test_processing()) { + if (test_parsing() || test_processing() || test_tokenizer()) { return 1; } diff --git a/plugins/header_rewrite/lulu.cc b/plugins/header_rewrite/lulu.cc index 78b70f0c7eb..b4eb9f6341c 100644 --- a/plugins/header_rewrite/lulu.cc +++ b/plugins/header_rewrite/lulu.cc @@ -71,6 +71,10 @@ getIP(sockaddr const *s_sockaddr) uint16_t getPort(sockaddr const *s_sockaddr) { + if (!s_sockaddr) { + return 0; + } + switch (s_sockaddr->sa_family) { case AF_INET: { const struct sockaddr_in *s_sockaddr_in = reinterpret_cast(s_sockaddr); diff --git a/plugins/header_rewrite/lulu.h b/plugins/header_rewrite/lulu.h index c0c20f06380..cadc9023742 100644 --- a/plugins/header_rewrite/lulu.h +++ b/plugins/header_rewrite/lulu.h @@ -35,8 +35,3 @@ uint16_t getPort(sockaddr const *s_sockaddr); extern const char PLUGIN_NAME[]; extern const char PLUGIN_NAME_DBG[]; - -// From google styleguide: http://google-styleguide.googlecode.com/svn/trunk/cppguide.xml -#define DISALLOW_COPY_AND_ASSIGN(TypeName) \ - TypeName(const TypeName &) = delete; \ - void operator=(const TypeName &) = delete diff --git a/plugins/header_rewrite/matcher.h b/plugins/header_rewrite/matcher.h index 3d8e71eece9..83bb459072c 100644 --- a/plugins/header_rewrite/matcher.h +++ b/plugins/header_rewrite/matcher.h @@ -46,11 +46,12 @@ class Matcher explicit Matcher(const MatcherOps op) : _op(op) { TSDebug(PLUGIN_NAME_DBG, "Calling CTOR for Matcher"); } virtual ~Matcher() { TSDebug(PLUGIN_NAME_DBG, "Calling DTOR for Matcher"); } + // noncopyable + Matcher(const Matcher &) = delete; + void operator=(const Matcher &) = delete; + protected: const MatcherOps _op; - -private: - DISALLOW_COPY_AND_ASSIGN(Matcher); }; // Template class to match on various types of data @@ -66,7 +67,7 @@ template class Matchers : public Matcher }; void - setRegex(const std::string /* data ATS_UNUSED */) + setRegex(const std::string & /* data ATS_UNUSED */) { if (!helper.setRegexMatch(_data)) { std::stringstream ss; diff --git a/plugins/header_rewrite/operator.h b/plugins/header_rewrite/operator.h index b11a1de4e08..04821fcd330 100644 --- a/plugins/header_rewrite/operator.h +++ b/plugins/header_rewrite/operator.h @@ -44,6 +44,11 @@ class Operator : public Statement { public: Operator() { TSDebug(PLUGIN_NAME_DBG, "Calling CTOR for Operator"); } + + // noncopyable + Operator(const Operator &) = delete; + void operator=(const Operator &) = delete; + OperModifiers get_oper_modifiers() const; void initialize(Parser &p) override; @@ -60,8 +65,6 @@ class Operator : public Statement virtual void exec(const Resources &res) const = 0; private: - DISALLOW_COPY_AND_ASSIGN(Operator); - OperModifiers _mods = OPER_NONE; }; @@ -73,13 +76,15 @@ class OperatorHeaders : public Operator { public: OperatorHeaders() { TSDebug(PLUGIN_NAME_DBG, "Calling CTOR for OperatorHeaders"); } + + // noncopyable + OperatorHeaders(const OperatorHeaders &) = delete; + void operator=(const OperatorHeaders &) = delete; + void initialize(Parser &p) override; protected: std::string _header; - -private: - DISALLOW_COPY_AND_ASSIGN(OperatorHeaders); }; /////////////////////////////////////////////////////////////////////////////// @@ -90,11 +95,13 @@ class OperatorCookies : public Operator { public: OperatorCookies() { TSDebug(PLUGIN_NAME_DBG, "Calling CTOR for OperatorCookies"); } + + // noncopyable + OperatorCookies(const OperatorCookies &) = delete; + void operator=(const OperatorCookies &) = delete; + void initialize(Parser &p) override; protected: std::string _cookie; - -private: - DISALLOW_COPY_AND_ASSIGN(OperatorCookies); }; diff --git a/plugins/header_rewrite/operators.cc b/plugins/header_rewrite/operators.cc index 3b57f7f2158..c2050116b40 100644 --- a/plugins/header_rewrite/operators.cc +++ b/plugins/header_rewrite/operators.cc @@ -25,7 +25,6 @@ #include "ts/ts.h" #include "operators.h" -#include "expander.h" #include "ts/apidefs.h" // OperatorConfig @@ -355,11 +354,6 @@ OperatorSetRedirect::exec(const Resources &res) const _location.append_value(value, res); - if (_location.need_expansion()) { - VariableExpander ve(value); - value = ve.expand(res); - } - bool remap = false; if (nullptr != res._rri) { remap = true; @@ -543,12 +537,6 @@ OperatorAddHeader::exec(const Resources &res) const _value.append_value(value, res); - if (_value.need_expansion()) { - VariableExpander ve(value); - - value = ve.expand(res); - } - // Never set an empty header (I don't think that ever makes sense?) if (value.empty()) { TSDebug(PLUGIN_NAME, "Would set header %s to an empty value, skipping", _header.c_str()); @@ -585,12 +573,6 @@ OperatorSetHeader::exec(const Resources &res) const _value.append_value(value, res); - if (_value.need_expansion()) { - VariableExpander ve(value); - - value = ve.expand(res); - } - // Never set an empty header (I don't think that ever makes sense?) if (value.empty()) { TSDebug(PLUGIN_NAME, "Would set header %s to an empty value, skipping", _header.c_str()); @@ -718,12 +700,6 @@ OperatorAddCookie::exec(const Resources &res) const _value.append_value(value, res); - if (_value.need_expansion()) { - VariableExpander ve(value); - - value = ve.expand(res); - } - if (res.bufp && res.hdr_loc) { TSDebug(PLUGIN_NAME, "OperatorAddCookie::exec() invoked on cookie %s", _cookie.c_str()); TSMLoc field_loc; @@ -769,12 +745,6 @@ OperatorSetCookie::exec(const Resources &res) const _value.append_value(value, res); - if (_value.need_expansion()) { - VariableExpander ve(value); - - value = ve.expand(res); - } - if (res.bufp && res.hdr_loc) { TSDebug(PLUGIN_NAME, "OperatorSetCookie::exec() invoked on cookie %s", _cookie.c_str()); TSMLoc field_loc; diff --git a/plugins/header_rewrite/operators.h b/plugins/header_rewrite/operators.h index 2ff0d847977..f1d24fab342 100644 --- a/plugins/header_rewrite/operators.h +++ b/plugins/header_rewrite/operators.h @@ -36,14 +36,17 @@ class OperatorSetConfig : public Operator { public: OperatorSetConfig() { TSDebug(PLUGIN_NAME_DBG, "Calling CTOR for OperatorSetConfig"); } + + // noncopyable + OperatorSetConfig(const OperatorSetConfig &) = delete; + void operator=(const OperatorSetConfig &) = delete; + void initialize(Parser &p) override; protected: void exec(const Resources &res) const override; private: - DISALLOW_COPY_AND_ASSIGN(OperatorSetConfig); - TSOverridableConfigKey _key = TS_CONFIG_NULL; TSRecordDataType _type = TS_RECORDDATATYPE_NULL; @@ -55,6 +58,11 @@ class OperatorSetStatus : public Operator { public: OperatorSetStatus() { TSDebug(PLUGIN_NAME_DBG, "Calling CTOR for OperatorSetStatus"); } + + // noncopyable + OperatorSetStatus(const OperatorSetStatus &) = delete; + void operator=(const OperatorSetStatus &) = delete; + void initialize(Parser &p) override; protected: @@ -62,8 +70,6 @@ class OperatorSetStatus : public Operator void exec(const Resources &res) const override; private: - DISALLOW_COPY_AND_ASSIGN(OperatorSetStatus); - Value _status; const char *_reason = nullptr; int _reason_len = 0; @@ -73,6 +79,11 @@ class OperatorSetStatusReason : public Operator { public: OperatorSetStatusReason() { TSDebug(PLUGIN_NAME_DBG, "Calling CTOR for OperatorSetStatusReason"); } + + // noncopyable + OperatorSetStatusReason(const OperatorSetStatusReason &) = delete; + void operator=(const OperatorSetStatusReason &) = delete; + void initialize(Parser &p) override; protected: @@ -80,8 +91,6 @@ class OperatorSetStatusReason : public Operator void exec(const Resources &res) const override; private: - DISALLOW_COPY_AND_ASSIGN(OperatorSetStatusReason); - Value _reason; }; @@ -89,14 +98,17 @@ class OperatorSetDestination : public Operator { public: OperatorSetDestination() { TSDebug(PLUGIN_NAME_DBG, "Calling CTOR for OperatorSetDestination"); } + + // noncopyable + OperatorSetDestination(const OperatorSetDestination &) = delete; + void operator=(const OperatorSetDestination &) = delete; + void initialize(Parser &p) override; protected: void exec(const Resources &res) const override; private: - DISALLOW_COPY_AND_ASSIGN(OperatorSetDestination); - UrlQualifiers _url_qual = URL_QUAL_NONE; Value _value; }; @@ -105,6 +117,11 @@ class OperatorSetRedirect : public Operator { public: OperatorSetRedirect() { TSDebug(PLUGIN_NAME_DBG, "Calling CTOR for OperatorSetRedirect"); } + + // noncopyable + OperatorSetRedirect(const OperatorSetRedirect &) = delete; + void operator=(const OperatorSetRedirect &) = delete; + void initialize(Parser &p) override; TSHttpStatus @@ -123,8 +140,6 @@ class OperatorSetRedirect : public Operator void exec(const Resources &res) const override; private: - DISALLOW_COPY_AND_ASSIGN(OperatorSetRedirect); - Value _status; Value _location; }; @@ -134,25 +149,29 @@ class OperatorNoOp : public Operator public: OperatorNoOp() { TSDebug(PLUGIN_NAME_DBG, "Calling CTOR for OperatorNoOp"); } + // noncopyable + OperatorNoOp(const OperatorNoOp &) = delete; + void operator=(const OperatorNoOp &) = delete; + protected: void exec(const Resources & /* res ATS_UNUSED */) const override{}; - -private: - DISALLOW_COPY_AND_ASSIGN(OperatorNoOp); }; class OperatorSetTimeoutOut : public Operator { public: OperatorSetTimeoutOut() { TSDebug(PLUGIN_NAME_DBG, "Calling CTOR for OperatorSetTimeoutOut"); } + + // noncopyable + OperatorSetTimeoutOut(const OperatorSetTimeoutOut &) = delete; + void operator=(const OperatorSetTimeoutOut &) = delete; + void initialize(Parser &p) override; protected: void exec(const Resources &res) const override; private: - DISALLOW_COPY_AND_ASSIGN(OperatorSetTimeoutOut); - enum TimeoutOutType { TO_OUT_UNDEFINED, TO_OUT_ACTIVE, @@ -169,14 +188,17 @@ class OperatorSkipRemap : public Operator { public: OperatorSkipRemap() { TSDebug(PLUGIN_NAME_DBG, "Calling CTOR for OperatorSkipRemap"); } + + // noncopyable + OperatorSkipRemap(const OperatorSkipRemap &) = delete; + void operator=(const OperatorSkipRemap &) = delete; + void initialize(Parser &p) override; protected: void exec(const Resources &res) const override; private: - DISALLOW_COPY_AND_ASSIGN(OperatorSkipRemap); - bool _skip_remap = false; }; @@ -186,25 +208,29 @@ class OperatorRMHeader : public OperatorHeaders public: OperatorRMHeader() { TSDebug(PLUGIN_NAME_DBG, "Calling CTOR for OperatorRMHeader"); } + // noncopyable + OperatorRMHeader(const OperatorRMHeader &) = delete; + void operator=(const OperatorRMHeader &) = delete; + protected: void exec(const Resources &res) const override; - -private: - DISALLOW_COPY_AND_ASSIGN(OperatorRMHeader); }; class OperatorAddHeader : public OperatorHeaders { public: OperatorAddHeader() { TSDebug(PLUGIN_NAME_DBG, "Calling CTOR for OperatorAddHeader"); } + + // noncopyable + OperatorAddHeader(const OperatorAddHeader &) = delete; + void operator=(const OperatorAddHeader &) = delete; + void initialize(Parser &p) override; protected: void exec(const Resources &res) const override; private: - DISALLOW_COPY_AND_ASSIGN(OperatorAddHeader); - Value _value; }; @@ -212,14 +238,17 @@ class OperatorSetHeader : public OperatorHeaders { public: OperatorSetHeader() { TSDebug(PLUGIN_NAME_DBG, "Calling CTOR for OperatorSetHeader"); } + + // noncopyable + OperatorSetHeader(const OperatorSetHeader &) = delete; + void operator=(const OperatorSetHeader &) = delete; + void initialize(Parser &p) override; protected: void exec(const Resources &res) const override; private: - DISALLOW_COPY_AND_ASSIGN(OperatorSetHeader); - Value _value; }; @@ -227,14 +256,17 @@ class OperatorCounter : public Operator { public: OperatorCounter() { TSDebug(PLUGIN_NAME_DBG, "Calling CTOR for OperatorCounter"); } + + // noncopyable + OperatorCounter(const OperatorCounter &) = delete; + void operator=(const OperatorCounter &) = delete; + void initialize(Parser &p) override; protected: void exec(const Resources &res) const override; private: - DISALLOW_COPY_AND_ASSIGN(OperatorCounter); - std::string _counter_name; int _counter = TS_ERROR; }; @@ -244,25 +276,29 @@ class OperatorRMCookie : public OperatorCookies public: OperatorRMCookie() { TSDebug(PLUGIN_NAME_DBG, "Calling CTOR for OperatorRMCookie"); } + // noncopyable + OperatorRMCookie(const OperatorRMCookie &) = delete; + void operator=(const OperatorRMCookie &) = delete; + protected: void exec(const Resources &res) const override; - -private: - DISALLOW_COPY_AND_ASSIGN(OperatorRMCookie); }; class OperatorAddCookie : public OperatorCookies { public: OperatorAddCookie() { TSDebug(PLUGIN_NAME_DBG, "Calling CTOR for OperatorAddCookie"); } + + // noncopyable + OperatorAddCookie(const OperatorAddCookie &) = delete; + void operator=(const OperatorAddCookie &) = delete; + void initialize(Parser &p) override; protected: void exec(const Resources &res) const override; private: - DISALLOW_COPY_AND_ASSIGN(OperatorAddCookie); - Value _value; }; @@ -270,14 +306,17 @@ class OperatorSetCookie : public OperatorCookies { public: OperatorSetCookie() { TSDebug(PLUGIN_NAME_DBG, "Calling CTOR for OperatorSetCookie"); } + + // noncopyable + OperatorSetCookie(const OperatorSetCookie &) = delete; + void operator=(const OperatorSetCookie &) = delete; + void initialize(Parser &p) override; protected: void exec(const Resources &res) const override; private: - DISALLOW_COPY_AND_ASSIGN(OperatorSetCookie); - Value _value; }; @@ -297,6 +336,11 @@ class OperatorSetConnDSCP : public Operator { public: OperatorSetConnDSCP() { TSDebug(PLUGIN_NAME_DBG, "Calling CTOR for OperatorSetConnDSCP"); } + + // noncopyable + OperatorSetConnDSCP(const OperatorSetConnDSCP &) = delete; + void operator=(const OperatorSetConnDSCP &) = delete; + void initialize(Parser &p) override; protected: @@ -304,8 +348,6 @@ class OperatorSetConnDSCP : public Operator void exec(const Resources &res) const override; private: - DISALLOW_COPY_AND_ASSIGN(OperatorSetConnDSCP); - Value _ds_value; }; @@ -313,6 +355,11 @@ class OperatorSetConnMark : public Operator { public: OperatorSetConnMark() { TSDebug(PLUGIN_NAME_DBG, "Calling CTOR for OperatorSetConnMark"); } + + // noncopyable + OperatorSetConnMark(const OperatorSetConnMark &) = delete; + void operator=(const OperatorSetConnMark &) = delete; + void initialize(Parser &p) override; protected: @@ -320,8 +367,6 @@ class OperatorSetConnMark : public Operator void exec(const Resources &res) const override; private: - DISALLOW_COPY_AND_ASSIGN(OperatorSetConnMark); - Value _ds_value; }; @@ -329,12 +374,14 @@ class OperatorSetDebug : public Operator { public: OperatorSetDebug() { TSDebug(PLUGIN_NAME_DBG, "Calling CTOR for OperatorSetDebug"); } + + // noncopyable + OperatorSetDebug(const OperatorSetDebug &) = delete; + void operator=(const OperatorSetDebug &) = delete; + void initialize(Parser &p) override; protected: void initialize_hooks() override; void exec(const Resources &res) const override; - -private: - DISALLOW_COPY_AND_ASSIGN(OperatorSetDebug); }; diff --git a/plugins/header_rewrite/parser.cc b/plugins/header_rewrite/parser.cc index fc821d3483c..cc7a70c52b7 100644 --- a/plugins/header_rewrite/parser.cc +++ b/plugins/header_rewrite/parser.cc @@ -28,9 +28,9 @@ #include "parser.h" -enum ParserState { PARSER_DEFAULT, PARSER_IN_QUOTE, PARSER_IN_REGEX }; +enum ParserState { PARSER_DEFAULT, PARSER_IN_QUOTE, PARSER_IN_REGEX, PARSER_IN_EXPANSION }; -Parser::Parser(const std::string &original_line, bool preserve_quotes) : _cond(false), _empty(false) +Parser::Parser(const std::string &original_line) : _cond(false), _empty(false) { std::string line = original_line; ParserState state = PARSER_DEFAULT; @@ -39,8 +39,7 @@ Parser::Parser(const std::string &original_line, bool preserve_quotes) : _cond(f size_t cur_token_length = 0; for (size_t i = 0; i < line.size(); ++i) { - if ((state == PARSER_DEFAULT) && - (std::isspace(line[i]) || ((line[i] == '=') || (line[i] == '>') || (line[i] == '<') || (line[i] == '+')))) { + if ((state == PARSER_DEFAULT) && (std::isspace(line[i]) || ((line[i] == '=')))) { if (extracting_token) { cur_token_length = i - cur_token_start; if (cur_token_length > 0) { @@ -78,11 +77,7 @@ Parser::Parser(const std::string &original_line, bool preserve_quotes) : _cond(f cur_token_start = i + 1; // Eat the leading quote } else if ((state == PARSER_IN_QUOTE) && extracting_token) { cur_token_length = i - cur_token_start; - if (preserve_quotes) { - _tokens.push_back(line.substr(cur_token_start - 1, i - cur_token_start + 2)); - } else { - _tokens.push_back(line.substr(cur_token_start, cur_token_length)); - } + _tokens.push_back(line.substr(cur_token_start, cur_token_length)); state = PARSER_DEFAULT; extracting_token = false; } else { @@ -99,8 +94,8 @@ Parser::Parser(const std::string &original_line, bool preserve_quotes) : _cond(f break; } - if ((line[i] == '=') || (line[i] == '>') || (line[i] == '<') || (line[i] == '+')) { - // These are always a seperate token + if ((line[i] == '=') || (line[i] == '+')) { + // These are always a separate token _tokens.push_back(std::string(1, line[i])); continue; } @@ -202,7 +197,7 @@ Parser::preprocess(std::vector tokens) } } else { // Syntax error - TSError("[%s] mods have to be embraced in []", PLUGIN_NAME); + TSError("[%s] mods have to be enclosed in []", PLUGIN_NAME); return; } } @@ -251,3 +246,57 @@ Parser::get_tokens() const { return _tokens; } + +SimpleTokenizer::SimpleTokenizer(const std::string &original_line) +{ + std::string line = original_line; + ParserState state = PARSER_DEFAULT; + bool extracting_token = false; + off_t cur_token_start = 0; + size_t cur_token_length = 0; + + for (size_t i = 0; i < line.size(); ++i) { + extracting_token = true; + switch (state) { + case PARSER_DEFAULT: + if ((line[i] == '{') || (line[i] == '<')) { + if (line[i - 1] == '%') { + // pickup what we currently have + cur_token_length = i - cur_token_start - 1; + if (cur_token_length > 0) { + _tokens.push_back(line.substr(cur_token_start, cur_token_length)); + } + + cur_token_start = i - 1; + state = PARSER_IN_EXPANSION; + extracting_token = false; + } + } + break; + case PARSER_IN_EXPANSION: + if ((line[i] == '}') || (line[i] == '>')) { + cur_token_length = i - cur_token_start + 1; + if (cur_token_length > 0) { + _tokens.push_back(line.substr(cur_token_start, cur_token_length)); + } + cur_token_start = i + 1; + state = PARSER_DEFAULT; + extracting_token = false; + } + break; + default: + break; + } + } + + // take what was left behind + if (extracting_token) { + _tokens.push_back(line.substr(cur_token_start)); + } +} + +const std::vector & +SimpleTokenizer::get_tokens() const +{ + return _tokens; +} diff --git a/plugins/header_rewrite/parser.h b/plugins/header_rewrite/parser.h index d82203d8b7b..385c824b833 100644 --- a/plugins/header_rewrite/parser.h +++ b/plugins/header_rewrite/parser.h @@ -33,7 +33,11 @@ class Parser { public: - explicit Parser(const std::string &line, bool preserve_quotes = false); + explicit Parser(const std::string &line); + + // noncopyable + Parser(const Parser &) = delete; + void operator=(const Parser &) = delete; bool empty() const @@ -76,7 +80,6 @@ class Parser private: void preprocess(std::vector tokens); - DISALLOW_COPY_AND_ASSIGN(Parser); bool _cond; bool _empty; @@ -88,3 +91,18 @@ class Parser protected: std::vector _tokens; }; + +class SimpleTokenizer +{ +public: + explicit SimpleTokenizer(const std::string &line); + + // noncopyable + SimpleTokenizer(const SimpleTokenizer &) = delete; + void operator=(const SimpleTokenizer &) = delete; + + const std::vector &get_tokens() const; + +protected: + std::vector _tokens; +}; diff --git a/plugins/header_rewrite/regex_helper.cc b/plugins/header_rewrite/regex_helper.cc index c07d117081b..8b30361123e 100644 --- a/plugins/header_rewrite/regex_helper.cc +++ b/plugins/header_rewrite/regex_helper.cc @@ -40,18 +40,6 @@ regexHelper::setRegexMatch(const std::string &s) return true; } -const std::string & -regexHelper::getRegexString() const -{ - return regexString; -} - -int -regexHelper::getRegexCcount() const -{ - return regexCcount; -} - int regexHelper::regexMatch(const char *str, int len, int ovector[]) const { diff --git a/plugins/header_rewrite/regex_helper.h b/plugins/header_rewrite/regex_helper.h index 8f812181805..d156fee730a 100644 --- a/plugins/header_rewrite/regex_helper.h +++ b/plugins/header_rewrite/regex_helper.h @@ -39,8 +39,6 @@ class regexHelper } bool setRegexMatch(const std::string &s); - const std::string &getRegexString() const; - int getRegexCcount() const; int regexMatch(const char *, int, int ovector[]) const; private: diff --git a/plugins/header_rewrite/resources.cc b/plugins/header_rewrite/resources.cc index cc1379e93b4..bc861c6c2b7 100644 --- a/plugins/header_rewrite/resources.cc +++ b/plugins/header_rewrite/resources.cc @@ -84,7 +84,7 @@ Resources::gather(const ResourceIDs ids, TSHttpHookID hook) return; } if (ids & RSRC_RESPONSE_STATUS) { - TSDebug(PLUGIN_NAME, "\tAdding TXN client esponse status resource"); + TSDebug(PLUGIN_NAME, "\tAdding TXN client response status resource"); resp_status = TSHttpHdrStatusGet(bufp, hdr_loc); } } diff --git a/plugins/header_rewrite/resources.h b/plugins/header_rewrite/resources.h index d9d735aaa5e..ee5e0f0e6ee 100644 --- a/plugins/header_rewrite/resources.h +++ b/plugins/header_rewrite/resources.h @@ -59,6 +59,11 @@ class Resources } ~Resources() { destroy(); } + + // noncopyable + Resources(const Resources &) = delete; + void operator=(const Resources &) = delete; + void gather(const ResourceIDs ids, TSHttpHookID hook); bool ready() const @@ -78,7 +83,6 @@ class Resources private: void destroy(); - DISALLOW_COPY_AND_ASSIGN(Resources); bool _ready = false; }; diff --git a/plugins/header_rewrite/ruleset.h b/plugins/header_rewrite/ruleset.h index c963495b458..b7d2bd2b96c 100644 --- a/plugins/header_rewrite/ruleset.h +++ b/plugins/header_rewrite/ruleset.h @@ -45,6 +45,10 @@ class RuleSet delete _oper; } + // noncopyable + RuleSet(const RuleSet &) = delete; + void operator=(const RuleSet &) = delete; + // No reason to inline these void append(RuleSet *rule); bool add_condition(Parser &p, const char *filename, int lineno); @@ -107,8 +111,6 @@ class RuleSet RuleSet *next = nullptr; // Linked list private: - DISALLOW_COPY_AND_ASSIGN(RuleSet); - Condition *_cond = nullptr; // First pre-condition (linked list) Operator *_oper = nullptr; // First operator (linked list) TSHttpHookID _hook = TS_HTTP_READ_RESPONSE_HDR_HOOK; // Which hook is this rule for diff --git a/plugins/header_rewrite/statement.h b/plugins/header_rewrite/statement.h index e5aa291f5cf..7a006ba9455 100644 --- a/plugins/header_rewrite/statement.h +++ b/plugins/header_rewrite/statement.h @@ -104,6 +104,10 @@ class Statement delete _next; } + // noncopyable + Statement(const Statement &) = delete; + void operator=(const Statement &) = delete; + // Which hook are we adding this statement to? bool set_hook(TSHttpHookID hook); TSHttpHookID @@ -152,8 +156,6 @@ class Statement Statement *_next = nullptr; // Linked list private: - DISALLOW_COPY_AND_ASSIGN(Statement); - ResourceIDs _rsrc = RSRC_NONE; bool _initialized = false; TSHttpHookID _hook = TS_HTTP_READ_RESPONSE_HDR_HOOK; diff --git a/plugins/header_rewrite/value.cc b/plugins/header_rewrite/value.cc index ef15e43ffc6..5059a009755 100644 --- a/plugins/header_rewrite/value.cc +++ b/plugins/header_rewrite/value.cc @@ -33,8 +33,8 @@ Value::~Value() { TSDebug(PLUGIN_NAME_DBG, "Calling DTOR for Value"); - for (auto it = _cond_vals.begin(); it != _cond_vals.end(); it++) { - delete *it; + for (auto &_cond_val : _cond_vals) { + delete _cond_val; } } @@ -43,28 +43,28 @@ Value::set_value(const std::string &val) { _value = val; - if (_value.find("%{") != std::string::npos || _value.find("%<") != std::string::npos || _value.find("\"") != std::string::npos) { - Parser parser(_value); - auto tokens = parser.get_tokens(); - for (auto it = tokens.begin(); it != tokens.end(); it++) { - Parser tparser(*it); + if (_value.find("%{") != std::string::npos) { + SimpleTokenizer tokenizer(_value); + auto tokens = tokenizer.get_tokens(); + for (auto token : tokens) { Condition *tcond_val = nullptr; - if ((*it).substr(0, 2) == "%<") { - tcond_val = new ConditionExpandableString(*it); - } else if ((*it) == "+") { - // Skip concat token - continue; - } else { - tcond_val = condition_factory(tparser.get_op()); - if (tcond_val) { - tcond_val->initialize(tparser); - } else { - tcond_val = new ConditionStringLiteral(*it); + if (token.substr(0, 2) == "%{") { + std::string cond_token = token.substr(2, token.size() - 3); + + if ((tcond_val = condition_factory(cond_token))) { + Parser parser(_value); + + tcond_val->initialize(parser); } + } else { + tcond_val = new ConditionStringLiteral(token); + } + + if (tcond_val) { + _cond_vals.push_back(tcond_val); } - _cond_vals.push_back(tcond_val); } } else { _int_value = strtol(_value.c_str(), nullptr, 10); diff --git a/plugins/header_rewrite/value.h b/plugins/header_rewrite/value.h index 99859aaff5a..c237abba4ff 100644 --- a/plugins/header_rewrite/value.h +++ b/plugins/header_rewrite/value.h @@ -41,7 +41,11 @@ class Value : Statement public: Value() { TSDebug(PLUGIN_NAME_DBG, "Calling CTOR for Value"); } - virtual ~Value(); + ~Value() override; + + // noncopyable + Value(const Value &) = delete; + void operator=(const Value &) = delete; void set_value(const std::string &val); @@ -49,8 +53,8 @@ class Value : Statement append_value(std::string &s, const Resources &res) const { if (!_cond_vals.empty()) { - for (auto it = _cond_vals.begin(); it != _cond_vals.end(); it++) { - (*it)->append_value(s, res); + for (auto _cond_val : _cond_vals) { + _cond_val->append_value(s, res); } } else { s += _value; @@ -87,16 +91,7 @@ class Value : Statement return _value.empty(); } - bool - need_expansion() const - { - return _need_expander; - } - private: - DISALLOW_COPY_AND_ASSIGN(Value); - - bool _need_expander = false; int _int_value = 0; double _float_value = 0.0; std::string _value; diff --git a/plugins/healthchecks/healthchecks.c b/plugins/healthchecks/healthchecks.c index c2fe0955a5b..37d7d73ebde 100644 --- a/plugins/healthchecks/healthchecks.c +++ b/plugins/healthchecks/healthchecks.c @@ -206,7 +206,7 @@ hc_thread(void *data ATS_UNUSED) do { HCFileData *next = fdata->_next; - TSDebug(PLUGIN_NAME, "Cleaning up entry from frelist"); + TSDebug(PLUGIN_NAME, "Cleaning up entry from freelist"); TSfree(fdata); fdata = next; } while (fdata); @@ -458,7 +458,7 @@ hc_process_write(TSCont contp, TSEvent event, HCState *my_state) } TSVIONBytesSet(my_state->write_vio, my_state->output_bytes); TSVIOReenable(my_state->write_vio); - } else if (TS_EVENT_VCONN_WRITE_COMPLETE) { + } else if (event == TS_EVENT_VCONN_WRITE_COMPLETE) { cleanup(contp, my_state); } else if (event == TS_EVENT_ERROR) { TSError("[healthchecks] hc_process_write: Received TS_EVENT_ERROR"); @@ -477,7 +477,7 @@ hc_process_accept(TSCont contp, HCState *my_state) my_state->read_vio = TSVConnRead(my_state->net_vc, contp, my_state->req_buffer, INT64_MAX); } -/* Imlement the server intercept */ +/* Implement the server intercept */ static int hc_intercept(TSCont contp, TSEvent event, void *edata) { @@ -512,7 +512,7 @@ health_check_origin(TSCont contp ATS_UNUSED, TSEvent event ATS_UNUSED, void *eda int path_len = 0; const char *path = TSUrlPathGet(reqp, url_loc, &path_len); - /* Short circuit the / path, common case, and we won't allow healthecks on / */ + /* Short circuit the / path, common case, and we won't allow healthchecks on / */ if (!path || !path_len) { goto cleanup; } diff --git a/plugins/lua/ts_lua.c b/plugins/lua/ts_lua.c index 0d8af2c3fa5..5aef7b551ef 100644 --- a/plugins/lua/ts_lua.c +++ b/plugins/lua/ts_lua.c @@ -63,11 +63,14 @@ TSRemapInit(TSRemapInterface *api_info, char *errbuf, int errbuf_size) TSReturnCode TSRemapNewInstance(int argc, char *argv[], void **ih, char *errbuf, int errbuf_size) { - int fn; int ret; + char script[TS_LUA_MAX_SCRIPT_FNAME_LENGTH]; + char *inline_script = ""; + int fn = 0; int states = TS_LUA_MAX_STATE_COUNT; static const struct option longopt[] = { {"states", required_argument, 0, 's'}, + {"inline", required_argument, 0, 'i'}, {0, 0, 0, 0}, }; @@ -84,6 +87,8 @@ TSRemapNewInstance(int argc, char *argv[], void **ih, char *errbuf, int errbuf_s TSDebug(TS_LUA_DEBUG_TAG, "[%s] setting number of lua VM [%d]", __FUNCTION__, states); // set state break; + case 'i': + inline_script = optarg; } if (opt == -1) { @@ -97,17 +102,24 @@ TSRemapNewInstance(int argc, char *argv[], void **ih, char *errbuf, int errbuf_s return TS_ERROR; } - if (argc - optind < 1) { + if (argc - optind > 0) { + fn = 1; + if (argv[optind][0] == '/') { + snprintf(script, sizeof(script), "%s", argv[optind]); + } else { + snprintf(script, sizeof(script), "%s/%s", TSConfigDirGet(), argv[optind]); + } + } + + if (strlen(inline_script) == 0 && argc - optind < 1) { strncpy(errbuf, "[TSRemapNewInstance] - lua script file or string is required !!", errbuf_size - 1); errbuf[errbuf_size - 1] = '\0'; return TS_ERROR; } - fn = 1; - - if (argv[optind][0] != '/') { - fn = 0; - } else if (strlen(argv[optind]) >= TS_LUA_MAX_SCRIPT_FNAME_LENGTH - 16) { + if (strlen(script) >= TS_LUA_MAX_SCRIPT_FNAME_LENGTH - 16) { + strncpy(errbuf, "[TSRemapNewInstance] - lua script file name too long !!", errbuf_size - 1); + errbuf[errbuf_size - 1] = '\0'; return TS_ERROR; } @@ -116,8 +128,7 @@ TSRemapNewInstance(int argc, char *argv[], void **ih, char *errbuf, int errbuf_s // check to make sure it is a lua file and there is no parameter for the lua file if (fn && (argc - optind < 2)) { TSDebug(TS_LUA_DEBUG_TAG, "[%s] checking if script has been registered", __FUNCTION__); - char script[TS_LUA_MAX_SCRIPT_FNAME_LENGTH]; - snprintf(script, TS_LUA_MAX_SCRIPT_FNAME_LENGTH, "%s", argv[optind]); + // we only need to check the first lua VM for script registration conf = ts_lua_script_registered(ts_lua_main_ctx_array[0].lua, script); } @@ -138,9 +149,9 @@ TSRemapNewInstance(int argc, char *argv[], void **ih, char *errbuf, int errbuf_s conf->init_func = 0; if (fn) { - snprintf(conf->script, TS_LUA_MAX_SCRIPT_FNAME_LENGTH, "%s", argv[optind]); + snprintf(conf->script, TS_LUA_MAX_SCRIPT_FNAME_LENGTH, "%s", script); } else { - conf->content = argv[optind]; + conf->content = inline_script; } ts_lua_init_instance(conf); @@ -508,7 +519,11 @@ TSPluginInit(int argc, const char *argv[]) conf->remap = 0; conf->states = states; - snprintf(conf->script, TS_LUA_MAX_SCRIPT_FNAME_LENGTH, "%s", argv[optind]); + if (argv[optind][0] == '/') { + snprintf(conf->script, TS_LUA_MAX_SCRIPT_FNAME_LENGTH, "%s", argv[optind]); + } else { + snprintf(conf->script, TS_LUA_MAX_SCRIPT_FNAME_LENGTH, "%s/%s", TSConfigDirGet(), argv[optind]); + } ts_lua_init_instance(conf); diff --git a/plugins/lua/ts_lua_cached_response.c b/plugins/lua/ts_lua_cached_response.c index 28a8ef0b5eb..0fa5e174c0f 100644 --- a/plugins/lua/ts_lua_cached_response.c +++ b/plugins/lua/ts_lua_cached_response.c @@ -163,7 +163,7 @@ ts_lua_cached_response_header_get(lua_State *L) next_field_loc = TSMimeHdrFieldNextDup(http_ctx->cached_response_bufp, http_ctx->cached_response_hdrp, field_loc); lua_pushlstring(L, val, val_len); count++; - // multiple headers with the same name must be semantically the same as one value which is comma seperated + // multiple headers with the same name must be semantically the same as one value which is comma separated if (next_field_loc != TS_NULL_MLOC) { lua_pushlstring(L, ",", 1); count++; diff --git a/plugins/lua/ts_lua_client_request.c b/plugins/lua/ts_lua_client_request.c index aef49cac10e..b851eb19d7e 100644 --- a/plugins/lua/ts_lua_client_request.c +++ b/plugins/lua/ts_lua_client_request.c @@ -163,7 +163,7 @@ ts_lua_client_request_header_get(lua_State *L) next_field_loc = TSMimeHdrFieldNextDup(http_ctx->client_request_bufp, http_ctx->client_request_hdrp, field_loc); lua_pushlstring(L, val, val_len); count++; - // multiple headers with the same name must be semantically the same as one value which is comma seperated + // multiple headers with the same name must be semantically the same as one value which is comma separated if (next_field_loc != TS_NULL_MLOC) { lua_pushlstring(L, ",", 1); count++; @@ -483,6 +483,7 @@ ts_lua_client_request_set_url_port(lua_State *L) GET_HTTP_CONTEXT(http_ctx, L); + // NOLINTNEXTLINE port = luaL_checkint(L, 1); TSUrlPortSet(http_ctx->client_request_bufp, http_ctx->client_request_url, port); diff --git a/plugins/lua/ts_lua_client_response.c b/plugins/lua/ts_lua_client_response.c index d5c656c16f4..224e5e45f5f 100644 --- a/plugins/lua/ts_lua_client_response.c +++ b/plugins/lua/ts_lua_client_response.c @@ -108,7 +108,7 @@ ts_lua_client_response_header_get(lua_State *L) next_field_loc = TSMimeHdrFieldNextDup(http_ctx->client_response_bufp, http_ctx->client_response_hdrp, field_loc); lua_pushlstring(L, val, val_len); count++; - // multiple headers with the same name must be semantically the same as one value which is comma seperated + // multiple headers with the same name must be semantically the same as one value which is comma separated if (next_field_loc != TS_NULL_MLOC) { lua_pushlstring(L, ",", 1); count++; @@ -316,6 +316,7 @@ ts_lua_client_response_set_status(lua_State *L) TS_LUA_CHECK_CLIENT_RESPONSE_HDR(http_ctx); + // NOLINTNEXTLINE status = luaL_checkint(L, 1); reason = TSHttpHdrReasonLookup(status); diff --git a/plugins/lua/ts_lua_http_config.c b/plugins/lua/ts_lua_http_config.c index d7f25c61e3e..c7f46cbd4ac 100644 --- a/plugins/lua/ts_lua_http_config.c +++ b/plugins/lua/ts_lua_http_config.c @@ -134,6 +134,12 @@ typedef enum { TS_LUA_CONFIG_HTTP_ALLOW_MULTI_RANGE = TS_CONFIG_HTTP_ALLOW_MULTI_RANGE, TS_LUA_CONFIG_HTTP_REQUEST_BUFFER_ENABLED = TS_CONFIG_HTTP_REQUEST_BUFFER_ENABLED, TS_LUA_CONFIG_HTTP_ALLOW_HALF_OPEN = TS_CONFIG_HTTP_ALLOW_HALF_OPEN, + TS_LUA_CONFIG_SSL_CLIENT_VERIFY_SERVER = TS_CONFIG_SSL_CLIENT_VERIFY_SERVER, + TS_LUA_CONFIG_SSL_CLIENT_VERIFY_SERVER_POLICY = TS_CONFIG_SSL_CLIENT_VERIFY_SERVER_POLICY, + TS_LUA_CONFIG_SSL_CLIENT_VERIFY_SERVER_PROPERTIES = TS_CONFIG_SSL_CLIENT_VERIFY_SERVER_PROPERTIES, + TS_LUA_CONFIG_SSL_CLIENT_SNI_POLICY = TS_CONFIG_SSL_CLIENT_SNI_POLICY, + TS_LUA_CONFIG_SSL_CLIENT_PRIVATE_KEY_FILENAME = TS_CONFIG_SSL_CLIENT_PRIVATE_KEY_FILENAME, + TS_LUA_CONFIG_SSL_CLIENT_CA_CERT_FILENAME = TS_CONFIG_SSL_CLIENT_CA_CERT_FILENAME, TS_LUA_CONFIG_LAST_ENTRY = TS_CONFIG_LAST_ENTRY, } TSLuaOverridableConfigKey; @@ -258,6 +264,12 @@ ts_lua_var_item ts_lua_http_config_vars[] = { TS_LUA_MAKE_VAR_ITEM(TS_LUA_CONFIG_HTTP_ALLOW_MULTI_RANGE), TS_LUA_MAKE_VAR_ITEM(TS_LUA_CONFIG_HTTP_REQUEST_BUFFER_ENABLED), TS_LUA_MAKE_VAR_ITEM(TS_LUA_CONFIG_HTTP_ALLOW_HALF_OPEN), + TS_LUA_MAKE_VAR_ITEM(TS_CONFIG_SSL_CLIENT_VERIFY_SERVER), + TS_LUA_MAKE_VAR_ITEM(TS_CONFIG_SSL_CLIENT_VERIFY_SERVER_POLICY), + TS_LUA_MAKE_VAR_ITEM(TS_CONFIG_SSL_CLIENT_VERIFY_SERVER_PROPERTIES), + TS_LUA_MAKE_VAR_ITEM(TS_CONFIG_SSL_CLIENT_SNI_POLICY), + TS_LUA_MAKE_VAR_ITEM(TS_CONFIG_SSL_CLIENT_PRIVATE_KEY_FILENAME), + TS_LUA_MAKE_VAR_ITEM(TS_CONFIG_SSL_CLIENT_CA_CERT_FILENAME), TS_LUA_MAKE_VAR_ITEM(TS_LUA_CONFIG_HTTP_PER_SERVER_CONNECTION_MAX), TS_LUA_MAKE_VAR_ITEM(TS_LUA_CONFIG_HTTP_PER_SERVER_CONNECTION_MATCH), TS_LUA_MAKE_VAR_ITEM(TS_LUA_CONFIG_LAST_ENTRY), diff --git a/plugins/lua/ts_lua_http_intercept.c b/plugins/lua/ts_lua_http_intercept.c index 048f52d9182..d794231c6c5 100644 --- a/plugins/lua/ts_lua_http_intercept.c +++ b/plugins/lua/ts_lua_http_intercept.c @@ -406,7 +406,7 @@ ts_lua_flush_wakeup(ts_lua_http_intercept_ctx *ictx) ci = &ictx->cinfo; contp = TSContCreate(ts_lua_flush_wakeup_handler, ci->mutex); - action = TSContSchedule(contp, 0, TS_THREAD_POOL_DEFAULT); + action = TSContScheduleOnPool(contp, 0, TS_THREAD_POOL_NET); ai = ts_lua_async_create_item(contp, ts_lua_flush_cleanup, (void *)action, ci); TSContDataSet(contp, ai); diff --git a/plugins/lua/ts_lua_misc.c b/plugins/lua/ts_lua_misc.c index 8da3a160aa0..e12e5a1fb1b 100644 --- a/plugins/lua/ts_lua_misc.c +++ b/plugins/lua/ts_lua_misc.c @@ -211,7 +211,7 @@ ts_lua_schedule(lua_State *L) nci->contp = contp; nci->mutex = ci->mutex; - TSContSchedule(contp, sec * 1000, entry); + TSContScheduleOnPool(contp, sec * 1000, entry); return 0; } @@ -287,7 +287,7 @@ ts_lua_sleep(lua_State *L) } contp = TSContCreate(ts_lua_sleep_handler, ci->mutex); - action = TSContSchedule(contp, sec * 1000, TS_THREAD_POOL_DEFAULT); + action = TSContScheduleOnPool(contp, sec * 1000, TS_THREAD_POOL_NET); ai = ts_lua_async_create_item(contp, ts_lua_sleep_cleanup, (void *)action, ci); TSContDataSet(contp, ai); diff --git a/plugins/lua/ts_lua_server_request.c b/plugins/lua/ts_lua_server_request.c index d0b26e0800b..608f39e3e4e 100644 --- a/plugins/lua/ts_lua_server_request.c +++ b/plugins/lua/ts_lua_server_request.c @@ -188,7 +188,7 @@ ts_lua_server_request_header_get(lua_State *L) next_field_loc = TSMimeHdrFieldNextDup(http_ctx->server_request_bufp, http_ctx->server_request_hdrp, field_loc); lua_pushlstring(L, val, val_len); count++; - // multiple headers with the same name must be semantically the same as one value which is comma seperated + // multiple headers with the same name must be semantically the same as one value which is comma separated if (next_field_loc != TS_NULL_MLOC) { lua_pushlstring(L, ",", 1); count++; diff --git a/plugins/lua/ts_lua_server_response.c b/plugins/lua/ts_lua_server_response.c index 107c9b4b616..f2c78d49ab8 100644 --- a/plugins/lua/ts_lua_server_response.c +++ b/plugins/lua/ts_lua_server_response.c @@ -181,7 +181,7 @@ ts_lua_server_response_header_get(lua_State *L) next_field_loc = TSMimeHdrFieldNextDup(http_ctx->server_response_bufp, http_ctx->server_response_hdrp, field_loc); lua_pushlstring(L, val, val_len); count++; - // multiple headers with the same name must be semantically the same as one value which is comma seperated + // multiple headers with the same name must be semantically the same as one value which is comma separated if (next_field_loc != TS_NULL_MLOC) { lua_pushlstring(L, ",", 1); count++; @@ -299,6 +299,7 @@ ts_lua_server_response_set_status(lua_State *L) TS_LUA_CHECK_SERVER_RESPONSE_HDR(http_ctx); + // NOLINTNEXTLINE status = luaL_checkint(L, 1); reason = TSHttpHdrReasonLookup(status); diff --git a/plugins/lua/ts_lua_transform.c b/plugins/lua/ts_lua_transform.c index 56c27fc51bd..57da47d727c 100644 --- a/plugins/lua/ts_lua_transform.c +++ b/plugins/lua/ts_lua_transform.c @@ -89,7 +89,7 @@ ts_lua_transform_handler(TSCont contp, ts_lua_http_transform_ctx *transform_ctx, empty_input = 0; if (!TSVIOBufferGet(input_vio)) { if (transform_ctx->output.vio) { - TSDebug(TS_LUA_DEBUG_TAG, "[%s] reenabling ouput VIO after input VIO does not exist", __FUNCTION__); + TSDebug(TS_LUA_DEBUG_TAG, "[%s] reenabling output VIO after input VIO does not exist", __FUNCTION__); TSVIONBytesSet(transform_ctx->output.vio, transform_ctx->total); TSVIOReenable(transform_ctx->output.vio); return 0; diff --git a/plugins/lua/ts_lua_util.c b/plugins/lua/ts_lua_util.c index b72b076f004..34d2b1ea54e 100644 --- a/plugins/lua/ts_lua_util.c +++ b/plugins/lua/ts_lua_util.c @@ -201,7 +201,7 @@ ts_lua_add_module(ts_lua_instance_conf *conf, ts_lua_main_ctx *arr, int n, int a if (conf->content) { if (luaL_loadstring(L, conf->content)) { - snprintf(errbuf, errbuf_size, "[%s] luaL_loadstring %s failed: %s", __FUNCTION__, conf->script, lua_tostring(L, -1)); + snprintf(errbuf, errbuf_size, "[%s] luaL_loadstring failed: %s", __FUNCTION__, lua_tostring(L, -1)); lua_pop(L, 1); TSMutexUnlock(arr[i].mutexp); return -1; diff --git a/plugins/regex_remap/regex_remap.cc b/plugins/regex_remap/regex_remap.cc index 484a0d9ca68..28b5223c8a4 100644 --- a/plugins/regex_remap/regex_remap.cc +++ b/plugins/regex_remap/regex_remap.cc @@ -75,21 +75,7 @@ enum ExtraSubstitutions { // length calculations (we need all of them). // struct UrlComponents { - UrlComponents() - : scheme(nullptr), - host(nullptr), - path(nullptr), - query(nullptr), - matrix(nullptr), - port(0), - scheme_len(0), - host_len(0), - path_len(0), - query_len(0), - matrix_len(0), - url_len(0) - { - } + UrlComponents() {} void populate(TSRemapRequestInfo *rri) @@ -104,20 +90,20 @@ struct UrlComponents { url_len = scheme_len + host_len + path_len + query_len + matrix_len + 32; } - const char *scheme; - const char *host; - const char *path; - const char *query; - const char *matrix; - int port; + const char *scheme = nullptr; + const char *host = nullptr; + const char *path = nullptr; + const char *query = nullptr; + const char *matrix = nullptr; + int port = 0; - int scheme_len; - int host_len; - int path_len; - int query_len; - int matrix_len; + int scheme_len = 0; + int host_len = 0; + int path_len = 0; + int query_len = 0; + int matrix_len = 0; - int url_len; // Full length, of all components + int url_len = 0; // Full length, of all components }; /////////////////////////////////////////////////////////////////////////////// @@ -643,29 +629,17 @@ RemapRegex::substitute(char dest[], const char *src, const int ovector[], const // Hold one remap instance struct RemapInstance { - RemapInstance() - : first(nullptr), - last(nullptr), - profile(false), - method(false), - query_string(true), - matrix_params(false), - host(false), - hits(0), - misses(0), - filename("unknown") - { - } - - RemapRegex *first; - RemapRegex *last; - bool profile; - bool method; - bool query_string; - bool matrix_params; - bool host; - int hits; - int misses; + RemapInstance() : filename("unknown") {} + + RemapRegex *first = nullptr; + RemapRegex *last = nullptr; + bool profile = false; + bool method = false; + bool query_string = true; + bool matrix_params = false; + bool host = false; + int hits = 0; + int misses = 0; std::string filename; }; diff --git a/plugins/regex_revalidate/regex_revalidate.c b/plugins/regex_revalidate/regex_revalidate.c index 56713ea4df9..d7ff7065639 100644 --- a/plugins/regex_revalidate/regex_revalidate.c +++ b/plugins/regex_revalidate/regex_revalidate.c @@ -19,8 +19,7 @@ limitations under the License. */ -#include "ts/ts.h" -#include "ts/remap.h" +#include #include #include @@ -32,7 +31,6 @@ #include #include #include -#include #ifdef HAVE_PCRE_PCRE_H #include @@ -59,13 +57,6 @@ ts_free(void *s) return TSfree(s); } -static void -setup_memory_allocation() -{ - pcre_malloc = &ts_malloc; - pcre_free = &ts_free; -} - typedef struct invalidate_t { const char *regex_text; pcre *regex; @@ -375,7 +366,7 @@ config_handler(TSCont cont, TSEvent event, void *edata) if (iptr) { free_cont = TSContCreate(free_handler, TSMutexCreate()); TSContDataSet(free_cont, (void *)iptr); - TSContSchedule(free_cont, FREE_TMOUT, TS_THREAD_POOL_TASK); + TSContScheduleOnPool(free_cont, FREE_TMOUT, TS_THREAD_POOL_TASK); } } else { TSDebug(LOG_PREFIX, "No Changes"); @@ -386,7 +377,10 @@ config_handler(TSCont cont, TSEvent event, void *edata) TSMutexUnlock(mutex); - TSContSchedule(cont, CONFIG_TMOUT, TS_THREAD_POOL_TASK); + // Don't reschedule for TS_EVENT_MGMT_UPDATE + if (event == TS_EVENT_TIMEOUT) { + TSContScheduleOnPool(cont, CONFIG_TMOUT, TS_THREAD_POOL_TASK); + } return 0; } @@ -460,23 +454,44 @@ main_handler(TSCont cont, TSEvent event, void *edata) return 0; } -TSRemapStatus -TSRemapDoRemap(void *ih, TSHttpTxn txnp, TSRemapRequestInfo *rri) +static bool +check_ts_version() { - TSCont main_cont = NULL; + const char *ts_version = TSTrafficServerVersionGet(); - main_cont = TSContCreate(main_handler, NULL); - TSContDataSet(main_cont, ih); - TSHttpTxnHookAdd(txnp, TS_HTTP_SEND_REQUEST_HDR_HOOK, main_cont); + if (ts_version) { + int major_ts_version = 0; + int minor_ts_version = 0; + int micro_ts_version = 0; + + if (sscanf(ts_version, "%d.%d.%d", &major_ts_version, &minor_ts_version, µ_ts_version) != 3) { + return false; + } - return TSREMAP_NO_REMAP; + if ((TS_VERSION_MAJOR == major_ts_version) && (TS_VERSION_MINOR == minor_ts_version) && + (TS_VERSION_MICRO == micro_ts_version)) { + return true; + } + } + + return false; } -static bool -configure_plugin_state(plugin_state_t *pstate, int argc, const char **argv, bool *const disable_timed_reload) +void +TSPluginInit(int argc, const char *argv[]) { + TSPluginRegistrationInfo info; + TSCont main_cont, config_cont; + plugin_state_t *pstate; + invalidate_t *iptr = NULL; + bool disable_timed_reload = false; + + TSDebug(LOG_PREFIX, "Starting plugin init"); + + pstate = (plugin_state_t *)TSmalloc(sizeof(plugin_state_t)); + init_plugin_state_t(pstate); + int c; - invalidate_t *iptr = NULL; static const struct option longopts[] = {{"config", required_argument, NULL, 'c'}, {"log", required_argument, NULL, 'l'}, {"disable-timed-reload", no_argument, NULL, 'd'}, @@ -486,19 +501,16 @@ configure_plugin_state(plugin_state_t *pstate, int argc, const char **argv, bool switch (c) { case 'c': pstate->config_file = TSstrdup(optarg); - TSDebug(LOG_PREFIX, "Config File: %s", pstate->config_file); break; case 'l': if (TS_SUCCESS == TSTextLogObjectCreate(optarg, TS_LOG_MODE_ADD_TIMESTAMP, &pstate->log)) { TSTextLogObjectRollingEnabledSet(pstate->log, 1); TSTextLogObjectRollingIntervalSecSet(pstate->log, LOG_ROLL_INTERVAL); TSTextLogObjectRollingOffsetHrSet(pstate->log, LOG_ROLL_OFFSET); - TSDebug(LOG_PREFIX, "Logging Mode enabled"); } break; case 'd': - *disable_timed_reload = true; - TSDebug(LOG_PREFIX, "Timed reload disabled (disable-timed-reload)"); + disable_timed_reload = true; break; default: break; @@ -507,7 +519,8 @@ configure_plugin_state(plugin_state_t *pstate, int argc, const char **argv, bool if (!pstate->config_file) { TSError("[regex_revalidate] Plugin requires a --config option along with a config file name"); - return false; + free_plugin_state_t(pstate); + return; } if (!load_config(pstate, &iptr)) { @@ -517,104 +530,6 @@ configure_plugin_state(plugin_state_t *pstate, int argc, const char **argv, bool list_config(pstate, iptr); } - return true; -} - -TSReturnCode -TSRemapNewInstance(int argc, char *argv[], void **ih, char *errbuf, int errbuf_size) -{ - TSCont config_cont = NULL; - plugin_state_t *pstate = NULL; - bool disable_timed_reload = false; - - TSDebug(LOG_PREFIX, "Starting remap init"); - pstate = (plugin_state_t *)TSmalloc(sizeof(plugin_state_t)); - init_plugin_state_t(pstate); - - if (!configure_plugin_state(pstate, argc - 1, (const char **)(argv + 1), &disable_timed_reload)) { - free_plugin_state_t(pstate); - TSError("[regex_revalidate] Remap plugin registration failed"); - return TS_ERROR; - } - - *ih = (void *)pstate; - - config_cont = TSContCreate(config_handler, TSMutexCreate()); - TSContDataSet(config_cont, (void *)pstate); - - TSMgmtUpdateRegister(config_cont, LOG_PREFIX); - - if (!disable_timed_reload) { - TSContSchedule(config_cont, CONFIG_TMOUT, TS_THREAD_POOL_TASK); - } - - TSDebug(LOG_PREFIX, "Remap plugin registration succeeded"); - - return TS_SUCCESS; -} - -void -TSRemapDeleteInstance(void *ih) -{ - if (NULL != ih) { - plugin_state_t *const pstate = (plugin_state_t *)ih; - free_plugin_state_t(pstate); - } -} - -static bool -check_ts_version() -{ - const char *ts_version = TSTrafficServerVersionGet(); - - if (ts_version) { - int major_ts_version = 0; - int minor_ts_version = 0; - int micro_ts_version = 0; - - if (sscanf(ts_version, "%d.%d.%d", &major_ts_version, &minor_ts_version, µ_ts_version) != 3) { - return false; - } - - if ((TS_VERSION_MAJOR == major_ts_version)) { - return true; - } - } - - return false; -} - -TSReturnCode -TSRemapInit(TSRemapInterface *api_info, char *errbug, int errbuf_size) -{ - setup_memory_allocation(); - - if (!check_ts_version()) { - TSError("[regex_revalidate] Plugin requires Traffic Server %d", TS_VERSION_MAJOR); - return TS_ERROR; - } - - return TS_SUCCESS; -} - -void -TSPluginInit(int argc, const char *argv[]) -{ - TSPluginRegistrationInfo info; - TSCont main_cont, config_cont; - plugin_state_t *pstate = NULL; - bool disable_timed_reload = false; - - TSDebug(LOG_PREFIX, "Starting plugin init"); - - pstate = (plugin_state_t *)TSmalloc(sizeof(plugin_state_t)); - init_plugin_state_t(pstate); - - if (!configure_plugin_state(pstate, argc, argv, &disable_timed_reload)) { - free_plugin_state_t(pstate); - return; - } - info.plugin_name = LOG_PREFIX; info.vendor_name = "Apache Software Foundation"; info.support_email = "dev@trafficserver.apache.org"; @@ -629,12 +544,13 @@ TSPluginInit(int argc, const char *argv[]) } if (!check_ts_version()) { - TSError("[regex_revalidate] Plugin requires Traffic Server %d", TS_VERSION_MAJOR); + TSError("[regex_revalidate] Plugin requires Traffic Server %d.%d.%d", TS_VERSION_MAJOR, TS_VERSION_MINOR, TS_VERSION_MICRO); free_plugin_state_t(pstate); return; } - setup_memory_allocation(); + pcre_malloc = &ts_malloc; + pcre_free = &ts_free; main_cont = TSContCreate(main_handler, NULL); TSContDataSet(main_cont, (void *)pstate); @@ -646,7 +562,7 @@ TSPluginInit(int argc, const char *argv[]) TSMgmtUpdateRegister(config_cont, LOG_PREFIX); if (!disable_timed_reload) { - TSContSchedule(config_cont, CONFIG_TMOUT, TS_THREAD_POOL_TASK); + TSContScheduleOnPool(config_cont, CONFIG_TMOUT, TS_THREAD_POOL_TASK); } TSDebug(LOG_PREFIX, "Plugin Init Complete"); diff --git a/plugins/s3_auth/aws_auth_v4.cc b/plugins/s3_auth/aws_auth_v4.cc index 61f595de412..1df1a6175ec 100644 --- a/plugins/s3_auth/aws_auth_v4.cc +++ b/plugins/s3_auth/aws_auth_v4.cc @@ -103,32 +103,52 @@ uriEncode(const String &in, bool isObjectName) } /** - * @brief URI-decode a character string (AWS specific version, see spec) + * @brief checks if the string is URI-encoded (AWS specific encoding version, see spec) * * @see AWS spec: http://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-header-based-auth.html * - * @todo Consider reusing / converting to TSStringPercentDecode() - * Currently we don't build a library/archive so we could link with the unit-test binary. Also using - * different sets of encode/decode functions during runtime and unit-testing did not seem as a good idea. - * @param in string to be URI decoded - * @return encoded string. + * @note According to the following RFC if the string is encoded and contains '%' it should + * be followed by 2 hexadecimal symbols otherwise '%' should be encoded with %25: + * https://tools.ietf.org/html/rfc3986#section-2.1 + * + * @param in string to be URI checked + * @param isObjectName if true encoding didn't encode '/', kept it as it is. + * @return true if encoded, false not encoded. */ -String -uriDecode(const String &in) +bool +isUriEncoded(const String &in, bool isObjectName) { - std::string result; - result.reserve(in.length()); - size_t i = 0; - while (i < in.length()) { - if (in[i] == '%') { - result += static_cast(std::stoi(in.substr(i + 1, 2), nullptr, 16)); - i += 3; - } else { - result += in[i]; - i++; + for (size_t pos = 0; pos < in.length(); pos++) { + char c = in[pos]; + + if (isalnum(c) || c == '-' || c == '_' || c == '.' || c == '~') { + /* found a unreserved character which should not have been be encoded regardless + * 'A'-'Z', 'a'-'z', '0'-'9', '-', '.', '_', and '~'. */ + continue; + } + + if (' ' == c) { + /* space should have been encoded with %20 if the string was encoded */ + return false; + } + + if ('/' == c && !isObjectName) { + /* if this is not an object name '/' should have been encoded */ + return false; + } + + if ('%' == c) { + if (pos + 2 < in.length() && std::isxdigit(in[pos + 1]) && std::isxdigit(in[pos + 2])) { + /* if string was encoded we should have exactly 2 hexadecimal chars following it */ + return true; + } else { + /* lonely '%' should have been encoded with %25 according to the RFC so likely not encoded */ + return false; + } } } - return result; + + return false; } /** @@ -290,10 +310,7 @@ getCanonicalRequestSha256Hash(TsInterface &api, bool signPayload, const StringSe paramNames.insert(encodedParam); - /* Look for '%' first trying to avoid as many uri-decode calls as possible. - * it is hard to estimate which is more likely use-case - (1) URIs with uri-encoded query parameter - * values or (2) with unencoded which defines the success of this optimization */ - if (nullptr == memchr(value.c_str(), '%', value.length()) || 0 == uriDecode(value).compare(value)) { + if (!isUriEncoded(value, /* isObjectName */ false)) { /* Not URI-encoded */ paramsMap[encodedParam] = uriEncode(value, /* isObjectName */ false); } else { diff --git a/plugins/s3_auth/aws_auth_v4.h b/plugins/s3_auth/aws_auth_v4.h index d0eac44a1f6..aa929de57be 100644 --- a/plugins/s3_auth/aws_auth_v4.h +++ b/plugins/s3_auth/aws_auth_v4.h @@ -25,7 +25,7 @@ #pragma once #include /* transform() */ -#include /* soze_t */ +#include /* size_t */ #include /* std::string */ #include /* std::stringstream */ #include /* std::map */ diff --git a/plugins/s3_auth/aws_auth_v4_wrap.h b/plugins/s3_auth/aws_auth_v4_wrap.h index a4351478c61..72221c3b89b 100644 --- a/plugins/s3_auth/aws_auth_v4_wrap.h +++ b/plugins/s3_auth/aws_auth_v4_wrap.h @@ -18,7 +18,7 @@ /** * @file aws_auth_v4_ts.h - * @brief TS API adaptor and header iterator using the TS API which are swapped with mocks during testing. + * @brief TS API adapter and header iterator using the TS API which are swapped with mocks during testing. * @see aws_auth_v4.h */ @@ -28,7 +28,7 @@ class HeaderIterator { public: - HeaderIterator() : _bufp(nullptr), _hdrs(TS_NULL_MLOC), _field(TS_NULL_MLOC) {} + HeaderIterator() : _hdrs(TS_NULL_MLOC), _field(TS_NULL_MLOC) {} HeaderIterator(TSMBuffer bufp, TSMLoc hdrs, TSMLoc field) : _bufp(bufp), _hdrs(hdrs), _field(field) {} HeaderIterator(const HeaderIterator &it) { @@ -81,7 +81,7 @@ class HeaderIterator { return TSMimeHdrFieldValueStringGet(_bufp, _hdrs, _field, -1, len); } - TSMBuffer _bufp; + TSMBuffer _bufp = nullptr; TSMLoc _hdrs; TSMLoc _field; }; diff --git a/plugins/s3_auth/s3_auth.cc b/plugins/s3_auth/s3_auth.cc index de79dfe9a2e..77e86fb3057 100644 --- a/plugins/s3_auth/s3_auth.cc +++ b/plugins/s3_auth/s3_auth.cc @@ -41,7 +41,7 @@ #include #include "tscore/ink_config.h" -// Special snowflake here, only availbale when building inside the ATS source tree. +// Special snowflake here, only available when building inside the ATS source tree. #include "tscore/ink_atomic.h" #include "aws_auth_v4.h" @@ -137,7 +137,7 @@ loadRegionMap(StringMap &m, const String &filename) } /////////////////////////////////////////////////////////////////////////////// -// Cache for the secrets file, to avoid reading / loding them repeatedly on +// Cache for the secrets file, to avoid reading / loading them repeatedly on // a reload of remap.config. This gets cached for 60s (not configurable). // class S3Config; @@ -463,7 +463,7 @@ S3Config::parse_config(const std::string &config_fname) continue; } - // Skip trailig white spaces + // Skip trailing white spaces pos2 = pos1; pos1 = pos2 + strlen(pos2) - 1; while ((pos1 > pos2) && isspace(*pos1)) { @@ -647,7 +647,7 @@ S3Request::set_header(const char *header, int header_len, const char *val, int v return ret; } -// dst poinsts to starting offset of dst buffer +// dst points to starting offset of dst buffer // dst_len remaining space in buffer static size_t str_concat(char *dst, size_t dst_len, const char *src, size_t src_len) @@ -909,7 +909,7 @@ event_handler(TSCont cont, TSEvent event, void *edata) } if (TS_HTTP_STATUS_OK == status) { - TSDebug(PLUGIN_NAME, "Succesfully signed the AWS S3 URL"); + TSDebug(PLUGIN_NAME, "Successfully signed the AWS S3 URL"); } else { TSDebug(PLUGIN_NAME, "Failed to sign the AWS S3 URL, status = %d", status); TSHttpTxnStatusSet(txnp, status); @@ -1060,7 +1060,7 @@ TSRemapDoRemap(void *ih, TSHttpTxn txnp, TSRemapRequestInfo * /* rri */) if (s3) { TSAssert(s3->valid()); - s3->acquire(); // Increasement ref-count + s3->acquire(); // Increase ref-count // Now schedule the continuation to update the URL when going to origin. // Note that in most cases, this is a No-Op, assuming you have reasonable // cache hit ratio. However, the scheduling is next to free (very cheap). diff --git a/plugins/s3_auth/unit_tests/test_aws_auth_v4.cc b/plugins/s3_auth/unit_tests/test_aws_auth_v4.cc index 7dd4f600880..408ce21f2d8 100644 --- a/plugins/s3_auth/unit_tests/test_aws_auth_v4.cc +++ b/plugins/s3_auth/unit_tests/test_aws_auth_v4.cc @@ -69,37 +69,78 @@ TEST_CASE("uriEncode(): encode reserved chars in an object name", "[AWS][auth][u CHECK_FALSE(encoded.compare("%20/%21%22%23%24%25%26%27%28%29%2A%2B%2C%3A%3B%3C%3D%3E%3F%40%5B%5C%5D%5E%60%7B%7C%7D")); } -TEST_CASE("uriDecode(): decode empty input", "[AWS][auth][utility]") +TEST_CASE("isUriEncoded(): check an empty input", "[AWS][auth][utility]") { - String encoded(""); - String decoded = uriDecode(encoded); - CHECK(0 == decoded.length()); /* 0 encoded because of the invalid input */ + CHECK(false == isUriEncoded("")); } -TEST_CASE("uriDecode(): decode unreserved chars", "[AWS][auth][utility]") +TEST_CASE("isUriEncoded(): '%' and nothing else", "[AWS][auth][utility]") +{ + CHECK(false == isUriEncoded("%")); +} + +TEST_CASE("isUriEncoded(): '%' but no hex digits", "[AWS][auth][utility]") +{ + CHECK(false == isUriEncoded("XXX%XXX")); +} + +TEST_CASE("isUriEncoded(): '%' but only one hex digit", "[AWS][auth][utility]") +{ + CHECK(false == isUriEncoded("XXXXX%1XXXXXX")); + CHECK(false == isUriEncoded("XXX%1")); // test end of string case +} + +TEST_CASE("isUriEncoded(): '%' and 2 hex digit", "[AWS][auth][utility]") +{ + CHECK(true == isUriEncoded("XXX%12XXX")); + CHECK(true == isUriEncoded("XXX%12")); // test end of string case +} + +TEST_CASE("isUriEncoded(): space not encoded", "[AWS][auth][utility]") +{ + // Having a space always means it was not encoded. + CHECK(false == isUriEncoded("XXXXX XXXXXX")); +} + +TEST_CASE("isUriEncoded(): '/' in strings which are not object names", "[AWS][auth][utility]") +{ + // This is not an object name so if we have '/' => the string was not encoded. + CHECK(false == isUriEncoded("XXXXX/XXXXXX", /* isObjectName */ false)); + + // There is no '/' and '%2F' shows that it was encoded. + CHECK(true == isUriEncoded("XXXXX%2FXXXXXX", /* isObjectName */ false)); + + // This is not an object name so if we have '/' => the string was not encoded despite '%20' in it. + CHECK(false == isUriEncoded("XXXXX/%20XXXXX", /* isObjectName */ false)); +} + +TEST_CASE("isUriEncoded(): '/' in strings that are object names", "[AWS][auth][utility]") +{ + // This is an object name so having '/' is normal but not enough to conclude if it is encoded or not. + CHECK(false == isUriEncoded("XXXXX/XXXXXX", /* isObjectName */ true)); + + // There is no '/' and '%2F' shows it is encoded. + CHECK(true == isUriEncoded("XXXXX%2FXXXXXX", /* isObjectName */ true)); + + // This is an object name so having '/' is normal and because of '%20' we can conclude it was encoded. + CHECK(true == isUriEncoded("XXXXX/%20XXXXX", /* isObjectName */ true)); +} + +TEST_CASE("isUriEncoded(): no reserved chars in the input", "[AWS][auth][utility]") { const String encoded = "ABCDEFGHIJKLMNOPQRSTUVWXYZ" "abcdefghijklmnopqrstuvwxyz" "0123456789" "-._~"; - String decoded = uriDecode(encoded); - - CHECK(encoded.length() == encoded.length()); - CHECK_FALSE(decoded.compare("ABCDEFGHIJKLMNOPQRSTUVWXYZ" - "abcdefghijklmnopqrstuvwxyz" - "0123456789" - "-._~")); + CHECK(false == isUriEncoded(encoded)); } -TEST_CASE("uriDecode(): decode reserved chars", "[AWS][auth][utility]") +TEST_CASE("isUriEncoded(): reserved chars in the input", "[AWS][auth][utility]") { - const String encoded = - "%20%2F%21%22%23%24%25%26%27%28%29%2A%2B%2C%3A%3B%3C%3D%3E%3F%40%5B%5C%5D%5E%60%7B%7C%7D"; /* some printable but - reserved chars */ - String decoded = uriDecode(encoded); + // some printable but reserved chars " /!\"#$%&'()*+,:;<=>?@[\\]^`{|}" + const String encoded = "%20%2F%21%22%23%24%25%26%27%28%29%2A%2B%2C%3A%3B%3C%3D%3E%3F%40%5B%5C%5D%5E%60%7B%7C%7D"; - CHECK(3 * decoded.length() == encoded.length()); /* size of "%NN" = 3 */ - CHECK_FALSE(decoded.compare(" /!\"#$%&'()*+,:;<=>?@[\\]^`{|}")); + CHECK(true == isUriEncoded(encoded)); } /* base16Encode() ************************************************************************************************************** */ @@ -323,7 +364,7 @@ ValidateBench(TsInterface &api, bool signPayload, time_t *now, const char *bench CAPTURE(signedHeaders); CHECK_FALSE(signedHeaders.compare(bench[6])); - /* Test the formating of the date and time */ + /* Test the formatting of the date and time */ char dateTime[sizeof("20170428T010203Z")]; size_t dateTimeLen = getIso8601Time(now, dateTime, sizeof(dateTime)); CAPTURE(String(dateTime, dateTimeLen)); diff --git a/plugins/s3_auth/unit_tests/test_aws_auth_v4.h b/plugins/s3_auth/unit_tests/test_aws_auth_v4.h index 5b0ea30dd96..134151797b4 100644 --- a/plugins/s3_auth/unit_tests/test_aws_auth_v4.h +++ b/plugins/s3_auth/unit_tests/test_aws_auth_v4.h @@ -121,7 +121,7 @@ class MockTsInterface : public TsInterface /* Expose the following methods only to the unit tests */ String base16Encode(const char *in, size_t inLen); String uriEncode(const String &in, bool isObjectName = false); -String uriDecode(const String &in); +bool isUriEncoded(const String &in, bool isObjectName = false); String lowercase(const char *in, size_t inLen); const char *trimWhiteSpaces(const char *in, size_t inLen, size_t &newLen); diff --git a/plugins/stats_over_http/stats_over_http.c b/plugins/stats_over_http/stats_over_http.c index 919c394f62d..3f44b688678 100644 --- a/plugins/stats_over_http/stats_over_http.c +++ b/plugins/stats_over_http/stats_over_http.c @@ -205,7 +205,7 @@ stats_process_write(TSCont contp, TSEvent event, stats_state *my_state) TSVIONBytesSet(my_state->write_vio, my_state->output_bytes); } TSVIOReenable(my_state->write_vio); - } else if (TS_EVENT_VCONN_WRITE_COMPLETE) { + } else if (event == TS_EVENT_VCONN_WRITE_COMPLETE) { stats_cleanup(contp, my_state); } else if (event == TS_EVENT_ERROR) { TSError("[%s] stats_process_write: Received TS_EVENT_ERROR", PLUGIN_NAME); diff --git a/plugins/tcpinfo/tcpinfo.cc b/plugins/tcpinfo/tcpinfo.cc index dccb4c22300..e8e498e668e 100644 --- a/plugins/tcpinfo/tcpinfo.cc +++ b/plugins/tcpinfo/tcpinfo.cc @@ -53,11 +53,11 @@ // Log format headers. These are emitted once at the start of a log file. Note that we // carefully order the fields so the field ordering is compatible. This lets you change -// the verbosity without breaking a perser that is moderately robust. +// the verbosity without breaking a parser that is moderately robust. static const char *tcpi_headers[] = { "timestamp event client server rtt", "timestamp event client server rtt rttvar last_sent last_recv " - "snd_ssthresh rcv_ssthresh unacked sacked lost retrans fackets all_retrans", + "snd_cwnd snd_ssthresh rcv_ssthresh unacked sacked lost retrans fackets all_retrans", }; struct Config { @@ -254,7 +254,7 @@ parse_unsigned(const char *str, unsigned long &lval) } if (end && *end != '\0') { - // Not all charaters consumed. + // Not all characters consumed. return false; } diff --git a/plugins/xdebug/xdebug.cc b/plugins/xdebug/xdebug.cc index 29751e6bd36..a2d54b8becf 100644 --- a/plugins/xdebug/xdebug.cc +++ b/plugins/xdebug/xdebug.cc @@ -26,13 +26,15 @@ #include #include #include +#include #include #include "tscore/ink_defs.h" #include "tscpp/util/PostScript.h" #include "tscpp/util/TextView.h" -#define DEBUG_TAG_LOG_HEADERS "xdebug.headers" +#include "xdebug_headers.cc" +#include "xdebug_transforms.cc" static struct { const char *str; @@ -47,9 +49,11 @@ enum { XHEADER_X_TRANSACTION_ID = 1u << 6, XHEADER_X_DUMP_HEADERS = 1u << 7, XHEADER_X_REMAP = 1u << 8, + XHEADER_X_PROBE_HEADERS = 1u << 9, }; static int XArgIndex = 0; +static int BodyBuilderArgIndex = 0; static TSCont XInjectHeadersCont = nullptr; static TSCont XDeleteDebugHdrCont = nullptr; @@ -321,48 +325,6 @@ InjectTxnUuidHeader(TSHttpTxn txn, TSMBuffer buffer, TSMLoc hdr) } } -/////////////////////////////////////////////////////////////////////////// -// Dump a header on stderr, useful together with TSDebug(). -void -log_headers(TSHttpTxn txn, TSMBuffer bufp, TSMLoc hdr_loc, const char *msg_type) -{ - if (!TSIsDebugTagSet(DEBUG_TAG_LOG_HEADERS)) { - return; - } - - TSIOBuffer output_buffer; - TSIOBufferReader reader; - TSIOBufferBlock block; - const char *block_start; - int64_t block_avail; - - std::stringstream ss; - ss << "TxnID:" << TSHttpTxnIdGet(txn) << " " << msg_type << " Headers are..."; - - output_buffer = TSIOBufferCreate(); - reader = TSIOBufferReaderAlloc(output_buffer); - - /* This will print just MIMEFields and not the http request line */ - TSMimeHdrPrint(bufp, hdr_loc, output_buffer); - - /* We need to loop over all the buffer blocks, there can be more than 1 */ - block = TSIOBufferReaderStart(reader); - do { - block_start = TSIOBufferBlockReadStart(block, reader, &block_avail); - if (block_avail > 0) { - ss << "\n" << std::string(block_start, static_cast(block_avail)); - } - TSIOBufferReaderConsume(reader, block_avail); - block = TSIOBufferReaderStart(reader); - } while (block && block_avail != 0); - - /* Free up the TSIOBuffer that we used to print out the header */ - TSIOBufferReaderFree(reader); - TSIOBufferDestroy(output_buffer); - - TSDebug(DEBUG_TAG_LOG_HEADERS, "%s", ss.str().c_str()); -} - static int XInjectResponseHeaders(TSCont /* contp */, TSEvent event, void *edata) { @@ -401,12 +363,25 @@ XInjectResponseHeaders(TSCont /* contp */, TSEvent event, void *edata) InjectTxnUuidHeader(txn, buffer, hdr); } + if (xheaders & XHEADER_X_REMAP) { + InjectRemapHeader(txn, buffer, hdr); + } + + // intentionally placed after all injected headers. + if (xheaders & XHEADER_X_DUMP_HEADERS) { log_headers(txn, buffer, hdr, "ClientResponse"); } - if (xheaders & XHEADER_X_REMAP) { - InjectRemapHeader(txn, buffer, hdr); + if (xheaders & XHEADER_X_PROBE_HEADERS) { + BodyBuilder *data = static_cast(TSHttpTxnArgGet(txn, BodyBuilderArgIndex)); + TSDebug("xdebug_transform", "XInjectResponseHeaders(): client resp header ready"); + if (data == nullptr) { + TSHttpTxnReenable(txn, TS_EVENT_HTTP_ERROR); + return TS_ERROR; + } + data->hdr_ready = true; + writePostBody(data); } done: @@ -519,44 +494,36 @@ XScanRequestHeaders(TSCont /* contp */, TSEvent event, void *edata) } else if (header_field_eq("diags", value, vsize)) { // Enable diagnostics for DebugTxn()'s only TSHttpTxnDebugSet(txn, 1); - } else if (header_field_eq("log-headers", value, vsize)) { - xheaders |= XHEADER_X_DUMP_HEADERS; - log_headers(txn, buffer, hdr, "ClientRequest"); - - // dump on server request - auto send_req_dump = [](TSCont /* contp */, TSEvent event, void *edata) -> int { - TSHttpTxn txn = (TSHttpTxn)edata; - TSMBuffer buffer; - TSMLoc hdr; - if (TSHttpTxnServerReqGet(txn, &buffer, &hdr) == TS_SUCCESS) { - // re-add header "X-Debug: log-headers", but only once - TSMLoc dst = TSMimeHdrFieldFind(buffer, hdr, xDebugHeader.str, xDebugHeader.len); - if (dst == TS_NULL_MLOC) { - if (TSMimeHdrFieldCreateNamed(buffer, hdr, xDebugHeader.str, xDebugHeader.len, &dst) == TS_SUCCESS) { - TSReleaseAssert(TSMimeHdrFieldAppend(buffer, hdr, dst) == TS_SUCCESS); - TSReleaseAssert(TSMimeHdrFieldValueStringInsert(buffer, hdr, dst, 0 /* idx */, "log-headers", - lengthof("log-headers")) == TS_SUCCESS); - log_headers(txn, buffer, hdr, "ServerRequest"); - } - } - } - TSHttpTxnReenable(txn, TS_EVENT_HTTP_CONTINUE); - return TS_EVENT_NONE; - }; - TSHttpTxnHookAdd(txn, TS_HTTP_SEND_REQUEST_HDR_HOOK, TSContCreate(send_req_dump, nullptr)); - - // dump on server response - auto read_resp_dump = [](TSCont /* contp */, TSEvent event, void *edata) -> int { - TSHttpTxn txn = (TSHttpTxn)edata; - TSMBuffer buffer; - TSMLoc hdr; - if (TSHttpTxnServerRespGet(txn, &buffer, &hdr) == TS_SUCCESS) { - log_headers(txn, buffer, hdr, "ServerResponse"); - } - TSHttpTxnReenable(txn, TS_EVENT_HTTP_CONTINUE); + + } else if (header_field_eq("probe", value, vsize)) { + xheaders |= XHEADER_X_PROBE_HEADERS; + + // prefix request headers and postfix response headers + BodyBuilder *data = new BodyBuilder(); + data->txn = txn; + + TSVConn connp = TSTransformCreate(body_transform, txn); + TSContDataSet(connp, data); + TSHttpTxnHookAdd(txn, TS_HTTP_RESPONSE_TRANSFORM_HOOK, connp); + + // store data pointer in txnarg to use in global cont XInjectResponseHeaders + TSHttpTxnArgSet(txn, BodyBuilderArgIndex, data); + + // create a self-cleanup on close + auto cleanupBodyBuilder = [](TSCont /* contp */, TSEvent event, void *edata) -> int { + TSHttpTxn txn = (TSHttpTxn)edata; + BodyBuilder *data = static_cast(TSHttpTxnArgGet(txn, BodyBuilderArgIndex)); + delete data; return TS_EVENT_NONE; }; - TSHttpTxnHookAdd(txn, TS_HTTP_READ_RESPONSE_HDR_HOOK, TSContCreate(read_resp_dump, nullptr)); + TSHttpTxnHookAdd(txn, TS_HTTP_TXN_CLOSE_HOOK, TSContCreate(cleanupBodyBuilder, nullptr)); + + // disable writing to cache because we are injecting data into the body. + TSHttpTxnReqCacheableSet(txn, 0); + TSHttpTxnRespCacheableSet(txn, 0); + TSHttpTxnServerRespNoStoreSet(txn, 1); + TSHttpTxnTransformedRespCache(txn, 0); + TSHttpTxnUntransformedRespCache(txn, 0); } else if (isFwdFieldValue(std::string_view(value, vsize), fwdCnt)) { if (fwdCnt > 0) { @@ -668,7 +635,10 @@ TSPluginInit(int argc, const char *argv[]) // Setup the global hook TSReleaseAssert(TSHttpTxnArgIndexReserve("xdebug", "xdebug header requests", &XArgIndex) == TS_SUCCESS); + TSReleaseAssert(TSHttpTxnArgIndexReserve("bodyTransform", "BodyBuilder*", &XArgIndex) == TS_SUCCESS); TSReleaseAssert(XInjectHeadersCont = TSContCreate(XInjectResponseHeaders, nullptr)); TSReleaseAssert(XDeleteDebugHdrCont = TSContCreate(XDeleteDebugHdr, nullptr)); TSHttpHookAdd(TS_HTTP_READ_REQUEST_HDR_HOOK, TSContCreate(XScanRequestHeaders, nullptr)); + + gethostname(Hostname, 1024); } diff --git a/plugins/xdebug/xdebug_headers.cc b/plugins/xdebug/xdebug_headers.cc new file mode 100644 index 00000000000..aef6871146a --- /dev/null +++ b/plugins/xdebug/xdebug_headers.cc @@ -0,0 +1,159 @@ +/** @file + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#define DEBUG_TAG_LOG_HEADERS "xdebug.headers" + +std::string_view +escape_char_for_json(char const &c, bool &parsing_key) +{ + switch (c) { + case '\'': + return {"\\\'"}; + case '"': + return {"\\\""}; + case '\\': + return {"\\\\"}; + case '\b': + return {"\\b"}; + case '\f': + return {"\\f"}; + case '\t': + return {"\\t"}; + + // Special header reformatting + case '\r': + return {""}; + case '\n': + parsing_key = true; + return {"',\r\n\t'"}; // replace new line with pair delimiter + case ':': + if (parsing_key) { + return {"' : "}; // replace colon after key with quote + colon + } + return {":"}; + case ' ': + if (parsing_key) { + parsing_key = false; + return {"'"}; // replace first space after the key to be a quote + } + return {" "}; + default: + return {&c, 1}; + } +} + +/////////////////////////////////////////////////////////////////////////// +// Dump a header on stderr, useful together with TSDebug(). +void +print_headers(TSHttpTxn txn, TSMBuffer bufp, TSMLoc hdr_loc, std::stringstream &ss) +{ + TSIOBuffer output_buffer; + TSIOBufferReader reader; + TSIOBufferBlock block; + const char *block_start; + int64_t block_avail; + bool parsing_key = true; + size_t print_rewind = ss.str().length(); + output_buffer = TSIOBufferCreate(); + reader = TSIOBufferReaderAlloc(output_buffer); + + ss << "\t'"; + /* This will print just MIMEFields and not the http request line */ + TSMimeHdrPrint(bufp, hdr_loc, output_buffer); + + /* We need to loop over all the buffer blocks, there can be more than 1 */ + block = TSIOBufferReaderStart(reader); + do { + block_start = TSIOBufferBlockReadStart(block, reader, &block_avail); + for (const char *c = block_start; c < block_start + block_avail; ++c) { + bool was_parsing_key = parsing_key; + ss << escape_char_for_json(*c, parsing_key); + if (parsing_key && !was_parsing_key) { + print_rewind = ss.str().length() - 1; + } + } + TSIOBufferReaderConsume(reader, block_avail); + block = TSIOBufferReaderStart(reader); + } while (block && block_avail != 0); + + ss.seekp(print_rewind); + + /* Free up the TSIOBuffer that we used to print out the header */ + TSIOBufferReaderFree(reader); + TSIOBufferDestroy(output_buffer); + + TSDebug(DEBUG_TAG_LOG_HEADERS, "%s", ss.str().c_str()); +} + +void +log_headers(TSHttpTxn txn, TSMBuffer bufp, TSMLoc hdr_loc, char const *type_msg) +{ + if (TSIsDebugTagSet(DEBUG_TAG_LOG_HEADERS)) { + std::stringstream output; + print_headers(txn, bufp, hdr_loc, output); + TSDebug(DEBUG_TAG_LOG_HEADERS, "\n=============\n %s headers are... \n %s", type_msg, output.str().c_str()); + } +} + +void +print_request_headers(TSHttpTxn txn, std::stringstream &output) +{ + TSMBuffer buf_c, buf_s; + TSMLoc hdr_loc; + if (TSHttpTxnClientReqGet(txn, &buf_c, &hdr_loc) == TS_SUCCESS) { + output << "{'type':'request', 'side':'client', 'headers': {\n"; + print_headers(txn, buf_c, hdr_loc, output); + output << "}}"; + TSHandleMLocRelease(buf_c, TS_NULL_MLOC, hdr_loc); + } + if (TSHttpTxnServerReqGet(txn, &buf_s, &hdr_loc) == TS_SUCCESS) { + output << ",{'type':'request', 'side':'server', 'headers': {\n"; + print_headers(txn, buf_s, hdr_loc, output); + output << "}}"; + TSHandleMLocRelease(buf_s, TS_NULL_MLOC, hdr_loc); + } +} + +void +print_response_headers(TSHttpTxn txn, std::stringstream &output) +{ + TSMBuffer buf_c, buf_s; + TSMLoc hdr_loc; + if (TSHttpTxnServerRespGet(txn, &buf_s, &hdr_loc) == TS_SUCCESS) { + output << "{'type':'response', 'side':'server', 'headers': {\n"; + print_headers(txn, buf_s, hdr_loc, output); + output << "}},"; + TSHandleMLocRelease(buf_s, TS_NULL_MLOC, hdr_loc); + } + if (TSHttpTxnClientRespGet(txn, &buf_c, &hdr_loc) == TS_SUCCESS) { + output << "{'type':'response', 'side':'client', 'headers': {\n"; + print_headers(txn, buf_c, hdr_loc, output); + output << "}}"; + TSHandleMLocRelease(buf_c, TS_NULL_MLOC, hdr_loc); + } +} diff --git a/plugins/xdebug/xdebug_transforms.cc b/plugins/xdebug/xdebug_transforms.cc new file mode 100644 index 00000000000..a5bc75a0a73 --- /dev/null +++ b/plugins/xdebug/xdebug_transforms.cc @@ -0,0 +1,158 @@ +/** @file + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#include +#include +#include +#include +#include + +#include "ts/ts.h" + +static const std::string_view MultipartBoundary{"\r\n--- ATS xDebug Probe Injection Boundary ---\r\n\r\n"}; + +struct BodyBuilder { + TSVIO output_vio = nullptr; + TSIOBuffer output_buffer = nullptr; + TSIOBufferReader output_reader = nullptr; + bool wrote_prebody = false; + bool wrote_body = false; + bool hdr_ready = false; + std::atomic_flag wrote_postbody; + + int64_t nbytes = 0; + TSHttpTxn txn = nullptr; +}; + +static char Hostname[1024]; + +static std::string +getPreBody(TSHttpTxn txn) +{ + std::stringstream output; + output << "{'xDebugProbeAt' : '" << Hostname << "'\n 'captured':["; + print_request_headers(txn, output); + output << "\n ]\n}"; + output << MultipartBoundary; + return output.str(); +} + +static std::string +getPostBody(TSHttpTxn txn) +{ + std::stringstream output; + output << MultipartBoundary; + output << "{'xDebugProbeAt' : '" << Hostname << "'\n 'captured':["; + print_response_headers(txn, output); + output << "\n ]\n}"; + return output.str(); +} + +static void +writePostBody(BodyBuilder *data) +{ + if (data->wrote_body && data->hdr_ready && !data->wrote_postbody.test_and_set()) { + TSDebug("xdebug_transform", "body_transform(): Writing postbody headers..."); + std::string postbody = getPostBody(data->txn); + TSIOBufferWrite(data->output_buffer, postbody.data(), postbody.length()); + data->nbytes += postbody.length(); + TSVIONBytesSet(data->output_vio, data->nbytes); + TSVIOReenable(data->output_vio); + } +} + +static int +body_transform(TSCont contp, TSEvent event, void *edata) +{ + BodyBuilder *data = static_cast(TSContDataGet(contp)); + if (!data) { + TSContDestroy(contp); + return TS_ERROR; + } + if (TSVConnClosedGet(contp)) { + // write connection destroyed. cleanup. + delete data; + TSContDestroy(contp); + return 0; + } + + TSVIO src_vio = TSVConnWriteVIOGet(contp); + + switch (event) { + case TS_EVENT_ERROR: { + // Notify input vio of this error event + TSContCall(TSVIOContGet(src_vio), TS_EVENT_ERROR, src_vio); + return 0; + } + case TS_EVENT_VCONN_WRITE_COMPLETE: { + TSVConnShutdown(TSTransformOutputVConnGet(contp), 0, 1); + return 0; + } + case TS_EVENT_VCONN_WRITE_READY: + TSDebug("xdebug_transform", "body_transform(): Event is TS_EVENT_VCONN_WRITE_READY"); + // fall through + default: + if (!data->output_buffer) { + data->output_buffer = TSIOBufferCreate(); + data->output_reader = TSIOBufferReaderAlloc(data->output_buffer); + data->output_vio = TSVConnWrite(TSTransformOutputVConnGet(contp), contp, data->output_reader, INT64_MAX); + } + + if (data->wrote_prebody == false) { + TSDebug("xdebug_transform", "body_transform(): Writing prebody headers..."); + std::string prebody = getPreBody(data->txn); + TSIOBufferWrite(data->output_buffer, prebody.data(), prebody.length()); // write prebody + data->wrote_prebody = true; + data->nbytes += prebody.length(); + } + + TSIOBuffer src_buf = TSVIOBufferGet(src_vio); + + if (!src_buf) { + // upstream continuation shuts down write operation. + data->wrote_body = true; + writePostBody(data); + return 0; + } + + int64_t towrite = TSVIONTodoGet(src_vio); + TSDebug("xdebug_transform", "body_transform(): %" PRId64 " bytes of body is expected", towrite); + int64_t avail = TSIOBufferReaderAvail(TSVIOReaderGet(src_vio)); + towrite = towrite > avail ? avail : towrite; + if (towrite > 0) { + TSIOBufferCopy(TSVIOBufferGet(data->output_vio), TSVIOReaderGet(src_vio), towrite, 0); + TSIOBufferReaderConsume(TSVIOReaderGet(src_vio), towrite); + TSVIONDoneSet(src_vio, TSVIONDoneGet(src_vio) + towrite); + TSDebug("xdebug_transform", "body_transform(): writing %" PRId64 " bytes of body", towrite); + } + + if (TSVIONTodoGet(src_vio) > 0) { + TSVIOReenable(data->output_vio); + TSContCall(TSVIOContGet(src_vio), TS_EVENT_VCONN_WRITE_READY, src_vio); + } else { + // End of src vio + // Write post body content and update output VIO + data->wrote_body = true; + data->nbytes += TSVIONDoneGet(src_vio); + writePostBody(data); + TSContCall(TSVIOContGet(src_vio), TS_EVENT_VCONN_WRITE_COMPLETE, src_vio); + } + } + return 0; +} diff --git a/proxy/CacheControl.cc b/proxy/CacheControl.cc index 5d5da5efde3..423584e96b5 100644 --- a/proxy/CacheControl.cc +++ b/proxy/CacheControl.cc @@ -144,17 +144,21 @@ initCacheControl() // // Called when the cache.conf file changes. Since it called // infrequently, we do the load of new file as blocking I/O and -// lock aquire is also blocking +// lock acquire is also blocking // void reloadCacheControl() { + Note("cache.config loading ..."); + CC_table *newTable; Debug("cache_control", "cache.config updated, reloading"); eventProcessor.schedule_in(new CC_FreerContinuation(CacheControlTable), CACHE_CONTROL_TIMEOUT, ET_CACHE); newTable = new CC_table("proxy.config.cache.control.filename", modulePrefix, &http_dest_tags); ink_atomic_swap(&CacheControlTable, newTable); + + Note("cache.config finished loading"); } void @@ -368,9 +372,12 @@ CacheControlRecord::UpdateMatch(CacheControlResult *result, RequestData *rdata) break; case CC_NEVER_CACHE: if (this->CheckForMatch(h_rdata, result->never_line) == true) { - result->never_cache = true; - result->never_line = this->line_num; - match = true; + // ttl-in-cache overrides never-cache + if (result->ttl_line == -1) { + result->never_cache = true; + result->never_line = this->line_num; + match = true; + } } break; case CC_STANDARD_CACHE: diff --git a/proxy/CacheControl.h b/proxy/CacheControl.h index 1900a9594ec..3c088b6cabf 100644 --- a/proxy/CacheControl.h +++ b/proxy/CacheControl.h @@ -72,11 +72,11 @@ class CacheControlResult int revalidate_after; int pin_in_cache_for; int ttl_in_cache; - bool never_cache; - bool ignore_client_no_cache; - bool ignore_server_no_cache; - bool ignore_client_cc_max_age; - int cache_responses_to_cookies; ///< Override for caching cookied responses. + bool never_cache = false; + bool ignore_client_no_cache = false; + bool ignore_server_no_cache = false; + bool ignore_client_cc_max_age = true; + int cache_responses_to_cookies = -1; ///< Override for caching cookied responses. // Data for internal use only // @@ -86,29 +86,17 @@ class CacheControlResult // be overriden by something that appeared // earlier in the the config file // - int reval_line; - int never_line; - int pin_line; - int ttl_line; - int ignore_client_line; - int ignore_server_line; + int reval_line = -1; + int never_line = -1; + int pin_line = -1; + int ttl_line = -1; + int ignore_client_line = -1; + int ignore_server_line = -1; }; inline CacheControlResult::CacheControlResult() - : revalidate_after(CC_UNSET_TIME), - pin_in_cache_for(CC_UNSET_TIME), - ttl_in_cache(CC_UNSET_TIME), - never_cache(false), - ignore_client_no_cache(false), - ignore_server_no_cache(false), - ignore_client_cc_max_age(true), - cache_responses_to_cookies(-1), // do not change value - reval_line(-1), - never_line(-1), - pin_line(-1), - ttl_line(-1), - ignore_client_line(-1), - ignore_server_line(-1) + : revalidate_after(CC_UNSET_TIME), pin_in_cache_for(CC_UNSET_TIME), ttl_in_cache(CC_UNSET_TIME) + { } @@ -116,17 +104,15 @@ class CacheControlRecord : public ControlBase { public: CacheControlRecord(); - CacheControlType directive; - int time_arg; - int cache_responses_to_cookies; + CacheControlType directive = CC_INVALID; + int time_arg = 0; + int cache_responses_to_cookies = -1; Result Init(matcher_line *line_info); inkcoreapi void UpdateMatch(CacheControlResult *result, RequestData *rdata); void Print(); }; -inline CacheControlRecord::CacheControlRecord() : ControlBase(), directive(CC_INVALID), time_arg(0), cache_responses_to_cookies(-1) -{ -} +inline CacheControlRecord::CacheControlRecord() : ControlBase() {} // // API to outside world diff --git a/proxy/ControlBase.h b/proxy/ControlBase.h index 844276f8285..76ffb25861d 100644 --- a/proxy/ControlBase.h +++ b/proxy/ControlBase.h @@ -77,7 +77,7 @@ class ControlBase bool CheckModifiers(HttpRequestData *request_data); bool CheckForMatch(HttpRequestData *request_data, int last_number); void Print(); - int line_num; + int line_num = 0; Modifier *findModOfType(Modifier::Type t) const; protected: @@ -97,7 +97,7 @@ class ControlBase void clear(); }; -inline ControlBase::ControlBase() : line_num(0) {} +inline ControlBase::ControlBase() {} inline bool ControlBase::CheckForMatch(HttpRequestData *request_data, int last_number) diff --git a/proxy/ControlMatcher.h b/proxy/ControlMatcher.h index 16dc8ecb600..50cb2916e6e 100644 --- a/proxy/ControlMatcher.h +++ b/proxy/ControlMatcher.h @@ -86,7 +86,6 @@ #pragma once -#include "tscore/DynArray.h" #include "tscore/IpMap.h" #include "tscore/Result.h" #include "tscore/MatcherUtils.h" @@ -139,31 +138,23 @@ class HttpRequestData : public RequestData inkcoreapi sockaddr const *get_client_ip() override; HttpRequestData() - : hdr(nullptr), - hostname_str(nullptr), - api_info(nullptr), - xact_start(0), - incoming_port(0), - tag(nullptr), - internal_txn(false), - cache_info_lookup_url(nullptr), - cache_info_parent_selection_url(nullptr) + { ink_zero(src_ip); ink_zero(dest_ip); } - HTTPHdr *hdr; - char *hostname_str; - HttpApiInfo *api_info; - time_t xact_start; + HTTPHdr *hdr = nullptr; + char *hostname_str = nullptr; + HttpApiInfo *api_info = nullptr; + time_t xact_start = 0; IpEndpoint src_ip; IpEndpoint dest_ip; - uint16_t incoming_port; - char *tag; - bool internal_txn; - URL **cache_info_lookup_url; - URL **cache_info_parent_selection_url; + uint16_t incoming_port = 0; + char *tag = nullptr; + bool internal_txn = false; + URL **cache_info_lookup_url = nullptr; + URL **cache_info_parent_selection_url = nullptr; }; // Mixin class for shared info across all templates. This just wraps the diff --git a/proxy/HostStatus.h b/proxy/HostStatus.h index 8257913dfde..f349b903f8b 100644 --- a/proxy/HostStatus.h +++ b/proxy/HostStatus.h @@ -30,7 +30,7 @@ #pragma once -#include +#include #include #include "tscore/ink_rwlock.h" #include "records/P_RecProcess.h" @@ -50,11 +50,12 @@ struct HostStatRec_t { }; struct Reasons { - static constexpr const char *ACTIVE = "active"; - static constexpr const char *LOCAL = "local"; - static constexpr const char *MANUAL = "manual"; + static constexpr const char *ACTIVE = "active"; + static constexpr const char *LOCAL = "local"; + static constexpr const char *MANUAL = "manual"; + static constexpr const char *SELF_DETECT = "self_detect"; - static constexpr const char *reasons[3] = {ACTIVE, LOCAL, MANUAL}; + static constexpr const char *reasons[4] = {ACTIVE, LOCAL, MANUAL, SELF_DETECT}; static bool validReason(const char *reason) @@ -85,6 +86,7 @@ struct HostStatus { void setHostStatus(const char *name, const HostStatus_t status, const unsigned int down_time, const char *reason); HostStatus_t getHostStatus(const char *name); void createHostStat(const char *name); + void loadHostStatusFromStats(); int getHostStatId(const char *name); private: diff --git a/proxy/IPAllow.cc b/proxy/IPAllow.cc index ed208d99b72..8adadb7a3e4 100644 --- a/proxy/IPAllow.cc +++ b/proxy/IPAllow.cc @@ -77,12 +77,14 @@ IpAllow::reconfigure() { self_type *new_table; - Note("ip_allow.config updated, reloading"); + Note("ip_allow.config loading ..."); new_table = new self_type("proxy.config.cache.ip_allow.filename"); new_table->BuildTable(); configid = configProcessor.set(configid, new_table); + + Note("ip_allow.config finished loading"); } IpAllow * @@ -131,10 +133,7 @@ IpAllow::match(sockaddr const *ip, match_key_t key) // End API functions // -IpAllow::IpAllow(const char *config_var) -{ - config_file_path = RecConfigReadConfigPath(config_var); -} +IpAllow::IpAllow(const char *config_var) : config_file_path(RecConfigReadConfigPath(config_var)) {} void IpAllow::PrintMap(IpMap *map) diff --git a/proxy/IPAllow.h b/proxy/IPAllow.h index 2fedf29f87f..112d9774d2d 100644 --- a/proxy/IPAllow.h +++ b/proxy/IPAllow.h @@ -98,8 +98,8 @@ class IpAllow : public ConfigInfo using self_type = ACL; ///< Self reference type. public: ACL() = default; - ACL(const self_type &) = delete; // no copies. - ACL(self_type &&that) noexcept; // move allowed. + ACL(const self_type &) = delete; // no copies. + explicit ACL(self_type &&that) noexcept; // move allowed. ~ACL(); self_type &operator=(const self_type &) = delete; @@ -131,7 +131,7 @@ class IpAllow : public ConfigInfo IpAllow *_config{nullptr}; ///< The backing configuration. }; - IpAllow(const char *config_var); + explicit IpAllow(const char *config_var); void Print(); diff --git a/proxy/InkAPIInternal.h b/proxy/InkAPIInternal.h index c2bd1a86ca4..17ba528defb 100644 --- a/proxy/InkAPIInternal.h +++ b/proxy/InkAPIInternal.h @@ -53,20 +53,13 @@ enum CacheInfoMagic { struct CacheInfo { CryptoHash cache_key; - CacheFragType frag_type; - char *hostname; - int len; - time_t pin_in_cache; - CacheInfoMagic magic; + CacheFragType frag_type = CACHE_FRAG_TYPE_NONE; + char *hostname = nullptr; + int len = 0; + time_t pin_in_cache = 0; + CacheInfoMagic magic = CACHE_INFO_MAGIC_ALIVE; - CacheInfo() - { - frag_type = CACHE_FRAG_TYPE_NONE; - hostname = nullptr; - len = 0; - pin_in_cache = 0; - magic = CACHE_INFO_MAGIC_ALIVE; - } + CacheInfo() {} }; class FileImpl @@ -135,7 +128,6 @@ class APIHooks APIHook *get() const; void clear(); bool is_empty() const; - void invoke(int event, void *data); private: Que(APIHook, m_link) m_hooks; @@ -147,13 +139,6 @@ APIHooks::is_empty() const return nullptr == m_hooks.head; } -inline void -APIHooks::invoke(int event, void *data) -{ - for (APIHook *hook = m_hooks.head; nullptr != hook; hook = hook->next()) - hook->invoke(event, data); -} - /** Container for API hooks for a specific feature. This is an array of hook lists, each identified by a numeric identifier (id). Each array element is a list of all @@ -164,7 +149,7 @@ APIHooks::invoke(int event, void *data) maximum hook ID so the valid ids are 0..(N-1) in the standard C array style. */ template class FeatureAPIHooks { @@ -197,19 +182,19 @@ class FeatureAPIHooks bool has_hooks_for(ID id) const; private: - bool hooks_p; ///< Flag for (not) empty container. + bool hooks_p = false; ///< Flag for (not) empty container. /// The array of hooks lists. APIHooks m_hooks[N]; }; -template FeatureAPIHooks::FeatureAPIHooks() : hooks_p(false) {} +template FeatureAPIHooks::FeatureAPIHooks() {} -template FeatureAPIHooks::~FeatureAPIHooks() +template FeatureAPIHooks::~FeatureAPIHooks() { this->clear(); } -template +template void FeatureAPIHooks::clear() { @@ -219,7 +204,7 @@ FeatureAPIHooks::clear() hooks_p = false; } -template +template void FeatureAPIHooks::prepend(ID id, INKContInternal *cont) { @@ -229,7 +214,7 @@ FeatureAPIHooks::prepend(ID id, INKContInternal *cont) } } -template +template void FeatureAPIHooks::append(ID id, INKContInternal *cont) { @@ -239,14 +224,14 @@ FeatureAPIHooks::append(ID id, INKContInternal *cont) } } -template +template APIHook * FeatureAPIHooks::get(ID id) const { return likely(is_valid(id)) ? m_hooks[id].get() : nullptr; } -template +template void FeatureAPIHooks::invoke(ID id, int event, void *data) { @@ -255,14 +240,14 @@ FeatureAPIHooks::invoke(ID id, int event, void *data) } } -template +template bool FeatureAPIHooks::has_hooks() const { return hooks_p; } -template +template bool FeatureAPIHooks::is_valid(ID id) { @@ -273,21 +258,26 @@ class HttpAPIHooks : public FeatureAPIHooks { }; -typedef enum { - TS_SSL_INTERNAL_FIRST_HOOK, - TS_VCONN_START_INTERNAL_HOOK = TS_SSL_INTERNAL_FIRST_HOOK, - TS_VCONN_CLOSE_INTERNAL_HOOK, - TS_SSL_CERT_INTERNAL_HOOK, - TS_SSL_SERVERNAME_INTERNAL_HOOK, - TS_SSL_VERIFY_SERVER_INTERNAL_HOOK, - TS_SSL_VERIFY_CLIENT_INTERNAL_HOOK, - TS_SSL_SESSION_INTERNAL_HOOK, - TS_VCONN_OUTBOUND_START_INTERNAL_HOOK, - TS_VCONN_OUTBOUND_CLOSE_INTERNAL_HOOK, - TS_SSL_INTERNAL_LAST_HOOK -} TSSslHookInternalID; - -class SslAPIHooks : public FeatureAPIHooks +class TSSslHookInternalID +{ +public: + explicit constexpr TSSslHookInternalID(TSHttpHookID id) : _id(id - TS_SSL_FIRST_HOOK) {} + + constexpr operator int() const { return _id; } + + static const int NUM = TS_SSL_LAST_HOOK - TS_SSL_FIRST_HOOK + 1; + + constexpr bool + is_in_bounds() const + { + return (_id >= 0) && (_id < NUM); + } + +private: + const int _id; +}; + +class SslAPIHooks : public FeatureAPIHooks { }; @@ -298,7 +288,7 @@ class LifecycleAPIHooks : public FeatureAPIHooksmutex.get()), m_cont(contp) + explicit ConfigUpdateCallback(INKContInternal *contp) : Continuation(contp->mutex.get()), m_cont(contp) { SET_HANDLER(&ConfigUpdateCallback::event_handler); } diff --git a/proxy/Makefile.am b/proxy/Makefile.am index 0bd93d99822..1965837408d 100644 --- a/proxy/Makefile.am +++ b/proxy/Makefile.am @@ -62,7 +62,7 @@ libproxy_a_SOURCES = \ ProxyClientSession.cc \ ProxyClientSession.h \ ProxyClientTransaction.cc \ - ProxyClientTransaction \ + ProxyClientTransaction.h \ ReverseProxy.cc \ ReverseProxy.h \ StatPages.cc \ @@ -164,3 +164,6 @@ install-data-hook: chown -R $(pkgsysuser):$(pkgsysgroup) $(DESTDIR)$(pkgsysconfdir) $(DESTDIR)$(pkgdatadir);\ fi -echo " $(PACKAGE_VERSION)" > $(DESTDIR)$(pkgsysconfdir)/trafficserver-release + +clang-tidy-local: $(DIST_SOURCES) + $(CXX_Clang_Tidy) diff --git a/proxy/ParentSelection.cc b/proxy/ParentSelection.cc index df3f368ee95..6d3d0e55992 100644 --- a/proxy/ParentSelection.cc +++ b/proxy/ParentSelection.cc @@ -279,6 +279,8 @@ ParentConfig::startup() void ParentConfig::reconfigure() { + Note("parent.config loading ..."); + ParentConfigParams *params = nullptr; // Allocate parent table @@ -292,6 +294,8 @@ ParentConfig::reconfigure() if (is_debug_tag_set("parent_config")) { ParentConfig::print(); } + + Note("parent.config finished loading"); } // void ParentConfig::print @@ -368,8 +372,7 @@ ParentRecord::PreProcessParents(const char *val, const int line_num, char *buf, continue; } else { Debug("parent_select", "token: %s, matches this machine. Marking down self from parent list at line %d", fqdn, line_num); - hs.createHostStat(fqdn); - hs.setHostStatus(fqdn, HostStatus_t::HOST_STATUS_DOWN, 0, Reasons::MANUAL); + hs.setHostStatus(fqdn, HostStatus_t::HOST_STATUS_DOWN, 0, Reasons::SELF_DETECT); } } } else { @@ -381,8 +384,7 @@ ParentRecord::PreProcessParents(const char *val, const int line_num, char *buf, } else { Debug("parent_select", "token: %s, matches this machine. Marking down self from parent list at line %d", token, line_num); - hs.createHostStat(token); - hs.setHostStatus(token, HostStatus_t::HOST_STATUS_DOWN, 0, Reasons::MANUAL); + hs.setHostStatus(token, HostStatus_t::HOST_STATUS_DOWN, 0, Reasons::SELF_DETECT); } } } @@ -413,7 +415,7 @@ ParentRecord::ProcessParents(char *val, bool isPrimary) int numTok = 0; const char *current = nullptr; int port = 0; - char *tmp = nullptr, *tmp2 = nullptr; + char *tmp = nullptr, *tmp2 = nullptr, *tmp3 = nullptr; const char *errPtr = nullptr; float weight = 1.0; @@ -466,23 +468,27 @@ ParentRecord::ProcessParents(char *val, bool isPrimary) } } + tmp3 = (char *)strchr(current, '&'); + // Make sure that is no garbage beyond the parent - // port or weight - char *scan; - if (tmp2) { - scan = tmp2 + 1; - } else { - scan = tmp + 1; - } - for (; *scan != '\0' && (ParseRules::is_digit(*scan) || *scan == '.'); scan++) { - ; - } - for (; *scan != '\0' && ParseRules::is_wslfcr(*scan); scan++) { - ; - } - if (*scan != '\0') { - errPtr = "Garbage trailing entry or invalid separator"; - goto MERROR; + // port or weight + if (!tmp3) { + char *scan; + if (tmp2) { + scan = tmp2 + 1; + } else { + scan = tmp + 1; + } + for (; *scan != '\0' && (ParseRules::is_digit(*scan) || *scan == '.'); scan++) { + ; + } + for (; *scan != '\0' && ParseRules::is_wslfcr(*scan); scan++) { + ; + } + if (*scan != '\0') { + errPtr = "Garbage trailing entry or invalid separator"; + goto MERROR; + } } // Check to make sure that the string will fit in the // pRecord @@ -490,7 +496,7 @@ ParentRecord::ProcessParents(char *val, bool isPrimary) errPtr = "Parent hostname is too long"; goto MERROR; } else if (tmp - current == 0) { - errPtr = "Parent string is emtpy"; + errPtr = "Parent string is empty"; goto MERROR; } // Update the pRecords @@ -505,7 +511,13 @@ ParentRecord::ProcessParents(char *val, bool isPrimary) this->parents[i].name = this->parents[i].hostname; this->parents[i].available = true; this->parents[i].weight = weight; - hs.createHostStat(this->parents[i].hostname); + if (tmp3) { + memcpy(this->parents[i].hash_string, tmp3 + 1, strlen(tmp3)); + this->parents[i].name = this->parents[i].hash_string; + } + if (hs.getHostStatus(this->parents[i].hostname) == HostStatus_t::HOST_STATUS_INIT) { + hs.setHostStatus(this->parents[i].hostname, HOST_STATUS_UP, 0, Reasons::MANUAL); + } } else { memcpy(this->secondary_parents[i].hostname, current, tmp - current); this->secondary_parents[i].hostname[tmp - current] = '\0'; @@ -517,8 +529,15 @@ ParentRecord::ProcessParents(char *val, bool isPrimary) this->secondary_parents[i].name = this->secondary_parents[i].hostname; this->secondary_parents[i].available = true; this->secondary_parents[i].weight = weight; - hs.createHostStat(this->secondary_parents[i].hostname); + if (tmp3) { + memcpy(this->secondary_parents[i].hash_string, tmp3 + 1, strlen(tmp3)); + this->secondary_parents[i].name = this->secondary_parents[i].hash_string; + } + if (hs.getHostStatus(this->secondary_parents[i].hostname) == HostStatus_t::HOST_STATUS_INIT) { + hs.setHostStatus(this->secondary_parents[i].hostname, HOST_STATUS_UP, 0, Reasons::MANUAL); + } } + tmp3 = nullptr; } if (isPrimary) { @@ -803,7 +822,7 @@ ParentRecord::Print() { printf("\t\t"); for (int i = 0; i < num_parents; i++) { - printf(" %s:%d ", parents[i].hostname, parents[i].port); + printf(" %s:%d|%f&%s ", parents[i].hostname, parents[i].port, parents[i].weight, parents[i].name); } printf(" direct=%s\n", (go_direct == true) ? "true" : "false"); printf(" parent_is_proxy=%s\n", (parent_is_proxy == true) ? "true" : "false"); @@ -882,6 +901,8 @@ setup_socks_servers(ParentRecord *rec_arr, int len) void SocksServerConfig::reconfigure() { + Note("socks.config loading ..."); + char *default_val = nullptr; int retry_time = 30; int fail_threshold; @@ -923,6 +944,8 @@ SocksServerConfig::reconfigure() if (is_debug_tag_set("parent_config")) { SocksServerConfig::print(); } + + Note("socks.config finished loading"); } void @@ -1719,6 +1742,52 @@ EXCLUSIVE_REGRESSION_TEST(PARENTSELECTION)(RegressionTest * /* t ATS_UNUSED */, sleep(1); RE(verify(result, PARENT_FAIL, nullptr, 80), 208); + // Tests 209 through 211 test that host selection is based upon the hash_string + + // Test 209 + // fuzzy { curly larry, moe } fluffy + tbl[0] = '\0'; + ST(209); + T("dest_domain=stooges.net parent=curly:80|1.0&myhash;joe:80|1.0&hishash;larry:80|1.0&ourhash " + "round_robin=consistent_hash go_direct=false\n"); + REBUILD; + REINIT; + br(request, "i.am.stooges.net"); + FP; + RE(verify(result, PARENT_SPECIFIED, "larry", 80), 209); + + // Test 210 + // fuzzy { curly larry, moe } fluffy + tbl[0] = '\0'; + ST(210); + T("dest_domain=stooges.net parent=curly:80|1.0&ourhash;joe:80|1.0&hishash;larry:80|1.0&myhash " + "round_robin=consistent_hash go_direct=false\n"); + REBUILD; + REINIT; + br(request, "i.am.stooges.net"); + FP; + RE(verify(result, PARENT_SPECIFIED, "curly", 80), 210); + + // Test 211 + // fuzzy { curly larry, moe } fluffy + tbl[0] = '\0'; + ST(211); + T("dest_domain=stooges.net parent=curly:80|1.0&ourhash;joe:80|1.0&hishash;larry:80|1.0&myhash " + "secondary_parent=carol:80|1.0&ourhash;betty:80|1.0&hishash;donna:80|1.0&myhash " + "round_robin=consistent_hash go_direct=false\n"); + REBUILD; + REINIT; + _st.setHostStatus("curly", HOST_STATUS_DOWN, 0, Reasons::MANUAL); + br(request, "i.am.stooges.net"); + FP; + RE(verify(result, PARENT_SPECIFIED, "carol", 80), 211); + + // cleanup, allow changes to be persisted to records.snap + // so that subsequent test runs do not fail unexpectedly. + _st.setHostStatus("curly", HOST_STATUS_UP, 0, Reasons::MANUAL); + _st.setHostStatus("fuzzy", HOST_STATUS_UP, 0, Reasons::MANUAL); + sleep(2); + delete request; delete result; delete params; diff --git a/proxy/ParentSelection.h b/proxy/ParentSelection.h index 3b8fbd22b38..c0f3802df8c 100644 --- a/proxy/ParentSelection.h +++ b/proxy/ParentSelection.h @@ -102,6 +102,7 @@ struct pRecord : ATSConsistentHashNode { const char *scheme; // for which parent matches (if any) int idx; float weight; + char hash_string[MAXDNAME + 1]; }; typedef ControlMatcher P_table; diff --git a/proxy/Plugin.cc b/proxy/Plugin.cc index 174c6d07390..abecdc8ed69 100644 --- a/proxy/Plugin.cc +++ b/proxy/Plugin.cc @@ -51,10 +51,7 @@ using init_func_t = void (*)(int, char **); DLL plugin_reg_list; PluginRegInfo *plugin_reg_current = nullptr; -PluginRegInfo::PluginRegInfo() - : plugin_registered(false), plugin_path(nullptr), plugin_name(nullptr), vendor_name(nullptr), support_email(nullptr), dlh(nullptr) -{ -} +PluginRegInfo::PluginRegInfo() {} PluginRegInfo::~PluginRegInfo() { @@ -234,10 +231,11 @@ plugin_init(bool validateOnly) INIT_ONCE = false; } + Note("plugin.config loading ..."); path = RecConfigReadConfigPath(nullptr, "plugin.config"); fd = open(path, O_RDONLY); if (fd < 0) { - Warning("unable to open plugin config file '%s': %d, %s", (const char *)path, errno, strerror(errno)); + Warning("plugin.config failed to load: %d, %s", errno, strerror(errno)); return false; } @@ -312,5 +310,10 @@ plugin_init(bool validateOnly) } close(fd); + if (retVal) { + Note("plugin.config finished loading"); + } else { + Error("plugin.config failed to load"); + } return retVal; } diff --git a/proxy/Plugin.h b/proxy/Plugin.h index 349a1482fa7..09c3de69fca 100644 --- a/proxy/Plugin.h +++ b/proxy/Plugin.h @@ -29,14 +29,14 @@ struct PluginRegInfo { PluginRegInfo(); ~PluginRegInfo(); - bool plugin_registered; - char *plugin_path; + bool plugin_registered = false; + char *plugin_path = nullptr; - char *plugin_name; - char *vendor_name; - char *support_email; + char *plugin_name = nullptr; + char *vendor_name = nullptr; + char *support_email = nullptr; - void *dlh; + void *dlh = nullptr; LINK(PluginRegInfo, link); }; diff --git a/proxy/PluginVC.cc b/proxy/PluginVC.cc index 85dd7fd9ccc..569ea851127 100644 --- a/proxy/PluginVC.cc +++ b/proxy/PluginVC.cc @@ -310,6 +310,7 @@ PluginVC::reenable(VIO *vio) { ink_assert(!closed); ink_assert(magic == PLUGIN_VC_MAGIC_ALIVE); + ink_assert(vio->mutex->thread_holding == this_ethread()); Ptr sm_mutex = vio->mutex; SCOPED_MUTEX_LOCK(lock, sm_mutex, this_ethread()); @@ -332,6 +333,7 @@ PluginVC::reenable_re(VIO *vio) { ink_assert(!closed); ink_assert(magic == PLUGIN_VC_MAGIC_ALIVE); + ink_assert(vio->mutex->thread_holding == this_ethread()); Debug("pvc", "[%u] %s: reenable_re %s", core_obj->id, PVC_TYPE, (vio->op == VIO::WRITE) ? "Write" : "Read"); @@ -473,25 +475,13 @@ PluginVC::process_write_side(bool other_side_call) MIOBuffer *core_buffer = (vc_type == PLUGIN_VC_ACTIVE) ? core_obj->a_to_p_buffer : core_obj->p_to_a_buffer; + Debug("pvc", "[%u] %s: process_write_side", core_obj->id, PVC_TYPE); need_write_process = false; + // Check write_state if (write_state.vio.op != VIO::WRITE || closed || write_state.shutdown) { return; } - // Acquire the lock of the write side continuation - EThread *my_ethread = mutex->thread_holding; - ink_assert(my_ethread != nullptr); - MUTEX_TRY_LOCK(lock, write_state.vio.mutex, my_ethread); - if (!lock.is_locked()) { - Debug("pvc_event", "[%u] %s: process_write_side lock miss, retrying", core_obj->id, PVC_TYPE); - - need_write_process = true; - setup_event_cb(PVC_LOCK_RETRY_TIME, &core_lock_retry_event); - return; - } - - Debug("pvc", "[%u] %s: process_write_side", core_obj->id, PVC_TYPE); - need_write_process = false; // Check the state of our write buffer as well as ntodo int64_t ntodo = write_state.vio.ntodo(); @@ -552,6 +542,30 @@ PluginVC::process_write_side(bool other_side_call) // Wake up the read side on the other side to process these bytes if (!other_side->closed) { if (!other_side_call) { + /* To clear the `need_read_process`, the mutexes must be obtained: + * + * - PluginVC::mutex + * - PluginVC::read_state.vio.mutex + * + */ + if (other_side->read_state.vio.op != VIO::READ || other_side->closed || other_side->read_state.shutdown) { + // Just return, no touch on `other_side->need_read_process`. + return; + } + // Acquire the lock of the read side continuation + EThread *my_ethread = mutex->thread_holding; + ink_assert(my_ethread != nullptr); + MUTEX_TRY_LOCK(lock, other_side->read_state.vio.mutex, my_ethread); + if (!lock.is_locked()) { + Debug("pvc_event", "[%u] %s: process_read_side from other side lock miss, retrying", other_side->core_obj->id, + ((other_side->vc_type == PLUGIN_VC_ACTIVE) ? "Active" : "Passive")); + + // set need_read_process to enforce the read processing + other_side->need_read_process = true; + other_side->setup_event_cb(PVC_LOCK_RETRY_TIME, &other_side->core_lock_retry_event); + return; + } + other_side->process_read_side(true); } else { other_side->read_state.vio.reenable(); @@ -587,28 +601,11 @@ PluginVC::process_read_side(bool other_side_call) core_reader = core_obj->a_to_p_reader; } - need_read_process = false; - - if (read_state.vio.op != VIO::READ || closed) { - return; - } - // Acquire the lock of the read side continuation - EThread *my_ethread = mutex->thread_holding; - ink_assert(my_ethread != nullptr); - MUTEX_TRY_LOCK(lock, read_state.vio.mutex, my_ethread); - if (!lock.is_locked()) { - Debug("pvc_event", "[%u] %s: process_read_side lock miss, retrying", core_obj->id, PVC_TYPE); - - need_read_process = true; - setup_event_cb(PVC_LOCK_RETRY_TIME, &core_lock_retry_event); - return; - } - Debug("pvc", "[%u] %s: process_read_side", core_obj->id, PVC_TYPE); need_read_process = false; - // Check read_state.shutdown after the lock has been obtained. - if (read_state.shutdown) { + // Check read_state + if (read_state.vio.op != VIO::READ || closed || read_state.shutdown) { return; } @@ -668,6 +665,30 @@ PluginVC::process_read_side(bool other_side_call) // intermediate buffer if (!other_side->closed) { if (!other_side_call) { + /* To clear the `need_write_process`, the mutexes must be obtained: + * + * - PluginVC::mutex + * - PluginVC::write_state.vio.mutex + * + */ + if (other_side->write_state.vio.op != VIO::WRITE || other_side->closed || other_side->write_state.shutdown) { + // Just return, no touch on `other_side->need_write_process`. + return; + } + // Acquire the lock of the write side continuation + EThread *my_ethread = mutex->thread_holding; + ink_assert(my_ethread != nullptr); + MUTEX_TRY_LOCK(lock, other_side->write_state.vio.mutex, my_ethread); + if (!lock.is_locked()) { + Debug("pvc_event", "[%u] %s: process_write_side from other side lock miss, retrying", other_side->core_obj->id, + ((other_side->vc_type == PLUGIN_VC_ACTIVE) ? "Active" : "Passive")); + + // set need_write_process to enforce the write processing + other_side->need_write_process = true; + other_side->setup_event_cb(PVC_LOCK_RETRY_TIME, &other_side->core_lock_retry_event); + return; + } + other_side->process_write_side(true); } else { other_side->write_state.vio.reenable(); @@ -936,6 +957,18 @@ PluginVC::set_remote_addr() } } +void +PluginVC::set_remote_addr(const sockaddr * /* new_sa ATS_UNUSED */) +{ + return; +} + +void +PluginVC::set_mptcp_state() +{ + return; +} + int PluginVC::set_tcp_init_cwnd(int /* init_cwnd ATS_UNUSED */) { @@ -1250,15 +1283,15 @@ class PVCTestDriver : public NetTestDriver ~PVCTestDriver() override; void start_tests(RegressionTest *r_arg, int *pstatus_arg); - void run_next_test(); + bool run_next_test(); int main_handler(int event, void *data); private: - unsigned i; - unsigned completions_received; + unsigned i = 0; + unsigned completions_received = 0; }; -PVCTestDriver::PVCTestDriver() : NetTestDriver(), i(0), completions_received(0) {} +PVCTestDriver::PVCTestDriver() : NetTestDriver() {} PVCTestDriver::~PVCTestDriver() { @@ -1274,12 +1307,12 @@ PVCTestDriver::start_tests(RegressionTest *r_arg, int *pstatus_arg) r = r_arg; pstatus = pstatus_arg; - run_next_test(); - - SET_HANDLER(&PVCTestDriver::main_handler); + if (run_next_test()) { + SET_HANDLER(&PVCTestDriver::main_handler); + } } -void +bool PVCTestDriver::run_next_test() { unsigned a_index = i * 2; @@ -1293,7 +1326,7 @@ PVCTestDriver::run_next_test() *pstatus = REGRESSION_TEST_FAILED; } delete this; - return; + return false; } completions_received = 0; i++; @@ -1308,6 +1341,8 @@ PVCTestDriver::run_next_test() PluginVC *a_vc = core->connect(); a->init_test(NET_VC_TEST_ACTIVE, this, a_vc, r, &netvc_tests_def[a_index], "PluginVC", "pvc_test_detail"); + + return true; } int diff --git a/proxy/PluginVC.h b/proxy/PluginVC.h index e3603829f60..6ee6b4db041 100644 --- a/proxy/PluginVC.h +++ b/proxy/PluginVC.h @@ -44,10 +44,10 @@ class PluginVCCore; struct PluginVCState { PluginVCState(); VIO vio; - bool shutdown; + bool shutdown = false; }; -inline PluginVCState::PluginVCState() : vio(), shutdown(false) {} +inline PluginVCState::PluginVCState() : vio() {} enum PluginVC_t { PLUGIN_VC_UNKNOWN, @@ -101,6 +101,8 @@ class PluginVC : public NetVConnection, public PluginIdentity SOCKET get_socket() override; void set_local_addr() override; void set_remote_addr() override; + void set_remote_addr(const sockaddr *) override; + void set_mptcp_state() override; int set_tcp_init_cwnd(int init_cwnd) override; int set_tcp_congestion_control(int) override; @@ -246,37 +248,27 @@ class PluginVCCore : public Continuation void init(); void destroy(); - Continuation *connect_to; - bool connected; + Continuation *connect_to = nullptr; + bool connected = false; - MIOBuffer *p_to_a_buffer; - IOBufferReader *p_to_a_reader; + MIOBuffer *p_to_a_buffer = nullptr; + IOBufferReader *p_to_a_reader = nullptr; - MIOBuffer *a_to_p_buffer; - IOBufferReader *a_to_p_reader; + MIOBuffer *a_to_p_buffer = nullptr; + IOBufferReader *a_to_p_reader = nullptr; IpEndpoint passive_addr_struct; IpEndpoint active_addr_struct; - void *passive_data; - void *active_data; + void *passive_data = nullptr; + void *active_data = nullptr; static int32_t nextid; - unsigned id; + unsigned id = 0; }; -inline PluginVCCore::PluginVCCore() - : active_vc(this), - passive_vc(this), - connect_to(nullptr), - connected(false), - p_to_a_buffer(nullptr), - p_to_a_reader(nullptr), - a_to_p_buffer(nullptr), - a_to_p_reader(nullptr), - passive_data(nullptr), - active_data(nullptr), - id(0) +inline PluginVCCore::PluginVCCore() : active_vc(this), passive_vc(this) + { memset(&active_addr_struct, 0, sizeof active_addr_struct); memset(&passive_addr_struct, 0, sizeof passive_addr_struct); diff --git a/proxy/ProtocolProbeSessionAccept.cc b/proxy/ProtocolProbeSessionAccept.cc index 8d332941d6d..31b6e52ad4d 100644 --- a/proxy/ProtocolProbeSessionAccept.cc +++ b/proxy/ProtocolProbeSessionAccept.cc @@ -25,6 +25,8 @@ #include "I_Machine.h" #include "ProtocolProbeSessionAccept.h" #include "http2/HTTP2.h" +#include "ProxyProtocol.h" +#include "I_NetVConnection.h" static bool proto_is_http2(IOBufferReader *reader) @@ -90,6 +92,44 @@ struct ProtocolProbeTrampoline : public Continuation, public ProtocolProbeSessio goto done; } + // if proxy_protocol is enabled via port descriptor AND the src IP is in + // the trusted whitelist for proxy protocol, then check to see if it is + // present + + IpMap *pp_ipmap; + pp_ipmap = probeParent->proxy_protocol_ipmap; + + if (netvc->get_is_proxy_protocol()) { + Debug("proxyprotocol", "ioCompletionEvent: proxy protocol is enabled on this port"); + if (pp_ipmap->count() > 0) { + Debug("proxyprotocol", "ioCompletionEvent: proxy protocol has a configured whitelist of trusted IPs - checking"); + void *payload = nullptr; + if (!pp_ipmap->contains(netvc->get_remote_addr(), &payload)) { + Debug("proxyprotocol", + "ioCompletionEvent: proxy protocol src IP is NOT in the configured whitelist of trusted IPs - closing connection"); + goto done; + } else { + char new_host[INET6_ADDRSTRLEN]; + Debug("proxyprotocol", "ioCompletionEvent: Source IP [%s] is trusted in the whitelist for proxy protocol", + ats_ip_ntop(netvc->get_remote_addr(), new_host, sizeof(new_host))); + } + } else { + Debug("proxyprotocol", + "ioCompletionEvent: proxy protocol DOES NOT have a configured whitelist of trusted IPs but proxy protocol is " + "ernabled on this port - processing all connections"); + } + + if (http_has_proxy_v1(reader, netvc)) { + Debug("proxyprotocol", "ioCompletionEvent: http has proxy_v1 header"); + netvc->set_remote_addr(netvc->get_proxy_protocol_src_addr()); + } else { + Debug("proxyprotocol", + "ioCompletionEvent: proxy protocol was enabled, but required header was not present in the transaction - " + "closing connection"); + goto done; + } + } // end of Proxy Protocol processing + if (proto_is_http2(reader)) { key = PROTO_HTTP2; } else { diff --git a/proxy/ProtocolProbeSessionAccept.h b/proxy/ProtocolProbeSessionAccept.h index 8ba84201760..c58dbe31bd2 100644 --- a/proxy/ProtocolProbeSessionAccept.h +++ b/proxy/ProtocolProbeSessionAccept.h @@ -53,6 +53,8 @@ class ProtocolProbeSessionAccept : public SessionAccept, public ProtocolProbeSes ProtocolProbeSessionAccept(const ProtocolProbeSessionAccept &) = delete; // disabled ProtocolProbeSessionAccept &operator=(const ProtocolProbeSessionAccept &) = delete; // disabled + IpMap *proxy_protocol_ipmap = nullptr; + private: int mainEvent(int event, void *netvc) override; diff --git a/proxy/ProxyClientSession.cc b/proxy/ProxyClientSession.cc index 0c71d12be10..214295a7d9c 100644 --- a/proxy/ProxyClientSession.cc +++ b/proxy/ProxyClientSession.cc @@ -119,29 +119,20 @@ ProxyClientSession::state_api_callout(int event, void *data) } if (this->api_current) { - bool plugin_lock = false; - APIHook *hook = this->api_current; - Ptr plugin_mutex; - - if (hook->m_cont->mutex) { - plugin_mutex = hook->m_cont->mutex; - plugin_lock = MUTEX_TAKE_TRY_LOCK(hook->m_cont->mutex, mutex->thread_holding); - if (!plugin_lock) { - SET_HANDLER(&ProxyClientSession::state_api_callout); - if (!schedule_event) { // Don't bother to schedule is there is already one out. - schedule_event = mutex->thread_holding->schedule_in(this, HRTIME_MSECONDS(10)); - } - return 0; + APIHook *hook = this->api_current; + + MUTEX_TRY_LOCK(lock, hook->m_cont->mutex, mutex->thread_holding); + // Have a mutex but did't get the lock, reschedule + if (!lock.is_locked()) { + SET_HANDLER(&ProxyClientSession::state_api_callout); + if (!schedule_event) { // Don't bother to schedule is there is already one out. + schedule_event = mutex->thread_holding->schedule_in(this, HRTIME_MSECONDS(10)); } + return 0; } this->api_current = this->api_current->next(); hook->invoke(eventmap[this->api_hookid], this); - - if (plugin_lock) { - Mutex_unlock(plugin_mutex, this_ethread()); - } - return 0; } } @@ -170,7 +161,7 @@ ProxyClientSession::do_api_callout(TSHttpHookID id) this->api_scope = API_HOOK_SCOPE_GLOBAL; this->api_current = nullptr; - if (this->hooks_on && this->has_hooks()) { + if (this->has_hooks()) { SET_HANDLER(&ProxyClientSession::state_api_callout); this->state_api_callout(EVENT_NONE, nullptr); } else { @@ -202,8 +193,7 @@ ProxyClientSession::handle_api_return(int event) break; } default: - Error("received invalid session hook %s (%d)", HttpDebugNames::get_api_hook_name(hookid), hookid); - ink_release_assert(false); + Fatal("received invalid session hook %s (%d)", HttpDebugNames::get_api_hook_name(hookid), hookid); break; } } diff --git a/proxy/ProxyClientSession.h b/proxy/ProxyClientSession.h index 9637a9a4c50..7c6acf9f6a9 100644 --- a/proxy/ProxyClientSession.h +++ b/proxy/ProxyClientSession.h @@ -25,6 +25,7 @@ #include "tscore/ink_platform.h" #include "tscore/ink_resolver.h" +#include "tscore/TSSystemState.h" #include #include "P_Net.h" #include "InkAPIInternal.h" @@ -69,6 +70,7 @@ struct ProxyError { uint32_t code = 0; }; +/// Abstract class for HttpSM to interface with any session class ProxyClientSession : public VConnection { public: @@ -78,7 +80,7 @@ class ProxyClientSession : public VConnection virtual void free(); virtual void start() = 0; - virtual void new_connection(NetVConnection *new_vc, MIOBuffer *iobuf, IOBufferReader *reader, bool backdoor) = 0; + virtual void new_connection(NetVConnection *new_vc, MIOBuffer *iobuf, IOBufferReader *reader) = 0; virtual NetVConnection *get_netvc() const = 0; @@ -131,12 +133,6 @@ class ProxyClientSession : public VConnection return this->debug_on; } - bool - hooks_enabled() const - { - return this->hooks_on; - } - bool has_hooks() const { @@ -152,11 +148,7 @@ class ProxyClientSession : public VConnection bool is_draining() const { - RecInt draining; - if (RecGetRecordInt("proxy.node.config.draining", &draining) != REC_ERR_OKAY) { - return false; - } - return draining != 0; + return TSSystemState::is_draining(); } // Initiate an API hook invocation. @@ -312,7 +304,6 @@ class ProxyClientSession : public VConnection // Session specific debug flag. bool debug_on = false; - bool hooks_on = true; bool in_destroy = false; int64_t con_id = 0; diff --git a/proxy/ProxyClientTransaction.cc b/proxy/ProxyClientTransaction.cc index 17c1c93f5e4..71323a4e202 100644 --- a/proxy/ProxyClientTransaction.cc +++ b/proxy/ProxyClientTransaction.cc @@ -27,15 +27,7 @@ #define HttpTxnDebug(fmt, ...) SsnDebug(this, "http_txn", fmt, __VA_ARGS__) -ProxyClientTransaction::ProxyClientTransaction() - : VConnection(nullptr), - parent(nullptr), - current_reader(nullptr), - sm_reader(nullptr), - host_res_style(HOST_RES_NONE), - restart_immediate(false) -{ -} +ProxyClientTransaction::ProxyClientTransaction() : VConnection(nullptr) {} void ProxyClientTransaction::new_transaction() @@ -57,6 +49,7 @@ ProxyClientTransaction::new_transaction() current_reader->plugin_id = pi->getPluginId(); } + this->increment_client_transactions_stat(); current_reader->attach_client_session(this, sm_reader); } @@ -66,6 +59,8 @@ ProxyClientTransaction::release(IOBufferReader *r) HttpTxnDebug("[%" PRId64 "] session released by sm [%" PRId64 "]", parent ? parent->connection_id() : 0, current_reader ? current_reader->sm_id : 0); + this->decrement_client_transactions_stat(); + // Pass along the release to the session if (parent) { parent->release(this); diff --git a/proxy/ProxyClientTransaction.h b/proxy/ProxyClientTransaction.h index 4dc46aad445..c4ebfe0382f 100644 --- a/proxy/ProxyClientTransaction.h +++ b/proxy/ProxyClientTransaction.h @@ -98,11 +98,6 @@ class ProxyClientTransaction : public VConnection { return parent ? parent->debug() : false; } - bool - hooks_enabled() const - { - return parent ? parent->hooks_enabled() : false; - } APIHook * ssn_hook_get(TSHttpHookID id) const @@ -267,19 +262,22 @@ class ProxyClientTransaction : public VConnection void set_rx_error_code(ProxyError e); void set_tx_error_code(ProxyError e); + virtual void increment_client_transactions_stat() = 0; + virtual void decrement_client_transactions_stat() = 0; + protected: - ProxyClientSession *parent; - HttpSM *current_reader; - IOBufferReader *sm_reader; + ProxyClientSession *parent = nullptr; + HttpSM *current_reader = nullptr; + IOBufferReader *sm_reader = nullptr; /// DNS resolution preferences. - HostResStyle host_res_style; + HostResStyle host_res_style = HOST_RES_NONE; /// Local outbound address control. in_port_t outbound_port{0}; IpAddr outbound_ip4; IpAddr outbound_ip6; - bool restart_immediate; + bool restart_immediate = false; private: }; diff --git a/proxy/README-stats.otl b/proxy/README-stats.otl index 80648346d0c..e57263fc0a1 100644 --- a/proxy/README-stats.otl +++ b/proxy/README-stats.otl @@ -521,7 +521,6 @@ Adam Beguelin wrote: > o Cache/Freshness > o Cache/Variable Content > o HostDB page -> o Logging/Log Collation --------------------------------------------------------------------------------- --------------------------------------------------------------------------------- --------------------------------------------------------------------------------- diff --git a/proxy/RegressionSM.cc b/proxy/RegressionSM.cc index b31368e7867..a6f28ebe88c 100644 --- a/proxy/RegressionSM.cc +++ b/proxy/RegressionSM.cc @@ -120,16 +120,15 @@ r_sequential(RegressionTest *t, RegressionSM *sm, ...) RegressionSM *new_sm = new RegressionSM(t); va_list ap; va_start(ap, sm); - new_sm->parallel = false; - new_sm->repeat = false; - new_sm->ichild = 0; - new_sm->nchildren = 0; - new_sm->nwaiting = 0; - while (nullptr != sm) { - new_sm->children(new_sm->nchildren++) = sm; - sm = va_arg(ap, RegressionSM *); + new_sm->parallel = false; + new_sm->repeat = false; + new_sm->ichild = 0; + new_sm->nwaiting = 0; + while (sm) { + new_sm->children.push_back(sm); + sm = va_arg(ap, RegressionSM *); } - new_sm->n = new_sm->nchildren; + new_sm->n = new_sm->children.size(); va_end(ap); return new_sm; } @@ -141,10 +140,9 @@ r_sequential(RegressionTest *t, int an, RegressionSM *sm) new_sm->parallel = false; new_sm->repeat = true; new_sm->ichild = 0; - new_sm->nchildren = 1; - new_sm->children(0) = sm; - new_sm->nwaiting = 0; - new_sm->n = an; + new_sm->children.push_back(sm); + new_sm->nwaiting = 0; + new_sm->n = an; return new_sm; } @@ -154,16 +152,15 @@ r_parallel(RegressionTest *t, RegressionSM *sm, ...) RegressionSM *new_sm = new RegressionSM(t); va_list ap; va_start(ap, sm); - new_sm->parallel = true; - new_sm->repeat = false; - new_sm->ichild = 0; - new_sm->nchildren = 0; - new_sm->nwaiting = 0; + new_sm->parallel = true; + new_sm->repeat = false; + new_sm->ichild = 0; + new_sm->nwaiting = 0; while (sm) { - new_sm->children(new_sm->nchildren++) = sm; - sm = va_arg(ap, RegressionSM *); + new_sm->children.push_back(sm); + sm = va_arg(ap, RegressionSM *); } - new_sm->n = new_sm->nchildren; + new_sm->n = new_sm->children.size(); va_end(ap); return new_sm; } @@ -175,10 +172,9 @@ r_parallel(RegressionTest *t, int an, RegressionSM *sm) new_sm->parallel = true; new_sm->repeat = true; new_sm->ichild = 0; - new_sm->nchildren = 1; - new_sm->children(0) = sm; - new_sm->nwaiting = 0; - new_sm->n = an; + new_sm->children.push_back(sm); + new_sm->nwaiting = 0; + new_sm->n = an; return new_sm; } @@ -227,15 +223,14 @@ RegressionSM::RegressionSM(const RegressionSM &ao) : Continuation(ao) { RegressionSM &o = *(RegressionSM *)&ao; - t = o.t; - status = o.status; - pstatus = o.pstatus; - parent = &o; - nwaiting = o.nwaiting; - nchildren = o.nchildren; + t = o.t; + status = o.status; + pstatus = o.pstatus; + parent = &o; + nwaiting = o.nwaiting; - for (intptr_t i = 0; i < nchildren; i++) { - children(i) = o.children[i]->clone(); + for (auto &i : o.children) { + children.push_back(i->clone()); } n = o.n; diff --git a/proxy/RegressionSM.h b/proxy/RegressionSM.h index d643de37a8a..31cabf807bd 100644 --- a/proxy/RegressionSM.h +++ b/proxy/RegressionSM.h @@ -25,7 +25,6 @@ #include "I_EventSystem.h" #include "tscore/Regression.h" -#include "tscore/DynArray.h" /* Regression Test Composition State Machine @@ -50,17 +49,17 @@ struct RegressionSM : public Continuation { void run_in(int *pstatus, ink_hrtime t); // internal - int status = REGRESSION_TEST_INPROGRESS; - int *pstatus = nullptr; - RegressionSM *parent = nullptr; - int nwaiting = 0; - int nchildren = 0; - DynArray children = nullptr; - intptr_t n = 0; - intptr_t ichild = 0; - bool parallel = false; - bool repeat = false; - Action *pending_action = nullptr; + int status = REGRESSION_TEST_INPROGRESS; + int *pstatus = nullptr; + RegressionSM *parent = nullptr; + int nwaiting = 0; + int nchildren = 0; + std::vector children; + intptr_t n = 0; + intptr_t ichild = 0; + bool parallel = false; + bool repeat = false; + Action *pending_action = nullptr; int regression_sm_start(int event, void *data); int regression_sm_waiting(int event, void *data); diff --git a/proxy/ReverseProxy.cc b/proxy/ReverseProxy.cc index 81f18a5959b..2ab3d036065 100644 --- a/proxy/ReverseProxy.cc +++ b/proxy/ReverseProxy.cc @@ -43,8 +43,7 @@ // Global Ptrs static Ptr reconfig_mutex; -UrlRewrite *rewrite_table = nullptr; -remap_plugin_info *remap_pi_list = nullptr; // We never reload the remap plugins, just append to 'em. +UrlRewrite *rewrite_table = nullptr; // Tokens for the Callback function #define FILE_CHANGED 0 @@ -64,9 +63,11 @@ init_reverse_proxy() reconfig_mutex = new_ProxyMutex(); rewrite_table = new UrlRewrite(); - if (!rewrite_table->is_valid()) { - Fatal("unable to load remap.config"); + Note("remap.config loading ..."); + if (!rewrite_table->load()) { + Fatal("remap.config failed to load"); } + Note("remap.config finished loading"); REC_RegisterConfigUpdateFunc("proxy.config.url_remap.filename", url_rewrite_CB, (void *)FILE_CHANGED); REC_RegisterConfigUpdateFunc("proxy.config.proxy_name", url_rewrite_CB, (void *)TSNAME_CHANGED); @@ -117,6 +118,12 @@ struct UR_UpdateContinuation : public Continuation { } }; +bool +urlRewriteVerify() +{ + return UrlRewrite().load(); +} + /** Called when the remap.config file changes. Since it called infrequently, we do the load of new file as blocking I/O and lock aquire is also @@ -126,26 +133,34 @@ struct UR_UpdateContinuation : public Continuation { bool reloadUrlRewrite() { - UrlRewrite *newTable; + UrlRewrite *newTable, *oldTable; + Note("remap.config loading ..."); Debug("url_rewrite", "remap.config updated, reloading..."); newTable = new UrlRewrite(); - if (newTable->is_valid()) { - static const char *msg = "remap.config done reloading!"; + if (newTable->load()) { + static const char *msg = "remap.config finished loading"; // Hold at least one lease, until we reload the configuration newTable->acquire(); - ink_atomic_swap(&rewrite_table, newTable)->release(); // Swap configurations, and release the old one + // Swap configurations + oldTable = ink_atomic_swap(&rewrite_table, newTable); + + ink_assert(oldTable != nullptr); + + // Release the old one + oldTable->release(); + Debug("url_rewrite", "%s", msg); Note("%s", msg); return true; } else { - static const char *msg = "failed to reload remap.config, not replacing!"; + static const char *msg = "remap.config failed to load"; delete newTable; Debug("url_rewrite", "%s", msg); - Warning("%s", msg); + Error("%s", msg); return false; } } diff --git a/proxy/ReverseProxy.h b/proxy/ReverseProxy.h index 0ba5bc3de0c..49f2e21ac58 100644 --- a/proxy/ReverseProxy.h +++ b/proxy/ReverseProxy.h @@ -46,7 +46,6 @@ class url_mapping; struct host_hdr_info; extern UrlRewrite *rewrite_table; -extern remap_plugin_info *remap_pi_list; // API Functions int init_reverse_proxy(); @@ -56,5 +55,6 @@ bool response_url_remap(HTTPHdr *response_header, UrlRewrite *table); // Reload Functions bool reloadUrlRewrite(); +bool urlRewriteVerify(); int url_rewrite_CB(const char *name, RecDataT data_type, RecData data, void *cookie); diff --git a/proxy/StatPages.h b/proxy/StatPages.h index e337cbb65ee..cbce9cb703b 100644 --- a/proxy/StatPages.h +++ b/proxy/StatPages.h @@ -64,13 +64,13 @@ typedef Action *(*StatPagesFunc)(Continuation *cont, HTTPHdr *header); struct StatPageData { - char *data; - char *type; - int length; + char *data = nullptr; + char *type = nullptr; + int length = 0; - StatPageData() : data(nullptr), type(nullptr), length(0) {} - StatPageData(char *adata) : data(adata), type(nullptr) { length = strlen(adata); } - StatPageData(char *adata, int alength) : data(adata), type(nullptr), length(alength) {} + StatPageData() {} + StatPageData(char *adata) : data(adata) { length = strlen(adata); } + StatPageData(char *adata, int alength) : data(adata), length(alength) {} }; struct StatPagesManager { @@ -97,17 +97,17 @@ class BaseStatPagesHandler : public Continuation ~BaseStatPagesHandler() override { resp_clear(); }; protected: - inkcoreapi void resp_clear(void); + inkcoreapi void resp_clear(); inkcoreapi void resp_add(const char *fmt, ...); - inkcoreapi void resp_add_sep(void); + inkcoreapi void resp_add_sep(); inkcoreapi void resp_begin(const char *title); - inkcoreapi void resp_end(void); - void resp_begin_numbered(void); - void resp_end_numbered(void); - inkcoreapi void resp_begin_unnumbered(void); - inkcoreapi void resp_end_unnumbered(void); - inkcoreapi void resp_begin_item(void); - void resp_end_item(void); + inkcoreapi void resp_end(); + void resp_begin_numbered(); + void resp_end_numbered(); + inkcoreapi void resp_begin_unnumbered(); + inkcoreapi void resp_end_unnumbered(); + inkcoreapi void resp_begin_item(); + void resp_end_item(); inkcoreapi void resp_begin_table(int border, int columns, int percent); inkcoreapi void resp_end_table(); inkcoreapi void resp_begin_row(); diff --git a/proxy/Transform.cc b/proxy/Transform.cc index 516f8e94b75..3049305812b 100644 --- a/proxy/Transform.cc +++ b/proxy/Transform.cc @@ -524,8 +524,7 @@ TransformVConnection::backlog(uint64_t limit) /*------------------------------------------------------------------------- -------------------------------------------------------------------------*/ -TransformControl::TransformControl() - : Continuation(new_ProxyMutex()), m_hooks(), m_tvc(nullptr), m_read_buf(nullptr), m_write_buf(nullptr) +TransformControl::TransformControl() : Continuation(new_ProxyMutex()), m_hooks() { SET_HANDLER(&TransformControl::handle_event); @@ -727,7 +726,7 @@ NullTransform::handle_event(int event, void *edata) /*------------------------------------------------------------------------- -------------------------------------------------------------------------*/ -#ifdef TS_HAS_TESTS +#if TS_HAS_TESTS void TransformTest::run() { @@ -782,7 +781,11 @@ RangeTransform::handle_event(int event, void *edata) if (m_closed) { if (m_deletable) { - Debug("http_trans", "RangeTransform destroy: %p ndone=%" PRId64, this, m_output_vio ? m_output_vio->ndone : 0); + if (m_output_vc != nullptr) { + Debug("http_trans", "RangeTransform destroy: %p ndone=%" PRId64, this, m_output_vio ? m_output_vio->ndone : 0); + } else { + Debug("http_trans", "RangeTransform destroy"); + } delete this; } } else { diff --git a/proxy/Transform.h b/proxy/Transform.h index 5789b98a7dc..2030eb6df03 100644 --- a/proxy/Transform.h +++ b/proxy/Transform.h @@ -30,10 +30,10 @@ #define TRANSFORM_READ_READY (TRANSFORM_EVENTS_START + 0) typedef struct _RangeRecord { - _RangeRecord() : _start(-1), _end(-1), _done_byte(-1) {} - int64_t _start; - int64_t _end; - int64_t _done_byte; + _RangeRecord() {} + int64_t _start = -1; + int64_t _end = -1; + int64_t _done_byte = -1; } RangeRecord; class TransformProcessor @@ -48,7 +48,7 @@ class TransformProcessor int content_type_len, int64_t content_length); }; -#ifdef TS_HAS_TESTS +#if TS_HAS_TESTS class TransformTest { public: diff --git a/proxy/TransformInternal.h b/proxy/TransformInternal.h index 754b241aa51..02b862fb4ac 100644 --- a/proxy/TransformInternal.h +++ b/proxy/TransformInternal.h @@ -55,21 +55,21 @@ class TransformVConnection : public TransformVCChain { public: TransformVConnection(Continuation *cont, APIHook *hooks); - ~TransformVConnection(); + ~TransformVConnection() override; int handle_event(int event, void *edata); - VIO *do_io_read(Continuation *c, int64_t nbytes, MIOBuffer *buf); - VIO *do_io_write(Continuation *c, int64_t nbytes, IOBufferReader *buf, bool owner = false); - void do_io_close(int lerrno = -1); - void do_io_shutdown(ShutdownHowTo_t howto); + VIO *do_io_read(Continuation *c, int64_t nbytes, MIOBuffer *buf) override; + VIO *do_io_write(Continuation *c, int64_t nbytes, IOBufferReader *buf, bool owner = false) override; + void do_io_close(int lerrno = -1) override; + void do_io_shutdown(ShutdownHowTo_t howto) override; - void reenable(VIO *vio); + void reenable(VIO *vio) override; /** Compute the backlog. @return The actual backlog, or a value at least @a limit. */ - virtual uint64_t backlog(uint64_t limit = UINT64_MAX); + uint64_t backlog(uint64_t limit = UINT64_MAX) override; public: VConnection *m_transform; @@ -87,16 +87,16 @@ class TransformControl : public Continuation public: APIHooks m_hooks; - VConnection *m_tvc; - IOBufferReader *m_read_buf; - MIOBuffer *m_write_buf; + VConnection *m_tvc = nullptr; + IOBufferReader *m_read_buf = nullptr; + MIOBuffer *m_write_buf = nullptr; }; class NullTransform : public INKVConnInternal { public: NullTransform(ProxyMutex *mutex); - ~NullTransform(); + ~NullTransform() override; int handle_event(int event, void *edata); @@ -111,7 +111,7 @@ class RangeTransform : public INKVConnInternal public: RangeTransform(ProxyMutex *mutex, RangeRecord *ranges, int num_fields, HTTPHdr *transform_resp, const char *content_type, int content_type_len, int64_t content_length); - ~RangeTransform(); + ~RangeTransform() override; int handle_event(int event, void *edata); diff --git a/proxy/hdrs/HTTP.cc b/proxy/hdrs/HTTP.cc index 3858bd0cafa..0bbde741c23 100644 --- a/proxy/hdrs/HTTP.cc +++ b/proxy/hdrs/HTTP.cc @@ -592,21 +592,21 @@ http_hdr_describe(HdrHeapObjImpl *raw, bool recurse) -------------------------------------------------------------------------*/ int -http_hdr_length_get(HTTPHdrImpl *hdr) +HTTPHdr::length_get() const { int length = 0; - if (hdr->m_polarity == HTTP_TYPE_REQUEST) { - if (hdr->u.req.m_ptr_method) { - length = hdr->u.req.m_len_method; + if (m_http->m_polarity == HTTP_TYPE_REQUEST) { + if (m_http->u.req.m_ptr_method) { + length = m_http->u.req.m_len_method; } else { length = 0; } length += 1; // " " - if (hdr->u.req.m_url_impl) { - length += url_length_get(hdr->u.req.m_url_impl); + if (m_http->u.req.m_url_impl) { + length += url_length_get(m_http->u.req.m_url_impl); } length += 1; // " " @@ -614,9 +614,9 @@ http_hdr_length_get(HTTPHdrImpl *hdr) length += 8; // HTTP/%d.%d length += 2; // "\r\n" - } else if (hdr->m_polarity == HTTP_TYPE_RESPONSE) { - if (hdr->u.resp.m_ptr_reason) { - length = hdr->u.resp.m_len_reason; + } else if (m_http->m_polarity == HTTP_TYPE_RESPONSE) { + if (m_http->u.resp.m_ptr_reason) { + length = m_http->u.resp.m_len_reason; } else { length = 0; } @@ -632,7 +632,7 @@ http_hdr_length_get(HTTPHdrImpl *hdr) length += 2; // "\r\n" } - length += mime_hdr_length_get(hdr->m_fields_impl); + length += mime_hdr_length_get(m_http->m_fields_impl); return length; } @@ -708,7 +708,7 @@ http_hdr_url_set(HdrHeap *heap, HTTPHdrImpl *hh, URLImpl *url) int url_string_length = url->strings_length(); heap->m_read_write_heap = new_HdrStrHeap(url_string_length); } - hh->u.req.m_url_impl->move_strings(heap->m_read_write_heap.get()); + hh->u.req.m_url_impl->rehome_strings(heap); } else { hh->u.req.m_url_impl = url; } @@ -834,15 +834,6 @@ http_hdr_reason_lookup(unsigned status) return nullptr; } -/*------------------------------------------------------------------------- - -------------------------------------------------------------------------*/ - -void -_http_parser_init(HTTPParser *parser) -{ - parser->m_parsing_http = true; -} - ////////////////////////////////////////////////////// // init first time structure setup // // clear resets an already-initialized structure // @@ -851,14 +842,14 @@ _http_parser_init(HTTPParser *parser) void http_parser_init(HTTPParser *parser) { - _http_parser_init(parser); + parser->m_parsing_http = true; mime_parser_init(&parser->m_mime_parser); } void http_parser_clear(HTTPParser *parser) { - _http_parser_init(parser); + parser->m_parsing_http = true; mime_parser_clear(&parser->m_mime_parser); } @@ -903,17 +894,21 @@ http_parser_parse_req(HTTPParser *parser, HdrHeap *heap, HTTPHdrImpl *hh, const const char *version_start; const char *version_end; + ts::TextView text, parsed; + real_end = end; start: hh->m_polarity = HTTP_TYPE_REQUEST; // Make sure the line is not longer than 64K - if (scanner->m_line_length >= UINT16_MAX) { + if (scanner->get_buffered_line_size() >= UINT16_MAX) { return PARSE_RESULT_ERROR; } - err = mime_scanner_get(scanner, start, real_end, &line_start, &end, &line_is_real, eof, MIME_SCANNER_TYPE_LINE); + text.assign(*start, real_end); + err = scanner->get(text, parsed, line_is_real, eof, MIMEScanner::LINE); + *start = text.data(); if (err < 0) { return err; } @@ -926,9 +921,9 @@ http_parser_parse_req(HTTPParser *parser, HdrHeap *heap, HTTPHdrImpl *hh, const return err; } - cur = line_start; - ink_assert((end - cur) >= 0); - ink_assert((end - cur) < UINT16_MAX); + ink_assert(parsed.size() < UINT16_MAX); + line_start = cur = parsed.data(); + end = parsed.data_end(); must_copy_strings = (must_copy_strings || (!line_is_real)); @@ -1253,11 +1248,14 @@ http_parser_parse_resp(HTTPParser *parser, HdrHeap *heap, HTTPHdrImpl *hh, const hh->m_polarity = HTTP_TYPE_RESPONSE; // Make sure the line is not longer than 64K - if (scanner->m_line_length >= UINT16_MAX) { + if (scanner->get_buffered_line_size() >= UINT16_MAX) { return PARSE_RESULT_ERROR; } - err = mime_scanner_get(scanner, start, real_end, &line_start, &end, &line_is_real, eof, MIME_SCANNER_TYPE_LINE); + ts::TextView text{*start, real_end}; + ts::TextView parsed; + err = scanner->get(text, parsed, line_is_real, eof, MIMEScanner::LINE); + *start = text.data(); if (err < 0) { return err; } @@ -1265,9 +1263,9 @@ http_parser_parse_resp(HTTPParser *parser, HdrHeap *heap, HTTPHdrImpl *hh, const return err; } - cur = line_start; - ink_assert((end - cur) >= 0); - ink_assert((end - cur) < UINT16_MAX); + ink_assert(parsed.size() < UINT16_MAX); + line_start = cur = parsed.data(); + end = parsed.data_end(); must_copy_strings = (must_copy_strings || (!line_is_real)); @@ -1384,11 +1382,7 @@ http_parser_parse_resp(HTTPParser *parser, HdrHeap *heap, HTTPHdrImpl *hh, const eoh: *start = old_start; - if (parser->m_allow_non_http) { - return PARSE_RESULT_DONE; - } else { - return PARSE_RESULT_ERROR; - } + return PARSE_RESULT_ERROR; // This used to return PARSE_RESULT_DONE by default before done: if (!version_start || !version_end) { @@ -1702,6 +1696,7 @@ class UrlPrintHack ink_assert(nullptr == ui->m_ptr_port); // shouldn't be set if not in URL. ui->m_ptr_port = m_port_buff; ui->m_len_port = snprintf(m_port_buff, sizeof(m_port_buff), "%d", hdr->m_port); + ui->m_port = hdr->m_port; m_port_modified_p = true; } else { m_port_modified_p = false; @@ -1725,6 +1720,7 @@ class UrlPrintHack if (m_port_modified_p) { ui->m_len_port = 0; ui->m_ptr_port = nullptr; + ui->m_port = 0; } if (m_host_modified_p) { ui->m_len_host = 0; @@ -1877,19 +1873,8 @@ ClassAllocator httpCacheAltAllocator("httpCacheAltAllocator"); -------------------------------------------------------------------------*/ int constexpr HTTPCacheAlt::N_INTEGRAL_FRAG_OFFSETS; -HTTPCacheAlt::HTTPCacheAlt() - : m_magic(CACHE_ALT_MAGIC_ALIVE), - m_writeable(1), - m_unmarshal_len(-1), - m_id(-1), - m_rid(-1), - m_request_hdr(), - m_response_hdr(), - m_request_sent_time(0), - m_response_received_time(0), - m_frag_offset_count(0), - m_frag_offsets(nullptr), - m_ext_buffer(nullptr) +HTTPCacheAlt::HTTPCacheAlt() : m_request_hdr(), m_response_hdr() + { memset(&m_object_key[0], 0, CRYPTO_HASH_SIZE); m_object_size[0] = 0; @@ -1961,7 +1946,7 @@ HTTPCacheAlt::copy_frag_offsets_from(HTTPCacheAlt *src) } } -const int HTTP_ALT_MARSHAL_SIZE = ROUND(sizeof(HTTPCacheAlt), HDR_PTR_SIZE); +const int HTTP_ALT_MARSHAL_SIZE = HdrHeapMarshalBlocks{ts::round_up(sizeof(HTTPCacheAlt))}; void HTTPInfo::create() @@ -2002,7 +1987,6 @@ HTTPInfo::marshal_length() } if (m_alt->m_frag_offset_count > HTTPCacheAlt::N_INTEGRAL_FRAG_OFFSETS) { - len -= sizeof(m_alt->m_integral_frag_offsets); len += sizeof(FragOffset) * m_alt->m_frag_offset_count; } @@ -2017,23 +2001,11 @@ HTTPInfo::marshal(char *buf, int len) HTTPCacheAlt *marshal_alt = (HTTPCacheAlt *)buf; // non-zero only if the offsets are external. Otherwise they get // marshalled along with the alt struct. - int frag_len = (0 == m_alt->m_frag_offset_count || m_alt->m_frag_offsets == m_alt->m_integral_frag_offsets) ? - 0 : - sizeof(HTTPCacheAlt::FragOffset) * m_alt->m_frag_offset_count; - ink_assert(m_alt->m_magic == CACHE_ALT_MAGIC_ALIVE); // Make sure the buffer is aligned // ink_assert(((intptr_t)buf) & 0x3 == 0); - // If we have external fragment offsets, copy the initial ones - // into the integral data. - if (frag_len) { - memcpy(m_alt->m_integral_frag_offsets, m_alt->m_frag_offsets, sizeof(m_alt->m_integral_frag_offsets)); - frag_len -= sizeof(m_alt->m_integral_frag_offsets); - // frag_len should never be non-zero at this point, as the offsets - // should be external only if too big for the internal table. - } // Memcpy the whole object so that we can use it // live later. This involves copying a few // extra bytes now but will save copying any @@ -2046,11 +2018,11 @@ HTTPInfo::marshal(char *buf, int len) buf += HTTP_ALT_MARSHAL_SIZE; used += HTTP_ALT_MARSHAL_SIZE; - if (frag_len > 0) { + if (m_alt->m_frag_offset_count > HTTPCacheAlt::N_INTEGRAL_FRAG_OFFSETS) { marshal_alt->m_frag_offsets = static_cast(reinterpret_cast(used)); - memcpy(buf, m_alt->m_frag_offsets + HTTPCacheAlt::N_INTEGRAL_FRAG_OFFSETS, frag_len); - buf += frag_len; - used += frag_len; + memcpy(buf, m_alt->m_frag_offsets, m_alt->m_frag_offset_count * sizeof(FragOffset)); + buf += m_alt->m_frag_offset_count * sizeof(FragOffset); + used += m_alt->m_frag_offset_count * sizeof(FragOffset); } else { marshal_alt->m_frag_offsets = nullptr; } @@ -2108,6 +2080,73 @@ HTTPInfo::unmarshal(char *buf, int len, RefCountObj *block_ref) ink_assert(alt->m_writeable == 0); len -= HTTP_ALT_MARSHAL_SIZE; + if (alt->m_frag_offset_count > HTTPCacheAlt::N_INTEGRAL_FRAG_OFFSETS) { + alt->m_frag_offsets = reinterpret_cast(buf + reinterpret_cast(alt->m_frag_offsets)); + len -= sizeof(FragOffset) * alt->m_frag_offset_count; + ink_assert(len >= 0); + } else if (alt->m_frag_offset_count > 0) { + alt->m_frag_offsets = alt->m_integral_frag_offsets; + } else { + alt->m_frag_offsets = nullptr; // should really already be zero. + } + + HdrHeap *heap = (HdrHeap *)(alt->m_request_hdr.m_heap ? (buf + (intptr_t)alt->m_request_hdr.m_heap) : nullptr); + HTTPHdrImpl *hh = nullptr; + int tmp; + if (heap != nullptr) { + tmp = heap->unmarshal(len, HDR_HEAP_OBJ_HTTP_HEADER, (HdrHeapObjImpl **)&hh, block_ref); + if (hh == nullptr || tmp < 0) { + ink_assert(!"HTTPInfo::request unmarshal failed"); + return -1; + } + len -= tmp; + alt->m_request_hdr.m_heap = heap; + alt->m_request_hdr.m_http = hh; + alt->m_request_hdr.m_mime = hh->m_fields_impl; + alt->m_request_hdr.m_url_cached.m_heap = heap; + } + + heap = (HdrHeap *)(alt->m_response_hdr.m_heap ? (buf + (intptr_t)alt->m_response_hdr.m_heap) : nullptr); + if (heap != nullptr) { + tmp = heap->unmarshal(len, HDR_HEAP_OBJ_HTTP_HEADER, (HdrHeapObjImpl **)&hh, block_ref); + if (hh == nullptr || tmp < 0) { + ink_assert(!"HTTPInfo::response unmarshal failed"); + return -1; + } + len -= tmp; + + alt->m_response_hdr.m_heap = heap; + alt->m_response_hdr.m_http = hh; + alt->m_response_hdr.m_mime = hh->m_fields_impl; + } + + alt->m_unmarshal_len = orig_len - len; + + return alt->m_unmarshal_len; +} + +int +HTTPInfo::unmarshal_v24_1(char *buf, int len, RefCountObj *block_ref) +{ + HTTPCacheAlt *alt = (HTTPCacheAlt *)buf; + int orig_len = len; + + if (alt->m_magic == CACHE_ALT_MAGIC_ALIVE) { + // Already unmarshaled, must be a ram cache + // it + ink_assert(alt->m_unmarshal_len > 0); + ink_assert(alt->m_unmarshal_len <= len); + return alt->m_unmarshal_len; + } else if (alt->m_magic != CACHE_ALT_MAGIC_MARSHALED) { + ink_assert(!"HTTPInfo::unmarshal bad magic"); + return -1; + } + + ink_assert(alt->m_unmarshal_len < 0); + alt->m_magic = CACHE_ALT_MAGIC_ALIVE; + ink_assert(alt->m_writeable == 0); + len -= HTTP_ALT_MARSHAL_SIZE; + if (alt->m_frag_offset_count > HTTPCacheAlt::N_INTEGRAL_FRAG_OFFSETS) { // stuff that didn't fit in the integral slots. int extra = sizeof(FragOffset) * alt->m_frag_offset_count - sizeof(alt->m_integral_frag_offsets); diff --git a/proxy/hdrs/HTTP.h b/proxy/hdrs/HTTP.h index 682a65a3557..8636cf92d65 100644 --- a/proxy/hdrs/HTTP.h +++ b/proxy/hdrs/HTTP.h @@ -322,8 +322,7 @@ struct HTTPValTE { }; struct HTTPParser { - bool m_parsing_http = false; - bool m_allow_non_http = false; + bool m_parsing_http = false; MIMEParser m_mime_parser; }; @@ -424,9 +423,6 @@ inkcoreapi int http_hdr_print(HdrHeap *heap, HTTPHdrImpl *hh, char *buf, int buf void http_hdr_describe(HdrHeapObjImpl *obj, bool recurse = true); -int http_hdr_length_get(HTTPHdrImpl *hh); -// HTTPType http_hdr_type_get (HTTPHdrImpl *hh); - // int32_t http_hdr_version_get (HTTPHdrImpl *hh); inkcoreapi void http_hdr_version_set(HTTPHdrImpl *hh, int32_t ver); @@ -521,7 +517,7 @@ class HTTPHdr : public MIMEHdr int print(char *buf, int bufsize, int *bufindex, int *dumpoffset); - int length_get(); + int length_get() const; HTTPType type_get() const; @@ -844,16 +840,6 @@ HTTPHdr::print(char *buf, int bufsize, int *bufindex, int *dumpoffset) return http_hdr_print(m_heap, m_http, buf, bufsize, bufindex, dumpoffset); } -/*------------------------------------------------------------------------- - -------------------------------------------------------------------------*/ - -inline int -HTTPHdr::length_get() -{ - ink_assert(valid()); - return http_hdr_length_get(m_http); -} - /*------------------------------------------------------------------------- -------------------------------------------------------------------------*/ @@ -1250,7 +1236,7 @@ HTTPHdr::is_cache_control_set(const char *cc_directive_wks) ink_assert(valid()); ink_assert(hdrtoken_is_wks(cc_directive_wks)); - HdrTokenHeapPrefix *prefix = hdrtoken_wks_to_prefix(cc_directive_wks); + const HdrTokenHeapPrefix *prefix = hdrtoken_wks_to_prefix(cc_directive_wks); ink_assert(prefix->wks_token_type == HDRTOKEN_TYPE_CACHE_CONTROL); uint32_t cc_mask = prefix->wks_type_specific.u.cache_control.cc_mask; @@ -1306,18 +1292,18 @@ struct HTTPCacheAlt { void copy_frag_offsets_from(HTTPCacheAlt *src); void destroy(); - uint32_t m_magic; + uint32_t m_magic = CACHE_ALT_MAGIC_ALIVE; // Writeable is set to true is we reside // in a buffer owned by this structure. // INVARIANT: if own the buffer this HttpCacheAlt // we also own the buffers for the request & // response headers - int32_t m_writeable; - int32_t m_unmarshal_len; + int32_t m_writeable = 1; + int32_t m_unmarshal_len = -1; - int32_t m_id; - int32_t m_rid; + int32_t m_id = -1; + int32_t m_rid = -1; int32_t m_object_key[sizeof(CryptoHash) / sizeof(int32_t)]; int32_t m_object_size[2]; @@ -1325,12 +1311,12 @@ struct HTTPCacheAlt { HTTPHdr m_request_hdr; HTTPHdr m_response_hdr; - time_t m_request_sent_time; - time_t m_response_received_time; + time_t m_request_sent_time = 0; + time_t m_response_received_time = 0; /// # of fragment offsets in this alternate. /// @note This is one less than the number of fragments. - int m_frag_offset_count; + int m_frag_offset_count = 0; /// Type of offset for a fragment. typedef uint64_t FragOffset; /// Table of fragment offsets. @@ -1338,7 +1324,7 @@ struct HTTPCacheAlt { /// first byte past the end of fragment 0 which is also the first /// byte of fragment 1. For this reason there is no fragment offset /// for the last fragment. - FragOffset *m_frag_offsets; + FragOffset *m_frag_offsets = nullptr; /// # of fragment offsets built in to object. static int constexpr N_INTEGRAL_FRAG_OFFSETS = 4; /// Integral fragment offset table. @@ -1351,7 +1337,7 @@ struct HTTPCacheAlt { // We don't want to use a ref count ptr (Ptr<>) // since our ownership model requires explicit // destroys and ref count pointers defeat this - RefCountObj *m_ext_buffer; + RefCountObj *m_ext_buffer = nullptr; }; class HTTPInfo @@ -1359,9 +1345,9 @@ class HTTPInfo public: typedef HTTPCacheAlt::FragOffset FragOffset; ///< Import type. - HTTPCacheAlt *m_alt; + HTTPCacheAlt *m_alt = nullptr; - HTTPInfo() : m_alt(nullptr) {} + HTTPInfo() {} ~HTTPInfo() { clear(); } void clear() @@ -1389,6 +1375,7 @@ class HTTPInfo inkcoreapi int marshal_length(); inkcoreapi int marshal(char *buf, int len); static int unmarshal(char *buf, int len, RefCountObj *block_ref); + static int unmarshal_v24_1(char *buf, int len, RefCountObj *block_ref); void set_buffer_reference(RefCountObj *block_ref); int get_handle(char *buf, int len); diff --git a/proxy/hdrs/HdrHeap.cc b/proxy/hdrs/HdrHeap.cc index 0de45b212c9..5dfd44d8051 100644 --- a/proxy/hdrs/HdrHeap.cc +++ b/proxy/hdrs/HdrHeap.cc @@ -37,12 +37,11 @@ #include "HTTP.h" #include "I_EventSystem.h" -#define MAX_LOST_STR_SPACE 1024 +constexpr size_t MAX_LOST_STR_SPACE = 1024; -Allocator hdrHeapAllocator("hdrHeap", HDR_HEAP_DEFAULT_SIZE); -static HdrHeap proto_heap; +Allocator hdrHeapAllocator("hdrHeap", HdrHeap::DEFAULT_SIZE); -Allocator strHeapAllocator("hdrStrHeap", HDR_STR_HEAP_DEFAULT_SIZE); +Allocator strHeapAllocator("hdrStrHeap", HdrStrHeap::DEFAULT_SIZE); /*------------------------------------------------------------------------- -------------------------------------------------------------------------*/ @@ -115,18 +114,13 @@ HdrHeap * new_HdrHeap(int size) { HdrHeap *h; - if (size <= HDR_HEAP_DEFAULT_SIZE) { - size = HDR_HEAP_DEFAULT_SIZE; + if (size <= HdrHeap::DEFAULT_SIZE) { + size = HdrHeap::DEFAULT_SIZE; h = (HdrHeap *)(THREAD_ALLOC(hdrHeapAllocator, this_ethread())); } else { h = (HdrHeap *)ats_malloc(size); } - // Debug("hdrs", "Allocated header heap in size %d", size); - - // Patch virtual function table ptr - *((void **)h) = *((void **)&proto_heap); - h->m_size = size; h->init(); @@ -144,12 +138,12 @@ new_HdrStrHeap(int requested_size) int alloc_size = requested_size + sizeof(HdrStrHeap); HdrStrHeap *sh; - if (alloc_size <= HDR_STR_HEAP_DEFAULT_SIZE) { - alloc_size = HDR_STR_HEAP_DEFAULT_SIZE; + if (alloc_size <= HdrStrHeap::DEFAULT_SIZE) { + alloc_size = HdrStrHeap::DEFAULT_SIZE; sh = (HdrStrHeap *)(THREAD_ALLOC(strHeapAllocator, this_ethread())); } else { - alloc_size = ROUND(alloc_size, HDR_STR_HEAP_DEFAULT_SIZE * 2); - sh = (HdrStrHeap *)ats_malloc(alloc_size); + alloc_size = ts::round_up(alloc_size); + sh = static_cast(ats_malloc(alloc_size)); } // Debug("hdrs", "Allocated string heap in size %d", alloc_size); @@ -158,8 +152,8 @@ new_HdrStrHeap(int requested_size) sh = new (sh) HdrStrHeap(); sh->m_heap_size = alloc_size; - sh->m_free_size = alloc_size - STR_HEAP_HDR_SIZE; - sh->m_free_start = ((char *)sh) + STR_HEAP_HDR_SIZE; + sh->m_free_size = alloc_size - sizeof(HdrStrHeap); + sh->m_free_start = reinterpret_cast(sh + 1); ink_assert(sh->refcount() == 0); @@ -180,7 +174,7 @@ HdrHeap::destroy() i.m_ref_count_ptr = nullptr; } - if (m_size == HDR_HEAP_DEFAULT_SIZE) { + if (m_size == HdrHeap::DEFAULT_SIZE) { THREAD_FREE(this, hdrHeapAllocator, this_thread()); } else { ats_free(this); @@ -195,9 +189,9 @@ HdrHeap::allocate_obj(int nbytes, int type) ink_assert(m_writeable); - nbytes = ROUND(nbytes, HDR_PTR_SIZE); + nbytes = HdrHeapMarshalBlocks{ts::round_up(nbytes)}; - if (nbytes > (int)HDR_MAX_ALLOC_SIZE) { + if (nbytes > static_cast(HDR_MAX_ALLOC_SIZE)) { ink_assert(!"alloc too big"); return nullptr; } @@ -259,7 +253,7 @@ HdrHeap::allocate_str(int nbytes) // First check to see if we have a read/write // string heap if (!m_read_write_heap) { - int next_size = (last_size * 2) - STR_HEAP_HDR_SIZE; + int next_size = (last_size * 2) - sizeof(HdrStrHeap); next_size = next_size > nbytes ? next_size : nbytes; m_read_write_heap = new_HdrStrHeap(next_size); } @@ -568,7 +562,7 @@ HdrHeap::marshal_length() } } - len = ROUND(len, HDR_PTR_SIZE); + len = HdrHeapMarshalBlocks(ts::round_up(len)); return len; } @@ -616,6 +610,8 @@ HdrHeap::marshal(char *buf, int len) int ptr_xl_size = 2; MarshalXlate static_table[2]; MarshalXlate *ptr_xlation = static_table; + // need to initialize it here because of those gotos + MarshalXlate str_xlation[HDR_BUF_RONLY_HEAPS + 1]; // Let's start by skipping over the header block // and copying the pointer blocks to marshalled @@ -628,7 +624,6 @@ HdrHeap::marshal(char *buf, int len) // Variables used later on. Sunpro doesn't like // bypassing initializations with gotos int used; - int i; HdrHeap *unmarshal_hdr = this; @@ -664,7 +659,7 @@ HdrHeap::marshal(char *buf, int len) // Now that we've got the pointer blocks marshaled // we can fill in the header on marshalled block marshal_hdr->m_free_start = nullptr; - marshal_hdr->m_data_start = (char *)HDR_HEAP_HDR_SIZE; // offset + marshal_hdr->m_data_start = reinterpret_cast(HDR_HEAP_HDR_SIZE.value()); // offset marshal_hdr->m_magic = HDR_BUF_MAGIC_MARSHALED; marshal_hdr->m_writeable = false; marshal_hdr->m_size = ptr_heap_size + HDR_HEAP_HDR_SIZE; @@ -677,7 +672,7 @@ HdrHeap::marshal(char *buf, int len) marshal_hdr->m_ronly_heap[0].m_heap_start = (char *)(intptr_t)marshal_hdr->m_size; // offset marshal_hdr->m_ronly_heap[0].m_ref_count_ptr.detach(); - for (int i = 1; i < HDR_BUF_RONLY_HEAPS; i++) { + for (unsigned i = 1; i < HDR_BUF_RONLY_HEAPS; ++i) { marshal_hdr->m_ronly_heap[i].m_heap_start = nullptr; } @@ -690,7 +685,6 @@ HdrHeap::marshal(char *buf, int len) // is too big and only copy over live strings if it is. May // not be too much of a problem since I've prevented too much // lost string space both in string alloc and inherit - MarshalXlate str_xlation[HDR_BUF_RONLY_HEAPS + 1]; if (m_read_write_heap) { char *copy_start = ((char *)m_read_write_heap.get()) + sizeof(HdrStrHeap); @@ -713,25 +707,25 @@ HdrHeap::marshal(char *buf, int len) str_heaps++; } - for (i = 0; i < HDR_BUF_RONLY_HEAPS; i++) { - if (m_ronly_heap[i].m_heap_start != nullptr) { - if (m_ronly_heap[i].m_heap_len > len) { + for (auto &i : m_ronly_heap) { + if (i.m_heap_start != nullptr) { + if (i.m_heap_len > len) { goto Failed; } - memcpy(b, m_ronly_heap[i].m_heap_start, m_ronly_heap[i].m_heap_len); + memcpy(b, i.m_heap_start, i.m_heap_len); // Add translation table entry for string heaps // FIX ME - possible offset overflow issues? - str_xlation[str_heaps].start = m_ronly_heap[i].m_heap_start; - str_xlation[str_heaps].end = m_ronly_heap[i].m_heap_start + m_ronly_heap[i].m_heap_len; + str_xlation[str_heaps].start = i.m_heap_start; + str_xlation[str_heaps].end = i.m_heap_start + i.m_heap_len; str_xlation[str_heaps].offset = str_xlation[str_heaps].start - (b - buf); ink_assert(str_xlation[str_heaps].start <= str_xlation[str_heaps].end); str_heaps++; - b += m_ronly_heap[i].m_heap_len; - len -= m_ronly_heap[i].m_heap_len; - str_size += m_ronly_heap[i].m_heap_len; + b += i.m_heap_len; + len -= i.m_heap_len; + str_size += i.m_heap_len; } } @@ -790,7 +784,7 @@ HdrHeap::marshal(char *buf, int len) // Add up the total bytes used used = ptr_heap_size + str_size + HDR_HEAP_HDR_SIZE; - used = ROUND(used, HDR_PTR_SIZE); + used = HdrHeapMarshalBlocks(ts::round_up(used)); #ifdef HDR_HEAP_CHECKSUMS { @@ -950,7 +944,7 @@ HdrHeap::unmarshal(int buf_length, int obj_type, HdrHeapObjImpl **found_obj, Ref // Nothing to do break; default: - fprintf(stderr, "WARNING: Unmarshal failed due to unknow obj type %d after %d bytes", (int)obj->m_type, + fprintf(stderr, "WARNING: Unmarshal failed due to unknown obj type %d after %d bytes", (int)obj->m_type, (int)(obj_data - (char *)this)); dump_heap(unmarshal_size); return -1; @@ -961,14 +955,14 @@ HdrHeap::unmarshal(int buf_length, int obj_type, HdrHeapObjImpl **found_obj, Ref m_magic = HDR_BUF_MAGIC_ALIVE; - unmarshal_size = ROUND(unmarshal_size, HDR_PTR_SIZE); + unmarshal_size = HdrHeapMarshalBlocks(ts::round_up(unmarshal_size)); return unmarshal_size; } inline bool -HdrHeap::attach_str_heap(char *h_start, int h_len, RefCountObj *h_ref_obj, int *index) +HdrHeap::attach_str_heap(char const *h_start, int h_len, RefCountObj *h_ref_obj, int *index) { - if (*index >= HDR_BUF_RONLY_HEAPS) { + if (*index >= static_cast(HDR_BUF_RONLY_HEAPS)) { return false; } @@ -1010,14 +1004,13 @@ HdrHeap::inherit_string_heaps(const HdrHeap *inherit_from) return; } - int index; int first_free = HDR_BUF_RONLY_HEAPS; // default is out of array bounds int free_slots = 0; int inherit_str_size = 0; ink_assert(m_writeable); // Find the number of free heap slots & the first open index - for (index = 0; index < HDR_BUF_RONLY_HEAPS; index++) { + for (unsigned index = 0; index < HDR_BUF_RONLY_HEAPS; ++index) { if (m_ronly_heap[index].m_heap_start == nullptr) { if (first_free == HDR_BUF_RONLY_HEAPS) { first_free = index; @@ -1031,10 +1024,10 @@ HdrHeap::inherit_string_heaps(const HdrHeap *inherit_from) free_slots--; inherit_str_size = inherit_from->m_read_write_heap->m_heap_size; } - for (index = 0; index < HDR_BUF_RONLY_HEAPS; index++) { - if (inherit_from->m_ronly_heap[index].m_heap_start != nullptr) { + for (const auto &index : inherit_from->m_ronly_heap) { + if (index.m_heap_start != nullptr) { free_slots--; - inherit_str_size += inherit_from->m_ronly_heap[index].m_heap_len; + inherit_str_size += index.m_heap_len; } else { // Heaps are allocated from the front of the array, so if // we hit a NULL, we know we can stop @@ -1059,8 +1052,8 @@ HdrHeap::inherit_string_heaps(const HdrHeap *inherit_from) // Copy over read/write string heap if it exists if (inherit_from->m_read_write_heap) { int str_size = - inherit_from->m_read_write_heap->m_heap_size - STR_HEAP_HDR_SIZE - inherit_from->m_read_write_heap->m_free_size; - ink_release_assert(attach_str_heap(((char *)inherit_from->m_read_write_heap.get()) + STR_HEAP_HDR_SIZE, str_size, + inherit_from->m_read_write_heap->m_heap_size - sizeof(HdrStrHeap) - inherit_from->m_read_write_heap->m_free_size; + ink_release_assert(attach_str_heap(reinterpret_cast(inherit_from->m_read_write_heap.get() + 1), str_size, inherit_from->m_read_write_heap.get(), &first_free)); } // Copy over read only string heaps @@ -1120,7 +1113,7 @@ HdrHeap::dump_heap(int len) void HdrStrHeap::free() { - if (m_heap_size == HDR_STR_HEAP_DEFAULT_SIZE) { + if (m_heap_size == HdrStrHeap::DEFAULT_SIZE) { THREAD_FREE(this, strHeapAllocator, this_thread()); } else { ats_free(this); @@ -1157,8 +1150,8 @@ HdrStrHeap::expand(char *ptr, int old_size, int new_size) { unsigned int expand_size = new_size - old_size; - ink_assert(ptr >= ((char *)this) + STR_HEAP_HDR_SIZE); - ink_assert(ptr < ((char *)this) + m_heap_size); + ink_assert(ptr >= reinterpret_cast(this + 1)); + ink_assert(ptr < reinterpret_cast(this) + m_heap_size); if (ptr + old_size == m_free_start && expand_size <= m_free_size) { m_free_start += expand_size; @@ -1169,18 +1162,6 @@ HdrStrHeap::expand(char *ptr, int old_size, int new_size) } } -StrHeapDesc::StrHeapDesc() -{ - m_heap_start = nullptr; - m_heap_len = 0; - m_locked = false; -} - -struct StrTest { - char *ptr; - int len; -}; - #if TS_HAS_TESTS #include "tscore/TestBox.h" REGRESSION_TEST(HdrHeap_Coalesce)(RegressionTest *t, int /* atype ATS_UNUSED */, int *pstatus) @@ -1191,9 +1172,9 @@ REGRESSION_TEST(HdrHeap_Coalesce)(RegressionTest *t, int /* atype ATS_UNUSED */, * demotion of rw heaps to ronly heaps, and finally the coalesce and evacuate behaviours. */ - // The amount of space we will need to overflow the StrHdrHeap is HDR_STR_HEAP_DEFAULT_SIZE - STR_HEAP_HDR_SIZE - size_t next_rw_heap_size = HDR_STR_HEAP_DEFAULT_SIZE; - size_t next_required_overflow_size = next_rw_heap_size - STR_HEAP_HDR_SIZE; + // The amount of space we will need to overflow the StrHdrHeap is HdrStrHeap::DEFAULT_SIZE - sizeof(HdrStrHeap) + size_t next_rw_heap_size = HdrStrHeap::DEFAULT_SIZE; + size_t next_required_overflow_size = next_rw_heap_size - sizeof(HdrStrHeap); char buf[next_required_overflow_size]; for (unsigned int i = 0; i < sizeof(buf); ++i) { buf[i] = ('a' + (i % 26)); @@ -1206,15 +1187,15 @@ REGRESSION_TEST(HdrHeap_Coalesce)(RegressionTest *t, int /* atype ATS_UNUSED */, tb.check(heap->m_read_write_heap.get() == nullptr, "Checking that we have no rw heap."); url_path_set(heap, url, buf, next_required_overflow_size, true); tb.check(heap->m_read_write_heap->m_free_size == 0, "Checking that we've completely consumed the rw heap"); - for (int i = 0; i < HDR_BUF_RONLY_HEAPS; ++i) { + for (unsigned i = 0; i < HDR_BUF_RONLY_HEAPS; ++i) { tb.check(heap->m_ronly_heap[i].m_heap_start == (char *)nullptr, "Checking ronly_heap[%d] is NULL", i); } // Now we have no ronly heaps in use and a completely full rwheap, so we will test that // we demote to ronly heaps HDR_BUF_RONLY_HEAPS times. - for (int ronly_heap = 0; ronly_heap < HDR_BUF_RONLY_HEAPS; ++ronly_heap) { + for (unsigned ronly_heap = 0; ronly_heap < HDR_BUF_RONLY_HEAPS; ++ronly_heap) { next_rw_heap_size = 2 * heap->m_read_write_heap->m_heap_size; - next_required_overflow_size = next_rw_heap_size - STR_HEAP_HDR_SIZE; + next_required_overflow_size = next_rw_heap_size - sizeof(HdrStrHeap); char buf2[next_required_overflow_size]; for (unsigned int i = 0; i < sizeof(buf2); ++i) { buf2[i] = ('a' + (i % 26)); @@ -1227,13 +1208,13 @@ REGRESSION_TEST(HdrHeap_Coalesce)(RegressionTest *t, int /* atype ATS_UNUSED */, (int)next_rw_heap_size); tb.check(heap->m_read_write_heap->m_free_size == 0, "Checking that we've completely consumed the rw heap"); tb.check(heap->m_ronly_heap[ronly_heap].m_heap_start != nullptr, "Checking that we properly demoted the previous rw heap"); - for (int i = ronly_heap + 1; i < HDR_BUF_RONLY_HEAPS; ++i) { + for (unsigned i = ronly_heap + 1; i < HDR_BUF_RONLY_HEAPS; ++i) { tb.check(heap->m_ronly_heap[i].m_heap_start == nullptr, "Checking ronly_heap[%d] is NULL", i); } } // We will rerun these checks after we introduce a non-copied string to make sure we didn't already coalesce - for (int i = 0; i < HDR_BUF_RONLY_HEAPS; ++i) { + for (unsigned i = 0; i < HDR_BUF_RONLY_HEAPS; ++i) { tb.check(heap->m_ronly_heap[i].m_heap_start != (char *)nullptr, "Pre non-copied string: Checking ronly_heap[%d] is NOT NULL", i); } @@ -1251,7 +1232,7 @@ REGRESSION_TEST(HdrHeap_Coalesce)(RegressionTest *t, int /* atype ATS_UNUSED */, "Checking that the aliased string shows having proper length"); tb.check(aliased_str_url->m_ptr_path == buf3, "Checking that the aliased string is correctly pointing at buf"); - for (int i = 0; i < HDR_BUF_RONLY_HEAPS; ++i) { + for (unsigned i = 0; i < HDR_BUF_RONLY_HEAPS; ++i) { tb.check(heap->m_ronly_heap[i].m_heap_start != (char *)nullptr, "Post non-copied string: Checking ronly_heap[%d] is NOT NULL", i); } diff --git a/proxy/hdrs/HdrHeap.h b/proxy/hdrs/HdrHeap.h index faa8c488910..825d19c4cde 100644 --- a/proxy/hdrs/HdrHeap.h +++ b/proxy/hdrs/HdrHeap.h @@ -36,28 +36,21 @@ #include "tscore/ink_defs.h" #include "tscore/ink_assert.h" #include "tscore/Arena.h" +#include "tscore/Scalar.h" #include "HdrToken.h" // Objects in the heap must currently be aligned to 8 byte boundaries, // so their (address & HDR_PTR_ALIGNMENT_MASK) == 0 -#define HDR_PTR_SIZE (sizeof(uint64_t)) -#define HDR_PTR_ALIGNMENT_MASK ((HDR_PTR_SIZE)-1L) - -#define ROUND(x, l) (((x) + ((l)-1L)) & ~((l)-1L)) +static constexpr size_t HDR_PTR_SIZE = sizeof(uint64_t); +static constexpr size_t HDR_PTR_ALIGNMENT_MASK = HDR_PTR_SIZE - 1L; +using HdrHeapMarshalBlocks = ts::Scalar; // A many of the operations regarding read-only str // heaps are hand unrolled in the code. Chaning // this value requires a full pass through HdrBuf.cc // to fix the unrolled operations -#define HDR_BUF_RONLY_HEAPS 3 - -#define HDR_HEAP_DEFAULT_SIZE 2048 -#define HDR_STR_HEAP_DEFAULT_SIZE 2048 - -#define HDR_MAX_ALLOC_SIZE (HDR_HEAP_DEFAULT_SIZE - sizeof(HdrHeap)) -#define HDR_HEAP_HDR_SIZE ROUND(sizeof(HdrHeap), HDR_PTR_SIZE) -#define STR_HEAP_HDR_SIZE sizeof(HdrStrHeap) +static constexpr unsigned HDR_BUF_RONLY_HEAPS = 3; class CoreUtils; class IOBufferBlock; @@ -140,6 +133,8 @@ enum { class HdrStrHeap : public RefCountObj { public: + static constexpr int DEFAULT_SIZE = 2048; + void free() override; char *allocate(int nbytes); @@ -150,19 +145,22 @@ class HdrStrHeap : public RefCountObj char *m_free_start; uint32_t m_free_size; - bool - contains(const char *str) const - { - return (str >= ((const char *)this + STR_HEAP_HDR_SIZE) && str < ((const char *)this + m_heap_size)); - } + bool contains(const char *str) const; }; +inline bool +HdrStrHeap::contains(const char *str) const +{ + return reinterpret_cast(this + 1) <= str && str < reinterpret_cast(this) + m_heap_size; +} + struct StrHeapDesc { - StrHeapDesc(); + StrHeapDesc() = default; + Ptr m_ref_count_ptr; - char *m_heap_start; - int32_t m_heap_len; - bool m_locked; + char const *m_heap_start = nullptr; + int32_t m_heap_len = 0; + bool m_locked = false; bool contains(const char *str) const @@ -176,6 +174,8 @@ class HdrHeap friend class CoreUtils; public: + static constexpr int DEFAULT_SIZE = 2048; + void init(); inkcoreapi void destroy(); @@ -206,20 +206,20 @@ class HdrHeap // by a heap consolidation. Does NOT lock for Multi-Threaed // access! void - lock_ronly_str_heap(int i) + lock_ronly_str_heap(unsigned i) { m_ronly_heap[i].m_locked = true; } void - unlock_ronly_str_heap(int i) + unlock_ronly_str_heap(unsigned i) { m_ronly_heap[i].m_locked = false; // INKqa11238 // Move slot i to the first available slot in m_ronly_heap[]. // The move is necessary because the rest of the code assumes // heaps are always allocated in order. - for (int j = 0; j < i; j++) { + for (unsigned j = 0; j < i; j++) { if (m_ronly_heap[j].m_heap_start == nullptr) { // move slot i to slot j m_ronly_heap[j].m_ref_count_ptr = m_ronly_heap[i].m_ref_count_ptr; @@ -230,8 +230,29 @@ class HdrHeap m_ronly_heap[i].m_heap_start = nullptr; m_ronly_heap[i].m_heap_len = 0; m_ronly_heap[i].m_locked = false; + break; // Did the move, time to go. + } + } + } + + // Working function to copy strings into a new heap + // Unlike the HDR_MOVE_STR macro, this function will call + // allocate_str which will update the new_heap to create more space + // if there is not originally sufficient space + inline std::string_view + localize(const std::string_view &string) + { + auto length = string.length(); + if (length > 0) { + char *new_str = this->allocate_str(length); + if (new_str) { + memcpy(new_str, string.data(), length); + } else { + length = 0; } + return {new_str, length}; } + return {nullptr, 0}; } // Sanity Check Functions @@ -264,7 +285,7 @@ class HdrHeap void coalesce_str_heaps(int incoming_size = 0); void evacuate_from_str_heaps(HdrStrHeap *new_heap); size_t required_space_for_evacuation(); - bool attach_str_heap(char *h_start, int h_len, RefCountObj *h_ref_obj, int *index); + bool attach_str_heap(char const *h_start, int h_len, RefCountObj *h_ref_obj, int *index); /** Struct to prevent garbage collection on heaps. This bumps the reference count to the heap containing the pointer @@ -301,6 +322,9 @@ class HdrHeap int m_lost_string_space; }; +static constexpr HdrHeapMarshalBlocks HDR_HEAP_HDR_SIZE{ts::round_up(sizeof(HdrHeap))}; +static constexpr size_t HDR_MAX_ALLOC_SIZE = HdrHeap::DEFAULT_SIZE - HDR_HEAP_HDR_SIZE; + inline void HdrHeap::free_string(const char *s, int len) { @@ -317,14 +341,15 @@ HdrHeap::unmarshal_size() const // struct MarshalXlate { - char *start; - char *end; - char *offset; + char const *start = nullptr; + char const *end = nullptr; + char const *offset = nullptr; + MarshalXlate() {} }; struct HeapCheck { - char *start; - char *end; + char const *start; + char const *end; }; // Nasty macro to do string marshalling @@ -433,7 +458,7 @@ struct HeapCheck { // struct HdrHeapSDKHandle { public: - HdrHeapSDKHandle() : m_heap(nullptr) {} + HdrHeapSDKHandle() {} ~HdrHeapSDKHandle() { clear(); } // clear() only deallocates chained SDK return values // The underlying MBuffer is left untouched @@ -446,7 +471,7 @@ struct HdrHeapSDKHandle { void set(const HdrHeapSDKHandle *from); const char *make_sdk_string(const char *raw_str, int raw_str_len); - HdrHeap *m_heap; + HdrHeap *m_heap = nullptr; // In order to prevent gratitous refcounting, // automatic C++ copies are disabled! @@ -477,6 +502,6 @@ HdrHeapSDKHandle::set(const HdrHeapSDKHandle *from) } HdrStrHeap *new_HdrStrHeap(int requested_size); -inkcoreapi HdrHeap *new_HdrHeap(int size = HDR_HEAP_DEFAULT_SIZE); +inkcoreapi HdrHeap *new_HdrHeap(int size = HdrHeap::DEFAULT_SIZE); void hdr_heap_test(); diff --git a/proxy/hdrs/HdrTSOnly.cc b/proxy/hdrs/HdrTSOnly.cc index 6841b1e9837..55e2d9351fb 100644 --- a/proxy/hdrs/HdrTSOnly.cc +++ b/proxy/hdrs/HdrTSOnly.cc @@ -170,30 +170,29 @@ HdrHeap::attach_block(IOBufferBlock *b, const char *use_start) RETRY: - // It's my contention that since heaps are add to the - // first available slot, one you find an empty slot - // it's not possible that a heap ptr for this block - // exists in a later slot - for (int i = 0; i < HDR_BUF_RONLY_HEAPS; i++) { - if (m_ronly_heap[i].m_heap_start == nullptr) { + // It's my contention that since heaps are add to the first available slot, one you find an empty + // slot it's not possible that a heap ptr for this block exists in a later slot + + for (auto &heap : m_ronly_heap) { + if (heap.m_heap_start == nullptr) { // Add block to heap in this slot - m_ronly_heap[i].m_heap_start = (char *)use_start; - m_ronly_heap[i].m_heap_len = (int)(b->end() - b->start()); - m_ronly_heap[i].m_ref_count_ptr = b->data.object(); + heap.m_heap_start = static_cast(use_start); + heap.m_heap_len = static_cast(b->end() - b->start()); + heap.m_ref_count_ptr = b->data.object(); // printf("Attaching block at %X for %d in slot %d\n", // m_ronly_heap[i].m_heap_start, // m_ronly_heap[i].m_heap_len, // i); - return i; - } else if (m_ronly_heap[i].m_heap_start == b->buf()) { + return &heap - m_ronly_heap; + } else if (heap.m_heap_start == b->buf()) { // This block is already on the heap so just extend // it's range - m_ronly_heap[i].m_heap_len = (int)(b->end() - b->buf()); + heap.m_heap_len = static_cast(b->end() - b->buf()); // printf("Extending block at %X to %d in slot %d\n", // m_ronly_heap[i].m_heap_start, // m_ronly_heap[i].m_heap_len, // i); - return i; + return &heap - m_ronly_heap; } } diff --git a/proxy/hdrs/HdrTest.cc b/proxy/hdrs/HdrTest.cc index ee3f93db134..875e4e1a738 100644 --- a/proxy/hdrs/HdrTest.cc +++ b/proxy/hdrs/HdrTest.cc @@ -74,7 +74,6 @@ HdrTest::go(RegressionTest *t, int /* atype ATS_UNUSED */) status = status & test_url(); status = status & test_arena(); status = status & test_regex(); - status = status & test_http_parser_eos_boundary_cases(); status = status & test_http_mutation(); status = status & test_mime(); status = status & test_http(); @@ -517,84 +516,6 @@ HdrTest::test_mime() return (failures_to_status("test_mime", 0)); } -/*------------------------------------------------------------------------- - -------------------------------------------------------------------------*/ - -int -HdrTest::test_http_parser_eos_boundary_cases() -{ - struct { - const char *msg; - int expected_result; - int expected_bytes_consumed; - } tests[] = { - {"GET /index.html HTTP/1.0\r\n", PARSE_RESULT_DONE, 26}, - {"GET /index.html HTTP/1.0\r\n\r\n***BODY****", PARSE_RESULT_DONE, 28}, - {"GET /index.html HTTP/1.0\r\nUser-Agent: foobar\r\n\r\n***BODY****", PARSE_RESULT_DONE, 48}, - {"GET", PARSE_RESULT_ERROR, 3}, - {"GET /index.html", PARSE_RESULT_ERROR, 15}, - {"GET /index.html\r\n", PARSE_RESULT_ERROR, 17}, - {"GET /index.html HTTP/1.0", PARSE_RESULT_ERROR, 24}, - {"GET /index.html HTTP/1.0\r", PARSE_RESULT_ERROR, 25}, - {"GET /index.html HTTP/1.0\n", PARSE_RESULT_DONE, 25}, - {"GET /index.html HTTP/1.0\n\n", PARSE_RESULT_DONE, 26}, - {"GET /index.html HTTP/1.0\r\n\r\n", PARSE_RESULT_DONE, 28}, - {"GET /index.html HTTP/1.0\r\nUser-Agent: foobar", PARSE_RESULT_ERROR, 44}, - {"GET /index.html HTTP/1.0\r\nUser-Agent: foobar\n", PARSE_RESULT_DONE, 45}, - {"GET /index.html HTTP/1.0\r\nUser-Agent: foobar\r\n", PARSE_RESULT_DONE, 46}, - {"GET /index.html HTTP/1.0\r\nUser-Agent: foobar\r\n\r\n", PARSE_RESULT_DONE, 48}, - {"GET /index.html HTTP/1.0\nUser-Agent: foobar\n", PARSE_RESULT_DONE, 44}, - {"GET /index.html HTTP/1.0\nUser-Agent: foobar\nBoo: foo\n", PARSE_RESULT_DONE, 53}, - {"GET /index.html HTTP/1.0\r\nUser-Agent: foobar\r\n", PARSE_RESULT_DONE, 46}, - {"GET /index.html HTTP/1.0\r\n", PARSE_RESULT_DONE, 26}, - {"", PARSE_RESULT_ERROR, 0}, - {nullptr, 0, 0}, - }; - - int i, ret, bytes_consumed; - const char *orig_start; - const char *start; - const char *end; - HTTPParser parser; - - int failures = 0; - - bri_box("test_http_parser_eos_boundary_cases"); - - http_parser_init(&parser); - - for (i = 0; tests[i].msg != nullptr; i++) { - HTTPHdr req_hdr; - - start = tests[i].msg; - end = start + strlen(start); // 1 character past end of string - - req_hdr.create(HTTP_TYPE_REQUEST); - - http_parser_clear(&parser); - - orig_start = start; - ret = req_hdr.parse_req(&parser, &start, end, true); - bytes_consumed = (int)(start - orig_start); - - printf("======== test %d (length=%d, consumed=%d)\n", i, (int)strlen(tests[i].msg), bytes_consumed); - printf("[%s]\n", tests[i].msg); - printf("\n["); - req_hdr.print(nullptr, 0, nullptr, nullptr); - printf("]\n\n"); - - if ((ret != tests[i].expected_result) || (bytes_consumed != tests[i].expected_bytes_consumed)) { - ++failures; - printf("FAILED: test %d: retval , eaten \n\n", i, tests[i].expected_result, ret, - tests[i].expected_bytes_consumed, bytes_consumed); - } - - req_hdr.destroy(); - } - - return (failures_to_status("test_http_parser_eos_boundary_cases", failures)); -} - /*------------------------------------------------------------------------- -------------------------------------------------------------------------*/ diff --git a/proxy/hdrs/HdrTest.h b/proxy/hdrs/HdrTest.h index 91a979c9c4a..f9b29d9fed2 100644 --- a/proxy/hdrs/HdrTest.h +++ b/proxy/hdrs/HdrTest.h @@ -39,9 +39,9 @@ class HdrTest { public: - RegressionTest *rtest; + RegressionTest *rtest = nullptr; - HdrTest() : rtest(nullptr){}; + HdrTest(){}; ~HdrTest(){}; int go(RegressionTest *t, int atype); @@ -51,7 +51,6 @@ class HdrTest int test_parse_date(); int test_format_date(); int test_url(); - int test_http_parser_eos_boundary_cases(); int test_arena(); int test_regex(); int test_accept_language_match(); diff --git a/proxy/hdrs/HdrToken.cc b/proxy/hdrs/HdrToken.cc index 55fe5200a6f..77759924903 100644 --- a/proxy/hdrs/HdrToken.cc +++ b/proxy/hdrs/HdrToken.cc @@ -522,7 +522,7 @@ hdrtoken_tokenize_dfa(const char *string, int string_len, const char **wks_strin { int wks_idx; - wks_idx = hdrtoken_strs_dfa->match(string, string_len); + wks_idx = hdrtoken_strs_dfa->match({string, size_t(string_len)}); if (wks_idx < 0) { wks_idx = -1; diff --git a/proxy/hdrs/HdrToken.h b/proxy/hdrs/HdrToken.h index ab5f4fb5beb..8a50a16f122 100644 --- a/proxy/hdrs/HdrToken.h +++ b/proxy/hdrs/HdrToken.h @@ -137,11 +137,12 @@ hdrtoken_is_valid_wks_idx(int wks_idx) /*------------------------------------------------------------------------- -------------------------------------------------------------------------*/ +// ToDo: This, and dependencies / users should probalby be const HdrTokenHeapPrefix * IMO. inline HdrTokenHeapPrefix * hdrtoken_wks_to_prefix(const char *wks) { ink_assert(hdrtoken_is_wks(wks)); - return ((HdrTokenHeapPrefix *)(wks - sizeof(HdrTokenHeapPrefix))); + return reinterpret_cast(const_cast(wks) - sizeof(HdrTokenHeapPrefix)); } /*------------------------------------------------------------------------- diff --git a/proxy/hdrs/HdrUtils.cc b/proxy/hdrs/HdrUtils.cc index 37a00acda71..f6ee4d4d895 100644 --- a/proxy/hdrs/HdrUtils.cc +++ b/proxy/hdrs/HdrUtils.cc @@ -1,24 +1,21 @@ /** @file - A brief file description + Utilities for HTTP MIME headers. - @section license License + @section license License - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at + Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. + See the NOTICE file distributed with this work for additional information regarding copyright + ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance with the License. You may obtain a + copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. + Unless required by applicable law or agreed to in writing, software distributed under the License + is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + or implied. See the License for the specific language governing permissions and limitations under + the License. */ /**************************************************************************** @@ -34,147 +31,99 @@ #include "tscore/ink_platform.h" #include "HdrUtils.h" -#define GETNEXT() \ - { \ - cur += 1; \ - if (cur >= end) { \ - goto done; \ - } \ - } - void HdrCsvIter::find_csv() { - const char *cur, *end, *last_data, *csv_start; - -RETRY: - - cur = m_csv_start; - end = m_value_start + m_value_len; - last_data = nullptr; - csv_start = nullptr; - - if (cur >= end) { - goto done; - } -skip_leading_whitespace: - if (ParseRules::is_ws(*cur)) { - GETNEXT(); - goto skip_leading_whitespace; - } - csv_start = cur; -parse_value: - // ink_assert((',' > '"') && (',' > ' ') && (',' > '\t')); - // Cookie/Set-Cookie use ';' as the separator - if (m_separator == ',') { - while ((cur < end - 1) && (*cur > ',')) { - last_data = cur; - cur++; + char sep_list[3] = {'"', m_separator, 0}; + + m_csv.clear(); + + while (m_cur_field) { + while (m_value) { + bool in_quote_p = false; + // Index of next interesting character. + TextView::size_type idx = 0; + // Characters of interest in a null terminated string. + while (idx < m_value.size()) { + // Next character of interest. + idx = m_value.find_first_of(sep_list, idx); + if (TextView::npos == idx) { + // no more, consume all of @a src. + break; + } else if ('"' == m_value[idx]) { + if (in_quote_p) { + // quoted-pair only allowed inside a quoted-string. + // Can always back up because if @a in_quote_p there's a least a preceding quote. + if (m_value[idx - 1] != '\\') { + in_quote_p = false; + } + } else { + in_quote_p = true; + } + ++idx; + } else if (m_separator == m_value[idx]) { // separator. + if (in_quote_p) { + // quoted separator, skip and continue. + ++idx; + } else { + // found token, finish up. + break; + } + } + } + + // Trim and then see if there's anything left. + m_csv = m_value.take_prefix_at(idx).trim_if(&ParseRules::is_ws); + if (m_csv && '"' == m_csv[0]) { + m_csv.remove_prefix(1); + } + if (m_csv && '"' == m_csv[m_csv.size() - 1]) { + m_csv.remove_suffix(1); + } + // If there's any value after that, we're done. + if (m_csv) { + return; + } } - } - - if (*cur == m_separator) { - goto done; - } - if (*cur == '\"') { - // If the quote come before any text - // skip it - if (cur == csv_start) { - csv_start++; + // Current field is exhausted, try the next if following dupes. + if (m_follow_dups && m_cur_field->m_next_dup) { + this->field_init(m_cur_field->m_next_dup); + } else { + m_cur_field = nullptr; } - GETNEXT(); - goto parse_value_quote; - } - if (!ParseRules::is_ws(*cur)) { - last_data = cur; } - GETNEXT(); - goto parse_value; -parse_value_quote: - if ((*cur == '\"') && (cur[-1] != '\\')) { - GETNEXT(); - goto parse_value; - } - last_data = cur; - GETNEXT(); - goto parse_value_quote; -done: - m_csv_end = cur; - m_csv_start = csv_start; - - if (last_data) { - m_csv_len = (int)(last_data - csv_start) + 1; - } else { - // Nothing found. See if there is another - // field in the dup list - if (m_cur_field->m_next_dup && m_follow_dups) { - field_init(m_cur_field->m_next_dup); - goto RETRY; - } +} - m_csv_len = 0; +ts::TextView +HdrCsvIter::get_nth(MIMEField *field, int nth, bool follow_dups) +{ + ink_assert(nth >= 0); + int i = 0; + + auto tv = get_first(field, follow_dups); // index zero sub-value. + while (tv && nth > i) { + tv = get_next(); + ++i; } + return tv; } const char * HdrCsvIter::get_nth(MIMEField *field, int *len, int n, bool follow_dups) { - const char *s; - int i, l; - - ink_assert(n >= 0); - i = 0; - - s = get_first(field, &l, follow_dups); // get index 0 - while (s && (n > i)) { - s = get_next(&l); - i++; - } // if had index i, but want n > i, get next - *len = (s ? l : 0); // length is zero if NULL string - return (s); + auto tv = this->get_nth(field, n, follow_dups); + *len = int(tv.size()); + return tv.data(); } int HdrCsvIter::count_values(MIMEField *field, bool follow_dups) { - const char *s; - int count, l; - - count = 0; - s = get_first(field, &l, follow_dups); // get index 0 - while (s) { - s = get_next(&l); + int count = 0; + auto val = get_first(field, follow_dups); // get index 0 + while (val) { + val = get_next(); ++count; - } // get next - return (count); -} - -/* -int main() { - - char* tests[] = {"\"I\", \"hate\", \"strings\"", - "This, is a , test", - "a,b,c,d,e,f,g,h,i,j,k,l,m,n,o,p", - "\"This is,\" a test" - }; - - for (int i = 0; i < 4; i++) { - printf ("Testing: %s\n", tests[i]); - - HdrCsvIter iter; - int len; - const char* v = iter.get_first(tests[i], strlen(tests[i]), &len); - - while (v) { - char* str_v = (char*)ats_malloc(len+1); - memcpy(str_v, v, len); - str_v[len] = '\0'; - printf ("%s|", str_v); - v = iter.get_next(&len); - } - - printf("\n\n"); - } - + } + return count; } -*/ diff --git a/proxy/hdrs/HdrUtils.h b/proxy/hdrs/HdrUtils.h index df40737feb7..2d128c73e75 100644 --- a/proxy/hdrs/HdrUtils.h +++ b/proxy/hdrs/HdrUtils.h @@ -33,68 +33,98 @@ #pragma once +#include "tscpp/util/TextView.h" #include "tscore/ParseRules.h" #include "MIME.h" -// csv = comma separated value +/** Accessor class to iterate over values in a multi-valued field. + * + * This implements the logic for quoted strings as specified in the RFC. + */ class HdrCsvIter { + using TextView = ts::TextView; + public: - // MIME standard separator ',' is used as the default value - // Set-cookie/Cookie uses ';' - HdrCsvIter(const char s = ',') - : m_value_start(nullptr), - m_value_len(0), - m_bytes_consumed(0), - m_follow_dups(false), - m_csv_start(nullptr), - m_csv_len(0), - m_csv_end(nullptr), - m_csv_index(0), - m_cur_field(nullptr), - m_separator(s) - { - } + /** Construct the iterator in the initial state. + * + * @param s The separator character for sub-values. + */ + HdrCsvIter(char s = ',') : m_separator(s) {} + + /** Get the first sub-value. + * + * @param m The multi-valued field. + * @param follow_dups Continue on to duplicate fields flag. + * @return A view of the first sub-value in multi-valued data. + */ + TextView get_first(const MIMEField *m, bool follow_dups = true); const char *get_first(const MIMEField *m, int *len, bool follow_dups = true); + + /** Get the next sub-value. + * + * @return A view of the next subvalue, or an empty view if no more values. + * + * If @a follow_dups was set in the constructor, this will continue on to additional fields + * if those fields have the same name as the original field (e.g, are duplicates). + */ + TextView get_next(); const char *get_next(int *len); + + /** Get the current sub-value. + * + * @return A view of the current subvalue, or an empty view if no more values. + * + * The state of the iterator is not modified. + */ + TextView get_current(); const char *get_current(int *len); + /** Get the @a nth sub-value in the field @a m. + * + * @param m Field. + * @param nth Index of the target sub-value. + * @param follow_dups Follow duplicate fields if necessary. + * @return The subvalue at index @a n, or an empty view if that does not exist. + */ + TextView get_nth(MIMEField *m, int nth, bool follow_dups = true); const char *get_nth(MIMEField *m, int *len, int n, bool follow_dups = true); - int count_values(MIMEField *field, bool follow_dups = true); - int get_index(); + int count_values(MIMEField *field, bool follow_dups = true); - int get_first_int(MIMEField *m, int *valid = nullptr); - int get_next_int(int *valid = nullptr); + /** Get the first sub-value as an integer. + * + * @param m Field with the value. + * @param result [out] Set to the integer sub-value. + * @return @c true if there was an integer and @a result was set, @c false otherwise. + */ + bool get_first_int(MIMEField *m, int &result); + + /** Get the next subvalue as an integer. + * + * @param result [out] Set to the integer sub-value. + * @return @c true if there was an integer and @a result was set, @c false otherwise. + */ + bool get_next_int(int &result); private: void find_csv(); - const char *m_value_start; - int m_value_len; - int m_bytes_consumed; - bool m_follow_dups; - - // m_csv_start - the start of the current comma separated value - // leading white space, and leading quotes have - // been skipped over - // m_csv_len - the length of the current comma separated value - // not including leading whitespace, trailing - // whitespace (unless quoted) or the terminating - // comma, or trailing quotes have been removed - // m_csv_end - the terminating comma of the csv unit. Either - // the terminating comma or the final character - // if this is the last csv in the string - // m_cvs_index - the integer index of current csv starting - // at zero - const char *m_csv_start; - int m_csv_len; - const char *m_csv_end; - int m_csv_index; - const MIMEField *m_cur_field; - - // for the Cookie/Set-cookie headers, the separator is ';' - const char m_separator; + /// The current field value. + TextView m_value; + + /// Whether duplicates are being followed. + bool m_follow_dups = false; + + /// The current sub-value. + TextView m_csv; + + /// The field containing the current sub-value. + const MIMEField *m_cur_field = nullptr; + + /// Separator for sub-values. + /// for the Cookie/Set-cookie headers, the separator is ';' + const char m_separator; // required constructor parameter, no initialization here. void field_init(const MIMEField *m); }; @@ -102,86 +132,83 @@ class HdrCsvIter inline void HdrCsvIter::field_init(const MIMEField *m) { - m_cur_field = m; - m_value_start = m->m_ptr_value; - m_value_len = m->m_len_value; - m_csv_start = m_value_start; + m_cur_field = m; + m_value.assign(m->m_ptr_value, m->m_len_value); } inline const char * HdrCsvIter::get_first(const MIMEField *m, int *len, bool follow_dups) { - field_init(m); + auto tv = this->get_first(m, follow_dups); + *len = static_cast(tv.size()); + return tv.data(); +} +inline ts::TextView +HdrCsvIter::get_first(const MIMEField *m, bool follow_dups) +{ + field_init(m); m_follow_dups = follow_dups; + this->find_csv(); + return m_csv; +} - m_bytes_consumed = 0; - m_csv_index = -1; - - if (m_csv_start) { - find_csv(); - } else { - m_csv_len = 0; - } - - *len = m_csv_len; - return m_csv_start; +inline ts::TextView +HdrCsvIter::get_next() +{ + this->find_csv(); + return m_csv; } inline const char * HdrCsvIter::get_next(int *len) { - if (m_csv_start) { - // Skip past the current csv - m_csv_start = m_csv_end + 1; - find_csv(); - } + auto tv = this->get_next(); + *len = static_cast(tv.size()); + return tv.data(); +} - *len = m_csv_len; - return m_csv_start; +inline ts::TextView +HdrCsvIter::get_current() +{ + return m_csv; } inline const char * HdrCsvIter::get_current(int *len) { - *len = m_csv_len; - return m_csv_start; + *len = static_cast(m_csv.size()); + return m_csv.data(); } -inline int -HdrCsvIter::get_first_int(MIMEField *m, int *valid) +inline bool +HdrCsvIter::get_first_int(MIMEField *m, int &result) { - int len; - const char *r = get_first(m, &len); - - if (r) { - if (valid) { - *valid = 1; + auto val = this->get_first(m); + + if (val) { + TextView parsed; + int n = ts::svtoi(val, &parsed); + if (parsed) { + result = n; + return true; } - return ink_atoi(r, len); - } else { - if (valid) { - *valid = 0; - } - return 0; } + return false; } -inline int -HdrCsvIter::get_next_int(int *valid) +inline bool +HdrCsvIter::get_next_int(int &result) { - int len; - const char *r = get_next(&len); - - if (r) { - if (valid) { - *valid = 1; - } - return ink_atoi(r, len); - } else { - if (valid) { - *valid = 0; + auto val = this->get_next(); + + if (val) { + TextView parsed; + int n = ts::svtoi(val, &parsed); + if (parsed) { + result = n; + return true; } - return 0; } + return false; } diff --git a/proxy/hdrs/HttpCompat.h b/proxy/hdrs/HttpCompat.h index 3bf0dc334a3..8c237995f83 100644 --- a/proxy/hdrs/HttpCompat.h +++ b/proxy/hdrs/HttpCompat.h @@ -25,7 +25,6 @@ #include "tscore/ink_string++.h" #include "MIME.h" -#include "tscore/RawHashTable.h" #include "tscore/Diags.h" class HttpCompat diff --git a/proxy/hdrs/MIME.cc b/proxy/hdrs/MIME.cc index c2a22c8c257..73d5b2bec74 100644 --- a/proxy/hdrs/MIME.cc +++ b/proxy/hdrs/MIME.cc @@ -34,6 +34,8 @@ #include "HdrUtils.h" #include "HttpCompat.h" +using ts::TextView; + /*********************************************************************** * * * C O M P I L E O P T I O N S * @@ -1476,9 +1478,8 @@ mime_hdr_field_attach(MIMEHdrImpl *mh, MIMEField *field, int check_for_dups, MIM ////////////////////////////////////////////////// if (check_for_dups || (prev_dup && (!prev_dup->is_dup_head()))) { - int length; - const char *name = mime_field_name_get(field, &length); - prev_dup = mime_hdr_field_find(mh, name, length); + std::string_view name{field->name_get()}; + prev_dup = mime_hdr_field_find(mh, name.data(), static_cast(name.size())); ink_assert((prev_dup == nullptr) || (prev_dup->is_dup_head())); } @@ -1595,9 +1596,8 @@ mime_hdr_field_detach(MIMEHdrImpl *mh, MIMEField *field, bool detach_all_dups) } } else // need to walk list to find and patch out from predecessor { - int name_length; - const char *name = mime_field_name_get(field, &name_length); - MIMEField *prev = mime_hdr_field_find(mh, name, name_length); + std::string_view name{field->name_get()}; + MIMEField *prev = mime_hdr_field_find(mh, name.data(), static_cast(name.size())); while (prev && (prev->m_next_dup != field)) { prev = prev->m_next_dup; @@ -1726,15 +1726,13 @@ mime_field_destroy(MIMEHdrImpl * /* mh ATS_UNUSED */, MIMEField *field) field->m_readiness = MIME_FIELD_SLOT_READINESS_DELETED; } -const char * -mime_field_name_get(const MIMEField *field, int *length) +std::string_view +MIMEField::name_get() const { - *length = field->m_len_name; - if (field->m_wks_idx >= 0) { - return hdrtoken_index_to_wks(field->m_wks_idx); - } else { - return field->m_ptr_name; + if (m_wks_idx >= 0) { + return {hdrtoken_index_to_wks(m_wks_idx), m_len_name}; } + return {m_ptr_name, m_len_name}; } void @@ -1781,45 +1779,42 @@ MIMEField::value_get_index(const char *value, int length) const return retval; } -const char * -mime_field_value_get(const MIMEField *field, int *length) +std::string_view +MIMEField::value_get() const { - *length = field->m_len_value; - return field->m_ptr_value; + return {m_ptr_value, m_len_value}; } int32_t mime_field_value_get_int(const MIMEField *field) { - int length; - const char *str = mime_field_value_get(field, &length); + std::string_view value{field->value_get()}; - return mime_parse_int(str, str + length); + return mime_parse_int(value.data(), value.data() + value.size()); } uint32_t mime_field_value_get_uint(const MIMEField *field) { - int length; - const char *str = mime_field_value_get(field, &length); - return mime_parse_uint(str, str + length); + std::string_view value{field->value_get()}; + + return mime_parse_uint(value.data(), value.data() + value.size()); } int64_t mime_field_value_get_int64(const MIMEField *field) { - int length; - const char *str = mime_field_value_get(field, &length); + std::string_view value{field->value_get()}; - return mime_parse_int64(str, str + length); + return mime_parse_int64(value.data(), value.data() + value.size()); } time_t mime_field_value_get_date(const MIMEField *field) { - int length; - const char *str = mime_field_value_get(field, &length); - return mime_parse_date(str, str + length); + std::string_view value{field->value_get()}; + + return mime_parse_date(value.data(), value.data() + value.size()); } const char * @@ -1828,10 +1823,9 @@ mime_field_value_get_comma_val(const MIMEField *field, int *length, int idx) // some fields (like Date) contain commas but should not be ripped apart if (!field->supports_commas()) { if (idx == 0) { - return mime_field_value_get(field, length); - } else { - return nullptr; + return field->value_get(length); } + return nullptr; } else { Str *str; StrList list(false); @@ -1864,16 +1858,13 @@ mime_field_value_get_comma_val_count(const MIMEField *field) int mime_field_value_get_comma_list(const MIMEField *field, StrList *list) { - const char *str; - int len; - - str = mime_field_value_get(field, &len); + std::string_view value{field->value_get()}; // if field doesn't support commas, don't rip apart. if (!field->supports_commas()) { - list->append_string(str, len); + list->append_string(value.data(), static_cast(value.size())); } else { - HttpCompat::parse_tok_list(list, 1, str, len, ','); + HttpCompat::parse_tok_list(list, 1, value.data(), static_cast(value.size()), ','); } return list->count; @@ -2313,215 +2304,161 @@ MIMEHdr::get_host_port_values(const char **host_ptr, ///< Pointer to host. * P A R S E R * * * ***********************************************************************/ -void -_mime_scanner_init(MIMEScanner *scanner) -{ - scanner->m_line = nullptr; - scanner->m_line_size = 0; - scanner->m_line_length = 0; - scanner->m_state = MIME_PARSE_BEFORE; -} -////////////////////////////////////////////////////// -// init first time structure setup // -// clear resets an already-initialized structure // -////////////////////////////////////////////////////// void -mime_scanner_init(MIMEScanner *scanner) +MIMEScanner::init() { - _mime_scanner_init(scanner); + m_state = INITIAL_PARSE_STATE; + // Ugly, but required because of how proxy allocation works - that leaves the instance in a + // random state, so even assigning to it can crash. Because this method substitutes for a real + // constructor in the proxy allocation system, call the CTOR here. Any memory that gets allocated + // is supposed to be cleaned up by calling @c clear on this object. + new (&m_line) std::string; } -// clear is to reset an already initialized structure -void -mime_scanner_clear(MIMEScanner *scanner) +MIMEScanner & +MIMEScanner::append(TextView text) { - ats_free(scanner->m_line); - _mime_scanner_init(scanner); -} - -void -mime_scanner_append(MIMEScanner *scanner, const char *data, int data_size) -{ - int free_size = scanner->m_line_size - scanner->m_line_length; - - ////////////////////////////////////////////////////// - // if not enough space, allocate or grow the buffer // - ////////////////////////////////////////////////////// - if (data_size > free_size) { // need to allocate/grow the buffer - if (scanner->m_line_size == 0) { // buffer should be at least 128 bytes - scanner->m_line_size = 128; - } - - while (free_size < data_size) { // grow buffer by powers of 2 - scanner->m_line_size *= 2; - free_size = scanner->m_line_size - scanner->m_line_length; - } - - if (scanner->m_line == nullptr) { // if no buffer yet, allocate one - scanner->m_line = (char *)ats_malloc(scanner->m_line_size); - } else { - scanner->m_line = (char *)ats_realloc(scanner->m_line, scanner->m_line_size); - } - } - //////////////////////////////////////////////// - // append new data onto the end of the buffer // - //////////////////////////////////////////////// - - memcpy(&(scanner->m_line[scanner->m_line_length]), data, data_size); - scanner->m_line_length += data_size; + m_line += text; + return *this; } ParseResult -mime_scanner_get(MIMEScanner *S, const char **raw_input_s, const char *raw_input_e, const char **output_s, const char **output_e, - bool *output_shares_raw_input, - bool raw_input_eof, ///< All data has been received for this header. - int raw_input_scan_type) +MIMEScanner::get(TextView &input, TextView &output, bool &output_shares_input, bool eof_p, ScanType scan_type) { - const char *raw_input_c, *lf_ptr; ParseResult zret = PARSE_RESULT_CONT; // Need this for handling dangling CR. - static const char RAW_CR = ParseRules::CHAR_CR; - - ink_assert((raw_input_s != nullptr) && (*raw_input_s != nullptr)); - ink_assert(raw_input_e != nullptr); + static const char RAW_CR{ParseRules::CHAR_CR}; - raw_input_c = *raw_input_s; - - while (PARSE_RESULT_CONT == zret && raw_input_c < raw_input_e) { - ptrdiff_t runway = raw_input_e - raw_input_c; // remaining input. - switch (S->m_state) { + auto text = input; + while (PARSE_RESULT_CONT == zret && !text.empty()) { + switch (m_state) { case MIME_PARSE_BEFORE: // waiting to find a field. - if (ParseRules::is_cr(*raw_input_c)) { - ++raw_input_c; - if (runway >= 2 && ParseRules::is_lf(*raw_input_c)) { + m_line.resize(0); // any caller should already be done with the buffer + if (ParseRules::is_cr(*text)) { + ++text; + if (!text.empty() && ParseRules::is_lf(*text)) { // optimize a bit - this happens >99% of the time after a CR. - ++raw_input_c; + ++text; zret = PARSE_RESULT_DONE; } else { - S->m_state = MIME_PARSE_FOUND_CR; + m_state = MIME_PARSE_FOUND_CR; } - } else if (ParseRules::is_lf(*raw_input_c)) { - ++raw_input_c; + } else if (ParseRules::is_lf(*text)) { + ++text; zret = PARSE_RESULT_DONE; // Required by regression test. } else { // consume this character in the next state. - S->m_state = MIME_PARSE_INSIDE; + m_state = MIME_PARSE_INSIDE; } break; case MIME_PARSE_FOUND_CR: - // Looking for a field and found a CR, which should mean terminating - // the header. Note that we've left the CR in the input so we have - // to skip over it. - if (ParseRules::is_lf(*raw_input_c)) { - // Header terminated. - ++raw_input_c; + // Looking for a field and found a CR, which should mean terminating the header. + if (ParseRules::is_lf(*text)) { + ++text; zret = PARSE_RESULT_DONE; } else { - // This really should be an error (spec doesn't permit lone CR) - // but the regression tests require it. - mime_scanner_append(S, &RAW_CR, 1); - S->m_state = MIME_PARSE_INSIDE; + // This really should be an error (spec doesn't permit lone CR) but the regression tests + // require it. + this->append({&RAW_CR, 1}); + m_state = MIME_PARSE_INSIDE; } break; - case MIME_PARSE_INSIDE: - lf_ptr = static_cast(memchr(raw_input_c, ParseRules::CHAR_LF, runway)); - if (lf_ptr) { - raw_input_c = lf_ptr + 1; - if (MIME_SCANNER_TYPE_LINE == raw_input_scan_type) { - zret = PARSE_RESULT_OK; - S->m_state = MIME_PARSE_BEFORE; + case MIME_PARSE_INSIDE: { + auto lf_off = text.find(ParseRules::CHAR_LF); + if (lf_off != TextView::npos) { + text.remove_prefix(lf_off + 1); // drop up to and including LF + if (LINE == scan_type) { + zret = PARSE_RESULT_OK; + m_state = MIME_PARSE_BEFORE; } else { - S->m_state = MIME_PARSE_AFTER; + m_state = MIME_PARSE_AFTER; // looking for line folding. } - } else { - raw_input_c = raw_input_e; // grab all that's available. + } else { // no EOL, consume all text without changing state. + text.remove_prefix(text.size()); } - break; + } break; case MIME_PARSE_AFTER: - // After a LF. Might be the end or a continuation. - if (ParseRules::is_ws(*raw_input_c)) { - char *unfold = const_cast(raw_input_c - 1); - - *unfold-- = ' '; + // After a LF, the next line might be a continuation / folded line. That's indicated by a + // starting whitespace. If that's the case, back up over the preceding CR/LF with space and + // pretend it's the same line. + if (ParseRules::is_ws(*text)) { // folded line. + char *unfold = const_cast(text.data() - 1); + *unfold-- = ' '; if (ParseRules::is_cr(*unfold)) { *unfold = ' '; } - S->m_state = MIME_PARSE_INSIDE; // back inside the field. + m_state = MIME_PARSE_INSIDE; // back inside the field. } else { - S->m_state = MIME_PARSE_BEFORE; // field terminated. - zret = PARSE_RESULT_OK; + m_state = MIME_PARSE_BEFORE; // field terminated. + zret = PARSE_RESULT_OK; } break; } } - ptrdiff_t data_size = raw_input_c - *raw_input_s; + TextView parsed_text{input.data(), text.data()}; + bool save_parsed_text_p = !parsed_text.empty(); if (PARSE_RESULT_CONT == zret) { - // data ran out before we got a clear final result. - // There a number of things we need to check and possibly adjust - // that result. It's less complex to do this cleanup than handle - // in the parser state machine. - if (raw_input_eof) { + // data ran out before we got a clear final result. There a number of things we need to check + // and possibly adjust that result. It's less complex to do this cleanup than handle all of + // these checks in the parser state machine. + if (eof_p) { // Should never return PARSE_CONT if we've hit EOF. - if (0 == data_size) { + if (parsed_text.empty()) { // all input previously consumed. If we're between fields, that's cool. - if (MIME_PARSE_INSIDE != S->m_state) { - S->m_state = MIME_PARSE_BEFORE; // probably not needed... - zret = PARSE_RESULT_DONE; + if (MIME_PARSE_INSIDE != m_state) { + m_state = MIME_PARSE_BEFORE; // probably not needed... + zret = PARSE_RESULT_DONE; } else { zret = PARSE_RESULT_ERROR; // unterminated field. } - } else if (MIME_PARSE_AFTER == S->m_state) { - // Special case it seems - need to accept the final field - // even if there's no header terminating CR LF. We check for - // absolute end of input because otherwise this might be - // a multiline field where we haven't seen the next leading space. - S->m_state = MIME_PARSE_BEFORE; - zret = PARSE_RESULT_OK; + } else if (MIME_PARSE_AFTER == m_state) { + // Special case it seems - need to accept the final field even if there's no header + // terminating CR LF. This is only reasonable after absolute end of input (EOF) because + // otherwise this might be a multiline field where we haven't seen the next leading space. + m_state = MIME_PARSE_BEFORE; + zret = PARSE_RESULT_OK; } else { // Partial input, no field / line CR LF zret = PARSE_RESULT_ERROR; // Unterminated field. } - } else if (data_size) { - if (MIME_PARSE_INSIDE == S->m_state) { + } else if (!parsed_text.empty()) { + if (MIME_PARSE_INSIDE == m_state) { // Inside a field but more data is expected. Save what we've got. - mime_scanner_append(S, *raw_input_s, data_size); - data_size = 0; // Don't append again. - } else if (MIME_PARSE_AFTER == S->m_state) { + this->append(parsed_text); // Do this here to force appending. + save_parsed_text_p = false; // don't double append. + } else if (MIME_PARSE_AFTER == m_state) { // After a field but we still have data. Need to parse it too. - S->m_state = MIME_PARSE_BEFORE; - zret = PARSE_RESULT_OK; + m_state = MIME_PARSE_BEFORE; + zret = PARSE_RESULT_OK; } } } - if (data_size && S->m_line_length) { + if (save_parsed_text_p && !m_line.empty()) { // If we're already accumulating, continue to do so if we have data. - mime_scanner_append(S, *raw_input_s, data_size); + this->append(parsed_text); } - // No sharing if we've accumulated data (really, force this to make compiler shut up). - *output_shares_raw_input = 0 == S->m_line_length; // adjust out arguments. + output_shares_input = true; if (PARSE_RESULT_CONT != zret) { - if (0 != S->m_line_length) { - *output_s = S->m_line; - *output_e = *output_s + S->m_line_length; - S->m_line_length = 0; + if (!m_line.empty()) { + output = m_line; // cleared when called with state MIME_PARSE_BEFORE + output_shares_input = false; } else { - *output_s = *raw_input_s; - *output_e = raw_input_c; + output = parsed_text; } } - // Make sure there are no '\0' in the input scanned so far - if (zret != PARSE_RESULT_ERROR && memchr(*raw_input_s, '\0', raw_input_c - *raw_input_s) != nullptr) { + // Make sure there are no null characters in the input scanned so far + if (zret != PARSE_RESULT_ERROR && TextView::npos != parsed_text.find('\0')) { zret = PARSE_RESULT_ERROR; } - *raw_input_s = raw_input_c; // mark input consumed. + input.remove_prefix(parsed_text.size()); return zret; } @@ -2539,14 +2476,14 @@ _mime_parser_init(MIMEParser *parser) void mime_parser_init(MIMEParser *parser) { - mime_scanner_init(&parser->m_scanner); + parser->m_scanner.init(); _mime_parser_init(parser); } void mime_parser_clear(MIMEParser *parser) { - mime_scanner_clear(&parser->m_scanner); + parser->m_scanner.clear(); _mime_parser_init(parser); } @@ -2556,17 +2493,6 @@ mime_parser_parse(MIMEParser *parser, HdrHeap *heap, MIMEHdrImpl *mh, const char { ParseResult err; bool line_is_real; - const char *colon; - const char *line_c; - const char *line_s = nullptr; - const char *line_e = nullptr; - const char *field_name_first; - const char *field_name_last; - const char *field_value_first; - const char *field_value_last; - const char *field_line_first; - const char *field_line_last; - int field_name_length, field_value_length; MIMEScanner *scanner = &parser->m_scanner; @@ -2575,22 +2501,23 @@ mime_parser_parse(MIMEParser *parser, HdrHeap *heap, MIMEHdrImpl *mh, const char // get a name:value line, with all continuation lines glued into one line // //////////////////////////////////////////////////////////////////////////// - err = mime_scanner_get(scanner, real_s, real_e, &line_s, &line_e, &line_is_real, eof, MIME_SCANNER_TYPE_FIELD); + TextView text{*real_s, real_e}; + TextView parsed; + err = scanner->get(text, parsed, line_is_real, eof, MIMEScanner::FIELD); + *real_s = text.data(); if (err != PARSE_RESULT_OK) { return err; } - line_c = line_s; - ////////////////////////////////////////////////// // if got a LF or CR on its own, end the header // ////////////////////////////////////////////////// - if ((line_e - line_c >= 2) && (line_c[0] == ParseRules::CHAR_CR) && (line_c[1] == ParseRules::CHAR_LF)) { + if ((parsed.size() >= 2) && (parsed[0] == ParseRules::CHAR_CR) && (parsed[1] == ParseRules::CHAR_LF)) { return PARSE_RESULT_DONE; } - if ((line_e - line_c >= 1) && (line_c[0] == ParseRules::CHAR_LF)) { + if ((parsed.size() >= 1) && (parsed[0] == ParseRules::CHAR_LF)) { return PARSE_RESULT_DONE; } @@ -2598,27 +2525,23 @@ mime_parser_parse(MIMEParser *parser, HdrHeap *heap, MIMEHdrImpl *mh, const char // find pointers into the name:value field // ///////////////////////////////////////////// - field_line_first = line_c; - field_line_last = line_e - 1; - - // find name first - field_name_first = line_c; /** * Fix for INKqa09141. The is_token function fails for '@' character. * Header names starting with '@' signs are valid headers. Hence we * have to add one more check to see if the first parameter is '@' * character then, the header name is valid. **/ - if ((!ParseRules::is_token(*field_name_first)) && (*field_name_first != '@')) { + if ((!ParseRules::is_token(*parsed)) && (*parsed != '@')) { continue; // toss away garbage line } // find name last - colon = (char *)memchr(line_c, ':', (line_e - line_c)); - if (!colon) { + auto field_value = parsed; // need parsed as is later on. + auto field_name = field_value.split_prefix_at(':'); + if (field_name.empty()) { continue; // toss away garbage line } - field_name_last = colon - 1; + // RFC7230 section 3.2.4: // No whitespace is allowed between the header field-name and colon. In // the past, differences in the handling of such whitespace have led to @@ -2626,57 +2549,44 @@ mime_parser_parse(MIMEParser *parser, HdrHeap *heap, MIMEHdrImpl *mh, const char // server MUST reject any received request message that contains // whitespace between a header field-name and colon with a response code // of 400 (Bad Request). - if ((field_name_last >= field_name_first) && is_ws(*field_name_last)) { + if (is_ws(field_name.back())) { return PARSE_RESULT_ERROR; } // find value first - field_value_first = colon + 1; - while ((field_value_first < line_e) && is_ws(*field_value_first)) { - ++field_value_first; - } - - // find_value_last - field_value_last = line_e - 1; - while ((field_value_last >= field_value_first) && ParseRules::is_wslfcr(*field_value_last)) { - --field_value_last; - } - - field_name_length = (int)(field_name_last - field_name_first + 1); - field_value_length = (int)(field_value_last - field_value_first + 1); + field_value.ltrim_if(&ParseRules::is_ws); + field_value.rtrim_if(&ParseRules::is_wslfcr); // Make sure the name or value is not longer than 64K - if (field_name_length >= UINT16_MAX || field_value_length >= UINT16_MAX) { + if (field_name.size() >= UINT16_MAX || field_value.size() >= UINT16_MAX) { return PARSE_RESULT_ERROR; } - int total_line_length = (int)(field_line_last - field_line_first + 1); + // int total_line_length = (int)(field_line_last - field_line_first + 1); ////////////////////////////////////////////////////////////////////// // if we can't leave the name & value in the real buffer, copy them // ////////////////////////////////////////////////////////////////////// if (must_copy_strings || (!line_is_real)) { - int length = total_line_length; - char *dup = heap->duplicate_str(field_name_first, length); - intptr_t delta = dup - field_name_first; - - field_name_first += delta; - field_value_first += delta; + char *dup = heap->duplicate_str(parsed.data(), parsed.size()); + ptrdiff_t delta = dup - parsed.data(); + field_name.assign(field_name.data() + delta, field_name.size()); + field_value.assign(field_value.data() + delta, field_value.size()); } /////////////////////// // tokenize the name // /////////////////////// - int field_name_wks_idx = hdrtoken_tokenize(field_name_first, field_name_length); + int field_name_wks_idx = hdrtoken_tokenize(field_name.data(), field_name.size()); /////////////////////////////////////////// // build and insert the new field object // /////////////////////////////////////////// MIMEField *field = mime_field_create(heap, mh); - mime_field_name_value_set(heap, mh, field, field_name_wks_idx, field_name_first, field_name_length, field_value_first, - field_value_length, true, total_line_length, false); + mime_field_name_value_set(heap, mh, field, field_name_wks_idx, field_name.data(), field_name.size(), field_value.data(), + field_value.size(), true, parsed.size(), false); mime_hdr_field_attach(mh, field, 1, nullptr); } } @@ -3151,12 +3061,18 @@ mime_parse_int(const char *buf, const char *end) return 0; } - if (is_digit(*buf)) // fast case - { + if (is_digit(*buf)) { // fast case num = *buf++ - '0'; while ((buf != end) && is_digit(*buf)) { - num = (num * 10) + (*buf++ - '0'); + if (num != INT_MAX) { + int new_num = (num * 10) + (*buf++ - '0'); + + num = (new_num < num ? INT_MAX : new_num); // Check for overflow + } else { + ++buf; // Skip the remaining (valid) digits since we reached MAX/MIN_INT + } } + return num; } else { num = 0; @@ -3173,7 +3089,13 @@ mime_parse_int(const char *buf, const char *end) // NOTE: we first compute the value as negative then correct the // sign back to positive. This enables us to correctly parse MININT. while ((buf != end) && is_digit(*buf)) { - num = (num * 10) - (*buf++ - '0'); + if (num != INT_MIN) { + int new_num = (num * 10) - (*buf++ - '0'); + + num = (new_num > num ? INT_MIN : new_num); // Check for overflow, so to speak, see above re: negative + } else { + ++buf; // Skip the remaining (valid) digits since we reached MAX/MIN_INT + } } if (!negative) { @@ -3293,6 +3215,7 @@ int mime_parse_rfc822_date_fastcase(const char *buf, int length, struct tm *tp) { unsigned int three_char_wday, three_char_mon; + std::string_view view{buf, size_t(length)}; ink_assert(length >= 29); ink_assert(!is_ws(buf[0])); @@ -3323,7 +3246,7 @@ mime_parse_rfc822_date_fastcase(const char *buf, int length, struct tm *tp) } } if (tp->tm_wday < 0) { - tp->tm_wday = day_names_dfa->match(buf, length); + tp->tm_wday = day_names_dfa->match(view); if (tp->tm_wday < 0) { return 0; } @@ -3376,7 +3299,7 @@ mime_parse_rfc822_date_fastcase(const char *buf, int length, struct tm *tp) } } if (tp->tm_mon < 0) { - tp->tm_mon = month_names_dfa->match(buf, length); + tp->tm_mon = month_names_dfa->match(view); if (tp->tm_mon < 0) { return 0; } @@ -3507,7 +3430,7 @@ mime_parse_date(const char *buf, const char *end) return t; } -int +bool mime_parse_day(const char *&buf, const char *end, int *day) { const char *e; @@ -3521,16 +3444,16 @@ mime_parse_day(const char *&buf, const char *end, int *day) e += 1; } - *day = day_names_dfa->match(buf, e - buf); + *day = day_names_dfa->match({buf, size_t(e - buf)}); if (*day < 0) { - return 0; + return false; } else { buf = e; - return 1; + return true; } } -int +bool mime_parse_month(const char *&buf, const char *end, int *month) { const char *e; @@ -3544,22 +3467,22 @@ mime_parse_month(const char *&buf, const char *end, int *month) e += 1; } - *month = month_names_dfa->match(buf, e - buf); + *month = month_names_dfa->match({buf, size_t(e - buf)}); if (*month < 0) { - return 0; + return false; } else { buf = e; - return 1; + return true; } } -int +bool mime_parse_mday(const char *&buf, const char *end, int *mday) { return mime_parse_integer(buf, end, mday); } -int +bool mime_parse_year(const char *&buf, const char *end, int *year) { int val; @@ -3569,7 +3492,7 @@ mime_parse_year(const char *&buf, const char *end, int *year) } if ((buf == end) || (*buf == '\0')) { - return 0; + return false; } val = 0; @@ -3586,59 +3509,88 @@ mime_parse_year(const char *&buf, const char *end, int *year) *year = val; - return 1; + return true; } -int +bool mime_parse_time(const char *&buf, const char *end, int *hour, int *min, int *sec) { if (!mime_parse_integer(buf, end, hour)) { - return 0; + return false; } if (!mime_parse_integer(buf, end, min)) { - return 0; + return false; } if (!mime_parse_integer(buf, end, sec)) { - return 0; + return false; } - return 1; + return true; } -// TODO: Do we really need mime_parse_int() and mime_parse_integer() ? I know -// they have slightly different prototypes, but still... -int +// This behaves slightly different than mime_parse_int(), int that we actually +// return a "bool" for success / failure on "reasonable" parsing. This kinda +// dumb, because we have two interfaces, where one does not move along the +// buf pointer, but this one does (and the ones using this function do). +bool mime_parse_integer(const char *&buf, const char *end, int *integer) { - int val; - bool negative; - - negative = false; - while ((buf != end) && *buf && !is_digit(*buf) && (*buf != '-')) { buf += 1; } if ((buf == end) || (*buf == '\0')) { - return 0; + return false; } - if (*buf == '-') { - negative = true; - buf += 1; - } + int32_t num; + bool negative; - val = 0; - while ((buf != end) && is_digit(*buf)) { - val = (val * 10) + (*buf++ - '0'); - } + // This code is copied verbatim from mime_parse_int ... Sigh. Maybe amc is right, and + // we really need to clean this up. But, as such, we should redo all these interfaces, + // and that's a big undertaking (and we'd want to move these strings all to string_view's). + if (is_digit(*buf)) { // fast case + num = *buf++ - '0'; + while ((buf != end) && is_digit(*buf)) { + if (num != INT_MAX) { + int new_num = (num * 10) + (*buf++ - '0'); - if (negative) { - *integer = -val; + num = (new_num < num ? INT_MAX : new_num); // Check for overflow + } else { + ++buf; // Skip the remaining (valid) digits since we reached MAX/MIN_INT + } + } } else { - *integer = val; + num = 0; + negative = false; + + while ((buf != end) && ParseRules::is_space(*buf)) { + buf += 1; + } + + if ((buf != end) && (*buf == '-')) { + negative = true; + buf += 1; + } + // NOTE: we first compute the value as negative then correct the + // sign back to positive. This enables us to correctly parse MININT. + while ((buf != end) && is_digit(*buf)) { + if (num != INT_MIN) { + int new_num = (num * 10) - (*buf++ - '0'); + + num = (new_num > num ? INT_MIN : new_num); // Check for overflow, so to speak, see above re: negative + } else { + ++buf; // Skip the remaining (valid) digits since we reached MAX/MIN_INT + } + } + + if (!negative) { + num = -num; + } } - return 1; + *integer = num; + + return true; } /*********************************************************************** diff --git a/proxy/hdrs/MIME.h b/proxy/hdrs/MIME.h index d95f147d9f8..6ff23f77220 100644 --- a/proxy/hdrs/MIME.h +++ b/proxy/hdrs/MIME.h @@ -24,6 +24,8 @@ #pragma once #include +#include +#include #include "tscore/ink_assert.h" #include "tscore/ink_apidefs.h" @@ -32,6 +34,8 @@ #include "HdrHeap.h" #include "HdrToken.h" +#include "tscpp/util/TextView.h" + /*********************************************************************** * * * Defines * @@ -57,9 +61,6 @@ enum MimeParseState { MIME_PARSE_AFTER, ///< After a field. }; -#define MIME_SCANNER_TYPE_LINE 0 -#define MIME_SCANNER_TYPE_FIELD 1 - /*********************************************************************** * * * Assertions * @@ -116,7 +117,7 @@ struct MIMEField { bool is_cooked() { - return (m_flags & MIME_FIELD_SLOT_FLAGS_COOKED); + return (m_flags & MIME_FIELD_SLOT_FLAGS_COOKED) ? true : false; } bool @@ -134,12 +135,14 @@ struct MIMEField { bool supports_commas() const { - if (m_wks_idx >= 0) + if (m_wks_idx >= 0) { return (hdrtoken_index_to_flags(m_wks_idx) & MIME_FLAGS_COMMAS); - else - return true; // by default, assume supports commas + } + return true; // by default, assume supports commas } + /// @return The name of @a this field. + std::string_view name_get() const; const char *name_get(int *length) const; /** Find the index of the value in the multi-value field. @@ -156,7 +159,10 @@ struct MIMEField { */ int value_get_index(const char *value, int length) const; + /// @return The value of @a this field. + std::string_view value_get() const; const char *value_get(int *length) const; + int32_t value_get_int() const; uint32_t value_get_uint() const; int64_t value_get_int64() const; @@ -277,13 +283,75 @@ struct MIMEHdrImpl : public HdrHeapObjImpl { * * ***********************************************************************/ +/** A pre-parser used to extract MIME "lines" from raw input for further parsing. + * + * This maintains an internal line buffer which is used to keeping content between calls + * when the parse has not yet completed. + * + */ struct MIMEScanner { - char *m_line; // buffered line being built up - int m_line_length; // size of real live data in buffer - int m_line_size; // total allocated size of buffer - MimeParseState m_state; ///< Parsing machine state. + using self_type = MIMEScanner; ///< Self reference type. +public: + /// Type of input scanning. + enum ScanType { + LINE = 0, ///< Scan a single line. + FIELD = 1, ///< Scan with line folding enabled. + }; + + void init(); ///< Pseudo-constructor required by proxy allocation. + void clear(); ///< Pseudo-destructor required by proxy allocation. + + /// @return The size of the internal line buffer. + size_t get_buffered_line_size() const; + + /** Scan @a input for MIME data delimited by CR/LF end of line markers. + * + * @param input [in,out] Text to scan. + * @param output [out] Parsed text from @a input, if any. + * @param output_shares_input [out] Whether @a output is in @a input. + * @param eof_p [in] The source for @a input is done, no more data will ever be available. + * @param scan_type [in] Whether to check for line folding. + * @return The result of scanning. + * + * @a input is updated to remove text that was scanned. @a output is updated to be a view of the + * scanned @a input. This is separate because @a output may be a view of @a input or a view of the + * internal line buffer. Which of these cases obtains is returned in @a output_shares_input. This + * is @c true if @a output is a view of @a input, and @c false if @a output is a view of the + * internal buffer, but is only set if the result is not @c PARSE_RESULT_CONT (that is, it is not + * set until scanning has completed). If @a scan_type is @c FIELD then folded lines are + * accumulated in to a single line stored in the internal buffer. Otherwise the scanning + * terminates at the first CR/LF. + */ + ParseResult get(ts::TextView &input, ts::TextView &output, bool &output_shares_input, bool eof_p, ScanType scan_type); + +protected: + /** Append @a text to the internal buffer. + * + * @param text Text to append. + * @return @a this + * + * A copy of @a text is appended to the internal line buffer. + */ + self_type &append(ts::TextView text); + + static constexpr MimeParseState INITIAL_PARSE_STATE = MIME_PARSE_BEFORE; + std::string m_line; ///< Internally buffered line data for field coalescence. + MimeParseState m_state{INITIAL_PARSE_STATE}; ///< Parsing machine state. }; +inline size_t +MIMEScanner::get_buffered_line_size() const +{ + return m_line.size(); +} + +inline void +MIMEScanner::clear() +{ + m_line.clear(); + m_state = INITIAL_PARSE_STATE; +} + struct MIMEParser { MIMEScanner m_scanner; int32_t m_field; @@ -654,11 +722,9 @@ inkcoreapi MIMEField *mime_hdr_prepare_for_value_set(HdrHeap *heap, MIMEHdrImpl void mime_field_destroy(MIMEHdrImpl *mh, MIMEField *field); -const char *mime_field_name_get(const MIMEField *field, int *length); void mime_field_name_set(HdrHeap *heap, MIMEHdrImpl *mh, MIMEField *field, int16_t name_wks_idx_or_neg1, const char *name, int length, bool must_copy_string); -inkcoreapi const char *mime_field_value_get(const MIMEField *field, int *length); int32_t mime_field_value_get_int(const MIMEField *field); uint32_t mime_field_value_get_uint(const MIMEField *field); int64_t mime_field_value_get_int64(const MIMEField *field); @@ -688,12 +754,6 @@ void mime_field_name_value_set(HdrHeap *heap, MIMEHdrImpl *mh, MIMEField *field, void mime_field_value_append(HdrHeap *heap, MIMEHdrImpl *mh, MIMEField *field, const char *value, int length, bool prepend_comma, const char separator); -void mime_scanner_init(MIMEScanner *scanner); -void mime_scanner_clear(MIMEScanner *scanner); -void mime_scanner_append(MIMEScanner *scanner, const char *data, int data_size); -ParseResult mime_scanner_get(MIMEScanner *S, const char **raw_input_s, const char *raw_input_e, const char **output_s, - const char **output_e, bool *output_shares_raw_input, bool raw_input_eof, int raw_input_scan_type); - void mime_parser_init(MIMEParser *parser); void mime_parser_clear(MIMEParser *parser); ParseResult mime_parser_parse(MIMEParser *parser, HdrHeap *heap, MIMEHdrImpl *mh, const char **real_s, const char *real_e, @@ -724,12 +784,12 @@ uint32_t mime_parse_uint(const char *buf, const char *end = nullptr); int64_t mime_parse_int64(const char *buf, const char *end = nullptr); int mime_parse_rfc822_date_fastcase(const char *buf, int length, struct tm *tp); time_t mime_parse_date(const char *buf, const char *end = nullptr); -int mime_parse_day(const char *&buf, const char *end, int *day); -int mime_parse_month(const char *&buf, const char *end, int *month); -int mime_parse_mday(const char *&buf, const char *end, int *mday); -int mime_parse_year(const char *&buf, const char *end, int *year); -int mime_parse_time(const char *&buf, const char *end, int *hour, int *min, int *sec); -int mime_parse_integer(const char *&buf, const char *end, int *integer); +bool mime_parse_day(const char *&buf, const char *end, int *day); +bool mime_parse_month(const char *&buf, const char *end, int *month); +bool mime_parse_mday(const char *&buf, const char *end, int *mday); +bool mime_parse_year(const char *&buf, const char *end, int *year); +bool mime_parse_time(const char *&buf, const char *end, int *hour, int *min, int *sec); +bool mime_parse_integer(const char *&buf, const char *end, int *integer); /*********************************************************************** * * @@ -743,7 +803,9 @@ int mime_parse_integer(const char *&buf, const char *end, int *integer); inline const char * MIMEField::name_get(int *length) const { - return (mime_field_name_get(this, length)); + auto name{this->name_get()}; + *length = int(name.size()); + return name.data(); } /*------------------------------------------------------------------------- @@ -752,11 +814,10 @@ MIMEField::name_get(int *length) const inline void MIMEField::name_set(HdrHeap *heap, MIMEHdrImpl *mh, const char *name, int length) { - int16_t name_wks_idx; const char *name_wks; if (hdrtoken_is_wks(name)) { - name_wks_idx = hdrtoken_wks_to_index(name); + int16_t name_wks_idx = hdrtoken_wks_to_index(name); mime_field_name_set(heap, mh, this, name_wks_idx, name, length, true); } else { int field_name_wks_idx = hdrtoken_tokenize(name, length, &name_wks); @@ -787,37 +848,39 @@ MIMEField::name_is_valid() const inline const char * MIMEField::value_get(int *length) const { - return (mime_field_value_get(this, length)); + auto value{this->value_get()}; + *length = int(value.size()); + return value.data(); } inline int32_t MIMEField::value_get_int() const { - return (mime_field_value_get_int(this)); + return mime_field_value_get_int(this); } inline uint32_t MIMEField::value_get_uint() const { - return (mime_field_value_get_uint(this)); + return mime_field_value_get_uint(this); } inline int64_t MIMEField::value_get_int64() const { - return (mime_field_value_get_int64(this)); + return mime_field_value_get_int64(this); } inline time_t MIMEField::value_get_date() const { - return (mime_field_value_get_date(this)); + return mime_field_value_get_date(this); } inline int MIMEField::value_get_comma_list(StrList *list) const { - return (mime_field_value_get_comma_list(this, list)); + return mime_field_value_get_comma_list(this, list); } /*------------------------------------------------------------------------- @@ -901,9 +964,9 @@ MIMEField::has_dups() const ***********************************************************************/ struct MIMEFieldIter { - MIMEFieldIter() : m_slot(0), m_block(nullptr) {} - uint32_t m_slot; - MIMEFieldBlockImpl *m_block; + MIMEFieldIter() {} + uint32_t m_slot = 0; + MIMEFieldBlockImpl *m_block = nullptr; }; /*------------------------------------------------------------------------- @@ -952,6 +1015,7 @@ class MIMEHdr : public HdrHeapSDKHandle int value_get_index(const char *name, int name_length, const char *value, int value_length) const; const char *value_get(const char *name, int name_length, int *value_length) const; + std::string_view value_get(std::string_view const &name) const; // Convenience overload. int32_t value_get_int(const char *name, int name_length) const; uint32_t value_get_uint(const char *name, int name_length) const; int64_t value_get_int64(const char *name, int name_length) const; @@ -1104,7 +1168,7 @@ MIMEHdr::field_create(const char *name, int length) mime_field_name_set(m_heap, m_mime, field, field_name_wks_idx, name, length, true); } - return (field); + return field; } /*------------------------------------------------------------------------- @@ -1239,10 +1303,11 @@ inline int MIMEHdr::value_get_index(const char *name, int name_length, const char *value, int value_length) const { const MIMEField *field = field_find(name, name_length); - if (field) + + if (field) { return field->value_get_index(value, value_length); - else - return -1; + } + return -1; } /*------------------------------------------------------------------------- @@ -1251,13 +1316,23 @@ MIMEHdr::value_get_index(const char *name, int name_length, const char *value, i inline const char * MIMEHdr::value_get(const char *name, int name_length, int *value_length_return) const { - // ink_assert(valid()); const MIMEField *field = field_find(name, name_length); - if (field) - return (mime_field_value_get(field, value_length_return)); - else - return (nullptr); + if (field) { + return field->value_get(value_length_return); + } + return nullptr; +} + +inline std::string_view +MIMEHdr::value_get(std::string_view const &name) const +{ + MIMEField const *field = field_find(name.data(), name.size()); + + if (field) { + return field->value_get(); + } + return {}; } inline int32_t @@ -1265,10 +1340,10 @@ MIMEHdr::value_get_int(const char *name, int name_length) const { const MIMEField *field = field_find(name, name_length); - if (field) - return (mime_field_value_get_int(field)); - else - return (0); + if (field) { + return mime_field_value_get_int(field); + } + return 0; } inline uint32_t @@ -1276,10 +1351,10 @@ MIMEHdr::value_get_uint(const char *name, int name_length) const { const MIMEField *field = field_find(name, name_length); - if (field) - return (mime_field_value_get_uint(field)); - else - return (0); + if (field) { + return mime_field_value_get_uint(field); + } + return 0; } inline int64_t @@ -1287,10 +1362,10 @@ MIMEHdr::value_get_int64(const char *name, int name_length) const { const MIMEField *field = field_find(name, name_length); - if (field) - return (mime_field_value_get_int64(field)); - else - return (0); + if (field) { + return mime_field_value_get_int64(field); + } + return 0; } inline time_t @@ -1298,10 +1373,10 @@ MIMEHdr::value_get_date(const char *name, int name_length) const { const MIMEField *field = field_find(name, name_length); - if (field) - return (mime_field_value_get_date(field)); - else - return (0); + if (field) { + return mime_field_value_get_date(field); + } + return 0; } inline int @@ -1309,10 +1384,10 @@ MIMEHdr::value_get_comma_list(const char *name, int name_length, StrList *list) { const MIMEField *field = field_find(name, name_length); - if (field) - return (field->value_get_comma_list(list)); - else - return (0); + if (field) { + return field->value_get_comma_list(list); + } + return 0; } /*------------------------------------------------------------------------- @@ -1476,7 +1551,7 @@ MIMEHdr::get_age() inline int64_t MIMEHdr::get_content_length() const { - return (value_get_int64(MIME_FIELD_CONTENT_LENGTH, MIME_LEN_CONTENT_LENGTH)); + return value_get_int64(MIME_FIELD_CONTENT_LENGTH, MIME_LEN_CONTENT_LENGTH); } /*------------------------------------------------------------------------- @@ -1485,7 +1560,7 @@ MIMEHdr::get_content_length() const inline time_t MIMEHdr::get_date() { - return (value_get_date(MIME_FIELD_DATE, MIME_LEN_DATE)); + return value_get_date(MIME_FIELD_DATE, MIME_LEN_DATE); } /*------------------------------------------------------------------------- @@ -1494,7 +1569,7 @@ MIMEHdr::get_date() inline time_t MIMEHdr::get_expires() { - return (value_get_date(MIME_FIELD_EXPIRES, MIME_LEN_EXPIRES)); + return value_get_date(MIME_FIELD_EXPIRES, MIME_LEN_EXPIRES); } /*------------------------------------------------------------------------- @@ -1503,7 +1578,7 @@ MIMEHdr::get_expires() inline time_t MIMEHdr::get_if_modified_since() { - return (value_get_date(MIME_FIELD_IF_MODIFIED_SINCE, MIME_LEN_IF_MODIFIED_SINCE)); + return value_get_date(MIME_FIELD_IF_MODIFIED_SINCE, MIME_LEN_IF_MODIFIED_SINCE); } /*------------------------------------------------------------------------- @@ -1512,7 +1587,7 @@ MIMEHdr::get_if_modified_since() inline time_t MIMEHdr::get_if_unmodified_since() { - return (value_get_date(MIME_FIELD_IF_UNMODIFIED_SINCE, MIME_LEN_IF_UNMODIFIED_SINCE)); + return value_get_date(MIME_FIELD_IF_UNMODIFIED_SINCE, MIME_LEN_IF_UNMODIFIED_SINCE); } /*------------------------------------------------------------------------- @@ -1521,7 +1596,7 @@ MIMEHdr::get_if_unmodified_since() inline time_t MIMEHdr::get_last_modified() { - return (value_get_date(MIME_FIELD_LAST_MODIFIED, MIME_LEN_LAST_MODIFIED)); + return value_get_date(MIME_FIELD_LAST_MODIFIED, MIME_LEN_LAST_MODIFIED); } /*------------------------------------------------------------------------- @@ -1530,7 +1605,7 @@ MIMEHdr::get_last_modified() inline time_t MIMEHdr::get_if_range_date() { - return (value_get_date(MIME_FIELD_IF_RANGE, MIME_LEN_IF_RANGE)); + return value_get_date(MIME_FIELD_IF_RANGE, MIME_LEN_IF_RANGE); } /*------------------------------------------------------------------------- @@ -1539,7 +1614,7 @@ MIMEHdr::get_if_range_date() inline int32_t MIMEHdr::get_max_forwards() { - return (value_get_int(MIME_FIELD_MAX_FORWARDS, MIME_LEN_MAX_FORWARDS)); + return value_get_int(MIME_FIELD_MAX_FORWARDS, MIME_LEN_MAX_FORWARDS); } /*------------------------------------------------------------------------- @@ -1551,7 +1626,7 @@ MIMEHdr::get_warning(int idx) (void)idx; // FIXME: what do we do here? ink_release_assert(!"unimplemented"); - return (0); + return 0; } /*------------------------------------------------------------------------- @@ -1560,7 +1635,7 @@ MIMEHdr::get_warning(int idx) inline uint32_t MIMEHdr::get_cooked_cc_mask() { - return (m_mime->m_cooked_stuff.m_cache_control.m_mask); + return m_mime->m_cooked_stuff.m_cache_control.m_mask; } /*------------------------------------------------------------------------- @@ -1569,7 +1644,7 @@ MIMEHdr::get_cooked_cc_mask() inline int32_t MIMEHdr::get_cooked_cc_max_age() { - return (m_mime->m_cooked_stuff.m_cache_control.m_secs_max_age); + return m_mime->m_cooked_stuff.m_cache_control.m_secs_max_age; } /*------------------------------------------------------------------------- @@ -1578,7 +1653,7 @@ MIMEHdr::get_cooked_cc_max_age() inline int32_t MIMEHdr::get_cooked_cc_s_maxage() { - return (m_mime->m_cooked_stuff.m_cache_control.m_secs_s_maxage); + return m_mime->m_cooked_stuff.m_cache_control.m_secs_s_maxage; } /*------------------------------------------------------------------------- @@ -1587,7 +1662,7 @@ MIMEHdr::get_cooked_cc_s_maxage() inline int32_t MIMEHdr::get_cooked_cc_max_stale() { - return (m_mime->m_cooked_stuff.m_cache_control.m_secs_max_stale); + return m_mime->m_cooked_stuff.m_cache_control.m_secs_max_stale; } /*------------------------------------------------------------------------- @@ -1596,7 +1671,7 @@ MIMEHdr::get_cooked_cc_max_stale() inline int32_t MIMEHdr::get_cooked_cc_min_fresh() { - return (m_mime->m_cooked_stuff.m_cache_control.m_secs_min_fresh); + return m_mime->m_cooked_stuff.m_cache_control.m_secs_min_fresh; } /*------------------------------------------------------------------------- @@ -1605,7 +1680,7 @@ MIMEHdr::get_cooked_cc_min_fresh() inline bool MIMEHdr::get_cooked_pragma_no_cache() { - return (m_mime->m_cooked_stuff.m_pragma.m_no_cache); + return m_mime->m_cooked_stuff.m_pragma.m_no_cache; } /*------------------------------------------------------------------------- diff --git a/proxy/hdrs/Makefile.am b/proxy/hdrs/Makefile.am index 42dad696a94..75c33860eb4 100644 --- a/proxy/hdrs/Makefile.am +++ b/proxy/hdrs/Makefile.am @@ -59,12 +59,11 @@ load_http_hdr_SOURCES = \ load_http_hdr_LDADD = -L. -lhdrs \ $(top_builddir)/src/tscore/libtscore.la \ - $(top_builddir)/src/tscpp/util/libtscpputil.la \ - @LIBTCL@ + $(top_builddir)/src/tscpp/util/libtscpputil.la -check_PROGRAMS = test_mime +check_PROGRAMS = test_mime test_proxy_hdrs -TESTS = test_mime +TESTS = $(check_PROGRAMS) test_mime_LDADD = -L. -lhdrs \ $(top_builddir)/src/tscore/libtscore.la \ @@ -74,10 +73,30 @@ test_mime_LDADD = -L. -lhdrs \ $(top_builddir)/mgmt/libmgmt_p.la \ $(top_builddir)/proxy/shared/libUglyLogStubs.a \ @HWLOC_LIBS@ \ - @LIBTCL@ @LIBCAP@ + @LIBCAP@ test_mime_SOURCES = test_mime.cc +test_proxy_hdrs_CPPFLAGS = $(AM_CPPFLAGS)\ + -I$(abs_top_srcdir)/tests/include + +test_proxy_hdrs_SOURCES = \ + unit_tests/unit_test_main.cc \ + unit_tests/test_Hdrs.cc \ + unit_tests/test_HdrUtils.cc + +test_proxy_hdrs_LDADD = \ + $(top_builddir)/src/tscore/libtscore.la \ + -L. -lhdrs \ + $(top_builddir)/src/tscore/libtscore.la \ + $(top_builddir)/src/tscpp/util/libtscpputil.la \ + $(top_builddir)/iocore/eventsystem/libinkevent.a \ + $(top_builddir)/lib/records/librecords_p.a \ + $(top_builddir)/mgmt/libmgmt_p.la \ + $(top_builddir)/proxy/shared/libUglyLogStubs.a \ + @HWLOC_LIBS@ \ + @LIBCAP@ + #test_UNUSED_SOURCES = \ # test_urlhash.cc diff --git a/proxy/hdrs/URL.cc b/proxy/hdrs/URL.cc index 8f94329796d..6137add2df6 100644 --- a/proxy/hdrs/URL.cc +++ b/proxy/hdrs/URL.cc @@ -348,6 +348,21 @@ URLImpl::unmarshal(intptr_t offset) // HDR_UNMARSHAL_STR(m_ptr_printed_string, offset); } +void +URLImpl::rehome_strings(HdrHeap *new_heap) +{ + m_ptr_scheme = new_heap->localize({m_ptr_scheme, m_len_scheme}).data(); + m_ptr_user = new_heap->localize({m_ptr_user, m_len_user}).data(); + m_ptr_password = new_heap->localize({m_ptr_password, m_len_password}).data(); + m_ptr_host = new_heap->localize({m_ptr_host, m_len_host}).data(); + m_ptr_port = new_heap->localize({m_ptr_port, m_len_port}).data(); + m_ptr_path = new_heap->localize({m_ptr_path, m_len_path}).data(); + m_ptr_params = new_heap->localize({m_ptr_params, m_len_params}).data(); + m_ptr_query = new_heap->localize({m_ptr_query, m_len_query}).data(); + m_ptr_fragment = new_heap->localize({m_ptr_fragment, m_len_fragment}).data(); + m_ptr_printed_string = new_heap->localize({m_ptr_printed_string, m_len_printed_string}).data(); +} + void URLImpl::move_strings(HdrStrHeap *new_heap) { @@ -684,24 +699,6 @@ url_string_get_buf(URLImpl *url, char *dstbuf, int dstbuf_size, int *length) return buf; } -/*------------------------------------------------------------------------- - -------------------------------------------------------------------------*/ - -const char * -url_scheme_get(URLImpl *url, int *length) -{ - const char *str; - - if (url->m_scheme_wks_idx >= 0) { - str = hdrtoken_index_to_wks(url->m_scheme_wks_idx); - *length = hdrtoken_index_to_length(url->m_scheme_wks_idx); - } else { - str = url->m_ptr_scheme; - *length = url->m_len_scheme; - } - return str; -} - /*------------------------------------------------------------------------- -------------------------------------------------------------------------*/ diff --git a/proxy/hdrs/URL.h b/proxy/hdrs/URL.h index d1fa5822d11..0cecc705a25 100644 --- a/proxy/hdrs/URL.h +++ b/proxy/hdrs/URL.h @@ -80,6 +80,7 @@ struct URLImpl : public HdrHeapObjImpl { int marshal(MarshalXlate *str_xlate, int num_xlate); void unmarshal(intptr_t offset); void move_strings(HdrStrHeap *new_heap); + void rehome_strings(HdrHeap *new_heap); size_t strings_length(); // Sanity Check Functions @@ -151,9 +152,6 @@ extern int URL_LEN_MMS; extern int URL_LEN_MMSU; extern int URL_LEN_MMST; -/* Private */ -void url_adjust(MarshalXlate *str_xlate, int num_xlate); - /* Public */ bool validate_host_name(std::string_view addr); void url_init(); @@ -176,7 +174,6 @@ char *url_string_get_ref(HdrHeap *heap, URLImpl *url, int *length); void url_called_set(URLImpl *url); char *url_string_get_buf(URLImpl *url, char *dstbuf, int dstbuf_size, int *length); -const char *url_scheme_get(URLImpl *url, int *length); void url_CryptoHash_get(const URLImpl *url, CryptoHash *hash, cache_generation_t generation = -1); void url_host_CryptoHash_get(URLImpl *url, CryptoHash *hash); const char *url_scheme_set(HdrHeap *heap, URLImpl *url, const char *value, int value_wks_idx, int length, bool copy_string); @@ -226,7 +223,7 @@ url_canonicalize_port(int type, int port) class URL : public HdrHeapSDKHandle { public: - URLImpl *m_url_impl; + URLImpl *m_url_impl = nullptr; URL(); ~URL(); @@ -252,6 +249,7 @@ class URL : public HdrHeapSDKHandle void host_hash_get(CryptoHash *hash); const char *scheme_get(int *length); + const std::string_view scheme_get(); int scheme_get_wksidx(); void scheme_set(const char *value, int length); @@ -292,7 +290,7 @@ class URL : public HdrHeapSDKHandle /*------------------------------------------------------------------------- -------------------------------------------------------------------------*/ -inline URL::URL() : m_url_impl(nullptr) {} +inline URL::URL() {} /*------------------------------------------------------------------------- -------------------------------------------------------------------------*/ @@ -447,11 +445,25 @@ URL::host_hash_get(CryptoHash *hash) /*------------------------------------------------------------------------- -------------------------------------------------------------------------*/ +inline const std::string_view +URL::scheme_get() +{ + ink_assert(valid()); + + if (m_url_impl->m_scheme_wks_idx >= 0) { + return std::string_view{hdrtoken_index_to_wks(m_url_impl->m_scheme_wks_idx), + static_cast(hdrtoken_index_to_length(m_url_impl->m_scheme_wks_idx))}; + } else { + return std::string_view{m_url_impl->m_ptr_scheme, m_url_impl->m_len_scheme}; + } +} + inline const char * URL::scheme_get(int *length) { - ink_assert(valid()); - return (url_scheme_get(m_url_impl, length)); + std::string_view ret = this->scheme_get(); + *length = ret.size(); + return ret.data(); } inline int diff --git a/proxy/hdrs/test_mime.cc b/proxy/hdrs/test_mime.cc index b5055f6bb27..eba287d1b19 100644 --- a/proxy/hdrs/test_mime.cc +++ b/proxy/hdrs/test_mime.cc @@ -52,6 +52,47 @@ REGRESSION_TEST(MIME)(RegressionTest *t, int /* atype ATS_UNUSED */, int *pstatu hdr.destroy(); } +REGRESSION_TEST(MIME_PARSERS)(RegressionTest *t, int /* atype ATS_UNUSED */, int *pstatus) +{ + const char *end; + int value; + TestBox box(t, pstatus); + box = REGRESSION_TEST_PASSED; + + std::vector> tests = {{"0", 0}, + {"1234", 1234}, + {"-1234", -1234}, + {"2147483647", 2147483647}, + {"-2147483648", 2147483648}, + {"2147483648", INT_MAX}, + {"-2147483649", INT_MIN}, + {"2147483647", INT_MAX}, + {"-2147483648", INT_MIN}, + {"999999999999", INT_MAX}, + {"-999999999999", INT_MIN}}; + + for (const auto &it : tests) { + auto [buf, val] = it; + + end = buf + strlen(buf); + box.check(mime_parse_int(buf, end) == val, "Failed mime_parse_int"); + box.check(mime_parse_integer(buf, end, &value), "Failed mime_parse_integer call"); + box.check(value == val, "Failed mime_parse_integer value"); + } + + // Also check the date parser, which relies heavily on the mime_parse_integer() function + const char *date1 = "Sun, 05 Dec 1999 08:49:37 GMT"; + const char *date2 = "Sunday, 05-Dec-1999 08:49:37 GMT"; + + int d1 = mime_parse_date(date1, date1 + strlen(date1)); + int d2 = mime_parse_date(date2, date2 + strlen(date2)); + + box.check(d1 == d2, "Failed mime_parse_date"); + + printf("Date1: %d\n", d1); + printf("Date2: %d\n", d2); +} + int main(int argc, const char **argv) { diff --git a/proxy/hdrs/unit_tests/test_HdrUtils.cc b/proxy/hdrs/unit_tests/test_HdrUtils.cc new file mode 100644 index 00000000000..fe085c98757 --- /dev/null +++ b/proxy/hdrs/unit_tests/test_HdrUtils.cc @@ -0,0 +1,122 @@ +/** @file + + Catch-based tests for HdrsUtils.cc + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. + See the NOTICE file distributed with this work for additional information regarding copyright + ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance with the License. You may obtain a + copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software distributed under the License + is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + or implied. See the License for the specific language governing permissions and limitations under + the License. + */ + +#include +#include +#include +#include +#include +#include + +#include "catch.hpp" + +#include "HdrHeap.h" +#include "MIME.h" +#include "HdrUtils.h" + +TEST_CASE("HdrUtils", "[proxy][hdrutils]") +{ + static constexpr ts::TextView text{"One: alpha\r\n" + "Two: alpha, bravo\r\n" + "Three: zwoop, \"A,B\" , , phil , \"unterminated\r\n" + "Five: alpha, bravo, charlie\r\n" + "Four: itchi, \"ni, \\\"san\" , \"\" , \"\r\n" + "Five: delta, echo\r\n" + "\r\n"}; + + static constexpr std::string_view ONE_TAG{"One"}; + static constexpr std::string_view TWO_TAG{"Two"}; + static constexpr std::string_view THREE_TAG{"Three"}; + static constexpr std::string_view FOUR_TAG{"Four"}; + static constexpr std::string_view FIVE_TAG{"Five"}; + + HdrHeap *heap = new_HdrHeap(HdrHeap::DEFAULT_SIZE + 64); + MIMEParser parser; + char const *real_s = text.data(); + char const *real_e = text.data_end(); + MIMEHdr mime; + + mime.create(heap); + mime_parser_init(&parser); + + auto result = mime_parser_parse(&parser, heap, mime.m_mime, &real_s, real_e, false, true); + REQUIRE(PARSE_RESULT_DONE == result); + + HdrCsvIter iter; + + MIMEField *field{mime.field_find(ONE_TAG.data(), int(ONE_TAG.size()))}; + REQUIRE(field != nullptr); + + auto value = iter.get_first(field); + REQUIRE(value == "alpha"); + + field = mime.field_find(TWO_TAG.data(), int(TWO_TAG.size())); + value = iter.get_first(field); + REQUIRE(value == "alpha"); + value = iter.get_next(); + REQUIRE(value == "bravo"); + value = iter.get_next(); + REQUIRE(value.empty()); + + field = mime.field_find(THREE_TAG.data(), int(THREE_TAG.size())); + value = iter.get_first(field); + REQUIRE(value == "zwoop"); + value = iter.get_next(); + REQUIRE(value == "A,B"); // quotes escape separator, and are stripped. + value = iter.get_next(); + REQUIRE(value == "phil"); + value = iter.get_next(); + REQUIRE(value == "unterminated"); + value = iter.get_next(); + REQUIRE(value.empty()); + + field = mime.field_find(FOUR_TAG.data(), int(FOUR_TAG.size())); + value = iter.get_first(field); + REQUIRE(value == "itchi"); + value = iter.get_next(); + REQUIRE(value == "ni, \\\"san"); // verify escaped quotes are passed through. + value = iter.get_next(); + REQUIRE(value.empty()); + + // Check that duplicates are handled correctly. + field = mime.field_find(FIVE_TAG.data(), int(FIVE_TAG.size())); + value = iter.get_first(field); + REQUIRE(value == "alpha"); + value = iter.get_next(); + REQUIRE(value == "bravo"); + value = iter.get_next(); + REQUIRE(value == "charlie"); + value = iter.get_next(); + REQUIRE(value == "delta"); + value = iter.get_next(); + REQUIRE(value == "echo"); + value = iter.get_next(); + REQUIRE(value.empty()); + + field = mime.field_find(FIVE_TAG.data(), int(FIVE_TAG.size())); + value = iter.get_first(field, false); + REQUIRE(value == "alpha"); + value = iter.get_next(); + REQUIRE(value == "bravo"); + value = iter.get_next(); + REQUIRE(value == "charlie"); + value = iter.get_next(); + REQUIRE(value.empty()); +} diff --git a/proxy/hdrs/unit_tests/test_Hdrs.cc b/proxy/hdrs/unit_tests/test_Hdrs.cc new file mode 100644 index 00000000000..a471339f8ad --- /dev/null +++ b/proxy/hdrs/unit_tests/test_Hdrs.cc @@ -0,0 +1,117 @@ +/** @file + + Catch-based unit tests for various header logic. + This replaces the old regression tests in HdrTest.cc. + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. + See the NOTICE file distributed with this work for additional information regarding copyright + ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance with the License. You may obtain a + copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software distributed under the License + is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + or implied. See the License for the specific language governing permissions and limitations under + the License. + */ + +#include +#include +#include +#include +#include +#include +#include + +#include "catch.hpp" + +#include "HTTP.h" + +// replaces test_http_parser_eos_boundary_cases +TEST_CASE("HdrTest", "[proxy][hdrtest]") +{ + struct Test { + ts::TextView msg; + int expected_result; + int expected_bytes_consumed; + }; + static const std::array tests = {{ + {"GET /index.html HTTP/1.0\r\n", PARSE_RESULT_DONE, 26}, + {"GET /index.html HTTP/1.0\r\n\r\n***BODY****", PARSE_RESULT_DONE, 28}, + {"GET /index.html HTTP/1.0\r\nUser-Agent: foobar\r\n\r\n***BODY****", PARSE_RESULT_DONE, 48}, + {"GET", PARSE_RESULT_ERROR, 3}, + {"GET /index.html", PARSE_RESULT_ERROR, 15}, + {"GET /index.html\r\n", PARSE_RESULT_ERROR, 17}, + {"GET /index.html HTTP/1.0", PARSE_RESULT_ERROR, 24}, + {"GET /index.html HTTP/1.0\r", PARSE_RESULT_ERROR, 25}, + {"GET /index.html HTTP/1.0\n", PARSE_RESULT_DONE, 25}, + {"GET /index.html HTTP/1.0\n\n", PARSE_RESULT_DONE, 26}, + {"GET /index.html HTTP/1.0\r\n\r\n", PARSE_RESULT_DONE, 28}, + {"GET /index.html HTTP/1.0\r\nUser-Agent: foobar", PARSE_RESULT_ERROR, 44}, + {"GET /index.html HTTP/1.0\r\nUser-Agent: foobar\n", PARSE_RESULT_DONE, 45}, + {"GET /index.html HTTP/1.0\r\nUser-Agent: foobar\r\n", PARSE_RESULT_DONE, 46}, + {"GET /index.html HTTP/1.0\r\nUser-Agent: foobar\r\n\r\n", PARSE_RESULT_DONE, 48}, + {"GET /index.html HTTP/1.0\nUser-Agent: foobar\n", PARSE_RESULT_DONE, 44}, + {"GET /index.html HTTP/1.0\nUser-Agent: foobar\nBoo: foo\n", PARSE_RESULT_DONE, 53}, + {"GET /index.html HTTP/1.0\r\nUser-Agent: foobar\r\n", PARSE_RESULT_DONE, 46}, + {"GET /index.html HTTP/1.0\r\n", PARSE_RESULT_DONE, 26}, + {"", PARSE_RESULT_ERROR, 0}, + }}; + + HTTPParser parser; + + http_parser_init(&parser); + + for (auto const &test : tests) { + HTTPHdr req_hdr; + HdrHeap *heap = new_HdrHeap(HdrHeap::DEFAULT_SIZE + 64); // extra to prevent proxy allocation. + + req_hdr.create(HTTP_TYPE_REQUEST, heap); + + http_parser_clear(&parser); + + auto start = test.msg.data(); + auto ret = req_hdr.parse_req(&parser, &start, test.msg.data_end(), true); + auto bytes_consumed = start - test.msg.data(); + + REQUIRE(bytes_consumed == test.expected_bytes_consumed); + REQUIRE(ret == test.expected_result); + + req_hdr.destroy(); + } +} + +TEST_CASE("MIMEScanner_fragments", "[proxy][mimescanner_fragments]") +{ + constexpr ts::TextView const message = "GET /index.html HTTP/1.0\r\n"; + + struct Fragment { + ts::TextView msg; + bool shares_input; + int expected_result; + }; + constexpr std::array const fragments = {{ + {message.substr(0, 11), true, PARSE_RESULT_CONT}, + {message.substr(11, 11), true, PARSE_RESULT_CONT}, + {message.substr(22), false, PARSE_RESULT_OK}, + }}; + + MIMEScanner scanner; + ts::TextView output; // only set on last call + + for (auto const &frag : fragments) { + ts::TextView input = frag.msg; + bool got_shares_input = !frag.shares_input; + constexpr bool const is_eof = false; + ParseResult const got_res = scanner.get(input, output, got_shares_input, is_eof, MIMEScanner::LINE); + + REQUIRE(frag.expected_result == got_res); + REQUIRE(frag.shares_input == got_shares_input); + } + + REQUIRE(message == output); +} diff --git a/proxy/logging/LogCollationAccept.h b/proxy/hdrs/unit_tests/unit_test_main.cc similarity index 65% rename from proxy/logging/LogCollationAccept.h rename to proxy/hdrs/unit_tests/unit_test_main.cc index 315ca8905ce..636c5ccbe5d 100644 --- a/proxy/logging/LogCollationAccept.h +++ b/proxy/hdrs/unit_tests/unit_test_main.cc @@ -1,6 +1,6 @@ /** @file - A brief file description + This file used for catch based tests. It is the main() stub. @section license License @@ -21,22 +21,24 @@ limitations under the License. */ -#pragma once +#include "HTTP.h" -#include "P_EventSystem.h" -#include "P_Net.h" +#define CATCH_CONFIG_RUNNER +#include "catch.hpp" -struct LogCollationAccept : public Continuation { -public: - LogCollationAccept(int port); - ~LogCollationAccept() override; +extern int cmd_disable_pfreelist; - int accept_event(int event, NetVConnection *net_vc); +int +main(int argc, char *argv[]) +{ + // No thread setup, forbid use of thread local allocators. + cmd_disable_pfreelist = true; + // Get all of the HTTP WKS items populated. + http_init(); -private: - int m_port; - Action *m_accept_action = nullptr; - Event *m_pending_event = nullptr; -}; + int result = Catch::Session().run(argc, argv); -typedef int (LogCollationAccept::*LogCollationAcceptHandler)(int, void *); + // global clean-up... + + return result; +} diff --git a/proxy/http/Http1ClientSession.cc b/proxy/http/Http1ClientSession.cc index c4c3572c562..10ac26c3ae2 100644 --- a/proxy/http/Http1ClientSession.cc +++ b/proxy/http/Http1ClientSession.cc @@ -62,24 +62,7 @@ ink_mutex debug_cs_list_mutex; ClassAllocator http1ClientSessionAllocator("http1ClientSessionAllocator"); -Http1ClientSession::Http1ClientSession() - : client_vc(nullptr), - magic(HTTP_CS_MAGIC_DEAD), - transact_count(0), - tcp_init_cwnd_set(false), - half_close(false), - conn_decrease(false), - read_buffer(nullptr), - sm_reader(nullptr), - read_state(HCS_INIT), - ka_vio(nullptr), - slave_ka_vio(nullptr), - bound_ss(nullptr), - released_transactions(0), - f_outbound_transparent(false), - f_transparent_passthrough(false) -{ -} +Http1ClientSession::Http1ClientSession() {} void Http1ClientSession::destroy() @@ -145,7 +128,7 @@ Http1ClientSession::free() } void -Http1ClientSession::new_connection(NetVConnection *new_vc, MIOBuffer *iobuf, IOBufferReader *reader, bool backdoor) +Http1ClientSession::new_connection(NetVConnection *new_vc, MIOBuffer *iobuf, IOBufferReader *reader) { ink_assert(new_vc != nullptr); ink_assert(client_vc == nullptr); @@ -159,9 +142,6 @@ Http1ClientSession::new_connection(NetVConnection *new_vc, MIOBuffer *iobuf, IOB MUTEX_TRY_LOCK(lock, mutex, this_ethread()); ink_assert(lock.is_locked()); - // Disable hooks for backdoor connections. - this->hooks_on = !backdoor; - // Unique client session identifier. con_id = ProxyClientSession::next_connection_id(); diff --git a/proxy/http/Http1ClientSession.h b/proxy/http/Http1ClientSession.h index b81d41be5f9..3fafd2d451a 100644 --- a/proxy/http/Http1ClientSession.h +++ b/proxy/http/Http1ClientSession.h @@ -31,7 +31,6 @@ #pragma once -//#include "libts.h" #include "P_Net.h" #include "InkAPIInternal.h" #include "HTTP.h" @@ -65,7 +64,7 @@ class Http1ClientSession : public ProxyClientSession this->release(&trans); } - void new_connection(NetVConnection *new_vc, MIOBuffer *iobuf, IOBufferReader *reader, bool backdoor) override; + void new_connection(NetVConnection *new_vc, MIOBuffer *iobuf, IOBufferReader *reader) override; // Implement VConnection interface. VIO *do_io_read(Continuation *c, int64_t nbytes = INT64_MAX, MIOBuffer *buf = nullptr) override; @@ -184,33 +183,33 @@ class Http1ClientSession : public ProxyClientSession HCS_CLOSED, }; - NetVConnection *client_vc; - int magic; - int transact_count; - bool tcp_init_cwnd_set; - bool half_close; - bool conn_decrease; + NetVConnection *client_vc = nullptr; + int magic = HTTP_SS_MAGIC_DEAD; + int transact_count = 0; + bool tcp_init_cwnd_set = false; + bool half_close = false; + bool conn_decrease = false; - MIOBuffer *read_buffer; - IOBufferReader *sm_reader; + MIOBuffer *read_buffer = nullptr; + IOBufferReader *sm_reader = nullptr; - C_Read_State read_state; + C_Read_State read_state = HCS_INIT; - VIO *ka_vio; - VIO *slave_ka_vio; + VIO *ka_vio = nullptr; + VIO *slave_ka_vio = nullptr; - HttpServerSession *bound_ss; + HttpServerSession *bound_ss = nullptr; - int released_transactions; + int released_transactions = 0; public: // Link debug_link; LINK(Http1ClientSession, debug_link); /// Set outbound connection to transparent. - bool f_outbound_transparent; + bool f_outbound_transparent = false; /// Transparently pass-through non-HTTP traffic. - bool f_transparent_passthrough; + bool f_transparent_passthrough = false; Http1ClientTransaction trans; }; diff --git a/proxy/http/Http1ClientTransaction.cc b/proxy/http/Http1ClientTransaction.cc index 5c5f1759b90..d8a78abbfe5 100644 --- a/proxy/http/Http1ClientTransaction.cc +++ b/proxy/http/Http1ClientTransaction.cc @@ -49,15 +49,17 @@ Http1ClientTransaction::release(IOBufferReader *r) void Http1ClientTransaction::set_parent(ProxyClientSession *new_parent) { - parent = new_parent; Http1ClientSession *http1_parent = dynamic_cast(new_parent); + if (http1_parent) { outbound_port = http1_parent->outbound_port; outbound_ip4 = http1_parent->outbound_ip4; outbound_ip6 = http1_parent->outbound_ip6; outbound_transparent = http1_parent->f_outbound_transparent; + super_type::set_parent(new_parent); + } else { + parent = nullptr; } - super_type::set_parent(new_parent); } void @@ -78,3 +80,15 @@ Http1ClientTransaction::allow_half_open() const } return false; } + +void +Http1ClientTransaction::increment_client_transactions_stat() +{ + HTTP_INCREMENT_DYN_STAT(http_current_client_transactions_stat); +} + +void +Http1ClientTransaction::decrement_client_transactions_stat() +{ + HTTP_DECREMENT_DYN_STAT(http_current_client_transactions_stat); +} diff --git a/proxy/http/Http1ClientTransaction.h b/proxy/http/Http1ClientTransaction.h index 8a876660f8e..c0dbab48ef4 100644 --- a/proxy/http/Http1ClientTransaction.h +++ b/proxy/http/Http1ClientTransaction.h @@ -127,6 +127,9 @@ class Http1ClientTransaction : public ProxyClientTransaction return get_transact_count(); } + void increment_client_transactions_stat() override; + void decrement_client_transactions_stat() override; + protected: bool outbound_transparent{false}; }; diff --git a/proxy/http/HttpBodyFactory.cc b/proxy/http/HttpBodyFactory.cc index 53edaa6864b..9d5433b53d7 100644 --- a/proxy/http/HttpBodyFactory.cc +++ b/proxy/http/HttpBodyFactory.cc @@ -190,50 +190,29 @@ HttpBodyFactory::fabricate_with_old_api(const char *type, HttpTransact::State *c } unlock(); - return (buffer); + return buffer; } void HttpBodyFactory::dump_template_tables(FILE *fp) { - RawHashTable *h1, *h2; - RawHashTable_Key k1, k2; - RawHashTable_Value v1, v2; - RawHashTable_Binding *b1, *b2; - RawHashTable_IteratorState i1, i2; - HttpBodySet *body_set; - lock(); - - h1 = table_of_sets; - - if (h1 != nullptr) { - /////////////////////////////////////////// - // loop over set->body-types hash table // - /////////////////////////////////////////// - - for (b1 = h1->firstBinding(&i1); b1 != nullptr; b1 = h1->nextBinding(&i1)) { - k1 = table_of_sets->getKeyFromBinding(b1); - v1 = table_of_sets->getValueFromBinding(b1); - body_set = (HttpBodySet *)v1; - - if (body_set != nullptr) { - fprintf(fp, "set %s: name '%s', lang '%s', charset '%s'\n", k1, body_set->set_name, body_set->content_language, - body_set->content_charset); + if (table_of_sets) { + for (const auto &it1 : *table_of_sets.get()) { + HttpBodySet *body_set = static_cast(it1.second); + if (body_set) { + fprintf(fp, "set %s: name '%s', lang '%s', charset '%s'\n", it1.first.c_str(), body_set->set_name, + body_set->content_language, body_set->content_charset); /////////////////////////////////////////// // loop over body-types->body hash table // /////////////////////////////////////////// ink_assert(body_set->is_sane()); - h2 = body_set->table_of_pages; - - for (b2 = h2->firstBinding(&i2); b2 != nullptr; b2 = h2->nextBinding(&i2)) { - k2 = table_of_sets->getKeyFromBinding(b2); - v2 = table_of_sets->getValueFromBinding(b2); - HttpBodyTemplate *t = (HttpBodyTemplate *)v2; - - fprintf(fp, " %-30s: %" PRId64 " bytes\n", k2, t->byte_count); + if (body_set->table_of_pages) { + for (const auto &it2 : *body_set->table_of_pages.get()) { + fprintf(fp, " %-30s: %" PRId64 " bytes\n", it2.first.c_str(), it2.second->byte_count); + } } } } @@ -254,7 +233,7 @@ config_callback(const char * /* name ATS_UNUSED */, RecDataT /* data_type ATS_UN { HttpBodyFactory *body_factory = (HttpBodyFactory *)cookie; body_factory->reconfigure(); - return (0); + return 0; } void @@ -372,7 +351,7 @@ HttpBodyFactory::HttpBodyFactory() HttpBodyFactory::~HttpBodyFactory() { // FIX: need to implement destructor - delete table_of_sets; + table_of_sets.reset(nullptr); } // LOCKING: must be called with lock taken @@ -457,13 +436,12 @@ const char * HttpBodyFactory::determine_set_by_host(HttpTransact::State *context) { const char *set; - RawHashTable_Value v; int host_len = context->hh_info.host_len; char host_buffer[host_len + 1]; strncpy(host_buffer, context->hh_info.request_host, host_len); host_buffer[host_len] = '\0'; - if (table_of_sets->getValue((RawHashTable_Key)host_buffer, &v)) { - set = table_of_sets->getKeyFromBinding(table_of_sets->getCurrentBinding((RawHashTable_Key)host_buffer)); + if (auto it = table_of_sets->find(host_buffer); it != table_of_sets->end()) { + set = it->first.c_str(); } else { set = "default"; } @@ -471,8 +449,9 @@ HttpBodyFactory::determine_set_by_host(HttpTransact::State *context) } const char * -HttpBodyFactory::determine_set_by_language(RawHashTable *table_of_sets, StrList *acpt_language_list, StrList *acpt_charset_list, - float *Q_best_ptr, int *La_best_ptr, int *Lc_best_ptr, int *I_best_ptr) +HttpBodyFactory::determine_set_by_language(std::unique_ptr &table_of_sets, StrList *acpt_language_list, + StrList *acpt_charset_list, float *Q_best_ptr, int *La_best_ptr, int *Lc_best_ptr, + int *I_best_ptr) { float Q, Ql, Qc, Q_best; int I, Idummy, I_best; @@ -480,13 +459,6 @@ HttpBodyFactory::determine_set_by_language(RawHashTable *table_of_sets, StrList int is_the_default_set; const char *set_best; - RawHashTable_Key k1; - RawHashTable_Value v1; - RawHashTable_Binding *b1; - RawHashTable_IteratorState i1; - RawHashTable *table_of_pages; - HttpBodySetRawData *body_set; - set_best = "default"; Q_best = 0.00001; La_best = 0; @@ -506,20 +478,15 @@ HttpBodyFactory::determine_set_by_language(RawHashTable *table_of_sets, StrList goto done; } - if (table_of_sets != nullptr) { + if (table_of_sets) { /////////////////////////////////////////// // loop over set->body-types hash table // /////////////////////////////////////////// + for (const auto &it : *table_of_sets.get()) { + const char *set_name = it.first.c_str(); + HttpBodySetRawData *body_set = it.second; - for (b1 = table_of_sets->firstBinding(&i1); b1 != nullptr; b1 = table_of_sets->nextBinding(&i1)) { - k1 = table_of_sets->getKeyFromBinding(b1); - v1 = table_of_sets->getValueFromBinding(b1); - const char *set_name = (const char *)k1; - - body_set = (HttpBodySetRawData *)v1; - table_of_pages = body_set->table_of_pages; - - if ((set_name == nullptr) || (table_of_pages == nullptr)) { + if ((it.first.empty()) || (body_set->table_of_pages == nullptr)) { continue; } @@ -635,7 +602,7 @@ HttpBodyFactory::determine_set_by_language(RawHashTable *table_of_sets, StrList *La_best_ptr = La_best; *Lc_best_ptr = Lc_best; *I_best_ptr = I_best; - return (set_best); + return set_best; } // LOCKING: must be called with lock taken @@ -648,46 +615,42 @@ HttpBodyFactory::determine_set_by_language(StrList *acpt_language_list, StrList set_best = determine_set_by_language(table_of_sets, acpt_language_list, acpt_charset_list, &Q_best, &La_best, &Lc_best, &I_best); - return (set_best); + return set_best; } // LOCKING: must be called with lock taken HttpBodyTemplate * HttpBodyFactory::find_template(const char *set, const char *type, HttpBodySet **body_set_return) { - RawHashTable_Value v; - Debug("body_factory", "calling find_template(%s,%s)", set, type); *body_set_return = nullptr; - if (table_of_sets == nullptr) { - return (nullptr); + if (table_of_sets == nullptr || !set || !type) { + return nullptr; } - if (table_of_sets->getValue((RawHashTable_Key)set, &v)) { - HttpBodySet *body_set = (HttpBodySet *)v; - RawHashTable *table_of_types = body_set->table_of_pages; - - if (table_of_types == nullptr) { - return (nullptr); + if (auto it = table_of_sets->find(set); it != table_of_sets->end()) { + HttpBodySet *body_set = static_cast(it->second); + if (body_set->table_of_pages == nullptr) { + return nullptr; } - if (table_of_types->getValue((RawHashTable_Key)type, &v)) { - HttpBodyTemplate *t = (HttpBodyTemplate *)v; + if (auto it_page = body_set->table_of_pages->find(type); it_page != body_set->table_of_pages->end()) { + HttpBodyTemplate *t = it_page->second; if ((t == nullptr) || (!t->is_sane())) { - return (nullptr); + return nullptr; } *body_set_return = body_set; Debug("body_factory", "find_template(%s,%s) -> (file %s, length %" PRId64 ", lang '%s', charset '%s')", set, type, t->template_pathname, t->byte_count, body_set->content_language, body_set->content_charset); - return (t); + return t; } } Debug("body_factory", "find_template(%s,%s) -> NULL", set, type); - return (nullptr); + return nullptr; } // LOCKING: must be called with lock taken @@ -705,17 +668,17 @@ HttpBodyFactory::is_response_suppressed(HttpTransact::State *context) } else */ if (response_suppression_mode == 0) { - return (false); + return false; } else if (response_suppression_mode == 1) { - return (true); + return true; } else if (response_suppression_mode == 2) { if (context->req_flavor == HttpTransact::REQ_FLAVOR_INTERCEPTED) { - return (true); + return true; } else { - return (false); + return false; } } else { - return (false); + return false; } } @@ -723,69 +686,43 @@ HttpBodyFactory::is_response_suppressed(HttpTransact::State *context) void HttpBodyFactory::nuke_template_tables() { - RawHashTable *h1, *h2; - RawHashTable_Value v1, v2; - RawHashTable_Binding *b1, *b2; - RawHashTable_IteratorState i1, i2; - HttpBodySet *body_set; - HttpBodyTemplate *hbt; - - h1 = table_of_sets; - - if (h1) { + if (table_of_sets) { Debug("body_factory", "deleting pre-existing template tables"); } else { Debug("body_factory", "no pre-existing template tables"); } - if (h1 != nullptr) { + if (table_of_sets) { /////////////////////////////////////////// // loop over set->body-types hash table // /////////////////////////////////////////// - - for (b1 = h1->firstBinding(&i1); b1 != nullptr; b1 = h1->nextBinding(&i1)) { - v1 = h1->getValueFromBinding(b1); - - body_set = (HttpBodySet *)v1; + for (const auto &it : *table_of_sets.get()) { + HttpBodySet *body_set = static_cast(it.second); ink_assert(body_set->is_sane()); - h2 = body_set->table_of_pages; - - if (h2 != nullptr) { - body_set->table_of_pages = nullptr; - + if (body_set->table_of_pages) { /////////////////////////////////////////// // loop over body-types->body hash table // /////////////////////////////////////////// - - for (b2 = h2->firstBinding(&i2); b2 != nullptr; b2 = h2->nextBinding(&i2)) { - v2 = h2->getValueFromBinding(b2); - if (v2) { - // need a cast here - hbt = (HttpBodyTemplate *)v2; - delete hbt; - } + for (const auto &it_page : *body_set->table_of_pages.get()) { + delete it_page.second; } - - delete h2; + body_set->table_of_pages.reset(nullptr); } - delete body_set; } - delete h1; + table_of_sets.reset(nullptr); } - - table_of_sets = nullptr; } // LOCKING: must be called with lock taken -RawHashTable * +std::unique_ptr HttpBodyFactory::load_sets_from_directory(char *set_dir) { DIR *dir; struct dirent *dirEntry; - RawHashTable *new_table_of_sets; + std::unique_ptr new_table_of_sets; if (set_dir == nullptr) { - return (nullptr); + return nullptr; } Debug("body_factory", "load_sets_from_directory(%s)", set_dir); @@ -798,10 +735,10 @@ HttpBodyFactory::load_sets_from_directory(char *set_dir) if (dir == nullptr) { Warning("can't open response template directory '%s' (%s)", set_dir, (strerror(errno) ? strerror(errno) : "unknown reason")); Warning("no response templates --- using default error pages"); - return (nullptr); + return nullptr; } - new_table_of_sets = new RawHashTable(RawHashTable_KeyType_String); + new_table_of_sets.reset(new HttpBodyFactory::BodySetTable); ////////////////////////////////////////// // loop over each language subdirectory // @@ -837,13 +774,13 @@ HttpBodyFactory::load_sets_from_directory(char *set_dir) HttpBodySet *body_set = load_body_set_from_directory(dirEntry->d_name, subdir); if (body_set != nullptr) { Debug("body_factory", " %s -> %p", dirEntry->d_name, body_set); - new_table_of_sets->setValue((RawHashTable_Key)(dirEntry->d_name), (RawHashTable_Value)body_set); + new_table_of_sets->emplace(dirEntry->d_name, body_set); } } closedir(dir); - return (new_table_of_sets); + return new_table_of_sets; } // LOCKING: must be called with lock taken @@ -931,7 +868,7 @@ HttpBodyFactory::load_body_set_from_directory(char *set_name, char *tmpl_dir) } closedir(dir); - return (body_set); + return body_set; } //////////////////////////////////////////////////////////////////////// @@ -956,9 +893,7 @@ HttpBodySet::~HttpBodySet() ats_free(set_name); ats_free(content_language); ats_free(content_charset); - if (table_of_pages) { - delete table_of_pages; - } + table_of_pages.reset(nullptr); } int @@ -972,15 +907,11 @@ HttpBodySet::init(char *set, char *dir) ink_filepath_make(info_path, sizeof(info_path), dir, ".body_factory_info"); fd = open(info_path, O_RDONLY); if (fd < 0) { - return (-1); + return -1; } this->set_name = ats_strdup(set); - - if (this->table_of_pages) { - delete (this->table_of_pages); - } - this->table_of_pages = new RawHashTable(RawHashTable_KeyType_String); + this->table_of_pages.reset(new TemplateTable); lineno = 0; @@ -1079,37 +1010,37 @@ HttpBodySet::init(char *set, char *dir) } close(fd); - return (lines_added); + return lines_added; } HttpBodyTemplate * HttpBodySet::get_template_by_name(const char *name) { - RawHashTable_Value v; - Debug("body_factory", " calling get_template_by_name(%s)", name); if (table_of_pages == nullptr) { - return (nullptr); + return nullptr; } - if (table_of_pages->getValue((RawHashTable_Key)name, &v)) { - HttpBodyTemplate *t = (HttpBodyTemplate *)v; + if (auto it = table_of_pages->find(name); it != table_of_pages->end()) { + HttpBodyTemplate *t = it->second; if ((t == nullptr) || (!t->is_sane())) { - return (nullptr); + return nullptr; } Debug("body_factory", " get_template_by_name(%s) -> (file %s, length %" PRId64 ")", name, t->template_pathname, t->byte_count); - return (t); + return t; } Debug("body_factory", " get_template_by_name(%s) -> NULL", name); - return (nullptr); + return nullptr; } void HttpBodySet::set_template_by_name(const char *name, HttpBodyTemplate *t) { - table_of_pages->setValue((RawHashTable_Key)name, (RawHashTable_Value)t); + if (name) { + table_of_pages->emplace(name, t); + } } //////////////////////////////////////////////////////////////////////// @@ -1158,10 +1089,10 @@ HttpBodyTemplate::load_from_file(char *dir, char *file) // coverity[fs_check_call] status = stat(path, &stat_buf); if (status != 0) { - return (0); + return 0; } if (!S_ISREG(stat_buf.st_mode)) { - return (0); + return 0; } /////////////////// @@ -1171,7 +1102,7 @@ HttpBodyTemplate::load_from_file(char *dir, char *file) // coverity[toctou] fd = open(path, O_RDONLY); if (fd < 0) { - return (0); + return 0; } //////////////////////////////////////// @@ -1192,7 +1123,7 @@ HttpBodyTemplate::load_from_file(char *dir, char *file) Warning("reading template file '%s', got %" PRId64 " bytes instead of %" PRId64 " (%s)", path, bytes_read, new_byte_count, (strerror(errno) ? strerror(errno) : "unknown error")); ats_free(new_template_buffer); - return (0); + return 0; } Debug("body_factory", " read %" PRId64 " bytes from '%s'", new_byte_count, path); @@ -1206,7 +1137,7 @@ HttpBodyTemplate::load_from_file(char *dir, char *file) byte_count = new_byte_count; template_pathname = ats_strdup(path); - return (1); + return 1; } char * @@ -1224,5 +1155,5 @@ HttpBodyTemplate::build_instantiated_buffer(HttpTransact::State *context, int64_ Debug("body_factory_instantiation", " after instantiation: [%s]", buffer); Debug("body_factory", " returning %" PRId64 " byte instantiated buffer", *buflen_return); - return (buffer); + return buffer; } diff --git a/proxy/http/HttpBodyFactory.h b/proxy/http/HttpBodyFactory.h index 5d78fb46b20..165b24017a2 100644 --- a/proxy/http/HttpBodyFactory.h +++ b/proxy/http/HttpBodyFactory.h @@ -61,9 +61,11 @@ #include "HttpConfig.h" #include "HttpCompat.h" #include "HttpTransact.h" -#include "tscore/RawHashTable.h" #include "tscore/ink_sprintf.h" +#include +#include + #define HTTP_BODY_TEMPLATE_MAGIC 0xB0DFAC00 #define HTTP_BODY_SET_MAGIC 0xB0DFAC55 #define HTTP_BODY_FACTORY_MAGIC 0xB0DFACFF @@ -109,11 +111,12 @@ class HttpBodyTemplate //////////////////////////////////////////////////////////////////////// struct HttpBodySetRawData { - unsigned int magic = 0; + using TemplateTable = std::unordered_map; + unsigned int magic = 0; char *set_name; char *content_language; char *content_charset; - RawHashTable *table_of_pages; + std::unique_ptr table_of_pages; }; //////////////////////////////////////////////////////////////////////// @@ -162,6 +165,7 @@ class HttpBodySet : public HttpBodySetRawData class HttpBodyFactory { public: + using BodySetTable = std::unordered_map; HttpBodyFactory(); ~HttpBodyFactory(); @@ -197,8 +201,9 @@ class HttpBodyFactory void dump_template_tables(FILE *fp = stderr); void reconfigure(); - static const char *determine_set_by_language(RawHashTable *table_of_sets, StrList *acpt_language_list, StrList *acpt_charset_list, - float *Q_best_ptr, int *La_best_ptr, int *Lc_best_ptr, int *I_best_ptr); + static const char *determine_set_by_language(std::unique_ptr &table_of_sets, StrList *acpt_language_list, + StrList *acpt_charset_list, float *Q_best_ptr, int *La_best_ptr, int *Lc_best_ptr, + int *I_best_ptr); private: char *fabricate(StrList *acpt_language_list, StrList *acpt_charset_list, const char *type, HttpTransact::State *context, @@ -225,7 +230,7 @@ class HttpBodyFactory // initialization methods // //////////////////////////// void nuke_template_tables(); - RawHashTable *load_sets_from_directory(char *set_dir); + std::unique_ptr load_sets_from_directory(char *set_dir); HttpBodySet *load_body_set_from_directory(char *set_name, char *tmpl_dir); ///////////////////////////////////////////////// @@ -254,6 +259,6 @@ class HttpBodyFactory //////////////////// unsigned int magic = HTTP_BODY_FACTORY_MAGIC; // magic for sanity checks/debugging ink_mutex mutex; // prevents reconfig/read races - bool callbacks_established = false; // all config variables present - RawHashTable *table_of_sets = nullptr; // sets of template hash tables + bool callbacks_established = false; // all config variables present + std::unique_ptr table_of_sets; // sets of template hash tables }; diff --git a/proxy/http/HttpCacheSM.cc b/proxy/http/HttpCacheSM.cc index 52190bc4c17..8a859f09a0a 100644 --- a/proxy/http/HttpCacheSM.cc +++ b/proxy/http/HttpCacheSM.cc @@ -45,7 +45,7 @@ Debug("http_cache", "[%" PRId64 "] [%s, %s]", master_sm->sm_id, #state_name, HttpDebugNames::get_event_name(event)); \ } -HttpCacheAction::HttpCacheAction() : sm(nullptr) {} +HttpCacheAction::HttpCacheAction() {} void HttpCacheAction::cancel(Continuation *c) @@ -61,25 +61,9 @@ HttpCacheAction::cancel(Continuation *c) HttpCacheSM::HttpCacheSM() : Continuation(nullptr), - cache_read_vc(nullptr), - cache_write_vc(nullptr), - read_locked(false), - write_locked(false), - readwhilewrite_inprogress(false), - master_sm(nullptr), - pending_action(nullptr), - captive_action(), - open_read_cb(false), - open_write_cb(false), - open_read_tries(0), - read_request_hdr(nullptr), - http_params(nullptr), - read_pin_in_cache(0), - retry_write(true), - open_write_tries(0), - lookup_url(nullptr), - lookup_max_recursive(0), - current_lookup_level(0) + + captive_action() + { } diff --git a/proxy/http/HttpCacheSM.h b/proxy/http/HttpCacheSM.h index 435279fb436..89b8542ec58 100644 --- a/proxy/http/HttpCacheSM.h +++ b/proxy/http/HttpCacheSM.h @@ -49,7 +49,7 @@ struct HttpCacheAction : public Action { { sm = sm_arg; }; - HttpCacheSM *sm; + HttpCacheSM *sm = nullptr; }; class HttpCacheSM : public Continuation @@ -70,16 +70,16 @@ class HttpCacheSM : public Continuation Action *open_write(const HttpCacheKey *key, URL *url, HTTPHdr *request, CacheHTTPInfo *old_info, time_t pin_in_cache, bool retry, bool allow_multiple); - CacheVConnection *cache_read_vc; - CacheVConnection *cache_write_vc; + CacheVConnection *cache_read_vc = nullptr; + CacheVConnection *cache_write_vc = nullptr; - bool read_locked; - bool write_locked; + bool read_locked = false; + bool write_locked = false; // Flag to check whether read-while-write is in progress or not - bool readwhilewrite_inprogress; + bool readwhilewrite_inprogress = false; - HttpSM *master_sm; - Action *pending_action; + HttpSM *master_sm = nullptr; + Action *pending_action = nullptr; // Function to set readwhilewrite_inprogress flag inline void @@ -190,24 +190,24 @@ class HttpCacheSM : public Continuation int state_cache_open_write(int event, void *data); HttpCacheAction captive_action; - bool open_read_cb; - bool open_write_cb; + bool open_read_cb = false; + bool open_write_cb = false; // Open read parameters - int open_read_tries; - HTTPHdr *read_request_hdr; - OverridableHttpConfigParams *http_params; - time_t read_pin_in_cache; + int open_read_tries = 0; + HTTPHdr *read_request_hdr = nullptr; + OverridableHttpConfigParams *http_params = nullptr; + time_t read_pin_in_cache = 0; // Open write parameters - bool retry_write; - int open_write_tries; + bool retry_write = true; + int open_write_tries = 0; // Common parameters - URL *lookup_url; + URL *lookup_url = nullptr; HttpCacheKey cache_key; // to keep track of multiple cache lookups - int lookup_max_recursive; - int current_lookup_level; + int lookup_max_recursive = 0; + int current_lookup_level = 0; }; diff --git a/proxy/http/HttpConfig.cc b/proxy/http/HttpConfig.cc index 13b11258fe2..2ba767c5746 100644 --- a/proxy/http/HttpConfig.cc +++ b/proxy/http/HttpConfig.cc @@ -921,7 +921,7 @@ set_negative_caching_list(const char *name, RecDataT dtype, RecData data, HttpCo } else if (n <= 0 || n >= HTTP_STATUS_NUMBER) { Error("Invalid status code '%.*s' for negative caching: out of range", static_cast(token.size()), token.data()); } else { - set[n] = 1; + set[n] = true; } } } @@ -961,6 +961,7 @@ load_negative_caching_var(RecRecord const *r, void *cookie) void HttpConfig::startup() { + extern void SSLConfigInit(IpMap * map); http_rsb = RecAllocateRawStatBlock((int)http_stat_count); register_stat_callbacks(); @@ -978,6 +979,8 @@ HttpConfig::startup() RecHttpLoadIp("proxy.local.incoming_ip_to_bind", c.inbound_ip4, c.inbound_ip6); RecHttpLoadIp("proxy.local.outgoing_ip_to_bind", c.outbound_ip4, c.outbound_ip6); + RecHttpLoadIpMap("proxy.config.http.proxy_protocol_whitelist", c.config_proxy_protocol_ipmap); + SSLConfigInit(&c.config_proxy_protocol_ipmap); HttpEstablishStaticConfigLongLong(c.server_max_connections, "proxy.config.http.server_max_connections"); HttpEstablishStaticConfigLongLong(c.max_websocket_connections, "proxy.config.http.websocket.max_number_of_connections"); @@ -1149,7 +1152,6 @@ HttpConfig::startup() HttpEstablishStaticConfigByte(c.send_100_continue_response, "proxy.config.http.send_100_continue_response"); HttpEstablishStaticConfigByte(c.disallow_post_100_continue, "proxy.config.http.disallow_post_100_continue"); - HttpEstablishStaticConfigByte(c.parser_allow_non_http, "proxy.config.http.parse.allow_non_http"); HttpEstablishStaticConfigByte(c.keepalive_internal_vc, "proxy.config.http.keepalive_internal_vc"); @@ -1219,9 +1221,15 @@ HttpConfig::startup() HttpEstablishStaticConfigLongLong(c.post_copy_size, "proxy.config.http.post_copy_size"); HttpEstablishStaticConfigStringAlloc(c.redirect_actions_string, "proxy.config.http.redirect.actions"); + HttpEstablishStaticConfigStringAlloc(c.oride.ssl_client_sni_policy, "proxy.config.ssl.client.sni_policy"); + OutboundConnTrack::config_init(&c.outbound_conntrack, &c.oride.outbound_conntrack); - http_config_cont->dispatchEvent(EVENT_NONE, nullptr); + MUTEX_TRY_LOCK(lock, http_config_cont->mutex, this_ethread()); + if (!lock.is_locked()) { + ink_release_assert(0); + } + http_config_cont->handleEvent(EVENT_NONE, nullptr); return; } @@ -1430,7 +1438,6 @@ HttpConfig::reconfigure() params->send_100_continue_response = INT_TO_BOOL(m_master.send_100_continue_response); params->disallow_post_100_continue = INT_TO_BOOL(m_master.disallow_post_100_continue); - params->parser_allow_non_http = INT_TO_BOOL(m_master.parser_allow_non_http); params->keepalive_internal_vc = INT_TO_BOOL(m_master.keepalive_internal_vc); params->oride.cache_open_write_fail_action = m_master.oride.cache_open_write_fail_action; @@ -1488,6 +1495,8 @@ HttpConfig::reconfigure() params->redirect_actions_string = ats_strdup(m_master.redirect_actions_string); params->redirect_actions_map = parse_redirect_actions(params->redirect_actions_string, params->redirect_actions_self_action); + params->oride.ssl_client_sni_policy = ats_strdup(m_master.oride.ssl_client_sni_policy); + params->negative_caching_list = m_master.negative_caching_list; m_id = configProcessor.set(m_id, params); diff --git a/proxy/http/HttpConfig.h b/proxy/http/HttpConfig.h index fd61d074243..0bb0a354d64 100644 --- a/proxy/http/HttpConfig.h +++ b/proxy/http/HttpConfig.h @@ -358,11 +358,11 @@ extern RecRawStatBlock *http_rsb; // (corresponds to a "*" in the config file) ///////////////////////////////////////////////////////////// struct HttpConfigPortRange { - int low; - int high; - HttpConfigPortRange *next; + int low = 0; + int high = 0; + HttpConfigPortRange *next = nullptr; - HttpConfigPortRange() : low(0), high(0), next(nullptr) {} + HttpConfigPortRange() {} ~HttpConfigPortRange() { if (next) @@ -436,172 +436,53 @@ static std::map action_map = { // and State (txn) structure. It allows for certain configs // to be overridable per transaction more easily. struct OverridableHttpConfigParams { - OverridableHttpConfigParams() - : maintain_pristine_host_hdr(1), - chunking_enabled(1), - negative_caching_enabled(0), - negative_revalidating_enabled(0), - cache_when_to_revalidate(0), - keep_alive_enabled_in(1), - keep_alive_enabled_out(1), - keep_alive_post_out(1), - server_session_sharing_match(TS_SERVER_SESSION_SHARING_MATCH_BOTH), - auth_server_session_private(1), - fwd_proxy_auth_to_parent(0), - uncacheable_requests_bypass_parent(1), - attach_server_session_to_client(0), - forward_connect_method(0), - insert_age_in_response(1), - anonymize_remove_from(0), - anonymize_remove_referer(0), - anonymize_remove_user_agent(0), - anonymize_remove_cookie(0), - anonymize_remove_client_ip(0), - anonymize_insert_client_ip(1), - proxy_response_server_enabled(1), - proxy_response_hsts_include_subdomains(0), - insert_squid_x_forwarded_for(1), - insert_forwarded(HttpForwarded::OptionBitSet()), - send_http11_requests(1), - cache_http(1), - cache_ignore_client_no_cache(1), - cache_ignore_client_cc_max_age(0), - cache_ims_on_client_no_cache(1), - cache_ignore_server_no_cache(0), - cache_responses_to_cookies(1), - cache_ignore_auth(0), - cache_urls_that_look_dynamic(1), - cache_required_headers(2), - cache_range_lookup(1), - cache_range_write(0), - allow_multi_range(0), - cache_enable_default_vary_headers(0), - ignore_accept_mismatch(0), - ignore_accept_language_mismatch(0), - ignore_accept_encoding_mismatch(0), - ignore_accept_charset_mismatch(0), - insert_request_via_string(1), - insert_response_via_string(0), - doc_in_cache_skip_dns(1), - flow_control_enabled(0), - normalize_ae(0), - srv_enabled(0), - parent_failures_update_hostdb(0), - cache_open_write_fail_action(0), - post_check_content_length_enabled(1), - request_buffer_enabled(0), - allow_half_open(1), - ssl_client_verify_server(0), - redirect_use_orig_cache_key(0), - number_of_redirections(0), - proxy_response_hsts_max_age(-1), - negative_caching_lifetime(1800), - negative_revalidating_lifetime(1800), - sock_recv_buffer_size_out(0), - sock_send_buffer_size_out(0), - sock_option_flag_out(0), - sock_packet_mark_out(0), - sock_packet_tos_out(0), - server_tcp_init_cwnd(0), - request_hdr_max_size(131072), - response_hdr_max_size(131072), - cache_heuristic_min_lifetime(3600), - cache_heuristic_max_lifetime(86400), - cache_guaranteed_min_lifetime(0), - cache_guaranteed_max_lifetime(31536000), - cache_max_stale_age(604800), - keep_alive_no_activity_timeout_in(120), - keep_alive_no_activity_timeout_out(120), - transaction_no_activity_timeout_in(30), - transaction_no_activity_timeout_out(30), - transaction_active_timeout_out(0), - transaction_active_timeout_in(900), - websocket_active_timeout(3600), - websocket_inactive_timeout(600), - connect_attempts_max_retries(0), - connect_attempts_max_retries_dead_server(3), - connect_attempts_rr_retries(3), - connect_attempts_timeout(30), - post_connect_attempts_timeout(1800), - parent_connect_attempts(4), - parent_retry_time(300), - parent_fail_threshold(10), - per_parent_connect_attempts(2), - parent_connect_timeout(30), - down_server_timeout(300), - client_abort_threshold(10), - max_cache_open_read_retries(-1), - cache_open_read_retry_time(10), - cache_generation_number(-1), - max_cache_open_write_retries(1), - background_fill_active_timeout(60), - http_chunking_size(4096), - flow_high_water_mark(0), - flow_low_water_mark(0), - default_buffer_size_index(8), - default_buffer_water_mark(32768), - slow_log_threshold(0), - body_factory_template_base(nullptr), - body_factory_template_base_len(0), - proxy_response_server_string(nullptr), - proxy_response_server_string_len(0), - global_user_agent_header(nullptr), - global_user_agent_header_size(0), - cache_heuristic_lm_factor(0.10), - background_fill_threshold(0.5), - client_cert_filename(nullptr), - client_cert_filepath(nullptr), - cache_vary_default_text(nullptr), - cache_vary_default_images(nullptr), - cache_vary_default_other(nullptr) - { - } + OverridableHttpConfigParams() : insert_forwarded(HttpForwarded::OptionBitSet()) {} // A simple rules here: // * Place all MgmtByte configs before all other configs - MgmtByte maintain_pristine_host_hdr; - MgmtByte chunking_enabled; + MgmtByte maintain_pristine_host_hdr = 1; + MgmtByte chunking_enabled = 1; //////////////////////////////// // Negative Response Caching // //////////////////////////////// - MgmtByte negative_caching_enabled; - MgmtByte negative_revalidating_enabled; + MgmtByte negative_caching_enabled = 0; + MgmtByte negative_revalidating_enabled = 0; - MgmtByte cache_when_to_revalidate; + MgmtByte cache_when_to_revalidate = 0; - MgmtByte keep_alive_enabled_in; - MgmtByte keep_alive_enabled_out; - MgmtByte keep_alive_post_out; // share server sessions for post + MgmtByte keep_alive_enabled_in = 1; + MgmtByte keep_alive_enabled_out = 1; + MgmtByte keep_alive_post_out = 1; // share server sessions for post - MgmtByte server_session_sharing_match; + MgmtByte server_session_sharing_match = TS_SERVER_SESSION_SHARING_MATCH_BOTH; // MgmtByte share_server_sessions; - MgmtByte auth_server_session_private; - MgmtByte fwd_proxy_auth_to_parent; - MgmtByte uncacheable_requests_bypass_parent; - MgmtByte attach_server_session_to_client; + MgmtByte auth_server_session_private = 1; + MgmtByte fwd_proxy_auth_to_parent = 0; + MgmtByte uncacheable_requests_bypass_parent = 1; + MgmtByte attach_server_session_to_client = 0; - MgmtByte forward_connect_method; + MgmtByte forward_connect_method = 0; - MgmtByte insert_age_in_response; + MgmtByte insert_age_in_response = 1; /////////////////////////////////////////////////////////////////// // Privacy: fields which are removed from the user agent request // /////////////////////////////////////////////////////////////////// - MgmtByte anonymize_remove_from; - MgmtByte anonymize_remove_referer; - MgmtByte anonymize_remove_user_agent; - MgmtByte anonymize_remove_cookie; - MgmtByte anonymize_remove_client_ip; - MgmtByte anonymize_insert_client_ip; + MgmtByte anonymize_remove_from = 0; + MgmtByte anonymize_remove_referer = 0; + MgmtByte anonymize_remove_user_agent = 0; + MgmtByte anonymize_remove_cookie = 0; + MgmtByte anonymize_remove_client_ip = 0; + MgmtByte anonymize_insert_client_ip = 1; - MgmtByte proxy_response_server_enabled; - MgmtByte proxy_response_hsts_include_subdomains; + MgmtByte proxy_response_server_enabled = 1; + MgmtByte proxy_response_hsts_include_subdomains = 0; ///////////////////// // X-Forwarded-For // ///////////////////// - MgmtByte insert_squid_x_forwarded_for; + MgmtByte insert_squid_x_forwarded_for = 1; /////////////// // Forwarded // @@ -611,193 +492,197 @@ struct OverridableHttpConfigParams { ////////////////////// // Version Hell // ////////////////////// - MgmtByte send_http11_requests; + MgmtByte send_http11_requests = 1; /////////////////// // cache control // /////////////////// - MgmtByte cache_http; - MgmtByte cache_ignore_client_no_cache; - MgmtByte cache_ignore_client_cc_max_age; - MgmtByte cache_ims_on_client_no_cache; - MgmtByte cache_ignore_server_no_cache; - MgmtByte cache_responses_to_cookies; - MgmtByte cache_ignore_auth; - MgmtByte cache_urls_that_look_dynamic; - MgmtByte cache_required_headers; - MgmtByte cache_range_lookup; - MgmtByte cache_range_write; - MgmtByte allow_multi_range; - - MgmtByte cache_enable_default_vary_headers; - - MgmtByte ignore_accept_mismatch; - MgmtByte ignore_accept_language_mismatch; - MgmtByte ignore_accept_encoding_mismatch; - MgmtByte ignore_accept_charset_mismatch; - - MgmtByte insert_request_via_string; - MgmtByte insert_response_via_string; + MgmtByte cache_http = 1; + MgmtByte cache_ignore_client_no_cache = 1; + MgmtByte cache_ignore_client_cc_max_age = 0; + MgmtByte cache_ims_on_client_no_cache = 1; + MgmtByte cache_ignore_server_no_cache = 0; + MgmtByte cache_responses_to_cookies = 1; + MgmtByte cache_ignore_auth = 0; + MgmtByte cache_urls_that_look_dynamic = 1; + MgmtByte cache_required_headers = 2; + MgmtByte cache_range_lookup = 1; + MgmtByte cache_range_write = 0; + MgmtByte allow_multi_range = 0; + + MgmtByte cache_enable_default_vary_headers = 0; + + MgmtByte ignore_accept_mismatch = 0; + MgmtByte ignore_accept_language_mismatch = 0; + MgmtByte ignore_accept_encoding_mismatch = 0; + MgmtByte ignore_accept_charset_mismatch = 0; + + MgmtByte insert_request_via_string = 1; + MgmtByte insert_response_via_string = 0; ////////////////////// // DOC IN CACHE NO DNS// ////////////////////// - MgmtByte doc_in_cache_skip_dns; - MgmtByte flow_control_enabled; + MgmtByte doc_in_cache_skip_dns = 1; + MgmtByte flow_control_enabled = 0; //////////////////////////////// // Optimize gzip alternates // //////////////////////////////// - MgmtByte normalize_ae; + MgmtByte normalize_ae = 0; ////////////////////////// // hostdb/dns variables // ////////////////////////// - MgmtByte srv_enabled; - MgmtByte parent_failures_update_hostdb; + MgmtByte srv_enabled = 0; + MgmtByte parent_failures_update_hostdb = 0; - MgmtByte cache_open_write_fail_action; + MgmtByte cache_open_write_fail_action = 0; //////////////////////// // Check Post request // //////////////////////// - MgmtByte post_check_content_length_enabled; + MgmtByte post_check_content_length_enabled = 1; //////////////////////////////////////////////// // Buffer post body before connecting servers // //////////////////////////////////////////////// - MgmtByte request_buffer_enabled; + MgmtByte request_buffer_enabled = 0; ///////////////////////////////////////////////// // Keep connection open after client sends FIN // ///////////////////////////////////////////////// - MgmtByte allow_half_open; + MgmtByte allow_half_open = 1; ///////////////////////////// // server verification mode// ///////////////////////////// - MgmtByte ssl_client_verify_server; + MgmtByte ssl_client_verify_server = 0; + char *ssl_client_verify_server_policy = nullptr; + char *ssl_client_verify_server_properties = nullptr; + char *ssl_client_sni_policy = nullptr; ////////////////// // Redirection // ////////////////// - MgmtByte redirect_use_orig_cache_key; - MgmtInt number_of_redirections; + MgmtByte redirect_use_orig_cache_key = 0; + MgmtInt number_of_redirections = 0; - MgmtInt proxy_response_hsts_max_age; + MgmtInt proxy_response_hsts_max_age = -1; //////////////////////////////// // Negative cache lifetimes // //////////////////////////////// - MgmtInt negative_caching_lifetime; - MgmtInt negative_revalidating_lifetime; + MgmtInt negative_caching_lifetime = 1800; + MgmtInt negative_revalidating_lifetime = 1800; /////////////////////////////////////// // origin server connection settings // /////////////////////////////////////// - MgmtInt sock_recv_buffer_size_out; - MgmtInt sock_send_buffer_size_out; - MgmtInt sock_option_flag_out; - MgmtInt sock_packet_mark_out; - MgmtInt sock_packet_tos_out; + MgmtInt sock_recv_buffer_size_out = 0; + MgmtInt sock_send_buffer_size_out = 0; + MgmtInt sock_option_flag_out = 0; + MgmtInt sock_packet_mark_out = 0; + MgmtInt sock_packet_tos_out = 0; /////////////////////////////// // Initial congestion window // /////////////////////////////// - MgmtInt server_tcp_init_cwnd; + MgmtInt server_tcp_init_cwnd = 0; /////////////// // Hdr Limit // /////////////// - MgmtInt request_hdr_max_size; - MgmtInt response_hdr_max_size; + MgmtInt request_hdr_max_size = 131072; + MgmtInt response_hdr_max_size = 131072; ///////////////////// // cache variables // ///////////////////// - MgmtInt cache_heuristic_min_lifetime; - MgmtInt cache_heuristic_max_lifetime; - MgmtInt cache_guaranteed_min_lifetime; - MgmtInt cache_guaranteed_max_lifetime; - MgmtInt cache_max_stale_age; + MgmtInt cache_heuristic_min_lifetime = 3600; + MgmtInt cache_heuristic_max_lifetime = 86400; + MgmtInt cache_guaranteed_min_lifetime = 0; + MgmtInt cache_guaranteed_max_lifetime = 31536000; + MgmtInt cache_max_stale_age = 604800; /////////////////////////////////////////////////// // connection variables. timeouts are in seconds // /////////////////////////////////////////////////// - MgmtInt keep_alive_no_activity_timeout_in; - MgmtInt keep_alive_no_activity_timeout_out; - MgmtInt transaction_no_activity_timeout_in; - MgmtInt transaction_no_activity_timeout_out; - MgmtInt transaction_active_timeout_out; - MgmtInt transaction_active_timeout_in; - MgmtInt websocket_active_timeout; - MgmtInt websocket_inactive_timeout; + MgmtInt keep_alive_no_activity_timeout_in = 120; + MgmtInt keep_alive_no_activity_timeout_out = 120; + MgmtInt transaction_no_activity_timeout_in = 30; + MgmtInt transaction_no_activity_timeout_out = 30; + MgmtInt transaction_active_timeout_out = 0; + MgmtInt transaction_active_timeout_in = 900; + MgmtInt websocket_active_timeout = 3600; + MgmtInt websocket_inactive_timeout = 600; //////////////////////////////////// // origin server connect attempts // //////////////////////////////////// - MgmtInt connect_attempts_max_retries; - MgmtInt connect_attempts_max_retries_dead_server; - MgmtInt connect_attempts_rr_retries; - MgmtInt connect_attempts_timeout; - MgmtInt post_connect_attempts_timeout; + MgmtInt connect_attempts_max_retries = 0; + MgmtInt connect_attempts_max_retries_dead_server = 3; + MgmtInt connect_attempts_rr_retries = 3; + MgmtInt connect_attempts_timeout = 30; + MgmtInt post_connect_attempts_timeout = 1800; //////////////////////////////////// // parent proxy connect attempts // /////////////////////////////////// - MgmtInt parent_connect_attempts; - MgmtInt parent_retry_time; - MgmtInt parent_fail_threshold; - MgmtInt per_parent_connect_attempts; - MgmtInt parent_connect_timeout; + MgmtInt parent_connect_attempts = 4; + MgmtInt parent_retry_time = 300; + MgmtInt parent_fail_threshold = 10; + MgmtInt per_parent_connect_attempts = 2; + MgmtInt parent_connect_timeout = 30; - MgmtInt down_server_timeout; - MgmtInt client_abort_threshold; + MgmtInt down_server_timeout = 300; + MgmtInt client_abort_threshold = 10; // open read failure retries. - MgmtInt max_cache_open_read_retries; - MgmtInt cache_open_read_retry_time; // time is in mseconds - MgmtInt cache_generation_number; + MgmtInt max_cache_open_read_retries = -1; + MgmtInt cache_open_read_retry_time = 10; // time is in mseconds + MgmtInt cache_generation_number = -1; // open write failure retries. - MgmtInt max_cache_open_write_retries; + MgmtInt max_cache_open_write_retries = 1; - MgmtInt background_fill_active_timeout; + MgmtInt background_fill_active_timeout = 60; - MgmtInt http_chunking_size; // Maximum chunk size for chunked output. - MgmtInt flow_high_water_mark; ///< Flow control high water mark. - MgmtInt flow_low_water_mark; ///< Flow control low water mark. + MgmtInt http_chunking_size = 4096; // Maximum chunk size for chunked output. + MgmtInt flow_high_water_mark = 0; ///< Flow control high water mark. + MgmtInt flow_low_water_mark = 0; ///< Flow control low water mark. - MgmtInt default_buffer_size_index; - MgmtInt default_buffer_water_mark; - MgmtInt slow_log_threshold; + MgmtInt default_buffer_size_index = 8; + MgmtInt default_buffer_water_mark = 32768; + MgmtInt slow_log_threshold = 0; OutboundConnTrack::TxnConfig outbound_conntrack; /////////////////////////////////////////////////////////////////// // Server header // /////////////////////////////////////////////////////////////////// - char *body_factory_template_base; - size_t body_factory_template_base_len; - char *proxy_response_server_string; // This does not get free'd by us! - size_t proxy_response_server_string_len; // Updated when server_string is set. + char *body_factory_template_base = nullptr; + size_t body_factory_template_base_len = 0; + char *proxy_response_server_string = nullptr; // This does not get free'd by us! + size_t proxy_response_server_string_len = 0; // Updated when server_string is set. /////////////////////////////////////////////////////////////////// // Global User Agent header // /////////////////////////////////////////////////////////////////// - char *global_user_agent_header; // This does not get free'd by us! - size_t global_user_agent_header_size; // Updated when user_agent is set. + char *global_user_agent_header = nullptr; // This does not get free'd by us! + size_t global_user_agent_header_size = 0; // Updated when user_agent is set. - MgmtFloat cache_heuristic_lm_factor; - MgmtFloat background_fill_threshold; + MgmtFloat cache_heuristic_lm_factor = 0.10; + MgmtFloat background_fill_threshold = 0.5; // Various strings, good place for them here ... - char *client_cert_filename; - char *client_cert_filepath; + char *ssl_client_cert_filename = nullptr; + char *ssl_client_private_key_filename = nullptr; + char *ssl_client_ca_cert_filename = nullptr; - char *cache_vary_default_text; - char *cache_vary_default_images; - char *cache_vary_default_other; + char *cache_vary_default_text = nullptr; + char *cache_vary_default_images = nullptr; + char *cache_vary_default_other = nullptr; }; ///////////////////////////////////////////////////////////// @@ -828,6 +713,8 @@ struct HttpConfigParams : public ConfigInfo { public: IpAddr inbound_ip4, inbound_ip6; IpAddr outbound_ip4, outbound_ip6; + IpAddr proxy_protocol_ip4, proxy_protocol_ip6; + IpMap config_proxy_protocol_ipmap; MgmtInt server_max_connections = 0; MgmtInt origin_min_keep_alive_connections = 0; // TODO: This one really ought to be overridable, but difficult right now. @@ -894,7 +781,6 @@ struct HttpConfigParams : public ConfigInfo { MgmtByte send_100_continue_response = 0; MgmtByte disallow_post_100_continue = 0; - MgmtByte parser_allow_non_http = 1; MgmtByte keepalive_internal_vc = 0; MgmtByte server_session_sharing_pool = TS_SERVER_SESSION_SHARING_POOL_THREAD; @@ -962,14 +848,16 @@ inline HttpConfigParams::~HttpConfigParams() ats_free(oride.body_factory_template_base); ats_free(oride.proxy_response_server_string); ats_free(oride.global_user_agent_header); - ats_free(oride.client_cert_filename); - ats_free(oride.client_cert_filepath); + ats_free(oride.ssl_client_cert_filename); + ats_free(oride.ssl_client_private_key_filename); + ats_free(oride.ssl_client_ca_cert_filename); ats_free(oride.cache_vary_default_text); ats_free(oride.cache_vary_default_images); ats_free(oride.cache_vary_default_other); ats_free(connect_ports_string); ats_free(reverse_proxy_no_host_redirect); ats_free(redirect_actions_string); + ats_free(oride.ssl_client_sni_policy); delete connect_ports; delete redirect_actions_map; diff --git a/proxy/http/HttpConnectionCount.cc b/proxy/http/HttpConnectionCount.cc index 3b596354312..807f6c9bebe 100644 --- a/proxy/http/HttpConnectionCount.cc +++ b/proxy/http/HttpConnectionCount.cc @@ -33,13 +33,9 @@ OutboundConnTrack::Imp OutboundConnTrack::_imp; OutboundConnTrack::GlobalConfig *OutboundConnTrack::_global_config{nullptr}; -const MgmtConverter OutboundConnTrack::MAX_CONV{ +const MgmtConverter OutboundConnTrack::MAX_CONV( [](void *data) -> MgmtInt { return static_cast(*static_cast(data)); }, - [](void *data, MgmtInt i) -> void { *static_cast(data) = static_cast(i); }, - nullptr, - nullptr, - nullptr, - nullptr}; + [](void *data, MgmtInt i) -> void { *static_cast(data) = static_cast(i); }); // Do integer and string conversions. const MgmtConverter OutboundConnTrack::MATCH_CONV{ diff --git a/proxy/http/HttpDebugNames.cc b/proxy/http/HttpDebugNames.cc index a67916710ad..f99e5b1c135 100644 --- a/proxy/http/HttpDebugNames.cc +++ b/proxy/http/HttpDebugNames.cc @@ -28,6 +28,8 @@ #include "Transform.h" #include "HttpSM.h" #include "HttpUpdateSM.h" +#include +#include //---------------------------------------------------------------------------- const char * @@ -103,119 +105,131 @@ const char * HttpDebugNames::get_event_name(int event) { switch (event) { - ///////////////////////// - // VCONNECTION EVENTS // - ///////////////////////// - case VC_EVENT_NONE: - return ("VC_EVENT_NONE"); - case VC_EVENT_IMMEDIATE: - return ("VC_EVENT_IMMEDIATE"); + case EVENT_NONE: + static_assert(static_cast(EVENT_NONE) == static_cast(VC_EVENT_NONE)); + return "EVENT_NONE/VC_EVENT_NONE"; + case EVENT_IMMEDIATE: + static_assert(static_cast(EVENT_IMMEDIATE) == static_cast(TS_EVENT_IMMEDIATE)); + static_assert(static_cast(EVENT_IMMEDIATE) == static_cast(VC_EVENT_IMMEDIATE)); + return "EVENT_IMMEDIATE/TS_EVENT_IMMEDIATE/VC_EVENT_IMMEDIATE"; + case EVENT_ERROR: + static_assert(static_cast(EVENT_ERROR) == static_cast(TS_EVENT_ERROR)); + static_assert(static_cast(EVENT_ERROR) == static_cast(VC_EVENT_ERROR)); + return "EVENT_ERROR/TS_EVENT_ERROR/VC_EVENT_ERROR"; + case EVENT_INTERVAL: + return "EVENT_INTERVAL"; case VC_EVENT_READ_READY: - return ("VC_EVENT_READ_READY"); + static_assert(static_cast(VC_EVENT_READ_READY) == static_cast(TS_EVENT_VCONN_READ_READY)); + return "VC_EVENT_READ_READY/TS_EVENT_VCONN_READ_READY"; case VC_EVENT_WRITE_READY: - return ("VC_EVENT_WRITE_READY"); + static_assert(static_cast(VC_EVENT_WRITE_READY) == static_cast(TS_EVENT_VCONN_WRITE_READY)); + return "VC_EVENT_WRITE_READY/TS_EVENT_VCONN_WRITE_READY"; case VC_EVENT_READ_COMPLETE: - return ("VC_EVENT_READ_COMPLETE"); + static_assert(static_cast(VC_EVENT_READ_COMPLETE) == static_cast(TS_EVENT_VCONN_READ_COMPLETE)); + return "VC_EVENT_READ_COMPLETE/TS_EVENT_VCONN_READ_COMPLETE"; case VC_EVENT_WRITE_COMPLETE: - return ("VC_EVENT_WRITE_COMPLETE"); + static_assert(static_cast(VC_EVENT_WRITE_COMPLETE) == static_cast(TS_EVENT_VCONN_WRITE_COMPLETE)); + return "VC_EVENT_WRITE_COMPLETE/TS_EVENT_VCONN_WRITE_COMPLETE"; case VC_EVENT_EOS: - return ("VC_EVENT_EOS"); - case VC_EVENT_ERROR: - return ("VC_EVENT_ERROR"); + static_assert(static_cast(VC_EVENT_EOS) == static_cast(TS_EVENT_VCONN_EOS)); + return "VC_EVENT_EOS/TS_EVENT_VCONN_EOS"; case VC_EVENT_INACTIVITY_TIMEOUT: - return ("VC_EVENT_INACTIVITY_TIMEOUT"); + static_assert(static_cast(VC_EVENT_INACTIVITY_TIMEOUT) == static_cast(TS_EVENT_VCONN_INACTIVITY_TIMEOUT)); + return "VC_EVENT_INACTIVITY_TIMEOUT/TS_EVENT_VCONN_INACTIVITY_TIMEOUT"; case VC_EVENT_ACTIVE_TIMEOUT: - return ("VC_EVENT_ACTIVE_TIMEOUT"); - case EVENT_INTERVAL: - return ("VC_EVENT_INTERVAL"); + static_assert(static_cast(VC_EVENT_ACTIVE_TIMEOUT) == static_cast(TS_EVENT_VCONN_ACTIVE_TIMEOUT)); + return "VC_EVENT_ACTIVE_TIMEOUT/TS_EVENT_VCONN_ACTIVE_TIMEOUT"; ///////////////// // NET EVENTS // ///////////////// case NET_EVENT_OPEN: - return ("NET_EVENT_OPEN"); + static_assert(static_cast(NET_EVENT_OPEN) == static_cast(TS_EVENT_NET_CONNECT)); + return "NET_EVENT_OPEN/TS_EVENT_NET_CONNECT"; case NET_EVENT_ACCEPT: - return ("NET_EVENT_ACCEPT"); + static_assert(static_cast(NET_EVENT_ACCEPT) == static_cast(TS_EVENT_NET_ACCEPT)); + return "NET_EVENT_ACCEPT/TS_EVENT_NET_ACCEPT"; case NET_EVENT_OPEN_FAILED: - return ("NET_EVENT_OPEN_FAILED"); + static_assert(static_cast(NET_EVENT_OPEN_FAILED) == static_cast(TS_EVENT_NET_CONNECT_FAILED)); + return "NET_EVENT_OPEN_FAILED/TS_EVENT_NET_CONNECT_FAILED"; //////////////////// // HOSTDB EVENTS // //////////////////// case EVENT_HOST_DB_LOOKUP: - return ("EVENT_HOST_DB_LOOKUP"); - + static_assert(static_cast(EVENT_HOST_DB_LOOKUP) == static_cast(TS_EVENT_HOST_LOOKUP)); + return "EVENT_HOST_DB_LOOKUP/TS_EVENT_HOST_LOOKUP"; case EVENT_HOST_DB_GET_RESPONSE: - return ("EVENT_HOST_DB_GET_RESPONSE"); - - //////////////////// - // HOSTDB EVENTS // - //////////////////// + return "EVENT_HOST_DB_GET_RESPONSE"; case EVENT_SRV_LOOKUP: - return ("EVENT_SRV_LOOKUP"); - + return "EVENT_SRV_LOOKUP"; case EVENT_SRV_IP_REMOVED: - return ("EVENT_SRV_IP_REMOVED"); - + return "EVENT_SRV_IP_REMOVED"; case EVENT_SRV_GET_RESPONSE: - return ("EVENT_SRV_GET_RESPONSE"); + return "EVENT_SRV_GET_RESPONSE"; //////////////////// // DNS EVENTS // //////////////////// case DNS_EVENT_LOOKUP: - return ("DNS_EVENT_LOOKUP"); - - //////////////////// - // CACHE EVENTS // - //////////////////// + return "DNS_EVENT_LOOKUP"; - case CACHE_EVENT_LOOKUP: - return ("CACHE_EVENT_LOOKUP"); + //////////////////// + // CACHE EVENTS // + //////////////////// case CACHE_EVENT_LOOKUP_FAILED: - return ("CACHE_EVENT_LOOKUP_FAILED"); + return "CACHE_EVENT_LOOKUP_FAILED"; case CACHE_EVENT_OPEN_READ: - return ("CACHE_EVENT_OPEN_READ"); + static_assert(static_cast(CACHE_EVENT_OPEN_READ) == static_cast(TS_EVENT_CACHE_OPEN_READ)); + return "CACHE_EVENT_OPEN_READ/TS_EVENT_CACHE_OPEN_READ"; case CACHE_EVENT_OPEN_READ_FAILED: - return ("CACHE_EVENT_OPEN_READ_FAILED"); + static_assert(static_cast(CACHE_EVENT_OPEN_READ_FAILED) == static_cast(TS_EVENT_CACHE_OPEN_READ_FAILED)); + return "CACHE_EVENT_OPEN_READ_FAILED/TS_EVENT_CACHE_OPEN_READ_FAILED"; case CACHE_EVENT_OPEN_WRITE: - return ("CACHE_EVENT_OPEN_WRITE"); + static_assert(static_cast(CACHE_EVENT_OPEN_WRITE) == static_cast(TS_EVENT_CACHE_OPEN_WRITE)); + return "CACHE_EVENT_OPEN_WRITE/TS_EVENT_CACHE_OPEN_WRITE"; case CACHE_EVENT_OPEN_WRITE_FAILED: - return ("CACHE_EVENT_OPEN_WRITE_FAILED"); + static_assert(static_cast(CACHE_EVENT_OPEN_WRITE_FAILED) == static_cast(TS_EVENT_CACHE_OPEN_WRITE_FAILED)); + return "CACHE_EVENT_OPEN_WRITE_FAILED/TS_EVENT_CACHE_OPEN_WRITE_FAILED"; case CACHE_EVENT_REMOVE: - return ("CACHE_EVENT_REMOVE"); + static_assert(static_cast(CACHE_EVENT_REMOVE) == static_cast(TS_EVENT_CACHE_REMOVE)); + return "CACHE_EVENT_REMOVE/TS_EVENT_CACHE_REMOVE"; case CACHE_EVENT_REMOVE_FAILED: - return ("CACHE_EVENT_REMOVE_FAILED"); + static_assert(static_cast(CACHE_EVENT_REMOVE_FAILED) == static_cast(TS_EVENT_CACHE_REMOVE_FAILED)); + return "CACHE_EVENT_REMOVE_FAILED/TS_EVENT_CACHE_REMOVE_FAILED"; case CACHE_EVENT_UPDATE: - return ("CACHE_EVENT_UPDATE"); + return "CACHE_EVENT_UPDATE"; case CACHE_EVENT_UPDATE_FAILED: - return ("CACHE_EVENT_UPDATE_FAILED"); + return "CACHE_EVENT_UPDATE_FAILED"; case STAT_PAGE_SUCCESS: - return ("STAT_PAGE_SUCCESS"); + return "STAT_PAGE_SUCCESS"; case STAT_PAGE_FAILURE: - return ("STAT_PAGE_FAILURE"); + return "STAT_PAGE_FAILURE"; case TRANSFORM_READ_READY: - return ("TRANSFORM_READ_READY"); + static_assert(static_cast(TRANSFORM_READ_READY) == static_cast(TS_EVENT_SSL_SESSION_GET)); + return "TRANSFORM_READ_READY/TS_EVENT_SSL_SESSION_GET"; ///////////////////////// - // HttpTunnel Events // + // HttpTunnel Events // ///////////////////////// case HTTP_TUNNEL_EVENT_DONE: - return ("HTTP_TUNNEL_EVENT_DONE"); + return "HTTP_TUNNEL_EVENT_DONE"; case HTTP_TUNNEL_EVENT_PRECOMPLETE: - return ("HTTP_TUNNEL_EVENT_PRECOMPLETE"); + return "HTTP_TUNNEL_EVENT_PRECOMPLETE"; case HTTP_TUNNEL_EVENT_CONSUMER_DETACH: - return ("HTTP_TUNNEL_EVENT_CONSUMER_DETACH"); + return "HTTP_TUNNEL_EVENT_CONSUMER_DETACH"; - ////////////////////////////// + ///////////////////////////// // Plugin Events - ////////////////////////////// + ///////////////////////////// case HTTP_API_CONTINUE: - return ("HTTP_API_CONTINUE"); + static_assert(static_cast(HTTP_API_CONTINUE) == static_cast(TS_EVENT_HTTP_CONTINUE)); + return "HTTP_API_CONTINUE/TS_EVENT_HTTP_CONTINUE"; case HTTP_API_ERROR: - return ("HTTP_API_ERROR"); + static_assert(static_cast(HTTP_API_ERROR) == static_cast(TS_EVENT_HTTP_ERROR)); + return "HTTP_API_ERROR/TS_EVENT_HTTP_ERROR"; /////////////////////////////// // Scheduled Update Events @@ -232,6 +246,139 @@ HttpDebugNames::get_event_name(int event) return "HTTP_SCH_UPDATE_EVENT_ERROR"; case HTTP_SCH_UPDATE_EVENT_NO_ACTION: return "HTTP_SCH_UPDATE_EVENT_NO_ACTION"; + + case TS_EVENT_NET_ACCEPT_FAILED: + return "TS_EVENT_NET_ACCEPT_FAILED"; + case TS_EVENT_INTERNAL_206: + return "TS_EVENT_INTERNAL_206"; + case TS_EVENT_INTERNAL_207: + return "TS_EVENT_INTERNAL_207"; + case TS_EVENT_INTERNAL_208: + return "TS_EVENT_INTERNAL_208"; + case TS_EVENT_INTERNAL_209: + return "TS_EVENT_INTERNAL_209"; + case TS_EVENT_INTERNAL_210: + return "TS_EVENT_INTERNAL_210"; + case TS_EVENT_INTERNAL_211: + return "TS_EVENT_INTERNAL_211"; + case TS_EVENT_INTERNAL_212: + return "TS_EVENT_INTERNAL_212"; + case TS_EVENT_CACHE_SCAN: + return "TS_EVENT_CACHE_SCAN"; + case TS_EVENT_CACHE_SCAN_FAILED: + return "TS_EVENT_CACHE_SCAN_FAILED"; + case TS_EVENT_CACHE_SCAN_OBJECT: + return "TS_EVENT_CACHE_SCAN_OBJECT"; + case TS_EVENT_CACHE_SCAN_OPERATION_BLOCKED: + return "TS_EVENT_CACHE_SCAN_OPERATION_BLOCKED"; + case TS_EVENT_CACHE_SCAN_OPERATION_FAILED: + return "TS_EVENT_CACHE_SCAN_OPERATION_FAILED"; + case TS_EVENT_CACHE_SCAN_DONE: + return "TS_EVENT_CACHE_SCAN_DONE"; + case TS_EVENT_CACHE_LOOKUP: + return "TS_EVENT_CACHE_LOOKUP"; + case TS_EVENT_CACHE_READ: + return "TS_EVENT_CACHE_READ"; + case TS_EVENT_CACHE_DELETE: + return "TS_EVENT_CACHE_DELETE"; + case TS_EVENT_CACHE_WRITE: + return "TS_EVENT_CACHE_WRITE"; + case TS_EVENT_CACHE_WRITE_HEADER: + return "TS_EVENT_CACHE_WRITE_HEADER"; + case TS_EVENT_CACHE_CLOSE: + return "TS_EVENT_CACHE_CLOSE"; + case TS_EVENT_CACHE_LOOKUP_READY: + return "TS_EVENT_CACHE_LOOKUP_READY"; + case TS_EVENT_CACHE_LOOKUP_COMPLETE: + return "TS_EVENT_CACHE_LOOKUP_COMPLETE"; + case TS_EVENT_CACHE_READ_READY: + return "TS_EVENT_CACHE_READ_READY"; + case TS_EVENT_CACHE_READ_COMPLETE: + return "TS_EVENT_CACHE_READ_COMPLETE"; + case TS_EVENT_INTERNAL_1200: + return "TS_EVENT_INTERNAL_1200"; + case TS_EVENT_SSL_SESSION_NEW: + return "TS_EVENT_SSL_SESSION_NEW"; + case TS_EVENT_SSL_SESSION_REMOVE: + return "TS_EVENT_SSL_SESSION_REMOVE"; + case TS_EVENT_AIO_DONE: + return "TS_EVENT_AIO_DONE"; + case TS_EVENT_HTTP_READ_REQUEST_HDR: + return "TS_EVENT_HTTP_READ_REQUEST_HDR"; + case TS_EVENT_HTTP_OS_DNS: + return "TS_EVENT_HTTP_OS_DNS"; + case TS_EVENT_HTTP_SEND_REQUEST_HDR: + return "TS_EVENT_HTTP_SEND_REQUEST_HDR"; + case TS_EVENT_HTTP_READ_CACHE_HDR: + return "TS_EVENT_HTTP_READ_CACHE_HDR"; + case TS_EVENT_HTTP_READ_RESPONSE_HDR: + return "TS_EVENT_HTTP_READ_RESPONSE_HDR"; + case TS_EVENT_HTTP_SEND_RESPONSE_HDR: + return "TS_EVENT_HTTP_SEND_RESPONSE_HDR"; + case TS_EVENT_HTTP_REQUEST_TRANSFORM: + return "TS_EVENT_HTTP_REQUEST_TRANSFORM"; + case TS_EVENT_HTTP_RESPONSE_TRANSFORM: + return "TS_EVENT_HTTP_RESPONSE_TRANSFORM"; + case TS_EVENT_HTTP_SELECT_ALT: + return "TS_EVENT_HTTP_SELECT_ALT"; + case TS_EVENT_HTTP_TXN_START: + return "TS_EVENT_HTTP_TXN_START"; + case TS_EVENT_HTTP_TXN_CLOSE: + return "TS_EVENT_HTTP_TXN_CLOSE"; + case TS_EVENT_HTTP_SSN_START: + return "TS_EVENT_HTTP_SSN_START"; + case TS_EVENT_HTTP_SSN_CLOSE: + return "TS_EVENT_HTTP_SSN_CLOSE"; + case TS_EVENT_HTTP_CACHE_LOOKUP_COMPLETE: + return "TS_EVENT_HTTP_CACHE_LOOKUP_COMPLETE"; + case TS_EVENT_HTTP_PRE_REMAP: + return "TS_EVENT_HTTP_PRE_REMAP"; + case TS_EVENT_HTTP_POST_REMAP: + return "TS_EVENT_HTTP_POST_REMAP"; + case TS_EVENT_LIFECYCLE_PORTS_INITIALIZED: + return "TS_EVENT_LIFECYCLE_PORTS_INITIALIZED"; + case TS_EVENT_LIFECYCLE_PORTS_READY: + return "TS_EVENT_LIFECYCLE_PORTS_READY"; + case TS_EVENT_LIFECYCLE_CACHE_READY: + return "TS_EVENT_LIFECYCLE_CACHE_READY"; + case TS_EVENT_LIFECYCLE_SERVER_SSL_CTX_INITIALIZED: + return "TS_EVENT_LIFECYCLE_SERVER_SSL_CTX_INITIALIZED"; + case TS_EVENT_LIFECYCLE_CLIENT_SSL_CTX_INITIALIZED: + return "TS_EVENT_LIFECYCLE_CLIENT_SSL_CTX_INITIALIZED"; + case TS_EVENT_LIFECYCLE_TASK_THREADS_READY: + return "TS_EVENT_LIFECYCLE_TASK_THREADS_READY"; + case TS_EVENT_LIFECYCLE_SHUTDOWN: + return "TS_EVENT_LIFECYCLE_SHUTDOWN"; + case TS_EVENT_VCONN_START: + return "TS_EVENT_VCONN_START"; + case TS_EVENT_VCONN_CLOSE: + return "TS_EVENT_VCONN_CLOSE"; + case TS_EVENT_LIFECYCLE_MSG: + return "TS_EVENT_LIFECYCLE_MSG"; + case TS_EVENT_HTTP_REQUEST_BUFFER_COMPLETE: + return "TS_EVENT_HTTP_REQUEST_BUFFER_COMPLETE"; + case TS_EVENT_MGMT_UPDATE: + return "TS_EVENT_MGMT_UPDATE"; + case TS_EVENT_INTERNAL_60200: + return "TS_EVENT_INTERNAL_60200"; + case TS_EVENT_INTERNAL_60201: + return "TS_EVENT_INTERNAL_60201"; + case TS_EVENT_INTERNAL_60202: + return "TS_EVENT_INTERNAL_60202"; + case TS_EVENT_SSL_CLIENT_HELLO: + return "TS_EVENT_SSL_CLIENT_HELLO"; + case TS_EVENT_SSL_CERT: + return "TS_EVENT_SSL_CERT"; + case TS_EVENT_SSL_SERVERNAME: + return "TS_EVENT_SSL_SERVERNAME"; + case TS_EVENT_SSL_VERIFY_SERVER: + return "TS_EVENT_SSL_VERIFY_SERVER"; + case TS_EVENT_SSL_VERIFY_CLIENT: + return "TS_EVENT_SSL_VERIFY_CLIENT"; + case TS_EVENT_VCONN_OUTBOUND_START: + return "TS_EVENT_VCONN_OUTBOUND_START"; + case TS_EVENT_VCONN_OUTBOUND_CLOSE: + return "TS_EVENT_VCONN_OUTBOUND_CLOSE"; } return ("unknown event"); @@ -462,6 +609,8 @@ HttpDebugNames::get_api_hook_name(TSHttpHookID t) return "TS_VCONN_START_HOOK"; case TS_VCONN_CLOSE_HOOK: return "TS_VCONN_CLOSE_HOOK"; + case TS_SSL_CLIENT_HELLO_HOOK: + return "TS_SSL_CLIENT_HELLO_HOOK"; case TS_SSL_CERT_HOOK: return "TS_SSL_CERT_HOOK"; case TS_SSL_SERVERNAME_HOOK: diff --git a/proxy/http/HttpPages.h b/proxy/http/HttpPages.h index b51e45ed9d8..7b9fea2337f 100644 --- a/proxy/http/HttpPages.h +++ b/proxy/http/HttpPages.h @@ -35,7 +35,6 @@ #include "tscore/ink_platform.h" #include "P_EventSystem.h" -#include "tscore/DynArray.h" #include "HTTP.h" #include "StatPages.h" #include "HttpSM.h" diff --git a/proxy/http/HttpProxyServerMain.cc b/proxy/http/HttpProxyServerMain.cc index a342e276f31..631d54201d5 100644 --- a/proxy/http/HttpProxyServerMain.cc +++ b/proxy/http/HttpProxyServerMain.cc @@ -119,12 +119,12 @@ ssl_unregister_protocol(const char *protocol, Continuation *contp) */ struct HttpProxyAcceptor { /// Accept continuation. - Continuation *_accept; + Continuation *_accept = nullptr; /// Options for @c NetProcessor. NetProcessor::AcceptOptions _net_opt; /// Default constructor. - HttpProxyAcceptor() : _accept(nullptr) {} + HttpProxyAcceptor() {} }; /** Global acceptors. @@ -151,6 +151,10 @@ make_net_accept_options(const HttpProxyPort *port, unsigned nthreads) REC_ReadConfigInteger(net.recv_bufsize, "proxy.config.net.sock_recv_buffer_size_in"); REC_ReadConfigInteger(net.send_bufsize, "proxy.config.net.sock_send_buffer_size_in"); REC_ReadConfigInteger(net.sockopt_flags, "proxy.config.net.sock_option_flag_in"); + REC_ReadConfigInteger(net.defer_accept, "proxy.config.net.defer_accept"); +#ifdef TCP_INIT_CWND + REC_ReadConfigInteger(net.init_cwnd, "proxy.config.http.server_tcp_init_cwnd"); +#endif #ifdef TCP_FASTOPEN REC_ReadConfigInteger(net.tfo_queue_length, "proxy.config.net.sock_option_tfo_queue_size_in"); @@ -158,8 +162,10 @@ make_net_accept_options(const HttpProxyPort *port, unsigned nthreads) if (port) { net.f_inbound_transparent = port->m_inbound_transparent_p; + net.f_mptcp = port->m_mptcp; net.ip_family = port->m_family; net.local_port = port->m_port; + net.f_proxy_protocol = port->m_proxy_protocol; if (port->m_inbound_ip.isValid()) { net.local_ip = port->m_inbound_ip; @@ -169,7 +175,6 @@ make_net_accept_options(const HttpProxyPort *port, unsigned nthreads) net.local_ip = HttpConfig::m_master.inbound_ip4; } } - return net; } @@ -209,6 +214,7 @@ MakeHttpProxyAcceptor(HttpProxyAcceptor &acceptor, HttpProxyPort &port, unsigned ProtocolProbeSessionAccept *probe = new ProtocolProbeSessionAccept(); HttpSessionAccept *http = nullptr; // don't allocate this unless it will be used. probe->proxyPort = &port; + probe->proxy_protocol_ipmap = &HttpConfig::m_master.config_proxy_protocol_ipmap; if (port.m_session_protocol_preference.intersects(HTTP_PROTOCOL_SET)) { http = new HttpSessionAccept(accept_opt); @@ -367,22 +373,6 @@ start_HttpProxyServer() } } -void -start_HttpProxyServerBackDoor(int port, int accept_threads) -{ - NetProcessor::AcceptOptions opt; - HttpSessionAccept::Options ha_opt; - - opt.local_port = port; - opt.accept_threads = accept_threads; - opt.localhost_only = true; - ha_opt.backdoor = true; - opt.backdoor = true; - - // The backdoor only binds the loopback interface - netProcessor.main_accept(new HttpSessionAccept(ha_opt), NO_FD, opt); -} - void stop_HttpProxyServer() { diff --git a/proxy/http/HttpProxyServerMain.h b/proxy/http/HttpProxyServerMain.h index 33f19ec73ee..96c461abecc 100644 --- a/proxy/http/HttpProxyServerMain.h +++ b/proxy/http/HttpProxyServerMain.h @@ -47,8 +47,6 @@ void start_HttpProxyServer(); void stop_HttpProxyServer(); -void start_HttpProxyServerBackDoor(int port, int accept_threads = 0); - NetProcessor::AcceptOptions make_net_accept_options(const HttpProxyPort *port, unsigned nthreads); extern std::mutex proxyServerMutex; diff --git a/proxy/http/HttpSM.cc b/proxy/http/HttpSM.cc index 58cf9d5a50f..05630849e54 100644 --- a/proxy/http/HttpSM.cc +++ b/proxy/http/HttpSM.cc @@ -49,6 +49,7 @@ #include #include #include +#include #define DEFAULT_RESPONSE_BUFFER_SIZE_INDEX 6 // 8K #define DEFAULT_REQUEST_BUFFER_SIZE_INDEX 6 // 8K @@ -70,7 +71,6 @@ #define USE_NEW_EMPTY_MIOBUFFER extern int cache_config_read_while_writer; -extern TunnelHashMap TunnelMap; // stores the name of the servers to tunnel to // We have a debugging list that can use to find stuck // state machines @@ -341,7 +341,6 @@ HttpSM::init() t_state.force_dns = (ip_rule_in_CacheControlTable() || t_state.parent_params->parent_table->ipMatch || !(t_state.txn_conf->doc_in_cache_skip_dns) || !(t_state.txn_conf->cache_http)); - http_parser.m_allow_non_http = t_state.http_config_param->parser_allow_non_http; http_parser_init(&http_parser); SET_HANDLER(&HttpSM::main_handler); @@ -461,14 +460,20 @@ HttpSM::attach_client_session(ProxyClientTransaction *client_vc, IOBufferReader _client_transaction_id = ua_txn->get_transaction_id(); { auto p = ua_txn->get_parent(); + if (p) { _client_connection_id = p->connection_id(); } } - // Collect log & stats information - client_tcp_reused = !(ua_txn->is_first_transaction()); + // Collect log & stats information. We've already verified that the netvc is !nullptr above, + // and netvc == ua_txn->get_netvc(). SSLNetVConnection *ssl_vc = dynamic_cast(netvc); + + is_internal = netvc->get_is_internal_request(); + mptcp_state = netvc->get_mptcp_state(); + client_tcp_reused = !(ua_txn->is_first_transaction()); + if (ssl_vc != nullptr) { client_connection_is_ssl = true; client_ssl_reused = ssl_vc->getSSLSessionCacheHit(); @@ -487,7 +492,6 @@ HttpSM::attach_client_session(ProxyClientTransaction *client_vc, IOBufferReader ink_release_assert(ua_txn->get_half_close_flag() == false); mutex = client_vc->mutex; - HTTP_INCREMENT_DYN_STAT(http_current_client_transactions_stat); if (ua_txn->debug()) { debug_on = true; } @@ -504,7 +508,6 @@ HttpSM::attach_client_session(ProxyClientTransaction *client_vc, IOBufferReader ats_ip_copy(&t_state.client_info.dst_addr, netvc->get_local_addr()); t_state.client_info.dst_addr.port() = netvc->get_local_port(); t_state.client_info.is_transparent = netvc->get_is_transparent(); - t_state.backdoor_request = !client_vc->hooks_enabled(); t_state.client_info.port_attribute = static_cast(netvc->attributes); // Record api hook set state @@ -518,7 +521,7 @@ HttpSM::attach_client_session(ProxyClientTransaction *client_vc, IOBufferReader http_parser_init(&http_parser); // Prepare raw reader which will live until we are sure this is HTTP indeed - if (is_transparent_passthrough_allowed()) { + if (is_transparent_passthrough_allowed() || (ssl_vc && ssl_vc->decrypt_tunnel())) { ua_raw_buffer_reader = buffer_reader->clone(); } @@ -556,7 +559,7 @@ HttpSM::setup_client_read_request_header() ua_entry->read_vio = ua_txn->do_io_read(this, INT64_MAX, ua_buffer_reader->mbuf); // The header may already be in the buffer if this // a request from a keep-alive connection - dispatchEvent(VC_EVENT_READ_READY, ua_entry->read_vio); + handleEvent(VC_EVENT_READ_READY, ua_entry->read_vio); } void @@ -565,7 +568,7 @@ HttpSM::setup_blind_tunnel_port() NetVConnection *netvc = ua_txn->get_netvc(); SSLNetVConnection *ssl_vc = dynamic_cast(netvc); int host_len; - if (ssl_vc && ssl_vc->GetSNIMapping()) { + if (ssl_vc) { if (!t_state.hdr_info.client_request.url_get()->host_get(&host_len)) { // the URL object has not been created in the start of the transaction. Hence, we need to create the URL here URL u; @@ -575,10 +578,11 @@ HttpSM::setup_blind_tunnel_port() t_state.hdr_info.client_request.url_create(&u); u.scheme_set(URL_SCHEME_TUNNEL, URL_LEN_TUNNEL); t_state.hdr_info.client_request.url_set(&u); - if (auto it = TunnelMap.find(ssl_vc->serverName); it != TunnelMap.end()) { - t_state.hdr_info.client_request.url_get()->host_set(it->second.hostname.c_str(), it->second.hostname.size()); - if (it->second.port > 0) { - t_state.hdr_info.client_request.url_get()->port_set(it->second.port); + if (ssl_vc->has_tunnel_destination()) { + const char *tunnel_host = ssl_vc->get_tunnel_host(); + t_state.hdr_info.client_request.url_get()->host_set(tunnel_host, strlen(tunnel_host)); + if (ssl_vc->get_tunnel_port() > 0) { + t_state.hdr_info.client_request.url_get()->port_set(ssl_vc->get_tunnel_port()); } else { t_state.hdr_info.client_request.url_get()->port_set(t_state.state_machine->ua_txn->get_netvc()->get_local_port()); } @@ -664,7 +668,7 @@ HttpSM::state_read_client_request_header(int event, void *data) // We need to handle EOS as well as READ_READY because the client // may have sent all of the data already followed by a fIN and that // should be OK. - if (is_transparent_passthrough_allowed() && ua_raw_buffer_reader != nullptr) { + if (ua_raw_buffer_reader != nullptr) { bool do_blind_tunnel = false; // If we had a parse error and we're done reading data // blind tunnel @@ -686,7 +690,7 @@ HttpSM::state_read_client_request_header(int event, void *data) // Turn off read eventing until we get the // blind tunnel infrastructure set up if (netvc) { - netvc->do_io_read(this, 0, nullptr); + netvc->do_io_read(nullptr, 0, nullptr); } /* establish blind tunnel */ @@ -888,7 +892,7 @@ HttpSM::state_watch_for_client_abort(int event, void *data) "[%" PRId64 "] [watch_for_client_abort] " "forwarding event %s to tunnel", sm_id, HttpDebugNames::get_event_name(event)); - tunnel.dispatchEvent(event, c->write_vio); + tunnel.handleEvent(event, c->write_vio); return 0; } else { tunnel.kill_tunnel(); @@ -988,9 +992,6 @@ HttpSM::state_read_push_response_header(int event, void *data) ink_assert(ua_entry->read_vio == (VIO *)data); ink_assert(t_state.current.server == nullptr); - int64_t data_size = 0; - int64_t bytes_used = 0; - switch (event) { case VC_EVENT_EOS: ua_entry->eos = true; @@ -1014,17 +1015,16 @@ HttpSM::state_read_push_response_header(int event, void *data) while (ua_buffer_reader->read_avail() && state == PARSE_RESULT_CONT) { const char *start = ua_buffer_reader->start(); const char *tmp = start; - data_size = ua_buffer_reader->block_read_avail(); + int64_t data_size = ua_buffer_reader->block_read_avail(); ink_assert(data_size >= 0); ///////////////////// // tokenize header // ///////////////////// - state = - t_state.hdr_info.server_response.parse_resp(&http_parser, &tmp, tmp + data_size, false // Only call w/ eof when data exhausted - ); + state = t_state.hdr_info.server_response.parse_resp(&http_parser, &tmp, tmp + data_size, + false); // Only call w/ eof when data exhausted - bytes_used = tmp - start; + int64_t bytes_used = tmp - start; ink_release_assert(bytes_used <= data_size); ua_buffer_reader->consume(bytes_used); @@ -1384,18 +1384,18 @@ plugins required to work with sni_routing. NetVConnection *netvc = ua_txn->get_netvc(); SSLNetVConnection *ssl_vc = dynamic_cast(netvc); - if (ssl_vc && ssl_vc->GetSNIMapping()) { - if (auto it = TunnelMap.find(ssl_vc->serverName); it != TunnelMap.end()) { - t_state.hdr_info.client_request.url_get()->host_set(it->second.hostname.c_str(), it->second.hostname.size()); - if (it->second.port > 0) { - t_state.hdr_info.client_request.url_get()->port_set(it->second.port); - } else { - t_state.hdr_info.client_request.url_get()->port_set(t_state.state_machine->ua_txn->get_netvc()->get_local_port()); - } + if (ssl_vc && ssl_vc->has_tunnel_destination()) { + const char *tunnel_host = ssl_vc->get_tunnel_host(); + t_state.hdr_info.client_request.url_get()->host_set(tunnel_host, strlen(tunnel_host)); + ushort tunnel_port = ssl_vc->get_tunnel_port(); + if (tunnel_port > 0) { + t_state.hdr_info.client_request.url_get()->port_set(tunnel_port); } else { - t_state.hdr_info.client_request.url_get()->host_set(ssl_vc->serverName, strlen(ssl_vc->serverName)); t_state.hdr_info.client_request.url_get()->port_set(t_state.state_machine->ua_txn->get_netvc()->get_local_port()); } + } else if (ssl_vc) { + t_state.hdr_info.client_request.url_get()->host_set(ssl_vc->serverName, strlen(ssl_vc->serverName)); + t_state.hdr_info.client_request.url_get()->port_set(t_state.state_machine->ua_txn->get_netvc()->get_local_port()); } } // FALLTHROUGH @@ -1429,34 +1429,19 @@ plugins required to work with sni_routing. callout_state = HTTP_API_IN_CALLOUT; } - /* The MUTEX_TRY_LOCK macro was changed so - that it can't handle NULL mutex'es. The plugins - can use null mutexes so we have to do this manually. - We need to take a smart pointer to the mutex since - the plugin could release it's mutex while we're on - the callout - */ - bool plugin_lock; - Ptr plugin_mutex; - if (cur_hook->m_cont->mutex) { - plugin_mutex = cur_hook->m_cont->mutex; - plugin_lock = MUTEX_TAKE_TRY_LOCK(cur_hook->m_cont->mutex, mutex->thread_holding); - - if (!plugin_lock) { - api_timer = -Thread::get_hrtime_updated(); - HTTP_SM_SET_DEFAULT_HANDLER(&HttpSM::state_api_callout); - ink_assert(pending_action == nullptr); - pending_action = mutex->thread_holding->schedule_in(this, HRTIME_MSECONDS(10)); - // Should @a callout_state be reset back to HTTP_API_NO_CALLOUT here? Because the default - // handler has been changed the value isn't important to the rest of the state machine - // but not resetting means there is no way to reliably detect re-entrance to this state with an - // outstanding callout. - return 0; - } - } else { - plugin_lock = false; + MUTEX_TRY_LOCK(lock, cur_hook->m_cont->mutex, mutex->thread_holding); + // Have a mutex but didn't get the lock, reschedule + if (!lock.is_locked()) { + api_timer = -Thread::get_hrtime_updated(); + HTTP_SM_SET_DEFAULT_HANDLER(&HttpSM::state_api_callout); + ink_assert(pending_action == nullptr); + pending_action = mutex->thread_holding->schedule_in(this, HRTIME_MSECONDS(10)); + // Should @a callout_state be reset back to HTTP_API_NO_CALLOUT here? Because the default + // handler has been changed the value isn't important to the rest of the state machine + // but not resetting means there is no way to reliably detect re-entrance to this state with an + // outstanding callout. + return 0; } - SMDebug("http", "[%" PRId64 "] calling plugin on hook %s at hook %p", sm_id, HttpDebugNames::get_api_hook_name(cur_hook_id), cur_hook); @@ -1474,11 +1459,6 @@ plugins required to work with sni_routing. // tracking a non-complete callout from a chain so just let it ride. It will get cleaned // up in state_api_callback when the plugin re-enables this transaction. } - - if (plugin_lock) { - Mutex_unlock(plugin_mutex, mutex->thread_holding); - } - return 0; } } @@ -1578,14 +1558,17 @@ void HttpSM::handle_api_return() { switch (t_state.api_next_action) { - case HttpTransact::SM_ACTION_API_SM_START: - if (t_state.client_info.port_attribute == HttpProxyPort::TRANSPORT_BLIND_TUNNEL) { + case HttpTransact::SM_ACTION_API_SM_START: { + NetVConnection *netvc = ua_txn->get_netvc(); + SSLNetVConnection *ssl_vc = dynamic_cast(netvc); + bool forward_dest = ssl_vc != nullptr && ssl_vc->decrypt_tunnel(); + if (t_state.client_info.port_attribute == HttpProxyPort::TRANSPORT_BLIND_TUNNEL || forward_dest) { setup_blind_tunnel_port(); } else { setup_client_read_request_header(); } return; - + } case HttpTransact::SM_ACTION_API_CACHE_LOOKUP_COMPLETE: case HttpTransact::SM_ACTION_API_READ_CACHE_HDR: if (t_state.api_cleanup_cache_read && t_state.api_update_cached_object != HttpTransact::UPDATE_CACHED_OBJECT_PREPARE) { @@ -1629,7 +1612,7 @@ HttpSM::handle_api_return() state_remove_from_list(EVENT_NONE, nullptr); return; default: - ink_release_assert("! Not reached"); + ink_release_assert(!"Not reached"); break; } @@ -1732,21 +1715,23 @@ HttpSM::state_http_server_open(int event, void *data) pending_action = nullptr; } milestones[TS_MILESTONE_SERVER_CONNECT_END] = Thread::get_hrtime(); - HttpServerSession *session; - NetVConnection *netvc = nullptr; + NetVConnection *netvc = nullptr; switch (event) { case NET_EVENT_OPEN: { - session = (TS_SERVER_SESSION_SHARING_POOL_THREAD == t_state.http_config_param->server_session_sharing_pool) ? - THREAD_ALLOC_INIT(httpServerSessionAllocator, mutex->thread_holding) : - httpServerSessionAllocator.alloc(); + HttpServerSession *session = (TS_SERVER_SESSION_SHARING_POOL_THREAD == t_state.http_config_param->server_session_sharing_pool) ? + THREAD_ALLOC_INIT(httpServerSessionAllocator, mutex->thread_holding) : + httpServerSessionAllocator.alloc(); session->sharing_pool = static_cast(t_state.http_config_param->server_session_sharing_pool); session->sharing_match = static_cast(t_state.txn_conf->server_session_sharing_match); netvc = static_cast(data); session->attach_hostname(t_state.current.server->name); UnixNetVConnection *vc = static_cast(data); - ink_release_assert(pending_action == nullptr || pending_action == vc->get_action()); + // Since the UnixNetVConnection::action_ or SocksEntry::action_ may be returned from netProcessor.connect_re, and the + // SocksEntry::action_ will be copied into UnixNetVConnection::action_ before call back NET_EVENT_OPEN from SocksEntry::free(), + // so we just compare the Continuation between pending_action and VC's action_. + ink_release_assert(pending_action == nullptr || pending_action->continuation == vc->get_action()->continuation); pending_action = nullptr; session->new_connection(vc); @@ -1782,6 +1767,7 @@ HttpSM::state_http_server_open(int event, void *data) } return 0; } + case VC_EVENT_READ_COMPLETE: case VC_EVENT_WRITE_READY: case VC_EVENT_WRITE_COMPLETE: // Update the time out to the regular connection timeout. @@ -2266,7 +2252,7 @@ HttpSM::state_hostdb_lookup(int event, void *data) opt.host_res_style = ua_txn->get_host_res_style(); Action *dns_lookup_action_handle = - hostDBProcessor.getbyname_imm(this, (process_hostdb_info_pfn)&HttpSM::process_hostdb_info, host_name, 0, opt); + hostDBProcessor.getbyname_imm(this, (cb_process_result_pfn)&HttpSM::process_hostdb_info, host_name, 0, opt); if (dns_lookup_action_handle != ACTION_RESULT_DONE) { ink_assert(!pending_action); pending_action = dns_lookup_action_handle; @@ -2560,7 +2546,7 @@ HttpSM::state_cache_open_read(int event, void *data) break; default: - ink_release_assert("!Unknown event"); + ink_release_assert(!"Unknown event"); break; } @@ -3238,8 +3224,7 @@ HttpSM::tunnel_handler_ua(int event, HttpTunnelConsumer *c) c->write_success = true; t_state.client_info.abort = HttpTransact::DIDNOT_ABORT; if (t_state.client_info.keep_alive == HTTP_KEEPALIVE) { - if (ua_txn->allow_half_open() && - (t_state.www_auth_content != HttpTransact::CACHE_AUTH_SERVE || ua_txn->get_server_session())) { + if (t_state.www_auth_content != HttpTransact::CACHE_AUTH_SERVE || ua_txn->get_server_session()) { // successful keep-alive close_connection = false; } @@ -3293,18 +3278,14 @@ HttpSM::tunnel_handler_ua(int event, HttpTunnelConsumer *c) // set the ua_txn into half close mode // only external POSTs should be subject to this logic; ruling out internal POSTs here - bool is_eligible_post_request = (t_state.method == HTTP_WKSIDX_POST); - if (is_eligible_post_request) { - NetVConnection *vc = ua_txn->get_netvc(); - if (vc) { - is_eligible_post_request &= !vc->get_is_internal_request(); - } - } - if ((is_eligible_post_request || t_state.client_info.pipeline_possible == true) && ua_txn->allow_half_open() && - c->producer->vc_type != HT_STATIC && event == VC_EVENT_WRITE_COMPLETE) { + bool is_eligible_post_request = ((t_state.method == HTTP_WKSIDX_POST) && !is_internal); + + if ((is_eligible_post_request || t_state.client_info.pipeline_possible == true) && c->producer->vc_type != HT_STATIC && + event == VC_EVENT_WRITE_COMPLETE) { ua_txn->set_half_close_flag(true); } + vc_table.remove_entry(this->ua_entry); ua_txn->do_io_close(); } else { ink_assert(ua_buffer_reader != nullptr); @@ -3977,7 +3958,7 @@ HttpSM::state_remap_request(int event, void * /* data ATS_UNUSED */) } default: - ink_assert("Unexpected event inside state_remap_request"); + ink_assert(!"Unexpected event inside state_remap_request"); break; } @@ -4012,7 +3993,7 @@ HttpSM::do_remap_request(bool run_inline) if (!ret) { SMDebug("url_rewrite", "Could not find a valid remapping entry for this request [%" PRId64 "]", sm_id); if (!run_inline) { - dispatchEvent(EVENT_REMAP_COMPLETE, nullptr); + handleEvent(EVENT_REMAP_COMPLETE, nullptr); } return; } @@ -4063,7 +4044,7 @@ HttpSM::do_hostdb_lookup() opt.timeout = t_state.api_txn_dns_timeout_value; } Action *srv_lookup_action_handle = - hostDBProcessor.getSRVbyname_imm(this, (process_srv_info_pfn)&HttpSM::process_srv_info, d, 0, opt); + hostDBProcessor.getSRVbyname_imm(this, (cb_process_result_pfn)&HttpSM::process_srv_info, d, 0, opt); if (srv_lookup_action_handle != ACTION_RESULT_DONE) { ink_assert(!pending_action); @@ -4080,7 +4061,7 @@ HttpSM::do_hostdb_lookup() opt.host_res_style = ua_txn->get_host_res_style(); Action *dns_lookup_action_handle = - hostDBProcessor.getbyname_imm(this, (process_hostdb_info_pfn)&HttpSM::process_hostdb_info, host_name, 0, opt); + hostDBProcessor.getbyname_imm(this, (cb_process_result_pfn)&HttpSM::process_hostdb_info, host_name, 0, opt); if (dns_lookup_action_handle != ACTION_RESULT_DONE) { ink_assert(!pending_action); pending_action = dns_lookup_action_handle; @@ -4094,10 +4075,14 @@ HttpSM::do_hostdb_lookup() // If there is not a current server, we must be looking up the origin // server at the beginning of the transaction - int server_port = t_state.current.server ? - t_state.current.server->dst_addr.host_order_port() : - t_state.server_info.dst_addr.isValid() ? t_state.server_info.dst_addr.host_order_port() : - t_state.hdr_info.client_request.port_get(); + int server_port = 0; + if (t_state.current.server && t_state.current.server->dst_addr.isValid()) { + server_port = t_state.current.server->dst_addr.host_order_port(); + } else if (t_state.server_info.dst_addr.isValid()) { + server_port = t_state.server_info.dst_addr.host_order_port(); + } else { + server_port = t_state.hdr_info.client_request.port_get(); + } if (t_state.api_txn_dns_timeout_value != -1) { SMDebug("http_timeout", "beginning DNS lookup. allowing %d mseconds for DNS lookup", t_state.api_txn_dns_timeout_value); @@ -4110,7 +4095,7 @@ HttpSM::do_hostdb_lookup() opt.timeout = (t_state.api_txn_dns_timeout_value != -1) ? t_state.api_txn_dns_timeout_value : 0; opt.host_res_style = ua_txn->get_host_res_style(); - Action *dns_lookup_action_handle = hostDBProcessor.getbyname_imm(this, (process_hostdb_info_pfn)&HttpSM::process_hostdb_info, + Action *dns_lookup_action_handle = hostDBProcessor.getbyname_imm(this, (cb_process_result_pfn)&HttpSM::process_hostdb_info, t_state.dns_info.lookup_name, 0, opt); if (dns_lookup_action_handle != ACTION_RESULT_DONE) { @@ -4285,7 +4270,15 @@ HttpSM::parse_range_and_compare(MIMEField *field, int64_t content_length) start = -1; } else { for (start = 0; s < e && *s >= '0' && *s <= '9'; ++s) { - start = start * 10 + (*s - '0'); + // check the int64 overflow in case of high gcc with O3 option + // thinking the start is always positive + int64_t new_start = start * 10 + (*s - '0'); + + if (new_start < start) { // Overflow + t_state.range_setup = HttpTransact::RANGE_NONE; + goto Lfaild; + } + start = new_start; } // skip last white spaces for (; s < e && ParseRules::is_ws(*s); ++s) { @@ -4317,7 +4310,15 @@ HttpSM::parse_range_and_compare(MIMEField *field, int64_t content_length) end = content_length - 1; } else { for (end = 0; s < e && *s >= '0' && *s <= '9'; ++s) { - end = end * 10 + (*s - '0'); + // check the int64 overflow in case of high gcc with O3 option + // thinking the start is always positive + int64_t new_end = end * 10 + (*s - '0'); + + if (new_end < end) { // Overflow + t_state.range_setup = HttpTransact::RANGE_NONE; + goto Lfaild; + } + end = new_end; } // skip last white spaces for (; s < e && ParseRules::is_ws(*s); ++s) { @@ -4395,8 +4396,6 @@ HttpSM::parse_range_and_compare(MIMEField *field, int64_t content_length) void HttpSM::calculate_output_cl(int64_t num_chars_for_ct, int64_t num_chars_for_cl) { - int i; - if (t_state.range_setup != HttpTransact::RANGE_REQUESTED && t_state.range_setup != HttpTransact::RANGE_NOT_TRANSFORM_REQUESTED) { return; } @@ -4406,7 +4405,7 @@ HttpSM::calculate_output_cl(int64_t num_chars_for_ct, int64_t num_chars_for_cl) if (t_state.num_range_fields == 1) { t_state.range_output_cl = t_state.ranges[0]._end - t_state.ranges[0]._start + 1; } else { - for (i = 0; i < t_state.num_range_fields; i++) { + for (int i = 0; i < t_state.num_range_fields; i++) { if (t_state.ranges[i]._start >= 0) { t_state.range_output_cl += boundary_size; t_state.range_output_cl += sub_header_size + num_chars_for_ct; @@ -4443,9 +4442,6 @@ void HttpSM::do_range_setup_if_necessary() { MIMEField *field; - INKVConnInternal *range_trans; - int field_content_type_len = -1; - const char *content_type; ink_assert(t_state.cache_info.object_read != nullptr); @@ -4489,11 +4485,14 @@ HttpSM::do_range_setup_if_necessary() // We have to do the transform on (allowed) multi-range request, *or* if the VC is not pread capable if (do_transform) { if (api_hooks.get(TS_HTTP_RESPONSE_TRANSFORM_HOOK) == nullptr) { + int field_content_type_len = -1; + const char *content_type = t_state.cache_info.object_read->response_get()->value_get( + MIME_FIELD_CONTENT_TYPE, MIME_LEN_CONTENT_TYPE, &field_content_type_len); + Debug("http_trans", "Unable to accelerate range request, fallback to transform"); - content_type = t_state.cache_info.object_read->response_get()->value_get(MIME_FIELD_CONTENT_TYPE, MIME_LEN_CONTENT_TYPE, - &field_content_type_len); + // create a Range: transform processor for requests of type Range: bytes=1-2,4-5,10-100 (eg. multiple ranges) - range_trans = transformProcessor.range_transform( + INKVConnInternal *range_trans = transformProcessor.range_transform( mutex.get(), t_state.ranges, t_state.num_range_fields, &t_state.hdr_info.transform_response, content_type, field_content_type_len, t_state.cache_info.object_read->object_size_get()); api_hooks.append(TS_HTTP_RESPONSE_TRANSFORM_HOOK, range_trans); @@ -4618,7 +4617,7 @@ HttpSM::do_cache_prepare_update() void HttpSM::do_cache_prepare_action(HttpCacheSM *c_sm, CacheHTTPInfo *object_read_info, bool retry, bool allow_multiple) { - URL *o_url, *c_url, *s_url; + URL *o_url, *s_url; bool restore_client_request = false; ink_assert(!pending_action); @@ -4640,7 +4639,7 @@ HttpSM::do_cache_prepare_action(HttpCacheSM *c_sm, CacheHTTPInfo *object_read_in // modify client request to make it have the url we are going to // store into the cache if (restore_client_request) { - c_url = t_state.hdr_info.client_request.url_get(); + URL *c_url = t_state.hdr_info.client_request.url_get(); s_url->copy(c_url); } @@ -4668,6 +4667,46 @@ HttpSM::send_origin_throttled_response() call_transact_and_set_next_state(HttpTransact::HandleResponse); } +static void +set_tls_options(NetVCOptions &opt, OverridableHttpConfigParams *txn_conf) +{ + char *verify_server = nullptr; + if (txn_conf->ssl_client_verify_server_policy == nullptr) { + opt.verifyServerPolicy = YamlSNIConfig::Policy::UNSET; + } else { + verify_server = txn_conf->ssl_client_verify_server_policy; + if (strcmp(verify_server, "DISABLED") == 0) { + opt.verifyServerPolicy = YamlSNIConfig::Policy::DISABLED; + } else if (strcmp(verify_server, "PERMISSIVE") == 0) { + opt.verifyServerPolicy = YamlSNIConfig::Policy::PERMISSIVE; + } else if (strcmp(verify_server, "ENFORCED") == 0) { + opt.verifyServerPolicy = YamlSNIConfig::Policy::ENFORCED; + } else { + Warning("%s is invalid for proxy.config.ssl.client.verify.server.policy. Should be one of DISABLED, PERMISSIVE, or ENFORCED", + verify_server); + opt.verifyServerPolicy = YamlSNIConfig::Policy::UNSET; + } + } + if (txn_conf->ssl_client_verify_server_properties == nullptr) { + opt.verifyServerProperties = YamlSNIConfig::Property::UNSET; + } else { + verify_server = txn_conf->ssl_client_verify_server_properties; + if (strcmp(verify_server, "SIGNATURE") == 0) { + opt.verifyServerProperties = YamlSNIConfig::Property::SIGNATURE_MASK; + } else if (strcmp(verify_server, "NAME") == 0) { + opt.verifyServerProperties = YamlSNIConfig::Property::NAME_MASK; + } else if (strcmp(verify_server, "ALL") == 0) { + opt.verifyServerProperties = YamlSNIConfig::Property::ALL_MASK; + } else if (strcmp(verify_server, "NONE") == 0) { + opt.verifyServerProperties = YamlSNIConfig::Property::NONE; + } else { + Warning("%s is invalid for proxy.config.ssl.client.verify.server.properties. Should be one of SIGNATURE, NAME, or ALL", + verify_server); + opt.verifyServerProperties = YamlSNIConfig::Property::NONE; + } + } +} + ////////////////////////////////////////////////////////////////////////// // // HttpSM::do_http_server_open() @@ -4956,6 +4995,8 @@ HttpSM::do_http_server_open(bool raw) t_state.txn_conf->sock_option_flag_out, t_state.txn_conf->sock_packet_mark_out, t_state.txn_conf->sock_packet_tos_out); + set_tls_options(opt, t_state.txn_conf); + opt.ip_family = ip_family; if (ua_txn) { @@ -5000,14 +5041,22 @@ HttpSM::do_http_server_open(bool raw) if (scheme_to_use == URL_WKSIDX_HTTPS || HttpTransactHeaders::is_method_idempotent(t_state.method)) { opt.f_tcp_fastopen = (t_state.txn_conf->sock_option_flag_out & NetVCOptions::SOCK_OPT_TCP_FAST_OPEN); } + opt.ssl_client_cert_name = t_state.txn_conf->ssl_client_cert_filename; + opt.ssl_client_private_key_name = t_state.txn_conf->ssl_client_private_key_filename; + opt.ssl_client_ca_cert_name = t_state.txn_conf->ssl_client_ca_cert_filename; if (scheme_to_use == URL_WKSIDX_HTTPS) { SMDebug("http", "calling sslNetProcessor.connect_re"); - int len = 0; - const char *host = t_state.hdr_info.server_request.host_get(&len); - if (host && len > 0) { - opt.set_sni_servername(host, len); + int len = 0; + if (t_state.txn_conf->ssl_client_sni_policy != nullptr && !strcmp(t_state.txn_conf->ssl_client_sni_policy, "remap")) { + len = strlen(t_state.server_info.name); + opt.set_sni_servername(t_state.server_info.name, len); + } else { // Do the default of host header for SNI + const char *host = t_state.hdr_info.server_request.host_get(&len); + if (host && len > 0) { + opt.set_sni_servername(host, len); + } } if (t_state.server_info.name) { opt.set_ssl_servername(t_state.server_info.name); @@ -5034,11 +5083,6 @@ HttpSM::do_http_server_open(bool raw) void HttpSM::do_api_callout_internal() { - if (t_state.backdoor_request) { - handle_api_return(); - return; - } - switch (t_state.api_next_action) { case HttpTransact::SM_ACTION_API_SM_START: cur_hook_id = TS_HTTP_TXN_START_HOOK; @@ -5145,9 +5189,12 @@ HttpSM::mark_host_failure(HostDBInfo *info, time_t time_down) if (info->app.http_data.last_failure == 0) { char *url_str = t_state.hdr_info.client_request.url_string_get(&t_state.arena, nullptr); Log::error("%s", lbw() - .print("CONNECT: could not connect to {} for '{}' (setting last failure time) connect_result={}\0", - t_state.current.server->dst_addr, url_str ? url_str : "", - ts::bwf::Errno(t_state.current.server->connect_result)) + .clip(1) + .print("CONNECT Error: {} connecting to {} for '{}' (setting last failure time)", + ts::bwf::Errno(t_state.current.server->connect_result), t_state.current.server->dst_addr, + ts::bwf::FirstOf(url_str, "")) + .extend(1) + .write('\0') .data()); if (url_str) { @@ -5422,13 +5469,13 @@ HttpSM::handle_server_setup_error(int event, void *data) ua_producer->alive = false; ua_producer->handler_state = HTTP_SM_POST_SERVER_FAIL; - tunnel.dispatchEvent(VC_EVENT_ERROR, c->write_vio); + tunnel.handleEvent(VC_EVENT_ERROR, c->write_vio); return; } } else { // c could be null here as well if (c != nullptr) { - tunnel.dispatchEvent(event, c->write_vio); + tunnel.handleEvent(event, c->write_vio); return; } } @@ -5542,7 +5589,7 @@ HttpSM::setup_transform_to_server_transfer() } void -HttpSM::do_drain_request_body() +HttpSM::do_drain_request_body(HTTPHdr &response) { int64_t content_length = t_state.hdr_info.client_request.get_content_length(); int64_t avail = ua_buffer_reader->read_avail(); @@ -5567,7 +5614,7 @@ HttpSM::do_drain_request_body() close_connection: t_state.client_info.keep_alive = HTTP_NO_KEEPALIVE; - t_state.hdr_info.client_response.value_set(MIME_FIELD_CONNECTION, MIME_LEN_CONNECTION, "close", 5); + response.value_set(MIME_FIELD_CONNECTION, MIME_LEN_CONNECTION, "close", 5); } void @@ -5805,16 +5852,15 @@ HttpSM::issue_cache_update() int HttpSM::write_header_into_buffer(HTTPHdr *h, MIOBuffer *b) { - int bufindex; int dumpoffset; - int done, tmp; - IOBufferBlock *block; + int done; dumpoffset = 0; do { - bufindex = 0; - tmp = dumpoffset; - block = b->get_current_block(); + IOBufferBlock *block = b->get_current_block(); + int bufindex = 0; + int tmp = dumpoffset; + ink_assert(block->write_avail() > 0); done = h->print(block->start(), block->write_avail(), &bufindex, &tmp); dumpoffset += bufindex; @@ -5860,30 +5906,9 @@ HttpSM::attach_server_session(HttpServerSession *s) // Get server and client connections UnixNetVConnection *server_vc = dynamic_cast(server_session->get_netvc()); UnixNetVConnection *client_vc = (UnixNetVConnection *)(ua_txn->get_netvc()); - SSLNetVConnection *ssl_vc = dynamic_cast(client_vc); // Verifying that the user agent and server sessions/transactions are operating on the same thread. ink_release_assert(!server_vc || !client_vc || server_vc->thread == client_vc->thread); - bool associated_connection = false; - if (server_vc) { // if server_vc isn't a PluginVC - if (ssl_vc) { // if incoming connection is SSL - bool client_trace = ssl_vc->getSSLTrace(); - if (client_trace) { - // get remote address and port to mark corresponding traces - const sockaddr *remote_addr = ssl_vc->get_remote_addr(); - uint16_t remote_port = ssl_vc->get_remote_port(); - server_vc->setOriginTrace(true); - server_vc->setOriginTraceAddr(remote_addr); - server_vc->setOriginTracePort(remote_port); - associated_connection = true; - } - } - } - if (!associated_connection && server_vc) { - server_vc->setOriginTrace(false); - server_vc->setOriginTraceAddr(nullptr); - server_vc->setOriginTracePort(0); - } // set flag for server session is SSL SSLNetVConnection *server_ssl_vc = dynamic_cast(server_vc); @@ -6300,7 +6325,6 @@ HttpSM::setup_internal_transfer(HttpSMHandler handler_arg) int HttpSM::find_http_resp_buffer_size(int64_t content_length) { - int64_t buf_size; int64_t alloc_index; if (content_length == HTTP_UNDEFINED_CL) { @@ -6311,6 +6335,8 @@ HttpSM::find_http_resp_buffer_size(int64_t content_length) alloc_index = DEFAULT_RESPONSE_BUFFER_SIZE_INDEX; } } else { + int64_t buf_size; + #ifdef WRITE_AND_TRANSFER buf_size = HTTP_HEADER_BUFFER_SIZE + content_length - index_to_buffer_size(HTTP_SERVER_RESP_HDR_BUFFER_INDEX); #else @@ -6372,10 +6398,7 @@ HttpSM::server_transfer_init(MIOBuffer *buf, int hdr_size) if (server_response_pre_read_bytes == to_copy && server_buffer_reader->read_avail() > 0) { t_state.current.server->keep_alive = HTTP_NO_KEEPALIVE; } -#ifdef LAZY_BUF_ALLOC - // reset the server session buffer - server_session->reset_read_buffer(); -#endif + return nbytes; } @@ -6890,7 +6913,6 @@ HttpSM::kill_this() SMDebug("http", "[%" PRId64 "] deallocating sm", sm_id); // authAdapter.destroyState(); - HTTP_DECREMENT_DYN_STAT(http_current_client_transactions_stat); destroy(); } } @@ -7388,11 +7410,10 @@ HttpSM::set_next_state() release_server_session(true); t_state.source = HttpTransact::SOURCE_CACHE; - do_drain_request_body(); - if (transform_info.vc) { ink_assert(t_state.hdr_info.client_response.valid() == 0); ink_assert((t_state.hdr_info.transform_response.valid() ? true : false) == true); + do_drain_request_body(t_state.hdr_info.transform_response); t_state.hdr_info.cache_response.create(HTTP_TYPE_RESPONSE); t_state.hdr_info.cache_response.copy(&t_state.hdr_info.transform_response); @@ -7401,6 +7422,7 @@ HttpSM::set_next_state() tunnel.tunnel_run(p); } else { ink_assert((t_state.hdr_info.client_response.valid() ? true : false) == true); + do_drain_request_body(t_state.hdr_info.client_response); t_state.hdr_info.cache_response.create(HTTP_TYPE_RESPONSE); t_state.hdr_info.cache_response.copy(&t_state.hdr_info.client_response); @@ -7577,7 +7599,7 @@ HttpSM::set_next_state() } default: { - ink_release_assert("!Unknown next action"); + ink_release_assert(!"Unknown next action"); } } } @@ -7641,7 +7663,7 @@ HttpSM::do_redirect() HTTP_INCREMENT_DYN_STAT(http_total_x_redirect_stat); } else { // get the location header and setup the redirect - int redir_len; + int redir_len = 0; char *redir_url = (char *)t_state.hdr_info.client_response.value_get(MIME_FIELD_LOCATION, MIME_LEN_LOCATION, &redir_len); redirect_request(redir_url, redir_len); } @@ -7655,7 +7677,7 @@ HttpSM::do_redirect() } void -HttpSM::redirect_request(const char *redirect_url, const int redirect_len) +HttpSM::redirect_request(const char *arg_redirect_url, const int arg_redirect_len) { SMDebug("http_redirect", "[HttpSM::redirect_request]"); // get a reference to the client request header and client url and check to see if the url is valid @@ -7698,30 +7720,27 @@ HttpSM::redirect_request(const char *redirect_url, const int redirect_len) t_state.redirect_info.redirect_in_process = true; // set the passed in location url and parse it - URL &redirectUrl = t_state.redirect_info.redirect_url; - if (!redirectUrl.valid()) { - redirectUrl.create(nullptr); - } + URL redirectUrl; + redirectUrl.create(nullptr); - // reset the path from previous redirects (if any) - t_state.redirect_info.redirect_url.path_set(nullptr, 0); - - // redirectUrl.user_set(redirect_url, redirect_len); - redirectUrl.parse(redirect_url, redirect_len); + redirectUrl.parse(arg_redirect_url, arg_redirect_len); { int _scheme_len = -1; int _host_len = -1; - if (redirectUrl.scheme_get(&_scheme_len) == nullptr && redirectUrl.host_get(&_host_len) != nullptr && redirect_url[0] != '/') { + if (redirectUrl.scheme_get(&_scheme_len) == nullptr && redirectUrl.host_get(&_host_len) != nullptr && + arg_redirect_url[0] != '/') { // RFC7230 § 5.5 // The redirect URL lacked a scheme and so it is a relative URL. // The redirect URL did not begin with a slash, so we parsed some or all // of the the relative URI path as the host. // Prepend a slash and parse again. - char redirect_url_leading_slash[redirect_len + 1]; + char redirect_url_leading_slash[arg_redirect_len + 1]; redirect_url_leading_slash[0] = '/'; - memcpy(redirect_url_leading_slash + 1, redirect_url, redirect_len + 1); + if (arg_redirect_len > 0) { + memcpy(redirect_url_leading_slash + 1, arg_redirect_url, arg_redirect_len); + } url_nuke_proxy_stuff(redirectUrl.m_url_impl); - redirectUrl.parse(redirect_url_leading_slash, redirect_len + 1); + redirectUrl.parse(redirect_url_leading_slash, arg_redirect_len + 1); } } @@ -7734,6 +7753,8 @@ HttpSM::redirect_request(const char *redirect_url, const int redirect_len) // copy the redirect url to the client url clientUrl.copy(&redirectUrl); + redirectUrl.destroy(); + //(bug 2540703) Clear the previous response if we will attempt the redirect if (t_state.hdr_info.client_response.valid()) { // XXX - doing a destroy() for now, we can do a fileds_clear() if we have performance issue @@ -7773,9 +7794,9 @@ HttpSM::redirect_request(const char *redirect_url, const int redirect_len) bool noPortInHost = HttpConfig::m_master.redirection_host_no_port; - bool isRedirectUrlOriginForm = clientUrl.m_url_impl->m_len_scheme <= 0 && clientUrl.m_url_impl->m_len_user <= 0 && - clientUrl.m_url_impl->m_len_password <= 0 && clientUrl.m_url_impl->m_len_host <= 0 && - clientUrl.m_url_impl->m_len_port <= 0; + bool isRedirectUrlOriginForm = !clientUrl.m_url_impl->m_len_scheme && !clientUrl.m_url_impl->m_len_user && + !clientUrl.m_url_impl->m_len_password && !clientUrl.m_url_impl->m_len_host && + !clientUrl.m_url_impl->m_len_port; // check to see if the client request passed a host header, if so copy the host and port from the redirect url and // make a new host header diff --git a/proxy/http/HttpSM.h b/proxy/http/HttpSM.h index a26e2e17ac0..fe317290959 100644 --- a/proxy/http/HttpSM.h +++ b/proxy/http/HttpSM.h @@ -29,9 +29,11 @@ ****************************************************************************/ - #pragma once +#include +#include + #include "tscore/ink_platform.h" #include "P_EventSystem.h" #include "HttpCacheSM.h" @@ -41,22 +43,7 @@ #include "InkAPIInternal.h" #include "../ProxyClientTransaction.h" #include "HdrUtils.h" -#include #include "tscore/History.h" -//#include "AuthHttpAdapter.h" - -/* Enable LAZY_BUF_ALLOC to delay allocation of buffers until they - * are actually required. - * Enabling LAZY_BUF_ALLOC, stop Http code from allocation space - * for header buffer and tunnel buffer. The allocation is done by - * the net code in read_from_net when data is actually written into - * the buffer. By allocating memory only when it is required we can - * reduce the memory consumed by TS process. - * - * IMPORTANT NOTE: enable/disable LAZY_BUF_ALLOC in HttpServerSession.h - * as well. - */ -#define LAZY_BUF_ALLOC #define HTTP_API_CONTINUE (INK_API_EVENT_EVENTS_START + 0) #define HTTP_API_ERROR (INK_API_EVENT_EVENTS_START + 1) @@ -112,7 +99,7 @@ struct HttpVCTableEntry { struct HttpVCTable { static const int vc_table_max_entries = 4; - HttpVCTable(HttpSM *); + explicit HttpVCTable(HttpSM *); HttpVCTableEntry *new_entry(); HttpVCTableEntry *find_entry(VConnection *); @@ -139,10 +126,10 @@ HttpVCTable::is_table_clear() const } struct HttpTransformInfo { - HttpVCTableEntry *entry; - VConnection *vc; + HttpVCTableEntry *entry = nullptr; + VConnection *vc = nullptr; - HttpTransformInfo() : entry(nullptr), vc(nullptr) {} + HttpTransformInfo() {} }; enum { @@ -323,12 +310,12 @@ class HttpSM : public Continuation // YTS Team, yamsat Plugin bool enable_redirection = false; // To check if redirection is enabled + bool post_failed = false; // Added to identify post failure + bool debug_on = false; // Transaction specific debug flag char *redirect_url = nullptr; // url for force redirect (provide users a functionality to redirect to another url when needed) int redirect_url_len = 0; - int redirection_tries = 0; // To monitor number of redirections - int64_t transfered_bytes = 0; // Added to calculate POST data - bool post_failed = false; // Added to identify post failure - bool debug_on = false; // Transaction specific debug flag + int redirection_tries = 0; // To monitor number of redirections + int64_t transfered_bytes = 0; // Added to calculate POST data // Tunneling request to plugin HttpPluginTunnel_t plugin_tunnel_type = HTTP_NO_PLUGIN_TUNNEL; @@ -348,9 +335,9 @@ class HttpSM : public Continuation void postbuf_copy_partial_data(); void postbuf_init(IOBufferReader *ua_reader); void set_postbuf_done(bool done); + IOBufferReader *get_postbuf_clone_reader(); bool get_postbuf_done(); bool is_postbuf_valid(); - IOBufferReader *get_postbuf_clone_reader(); protected: int reentrancy_count = 0; @@ -479,7 +466,7 @@ class HttpSM : public Continuation void do_api_callout_internal(); void do_redirect(); void redirect_request(const char *redirect_url, const int redirect_len); - void do_drain_request_body(); + void do_drain_request_body(HTTPHdr &response); void wait_for_full_body(); @@ -549,16 +536,17 @@ class HttpSM : public Continuation int pushed_response_hdr_bytes = 0; int64_t pushed_response_body_bytes = 0; bool client_tcp_reused = false; - // Info about client's SSL connection. - bool client_ssl_reused = false; - bool client_connection_is_ssl = false; + bool client_ssl_reused = false; + bool client_connection_is_ssl = false; + bool is_internal = false; + bool server_connection_is_ssl = false; + bool is_waiting_for_full_body = false; + bool is_using_post_buffer = false; + std::optional mptcp_state; // Don't initialize, that marks it as "not defined". const char *client_protocol = "-"; const char *client_sec_protocol = "-"; const char *client_cipher_suite = "-"; int server_transact_count = 0; - bool server_connection_is_ssl = false; - bool is_waiting_for_full_body = false; - bool is_using_post_buffer = false; TransactionMilestones milestones; ink_hrtime api_timer = 0; diff --git a/proxy/http/HttpServerSession.cc b/proxy/http/HttpServerSession.cc index cadbcc0e085..9c77ac3eceb 100644 --- a/proxy/http/HttpServerSession.cc +++ b/proxy/http/HttpServerSession.cc @@ -74,11 +74,9 @@ HttpServerSession::new_connection(NetVConnection *new_vc) magic = HTTP_SS_MAGIC_ALIVE; HTTP_SUM_GLOBAL_DYN_STAT(http_current_server_connections_stat, 1); // Update the true global stat HTTP_INCREMENT_DYN_STAT(http_total_server_connections_stat); -#ifdef LAZY_BUF_ALLOC - read_buffer = new_empty_MIOBuffer(HTTP_SERVER_RESP_HDR_BUFFER_INDEX); -#else + read_buffer = new_MIOBuffer(HTTP_SERVER_RESP_HDR_BUFFER_INDEX); -#endif + buf_reader = read_buffer->alloc_reader(); Debug("http_ss", "[%" PRId64 "] session born, netvc %p", con_id, new_vc); state = HSS_INIT; diff --git a/proxy/http/HttpServerSession.h b/proxy/http/HttpServerSession.h index 5e4224af68d..03ff0150031 100644 --- a/proxy/http/HttpServerSession.h +++ b/proxy/http/HttpServerSession.h @@ -31,17 +31,6 @@ ****************************************************************************/ #pragma once -/* Enable LAZY_BUF_ALLOC to delay allocation of buffers until they - * are actually required. - * Enabling LAZY_BUF_ALLOC, stop Http code from allocation space - * for header buffer and tunnel buffer. The allocation is done by - * the net code in read_from_net when data is actually written into - * the buffer. By allocating memory only when it is required we can - * reduce the memory consumed by TS process. - * - * IMPORTANT NOTE: enable/disable LAZY_BUF_ALLOC in HttpSM.h as well. - */ -#define LAZY_BUF_ALLOC #include "P_Net.h" @@ -86,16 +75,6 @@ class HttpServerSession : public VConnection */ void enable_outbound_connection_tracking(OutboundConnTrack::Group *group); - void - reset_read_buffer(void) - { - ink_assert(read_buffer->_writer); - ink_assert(buf_reader != nullptr); - read_buffer->dealloc_all_readers(); - read_buffer->_writer = nullptr; - buf_reader = read_buffer->alloc_reader(); - } - IOBufferReader * get_reader() { diff --git a/proxy/http/HttpSessionAccept.cc b/proxy/http/HttpSessionAccept.cc index ac24ef86cb4..a13f8f30f1b 100644 --- a/proxy/http/HttpSessionAccept.cc +++ b/proxy/http/HttpSessionAccept.cc @@ -33,16 +33,10 @@ HttpSessionAccept::accept(NetVConnection *netvc, MIOBuffer *iobuf, IOBufferReade IpAllow::ACL acl; ip_port_text_buffer ipb; - // The backdoor port is now only bound to "localhost", so no - // reason to check for if it's incoming from "localhost" or not. - if (backdoor) { - acl = IpAllow::makeAllowAllACL(); - } else { - acl = IpAllow::match(client_ip, IpAllow::SRC_ADDR); - if (!acl.isValid()) { // if there's no ACL, it's a hard deny. - Warning("client '%s' prohibited by ip-allow policy", ats_ip_ntop(client_ip, ipb, sizeof(ipb))); - return false; - } + acl = IpAllow::match(client_ip, IpAllow::SRC_ADDR); + if (!acl.isValid()) { // if there's no ACL, it's a hard deny. + Warning("client '%s' prohibited by ip-allow policy", ats_ip_ntop(client_ip, ipb, sizeof(ipb))); + return false; } // Set the transport type if not already set @@ -66,7 +60,7 @@ HttpSessionAccept::accept(NetVConnection *netvc, MIOBuffer *iobuf, IOBufferReade new_session->host_res_style = ats_host_res_from(client_ip->sa_family, host_res_preference); new_session->acl = std::move(acl); - new_session->new_connection(netvc, iobuf, reader, backdoor); + new_session->new_connection(netvc, iobuf, reader); return true; } diff --git a/proxy/http/HttpSessionAccept.h b/proxy/http/HttpSessionAccept.h index 49d8350e849..362dd8bfb86 100644 --- a/proxy/http/HttpSessionAccept.h +++ b/proxy/http/HttpSessionAccept.h @@ -58,7 +58,7 @@ class HttpSessionAcceptOptions HttpSessionAcceptOptions(); // Connection type (HttpProxyPort::TransportType) - int transport_type; + int transport_type = 0; /// Set the transport type. self &setTransportType(int); /// Local address to bind for outbound connections. @@ -70,21 +70,17 @@ class HttpSessionAcceptOptions /// Set the outbound IP address to @a ip. self &setOutboundIp(IpEndpoint *ip); /// Local port for outbound connection. - uint16_t outbound_port; + uint16_t outbound_port = 0; /// Set outbound port. self &setOutboundPort(uint16_t); /// Outbound transparent. - bool f_outbound_transparent; + bool f_outbound_transparent = false; /// Set outbound transparency. self &setOutboundTransparent(bool); /// Transparent pass-through. - bool f_transparent_passthrough; + bool f_transparent_passthrough = false; /// Set transparent passthrough. self &setTransparentPassthrough(bool); - /// Accepting backdoor connections. - bool backdoor; - /// Set backdoor accept. - self &setBackdoor(bool); /// Host address resolution preference order. HostResPreferenceOrder host_res_preference; /// Set the host query preference. @@ -96,7 +92,7 @@ class HttpSessionAcceptOptions }; inline HttpSessionAcceptOptions::HttpSessionAcceptOptions() - : transport_type(0), outbound_port(0), f_outbound_transparent(false), f_transparent_passthrough(false), backdoor(false) + { memcpy(host_res_preference, host_res_default_preference_order, sizeof(host_res_preference)); } @@ -149,13 +145,6 @@ HttpSessionAcceptOptions::setTransparentPassthrough(bool flag) return *this; } -inline HttpSessionAcceptOptions & -HttpSessionAcceptOptions::setBackdoor(bool flag) -{ - backdoor = flag; - return *this; -} - inline HttpSessionAcceptOptions & HttpSessionAcceptOptions::setHostResPreference(HostResPreferenceOrder const order) { diff --git a/proxy/http/HttpSessionManager.cc b/proxy/http/HttpSessionManager.cc index c39f949124b..d95a79decdc 100644 --- a/proxy/http/HttpSessionManager.cc +++ b/proxy/http/HttpSessionManager.cc @@ -212,7 +212,7 @@ ServerSessionPool::eventHandler(int event, void *data) if (connection_count_below_min) { Debug("http_ss", "[%" PRId64 "] [session_bucket] session received io notice [%s], " - "reseting timeout to maintain minimum number of connections", + "resetting timeout to maintain minimum number of connections", s->con_id, HttpDebugNames::get_event_name(event)); s->get_netvc()->set_inactivity_timeout(s->get_netvc()->get_inactivity_timeout()); s->get_netvc()->set_active_timeout(s->get_netvc()->get_active_timeout()); @@ -320,10 +320,11 @@ HttpSessionManager::acquire_session(Continuation * /* cont ATS_UNUSED */, sockad // client session { // Now check to see if we have a connection in our shared connection pool - EThread *ethread = this_ethread(); - ProxyMutex *pool_mutex = (TS_SERVER_SESSION_SHARING_POOL_THREAD == sm->t_state.http_config_param->server_session_sharing_pool) ? - ethread->server_session_pool->mutex.get() : - m_g_pool->mutex.get(); + EThread *ethread = this_ethread(); + Ptr pool_mutex = + (TS_SERVER_SESSION_SHARING_POOL_THREAD == sm->t_state.http_config_param->server_session_sharing_pool) ? + ethread->server_session_pool->mutex : + m_g_pool->mutex; MUTEX_TRY_LOCK(lock, pool_mutex, ethread); if (lock.is_locked()) { if (TS_SERVER_SESSION_SHARING_POOL_THREAD == sm->t_state.http_config_param->server_session_sharing_pool) { diff --git a/proxy/http/HttpSessionManager.h b/proxy/http/HttpSessionManager.h index f83196aad0e..4112e3b8ae7 100644 --- a/proxy/http/HttpSessionManager.h +++ b/proxy/http/HttpSessionManager.h @@ -101,7 +101,7 @@ class ServerSessionPool : public Continuation class HttpSessionManager { public: - HttpSessionManager() : m_g_pool(nullptr) {} + HttpSessionManager() {} ~HttpSessionManager() {} HSMresult_t acquire_session(Continuation *cont, sockaddr const *addr, const char *hostname, ProxyClientTransaction *ua_txn, HttpSM *sm); @@ -113,7 +113,7 @@ class HttpSessionManager private: /// Global pool, used if not per thread pools. /// @internal We delay creating this because the session manager is created during global statics init. - ServerSessionPool *m_g_pool; + ServerSessionPool *m_g_pool = nullptr; }; extern HttpSessionManager httpSessionManager; diff --git a/proxy/http/HttpTransact.cc b/proxy/http/HttpTransact.cc index 3c92d0265ce..ed30844b4c4 100644 --- a/proxy/http/HttpTransact.cc +++ b/proxy/http/HttpTransact.cc @@ -5571,7 +5571,12 @@ HttpTransact::initialize_state_variables_from_request(State *s, HTTPHdr *obsolet ats_ip_copy(&s->request_data.src_ip, &s->client_info.src_addr); memset(&s->request_data.dest_ip, 0, sizeof(s->request_data.dest_ip)); if (vc) { - s->request_data.incoming_port = vc->get_local_port(); + s->request_data.incoming_port = vc->get_local_port(); + s->pp_info.proxy_protocol_version = vc->get_proxy_protocol_version(); + if (s->pp_info.proxy_protocol_version != NetVConnection::ProxyProtocolVersion::UNDEFINED) { + ats_ip_copy(s->pp_info.src_addr, vc->pp_info.src_addr); + ats_ip_copy(s->pp_info.dst_addr, vc->pp_info.dst_addr); + } } s->request_data.xact_start = s->client_request_time; s->request_data.api_info = &s->api_info; @@ -6617,7 +6622,13 @@ HttpTransact::handle_content_length_header(State *s, HTTPHdr *header, HTTPHdr *b TxnDebug("http_trans", "[handle_content_length_header] RESPONSE cont len in hdr is %" PRId64, header->get_content_length()); } else { // No content length header. - if (s->source == SOURCE_CACHE) { + // If the source is cache or server returned 304 response, + // we can try to get the content length based on object size. + // Also, we should check the scenario of server sending't a unexpected 304 response for a non conditional request( no cached + // object ) + if (s->source == SOURCE_CACHE || + (s->source == SOURCE_HTTP_ORIGIN_SERVER && s->hdr_info.server_response.status_get() == HTTP_STATUS_NOT_MODIFIED && + s->cache_info.object_read != nullptr)) { // If there is no content-length header, we can // insert one since the cache knows definitely // how long the object is unless we're in a @@ -6632,18 +6643,14 @@ HttpTransact::handle_content_length_header(State *s, HTTPHdr *header, HTTPHdr *b } else if (s->range_setup == RANGE_NOT_TRANSFORM_REQUESTED) { // if we are doing a single Range: request, calculate the new // C-L: header + // either the object is in cache or origin returned a 304 Not Modified response. We can still turn this into a proper Range + // response from the cached object. change_response_header_because_of_range_request(s, header); s->hdr_info.trust_response_cl = true; } else { header->set_content_length(cl); s->hdr_info.trust_response_cl = true; } - } else if (s->source == SOURCE_HTTP_ORIGIN_SERVER && s->hdr_info.server_response.status_get() == HTTP_STATUS_NOT_MODIFIED && - s->range_setup == RANGE_NOT_TRANSFORM_REQUESTED) { - // In this case, we had a cached object, possibly chunked encoded (so we don't have a CL: header), but the origin did a - // 304 Not Modified response. We can still turn this into a proper Range response from the cached object. - change_response_header_because_of_range_request(s, header); - s->hdr_info.trust_response_cl = true; } else { // Check to see if there is no content length // header because the response precludes a @@ -7204,11 +7211,16 @@ HttpTransact::what_is_document_freshness(State *s, HTTPHdr *client_request, HTTP current_age = HttpTransactHeaders::calculate_document_age(s->request_sent_time, s->response_received_time, cached_obj_response, response_date, s->current.now); - // Overflow ? + // First check overflow status + // Second if current_age is under the max, use the smaller value + // Finally we take the max of current age or guaranteed max, this ensures it will + // age out properly, otherwise a doc will never expire if guaranteed < document max-age if (current_age < 0) { current_age = s->txn_conf->cache_guaranteed_max_lifetime; - } else { + } else if (current_age < s->txn_conf->cache_guaranteed_max_lifetime) { current_age = std::min((time_t)s->txn_conf->cache_guaranteed_max_lifetime, current_age); + } else { + current_age = std::max((time_t)s->txn_conf->cache_guaranteed_max_lifetime, current_age); } TxnDebug("http_match", "[what_is_document_freshness] fresh_limit: %d current_age: %" PRId64, fresh_limit, (int64_t)current_age); @@ -7324,7 +7336,7 @@ HttpTransact::what_is_document_freshness(State *s, HTTPHdr *client_request, HTTP // now, see if the age is "fresh enough" // /////////////////////////////////////////// - if (do_revalidate || current_age > age_limit) { // client-modified limit + if (do_revalidate || !age_limit || current_age > age_limit) { // client-modified limit TxnDebug("http_match", "[..._document_freshness] document needs revalidate/too old; " "returning FRESHNESS_STALE"); return (FRESHNESS_STALE); @@ -7544,13 +7556,7 @@ HttpTransact::build_request(State *s, HTTPHdr *base_request, HTTPHdr *outgoing_r // // notice that currently, based_request IS client_request if (base_request == &s->hdr_info.client_request) { - if (s->redirect_info.redirect_in_process) { - // this is for auto redirect - URL *r_url = &s->redirect_info.redirect_url; - - ink_assert(r_url->valid()); - base_request->url_get()->copy(r_url); - } else { + if (!s->redirect_info.redirect_in_process) { // this is for multiple cache lookup URL *o_url = &s->cache_info.original_url; @@ -7974,9 +7980,9 @@ HttpTransact::build_error_response(State *s, HTTPStatus status_code, const char int64_t len; char *new_msg; - new_msg = body_factory->fabricate_with_old_api(error_body_type, s, 8192, &len, body_language, sizeof(body_language), body_type, - sizeof(body_type), s->internal_msg_buffer_size, - s->internal_msg_buffer_size ? s->internal_msg_buffer : nullptr); + new_msg = body_factory->fabricate_with_old_api( + error_body_type, s, s->http_config_param->body_factory_response_max_size, &len, body_language, sizeof(body_language), body_type, + sizeof(body_type), s->internal_msg_buffer_size, s->internal_msg_buffer_size ? s->internal_msg_buffer : nullptr); // After the body factory is called, a new "body" is allocated, and we must replace it. It is // unfortunate that there's no way to avoid this fabrication even when there is no substitutions... @@ -8696,42 +8702,34 @@ HttpTransact::update_size_and_time_stats(State *s, ink_hrtime total_time, ink_hr void HttpTransact::delete_warning_value(HTTPHdr *to_warn, HTTPWarningCode warning_code) { - int w_code = (int)warning_code; + int w_code = static_cast(warning_code); MIMEField *field = to_warn->field_find(MIME_FIELD_WARNING, MIME_LEN_WARNING); - ; // Loop over the values to see if we need to do anything if (field) { HdrCsvIter iter; - - int valid; int val_code; - - const char *value_str; - int value_len; - MIMEField *new_field = nullptr; - val_code = iter.get_first_int(field, &valid); - while (valid) { + bool valid_p = iter.get_first_int(field, val_code); + + while (valid_p) { if (val_code == w_code) { - // Ok, found the value we're look to delete - // Look over and create a new field - // appending all elements that are not this - // value - val_code = iter.get_first_int(field, &valid); + // Ok, found the value we're look to delete Look over and create a new field appending all + // elements that are not this value + valid_p = iter.get_first_int(field, val_code); - while (valid) { + while (valid_p) { if (val_code != warning_code) { - value_str = iter.get_current(&value_len); + auto value = iter.get_current(); if (new_field) { - new_field->value_append(to_warn->m_heap, to_warn->m_mime, value_str, value_len, true); + new_field->value_append(to_warn->m_heap, to_warn->m_mime, value.data(), value.size(), true); } else { new_field = to_warn->field_create(); - to_warn->field_value_set(new_field, value_str, value_len); + to_warn->field_value_set(new_field, value.data(), value.size()); } } - val_code = iter.get_next_int(&valid); + valid_p = iter.get_next_int(val_code); } to_warn->field_delete(MIME_FIELD_WARNING, MIME_LEN_WARNING); @@ -8742,8 +8740,7 @@ HttpTransact::delete_warning_value(HTTPHdr *to_warn, HTTPWarningCode warning_cod return; } - - val_code = iter.get_next_int(&valid); + valid_p = iter.get_next_int(val_code); } } } diff --git a/proxy/http/HttpTransact.h b/proxy/http/HttpTransact.h index 031a0a0a2cb..af40b1530bd 100644 --- a/proxy/http/HttpTransact.h +++ b/proxy/http/HttpTransact.h @@ -530,7 +530,6 @@ class HttpTransact typedef struct _RedirectInfo { bool redirect_in_process = false; URL original_url; - URL redirect_url; _RedirectInfo() {} } RedirectInfo; @@ -714,7 +713,6 @@ class HttpTransact bool cdn_remap_complete = false; bool first_dns_lookup = true; - bool backdoor_request = false; // internal HttpRequestData request_data; ParentConfigParams *parent_params = nullptr; ParentResult parent_result; @@ -770,8 +768,8 @@ class HttpTransact // INK API/Remap API plugin interface void *remap_plugin_instance = nullptr; void *user_args[TS_HTTP_MAX_USER_ARG]; - remap_plugin_info::_tsremap_os_response *fp_tsremap_os_response = nullptr; - HTTPStatus http_return_code = HTTP_STATUS_NONE; + RemapPluginInfo::OS_Response_F *fp_tsremap_os_response = nullptr; + HTTPStatus http_return_code = HTTP_STATUS_NONE; int api_txn_active_timeout_value = -1; int api_txn_connect_timeout_value = -1; @@ -883,7 +881,6 @@ class HttpTransact cache_info.object_store.destroy(); cache_info.transform_store.destroy(); redirect_info.original_url.destroy(); - redirect_info.redirect_url.destroy(); url_map.clear(); arena.reset(); @@ -921,6 +918,9 @@ class HttpTransact } internal_msg_buffer_size = 0; } + + NetVConnection::ProxyProtocol pp_info; + }; // End of State struct. static void HandleBlindTunnel(State *s); @@ -1081,15 +1081,17 @@ class HttpTransact typedef void (*TransactEntryFunc_t)(HttpTransact::State *s); -//////////////////////////////////////////////////////// -// the spec says about message body the following: // -// All responses to the HEAD request method MUST NOT // -// include a message-body, even though the presence // -// of entity-header fields might lead one to believe // -// they do. All 1xx (informational), 204 (no content),// -// and 304 (not modified) responses MUST NOT include // -// a message-body. // -//////////////////////////////////////////////////////// +/* The spec says about message body the following: + * + * All responses to the HEAD and CONNECT request method + * MUST NOT include a message-body, even though the presence + * of entity-header fields might lead one to believe they do. + * + * All 1xx (informational), 204 (no content), and 304 (not modified) + * responses MUST NOT include a message-body. + * + * Refer : [https://tools.ietf.org/html/rfc7231#section-4.3.6] + */ inline bool is_response_body_precluded(HTTPStatus status_code) { @@ -1105,11 +1107,11 @@ is_response_body_precluded(HTTPStatus status_code) inline bool is_response_body_precluded(HTTPStatus status_code, int method) { - if ((method == HTTP_WKSIDX_HEAD) || is_response_body_precluded(status_code)) { + if ((method == HTTP_WKSIDX_HEAD) || (method == HTTP_WKSIDX_CONNECT) || is_response_body_precluded(status_code)) { return true; } else { return false; } } -inkcoreapi extern ink_time_t ink_local_time(void); +inkcoreapi extern ink_time_t ink_local_time(); diff --git a/proxy/http/HttpTransactCache.cc b/proxy/http/HttpTransactCache.cc index 0b5ecdf92f1..56adb7ba623 100644 --- a/proxy/http/HttpTransactCache.cc +++ b/proxy/http/HttpTransactCache.cc @@ -1288,6 +1288,33 @@ HttpTransactCache::match_response_to_request_conditionals(HTTPHdr *request, HTTP return response->status_get(); } + // If-None-Match: may match weakly // + if (request->presence(MIME_PRESENCE_IF_NONE_MATCH)) { + int raw_etags_len, comma_sep_tag_list_len; + const char *raw_etags = response->value_get(MIME_FIELD_ETAG, MIME_LEN_ETAG, &raw_etags_len); + const char *comma_sep_tag_list = nullptr; + + if (raw_etags) { + comma_sep_tag_list = request->value_get(MIME_FIELD_IF_NONE_MATCH, MIME_LEN_IF_NONE_MATCH, &comma_sep_tag_list_len); + if (!comma_sep_tag_list) { + comma_sep_tag_list = ""; + comma_sep_tag_list_len = 0; + } + + //////////////////////////////////////////////////////////////////////// + // If we have an etag and a if-none-match, we are talking to someone // + // who is doing a 1.1 revalidate. Since this is a GET request with no // + // sub-ranges, we can do a weak validation. // + //////////////////////////////////////////////////////////////////////// + if (do_strings_match_weakly(raw_etags, raw_etags_len, comma_sep_tag_list, comma_sep_tag_list_len)) { + // the response already failed If-modified-since (if one exists) + return HTTP_STATUS_NOT_MODIFIED; + } else { + return response->status_get(); + } + } + } + // If-Modified-Since // if (request->presence(MIME_PRESENCE_IF_MODIFIED_SINCE)) { if (response->presence(MIME_PRESENCE_LAST_MODIFIED)) { @@ -1316,38 +1343,6 @@ HttpTransactCache::match_response_to_request_conditionals(HTTPHdr *request, HTTP response_code = HTTP_STATUS_NOT_MODIFIED; } - - // we cannot return NOT_MODIFIED yet, need to check If-none-match - if (!request->presence(MIME_PRESENCE_IF_NONE_MATCH)) { - return response_code; - } - } - - // If-None-Match: may match weakly // - if (request->presence(MIME_PRESENCE_IF_NONE_MATCH)) { - int raw_etags_len, comma_sep_tag_list_len; - const char *raw_etags = response->value_get(MIME_FIELD_ETAG, MIME_LEN_ETAG, &raw_etags_len); - const char *comma_sep_tag_list = nullptr; - - if (raw_etags) { - comma_sep_tag_list = request->value_get(MIME_FIELD_IF_NONE_MATCH, MIME_LEN_IF_NONE_MATCH, &comma_sep_tag_list_len); - if (!comma_sep_tag_list) { - comma_sep_tag_list = ""; - comma_sep_tag_list_len = 0; - } - - //////////////////////////////////////////////////////////////////////// - // If we have an etag and a if-none-match, we are talking to someone // - // who is doing a 1.1 revalidate. Since this is a GET request with no // - // sub-ranges, we can do a weak validation. // - //////////////////////////////////////////////////////////////////////// - if (do_strings_match_weakly(raw_etags, raw_etags_len, comma_sep_tag_list, comma_sep_tag_list_len)) { - // the response already failed If-modified-since (if one exists) - return HTTP_STATUS_NOT_MODIFIED; - } else { - return response->status_get(); - } - } } // There is no If-none-match, and If-modified-since failed, diff --git a/proxy/http/HttpTransactHeaders.cc b/proxy/http/HttpTransactHeaders.cc index 95f662626c7..5f18843b4d8 100644 --- a/proxy/http/HttpTransactHeaders.cc +++ b/proxy/http/HttpTransactHeaders.cc @@ -1155,25 +1155,19 @@ HttpTransactHeaders::add_forwarded_field_to_request(HttpTransact::State *s, HTTP if (n_proto > 0) { auto Conn = [&](HttpForwarded::Option opt, HttpTransactHeaders::ProtocolStackDetail detail) -> void { - if (optSet[opt]) { - int revert = hdr.size(); + if (optSet[opt] && hdr.remaining() > 0) { + ts::FixedBufferWriter lw{hdr.auxBuffer(), hdr.remaining()}; if (hdr.size()) { - hdr << ';'; + lw << ';'; } - hdr << "connection="; + lw << "connection="; int numChars = - HttpTransactHeaders::write_hdr_protocol_stack(hdr.auxBuffer(), hdr.remaining(), detail, protoBuf.data(), n_proto, '-'); - if (numChars > 0) { - hdr.fill(size_t(numChars)); - } - - if ((numChars <= 0) or (hdr.size() >= hdr.capacity())) { - // Remove parameter with potentially incomplete value. - // - hdr.reduce(revert); + HttpTransactHeaders::write_hdr_protocol_stack(lw.auxBuffer(), lw.remaining(), detail, protoBuf.data(), n_proto, '-'); + if (numChars > 0 && !lw.fill(size_t(numChars)).error()) { + hdr.fill(lw.size()); } } }; diff --git a/proxy/http/HttpTunnel.cc b/proxy/http/HttpTunnel.cc index 86ca78be6b8..8e37e72d321 100644 --- a/proxy/http/HttpTunnel.cc +++ b/proxy/http/HttpTunnel.cc @@ -44,26 +44,7 @@ static const char *const CHUNK_HEADER_FMT = "%" PRIx64 "\r\n"; // a block in the input stream. static int const CHUNK_IOBUFFER_SIZE_INDEX = MIN_IOBUFFER_SIZE; -ChunkedHandler::ChunkedHandler() - : action(ACTION_UNSET), - chunked_reader(nullptr), - dechunked_buffer(nullptr), - dechunked_size(0), - dechunked_reader(nullptr), - chunked_buffer(nullptr), - chunked_size(0), - truncation(false), - skip_bytes(0), - state(CHUNK_READ_CHUNK), - cur_chunk_size(0), - bytes_left(0), - last_server_event(VC_EVENT_NONE), - running_sum(0), - num_digits(0), - max_chunk_size(DEFAULT_MAX_CHUNK_SIZE), - max_chunk_header_len(0) -{ -} +ChunkedHandler::ChunkedHandler() : max_chunk_size(DEFAULT_MAX_CHUNK_SIZE) {} void ChunkedHandler::init(IOBufferReader *buffer_in, HttpTunnelProducer *p) @@ -393,32 +374,7 @@ ChunkedHandler::generate_chunked_content() return false; } -HttpTunnelProducer::HttpTunnelProducer() - : consumer_list(), - self_consumer(nullptr), - vc(nullptr), - vc_handler(nullptr), - read_vio(nullptr), - read_buffer(nullptr), - buffer_start(nullptr), - vc_type(HT_HTTP_SERVER), - chunking_action(TCA_PASSTHRU_DECHUNKED_CONTENT), - do_chunking(false), - do_dechunking(false), - do_chunked_passthru(false), - init_bytes_done(0), - nbytes(0), - ntodo(0), - bytes_read(0), - handler_state(0), - last_event(0), - num_consumers(0), - alive(false), - read_success(false), - flow_control_source(nullptr), - name(nullptr) -{ -} +HttpTunnelProducer::HttpTunnelProducer() : consumer_list() {} uint64_t HttpTunnelProducer::backlog(uint64_t limit) @@ -484,23 +440,7 @@ HttpTunnelProducer::set_throttle_src(HttpTunnelProducer *srcp) } } -HttpTunnelConsumer::HttpTunnelConsumer() - : link(), - producer(nullptr), - self_producer(nullptr), - vc_type(HT_HTTP_CLIENT), - vc(nullptr), - buffer_reader(nullptr), - vc_handler(nullptr), - write_vio(nullptr), - skip_bytes(0), - bytes_written(0), - handler_state(0), - alive(false), - write_success(false), - name(nullptr) -{ -} +HttpTunnelConsumer::HttpTunnelConsumer() : link() {} HttpTunnel::HttpTunnel() : Continuation(nullptr) {} @@ -529,11 +469,11 @@ HttpTunnel::reset() { ink_assert(active == false); #ifdef DEBUG - for (int i = 0; i < MAX_PRODUCERS; ++i) { - ink_assert(producers[i].alive == false); + for (auto &producer : producers) { + ink_assert(producer.alive == false); } - for (int j = 0; j < MAX_CONSUMERS; ++j) { - ink_assert(consumers[j].alive == false); + for (auto &consumer : consumers) { + ink_assert(consumer.alive == false); } #endif @@ -776,7 +716,7 @@ HttpTunnel::tunnel_run(HttpTunnelProducer *p_arg) // back to say we are done if (!is_tunnel_alive()) { active = false; - sm->dispatchEvent(HTTP_TUNNEL_EVENT_DONE, this); + sm->handleEvent(HTTP_TUNNEL_EVENT_DONE, this); } } @@ -1272,10 +1212,8 @@ HttpTunnel::consumer_reenable(HttpTunnelConsumer *c) { HttpTunnelProducer *p = c->producer; - if (p && p->alive -#ifndef LAZY_BUF_ALLOC - && p->read_buffer->write_avail() > 0 -#endif + if (p && p->alive && p->read_buffer->write_avail() > 0 + ) { // Only do flow control if enabled and the producer is an external // source. Otherwise disable by making the backlog zero. Because @@ -1398,9 +1336,9 @@ HttpTunnel::consumer_handler(int event, HttpTunnelConsumer *c) // updating the buffer state for the VConnection // that is being reenabled if (p->alive && p->read_vio -#ifndef LAZY_BUF_ALLOC + && p->read_buffer->write_avail() > 0 -#endif + ) { if (p->is_throttled()) { this->consumer_reenable(c); @@ -1640,7 +1578,7 @@ HttpTunnel::main_handler(int event, void *data) if (reentrancy_count == 1) { reentrancy_count = 0; active = false; - sm->dispatchEvent(HTTP_TUNNEL_EVENT_DONE, this); + sm->handleEvent(HTTP_TUNNEL_EVENT_DONE, this); return EVENT_DONE; } else { call_sm = true; diff --git a/proxy/http/HttpTunnel.h b/proxy/http/HttpTunnel.h index e47160d9a0c..272b33e674f 100644 --- a/proxy/http/HttpTunnel.h +++ b/proxy/http/HttpTunnel.h @@ -94,27 +94,27 @@ struct ChunkedHandler { enum Action { ACTION_DOCHUNK = 0, ACTION_DECHUNK, ACTION_PASSTHRU, ACTION_UNSET }; - Action action; + Action action = ACTION_UNSET; - IOBufferReader *chunked_reader; - MIOBuffer *dechunked_buffer; - int64_t dechunked_size; + IOBufferReader *chunked_reader = nullptr; + MIOBuffer *dechunked_buffer = nullptr; + int64_t dechunked_size = 0; - IOBufferReader *dechunked_reader; - MIOBuffer *chunked_buffer; - int64_t chunked_size; + IOBufferReader *dechunked_reader = nullptr; + MIOBuffer *chunked_buffer = nullptr; + int64_t chunked_size = 0; - bool truncation; - int64_t skip_bytes; + bool truncation = false; + int64_t skip_bytes = 0; - ChunkedState state; - int64_t cur_chunk_size; - int64_t bytes_left; - int last_server_event; + ChunkedState state = CHUNK_READ_CHUNK; + int64_t cur_chunk_size = 0; + int64_t bytes_left = 0; + int last_server_event = VC_EVENT_NONE; // Parsing Info - int running_sum; - int num_digits; + int running_sum = 0; + int num_digits = 0; /// @name Output data. //@{ @@ -125,7 +125,7 @@ struct ChunkedHandler { /// It holds the header for a maximal sized chunk which will cover /// almost all output chunks. char max_chunk_header[16]; - int max_chunk_header_len; + int max_chunk_header_len = 0; //@} ChunkedHandler(); @@ -152,22 +152,22 @@ struct HttpTunnelConsumer { HttpTunnelConsumer(); LINK(HttpTunnelConsumer, link); - HttpTunnelProducer *producer; - HttpTunnelProducer *self_producer; + HttpTunnelProducer *producer = nullptr; + HttpTunnelProducer *self_producer = nullptr; - HttpTunnelType_t vc_type; - VConnection *vc; - IOBufferReader *buffer_reader; - HttpConsumerHandler vc_handler; - VIO *write_vio; + HttpTunnelType_t vc_type = HT_HTTP_CLIENT; + VConnection *vc = nullptr; + IOBufferReader *buffer_reader = nullptr; + HttpConsumerHandler vc_handler = nullptr; + VIO *write_vio = nullptr; - int64_t skip_bytes; // bytes to skip at beginning of stream - int64_t bytes_written; // total bytes written to the vc - int handler_state; // state used the handlers + int64_t skip_bytes = 0; // bytes to skip at beginning of stream + int64_t bytes_written = 0; // total bytes written to the vc + int handler_state = 0; // state used the handlers - bool alive; - bool write_success; - const char *name; + bool alive = false; + bool write_success = false; + const char *name = nullptr; /** Check if this consumer is downstream from @a vc. @return @c true if any producer in the tunnel eventually feeds @@ -184,37 +184,37 @@ struct HttpTunnelProducer { HttpTunnelProducer(); DLL consumer_list; - HttpTunnelConsumer *self_consumer; - VConnection *vc; - HttpProducerHandler vc_handler; - VIO *read_vio; - MIOBuffer *read_buffer; - IOBufferReader *buffer_start; - HttpTunnelType_t vc_type; + HttpTunnelConsumer *self_consumer = nullptr; + VConnection *vc = nullptr; + HttpProducerHandler vc_handler = nullptr; + VIO *read_vio = nullptr; + MIOBuffer *read_buffer = nullptr; + IOBufferReader *buffer_start = nullptr; + HttpTunnelType_t vc_type = HT_HTTP_SERVER; ChunkedHandler chunked_handler; - TunnelChunkingAction_t chunking_action; + TunnelChunkingAction_t chunking_action = TCA_PASSTHRU_DECHUNKED_CONTENT; - bool do_chunking; - bool do_dechunking; - bool do_chunked_passthru; + bool do_chunking = false; + bool do_dechunking = false; + bool do_chunked_passthru = false; - int64_t init_bytes_done; // bytes passed in buffer - int64_t nbytes; // total bytes (client's perspective) - int64_t ntodo; // what this vc needs to do - int64_t bytes_read; // total bytes read from the vc - int handler_state; // state used the handlers - int last_event; ///< Tracking for flow control restarts. + int64_t init_bytes_done = 0; // bytes passed in buffer + int64_t nbytes = 0; // total bytes (client's perspective) + int64_t ntodo = 0; // what this vc needs to do + int64_t bytes_read = 0; // total bytes read from the vc + int handler_state = 0; // state used the handlers + int last_event = 0; ///< Tracking for flow control restarts. - int num_consumers; + int num_consumers = 0; - bool alive; - bool read_success; + bool alive = false; + bool read_success = false; /// Flag and pointer for active flow control throttling. /// If this is set, it points at the source producer that is under flow control. /// If @c NULL then data flow is not being throttled. - HttpTunnelProducer *flow_control_source; - const char *name; + HttpTunnelProducer *flow_control_source = nullptr; + const char *name = nullptr; /** Get the largest number of bytes any consumer has not consumed. Use @a limit if you only need to check if the backlog is at least @a limit. @@ -261,9 +261,9 @@ class HttpTunnel : public Continuation // Default value for high and low water marks. static uint64_t const DEFAULT_WATER_MARK = 1 << 16; - uint64_t high_water; ///< Buffered data limit - throttle if more than this. - uint64_t low_water; ///< Unthrottle if less than this buffered. - bool enabled_p; ///< Flow control state (@c false means disabled). + uint64_t high_water; ///< Buffered data limit - throttle if more than this. + uint64_t low_water; ///< Unthrottle if less than this buffered. + bool enabled_p = false; ///< Flow control state (@c false means disabled). /// Default constructor. FlowControl(); @@ -517,7 +517,7 @@ inline bool HttpTunnelConsumer::is_downstream_from(VConnection *vc) { HttpTunnelProducer *p = producer; - HttpTunnelConsumer *c; + while (p) { if (p->vc == vc) { return true; @@ -525,7 +525,8 @@ HttpTunnelConsumer::is_downstream_from(VConnection *vc) // The producer / consumer chain can contain a cycle in the case // of a blind tunnel so give up if we find ourself (the original // consumer). - c = p->self_consumer; + HttpTunnelConsumer *c = p->self_consumer; + p = (c && c != this) ? c->producer : nullptr; } return false; @@ -575,4 +576,4 @@ HttpTunnelProducer::unthrottle() } } -inline HttpTunnel::FlowControl::FlowControl() : high_water(DEFAULT_WATER_MARK), low_water(DEFAULT_WATER_MARK), enabled_p(false) {} +inline HttpTunnel::FlowControl::FlowControl() : high_water(DEFAULT_WATER_MARK), low_water(DEFAULT_WATER_MARK) {} diff --git a/proxy/http/HttpUpdateSM.cc b/proxy/http/HttpUpdateSM.cc index a6ae057dc45..fa89ac58663 100644 --- a/proxy/http/HttpUpdateSM.cc +++ b/proxy/http/HttpUpdateSM.cc @@ -42,7 +42,7 @@ ClassAllocator httpUpdateSMAllocator("httpUpdateSMAllocator"); Debug("http", "[%" PRId64 "] [%s, %s]", sm_id, #state_name, HttpDebugNames::get_event_name(event)); \ } -HttpUpdateSM::HttpUpdateSM() : cb_occured(false), cb_cont(nullptr), cb_action(), cb_event(HTTP_SCH_UPDATE_EVENT_ERROR) {} +HttpUpdateSM::HttpUpdateSM() : cb_action(), cb_event(HTTP_SCH_UPDATE_EVENT_ERROR) {} void HttpUpdateSM::destroy() @@ -72,7 +72,6 @@ HttpUpdateSM::start_scheduled_update(Continuation *cont, HTTPHdr *request) // Fix ME: What should these be set to since there is not a // real client ats_ip4_set(&t_state.client_info.src_addr, htonl(INADDR_LOOPBACK), 0); - t_state.backdoor_request = false; t_state.client_info.port_attribute = HttpProxyPort::TRANSPORT_DEFAULT; t_state.req_flavor = HttpTransact::REQ_FLAVOR_SCHEDULED_UPDATE; diff --git a/proxy/http/HttpUpdateSM.h b/proxy/http/HttpUpdateSM.h index 67662937036..a55258923ce 100644 --- a/proxy/http/HttpUpdateSM.h +++ b/proxy/http/HttpUpdateSM.h @@ -50,20 +50,20 @@ class HttpUpdateSM : public HttpSM HttpUpdateSM(); static HttpUpdateSM *allocate(); - void destroy(); + void destroy() override; Action *start_scheduled_update(Continuation *cont, HTTPHdr *req); // private: - bool cb_occured; - Continuation *cb_cont; + bool cb_occured = false; + Continuation *cb_cont = nullptr; Action cb_action; int cb_event; protected: - void handle_api_return(); - void set_next_state(); - int kill_this_async_hook(int event, void *data); + void handle_api_return() override; + void set_next_state() override; + int kill_this_async_hook(int event, void *data) override; }; inline HttpUpdateSM * diff --git a/proxy/http/Makefile.am b/proxy/http/Makefile.am index 0706127c8f8..6f5733229b0 100644 --- a/proxy/http/Makefile.am +++ b/proxy/http/Makefile.am @@ -107,7 +107,7 @@ test_proxy_http_LDADD = \ $(top_builddir)/mgmt/libmgmt_p.la \ $(top_builddir)/iocore/utils/libinkutils.a \ @HWLOC_LIBS@ \ - @LIBTCL@ @LIBCAP@ + @LIBCAP@ clang-tidy-local: $(libhttp_a_SOURCES) $(noinst_HEADERS) $(CXX_Clang_Tidy) diff --git a/proxy/http/remap/AclFiltering.cc b/proxy/http/remap/AclFiltering.cc index 16bfe592333..64673694feb 100644 --- a/proxy/http/remap/AclFiltering.cc +++ b/proxy/http/remap/AclFiltering.cc @@ -51,8 +51,7 @@ acl_filter_rule::reset() internal = 0; } -acl_filter_rule::acl_filter_rule() - : next(nullptr), filter_name(nullptr), allow_flag(1), src_ip_valid(0), active_queue_flag(0), internal(0), argc(0) +acl_filter_rule::acl_filter_rule() : allow_flag(1), src_ip_valid(0), active_queue_flag(0), internal(0) { standard_method_lookup.resize(HTTP_WKSIDX_METHODS_CNT); ink_zero(argv); diff --git a/proxy/http/remap/AclFiltering.h b/proxy/http/remap/AclFiltering.h index f9cb8c8d80b..ceba59a7c2c 100644 --- a/proxy/http/remap/AclFiltering.h +++ b/proxy/http/remap/AclFiltering.h @@ -64,19 +64,19 @@ struct src_ip_info_t { class acl_filter_rule { private: - void reset(void); + void reset(); public: - acl_filter_rule *next; - char *filter_name; // optional filter name - unsigned int allow_flag : 1, // action allow deny - src_ip_valid : 1, // src_ip range valid + acl_filter_rule *next = nullptr; + char *filter_name = nullptr; // optional filter name + unsigned int allow_flag : 1, // action allow deny + src_ip_valid : 1, // src_ip range valid in_ip_valid : 1, active_queue_flag : 1, // filter is in active state (used by .useflt directive) internal : 1; // filter internal HTTP requests // we need arguments as string array for directive processing - int argc; // argument counter (only for filter defs) + int argc = 0; // argument counter (only for filter defs) char *argv[ACL_FILTER_MAX_ARGV]; // argument strings (only for filter defs) // methods @@ -91,14 +91,14 @@ class acl_filter_rule src_ip_info_t src_ip_array[ACL_FILTER_MAX_SRC_IP]; // in_ip - int in_ip_cnt; // how many valid dst_ip rules we have + int in_ip_cnt; // how many valid dest_ip rules we have src_ip_info_t in_ip_array[ACL_FILTER_MAX_IN_IP]; acl_filter_rule(); ~acl_filter_rule(); void name(const char *_name = nullptr); int add_argv(int _argc, char *_argv[]); - void print(void); + void print(); static acl_filter_rule *find_byname(acl_filter_rule *list, const char *name); static void delete_byname(acl_filter_rule **list, const char *name); diff --git a/proxy/http/remap/RemapConfig.cc b/proxy/http/remap/RemapConfig.cc index 0feefdf6d00..8d5948f7796 100644 --- a/proxy/http/remap/RemapConfig.cc +++ b/proxy/http/remap/RemapConfig.cc @@ -79,7 +79,7 @@ clear_xstr_array(char *v[], size_t vsize) } BUILD_TABLE_INFO::BUILD_TABLE_INFO() - : remap_optflg(0), paramc(0), argc(0), ip_allow_check_enabled_p(true), accept_check_p(true), rules_list(nullptr), rewrite(nullptr) + { memset(this->paramv, 0, sizeof(this->paramv)); memset(this->argv, 0, sizeof(this->argv)); @@ -163,7 +163,6 @@ parse_define_directive(const char *directive, BUILD_TABLE_INFO *bti, char *errbu { bool flg; acl_filter_rule *rp; - acl_filter_rule **rpp; const char *cstr = nullptr; if (bti->paramc < 2) { @@ -181,6 +180,7 @@ parse_define_directive(const char *directive, BUILD_TABLE_INFO *bti, char *errbu // coverity[alloc_arg] if ((cstr = remap_validate_filter_args(&rp, (const char **)bti->argv, bti->argc, errbuf, errbufsize)) == nullptr && rp) { if (flg) { // new filter - add to list + acl_filter_rule **rpp = nullptr; Debug("url_rewrite", "[parse_directive] new rule \"%s\" was created", bti->paramv[1]); for (rpp = &bti->rules_list; *rpp; rpp = &((*rpp)->next)) { ; @@ -400,7 +400,7 @@ remap_parse_directive(BUILD_TABLE_INFO *bti, char *errbuf, size_t errbufsize) const char *directive = nullptr; // Check arguments - if (unlikely(!bti || !errbuf || errbufsize <= 0 || !bti->paramc || (directive = bti->paramv[0]) == nullptr)) { + if (unlikely(!bti || !errbuf || errbufsize == 0 || !bti->paramc || (directive = bti->paramv[0]) == nullptr)) { Debug("url_rewrite", "[parse_directive] Invalid argument(s)"); return "Invalid argument(s)"; } @@ -420,8 +420,6 @@ const char * remap_validate_filter_args(acl_filter_rule **rule_pp, const char **argv, int argc, char *errStrBuf, size_t errStrBufSize) { acl_filter_rule *rule; - unsigned long ul; - const char *argptr; src_ip_info_t *ipi; int i, j; bool new_rule_flg = false; @@ -450,8 +448,10 @@ remap_validate_filter_args(acl_filter_rule **rule_pp, const char **argv, int arg } for (i = 0; i < argc; i++) { + unsigned long ul; bool hasarg; + const char *argptr; if ((ul = remap_check_option((const char **)&argv[i], 1, 0, nullptr, &argptr)) == 0) { Debug("url_rewrite", "[validate_filter_args] Unknown remap option - %s", argv[i]); snprintf(errStrBuf, errStrBufSize, "Unknown option - \"%s\"", argv[i]); @@ -529,7 +529,7 @@ remap_validate_filter_args(acl_filter_rule **rule_pp, const char **argv, int arg } } - if (ul & REMAP_OPTFLG_IN_IP) { /* "dst_ip=" option */ + if (ul & REMAP_OPTFLG_IN_IP) { /* "dest_ip=" option */ if (rule->in_ip_cnt >= ACL_FILTER_MAX_IN_IP) { Debug("url_rewrite", "[validate_filter_args] Too many \"in_ip=\" filters"); snprintf(errStrBuf, errStrBufSize, "Defined more than %d \"in_ip=\" filters!", ACL_FILTER_MAX_IN_IP); @@ -714,12 +714,11 @@ remap_load_plugin(const char **argv, int argc, url_mapping *mp, char *errbuf, in { TSRemapInterface ri; struct stat stat_buf; - remap_plugin_info *pi; + RemapPluginInfo *pi; char *c, *err, tmpbuf[2048], default_path[PATH_NAME_MAX]; const char *new_argv[1024]; char *parv[1024]; - int idx = 0, retcode = 0; - int parc = 0; + int idx = 0; *plugin_found_at = 0; @@ -775,13 +774,9 @@ remap_load_plugin(const char **argv, int argc, url_mapping *mp, char *errbuf, in Debug("remap_plugin", "using path %s for plugin", c); - if (!remap_pi_list || (pi = remap_pi_list->find_by_path(c)) == nullptr) { - pi = new remap_plugin_info(c); - if (!remap_pi_list) { - remap_pi_list = pi; - } else { - remap_pi_list->add_to_list(pi); - } + if ((pi = RemapPluginInfo::find_by_path(c)) == nullptr) { + pi = new RemapPluginInfo(ts::file::path(c)); + RemapPluginInfo::add_to_list(pi); Debug("remap_plugin", "New remap plugin info created for \"%s\"", c); { @@ -789,7 +784,7 @@ remap_load_plugin(const char **argv, int argc, url_mapping *mp, char *errbuf, in REC_ReadConfigInteger(elevate_access, "proxy.config.plugin.load_elevated"); ElevateAccess access(elevate_access ? ElevateAccess::FILE_PRIVILEGE : 0); - if ((pi->dlh = dlopen(c, RTLD_NOW)) == nullptr) { + if ((pi->dl_handle = dlopen(c, RTLD_NOW)) == nullptr) { #if defined(freebsd) || defined(openbsd) err = (char *)dlerror(); #else @@ -798,30 +793,29 @@ remap_load_plugin(const char **argv, int argc, url_mapping *mp, char *errbuf, in snprintf(errbuf, errbufsize, "Can't load plugin \"%s\" - %s", c, err ? err : "Unknown dlopen() error"); return -4; } - pi->fp_tsremap_init = reinterpret_cast(dlsym(pi->dlh, TSREMAP_FUNCNAME_INIT)); - pi->fp_tsremap_config_reload = - reinterpret_cast(dlsym(pi->dlh, TSREMAP_FUNCNAME_CONFIG_RELOAD)); - pi->fp_tsremap_done = reinterpret_cast(dlsym(pi->dlh, TSREMAP_FUNCNAME_DONE)); - pi->fp_tsremap_new_instance = - reinterpret_cast(dlsym(pi->dlh, TSREMAP_FUNCNAME_NEW_INSTANCE)); - pi->fp_tsremap_delete_instance = - reinterpret_cast(dlsym(pi->dlh, TSREMAP_FUNCNAME_DELETE_INSTANCE)); - pi->fp_tsremap_do_remap = reinterpret_cast(dlsym(pi->dlh, TSREMAP_FUNCNAME_DO_REMAP)); - pi->fp_tsremap_os_response = - reinterpret_cast(dlsym(pi->dlh, TSREMAP_FUNCNAME_OS_RESPONSE)); - - if (!pi->fp_tsremap_init) { + pi->init_cb = reinterpret_cast(dlsym(pi->dl_handle, TSREMAP_FUNCNAME_INIT)); + pi->config_reload_cb = reinterpret_cast(dlsym(pi->dl_handle, TSREMAP_FUNCNAME_CONFIG_RELOAD)); + pi->done_cb = reinterpret_cast(dlsym(pi->dl_handle, TSREMAP_FUNCNAME_DONE)); + pi->new_instance_cb = + reinterpret_cast(dlsym(pi->dl_handle, TSREMAP_FUNCNAME_NEW_INSTANCE)); + pi->delete_instance_cb = + reinterpret_cast(dlsym(pi->dl_handle, TSREMAP_FUNCNAME_DELETE_INSTANCE)); + pi->do_remap_cb = reinterpret_cast(dlsym(pi->dl_handle, TSREMAP_FUNCNAME_DO_REMAP)); + pi->os_response_cb = reinterpret_cast(dlsym(pi->dl_handle, TSREMAP_FUNCNAME_OS_RESPONSE)); + + int retcode = 0; + if (!pi->init_cb) { snprintf(errbuf, errbufsize, R"(Can't find "%s" function in remap plugin "%s")", TSREMAP_FUNCNAME_INIT, c); retcode = -10; - } else if (!pi->fp_tsremap_new_instance && pi->fp_tsremap_delete_instance) { + } else if (!pi->new_instance_cb && pi->delete_instance_cb) { snprintf(errbuf, errbufsize, R"(Can't find "%s" function in remap plugin "%s" which is required if "%s" function exists)", TSREMAP_FUNCNAME_NEW_INSTANCE, c, TSREMAP_FUNCNAME_DELETE_INSTANCE); retcode = -11; - } else if (!pi->fp_tsremap_do_remap) { + } else if (!pi->do_remap_cb) { snprintf(errbuf, errbufsize, R"(Can't find "%s" function in remap plugin "%s")", TSREMAP_FUNCNAME_DO_REMAP, c); retcode = -12; - } else if (pi->fp_tsremap_new_instance && !pi->fp_tsremap_delete_instance) { + } else if (pi->new_instance_cb && !pi->delete_instance_cb) { snprintf(errbuf, errbufsize, R"(Can't find "%s" function in remap plugin "%s" which is required if "%s" function exists)", TSREMAP_FUNCNAME_DELETE_INSTANCE, c, TSREMAP_FUNCNAME_NEW_INSTANCE); @@ -831,16 +825,16 @@ remap_load_plugin(const char **argv, int argc, url_mapping *mp, char *errbuf, in if (errbuf && errbufsize > 0) { Debug("remap_plugin", "%s", errbuf); } - dlclose(pi->dlh); - pi->dlh = nullptr; + dlclose(pi->dl_handle); + pi->dl_handle = nullptr; return retcode; } memset(&ri, 0, sizeof(ri)); ri.size = sizeof(ri); ri.tsremap_version = TSREMAP_VERSION; - if (pi->fp_tsremap_init(&ri, tmpbuf, sizeof(tmpbuf) - 1) != TS_SUCCESS) { - snprintf(errbuf, errbufsize, "Failed to initialize plugin \"%s\": %s", pi->path, + if (pi->init_cb(&ri, tmpbuf, sizeof(tmpbuf) - 1) != TS_SUCCESS) { + snprintf(errbuf, errbufsize, "Failed to initialize plugin \"%s\": %s", pi->path.c_str(), tmpbuf[0] ? tmpbuf : "Unknown plugin error"); return -5; } @@ -848,7 +842,7 @@ remap_load_plugin(const char **argv, int argc, url_mapping *mp, char *errbuf, in Debug("remap_plugin", "Remap plugin \"%s\" - initialization completed", c); } - if (!pi->dlh) { + if (!pi->dl_handle) { snprintf(errbuf, errbufsize, "Can't load plugin \"%s\"", c); return -6; } @@ -857,10 +851,12 @@ remap_load_plugin(const char **argv, int argc, url_mapping *mp, char *errbuf, in snprintf(errbuf, errbufsize, "Can't load fromURL from URL class"); return -7; } + + int parc = 0; parv[parc++] = ats_strdup(err); ats_free(err); - if ((err = mp->toUrl.string_get(nullptr)) == nullptr) { + if ((err = mp->toURL.string_get(nullptr)) == nullptr) { snprintf(errbuf, errbufsize, "Can't load toURL from URL class"); return -7; } @@ -889,7 +885,7 @@ remap_load_plugin(const char **argv, int argc, url_mapping *mp, char *errbuf, in Debug("url_rewrite", "Argument %d: %s", k, argv[k]); } - Debug("url_rewrite", "Viewing parsed plugin parameters for %s: [%d]", pi->path, *plugin_found_at); + Debug("url_rewrite", "Viewing parsed plugin parameters for %s: [%d]", pi->path.c_str(), *plugin_found_at); for (int k = 0; k < parc; k++) { Debug("url_rewrite", "Argument %d: %s", k, parv[k]); } @@ -898,7 +894,7 @@ remap_load_plugin(const char **argv, int argc, url_mapping *mp, char *errbuf, in void *ih = nullptr; TSReturnCode res = TS_SUCCESS; - if (pi->fp_tsremap_new_instance) { + if (pi->new_instance_cb) { #if (!defined(kfreebsd) && defined(freebsd)) || defined(darwin) optreset = 1; #endif @@ -910,7 +906,7 @@ remap_load_plugin(const char **argv, int argc, url_mapping *mp, char *errbuf, in opterr = 0; optarg = nullptr; - res = pi->fp_tsremap_new_instance(parc, parv, &ih, tmpbuf, sizeof(tmpbuf) - 1); + res = pi->new_instance_cb(parc, parv, &ih, tmpbuf, sizeof(tmpbuf) - 1); } Debug("remap_plugin", "done creating new plugin instance"); @@ -966,7 +962,7 @@ process_regex_mapping_config(const char *from_host_lower, url_mapping *new_mappi goto lFail; } - to_host = new_mapping->toUrl.host_get(&to_host_len); + to_host = new_mapping->toURL.host_get(&to_host_len); for (int i = 0; i < (to_host_len - 1); ++i) { if (to_host[i] == '$') { if (substitution_count > UrlRewrite::MAX_REGEX_SUBS) { @@ -987,7 +983,7 @@ process_regex_mapping_config(const char *from_host_lower, url_mapping *new_mappi // so the regex itself is stored in fromURL.host; string to match // will be in the request; string to use for substitutions will be // in this buffer - str = new_mapping->toUrl.host_get(&str_index); // reusing str and str_index + str = new_mapping->toURL.host_get(&str_index); // reusing str and str_index reg_map->to_url_host_template_len = str_index; reg_map->to_url_host_template = static_cast(ats_malloc(str_index)); memcpy(reg_map->to_url_host_template, str, str_index); @@ -1156,10 +1152,9 @@ remap_parse_config_bti(const char *path, BUILD_TABLE_INFO *bti) new_mapping->map_id = 0; if ((bti->remap_optflg & REMAP_OPTFLG_MAP_ID) != 0) { int idx = 0; - char *c; int ret = remap_check_option((const char **)bti->argv, bti->argc, REMAP_OPTFLG_MAP_ID, &idx); if (ret & REMAP_OPTFLG_MAP_ID) { - c = strchr(bti->argv[idx], (int)'='); + char *c = strchr(bti->argv[idx], (int)'='); new_mapping->map_id = (unsigned int)atoi(++c); } } @@ -1191,8 +1186,8 @@ remap_parse_config_bti(const char *path, BUILD_TABLE_INFO *bti) map_to_start = map_to; tmp = map_to; - new_mapping->toUrl.create(nullptr); - rparse = new_mapping->toUrl.parse_no_path_component_breakdown(tmp, length); + new_mapping->toURL.create(nullptr); + rparse = new_mapping->toURL.parse_no_path_component_breakdown(tmp, length); map_to_start[origLength] = '\0'; // Unwhack if (rparse != PARSE_RESULT_DONE) { @@ -1208,7 +1203,7 @@ remap_parse_config_bti(const char *path, BUILD_TABLE_INFO *bti) fromScheme = new_mapping->fromURL.scheme_get(&fromSchemeLen); new_mapping->wildcard_from_scheme = true; } - toScheme = new_mapping->toUrl.scheme_get(&toSchemeLen); + toScheme = new_mapping->toURL.scheme_get(&toSchemeLen); // Include support for HTTPS scheme // includes support for FILE scheme @@ -1287,7 +1282,7 @@ remap_parse_config_bti(const char *path, BUILD_TABLE_INFO *bti) } } - toHost = new_mapping->toUrl.host_get(&toHostLen); + toHost = new_mapping->toURL.host_get(&toHostLen); if (toHost == nullptr || toHostLen <= 0) { errStr = "The remap destinations require a hostname"; goto MAP_ERROR; @@ -1346,8 +1341,8 @@ remap_parse_config_bti(const char *path, BUILD_TABLE_INFO *bti) u_mapping->fromURL.create(nullptr); u_mapping->fromURL.copy(&new_mapping->fromURL); u_mapping->fromURL.host_set(ipb, strlen(ipb)); - u_mapping->toUrl.create(nullptr); - u_mapping->toUrl.copy(&new_mapping->toUrl); + u_mapping->toURL.create(nullptr); + u_mapping->toURL.copy(&new_mapping->toURL); if (bti->paramv[3] != nullptr) { u_mapping->tag = ats_strdup(&(bti->paramv[3][0])); @@ -1426,9 +1421,7 @@ remap_parse_config(const char *path, UrlRewrite *rewrite) // If this happens to be a config reload, the list of loaded remap plugins is non-empty, and we // can signal all these plugins that a reload has begun. - if (remap_pi_list) { - remap_pi_list->indicate_reload(); - } + RemapPluginInfo::indicate_reload(); bti.rewrite = rewrite; return remap_parse_config_bti(path, &bti); } diff --git a/proxy/http/remap/RemapConfig.h b/proxy/http/remap/RemapConfig.h index e0466406e1d..8cac3d821b3 100644 --- a/proxy/http/remap/RemapConfig.h +++ b/proxy/http/remap/RemapConfig.h @@ -46,16 +46,16 @@ struct BUILD_TABLE_INFO { BUILD_TABLE_INFO(); ~BUILD_TABLE_INFO(); - unsigned long remap_optflg; - int paramc; - int argc; + unsigned long remap_optflg = 0; + int paramc = 0; + int argc = 0; char *paramv[BUILD_TABLE_MAX_ARGS]; char *argv[BUILD_TABLE_MAX_ARGS]; - bool ip_allow_check_enabled_p; - bool accept_check_p; - acl_filter_rule *rules_list; // all rules defined in config files as .define_filter foobar @src_ip=..... - UrlRewrite *rewrite; // Pointer to the UrlRewrite object we are parsing for. + bool ip_allow_check_enabled_p = true; + bool accept_check_p = true; + acl_filter_rule *rules_list = nullptr; // all rules defined in config files as .define_filter foobar @src_ip=..... + UrlRewrite *rewrite = nullptr; // Pointer to the UrlRewrite object we are parsing for. // Clear the argument vector. void reset(); diff --git a/proxy/http/remap/RemapPluginInfo.cc b/proxy/http/remap/RemapPluginInfo.cc index ad0b06c9e96..db6dfe59498 100644 --- a/proxy/http/remap/RemapPluginInfo.cc +++ b/proxy/http/remap/RemapPluginInfo.cc @@ -1,6 +1,6 @@ /** @file - A brief file description + Information about remap plugin libraries. @section license License @@ -25,102 +25,56 @@ #include "tscore/ink_string.h" #include "tscore/ink_memory.h" -remap_plugin_info::remap_plugin_info(char *_path) - : next(nullptr), - path(nullptr), - path_size(0), - dlh(nullptr), - fp_tsremap_init(nullptr), - fp_tsremap_config_reload(nullptr), - fp_tsremap_done(nullptr), - fp_tsremap_new_instance(nullptr), - fp_tsremap_delete_instance(nullptr), - fp_tsremap_do_remap(nullptr), - fp_tsremap_os_response(nullptr) -{ - // coverity did not see ats_free - // coverity[ctor_dtor_leak] - if (_path && likely((path = ats_strdup(_path)) != nullptr)) { - path_size = strlen(path); - } -} +RemapPluginInfo::List RemapPluginInfo::g_list; -remap_plugin_info::~remap_plugin_info() +RemapPluginInfo::RemapPluginInfo(ts::file::path &&library_path) : path(std::move(library_path)) {} + +RemapPluginInfo::~RemapPluginInfo() { - ats_free(path); - if (dlh) { - dlclose(dlh); + if (dl_handle) { + dlclose(dl_handle); } } // // Find a plugin by path from our linked list // -remap_plugin_info * -remap_plugin_info::find_by_path(char *_path) +RemapPluginInfo * +RemapPluginInfo::find_by_path(std::string_view library_path) { - int _path_size = 0; - remap_plugin_info *pi = nullptr; - - if (likely(_path && (_path_size = strlen(_path)) > 0)) { - for (pi = this; pi; pi = pi->next) { - if (pi->path && pi->path_size == _path_size && !strcmp(pi->path, _path)) { - break; - } - } - } - - return pi; + auto spot = std::find_if(g_list.begin(), g_list.end(), + [&](self_type const &info) -> bool { return 0 == library_path.compare(info.path.view()); }); + return spot == g_list.end() ? nullptr : static_cast(spot); } // // Add a plugin to the linked list // void -remap_plugin_info::add_to_list(remap_plugin_info *pi) +RemapPluginInfo::add_to_list(RemapPluginInfo *pi) { - remap_plugin_info *p = this; - - if (likely(pi)) { - while (p->next) { - p = p->next; - } - - p->next = pi; - pi->next = nullptr; - } + g_list.append(pi); } // -// Remove and delete all plugins from a list, including ourselves. +// Remove and delete all plugins from a list. // void -remap_plugin_info::delete_my_list() +RemapPluginInfo::delete_list() { - remap_plugin_info *p = this->next; - - while (p) { - remap_plugin_info *tmp = p; - - p = p->next; - delete tmp; - } - - delete this; + g_list.apply([](self_type *info) -> void { delete info; }); + g_list.clear(); } // // Tell all plugins (that so wish) that remap.config is being reloaded // void -remap_plugin_info::indicate_reload() +RemapPluginInfo::indicate_reload() { - remap_plugin_info *p = this; - - while (p) { - if (p->fp_tsremap_config_reload) { - p->fp_tsremap_config_reload(); + g_list.apply([](self_type *info) -> void { + if (info->config_reload_cb) { + info->config_reload_cb(); } - p = p->next; - } + }); } diff --git a/proxy/http/remap/RemapPluginInfo.h b/proxy/http/remap/RemapPluginInfo.h index a83a48500a9..7802b5c252d 100644 --- a/proxy/http/remap/RemapPluginInfo.h +++ b/proxy/http/remap/RemapPluginInfo.h @@ -1,6 +1,6 @@ /** @file - A brief file description + Information about remap plugins. @section license License @@ -23,51 +23,67 @@ #pragma once #include "tscore/ink_platform.h" - +#include "tscpp/util/IntrusiveDList.h" +#include "tscore/ts_file.h" #include "ts/apidefs.h" #include "ts/remap.h" -#define TSREMAP_FUNCNAME_INIT "TSRemapInit" -#define TSREMAP_FUNCNAME_CONFIG_RELOAD "TSRemapConfigReload" -#define TSREMAP_FUNCNAME_DONE "TSRemapDone" -#define TSREMAP_FUNCNAME_NEW_INSTANCE "TSRemapNewInstance" -#define TSREMAP_FUNCNAME_DELETE_INSTANCE "TSRemapDeleteInstance" -#define TSREMAP_FUNCNAME_DO_REMAP "TSRemapDoRemap" -#define TSREMAP_FUNCNAME_OS_RESPONSE "TSRemapOSResponse" +static constexpr const char *const TSREMAP_FUNCNAME_INIT = "TSRemapInit"; +static constexpr const char *const TSREMAP_FUNCNAME_CONFIG_RELOAD = "TSRemapConfigReload"; +static constexpr const char *const TSREMAP_FUNCNAME_DONE = "TSRemapDone"; +static constexpr const char *const TSREMAP_FUNCNAME_NEW_INSTANCE = "TSRemapNewInstance"; +static constexpr const char *const TSREMAP_FUNCNAME_DELETE_INSTANCE = "TSRemapDeleteInstance"; +static constexpr const char *const TSREMAP_FUNCNAME_DO_REMAP = "TSRemapDoRemap"; +static constexpr const char *const TSREMAP_FUNCNAME_OS_RESPONSE = "TSRemapOSResponse"; -/** - * - **/ -class remap_plugin_info +/** Information for a remap plugin. + * This stores the name of the library file and the callback entry points. + */ +class RemapPluginInfo { public: - typedef TSReturnCode _tsremap_init(TSRemapInterface *api_info, char *errbuf, int errbuf_size); - typedef void _tsremap_config_reload(); - typedef void _tsremap_done(void); - typedef TSReturnCode _tsremap_new_instance(int argc, char *argv[], void **ih, char *errbuf, int errbuf_size); - typedef void _tsremap_delete_instance(void *); - typedef TSRemapStatus _tsremap_do_remap(void *ih, TSHttpTxn rh, TSRemapRequestInfo *rri); - typedef void _tsremap_os_response(void *ih, TSHttpTxn rh, int os_response_type); + using self_type = RemapPluginInfo; ///< Self reference type. + + /// Initialization function, called on library load. + using Init_F = TSReturnCode(TSRemapInterface *api_info, char *errbuf, int errbuf_size); + /// Reload function, called to inform the plugin of a configuration reload. + using Reload_F = void(); + /// Called when remapping for a transaction has finished. + using Done_F = void(); + /// Create an rule instance. + using New_Instance_F = TSReturnCode(int argc, char *argv[], void **ih, char *errbuf, int errbuf_size); + /// Delete a rule instance. + using Delete_Instance_F = void(void *ih); + /// Perform remap. + using Do_Remap_F = TSRemapStatus(void *ih, TSHttpTxn rh, TSRemapRequestInfo *rri); + /// I have no idea what this is for. + using OS_Response_F = void(void *ih, TSHttpTxn rh, int os_response_type); + + self_type *_next = nullptr; + self_type *_prev = nullptr; + using Linkage = ts::IntrusiveLinkage; + using List = ts::IntrusiveDList; + + ts::file::path path; + void *dl_handle = nullptr; /* "handle" for the dynamic library */ + Init_F *init_cb = nullptr; + Reload_F *config_reload_cb = nullptr; + Done_F *done_cb = nullptr; + New_Instance_F *new_instance_cb = nullptr; + Delete_Instance_F *delete_instance_cb = nullptr; + Do_Remap_F *do_remap_cb = nullptr; + OS_Response_F *os_response_cb = nullptr; - remap_plugin_info *next; - char *path; - int path_size; - void *dlh; /* "handle" for the dynamic library */ - _tsremap_init *fp_tsremap_init; - _tsremap_config_reload *fp_tsremap_config_reload; - _tsremap_done *fp_tsremap_done; - _tsremap_new_instance *fp_tsremap_new_instance; - _tsremap_delete_instance *fp_tsremap_delete_instance; - _tsremap_do_remap *fp_tsremap_do_remap; - _tsremap_os_response *fp_tsremap_os_response; + explicit RemapPluginInfo(ts::file::path &&library_path); + ~RemapPluginInfo(); - remap_plugin_info(char *_path); - ~remap_plugin_info(); + static self_type *find_by_path(std::string_view library_path); + static void add_to_list(self_type *pi); + static void delete_list(); + static void indicate_reload(); - remap_plugin_info *find_by_path(char *_path); - void add_to_list(remap_plugin_info *pi); - void delete_my_list(); - void indicate_reload(); + /// Singleton list of remap plugin info instances. + static List g_list; }; /** diff --git a/proxy/http/remap/RemapPlugins.cc b/proxy/http/remap/RemapPlugins.cc index ebddec2bacd..6a664450fb8 100644 --- a/proxy/http/remap/RemapPlugins.cc +++ b/proxy/http/remap/RemapPlugins.cc @@ -26,7 +26,7 @@ ClassAllocator pluginAllocator("RemapPluginsAlloc"); TSRemapStatus -RemapPlugins::run_plugin(remap_plugin_info *plugin) +RemapPlugins::run_plugin(RemapPluginInfo *plugin) { ink_assert(_s); @@ -51,11 +51,11 @@ RemapPlugins::run_plugin(remap_plugin_info *plugin) // Prepare State for the future if (_cur == 0) { - _s->fp_tsremap_os_response = plugin->fp_tsremap_os_response; + _s->fp_tsremap_os_response = plugin->os_response_cb; _s->remap_plugin_instance = ih; } - plugin_retcode = plugin->fp_tsremap_do_remap(ih, reinterpret_cast(_s->state_machine), &rri); + plugin_retcode = plugin->do_remap_cb(ih, reinterpret_cast(_s->state_machine), &rri); // TODO: Deal with negative return codes here if (plugin_retcode < 0) { plugin_retcode = TSREMAP_NO_REMAP; @@ -78,54 +78,45 @@ RemapPlugins::run_plugin(remap_plugin_info *plugin) there actually *is* something to do). */ -int +bool RemapPlugins::run_single_remap() { url_mapping *map = _s->url_map.getMapping(); - remap_plugin_info *plugin = map->get_plugin(_cur); // get the nth plugin in our list of plugins + RemapPluginInfo *plugin = map->get_plugin(_cur); // get the nth plugin in our list of plugins TSRemapStatus plugin_retcode = TSREMAP_NO_REMAP; - + bool zret = true; // default - last iteration. Debug("url_rewrite", "running single remap rule id %d for the %d%s time", map->map_id, _cur, _cur == 1 ? "st" : _cur == 2 ? "nd" : _cur == 3 ? "rd" : "th"); + if (0 == _cur) { + Debug("url_rewrite", "setting the remapped url by copying from mapping rule"); + url_rewrite_remap_request(_s->url_map, _request_url, _s->hdr_info.client_request.method_get_wksidx()); + } + // There might not be a plugin if we are a regular non-plugin map rule. In that case, we will fall through // and do the default mapping and then stop. if (plugin) { plugin_retcode = run_plugin(plugin); } - _cur++; + ++_cur; - // If the plugin redirected, we need to end the remap chain now. - if (_s->remap_redirect) { - return 1; - } - - if (TSREMAP_NO_REMAP == plugin_retcode || TSREMAP_NO_REMAP_STOP == plugin_retcode) { - // After running the first plugin, rewrite the request URL. This is doing the default rewrite rule - // to handle the case where no plugin ever rewrites. - // - // XXX we could probably optimize this a bit more by keeping a flag and only rewriting the request URL - // if no plugin has rewritten it already. - if (_cur == 1) { - Debug("url_rewrite", "plugin did not change host, port or path, copying from mapping rule"); - url_rewrite_remap_request(_s->url_map, _request_url, _s->hdr_info.client_request.method_get_wksidx()); + // If the plugin redirected, we need to end the remap chain now. Otherwise see what's next. + if (!_s->remap_redirect) { + if (TSREMAP_DID_REMAP_STOP == plugin_retcode || TSREMAP_DID_REMAP == plugin_retcode) { + ++_rewritten; } - } - if (TSREMAP_NO_REMAP_STOP == plugin_retcode || TSREMAP_DID_REMAP_STOP == plugin_retcode) { - Debug("url_rewrite", "breaking remap plugin chain since last plugin said we should stop"); - return 1; - } - - if (_cur >= map->plugin_count()) { - // Normally, we would callback into this function but we dont have anything more to do! - Debug("url_rewrite", "completed all remap plugins for rule id %d", map->map_id); - return 1; + if (TSREMAP_NO_REMAP_STOP == plugin_retcode || TSREMAP_DID_REMAP_STOP == plugin_retcode) { + Debug("url_rewrite", "breaking remap plugin chain since last plugin said we should stop after %d rewrites", _rewritten); + } else if (_cur >= map->plugin_count()) { + Debug("url_rewrite", "completed all remap plugins for rule id %d, changed by %d plugins", map->map_id, _rewritten); + } else { + Debug("url_rewrite", "completed single remap, attempting another via immediate callback"); + zret = false; // not done yet. + } } - - Debug("url_rewrite", "completed single remap, attempting another via immediate callback"); - return 0; + return zret; } int @@ -136,8 +127,6 @@ RemapPlugins::run_remap(int event, Event *e) ink_assert(action.continuation); ink_assert(action.continuation); - int ret = 0; - /* make sure we weren't cancelled */ if (action.cancelled) { mutex.clear(); @@ -148,14 +137,14 @@ RemapPlugins::run_remap(int event, Event *e) switch (event) { case EVENT_IMMEDIATE: Debug("url_rewrite", "handling immediate event inside RemapPlugins::run_remap"); - ret = run_single_remap(); /** - * If ret !=0 then we are done with this processor and we call back into the SM; - * otherwise, we call this function again immediately (which really isn't immediate) - * thru the eventProcessor, thus forcing another run of run_single_remap() which will + * If @c run_single_remap returns @c true then we are done with this processor and we call back + * into the SM; otherwise, we call this function again immediately (which really isn't + * immediate) thru the eventProcessor, thus forcing another run of run_single_remap() which will * then operate on _request_url, etc performing additional remaps (mainly another plugin run) + * **/ - if (ret) { + if (run_single_remap()) { action.continuation->handleEvent(EVENT_REMAP_COMPLETE, nullptr); mutex.clear(); action.mutex.clear(); diff --git a/proxy/http/remap/RemapPlugins.h b/proxy/http/remap/RemapPlugins.h index bd7647710c4..421b55ca51e 100644 --- a/proxy/http/remap/RemapPlugins.h +++ b/proxy/http/remap/RemapPlugins.h @@ -38,13 +38,12 @@ * A class that represents a queue of plugins to run **/ struct RemapPlugins : public Continuation { - RemapPlugins() : _cur(0) {} + RemapPlugins() = default; RemapPlugins(HttpTransact::State *s, URL *u, HTTPHdr *h, host_hdr_info *hi) - : _cur(0), _s(s), _request_url(u), _request_header(h), _hh_ptr(hi) + : _s(s), _request_url(u), _request_header(h), _hh_ptr(hi) { } - ~RemapPlugins() override { _cur = 0; } // Some basic setters void setState(HttpTransact::State *state) @@ -68,13 +67,14 @@ struct RemapPlugins : public Continuation { } int run_remap(int event, Event *e); - int run_single_remap(); - TSRemapStatus run_plugin(remap_plugin_info *plugin); + bool run_single_remap(); + TSRemapStatus run_plugin(RemapPluginInfo *plugin); Action action; private: - unsigned int _cur = 0; + unsigned _cur = 0; + unsigned _rewritten = 0; HttpTransact::State *_s = nullptr; URL *_request_url = nullptr; HTTPHdr *_request_header = nullptr; diff --git a/proxy/http/remap/RemapProcessor.cc b/proxy/http/remap/RemapProcessor.cc index e22b710f882..05f47236e8e 100644 --- a/proxy/http/remap/RemapProcessor.cc +++ b/proxy/http/remap/RemapProcessor.cc @@ -144,11 +144,10 @@ RemapProcessor::finish_remap(HttpTransact::State *s, UrlRewrite *table) HTTPHdr *request_header = &s->hdr_info.client_request; URL *request_url = request_header->url_get(); char **redirect_url = &s->remap_redirect; - char host_hdr_buf[TS_MAX_HOST_NAME_LEN], tmp_referer_buf[4096], tmp_redirect_buf[4096], tmp_buf[2048], *c; + char tmp_referer_buf[4096], tmp_redirect_buf[4096], tmp_buf[2048]; const char *remapped_host; - int remapped_host_len, remapped_port, tmp; + int remapped_host_len, tmp; int from_len; - bool remap_found = false; referer_info *ri; map = s->url_map.getMapping(); @@ -189,8 +188,9 @@ RemapProcessor::finish_remap(HttpTransact::State *s, UrlRewrite *table) if ((s->filter_mask & URL_REMAP_FILTER_REDIRECT_FMT) != 0 && map->redir_chunk_list) { redirect_tag_str *rc; tmp_redirect_buf[(tmp = 0)] = 0; + for (rc = map->redir_chunk_list; rc; rc = rc->next) { - c = nullptr; + char *c = nullptr; switch (rc->type) { case 's': c = rc->chunk_str; @@ -233,8 +233,6 @@ RemapProcessor::finish_remap(HttpTransact::State *s, UrlRewrite *table) } } - remap_found = true; - // We also need to rewrite the "Host:" header if it exists and // pristine host hdr is not enabled int host_len; @@ -253,8 +251,10 @@ RemapProcessor::finish_remap(HttpTransact::State *s, UrlRewrite *table) // temporary buffer has adequate length // remapped_host = request_url->host_get(&remapped_host_len); - remapped_port = request_url->port_get_raw(); + int remapped_port = request_url->port_get_raw(); + + char host_hdr_buf[TS_MAX_HOST_NAME_LEN]; if (TS_MAX_HOST_NAME_LEN > remapped_host_len) { tmp = remapped_host_len; memcpy(host_hdr_buf, remapped_host, remapped_host_len); @@ -280,7 +280,7 @@ RemapProcessor::finish_remap(HttpTransact::State *s, UrlRewrite *table) request_header->mark_target_dirty(); - return remap_found; + return true; } Action * @@ -318,11 +318,9 @@ RemapProcessor::perform_remap(Continuation *cont, HttpTransact::State *s) return &plugins->action; } else { RemapPlugins plugins(s, request_url, request_header, hh_info); - int ret = 0; - do { - ret = plugins.run_single_remap(); - } while (ret == 0); + while (!plugins.run_single_remap()) + ; // EMPTY return ACTION_RESULT_DONE; } diff --git a/proxy/http/remap/RemapProcessor.h b/proxy/http/remap/RemapProcessor.h index f3665b8a50f..d5e5aced5b2 100644 --- a/proxy/http/remap/RemapProcessor.h +++ b/proxy/http/remap/RemapProcessor.h @@ -38,7 +38,7 @@ class RemapProcessor : public Processor { public: - RemapProcessor() : ET_REMAP(0), _use_separate_remap_thread(false) {} + RemapProcessor() {} ~RemapProcessor() override {} bool setup_for_remap(HttpTransact::State *s, UrlRewrite *table); bool finish_remap(HttpTransact::State *s, UrlRewrite *table); @@ -58,8 +58,8 @@ class RemapProcessor : public Processor } private: - EventType ET_REMAP; - bool _use_separate_remap_thread; + EventType ET_REMAP = 0; + bool _use_separate_remap_thread = false; }; /** diff --git a/proxy/http/remap/UrlMapping.cc b/proxy/http/remap/UrlMapping.cc index 5ff1d541772..782b3e2ac56 100644 --- a/proxy/http/remap/UrlMapping.cc +++ b/proxy/http/remap/UrlMapping.cc @@ -30,7 +30,7 @@ * **/ bool -url_mapping::add_plugin(remap_plugin_info *i, void *ih) +url_mapping::add_plugin(RemapPluginInfo *i, void *ih) { _plugin_list.push_back(i); _instance_data.push_back(ih); @@ -41,7 +41,7 @@ url_mapping::add_plugin(remap_plugin_info *i, void *ih) /** * **/ -remap_plugin_info * +RemapPluginInfo * url_mapping::get_plugin(std::size_t index) const { Debug("url_rewrite", "get_plugin says we have %zu plugins and asking for plugin %zu", plugin_count(), index); @@ -66,11 +66,11 @@ url_mapping::get_instance(std::size_t index) const void url_mapping::delete_instance(unsigned int index) { - void *ih = get_instance(index); - remap_plugin_info *p = get_plugin(index); + void *ih = get_instance(index); + RemapPluginInfo *p = get_plugin(index); - if (ih && p && p->fp_tsremap_delete_instance) { - p->fp_tsremap_delete_instance(ih); + if (ih && p && p->delete_instance_cb) { + p->delete_instance_cb(ih); } } @@ -111,7 +111,7 @@ url_mapping::~url_mapping() // Destroy the URLs fromURL.destroy(); - toUrl.destroy(); + toURL.destroy(); } void @@ -120,7 +120,7 @@ url_mapping::Print() char from_url_buf[131072], to_url_buf[131072]; fromURL.string_get_buf(from_url_buf, (int)sizeof(from_url_buf)); - toUrl.string_get_buf(to_url_buf, (int)sizeof(to_url_buf)); + toURL.string_get_buf(to_url_buf, (int)sizeof(to_url_buf)); printf("\t %s %s=> %s %s <%s> [plugins %s enabled; running with %zu plugins]\n", from_url_buf, unique ? "(unique)" : "", to_url_buf, homePageRedirect ? "(R)" : "", tag ? tag : "", plugin_count() > 0 ? "are" : "not", plugin_count()); } @@ -132,12 +132,12 @@ redirect_tag_str * redirect_tag_str::parse_format_redirect_url(char *url) { char *c; - redirect_tag_str *r, **rr; + redirect_tag_str *r; redirect_tag_str *list = nullptr; - char type = 0; if (url && *url) { - for (rr = &list; *(c = url) != 0;) { + for (redirect_tag_str **rr = &list; *(c = url) != 0;) { + char type = 0; for (type = 's'; *c; c++) { if (c[0] == '%') { char tmp_type = (char)tolower((int)c[1]); @@ -162,7 +162,6 @@ redirect_tag_str::parse_format_redirect_url(char *url) } (*rr = r)->next = nullptr; rr = &(r->next); - // printf("\t***********'%c' - '%s'*******\n",r->type,r->chunk_str ? r->chunk_str : ""); } else { break; /* memory allocation error */ } diff --git a/proxy/http/remap/UrlMapping.h b/proxy/http/remap/UrlMapping.h index 269d0143ef7..1b55e52b72f 100644 --- a/proxy/http/remap/UrlMapping.h +++ b/proxy/http/remap/UrlMapping.h @@ -55,7 +55,7 @@ class referer_info class redirect_tag_str { public: - redirect_tag_str() : next(nullptr), chunk_str(nullptr), type(0) {} + redirect_tag_str() {} ~redirect_tag_str() { type = 0; @@ -65,9 +65,9 @@ class redirect_tag_str } } - redirect_tag_str *next; - char *chunk_str; - char type; /* s - string, r - referer, t - url_to, f - url_from, o - origin url */ + redirect_tag_str *next = nullptr; + char *chunk_str = nullptr; + char type = 0; /* s - string, r - referer, t - url_to, f - url_from, o - origin url */ static redirect_tag_str *parse_format_redirect_url(char *url); }; @@ -79,8 +79,8 @@ class url_mapping public: ~url_mapping(); - bool add_plugin(remap_plugin_info *i, void *ih); - remap_plugin_info *get_plugin(std::size_t) const; + bool add_plugin(RemapPluginInfo *i, void *ih); + RemapPluginInfo *get_plugin(std::size_t) const; void *get_instance(std::size_t) const; std::size_t @@ -94,7 +94,7 @@ class url_mapping int from_path_len = 0; URL fromURL; - URL toUrl; // Default TO-URL (from remap.config) + URL toURL; // Default TO-URL (from remap.config) bool homePageRedirect = false; bool unique = false; // INKqa11970 - unique mapping bool default_redirect_url = false; @@ -122,7 +122,7 @@ class url_mapping }; private: - std::vector _plugin_list; + std::vector _plugin_list; std::vector _instance_data; int _rank = 0; }; @@ -134,8 +134,8 @@ class url_mapping class UrlMappingContainer { public: - UrlMappingContainer() : _mapping(nullptr), _toURLPtr(nullptr), _heap(nullptr) {} - explicit UrlMappingContainer(HdrHeap *heap) : _mapping(nullptr), _toURLPtr(nullptr), _heap(heap) {} + UrlMappingContainer() {} + explicit UrlMappingContainer(HdrHeap *heap) : _heap(heap) {} ~UrlMappingContainer() { deleteToURL(); } URL * getToURL() const @@ -159,7 +159,7 @@ class UrlMappingContainer { deleteToURL(); _mapping = m; - _toURLPtr = m ? &(m->toUrl) : nullptr; + _toURLPtr = m ? &(m->toURL) : nullptr; } void @@ -200,8 +200,8 @@ class UrlMappingContainer UrlMappingContainer &operator=(const UrlMappingContainer &rhs) = delete; private: - url_mapping *_mapping; - URL *_toURLPtr; + url_mapping *_mapping = nullptr; + URL *_toURLPtr = nullptr; URL _toURL; - HdrHeap *_heap; + HdrHeap *_heap = nullptr; }; diff --git a/proxy/http/remap/UrlRewrite.cc b/proxy/http/remap/UrlRewrite.cc index 3f3d0ed4cb4..6f2e83d1b95 100644 --- a/proxy/http/remap/UrlRewrite.cc +++ b/proxy/http/remap/UrlRewrite.cc @@ -1,6 +1,6 @@ /** @file - A brief file description + URL rewriting. @section license License @@ -49,20 +49,8 @@ SetHomePageRedirectFlag(url_mapping *new_mapping, URL &new_to_url) new_mapping->homePageRedirect = (from_path && !to_path) ? true : false; } -// -// CTOR / DTOR for the UrlRewrite class. -// -UrlRewrite::UrlRewrite() - : nohost_rules(0), - reverse_proxy(0), - ts_name(nullptr), - http_default_redirect_url(nullptr), - num_rules_forward(0), - num_rules_reverse(0), - num_rules_redirect_permanent(0), - num_rules_redirect_temporary(0), - num_rules_forward_with_recv_port(0), - _valid(false) +bool +UrlRewrite::load() { ats_scoped_str config_file_path; @@ -70,7 +58,7 @@ UrlRewrite::UrlRewrite() if (!config_file_path) { pmgmt->signalManager(MGMT_SIGNAL_CONFIG_ERROR, "Unable to find proxy.config.url_remap.filename"); Warning("%s Unable to locate remap.config. No remappings in effect", modulePrefix); - return; + return false; } this->ts_name = nullptr; @@ -99,6 +87,7 @@ UrlRewrite::UrlRewrite() } else { Warning("something failed during BuildTable() -- check your remap plugins!"); } + return _valid; } UrlRewrite::~UrlRewrite() @@ -144,8 +133,8 @@ UrlRewrite::SetupBackdoorMapping() mapping->fromURL.parse(from_url, sizeof(from_url) - 1); mapping->fromURL.scheme_set(URL_SCHEME_HTTP, URL_LEN_HTTP); - mapping->toUrl.create(nullptr); - mapping->toUrl.parse(to_url, sizeof(to_url) - 1); + mapping->toURL.create(nullptr); + mapping->toURL.parse(to_url, sizeof(to_url) - 1); return mapping; } @@ -155,8 +144,8 @@ void UrlRewrite::_destroyTable(std::unique_ptr &h_table) { if (h_table) { - for (auto it = h_table->begin(); it != h_table->end(); ++it) { - delete it->second; + for (auto &it : *h_table) { + delete it.second; } } } @@ -194,8 +183,8 @@ void UrlRewrite::PrintStore(MappingsStore &store) { if (store.hash_lookup) { - for (auto it = store.hash_lookup->begin(); it != store.hash_lookup->end(); ++it) { - it->second->Print(); + for (auto &it : *store.hash_lookup) { + it.second->Print(); } } @@ -599,7 +588,7 @@ UrlRewrite::InsertMapping(mapping_type maptype, url_mapping *new_mapping, RegexM success = _addToStore(forward_mappings, new_mapping, reg_map, src_host, is_cur_mapping_regex, num_rules_forward); if (success) { // @todo: is this applicable to regex mapping too? - SetHomePageRedirectFlag(new_mapping, new_mapping->toUrl); + SetHomePageRedirectFlag(new_mapping, new_mapping->toURL); } break; case REVERSE_MAP: @@ -641,7 +630,7 @@ UrlRewrite::InsertForwardMapping(mapping_type maptype, url_mapping *mapping, con case FORWARD_MAP: case FORWARD_MAP_REFERER: case FORWARD_MAP_WITH_RECV_PORT: - SetHomePageRedirectFlag(mapping, mapping->toUrl); + SetHomePageRedirectFlag(mapping, mapping->toURL); break; default: break; @@ -662,8 +651,6 @@ UrlRewrite::InsertForwardMapping(mapping_type maptype, url_mapping *mapping, con int UrlRewrite::BuildTable(const char *path) { - BUILD_TABLE_INFO bti; - ink_assert(forward_mappings.empty()); ink_assert(reverse_mappings.empty()); ink_assert(permanent_redirects.empty()); @@ -897,7 +884,8 @@ UrlRewrite::_regexMappingLookup(RegexMappingList ®ex_mappings, URL *request_u } int matches_info[MAX_REGEX_SUBS * 3]; - bool match_result = list_iter->regular_expression.exec(request_host, request_host_len, matches_info, countof(matches_info)); + bool match_result = + list_iter->regular_expression.exec(std::string_view(request_host, request_host_len), matches_info, countof(matches_info)); if (match_result == true) { Debug("url_rewrite_regex", @@ -913,7 +901,7 @@ UrlRewrite::_regexMappingLookup(RegexMappingList ®ex_mappings, URL *request_u // Expand substitutions in the host field from the stored template buf_len = _expandSubstitutions(matches_info, list_iter, request_host, buf, sizeof(buf)); URL *expanded_url = mapping_container.createNewToURL(); - expanded_url->copy(&((list_iter->url_map)->toUrl)); + expanded_url->copy(&((list_iter->url_map)->toURL)); expanded_url->host_set(buf, buf_len); Debug("url_rewrite_regex", "Expanded toURL to [%.*s]", expanded_url->length_get(), expanded_url->string_get_ref()); diff --git a/proxy/http/remap/UrlRewrite.h b/proxy/http/remap/UrlRewrite.h index dafb1d31799..89661caecae 100644 --- a/proxy/http/remap/UrlRewrite.h +++ b/proxy/http/remap/UrlRewrite.h @@ -1,6 +1,6 @@ /** @file - A brief file description + URL rewriting. @section license License @@ -57,10 +57,24 @@ class UrlRewrite : public RefCountObj { public: using URLTable = std::unordered_map; - UrlRewrite(); + UrlRewrite() = default; ~UrlRewrite() override; + /** Load the configuration. + * + * This access data in librecords to obtain the information needed for loading the configuration. + * + * @return @c true if the instance state is valid, @c false if not. + */ + bool load(); + + /** Build the internal url write tables. + * + * @param path Path to configuration file. + * @return 0 on success, non-zero error code on failure. + */ int BuildTable(const char *path); + mapping_type Remap_redirect(HTTPHdr *request_header, URL *redirect_url); bool ReverseMap(HTTPHdr *response_header); void SetReverseFlag(int flag); @@ -182,20 +196,20 @@ class UrlRewrite : public RefCountObj mapping_container); } - int nohost_rules; - int reverse_proxy; + int nohost_rules = 0; + int reverse_proxy = 0; - char *ts_name; // Used to send redirects when no host info + char *ts_name = nullptr; // Used to send redirects when no host info - char *http_default_redirect_url; // Used if redirect in "referer" filtering was not defined properly - int num_rules_forward; - int num_rules_reverse; - int num_rules_redirect_permanent; - int num_rules_redirect_temporary; - int num_rules_forward_with_recv_port; + char *http_default_redirect_url = nullptr; // Used if redirect in "referer" filtering was not defined properly + int num_rules_forward = 0; + int num_rules_reverse = 0; + int num_rules_redirect_permanent = 0; + int num_rules_redirect_temporary = 0; + int num_rules_forward_with_recv_port = 0; private: - bool _valid; + bool _valid = false; bool _mappingLookup(MappingsStore &mappings, URL *request_url, int request_port, const char *request_host, int request_host_len, UrlMappingContainer &mapping_container); diff --git a/proxy/http/test_http_client.pl b/proxy/http/test_http_client.pl index 68e6d9dde52..24a62811e3d 100644 --- a/proxy/http/test_http_client.pl +++ b/proxy/http/test_http_client.pl @@ -28,7 +28,6 @@ sub make_doc_filename($); sub make_doc_http_filename($); - ########################################################### # # global configuration parameters @@ -44,54 +43,39 @@ ########################################################### sub process_input_http_requests_file($$$) { - my ($filename, $proxy_name, $proxy_port) = @_; - my ($input, $host_name, $host_port, $request, $line); - - #open input file for read - unless (open input, "<$filename") - { - print "cannot open $filename: $!\n"; - return; - } - - while ($line = ) - { - $request .= $line; - #replace \n with \r\n - if (not $line =~ m/\r/) - { - $line =~ s/\n/\r\n/; - } - if ($line =~ m/host/i) - { - ($_, $host_name, $host_port) = split( /:/, $line, 3); - if (not $host_port) - { - $host_port = 80; - } - } - elsif (length($line) <= 2 && $line == "\n") - { - $request .= $line; - if ($proxy_name and $proxy_port) - { - $request = make_proxy_request( - $request, - $host_name, - $host_port, - $proxy_name, - $proxy_port); - spawn_http_request($proxy_name, $proxy_port, $request); - } - else - { - print $request; - spawn_http_request($host_name, $host_port, $request); - } - $request = ""; - } - } - return; + my ($filename, $proxy_name, $proxy_port) = @_; + my ($input, $host_name, $host_port, $request, $line); + + #open input file for read + unless (open input, "<$filename") { + print "cannot open $filename: $!\n"; + return; + } + + while ($line = ) { + $request .= $line; + #replace \n with \r\n + if (not $line =~ m/\r/) { + $line =~ s/\n/\r\n/; + } + if ($line =~ m/host/i) { + ($_, $host_name, $host_port) = split(/:/, $line, 3); + if (not $host_port) { + $host_port = 80; + } + } elsif (length($line) <= 2 && $line == "\n") { + $request .= $line; + if ($proxy_name and $proxy_port) { + $request = make_proxy_request($request, $host_name, $host_port, $proxy_name, $proxy_port); + spawn_http_request($proxy_name, $proxy_port, $request); + } else { + print $request; + spawn_http_request($host_name, $host_port, $request); + } + $request = ""; + } + } + return; } ########################################################### # @@ -100,21 +84,18 @@ ($$$) ########################################################### sub spawn_http_request($$$) { - my($hostname, $hostport, $request) = @_; - - my ($pid); - if (!defined ($pid = fork)) - { - print "fork failed", "\n"; - exit; - } - elsif ($pid) - { # parent - return; - } - # else, I am the child - do_http_request ($hostname, $hostport, $request); - exit; + my ($hostname, $hostport, $request) = @_; + + my ($pid); + if (!defined($pid = fork)) { + print "fork failed", "\n"; + exit; + } elsif ($pid) { # parent + return; + } + # else, I am the child + do_http_request($hostname, $hostport, $request); + exit; } ########################################################### # @@ -123,33 +104,31 @@ ($$$) ########################################################### sub spawn_http_request($$$) { - my ($hostname, $hostport, $request) = @_; - my ($line); - my ($iaddr, $paddr, $proto); - - $hostname =~ s/\s//g; - - $iaddr = inet_aton($hostname) or die "no host: $hostname", "\n"; - $paddr = sockaddr_in($hostport, $iaddr); - $proto = getprotobyname('tcp'); - - unless (socket(Host, PF_INET, SOCK_STREAM, $proto)) - { - print "socket: $!", "\n"; - exit; - } - unless (connect(Host, $paddr)) - { - print "connect: $!", "\n"; - exit; - } - syswrite Host, $request, length($request); - #process response - process_http_response($request, $Host, 1, 1); - print "request is done\n"; - close (Host); - - return; + my ($hostname, $hostport, $request) = @_; + my ($line); + my ($iaddr, $paddr, $proto); + + $hostname =~ s/\s//g; + + $iaddr = inet_aton($hostname) or die "no host: $hostname", "\n"; + $paddr = sockaddr_in($hostport, $iaddr); + $proto = getprotobyname('tcp'); + + unless (socket(Host, PF_INET, SOCK_STREAM, $proto)) { + print "socket: $!", "\n"; + exit; + } + unless (connect(Host, $paddr)) { + print "connect: $!", "\n"; + exit; + } + syswrite Host, $request, length($request); + #process response + process_http_response($request, $Host, 1, 1); + print "request is done\n"; + close(Host); + + return; } ########################################################### # @@ -165,62 +144,52 @@ ($$$) ########################################################### sub process_http_response($$$$) { - my ($request, $Host, $save_doc_flag, $save_http_flag) = @_; - my ($doc_filename, $http_filename); - - my ($doc_filename) = make_doc_filename($request); - my ($doc_http_filename) = make_doc_http_filename($request); - - print $doc_filename, ' ', $doc_http_filename, "\n"; - - my ($doc_file, $doc_http_file); - ######################## - # open files for write # - ######################## - if ($save_doc_flag) - { - unless (open doc_file, ">$doc_filename") - { - print "cannot open $doc_filename for write", "\n"; - return; - } - } - if ($save_http_flag) - { - unless (open doc_http_file, ">$doc_http_filename") - { - print "cannot open $doc_http_filename for write", "\n"; - return; - } - } - ############################## - # write http header and body # - ############################## - my ($http_header) = 1; - my ($doc_body) = 0; - my ($line); - - while ($line = ) - { - if ($http_header) - { - if ($save_http_flag) - { - print doc_http_file $line; - } - if (length($line) <= 2 && $line == "\n") - { - close doc_http_file; - $http_header = 0; - $doc_body = 1; - } - } - elsif ($save_doc_flag) - { - print doc_file $line; - } - } - return; + my ($request, $Host, $save_doc_flag, $save_http_flag) = @_; + my ($doc_filename, $http_filename); + + my ($doc_filename) = make_doc_filename($request); + my ($doc_http_filename) = make_doc_http_filename($request); + + print $doc_filename, ' ', $doc_http_filename, "\n"; + + my ($doc_file, $doc_http_file); + ######################## + # open files for write # + ######################## + if ($save_doc_flag) { + unless (open doc_file, ">$doc_filename") { + print "cannot open $doc_filename for write", "\n"; + return; + } + } + if ($save_http_flag) { + unless (open doc_http_file, ">$doc_http_filename") { + print "cannot open $doc_http_filename for write", "\n"; + return; + } + } + ############################## + # write http header and body # + ############################## + my ($http_header) = 1; + my ($doc_body) = 0; + my ($line); + + while ($line = ) { + if ($http_header) { + if ($save_http_flag) { + print doc_http_file $line; + } + if (length($line) <= 2 && $line == "\n") { + close doc_http_file; + $http_header = 0; + $doc_body = 1; + } + } elsif ($save_doc_flag) { + print doc_file $line; + } + } + return; } ########################################################### # @@ -234,21 +203,20 @@ ($$$$) ########################################################### sub make_proxy_request($$$$$) { - my ($request, $host_name, $host_port, $proxy_name, $proxy_port) = @_; - my ($proxy_request) = $request; + my ($request, $host_name, $host_port, $proxy_name, $proxy_port) = @_; + my ($proxy_request) = $request; - my ($url_prefix) = "http:\/\/$host_name\/"; - $url_prefix =~ s/\s//g; + my ($url_prefix) = "http:\/\/$host_name\/"; + $url_prefix =~ s/\s//g; - if ($host_port != 80) - { - $url_prefix .= ":$host_port\/"; - } - $url_prefix =~ s/\s//g; + if ($host_port != 80) { + $url_prefix .= ":$host_port\/"; + } + $url_prefix =~ s/\s//g; - $proxy_request =~ s/\//$url_prefix/; + $proxy_request =~ s/\//$url_prefix/; - return ($proxy_request); + return ($proxy_request); } ########################################################### # @@ -258,45 +226,44 @@ ($$$$$) ########################################################### sub make_doc_filename($) { - my ($request) = @_; - my ($doc_filename); - my ($host_name); - - ($_, $host_name) = split (/host:/i, $request, 2); - ($host_name, $_) = split (/ /, $host_name); - #replace every . with _ - $host_name =~ s/\./_/g; - - print $request, "\n"; - print $host_name, "\n"; - - ($_, $doc_filename) = split (/ /, $request, 2); - #remove scheme://host_name if this is a proxy request -# if ($doc_filename =~ m/:\/\//) -# { -# -# } -# -# @@@@@@@@ + my ($request) = @_; + my ($doc_filename); + my ($host_name); + + ($_, $host_name) = split(/host:/i, $request, 2); + ($host_name, $_) = split(/ /, $host_name); + #replace every . with _ + $host_name =~ s/\./_/g; - ($_, $doc_filename) = split (/\//, $doc_filename, 2); - $doc_filename =~ s/\//_/g; - #remove any white spaces - $doc_filename =~ s/\s//g; + print $request, "\n"; + print $host_name, "\n"; - print "doc name is: ", $doc_filename, "\n"; + ($_, $doc_filename) = split(/ /, $request, 2); + #remove scheme://host_name if this is a proxy request + # if ($doc_filename =~ m/:\/\//) + # { + # + # } + # + # @@@@@@@@ - if (length($doc_filename) <= 1) - { - $doc_filename = 'default.html'; - } + ($_, $doc_filename) = split(/\//, $doc_filename, 2); + $doc_filename =~ s/\//_/g; + #remove any white spaces + $doc_filename =~ s/\s//g; - $doc_filename = $host_name . '_' . $doc_filename; + print "doc name is: ", $doc_filename, "\n"; - #remove any white spaces - $doc_filename =~ s/\s//g; + if (length($doc_filename) <= 1) { + $doc_filename = 'default.html'; + } - return ($doc_filename); + $doc_filename = $host_name . '_' . $doc_filename; + + #remove any white spaces + $doc_filename =~ s/\s//g; + + return ($doc_filename); } ########################################################### # @@ -305,43 +272,34 @@ ($) ########################################################### sub make_doc_http_filename($) { - my ($request) = @_; - my ($doc_http_filename); + my ($request) = @_; + my ($doc_http_filename); - $doc_http_filename = make_doc_filename($request); - $doc_http_filename .= '.http'; + $doc_http_filename = make_doc_filename($request); + $doc_http_filename .= '.http'; - return ($doc_http_filename); + return ($doc_http_filename); } ########################################################### # # main entry point # ########################################################### -if ($#ARGV != 1 and $#ARGV != 3) -{ - print 'no proxy : test_http_client ', "\n"; - print 'use proxy: test_http_client '; - print ' ', "\n"; - exit; +if ($#ARGV != 1 and $#ARGV != 3) { + print 'no proxy : test_http_client ', "\n"; + print 'use proxy: test_http_client '; + print ' ', "\n"; + exit; } -if ($#ARGV == 1) -{ - my ($infile, $nusers) = @ARGV; - process_input_http_requests_file($infile, "", ""); -} -elsif ($#ARGV == 3) -{ - my ($infile, $nusers, $proxy_name, $proxy_port) = @ARGV; - process_input_http_requests_file($infile, $proxy_name, $proxy_port); +if ($#ARGV == 1) { + my ($infile, $nusers) = @ARGV; + process_input_http_requests_file($infile, "", ""); +} elsif ($#ARGV == 3) { + my ($infile, $nusers, $proxy_name, $proxy_port) = @ARGV; + process_input_http_requests_file($infile, $proxy_name, $proxy_port); } print "\n"; exit; - - - - - diff --git a/proxy/http/test_proxy.pl b/proxy/http/test_proxy.pl index 94de3752ee4..646547703e7 100644 --- a/proxy/http/test_proxy.pl +++ b/proxy/http/test_proxy.pl @@ -29,16 +29,15 @@ sub make_doc_filename($); sub make_doc_http_filename($); - ########################################################### # # global configuration parameters # ########################################################### -glob ($number_of_users) = 1; -glob ($save_http_doc) = 0; #if false (0) don't save a copy - #of the doc and header files. -glob ($method) = "GET"; #method to use in http requests +glob($number_of_users) = 1; +glob($save_http_doc) = 0; #if false (0) don't save a copy + #of the doc and header files. +glob($method) = "GET"; #method to use in http requests ########################################################### # @@ -48,13 +47,13 @@ ########################################################### sub compare_files($$$) { - my ($dfile, $pfile, $log_file) = @_; - @args = ("diff", $dfile, $pfile, ">>", $log_file); + my ($dfile, $pfile, $log_file) = @_; + @args = ("diff", $dfile, $pfile, ">>", $log_file); - #diff returns 0 if files are identical - $is_diff = system (@args); + #diff returns 0 if files are identical + $is_diff = system(@args); - return ($is_diff); + return ($is_diff); } ########################################################### # @@ -68,21 +67,18 @@ ($$$) ########################################################### sub spawn_task($$$$) { - my($hostname, $hostport, $request, $run_task) = @_; - - my ($pid); - if (!defined ($pid = fork)) - { - print "fork failed", "\n"; - exit; - } - elsif ($pid) - { # parent - return; - } - # else, I am the child - run_task ($hostname, $hostport, $request); - exit; + my ($hostname, $hostport, $request, $run_task) = @_; + + my ($pid); + if (!defined($pid = fork)) { + print "fork failed", "\n"; + exit; + } elsif ($pid) { # parent + return; + } + # else, I am the child + run_task($hostname, $hostport, $request); + exit; } ########################################################### # @@ -91,236 +87,214 @@ ($$$$) ########################################################### sub run_proxy_keep_alive { - my ($proxy_host_name, $proxy_port, -} + my ( + $proxy_host_name, $proxy_port,; + } -@@@@@@@@@@ + @@@@@@@@@@ ########################################################### -# -# subroutine: do_http_request hostname request -# + # + # subroutine: do_http_request hostname request + # ########################################################### -sub do_http_request($$$) -{ - my ($hostname, $hostport, $request) = @_; - my ($line); - my ($iaddr, $paddr, $proto); - - $hostname =~ s/\s//g; - - $iaddr = inet_aton($hostname) or die "no host: $hostname", "\n"; - $paddr = sockaddr_in($hostport, $iaddr); - $proto = getprotobyname('tcp'); - - unless (socket(Host, PF_INET, SOCK_STREAM, $proto)) - { - print "socket: $!", "\n"; - exit; - } - unless (connect(Host, $paddr)) - { - print "connect: $!", "\n"; - exit; - } - syswrite Host, $request, length($request); - #process response - process_http_response($request, $Host, 1, 1); - print "request is done\n"; - close (Host); - - return; -} + sub do_http_request($$$) + { + my ($hostname, $hostport, $request) = @_; + my ($line); + my ($iaddr, $paddr, $proto); + + $hostname =~ s/\s//g; + + $iaddr = inet_aton($hostname) or die "no host: $hostname", "\n"; + $paddr = sockaddr_in($hostport, $iaddr); + $proto = getprotobyname('tcp'); + + unless (socket(Host, PF_INET, SOCK_STREAM, $proto)) { + print "socket: $!", "\n"; + exit; + } + unless (connect(Host, $paddr)) { + print "connect: $!", "\n"; + exit; + } + syswrite Host, $request, length($request); + #process response + process_http_response($request, $Host, 1, 1); + print "request is done\n"; + close(Host); + + return; + } ########################################################### -# -# subroutine: process_http_response -# request, -# host_socket, -# save_doc_flag, -# save_http_flag -# -# options for save doc -# - save http response header in doc.http -# - save http doc in a unique file + # + # subroutine: process_http_response + # request, + # host_socket, + # save_doc_flag, + # save_http_flag + # + # options for save doc + # - save http response header in doc.http + # - save http doc in a unique file ########################################################### -sub process_http_response($$$$) -{ - my ($request, $Host, $save_doc_flag, $save_http_flag) = @_; - my ($doc_filename, $http_filename); - - my ($doc_filename) = make_doc_filename($request); - my ($doc_http_filename) = make_doc_http_filename($request); - - print $doc_filename, ' ', $doc_http_filename, "\n"; - - my ($doc_file, $doc_http_file); - ######################## - # open files for write # - ######################## - if ($save_doc_flag) - { - unless (open doc_file, ">$doc_filename") - { - print "cannot open $doc_filename for write", "\n"; - return; - } - } - if ($save_http_flag) - { - unless (open doc_http_file, ">$doc_http_filename") - { - print "cannot open $doc_http_filename for write", "\n"; - return; - } - } - ############################## - # write http header and body # - ############################## - my ($http_header) = 1; - my ($doc_body) = 0; - my ($line); - - while ($line = ) - { - if ($http_header) - { - if ($save_http_flag) - { - print doc_http_file $line; - } - if (length($line) <= 2 && $line == "\n") - { - close doc_http_file; - $http_header = 0; - $doc_body = 1; - } - } - elsif ($save_doc_flag) - { - print doc_file $line; - } - } - return; -} + sub process_http_response($$$$) + { + my ($request, $Host, $save_doc_flag, $save_http_flag) = @_; + my ($doc_filename, $http_filename); + + my ($doc_filename) = make_doc_filename($request); + my ($doc_http_filename) = make_doc_http_filename($request); + + print $doc_filename, ' ', $doc_http_filename, "\n"; + + my ($doc_file, $doc_http_file); + ######################## + # open files for write # + ######################## + if ($save_doc_flag) { + unless (open doc_file, ">$doc_filename") { + print "cannot open $doc_filename for write", "\n"; + return; + } + } + if ($save_http_flag) { + unless (open doc_http_file, ">$doc_http_filename") { + print "cannot open $doc_http_filename for write", "\n"; + return; + } + } + ############################## + # write http header and body # + ############################## + my ($http_header) = 1; + my ($doc_body) = 0; + my ($line); + + while ($line = ) { + if ($http_header) { + if ($save_http_flag) { + print doc_http_file $line; + } + if (length($line) <= 2 && $line == "\n") { + close doc_http_file; + $http_header = 0; + $doc_body = 1; + } + } elsif ($save_doc_flag) { + print doc_file $line; + } + } + return; + } ########################################################### -# -# subroutine: make_proxy_request -# request -# host_name -# host_port -# proxy_name -# proxy_port -# + # + # subroutine: make_proxy_request + # request + # host_name + # host_port + # proxy_name + # proxy_port + # ########################################################### -sub make_proxy_request($$$$$) -{ - my ($request, $host_name, $host_port, $proxy_name, $proxy_port) = @_; - my ($proxy_request) = $request; + sub make_proxy_request($$$$$) + { + my ($request, $host_name, $host_port, $proxy_name, $proxy_port) = @_; + my ($proxy_request) = $request; - my ($url_prefix) = "http:\/\/$host_name\/"; - $url_prefix =~ s/\s//g; + my ($url_prefix) = "http:\/\/$host_name\/"; + $url_prefix =~ s/\s//g; - if ($host_port != 80) - { - $url_prefix .= ":$host_port\/"; - } - $url_prefix =~ s/\s//g; + if ($host_port != 80) { + $url_prefix .= ":$host_port\/"; + } + $url_prefix =~ s/\s//g; - $proxy_request =~ s/\//$url_prefix/; + $proxy_request =~ s/\//$url_prefix/; - return ($proxy_request); -} + return ($proxy_request); + } ########################################################### -# -# subroutine: make_doc_filename request -# -# file name is: + # + # subroutine: make_doc_filename request + # + # file name is: ########################################################### -sub make_doc_filename($) -{ - my ($request) = @_; - my ($doc_filename); - my ($host_name); - - ($_, $host_name) = split (/host:/i, $request, 2); - ($host_name, $_) = split (/ /, $host_name); - #replace every . with _ - $host_name =~ s/\./_/g; - - print $request, "\n"; - print $host_name, "\n"; - - ($_, $doc_filename) = split (/ /, $request, 2); - #remove scheme://host_name if this is a proxy request -# if ($doc_filename =~ m/:\/\//) -# { -# -# } -# -# @@@@@@@@ - - ($_, $doc_filename) = split (/\//, $doc_filename, 2); - $doc_filename =~ s/\//_/g; - #remove any white spaces - $doc_filename =~ s/\s//g; - - print "doc name is: ", $doc_filename, "\n"; - - if (length($doc_filename) <= 1) - { - $doc_filename = 'default.html'; - } - - $doc_filename = $host_name . '_' . $doc_filename; - - #remove any white spaces - $doc_filename =~ s/\s//g; - - return ($doc_filename); -} + sub make_doc_filename($) + { + my ($request) = @_; + my ($doc_filename); + my ($host_name); + + ($_, $host_name) = split(/host:/i, $request, 2); + ($host_name, $_) = split(/ /, $host_name); + #replace every . with _ + $host_name =~ s/\./_/g; + + print $request, "\n"; + print $host_name, "\n"; + + ($_, $doc_filename) = split(/ /, $request, 2); + #remove scheme://host_name if this is a proxy request + # if ($doc_filename =~ m/:\/\//) + # { + # + # } + # + # @@@@@@@@ + + ($_, $doc_filename) = split(/\//, $doc_filename, 2); + $doc_filename =~ s/\//_/g; + #remove any white spaces + $doc_filename =~ s/\s//g; + + print "doc name is: ", $doc_filename, "\n"; + + if (length($doc_filename) <= 1) { + $doc_filename = 'default.html'; + } + + $doc_filename = $host_name . '_' . $doc_filename; + + #remove any white spaces + $doc_filename =~ s/\s//g; + + return ($doc_filename); + } ########################################################### -# -# subroutine: make_doc_filename request -# + # + # subroutine: make_doc_filename request + # ########################################################### -sub make_doc_http_filename($) -{ - my ($request) = @_; - my ($doc_http_filename); + sub make_doc_http_filename($) + { + my ($request) = @_; + my ($doc_http_filename); - $doc_http_filename = make_doc_filename($request); - $doc_http_filename .= '.http'; + $doc_http_filename = make_doc_filename($request); + $doc_http_filename .= '.http'; - return ($doc_http_filename); -} + return ($doc_http_filename); + } ########################################################### -# -# main entry point -# + # + # main entry point + # ########################################################### -if ($#ARGV != 1 and $#ARGV != 3) -{ - print 'no proxy : test_http_client ', "\n"; - print 'use proxy: test_http_client '; - print ' ', "\n"; - exit; -} - -if ($#ARGV == 1) -{ - my ($infile, $nusers) = @ARGV; - process_input_http_requests_file($infile, "", ""); -} -elsif ($#ARGV == 3) -{ - my ($infile, $nusers, $proxy_name, $proxy_port) = @ARGV; - process_input_http_requests_file($infile, $proxy_name, $proxy_port); -} - -print "\n"; -exit; - - - - - + if ($#ARGV != 1 and $#ARGV != 3) { + print 'no proxy : test_http_client ', "\n"; + print 'use proxy: test_http_client '; + print ' ', "\n"; + exit; + } + + if ($#ARGV == 1) { + my ($infile, $nusers) = @ARGV; + process_input_http_requests_file($infile, "", ""); + } elsif ($#ARGV == 3) { + my ($infile, $nusers, $proxy_name, $proxy_port) = @ARGV; + process_input_http_requests_file($infile, $proxy_name, $proxy_port); + } + + print "\n"; + exit; diff --git a/proxy/http/test_socket_close.cc b/proxy/http/test_socket_close.cc index 4be9c58a121..46001403c06 100644 --- a/proxy/http/test_socket_close.cc +++ b/proxy/http/test_socket_close.cc @@ -93,7 +93,7 @@ struct State { int64_t nbytes_write; // number of bytes to write intte_t nbytes_read; // number of bytes to read - State() : state(STATE_IDLE), tasks_count(0) {} + State() : state(STATE_IDLE), tasks_count(0), nbytes_write(0), {} }; struct Conn { @@ -241,7 +241,6 @@ state_act(Conn *c) void state_act_task(Conn *c) { - int error; char write_ch = 'T', read_ch; int r; @@ -332,7 +331,6 @@ int do_connect(Conn *from, Conn *to) { assert(to->listen_s > 0); - int error; // create a non-blocking socket if ((from->s = create_nonblocking_socket()) < 0) { @@ -341,7 +339,7 @@ do_connect(Conn *from, Conn *to) } // connect if (connect(from->s, (struct sockaddr *)&to->addr, sizeof(to->addr)) < 0) { - error = -errno; + int error = -errno; if (error != -EINPROGRESS) { ::close(from->s); from->state.state = STATE_ERROR; @@ -463,11 +461,8 @@ create_nonblocking_socket() int set_nonblocking_socket(int s) { - int error = 0; - int on = 1; - if (fcntl(s, F_SETFL, O_NDELAY) < 0) { - error = -errno; + int error = -errno; ::close(s); cout << "fcntl F_SETFD O_NDELAY failed (" << error << ")" << endl; return (error); @@ -484,7 +479,7 @@ set_nonblocking_socket(int s) int do_shutdown(int s, Task_t task) { - int howto, error; + int howto; switch (task) { case TASK_SHUTDOWN_OUTPUT: @@ -503,7 +498,7 @@ do_shutdown(int s, Task_t task) break; } if (shutdown(s, howto) < 0) { - error = -errno; + int error = -errno; cout << "shutdown failed (" << error << ")" << endl; return (error); } diff --git a/proxy/http/unit_tests/test_ForwardedConfig.cc b/proxy/http/unit_tests/test_ForwardedConfig.cc index 72b21d6d9ef..02f2ff7b620 100644 --- a/proxy/http/unit_tests/test_ForwardedConfig.cc +++ b/proxy/http/unit_tests/test_ForwardedConfig.cc @@ -36,7 +36,7 @@ using namespace HttpForwarded; class OptionBitSetListInit : public OptionBitSet { public: - OptionBitSetListInit(std::initializer_list il) + explicit OptionBitSetListInit(std::initializer_list il) { for (std::size_t i : il) { this->set(i); @@ -68,7 +68,7 @@ class XS std::string s; public: - XS(const char *in) : s{nextWs()} + explicit XS(const char *in) : s{nextWs()} { bool upper{true}; for (; *in; ++in) { @@ -92,7 +92,7 @@ class XS }; void -test(const char *spec, const char *reqErr, OptionBitSet bS) +test(const char *spec, const char *reqErr, const OptionBitSet &bS) { ts::LocalBufferWriter<1024> error; diff --git a/proxy/http/unit_tests/test_error_page_selection.cc b/proxy/http/unit_tests/test_error_page_selection.cc index 483c8e2c350..004aff537bf 100644 --- a/proxy/http/unit_tests/test_error_page_selection.cc +++ b/proxy/http/unit_tests/test_error_page_selection.cc @@ -74,16 +74,16 @@ TEST_CASE("error page selection test", "[http]") int nsets = sizeof(sets) / sizeof(sets[0]); int ntests = sizeof(tests) / sizeof(tests[0]); // (1) build fake hash table of sets - RawHashTable *table_of_sets = new RawHashTable(RawHashTable_KeyType_String); + std::unique_ptr table_of_sets; + table_of_sets.reset(new HttpBodyFactory::BodySetTable); for (i = 0; i < nsets; i++) { - HttpBodySetRawData *body_set; - body_set = (HttpBodySetRawData *)ats_malloc(sizeof(HttpBodySetRawData)); - body_set->magic = 0; - body_set->set_name = (char *)(sets[i].set_name); - body_set->content_language = (char *)(sets[i].content_language); - body_set->content_charset = (char *)(sets[i].content_charset); - body_set->table_of_pages = (RawHashTable *)1; // hack --- can't be NULL - table_of_sets->setValue((RawHashTable_Key)(body_set->set_name), (RawHashTable_Value)body_set); + HttpBodySetRawData *body_set = new HttpBodySetRawData; + body_set->magic = 0; + body_set->set_name = strdup(sets[i].set_name); + body_set->content_language = strdup(sets[i].content_language); + body_set->content_charset = strdup(sets[i].content_charset); + body_set->table_of_pages.reset(new HttpBodySetRawData::TemplateTable); + table_of_sets->emplace(body_set->set_name, body_set); } // (2) for each test, parse accept headers into lists, and test matching for (i = 0; i < ntests; i++) { @@ -104,5 +104,12 @@ TEST_CASE("error page selection test", "[http]") REQUIRE(La_best == tests[i].expected_La); REQUIRE(I_best == tests[i].expected_I); } - delete table_of_sets; + for (const auto &it : *table_of_sets.get()) { + ats_free(it.second->set_name); + ats_free(it.second->content_language); + ats_free(it.second->content_charset); + it.second->table_of_pages.reset(nullptr); + delete it.second; + } + table_of_sets.reset(nullptr); } diff --git a/proxy/http2/HPACK.cc b/proxy/http2/HPACK.cc index 1c45d1e178e..5abe3de420c 100644 --- a/proxy/http2/HPACK.cc +++ b/proxy/http2/HPACK.cc @@ -385,7 +385,7 @@ bool HpackDynamicTable::update_maximum_size(uint32_t new_size) { while (_current_size > new_size) { - if (_headers.size() <= 0) { + if (_headers.size() == 0) { return false; } int last_name_len, last_value_len; @@ -450,7 +450,7 @@ encode_string(uint8_t *buf_start, const uint8_t *buf_end, const char *value, siz int64_t data_len = 0; // TODO Choose whether to use Huffman encoding wisely - + // cppcheck-suppress knownConditionTrueFalse; leaving "use_huffman" for wise huffman usage in the future if (use_huffman && value_len) { data = static_cast(ats_malloc(value_len * 4)); if (data == nullptr) { diff --git a/proxy/http2/HPACK.h b/proxy/http2/HPACK.h index 9d0f9f56295..78efb296b26 100644 --- a/proxy/http2/HPACK.h +++ b/proxy/http2/HPACK.h @@ -55,10 +55,10 @@ enum class HpackMatch { // Result of looking for a header field in IndexingTable struct HpackLookupResult { - HpackLookupResult() : index(0), index_type(HpackIndex::NONE), match_type(HpackMatch::NONE) {} - int index; - HpackIndex index_type; - HpackMatch match_type; + HpackLookupResult() {} + int index = 0; + HpackIndex index_type = HpackIndex::NONE; + HpackMatch match_type = HpackMatch::NONE; }; class MIMEFieldWrapper @@ -105,7 +105,7 @@ class MIMEFieldWrapper class HpackDynamicTable { public: - HpackDynamicTable(uint32_t size) : _current_size(0), _maximum_size(size) + explicit HpackDynamicTable(uint32_t size) : _current_size(0), _maximum_size(size) { _mhdr = new MIMEHdr(); _mhdr->create(); @@ -119,6 +119,10 @@ class HpackDynamicTable delete _mhdr; } + // noncopyable + HpackDynamicTable(HpackDynamicTable &) = delete; + HpackDynamicTable &operator=(const HpackDynamicTable &) = delete; + const MIMEField *get_header_field(uint32_t index) const; void add_header_field(const MIMEField *field); @@ -140,8 +144,13 @@ class HpackDynamicTable class HpackIndexingTable { public: - HpackIndexingTable(uint32_t size) { _dynamic_table = new HpackDynamicTable(size); } + explicit HpackIndexingTable(uint32_t size) { _dynamic_table = new HpackDynamicTable(size); } ~HpackIndexingTable() { delete _dynamic_table; } + + // noncopyable + HpackIndexingTable(HpackIndexingTable &) = delete; + HpackIndexingTable &operator=(const HpackIndexingTable &) = delete; + HpackLookupResult lookup(const MIMEFieldWrapper &field) const; HpackLookupResult lookup(const char *name, int name_len, const char *value, int value_len) const; int get_header_field(uint32_t index, MIMEFieldWrapper &header_field) const; diff --git a/proxy/http2/HTTP2.cc b/proxy/http2/HTTP2.cc index 1ce0a9dc824..8268cd5dd2e 100644 --- a/proxy/http2/HTTP2.cc +++ b/proxy/http2/HTTP2.cc @@ -62,6 +62,7 @@ static const char *const HTTP2_STAT_SESSION_DIE_ACTIVE_NAME = "pro static const char *const HTTP2_STAT_SESSION_DIE_INACTIVE_NAME = "proxy.process.http2.session_die_inactive"; static const char *const HTTP2_STAT_SESSION_DIE_EOS_NAME = "proxy.process.http2.session_die_eos"; static const char *const HTTP2_STAT_SESSION_DIE_ERROR_NAME = "proxy.process.http2.session_die_error"; +static const char *const HTTP2_STAT_SESSION_DIE_HIGH_ERROR_RATE_NAME = "proxy.process.http2.session_die_high_error_rate"; union byte_pointer { byte_pointer(void *p) : ptr(p) {} @@ -88,6 +89,7 @@ write_and_advance(byte_pointer &dst, uint32_t src) { byte_addressable_value pval; + // cppcheck-suppress unreadVariable ; it's an union and be read as pval.bytes pval.value = htonl(src); memcpy(dst.u8, pval.bytes, sizeof(pval.bytes)); dst.u8 += sizeof(pval.bytes); @@ -98,6 +100,7 @@ write_and_advance(byte_pointer &dst, uint16_t src) { byte_addressable_value pval; + // cppcheck-suppress unreadVariable ; it's an union and be read as pval.bytes pval.value = htons(src); memcpy(dst.u8, pval.bytes, sizeof(pval.bytes)); dst.u8 += sizeof(pval.bytes); @@ -217,6 +220,7 @@ http2_write_frame_header(const Http2FrameHeader &hdr, IOVec iov) } byte_addressable_value length; + // cppcheck-suppress unreadVariable ; it's an union and be read as pval.bytes length.value = htonl(hdr.length); // MSB length.bytes[0] is unused. write_and_advance(ptr, length.bytes[1]); @@ -416,8 +420,8 @@ http2_convert_header_from_2_to_1_1(HTTPHdr *headers) ink_assert(http_hdr_type_get(headers->m_http) != HTTP_TYPE_UNKNOWN); if (http_hdr_type_get(headers->m_http) == HTTP_TYPE_REQUEST) { - const char *scheme, *authority, *path, *method; - int scheme_len, authority_len, path_len, method_len; + const char *scheme, *authority, *path; + int scheme_len, authority_len, path_len; // Get values of :scheme, :authority and :path to assemble requested URL if ((field = headers->field_find(HTTP2_VALUE_SCHEME, HTTP2_LEN_SCHEME)) != nullptr && field->value_is_valid()) { @@ -453,7 +457,8 @@ http2_convert_header_from_2_to_1_1(HTTPHdr *headers) // Get value of :method if ((field = headers->field_find(HTTP2_VALUE_METHOD, HTTP2_LEN_METHOD)) != nullptr && field->value_is_valid()) { - method = field->value_get(&method_len); + int method_len; + const char *method = field->value_get(&method_len); int method_wks_idx = hdrtoken_tokenize(method, method_len); http_hdr_method_set(headers->m_heap, headers->m_http, method, method_wks_idx, method_len, false); @@ -477,11 +482,9 @@ http2_convert_header_from_2_to_1_1(HTTPHdr *headers) headers->field_delete(HTTP2_VALUE_AUTHORITY, HTTP2_LEN_AUTHORITY); headers->field_delete(HTTP2_VALUE_PATH, HTTP2_LEN_PATH); } else { - int status_len; - const char *status; - if ((field = headers->field_find(HTTP2_VALUE_STATUS, HTTP2_LEN_STATUS)) != nullptr) { - status = field->value_get(&status_len); + int status_len; + const char *status = field->value_get(&status_len); headers->status_set(http_parse_status(status, status + status_len)); } else { return PARSE_RESULT_ERROR; @@ -493,8 +496,8 @@ http2_convert_header_from_2_to_1_1(HTTPHdr *headers) // Check validity of all names and values MIMEFieldIter iter; - for (const MIMEField *field = headers->iter_get_first(&iter); field != nullptr; field = headers->iter_get_next(&iter)) { - if (!field->name_is_valid() || !field->value_is_valid()) { + for (auto *mf = headers->iter_get_first(&iter); mf != nullptr; mf = headers->iter_get_next(&iter)) { + if (!mf->name_is_valid() || !mf->value_is_valid()) { return PARSE_RESULT_ERROR; } } @@ -614,7 +617,7 @@ http2_decode_header_blocks(HTTPHdr *hdr, const uint8_t *buf_start, const uint32_ const char *value; int len; bool is_trailing_header = trailing_header; - int64_t result = hpack_decode_header_block(handle, hdr, buf_start, buf_len, Http2::max_request_header_size, maximum_table_size); + int64_t result = hpack_decode_header_block(handle, hdr, buf_start, buf_len, Http2::max_header_list_size, maximum_table_size); if (result < 0) { if (result == HPACK_ERROR_COMPRESSION_ERROR) { @@ -726,12 +729,12 @@ uint32_t Http2::initial_window_size = 1048576; uint32_t Http2::max_frame_size = 16384; uint32_t Http2::header_table_size = 4096; uint32_t Http2::max_header_list_size = 4294967295; -uint32_t Http2::max_request_header_size = 131072; uint32_t Http2::accept_no_activity_timeout = 120; uint32_t Http2::no_activity_timeout_in = 120; uint32_t Http2::active_timeout_in = 0; uint32_t Http2::push_diary_size = 256; uint32_t Http2::zombie_timeout_in = 0; +float Http2::stream_error_rate_threshold = 0.1; void Http2::init() @@ -744,12 +747,12 @@ Http2::init() REC_EstablishStaticConfigInt32U(max_frame_size, "proxy.config.http2.max_frame_size"); REC_EstablishStaticConfigInt32U(header_table_size, "proxy.config.http2.header_table_size"); REC_EstablishStaticConfigInt32U(max_header_list_size, "proxy.config.http2.max_header_list_size"); - REC_EstablishStaticConfigInt32U(max_request_header_size, "proxy.config.http.request_header_max_size"); REC_EstablishStaticConfigInt32U(accept_no_activity_timeout, "proxy.config.http2.accept_no_activity_timeout"); REC_EstablishStaticConfigInt32U(no_activity_timeout_in, "proxy.config.http2.no_activity_timeout_in"); REC_EstablishStaticConfigInt32U(active_timeout_in, "proxy.config.http2.active_timeout_in"); REC_EstablishStaticConfigInt32U(push_diary_size, "proxy.config.http2.push_diary_size"); REC_EstablishStaticConfigInt32U(zombie_timeout_in, "proxy.config.http2.zombie_debug_timeout_in"); + REC_EstablishStaticConfigFloat(stream_error_rate_threshold, "proxy.config.http2.stream_error_rate_threshold"); // If any settings is broken, ATS should not start ink_release_assert(http2_settings_parameter_is_valid({HTTP2_SETTINGS_MAX_CONCURRENT_STREAMS, max_concurrent_streams_in})); @@ -798,6 +801,8 @@ Http2::init() static_cast(HTTP2_STAT_SESSION_DIE_INACTIVE), RecRawStatSyncSum); RecRegisterRawStat(http2_rsb, RECT_PROCESS, HTTP2_STAT_SESSION_DIE_ERROR_NAME, RECD_INT, RECP_PERSISTENT, static_cast(HTTP2_STAT_SESSION_DIE_ERROR), RecRawStatSyncSum); + RecRegisterRawStat(http2_rsb, RECT_PROCESS, HTTP2_STAT_SESSION_DIE_HIGH_ERROR_RATE_NAME, RECD_INT, RECP_PERSISTENT, + static_cast(HTTP2_STAT_SESSION_DIE_HIGH_ERROR_RATE), RecRawStatSyncSum); } #if TS_HAS_TESTS diff --git a/proxy/http2/HTTP2.h b/proxy/http2/HTTP2.h index dcf895bf89c..e705d74705a 100644 --- a/proxy/http2/HTTP2.h +++ b/proxy/http2/HTTP2.h @@ -83,6 +83,7 @@ enum { HTTP2_STAT_SESSION_DIE_INACTIVE, HTTP2_STAT_SESSION_DIE_EOS, HTTP2_STAT_SESSION_DIE_ERROR, + HTTP2_STAT_SESSION_DIE_HIGH_ERROR_RATE, HTTP2_N_STATS // Terminal counter, NOT A STAT INDEX. }; @@ -258,28 +259,25 @@ struct Http2SettingsParameter { // [RFC 7540] 6.3 PRIORITY Format struct Http2Priority { - Http2Priority() - : exclusive_flag(false), weight(HTTP2_PRIORITY_DEFAULT_WEIGHT), stream_dependency(HTTP2_PRIORITY_DEFAULT_STREAM_DEPENDENCY) - { - } + Http2Priority() : weight(HTTP2_PRIORITY_DEFAULT_WEIGHT), stream_dependency(HTTP2_PRIORITY_DEFAULT_STREAM_DEPENDENCY) {} - bool exclusive_flag; + bool exclusive_flag = false; uint8_t weight; uint32_t stream_dependency; }; // [RFC 7540] 6.2 HEADERS Format struct Http2HeadersParameter { - Http2HeadersParameter() : pad_length(0) {} - uint8_t pad_length; + Http2HeadersParameter() {} + uint8_t pad_length = 0; Http2Priority priority; }; // [RFC 7540] 6.8 GOAWAY Format struct Http2Goaway { - Http2Goaway() : last_streamid(0), error_code(Http2ErrorCode::HTTP2_ERROR_NO_ERROR) {} - Http2StreamId last_streamid; - Http2ErrorCode error_code; + Http2Goaway() {} + Http2StreamId last_streamid = 0; + Http2ErrorCode error_code = Http2ErrorCode::HTTP2_ERROR_NO_ERROR; // NOTE: we don't (de)serialize the variable length debug data at this layer // because there's @@ -295,9 +293,9 @@ struct Http2RstStream { // [RFC 7540] 6.6 PUSH_PROMISE Format struct Http2PushPromise { - Http2PushPromise() : pad_length(0), promised_streamid(0) {} - uint8_t pad_length; - Http2StreamId promised_streamid; + Http2PushPromise() {} + uint8_t pad_length = 0; + Http2StreamId promised_streamid = 0; }; static inline bool @@ -372,12 +370,12 @@ class Http2 static uint32_t max_frame_size; static uint32_t header_table_size; static uint32_t max_header_list_size; - static uint32_t max_request_header_size; static uint32_t accept_no_activity_timeout; static uint32_t no_activity_timeout_in; static uint32_t active_timeout_in; static uint32_t push_diary_size; static uint32_t zombie_timeout_in; + static float stream_error_rate_threshold; static void init(); }; diff --git a/proxy/http2/Http2ClientSession.cc b/proxy/http2/Http2ClientSession.cc index ed830a9d44d..35d60acb660 100644 --- a/proxy/http2/Http2ClientSession.cc +++ b/proxy/http2/Http2ClientSession.cc @@ -25,8 +25,14 @@ #include "HttpDebugNames.h" #include "tscore/ink_base64.h" +#define REMEMBER(e, r) \ + { \ + this->remember(MakeSourceLocation(), e, r); \ + } + #define STATE_ENTER(state_name, event) \ do { \ + REMEMBER(event, this->recursion) \ SsnDebug(this, "http2_cs", "[%" PRId64 "] [%s, %s]", this->connection_id(), #state_name, \ HttpDebugNames::get_event_name(event)); \ } while (0) @@ -35,6 +41,7 @@ #define HTTP2_SET_SESSION_HANDLER(handler) \ do { \ + REMEMBER(NO_EVENT, this->recursion); \ this->session_handler = (handler); \ } while (0) @@ -65,6 +72,7 @@ Http2ClientSession::destroy() { if (!in_destroy) { in_destroy = true; + REMEMBER(NO_EVENT, this->recursion) Http2SsnDebug("session destroy"); // Let everyone know we are going down do_api_callout(TS_HTTP_SSN_CLOSE_HOOK); @@ -87,6 +95,7 @@ Http2ClientSession::free() return; } + REMEMBER(NO_EVENT, this->recursion) Http2SsnDebug("session free"); HTTP2_DECREMENT_THREAD_DYN_STAT(HTTP2_STAT_CURRENT_CLIENT_SESSION_COUNT, this->mutex->thread_holding); @@ -94,25 +103,37 @@ Http2ClientSession::free() // Update stats on how we died. May want to eliminate this. Was useful for // tracking down which cases we were having problems cleaning up. But for general // use probably not worth the effort - switch (dying_event) { - case VC_EVENT_NONE: - HTTP2_INCREMENT_THREAD_DYN_STAT(HTTP2_STAT_SESSION_DIE_DEFAULT, this_ethread()); - break; - case VC_EVENT_ACTIVE_TIMEOUT: - HTTP2_INCREMENT_THREAD_DYN_STAT(HTTP2_STAT_SESSION_DIE_ACTIVE, this_ethread()); - break; - case VC_EVENT_INACTIVITY_TIMEOUT: - HTTP2_INCREMENT_THREAD_DYN_STAT(HTTP2_STAT_SESSION_DIE_INACTIVE, this_ethread()); - break; - case VC_EVENT_ERROR: - HTTP2_INCREMENT_THREAD_DYN_STAT(HTTP2_STAT_SESSION_DIE_ERROR, this_ethread()); - break; - case VC_EVENT_EOS: - HTTP2_INCREMENT_THREAD_DYN_STAT(HTTP2_STAT_SESSION_DIE_EOS, this_ethread()); - break; - default: - HTTP2_INCREMENT_THREAD_DYN_STAT(HTTP2_STAT_SESSION_DIE_OTHER, this_ethread()); - break; + if (cause_of_death != Http2SessionCod::NOT_PROVIDED) { + switch (cause_of_death) { + case Http2SessionCod::HIGH_ERROR_RATE: + HTTP2_INCREMENT_THREAD_DYN_STAT(HTTP2_STAT_SESSION_DIE_HIGH_ERROR_RATE, this_ethread()); + break; + case Http2SessionCod::NOT_PROVIDED: + // Can't happen but this case is here to not have default case. + HTTP2_INCREMENT_THREAD_DYN_STAT(HTTP2_STAT_SESSION_DIE_OTHER, this_ethread()); + break; + } + } else { + switch (dying_event) { + case VC_EVENT_NONE: + HTTP2_INCREMENT_THREAD_DYN_STAT(HTTP2_STAT_SESSION_DIE_DEFAULT, this_ethread()); + break; + case VC_EVENT_ACTIVE_TIMEOUT: + HTTP2_INCREMENT_THREAD_DYN_STAT(HTTP2_STAT_SESSION_DIE_ACTIVE, this_ethread()); + break; + case VC_EVENT_INACTIVITY_TIMEOUT: + HTTP2_INCREMENT_THREAD_DYN_STAT(HTTP2_STAT_SESSION_DIE_INACTIVE, this_ethread()); + break; + case VC_EVENT_ERROR: + HTTP2_INCREMENT_THREAD_DYN_STAT(HTTP2_STAT_SESSION_DIE_ERROR, this_ethread()); + break; + case VC_EVENT_EOS: + HTTP2_INCREMENT_THREAD_DYN_STAT(HTTP2_STAT_SESSION_DIE_EOS, this_ethread()); + break; + default: + HTTP2_INCREMENT_THREAD_DYN_STAT(HTTP2_STAT_SESSION_DIE_OTHER, this_ethread()); + break; + } } ink_release_assert(this->client_vc == nullptr); @@ -151,15 +172,12 @@ Http2ClientSession::start() } void -Http2ClientSession::new_connection(NetVConnection *new_vc, MIOBuffer *iobuf, IOBufferReader *reader, bool backdoor) +Http2ClientSession::new_connection(NetVConnection *new_vc, MIOBuffer *iobuf, IOBufferReader *reader) { ink_assert(new_vc->mutex->thread_holding == this_ethread()); HTTP2_INCREMENT_THREAD_DYN_STAT(HTTP2_STAT_CURRENT_CLIENT_SESSION_COUNT, new_vc->mutex->thread_holding); HTTP2_INCREMENT_THREAD_DYN_STAT(HTTP2_STAT_TOTAL_CLIENT_CONNECTION_COUNT, new_vc->mutex->thread_holding); - // HTTP/2 for the backdoor connections? Let's not deal woth that yet. - ink_release_assert(backdoor == false); - // Unique client session identifier. this->con_id = ProxyClientSession::next_connection_id(); this->client_vc = new_vc; @@ -195,9 +213,9 @@ Http2ClientSession::set_upgrade_context(HTTPHdr *h) int svlen; const char *sv = settings->value_get(&svlen); - // Maybe size of data decoded by Base64URL is lower than size of encoded data. - unsigned char out_buf[svlen]; if (sv && svlen > 0) { + // Maybe size of data decoded by Base64URL is lower than size of encoded data. + unsigned char out_buf[svlen]; size_t decoded_len; ats_base64_decode(sv, svlen, out_buf, svlen, &decoded_len); for (size_t nbytes = 0; nbytes < decoded_len; nbytes += HTTP2_SETTINGS_PARAMETER_LEN) { @@ -256,6 +274,7 @@ Http2ClientSession::do_io_shutdown(ShutdownHowTo_t howto) void Http2ClientSession::do_io_close(int alerrno) { + REMEMBER(NO_EVENT, this->recursion) Http2SsnDebug("session closed"); ink_assert(this->mutex->thread_holding == this_ethread()); @@ -275,6 +294,8 @@ Http2ClientSession::do_io_close(int alerrno) SCOPED_MUTEX_LOCK(lock, this->connection_state.mutex, this_ethread()); this->connection_state.release_stream(nullptr); } + + this->clear_session_active(); } void @@ -317,7 +338,7 @@ Http2ClientSession::main_event_handler(int event, void *edata) } case HTTP2_SESSION_EVENT_XMIT: { - Http2Frame *frame = (Http2Frame *)edata; + Http2Frame *frame = static_cast(edata); total_write_len += frame->size(); write_vio->nbytes = total_write_len; frame->xmit(this->write_buffer); @@ -351,13 +372,25 @@ Http2ClientSession::main_event_handler(int event, void *edata) break; } - if (!this->is_draining()) { + if (!this->is_draining() && this->connection_state.get_shutdown_reason() == Http2ErrorCode::HTTP2_ERROR_MAX) { this->connection_state.set_shutdown_state(HTTP2_SHUTDOWN_NONE); } - // For a case we already checked Connection header and it didn't exist - if (this->is_draining() && this->connection_state.get_shutdown_state() == HTTP2_SHUTDOWN_NONE) { - this->connection_state.set_shutdown_state(HTTP2_SHUTDOWN_NOT_INITIATED); + if (this->connection_state.get_shutdown_state() == HTTP2_SHUTDOWN_NONE) { + if (this->is_draining()) { // For a case we already checked Connection header and it didn't exist + Http2SsnDebug("Preparing for graceful shutdown because of draining state"); + this->connection_state.set_shutdown_state(HTTP2_SHUTDOWN_NOT_INITIATED); + } else if (this->connection_state.get_stream_error_rate() > + Http2::stream_error_rate_threshold) { // For a case many stream errors happened + ip_port_text_buffer ipb; + const char *client_ip = ats_ip_ntop(get_client_addr(), ipb, sizeof(ipb)); + Error("HTTP/2 session error client_ip=%s session_id=%" PRId64 + " closing a connection, because its stream error rate (%f) exceeded the threshold (%f)", + client_ip, connection_id(), this->connection_state.get_stream_error_rate(), Http2::stream_error_rate_threshold); + Http2SsnDebug("Preparing for graceful shutdown because of a high stream error rate"); + cause_of_death = Http2SessionCod::HIGH_ERROR_RATE; + this->connection_state.set_shutdown_state(HTTP2_SHUTDOWN_NOT_INITIATED, Http2ErrorCode::HTTP2_ERROR_ENHANCE_YOUR_CALM); + } } if (this->connection_state.get_shutdown_state() == HTTP2_SHUTDOWN_NOT_INITIATED) { @@ -510,6 +543,11 @@ Http2ClientSession::state_process_frame_read(int event, VIO *vio, bool inside_fr } while (this->sm_reader->read_avail() >= (int64_t)HTTP2_FRAME_HEADER_LEN) { + // Cancel reading if there was an error + if (connection_state.tx_error_code.code != static_cast(Http2ErrorCode::HTTP2_ERROR_NO_ERROR)) { + Http2SsnDebug("reading a frame has been canceled (%u)", connection_state.tx_error_code.code); + break; + } // Return if there was an error Http2ErrorCode err; if (do_start_frame_read(err) < 0) { @@ -551,3 +589,9 @@ Http2ClientSession::decrement_current_active_client_connections_stat() { HTTP2_DECREMENT_THREAD_DYN_STAT(HTTP2_STAT_CURRENT_ACTIVE_CLIENT_CONNECTION_COUNT, this_ethread()); } + +void +Http2ClientSession::remember(const SourceLocation &location, int event, int reentrant) +{ + this->_history.push_back(location, event, reentrant); +} diff --git a/proxy/http2/Http2ClientSession.h b/proxy/http2/Http2ClientSession.h index 51f709e4e71..b6d1faed0f3 100644 --- a/proxy/http2/Http2ClientSession.h +++ b/proxy/http2/Http2ClientSession.h @@ -29,6 +29,7 @@ #include "Http2ConnectionState.h" #include #include "tscore/ink_inet.h" +#include "tscore/History.h" // Name Edata Description // HTTP2_SESSION_EVENT_INIT Http2ClientSession * HTTP/2 session is born @@ -43,11 +44,16 @@ #define HTTP2_SESSION_EVENT_SHUTDOWN_INIT (HTTP2_SESSION_EVENTS_START + 5) #define HTTP2_SESSION_EVENT_SHUTDOWN_CONT (HTTP2_SESSION_EVENTS_START + 6) +enum class Http2SessionCod : int { + NOT_PROVIDED, + HIGH_ERROR_RATE, +}; + size_t const HTTP2_HEADER_BUFFER_SIZE_INDEX = CLIENT_CONNECTION_FIRST_READ_BUFFER_SIZE_INDEX; // To support Upgrade: h2c struct Http2UpgradeContext { - Http2UpgradeContext() : req_header(nullptr) {} + Http2UpgradeContext() {} ~Http2UpgradeContext() { if (req_header) { @@ -57,7 +63,7 @@ struct Http2UpgradeContext { } // Modified request header - HTTPHdr *req_header; + HTTPHdr *req_header = nullptr; // Decoded HTTP2-Settings Header Field Http2ConnectionSettings client_settings; @@ -66,17 +72,8 @@ struct Http2UpgradeContext { class Http2Frame { public: - Http2Frame(const Http2FrameHeader &h, IOBufferReader *r) - { - this->hdr = h; - this->ioreader = r; - } - - Http2Frame(Http2FrameType type, Http2StreamId streamid, uint8_t flags) - { - this->hdr = {0, (uint8_t)type, flags, streamid}; - this->ioreader = nullptr; - } + Http2Frame(const Http2FrameHeader &h, IOBufferReader *r) : hdr(h), ioreader(r) {} + Http2Frame(Http2FrameType type, Http2StreamId streamid, uint8_t flags) : hdr({0, (uint8_t)type, flags, streamid}) {} IOBufferReader * reader() const @@ -149,7 +146,7 @@ class Http2Frame private: Http2FrameHeader hdr; // frame header Ptr ioblock; // frame payload - IOBufferReader *ioreader; + IOBufferReader *ioreader = nullptr; }; class Http2ClientSession : public ProxyClientSession @@ -164,7 +161,7 @@ class Http2ClientSession : public ProxyClientSession void start() override; void destroy() override; void free() override; - void new_connection(NetVConnection *new_vc, MIOBuffer *iobuf, IOBufferReader *reader, bool backdoor) override; + void new_connection(NetVConnection *new_vc, MIOBuffer *iobuf, IOBufferReader *reader) override; bool ready_to_free() const @@ -304,6 +301,9 @@ class Http2ClientSession : public ProxyClientSession return write_buffer->max_read_avail(); } + // Record history from Http2ConnectionState + void remember(const SourceLocation &location, int event, int reentrant = NO_REENTRANT); + // noncopyable Http2ClientSession(Http2ClientSession &) = delete; Http2ClientSession &operator=(const Http2ClientSession &) = delete; @@ -333,14 +333,17 @@ class Http2ClientSession : public ProxyClientSession IpEndpoint cached_client_addr; IpEndpoint cached_local_addr; + History _history; + // For Upgrade: h2c Http2UpgradeContext upgrade_context; - VIO *write_vio = nullptr; - int dying_event = 0; - bool kill_me = false; - bool half_close_local = false; - int recursion = 0; + VIO *write_vio = nullptr; + int dying_event = 0; + bool kill_me = false; + Http2SessionCod cause_of_death = Http2SessionCod::NOT_PROVIDED; + bool half_close_local = false; + int recursion = 0; std::unordered_set h2_pushed_urls; }; diff --git a/proxy/http2/Http2ConnectionState.cc b/proxy/http2/Http2ConnectionState.cc index a7cf6e738da..afc1d2488e4 100644 --- a/proxy/http2/Http2ConnectionState.cc +++ b/proxy/http2/Http2ConnectionState.cc @@ -26,8 +26,16 @@ #include "Http2ClientSession.h" #include "Http2Stream.h" #include "Http2DebugNames.h" +#include "HttpDebugNames.h" #include +#define REMEMBER(e, r) \ + { \ + if (this->ua_session) { \ + this->ua_session->remember(MakeSourceLocation(), e, r); \ + } \ + } + #define Http2ConDebug(ua_session, fmt, ...) \ SsnDebug(ua_session, "http2_con", "[%" PRId64 "] " fmt, ua_session->connection_id(), ##__VA_ARGS__); @@ -237,7 +245,7 @@ rcv_headers_frame(Http2ConnectionState &cstate, const Http2Frame &frame) // keep track of how many bytes we get in the frame stream->request_header_length += payload_length; - if (stream->request_header_length > Http2::max_request_header_size) { + if (stream->request_header_length > Http2::max_header_list_size) { return Http2Error(Http2ErrorClass::HTTP2_ERROR_CLASS_STREAM, Http2ErrorCode::HTTP2_ERROR_PROTOCOL_ERROR, "recv headers payload for headers greater than header length"); } @@ -800,7 +808,7 @@ rcv_continuation_frame(Http2ConnectionState &cstate, const Http2Frame &frame) // keep track of how many bytes we get in the frame stream->request_header_length += payload_length; - if (stream->request_header_length > Http2::max_request_header_size) { + if (stream->request_header_length > Http2::max_header_list_size) { return Http2Error(Http2ErrorClass::HTTP2_ERROR_CLASS_CONNECTION, Http2ErrorCode::HTTP2_ERROR_PROTOCOL_ERROR, "continuation payload for headers exceeded"); } @@ -876,7 +884,8 @@ Http2ConnectionState::main_event_handler(int event, void *edata) // Initialize HTTP/2 Connection case HTTP2_SESSION_EVENT_INIT: { ink_assert(this->ua_session == nullptr); - this->ua_session = (Http2ClientSession *)edata; + this->ua_session = static_cast(edata); + REMEMBER(event, this->recursion); // [RFC 7540] 3.5. HTTP/2 Connection Preface. Upon establishment of a TCP connection and // determination that HTTP/2 will be used by both peers, each endpoint MUST @@ -902,6 +911,7 @@ Http2ConnectionState::main_event_handler(int event, void *edata) // Finalize HTTP/2 Connection case HTTP2_SESSION_EVENT_FINI: { SCOPED_MUTEX_LOCK(lock, this->mutex, this_ethread()); + REMEMBER(event, this->recursion); ink_assert(this->fini_received == false); this->fini_received = true; @@ -911,6 +921,7 @@ Http2ConnectionState::main_event_handler(int event, void *edata) } break; case HTTP2_SESSION_EVENT_XMIT: { + REMEMBER(event, this->recursion); SCOPED_MUTEX_LOCK(lock, this->mutex, this_ethread()); send_data_frames_depends_on_priority(); _scheduled = false; @@ -918,7 +929,8 @@ Http2ConnectionState::main_event_handler(int event, void *edata) // Parse received HTTP/2 frames case HTTP2_SESSION_EVENT_RECV: { - const Http2Frame *frame = (Http2Frame *)edata; + REMEMBER(event, this->recursion); + const Http2Frame *frame = static_cast(edata); const Http2StreamId stream_id = frame->header().streamid; Http2Error error; @@ -964,6 +976,7 @@ Http2ConnectionState::main_event_handler(int event, void *edata) // Initiate a gracefull shutdown case HTTP2_SESSION_EVENT_SHUTDOWN_INIT: { + REMEMBER(event, this->recursion); ink_assert(shutdown_state == HTTP2_SHUTDOWN_NOT_INITIATED); shutdown_state = HTTP2_SHUTDOWN_INITIATED; // [RFC 7540] 6.8. GOAWAY @@ -977,12 +990,16 @@ Http2ConnectionState::main_event_handler(int event, void *edata) // Continue a gracefull shutdown case HTTP2_SESSION_EVENT_SHUTDOWN_CONT: { + REMEMBER(event, this->recursion); ink_assert(shutdown_state == HTTP2_SHUTDOWN_INITIATED); shutdown_cont_event = nullptr; shutdown_state = HTTP2_SHUTDOWN_IN_PROGRESS; // [RFC 7540] 6.8. GOAWAY // ..., the server can send another GOAWAY frame with an updated last stream identifier - send_goaway_frame(latest_streamid_in, Http2ErrorCode::HTTP2_ERROR_NO_ERROR); + if (shutdown_reason == Http2ErrorCode::HTTP2_ERROR_MAX) { + shutdown_reason = Http2ErrorCode::HTTP2_ERROR_NO_ERROR; + } + send_goaway_frame(latest_streamid_in, shutdown_reason); // Stop creating new streams SCOPED_MUTEX_LOCK(lock, this->ua_session->mutex, this_ethread()); this->ua_session->set_half_close_local_flag(true); @@ -1010,8 +1027,10 @@ Http2ConnectionState::main_event_handler(int event, void *edata) } int -Http2ConnectionState::state_closed(int /* event */, void *edata) +Http2ConnectionState::state_closed(int event, void *edata) { + REMEMBER(event, this->recursion); + if (edata == zombie_event) { // Zombie session is still around. Assert! ink_release_assert(zombie_event == nullptr); @@ -1119,14 +1138,15 @@ Http2ConnectionState::find_stream(Http2StreamId id) const void Http2ConnectionState::restart_streams() { - // This is a static variable, so it is shared in Http2ConnectionState instances and will get incremented in subsequent calls. - // It doesn't need to be initialized with rand() nor time(), and doesn't need to be accessed with a lock, because it doesn't need - // that randomness and accuracy. - static uint16_t starting_point = 0; - - Http2Stream *s = stream_list.head; - Http2Stream *end = s; + Http2Stream *s = stream_list.head; if (s) { + Http2Stream *end = s; + + // This is a static variable, so it is shared in Http2ConnectionState instances and will get incremented in subsequent calls. + // It doesn't need to be initialized with rand() nor time(), and doesn't need to be accessed with a lock, because it doesn't + // need that randomness and accuracy. + static uint16_t starting_point = 0; + // Change the start point randomly for (int i = starting_point % total_client_streams_count; i >= 0; --i) { end = static_cast(end->link.next ? end->link.next : stream_list.head); @@ -1195,6 +1215,7 @@ Http2ConnectionState::delete_stream(Http2Stream *stream) } Http2StreamDebug(ua_session, stream->get_id(), "Delete stream"); + REMEMBER(NO_EVENT, this->recursion); if (Http2::stream_priority_enabled) { Http2DependencyTree::Node *node = stream->priority_node; @@ -1236,27 +1257,18 @@ Http2ConnectionState::delete_stream(Http2Stream *stream) void Http2ConnectionState::release_stream(Http2Stream *stream) { + REMEMBER(NO_EVENT, this->recursion) + if (stream) { // Decrement total_client_streams_count here, because it's a counter include streams in the process of shutting down. // Other counters (client_streams_in_count/client_streams_out_count) are already decremented in delete_stream(). --total_client_streams_count; } - if (ua_session) { - SCOPED_MUTEX_LOCK(lock, this->ua_session->mutex, this_ethread()); - if (!ua_session) { - // Workaround fix for GitHub #4504. The `ua_session` could be freed while waiting for acquiring the above lock. - return; - } + SCOPED_MUTEX_LOCK(lock, this->mutex, this_ethread()); + if (this->ua_session) { + ink_assert(this->mutex == ua_session->mutex); - // If the number of clients is 0 and ua_session is active, then mark the connection as inactive - if (total_client_streams_count == 0 && ua_session->is_active()) { - ua_session->clear_session_active(); - UnixNetVConnection *vc = static_cast(ua_session->get_netvc()); - if (vc && vc->active_timeout_in == 0) { - vc->add_to_keep_alive_queue(); - } - } if (total_client_streams_count == 0) { if (fini_received) { // We were shutting down, go ahead and terminate the session @@ -1269,6 +1281,15 @@ Http2ConnectionState::release_stream(Http2Stream *stream) // ua_session = nullptr; } else if (shutdown_state == HTTP2_SHUTDOWN_IN_PROGRESS && fini_event == nullptr) { fini_event = this_ethread()->schedule_imm_local((Continuation *)this, HTTP2_SESSION_EVENT_FINI); + } else if (ua_session->is_active()) { + // If the number of clients is 0, HTTP2_SESSION_EVENT_FINI is not received or sent, and ua_session is active, + // then mark the connection as inactive + ua_session->clear_session_active(); + UnixNetVConnection *vc = static_cast(ua_session->get_netvc()); + if (vc && vc->active_timeout_in == 0) { + // With heavy traffic, ua_session could be destroyed. Do not touch ua_session after this. + vc->add_to_keep_alive_queue(); + } } else { schedule_zombie_event(); } @@ -1358,7 +1379,6 @@ Http2ConnectionState::send_a_data_frame(Http2Stream *stream, size_t &payload_len const ssize_t window_size = std::min(this->client_rwnd, stream->client_rwnd); const size_t buf_len = BUFFER_SIZE_FOR_INDEX(buffer_size_index[HTTP2_FRAME_TYPE_DATA]); const size_t write_available_size = std::min(buf_len, static_cast(window_size)); - size_t read_available_size = 0; payload_length = 0; uint8_t flags = 0x00; @@ -1367,23 +1387,21 @@ Http2ConnectionState::send_a_data_frame(Http2Stream *stream, size_t &payload_len SCOPED_MUTEX_LOCK(stream_lock, stream->mutex, this_ethread()); - if (current_reader) { - read_available_size = static_cast(current_reader->read_avail()); - } else { + if (!current_reader) { Http2StreamDebug(this->ua_session, stream->get_id(), "couldn't get data reader"); return Http2SendDataFrameResult::ERROR; } // Select appropriate payload length - if (read_available_size > 0) { + if (current_reader->is_read_avail_more_than(0)) { // We only need to check for window size when there is a payload if (window_size <= 0) { Http2StreamDebug(this->ua_session, stream->get_id(), "No window"); return Http2SendDataFrameResult::NO_WINDOW; } // Copy into the payload buffer. Seems like we should be able to skip this copy step - payload_length = std::min(read_available_size, write_available_size); - current_reader->memcpy(payload_buffer, static_cast(payload_length)); + payload_length = write_available_size; + payload_length = current_reader->read(payload_buffer, static_cast(write_available_size)); } else { payload_length = 0; } @@ -1396,7 +1414,7 @@ Http2ConnectionState::send_a_data_frame(Http2Stream *stream, size_t &payload_len return Http2SendDataFrameResult::NO_PAYLOAD; } - if (stream->is_body_done() && read_available_size <= write_available_size) { + if (stream->is_body_done() && !current_reader->is_read_avail_more_than(0)) { flags |= HTTP2_FLAGS_DATA_END_STREAM; } @@ -1414,7 +1432,6 @@ Http2ConnectionState::send_a_data_frame(Http2Stream *stream, size_t &payload_len data.finalize(payload_length); stream->update_sent_count(payload_length); - current_reader->consume(payload_length); // xmit event SCOPED_MUTEX_LOCK(lock, this->ua_session->mutex, this_ethread()); @@ -1536,14 +1553,14 @@ Http2ConnectionState::send_headers_frame(Http2Stream *stream) if (sent + payload_length == header_blocks_size) { flags |= HTTP2_FLAGS_CONTINUATION_END_HEADERS; } - Http2Frame headers(HTTP2_FRAME_TYPE_CONTINUATION, stream->get_id(), flags); - headers.alloc(buffer_size_index[HTTP2_FRAME_TYPE_CONTINUATION]); - http2_write_headers(buf + sent, payload_length, headers.write()); - headers.finalize(payload_length); - stream->change_state(headers.header().type, headers.header().flags); + Http2Frame continuation_frame(HTTP2_FRAME_TYPE_CONTINUATION, stream->get_id(), flags); + continuation_frame.alloc(buffer_size_index[HTTP2_FRAME_TYPE_CONTINUATION]); + http2_write_headers(buf + sent, payload_length, continuation_frame.write()); + continuation_frame.finalize(payload_length); + stream->change_state(continuation_frame.header().type, continuation_frame.header().flags); // xmit event SCOPED_MUTEX_LOCK(lock, this->ua_session->mutex, this_ethread()); - this->ua_session->handleEvent(HTTP2_SESSION_EVENT_XMIT, &headers); + this->ua_session->handleEvent(HTTP2_SESSION_EVENT_XMIT, &continuation_frame); sent += payload_length; } @@ -1613,15 +1630,15 @@ Http2ConnectionState::send_push_promise_frame(Http2Stream *stream, URL &url, con payload_length = BUFFER_SIZE_FOR_INDEX(buffer_size_index[HTTP2_FRAME_TYPE_PUSH_PROMISE]) - sizeof(push_promise.promised_streamid); } - Http2Frame headers(HTTP2_FRAME_TYPE_PUSH_PROMISE, stream->get_id(), flags); - headers.alloc(buffer_size_index[HTTP2_FRAME_TYPE_PUSH_PROMISE]); + Http2Frame push_promise_frame(HTTP2_FRAME_TYPE_PUSH_PROMISE, stream->get_id(), flags); + push_promise_frame.alloc(buffer_size_index[HTTP2_FRAME_TYPE_PUSH_PROMISE]); Http2StreamId id = this->get_latest_stream_id_out() + 2; push_promise.promised_streamid = id; - http2_write_push_promise(push_promise, buf, payload_length, headers.write()); - headers.finalize(sizeof(push_promise.promised_streamid) + payload_length); + http2_write_push_promise(push_promise, buf, payload_length, push_promise_frame.write()); + push_promise_frame.finalize(sizeof(push_promise.promised_streamid) + payload_length); // xmit event SCOPED_MUTEX_LOCK(lock, this->ua_session->mutex, this_ethread()); - this->ua_session->handleEvent(HTTP2_SESSION_EVENT_XMIT, &headers); + this->ua_session->handleEvent(HTTP2_SESSION_EVENT_XMIT, &push_promise_frame); sent += payload_length; // Send CONTINUATION frames @@ -1633,13 +1650,13 @@ Http2ConnectionState::send_push_promise_frame(Http2Stream *stream, URL &url, con if (sent + payload_length == header_blocks_size) { flags |= HTTP2_FLAGS_CONTINUATION_END_HEADERS; } - Http2Frame headers(HTTP2_FRAME_TYPE_CONTINUATION, stream->get_id(), flags); - headers.alloc(buffer_size_index[HTTP2_FRAME_TYPE_CONTINUATION]); - http2_write_headers(buf + sent, payload_length, headers.write()); - headers.finalize(payload_length); + Http2Frame continuation_frame(HTTP2_FRAME_TYPE_CONTINUATION, stream->get_id(), flags); + continuation_frame.alloc(buffer_size_index[HTTP2_FRAME_TYPE_CONTINUATION]); + http2_write_headers(buf + sent, payload_length, continuation_frame.write()); + continuation_frame.finalize(payload_length); // xmit event SCOPED_MUTEX_LOCK(lock, this->ua_session->mutex, this_ethread()); - this->ua_session->handleEvent(HTTP2_SESSION_EVENT_XMIT, &headers); + this->ua_session->handleEvent(HTTP2_SESSION_EVENT_XMIT, &continuation_frame); sent += payload_length; } ats_free(buf); @@ -1680,6 +1697,7 @@ Http2ConnectionState::send_rst_stream_frame(Http2StreamId id, Http2ErrorCode ec) if (ec != Http2ErrorCode::HTTP2_ERROR_NO_ERROR) { HTTP2_INCREMENT_THREAD_DYN_STAT(HTTP2_STAT_STREAM_ERRORS_COUNT, this_ethread()); + ++stream_error_count; } Http2Frame rst_stream(HTTP2_FRAME_TYPE_RST_STREAM, id, 0); @@ -1776,6 +1794,8 @@ Http2ConnectionState::send_ping_frame(Http2StreamId id, uint8_t flag, const uint void Http2ConnectionState::send_goaway_frame(Http2StreamId id, Http2ErrorCode ec) { + ink_assert(this->ua_session != nullptr); + Http2ConDebug(ua_session, "Send GOAWAY frame, last_stream_id: %d", id); if (ec != Http2ErrorCode::HTTP2_ERROR_NO_ERROR) { @@ -1785,8 +1805,6 @@ Http2ConnectionState::send_goaway_frame(Http2StreamId id, Http2ErrorCode ec) Http2Frame frame(HTTP2_FRAME_TYPE_GOAWAY, 0, 0); Http2Goaway goaway; - ink_assert(this->ua_session != nullptr); - goaway.last_streamid = id; goaway.error_code = ec; diff --git a/proxy/http2/Http2ConnectionState.h b/proxy/http2/Http2ConnectionState.h index bb2a2b2b7b0..57cd3685bc4 100644 --- a/proxy/http2/Http2ConnectionState.h +++ b/proxy/http2/Http2ConnectionState.h @@ -23,6 +23,7 @@ #pragma once +#include #include "HTTP2.h" #include "HPACK.h" #include "Http2Stream.h" @@ -84,7 +85,7 @@ class Http2ConnectionSettings if (0 < id && id < HTTP2_SETTINGS_MAX) { return this->settings[indexof(id)] = value; } else { - ink_assert(!"Bad Settings Identifier"); + // Do nothing - 6.5.3 Unsupported parameters MUST be ignored. } return 0; @@ -141,10 +142,12 @@ class Http2ConnectionState : public Continuation } cleanup_streams(); - mutex = nullptr; // magic happens - assigning to nullptr frees the ProxyMutex delete local_hpack_handle; + local_hpack_handle = nullptr; delete remote_hpack_handle; + remote_hpack_handle = nullptr; delete dependency_tree; + dependency_tree = nullptr; this->ua_session = nullptr; if (fini_event) { @@ -153,6 +156,8 @@ class Http2ConnectionState : public Continuation if (zombie_event) { zombie_event->cancel(); } + // release the mutex after the events are cancelled and sessions are destroyed. + mutex = nullptr; // magic happens - assigning to nullptr frees the ProxyMutex } // Event handlers @@ -216,6 +221,23 @@ class Http2ConnectionState : public Continuation return client_streams_in_count; } + double + get_stream_error_rate() const + { + int total = get_stream_requests(); + if (total > 0) { + return (double)stream_error_count / (double)total; + } else { + return 0; + } + } + + Http2ErrorCode + get_shutdown_reason() const + { + return shutdown_reason; + } + // Connection level window size ssize_t client_rwnd = HTTP2_INITIAL_WINDOW_SIZE; ssize_t server_rwnd = Http2::initial_window_size; @@ -262,9 +284,10 @@ class Http2ConnectionState : public Continuation } void - set_shutdown_state(Http2ShutdownState state) + set_shutdown_state(Http2ShutdownState state, Http2ErrorCode reason = Http2ErrorCode::HTTP2_ERROR_NO_ERROR) { - shutdown_state = state; + shutdown_state = state; + shutdown_reason = reason; } // noncopyable @@ -300,16 +323,19 @@ class Http2ConnectionState : public Continuation Queue stream_list; Http2StreamId latest_streamid_in = 0; Http2StreamId latest_streamid_out = 0; - int stream_requests = 0; + std::atomic stream_requests = 0; // Counter for current active streams which is started by client - uint32_t client_streams_in_count = 0; + std::atomic client_streams_in_count = 0; // Counter for current acive streams which is started by server - uint32_t client_streams_out_count = 0; + std::atomic client_streams_out_count = 0; // Counter for current active streams and streams in the process of shutting down - uint32_t total_client_streams_count = 0; + std::atomic total_client_streams_count = 0; + + // Counter for stream errors ATS sent + uint32_t stream_error_count = 0; // NOTE: Id of stream which MUST receive CONTINUATION frame. // - [RFC 7540] 6.2 HEADERS @@ -323,6 +349,7 @@ class Http2ConnectionState : public Continuation bool fini_received = false; int recursion = 0; Http2ShutdownState shutdown_state = HTTP2_SHUTDOWN_NONE; + Http2ErrorCode shutdown_reason = Http2ErrorCode::HTTP2_ERROR_MAX; Event *shutdown_cont_event = nullptr; Event *fini_event = nullptr; Event *zombie_event = nullptr; diff --git a/proxy/http2/Http2DependencyTree.h b/proxy/http2/Http2DependencyTree.h index 7b71303b113..ec12acb17cf 100644 --- a/proxy/http2/Http2DependencyTree.h +++ b/proxy/http2/Http2DependencyTree.h @@ -41,7 +41,7 @@ namespace Http2DependencyTree class Node { public: - Node(void *t = nullptr) : t(t) + explicit Node(void *t = nullptr) : t(t) { entry = new PriorityQueueEntry(this); queue = new PriorityQueue(); @@ -71,6 +71,9 @@ class Node } } + Node(const Node &) = delete; + Node &operator=(const Node &) = delete; + LINK(Node, link); bool @@ -78,6 +81,7 @@ class Node { return point < n.point; } + bool operator>(const Node &n) const { @@ -111,7 +115,7 @@ class Node template class Tree { public: - Tree(uint32_t max_concurrent_streams) : _max_depth(MIN(max_concurrent_streams, HTTP2_DEPENDENCY_TREE_MAX_DEPTH)) + explicit Tree(uint32_t max_concurrent_streams) : _max_depth(MIN(max_concurrent_streams, HTTP2_DEPENDENCY_TREE_MAX_DEPTH)) { _ancestors.resize(_max_ancestors); } @@ -119,10 +123,10 @@ template class Tree Node *find(uint32_t id, bool *is_max_leaf = nullptr); Node *find_shadow(uint32_t id, bool *is_max_leaf = nullptr); Node *add(uint32_t parent_id, uint32_t id, uint32_t weight, bool exclusive, T t, bool shadow = false); - void remove(Node *node); - void reprioritize(uint32_t new_parent_id, uint32_t id, bool exclusive); - void reprioritize(Node *node, uint32_t id, bool exclusive); + Node *reprioritize(uint32_t id, uint32_t new_parent_id, bool exclusive); + Node *reprioritize(Node *node, uint32_t id, bool exclusive); Node *top(); + void remove(Node *node); void activate(Node *node); void deactivate(Node *node, uint32_t sent); void update(Node *node, uint32_t sent); @@ -143,7 +147,7 @@ template class Tree void _dump(Node *node, std::ostream &output) const; Node *_find(Node *node, uint32_t id, uint32_t depth = 1, bool *is_max_leaf = nullptr); Node *_top(Node *node); - void _change_parent(Node *new_parent, Node *node, bool exclusive); + void _change_parent(Node *node, Node *new_parent, bool exclusive); bool in_parent_chain(Node *maybe_parent, Node *target); Node *_root = new Node(this); @@ -177,7 +181,7 @@ template void Tree::_dump(Node *node, std::ostream &output) const { - output << "{ \"id\":\"" << node->id << "/" << node->weight << "/" << node->point << "/" << ((node->t != nullptr) ? "1" : "0") + output << R"({ "id":")" << node->id << "/" << node->weight << "/" << node->point << "/" << ((node->t != nullptr) ? "1" : "0") << "/" << ((node->active) ? "a" : "d") << "\","; // Dump the children output << " \"c\":["; @@ -269,7 +273,7 @@ Tree::add(uint32_t parent_id, uint32_t id, uint32_t weight, bool exclusive, T node->weight = weight; node->shadow = false; // Move the shadow node into the proper position in the tree - reprioritize(node, parent_id, exclusive); + node = reprioritize(node, parent_id, exclusive); return node; } @@ -389,36 +393,36 @@ Tree::remove(Node *node) } template -void +Node * Tree::reprioritize(uint32_t id, uint32_t new_parent_id, bool exclusive) { Node *node = find(id); if (node == nullptr) { - return; + return node; } - reprioritize(node, new_parent_id, exclusive); + return reprioritize(node, new_parent_id, exclusive); } template -void +Node * Tree::reprioritize(Node *node, uint32_t new_parent_id, bool exclusive) { if (node == nullptr) { - return; + return node; } Node *old_parent = node->parent; if (old_parent->id == new_parent_id) { // Do nothing - return; + return node; } // should not change the root node ink_assert(node->parent); Node *new_parent = find(new_parent_id); if (new_parent == nullptr) { - return; + return node; } // If node is dependent on the new parent, must move the new parent first if (new_parent_id != 0 && in_parent_chain(node, new_parent)) { @@ -429,7 +433,10 @@ Tree::reprioritize(Node *node, uint32_t new_parent_id, bool exclusive) // delete the shadow node if (node->is_shadow() && node->children.empty() && node->queue->empty()) { remove(node); + return nullptr; } + + return node; } template diff --git a/proxy/http2/Http2SessionAccept.cc b/proxy/http2/Http2SessionAccept.cc index 165eb5e9baa..0fd55f848dd 100644 --- a/proxy/http2/Http2SessionAccept.cc +++ b/proxy/http2/Http2SessionAccept.cc @@ -59,7 +59,7 @@ Http2SessionAccept::accept(NetVConnection *netvc, MIOBuffer *iobuf, IOBufferRead new_session->outbound_ip4 = options.outbound_ip4; new_session->outbound_ip6 = options.outbound_ip6; new_session->outbound_port = options.outbound_port; - new_session->new_connection(netvc, iobuf, reader, false /* backdoor */); + new_session->new_connection(netvc, iobuf, reader); return true; } @@ -67,12 +67,11 @@ Http2SessionAccept::accept(NetVConnection *netvc, MIOBuffer *iobuf, IOBufferRead int Http2SessionAccept::mainEvent(int event, void *data) { - NetVConnection *netvc; ink_release_assert(event == NET_EVENT_ACCEPT || event == EVENT_ERROR); ink_release_assert((event == NET_EVENT_ACCEPT) ? (data != nullptr) : (1)); if (event == NET_EVENT_ACCEPT) { - netvc = static_cast(data); + NetVConnection *netvc = static_cast(data); if (!this->accept(netvc, nullptr, nullptr)) { netvc->do_io_close(); } diff --git a/proxy/http2/Http2Stream.cc b/proxy/http2/Http2Stream.cc index 1f3cd887dd0..46614a63847 100644 --- a/proxy/http2/Http2Stream.cc +++ b/proxy/http2/Http2Stream.cc @@ -26,6 +26,11 @@ #include "Http2ClientSession.h" #include "../http/HttpSM.h" +#define REMEMBER(e, r) \ + { \ + this->_history.push_back(MakeSourceLocation(), e, r); \ + } + #define Http2StreamDebug(fmt, ...) \ SsnDebug(parent, "http2_stream", "[%" PRId64 "] [%u] " fmt, parent->connection_id(), this->get_id(), ##__VA_ARGS__); @@ -34,19 +39,17 @@ ClassAllocator http2StreamAllocator("http2StreamAllocator"); int Http2Stream::main_event_handler(int event, void *edata) { - Event *e = static_cast(edata); + SCOPED_MUTEX_LOCK(lock, this->mutex, this_ethread()); + REMEMBER(event, this->reentrancy_count); - Thread *this_thread = this_ethread(); - if (this->get_thread() != this_thread) { - // Send on to the owning thread - if (cross_thread_event == nullptr) { - cross_thread_event = this->get_thread()->schedule_imm(this, event, edata); - } + if (!this->_switch_thread_if_not_on_right_thread(event, edata)) { + // Not on the right thread return 0; } + ink_release_assert(this->_thread == this_ethread()); + + Event *e = static_cast(edata); reentrancy_count++; - ink_release_assert(this->get_thread() == this_ethread()); - SCOPED_MUTEX_LOCK(lock, this->mutex, this_ethread()); if (e == cross_thread_event) { cross_thread_event = nullptr; } else if (e == active_event) { @@ -151,11 +154,10 @@ Http2Stream::send_request(Http2ConnectionState &cstate) int bufindex; int dumpoffset = 0; int done, tmp; - IOBufferBlock *block; do { - bufindex = 0; - tmp = dumpoffset; - block = request_buffer.get_current_block(); + bufindex = 0; + tmp = dumpoffset; + IOBufferBlock *block = request_buffer.get_current_block(); if (!block) { request_buffer.add_block(); block = request_buffer.get_current_block(); @@ -323,6 +325,7 @@ Http2Stream::do_io_close(int /* flags */) super::release(nullptr); if (!closed) { + REMEMBER(NO_EVENT, this->reentrancy_count); Http2StreamDebug("do_io_close"); // When we get here, the SM has initiated the shutdown. Either it received a WRITE_COMPLETE, or it is shutting down. Any @@ -374,6 +377,8 @@ void Http2Stream::terminate_if_possible() { if (terminate_stream && reentrancy_count == 0) { + REMEMBER(NO_EVENT, this->reentrancy_count); + Http2ClientSession *h2_parent = static_cast(parent); SCOPED_MUTEX_LOCK(lock, h2_parent->connection_state.mutex, this_ethread()); h2_parent->connection_state.delete_stream(this); @@ -387,6 +392,7 @@ Http2Stream::initiating_close() { if (!closed) { SCOPED_MUTEX_LOCK(lock, this->mutex, this_ethread()); + REMEMBER(NO_EVENT, this->reentrancy_count); Http2StreamDebug("initiating_close"); // Set the state of the connection to closed @@ -455,6 +461,7 @@ Http2Stream::send_tracked_event(Event *event, int send_event, VIO *vio) } if (event == nullptr) { + REMEMBER(send_event, this->reentrancy_count); event = this_ethread()->schedule_imm(this, send_event, vio); } @@ -467,15 +474,13 @@ Http2Stream::update_read_request(int64_t read_len, bool call_update, bool check_ if (closed || parent == nullptr || current_reader == nullptr || read_vio.mutex == nullptr) { return; } - if (this->get_thread() != this_ethread()) { - SCOPED_MUTEX_LOCK(stream_lock, this->mutex, this_ethread()); - if (cross_thread_event == nullptr) { - // Send to the right thread - cross_thread_event = this->get_thread()->schedule_imm(this, VC_EVENT_READ_READY, nullptr); - } + + if (!this->_switch_thread_if_not_on_right_thread(VC_EVENT_READ_READY, nullptr)) { + // Not on the right thread return; } - ink_release_assert(this->get_thread() == this_ethread()); + ink_release_assert(this->_thread == this_ethread()); + SCOPED_MUTEX_LOCK(lock, read_vio.mutex, this_ethread()); if (read_vio.nbytes > 0 && read_vio.ndone <= read_vio.nbytes) { // If this vio has a different buffer, we must copy @@ -532,15 +537,13 @@ Http2Stream::update_write_request(IOBufferReader *buf_reader, int64_t write_len, (buf_reader == nullptr && write_len == 0)) { return; } - if (this->get_thread() != this_ethread()) { - SCOPED_MUTEX_LOCK(stream_lock, this->mutex, this_ethread()); - if (cross_thread_event == nullptr) { - // Send to the right thread - cross_thread_event = this->get_thread()->schedule_imm(this, VC_EVENT_WRITE_READY, nullptr); - } + + if (!this->_switch_thread_if_not_on_right_thread(VC_EVENT_WRITE_READY, nullptr)) { + // Not on the right thread return; } - ink_release_assert(this->get_thread() == this_ethread()); + ink_release_assert(this->_thread == this_ethread()); + Http2ClientSession *parent = static_cast(this->get_parent()); SCOPED_MUTEX_LOCK(lock, write_vio.mutex, this_ethread()); @@ -550,7 +553,7 @@ Http2Stream::update_write_request(IOBufferReader *buf_reader, int64_t write_len, if (this->chunked) { if (chunked_handler.dechunked_buffer && chunked_handler.dechunked_buffer->max_read_avail() > HTTP2_MAX_BUFFER_USAGE) { if (buffer_full_write_event == nullptr) { - buffer_full_write_event = get_thread()->schedule_imm(this, VC_EVENT_WRITE_READY); + buffer_full_write_event = _thread->schedule_imm(this, VC_EVENT_WRITE_READY); } } else { this->response_process_data(is_done); @@ -599,7 +602,7 @@ Http2Stream::update_write_request(IOBufferReader *buf_reader, int64_t write_len, if (memcmp(HTTP_VALUE_CLOSE, value, HTTP_LEN_CLOSE) == 0) { SCOPED_MUTEX_LOCK(lock, parent->connection_state.mutex, this_ethread()); if (parent->connection_state.get_shutdown_state() == HTTP2_SHUTDOWN_NONE) { - parent->connection_state.set_shutdown_state(HTTP2_SHUTDOWN_NOT_INITIATED); + parent->connection_state.set_shutdown_state(HTTP2_SHUTDOWN_NOT_INITIATED, Http2ErrorCode::HTTP2_ERROR_NO_ERROR); } } } @@ -712,6 +715,7 @@ Http2Stream::reenable(VIO *vio) void Http2Stream::destroy() { + REMEMBER(NO_EVENT, this->reentrancy_count); Http2StreamDebug("Destroy stream, sent %" PRIu64 " bytes", this->bytes_sent); SCOPED_MUTEX_LOCK(lock, this->mutex, this_ethread()); // Clean up after yourself if this was an EOS @@ -733,7 +737,6 @@ Http2Stream::destroy() // Clean up the write VIO in case of inactivity timeout this->do_io_write(nullptr, 0, nullptr); - HTTP2_DECREMENT_THREAD_DYN_STAT(HTTP2_STAT_CURRENT_CLIENT_STREAM_COUNT, _thread); ink_hrtime end_time = Thread::get_hrtime(); HTTP2_SUM_THREAD_DYN_STAT(HTTP2_STAT_TOTAL_TRANSACTIONS_TIME, _thread, end_time - _start_time); _req_header.destroy(); @@ -757,24 +760,6 @@ Http2Stream::destroy() THREAD_FREE(this, http2StreamAllocator, this_ethread()); } -bool -check_stream_thread(Continuation *cont) -{ - Http2Stream *stream = dynamic_cast(cont); - if (stream) { - return stream->get_thread() == this_ethread(); - } else { - return true; - } -} - -bool -check_continuation(Continuation *cont) -{ - Http2Stream *stream = dynamic_cast(cont); - return stream == nullptr; -} - void Http2Stream::response_initialize_data_handling(bool &is_done) { @@ -905,3 +890,30 @@ Http2Stream::release(IOBufferReader *r) current_reader = nullptr; // State machine is on its own way down. this->do_io_close(); } + +void +Http2Stream::increment_client_transactions_stat() +{ + HTTP2_INCREMENT_THREAD_DYN_STAT(HTTP2_STAT_CURRENT_CLIENT_STREAM_COUNT, _thread); + HTTP2_INCREMENT_THREAD_DYN_STAT(HTTP2_STAT_TOTAL_CLIENT_STREAM_COUNT, _thread); +} + +void +Http2Stream::decrement_client_transactions_stat() +{ + HTTP2_DECREMENT_THREAD_DYN_STAT(HTTP2_STAT_CURRENT_CLIENT_STREAM_COUNT, _thread); +} + +bool +Http2Stream::_switch_thread_if_not_on_right_thread(int event, void *edata) +{ + if (this->_thread != this_ethread()) { + SCOPED_MUTEX_LOCK(stream_lock, this->mutex, this_ethread()); + if (cross_thread_event == nullptr) { + // Send to the right thread + cross_thread_event = this->_thread->schedule_imm(this, event, edata); + } + return false; + } + return true; +} diff --git a/proxy/http2/Http2Stream.h b/proxy/http2/Http2Stream.h index e9950906536..71fc00d7c2e 100644 --- a/proxy/http2/Http2Stream.h +++ b/proxy/http2/Http2Stream.h @@ -28,6 +28,7 @@ #include "Http2DebugNames.h" #include "../http/HttpTunnel.h" // To get ChunkedHandler #include "Http2DependencyTree.h" +#include "tscore/History.h" class Http2Stream; class Http2ConnectionState; @@ -40,6 +41,8 @@ class Http2Stream : public ProxyClientTransaction typedef ProxyClientTransaction super; ///< Parent type. Http2Stream(Http2StreamId sid = 0, ssize_t initial_rwnd = Http2::initial_window_size) : client_rwnd(initial_rwnd), _id(sid) { + http_parser_init(&http_parser); + SET_HANDLER(&Http2Stream::main_event_handler); } @@ -50,16 +53,12 @@ class Http2Stream : public ProxyClientTransaction _start_time = Thread::get_hrtime(); _thread = this_ethread(); this->client_rwnd = initial_rwnd; - HTTP2_INCREMENT_THREAD_DYN_STAT(HTTP2_STAT_CURRENT_CLIENT_STREAM_COUNT, _thread); - HTTP2_INCREMENT_THREAD_DYN_STAT(HTTP2_STAT_TOTAL_CLIENT_STREAM_COUNT, _thread); sm_reader = request_reader = request_buffer.alloc_reader(); - http_parser_init(&http_parser); // FIXME: Are you sure? every "stream" needs request_header? _req_header.create(HTTP_TYPE_REQUEST); response_header.create(HTTP_TYPE_RESPONSE); } - ~Http2Stream() { this->destroy(); } int main_event_handler(int event, void *edata); void destroy() override; @@ -108,12 +107,6 @@ class Http2Stream : public ProxyClientTransaction bool change_state(uint8_t type, uint8_t flags); - void - set_id(Http2StreamId sid) - { - _id = sid; - } - void update_initial_rwnd(Http2WindowSize new_size) { @@ -186,12 +179,6 @@ class Http2Stream : public ProxyClientTransaction MIOBuffer request_buffer = CLIENT_CONNECTION_FIRST_READ_BUFFER_SIZE_INDEX; Http2DependencyTree::Node *priority_node = nullptr; - EThread * - get_thread() - { - return _thread; - } - IOBufferReader *response_get_data_reader() const; bool response_is_chunked() const @@ -234,6 +221,9 @@ class Http2Stream : public ProxyClientTransaction return is_first_transaction_flag; } + void increment_client_transactions_stat() override; + void decrement_client_transactions_stat() override; + private: void response_initialize_data_handling(bool &is_done); void response_process_data(bool &is_done); @@ -241,6 +231,13 @@ class Http2Stream : public ProxyClientTransaction Event *send_tracked_event(Event *event, int send_event, VIO *vio); void send_response_body(bool call_update); + /** + * Check if this thread is the right thread to process events for this + * continuation. + * Return true if the caller can continue event processing. + */ + bool _switch_thread_if_not_on_right_thread(int event, void *edata); + HTTPParser http_parser; ink_hrtime _start_time = 0; EThread *_thread = nullptr; @@ -251,6 +248,8 @@ class Http2Stream : public ProxyClientTransaction VIO read_vio; VIO write_vio; + History _history; + bool trailing_header = false; bool body_done = false; bool chunked = false; @@ -298,6 +297,3 @@ class Http2Stream : public ProxyClientTransaction }; extern ClassAllocator http2StreamAllocator; - -extern bool check_continuation(Continuation *cont); -extern bool check_stream_thread(Continuation *cont); diff --git a/proxy/http2/HuffmanCodec.cc b/proxy/http2/HuffmanCodec.cc index 0c103731c14..562fa36b88a 100644 --- a/proxy/http2/HuffmanCodec.cc +++ b/proxy/http2/HuffmanCodec.cc @@ -93,12 +93,12 @@ static Node * make_huffman_tree() { Node *root = make_huffman_tree_node(); - Node *current; - uint32_t bit_len; + // insert leafs for each ascii code for (unsigned i = 0; i < countof(huffman_table); i++) { - bit_len = huffman_table[i].bit_len; - current = root; + uint32_t bit_len = huffman_table[i].bit_len; + Node *current = root; + while (bit_len > 0) { if (huffman_table[i].code_as_hex & (1 << (bit_len - 1))) { if (!current->right) { @@ -116,6 +116,7 @@ make_huffman_tree() current->ascii_code = i; current->leaf_node = true; } + return root; } diff --git a/proxy/http2/Makefile.am b/proxy/http2/Makefile.am index ad08f7e7318..47cbb8e7577 100644 --- a/proxy/http2/Makefile.am +++ b/proxy/http2/Makefile.am @@ -95,7 +95,6 @@ test_HPACK_LDADD = \ $(top_builddir)/lib/records/librecords_p.a \ $(top_builddir)/mgmt/libmgmt_p.la \ $(top_builddir)/proxy/shared/libUglyLogStubs.a \ - @LIBTCL@ \ @HWLOC_LIBS@ test_HPACK_SOURCES = \ diff --git a/proxy/http2/RegressionHPACK.cc b/proxy/http2/RegressionHPACK.cc index 7b2de58141a..e0cdadd6d50 100644 --- a/proxy/http2/RegressionHPACK.cc +++ b/proxy/http2/RegressionHPACK.cc @@ -47,7 +47,7 @@ const static struct { uint8_t *encoded_field; int encoded_field_len; int prefix; -} integer_test_case[] = {{10, (uint8_t *)"\x0A", 1, 5}, {1337, (uint8_t *)"\x1F\x9A\x0A", 3, 5}, {42, (uint8_t *)"\x2A", 1, 8}}; +} integer_test_case[] = {{10, (uint8_t *)"\x0A", 1, 5}, {1337, (uint8_t *)"\x1F\x9A\x0A", 3, 5}, {42, (uint8_t *)R"(*)", 1, 8}}; // Example: custom-key: custom-header const static struct { diff --git a/proxy/http2/test_HPACK.cc b/proxy/http2/test_HPACK.cc index b1d4ed459d1..be29e8fdd6a 100644 --- a/proxy/http2/test_HPACK.cc +++ b/proxy/http2/test_HPACK.cc @@ -67,10 +67,9 @@ int unpack(string &packed, uint8_t *unpacked) { int n = packed.length() / 2; - int u, l; for (int i = 0; i < n; ++i) { - u = packed[i * 2]; - l = packed[i * 2 + 1]; + int u = packed[i * 2]; + int l = packed[i * 2 + 1]; unpacked[i] = (((u >= 'a') ? u - 'a' + 10 : u - '0') << 4) + ((l >= 'a') ? l - 'a' + 10 : l - '0'); } return n; @@ -129,21 +128,21 @@ print_difference(const char *a_str, const int a_str_len, const char *b_str, cons int compare_header_fields(HTTPHdr *a, HTTPHdr *b) { - const char *a_str, *b_str; - int a_str_len, b_str_len; + // compare fields count + if (a->fields_count() != b->fields_count()) { + return -1; + } + MIMEFieldIter a_iter, b_iter; const MIMEField *a_field = a->iter_get_first(&a_iter); const MIMEField *b_field = b->iter_get_first(&b_iter); - // compare fields count - if (a->fields_count() != b->fields_count()) { - return -1; - } - for (; a_field != nullptr; a_field = a->iter_get_next(&a_iter), b_field = b->iter_get_next(&b_iter)) { + while (a_field != nullptr && b_field != nullptr) { + int a_str_len, b_str_len; // compare header name - a_str = a_field->name_get(&a_str_len); - b_str = b_field->name_get(&b_str_len); + const char *a_str = a_field->name_get(&a_str_len); + const char *b_str = b_field->name_get(&b_str_len); if (a_str_len != b_str_len) { if (memcmp(a_str, b_str, a_str_len) != 0) { print_difference(a_str, a_str_len, b_str, b_str_len); @@ -159,6 +158,9 @@ compare_header_fields(HTTPHdr *a, HTTPHdr *b) return -1; } } + + a_field = a->iter_get_next(&a_iter); + b_field = b->iter_get_next(&b_iter); } return 0; diff --git a/proxy/http2/unit_tests/test_Http2DependencyTree.cc b/proxy/http2/unit_tests/test_Http2DependencyTree.cc index b72bbce4dcd..6a00b1ae4c3 100644 --- a/proxy/http2/unit_tests/test_Http2DependencyTree.cc +++ b/proxy/http2/unit_tests/test_Http2DependencyTree.cc @@ -282,7 +282,7 @@ TEST_CASE("Http2DependencyTree_Chrome_50", "[http2][Http2DependencyTree]") ostringstream oss; - for (int i = 0; i < 108; ++i) { + for (int index = 0; index < 108; ++index) { Node *node = tree->top(); oss << static_cast(node->t)->c_str(); @@ -336,7 +336,7 @@ TEST_CASE("Http2DependencyTree_Chrome_51", "[http2][Http2DependencyTree]") ostringstream oss; - for (int i = 0; i < 9; ++i) { + for (int index = 0; index < 9; ++index) { Node *node = tree->top(); if (node != nullptr) { oss << static_cast(node->t)->c_str(); @@ -352,7 +352,7 @@ TEST_CASE("Http2DependencyTree_Chrome_51", "[http2][Http2DependencyTree]") tree->activate(node_f); tree->activate(node_h); - for (int i = 0; i < 9; ++i) { + for (int index = 0; index < 9; ++index) { Node *node = tree->top(); if (node != nullptr) { oss << static_cast(node->t)->c_str(); diff --git a/proxy/logging/Log.cc b/proxy/logging/Log.cc index 3d11a750e10..090b6a5c695 100644 --- a/proxy/logging/Log.cc +++ b/proxy/logging/Log.cc @@ -33,6 +33,7 @@ ***************************************************************************/ #include "tscore/ink_platform.h" +#include "tscore/TSSystemState.h" #include "P_EventSystem.h" #include "P_Net.h" #include "I_Machine.h" @@ -43,13 +44,11 @@ #include "LogFilter.h" #include "LogFormat.h" #include "LogFile.h" -#include "LogHost.h" #include "LogObject.h" #include "LogConfig.h" #include "LogBuffer.h" #include "LogUtils.h" #include "Log.h" -#include "LogSock.h" #include "tscore/SimpleTokenizer.h" #include "tscore/ink_apidefs.h" @@ -68,14 +67,8 @@ EventNotify *Log::preproc_notify; EventNotify *Log::flush_notify; InkAtomicList *Log::flush_data_list; -// Collate thread stuff -EventNotify Log::collate_notify; -ink_thread Log::collate_thread; -int Log::collation_accept_file_descriptor; -int Log::collation_preproc_threads; -int Log::collation_port; - // Log private objects +int Log::preproc_threads; int Log::init_status = 0; int Log::config_flags = 0; bool Log::logging_mode_changed = false; @@ -226,7 +219,7 @@ Log::periodic_tasks(long time_now) // so that log objects are flushed // change_configuration(); - } else if (logging_mode > LOG_MODE_NONE || config->collation_mode == Log::COLLATION_HOST || config->has_api_objects()) { + } else if (logging_mode > LOG_MODE_NONE || config->has_api_objects()) { Debug("log-periodic", "Performing periodic tasks"); Debug("log-periodic", "Periodic task interval = %d", periodic_tasks_interval); @@ -277,7 +270,10 @@ struct LoggingPreprocContinuation : public Continuation { return 0; } - LoggingPreprocContinuation(int idx) : Continuation(nullptr), m_idx(idx) { SET_HANDLER(&LoggingPreprocContinuation::mainEvent); } + explicit LoggingPreprocContinuation(int idx) : Continuation(nullptr), m_idx(idx) + { + SET_HANDLER(&LoggingPreprocContinuation::mainEvent); + } }; struct LoggingFlushContinuation : public Continuation { @@ -290,18 +286,10 @@ struct LoggingFlushContinuation : public Continuation { return 0; } - LoggingFlushContinuation(int idx) : Continuation(nullptr), m_idx(idx) { SET_HANDLER(&LoggingFlushContinuation::mainEvent); } -}; - -struct LoggingCollateContinuation : public Continuation { - int - mainEvent(int /* event ATS_UNUSED */, void * /* data ATS_UNUSED */) + explicit LoggingFlushContinuation(int idx) : Continuation(nullptr), m_idx(idx) { - Log::collate_thread_main(nullptr); - return 0; + SET_HANDLER(&LoggingFlushContinuation::mainEvent); } - - LoggingCollateContinuation() : Continuation(nullptr) { SET_HANDLER(&LoggingCollateContinuation::mainEvent); } }; /*------------------------------------------------------------------------- @@ -497,6 +485,16 @@ Log::init_fields() global_field_list.add(field, false); field_symbol_hash.emplace("cqssr", field); + field = new LogField("client_req_is_internal", "cqint", LogField::sINT, &LogAccess::marshal_client_req_is_internal, + &LogAccess::unmarshal_int_to_str); + global_field_list.add(field, false); + field_symbol_hash.emplace("cqint", field); + + field = new LogField("client_req_mptcp", "cqmpt", LogField::sINT, &LogAccess::marshal_client_req_mptcp_state, + &LogAccess::unmarshal_int_to_str); + global_field_list.add(field, false); + field_symbol_hash.emplace("cqmpt", field); + field = new LogField("client_sec_protocol", "cqssv", LogField::STRING, &LogAccess::marshal_client_security_protocol, (LogField::UnmarshalFunc)&LogAccess::unmarshal_str); global_field_list.add(field, false); @@ -535,6 +533,11 @@ Log::init_fields() global_field_list.add(field, false); field_symbol_hash.emplace("ctec", field); + field = new LogField("client_request_all_header_fields", "cqah", LogField::STRING, + &LogAccess::marshal_client_req_all_header_fields, &LogUtils::unmarshalMimeHdr); + global_field_list.add(field, false); + field_symbol_hash.emplace("cqah", field); + // proxy -> client fields field = new LogField("proxy_resp_content_type", "psct", LogField::STRING, &LogAccess::marshal_proxy_resp_content_type, (LogField::UnmarshalFunc)&LogAccess::unmarshal_str); @@ -632,6 +635,11 @@ Log::init_fields() global_field_list.add(field, false); field_symbol_hash.emplace("chm", field); + field = new LogField("proxy_response_all_header_fields", "psah", LogField::STRING, + &LogAccess::marshal_proxy_resp_all_header_fields, &LogUtils::unmarshalMimeHdr); + global_field_list.add(field, false); + field_symbol_hash.emplace("psah", field); + // proxy -> server fields field = new LogField("proxy_req_header_len", "pqhl", LogField::sINT, &LogAccess::marshal_proxy_req_header_len, &LogAccess::unmarshal_int_to_str); @@ -710,6 +718,11 @@ Log::init_fields() global_field_list.add(field, false); field_symbol_hash.emplace("pqssl", field); + field = new LogField("proxy_request_all_header_fields", "pqah", LogField::STRING, &LogAccess::marshal_proxy_req_all_header_fields, + &LogUtils::unmarshalMimeHdr); + global_field_list.add(field, false); + field_symbol_hash.emplace("pqah", field); + // server -> proxy fields field = new LogField("server_host_ip", "shi", LogField::IP, &LogAccess::marshal_server_host_ip, &LogAccess::unmarshal_ip_to_str); @@ -776,6 +789,11 @@ Log::init_fields() global_field_list.add(field, false); field_symbol_hash.emplace("sca", field); + field = new LogField("origin_response_all_header_fields", "ssah", LogField::STRING, + &LogAccess::marshal_server_resp_all_header_fields, &LogUtils::unmarshalMimeHdr); + global_field_list.add(field, false); + field_symbol_hash.emplace("ssah", field); + field = new LogField("cached_resp_status_code", "csssc", LogField::sINT, &LogAccess::marshal_cache_resp_status_code, &LogAccess::unmarshal_http_status); global_field_list.add(field, false); @@ -801,6 +819,11 @@ Log::init_fields() global_field_list.add(field, false); field_symbol_hash.emplace("csshv", field); + field = new LogField("cache_origin_response_all_header_fields", "cssah", LogField::STRING, + &LogAccess::marshal_cache_resp_all_header_fields, &LogUtils::unmarshalMimeHdr); + global_field_list.add(field, false); + field_symbol_hash.emplace("cssah", field); + field = new LogField("client_retry_after_time", "crat", LogField::sINT, &LogAccess::marshal_client_retry_after_time, &LogAccess::unmarshal_int_to_str); global_field_list.add(field, false); @@ -893,8 +916,7 @@ Log::handle_periodic_tasks_int_change(const char * /* name ATS_UNUSED */, RecDat void Log::init(int flags) { - collation_preproc_threads = 1; - collation_accept_file_descriptor = NO_FD; + preproc_threads = 1; // store the configuration flags // @@ -915,33 +937,28 @@ Log::init(int flags) LogConfig::register_stat_callbacks(); config->read_configuration_variables(); - collation_port = config->collation_port; - collation_preproc_threads = config->collation_preproc_threads; - - if (config_flags & STANDALONE_COLLATOR) { - logging_mode = LOG_MODE_TRANSACTIONS; + preproc_threads = config->preproc_threads; + + int val = (int)REC_ConfigReadInteger("proxy.config.log.logging_enabled"); + if (val < LOG_MODE_NONE || val > LOG_MODE_FULL) { + logging_mode = LOG_MODE_FULL; + Warning("proxy.config.log.logging_enabled has an invalid " + "value, setting it to %d", + logging_mode); } else { - int val = (int)REC_ConfigReadInteger("proxy.config.log.logging_enabled"); - if (val < LOG_MODE_NONE || val > LOG_MODE_FULL) { - logging_mode = LOG_MODE_FULL; - Warning("proxy.config.log.logging_enabled has an invalid " - "value, setting it to %d", - logging_mode); - } else { - logging_mode = (LoggingMode)val; - } - // periodic task interval are set on a per instance basis - MgmtInt pti = REC_ConfigReadInteger("proxy.config.log.periodic_tasks_interval"); - if (pti <= 0) { - Error("proxy.config.log.periodic_tasks_interval = %" PRId64 " is invalid", pti); - Note("falling back to default periodic tasks interval = %d", PERIODIC_TASKS_INTERVAL_FALLBACK); - periodic_tasks_interval = PERIODIC_TASKS_INTERVAL_FALLBACK; - } else { - periodic_tasks_interval = static_cast(pti); - } - - REC_RegisterConfigUpdateFunc("proxy.config.log.periodic_tasks_interval", &Log::handle_periodic_tasks_int_change, nullptr); + logging_mode = (LoggingMode)val; + } + // periodic task interval are set on a per instance basis + MgmtInt pti = REC_ConfigReadInteger("proxy.config.log.periodic_tasks_interval"); + if (pti <= 0) { + Error("proxy.config.log.periodic_tasks_interval = %" PRId64 " is invalid", pti); + Note("falling back to default periodic tasks interval = %d", PERIODIC_TASKS_INTERVAL_FALLBACK); + periodic_tasks_interval = PERIODIC_TASKS_INTERVAL_FALLBACK; + } else { + periodic_tasks_interval = static_cast(pti); } + + REC_RegisterConfigUpdateFunc("proxy.config.log.periodic_tasks_interval", &Log::handle_periodic_tasks_int_change, nullptr); } // if remote management is enabled, do all necessary initialization to @@ -950,8 +967,6 @@ Log::init(int flags) if (!(config_flags & NO_REMOTE_MANAGEMENT)) { REC_RegisterConfigUpdateFunc("proxy.config.log.logging_enabled", &Log::handle_logging_mode_change, nullptr); - REC_RegisterConfigUpdateFunc("proxy.local.log.collation_mode", &Log::handle_logging_mode_change, nullptr); - // Clear any stat values that need to be reset on startup // RecSetRawStatSum(log_rsb, log_stat_log_files_open_stat, 0); @@ -961,9 +976,6 @@ Log::init(int flags) init_fields(); if (!(config_flags & LOGCAT)) { Debug("log-config", "Log::init(): logging_mode = %d init status = %d", logging_mode, init_status); - if (config_flags & STANDALONE_COLLATOR) { - config->collation_mode = Log::COLLATION_HOST; - } config->init(); init_when_enabled(); } @@ -976,26 +988,24 @@ Log::init_when_enabled() ink_release_assert(config->initialized == true); if (!(init_status & FULLY_INITIALIZED)) { - if (!(config_flags & STANDALONE_COLLATOR)) { - // register callbacks - // - if (!(config_flags & NO_REMOTE_MANAGEMENT)) { - LogConfig::register_config_callbacks(); - } - - LogConfig::register_mgmt_callbacks(); + // register callbacks + // + if (!(config_flags & NO_REMOTE_MANAGEMENT)) { + LogConfig::register_config_callbacks(); } + + LogConfig::register_mgmt_callbacks(); // setup global scrap object // global_scrap_format = MakeTextLogFormat(); global_scrap_object = new LogObject(global_scrap_format, Log::config->logfile_dir, "scrapfile.log", LOG_FILE_BINARY, nullptr, - Log::config->rolling_enabled, Log::config->collation_preproc_threads, Log::config->rolling_interval_sec, + Log::config->rolling_enabled, Log::config->preproc_threads, Log::config->rolling_interval_sec, Log::config->rolling_offset_hr, Log::config->rolling_size_mb); - // create the flush thread and the collation thread + // create the flush thread create_threads(); - eventProcessor.schedule_every(new PeriodicWakeup(collation_preproc_threads, 1), HRTIME_SECOND, ET_CALL); + eventProcessor.schedule_every(new PeriodicWakeup(preproc_threads, 1), HRTIME_SECOND, ET_CALL); init_status |= FULLY_INITIALIZED; } @@ -1010,7 +1020,7 @@ void Log::create_threads() { char desc[64]; - preproc_notify = new EventNotify[collation_preproc_threads]; + preproc_notify = new EventNotify[preproc_threads]; size_t stacksize; REC_ReadConfigInteger(stacksize, "proxy.config.thread.default.stacksize"); @@ -1019,7 +1029,7 @@ Log::create_threads() // // no need for the conditional var since it will be relying on // on the event system. - for (int i = 0; i < collation_preproc_threads; i++) { + for (int i = 0; i < preproc_threads; i++) { Continuation *preproc_cont = new LoggingPreprocContinuation(i); sprintf(desc, "[LOG_PREPROC %d]", i); eventProcessor.spawn_thread(preproc_cont, desc, stacksize); @@ -1213,14 +1223,13 @@ Log::preproc_thread_main(void *args) Log::preproc_notify[idx].lock(); while (true) { - if (unlikely(shutdown_event_system == true)) { + if (TSSystemState::is_event_system_shut_down()) { return nullptr; } - size_t buffers_preproced = 0; - LogConfig *current = (LogConfig *)configProcessor.get(log_configid); + LogConfig *current = static_cast(configProcessor.get(log_configid)); if (likely(current)) { - buffers_preproced = current->log_object_manager.preproc_buffers(idx); + size_t buffers_preproced = current->log_object_manager.preproc_buffers(idx); // config->increment_space_used(bytes_to_disk); // TODO: the bytes_to_disk should be set to Log @@ -1256,10 +1265,10 @@ Log::flush_thread_main(void * /* args ATS_UNUSED */) Log::flush_notify->lock(); while (true) { - if (unlikely(shutdown_event_system == true)) { + if (TSSystemState::is_event_system_shut_down()) { return nullptr; } - fdata = (LogFlushData *)ink_atomiclist_popall(flush_data_list); + fdata = static_cast(ink_atomiclist_popall(flush_data_list)); // invert the list // @@ -1276,7 +1285,7 @@ Log::flush_thread_main(void * /* args ATS_UNUSED */) LogFile *logfile = fdata->m_logfile.get(); if (logfile->m_file_format == LOG_FILE_BINARY) { - logbuffer = (LogBuffer *)fdata->m_data; + logbuffer = static_cast(fdata->m_data); LogBufferHeader *buffer_header = logbuffer->header(); buf = (char *)buffer_header; @@ -1359,166 +1368,3 @@ Log::flush_thread_main(void * /* args ATS_UNUSED */) Log::flush_notify->unlock(); return nullptr; } - -/*------------------------------------------------------------------------- - Log::collate_thread_main - - This function defines the functionality of the log collation thread, - whose purpose is to collate log buffers from other nodes. - -------------------------------------------------------------------------*/ - -void * -Log::collate_thread_main(void * /* args ATS_UNUSED */) -{ - LogSock *sock; - LogBufferHeader *header; - LogFormat *format; - LogObject *obj; - int bytes_read; - int sock_id; - int new_client; - - Debug("log-thread", "Log collation thread is alive ..."); - - Log::collate_notify.lock(); - - while (true) { - ink_assert(Log::config != nullptr); - - // wait on the collation condition variable until we're sure that - // we're a collation host. The while loop guards against spurious - // wake-ups. - // - while (!Log::config->am_collation_host()) { - Log::collate_notify.wait(); - } - - // Ok, at this point we know we're a log collation host, so get to - // work. We still need to keep checking whether we're a collation - // host to account for a reconfiguration. - // - Debug("log-sock", "collation thread starting, creating LogSock"); - sock = new LogSock(LogSock::LS_CONST_MAX_CONNS); - ink_assert(sock != nullptr); - - if (sock->listen(Log::config->collation_port) != 0) { - LogUtils::manager_alarm(LogUtils::LOG_ALARM_ERROR, "Collation server error; could not listen on port %d", - Log::config->collation_port); - Warning("Collation server error; could not listen on port %d", Log::config->collation_port); - delete sock; - // - // go to sleep ... - // - Log::collate_notify.wait(); - continue; - } - - while (true) { - if (!Log::config->am_collation_host()) { - break; - } - - if (sock->pending_connect(0)) { - Debug("log-sock", "pending connection ..."); - if ((new_client = sock->accept()) < 0) { - Debug("log-sock", "error accepting new collation client"); - } else { - Debug("log-sock", "connection %d accepted", new_client); - if (!sock->authorized_client(new_client, Log::config->collation_secret)) { - Warning("Unauthorized client connecting to " - "log collation port; connection refused."); - sock->close(new_client); - } - } - } - - sock->check_connections(); - - if (!sock->pending_message_any(&sock_id, 0)) { - continue; - } - - Debug("log-sock", "pending message ..."); - header = (LogBufferHeader *)sock->read_alloc(sock_id, &bytes_read); - if (!header) { - Debug("log-sock", "Error reading LogBuffer from collation client"); - continue; - } - - if (header->version != LOG_SEGMENT_VERSION) { - Note("Invalid LogBuffer received; invalid version - buffer = %u, current = %u", header->version, LOG_SEGMENT_VERSION); - delete[] header; - continue; - } - - Debug("log-sock", "message accepted, size = %d", bytes_read); - - obj = match_logobject(header); - if (!obj) { - Note("LogObject not found with fieldlist id; " - "writing LogBuffer to scrap file"); - obj = global_scrap_object; - } - - format = obj->m_format; - Debug("log-sock", "Using format '%s'", format->name()); - - delete[] header; - } - - Debug("log", "no longer collation host, deleting LogSock"); - delete sock; - } - - /* NOTREACHED */ - Log::collate_notify.unlock(); - return nullptr; -} - -/*------------------------------------------------------------------------- - Log::match_logobject - - This routine matches the given buffer with the local list of LogObjects. - If a match cannot be found, then we'll try to construct a local LogObject - using the information provided in the header. If all else fails, we - return NULL. - -------------------------------------------------------------------------*/ - -LogObject * -Log::match_logobject(LogBufferHeader *header) -{ - if (!header) { - return nullptr; - } - - LogObject *obj; - obj = Log::config->log_object_manager.get_object_with_signature(header->log_object_signature); - - if (!obj) { - // object does not exist yet, create it - // - LogFormat fmt("__collation_format__", header->fmt_fieldlist(), header->fmt_printf()); - - if (fmt.valid()) { - LogFileFormat file_format = header->log_object_flags & LogObject::BINARY ? - LOG_FILE_BINARY : - (header->log_object_flags & LogObject::WRITES_TO_PIPE ? LOG_FILE_PIPE : LOG_FILE_ASCII); - - obj = new LogObject(&fmt, Log::config->logfile_dir, header->log_filename(), file_format, nullptr, - (Log::RollingEnabledValues)Log::config->rolling_enabled, Log::config->collation_preproc_threads, - Log::config->rolling_interval_sec, Log::config->rolling_offset_hr, Log::config->rolling_size_mb, true); - - obj->set_remote_flag(); - - if (Log::config->log_object_manager.manage_object(obj)) { - // object manager can't solve filename conflicts - // delete the object and return NULL - // - delete obj; - obj = nullptr; - } - } - } - - return obj; -} diff --git a/proxy/logging/Log.h b/proxy/logging/Log.h index 00e0d1bb8f5..01700a60818 100644 --- a/proxy/logging/Log.h +++ b/proxy/logging/Log.h @@ -87,7 +87,7 @@ class LogFlushData { switch (m_logfile->m_file_format) { case LOG_FILE_BINARY: - logbuffer = (LogBuffer *)m_data; + logbuffer = static_cast(m_data); LogBuffer::destroy(logbuffer); break; case LOG_FILE_ASCII: @@ -132,12 +132,9 @@ class Log enum ConfigFlags { NO_REMOTE_MANAGEMENT = 1, - STANDALONE_COLLATOR = 2, LOGCAT = 4, }; - enum CollationMode { NO_COLLATION = 0, COLLATION_HOST, N_COLLATION_MODES }; - enum RollingEnabledValues { NO_ROLLING = 0, ROLL_ON_TIME_ONLY, @@ -195,14 +192,7 @@ class Log static InkAtomicList *flush_data_list; static void *flush_thread_main(void *args); - // collation thread stuff - static EventNotify collate_notify; - static ink_thread collate_thread; - static int collation_preproc_threads; - static int collation_accept_file_descriptor; - static int collation_port; - static void *collate_thread_main(void *args); - static LogObject *match_logobject(LogBufferHeader *header); + static int preproc_threads; // reconfiguration stuff static void change_configuration(); diff --git a/proxy/logging/LogAccess.cc b/proxy/logging/LogAccess.cc index 586676b09b8..43e5c751e86 100644 --- a/proxy/logging/LogAccess.cc +++ b/proxy/logging/LogAccess.cc @@ -359,6 +359,15 @@ LogAccess::marshal_str(char *dest, const char *source, int padded_len) #endif } +/*------------------------------------------------------------------------- + -------------------------------------------------------------------------*/ + +int +LogAccess::marshal_client_req_all_header_fields(char *buf) +{ + return LogUtils::marshalMimeHdr(m_client_request, buf); +} + /*------------------------------------------------------------------------- LogAccess::marshal_mem @@ -429,7 +438,7 @@ LogAccess::marshal_ip(char *dest, sockaddr const *ip) } inline int -LogAccess::unmarshal_with_map(int64_t code, char *dest, int len, Ptr map, const char *msg) +LogAccess::unmarshal_with_map(int64_t code, char *dest, int len, const Ptr &map, const char *msg) { long int codeStrLen = 0; @@ -476,6 +485,18 @@ LogAccess::unmarshal_int(char **buf) return val; } +/*------------------------------------------------------------------------- + -------------------------------------------------------------------------*/ + +int +LogAccess::marshal_proxy_resp_all_header_fields(char *buf) +{ + return LogUtils::marshalMimeHdr(m_proxy_response, buf); +} + +/*------------------------------------------------------------------------- + -------------------------------------------------------------------------*/ + /*------------------------------------------------------------------------- unmarshal_itoa @@ -490,23 +511,27 @@ LogAccess::unmarshal_itoa(int64_t val, char *dest, int field_width, char leading { ink_assert(dest != nullptr); - char *p = dest; + char *p = dest; + bool negative = false; - if (val <= 0) { - *p-- = '0'; - while (dest - p < field_width) { - *p-- = leading_char; - } - return (int)(dest - p); + if (val < 0) { + negative = true; + val = -val; } - while (val) { + do { *p-- = '0' + (val % 10); val /= 10; - } + } while (val); + while (dest - p < field_width) { *p-- = leading_char; } + + if (negative) { + *p-- = '-'; + } + return (int)(dest - p); } @@ -586,6 +611,18 @@ LogAccess::unmarshal_int_to_str_hex(char **buf, char *dest, int len) return -1; } +/*------------------------------------------------------------------------- + -------------------------------------------------------------------------*/ + +int +LogAccess::marshal_proxy_req_all_header_fields(char *buf) +{ + return LogUtils::marshalMimeHdr(m_proxy_request, buf); +} + +/*------------------------------------------------------------------------- + -------------------------------------------------------------------------*/ + /*------------------------------------------------------------------------- LogAccess::unmarshal_str @@ -673,6 +710,18 @@ LogAccess::unmarshal_int_to_time_str(char **buf, char *dest, int len) return strlen; } +/*------------------------------------------------------------------------- + -------------------------------------------------------------------------*/ + +int +LogAccess::marshal_server_resp_all_header_fields(char *buf) +{ + return LogUtils::marshalMimeHdr(m_server_response, buf); +} + +/*------------------------------------------------------------------------- + -------------------------------------------------------------------------*/ + int LogAccess::unmarshal_int_to_netscape_str(char **buf, char *dest, int len) { @@ -702,6 +751,13 @@ LogAccess::unmarshal_http_method (char **buf, char *dest, int len) return unmarshal_str (buf, dest, len); } */ + +int +LogAccess::marshal_cache_resp_all_header_fields(char *buf) +{ + return LogUtils::marshalMimeHdr(m_cache_response, buf); +} + /*------------------------------------------------------------------------- LogAccess::unmarshal_http_version @@ -899,7 +955,7 @@ LogAccess::unmarshal_ip_to_hex(char **buf, char *dest, int len) -------------------------------------------------------------------------*/ int -LogAccess::unmarshal_hierarchy(char **buf, char *dest, int len, Ptr map) +LogAccess::unmarshal_hierarchy(char **buf, char *dest, int len, const Ptr &map) { ink_assert(buf != nullptr); ink_assert(*buf != nullptr); @@ -917,7 +973,7 @@ LogAccess::unmarshal_hierarchy(char **buf, char *dest, int len, Ptr map) +LogAccess::unmarshal_finish_status(char **buf, char *dest, int len, const Ptr &map) { ink_assert(buf != nullptr); ink_assert(*buf != nullptr); @@ -935,7 +991,7 @@ LogAccess::unmarshal_finish_status(char **buf, char *dest, int len, Ptr map) +LogAccess::unmarshal_cache_code(char **buf, char *dest, int len, const Ptr &map) { ink_assert(buf != nullptr); ink_assert(*buf != nullptr); @@ -953,7 +1009,7 @@ LogAccess::unmarshal_cache_code(char **buf, char *dest, int len, Ptr map) +LogAccess::unmarshal_cache_hit_miss(char **buf, char *dest, int len, const Ptr &map) { ink_assert(buf != nullptr); ink_assert(*buf != nullptr); @@ -963,7 +1019,7 @@ LogAccess::unmarshal_cache_hit_miss(char **buf, char *dest, int len, Ptr map) +LogAccess::unmarshal_cache_write_code(char **buf, char *dest, int len, const Ptr &map) { ink_assert(buf != nullptr); ink_assert(*buf != nullptr); @@ -1276,9 +1332,6 @@ LogAccess::validate_unmapped_url() void LogAccess::validate_unmapped_url_path() { - int len; - char *c; - if (m_client_req_unmapped_url_path_str == nullptr && m_client_req_unmapped_url_host_str == nullptr) { // Use unmapped canonical URL as default m_client_req_unmapped_url_path_str = m_client_req_unmapped_url_canon_str; @@ -1287,7 +1340,9 @@ LogAccess::validate_unmapped_url_path() m_client_req_unmapped_url_host_str = INVALID_STR; if (m_client_req_unmapped_url_path_len >= 6) { // xxx:// - minimum schema size - c = (char *)memchr((void *)m_client_req_unmapped_url_path_str, ':', m_client_req_unmapped_url_path_len - 1); + int len; + char *c = (char *)memchr((void *)m_client_req_unmapped_url_path_str, ':', m_client_req_unmapped_url_path_len - 1); + if (c && (len = (int)(c - m_client_req_unmapped_url_path_str)) <= 5) { // 5 - max schema size if (len + 2 <= m_client_req_unmapped_url_canon_len && c[1] == '/' && c[2] == '/') { len += 3; // Skip "://" @@ -1637,37 +1692,49 @@ int LogAccess::marshal_client_req_tcp_reused(char *buf) { if (buf) { - int64_t tcp_reused; - tcp_reused = m_http_sm->client_tcp_reused; - marshal_int(buf, tcp_reused); + marshal_int(buf, m_http_sm->client_tcp_reused ? 1 : 0); } return INK_MIN_ALIGN; } -/*------------------------------------------------------------------------- - -------------------------------------------------------------------------*/ - int LogAccess::marshal_client_req_is_ssl(char *buf) { if (buf) { - int64_t is_ssl; - is_ssl = m_http_sm->client_connection_is_ssl; - marshal_int(buf, is_ssl); + marshal_int(buf, m_http_sm->client_connection_is_ssl ? 1 : 0); } return INK_MIN_ALIGN; } -/*------------------------------------------------------------------------- - -------------------------------------------------------------------------*/ - int LogAccess::marshal_client_req_ssl_reused(char *buf) { if (buf) { - int64_t ssl_session_reused; - ssl_session_reused = m_http_sm->client_ssl_reused; - marshal_int(buf, ssl_session_reused); + marshal_int(buf, m_http_sm->client_ssl_reused ? 1 : 0); + } + return INK_MIN_ALIGN; +} + +int +LogAccess::marshal_client_req_is_internal(char *buf) +{ + if (buf) { + marshal_int(buf, m_http_sm->is_internal ? 1 : 0); + } + return INK_MIN_ALIGN; +} + +int +LogAccess::marshal_client_req_mptcp_state(char *buf) +{ + if (buf) { + int val = -1; + + if (m_http_sm->mptcp_state.has_value()) { + val = m_http_sm->mptcp_state.value() ? 1 : 0; + } else { + } + marshal_int(buf, val); } return INK_MIN_ALIGN; } @@ -2197,9 +2264,7 @@ int LogAccess::marshal_server_resp_time_ms(char *buf) { if (buf) { - ink_hrtime elapsed = m_http_sm->milestones[TS_MILESTONE_SERVER_CLOSE] - m_http_sm->milestones[TS_MILESTONE_SERVER_CONNECT]; - int64_t val = (int64_t)ink_hrtime_to_msec(elapsed); - marshal_int(buf, val); + marshal_int(buf, m_http_sm->milestones.difference_msec(TS_MILESTONE_SERVER_CONNECT, TS_MILESTONE_SERVER_CLOSE)); } return INK_MIN_ALIGN; } @@ -2208,9 +2273,8 @@ int LogAccess::marshal_server_resp_time_s(char *buf) { if (buf) { - ink_hrtime elapsed = m_http_sm->milestones[TS_MILESTONE_SERVER_CLOSE] - m_http_sm->milestones[TS_MILESTONE_SERVER_CONNECT]; - int64_t val = (int64_t)ink_hrtime_to_sec(elapsed); - marshal_int(buf, val); + marshal_int(buf, + static_cast(m_http_sm->milestones.difference_sec(TS_MILESTONE_SERVER_CONNECT, TS_MILESTONE_SERVER_CLOSE))); } return INK_MIN_ALIGN; } @@ -2393,9 +2457,7 @@ int LogAccess::marshal_transfer_time_ms(char *buf) { if (buf) { - ink_hrtime elapsed = m_http_sm->milestones[TS_MILESTONE_SM_FINISH] - m_http_sm->milestones[TS_MILESTONE_SM_START]; - int64_t val = (int64_t)ink_hrtime_to_msec(elapsed); - marshal_int(buf, val); + marshal_int(buf, m_http_sm->milestones.difference_msec(TS_MILESTONE_SM_START, TS_MILESTONE_SM_FINISH)); } return INK_MIN_ALIGN; } @@ -2404,9 +2466,7 @@ int LogAccess::marshal_transfer_time_s(char *buf) { if (buf) { - ink_hrtime elapsed = m_http_sm->milestones[TS_MILESTONE_SM_FINISH] - m_http_sm->milestones[TS_MILESTONE_SM_START]; - int64_t val = (int64_t)ink_hrtime_to_sec(elapsed); - marshal_int(buf, val); + marshal_int(buf, static_cast(m_http_sm->milestones.difference_sec(TS_MILESTONE_SM_START, TS_MILESTONE_SM_FINISH))); } return INK_MIN_ALIGN; } @@ -2718,8 +2778,7 @@ int LogAccess::marshal_milestone_diff(TSMilestonesType ms1, TSMilestonesType ms2, char *buf) { if (buf) { - ink_hrtime elapsed = m_http_sm->milestones.elapsed(ms2, ms1); - int64_t val = (int64_t)ink_hrtime_to_msec(elapsed); + int64_t val = m_http_sm->milestones.difference_msec(ms2, ms1); marshal_int(buf, val); } return INK_MIN_ALIGN; diff --git a/proxy/logging/LogAccess.h b/proxy/logging/LogAccess.h index f402ad97186..1d35627904d 100644 --- a/proxy/logging/LogAccess.h +++ b/proxy/logging/LogAccess.h @@ -118,7 +118,7 @@ class LogAccess { } - LogAccess(HttpSM *sm); + explicit LogAccess(HttpSM *sm); inkcoreapi ~LogAccess() {} inkcoreapi void init(); @@ -149,6 +149,8 @@ class LogAccess inkcoreapi int marshal_client_req_tcp_reused(char *); // INT inkcoreapi int marshal_client_req_is_ssl(char *); // INT inkcoreapi int marshal_client_req_ssl_reused(char *); // INT + inkcoreapi int marshal_client_req_is_internal(char *); // INT + inkcoreapi int marshal_client_req_mptcp_state(char *); // INT inkcoreapi int marshal_client_security_protocol(char *); // STR inkcoreapi int marshal_client_security_cipher_suite(char *); // STR inkcoreapi int marshal_client_finish_status_code(char *); // INT @@ -156,60 +158,65 @@ class LogAccess inkcoreapi int marshal_client_req_uuid(char *); // STR inkcoreapi int marshal_client_rx_error_code(char *); // STR inkcoreapi int marshal_client_tx_error_code(char *); // STR + inkcoreapi int marshal_client_req_all_header_fields(char *); // STR // // proxy -> client fields // - inkcoreapi int marshal_proxy_resp_content_type(char *); // STR - inkcoreapi int marshal_proxy_resp_reason_phrase(char *); // STR - inkcoreapi int marshal_proxy_resp_squid_len(char *); // INT - inkcoreapi int marshal_proxy_resp_content_len(char *); // INT - inkcoreapi int marshal_proxy_resp_status_code(char *); // INT - inkcoreapi int marshal_proxy_resp_header_len(char *); // INT - inkcoreapi int marshal_proxy_finish_status_code(char *); // INT - inkcoreapi int marshal_cache_result_code(char *); // INT - inkcoreapi int marshal_cache_result_subcode(char *); // INT - inkcoreapi int marshal_proxy_host_port(char *); // INT - inkcoreapi int marshal_cache_hit_miss(char *); // INT + inkcoreapi int marshal_proxy_resp_content_type(char *); // STR + inkcoreapi int marshal_proxy_resp_reason_phrase(char *); // STR + inkcoreapi int marshal_proxy_resp_squid_len(char *); // INT + inkcoreapi int marshal_proxy_resp_content_len(char *); // INT + inkcoreapi int marshal_proxy_resp_status_code(char *); // INT + inkcoreapi int marshal_proxy_resp_header_len(char *); // INT + inkcoreapi int marshal_proxy_finish_status_code(char *); // INT + inkcoreapi int marshal_cache_result_code(char *); // INT + inkcoreapi int marshal_cache_result_subcode(char *); // INT + inkcoreapi int marshal_proxy_host_port(char *); // INT + inkcoreapi int marshal_cache_hit_miss(char *); // INT + inkcoreapi int marshal_proxy_resp_all_header_fields(char *); // STR // // proxy -> server fields // - inkcoreapi int marshal_proxy_req_header_len(char *); // INT - inkcoreapi int marshal_proxy_req_squid_len(char *); // INT - inkcoreapi int marshal_proxy_req_content_len(char *); // INT - inkcoreapi int marshal_proxy_req_server_ip(char *); // INT - inkcoreapi int marshal_proxy_req_server_port(char *); // INT - inkcoreapi int marshal_proxy_hierarchy_route(char *); // INT - inkcoreapi int marshal_next_hop_ip(char *); // STR - inkcoreapi int marshal_next_hop_port(char *); // INT - inkcoreapi int marshal_proxy_host_name(char *); // STR - inkcoreapi int marshal_proxy_host_ip(char *); // STR - inkcoreapi int marshal_proxy_req_is_ssl(char *); // INT + inkcoreapi int marshal_proxy_req_header_len(char *); // INT + inkcoreapi int marshal_proxy_req_squid_len(char *); // INT + inkcoreapi int marshal_proxy_req_content_len(char *); // INT + inkcoreapi int marshal_proxy_req_server_ip(char *); // INT + inkcoreapi int marshal_proxy_req_server_port(char *); // INT + inkcoreapi int marshal_proxy_hierarchy_route(char *); // INT + inkcoreapi int marshal_next_hop_ip(char *); // STR + inkcoreapi int marshal_next_hop_port(char *); // INT + inkcoreapi int marshal_proxy_host_name(char *); // STR + inkcoreapi int marshal_proxy_host_ip(char *); // STR + inkcoreapi int marshal_proxy_req_is_ssl(char *); // INT + inkcoreapi int marshal_proxy_req_all_header_fields(char *); // STR // // server -> proxy fields // - inkcoreapi int marshal_server_host_ip(char *); // INT - inkcoreapi int marshal_server_host_name(char *); // STR - inkcoreapi int marshal_server_resp_status_code(char *); // INT - inkcoreapi int marshal_server_resp_squid_len(char *); // INT - inkcoreapi int marshal_server_resp_content_len(char *); // INT - inkcoreapi int marshal_server_resp_header_len(char *); // INT - inkcoreapi int marshal_server_resp_http_version(char *); // INT - inkcoreapi int marshal_server_resp_time_ms(char *); // INT - inkcoreapi int marshal_server_resp_time_s(char *); // INT - inkcoreapi int marshal_server_transact_count(char *); // INT - inkcoreapi int marshal_server_connect_attempts(char *); // INT + inkcoreapi int marshal_server_host_ip(char *); // INT + inkcoreapi int marshal_server_host_name(char *); // STR + inkcoreapi int marshal_server_resp_status_code(char *); // INT + inkcoreapi int marshal_server_resp_squid_len(char *); // INT + inkcoreapi int marshal_server_resp_content_len(char *); // INT + inkcoreapi int marshal_server_resp_header_len(char *); // INT + inkcoreapi int marshal_server_resp_http_version(char *); // INT + inkcoreapi int marshal_server_resp_time_ms(char *); // INT + inkcoreapi int marshal_server_resp_time_s(char *); // INT + inkcoreapi int marshal_server_transact_count(char *); // INT + inkcoreapi int marshal_server_connect_attempts(char *); // INT + inkcoreapi int marshal_server_resp_all_header_fields(char *); // STR // // cache -> client fields // - inkcoreapi int marshal_cache_resp_status_code(char *); // INT - inkcoreapi int marshal_cache_resp_squid_len(char *); // INT - inkcoreapi int marshal_cache_resp_content_len(char *); // INT - inkcoreapi int marshal_cache_resp_header_len(char *); // INT - inkcoreapi int marshal_cache_resp_http_version(char *); // INT + inkcoreapi int marshal_cache_resp_status_code(char *); // INT + inkcoreapi int marshal_cache_resp_squid_len(char *); // INT + inkcoreapi int marshal_cache_resp_content_len(char *); // INT + inkcoreapi int marshal_cache_resp_header_len(char *); // INT + inkcoreapi int marshal_cache_resp_http_version(char *); // INT + inkcoreapi int marshal_cache_resp_all_header_fields(char *); // STR inkcoreapi void set_client_req_url(char *, int); // STR inkcoreapi void set_client_req_url_canon(char *, int); // STR @@ -294,14 +301,14 @@ class LogAccess static int unmarshal_ip(char **buf, IpEndpoint *dest); static int unmarshal_ip_to_str(char **buf, char *dest, int len); static int unmarshal_ip_to_hex(char **buf, char *dest, int len); - static int unmarshal_hierarchy(char **buf, char *dest, int len, Ptr map); - static int unmarshal_finish_status(char **buf, char *dest, int len, Ptr map); - static int unmarshal_cache_code(char **buf, char *dest, int len, Ptr map); - static int unmarshal_cache_hit_miss(char **buf, char *dest, int len, Ptr map); - static int unmarshal_cache_write_code(char **buf, char *dest, int len, Ptr map); + static int unmarshal_hierarchy(char **buf, char *dest, int len, const Ptr &map); + static int unmarshal_finish_status(char **buf, char *dest, int len, const Ptr &map); + static int unmarshal_cache_code(char **buf, char *dest, int len, const Ptr &map); + static int unmarshal_cache_hit_miss(char **buf, char *dest, int len, const Ptr &map); + static int unmarshal_cache_write_code(char **buf, char *dest, int len, const Ptr &map); static int unmarshal_client_protocol_stack(char **buf, char *dest, int len, Ptr map); - static int unmarshal_with_map(int64_t code, char *dest, int len, Ptr map, const char *msg = nullptr); + static int unmarshal_with_map(int64_t code, char *dest, int len, const Ptr &map, const char *msg = nullptr); static int unmarshal_record(char **buf, char *dest, int len); @@ -353,10 +360,10 @@ class LogAccess char *m_cache_lookup_url_canon_str; int m_cache_lookup_url_canon_len; - void validate_unmapped_url(void); - void validate_unmapped_url_path(void); + void validate_unmapped_url(); + void validate_unmapped_url_path(); - void validate_lookup_url(void); + void validate_lookup_url(); }; inline int diff --git a/proxy/logging/LogBuffer.cc b/proxy/logging/LogBuffer.cc index e12cd21a87a..b35669406cb 100644 --- a/proxy/logging/LogBuffer.cc +++ b/proxy/logging/LogBuffer.cc @@ -38,7 +38,6 @@ #include "LogFormat.h" #include "LogUtils.h" #include "LogFile.h" -#include "LogHost.h" #include "LogObject.h" #include "LogAccess.h" #include "LogConfig.h" @@ -261,7 +260,7 @@ LogBuffer::checkout_write(size_t *write_offset, size_t write_size) } if (switch_state(old_s, new_s)) { - // we succeded in setting the new state + // we succeeded in setting the new state break; } } diff --git a/proxy/logging/LogBuffer.h b/proxy/logging/LogBuffer.h index 3981c2907fc..1c64a18c4f6 100644 --- a/proxy/logging/LogBuffer.h +++ b/proxy/logging/LogBuffer.h @@ -261,9 +261,9 @@ class LogBufferList ~LogBufferList(); void add(LogBuffer *lb); - LogBuffer *get(void); + LogBuffer *get(); int - get_size(void) + get_size() { return m_size; } diff --git a/proxy/logging/LogCollationAccept.cc b/proxy/logging/LogCollationAccept.cc deleted file mode 100644 index 3e9414396ad..00000000000 --- a/proxy/logging/LogCollationAccept.cc +++ /dev/null @@ -1,105 +0,0 @@ -/** @file - - A brief file description - - @section license License - - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - */ - -//------------------------------------------------------------------------- -// include files -//------------------------------------------------------------------------- - -#include "tscore/ink_platform.h" -#include "P_EventSystem.h" - -#include "Log.h" -#include "LogCollationAccept.h" -#include "LogCollationHostSM.h" - -//------------------------------------------------------------------------- -// LogCollationAccept::LogCollationAccept -//------------------------------------------------------------------------- - -LogCollationAccept::LogCollationAccept(int port) : Continuation(new_ProxyMutex()), m_port(port) -{ - NetProcessor::AcceptOptions opt; - SET_HANDLER((LogCollationAcceptHandler)&LogCollationAccept::accept_event); - // work around for iocore problem where _pre_fetch_buffer can get - // appended to itself if multiple do_io_reads are called requesting - // small amounts of data. Most arguments are default except for the - // last one which we will set to true. - // [amc] That argument is ignored so I dropped it. - opt.local_port = m_port; - opt.ip_family = AF_INET; - opt.accept_threads = 0; - m_accept_action = netProcessor.accept(this, opt); - ink_assert(nullptr != m_accept_action); -} - -//------------------------------------------------------------------------- -// LogCollationAccept::~LogCollationAccept -//------------------------------------------------------------------------- - -LogCollationAccept::~LogCollationAccept() -{ - Debug("log-collation", "LogCollationAccept::~LogCollationAccept"); - - // stop the netProcessor - if (m_accept_action) { - m_accept_action->cancel(); - m_accept_action = nullptr; - - Debug("log-collation", - "closing Log::collation_accept_file_descriptor " - "(%d)", - Log::collation_accept_file_descriptor); - if (::close(Log::collation_accept_file_descriptor) < 0) { - Error("error closing collate listen file descriptor [%d]: %s", Log::collation_accept_file_descriptor, strerror(errno)); - } else { - Log::collation_accept_file_descriptor = NO_FD; - } - } else { - ink_assert(!"[ERROR] m_accept_action is NULL"); - } - - // stop the eventProcessor - // ... but what if there's more than one pending? - if (m_pending_event && (m_pending_event != ACTION_RESULT_DONE)) { - m_pending_event->cancel(); - } -} - -//------------------------------------------------------------------------- -// LogCollationAccept::accept_event -//------------------------------------------------------------------------- - -int -LogCollationAccept::accept_event(int event, NetVConnection *net_vc) -{ - switch (event) { - case NET_EVENT_ACCEPT: - new LogCollationHostSM(net_vc); - break; - - default: - ink_assert(!"[ERROR] Unexpected Event"); - } - - return EVENT_CONT; -} diff --git a/proxy/logging/LogCollationClientSM.cc b/proxy/logging/LogCollationClientSM.cc deleted file mode 100644 index 73051ccec94..00000000000 --- a/proxy/logging/LogCollationClientSM.cc +++ /dev/null @@ -1,716 +0,0 @@ -/** @file - - A brief file description - - @section license License - - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - */ - -//------------------------------------------------------------------------- -// include files -//------------------------------------------------------------------------- - -#include "tscore/ink_platform.h" - -#include -#include -#include -#include -#include - -#include "P_EventSystem.h" -#include "P_Net.h" - -#include "LogUtils.h" -#include "LogSock.h" -#include "LogField.h" -#include "LogFile.h" -#include "LogFormat.h" -#include "LogBuffer.h" -#include "LogHost.h" -#include "LogObject.h" -#include "LogConfig.h" -#include "Log.h" - -#include "LogCollationClientSM.h" - -//------------------------------------------------------------------------- -// statics -//------------------------------------------------------------------------- - -int LogCollationClientSM::ID = 0; - -//------------------------------------------------------------------------- -// LogCollationClientSM::LogCollationClientSM -//------------------------------------------------------------------------- - -LogCollationClientSM::LogCollationClientSM(LogHost *log_host) : Continuation(new_ProxyMutex()), m_log_host(log_host), m_id(ID++) -{ - Debug("log-coll", "[%d]client::constructor", m_id); - - ink_assert(m_log_host != nullptr); - - // allocate send_list before we do anything - // we can accept logs to send before we're fully initialized - m_buffer_send_list = new LogBufferList(); - ink_assert(m_buffer_send_list != nullptr); - - SET_HANDLER((LogCollationClientSMHandler)&LogCollationClientSM::client_handler); - client_init(LOG_COLL_EVENT_SWITCH, nullptr); -} - -//------------------------------------------------------------------------- -// LogCollationClientSM::~LogCollationClientSM -//------------------------------------------------------------------------- - -LogCollationClientSM::~LogCollationClientSM() -{ - Debug("log-coll", "[%d]client::destructor", m_id); - - ink_mutex_acquire(&(mutex->the_mutex)); - client_done(LOG_COLL_EVENT_SWITCH, nullptr); - ink_mutex_release(&(mutex->the_mutex)); -} - -//------------------------------------------------------------------------- -//------------------------------------------------------------------------- -// -// handler -// -//------------------------------------------------------------------------- -//------------------------------------------------------------------------- - -//------------------------------------------------------------------------- -// LogCollationClientSM::client_handler -//------------------------------------------------------------------------- - -int -LogCollationClientSM::client_handler(int event, void *data) -{ - switch (m_client_state) { - case LOG_COLL_CLIENT_AUTH: - return client_auth(event, (VIO *)data); - case LOG_COLL_CLIENT_DNS: - return client_dns(event, (HostDBInfo *)data); - case LOG_COLL_CLIENT_DONE: - return client_done(event, data); - case LOG_COLL_CLIENT_FAIL: - return client_fail(event, data); - case LOG_COLL_CLIENT_IDLE: - return client_idle(event, data); - case LOG_COLL_CLIENT_INIT: - return client_init(event, data); - case LOG_COLL_CLIENT_OPEN: - return client_open(event, (NetVConnection *)data); - case LOG_COLL_CLIENT_SEND: - return client_send(event, (VIO *)data); - default: - ink_assert(!"unexpcted state"); - return EVENT_CONT; - } -} - -//------------------------------------------------------------------------- -//------------------------------------------------------------------------- -// -// pubic interface -// -//------------------------------------------------------------------------- -//------------------------------------------------------------------------- - -//------------------------------------------------------------------------- -// LogCollationClientSM::send -//------------------------------------------------------------------------- - -int -LogCollationClientSM::send(LogBuffer *log_buffer) -{ - ip_port_text_buffer ipb; - - // take lock (can block on call because we're on our own thread) - ink_mutex_acquire(&(mutex->the_mutex)); - - Debug("log-coll", "[%d]client::send", m_id); - - // deny if state is DONE or FAIL - if (m_client_state == LOG_COLL_CLIENT_DONE || m_client_state == LOG_COLL_CLIENT_FAIL) { - Debug("log-coll", "[%d]client::send - DONE/FAIL state; rejecting", m_id); - ink_mutex_release(&(mutex->the_mutex)); - return 0; - } - // only allow send if m_flow is ALLOW - if (m_flow == LOG_COLL_FLOW_DENY) { - Debug("log-coll", "[%d]client::send - m_flow = DENY; rejecting", m_id); - ink_mutex_release(&(mutex->the_mutex)); - return 0; - } - // add log_buffer to m_buffer_send_list - ink_assert(log_buffer != nullptr); - ink_assert(m_buffer_send_list != nullptr); - m_buffer_send_list->add(log_buffer); - Debug("log-coll", "[%d]client::send - new log_buffer to send_list", m_id); - - // disable m_flow if there's too much work to do now - ink_assert(m_flow == LOG_COLL_FLOW_ALLOW); - if (m_buffer_send_list->get_size() >= Log::config->collation_max_send_buffers) { - Debug("log-coll", "[%d]client::send - m_flow = DENY", m_id); - Note("[log-coll] send-queue full; orphaning logs " - "[%s:%u]", - m_log_host->ip_addr().toString(ipb, sizeof(ipb)), m_log_host->port()); - m_flow = LOG_COLL_FLOW_DENY; - } - // compute return value - // must be done before call to client_send. log_buffer may - // be converted to network order during that call. - LogBufferHeader *log_buffer_header = log_buffer->header(); - ink_assert(log_buffer_header != nullptr); - int bytes_to_write = log_buffer_header->byte_count; - - // re-initiate sending if currently idle - if (m_client_state == LOG_COLL_CLIENT_IDLE) { - m_client_state = LOG_COLL_CLIENT_SEND; - ink_assert(m_pending_event == nullptr); - m_pending_event = eventProcessor.schedule_imm(this); - // eventProcessor.schedule_imm(this); - // client_send(LOG_COLL_EVENT_SWITCH, NULL); - } - - ink_mutex_release(&(mutex->the_mutex)); - return bytes_to_write; -} - -//------------------------------------------------------------------------- -//------------------------------------------------------------------------- -// -// client states -// -//------------------------------------------------------------------------- -//------------------------------------------------------------------------- - -//------------------------------------------------------------------------- -// LogCollationClientSM::client_auth -// next: client_fail || client_send -//------------------------------------------------------------------------- - -int -LogCollationClientSM::client_auth(int event, VIO * /* vio ATS_UNUSED */) -{ - ip_port_text_buffer ipb; - - Debug("log-coll", "[%d]client::client_auth", m_id); - - switch (event) { - case LOG_COLL_EVENT_SWITCH: { - Debug("log-coll", "[%d]client::client_auth - SWITCH", m_id); - m_client_state = LOG_COLL_CLIENT_AUTH; - - NetMsgHeader nmh; - int bytes_to_send = (int)strlen(Log::config->collation_secret); - nmh.msg_bytes = bytes_to_send; - - // memory copies, I know... but it happens rarely!!! ^_^ - ink_assert(m_auth_buffer != nullptr); - m_auth_buffer->write((char *)&nmh, sizeof(NetMsgHeader)); - m_auth_buffer->write(Log::config->collation_secret, bytes_to_send); - bytes_to_send += sizeof(NetMsgHeader); - - Debug("log-coll", "[%d]client::client_auth - do_io_write(%d)", m_id, bytes_to_send); - ink_assert(m_host_vc != nullptr); - m_host_vio = m_host_vc->do_io_write(this, bytes_to_send, m_auth_reader); - ink_assert(m_host_vio != nullptr); - - return EVENT_CONT; - } - - case VC_EVENT_WRITE_READY: - Debug("log-coll", "[%d]client::client_auth - WRITE_READY", m_id); - return EVENT_CONT; - - case VC_EVENT_WRITE_COMPLETE: - Debug("log-coll", "[%d]client::client_auth - WRITE_COMPLETE", m_id); - - Note("[log-coll] host up [%s:%u]", m_log_host->ip_addr().toString(ipb, sizeof(ipb)), m_log_host->port()); - m_host_is_up = true; - - return client_send(LOG_COLL_EVENT_SWITCH, nullptr); - - case VC_EVENT_ACTIVE_TIMEOUT: - case VC_EVENT_INACTIVITY_TIMEOUT: - case VC_EVENT_EOS: - case VC_EVENT_ERROR: { - Debug("log-coll", "[%d]client::client_auth - TIMEOUT|EOS|ERROR", m_id); - int64_t read_avail = m_auth_reader->read_avail(); - - if (read_avail > 0) { - Debug("log-coll", "[%d]client::client_auth - consuming unsent data", m_id); - m_auth_reader->consume(read_avail); - } - - return client_fail(LOG_COLL_EVENT_SWITCH, nullptr); - } - - default: - ink_assert(!"unexpected event"); - return EVENT_CONT; - } -} - -//------------------------------------------------------------------------- -// LogCollationClientSM::client_dns -// next: client_open || client_done -//------------------------------------------------------------------------- -int -LogCollationClientSM::client_dns(int event, HostDBInfo *hostdb_info) -{ - Debug("log-coll", "[%d]client::client_dns", m_id); - - switch (event) { - case LOG_COLL_EVENT_SWITCH: - m_client_state = LOG_COLL_CLIENT_DNS; - if (m_log_host->m_name == nullptr) { - return client_done(LOG_COLL_EVENT_SWITCH, nullptr); - } - hostDBProcessor.getbyname_re(this, m_log_host->m_name, 0, - HostDBProcessor::Options().setFlags(HostDBProcessor::HOSTDB_FORCE_DNS_RELOAD)); - return EVENT_CONT; - - case EVENT_HOST_DB_LOOKUP: - if (hostdb_info == nullptr) { - return client_done(LOG_COLL_EVENT_SWITCH, nullptr); - } - m_log_host->m_ip.assign(hostdb_info->ip()); - m_log_host->m_ip.toString(m_log_host->m_ipstr, sizeof(m_log_host->m_ipstr)); - - return client_open(LOG_COLL_EVENT_SWITCH, nullptr); - - default: - ink_assert(!"unexpected event"); - return EVENT_CONT; - } -} - -//------------------------------------------------------------------------- -// LogCollationClientSM::client_done -// next: -//------------------------------------------------------------------------- - -int -LogCollationClientSM::client_done(int event, void * /* data ATS_UNUSED */) -{ - ip_port_text_buffer ipb; - - Debug("log-coll", "[%d]client::client_done", m_id); - - switch (event) { - case LOG_COLL_EVENT_SWITCH: - m_client_state = LOG_COLL_CLIENT_DONE; - - Note("[log-coll] client shutdown [%s:%u]", m_log_host->ip_addr().toString(ipb, sizeof(ipb)), m_log_host->port()); - - // close connections - if (m_host_vc) { - Debug("log-coll", "[%d]client::client_done - disconnecting!", m_id); - // do I need to delete this??? - m_host_vc->do_io_close(0); - m_host_vc = nullptr; - } - // flush unsent logs to orphan - flush_to_orphan(); - - // cancel any pending events/actions - if (m_pending_action != nullptr) { - m_pending_action->cancel(); - } - if (m_pending_event != nullptr) { - m_pending_event->cancel(); - } - // free memory - if (m_auth_buffer) { - if (m_auth_reader) { - m_auth_buffer->dealloc_reader(m_auth_reader); - } - free_MIOBuffer(m_auth_buffer); - } - if (m_send_buffer) { - if (m_send_reader) { - m_send_buffer->dealloc_reader(m_send_reader); - } - free_MIOBuffer(m_send_buffer); - } - if (m_abort_buffer) { - free_MIOBuffer(m_abort_buffer); - } - if (m_buffer_send_list) { - delete m_buffer_send_list; - } - - return EVENT_DONE; - - default: - ink_assert(!"unexpected event"); - return EVENT_DONE; - } -} - -//------------------------------------------------------------------------- -// LogCollationClientSM::client_fail -// next: client_fail || client_open -//------------------------------------------------------------------------- - -int -LogCollationClientSM::client_fail(int event, void * /* data ATS_UNUSED */) -{ - ip_port_text_buffer ipb; - - Debug("log-coll", "[%d]client::client_fail", m_id); - - switch (event) { - case LOG_COLL_EVENT_SWITCH: - Debug("log-coll", "[%d]client::client_fail - SWITCH", m_id); - m_client_state = LOG_COLL_CLIENT_FAIL; - - // avoid flooding log when host is down - if (m_host_is_up) { - Note("[log-coll] host down [%s:%u]", m_log_host->ip_addr().toString(ipb, sizeof ipb), m_log_host->m_port); - char msg_buf[128]; - snprintf(msg_buf, sizeof(msg_buf), "Collation host %s:%u down", m_log_host->ip_addr().toString(ipb, sizeof ipb), - m_log_host->m_port); - RecSignalManager(MGMT_SIGNAL_SAC_SERVER_DOWN, msg_buf); - m_host_is_up = false; - } - - // close our NetVConnection (do I need to delete this) - if (m_host_vc) { - m_host_vc->do_io_close(0); - m_host_vc = nullptr; - } - // flush unsent logs to orphan - flush_to_orphan(); - - // call back in collation_retry_sec seconds - ink_assert(m_pending_event == nullptr); - m_pending_event = eventProcessor.schedule_in(this, HRTIME_SECONDS(Log::config->collation_retry_sec)); - - return EVENT_CONT; - - case EVENT_INTERVAL: - Debug("log-coll", "[%d]client::client_fail - INTERVAL", m_id); - m_pending_event = nullptr; - return client_open(LOG_COLL_EVENT_SWITCH, nullptr); - - default: - ink_assert(!"unexpected event"); - return EVENT_CONT; - } -} - -//------------------------------------------------------------------------- -// LogCollationClientSM::client_idle -// next: client_send -//------------------------------------------------------------------------- - -int -LogCollationClientSM::client_idle(int event, void * /* data ATS_UNUSED */) -{ - Debug("log-coll", "[%d]client::client_idle", m_id); - - switch (event) { - case LOG_COLL_EVENT_SWITCH: - m_client_state = LOG_COLL_CLIENT_IDLE; - return EVENT_CONT; - - case VC_EVENT_ACTIVE_TIMEOUT: - case VC_EVENT_INACTIVITY_TIMEOUT: - case VC_EVENT_EOS: - case VC_EVENT_ERROR: - Debug("log-coll", "[%d]client::client_idle - TIMEOUT|EOS|ERROR", m_id); - return client_fail(LOG_COLL_EVENT_SWITCH, nullptr); - - default: - ink_assert(!"unexpcted state"); - return EVENT_CONT; - } -} - -//------------------------------------------------------------------------- -// LogCollationClientSM::client_init -// next: client_dns -//------------------------------------------------------------------------- - -int -LogCollationClientSM::client_init(int event, void * /* data ATS_UNUSED */) -{ - Debug("log-coll", "[%d]client::client_init", m_id); - - switch (event) { - case LOG_COLL_EVENT_SWITCH: - m_client_state = LOG_COLL_CLIENT_INIT; - ink_assert(m_pending_event == nullptr); - ink_mutex_acquire(&(mutex->the_mutex)); - m_pending_event = eventProcessor.schedule_imm(this); - ink_mutex_release(&(mutex->the_mutex)); - return EVENT_CONT; - - case EVENT_IMMEDIATE: - // callback complete, reset m_pending_event - m_pending_event = nullptr; - - // allocate buffers - m_auth_buffer = new_MIOBuffer(); - ink_assert(m_auth_buffer != nullptr); - m_auth_reader = m_auth_buffer->alloc_reader(); - ink_assert(m_auth_reader != nullptr); - m_send_buffer = new_MIOBuffer(); - ink_assert(m_send_buffer != nullptr); - m_send_reader = m_send_buffer->alloc_reader(); - ink_assert(m_send_reader != nullptr); - m_abort_buffer = new_MIOBuffer(); - ink_assert(m_abort_buffer != nullptr); - - // if we don't have an ip already, switch to client_dns - if (!m_log_host->ip_addr().isValid()) { - return client_dns(LOG_COLL_EVENT_SWITCH, nullptr); - } else { - return client_open(LOG_COLL_EVENT_SWITCH, nullptr); - } - - default: - ink_assert(!"unexpected state"); - return EVENT_CONT; - } -} - -//------------------------------------------------------------------------- -// LogCollationClientSM::client_open -// next: client_auth || client_fail -//------------------------------------------------------------------------- - -int -LogCollationClientSM::client_open(int event, NetVConnection *net_vc) -{ - RecInt rec_timeout; - int timeout = 86400; - ip_port_text_buffer ipb; - Debug("log-coll", "[%d]client::client_open", m_id); - - switch (event) { - case LOG_COLL_EVENT_SWITCH: - Debug("log-coll", "[%d]client::client_open - SWITCH", m_id); - m_client_state = LOG_COLL_CLIENT_OPEN; - - { - IpEndpoint target; - target.assign(m_log_host->ip_addr(), htons(m_log_host->port())); - ink_assert(target.isValid()); - Action *connect_action_handle = netProcessor.connect_re(this, &target.sa); - - if (connect_action_handle != ACTION_RESULT_DONE) { - ink_assert(!m_pending_action); - m_pending_action = connect_action_handle; - } - } - - return EVENT_CONT; - - case NET_EVENT_OPEN: - Debug("log-coll", "[%d]client::client_open - %s:%u", m_id, m_log_host->ip_addr().toString(ipb, sizeof ipb), m_log_host->port()); - - // callback complete, reset m_pending_action - m_pending_action = nullptr; - - ink_assert(net_vc != nullptr); - m_host_vc = net_vc; - - // assign an explicit inactivity timeout so that it will not get the default value later - if (RecGetRecordInt("proxy.config.log.collation_client_timeout", &rec_timeout) == REC_ERR_OKAY) { - timeout = rec_timeout; - } - m_host_vc->set_inactivity_timeout(HRTIME_SECONDS(timeout)); - - // setup a client reader just for detecting a host disconnnect - // (iocore should call back this function with and EOS/ERROR) - m_abort_vio = m_host_vc->do_io_read(this, 1, m_abort_buffer); - - // change states - return client_auth(LOG_COLL_EVENT_SWITCH, nullptr); - - case NET_EVENT_OPEN_FAILED: - Debug("log-coll", "[%d]client::client_open - OPEN_FAILED", m_id); - // callback complete, reset m_pending_pending action - m_pending_action = nullptr; - return client_fail(LOG_COLL_EVENT_SWITCH, nullptr); - - default: - ink_assert(!"unexpected event"); - return EVENT_CONT; - } -} - -//------------------------------------------------------------------------- -// LogCollationClientSM::client_send -// next: client_fail || client_idle || client_send -//------------------------------------------------------------------------- - -int -LogCollationClientSM::client_send(int event, VIO * /* vio ATS_UNUSED */) -{ - ip_port_text_buffer ipb; - - Debug("log-coll", "[%d]client::client_send", m_id); - - switch (event) { - case EVENT_IMMEDIATE: - Debug("log-coll", "[%d]client::client_send - EVENT_IMMEDIATE", m_id); - // callback complete, reset m_pending_event - m_pending_event = nullptr; - // fallthrough - - case LOG_COLL_EVENT_SWITCH: { - Debug("log-coll", "[%d]client::client_send - SWITCH", m_id); - m_client_state = LOG_COLL_CLIENT_SEND; - - // get a buffer off our queue - ink_assert(m_buffer_send_list != nullptr); - ink_assert(m_buffer_in_iocore == nullptr); - if ((m_buffer_in_iocore = m_buffer_send_list->get()) == nullptr) { - return client_idle(LOG_COLL_EVENT_SWITCH, nullptr); - } - Debug("log-coll", "[%d]client::client_send - send_list to m_buffer_in_iocore", m_id); - Debug("log-coll", "[%d]client::client_send - send_list_size(%d)", m_id, m_buffer_send_list->get_size()); - - // enable m_flow if we're out of work to do - if (m_flow == LOG_COLL_FLOW_DENY && m_buffer_send_list->get_size() == 0) { - Debug("log-coll", "[%d]client::client_send - m_flow = ALLOW", m_id); - Note("[log-coll] send-queue clear; resuming collation [%s:%u]", m_log_host->ip_addr().toString(ipb, sizeof ipb), - m_log_host->port()); - m_flow = LOG_COLL_FLOW_ALLOW; - } - // future work: - // Wrap the buffer in a io_buffer_block and send directly to - // do_io_write to save a memory copy. But for now, just - // write the lame way. - -#if defined(LOG_BUFFER_TRACKING) - Debug("log-buftrak", "[%d]client::client_send - network write begin", m_buffer_in_iocore->header()->id); -#endif // defined(LOG_BUFFER_TRACKING) - - // prepare to send data - ink_assert(m_buffer_in_iocore != nullptr); - LogBufferHeader *log_buffer_header = m_buffer_in_iocore->header(); - ink_assert(log_buffer_header != nullptr); - NetMsgHeader nmh; - int bytes_to_send = log_buffer_header->byte_count; - nmh.msg_bytes = bytes_to_send; - // TODO: We currently don't try to make the log buffers handle little vs big endian. TS-1156. - // m_buffer_in_iocore->convert_to_network_order(); - - RecIncrRawStat(log_rsb, mutex->thread_holding, log_stat_num_sent_to_network_stat, log_buffer_header->entry_count); - - RecIncrRawStat(log_rsb, mutex->thread_holding, log_stat_bytes_sent_to_network_stat, log_buffer_header->byte_count); - - // copy into m_send_buffer - ink_assert(m_send_buffer != nullptr); - m_send_buffer->write((char *)&nmh, sizeof(NetMsgHeader)); - m_send_buffer->write((char *)log_buffer_header, bytes_to_send); - bytes_to_send += sizeof(NetMsgHeader); - - // send m_send_buffer to iocore - Debug("log-coll", "[%d]client::client_send - do_io_write(%d)", m_id, bytes_to_send); - ink_assert(m_host_vc != nullptr); - m_host_vio = m_host_vc->do_io_write(this, bytes_to_send, m_send_reader); - ink_assert(m_host_vio != nullptr); - } - return EVENT_CONT; - - case VC_EVENT_WRITE_READY: - Debug("log-coll", "[%d]client::client_send - WRITE_READY", m_id); - return EVENT_CONT; - - case VC_EVENT_WRITE_COMPLETE: - Debug("log-coll", "[%d]client::client_send - WRITE_COMPLETE", m_id); - - ink_assert(m_buffer_in_iocore != nullptr); -#if defined(LOG_BUFFER_TRACKING) - Debug("log-buftrak", "[%d]client::client_send - network write complete", m_buffer_in_iocore->header()->id); -#endif // defined(LOG_BUFFER_TRACKING) - - // done with the buffer, delete it - Debug("log-coll", "[%d]client::client_send - m_buffer_in_iocore[%p] to delete_list", m_id, m_buffer_in_iocore); - LogBuffer::destroy(m_buffer_in_iocore); - m_buffer_in_iocore = nullptr; - - // switch back to client_send - return client_send(LOG_COLL_EVENT_SWITCH, nullptr); - - case VC_EVENT_ACTIVE_TIMEOUT: - case VC_EVENT_INACTIVITY_TIMEOUT: - case VC_EVENT_EOS: - case VC_EVENT_ERROR: { - Debug("log-coll", "[%d]client::client_send - TIMEOUT|EOS|ERROR", m_id); - int64_t read_avail = m_send_reader->read_avail(); - - if (read_avail > 0) { - Debug("log-coll", "[%d]client::client_send - consuming unsent data", m_id); - m_send_reader->consume(read_avail); - } - - return client_fail(LOG_COLL_EVENT_SWITCH, nullptr); - } - - default: - Debug("log-coll", "[%d]client::client_send - default", m_id); - return client_fail(LOG_COLL_EVENT_SWITCH, nullptr); - } -} - -//------------------------------------------------------------------------- -//------------------------------------------------------------------------- -// -// support functions -// -//------------------------------------------------------------------------- -//------------------------------------------------------------------------- - -//------------------------------------------------------------------------- -// LogCollationClientSM::flush_to_orphan -//------------------------------------------------------------------------- -void -LogCollationClientSM::flush_to_orphan() -{ - Debug("log-coll", "[%d]client::flush_to_orphan", m_id); - - // if in middle of a write, flush buffer_in_iocore to orphan - if (m_buffer_in_iocore != nullptr) { - Debug("log-coll", "[%d]client::flush_to_orphan - m_buffer_in_iocore to oprhan", m_id); - // TODO: We currently don't try to make the log buffers handle little vs big endian. TS-1156. - // m_buffer_in_iocore->convert_to_host_order(); - m_log_host->orphan_write_and_try_delete(m_buffer_in_iocore); - m_buffer_in_iocore = nullptr; - } - // flush buffers in send_list to orphan - LogBuffer *log_buffer; - ink_assert(m_buffer_send_list != nullptr); - while ((log_buffer = m_buffer_send_list->get()) != nullptr) { - Debug("log-coll", "[%d]client::flush_to_orphan - send_list to orphan", m_id); - m_log_host->orphan_write_and_try_delete(log_buffer); - } - - // Now send_list is empty, let's update m_flow to ALLOW status - Debug("log-coll", "[%d]client::client_send - m_flow = ALLOW", m_id); - m_flow = LOG_COLL_FLOW_ALLOW; -} diff --git a/proxy/logging/LogCollationClientSM.h b/proxy/logging/LogCollationClientSM.h deleted file mode 100644 index e10cc1fe5fe..00000000000 --- a/proxy/logging/LogCollationClientSM.h +++ /dev/null @@ -1,116 +0,0 @@ -/** @file - - A brief file description - - @section license License - - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - */ - -#pragma once - -//------------------------------------------------------------------------- -// includes -//------------------------------------------------------------------------- -#include "P_HostDB.h" -#include "P_Net.h" -#include "LogCollationBase.h" - -//------------------------------------------------------------------------- -// pre-declarations -//------------------------------------------------------------------------- - -class LogBuffer; -class LogHost; - -//------------------------------------------------------------------------- -// LogCollationClientSM -//------------------------------------------------------------------------- - -class LogCollationClientSM : public LogCollationBase, public Continuation -{ -public: - LogCollationClientSM(LogHost *log_host); - ~LogCollationClientSM() override; - - int client_handler(int event, void *data); - - // public interface (for LogFile) - int send(LogBuffer *log_buffer); - -private: - enum ClientState { - LOG_COLL_CLIENT_START, - LOG_COLL_CLIENT_AUTH, - LOG_COLL_CLIENT_DNS, - LOG_COLL_CLIENT_DONE, - LOG_COLL_CLIENT_FAIL, - LOG_COLL_CLIENT_IDLE, - LOG_COLL_CLIENT_INIT, - LOG_COLL_CLIENT_OPEN, - LOG_COLL_CLIENT_SEND - }; - - enum ClientFlowControl { - LOG_COLL_FLOW_ALLOW, - LOG_COLL_FLOW_DENY, - }; - -private: - // client states - int client_auth(int event, VIO *vio); - int client_dns(int event, HostDBInfo *hostdb_info); - int client_done(int event, void *data); - int client_fail(int event, void *data); - int client_idle(int event, void *data); - int client_init(int event, void *data); - int client_open(int event, NetVConnection *net_vc); - int client_send(int event, VIO *vio); - ClientState m_client_state = LOG_COLL_CLIENT_START; - - // support functions - void flush_to_orphan(); - - // iocore stuff (two buffers to avoid races) - NetVConnection *m_host_vc = nullptr; - VIO *m_host_vio = nullptr; - MIOBuffer *m_auth_buffer = nullptr; - IOBufferReader *m_auth_reader = nullptr; - MIOBuffer *m_send_buffer = nullptr; - IOBufferReader *m_send_reader = nullptr; - Action *m_pending_action = nullptr; - Event *m_pending_event = nullptr; - - // to detect server closes (there's got to be a better way to do this) - VIO *m_abort_vio = nullptr; - MIOBuffer *m_abort_buffer = nullptr; - bool m_host_is_up = false; - - // send stuff - LogBufferList *m_buffer_send_list = nullptr; - LogBuffer *m_buffer_in_iocore = nullptr; - ClientFlowControl m_flow = LOG_COLL_FLOW_ALLOW; - - // back pointer to LogHost container - LogHost *m_log_host; - - // debugging - static int ID; - int m_id = 0; -}; - -typedef int (LogCollationClientSM::*LogCollationClientSMHandler)(int, void *); diff --git a/proxy/logging/LogCollationHostSM.cc b/proxy/logging/LogCollationHostSM.cc deleted file mode 100644 index 2962271a93d..00000000000 --- a/proxy/logging/LogCollationHostSM.cc +++ /dev/null @@ -1,527 +0,0 @@ -/** @file - - A brief file description - - @section license License - - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - */ - -//------------------------------------------------------------------------- -// include files -//------------------------------------------------------------------------- - -#include "tscore/ink_config.h" - -#include -#include -#include -#include -#include - -#include "P_EventSystem.h" -#include "P_Net.h" - -#include "LogUtils.h" -#include "LogSock.h" -#include "LogField.h" -#include "LogFile.h" -#include "LogFormat.h" -#include "LogBuffer.h" -#include "LogHost.h" -#include "LogObject.h" -#include "LogConfig.h" -#include "Log.h" - -#include "LogCollationHostSM.h" - -//------------------------------------------------------------------------- -// statics -//------------------------------------------------------------------------- - -int LogCollationHostSM::ID = 0; - -//------------------------------------------------------------------------- -// LogCollationHostSM::LogCollationHostSM -//------------------------------------------------------------------------- - -LogCollationHostSM::LogCollationHostSM(NetVConnection *client_vc) - : Continuation(new_ProxyMutex()), - m_client_vc(client_vc), - m_client_vio(nullptr), - m_client_buffer(nullptr), - m_client_reader(nullptr), - m_pending_event(nullptr), - m_read_buffer(nullptr), - m_read_bytes_wanted(0), - m_read_bytes_received(0), - m_read_buffer_fast_allocator_size(-1), - m_client_ip(0), - m_client_port(0), - m_id(ID++) -{ - RecInt rec_timeout; - int timeout = 86390; - Debug("log-coll", "[%d]host::constructor", m_id); - - ink_assert(m_client_vc != nullptr); - - // assign an explicit inactivity timeout so that it will not get the default value later - if (RecGetRecordInt("proxy.config.log.collation_host_timeout", &rec_timeout) == REC_ERR_OKAY) { - timeout = rec_timeout; - } - m_client_vc->set_inactivity_timeout(HRTIME_SECONDS(timeout)); - - // get client info - m_client_ip = m_client_vc->get_remote_ip(); - m_client_port = m_client_vc->get_remote_port(); - Note("[log-coll] client connected [%d.%d.%d.%d:%d]", ((unsigned char *)(&m_client_ip))[0], ((unsigned char *)(&m_client_ip))[1], - ((unsigned char *)(&m_client_ip))[2], ((unsigned char *)(&m_client_ip))[3], m_client_port); - - SET_HANDLER((LogCollationHostSMHandler)&LogCollationHostSM::host_handler); - host_init(LOG_COLL_EVENT_SWITCH, nullptr); -} - -void -LogCollationHostSM::freeReadBuffer() -{ - if (m_read_buffer) { - if (m_read_buffer_fast_allocator_size >= 0) { - ioBufAllocator[m_read_buffer_fast_allocator_size].free_void(m_read_buffer); - } else { - ats_free(m_read_buffer); - } - m_read_buffer = nullptr; - } -} - -//------------------------------------------------------------------------- -//------------------------------------------------------------------------- -// -// handlers -// -//------------------------------------------------------------------------- -//------------------------------------------------------------------------- - -//------------------------------------------------------------------------- -// LogCollationHostSM::host_handler -//------------------------------------------------------------------------- - -int -LogCollationHostSM::host_handler(int event, void *data) -{ - switch (m_host_state) { - case LOG_COLL_HOST_AUTH: - return host_auth(event, data); - case LOG_COLL_HOST_DONE: - return host_done(event, data); - case LOG_COLL_HOST_INIT: - return host_init(event, data); - case LOG_COLL_HOST_RECV: - return host_recv(event, data); - default: - ink_assert(!"unexpected state"); - return EVENT_CONT; - } -} - -//------------------------------------------------------------------------- -// LogCollationHostSM::read_handler -//------------------------------------------------------------------------- - -int -LogCollationHostSM::read_handler(int event, void *data) -{ - switch (m_read_state) { - case LOG_COLL_READ_BODY: - return read_body(event, (VIO *)data); - case LOG_COLL_READ_HDR: - return read_hdr(event, (VIO *)data); - default: - ink_assert(!"unexpected state"); - return EVENT_CONT; - } -} - -//------------------------------------------------------------------------- -//------------------------------------------------------------------------- -// -// host states -// -//------------------------------------------------------------------------- -//------------------------------------------------------------------------- - -//------------------------------------------------------------------------- -// LogCollationHostSM::host_auth -// next: host_done || host_recv -//------------------------------------------------------------------------- - -int -LogCollationHostSM::host_auth(int event, void * /* data ATS_UNUSED */) -{ - Debug("log-coll", "[%d]host::host_auth", m_id); - - switch (event) { - case LOG_COLL_EVENT_SWITCH: - Debug("log-coll", "[%d]host::host_auth - SWITCH", m_id); - m_host_state = LOG_COLL_HOST_AUTH; - return read_start(); - - case LOG_COLL_EVENT_READ_COMPLETE: - Debug("log-coll", "[%d]host::host_auth - READ_COMPLETE", m_id); - { - // compare authorization secrets - ink_assert(m_read_buffer != nullptr); - int diff = strncmp(m_read_buffer, Log::config->collation_secret, m_read_bytes_received); - freeReadBuffer(); - if (!diff) { - Debug("log-coll", "[%d]host::host_auth - authenticated!", m_id); - return host_recv(LOG_COLL_EVENT_SWITCH, nullptr); - } else { - Debug("log-coll", "[%d]host::host_auth - authenticated failed!", m_id); - Note("[log-coll] authentication failed [%d.%d.%d.%d:%d]", ((unsigned char *)(&m_client_ip))[0], - ((unsigned char *)(&m_client_ip))[1], ((unsigned char *)(&m_client_ip))[2], ((unsigned char *)(&m_client_ip))[3], - m_client_port); - return host_done(LOG_COLL_EVENT_SWITCH, nullptr); - } - } - - case LOG_COLL_EVENT_ERROR: - Debug("log-coll", "[%d]host::host_auth - ERROR", m_id); - return host_done(LOG_COLL_EVENT_SWITCH, nullptr); - - default: - ink_assert(!"unexpected state"); - return EVENT_CONT; - } -} - -//------------------------------------------------------------------------- -// LogCollationHostSM::host_done -// next: none -//------------------------------------------------------------------------- - -int -LogCollationHostSM::host_done(int /* event ATS_UNUSED */, void * /* data ATS_UNUSED */) -{ - Debug("log-coll", "[%d]host::host_done", m_id); - - // close connections - if (m_client_vc) { - Debug("log-coll", "[%d]host::host_done - disconnecting!", m_id); - m_client_vc->do_io_close(); - m_client_vc = nullptr; - Note("[log-coll] client disconnected [%d.%d.%d.%d:%d]", ((unsigned char *)(&m_client_ip))[0], - ((unsigned char *)(&m_client_ip))[1], ((unsigned char *)(&m_client_ip))[2], ((unsigned char *)(&m_client_ip))[3], - m_client_port); - } - // free memory - if (m_client_buffer) { - if (m_client_reader) { - m_client_buffer->dealloc_reader(m_client_reader); - } - free_MIOBuffer(m_client_buffer); - } - // delete this state machine and return - delete this; - return EVENT_DONE; -} - -//------------------------------------------------------------------------- -// LogCollationHostSM::host_init -// next: host_auth || host_done -//------------------------------------------------------------------------- - -int -LogCollationHostSM::host_init(int event, void * /* data ATS_UNUSED */) -{ - Debug("log-coll", "[%d]host::host_init", m_id); - - switch (event) { - case LOG_COLL_EVENT_SWITCH: - m_host_state = LOG_COLL_HOST_INIT; - m_pending_event = eventProcessor.schedule_imm(this); - return EVENT_CONT; - - case EVENT_IMMEDIATE: - // allocate memory - m_client_buffer = new_MIOBuffer(); - ink_assert(m_client_buffer != nullptr); - m_client_reader = m_client_buffer->alloc_reader(); - ink_assert(m_client_reader != nullptr); - return host_auth(LOG_COLL_EVENT_SWITCH, nullptr); - - default: - ink_assert(!"unexpected state"); - return EVENT_DONE; - } -} - -//------------------------------------------------------------------------- -// LogCollationHostSM::host_recv -// next: host_done || host_recv -//------------------------------------------------------------------------- - -int -LogCollationHostSM::host_recv(int event, void * /* data ATS_UNUSED */) -{ - Debug("log-coll", "[%d]host::host_recv", m_id); - - switch (event) { - case LOG_COLL_EVENT_SWITCH: - Debug("log-coll", "[%d]host::host_recv - SWITCH", m_id); - m_host_state = LOG_COLL_HOST_RECV; - return read_start(); - - case LOG_COLL_EVENT_READ_COMPLETE: - Debug("log-coll", "[%d]host::host_recv - READ_COMPLETE", m_id); - { - // grab the log_buffer - LogBufferHeader *log_buffer_header; - LogBuffer *log_buffer; - LogFormat *log_format; - LogObject *log_object; - unsigned version; - - ink_assert(m_read_buffer != nullptr); - ink_assert(m_read_bytes_received >= (int64_t)sizeof(LogBufferHeader)); - log_buffer_header = (LogBufferHeader *)m_read_buffer; - - // convert the buffer we just received to host order - // TODO: We currently don't try to make the log buffers handle little vs big endian. TS-1156. - // LogBuffer::convert_to_host_order(log_buffer_header); - - version = log_buffer_header->version; - if (version != LOG_SEGMENT_VERSION) { - Note("[log-coll] invalid LogBuffer received; invalid version - " - "buffer = %u, current = %u", - version, LOG_SEGMENT_VERSION); - freeReadBuffer(); - - } else { - log_object = Log::match_logobject(log_buffer_header); - if (!log_object) { - Note("[log-coll] LogObject not found with fieldlist id; " - "writing LogBuffer to scrap file"); - log_object = Log::global_scrap_object; - } - log_format = log_object->m_format; - Debug("log-coll", "[%d]host::host_recv - using format '%s'", m_id, log_format->name()); - - // make a new LogBuffer (log_buffer_header plus subsequent - // buffer already converted to host order) and add it to the - // object's flush queue - // - log_buffer = new LogBuffer(log_object, log_buffer_header); - - RecIncrRawStat(log_rsb, mutex->thread_holding, log_stat_num_received_from_network_stat, log_buffer_header->entry_count); - - RecIncrRawStat(log_rsb, mutex->thread_holding, log_stat_bytes_received_from_network_stat, log_buffer_header->byte_count); - - int idx = log_object->add_to_flush_queue(log_buffer); - Log::preproc_notify[idx].signal(); - } - -#if defined(LOG_BUFFER_TRACKING) - Debug("log-buftrak", "[%d]host::host_recv - network read complete", log_buffer_header->id); -#endif // defined(LOG_BUFFER_TRACKING) - - // get ready for next read (memory may not be freed!!!) - m_read_buffer = nullptr; - - return host_recv(LOG_COLL_EVENT_SWITCH, nullptr); - } - - case LOG_COLL_EVENT_ERROR: - Debug("log-coll", "[%d]host::host_recv - ERROR", m_id); - return host_done(LOG_COLL_EVENT_SWITCH, nullptr); - - default: - ink_assert(!"unexpected state"); - return EVENT_DONE; - } -} - -//------------------------------------------------------------------------- -//------------------------------------------------------------------------- -// -// read states -// -//------------------------------------------------------------------------- -//------------------------------------------------------------------------- - -//------------------------------------------------------------------------- -// LogCollationHostSM::read_start -// next: read_hdr -//------------------------------------------------------------------------- - -int -LogCollationHostSM::read_start() -{ - Debug("log-coll", "[%d]host::read_start", m_id); - - SET_HANDLER((LogCollationHostSMHandler)&LogCollationHostSM::read_handler); - if (m_read_buffer) { - ink_assert(!"m_read_buffer still points to something, doh!"); - } - return read_hdr(LOG_COLL_EVENT_SWITCH, nullptr); -} - -//------------------------------------------------------------------------- -// LogCollationHostSM::read_hdr -// next: read_body || read_done -//------------------------------------------------------------------------- - -int -LogCollationHostSM::read_hdr(int event, VIO *vio) -{ - Debug("log-coll", "[%d]host::read_hdr", m_id); - - switch (event) { - case LOG_COLL_EVENT_SWITCH: - Debug("log-coll", "[%d]host:read_hdr - SWITCH", m_id); - m_read_state = LOG_COLL_READ_HDR; - - m_read_bytes_wanted = sizeof(NetMsgHeader); - m_read_bytes_received = 0; - m_read_buffer = (char *)&m_net_msg_header; - ink_assert(m_client_vc != nullptr); - Debug("log-coll", "[%d]host:read_hdr - do_io_read(%" PRId64 ")", m_id, m_read_bytes_wanted); - m_client_vio = m_client_vc->do_io_read(this, m_read_bytes_wanted, m_client_buffer); - ink_assert(m_client_vio != nullptr); - return EVENT_CONT; - - case VC_EVENT_IMMEDIATE: - Debug("log-coll", "[%d]host::read_hdr - IMMEDIATE", m_id); - return EVENT_CONT; - - case VC_EVENT_READ_READY: - Debug("log-coll", "[%d]host::read_hdr - READ_READY", m_id); - read_partial(vio); - return EVENT_CONT; - - case VC_EVENT_READ_COMPLETE: - Debug("log-coll", "[%d]host::read_hdr - READ_COMPLETE", m_id); - read_partial(vio); - ink_assert(m_read_bytes_wanted == m_read_bytes_received); - return read_body(LOG_COLL_EVENT_SWITCH, nullptr); - - case VC_EVENT_ACTIVE_TIMEOUT: - case VC_EVENT_INACTIVITY_TIMEOUT: - case VC_EVENT_EOS: - case VC_EVENT_ERROR: - Debug("log-coll", "[%d]host::read_hdr - TIMEOUT|EOS|ERROR", m_id); - return read_done(LOG_COLL_EVENT_ERROR, nullptr); - - default: - Debug("log-coll", "[%d]host::read_hdr - default %d", m_id, event); - return read_done(LOG_COLL_EVENT_ERROR, nullptr); - } -} - -//------------------------------------------------------------------------- -// LogCollationHostSM::read_body -// next: read_body || read_done -//------------------------------------------------------------------------- - -int -LogCollationHostSM::read_body(int event, VIO *vio) -{ - Debug("log-coll", "[%d]host::read_body", m_id); - - switch (event) { - case LOG_COLL_EVENT_SWITCH: - Debug("log-coll", "[%d]host:read_body - SWITCH", m_id); - m_read_state = LOG_COLL_READ_BODY; - - m_read_bytes_wanted = m_net_msg_header.msg_bytes; - ink_assert(m_read_bytes_wanted > 0); - m_read_bytes_received = 0; - if (m_read_bytes_wanted <= max_iobuffer_size) { - m_read_buffer_fast_allocator_size = buffer_size_to_index(m_read_bytes_wanted); - m_read_buffer = (char *)ioBufAllocator[m_read_buffer_fast_allocator_size].alloc_void(); - } else { - m_read_buffer_fast_allocator_size = -1; - m_read_buffer = (char *)ats_malloc(m_read_bytes_wanted); - } - ink_assert(m_read_buffer != nullptr); - ink_assert(m_client_vc != nullptr); - Debug("log-coll", "[%d]host:read_body - do_io_read(%" PRId64 ")", m_id, m_read_bytes_wanted); - m_client_vio = m_client_vc->do_io_read(this, m_read_bytes_wanted, m_client_buffer); - ink_assert(m_client_vio != nullptr); - return EVENT_CONT; - - case VC_EVENT_IMMEDIATE: - Debug("log-coll", "[%d]host::read_body - IMMEDIATE", m_id); - return EVENT_CONT; - - case VC_EVENT_READ_READY: - Debug("log-coll", "[%d]host::read_body - READ_READY", m_id); - read_partial(vio); - return EVENT_CONT; - - case VC_EVENT_READ_COMPLETE: - Debug("log-coll", "[%d]host::read_body - READ_COMPLETE", m_id); - read_partial(vio); - ink_assert(m_read_bytes_wanted == m_read_bytes_received); - return read_done(LOG_COLL_EVENT_READ_COMPLETE, nullptr); - - case VC_EVENT_ACTIVE_TIMEOUT: - case VC_EVENT_INACTIVITY_TIMEOUT: - case VC_EVENT_EOS: - case VC_EVENT_ERROR: - Debug("log-coll", "[%d]host::read_body - TIMEOUT|EOS|ERROR", m_id); - return read_done(LOG_COLL_EVENT_ERROR, nullptr); - - default: - Debug("log-coll", "[%d]host::read_body - default %d", m_id, event); - return read_done(LOG_COLL_EVENT_ERROR, nullptr); - } -} - -//------------------------------------------------------------------------- -// LogCollationHostSM::read_done -// next: give control back to host state-machine -//------------------------------------------------------------------------- - -int -LogCollationHostSM::read_done(int event, void * /* data ATS_UNUSED */) -{ - SET_HANDLER((LogCollationHostSMHandler)&LogCollationHostSM::host_handler); - return host_handler(event, nullptr); -} - -//------------------------------------------------------------------------- -// LogCollationHostSM::read_partial -//------------------------------------------------------------------------- - -void -LogCollationHostSM::read_partial(VIO *vio) -{ - // checks - ink_assert(vio != nullptr); - ink_assert(vio->vc_server == m_client_vc); - ink_assert(m_client_buffer != nullptr); - ink_assert(m_client_reader != nullptr); - - // careful not to read more than we have memory for - char *p = &(m_read_buffer[m_read_bytes_received]); - int64_t bytes_wanted_now = m_read_bytes_wanted - m_read_bytes_received; - int64_t bytes_received_now = m_client_reader->read(p, bytes_wanted_now); - - m_read_bytes_received += bytes_received_now; -} diff --git a/proxy/logging/LogCollationHostSM.h b/proxy/logging/LogCollationHostSM.h deleted file mode 100644 index d7038fca0b6..00000000000 --- a/proxy/logging/LogCollationHostSM.h +++ /dev/null @@ -1,109 +0,0 @@ -/** @file - - A brief file description - - @section license License - - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - */ - -#pragma once - -//------------------------------------------------------------------------- -// includes -//------------------------------------------------------------------------- - -#include "P_EventSystem.h" -#include "LogCollationBase.h" - -//------------------------------------------------------------------------- -// pre-declarations -//------------------------------------------------------------------------- - -struct LogBufferHeader; - -//------------------------------------------------------------------------- -// LogCollationHostSM -//------------------------------------------------------------------------- - -class LogCollationHostSM : public LogCollationBase, public Continuation -{ -public: - LogCollationHostSM(NetVConnection *client_vc); - - // handlers - int host_handler(int event, void *data); - int read_handler(int event, void *data); - -private: - enum HostState { - LOG_COLL_HOST_NULL, - LOG_COLL_HOST_AUTH, - LOG_COLL_HOST_DONE, - LOG_COLL_HOST_INIT, - LOG_COLL_HOST_RECV, - }; - - enum ReadState { - LOG_COLL_READ_NULL, - LOG_COLL_READ_BODY, - LOG_COLL_READ_HDR, - }; - -private: - // host states - int host_init(int event, void *data); - int host_auth(int event, void *data); - int host_recv(int event, void *data); - int host_done(int event, void *data); - HostState m_host_state; - - // read states - int read_hdr(int event, VIO *vio); - int read_body(int event, VIO *vio); - int read_done(int event, void *data); - int read_start(); - void freeReadBuffer(); - ReadState m_read_state; - - // helper for read states - void read_partial(VIO *vio); - - // iocore stuff - NetVConnection *m_client_vc; - VIO *m_client_vio; - MIOBuffer *m_client_buffer; - IOBufferReader *m_client_reader; - Event *m_pending_event; - - // read_state stuff - NetMsgHeader m_net_msg_header; - char *m_read_buffer; - int64_t m_read_bytes_wanted; - int64_t m_read_bytes_received; - int64_t m_read_buffer_fast_allocator_size; - - // client info - int m_client_ip; - int m_client_port; - - // debugging - static int ID; - int m_id; -}; - -typedef int (LogCollationHostSM::*LogCollationHostSMHandler)(int, void *); diff --git a/proxy/logging/LogConfig.cc b/proxy/logging/LogConfig.cc index 757c0bb4ccf..2644011f3d5 100644 --- a/proxy/logging/LogConfig.cc +++ b/proxy/logging/LogConfig.cc @@ -41,14 +41,11 @@ #include "LogFormat.h" #include "LogFile.h" #include "LogBuffer.h" -#include "LogHost.h" #include "LogObject.h" #include "LogConfig.h" #include "LogUtils.h" #include "tscore/SimpleTokenizer.h" -#include "LogCollationAccept.h" - #include "YamlLogConfig.h" #define DISK_IS_CONFIG_FULL_MESSAGE \ @@ -75,22 +72,14 @@ LogConfig::setup_default_values() } hostname = ats_strdup(name); - log_buffer_size = (int)(10 * LOG_KILOBYTE); - max_secs_per_buffer = 5; - max_space_mb_for_logs = 100; - max_space_mb_for_orphan_logs = 25; - max_space_mb_headroom = 10; - logfile_perm = 0644; - logfile_dir = ats_strdup("."); - - collation_mode = Log::NO_COLLATION; - collation_host = ats_strdup("none"); - collation_port = 0; - collation_host_tagged = false; - collation_preproc_threads = 1; - collation_secret = ats_strdup("foobar"); - collation_retry_sec = 0; - collation_max_send_buffers = 0; + log_buffer_size = (int)(10 * LOG_KILOBYTE); + max_secs_per_buffer = 5; + max_space_mb_for_logs = 100; + max_space_mb_headroom = 10; + logfile_perm = 0644; + logfile_dir = ats_strdup("."); + + preproc_threads = 1; rolling_enabled = Log::NO_ROLLING; rolling_interval_sec = 86400; // 24 hours @@ -103,19 +92,14 @@ LogConfig::setup_default_values() file_stat_frequency = 16; space_used_frequency = 900; - use_orphan_log_space_value = false; - ascii_buffer_size = 4 * 9216; max_line_size = 9216; // size of pipe buffer for SunOS 5.6 } -void * -LogConfig::reconfigure_mgmt_variables(void * /* token ATS_UNUSED */, char * /* data_raw ATS_UNUSED */, - int /* data_len ATS_UNUSED */) +void LogConfig::reconfigure_mgmt_variables(ts::MemSpan) { Note("received log reconfiguration event, rolling now"); Log::config->roll_log_files_now = true; - return nullptr; } void @@ -139,11 +123,6 @@ LogConfig::read_configuration_variables() max_space_mb_for_logs = val; } - val = (int)REC_ConfigReadInteger("proxy.config.log.max_space_mb_for_orphan_logs"); - if (val > 0) { - max_space_mb_for_orphan_logs = val; - } - val = (int)REC_ConfigReadInteger("proxy.config.log.max_space_mb_headroom"); if (val > 0) { max_space_mb_headroom = val; @@ -172,45 +151,9 @@ LogConfig::read_configuration_variables() ::exit(1); } - // COLLATION - val = (int)REC_ConfigReadInteger("proxy.local.log.collation_mode"); - // do not restrict value so that error message is logged if - // collation_mode is out of range - collation_mode = val; - - ptr = REC_ConfigReadString("proxy.config.log.collation_host"); - if (ptr != nullptr) { - ats_free(collation_host); - collation_host = ptr; - } - - val = (int)REC_ConfigReadInteger("proxy.config.log.collation_port"); - if (val >= 0) { - collation_port = val; - } - - val = (int)REC_ConfigReadInteger("proxy.config.log.collation_host_tagged"); - collation_host_tagged = (val > 0); - - val = (int)REC_ConfigReadInteger("proxy.config.log.collation_preproc_threads"); + val = (int)REC_ConfigReadInteger("proxy.config.log.preproc_threads"); if (val > 0 && val <= 128) { - collation_preproc_threads = val; - } - - ptr = REC_ConfigReadString("proxy.config.log.collation_secret"); - if (ptr != nullptr) { - ats_free(collation_secret); - collation_secret = ptr; - } - - val = (int)REC_ConfigReadInteger("proxy.config.log.collation_retry_sec"); - if (val >= 0) { - collation_retry_sec = val; - } - - val = (int)REC_ConfigReadInteger("proxy.config.log.collation_max_send_buffers"); - if (val >= 0) { - collation_max_send_buffers = val; + preproc_threads = val; } // ROLLING @@ -287,18 +230,8 @@ LogConfig::read_configuration_variables() -------------------------------------------------------------------------*/ // TODO: Is UINT_MAX here really correct? -LogConfig::LogConfig() - : initialized(false), - reconfiguration_needed(false), - logging_space_exhausted(false), - m_space_used(0), - m_partition_space_left((int64_t)UINT_MAX), - m_log_collation_accept(nullptr), - m_disk_full(false), - m_disk_low(false), - m_partition_full(false), - m_partition_low(false), - m_log_directory_inaccessible(false) +LogConfig::LogConfig() : m_partition_space_left((int64_t)UINT_MAX) + { // Setup the default values for all LogConfig public variables so that // a LogConfig object is valid upon return from the constructor even @@ -315,77 +248,7 @@ LogConfig::LogConfig() LogConfig::~LogConfig() { - // we don't delete the log collation accept because it may be transferred - // to another LogConfig object - // - // delete m_log_collation_accept; - - ats_free(hostname); ats_free(logfile_dir); - ats_free(collation_host); - ats_free(collation_secret); -} - -/*------------------------------------------------------------------------- - LogConfig::setup_collation - -------------------------------------------------------------------------*/ - -void -LogConfig::setup_collation(LogConfig *prev_config) -{ - // Set-up the collation status, but only if collation is enabled and - // there are valid entries for the collation host and port. - // - if (collation_mode < Log::NO_COLLATION || collation_mode >= Log::N_COLLATION_MODES) { - Note("Invalid value %d for proxy.local.log.collation_mode" - " configuration variable (valid range is from %d to %d)\n" - "Log collation disabled", - collation_mode, Log::NO_COLLATION, Log::N_COLLATION_MODES - 1); - } else if (collation_mode == Log::NO_COLLATION) { - // if the previous configuration had a collation accept, delete it - // - if (prev_config && prev_config->m_log_collation_accept) { - delete prev_config->m_log_collation_accept; - prev_config->m_log_collation_accept = nullptr; - } - } else { - Warning("Log collation is deprecated as of ATS v8.0.0!"); - if (!collation_port) { - Note("Cannot activate log collation, %d is an invalid collation port", collation_port); - } else if (collation_mode > Log::COLLATION_HOST && strcmp(collation_host, "none") == 0) { - Note("Cannot activate log collation, \"%s\" is an invalid collation host", collation_host); - } else { - if (collation_mode == Log::COLLATION_HOST) { - ink_assert(m_log_collation_accept == nullptr); - - if (prev_config && prev_config->m_log_collation_accept) { - if (prev_config->collation_port == collation_port) { - m_log_collation_accept = prev_config->m_log_collation_accept; - } else { - delete prev_config->m_log_collation_accept; - } - } - - if (!m_log_collation_accept) { - Log::collation_port = collation_port; - m_log_collation_accept = new LogCollationAccept(collation_port); - } - Debug("log", "I am a collation host listening on port %d.", collation_port); - } else { - Debug("log", - "I am a collation client (%d)." - " My collation host is %s:%d", - collation_mode, collation_host, collation_port); - } - - Debug("log", "using iocore log collation"); - if (collation_host_tagged) { - LogFormat::turn_tagging_on(); - } else { - LogFormat::turn_tagging_off(); - } - } - } } /*------------------------------------------------------------------------- @@ -399,8 +262,6 @@ LogConfig::init(LogConfig *prev_config) ink_assert(!initialized); - setup_collation(prev_config); - update_space_used(); // create log objects @@ -417,7 +278,7 @@ LogConfig::init(LogConfig *prev_config) Debug("log", "creating predefined error log object"); errlog = new LogObject(fmt.get(), logfile_dir, "error.log", LOG_FILE_ASCII, nullptr, (Log::RollingEnabledValues)rolling_enabled, - collation_preproc_threads, rolling_interval_sec, rolling_offset_hr, rolling_size_mb); + preproc_threads, rolling_interval_sec, rolling_offset_hr, rolling_size_mb); log_object_manager.manage_object(errlog); errlog->set_fmt_timestamps(); @@ -438,16 +299,6 @@ LogConfig::init(LogConfig *prev_config) ink_atomic_swap(&Log::error_log, errlog); - // determine if we should use the orphan log space value or not - // we use it if all objects are collation clients, or if some are and - // the specified space for collation is larger than that for local files - // - size_t num_collation_clients = log_object_manager.get_num_collation_clients(); - use_orphan_log_space_value = (num_collation_clients == 0 ? false : - (log_object_manager.get_num_objects() == num_collation_clients ? - true : - max_space_mb_for_orphan_logs > max_space_mb_for_logs)); - initialized = true; } @@ -467,18 +318,12 @@ LogConfig::display(FILE *fd) fprintf(fd, " log_buffer_size = %d\n", log_buffer_size); fprintf(fd, " max_secs_per_buffer = %d\n", max_secs_per_buffer); fprintf(fd, " max_space_mb_for_logs = %d\n", max_space_mb_for_logs); - fprintf(fd, " max_space_mb_for_orphan_logs = %d\n", max_space_mb_for_orphan_logs); - fprintf(fd, " use_orphan_log_space_value = %d\n", use_orphan_log_space_value); fprintf(fd, " max_space_mb_headroom = %d\n", max_space_mb_headroom); fprintf(fd, " hostname = %s\n", hostname); fprintf(fd, " logfile_dir = %s\n", logfile_dir); fprintf(fd, " logfile_perm = 0%o\n", logfile_perm); - fprintf(fd, " collation_mode = %d\n", collation_mode); - fprintf(fd, " collation_host = %s\n", collation_host); - fprintf(fd, " collation_port = %d\n", collation_port); - fprintf(fd, " collation_host_tagged = %d\n", collation_host_tagged); - fprintf(fd, " collation_preproc_threads = %d\n", collation_preproc_threads); - fprintf(fd, " collation_secret = %s\n", collation_secret); + + fprintf(fd, " preproc_threads = %d\n", preproc_threads); fprintf(fd, " rolling_enabled = %d\n", rolling_enabled); fprintf(fd, " rolling_interval_sec = %d\n", rolling_interval_sec); fprintf(fd, " rolling_offset_hr = %d\n", rolling_offset_hr); @@ -556,29 +401,11 @@ void LogConfig::register_config_callbacks() { static const char *names[] = { - "proxy.config.log.log_buffer_size", - "proxy.config.log.max_secs_per_buffer", - "proxy.config.log.max_space_mb_for_logs", - "proxy.config.log.max_space_mb_for_orphan_logs", - "proxy.config.log.max_space_mb_headroom", - "proxy.config.log.logfile_perm", - "proxy.config.log.hostname", - "proxy.config.log.logfile_dir", - "proxy.local.log.collation_mode", - "proxy.config.log.collation_host", - "proxy.config.log.collation_port", - "proxy.config.log.collation_host_tagged", - "proxy.config.log.collation_secret", - "proxy.config.log.collation_retry_sec", - "proxy.config.log.collation_max_send_buffers", - "proxy.config.log.rolling_enabled", - "proxy.config.log.rolling_interval_sec", - "proxy.config.log.rolling_offset_hr", - "proxy.config.log.rolling_size_mb", - "proxy.config.log.auto_delete_rolled_files", - "proxy.config.log.config.filename", - "proxy.config.log.sampling_frequency", - "proxy.config.log.file_stat_frequency", + "proxy.config.log.log_buffer_size", "proxy.config.log.max_secs_per_buffer", "proxy.config.log.max_space_mb_for_logs", + "proxy.config.log.max_space_mb_headroom", "proxy.config.log.logfile_perm", "proxy.config.log.hostname", + "proxy.config.log.logfile_dir", "proxy.config.log.rolling_enabled", "proxy.config.log.rolling_interval_sec", + "proxy.config.log.rolling_offset_hr", "proxy.config.log.rolling_size_mb", "proxy.config.log.auto_delete_rolled_files", + "proxy.config.log.config.filename", "proxy.config.log.sampling_frequency", "proxy.config.log.file_stat_frequency", "proxy.config.log.space_used_frequency", }; @@ -668,7 +495,7 @@ LogConfig::register_stat_callbacks() void LogConfig::register_mgmt_callbacks() { - RecRegisterManagerCb(REC_EVENT_ROLL_LOG_FILES, &LogConfig::reconfigure_mgmt_variables, nullptr); + RecRegisterManagerCb(REC_EVENT_ROLL_LOG_FILES, &LogConfig::reconfigure_mgmt_variables); } /*------------------------------------------------------------------------- @@ -972,14 +799,14 @@ LogConfig::evaluate_config() return false; } - Note("loading logging.yaml"); + Note("logging.yaml loading ..."); YamlLogConfig y(this); bool zret = y.parse(path.get()); if (zret) { - Note("logging.yaml done reloading!"); + Note("logging.yaml finished loading"); } else { - Note("failed to reload logging.yaml"); + Note("logging.yaml failed to load"); } return zret; diff --git a/proxy/logging/LogConfig.h b/proxy/logging/LogConfig.h index d8efcf3559b..c36d38b0681 100644 --- a/proxy/logging/LogConfig.h +++ b/proxy/logging/LogConfig.h @@ -31,6 +31,7 @@ #include "records/P_RecProcess.h" #include "ProxyConfig.h" #include "LogObject.h" +#include "tscpp/util/MemSpan.h" /* Instead of enumerating the stats in DynamicStats.h, each module needs to enumerate its stats separately and register them with librecords @@ -76,7 +77,6 @@ enum { extern RecRawStatBlock *log_rsb; struct dirent; -struct LogCollationAccept; /*------------------------------------------------------------------------- LogDeleteCandidate, LogDeletingInfo&Descriptor @@ -187,12 +187,6 @@ class LogConfig : public ConfigInfo bool space_to_write(int64_t bytes_to_write) const; - bool - am_collation_host() const - { - return collation_mode == Log::COLLATION_HOST; - } - bool space_is_short() const { @@ -210,12 +204,12 @@ class LogConfig : public ConfigInfo void read_configuration_variables(); // CVR This is the mgmt callback function, hence all the strange arguments - static void *reconfigure_mgmt_variables(void *token, char *data_raw, int data_len); + static void reconfigure_mgmt_variables(ts::MemSpan); int get_max_space_mb() const { - return (use_orphan_log_space_value ? max_space_mb_for_orphan_logs : max_space_mb_for_logs); + return max_space_mb_for_logs; } void @@ -231,10 +225,10 @@ class LogConfig : public ConfigInfo } public: - bool initialized; - bool reconfiguration_needed; - bool logging_space_exhausted; - int64_t m_space_used; + bool initialized = false; + bool reconfiguration_needed = false; + bool logging_space_exhausted = false; + int64_t m_space_used = 0; int64_t m_partition_space_left; bool roll_log_files_now; // signal that files must be rolled @@ -246,15 +240,11 @@ class LogConfig : public ConfigInfo int log_buffer_size; int max_secs_per_buffer; int max_space_mb_for_logs; - int max_space_mb_for_orphan_logs; int max_space_mb_headroom; int logfile_perm; - int collation_mode; - int collation_port; - bool collation_host_tagged; - int collation_preproc_threads; - int collation_retry_sec; - int collation_max_send_buffers; + + int preproc_threads; + Log::RollingEnabledValues rolling_enabled; int rolling_interval_sec; int rolling_offset_hr; @@ -273,28 +263,18 @@ class LogConfig : public ConfigInfo char *hostname; char *logfile_dir; - char *collation_host; - char *collation_secret; private: bool evaluate_config(); void setup_default_values(); - void setup_collation(LogConfig *prev_config); private: - // if true, use max_space_mb_for_orphan_logs to determine the amount - // of space that logging can use, otherwise use max_space_mb_for_logs - // - bool use_orphan_log_space_value; - - LogCollationAccept *m_log_collation_accept; - - bool m_disk_full; - bool m_disk_low; - bool m_partition_full; - bool m_partition_low; - bool m_log_directory_inaccessible; + bool m_disk_full = false; + bool m_disk_low = false; + bool m_partition_full = false; + bool m_partition_low = false; + bool m_log_directory_inaccessible = false; // noncopyable // -- member functions not allowed -- diff --git a/proxy/logging/LogField.cc b/proxy/logging/LogField.cc index 0b1ce143580..563638d55e3 100644 --- a/proxy/logging/LogField.cc +++ b/proxy/logging/LogField.cc @@ -701,7 +701,7 @@ LogField::fieldlist_contains_aggregates(const char *fieldlist) heap with "new" and that each element is on at most ONE list. To enforce this, items are copied by default, using the copy ctor. -------------------------------------------------------------------------*/ -LogFieldList::LogFieldList() : m_marshal_len(0) {} +LogFieldList::LogFieldList() {} LogFieldList::~LogFieldList() { diff --git a/proxy/logging/LogField.h b/proxy/logging/LogField.h index 236258b141c..e02dba36ab9 100644 --- a/proxy/logging/LogField.h +++ b/proxy/logging/LogField.h @@ -50,7 +50,7 @@ struct LogSlice { // Initialize LogSlice by slice notation, // the str looks like: "xxx[0:30]". // - LogSlice(char *str); + explicit LogSlice(char *str); // // Convert slice notation to target string's offset, @@ -79,7 +79,7 @@ class LogField typedef int (LogAccess::*MarshalFunc)(char *buf); typedef int (*UnmarshalFunc)(char **buf, char *dest, int len); typedef int (*UnmarshalFuncWithSlice)(char **buf, char *dest, int len, LogSlice *slice); - typedef int (*UnmarshalFuncWithMap)(char **buf, char *dest, int len, Ptr map); + typedef int (*UnmarshalFuncWithMap)(char **buf, char *dest, int len, const Ptr &map); typedef void (LogAccess::*SetFunc)(char *buf, int len); enum Type { @@ -177,7 +177,7 @@ class LogField void set_aggregate_op(Aggregate agg_op); void update_aggregate(int64_t val); - static void init_milestone_container(void); + static void init_milestone_container(); static Container valid_container_name(char *name); static Aggregate valid_aggregate_name(char *name); static bool fieldlist_contains_aggregates(const char *fieldlist); @@ -264,7 +264,7 @@ class LogFieldList LogFieldList &operator=(const LogFieldList &rhs) = delete; private: - unsigned m_marshal_len; + unsigned m_marshal_len = 0; Queue m_field_list; std::string _badSymbols; }; diff --git a/proxy/logging/LogFieldAliasMap.cc b/proxy/logging/LogFieldAliasMap.cc index 4e186d061b7..caf68a94599 100644 --- a/proxy/logging/LogFieldAliasMap.cc +++ b/proxy/logging/LogFieldAliasMap.cc @@ -44,7 +44,6 @@ LogFieldAliasTable::init(size_t numPairs, ...) size_t n; va_list ap; va_start(ap, numPairs); - char *name; /* A note on the varargs - Although IntType is used internally the compiler doesn't know that @@ -73,7 +72,7 @@ LogFieldAliasTable::init(size_t numPairs, ...) for (n = 0; n < numPairs; n++) { IntType val = va_arg(ap, int); size_t i = val - m_min; - name = va_arg(ap, char *); + char *name = va_arg(ap, char *); m_table[i].name = ats_strdup(name); m_table[i].length = strlen(name); diff --git a/proxy/logging/LogFieldAliasMap.h b/proxy/logging/LogFieldAliasMap.h index 1d605770253..673cdf749a6 100644 --- a/proxy/logging/LogFieldAliasMap.h +++ b/proxy/logging/LogFieldAliasMap.h @@ -103,11 +103,11 @@ table->init(3, 1, "one", 2, "two", 7, "seven") *****************************************************************************/ struct LogFieldAliasTableEntry { - bool valid; // entry in table is valid - char *name; // the string equivalent - size_t length; // the length of the string + bool valid = false; // entry in table is valid + char *name = nullptr; // the string equivalent + size_t length = 0; // the length of the string - LogFieldAliasTableEntry() : valid(false), name(nullptr), length(0) {} + LogFieldAliasTableEntry() {} ~LogFieldAliasTableEntry() { if (name) { @@ -119,13 +119,13 @@ struct LogFieldAliasTableEntry { class LogFieldAliasTable : public LogFieldAliasMap { private: - IntType m_min; // minimum numeric value - IntType m_max; // maximum numeric value - IntType m_entries; // number of entries in table - LogFieldAliasTableEntry *m_table; // array of table entries + IntType m_min = 0; // minimum numeric value + IntType m_max = 0; // maximum numeric value + IntType m_entries = 0; // number of entries in table + LogFieldAliasTableEntry *m_table = nullptr; // array of table entries public: - LogFieldAliasTable() : m_min(0), m_max(0), m_entries(0), m_table(nullptr) {} + LogFieldAliasTable() {} ~LogFieldAliasTable() override { delete[] m_table; } void init(size_t numPairs, ...); @@ -183,34 +183,4 @@ class LogFieldAliasTable : public LogFieldAliasMap } }; -/***************************************************************************** - -The LogFieldAliasTimehex class implements a LogFieldAliasMap that converts time -from their integer value to the "hex" notation and back. - - *****************************************************************************/ - -class LogFieldAliasTimeHex : public LogFieldAliasMap -{ -public: - int - asInt(char *str, IntType *time, bool /* case_sensitive ATS_UNUSED */) const override - { - unsigned long a; - // coverity[secure_coding] - if (sscanf(str, "%lx", (unsigned long *)&a) == 1) { - *time = (IntType)a; - return ALL_OK; - } else { - return INVALID_STRING; - } - } - - int - asString(IntType time, char *buf, size_t bufLen, size_t *numCharsPtr = nullptr) const override - { - return (LogUtils::timestamp_to_hex_str(time, buf, bufLen, numCharsPtr) ? BUFFER_TOO_SMALL : ALL_OK); - } -}; - // LOG_FIELD_ALIAS_MAP_H diff --git a/proxy/logging/LogFile.cc b/proxy/logging/LogFile.cc index 7d70bbdde4a..3f9a5819b63 100644 --- a/proxy/logging/LogFile.cc +++ b/proxy/logging/LogFile.cc @@ -38,7 +38,6 @@ #include "P_EventSystem.h" #include "I_Machine.h" -#include "LogSock.h" #include "tscore/BaseLogFile.h" #include "LogField.h" @@ -46,7 +45,6 @@ #include "LogFormat.h" #include "LogBuffer.h" #include "LogFile.h" -#include "LogHost.h" #include "LogObject.h" #include "LogUtils.h" #include "LogConfig.h" diff --git a/proxy/logging/LogFile.h b/proxy/logging/LogFile.h index f439cb23ef1..fe3b1102e5f 100644 --- a/proxy/logging/LogFile.h +++ b/proxy/logging/LogFile.h @@ -29,7 +29,6 @@ #include "tscore/ink_platform.h" #include "LogBufferSink.h" -class LogSock; class LogBuffer; struct LogBufferHeader; class LogObject; diff --git a/proxy/logging/LogFilter.cc b/proxy/logging/LogFilter.cc index e863194cef2..8b85394e883 100644 --- a/proxy/logging/LogFilter.cc +++ b/proxy/logging/LogFilter.cc @@ -34,7 +34,6 @@ #include "LogFormat.h" #include "LogFile.h" #include "LogBuffer.h" -#include "LogHost.h" #include "LogObject.h" #include "LogConfig.h" #include "Log.h" @@ -927,7 +926,7 @@ filters_are_equal(LogFilter *filt1, LogFilter *filt2) add() function is overloaded for each sub-type of LogFilter. -------------------------------------------------------------------------*/ -LogFilterList::LogFilterList() : m_does_conjunction(true) {} +LogFilterList::LogFilterList() {} /*------------------------------------------------------------------------- -------------------------------------------------------------------------*/ diff --git a/proxy/logging/LogFilter.h b/proxy/logging/LogFilter.h index 7d9d6a115e5..94d5ff7e7ba 100644 --- a/proxy/logging/LogFilter.h +++ b/proxy/logging/LogFilter.h @@ -301,7 +301,7 @@ class LogFilterList private: Queue m_filter_list; - bool m_does_conjunction; + bool m_does_conjunction = true; // If m_does_conjunction = true // toss_this_entry returns true // if ANY filter tosses entry away. diff --git a/proxy/logging/LogFormat.cc b/proxy/logging/LogFormat.cc index e4571d5b644..e87e1542122 100644 --- a/proxy/logging/LogFormat.cc +++ b/proxy/logging/LogFormat.cc @@ -40,7 +40,6 @@ #include "LogField.h" #include "LogFilter.h" #include "LogFormat.h" -#include "LogHost.h" #include "LogBuffer.h" #include "LogObject.h" #include "LogConfig.h" @@ -201,32 +200,6 @@ LogFormat::LogFormat(const char *name, const char *format_str, unsigned interval m_format_type = format_str ? LOG_FORMAT_CUSTOM : LOG_FORMAT_TEXT; } -//----------------------------------------------------------------------------- -// This constructor is used only in Log::match_logobject -// -// It is awkward because it does not take a format_str but a fieldlist_str -// and a printf_str. These should be derived from a format_str but a -// LogBufferHeader does not store the format_str, if it did, we could probably -// delete this. -// -LogFormat::LogFormat(const char *name, const char *fieldlist_str, const char *printf_str, unsigned interval_sec) - : m_interval_sec(0), - m_interval_next(0), - m_agg_marshal_space(nullptr), - m_valid(false), - m_name_str(nullptr), - m_name_id(0), - m_fieldlist_str(nullptr), - m_fieldlist_id(0), - m_field_count(0), - m_printf_str(nullptr), - m_aggregate(false), - m_format_str(nullptr) -{ - init_variables(name, fieldlist_str, printf_str, interval_sec); - m_format_type = LOG_FORMAT_CUSTOM; -} - /*------------------------------------------------------------------------- LogFormat::LogFormat diff --git a/proxy/logging/LogHost.cc b/proxy/logging/LogHost.cc deleted file mode 100644 index 649682b1cbb..00000000000 --- a/proxy/logging/LogHost.cc +++ /dev/null @@ -1,466 +0,0 @@ -/** @file - - A brief file description - - @section license License - - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - */ - -/*************************************************************************** - LogHost.cc - - - ***************************************************************************/ -#include "tscore/ink_platform.h" - -#include "LogUtils.h" -#include "LogSock.h" -#include "LogField.h" -#include "LogFile.h" -#include "LogFormat.h" -#include "LogBuffer.h" -#include "LogHost.h" -#include "LogObject.h" -#include "LogConfig.h" -#include "Log.h" - -#include "LogCollationClientSM.h" - -#define PING true -#define NOPING false - -static Ptr -make_orphan_logfile(LogHost *lh, const char *filename) -{ - const char *ext = "orphan"; - unsigned name_len = (unsigned)(strlen(filename) + strlen(lh->name()) + strlen(ext) + 16); - char *name_buf = (char *)ats_malloc(name_len); - - // NT: replace ':'s with '-'s. This change is necessary because - // NT doesn't like filenames with ':'s in them. ^_^ - snprintf(name_buf, name_len, "%s%s%s-%u.%s", filename, LOGFILE_SEPARATOR_STRING, lh->name(), lh->port(), ext); - - // XXX should check for conflicts with orphan filename - - Ptr orphan(new LogFile(name_buf, nullptr, LOG_FILE_ASCII, lh->signature())); - - ats_free(name_buf); - return orphan; -} - -/*------------------------------------------------------------------------- - LogHost - -------------------------------------------------------------------------*/ - -LogHost::LogHost(const char *object_filename, uint64_t object_signature) - : m_object_filename(ats_strdup(object_filename)), - m_object_signature(object_signature), - m_port(0), - m_name(nullptr), - m_sock(nullptr), - m_sock_fd(-1), - m_connected(false), - m_orphan_file(nullptr), - m_log_collation_client_sm(nullptr) -{ - ink_zero(m_ip); - ink_zero(m_ipstr); -} - -LogHost::LogHost(const LogHost &rhs) - : m_object_filename(ats_strdup(rhs.m_object_filename)), - m_object_signature(rhs.m_object_signature), - m_ip(rhs.m_ip), - m_port(0), - m_name(ats_strdup(rhs.m_name)), - m_sock(nullptr), - m_sock_fd(-1), - m_connected(false), - m_orphan_file(nullptr), - m_log_collation_client_sm(nullptr) -{ - memcpy(m_ipstr, rhs.m_ipstr, sizeof(m_ipstr)); - m_orphan_file = make_orphan_logfile(this, m_object_filename); -} - -LogHost::~LogHost() -{ - clear(); - ats_free(m_object_filename); -} - -// -// There are 3 ways to establish a LogHost: -// - by "hostname:port" or IP:port", where IP is a string of the -// form "xxx.xxx.xxx.xxx". -// - by specifying a hostname and a port (as separate arguments). -// - by specifying an ip and a port (as separate arguments). -// -bool -LogHost::set_name_port(const char *hostname, unsigned int pt) -{ - if (!hostname || hostname[0] == 0) { - Note("Cannot establish LogHost with NULL hostname"); - return false; - } - - clear(); // remove all previous state for this LogHost - - m_name = ats_strdup(hostname); - m_port = pt; - - Debug("log-host", "LogHost established as %s:%u", this->name(), this->port()); - - m_orphan_file = make_orphan_logfile(this, m_object_filename); - return true; -} - -bool -LogHost::set_ipstr_port(const char *ipstr, unsigned int pt) -{ - if (!ipstr || ipstr[0] == 0) { - Note("Cannot establish LogHost with NULL ipstr"); - return false; - } - - clear(); // remove all previous state for this LogHost - - if (0 != m_ip.load(ipstr)) { - Note("Log host failed to parse IP address %s", ipstr); - } - - m_port = pt; - ink_strlcpy(m_ipstr, ipstr, sizeof(m_ipstr)); - m_name = ats_strdup(ipstr); - - Debug("log-host", "LogHost established as %s:%u", name(), pt); - - m_orphan_file = make_orphan_logfile(this, m_object_filename); - return true; -} - -bool -LogHost::set_name_or_ipstr(const char *name_or_ip) -{ - if (name_or_ip && name_or_ip[0] != '\0') { - std::string_view addr, port; - if (ats_ip_parse(std::string_view(name_or_ip), &addr, &port) == 0) { - uint16_t p = port.empty() ? Log::config->collation_port : atoi(port.data()); - char *n = const_cast(addr.data()); - // Force termination. We know we can do this because the address - // string is followed by either a nul or a colon. - n[addr.size()] = 0; - if (AF_UNSPEC == ats_ip_check_characters(addr)) { - return set_name_port(n, p); - } else { - return set_ipstr_port(n, p); - } - } - } - - return false; -} - -bool -LogHost::connected(bool ping) -{ - if (m_connected && m_sock && m_sock_fd >= 0) { - if (m_sock->is_connected(m_sock_fd, ping)) { - return true; - } - } - return false; -} - -bool -LogHost::connect() -{ - if (!m_ip.isValid()) { - Note("Cannot connect to LogHost; host IP has not been established"); - return false; - } - - if (connected(PING)) { - return true; - } - - IpEndpoint target; - ip_port_text_buffer ipb; - target.assign(m_ip, htons(m_port)); - - if (is_debug_tag_set("log-host")) { - Debug("log-host", "Connecting to LogHost %s", ats_ip_nptop(&target, ipb, sizeof ipb)); - } - - disconnect(); // make sure connection members are initialized - - if (m_sock == nullptr) { - m_sock = new LogSock(); - ink_assert(m_sock != nullptr); - } - m_sock_fd = m_sock->connect(&target.sa); - if (m_sock_fd < 0) { - Note("Connection to LogHost %s failed", ats_ip_nptop(&target, ipb, sizeof ipb)); - return false; - } - m_connected = true; - - if (!authenticated()) { - Note("Authentication to LogHost %s failed", ats_ip_nptop(&target, ipb, sizeof ipb)); - disconnect(); - return false; - } - - return true; -} - -void -LogHost::disconnect() -{ - if (m_sock && m_sock_fd >= 0) { - m_sock->close(m_sock_fd); - m_sock_fd = -1; - } - if (m_log_collation_client_sm) { - delete m_log_collation_client_sm; - m_log_collation_client_sm = nullptr; - } - m_connected = false; -} - -// -// preprocess the given buffer data before sent to target host -// and try to delete it when its reference become zero. -// -bool -LogHost::preproc_and_try_delete(LogBuffer *&lb) -{ - if (lb == nullptr) { - Note("Cannot write LogBuffer to LogHost %s; LogBuffer is NULL", name()); - return false; - } - - LogBufferHeader *buffer_header = lb->header(); - if (buffer_header == nullptr) { - Note("Cannot write LogBuffer to LogHost %s; LogBufferHeader is NULL", name()); - goto done; - } - - if (buffer_header->entry_count == 0) { - // no bytes to write - goto done; - } - - // create a new collation client if necessary - if (m_log_collation_client_sm == nullptr) { - m_log_collation_client_sm = new LogCollationClientSM(this); - ink_assert(m_log_collation_client_sm != nullptr); - } - - // send log_buffer - if (m_log_collation_client_sm->send(lb) <= 0) { - goto done; - } - - return true; - -done: - LogBuffer::destroy(lb); - return false; -} - -// -// write the given buffer data to orphan file and -// try to delete it when its reference become zero. -// -void -LogHost::orphan_write_and_try_delete(LogBuffer *&lb) -{ - RecIncrRawStat(log_rsb, this_thread()->mutex->thread_holding, log_stat_num_lost_before_sent_to_network_stat, - lb->header()->entry_count); - - RecIncrRawStat(log_rsb, this_thread()->mutex->thread_holding, log_stat_bytes_lost_before_sent_to_network_stat, - lb->header()->byte_count); - - if (!Log::config->logging_space_exhausted) { - Debug("log-host", "Sending LogBuffer to orphan file %s", m_orphan_file->get_name()); - m_orphan_file->preproc_and_try_delete(lb); - } else { - Debug("log-host", "logging space exhausted, failed to write orphan file, drop(%" PRIu32 ") bytes", lb->header()->byte_count); - } - LogBuffer::destroy(lb); -} - -void -LogHost::display(FILE *fd) -{ - fprintf(fd, "LogHost: %s:%u, %s\n", name(), port(), (connected(NOPING)) ? "connected" : "not connected"); - - LogHost *host = this; - while (host->failover_link.next != nullptr) { - fprintf(fd, "Failover: %s:%u, %s\n", host->name(), host->port(), (host->connected(NOPING)) ? "connected" : "not connected"); - host = host->failover_link.next; - } -} - -void -LogHost::clear() -{ - // close an established connection and clear the state of this host - - disconnect(); - - ats_free(m_name); - delete m_sock; - m_orphan_file.clear(); - - ink_zero(m_ip); - m_port = 0; - ink_zero(m_ipstr); - m_name = nullptr; - m_sock = nullptr; - m_sock_fd = -1; - m_connected = false; -} - -bool -LogHost::authenticated() -{ - if (!connected(NOPING)) { - Note("Cannot authenticate LogHost %s; not connected", name()); - return false; - } - - Debug("log-host", "Authenticating LogHost %s ...", name()); - char *auth_key = Log::config->collation_secret; - unsigned auth_key_len = (unsigned)::strlen(auth_key) + 1; // incl null - int bytes = m_sock->write(m_sock_fd, auth_key, auth_key_len); - if ((unsigned)bytes != auth_key_len) { - Debug("log-host", "... bad write on authenticate"); - return false; - } - - Debug("log-host", "... authenticated"); - return true; -} - -/*------------------------------------------------------------------------- - LogHostList - -------------------------------------------------------------------------*/ - -LogHostList::LogHostList() {} - -LogHostList::~LogHostList() -{ - clear(); -} - -void -LogHostList::add(LogHost *object, bool copy) -{ - ink_assert(object != nullptr); - if (copy) { - m_host_list.enqueue(new LogHost(*object)); - } else { - m_host_list.enqueue(object); - } -} - -unsigned -LogHostList::count() -{ - unsigned cnt = 0; - for (LogHost *host = first(); host; host = next(host)) { - cnt++; - } - return cnt; -} - -void -LogHostList::clear() -{ - LogHost *host; - while ((host = m_host_list.dequeue())) { - delete host; - } -} - -int -LogHostList::preproc_and_try_delete(LogBuffer *lb) -{ - int success = false; - unsigned nr; - bool need_orphan = true; - LogHost *available_host = nullptr; - - ink_release_assert(lb->m_references == 0); - - nr = count(); - ink_atomic_increment(&lb->m_references, 1); - - for (LogHost *host = first(); host && nr; host = next(host)) { - LogHost *lh = host; - available_host = lh; - - do { - ink_atomic_increment(&lb->m_references, 1); - success = lh->preproc_and_try_delete(lb); - need_orphan = need_orphan && (success == false); - } while (lb && (success == false) && (lh = lh->failover_link.next)); - - nr--; - } - - if (lb != nullptr && need_orphan && available_host) { - ink_atomic_increment(&lb->m_references, 1); - available_host->orphan_write_and_try_delete(lb); - } - - if (lb != nullptr) { - LogBuffer::destroy(lb); - } - return 0; -} - -void -LogHostList::display(FILE *fd) -{ - for (LogHost *host = first(); host; host = next(host)) { - host->display(fd); - } -} - -bool -LogHostList::operator==(LogHostList &rhs) -{ - LogHost *host; - for (host = first(); host; host = next(host)) { - LogHost *rhs_host; - for (rhs_host = rhs.first(); rhs_host; rhs_host = next(host)) { - if ((host->port() == rhs_host->port() && host->ip_addr().isValid() && host->ip_addr() == rhs_host->ip_addr()) || - (host->name() && rhs_host->name() && (strcmp(host->name(), rhs_host->name()) == 0)) || - (*(host->ipstr()) && *(rhs_host->ipstr()) && (strcmp(host->ipstr(), rhs_host->ipstr()) == 0))) { - break; - } - } - if (rhs_host == nullptr) { - return false; - } - } - return true; -} diff --git a/proxy/logging/LogHost.h b/proxy/logging/LogHost.h deleted file mode 100644 index cb62e107fab..00000000000 --- a/proxy/logging/LogHost.h +++ /dev/null @@ -1,165 +0,0 @@ -/** @file - - A brief file description - - @section license License - - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - */ -#pragma once - -class LogSock; -class LogBuffer; -class LogCollationClientSM; - -#include "LogBufferSink.h" - -/*------------------------------------------------------------------------- - LogHost - This object corresponds to a named log collation host. - -------------------------------------------------------------------------*/ -class LogHost -{ - friend class LogCollationClientSM; - -public: - LogHost(const char *object_filename, uint64_t object_signature); - LogHost(const LogHost &); - ~LogHost(); - - bool set_name_or_ipstr(const char *name_or_ipstr); - bool set_ipstr_port(const char *ipstr, unsigned int port); - bool set_name_port(const char *hostname, unsigned int port); - - bool connected(bool ping); - bool connect(); - void disconnect(); - // - // preprocess the given buffer data before sent to target host - // and try to delete it when its reference become zero. - // - bool preproc_and_try_delete(LogBuffer *&lb); - - // - // write the given buffer data to orphan file and - // try to delete it when its reference become zero. - // - void orphan_write_and_try_delete(LogBuffer *&lb); - - const char * - name() const - { - return m_name ? m_name : "UNKNOWN"; - } - - uint64_t - signature() const - { - return m_object_signature; - } - - IpAddr const & - ip_addr() const - { - return m_ip; - } - - in_port_t - port() const - { - return m_port; - } - - const char * - ipstr() const - { - return m_ipstr; - } - - void display(FILE *fd = stdout); - - LogFile * - get_orphan_logfile() const - { - return m_orphan_file.get(); - } - -private: - void clear(); - bool authenticated(); - -private: - char *m_object_filename; - uint64_t m_object_signature; - IpAddr m_ip; // IP address, network order. - in_port_t m_port; // IP port, host order. - ip_text_buffer m_ipstr; - char *m_name; - LogSock *m_sock; - int m_sock_fd; - bool m_connected; - Ptr m_orphan_file; - LogCollationClientSM *m_log_collation_client_sm; - -public: - LINK(LogHost, link); - SLINK(LogHost, failover_link); - - // noncopyable - LogHost &operator=(const LogHost &) = delete; - -private: - // -- member functions not allowed -- - LogHost(); -}; - -/*------------------------------------------------------------------------- - LogHostList - -------------------------------------------------------------------------*/ -class LogHostList : public LogBufferSink -{ -public: - LogHostList(); - ~LogHostList() override; - - void add(LogHost *host, bool copy = true); - unsigned count(); - void clear(); - int preproc_and_try_delete(LogBuffer *lb) override; - - LogHost * - first() - { - return m_host_list.head; - } - - LogHost * - next(LogHost *here) - { - return (here->link).next; - } - - void display(FILE *fd = stdout); - bool operator==(LogHostList &rhs); - - // -- member functions not allowed -- - LogHostList(const LogHostList &) = delete; - LogHostList &operator=(const LogHostList &) = delete; - -private: - Queue m_host_list; -}; diff --git a/proxy/logging/LogObject.cc b/proxy/logging/LogObject.cc index c745bddb277..cb91f5b6610 100644 --- a/proxy/logging/LogObject.cc +++ b/proxy/logging/LogObject.cc @@ -91,8 +91,7 @@ LogBufferManager::preproc_buffers(LogBufferSink *sink) LogObject::LogObject(const LogFormat *format, const char *log_dir, const char *basename, LogFileFormat file_format, const char *header, Log::RollingEnabledValues rolling_enabled, int flush_threads, int rolling_interval_sec, int rolling_offset_hr, int rolling_size_mb, bool auto_created) - : m_auto_created(auto_created), - m_alt_filename(nullptr), + : m_alt_filename(nullptr), m_flags(0), m_signature(0), m_flush_threads(flush_threads), @@ -117,9 +116,6 @@ LogObject::LogObject(const LogFormat *format, const char *log_dir, const char *b // compute_signature is a static function m_signature = compute_signature(m_format, m_basename, m_flags); - // by default, create a LogFile for this object, if a loghost is - // later specified, then we will delete the LogFile object - // m_logFile = new LogFile(m_filename, header, file_format, m_signature, Log::config->ascii_buffer_size, Log::config->max_line_size); LogBuffer *b = new LogBuffer(this, Log::config->log_buffer_size); @@ -133,7 +129,6 @@ LogObject::LogObject(const LogFormat *format, const char *log_dir, const char *b LogObject::LogObject(LogObject &rhs) : RefCountObj(rhs), - m_auto_created(rhs.m_auto_created), m_basename(ats_strdup(rhs.m_basename)), m_filename(ats_strdup(rhs.m_filename)), m_alt_filename(ats_strdup(rhs.m_alt_filename)), @@ -162,11 +157,6 @@ LogObject::LogObject(LogObject &rhs) add_filter(filter); } - LogHost *host; - for (host = rhs.m_host_list.first(); host; host = rhs.m_host_list.next(host)) { - add_loghost(host); - } - // copy gets a fresh log buffer // LogBuffer *b = new LogBuffer(this, Log::config->log_buffer_size); @@ -184,13 +174,6 @@ LogObject::~LogObject() Debug("log-config", "entering LogObject destructor, this=%p", this); preproc_buffers(); - - // here we need to free LogHost if it is remote logging. - if (is_collation_client()) { - if (m_host_list.count()) { - m_host_list.clear(); - } - } ats_free(m_basename); ats_free(m_filename); ats_free(m_alt_filename); @@ -302,23 +285,6 @@ LogObject::set_filter_list(const LogFilterList &list, bool copy) m_filter_list.set_conjunction(list.does_conjunction()); } -void -LogObject::add_loghost(LogHost *host, bool copy) -{ - if (!host) { - return; - } - m_host_list.add(host, copy); - - // A LogObject either writes to a file, or sends to a collation host, but - // not both. By default, it writes to a file. If a LogHost is specified, - // then clear the intelligent Ptr containing LogFile. - // - m_logFile.clear(); - - Debug("log", "added log host %p to object %p for target %s:%d", host, this, host->name(), host->port()); -} - // we conpute the object signature from the fieldlist_str and the printf_str // of the LogFormat rather than from the format_str because the format_str // is not part of a LogBuffer header @@ -355,11 +321,8 @@ LogObject::display(FILE *fd) "flags = %u\n" "signature = %" PRIu64 "\n", this, m_format->name(), m_format, m_basename, m_flags, m_signature); - if (is_collation_client()) { - m_host_list.display(fd); - } else { - fprintf(fd, "full path = %s\n", get_full_filename()); - } + + fprintf(fd, "full path = %s\n", get_full_filename()); m_filter_list.display(fd); fprintf(fd, "++++++++++++++++++++++++++++++++++++++++++++++++++++++++\n"); } @@ -409,7 +372,7 @@ LogObject::_checkout_write(size_t *write_offset, size_t bytes_needed) switch (result_code) { case LogBuffer::LB_OK: - // checkout succeded + // checkout succeeded retry = false; break; @@ -532,6 +495,13 @@ LogObject::va_log(LogAccess *lad, const char *fmt, va_list ap) int LogObject::log(LogAccess *lad, const char *text_entry) +{ + // Clang doesn't like initializing a view with nullptr, have to check. + return this->log(lad, std::string_view{text_entry ? text_entry : ""}); +} + +int +LogObject::log(LogAccess *lad, std::string_view text_entry) { LogBuffer *buffer; size_t offset = 0; // prevent warning @@ -546,7 +516,7 @@ LogObject::log(LogAccess *lad, const char *text_entry) } // this verification must be done here in order to avoid 'dead' LogBuffers // with none zero 'in usage' counters (see _checkout_write for more details) - if (!lad && !text_entry) { + if (!lad && text_entry.empty()) { Note("Call to LogAccess without LAD or text entry; skipping"); return Log::FAIL; } @@ -596,8 +566,8 @@ LogObject::log(LogAccess *lad, const char *text_entry) bytes_needed = m_format->field_count() * INK_MIN_ALIGN; } else if (lad) { bytes_needed = m_format->m_field_list.marshal_len(lad); - } else if (text_entry) { - bytes_needed = LogAccess::strlen(text_entry); + } else if (!text_entry.empty()) { + bytes_needed = INK_ALIGN_DEFAULT(text_entry.size() + 1); // must include null terminator. } if (bytes_needed == 0) { @@ -630,8 +600,10 @@ LogObject::log(LogAccess *lad, const char *text_entry) } else if (lad) { bytes_used = m_format->m_field_list.marshal(lad, &(*buffer)[offset]); ink_assert(bytes_needed >= bytes_used); - } else if (text_entry) { - ink_strlcpy(&(*buffer)[offset], text_entry, bytes_needed); + } else if (!text_entry.empty()) { + char *dst = &(*buffer)[offset]; + memcpy(dst, text_entry.data(), text_entry.size()); + memset(dst + text_entry.size(), 0, bytes_needed - text_entry.size()); } buffer->checkin_write(offset); @@ -776,15 +748,8 @@ LogObject::_roll_files(long last_roll_time, long time_now) if (!writes_to_pipe()) { num_rolled += m_logFile->roll(last_roll_time, time_now); } - } else { - LogHost *host; - for (host = m_host_list.first(); host; host = m_host_list.next(host)) { - LogFile *orphan_logfile = host->get_orphan_logfile(); - if (orphan_logfile) { - num_rolled += orphan_logfile->roll(last_roll_time, time_now); - } - } } + m_last_roll_time = time_now; return num_rolled; } @@ -886,14 +851,10 @@ LogObjectManager::_manage_object(LogObject *log_object, bool is_api_object, int ACQUIRE_API_MUTEX("A LogObjectManager::_manage_object"); } - bool col_client = log_object->is_collation_client(); - int retVal = _solve_internal_filename_conflicts(log_object, maxConflicts); + int retVal = _solve_internal_filename_conflicts(log_object, maxConflicts); if (retVal == NO_FILENAME_CONFLICTS) { - // check for external conflicts only if the object is not a collation - // client - // - if (col_client || (retVal = _solve_filename_conflicts(log_object, maxConflicts), retVal == NO_FILENAME_CONFLICTS)) { + if (retVal = _solve_filename_conflicts(log_object, maxConflicts), retVal == NO_FILENAME_CONFLICTS) { // do filesystem checks // { @@ -911,16 +872,15 @@ LogObjectManager::_manage_object(LogObject *log_object, bool is_api_object, int Debug("log", "LogObjectManager managing object %s (%s) " "[signature = %" PRIu64 ", address = %p]", - log_object->get_base_filename(), col_client ? "collation client" : log_object->get_full_filename(), - log_object->get_signature(), log_object); + log_object->get_base_filename(), log_object->get_full_filename(), log_object->get_signature(), log_object); if (log_object->has_alternate_name()) { - Warning("The full path for the (%s) LogObject %s " + Warning("The full path for the (%s) LogObject " "with signature %" PRIu64 " " "has been set to %s rather than %s because the latter " "is being used by another LogObject", - log_object->receives_remote_data() ? "remote" : "local", log_object->get_base_filename(), - log_object->get_signature(), log_object->get_full_filename(), log_object->get_original_filename()); + log_object->get_base_filename(), log_object->get_signature(), log_object->get_full_filename(), + log_object->get_original_filename()); } } } @@ -1054,14 +1014,12 @@ bool LogObjectManager::_has_internal_filename_conflict(const char *filename, LogObjectList &objects) { for (auto &object : objects) { - if (!object->is_collation_client()) { - // an internal conflict exists if two objects request the - // same filename, regardless of the object signatures, since - // two objects writing to the same file would produce a - // log with duplicate entries and non monotonic timestamps - if (strcmp(object->get_full_filename(), filename) == 0) { - return true; - } + // an internal conflict exists if two objects request the + // same filename, regardless of the object signatures, since + // two objects writing to the same file would produce a + // log with duplicate entries and non monotonic timestamps + if (strcmp(object->get_full_filename(), filename) == 0) { + return true; } } return false; @@ -1180,7 +1138,7 @@ LogObjectManager::open_local_pipes() // for (unsigned i = 0; i < this->_objects.size(); i++) { LogObject *obj = _objects[i]; - if (obj->writes_to_pipe() && !obj->is_collation_client()) { + if (obj->writes_to_pipe()) { obj->m_logFile->open_file(); } } @@ -1290,19 +1248,6 @@ LogObjectManager::find_by_format_name(const char *name) const return nullptr; } -unsigned -LogObjectManager::get_num_collation_clients() const -{ - unsigned coll_clients = 0; - - for (auto _object : this->_objects) { - if (_object && _object->is_collation_client()) { - ++coll_clients; - } - } - return coll_clients; -} - int LogObjectManager::log(LogAccess *lad) { @@ -1310,15 +1255,6 @@ LogObjectManager::log(LogAccess *lad) ProxyMutex *mutex = this_thread()->mutex.get(); for (unsigned i = 0; i < this->_objects.size(); i++) { - // - // Auto created LogObject is only applied to LogBuffer - // data received from network in collation host. It should - // be ignored here. - // - if (_objects[i]->m_auto_created) { - continue; - } - ret |= _objects[i]->log(lad); } @@ -1338,7 +1274,7 @@ LogObjectManager::log(LogAccess *lad) } else if (likely(ret & Log::SKIP)) { RecIncrRawStat(log_rsb, mutex->thread_holding, log_stat_event_log_access_skip_stat, 1); } else { - ink_release_assert("Unexpected result"); + ink_release_assert(!"Unexpected result"); } return ret; diff --git a/proxy/logging/LogObject.h b/proxy/logging/LogObject.h index 3dc50e278d0..73bc3298078 100644 --- a/proxy/logging/LogObject.h +++ b/proxy/logging/LogObject.h @@ -28,7 +28,6 @@ #include "LogFile.h" #include "LogFormat.h" #include "LogFilter.h" -#include "LogHost.h" #include "LogBuffer.h" #include "LogAccess.h" #include "LogFilter.h" @@ -66,10 +65,10 @@ class LogBufferManager { private: ASLL(LogBuffer, write_link) write_list; - int _num_flush_buffers; + int _num_flush_buffers = 0; public: - LogBufferManager() : _num_flush_buffers(0) {} + LogBufferManager() {} inline void add_to_flush_queue(LogBuffer *buffer) { @@ -87,14 +86,11 @@ class LogObject : public RefCountObj public: enum LogObjectFlags { BINARY = 1, - REMOTE_DATA = 2, WRITES_TO_PIPE = 4, LOG_OBJECT_FMT_TIMESTAMP = 8, // always format a timestamp into each log line (for raw text logs) }; // BINARY: log is written in binary format (rather than ascii) - // REMOTE_DATA: object receives data from remote collation clients, so - // it should not be destroyed during a reconfiguration // WRITES_TO_PIPE: object writes to a named pipe rather than to a file LogObject(const LogFormat *format, const char *log_dir, const char *basename, LogFileFormat file_format, const char *header, @@ -105,13 +101,7 @@ class LogObject : public RefCountObj void add_filter(LogFilter *filter, bool copy = true); void set_filter_list(const LogFilterList &list, bool copy = true); - void add_loghost(LogHost *host, bool copy = true); - inline void - set_remote_flag() - { - m_flags |= REMOTE_DATA; - }; inline void set_fmt_timestamps() { @@ -119,6 +109,17 @@ class LogObject : public RefCountObj } int log(LogAccess *lad, const char *text_entry = nullptr); + + /** Log the @a text_entry. + * + * @param lad Log accessor. + * @param text_entry Literal text to log. + * @return Result - value from Log::ReturnCodeFlags. + * + * @see Log::ReturnCodeFlags. + */ + int log(LogAccess *lad, std::string_view text_entry); + int va_log(LogAccess *lad, const char *fmt, va_list ap); unsigned roll_files(long time_now = 0); @@ -142,11 +143,8 @@ class LogObject : public RefCountObj idx = m_buffer_manager_idx++ % m_flush_threads; } - if (m_logFile) { - nfb = m_buffer_manager[idx].preproc_buffers(m_logFile.get()); - } else { - nfb = m_buffer_manager[idx].preproc_buffers(&m_host_list); - } + nfb = m_buffer_manager[idx].preproc_buffers(m_logFile.get()); + return nfb; } @@ -215,20 +213,10 @@ class LogObject : public RefCountObj _setup_rolling(m_rolling_enabled, m_rolling_interval_sec, m_rolling_offset_hr, rolling_size_mb); } - inline bool - is_collation_client() const - { - return (m_logFile ? false : true); - } - inline bool - receives_remote_data() const - { - return m_flags & REMOTE_DATA ? true : false; - } inline bool writes_to_pipe() const { - return m_flags & WRITES_TO_PIPE ? true : false; + return (m_flags & WRITES_TO_PIPE) ? true : false; } inline bool writes_to_disk() @@ -265,11 +253,9 @@ class LogObject : public RefCountObj bool operator==(LogObject &rhs); public: - bool m_auto_created; LogFormat *m_format; Ptr m_logFile; LogFilterList m_filter_list; - LogHostList m_host_list; private: char *m_basename; // the name of the file associated @@ -415,41 +401,19 @@ class LogObjectManager { return _objects.size(); } - unsigned get_num_collation_clients() const; }; inline bool LogObject::operator==(LogObject &old) { - if (!receives_remote_data() && !old.receives_remote_data()) { - return (get_signature() == old.get_signature() && - (is_collation_client() && old.is_collation_client() ? - m_host_list == old.m_host_list : - m_logFile && old.m_logFile && strcmp(m_logFile->get_name(), old.m_logFile->get_name()) == 0) && - (m_filter_list == old.m_filter_list) && - (m_rolling_interval_sec == old.m_rolling_interval_sec && m_rolling_offset_hr == old.m_rolling_offset_hr && - m_rolling_size_mb == old.m_rolling_size_mb)); - } - return false; + return (get_signature() == old.get_signature() && m_logFile && old.m_logFile && + strcmp(m_logFile->get_name(), old.m_logFile->get_name()) == 0 && (m_filter_list == old.m_filter_list) && + (m_rolling_interval_sec == old.m_rolling_interval_sec && m_rolling_offset_hr == old.m_rolling_offset_hr && + m_rolling_size_mb == old.m_rolling_size_mb)); } inline off_t LogObject::get_file_size_bytes() { - if (m_logFile) { - return m_logFile->get_size_bytes(); - } else { - off_t max_size = 0; - LogHost *host; - for (host = m_host_list.first(); host; host = m_host_list.next(host)) { - LogFile *orphan_logfile = host->get_orphan_logfile(); - if (orphan_logfile) { - off_t s = orphan_logfile->get_size_bytes(); - if (s > max_size) { - max_size = s; - } - } - } - return max_size; - } + return m_logFile->get_size_bytes(); } diff --git a/proxy/logging/LogSock.cc b/proxy/logging/LogSock.cc deleted file mode 100644 index 4a37d0c8c6f..00000000000 --- a/proxy/logging/LogSock.cc +++ /dev/null @@ -1,736 +0,0 @@ -/** @file - - A brief file description - - @section license License - - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - */ - -#include "tscore/ink_inet.h" -#include "tscore/ink_string.h" -#include "P_EventSystem.h" - -#include "LogSock.h" -#include "LogUtils.h" - -static const int LS_SOCKTYPE = SOCK_STREAM; -static const int LS_PROTOCOL = 0; - -/** - LogSock - - The constructor establishes the connection table (ct) and initializes the - first entry of the table (index 0) to be the port on which new - connections are accepted. -*/ -LogSock::LogSock(int max_connects) : ct((ConnectTable *)nullptr), m_accept_connections(false), m_max_connections(max_connects + 1) -{ - ink_assert(m_max_connections > 0); - - // - // allocate space for the connection table. - // - ct = new ConnectTable[m_max_connections]; - ink_assert(ct != nullptr); - for (int i = 0; i < m_max_connections; ++i) { - init_cid(i, nullptr, 0, -1, LogSock::LS_STATE_UNUSED); - } - - Debug("log-sock", "LogSocket established"); -} - -/** - LogSock::~LogSock - - Shut down all connections and delete memory for the tables. -*/ -LogSock::~LogSock() -{ - Debug("log-sock", "shutting down LogSocket on [%s:%d]", ct[0].host, ct[0].port); - - this->close(); // close all connections - this->close(0); // close accept socket - delete[] ct; // delete the connection table -} - -/** - LogSock::listen - - This routine sets up the LogSock to begin accepting connections on the - given @param accept port. A maximum number of connections is also specified, - which is used to establish the size of the listen queue. - - @Return zero if all goes well, -1 otherwise. -*/ -int -LogSock::listen(int accept_port, int family) -{ - IpEndpoint bind_addr; - int size = sizeof(bind_addr); - char this_host[MAXDNAME]; - int ret; - ats_scoped_fd accept_sd; - - Debug("log-sock", "Listening ..."); - - // Set up local address for bind. - bind_addr.setToAnyAddr(family); - if (!bind_addr.isValid()) { - Warning("Could not set up socket - invalid address family %d", family); - return -1; - } - bind_addr.port() = htons(accept_port); - size = ats_ip_size(&bind_addr.sa); - - // - // create the socket for accepting new connections - // - accept_sd = ::socket(family, LS_SOCKTYPE, LS_PROTOCOL); - if (accept_sd < 0) { - Warning("Could not create a socket for family %d: %s", family, strerror(errno)); - return -1; - } - // - // Set socket options (NO_LINGER, TCP_NODELAY, SO_REUSEADDR) - // - // CLOSE ON EXEC - if ((ret = safe_fcntl(accept_sd, F_SETFD, FD_CLOEXEC)) < 0) { - Warning("Could not set option CLOSE ON EXEC on socket (%d): %s", ret, strerror(errno)); - return -1; - } - // NO_LINGER - struct linger l; - l.l_onoff = 0; - l.l_linger = 0; - if ((ret = safe_setsockopt(accept_sd, SOL_SOCKET, SO_LINGER, (char *)&l, sizeof(l))) < 0) { - Warning("Could not set option NO_LINGER on socket (%d): %s", ret, strerror(errno)); - return -1; - } - // REUSEADDR - if ((ret = safe_setsockopt(accept_sd, SOL_SOCKET, SO_REUSEADDR, SOCKOPT_ON, sizeof(int))) < 0) { - Warning("Could not set option REUSEADDR on socket (%d): %s", ret, strerror(errno)); - return -1; - } - - // Bind to local address. - if ((ret = safe_bind(accept_sd, &bind_addr.sa, size)) < 0) { - Warning("Could not bind port: %s", strerror(errno)); - return -1; - } - - if ((ret = safe_setsockopt(accept_sd, IPPROTO_TCP, TCP_NODELAY, SOCKOPT_ON, sizeof(int))) < 0) { - Warning("Could not set option TCP_NODELAY on socket (%d): %s", ret, strerror(errno)); - return -1; - } - - if ((ret = safe_setsockopt(accept_sd, SOL_SOCKET, SO_KEEPALIVE, SOCKOPT_ON, sizeof(int))) < 0) { - Warning("Could not set option SO_KEEPALIVE on socket (%d): %s", ret, strerror(errno)); - return -1; - } - - // - // if the accept_port argument was zero, then the system just picked - // one for us, so we need to find out what it was and record it in the - // connection table correctly. - // - if (accept_port == 0) { - ret = safe_getsockname(accept_sd, &bind_addr.sa, &size); - if (ret == 0) { - accept_port = ntohs(bind_addr.port()); - } - } - // - // establish the listen queue for incomming connections - // - if ((ret = safe_listen(accept_sd, m_max_connections)) < 0) { - Warning("Could not establish listen queue: %s", strerror(errno)); - return -1; - } - // - // initialize the first entry of the table for accepting incoming - // connection requests. - // - if (gethostname(&this_host[0], MAXDNAME) != 0) { - snprintf(this_host, sizeof(this_host), "unknown-host"); - } - init_cid(0, this_host, accept_port, accept_sd, LogSock::LS_STATE_INCOMING); - - m_accept_connections = true; - Debug("log-sock", "LogSocket established on [%s:%d]", this_host, accept_port); - - accept_sd.release(); - return 0; -} - -/** - LogSock::accept - - Accept a new connection. This is a blocking operation, so you may want - to use one of the non-blocking pending_XXX calls to see if there is a - connection first. - @return This returns the table index for the new connection. -*/ -int -LogSock::accept() -{ - int cid, connect_sd; - IpEndpoint connect_addr; - socklen_t size = sizeof(connect_addr); - in_port_t connect_port; - - if (!m_accept_connections || ct[0].sd < 0) { - return LogSock::LS_ERROR_NO_CONNECTION; - } - - cid = new_cid(); - if (cid < 0) { - return LogSock::LS_ERROR_CONNECT_TABLE_FULL; - } - - Debug("log-sock", "waiting to accept a new connection"); - - connect_sd = ::accept(ct[0].sd, &connect_addr.sa, &size); - if (connect_sd < 0) { - return LogSock::LS_ERROR_ACCEPT; - } - connect_port = ntohs(connect_addr.port()); - - init_cid(cid, nullptr, connect_port, connect_sd, LogSock::LS_STATE_INCOMING); - - Debug("log-sock", "new connection accepted, cid = %d, port = %d", cid, connect_port); - - return cid; -} - -/** - LogSock::connect - - Establish a new connection to another machine [host:port], and place this - information into the connection and poll tables. -*/ -int -LogSock::connect(sockaddr const *ip) -{ - int cid, ret; - ats_scoped_fd connect_sd; - uint16_t port; - - if (!ats_is_ip(ip)) { - Note("Invalid host IP or port number for connection"); - return LogSock::LS_ERROR_NO_SUCH_HOST; - } - port = ntohs(ats_ip_port_cast(ip)); - - ip_port_text_buffer ipstr; - Debug("log-sock", "connecting to [%s:%d]", ats_ip_nptop(ip, ipstr, sizeof(ipstr)), port); - - // get an index into the connection table - cid = new_cid(); - if (cid < 0) { - Note("No more connections allowed for this socket"); - return LogSock::LS_ERROR_CONNECT_TABLE_FULL; - } - // initialize a new socket descriptor - connect_sd = ::socket(ip->sa_family, LS_SOCKTYPE, LS_PROTOCOL); - if (connect_sd < 0) { - Note("Error initializing socket for connection: %d", static_cast(connect_sd)); - return LogSock::LS_ERROR_SOCKET; - } - - if ((ret = safe_setsockopt(connect_sd, IPPROTO_TCP, TCP_NODELAY, SOCKOPT_ON, sizeof(int))) < 0) { - Note("Could not set option TCP_NODELAY on socket (%d): %s", ret, strerror(errno)); - return -1; - } - - if ((ret = safe_setsockopt(connect_sd, SOL_SOCKET, SO_KEEPALIVE, SOCKOPT_ON, sizeof(int))) < 0) { - Note("Could not set option SO_KEEPALIVE on socket (%d): %s", ret, strerror(errno)); - return -1; - } - - // attempt to connect - if (::connect(connect_sd, ip, ats_ip_size(ip)) != 0) { - Note("Failure to connect"); - return LogSock::LS_ERROR_CONNECT; - } - - init_cid(cid, ipstr, port, connect_sd, LogSock::LS_STATE_OUTGOING); - - Debug("log-sock", "outgoing connection to [%s:%d] established, fd = %d", ipstr, port, cid); - - connect_sd.release(); - return cid; -} - -/** - LogSock::pending_data - - This private routine checks for incoming data on some of the socket - descriptors. - @return Returns true if there is something incoming, with *cid - set to the index corresponding to the incoming socket. -*/ -bool -LogSock::pending_data(int *cid, int timeout_msec, bool include_connects) -{ - int start_index, ret, n_poll_fds, i; - static struct pollfd fds[LS_CONST_MAX_CONNS]; - int fd_to_cid[LS_CONST_MAX_CONNS]; - - ink_assert(m_max_connections <= (LS_CONST_MAX_CONNS + 1)); - ink_assert(cid != nullptr); - ink_assert(timeout_msec >= 0); - - // - // we'll use the poll() routine, which replaces the select routine - // to support a larger number of socket descriptors. to use poll, - // we need to set-up a pollfd array for the socket descriptors - // that will be polled. - // - - if (*cid >= 0) { // look for data on this specific socket - - ink_assert(*cid < m_max_connections); - fds[0].fd = ct[*cid].sd; - fds[0].events = POLLIN; - fds[0].revents = 0; - fd_to_cid[0] = *cid; - n_poll_fds = 1; - - } else { // look for data on any INCOMING socket - - if (include_connects) { - start_index = 0; - } else { - start_index = 1; - } - n_poll_fds = 0; - for (i = start_index; i < m_max_connections; i++) { - if (ct[i].state == LogSock::LS_STATE_INCOMING) { - fds[n_poll_fds].fd = ct[i].sd; - fds[n_poll_fds].events = POLLIN; - fds[n_poll_fds].revents = 0; - fd_to_cid[n_poll_fds] = i; - n_poll_fds++; - } - } - } - - if (n_poll_fds == 0) { - return false; - } - - ret = ::poll(fds, n_poll_fds, timeout_msec); - - if (ret == 0) { - return false; // timeout - } else if (ret < 0) { - Debug("log-sock", "error on poll"); - return false; // error - } - // - // a positive return value indicates how many descriptors had something - // waiting on them. We only care about finding one of them, so we'll - // look for the first one with an revents flag set to POLLIN. - // - - for (i = 0; i < n_poll_fds; i++) { - if (fds[i].revents & POLLIN) { - *cid = fd_to_cid[i]; - Debug("log-sock", "poll successful on index %d", *cid); - return true; - } - } - - Debug("log-sock", "invalid revents in the poll table"); - return false; -} - -/** - LogSock::pending_any - - Check for incomming data on any of the INCOMING sockets. -*/ -bool -LogSock::pending_any(int *cid, int timeout_msec) -{ - ink_assert(cid != nullptr); - *cid = -1; - if (m_accept_connections) { - return pending_data(cid, timeout_msec, true); - } else { - return pending_data(cid, timeout_msec, false); - } -} - -/*------------------------------------------------------------------------- - LogSock::pending_message_any - - Check for an incomming message on any of the INCOMING sockets, aside from - the socket reserved for accepting new connections. - -------------------------------------------------------------------------*/ - -bool -LogSock::pending_message_any(int *cid, int timeout_msec) -{ - ink_assert(cid != nullptr); - *cid = -1; - return pending_data(cid, timeout_msec, false); -} - -/** - LogSock::pending_message_on - - Check for incomming data on the specified socket. -*/ -bool -LogSock::pending_message_on(int cid, int timeout_msec) -{ - return pending_data(&cid, timeout_msec, false); -} - -/** - LogSock::pending_connect - - Check for an incoming connection request on the socket reserved for that - (cid = 0). -*/ -bool -LogSock::pending_connect(int timeout_msec) -{ - int cid = 0; - if (m_accept_connections) { - return pending_data(&cid, timeout_msec, true); - } else { - return false; - } -} - -/** - LogSock::close - - Close one (cid specified) or all (no argument) sockets, except for the - incomming connection socket. -*/ -void -LogSock::close(int cid) -{ - ink_assert(cid >= 0 && cid < m_max_connections); - - Debug("log-sock", "closing connection for cid %d", cid); - - if (ct[cid].state != LogSock::LS_STATE_UNUSED) { - ::close(ct[cid].sd); - delete ct[cid].host; - ct[cid].state = LogSock::LS_STATE_UNUSED; - } -} - -void -LogSock::close() -{ - for (int i = 1; i < m_max_connections; i++) { - this->close(i); - } -} - -/** - LogSock::write - - Write data onto the socket corresponding to the given cid. Return the - number of bytes actually written. -*/ -int -LogSock::write(int cid, void *buf, int bytes) -{ - LogSock::MsgHeader header = {0}; - header.msg_bytes = 0; - int ret; - - ink_assert(cid >= 0 && cid < m_max_connections); - - if (buf == nullptr || bytes == 0) { - return 0; - } - - if (ct[cid].state != LogSock::LS_STATE_OUTGOING) { - return LogSock::LS_ERROR_STATE; - } - - Debug("log-sock", "Sending %d bytes to cid %d", bytes, cid); - - // - // send the message header - // - Debug("log-sock", " sending header (%zu bytes)", sizeof(LogSock::MsgHeader)); - header.msg_bytes = bytes; - ret = ::send(ct[cid].sd, (char *)&header, sizeof(LogSock::MsgHeader), 0); - if (ret != sizeof(LogSock::MsgHeader)) { - return LogSock::LS_ERROR_WRITE; - } - // - // send the actual data - // - Debug("log-sock", " sending data (%d bytes)", bytes); - return ::send(ct[cid].sd, (char *)buf, bytes, 0); -} - -/** - LogSock::read - - Read data from the specified connection. This is a blocking call, so you - may want to use one of the pending_XXX calls to see if there is anything - to read first. Returns number of bytes read. -*/ -int -LogSock::read(int cid, void *buf, unsigned maxsize) -{ - LogSock::MsgHeader header; - unsigned size; - - ink_assert(cid >= 0 && cid < m_max_connections); - ink_assert(buf != nullptr); - - if (ct[cid].state != LogSock::LS_STATE_INCOMING) { - return LogSock::LS_ERROR_STATE; - } - - Debug("log-sock", "reading data from cid %d", cid); - - if (read_header(ct[cid].sd, &header) < 0) { - return LogSock::LS_ERROR_READ; - } - - size = ((unsigned)header.msg_bytes < maxsize) ? (unsigned)header.msg_bytes : maxsize; - return read_body(ct[cid].sd, buf, size); -} - -/** - LogSock::read_alloc - - This routine reads data from the specified connection, and returns a - pointer to newly allocated space (allocated with new) containing the - data. The number of bytes read is set in the argument size, which is - expected to be a pointer to an int. -*/ -void * -LogSock::read_alloc(int cid, int *size) -{ - LogSock::MsgHeader header; - char *data; - - ink_assert(cid >= 0 && cid < m_max_connections); - - if (ct[cid].state != LogSock::LS_STATE_INCOMING) { - return nullptr; - } - - Debug("log-sock", "reading data from cid %d", cid); - - if (read_header(ct[cid].sd, &header) < 0) { - return nullptr; - } - - data = new char[header.msg_bytes]; - ink_assert(data != nullptr); - - if ((*size = read_body(ct[cid].sd, data, header.msg_bytes)) < 0) { - delete[] data; - data = nullptr; - } - - return data; -} - -/** - */ -bool -LogSock::is_connected(int cid, bool ping) const -{ - int i, j, flags; - - ink_assert(cid >= 0 && cid < m_max_connections); - - if (ct[cid].state == LogSock::LS_STATE_UNUSED) { - return false; - } - - if (ping) { - flags = fcntl(ct[cid].sd, F_GETFL); - ::fcntl(ct[cid].sd, F_SETFL, O_NONBLOCK); - j = ::recv(ct[cid].sd, (char *)&i, sizeof(int), MSG_PEEK); - ::fcntl(ct[cid].sd, F_SETFL, flags); - if (j != 0) { - return true; - } else { - return false; - } - } else { - return (ct[cid].sd >= 0); - } -} - -/** - */ -void -LogSock::check_connections() -{ - for (int i = 1; i < m_max_connections; i++) { - if (ct[i].state == LogSock::LS_STATE_INCOMING) { - if (!is_connected(i, true)) { - Debug("log-sock", "Connection %d is no longer connected", i); - close(i); - } - } - } -} - -/** - This routine will check to ensure that the client connecting is - authorized to use the log collation port. To authorize, the client is - expected to send the logging secret string. -*/ -bool -LogSock::authorized_client(int cid, char *key) -{ - // - // Wait for up to 5 seconds for the client to authenticate - // - if (!pending_message_on(cid, 5000)) { - return false; - } - // - // Ok, the client has a pending message, so check to see if it matches - // the given key. - // - char buf[1024]; - int size = this->read(cid, buf, 1024); - ink_assert(size >= 0 && size <= 1024); - - if (strncmp(buf, key, size) == 0) { - return true; - } - - return false; -} - -/** - */ -char * -LogSock::connected_host(int cid) -{ - ink_assert(cid >= 0 && cid < m_max_connections); - return ct[cid].host; -} - -/** - */ -int -LogSock::connected_port(int cid) -{ - ink_assert(cid >= 0 && cid < m_max_connections); - return ct[cid].port; -} - -/*------------------------------------------------------------------------- - LOCAL ROUTINES - -------------------------------------------------------------------------*/ - -/** - LogSock::new_cid -*/ -int -LogSock::new_cid() -{ - int cid = -1; - - for (int i = 1; i < m_max_connections; i++) { - if (ct[i].state == LogSock::LS_STATE_UNUSED) { - cid = i; - break; - } - } - - return cid; -} - -/** - LogSock::init_cid -*/ -void -LogSock::init_cid(int cid, char *host, int port, int sd, LogSock::State state) -{ - ink_assert(cid >= 0 && cid < m_max_connections); - // host can be NULL if it's not known - ink_assert(port >= 0); - // sd can be -1 to indicate no connection yet - ink_assert(state >= 0 && state < LogSock::LS_N_STATES); - - if (host != nullptr) { - const size_t host_size = strlen(host) + 1; - ct[cid].host = new char[host_size]; - ink_strlcpy(ct[cid].host, host, host_size); - } else { - ct[cid].host = nullptr; - } - - ct[cid].port = port; - ct[cid].sd = sd; - ct[cid].state = state; -} - -/** - */ -int -LogSock::read_header(int sd, LogSock::MsgHeader *header) -{ - ink_assert(sd >= 0); - ink_assert(header != nullptr); - - int bytes = ::recv(sd, (char *)header, sizeof(LogSock::MsgHeader), 0); - if (bytes != sizeof(LogSock::MsgHeader)) { - return -1; - } - - return bytes; -} - -/** - */ -int -LogSock::read_body(int sd, void *buf, int bytes) -{ - ink_assert(sd >= 0); - ink_assert(buf != nullptr); - ink_assert(bytes >= 0); - - if (bytes == 0) { - return 0; - } - - unsigned bytes_left = bytes; - unsigned bytes_read; - char *to = (char *)buf; - - while (bytes_left) { - bytes_read = ::recv(sd, to, bytes_left, 0); - to += bytes_read; - bytes_left -= bytes_read; - } - - return bytes; -} diff --git a/proxy/logging/LogSock.h b/proxy/logging/LogSock.h deleted file mode 100644 index eda5d1a126e..00000000000 --- a/proxy/logging/LogSock.h +++ /dev/null @@ -1,128 +0,0 @@ -/** @file - - A brief file description - - @section license License - - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - */ - -#pragma once - -#include "tscore/ink_platform.h" - -/*------------------------------------------------------------------------- - LogSock - - This class implements a multiplexed socket class that supports both - client and server functionality. - -------------------------------------------------------------------------*/ - -class LogSock -{ -public: - enum Constant { - LS_CONST_PACKETSIZE = 1024, - LS_CONST_MAX_CONNS = 256, - }; - - enum Err { - LS_ERROR_UNKNOWN = -1, - LS_ERROR_CONNECT_TABLE_FULL = -3, - LS_ERROR_SOCKET = -4, - LS_ERROR_BIND = -5, - LS_ERROR_CONNECT = -6, - LS_ERROR_ACCEPT = -7, - LS_ERROR_NO_SUCH_HOST = -8, - LS_ERROR_NO_CONNECTION = -9, - LS_ERROR_STATE = -10, - LS_ERROR_WRITE = -11, - LS_ERROR_READ = -12 - }; - - enum State { - LS_STATE_UNUSED = 0, - LS_STATE_INCOMING, - LS_STATE_OUTGOING, - LS_N_STATES, - }; - - LogSock(int max_connects = 1); - ~LogSock(); - - bool pending_any(int *cid, int timeout_msec = 0); - bool pending_message_any(int *cid, int timeout_msec = 0); - bool pending_message_on(int cid, int timeout_msec = 0); - bool pending_connect(int timeout_msec = 0); - - int listen(int accept_port, int family = AF_INET); - int accept(); - int connect(sockaddr const *ip); - - void close(int cid); // this connection - void close(); // all connections - - int write(int cid, void *buf, int bytes); - - int read(int cid, void *buf, unsigned maxsize); - void *read_alloc(int cid, int *size); - - char * - on_host() - { - return ct[0].host; - } - - int - on_port() - { - return ct[0].port; - } - - bool is_connected(int cid, bool ping = false) const; - void check_connections(); - bool authorized_client(int cid, char *key); - char *connected_host(int cid); - int connected_port(int cid); - - // noncopyable - LogSock(const LogSock &) = delete; - LogSock &operator=(const LogSock &) = delete; - -private: - struct ConnectTable { - char *host; // hostname for this connection - int port; // port number for this connection - int sd; // socket descriptor for this connection - State state; // state of this entry - }; - - struct MsgHeader { - int msg_bytes; // length of the following message - }; - - bool pending_data(int *cid, int timeout_msec, bool include_connects); - int new_cid(); - void init_cid(int cid, char *host, int port, int sd, State state); - int read_header(int sd, MsgHeader *header); - int read_body(int sd, void *buf, int bytes); - - ConnectTable *ct; // list of all connections; index 0 is - // the accept port. - bool m_accept_connections; // do we accept new connections? - int m_max_connections; // max size of all tables -}; diff --git a/proxy/logging/LogUtils.cc b/proxy/logging/LogUtils.cc index 94becf250ac..61f2891222c 100644 --- a/proxy/logging/LogUtils.cc +++ b/proxy/logging/LogUtils.cc @@ -21,8 +21,22 @@ See the License for the specific language governing permissions and limitations under the License. */ + +#include #include "tscore/ink_config.h" #include "tscore/ink_string.h" +#include +#include + +#ifdef TEST_LOG_UTILS + +#include "unit-tests/test_LogUtils.h" + +#else + +#include + +#endif #include #include @@ -30,6 +44,8 @@ #include #include #include +#include +#include #include #include @@ -83,12 +99,11 @@ char * LogUtils::timestamp_to_netscape_str(long timestamp) { static char timebuf[64]; // NOTE: not MT safe - static char gmtstr[16]; static long last_timestamp = 0; - static char bad_time[] = "Bad timestamp"; // safety check if (timestamp < 0) { + static char bad_time[] = "Bad timestamp"; return bad_time; } // @@ -118,6 +133,8 @@ LogUtils::timestamp_to_netscape_str(long timestamp) offset = zone / -60; sign = '+'; } + + static char gmtstr[16]; int glen = snprintf(gmtstr, 16, "%c%.2d%.2d", sign, offset / 60, offset % 60); strftime(timebuf, 64 - glen, "%d/%b/%Y:%H:%M:%S ", tms); @@ -139,10 +156,10 @@ LogUtils::timestamp_to_date_str(long timestamp) { static char timebuf[64]; // NOTE: not MT safe static long last_timestamp = 0; - static char bad_time[] = "Bad timestamp"; // safety check if (timestamp < 0) { + static char bad_time[] = "Bad timestamp"; return bad_time; } // @@ -171,10 +188,10 @@ LogUtils::timestamp_to_time_str(long timestamp) { static char timebuf[64]; // NOTE: not MT safe static long last_timestamp = 0; - static char bad_time[] = "Bad timestamp"; // safety check if (timestamp < 0) { + static char bad_time[] = "Bad timestamp"; return bad_time; } // @@ -205,13 +222,13 @@ void LogUtils::manager_alarm(LogUtils::AlarmType alarm_type, const char *msg, ...) { char msg_buf[LOG_MAX_FORMATTED_LINE]; - va_list ap; ink_assert(alarm_type >= 0 && alarm_type < LogUtils::LOG_ALARM_N_TYPES); if (msg == nullptr) { snprintf(msg_buf, sizeof(msg_buf), "No Message"); } else { + va_list ap; va_start(ap, msg); vsnprintf(msg_buf, LOG_MAX_FORMATTED_LINE, msg, ap); va_end(ap); @@ -434,42 +451,6 @@ LogUtils::remove_content_type_attributes(char *type_str, int *type_len) } } -/*------------------------------------------------------------------------- - LogUtils::timestamp_to_hex_str - - This routine simply writes the given timestamp integer [time_t] in the equivalent - hexadecimal string format "xxxxxxxxxx" into the provided buffer [buf] of - size [bufLen]. - - It returns 1 if the provided buffer is not big enough to hold the - equivalent ip string (and its null terminator), and 0 otherwise. - If the buffer is not big enough, only the ip "segments" that completely - fit into it are written, and the buffer is null terminated. - -------------------------------------------------------------------------*/ - -int -LogUtils::timestamp_to_hex_str(unsigned ip, char *buf, size_t bufLen, size_t *numCharsPtr) -{ - static const char *table = "0123456789abcdef@"; - int retVal = 1; - int shift = 28; - if (buf && bufLen > 0) { - if (bufLen > 8) { - bufLen = 8; - } - for (retVal = 0; retVal < (int)bufLen;) { - buf[retVal++] = (char)table[((ip >> shift) & 0xf)]; - shift -= 4; - } - - if (numCharsPtr) { - *numCharsPtr = (size_t)retVal; - } - retVal = (retVal == 8) ? 0 : 1; - } - return retVal; -} - /* int LogUtils::ip_to_str (unsigned ip, char *str, unsigned len) @@ -607,3 +588,158 @@ LogUtils::file_is_writeable(const char *full_filename, off_t *size_bytes, bool * return ret_val; } + +namespace +{ +// Get a string out of a MIMEField using one of its member funcitions, and put it into a buffer writer, terminated with a nul. +// +void +marshalStr(ts::FixedBufferWriter &bw, const MIMEField &mf, const char *(MIMEField::*get_func)(int *length) const) +{ + int length; + const char *data = (mf.*get_func)(&length); + + if (!data or (*data == '\0')) { + // Empty string. This is a problem, since it would result in two successive nul characters, which indicates the end of the + // marshaled hearer. Change the string to a single blank character. + + static const char Blank[] = " "; + data = Blank; + length = 1; + } + + bw << std::string_view(data, length) << '\0'; +} + +void +unmarshalStr(ts::FixedBufferWriter &bw, const char *&data) +{ + bw << '{'; + + while (*data) { + bw << *(data++); + } + + // Skip over terminal nul. + ++data; + + bw << '}'; +} + +} // end anonymous namespace + +namespace LogUtils +{ +// Marshals header tags and values together, with a single terminating nul character. Returns buffer space required. 'buf' points +// to where to put the marshaled data. If 'buf' is null, no data is marshaled, but the function returns the amount of space that +// would have been used. +// +int +marshalMimeHdr(MIMEHdr *hdr, char *buf) +{ + std::size_t bwSize = buf ? SIZE_MAX : 0; + + ts::FixedBufferWriter bw(buf, bwSize); + + if (hdr) { + MIMEFieldIter mfIter; + const MIMEField *mfp = hdr->iter_get_first(&mfIter); + + while (mfp) { + marshalStr(bw, *mfp, &MIMEField::name_get); + marshalStr(bw, *mfp, &MIMEField::value_get); + + mfp = hdr->iter_get_next(&mfIter); + } + } + + bw << '\0'; + + return int(INK_ALIGN_DEFAULT(bw.extent())); +} + +// Unmarshaled/printable format is {{{tag1}:{value1}}{{tag2}:{value2}} ... } +// +int +unmarshalMimeHdr(char **buf, char *dest, int destLength) +{ + ink_assert(*buf != nullptr); + + const char *data = *buf; + + ink_assert(data != nullptr); + + ts::FixedBufferWriter bw(dest, destLength); + + bw << '{'; + + int pairEndFallback{0}, pairEndFallback2{0}, pairSeparatorFallback{0}; + + while (*data) { + if (!bw.error()) { + pairEndFallback2 = pairEndFallback; + pairEndFallback = bw.size(); + } + + // Add open bracket of pair. + // + bw << '{'; + + // Unmarshal field name. + unmarshalStr(bw, data); + + bw << ':'; + + if (!bw.error()) { + pairSeparatorFallback = bw.size(); + } + + // Unmarshal field value. + unmarshalStr(bw, data); + + // Add close bracket of pair. + bw << '}'; + + } // end for loop + + bw << '}'; + + if (bw.error()) { + // The output buffer wasn't big enough. + + static std::string_view FULL_ELLIPSES("...}}}"); + + if ((pairSeparatorFallback > pairEndFallback) and ((pairSeparatorFallback + 7) <= destLength)) { + // In the report, we can show the existence of the last partial tag/value pair, and maybe part of the value. If we only + // show part of the value, we want to end it with an elipsis, to make it clear it's not complete. + + bw.reduce(destLength - FULL_ELLIPSES.size()); + bw << FULL_ELLIPSES; + + } else if (pairEndFallback and (pairEndFallback < destLength)) { + bw.reduce(pairEndFallback); + bw << '}'; + + } else if ((pairSeparatorFallback > pairEndFallback2) and ((pairSeparatorFallback + 7) <= destLength)) { + bw.reduce(destLength - FULL_ELLIPSES.size()); + bw << FULL_ELLIPSES; + + } else if (pairEndFallback2 and (pairEndFallback2 < destLength)) { + bw.reduce(pairEndFallback2); + bw << '}'; + + } else if (destLength > 1) { + bw.reduce(1); + bw << '}'; + + } else { + bw.reduce(0); + } + } + + *buf += INK_ALIGN_DEFAULT(data - *buf + 1); + + return bw.size(); +} + +} // end namespace LogUtils diff --git a/proxy/logging/LogUtils.h b/proxy/logging/LogUtils.h index 96cd05148b0..2860db1b63f 100644 --- a/proxy/logging/LogUtils.h +++ b/proxy/logging/LogUtils.h @@ -27,6 +27,8 @@ #include "tscore/ink_platform.h" #include "tscore/Arena.h" +class MIMEHdr; + namespace LogUtils { enum AlarmType { @@ -59,4 +61,16 @@ int timestamp_to_hex_str(unsigned timestamp, char *str, size_t len, size_t *n_ch int seconds_to_next_roll(time_t time_now, int rolling_offset, int rolling_interval); int file_is_writeable(const char *full_filename, off_t *size_bytes = nullptr, bool *has_size_limit = nullptr, uint64_t *current_size_limit_bytes = nullptr); + +// Marshals header tags and values together, with a single terminating nul character. Returns buffer space required. 'buf' points +// to where to put the marshaled data. If 'buf' is null, no data is marshaled, but the function returns the amount of space that +// would have been used. +int marshalMimeHdr(MIMEHdr *hdr, char *buf); + +// Unmarshelled/printable format is {{{tag1}:{value1}}{{tag2}:{value2}} ... } +// +// Returns -1 if data corruption is detected, otherwise the actual amount of data put into the 'dest' buffer. '*buf' is advanced +// to byte after the last byte consumed. +int unmarshalMimeHdr(char **buf, char *dest, int destLength); + }; // namespace LogUtils diff --git a/proxy/logging/Makefile.am b/proxy/logging/Makefile.am index cac0eb4891b..abf8e866cfc 100644 --- a/proxy/logging/Makefile.am +++ b/proxy/logging/Makefile.am @@ -34,7 +34,7 @@ AM_CPPFLAGS += \ EXTRA_DIST = LogStandalone.cc -noinst_LIBRARIES = liblogging.a liblogcollation.a +noinst_LIBRARIES = liblogging.a liblogging_a_SOURCES = \ Log.cc \ @@ -56,41 +56,47 @@ liblogging_a_SOURCES = \ LogFilter.h \ LogFormat.cc \ LogFormat.h \ - LogHost.cc \ - LogHost.h \ LogLimits.h \ LogObject.cc \ LogObject.h \ - LogSock.cc \ - LogSock.h \ LogUtils.cc \ LogUtils.h \ YamlLogConfig.cc \ YamlLogConfigDecoders.cc \ YamlLogConfig.h -liblogcollation_a_SOURCES = \ - LogCollationAccept.cc \ - LogCollationAccept.h \ - LogCollationBase.h \ - LogCollationClientSM.cc \ - LogCollationClientSM.h \ - LogCollationHostSM.cc \ - LogCollationHostSM.h - check_PROGRAMS = \ - test_LogUtils + test_LogUtils \ + test_LogUtils2 TESTS = \ - test_LogUtils + test_LogUtils \ + test_LogUtils2 + +test_LogUtils_CPPFLAGS = $(AM_CPPFLAGS)\ + -DTEST_LOG_UTILS + +test_LogUtils_SOURCES = \ + test_LogUtils.cc test_LogUtils_LDADD = \ $(top_builddir)/src/tscore/libtscore.la \ $(top_builddir)/src/tscpp/util/libtscpputil.la \ $(top_builddir)/iocore/eventsystem/libinkevent.a -test_LogUtils_SOURCES = \ - test_LogUtils.cc +test_LogUtils2_CPPFLAGS = $(AM_CPPFLAGS)\ + -DTEST_LOG_UTILS \ + -I$(abs_top_srcdir)/tests/include + +test_LogUtils2_SOURCES = \ + LogUtils.cc \ + unit-tests/BufferWriterFormat.cc \ + unit-tests/test_LogUtils2.cc + +test_LogUtils2_LDADD = \ + $(top_builddir)/src/tscore/libtscore.la \ + $(top_builddir)/src/tscpp/util/libtscpputil.la \ + $(top_builddir)/iocore/eventsystem/libinkevent.a -clang-tidy-local: $(liblogging_a_SOURCES) $(liblogcollation_a_SOURCES) $(EXTRA_DIST) +clang-tidy-local: $(liblogging_a_SOURCES) $(EXTRA_DIST) $(CXX_Clang_Tidy) diff --git a/proxy/logging/YamlLogConfig.cc b/proxy/logging/YamlLogConfig.cc index b3fd5bcdc8f..db3580f7868 100644 --- a/proxy/logging/YamlLogConfig.cc +++ b/proxy/logging/YamlLogConfig.cc @@ -60,8 +60,8 @@ YamlLogConfig::loadLogConfig(const char *cfgFilename) } auto formats = config["formats"]; - for (auto it = formats.begin(); it != formats.end(); ++it) { - auto fmt = it->as>().release(); + for (auto const &node : formats) { + auto fmt = node.as>().release(); if (fmt->valid()) { cfg->format_list.add(fmt, false); @@ -76,8 +76,8 @@ YamlLogConfig::loadLogConfig(const char *cfgFilename) } auto filters = config["filters"]; - for (auto it = filters.begin(); it != filters.end(); ++it) { - auto filter = it->as>().release(); + for (auto const &node : filters) { + auto filter = node.as>().release(); if (filter) { cfg->filter_list.add(filter, false); @@ -90,8 +90,8 @@ YamlLogConfig::loadLogConfig(const char *cfgFilename) } auto logs = config["logs"]; - for (auto it = logs.begin(); it != logs.end(); ++it) { - auto obj = decodeLogObject(*it); + for (auto const &node : logs) { + auto obj = decodeLogObject(node); if (obj) { cfg->log_object_manager.manage_object(obj); } @@ -99,29 +99,31 @@ YamlLogConfig::loadLogConfig(const char *cfgFilename) return true; } -TsEnumDescriptor ROLLING_MODE = {{{"none", 0}, {"time", 1}, {"size", 2}, {"both", 3}, {"any", 4}}}; +TsEnumDescriptor ROLLING_MODE_TEXT = {{{"none", 0}, {"time", 1}, {"size", 2}, {"both", 3}, {"any", 4}}}; +TsEnumDescriptor ROLLING_MODE_LUA = { + {{"log.roll.none", 0}, {"log.roll.time", 1}, {"log.roll.size", 2}, {"log.roll.both", 3}, {"log.roll.any", 4}}}; std::set valid_log_object_keys = { - "filename", "format", "mode", "header", "rolling_enabled", "rolling_interval_sec", - "rolling_offset_hr", "rolling_size_mb", "filters", "collation_hosts", "min_count"}; + "filename", "format", "mode", "header", "rolling_enabled", "rolling_interval_sec", + "rolling_offset_hr", "rolling_size_mb", "filters", "min_count"}; LogObject * YamlLogConfig::decodeLogObject(const YAML::Node &node) { - for (auto &&item : node) { + for (auto const &item : node) { if (std::none_of(valid_log_object_keys.begin(), valid_log_object_keys.end(), [&item](std::string s) { return s == item.first.as(); })) { - throw std::runtime_error("log: unsupported key '" + item.first.as() + "'"); + throw YAML::ParserException(item.Mark(), "log: unsupported key '" + item.first.as() + "'"); } } if (!node["format"]) { - throw std::runtime_error("missing 'format' argument"); + throw YAML::ParserException(node.Mark(), "missing 'format' argument"); } std::string format = node["format"].as(); if (!node["filename"]) { - throw std::runtime_error("missing 'filename' argument"); + throw YAML::ParserException(node.Mark(), "missing 'filename' argument"); } std::string header; @@ -153,9 +155,15 @@ YamlLogConfig::decodeLogObject(const YAML::Node &node) if (node["rolling_enabled"]) { auto value = node["rolling_enabled"].as(); - obj_rolling_enabled = ROLLING_MODE.get(value); + obj_rolling_enabled = ROLLING_MODE_TEXT.get(value); if (obj_rolling_enabled < 0) { - throw std::runtime_error("unknown value " + value); + obj_rolling_enabled = ROLLING_MODE_LUA.get(value); + if (obj_rolling_enabled < 0) { + obj_rolling_enabled = node["rolling_enabled"].as(); + if (obj_rolling_enabled < Log::NO_ROLLING || obj_rolling_enabled > Log::ROLL_ON_TIME_AND_SIZE) { + throw YAML::ParserException(node["rolling_enabled"].Mark(), "unknown value " + value); + } + } } } if (node["rolling_interval_sec"]) { @@ -175,7 +183,7 @@ YamlLogConfig::decodeLogObject(const YAML::Node &node) } auto logObject = new LogObject(fmt, Log::config->logfile_dir, filename.c_str(), file_type, header.c_str(), - (Log::RollingEnabledValues)obj_rolling_enabled, Log::config->collation_preproc_threads, + (Log::RollingEnabledValues)obj_rolling_enabled, Log::config->preproc_threads, obj_rolling_interval_sec, obj_rolling_offset_hr, obj_rolling_size_mb); // Generate LogDeletingInfo entry for later use @@ -202,66 +210,18 @@ YamlLogConfig::decodeLogObject(const YAML::Node &node) } if (!filters.IsSequence()) { - throw std::runtime_error("'filters' should be a list"); + throw YAML::ParserException(filters.Mark(), "'filters' should be a list"); } - for (auto &&filter : filters) { - const char *filter_name = filter.as().c_str(); - LogFilter *f = cfg->filter_list.find_by_name(filter_name); + for (auto const &filter : filters) { + std::string filter_name = filter.as().c_str(); + LogFilter *f = cfg->filter_list.find_by_name(filter_name.c_str()); if (!f) { - Warning("Filter %s is not a known filter; cannot add to this LogObject", filter_name); + Warning("Filter %s is not a known filter; cannot add to this LogObject", filter_name.c_str()); } else { logObject->add_filter(f); } } - auto collation_host_list = node["collation_hosts"]; - if (!collation_host_list) { - return logObject; - } - - if (!collation_host_list.IsSequence()) { - throw std::runtime_error("'collation_hosts' should be a list of collation_host objects"); - } - - for (auto &&collation_host : collation_host_list) { - if (!collation_host["host"]) { - Warning("no collation 'host' name; cannot add this Collation host"); - continue; - } - - auto collation_host_name = collation_host["host"].as(); - - LogHost *lh = new LogHost(logObject->get_full_filename(), logObject->get_signature()); - if (!lh->set_name_or_ipstr(collation_host_name.c_str())) { - Warning("Could not set \"%s\" as collation host", collation_host_name.c_str()); - delete lh; - continue; - } - - logObject->add_loghost(lh, false); - if (!collation_host["failover"]) { - continue; - } - - if (!collation_host["failover"].IsSequence()) { - delete lh; - throw std::runtime_error("'failover' should be a list of host names"); - } - - LogHost *prev = lh; - for (auto &&failover_host : collation_host["failover"]) { - auto failover_host_name = failover_host.as(); - LogHost *flh = new LogHost(logObject->get_full_filename(), logObject->get_signature()); - if (!flh->set_name_or_ipstr(failover_host_name.c_str())) { - Warning("Could not set \"%s\" as a failover host", failover_host_name.c_str()); - delete flh; - continue; - } - prev->failover_link.next = flh; - prev = flh; - } - } - return logObject; } diff --git a/proxy/logging/YamlLogConfigDecoders.cc b/proxy/logging/YamlLogConfigDecoders.cc index d1853c1145d..6146aca09c6 100644 --- a/proxy/logging/YamlLogConfigDecoders.cc +++ b/proxy/logging/YamlLogConfigDecoders.cc @@ -39,12 +39,12 @@ convert>::decode(const Node &node, std::unique_ptr(); })) { - throw std::runtime_error("format: unsupported key '" + item.first.as() + "'"); + throw YAML::ParserException(node.Mark(), "format: unsupported key '" + item.first.as() + "'"); } } if (!node["format"]) { - throw std::runtime_error("missing 'format' argument"); + throw YAML::ParserException(node.Mark(), "missing 'format' argument"); } std::string format = node["format"].as(); @@ -80,14 +80,14 @@ convert>::decode(const Node &node, std::unique_ptr(); })) { - throw std::runtime_error("filter: unsupported key '" + item.first.as() + "'"); + throw YAML::ParserException(node.Mark(), "filter: unsupported key '" + item.first.as() + "'"); } } // we require all keys for LogFilter for (auto &&item : valid_log_filter_keys) { if (!node[item]) { - throw std::runtime_error("missing '" + item + "' argument"); + throw YAML::ParserException(node.Mark(), "missing '" + item + "' argument"); } } diff --git a/proxy/logging/test_LogUtils.cc b/proxy/logging/test_LogUtils.cc index 0f39524ceaa..02b8cfee6b7 100644 --- a/proxy/logging/test_LogUtils.cc +++ b/proxy/logging/test_LogUtils.cc @@ -25,6 +25,15 @@ #include "LogUtils.cc" #include +#if 0 +// Stub +EThread * +this_ethread() +{ + return nullptr; +} +#endif + REGRESSION_TEST(LogUtils_pure_escapify_url)(RegressionTest *t, int /* atype ATS_UNUSED */, int *pstatus) { TestBox box(t, pstatus); diff --git a/iocore/eventsystem/Continuation.cc b/proxy/logging/unit-tests/BufferWriterFormat.cc similarity index 65% rename from iocore/eventsystem/Continuation.cc rename to proxy/logging/unit-tests/BufferWriterFormat.cc index c6936322dca..d2c3e39015b 100644 --- a/iocore/eventsystem/Continuation.cc +++ b/proxy/logging/unit-tests/BufferWriterFormat.cc @@ -1,6 +1,6 @@ /** @file - Contination.cc + includes BufferWriterFormat.cc from TS utilities. @section license License @@ -21,22 +21,4 @@ limitations under the License. */ -#include "I_EventSystem.h" -#include "I_Continuation.h" - -int -Continuation::dispatchEvent(int event, void *data) -{ - if (mutex) { - EThread *t = this_ethread(); - MUTEX_TRY_LOCK(lock, this->mutex, t); - if (!lock.is_locked()) { - t->schedule_imm(this, event, data); - return 0; - } else { - return (this->*handler)(event, data); - } - } else { - return (this->*handler)(event, data); - } -} +#include "../../../src/tscore/BufferWriterFormat.cc" diff --git a/proxy/logging/unit-tests/test_LogUtils.h b/proxy/logging/unit-tests/test_LogUtils.h new file mode 100644 index 00000000000..3c9ed225932 --- /dev/null +++ b/proxy/logging/unit-tests/test_LogUtils.h @@ -0,0 +1,73 @@ +/** @file + + Header file for shared declarations/definitions for test_LogUtils2.cc and LogUtils.h for unit testing. + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#pragma once + +struct MIMEField { + const char *tag, *value; + + const char * + name_get(int *length) const + { + *length = strlen(tag); + return tag; + } + const char * + value_get(int *length) const + { + *length = strlen(value); + return value; + } +}; + +struct MIMEFieldIter { +}; + +class MIMEHdr +{ +public: + MIMEHdr(const MIMEField *first, int count) : _first(first), _count(count), _idx(0) {} + + const MIMEField * + iter_get_first(MIMEFieldIter *) + { + return _idx < _count ? _first + _idx : nullptr; + } + const MIMEField * + iter_get_next(MIMEFieldIter *) + { + ++_idx; + return iter_get_first(nullptr); + } + + void + reset() + { + _idx = 0; + } + +private: + const MIMEField *const _first; + const int _count; + int _idx; +}; diff --git a/proxy/logging/unit-tests/test_LogUtils2.cc b/proxy/logging/unit-tests/test_LogUtils2.cc new file mode 100644 index 00000000000..31376b3f4d0 --- /dev/null +++ b/proxy/logging/unit-tests/test_LogUtils2.cc @@ -0,0 +1,132 @@ +/** @file + + Catch-based tests for LogUtils.h. + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#include +#include + +#include + +#include "test_LogUtils.h" + +#define CATCH_CONFIG_MAIN +#include "catch.hpp" + +#include + +using namespace LogUtils; + +namespace +{ +void +test(const MIMEField *pairs, int numPairs, const char *asciiResult, int extraUnmarshalSpace = 0) +{ + char binBuf[1500], asciiBuf[1500]; + + MIMEHdr hdr{pairs, numPairs}; + + int binAlignSize = marshalMimeHdr(numPairs ? &hdr : nullptr, nullptr); + + REQUIRE(binAlignSize < sizeof(binBuf)); + + hdr.reset(); + + REQUIRE(marshalMimeHdr(numPairs ? &hdr : nullptr, binBuf) == binAlignSize); + + int binSize{1}; + + if (binBuf[0]) { + for (; binBuf[binSize] or binBuf[binSize + 1]; ++binSize) { + } + + binSize += 2; + + } else { + binSize = 1; + } + + REQUIRE(INK_ALIGN_DEFAULT(binSize) == binAlignSize); + + char *bp = binBuf; + + int asciiSize = unmarshalMimeHdr(&bp, asciiBuf, std::strlen(asciiResult) + extraUnmarshalSpace); + + REQUIRE(asciiSize == std::strlen(asciiResult)); + + REQUIRE((bp - binBuf) == binAlignSize); + + REQUIRE(std::memcmp(asciiBuf, asciiResult, asciiSize) == 0); +} + +} // namespace + +TEST_CASE("LogUtilsHttp", "[LUHP]") +{ +#define X "12345678" +#define X2 X X +#define X3 X2 X2 +#define X4 X3 X3 +#define X5 X4 X4 +#define X6 X5 X5 +#define X7 X6 X6 +#define X8 X7 X7 + + const MIMEField pairs[] = {{"Argh", "Ugh"}, {"Argh2", "UghUgh"}, {"alltogethernow", X8}}; + + test(pairs, 1, "{{{Argh}:{Ugh}}}"); + test(pairs, 2, "{{{Argh}:{Ugh}}{{Argh2}:{UghUgh}}}"); + test(pairs, 2, "{{{Argh}:{Ugh}}{{Argh2}:{Ug...}}}"); + test(pairs, 2, "{{{Argh}:{Ugh}}{{Argh2}:{U...}}}"); + test(pairs, 2, "{{{Argh}:{Ugh}}{{Argh2}:{...}}}"); + test(pairs, 2, "{{{Argh}:{Ugh}}}"); + test(pairs, 2, "{{{Argh}:{Ugh}}}", 1); + test(pairs, 2, "{{{Argh}:{Ugh}}}", sizeof("{{Argh2}:{...}}") - 2); + test(pairs, 3, "{{{Argh}:{Ugh}}{{Argh2}:{UghUgh}}{{alltogethernow}:{" X8 "}}}"); + + test(pairs, 3, "{{{Argh}:{Ugh}}{{Argh2}:{UghUgh}}}"); + test(pairs, 3, "{{{Argh}:{Ugh}}{{Argh2}:{Ug...}}}"); + test(pairs, 3, "{{{Argh}:{Ugh}}{{Argh2}:{U...}}}"); + test(pairs, 3, "{{{Argh}:{Ugh}}{{Argh2}:{...}}}"); + test(pairs, 3, "{{{Argh}:{Ugh}}}"); + test(pairs, 3, "{{{Argh}:{Ugh}}}", 1); + test(pairs, 3, "{{{Argh}:{Ugh}}}", sizeof("{{Argh2}:{...}}") - 2); + + test(nullptr, 0, "{}"); + test(nullptr, 0, ""); + test(nullptr, 0, "", 1); +} + +#include +#include + +void +_ink_assert(const char *a, const char *f, int line) +{ + std::cout << a << '\n' << f << '\n' << line << '\n'; + + std::exit(1); +} + +void +RecSignalManager(int, char const *, std::size_t) +{ +} diff --git a/proxy/shared/DiagsConfig.cc b/proxy/shared/DiagsConfig.cc index ebdedeb70f6..98848ea4a24 100644 --- a/proxy/shared/DiagsConfig.cc +++ b/proxy/shared/DiagsConfig.cc @@ -265,14 +265,11 @@ DiagsConfig::RegisterDiagConfig() } DiagsConfig::DiagsConfig(const char *prefix_string, const char *filename, const char *tags, const char *actions, bool use_records) - : diags_log(nullptr) + : callbacks_established(false), diags_log(nullptr), diags(nullptr) { char diags_logpath[PATH_NAME_MAX]; ats_scoped_str logpath; - callbacks_established = false; - diags = nullptr; - //////////////////////////////////////////////////////////////////// // If we aren't using the manager records for configuation // // just build the tables based on command line parameters and // diff --git a/proxy/shared/UglyLogStubs.cc b/proxy/shared/UglyLogStubs.cc index ce4d97be2b4..69523896125 100644 --- a/proxy/shared/UglyLogStubs.cc +++ b/proxy/shared/UglyLogStubs.cc @@ -75,22 +75,6 @@ Machine::instance() return nullptr; } -#include "LogCollationAccept.h" -LogCollationAccept::LogCollationAccept(int port) : Continuation(new_ProxyMutex()), m_port(port) {} -LogCollationAccept::~LogCollationAccept() {} - -#include "LogCollationClientSM.h" -LogCollationClientSM::LogCollationClientSM(LogHost *log_host) : Continuation(new_ProxyMutex()), m_log_host(log_host) {} - -LogCollationClientSM::~LogCollationClientSM() {} - -int -LogCollationClientSM::send(LogBuffer * /* log_buffer ATS_UNUSED */) -{ - ink_release_assert(false); - return 0; -} - NetAccept * UnixNetProcessor::createNetAccept(const NetProcessor::AcceptOptions &opt) { @@ -104,6 +88,12 @@ UnixNetProcessor::init() ink_release_assert(false); } +void +UnixNetProcessor::init_socks() +{ + ink_release_assert(false); +} + // TODO: The following was necessary only for Solaris, should examine more. NetVCOptions const Connection::DEFAULT_OPTIONS; NetProcessor::AcceptOptions const NetProcessor::DEFAULT_ACCEPT_OPTIONS; diff --git a/src/Makefile.am b/src/Makefile.am index 8e42189a73d..0d7f07973ae 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -43,4 +43,3 @@ include traffic_logcat/Makefile.inc clang-tidy-local: $(DIST_SOURCES) $(CXX_Clang_Tidy) - $(CC_Clang_Tidy) diff --git a/src/traffic_cache_tool/CacheDefs.cc b/src/traffic_cache_tool/CacheDefs.cc index 820c5d10b89..e84fd23db8b 100644 --- a/src/traffic_cache_tool/CacheDefs.cc +++ b/src/traffic_cache_tool/CacheDefs.cc @@ -237,9 +237,9 @@ Stripe::InitializeMeta() // memset(this->raw_dir, 0, dir_len); for (auto &i : _meta) { for (auto &j : i) { - j.magic = StripeMeta::MAGIC; - j.version.ink_major = ts::CACHE_DB_MAJOR_VERSION; - j.version.ink_minor = ts::CACHE_DB_MINOR_VERSION; + j.magic = StripeMeta::MAGIC; + j.version._major = ts::CACHE_DB_MAJOR_VERSION; + j.version._minor = ts::CACHE_DB_MINOR_VERSION; j.agg_pos = j.last_write_pos = j.write_pos = this->_content; j.phase = j.cycle = j.sync_serial = j.write_serial = j.dirty = 0; j.create_time = time(nullptr); @@ -264,8 +264,8 @@ bool Stripe::validateMeta(StripeMeta const *meta) { // Need to be bit more robust at some point. - return StripeMeta::MAGIC == meta->magic && meta->version.ink_major <= ts::CACHE_DB_MAJOR_VERSION && - meta->version.ink_minor <= 2 // This may have always been zero, actually. + return StripeMeta::MAGIC == meta->magic && meta->version._major <= ts::CACHE_DB_MAJOR_VERSION && + meta->version._minor <= 2 // This may have always been zero, actually. ; } @@ -874,7 +874,7 @@ Errata Stripe::loadMeta() { // Read from disk in chunks of this size. This needs to be a multiple of both the - // store block size and the directory entry size so neither goes acrss read boundaries. + // store block size and the directory entry size so neither goes across read boundaries. // Beyond that the value should be in the ~10MB range for what I guess is best performance // vs. blocking production disk I/O on a live system. constexpr static int64_t N = (1 << 8) * CacheStoreBlocks::SCALE * sizeof(CacheDirEntry); diff --git a/src/traffic_cache_tool/CacheDefs.h b/src/traffic_cache_tool/CacheDefs.h index 4a980827a49..8b42a92ba40 100644 --- a/src/traffic_cache_tool/CacheDefs.h +++ b/src/traffic_cache_tool/CacheDefs.h @@ -300,11 +300,11 @@ struct url_matcher { for (i = 0; i < count; i++) { std::cout << "regex " << patterns[i] << std::endl; } - if (regex.compile(patterns, count) != 0) { + if (regex.compile(patterns, count) != count) { std::cout << "Check your regular expression" << std::endl; } - if (port.compile(R"([0-9]+$)") != 0) { + if (!port.compile(R"([0-9]+$)")) { std::cout << "Check your regular expression" << std::endl; return; } @@ -313,11 +313,11 @@ struct url_matcher { url_matcher() { - if (regex.compile(R"(^(https?\:\/\/)") != 0) { + if (!regex.compile(R"(^(https?\:\/\/)")) { std::cout << "Check your regular expression" << std::endl; return; } - if (port.compile(R"([0-9]+$)") != 0) { + if (!port.compile(R"([0-9]+$)")) { std::cout << "Check your regular expression" << std::endl; return; } @@ -328,19 +328,12 @@ struct url_matcher { uint8_t match(const char *hostname) const { - if (regex.match(hostname) != -1) { - return 1; - } - - return 0; + return regex.match(hostname) ? 1 : 0; } uint8_t portmatch(const char *hostname, int length) const { - if (port.match(hostname, length) != -1) { - return 1; - } - return 0; + return port.match({hostname, size_t(length)}) ? 1 : 0; } private: @@ -405,7 +398,7 @@ dir_from_offset(int64_t i, CacheDirEntry *seg) { #if DIR_DEPTH < 5 if (!i) - return 0; + return nullptr; return dir_in_seg(seg, i); #else i = i + ((i - 1) / (DIR_DEPTH - 1)); @@ -474,7 +467,7 @@ struct Span { /// Local copy of serialized header data stored on in the span. std::unique_ptr _header; /// Live information about stripes. - /// Seeded from @a _header and potentially agumented with direct probing. + /// Seeded from @a _header and potentially augmented with direct probing. std::list _stripes; }; /* --------------------------------------------------------------------------------------- */ diff --git a/src/traffic_cache_tool/CacheScan.cc b/src/traffic_cache_tool/CacheScan.cc index 06be5355217..d46572b6cb9 100644 --- a/src/traffic_cache_tool/CacheScan.cc +++ b/src/traffic_cache_tool/CacheScan.cc @@ -29,7 +29,7 @@ // using namespace ct; -const int HTTP_ALT_MARSHAL_SIZE = ROUND(sizeof(HTTPCacheAlt), HDR_PTR_SIZE); +constexpr HdrHeapMarshalBlocks HTTP_ALT_MARSHAL_SIZE = ts::round_up(sizeof(HTTPCacheAlt)); namespace ct { @@ -208,7 +208,7 @@ CacheScan::unmarshal(HdrHeap *hh, int buf_length, int obj_type, HdrHeapObjImpl * while (obj_data < hh->m_free_start) { HdrHeapObjImpl *obj = (HdrHeapObjImpl *)obj_data; if (!obj_is_aligned(obj)) { - std::cout << "Invalid alignmgnt of object of type HdrHeapObjImpl" << std::endl; + std::cout << "Invalid alignment of object of type HdrHeapObjImpl" << std::endl; return zret; } @@ -248,8 +248,7 @@ CacheScan::unmarshal(HdrHeap *hh, int buf_length, int obj_type, HdrHeapObjImpl * hh->m_magic = HDR_BUF_MAGIC_ALIVE; - int unmarshal_length = ROUND(hh->unmarshal_size(), HDR_PTR_SIZE); - return unmarshal_length; + return HdrHeapMarshalBlocks(ts::round_up(hh->unmarshal_size())); } Errata @@ -260,7 +259,7 @@ CacheScan::unmarshal(char *buf, int len, RefCountObj *block_ref) int orig_len = len; if (alt->m_magic == CACHE_ALT_MAGIC_ALIVE) { - // Already unmarshaled, must be a ram cache + // Already unmarshalled, must be a ram cache // it ink_assert(alt->m_unmarshal_len > 0); ink_assert(alt->m_unmarshal_len <= len); diff --git a/src/traffic_cache_tool/CacheTool.cc b/src/traffic_cache_tool/CacheTool.cc index f16a1e86a6e..4c60dd9730f 100644 --- a/src/traffic_cache_tool/CacheTool.cc +++ b/src/traffic_cache_tool/CacheTool.cc @@ -618,9 +618,8 @@ Cache::dumpSpans(SpanDumpDepth depth) for (int i = 0; i < 2; i++) { for (int j = 0; j < 2; j++) { std::cout << "\n" << MetaCopy[i] << ":" << MetaType[j] << "\n" << std::endl; - std::cout << " Magic:" << stripe->_meta[i][j].magic - << "\n version: ink_major: " << stripe->_meta[i][j].version.ink_major - << "\n version: ink_minor: " << stripe->_meta[i][j].version.ink_minor + std::cout << " Magic:" << stripe->_meta[i][j].magic << "\n version: major: " << stripe->_meta[i][j].version._major + << "\n version: minor: " << stripe->_meta[i][j].version._minor << "\n create_time: " << stripe->_meta[i][j].create_time << "\n write_pos: " << stripe->_meta[i][j].write_pos << "\n last_write_pos: " << stripe->_meta[i][j].last_write_pos @@ -871,7 +870,7 @@ Span::updateHeader() zret.push(0, errno, "Failed to update span - ", strerror(errno)); } } else { - std::cout << "Writing not enabled, no updates perfomed" << std::endl; + std::cout << "Writing not enabled, no updates performed" << std::endl; } return zret; } @@ -981,7 +980,7 @@ Cache::build_stripe_hash_table() ttable[i] = VOL_HASH_EMPTY; } - // generate random numbers proportaion to allocation + // generate random numbers proportional to allocation rtable_pair *rtable = (rtable_pair *)ats_malloc(sizeof(rtable_pair) * rtable_size); int rindex = 0; for (int i = 0; i < num_stripes; i++) { @@ -1220,7 +1219,7 @@ dir_check() } void -walk_bucket_chain(std::string devicePath) +walk_bucket_chain(const std::string &devicePath) { Cache cache; if ((err = cache.loadSpan(SpanFile))) { @@ -1238,7 +1237,7 @@ walk_bucket_chain(std::string devicePath) } void -Clear_Span(std::string devicePath) +Clear_Span(const std::string &devicePath) { Cache cache; if ((err = cache.loadSpan(SpanFile))) { @@ -1254,7 +1253,7 @@ Clear_Span(std::string devicePath) } void -Check_Freelist(std::string devicePath) +Check_Freelist(const std::string &devicePath) { Cache cache; if ((err = cache.loadSpan(SpanFile))) { diff --git a/src/traffic_cache_tool/Makefile.inc b/src/traffic_cache_tool/Makefile.inc index 82063805082..8af4112533c 100644 --- a/src/traffic_cache_tool/Makefile.inc +++ b/src/traffic_cache_tool/Makefile.inc @@ -48,4 +48,9 @@ traffic_cache_tool_traffic_cache_tool_LDADD = \ $(top_builddir)/src/tscore/.libs/Regex.o \ $(top_builddir)/src/tscore/.libs/CryptoHash.o \ $(top_builddir)/src/tscore/.libs/MMH.o \ - @OPENSSL_LIBS@ @LIBPCRE@ @LIBTCL@ + $(top_builddir)/src/tscore/.libs/Version.o \ + $(top_builddir)/src/tscore/.libs/Regression.o \ + $(top_builddir)/src/tscore/.libs/ink_args.o \ + $(top_builddir)/src/tscore/.libs/ParseRules.o \ + $(top_builddir)/src/tscore/.libs/SourceLocation.o \ + @OPENSSL_LIBS@ @LIBPCRE@ diff --git a/src/traffic_crashlog/Makefile.inc b/src/traffic_crashlog/Makefile.inc index b7a408b5499..f665ab63d06 100644 --- a/src/traffic_crashlog/Makefile.inc +++ b/src/traffic_crashlog/Makefile.inc @@ -44,4 +44,4 @@ traffic_crashlog_traffic_crashlog_LDADD = \ $(top_builddir)/mgmt/api/libtsmgmt.la \ $(top_builddir)/src/tscore/libtscore.la \ $(top_builddir)/src/tscpp/util/libtscpputil.la \ - @LIBTCL@ @HWLOC_LIBS@ + @HWLOC_LIBS@ diff --git a/src/traffic_crashlog/traffic_crashlog.cc b/src/traffic_crashlog/traffic_crashlog.cc index c2848f9491e..42aff7aebf9 100644 --- a/src/traffic_crashlog/traffic_crashlog.cc +++ b/src/traffic_crashlog/traffic_crashlog.cc @@ -198,7 +198,7 @@ main(int /* argc ATS_UNUSED */, const char **argv) mgmterr = TSInit(nullptr, (TSInitOptionT)(TS_MGMT_OPT_NO_EVENTS | TS_MGMT_OPT_NO_SOCK_TESTS)); if (mgmterr != TS_ERR_OKAY) { char *msg = TSGetErrorMessage(mgmterr); - Warning("failed to intialize management API: %s", msg); + Warning("failed to initialize management API: %s", msg); TSfree(msg); } diff --git a/src/traffic_ctl/Makefile.inc b/src/traffic_ctl/Makefile.inc index 1efd85e6ab9..446e56539f9 100644 --- a/src/traffic_ctl/Makefile.inc +++ b/src/traffic_ctl/Makefile.inc @@ -47,4 +47,4 @@ traffic_ctl_traffic_ctl_LDADD = \ $(top_builddir)/mgmt/api/libtsmgmt.la \ $(top_builddir)/src/tscore/libtscore.la \ $(top_builddir)/src/tscpp/util/libtscpputil.la \ - @LIBTCL@ @HWLOC_LIBS@ + @HWLOC_LIBS@ diff --git a/src/traffic_ctl/alarm.cc b/src/traffic_ctl/alarm.cc index d3229f386ac..d8df03a5ad7 100644 --- a/src/traffic_ctl/alarm.cc +++ b/src/traffic_ctl/alarm.cc @@ -41,46 +41,38 @@ struct AlarmListPolicy { using CtrlAlarmList = CtrlMgmtList; -static int -alarm_list(unsigned argc, const char **argv) +void +CtrlEngine::alarm_list() { TSMgmtError error; CtrlAlarmList alarms; - if (!CtrlProcessArguments(argc, argv, nullptr, 0) || n_file_arguments != 0) { - return CtrlCommandUsage("alarm list", nullptr, 0); - } - error = TSActiveEventGetMlt(alarms.list); if (error != TS_ERR_OKAY) { CtrlMgmtError(error, "failed to fetch active alarms"); - return CTRL_EX_ERROR; + status_code = CTRL_EX_ERROR; + return; } while (!alarms.empty()) { char *a = alarms.next(); - printf("%s\n", a); + std::cout << a << std::endl; TSfree(a); } - - return CTRL_EX_OK; } -static int -alarm_clear(unsigned argc, const char **argv) +void +CtrlEngine::alarm_clear() { TSMgmtError error; CtrlAlarmList alarms; - if (!CtrlProcessArguments(argc, argv, nullptr, 0) || n_file_arguments != 0) { - return CtrlCommandUsage("alarm clear", nullptr, 0); - } - // First get the active alarms ... error = TSActiveEventGetMlt(alarms.list); if (error != TS_ERR_OKAY) { CtrlMgmtError(error, "failed to fetch active alarms"); - return CTRL_EX_ERROR; + status_code = CTRL_EX_ERROR; + return; } // Now resolve them all ... @@ -91,49 +83,26 @@ alarm_clear(unsigned argc, const char **argv) if (error != TS_ERR_OKAY) { CtrlMgmtError(error, "failed to resolve %s", a); TSfree(a); - return CTRL_EX_ERROR; + status_code = CTRL_EX_ERROR; + return; } TSfree(a); } - - return CTRL_EX_OK; } -static int -alarm_resolve(unsigned argc, const char **argv) +void +CtrlEngine::alarm_resolve() { TSMgmtError error; CtrlAlarmList alarms; - if (!CtrlProcessArguments(argc, argv, nullptr, 0) || n_file_arguments == 0) { - return CtrlCommandUsage("alarm resolve ALARM [ALARM ...]", nullptr, 0); - } - - for (unsigned i = 0; i < n_file_arguments; ++i) { - error = TSEventResolve(file_arguments[i]); + for (const auto &it : arguments.get("resolve")) { + error = TSEventResolve(it.c_str()); if (error != TS_ERR_OKAY) { - CtrlMgmtError(error, "failed to resolve %s", file_arguments[i]); - return CTRL_EX_ERROR; + CtrlMgmtError(error, "failed to resolve %s", it.c_str()); + status_code = CTRL_EX_ERROR; + return; } } - - return CTRL_EX_OK; -} - -int -subcommand_alarm(unsigned argc, const char **argv) -{ - const subcommand commands[] = { - {alarm_clear, "clear", "Clear all current alarms"}, - {alarm_list, "list", "List all current alarms"}, - - // Note that we separate resolve one from resolve all for the same reasons that - // we have "metric zero" and "metric clear". - {alarm_resolve, "resolve", "Resolve the listed alarms"}, - /* XXX describe a specific alarm? */ - /* XXX raise an alarm? */ - }; - - return CtrlGenericSubcommand("alarm", commands, countof(commands), argc, argv); } diff --git a/src/traffic_ctl/config.cc b/src/traffic_ctl/config.cc index 35c73035f6d..61af1998bc5 100644 --- a/src/traffic_ctl/config.cc +++ b/src/traffic_ctl/config.cc @@ -63,7 +63,7 @@ rec_typeof(int rec_type) return "FLOAT"; case TS_REC_STRING: return "STRING"; - case TS_REC_UNDEFINED: /* fallthru */ + case TS_REC_UNDEFINED: /* fallthrough */ default: return "UNDEFINED"; } @@ -98,7 +98,7 @@ rec_accessof(int rec_access) return "no access"; case RECA_READ_ONLY: return "read only"; - case RECA_NULL: /* fallthru */ + case RECA_NULL: /* fallthrough */ default: return "default"; } @@ -115,7 +115,7 @@ rec_updateof(int rec_updatetype) return "static, restart traffic_server"; case RECU_RESTART_TM: return "static, restart traffic_manager"; - case RECU_NULL: /* fallthru */ + case RECU_NULL: /* fallthrough */ default: return "none"; } @@ -132,7 +132,7 @@ rec_checkof(int rec_checktype) return "integer with a specified range"; case RECC_IP: return "IP address"; - case RECC_NULL: /* fallthru */ + case RECC_NULL: /* fallthrough */ default: return "none"; } @@ -181,179 +181,143 @@ format_record(const CtrlMgmtRecord &record, bool recfmt) CtrlMgmtRecordValue value(record); if (recfmt) { - printf("%s %s %s %s\n", rec_labelof(record.rclass()), record.name(), rec_typeof(record.type()), value.c_str()); + std::cout << rec_labelof(record.rclass()) << ' ' << record.name() << ' ' << rec_typeof(record.type()) << ' ' << value.c_str() + << std::endl; } else { - printf("%s: %s\n", record.name(), value.c_str()); + std::cout << record.name() << ": " << value.c_str() << std::endl; } } -static int -config_get(unsigned argc, const char **argv) +void +CtrlEngine::config_get() { - int recfmt = 0; - const ArgumentDescription opts[] = { - {"records", '-', "Emit output in records.config format", "F", &recfmt, nullptr, nullptr}, - }; - - if (!CtrlProcessArguments(argc, argv, opts, countof(opts)) || n_file_arguments < 1) { - return CtrlCommandUsage("config get [OPTIONS] RECORD [RECORD ...]", opts, countof(opts)); - } - - for (unsigned i = 0; i < n_file_arguments; ++i) { + for (const auto &it : arguments.get("get")) { CtrlMgmtRecord record; TSMgmtError error; - error = record.fetch(file_arguments[i]); + error = record.fetch(it.c_str()); if (error != TS_ERR_OKAY) { - CtrlMgmtError(error, "failed to fetch %s", file_arguments[i]); - return CTRL_EX_ERROR; + CtrlMgmtError(error, "failed to fetch %s", it.c_str()); + status_code = CTRL_EX_ERROR; + return; } if (REC_TYPE_IS_CONFIG(record.rclass())) { - format_record(record, recfmt); + format_record(record, arguments.get("records")); } } - - return CTRL_EX_OK; } -static int -config_describe(unsigned argc, const char **argv) +void +CtrlEngine::config_describe() { - if (!CtrlProcessArguments(argc, argv, nullptr, 0) || n_file_arguments < 1) { - return CtrlCommandUsage("config describe RECORD [RECORD ...]"); - } - - for (unsigned i = 0; i < n_file_arguments; ++i) { + for (const auto &it : arguments.get("describe")) { TSConfigRecordDescription desc; TSMgmtError error; ink_zero(desc); - error = TSConfigRecordDescribe(file_arguments[i], 0 /* flags */, &desc); + error = TSConfigRecordDescribe(it.c_str(), 0 /* flags */, &desc); if (error != TS_ERR_OKAY) { - CtrlMgmtError(error, "failed to describe %s", file_arguments[i]); - return CTRL_EX_ERROR; + CtrlMgmtError(error, "failed to describe %s", it.c_str()); + status_code = CTRL_EX_ERROR; + return; } - printf("%-16s: %s\n", "Name", desc.rec_name); - printf("%-16s: %s\n", "Current Value", CtrlMgmtRecordValue(desc.rec_type, desc.rec_value).c_str()); - printf("%-16s: %s\n", "Default Value", CtrlMgmtRecordValue(desc.rec_type, desc.rec_default).c_str()); - printf("%-16s: %s\n", "Record Type", rec_classof(desc.rec_class)); - printf("%-16s: %s\n", "Data Type", rec_typeof(desc.rec_type)); - printf("%-16s: %s\n", "Access Control ", rec_accessof(desc.rec_access)); - printf("%-16s: %s\n", "Update Type", rec_updateof(desc.rec_updatetype)); - printf("%-16s: 0x%" PRIx64 "\n", "Update Status", desc.rec_update); - printf("%-16s: %s\n", "Source", rec_sourceof(desc.rec_source)); + std::cout << "Name: " << desc.rec_name << std::endl; + std::cout << "Current Value: " << CtrlMgmtRecordValue(desc.rec_type, desc.rec_value).c_str() << std::endl; + std::cout << "Default Value: " << CtrlMgmtRecordValue(desc.rec_type, desc.rec_default).c_str() << std::endl; + std::cout << "Record Type: " << rec_classof(desc.rec_class) << std::endl; + std::cout << "Data Type: " << rec_typeof(desc.rec_type) << std::endl; + std::cout << "Access Control: " << rec_accessof(desc.rec_access) << std::endl; + std::cout << "Update Type: " << rec_updateof(desc.rec_updatetype) << std::endl; + std::cout << "Update Status: " << desc.rec_update << std::endl; + std::cout << "Source: " << rec_sourceof(desc.rec_source) << std::endl; if (strlen(desc.rec_checkexpr)) { - printf("%-16s: %s, '%s'\n", "Syntax Check", rec_checkof(desc.rec_checktype), desc.rec_checkexpr); + std::cout << "Syntax Check: " << rec_checkof(desc.rec_checktype) << desc.rec_checkexpr << std::endl; } else { - printf("%-16s: %s\n", "Syntax Check", rec_checkof(desc.rec_checktype)); + std::cout << "Syntax Check: " << rec_checkof(desc.rec_checktype) << std::endl; } - - printf("%-16s: %" PRId64 "\n", "Version", desc.rec_version); - printf("%-16s: %" PRId64 "\n", "Order", desc.rec_order); - printf("%-16s: %" PRId64 "\n", "Raw Stat Block", desc.rec_rsb); + std::cout << "Version: " << desc.rec_version << std::endl; + std::cout << "Order: " << desc.rec_order << std::endl; + std::cout << "Raw Stat Block: " << desc.rec_rsb << std::endl; TSConfigRecordDescriptionFree(&desc); } - - return CTRL_EX_OK; } -static int -config_set(unsigned argc, const char **argv) +void +CtrlEngine::config_set() { TSMgmtError error; TSActionNeedT action; - - if (!CtrlProcessArguments(argc, argv, nullptr, 0) || n_file_arguments != 2) { - return CtrlCommandUsage("config set RECORD VALUE"); - } - - error = TSRecordSet(file_arguments[0], file_arguments[1], &action); + auto set_data = arguments.get("set"); + const char *rec_name = set_data[0].c_str(); + const char *rec_val = set_data[1].c_str(); + error = TSRecordSet(rec_name, rec_val, &action); if (error != TS_ERR_OKAY) { - CtrlMgmtError(error, "failed to set %s", file_arguments[0]); - return CTRL_EX_ERROR; + CtrlMgmtError(error, "failed to set %s", rec_name); + status_code = CTRL_EX_ERROR; + return; } switch (action) { case TS_ACTION_SHUTDOWN: - printf("set %s, full shutdown required\n", file_arguments[0]); + std::cout << "set " << rec_name << ", full shutdown required" << std::endl; break; case TS_ACTION_RESTART: - printf("set %s, restart required\n", file_arguments[0]); + std::cout << "set " << rec_name << ", restart required" << std::endl; break; case TS_ACTION_RECONFIGURE: - printf("set %s, please wait 10 seconds for traffic server to sync configuration, restart is not required\n", file_arguments[0]); + std::cout << "set " << rec_name << ", please wait 10 seconds for traffic server to sync configuration, restart is not required" + << std::endl; break; case TS_ACTION_DYNAMIC: default: - printf("set %s\n", file_arguments[0]); + printf("set %s\n", rec_name); break; } - - return CTRL_EX_OK; } -static int -config_match(unsigned argc, const char **argv) +void +CtrlEngine::config_match() { - int recfmt = 0; - const ArgumentDescription opts[] = { - {"records", '-', "Emit output in records.config format", "F", &recfmt, nullptr, nullptr}, - }; - - if (!CtrlProcessArguments(argc, argv, opts, countof(opts)) || n_file_arguments < 1) { - return CtrlCommandUsage("config match [OPTIONS] REGEX [REGEX ...]", opts, countof(opts)); - } - - for (unsigned i = 0; i < n_file_arguments; ++i) { + for (const auto &it : arguments.get("match")) { CtrlMgmtRecordList reclist; TSMgmtError error; // XXX filter the results to only match configuration records. - error = reclist.match(file_arguments[i]); + error = reclist.match(it.c_str()); if (error != TS_ERR_OKAY) { - CtrlMgmtError(error, "failed to fetch %s", file_arguments[i]); - return CTRL_EX_ERROR; + CtrlMgmtError(error, "failed to fetch %s", it.c_str()); + status_code = CTRL_EX_ERROR; + return; } while (!reclist.empty()) { CtrlMgmtRecord record(reclist.next()); if (REC_TYPE_IS_CONFIG(record.rclass())) { - format_record(record, recfmt); + format_record(record, arguments.get("records")); } } } - - return CTRL_EX_OK; } -static int -config_reload(unsigned argc, const char **argv) +void +CtrlEngine::config_reload() { - if (!CtrlProcessArguments(argc, argv, nullptr, 0) || n_file_arguments != 0) { - return CtrlCommandUsage("config reload"); - } - TSMgmtError error = TSReconfigure(); if (error != TS_ERR_OKAY) { CtrlMgmtError(error, "configuration reload request failed"); - return CTRL_EX_ERROR; + status_code = CTRL_EX_ERROR; + return; } - - return CTRL_EX_OK; } -static int -config_status(unsigned argc, const char **argv) +void +CtrlEngine::config_status() { - if (!CtrlProcessArguments(argc, argv, nullptr, 0) || n_file_arguments != 0) { - return CtrlCommandUsage("config status"); - } - CtrlMgmtRecord version; CtrlMgmtRecord configtime; CtrlMgmtRecord starttime; @@ -368,77 +332,57 @@ config_status(unsigned argc, const char **argv) CTRL_MGMT_CHECK(proxy.fetch("proxy.node.config.restart_required.proxy")); CTRL_MGMT_CHECK(manager.fetch("proxy.node.config.restart_required.manager")); - printf("%s\n", CtrlMgmtRecordValue(version).c_str()); - printf("Started at %s", timestr((time_t)starttime.as_int()).c_str()); - printf("Last reconfiguration at %s", timestr((time_t)configtime.as_int()).c_str()); - printf("%s\n", reconfig.as_int() ? "Reconfiguration required" : "Configuration is current"); + std::cout << CtrlMgmtRecordValue(version).c_str() << std::endl; + std::cout << "Started at " << timestr((time_t)starttime.as_int()).c_str(); + std::cout << "Last reconfiguration at " << timestr((time_t)configtime.as_int()).c_str(); + std::cout << (reconfig.as_int() ? "Reconfiguration required" : "Configuration is current") << std::endl; if (proxy.as_int()) { - printf("traffic_server requires restarting\n"); + std::cout << "traffic_server requires restarting" << std::endl; } if (manager.as_int()) { - printf("traffic_manager requires restarting\n"); + std::cout << "traffic_manager requires restarting\n" << std::endl; } - - return CTRL_EX_OK; } -static int -config_defaults(unsigned argc, const char **argv) +void +CtrlEngine::config_defaults() { - int recfmt = 0; - const ArgumentDescription opts[] = { - {"records", '-', "Emit output in records.config format", "F", &recfmt, nullptr, nullptr}, - }; - - if (!CtrlProcessArguments(argc, argv, opts, countof(opts)) || n_file_arguments != 0) { - return CtrlCommandUsage("config diff [OPTIONS]"); - } - TSMgmtError error; CtrlMgmtRecordDescriptionList descriptions; error = descriptions.match(".*"); if (error != TS_ERR_OKAY) { CtrlMgmtError(error, "failed to fetch record metadata"); - return CTRL_EX_ERROR; + status_code = CTRL_EX_ERROR; + return; } while (!descriptions.empty()) { TSConfigRecordDescription *desc = descriptions.next(); CtrlMgmtRecordValue deflt(desc->rec_type, desc->rec_default); - if (recfmt) { - printf("%s %s %s %s\n", rec_labelof(desc->rec_class), desc->rec_name, rec_typeof(desc->rec_type), deflt.c_str()); + if (arguments.get("records")) { + std::cout << rec_labelof(desc->rec_class) << ' ' << desc->rec_name << ' ' << rec_typeof(desc->rec_type) << ' ' + << deflt.c_str() << std::endl; } else { - printf("%s: %s\n", desc->rec_name, deflt.c_str()); + std::cout << desc->rec_name << ": " << deflt.c_str() << std::endl; } - TSConfigRecordDescriptionDestroy(desc); } - - return CTRL_EX_OK; } -static int -config_diff(unsigned argc, const char **argv) +void +CtrlEngine::config_diff() { - int recfmt = 0; - const ArgumentDescription opts[] = { - {"records", '-', "Emit output in records.config format", "F", &recfmt, nullptr, nullptr}, - }; - - if (!CtrlProcessArguments(argc, argv, opts, countof(opts)) || n_file_arguments != 0) { - return CtrlCommandUsage("config diff [OPTIONS]"); - } - TSMgmtError error; CtrlMgmtRecordDescriptionList descriptions; error = descriptions.match(".*"); if (error != TS_ERR_OKAY) { CtrlMgmtError(error, "failed to fetch record metadata"); - return CTRL_EX_ERROR; + status_code = CTRL_EX_ERROR; + return; } while (!descriptions.empty()) { @@ -468,35 +412,16 @@ config_diff(unsigned argc, const char **argv) CtrlMgmtRecordValue current(desc->rec_type, desc->rec_value); CtrlMgmtRecordValue deflt(desc->rec_type, desc->rec_default); - if (recfmt) { - printf("%s %s %s %s # default: %s\n", rec_labelof(desc->rec_class), desc->rec_name, rec_typeof(desc->rec_type), - current.c_str(), deflt.c_str()); + if (arguments.get("records")) { + std::cout << rec_labelof(desc->rec_class) << ' ' << desc->rec_name << ' ' << rec_typeof(desc->rec_type) << ' ' + << current.c_str() << " # default: " << deflt.c_str() << std::endl; } else { - printf("%s has changed\n", desc->rec_name); - printf("\t%-16s: %s\n", "Current Value", current.c_str()); - printf("\t%-16s: %s\n", "Default Value", deflt.c_str()); + std::cout << desc->rec_name << " has changed" << std::endl; + std::cout << "\tCurrent Value: " << current.c_str() << std::endl; + std::cout << "\tDefault Value: " << deflt.c_str() << std::endl; } } TSConfigRecordDescriptionDestroy(desc); } - - return CTRL_EX_OK; -} - -int -subcommand_config(unsigned argc, const char **argv) -{ - const subcommand commands[] = { - {config_defaults, "defaults", "Show default information configuration values"}, - {config_describe, "describe", "Show detailed information about configuration values"}, - {config_diff, "diff", "Show non-default configuration values"}, - {config_get, "get", "Get one or more configuration values"}, - {config_match, "match", "Get configuration matching a regular expression"}, - {config_reload, "reload", "Request a configuration reload"}, - {config_set, "set", "Set a configuration value"}, - {config_status, "status", "Check the configuration status"}, - }; - - return CtrlGenericSubcommand("config", commands, countof(commands), argc, argv); } diff --git a/src/traffic_ctl/host.cc b/src/traffic_ctl/host.cc index 1c69fa30480..2b23bc9dbb6 100644 --- a/src/traffic_ctl/host.cc +++ b/src/traffic_ctl/host.cc @@ -25,121 +25,78 @@ #include "HostStatus.h" #include "records/P_RecUtils.h" -static int -status_get(unsigned argc, const char **argv) +void +CtrlEngine::status_get() { - if (!CtrlProcessArguments(argc, argv, nullptr, 0) || n_file_arguments < 1) { - return CtrlCommandUsage("host status HOST [HOST ...]", nullptr, 0); - } - - for (unsigned i = 0; i < n_file_arguments; ++i) { + for (const auto &it : arguments.get("status")) { CtrlMgmtRecord record; TSMgmtError error; - std::string str = stat_prefix + file_arguments[i]; + std::string str = stat_prefix + it; for (const char *_reason_tag : Reasons::reasons) { std::string _stat = str + "_" + _reason_tag; error = record.fetch(_stat.c_str()); if (error != TS_ERR_OKAY) { - CtrlMgmtError(error, "failed to fetch %s", file_arguments[i]); - return CTRL_EX_ERROR; + CtrlMgmtError(error, "failed to fetch %s", it.c_str()); + status_code = CTRL_EX_ERROR; + return; } if (REC_TYPE_IS_STAT(record.rclass())) { - printf("%s %s\n", record.name(), CtrlMgmtRecordValue(record).c_str()); + std::cout << record.name() << ' ' << CtrlMgmtRecordValue(record).c_str() << std::endl; } } } - - return CTRL_EX_OK; } -static int -status_down(unsigned argc, const char **argv) +void +CtrlEngine::status_down() { - int down_time = 0; - char *reason = nullptr; - const char *usage = "host down HOST [OPTIONS]"; - - const ArgumentDescription opts[] = { - {"time", 'I', "number of seconds that a host is marked down", "I", &down_time, nullptr, nullptr}, - // memory is allocated for 'reason', if this option is used - {"reason", '-', "reason for marking the host down, one of 'manual|active|local'", "S*", &reason, nullptr, nullptr}, - }; - - if (!CtrlProcessArguments(argc, argv, opts, countof(opts)) || n_file_arguments < 1) { - return CtrlCommandUsage(usage, opts, countof(opts)); - } + int down_time = 0; + std::string reason = arguments.get("reason").value(); // if reason is not set, set it to manual (default) - if (reason == nullptr) { - reason = ats_strdup(Reasons::MANUAL); + if (reason.empty()) { + reason = Reasons::MANUAL; } - if (!Reasons::validReason(reason)) { - fprintf(stderr, "\nInvalid reason: '%s'\n\n", reason); - return CtrlCommandUsage(usage, opts, countof(opts)); + if (!Reasons::validReason(reason.c_str())) { + fprintf(stderr, "\nInvalid reason: '%s'\n\n", reason.c_str()); + parser.help_message(); } TSMgmtError error = TS_ERR_OKAY; - for (unsigned i = 0; i < n_file_arguments; ++i) { - error = TSHostStatusSetDown(file_arguments[i], down_time, reason); + for (const auto &it : arguments.get("down")) { + error = TSHostStatusSetDown(it.c_str(), down_time, reason.c_str()); if (error != TS_ERR_OKAY) { - CtrlMgmtError(error, "failed to set %s", file_arguments[i]); - return CTRL_EX_ERROR; + CtrlMgmtError(error, "failed to set %s", it.c_str()); + status_code = CTRL_EX_ERROR; + return; } } - ats_free(reason); - - return CTRL_EX_OK; } -static int -status_up(unsigned argc, const char **argv) +void +CtrlEngine::status_up() { - char *reason = nullptr; - const char *usage = "host up HOST [OPTIONS]"; - - const ArgumentDescription opts[] = { - // memory is allocated for 'reason', if this option is used - {"reason", '-', "reason for marking the host up, one of 'manual|active|local'", "S*", &reason, nullptr, nullptr}, - }; - - if (!CtrlProcessArguments(argc, argv, opts, countof(opts)) || n_file_arguments < 1) { - return CtrlCommandUsage(usage, nullptr, 0); - } + std::string reason = arguments.get("reason").value(); // if reason is not set, set it to manual (default) - if (reason == nullptr) { - reason = ats_strdup(Reasons::MANUAL); + if (reason.empty()) { + reason = Reasons::MANUAL; } - if (!Reasons::validReason(reason)) { - fprintf(stderr, "\nInvalid reason: '%s'\n\n", reason); - return CtrlCommandUsage(usage, opts, countof(opts)); + if (!Reasons::validReason(reason.c_str())) { + fprintf(stderr, "\nInvalid reason: '%s'\n\n", reason.c_str()); + parser.help_message(); } TSMgmtError error; - for (unsigned i = 0; i < n_file_arguments; ++i) { - error = TSHostStatusSetUp(file_arguments[i], 0, reason); + for (const auto &it : arguments.get("up")) { + error = TSHostStatusSetUp(it.c_str(), 0, reason.c_str()); if (error != TS_ERR_OKAY) { - CtrlMgmtError(error, "failed to set %s", file_arguments[i]); - return CTRL_EX_ERROR; + CtrlMgmtError(error, "failed to set %s", it.c_str()); + status_code = CTRL_EX_ERROR; + return; } } - ats_free(reason); - - return CTRL_EX_OK; -} - -int -subcommand_host(unsigned argc, const char **argv) -{ - const subcommand commands[] = { - {status_get, "status", "Get one or more host statuses"}, - {status_down, "down", "Set down one or more host(s) "}, - {status_up, "up", "Set up one or more host(s) "}, - - }; - - return CtrlGenericSubcommand("host", commands, countof(commands), argc, argv); } diff --git a/src/traffic_ctl/metric.cc b/src/traffic_ctl/metric.cc index 68796b5e62d..1076aceb293 100644 --- a/src/traffic_ctl/metric.cc +++ b/src/traffic_ctl/metric.cc @@ -24,103 +24,72 @@ #include "traffic_ctl.h" #include "records/P_RecUtils.h" -static int -metric_get(unsigned argc, const char **argv) +void +CtrlEngine::metric_get() { - if (!CtrlProcessArguments(argc, argv, nullptr, 0) || n_file_arguments < 1) { - return CtrlCommandUsage("metric get METRIC [METRIC ...]", nullptr, 0); - } - - for (unsigned i = 0; i < n_file_arguments; ++i) { + for (const auto &it : arguments.get("get")) { CtrlMgmtRecord record; TSMgmtError error; - error = record.fetch(file_arguments[i]); + error = record.fetch(it.c_str()); if (error != TS_ERR_OKAY) { - CtrlMgmtError(error, "failed to fetch %s", file_arguments[i]); - return CTRL_EX_ERROR; + CtrlMgmtError(error, "failed to fetch %s", it.c_str()); + status_code = CTRL_EX_ERROR; + return; } if (REC_TYPE_IS_STAT(record.rclass())) { - printf("%s %s\n", record.name(), CtrlMgmtRecordValue(record).c_str()); + std::cout << record.name() << ' ' << CtrlMgmtRecordValue(record).c_str() << std::endl; } } - - return CTRL_EX_OK; } -static int -metric_match(unsigned argc, const char **argv) +void +CtrlEngine::metric_match() { - if (!CtrlProcessArguments(argc, argv, nullptr, 0) || n_file_arguments < 1) { - return CtrlCommandUsage("metric match [OPTIONS] REGEX [REGEX ...]", nullptr, 0); - } - - for (unsigned i = 0; i < n_file_arguments; ++i) { + for (const auto &it : arguments.get("match")) { CtrlMgmtRecordList reclist; TSMgmtError error; - error = reclist.match(file_arguments[i]); + error = reclist.match(it.c_str()); if (error != TS_ERR_OKAY) { - CtrlMgmtError(error, "failed to fetch %s", file_arguments[i]); - return CTRL_EX_ERROR; + CtrlMgmtError(error, "failed to fetch %s", it.c_str()); + status_code = CTRL_EX_ERROR; + return; } while (!reclist.empty()) { CtrlMgmtRecord record(reclist.next()); if (REC_TYPE_IS_STAT(record.rclass())) { - printf("%s %s\n", record.name(), CtrlMgmtRecordValue(record).c_str()); + std::cout << record.name() << ' ' << CtrlMgmtRecordValue(record).c_str() << std::endl; } } } - - return CTRL_EX_OK; } -static int -metric_clear(unsigned argc, const char **argv) +void +CtrlEngine::metric_clear() { TSMgmtError error; error = TSStatsReset(nullptr); if (error != TS_ERR_OKAY) { CtrlMgmtError(error, "failed to clear metrics"); - return CTRL_EX_ERROR; + status_code = CTRL_EX_ERROR; + return; } - - return CTRL_EX_OK; } -static int -metric_zero(unsigned argc, const char **argv) +void +CtrlEngine::metric_zero() { TSMgmtError error; - for (unsigned i = 0; i < n_file_arguments; ++i) { - error = TSStatsReset(file_arguments[i]); + for (const auto &it : arguments.get("zero")) { + error = TSStatsReset(it.c_str()); if (error != TS_ERR_OKAY) { - CtrlMgmtError(error, "failed to clear %s", file_arguments[i]); - return CTRL_EX_ERROR; + CtrlMgmtError(error, "failed to clear %s", it.c_str()); + status_code = CTRL_EX_ERROR; } } - - return CTRL_EX_OK; -} - -int -subcommand_metric(unsigned argc, const char **argv) -{ - const subcommand commands[] = { - {metric_get, "get", "Get one or more metric values"}, - {metric_clear, "clear", "Clear all metric values"}, - {CtrlUnimplementedCommand, "describe", "Show detailed information about one or more metric values"}, - {metric_match, "match", "Get metrics matching a regular expression"}, - {CtrlUnimplementedCommand, "monitor", "Display the value of a metric over time"}, - - // We could allow clearing all the metrics in the "clear" subcommand, but that seems error-prone. It - // would be too easy to just expect a help message and accidentally nuke all the metrics. - {metric_zero, "zero", "Clear one or more metric values"}, - }; - - return CtrlGenericSubcommand("metric", commands, countof(commands), argc, argv); } diff --git a/src/traffic_ctl/plugin.cc b/src/traffic_ctl/plugin.cc index cec12fac9b5..8935ee7b1de 100644 --- a/src/traffic_ctl/plugin.cc +++ b/src/traffic_ctl/plugin.cc @@ -23,30 +23,16 @@ #include "traffic_ctl.h" -static int -plugin_msg(unsigned argc, const char **argv) +void +CtrlEngine::plugin_msg() { - if (!CtrlProcessArguments(argc, argv, nullptr, 0) || n_file_arguments != 2) { - return CtrlCommandUsage("plugin msg TAG DATA"); - } - TSMgmtError error; + auto msg_data = arguments.get("msg"); - error = TSLifecycleMessage(file_arguments[0], file_arguments[1], strlen(file_arguments[1]) + 1); + error = TSLifecycleMessage(msg_data[0].c_str(), msg_data[1].c_str(), msg_data[1].size() + 1); if (error != TS_ERR_OKAY) { - CtrlMgmtError(error, "message '%s' not sent", file_arguments[0]); - return CTRL_EX_ERROR; + CtrlMgmtError(error, "message '%s' not sent", msg_data[0].c_str()); + status_code = CTRL_EX_ERROR; + return; } - - return CTRL_EX_OK; -} - -int -subcommand_plugin(unsigned argc, const char **argv) -{ - const subcommand commands[] = { - {plugin_msg, "msg", "Send message to plugins - a TAG and the message DATA"}, - }; - - return CtrlGenericSubcommand("plugin", commands, countof(commands), argc, argv); } diff --git a/src/traffic_ctl/server.cc b/src/traffic_ctl/server.cc index d41064ffa61..58271ba4de4 100644 --- a/src/traffic_ctl/server.cc +++ b/src/traffic_ctl/server.cc @@ -23,30 +23,17 @@ #include "traffic_ctl.h" -static int drain = 0; -static int manager = 0; - -static int -restart(unsigned argc, const char **argv) +void +CtrlEngine::server_restart() { TSMgmtError error; - const char *usage = "server restart [OPTIONS]"; - unsigned flags = TS_RESTART_OPT_NONE; - - const ArgumentDescription opts[] = { - {"drain", '-', "Wait for client connections to drain before restarting", "F", &drain, nullptr, nullptr}, - {"manager", '-', "Restart traffic_manager as well as traffic_server", "F", &manager, nullptr, nullptr}, - }; - - if (!CtrlProcessArguments(argc, argv, opts, countof(opts)) || n_file_arguments != 0) { - return CtrlCommandUsage(usage, opts, countof(opts)); - } + unsigned flags = TS_RESTART_OPT_NONE; - if (drain) { + if (arguments.get("drain")) { flags |= TS_RESTART_OPT_DRAIN; } - if (manager) { + if (arguments.get("manager")) { error = TSRestart(flags); } else { error = TSBounce(flags); @@ -54,79 +41,51 @@ restart(unsigned argc, const char **argv) if (error != TS_ERR_OKAY) { CtrlMgmtError(error, "server restart failed"); - return CTRL_EX_ERROR; + status_code = CTRL_EX_ERROR; + return; } - - return CTRL_EX_OK; -} - -static int -server_restart(unsigned argc, const char **argv) -{ - return restart(argc, argv); } -static int -server_backtrace(unsigned argc, const char **argv) +void +CtrlEngine::server_backtrace() { TSMgmtError error; TSString trace = nullptr; - if (!CtrlProcessArguments(argc, argv, nullptr, 0) || n_file_arguments != 0) { - return CtrlCommandUsage("server backtrace"); - } - error = TSProxyBacktraceGet(0, &trace); if (error != TS_ERR_OKAY) { CtrlMgmtError(error, "server backtrace failed"); - return CTRL_EX_ERROR; + status_code = CTRL_EX_ERROR; + return; } - printf("%s\n", trace); + std::cout << trace << std::endl; TSfree(trace); - return CTRL_EX_OK; } -static int -server_status(unsigned argc, const char **argv) +void +CtrlEngine::server_status() { - if (!CtrlProcessArguments(argc, argv, nullptr, 0) || n_file_arguments != 0) { - return CtrlCommandUsage("server status"); - } - switch (TSProxyStateGet()) { case TS_PROXY_ON: - printf("Proxy -- on\n"); + std::cout << "Proxy -- on" << std::endl; break; case TS_PROXY_OFF: - printf("Proxy -- off\n"); + std::cout << "Proxy -- off" << std::endl; break; case TS_PROXY_UNDEFINED: - printf("Proxy status undefined\n"); + std::cout << "Proxy status undefined" << std::endl; break; } - - // XXX Surely we can report more useful status that this !?!! - - return CTRL_EX_OK; } -static int -server_stop(unsigned argc, const char **argv) +void +CtrlEngine::server_stop() { TSMgmtError error; - const char *usage = "server stop [OPTIONS]"; - unsigned flags = TS_RESTART_OPT_NONE; + unsigned flags = TS_RESTART_OPT_NONE; - const ArgumentDescription opts[] = { - {"drain", '-', "Wait for client connections to drain before stopping", "F", &drain, nullptr, nullptr}, - }; - - if (!CtrlProcessArguments(argc, argv, opts, countof(opts)) || n_file_arguments != 0) { - return CtrlCommandUsage(usage, opts, countof(opts)); - } - - if (drain) { + if (arguments.get("drain")) { flags |= TS_STOP_OPT_DRAIN; } @@ -134,62 +93,36 @@ server_stop(unsigned argc, const char **argv) if (error != TS_ERR_OKAY) { CtrlMgmtError(error, "server stop failed"); - return CTRL_EX_ERROR; + status_code = CTRL_EX_ERROR; + return; } - - return CTRL_EX_OK; } -static int -server_start(unsigned argc, const char **argv) +void +CtrlEngine::server_start() { TSMgmtError error; - int cache = 0; - int hostdb = 0; unsigned clear = TS_CACHE_CLEAR_NONE; - const ArgumentDescription opts[] = { - {"clear-cache", '-', "Clear the disk cache on startup", "F", &cache, nullptr, nullptr}, - {"clear-hostdb", '-', "Clear the DNS cache on startup", "F", &hostdb, nullptr, nullptr}, - }; - - if (!CtrlProcessArguments(argc, argv, opts, countof(opts)) || n_file_arguments != 0) { - return CtrlCommandUsage("server start [OPTIONS]", opts, countof(opts)); - } - - clear |= cache ? TS_CACHE_CLEAR_CACHE : TS_CACHE_CLEAR_NONE; - clear |= hostdb ? TS_CACHE_CLEAR_HOSTDB : TS_CACHE_CLEAR_NONE; + clear |= arguments.get("clear-cache") ? TS_CACHE_CLEAR_CACHE : TS_CACHE_CLEAR_NONE; + clear |= arguments.get("clear-hostdb") ? TS_CACHE_CLEAR_HOSTDB : TS_CACHE_CLEAR_NONE; error = TSProxyStateSet(TS_PROXY_ON, clear); if (error != TS_ERR_OKAY) { CtrlMgmtError(error, "server start failed"); - return CTRL_EX_ERROR; + status_code = CTRL_EX_ERROR; + return; } - - return CTRL_EX_OK; } -static int -server_drain(unsigned argc, const char **argv) +void +CtrlEngine::server_drain() { TSMgmtError error; - const char *usage = "server drain [OPTIONS]"; - - int no_new_connection = 0; - int undo = 0; - const ArgumentDescription opts[] = { - {"no-new-connection", 'N', "Wait for new connections down to threshold before starting draining", "F", &no_new_connection, - nullptr, nullptr}, - {"undo", 'U', "Recover server from the drain mode", "F", &undo, nullptr, nullptr}, - }; - - if (!CtrlProcessArguments(argc, argv, opts, countof(opts)) || n_file_arguments != 0) { - return CtrlCommandUsage(usage, opts, countof(opts)); - } - if (undo) { + if (arguments.get("undo")) { error = TSDrain(TS_DRAIN_OPT_UNDO); - } else if (no_new_connection) { + } else if (arguments.get("no-new-connection")) { error = TSDrain(TS_DRAIN_OPT_IDLE); } else { error = TSDrain(TS_DRAIN_OPT_NONE); @@ -197,21 +130,7 @@ server_drain(unsigned argc, const char **argv) if (error != TS_ERR_OKAY) { CtrlMgmtError(error, "server drain failed"); - return CTRL_EX_ERROR; + status_code = CTRL_EX_ERROR; + return; } - - return CTRL_EX_OK; -} - -int -subcommand_server(unsigned argc, const char **argv) -{ - const subcommand commands[] = {{server_backtrace, "backtrace", "Show a full stack trace of the traffic_server process"}, - {server_restart, "restart", "Restart Traffic Server"}, - {server_start, "start", "Start the proxy"}, - {server_status, "status", "Show the proxy status"}, - {server_stop, "stop", "Stop the proxy"}, - {server_drain, "drain", "Drain the requests"}}; - - return CtrlGenericSubcommand("server", commands, countof(commands), argc, argv); } diff --git a/src/traffic_ctl/storage.cc b/src/traffic_ctl/storage.cc index ebd96071eba..1e55bc94fde 100644 --- a/src/traffic_ctl/storage.cc +++ b/src/traffic_ctl/storage.cc @@ -23,33 +23,18 @@ #include "traffic_ctl.h" -static int -storage_offline(unsigned argc, const char **argv) +void +CtrlEngine::storage_offline() { - if (!CtrlProcessArguments(argc, argv, nullptr, 0) || n_file_arguments == 0) { - return CtrlCommandUsage("storage offline DEVICE [DEVICE ...]"); - } - - for (unsigned i = 0; i < n_file_arguments; ++i) { + auto offline_data = arguments.get("offline"); + for (const auto &it : offline_data) { TSMgmtError error; - error = TSStorageDeviceCmdOffline(file_arguments[i]); + error = TSStorageDeviceCmdOffline(it.c_str()); if (error != TS_ERR_OKAY) { - CtrlMgmtError(error, "failed to take %s offline", file_arguments[0]); - return CTRL_EX_ERROR; + CtrlMgmtError(error, "failed to take %s offline", offline_data[0].c_str()); + status_code = CTRL_EX_ERROR; + return; } } - - return CTRL_EX_OK; -} - -int -subcommand_storage(unsigned argc, const char **argv) -{ - const subcommand commands[] = { - {storage_offline, "offline", "Take one or more storage volumes offline"}, - {CtrlUnimplementedCommand, "status", "Show the storage configuration"}, - }; - - return CtrlGenericSubcommand("storage", commands, countof(commands), argc, argv); } diff --git a/src/traffic_ctl/traffic_ctl.cc b/src/traffic_ctl/traffic_ctl.cc index be61cf096dd..a826abb0cbf 100644 --- a/src/traffic_ctl/traffic_ctl.cc +++ b/src/traffic_ctl/traffic_ctl.cc @@ -28,8 +28,6 @@ #include "tscore/I_Layout.h" #include "tscore/runroot.h" -AppVersionInfo CtrlVersionInfo; - const char * CtrlMgmtRecord::name() const { @@ -145,120 +143,137 @@ CtrlMgmtError(TSMgmtError err, const char *fmt, ...) } } -int -CtrlSubcommandUsage(const char *name, const subcommand *cmds, unsigned ncmds, const ArgumentDescription *desc, unsigned ndesc) -{ - const char *opt = ndesc ? "[OPTIONS]" : ""; - const char *sep = (ndesc && name) ? " " : ""; - - fprintf(stderr, "Usage: traffic_ctl %s%s%s CMD [ARGS ...]\n\nSubcommands:\n", name ? name : "", sep, opt); - - for (unsigned i = 0; i < ncmds; ++i) { - fprintf(stderr, " %-16s%s\n", cmds[i].name, cmds[i].help); - } - - if (ndesc) { - usage(desc, ndesc, "\nOptions:"); - } - - return CTRL_EX_USAGE; -} - -int -CtrlCommandUsage(const char *msg, const ArgumentDescription *desc, unsigned ndesc) -{ - fprintf(stderr, "Usage: traffic_ctl %s\n", msg); - - if (ndesc) { - usage(desc, ndesc, "\nOptions:"); - } - - return CTRL_EX_USAGE; -} - -bool -CtrlProcessArguments(int /* argc */, const char **argv, const ArgumentDescription *desc, unsigned ndesc) -{ - n_file_arguments = 0; - return process_args_ex(&CtrlVersionInfo, desc, ndesc, argv); -} - -int -CtrlUnimplementedCommand(unsigned /* argc */, const char **argv) +void +CtrlEngine::CtrlUnimplementedCommand(std::string_view command) { - fprintf(stderr, "'%s' command is not implemented\n", *argv); - return CTRL_EX_UNIMPLEMENTED; + fprintf(stderr, "'%s' command is not implemented\n", command.data()); + status_code = CTRL_EX_UNIMPLEMENTED; } -int -CtrlGenericSubcommand(const char *name, const subcommand *cmds, unsigned ncmds, unsigned argc, const char **argv) -{ - CtrlCommandLine cmdline; - - // Process command line arguments and dump into variables - if (!CtrlProcessArguments(argc, argv, nullptr, 0) || n_file_arguments < 1) { - return CtrlSubcommandUsage(name, cmds, ncmds, nullptr, 0); - } - - cmdline.init(n_file_arguments, file_arguments); - - for (unsigned i = 0; i < ncmds; ++i) { - if (strcmp(file_arguments[0], cmds[i].name) == 0) { - return cmds[i].handler(cmdline.argc(), cmdline.argv()); - } - } - - return CtrlSubcommandUsage(name, cmds, ncmds, nullptr, 0); -} - -static const subcommand commands[] = { - {subcommand_alarm, "alarm", "Manipulate alarms"}, - {subcommand_config, "config", "Manipulate configuration records"}, - {subcommand_metric, "metric", "Manipulate performance metrics"}, - {subcommand_server, "server", "Stop, restart and examine the server"}, - {subcommand_storage, "storage", "Manipulate cache storage"}, - {subcommand_plugin, "plugin", "Interact with plugins"}, - {subcommand_host, "host", "Interact with host status"}, -}; - int main(int argc, const char **argv) { - CtrlCommandLine cmdline; - int debug = false; - - CtrlVersionInfo.setup(PACKAGE_NAME, "traffic_ctl", PACKAGE_VERSION, __DATE__, __TIME__, BUILD_MACHINE, BUILD_PERSON, ""); - program_name = CtrlVersionInfo.AppStr; - - ArgumentDescription argument_descriptions[] = { - {"debug", '-', "Enable debugging output", "F", &debug, nullptr, nullptr}, - {"help", 'h', "Print usage information", nullptr, nullptr, nullptr, - [](const ArgumentDescription *args, unsigned nargs, const char *arg_unused) { - CtrlSubcommandUsage(nullptr, commands, countof(commands), args, nargs); - }}, - VERSION_ARGUMENT_DESCRIPTION(), - RUNROOT_ARGUMENT_DESCRIPTION(), - }; + CtrlEngine engine; + + engine.parser.add_global_usage("traffic_ctl [OPTIONS] CMD [ARGS ...]"); + engine.parser.require_commands(); + + engine.parser.add_option("--debug", "", "Enable debugging output") + .add_option("--version", "-V", "Print version string") + .add_option("--help", "-h", "Print usage information") + .add_option("--run-root", "", "using TS_RUNROOT as sandbox", "TS_RUNROOT", 1); + + auto &alarm_command = engine.parser.add_command("alarm", "Manipulate alarms").require_commands(); + auto &config_command = engine.parser.add_command("config", "Manipulate configuration records").require_commands(); + auto &metric_command = engine.parser.add_command("metric", "Manipulate performance metrics").require_commands(); + auto &server_command = engine.parser.add_command("server", "Stop, restart and examine the server").require_commands(); + auto &storage_command = engine.parser.add_command("storage", "Manipulate cache storage").require_commands(); + auto &plugin_command = engine.parser.add_command("plugin", "Interact with plugins").require_commands(); + auto &host_command = engine.parser.add_command("host", "Interact with host status").require_commands(); + + // alarm commands + alarm_command.add_command("clear", "Clear all current alarms", [&]() { engine.alarm_clear(); }) + .add_example_usage("traffic_ctl alarm clear"); + alarm_command.add_command("list", "List all current alarms", [&]() { engine.alarm_list(); }) + .add_example_usage("traffic_ctl alarm list"); + alarm_command.add_command("resolve", "Resolve the listed alarms", "", MORE_THAN_ONE_ARG_N, [&]() { engine.alarm_resolve(); }) + .add_example_usage("traffic_ctl alarm resolve ALARM [ALARM ...]"); + + // config commands + config_command.add_command("defaults", "Show default information configuration values", [&]() { engine.config_defaults(); }) + .add_example_usage("traffic_ctl config defaults [OPTIONS]") + .add_option("--records", "", "Emit output in records.config format"); + config_command + .add_command("describe", "Show detailed information about configuration values", "", MORE_THAN_ONE_ARG_N, + [&]() { engine.config_describe(); }) + .add_example_usage("traffic_ctl config describe RECORD [RECORD ...]"); + config_command.add_command("diff", "Show non-default configuration values", [&]() { engine.config_diff(); }) + .add_example_usage("traffic_ctl config diff [OPTIONS]") + .add_option("--records", "", "Emit output in records.config format"); + config_command.add_command("get", "Get one or more configuration values", "", MORE_THAN_ONE_ARG_N, [&]() { engine.config_get(); }) + .add_example_usage("traffic_ctl config get [OPTIONS] RECORD [RECORD ...]") + .add_option("--records", "", "Emit output in records.config format"); + config_command + .add_command("match", "Get configuration matching a regular expression", "", MORE_THAN_ONE_ARG_N, + [&]() { engine.config_match(); }) + .add_example_usage("traffic_ctl config match [OPTIONS] REGEX [REGEX ...]") + .add_option("--records", "", "Emit output in records.config format"); + config_command.add_command("reload", "Request a configuration reload", [&]() { engine.config_reload(); }) + .add_example_usage("traffic_ctl config reload"); + config_command.add_command("status", "Check the configuration status", [&]() { engine.config_status(); }) + .add_example_usage("traffic_ctl config status"); + config_command.add_command("set", "Set a configuration value", "", 2, [&]() { engine.config_set(); }) + .add_example_usage("traffic_ctl config set RECORD VALUE"); + + // host commands + host_command.add_command("status", "Get one or more host statuses", "", MORE_THAN_ONE_ARG_N, [&]() { engine.status_get(); }) + .add_example_usage("traffic_ctl host status HOST [HOST ...]"); + host_command.add_command("down", "Set down one or more host(s)", "", MORE_THAN_ONE_ARG_N, [&]() { engine.status_down(); }) + .add_example_usage("traffic_ctl host down HOST [OPTIONS]") + .add_option("--time", "-I", "number of seconds that a host is marked down", "", 1) + .add_option("--reason", "", "reason for marking the host down, one of 'manual|active|local", "", 1); + host_command.add_command("up", "Set up one or more host(s)", "", MORE_THAN_ONE_ARG_N, [&]() { engine.status_up(); }) + .add_example_usage("traffic_ctl host up METRIC value") + .add_option("--reason", "", "reason for marking the host up, one of 'manual|active|local", "", 1); + + // metric commands + metric_command.add_command("get", "Get one or more metric values", "", MORE_THAN_ONE_ARG_N, [&]() { engine.metric_get(); }) + .add_example_usage("traffic_ctl metric get METRIC [METRIC ...]"); + metric_command.add_command("clear", "Clear all metric values", [&]() { engine.metric_clear(); }); + metric_command.add_command("describe", "Show detailed information about one or more metric values", "", MORE_THAN_ONE_ARG_N, + [&]() { engine.CtrlUnimplementedCommand("describe"); }); // not implemented + metric_command.add_command("match", "Get metrics matching a regular expression", "", MORE_THAN_ZERO_ARG_N, + [&]() { engine.metric_match(); }); + metric_command.add_command("monitor", "Display the value of a metric over time", "", MORE_THAN_ZERO_ARG_N, + [&]() { engine.CtrlUnimplementedCommand("monitor"); }); // not implemented + metric_command.add_command("zero", "Clear one or more metric values", "", MORE_THAN_ONE_ARG_N, [&]() { engine.metric_zero(); }); + + // plugin command + plugin_command.add_command("msg", "Send message to plugins - a TAG and the message DATA", "", 2, [&]() { engine.plugin_msg(); }) + .add_example_usage("traffic_ctl plugin msg TAG DATA"); + + // server commands + server_command.add_command("backtrace", "Show a full stack trace of the traffic_server process", + [&]() { engine.server_backtrace(); }); + server_command.add_command("restart", "Restart Traffic Server", [&]() { engine.server_backtrace(); }) + .add_example_usage("traffic_ctl server restart [OPTIONS]") + .add_option("--drain", "", "Wait for client connections to drain before restarting") + .add_option("--manager", "", "Restart traffic_manager as well as traffic_server"); + server_command.add_command("start", "Start the proxy", [&]() { engine.server_start(); }) + .add_example_usage("traffic_ctl server start [OPTIONS]") + .add_option("--clear-cache", "", "Clear the disk cache on startup") + .add_option("--clear-hostdb", "", "Clear the DNS cache on startup"); + server_command.add_command("status", "Show the proxy status", [&]() { engine.server_status(); }) + .add_example_usage("traffic_ctl server status"); + server_command.add_command("stop", "Stop the proxy", [&]() { engine.server_stop(); }) + .add_example_usage("traffic_ctl server stop [OPTIONS]") + .add_option("--drain", "", "Wait for client connections to drain before stopping"); + server_command.add_command("drain", "Drain the requests", [&]() { engine.server_drain(); }) + .add_example_usage("traffic_ctl server drain [OPTIONS]") + .add_option("--no-new-connection", "-N", "Wait for new connections down to threshold before starting draining") + .add_option("--undo", "-U", "Recover server from the drain mode"); + + // storage commands + storage_command + .add_command("offline", "Take one or more storage volumes offline", "", MORE_THAN_ONE_ARG_N, + [&]() { engine.storage_offline(); }) + .add_example_usage("storage offline DEVICE [DEVICE ...]"); + storage_command.add_command("status", "Show the storage configuration", "", MORE_THAN_ZERO_ARG_N, + [&]() { engine.CtrlUnimplementedCommand("status"); }); // not implemented + + // parse the arguments + engine.arguments = engine.parser.parse(argv); BaseLogFile *base_log_file = new BaseLogFile("stderr"); - diags = new Diags(program_name, "" /* tags */, "" /* actions */, base_log_file); + diags = new Diags("traffic_ctl", "" /* tags */, "" /* actions */, base_log_file); - // Process command line arguments and dump into variables - if (!CtrlProcessArguments(argc, argv, argument_descriptions, countof(argument_descriptions))) { - return CtrlSubcommandUsage(nullptr, commands, countof(commands), argument_descriptions, countof(argument_descriptions)); - } - - if (debug) { + if (engine.arguments.get("debug")) { diags->activate_taglist("traffic_ctl", DiagsTagType_Debug); diags->config.enabled[DiagsTagType_Debug] = true; diags->show_location = SHOW_LOCATION_DEBUG; } - if (n_file_arguments < 1) { - return CtrlSubcommandUsage(nullptr, commands, countof(commands), argument_descriptions, countof(argument_descriptions)); - } - - runroot_handler(argv); + argparser_runroot_handler(engine.arguments.get("--run-root").value(), argv[0]); Layout::create(); RecProcessInit(RECM_STAND_ALONE, diags); LibRecordsConfigInit(); @@ -272,16 +287,10 @@ main(int argc, const char **argv) // error. TSInit(rundir, static_cast(TS_MGMT_OPT_NO_EVENTS | TS_MGMT_OPT_NO_SOCK_TESTS)); - for (unsigned i = 0; i < countof(commands); ++i) { - if (strcmp(file_arguments[0], commands[i].name) == 0) { - CtrlCommandLine cmdline; - - cmdline.init(n_file_arguments, file_arguments); - return commands[i].handler(cmdline.argc(), cmdline.argv()); - } - } + engine.arguments.invoke(); // Done with the mgmt API. TSTerminate(); - return CtrlSubcommandUsage(nullptr, commands, countof(commands), argument_descriptions, countof(argument_descriptions)); + + return engine.status_code; } diff --git a/src/traffic_ctl/traffic_ctl.h b/src/traffic_ctl/traffic_ctl.h index 991c790bfa8..94073f6323e 100644 --- a/src/traffic_ctl/traffic_ctl.h +++ b/src/traffic_ctl/traffic_ctl.h @@ -30,28 +30,22 @@ #include "tscore/ink_args.h" #include "tscore/I_Version.h" #include "tscore/BaseLogFile.h" +#include "tscore/ArgParser.h" #include #include +#include -struct subcommand { - int (*handler)(unsigned, const char **); - const char *name; - const char *help; -}; - -extern AppVersionInfo CtrlVersionInfo; +// Exit status codes, following BSD's sysexits(3) +constexpr int CTRL_EX_OK = 0; +constexpr int CTRL_EX_ERROR = 2; +constexpr int CTRL_EX_UNIMPLEMENTED = 3; +constexpr int CTRL_EX_USAGE = EX_USAGE; +constexpr int CTRL_EX_UNAVAILABLE = 69; #define CtrlDebug(...) Debug("traffic_ctl", __VA_ARGS__) void CtrlMgmtError(TSMgmtError err, const char *fmt, ...) TS_PRINTFLIKE(2, 3); -int CtrlSubcommandUsage(const char *name, const subcommand *cmds, unsigned ncmds, const ArgumentDescription *desc, unsigned ndesc); -int CtrlCommandUsage(const char *msg, const ArgumentDescription *desc = nullptr, unsigned ndesc = 0); - -bool CtrlProcessArguments(int argc, const char **argv, const ArgumentDescription *desc, unsigned ndesc); -int CtrlUnimplementedCommand(unsigned argc, const char **argv); - -int CtrlGenericSubcommand(const char *, const subcommand *cmds, unsigned ncmds, unsigned argc, const char **argv); #define CTRL_MGMT_CHECK(expr) \ do { \ @@ -59,7 +53,8 @@ int CtrlGenericSubcommand(const char *, const subcommand *cmds, unsigned ncmds, if (e != TS_ERR_OKAY) { \ CtrlDebug("%s failed with status %d", #expr, e); \ CtrlMgmtError(e, nullptr); \ - return CTRL_EX_ERROR; \ + status_code = CTRL_EX_ERROR; \ + return; \ } \ } while (0) @@ -167,47 +162,59 @@ struct CtrlMgmtRecordList : CtrlMgmtList { TSMgmtError match(const char *); }; -struct CtrlCommandLine { - CtrlCommandLine() { this->args.push_back(nullptr); } - void - init(unsigned argc, const char **argv) - { - this->args.clear(); - for (unsigned i = 0; i < argc; ++i) { - this->args.push_back(argv[i]); - } - - // Always nullptr-terminate to keep ink_args happy. Note that we adjust arg() accordingly. - this->args.push_back(nullptr); - } - - unsigned - argc() - { - return args.size() - 1; - } - - const char ** - argv() - { - return &args[0]; - } - -private: - std::vector args; +// this is a engine for traffic_ctl containing the ArgParser and all the methods +// it also has a status code which can be set by these methods to return +struct CtrlEngine { + // the parser for traffic_ctl + ts::ArgParser parser; + // parsed arguments + ts::Arguments arguments; + // the return status code from functions + // By default it is set to CTRL_EX_OK so we don't need to set it + // in each method when they finish successfully. + int status_code = CTRL_EX_OK; + + // All traffic_ctl methods: + // unimplemented command + void CtrlUnimplementedCommand(std::string_view command); + + // alarm methods + void alarm_list(); + void alarm_clear(); + void alarm_resolve(); + + // config methods + void config_defaults(); + void config_diff(); + void config_status(); + void config_reload(); + void config_match(); + void config_get(); + void config_set(); + void config_describe(); + + // host methods + void status_get(); + void status_down(); + void status_up(); + + // metric methods + void metric_get(); + void metric_match(); + void metric_clear(); + void metric_zero(); + + // metric methods + void plugin_msg(); + + // server methods + void server_restart(); + void server_backtrace(); + void server_status(); + void server_stop(); + void server_start(); + void server_drain(); + + // storage methods + void storage_offline(); }; - -int subcommand_alarm(unsigned argc, const char **argv); -int subcommand_config(unsigned argc, const char **argv); -int subcommand_metric(unsigned argc, const char **argv); -int subcommand_server(unsigned argc, const char **argv); -int subcommand_storage(unsigned argc, const char **argv); -int subcommand_plugin(unsigned argc, const char **argv); -int subcommand_host(unsigned argc, const char **argv); - -// Exit status codes, following BSD's sysexits(3) -#define CTRL_EX_OK 0 -#define CTRL_EX_ERROR 2 -#define CTRL_EX_UNIMPLEMENTED 3 -#define CTRL_EX_USAGE EX_USAGE -#define CTRL_EX_UNAVAILABLE 69 diff --git a/src/traffic_layout/Makefile.inc b/src/traffic_layout/Makefile.inc index e22f3dec0b3..fd37a51123e 100644 --- a/src/traffic_layout/Makefile.inc +++ b/src/traffic_layout/Makefile.inc @@ -29,7 +29,7 @@ traffic_layout_traffic_layout_CPPFLAGS = \ traffic_layout_traffic_layout_LDFLAGS = \ $(AM_LDFLAGS) \ - @YAMLCPP_LDFLAGS@ + @YAMLCPP_LDFLAGS@ $(BROTLIENC_LIB) traffic_layout_traffic_layout_SOURCES = \ traffic_layout/traffic_layout.cc \ @@ -47,4 +47,4 @@ traffic_layout_traffic_layout_LDADD = \ $(top_builddir)/iocore/eventsystem/libinkevent.a \ $(top_builddir)/src/tscore/libtscore.la \ $(top_builddir)/src/tscpp/util/libtscpputil.la \ - @LIBTCL@ @HWLOC_LIBS@ @YAMLCPP_LIBS@ + @HWLOC_LIBS@ @YAMLCPP_LIBS@ @LIBLZMA@ diff --git a/src/traffic_layout/README b/src/traffic_layout/README index 0c65d6164b3..c7b88b8bfb0 100644 --- a/src/traffic_layout/README +++ b/src/traffic_layout/README @@ -2,4 +2,4 @@ traffic_layout is a easy-to-use tool to: - show the location of various installation paths and resources. - deal with traffic server runroot. -For detailed information, please refer to doc/appendices/command-line/traffic_layout.en.rst. +For detailed information, please refer to doc/appendices/command-line/traffic_layout.en and doc/developer-guide/layout/runroot.en. diff --git a/src/traffic_layout/engine.cc b/src/traffic_layout/engine.cc index ea2c96b7b4a..fc994b9c74e 100644 --- a/src/traffic_layout/engine.cc +++ b/src/traffic_layout/engine.cc @@ -39,16 +39,20 @@ #include #include #include +#include static const long MAX_LOGIN = ink_login_name_max(); static constexpr int MAX_GROUP_NUM = 32; -// for nftw check_directory -std::string directory_check; +// Personally I don't really like the way that nftw needs global variables. Right now there is no other options. +// This iteration through directory can be avoided after std::filesystem is in. +// for nftw check_directory +static std::string empty_check_directory; // for 'verify --fix' nftw iteration -int permission; -std::string cur_fix_dir; +static PermissionEntry permission_entry; +// if fix_flag is set, permission handler will perform fixing operation +static bool fix_flag = false; // the function for the checking of the yaml file in the passed in path // if found return the path, if not return empty string @@ -85,51 +89,27 @@ check_parent_path(const std::string &path) return {}; } -// check if we can create the runroot using path -// return true if the path is good to use -static bool -check_run_path(const std::string &arg, const bool forceflag) -{ - if (arg.empty()) { - return false; - } - // the condition of force create - if (is_directory(arg) && forceflag) { - std::cout << "Forcing creating runroot ..." << std::endl; - return true; - } - - // if directory already exist - if (is_directory(arg)) { - return true; - } else { - // try to create & remove - if (!create_directory(arg)) { - return false; - } - remove_directory(arg); - return true; - } -} - // handle the path of the engine during parsing static std::string path_handler(const std::string &path, bool run_flag, const std::string &command) { + std::string cur_working_dir = ""; char cwd[PATH_MAX]; - if (getcwd(cwd, sizeof(cwd)) == nullptr) { - ink_fatal("unexcepted failure from getcwd() code=%d", errno); + if (!getcwd(cwd, sizeof(cwd))) { + ink_warning("unexpected failure from getcwd() - %s", strerror(errno)); + } else { + cur_working_dir = cwd; } if (run_flag) { // no path passed in, use cwd if (path.empty()) { - return cwd; + return cur_working_dir; } if (path[0] == '/') { return path; } else { - return Layout::relative_to(cwd, path); + return Layout::relative_to(cur_working_dir, path); } } @@ -140,7 +120,7 @@ path_handler(const std::string &path, bool run_flag, const std::string &command) if (path[0] == '/') { cmdpath = check_path(path); } else { - cmdpath = check_path(Layout::relative_to(cwd, path)); + cmdpath = check_path(Layout::relative_to(cur_working_dir, path)); } if (!cmdpath.empty()) { return cmdpath; @@ -148,7 +128,7 @@ path_handler(const std::string &path, bool run_flag, const std::string &command) } // 2. find cwd or parent path of cwd to check - std::string cwdpath = check_parent_path(cwd); + std::string cwdpath = check_parent_path(cur_working_dir); if (!cwdpath.empty()) { return cwdpath; } @@ -157,8 +137,9 @@ path_handler(const std::string &path, bool run_flag, const std::string &command) char RealBinPath[PATH_MAX] = {0}; if (!command.empty() && realpath(command.c_str(), RealBinPath) != nullptr) { std::string bindir = RealBinPath; - bindir = bindir.substr(0, bindir.find_last_of("/")); // getting the bin dir not executable path - bindir = check_parent_path(bindir); + + bindir = bindir.substr(0, bindir.find_last_of("/")); // getting the bin dir not executable path + bindir = check_parent_path(bindir); if (!bindir.empty()) { return bindir; } @@ -167,29 +148,11 @@ path_handler(const std::string &path, bool run_flag, const std::string &command) return path; } -// if directory is not empty, ask input from user Y/N +// check if there is any directory inside empty_check_directory to see if it is empty or not static int -check_directory(const char *path, const struct stat *s, int flag, struct FTW *f) +check_directory_empty(const char *path, const struct stat *s, int flag) { - std::string checkpath = path; - if (checkpath != directory_check) { - // check for valid Y/N - for (int i = 0; i < 3; i++) { - std::cout << "Are you sure to create runroot inside a non-empty directory Y/N: "; - std::string input; - std::cin >> input; - if (input == "Y" || input == "y") { - // terminate nftw - return 1; - } - if (input == "N" || input == "n") { - exit(0); - } - } - ink_error("Invalid input Y/N"); - exit(70); - } - return 0; + return std::string(path) != empty_check_directory ? -1 : 0; } void @@ -199,6 +162,8 @@ LayoutEngine::info() if (arguments.get("features")) { produce_features(json); + } else if (arguments.get("versions")) { + produce_versions(json); } else { produce_layout(json); } @@ -208,14 +173,20 @@ void LayoutEngine::create_runroot() { // set up options - std::string path = path_handler(arguments.get("path").value(), true, _argv[0]); + std::string path = path_handler(arguments.get("path").value(), true, _argv[0]); + if (path.empty()) { + ink_error("Path not valid for creating"); + status_code = EX_SOFTWARE; + return; + } bool force_flag = arguments.get("force"); bool abs_flag = arguments.get("absolute"); std::string layout_file = arguments.get("layout").value(); if (layout_file.find("runroot.yaml") != std::string::npos) { ink_error( "'runroot.yaml' is a potentially dangerous name for '--layout' option.\nPlease set other name to the file for '--layout'"); - exit(70); + status_code = EX_SOFTWARE; + return; } // deal with the copy style CopyStyle copy_style = HARD; @@ -230,13 +201,10 @@ LayoutEngine::create_runroot() copy_style = HARD; } else { ink_error("Unknown copy style: '%s'", style.c_str()); - exit(70); + status_code = EX_USAGE; + return; } } - // check validity of the path - if (!check_run_path(path, force_flag)) { - ink_fatal("Failed to create runroot with path '%s'", path.c_str()); - } std::string original_root = Layout::get()->prefix; std::string ts_runroot = path; // check for existing runroot to use rather than create new one @@ -247,15 +215,35 @@ LayoutEngine::create_runroot() return; } if (!force_flag && !check_parent_path(ts_runroot).empty()) { - ink_fatal("Cannot create runroot inside another runroot"); + ink_error("Cannot create runroot inside another runroot"); + status_code = EX_SOFTWARE; + return; } std::cout << "creating runroot - " + ts_runroot << std::endl; // check if directory is empty when --force is not used if (is_directory(ts_runroot) && !force_flag) { - directory_check = ts_runroot; - nftw(ts_runroot.c_str(), check_directory, OPEN_MAX_FILE, FTW_DEPTH); + empty_check_directory = ts_runroot; + if (ftw(ts_runroot.c_str(), check_directory_empty, OPEN_MAX_FILE) != 0) { + // if the directory is not empty, check for valid Y/N + for (int i = 0; i < 3; i++) { + std::cout << "Are you sure to create runroot inside a non-empty directory Y/N: "; + std::string input; + std::cin >> input; + if (input == "Y" || input == "y") { + break; + } + if (input == "N" || input == "n") { + return; + } + if (i == 2) { + ink_error("Invalid input Y/N"); + status_code = EX_SOFTWARE; + return; + } + } + } } // create new root & copy from original to new runroot. then fill in the map @@ -281,10 +269,10 @@ LayoutEngine::create_runroot() if (layout_file.size() != 0) { try { YAML::Node yamlfile = YAML::LoadFile(layout_file); - for (auto &&it : yamlfile) { + for (const auto &it : yamlfile) { std::string key = it.first.as(); std::string value = it.second.as(); - auto &&iter = new_map.find(key); + auto iter = new_map.find(key); if (iter != new_map.end()) { iter->second = value; } else { @@ -303,7 +291,7 @@ LayoutEngine::create_runroot() // symlink the executables // set up path_map for yaml to emit key-value pairs RunrootMapType path_map; - for (auto &&it : original_map) { + for (const auto &it : original_map) { // path handle std::string join_path; if (it.second[0] && it.second[0] == '/') { @@ -345,7 +333,7 @@ LayoutEngine::create_runroot() YAML::Emitter yamlfile; // emit key value pairs to the yaml file yamlfile << YAML::BeginMap; - for (auto &&it : dir_vector) { + for (const auto &it : dir_vector) { yamlfile << YAML::Key << it; yamlfile << YAML::Value << path_map[it]; } @@ -367,18 +355,14 @@ LayoutEngine::remove_runroot() std::string path = path_handler(arguments.get("path").value(), false, _argv[0]); // check validity of the path if (path.empty()) { - ink_fatal("Path not valid (runroot.yaml not found)"); + ink_error("Path not valid (runroot.yaml not found)"); + status_code = EX_IOERR; + return; } std::string clean_root = path; append_slash(clean_root); - char cwd[PATH_MAX]; - if (getcwd(cwd, sizeof(cwd)) == nullptr) { - ink_fatal("unexcepted failure from getcwd() code=%d", errno); - } - std::string cur_working_dir = cwd; - if (arguments.get("force")) { // the force remove std::cout << "Forcing removing runroot ..." << std::endl; @@ -390,11 +374,20 @@ LayoutEngine::remove_runroot() RunrootMapType map = runroot_map(Layout::relative_to(clean_root, "runroot.yaml")); map.erase(LAYOUT_PREFIX); map.erase(LAYOUT_EXEC_PREFIX); - for (auto &&it : map) { + // get current working directory + std::string cur_working_dir = ""; + char cwd[PATH_MAX]; + if (getcwd(cwd, sizeof(cwd)) == nullptr) { + ink_warning("unexpected failure from getcwd() - %s", strerror(errno)); + } else { + cur_working_dir = cwd; + } + for (const auto &it : map) { std::string dir = it.second; append_slash(dir); // get the directory to remove: prefix/etc/trafficserver -> prefix/etc dir = dir.substr(0, dir.substr(clean_root.size()).find_first_of("/") + clean_root.size()); + // don't remove cwd if (cur_working_dir != dir) { remove_directory(dir); } else { @@ -418,230 +411,250 @@ LayoutEngine::remove_runroot() } } -// return an array containing gid and uid of provided user -static std::pair -get_giduid(std::string &user) -{ - passwd *pwd; - if (user[0] == '#') { - // Numeric user notation. - uid_t uid = (uid_t)atoi(&user[1]); - pwd = getpwuid(uid); - } else { - pwd = getpwnam(user.c_str()); - } - if (!pwd) { - // null ptr - ink_fatal("missing password database entry for '%s'", user.c_str()); - } - user = pwd->pw_name; - std::pair giduid = {pwd->pw_gid, pwd->pw_uid}; - return giduid; -} - -// output read permission -static void -output_read_permission(const std::string &permission) +// check permission for verify +static int +permission_handler(const char *path, const struct stat *s, int flag) { - if (permission[0] == '1') { - std::cout << "\tRead Permission: \033[1;32mPASSED\033[0m" << std::endl; - } else { - std::cout << "\tRead Permission: \033[1;31mFAILED\033[0m" << std::endl; + std::string cur_directory = permission_entry.name; + // filter traffic server related files + if (!filter_ts_files(cur_directory, path)) { + return 0; } -} - -// output write permission -static void -output_write_permission(const std::string &permission) -{ - if (permission[1] == '1') { - std::cout << "\tWrite Permission: \033[1;32mPASSED\033[0m" << std::endl; - } else { - std::cout << "\tWrite Permission: \033[1;31mFAILED\033[0m" << std::endl; + if (flag == FTW_NS || !s) { + ink_warning("unable to stat() destination path %s - %s", path, strerror(errno)); + return -1; } -} -// output execute permission -static void -output_execute_permission(const std::string &permission) -{ - if (permission[2] == '1') { - std::cout << "\tExecute Permission: \033[1;32mPASSED\033[0m" << std::endl; + int ret = 0; + // --------- for directories --------- + if (flag == FTW_D || flag == FTW_DNR) { + // always need read permission for directories + if (!(s->st_mode & permission_entry.r_mode)) { + if (fix_flag) { + if (chmod(path, s->st_mode | permission_entry.r_mode) < 0) { + ink_warning("Unable to change file mode on %s - %s", path, strerror(errno)); + } else { + std::cout << "Fixed read permission: " << path << std::endl; + } + } else { + std::cout << "Read permission failed for directory: " << path << std::endl; + ret = -1; + } + } + // need write permission for logdir, runtimedir and cachedir + if (cur_directory == LAYOUT_LOGDIR || cur_directory == LAYOUT_RUNTIMEDIR || cur_directory == LAYOUT_CACHEDIR) { + if (!(s->st_mode & permission_entry.w_mode)) { + if (fix_flag) { + if (chmod(path, s->st_mode | permission_entry.w_mode) < 0) { + ink_warning("Unable to change file mode on %s - %s", path, strerror(errno)); + } else { + std::cout << "Fixed write permission: " << path << std::endl; + } + } else { + std::cout << "Write permission failed for directory: " << path << std::endl; + ret = -1; + } + } + } + // always need execute permission for directories + if (!(s->st_mode & permission_entry.e_mode)) { + if (fix_flag) { + if (chmod(path, s->st_mode | permission_entry.e_mode) < 0) { + ink_warning("Unable to change file mode on %s - %s", path, strerror(errno)); + } else { + std::cout << "Fixed execute permission: " << path << std::endl; + } + } else { + std::cout << "Execute permission failed for directory: " << path << std::endl; + ret = -1; + } + } } else { - std::cout << "\tExecute Permission: \033[1;31mFAILED\033[0m" << std::endl; + // --------- for files --------- + // always need read permission for all files + if (!(s->st_mode & permission_entry.r_mode)) { + if (fix_flag) { + if (chmod(path, s->st_mode | permission_entry.r_mode) < 0) { + ink_warning("Unable to change file mode on %s - %s", path, strerror(errno)); + } else { + std::cout << "Fixed read permission: " << path << std::endl; + } + } else { + std::cout << "Read permission failed for file " << path << std::endl; + ret = -1; + } + } + // need write permission for files in logdir, runtimedir and cachedir + if (cur_directory == LAYOUT_LOGDIR || cur_directory == LAYOUT_RUNTIMEDIR || cur_directory == LAYOUT_CACHEDIR) { + if (!(s->st_mode & permission_entry.w_mode)) { + if (fix_flag) { + if (chmod(path, s->st_mode | permission_entry.w_mode) < 0) { + ink_warning("Unable to change file mode on %s - %s", path, strerror(errno)); + } else { + std::cout << "Fixed write permission: " << path << std::endl; + } + } else { + std::cout << "Write permission failed for file " << path << std::endl; + ret = -1; + } + } + } + // execute permission on files in bindir, sbindir, libdir and libexecdir + if (cur_directory == LAYOUT_BINDIR || cur_directory == LAYOUT_SBINDIR || cur_directory == LAYOUT_LIBDIR || + cur_directory == LAYOUT_LIBEXECDIR) { + std::string tmp_path = path; + // skip the files in perl5 and pkgconfig from libdir + if (tmp_path.find("/perl5/") != std::string::npos || tmp_path.find("/pkgconfig/") != std::string::npos) { + return 0; + } + if (!(s->st_mode & permission_entry.e_mode)) { + if (fix_flag) { + if (chmod(path, s->st_mode | permission_entry.e_mode) < 0) { + ink_warning("Unable to change file mode on %s - %s", path, strerror(errno)); + } else { + std::cout << "Fixed execute permission: " << path << std::endl; + } + } else { + std::cout << "Execute permission failed for file: " << path << std::endl; + ret = -1; + } + } + } } + return ret; } -// chmod the file permission -static int -chmod_files_permission(const char *path, const struct stat *s, int flag, struct FTW *f) +// used for prefix, exec_prefix and localstatedir +// only check the read/execute permission on those directories +static bool +check_directory_permission(const char *path) { - // ----- filter traffic server related files ----- - if (!filter_ts_files(cur_fix_dir, path)) { - return 0; - } struct stat stat_buffer; if (stat(path, &stat_buffer) < 0) { - ink_warning("unable to stat() destination path %s: %s", path, strerror(errno)); - return 0; + ink_warning("unable to stat() destination path %s - %s", path, strerror(errno)); + return false; } - if (chmod(path, stat_buffer.st_mode | permission) < 0) { - ink_warning("Unable to change file mode on %s: %s", path, strerror(errno)); + if (!(stat_buffer.st_mode & permission_entry.r_mode)) { + std::cout << "Read permission failed for: " << path << std::endl; + return false; } - return 0; -} - -// the fixing permission of runroot used by verify command -static void -fix_runroot(RunrootMapType &path_map, RunrootMapType &permission_map, RunrootMapType &usergroup_map) -{ - if (getuid() != 0) { - ink_error("To fix permission issues, root privileges are required.\nPlease run with sudo."); - exit(70); - } - for (auto &&it : permission_map) { - std::string name = it.first; - std::string usergroup = usergroup_map[name]; - std::string path = path_map[name]; - std::cout << "Fixing " + name + "..." << std::endl; - - int read_permission; - int write_permission; - int execute_permission; - if (usergroup == "owner") { - read_permission = S_IRUSR; - write_permission = S_IWUSR; - execute_permission = S_IXUSR; - } else if (usergroup == "group") { - read_permission = S_IRGRP; - write_permission = S_IWGRP; - execute_permission = S_IXGRP; - } else { - read_permission = S_IROTH; - write_permission = S_IWOTH; - execute_permission = S_IXOTH; - } - // fix read permission - permission = read_permission; - cur_fix_dir = name; - nftw(path.c_str(), chmod_files_permission, OPEN_MAX_FILE, FTW_DEPTH); - // fix write permission - if (name == LAYOUT_LOGDIR || name == LAYOUT_RUNTIMEDIR || name == LAYOUT_CACHEDIR) { - permission = write_permission; - nftw(path.c_str(), chmod_files_permission, OPEN_MAX_FILE, FTW_DEPTH); - } - // fix execute permission - if (name == LAYOUT_BINDIR || name == LAYOUT_SBINDIR || name == LAYOUT_LIBDIR || name == LAYOUT_LIBEXECDIR) { - permission = execute_permission; - nftw(path.c_str(), chmod_files_permission, OPEN_MAX_FILE, FTW_DEPTH); - } + if (!(stat_buffer.st_mode & permission_entry.e_mode)) { + std::cout << "Execute permission failed for: " << path << std::endl; + return false; } + return true; } +#if defined(darwin) +// on Darwin, getgrouplist() takes int. +using gid_type = int; +#else +using gid_type = gid_t; +#endif + // helper function to check if user is from the same group of path_gid static bool -from_group(std::string const &user, gid_t group_id, gid_t path_gid) +from_group(const char *user, gid_type group_id, gid_type path_gid) { int ngroups = MAX_GROUP_NUM; - // Retrieve group list -#if defined(darwin) - // on Darwin, getgrouplist() takes int. - int groups[ngroups]; - if (getgrouplist(user.c_str(), static_cast(group_id), groups, &ngroups) == -1) { - ink_fatal("Failed to get group list as user '%s'\n", user.c_str()); - } - for (int i = 0; i < ngroups; i++) { - if (static_cast(path_gid) == groups[i]) { - return true; - } - } -#else - gid_t groups[ngroups]; - if (getgrouplist(user.c_str(), group_id, groups, &ngroups) == -1) { - ink_fatal("Failed to get group list as user '%s'\n", user.c_str()); + gid_type groups[ngroups]; + if (getgrouplist(user, group_id, groups, &ngroups) == -1) { + ink_warning("Unable to get group list as user '%s'\n", user); + return false; } for (int i = 0; i < ngroups; i++) { if (path_gid == groups[i]) { return true; } } -#endif return false; } // set permission to the map in verify runroot static void -set_permission(std::vector const &dir_vector, RunrootMapType &path_map, RunrootMapType &permission_map, - RunrootMapType &usergroup_map, std::string &user) +set_permission(PermissionMapType &permission_map, const struct passwd *pwd) { - // active group and user of the path - std::pair giduid = get_giduid(user); - - gid_t ts_gid = giduid.first; - uid_t ts_uid = giduid.second; - - struct stat stat_buffer; + gid_t ts_gid = pwd->pw_gid; + uid_t ts_uid = pwd->pw_uid; + bool new_line = false; // set up permission map for all permissions - for (auto &&it : dir_vector) { - std::string name = it; - std::string value = path_map[name]; - - if (name == LAYOUT_PREFIX || name == LAYOUT_EXEC_PREFIX) { - continue; - } + for (auto &it : permission_map) { + std::string name = it.first; + std::string value = it.second.path; + struct stat stat_buffer; if (stat(value.c_str(), &stat_buffer) < 0) { - ink_warning("unable to stat() destination path %s: %s", value.c_str(), strerror(errno)); + ink_warning("unable to stat() destination path %s - %s", value.c_str(), strerror(errno)); + it.second.result = false; + new_line = true; continue; } gid_t path_gid = stat_buffer.st_gid; uid_t path_uid = stat_buffer.st_uid; - permission_map[name] = "000"; // default rwx all 0 if (ts_uid == path_uid) { - // check for owner permission of st_mode - usergroup_map[name] = "owner"; - if (stat_buffer.st_mode & S_IRUSR) { - permission_map[name][0] = '1'; - } - if (stat_buffer.st_mode & S_IWUSR) { - permission_map[name][1] = '1'; - } - if (stat_buffer.st_mode & S_IXUSR) { - permission_map[name][2] = '1'; - } - } else if (from_group(user, ts_gid, path_gid)) { // current user is in the path_gid group - // check for group permission of st_mode - usergroup_map[name] = "group"; - if (stat_buffer.st_mode & S_IRGRP) { - permission_map[name][0] = '1'; - } - if (stat_buffer.st_mode & S_IWGRP) { - permission_map[name][1] = '1'; - } - if (stat_buffer.st_mode & S_IXGRP) { - permission_map[name][2] = '1'; - } + it.second.r_mode = S_IRUSR; + it.second.w_mode = S_IWUSR; + it.second.e_mode = S_IXUSR; + } else if (from_group(pwd->pw_name, ts_gid, path_gid)) { + it.second.r_mode = S_IRGRP; + it.second.w_mode = S_IWGRP; + it.second.e_mode = S_IXGRP; } else { - // check for others permission of st_mode - usergroup_map[name] = "others"; - if (stat_buffer.st_mode & S_IROTH) { - permission_map[name][0] = '1'; - } - if (stat_buffer.st_mode & S_IWOTH) { - permission_map[name][1] = '1'; + it.second.r_mode = S_IROTH; + it.second.w_mode = S_IWOTH; + it.second.e_mode = S_IXOTH; + } + // set the result in permission entry + permission_entry = it.second; + it.second.result = true; + // Special case with prefix, exec_prefix and localstatedir. We only need to check the directory itself + // but not the files within because they are just container dir for others. + if (name == LAYOUT_PREFIX || name == LAYOUT_EXEC_PREFIX || name == LAYOUT_LOCALSTATEDIR) { + if (!check_directory_permission(value.c_str())) { + it.second.result = false; + new_line = true; } - if (stat_buffer.st_mode & S_IXOTH) { - permission_map[name][2] = '1'; + } else { + // go through all files to check permission + if (ftw(value.c_str(), permission_handler, OPEN_MAX_FILE) != 0) { + it.second.result = false; + new_line = true; } } } + if (new_line) { + std::cout << std::endl; // print a new line after the failed permission messages + } +} + +// the fixing permission of runroot used by verify command +static void +fix_runroot(PermissionMapType &permission_map, const struct passwd *pwd) +{ + fix_flag = true; + for (const auto &it : permission_map) { + std::string name = it.first; + std::string value = it.second.path; + permission_entry = permission_map[name]; + ftw(value.c_str(), permission_handler, OPEN_MAX_FILE); + } + fix_flag = false; + set_permission(permission_map, pwd); } void LayoutEngine::verify_runroot() { + // require sudo permission for --fix + if (arguments.get("fix") && getuid() != 0) { + ink_error("To fix permission issues, root privilege is required.\nPlease run with sudo."); + status_code = EX_SOFTWARE; + return; + } + + // retrieve information std::string path = path_handler(arguments.get("path").value(), false, _argv[0]); std::string user; char user_buf[MAX_LOGIN + 1]; @@ -656,49 +669,53 @@ LayoutEngine::verify_runroot() user = TS_PKGSYSUSER; } } + // Numeric user notation for user + if (user[0] == '#') { + struct passwd *pwd = getpwuid(atoi(&user[1])); + if (!pwd) { + ink_error("No user found under id '%s'", user.c_str()); + status_code = EX_OSERR; + return; + } + user = pwd->pw_name; + } + std::cout << "Verifying permission as user: \x1b[1m" << user << "\x1b[0m" << std::endl << std::endl; + // try to find the user from password file + struct passwd *pwd = getpwnam(user.c_str()); + if (!pwd) { + ink_error("No user found under name '%s'", user.c_str()); + status_code = EX_OSERR; + return; + } // put paths from yaml file or default paths to path_map RunrootMapType path_map; if (path.empty()) { path_map = runroot_map_default(); - std::cout << "Verifying default build ..." << std::endl; } else { path_map = runroot_map(Layout::relative_to(path, "runroot.yaml")); } - - RunrootMapType permission_map; // contains rwx bits - RunrootMapType usergroup_map; // map: owner, group, others - - set_permission(dir_vector, path_map, permission_map, usergroup_map, user); - - std::cout << "Verifying as user: " << user << std::endl << std::endl; - - // if --fix is used - if (arguments.get("fix")) { - fix_runroot(path_map, permission_map, usergroup_map); - set_permission(dir_vector, path_map, permission_map, usergroup_map, user); - } else { - std::cout << "Note: This only shows directory permissions, please use '--fix' to correct permissions on all files" << std::endl; + // setup the permission map + PermissionMapType permission_map; + for (const auto &it : dir_vector) { + permission_map[it].name = it; + permission_map[it].path = path_map[it]; + } + // root always has full access and no need to check for root + if (user != "root") { + set_permission(permission_map, pwd); + // if --fix is used + if (arguments.get("fix")) { + fix_runroot(permission_map, pwd); + } } // display pass or fail for permission required - for (uint i = 2; i < dir_vector.size(); i++) { - std::string name = dir_vector[i]; - std::string permission = permission_map[dir_vector[i]]; - std::cout << name << ": \x1b[1m" + path_map[name] + "\x1b[0m" << std::endl; - - // output read permission - if (name == LAYOUT_LOCALSTATEDIR || name == LAYOUT_INCLUDEDIR || name == LAYOUT_SYSCONFDIR || name == LAYOUT_DATADIR) { - output_read_permission(permission); - } - // output write permission - if (name == LAYOUT_LOGDIR || name == LAYOUT_RUNTIMEDIR || name == LAYOUT_CACHEDIR) { - output_read_permission(permission); - output_write_permission(permission); - } - // output execute permission - if (name == LAYOUT_BINDIR || name == LAYOUT_SBINDIR || name == LAYOUT_LIBDIR || name == LAYOUT_LIBEXECDIR) { - output_read_permission(permission); - output_execute_permission(permission); + for (const auto &it : dir_vector) { + if (permission_map[it].result) { + std::cout << it << ": \x1b[1m" << path_map[it] << "\x1b[0m \033[1;32mPASSED\033[0m" << std::endl; + } else { + std::cout << it << ": \x1b[1m" << path_map[it] << "\x1b[0m \033[1;31mFAILED\033[0m" << std::endl; + status_code = EX_SOFTWARE; } } } diff --git a/src/traffic_layout/engine.h b/src/traffic_layout/engine.h index 9154f1b8358..91098e7d42e 100644 --- a/src/traffic_layout/engine.h +++ b/src/traffic_layout/engine.h @@ -23,12 +23,27 @@ #pragma once +#include #include "tscore/ArgParser.h" #include -typedef std::unordered_map RunrootMapType; +// used by runroot verify +struct PermissionEntry { + std::string name; // sysconfdir, libdir ... + std::string path; // real path of the directory + // required permission + mode_t r_mode; + mode_t w_mode; + mode_t e_mode; + // result set by set_permission() + bool result = true; +}; + +// this map contain the corresponding permission information of directories +// PermissionEntry contains the read/write/execute mode and the result of output +using PermissionMapType = std::unordered_map; -// structure for informaiton of the runroot passing around +// structure for information of the runroot passing around struct LayoutEngine { // default output of all layouts void info(); @@ -47,6 +62,8 @@ struct LayoutEngine { ts::ArgParser parser; // parsed arguments ts::Arguments arguments; - // mordern argv + // modern argv std::vector _argv; + + int status_code = 0; }; diff --git a/src/traffic_layout/file_system.cc b/src/traffic_layout/file_system.cc index 644b68a90d8..f75fcebc0ed 100644 --- a/src/traffic_layout/file_system.cc +++ b/src/traffic_layout/file_system.cc @@ -73,15 +73,15 @@ create_directory(const std::string &dir) int ret = 0, pos = 0, pos1 = 0; if ((s[0] == '.') || (s[0] == '/')) { - pos1 = s.find("/") + 1; + pos1 = s.find('/') + 1; } - pos = s.find("/", pos1); + pos = s.find('/', pos1); ret = mkdir(s.substr(0, pos).c_str(), S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH); pos1 = pos + 1; // create directory one layer by one layer while (true) { - pos = s.find("/", pos1); + pos = s.find('/', pos1); if ((size_t)pos == s.npos) { break; } @@ -192,7 +192,7 @@ filter_ts_files(const std::string &dir, const std::string &dst_path) // ----- filter traffic server related files ----- if (dir == LAYOUT_BINDIR || dir == LAYOUT_SBINDIR) { // check if executable is in the list of traffic server executables. If not, end the copying. - if (executables.find(dst_path.substr(dst_path.find_last_of("/") + 1)) == executables.end()) { + if (executables.find(dst_path.substr(dst_path.find_last_of('/') + 1)) == executables.end()) { return false; } } @@ -247,7 +247,7 @@ ts_copy_function(const char *src_path, const struct stat *sb, int flag) // if the file already exist, overwrite it if (exists(dst_path)) { if (remove(dst_path.c_str())) { - ink_error("overwrite file falied during copy"); + ink_error("overwrite file failed during copy"); } } // hardlink bin executable @@ -271,7 +271,7 @@ ts_copy_function(const char *src_path, const struct stat *sb, int flag) std::ofstream dst(dst_path, std::ios::binary); dst << src.rdbuf(); if (chmod(dst_path.c_str(), sb->st_mode) == -1) { - ink_warning("failed chomd the destination path: %s", strerror(errno)); + ink_warning("failed chmod the destination path: %s", strerror(errno)); } } return 0; diff --git a/src/traffic_layout/file_system.h b/src/traffic_layout/file_system.h index 2d1210c4ff6..1f174c200cb 100644 --- a/src/traffic_layout/file_system.h +++ b/src/traffic_layout/file_system.h @@ -32,7 +32,7 @@ // full copy, hardlink, softlink enum CopyStyle { FULL, HARD, SOFT }; -// append slash & remove slash of path for convinient use +// append slash & remove slash of path for convenient use void append_slash(std::string &path); // for file system diff --git a/src/traffic_layout/info.cc b/src/traffic_layout/info.cc index f67a737fd20..184049d1abf 100644 --- a/src/traffic_layout/info.cc +++ b/src/traffic_layout/info.cc @@ -21,11 +21,25 @@ limitations under the License. */ +#include #include "tscore/I_Layout.h" +#include "tscore/BufferWriter.h" #include "records/I_RecProcess.h" #include "RecordsConfig.h" #include "info.h" +#if HAVE_ZLIB_H +#include +#endif + +#if HAVE_LZMA_H +#include +#endif + +#if HAVE_BROTLI_ENCODE_H +#include +#endif + // Produce output about compile time features, useful for checking how things were built, as well // as for our TSQA test harness. static void @@ -59,17 +73,17 @@ produce_features(bool json) print_feature("BUILD_PERSON", BUILD_PERSON, json); print_feature("BUILD_GROUP", BUILD_GROUP, json); print_feature("BUILD_NUMBER", BUILD_NUMBER, json); -#ifdef HAVE_ZLIB_H +#if HAVE_ZLIB_H print_feature("TS_HAS_LIBZ", 1, json); #else print_feature("TS_HAS_LIBZ", 0, json); #endif -#ifdef HAVE_LZMA_H +#if HAVE_LZMA_H print_feature("TS_HAS_LZMA", 1, json); #else print_feature("TS_HAS_LZMA", 0, json); #endif -#ifdef HAVE_BROTLI_ENCODE_H +#if HAVE_BROTLI_ENCODE_H print_feature("TS_HAS_BROTLI", 1, json); #else print_feature("TS_HAS_BROTLI", 0, json); @@ -89,11 +103,7 @@ produce_features(bool json) print_feature("TS_HAS_SO_MARK", TS_HAS_SO_MARK, json); print_feature("TS_HAS_IP_TOS", TS_HAS_IP_TOS, json); print_feature("TS_USE_HWLOC", TS_USE_HWLOC, json); - print_feature("TS_USE_TLS_NPN", TS_USE_TLS_NPN, json); - print_feature("TS_USE_TLS_ALPN", TS_USE_TLS_ALPN, json); - print_feature("TS_USE_CERT_CB", TS_USE_CERT_CB, json); print_feature("TS_USE_SET_RBIO", TS_USE_SET_RBIO, json); - print_feature("TS_USE_TLS_ECKEY", TS_USE_TLS_ECKEY, json); print_feature("TS_USE_LINUX_NATIVE_AIO", TS_USE_LINUX_NATIVE_AIO, json); print_feature("TS_HAS_SO_PEERCRED", TS_HAS_SO_PEERCRED, json); print_feature("TS_USE_REMOTE_UNWINDING", TS_USE_REMOTE_UNWINDING, json); @@ -156,3 +166,48 @@ produce_layout(bool json) printf("}\n"); } } + +void +produce_versions(bool json) +{ + using LBW = ts::LocalBufferWriter<128>; + static const std::string_view undef{"undef"}; + + if (json) { + printf("{\n"); + } + + print_var("openssl", LBW().print("{:#x}", OPENSSL_VERSION_NUMBER).view(), json); + print_var("openssl_str", LBW().print(OPENSSL_VERSION_TEXT).view(), json); + print_var("pcre", LBW().print("{}.{}", PCRE_MAJOR, PCRE_MINOR).view(), json); + // These are optional, for now at least. +#if TS_USE_HWLOC + print_var("hwloc", LBW().print("{:#x}", HWLOC_API_VERSION).view(), json); + print_var("hwloc.run", LBW().print("{:#x}", hwloc_get_api_version()).view(), json); +#else + print_var("hwloc", undef, json); +#endif +#if HAVE_ZLIB_H + print_var("libz", LBW().print("{}", ZLIB_VERSION).view(), json); +#else + print_var("libz", undef, json); +#endif +#if HAVE_LZMA_H + print_var("lzma", LBW().print("{}", LZMA_VERSION_STRING).view(), json); + print_var("lzma.run", LBW().print("{}", lzma_version_string()).view(), json); +#else + print_var("lzma", undef, json); +#endif +#if HAVE_BROTLI_ENCODE_H + print_var("brotli", LBW().print("{:#x}", BrotliEncoderVersion()).view(), json); +#else + print_var("brotli", undef, json); +#endif + + // This should always be last + print_var("traffic-server", LBW().print(TS_VERSION_STRING).view(), json, true); + + if (json) { + printf("}\n"); + } +} diff --git a/src/traffic_layout/info.h b/src/traffic_layout/info.h index c70f0ec8756..fc3831c0454 100644 --- a/src/traffic_layout/info.h +++ b/src/traffic_layout/info.h @@ -24,5 +24,5 @@ // the original traffic_layout void produce_features(bool json); - void produce_layout(bool json); +void produce_versions(bool json); diff --git a/src/traffic_layout/traffic_layout.cc b/src/traffic_layout/traffic_layout.cc index a0095d142e0..935a6a77c89 100644 --- a/src/traffic_layout/traffic_layout.cc +++ b/src/traffic_layout/traffic_layout.cc @@ -40,12 +40,14 @@ main(int argc, const char **argv) engine.parser.add_global_usage("traffic_layout CMD [OPTIONS]"); // global options - engine.parser.add_option("--help", "-h", "Print usage information"); - engine.parser.add_option("--run-root", "", "using TS_RUNROOT as sandbox", "", 1); + engine.parser.add_option("--help", "-h", "Print usage information") + .add_option("--run-root", "", "using TS_RUNROOT as sandbox", "TS_RUNROOT", 1) + .add_option("--version", "-V", "Print version string"); // info command engine.parser.add_command("info", "Show the layout as default", [&]() { engine.info(); }) .add_option("--features", "", "Show the compiled features") + .add_option("--versions", "", "Show various library and other versioning information") .add_option("--json", "-j", "Produce output in JSON format (when supported)") .set_default(); // init command @@ -67,10 +69,11 @@ main(int argc, const char **argv) engine.arguments = engine.parser.parse(argv); - runroot_handler(argv, engine.arguments.get("json")); + auto runroot_arg = engine.arguments.get("run-root"); + argparser_runroot_handler(runroot_arg.value(), argv[0], engine.arguments.get("json")); Layout::create(); engine.arguments.invoke(); - return 0; + return engine.status_code; } diff --git a/src/traffic_logcat/Makefile.inc b/src/traffic_logcat/Makefile.inc index 4c962424051..fe77562a4c6 100644 --- a/src/traffic_logcat/Makefile.inc +++ b/src/traffic_logcat/Makefile.inc @@ -50,6 +50,6 @@ traffic_logcat_traffic_logcat_LDADD = \ $(top_builddir)/src/tscpp/util/libtscpputil.la traffic_logcat_traffic_logcat_LDADD += \ - @LIBTCL@ @HWLOC_LIBS@ \ + @HWLOC_LIBS@ \ @YAMLCPP_LIBS@ \ @LIBPROFILER@ -lm diff --git a/src/traffic_logcat/logcat.cc b/src/traffic_logcat/logcat.cc index 879938c47c0..fef226b3dc2 100644 --- a/src/traffic_logcat/logcat.cc +++ b/src/traffic_logcat/logcat.cc @@ -38,12 +38,10 @@ #include "LogFilter.h" #include "LogFormat.h" #include "LogFile.h" -#include "LogHost.h" #include "LogObject.h" #include "LogConfig.h" #include "LogBuffer.h" #include "LogUtils.h" -#include "LogSock.h" #include "Log.h" // logcat-specific command-line flags @@ -198,7 +196,7 @@ process_file(int in_fd, int out_fd) // see if there is an alternate format request from the command // line // - const char *alt_format = NULL; + const char *alt_format = nullptr; // convert the buffer to ascii entries and place onto stdout // if (header->fmt_fieldlist()) { @@ -218,7 +216,7 @@ open_output_file(char *output_file) if (access(output_file, F_OK)) { if (errno != ENOENT) { fprintf(stderr, "Error accessing output file %s: ", output_file); - perror(0); + perror(nullptr); file_desc = -1; } } else { @@ -235,7 +233,7 @@ open_output_file(char *output_file) if (file_desc < 0) { fprintf(stderr, "Error while opening output file %s: ", output_file); - perror(0); + perror(nullptr); } } @@ -303,7 +301,7 @@ main(int /* argc ATS_UNUSED */, const char *argv[]) int in_fd = open(file_arguments[i], O_RDONLY); if (in_fd < 0) { fprintf(stderr, "Error opening input file %s: ", file_arguments[i]); - perror(0); + perror(nullptr); error = DATA_PROCESSING_ERROR; } else { #if HAVE_POSIX_FADVISE diff --git a/src/traffic_logstats/Makefile.inc b/src/traffic_logstats/Makefile.inc index 89260840107..12f8335692f 100644 --- a/src/traffic_logstats/Makefile.inc +++ b/src/traffic_logstats/Makefile.inc @@ -54,6 +54,6 @@ traffic_logstats_traffic_logstats_LDADD = \ $(top_builddir)/src/tscpp/util/libtscpputil.la traffic_logstats_traffic_logstats_LDADD += \ - @LIBTCL@ @HWLOC_LIBS@ \ + @HWLOC_LIBS@ \ @YAMLCPP_LIBS@ \ @LIBPROFILER@ -lm diff --git a/src/traffic_logstats/logstats.cc b/src/traffic_logstats/logstats.cc index 72885e1bc65..d396c4ed3c1 100644 --- a/src/traffic_logstats/logstats.cc +++ b/src/traffic_logstats/logstats.cc @@ -599,39 +599,25 @@ struct CommandLineArgs { char log_file[1024]; char origin_file[1024]; char origin_list[MAX_ORIG_STRING]; - int max_origins; + int max_origins = 0; char state_tag[1024]; - int64_t min_hits; - int max_age; + int64_t min_hits = 0; + int max_age = 0; int line_len; - int incremental; // Do an incremental run - int tail; // Tail the log file - int summary; // Summary only - int json; // JSON output - int cgi; // CGI output (typically with json) - int urls; // Produce JSON output of URL stats, arg is LRU size - int show_urls; // Max URLs to show - int as_object; // Show the URL stats as a single JSON object (not array) - int concise; // Eliminate metrics that can be inferred by other values - int report_per_user; // A flag to aggregate and report stats per user instead of per host if 'true' (default 'false') - int no_format_check; // A flag to skip the log format check if any of the fields is not a standard squid log format field. - - CommandLineArgs() - : max_origins(0), - min_hits(0), - max_age(0), - line_len(DEFAULT_LINE_LEN), - incremental(0), - tail(0), - summary(0), - json(0), - cgi(0), - urls(0), - show_urls(0), - as_object(0), - concise(0), - report_per_user(0), - no_format_check(0) + int incremental = 0; // Do an incremental run + int tail = 0; // Tail the log file + int summary = 0; // Summary only + int json = 0; // JSON output + int cgi = 0; // CGI output (typically with json) + int urls = 0; // Produce JSON output of URL stats, arg is LRU size + int show_urls = 0; // Max URLs to show + int as_object = 0; // Show the URL stats as a single JSON object (not array) + int concise = 0; // Eliminate metrics that can be inferred by other values + int report_per_user = 0; // A flag to aggregate and report stats per user instead of per host if 'true' (default 'false') + int no_format_check = 0; // A flag to skip the log format check if any of the fields is not a standard squid log format field. + + CommandLineArgs() : line_len(DEFAULT_LINE_LEN) + { log_file[0] = '\0'; origin_file[0] = '\0'; @@ -731,10 +717,10 @@ enum ExitLevel { }; struct ExitStatus { - ExitLevel level; + ExitLevel level = EXIT_OK; char notice[1024]; - ExitStatus() : level(EXIT_OK) { memset(notice, 0, sizeof(notice)); } + ExitStatus() { memset(notice, 0, sizeof(notice)); } void set(ExitLevel l, const char *n = nullptr) { @@ -753,7 +739,7 @@ struct ExitStatus { } void - append(const std::string s) + append(const std::string &s) { ink_strlcat(notice, s.c_str(), sizeof(notice)); } @@ -1121,7 +1107,7 @@ update_codes(OriginStats *stat, int code, int size) inline void update_methods(OriginStats *stat, int method, int size) { - // We're so loppsided on GETs, so makes most sense to test 'out of order'. + // We're so lopsided on GETs, so makes most sense to test 'out of order'. switch (method) { case METHOD_GET: update_counter(stat->methods.get, size); @@ -1828,7 +1814,7 @@ process_file(int in_fd, off_t offset, unsigned max_age) unsigned second_read_size = sizeof(LogBufferHeader) - first_read_size; nread = read(in_fd, &buffer[first_read_size], second_read_size); if (!nread || EOF == nread) { - Debug("logstats", "Second read of header failed (attemped %d bytes at offset %d, got nothing), errno=%d.", second_read_size, + Debug("logstats", "Second read of header failed (attempted %d bytes at offset %d, got nothing), errno=%d.", second_read_size, first_read_size, errno); return 1; } @@ -2181,7 +2167,7 @@ print_detail_stats(const OriginStats *stat, bool json, bool concise) if (!json) { std::cout << std::endl << std::endl; - // Protocol familes + // Protocol families format_detail_header("Protocols"); } diff --git a/src/traffic_manager/AddConfigFilesHere.cc b/src/traffic_manager/AddConfigFilesHere.cc index dc816b00903..0ace074e4ea 100644 --- a/src/traffic_manager/AddConfigFilesHere.cc +++ b/src/traffic_manager/AddConfigFilesHere.cc @@ -54,7 +54,7 @@ registerFile(const char *configName, const char *defaultName) // // initializeRegistry() // -// Code to initialze of registry of objects that represent +// Code to initialize of registry of objects that represent // Web Editable configuration files // // thread-safe: NO! - Should only be executed once from the main diff --git a/src/traffic_manager/Makefile.inc b/src/traffic_manager/Makefile.inc index de2d62584ad..19dfb927568 100644 --- a/src/traffic_manager/Makefile.inc +++ b/src/traffic_manager/Makefile.inc @@ -49,7 +49,7 @@ traffic_manager_traffic_manager_LDADD = \ $(top_builddir)/lib/records/librecords_lm.a \ $(top_builddir)/proxy/shared/libdiagsconfig.a \ $(LIBUNWIND_LIBS) \ - @LIBPCRE@ @LIBTCL@ @LIBCAP@ @HWLOC_LIBS@ \ + @LIBPCRE@ @LIBCAP@ @HWLOC_LIBS@ \ @YAMLCPP_LIBS@ -lm diff --git a/src/traffic_manager/traffic_manager.cc b/src/traffic_manager/traffic_manager.cc index d167ec81f7c..f7d3835afdc 100644 --- a/src/traffic_manager/traffic_manager.cc +++ b/src/traffic_manager/traffic_manager.cc @@ -84,6 +84,7 @@ static inkcoreapi DiagsConfig *diagsConfig; static char debug_tags[1024] = ""; static char action_tags[1024] = ""; static int proxy_off = false; +static int listen_off = false; static char bind_stdout[512] = ""; static char bind_stderr[512] = ""; @@ -142,7 +143,7 @@ rotateLogs() if (kill(tspid, SIGUSR2) != 0) { mgmt_log("Could not send SIGUSR2 to TS: %s", strerror(errno)); } else { - mgmt_log("Succesfully sent SIGUSR2 to TS!"); + mgmt_log("Successfully sent SIGUSR2 to TS!"); } } } @@ -308,7 +309,7 @@ initSignalHandlers() // Block the delivery of any signals we are not catching // - // except for SIGALARM since we use it + // except for SIGALRM since we use it // to break out of deadlock on semaphore // we share with the proxy // @@ -491,6 +492,7 @@ main(int argc, const char **argv) ArgumentDescription argument_descriptions[] = { {"proxyOff", '-', "Disable proxy", "F", &proxy_off, nullptr, nullptr}, + {"listenOff", '-', "Disable traffic manager listen to proxy ports", "F", &listen_off, nullptr, nullptr}, {"path", '-', "Path to the management socket", "S*", &mgmt_path, nullptr, nullptr}, {"recordsConf", '-', "Path to records.config", "S*", &recs_conf, nullptr, nullptr}, {"tsArgs", '-', "Additional arguments for traffic_server", "S*", &tsArgs, nullptr, nullptr}, @@ -546,7 +548,7 @@ main(int argc, const char **argv) init_dirs(); // setup critical directories, needs LibRecords - if (RecGetRecordString("proxy.config.admin.user_id", userToRunAs, sizeof(userToRunAs)) != TS_ERR_OKAY || + if (RecGetRecordString("proxy.config.admin.user_id", userToRunAs, sizeof(userToRunAs)) != REC_ERR_OKAY || strlen(userToRunAs) == 0) { mgmt_fatal(0, "proxy.config.admin.user_id is not set\n"); } @@ -572,7 +574,7 @@ main(int argc, const char **argv) #endif ts_host_res_global_init(); ts_session_protocol_well_known_name_indices_init(); - lmgmt = new LocalManager(proxy_off == false); + lmgmt = new LocalManager(proxy_off == false, listen_off == false); RecLocalInitMessage(); lmgmt->initAlarm(); diff --git a/src/traffic_server/CoreUtils.cc b/src/traffic_server/CoreUtils.cc index ac628b5e68d..0ae7ba39632 100644 --- a/src/traffic_server/CoreUtils.cc +++ b/src/traffic_server/CoreUtils.cc @@ -100,9 +100,9 @@ int program_counter = 0; #if defined(darwin) || defined(freebsd) || defined(solaris) || defined(openbsd) // FIXME: solaris x86 // TODO: Cleanup multiple includes -#include -#include -#include +#include +#include +#include #include "tscore/ink_platform.h" #include "CoreUtils.h" #endif /* darwin || freebsd || solaris */ @@ -116,7 +116,7 @@ int program_counter = 0; bool inTable; FILE *fp; memTable default_memTable = {0, 0, 0}; -DynArray arrayMem(&default_memTable, 0); +std::vector arrayMem(0, default_memTable); HTTPHdrImpl *global_http; HttpSM *last_seen_http_sm = nullptr; @@ -172,39 +172,11 @@ CoreUtils::insert_table(intptr_t vaddr1, intptr_t offset1, intptr_t fsize1) m.fsize = fsize1; #endif - if (arrayMem.length() == 0) { - arrayMem(0); - arrayMem[(intptr_t)0].vaddr = vaddr1; - arrayMem[(intptr_t)0].offset = offset1; - arrayMem[(intptr_t)0].fsize = fsize1; + if (arrayMem.empty()) { + arrayMem.push_back({vaddr1, offset1, fsize1}); } else { - intptr_t index = find_vaddr(vaddr1, arrayMem.length(), 0); - if (index == arrayMem.length()) { - arrayMem(index); - arrayMem[index].vaddr = vaddr1; - arrayMem[index].offset = offset1; - arrayMem[index].fsize = fsize1; - } else if (index == 0) { - arrayMem(arrayMem.length()); - for (intptr_t i = 0; i < arrayMem.length(); i++) { - arrayMem[arrayMem.length() - i - 1].vaddr = arrayMem[arrayMem.length() - i - 2].vaddr; - arrayMem[arrayMem.length() - i - 1].offset = arrayMem[arrayMem.length() - i - 2].offset; - arrayMem[arrayMem.length() - i - 1].fsize = arrayMem[arrayMem.length() - i - 2].fsize; - } - arrayMem[(intptr_t)0].vaddr = vaddr1; - arrayMem[(intptr_t)0].offset = offset1; - arrayMem[(intptr_t)0].fsize = fsize1; - } else { - arrayMem(arrayMem.length()); - for (intptr_t i = 1; i < arrayMem.length() - index; i++) { - arrayMem[arrayMem.length() - i].vaddr = arrayMem[arrayMem.length() - i - 1].vaddr; - arrayMem[arrayMem.length() - i].offset = arrayMem[arrayMem.length() - i - 1].offset; - arrayMem[arrayMem.length() - i].fsize = arrayMem[arrayMem.length() - i - 1].fsize; - } - arrayMem[index].vaddr = vaddr1; - arrayMem[index].offset = offset1; - arrayMem[index].fsize = fsize1; - } + unsigned index = find_vaddr(vaddr1, arrayMem.size(), 0); + arrayMem.insert(arrayMem.begin() + index, {vaddr1, offset1, fsize1}); } } @@ -213,7 +185,7 @@ CoreUtils::insert_table(intptr_t vaddr1, intptr_t offset1, intptr_t fsize1) intptr_t CoreUtils::read_from_core(intptr_t vaddr, intptr_t bytes, char *buf) { - intptr_t index = find_vaddr(vaddr, arrayMem.length(), 0); + intptr_t index = find_vaddr(vaddr, arrayMem.size(), 0); intptr_t vadd = arrayMem[index - 1].vaddr; intptr_t offset = arrayMem[index - 1].offset; intptr_t size = arrayMem[index - 1].fsize; @@ -253,7 +225,7 @@ void CoreUtils::get_base_frame(intptr_t framep, core_stack_state *coress) { // finds vaddress less than framep - intptr_t index = find_vaddr(framep, arrayMem.length(), 0); + intptr_t index = find_vaddr(framep, arrayMem.size(), 0); intptr_t vadd = arrayMem[index - 1].vaddr; intptr_t off = arrayMem[index - 1].offset; intptr_t off2 = std::abs(vadd - framep); @@ -294,7 +266,7 @@ CoreUtils::get_next_frame(core_stack_state *coress) intptr_t i = 0; intptr_t framep = coress->framep; - intptr_t index = find_vaddr(framep, arrayMem.length(), 0); + intptr_t index = find_vaddr(framep, arrayMem.size(), 0); // finds vaddress less than framep intptr_t vadd = arrayMem[index - 1].vaddr; @@ -490,19 +462,14 @@ CoreUtils::load_http_hdr(HTTPHdr *core_hdr, HTTPHdr *live_hdr) { char buf[sizeof(char) * sizeof(HdrHeap)]; // Load HdrHeap chain - HTTPHdr *http_hdr = core_hdr; - HdrHeap *heap = (HdrHeap *)core_hdr->m_heap; - HdrHeap *heap_ptr = (HdrHeap *)http_hdr->m_heap; - intptr_t ptr_heaps = 0; - intptr_t ptr_heap_size = 0; - intptr_t ptr_xl_size = 2; - intptr_t str_size = 0; - intptr_t str_heaps = 0; - MarshalXlate default_MarshalXlate = {nullptr, nullptr, nullptr}; - DynArray ptr_xlation(&default_MarshalXlate, 2); - // MarshalXlate static_table[2]; - // MarshalXlate* ptr_xlation = static_table; - intptr_t used; + HTTPHdr *http_hdr = core_hdr; + HdrHeap *heap = (HdrHeap *)core_hdr->m_heap; + HdrHeap *heap_ptr = (HdrHeap *)http_hdr->m_heap; + intptr_t ptr_heaps = 0; + intptr_t ptr_heap_size = 0; + intptr_t str_size = 0; + intptr_t str_heaps = 0; + std::vector ptr_xlation(2); intptr_t i; intptr_t copy_size; @@ -537,8 +504,8 @@ CoreUtils::load_http_hdr(HTTPHdr *core_hdr, HTTPHdr *live_hdr) ::exit(0); } // Expand ptr xlation table if necessary - if (ptr_heaps >= ptr_xl_size) { - ptr_xlation(ptr_heaps); + if (static_cast(ptr_heaps) >= ptr_xlation.size()) { + ptr_xlation.resize(ptr_heaps + 1); } char *data, *free, *off; @@ -574,7 +541,7 @@ CoreUtils::load_http_hdr(HTTPHdr *core_hdr, HTTPHdr *live_hdr) swizzle_heap->m_ronly_heap[0].m_heap_start = (char *)(intptr_t)swizzle_heap->m_size; // offset swizzle_heap->m_ronly_heap[0].m_ref_count_ptr.m_ptr = nullptr; - for (int i = 1; i < HDR_BUF_RONLY_HEAPS; i++) { + for (unsigned i = 1; i < HDR_BUF_RONLY_HEAPS; ++i) { swizzle_heap->m_ronly_heap[i].m_heap_start = nullptr; } @@ -647,18 +614,18 @@ CoreUtils::load_http_hdr(HTTPHdr *core_hdr, HTTPHdr *live_hdr) } break; case HDR_HEAP_OBJ_HTTP_HEADER: - if (((HTTPHdrImpl *)obj)->marshal(ptr_xlation, ptr_heaps, str_xlation, str_heaps) < 0) { + if (((HTTPHdrImpl *)obj)->marshal(&ptr_xlation[0], ptr_heaps, str_xlation, str_heaps) < 0) { goto Failed; } live_hdr->m_http = (HTTPHdrImpl *)obj; break; case HDR_HEAP_OBJ_FIELD_BLOCK: - if (((MIMEFieldBlockImpl *)obj)->marshal(ptr_xlation, ptr_heaps, str_xlation, str_heaps) < 0) { + if (((MIMEFieldBlockImpl *)obj)->marshal(&ptr_xlation[0], ptr_heaps, str_xlation, str_heaps) < 0) { goto Failed; } break; case HDR_HEAP_OBJ_MIME_HEADER: - if (((MIMEHdrImpl *)obj)->marshal(ptr_xlation, ptr_heaps, str_xlation, str_heaps)) { + if (((MIMEHdrImpl *)obj)->marshal(&ptr_xlation[0], ptr_heaps, str_xlation, str_heaps)) { goto Failed; } break; @@ -680,10 +647,7 @@ CoreUtils::load_http_hdr(HTTPHdr *core_hdr, HTTPHdr *live_hdr) } // Add up the total bytes used - used = ptr_heap_size + str_size + HDR_HEAP_HDR_SIZE; - used = ROUND(used, HDR_PTR_SIZE); - - return used; + return HdrHeapMarshalBlocks{ts::round_up(ptr_heap_size + str_size + HDR_HEAP_HDR_SIZE)}; Failed: swizzle_heap->m_magic = HDR_BUF_MAGIC_CORRUPT; diff --git a/src/traffic_server/CoreUtils.h b/src/traffic_server/CoreUtils.h index f4817290523..b3d2a77135b 100644 --- a/src/traffic_server/CoreUtils.h +++ b/src/traffic_server/CoreUtils.h @@ -38,13 +38,12 @@ #include #include #include -#include "tscore/DynArray.h" #define SP_REGNUM 15 /* Contains address of top of stack USP */ #define PC_REGNUM 12 /* Contains program counter EIP */ #define FP_REGNUM 5 /* Virtual frame pointer EBP */ -#define NO_OF_ARGS \ - 10 /* The argument depth upto which we would be looking into \ +#define NO_OF_ARGS \ + 10 /* The argument depth up to which we would be looking into \ the stack */ // contains local and in registers, frame pointer, and stack base @@ -56,14 +55,14 @@ struct core_stack_state { #endif // linux check #if defined(darwin) || defined(freebsd) || defined(solaris) || defined(openbsd) // FIXME: solaris x86 -#include +#include #include -#include -#include -#include +#include +#include +#include -#define NO_OF_ARGS \ - 10 /* The argument depth upto which we would be looking into \ +#define NO_OF_ARGS \ + 10 /* The argument depth up to which we would be looking into \ the stack */ // contains local and in registers, frame pointer, and stack base diff --git a/src/traffic_server/EventName.cc b/src/traffic_server/EventName.cc index e45be296351..1eb9b58ec74 100644 --- a/src/traffic_server/EventName.cc +++ b/src/traffic_server/EventName.cc @@ -26,7 +26,6 @@ #include #include "P_EventSystem.h" -// #include "I_Disk.h" unused #include "I_Cache.h" #include "I_Net.h" #include "I_HostDB.h" diff --git a/src/traffic_server/FetchSM.cc b/src/traffic_server/FetchSM.cc index 436405fa34e..fd953c01fab 100644 --- a/src/traffic_server/FetchSM.cc +++ b/src/traffic_server/FetchSM.cc @@ -376,7 +376,7 @@ FetchSM::get_info_from_buffer(IOBufferReader *reader) info = (char *)ats_malloc(sizeof(char) * (read_avail + 1)); client_response = info; - // To maintain backwards compatability we don't allow chunking when it's not streaming. + // To maintain backwards compatibility we don't allow chunking when it's not streaming. if (!(fetch_flags & TS_FETCH_FLAGS_STREAM) || !check_chunked()) { /* Read the data out of the reader */ while (read_avail > 0) { diff --git a/src/traffic_server/FetchSM.h b/src/traffic_server/FetchSM.h index dc7f89bfc8c..0e57e4ce8b4 100644 --- a/src/traffic_server/FetchSM.h +++ b/src/traffic_server/FetchSM.h @@ -66,7 +66,7 @@ class FetchSM : public Continuation mutex = new_ProxyMutex(); // - // We had dropped response_buffer/respone_reader to avoid unnecessary + // We had dropped response_buffer/response_reader to avoid unnecessary // memory copying. But for the original TSFetchURL() API, PluginVC may // stop adding data to resp_buffer when the pending data in resp_buffer // reach its water_mark. @@ -130,7 +130,7 @@ class FetchSM : public Continuation { return req_reader->read_avail(); } - /// Check if the comma supproting MIME field @a name has @a value in it. + /// Check if the comma supporting MIME field @a name has @a value in it. bool check_for_field_value(const char *name, size_t name_len, char const *value, size_t value_len); bool has_body(); diff --git a/src/traffic_server/HostStatus.cc b/src/traffic_server/HostStatus.cc index 365e2e0f7b7..0facb4cc222 100644 --- a/src/traffic_server/HostStatus.cc +++ b/src/traffic_server/HostStatus.cc @@ -37,16 +37,18 @@ getStatName(std::string &stat_name, const char *name, const char *reason) } } -static void * -mgmt_host_status_up_callback(void *x, char *data, int len) +static void +mgmt_host_status_up_callback(ts::MemSpan span) { MgmtInt op; MgmtMarshallString name; MgmtMarshallInt down_time; MgmtMarshallString reason; std::string reason_stat; + char *data = static_cast(span.data()); + auto len = span.size(); static const MgmtMarshallType fields[] = {MGMT_MARSHALL_INT, MGMT_MARSHALL_STRING, MGMT_MARSHALL_STRING, MGMT_MARSHALL_INT}; - Debug("host_statuses", "%s:%s:%d - data: %s, len: %d\n", __FILE__, __func__, __LINE__, data, len); + Debug("host_statuses", "%s:%s:%d - data: %s, len: %ld\n", __FILE__, __func__, __LINE__, data, len); if (mgmt_message_parse(data, len, fields, countof(fields), &op, &name, &reason, &down_time) == -1) { Error("Plugin message - RPC parsing error - message discarded."); @@ -61,19 +63,20 @@ mgmt_host_status_up_callback(void *x, char *data, int len) } hs.setHostStatus(name, HostStatus_t::HOST_STATUS_UP, down_time, reason); } - return nullptr; } -static void * -mgmt_host_status_down_callback(void *x, char *data, int len) +static void +mgmt_host_status_down_callback(ts::MemSpan span) { MgmtInt op; MgmtMarshallString name; MgmtMarshallInt down_time; MgmtMarshallString reason; std::string reason_stat; + char *data = static_cast(span.data()); + auto len = span.size(); static const MgmtMarshallType fields[] = {MGMT_MARSHALL_INT, MGMT_MARSHALL_STRING, MGMT_MARSHALL_STRING, MGMT_MARSHALL_INT}; - Debug("host_statuses", "%s:%s:%d - data: %s, len: %d\n", __FILE__, __func__, __LINE__, data, len); + Debug("host_statuses", "%s:%s:%d - data: %s, len: %ld\n", __FILE__, __func__, __LINE__, data, len); if (mgmt_message_parse(data, len, fields, countof(fields), &op, &name, &reason, &down_time) == -1) { Error("Plugin message - RPC parsing error - message discarded."); @@ -89,15 +92,42 @@ mgmt_host_status_down_callback(void *x, char *data, int len) } hs.setHostStatus(name, HostStatus_t::HOST_STATUS_DOWN, down_time, reason); } - return nullptr; +} + +static void +handle_record_read(const RecRecord *rec, void *edata) +{ + HostStatus &hs = HostStatus::instance(); + std::string hostname; + std::string reason; + + if (rec) { + // parse the hostname from the stat name + char *s = const_cast(rec->name); + // 1st move the pointer past the stat prefix. + s += strlen(stat_prefix.c_str()); + hostname = s; + // parse the reason from the stat name. + reason = hostname.substr(hostname.find('_')); + reason.erase(0, 1); + // erase the reason tag + hostname.erase(hostname.find('_')); + + // if the data loaded from stats indicates that the host was down, + // then update the state so that the host remains down until + // specifically marked up using traffic_ctl. + if (rec->data.rec_int == 0 && Reasons::validReason(reason.c_str())) { + hs.setHostStatus(hostname.c_str(), HOST_STATUS_DOWN, 0, reason.c_str()); + } + } } HostStatus::HostStatus() { ink_rwlock_init(&host_status_rwlock); ink_rwlock_init(&host_statids_rwlock); - pmgmt->registerMgmtCallback(MGMT_EVENT_HOST_STATUS_UP, mgmt_host_status_up_callback, nullptr); - pmgmt->registerMgmtCallback(MGMT_EVENT_HOST_STATUS_DOWN, mgmt_host_status_down_callback, nullptr); + pmgmt->registerMgmtCallback(MGMT_EVENT_HOST_STATUS_UP, &mgmt_host_status_up_callback); + pmgmt->registerMgmtCallback(MGMT_EVENT_HOST_STATUS_DOWN, &mgmt_host_status_down_callback); host_status_rsb = RecAllocateRawStatBlock((int)TS_MAX_API_STATS); } @@ -111,6 +141,14 @@ HostStatus::~HostStatus() ink_rwlock_destroy(&host_statids_rwlock); } +void +HostStatus::loadHostStatusFromStats() +{ + if (RecLookupMatchingRecords(RECT_ALL, stat_prefix.c_str(), handle_record_read, nullptr) != REC_ERR_OKAY) { + Error("[HostStatus] - While loading HostStatus stats, there was an Error reading HostStatus stats."); + } +} + void HostStatus::setHostStatus(const char *name, HostStatus_t status, const unsigned int down_time, const char *reason) { @@ -118,6 +156,10 @@ HostStatus::setHostStatus(const char *name, HostStatus_t status, const unsigned getStatName(reason_stat, name, reason); + if (getHostStatId(reason_stat.c_str()) == -1) { + createHostStat(name); + } + int stat_id = getHostStatId(reason_stat.c_str()); // update the stats @@ -197,7 +239,7 @@ HostStatus::createHostStat(const char *name) std::string reason_stat; getStatName(reason_stat, name, i); if (hosts_stats_ids.find(reason_stat) == hosts_stats_ids.end()) { - RecRegisterRawStat(host_status_rsb, RECT_PROCESS, (reason_stat).c_str(), RECD_INT, RECP_NON_PERSISTENT, (int)next_stat_id, + RecRegisterRawStat(host_status_rsb, RECT_PROCESS, (reason_stat).c_str(), RECD_INT, RECP_PERSISTENT, (int)next_stat_id, RecRawStatSyncSum); RecSetRawStatCount(host_status_rsb, next_stat_id, 1); RecSetRawStatSum(host_status_rsb, next_stat_id, 1); @@ -210,7 +252,6 @@ HostStatus::createHostStat(const char *name) } } ink_rwlock_unlock(&host_statids_rwlock); - setHostStatus(name, HostStatus_t::HOST_STATUS_UP, 0, nullptr); } int diff --git a/src/traffic_server/InkAPI.cc b/src/traffic_server/InkAPI.cc index fc18b721c08..f2a2bd59e7b 100644 --- a/src/traffic_server/InkAPI.cc +++ b/src/traffic_server/InkAPI.cc @@ -95,10 +95,6 @@ extern AppVersionInfo appVersionInfo; static int api_rsb_index; static RecRawStatBlock *api_rsb; -static std::type_info const &TYPE_INFO_MGMT_INT = typeid(MgmtInt); -static std::type_info const &TYPE_INFO_MGMT_BYTE = typeid(MgmtByte); -static std::type_info const &TYPE_INFO_MGMT_FLOAT = typeid(MgmtFloat); - /** Reservation for a user arg. */ struct UserArg { @@ -1295,7 +1291,12 @@ APIHook::invoke(int event, void *edata) ink_assert(!"not reached"); } } - return m_cont->dispatchEvent(event, edata); + MUTEX_TRY_LOCK(lock, m_cont->mutex, this_ethread()); + if (!lock.is_locked()) { + // If we cannot get the lock, the caller needs to restructure to handle rescheduling + ink_release_assert(0); + } + return m_cont->handleEvent(event, edata); } APIHook * @@ -1772,28 +1773,28 @@ TShrtime() //////////////////////////////////////////////////////////////////// const char * -TSInstallDirGet(void) +TSInstallDirGet() { static std::string prefix = Layout::get()->prefix; return prefix.c_str(); } const char * -TSConfigDirGet(void) +TSConfigDirGet() { static std::string sysconfdir = RecConfigReadConfigDir(); return sysconfdir.c_str(); } const char * -TSRuntimeDirGet(void) +TSRuntimeDirGet() { static std::string runtimedir = RecConfigReadRuntimeDir(); return runtimedir.c_str(); } const char * -TSTrafficServerVersionGet(void) +TSTrafficServerVersionGet() { return traffic_server_version; } @@ -1815,7 +1816,7 @@ TSTrafficServerVersionGetPatch() } const char * -TSPluginDirGet(void) +TSPluginDirGet() { static std::string path = RecConfigReadPluginDir(); return path.c_str(); @@ -1957,7 +1958,7 @@ TSHandleMLocRelease(TSMBuffer bufp, TSMLoc parent, TSMLoc mloc) // TSMBuffer: pointers to HdrHeapSDKHandle objects TSMBuffer -TSMBufferCreate(void) +TSMBufferCreate() { TSMBuffer bufp; HdrHeapSDKHandle *new_heap = new HdrHeapSDKHandle; @@ -2451,7 +2452,7 @@ TSIpStringToAddr(const char *str, size_t str_len, sockaddr *addr) /**************/ TSMimeParser -TSMimeParserCreate(void) +TSMimeParserCreate() { TSMimeParser parser = reinterpret_cast(ats_malloc(sizeof(MIMEParser))); @@ -2675,7 +2676,7 @@ TSMimeFieldValueGet(TSMBuffer /* bufp ATS_UNUSED */, TSMLoc field_obj, int idx, if (idx >= 0) { return mime_field_value_get_comma_val(handle->field_ptr, value_len_ptr, idx); } else { - return mime_field_value_get(handle->field_ptr, value_len_ptr); + return handle->field_ptr->value_get(value_len_ptr); } } @@ -3126,8 +3127,8 @@ TSMimeHdrFieldNameGet(TSMBuffer bufp, TSMLoc hdr, TSMLoc field, int *length) sdk_assert(sdk_sanity_check_field_handle(field, hdr) == TS_SUCCESS); sdk_assert(sdk_sanity_check_null_ptr((void *)length) == TS_SUCCESS); - MIMEFieldSDKHandle *handle = (MIMEFieldSDKHandle *)field; - return mime_field_name_get(handle->field_ptr, length); + MIMEFieldSDKHandle *handle = reinterpret_cast(field); + return handle->field_ptr->name_get(length); } TSReturnCode @@ -3509,7 +3510,7 @@ TSMimeHdrFieldValueDateInsert(TSMBuffer bufp, TSMLoc hdr, TSMLoc field, time_t v char tmp[33]; int len = mime_format_date(tmp, value); - // idx ignored, overwrite all exisiting values + // idx ignored, overwrite all existing values // (void)TSMimeFieldValueInsert(bufp, field_obj, tmp, len, idx); (void)TSMimeFieldValueSet(bufp, field, -1, tmp, len); return TS_SUCCESS; @@ -3542,7 +3543,7 @@ TSMimeHdrFieldValueDelete(TSMBuffer bufp, TSMLoc hdr, TSMLoc field, int idx) /* HttpParser */ /**************/ TSHttpParser -TSHttpParserCreate(void) +TSHttpParserCreate() { TSHttpParser parser = reinterpret_cast(ats_malloc(sizeof(HTTPParser))); http_parser_init((HTTPParser *)parser); @@ -4034,7 +4035,7 @@ sdk_sanity_check_cachekey(TSCacheKey key) } TSCacheKey -TSCacheKeyCreate(void) +TSCacheKeyCreate() { TSCacheKey key = (TSCacheKey) new CacheInfo(); @@ -4244,7 +4245,7 @@ TSCacheHttpInfoDestroy(TSCacheHttpInfo infop) } TSCacheHttpInfo -TSCacheHttpInfoCreate(void) +TSCacheHttpInfoCreate() { CacheHTTPInfo *info = new CacheHTTPInfo; info->create(); @@ -4390,16 +4391,45 @@ TSContDataGet(TSCont contp) } TSAction -TSContSchedule(TSCont contp, ink_hrtime timeout, TSThreadPool tp) +TSContSchedule(TSCont contp, TSHRTime timeout) { sdk_assert(sdk_sanity_check_iocore_structure(contp) == TS_SUCCESS); FORCE_PLUGIN_SCOPED_MUTEX(contp); - INKContInternal *i = (INKContInternal *)contp; + INKContInternal *i = reinterpret_cast(contp); + + if (ink_atomic_increment(static_cast(&i->m_event_count), 1) < 0) { + ink_assert(!"not reached"); + } + + EThread *eth = i->getThreadAffinity(); + if (eth == nullptr) { + return nullptr; + } + TSAction action; + if (timeout == 0) { + action = reinterpret_cast(eth->schedule_imm(i)); + } else { + action = reinterpret_cast(eth->schedule_in(i, HRTIME_MSECONDS(timeout))); + } - if (ink_atomic_increment((int *)&i->m_event_count, 1) < 0) { + /* This is a hack. Should be handled in ink_types */ + action = (TSAction)((uintptr_t)action | 0x1); + return action; +} + +TSAction +TSContScheduleOnPool(TSCont contp, TSHRTime timeout, TSThreadPool tp) +{ + sdk_assert(sdk_sanity_check_iocore_structure(contp) == TS_SUCCESS); + + FORCE_PLUGIN_SCOPED_MUTEX(contp); + + INKContInternal *i = reinterpret_cast(contp); + + if (ink_atomic_increment(static_cast(&i->m_event_count), 1) < 0) { ink_assert(!"not reached"); } @@ -4407,7 +4437,6 @@ TSContSchedule(TSCont contp, ink_hrtime timeout, TSThreadPool tp) switch (tp) { case TS_THREAD_POOL_NET: - case TS_THREAD_POOL_DEFAULT: etype = ET_NET; break; case TS_THREAD_POOL_TASK: @@ -4430,28 +4459,85 @@ TSContSchedule(TSCont contp, ink_hrtime timeout, TSThreadPool tp) break; } + TSAction action; if (timeout == 0) { action = reinterpret_cast(eventProcessor.schedule_imm(i, etype)); } else { action = reinterpret_cast(eventProcessor.schedule_in(i, HRTIME_MSECONDS(timeout), etype)); } - /* This is a hack. SHould be handled in ink_types */ + /* This is a hack. Should be handled in ink_types */ action = (TSAction)((uintptr_t)action | 0x1); return action; } TSAction -TSContScheduleEvery(TSCont contp, ink_hrtime every, TSThreadPool tp) +TSContScheduleOnThread(TSCont contp, TSHRTime timeout, TSEventThread ethread) { + ink_release_assert(ethread != nullptr); + sdk_assert(sdk_sanity_check_iocore_structure(contp) == TS_SUCCESS); FORCE_PLUGIN_SCOPED_MUTEX(contp); - INKContInternal *i = (INKContInternal *)contp; + INKContInternal *i = reinterpret_cast(contp); + + if (ink_atomic_increment(static_cast(&i->m_event_count), 1) < 0) { + ink_assert(!"not reached"); + } + + EThread *eth = reinterpret_cast(ethread); + if (i->getThreadAffinity() == nullptr) { + i->setThreadAffinity(eth); + } + TSAction action; + if (timeout == 0) { + action = reinterpret_cast(eth->schedule_imm(i)); + } else { + action = reinterpret_cast(eth->schedule_in(i, HRTIME_MSECONDS(timeout))); + } - if (ink_atomic_increment((int *)&i->m_event_count, 1) < 0) { + /* This is a hack. Should be handled in ink_types */ + action = (TSAction)((uintptr_t)action | 0x1); + return action; +} + +TSAction +TSContScheduleEvery(TSCont contp, TSHRTime every /* millisecs */) +{ + sdk_assert(sdk_sanity_check_iocore_structure(contp) == TS_SUCCESS); + + FORCE_PLUGIN_SCOPED_MUTEX(contp); + + INKContInternal *i = reinterpret_cast(contp); + + if (ink_atomic_increment(static_cast(&i->m_event_count), 1) < 0) { + ink_assert(!"not reached"); + } + + EThread *eth = i->getThreadAffinity(); + if (eth == nullptr) { + return nullptr; + } + + TSAction action = reinterpret_cast(eth->schedule_every(i, HRTIME_MSECONDS(every))); + + /* This is a hack. Should be handled in ink_types */ + action = (TSAction)((uintptr_t)action | 0x1); + return action; +} + +TSAction +TSContScheduleEveryOnPool(TSCont contp, TSHRTime every, TSThreadPool tp) +{ + sdk_assert(sdk_sanity_check_iocore_structure(contp) == TS_SUCCESS); + + FORCE_PLUGIN_SCOPED_MUTEX(contp); + + INKContInternal *i = reinterpret_cast(contp); + + if (ink_atomic_increment(static_cast(&i->m_event_count), 1) < 0) { ink_assert(!"not reached"); } @@ -4459,7 +4545,6 @@ TSContScheduleEvery(TSCont contp, ink_hrtime every, TSThreadPool tp) switch (tp) { case TS_THREAD_POOL_NET: - case TS_THREAD_POOL_DEFAULT: etype = ET_NET; break; case TS_THREAD_POOL_TASK: @@ -4470,15 +4555,84 @@ TSContScheduleEvery(TSCont contp, ink_hrtime every, TSThreadPool tp) break; } - action = reinterpret_cast(eventProcessor.schedule_every(i, HRTIME_MSECONDS(every), etype)); + TSAction action = reinterpret_cast(eventProcessor.schedule_every(i, HRTIME_MSECONDS(every), etype)); + + /* This is a hack. Should be handled in ink_types */ + action = (TSAction)((uintptr_t)action | 0x1); + return action; +} + +TSAction +TSContScheduleEveryOnThread(TSCont contp, TSHRTime every /* millisecs */, TSEventThread ethread) +{ + ink_release_assert(ethread != nullptr); + + sdk_assert(sdk_sanity_check_iocore_structure(contp) == TS_SUCCESS); + + FORCE_PLUGIN_SCOPED_MUTEX(contp); + + INKContInternal *i = reinterpret_cast(contp); + + if (ink_atomic_increment(static_cast(&i->m_event_count), 1) < 0) { + ink_assert(!"not reached"); + } + + EThread *eth = reinterpret_cast(ethread); + if (i->getThreadAffinity() == nullptr) { + i->setThreadAffinity(eth); + } + + TSAction action = reinterpret_cast(eth->schedule_every(i, HRTIME_MSECONDS(every))); - /* This is a hack. SHould be handled in ink_types */ + /* This is a hack. Should be handled in ink_types */ action = (TSAction)((uintptr_t)action | 0x1); return action; } +TSReturnCode +TSContThreadAffinitySet(TSCont contp, TSEventThread ethread) +{ + ink_release_assert(ethread != nullptr); + + sdk_assert(sdk_sanity_check_iocore_structure(contp) == TS_SUCCESS); + + FORCE_PLUGIN_SCOPED_MUTEX(contp); + + INKContInternal *i = reinterpret_cast(contp); + EThread *thread_affinity = reinterpret_cast(ethread); + + if (i->setThreadAffinity(thread_affinity)) { + return TS_SUCCESS; + } + return TS_ERROR; +} + +TSEventThread +TSContThreadAffinityGet(TSCont contp) +{ + sdk_assert(sdk_sanity_check_iocore_structure(contp) == TS_SUCCESS); + + FORCE_PLUGIN_SCOPED_MUTEX(contp); + + INKContInternal *i = reinterpret_cast(contp); + + return reinterpret_cast(i->getThreadAffinity()); +} + +void +TSContThreadAffinityClear(TSCont contp) +{ + sdk_assert(sdk_sanity_check_iocore_structure(contp) == TS_SUCCESS); + + FORCE_PLUGIN_SCOPED_MUTEX(contp); + + INKContInternal *i = reinterpret_cast(contp); + + i->clearThreadAffinity(); +} + TSAction -TSHttpSchedule(TSCont contp, TSHttpTxn txnp, ink_hrtime timeout) +TSHttpSchedule(TSCont contp, TSHttpTxn txnp, TSHRTime timeout) { sdk_assert(sdk_sanity_check_iocore_structure(contp) == TS_SUCCESS); @@ -4510,7 +4664,12 @@ int TSContCall(TSCont contp, TSEvent event, void *edata) { Continuation *c = (Continuation *)contp; - return c->dispatchEvent((int)event, edata); + MUTEX_TRY_LOCK(lock, c->mutex, this_ethread()); + if (!lock.is_locked()) { + // If we cannot get the lock, the caller needs to restructure to handle rescheduling + ink_release_assert(0); + } + return c->handleEvent((int)event, edata); } TSMutex @@ -4533,8 +4692,8 @@ TSHttpHookAdd(TSHttpHookID id, TSCont contp) icontp = reinterpret_cast(contp); - if (id >= TS_SSL_FIRST_HOOK && id <= TS_SSL_LAST_HOOK) { - TSSslHookInternalID internalId = static_cast(id - TS_SSL_FIRST_HOOK); + TSSslHookInternalID internalId{id}; + if (internalId.is_in_bounds()) { ssl_hooks->append(internalId, icontp); } else { // Follow through the regular HTTP hook framework http_global_hooks->append(id, icontp); @@ -4874,9 +5033,10 @@ TSHttpTxnCachedRespGet(TSHttpTxn txnp, TSMBuffer *bufp, TSMLoc *obj) HdrHeapSDKHandle **handle = &(sm->t_state.cache_resp_hdr_heap_handle); if (*handle == nullptr) { - *handle = (HdrHeapSDKHandle *)sm->t_state.arena.alloc(sizeof(HdrHeapSDKHandle)); - (*handle)->m_heap = cached_hdr->m_heap; + *handle = (HdrHeapSDKHandle *)sm->t_state.arena.alloc(sizeof(HdrHeapSDKHandle)); } + // Always reset the m_heap to make sure the heap is not stale + (*handle)->m_heap = cached_hdr->m_heap; *(reinterpret_cast(bufp)) = *handle; *obj = reinterpret_cast(cached_hdr->m_http); @@ -5104,57 +5264,6 @@ TSHttpTxnCacheLookupUrlSet(TSHttpTxn txnp, TSMBuffer bufp, TSMLoc obj) return TS_SUCCESS; } -/* - * TSHttpTxnRedirectRequest is very odd. It is only in experimental.h. - * It is not used in any checked in code. We should probably remove this. - * SKH 1/15/2015 - */ -TSReturnCode -TSHttpTxnRedirectRequest(TSHttpTxn txnp, TSMBuffer bufp, TSMLoc url_loc) -{ - sdk_assert(sdk_sanity_check_txn(txnp) == TS_SUCCESS); - sdk_assert(sdk_sanity_check_mbuffer(bufp) == TS_SUCCESS); - sdk_assert(sdk_sanity_check_url_handle(url_loc) == TS_SUCCESS); - - URL u, *o_url, *r_url, *client_url; - HttpSM *sm = (HttpSM *)txnp; - HttpTransact::State *s = &(sm->t_state); - - u.m_heap = ((HdrHeapSDKHandle *)bufp)->m_heap; - u.m_url_impl = (URLImpl *)url_loc; - if (!u.valid()) { - return TS_ERROR; - } - - client_url = s->hdr_info.client_request.url_get(); - if (!(client_url->valid())) { - return TS_ERROR; - } - - s->redirect_info.redirect_in_process = true; - o_url = &(s->redirect_info.original_url); - if (!o_url->valid()) { - o_url->create(nullptr); - o_url->copy(client_url); - } - client_url->copy(&u); - - r_url = &(s->redirect_info.redirect_url); - if (!r_url->valid()) { - r_url->create(nullptr); - } - r_url->copy(&u); - - s->hdr_info.server_request.destroy(); - - s->request_sent_time = 0; - s->response_received_time = 0; - s->cache_info.write_lock_state = HttpTransact::CACHE_WL_INIT; - s->next_action = HttpTransact::SM_ACTION_REDIRECT_READ; - - return TS_SUCCESS; -} - /** * timeout is in msec * overrides as proxy.config.http.transaction_active_timeout_out @@ -5806,13 +5915,14 @@ TSHttpTxnReenable(TSHttpTxn txnp, TSEvent event) // If this function is being executed on a thread created by the API // which is DEDICATED, the continuation needs to be called back on a // REGULAR thread. - if (eth == nullptr || eth->tt != REGULAR) { + if (eth == nullptr || eth->tt != REGULAR || !eth->is_event_type(ET_NET)) { eventProcessor.schedule_imm(new TSHttpSMCallback(sm, event), ET_NET); } else { MUTEX_TRY_LOCK(trylock, sm->mutex, eth); if (!trylock.is_locked()) { eventProcessor.schedule_imm(new TSHttpSMCallback(sm, event), ET_NET); } else { + ink_assert(eth->is_event_type(ET_NET)); sm->state_api_callback((int)event, nullptr); } } @@ -6254,7 +6364,7 @@ TSHttpTxnLookingUpTypeGet(TSHttpTxn txnp) } int -TSHttpCurrentClientConnectionsGet(void) +TSHttpCurrentClientConnectionsGet() { int64_t S; @@ -6263,7 +6373,7 @@ TSHttpCurrentClientConnectionsGet(void) } int -TSHttpCurrentActiveClientConnectionsGet(void) +TSHttpCurrentActiveClientConnectionsGet() { int64_t S; @@ -6272,7 +6382,7 @@ TSHttpCurrentActiveClientConnectionsGet(void) } int -TSHttpCurrentIdleClientConnectionsGet(void) +TSHttpCurrentIdleClientConnectionsGet() { int64_t total = 0; int64_t active = 0; @@ -6288,7 +6398,7 @@ TSHttpCurrentIdleClientConnectionsGet(void) } int -TSHttpCurrentCacheConnectionsGet(void) +TSHttpCurrentCacheConnectionsGet() { int64_t S; @@ -6297,7 +6407,7 @@ TSHttpCurrentCacheConnectionsGet(void) } int -TSHttpCurrentServerConnectionsGet(void) +TSHttpCurrentServerConnectionsGet() { int64_t S; @@ -6445,7 +6555,7 @@ TSActionCancel(TSAction actionp) Action *a; INKContInternal *i; - /* This is a hack. SHould be handled in ink_types */ + /* This is a hack. Should be handled in ink_types */ if ((uintptr_t)actionp & 0x1) { a = (Action *)((uintptr_t)actionp - 1); i = (INKContInternal *)a->continuation; @@ -6858,7 +6968,6 @@ extern bool ssl_register_protocol(const char *, Continuation *); extern bool ssl_unregister_protocol(const char *, Continuation *); TSReturnCode -#if TS_USE_TLS_NPN TSNetAcceptNamedProtocol(TSCont contp, const char *protocol) { sdk_assert(protocol != nullptr); @@ -6872,12 +6981,6 @@ TSNetAcceptNamedProtocol(TSCont contp, const char *protocol) return TS_SUCCESS; } -#else /* TS_USE_TLS_NPN */ -TSNetAcceptNamedProtocol(TSCont, const char *) -{ - return TS_ERROR; -} -#endif /* TS_USE_TLS_NPN */ /* DNS Lookups */ TSAction @@ -7154,10 +7257,9 @@ TSTextLogObjectCreate(const char *filename, int mode, TSTextLogObject *new_objec return TS_ERROR; } - TextLogObject *tlog = - new TextLogObject(filename, Log::config->logfile_dir, (bool)mode & TS_LOG_MODE_ADD_TIMESTAMP, nullptr, - Log::config->rolling_enabled, Log::config->collation_preproc_threads, Log::config->rolling_interval_sec, - Log::config->rolling_offset_hr, Log::config->rolling_size_mb); + TextLogObject *tlog = new TextLogObject( + filename, Log::config->logfile_dir, (bool)mode & TS_LOG_MODE_ADD_TIMESTAMP, nullptr, Log::config->rolling_enabled, + Log::config->preproc_threads, Log::config->rolling_interval_sec, Log::config->rolling_offset_hr, Log::config->rolling_size_mb); if (tlog == nullptr) { *new_object = nullptr; return TS_ERROR; @@ -7352,7 +7454,7 @@ TSMatcherExtractIPRange(char *match_str, sockaddr *addr1, sockaddr *addr2) } TSMatcherLine -TSMatcherLineCreate(void) +TSMatcherLineCreate() { return reinterpret_cast(ats_malloc(sizeof(matcher_line))); } @@ -7405,7 +7507,7 @@ TSMgmtConfigIntSet(const char *var_name, TSMgmtInt value) // tell manager to set the configuration; note that this is not // transactional (e.g. we return control to the plugin before the - // value is commited to disk by the manager) + // value is committed to disk by the manager) RecSignalManager(MGMT_SIGNAL_PLUGIN_SET_CONFIG, buffer); return TS_SUCCESS; @@ -7837,50 +7939,63 @@ TSSkipRemappingSet(TSHttpTxn txnp, int flag) * to this API handling, with the rest of the code base using the natural types. */ +/// Unhandled API conversions. +/// Because the code around the specially handled types still uses this in the default case, +/// it must compile for those cases. To indicate unhandled, return @c nullptr for @a conv. +/// @internal This should be a temporary state, eventually the other cases should be handled +/// via specializations here. +/// @internal C++ note - THIS MUST BE FIRST IN THE DECLARATIONS or it might be falsely used. template inline void * _memberp_to_generic(T *ptr, MgmtConverter const *&conv) { - static const MgmtConverter IntConverter{ - [](void *data) -> MgmtInt { return *static_cast(data); }, - [](void *data, MgmtInt i) -> void { *static_cast(data) = i; }, - nullptr, - nullptr, // float - nullptr, - nullptr // string - }; + conv = nullptr; + return ptr; +} - static const MgmtConverter ByteConverter{ - [](void *data) -> MgmtInt { return *static_cast(data); }, - [](void *data, MgmtInt i) -> void { *static_cast(data) = i; }, - nullptr, - nullptr, // float - nullptr, - nullptr // string - }; +/// API conversion for @c MgmtInt, identify conversion as integer. +inline void * +_memberp_to_generic(MgmtInt *ptr, MgmtConverter const *&conv) +{ + static const MgmtConverter converter([](void *data) -> MgmtInt { return *static_cast(data); }, + [](void *data, MgmtInt i) -> void { *static_cast(data) = i; }); - static const MgmtConverter FloatConverter{ - nullptr, // int - nullptr, - [](void *data) -> MgmtFloat { return *static_cast(data); }, - [](void *data, MgmtFloat f) -> void { *static_cast(data) = f; }, - nullptr, - nullptr // string - }; + conv = &converter; + return ptr; +} - // For now, strings are special. +/// API conversion for @c MgmtByte, handles integer / byte size differences. +inline void * +_memberp_to_generic(MgmtByte *ptr, MgmtConverter const *&conv) +{ + static const MgmtConverter converter{[](void *data) -> MgmtInt { return *static_cast(data); }, + [](void *data, MgmtInt i) -> void { *static_cast(data) = i; }}; - auto type = &typeid(T); - if (*type == TYPE_INFO_MGMT_INT) { - conv = &IntConverter; - } else if (*type == TYPE_INFO_MGMT_BYTE) { - conv = &ByteConverter; - } else if (*type == TYPE_INFO_MGMT_FLOAT) { - conv = &FloatConverter; - } else { - conv = nullptr; - } + conv = &converter; + return ptr; +} +/// API conversion for @c MgmtFloat, identity conversion as float. +inline void * +_memberp_to_generic(MgmtFloat *ptr, MgmtConverter const *&conv) +{ + static const MgmtConverter converter{[](void *data) -> MgmtFloat { return *static_cast(data); }, + [](void *data, MgmtFloat f) -> void { *static_cast(data) = f; }}; + + conv = &converter; + return ptr; +} + +/// API conversion for arbitrary enum. +/// Handle casting to and from the enum type @a E. +template +inline auto +_memberp_to_generic(MgmtFloat *ptr, MgmtConverter const *&conv) -> typename std::enable_if::value, void *>::type +{ + static const MgmtConverter converter{[](void *data) -> MgmtInt { return static_cast(*static_cast(data)); }, + [](void *data, MgmtInt i) -> void { *static_cast(data) = static_cast(i); }}; + + conv = &converter; return ptr; } @@ -8180,11 +8295,17 @@ _conf_to_memberp(TSOverridableConfigKey conf, OverridableHttpConfigParams *overr case TS_CONFIG_HTTP_FORWARD_CONNECT_METHOD: ret = _memberp_to_generic(&overridableHttpConfig->forward_connect_method, conv); break; - case TS_CONFIG_SSL_CERT_FILENAME: - ret = _memberp_to_generic(&overridableHttpConfig->client_cert_filename, conv); + case TS_CONFIG_SSL_CLIENT_VERIFY_SERVER: + ret = _memberp_to_generic(&overridableHttpConfig->ssl_client_verify_server, conv); break; + case TS_CONFIG_SSL_CLIENT_VERIFY_SERVER_POLICY: + case TS_CONFIG_SSL_CLIENT_VERIFY_SERVER_PROPERTIES: + case TS_CONFIG_SSL_CLIENT_SNI_POLICY: + case TS_CONFIG_SSL_CLIENT_CERT_FILENAME: case TS_CONFIG_SSL_CERT_FILEPATH: - ret = _memberp_to_generic(&overridableHttpConfig->client_cert_filepath, conv); + case TS_CONFIG_SSL_CLIENT_PRIVATE_KEY_FILENAME: + case TS_CONFIG_SSL_CLIENT_CA_CERT_FILENAME: + // String, must be handled elsewhere break; case TS_CONFIG_PARENT_FAILURES_UPDATE_HOSTDB: ret = _memberp_to_generic(&overridableHttpConfig->parent_failures_update_hostdb, conv); @@ -8262,11 +8383,11 @@ TSHttpTxnConfigIntSet(TSHttpTxn txnp, TSOverridableConfigKey conf, TSMgmtInt val void *dest = _conf_to_memberp(conf, s->t_state.txn_conf, conv); - if (!dest || !conv->set_int) { + if (!dest || !conv->store_int) { return TS_ERROR; } - conv->set_int(dest, value); + conv->store_int(dest, value); return TS_SUCCESS; } @@ -8281,11 +8402,11 @@ TSHttpTxnConfigIntGet(TSHttpTxn txnp, TSOverridableConfigKey conf, TSMgmtInt *va MgmtConverter const *conv; void *src = _conf_to_memberp(conf, s->t_state.txn_conf, conv); - if (!src || !conv->get_int) { + if (!src || !conv->load_int) { return TS_ERROR; } - *value = conv->get_int(src); + *value = conv->load_int(src); return TS_SUCCESS; } @@ -8302,11 +8423,11 @@ TSHttpTxnConfigFloatSet(TSHttpTxn txnp, TSOverridableConfigKey conf, TSMgmtFloat void *dest = _conf_to_memberp(conf, s->t_state.txn_conf, conv); - if (!dest || !conv->set_float) { + if (!dest || !conv->store_float) { return TS_ERROR; } - conv->set_float(dest, value); + conv->store_float(dest, value); return TS_SUCCESS; } @@ -8320,10 +8441,10 @@ TSHttpTxnConfigFloatGet(TSHttpTxn txnp, TSOverridableConfigKey conf, TSMgmtFloat MgmtConverter const *conv; void *src = _conf_to_memberp(conf, reinterpret_cast(txnp)->t_state.txn_conf, conv); - if (!src || !conv->get_float) { + if (!src || !conv->load_float) { return TS_ERROR; } - *value = conv->get_float(src); + *value = conv->load_float(src); return TS_SUCCESS; } @@ -8369,16 +8490,6 @@ TSHttpTxnConfigStringSet(TSHttpTxn txnp, TSOverridableConfigKey conf, const char s->t_state.txn_conf->body_factory_template_base_len = 0; } break; - case TS_CONFIG_SSL_CERT_FILENAME: - if (value && length > 0) { - s->t_state.txn_conf->client_cert_filename = const_cast(value); - } - break; - case TS_CONFIG_SSL_CERT_FILEPATH: - if (value && length > 0) { - s->t_state.txn_conf->client_cert_filepath = const_cast(value); - } - break; case TS_CONFIG_HTTP_INSERT_FORWARDED: if (value && length > 0) { ts::LocalBufferWriter<1024> error; @@ -8390,11 +8501,44 @@ TSHttpTxnConfigStringSet(TSHttpTxn txnp, TSOverridableConfigKey conf, const char } } break; + case TS_CONFIG_SSL_CLIENT_VERIFY_SERVER_POLICY: + if (value && length > 0) { + s->t_state.txn_conf->ssl_client_verify_server_policy = const_cast(value); + } + break; + case TS_CONFIG_SSL_CLIENT_VERIFY_SERVER_PROPERTIES: + if (value && length > 0) { + s->t_state.txn_conf->ssl_client_verify_server_properties = const_cast(value); + } + break; + case TS_CONFIG_SSL_CLIENT_SNI_POLICY: + if (value && length > 0) { + s->t_state.txn_conf->ssl_client_sni_policy = const_cast(value); + } + break; + case TS_CONFIG_SSL_CLIENT_CERT_FILENAME: + if (value && length > 0) { + s->t_state.txn_conf->ssl_client_cert_filename = const_cast(value); + } + break; + case TS_CONFIG_SSL_CLIENT_PRIVATE_KEY_FILENAME: + if (value && length > 0) { + s->t_state.txn_conf->ssl_client_private_key_filename = const_cast(value); + } + break; + case TS_CONFIG_SSL_CLIENT_CA_CERT_FILENAME: + if (value && length > 0) { + s->t_state.txn_conf->ssl_client_ca_cert_filename = const_cast(value); + } + break; + case TS_CONFIG_SSL_CERT_FILEPATH: + /* noop */ + break; default: { MgmtConverter const *conv; void *dest = _conf_to_memberp(conf, s->t_state.txn_conf, conv); - if (dest != nullptr && conv != nullptr && conv->set_string) { - conv->set_string(dest, std::string_view(value, length)); + if (dest != nullptr && conv != nullptr && conv->store_string) { + conv->store_string(dest, std::string_view(value, length)); } else { return TS_ERROR; } @@ -8430,8 +8574,8 @@ TSHttpTxnConfigStringGet(TSHttpTxn txnp, TSOverridableConfigKey conf, const char default: { MgmtConverter const *conv; void *src = _conf_to_memberp(conf, sm->t_state.txn_conf, conv); - if (src != nullptr && conv != nullptr && conv->get_string) { - auto sv = conv->get_string(src); + if (src != nullptr && conv != nullptr && conv->load_string) { + auto sv = conv->load_string(src); *value = sv.data(); *length = sv.size(); } else { @@ -8464,7 +8608,6 @@ static const std::unordered_map(SSLCreateServerContext(config)); -#ifdef TS_USE_TLS_OCSP +#if TS_USE_TLS_OCSP if (ret && SSLConfigParams::ssl_ocsp_enabled && cert && certname) { if (SSL_CTX_set_tlsext_status_cb(reinterpret_cast(ret), ssl_callback_ocsp_stapling)) { - if (!ssl_stapling_init_cert(reinterpret_cast(ret), reinterpret_cast(cert), certname)) { + if (!ssl_stapling_init_cert(reinterpret_cast(ret), reinterpret_cast(cert), certname, rsp_file)) { Warning("failed to configure SSL_CTX for OCSP Stapling info for certificate at %s", (const char *)certname); } } @@ -9032,7 +9183,7 @@ TSSslSessionRemove(const TSSslSessionID *session_id) // APIs for managing and using UUIDs. TSUuid -TSUuidCreate(void) +TSUuidCreate() { ATSUuid *uuid = new ATSUuid(); return (TSUuid)uuid; @@ -9072,7 +9223,7 @@ TSUuidInitialize(TSUuid uuid, TSUuidVersion v) } TSUuid -TSProcessUuidGet(void) +TSProcessUuidGet() { Machine *machine = Machine::instance(); return (TSUuid)(&machine->uuid); diff --git a/src/traffic_server/InkAPITest.cc b/src/traffic_server/InkAPITest.cc index 886a176813f..00daa08250f 100644 --- a/src/traffic_server/InkAPITest.cc +++ b/src/traffic_server/InkAPITest.cc @@ -160,7 +160,7 @@ static char *generate_response(const char *request); static int get_request_id(TSHttpTxn txnp); /* client side */ -static ClientTxn *synclient_txn_create(void); +static ClientTxn *synclient_txn_create(); static int synclient_txn_delete(ClientTxn *txn); static void synclient_txn_close(ClientTxn *txn); static int synclient_txn_send_request(ClientTxn *txn, char *request); @@ -438,7 +438,7 @@ generate_response(const char *request) break; } } else { - /* Didin't recognize a testcase request. send the default response */ + /* Didn't recognize a testcase request. send the default response */ snprintf(response, RESPONSE_MAX_SIZE + 1, HTTP_RESPONSE_DEFAULT_FORMAT, test_case); } @@ -503,7 +503,7 @@ get_response_id(TSHttpTxn txnp) ////////////////////////////////////////////////////////////////////////////// static ClientTxn * -synclient_txn_create(void) +synclient_txn_create() { const HttpProxyPort *proxy_port; @@ -1202,7 +1202,7 @@ SDK_RPRINT(RegressionTest *t, const char *api_name, const char *testcase_name, i REGRESSION_TEST_INPROGRESS REGRESSION_TEST_FAILED REGRESSION_TEST_NOT_RUN - Note: pstatus is polled and can be used for asynchroneous tests. + Note: pstatus is polled and can be used for asynchronous tests. */ @@ -1595,7 +1595,7 @@ REGRESSION_TEST(SDK_API_TSPortDescriptor)(RegressionTest *test, int /* atype ATS // (OBJECT_SIZE/2, then OBJECT_SIZE-100 and finally OBJECT_SIZE) // - read object from the cache // - remove it from the cache -// - try to read it (should faild) +// - try to read it (should fail) #define OBJECT_SIZE 100000 // size of the object we'll write/read/remove in cache @@ -1882,7 +1882,7 @@ cache_handler(TSCont contp, TSEvent event, void *data) // now waiting for 100ms to make sure the key is // written in directory remove the content - TSContSchedule(contp, 100, TS_THREAD_POOL_DEFAULT); + TSContScheduleOnPool(contp, 100, TS_THREAD_POOL_NET); } return 1; @@ -2463,7 +2463,7 @@ REGRESSION_TEST(SDK_API_TSActionCancel)(RegressionTest *test, int /* atype ATS_U TSMutex cont_mutex = TSMutexCreate(); TSCont contp = TSContCreate(action_cancel_handler, cont_mutex); - TSAction actionp = TSContSchedule(contp, 10000, TS_THREAD_POOL_DEFAULT); + TSAction actionp = TSContScheduleOnPool(contp, 10000, TS_THREAD_POOL_NET); TSMutexLock(cont_mutex); if (TSActionDone(actionp)) { @@ -2475,7 +2475,7 @@ REGRESSION_TEST(SDK_API_TSActionCancel)(RegressionTest *test, int /* atype ATS_U } TSMutexUnlock(cont_mutex); - TSContSchedule(contp, 0, TS_THREAD_POOL_DEFAULT); + TSContScheduleOnPool(contp, 0, TS_THREAD_POOL_NET); } ////////////////////////////////////////////// @@ -2594,7 +2594,7 @@ REGRESSION_TEST(SDK_API_TSContDataGet)(RegressionTest *test, int /* atype ATS_UN TSContDataSet(contp, (void *)my_data); - TSContSchedule(contp, 0, TS_THREAD_POOL_DEFAULT); + TSContScheduleOnPool(contp, 0, TS_THREAD_POOL_NET); } ////////////////////////////////////////////// @@ -2621,7 +2621,7 @@ REGRESSION_TEST(SDK_API_TSContMutexGet)(RegressionTest *test, int /* atype ATS_U SDK_RPRINT(test, "TSContMutexGet", "TestCase1", TC_PASS, "ok"); test_passed = true; } else { - SDK_RPRINT(test, "TSContMutexGet", "TestCase1", TC_FAIL, "Continutation's mutex corrupted"); + SDK_RPRINT(test, "TSContMutexGet", "TestCase1", TC_FAIL, "Continuation's mutex corrupted"); } // Status of the whole test @@ -2633,7 +2633,7 @@ REGRESSION_TEST(SDK_API_TSContMutexGet)(RegressionTest *test, int /* atype ATS_U ////////////////////////////////////////////// // SDK_API_TSCont // -// Unit Test for API: TSContSchedule +// Unit Test for API: TSContScheduleOnPool ////////////////////////////////////////////// // this is needed for asynchronous APIs @@ -2649,15 +2649,15 @@ cont_schedule_handler(TSCont contp, TSEvent event, void * /* edata ATS_UNUSED */ { if (event == TS_EVENT_IMMEDIATE) { // Test Case 1 - SDK_RPRINT(SDK_ContSchedule_test, "TSContSchedule", "TestCase1", TC_PASS, "ok"); + SDK_RPRINT(SDK_ContSchedule_test, "TSContScheduleOnPool", "TestCase1", TC_PASS, "ok"); tc1_count++; } else if (event == TS_EVENT_TIMEOUT) { // Test Case 2 - SDK_RPRINT(SDK_ContSchedule_test, "TSContSchedule", "TestCase2", TC_PASS, "ok"); + SDK_RPRINT(SDK_ContSchedule_test, "TSContScheduleOnPool", "TestCase2", TC_PASS, "ok"); tc2_count++; } else { // If we receive a bad event, it's a failure - SDK_RPRINT(SDK_ContSchedule_test, "TSContSchedule", "TestCase1|2", TC_FAIL, "received unexpected event number %d", event); + SDK_RPRINT(SDK_ContSchedule_test, "TSContScheduleOnPool", "TestCase1|2", TC_FAIL, "received unexpected event number %d", event); *SDK_ContSchedule_pstatus = REGRESSION_TEST_FAILED; return 0; } @@ -3005,7 +3005,7 @@ REGRESSION_TEST(SDK_API_TSIOBufferBlockNext)(RegressionTest *test, int /* atype TSIOBufferReader readerp = TSIOBufferReaderAlloc(bufp); TSIOBufferBlock blockp = TSIOBufferReaderStart(readerp); - // TODO: This is probaby not the best of regression tests right now ... + // TODO: This is probably not the best of regression tests right now ... // Note that this assumes block size is > sizeof(int) bytes. if (TSIOBufferBlockNext(blockp) == nullptr) { SDK_RPRINT(test, "TSIOBufferBlockNext", "TestCase1", TC_PASS, "ok"); @@ -3035,10 +3035,10 @@ REGRESSION_TEST(SDK_API_TSContSchedule)(RegressionTest *test, int /* atype ATS_U TSCont contp2 = TSContCreate(cont_schedule_handler, TSMutexCreate()); // Test Case 1: schedule immediate - TSContSchedule(contp, 0, TS_THREAD_POOL_DEFAULT); + TSContScheduleOnPool(contp, 0, TS_THREAD_POOL_NET); // Test Case 2: schedule in 10ms - TSContSchedule(contp2, 10, TS_THREAD_POOL_DEFAULT); + TSContScheduleOnPool(contp2, 10, TS_THREAD_POOL_NET); } ////////////////////////////////////////////////////////////////////////////// @@ -3507,7 +3507,7 @@ mytest_handler(TSCont contp, TSEvent event, void *data) case TS_EVENT_TIMEOUT: /* Browser still waiting the response ? */ if (test->browser->status == REQUEST_INPROGRESS) { - TSContSchedule(contp, 25, TS_THREAD_POOL_DEFAULT); + TSContScheduleOnPool(contp, 25, TS_THREAD_POOL_NET); } /* Browser got the response. test is over. clean up */ else { @@ -3600,7 +3600,7 @@ EXCLUSIVE_REGRESSION_TEST(SDK_API_HttpHookAdd)(RegressionTest *test, int /* atyp /* Wait until transaction is done */ if (socktest->browser->status == REQUEST_INPROGRESS) { - TSContSchedule(cont, 25, TS_THREAD_POOL_DEFAULT); + TSContScheduleOnPool(cont, 25, TS_THREAD_POOL_NET); } return; @@ -3836,7 +3836,7 @@ REGRESSION_TEST(SDK_API_TSUrl)(RegressionTest *test, int /* atype ATS_UNUSED */, SDK_RPRINT(test, "TSUrlPasswordSet", "TestCase1", TC_FAIL, "Returned TS_ERROR"); } else { password_get = TSUrlPasswordGet(bufp1, url_loc1, &length); - if (((password_get == nullptr) && (password == nullptr)) || (strncmp(password_get, password, length) == 0)) { + if ((password_get == nullptr) || (strncmp(password_get, password, length) == 0)) { SDK_RPRINT(test, "TSUrlPasswordSet&Get", "TestCase1", TC_PASS, "ok"); test_passed_password = true; } else { @@ -4225,7 +4225,7 @@ REGRESSION_TEST(SDK_API_TSHttpHdr)(RegressionTest *test, int /* atype ATS_UNUSED SDK_RPRINT(test, "TSHttpHdrUrlSet&Get", "TestCase1", TC_FAIL, "TSHttpHdrUrlSet returns TS_ERROR"); } else { if (TSHttpHdrUrlGet(bufp1, hdr_loc1, &url_loc_Get) != TS_SUCCESS) { - SDK_RPRINT(test, "TSHttpHdrUrlSet&Get", "TestCase1", TC_FAIL, "TSHttpHdrUrlGet retuns TS_ERROR"); + SDK_RPRINT(test, "TSHttpHdrUrlSet&Get", "TestCase1", TC_FAIL, "TSHttpHdrUrlGet returns TS_ERROR"); } else { if (url_loc == url_loc_Get) { SDK_RPRINT(test, "TSHttpHdrUrlSet&Get", "TestCase1", TC_PASS, "ok"); @@ -6361,7 +6361,7 @@ log_test_handler(TSCont contp, TSEvent event, void * /* edata ATS_UNUSED */) *(data->pstatus) = REGRESSION_TEST_PASSED; SDK_RPRINT(data->test, "TSTextLogObject", "TestCase1", TC_PASS, "ok"); - // figure out the matainfo file for cleanup. + // figure out the metainfo file for cleanup. // code from MetaInfo::_build_name(const char *filename) int i = -1, l = 0; char c; @@ -6443,7 +6443,7 @@ REGRESSION_TEST(SDK_API_TSTextLog)(RegressionTest *test, int /* atype ATS_UNUSED data->log = log; TSContDataSet(log_test_cont, data); - TSContSchedule(log_test_cont, 6000, TS_THREAD_POOL_DEFAULT); + TSContScheduleOnPool(log_test_cont, 6000, TS_THREAD_POOL_NET); return; } @@ -6620,6 +6620,7 @@ typedef enum { ORIG_TS_SSL_FIRST_HOOK, ORIG_TS_VCONN_START_HOOK = ORIG_TS_SSL_FIRST_HOOK, ORIG_TS_VCONN_CLOSE_HOOK, + ORIG_TS_SSL_CLIENT_HELLO_HOOK, ORIG_TS_SSL_SNI_HOOK, ORIG_TS_SSL_SERVERNAME_HOOK, ORIG_TS_SSL_VERIFY_SERVER_HOOK, @@ -6682,7 +6683,7 @@ typedef enum { ORIG_TS_EVENT_HTTP_SSN_CLOSE = 60014, ORIG_TS_EVENT_HTTP_CACHE_LOOKUP_COMPLETE = 60015, - ORIG_TS_EVENT_MGMT_UPDATE = 60100 + ORIG_TS_EVENT_MGMT_UPDATE = 60300 } ORIG_TSEvent; typedef enum { @@ -7000,7 +7001,7 @@ ssn_handler(TSCont contp, TSEvent event, void *edata) case TS_EVENT_TIMEOUT: /* Browser still waiting the response ? */ if (data->browser->status == REQUEST_INPROGRESS) { - TSContSchedule(contp, 25, TS_THREAD_POOL_DEFAULT); + TSContScheduleOnPool(contp, 25, TS_THREAD_POOL_NET); } /* Browser got the response. test is over. clean up */ else { @@ -7085,7 +7086,7 @@ EXCLUSIVE_REGRESSION_TEST(SDK_API_HttpSsn)(RegressionTest *test, int /* atype AT /* Wait until transaction is done */ if (socktest->browser->status == REQUEST_INPROGRESS) { - TSContSchedule(cont, 25, TS_THREAD_POOL_DEFAULT); + TSContScheduleOnPool(cont, 25, TS_THREAD_POOL_NET); } return; @@ -7244,13 +7245,13 @@ parent_proxy_handler(TSCont contp, TSEvent event, void *edata) if (ptest->configured) { // If we are still in progress, reschedule. rprintf(ptest->regtest, "waiting for response\n"); - TSContSchedule(contp, 100, TS_THREAD_POOL_DEFAULT); + TSContScheduleOnPool(contp, 100, TS_THREAD_POOL_NET); break; } if (!ptest->parent_routing_enabled()) { rprintf(ptest->regtest, "waiting for configuration\n"); - TSContSchedule(contp, 100, TS_THREAD_POOL_DEFAULT); + TSContScheduleOnPool(contp, 100, TS_THREAD_POOL_NET); break; } @@ -7330,7 +7331,7 @@ EXCLUSIVE_REGRESSION_TEST(SDK_API_HttpParentProxySet_Fail)(RegressionTest *test, ptest->os = synserver_create(SYNSERVER_LISTEN_PORT, TSContCreate(synserver_vc_refuse, TSMutexCreate())); synserver_start(ptest->os); - TSContSchedule(cont, 25, TS_THREAD_POOL_DEFAULT); + TSContScheduleOnPool(cont, 25, TS_THREAD_POOL_NET); } EXCLUSIVE_REGRESSION_TEST(SDK_API_HttpParentProxySet_Success)(RegressionTest *test, int level, int *pstatus) @@ -7362,7 +7363,7 @@ EXCLUSIVE_REGRESSION_TEST(SDK_API_HttpParentProxySet_Success)(RegressionTest *te ptest->os = synserver_create(SYNSERVER_LISTEN_PORT, TSContCreate(synserver_vc_accept, TSMutexCreate())); synserver_start(ptest->os); - TSContSchedule(cont, 25, TS_THREAD_POOL_DEFAULT); + TSContScheduleOnPool(cont, 25, TS_THREAD_POOL_NET); } ///////////////////////////////////////////////////// @@ -7485,12 +7486,12 @@ cache_hook_handler(TSCont contp, TSEvent event, void *edata) /* Browser still waiting the response ? */ if (data->first_time == true) { if (data->browser1->status == REQUEST_INPROGRESS) { - TSContSchedule(contp, 25, TS_THREAD_POOL_DEFAULT); + TSContScheduleOnPool(contp, 25, TS_THREAD_POOL_NET); return 0; } } else { if (data->browser2->status == REQUEST_INPROGRESS) { - TSContSchedule(contp, 25, TS_THREAD_POOL_DEFAULT); + TSContScheduleOnPool(contp, 25, TS_THREAD_POOL_NET); return 0; } } @@ -7508,7 +7509,7 @@ cache_hook_handler(TSCont contp, TSEvent event, void *edata) /* Send another similar client request */ synclient_txn_send_request(data->browser2, data->request); ink_assert(REQUEST_INPROGRESS == data->browser2->status); - TSContSchedule(contp, 25, TS_THREAD_POOL_DEFAULT); + TSContScheduleOnPool(contp, 25, TS_THREAD_POOL_NET); return 0; } @@ -7577,7 +7578,7 @@ EXCLUSIVE_REGRESSION_TEST(SDK_API_HttpTxnCache)(RegressionTest *test, int /* aty synclient_txn_send_request(socktest->browser1, socktest->request); /* Wait until transaction is done */ - TSContSchedule(cont, 25, TS_THREAD_POOL_DEFAULT); + TSContScheduleOnPool(cont, 25, TS_THREAD_POOL_NET); return; } @@ -7951,37 +7952,37 @@ transform_hook_handler(TSCont contp, TSEvent event, void *edata) switch (data->req_no) { case 1: if (data->browser1->status == REQUEST_INPROGRESS) { - TSContSchedule(contp, 25, TS_THREAD_POOL_DEFAULT); + TSContScheduleOnPool(contp, 25, TS_THREAD_POOL_NET); return 0; } data->req_no++; Debug(UTDBG_TAG "_transform", "Running Browser 2"); synclient_txn_send_request(data->browser2, data->request2); - TSContSchedule(contp, 25, TS_THREAD_POOL_DEFAULT); + TSContScheduleOnPool(contp, 25, TS_THREAD_POOL_NET); return 0; case 2: if (data->browser2->status == REQUEST_INPROGRESS) { - TSContSchedule(contp, 25, TS_THREAD_POOL_DEFAULT); + TSContScheduleOnPool(contp, 25, TS_THREAD_POOL_NET); return 0; } data->req_no++; Debug(UTDBG_TAG "_transform", "Running Browser 3"); synclient_txn_send_request(data->browser3, data->request1); - TSContSchedule(contp, 25, TS_THREAD_POOL_DEFAULT); + TSContScheduleOnPool(contp, 25, TS_THREAD_POOL_NET); return 0; case 3: if (data->browser3->status == REQUEST_INPROGRESS) { - TSContSchedule(contp, 25, TS_THREAD_POOL_DEFAULT); + TSContScheduleOnPool(contp, 25, TS_THREAD_POOL_NET); return 0; } data->req_no++; Debug(UTDBG_TAG "_transform", "Running Browser 4"); synclient_txn_send_request(data->browser4, data->request2); - TSContSchedule(contp, 25, TS_THREAD_POOL_DEFAULT); + TSContScheduleOnPool(contp, 25, TS_THREAD_POOL_NET); return 0; case 4: if (data->browser4->status == REQUEST_INPROGRESS) { - TSContSchedule(contp, 25, TS_THREAD_POOL_DEFAULT); + TSContScheduleOnPool(contp, 25, TS_THREAD_POOL_NET); return 0; } synserver_delete(data->os); @@ -8120,7 +8121,7 @@ EXCLUSIVE_REGRESSION_TEST(SDK_API_HttpTxnTransform)(RegressionTest *test, int /* // synclient_txn_send_request(socktest->browser2, socktest->request2); /* Wait until transaction is done */ - TSContSchedule(cont, 25, TS_THREAD_POOL_DEFAULT); + TSContScheduleOnPool(cont, 25, TS_THREAD_POOL_NET); return; } @@ -8232,12 +8233,12 @@ altinfo_hook_handler(TSCont contp, TSEvent event, void *edata) /* Browser still waiting the response ? */ if (data->first_time == true) { if ((data->browser1->status == REQUEST_INPROGRESS) || (data->browser2->status == REQUEST_INPROGRESS)) { - TSContSchedule(contp, 25, TS_THREAD_POOL_DEFAULT); + TSContScheduleOnPool(contp, 25, TS_THREAD_POOL_NET); return 0; } } else { if (data->browser3->status == REQUEST_INPROGRESS) { - TSContSchedule(contp, 25, TS_THREAD_POOL_DEFAULT); + TSContScheduleOnPool(contp, 25, TS_THREAD_POOL_NET); return 0; } } @@ -8257,7 +8258,7 @@ altinfo_hook_handler(TSCont contp, TSEvent event, void *edata) /* Register to HTTP hooks that are called in case of alternate selection */ TSHttpHookAdd(TS_HTTP_SELECT_ALT_HOOK, contp); - TSContSchedule(contp, 25, TS_THREAD_POOL_DEFAULT); + TSContScheduleOnPool(contp, 25, TS_THREAD_POOL_NET); return 0; } @@ -8336,7 +8337,7 @@ EXCLUSIVE_REGRESSION_TEST(SDK_API_HttpAltInfo)(RegressionTest *test, int /* atyp synclient_txn_send_request(socktest->browser2, socktest->request2); /* Wait until transaction is done */ - TSContSchedule(cont, 25, TS_THREAD_POOL_DEFAULT); + TSContScheduleOnPool(cont, 25, TS_THREAD_POOL_NET); return; } @@ -8362,7 +8363,7 @@ EXCLUSIVE_REGRESSION_TEST(SDK_API_HttpAltInfo)(RegressionTest *test, int /* atyp ////////////////////////////////////////////// // Important: we create servers listening on different port than the default one -// to make sure our synthetix servers are called +// to make sure our synthetic servers are called #define TEST_CASE_CONNECT_ID1 9 // TSHttpTxnIntercept #define TEST_CASE_CONNECT_ID2 10 // TSHttpTxnServerIntercept @@ -8425,7 +8426,7 @@ cont_test_handler(TSCont contp, TSEvent event, void *edata) /* Browser still waiting the response ? */ if (data->browser->status == REQUEST_INPROGRESS) { TSDebug(UTDBG_TAG, "Browser still waiting response..."); - TSContSchedule(contp, 25, TS_THREAD_POOL_DEFAULT); + TSContScheduleOnPool(contp, 25, TS_THREAD_POOL_NET); } /* Browser got the response */ else { @@ -8517,7 +8518,7 @@ EXCLUSIVE_REGRESSION_TEST(SDK_API_TSHttpConnectIntercept)(RegressionTest *test, synclient_txn_send_request_to_vc(data->browser, data->request, data->vc); /* Wait until transaction is done */ - TSContSchedule(cont_test, 25, TS_THREAD_POOL_DEFAULT); + TSContScheduleOnPool(cont_test, 25, TS_THREAD_POOL_NET); return; } @@ -8526,7 +8527,7 @@ EXCLUSIVE_REGRESSION_TEST(SDK_API_TSHttpConnectServerIntercept)(RegressionTest * { *pstatus = REGRESSION_TEST_INPROGRESS; - TSDebug(UTDBG_TAG, "Starting test TSHttpConnectServerintercept"); + TSDebug(UTDBG_TAG, "Starting test TSHttpConnectServerIntercept"); TSCont cont_test = TSContCreate(cont_test_handler, TSMutexCreate()); ConnectTestData *data = (ConnectTestData *)TSmalloc(sizeof(ConnectTestData)); @@ -8556,7 +8557,7 @@ EXCLUSIVE_REGRESSION_TEST(SDK_API_TSHttpConnectServerIntercept)(RegressionTest * synclient_txn_send_request_to_vc(data->browser, data->request, data->vc); /* Wait until transaction is done */ - TSContSchedule(cont_test, 25, TS_THREAD_POOL_DEFAULT); + TSContScheduleOnPool(cont_test, 25, TS_THREAD_POOL_NET); return; } @@ -8689,7 +8690,13 @@ std::array SDK_Overridable_Configs = { "proxy.config.http.request_buffer_enabled", "proxy.config.http.allow_half_open", OutboundConnTrack::CONFIG_VAR_MAX, - OutboundConnTrack::CONFIG_VAR_MATCH}}; + OutboundConnTrack::CONFIG_VAR_MATCH, + "proxy.config.ssl.client.verify.server", + "proxy.config.ssl.client.verify.server.policy", + "proxy.config.ssl.client.verify.server.properties", + "proxy.config.ssl.client.sni_policy", + "proxy.config.ssl.client.private_key.filename", + "proxy.config.ssl.client.CA.cert.filename"}}; REGRESSION_TEST(SDK_API_OVERRIDABLE_CONFIGS)(RegressionTest *test, int /* atype ATS_UNUSED */, int *pstatus) { @@ -8978,7 +8985,7 @@ REGRESSION_TEST(SDK_API_DEBUG_NAME_LOOKUPS)(RegressionTest *test, int /* atype A bool success = true; const char state_name[] = "INACTIVE_TIMEOUT"; const char hook_name[] = "TS_HTTP_READ_RESPONSE_HDR_HOOK"; - const char event_name[] = "VC_EVENT_IMMEDIATE"; + const char event_name[] = "TS_EVENT_IMMEDIATE"; const char *str; *pstatus = REGRESSION_TEST_INPROGRESS; @@ -9002,9 +9009,9 @@ REGRESSION_TEST(SDK_API_DEBUG_NAME_LOOKUPS)(RegressionTest *test, int /* atype A } str = TSHttpEventNameLookup(TS_EVENT_IMMEDIATE); - if ((strlen(str) != strlen(event_name) || strcmp(str, event_name))) { - SDK_RPRINT(test, "TSHttpEventNameLookup", "TestCase1", TC_FAIL, "Failed on %d, expected %s, got %s", TS_EVENT_IMMEDIATE, - hook_name, str); + if (strstr(str, event_name) == nullptr) { + SDK_RPRINT(test, "TSHttpEventNameLookup", "TestCase1", TC_FAIL, "Failed on %d, expected %s to be within %s", TS_EVENT_IMMEDIATE, + event_name, str); success = false; } else { SDK_RPRINT(test, "TSHttpEventNameLookup", "TestCase1", TC_PASS, "ok"); @@ -9066,7 +9073,7 @@ REGRESSION_TEST(SDK_API_UUID)(RegressionTest *test, int /* atype ATS_UNUSED */, // Test TSUuidCreate if (!(uuid = TSUuidCreate())) { - SDK_RPRINT(test, "TSUuidCreate", "TestCase1", TC_FAIL, "Failed to crete a UUID object"); + SDK_RPRINT(test, "TSUuidCreate", "TestCase1", TC_FAIL, "Failed to create a UUID object"); *pstatus = REGRESSION_TEST_FAILED; return; } else { @@ -9168,7 +9175,7 @@ REGRESSION_TEST(SDK_API_TSSslServerContextCreate)(RegressionTest *test, int leve TSSslContext ctx; // See TS-4769: TSSslServerContextCreate always returns null. - ctx = TSSslServerContextCreate(nullptr, nullptr); + ctx = TSSslServerContextCreate(nullptr, nullptr, nullptr); *pstatus = ctx ? REGRESSION_TEST_PASSED : REGRESSION_TEST_FAILED; TSSslContextDestroy(ctx); diff --git a/src/traffic_server/InkIOCoreAPI.cc b/src/traffic_server/InkIOCoreAPI.cc index d3dc3bf35b9..d8c3fda4452 100644 --- a/src/traffic_server/InkIOCoreAPI.cc +++ b/src/traffic_server/InkIOCoreAPI.cc @@ -54,7 +54,7 @@ sdk_sanity_check_mutex(TSMutex mutex) return TS_ERROR; } - ProxyMutex *mutexp = (ProxyMutex *)mutex; + ProxyMutex *mutexp = reinterpret_cast(mutex); if (mutexp->refcount() < 0) { return TS_ERROR; @@ -227,6 +227,12 @@ TSThreadSelf(void) return ithread; } +TSEventThread +TSEventThreadSelf(void) +{ + return reinterpret_cast(this_event_thread()); +} + //////////////////////////////////////////////////////////////////// // // Mutexes @@ -236,6 +242,7 @@ TSMutex TSMutexCreate() { ProxyMutex *mutexp = new_ProxyMutex(); + mutexp->refcount_inc(); // TODO: Remove this when allocations can never fail. sdk_assert(sdk_sanity_check_mutex((TSMutex)mutexp) == TS_SUCCESS); @@ -247,9 +254,12 @@ void TSMutexDestroy(TSMutex m) { sdk_assert(sdk_sanity_check_mutex(m) == TS_SUCCESS); - ink_release_assert(((ProxyMutex *)m)->refcount() == 0); - - ((ProxyMutex *)m)->free(); + ProxyMutex *mutexp = reinterpret_cast(m); + // Decrement the refcount added in TSMutexCreate. Delete if this + // was the last ref count + if (mutexp && mutexp->refcount_dec() == 0) { + mutexp->free(); + } } /* The following two APIs are for Into work, actually, APIs of Mutex @@ -286,21 +296,24 @@ void TSMutexLock(TSMutex mutexp) { sdk_assert(sdk_sanity_check_mutex(mutexp) == TS_SUCCESS); - MUTEX_TAKE_LOCK((ProxyMutex *)mutexp, this_ethread()); + Ptr proxy_mutex(reinterpret_cast(mutexp)); + MUTEX_TAKE_LOCK(proxy_mutex, this_ethread()); } TSReturnCode TSMutexLockTry(TSMutex mutexp) { sdk_assert(sdk_sanity_check_mutex(mutexp) == TS_SUCCESS); - return (MUTEX_TAKE_TRY_LOCK((ProxyMutex *)mutexp, this_ethread()) ? TS_SUCCESS : TS_ERROR); + Ptr proxy_mutex(reinterpret_cast(mutexp)); + return (MUTEX_TAKE_TRY_LOCK(proxy_mutex, this_ethread()) ? TS_SUCCESS : TS_ERROR); } void TSMutexUnlock(TSMutex mutexp) { sdk_assert(sdk_sanity_check_mutex(mutexp) == TS_SUCCESS); - MUTEX_UNTAKE_LOCK((ProxyMutex *)mutexp, this_ethread()); + Ptr proxy_mutex(reinterpret_cast(mutexp)); + MUTEX_UNTAKE_LOCK(proxy_mutex, this_ethread()); } /* VIOs */ @@ -403,7 +416,7 @@ TSVIOMutexGet(TSVIO viop) sdk_assert(sdk_sanity_check_iocore_structure(viop) == TS_SUCCESS); VIO *vio = (VIO *)viop; - return (TSMutex)(vio->mutex.get()); + return reinterpret_cast(vio->mutex.get()); } /* High Resolution Time */ diff --git a/src/traffic_server/Makefile.inc b/src/traffic_server/Makefile.inc index 98966c383e8..a18d6cec51d 100644 --- a/src/traffic_server/Makefile.inc +++ b/src/traffic_server/Makefile.inc @@ -64,7 +64,6 @@ traffic_server_traffic_server_LDADD = \ $(top_builddir)/proxy/http/remap/libhttp_remap.a \ $(top_builddir)/proxy/http2/libhttp2.a \ $(top_builddir)/proxy/logging/liblogging.a \ - $(top_builddir)/proxy/logging/liblogcollation.a \ $(top_builddir)/proxy/hdrs/libhdrs.a \ $(top_builddir)/proxy/shared/libdiagsconfig.a \ $(top_builddir)/mgmt/libmgmt_p.la \ @@ -83,7 +82,6 @@ traffic_server_traffic_server_LDADD = \ $(top_builddir)/lib/tsconfig/libtsconfig.la \ @HWLOC_LIBS@ \ @LIBPCRE@ \ - @LIBTCL@ \ @LIBRESOLV@ \ @LIBZ@ \ @LIBLZMA@ \ diff --git a/src/traffic_server/SocksProxy.cc b/src/traffic_server/SocksProxy.cc index 77208ee4669..824e395a75f 100644 --- a/src/traffic_server/SocksProxy.cc +++ b/src/traffic_server/SocksProxy.cc @@ -22,7 +22,7 @@ */ /* - This implements SOCKS server. We intecept the http traffic and send it + This implements SOCKS server. We intercept the http traffic and send it through HTTP. Others are tunneled through directly to the socks server. @@ -403,7 +403,7 @@ SocksProxy::sendResp(bool granted) // In SOCKS 4, IP addr and Dest Port fields are ignored. // In SOCKS 5, IP addr and Dest Port are the ones we use to connect to the // real host. In our case, it does not make sense, since we may not - // connect at all. Set these feilds to zeros. Any socks client which uses + // connect at all. Set these fields to zeros. Any socks client which uses // these breaks caching. buf->reset(); @@ -446,7 +446,7 @@ SocksProxy::setupHttpRequest(unsigned char *p) break; case SOCKS_ATYPE_FQHN: - // This is stored as a zero terminicated string + // This is stored as a zero terminated string a->addr.buf = (unsigned char *)ats_malloc(p[4] + 1); memcpy(a->addr.buf, &p[5], p[4]); a->addr.buf[p[4]] = 0; diff --git a/src/traffic_server/traffic_server.cc b/src/traffic_server/traffic_server.cc index 0eb340299b4..5674f7054df 100644 --- a/src/traffic_server/traffic_server.cc +++ b/src/traffic_server/traffic_server.cc @@ -96,6 +96,7 @@ extern "C" int plock(int); #include "HTTP2.h" #include "tscore/ink_config.h" #include "P_SSLSNI.h" +#include "P_SSLClientUtils.h" #include "tscore/ink_cap.h" @@ -114,13 +115,14 @@ extern "C" int plock(int); static const long MAX_LOGIN = ink_login_name_max(); -static void *mgmt_restart_shutdown_callback(void *, char *, int data_len); -static void *mgmt_drain_callback(void *, char *, int data_len); -static void *mgmt_storage_device_cmd_callback(void *x, char *data, int len); -static void *mgmt_lifecycle_msg_callback(void *x, char *data, int len); +static void mgmt_restart_shutdown_callback(ts::MemSpan); +static void mgmt_drain_callback(ts::MemSpan); +static void mgmt_storage_device_cmd_callback(int cmd, std::string_view const &arg); +static void mgmt_lifecycle_msg_callback(ts::MemSpan); static void init_ssl_ctx_callback(void *ctx, bool server); static void load_ssl_file_callback(const char *ssl_file, unsigned int options); static void load_remap_file_callback(const char *remap_file); +static void task_threads_started_callback(); // We need these two to be accessible somewhere else now int num_of_net_threads = ink_number_of_processors(); @@ -218,8 +220,17 @@ struct AutoStopCont : public Continuation { int mainEvent(int /* event */, Event * /* e */) { + TSSystemState::stop_ssl_handshaking(); + + APIHook *hook = lifecycle_hooks->get(TS_LIFECYCLE_SHUTDOWN_HOOK); + while (hook) { + SCOPED_MUTEX_LOCK(lock, hook->m_cont->mutex, this_ethread()); + hook->invoke(TS_EVENT_LIFECYCLE_SHUTDOWN, nullptr); + hook = hook->next(); + } + pmgmt->stop(); - shutdown_event_system = true; + TSSystemState::shut_down_event_system(); delete this; return EVENT_CONT; } @@ -277,6 +288,7 @@ class SignalContinuation : public Continuation RecInt timeout = 0; if (RecGetRecordInt("proxy.config.stop.shutdown_timeout", &timeout) == REC_ERR_OKAY && timeout) { RecSetRecordInt("proxy.node.config.draining", 1, REC_SOURCE_DEFAULT); + TSSystemState::drain(true); if (!remote_management_flag) { // Close listening sockets here only if TS is running standalone RecInt close_sockets = 0; @@ -381,7 +393,7 @@ class DiagsLogContinuation : public Continuation class MemoryLimit : public Continuation { public: - MemoryLimit() : Continuation(new_ProxyMutex()), _memory_limit(0) + MemoryLimit() : Continuation(new_ProxyMutex()) { memset(&_usage, 0, sizeof(_usage)); SET_HANDLER(&MemoryLimit::periodic); @@ -431,7 +443,7 @@ class MemoryLimit : public Continuation } private: - int64_t _memory_limit; + int64_t _memory_limit = 0; struct rusage _usage; }; @@ -628,7 +640,7 @@ initialize_process_manager() RECP_NON_PERSISTENT); } -#define CMD_ERROR -2 // serious error, exit maintaince mode +#define CMD_ERROR -2 // serious error, exit maintenance mode #define CMD_FAILED -1 // error, but recoverable #define CMD_OK 0 // ok, or minor (user) error #define CMD_HELP 1 // ok, print help @@ -686,11 +698,11 @@ CB_After_Cache_Init() #if TS_ENABLE_FIPS == 0 // Check for cache BC after the cache is initialized and before listen, if possible. - if (cacheProcessor.min_stripe_version.ink_major < CACHE_DB_MAJOR_VERSION) { + if (cacheProcessor.min_stripe_version._major < CACHE_DB_MAJOR_VERSION) { // Versions before 23 need the MMH hash. - if (cacheProcessor.min_stripe_version.ink_major < 23) { + if (cacheProcessor.min_stripe_version._major < 23) { Debug("cache_bc", "Pre 4.0 stripe (cache version %d.%d) found, forcing MMH hash for cache URLs", - cacheProcessor.min_stripe_version.ink_major, cacheProcessor.min_stripe_version.ink_minor); + cacheProcessor.min_stripe_version._major, cacheProcessor.min_stripe_version._minor); URLHashContext::Setting = URLHashContext::MMH; } } @@ -848,7 +860,7 @@ cmd_verify(char * /* cmd ATS_UNUSED */) Layout::get()->update_sysconfdir(conf_dir); } - if (!reloadUrlRewrite()) { + if (!urlRewriteVerify()) { exitStatus |= (1 << 0); fprintf(stderr, "ERROR: Failed to load remap.config, exitStatus %d\n\n", exitStatus); } else { @@ -1155,18 +1167,18 @@ struct ShowStats : public Continuation { #ifdef ENABLE_TIME_TRACE FILE *fp; #endif - int cycle; - int64_t last_cc; - int64_t last_rb; - int64_t last_w; - int64_t last_r; - int64_t last_wb; - int64_t last_nrb; - int64_t last_nw; - int64_t last_nr; - int64_t last_nwb; - int64_t last_p; - int64_t last_o; + int cycle = 0; + int64_t last_cc = 0; + int64_t last_rb = 0; + int64_t last_w = 0; + int64_t last_r = 0; + int64_t last_wb = 0; + int64_t last_nrb = 0; + int64_t last_nw = 0; + int64_t last_nr = 0; + int64_t last_nwb = 0; + int64_t last_p = 0; + int64_t last_o = 0; int mainEvent(int event, Event *e) { @@ -1266,20 +1278,8 @@ struct ShowStats : public Continuation { #endif return EVENT_CONT; } - ShowStats() - : Continuation(nullptr), - cycle(0), - last_cc(0), - last_rb(0), - last_w(0), - last_r(0), - last_wb(0), - last_nrb(0), - last_nw(0), - last_nr(0), - last_nwb(0), - last_p(0), - last_o(0) + ShowStats() : Continuation(nullptr) + { SET_HANDLER(&ShowStats::mainEvent); #ifdef ENABLE_TIME_TRACE @@ -1334,9 +1334,9 @@ init_http_header() #if TS_HAS_TESTS struct RegressionCont : public Continuation { - int initialized; - int waits; - int started; + int initialized = 0; + int waits = 0; + int started = 0; int mainEvent(int event, Event *e) @@ -1359,15 +1359,13 @@ struct RegressionCont : public Continuation { return EVENT_CONT; } + TSSystemState::shut_down_event_system(); fprintf(stderr, "REGRESSION_TEST DONE: %s\n", regression_status_string(res)); ::exit(res == REGRESSION_TEST_PASSED ? 0 : 1); return EVENT_CONT; } - RegressionCont() : Continuation(new_ProxyMutex()), initialized(0), waits(0), started(0) - { - SET_HANDLER(&RegressionCont::mainEvent); - } + RegressionCont() : Continuation(new_ProxyMutex()) { SET_HANDLER(&RegressionCont::mainEvent); } }; static void @@ -1794,13 +1792,15 @@ main(int /* argc ATS_UNUSED */, const char **argv) REC_ReadConfigInteger(thread_max_heartbeat_mseconds, "proxy.config.thread.max_heartbeat_mseconds"); - ink_event_system_init(makeModuleVersion(1, 0, PRIVATE_MODULE_HEADER)); - ink_net_init(makeModuleVersion(1, 0, PRIVATE_MODULE_HEADER)); - ink_aio_init(makeModuleVersion(1, 0, PRIVATE_MODULE_HEADER)); - ink_cache_init(makeModuleVersion(1, 0, PRIVATE_MODULE_HEADER)); - ink_hostdb_init(makeModuleVersion(HOSTDB_MODULE_MAJOR_VERSION, HOSTDB_MODULE_MINOR_VERSION, PRIVATE_MODULE_HEADER)); - ink_dns_init(makeModuleVersion(HOSTDB_MODULE_MAJOR_VERSION, HOSTDB_MODULE_MINOR_VERSION, PRIVATE_MODULE_HEADER)); - ink_split_dns_init(makeModuleVersion(1, 0, PRIVATE_MODULE_HEADER)); + ink_event_system_init(ts::ModuleVersion(1, 0, ts::ModuleVersion::PRIVATE)); + ink_net_init(ts::ModuleVersion(1, 0, ts::ModuleVersion::PRIVATE)); + ink_aio_init(ts::ModuleVersion(1, 0, ts::ModuleVersion::PRIVATE)); + ink_cache_init(ts::ModuleVersion(1, 0, ts::ModuleVersion::PRIVATE)); + ink_hostdb_init( + ts::ModuleVersion(HOSTDB_MODULE_INTERNAL_VERSION._major, HOSTDB_MODULE_INTERNAL_VERSION._minor, ts::ModuleVersion::PRIVATE)); + ink_dns_init( + ts::ModuleVersion(HOSTDB_MODULE_INTERNAL_VERSION._major, HOSTDB_MODULE_INTERNAL_VERSION._minor, ts::ModuleVersion::PRIVATE)); + ink_split_dns_init(ts::ModuleVersion(1, 0, ts::ModuleVersion::PRIVATE)); naVecMutex = new_ProxyMutex(); @@ -1864,8 +1864,9 @@ main(int /* argc ATS_UNUSED */, const char **argv) RecProcessStart(); initCacheControl(); IpAllow::startup(); + HostStatus::instance().loadHostStatusFromStats(); + netProcessor.init_socks(); ParentConfig::startup(); - HostStatus::instance(); #ifdef SPLIT_DNS SplitDNSConfig::startup(); #endif @@ -1955,22 +1956,25 @@ main(int /* argc ATS_UNUSED */, const char **argv) RecConfigWarnIfUnregistered(); // "Task" processor, possibly with its own set of task threads + tasksProcessor.register_event_type(); + eventProcessor.thread_group[ET_TASK]._afterStartCallback = task_threads_started_callback; tasksProcessor.start(num_task_threads, stacksize); if (netProcessor.socks_conf_stuff->accept_enabled) { start_SocksProxy(netProcessor.socks_conf_stuff->accept_port); } - pmgmt->registerMgmtCallback(MGMT_EVENT_SHUTDOWN, mgmt_restart_shutdown_callback, nullptr); - pmgmt->registerMgmtCallback(MGMT_EVENT_RESTART, mgmt_restart_shutdown_callback, nullptr); - pmgmt->registerMgmtCallback(MGMT_EVENT_DRAIN, mgmt_drain_callback, nullptr); + pmgmt->registerMgmtCallback(MGMT_EVENT_SHUTDOWN, &mgmt_restart_shutdown_callback); + pmgmt->registerMgmtCallback(MGMT_EVENT_RESTART, &mgmt_restart_shutdown_callback); + pmgmt->registerMgmtCallback(MGMT_EVENT_DRAIN, &mgmt_drain_callback); // Callback for various storage commands. These all go to the same function so we // pass the event code along so it can do the right thing. We cast that to first // just to be safe because the value is a #define, not a typed value. - pmgmt->registerMgmtCallback(MGMT_EVENT_STORAGE_DEVICE_CMD_OFFLINE, mgmt_storage_device_cmd_callback, - reinterpret_cast(static_cast(MGMT_EVENT_STORAGE_DEVICE_CMD_OFFLINE))); - pmgmt->registerMgmtCallback(MGMT_EVENT_LIFECYCLE_MESSAGE, mgmt_lifecycle_msg_callback, nullptr); + pmgmt->registerMgmtCallback(MGMT_EVENT_STORAGE_DEVICE_CMD_OFFLINE, [](ts::MemSpan span) -> void { + mgmt_storage_device_cmd_callback(MGMT_EVENT_STORAGE_DEVICE_CMD_OFFLINE, std::string_view{span}); + }); + pmgmt->registerMgmtCallback(MGMT_EVENT_LIFECYCLE_MESSAGE, &mgmt_lifecycle_msg_callback); ink_set_thread_name("[TS_MAIN]"); @@ -1993,7 +1997,7 @@ main(int /* argc ATS_UNUSED */, const char **argv) } #endif - while (!shutdown_event_system) { + while (!TSSystemState::is_event_system_shut_down()) { sleep(1); } @@ -2015,42 +2019,37 @@ REGRESSION_TEST(Hdrs)(RegressionTest *t, int atype, int *pstatus) } #endif -static void * -mgmt_restart_shutdown_callback(void *, char *, int /* data_len ATS_UNUSED */) +static void mgmt_restart_shutdown_callback(ts::MemSpan) { sync_cache_dir_on_shutdown(); - return nullptr; } -static void * -mgmt_drain_callback(void *, char *arg, int len) +static void +mgmt_drain_callback(ts::MemSpan span) { - ink_assert(len > 1 && (arg[0] == '0' || arg[0] == '1')); - RecSetRecordInt("proxy.node.config.draining", arg[0] == '1', REC_SOURCE_DEFAULT); - return nullptr; + char *arg = static_cast(span.data()); + TSSystemState::drain(span.size() == 2 && arg[0] == '1'); + RecSetRecordInt("proxy.node.config.draining", TSSystemState::is_draining() ? 1 : 0, REC_SOURCE_DEFAULT); } -static void * -mgmt_storage_device_cmd_callback(void *data, char *arg, int len) +static void +mgmt_storage_device_cmd_callback(int cmd, std::string_view const &arg) { // data is the device name to control - CacheDisk *d = cacheProcessor.find_by_path(arg, len); - // Actual command is in @a data. - intptr_t cmd = reinterpret_cast(data); + CacheDisk *d = cacheProcessor.find_by_path(arg.data(), int(arg.size())); if (d) { switch (cmd) { case MGMT_EVENT_STORAGE_DEVICE_CMD_OFFLINE: - Debug("server", "Marking %.*s offline", len, arg); + Debug("server", "Marking %.*s offline", int(arg.size()), arg.data()); cacheProcessor.mark_storage_offline(d, /* admin */ true); break; } } - return nullptr; } -static void * -mgmt_lifecycle_msg_callback(void *, char *data, int len) +static void +mgmt_lifecycle_msg_callback(ts::MemSpan span) { APIHook *hook = lifecycle_hooks->get(TS_LIFECYCLE_MSG_HOOK); TSPluginMsg msg; @@ -2059,7 +2058,7 @@ mgmt_lifecycle_msg_callback(void *, char *data, int len) MgmtMarshallData payload; static const MgmtMarshallType fields[] = {MGMT_MARSHALL_INT, MGMT_MARSHALL_STRING, MGMT_MARSHALL_DATA}; - if (mgmt_message_parse(data, len, fields, countof(fields), &op, &tag, &payload) == -1) { + if (mgmt_message_parse(span.data(), span.size(), fields, countof(fields), &op, &tag, &payload) == -1) { Error("Plugin message - RPC parsing error - message discarded."); } else { msg.tag = tag; @@ -2071,7 +2070,6 @@ mgmt_lifecycle_msg_callback(void *, char *data, int len) hook = hook->next(); } } - return nullptr; } static void @@ -2098,3 +2096,13 @@ load_remap_file_callback(const char *remap_file) { pmgmt->signalConfigFileChild("remap.config", remap_file, CONFIG_FLAG_UNVERSIONED); } + +static void +task_threads_started_callback() +{ + APIHook *hook = lifecycle_hooks->get(TS_LIFECYCLE_TASK_THREADS_READY_HOOK); + while (hook) { + hook->invoke(TS_EVENT_LIFECYCLE_TASK_THREADS_READY, nullptr); + hook = hook->next(); + } +} diff --git a/src/traffic_top/Makefile.inc b/src/traffic_top/Makefile.inc index d3bced9e5d8..7d0a820fb95 100644 --- a/src/traffic_top/Makefile.inc +++ b/src/traffic_top/Makefile.inc @@ -49,6 +49,6 @@ traffic_top_traffic_top_LDADD = \ $(top_builddir)/src/tscpp/util/libtscpputil.la \ @CURL_LIBS@ \ @CURSES_LIBS@ \ - @LIBTCL@ @HWLOC_LIBS@ + @HWLOC_LIBS@ endif diff --git a/src/traffic_top/stats.h b/src/traffic_top/stats.h index 7dc3d9773c7..1789fd0ed02 100644 --- a/src/traffic_top/stats.h +++ b/src/traffic_top/stats.h @@ -73,7 +73,7 @@ class Stats // set the host size_t start = _url.find(":"); - size_t end = _url.find("/", start + 3); + size_t end = _url.find('/', start + 3); _host = _url.substr(start + 3, end - start - 3); end = _host.find(":"); if (end != string::npos) { @@ -111,12 +111,33 @@ class Stats lookup_table.insert(make_pair("dns_entry", LookupItem("DNS Entry", "proxy.process.hostdb.cache.current_items", 1))); lookup_table.insert(make_pair("dns_hits", LookupItem("DNS Hits", "proxy.process.hostdb.total_hits", 2))); lookup_table.insert(make_pair("dns_lookups", LookupItem("DNS Lookups", "proxy.process.hostdb.total_lookups", 2))); + + // Incoming HTTP/1.1 and HTTP/2 connections - some metrics are HTTP version specific lookup_table.insert(make_pair("client_req", LookupItem("Requests", "proxy.process.http.incoming_requests", 2))); - lookup_table.insert(make_pair("client_conn", LookupItem("New Conn", "proxy.process.http.total_client_connections", 2))); + + // total_client_connections + lookup_table.insert( + make_pair("client_conn_h1", LookupItem("New Conn HTTP/1.x", "proxy.process.http.total_client_connections", 2))); + lookup_table.insert( + make_pair("client_conn_h2", LookupItem("New Conn HTTP/2", "proxy.process.http2.total_client_connections", 2))); + lookup_table.insert(make_pair("client_conn", LookupItem("New Conn", "client_conn_h1", "client_conn_h2", 6))); + + // requests / connections lookup_table.insert(make_pair("client_req_conn", LookupItem("Req/Conn", "client_req", "client_conn", 3))); - lookup_table.insert(make_pair("client_curr_conn", LookupItem("Curr Conn", "proxy.process.http.current_client_connections", 1))); + + // current_client_connections + lookup_table.insert( + make_pair("client_curr_conn_h1", LookupItem("Curr Conn HTTP/1.x", "proxy.process.http.current_client_connections", 1))); lookup_table.insert( - make_pair("client_actv_conn", LookupItem("Active Con", "proxy.process.http.current_active_client_connections", 1))); + make_pair("client_curr_conn_h2", LookupItem("Curr Conn HTTP/2", "proxy.process.http2.current_client_connections", 1))); + lookup_table.insert(make_pair("client_curr_conn", LookupItem("Curr Conn", "client_curr_conn_h1", "client_curr_conn_h2", 6))); + + // current_active_client_connections + lookup_table.insert(make_pair("client_actv_conn_h1", + LookupItem("Active Con HTTP/1.x", "proxy.process.http.current_active_client_connections", 1))); + lookup_table.insert(make_pair("client_actv_conn_h2", + LookupItem("Active Con HTTP/2", "proxy.process.http2.current_active_client_connections", 1))); + lookup_table.insert(make_pair("client_actv_conn", LookupItem("Active Con", "client_actv_conn_h1", "client_actv_conn_h2", 6))); lookup_table.insert(make_pair("server_req", LookupItem("Requests", "proxy.process.http.outgoing_requests", 2))); lookup_table.insert(make_pair("server_conn", LookupItem("New Conn", "proxy.process.http.total_server_connections", 2))); diff --git a/src/traffic_top/traffic_top.cc b/src/traffic_top/traffic_top.cc index 4021cc77d91..ffb4b3dd3aa 100644 --- a/src/traffic_top/traffic_top.cc +++ b/src/traffic_top/traffic_top.cc @@ -57,6 +57,7 @@ #include "tscore/ink_args.h" #include "records/I_RecProcess.h" #include "RecordsConfig.h" +#include "tscore/runroot.h" using namespace std; @@ -246,7 +247,7 @@ help(const string &host, const string &version) attron(A_BOLD); mvprintw(7, 0, "Definitions:"); attroff(A_BOLD); - mvprintw(8, 0, "Fresh => Requests that were servered by fresh entries in cache"); + mvprintw(8, 0, "Fresh => Requests that were served by fresh entries in cache"); mvprintw(9, 0, "Revalidate => Requests that contacted the origin to verify if still valid"); mvprintw(10, 0, "Cold => Requests that were not in cache at all"); mvprintw(11, 0, "Changed => Requests that required entries in cache to be updated"); @@ -407,6 +408,7 @@ main(int argc, const char **argv) process_args(&version, argument_descriptions, countof(argument_descriptions), argv, USAGE); + runroot_handler(argv); Layout::create(); RecProcessInit(RECM_STAND_ALONE, nullptr /* diags */); LibRecordsConfigInit(); diff --git a/src/traffic_via/test_traffic_via b/src/traffic_via/test_traffic_via index fb9fb0347b6..9b3995f659b 100755 --- a/src/traffic_via/test_traffic_via +++ b/src/traffic_via/test_traffic_via @@ -20,11 +20,16 @@ set -e # exit on error TMPDIR=${TMPDIR:-/tmp} tmpfile=$(mktemp "$TMPDIR/via.XXXXXX") -srcdir=$(cd $srcdir && pwd) +if [ ! -z "$srcdir"]; then + srcdir=$(cd $srcdir && pwd) +else + srcdir=$(pwd) +fi find $srcdir/tests -type f | while read f ; do name=$(basename "$f") + echo "testing $name" ./traffic_via "$name" > "$tmpfile" 2>&1 || true diff -u "$tmpfile" "$srcdir/tests/$name" done diff --git a/src/traffic_via/tests/[u c s f p eS;tNc p s ] b/src/traffic_via/tests/[u c s f p eS;tNc p s ] index 95cf7a7d9d5..e662dc0e814 100644 --- a/src/traffic_via/tests/[u c s f p eS;tNc p s ] +++ b/src/traffic_via/tests/[u c s f p eS;tNc p s ] @@ -1,4 +1,4 @@ -Via header is [u c s f p eS;tNc p s ], Length is 24 +Via header is [u c s f p eS;tNc p s ], Length is 22 Via Header Details: Request headers received from client :unknown Result of Traffic Server cache lookup for URL :no cache lookup diff --git a/src/traffic_via/tests/[uIcRs f p eN;t cCHp s ] b/src/traffic_via/tests/[uIcRs f p eN;t cCHp s ] index a5f3238c4b0..0755883b90f 100644 --- a/src/traffic_via/tests/[uIcRs f p eN;t cCHp s ] +++ b/src/traffic_via/tests/[uIcRs f p eN;t cCHp s ] @@ -1,4 +1,4 @@ -Via header is [uIcRs f p eN;t cCHp s ], Length is 24 +Via header is [uIcRs f p eN;t cCHp s ], Length is 22 Via Header Details: Request headers received from client :IMS Result of Traffic Server cache lookup for URL :in cache, fresh Ram hit (a cache "HIT") diff --git a/src/traffic_via/tests/[uIcRs f p eN;t cCNp s ] b/src/traffic_via/tests/[uIcRs f p eN;t cCNp s ] index 6af0055ed2e..6ecfc64517a 100644 --- a/src/traffic_via/tests/[uIcRs f p eN;t cCNp s ] +++ b/src/traffic_via/tests/[uIcRs f p eN;t cCNp s ] @@ -1,4 +1,4 @@ -Via header is [uIcRs f p eN;t cCNp s ], Length is 24 +Via header is [uIcRs f p eN;t cCNp s ], Length is 22 Via Header Details: Request headers received from client :IMS Result of Traffic Server cache lookup for URL :in cache, fresh Ram hit (a cache "HIT") diff --git a/src/traffic_via/tests/[uScMsSf pSeN;t cCMp sS] b/src/traffic_via/tests/[uScMsSf pSeN;t cCMp sS] index 1e494c63175..6ee096a1889 100644 --- a/src/traffic_via/tests/[uScMsSf pSeN;t cCMp sS] +++ b/src/traffic_via/tests/[uScMsSf pSeN;t cCMp sS] @@ -1,4 +1,4 @@ -Via header is [uScMsSf pSeN;t cCMp sS], Length is 24 +Via header is [uScMsSf pSeN;t cCMp sS], Length is 22 Via Header Details: Request headers received from client :simple request (not conditional) Result of Traffic Server cache lookup for URL :miss (a cache "MISS") diff --git a/src/traffic_via/tests/[uScRs f p eN;t cCHp s ] b/src/traffic_via/tests/[uScRs f p eN;t cCHp s ] index d7d394c6e59..ea96fdb54fa 100644 --- a/src/traffic_via/tests/[uScRs f p eN;t cCHp s ] +++ b/src/traffic_via/tests/[uScRs f p eN;t cCHp s ] @@ -1,4 +1,4 @@ -Via header is [uScRs f p eN;t cCHp s ], Length is 24 +Via header is [uScRs f p eN;t cCHp s ], Length is 22 Via Header Details: Request headers received from client :simple request (not conditional) Result of Traffic Server cache lookup for URL :in cache, fresh Ram hit (a cache "HIT") diff --git a/src/traffic_via/tests/long rubbish via code2 b/src/traffic_via/tests/long rubbish via code2 index d3abd3d95ba..37366829fbe 100644 --- a/src/traffic_via/tests/long rubbish via code2 +++ b/src/traffic_via/tests/long rubbish via code2 @@ -12,6 +12,6 @@ traffic_via: Invalid VIA header character: i traffic_via: Invalid VIA header character: a traffic_via: Invalid VIA header character: o traffic_via: Invalid VIA header character: d -Via header is long rubbish via code2, Length is 22 +Via header is [long rubbish via code2], Length is 22 Via Header Details: Error codes (if any) :Invalid sequence diff --git a/src/traffic_via/tests/rubbish b/src/traffic_via/tests/rubbish index 348f372e606..599670db921 100644 --- a/src/traffic_via/tests/rubbish +++ b/src/traffic_via/tests/rubbish @@ -1,4 +1,4 @@ -Via header is rubbish, Length is 7 +Via header is [rubbish], Length is 7 Invalid VIA header. VIA header length should be 6 or 22 characters Valid via header format is [ucsfpe:tcps] diff --git a/src/traffic_via/tests/short b/src/traffic_via/tests/short index c0f098ce3bc..76d85bfb117 100644 --- a/src/traffic_via/tests/short +++ b/src/traffic_via/tests/short @@ -2,5 +2,5 @@ traffic_via: Invalid VIA header character: h traffic_via: Invalid VIA header character: o traffic_via: Invalid VIA header character: r traffic_via: Invalid VIA header character: t -Via header is short, Length is 5 +Via header is [short], Length is 5 Via Header Details: diff --git a/src/traffic_via/traffic_via.cc b/src/traffic_via/traffic_via.cc index b755f2c6916..cab50985205 100644 --- a/src/traffic_via/traffic_via.cc +++ b/src/traffic_via/traffic_via.cc @@ -224,7 +224,6 @@ decodeViaHeader(const char *str) memcpy(Via, str, viaHdrLength); Via[viaHdrLength] = '\0'; // null terminate - printf("Via header is %s, Length is %zu\n", Via, viaHdrLength); // Via header inside square brackets if (Via[0] == '[' && Via[viaHdrLength - 1] == ']') { @@ -233,6 +232,8 @@ decodeViaHeader(const char *str) Via[viaHdrLength] = '\0'; // null terminate the string after trimming } + printf("Via header is [%s], Length is %zu\n", Via, viaHdrLength); + if (viaHdrLength == 5) { Via = strcat(Via, " "); // Add one space character before decoding via header ++viaHdrLength; diff --git a/src/traffic_wccp/wccp_client.cc b/src/traffic_wccp/wccp_client.cc index aca1bc94aad..6f2c691b816 100644 --- a/src/traffic_wccp/wccp_client.cc +++ b/src/traffic_wccp/wccp_client.cc @@ -20,9 +20,9 @@ limitations under the License. */ -#include +#include #include -#include +#include #include #include #include @@ -49,7 +49,7 @@ bool do_daemon = false; static const char USAGE_TEXT[] = "%s\n" "--address IP address to bind.\n" - "--router Booststrap IP address for routers.\n" + "--router Bootstrap IP address for routers.\n" "--service Path to service group definitions.\n" "--debug Print debugging information.\n" "--daemon Run as daemon.\n" diff --git a/src/tscore/Arena.cc b/src/tscore/Arena.cc index 23469e08b4f..1667923d4f5 100644 --- a/src/tscore/Arena.cc +++ b/src/tscore/Arena.cc @@ -133,12 +133,12 @@ Arena::free(void *mem, size_t size) b = m_blocks; while (b->next) { + if (b->m_water_level == ((char *)mem + size)) { + b->m_water_level = (char *)mem; + return; + } b = b->next; } - - if (b->m_water_level == ((char *)mem + size)) { - b->m_water_level = (char *)mem; - } } } diff --git a/src/tscore/ArgParser.cc b/src/tscore/ArgParser.cc index 349eb657d61..b7cb4af0219 100644 --- a/src/tscore/ArgParser.cc +++ b/src/tscore/ArgParser.cc @@ -22,15 +22,23 @@ */ #include "tscore/ArgParser.h" +#include "tscore/ink_file.h" +#include "tscore/I_Version.h" #include #include #include +#include +#include std::string global_usage; std::string parser_program_name; std::string default_command; +// by default return EX_USAGE(64) when usage is called. +// if -h or --help is called specifically, return 0 +int usage_return_code = EX_USAGE; + namespace ts { ArgParser::ArgParser() {} @@ -76,7 +84,7 @@ ArgParser::add_global_usage(std::string const &usage) void ArgParser::help_message(std::string_view err) const { - return _top_level_command.help_message(err); + _top_level_command.help_message(err); } // a graceful way to output help message @@ -103,6 +111,17 @@ ArgParser::Command::help_message(std::string_view err) const if (!_example_usage.empty()) { std::cout << "\nExample Usage: " << _example_usage << std::endl; } + // standard return code + exit(usage_return_code); +} + +void +ArgParser::Command::version_message() const +{ + // unified version message of ATS + AppVersionInfo appVersionInfo; + appVersionInfo.setup(PACKAGE_NAME, _name.c_str(), PACKAGE_VERSION, __DATE__, __TIME__, BUILD_MACHINE, BUILD_PERSON, ""); + ink_fputln(stdout, appVersionInfo.FullVersionInfoStr); exit(0); } @@ -143,7 +162,7 @@ ArgParser::parse(const char **argv) parser_program_name = _argv[0]; Arguments ret; // the parsed arg object to return AP_StrVec args = _argv; - // call the recrusive parse method in Command + // call the recursive parse method in Command if (!_top_level_command.parse(ret, args)) { // deal with default command if (!default_command.empty()) { @@ -154,11 +173,20 @@ ArgParser::parse(const char **argv) }; // if there is anything left, then output usage if (!args.empty()) { - std::string msg = "Unkown command, option or args:"; + std::string msg = "Unknown command, option or args:"; for (const auto &it : args) { msg = msg + " '" + it + "'"; } - _top_level_command.help_message(msg); + // find the correct level to output help message + ArgParser::Command *command = &_top_level_command; + for (unsigned i = 1; i < _argv.size(); i++) { + auto it = command->_subcommand_list.find(_argv[i]); + if (it == command->_subcommand_list.end()) { + break; + } + command = &it->second; + } + command->help_message(msg); } return ret; } @@ -172,7 +200,7 @@ ArgParser::require_commands() void ArgParser::set_error(std::string e) { - _error_msg = e; + _error_msg = std::move(e); } std::string @@ -282,9 +310,9 @@ void ArgParser::Command::output_command(std::ostream &out, std::string const &prefix) const { if (_name != parser_program_name) { - // a nicely formated way to output command usage + // a nicely formatted way to output command usage std::string msg = prefix + _name; - // nicely formated output + // nicely formatted output if (!_description.empty()) { if (INDENT_ONE - static_cast(msg.size()) < 0) { // if the command msg is too long @@ -398,10 +426,14 @@ ArgParser::Command::append_option_data(Arguments &ret, AP_StrVec &args, int inde i -= 1; } } else { + // output version message + if ((args[i] == "--version" || args[i] == "-V") && _option_list.find("--version") != _option_list.end()) { + version_message(); + } // output help message - if (args[i] == "--help" || args[i] == "-h") { + if ((args[i] == "--help" || args[i] == "-h") && _option_list.find("--help") != _option_list.end()) { ArgParser::Command *command = this; - // find the correct level to output help messsage + // find the correct level to output help message for (unsigned i = 1; i < args.size(); i++) { auto it = command->_subcommand_list.find(args[i]); if (it == command->_subcommand_list.end()) { @@ -409,6 +441,7 @@ ArgParser::Command::append_option_data(Arguments &ret, AP_StrVec &args, int inde } command = &it->second; } + usage_return_code = 0; command->help_message(); } // deal with normal --arg val1 val2 ... @@ -594,7 +627,7 @@ std::string const & ArgumentData::at(unsigned index) const { if (index >= _values.size()) { - throw std::out_of_range("argument not fonud at index: " + std::to_string(index)); + throw std::out_of_range("argument not found at index: " + std::to_string(index)); } return _values.at(index); } diff --git a/src/tscore/BaseLogFile.cc b/src/tscore/BaseLogFile.cc index 64968153605..682c32a1a6a 100644 --- a/src/tscore/BaseLogFile.cc +++ b/src/tscore/BaseLogFile.cc @@ -24,7 +24,7 @@ #include "tscore/BaseLogFile.h" /* - * This consturctor creates a BaseLogFile based on a given name. + * This constructor creates a BaseLogFile based on a given name. * This is the most common way BaseLogFiles are created. */ BaseLogFile::BaseLogFile(const char *name) : m_name(ats_strdup(name)) @@ -33,7 +33,7 @@ BaseLogFile::BaseLogFile(const char *name) : m_name(ats_strdup(name)) } /* - * This consturctor creates a BaseLogFile based on a given name. + * This constructor creates a BaseLogFile based on a given name. * Similar to above constructor, but is overloaded with the object signature */ BaseLogFile::BaseLogFile(const char *name, uint64_t sig) : m_name(ats_strdup(name)), m_signature(sig), m_has_signature(true) @@ -46,15 +46,13 @@ BaseLogFile::BaseLogFile(const char *name, uint64_t sig) : m_name(ats_strdup(nam * This copy constructor creates a BaseLogFile based on a given copy. */ BaseLogFile::BaseLogFile(const BaseLogFile ©) - : m_fp(nullptr), - m_start_time(copy.m_start_time), - m_end_time(0L), - m_bytes_written(0), + : m_start_time(copy.m_start_time), + m_name(ats_strdup(copy.m_name)), m_hostname(ats_strdup(copy.m_hostname)), - m_is_regfile(false), + m_is_init(copy.m_is_init), - m_meta_info(nullptr), + m_signature(copy.m_signature), m_has_signature(copy.m_has_signature) { @@ -157,7 +155,7 @@ BaseLogFile::roll(long interval_start, long interval_end) // produce overlapping filenames (the problem is that we have // no easy way of keeping track of the timestamp of the first // transaction - log_log_trace("in BaseLogFile::roll(..), didn't use metadata starttime, used earlist available starttime\n"); + log_log_trace("in BaseLogFile::roll(..), didn't use metadata starttime, used earliest available starttime\n"); if (interval_start == 0) { start = m_start_time; } else { @@ -206,7 +204,7 @@ BaseLogFile::roll(long interval_start, long interval_end) } /* - * The more convienent rolling function. Intended use is for less + * The more convenient rolling function. Intended use is for less * critical logs such as diags.log or traffic.out, since _exact_ * timestamps may be less important * @@ -333,7 +331,7 @@ BaseLogFile::open_file(int perm) } } - // set m_bytes_written to force the rolling based on filesize. + // set m_bytes_written to force the rolling based on file size. m_bytes_written = fseek(m_fp, 0, SEEK_CUR); log_log_trace("BaseLogFile %s is now open (fd=%d)\n", m_name.get(), fileno(m_fp)); @@ -537,7 +535,7 @@ BaseMetaInfo::_write_to_file() // TODO modify this runtime check so that it is not an assertion ink_release_assert(n <= BUF_SIZE); if (write(fd, _buffer, n) == -1) { - log_log_error("Could not write object_signaure\n"); + log_log_error("Could not write object_signature\n"); } log_log_trace("BaseMetaInfo::_write_to_file\n" "\tfilename = %s\n" diff --git a/src/tscore/BufferWriterFormat.cc b/src/tscore/BufferWriterFormat.cc index d5152e6f24c..54eafdf7a29 100644 --- a/src/tscore/BufferWriterFormat.cc +++ b/src/tscore/BufferWriterFormat.cc @@ -23,6 +23,7 @@ #include "tscore/BufferWriter.h" #include "tscore/bwf_std_format.h" +#include "tscore/ink_thread.h" #include #include #include @@ -31,6 +32,7 @@ #include #include #include +#include using namespace std::literals; @@ -97,12 +99,11 @@ BWFSpec::Property::Property() } /// Parse a format specification. -BWFSpec::BWFSpec(TextView fmt) +BWFSpec::BWFSpec(TextView fmt) : _name(fmt.take_prefix_at(':')) { TextView num; // temporary for number parsing. intmax_t n; - _name = fmt.take_prefix_at(':'); // if it's parsable as a number, treat it as an index. n = tv_to_positive_decimal(_name, &num); if (num.size()) { @@ -238,7 +239,7 @@ namespace bw_fmt size_t delta = min - extent; char *base = w.auxBuffer(); // should be first byte of @a lw e.g. lw.data() - avoid const_cast. char *limit = base + lw.capacity(); // first invalid byte. - char *dst; // used to track memory operation targest; + char *dst; // used to track memory operation target; char *last; // track limit of memory operation. size_t d2; switch (spec._align) { @@ -451,7 +452,7 @@ namespace bw_fmt width -= static_cast(n); std::string_view digits{buff + sizeof(buff) - n, n}; - if (spec._align == BWFSpec::Align::SIGN) { // custom for signed case because prefix and digits are seperated. + if (spec._align == BWFSpec::Align::SIGN) { // custom for signed case because prefix and digits are separated. if (neg) { w.write(neg); } @@ -481,9 +482,9 @@ namespace bw_fmt return w; } - /// Format for floating point values. Seperates floating point into a whole number and a + /// Format for floating point values. Separates floating point into a whole number and a /// fraction. The fraction is converted into an unsigned integer based on the specified - /// precision, spec._prec. ie. 3.1415 with precision two is seperated into two unsigned + /// precision, spec._prec. ie. 3.1415 with precision two is separated into two unsigned /// integers 3 and 14. The different pieces are assembled and placed into the BufferWriter. /// The default is two decimal places. ie. X.XX. The value is always written in base 10. /// @@ -586,7 +587,7 @@ namespace bw_fmt /// Write out the @a data as hexadecimal, using @a digits as the conversion. void - Hex_Dump(BufferWriter &w, std::string_view data, const char *digits) + Format_As_Hex(BufferWriter &w, std::string_view data, const char *digits) { const char *ptr = data.data(); for (auto n = data.size(); n > 0; --n) { @@ -607,14 +608,7 @@ bwformat(BufferWriter &w, BWFSpec const &spec, std::string_view sv) } if ('x' == spec._type || 'X' == spec._type) { - const char *digits = 'x' == spec._type ? bw_fmt::LOWER_DIGITS : bw_fmt::UPPER_DIGITS; - width -= sv.size() * 2; - if (spec._radix_lead_p) { - w.write('0'); - w.write(spec._type); - width -= 2; - } - bw_fmt::Write_Aligned(w, [&w, &sv, digits]() { bw_fmt::Hex_Dump(w, sv, digits); }, spec._align, width, spec._fill, 0); + return bwformat(w, spec, bwf::detail::MemDump(sv.data(), sv.size())); } else { width -= sv.size(); bw_fmt::Write_Aligned(w, [&w, &sv]() { w.write(sv); }, spec._align, width, spec._fill, 0); @@ -627,12 +621,7 @@ bwformat(BufferWriter &w, BWFSpec const &spec, MemSpan const &span) { static const BWFormat default_fmt{"{:#x}@{:p}"}; if (spec._ext.size() && 'd' == spec._ext.front()) { - const char *digits = 'X' == spec._type ? bw_fmt::UPPER_DIGITS : bw_fmt::LOWER_DIGITS; - if (spec._radix_lead_p) { - w.write('0'); - w.write(digits[33]); - } - bw_fmt::Hex_Dump(w, span.view(), digits); + bwformat(w, spec, bwf::detail::MemDump(span.data(), span.size())); } else { w.print(default_fmt, span.size(), span.data()); } @@ -957,6 +946,28 @@ bwformat(BufferWriter &w, BWFSpec const &spec, bwf::OptionalAffix const &opts) return w.write(opts._prefix).write(opts._text).write(opts._suffix); } +BufferWriter & +bwformat(BufferWriter &w, BWFSpec const &spec, bwf::detail::MemDump const &hex) +{ + char fmt_type = spec._type; + const char *digits = bw_fmt::UPPER_DIGITS; + + if ('X' != fmt_type) { + fmt_type = 'x'; + digits = bw_fmt::LOWER_DIGITS; + } + + int width = int(spec._min) - hex._view.size() * 2; // amount left to fill. + if (spec._radix_lead_p) { + w.write('0'); + w.write(fmt_type); + width -= 2; + } + bw_fmt::Write_Aligned(w, [&w, &hex, digits]() { bw_fmt::Format_As_Hex(w, hex._view, digits); }, spec._align, width, spec._fill, + 0); + return w; +} + } // namespace ts namespace @@ -964,12 +975,14 @@ namespace void BWF_Timestamp(ts::BufferWriter &w, ts::BWFSpec const &spec) { - // Unfortunately need to write to a temporary buffer or the sizing isn't correct if @a w is clipped - // because @c strftime returns 0 if the buffer isn't large enough. - char buff[32]; - std::time_t t = std::time(nullptr); - auto n = strftime(buff, sizeof(buff), "%Y %b %d %H:%M:%S", std::localtime(&t)); - w.write(buff, n); + auto now = std::chrono::system_clock::now(); + auto epoch = std::chrono::system_clock::to_time_t(now); + ts::LocalBufferWriter<48> lw; + + ctime_r(&epoch, lw.auxBuffer()); + lw.fill(19); // clip trailing stuff, do not want. + lw.print(".{:03}", std::chrono::time_point_cast(now).time_since_epoch().count() % 1000); + w.write(lw.view().substr(4)); } void @@ -993,13 +1006,9 @@ BWF_ThreadID(ts::BufferWriter &w, ts::BWFSpec const &spec) void BWF_ThreadName(ts::BufferWriter &w, ts::BWFSpec const &spec) { -#if defined(__FreeBSD_version) - bwformat(w, spec, "thread"sv); // no thread names in FreeBSD. -#else char name[32]; // manual says at least 16, bump that up a bit. - pthread_getname_np(pthread_self(), name, sizeof(name)); + ink_get_thread_name(name, sizeof(name)); bwformat(w, spec, std::string_view{name}); -#endif } static bool BW_INITIALIZED __attribute__((unused)) = []() -> bool { diff --git a/src/tscore/CryptoHash.cc b/src/tscore/CryptoHash.cc index fec88fb84e0..b422c85493a 100644 --- a/src/tscore/CryptoHash.cc +++ b/src/tscore/CryptoHash.cc @@ -55,7 +55,7 @@ CryptoContext::CryptoContext() break; #endif default: - ink_release_assert("Invalid global URL hash context"); + ink_release_assert(!"Invalid global URL hash context"); }; #if TS_ENABLE_FIPS == 0 static_assert(CryptoContext::OBJ_SIZE >= sizeof(MD5Context), "bad OBJ_SIZE"); @@ -69,7 +69,7 @@ CryptoContext::CryptoContext() @brief Converts a hash to a null-terminated string Externalizes an hash as a null-terminated string into the first argument. - Does so without intenal procedure calls. + Does so without internal procedure calls. Side Effects: none. Reentrancy: n/a. Thread Safety: safe. diff --git a/src/tscore/Diags.cc b/src/tscore/Diags.cc index 8d2cd02a185..6c85e578469 100644 --- a/src/tscore/Diags.cc +++ b/src/tscore/Diags.cc @@ -34,6 +34,8 @@ ****************************************************************************/ +#include "tscore/BufferWriter.h" +#include "tscore/bwf_std_format.h" #include "tscore/ink_platform.h" #include "tscore/ink_memory.h" #include "tscore/ink_defs.h" @@ -202,7 +204,7 @@ Diags::~Diags() // // This routine outputs to all of the output targets enabled for this // debugging level in config.outputs[diags_level]. Many higher level -// diagnosting printing routines are built upon print_va, including: +// diagnostics printing routines are built upon print_va, including: // // void print(...) // void log_va(...) @@ -215,76 +217,29 @@ Diags::print_va(const char *debug_tag, DiagsLevel diags_level, const SourceLocat va_list ap) const { ink_release_assert(diags_level < DiagsLevel_Count); - - using ts::LocalBufferWriter; - LocalBufferWriter<1024> format_writer; + ts::LocalBufferWriter<1024> format_writer; // Save room for optional newline and terminating NUL bytes. format_writer.clip(2); - ////////////////////// - // append timestamp // - ////////////////////// - { - struct timeval tp = ink_gettimeofday(); - time_t cur_clock = (time_t)tp.tv_sec; - char timestamp_buf[48]; - char *buffer = ink_ctime_r(&cur_clock, timestamp_buf); - - int num_bytes_written = snprintf(&(timestamp_buf[19]), (sizeof(timestamp_buf) - 20), ".%03d", (int)(tp.tv_usec / 1000)); - - if (num_bytes_written > 0) { - format_writer.write('['); - format_writer.write(buffer + 4, strlen(buffer + 4)); - format_writer.write("] ", 2); - } - } - - size_t timestamp_end_offset = format_writer.size(); + format_writer.print("[{timestamp}] "); + auto timestamp_offset = format_writer.size(); - /////////////////////// - // add the thread id // - /////////////////////// - format_writer.fill( - snprintf(format_writer.auxBuffer(), format_writer.remaining(), "{0x%" PRIx64 "} ", (uint64_t)ink_thread_self())); - - ////////////////////////////////// - // append the diag level prefix // - ////////////////////////////////// - - format_writer.write(level_name(diags_level), strlen(level_name(diags_level))); - format_writer.write(": ", 2); - - ///////////////////////////// - // append location, if any // - ///////////////////////////// + format_writer.print("{thread-name}"); + format_writer.print(" {}: ", level_name(diags_level)); if (location(loc, show_location, diags_level)) { - char *lp, buf[256]; - lp = loc->str(buf, sizeof(buf)); - if (lp) { - format_writer.write('<'); - format_writer.write(lp, std::min(strlen(lp), sizeof(buf))); - format_writer.write("> ", 2); - } + format_writer.print("<{}> ", *loc); } - ////////////////////////// - // append debugging tag // - ////////////////////////// - if (debug_tag != nullptr) { - format_writer.write('('); - format_writer.write(debug_tag, strlen(debug_tag)); - format_writer.write(") ", 2); + if (debug_tag) { + format_writer.print("({}) ", debug_tag); } - ////////////////////////////////////////////////////// - // append original format string, ensure there is a // - // newline, and NUL terminate // - ////////////////////////////////////////////////////// - format_writer.write(format_string, strlen(format_string)); - format_writer.extend(2); - if (format_writer.data()[format_writer.size() - 1] != '\n') { + format_writer.print("{}", format_string); + + format_writer.extend(2); // restore the space for required termination. + if (format_writer.view().back() != '\n') { // safe because always some chars in the buffer. format_writer.write('\n'); } format_writer.write('\0'); @@ -360,7 +315,7 @@ Diags::print_va(const char *debug_tag, DiagsLevel diags_level, const SourceLocat priority = LOG_NOTICE; break; } - vsnprintf(syslog_buffer, sizeof(syslog_buffer), format_writer.data() + timestamp_end_offset, ap); + vsnprintf(syslog_buffer, sizeof(syslog_buffer), format_writer.data() + timestamp_offset, ap); syslog(priority, "%s", syslog_buffer); } diff --git a/src/tscore/EventNotify.cc b/src/tscore/EventNotify.cc index dc1e1521573..a18decc3021 100644 --- a/src/tscore/EventNotify.cc +++ b/src/tscore/EventNotify.cc @@ -33,7 +33,7 @@ #ifdef HAVE_EVENTFD #include -#include +#include #include #endif diff --git a/src/tscore/HashMD5.cc b/src/tscore/HashMD5.cc index 7f1dedc1b80..f8b4ff5642e 100644 --- a/src/tscore/HashMD5.cc +++ b/src/tscore/HashMD5.cc @@ -23,7 +23,7 @@ #include "tscore/ink_config.h" #include "tscore/HashMD5.h" -ATSHashMD5::ATSHashMD5() : md_len(0), finalized(false) +ATSHashMD5::ATSHashMD5() { ctx = EVP_MD_CTX_new(); int ret = EVP_DigestInit_ex(ctx, EVP_md5(), nullptr); diff --git a/src/tscore/HostLookup.cc b/src/tscore/HostLookup.cc index d6a7439b5e4..b2c7166baf6 100644 --- a/src/tscore/HostLookup.cc +++ b/src/tscore/HostLookup.cc @@ -49,7 +49,7 @@ namespace bool domaincmp(string_view hostname, string_view domain) { - // Check to see if were passed emtpy strings for either + // Check to see if were passed empty strings for either // argument. Empty strings do not match anything // if (domain.empty() || hostname.empty()) { @@ -150,7 +150,7 @@ hostcmp(string_view lhs, string_view rhs) // '_' is also included although it is not in the spec (RFC 883) // // Uppercase and lowercase "a-z" both map to same indexes -// since hostnames are not case sensative +// since hostnames are not case sensitive // // Illegal characters map to 255 // @@ -177,7 +177,7 @@ static const unsigned char asciiToTable[256] = { 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255}; -// Number of legal characters in the acssiToTable array +// Number of legal characters in the asciiToTable array static const int numLegalChars = 38; // struct CharIndexBlock @@ -193,7 +193,7 @@ struct CharIndexBlock { }; // class CharIndex - A constant time string matcher intended for -// short strings in a sparsely populated DNS paritition +// short strings in a sparsely populated DNS partition // // Creates a look up table for character in data string // @@ -307,7 +307,7 @@ CharIndex::Insert(string_view match_data, HostBranch *toInsert) // Check to see if are at the level we supposed be at if (match_data.size() == 1) { - // The slot should always be emtpy, no duplicate keys are allowed + // The slot should always be empty, no duplicate keys are allowed ink_assert(cur->array[index].branch == nullptr); cur->array[index].branch = toInsert; break; @@ -458,7 +458,7 @@ CharIndex::iterator::operator!=(const self_type &that) const // class HostArray // -// Is a fixed size array for holding HostBrach* +// Is a fixed size array for holding HostBranch* // Allows only sequential access to data // @@ -656,7 +656,7 @@ HostLookup::TableNewLevel(HostBranch *from, string_view level_data) // HostBranch* HostLookup::InsertBranch(HostBranch* insert_to, const char* level_data) // // -// Abstrction to place a new node for level_data below node +// Abstraction to place a new node for level_data below node // insert to. Inserts into any of the data types used by // by class HostMatcher // diff --git a/src/tscore/IntrusivePtrTest.cc b/src/tscore/IntrusivePtrTest.cc index c7cb58c3f1c..85bd7e3f063 100644 --- a/src/tscore/IntrusivePtrTest.cc +++ b/src/tscore/IntrusivePtrTest.cc @@ -26,7 +26,7 @@ #include "tscore/TestBox.h" namespace -{ // Hide our local defintions +{ // Hide our local definitions // Test class for pointers and lists. class A : public IntrusivePtrCounter diff --git a/src/tscore/IpMap.cc b/src/tscore/IpMap.cc index ed206572c3c..89f11434986 100644 --- a/src/tscore/IpMap.cc +++ b/src/tscore/IpMap.cc @@ -1,6 +1,3 @@ -#include "tscore/IpMap.h" -#include "tscore/ink_inet.h" - /** @file IP address map support. @@ -46,6 +43,10 @@ before we had IpAddr as a type. */ +#include "tscore/IpMap.h" +#include "tscore/ink_inet.h" +#include "tscore/BufferWriter.h" + namespace ts { namespace detail @@ -139,8 +140,9 @@ namespace detail using ArgType = typename N::ArgType; ///< Import type. using Metric = typename N::Metric; ///< Import type.g482 - IpMapBase() : _root(nullptr) {} - ~IpMapBase() { this->clear(); } + IpMapBase() = default; + IpMapBase(self_type &&that) : _root(that._root), _list(std::move(that._list)) { that._root = nullptr; } + ~IpMapBase(); /** Mark a range. All addresses in the range [ @a min , @a max ] are marked with @a data. @return This object. @@ -225,9 +227,13 @@ namespace detail /// @return The number of distinct ranges. size_t count() const; - /// Print all spans. - /// @return This map. - self_type &print(); + /** Generate formatted output. + * + * @param w Destination of the output. + * @param spec Format specification. + * @return @a w. + */ + ts::BufferWriter &describe(ts::BufferWriter &w, ts::BWFSpec const &spec) const; // Helper methods. N * @@ -266,7 +272,7 @@ namespace detail return static_cast(_list.tail()); } - N *_root; ///< Root node. + N *_root = nullptr; ///< Root node. /// In order list of nodes. /// For ugly compiler reasons, this is a list of base class pointers /// even though we really store @a N instances on it. @@ -283,7 +289,6 @@ namespace detail } }; using NodeList = ts::IntrusiveDList; - // typedef ts::IntrusiveDList NodeList; /// This keeps track of all allocated nodes in order. /// Iteration depends on this list being maintained. NodeList _list; @@ -377,8 +382,11 @@ namespace detail if (n->_data == payload) { if (x) { if (n->_max <= max) { - // next range is covered, so we can remove and continue. - this->remove(n); +// next range is covered, so we can remove and continue. +#if defined(__clang_analyzer__) + ink_assert(x != n) +#endif + this->remove(n); n = next(x); } else if (n->_min <= max_plus1) { // Overlap or adjacent with larger max - absorb and finish. @@ -460,17 +468,14 @@ namespace detail Metric max_plus = N::deref(max); N::inc(max_plus); - /* Some subtlety - for IPv6 we overload the compare operators to do - the right thing, but we can't overload pointer - comparisons. Therefore we carefully never compare pointers in - this logic. Only @a min and @a max can be pointers, everything - else is an instance or a reference. Since there's no good reason - to compare @a min and @a max this isn't particularly tricky, but - it's good to keep in mind. If we were somewhat more clever, we - would provide static less than and equal operators in the - template class @a N and convert all the comparisons to use only - those two via static function call. - */ + /* Some subtlety - for IPv6 we overload the compare operators to do the right thing, but we + * can't overload pointer comparisons. Therefore we carefully never compare pointers in this + * logic. Only @a min and @a max can be pointers, everything else is an instance or a reference. + * Since there's no good reason to compare @a min and @a max this isn't particularly tricky, but + * it's good to keep in mind. If we were somewhat more clever, we would provide static less than + * and equal operators in the template class @a N and convert all the comparisons to use only + * those two via static function call. + */ /* We have lots of special cases here primarily to minimize memory allocation by re-using an existing node as often as possible. @@ -640,10 +645,14 @@ namespace detail void IpMapBase::insert_before(N *spot, N *n) { - N *c = left(spot); - if (!c) { + if (left(spot) == nullptr) { spot->setChild(n, N::LEFT); } else { +// If there's a left child, there's a previous node, therefore spot->_prev is valid. +// Clang analyzer doesn't realize this so it generates a false positive. +#if defined(__clang_analyzer__) + ink_assert(spot->_prev != nullptr); +#endif spot->_prev->setChild(n, N::RIGHT); } @@ -734,22 +743,27 @@ namespace detail } template - IpMapBase & - IpMapBase::print() + ts::BufferWriter & + IpMapBase::describe(ts::BufferWriter &w, ts::BWFSpec const &spec) const { -#if 0 - for ( Node* n = _list.head() ; n ; n = n->_next ) { - std::cout - << n << ": " << n->_min << '-' << n->_max << " [" << n->_data << "] " - << (n->_color == Node::BLACK ? "Black " : "Red ") << "P=" << n->_parent << " L=" << n->_left << " R=" << n->_right - << std::endl; - } -#endif - return *this; + auto pos = w.extent(); + for (auto const &rb_node : _list) { + N const &n{static_cast(rb_node)}; + if (w.extent() > pos) { + w.write(','); + } + w.print("{::a}-{::a}={}", n.min(), n.max(), n._data); + if (std::string_view::npos != spec._ext.find('x')) { + w.print("[{};^{};<{};>{}]", n._color == N::BLACK ? "Black" : "Red", n._parent, n._left, n._right); + } + } + return w; } + template IpMapBase::~IpMapBase() { this->clear(); } + //---------------------------------------------------------------------------- - typedef Interval Ip4Span; + using Ip4Span = Interval; /** Node for IPv4 map. We store the address in host order in the @a _min and @a _max @@ -835,7 +849,7 @@ namespace detail { return this->setMin(min + 1); } - /** Decremement the maximum value in place. + /** decrement the maximum value in place. @return This object. */ self_type & @@ -911,7 +925,13 @@ namespace detail /// is to use a pointer, not a reference. using ArgType = const ts::detail::Interval::Metric *; - /// Construct from pointers. + /** Construct from the argument type. + * + * @param min Minimum value in the range. + * @param max Maximum value in the range (inclusive). + * @param data Data to attach to the range. + */ + Ip6Node(ArgType min, ///< Minimum address (network order). ArgType max, ///< Maximum address (network order). void *data ///< Client data. @@ -919,30 +939,36 @@ namespace detail : Node(data), Ip6Span(*min, *max) { } - /// Construct with values. - Ip6Node(Metric const &min, ///< Minimum address (network order). - Metric const &max, ///< Maximum address (network order). - void *data ///< Client data. - ) - : Node(data), Ip6Span(min, max) - { - } + + /** Construct from the underlying @c Metric type @a min to @a max + * + * @param min Minimum value in the range. + * @param max Maximum value in the range (inclusive). + * @param data Data to attach to the range. + */ + Ip6Node(Metric const &min, Metric const &max, void *data) : Node(data), Ip6Span(min, max) {} + /// @return The minimum value of the interval. sockaddr const * min() const override { return ats_ip_sa_cast(&_min); } + /// @return The maximum value of the interval. sockaddr const * max() const override { return ats_ip_sa_cast(&_max); } - /// Set the client data. + + /** Set the client @a data. + * + * @param data Client data. + * @return @a this + */ self_type & - setData(void *data ///< Client data. - ) override + setData(void *data) override { _data = data; return *this; @@ -1009,7 +1035,7 @@ namespace detail inc(_min); return *this; } - /** Decremement the maximum value in place. + /** Decrement the maximum value in place. @return This object. */ self_type & @@ -1018,7 +1044,7 @@ namespace detail dec(_max); return *this; } - /** Increment the mininimum value in place. + /** Increment the minimum value in place. @return This object. */ self_type & @@ -1079,8 +1105,33 @@ namespace detail friend class ::IpMap; }; } // namespace detail + +template +inline BufferWriter & +bwformat(BufferWriter &w, BWFSpec const &spec, detail::IpMapBase const &map) +{ + return map.describe(w, spec); +} + } // namespace ts //---------------------------------------------------------------------------- +IpMap::IpMap(IpMap::self_type &&that) noexcept : _m4(that._m4), _m6(that._m6) +{ + that._m4 = nullptr; + that._m6 = nullptr; +} + +IpMap::self_type & +IpMap::operator=(IpMap::self_type &&that) +{ + if (&that != this) { + this->clear(); + std::swap(_m4, that._m4); + std::swap(_m6, that._m6); + } + return *this; +} + IpMap::~IpMap() { delete _m4; @@ -1262,5 +1313,27 @@ IpMap::iterator::operator--() return *this; } +ts::BufferWriter & +IpMap::describe(ts::BufferWriter &w, ts::BWFSpec const &spec) const +{ + w.write("IPv4 "); + if (_m4) { + bwformat(w, spec, *_m4); + } else { + w.write("N/A"); + } + w.write("\n"); + + w.write("IPv6 "); + if (_m6) { + bwformat(w, spec, *_m6); + } else { + w.write("N/A"); + } + w.write("\n"); + + return w; +} + //---------------------------------------------------------------------------- //---------------------------------------------------------------------------- diff --git a/src/tscore/JeAllocator.cc b/src/tscore/JeAllocator.cc index 214ffc0223e..fd1b418a8bf 100644 --- a/src/tscore/JeAllocator.cc +++ b/src/tscore/JeAllocator.cc @@ -92,7 +92,7 @@ JemallocNodumpAllocator::extend_and_setup_arena() } /** - * This will retain the orignal functionality if + * This will retain the original functionality if * !defined(JEMALLOC_NODUMP_ALLOCATOR_SUPPORTED) */ void * @@ -121,7 +121,7 @@ JemallocNodumpAllocator::allocate(InkFreeList *f) } /** - * This will retain the orignal functionality if + * This will retain the original functionality if * !defined(JEMALLOC_NODUMP_ALLOCATOR_SUPPORTED) */ void diff --git a/src/tscore/Makefile.am b/src/tscore/Makefile.am index 367cf8e900f..714e0bdb908 100644 --- a/src/tscore/Makefile.am +++ b/src/tscore/Makefile.am @@ -21,7 +21,7 @@ include $(top_srcdir)/build/tidy.mk noinst_PROGRAMS = mkdfa CompileParseRules check_PROGRAMS = test_atomic test_freelist test_geometry test_X509HostnameValidator test_tscore -TESTS_ENVIRONMENT = LSAN_OPTIONS=suppressions=suppression.txt +TESTS_ENVIRONMENT = LSAN_OPTIONS=suppressions=$(abs_top_srcdir)/ci/asan_leak_suppression/unit_tests.txt TESTS = $(check_PROGRAMS) @@ -30,7 +30,6 @@ lib_LTLIBRARIES = libtscore.la AM_CPPFLAGS += \ $(iocore_include_dirs) \ -I$(abs_top_srcdir)/include \ - -I$(abs_top_srcdir)/include/records \ -I$(abs_top_srcdir)/lib \ $(TS_INCLUDES) \ @YAMLCPP_INCLUDES@ @@ -42,175 +41,80 @@ libtscore_la_LIBADD = \ @LIBOBJS@ \ @LIBPCRE@ \ @OPENSSL_LIBS@ \ - @LIBTCL@ \ @LIBRESOLV@ \ @LIBCAP@ \ @YAMLCPP_LIBS@ \ -lc libtscore_la_SOURCES = \ - Allocator.h \ AcidPtr.cc \ - AcidPtr.h \ Arena.cc \ - Arena.h \ ArgParser.cc \ - ArgParser.h \ BaseLogFile.cc \ - BaseLogFile.h \ - BufferWriter.h \ - BufferWriterForward.h \ BufferWriterFormat.cc \ ConsistentHash.cc \ - ConsistentHash.h \ ContFlags.cc \ - ContFlags.h \ CryptoHash.cc \ - CryptoHash.h \ - defalloc.h \ Diags.cc \ - Diags.h \ - DynArray.h \ EventNotify.cc \ - EventNotify.h \ - Extendible.h \ fastlz.c \ - fastlz.h \ Hash.cc \ HashFNV.cc \ - HashFNV.h \ - Hash.h \ HashMD5.cc \ - HashMD5.h \ HashSip.cc \ - HashSip.h \ - History.h \ HostLookup.cc \ - HostLookup.h \ hugepages.cc \ - hugepages.h \ - I_Layout.h \ - ink_aiocb.h \ - ink_align.h \ - ink_apidefs.h \ ink_args.cc \ - ink_args.h \ ink_assert.cc \ - ink_assert.h \ - ink_atomic.h \ ink_base64.cc \ - ink_base64.h \ ink_cap.cc \ - ink_cap.h \ ink_code.cc \ - ink_code.h \ ink_defs.cc \ - ink_defs.h \ InkErrno.cc \ - InkErrno.h \ ink_error.cc \ - ink_error.h \ - ink_exception.h \ ink_file.cc \ - ink_file.h \ - ink_hash_table.cc \ - ink_hash_table.h \ ink_hrtime.cc \ - ink_hrtime.h \ ink_inet.cc \ - ink_inet.h \ - ink_inout.h \ - ink_llqueue.h \ - ink_lockfile.h \ - INK_MD5.h \ ink_memory.cc \ - ink_memory.h \ ink_mutex.cc \ - ink_mutex.h \ - ink_platform.h \ ink_queue.cc \ - ink_queue.h \ ink_queue_utils.cc \ ink_rand.cc \ - ink_rand.h \ ink_res_init.cc \ ink_res_mkquery.cc \ - ink_resolver.h \ ink_resource.cc \ - ink_resource.h \ ink_rwlock.cc \ - ink_rwlock.h \ ink_sock.cc \ - ink_sock.h \ ink_sprintf.cc \ - ink_sprintf.h \ ink_stack_trace.cc \ - ink_stack_trace.h \ ink_string.cc \ ink_string++.cc \ - ink_string.h \ - ink_string++.h \ ink_sys_control.cc \ - ink_sys_control.h \ ink_syslog.cc \ - ink_syslog.h \ ink_thread.cc \ - ink_thread.h \ ink_time.cc \ - ink_time.h \ ink_uuid.cc \ - ink_uuid.h \ IpMap.cc \ IpMapConf.cc \ - IpMapConf.h \ - IpMap.h \ - I_Version.h \ - JeAllocator.h \ JeAllocator.cc \ Layout.cc \ - List.h \ llqueue.cc \ lockfile.cc \ - Map.h \ MatcherUtils.cc \ - MatcherUtils.h \ - MemSpan.h \ MemArena.cc \ - MemArena.h \ MMH.cc \ - MMH.h \ - MT_hashtable.h \ ParseRules.cc \ - ParseRules.h \ - PriorityQueue.h \ - Ptr.h \ - RawHashTable.cc \ - RawHashTable.h \ RbTree.cc \ - RbTree.h \ Regex.cc \ - Regex.h \ Regression.cc \ - Regression.h \ - Result.h \ runroot.cc \ - runroot.h \ signals.cc \ - signals.h \ - SimpleTokenizer.h \ SourceLocation.cc \ - SourceLocation.h \ - TestBox.h \ TextBuffer.cc \ - TextBuffer.h \ Tokenizer.cc \ - Tokenizer.h \ - Trie.h \ - TsBuffer.h \ - ts_file. h ts_file.cc \ + ts_file.cc \ Version.cc \ - X509HostnameValidator.cc \ - X509HostnameValidator.h + X509HostnameValidator.cc BufferWriterFormat.o : AM_CPPFLAGS += -Wno-char-subscripts @@ -229,16 +133,16 @@ ParseRulesCType: CompileParseRules mkdfa_SOURCES = mkdfa.c test_atomic_SOURCES = test_atomic.cc -test_atomic_LDADD = libtscore.la $(top_builddir)/src/tscpp/util/libtscpputil.la @LIBTCL@ @LIBPCRE@ +test_atomic_LDADD = libtscore.la $(top_builddir)/src/tscpp/util/libtscpputil.la @LIBPCRE@ test_freelist_SOURCES = test_freelist.cc -test_freelist_LDADD = libtscore.la $(top_builddir)/src/tscpp/util/libtscpputil.la @LIBTCL@ @LIBPCRE@ +test_freelist_LDADD = libtscore.la $(top_builddir)/src/tscpp/util/libtscpputil.la @LIBPCRE@ test_geometry_SOURCES = test_geometry.cc -test_geometry_LDADD = libtscore.la $(top_builddir)/src/tscpp/util/libtscpputil.la @LIBTCL@ @LIBPCRE@ +test_geometry_LDADD = libtscore.la $(top_builddir)/src/tscpp/util/libtscpputil.la @LIBPCRE@ test_X509HostnameValidator_CPPFLAGS = $(AM_CPPFLAGS) -I$(abs_top_srcdir)/tests/include -test_X509HostnameValidator_LDADD = libtscore.la $(top_builddir)/src/tscpp/util/libtscpputil.la @LIBTCL@ @LIBPCRE@ @OPENSSL_LIBS@ +test_X509HostnameValidator_LDADD = libtscore.la $(top_builddir)/src/tscpp/util/libtscpputil.la @LIBPCRE@ @OPENSSL_LIBS@ test_X509HostnameValidator_SOURCES = unit_tests/test_X509HostnameValidator.cc test_tscore_CPPFLAGS = $(AM_CPPFLAGS)\ @@ -260,7 +164,6 @@ test_tscore_SOURCES = \ unit_tests/test_IntrusivePtr.cc \ unit_tests/test_IpMap.cc \ unit_tests/test_layout.cc \ - unit_tests/test_Map.cc \ unit_tests/test_List.cc \ unit_tests/test_MemArena.cc \ unit_tests/test_MT_hashtable.cc \ @@ -269,8 +172,7 @@ test_tscore_SOURCES = \ unit_tests/test_Regex.cc \ unit_tests/test_Scalar.cc \ unit_tests/test_scoped_resource.cc \ - unit_tests/test_ts_file.cc \ - unit_tests/test_Vec.cc + unit_tests/test_ts_file.cc CompileParseRules_SOURCES = CompileParseRules.cc diff --git a/src/tscore/MatcherUtils.cc b/src/tscore/MatcherUtils.cc index 7e7648c0275..cb9822a2d75 100644 --- a/src/tscore/MatcherUtils.cc +++ b/src/tscore/MatcherUtils.cc @@ -44,7 +44,7 @@ // off the heap (via ats_malloc() ) Returns a pointer to the buffer // is successful and nullptr otherwise. // -// CALLEE is responsibled for deallocating the buffer via ats_free() +// CALLEE is responsible for deallocating the buffer via ats_free() // char * readIntoBuffer(const char *file_path, const char *module_name, int *read_size_ptr) @@ -495,7 +495,7 @@ parseConfigLine(char *line, matcher_line *p_line, const matcher_tags *tags) case PARSE_VAL: if (inQuote == true) { if (*s == '\\') { - // The next character is esacped + // The next character is escaped // // To remove the escaped character // we need to copy diff --git a/src/tscore/ParseRules.cc b/src/tscore/ParseRules.cc index a3b0f78a568..8958d39443d 100644 --- a/src/tscore/ParseRules.cc +++ b/src/tscore/ParseRules.cc @@ -75,7 +75,7 @@ ink_atoi64(const char *str) while (*str && ParseRules::is_digit(*str)) { num = (num * 10) - (*str++ - '0'); } -#if USE_SI_MULTILIERS +#if USE_SI_MULTIPLIERS if (*str) { if (*str == 'K') { num = num * (1LL << 10); @@ -113,7 +113,7 @@ ink_atoui64(const char *str) while (*str && ParseRules::is_digit(*str)) { num = (num * 10) + (*str++ - '0'); } -#if USE_SI_MULTILIERS +#if USE_SI_MULTIPLIERS if (*str) { if (*str == 'K') { num = num * (1LL << 10); @@ -165,7 +165,7 @@ ink_atoi64(const char *str, int len) num = (num * 10) - (*str++ - '0'); len--; } -#if USE_SI_MULTILIERS +#if USE_SI_MULTIPLIERS if (len > 0 && *str) { if (*str == 'K') { num = num * (1 << 10); diff --git a/src/tscore/Regex.cc b/src/tscore/Regex.cc index 1a6cbedff00..d0a7a8bd864 100644 --- a/src/tscore/Regex.cc +++ b/src/tscore/Regex.cc @@ -21,6 +21,8 @@ limitations under the License. */ +#include + #include "tscore/ink_platform.h" #include "tscore/ink_thread.h" #include "tscore/ink_memory.h" @@ -48,6 +50,12 @@ get_jit_stack(void *data ATS_UNUSED) } #endif +Regex::Regex(Regex &&that) noexcept : regex(that.regex), regex_extra(that.regex_extra) +{ + that.regex = nullptr; + that.regex_extra = nullptr; +} + bool Regex::compile(const char *pattern, const unsigned flags) { @@ -101,25 +109,19 @@ Regex::get_capture_count() } bool -Regex::exec(const char *str) -{ - return exec(str, strlen(str)); -} - -bool -Regex::exec(const char *str, int length) +Regex::exec(std::string_view const &str) { - int ovector[30]; - return exec(str, length, ovector, countof(ovector)); + std::array ovector; + return this->exec(str, ovector.data(), ovector.size()); } bool -Regex::exec(const char *str, int length, int *ovector, int ovecsize) +Regex::exec(std::string_view const &str, int *ovector, int ovecsize) { int rv; - rv = pcre_exec(regex, regex_extra, str, length, 0, 0, ovector, ovecsize); - return rv > 0 ? true : false; + rv = pcre_exec(regex, regex_extra, str.data(), int(str.size()), 0, 0, ovector, ovecsize); + return rv > 0; } Regex::~Regex() @@ -136,113 +138,63 @@ Regex::~Regex() } } -DFA::~DFA() -{ - dfa_pattern *p = _my_patterns; - dfa_pattern *t; +DFA::~DFA() {} - while (p) { - if (p->_re) { - delete p->_re; - } - if (p->_p) { - ats_free(p->_p); - } - t = p->_next; - ats_free(p); - p = t; - } -} - -dfa_pattern * -DFA::build(const char *pattern, unsigned flags) +bool +DFA::build(std::string_view const &pattern, unsigned flags) { - dfa_pattern *ret; - int rv; + Regex rxp; + std::string string{pattern}; if (!(flags & RE_UNANCHORED)) { flags |= RE_ANCHORED; } - ret = (dfa_pattern *)ats_malloc(sizeof(dfa_pattern)); - ret->_p = nullptr; - - ret->_re = new Regex(); - rv = ret->_re->compile(pattern, flags); - if (rv == -1) { - delete ret->_re; - ats_free(ret); - return nullptr; + if (!rxp.compile(string.c_str(), flags)) { + return false; } - - ret->_idx = 0; - ret->_p = ats_strndup(pattern, strlen(pattern)); - ret->_next = nullptr; - return ret; + _patterns.emplace_back(std::move(rxp), std::move(string)); + return true; } int -DFA::compile(const char *pattern, unsigned flags) +DFA::compile(std::string_view const &pattern, unsigned flags) { - ink_assert(_my_patterns == nullptr); - _my_patterns = build(pattern, flags); - if (_my_patterns) { - return 0; - } else { - return -1; - } + ink_assert(_patterns.empty()); + this->build(pattern, flags); + return _patterns.size(); } int -DFA::compile(const char **patterns, int npatterns, unsigned flags) +DFA::compile(std::string_view *patterns, int npatterns, unsigned flags) { - const char *pattern; - dfa_pattern *ret = nullptr; - dfa_pattern *end = nullptr; - int i; - - for (i = 0; i < npatterns; i++) { - pattern = patterns[i]; - ret = build(pattern, flags); - if (!ret) { - continue; - } - - if (!_my_patterns) { - _my_patterns = ret; - _my_patterns->_next = nullptr; - _my_patterns->_idx = i; - } else { - end = _my_patterns; - while (end->_next) { - end = end->_next; - } - end->_next = ret; // add to end - ret->_idx = i; - } + _patterns.reserve(npatterns); // try to pre-allocate. + for (int i = 0; i < npatterns; ++i) { + this->build(patterns[i], flags); } - - return 0; + return _patterns.size(); } int -DFA::match(const char *str) const +DFA::compile(const char **patterns, int npatterns, unsigned flags) { - return match(str, strlen(str)); + _patterns.reserve(npatterns); // try to pre-allocate. + for (int i = 0; i < npatterns; ++i) { + this->build(patterns[i], flags); + } + return _patterns.size(); } int -DFA::match(const char *str, int length) const +DFA::match(std::string_view const &str) const { - int rc; - dfa_pattern *p = _my_patterns; - - while (p) { - rc = p->_re->exec(str, length); - if (rc > 0) { - return p->_idx; + // This is ugly, but the external interface needs to be @c const even though it's not really. + // This handles making the iterator non-const. + auto &pv{const_cast(_patterns)}; + for (auto spot = pv.begin(), limit = pv.end(); spot != limit; ++spot) { + if (spot->_re.exec(str)) { + return spot - _patterns.begin(); } - p = p->_next; } return -1; diff --git a/src/tscore/TextBuffer.cc b/src/tscore/TextBuffer.cc index ec62ca40c46..ef2b0b1750f 100644 --- a/src/tscore/TextBuffer.cc +++ b/src/tscore/TextBuffer.cc @@ -28,7 +28,7 @@ /**************************************************************************** * - * TextBuffer.cc - A self-expanding buffer, primarly meant for strings + * TextBuffer.cc - A self-expanding buffer, primarily meant for strings * * * @@ -40,7 +40,7 @@ TextBuffer::TextBuffer(int size) nextAdd = nullptr; currentSize = spaceLeft = 0; if (size > 0) { - // Insitute a minimum size + // Institute a minimum size if (size < 1024) { size = 1024; } @@ -162,8 +162,8 @@ TextBuffer::rawReadFromFile(int fd) { int readSize; - // Check to see if we have got a resonable amount of space left in our - // buffer, if not try to get somemore + // Check to see if we have got a reasonable amount of space left in our + // buffer, if not try to get some more if (spaceLeft < 4096) { if (enlargeBuffer(4096) == -1) { return -1; @@ -198,15 +198,15 @@ TextBuffer::slurp(int fd) // int TextBuffer::readFromFD(int fd) // // Issues a single read command on the file -// descritor passed in. Attempts to read a minimum of +// descriptor passed in. Attempts to read a minimum of // 512 bytes from file descriptor passed. int TextBuffer::readFromFD(int fd) { int readSize; - // Check to see if we have got a resonable amount of space left in our - // buffer, if not try to get somemore + // Check to see if we have got a reasonable amount of space left in our + // buffer, if not try to get some more if (spaceLeft < 512) { if (enlargeBuffer(512) == -1) { return -1; diff --git a/src/tscore/Tokenizer.cc b/src/tscore/Tokenizer.cc index b538facc9fd..b7c7f790f56 100644 --- a/src/tscore/Tokenizer.cc +++ b/src/tscore/Tokenizer.cc @@ -192,7 +192,7 @@ Tokenizer::Initialize(char *str, unsigned opt) quoteFound = false; - // Check to see if we stoped due to a maxToken limit + // Check to see if we stopped due to a maxToken limit if (max_limit_hit == true) { if (options & ALLOW_EMPTY_TOKS) { // Go till either we hit a delimiter or we've diff --git a/src/tscore/X509HostnameValidator.cc b/src/tscore/X509HostnameValidator.cc index ab3a7800f5e..73f03b01428 100644 --- a/src/tscore/X509HostnameValidator.cc +++ b/src/tscore/X509HostnameValidator.cc @@ -166,7 +166,7 @@ wildcard_match(const unsigned char *prefix, size_t prefix_len, const unsigned ch * permitted characters and only matches a single label */ for (p = wildcard_start; p != wildcard_end; ++p) { - if (!(('0' <= *p && *p <= '9') || ('A' <= *p && *p <= 'Z') || ('a' <= *p && *p <= 'z') || *p == '-')) { + if (!(('a' <= *p && *p <= 'z') || ('A' <= *p && *p <= 'Z') || ('0' <= *p && *p <= '9') || *p == '-' || *p == '_')) { return false; } } diff --git a/src/tscore/ink_base64.cc b/src/tscore/ink_base64.cc index 47f2e6a8745..92dfa132660 100644 --- a/src/tscore/ink_base64.cc +++ b/src/tscore/ink_base64.cc @@ -101,7 +101,7 @@ ats_base64_encode(const char *inBuffer, size_t inBufferSize, char *outBuffer, si } /*------------------------------------------------------------------------- - This is a reentrant, and malloc free implemetnation of ats_base64_decode. + This is a reentrant, and malloc free implementation of ats_base64_decode. -------------------------------------------------------------------------*/ #ifdef DECODE #undef DECODE diff --git a/src/tscore/ink_cap.cc b/src/tscore/ink_cap.cc index f33b732e972..7fd3289eb82 100644 --- a/src/tscore/ink_cap.cc +++ b/src/tscore/ink_cap.cc @@ -486,8 +486,7 @@ ElevateAccess::releasePrivilege() #endif ElevateAccess::ElevateAccess(unsigned lvl) - : elevated(false), - saved_uid(geteuid()), + : saved_uid(geteuid()), level(lvl) #if TS_USE_POSIX_CAP , diff --git a/src/tscore/ink_file.cc b/src/tscore/ink_file.cc index d737ad8b2d1..c0003c867eb 100644 --- a/src/tscore/ink_file.cc +++ b/src/tscore/ink_file.cc @@ -174,7 +174,7 @@ ink_filepath_merge(char *path, int pathsz, const char *rootpath, const char *add } else { // If INK_FILEPATH_NOTABSOLUTE is specified, the caller // requires a relative result. If the rootpath is - // ommitted, we do not retrieve the working path, + // omitted, we do not retrieve the working path, // if rootpath was supplied as absolute then fail. // if (flags & INK_FILEPATH_NOTABSOLUTE) { @@ -490,7 +490,7 @@ ink_file_get_geometry(int fd ATS_UNUSED, ink_device_geometry &geometry) #if defined(BLKALIGNOFF) // BLKALIGNOFF gets the number of bytes needed to align the I/Os to the block device with // and underlying block devices. This might be non-zero when you are using a logical volume - // backed by JBOD or RAID device(s). BLKALIGNOFF was addeed in 2.6.32, so it's not present in + // backed by JBOD or RAID device(s). BLKALIGNOFF was added in 2.6.32, so it's not present in // RHEL 5. if (ioctl(fd, BLKALIGNOFF, &arg.u32) == 0) { geometry.alignsz = arg.u32; diff --git a/src/tscore/ink_hash_table.cc b/src/tscore/ink_hash_table.cc deleted file mode 100644 index d4cf6e7eab1..00000000000 --- a/src/tscore/ink_hash_table.cc +++ /dev/null @@ -1,425 +0,0 @@ -/** @file - - A brief file description - - @section license License - - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - */ - -/**************************************************************************** - - ink_hash_table.c - - This file implements hash tables. This allows us to provide alternative - implementations of hash tables. - - ****************************************************************************/ - -#include "tscore/ink_error.h" -#include "tscore/ink_hash_table.h" -#include "tscore/ink_memory.h" - -/*===========================================================================* - - This is the Tcl implementation of InkHashTable - - *===========================================================================*/ - -/*---------------------------------------------------------------------------* - - InkHashTable *ink_hash_table_create(InkHashTableKeyType key_type) - - This routine allocates an initializes an empty InkHashTable, and returns a - pointer to the new table. The argument indicates whether keys - are represented as strings, or as words. Legal values are - InkHashTableKeyType_String and InkHashTableKeyType_Word. - - *---------------------------------------------------------------------------*/ - -InkHashTable * -ink_hash_table_create(InkHashTableKeyType key_type) -{ - InkHashTable *ht_ptr; - Tcl_HashTable *tcl_ht_ptr; - int tcl_key_type; - - tcl_ht_ptr = (Tcl_HashTable *)ats_malloc(sizeof(Tcl_HashTable)); - - if (key_type == InkHashTableKeyType_String) { - tcl_key_type = TCL_STRING_KEYS; - } else if (key_type == InkHashTableKeyType_Word) { - tcl_key_type = TCL_ONE_WORD_KEYS; - } else { - ink_fatal("ink_hash_table_create: bad key_type %d", key_type); - } - - Tcl_InitHashTable(tcl_ht_ptr, tcl_key_type); - - ht_ptr = (InkHashTable *)tcl_ht_ptr; - return (ht_ptr); -} /* End ink_hash_table_create */ - -/*---------------------------------------------------------------------------* - - void ink_hash_table_destroy(InkHashTable *ht_ptr) - - This routine takes a hash table , and frees its storage. - - *---------------------------------------------------------------------------*/ - -InkHashTable * -ink_hash_table_destroy(InkHashTable *ht_ptr) -{ - Tcl_HashTable *tcl_ht_ptr; - - tcl_ht_ptr = (Tcl_HashTable *)ht_ptr; - Tcl_DeleteHashTable(tcl_ht_ptr); - ats_free(tcl_ht_ptr); - return (InkHashTable *)nullptr; -} /* End ink_hash_table_destroy */ - -/*---------------------------------------------------------------------------* - - void ink_hash_table_destroy_and_free_values(InkHashTable *ht_ptr) - - This routine takes a hash table , and frees its storage, after - first calling ink_free on all the values. You better darn well make sure the - values have been dynamically allocated. - - *---------------------------------------------------------------------------*/ - -static int -_ink_hash_table_free_entry_value(InkHashTable *ht_ptr, InkHashTableEntry *e) -{ - InkHashTableValue value; - - value = ink_hash_table_entry_value(ht_ptr, e); - if (value != nullptr) { - ats_free(value); - } - - return (0); -} /* End _ink_hash_table_free_entry_value */ - -InkHashTable * -ink_hash_table_destroy_and_free_values(InkHashTable *ht_ptr) -{ - ink_hash_table_map(ht_ptr, _ink_hash_table_free_entry_value); - ink_hash_table_destroy(ht_ptr); - return (InkHashTable *)nullptr; -} /* End ink_hash_table_destroy_and_free_values */ - -/*---------------------------------------------------------------------------* - - int ink_hash_table_isbound(InkHashTable *ht_ptr, InkHashTableKey key) - - This routine takes a hash table , a key , and returns 1 - if the value is bound in the hash table, 0 otherwise. - - *---------------------------------------------------------------------------*/ - -int -ink_hash_table_isbound(InkHashTable *ht_ptr, const char *key) -{ - InkHashTableEntry *he_ptr; - - he_ptr = ink_hash_table_lookup_entry(ht_ptr, key); - return ((he_ptr == nullptr) ? 0 : 1); -} /* End ink_hash_table_isbound */ - -/*---------------------------------------------------------------------------* - int ink_hash_table_lookup(InkHashTable *ht_ptr, - InkHashTableKey key, - InkHashTableValue *value_ptr) - - This routine takes a hash table , a key , and stores the - value bound to the key by reference through . If no binding is - found, 0 is returned, else 1 is returned. - - *---------------------------------------------------------------------------*/ - -int -ink_hash_table_lookup(InkHashTable *ht_ptr, const char *key, InkHashTableValue *value_ptr) -{ - InkHashTableEntry *he_ptr; - InkHashTableValue value; - - he_ptr = ink_hash_table_lookup_entry(ht_ptr, key); - if (he_ptr == nullptr) { - return (0); - } - - value = ink_hash_table_entry_value(ht_ptr, he_ptr); - *value_ptr = value; - return (1); -} /* End ink_hash_table_lookup */ - -/*---------------------------------------------------------------------------* - - int ink_hash_table_delete(InkHashTable *ht_ptr, InkHashTableKey key) - - This routine takes a hash table and a key , and deletes the - binding for the in the hash table if it exists. This routine - returns 1 if the key existed, else 0. - - *---------------------------------------------------------------------------*/ - -int -ink_hash_table_delete(InkHashTable *ht_ptr, const char *key) -{ - char *tcl_key; - Tcl_HashTable *tcl_ht_ptr; - Tcl_HashEntry *tcl_he_ptr; - - tcl_key = (char *)key; - tcl_ht_ptr = (Tcl_HashTable *)ht_ptr; - tcl_he_ptr = Tcl_FindHashEntry(tcl_ht_ptr, tcl_key); - - if (!tcl_he_ptr) { - return (0); - } - Tcl_DeleteHashEntry(tcl_he_ptr); - - return (1); -} /* End ink_hash_table_delete */ - -/*---------------------------------------------------------------------------* - - InkHashTableEntry *ink_hash_table_lookup_entry(InkHashTable *ht_ptr, - InkHashTableKey key) - - This routine takes a hash table and a key , and returns the - entry matching the key, or nullptr otherwise. - - *---------------------------------------------------------------------------*/ - -InkHashTableEntry * -ink_hash_table_lookup_entry(InkHashTable *ht_ptr, const char *key) -{ - Tcl_HashTable *tcl_ht_ptr; - Tcl_HashEntry *tcl_he_ptr; - InkHashTableEntry *he_ptr; - - tcl_ht_ptr = (Tcl_HashTable *)ht_ptr; - tcl_he_ptr = Tcl_FindHashEntry(tcl_ht_ptr, key); - he_ptr = (InkHashTableEntry *)tcl_he_ptr; - - return (he_ptr); -} /* End ink_hash_table_lookup_entry */ - -/*---------------------------------------------------------------------------* - - InkHashTableEntry *ink_hash_table_get_entry(InkHashTable *ht_ptr, - InkHashTableKey key, - int *new_value) - - This routine takes a hash table and a key , and returns the - entry matching the key, or creates, binds, and returns a new entry. - If the binding already existed, *new is set to 0, else 1. - - *---------------------------------------------------------------------------*/ - -InkHashTableEntry * -ink_hash_table_get_entry(InkHashTable *ht_ptr, const char *key, int *new_value) -{ - Tcl_HashTable *tcl_ht_ptr; - Tcl_HashEntry *tcl_he_ptr; - - tcl_ht_ptr = (Tcl_HashTable *)ht_ptr; - tcl_he_ptr = Tcl_CreateHashEntry(tcl_ht_ptr, key, new_value); - - if (tcl_he_ptr == nullptr) { - ink_fatal("%s: Tcl_CreateHashEntry returned nullptr", "ink_hash_table_get_entry"); - } - - return ((InkHashTableEntry *)tcl_he_ptr); -} /* End ink_hash_table_get_entry */ - -/*---------------------------------------------------------------------------* - - void ink_hash_table_set_entry(InkHashTable *ht_ptr, - InkHashTableEntry *he_ptr, - InkHashTableValue value) - - This routine takes a hash table , a hash table entry , - and changes the value field of the entry to . - - *---------------------------------------------------------------------------*/ - -void -ink_hash_table_set_entry(InkHashTable *ht_ptr, InkHashTableEntry *he_ptr, InkHashTableValue value) -{ - (void)ht_ptr; - ClientData tcl_value; - Tcl_HashEntry *tcl_he_ptr; - - tcl_value = (ClientData)value; - tcl_he_ptr = (Tcl_HashEntry *)he_ptr; - Tcl_SetHashValue(tcl_he_ptr, tcl_value); -} /* End ink_hash_table_set_entry */ - -/*---------------------------------------------------------------------------* - - void ink_hash_table_insert(InkHashTable *ht_ptr, - InkHashTableKey key, - InkHashTableValue value) - - This routine takes a hash table , a key , and binds the value - to the key, replacing any previous binding, if any. - - *---------------------------------------------------------------------------*/ - -void -ink_hash_table_insert(InkHashTable *ht_ptr, const char *key, InkHashTableValue value) -{ - int new_value; - InkHashTableEntry *he_ptr; - - he_ptr = ink_hash_table_get_entry(ht_ptr, key, &new_value); - ink_hash_table_set_entry(ht_ptr, he_ptr, value); -} /* End ink_hash_table_insert */ - -/*---------------------------------------------------------------------------* - - void ink_hash_table_map(InkHashTable *ht_ptr, InkHashTableEntryFunction map) - - This routine takes a hash table and a function pointer , and - applies the function to each entry in the hash table. The function - should return 0 normally, otherwise the iteration will stop. - - *---------------------------------------------------------------------------*/ - -void -ink_hash_table_map(InkHashTable *ht_ptr, InkHashTableEntryFunction map) -{ - int retcode; - InkHashTableEntry *e; - InkHashTableIteratorState state; - - for (e = ink_hash_table_iterator_first(ht_ptr, &state); e != nullptr; e = ink_hash_table_iterator_next(ht_ptr, &state)) { - retcode = (*map)(ht_ptr, e); - if (retcode != 0) { - break; - } - } -} /* End ink_hash_table_map */ - -/*---------------------------------------------------------------------------* - - InkHashTableKey ink_hash_table_entry_key(InkHashTable *ht_ptr, - InkHashTableEntry *entry_ptr) - - This routine takes a hash table and a pointer to a hash table - entry , and returns the key portion of the entry. - - *---------------------------------------------------------------------------*/ - -InkHashTableKey -ink_hash_table_entry_key(InkHashTable *ht_ptr, InkHashTableEntry *entry_ptr) -{ - char *tcl_key; - - tcl_key = (char *)Tcl_GetHashKey((Tcl_HashTable *)ht_ptr, (Tcl_HashEntry *)entry_ptr); - return ((InkHashTableKey)tcl_key); -} /* End ink_hash_table_entry_key */ - -/*---------------------------------------------------------------------------* - - InkHashTableValue ink_hash_table_entry_value(InkHashTable *ht_ptr, - InkHashTableEntry *entry_ptr) - - This routine takes a hash table and a pointer to a hash table - entry , and returns the value portion of the entry. - - *---------------------------------------------------------------------------*/ - -InkHashTableValue -ink_hash_table_entry_value(InkHashTable *ht_ptr, InkHashTableEntry *entry_ptr) -{ - (void)ht_ptr; - ClientData tcl_value; - - tcl_value = Tcl_GetHashValue((Tcl_HashEntry *)entry_ptr); - return ((InkHashTableValue)tcl_value); -} /* End ink_hash_table_entry_value */ - -/*---------------------------------------------------------------------------* - - void ink_hash_table_dump_strings(InkHashTable *ht_ptr) - - This routine takes a hash table of string values, and dumps the keys and - string values to stdout. It is the caller's responsibility to ensure that - both the key and the value are NUL terminated strings. - - *---------------------------------------------------------------------------*/ - -static int -DumpStringEntry(InkHashTable *ht_ptr, InkHashTableEntry *e) -{ - InkHashTableKey key; - InkHashTableValue value; - - key = ink_hash_table_entry_key(ht_ptr, e); - value = ink_hash_table_entry_value(ht_ptr, e); - - fprintf(stderr, "key = '%s', value = '%s'\n", (char *)key, (char *)value); - - return (0); -} - -void -ink_hash_table_dump_strings(InkHashTable *ht_ptr) -{ - ink_hash_table_map(ht_ptr, DumpStringEntry); -} /* End ink_hash_table_dump_strings */ - -/*---------------------------------------------------------------------------* - - void ink_hash_table_replace_string(InkHashTable *ht_ptr, - char *string_key, char *string_value) - - This conveninece routine is intended for hash tables with keys of type - InkHashTableKeyType_String, and values being dynamically allocated strings. - This routine binds to a copy of , and any - previous bound value is deallocated. - - *---------------------------------------------------------------------------*/ - -void -ink_hash_table_replace_string(InkHashTable *ht_ptr, char *string_key, char *string_value) -{ - int new_value; - char *old_str; - InkHashTableEntry *he_ptr; - - /* - * The following line will flag a type-conversion warning on the - * DEC Alpha, but that message can be ignored, since we're - * still dealing with pointers, and we aren't loosing any bits. - */ - - he_ptr = ink_hash_table_get_entry(ht_ptr, (InkHashTableKey)string_key, &new_value); - if (new_value == 0) { - old_str = (char *)ink_hash_table_entry_value(ht_ptr, he_ptr); - if (old_str) { - ats_free(old_str); - } - } - - ink_hash_table_set_entry(ht_ptr, he_ptr, (InkHashTableValue)(ats_strdup(string_value))); -} /* End ink_hash_table_replace_string */ diff --git a/src/tscore/ink_hrtime.cc b/src/tscore/ink_hrtime.cc index 3a328c21441..9d8eff8ef98 100644 --- a/src/tscore/ink_hrtime.cc +++ b/src/tscore/ink_hrtime.cc @@ -37,7 +37,7 @@ #include #include #endif -#include +#include #include char * diff --git a/src/tscore/ink_inet.cc b/src/tscore/ink_inet.cc index 12524878b77..3f276c4e89f 100644 --- a/src/tscore/ink_inet.cc +++ b/src/tscore/ink_inet.cc @@ -841,7 +841,7 @@ bwformat(BufferWriter &w, BWFSpec const &spec, IpAddr const &addr) if (spec.has_numeric_type()) { bwformat(w, local_spec, static_cast(addr.family())); } else { - bwformat(w, local_spec, addr.family()); + bwformat(w, local_spec, ats_ip_family_name(addr.family())); } } return w; @@ -937,4 +937,13 @@ bwformat(BufferWriter &w, BWFSpec const &spec, sockaddr const *addr) return w; } +namespace bwf +{ + detail::MemDump + Hex_Dump(IpEndpoint const &addr) + { + return detail::MemDump(ats_ip_addr8_cast(&addr), ats_ip_addr_size(&addr)); + } +} // namespace bwf + } // namespace ts diff --git a/src/tscore/ink_memory.cc b/src/tscore/ink_memory.cc index 7ba65bd696d..aca92b80142 100644 --- a/src/tscore/ink_memory.cc +++ b/src/tscore/ink_memory.cc @@ -33,7 +33,7 @@ #include #if defined(linux) -// XXX: SHouldn't that be part of CPPFLAGS? +// XXX: Shouldn't that be part of CPPFLAGS? #ifndef _XOPEN_SOURCE #define _XOPEN_SOURCE 600 #endif @@ -168,7 +168,7 @@ ats_msync(caddr_t addr, size_t len, caddr_t end, int flags) // align start back to page boundary caddr_t a = (caddr_t)(((uintptr_t)addr) & ~(pagesize - 1)); - // align length to page boundry covering region + // align length to page boundary covering region size_t l = (len + (addr - a) + (pagesize - 1)) & ~(pagesize - 1); if ((a + l) > end) { l = end - a; // strict limit @@ -271,9 +271,21 @@ _xstrdup(const char *str, int length, const char * /* path ATS_UNUSED */) *newstr = '\0'; } else { strncpy(newstr, str, length); // we cannot do length + 1 because the string isn't - newstr[length] = '\0'; // guaranteeed to be null terminated! + newstr[length] = '\0'; // guaranteed to be null terminated! } return newstr; } return nullptr; } + +ats_scoped_str & +ats_scoped_str::operator=(std::string_view s) +{ + this->clear(); + if (!s.empty()) { + _r = static_cast(ats_malloc(s.size() + 1)); + memcpy(_r, s.data(), s.size()); + _r[s.size()] = '\0'; + } + return *this; +} diff --git a/src/tscore/ink_queue.cc b/src/tscore/ink_queue.cc index f6e8787f177..0a6ef05d1bf 100644 --- a/src/tscore/ink_queue.cc +++ b/src/tscore/ink_queue.cc @@ -91,10 +91,8 @@ static const ink_freelist_ops malloc_ops = {malloc_new, malloc_free, malloc_bu static const ink_freelist_ops freelist_ops = {freelist_new, freelist_free, freelist_bulkfree}; static const ink_freelist_ops *default_ops = &freelist_ops; -const ink_freelist_ops *freelist_global_ops = default_ops; -const ink_freelist_ops *freelist_class_ops = default_ops; - -static ink_freelist_list *freelists = nullptr; +static ink_freelist_list *freelists = nullptr; +static const ink_freelist_ops *freelist_global_ops = default_ops; const InkFreeListOps * ink_freelist_malloc_ops() @@ -109,15 +107,13 @@ ink_freelist_freelist_ops() } void -ink_freelist_init_ops(int nofl_global, int nofl_class) +ink_freelist_init_ops(int nofl_class, int nofl_proxy) { // This *MUST* only be called at startup before any freelists allocate anything. We will certainly crash if object // allocated from the freelist are freed by malloc. ink_release_assert(freelist_global_ops == default_ops); - ink_release_assert(freelist_class_ops == default_ops); - freelist_global_ops = nofl_global ? ink_freelist_malloc_ops() : ink_freelist_freelist_ops(); - freelist_class_ops = nofl_class ? ink_freelist_malloc_ops() : ink_freelist_freelist_ops(); + freelist_global_ops = (nofl_class || nofl_proxy) ? ink_freelist_malloc_ops() : ink_freelist_freelist_ops(); } void @@ -184,12 +180,11 @@ int fake_global_for_ink_queue = 0; #endif void * -ink_freelist_new(InkFreeList *f, const InkFreeListOps *ops) +ink_freelist_new(InkFreeList *f) { - ink_assert(ops != nullptr); void *ptr; - if (likely(ptr = ops->fl_new(f))) { + if (likely(ptr = freelist_global_ops->fl_new(f))) { ink_atomic_increment((int *)&f->used, 1); } @@ -275,13 +270,11 @@ malloc_new(InkFreeList *f) } void -ink_freelist_free(InkFreeList *f, void *item, const InkFreeListOps *ops) +ink_freelist_free(InkFreeList *f, void *item) { - ink_assert(ops != nullptr); - if (likely(item != nullptr)) { ink_assert(f->used != 0); - ops->fl_free(f, item); + freelist_global_ops->fl_free(f, item); ink_atomic_decrement((int *)&f->used, 1); } } @@ -334,12 +327,11 @@ malloc_free(InkFreeList *f, void *item) } void -ink_freelist_free_bulk(InkFreeList *f, void *head, void *tail, size_t num_item, const InkFreeListOps *ops) +ink_freelist_free_bulk(InkFreeList *f, void *head, void *tail, size_t num_item) { - ink_assert(ops != nullptr); ink_assert(f->used >= num_item); - ops->fl_bulkfree(f, head, tail, num_item); + freelist_global_ops->fl_bulkfree(f, head, tail, num_item); ink_atomic_decrement((int *)&f->used, num_item); } diff --git a/src/tscore/ink_queue_utils.cc b/src/tscore/ink_queue_utils.cc index 4246a3b58df..36fcdae2c6f 100644 --- a/src/tscore/ink_queue_utils.cc +++ b/src/tscore/ink_queue_utils.cc @@ -34,10 +34,10 @@ * problem resulted in the discovery that gcc was spitting out the * "ldd" (load double) instruction for loading of the 64 bit field "data" * while CC was spitting out two "ld" (load) instructions. The old code - * was calling item.data = head.data on sparcs and not putting any restriction + * was calling item.data = head.data on Sparcs and not putting any restriction * on the order of loading of the fields. * - * This is a problem on the sparcs because the "pointer" field was being loaded + * This is a problem on the Sparcs because the "pointer" field was being loaded * before the "version" field. This can result in a very subtle race condition * which subverts the addition of the "version" field. * @@ -50,7 +50,7 @@ * next.version = item.version ++; * cas64(head, item, next) - * Note, that the cas64 call will be succesful and the next.ptr will probably + * Note, that the cas64 call will be successful and the next.ptr will probably * be a pointer into the vtable entry. The next alloc will result in a write into * the vtable area. * diff --git a/src/tscore/ink_res_init.cc b/src/tscore/ink_res_init.cc index 962fc6b6070..be4dccce24d 100644 --- a/src/tscore/ink_res_init.cc +++ b/src/tscore/ink_res_init.cc @@ -129,7 +129,7 @@ ink_res_setservers(ink_res_state statp, IpEndpoint const *set, int cnt) /* The goal here seems to be to compress the source list (@a set) by squeezing out invalid addresses. We handle the special case where - the destination and sourcea are the same. + the destination and source are the same. */ int nserv = 0; for (IpEndpoint const *limit = set + cnt; nserv < INK_MAXNS && set < limit; ++set) { @@ -275,7 +275,7 @@ ink_res_randomid() * there will have precedence. Otherwise, the server address is set to * INADDR_ANY and the default domain name comes from the gethostname(). * - * An interrim version of this code (BIND 4.9, pre-4.4BSD) used 127.0.0.1 + * An interim version of this code (BIND 4.9, pre-4.4BSD) used 127.0.0.1 * rather than INADDR_ANY ("0.0.0.0") as the default name server address * since it was noted that INADDR_ANY actually meant ``the first interface * you "ifconfig"'d at boot time'' and if this was a SLIP or PPP interface, @@ -382,7 +382,7 @@ ink_res_init(ink_res_state statp, ///< State object to update. } /* --------------------------------------------- - Default domain name and doamin Search list: + Default domain name and domain Search list: if we are supplied a default domain name, and/or search list we will use it. Otherwise, @@ -427,7 +427,7 @@ ink_res_init(ink_res_state statp, ///< State object to update. } /* ------------------------------------------- - we must be provided with atleast a named! + we must be provided with at least a named! ------------------------------------------- */ if (pHostList) { if (pHostListSize > INK_MAXNS) { diff --git a/src/tscore/ink_res_mkquery.cc b/src/tscore/ink_res_mkquery.cc index 4ae16427763..fdd0aa1375e 100644 --- a/src/tscore/ink_res_mkquery.cc +++ b/src/tscore/ink_res_mkquery.cc @@ -217,7 +217,7 @@ labellen(const u_char *lp) } return ((bitlen + 7) / 8 + 1); } - return (-1); /*%< unknwon ELT */ + return (-1); /*%< unknown ELT */ } return (l); } @@ -278,7 +278,7 @@ decode_bitstring(const unsigned char **cpp, char *dn, const char *eom) /*% * Thinking in noninternationalized USASCII (per the DNS spec), - * is this characted special ("in need of quoting") ? + * is this character special ("in need of quoting") ? * * return: *\li boolean. diff --git a/src/tscore/ink_resource.cc b/src/tscore/ink_resource.cc index 08ba118983b..c557ad29e0a 100644 --- a/src/tscore/ink_resource.cc +++ b/src/tscore/ink_resource.cc @@ -39,7 +39,7 @@ ink_mutex ResourceTracker::resourceLock = PTHREAD_MUTEX_INITIALIZER; class Resource { public: - Resource() : _incrementCount(0), _decrementCount(0), _value(0), _symbol(nullptr) { _name[0] = '\0'; } + Resource() { _name[0] = '\0'; } void increment(const int64_t size); int64_t getValue() const @@ -86,10 +86,10 @@ class Resource } private: - int64_t _incrementCount; - int64_t _decrementCount; - int64_t _value; - const void *_symbol; + int64_t _incrementCount = 0; + int64_t _decrementCount = 0; + int64_t _value = 0; + const void *_symbol = nullptr; char _name[128]; }; diff --git a/src/tscore/ink_sock.cc b/src/tscore/ink_sock.cc index c9406205f9f..ccb5d9995aa 100644 --- a/src/tscore/ink_sock.cc +++ b/src/tscore/ink_sock.cc @@ -50,7 +50,7 @@ check_valid_sockaddr(sockaddr *sa, char *file, int line) if (port > 20000) { cerr << "[byteordering] In " << file << ", line " << line << " the IP port "; cerr << "was found to be " << port << "(in host byte order).\n"; - cerr << "[byteordering] This seems inplausible, so check for byte order problems\n"; + cerr << "[byteordering] This seems implausible, so check for byte order problems\n"; } } #else diff --git a/src/tscore/ink_string++.cc b/src/tscore/ink_string++.cc index bc8db8ac0d9..eac0718bac6 100644 --- a/src/tscore/ink_string++.cc +++ b/src/tscore/ink_string++.cc @@ -177,7 +177,7 @@ StrListOverflow::create_heap(int user_size) // matter since we are talking about strings but since this is a // last minute emergency bug fix, I'm not take any changes. If // allocations are not of aligned values then subsequents allocations - // aren't aligned, again mirroring the previous implemnetation + // aren't aligned, again mirroring the previous implementation int total_size = overflow_head_hdr_size + user_size; StrListOverflow *o = (StrListOverflow *)ats_malloc(total_size); diff --git a/src/tscore/ink_time.cc b/src/tscore/ink_time.cc index c8145cd01e8..ba36a1238bc 100644 --- a/src/tscore/ink_time.cc +++ b/src/tscore/ink_time.cc @@ -107,7 +107,7 @@ struct dtconv { /* * The man page for cftime lies. It claims that it is thread safe. * Instead, it silently trashes the heap (by freeing things more than - * once) when used in a mulithreaded program. Gack! + * once) when used in a multithreaded program. Gack! */ int cftime_replacement(char *s, int maxsize, const char *format, const time_t *clock) diff --git a/src/tscore/ink_uuid.cc b/src/tscore/ink_uuid.cc index 92266cdf23c..80940ad96b8 100644 --- a/src/tscore/ink_uuid.cc +++ b/src/tscore/ink_uuid.cc @@ -52,11 +52,13 @@ ATSUuid::initialize(TSUuidVersion v) // Copy assignment ATSUuid & -ATSUuid::operator=(const ATSUuid other) +ATSUuid::operator=(const ATSUuid &other) { - memcpy(_uuid.data, other._uuid.data, sizeof(_uuid.data)); - memcpy(_string, other._string, sizeof(_string)); - _version = other._version; + if (this != &other) { // Self assignment guard + memcpy(_uuid.data, other._uuid.data, sizeof(_uuid.data)); + memcpy(_string, other._string, sizeof(_string)); + _version = other._version; + } return *this; } diff --git a/src/tscore/load_http_hdr.cc b/src/tscore/load_http_hdr.cc index b565438acf8..c9d840f6cb1 100644 --- a/src/tscore/load_http_hdr.cc +++ b/src/tscore/load_http_hdr.cc @@ -128,7 +128,7 @@ load_buffer(int fd, hdr_type h_type) fprintf(stderr, "Failed to read data file : %s\n", strerror(errno)); exit(1); } else if (done == 0) { - fprintf(stderr, "EOF encounted\n", strerror(errno)); + fprintf(stderr, "EOF encountered\n", strerror(errno)); exit(1); } diff --git a/src/tscore/lockfile.cc b/src/tscore/lockfile.cc index b3782682af8..a0c87cecaf5 100644 --- a/src/tscore/lockfile.cc +++ b/src/tscore/lockfile.cc @@ -24,7 +24,7 @@ #include "tscore/ink_platform.h" #include "tscore/ink_lockfile.h" -#define LOCKFILE_BUF_LEN 16 // 16 bytes should be enought for a pid +#define LOCKFILE_BUF_LEN 16 // 16 bytes should be enough for a pid int Lockfile::Open(pid_t *holding_pid) @@ -99,7 +99,7 @@ Lockfile::Open(pid_t *holding_pid) FAIL(0); } // If we did get the lock, then set the close on exec flag so that - // we don't accidently pass the file descriptor to a child process + // we don't accidentally pass the file descriptor to a child process // when we do a fork/exec. do { err = fcntl(fd, F_GETFD, 0); diff --git a/src/tscore/runroot.cc b/src/tscore/runroot.cc index 5d786d9f5fb..7736a97629f 100644 --- a/src/tscore/runroot.cc +++ b/src/tscore/runroot.cc @@ -96,49 +96,15 @@ get_parent_yaml_path(const std::string &path) if (!yaml_file.empty()) { return yaml_file; } - whole_path = whole_path.substr(0, whole_path.find_last_of("/")); + whole_path = whole_path.substr(0, whole_path.find_last_of('/')); } return {}; } -// handler for ts runroot -// this function set up runroot_file -void -runroot_handler(const char **argv, bool json) +static void +runroot_extra_handling(const char *executable, bool json) { - std::string prefix = "--run-root"; std::string path; - - // check if we have --run-root... - std::string arg = {}; - - int i = 0; - while (argv[i]) { - std::string_view command = argv[i]; - if (command.substr(0, prefix.size()) == prefix) { - arg = command.data(); - break; - } - i++; - } - - // if --run-root is provided - if (!arg.empty() && arg != prefix) { - // 1. pass in path - prefix += "="; - std::string value = arg.substr(prefix.size(), arg.size() - 1); - path = get_yaml_path(value); - if (!path.empty()) { - if (!json) { - ink_notice("using command line path as RUNROOT"); - } - runroot_file = path; - return; - } else if (!json) { - ink_warning("Unable to access runroot: '%s'", value.c_str()); - } - } - // 2. check Environment variable char *env_val = getenv("TS_RUNROOT"); if (env_val) { @@ -153,7 +119,6 @@ runroot_handler(const char **argv, bool json) ink_warning("Unable to access runroot: '%s' from $TS_RUNROOT", env_val); } } - // 3. find cwd or parent path of cwd to check char cwd[PATH_MAX] = {0}; if (getcwd(cwd, sizeof(cwd)) != nullptr) { @@ -166,12 +131,11 @@ runroot_handler(const char **argv, bool json) return; } } - // 4. installed executable char RealBinPath[PATH_MAX] = {0}; - if ((argv[0] != nullptr) && realpath(argv[0], RealBinPath) != nullptr) { + if ((executable != nullptr) && realpath(executable, RealBinPath) != nullptr) { std::string bindir = RealBinPath; - bindir = bindir.substr(0, bindir.find_last_of("/")); // getting the bin dir not executable path + bindir = bindir.substr(0, bindir.find_last_of('/')); // getting the bin dir not executable path path = get_parent_yaml_path(bindir); if (!path.empty()) { runroot_file = path; @@ -185,6 +149,68 @@ runroot_handler(const char **argv, bool json) return; } +// This is a temporary approach to handle runroot with migration to ArgParser. +// When all program use ArgParser, we can remove the runroot_handler below and replace it with this one. +void +argparser_runroot_handler(std::string const &value, const char *executable, bool json) +{ + // 1.if --run-root is provided + if (!value.empty()) { + std::string path = get_yaml_path(value); + if (!path.empty()) { + if (!json) { + ink_notice("using command line path as RUNROOT"); + } + runroot_file = path; + return; + } else if (!json) { + ink_warning("Unable to access runroot: '%s'", value.c_str()); + } + } + runroot_extra_handling(executable, json); +} + +// handler for ts runroot +// this function set up runroot_file +void +runroot_handler(const char **argv, bool json) +{ + std::string prefix = "--run-root"; + std::string path; + + // check if we have --run-root... + std::string arg = {}; + + int i = 0; + while (argv[i]) { + std::string_view command = argv[i]; + if (command.substr(0, prefix.size()) == prefix) { + arg = command.data(); + break; + } + i++; + } + + // if --run-root is provided + if (!arg.empty() && arg != prefix) { + // 1. pass in path + prefix += "="; + std::string value = arg.substr(prefix.size(), arg.size() - 1); + path = get_yaml_path(value); + if (!path.empty()) { + if (!json) { + ink_notice("using command line path as RUNROOT"); + } + runroot_file = path; + return; + } else if (!json) { + ink_warning("Unable to access runroot: '%s'", value.c_str()); + } + } + + runroot_extra_handling(argv[0], json); +} + // return a map of all path with default layout std::unordered_map runroot_map_default() @@ -216,7 +242,7 @@ runroot_map(const std::string &file) RunrootMapType map; try { YAML::Node yamlfile = YAML::LoadFile(file); - std::string prefix = file.substr(0, file.find_last_of("/")); + std::string prefix = file.substr(0, file.find_last_of('/')); for (const auto &it : yamlfile) { // key value pairs of dirs diff --git a/src/tscore/suppression.txt b/src/tscore/suppression.txt deleted file mode 100644 index d76f93e1824..00000000000 --- a/src/tscore/suppression.txt +++ /dev/null @@ -1,4 +0,0 @@ -leak:CRYPTO_malloc -leak:CRYPTO_realloc -leak:ConsCell -leak:Vec, DefaultAlloc, 2>::set_expand diff --git a/src/tscore/test_freelist.cc b/src/tscore/test_freelist.cc index 159951017ea..3fdc27ab20d 100644 --- a/src/tscore/test_freelist.cc +++ b/src/tscore/test_freelist.cc @@ -37,13 +37,12 @@ test(void *d) id = (intptr_t)d; - time_t start = time(nullptr); - int count = 0; - const InkFreeListOps *ops = ink_freelist_freelist_ops(); + time_t start = time(nullptr); + int count = 0; for (;;) { - m1 = ink_freelist_new(flist, ops); - m2 = ink_freelist_new(flist, ops); - m3 = ink_freelist_new(flist, ops); + m1 = ink_freelist_new(flist); + m2 = ink_freelist_new(flist); + m3 = ink_freelist_new(flist); if ((m1 == m2) || (m1 == m3) || (m2 == m3)) { printf("0x%08" PRIx64 " 0x%08" PRIx64 " 0x%08" PRIx64 "\n", (uint64_t)(uintptr_t)m1, (uint64_t)(uintptr_t)m2, @@ -55,9 +54,9 @@ test(void *d) memset(m2, id, 64); memset(m3, id, 64); - ink_freelist_free(flist, m1, ops); - ink_freelist_free(flist, m2, ops); - ink_freelist_free(flist, m3, ops); + ink_freelist_free(flist, m1); + ink_freelist_free(flist, m2); + ink_freelist_free(flist, m3); // break out of the test if we have run more then 60 seconds if (++count % 1000 == 0 && (start + 60) < time(nullptr)) { diff --git a/src/tscore/unit_tests/test_AcidPtr.cc b/src/tscore/unit_tests/test_AcidPtr.cc index e3f89017ae0..4f7e98a4e56 100644 --- a/src/tscore/unit_tests/test_AcidPtr.cc +++ b/src/tscore/unit_tests/test_AcidPtr.cc @@ -85,7 +85,7 @@ TEST_CASE("AcidPtr Isolation") { AcidCommitPtr w = p; *w += 1; - CHECK(*p.getPtr() == 40); // new value not commited until end of scope + CHECK(*p.getPtr() == 40); // new value not committed until end of scope } CHECK(*p.getPtr() == 41); { @@ -94,3 +94,42 @@ TEST_CASE("AcidPtr Isolation") } CHECK(*p.getPtr() == 42); } + +TEST_CASE("AcidPtr persistence") +{ + AcidPtr p(new int(40)); + std::shared_ptr r1, r2, r3, r4; + REQUIRE(p.getPtr() != nullptr); + r1 = p.getPtr(); + { + AcidCommitPtr w = p; + r2 = p.getPtr(); + *w += 1; // update p at end of scope + } + r3 = p.getPtr(); + { + *AcidCommitPtr(p) += 1; // leaves scope immediately if not named. + r4 = p.getPtr(); + } + CHECK(*r1 == 40); // references to data are still valid, but inconsistent. (todo: rename AcidPtr to AiPtr?) + CHECK(*r2 == 40); + CHECK(*r3 == 41); + CHECK(*r4 == 42); +} + +TEST_CASE("AcidPtr Abort") +{ + AcidPtr p; + { + AcidCommitPtr w(p); + *w = 40; + } + CHECK(*p.getPtr() == 40); + { + AcidCommitPtr w = p; + *w += 1; + w.abort(); + CHECK(w == nullptr); + } + CHECK(*p.getPtr() == 40); +} diff --git a/src/tscore/unit_tests/test_BufferWriterFormat.cc b/src/tscore/unit_tests/test_BufferWriterFormat.cc index e8dee671a08..d6f28587f0f 100644 --- a/src/tscore/unit_tests/test_BufferWriterFormat.cc +++ b/src/tscore/unit_tests/test_BufferWriterFormat.cc @@ -25,6 +25,7 @@ #include "../../../tests/include/catch.hpp" #include #include +#include #include "tscore/BufferWriter.h" #include "tscore/bwf_std_format.h" #include "tscpp/util/MemSpan.h" @@ -306,6 +307,14 @@ TEST_CASE("bwstring", "[bwprint][bwstring]") ts::bwprint(out, fmt, std::string_view(), "Leif", "confused"); REQUIRE(out == "Did you know? Leif is confused"); } + + char const *null_string{nullptr}; + ts::bwprint(s, "Null {0:x}.{0}", null_string); + REQUIRE(s == "Null 0x0."); + ts::bwprint(s, "Null {0:X}.{0}", nullptr); + REQUIRE(s == "Null 0X0."); + ts::bwprint(s, "Null {0:p}.{0:P}.{0:s}.{0:S}", null_string); + REQUIRE(s == "Null 0x0.0X0.null.NULL"); } TEST_CASE("BWFormat integral", "[bwprint][bwformat]") @@ -513,7 +522,7 @@ TEST_CASE("bwstring std formats", "[libts][bwprint]") w.print("{}", ts::bwf::Errno(13)); REQUIRE(w.view() == "EACCES: Permission denied [13]"sv); w.reset().print("{}", ts::bwf::Errno(134)); - REQUIRE(w.view().substr(0, 22) == "Unknown: Unknown error"sv); + REQUIRE(w.view().substr(0, 9) == "Unknown: "sv); time_t t = 1528484137; // default is GMT @@ -566,6 +575,25 @@ TEST_CASE("bwstring std formats", "[libts][bwprint]") REQUIRE(w.view() == "name = Evil Dave"); w.reset().print("name = {}", ts::bwf::FirstOf(empty, empty, s3, empty, s2, s1)); REQUIRE(w.view() == "name = Leif"); + + unsigned v = ntohl(0xdeadbeef); + w.reset().print("{}", ts::bwf::Hex_Dump(v)); + REQUIRE(w.view() == "deadbeef"); + w.reset().print("{:x}", ts::bwf::Hex_Dump(v)); + REQUIRE(w.view() == "deadbeef"); + w.reset().print("{:X}", ts::bwf::Hex_Dump(v)); + REQUIRE(w.view() == "DEADBEEF"); + w.reset().print("{:#X}", ts::bwf::Hex_Dump(v)); + REQUIRE(w.view() == "0XDEADBEEF"); + w.reset().print("{} bytes {} digits {}", sizeof(double), std::numeric_limits::digits10, ts::bwf::Hex_Dump(2.718281828)); + REQUIRE(w.view() == "8 bytes 15 digits 9b91048b0abf0540"); + + INK_MD5 md5; + w.reset().print("{}", ts::bwf::Hex_Dump(md5)); + REQUIRE(w.view() == "00000000000000000000000000000000"); + CryptoContext().hash_immediate(md5, s2.data(), s2.size()); + w.reset().print("{}", ts::bwf::Hex_Dump(md5)); + REQUIRE(w.view() == "f240ccd7a95c7ec66d6c111e2925b23e"); } // Normally there's no point in running the performance tests, but it's worth keeping the code diff --git a/src/tscore/unit_tests/test_Extendible.cc b/src/tscore/unit_tests/test_Extendible.cc index 6a0900ecf28..31669292725 100644 --- a/src/tscore/unit_tests/test_Extendible.cc +++ b/src/tscore/unit_tests/test_Extendible.cc @@ -209,13 +209,28 @@ TEST_CASE("Extendible", "") CHECK(testField::alive == 0); } + INFO("AcidPtr AcidCommitPtr malloc ptr int"); + { + void *mem = malloc(sizeof(AcidPtr)); + AcidPtr &reader = *(new (mem) AcidPtr); + { + auto writer = reader.startCommit(); + CHECK(*writer == 0); + *writer = 1; + CHECK(*writer == 1); + CHECK(*reader.getPtr().get() == 0); + // end of scope writer, commit to reader + } + CHECK(*reader.getPtr().get() == 1); + reader.~AcidPtr(); + free(mem); + } INFO("AcidPtr AcidCommitPtr casting"); { - void *mem = malloc(sizeof(AcidPtr)); - new (mem) AcidPtr(); - AcidPtr &reader = *static_cast *>(mem); + void *mem = malloc(sizeof(AcidPtr)); + AcidPtr &reader = *(new (mem) AcidPtr); { - AcidCommitPtr writer = AcidCommitPtr(reader); + auto writer = reader.startCommit(); CHECK(writer->arr[0] == 1); CHECK(reader.getPtr()->arr[0] == 1); writer->arr[0] = 99; @@ -223,6 +238,7 @@ TEST_CASE("Extendible", "") CHECK(reader.getPtr()->arr[0] == 1); } CHECK(reader.getPtr()->arr[0] == 99); + reader.~AcidPtr(); free(mem); } INFO("ACIDPTR block-free reader") diff --git a/src/tscore/unit_tests/test_IntrusiveHashMap.cc b/src/tscore/unit_tests/test_IntrusiveHashMap.cc index a0521a27a21..f8f5b882e23 100644 --- a/src/tscore/unit_tests/test_IntrusiveHashMap.cc +++ b/src/tscore/unit_tests/test_IntrusiveHashMap.cc @@ -90,9 +90,10 @@ TEST_CASE("IntrusiveHashMap", "[libts][IntrusiveHashMap]") map.insert(new Thing("dave")); map.insert(new Thing("persia")); REQUIRE(map.count() == 3); - for (auto &thing : map) { - delete &thing; - } + // Need to be bit careful cleaning up, since the link pointers are in the objects and deleting + // the object makes it unsafe to use an iterator referencing that object. For a full cleanup, + // the best option is to first delete everything, then clean up the map. + map.apply([](Thing *thing) { delete thing; }); map.clear(); REQUIRE(map.count() == 0); @@ -111,9 +112,9 @@ TEST_CASE("IntrusiveHashMap", "[libts][IntrusiveHashMap]") REQUIRE(map.bucket_count() > nb); for (auto &thing : map) { REQUIRE(0 == marks[thing._n]); - marks[thing._n] = 1; + marks[thing._n] = true; } - marks[0] = 1; + marks[0] = true; REQUIRE(marks.all()); map.insert(new Thing("dup"sv, 79)); map.insert(new Thing("dup"sv, 80)); @@ -127,9 +128,11 @@ TEST_CASE("IntrusiveHashMap", "[libts][IntrusiveHashMap]") Map::iterator idx; // Erase all the non-"dup" and see if the range is still correct. - map.apply([&map](Thing &thing) { - if (thing._payload != "dup"sv) - map.erase(map.iterator_for(&thing)); + map.apply([&map](Thing *thing) { + if (thing->_payload != "dup"sv) { + map.erase(map.iterator_for(thing)); + delete thing; + } }); r = map.equal_range("dup"sv); REQUIRE(r.first != r.second); @@ -154,7 +157,7 @@ TEST_CASE("IntrusiveHashMap", "[libts][IntrusiveHashMap]") // Some more involved tests. TEST_CASE("IntrusiveHashMapManyStrings", "[IntrusiveHashMap]") { - std::vector strings; + std::vector strings; std::uniform_int_distribution char_gen{'a', 'z'}; std::uniform_int_distribution length_gen{20, 40}; @@ -166,12 +169,12 @@ TEST_CASE("IntrusiveHashMapManyStrings", "[IntrusiveHashMap]") strings.reserve(N); for (int i = 0; i < N; ++i) { auto len = length_gen(randu); - char *s = static_cast(malloc(len + 1)); + std::string s; + s.reserve(len); for (decltype(len) j = 0; j < len; ++j) { - s[j] = char_gen(randu); + s += char_gen(randu); } - s[len] = 0; - strings.push_back({s, size_t(len)}); + strings.push_back(s); } // Fill the IntrusiveHashMap @@ -232,4 +235,6 @@ TEST_CASE("IntrusiveHashMapManyStrings", "[IntrusiveHashMap]") } } REQUIRE(miss_p == false); + + ihm.apply([](Thing *thing) { delete thing; }); }; diff --git a/src/tscore/unit_tests/test_IntrusivePtr.cc b/src/tscore/unit_tests/test_IntrusivePtr.cc index c2fcf7e8190..d4a130aaac7 100644 --- a/src/tscore/unit_tests/test_IntrusivePtr.cc +++ b/src/tscore/unit_tests/test_IntrusivePtr.cc @@ -28,7 +28,7 @@ struct Thing : public ts::IntrusivePtrCounter { Thing() { ++_count; } - ~Thing() { --_count; } + virtual ~Thing() { --_count; } std::string _name; static int _count; // instance count. }; @@ -96,7 +96,7 @@ TEST_CASE("IntrusivePtr", "[libts][IntrusivePtr]") // List test. TEST_CASE("IntrusivePtr List", "[libts][IntrusivePtr]") { - // The clang analyzer claims this type of list manipularion leads to use after free because of + // The clang analyzer claims this type of list manipulation leads to use after free because of // premature class destruction but these tests verify that is a false positive. using LP = ts::IntrusivePtr; diff --git a/src/tscore/unit_tests/test_IpMap.cc b/src/tscore/unit_tests/test_IpMap.cc index f8a288663b4..07debf88b06 100644 --- a/src/tscore/unit_tests/test_IpMap.cc +++ b/src/tscore/unit_tests/test_IpMap.cc @@ -24,6 +24,7 @@ #include "tscore/IpMap.h" #include #include +#include std::ostream & operator<<(std::ostream &s, IpEndpoint const &addr) @@ -540,11 +541,11 @@ TEST_CASE("IpMap CloseIntersection", "[libts][ipmap]") void *const markB = reinterpret_cast(2); void *const markC = reinterpret_cast(3); void *const markD = reinterpret_cast(4); - // void *mark; // for retrieval IpEndpoint a_1_l, a_1_u, a_2_l, a_2_u, a_3_l, a_3_u, a_4_l, a_4_u, a_5_l, a_5_u, a_6_l, a_6_u, a_7_l, a_7_u; IpEndpoint b_1_l, b_1_u; IpEndpoint c_1_l, c_1_u, c_2_l, c_2_u, c_3_l, c_3_u; + IpEndpoint c_3_m; IpEndpoint d_1_l, d_1_u, d_2_l, d_2_u; IpEndpoint a_1_m; @@ -573,6 +574,7 @@ TEST_CASE("IpMap CloseIntersection", "[libts][ipmap]") ats_ip_pton("123.90.112.0", &c_2_l); ats_ip_pton("123.90.119.255", &c_2_u); ats_ip_pton("123.90.132.0", &c_3_l); + ats_ip_pton("123.90.134.157", &c_3_m); ats_ip_pton("123.90.135.255", &c_3_u); ats_ip_pton("123.82.196.0", &d_1_l); @@ -590,16 +592,42 @@ TEST_CASE("IpMap CloseIntersection", "[libts][ipmap]") CHECK_THAT(map, IsMarkedAt(a_1_m)); map.mark(b_1_l, b_1_u, markB); - CHECK_THAT(map, IsMarkedAt(a_1_m)); + CHECK_THAT(map, IsMarkedWith(a_1_m, markA)); map.mark(c_1_l, c_1_u, markC); map.mark(c_2_l, c_2_u, markC); map.mark(c_3_l, c_3_u, markC); - CHECK_THAT(map, IsMarkedAt(a_1_m)); + CHECK_THAT(map, IsMarkedWith(a_1_m, markA)); map.mark(d_1_l, d_1_u, markD); map.mark(d_2_l, d_2_u, markD); CHECK_THAT(map, IsMarkedAt(a_1_m)); + CHECK_THAT(map, IsMarkedWith(b_1_u, markB)); + CHECK_THAT(map, IsMarkedWith(c_3_m, markC)); + CHECK_THAT(map, IsMarkedWith(d_2_l, markD)); CHECK(map.count() == 13); -} + + // Check move constructor. + IpMap m2{std::move(map)}; + // Original map should be empty. + REQUIRE(map.count() == 0); + // Do all these again on the destination map. + CHECK_THAT(m2, IsMarkedWith(a_1_m, markA)); + CHECK_THAT(m2, IsMarkedWith(a_1_m, markA)); + CHECK_THAT(m2, IsMarkedWith(a_1_m, markA)); + CHECK_THAT(m2, IsMarkedWith(a_1_m, markA)); + CHECK_THAT(m2, IsMarkedWith(b_1_u, markB)); + CHECK_THAT(m2, IsMarkedWith(c_3_m, markC)); + CHECK_THAT(m2, IsMarkedWith(d_2_l, markD)); + CHECK(m2.count() == 13); + +#if 0 + ts::LocalBufferWriter<1024> w; + std::cout << "Basic map dump" << std::endl; + std::cout << w.print("{}", m2).view() << std::endl; + w.reset(); + std::cout << "With tree detail" << std::endl; + std::cout << w.print("{::x}", m2).view() << std::endl; +#endif +}; diff --git a/src/tscore/unit_tests/test_Map.cc b/src/tscore/unit_tests/test_Map.cc deleted file mode 100644 index a0a98860774..00000000000 --- a/src/tscore/unit_tests/test_Map.cc +++ /dev/null @@ -1,215 +0,0 @@ -/** @file - - Test code for the Map templates. - - @section license License - - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -#include "catch.hpp" - -#include -#include "tscore/Map.h" -#include - -using cchar = const char; - -struct Item { - LINK(Item, m_link); - struct Hash { - using ID = uint32_t; - using Key = uint32_t; - using Value = Item; - using ListHead = DLL; - - static ID - hash(Key key) - { - return key; - } - static Key key(Value *); - static bool equal(Key lhs, Key rhs); - }; - - uint32_t _key; - uint32_t _value; - - Item(uint32_t x) : _key(x), _value(x) {} - Item(uint32_t key, uint32_t value) : _key(key), _value(value) {} -}; - -uint32_t -Item::Hash::key(Value *v) -{ - return v->_key; -} -bool -Item::Hash::equal(Key lhs, Key rhs) -{ - return lhs == rhs; -} - -using Table = TSHashTable; - -class testHashMap -{ -private: - HashMap testsh; - -public: - int - get(cchar *ch) const - { - return testsh.get(ch); - } - - void - put(cchar *key, int v) - { - testsh.put(key, v); - } -}; - -TEST_CASE("test Map", "[libts][Map]") -{ - typedef Map SSMap; - typedef MapElem SSMapElem; - testHashMap testsh; -#define form_SSMap(_p, _v) form_Map(SSMapElem, _p, _v) - SSMap ssm; - ssm.put("a", "A"); - ssm.put("b", "B"); - ssm.put("c", "C"); - ssm.put("d", "D"); - form_SSMap(x, ssm) - { /* nop */ - } - - /* - if ((ssm).n) - for (SSMapElem *qq__x = (SSMapElem*)0, *x = &(ssm).v[0]; - ((intptr_t)(qq__x) < (ssm).n) && ((x = &(ssm).v[(intptr_t)qq__x]) || 1); - qq__x = (SSMapElem*)(((intptr_t)qq__x) + 1)) - if ((x)->key) { - // nop - } - */ - - cchar *hi = "hi", *ho = "ho", *hum = "hum", *hhi = "hhi"; - - ++hhi; - HashMap sh; - sh.put(hi, 1); - sh.put(ho, 2); - sh.put(hum, 3); - sh.put(hhi, 4); - REQUIRE(sh.get(hi) == 4); - REQUIRE(sh.get(ho) == 2); - REQUIRE(sh.get(hum) == 3); - sh.put("aa", 5); - sh.put("ab", 6); - sh.put("ac", 7); - sh.put("ad", 8); - sh.put("ae", 9); - sh.put("af", 10); - REQUIRE(sh.get(hi) == 4); - REQUIRE(sh.get(ho) == 2); - REQUIRE(sh.get(hum) == 3); - REQUIRE(sh.get("af") == 10); - REQUIRE(sh.get("ac") == 7); - - HashMap sh2(-99); // return -99 if key not found - sh2.put("aa", 15); - sh2.put("ab", 16); - testsh.put("aa", 15); - testsh.put("ab", 16); - REQUIRE(sh2.get("aa") == 15); - REQUIRE(sh2.get("ac") == -99); - REQUIRE(testsh.get("aa") == 15); - - // test_TSHashTable - static uint32_t const N = 270; - Table t; - Item *item = nullptr; - Table::Location loc; - std::list to_delete; - - for (uint32_t i = 1; i <= N; ++i) { - item = new Item(i); - t.insert(item); - to_delete.push_back(item); - } - - int failures = 0; - for (uint32_t i = 1; i <= N; ++i) { - Table::Location l = t.find(i); - if (!l.isValid() || i != l->_value) { - failures++; - } - } - REQUIRE(failures == 0); - - REQUIRE(!(t.find(N * 2).isValid())); - - loc = t.find(N / 2 | 1); - if (loc) { - t.remove(loc); - } else { - REQUIRE(!"Did not find expected value"); - } - - if (!loc) { - ; // compiler check. - } - - REQUIRE(!(t.find(N / 2 | 1).isValid())); - - for (uint32_t i = 1; i <= N; i += 2) { - t.remove(i); - } - - failures = 0; - for (uint32_t i = 1; i <= N; ++i) { - Table::Location l = t.find(i); - if (1 & i) { - if (l.isValid()) { - failures++; - } - } else { - if (!l.isValid()) { - failures++; - } - } - } - REQUIRE(failures == 0); - - int n = 0; - failures = 0; - for (Table::iterator spot = t.begin(), limit = t.end(); spot != limit; ++spot) { - ++n; - if ((spot->_value & 1) != 0) { - failures++; - } - } - REQUIRE(failures == 0); - REQUIRE(n == N / 2); - - for (auto it : to_delete) { - delete it; - } -} diff --git a/src/tscore/unit_tests/test_PriorityQueue.cc b/src/tscore/unit_tests/test_PriorityQueue.cc index 81a1dd2ffa3..b5144044450 100644 --- a/src/tscore/unit_tests/test_PriorityQueue.cc +++ b/src/tscore/unit_tests/test_PriorityQueue.cc @@ -21,7 +21,7 @@ limitations under the License. */ -#include +#include #include "tscore/PriorityQueue.h" #include "catch.hpp" diff --git a/src/tscore/unit_tests/test_Scalar.cc b/src/tscore/unit_tests/test_Scalar.cc index 3f8ac69ba9b..d38c2e80119 100644 --- a/src/tscore/unit_tests/test_Scalar.cc +++ b/src/tscore/unit_tests/test_Scalar.cc @@ -22,10 +22,8 @@ */ #include + #include "tscore/Scalar.h" -//#include -//#include -//#include using Bytes = ts::Scalar<1, off_t>; using Paragraphs = ts::Scalar<16, off_t>; @@ -76,6 +74,24 @@ TEST_CASE("Scalar", "[libts][Scalar]") sz_b = sz; // Should be OK because SCALE_1 is an integer multiple of SCALE_2 // sz = sz_b; // Should not compile. REQUIRE(sz_b.count() == 119 * (SCALE_1 / SCALE_2)); + + // Test generic rounding. + REQUIRE(120 == ts::round_up<10>(118)); + REQUIRE(120 == ts::round_up<10>(120)); + REQUIRE(130 == ts::round_up<10>(121)); + + REQUIRE(110 == ts::round_down<10>(118)); + REQUIRE(120 == ts::round_down<10>(120)); + REQUIRE(120 == ts::round_down<10>(121)); + + REQUIRE(1200 == ts::round_up<100>(1108)); + REQUIRE(1200 == ts::round_up<100>(1200)); + REQUIRE(1300 == ts::round_up<100>(1201)); + + REQUIRE(100 == ts::round_down<100>(118)); + REQUIRE(1100 == ts::round_down<100>(1108)); + REQUIRE(1200 == ts::round_down<100>(1200)); + REQUIRE(1200 == ts::round_down<100>(1208)); } TEST_CASE("Scalar Factors", "[libts][Scalar][factors]") diff --git a/src/tscore/unit_tests/test_Vec.cc b/src/tscore/unit_tests/test_Vec.cc deleted file mode 100644 index 0ae6af65b98..00000000000 --- a/src/tscore/unit_tests/test_Vec.cc +++ /dev/null @@ -1,388 +0,0 @@ -/* -*-Mode: c++;-*- - Various vector related code. - - @section license License - - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -/* UnionFind after Tarjan */ - -#include "catch.hpp" - -#include -#include -#include "tscore/ink_assert.h" -#include "tscore/Map.h" - -// Intervals store sets in interval format (e.g. [1..10][12..12]). -// Inclusion test is by binary search on intervals. -// Deletion is not supported -class Intervals : public Vec -{ -public: - void insert(int n); - bool in(int n) const; -}; - -// UnionFind supports fast unify and finding of -// 'representitive elements'. -// Elements are numbered from 0 to N-1. -class UnionFind : public Vec -{ -public: - // set number of elements, initialized to singletons, may be called repeatedly to increase size - void size(int n); - // return representitive element - int find(int n); - // unify the sets containing the two elements - void unify(int n, int m); -}; - -// binary search over intervals -static int -i_find(const Intervals *i, int x) -{ - ink_assert(i->n); - int l = 0, h = i->n; -Lrecurse: - if (h <= l + 2) { - if (h <= l) { - return -(l + 1); - } - if (x < i->v[l] || x > i->v[l + 1]) { - return -(l + 1); - } - return h; - } - int m = (((h - l) / 4) * 2) + l; - if (x > i->v[m + 1]) { - l = m; - goto Lrecurse; - } - if (x < i->v[m]) { - h = m; - goto Lrecurse; - } - return (l + 1); -} - -bool -Intervals::in(int x) const -{ - if (!n) { - return false; - } - if (i_find(this, x) > 0) { - return true; - } - return false; -} - -// insert into interval with merge -void -Intervals::insert(int x) -{ - if (!n) { - add(x); - add(x); - return; - } - int l = i_find(this, x); - if (l > 0) { - return; - } - l = -l - 1; - - if (x > v[l + 1]) { - if (x == v[l + 1] + 1) { - v[l + 1]++; - goto Lmerge; - } - l += 2; - if (l < (int)n) { - if (x == v[l] - 1) { - v[l]--; - goto Lmerge; - } - } - goto Lmore; - } else { - ink_assert(x < v[l]); - if (x == v[l] - 1) { - v[l]--; - goto Lmerge; - } - if (!l) { - goto Lmore; - } - l -= 2; - if (x == v[l + 1] + 1) { - v[l + 1]++; - goto Lmerge; - } - } -Lmore: - fill(n + 2); - if (n - 2 - l > 0) { - memmove(v + l + 2, v + l, sizeof(int) * (n - 2 - l)); - } - v[l] = x; - v[l + 1] = x; - return; -Lmerge: - if (l) { - if (v[l] - v[l - 1] < 2) { - l -= 2; - goto Ldomerge; - } - } - if (l < (int)(n - 2)) { - if (v[l + 2] - v[l + 1] < 2) { - goto Ldomerge; - } - } - return; -Ldomerge: - memmove(v + l + 1, v + l + 3, sizeof(int) * (n - 3 - l)); - n -= 2; - goto Lmerge; -} - -void -UnionFind::size(int s) -{ - size_t nn = n; - fill(s); - for (size_t i = nn; i < n; i++) { - v[i] = -1; - } -} - -int -UnionFind::find(int n) -{ - int i, t; - for (i = n; v[i] >= 0; i = v[i]) { - ; - } - while (v[n] >= 0) { - t = n; - n = v[n]; - v[t] = i; - } - return i; -} - -void -UnionFind::unify(int n, int m) -{ - n = find(n); - m = find(m); - if (n != m) { - if (v[m] < v[n]) { - v[m] += (v[n] - 1); - v[n] = m; - } else { - v[n] += (v[m] - 1); - v[m] = n; - } - } -} - -TEST_CASE("test append", "[Vec]") -{ - static const char value[] = "this is a string"; - unsigned int len = (int)sizeof(value) - 1; - - Vec str; - - str.append(value, 0); - REQUIRE(str.length() == 0); - - str.append(value, len); - REQUIRE(memcmp(&str[0], value, len) == 0); - REQUIRE(str.length() == len); - - str.clear(); - REQUIRE(str.length() == 0); - - int failures = 0; - for (unsigned i = 0; i < 1000; ++i) { - str.append(value, len); - if (memcmp(&str[i * len], value, len) != 0) { - failures++; - } - } - REQUIRE(failures == 0); - - REQUIRE(str.length() == 1000 * len); -} - -TEST_CASE("test basic", "[libts][Vec]") -{ - Vec v, vv, vvv; - int tt = 99 * 50, t = 0; - - for (size_t i = 0; i < 100; i++) { - v.add((void *)(intptr_t)i); - } - for (size_t i = 0; i < 100; i++) { - t += (int)(intptr_t)v.v[i]; - } - REQUIRE(t == tt); - - t = 0; - for (size_t i = 1; i < 100; i++) { - vv.set_add((void *)(intptr_t)i); - } - for (size_t i = 1; i < 100; i++) { - vvv.set_add((void *)(intptr_t)i); - } - for (size_t i = 1; i < 100; i++) { - vvv.set_add((void *)(intptr_t)(i * 1000)); - } - vv.set_union(vvv); - for (size_t i = 0; i < vv.n; i++) { - if (vv.v[i]) { - t += (int)(intptr_t)vv.v[i]; - } - } - REQUIRE(t == tt + 1000 * tt); - - v.clear(); - v.reserve(1000); - t = 0; - for (size_t i = 0; i < 1000; i++) { - v.add((void *)(intptr_t)i); - } - for (size_t i = 0; i < 1000; i++) { - t += (int)(intptr_t)v.v[i]; - } - REQUIRE(t == 999 * 500); - - Intervals in; - in.insert(1); - REQUIRE(in.n == 2); - in.insert(2); - REQUIRE(in.n == 2); - in.insert(6); - REQUIRE(in.n == 4); - in.insert(7); - REQUIRE(in.n == 4); - in.insert(9); - REQUIRE(in.n == 6); - in.insert(4); - REQUIRE(in.n == 8); - in.insert(5); - REQUIRE(in.n == 6); - in.insert(3); - REQUIRE(in.n == 4); - in.insert(8); - REQUIRE(in.n == 2); - - UnionFind uf; - uf.size(4); - uf.unify(0, 1); - uf.unify(2, 3); - REQUIRE(uf.find(2) == uf.find(3)); - REQUIRE(uf.find(0) == uf.find(1)); - REQUIRE(uf.find(0) != uf.find(3)); - REQUIRE(uf.find(1) != uf.find(3)); - REQUIRE(uf.find(1) != uf.find(2)); - REQUIRE(uf.find(0) != uf.find(2)); - uf.unify(1, 2); - REQUIRE(uf.find(0) == uf.find(3)); - REQUIRE(uf.find(1) == uf.find(3)); -} - -static bool -compare(void *a, void *b) -{ - return a < b; -} - -TEST_CASE("test sort", "[libts][Vec]") -{ - Vec v; - for (long i = 1; i <= 1000; ++i) { - v.add(reinterpret_cast(static_cast(((i * 149) % 1000) + 1))); - } - v.qsort(&compare); - int failures = 0; - for (int i = 0; i < 1000; ++i) { - if (reinterpret_cast(static_cast(i + 1)) != v[i]) { - failures++; - } - } - REQUIRE(failures == 0); - - v.clear(); - for (long i = 1; i <= 1000000; ++i) { - v.add(reinterpret_cast(static_cast(((i * 51511) % 1000000) + 1))); - } - v.qsort(&compare); - failures = 0; - for (long i = 0; i < 1000000; ++i) { - if (reinterpret_cast(static_cast(i + 1)) != v[i]) { - failures++; - } - } - REQUIRE(failures == 0); - - v.clear(); - for (long i = 1; i <= 1000000; ++i) { - // This should be every number 1..500000 twice. - v.add(reinterpret_cast(static_cast(((i * 199999) % 500000) + 1))); - } - v.qsort(&compare); - failures = 0; - for (long i = 0; i < 1000000; ++i) { - if (reinterpret_cast(static_cast((i / 2) + 1)) != v[i]) { - failures++; - } - } - REQUIRE(failures == 0); - - // Very long array, already sorted. This is what broke before. - v.clear(); - for (long i = 1; i <= 10000000; ++i) { - v.add(reinterpret_cast(static_cast(i))); - } - v.qsort(&compare); - failures = 0; - for (long i = 0; i < 10000000; ++i) { - if (reinterpret_cast(static_cast(i + 1)) != v[i]) { - failures++; - } - } - REQUIRE(failures == 0); - - // very long, reverse sorted. - v.clear(); - for (long i = 10000000; i >= 1; --i) { - v.add(reinterpret_cast(static_cast(i))); - } - v.qsort(&compare); - failures = 0; - for (long i = 0; i < 10000000; ++i) { - if (reinterpret_cast(static_cast(i + 1)) != v[i]) { - failures++; - } - } - REQUIRE(failures == 0); -} diff --git a/src/tscore/unit_tests/test_arena.cc b/src/tscore/unit_tests/test_arena.cc index dbf97ad618b..01f4217f0ed 100644 --- a/src/tscore/unit_tests/test_arena.cc +++ b/src/tscore/unit_tests/test_arena.cc @@ -28,7 +28,7 @@ Description: A short test program intended to be used with Purify to detect problems - with the arnea code + with the arena code ****************************************************************************/ diff --git a/src/tscore/unit_tests/test_ink_inet.cc b/src/tscore/unit_tests/test_ink_inet.cc index bd5749818b8..4fd47475610 100644 --- a/src/tscore/unit_tests/test_ink_inet.cc +++ b/src/tscore/unit_tests/test_ink_inet.cc @@ -219,6 +219,10 @@ TEST_CASE("inet formatting", "[libts][ink_inet][bwformat]") REQUIRE(w.view() == "172. 17. 99.231"); w.reset().print("{::=a}", ep); REQUIRE(w.view() == "172.017.099.231"); + w.reset().print("{}", ts::bwf::Hex_Dump(ep)); + REQUIRE(w.view() == "ac1163e7"); + w.reset().print("{:#X}", ts::bwf::Hex_Dump(ep)); + REQUIRE(w.view() == "0XAC1163E7"); // Documentation examples REQUIRE(0 == ats_ip_pton(addr_7, &ep.sa)); @@ -247,4 +251,8 @@ TEST_CASE("inet formatting", "[libts][ink_inet][bwformat]") w.reset().print("{:p}", reinterpret_cast(0x1337beef)); REQUIRE(w.view() == "0x1337beef"); + + ats_ip_pton(addr_1, &ep.sa); + w.reset().print("{}", ts::bwf::Hex_Dump(ep)); + REQUIRE(w.view() == "ffee00000000000024c333493cee0143"); } diff --git a/src/tscpp/api/AsyncHttpFetch.cc b/src/tscpp/api/AsyncHttpFetch.cc index 64239946912..add00c70b5a 100644 --- a/src/tscpp/api/AsyncHttpFetch.cc +++ b/src/tscpp/api/AsyncHttpFetch.cc @@ -222,7 +222,7 @@ AsyncHttpFetch::run() request_str += "\r\n"; request_str += state_->request_body_; - LOG_DEBUG("Issing (non-streaming) TSFetchUrl with request\n[%s]", request_str.c_str()); + LOG_DEBUG("Issuing (non-streaming) TSFetchUrl with request\n[%s]", request_str.c_str()); TSFetchUrl(request_str.c_str(), request_str.size(), reinterpret_cast(&addr), fetchCont, AFTER_BODY, event_ids); } else { diff --git a/src/tscpp/api/AsyncTimer.cc b/src/tscpp/api/AsyncTimer.cc index 9fab335b870..9387409cb6c 100644 --- a/src/tscpp/api/AsyncTimer.cc +++ b/src/tscpp/api/AsyncTimer.cc @@ -56,7 +56,7 @@ handleTimerEvent(TSCont cont, TSEvent event, void *edata) state->initial_timer_action_ = nullptr; // mark it so that it won't be canceled later on if (state->type_ == AsyncTimer::TYPE_PERIODIC) { LOG_DEBUG("Scheduling periodic event now"); - state->periodic_timer_action_ = TSContScheduleEvery(state->cont_, state->period_in_ms_, state->thread_pool_); + state->periodic_timer_action_ = TSContScheduleEveryOnPool(state->cont_, state->period_in_ms_, state->thread_pool_); } } if (!state->dispatch_controller_->dispatch()) { @@ -88,10 +88,10 @@ AsyncTimer::run() } if (one_off_timeout_in_ms) { LOG_DEBUG("Scheduling initial/one-off event"); - state_->initial_timer_action_ = TSContSchedule(state_->cont_, one_off_timeout_in_ms, state_->thread_pool_); + state_->initial_timer_action_ = TSContScheduleOnPool(state_->cont_, one_off_timeout_in_ms, state_->thread_pool_); } else if (regular_timeout_in_ms) { LOG_DEBUG("Scheduling regular timer events"); - state_->periodic_timer_action_ = TSContScheduleEvery(state_->cont_, regular_timeout_in_ms, state_->thread_pool_); + state_->periodic_timer_action_ = TSContScheduleEveryOnPool(state_->cont_, regular_timeout_in_ms, state_->thread_pool_); } } diff --git a/src/tscpp/api/GzipDeflateTransformation.cc b/src/tscpp/api/GzipDeflateTransformation.cc index 9832ec1f8ca..d371e3076d1 100644 --- a/src/tscpp/api/GzipDeflateTransformation.cc +++ b/src/tscpp/api/GzipDeflateTransformation.cc @@ -134,7 +134,7 @@ GzipDeflateTransformation::handleInputComplete() // We will flush out anything that's remaining in the gzip buffer int status = Z_OK; int iteration = 0; - const int buffer_size = 1024; // 1024 bytes is usually more than enough for the epilouge + const int buffer_size = 1024; // 1024 bytes is usually more than enough for the epilogue unsigned char buffer[buffer_size]; /* Deflate remaining data */ @@ -154,7 +154,7 @@ GzipDeflateTransformation::handleInputComplete() bytes_to_write, status); produce(std::string_view(reinterpret_cast(buffer), static_cast(bytes_to_write))); } else if (status != Z_STREAM_END) { - LOG_ERROR("Iteration %d: Gzip deflinate finalize produced an error '%d'", iteration, status); + LOG_ERROR("Iteration %d: Gzip deflate finalize produced an error '%d'", iteration, status); } } while (status == Z_OK); diff --git a/src/tscpp/api/Headers.cc b/src/tscpp/api/Headers.cc index 174588391ae..204d7a597a8 100644 --- a/src/tscpp/api/Headers.cc +++ b/src/tscpp/api/Headers.cc @@ -101,11 +101,11 @@ HeaderFieldName::operator!=(const std::string &field_name) * @private */ struct HeaderFieldValueIteratorState : noncopyable { - TSMBuffer hdr_buf_; - TSMLoc hdr_loc_; - TSMLoc field_loc_; - int index_; - HeaderFieldValueIteratorState() : hdr_buf_(nullptr), hdr_loc_(nullptr), field_loc_(nullptr), index_(0) {} + TSMBuffer hdr_buf_ = nullptr; + TSMLoc hdr_loc_ = nullptr; + TSMLoc field_loc_ = nullptr; + int index_ = 0; + HeaderFieldValueIteratorState() {} void reset(TSMBuffer bufp, TSMLoc hdr_loc, TSMLoc field_loc, int index) { diff --git a/src/tscpp/api/InterceptPlugin.cc b/src/tscpp/api/InterceptPlugin.cc index db72a59363d..b3ad83bb1b0 100644 --- a/src/tscpp/api/InterceptPlugin.cc +++ b/src/tscpp/api/InterceptPlugin.cc @@ -45,10 +45,10 @@ struct InterceptPlugin::State { TSVConn net_vc_ = nullptr; struct IoHandle { - TSVIO vio_; - TSIOBuffer buffer_; - TSIOBufferReader reader_; - IoHandle() : vio_(nullptr), buffer_(nullptr), reader_(nullptr){}; + TSVIO vio_ = nullptr; + TSIOBuffer buffer_ = nullptr; + TSIOBufferReader reader_ = nullptr; + IoHandle(){}; ~IoHandle() { if (reader_) { @@ -181,6 +181,17 @@ InterceptPlugin::getRequestHeaders() return state_->request_headers_; } +TSSslConnection +InterceptPlugin::getSslConnection() +{ + if (!state_->net_vc_) { + LOG_ERROR("Intercept Plugin is not ready to provide SSL Connection"); + return nullptr; + } + + return TSVConnSSLConnectionGet(state_->net_vc_); +} + bool InterceptPlugin::doRead() { @@ -359,7 +370,7 @@ handleEvents(TSCont cont, TSEvent pristine_event, void *pristine_edata) state->saved_event_ = event; state->saved_edata_ = edata; } - state->timeout_action_ = TSContSchedule(cont, 1, TS_THREAD_POOL_DEFAULT); + state->timeout_action_ = TSContScheduleOnPool(cont, 1, TS_THREAD_POOL_NET); return 0; } if (event == TS_EVENT_TIMEOUT) { // we have a saved event to restore diff --git a/src/tscpp/api/Logger.cc b/src/tscpp/api/Logger.cc index da94f313baa..7d44a0e3d4f 100644 --- a/src/tscpp/api/Logger.cc +++ b/src/tscpp/api/Logger.cc @@ -42,22 +42,17 @@ using atscppapi::Logger; */ struct atscppapi::LoggerState : noncopyable { std::string filename_; - bool add_timestamp_; - bool rename_file_; - Logger::LogLevel level_; - bool rolling_enabled_; - int rolling_interval_seconds_; - TSTextLogObject text_log_obj_; - bool initialized_; + bool add_timestamp_ = false; + bool rename_file_ = false; + Logger::LogLevel level_ = Logger::LOG_LEVEL_NO_LOG; + bool rolling_enabled_ = false; + int rolling_interval_seconds_ = -1; + TSTextLogObject text_log_obj_ = nullptr; + bool initialized_ = false; LoggerState() - : add_timestamp_(false), - rename_file_(false), - level_(Logger::LOG_LEVEL_NO_LOG), - rolling_enabled_(false), - rolling_interval_seconds_(-1), - text_log_obj_(nullptr), - initialized_(false){}; + + {}; ~LoggerState(){}; }; diff --git a/src/tscpp/api/Makefile.am b/src/tscpp/api/Makefile.am index 182c4322218..1232e92d287 100644 --- a/src/tscpp/api/Makefile.am +++ b/src/tscpp/api/Makefile.am @@ -15,6 +15,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +include $(top_srcdir)/build/tidy.mk + lib_LTLIBRARIES = libtscppapi.la libtscppapi_la_CPPFLAGS = $(AM_CPPFLAGS) -I $(abs_top_srcdir)/include @@ -47,3 +49,5 @@ libtscppapi_la_SOURCES = \ utils.cc \ utils_internal.cc +clang-tidy-local: $(DIST_SOURCES) + $(CXX_Clang_Tidy) diff --git a/src/tscpp/api/Request.cc b/src/tscpp/api/Request.cc index 83bd1901d93..74c45c744d8 100644 --- a/src/tscpp/api/Request.cc +++ b/src/tscpp/api/Request.cc @@ -32,24 +32,16 @@ using std::string; * @private */ struct atscppapi::RequestState : noncopyable { - TSMBuffer hdr_buf_; - TSMLoc hdr_loc_; - TSMLoc url_loc_; + TSMBuffer hdr_buf_ = nullptr; + TSMLoc hdr_loc_ = nullptr; + TSMLoc url_loc_ = nullptr; Url url_; Headers headers_; /* method and version are stored here for the case of an unbound request */ - HttpMethod method_; - HttpVersion version_; - bool destroy_buf_; - RequestState() - : hdr_buf_(nullptr), - hdr_loc_(nullptr), - url_loc_(nullptr), - method_(HTTP_METHOD_UNKNOWN), - version_(HTTP_VERSION_UNKNOWN), - destroy_buf_(false) - { - } + HttpMethod method_ = HTTP_METHOD_UNKNOWN; + HttpVersion version_ = HTTP_VERSION_UNKNOWN; + bool destroy_buf_ = false; + RequestState() {} }; Request::Request() @@ -204,10 +196,6 @@ Request::~Request() TSMLoc null_parent_loc = nullptr; TSHandleMLocRelease(state_->hdr_buf_, null_parent_loc, state_->url_loc_); TSMBufferDestroy(state_->hdr_buf_); - } else { - LOG_DEBUG("Destroying request object on hdr_buf=%p, hdr_loc=%p, url_loc=%p", state_->hdr_buf_, state_->hdr_loc_, - state_->url_loc_); - TSHandleMLocRelease(state_->hdr_buf_, state_->hdr_loc_, state_->url_loc_); } } delete state_; diff --git a/src/tscpp/api/Response.cc b/src/tscpp/api/Response.cc index bc2f0017059..7624e8861c2 100644 --- a/src/tscpp/api/Response.cc +++ b/src/tscpp/api/Response.cc @@ -32,10 +32,10 @@ namespace atscppapi * @private */ struct ResponseState : noncopyable { - TSMBuffer hdr_buf_; - TSMLoc hdr_loc_; + TSMBuffer hdr_buf_ = nullptr; + TSMLoc hdr_loc_ = nullptr; Headers headers_; - ResponseState() : hdr_buf_(nullptr), hdr_loc_(nullptr) {} + ResponseState() {} }; } // namespace atscppapi diff --git a/src/tscpp/api/Stat.cc b/src/tscpp/api/Stat.cc index b1422fa796d..0a28230cf3c 100644 --- a/src/tscpp/api/Stat.cc +++ b/src/tscpp/api/Stat.cc @@ -28,7 +28,7 @@ using namespace atscppapi; using std::string; -Stat::Stat() : stat_id_(TS_ERROR) +Stat::Stat() { // ATS Guarantees that stat ids will always be > 0. So we can use stat_id_ > 0 to // verify that this stat has been properly initialized. diff --git a/src/tscpp/api/Transaction.cc b/src/tscpp/api/Transaction.cc index 97e00dd6878..5beb03d7c31 100644 --- a/src/tscpp/api/Transaction.cc +++ b/src/tscpp/api/Transaction.cc @@ -413,7 +413,7 @@ namespace * @param constructor takes a function pointer of type GetterFunction * @param txn a TSHttpTxn * @param hdr_buf the address where the hdr buf will be stored - * @param hdr_loc the address where the mem loc will be storeds + * @param hdr_loc the address where the mem loc will be stored * @param name name of the entity - used for logging */ class initializeHandles diff --git a/src/tscpp/api/TransformationPlugin.cc b/src/tscpp/api/TransformationPlugin.cc index 435e498d6ab..e578657fc81 100644 --- a/src/tscpp/api/TransformationPlugin.cc +++ b/src/tscpp/api/TransformationPlugin.cc @@ -34,28 +34,27 @@ #define INT64_MAX (9223372036854775807LL) #endif -using namespace atscppapi; -using atscppapi::TransformationPlugin; - -namespace +namespace atscppapi { -class ResumeAfterPauseCont : public Continuation +namespace detail { -public: - ResumeAfterPauseCont() : Continuation() {} + class ResumeAfterPauseCont : public Continuation + { + public: + ResumeAfterPauseCont() : Continuation() {} - ResumeAfterPauseCont(Continuation::Mutex m) : Continuation(m) {} + ResumeAfterPauseCont(Continuation::Mutex m) : Continuation(m) {} -protected: - int _run(TSEvent event, void *edata) override; -}; + protected: + int _run(TSEvent event, void *edata) override; + }; -} // end anonymous namespace +} // end namespace detail /** * @private */ -struct atscppapi::TransformationPluginState : noncopyable, public ResumeAfterPauseCont { +struct TransformationPluginState : noncopyable, public detail::ResumeAfterPauseCont { TSVConn vconn_; Transaction &transaction_; TransformationPlugin &transformation_plugin_; @@ -70,7 +69,7 @@ struct atscppapi::TransformationPluginState : noncopyable, public ResumeAfterPau // We can only send a single WRITE_COMPLETE even though // we may receive an immediate event after we've sent a // write complete, so we'll keep track of whether or not we've - // sent the input end our write complte. + // sent the input end our write complete. bool input_complete_dispatched_; std::string request_xform_output_; // in case of request xform, data produced is buffered here @@ -93,7 +92,7 @@ struct atscppapi::TransformationPluginState : noncopyable, public ResumeAfterPau output_buffer_reader_ = TSIOBufferReaderAlloc(output_buffer_); }; - ~TransformationPluginState() + ~TransformationPluginState() override { if (output_buffer_reader_) { TSIOBufferReaderFree(output_buffer_reader_); @@ -107,8 +106,14 @@ struct atscppapi::TransformationPluginState : noncopyable, public ResumeAfterPau } }; +} // end namespace atscppapi + +using namespace atscppapi; + namespace { +using ResumeAfterPauseCont = atscppapi::detail::ResumeAfterPauseCont; + void cleanupTransformation(TSCont contp) { @@ -258,7 +263,7 @@ handleTransformationPluginEvents(TSCont contp, TSEvent event, void *edata) return 0; } - // All other events includign WRITE_READY will just attempt to transform more data. + // All other events including WRITE_READY will just attempt to transform more data. return handleTransformationPluginRead(state->vconn_, state); } @@ -421,7 +426,7 @@ TransformationPlugin::setOutputComplete() state_->txn_); // We're done without ever outputting anything, to correctly - // clean up we'll initiate a write then immeidately set it to 0 bytes done. + // clean up we'll initiate a write then immediately set it to 0 bytes done. state_->output_vio_ = TSVConnWrite(TSTransformOutputVConnGet(state_->vconn_), state_->vconn_, state_->output_buffer_reader_, 0); if (state_->output_vio_) { diff --git a/src/tscpp/api/utils_internal.cc b/src/tscpp/api/utils_internal.cc index f70f2c9855a..923c48d961a 100644 --- a/src/tscpp/api/utils_internal.cc +++ b/src/tscpp/api/utils_internal.cc @@ -79,7 +79,7 @@ handleTransactionEvents(TSCont cont, TSEvent event, void *edata) const std::list &plugins = utils::internal::getTransactionPlugins(transaction); for (auto plugin : plugins) { std::shared_ptr trans_mutex = utils::internal::getTransactionPluginMutex(*plugin); - LOG_DEBUG("Locking TransacitonPlugin mutex to delete transaction plugin at %p", plugin); + LOG_DEBUG("Locking TransactionPlugin mutex to delete transaction plugin at %p", plugin); trans_mutex->lock(); LOG_DEBUG("Locked Mutex...Deleting transaction plugin at %p", plugin); delete plugin; diff --git a/src/tscpp/util/Makefile.am b/src/tscpp/util/Makefile.am index 8f0f4428aa8..9e972b4efb8 100644 --- a/src/tscpp/util/Makefile.am +++ b/src/tscpp/util/Makefile.am @@ -16,6 +16,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +include $(top_srcdir)/build/tidy.mk + check_PROGRAMS = test_tscpputil TESTS = $(check_PROGRAMS) @@ -27,9 +29,7 @@ AM_CPPFLAGS += -I$(abs_top_srcdir)/include libtscpputil_la_LDFLAGS = -no-undefined -version-info @TS_LIBTOOL_VERSION@ libtscpputil_la_SOURCES = \ - IntrusiveDList.h \ - PostScript.h \ - TextView.h TextView.cc + TextView.cc test_tscpputil_CPPFLAGS = $(AM_CPPFLAGS)\ -I$(abs_top_srcdir)/tests/include @@ -48,4 +48,3 @@ clean-local: clang-tidy-local: $(DIST_SOURCES) $(CXX_Clang_Tidy) - diff --git a/src/tscpp/util/TextView.cc b/src/tscpp/util/TextView.cc index b6b1135eb62..78d668db90d 100644 --- a/src/tscpp/util/TextView.cc +++ b/src/tscpp/util/TextView.cc @@ -65,7 +65,7 @@ strcasecmp(const std::string_view &lhs, const std::string_view &rhs) } const int8_t ts::svtoi_convert[256] = { - /* [can't do this nicely because clang format won't allow exdented comments] + /* [can't do this nicely because clang format won't allow extended comments] 0 1 2 3 4 5 6 7 8 9 A B C D E F */ -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, // 00 @@ -131,7 +131,12 @@ ts::svtoi(TextView src, TextView *out, int base) break; default: while (src.size() && (0 <= (v = svtoi_convert[static_cast(*src)])) && v < base) { - zret = zret * base + v; + auto n = zret * base + v; + if (n < zret) { + zret = std::numeric_limits::max(); + break; // overflow, stop parsing. + } + zret = n; ++src; } break; @@ -148,7 +153,7 @@ ts::svtoi(TextView src, TextView *out, int base) return zret; } -// Do the template instantions. +// Do the template instantiations. template std::ostream &ts::TextView::stream_write(std::ostream &, const ts::TextView &) const; namespace std diff --git a/src/wccp/Makefile.am b/src/wccp/Makefile.am index 982f9f95c8a..5e57c031fea 100644 --- a/src/wccp/Makefile.am +++ b/src/wccp/Makefile.am @@ -41,3 +41,5 @@ libwccp_a_SOURCES = \ WccpStatic.cc \ WccpUtil.h +clang-tidy-local: $(DIST_SOURCES) + $(CXX_Clang_Tidy) diff --git a/src/wccp/WccpEndPoint.cc b/src/wccp/WccpEndPoint.cc index bbd45f36295..16e6377b054 100644 --- a/src/wccp/WccpEndPoint.cc +++ b/src/wccp/WccpEndPoint.cc @@ -721,7 +721,7 @@ CacheImpl::handleISeeYou(IpHeader const & /* ip_hdr ATS_UNUSED */, ts::Buffer co logf(LVL_DEBUG, "Received WCCP2_I_SEE_YOU for group %d.", group.m_svc.getSvcId()); - // Prefered address for router. + // Preferred address for router. uint32_t router_addr = msg.m_router_id.idElt().getAddr(); // Where we sent our packet. uint32_t to_addr = msg.m_router_id.getToAddr(); diff --git a/src/wccp/WccpLocal.h b/src/wccp/WccpLocal.h index 89dab2ebf2d..f73ce72aae9 100644 --- a/src/wccp/WccpLocal.h +++ b/src/wccp/WccpLocal.h @@ -25,6 +25,7 @@ #include "wccp/Wccp.h" #include "WccpUtil.h" #include "tscore/TsBuffer.h" +#include "ts/apidefs.h" // Needed for template use of byte ordering functions. #include #include @@ -179,7 +180,7 @@ struct RouterId { ); uint32_t m_addr; ///< Identifying router IP address. - uint32_t m_recv_id; ///< Recieve ID (sequence #). + uint32_t m_recv_id; ///< Receive ID (sequence #). }; /** Sect 5.7.1: Router Identity Element. @@ -485,7 +486,7 @@ class MaskValueSetElt ValueElt &operator[](int idx ///< Index of target element. ); //@} - /// Calcuate the size of an element with @a n values. + /// Calculate the size of an element with @a n values. static size_t calcSize(uint32_t n ///< Number of values. ); /// Get the size (length) of this element. @@ -1029,7 +1030,7 @@ class SecurityComp : public CompWithHeader static void setDefaultOption(Option opt ///< Type of security. ); - /// Set messsage local security key. + /// Set message local security key. self &setKey(const char *key ///< Shared key. ); @@ -1066,7 +1067,7 @@ class ServiceComp : public CompWithHeader struct raw_t : public super::raw_t, public ServiceGroup { }; - ServiceComp(); ///< Default constructor, no member intialization. + ServiceComp(); ///< Default constructor, no member initialization. /// @name Accessors //@{ @@ -1635,7 +1636,7 @@ class AltAssignComp : public CompWithHeader RouterAssignListElt m_routers; ///< Routers. }; - /// Force virtual desctructor. + /// Force virtual destructor. virtual ~AltAssignComp() {} /// @name Accessors //@{ @@ -1690,7 +1691,7 @@ class AltHashAssignComp : public AltAssignComp uint32_t getCacheCount() const; //@} - /// Force virtual desctructor. + /// Force virtual destructor. virtual ~AltHashAssignComp() {} /// Fill out the component from an @c Assignment. virtual self &fill(MsgBuffer &buffer, ///< Target storage. @@ -1723,7 +1724,7 @@ class AltMaskAssignComp : public AltAssignComp typedef AltMaskAssignComp self; ///< Self reference type. typedef AltAssignComp super; ///< Parent type. - /// Force virtual desctructor. + /// Force virtual destructor. virtual ~AltMaskAssignComp() {} /// Fill out the component from an @c Assignment. virtual self &fill(MsgBuffer &buffer, ///< Target storage. @@ -1774,7 +1775,7 @@ class CmdComp : public CompWithHeader //@} /// Write basic serialization data. - /// Elements must be filled in seperately and after invoking this method. + /// Elements must be filled in separately and after invoking this method. self &fill(MsgBuffer &buffer, ///< Component storage. cmd_t cmd, ///< Command type. uint32_t data ///< Command data. @@ -1860,7 +1861,7 @@ class QueryComp : public CompWithHeader uint32_t routerAddr, ///< Router identifying address. uint32_t toAddr, ///< Destination address. uint32_t cacheAddr, ///< Cache identifying address. - uint32_t recvId ///< Recieve ID. + uint32_t recvId ///< Receive ID. ); /// Validate an existing structure. @@ -2020,7 +2021,7 @@ class BaseMsg size_t getCount() const; /// Validate security option. - /// @note This presumes a sublcass has already successfully parsed. + /// @note This presumes a subclass has already successfully parsed. bool validateSecurity() const; // Common starting components for all messages. @@ -2072,7 +2073,7 @@ class ISeeYouMsg : public BaseMsg typedef ISeeYouMsg self; ///< Self reference type. /// Fill out message structure. - /// Router ID and view data must be filled in seperately. + /// Router ID and view data must be filled in separately. void fill(detail::router::GroupData const &group, ///< Service groupc context. SecurityOption sec_opt, ///< Security option. detail::Assignment &assign, ///< Cache assignment data. @@ -2212,7 +2213,7 @@ class Impl : public ts::IntrusivePtrCounter /// @return 0 for success, -errno on error. virtual int housekeeping() = 0; - /// Recieve and process a message. + /// Receive and process a message. /// @return 0 for success, -ERRNO on system error. virtual ts::Rv handleMessage(); @@ -2381,7 +2382,7 @@ namespace detail RouterBag::iterator findRouter(uint32_t addr ///< IP address of cache. ); - /// Set an intial router for a service group. + /// Set an initial router for a service group. self &seedRouter(uint32_t addr ///< IP address for router. ); /// Remove a seed router. @@ -2466,7 +2467,7 @@ class CacheImpl : public Impl ServiceGroup::Result *result = 0 ///< [out] Result for service creation. ); - /** Set an intial router for a service group. + /** Set an initial router for a service group. This is needed to bootstrap the protocol. If the router is already seeded, this call is silently ignored. */ diff --git a/src/wccp/WccpMsg.cc b/src/wccp/WccpMsg.cc index 0b64990481c..f054b862fe7 100644 --- a/src/wccp/WccpMsg.cc +++ b/src/wccp/WccpMsg.cc @@ -1796,7 +1796,7 @@ ISeeYouMsg::parse(ts::Buffer const &buffer) // Optional components. // Test for alternates here - // At most one of the asssignments but never both. + // At most one of the assignments but never both. // Can be omitted. m_assignment.parse(m_buffer); m_map.parse(m_buffer); diff --git a/src/wccp/WccpUtil.h b/src/wccp/WccpUtil.h index 3d986acda69..f8225257901 100644 --- a/src/wccp/WccpUtil.h +++ b/src/wccp/WccpUtil.h @@ -81,7 +81,7 @@ ts::Errata &logf(ts::Errata &err, ///< Target errata. ts::Errata log(ts::Errata::Code code, ///< Severity level. char const *text ///< Message text. ); -/// Return an Errata populated with a printf styleformatted string. +/// Return an Errata populated with a printf style formatted string. /// Use message ID 0. /// @return @a err. ts::Errata logf(ts::Errata::Code code, ///< Severity level. diff --git a/tests/README.md b/tests/README.md index 619c0c04249..0e092665de1 100644 --- a/tests/README.md +++ b/tests/README.md @@ -1,40 +1,45 @@ # Getting Started -This directory contains different tests for Apache Trafficserver. It is recommended that all test move to this common area under the correct location based on the type of test being added. +This directory contains different tests for Apache Trafficserver. It is recommended that all tests move to this common area under the correct location based on the type of test being added. ## Layout The current layout is: **gold_tests/** - contains all the TSQA v4 based tests that run on the Reusable Gold Testing System (AuTest) -**tools/** - contain programs used to help with testing. -In the future a directory called **"unit/"** will be added for adding unit tests based on some standardized testing system. +**tools/** - contains programs used to help with testing. +**include/** - contains headers used for unit testing. ## Scripts -To help with easy running of the tests, there is a autest.sh and bootstrap.py file. +To help with easy running of the tests, there is autest.sh and bootstrap.py. ### autest.sh -This file is a simple wrapper that will call the AuTest program in a python virtualenv. If the virtualenv is not setup it will try to install system. That will set up the Reusable Gold Testing System on most systems in a Python virtual environment. The wrapper add some basic options to the command to point to the location of the tests. Add --help for more details on options for running autest test system. +This file is a simple wrapper that will call the Reusable Gold Testing System (Autest) program in a python virtualenv. If the virtualenv is not setup, the script will try to install it on the system. That will set up the Autest on most systems in a Python virtual environment. The wrapper adds some basic options to the command to point to the location of the tests. Use --help for more details on options for running Autest. ### bootstrap.py -This script should try to install python35 or better on the system, and needed python packages for running the tests. +This script will try to install python35 or better on the system, and the needed python packages for running the tests. -# Advance setup +# Basic setup -AuTest can be install manually instead of using the wrapper script. The advange of this is that it is often easier to debug issues with the testing system, or the tests. There are two ways this can be done. -1. run the bootstrap script then source the path with a "source ./env-test/bin/activate" command. At this point autest command should run without the wrapper script -2. The other way is to make sure you install python 3.5 or better on your system. From there install these python packages ( ie pip install ): +AuTest can be run using the script file autest.sh listed above. Run the file from the tests/ directory followed by --ats-bin and the bin name. (ie ~/ats/bin) This will run the wrapper for the tests. See documentation for more details. + +# Advanced setup + +AuTest and the relevant tools can be install manually instead of using the wrapper script. By doing this, it is often easier to debug issues with the testing system, or the tests. There are two ways this can be done. +1. Run the bootstrap script then source the path with a "source ./env-test/bin/activate" command. At this point autest command should run without the wrapper script +2. Make sure you install python 3.5 or better on your system. From there install these python packages ( ie pip install ): - hyper - - git+https://bitbucket.org/dragon512/reusable-gold-testing-system.git + - git+https://bitbucket.org/autestsuite/reusable-gold-testing-system.git + - [traffic-replay](https://bitbucket.org/autestsuite/trafficreplay/src/master/) (This will automatically install [MicroDNS](https://bitbucket.org/autestsuite/microdns/src/master/), [MicroServer](https://bitbucket.org/autestsuite/microserver/src/master/), [TrafficReplayLibrary](https://bitbucket.org/autestsuite/trafficreplaylibrary/src/master/), and dnslib as part of the dependencies.) -# Writting tests for AuTest -When writting for the AuTest system please refer to the current documenation on the [online wiki](https://bitbucket.org/dragon512/reusable-gold-testing-system/wiki/Home) for general use of the system. +# Writing tests for AuTest +When writing for the AuTest system please refer to the current [Online Documentation](https://autestsuite.bitbucket.io/) for general use of the system. -## Documenation of AuTest extension for ATS. -Autest allows for the creation of extension to help specilaize and simplify test writting for a given application domian. Minus API addition the extension code will check that python 3.5 or better is used. There is also a new command line argumented added: +## Documentation of AuTest extension for ATS. +Autest allows for the creation of extensions to help specialize and simplify test writing for a given application domain. Minus API addition the extension code will check that python 3.5 or better is used. There is also a new command line argumented added specifically for Trafficserver: --ats-bin < path to bin directory > @@ -45,7 +50,7 @@ This command line argument will point to your build of ATS you want to test. At * command - optional argument defining what process to use. Defaults to traffic_server. * select_ports - have the testing system auto select the ports to use for this instance of ATS -This function will define a sandbox for an instance of trafficserver to run under. The function will return a AuTest process object that will have a number of files and variables define for making it easier to define a test. +This function will define a sandbox for an instance of trafficserver to run under. The function will return a AuTest process object that will have a number of files and variables defined to make it easier for test definition. #### Environment The environment of the process will have a number of added environment variables to control trafficserver running the in the sandbox location correctly. This can be used to easily setup other commands that should run under same environment. @@ -64,15 +69,14 @@ tr.Processes.Default.Env=ts.Env ``` #### Variables -These are the current variable that are define dynamically +These are the current variables that are defined dynamically for Trafficserver port - the ipv4 port to listen on -portv6 - the ipv4 port to listen on -manager_port - the manager port used. This is set even is select_port is False +portv6 - the ipv6 port to listen on admin_port - the admin port used. This is set even is select_port is False #### File objects -A number of file object are define to help with adding values to a given configuration value to for a test, or testing a value exists in a log file. File that are defined currently are: +A number of file objects are defined to help with adding values to a given configuration value to for a test, or testing a value exists in a log file. File that are defined currently are: ##### log files * squid.log @@ -291,11 +295,7 @@ ts.Disk.remap_config.AddLine( * TS_HAS_SO_MARK * TS_HAS_IP_TOS * TS_USE_HWLOC - * TS_USE_TLS_NPN - * TS_USE_TLS_ALPN - * TS_USE_CERT_CB * TS_USE_SET_RBIO - * TS_USE_TLS_ECKEY * TS_USE_LINUX_NATIVE_AIO * TS_HAS_SO_PEERCRED * TS_USE_REMOTE_UNWINDING @@ -308,7 +308,7 @@ ts.Disk.remap_config.AddLine( ```python #create the origin server process Test.SkipUnless( - Condition.HasATSFeature('TS_USE_TLS_ALPN'), + Condition.HasATSFeature('TS_USE_LINUX_NATIVE_AIO'), ) ``` diff --git a/tests/bootstrap.py b/tests/bootstrap.py index 4438b287603..cfef1a133cd 100755 --- a/tests/bootstrap.py +++ b/tests/bootstrap.py @@ -26,11 +26,12 @@ import sys pip_packages = [ - "autest==1.7.0", + "autest==1.7.2", "hyper", "requests", "dnslib", "httpbin", + "traffic-replay" # this should install TRLib, MicroServer, MicroDNS, Traffic-Replay ] diff --git a/tests/gold_tests/autest-site/conditions.test.ext b/tests/gold_tests/autest-site/conditions.test.ext index 9cac02f78b8..e60c06ca52c 100644 --- a/tests/gold_tests/autest-site/conditions.test.ext +++ b/tests/gold_tests/autest-site/conditions.test.ext @@ -19,6 +19,9 @@ def HasOpenSSLVersion(self, version): return self.EnsureVersion(["openssl","version"],min_version=version) +def HasCurlVersion(self, version): + return self.EnsureVersion(["curl","--version"],min_version=version) + def HasCurlFeature(self, feature): def default(output): @@ -60,6 +63,7 @@ def PluginExists(self, pluginname): ExtendCondition(HasOpenSSLVersion) ExtendCondition(HasATSFeature) +ExtendCondition(HasCurlVersion) ExtendCondition(HasCurlFeature) ExtendCondition(PluginExists) diff --git a/tests/gold_tests/autest-site/init.cli.ext b/tests/gold_tests/autest-site/init.cli.ext index 28ed5748e30..6d7e89b13e2 100644 --- a/tests/gold_tests/autest-site/init.cli.ext +++ b/tests/gold_tests/autest-site/init.cli.ext @@ -23,7 +23,7 @@ if sys.version_info < (3, 5, 0): host.WriteError( "You need python 3.5 or later to run these tests\n", show_stack=False) -autest_version ="1.7.0" +autest_version ="1.7.2" if AuTestVersion() < autest_version: host.WriteError( "Tests need AuTest version {ver} or better\n Please update AuTest:\n pip install --upgrade autest\n".format(ver=autest_version), show_stack=False) diff --git a/tests/gold_tests/autest-site/microDNS.test.ext b/tests/gold_tests/autest-site/microDNS.test.ext index e45a496c090..fdeffe00088 100644 --- a/tests/gold_tests/autest-site/microDNS.test.ext +++ b/tests/gold_tests/autest-site/microDNS.test.ext @@ -16,12 +16,16 @@ # See the License for the specific language governing permissions and # limitations under the License. -from ports import get_port import json import os import sys +import trlib.ipconstants as IPConstants +from ports import get_port + # AddRecord registers a list of ip address against hostname + + def AddRecord(hostname, list_ip_addr): record = dict() @@ -69,7 +73,6 @@ def addRecords(self, records=None, jsonFile=None): def MakeDNServer(obj, name, filename="dns_file.json", port=False, ip='INADDR_LOOPBACK', rr=False, default=None, options={}): - server_path = os.path.join(obj.Variables.AtsTestToolsDir, 'microDNS/uDNS.py') data_dir = os.path.join(obj.RunDirectory, name) filepath = os.path.join(data_dir, filename) obj.Variables.zone_file = filepath @@ -90,7 +93,7 @@ def MakeDNServer(obj, name, filename="dns_file.json", port=False, ip='INADDR_LOO p = obj.Processes.Process(name) if (port == False): port = get_port(p, "Port") - command = "python3 {0} {1} {2} {3}".format(server_path, ip, port, filepath) + command = "microdns {0} {1} {2}".format(ip, port, filepath) if rr: command += " --rr" @@ -101,10 +104,6 @@ def MakeDNServer(obj, name, filename="dns_file.json", port=False, ip='INADDR_LOO p.Variables.DataDir = data_dir p.ReturnCode = 0 - # to get the IP keywords in tools/lib - sys.path.append(obj.Variables.AtsTestToolsDir) - import lib.IPConstants as IPConstants - if IPConstants.isIPv6(ip): p.Ready = When.PortOpenv6(port) else: diff --git a/tests/gold_tests/autest-site/microserver.test.ext b/tests/gold_tests/autest-site/microserver.test.ext index 6fe93df9f52..bef027f09c7 100644 --- a/tests/gold_tests/autest-site/microserver.test.ext +++ b/tests/gold_tests/autest-site/microserver.test.ext @@ -16,14 +16,20 @@ # See the License for the specific language governing permissions and # limitations under the License. -from autest.api import AddWhenFunction -from ports import get_port import json import socket import ssl import time import sys +from autest.api import AddWhenFunction +from ports import get_port + +import trlib.ipconstants as IPConstants +from trlib import Transaction, Request, Response, Session + +DEFAULT_LOOKUP_KEY = '{PATH}' + def addMethod(self, testName, request_header, functionName): return @@ -52,51 +58,15 @@ def getHeaderFieldVal(request_header, field): return val # addResponse adds customized response with respect to request_header. request_header and response_header are both dictionaries +def addResponse(self, filename, request_header, response_header): + client_request = Request.fromRequestLine(request_header["headers"], request_header["body"], None if "options" not in request_header else request_header["options"]) + server_response = Response.fromRequestLine(response_header["headers"], response_header["body"], None if "options" not in response_header else response_header["options"]) + # timestamp field is left None because that needs to be revised for better implementation + txn = Transaction(client_request, None, server_response, None, None, None) -def addResponse(self, filename, request_header, response_header): - requestline = request_header["headers"].split("\r\n")[0] - host_ = "" - path_ = "" - if requestline: - url_part = requestline.split(" ") - if len(url_part) > 1: - if url_part[1].startswith("http"): - path_ = url_part[1].split("/", 2)[2] - host_, path_ = path_.split("/", 1) - else: - path_ = url_part[1].split("/", 1)[1] - - kpath = "" - - argsList = [] - keyslist = self.Variables.lookup_key.split("}") - for keystr in keyslist: - if keystr == '{PATH': - kpath = kpath + path_ - continue - if keystr == '{HOST': - kpath = kpath + host_ - continue - if keystr == '': # empty - continue - stringk = keystr.replace("{%", "") - argsList.append(stringk) - KeyList = [] - for argsL in argsList: - field_val = getHeaderFieldVal(request_header, argsL) - if field_val != None: - KeyList.append(field_val) - rl = "".join(KeyList) + kpath - txn = dict() - txn["timestamp"] = "" - txn["uuid"] = rl - txn["request"] = request_header - txn["response"] = response_header absFilepath = os.path.join(self.Variables.DataDir, filename) addTransactionToSession(txn, absFilepath) - # absFilepath=os.path.abspath(filename) - # self.Setup.CopyAs(absFilepath,self.Variables.DataDir) return # adds transaction in json format to the specified file @@ -110,18 +80,25 @@ def addTransactionToSession(txn, JFile): jf = open(JFile, 'r') jsondata = json.load(jf) + # hard coding only 1 session per file + # since for the purpose of testing, we don't need multiple sessions in a file if jsondata == None: - jsondata = dict() - jsondata["version"] = '0.2' - jsondata["timestamp"] = "1234567890.098" - jsondata["encoding"] = "url_encoded" - jsondata["txns"] = list() - jsondata["txns"].append(txn) + jsondata = {} + jsondata["sessions"] = [] + + jsondata["sessions"].append(Session(JFile.split("/")[-1], None, None, [txn]).toJSON()) + jsondata["meta"] = {} + jsondata["meta"]["version"] = "1.0" else: - jsondata["txns"].append(txn) + # hardcoding 0 because for testing we only have 1 session + jsondata["sessions"][0]["transactions"].append(txn.toJSON()) + with open(JFile, 'w+') as jf: jf.write(json.dumps(jsondata)) +def addSessionFromFiles(self, session_dir): + self.Setup.Copy(session_dir, self.Variables.DataDir) + # make headers with the key and values provided def makeHeader(self, requestString, **kwargs): @@ -132,73 +109,83 @@ def makeHeader(self, requestString, **kwargs): return headerStr -def uServerUpAndRunning(host, port, isSsl, isIPv6, clientcert='', clientkey=''): +def uServerUpAndRunning(serverHost, port, isSsl, isIPv6, request, clientcert='', clientkey=''): if isIPv6: plain_sock = socket.socket(socket.AF_INET6) else: plain_sock = socket.socket(socket.AF_INET) - if isSsl: - if clientcert != '' or clientkey != '': - sock = ssl.wrap_socket(plain_sock, keyfile=clientkey, certfile=clientcert) - else: - sock = ssl.wrap_socket(plain_sock) + if isSsl: + if clientcert != '' or clientkey != '': + sock = ssl.wrap_socket(plain_sock, keyfile=clientkey, certfile=clientcert) + else: + sock = ssl.wrap_socket(plain_sock) else: - sock = plain_sock + sock = plain_sock try: - sock.connect((host, port)) + sock.connect((serverHost, port)) except ConnectionRefusedError: return False - sock.sendall("GET /ruok HTTP/1.1\r\nHost: {}\r\n\r\n".format(host).encode()) - decoded_output='' + sock.sendall(request.encode()) + decoded_output = '' while True: + host.WriteDebug("??") output = sock.recv(4096) # suggested bufsize from docs.python.org + host.WriteDebug("!!") if len(output) <= 0: break else: - decoded_output+=output.decode() + decoded_output += output.decode() sock.close() sock = None - expected_response="HTTP/1.1 200 OK\r\nConnection: close\r\nContent-Length: 4\r\n\r\nimok" + expected_response = "HTTP/1.1 200 OK\r\nConnection: close\r\nContent-Length: 4\r\n\r\nimok" if decoded_output == expected_response: return True - raise RuntimeError('\n'.join([ - 'Got invalid response from microserver:', - '----', - decoded_output, - '----'])) -AddWhenFunction(uServerUpAndRunning) + host.WriteError('\n'.join([ + 'Got invalid response from microserver:', + '----', + decoded_output, + '----'])) + + +AddWhenFunction(uServerUpAndRunning) -def MakeOriginServer(obj, name, port=False, ip='INADDR_LOOPBACK', delay=False, ssl=False, lookup_key='{PATH}', mode='test', options={}, clientcert='', clientkey=''): - # to get the IP keywords in tools/lib - sys.path.append(obj.Variables.AtsTestToolsDir) - import lib.IPConstants as IPConstants - server_path = os.path.join(obj.Variables.AtsTestToolsDir, 'microServer/uWServer.py') +def MakeOriginServer(obj, name, port=None, s_port=None, ip='INADDR_LOOPBACK', delay=None, ssl=False, lookup_key=DEFAULT_LOOKUP_KEY, clientcert='', clientkey='', both=False, options={}): data_dir = os.path.join(obj.RunDirectory, name) - # create Process p = obj.Processes.Process(name) - if (port == False): - port = get_port(p, "Port") - ipaddr = IPConstants.getIP(ip) - if (delay == False): - delay = 0 + command = "microserver --data-dir {0} --ip_address {1} --lookupkey '{2}'".format(data_dir, ipaddr, lookup_key) + + if delay: + command += " --delay {0}".format(delay) + + if both or ssl: + if not s_port: + s_port = get_port(p, "SSL_Port") - command = "python3 {0} --data-dir {1} --port {2} --ip_address {3} --delay {4} -m test --lookupkey '{5}' -m {6}".format( - server_path, data_dir, port, ipaddr, delay, lookup_key, mode) + command += " --both" if both else " --ssl" + key = clientkey if clientkey else os.path.join(obj.Variables["AtsTestToolsDir"], "microserver", "ssl", "server.pem") + cert = clientcert if clientcert else os.path.join(obj.Variables["AtsTestToolsDir"], "microserver", "ssl", "server.crt") + command += " --key {0}".format(key) + command += " --cert {0}".format(cert) + command += " --s_port {0}".format(s_port) - if ssl: - command += " --ssl True" + # this might break if user specifies both both and ssl + if not ssl: # in both or HTTP only mode + if not port: + port = get_port(p, "Port") + + command += " --port {0}".format(port) for flag, value in options.items(): - command += " {} {}".format(flag, value) + command += " {} {}".format(flag, value if value else '') p.Command = command p.Setup.MakeDir(data_dir) @@ -206,20 +193,33 @@ def MakeOriginServer(obj, name, port=False, ip='INADDR_LOOPBACK', delay=False, s p.Variables.lookup_key = lookup_key AddMethodToInstance(p, addResponse) AddMethodToInstance(p, addTransactionToSession) + AddMethodToInstance(p, addSessionFromFiles) - # Set up health check. - addResponse(p, "healthcheck.json", { - "headers": "GET /ruok HTTP/1.1\r\nHost: {}\r\n\r\n".format(ipaddr), + custom_lookup_header = '' + keys = lookup_key.split("}") + + for key in keys: + if key not in ['{PATH', '{URL', '{HOST', '{%Host']: + k = key.replace("{%", "") + + if len(k) > 0: + custom_lookup_header += '{0}: healthcheck\r\n'.format(k) + + healthcheck_request = { + "headers": "GET /ruok HTTP/1.1\r\nHost: {0}\r\n{1}\r\n".format(ipaddr, custom_lookup_header), "timestamp": "1469733493.993", "body": "" - }, { + } + + # Set up health check. + addResponse(p, "healthcheck.json", healthcheck_request, { "headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": "imok", - "options": "skipHooks" + "options": {"skipHooks": None} }) - p.Ready = When.uServerUpAndRunning(ipaddr, port, ssl, IPConstants.isIPv6(ip), clientcert=clientcert, clientkey=clientkey) + p.Ready = When.uServerUpAndRunning(ipaddr, s_port if ssl else port, ssl, IPConstants.isIPv6(ip), healthcheck_request["headers"], clientcert=clientcert, clientkey=clientkey) p.ReturnCode = Any(None, 0) return p diff --git a/tests/gold_tests/autest-site/traffic_replay.test.ext b/tests/gold_tests/autest-site/traffic_replay.test.ext new file mode 100644 index 00000000000..340fae93d48 --- /dev/null +++ b/tests/gold_tests/autest-site/traffic_replay.test.ext @@ -0,0 +1,91 @@ +''' +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# default 'mixed' for connection type since it doesn't hurt +def Replay(obj, name, replay_dir, key=None, cert=None, conn_type='mixed', options={}): + # ATS setup - one line because we leave records and remap config to user + ts = obj.MakeATSProcess("ts", select_ports=False) # select ports can be disabled once we add ssl port selection in extension + + # TEMP + ts.Variables.ssl_port = 4443 + + ts.addSSLfile(os.path.join(obj.Variables["AtsTestToolsDir"], "microserver", "ssl", "server.pem")) + ts.addSSLfile(os.path.join(obj.Variables["AtsTestToolsDir"], "microserver", "ssl", "server.crt")) + + ts.Disk.ssl_multicert_config.AddLine( + 'dest_ip=* ssl_cert_name=server.pem ssl_key_name=server.pem' + ) + + # MicroServer setup - NOTE: expand to multiple microserver in future? + server = obj.MakeOriginServer("server", both=True, lookup_key='{%uuid}') + server.addSessionFromFiles(replay_dir) + + # MicroDNS setup + dns = obj.MakeDNServer("dns", default=['127.0.0.1']) + + # Traffic Replay setup + data_dir = os.path.join(obj.RunDirectory, name) + + # NOTE: we are forcing mixed connection types for now for the sake of simplicity + + # if conn_type != 'nossl': + # if not key: + # host.WriteError("Must provide SSL key to traffic-replay.") + + # if not cert: + # host.WriteError("Must provide SSL key to traffic-replay.") + + # if not ts.Variables.ssl_port: + # host.WriteError("Must set traffic server with an ssl port") + + # NOTE: does this need change? + hostIP = '127.0.0.1' + + if not key: + key = os.path.join(obj.Variables["AtsTestToolsDir"], "microserver", "ssl", "server.pem") + + if not cert: + cert = os.path.join(obj.Variables["AtsTestToolsDir"], "microserver", "ssl", "server.crt") + + command = 'traffic-replay --log_dir {0} --type {1} --verify --host {2} --port {3} --s_port {4} '.format(data_dir, conn_type, hostIP, ts.Variables.port, ts.Variables.ssl_port) + + if key: + command += "-k {0} ".format(key) + + if cert: + command += "--ca_cert {0} ".format(cert) + + if options: + for flag, value in options.items(): + command += "{} {} ".format(flag, value if value else '') + + tr = obj.AddTestRun(name) + tr.Command = command + # tr.Command = "echo Hi" + tr.Setup.MakeDir(data_dir) + tr.Setup.Copy(replay_dir, data_dir) + tr.Processes.Default.StartBefore(server) + tr.Processes.Default.StartBefore(ts, ready=When.PortOpen(ts.Variables.ssl_port)) + tr.Processes.Default.StartBefore(dns) + tr.ReturnCode = Any(None, 0) + tr.Processes.Default.Streams.All = Testers.ExcludesExpression("FAIL", "No fails allowed.") + + # return all the stuff in case user wants to do extra optimization + return (ts, server, dns, tr) + +AddTestRunSet(Replay) diff --git a/tests/gold_tests/autest-site/trafficserver.test.ext b/tests/gold_tests/autest-site/trafficserver.test.ext index 64ca9221900..7ae7626440a 100755 --- a/tests/gold_tests/autest-site/trafficserver.test.ext +++ b/tests/gold_tests/autest-site/trafficserver.test.ext @@ -254,15 +254,12 @@ def MakeATSProcess(obj, name, command='traffic_server', select_ports=True): else: p.Variables.port = 8080 p.Variables.portv6 = 8080 - get_port(p, "manager_port") get_port(p, "admin_port") # set the ports if select_ports: p.Env['PROXY_CONFIG_HTTP_SERVER_PORTS'] = "{0} {1}:ipv6".format( p.Variables.port, p.Variables.portv6) # your own listen port - p.Env['PROXY_CONFIG_PROCESS_MANAGER_MGMT_PORT'] = str( - p.Variables.manager_port) p.Env['PROXY_CONFIG_ADMIN_SYNTHETIC_PORT'] = str(p.Variables.admin_port) p.Env['PROXY_CONFIG_ADMIN_AUTOCONF_PORT'] = str( p.Variables.admin_port) # support pre ATS 6.x diff --git a/tests/gold_tests/cache/cache-control.test.py b/tests/gold_tests/cache/cache-control.test.py index ed0f24ecbae..e2c9fa4f534 100644 --- a/tests/gold_tests/cache/cache-control.test.py +++ b/tests/gold_tests/cache/cache-control.test.py @@ -53,6 +53,7 @@ 'proxy.config.http.response_via_str': 3, 'proxy.config.http.cache.http': 1, 'proxy.config.http.wait_for_cache': 1, + 'proxy.config.http.insert_age_in_response': 0, }) ts.Disk.remap_config.AddLine( diff --git a/tests/gold_tests/cache/gold/cache_and_req_body-hit.gold b/tests/gold_tests/cache/gold/cache_and_req_body-hit.gold index c10c0ba0705..34bb179c50b 100644 --- a/tests/gold_tests/cache/gold/cache_and_req_body-hit.gold +++ b/tests/gold_tests/cache/gold/cache_and_req_body-hit.gold @@ -2,7 +2,6 @@ HTTP/1.1 200 OK Cache-Control: max-age=10,public Content-Length: 11 Date: `` -Age: `` Connection: keep-alive Via: `` Server: `` diff --git a/tests/gold_tests/cache/gold/cache_and_req_body-miss.gold b/tests/gold_tests/cache/gold/cache_and_req_body-miss.gold index 0f8938e1c50..33efb4bac07 100644 --- a/tests/gold_tests/cache/gold/cache_and_req_body-miss.gold +++ b/tests/gold_tests/cache/gold/cache_and_req_body-miss.gold @@ -2,7 +2,6 @@ HTTP/1.1 200 OK Cache-Control: max-age=10,public Content-Length: 11 Date: `` -Age: `` Connection: keep-alive Via: `` Server: `` diff --git a/tests/gold_tests/cache/gold/cache_hit_stale.gold b/tests/gold_tests/cache/gold/cache_hit_stale.gold index 094162645e5..25a3d6670ca 100644 --- a/tests/gold_tests/cache/gold/cache_hit_stale.gold +++ b/tests/gold_tests/cache/gold/cache_hit_stale.gold @@ -2,7 +2,6 @@ HTTP/1.1 200 OK Cache-Control: max-age=10,public Content-Length: 11 Date: `` -Age: 0 Connection: keep-alive Via: `` Server: `` diff --git a/tests/gold_tests/cache/gold/cache_no_cc.gold b/tests/gold_tests/cache/gold/cache_no_cc.gold index 9bca02d5f45..7f50b005e62 100644 --- a/tests/gold_tests/cache/gold/cache_no_cc.gold +++ b/tests/gold_tests/cache/gold/cache_no_cc.gold @@ -1,7 +1,6 @@ HTTP/1.1 200 OK Content-Length: 14 Date: `` -Age: 0 Connection: keep-alive Via: `` Server: `` diff --git a/tests/gold_tests/chunked_encoding/chunked_encoding.test.py b/tests/gold_tests/chunked_encoding/chunked_encoding.test.py index f511a215ece..5803d626a26 100644 --- a/tests/gold_tests/chunked_encoding/chunked_encoding.test.py +++ b/tests/gold_tests/chunked_encoding/chunked_encoding.test.py @@ -83,7 +83,7 @@ 'map http://www.yetanotherexample.com http://127.0.0.1:{0}'.format(server3.Variables.Port) ) ts.Disk.remap_config.AddLine( - 'map https://www.anotherexample.com https://127.0.0.1:{0}'.format(server2.Variables.Port, ts.Variables.ssl_port) + 'map https://www.anotherexample.com https://127.0.0.1:{0}'.format(server2.Variables.SSL_Port, ts.Variables.ssl_port) ) diff --git a/tests/gold_tests/cont_schedule/gold/http_200.gold b/tests/gold_tests/cont_schedule/gold/http_200.gold new file mode 100644 index 00000000000..f3752f16420 --- /dev/null +++ b/tests/gold_tests/cont_schedule/gold/http_200.gold @@ -0,0 +1,9 @@ +`` +< HTTP/1.1 200 OK +< Date: `` +< Age: `` +< Transfer-Encoding: chunked +< Proxy-Connection: keep-alive +< Server: ATS/`` +< +`` diff --git a/tests/gold_tests/cont_schedule/gold/schedule.gold b/tests/gold_tests/cont_schedule/gold/schedule.gold new file mode 100644 index 00000000000..7b5d1f6a2b2 --- /dev/null +++ b/tests/gold_tests/cont_schedule/gold/schedule.gold @@ -0,0 +1,4 @@ +`` +``(TSContSchedule_test.check) pass [should be the same thread] +``(TSContSchedule_test.check) pass [should not be the same thread] +`` diff --git a/tests/gold_tests/cont_schedule/gold/schedule_on_pool.gold b/tests/gold_tests/cont_schedule/gold/schedule_on_pool.gold new file mode 100644 index 00000000000..df9cf85eaca --- /dev/null +++ b/tests/gold_tests/cont_schedule/gold/schedule_on_pool.gold @@ -0,0 +1,8 @@ +`` +``ET_NET``TSContScheduleOnPool handler 1`` +``ET_NET``TSContScheduleOnPool handler 1`` +``(TSContSchedule_test.check) pass [should not be the same thread] +``ET_TASK``TSContScheduleOnPool handler 2`` +``ET_TASK``TSContScheduleOnPool handler 2`` +``(TSContSchedule_test.check) pass [should be the same thread] +`` diff --git a/tests/gold_tests/cont_schedule/gold/schedule_on_thread.gold b/tests/gold_tests/cont_schedule/gold/schedule_on_thread.gold new file mode 100644 index 00000000000..7b5d1f6a2b2 --- /dev/null +++ b/tests/gold_tests/cont_schedule/gold/schedule_on_thread.gold @@ -0,0 +1,4 @@ +`` +``(TSContSchedule_test.check) pass [should be the same thread] +``(TSContSchedule_test.check) pass [should not be the same thread] +`` diff --git a/tests/gold_tests/cont_schedule/gold/thread_affinity.gold b/tests/gold_tests/cont_schedule/gold/thread_affinity.gold new file mode 100644 index 00000000000..09a79cf83ea --- /dev/null +++ b/tests/gold_tests/cont_schedule/gold/thread_affinity.gold @@ -0,0 +1,5 @@ +`` +``pass [affinity thread is not null] +``pass [affinity thread is cleared] +``pass [affinity thread is set] +`` diff --git a/tests/gold_tests/cont_schedule/schedule.test.py b/tests/gold_tests/cont_schedule/schedule.test.py new file mode 100644 index 00000000000..e2867ef1fd3 --- /dev/null +++ b/tests/gold_tests/cont_schedule/schedule.test.py @@ -0,0 +1,72 @@ +''' +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import os + +Test.Summary = 'Test TSContSchedule API' +Test.SkipUnless(Condition.HasProgram('curl', 'Curl need to be installed on system for this test to work')) + +Test.ContinueOnFail = True + +# Define default ATS +ts = Test.MakeATSProcess('ts') +server = Test.MakeOriginServer('server') + +Test.testName = '' +request_header = { + 'headers': 'GET / HTTP/1.1\r\nHost: www.example.com\r\n\r\n', + 'timestamp': '1469733493.993', + 'body': '' +} +response_header = { + 'headers': 'HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n', + 'timestamp': '1469733493.993', + 'body': '' +} +server.addResponse("sessionfile.log", request_header, response_header) + +ts.Disk.records_config.update({ + 'proxy.config.exec_thread.autoconfig': 0, + 'proxy.config.exec_thread.autoconfig.scale': 1.5, + 'proxy.config.exec_thread.limit': 32, + 'proxy.config.accept_threads': 1, + 'proxy.config.task_threads': 2, + 'proxy.config.diags.debug.enabled': 1, + 'proxy.config.diags.debug.tags': 'TSContSchedule_test' +}) +ts.Disk.remap_config.AddLine( + 'map / http://127.0.0.1:{0}'.format(server.Variables.Port) +) + +# Load plugin +Test.PreparePlugin(os.path.join(Test.Variables.AtsTestToolsDir, 'plugins', 'cont_schedule.cc'), ts) + +# www.example.com Host +tr = Test.AddTestRun() +tr.Processes.Default.Command = 'curl --proxy 127.0.0.1:{0} "http://www.example.com" -H "Proxy-Connection: Keep-Alive" --verbose'.format(ts.Variables.port) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.StartBefore(ts) +tr.Processes.Default.StartBefore(server) +tr.Processes.Default.Streams.stderr = 'gold/http_200.gold' +tr.StillRunningAfter = ts +tr.StillRunningAfter = server + +# Check Plugin Results +ts.Streams.All = "gold/schedule.gold" +ts.Streams.All += Testers.ExcludesExpression('fail', 'should not contain "fail"') diff --git a/tests/gold_tests/cont_schedule/schedule_on_pool.test.py b/tests/gold_tests/cont_schedule/schedule_on_pool.test.py new file mode 100644 index 00000000000..9fb54197b36 --- /dev/null +++ b/tests/gold_tests/cont_schedule/schedule_on_pool.test.py @@ -0,0 +1,72 @@ +''' +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import os + +Test.Summary = 'Test TSContScheduleOnPool API' +Test.SkipUnless(Condition.HasProgram('curl', 'Curl need to be installed on system for this test to work')) + +Test.ContinueOnFail = True + +# Define default ATS +ts = Test.MakeATSProcess('ts') +server = Test.MakeOriginServer('server') + +Test.testName = '' +request_header = { + 'headers': 'GET / HTTP/1.1\r\nHost: www.example.com\r\n\r\n', + 'timestamp': '1469733493.993', + 'body': '' +} +response_header = { + 'headers': 'HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n', + 'timestamp': '1469733493.993', + 'body': '' +} +server.addResponse("sessionfile.log", request_header, response_header) + +ts.Disk.records_config.update({ + 'proxy.config.exec_thread.autoconfig': 0, + 'proxy.config.exec_thread.autoconfig.scale': 1.5, + 'proxy.config.exec_thread.limit': 32, + 'proxy.config.accept_threads': 1, + 'proxy.config.task_threads': 2, + 'proxy.config.diags.debug.enabled': 1, + 'proxy.config.diags.debug.tags': 'TSContSchedule_test' +}) +ts.Disk.remap_config.AddLine( + 'map / http://127.0.0.1:{0}'.format(server.Variables.Port) +) + +# Load plugin +Test.PreparePlugin(os.path.join(Test.Variables.AtsTestToolsDir, 'plugins', 'cont_schedule.cc'), ts, 'pool') + +# www.example.com Host +tr = Test.AddTestRun() +tr.Processes.Default.Command = 'curl --proxy 127.0.0.1:{0} "http://www.example.com" -H "Proxy-Connection: Keep-Alive" --verbose'.format(ts.Variables.port) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.StartBefore(ts) +tr.Processes.Default.StartBefore(server) +tr.Processes.Default.Streams.stderr = 'gold/http_200.gold' +tr.StillRunningAfter = ts +tr.StillRunningAfter = server + +# Check Plugin Results +ts.Streams.All = "gold/schedule_on_pool.gold" +ts.Streams.All += Testers.ExcludesExpression('fail', 'should not contain "fail"') diff --git a/tests/gold_tests/cont_schedule/schedule_on_thread.test.py b/tests/gold_tests/cont_schedule/schedule_on_thread.test.py new file mode 100644 index 00000000000..5b5ab5f69d7 --- /dev/null +++ b/tests/gold_tests/cont_schedule/schedule_on_thread.test.py @@ -0,0 +1,72 @@ +''' +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import os + +Test.Summary = 'Test TSContScheduleOnThread API' +Test.SkipUnless(Condition.HasProgram('curl', 'Curl need to be installed on system for this test to work')) + +Test.ContinueOnFail = True + +# Define default ATS +ts = Test.MakeATSProcess('ts') +server = Test.MakeOriginServer('server') + +Test.testName = '' +request_header = { + 'headers': 'GET / HTTP/1.1\r\nHost: www.example.com\r\n\r\n', + 'timestamp': '1469733493.993', + 'body': '' +} +response_header = { + 'headers': 'HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n', + 'timestamp': '1469733493.993', + 'body': '' +} +server.addResponse("sessionfile.log", request_header, response_header) + +ts.Disk.records_config.update({ + 'proxy.config.exec_thread.autoconfig': 0, + 'proxy.config.exec_thread.autoconfig.scale': 1.5, + 'proxy.config.exec_thread.limit': 32, + 'proxy.config.accept_threads': 1, + 'proxy.config.task_threads': 2, + 'proxy.config.diags.debug.enabled': 1, + 'proxy.config.diags.debug.tags': 'TSContSchedule_test' +}) +ts.Disk.remap_config.AddLine( + 'map / http://127.0.0.1:{0}'.format(server.Variables.Port) +) + +# Load plugin +Test.PreparePlugin(os.path.join(Test.Variables.AtsTestToolsDir, 'plugins', 'cont_schedule.cc'), ts, 'thread') + +# www.example.com Host +tr = Test.AddTestRun() +tr.Processes.Default.Command = 'curl --proxy 127.0.0.1:{0} "http://www.example.com" -H "Proxy-Connection: Keep-Alive" --verbose'.format(ts.Variables.port) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.StartBefore(ts) +tr.Processes.Default.StartBefore(server) +tr.Processes.Default.Streams.stderr = 'gold/http_200.gold' +tr.StillRunningAfter = ts +tr.StillRunningAfter = server + +# Check Plugin Results +ts.Streams.All = "gold/schedule_on_thread.gold" +ts.Streams.All += Testers.ExcludesExpression('fail', 'should not contain "fail"') diff --git a/tests/gold_tests/cont_schedule/thread_affinity.test.py b/tests/gold_tests/cont_schedule/thread_affinity.test.py new file mode 100644 index 00000000000..8b91d5ac231 --- /dev/null +++ b/tests/gold_tests/cont_schedule/thread_affinity.test.py @@ -0,0 +1,72 @@ +''' +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import os + +Test.Summary = 'Test TSContThreadAffinity APIs' +Test.SkipUnless(Condition.HasProgram('curl', 'Curl need to be installed on system for this test to work')) + +Test.ContinueOnFail = True + +# Define default ATS +ts = Test.MakeATSProcess('ts') +server = Test.MakeOriginServer('server') + +Test.testName = '' +request_header = { + 'headers': 'GET / HTTP/1.1\r\nHost: www.example.com\r\n\r\n', + 'timestamp': '1469733493.993', + 'body': '' +} +response_header = { + 'headers': 'HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n', + 'timestamp': '1469733493.993', + 'body': '' +} +server.addResponse("sessionfile.log", request_header, response_header) + +ts.Disk.records_config.update({ + 'proxy.config.exec_thread.autoconfig': 0, + 'proxy.config.exec_thread.autoconfig.scale': 1.5, + 'proxy.config.exec_thread.limit': 32, + 'proxy.config.accept_threads': 1, + 'proxy.config.task_threads': 2, + 'proxy.config.diags.debug.enabled': 1, + 'proxy.config.diags.debug.tags': 'TSContSchedule_test' +}) +ts.Disk.remap_config.AddLine( + 'map / http://127.0.0.1:{0}'.format(server.Variables.Port) +) + +# Load plugin +Test.PreparePlugin(os.path.join(Test.Variables.AtsTestToolsDir, 'plugins', 'cont_schedule.cc'), ts, 'affinity') + +# www.example.com Host +tr = Test.AddTestRun() +tr.Processes.Default.Command = 'curl --proxy 127.0.0.1:{0} "http://www.example.com" -H "Proxy-Connection: Keep-Alive" --verbose'.format(ts.Variables.port) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.StartBefore(ts) +tr.Processes.Default.StartBefore(server) +tr.Processes.Default.Streams.stderr = 'gold/http_200.gold' +tr.StillRunningAfter = ts +tr.StillRunningAfter = server + +# Check Plugin Results +ts.Streams.All = "gold/thread_affinity.gold" +ts.Streams.All += Testers.ExcludesExpression('fail', 'should not contain "fail"') diff --git a/tests/gold_tests/continuations/double.test.py b/tests/gold_tests/continuations/double.test.py index b3f98d0823a..f6ba7053603 100644 --- a/tests/gold_tests/continuations/double.test.py +++ b/tests/gold_tests/continuations/double.test.py @@ -17,112 +17,125 @@ # limitations under the License. import os +import subprocess Test.Summary = ''' -Test transactions and sessions, making sure two continuations catch the same number of hooks. +Test transactions and sessions for http1, making sure the two continuations catch the same number of hooks. ''' -Test.SkipIf(Condition.true('This test errors frequently, and so it is disabled.')) + Test.SkipUnless( - Condition.HasProgram("curl", "Curl needs to be installed on system for this test to work") + Condition.HasProgram("curl", "Curl needs to be installed on system for this test to work"), ) Test.ContinueOnFail = True # Define default ATS -ts = Test.MakeATSProcess("ts", command="traffic_manager") - +ts = Test.MakeATSProcess("ts", select_ports=False, command="traffic_manager") server = Test.MakeOriginServer("server") +server2 = Test.MakeOriginServer("server2") Test.testName = "" -request_header = {"headers": "GET / HTTP/1.1\r\nHost: double.test\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +request_header = {"headers": "GET / HTTP/1.1\r\nHost: double_h2.test\r\n\r\n", "timestamp": "1469733493.993", "body": ""} # expected response from the origin server -response_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": ""} - -Test.PreparePlugin(os.path.join(Test.Variables.AtsTestToolsDir, 'plugins', 'continuations_verify.cc'), ts) +response_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\nContent-Length:0\r\n\r\n", + "timestamp": "1469733493.993", "body": ""} # add response to the server dictionary server.addResponse("sessionfile.log", request_header, response_header) + +# add port and remap rule +ts.Disk.remap_config.AddLine( + 'map / http://127.0.0.1:{0}'.format(server.Variables.Port) +) + ts.Disk.records_config.update({ 'proxy.config.diags.debug.enabled': 1, 'proxy.config.diags.debug.tags': 'continuations_verify.*', - 'proxy.config.http.cache.http' : 0, #disable cache to simply the test. - 'proxy.config.cache.enable_read_while_writer' : 0 + 'proxy.config.ssl.server.cert.path': '{0}'.format(ts.Variables.SSLDir), + 'proxy.config.ssl.server.private_key.path': '{0}'.format(ts.Variables.SSLDir), + 'proxy.config.http.cache.http': 0, # disable cache to simply the test. + 'proxy.config.cache.enable_read_while_writer': 0, + 'proxy.config.ssl.client.verify.server': 0, + 'proxy.config.ssl.server.cipher_suite': 'ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA384:AES128-GCM-SHA256:AES256-GCM-SHA384:ECDHE-RSA-RC4-SHA:ECDHE-RSA-AES128-SHA:ECDHE-RSA-AES256-SHA:RC4-SHA:RC4-MD5:AES128-SHA:AES256-SHA:DES-CBC3-SHA!SRP:!DSS:!PSK:!aNULL:!eNULL:!SSLv2', + 'proxy.config.http2.max_concurrent_streams_in': 65535 }) -ts.Disk.remap_config.AddLine( - 'map http://double.test:{0} http://127.0.0.1:{1}'.format(ts.Variables.port, server.Variables.Port) -) -cmd = 'curl -vs http://127.0.0.1:{0}'.format(ts.Variables.port) -numberOfRequests = 25 +# add plugin to assist with test metrics +Test.PreparePlugin(os.path.join(Test.Variables.AtsTestToolsDir, + 'plugins', 'continuations_verify.cc'), ts) + +comparator_command = ''' +if test "`traffic_ctl metric get continuations_verify.{0}.close.1 | cut -d ' ' -f 2`" -eq "`traffic_ctl metric get continuations_verify.{0}.close.2 | cut -d ' ' -f 2`" ; then\ + echo yes;\ + else \ + echo no; \ + fi; \ + traffic_ctl metric match continuations_verify + ''' + +cmd = 'curl -vs http://127.0.0.1:{0}/'.format(ts.Variables.port) +numberOfRequests = 55 tr = Test.AddTestRun() # Create a bunch of curl commands to be executed in parallel. Default.Process is set in SpawnCommands. -ps = tr.SpawnCommands(cmdstr=cmd, count=numberOfRequests) +# On Fedora 28/29, it seems that curl will occaisionally timeout after a couple seconds and return exitcode 2 +# Examinig the packet capture shows that Traffic Server dutifully sends the response +ps = tr.SpawnCommands(cmdstr=cmd, count=numberOfRequests, retcode=Any(0,2)) tr.Processes.Default.Env = ts.Env +tr.Processes.Default.ReturnCode = Any(0,2) # Execution order is: ts/server, ps(curl cmds), Default Process. tr.Processes.Default.StartBefore( server, ready=When.PortOpen(server.Variables.Port)) # Adds a delay once the ts port is ready. This is because we cannot test the ts state. -tr.Processes.Default.StartBefore(ts, ready=10) +tr.Processes.Default.StartBefore(Test.Processes.ts, ready=When.PortOpen(ts.Variables.port)) ts.StartAfter(*ps) server.StartAfter(*ps) tr.StillRunningAfter = ts -comparator_command = ''' -if test "`traffic_ctl metric get continuations_verify.{0}.close.1 | cut -d ' ' -f 2`" -eq "`traffic_ctl metric get continuations_verify.{0}.close.2 | cut -d ' ' -f 2`" ; then\ - echo yes;\ - else \ - echo no; \ - fi; - ''' +# Signal that all the curl processes have completed +tr = Test.AddTestRun("Curl Done") +tr.Processes.Default.Command = "traffic_ctl plugin msg done done" +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Env = ts.Env +tr.StillRunningAfter = ts -records = ts.Disk.File(os.path.join(ts.Variables.RUNTIMEDIR, "records.snap")) +# Parking this as a ready tester on a meaningless process +# To stall the test runs that check for the stats until the +# stats have propagated and are ready to read. +def make_done_stat_ready(tsenv): + def done_stat_ready(process, hasRunFor, **kw): + retval = subprocess.run("traffic_ctl metric get continuations_verify.test.done > done 2> /dev/null", shell=True, env=tsenv) + if retval.returncode == 0: + retval = subprocess.run("grep 1 done > /dev/null", shell = True, env=tsenv) + return retval.returncode == 0 + return done_stat_ready + # number of sessions/transactions opened and closed are equal -tr = Test.AddTestRun() -tr.DelayStart = 10 # wait for stats to be updated +tr = Test.AddTestRun("Check Ssn") +server2.StartupTimeout = 60 +# Again, here the imporant thing is the ready function not the server2 process +tr.Processes.Default.StartBefore(server2, ready=make_done_stat_ready(ts.Env)) tr.Processes.Default.Command = comparator_command.format('ssn') tr.Processes.Default.ReturnCode = 0 tr.Processes.Default.Env = ts.Env tr.Processes.Default.Streams.stdout = Testers.ContainsExpression("yes", 'should verify contents') +# Session and Txn's should be non-zero +tr.Processes.Default.Streams.stdout += Testers.ExcludesExpression("continations_verify.ssn.close.1 0", 'should be nonzero') +tr.Processes.Default.Streams.stdout += Testers.ExcludesExpression("continations_verify.ssn.close.2 0", 'should be nonzero') +tr.Processes.Default.Streams.stdout += Testers.ExcludesExpression("continations_verify.txn.close.1 0", 'should be nonzero') +tr.Processes.Default.Streams.stdout += Testers.ExcludesExpression("continations_verify.txn.close.2 0", 'should be nonzero') +tr.Processes.Default.Streams.stdout += Testers.ContainsExpression( + "continuations_verify.txn.close.1 {}".format(numberOfRequests), 'should be the number of transactions we made') +tr.Processes.Default.Streams.stdout += Testers.ContainsExpression( + "continuations_verify.txn.close.2 {}".format(numberOfRequests), 'should be the number of transactions we made') tr.StillRunningAfter = ts -# for debugging session number -ssn1 = tr.Processes.Process("session1", 'traffic_ctl metric get continuations_verify.ssn.close.1 > ssn1') -ssn2 = tr.Processes.Process("session2", 'traffic_ctl metric get continuations_verify.ssn.close.2 > ssn2') -ssn1.Env = ts.Env -ssn2.Env = ts.Env -tr.Processes.Default.StartBefore(ssn1) -tr.Processes.Default.StartBefore(ssn2) +tr.StillRunningAfter = server2 -tr = Test.AddTestRun() -tr.DelayStart = 10 # wait for stats to be updated +tr = Test.AddTestRun("Check Txn") tr.Processes.Default.Command = comparator_command.format('txn') tr.Processes.Default.ReturnCode = 0 tr.Processes.Default.Env = ts.Env tr.Processes.Default.Streams.stdout = Testers.ContainsExpression("yes", 'should verify contents') tr.StillRunningAfter = ts -# for debugging transaction number -txn1 = tr.Processes.Process("transaction1", 'traffic_ctl metric get continuations_verify.txn.close.1 > txn1') -txn2 = tr.Processes.Process("transaction2", 'traffic_ctl metric get continuations_verify.txn.close.2 > txn2') -txn1.Env = ts.Env -txn2.Env = ts.Env -tr.Processes.Default.StartBefore(txn1) -tr.Processes.Default.StartBefore(txn2) - -# session count is positive, -tr = Test.AddTestRun() -tr.DelayStart = 10 # wait for stats to be updated -tr.Processes.Default.Command = "traffic_ctl metric get continuations_verify.ssn.close.1" -tr.Processes.Default.ReturnCode = 0 -tr.Processes.Default.Env = ts.Env -tr.Processes.Default.Streams.stdout = Testers.ExcludesExpression(" 0", 'should be nonzero') -tr.StillRunningAfter = ts +tr.StillRunningAfter = server2 -# and we receive the same number of transactions as we asked it to make -tr = Test.AddTestRun() -tr.DelayStart = 10 # wait for stats to be updated -tr.Processes.Default.Command = "traffic_ctl metric get continuations_verify.txn.close.1" -tr.Processes.Default.ReturnCode = 0 -tr.Processes.Default.Env = ts.Env -tr.Processes.Default.Streams.stdout = Testers.ContainsExpression( - "continuations_verify.txn.close.1 {}".format(numberOfRequests), 'should be the number of transactions we made') -tr.StillRunningAfter = ts diff --git a/tests/gold_tests/continuations/double_h2.test.py b/tests/gold_tests/continuations/double_h2.test.py index 61da07d7932..86aace464a1 100644 --- a/tests/gold_tests/continuations/double_h2.test.py +++ b/tests/gold_tests/continuations/double_h2.test.py @@ -17,6 +17,7 @@ # limitations under the License. import os +import subprocess Test.Summary = ''' Test transactions and sessions for http2, making sure the two continuations catch the same number of hooks. ''' @@ -28,11 +29,12 @@ # Define default ATS ts = Test.MakeATSProcess("ts", select_ports=False, command="traffic_manager") server = Test.MakeOriginServer("server") +server2 = Test.MakeOriginServer("server2") Test.testName = "" request_header = {"headers": "GET / HTTP/1.1\r\nHost: double_h2.test\r\n\r\n", "timestamp": "1469733493.993", "body": ""} # expected response from the origin server -response_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", +response_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\nContent-Length:0\r\n\r\n", "timestamp": "1469733493.993", "body": ""} # add response to the server dictionary @@ -45,7 +47,7 @@ # add port and remap rule ts.Variables.ssl_port = 4443 ts.Disk.remap_config.AddLine( - 'map http://double_h2.test:{0} http://127.0.0.1:{1}'.format(ts.Variables.port, server.Variables.Port) + 'map / http://127.0.0.1:{0}'.format(server.Variables.Port) ) ts.Disk.ssl_multicert_config.AddLine( @@ -54,7 +56,7 @@ ts.Disk.records_config.update({ 'proxy.config.diags.debug.enabled': 1, - 'proxy.config.diags.debug.tags': 'continuations_verify.*', + 'proxy.config.diags.debug.tags': 'continuations_verify', 'proxy.config.ssl.server.cert.path': '{0}'.format(ts.Variables.SSLDir), 'proxy.config.ssl.server.private_key.path': '{0}'.format(ts.Variables.SSLDir), 'proxy.config.http.cache.http': 0, # disable cache to simply the test. @@ -75,7 +77,8 @@ echo yes;\ else \ echo no; \ - fi; + fi; \ + traffic_ctl metric match continuations_verify ''' # curl with http2 @@ -85,69 +88,66 @@ tr = Test.AddTestRun() # Create a bunch of curl commands to be executed in parallel. Default.Process is set in SpawnCommands. -ps = tr.SpawnCommands(cmdstr=cmd, count=numberOfRequests) +# On Fedora 28/29, it seems that curl will occaisionally timeout after a couple seconds and return exitcode 2 +# Examinig the packet capture shows that Traffic Server dutifully sends the response +ps = tr.SpawnCommands(cmdstr=cmd, count=numberOfRequests, retcode=Any(0,2)) tr.Processes.Default.Env = ts.Env +tr.Processes.Default.ReturnCode = Any(0,2) # Execution order is: ts/server, ps(curl cmds), Default Process. tr.Processes.Default.StartBefore( server, ready=When.PortOpen(server.Variables.Port)) # Adds a delay once the ts port is ready. This is because we cannot test the ts state. -tr.Processes.Default.StartBefore(ts, ready=10) +tr.Processes.Default.StartBefore(Test.Processes.ts, ready=When.PortOpen(ts.Variables.ssl_port)) ts.StartAfter(*ps) server.StartAfter(*ps) tr.StillRunningAfter = ts -# Watch the records snapshot file. -records = ts.Disk.File(os.path.join(ts.Variables.RUNTIMEDIR, "records.snap")) +# Signal that all the curl processes have completed +tr = Test.AddTestRun("Curl Done") +tr.Processes.Default.Command = "traffic_ctl plugin msg done done" +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Env = ts.Env +tr.StillRunningAfter = ts +# Parking this as a ready tester on a meaningless process +# To stall the test runs that check for the stats until the +# stats have propagated and are ready to read. +def make_done_stat_ready(tsenv): + def done_stat_ready(process, hasRunFor, **kw): + retval = subprocess.run("traffic_ctl metric get continuations_verify.test.done > done 2> /dev/null", shell=True, env=tsenv) + if retval.returncode == 0: + retval = subprocess.run("grep 1 done > /dev/null", shell = True, env=tsenv) + return retval.returncode == 0 + + return done_stat_ready + # number of sessions/transactions opened and closed are equal -tr = Test.AddTestRun() -tr.DelayStart = 10 # wait for stats to be updated +tr = Test.AddTestRun("Check Ssn") +server2.StartupTimeout = 60 +# Again, here the imporant thing is the ready function not the server2 process +tr.Processes.Default.StartBefore(server2, ready=make_done_stat_ready(ts.Env)) tr.Processes.Default.Command = comparator_command.format('ssn') tr.Processes.Default.ReturnCode = 0 tr.Processes.Default.Env = ts.Env tr.Processes.Default.Streams.stdout = Testers.ContainsExpression("yes", 'should verify contents') +# Session and Txn's should be non-zero +tr.Processes.Default.Streams.stdout += Testers.ExcludesExpression("continations_verify.ssn.close.1 0", 'should be nonzero') +tr.Processes.Default.Streams.stdout += Testers.ExcludesExpression("continations_verify.ssn.close.2 0", 'should be nonzero') +tr.Processes.Default.Streams.stdout += Testers.ExcludesExpression("continations_verify.txn.close.1 0", 'should be nonzero') +tr.Processes.Default.Streams.stdout += Testers.ExcludesExpression("continations_verify.txn.close.2 0", 'should be nonzero') +tr.Processes.Default.Streams.stdout += Testers.ContainsExpression( + "continuations_verify.txn.close.1 {}".format(numberOfRequests), 'should be the number of transactions we made') +tr.Processes.Default.Streams.stdout += Testers.ContainsExpression( + "continuations_verify.txn.close.2 {}".format(numberOfRequests), 'should be the number of transactions we made') tr.StillRunningAfter = ts +tr.StillRunningAfter = server2 -# for debugging session number -ssn1 = tr.Processes.Process("session1", 'traffic_ctl metric get continuations_verify.ssn.close.1 > ssn1') -ssn2 = tr.Processes.Process("session2", 'traffic_ctl metric get continuations_verify.ssn.close.2 > ssn2') -ssn1.Env = ts.Env -ssn2.Env = ts.Env -tr.Processes.Default.StartBefore(ssn1) -tr.Processes.Default.StartBefore(ssn2) - -tr = Test.AddTestRun() -tr.DelayStart = 10 # wait for stats to be updated +tr = Test.AddTestRun("Check Txn") tr.Processes.Default.Command = comparator_command.format('txn') tr.Processes.Default.ReturnCode = 0 tr.Processes.Default.Env = ts.Env tr.Processes.Default.Streams.stdout = Testers.ContainsExpression("yes", 'should verify contents') tr.StillRunningAfter = ts +tr.StillRunningAfter = server2 -# for debugging transaction number -txn1 = tr.Processes.Process("transaction1", 'traffic_ctl metric get continuations_verify.txn.close.1 > txn1') -txn2 = tr.Processes.Process("transaction2", 'traffic_ctl metric get continuations_verify.txn.close.2 > txn2') -txn1.Env = ts.Env -txn2.Env = ts.Env -tr.Processes.Default.StartBefore(txn1) -tr.Processes.Default.StartBefore(txn2) - -# session count is positive, -tr = Test.AddTestRun() -tr.DelayStart = 10 # wait for stats to be updated -tr.Processes.Default.Command = "traffic_ctl metric get continuations_verify.ssn.close.1" -tr.Processes.Default.ReturnCode = 0 -tr.Processes.Default.Env = ts.Env -tr.Processes.Default.Streams.stdout = Testers.ExcludesExpression(" 0", 'should be nonzero') -tr.StillRunningAfter = ts - -# and we receive the same number of transactions as we asked it to make -tr = Test.AddTestRun() -tr.DelayStart = 10 # wait for stats to be updated -tr.Processes.Default.Command = "traffic_ctl metric get continuations_verify.txn.close.1" -tr.Processes.Default.ReturnCode = 0 -tr.Processes.Default.Env = ts.Env -tr.Processes.Default.Streams.stdout = Testers.ContainsExpression( - "continuations_verify.txn.close.1 {}".format(numberOfRequests), 'should be the number of transactions we made') -tr.StillRunningAfter = ts diff --git a/tests/gold_tests/continuations/openclose.test.py b/tests/gold_tests/continuations/openclose.test.py index 32d45bdc8d1..452ea131513 100644 --- a/tests/gold_tests/continuations/openclose.test.py +++ b/tests/gold_tests/continuations/openclose.test.py @@ -17,6 +17,7 @@ # limitations under the License. import os +import subprocess Test.Summary = ''' Test transactions and sessions, making sure they open and close in the proper order. ''' @@ -29,12 +30,13 @@ ts = Test.MakeATSProcess("ts", command="traffic_manager") server = Test.MakeOriginServer("server") +server2 = Test.MakeOriginServer("server2") Test.testName = "" request_header = {"headers": "GET / HTTP/1.1\r\nHost: oc.test\r\n\r\n", "timestamp": "1469733493.993", "body": ""} # expected response from the origin server -response_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", +response_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\nContent-Length:0\r\n\r\n", "timestamp": "1469733493.993", "body": ""} Test.PreparePlugin(os.path.join(Test.Variables.AtsTestToolsDir, @@ -59,8 +61,11 @@ tr = Test.AddTestRun() # Create a bunch of curl commands to be executed in parallel. Default.Process is set in SpawnCommands. -ps = tr.SpawnCommands(cmdstr=cmd, count=numberOfRequests) +# On Fedora 28/29, it seems that curl will occaisionally timeout after a couple seconds and return exitcode 2 +# Examinig the packet capture shows that Traffic Server dutifully sends the response +ps = tr.SpawnCommands(cmdstr=cmd, count=numberOfRequests, retcode=Any(0,2)) tr.Processes.Default.Env = ts.Env +tr.Processes.Default.ReturnCode = Any(0,2) # Execution order is: ts/server, ps(curl cmds), Default Process. tr.Processes.Default.StartBefore( @@ -71,13 +76,30 @@ server.StartAfter(*ps) tr.StillRunningAfter = ts -# Watch the records snapshot file. -records = ts.Disk.File(os.path.join(ts.Variables.RUNTIMEDIR, "records.snap")) +# Signal that all the curl processes have completed +tr = Test.AddTestRun("Curl Done") +tr.Processes.Default.Command = "traffic_ctl plugin msg done done" +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Env = ts.Env +tr.StillRunningAfter = ts -# Check our work on traffic_ctl -# no errors happened, -tr = Test.AddTestRun() -tr.DelayStart = 10 +# Parking this as a ready tester on a meaningless process +# To stall the test runs that check for the stats until the +# stats have propagated and are ready to read. +def make_done_stat_ready(tsenv): + def done_stat_ready(process, hasRunFor, **kw): + retval = subprocess.run("traffic_ctl metric get ssntxnorder_verify.test.done > done 2> /dev/null", shell=True, env=tsenv) + if retval.returncode == 0: + retval = subprocess.run("grep 1 done > /dev/null", shell = True, env=tsenv) + return retval.returncode == 0 + + return done_stat_ready + +# number of sessions/transactions opened and closed are equal +tr = Test.AddTestRun("Check Ssn order errors") +server2.StartupTimeout = 60 +# Again, here the imporant thing is the ready function not the server2 process +tr.Processes.Default.StartBefore(server2, ready=make_done_stat_ready(ts.Env)) tr.Processes.Default.Command = 'traffic_ctl metric get ssntxnorder_verify.err' tr.Processes.Default.ReturnCode = 0 tr.Processes.Default.Env = ts.Env @@ -91,48 +113,33 @@ echo yes;\ else \ echo no; \ - fi; + fi; \ + traffic_ctl metric match ssntxnorder_verify ''' # number of sessions/transactions opened and closed are equal -tr = Test.AddTestRun() -tr.DelayStart = 10 +tr = Test.AddTestRun("Check for ssn open/close") tr.Processes.Default.Command = comparator_command.format('ssn') tr.Processes.Default.ReturnCode = 0 tr.Processes.Default.Env = ts.Env tr.Processes.Default.Streams.stdout = Testers.ContainsExpression( "yes", 'should verify contents') +tr.Processes.Default.Streams.stdout += Testers.ExcludesExpression( + "ssntxnorder_verify.ssn.start 0", 'should be nonzero') tr.StillRunningAfter = ts tr.StillRunningAfter = server -tr = Test.AddTestRun() -tr.DelayStart = 10 +tr = Test.AddTestRun("Check for txn/open/close") tr.Processes.Default.Command = comparator_command.format('txn') tr.Processes.Default.ReturnCode = 0 tr.Processes.Default.Env = ts.Env tr.Processes.Default.Streams.stdout = Testers.ContainsExpression( "yes", 'should verify contents') -tr.StillRunningAfter = ts -tr.StillRunningAfter = server - -# session count is positive, -tr = Test.AddTestRun() -tr.DelayStart = 10 -tr.Processes.Default.Command = "traffic_ctl metric get ssntxnorder_verify.ssn.start" -tr.Processes.Default.ReturnCode = 0 -tr.Processes.Default.Env = ts.Env -tr.Processes.Default.Streams.stdout = Testers.ExcludesExpression( - " 0", 'should be nonzero') -tr.StillRunningAfter = ts -tr.StillRunningAfter = server - +tr.Processes.Default.Streams.stdout += Testers.ExcludesExpression( + "ssntxnorder_verify.txn.start 0", 'should be nonzero') # and we receive the same number of transactions as we asked it to make -tr = Test.AddTestRun() -tr.DelayStart = 10 -tr.Processes.Default.Command = "traffic_ctl metric get ssntxnorder_verify.txn.start" -tr.Processes.Default.ReturnCode = 0 -tr.Processes.Default.Env = ts.Env -tr.Processes.Default.Streams.stdout = Testers.ContainsExpression( +tr.Processes.Default.Streams.stdout += Testers.ContainsExpression( "ssntxnorder_verify.txn.start {}".format(numberOfRequests), 'should be the number of transactions we made') tr.StillRunningAfter = ts tr.StillRunningAfter = server + diff --git a/tests/gold_tests/continuations/openclose_h2.test.py b/tests/gold_tests/continuations/openclose_h2.test.py index f14db70c79e..07aec811f7f 100644 --- a/tests/gold_tests/continuations/openclose_h2.test.py +++ b/tests/gold_tests/continuations/openclose_h2.test.py @@ -17,75 +17,67 @@ # limitations under the License. import os +import subprocess Test.Summary = ''' -Test transactions and sessions for http2, making sure they open and close in the proper order. +Test transactions and sessions over http2, making sure they open and close in the proper order. ''' -Test.SkipIf(Condition.true('This test errors frequently, and so it is disabled.')) Test.SkipUnless( Condition.HasProgram("curl", "Curl needs to be installed on system for this test to work"), Condition.HasCurlFeature('http2') ) -Test.ContinueOnFail = True + # Define default ATS ts = Test.MakeATSProcess("ts", select_ports=False, command="traffic_manager") +ts.Variables.ssl_port = 4443 + server = Test.MakeOriginServer("server") +server2 = Test.MakeOriginServer("server2") Test.testName = "" -request_header = {"headers": "GET / HTTP/1.1\r\nHost: oc_h2.test\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +request_header = {"headers": "GET / HTTP/1.1\r\nHost: oc.test\r\n\r\n", + "timestamp": "1469733493.993", "body": ""} # expected response from the origin server -response_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", +response_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\nContent-Length:0\r\n\r\n", "timestamp": "1469733493.993", "body": ""} -# add response to the server dictionary -server.addResponse("sessionfile.log", request_header, response_header) - +# add ssl materials like key, certificates for the server ts.addSSLfile("ssl/server.pem") ts.addSSLfile("ssl/server.key") -ts.Variables.ssl_port = 4443 -ts.Disk.remap_config.AddLine( - 'map http://oc_h2.test:{0} http://127.0.0.1:{1}'.format(ts.Variables.port, server.Variables.Port) -) - -ts.Disk.ssl_multicert_config.AddLine( - 'dest_ip=* ssl_cert_name=server.pem ssl_key_name=server.key' -) +Test.PreparePlugin(os.path.join(Test.Variables.AtsTestToolsDir, + 'plugins', 'ssntxnorder_verify.cc'), ts) +# add response to the server dictionary +server.addResponse("sessionfile.log", request_header, response_header) ts.Disk.records_config.update({ 'proxy.config.diags.debug.enabled': 1, - 'proxy.config.diags.debug.tags': 'ssntxnorder_verify.*', - 'proxy.config.ssl.server.cert.path': '{0}'.format(ts.Variables.SSLDir), - 'proxy.config.ssl.server.private_key.path': '{0}'.format(ts.Variables.SSLDir), + 'proxy.config.diags.debug.tags': 'ssntxnorder_verify', 'proxy.config.http.cache.http': 0, # disable cache to simply the test. 'proxy.config.cache.enable_read_while_writer': 0, - # enable ssl port + 'proxy.config.ssl.server.cert.path': '{0}'.format(ts.Variables.SSLDir), + 'proxy.config.ssl.server.private_key.path': '{0}'.format(ts.Variables.SSLDir), 'proxy.config.http.server_ports': '{0} {1}:proto=http2;http:ssl'.format(ts.Variables.port, ts.Variables.ssl_port), - 'proxy.config.ssl.client.verify.server': 0, - 'proxy.config.ssl.server.cipher_suite': 'ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA384:AES128-GCM-SHA256:AES256-GCM-SHA384:ECDHE-RSA-RC4-SHA:ECDHE-RSA-AES128-SHA:ECDHE-RSA-AES256-SHA:RC4-SHA:RC4-MD5:AES128-SHA:AES256-SHA:DES-CBC3-SHA!SRP:!DSS:!PSK:!aNULL:!eNULL:!SSLv2', - 'proxy.config.http2.max_concurrent_streams_in': 65535 }) -# add plugin to assist with test metrics -Test.PreparePlugin(os.path.join(Test.Variables.AtsTestToolsDir, - 'plugins', 'ssntxnorder_verify.cc'), ts) - -comparator_command = ''' -if test "`traffic_ctl metric get ssntxnorder_verify.{0}.start | cut -d ' ' -f 2`" -eq "`traffic_ctl metric get ssntxnorder_verify.{0}.close | cut -d ' ' -f 2`" ; then\ - echo yes;\ - else \ - echo no; \ - fi; - ''' +ts.Disk.remap_config.AddLine( + 'map http://oc.test:{0} http://127.0.0.1:{1}'.format( + ts.Variables.port, server.Variables.Port) +) +ts.Disk.ssl_multicert_config.AddLine( + 'dest_ip=* ssl_cert_name=server.pem ssl_key_name=server.key' +) -# curl with http2 -cmd = 'curl --http2 -k -vs https://127.0.0.1:{0}/'.format(ts.Variables.ssl_port) +cmd = 'curl -k --http2 -vs https://127.0.0.1:{0}'.format(ts.Variables.ssl_port) numberOfRequests = 25 tr = Test.AddTestRun() # Create a bunch of curl commands to be executed in parallel. Default.Process is set in SpawnCommands. -ps = tr.SpawnCommands(cmdstr=cmd, count=numberOfRequests) +# On Fedora 28/29, it seems that curl will occaisionally timeout after a couple seconds and return exitcode 2 +# Examinig the packet capture shows that Traffic Server dutifully sends the response +ps = tr.SpawnCommands(cmdstr=cmd, count=numberOfRequests, retcode=Any(0,2)) tr.Processes.Default.Env = ts.Env +tr.Processes.Default.ReturnCode = Any(0,2) # Execution order is: ts/server, ps(curl cmds), Default Process. tr.Processes.Default.StartBefore( @@ -96,13 +88,30 @@ server.StartAfter(*ps) tr.StillRunningAfter = ts -# Watch the records snapshot file. -records = ts.Disk.File(os.path.join(ts.Variables.RUNTIMEDIR, "records.snap")) +# Signal that all the curl processes have completed +tr = Test.AddTestRun("Curl Done") +tr.Processes.Default.Command = "traffic_ctl plugin msg done done" +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Env = ts.Env +tr.StillRunningAfter = ts -# Check our work on traffic_ctl -# no errors happened, -tr = Test.AddTestRun() -tr.DelayStart = 10 +# Parking this as a ready tester on a meaningless process +# To stall the test runs that check for the stats until the +# stats have propagated and are ready to read. +def make_done_stat_ready(tsenv): + def done_stat_ready(process, hasRunFor, **kw): + retval = subprocess.run("traffic_ctl metric get ssntxnorder_verify.test.done > done 2> /dev/null", shell=True, env=tsenv) + if retval.returncode == 0: + retval = subprocess.run("grep 1 done > /dev/null", shell = True, env=tsenv) + return retval.returncode == 0 + + return done_stat_ready + +# number of sessions/transactions opened and closed are equal +tr = Test.AddTestRun("Check Ssn order errors") +server2.StartupTimeout = 60 +# Again, here the imporant thing is the ready function not the server2 process +tr.Processes.Default.StartBefore(server2, ready=make_done_stat_ready(ts.Env)) tr.Processes.Default.Command = 'traffic_ctl metric get ssntxnorder_verify.err' tr.Processes.Default.ReturnCode = 0 tr.Processes.Default.Env = ts.Env @@ -111,45 +120,38 @@ tr.StillRunningAfter = ts tr.StillRunningAfter = server +comparator_command = ''' +if test "`traffic_ctl metric get ssntxnorder_verify.{0}.start | cut -d ' ' -f 2`" -eq "`traffic_ctl metric get ssntxnorder_verify.{0}.close | cut -d ' ' -f 2`" ; then\ + echo yes;\ + else \ + echo no; \ + fi; \ + traffic_ctl metric match ssntxnorder_verify + ''' + # number of sessions/transactions opened and closed are equal -tr = Test.AddTestRun() -tr.DelayStart = 10 +tr = Test.AddTestRun("Check for ssn open/close") tr.Processes.Default.Command = comparator_command.format('ssn') tr.Processes.Default.ReturnCode = 0 tr.Processes.Default.Env = ts.Env tr.Processes.Default.Streams.stdout = Testers.ContainsExpression( "yes", 'should verify contents') +tr.Processes.Default.Streams.stdout += Testers.ExcludesExpression( + "ssntxnorder_verify.ssn.start 0", 'should be nonzero') tr.StillRunningAfter = ts tr.StillRunningAfter = server -tr = Test.AddTestRun() -tr.DelayStart = 10 +tr = Test.AddTestRun("Check for txn/open/close") tr.Processes.Default.Command = comparator_command.format('txn') tr.Processes.Default.ReturnCode = 0 tr.Processes.Default.Env = ts.Env tr.Processes.Default.Streams.stdout = Testers.ContainsExpression( "yes", 'should verify contents') -tr.StillRunningAfter = ts -tr.StillRunningAfter = server - -# session count is positive, -tr = Test.AddTestRun() -tr.DelayStart = 10 -tr.Processes.Default.Command = "traffic_ctl metric get ssntxnorder_verify.ssn.start" -tr.Processes.Default.ReturnCode = 0 -tr.Processes.Default.Env = ts.Env -tr.Processes.Default.Streams.stdout = Testers.ExcludesExpression( - " 0", 'should be nonzero') -tr.StillRunningAfter = ts -tr.StillRunningAfter = server - +tr.Processes.Default.Streams.stdout += Testers.ExcludesExpression( + "ssntxnorder_verify.txn.start 0", 'should be nonzero') # and we receive the same number of transactions as we asked it to make -tr = Test.AddTestRun() -tr.DelayStart = 10 -tr.Processes.Default.Command = "traffic_ctl metric get ssntxnorder_verify.txn.start" -tr.Processes.Default.ReturnCode = 0 -tr.Processes.Default.Env = ts.Env -tr.Processes.Default.Streams.stdout = Testers.ContainsExpression( +tr.Processes.Default.Streams.stdout += Testers.ContainsExpression( "ssntxnorder_verify.txn.start {}".format(numberOfRequests), 'should be the number of transactions we made') tr.StillRunningAfter = ts tr.StillRunningAfter = server + diff --git a/tests/gold_tests/h2/gold/post_chunked.gold b/tests/gold_tests/h2/gold/post_chunked.gold index ad471007bd7..0ff06d17219 100644 --- a/tests/gold_tests/h2/gold/post_chunked.gold +++ b/tests/gold_tests/h2/gold/post_chunked.gold @@ -1 +1 @@ -0123456789 \ No newline at end of file +abbbbbbbbb \ No newline at end of file diff --git a/tests/gold_tests/h2/h2disable.test.py b/tests/gold_tests/h2/h2disable.test.py new file mode 100644 index 00000000000..ebeb68812ff --- /dev/null +++ b/tests/gold_tests/h2/h2disable.test.py @@ -0,0 +1,102 @@ +''' +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +Test.Summary = ''' +Test disabling H2 on a per domain basis +''' + +# need Curl +Test.SkipUnless( + Condition.HasProgram("curl", "Curl need to be installed on system for this test to work"), + Condition.HasCurlFeature('http2') +) + +# Define default ATS +ts = Test.MakeATSProcess("ts", select_ports=False) +server = Test.MakeOriginServer("server") + +request_header = {"headers": "GET / HTTP/1.1\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +response_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +server.addResponse("sessionlog.json", request_header, response_header) + +# add ssl materials like key, certificates for the server +ts.addSSLfile("ssl/server.pem") +ts.addSSLfile("ssl/server.key") + +ts.Variables.ssl_port = 4443 +ts.Disk.remap_config.AddLine( + 'map / http://127.0.0.1:{0}'.format(server.Variables.Port)) + +ts.Disk.ssl_multicert_config.AddLine( + 'dest_ip=* ssl_cert_name=server.pem ssl_key_name=server.key' +) + +# Case 1, global config policy=permissive properties=signature +# override for foo.com policy=enforced properties=all +ts.Disk.records_config.update({ + 'proxy.config.diags.debug.enabled': 0, + 'proxy.config.diags.debug.tags': 'http|ssl', + 'proxy.config.ssl.server.cert.path': '{0}'.format(ts.Variables.SSLDir), + 'proxy.config.ssl.server.private_key.path': '{0}'.format(ts.Variables.SSLDir), + # enable ssl port with http2 + 'proxy.config.http.server_ports': '{0} {1}:proto=http2;http:ssl'.format(ts.Variables.port, ts.Variables.ssl_port), + 'proxy.config.ssl.server.cipher_suite': 'ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA384:AES128-GCM-SHA256:AES256-GCM-SHA384:ECDHE-RSA-RC4-SHA:ECDHE-RSA-AES128-SHA:ECDHE-RSA-AES256-SHA:RC4-SHA:RC4-MD5:AES128-SHA:AES256-SHA:DES-CBC3-SHA!SRP:!DSS:!PSK:!aNULL:!eNULL:!SSLv2', + 'proxy.config.url_remap.pristine_host_hdr': 1 +}) + +ts.Disk.ssl_server_name_yaml.AddLines([ + '- fqdn: bar.com', + ' disable_h2: true', + '- fqdn: bob.*.com', + ' disable_h2: true', +]) + +tr = Test.AddTestRun("Negotiate-h2") +tr.Processes.Default.Command = "curl -v -k --resolve 'foo.com:{0}:127.0.0.1' https://foo.com:{0}".format(ts.Variables.ssl_port) +tr.ReturnCode = 0 +tr.Processes.Default.StartBefore(server) +tr.Processes.Default.StartBefore(Test.Processes.ts, ready=When.PortOpen(ts.Variables.ssl_port)) +tr.StillRunningAfter = server +tr.StillRunningAfter = ts +tr.Processes.Default.TimeOut = 5 +tr.Processes.Default.Streams.All = Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") +tr.Processes.Default.Streams.All += Testers.ContainsExpression("Using HTTP2", "Curl should negotiate HTTP2") +tr.TimeOut = 5 + +tr2 = Test.AddTestRun("Do not negotiate h2") +tr2.Processes.Default.Command = "curl -v -k --resolve 'bar.com:{0}:127.0.0.1' https://bar.com:{0}".format(ts.Variables.ssl_port) +tr2.ReturnCode = 0 +tr2.StillRunningAfter = server +tr2.Processes.Default.TimeOut = 5 +tr2.StillRunningAfter = ts +tr2.Processes.Default.Streams.All = Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") +tr2.Processes.Default.Streams.All += Testers.ExcludesExpression("Using HTTP2", "Curl should not negotiate HTTP2") +tr2.TimeOut = 5 + +tr2 = Test.AddTestRun("Do not negotiate h2") +tr2.Processes.Default.Command = "curl -v -k --resolve 'bob.foo.com:{0}:127.0.0.1' https://bob.foo.com:{0}".format(ts.Variables.ssl_port) +tr2.ReturnCode = 0 +tr2.StillRunningAfter = server +tr2.Processes.Default.TimeOut = 5 +tr2.StillRunningAfter = ts +tr2.Processes.Default.Streams.All = Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") +tr2.Processes.Default.Streams.All += Testers.ExcludesExpression("Using HTTP2", "Curl should not negotiate HTTP2") +tr2.TimeOut = 5 + + diff --git a/tests/gold_tests/h2/http2.test.py b/tests/gold_tests/h2/http2.test.py index 747ae0c8cba..5a841f54486 100644 --- a/tests/gold_tests/h2/http2.test.py +++ b/tests/gold_tests/h2/http2.test.py @@ -126,14 +126,15 @@ tr.Processes.Default.Streams.stdout = "gold/chunked.gold" tr.StillRunningAfter = server +# NOTE: Skipping this test run because traffic-replay doesn't currently support H2 # Test Case 4: Multiple request -client_path = os.path.join(Test.Variables.AtsTestToolsDir, 'traffic-replay/') -tr = Test.AddTestRun() -tr.Processes.Default.Command = "python3 {0} -type {1} -log_dir {2} -port {3} -host '127.0.0.1' -s_port {4} -v -colorize False".format( - client_path, 'h2', server.Variables.DataDir, ts.Variables.port, ts.Variables.ssl_port) -tr.Processes.Default.ReturnCode = 0 -tr.Processes.Default.Streams.stdout = "gold/replay.gold" -tr.StillRunningAfter = server +# client_path = os.path.join(Test.Variables.AtsTestToolsDir, 'traffic-replay/') +# tr = Test.AddTestRun() +# tr.Processes.Default.Command = "python3 {0} -type {1} -log_dir {2} -port {3} -host '127.0.0.1' -s_port {4} -v -colorize False".format( +# client_path, 'h2', server.Variables.DataDir, ts.Variables.port, ts.Variables.ssl_port) +# tr.Processes.Default.ReturnCode = 0 +# tr.Processes.Default.Streams.stdout = "gold/replay.gold" +# tr.StillRunningAfter = server # Test Case 5:h2_active_timeout tr = Test.AddTestRun() diff --git a/tests/gold_tests/headers/cache_and_error_nobody.gold b/tests/gold_tests/headers/cache_and_error_nobody.gold new file mode 100644 index 00000000000..8e10e6fe171 --- /dev/null +++ b/tests/gold_tests/headers/cache_and_error_nobody.gold @@ -0,0 +1,11 @@ +HTTP/1.1 404 Not Found +Content-Length: 0 +Cache-Control: max-age=3 +Date: `` +Age: `` +Connection: keep-alive +Via: `` +Server: `` +X-Cache-Key: http://127.0.0.1`` +X-Cache: `` + diff --git a/tests/gold_tests/headers/cache_and_req_body-hit-stale-206-etag.gold b/tests/gold_tests/headers/cache_and_req_body-hit-stale-206-etag.gold new file mode 100644 index 00000000000..8bc8a2a488a --- /dev/null +++ b/tests/gold_tests/headers/cache_and_req_body-hit-stale-206-etag.gold @@ -0,0 +1,13 @@ +HTTP/1.1 206 Partial Content +Cache-Control: max-age=1 +Content-Length: 2 +Date: `` +Etag: myetag +Age: `` +Connection: keep-alive +Via: `` +Server: `` +X-Cache-Key: http://127.0.0.1`` +X-Cache: hit-stale + +xx \ No newline at end of file diff --git a/tests/gold_tests/headers/cache_and_req_body-hit-stale-206.gold b/tests/gold_tests/headers/cache_and_req_body-hit-stale-206.gold new file mode 100644 index 00000000000..03cc794eb46 --- /dev/null +++ b/tests/gold_tests/headers/cache_and_req_body-hit-stale-206.gold @@ -0,0 +1,13 @@ +HTTP/1.1 206 Partial Content +Last-Modified: `` +Cache-Control: max-age=1 +Content-Length: 2 +Date: `` +Age: `` +Connection: keep-alive +Via: `` +Server: `` +X-Cache-Key: http://127.0.0.1`` +X-Cache: hit-stale + +xx \ No newline at end of file diff --git a/tests/gold_tests/headers/cache_and_req_body-hit-stale-INM.gold b/tests/gold_tests/headers/cache_and_req_body-hit-stale-INM.gold new file mode 100644 index 00000000000..2dc3eb84e90 --- /dev/null +++ b/tests/gold_tests/headers/cache_and_req_body-hit-stale-INM.gold @@ -0,0 +1,13 @@ +HTTP/1.1 200 OK +Cache-Control: max-age=1 +Content-Length: 3 +Date: `` +Etag: `` +Age: `` +Connection: keep-alive +Via: `` +Server: `` +X-Cache-Key: http://127.0.0.1`` +X-Cache: hit-stale + +xxx \ No newline at end of file diff --git a/tests/gold_tests/headers/cache_and_req_body-hit-stale.gold b/tests/gold_tests/headers/cache_and_req_body-hit-stale.gold new file mode 100644 index 00000000000..4e9f9e35c2b --- /dev/null +++ b/tests/gold_tests/headers/cache_and_req_body-hit-stale.gold @@ -0,0 +1,13 @@ +HTTP/1.1 200 OK +Last-Modified: `` +Cache-Control: max-age=1 +Content-Length: 3 +Date: `` +Age: `` +Connection: keep-alive +Via: `` +Server: `` +X-Cache-Key: http://127.0.0.1`` +X-Cache: hit-stale + +xxx \ No newline at end of file diff --git a/tests/gold_tests/headers/cache_and_req_body-hit.gold b/tests/gold_tests/headers/cache_and_req_body-hit.gold index 90d4a9743af..09c06a1ccd2 100644 --- a/tests/gold_tests/headers/cache_and_req_body-hit.gold +++ b/tests/gold_tests/headers/cache_and_req_body-hit.gold @@ -1,5 +1,6 @@ HTTP/1.1 200 OK -Cache-Control: max-age=300 +Last-Modified:`` +Cache-Control: max-age=1 Content-Length: 3 Date: `` Age: `` diff --git a/tests/gold_tests/headers/cache_and_req_body-hit_close.gold b/tests/gold_tests/headers/cache_and_req_body-hit_close.gold index ed944d21a9d..9ccf980e753 100644 --- a/tests/gold_tests/headers/cache_and_req_body-hit_close.gold +++ b/tests/gold_tests/headers/cache_and_req_body-hit_close.gold @@ -1,5 +1,6 @@ HTTP/1.1 200 OK -Cache-Control: max-age=300 +Last-Modified:`` +Cache-Control: max-age=1 Content-Length: 3 Date: `` Age: `` diff --git a/tests/gold_tests/headers/cache_and_req_body-miss.gold b/tests/gold_tests/headers/cache_and_req_body-miss.gold index aa174cdc2a6..7c8159d190c 100644 --- a/tests/gold_tests/headers/cache_and_req_body-miss.gold +++ b/tests/gold_tests/headers/cache_and_req_body-miss.gold @@ -1,11 +1,13 @@ HTTP/1.1 200 OK -Cache-Control: max-age=300 +Last-Modified: `` +Cache-Control: max-age=1 Content-Length: 3 Date: `` Age: `` Connection: keep-alive Via: `` Server: `` +X-Cache-Key: http://127.0.0.1`` X-Cache: miss xxx \ No newline at end of file diff --git a/tests/gold_tests/headers/cache_and_req_body.test.py b/tests/gold_tests/headers/cache_and_req_body.test.py index a952cc20221..8d3afc070e1 100644 --- a/tests/gold_tests/headers/cache_and_req_body.test.py +++ b/tests/gold_tests/headers/cache_and_req_body.test.py @@ -36,7 +36,7 @@ #**testname is required** testName = "" request_header = {"headers": "GET / HTTP/1.1\r\nHost: www.example.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} -response_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\nCache-Control: max-age=300\r\n\r\n", "timestamp": "1469733493.993", "body": "xxx"} +response_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\nLast-Modified: Tue, 08 May 2018 15:49:41 GMT\r\nCache-Control: max-age=1\r\n\r\n", "timestamp": "1469733493.993", "body": "xxx"} server.addResponse("sessionlog.json", request_header, response_header) # ATS Configuration @@ -82,4 +82,3 @@ tr.Processes.Default.ReturnCode = 0 tr.Processes.Default.Streams.stdout = "cache_and_req_body-hit_close.gold" tr.StillRunningAfter = ts - diff --git a/tests/gold_tests/headers/cache_and_req_nobody-hit-stale.gold b/tests/gold_tests/headers/cache_and_req_nobody-hit-stale.gold new file mode 100644 index 00000000000..46ca77ec996 --- /dev/null +++ b/tests/gold_tests/headers/cache_and_req_nobody-hit-stale.gold @@ -0,0 +1,11 @@ +HTTP/1.1 200 OK +Content-Length: 0 +Cache-Control: max-age=3 +Date: `` +Age: `` +Connection: keep-alive +Via: `` +Server: `` +X-Cache-Key: http://127.0.0.1`` +X-Cache: `` + diff --git a/tests/gold_tests/headers/cachedIMSRange.test.py b/tests/gold_tests/headers/cachedIMSRange.test.py new file mode 100644 index 00000000000..a8dabae52aa --- /dev/null +++ b/tests/gold_tests/headers/cachedIMSRange.test.py @@ -0,0 +1,168 @@ +''' +Test cached responses and requests with bodies +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import time +Test.Summary = ''' +Test revalidating cached objects +''' + +testName = "RevalidateCacheObject" + +# Needs Curl +Test.SkipUnless( + Condition.HasProgram("curl", "curl needs to be installed on system for this test to work"), +) +Test.ContinueOnFail = True + +# Set up Origin server +# request_header is from ATS to origin; response from Origin to ATS +# lookup_key is to make unique response in origin for header "UID" that will pass in ATS request +server = Test.MakeOriginServer("server",lookup_key="{%UID}") +# Initial request +request_header = {"headers": "GET / HTTP/1.1\r\nHost: www.example.com\r\nUID: Fill\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +response_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\nLast-Modified: Tue, 08 May 2018 15:49:41 GMT\r\nCache-Control: max-age=1\r\n\r\n", "timestamp": "1469733493.993", "body": "xxx"} +server.addResponse("sessionlog.json", request_header, response_header) +# IMS revalidation request +request_IMS_header = {"headers": "GET / HTTP/1.1\r\nUID: IMS\r\nIf-Modified-Since: Tue, 08 May 2018 15:49:41 GMT\r\nHost: www.example.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +response_IMS_header = {"headers": "HTTP/1.1 304 Not Modified\r\nConnection: close\r\nCache-Control: max-age=1\r\n\r\n", "timestamp": "1469733493.993", "body": None} +server.addResponse("sessionlog.json", request_IMS_header, response_IMS_header) + +# EtagFill +request_etagfill_header = {"headers": "GET /etag HTTP/1.1\r\nHost: www.example.com\r\nUID: EtagFill\r\n\r\n", "timestamp": "1469733493.993", "body": None} +response_etagfill_header = {"headers": "HTTP/1.1 200 OK\r\nETag: myetag\r\nConnection: close\r\nCache-Control: max-age=1\r\n\r\n", "timestamp": "1469733493.993", "body": "xxx"} +server.addResponse("sessionlog.json", request_etagfill_header, response_etagfill_header) +# INM revalidation +request_INM_header = {"headers": "GET /etag HTTP/1.1\r\nUID: INM\r\nIf-None-Match: myetag\r\nHost: www.example.com\r\n\r\n", "timestamp": "1469733493.993", "body": None} +response_INM_header = {"headers": "HTTP/1.1 304 Not Modified\r\nConnection: close\r\nETag: myetag\r\nCache-Control: max-age=1\r\n\r\n", "timestamp": "1469733493.993", "body": None} +server.addResponse("sessionlog.json", request_INM_header, response_INM_header) + +# object changed to 0 byte +request_noBody_header = {"headers": "GET / HTTP/1.1\r\nUID: noBody\r\nHost: www.example.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +response_noBody_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\nContent-Length: 0\r\nCache-Control: max-age=3\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +server.addResponse("sessionlog.json", request_noBody_header, response_noBody_header) + +# etag object now is a 404. Yeah, 404s don't usually have Cache-Control, but, ATS's default is to cache 404s for a while. +request_etagfill_header = {"headers": "GET /etag HTTP/1.1\r\nHost: www.example.com\r\nUID: EtagError\r\n\r\n", "timestamp": "1469733493.993", "body": None} +response_etagfill_header = {"headers": "HTTP/1.1 404 Not Found\r\nConnection: close\r\nContent-Length: 0\r\nCache-Control: max-age=3\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +server.addResponse("sessionlog.json", request_etagfill_header, response_etagfill_header) + +# ATS Configuration +ts = Test.MakeATSProcess("ts", select_ports=False) +ts.Disk.plugin_config.AddLine('xdebug.so') +ts.Disk.records_config.update({ + 'proxy.config.diags.debug.enabled': 1, + 'proxy.config.diags.debug.tags': 'http', + 'proxy.config.http.response_via_str': 3, + 'proxy.config.http.cache.http': 1, + 'proxy.config.http.wait_for_cache': 1, +}) + +ts.Disk.remap_config.AddLine( + 'map / http://127.0.0.1:{0}'.format(server.Variables.Port) +) + +# Test 0 - Fill a 3 byte object with Last-Modified time into cache. +tr = Test.AddTestRun() +tr.Processes.Default.StartBefore(server) +tr.Processes.Default.StartBefore(ts, ready=1) +tr.Processes.Default.Command = 'curl -s -D - -v --ipv4 --http1.1 -H"UID: Fill" -H "x-debug: x-cache,x-cache-key,via" -H "Host: www.example.com" http://localhost:{0}/'.format(ts.Variables.port) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Streams.stdout = "cache_and_req_body-miss.gold" +tr.StillRunningAfter = ts +tr.StillRunningAfter = server + +# Test 1 - Once it goes stale, fetch it again. We expect Origin to get IMS request, and serve a 304. We expect ATS to refresh the object, and give a 200 to user +tr = Test.AddTestRun() +tr.DelayStart=2 +tr.Processes.Default.Command = 'curl -s -D - -v --ipv4 --http1.1 -H"UID: IMS" -H "x-debug: x-cache,x-cache-key,via" -H "Host: www.example.com" http://localhost:{0}/'.format(ts.Variables.port) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Streams.stdout = "cache_and_req_body-hit-stale.gold" +tr.StillRunningAfter = ts +tr.StillRunningAfter = server + +# Test 2 - Once it goes stale, fetch it via a range request. We expect Origin to get IMS request, and serve a 304. We expect ATS to refresh the object, and give a 206 to user +tr = Test.AddTestRun() +tr.DelayStart=2 +tr.Processes.Default.Command = 'curl --range 0-1 -s -D - -v --ipv4 --http1.1 -H"UID: IMS" -H "x-debug: x-cache,x-cache-key,via" -H "Host: www.example.com" http://localhost:{0}/'.format(ts.Variables.port) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Streams.stdout = "cache_and_req_body-hit-stale-206.gold" +tr.StillRunningAfter = ts +tr.StillRunningAfter = server + +# Test 3 - Fill a new object with an Etag. Not checking the output here. +tr = Test.AddTestRun() +tr.Processes.Default.Command = 'curl -s -D - -v --ipv4 --http1.1 -H"UID: EtagFill" -H "x-debug: x-cache,x-cache-key,via" -H "Host: www.example.com" http://localhost:{0}/etag'.format(ts.Variables.port) +tr.Processes.Default.ReturnCode = 0 +tr.StillRunningAfter = ts +tr.StillRunningAfter = server + +# Test 4 - Once the etag object goes stale, fetch it again. We expect Origin to get INM request, and serve a 304. We expect ATS to refresh the object, and give a 200 to user +tr = Test.AddTestRun() +tr.DelayStart=2 +tr.Processes.Default.Command = 'curl -s -D - -v --ipv4 --http1.1 -H"UID: INM" -H "x-debug: x-cache,x-cache-key,via" -H "Host: www.example.com" http://localhost:{0}/etag'.format(ts.Variables.port) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Streams.stdout = "cache_and_req_body-hit-stale-INM.gold" +tr.StillRunningAfter = ts +tr.StillRunningAfter = server + +# Test 5 - Once the etag object goes stale, fetch it via a range request. We expect Origin to get INM request, and serve a 304. We expect ATS to refresh the object, and give a 206 to user +tr = Test.AddTestRun() +tr.DelayStart=2 +tr.Processes.Default.Command = 'curl --range 0-1 -s -D - -v --ipv4 --http1.1 -H"UID: INM" -H "x-debug: x-cache,x-cache-key,via" -H "Host: www.example.com" http://localhost:{0}/etag'.format(ts.Variables.port) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Streams.stdout = "cache_and_req_body-hit-stale-206-etag.gold" +tr.StillRunningAfter = ts +tr.StillRunningAfter = server + +# Test 6 - The origin changes the initial LMT object to 0 byte. We expect ATS to fetch and serve the new 0 byte object. +tr = Test.AddTestRun() +tr.DelayStart=3 +tr.Processes.Default.Command = 'curl -s -D - -v --ipv4 --http1.1 -H"UID: noBody" -H "x-debug: x-cache,x-cache-key,via" -H "Host: www.example.com" http://localhost:{0}/'.format(ts.Variables.port) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Streams.stdout = "cache_and_req_nobody-hit-stale.gold" +tr.StillRunningAfter = ts +tr.StillRunningAfter = server + +# Test 7 - Fetch the new 0 byte object again when fresh in cache to ensure its still a 0 byte object. +tr = Test.AddTestRun() +tr.DelayStart=3 +tr.Processes.Default.Command = 'curl -s -D - -v --ipv4 --http1.1 -H"UID: noBody" -H "x-debug: x-cache,x-cache-key,via" -H "Host: www.example.com" http://localhost:{0}/'.format(ts.Variables.port) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Streams.stdout = "cache_and_req_nobody-hit-stale.gold" +tr.StillRunningAfter = ts +tr.StillRunningAfter = server + +# Test 8 - The origin changes the etag object to 0 byte 404. We expect ATS to fetch and serve the 404 0 byte object. +tr = Test.AddTestRun() +tr.DelayStart=2 +tr.Processes.Default.Command = 'curl -s -D - -v --ipv4 --http1.1 -H"UID: EtagError" -H "x-debug: x-cache,x-cache-key,via" -H "Host: www.example.com" http://localhost:{0}/etag'.format(ts.Variables.port) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Streams.stdout = "cache_and_error_nobody.gold" +tr.StillRunningAfter = ts +tr.StillRunningAfter = server + +# Test 9 - Fetch the 0 byte etag object again when fresh in cache to ensure its still a 0 byte object +tr = Test.AddTestRun() +tr.DelayStart=2 +tr.Processes.Default.Command = 'curl -s -D - -v --ipv4 --http1.1 -H"UID: EtagError" -H "x-debug: x-cache,x-cache-key,via" -H "Host: www.example.com" http://localhost:{0}/etag'.format(ts.Variables.port) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Streams.stdout = "cache_and_error_nobody.gold" +tr.StillRunningAfter = ts +tr.StillRunningAfter = server diff --git a/tests/gold_tests/headers/domain-blacklist-30x.test.py b/tests/gold_tests/headers/domain-blacklist-30x.test.py index b8d90413f6d..19c972771c9 100644 --- a/tests/gold_tests/headers/domain-blacklist-30x.test.py +++ b/tests/gold_tests/headers/domain-blacklist-30x.test.py @@ -52,7 +52,7 @@ for x in (0, 301, 302, 307, 308): ts.Disk.MakeConfigFile("header_rewrite_rules_{0}.conf".format(x)).AddLine("""\ -set-redirect {0} "%" +set-redirect {0} "%{{CLIENT-URL}}" """.format(x)) Test.Setup.Copy(os.path.join(os.pardir, os.pardir, 'tools', 'tcp_client.py')) diff --git a/tests/gold_tests/headers/forwarded.gold b/tests/gold_tests/headers/forwarded.gold index 45451d62a54..939825a1372 100644 --- a/tests/gold_tests/headers/forwarded.gold +++ b/tests/gold_tests/headers/forwarded.gold @@ -31,11 +31,11 @@ for=127.0.0.1;by=unknown;by=Poxy_Proxy;__BY_EQUAL_UUID__;by=127.0.0.1;proto=http for=0.6.6.6 for=_argh, for=127.0.0.1;by=unknown;by=Poxy_Proxy;__BY_EQUAL_UUID__;by=127.0.0.1;proto=http;host=www.no-oride.com;connection=http;connection=http/1.0;connection=http/1.0-tcp-ipv4 - -for=127.0.0.1;by=unknown;by=Poxy_Proxy;__BY_EQUAL_UUID__;by=127.0.0.1;proto=https;host=www.no-oride.com;connection=https;connection=https/2;connection=http/1.1-h2-tls/1.2-tcp-ipv4 +for=127.0.0.1;by=unknown;by=Poxy_Proxy;__BY_EQUAL_UUID__;by=127.0.0.1;proto=https;host=www.no-oride.com;connection=https;connection=https/2;connection=http/1.1-h2-tls/1.{}-tcp-ipv4 - -for=127.0.0.1;by=unknown;by=Poxy_Proxy;__BY_EQUAL_UUID__;by=127.0.0.1;proto=https;host=www.no-oride.com;connection=https;connection=https/1.1;connection=http/1.1-tls/1.2-tcp-ipv4 +for=127.0.0.1;by=unknown;by=Poxy_Proxy;__BY_EQUAL_UUID__;by=127.0.0.1;proto=https;host=www.no-oride.com;connection=https;connection=https/1.1;connection=http/1.1-tls/1.{}-tcp-ipv4 - for="[::1]";by=unknown;by=Poxy_Proxy;__BY_EQUAL_UUID__;by="[::1]";proto=http;host=www.no-oride.com;connection=http;connection=http/1.1;connection=http/1.1-tcp-ipv6 - -for="[::1]";by=unknown;by=Poxy_Proxy;__BY_EQUAL_UUID__;by="[::1]";proto=https;host=www.no-oride.com;connection=https;connection=https/1.1;connection=http/1.1-tls/1.2-tcp-ipv6 +for="[::1]";by=unknown;by=Poxy_Proxy;__BY_EQUAL_UUID__;by="[::1]";proto=https;host=www.no-oride.com;connection=https;connection=https/1.1;connection=http/1.1-tls/1.{}-tcp-ipv6 - diff --git a/tests/gold_tests/headers/forwarded.test.py b/tests/gold_tests/headers/forwarded.test.py index e5854f2e752..eb95e382b4b 100644 --- a/tests/gold_tests/headers/forwarded.test.py +++ b/tests/gold_tests/headers/forwarded.test.py @@ -25,7 +25,6 @@ ''' Test.SkipUnless( - Condition.HasATSFeature('TS_USE_TLS_ALPN'), Condition.HasCurlFeature('http2'), Condition.HasCurlFeature('IPv6'), ) diff --git a/tests/gold_tests/headers/via.gold b/tests/gold_tests/headers/via.gold index 598ef376e75..01165cfe7e5 100644 --- a/tests/gold_tests/headers/via.gold +++ b/tests/gold_tests/headers/via.gold @@ -1,6 +1,6 @@ Via: http/1.1 = http/1.1 tcp ipv4 Via: http/1.0 = http/1.0 tcp ipv4 -Via: https/2 = http/1.1 h2 tls/1.2 tcp ipv4 -Via: https/1.1 = http/1.1 tls/1.2 tcp ipv4 +Via: https/2 = http/1.1 h2 tls/1.{} tcp ipv4 +Via: https/1.1 = http/1.1 tls/1.{} tcp ipv4 Via: http/1.1 = http/1.1 tcp ipv6 -Via: https/1.1 = http/1.1 tls/1.2 tcp ipv6 +Via: https/1.1 = http/1.1 tls/1.{} tcp ipv6 diff --git a/tests/gold_tests/headers/via.test.py b/tests/gold_tests/headers/via.test.py index 9746cdec606..1244b489647 100644 --- a/tests/gold_tests/headers/via.test.py +++ b/tests/gold_tests/headers/via.test.py @@ -26,7 +26,6 @@ ''' Test.SkipUnless( - Condition.HasATSFeature('TS_USE_TLS_ALPN'), Condition.HasCurlFeature('http2'), Condition.HasCurlFeature('IPv6') ) diff --git a/tests/gold_tests/logging/all_headers.test.py b/tests/gold_tests/logging/all_headers.test.py new file mode 100644 index 00000000000..cdb62758c52 --- /dev/null +++ b/tests/gold_tests/logging/all_headers.test.py @@ -0,0 +1,108 @@ +''' +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import subprocess + +Test.Summary = ''' +Test new "all headers" log fields +''' +# need Curl +Test.SkipUnless( + Condition.HasProgram( + "curl", "Curl need to be installed on system for this test to work"), + # Condition.IsPlatform("linux"), Don't see the need for this. +) + +# Define ATS. +# +ts = Test.MakeATSProcess("ts") + +# Define MicroServer. +# +server = Test.MakeOriginServer("server") + +request_header = {"headers": "GET / HTTP/1.1\r\nHost: does.not.matter\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +response_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\nCache-control: max-age=85000\r\n\r\n", + "timestamp": "1469733493.993", "body": "xxx"} +server.addResponse("sessionlog.json", request_header, response_header) + +ts.Disk.records_config.update({ + 'proxy.config.diags.debug.enabled': 0, + 'proxy.config.diags.debug.tags': 'http|dns', +}) + +ts.Disk.remap_config.AddLine( + 'map http://127.0.0.1:{0} http://127.0.0.1:{1}'.format(ts.Variables.port, server.Variables.Port) +) + +# Mix in a numeric log field. Hopefull this will detect any binary alignment problems. +# +ts.Disk.logging_yaml.AddLines( + ''' +formats: + - name: custom + format: " % % % % % % " +logs: + - filename: test_all_headers + format: custom +'''.split("\n") +) + +# Configure comparison of "sanitized" log file with gold file at end of test. +# +Test.Disk.File(os.path.join(ts.Variables.LOGDIR, 'test_all_headers.log.san'), + exists=True, content='gold/test_all_headers.gold') + + +def reallyLong(): + value = 'abcdefghijklmnop' + value = value + value + value = value + value + value = value + value + retval = "" + for i in range(3): + retval += ' -H "x-header{}: {}"'.format(i, value) + return retval + +tr = Test.AddTestRun() +tr.Processes.Default.StartBefore(server) +tr.Processes.Default.StartBefore(Test.Processes.ts) +tr.Processes.Default.Command = ( +'curl "http://127.0.0.1:{0}" --user-agent "007" --verbose '.format(ts.Variables.port) + reallyLong() +) +tr.Processes.Default.ReturnCode = 0 + +# Repeat same curl, will be answered from the ATS cache. +# +tr = Test.AddTestRun() +tr.Processes.Default.Command = ( +'curl "http://127.0.0.1:{0}" --user-agent "007" --verbose '.format(ts.Variables.port) + reallyLong() +) +tr.Processes.Default.ReturnCode = 0 + +# Delay to allow TS to flush report to disk, then "sanitize" generated log. +# +tr = Test.AddTestRun() +tr.DelayStart = 10 +tr.Processes.Default.Command = 'python {0} {3} < {1} > {2}'.format( + os.path.join(Test.TestDirectory, 'all_headers_sanitizer.py'), + os.path.join(ts.Variables.LOGDIR, 'test_all_headers.log'), + os.path.join(ts.Variables.LOGDIR, 'test_all_headers.log.san'), + server.Variables.Port) +tr.Processes.Default.ReturnCode = 0 diff --git a/tests/gold_tests/logging/all_headers_sanitizer.py b/tests/gold_tests/logging/all_headers_sanitizer.py new file mode 100644 index 00000000000..0e798df96da --- /dev/null +++ b/tests/gold_tests/logging/all_headers_sanitizer.py @@ -0,0 +1,43 @@ +''' +Sanitize the ATS-generated custom log file from the all_headers test. +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys +import re + +rexl = [] +rexl.append((re.compile(r"\{\{Date\}\:\{[^}]*\}\}"), "__DATE__")) +rexl.append((re.compile(r"\{\{Expires\}\:\{[^}]*\}\}"), "__EXPIRES__")) +rexl.append((re.compile(r"\{\{Last-Modified\}\:\{[^}]*\}\}"), "__LAST_MODIFIED__")) +rexl.append((re.compile(r"\{\{Server\}\:\{ATS/[0-9.]*\}\}"), "__ATS_SERVER__")) +rexl.append((re.compile(r"\{\{Server\}\:\{ECS [^}]*\}\}"), "__ECS_SERVER__")) +rexl.append((re.compile(r"\{\{Via\}\:\{[^}]*\}\}"), "__VIA__")) +rexl.append((re.compile(r"\{\{Server\}\:\{ApacheTrafficServer/[0-9.]*\}\}"), "__ATS2_SERVER__")) +rexl.append((re.compile(r"\{\{Age\}\:\{[0-9]*\}\}"), "__AGE__")) +rexl.append((re.compile(r"\:" + sys.argv[1]), "__TS_PORT__")) # 1st and only argument is TS client port + +# Handle inconsistencies which I think are caused by different revisions of the standard Python http.server.HTTPServer class. + +rexl.append((re.compile(r'\{"359670651[^"]*"\}'), '{"359670651__WEIRD__"}')) +rexl.append((re.compile(r'\{\{Accept-Ranges\}:\{bytes\}\}'), '')) + +for line in sys.stdin: + for rex, subStr in rexl: + line = rex.sub(subStr, line) + + print(line) diff --git a/tests/gold_tests/logging/ccid_ctid.test.py b/tests/gold_tests/logging/ccid_ctid.test.py index ef3f03122e4..b66a6d185bb 100644 --- a/tests/gold_tests/logging/ccid_ctid.test.py +++ b/tests/gold_tests/logging/ccid_ctid.test.py @@ -27,7 +27,6 @@ Condition.HasProgram( "curl", "Curl need to be installed on system for this test to work"), # Condition.IsPlatform("linux"), Don't see the need for this. - Condition.HasATSFeature('TS_USE_TLS_ALPN'), Condition.HasCurlFeature('http2') ) diff --git a/tests/gold_tests/logging/gold/test_all_headers.gold b/tests/gold_tests/logging/gold/test_all_headers.gold new file mode 100644 index 00000000000..3cb54e5b81e --- /dev/null +++ b/tests/gold_tests/logging/gold/test_all_headers.gold @@ -0,0 +1,4 @@ + {{{Host}:{127.0.0.1__TS_PORT__}}{{User-Agent}:{007}}{{Accept}:{*/*}}{{x-header0}:{abcdefghijklmnopabcdefghijklmnopabcdefghijklmnopabcdefghijklmnopabcdefghijklmnopabcdefghijklmnopabcdefghijklmnopabcdefghijklmnop}}{{x-header1}:{abcdefghijklmnopabcdefghijklmnopabcdefghijklmnopabcdefghijklmnopabcdefghijklmnopabcdefghijklmnopabcdefghijklmnopabcdefghijklmnop}}{{x-header2}:{abcdefghijklmnopabcdefghijklmnopabcdefghijklmnopabcdefghijklmnopabcdefghijklmnopabcdefghijklmnopabcdefghijklmnopabcdefghijklmnop}}} 200 {{{Cache-Control}:{max-age=85000}}{{Content-Length}:{3}}__DATE____AGE__{{Connection}:{keep-alive}}__ATS_SERVER__} {{{Connection}:{close}}{{Cache-Control}:{max-age=85000}}{{Content-Length}:{3}}__DATE__} {{{Host}:{127.0.0.1__TS_PORT__}}{{User-Agent}:{007}}{{Accept}:{*/*}}{{x-header0}:{abcdefghijklmnopabcdefghijklmnopabcdefghijklmnopabcdefghijklmnopabcdefghijklmnopabcdefghijklmnopabcdefghijklmnopabcdefghijklmnop}}{{x-header1}:{abcdefghijklmnopabcdefghijklmnopabcdefghijklmnopabcdefghijklmnopabcdefghijklmnopabcdefghijklmnopabcdefghijklmnopabcdefghijklmnop}}{{x-header2}:{abcdefghijklmnopabcdefghijklmnopabcdefghijklmnopabcdefghijklmnopabcdefghijklmnopabcdefghijklmnopabcdefghijklmnopabcdefghijklmnop}}{{Client-ip}:{127.0.0.1}}{{X-Forwarded-For}:{127.0.0.1}}__VIA__} {} + + {{{Host}:{127.0.0.1__TS_PORT__}}{{User-Agent}:{007}}{{Accept}:{*/*}}{{x-header0}:{abcdefghijklmnopabcdefghijklmnopabcdefghijklmnopabcdefghijklmnopabcdefghijklmnopabcdefghijklmnopabcdefghijklmnopabcdefghijklmnop}}{{x-header1}:{abcdefghijklmnopabcdefghijklmnopabcdefghijklmnopabcdefghijklmnopabcdefghijklmnopabcdefghijklmnopabcdefghijklmnopabcdefghijklmnop}}{{x-header2}:{abcdefghijklmnopabcdefghijklmnopabcdefghijklmnopabcdefghijklmnopabcdefghijklmnopabcdefghijklmnopabcdefghijklmnopabcdefghijklmnop}}} 200 {{{Cache-Control}:{max-age=85000}}{{Content-Length}:{3}}__DATE____AGE__{{Connection}:{keep-alive}}__ATS_SERVER__} {} {} {{{Cache-Control}:{max-age=85000}}{{Content-Length}:{3}}__DATE____AGE__{{Connection}:{keep-alive}}__ATS_SERVER__} + diff --git a/tests/gold_tests/pluginTest/cookie_remap/bucketcookie.test.py b/tests/gold_tests/pluginTest/cookie_remap/bucketcookie.test.py new file mode 100644 index 00000000000..86d4e50be16 --- /dev/null +++ b/tests/gold_tests/pluginTest/cookie_remap/bucketcookie.test.py @@ -0,0 +1,105 @@ +''' +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +Test.Summary = ''' + +''' +Test.SkipUnless(Condition.PluginExists('cookie_remap.so')) +# need Curl +Test.SkipUnless( + Condition.HasProgram("curl", "Curl need to be installed on system for this test to work") +) +Test.ContinueOnFail = True +Test.testName = "cookie_remap: cookie in bucket or not" + +# Define default ATS +ts = Test.MakeATSProcess("ts") + +# First server is run during first test and +# second server is run during second test +server = Test.MakeOriginServer("server", ip='127.0.0.10') + +request_header = {"headers": "GET /cookiematches HTTP/1.1\r\nHost: www.example.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +# expected response from the origin server +response_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": ""} + +# add response to the server dictionary +server.addResponse("sessionfile.log", request_header, response_header) + +server2 = Test.MakeOriginServer("server2", ip='127.0.0.11') +request_header2 = {"headers": "GET /cookiedoesntmatch HTTP/1.1\r\nHost: www.example.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +# expected response from the origin server +response_header2 = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": ""} + +# add response to the server dictionary +server2.addResponse("sessionfile.log", request_header2, response_header2) + +# Setup the remap configuration +config_path = os.path.join(Test.TestDirectory, "configs/bucketconfig.txt") +with open(config_path, 'r') as config_file: + config1 = config_file.read() + +ts.Disk.records_config.update({ + 'proxy.config.diags.debug.enabled': 1, + 'proxy.config.diags.debug.tags': 'cookie_remap.*|http.*|dns.*', +}) + +config1 = config1.replace("$PORT", str(server.Variables.Port)) +config1 = config1.replace("$ALTPORT", str(server2.Variables.Port)) + +ts.Disk.File(ts.Variables.CONFIGDIR +"/bucketconfig.txt", exists=False, id="config1") +ts.Disk.config1.WriteOn(config1) + +ts.Disk.remap_config.AddLine( + 'map http://www.example.com/magic http://shouldnothit.com @plugin=cookie_remap.so @pparam=config/bucketconfig.txt' +) + +# Cookie value in bucket +tr = Test.AddTestRun("cookie value in bucket") +tr.Processes.Default.Command = ''' +curl +--proxy 127.0.0.1:{0} +"http://www.example.com/magic" +-H"Cookie: fpbeta=333" +-H "Proxy-Connection: keep-alive" +--verbose +'''.format(ts.Variables.port) +tr.Processes.Default.ReturnCode = 0 +# time delay as proxy.config.http.wait_for_cache could be broken +tr.Processes.Default.StartBefore(server, ready=When.PortOpen(server.Variables.Port)) +tr.Processes.Default.StartBefore(Test.Processes.ts) +tr.StillRunningAfter = ts + +server.Streams.All = "gold/matchcookie.gold" + +# cookie value not in bucket +tr = Test.AddTestRun("cooke value not in bucket") +tr.Processes.Default.Command = ''' +curl +--proxy 127.0.0.1:{0} +"http://www.example.com/magic" +-H"Cookie: fpbeta=etc" +-H "Proxy-Connection: keep-alive" +--verbose +'''.format(ts.Variables.port) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.StartBefore(server2, ready=When.PortOpen(server2.Variables.Port)) +tr.StillRunningAfter = ts + +server2.Streams.All = "gold/wontmatchcookie.gold" diff --git a/tests/gold_tests/pluginTest/cookie_remap/collapseslashes.test.py b/tests/gold_tests/pluginTest/cookie_remap/collapseslashes.test.py new file mode 100644 index 00000000000..77d823a7649 --- /dev/null +++ b/tests/gold_tests/pluginTest/cookie_remap/collapseslashes.test.py @@ -0,0 +1,80 @@ +''' +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +Test.Summary = ''' + +''' +Test.SkipUnless(Condition.PluginExists('cookie_remap.so')) +# need Curl +Test.SkipUnless( + Condition.HasProgram("curl", "Curl need to be installed on system for this test to work") +) +Test.ContinueOnFail = True +Test.testName = "cookie_remap: plugin collapses consecutive slashes" +Test.SkipIf(Condition.true("Test is temporarily turned off, to be fixed according to an incompatible plugin API change (PR #4964)")) + +# Define default ATS +ts = Test.MakeATSProcess("ts") + +# We only run a server to capture ATS outbound requests +# and verify it collapsed the double // +server = Test.MakeOriginServer("server", ip='127.0.0.10') + +request_header = {"headers": "GET /i/like/cheetos?.done=http://finance.yahoo.com HTTP/1.1\r\nHost: www.example.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +# expected response from the origin server +response_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": ""} + +# add response to the server dictionary +server.addResponse("sessionfile.log", request_header, response_header) + +# Setup the remap configuration +config_path = os.path.join(Test.TestDirectory, "configs/collapseconfig.txt") +with open(config_path, 'r') as config_file: + config1 = config_file.read() + +ts.Disk.records_config.update({ + 'proxy.config.diags.debug.enabled': 1, + 'proxy.config.diags.debug.tags': 'cookie_remap.*|http.*|dns.*', +}) + +config1 = config1.replace("$PORT", str(server.Variables.Port)) + +ts.Disk.File(ts.Variables.CONFIGDIR +"/collapseconfig.txt", exists=False, id="config1") +ts.Disk.config1.WriteOn(config1) + +ts.Disk.remap_config.AddLine( + 'map http://www.example.com/magic http://shouldnothit.com @plugin=cookie_remap.so @pparam=config/collapseconfig.txt' +) + +tr = Test.AddTestRun("collapse consecutive forward slashes") +tr.Processes.Default.Command = ''' +curl +--proxy 127.0.0.1:{0} +"http://www.example.com/magic" +-H "Proxy-Connection: keep-alive" +--verbose +'''.format(ts.Variables.port) +tr.Processes.Default.ReturnCode = 0 +# time delay as proxy.config.http.wait_for_cache could be broken +tr.Processes.Default.StartBefore(server, ready=When.PortOpen(server.Variables.Port)) +tr.Processes.Default.StartBefore(Test.Processes.ts) +tr.StillRunningAfter = ts + +server.Streams.All = "gold/collapseslashes.gold" + diff --git a/tests/gold_tests/pluginTest/cookie_remap/configs/bucketconfig.txt b/tests/gold_tests/pluginTest/cookie_remap/configs/bucketconfig.txt new file mode 100644 index 00000000000..2af72d0d1da --- /dev/null +++ b/tests/gold_tests/pluginTest/cookie_remap/configs/bucketconfig.txt @@ -0,0 +1,9 @@ +# This is a test configuration + +# Do a regex against the cookie +op: + cookie: fpbeta + operation: bucket + bucket: 30/100 + sendto: http://127.0.0.10:$PORT/cookiematches + else: http://127.0.0.11:$ALTPORT/cookiedoesntmatch diff --git a/tests/gold_tests/pluginTest/cookie_remap/configs/collapseconfig.txt b/tests/gold_tests/pluginTest/cookie_remap/configs/collapseconfig.txt new file mode 100644 index 00000000000..11693a0a508 --- /dev/null +++ b/tests/gold_tests/pluginTest/cookie_remap/configs/collapseconfig.txt @@ -0,0 +1,6 @@ +# This is a test configuration + +# Do a regex against the request url +op: + regex: magic + sendto: http://127.0.0.10:$PORT/i/////////like/cheetos?.done=http://finance.yahoo.com diff --git a/tests/gold_tests/pluginTest/cookie_remap/configs/connectorconfig.txt b/tests/gold_tests/pluginTest/cookie_remap/configs/connectorconfig.txt new file mode 100644 index 00000000000..a35a1a6efb2 --- /dev/null +++ b/tests/gold_tests/pluginTest/cookie_remap/configs/connectorconfig.txt @@ -0,0 +1,16 @@ +# This is a test configuration + +# First 2 ops do a comparison against the cookie and the 3rd +# one does against the url +op: + cookie: fpbeta + operation: string + match: abcd + connector: and + cookie: icecream + operation: exists + connector: and + operation: regex + regex: magic + sendto: http://127.0.0.10:$PORT/cookiematches + else: http://127.0.0.11:$ALTPORT/cookiedoesntmatch diff --git a/tests/gold_tests/pluginTest/cookie_remap/configs/existsconfig.txt b/tests/gold_tests/pluginTest/cookie_remap/configs/existsconfig.txt new file mode 100644 index 00000000000..8b3189a86bc --- /dev/null +++ b/tests/gold_tests/pluginTest/cookie_remap/configs/existsconfig.txt @@ -0,0 +1,8 @@ +# This is a test configuration + +# Do a regex against the cookie +op: + cookie: fpbeta + operation: exists + sendto: http://127.0.0.10:$PORT/cookieexists + else: http://127.0.0.11:$ALTPORT/cookiedoesntexist diff --git a/tests/gold_tests/pluginTest/cookie_remap/configs/matchconfig.txt b/tests/gold_tests/pluginTest/cookie_remap/configs/matchconfig.txt new file mode 100644 index 00000000000..a7b850a6357 --- /dev/null +++ b/tests/gold_tests/pluginTest/cookie_remap/configs/matchconfig.txt @@ -0,0 +1,8 @@ +# This is a test configuration + +# Do a regex against the cookie +op: + cookie: fpbeta + match: magic + sendto: http://127.0.0.10:$PORT/cookiematches + else: http://127.0.0.11:$ALTPORT/cookiedoesntmatch diff --git a/tests/gold_tests/pluginTest/cookie_remap/configs/matchuriconfig.txt b/tests/gold_tests/pluginTest/cookie_remap/configs/matchuriconfig.txt new file mode 100644 index 00000000000..3272826ea71 --- /dev/null +++ b/tests/gold_tests/pluginTest/cookie_remap/configs/matchuriconfig.txt @@ -0,0 +1,17 @@ +# This is a test configuration + +# Do a regex against the request url +op: + match: dkfdkfkdfkdfkdfkdfk + sendto: http://nothere.com +op: + match: eioreioreioreioeio + sendto: http://nothereeither.com +op: + match: mmmdm,cmvmcnxcxncn + sendto: http://nothereaswell.com +op: + operation: regex + regex: thisispartofthepath + sendto: http://127.0.0.10:$PORT/cookiematches + else: http://127.0.0.11:$ALTPORT/cookiedoesntmatch diff --git a/tests/gold_tests/pluginTest/cookie_remap/configs/matrixconfig.txt b/tests/gold_tests/pluginTest/cookie_remap/configs/matrixconfig.txt new file mode 100644 index 00000000000..263c957359d --- /dev/null +++ b/tests/gold_tests/pluginTest/cookie_remap/configs/matrixconfig.txt @@ -0,0 +1,24 @@ +# This is a test configuration + +# Do a regex against the url + +op: + operation: regex + regex: eleventh + sendto: http://127.0.0.10:$PORT/$path/eleventh;matrix=2?query=true + +op: + operation: regex + regex: tenth + sendto: http://127.0.0.10:$PORT/$path/tenth;matrix=2 + +op: + operation: regex + regex: ninth + sendto: http://127.0.0.10:$PORT/$path/ninth?query=true + +op: + operation: regex + regex: eighth + sendto: http://127.0.0.10:$PORT/$path/eighth + diff --git a/tests/gold_tests/pluginTest/cookie_remap/configs/notexistsconfig.txt b/tests/gold_tests/pluginTest/cookie_remap/configs/notexistsconfig.txt new file mode 100644 index 00000000000..72a2c19eb5d --- /dev/null +++ b/tests/gold_tests/pluginTest/cookie_remap/configs/notexistsconfig.txt @@ -0,0 +1,8 @@ +# This is a test configuration + +# Do a regex against the cookie +op: + cookie: fpbeta + operation: not exists + sendto: http://127.0.0.10:$PORT/cookiedoesntexist + else: http://127.0.0.11:$ALTPORT/cookieexists diff --git a/tests/gold_tests/pluginTest/cookie_remap/configs/regexconfig.txt b/tests/gold_tests/pluginTest/cookie_remap/configs/regexconfig.txt new file mode 100644 index 00000000000..04e7fdd6d6f --- /dev/null +++ b/tests/gold_tests/pluginTest/cookie_remap/configs/regexconfig.txt @@ -0,0 +1,9 @@ +# This is a test configuration + +# Do a regex against the cookie +op: + cookie: fpbeta + operation: regex + regex: ilove-(\w+)-(\w+)-(\w+) + sendto: http://127.0.0.10:$PORT/regexmatches?cookies=$1-$2-$3 + else: http://127.0.0.11:$ALTPORT/regexdoesntmatch diff --git a/tests/gold_tests/pluginTest/cookie_remap/configs/statusconfig.txt b/tests/gold_tests/pluginTest/cookie_remap/configs/statusconfig.txt new file mode 100644 index 00000000000..6d66e8d12ff --- /dev/null +++ b/tests/gold_tests/pluginTest/cookie_remap/configs/statusconfig.txt @@ -0,0 +1,17 @@ +# This is a test configuration +# When status is set, ATS doesn't follow the sendto or else +# It just returns a canned response + +# Do a regex against the cookie +op: + cookie: fpbeta + match: magic + sendto: http://shouldnevergohere.com + status: 205 +op: + cookie: abracadabra + match: magic + sendto: http://shouldnevergohere.com + else: http://shouldnotgohereaswell.com + # When a else is present, status is associated with it + status: 400 diff --git a/tests/gold_tests/pluginTest/cookie_remap/configs/subcookie.txt b/tests/gold_tests/pluginTest/cookie_remap/configs/subcookie.txt new file mode 100644 index 00000000000..f34be180867 --- /dev/null +++ b/tests/gold_tests/pluginTest/cookie_remap/configs/subcookie.txt @@ -0,0 +1,17 @@ +# This is a test configuration + +# Do a regex against the cookie +op: + cookie: fpbeta.a + operation: string + match: 1 + connector: and + cookie: fpbeta.b + operation: string + match: 2 + connector: and + cookie: fpbeta.c + operation: string + match: 3 + sendto: http://127.0.0.10:$PORT/cookiematches + else: http://127.0.0.11:$ALTPORT/cookiedoesntmatch diff --git a/tests/gold_tests/pluginTest/cookie_remap/configs/substituteconfig.txt b/tests/gold_tests/pluginTest/cookie_remap/configs/substituteconfig.txt new file mode 100644 index 00000000000..120adc35f4e --- /dev/null +++ b/tests/gold_tests/pluginTest/cookie_remap/configs/substituteconfig.txt @@ -0,0 +1,21 @@ +# This is a test configuration + +# Do a regex against the cookie +op: + cookie: fpbeta + operation: exists + sendto: http://127.0.0.10:$PORT/photos/search?query=$path +op: + cookie: oxalpha + operation: exists + sendto: http://127.0.0.10:$PORT/photos/search?query=$unmatched_path +op: + cookie: acgamma + operation: exists + sendto: http://127.0.0.10:$PORT/photos/search/cr_substitutions?query=$cr_urlencode($cr_req_url) +# Regex against url and path is substituted in outgoing path +op: + operation: regex + regex: foobar + sendto: http://127.0.0.10:$PORT/photos/search/$path + diff --git a/tests/gold_tests/pluginTest/cookie_remap/connector.test.py b/tests/gold_tests/pluginTest/cookie_remap/connector.test.py new file mode 100644 index 00000000000..24edae3047d --- /dev/null +++ b/tests/gold_tests/pluginTest/cookie_remap/connector.test.py @@ -0,0 +1,106 @@ +''' +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +Test.Summary = ''' + +''' +Test.SkipUnless(Condition.PluginExists('cookie_remap.so')) +# need Curl +Test.SkipUnless( + Condition.HasProgram("curl", "Curl need to be installed on system for this test to work") +) +Test.ContinueOnFail = True +Test.testName = "cookie_remap: test connector" +Test.SkipIf(Condition.true("Test is temporarily turned off, to be fixed according to an incompatible plugin API change (PR #4964)")) + +# Define default ATS +ts = Test.MakeATSProcess("ts") + +# First server is run during first test and +# second server is run during second test +server = Test.MakeOriginServer("server", ip='127.0.0.10') + +request_header = {"headers": "GET /cookiematches HTTP/1.1\r\nHost: www.example.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +# expected response from the origin server +response_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": ""} + +# add response to the server dictionary +server.addResponse("sessionfile.log", request_header, response_header) + +server2 = Test.MakeOriginServer("server2", ip='127.0.0.11') +request_header2 = {"headers": "GET /cookiedoesntmatch HTTP/1.1\r\nHost: www.example.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +# expected response from the origin server +response_header2 = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": ""} + +# add response to the server dictionary +server2.addResponse("sessionfile.log", request_header2, response_header2) + +# Setup the remap configuration +config_path = os.path.join(Test.TestDirectory, "configs/connectorconfig.txt") +with open(config_path, 'r') as config_file: + config1 = config_file.read() + +ts.Disk.records_config.update({ + 'proxy.config.diags.debug.enabled': 1, + 'proxy.config.diags.debug.tags': 'cookie_remap.*|http.*|dns.*', +}) + +config1 = config1.replace("$PORT", str(server.Variables.Port)) +config1 = config1.replace("$ALTPORT", str(server2.Variables.Port)) + +ts.Disk.File(ts.Variables.CONFIGDIR +"/connectorconfig.txt", exists=False, id="config1") +ts.Disk.config1.WriteOn(config1) + +ts.Disk.remap_config.AddLine( + 'map http://www.example.com/magic http://shouldnothit.com @plugin=cookie_remap.so @pparam=config/connectorconfig.txt' +) + +# Positive test case that remaps because all connected operations pass +tr = Test.AddTestRun("cookie value matches") +tr.Processes.Default.Command = ''' +curl +--proxy 127.0.0.1:{0} +"http://www.example.com/magic" +-H"Cookie: fpbeta=abcd icecream=donteat" +-H "Proxy-Connection: keep-alive" +--verbose +'''.format(ts.Variables.port) +tr.Processes.Default.ReturnCode = 0 +# time delay as proxy.config.http.wait_for_cache could be broken +tr.Processes.Default.StartBefore(server, ready=When.PortOpen(server.Variables.Port)) +tr.Processes.Default.StartBefore(Test.Processes.ts) +tr.StillRunningAfter = ts + +server.Streams.All = "gold/matchcookie.gold" + +# Negative test case that doesn't remap because not all subops pass +tr = Test.AddTestRun("cookie value doesn't match") +tr.Processes.Default.Command = ''' +curl +--proxy 127.0.0.1:{0} +"http://www.example.com/magic" +-H"Cookie: fpbeta=somethingelse" +-H "Proxy-Connection: keep-alive" +--verbose +'''.format(ts.Variables.port) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.StartBefore(server2, ready=When.PortOpen(server2.Variables.Port)) +tr.StillRunningAfter = ts + +server2.Streams.All = "gold/wontmatchcookie.gold" diff --git a/tests/gold_tests/pluginTest/cookie_remap/existscookie.test.py b/tests/gold_tests/pluginTest/cookie_remap/existscookie.test.py new file mode 100644 index 00000000000..43598a7bfdc --- /dev/null +++ b/tests/gold_tests/pluginTest/cookie_remap/existscookie.test.py @@ -0,0 +1,105 @@ +''' +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +Test.Summary = ''' + +''' +Test.SkipUnless(Condition.PluginExists('cookie_remap.so')) +# need Curl +Test.SkipUnless( + Condition.HasProgram("curl", "Curl need to be installed on system for this test to work") +) +Test.ContinueOnFail = True +Test.testName = "cookie_remap: cookie exists" + +# Define default ATS +ts = Test.MakeATSProcess("ts") + +# First server is run during first test and +# second server is run during second test +server = Test.MakeOriginServer("server", ip='127.0.0.10') + +request_header = {"headers": "GET /cookieexists HTTP/1.1\r\nHost: www.example.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +# expected response from the origin server +response_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": ""} + +# add response to the server dictionary +server.addResponse("sessionfile.log", request_header, response_header) + +server2 = Test.MakeOriginServer("server2", ip='127.0.0.11') +request_header2 = {"headers": "GET /cookiedoesntexist HTTP/1.1\r\nHost: www.example.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +# expected response from the origin server +response_header2 = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": ""} + +# add response to the server dictionary +server2.addResponse("sessionfile.log", request_header2, response_header2) + +# Setup the remap configuration +config_path = os.path.join(Test.TestDirectory, "configs/existsconfig.txt") +with open(config_path, 'r') as config_file: + config1 = config_file.read() + +ts.Disk.records_config.update({ + 'proxy.config.diags.debug.enabled': 1, + 'proxy.config.diags.debug.tags': 'cookie_remap.*|http.*|dns.*', +}) + +config1 = config1.replace("$PORT", str(server.Variables.Port)) +config1 = config1.replace("$ALTPORT", str(server2.Variables.Port)) + +ts.Disk.File(ts.Variables.CONFIGDIR +"/existsconfig.txt", exists=False, id="config1") +ts.Disk.config1.WriteOn(config1) + +ts.Disk.remap_config.AddLine( + 'map http://www.example.com/magic http://shouldnothit.com @plugin=cookie_remap.so @pparam=config/existsconfig.txt' +) + +# Positive test case that remaps because cookie exists +tr = Test.AddTestRun("cookie fpbeta exists") +tr.Processes.Default.Command = ''' +curl +--proxy 127.0.0.1:{0} +"http://www.example.com/magic" +-H"Cookie: fpbeta=fdkfkdfkdfkdkdfkdfk" +-H "Proxy-Connection: keep-alive" +--verbose +'''.format(ts.Variables.port) +tr.Processes.Default.ReturnCode = 0 +# time delay as proxy.config.http.wait_for_cache could be broken +tr.Processes.Default.StartBefore(server, ready=When.PortOpen(server.Variables.Port)) +tr.Processes.Default.StartBefore(Test.Processes.ts) +tr.StillRunningAfter = ts + +server.Streams.All = "gold/existscookie.gold" + +# Negative test case that doesn't remap because cookie doesn't exist +tr = Test.AddTestRun("cooke fpbeta doesnt exist") +tr.Processes.Default.Command = ''' +curl +--proxy 127.0.0.1:{0} +"http://www.example.com/magic" +-H"Cookie: breadcrumb=etc" +-H "Proxy-Connection: keep-alive" +--verbose +'''.format(ts.Variables.port) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.StartBefore(server2, ready=When.PortOpen(server2.Variables.Port)) +tr.StillRunningAfter = ts + +server2.Streams.All = "gold/doesntexistcookie.gold" diff --git a/tests/gold_tests/pluginTest/cookie_remap/gold/collapseslashes.gold b/tests/gold_tests/pluginTest/cookie_remap/gold/collapseslashes.gold new file mode 100644 index 00000000000..254fdc612ca --- /dev/null +++ b/tests/gold_tests/pluginTest/cookie_remap/gold/collapseslashes.gold @@ -0,0 +1,3 @@ +`` +Serving GET /i/like/cheetos?.done=http://finance.yahoo.com... Finished +`` diff --git a/tests/gold_tests/pluginTest/cookie_remap/gold/doesntexistcookie.gold b/tests/gold_tests/pluginTest/cookie_remap/gold/doesntexistcookie.gold new file mode 100644 index 00000000000..204c488f4e9 --- /dev/null +++ b/tests/gold_tests/pluginTest/cookie_remap/gold/doesntexistcookie.gold @@ -0,0 +1,3 @@ +`` +Serving GET /cookiedoesntexist... Finished +`` diff --git a/tests/gold_tests/pluginTest/cookie_remap/gold/existscookie.gold b/tests/gold_tests/pluginTest/cookie_remap/gold/existscookie.gold new file mode 100644 index 00000000000..29680e34bc2 --- /dev/null +++ b/tests/gold_tests/pluginTest/cookie_remap/gold/existscookie.gold @@ -0,0 +1,3 @@ +`` +Serving GET /cookieexists... Finished +`` diff --git a/tests/gold_tests/pluginTest/cookie_remap/gold/matchcookie.gold b/tests/gold_tests/pluginTest/cookie_remap/gold/matchcookie.gold new file mode 100644 index 00000000000..53c5f864a69 --- /dev/null +++ b/tests/gold_tests/pluginTest/cookie_remap/gold/matchcookie.gold @@ -0,0 +1,3 @@ +`` +Serving GET /cookiematches... Finished +`` diff --git a/tests/gold_tests/pluginTest/cookie_remap/gold/matchcookie2.gold b/tests/gold_tests/pluginTest/cookie_remap/gold/matchcookie2.gold new file mode 100644 index 00000000000..2fed63291e6 --- /dev/null +++ b/tests/gold_tests/pluginTest/cookie_remap/gold/matchcookie2.gold @@ -0,0 +1,3 @@ +`` +Serving GET /cookiematches?a=1&b=2&c=3... Finished +`` diff --git a/tests/gold_tests/pluginTest/cookie_remap/gold/matchelsestatus.gold b/tests/gold_tests/pluginTest/cookie_remap/gold/matchelsestatus.gold new file mode 100644 index 00000000000..5c4de12cae2 --- /dev/null +++ b/tests/gold_tests/pluginTest/cookie_remap/gold/matchelsestatus.gold @@ -0,0 +1,3 @@ +`` +HTTP/1.1 400 Bad Request +`` diff --git a/tests/gold_tests/pluginTest/cookie_remap/gold/matchstatus.gold b/tests/gold_tests/pluginTest/cookie_remap/gold/matchstatus.gold new file mode 100644 index 00000000000..c396c1356fc --- /dev/null +++ b/tests/gold_tests/pluginTest/cookie_remap/gold/matchstatus.gold @@ -0,0 +1,3 @@ +`` +HTTP/1.1 205 Reset Content +`` diff --git a/tests/gold_tests/pluginTest/cookie_remap/gold/matrix.gold b/tests/gold_tests/pluginTest/cookie_remap/gold/matrix.gold new file mode 100644 index 00000000000..3989deaa61e --- /dev/null +++ b/tests/gold_tests/pluginTest/cookie_remap/gold/matrix.gold @@ -0,0 +1,11 @@ +`` +Serving GET /eighth/magic;matrix=1/eighth... Finished +`` +Serving GET /eighth/magic;matrix=1/eighth?hello=10... Finished +`` +Serving GET /tenth/magic/tenth;matrix=2... Finished +`` +Serving GET /tenth/magic/tenth;matrix=2?query=10... Finished +`` +Serving GET /eleventh/magic;matrix=4/eleventh;matrix=2?query=true... Finished +`` diff --git a/tests/gold_tests/pluginTest/cookie_remap/gold/regexdoesntmatch.gold b/tests/gold_tests/pluginTest/cookie_remap/gold/regexdoesntmatch.gold new file mode 100644 index 00000000000..6e357f14b25 --- /dev/null +++ b/tests/gold_tests/pluginTest/cookie_remap/gold/regexdoesntmatch.gold @@ -0,0 +1,3 @@ +`` +Serving GET /regexdoesntmatch... Finished +`` diff --git a/tests/gold_tests/pluginTest/cookie_remap/gold/regexmatches.gold b/tests/gold_tests/pluginTest/cookie_remap/gold/regexmatches.gold new file mode 100644 index 00000000000..0f3b6f30f53 --- /dev/null +++ b/tests/gold_tests/pluginTest/cookie_remap/gold/regexmatches.gold @@ -0,0 +1,3 @@ +`` +Serving GET /regexmatches?cookies=oreos-chipsahoy-icecream... Finished +`` diff --git a/tests/gold_tests/pluginTest/cookie_remap/gold/substitute.gold b/tests/gold_tests/pluginTest/cookie_remap/gold/substitute.gold new file mode 100644 index 00000000000..76ab353bda1 --- /dev/null +++ b/tests/gold_tests/pluginTest/cookie_remap/gold/substitute.gold @@ -0,0 +1,7 @@ +`` +Serving GET /photos/search?query=magic... Finished +`` +Serving GET /photos/search?query=/theunmatchedpath... Finished +`` +Serving GET /photos/search/magic/foobar... Finished +`` \ No newline at end of file diff --git a/tests/gold_tests/pluginTest/cookie_remap/gold/wontmatchcookie.gold b/tests/gold_tests/pluginTest/cookie_remap/gold/wontmatchcookie.gold new file mode 100644 index 00000000000..2257bd61371 --- /dev/null +++ b/tests/gold_tests/pluginTest/cookie_remap/gold/wontmatchcookie.gold @@ -0,0 +1,3 @@ +`` +Serving GET /cookiedoesntmatch... Finished +`` diff --git a/tests/gold_tests/pluginTest/cookie_remap/gold/wontmatchcookie2.gold b/tests/gold_tests/pluginTest/cookie_remap/gold/wontmatchcookie2.gold new file mode 100644 index 00000000000..83470c5b00f --- /dev/null +++ b/tests/gold_tests/pluginTest/cookie_remap/gold/wontmatchcookie2.gold @@ -0,0 +1,3 @@ +`` +Serving GET /cookiedoesntmatch?a=1&b=2&c=3... Finished +`` diff --git a/tests/gold_tests/pluginTest/cookie_remap/matchcookie.test.py b/tests/gold_tests/pluginTest/cookie_remap/matchcookie.test.py new file mode 100644 index 00000000000..cad59810ec3 --- /dev/null +++ b/tests/gold_tests/pluginTest/cookie_remap/matchcookie.test.py @@ -0,0 +1,105 @@ +''' +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +Test.Summary = ''' + +''' +Test.SkipUnless(Condition.PluginExists('cookie_remap.so')) +# need Curl +Test.SkipUnless( + Condition.HasProgram("curl", "Curl need to be installed on system for this test to work") +) +Test.ContinueOnFail = True +Test.testName = "cookie_remap: match cookie" + +# Define default ATS +ts = Test.MakeATSProcess("ts") + +# First server is run during first test and +# second server is run during second test +server = Test.MakeOriginServer("server", ip='127.0.0.10') + +request_header = {"headers": "GET /cookiematches?a=1&b=2&c=3 HTTP/1.1\r\nHost: www.example.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +# expected response from the origin server +response_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": ""} + +# add response to the server dictionary +server.addResponse("sessionfile.log", request_header, response_header) + +server2 = Test.MakeOriginServer("server2", ip='127.0.0.11') +request_header2 = {"headers": "GET /cookiedoesntmatch?a=1&b=2&c=3 HTTP/1.1\r\nHost: www.example.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +# expected response from the origin server +response_header2 = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": ""} + +# add response to the server dictionary +server2.addResponse("sessionfile.log", request_header2, response_header2) + +# Setup the remap configuration +config_path = os.path.join(Test.TestDirectory, "configs/matchconfig.txt") +with open(config_path, 'r') as config_file: + config1 = config_file.read() + +ts.Disk.records_config.update({ + 'proxy.config.diags.debug.enabled': 1, + 'proxy.config.diags.debug.tags': 'cookie_remap.*|http.*|dns.*', +}) + +config1 = config1.replace("$PORT", str(server.Variables.Port)) +config1 = config1.replace("$ALTPORT", str(server2.Variables.Port)) + +ts.Disk.File(ts.Variables.CONFIGDIR +"/matchconfig.txt", exists=False, id="config1") +ts.Disk.config1.WriteOn(config1) + +ts.Disk.remap_config.AddLine( + 'map http://www.example.com/magic http://shouldnothit.com @plugin=cookie_remap.so @pparam=config/matchconfig.txt' +) + +# Positive test case that remaps because cookie matches +tr = Test.AddTestRun("cookie value matches") +tr.Processes.Default.Command = ''' +curl \ +--proxy 127.0.0.1:{0} \ +"http://www.example.com/magic?a=1&b=2&c=3" \ +-H"Cookie: fpbeta=magic" \ +-H "Proxy-Connection: keep-alive" \ +--verbose \ +'''.format(ts.Variables.port) +tr.Processes.Default.ReturnCode = 0 +# time delay as proxy.config.http.wait_for_cache could be broken +tr.Processes.Default.StartBefore(server, ready=When.PortOpen(server.Variables.Port)) +tr.Processes.Default.StartBefore(Test.Processes.ts) +tr.StillRunningAfter = ts + +server.Streams.All = "gold/matchcookie2.gold" + +# Negative test case that doesn't remap because cookie doesn't match +tr = Test.AddTestRun("cookie regex doesn't match") +tr.Processes.Default.Command = ''' +curl \ +--proxy 127.0.0.1:{0} \ +"http://www.example.com/magic?a=1&b=2&c=3" \ +-H"Cookie: fpbeta=magit" \ +-H "Proxy-Connection: keep-alive" \ +--verbose \ +'''.format(ts.Variables.port) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.StartBefore(server2, ready=When.PortOpen(server2.Variables.Port)) +tr.StillRunningAfter = ts + +server2.Streams.All = "gold/wontmatchcookie2.gold" diff --git a/tests/gold_tests/pluginTest/cookie_remap/matchuri.test.py b/tests/gold_tests/pluginTest/cookie_remap/matchuri.test.py new file mode 100644 index 00000000000..bb968144372 --- /dev/null +++ b/tests/gold_tests/pluginTest/cookie_remap/matchuri.test.py @@ -0,0 +1,104 @@ +''' +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +Test.Summary = ''' + +''' +Test.SkipUnless(Condition.PluginExists('cookie_remap.so')) +# need Curl +Test.SkipUnless( + Condition.HasProgram("curl", "Curl need to be installed on system for this test to work") +) +Test.ContinueOnFail = True +Test.testName = "cookie_remap: match cookie" + +# Define default ATS +ts = Test.MakeATSProcess("ts") + +# First server is run during first test and +# second server is run during second test +server = Test.MakeOriginServer("server", ip='127.0.0.10') + +request_header = {"headers": "GET /cookiematches HTTP/1.1\r\nHost: www.example.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +# expected response from the origin server +response_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": ""} + +# add response to the server dictionary +server.addResponse("sessionfile.log", request_header, response_header) + +server2 = Test.MakeOriginServer("server2", ip='127.0.0.11') +request_header2 = {"headers": "GET /cookiedoesntmatch HTTP/1.1\r\nHost: www.example.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +# expected response from the origin server +response_header2 = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": ""} + +# add response to the server dictionary +server2.addResponse("sessionfile.log", request_header2, response_header2) + +# Setup the remap configuration +config_path = os.path.join(Test.TestDirectory, "configs/matchuriconfig.txt") +with open(config_path, 'r') as config_file: + config1 = config_file.read() + +ts.Disk.records_config.update({ + 'proxy.config.diags.debug.enabled': 1, + 'proxy.config.diags.debug.tags': 'cookie_remap.*|http.*|dns.*', +}) + +config1 = config1.replace("$PORT", str(server.Variables.Port)) +config1 = config1.replace("$ALTPORT", str(server2.Variables.Port)) + +ts.Disk.File(ts.Variables.CONFIGDIR +"/matchuriconfig.txt", exists=False, id="config1") +ts.Disk.config1.WriteOn(config1) + +ts.Disk.remap_config.AddLine( + 'map http://www.example.com/magic http://shouldnothit.com @plugin=cookie_remap.so @pparam=config/matchuriconfig.txt' +) + +# Positive test case, URI matches rule +tr = Test.AddTestRun("URI value matches") +tr.Processes.Default.Command = ''' +curl +--proxy 127.0.0.1:{0} +"http://www.example.com/magic/thisispartofthepath" +-H "Proxy-Connection: keep-alive" +--verbose +'''.format(ts.Variables.port) +tr.Processes.Default.ReturnCode = 0 +# time delay as proxy.config.http.wait_for_cache could be broken +tr.Processes.Default.StartBefore(server, ready=When.PortOpen(server.Variables.Port)) +tr.Processes.Default.StartBefore(Test.Processes.ts) +tr.StillRunningAfter = ts + +server.Streams.All = "gold/matchcookie.gold" + +# Negative test case that doesn't remap because URI doesn't match +tr = Test.AddTestRun("URI value doesn't match") +tr.Processes.Default.Command = ''' +curl +--proxy 127.0.0.1:{0} +"http://www.example.com/magic" +-H"Cookie: fpbeta=etc" +-H "Proxy-Connection: keep-alive" +--verbose +'''.format(ts.Variables.port) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.StartBefore(server2, ready=When.PortOpen(server2.Variables.Port)) +tr.StillRunningAfter = ts + +server2.Streams.All = "gold/wontmatchcookie.gold" diff --git a/tests/gold_tests/pluginTest/cookie_remap/matrixparams.test.py b/tests/gold_tests/pluginTest/cookie_remap/matrixparams.test.py new file mode 100644 index 00000000000..45ecb7b491e --- /dev/null +++ b/tests/gold_tests/pluginTest/cookie_remap/matrixparams.test.py @@ -0,0 +1,175 @@ +''' +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +Test.Summary = ''' + +''' +Test.SkipUnless(Condition.PluginExists('cookie_remap.so')) +# need Curl +Test.SkipUnless( + Condition.HasProgram("curl", "Curl need to be installed on system for this test to work") +) +Test.ContinueOnFail = True +Test.testName = "cookie_remap: Tests when matrix parameters are present" +Test.SkipIf(Condition.true("Test is temporarily turned off, to be fixed according to an incompatible plugin API change (PR #4964)")) + +# Define default ATS +ts = Test.MakeATSProcess("ts") + +# We just need a server to capture ATS outgoing requests +# so that we can verify the remap rules +# That's why I am not adding any canned request/response +server = Test.MakeOriginServer("server", ip='127.0.0.10') + +request_header = {"headers": "GET /eighth/magic;matrix=1/eighth HTTP/1.1\r\nHost: www.example.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +response_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": ""} + +server.addResponse("sessionfile.log", request_header, response_header) + +request_header_2 = {"headers": "GET /eighth/magic;matrix=1/eighth?hello=10 HTTP/1.1\r\nHost: www.example.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +response_header_2 = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": ""} + +server.addResponse("sessionfile.log", request_header_2, response_header_2) + +request_header_3 = {"headers": "GET /tenth/magic/tenth;matrix=2 HTTP/1.1\r\nHost: www.example.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +response_header_3 = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": ""} + +server.addResponse("sessionfile.log", request_header_3, response_header_3) + +request_header_4 = {"headers": "GET /tenth/magic/tenth;matrix=2?query=10 HTTP/1.1\r\nHost: www.example.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +response_header_4 = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": ""} + +server.addResponse("sessionfile.log", request_header_4, response_header_4) + +request_header_5 = {"headers": "GET /eleventh/magic;matrix=4/eleventh;matrix=2?query=true HTTP/1.1\r\nHost: www.example.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +response_header_5 = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": ""} + +server.addResponse("sessionfile.log", request_header_5, response_header_5) + +# Setup the remap configuration +config_path = os.path.join(Test.TestDirectory, "configs/matrixconfig.txt") +with open(config_path, 'r') as config_file: + config1 = config_file.read() + +ts.Disk.records_config.update({ + 'proxy.config.diags.debug.enabled': 1, + 'proxy.config.diags.debug.tags': 'cookie_remap.*|http.*|dns.*', +}) + +config1 = config1.replace("$PORT", str(server.Variables.Port)) + +ts.Disk.File(ts.Variables.CONFIGDIR +"/matrixconfig.txt", exists=False, id="config1") +ts.Disk.config1.WriteOn(config1) + +ts.Disk.remap_config.AddLine( + 'map http://www.example.com/eighth http://shouldnothit.com @plugin=cookie_remap.so @pparam=config/matrixconfig.txt' +) +ts.Disk.remap_config.AddLine( + 'map http://www.example.com/ninth http://shouldnothit.com @plugin=cookie_remap.so @pparam=config/matrixconfig.txt' +) +ts.Disk.remap_config.AddLine( + 'map http://www.example.com/tenth http://shouldnothit.com @plugin=cookie_remap.so @pparam=config/matrixconfig.txt' +) +ts.Disk.remap_config.AddLine( + 'map http://www.example.com/eleventh http://shouldnothit.com @plugin=cookie_remap.so @pparam=config/matrixconfig.txt' +) + +tr = Test.AddTestRun("path is substituted") +tr.Processes.Default.Command = ''' +curl \ +--proxy 127.0.0.1:{0} \ +"http://www.example.com/eighth/magic;matrix=1" \ +-H "Proxy-Connection: keep-alive" \ +--verbose \ +'''.format(ts.Variables.port) +tr.Processes.Default.ReturnCode = 0 +# time delay as proxy.config.http.wait_for_cache could be broken +tr.Processes.Default.StartBefore(server, ready=When.PortOpen(server.Variables.Port)) +tr.Processes.Default.StartBefore(Test.Processes.ts) +tr.StillRunningAfter = ts +tr.StillRunningAfter = server + +tr = Test.AddTestRun("path is substituted when matrix \ + and query is present") +tr.Processes.Default.Command = ''' +curl \ +--proxy 127.0.0.1:{0} \ +"http://www.example.com/eighth/magic;matrix=1?hello=10" \ +-H "Proxy-Connection: keep-alive" \ +--verbose \ +'''.format(ts.Variables.port) +tr.Processes.Default.ReturnCode = 0 +tr.StillRunningAfter = ts +tr.StillRunningAfter = server + +tr = Test.AddTestRun("Another $path substitution passing matrix \ + and query and replacing query") +tr.Processes.Default.Command = ''' +curl \ +--proxy 127.0.0.1:{0} \ +"http://www.example.com/ninth/magic;matrix=5?hello=16" \ +-H "Proxy-Connection: keep-alive" \ +--verbose \ +'''.format(ts.Variables.port) +tr.Processes.Default.ReturnCode = 0 +tr.StillRunningAfter = ts +tr.StillRunningAfter = server + +tr = Test.AddTestRun("$path substitution in sendto and \ + inserting matrix parameters in remap") +tr.Processes.Default.Command = ''' +curl \ +--proxy 127.0.0.1:{0} \ +"http://www.example.com/tenth/magic" \ +-H "Proxy-Connection: keep-alive" \ +--verbose \ +'''.format(ts.Variables.port) +tr.Processes.Default.ReturnCode = 0 +tr.StillRunningAfter = ts +tr.StillRunningAfter = server + +tr = Test.AddTestRun("inserting matrix params in remap and \ + passing along query string") +tr.Processes.Default.Command = ''' +curl \ +--proxy 127.0.0.1:{0} \ +"http://www.example.com/tenth/magic?query=10" \ +-H "Proxy-Connection: keep-alive" \ +--verbose \ +'''.format(ts.Variables.port) +tr.Processes.Default.ReturnCode = 0 +tr.StillRunningAfter = ts +tr.StillRunningAfter = server + +tr = Test.AddTestRun("Another test to verify matrix and query \ + params are passed along") +tr.Processes.Default.Command = ''' +curl \ +--proxy 127.0.0.1:{0} \ +"http://www.example.com/eleventh/magic;matrix=4?query=12" \ +-H "Proxy-Connection: keep-alive" \ +-H "Proxy-Connection: keep-alive" \ +--verbose \ +'''.format(ts.Variables.port) +tr.Processes.Default.ReturnCode = 0 +tr.StillRunningAfter = ts +tr.StillRunningAfter = server + +server.Streams.All = "gold/matrix.gold" + diff --git a/tests/gold_tests/pluginTest/cookie_remap/notexistscookie.test.py b/tests/gold_tests/pluginTest/cookie_remap/notexistscookie.test.py new file mode 100644 index 00000000000..c645ca58f1f --- /dev/null +++ b/tests/gold_tests/pluginTest/cookie_remap/notexistscookie.test.py @@ -0,0 +1,104 @@ +''' +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +Test.Summary = ''' + +''' +Test.SkipUnless(Condition.PluginExists('cookie_remap.so')) +# need Curl +Test.SkipUnless( + Condition.HasProgram("curl", "Curl need to be installed on system for this test to work") +) +Test.ContinueOnFail = True +Test.testName = "cookie_remap: cookie not exists" + +# Define default ATS +ts = Test.MakeATSProcess("ts") + +# First server is run during first test and +# second server is run during second test +server = Test.MakeOriginServer("server", ip='127.0.0.10') + +request_header = {"headers": "GET /cookiedoesntexist HTTP/1.1\r\nHost: www.example.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +# expected response from the origin server +response_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": ""} + +# add response to the server dictionary +server.addResponse("sessionfile.log", request_header, response_header) + +server2 = Test.MakeOriginServer("server2", ip='127.0.0.11') +request_header2 = {"headers": "GET /cookieexists HTTP/1.1\r\nHost: www.example.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +# expected response from the origin server +response_header2 = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": ""} + +# add response to the server dictionary +server2.addResponse("sessionfile.log", request_header2, response_header2) + +# Setup the remap configuration +config_path = os.path.join(Test.TestDirectory, "configs/notexistsconfig.txt") +with open(config_path, 'r') as config_file: + config1 = config_file.read() + +ts.Disk.records_config.update({ + 'proxy.config.diags.debug.enabled': 1, + 'proxy.config.diags.debug.tags': 'cookie_remap.*|http.*|dns.*', +}) + +config1 = config1.replace("$PORT", str(server.Variables.Port)) +config1 = config1.replace("$ALTPORT", str(server2.Variables.Port)) + +ts.Disk.File(ts.Variables.CONFIGDIR +"/notexistsconfig.txt", exists=False, id="config1") +ts.Disk.config1.WriteOn(config1) + +ts.Disk.remap_config.AddLine( + 'map http://www.example.com/magic http://shouldnothit.com @plugin=cookie_remap.so @pparam=config/notexistsconfig.txt' +) + +# Positive test case that remaps because cookie doesn't exist +tr = Test.AddTestRun("cookie fpbeta doesn't exist") +tr.Processes.Default.Command = ''' +curl +--proxy 127.0.0.1:{0} +"http://www.example.com/magic" +-H "Proxy-Connection: keep-alive" +--verbose +'''.format(ts.Variables.port) +tr.Processes.Default.ReturnCode = 0 +# time delay as proxy.config.http.wait_for_cache could be broken +tr.Processes.Default.StartBefore(server, ready=When.PortOpen(server.Variables.Port)) +tr.Processes.Default.StartBefore(Test.Processes.ts) +tr.StillRunningAfter = ts + +server.Streams.All = "gold/doesntexistcookie.gold" + +# Negative test case that doesn't remap because cookie exists +tr = Test.AddTestRun("cooke fpbeta exists") +tr.Processes.Default.Command = ''' +curl +--proxy 127.0.0.1:{0} +"http://www.example.com/magic" +-H"Cookie: fpbeta=etc" +-H "Proxy-Connection: keep-alive" +--verbose +'''.format(ts.Variables.port) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.StartBefore(server2, ready=When.PortOpen(server2.Variables.Port)) +tr.StillRunningAfter = ts + +server2.Streams.All = "gold/existscookie.gold" diff --git a/tests/gold_tests/pluginTest/cookie_remap/regexcookie.test.py b/tests/gold_tests/pluginTest/cookie_remap/regexcookie.test.py new file mode 100644 index 00000000000..ad78ec8b76c --- /dev/null +++ b/tests/gold_tests/pluginTest/cookie_remap/regexcookie.test.py @@ -0,0 +1,105 @@ +''' +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +Test.Summary = ''' + +''' +Test.SkipUnless(Condition.PluginExists('cookie_remap.so')) +# need Curl +Test.SkipUnless( + Condition.HasProgram("curl", "Curl need to be installed on system for this test to work") +) +Test.ContinueOnFail = True +Test.testName = "cookie_remap: cookie regex match and substition" + +# Define default ATS +ts = Test.MakeATSProcess("ts") + +# First server is run during first test and +# second server is run during second test +server = Test.MakeOriginServer("server", ip='127.0.0.10') + +request_header = {"headers": "GET /regexmatches?cookies=oreos-chipsahoy-icecream HTTP/1.1\r\nHost: www.example.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +# expected response from the origin server +response_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": ""} + +# add response to the server dictionary +server.addResponse("sessionfile.log", request_header, response_header) + +server2 = Test.MakeOriginServer("server2", ip='127.0.0.11') +request_header2 = {"headers": "GET /regexdoesntmatch HTTP/1.1\r\nHost: www.example.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +# expected response from the origin server +response_header2 = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": ""} + +# add response to the server dictionary +server2.addResponse("sessionfile.log", request_header2, response_header2) + +# Setup the remap configuration +config_path = os.path.join(Test.TestDirectory, "configs/regexconfig.txt") +with open(config_path, 'r') as config_file: + config1 = config_file.read() + +ts.Disk.records_config.update({ + 'proxy.config.diags.debug.enabled': 1, + 'proxy.config.diags.debug.tags': 'cookie_remap.*|http.*|dns.*', +}) + +config1 = config1.replace("$PORT", str(server.Variables.Port)) +config1 = config1.replace("$ALTPORT", str(server2.Variables.Port)) + +ts.Disk.File(ts.Variables.CONFIGDIR +"/regexconfig.txt", exists=False, id="config1") +ts.Disk.config1.WriteOn(config1) + +ts.Disk.remap_config.AddLine( + 'map http://www.example.com/magic http://shouldnothit.com @plugin=cookie_remap.so @pparam=config/regexconfig.txt' +) + +# Positive test case that remaps because cookie regex matches +tr = Test.AddTestRun("cookie regex matches") +tr.Processes.Default.Command = ''' +curl +--proxy 127.0.0.1:{0} +"http://www.example.com/magic" +-H"Cookie: fpbeta=ilove-oreos-chipsahoy-icecream" +-H "Proxy-Connection: keep-alive" +--verbose +'''.format(ts.Variables.port) +tr.Processes.Default.ReturnCode = 0 +# time delay as proxy.config.http.wait_for_cache could be broken +tr.Processes.Default.StartBefore(server, ready=When.PortOpen(server.Variables.Port)) +tr.Processes.Default.StartBefore(Test.Processes.ts) +tr.StillRunningAfter = ts + +server.Streams.All = "gold/regexmatches.gold" + +# Negative test case that doesn't remap because cookie regex doesn't match +tr = Test.AddTestRun("cookie regex doesn't match") +tr.Processes.Default.Command = ''' +curl +--proxy 127.0.0.1:{0} +"http://www.example.com/magic" +-H"Cookie: fpbeta=etc" +-H "Proxy-Connection: keep-alive" +--verbose +'''.format(ts.Variables.port) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.StartBefore(server2, ready=When.PortOpen(server2.Variables.Port)) +tr.StillRunningAfter = ts + +server2.Streams.All = "gold/regexdoesntmatch.gold" diff --git a/tests/gold_tests/pluginTest/cookie_remap/setstatus.test.py b/tests/gold_tests/pluginTest/cookie_remap/setstatus.test.py new file mode 100644 index 00000000000..bcbeb3ba2b6 --- /dev/null +++ b/tests/gold_tests/pluginTest/cookie_remap/setstatus.test.py @@ -0,0 +1,80 @@ +''' +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +Test.Summary = ''' + +''' +Test.SkipUnless(Condition.PluginExists('cookie_remap.so')) +# need Curl +Test.SkipUnless( + Condition.HasProgram("curl", "Curl need to be installed on system for this test to work") +) +Test.ContinueOnFail = True +Test.testName = "cookie_remap: set HTTP status of the response" + +# Define default ATS +ts = Test.MakeATSProcess("ts") + +# Setup the remap configuration +config_path = os.path.join(Test.TestDirectory, "configs/statusconfig.txt") +with open(config_path, 'r') as config_file: + config1 = config_file.read() + +ts.Disk.records_config.update({ + 'proxy.config.diags.debug.enabled': 1, + 'proxy.config.diags.debug.tags': 'cookie_remap.*|http.*|dns.*', +}) + +ts.Disk.File(ts.Variables.CONFIGDIR +"/statusconfig.txt", exists=False, id="config1") +ts.Disk.config1.WriteOn(config1) + +ts.Disk.remap_config.AddLine( + 'map http://www.example.com/magic http://shouldnothit.com @plugin=cookie_remap.so @pparam=config/statusconfig.txt' +) + +# Plugin sets the HTTP status because first rule matches +tr = Test.AddTestRun("Sets the status to 205") +tr.Processes.Default.Command = ''' +curl +--proxy 127.0.0.1:{0} +"http://www.example.com/magic" +-H"Cookie: fpbeta=magic" +-H "Proxy-Connection: keep-alive" +--verbose +'''.format(ts.Variables.port) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.StartBefore(Test.Processes.ts) +tr.StillRunningAfter = ts + +tr.Streams.All = "gold/matchstatus.gold" + +# Plugin sets the HTTP status because the else rule matches (i.e. no match) +tr = Test.AddTestRun("Sets the else status to 400") +tr.Processes.Default.Command = ''' +curl +--proxy 127.0.0.1:{0} +"http://www.example.com/magic" +-H "Proxy-Connection: keep-alive" +-H "Proxy-Connection: keep-alive" +--verbose +'''.format(ts.Variables.port) +tr.Processes.Default.ReturnCode = 0 +tr.StillRunningAfter = ts + +tr.Streams.All = "gold/matchelsestatus.gold" diff --git a/tests/gold_tests/pluginTest/cookie_remap/subcookie.test.py b/tests/gold_tests/pluginTest/cookie_remap/subcookie.test.py new file mode 100644 index 00000000000..23384f67f01 --- /dev/null +++ b/tests/gold_tests/pluginTest/cookie_remap/subcookie.test.py @@ -0,0 +1,102 @@ +''' +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +Test.Summary = ''' + +''' +Test.SkipUnless(Condition.PluginExists('cookie_remap.so')) +# need Curl +Test.SkipUnless( + Condition.HasProgram("curl", "Curl need to be installed on system for this test to work") +) +Test.ContinueOnFail = True +Test.testName = "cookie_remap: test connector" +Test.SkipIf(Condition.true("Test is temporarily turned off, to be fixed according to an incompatible plugin API change (PR #4964)")) + +# Define default ATS +ts = Test.MakeATSProcess("ts") + +# First server is run during first test and +# second server is run during second test +server = Test.MakeOriginServer("server", ip='127.0.0.10') + +request_header = {"headers": "GET /cookiematches HTTP/1.1\r\nHost: www.example.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +# expected response from the origin server +response_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": ""} + +# add response to the server dictionary +server.addResponse("sessionfile.log", request_header, response_header) + +server2 = Test.MakeOriginServer("server2", ip='127.0.0.11') +request_header2 = {"headers": "GET /cookiedoesntmatch HTTP/1.1\r\nHost: www.example.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +# expected response from the origin server +response_header2 = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": ""} + +# add response to the server dictionary +server2.addResponse("sessionfile.log", request_header2, response_header2) + +# Setup the remap configuration +config_path = os.path.join(Test.TestDirectory, "configs/subcookie.txt") +with open(config_path, 'r') as config_file: + config1 = config_file.read() + +ts.Disk.records_config.update({ + 'proxy.config.diags.debug.enabled': 1, + 'proxy.config.diags.debug.tags': 'cookie_remap.*|http.*|dns.*', +}) + +config1 = config1.replace("$PORT", str(server.Variables.Port)) +config1 = config1.replace("$ALTPORT", str(server2.Variables.Port)) + +ts.Disk.File(ts.Variables.CONFIGDIR +"/subcookie.txt", exists=False, id="config1") +ts.Disk.config1.WriteOn(config1) + +ts.Disk.remap_config.AddLine( + 'map http://www.example.com/magic http://shouldnothit.com @plugin=cookie_remap.so @pparam=config/subcookie.txt' +) + +# Positive test case that remaps because all connected operations pass +tr = Test.AddTestRun("cookie value matches") +# Unlike in other places I am using a single line string because the & seems to +# be interpreted by the autest framework or the shell (tried escaping with \) +tr.Processes.Default.Command = 'curl --proxy 127.0.0.1:{0} "http://www.example.com/magic" -H"Cookie: fpbeta=a=1&b=2&c=3" -H "Proxy-Connection: keep-alive" --verbose '.format(ts.Variables.port) +#tr.Processes.Default.Command = ''' +#curl +#--proxy 127.0.0.1:{0} +#"http://www.example.com/magic" +#-H\"Cookie: fpbeta=a=1&b=2&c=3\" +#-H "Proxy-Connection: keep-alive" +#--verbose +#'''.format(ts.Variables.port) +tr.Processes.Default.ReturnCode = 0 +# time delay as proxy.config.http.wait_for_cache could be broken +tr.Processes.Default.StartBefore(server, ready=When.PortOpen(server.Variables.Port)) +tr.Processes.Default.StartBefore(Test.Processes.ts) +tr.StillRunningAfter = ts + +server.Streams.All = "gold/matchcookie.gold" + +# Negative test case that doesn't remap because not all subops pass +tr = Test.AddTestRun("cookie value doesn't match") +tr.Processes.Default.Command = 'curl --proxy 127.0.0.1:{0} "http://www.example.com/magic" -H"Cookie: fpbeta=a=1&b=2&c=4" -H "Proxy-Connection: keep-alive" --verbose '.format(ts.Variables.port) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.StartBefore(server2, ready=When.PortOpen(server2.Variables.Port)) +tr.StillRunningAfter = ts + +server2.Streams.All = "gold/wontmatchcookie.gold" diff --git a/tests/gold_tests/pluginTest/cookie_remap/substitute.test.py b/tests/gold_tests/pluginTest/cookie_remap/substitute.test.py new file mode 100644 index 00000000000..bea8400cf10 --- /dev/null +++ b/tests/gold_tests/pluginTest/cookie_remap/substitute.test.py @@ -0,0 +1,126 @@ +''' +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +Test.Summary = ''' + +''' +Test.SkipUnless(Condition.PluginExists('cookie_remap.so')) +# need Curl +Test.SkipUnless( + Condition.HasProgram("curl", "Curl need to be installed on system for this test to work") +) +Test.ContinueOnFail = True +Test.testName = "cookie_remap: Substitute variables" +Test.SkipIf(Condition.true("Test is temporarily turned off, to be fixed according to an incompatible plugin API change (PR #4964)")) + +# Define default ATS +ts = Test.MakeATSProcess("ts") + +server = Test.MakeOriginServer("server", ip='127.0.0.10') + +request_header = {"headers": "GET /photos/search?query=magic HTTP/1.1\r\nHost: www.example.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +response_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": ""} + +server.addResponse("sessionfile.log", request_header, response_header) + +request_header_2 = {"headers": "GET /photos/search?query=/theunmatchedpath HTTP/1.1\r\nHost: www.example.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +response_header_2 = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": ""} + +server.addResponse("sessionfile.log", request_header_2, response_header_2) + +request_header_3 = {"headers": "GET /photos/search/magic/foobar HTTP/1.1\r\nHost: www.example.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +response_header_3 = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": ""} + +server.addResponse("sessionfile.log", request_header_3, response_header_3) + +# Setup the remap configuration +config_path = os.path.join(Test.TestDirectory, "configs/substituteconfig.txt") +with open(config_path, 'r') as config_file: + config1 = config_file.read() + +ts.Disk.records_config.update({ + 'proxy.config.diags.debug.enabled': 1, + 'proxy.config.diags.debug.tags': 'cookie_remap.*|http.*|dns.*', +}) + +config1 = config1.replace("$PORT", str(server.Variables.Port)) + +ts.Disk.File(ts.Variables.CONFIGDIR +"/substituteconfig.txt", exists=False, id="config1") +ts.Disk.config1.WriteOn(config1) + +ts.Disk.remap_config.AddLine( + 'map http://www.example.com/magic http://shouldnothit.com @plugin=cookie_remap.so @pparam=config/substituteconfig.txt' +) + +tr = Test.AddTestRun("Substitute $path in the dest query") +tr.Processes.Default.Command = ''' +curl +--proxy 127.0.0.1:{0} +"http://www.example.com/magic" +-H"Cookie: fpbeta=abcd" +-H "Proxy-Connection: keep-alive" +--verbose +'''.format(ts.Variables.port) +tr.Processes.Default.ReturnCode = 0 +# time delay as proxy.config.http.wait_for_cache could be broken +tr.Processes.Default.StartBefore(server, ready=When.PortOpen(server.Variables.Port)) +tr.Processes.Default.StartBefore(Test.Processes.ts) +tr.StillRunningAfter = ts +tr.StillRunningAfter = server + +tr = Test.AddTestRun("Substitute $unmatched_path in the dest query") +tr.Processes.Default.Command = ''' +curl +--proxy 127.0.0.1:{0} +"http://www.example.com/magic/theunmatchedpath" +-H"Cookie: oxalpha=3333" +-H "Proxy-Connection: keep-alive" +--verbose +'''.format(ts.Variables.port) +tr.Processes.Default.ReturnCode = 0 +tr.StillRunningAfter = ts +tr.StillRunningAfter = server + +tr = Test.AddTestRun("Substitute $cr_req_url using $cr_urlencode") +tr.Processes.Default.Command = ''' +curl +--proxy 127.0.0.1:{0} +"http://www.example.com/magic" +-H"Cookie: acgamma=dfndfdfd" +-H "Proxy-Connection: keep-alive" +--verbose +'''.format(ts.Variables.port) +tr.Processes.Default.ReturnCode = 0 +tr.StillRunningAfter = ts +tr.StillRunningAfter = server + +tr = Test.AddTestRun("Substitute $path as is in outgoing path") +tr.Processes.Default.Command = ''' +curl +--proxy 127.0.0.1:{0} +"http://www.example.com/magic/foobar" +-H "Proxy-Connection: keep-alive" +--verbose +'''.format(ts.Variables.port) +tr.Processes.Default.ReturnCode = 0 +tr.StillRunningAfter = ts +tr.StillRunningAfter = server + +server.Streams.All = "gold/substitute.gold" + diff --git a/tests/gold_tests/pluginTest/regex_revalidate/gold/regex_reval-hit.gold b/tests/gold_tests/pluginTest/regex_revalidate/gold/regex_reval-hit.gold new file mode 100644 index 00000000000..f98f8ac02c3 --- /dev/null +++ b/tests/gold_tests/pluginTest/regex_revalidate/gold/regex_reval-hit.gold @@ -0,0 +1,10 @@ +HTTP/1.1 200 OK +Etag: `` +Cache-Control: max-age=``,public +Content-Length: 3 +Date: `` +Connection: `` +Server: `` +X-Cache: hit-fresh +`` +`` diff --git a/tests/gold_tests/pluginTest/regex_revalidate/gold/regex_reval-miss.gold b/tests/gold_tests/pluginTest/regex_revalidate/gold/regex_reval-miss.gold new file mode 100644 index 00000000000..35ec6036eff --- /dev/null +++ b/tests/gold_tests/pluginTest/regex_revalidate/gold/regex_reval-miss.gold @@ -0,0 +1,10 @@ +HTTP/1.1 200 OK +Etag: `` +Cache-Control: max-age=``,public +Content-Length: 3 +Date: `` +Connection: `` +Server: `` +X-Cache: miss +`` +`` diff --git a/tests/gold_tests/pluginTest/regex_revalidate/gold/regex_reval-stale.gold b/tests/gold_tests/pluginTest/regex_revalidate/gold/regex_reval-stale.gold new file mode 100644 index 00000000000..ea29316b964 --- /dev/null +++ b/tests/gold_tests/pluginTest/regex_revalidate/gold/regex_reval-stale.gold @@ -0,0 +1,10 @@ +HTTP/1.1 200 OK +Etag: `` +Cache-Control: max-age=``,public +Content-Length: 3 +Date: `` +Connection: `` +Server: `` +X-Cache: hit-stale +`` +`` diff --git a/tests/gold_tests/pluginTest/regex_revalidate/regex_revalidate.test.py b/tests/gold_tests/pluginTest/regex_revalidate/regex_revalidate.test.py new file mode 100644 index 00000000000..bb2aa96fe6e --- /dev/null +++ b/tests/gold_tests/pluginTest/regex_revalidate/regex_revalidate.test.py @@ -0,0 +1,273 @@ +''' +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import time +Test.Summary = ''' +Test a basic regex_revalidate +''' + +## Test description: +# Load up cache, ensure fresh +# Create regex reval rule, config reload: +# ensure item is staled only once. +# Add a new rule, config reload: +# ensure item isn't restaled again, but rule still in effect. +# +# If the rule disappears from regex_revalidate.conf its still loaded!! +# A rule's expiry can't be changed after the fact! + +Test.SkipUnless( + Condition.HasProgram("curl", "Curl need to be installed on system for this test to work"), + Condition.PluginExists('regex_revalidate.so'), + Condition.PluginExists('xdebug.so') +) +Test.ContinueOnFail = False + +# configure origin server +server = Test.MakeOriginServer("server") + +# Define ATS and configure +ts = Test.MakeATSProcess("ts", command="traffic_manager") + +#**testname is required** +#testName = "regex_reval" + +# default root +request_header_0 = {"headers": + "GET / HTTP/1.1\r\n" + + "Host: www.example.com\r\n" + + "\r\n", + "timestamp": "1469733493.993", + "body": "", +} + +response_header_0 = {"headers": + "HTTP/1.1 200 OK\r\n" + + "Connection: close\r\n" + + "Cache-Control: max-age=300\r\n" + + "\r\n", + "timestamp": "1469733493.993", + "body": "xxx", +} + +# cache item path1 +request_header_1 = {"headers": + "GET /path1 HTTP/1.1\r\n" + + "Host: www.example.com\r\n" + + "\r\n", + "timestamp": "1469733493.993", + "body": "" +} +response_header_1 = {"headers": + "HTTP/1.1 200 OK\r\n" + + "Connection: close\r\n" + + 'Etag: "path1"\r\n' + + "Cache-Control: max-age=600,public\r\n" + + "\r\n", + "timestamp": "1469733493.993", + "body": "abc" +} + +# cache item path1a +request_header_2 = {"headers": + "GET /path1a HTTP/1.1\r\n" + + "Host: www.example.com\r\n" + + "\r\n", + "timestamp": "1469733493.993", + "body": "" +} +response_header_2 = {"headers": + "HTTP/1.1 200 OK\r\n" + + "Connection: close\r\n" + + 'Etag: "path1a"\r\n' + + "Cache-Control: max-age=600,public\r\n" + + "\r\n", + "timestamp": "1469733493.993", + "body": "cde" +} + +# cache item path2a +request_header_3 = {"headers": + "GET /path2a HTTP/1.1\r\n" + + "Host: www.example.com\r\n" + + "\r\n", + "timestamp": "1469733493.993", + "body": "" +} +response_header_3 = {"headers": + "HTTP/1.1 200 OK\r\n" + + "Connection: close\r\n" + + 'Etag: "path2a"\r\n' + + "Cache-Control: max-age=900,public\r\n" + + "\r\n", + "timestamp": "1469733493.993", + "body": "efg" +} + +server.addResponse("sessionlog.json", request_header_0, response_header_0) +server.addResponse("sessionlog.json", request_header_1, response_header_1) +server.addResponse("sessionlog.json", request_header_2, response_header_2) +server.addResponse("sessionlog.json", request_header_3, response_header_3) + +# Configure ATS server +ts.Disk.plugin_config.AddLine('xdebug.so') +ts.Disk.plugin_config.AddLine( + 'regex_revalidate.so -d -c regex_revalidate.conf' +) + +regex_revalidate_conf_path = os.path.join(ts.Variables.CONFIGDIR, 'regex_revalidate.conf') +curl_and_args = 'curl -s -D - -v -H "x-debug: x-cache" -H "Host: www.example.com"' + +path1_rule = 'path1 {}\n'.format(int(time.time()) + 600) + +# Define first revistion for when trafficserver starts +ts.Disk.File(regex_revalidate_conf_path, typename="ats:config").AddLines([ + "# Empty\n" +]) + +ts.Disk.remap_config.AddLine( + 'map / http://127.0.0.1:{}'.format(server.Variables.Port) +) + +# minimal configuration +ts.Disk.records_config.update({ + 'proxy.config.diags.debug.enabled': 1, + 'proxy.config.diags.debug.tags': 'regex_revalidate', +# 'proxy.config.diags.debug.enabled': 0, + 'proxy.config.http.cache.http': 1, + 'proxy.config.http.wait_for_cache': 1, + 'proxy.config.http.insert_age_in_response': 0, + 'proxy.config.http.response_via_str': 3, + 'proxy.config.http.server_ports': '{}'.format(ts.Variables.port), +}) + +# 0 Test - Load cache (miss) (path1) +tr = Test.AddTestRun("Cache miss path1") +tr.Processes.Default.StartBefore(server) +tr.Processes.Default.StartBefore(Test.Processes.ts, ready=1) +tr.Processes.Default.Command = curl_and_args + ' http://127.0.0.1:{}/path1'.format(ts.Variables.port) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Streams.stdout = "gold/regex_reval-miss.gold" +tr.StillRunningAfter = ts + +# 1 Test - Load cache (miss) for later test (path1a) +tr = Test.AddTestRun("Cache miss path1a") +tr.Processes.Default.Command = curl_and_args + ' http://127.0.0.1:{}/path1a'.format(ts.Variables.port) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Streams.stdout = "gold/regex_reval-miss.gold" +tr.StillRunningAfter = ts + +# 2 Test - Load cache (miss) for later test (path2a) +tr = Test.AddTestRun("Cache miss path2a") +tr.Processes.Default.Command = curl_and_args + ' http://127.0.0.1:{}/path2a'.format(ts.Variables.port) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Streams.stdout = "gold/regex_reval-miss.gold" +tr.StillRunningAfter = ts + +# 3 Test - Cache hit path1 +tr = Test.AddTestRun("Cache hit fresh path1") +tr.Processes.Default.Command = curl_and_args + ' http://127.0.0.1:{}/path1'.format(ts.Variables.port) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Streams.stdout = "gold/regex_reval-hit.gold" +tr.StillRunningAfter = ts + +# 4 Stage - Reload new regex_revalidate +tr = Test.AddTestRun("Reload config add path1") +tr.Disk.File(regex_revalidate_conf_path, typename="ats:config").AddLines([ + path1_rule +]) +tr.StillRunningAfter = ts +tr.StillRunningAfter = server +tr.Processes.Default.Command = 'traffic_ctl config reload' +# Need to copy over the environment so traffic_ctl knows where to find the unix domain socket +tr.Processes.Default.Env = ts.Env +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.TimeOut = 5 +tr.TimeOut = 5 + +# 5 Test - Revalidate path1 +tr = Test.AddTestRun("Revalidate stale path1") +tr.DelayStart = 5 +tr.Processes.Default.Command = curl_and_args + ' http://127.0.0.1:{}/path1'.format(ts.Variables.port) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Streams.stdout = "gold/regex_reval-stale.gold" +tr.StillRunningAfter = ts + +# 6 Test - Cache hit (path1) +tr = Test.AddTestRun("Cache hit fresh path1") +tr.Processes.Default.Command = curl_and_args + ' http://127.0.0.1:{}/path1'.format(ts.Variables.port) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Streams.stdout = "gold/regex_reval-hit.gold" +tr.StillRunningAfter = ts + +# 7 Stage - Reload new regex_revalidate +tr = Test.AddTestRun("Reload config add path2") +tr.Disk.File(regex_revalidate_conf_path, typename="ats:config").AddLines([ + path1_rule, + 'path2 {}\n'.format(int(time.time()) + 700) +]) +tr.StillRunningAfter = ts +tr.StillRunningAfter = server +tr.Processes.Default.Command = 'traffic_ctl config reload' +# Need to copy over the environment so traffic_ctl knows where to find the unix domain socket +tr.Processes.Default.Env = ts.Env +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.TimeOut = 5 +tr.TimeOut = 5 + +# 8 Test - Cache hit (path1) +tr = Test.AddTestRun("Cache hit fresh path1") +tr.DelayStart = 5 +tr.Processes.Default.Command = curl_and_args + ' http://127.0.0.1:{}/path1'.format(ts.Variables.port) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Streams.stdout = "gold/regex_reval-hit.gold" +tr.StillRunningAfter = ts + +# 9 Test - Cache stale (check rule is still loaded) (path1a) +tr = Test.AddTestRun("Revalidate stale path1a") +tr.Processes.Default.Command = curl_and_args + ' http://127.0.0.1:{}/path1a'.format(ts.Variables.port) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Streams.stdout = "gold/regex_reval-stale.gold" +tr.StillRunningAfter = ts + +# The C version of regex_revalidate doesn't allow an existing rule to +# be changed by a reload. + +# 10 Stage - regex_revalidate rewrite rule early expire +tr = Test.AddTestRun("Reload config change path2") +tr.Disk.File(regex_revalidate_conf_path, typename="ats:config").AddLines([ + path1_rule, + 'path2 {}\n'.format(int(time.time()) - 100), +]) +tr.StillRunningAfter = ts +tr.StillRunningAfter = server +tr.Processes.Default.Command = 'traffic_ctl config reload' +# Need to copy over the environment so traffic_ctl knows where to find the unix domain socket +tr.Processes.Default.Env = ts.Env +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.TimeOut = 5 +tr.TimeOut = 5 + +# 11 Test - Cache hit (path2a) +tr = Test.AddTestRun("Cache hit stale path2a") +tr.DelayStart = 5 +tr.Processes.Default.Command = curl_and_args + ' http://127.0.0.1:{}/path2a'.format(ts.Variables.port) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Streams.stdout = "gold/regex_reval-stale.gold" +tr.StillRunningAfter = ts diff --git a/tests/gold_tests/pluginTest/slice/curlsort.sh b/tests/gold_tests/pluginTest/slice/curlsort.sh new file mode 100755 index 00000000000..eabf5618ff2 --- /dev/null +++ b/tests/gold_tests/pluginTest/slice/curlsort.sh @@ -0,0 +1,33 @@ +#!/bin/sh + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +cmd='curl' +for arg in "$@"; do + case "$arg" in + *\'*) +# arg=`printf '%s' "$arg" | sed s/'/'\"'\"'/g"` + arg=`printf '%s' "$arg"` + ;; + *) : ;; + esac + cmd="$cmd '$arg'" +done + +cmd="$cmd -s -D /dev/stdout -o /dev/stderr" + +eval " $cmd" | sort diff --git a/tests/gold_tests/pluginTest/slice/gold/slice_200.stderr.gold b/tests/gold_tests/pluginTest/slice/gold/slice_200.stderr.gold new file mode 100644 index 00000000000..24ad29c1a82 --- /dev/null +++ b/tests/gold_tests/pluginTest/slice/gold/slice_200.stderr.gold @@ -0,0 +1 @@ +lets go surfin now`` diff --git a/tests/gold_tests/pluginTest/slice/gold/slice_200.stdout.gold b/tests/gold_tests/pluginTest/slice/gold/slice_200.stdout.gold new file mode 100644 index 00000000000..1e704c37b3d --- /dev/null +++ b/tests/gold_tests/pluginTest/slice/gold/slice_200.stdout.gold @@ -0,0 +1,8 @@ +`` +Cache-Control: `` +Connection: `` +Content-Length: 18 +Date: `` +Etag: "path" +HTTP/1.1 200 OK +Server: `` diff --git a/tests/gold_tests/pluginTest/slice/gold/slice_206.stderr.gold b/tests/gold_tests/pluginTest/slice/gold/slice_206.stderr.gold new file mode 100644 index 00000000000..24ad29c1a82 --- /dev/null +++ b/tests/gold_tests/pluginTest/slice/gold/slice_206.stderr.gold @@ -0,0 +1 @@ +lets go surfin now`` diff --git a/tests/gold_tests/pluginTest/slice/gold/slice_206.stdout.gold b/tests/gold_tests/pluginTest/slice/gold/slice_206.stdout.gold new file mode 100644 index 00000000000..d4bd2de8782 --- /dev/null +++ b/tests/gold_tests/pluginTest/slice/gold/slice_206.stdout.gold @@ -0,0 +1,9 @@ +`` +Cache-Control: `` +Connection: `` +Content-Length: 18 +Content-Range: bytes 0-17/18 +Date: `` +Etag: "path" +HTTP/1.1 206 Partial Content +Server: `` diff --git a/tests/gold_tests/pluginTest/slice/gold/slice_first.stderr.gold b/tests/gold_tests/pluginTest/slice/gold/slice_first.stderr.gold new file mode 100644 index 00000000000..10893bb0688 --- /dev/null +++ b/tests/gold_tests/pluginTest/slice/gold/slice_first.stderr.gold @@ -0,0 +1 @@ +lets go`` diff --git a/tests/gold_tests/pluginTest/slice/gold/slice_first.stdout.gold b/tests/gold_tests/pluginTest/slice/gold/slice_first.stdout.gold new file mode 100644 index 00000000000..235b074c618 --- /dev/null +++ b/tests/gold_tests/pluginTest/slice/gold/slice_first.stdout.gold @@ -0,0 +1,9 @@ +`` +Cache-Control: max-age=`` +Connection: `` +Content-Length: 7 +Content-Range: bytes 0-6/18 +Date: `` +Etag: `` +HTTP/1.1 206 Partial Content +Server: `` diff --git a/tests/gold_tests/pluginTest/slice/gold/slice_last.stderr.gold b/tests/gold_tests/pluginTest/slice/gold/slice_last.stderr.gold new file mode 100644 index 00000000000..6c9cadd2dfd --- /dev/null +++ b/tests/gold_tests/pluginTest/slice/gold/slice_last.stderr.gold @@ -0,0 +1 @@ + now`` diff --git a/tests/gold_tests/pluginTest/slice/gold/slice_last.stdout.gold b/tests/gold_tests/pluginTest/slice/gold/slice_last.stdout.gold new file mode 100644 index 00000000000..88fa42bcfba --- /dev/null +++ b/tests/gold_tests/pluginTest/slice/gold/slice_last.stdout.gold @@ -0,0 +1,9 @@ +`` +Cache-Control: max-age=`` +Connection: `` +Content-Length: 4 +Content-Range: bytes 14-17/18 +Date: `` +Etag: `` +HTTP/1.1 206 Partial Content +Server: `` diff --git a/tests/gold_tests/pluginTest/slice/gold/slice_mid.stderr.gold b/tests/gold_tests/pluginTest/slice/gold/slice_mid.stderr.gold new file mode 100644 index 00000000000..a9dc13c610e --- /dev/null +++ b/tests/gold_tests/pluginTest/slice/gold/slice_mid.stderr.gold @@ -0,0 +1 @@ +go surfin no`` diff --git a/tests/gold_tests/pluginTest/slice/gold/slice_mid.stdout.gold b/tests/gold_tests/pluginTest/slice/gold/slice_mid.stdout.gold new file mode 100644 index 00000000000..5067f1cce49 --- /dev/null +++ b/tests/gold_tests/pluginTest/slice/gold/slice_mid.stdout.gold @@ -0,0 +1,9 @@ +`` +Cache-Control: max-age=`` +Connection: `` +Content-Length: 12 +Content-Range: bytes 5-16/18 +Date: `` +Etag: `` +HTTP/1.1 206 Partial Content +Server: `` diff --git a/tests/gold_tests/pluginTest/slice/gold_error/crr.stderr.gold b/tests/gold_tests/pluginTest/slice/gold_error/crr.stderr.gold new file mode 100644 index 00000000000..fad05754105 --- /dev/null +++ b/tests/gold_tests/pluginTest/slice/gold_error/crr.stderr.gold @@ -0,0 +1 @@ +the quick`` diff --git a/tests/gold_tests/pluginTest/slice/gold_error/crr.stdout.gold b/tests/gold_tests/pluginTest/slice/gold_error/crr.stdout.gold new file mode 100644 index 00000000000..955a42451c0 --- /dev/null +++ b/tests/gold_tests/pluginTest/slice/gold_error/crr.stdout.gold @@ -0,0 +1,9 @@ +`` +Cache-Control: `` +Connection: `` +Content-Length: 19 +Date: `` +Etag: `` +HTTP/1.1 200 OK +Server: `` +Via: `` diff --git a/tests/gold_tests/pluginTest/slice/gold_error/etag.stderr.gold b/tests/gold_tests/pluginTest/slice/gold_error/etag.stderr.gold new file mode 100644 index 00000000000..fad05754105 --- /dev/null +++ b/tests/gold_tests/pluginTest/slice/gold_error/etag.stderr.gold @@ -0,0 +1 @@ +the quick`` diff --git a/tests/gold_tests/pluginTest/slice/gold_error/etag.stdout.gold b/tests/gold_tests/pluginTest/slice/gold_error/etag.stdout.gold new file mode 100644 index 00000000000..955a42451c0 --- /dev/null +++ b/tests/gold_tests/pluginTest/slice/gold_error/etag.stdout.gold @@ -0,0 +1,9 @@ +`` +Cache-Control: `` +Connection: `` +Content-Length: 19 +Date: `` +Etag: `` +HTTP/1.1 200 OK +Server: `` +Via: `` diff --git a/tests/gold_tests/pluginTest/slice/gold_error/lm.stderr.gold b/tests/gold_tests/pluginTest/slice/gold_error/lm.stderr.gold new file mode 100644 index 00000000000..fad05754105 --- /dev/null +++ b/tests/gold_tests/pluginTest/slice/gold_error/lm.stderr.gold @@ -0,0 +1 @@ +the quick`` diff --git a/tests/gold_tests/pluginTest/slice/gold_error/lm.stdout.gold b/tests/gold_tests/pluginTest/slice/gold_error/lm.stdout.gold new file mode 100644 index 00000000000..2d83f7e9fd3 --- /dev/null +++ b/tests/gold_tests/pluginTest/slice/gold_error/lm.stdout.gold @@ -0,0 +1,9 @@ +`` +Cache-Control: `` +Connection: `` +Content-Length: 19 +Date: `` +HTTP/1.1 200 OK +Last-Modified: `` +Server: `` +Via: `` diff --git a/tests/gold_tests/pluginTest/slice/gold_error/non206.stderr.gold b/tests/gold_tests/pluginTest/slice/gold_error/non206.stderr.gold new file mode 100644 index 00000000000..fad05754105 --- /dev/null +++ b/tests/gold_tests/pluginTest/slice/gold_error/non206.stderr.gold @@ -0,0 +1 @@ +the quick`` diff --git a/tests/gold_tests/pluginTest/slice/gold_error/non206.stdout.gold b/tests/gold_tests/pluginTest/slice/gold_error/non206.stdout.gold new file mode 100644 index 00000000000..86ee5ddcb2a --- /dev/null +++ b/tests/gold_tests/pluginTest/slice/gold_error/non206.stdout.gold @@ -0,0 +1,10 @@ +`` +Cache-Control: `` +Connection: `` +Content-Length: 19 +Date: `` +Etag: `` +HTTP/1.1 200 OK +Last-Modified: `` +Server: `` +Via: `` diff --git a/tests/gold_tests/pluginTest/slice/slice.test.py b/tests/gold_tests/pluginTest/slice/slice.test.py new file mode 100644 index 00000000000..8b57d48f526 --- /dev/null +++ b/tests/gold_tests/pluginTest/slice/slice.test.py @@ -0,0 +1,188 @@ +''' +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import time +Test.Summary = ''' +Basic slice plugin test +''' + +## Test description: +# Preload the cache with the entire asset to be range requested. +# Reload remap rule with slice plugin +# Request content through the slice plugin + +Test.SkipUnless( + Condition.HasProgram("curl", "Curl needs to be installed on system for this test to work"), + Condition.PluginExists('slice.so'), +) +Test.ContinueOnFail = False + +# configure origin server +server = Test.MakeOriginServer("server") + +# Define ATS and configure +ts = Test.MakeATSProcess("ts", command="traffic_manager") + +# default root +request_header_chk = {"headers": + "GET / HTTP/1.1\r\n" + + "Host: www.example.com\r\n" + + "\r\n", + "timestamp": "1469733493.993", + "body": "", +} + +response_header_chk = {"headers": + "HTTP/1.1 200 OK\r\n" + + "Connection: close\r\n" + + "\r\n", + "timestamp": "1469733493.993", + "body": "", +} + +server.addResponse("sessionlog.json", request_header_chk, response_header_chk) + +#block_bytes = 7 +body = "lets go surfin now" + +request_header = {"headers": + "GET /path HTTP/1.1\r\n" + + "Host: www.example.com\r\n" + + "\r\n", + "timestamp": "1469733493.993", + "body": "", +} + +response_header = {"headers": + "HTTP/1.1 200 OK\r\n" + + "Connection: close\r\n" + + 'Etag: "path"\r\n' + + "Cache-Control: max-age=500\r\n" + + "\r\n", + "timestamp": "1469733493.993", + "body": body, +} + +server.addResponse("sessionlog.json", request_header, response_header) + +ts.Setup.CopyAs('curlsort.sh', Test.RunDirectory) +curl_and_args = 'sh curlsort.sh -H "Host: www.example.com"' + +# set up whole asset fetch into cache +ts.Disk.remap_config.AddLine( + 'map / http://127.0.0.1:{}'.format(server.Variables.Port) +) + +# minimal configuration +ts.Disk.records_config.update({ + 'proxy.config.diags.debug.enabled': 1, + 'proxy.config.diags.debug.tags': 'slice', + 'proxy.config.http.cache.http': 1, + 'proxy.config.http.wait_for_cache': 1, + 'proxy.config.http.insert_age_in_response': 0, + 'proxy.config.http.response_via_str': 3, + 'proxy.config.http.server_ports': '{}'.format(ts.Variables.port), +}) + +# 0 Test - Prefetch entire asset into cache +tr = Test.AddTestRun("Fetch first slice range") +tr.Processes.Default.StartBefore(server) +tr.Processes.Default.StartBefore(Test.Processes.ts, ready=1) +tr.Processes.Default.Command = curl_and_args + ' http://127.0.0.1:{}/path'.format(ts.Variables.port) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Streams.stdout = "gold/slice_200.stdout.gold" +tr.Processes.Default.Streams.stderr = "gold/slice_200.stderr.gold" +tr.StillRunningAfter = ts + +block_bytes = 7 + +# 1 - Reconfigure remap.config with slice plugin +tr = Test.AddTestRun("Load Slice plugin") +remap_config_path = ts.Disk.remap_config.Name +tr.Disk.File(remap_config_path, typename="ats:config").AddLines([ + 'map / http://127.0.0.1:{}'.format(server.Variables.Port) + + ' @plugin=slice.so @pparam=--test-blockbytes={}'.format(block_bytes) +]) + +tr.StillRunningAfter = ts +tr.StillRunningAfter = server +tr.Processes.Default.Command = 'traffic_ctl config reload' +# Need to copy over the environment so traffic_ctl knows where to find the unix domain socket +tr.Processes.Default.Env = ts.Env +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.TimeOut = 5 +tr.TimeOut = 5 + +# 2 Test - First complete slice +tr = Test.AddTestRun("Fetch first slice range") +tr.DelayStart = 5 +tr.Processes.Default.Command = curl_and_args + ' http://127.0.0.1:{}/path'.format(ts.Variables.port) + ' -r 0-6' +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Streams.stdout = "gold/slice_first.stdout.gold" +tr.Processes.Default.Streams.stderr = "gold/slice_first.stderr.gold" +tr.StillRunningAfter = ts + +# 3 Test - Last slice auto +tr = Test.AddTestRun("Last slice -- 14-") +tr.Processes.Default.Command = curl_and_args + ' http://127.0.0.1:{}/path'.format(ts.Variables.port) + ' -r 14-' +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Streams.stdout = "gold/slice_last.stdout.gold" +tr.Processes.Default.Streams.stderr = "gold/slice_last.stderr.gold" +tr.StillRunningAfter = ts + +# 4 Test - Last slice exact +tr = Test.AddTestRun("Last slice 14-17") +tr.Processes.Default.Command = curl_and_args + ' http://127.0.0.1:{}/path'.format(ts.Variables.port) + ' -r 14-17' +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Streams.stdout = "gold/slice_last.stdout.gold" +tr.Processes.Default.Streams.stderr = "gold/slice_last.stderr.gold" +tr.StillRunningAfter = ts + +# 5 Test - Last slice truncated +tr = Test.AddTestRun("Last truncated slice 14-20") +tr.Processes.Default.Command = curl_and_args + ' http://127.0.0.1:{}/path'.format(ts.Variables.port) + ' -r 14-20' +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Streams.stdout = "gold/slice_last.stdout.gold" +tr.Processes.Default.Streams.stderr = "gold/slice_last.stderr.gold" +tr.StillRunningAfter = ts + +# 6 Test - Whole asset via slices +tr = Test.AddTestRun("Whole asset via slices") +tr.Processes.Default.Command = curl_and_args + ' http://127.0.0.1:{}/path'.format(ts.Variables.port) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Streams.stdout = "gold/slice_200.stdout.gold" +tr.Processes.Default.Streams.stderr = "gold/slice_200.stderr.gold" +tr.StillRunningAfter = ts + +# 7 Test - Whole asset via range +tr = Test.AddTestRun("Whole asset via range") +tr.Processes.Default.Command = curl_and_args + ' http://127.0.0.1:{}/path'.format(ts.Variables.port) + ' -r 0-' +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Streams.stdout = "gold/slice_206.stdout.gold" +tr.Processes.Default.Streams.stderr = "gold/slice_206.stderr.gold" +tr.StillRunningAfter = ts + +# 8 Test - Non aligned slice request +tr = Test.AddTestRun("Non aligned slice request") +tr.Processes.Default.Command = curl_and_args + ' http://127.0.0.1:{}/path'.format(ts.Variables.port) + ' -r 5-16' +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Streams.stdout = "gold/slice_mid.stdout.gold" +tr.Processes.Default.Streams.stderr = "gold/slice_mid.stderr.gold" +tr.StillRunningAfter = ts + diff --git a/tests/gold_tests/pluginTest/slice/slice_error.test.py b/tests/gold_tests/pluginTest/slice/slice_error.test.py new file mode 100644 index 00000000000..53cf194f26d --- /dev/null +++ b/tests/gold_tests/pluginTest/slice/slice_error.test.py @@ -0,0 +1,326 @@ +''' +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import time +Test.Summary = ''' +Slice plugin error.log test +''' + +## Test description: +# Preload the cache with the entire asset to be range requested. +# Reload remap rule with slice plugin +# Request content through the slice plugin + +Test.SkipUnless( + Condition.HasProgram("curl", "Curl needs to be installed on system for this test to work"), + Condition.PluginExists('slice.so'), +) +Test.ContinueOnFail = False + +# configure origin server +server = Test.MakeOriginServer("server", lookup_key="{%Range}{PATH}") + +# Define ATS and configure +ts = Test.MakeATSProcess("ts", command="traffic_manager") + +body = "the quick brown fox" # len 19 + +# default root +request_header_chk = {"headers": + "GET / HTTP/1.1\r\n" + + "Host: www.example.com\r\n" + + "Range: bytes=0-\r\n" + + "\r\n", + "timestamp": "1469733493.993", + "body": "", +} + +response_header_chk = {"headers": + "HTTP/1.1 206 Partial Content\r\n" + + "Connection: close\r\n" + + "\r\n", + "timestamp": "1469733493.993", + "body": body, +} + +server.addResponse("sessionlog.json", request_header_chk, response_header_chk) + +blockbytes = 9 + +range0 = "{}-{}".format(0, blockbytes - 1) +range1 = "{}-{}".format(blockbytes, (2 * blockbytes) - 1) + +body0 = body[0:blockbytes] +body1 = body[blockbytes:2 * blockbytes] + +# Mismatch etag + +request_header_etag0 = {"headers": + "GET /etag HTTP/1.1\r\n" + + "Host: www.example.com\r\n" + + "Range: bytes={}\r\n".format(range0) + + "X-Slicer-Info: full content request\r\n" + + "\r\n", + "timestamp": "1469733493.993", + "body": "", +} + +response_header_etag0 = {"headers": + "HTTP/1.1 206 Partial Content\r\n" + + "Connection: close\r\n" + + 'Etag: "etag0"\r\n' + + "Content-Range: bytes {}/{}\r\n".format(range0, len(body)) + + "Cache-Control: max-age=500\r\n" + + "\r\n", + "timestamp": "1469733493.993", + "body": body0, +} + +server.addResponse("sessionlog.json", request_header_etag0, response_header_etag0) + +request_header_etag1 = {"headers": + "GET /etag HTTP/1.1\r\n" + + "Host: www.example.com\r\n" + + "Range: bytes={}\r\n".format(range1) + + "X-Slicer-Info: full content request\r\n" + + "\r\n", + "timestamp": "1469733493.993", + "body": "", +} + +response_header_etag1 = {"headers": + "HTTP/1.1 206 Partial Content\r\n" + + "Connection: close\r\n" + + 'Etag: "etag1"\r\n' + + "Content-Range: bytes {}/{}\r\n".format(range1, len(body)) + + "Cache-Control: max-age=500\r\n" + + "\r\n", + "timestamp": "1469733493.993", + "body": body1, +} + +server.addResponse("sessionlog.json", request_header_etag1, response_header_etag1) + +# mismatch Last-Modified + +request_header_lm0 = {"headers": + "GET /lastmodified HTTP/1.1\r\n" + + "Host: www.example.com\r\n" + + "Range: bytes={}\r\n".format(range0) + + "X-Slicer-Info: full content request\r\n" + + "\r\n", + "timestamp": "1469733493.993", + "body": "", +} + +response_header_lm0 = {"headers": + "HTTP/1.1 206 Partial Content\r\n" + + "Connection: close\r\n" + + "Last-Modified: Tue, 08 May 2018 15:49:41 GMT\r\n" + + "Content-Range: bytes {}/{}\r\n".format(range0, len(body)) + + "Cache-Control: max-age=500\r\n" + + "\r\n", + "timestamp": "1469733493.993", + "body": body0, +} + +server.addResponse("sessionlog.json", request_header_lm0, response_header_lm0) + +request_header_lm1 = {"headers": + "GET /lastmodified HTTP/1.1\r\n" + + "Host: www.example.com\r\n" + + "Range: bytes={}\r\n".format(range1) + + "X-Slicer-Info: full content request\r\n" + + "\r\n", + "timestamp": "1469733493.993", + "body": "", +} + +response_header_lm1 = {"headers": + "HTTP/1.1 206 Partial Content\r\n" + + "Connection: close\r\n" + + "Last-Modified: Tue, 08 Apr 2019 18:00:00 GMT\r\n" + + "Content-Range: bytes {}/{}\r\n".format(range1, len(body)) + + "Cache-Control: max-age=500\r\n" + + "\r\n", + "timestamp": "1469733493.993", + "body": body1, +} + +server.addResponse("sessionlog.json", request_header_lm1, response_header_lm1) + +# non 206 slice block + +request_header_n206_0 = {"headers": + "GET /non206 HTTP/1.1\r\n" + + "Host: www.example.com\r\n" + + "Range: bytes={}\r\n".format(range0) + + "X-Slicer-Info: full content request\r\n" + + "\r\n", + "timestamp": "1469733493.993", + "body": "", +} + +response_header_n206_0 = {"headers": + "HTTP/1.1 206 Partial Content\r\n" + + "Connection: close\r\n" + + 'Etag: "etag"\r\n' + + "Last-Modified: Tue, 08 May 2018 15:49:41 GMT\r\n" + + "Content-Range: bytes {}/{}\r\n".format(range0, len(body)) + + "Cache-Control: max-age=500\r\n" + + "\r\n", + "timestamp": "1469733493.993", + "body": body0, +} + +server.addResponse("sessionlog.json", request_header_n206_0, response_header_n206_0) + +# mismatch content-range + +request_header_crr0 = {"headers": + "GET /crr HTTP/1.1\r\n" + + "Host: www.example.com\r\n" + + "Range: bytes={}\r\n".format(range0) + + "X-Slicer-Info: full content request\r\n" + + "\r\n", + "timestamp": "1469733493.993", + "body": "", +} + +response_header_crr0 = {"headers": + "HTTP/1.1 206 Partial Content\r\n" + + "Connection: close\r\n" + + "Etag: crr\r\n" + + "Content-Range: bytes {}/{}\r\n".format(range0, len(body)) + + "Cache-Control: max-age=500\r\n" + + "\r\n", + "timestamp": "1469733493.993", + "body": body0, +} + +server.addResponse("sessionlog.json", request_header_crr0, response_header_crr0) + +request_header_crr1 = {"headers": + "GET /crr HTTP/1.1\r\n" + + "Host: www.example.com\r\n" + + "Range: bytes={}\r\n".format(range1) + + "X-Slicer-Info: full content request\r\n" + + "\r\n", + "timestamp": "1469733493.993", + "body": "", +} + +response_header_crr1 = {"headers": + "HTTP/1.1 206 Partial Content\r\n" + + "Connection: close\r\n" + + "Etag: crr\r\n" + + "Content-Range: bytes {}/{}\r\n".format(range1, len(body) - 1) + + "Cache-Control: max-age=500\r\n" + + "\r\n", + "timestamp": "1469733493.993", + "body": body1, +} + +server.addResponse("sessionlog.json", request_header_crr1, response_header_crr1) + +ts.Setup.CopyAs('curlsort.sh', Test.RunDirectory) +curl_and_args = 'sh curlsort.sh -H "Host: www.example.com"' + +# set up whole asset fetch into cache +ts.Disk.remap_config.AddLine( + 'map / http://127.0.0.1:{}'.format(server.Variables.Port) + + ' @plugin=slice.so @pparam=--test-blockbytes={}'.format(blockbytes) +) + +# minimal configuration +ts.Disk.records_config.update({ + 'proxy.config.diags.debug.enabled': 0, + 'proxy.config.diags.debug.tags': 'slice', + 'proxy.config.http.cache.http': 0, + 'proxy.config.http.wait_for_cache': 0, + 'proxy.config.http.insert_age_in_response': 0, + 'proxy.config.http.insert_request_via_str': 0, + 'proxy.config.http.insert_response_via_str': 3, + 'proxy.config.http.server_ports': '{}'.format(ts.Variables.port), +}) + +# Override builtin error check as these cases will fail +# taken from the slice plug code +ts.Disk.diags_log.Content = Testers.ContainsExpression('reason="Mismatch block Etag"', "Mismatch block etag") +ts.Disk.diags_log.Content += Testers.ContainsExpression('reason="Mismatch block Last-Modified"', "Mismatch block Last-Modified") +ts.Disk.diags_log.Content += Testers.ContainsExpression('reason="Non 206 internal block response"', "Non 206 internal block response") +ts.Disk.diags_log.Content += Testers.ContainsExpression('reason="Mismatch/Bad block Content-Range"', "Mismatch/Bad block Content-Range") + +# 0 Test - Etag mismatch test +tr = Test.AddTestRun("Etag test") +tr.Processes.Default.StartBefore(server) +tr.Processes.Default.StartBefore(Test.Processes.ts, ready=1) +tr.Processes.Default.Command = curl_and_args + ' http://127.0.0.1:{}/etag'.format(ts.Variables.port) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Streams.stdout = "gold_error/etag.stdout.gold" +tr.Processes.Default.Streams.stderr = "gold_error/etag.stderr.gold" +tr.StillRunningAfter = ts + +# 1 Check - diags.log message +tr = Test.AddTestRun("Etag error check") +tr.Processes.Default.Command = "grep 'Mismatch block Etag' {}".format(ts.Disk.diags_log.Name) +tr.Processes.Default.ReturnCode = 0 +tr.StillRunningAfter = ts + +# 2 Test - Last Modified mismatch test +tr = Test.AddTestRun("Last-Modified test") +tr.Processes.Default.Command = curl_and_args + ' http://127.0.0.1:{}/lastmodified'.format(ts.Variables.port) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Streams.stdout = "gold_error/lm.stdout.gold" +tr.Processes.Default.Streams.stderr = "gold_error/lm.stderr.gold" +tr.StillRunningAfter = ts + +# 3 Check - diags.log message +tr = Test.AddTestRun("Last-Modified error check") +tr.Processes.Default.Command = "grep 'Mismatch block Last-Modified' {}".format(ts.Disk.diags_log.Name) +tr.Processes.Default.ReturnCode = 0 +tr.StillRunningAfter = ts + +# 4 Test - Non 206 mismatch test +tr = Test.AddTestRun("Non 206 test") +tr.Processes.Default.Command = curl_and_args + ' http://127.0.0.1:{}/non206'.format(ts.Variables.port) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Streams.stdout = "gold_error/non206.stdout.gold" +tr.Processes.Default.Streams.stderr = "gold_error/non206.stderr.gold" +tr.StillRunningAfter = ts + +# 3 Check - diags.log message +tr = Test.AddTestRun("Non 206 error check") +tr.Processes.Default.Command = "grep 'Non 206 internal block response' {}".format(ts.Disk.diags_log.Name) +tr.Processes.Default.ReturnCode = 0 +tr.StillRunningAfter = ts + +# 4 Test - Block content-range +tr = Test.AddTestRun("Content-Range test") +tr.Processes.Default.Command = curl_and_args + ' http://127.0.0.1:{}/crr'.format(ts.Variables.port) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Streams.stdout = "gold_error/crr.stdout.gold" +tr.Processes.Default.Streams.stderr = "gold_error/crr.stderr.gold" +tr.StillRunningAfter = ts + +# 3 Check - diags.log message +tr = Test.AddTestRun("Content-Range error check") +tr.Processes.Default.Command = "grep 'Mismatch/Bad block Content-Range' {}".format(ts.Disk.diags_log.Name) +tr.Processes.Default.ReturnCode = 0 +tr.StillRunningAfter = ts diff --git a/tests/tools/lib/IPConstants.py b/tests/gold_tests/pluginTest/sslheaders/observer.py similarity index 63% rename from tests/tools/lib/IPConstants.py rename to tests/gold_tests/pluginTest/sslheaders/observer.py index 0531137162b..673a56eb1a2 100644 --- a/tests/tools/lib/IPConstants.py +++ b/tests/gold_tests/pluginTest/sslheaders/observer.py @@ -1,4 +1,5 @@ ''' +Extract ssl-* headers and store in a log file for later verification. ''' # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file @@ -16,33 +17,15 @@ # See the License for the specific language governing permissions and # limitations under the License. -import ipaddress +log = open('sslheaders.log', 'w') -# convenience functions +def observe(headers): -IPkw = {'INADDR_LOOPBACK':'127.0.0.1', - 'IN6ADDR_LOOPBACK':'::1', - 'INADDR_ANY':'0.0.0.0', - 'IN6ADDR_ANY':'::'} + for h in headers.items(): + if h[0].lower().startswith('ssl-'): -def isIPv6(addr): - if addr in IPkw: - addr = IPkw[addr] - - - return ipaddress.ip_address(addr).version == 6 - - -def isIPv4(addr): - if addr in IPkw: - addr = IPkw[addr] - - return ipaddress.ip_address(addr).version == 4 - - -def getIP(addr): - if addr in IPkw: - addr = IPkw[addr] - - return str(ipaddress.ip_address(addr)) + log.write(h[0] + ": " + h[1] + "\n"); + log.write("-\n") + log.flush() +Hooks.register(Hooks.ReadRequestHook, observe) diff --git a/tests/gold_tests/pluginTest/sslheaders/ssl/server.key b/tests/gold_tests/pluginTest/sslheaders/ssl/server.key new file mode 100644 index 00000000000..9cdfc36aa5f --- /dev/null +++ b/tests/gold_tests/pluginTest/sslheaders/ssl/server.key @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQCZkEXSlZ+ZFKFg +CPpcDG39e73BuK6E5uE38q2PHh4DV0xcsJnIUx51viqLPwYughxfP0crHyBdXoHV +dW/3WX4gpiGrdiM/dvCouheo0DPaqUUJ2nZKVYh2M57oyeiuJidlKb7BGkfw3HWP +9TV7dVyGWok/cowjopqaLHJWxg/kh2KqvUBD0CHt9Kd1XvgXVmHwE7vCv0j5owv2 +MaExTsFb16uWmVLhl1gNHI2RqCX2yLaebH1DvtbLrit1XErjtaSYeJE9clVRaqT6 +vsvLOhyB5tA9WfZqfBYr/MHDeXQfrbIf+4Cp3aTpq5grc5InIMMH0eOk6/f/4tW+ +nq1lfszZAgMBAAECggEAYvYAqRbXRRVwca0Xel5gO2yU+tSDUw5esWlow8RK3yhR +A6KjV9+Iz6P/UsEIwMwEcLUcrgNfHgybau5Fe4dmqq+lHxQA3xNNP869FIMoB4/x +98mbVYgNau8VRztnAWOBG8ZtMZA4MFZCRMVm8+rL96E8tXCiMwzEyPo/rP/ymfhN +3GRunX+GhfIA79AYNbd7HMVL+cvWWUGUF5Bc5i1wXcLy4I7b9NYtv920BeCLzSFK +BypFB7ku/vKgTcBxe4yxThxPeXPwm4WFzGYKk/Afl1j8tVXCE2U4Y3yykfC0Qk6S +ECZbCKLO2Rxi9fclIDZBHWuKejZhdjHfjeNvZ2vLoQKBgQDJzLmkVLvWAxgl1yvF +U7gwqj/TzYqtVowbjEvTNEnPU1j/hIVI343SVV/EvJmif/iRUop6sRYfLsUjpMsH +CmPysNKL3UtgSYOxLs+0xLhG4OOQRpPSf/uvl9YyWY9G3AqiC7ScthkQjEhZa4c1 +eycYy0jr42kX0OL9MuIH9q0ENQKBgQDCzvGKMs8r5E/Qd3YB9VYB60dx+6G83AHZ +YqIelykObhCdxL9n4K+p4VKKLvgTcCOLYYIkBSWRJWR+ue3s3ey9+XWd2/q4Xvfh +TCjAuO2ibMV+y5ClNlW0fQ/doIVWSDbjO2tZW1jh7YWZ4CtuVrsEisv1sk3KltMB +MguhpTUylQKBgG6TfrncMFzxrx61C+gBmvEXqQffHfkjbnx94OKnSTaQ3jiNHhez +X9v8KhD8o1bWtpay2uyl8pA9qYqBdzqxZ9kJKSW4qd/mCIJjOy87iBpWint5IPD8 +biZmldlbF9ZlJnJq5ZnlclCN/er5r8oPZHoCkj+nieOh8294nUBt25ptAoGAMnPA +EIeaKgbmONpHgLhWPwb9KOL/f1cHT5KA5CVH58nPmdyTqcaCGCAX7Vu+ueIIApgN +SWDf2thxT3S9zuOm5YiO0oRfSZKm5f2AbHE4ciFzgKQd4PvSdH0TN9XT0oW/WVhR +NAI5YcHPIQvyk4/4vXNo4Uf9Z6NqIFwisQmFXoUCgYBK/ZI/HsFsvnR5MV0tFdGM +AdNe6bsQRSZkowoaPxuWbklE4Hn6QvwEmQg3ST2O+vCQV1f1yI6YiWYoptOYscJc +MSs/HxhhaaO5ZsiuPUO6WEPzpNb2CxuIGDDtl83VtUQyjxCmOb6pqqjwzFmZ2bsw +JNMaBCzokrJTxknvauCuSQ== +-----END PRIVATE KEY----- diff --git a/tests/gold_tests/pluginTest/sslheaders/ssl/server.pem b/tests/gold_tests/pluginTest/sslheaders/ssl/server.pem new file mode 100644 index 00000000000..2b56cc83ea2 --- /dev/null +++ b/tests/gold_tests/pluginTest/sslheaders/ssl/server.pem @@ -0,0 +1,21 @@ +-----BEGIN CERTIFICATE----- +MIIDZDCCAkygAwIBAgIJANod1+h9CtCaMA0GCSqGSIb3DQEBCwUAMEcxCzAJBgNV +BAYTAlVTMQswCQYDVQQIDAJJTDEPMA0GA1UECgwGQXBhY2hlMRowGAYDVQQDDBFy +YW5kb20uc2VydmVyLmNvbTAeFw0xODExMTkxNzEwMTlaFw0yODExMTYxNzEwMTla +MEcxCzAJBgNVBAYTAlVTMQswCQYDVQQIDAJJTDEPMA0GA1UECgwGQXBhY2hlMRow +GAYDVQQDDBFyYW5kb20uc2VydmVyLmNvbTCCASIwDQYJKoZIhvcNAQEBBQADggEP +ADCCAQoCggEBAJmQRdKVn5kUoWAI+lwMbf17vcG4roTm4TfyrY8eHgNXTFywmchT +HnW+Kos/Bi6CHF8/RysfIF1egdV1b/dZfiCmIat2Iz928Ki6F6jQM9qpRQnadkpV +iHYznujJ6K4mJ2UpvsEaR/DcdY/1NXt1XIZaiT9yjCOimposclbGD+SHYqq9QEPQ +Ie30p3Ve+BdWYfATu8K/SPmjC/YxoTFOwVvXq5aZUuGXWA0cjZGoJfbItp5sfUO+ +1suuK3VcSuO1pJh4kT1yVVFqpPq+y8s6HIHm0D1Z9mp8Fiv8wcN5dB+tsh/7gKnd +pOmrmCtzkicgwwfR46Tr9//i1b6erWV+zNkCAwEAAaNTMFEwHQYDVR0OBBYEFI2y +qm0+UAChDAnLrAINeFOuyUlhMB8GA1UdIwQYMBaAFI2yqm0+UAChDAnLrAINeFOu +yUlhMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZIhvcNAQELBQADggEBAA3ZNFbqxcOX +szS5A4EXCepyBJBFejEYy0CsvwQX/ai/pMrw5jqVeF0GAOTpBCVLddyY+ZV1arD2 +Pqi7Qwot9OxEZOzbCBiuMJGotruKgnWFQDHzJ9HA7KDQs270uNESAOG/xW9os9zN +MXApzqfBSR5EIQU5L3RtaiPzoKdQenGQUOj86s0Kon7snDSUzaA2VcfstMWgGvXP +JHtaVusULm0gry32cEap5G5UK+gII6DfLWgFwFGhHHmTz3mKjyGiJQ+09XBtu4lb +ENE+HGRBBA49dUKSr3kwErO4HyHnS0YrsTDnbYURCsGUDma12oijX2sCos6Q4zn8 +3svaouRrucw= +-----END CERTIFICATE----- diff --git a/tests/gold_tests/pluginTest/sslheaders/sslheaders.gold b/tests/gold_tests/pluginTest/sslheaders/sslheaders.gold new file mode 100644 index 00000000000..39cdd0ded6d --- /dev/null +++ b/tests/gold_tests/pluginTest/sslheaders/sslheaders.gold @@ -0,0 +1 @@ +- diff --git a/tests/gold_tests/pluginTest/sslheaders/sslheaders.test.py b/tests/gold_tests/pluginTest/sslheaders/sslheaders.test.py new file mode 100644 index 00000000000..c13d0da6117 --- /dev/null +++ b/tests/gold_tests/pluginTest/sslheaders/sslheaders.test.py @@ -0,0 +1,90 @@ +''' +Test the sslheaders plugin. +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +Test.Summary = ''' +Test sslheaders plugin. +''' + +Test.SkipUnless( + Condition.HasCurlFeature('http2'), +) + +Test.Disk.File('sslheaders.log').Content = 'sslheaders.gold' + +server = Test.MakeOriginServer("server", options={'--load': Test.TestDirectory + '/observer.py'}) + +request_header = { + "headers": "GET / HTTP/1.1\r\nHost: doesnotmatter\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +response_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +server.addResponse("sessionlog.json", request_header, response_header) + +ts = Test.MakeATSProcess("ts", select_ports=False) + +ts.addSSLfile("ssl/server.pem") +ts.addSSLfile("ssl/server.key") +# ts.addSSLfile("ssl/signer.pem") + +ts.Variables.ssl_port = 4443 + +ts.Disk.records_config.update({ + 'proxy.config.diags.debug.enabled': 0, + 'proxy.config.diags.debug.tags': 'http', + 'proxy.config.http.cache.http': 0, # Make sure each request is forwarded to the origin server. + 'proxy.config.proxy_name': 'Poxy_Proxy', # This will be the server name. + 'proxy.config.ssl.server.cert.path': '{0}'.format(ts.Variables.SSLDir), + 'proxy.config.ssl.server.private_key.path': '{0}'.format(ts.Variables.SSLDir), + 'proxy.config.http.server_ports': ( + 'ipv4:{0} ipv4:{1}:proto=http2;http:ssl ipv6:{0} ipv6:{1}:proto=http2;http:ssl' + .format(ts.Variables.port, ts.Variables.ssl_port)), +# 'proxy.config.ssl.client.verify.server': 0, +# 'proxy.config.ssl.server.cipher_suite': 'ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA384:AES128-GCM-SHA256:AES256-GCM-SHA384:ECDHE-RSA-RC4-SHA:ECDHE-RSA-AES128-SHA:ECDHE-RSA-AES256-SHA:RC4-SHA:RC4-MD5:AES128-SHA:AES256-SHA:DES-CBC3-SHA!SRP:!DSS:!PSK:!aNULL:!eNULL:!SSLv2', +# 'proxy.config.url_remap.pristine_host_hdr' : 1, +# 'proxy.config.ssl.client.certification_level': 2, +# 'proxy.config.ssl.CA.cert.filename': '{0}/signer.pem'.format(ts.Variables.SSLDir), +# 'proxy.config.ssl.TLSv1_3': 0 +}) + +ts.Disk.ssl_multicert_config.AddLine( + 'dest_ip=* ssl_cert_name=server.pem ssl_key_name=server.key' +) + +ts.Disk.remap_config.AddLine( + 'map http://bar.com http://127.0.0.1:{0}'.format(server.Variables.Port) +) +ts.Disk.remap_config.AddLine( + 'map https://bar.com http://127.0.0.1:{0}'.format(server.Variables.Port) +) + +ts.Disk.ssl_server_name_yaml.AddLines([ + '- fqdn: "*bar.com"', + ' verify_client: STRICT', +]) + +ts.Disk.plugin_config.AddLine( + 'sslheaders.so SSL-Client-ID=client.subject' +) + +tr = Test.AddTestRun() +tr.Processes.Default.StartBefore(server) +tr.Processes.Default.StartBefore(Test.Processes.ts, ready=When.PortOpen(ts.Variables.ssl_port)) +tr.Processes.Default.Command = ( + 'curl -H "SSL-Client-ID: My Fake Client ID" --verbose --ipv4 --insecure --header "Host: bar.com"' + + ' https://localhost:{}'.format(ts.Variables.ssl_port) +) +tr.Processes.Default.ReturnCode = 0 diff --git a/tests/gold_tests/pluginTest/test_hooks/log.gold b/tests/gold_tests/pluginTest/test_hooks/log.gold index 0c6258d6c44..b798539374a 100644 --- a/tests/gold_tests/pluginTest/test_hooks/log.gold +++ b/tests/gold_tests/pluginTest/test_hooks/log.gold @@ -1,11 +1,30 @@ -Global: event=unknown event -Global: event=unknown event -Session: event=unknown event -Global: event=unknown event -Session: event=unknown event -Transaction: event=unknown event -Global: event=unknown event -Session: event=unknown event -Transaction: event=unknown event -Global: event=unknown event -Session: event=unknown event +Global: event=TS_EVENT_HTTP_SSN_START +Global: event=TS_EVENT_HTTP_TXN_START +Session: event=TS_EVENT_HTTP_TXN_START +Global: event=TS_EVENT_HTTP_READ_REQUEST_HDR +Session: event=TS_EVENT_HTTP_READ_REQUEST_HDR +Transaction: event=TS_EVENT_HTTP_READ_REQUEST_HDR +Global: event=TS_EVENT_HTTP_TXN_CLOSE +Session: event=TS_EVENT_HTTP_TXN_CLOSE +Transaction: event=TS_EVENT_HTTP_TXN_CLOSE +Global: event=TS_EVENT_HTTP_SSN_CLOSE +Session: event=TS_EVENT_HTTP_SSN_CLOSE +Global: event=TS_EVENT_VCONN_START +Global: ssl flag=1 +Global: event=TS_EVENT_SSL_SERVERNAME +Global: ssl flag=1 +Global: event=TS_EVENT_SSL_CERT +Global: ssl flag=1 +Global: event=TS_EVENT_HTTP_SSN_START +Global: event=TS_EVENT_HTTP_TXN_START +Session: event=TS_EVENT_HTTP_TXN_START +Global: event=TS_EVENT_HTTP_READ_REQUEST_HDR +Session: event=TS_EVENT_HTTP_READ_REQUEST_HDR +Transaction: event=TS_EVENT_HTTP_READ_REQUEST_HDR +Global: event=TS_EVENT_HTTP_TXN_CLOSE +Session: event=TS_EVENT_HTTP_TXN_CLOSE +Transaction: event=TS_EVENT_HTTP_TXN_CLOSE +Global: event=TS_EVENT_HTTP_SSN_CLOSE +Session: event=TS_EVENT_HTTP_SSN_CLOSE +Global: event=TS_EVENT_VCONN_CLOSE +Global: ssl flag=1 diff --git a/tests/gold_tests/pluginTest/test_hooks/one.in b/tests/gold_tests/pluginTest/test_hooks/one.in deleted file mode 100644 index 48306bed26c..00000000000 --- a/tests/gold_tests/pluginTest/test_hooks/one.in +++ /dev/null @@ -1,4 +0,0 @@ -GET /argh HTTP/1.1 -Host: one -X-Debug: X-Remap - diff --git a/tests/gold_tests/pluginTest/test_hooks/ssl/server.key b/tests/gold_tests/pluginTest/test_hooks/ssl/server.key new file mode 100644 index 00000000000..4c7a661a6bd --- /dev/null +++ b/tests/gold_tests/pluginTest/test_hooks/ssl/server.key @@ -0,0 +1,15 @@ +-----BEGIN RSA PRIVATE KEY----- +MIICXQIBAAKBgQDWMHOiUF+ORmZjAxI8MWE9dblb7gQSJ36WCXlPFiFx6ynF+S1E +kXAYpIip5X0pzDUaIbLukxJUAAnOtMEO0PCgxJQUrEtRWh8wiJdbdQJF0Zs/9R+u +SUgb61f+mdTQvhqefBGx+xrpfAcgtcWiZuSA9Q3fvpDj5WOWSPWXBUuxywIDAQAB +AoGBAJPxRX2gjFAGWmQbU/YVmXfNH6navh8X/nx9sLeqrpE0AFeJI/ZPiqDKzMal +B43eSfNxwVi+ZxN0L1ICUbL9KKZvHs/QBxWLA1fGVAXrz7sRplEVvakPpTfHoEnv +sKaMWVKaK/S5WGbDhElb6zb/Lwo19DsIAPjGYqFvzFJBmobJAkEA9iSeTGkR9X26 +GywZoYrIMlRh34htOIRx1UUq88rFzdrCF21kQ4lhBIkX5OZMMy652i2gyak4OZTe +YewIv8jw9QJBAN7EQNHG8jPwXfVp91/fqxVQEfumuP2i6uiWWYQgZCmla2+0xcLZ +pMQ6sQEe10hhTrVnzHgAUVp50Ntn2jwBX78CQF09veGAI9d1Cxzj9cmmAvRd1r2Q +tp8kPOLnUsALXib+6WtqewLCdcf8DtsdClyRJMIraq85tRzK8fryKNZNzkkCQEgA +yS7FDj5JgCU15hZgFk1iPx3HCt44jZM2HaL+UUHAzRQjKxTLAl3G1rWVAWLMyQML +lORoveLvotl4HOruSsMCQQCAx9dV9JUSFoyc1CWILp/FgUH/se4cjQCThGO0DoQQ +vGTYmntY7j9WRJ9esQrjdD6Clw8zM/45GIBNwnXzqo7Z +-----END RSA PRIVATE KEY----- diff --git a/tests/tools/microServer/ssl/server.pem b/tests/gold_tests/pluginTest/test_hooks/ssl/server.pem similarity index 100% rename from tests/tools/microServer/ssl/server.pem rename to tests/gold_tests/pluginTest/test_hooks/ssl/server.pem diff --git a/tests/gold_tests/pluginTest/test_hooks/test_hooks.test.py b/tests/gold_tests/pluginTest/test_hooks/test_hooks.test.py index 6f79daef44f..82883036c51 100644 --- a/tests/gold_tests/pluginTest/test_hooks/test_hooks.test.py +++ b/tests/gold_tests/pluginTest/test_hooks/test_hooks.test.py @@ -18,6 +18,11 @@ Test TS API Hooks. ''' +Test.SkipUnless( + Condition.HasCurlFeature('http2'), +) +Test.ContinueOnFail = True + # test_hooks.so will output test logging to this file. Test.Env["OUTPUT_FILE"] = Test.RunDirectory + "/log.txt" @@ -28,41 +33,57 @@ response_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": "" } server.addResponse("sessionlog.json", request_header, response_header) -ts = Test.MakeATSProcess("ts") +ts = Test.MakeATSProcess("ts", select_ports=False) + +ts.addSSLfile("ssl/server.pem") +ts.addSSLfile("ssl/server.key") + +ts.Variables.ssl_port = 4443 ts.Disk.records_config.update({ + 'proxy.config.http.cache.http': 0, # Make sure each request is forwarded to the origin server. + 'proxy.config.proxy_name': 'Poxy_Proxy', # This will be the server name. + 'proxy.config.ssl.server.cert.path': '{0}'.format(ts.Variables.SSLDir), + 'proxy.config.ssl.server.private_key.path': '{0}'.format(ts.Variables.SSLDir), + 'proxy.config.http.server_ports': ( + 'ipv4:{0} ipv4:{1}:proto=http2;http:ssl'.format(ts.Variables.port, ts.Variables.ssl_port)), 'proxy.config.url_remap.remap_required': 0, - 'proxy.config.diags.debug.enabled': 1, - 'proxy.config.diags.debug.tags': 'test_hooks' - # 'proxy.config.diags.debug.tags': 'http|test_hooks' + 'proxy.config.diags.debug.enabled': 0, + 'proxy.config.diags.debug.tags': 'http|test_hooks', }) -ts.Disk.remap_config.AddLine( - "map http://one http://127.0.0.1:{0}".format(server.Variables.Port) +ts.Disk.ssl_multicert_config.AddLine( + 'dest_ip=* ssl_cert_name=server.pem ssl_key_name=server.key' ) Test.PreparePlugin(Test.Variables.AtsTestToolsDir + '/plugins/test_hooks.cc', ts) +ts.Disk.remap_config.AddLine( + "map http://one http://127.0.0.1:{0}".format(server.Variables.Port) +) +ts.Disk.remap_config.AddLine( + "map https://one http://127.0.0.1:{0}".format(server.Variables.Port) +) + tr = Test.AddTestRun() -tr.Processes.Default.StartBefore(Test.Processes.ts) -tr.Processes.Default.StartBefore(Test.Processes.server) -tr.Processes.Default.Command = "cp {}/tcp_client.py {}/tcp_client.py".format( - Test.Variables.AtsTestToolsDir, Test.RunDirectory) +# Probe server port to check if ready. +tr.Processes.Default.StartBefore(server, ready=When.PortOpen(server.Variables.Port)) +# Probe TS cleartext port to check if ready (probing TLS port causes spurious VCONN hook triggers). +tr.Processes.Default.StartBefore(Test.Processes.ts, ready=When.PortOpen(ts.Variables.port)) +# +tr.Processes.Default.Command = ( + 'curl --verbose --ipv4 --header "Host: one" http://localhost:{0}/argh'.format(ts.Variables.port) +) tr.Processes.Default.ReturnCode = 0 -def sendMsg(msgFile): - - tr = Test.AddTestRun() - tr.Processes.Default.Command = ( - "python {}/tcp_client.py 127.0.0.1 {} {}/{}.in".format( - Test.RunDirectory, ts.Variables.port, Test.TestDirectory, msgFile) - ) - tr.Processes.Default.ReturnCode = 0 - -sendMsg('one') +tr = Test.AddTestRun() +tr.Processes.Default.Command = ( + 'curl --verbose --ipv4 --http2 --insecure --header "Host: one" https://localhost:{0}/argh'.format(ts.Variables.ssl_port) +) +tr.Processes.Default.ReturnCode = 0 tr = Test.AddTestRun() -tr.Processes.Default.Command = "echo test log.gold" +tr.Processes.Default.Command = "echo check log" tr.Processes.Default.ReturnCode = 0 f = tr.Disk.File("log.txt") f.Content = "log.gold" diff --git a/tests/gold_tests/pluginTest/tsapi/log.gold b/tests/gold_tests/pluginTest/tsapi/log.gold new file mode 100644 index 00000000000..3960bfec884 --- /dev/null +++ b/tests/gold_tests/pluginTest/tsapi/log.gold @@ -0,0 +1,10 @@ +Global: event=TS_EVENT_HTTP_TXN_START +Global: event=TS_EVENT_HTTP_READ_REQUEST_HDR +TSHttpTxnEffectiveUrlStringGet(): http://myhost.test:SERVER_PORT/ +Transaction: event=TS_EVENT_HTTP_READ_REQUEST_HDR +TSHttpTxnEffectiveUrlStringGet(): http://myhost.test:SERVER_PORT/ +Global: event=TS_EVENT_HTTP_TXN_START +Global: event=TS_EVENT_HTTP_READ_REQUEST_HDR +TSHttpTxnEffectiveUrlStringGet(): https://myhost.test:SERVER_PORT/ +Transaction: event=TS_EVENT_HTTP_READ_REQUEST_HDR +TSHttpTxnEffectiveUrlStringGet(): https://myhost.test:SERVER_PORT/ diff --git a/tests/gold_tests/pluginTest/tsapi/ssl/server.key b/tests/gold_tests/pluginTest/tsapi/ssl/server.key new file mode 100644 index 00000000000..4c7a661a6bd --- /dev/null +++ b/tests/gold_tests/pluginTest/tsapi/ssl/server.key @@ -0,0 +1,15 @@ +-----BEGIN RSA PRIVATE KEY----- +MIICXQIBAAKBgQDWMHOiUF+ORmZjAxI8MWE9dblb7gQSJ36WCXlPFiFx6ynF+S1E +kXAYpIip5X0pzDUaIbLukxJUAAnOtMEO0PCgxJQUrEtRWh8wiJdbdQJF0Zs/9R+u +SUgb61f+mdTQvhqefBGx+xrpfAcgtcWiZuSA9Q3fvpDj5WOWSPWXBUuxywIDAQAB +AoGBAJPxRX2gjFAGWmQbU/YVmXfNH6navh8X/nx9sLeqrpE0AFeJI/ZPiqDKzMal +B43eSfNxwVi+ZxN0L1ICUbL9KKZvHs/QBxWLA1fGVAXrz7sRplEVvakPpTfHoEnv +sKaMWVKaK/S5WGbDhElb6zb/Lwo19DsIAPjGYqFvzFJBmobJAkEA9iSeTGkR9X26 +GywZoYrIMlRh34htOIRx1UUq88rFzdrCF21kQ4lhBIkX5OZMMy652i2gyak4OZTe +YewIv8jw9QJBAN7EQNHG8jPwXfVp91/fqxVQEfumuP2i6uiWWYQgZCmla2+0xcLZ +pMQ6sQEe10hhTrVnzHgAUVp50Ntn2jwBX78CQF09veGAI9d1Cxzj9cmmAvRd1r2Q +tp8kPOLnUsALXib+6WtqewLCdcf8DtsdClyRJMIraq85tRzK8fryKNZNzkkCQEgA +yS7FDj5JgCU15hZgFk1iPx3HCt44jZM2HaL+UUHAzRQjKxTLAl3G1rWVAWLMyQML +lORoveLvotl4HOruSsMCQQCAx9dV9JUSFoyc1CWILp/FgUH/se4cjQCThGO0DoQQ +vGTYmntY7j9WRJ9esQrjdD6Clw8zM/45GIBNwnXzqo7Z +-----END RSA PRIVATE KEY----- diff --git a/tests/gold_tests/pluginTest/tsapi/ssl/server.pem b/tests/gold_tests/pluginTest/tsapi/ssl/server.pem new file mode 100644 index 00000000000..58b9b9715b7 --- /dev/null +++ b/tests/gold_tests/pluginTest/tsapi/ssl/server.pem @@ -0,0 +1,32 @@ +-----BEGIN RSA PRIVATE KEY----- +MIICXQIBAAKBgQDWMHOiUF+ORmZjAxI8MWE9dblb7gQSJ36WCXlPFiFx6ynF+S1E +kXAYpIip5X0pzDUaIbLukxJUAAnOtMEO0PCgxJQUrEtRWh8wiJdbdQJF0Zs/9R+u +SUgb61f+mdTQvhqefBGx+xrpfAcgtcWiZuSA9Q3fvpDj5WOWSPWXBUuxywIDAQAB +AoGBAJPxRX2gjFAGWmQbU/YVmXfNH6navh8X/nx9sLeqrpE0AFeJI/ZPiqDKzMal +B43eSfNxwVi+ZxN0L1ICUbL9KKZvHs/QBxWLA1fGVAXrz7sRplEVvakPpTfHoEnv +sKaMWVKaK/S5WGbDhElb6zb/Lwo19DsIAPjGYqFvzFJBmobJAkEA9iSeTGkR9X26 +GywZoYrIMlRh34htOIRx1UUq88rFzdrCF21kQ4lhBIkX5OZMMy652i2gyak4OZTe +YewIv8jw9QJBAN7EQNHG8jPwXfVp91/fqxVQEfumuP2i6uiWWYQgZCmla2+0xcLZ +pMQ6sQEe10hhTrVnzHgAUVp50Ntn2jwBX78CQF09veGAI9d1Cxzj9cmmAvRd1r2Q +tp8kPOLnUsALXib+6WtqewLCdcf8DtsdClyRJMIraq85tRzK8fryKNZNzkkCQEgA +yS7FDj5JgCU15hZgFk1iPx3HCt44jZM2HaL+UUHAzRQjKxTLAl3G1rWVAWLMyQML +lORoveLvotl4HOruSsMCQQCAx9dV9JUSFoyc1CWILp/FgUH/se4cjQCThGO0DoQQ +vGTYmntY7j9WRJ9esQrjdD6Clw8zM/45GIBNwnXzqo7Z +-----END RSA PRIVATE KEY----- +-----BEGIN CERTIFICATE----- +MIICszCCAhwCCQD4jSkztmlO1TANBgkqhkiG9w0BAQsFADCBnTELMAkGA1UEBhMC +VVMxCzAJBgNVBAgTAklMMRIwEAYDVQQHEwlDaGFtcGFpZ24xDjAMBgNVBAoTBVlh +aG9vMQ0wCwYDVQQLEwRFZGdlMSgwJgYDVQQDEx9qdWljZXByb2R1Y2UuY29ycC5u +ZTEueWFob28uY29tMSQwIgYJKoZIhvcNAQkBFhVwZXJzaWEuYXppekB5YWhvby5j +b20wHhcNMTcwODI4MDM0NDQ1WhcNMjcwODI2MDM0NDQ1WjCBnTELMAkGA1UEBhMC +VVMxCzAJBgNVBAgTAklMMRIwEAYDVQQHEwlDaGFtcGFpZ24xDjAMBgNVBAoTBVlh +aG9vMQ0wCwYDVQQLEwRFZGdlMSgwJgYDVQQDEx9qdWljZXByb2R1Y2UuY29ycC5u +ZTEueWFob28uY29tMSQwIgYJKoZIhvcNAQkBFhVwZXJzaWEuYXppekB5YWhvby5j +b20wgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBANYwc6JQX45GZmMDEjwxYT11 +uVvuBBInfpYJeU8WIXHrKcX5LUSRcBikiKnlfSnMNRohsu6TElQACc60wQ7Q8KDE +lBSsS1FaHzCIl1t1AkXRmz/1H65JSBvrV/6Z1NC+Gp58EbH7Gul8ByC1xaJm5ID1 +Dd++kOPlY5ZI9ZcFS7HLAgMBAAEwDQYJKoZIhvcNAQELBQADgYEATX7975NdhIbJ +glda+sXI9a86GgOpiuKO+vKubRJQZA+UlPf2vHEONjC2+7Y1aZvZYaKYL74vxGky +zkgp6ANSPl45lqD632x0e1Z7vzW5TkqK1JB2/xH2WgDcQZmP0FuQHzVNs4GjghDr +HCp1+sQDhfPB4aLmLFeyN0TkhdH1N3M= +-----END CERTIFICATE----- diff --git a/tests/gold_tests/pluginTest/tsapi/tsapi.test.py b/tests/gold_tests/pluginTest/tsapi/tsapi.test.py new file mode 100644 index 00000000000..91574569a66 --- /dev/null +++ b/tests/gold_tests/pluginTest/tsapi/tsapi.test.py @@ -0,0 +1,92 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +Test.Summary = ''' +Test TS API. +''' + +Test.SkipUnless( + Condition.HasCurlFeature('http2'), +) +Test.ContinueOnFail = True + +# test_tsapi.so will output test logging to this file. +Test.Env["OUTPUT_FILE"] = Test.RunDirectory + "/log.txt" + +server = Test.MakeOriginServer("server") + +request_header = { + "headers": "GET / HTTP/1.1\r\nHost: doesnotmatter\r\n\r\n", "timestamp": "1469733493.993", "body": "" } +response_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": "112233" } +server.addResponse("sessionlog.json", request_header, response_header) + +ts = Test.MakeATSProcess("ts", select_ports=False) + +ts.addSSLfile("ssl/server.pem") +ts.addSSLfile("ssl/server.key") + +ts.Variables.ssl_port = 4443 + +ts.Disk.records_config.update({ + 'proxy.config.http.cache.http': 0, # Make sure each request is forwarded to the origin server. + 'proxy.config.proxy_name': 'Poxy_Proxy', # This will be the server name. + 'proxy.config.ssl.server.cert.path': '{0}'.format(ts.Variables.SSLDir), + 'proxy.config.ssl.server.private_key.path': '{0}'.format(ts.Variables.SSLDir), + 'proxy.config.http.server_ports': ( + 'ipv4:{0} ipv4:{1}:proto=http2;http:ssl'.format(ts.Variables.port, ts.Variables.ssl_port)), + 'proxy.config.url_remap.remap_required': 0, + 'proxy.config.diags.debug.enabled': 0, + 'proxy.config.diags.debug.tags': 'http|test_tsapi', +}) + +ts.Disk.ssl_multicert_config.AddLine( + 'dest_ip=* ssl_cert_name=server.pem ssl_key_name=server.key' +) + +ts.Disk.remap_config.AddLine( + "map http://myhost.test:{0} http://127.0.0.1:{0}".format(server.Variables.Port) +) +ts.Disk.remap_config.AddLine( + "map https://myhost.test:{0} http://127.0.0.1:{0}".format(server.Variables.Port) +) + +Test.PreparePlugin(Test.Variables.AtsTestToolsDir + '/plugins/test_tsapi.cc', ts) + +tr = Test.AddTestRun() +# Probe server port to check if ready. +tr.Processes.Default.StartBefore(server, ready=When.PortOpen(server.Variables.Port)) +# Probe TS cleartext port to check if ready. +tr.Processes.Default.StartBefore(Test.Processes.ts, ready=When.PortOpen(ts.Variables.port)) +# +tr.Processes.Default.Command = ( + 'curl --verbose --ipv4 --header "Host: myhost.test:{0}" http://localhost:{1}/'.format(server.Variables.Port, ts.Variables.port) +) +tr.Processes.Default.ReturnCode = 0 + +tr = Test.AddTestRun() +tr.Processes.Default.Command = ( + 'curl --verbose --ipv4 --http2 --insecure --header ' + + '"Host: myhost.test:{0}" https://localhost:{1}/'.format(server.Variables.Port, ts.Variables.ssl_port) +) +tr.Processes.Default.ReturnCode = 0 + +tr = Test.AddTestRun() +# Change server port number (which can vary) to a fixed string for compare to gold file. +tr.Processes.Default.Command = "sed 's/:{0}/:SERVER_PORT/' < {1}/log.txt > {1}/log2.txt".format( + server.Variables.Port, Test.RunDirectory) +tr.Processes.Default.ReturnCode = 0 +f = tr.Disk.File("log2.txt") +f.Content = "log.gold" diff --git a/tests/gold_tests/pluginTest/url_sig/url_sig.test.py b/tests/gold_tests/pluginTest/url_sig/url_sig.test.py index 19af447f0d6..3b83837e2b0 100644 --- a/tests/gold_tests/pluginTest/url_sig/url_sig.test.py +++ b/tests/gold_tests/pluginTest/url_sig/url_sig.test.py @@ -16,15 +16,16 @@ # See the License for the specific language governing permissions and # limitations under the License. +import hashlib +import hmac import os import subprocess Test.Summary = ''' Test url_sig plugin ''' -Test.SkipUnless( - Condition.HasATSFeature('TS_USE_TLS_ALPN'), -) +Test.ContinueOnFail = True +Test.SkipIf(Condition.true("Test is temporarily turned off, to be fixed according to an incompatible plugin API change (PR #4964)")) # Skip if plugins not present. Test.SkipUnless(Condition.PluginExists('url_sig.so')) @@ -195,16 +196,6 @@ # Success tests. -# No client / SHA1 / P=1 / URL not pristine / URL not altered. -# -tr = Test.AddTestRun() -tr.Processes.Default.ReturnCode = 0 -tr.Processes.Default.Command = ( - "curl --verbose --proxy http://127.0.0.1:{} 'http://one.two.three/".format(ts.Variables.port) + - "foo/abcde/qrstuvwxyz?E=33046618506&A=1&K=7&P=1&S=acae22b0e1ba6ea6fbb5d26018dbf152558e98cb'" + - LogTee -) - # With client / SHA1 / P=1 / URL pristine / URL not altered. # tr = Test.AddTestRun() @@ -245,13 +236,34 @@ LogTee ) +def sign(payload, key): + secret=bytes(key,'utf-8') + data=bytes(payload, 'utf-8') + md=bytes(hmac.new(secret, data, digestmod=hashlib.sha1).digest().hex(), 'utf-8') + return md.decode("utf-8") + +# No client / SHA1 / P=1 / URL not pristine / URL not altered. +# +path="foo/abcde/qrstuvwxyz?E=33046618506&A=1&K=7&P=1&S=" +to_sign="127.0.0.1:{}/".format(server.Variables.Port) + path +url="http://one.two.three/" + path + sign(to_sign, "dqsgopTSM_doT6iAysasQVUKaPykyb6e") + +tr = Test.AddTestRun() +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Command = ( + "curl --verbose --proxy http://127.0.0.1:{} '{}'".format(ts.Variables.port, url) + LogTee +) + # No client / SHA1 / P=1 / URL not pristine / URL not altered -- HTTPS. # +path="foo/abcde/qrstuvwxyz?E=33046618506&A=1&K=7&P=1&S=" +to_sign="127.0.0.1:{}/".format(server.Variables.Port) + path +url="https://127.0.0.1:{}/".format(ts.Variables.ssl_port) + path + sign(to_sign, "dqsgopTSM_doT6iAysasQVUKaPykyb6e") + tr = Test.AddTestRun() tr.Processes.Default.ReturnCode = 0 tr.Processes.Default.Command = ( - "curl --verbose --http1.1 --insecure --header 'Host: one.two.three' 'https://127.0.0.1:{}/".format(ts.Variables.ssl_port) + - "foo/abcde/qrstuvwxyz?E=33046618506&A=1&K=7&P=1&S=acae22b0e1ba6ea6fbb5d26018dbf152558e98cb'" + + "curl --verbose --http1.1 --insecure --header 'Host: one.two.three' '{}'".format(url) + LogTee + " ; grep -F -e '< HTTP' -e Authorization {0}/url_sig_long.log > {0}/url_sig_short.log ".format(ts.RunDirectory) ) diff --git a/tests/gold_tests/pluginTest/xdebug/x_remap/four.in b/tests/gold_tests/pluginTest/xdebug/x_remap/four.in new file mode 100644 index 00000000000..1982451bbe8 --- /dev/null +++ b/tests/gold_tests/pluginTest/xdebug/x_remap/four.in @@ -0,0 +1,4 @@ +GET /not_there HTTP/1.1 +Host: two +X-Debug: X-Remap + diff --git a/tests/gold_tests/pluginTest/xdebug/x_remap/out.gold b/tests/gold_tests/pluginTest/xdebug/x_remap/out.gold index e60a8aae9bf..0619bd2e5c8 100644 --- a/tests/gold_tests/pluginTest/xdebug/x_remap/out.gold +++ b/tests/gold_tests/pluginTest/xdebug/x_remap/out.gold @@ -58,6 +58,17 @@ X-Remap: from=http://three[0-9]+/, to=http://127.0.0.1:SERVER_PORT/ 0 +====== +HTTP/1.1 404 Not Found +Server: ATS/`` +Date: `` +Age: `` +Transfer-Encoding: chunked +Connection: keep-alive +X-Remap: from=http://two/, to=http://127.0.0.1:SERVER_PORT/ + +0 + ====== HTTP/1.1 200 OK Date: `` diff --git a/tests/gold_tests/pluginTest/xdebug/x_remap/x_remap.gold b/tests/gold_tests/pluginTest/xdebug/x_remap/x_remap.gold index 55e7f94dba5..d0baa817b03 100644 --- a/tests/gold_tests/pluginTest/xdebug/x_remap/x_remap.gold +++ b/tests/gold_tests/pluginTest/xdebug/x_remap/x_remap.gold @@ -4,6 +4,8 @@ X_DEBUG MISSING - X_DEBUG MISSING - +X_DEBUG MISSING +- X-Remap, fwd - X_DEBUG MISSING diff --git a/tests/gold_tests/pluginTest/xdebug/x_remap/x_remap.test.py b/tests/gold_tests/pluginTest/xdebug/x_remap/x_remap.test.py index e901098823c..fb070d0d3cb 100644 --- a/tests/gold_tests/pluginTest/xdebug/x_remap/x_remap.test.py +++ b/tests/gold_tests/pluginTest/xdebug/x_remap/x_remap.test.py @@ -68,6 +68,7 @@ def sendMsg(msgFile): sendMsg('one') sendMsg('two') sendMsg('three') +sendMsg('four') sendMsg('fwd1') sendMsg('fwd2') sendMsg('fwd3') diff --git a/tests/gold_tests/redirect/gold/redirect.gold b/tests/gold_tests/redirect/gold/redirect.gold index d6616ba55f9..9500bf976a0 100644 --- a/tests/gold_tests/redirect/gold/redirect.gold +++ b/tests/gold_tests/redirect/gold/redirect.gold @@ -1,4 +1,5 @@ HTTP/1.1 204 No Content +`` Age: `` Connection: keep-alive diff --git a/tests/gold_tests/remap/remap_https.test.py b/tests/gold_tests/remap/remap_https.test.py index 661863d5c34..efaa444b33d 100644 --- a/tests/gold_tests/remap/remap_https.test.py +++ b/tests/gold_tests/remap/remap_https.test.py @@ -61,7 +61,7 @@ 'map https://www.example.com:{1} http://127.0.0.1:{0}'.format(server.Variables.Port, ts.Variables.ssl_port) ) ts.Disk.remap_config.AddLine( - 'map https://www.anotherexample.com https://127.0.0.1:{0}'.format(server2.Variables.Port, ts.Variables.ssl_port) + 'map https://www.anotherexample.com https://127.0.0.1:{0}'.format(server2.Variables.SSL_Port,ts.Variables.ssl_port) ) diff --git a/tests/gold_tests/runroot/runroot_init.test.py b/tests/gold_tests/runroot/runroot_init.test.py index ec8a2f673bb..5e8e371a644 100644 --- a/tests/gold_tests/runroot/runroot_init.test.py +++ b/tests/gold_tests/runroot/runroot_init.test.py @@ -59,7 +59,6 @@ tr.Processes.Default.ReturnCode = 0 f = tr.Disk.File(os.path.join(path4, "runroot.yaml")) f.Exists = True -tr.Processes.Default.Streams.All = Testers.ContainsExpression("Forcing creating runroot", "force message") # create runroot with junk to guarantee only traffic server related files are copied bin_path = Test.Variables.BINDIR[Test.Variables.BINDIR.find(Test.Variables.PREFIX) + len(Test.Variables.PREFIX) + 1:] diff --git a/tests/gold_tests/runroot/runroot_verify.test.py b/tests/gold_tests/runroot/runroot_verify.test.py index a9253b4229b..d98058c8041 100644 --- a/tests/gold_tests/runroot/runroot_verify.test.py +++ b/tests/gold_tests/runroot/runroot_verify.test.py @@ -42,12 +42,7 @@ os.path.join(path, "bin"), "example bindir output") tr.Processes.Default.Streams.All = Testers.ContainsExpression( os.path.join(path, "var/log/trafficserver"), "example logdir output") -tr.Processes.Default.Streams.All = Testers.ContainsExpression( - "Read Permission: ", "read permission output") -tr.Processes.Default.Streams.All = Testers.ContainsExpression( - "Execute Permission: ", "execute permission output") -tr.Processes.Default.Streams.All = Testers.ContainsExpression( - "Write Permission: ", "write permission output") +tr.Processes.Default.Streams.All = Testers.ContainsExpression("PASSED", "contain passed message") # verify test #2 bin_path = Test.Variables.BINDIR[Test.Variables.BINDIR.find( @@ -60,9 +55,5 @@ os.path.join(path, "bin"), "example bindir output") tr.Processes.Default.Streams.All = Testers.ContainsExpression( os.path.join(path, "var/log/trafficserver"), "example logdir output") -tr.Processes.Default.Streams.All = Testers.ContainsExpression( - "Read Permission: ", "read permission output") -tr.Processes.Default.Streams.All = Testers.ContainsExpression( - "Execute Permission: ", "execute permission output") -tr.Processes.Default.Streams.All = Testers.ContainsExpression( - "Write Permission: ", "write permission output") +tr.Processes.Default.Streams.All = Testers.ContainsExpression("PASSED", "contain passed message") + diff --git a/tests/gold_tests/slow_post/slow_post.test.py b/tests/gold_tests/slow_post/slow_post.test.py index 906a87f14fd..c1e92f85374 100644 --- a/tests/gold_tests/slow_post/slow_post.test.py +++ b/tests/gold_tests/slow_post/slow_post.test.py @@ -53,9 +53,9 @@ def setupTS(self): self._ts.Disk.records_config.update({ 'proxy.config.diags.debug.enabled': 1, 'proxy.config.diags.debug.tags': 'http', - 'proxy.config.http.origin_max_connections': self._origin_max_connections, + 'proxy.config.http.per_server.connection.max': self._origin_max_connections, # Disable queueing when connection reaches limit - 'proxy.config.http.origin_max_connections_queue': 0, + 'proxy.config.http.per_server.connection.queue_size': 0, }) def run(self): diff --git a/tests/gold_tests/tls/gold/accesslog.gold b/tests/gold_tests/tls/gold/accesslog.gold new file mode 100644 index 00000000000..d00fc14a51a --- /dev/null +++ b/tests/gold_tests/tls/gold/accesslog.gold @@ -0,0 +1,8 @@ +1 0 +1 1 +1 0 +1 0 +1 0 +1 1 +1 0 +1 0 diff --git a/tests/gold_tests/tls/ssl-post.c b/tests/gold_tests/tls/ssl-post.c index cc8ea132aac..1b3a2fb2f68 100644 --- a/tests/gold_tests/tls/ssl-post.c +++ b/tests/gold_tests/tls/ssl-post.c @@ -89,6 +89,10 @@ spawn_same_session_send(void *arg) SSL_CTX *client_ctx = SSL_CTX_new(SSLv23_client_method()); SSL *ssl = SSL_new(client_ctx); +#if OPENSSL_VERSION_NUMBER >= 0x10100000 + SSL_set_max_proto_version(ssl, TLS1_2_VERSION); +#endif + SSL_set_session(ssl, tinfo->session); SSL_set_fd(ssl, sfd); @@ -282,6 +286,9 @@ main(int argc, char *argv[]) SSL_CTX *client_ctx = SSL_CTX_new(SSLv23_client_method()); SSL *ssl = SSL_new(client_ctx); +#if OPENSSL_VERSION_NUMBER >= 0x10100000 + SSL_set_max_proto_version(ssl, TLS1_2_VERSION); +#endif SSL_set_fd(ssl, sfd); int ret = SSL_connect(ssl); diff --git a/tests/gold_tests/tls/ssl/ca.ocsp.key b/tests/gold_tests/tls/ssl/ca.ocsp.key new file mode 100644 index 00000000000..ccae9f121a4 --- /dev/null +++ b/tests/gold_tests/tls/ssl/ca.ocsp.key @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEpQIBAAKCAQEAsg7JDZ8J/yG/bBxl/Td9urR78e0E/IwSfuujWbewn2fulEkF +jrvN8G4I8nmEG/Fg+2lMR2wKqUYYwbxKaq0tlprbZp/bB9QJqGdEqmX9HBKIPB6L +n0GFMwj5yG4etx2sx3scdHPaJBZ12IPljxJXPKsHurIY/tXa9WS6IhEwHgcNef+p +uVt8pNvmw35jyPMZhcXnN/Fs+bHX908SCrMgiwf4oKyPHFxh7WEqLVZj4Z2At7lH +On0wRHnYjFB7wKSMgM44eUjhjN6zOe4gWLGdfbtXgZibxgKLYfYL8YBzcPPChdCY +Evshn3RTbJ3IiwDHTAlBTchoDoBjV/gKlM8LbQIDAQABAoIBAQCMMeoJqIxFbrv6 +ko2XB2ceZ1cjz/xaIKu7dStDy8bsa/fEl44hqStoQCsZR6ZGHhK/QVRG9AGc8E0z +1V4+iiZX64wOTJU3n7MO/mhpRi45OTo9I8vJU9xdp5aMQnA6u8m7supfooxCV9Dn +7koEEWvQn9VRIUNe+uEQ0ANiKWhaauC4aODV3amNgmGTQVF1NKvcqdXbeQGCGHDy +HKoehDVlbaoKvzZ03dt0kL1sa+vqQmC6pNHlWQ9pchML6E+KBdaQrNCeRFHbLLiO +ULQg2KS0IF+kqbBrpBRd3lW4ZV2rglWZQ1998CJP0et3WXET14nqCTNfty3iTcUL +tXPGnXAdAoGBAM3BcnSWeEkTtY8Sxu3y/M/zB9wF7f+mHNE88ISWrdx/hJni4imV +diyzcnfyxTQc8uuett1UHA85SZuuDwdvVz7CD+UglmploFsaUvwl6Ytl05a2oFqT +n5dmnsGgLCpnpSNAZWpT6gyddQR3lyzCw+k5fkQVikPzcGJrd7mk27YfAoGBAN2J +1uYw3E7uiywjYlQXqrQ7L0Ouccw3wV0jcsSgDJGsffhTkPhn/BwZ2QQcldGR8TCB +Y6Er02Kqzk1mGFG94TPQy5C+xJzZv2KKq9LSrOVQszh+wozEruh7bKARQ/uPDLPw ++F/DWFe6/0mDb5gZf40pyk+njFh1UT7yGE2poVTzAoGBAMaDQ2hU3Iy05VC6rw9Y +hq4jLowLdIpYvCjsAKoLroa0yTynd8jjGPcb0u8DXVxgKcdGg+uagM/3V5tKHdnw +hF5aYXeRL05L6qC7DyGTenYxsikQ3jlFgI5URgtN/A6VnPAb6zzg5UlyiTncIBDh +gJ7+B2Ks3Y+dyepLAWItOoXFAoGBAMgahR2O7K/vD45it6I1bl81Rk/f9bH8eo/i +QPwRhMjgATiYYs29Px8yya7JExoktLKXbKJbr6fjmEyY90Z+ODhRVE39wiHbHN+p +WeInoTvQVNGmzZvQ3BvpwAglED7cyyCNfAsjq1wy7/w62EWOYoPjR3YDZOVRsn0k +t4cOvUa5AoGAL2LxuYaU5GESEv3MAtM4oQUsc0sHApGyy/hJzGBF/RXpy+U4kYBF +apAonFCQZ2VBr/t0XH0MW4GrRZoJsa1xWWhlXhARfOSWCRRS8tdTdir2TpLMmJJN +QakM9/QgfIQQ+J25XeLe16dRl2i4acG1BA2b1S1isC8HmV/wy4sGopA= +-----END RSA PRIVATE KEY----- diff --git a/tests/gold_tests/tls/ssl/ca.ocsp.pem b/tests/gold_tests/tls/ssl/ca.ocsp.pem new file mode 100644 index 00000000000..cd8114fb258 --- /dev/null +++ b/tests/gold_tests/tls/ssl/ca.ocsp.pem @@ -0,0 +1,20 @@ +-----BEGIN CERTIFICATE----- +MIIDWDCCAkCgAwIBAgIUUCa7m/ftLhb69KUqFfGBKwnw+lcwDQYJKoZIhvcNAQEL +BQAwRDELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAklMMQ8wDQYDVQQKEwZBcGFjaGUx +FzAVBgNVBAMTDmNhLmV4YW1wbGUuY29tMB4XDTE5MDUwMjEyNDAwMFoXDTQ5MDQy +NDEyNDAwMFowRDELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAklMMQ8wDQYDVQQKEwZB +cGFjaGUxFzAVBgNVBAMTDmNhLmV4YW1wbGUuY29tMIIBIjANBgkqhkiG9w0BAQEF +AAOCAQ8AMIIBCgKCAQEAsg7JDZ8J/yG/bBxl/Td9urR78e0E/IwSfuujWbewn2fu +lEkFjrvN8G4I8nmEG/Fg+2lMR2wKqUYYwbxKaq0tlprbZp/bB9QJqGdEqmX9HBKI +PB6Ln0GFMwj5yG4etx2sx3scdHPaJBZ12IPljxJXPKsHurIY/tXa9WS6IhEwHgcN +ef+puVt8pNvmw35jyPMZhcXnN/Fs+bHX908SCrMgiwf4oKyPHFxh7WEqLVZj4Z2A +t7lHOn0wRHnYjFB7wKSMgM44eUjhjN6zOe4gWLGdfbtXgZibxgKLYfYL8YBzcPPC +hdCYEvshn3RTbJ3IiwDHTAlBTchoDoBjV/gKlM8LbQIDAQABo0IwQDAOBgNVHQ8B +Af8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQUmAVXj2UeCyJRWe73 +dBL672OFRKIwDQYJKoZIhvcNAQELBQADggEBAEf1Jntnfg+4W4/u7FGGFgkjiQ9e +zS38xRsSyKhQ5pBipUHbkphbgwu0xXDa9jQkVJuwsk7xrMdvbKmbVx8DBg4OZKu+ +50eqw8/NTg401TVLKC+rJ5kcgUEGuZobegBhKtUqzuswQQF4F4KMTUreJBXWPeET +CxFyy04LAxeewEYLzJ/Ylw4jgosC+MlTlXmN41SxtQ5URjOOIPmHOZF9CzSHoJcz +qN84JT1cgcfd/gYgQD+r1pCPRjJrLU+5X3PCHHehNphIuU9cQ6KuJaCq1FABpovx +iIMiW56fzZ4Wbon3Sh0zuCxzvtBhtM5qOd+eP3pbIB6Ps+J9Dfkrpa/wVe0= +-----END CERTIFICATE----- diff --git a/tests/gold_tests/tls/ssl/combo-signed-foo.pem b/tests/gold_tests/tls/ssl/combo-signed-foo.pem new file mode 100644 index 00000000000..e3bf4cd29f1 --- /dev/null +++ b/tests/gold_tests/tls/ssl/combo-signed-foo.pem @@ -0,0 +1,47 @@ +-----BEGIN CERTIFICATE----- +MIIDCzCCAnQCCQC81MtBCwmQtzANBgkqhkiG9w0BAQsFADCBnTELMAkGA1UEBhMC +VVMxCzAJBgNVBAgTAklMMRIwEAYDVQQHEwlDaGFtcGFpZ24xDjAMBgNVBAoTBVlh +aG9vMQ0wCwYDVQQLEwRFZGdlMSgwJgYDVQQDEx9qdWljZXByb2R1Y2UuY29ycC5u +ZTEueWFob28uY29tMSQwIgYJKoZIhvcNAQkBFhVwZXJzaWEuYXppekB5YWhvby5j +b20wHhcNMTgxMDE1MTU1NjMzWhcNMjgxMDEyMTU1NjMzWjByMQswCQYDVQQGEwJV +UzELMAkGA1UECAwCSUwxEjAQBgNVBAcMCUNoYW1wYWlnbjEQMA4GA1UECgwHRXhh +bXBsZTEQMA4GA1UEAwwHZm9vLmNvbTEeMBwGCSqGSIb3DQEJARYPYm9iQGV4YW1w +bGUuY29tMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAyYXjrK9KOAtE +FXpaPL4mJfyI1r5I+GOmKI5zvxPU+R0n1sZEYpEU/F+8qrPQpa4zXGKDzTUD+b4J +dagCHosVK22WZJXGd5BxfzF3c7mFf/7k92H1q7Dk3X23LumnR7Qa/0HNMPRkmwaa +eNmQC8c42doWWaudV5ir3M+ef4Jv/WL5RhK877D85Ho3R+lNRini6hmmTqpFezdi +eMypicoj88K5kf/Mu5PvYwx0F/gNsGuYGogDBnSDPGk1fl7DTQL3rKb18l+1IcMQ +7MNHq9Bi7+LLGAq7uYRfrVHI3jgh8UpwqBAuOW9sw2RwEyy46+wCCedk3EqNZE2k +4qwDgIh0SwIDAQABMA0GCSqGSIb3DQEBCwUAA4GBALr2gm+KgveEcTXwURM0wxJC +m0yOR8w6MX8fxHKaekhJH1U84G64Ub0gbn2beOdLBQkG+4czLiOOOgyeukPaJJ81 +od2ooE7DrGUPGnbHYxW/70EtVF5nQEctcqpKNF/d04mVKrqI90919MJSxJ5KedHK +2H11+gUPwDWy/mAwJzEJ +-----END CERTIFICATE----- +-----BEGIN PRIVATE KEY----- +MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDJheOsr0o4C0QV +elo8viYl/IjWvkj4Y6YojnO/E9T5HSfWxkRikRT8X7yqs9ClrjNcYoPNNQP5vgl1 +qAIeixUrbZZklcZ3kHF/MXdzuYV//uT3YfWrsOTdfbcu6adHtBr/Qc0w9GSbBpp4 +2ZALxzjZ2hZZq51XmKvcz55/gm/9YvlGErzvsPzkejdH6U1GKeLqGaZOqkV7N2J4 +zKmJyiPzwrmR/8y7k+9jDHQX+A2wa5gaiAMGdIM8aTV+XsNNAvespvXyX7UhwxDs +w0er0GLv4ssYCru5hF+tUcjeOCHxSnCoEC45b2zDZHATLLjr7AIJ52TcSo1kTaTi +rAOAiHRLAgMBAAECggEAB7fXBnAYOZlE3EW5WwY1U9MeMotLJCg83uTFzhWmXHwf +YHxrdhL0aM4J3cfRP+cyFGG5hox3QINkvVrX6e+NugISdnu+BCpGDocIeigq0sIi +Zs8bp524xjrgXy2XuIlPV2NfxnY1vDI+jE5Y0/qnVMCjhn+qIQa53lUdTujh/SRR +3U7di+QMK4mdGwRnInos++ENy33A+2LqtUK8i0ERkzPFa1yMQEE4DOFPzZcW+jhK +arvzBwPIn37PZmL5oyiQB1YiGPGt4XNfPBwACTMYM8LlYBfEBHG77k3bMtUf0WqE +GctoT5SIe5+YbyrWkpfHgoKPxggH3I3TrFnVvqrKQQKBgQDmcV1YbuNEQLeif521 +iGqMgPQYmnpO6k27RsZrM9ikhIgm9bVJsOqnaYzQFeSfJ3eNLXYUL6IF8g46xddw +fDBtrEjDAA9OUkNRcizbeKF+GJRMtX11d4ZNbnG1wyMZYkArZGfraZBLHPEF1pya +2iFdVfokQCBpLmX7BMQEPePyuwKBgQDf33H9njf9oO0l9GfuWDvSoaV8GwqV9x55 +sFjggQYD/xqwEprrzr524X5Y2ZiTUpBu+kqqM8GYfm3bzBKkZU1rnjwxADUwBw8U +L2U/Z7Id3om8tAdzHOSI9d7mxWA8uTsScMm0IFv2l/XBQo1+AAJSD03pcsabr4Lf +SuJGmoFTsQKBgQDVzPASEC+DL5gwh75Gop5YZXwTJ5+6f+BGlM+avquNV/kKTIU6 +LY5IbMFcfjNzBicBMOCQsfDdG0rgdJYBovc7idCoOvH4dJJIimnb5fvPBfbxhKE1 +zwMn7ARL4xQ5hNKMb8eKvpJFXkCwbgE2GpNCCXbfEy/+5jFvx2gll1ZZ6QKBgG3J +OzJ/w796irHBQLKOzI+HvAq3jCJs9KICjCNUwql1EhZkmVqooZjVDkvuMbeVlsUF +s1XyWa852RAf7Mh38VakW6pACtVJsOhaMdG9PYkOWAeVVc3qzlwoDy6mfoJo6AIs +E45lDBRLAzbKN28h/AFYBgJEygcRNCHirEKphGCRAoGAEhcaxbmMo2fHBYuvOR1Q +ZAIq1EPvysDROUBHhdTJqN1wHsuJsmVJxX42+YHcZdjtgeCdjU3HMoyCnTaRxDee +K3VeB4PobN1WpQwFklFoqcvAhW6eicdZXme7ktK120NPQsXrmjgN6Lfg3PNjosn0 +tqSxQhQ4DrSf60fxx0/M/rw= +-----END PRIVATE KEY----- diff --git a/tests/gold_tests/tls/ssl/ocsp_response.der b/tests/gold_tests/tls/ssl/ocsp_response.der new file mode 100644 index 00000000000..fd6d9a631f6 Binary files /dev/null and b/tests/gold_tests/tls/ssl/ocsp_response.der differ diff --git a/tests/gold_tests/tls/ssl/responder.ocsp.key b/tests/gold_tests/tls/ssl/responder.ocsp.key new file mode 100644 index 00000000000..c71e4e55fda --- /dev/null +++ b/tests/gold_tests/tls/ssl/responder.ocsp.key @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEpAIBAAKCAQEAtjD9/eCvlUxtKoXVh43ab60uriYqkbzOTm2r0URwfXndQAWh +d35vA2goUosnNNKcssvh8SZFpNNtYvD/5r+4LZL09nlCOdJ/+JEqKDxR0WfXNCxd +UB2RwFnf1rMSAfyj5sT7bBYlPy2EX8LvbruGlzI00MBS1zLsYKNFWq2V1gu8rl0z +pFkJhhOJqXUUbINlfCH+NvLaO9NByjV1esYfHe4lYtnHxno9Q0vuxT1dobuE4pUV +6v+SnSg8VP0ZdFQSuuss0E412NNna3naBoZn14cBWkFjAtNxqNC+a9dHYxOZrsEo +joB8R1E96YZwY0oJ47/LmG/Sp5IQKWZWFZ4pnQIDAQABAoIBAD32gZujm/PJ72LD +67BThVPv8W1XG6k/LmcsE4Bzp1J1bNMGVzj9riHZfcU9AFONwa9peel7G7qIEa7R +yiafU7NkRJ3C9cwWlGFkdZMDmMwAZgefgwjpVZW2u1MYyeoVE2U730qOaZKIF3o+ +IRJnAspPT/kjP8liz1O6k67YVJpcBcp9nJruDJh8BZtgneb2z+di8aV6gi2d2Cfq +rPrAxlwDc1XiMgG+Auuq0OQs++K1GFAcy6pMutjIpXMDl3eAUddPjEPBuKlUiRO1 +GqR9d3YRWR+/KZV++RzdwPbZVjiK2STSrimsDfyufibD6xoQ3TuKtdL8UsSVdOkr +OORojd0CgYEAwfJ1EnJXhXmJhZj6qXE2Es61c2/JLXFbk0ryE7nIdmalU1rT7h0J +jPp+uzl1mtSRrx/aTesLOqHFefJqMOiTVjFO/7+J+fe7gsnHAvB9cHXLkSY6FN37 +p7PQ6yFlb+k8Biug+6tyuGv78MxADZMZ9pWc/AD4VQZlDkAygIFbVVcCgYEA8Hup +70Xv7O2lxnaPAhbqm1Ijm+ABDgObh8s1CaTO7F4upwmOSZ+EOrMdP6+vdPq0Hrm/ +KObaqRM5Vwi74hEoxKCoLulBslEnq2KbU0t22TnqCF+UBYledfFKgYTJmfvqGCxg +uVPQ1qZp//9yQxcrIhr4AwQDr3dX1Mb8VfH3TCsCgYEAkDHIMsfKJFVhFm/PZSzj +jAYdR88DnoKaGB9vbZUB4m2cWyW7TVxPXn5avK4SruN90Nr4vleTCKt/m5PMucIg +0MNmPaTVW4CA69NC3/+W84bQq4DlS+BimqOJH1e8CAE6/Edxr8sfRtgZ/0SMFsuY +UQmZJo8+ElDnzzmRkpMaKY8CgYA/KHtM+BU3KILtSJ3uco5TFJN9kKs2PwRN+bSI +P9yIf4PJIt1XwKk9sWTxIPb3xhAgMbBe0aKD3SSmEwklKlSGr5r8Fw7GAkJk5JTe +n2crTeaFJHT/r0A7wY9LzNAVvO+SQbV1dunWNgaI0VH1BNSzNFoGkLtXDgTnQQts +lwvX2QKBgQCjMNbYh+7TSe/L/ys/TG+tuFkNaprvrHeRYigI4U43rWKiY2EteYi/ +V/eVY/t7hWDnwV9FDo5q/v4dneWWgl2kJA8NlP/p65M+YrTY1nVRrbJX4+GIlxxY +yaGI4Ab1WBFsYkpHC4BpBommJV/znaOcNS/EBY3G3ZmW+EzGmiTuvw== +-----END RSA PRIVATE KEY----- diff --git a/tests/gold_tests/tls/ssl/responder.ocsp.pem b/tests/gold_tests/tls/ssl/responder.ocsp.pem new file mode 100644 index 00000000000..03753b8dd1f --- /dev/null +++ b/tests/gold_tests/tls/ssl/responder.ocsp.pem @@ -0,0 +1,22 @@ +-----BEGIN CERTIFICATE----- +MIIDlzCCAn+gAwIBAgIURxCF64GDk44IQeo7HFw5EB5jKNUwDQYJKoZIhvcNAQEL +BQAwRDELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAklMMQ8wDQYDVQQKEwZBcGFjaGUx +FzAVBgNVBAMTDmNhLmV4YW1wbGUuY29tMB4XDTE5MDUwMjEyNDAwMFoXDTQ5MDQy +NDEyNDAwMFowUDELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAklMMQ8wDQYDVQQKEwZB +cGFjaGUxIzAhBgNVBAMTGm9jc3AtcmVzcG9uZGVyLmV4YW1wbGUuY29tMIIBIjAN +BgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAtjD9/eCvlUxtKoXVh43ab60uriYq +kbzOTm2r0URwfXndQAWhd35vA2goUosnNNKcssvh8SZFpNNtYvD/5r+4LZL09nlC +OdJ/+JEqKDxR0WfXNCxdUB2RwFnf1rMSAfyj5sT7bBYlPy2EX8LvbruGlzI00MBS +1zLsYKNFWq2V1gu8rl0zpFkJhhOJqXUUbINlfCH+NvLaO9NByjV1esYfHe4lYtnH +xno9Q0vuxT1dobuE4pUV6v+SnSg8VP0ZdFQSuuss0E412NNna3naBoZn14cBWkFj +AtNxqNC+a9dHYxOZrsEojoB8R1E96YZwY0oJ47/LmG/Sp5IQKWZWFZ4pnQIDAQAB +o3UwczAOBgNVHQ8BAf8EBAMCB4AwEwYDVR0lBAwwCgYIKwYBBQUHAwkwDAYDVR0T +AQH/BAIwADAdBgNVHQ4EFgQUGOVVyRVQ7T9HG+yyb+evtoXW1/wwHwYDVR0jBBgw +FoAUmAVXj2UeCyJRWe73dBL672OFRKIwDQYJKoZIhvcNAQELBQADggEBAKSA6zdF +BFEy/gLQcGBsJ6IxnIpL3kijk/cdfMMraj+WgFpdr4utZXMI5kJ8s9C09nVqfYIk +os2XEUkpWOTbbXJDcgYjOtUy/3VPJ6BTELQYLR/IDISDiIwdehTtBOosHwuSoW1t +LnPQFtmZda6PvYfETKGBsAIKSM16GZQHJFrEKldWYALynjDAKwBN+SQDC6SHs48U +Md4V7XPm1GXysewBg+GIXZxPECBE7BstcQbOViiMY0ZvSbR8mMoxiSYLNA+2QrTu +QBFQPf2Bpyc10vSyM+8P+pZb++p9raK0BWqS8rNi0BP7OnakzpvZygwtwY8sRYM8 +AToAbPRxJMJaIAU= +-----END CERTIFICATE----- diff --git a/tests/gold_tests/tls/ssl/server.key b/tests/gold_tests/tls/ssl/server.key index 4c7a661a6bd..9cdfc36aa5f 100644 --- a/tests/gold_tests/tls/ssl/server.key +++ b/tests/gold_tests/tls/ssl/server.key @@ -1,15 +1,28 @@ ------BEGIN RSA PRIVATE KEY----- -MIICXQIBAAKBgQDWMHOiUF+ORmZjAxI8MWE9dblb7gQSJ36WCXlPFiFx6ynF+S1E -kXAYpIip5X0pzDUaIbLukxJUAAnOtMEO0PCgxJQUrEtRWh8wiJdbdQJF0Zs/9R+u -SUgb61f+mdTQvhqefBGx+xrpfAcgtcWiZuSA9Q3fvpDj5WOWSPWXBUuxywIDAQAB -AoGBAJPxRX2gjFAGWmQbU/YVmXfNH6navh8X/nx9sLeqrpE0AFeJI/ZPiqDKzMal -B43eSfNxwVi+ZxN0L1ICUbL9KKZvHs/QBxWLA1fGVAXrz7sRplEVvakPpTfHoEnv -sKaMWVKaK/S5WGbDhElb6zb/Lwo19DsIAPjGYqFvzFJBmobJAkEA9iSeTGkR9X26 -GywZoYrIMlRh34htOIRx1UUq88rFzdrCF21kQ4lhBIkX5OZMMy652i2gyak4OZTe -YewIv8jw9QJBAN7EQNHG8jPwXfVp91/fqxVQEfumuP2i6uiWWYQgZCmla2+0xcLZ -pMQ6sQEe10hhTrVnzHgAUVp50Ntn2jwBX78CQF09veGAI9d1Cxzj9cmmAvRd1r2Q -tp8kPOLnUsALXib+6WtqewLCdcf8DtsdClyRJMIraq85tRzK8fryKNZNzkkCQEgA -yS7FDj5JgCU15hZgFk1iPx3HCt44jZM2HaL+UUHAzRQjKxTLAl3G1rWVAWLMyQML -lORoveLvotl4HOruSsMCQQCAx9dV9JUSFoyc1CWILp/FgUH/se4cjQCThGO0DoQQ -vGTYmntY7j9WRJ9esQrjdD6Clw8zM/45GIBNwnXzqo7Z ------END RSA PRIVATE KEY----- +-----BEGIN PRIVATE KEY----- +MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQCZkEXSlZ+ZFKFg +CPpcDG39e73BuK6E5uE38q2PHh4DV0xcsJnIUx51viqLPwYughxfP0crHyBdXoHV +dW/3WX4gpiGrdiM/dvCouheo0DPaqUUJ2nZKVYh2M57oyeiuJidlKb7BGkfw3HWP +9TV7dVyGWok/cowjopqaLHJWxg/kh2KqvUBD0CHt9Kd1XvgXVmHwE7vCv0j5owv2 +MaExTsFb16uWmVLhl1gNHI2RqCX2yLaebH1DvtbLrit1XErjtaSYeJE9clVRaqT6 +vsvLOhyB5tA9WfZqfBYr/MHDeXQfrbIf+4Cp3aTpq5grc5InIMMH0eOk6/f/4tW+ +nq1lfszZAgMBAAECggEAYvYAqRbXRRVwca0Xel5gO2yU+tSDUw5esWlow8RK3yhR +A6KjV9+Iz6P/UsEIwMwEcLUcrgNfHgybau5Fe4dmqq+lHxQA3xNNP869FIMoB4/x +98mbVYgNau8VRztnAWOBG8ZtMZA4MFZCRMVm8+rL96E8tXCiMwzEyPo/rP/ymfhN +3GRunX+GhfIA79AYNbd7HMVL+cvWWUGUF5Bc5i1wXcLy4I7b9NYtv920BeCLzSFK +BypFB7ku/vKgTcBxe4yxThxPeXPwm4WFzGYKk/Afl1j8tVXCE2U4Y3yykfC0Qk6S +ECZbCKLO2Rxi9fclIDZBHWuKejZhdjHfjeNvZ2vLoQKBgQDJzLmkVLvWAxgl1yvF +U7gwqj/TzYqtVowbjEvTNEnPU1j/hIVI343SVV/EvJmif/iRUop6sRYfLsUjpMsH +CmPysNKL3UtgSYOxLs+0xLhG4OOQRpPSf/uvl9YyWY9G3AqiC7ScthkQjEhZa4c1 +eycYy0jr42kX0OL9MuIH9q0ENQKBgQDCzvGKMs8r5E/Qd3YB9VYB60dx+6G83AHZ +YqIelykObhCdxL9n4K+p4VKKLvgTcCOLYYIkBSWRJWR+ue3s3ey9+XWd2/q4Xvfh +TCjAuO2ibMV+y5ClNlW0fQ/doIVWSDbjO2tZW1jh7YWZ4CtuVrsEisv1sk3KltMB +MguhpTUylQKBgG6TfrncMFzxrx61C+gBmvEXqQffHfkjbnx94OKnSTaQ3jiNHhez +X9v8KhD8o1bWtpay2uyl8pA9qYqBdzqxZ9kJKSW4qd/mCIJjOy87iBpWint5IPD8 +biZmldlbF9ZlJnJq5ZnlclCN/er5r8oPZHoCkj+nieOh8294nUBt25ptAoGAMnPA +EIeaKgbmONpHgLhWPwb9KOL/f1cHT5KA5CVH58nPmdyTqcaCGCAX7Vu+ueIIApgN +SWDf2thxT3S9zuOm5YiO0oRfSZKm5f2AbHE4ciFzgKQd4PvSdH0TN9XT0oW/WVhR +NAI5YcHPIQvyk4/4vXNo4Uf9Z6NqIFwisQmFXoUCgYBK/ZI/HsFsvnR5MV0tFdGM +AdNe6bsQRSZkowoaPxuWbklE4Hn6QvwEmQg3ST2O+vCQV1f1yI6YiWYoptOYscJc +MSs/HxhhaaO5ZsiuPUO6WEPzpNb2CxuIGDDtl83VtUQyjxCmOb6pqqjwzFmZ2bsw +JNMaBCzokrJTxknvauCuSQ== +-----END PRIVATE KEY----- diff --git a/tests/gold_tests/tls/ssl/server.ocsp.key b/tests/gold_tests/tls/ssl/server.ocsp.key new file mode 100644 index 00000000000..03790391e89 --- /dev/null +++ b/tests/gold_tests/tls/ssl/server.ocsp.key @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEpAIBAAKCAQEA53sB/uEdZNJIFLiMJr+5KaB75lTCEBIg98FCCfxBPsL1XqNO +s3Tx1ihmCMeT6wnvxIi+RIIbf0ymSquImmkzAodZdW0MGs/G9Vo2HsjclfkOsZhN +M5k8/AXF/FaGOjGd/WOKZFGejhd6utzqOo+Bj+STm2u+bCNtI4IZF+icTri7rIIK +CMdZalN5Au33DqT+MuKhtTmRYs1Pw8Qf2inv311y7f5Ok/TvBnfbAU18Jo+Ga/M0 +94bVAMA8WH6RyABZZlr8VIn6e5YadFfHVt7GrMIO/2QV4mlCK7zEC2NamNNslf4c +n2NpQNQToV1UbOK3uH5SNtB6CqJDBR9sIPnTKwIDAQABAoIBADgrmjVeLQLNIB6f +FatFdMoMHmSrBphdvdBA/iRsKOzw5Be96xgS9agxD2lr/JHZTGxVfk4jgEaos+WE +sFY/1tfzPhsHhhtvdekNpfpcZWKjGBSyT3GI6sqBICT2XgX7Ckp1gByNzbrPKcH0 +X4YsUpU3MzZQs9mL0yz9odcyY9OZ/BMX2lHb/G/IzZqb2a5YCbew1eZyU3pMRs2X +POtKJgCfiKOGcI+Kv3gL8CWtMWkxZN6CeNF1GicndNLWBdW6KSPOjAWf7eBAu8cs +nC7gtpCcpqL2GVOx7rzuPPjTRY62Xqf1/2AjeJkAqkeBbqs9shlK4Nm76WvDsDp9 +V7ZxjpECgYEA8YX3lCy0Ba+VE9xmHrzIqBzl39s73du17hiWlNcI05fer3lKt9g6 +IINnH60nzYXYW3hKHoKBUJiop2/TI6/aaLTrlpJ/0lcFNLI2ZXOQ7fQ4s/iFyWrP +DkCE5Z1qlUNsPjCN1ZG4xMciGy83JtzjsQfoj3sHBUnrk8uqftp7hAMCgYEA9Vrw +j45Z9WKbBb/ilRv0UhkZ8uFYDut2udZZ2fp//Cb0C9ykdKeu6Q9p6MdFjdCPiNWi +DDx885arMmtJuRS4MGAt9qps4BIsOQAdsLW/5BB9hRR5nqYSaaOA/FZ/X2uFtmIk +5zjkHXktBBTqoVNVqGwnjeQyGqCpWxRXtNtTz7kCgYEA3IzfZnnj8oVB9x7+SfdO +rOWmrOMAKjpmSgQ+DbDHqKE4griaGIPloKcd1nlCrZUZ231fAble6QBekne1MRN2 +uMLtl1Q0URmR8WsD7WS45fJsjTvWv/U/Gt6j/SHgoGkvQSMJggtN1LObW4OkM2Lm +sVRtdAh+gr/b1dzX1nsg640CgYAWQbaavyYH7Xb0kZCDSDLkk6RX9Psg91kgyIIE +FQYxIHN48/3zGxbxy1UnKZR0pduvZPm7NG19R0imXTcl0+xVbxQcUR9pQBzE2u7W +jdYnYRuRy+awbo4zCQL1YP9S75UEk2iXlQCUb96WhTM3iTC3A4CfDXlCExrpyTGf +lVnH+QKBgQCGanhp+cL4gmbgFvu2KuLXGneK+gIRL3X9neLTq/0XoVE2pNiTZBne +bHVerfcogPXMSdFfCnjpWqx82UoXrQWghaOB9iLTPu+HH2SWWrvSle6JBYoF8iv8 +Y8/YKzlnZV62KpCPeKaal6RsPbNiPD5o6jONPKZmnxQeCa694ImLwg== +-----END RSA PRIVATE KEY----- diff --git a/tests/gold_tests/tls/ssl/server.ocsp.pem b/tests/gold_tests/tls/ssl/server.ocsp.pem new file mode 100644 index 00000000000..a8b14c0c736 --- /dev/null +++ b/tests/gold_tests/tls/ssl/server.ocsp.pem @@ -0,0 +1,23 @@ +-----BEGIN CERTIFICATE----- +MIID6TCCAtGgAwIBAgIUZyBINUwv8M98Rr31B9y2zXyuFDAwDQYJKoZIhvcNAQEL +BQAwRDELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAklMMQ8wDQYDVQQKEwZBcGFjaGUx +FzAVBgNVBAMTDmNhLmV4YW1wbGUuY29tMB4XDTE5MDUwMjEyNDAwMFoXDTQ5MDQy +NDEyNDAwMFowSDELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAklMMQ8wDQYDVQQKEwZB +cGFjaGUxGzAZBgNVBAMTEnNlcnZlci5leGFtcGxlLmNvbTCCASIwDQYJKoZIhvcN +AQEBBQADggEPADCCAQoCggEBAOd7Af7hHWTSSBS4jCa/uSmge+ZUwhASIPfBQgn8 +QT7C9V6jTrN08dYoZgjHk+sJ78SIvkSCG39MpkqriJppMwKHWXVtDBrPxvVaNh7I +3JX5DrGYTTOZPPwFxfxWhjoxnf1jimRRno4Xerrc6jqPgY/kk5trvmwjbSOCGRfo +nE64u6yCCgjHWWpTeQLt9w6k/jLiobU5kWLNT8PEH9op799dcu3+TpP07wZ32wFN +fCaPhmvzNPeG1QDAPFh+kcgAWWZa/FSJ+nuWGnRXx1bexqzCDv9kFeJpQiu8xAtj +WpjTbJX+HJ9jaUDUE6FdVGzit7h+UjbQegqiQwUfbCD50ysCAwEAAaOBzjCByzAO +BgNVHQ8BAf8EBAMCBaAwEwYDVR0lBAwwCgYIKwYBBQUHAwEwDAYDVR0TAQH/BAIw +ADAdBgNVHQ4EFgQUeLq8rzbtQim/wBI+OmgVScP+cWgwHwYDVR0jBBgwFoAUmAVX +j2UeCyJRWe73dBL672OFRKIwMQYIKwYBBQUHAQEEJTAjMCEGCCsGAQUFBzABhhVo +dHRwOi8vbG9jYWxob3N0Ojg4ODgwIwYDVR0RBBwwGoISc2VydmVyLmV4YW1wbGUu +Y29thwR/AAABMA0GCSqGSIb3DQEBCwUAA4IBAQCwLPIR8qzABd2OUvBOBXSWnXm2 +YiGY11805CISiDB+SGkRO+cd82wcjzI7bhPG9jtZ4ydrmOZ5FNmjtc0dgtmfnd3s +zGRQQ/cej5iWZbmWJOUKeCFZ1KC6c4DbwJe4iYekP0YIxcRK3hnRO1GBQ+zNwWs4 +74toxUp4dLUOuUaDZLsk0tsTqZO8alszpmIYNn1VzEguqE6FwBcFxNx110/dtH6P +chSOOG4m8ZJkZuV2PmdtOhqi4RdnUGIGS6kWjDYM6Qe+zworGzmNHv1iZMcaVvzm +YOQ54MTQ/V8o411Ckszzu1N0edC/WPBcE1ARrRMTN1hXzlGIwsLvGz51/lwd +-----END CERTIFICATE----- diff --git a/tests/gold_tests/tls/ssl/server.pem b/tests/gold_tests/tls/ssl/server.pem index 58b9b9715b7..2b56cc83ea2 100644 --- a/tests/gold_tests/tls/ssl/server.pem +++ b/tests/gold_tests/tls/ssl/server.pem @@ -1,32 +1,21 @@ ------BEGIN RSA PRIVATE KEY----- -MIICXQIBAAKBgQDWMHOiUF+ORmZjAxI8MWE9dblb7gQSJ36WCXlPFiFx6ynF+S1E -kXAYpIip5X0pzDUaIbLukxJUAAnOtMEO0PCgxJQUrEtRWh8wiJdbdQJF0Zs/9R+u -SUgb61f+mdTQvhqefBGx+xrpfAcgtcWiZuSA9Q3fvpDj5WOWSPWXBUuxywIDAQAB -AoGBAJPxRX2gjFAGWmQbU/YVmXfNH6navh8X/nx9sLeqrpE0AFeJI/ZPiqDKzMal -B43eSfNxwVi+ZxN0L1ICUbL9KKZvHs/QBxWLA1fGVAXrz7sRplEVvakPpTfHoEnv -sKaMWVKaK/S5WGbDhElb6zb/Lwo19DsIAPjGYqFvzFJBmobJAkEA9iSeTGkR9X26 -GywZoYrIMlRh34htOIRx1UUq88rFzdrCF21kQ4lhBIkX5OZMMy652i2gyak4OZTe -YewIv8jw9QJBAN7EQNHG8jPwXfVp91/fqxVQEfumuP2i6uiWWYQgZCmla2+0xcLZ -pMQ6sQEe10hhTrVnzHgAUVp50Ntn2jwBX78CQF09veGAI9d1Cxzj9cmmAvRd1r2Q -tp8kPOLnUsALXib+6WtqewLCdcf8DtsdClyRJMIraq85tRzK8fryKNZNzkkCQEgA -yS7FDj5JgCU15hZgFk1iPx3HCt44jZM2HaL+UUHAzRQjKxTLAl3G1rWVAWLMyQML -lORoveLvotl4HOruSsMCQQCAx9dV9JUSFoyc1CWILp/FgUH/se4cjQCThGO0DoQQ -vGTYmntY7j9WRJ9esQrjdD6Clw8zM/45GIBNwnXzqo7Z ------END RSA PRIVATE KEY----- -----BEGIN CERTIFICATE----- -MIICszCCAhwCCQD4jSkztmlO1TANBgkqhkiG9w0BAQsFADCBnTELMAkGA1UEBhMC -VVMxCzAJBgNVBAgTAklMMRIwEAYDVQQHEwlDaGFtcGFpZ24xDjAMBgNVBAoTBVlh -aG9vMQ0wCwYDVQQLEwRFZGdlMSgwJgYDVQQDEx9qdWljZXByb2R1Y2UuY29ycC5u -ZTEueWFob28uY29tMSQwIgYJKoZIhvcNAQkBFhVwZXJzaWEuYXppekB5YWhvby5j -b20wHhcNMTcwODI4MDM0NDQ1WhcNMjcwODI2MDM0NDQ1WjCBnTELMAkGA1UEBhMC -VVMxCzAJBgNVBAgTAklMMRIwEAYDVQQHEwlDaGFtcGFpZ24xDjAMBgNVBAoTBVlh -aG9vMQ0wCwYDVQQLEwRFZGdlMSgwJgYDVQQDEx9qdWljZXByb2R1Y2UuY29ycC5u -ZTEueWFob28uY29tMSQwIgYJKoZIhvcNAQkBFhVwZXJzaWEuYXppekB5YWhvby5j -b20wgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBANYwc6JQX45GZmMDEjwxYT11 -uVvuBBInfpYJeU8WIXHrKcX5LUSRcBikiKnlfSnMNRohsu6TElQACc60wQ7Q8KDE -lBSsS1FaHzCIl1t1AkXRmz/1H65JSBvrV/6Z1NC+Gp58EbH7Gul8ByC1xaJm5ID1 -Dd++kOPlY5ZI9ZcFS7HLAgMBAAEwDQYJKoZIhvcNAQELBQADgYEATX7975NdhIbJ -glda+sXI9a86GgOpiuKO+vKubRJQZA+UlPf2vHEONjC2+7Y1aZvZYaKYL74vxGky -zkgp6ANSPl45lqD632x0e1Z7vzW5TkqK1JB2/xH2WgDcQZmP0FuQHzVNs4GjghDr -HCp1+sQDhfPB4aLmLFeyN0TkhdH1N3M= +MIIDZDCCAkygAwIBAgIJANod1+h9CtCaMA0GCSqGSIb3DQEBCwUAMEcxCzAJBgNV +BAYTAlVTMQswCQYDVQQIDAJJTDEPMA0GA1UECgwGQXBhY2hlMRowGAYDVQQDDBFy +YW5kb20uc2VydmVyLmNvbTAeFw0xODExMTkxNzEwMTlaFw0yODExMTYxNzEwMTla +MEcxCzAJBgNVBAYTAlVTMQswCQYDVQQIDAJJTDEPMA0GA1UECgwGQXBhY2hlMRow +GAYDVQQDDBFyYW5kb20uc2VydmVyLmNvbTCCASIwDQYJKoZIhvcNAQEBBQADggEP +ADCCAQoCggEBAJmQRdKVn5kUoWAI+lwMbf17vcG4roTm4TfyrY8eHgNXTFywmchT +HnW+Kos/Bi6CHF8/RysfIF1egdV1b/dZfiCmIat2Iz928Ki6F6jQM9qpRQnadkpV +iHYznujJ6K4mJ2UpvsEaR/DcdY/1NXt1XIZaiT9yjCOimposclbGD+SHYqq9QEPQ +Ie30p3Ve+BdWYfATu8K/SPmjC/YxoTFOwVvXq5aZUuGXWA0cjZGoJfbItp5sfUO+ +1suuK3VcSuO1pJh4kT1yVVFqpPq+y8s6HIHm0D1Z9mp8Fiv8wcN5dB+tsh/7gKnd +pOmrmCtzkicgwwfR46Tr9//i1b6erWV+zNkCAwEAAaNTMFEwHQYDVR0OBBYEFI2y +qm0+UAChDAnLrAINeFOuyUlhMB8GA1UdIwQYMBaAFI2yqm0+UAChDAnLrAINeFOu +yUlhMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZIhvcNAQELBQADggEBAA3ZNFbqxcOX +szS5A4EXCepyBJBFejEYy0CsvwQX/ai/pMrw5jqVeF0GAOTpBCVLddyY+ZV1arD2 +Pqi7Qwot9OxEZOzbCBiuMJGotruKgnWFQDHzJ9HA7KDQs270uNESAOG/xW9os9zN +MXApzqfBSR5EIQU5L3RtaiPzoKdQenGQUOj86s0Kon7snDSUzaA2VcfstMWgGvXP +JHtaVusULm0gry32cEap5G5UK+gII6DfLWgFwFGhHHmTz3mKjyGiJQ+09XBtu4lb +ENE+HGRBBA49dUKSr3kwErO4HyHnS0YrsTDnbYURCsGUDma12oijX2sCos6Q4zn8 +3svaouRrucw= -----END CERTIFICATE----- diff --git a/tests/gold_tests/tls/ssl/signed-bob-bar.pem b/tests/gold_tests/tls/ssl/signed-bob-bar.pem new file mode 100644 index 00000000000..cca4c2fe93c --- /dev/null +++ b/tests/gold_tests/tls/ssl/signed-bob-bar.pem @@ -0,0 +1,18 @@ +-----BEGIN CERTIFICATE----- +MIIC2jCCAkMCCQC81MtBCwmQuTANBgkqhkiG9w0BAQsFADCBnTELMAkGA1UEBhMC +VVMxCzAJBgNVBAgTAklMMRIwEAYDVQQHEwlDaGFtcGFpZ24xDjAMBgNVBAoTBVlh +aG9vMQ0wCwYDVQQLEwRFZGdlMSgwJgYDVQQDEx9qdWljZXByb2R1Y2UuY29ycC5u +ZTEueWFob28uY29tMSQwIgYJKoZIhvcNAQkBFhVwZXJzaWEuYXppekB5YWhvby5j +b20wHhcNMTgxMTIwMTQ1NDEyWhcNMjgxMTE3MTQ1NDEyWjBBMQswCQYDVQQGEwJV +UzELMAkGA1UECAwCSUwxDzANBgNVBAoMBkFwYWNoZTEUMBIGA1UEAwwLYm9iLmJh +ci5jb20wggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDuzA6e4MZp98m/ +XuhRAZMCaq9fM5zRBDsY5fpx3zUkJwJtZa4ALk6Adusovfh3t6qYJJ47TItnNG9Z +nAvYQhOccZNkj8xbPytuj5pQzu3CAR/IsBBWtG8O0SLRHNVOmgu3bdXuJca+3LKR +X3qFJbReqd9OI9vGxrxKg9NPUeGpzAuUTplmfs8O70g8l0ZBJ7l/0VmsA5lUDZxV +NjcacVYHc+ApujuLz5drVRKeLxTrtc6DksP7uruT5ZVrWbL35hoviwUilugz9ZDI +SQl0T6cIus8CyUjs8/iJEln9AnxAZ8Muz5+pQ3G/4BsabKlEHf0+GYinXxiTMT1V +Cr9botZ/AgMBAAEwDQYJKoZIhvcNAQELBQADgYEAnPW5NsSCI95ETnx9b1ZJv8wo +EIWqxzD7Wm9CIL8gUmVGrEJpHrsfpuXEN/jKqVFxUr3d2ZI40P8BrO73BGHo2F0M +ISDxiRSs7tR5xl5WmmErzK8TpDHU9fhkCqlblk7lzPLalIXCQ0y8qDjACBbh3tfj +0grbVmTy66u6TxI5FTE= +-----END CERTIFICATE----- diff --git a/tests/gold_tests/tls/ssl/signed-bob-foo.pem b/tests/gold_tests/tls/ssl/signed-bob-foo.pem new file mode 100644 index 00000000000..05db399165d --- /dev/null +++ b/tests/gold_tests/tls/ssl/signed-bob-foo.pem @@ -0,0 +1,18 @@ +-----BEGIN CERTIFICATE----- +MIIC2jCCAkMCCQC81MtBCwmQujANBgkqhkiG9w0BAQsFADCBnTELMAkGA1UEBhMC +VVMxCzAJBgNVBAgTAklMMRIwEAYDVQQHEwlDaGFtcGFpZ24xDjAMBgNVBAoTBVlh +aG9vMQ0wCwYDVQQLEwRFZGdlMSgwJgYDVQQDEx9qdWljZXByb2R1Y2UuY29ycC5u +ZTEueWFob28uY29tMSQwIgYJKoZIhvcNAQkBFhVwZXJzaWEuYXppekB5YWhvby5j +b20wHhcNMTgxMTIwMTQ1NTA4WhcNMjgxMTE3MTQ1NTA4WjBBMQswCQYDVQQGEwJV +UzELMAkGA1UECAwCSUwxDzANBgNVBAoMBkFwYWNoZTEUMBIGA1UEAwwLYm9iLmZv +by5jb20wggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDJheOsr0o4C0QV +elo8viYl/IjWvkj4Y6YojnO/E9T5HSfWxkRikRT8X7yqs9ClrjNcYoPNNQP5vgl1 +qAIeixUrbZZklcZ3kHF/MXdzuYV//uT3YfWrsOTdfbcu6adHtBr/Qc0w9GSbBpp4 +2ZALxzjZ2hZZq51XmKvcz55/gm/9YvlGErzvsPzkejdH6U1GKeLqGaZOqkV7N2J4 +zKmJyiPzwrmR/8y7k+9jDHQX+A2wa5gaiAMGdIM8aTV+XsNNAvespvXyX7UhwxDs +w0er0GLv4ssYCru5hF+tUcjeOCHxSnCoEC45b2zDZHATLLjr7AIJ52TcSo1kTaTi +rAOAiHRLAgMBAAEwDQYJKoZIhvcNAQELBQADgYEAvrrtNDsSbjVhwOlAm7MFwbxp +EUh1jYk07C6vxz7dFAcEXh33Sx4LsD0sI2TSbV5swGtPqMVTxbA+Ok6S/2lfxUeT +h7B6GrWvHtzQdPZbkoTEyehH5fOFLchbwGbecugYLjAK9wtatDZhFnuKRq8aiqml +GuDJPQLfKB9APLyxIbA= +-----END CERTIFICATE----- diff --git a/tests/gold_tests/tls/ssl/wild-signed.pem b/tests/gold_tests/tls/ssl/wild-signed.pem new file mode 100644 index 00000000000..d6f25ff7086 --- /dev/null +++ b/tests/gold_tests/tls/ssl/wild-signed.pem @@ -0,0 +1,18 @@ +-----BEGIN CERTIFICATE----- +MIIC2TCCAkICCQCTw2t3s0sIrjANBgkqhkiG9w0BAQsFADCBnTELMAkGA1UEBhMC +VVMxCzAJBgNVBAgTAklMMRIwEAYDVQQHEwlDaGFtcGFpZ24xDjAMBgNVBAoTBVlh +aG9vMQ0wCwYDVQQLEwRFZGdlMSgwJgYDVQQDEx9qdWljZXByb2R1Y2UuY29ycC5u +ZTEueWFob28uY29tMSQwIgYJKoZIhvcNAQkBFhVwZXJzaWEuYXppekB5YWhvby5j +b20wHhcNMTkwMzExMTQ1ODEzWhcNMjkwMzA4MTQ1ODEzWjBAMQswCQYDVQQGEwJV +UzELMAkGA1UECAwCSUwxDzANBgNVBAoMBmFwYWNoZTETMBEGA1UEAwwKKi53aWxk +LmNvbTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAM51MZvJ4anfZkoF +w+yLI+qtSamz/nOEeaNYvpJIzfH/qG04hxujCMfodknWoElBcJfzJ6I+RTDev4DZ +QFiDaetwQv4/+H17fTc2RJR1PdOKA06rRfm76P6CrlRttWFYLfcUTooTUoQUBFGu +EFTBGWUpFWU4fVXbYKLT63PzEkCeeHU5wf7Xz88pTYI21ZWnQQt2ll+zXLgSHFCP +m9XO5Q/Dr++XpFI2joCYj1GL/y3mku26sdK6ZYxRo9oseYiVX8K9T6Oeo4O32CcT +3ZE4QMa1jXcb5r4Y6VBdneZkur10CW2+yDT9MX5dKBeeuB9UUTYClt7lJdncl5DH +GdLrV9kCAwEAATANBgkqhkiG9w0BAQsFAAOBgQCRpZoeIRSF83Iiy7COW59KG9qq +T9pSCbJnqJr9ri1usmgcO9mfb7lQt94RXCtNz+tF85ciDE3zXcM6nJyosgStQEbb +AkGcW0z29Lc+Owc6SYtN6o4fEgPM61SSABtdG26jhpzMjT8MZNXluqXJ6tR0HWki +Rep/Bm0POzeUeKb8GA== +-----END CERTIFICATE----- diff --git a/tests/gold_tests/tls/ssl/wild.key b/tests/gold_tests/tls/ssl/wild.key new file mode 100644 index 00000000000..a7b57205e2b --- /dev/null +++ b/tests/gold_tests/tls/ssl/wild.key @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDOdTGbyeGp32ZK +BcPsiyPqrUmps/5zhHmjWL6SSM3x/6htOIcbowjH6HZJ1qBJQXCX8yeiPkUw3r+A +2UBYg2nrcEL+P/h9e303NkSUdT3TigNOq0X5u+j+gq5UbbVhWC33FE6KE1KEFARR +rhBUwRllKRVlOH1V22Ci0+tz8xJAnnh1OcH+18/PKU2CNtWVp0ELdpZfs1y4EhxQ +j5vVzuUPw6/vl6RSNo6AmI9Ri/8t5pLturHSumWMUaPaLHmIlV/CvU+jnqODt9gn +E92ROEDGtY13G+a+GOlQXZ3mZLq9dAltvsg0/TF+XSgXnrgfVFE2Apbe5SXZ3JeQ +xxnS61fZAgMBAAECggEARyRdFtjHGRkxDzrTW5RKqRhTdNXgTYANxjrTWGccCFLX +f+NlsyFH6lLxR9pcW0HggYu5UY/xmbh39vdl09pcylNh0mjKwLqn2DmsAhgwWM+K ++jXMpBSbYfA4EEHJqaSQGj72HWAxI/Ad2OOJHxt3G8O/aqS/k7FHqHQsA0V0Oa2D +NFLscCQxzH3IzUhbhZhtoCrtNfFkxSgx7RQbHMHNxMqaNU3MtKafQ1KJqR+SuuQL +donKPL6A0Xi8MKOc8qsyrGtKL/+EEY0W77SRJAkY8N29X1oMNPnGzLsicarwhHLX +1KT+rxBISJ/tbGuTvaJNbCepvO6I6QLZshQZt3MuAQKBgQDvbdSoEzfDO3vcbphH +QMGjASMkGP7T/goCvFQwUGN03GtMnoHuYbbfpZ6iOcYVGefU6v0LDOqhczTwKFVq +44msdQt/T1k9A+wHp6/1SnxObJT8wJoGmJO9qryc0yoRYt3lv20jl3GSC5xMjrJL +2Ko9DwNZaZNTlOFLJzzOOSUqwQKBgQDcvy7zupQivgK90gFId/iRjXz0UuVWseGo +osDL+BV4w4jTltLUE6AMVRQCqF85O1spCcn3AKjU+kWj0fIcB38y2UZRv6Moy/u8 +w5fRAhHONqNPwb9yOLNplLdm7lOE6mm+nYVgauh13nca8bxeNKmRZe7tyEu2i/4o +kZqTecvrGQKBgAm/QuUEw0Rja4txxSlBbaChLzkM+3LN6MJrwFGnNCVRw9x+p3N4 +7uTz7R1VlMbPIyz71AlbIUIpWoJcYf3T/YrTyQAJzuw4+KbnILavrZfTu8z+Wkbi +d0FFbiBESHYkvDvaKytDww/bASXsuT11OJj7v3soXSMN8I4KruMGWIkBAoGBAMQl +GNo2ylQIlDUIul0jRPpIR2RtmBytmH6Yh0l2GdYhoJ2qIZGSEp+CpXIrG9ml1T2k +1hGlQ19jNqf27/NZ8ftDtskCyD6C6h9ziJ2OAjZCtGA1HyCmIz1IiKJsWEf9ZpKa +Mx5WQFIjp5+IdsEaeCWa9m/Qjv4YbHCt2DT8f2ZZAoGBALzd1ju4Sf5k+eP4QDO9 +jWRRRyjHIrT8RX4Hb3YZWJmW8edxjlV63GKVyuwv+PNyaP0hrLpVEw7h9tAYaIoL +DAjqph02ujDBe5oJrOMrO3IkxVqHj2PQFBnFR0MCbkQ9/+vyTt7ftmwXwQQFeDVW +YcOEYXCqeaAzjN9t0xidlkX7 +-----END PRIVATE KEY----- diff --git a/tests/gold_tests/tls/test-nc-s_client.sh b/tests/gold_tests/tls/test-nc-s_client.sh new file mode 100644 index 00000000000..a11a80c9329 --- /dev/null +++ b/tests/gold_tests/tls/test-nc-s_client.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +nc -l -p $1 -c 'echo -e "This is a reply"' -o test.out & +echo "This is a test" | openssl s_client -servername bar.com -connect localhost:$2 -ign_eof diff --git a/tests/gold_tests/tls/tls.test.py b/tests/gold_tests/tls/tls.test.py index f1ef8cdc7ed..4f02cbf1a6f 100644 --- a/tests/gold_tests/tls/tls.test.py +++ b/tests/gold_tests/tls/tls.test.py @@ -40,10 +40,11 @@ testName = "" header_count = 378 +#header_count = 78 header_string = "POST /post HTTP/1.1\r\nHost: www.example.com\r\nContent-Length:1000\r\n" -for i in range(0, 378): +for i in range(0, header_count): header_string = "{1}header{0}:{0}\r\n".format(i, header_string) header_string = "{0}\r\n".format(header_string) @@ -74,16 +75,15 @@ # enable ssl port 'proxy.config.http.server_ports': '{0} {1}:proto=http2;http:ssl'.format(ts.Variables.port, ts.Variables.ssl_port), 'proxy.config.ssl.client.verify.server': 0, + 'proxy.config.exec_thread.autoconfig.scale': 1.0, 'proxy.config.ssl.server.cipher_suite': 'ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA384:AES128-GCM-SHA256:AES256-GCM-SHA384:ECDHE-RSA-RC4-SHA:ECDHE-RSA-AES128-SHA:ECDHE-RSA-AES256-SHA:RC4-SHA:RC4-MD5:AES128-SHA:AES256-SHA:DES-CBC3-SHA!SRP:!DSS:!PSK:!aNULL:!eNULL:!SSLv2', }) tr = Test.AddTestRun("Run-Test") -tr.Command = './ssl-post 127.0.0.1 40 378 4443' +tr.Command = './ssl-post 127.0.0.1 40 {0} 4443'.format(header_count) tr.ReturnCode = 0 # time delay as proxy.config.http.wait_for_cache could be broken tr.Processes.Default.StartBefore(server) tr.Processes.Default.StartBefore(Test.Processes.ts, ready=When.PortOpen(ts.Variables.ssl_port)) tr.Processes.Default.Streams.stdout = "gold/ssl-post.gold" tr.StillRunningAfter = server -tr.Processes.Default.TimeOut = 5 -tr.TimeOut = 5 diff --git a/tests/gold_tests/tls/tls_check_cert_selection.test.py b/tests/gold_tests/tls/tls_check_cert_selection.test.py new file mode 100644 index 00000000000..5f2174bd26e --- /dev/null +++ b/tests/gold_tests/tls/tls_check_cert_selection.test.py @@ -0,0 +1,125 @@ +''' +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +Test.Summary = ''' +Test ATS offering different certificates based on SNI +''' + +# need Curl +Test.SkipUnless( + Condition.HasProgram("curl", "Curl need to be installed on system for this test to work") +) + +# Define default ATS +ts = Test.MakeATSProcess("ts", select_ports=False) +server = Test.MakeOriginServer("server", ssl=True) +dns = Test.MakeDNServer("dns") + +request_header = {"headers": "GET / HTTP/1.1\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +response_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +server.addResponse("sessionlog.json", request_header, response_header) + +# add ssl materials like key, certificates for the server +ts.addSSLfile("ssl/signed-foo.pem") +ts.addSSLfile("ssl/signed-foo.key") +ts.addSSLfile("ssl/signed-bar.pem") +ts.addSSLfile("ssl/signed2-bar.pem") +ts.addSSLfile("ssl/signed-bar.key") +ts.addSSLfile("ssl/server.pem") +ts.addSSLfile("ssl/server.key") +ts.addSSLfile("ssl/signer.pem") +ts.addSSLfile("ssl/signer.key") + +ts.Variables.ssl_port = 4443 +ts.Disk.remap_config.AddLine( + 'map / https://foo.com:{1}'.format(ts.Variables.ssl_port, server.Variables.SSL_Port)) + +ts.Disk.ssl_multicert_config.AddLines([ + 'dest_ip=127.0.0.1 ssl_cert_name=signed-foo.pem ssl_key_name=signed-foo.key', + 'ssl_cert_name=signed2-bar.pem ssl_key_name=signed-bar.key', + 'dest_ip=* ssl_cert_name=server.pem ssl_key_name=server.key' +]) + +# Case 1, global config policy=permissive properties=signature +# override for foo.com policy=enforced properties=all +ts.Disk.records_config.update({ + 'proxy.config.ssl.server.cert.path': '{0}'.format(ts.Variables.SSLDir), + 'proxy.config.ssl.server.private_key.path': '{0}'.format(ts.Variables.SSLDir), + 'proxy.config.http.server_ports': '{0} {1}:proto=http2;http:ssl'.format(ts.Variables.port, ts.Variables.ssl_port), + 'proxy.config.ssl.server.cipher_suite': 'ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA384:AES128-GCM-SHA256:AES256-GCM-SHA384:ECDHE-RSA-RC4-SHA:ECDHE-RSA-AES128-SHA:ECDHE-RSA-AES256-SHA:RC4-SHA:RC4-MD5:AES128-SHA:AES256-SHA:DES-CBC3-SHA!SRP:!DSS:!PSK:!aNULL:!eNULL:!SSLv2', + 'proxy.config.url_remap.pristine_host_hdr': 1, + 'proxy.config.dns.nameservers': '127.0.0.1:{0}'.format(dns.Variables.Port), + 'proxy.config.exec_thread.autoconfig.scale': 1.0, + 'proxy.config.dns.resolv_conf': 'NULL' +}) + +dns.addRecords(records={"foo.com.": ["127.0.0.1"]}) +dns.addRecords(records={"bar.com.": ["127.0.0.1"]}) + +# Should receive a bar.com cert +tr = Test.AddTestRun("bar.com cert") +tr.Setup.Copy("ssl/signer.pem") +tr.Setup.Copy("ssl/signer2.pem") +tr.Processes.Default.Command = "curl -v --cacert ./signer2.pem --resolve 'bar.com:{0}:127.0.0.1' https://bar.com:{0}".format(ts.Variables.ssl_port) +tr.ReturnCode = 0 +tr.Processes.Default.StartBefore(server) +tr.Processes.Default.StartBefore(dns) +tr.Processes.Default.StartBefore(Test.Processes.ts, ready=When.PortOpen(ts.Variables.ssl_port)) +tr.StillRunningAfter = server +tr.StillRunningAfter = ts +tr.Processes.Default.Streams.All = Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") +tr.Processes.Default.Streams.All += Testers.ContainsExpression("CN=bar.com", "Cert should contain bar.com") +tr.Processes.Default.Streams.All += Testers.ExcludesExpression("CN=foo.com", "Cert should not contain foo.com") +tr.Processes.Default.Streams.All += Testers.ContainsExpression("404", "Should make an exchange") + +# Should receive a foo.com cert +tr2 = Test.AddTestRun("foo.com cert") +tr2.Processes.Default.Command = "curl -v --cacert ./signer.pem --resolve 'foo.com:{0}:127.0.0.1' https://foo.com:{0}".format(ts.Variables.ssl_port) +tr2.ReturnCode = 0 +tr2.StillRunningAfter = server +tr2.StillRunningAfter = ts +tr2.Processes.Default.Streams.All = Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") +tr2.Processes.Default.Streams.All += Testers.ContainsExpression("CN=foo.com", "Cert should contain foo.com") +tr2.Processes.Default.Streams.All += Testers.ExcludesExpression("CN=bar.com", "Cert should not contain bar.com") +tr.Processes.Default.Streams.All += Testers.ContainsExpression("404", "Should make an exchange") + +# Should receive random.server.com +tr2 = Test.AddTestRun("random.server.com cert") +tr2.Processes.Default.Command = "curl -v -k --resolve 'random.server.com:{0}:127.0.0.1' https://random.server.com:{0}".format(ts.Variables.ssl_port) +tr2.ReturnCode = 0 +tr2.StillRunningAfter = server +tr2.StillRunningAfter = ts +tr2.Processes.Default.Streams.All = Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") +tr2.Processes.Default.Streams.All += Testers.ContainsExpression("CN=random.server.com", "Cert should contain random.server.com") +tr2.Processes.Default.Streams.All += Testers.ExcludesExpression("CN=foo.com", "Cert should not contain foo.com") +tr2.Processes.Default.Streams.All += Testers.ExcludesExpression("CN=bar.com", "Cert should not contain bar.com") +tr.Processes.Default.Streams.All += Testers.ContainsExpression("404", "Should make an exchange") + +# No SNI match should match specific IP address, foo.com +# SNI name and returned cert name will not match, so must use -k to avoid cert verification +tr2 = Test.AddTestRun("Bad SNI") +tr2.Processes.Default.Command = "curl -v -k --cacert ./signer.pem --resolve 'bad.sni.com:{0}:127.0.0.1' https://bad.sni.com:{0}".format(ts.Variables.ssl_port) +tr2.ReturnCode = 0 +tr2.StillRunningAfter = server +tr2.StillRunningAfter = ts +tr2.Processes.Default.Streams.All = Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") +tr2.Processes.Default.Streams.All += Testers.ContainsExpression("CN=foo.com", "Cert should contain foo.com") +tr2.Processes.Default.Streams.All += Testers.ExcludesExpression("CN=bar.com", "Cert should not contain bar.com") +tr.Processes.Default.Streams.All += Testers.ContainsExpression("404", "Should make an exchange") + diff --git a/tests/gold_tests/tls/tls_client_cert.test.py b/tests/gold_tests/tls/tls_client_cert.test.py index 1973031224d..6c6bcc8e3e7 100644 --- a/tests/gold_tests/tls/tls_client_cert.test.py +++ b/tests/gold_tests/tls/tls_client_cert.test.py @@ -18,6 +18,7 @@ # limitations under the License. import os +import subprocess import re Test.Summary = ''' @@ -29,8 +30,10 @@ ts = Test.MakeATSProcess("ts", command="traffic_manager", select_ports=False) cafile = "{0}/signer.pem".format(Test.RunDirectory) cafile2 = "{0}/signer2.pem".format(Test.RunDirectory) -server = Test.MakeOriginServer("server", ssl=True, options = { "--clientCA": cafile, "--clientverify": "true"}, clientcert="{0}/signed-foo.pem".format(Test.RunDirectory), clientkey="{0}/signed-foo.key".format(Test.RunDirectory)) -server2 = Test.MakeOriginServer("server2", ssl=True, options = { "--clientCA": cafile2, "--clientverify": "true"}, clientcert="{0}/signed2-bar.pem".format(Test.RunDirectory), clientkey="{0}/signed-bar.key".format(Test.RunDirectory)) +# --clientverify: "" empty string because microserver does store_true for argparse, but options is a dictionary +server = Test.MakeOriginServer("server", ssl=True, options = { "--clientCA": cafile, "--clientverify": ""}, clientcert="{0}/signed-foo.pem".format(Test.RunDirectory), clientkey="{0}/signed-foo.key".format(Test.RunDirectory)) +server2 = Test.MakeOriginServer("server2", ssl=True, options = { "--clientCA": cafile2, "--clientverify": ""}, clientcert="{0}/signed2-bar.pem".format(Test.RunDirectory), clientkey="{0}/signed-bar.key".format(Test.RunDirectory)) +server3 = Test.MakeOriginServer("server3") server.Setup.Copy("ssl/signer.pem") server.Setup.Copy("ssl/signer2.pem") server.Setup.Copy("ssl/signed-foo.pem") @@ -65,7 +68,7 @@ ts.Variables.ssl_port = 4443 ts.Disk.records_config.update({ 'proxy.config.diags.debug.enabled': 1, - 'proxy.config.diags.debug.tags': 'ssl|http', + 'proxy.config.diags.debug.tags': 'ssl_verify_test', 'proxy.config.ssl.server.cert.path': '{0}'.format(ts.Variables.SSLDir), 'proxy.config.ssl.server.private_key.path': '{0}'.format(ts.Variables.SSLDir), 'proxy.config.http.server_ports': '{0}'.format(ts.Variables.port), @@ -75,6 +78,7 @@ 'proxy.config.ssl.client.cert.filename': 'signed-foo.pem', 'proxy.config.ssl.client.private_key.path': '{0}'.format(ts.Variables.SSLDir), 'proxy.config.ssl.client.private_key.filename': 'signed-foo.key', + 'proxy.config.exec_thread.autoconfig.scale': 1.0, 'proxy.config.url_remap.pristine_host_hdr' : 1, }) @@ -83,10 +87,10 @@ ) ts.Disk.remap_config.AddLine( - 'map /case1 https://127.0.0.1:{0}/'.format(server.Variables.Port) + 'map /case1 https://127.0.0.1:{0}/'.format(server.Variables.SSL_Port) ) ts.Disk.remap_config.AddLine( - 'map /case2 https://127.0.0.1:{0}/'.format(server2.Variables.Port) + 'map /case2 https://127.0.0.1:{0}/'.format(server2.Variables.SSL_Port) ) ts.Disk.ssl_server_name_yaml.AddLine( @@ -107,9 +111,7 @@ tr.StillRunningAfter = server2 tr.Processes.Default.Command = "curl -H host:example.com http://127.0.0.1:{0}/case1".format(ts.Variables.port) tr.Processes.Default.ReturnCode = 0 -tr.Processes.Default.TimeOut = 5 tr.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could Not Connect", "Check response") -tr.TimeOut = 5 #Should fail trfail = Test.AddTestRun("Connect with first client cert to second server") @@ -118,9 +120,7 @@ trfail.StillRunningAfter = server2 trfail.Processes.Default.Command = 'curl -H host:example.com http://127.0.0.1:{0}/case2'.format(ts.Variables.port) trfail.Processes.Default.ReturnCode = 0 -trfail.Processes.Default.TimeOut = 5 trfail.Processes.Default.Streams.stdout = Testers.ContainsExpression("Could Not Connect", "Check response") -trfail.TimeOut = 5 # Should succeed trbar = Test.AddTestRun("Connect with signed2 bar to second server") @@ -129,9 +129,7 @@ trbar.StillRunningAfter = server2 trbar.Processes.Default.Command = "curl -H host:bar.com http://127.0.0.1:{0}/case2".format(ts.Variables.port) trbar.Processes.Default.ReturnCode = 0 -trbar.Processes.Default.TimeOut = 5 trbar.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could Not Connect", "Check response") -trbar.TimeOut = 5 #Should fail trbarfail = Test.AddTestRun("Connect with signed2 bar cert to first server") @@ -140,9 +138,7 @@ trbarfail.StillRunningAfter = server2 trbarfail.Processes.Default.Command = 'curl -H host:bar.com http://127.0.0.1:{0}/case1'.format(ts.Variables.port) trbarfail.Processes.Default.ReturnCode = 0 -trbarfail.Processes.Default.TimeOut = 5 trbarfail.Processes.Default.Streams.stdout = Testers.ContainsExpression("Could Not Connect", "Check response") -trbarfail.TimeOut = 5 tr2 = Test.AddTestRun("Update config files") # Update the SNI config @@ -158,8 +154,6 @@ # recreate the records.config with the cert filename changed tr2.Disk.File(recordspath, id = "records_config", typename="ats:config:records"), tr2.Disk.records_config.update({ - 'proxy.config.diags.debug.enabled': 1, - 'proxy.config.diags.debug.tags': 'ssl|http', 'proxy.config.ssl.server.cert.path': '{0}'.format(ts.Variables.SSLDir), 'proxy.config.ssl.server.private_key.path': '{0}'.format(ts.Variables.SSLDir), 'proxy.config.http.server_ports': '{0}'.format(ts.Variables.port), @@ -178,8 +172,20 @@ # Need to copy over the environment so traffic_ctl knows where to find the unix domain socket tr2.Processes.Default.Env = ts.Env tr2.Processes.Default.ReturnCode = 0 -tr2.Processes.Default.TimeOut = 5 -tr2.TimeOut = 5 + +# Parking this as a ready tester on a meaningless process +# Stall the test runs until the ssl_server_name reload has completed +# At that point the new ssl_server_name settings are ready to go +def ssl_server_name_reload_done(tsenv): + def done_reload(process, hasRunFor, **kw): + cmd = "grep 'ssl_server_name.yaml finished loading' {0} | wc -l > {1}/test.out".format(ts.Disk.diags_log.Name, Test.RunDirectory) + retval = subprocess.run(cmd, shell=True, env=tsenv) + if retval.returncode == 0: + cmd ="if [ -f {0}/test.out -a \"`cat {0}/test.out`\" = \"2\" ] ; then true; else false; fi".format(Test.RunDirectory) + retval = subprocess.run(cmd, shell = True, env=tsenv) + return retval.returncode == 0 + + return done_reload tr2reload = Test.AddTestRun("Reload config") tr2reload.StillRunningAfter = ts @@ -189,22 +195,18 @@ # Need to copy over the environment so traffic_ctl knows where to find the unix domain socket tr2reload.Processes.Default.Env = ts.Env tr2reload.Processes.Default.ReturnCode = 0 -tr2reload.Processes.Default.TimeOut = 5 -tr2reload.TimeOut = 5 #Should succeed tr3bar = Test.AddTestRun("Make request with other bar cert to first server") # Wait for the reload to complete -tr3bar.DelayStart = 10 +tr3bar.Processes.Default.StartBefore(server3, ready=ssl_server_name_reload_done(ts.Env)) tr3bar.StillRunningAfter = ts tr3bar.StillRunningAfter = server tr3bar.StillRunningAfter = server2 tr3bar.Processes.Default.Command = 'curl -H host:bar.com http://127.0.0.1:{0}/case1'.format(ts.Variables.port) tr3bar.Processes.Default.ReturnCode = 0 -tr3bar.Processes.Default.TimeOut = 5 tr3bar.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could Not Connect", "Check response") -tr3bar.TimeOut = 5 #Should fail tr3barfail = Test.AddTestRun("Make request with other bar cert to second server") @@ -213,9 +215,7 @@ tr3barfail.StillRunningAfter = server2 tr3barfail.Processes.Default.Command = 'curl -H host:bar.com http://127.0.0.1:{0}/case2'.format(ts.Variables.port) tr3barfail.Processes.Default.ReturnCode = 0 -tr3barfail.Processes.Default.TimeOut = 5 tr3barfail.Processes.Default.Streams.stdout = Testers.ContainsExpression("Could Not Connect", "Check response") -tr3barfail.TimeOut = 5 #Should succeed tr3 = Test.AddTestRun("Make request with other cert to second server") @@ -225,9 +225,7 @@ tr3.StillRunningAfter = server2 tr3.Processes.Default.Command = 'curl -H host:example.com http://127.0.0.1:{0}/case2'.format(ts.Variables.port) tr3.Processes.Default.ReturnCode = 0 -tr3.Processes.Default.TimeOut = 5 tr3.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could Not Connect", "Check response") -tr3.TimeOut = 5 #Should fail tr3fail = Test.AddTestRun("Make request with other cert to first server") @@ -236,9 +234,7 @@ tr3fail.StillRunningAfter = server2 tr3fail.Processes.Default.Command = 'curl -H host:example.com http://127.0.0.1:{0}/case1'.format(ts.Variables.port) tr3fail.Processes.Default.ReturnCode = 0 -tr3fail.Processes.Default.TimeOut = 5 tr3fail.Processes.Default.Streams.stdout = Testers.ContainsExpression("Could Not Connect", "Check response") -tr3fail.TimeOut = 5 # Test the case of updating certificate contents without changing file name. @@ -254,7 +250,6 @@ # Need to copy over the environment so traffic_ctl knows where to find the unix domain socket trupdate.Processes.Default.Env = ts.Env trupdate.Processes.Default.ReturnCode = 0 -trupdate.Processes.Default.TimeOut = 5 trreload = Test.AddTestRun("Reload config after renaming certs") trreload.StillRunningAfter = ts @@ -263,7 +258,6 @@ trreload.Processes.Default.Command = 'traffic_ctl config reload' trreload.Processes.Default.Env = ts.Env trreload.Processes.Default.ReturnCode = 0 -trreload.Processes.Default.TimeOut = 5 #Should succeed tr4bar = Test.AddTestRun("Make request with renamed bar cert to second server") @@ -274,9 +268,7 @@ tr4bar.StillRunningAfter = server2 tr4bar.Processes.Default.Command = 'curl -H host:bar.com http://127.0.0.1:{0}/case2'.format(ts.Variables.port) tr4bar.Processes.Default.ReturnCode = 0 -tr4bar.Processes.Default.TimeOut = 5 tr4bar.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could Not Connect", "Check response") -tr4bar.TimeOut = 5 #Should fail tr4barfail = Test.AddTestRun("Make request with renamed bar cert to first server") @@ -285,7 +277,6 @@ tr4barfail.StillRunningAfter = server2 tr4barfail.Processes.Default.Command = 'curl -H host:bar.com http://127.0.0.1:{0}/case1'.format(ts.Variables.port) tr4barfail.Processes.Default.ReturnCode = 0 -tr4barfail.Processes.Default.TimeOut = 5 tr4barfail.Processes.Default.Streams.stdout = Testers.ContainsExpression("Could Not Connect", "Check response") #Should succeed @@ -295,9 +286,7 @@ tr4.StillRunningAfter = server2 tr4.Processes.Default.Command = 'curl -H host:example.com http://127.0.0.1:{0}/case1'.format(ts.Variables.port) tr4.Processes.Default.ReturnCode = 0 -tr4.Processes.Default.TimeOut = 5 tr4.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could Not Connect", "Check response") -tr4.TimeOut = 5 #Should fail tr4fail = Test.AddTestRun("Make request with renamed foo cert to second server") @@ -306,9 +295,5 @@ tr4fail.StillRunningAfter = server2 tr4fail.Processes.Default.Command = 'curl -H host:example.com http://127.0.0.1:{0}/case2'.format(ts.Variables.port) tr4fail.Processes.Default.ReturnCode = 0 -tr4fail.Processes.Default.TimeOut = 5 tr4fail.Processes.Default.Streams.stdout = Testers.ContainsExpression("Could Not Connect", "Check response") -tr4fail.TimeOut = 5 -# Allow for error messages in diags -ts.Disk.diags_log.Content = Testers.ContainsExpression("ERROR", "Some connections should fail") diff --git a/tests/gold_tests/tls/tls_client_cert2.test.py b/tests/gold_tests/tls/tls_client_cert2.test.py new file mode 100644 index 00000000000..125e9690418 --- /dev/null +++ b/tests/gold_tests/tls/tls_client_cert2.test.py @@ -0,0 +1,179 @@ +''' +Test offering client cert to origin +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import re + +Test.Summary = ''' +Test client certs to origin selected via wildcard names in ssl_server_name +''' + +Test.SkipUnless(Condition.HasProgram("grep", "grep needs to be installed on system for this test to work")) + +ts = Test.MakeATSProcess("ts", command="traffic_server", select_ports=False) +cafile = "{0}/signer.pem".format(Test.RunDirectory) +cafile2 = "{0}/signer2.pem".format(Test.RunDirectory) +server = Test.MakeOriginServer("server", ssl=True, options = { "--clientCA": cafile, "--clientverify": ""}, clientcert="{0}/signed-foo.pem".format(Test.RunDirectory), clientkey="{0}/signed-foo.key".format(Test.RunDirectory)) +server2 = Test.MakeOriginServer("server2", ssl=True, options = { "--clientCA": cafile2, "--clientverify": ""}, clientcert="{0}/signed2-bar.pem".format(Test.RunDirectory), clientkey="{0}/signed-bar.key".format(Test.RunDirectory)) +server.Setup.Copy("ssl/signer.pem") +server.Setup.Copy("ssl/signer2.pem") +server.Setup.Copy("ssl/signed-foo.pem") +server.Setup.Copy("ssl/signed-foo.key") +server.Setup.Copy("ssl/signed2-foo.pem") +server.Setup.Copy("ssl/signed2-bar.pem") +server.Setup.Copy("ssl/signed-bar.key") +server2.Setup.Copy("ssl/signer.pem") +server2.Setup.Copy("ssl/signer2.pem") +server2.Setup.Copy("ssl/signed-foo.pem") +server2.Setup.Copy("ssl/signed-foo.key") +server2.Setup.Copy("ssl/signed2-foo.pem") +server2.Setup.Copy("ssl/signed2-bar.pem") +server2.Setup.Copy("ssl/signed-bar.key") + +request_header = {"headers": "GET / HTTP/1.1\r\nHost: example.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +response_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +server.addResponse("sessionlog.json", request_header, response_header) +request_header = {"headers": "GET / HTTP/1.1\r\nHost: bar.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +response_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +server.addResponse("sessionlog.json", request_header, response_header) + +ts.addSSLfile("ssl/server.pem") +ts.addSSLfile("ssl/server.key") +ts.addSSLfile("ssl/combo-signed-foo.pem") +ts.addSSLfile("ssl/signed-foo.pem") +ts.addSSLfile("ssl/signed-foo.key") +ts.addSSLfile("ssl/signed2-foo.pem") +ts.addSSLfile("ssl/signed-bar.pem") +ts.addSSLfile("ssl/signed2-bar.pem") +ts.addSSLfile("ssl/signed-bar.key") + +ts.Variables.ssl_port = 4443 +ts.Disk.records_config.update({ + 'proxy.config.ssl.server.cert.path': '{0}'.format(ts.Variables.SSLDir), + 'proxy.config.ssl.server.private_key.path': '{0}'.format(ts.Variables.SSLDir), + 'proxy.config.ssl.client.cert.path': '{0}'.format(ts.Variables.SSLDir), + 'proxy.config.ssl.client.private_key.path': '{0}'.format(ts.Variables.SSLDir), + 'proxy.config.http.server_ports': '{0}'.format(ts.Variables.port), + 'proxy.config.ssl.client.verify.server': 0, + 'proxy.config.ssl.server.cipher_suite': 'ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA384:AES128-GCM-SHA256:AES256-GCM-SHA384:ECDHE-RSA-RC4-SHA:ECDHE-RSA-AES128-SHA:ECDHE-RSA-AES256-SHA:RC4-SHA:RC4-MD5:AES128-SHA:AES256-SHA:DES-CBC3-SHA!SRP:!DSS:!PSK:!aNULL:!eNULL:!SSLv2', + 'proxy.config.exec_thread.autoconfig.scale': 1.0, + 'proxy.config.url_remap.pristine_host_hdr' : 1, +}) + +ts.Disk.ssl_multicert_config.AddLine( + 'dest_ip=* ssl_cert_name=server.pem ssl_key_name=server.key' +) + +ts.Disk.remap_config.AddLine( + 'map /case1 https://127.0.0.1:{0}/'.format(server.Variables.SSL_Port) +) +ts.Disk.remap_config.AddLine( + 'map /case2 https://127.0.0.1:{0}/'.format(server2.Variables.SSL_Port) +) + +ts.Disk.ssl_server_name_yaml.AddLines([ + '- fqdn: bob.bar.com', + ' client_cert: signed-bar.pem', + ' client_key: signed-bar.key', + '- fqdn: bob.*.com', + ' client_cert: {0}/combo-signed-foo.pem'.format(ts.Variables.SSLDir), + '- fqdn: "*bar.com"', + ' client_cert: {0}/signed2-bar.pem'.format(ts.Variables.SSLDir), + ' client_key: {0}/signed-bar.key'.format(ts.Variables.SSLDir), + '- fqdn: "foo.com"', + ' client_cert: {0}/signed2-foo.pem'.format(ts.Variables.SSLDir), + ' client_key: {0}/signed-foo.key'.format(ts.Variables.SSLDir), +]) + + +# Should succeed +tr = Test.AddTestRun("bob.bar.com to server 1") +tr.Processes.Default.StartBefore(Test.Processes.ts, ready=When.PortOpen(ts.Variables.port)) +tr.Processes.Default.StartBefore(server) +tr.Processes.Default.StartBefore(server2) +tr.StillRunningAfter = ts +tr.StillRunningAfter = server +tr.StillRunningAfter = server2 +tr.Processes.Default.Command = "curl -H host:bob.bar.com http://127.0.0.1:{0}/case1".format(ts.Variables.port) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could Not Connect", "Check response") + +#Should fail +trfail = Test.AddTestRun("bob.bar.com to server 2") +trfail.StillRunningAfter = ts +trfail.StillRunningAfter = server +trfail.StillRunningAfter = server2 +trfail.Processes.Default.Command = 'curl -H host:bob.bar.com http://127.0.0.1:{0}/case2'.format(ts.Variables.port) +trfail.Processes.Default.ReturnCode = 0 +trfail.Processes.Default.Streams.stdout = Testers.ContainsExpression("Could Not Connect", "Check response") + +# Should succeed +tr = Test.AddTestRun("bob.foo.com to server 1") +tr.StillRunningAfter = ts +tr.StillRunningAfter = server +tr.StillRunningAfter = server2 +tr.Processes.Default.Command = "curl -H host:bob.foo.com http://127.0.0.1:{0}/case1".format(ts.Variables.port) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could Not Connect", "Check response") + +#Should fail +trfail = Test.AddTestRun("bob.foo.com to server 2") +trfail.StillRunningAfter = ts +trfail.StillRunningAfter = server +trfail.StillRunningAfter = server2 +trfail.Processes.Default.Command = 'curl -H host:bob.foo.com http://127.0.0.1:{0}/case2'.format(ts.Variables.port) +trfail.Processes.Default.ReturnCode = 0 +trfail.Processes.Default.Streams.stdout = Testers.ContainsExpression("Could Not Connect", "Check response") + +# Should succeed +tr = Test.AddTestRun("random.bar.com to server 2") +tr.StillRunningAfter = ts +tr.StillRunningAfter = server +tr.StillRunningAfter = server2 +tr.Processes.Default.Command = "curl -H host:random.bar.com http://127.0.0.1:{0}/case2".format(ts.Variables.port) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could Not Connect", "Check response") + +#Should fail +trfail = Test.AddTestRun("random.bar.com to server 1") +trfail.StillRunningAfter = ts +trfail.StillRunningAfter = server +trfail.StillRunningAfter = server2 +trfail.Processes.Default.Command = 'curl -H host:random.bar.com http://127.0.0.1:{0}/case1'.format(ts.Variables.port) +trfail.Processes.Default.ReturnCode = 0 +trfail.Processes.Default.Streams.stdout = Testers.ContainsExpression("Could Not Connect", "Check response") + +# Should fail +tr = Test.AddTestRun("random.foo.com to server 2") +tr.StillRunningAfter = ts +tr.StillRunningAfter = server +tr.StillRunningAfter = server2 +tr.Processes.Default.Command = "curl -H host:random.foo.com http://127.0.0.1:{0}/case2".format(ts.Variables.port) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Streams.stdout = Testers.ContainsExpression("Could Not Connect", "Check response") + +#Should fail +trfail = Test.AddTestRun("random.foo.com to server 1") +trfail.StillRunningAfter = ts +trfail.StillRunningAfter = server +trfail.StillRunningAfter = server2 +trfail.Processes.Default.Command = 'curl -H host:random.foo.com http://127.0.0.1:{0}/case1'.format(ts.Variables.port) +trfail.Processes.Default.ReturnCode = 0 +trfail.Processes.Default.Streams.stdout = Testers.ContainsExpression("Could Not Connect", "Check response") + diff --git a/tests/gold_tests/tls/tls_client_cert_override.test.py b/tests/gold_tests/tls/tls_client_cert_override.test.py new file mode 100644 index 00000000000..e2fe10dd577 --- /dev/null +++ b/tests/gold_tests/tls/tls_client_cert_override.test.py @@ -0,0 +1,136 @@ +''' +Test offering client cert to origin +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import re + +Test.Summary = ''' +Test conf_remp to specify different client certificates to offer to the origin +''' + +Test.SkipUnless(Condition.HasProgram("grep", "grep needs to be installed on system for this test to work")) + +ts = Test.MakeATSProcess("ts", command="traffic_manager", select_ports=False) +cafile = "{0}/signer.pem".format(Test.RunDirectory) +cafile2 = "{0}/signer2.pem".format(Test.RunDirectory) +server = Test.MakeOriginServer("server", ssl=True, options = { "--clientCA": cafile, "--clientverify": ""}, clientcert="{0}/signed-foo.pem".format(Test.RunDirectory), clientkey="{0}/signed-foo.key".format(Test.RunDirectory)) +server2 = Test.MakeOriginServer("server2", ssl=True, options = { "--clientCA": cafile2, "--clientverify": ""}, clientcert="{0}/signed2-bar.pem".format(Test.RunDirectory), clientkey="{0}/signed-bar.key".format(Test.RunDirectory)) +server.Setup.Copy("ssl/signer.pem") +server.Setup.Copy("ssl/signer2.pem") +server.Setup.Copy("ssl/signed-foo.pem") +server.Setup.Copy("ssl/signed-foo.key") +server.Setup.Copy("ssl/signed2-foo.pem") +server.Setup.Copy("ssl/signed2-bar.pem") +server.Setup.Copy("ssl/signed-bar.key") +server2.Setup.Copy("ssl/signer.pem") +server2.Setup.Copy("ssl/signer2.pem") +server2.Setup.Copy("ssl/signed-foo.pem") +server2.Setup.Copy("ssl/signed-foo.key") +server2.Setup.Copy("ssl/signed2-foo.pem") +server2.Setup.Copy("ssl/signed2-bar.pem") +server2.Setup.Copy("ssl/signed-bar.key") + +request_header = {"headers": "GET / HTTP/1.1\r\nHost: example.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +response_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +server.addResponse("sessionlog.json", request_header, response_header) +request_header = {"headers": "GET / HTTP/1.1\r\nHost: bar.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +response_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +server.addResponse("sessionlog.json", request_header, response_header) + +ts.addSSLfile("ssl/server.pem") +ts.addSSLfile("ssl/server.key") +ts.addSSLfile("ssl/signed-foo.pem") +ts.addSSLfile("ssl/signed-foo.key") +ts.addSSLfile("ssl/signed2-foo.pem") +ts.addSSLfile("ssl/signed-bar.pem") +ts.addSSLfile("ssl/signed2-bar.pem") +ts.addSSLfile("ssl/signed-bar.key") + +ts.Variables.ssl_port = 4443 +ts.Disk.records_config.update({ + 'proxy.config.ssl.server.cert.path': '{0}'.format(ts.Variables.SSLDir), + 'proxy.config.ssl.server.private_key.path': '{0}'.format(ts.Variables.SSLDir), + 'proxy.config.http.server_ports': '{0}'.format(ts.Variables.port), + 'proxy.config.ssl.client.verify.server': 0, + 'proxy.config.ssl.server.cipher_suite': 'ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA384:AES128-GCM-SHA256:AES256-GCM-SHA384:ECDHE-RSA-RC4-SHA:ECDHE-RSA-AES128-SHA:ECDHE-RSA-AES256-SHA:RC4-SHA:RC4-MD5:AES128-SHA:AES256-SHA:DES-CBC3-SHA!SRP:!DSS:!PSK:!aNULL:!eNULL:!SSLv2', + 'proxy.config.ssl.client.cert.path': '{0}'.format(ts.Variables.SSLDir), + 'proxy.config.ssl.client.cert.filename': 'signed-foo.pem', + 'proxy.config.ssl.client.private_key.path': '{0}'.format(ts.Variables.SSLDir), + 'proxy.config.ssl.client.private_key.filename': 'signed-foo.key', + 'proxy.config.exec_thread.autoconfig.scale': 1.0, + 'proxy.config.url_remap.pristine_host_hdr' : 1, +}) + +ts.Disk.ssl_multicert_config.AddLine( + 'dest_ip=* ssl_cert_name=server.pem ssl_key_name=server.key' +) + +ts.Disk.remap_config.AddLine( + 'map /case1 https://127.0.0.1:{0}/ @plugin=conf_remap.so @pparam=proxy.config.ssl.client.cert.filename={1} plugin=conf_remap.so @pparam=proxy.config.ssl.client.private_key.filename={2}'.format(server.Variables.SSL_Port, "signed-foo.pem", "signed-foo.key") +) +ts.Disk.remap_config.AddLine( + 'map /badcase1 https://127.0.0.1:{0}/ @plugin=conf_remap.so @pparam=proxy.config.ssl.client.cert.filename={1} plugin=conf_remap.so @pparam=proxy.config.ssl.client.private_key.filename={2}'.format(server.Variables.SSL_Port, "signed2-foo.pem", "signed-foo.key") +) +ts.Disk.remap_config.AddLine( + 'map /case2 https://127.0.0.1:{0}/ @plugin=conf_remap.so @pparam=proxy.config.ssl.client.cert.filename={1} plugin=conf_remap.so @pparam=proxy.config.ssl.client.private_key.filename={2}'.format(server2.Variables.SSL_Port, "signed2-foo.pem", "signed-foo.key") +) +ts.Disk.remap_config.AddLine( + 'map /badcase2 https://127.0.0.1:{0}/ @plugin=conf_remap.so @pparam=proxy.config.ssl.client.cert.filename={1} plugin=conf_remap.so @pparam=proxy.config.ssl.client.private_key.filename={2}'.format(server2.Variables.SSL_Port, "signed-foo.pem", "signed-foo.key") +) + +# Should succeed +tr = Test.AddTestRun("Connect with correct client cert to first server") +tr.Processes.Default.StartBefore(Test.Processes.ts, ready=When.PortOpen(ts.Variables.port)) +tr.Processes.Default.StartBefore(server) +tr.Processes.Default.StartBefore(server2) +tr.StillRunningAfter = ts +tr.StillRunningAfter = server +tr.StillRunningAfter = server2 +tr.Processes.Default.Command = "curl -H host:example.com http://127.0.0.1:{0}/case1".format(ts.Variables.port) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could Not Connect", "Check response") + +#Should fail +trfail = Test.AddTestRun("Connect with bad client cert to first server") +trfail.StillRunningAfter = ts +trfail.StillRunningAfter = server +trfail.StillRunningAfter = server2 +trfail.Processes.Default.Command = 'curl -H host:example.com http://127.0.0.1:{0}/badcase1'.format(ts.Variables.port) +trfail.Processes.Default.ReturnCode = 0 +trfail.Processes.Default.Streams.stdout = Testers.ContainsExpression("Could Not Connect", "Check response") + +# Should succeed +trbar = Test.AddTestRun("Connect with correct client cert to second server") +trbar.StillRunningAfter = ts +trbar.StillRunningAfter = server +trbar.StillRunningAfter = server2 +trbar.Processes.Default.Command = "curl -H host:bar.com http://127.0.0.1:{0}/case2".format(ts.Variables.port) +trbar.Processes.Default.ReturnCode = 0 +trbar.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could Not Connect", "Check response") + +#Should fail +trbarfail = Test.AddTestRun("Connect with bad client cert to second server") +trbarfail.StillRunningAfter = ts +trbarfail.StillRunningAfter = server +trbarfail.StillRunningAfter = server2 +trbarfail.Processes.Default.Command = 'curl -H host:bar.com http://127.0.0.1:{0}/badcase2'.format(ts.Variables.port) +trbarfail.Processes.Default.ReturnCode = 0 +trbarfail.Processes.Default.Streams.stdout = Testers.ContainsExpression("Could Not Connect", "Check response") + + diff --git a/tests/gold_tests/tls/tls_client_verify.test.py b/tests/gold_tests/tls/tls_client_verify.test.py new file mode 100644 index 00000000000..e09ddabbdee --- /dev/null +++ b/tests/gold_tests/tls/tls_client_verify.test.py @@ -0,0 +1,178 @@ +''' +Test requiring certificate from user agent +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import re + +Test.Summary = ''' +Test various options for requiring certificate from client for mutual authentication TLS +''' + +Test.SkipUnless(Condition.HasProgram("grep", "grep needs to be installed on system for this test to work")) + +ts = Test.MakeATSProcess("ts", select_ports=False) +cafile = "{0}/signer.pem".format(Test.RunDirectory) +cafile2 = "{0}/signer2.pem".format(Test.RunDirectory) +server = Test.MakeOriginServer("server") + +request_header = {"headers": "GET / HTTP/1.1\r\nHost: example.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +response_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +server.addResponse("sessionlog.json", request_header, response_header) +request_header = {"headers": "GET / HTTP/1.1\r\nHost: bar.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +response_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +server.addResponse("sessionlog.json", request_header, response_header) + +ts.addSSLfile("ssl/server.pem") +ts.addSSLfile("ssl/server.key") +ts.addSSLfile("ssl/signer.pem") + +ts.Variables.ssl_port = 4443 +ts.Disk.records_config.update({ + 'proxy.config.ssl.server.cert.path': '{0}'.format(ts.Variables.SSLDir), + 'proxy.config.ssl.server.private_key.path': '{0}'.format(ts.Variables.SSLDir), + 'proxy.config.http.server_ports': '{0}:ssl'.format(ts.Variables.ssl_port), + 'proxy.config.ssl.client.verify.server': 0, + 'proxy.config.ssl.server.cipher_suite': 'ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA384:AES128-GCM-SHA256:AES256-GCM-SHA384:ECDHE-RSA-RC4-SHA:ECDHE-RSA-AES128-SHA:ECDHE-RSA-AES256-SHA:RC4-SHA:RC4-MD5:AES128-SHA:AES256-SHA:DES-CBC3-SHA!SRP:!DSS:!PSK:!aNULL:!eNULL:!SSLv2', + 'proxy.config.url_remap.pristine_host_hdr' : 1, + 'proxy.config.ssl.client.certification_level': 2, + 'proxy.config.ssl.CA.cert.filename': '{0}/signer.pem'.format(ts.Variables.SSLDir), + 'proxy.config.exec_thread.autoconfig.scale': 1.0, + 'proxy.config.ssl.TLSv1_3': 0 +}) + +ts.Disk.ssl_multicert_config.AddLine( + 'dest_ip=* ssl_cert_name=server.pem ssl_key_name=server.key' +) + +# Just map everything through to origin. This test is concentratign on the user-agent side +ts.Disk.remap_config.AddLine( + 'map / http://127.0.0.1:{0}/'.format(server.Variables.Port) +) + +# Scenario 1: Default no client cert required. cert required for bar.com +ts.Disk.ssl_server_name_yaml.AddLines([ + '- fqdn: bob.bar.com', + ' verify_client: NONE', + '- fqdn: bob.*.com', + ' verify_client: NONE', + '- fqdn: "*bar.com"', + ' verify_client: STRICT', +]) + +# to foo.com w/o client cert. Should fail +tr = Test.AddTestRun("Connect to foo.com without cert") +tr.Processes.Default.StartBefore(Test.Processes.ts, ready=When.PortOpen(ts.Variables.ssl_port)) +tr.Processes.Default.StartBefore(server) +tr.StillRunningAfter = ts +tr.StillRunningAfter = server +tr.Processes.Default.Command = "curl --tls-max 1.2 -k --resolve 'foo.com:{0}:127.0.0.1' https://foo.com:{0}/case1".format(ts.Variables.ssl_port) +tr.Processes.Default.ReturnCode = 35 + +tr = Test.AddTestRun("Connect to foo.com with bad cert") +tr.Setup.Copy("ssl/server.pem") +tr.Setup.Copy("ssl/server.key") +tr.StillRunningAfter = ts +tr.StillRunningAfter = server +tr.Processes.Default.Command = "curl --tls-max 1.2 -k --cert ./server.pem --key ./server.key --resolve 'foo.com:{0}:127.0.0.1' https://foo.com:{0}/case1".format(ts.Variables.ssl_port) +# Should fail with badly signed certs +tr.Processes.Default.ReturnCode = 35 + +tr = Test.AddTestRun("Connect to foo.com with cert") +tr.Setup.Copy("ssl/signed-foo.pem") +tr.Setup.Copy("ssl/signed-foo.key") +tr.StillRunningAfter = ts +tr.StillRunningAfter = server +tr.Processes.Default.Command = "curl --tls-max 1.2 -k --cert ./signed-foo.pem --key ./signed-foo.key --resolve 'foo.com:{0}:127.0.0.1' https://foo.com:{0}/case1".format(ts.Variables.ssl_port) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Streams.All = Testers.ExcludesExpression("error", "Check response") + +tr = Test.AddTestRun("Connect to bob.bar.com without cert") +tr.StillRunningAfter = ts +tr.StillRunningAfter = server +tr.Processes.Default.Command = "curl --tls-max 1.2 -k --resolve 'bob.bar.com:{0}:127.0.0.1' https://bob.bar.com:{0}/case1".format(ts.Variables.ssl_port) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Streams.All = Testers.ExcludesExpression("alert", "TLS handshake should succeed") + +tr = Test.AddTestRun("Connect to bob.bar.com with cert") +tr.Setup.Copy("ssl/signed-bob-bar.pem") +tr.Setup.Copy("ssl/signed-bar.key") +tr.StillRunningAfter = ts +tr.StillRunningAfter = server +tr.Processes.Default.Command = "curl --tls-max 1.2 -k --cert ./signed-bob-bar.pem --key ./signed-bar.key --resolve 'bob.bar.com:{0}:127.0.0.1' https://bob.bar.com:{0}/case1".format(ts.Variables.ssl_port) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Streams.All = Testers.ExcludesExpression("error", "Check response") + +tr = Test.AddTestRun("Connect to bob.bar.com with bad cert") +tr.Setup.Copy("ssl/server.pem") +tr.Setup.Copy("ssl/server.key") +tr.StillRunningAfter = ts +tr.StillRunningAfter = server +tr.Processes.Default.Command = "curl --tls-max 1.2 -k --cert ./server.pem --key ./server.key --resolve 'bob.bar.com:{0}:127.0.0.1' https://bob.bar.com:{0}/case1".format(ts.Variables.ssl_port) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Streams.All = Testers.ExcludesExpression("error", "Check response") + +tr = Test.AddTestRun("Connect to bob.foo.com without cert") +tr.StillRunningAfter = ts +tr.StillRunningAfter = server +tr.Processes.Default.Command = "curl --tls-max 1.2 -k --resolve 'bob.foo.com:{0}:127.0.0.1' https://bob.foo.com:{0}/case1".format(ts.Variables.ssl_port) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Streams.All = Testers.ExcludesExpression("alert", "TLS handshake should succeed") + +tr = Test.AddTestRun("Connect to bob.foo.com with cert") +tr.Setup.Copy("ssl/signed-bob-foo.pem") +tr.Setup.Copy("ssl/signed-foo.key") +tr.StillRunningAfter = ts +tr.StillRunningAfter = server +tr.Processes.Default.Command = "curl --tls-max 1.2 -k --cert ./signed-bob-foo.pem --key ./signed-foo.key --resolve 'bob.foo.com:{0}:127.0.0.1' https://bob.foo.com:{0}/case1".format(ts.Variables.ssl_port) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Streams.All = Testers.ExcludesExpression("error", "Check response") + +tr = Test.AddTestRun("Connect to bob.foo.com with bad cert") +tr.Setup.Copy("ssl/server.pem") +tr.Setup.Copy("ssl/server.key") +tr.StillRunningAfter = ts +tr.StillRunningAfter = server +tr.Processes.Default.Command = "curl --tls-max 1.2 -k --cert ./server.pem --key ./server.key --resolve 'bob.foo.com:{0}:127.0.0.1' https://bob.foo.com:{0}/case1".format(ts.Variables.ssl_port) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Streams.All = Testers.ExcludesExpression("error", "Check response") + +tr = Test.AddTestRun("Connect to bar.com without cert") +tr.StillRunningAfter = ts +tr.StillRunningAfter = server +tr.Processes.Default.Command = "curl --tls-max 1.2 -k --resolve 'bar.com:{0}:127.0.0.1' https://bar.com:{0}/case1".format(ts.Variables.ssl_port) +tr.Processes.Default.ReturnCode = 35 + +tr = Test.AddTestRun("Connect to bar.com with cert") +tr.Setup.Copy("ssl/signed-bar.pem") +tr.Setup.Copy("ssl/signed-bar.key") +tr.StillRunningAfter = ts +tr.StillRunningAfter = server +tr.Processes.Default.Command = "curl --tls-max 1.2 -k --cert ./signed-bar.pem --key ./signed-bar.key --resolve 'bar.com:{0}:127.0.0.1' https://bar.com:{0}/case1".format(ts.Variables.ssl_port) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Streams.All = Testers.ExcludesExpression("error", "TLS handshake should succeed") + +tr = Test.AddTestRun("Connect to bar.com with bad cert") +tr.Setup.Copy("ssl/server.pem") +tr.Setup.Copy("ssl/server.key") +tr.StillRunningAfter = ts +tr.StillRunningAfter = server +tr.Processes.Default.Command = "curl --tls-max 1.2 -k --cert ./server.pem --key ./server.key --resolve 'bar.com:{0}:127.0.0.1' https://bar.com:{0}/case1".format(ts.Variables.ssl_port) +tr.Processes.Default.ReturnCode = 35 + + diff --git a/tests/gold_tests/tls/tls_client_verify2.test.py b/tests/gold_tests/tls/tls_client_verify2.test.py new file mode 100644 index 00000000000..cf0286c598d --- /dev/null +++ b/tests/gold_tests/tls/tls_client_verify2.test.py @@ -0,0 +1,167 @@ +''' +Test requiring certificate from user agent +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import re + +Test.Summary = ''' +Test various options for requiring certificate from client for mutual authentication TLS +''' + +Test.SkipUnless(Condition.HasProgram("grep", "grep needs to be installed on system for this test to work")) + +ts = Test.MakeATSProcess("ts", select_ports=False) +cafile = "{0}/signer.pem".format(Test.RunDirectory) +cafile2 = "{0}/signer2.pem".format(Test.RunDirectory) +server = Test.MakeOriginServer("server") + +request_header = {"headers": "GET / HTTP/1.1\r\nHost: example.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +response_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +server.addResponse("sessionlog.json", request_header, response_header) +request_header = {"headers": "GET / HTTP/1.1\r\nHost: bar.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +response_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +server.addResponse("sessionlog.json", request_header, response_header) + +ts.addSSLfile("ssl/server.pem") +ts.addSSLfile("ssl/server.key") +ts.addSSLfile("ssl/signer.pem") + +ts.Variables.ssl_port = 4443 +ts.Disk.records_config.update({ + 'proxy.config.ssl.server.cert.path': '{0}'.format(ts.Variables.SSLDir), + 'proxy.config.ssl.server.private_key.path': '{0}'.format(ts.Variables.SSLDir), + 'proxy.config.http.server_ports': '{0}:ssl'.format(ts.Variables.ssl_port), + 'proxy.config.ssl.client.verify.server': 0, + 'proxy.config.ssl.server.cipher_suite': 'ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA384:AES128-GCM-SHA256:AES256-GCM-SHA384:ECDHE-RSA-RC4-SHA:ECDHE-RSA-AES128-SHA:ECDHE-RSA-AES256-SHA:RC4-SHA:RC4-MD5:AES128-SHA:AES256-SHA:DES-CBC3-SHA!SRP:!DSS:!PSK:!aNULL:!eNULL:!SSLv2', + 'proxy.config.url_remap.pristine_host_hdr' : 1, + 'proxy.config.ssl.client.certification_level': 0, + 'proxy.config.ssl.CA.cert.path': '', + 'proxy.config.exec_thread.autoconfig.scale': 1.0, + 'proxy.config.ssl.CA.cert.filename': '{0}/signer.pem'.format(ts.Variables.SSLDir) +}) + +ts.Disk.ssl_multicert_config.AddLine( + 'dest_ip=* ssl_cert_name=server.pem ssl_key_name=server.key' +) + +# Just map everything through to origin. This test is concentratign on the user-agent side +ts.Disk.remap_config.AddLine( + 'map / http://127.0.0.1:{0}/'.format(server.Variables.Port) +) + +# Scenario 1: Default no client cert required. cert required for bar.com +ts.Disk.ssl_server_name_yaml.AddLines([ + '- fqdn: bob.bar.com', + ' verify_client: STRICT', + '- fqdn: bob.*.com', + ' verify_client: STRICT', + '- fqdn: "*bar.com"', + ' verify_client: NONE', +]) + +# to foo.com w/o client cert. Should succeed +tr = Test.AddTestRun("Connect to foo.com without cert") +tr.Processes.Default.StartBefore(Test.Processes.ts, ready=When.PortOpen(ts.Variables.ssl_port)) +tr.Processes.Default.StartBefore(server) +tr.StillRunningAfter = ts +tr.StillRunningAfter = server +tr.Processes.Default.Command = "curl --tls-max 1.2 -k --resolve 'foo.com:{0}:127.0.0.1' https://foo.com:{0}/case1".format(ts.Variables.ssl_port) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Streams.All = Testers.ExcludesExpression("error", "Check response") + +tr = Test.AddTestRun("Connect to foo.com with cert") +tr.Setup.Copy("ssl/signed-foo.pem") +tr.Setup.Copy("ssl/signed-foo.key") +tr.StillRunningAfter = ts +tr.StillRunningAfter = server +tr.Processes.Default.Command = "curl --tls-max 1.2 -k --cert ./signed-foo.pem --key ./signed-foo.key --resolve 'foo.com:{0}:127.0.0.1' https://foo.com:{0}/case1".format(ts.Variables.ssl_port) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Streams.All = Testers.ExcludesExpression("error", "Check response") + +tr = Test.AddTestRun("Connect to bob.bar.com without cert") +tr.StillRunningAfter = ts +tr.StillRunningAfter = server +tr.Processes.Default.Command = "curl --tls-max 1.2 -k --resolve 'bob.bar.com:{0}:127.0.0.1' https://bob.bar.com:{0}/case1".format(ts.Variables.ssl_port) +tr.Processes.Default.ReturnCode = 35 + +tr = Test.AddTestRun("Connect to bob.bar.com with cert") +tr.Setup.Copy("ssl/signed-bob-bar.pem") +tr.Setup.Copy("ssl/signed-bar.key") +tr.StillRunningAfter = ts +tr.StillRunningAfter = server +tr.Processes.Default.Command = "curl --tls-max 1.2 -k --cert ./signed-bob-bar.pem --key ./signed-bar.key --resolve 'bob.bar.com:{0}:127.0.0.1' https://bob.bar.com:{0}/case1".format(ts.Variables.ssl_port) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Streams.All = Testers.ExcludesExpression("error", "TLS handshake should succeed") + +tr = Test.AddTestRun("Connect to bob.bar.com with bad cert") +tr.Setup.Copy("ssl/server.pem") +tr.Setup.Copy("ssl/server.key") +tr.StillRunningAfter = ts +tr.StillRunningAfter = server +tr.Processes.Default.Command = "curl --tls-max 1.2 -k --cert ./server.pem --key ./server.key --resolve 'bob.bar.com:{0}:127.0.0.1' https://bob.bar.com:{0}/case1".format(ts.Variables.ssl_port) +tr.Processes.Default.ReturnCode = 35 + +tr = Test.AddTestRun("Connect to bob.foo.com without cert") +tr.StillRunningAfter = ts +tr.StillRunningAfter = server +tr.Processes.Default.Command = "curl --tls-max 1.2 -k --resolve 'bob.foo.com:{0}:127.0.0.1' https://bob.foo.com:{0}/case1".format(ts.Variables.ssl_port) +tr.Processes.Default.ReturnCode = 35 + +tr = Test.AddTestRun("Connect to bob.foo.com with cert") +tr.Setup.Copy("ssl/signed-bob-foo.pem") +tr.Setup.Copy("ssl/signed-foo.key") +tr.StillRunningAfter = ts +tr.StillRunningAfter = server +tr.Processes.Default.Command = "curl --tls-max 1.2 -k --cert ./signed-bob-foo.pem --key ./signed-foo.key --resolve 'bob.foo.com:{0}:127.0.0.1' https://bob.foo.com:{0}/case1".format(ts.Variables.ssl_port) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Streams.All = Testers.ExcludesExpression("error", "TLS handshake should succeed") + +tr = Test.AddTestRun("Connect to bob.foo.com with bad cert") +tr.Setup.Copy("ssl/server.pem") +tr.Setup.Copy("ssl/server.key") +tr.StillRunningAfter = ts +tr.StillRunningAfter = server +tr.Processes.Default.Command = "curl --tls-max 1.2 -k --cert ./server.pem --key ./server.key --resolve 'bob.foo.com:{0}:127.0.0.1' https://bob.foo.com:{0}/case1".format(ts.Variables.ssl_port) +tr.Processes.Default.ReturnCode = 35 + +tr = Test.AddTestRun("Connect to bar.com without cert") +tr.StillRunningAfter = ts +tr.StillRunningAfter = server +tr.Processes.Default.Command = "curl --tls-max 1.2 -k --resolve 'bar.com:{0}:127.0.0.1' https://bar.com:{0}/case1".format(ts.Variables.ssl_port) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Streams.All = Testers.ExcludesExpression("alert", "TLS handshake should succeed") + +tr = Test.AddTestRun("Connect to bar.com with cert") +tr.Setup.Copy("ssl/signed-bar.pem") +tr.Setup.Copy("ssl/signed-bar.key") +tr.StillRunningAfter = ts +tr.StillRunningAfter = server +tr.Processes.Default.Command = "curl --tls-max 1.2 -k --cert ./signed-bar.pem --key ./signed-bar.key --resolve 'bar.com:{0}:127.0.0.1' https://bar.com:{0}/case1".format(ts.Variables.ssl_port) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Streams.All = Testers.ExcludesExpression("error", "Check response") + +tr = Test.AddTestRun("Connect to bar.com with bad cert") +tr.Setup.Copy("ssl/server.pem") +tr.Setup.Copy("ssl/server.key") +tr.StillRunningAfter = ts +tr.StillRunningAfter = server +tr.Processes.Default.Command = "curl --tls-max 1.2 -k --cert ./server.pem --key ./server.key --resolve 'bar.com:{0}:127.0.0.1' https://bar.com:{0}/case1".format(ts.Variables.ssl_port) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Streams.All = Testers.ExcludesExpression("alert unknown ca", "TLS handshake should succeed") + diff --git a/tests/gold_tests/tls/tls_client_versions.test.py b/tests/gold_tests/tls/tls_client_versions.test.py new file mode 100644 index 00000000000..91eea68fef1 --- /dev/null +++ b/tests/gold_tests/tls/tls_client_versions.test.py @@ -0,0 +1,101 @@ +''' +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +Test.Summary = ''' +Test TLS protocol offering based on SNI +''' + +# By default only offer TLSv1_2 +# for special doman foo.com only offer TLSv1 and TLSv1_1 + +# need Curl +Test.SkipUnless( + Condition.HasProgram("curl", "Curl need to be installed on system for this test to work"), + Condition.HasOpenSSLVersion("1.1.1") +) + +# Define default ATS +ts = Test.MakeATSProcess("ts", select_ports=False) +server = Test.MakeOriginServer("server", ssl=True) + +request_foo_header = {"headers": "GET / HTTP/1.1\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +response_foo_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": "foo ok"} +server.addResponse("sessionlog.json", request_foo_header, response_foo_header) + +# add ssl materials like key, certificates for the server +ts.addSSLfile("ssl/server.pem") +ts.addSSLfile("ssl/server.key") + +ts.Variables.ssl_port = 4443 + +# Need no remap rules. Everything should be proccessed by ssl_server_name + +# Make sure the TS server certs are different from the origin certs +ts.Disk.ssl_multicert_config.AddLine( + 'dest_ip=* ssl_cert_name=server.pem ssl_key_name=server.key' +) + +ts.Disk.records_config.update({ + 'proxy.config.ssl.server.cert.path': '{0}'.format(ts.Variables.SSLDir), + 'proxy.config.ssl.server.private_key.path': '{0}'.format(ts.Variables.SSLDir), + # enable ssl port + 'proxy.config.http.server_ports': '{0} {1}:proto=http2;http:ssl'.format(ts.Variables.port, ts.Variables.ssl_port), + 'proxy.config.ssl.server.cipher_suite': 'ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA384:AES128-GCM-SHA256:AES256-GCM-SHA384:ECDHE-RSA-RC4-SHA:ECDHE-RSA-AES128-SHA:ECDHE-RSA-AES256-SHA:RC4-SHA:RC4-MD5:AES128-SHA:AES256-SHA:DES-CBC3-SHA!SRP:!DSS:!PSK:!aNULL:!eNULL:!SSLv2', + 'proxy.config.ssl.client.CA.cert.path': '{0}'.format(ts.Variables.SSLDir), + 'proxy.config.url_remap.pristine_host_hdr': 1, + 'proxy.config.ssl.TLSv1': 0, + 'proxy.config.ssl.TLSv1_1': 0, + 'proxy.config.exec_thread.autoconfig.scale': 1.0, + 'proxy.config.ssl.TLSv1_2': 1 +}) + +# foo.com should only offer the older TLS protocols +# bar.com should terminate. +# empty SNI should tunnel to server_bar +ts.Disk.ssl_server_name_yaml.AddLines([ + '- fqdn: foo.com', + ' valid_tls_versions_in: [ TLSv1, TLSv1_1 ]' +]) + +# Target foo.com for TLSv1_2. Should fail +tr = Test.AddTestRun("foo.com TLSv1_2") +tr.Processes.Default.StartBefore(server) +tr.Processes.Default.StartBefore(Test.Processes.ts, ready=When.PortOpen(ts.Variables.ssl_port)) +tr.Processes.Default.Command = "curl -v --tls-max 1.2 --tlsv1.2 --resolve 'foo.com:{0}:127.0.0.1' -k https://foo.com:{0}".format(ts.Variables.ssl_port) +tr.ReturnCode = 35 +tr.StillRunningAfter = ts + +# Target foo.com for TLSv1. Should succeed +tr = Test.AddTestRun("foo.com TLSv1") +tr.Processes.Default.Command = "curl -v --tls-max 1.0 --tlsv1 --resolve 'foo.com:{0}:127.0.0.1' -k https://foo.com:{0}".format(ts.Variables.ssl_port) +tr.ReturnCode = 0 +tr.StillRunningAfter = ts + +# Target bar.com for TLSv1. Should fail +tr = Test.AddTestRun("bar.com TLSv1") +tr.Processes.Default.Command = "curl -v --tls-max 1.0 --tlsv1 --resolve 'bar.com:{0}:127.0.0.1' -k https://bar.com:{0}".format(ts.Variables.ssl_port) +tr.ReturnCode = 35 +tr.StillRunningAfter = ts + +# Target bar.com for TLSv1_2. Should succeed +tr = Test.AddTestRun("bar.com TLSv1_2") +tr.Processes.Default.Command = "curl -v --tls-max 1.2 --tlsv1.2 --resolve 'bar.com:{0}:127.0.0.1' -k https://bar.com:{0}".format(ts.Variables.ssl_port) +tr.ReturnCode = 0 +tr.StillRunningAfter = ts + diff --git a/tests/gold_tests/tls/tls_forward_nonhttp.test.py b/tests/gold_tests/tls/tls_forward_nonhttp.test.py new file mode 100644 index 00000000000..949f77458a2 --- /dev/null +++ b/tests/gold_tests/tls/tls_forward_nonhttp.test.py @@ -0,0 +1,74 @@ +''' +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +Test.Summary = ''' +Forwarding a non-HTTP protocol out of TLS +''' + +# need nc +Test.SkipUnless( + Condition.HasProgram("nc", "nc need to be installed on system for this test to work") +) + +# Define default ATS +ts = Test.MakeATSProcess("ts", select_ports=False) + +# add ssl materials like key, certificates for the server +ts.addSSLfile("ssl/server.pem") +ts.addSSLfile("ssl/server.key") + +ts.Variables.ssl_port = 4443 + +# Need no remap rules. Everything should be proccessed by ssl_server_name + +# Make sure the TS server certs are different from the origin certs +ts.Disk.ssl_multicert_config.AddLine( + 'dest_ip=* ssl_cert_name=server.pem ssl_key_name=server.key' +) + +# Case 1, global config policy=permissive properties=signature +# override for foo.com policy=enforced properties=all +ts.Disk.records_config.update({ + 'proxy.config.ssl.server.cert.path': '{0}'.format(ts.Variables.SSLDir), + 'proxy.config.ssl.server.private_key.path': '{0}'.format(ts.Variables.SSLDir), + # enable ssl port + 'proxy.config.http.server_ports': '{0} {1}:proto=http2;http:ssl'.format(ts.Variables.port, ts.Variables.ssl_port), + 'proxy.config.http.connect_ports': '{0} 4444'.format(ts.Variables.ssl_port), + 'proxy.config.ssl.server.cipher_suite': 'ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA384:AES128-GCM-SHA256:AES256-GCM-SHA384:ECDHE-RSA-RC4-SHA:ECDHE-RSA-AES128-SHA:ECDHE-RSA-AES256-SHA:RC4-SHA:RC4-MD5:AES128-SHA:AES256-SHA:DES-CBC3-SHA!SRP:!DSS:!PSK:!aNULL:!eNULL:!SSLv2', + 'proxy.config.exec_thread.autoconfig.scale': 1.0, + 'proxy.config.url_remap.pristine_host_hdr': 1 +}) + +# foo.com should not terminate. Just tunnel to server_foo +# bar.com should terminate. Forward its tcp stream to server_bar +ts.Disk.ssl_server_name_yaml.AddLines([ + "- fqdn: bar.com", + " forward_route: localhost:4444" + ]) + +tr = Test.AddTestRun("forward-non-http") +tr.Setup.Copy("test-nc-s_client.sh") +tr.Processes.Default.Command = "sh test-nc-s_client.sh 4444 4443" +tr.ReturnCode = 0 +tr.Processes.Default.StartBefore(Test.Processes.ts, ready=When.PortOpen(ts.Variables.ssl_port)) +tr.StillRunningAfter = ts +testout_path = os.path.join(Test.RunDirectory, "test.out") +tr.Disk.File(testout_path, id = "testout") +tr.Processes.Default.Streams.All += Testers.IncludesExpression("This is a reply", "s_client should get response") + diff --git a/tests/gold_tests/tls/tls_hooks_verify.test.py b/tests/gold_tests/tls/tls_hooks_verify.test.py index 88fd03737a9..50d94047bfe 100644 --- a/tests/gold_tests/tls/tls_hooks_verify.test.py +++ b/tests/gold_tests/tls/tls_hooks_verify.test.py @@ -38,6 +38,7 @@ ts.Variables.ssl_port = 4443 ts.Disk.records_config.update({ + # Test looks for debug output from the plugin 'proxy.config.diags.debug.enabled': 1, 'proxy.config.diags.debug.tags': 'ssl_verify_test', 'proxy.config.ssl.server.cert.path': '{0}'.format(ts.Variables.SSLDir), @@ -47,6 +48,7 @@ 'proxy.config.ssl.server.cipher_suite': 'ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA384:AES128-GCM-SHA256:AES256-GCM-SHA384:ECDHE-RSA-RC4-SHA:ECDHE-RSA-AES128-SHA:ECDHE-RSA-AES256-SHA:RC4-SHA:RC4-MD5:AES128-SHA:AES256-SHA:DES-CBC3-SHA!SRP:!DSS:!PSK:!aNULL:!eNULL:!SSLv2', 'proxy.config.ssl.client.verify.server.policy': 'ENFORCED', 'proxy.config.ssl.client.verify.server.properties': 'NONE', + 'proxy.config.exec_thread.autoconfig.scale': 1.0, 'proxy.config.url_remap.pristine_host_hdr': 1 }) @@ -55,13 +57,13 @@ ) ts.Disk.remap_config.AddLine( - 'map https://foo.com:{1}/ https://127.0.0.1:{0}'.format(server.Variables.Port,ts.Variables.ssl_port) + 'map https://foo.com:{1}/ https://127.0.0.1:{0}'.format(server.Variables.SSL_Port, ts.Variables.ssl_port) ) ts.Disk.remap_config.AddLine( - 'map https://bar.com:{1}/ https://127.0.0.1:{0}'.format(server.Variables.Port,ts.Variables.ssl_port) + 'map https://bar.com:{1}/ https://127.0.0.1:{0}'.format(server.Variables.SSL_Port, ts.Variables.ssl_port) ) ts.Disk.remap_config.AddLine( - 'map https://random.com:{1}/ https://127.0.0.1:{0}'.format(server.Variables.Port,ts.Variables.ssl_port) + 'map https://random.com:{1}/ https://127.0.0.1:{0}'.format(server.Variables.SSL_Port, ts.Variables.ssl_port) ) ts.Disk.ssl_server_name_yaml.AddLine( @@ -95,10 +97,8 @@ tr3.Processes.Default.ReturnCode = 0 tr3.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have failed") -# Over riding the built in ERROR check since we expect tr2 to fail -ts.Disk.diags_log.Content = Testers.ContainsExpression("WARNING: TS_EVENT_SSL_VERIFY_SERVER plugin failed the origin certificate check for 127.0.0.1. Action=Terminate SNI=random.com", "random.com should fail") +ts.Disk.diags_log.Content += Testers.ContainsExpression("WARNING: TS_EVENT_SSL_VERIFY_SERVER plugin failed the origin certificate check for 127.0.0.1. Action=Terminate SNI=random.com", "random.com should fail") ts.Disk.diags_log.Content += Testers.ContainsExpression("WARNING: TS_EVENT_SSL_VERIFY_SERVER plugin failed the origin certificate check for 127.0.0.1. Action=Continue SNI=bar.com", "bar.com should fail but continue") -ts.Disk.diags_log.Content += Testers.ContainsExpression("ERROR: SSL connection failed for 'random.com': .+?:certificate verify failed", "random.com should really fail") ts.Disk.diags_log.Content += Testers.ExcludesExpression("SNI=foo.com", "foo.com should not fail in any way") ts.Streams.All += Testers.ContainsExpression("Server verify callback 0 [\da-fx]+? - event is good SNI=foo.com good HS", "verify callback happens 2 times") diff --git a/tests/gold_tests/tls/tls_keepalive.test.py b/tests/gold_tests/tls/tls_keepalive.test.py new file mode 100644 index 00000000000..ff41e459ecd --- /dev/null +++ b/tests/gold_tests/tls/tls_keepalive.test.py @@ -0,0 +1,108 @@ +''' +Use pre-accept hook to verify that both requests are made over the same TLS session +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import re + +Test.Summary = ''' +Verify that the client-side keep alive is honored for TLS and different versions of HTTP +''' + +Test.SkipUnless( + Condition.HasProgram("curl", "Curl need to be installed on system for this test to work"), + Condition.HasCurlFeature('http2') +) + +ts = Test.MakeATSProcess("ts", select_ports=False) +server = Test.MakeOriginServer("server") +request_header = {"headers": "GET / HTTP/1.1\r\nHost: www.example.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +# desired response form the origin server +response_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +server.addResponse("sessionlog.json", request_header, response_header) + +ts.addSSLfile("ssl/server.pem") +ts.addSSLfile("ssl/server.key") + +ts.Variables.ssl_port = 4443 +ts.Disk.records_config.update({ + 'proxy.config.ssl.server.cert.path': '{0}'.format(ts.Variables.SSLDir), + 'proxy.config.ssl.server.private_key.path': '{0}'.format(ts.Variables.SSLDir), + # enable ssl port + 'proxy.config.http.server_ports': '{0}:ssl'.format(ts.Variables.ssl_port), + 'proxy.config.ssl.TLSv1_3': 0, + 'proxy.config.ssl.server.cipher_suite': 'ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA384:AES128-GCM-SHA256:AES256-GCM-SHA384:ECDHE-RSA-RC4-SHA:ECDHE-RSA-AES128-SHA:ECDHE-RSA-AES256-SHA:RC4-SHA:RC4-MD5:AES128-SHA:AES256-SHA:DES-CBC3-SHA!SRP:!DSS:!PSK:!aNULL:!eNULL:!SSLv2', + 'proxy.config.exec_thread.autoconfig.scale': 1.0, + 'proxy.config.log.max_secs_per_buffer': 1 +}) + +ts.Disk.ssl_multicert_config.AddLine( + 'dest_ip=* ssl_cert_name=server.pem ssl_key_name=server.key' +) + +ts.Disk.remap_config.AddLine( + 'map https://example.com:4443 http://127.0.0.1:{0}'.format(server.Variables.Port) +) + +ts.Disk.logging_yaml.AddLines([ + 'formats:', + '- name: testformat', + " format: '% %'", + "logs:", + "- mode: ascii", + " format: testformat", + " filename: squid" ]) + +Test.PreparePlugin(os.path.join(Test.Variables.AtsTestToolsDir, 'plugins', 'ssl_hook_test.cc'), ts, '-preaccept=1') + +tr = Test.AddTestRun("Test two HTTP/1.1 requests over one TLS connection") +tr.Processes.Default.StartBefore(server) +tr.Processes.Default.StartBefore(Test.Processes.ts, ready=When.PortOpen(ts.Variables.ssl_port)) +tr.StillRunningAfter = ts +tr.StillRunningAfter = server +tr.Processes.Default.Command = 'curl -k -v --http1.1 -H \'host:example.com:{0}\' https://127.0.0.1:{0} https://127.0.0.1:{0}'.format(ts.Variables.ssl_port) +tr.Processes.Default.ReturnCode = 0 + +tr = Test.AddTestRun("Test two HTTP/1.1 requests over two TLS connections") +tr.StillRunningAfter = ts +tr.StillRunningAfter = server +tr.Processes.Default.Command = 'curl -k -v --http1.1 -H \'host:example.com:{0}\' https://127.0.0.1:{0}; curl -k -v --http1.1 -H \'host:example.com:{0}\' https://127.0.0.1:{0}'.format(ts.Variables.ssl_port) +tr.Processes.Default.ReturnCode = 0 + +tr = Test.AddTestRun("Test two HTTP/2 requests over one TLS connection") +tr.StillRunningAfter = ts +tr.StillRunningAfter = server +tr.Processes.Default.Command = 'curl -k -v --http2 -H \'host:example.com:{0}\' https://127.0.0.1:{0} https://127.0.0.1:{0}'.format(ts.Variables.ssl_port) +tr.Processes.Default.ReturnCode = 0 + +tr = Test.AddTestRun("Test two HTTP/2 requests over two TLS connections") +tr.StillRunningAfter = ts +tr.StillRunningAfter = server +tr.Processes.Default.Command = 'curl -k -v --http2 -H \'host:example.com:{0}\' https://127.0.0.1:{0}; curl -k -v --http1.1 -H \'host:example.com:{0}\' https://127.0.0.1:{0}'.format(ts.Variables.ssl_port) +tr.Processes.Default.ReturnCode = 0 + +# Just a check to flush out the traffic log until we have a clean shutdown for traffic_server +tr = Test.AddTestRun("Wait for the access log to write out") +tr.DelayStart = 5 +tr.StillRunningAfter = ts +tr.StillRunningAfter = server +tr.Processes.Default.Command = 'ls' +tr.Processes.Default.ReturnCode = 0 + +ts.Disk.squid_log.Content = "gold/accesslog.gold" + diff --git a/tests/gold_tests/tls/tls_ocsp.test.py b/tests/gold_tests/tls/tls_ocsp.test.py new file mode 100644 index 00000000000..69f319e00e2 --- /dev/null +++ b/tests/gold_tests/tls/tls_ocsp.test.py @@ -0,0 +1,73 @@ +''' +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +Test.Summary = ''' +Test tls server prefetched OCSP responses +''' + +# curl --cert-status option has been introduced in version 7.41.0 +Test.SkipUnless( + Condition.HasCurlVersion("7.41.0") +) + +# Define default ATS +ts = Test.MakeATSProcess("ts", select_ports=False) +server = Test.MakeOriginServer("server") +request_header = {"headers": "GET / HTTP/1.1\r\nHost: example.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +# desired response form the origin server +response_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +server.addResponse("sessionlog.json", request_header, response_header) + +ts.addSSLfile("ssl/ca.ocsp.pem") +ts.addSSLfile("ssl/server.ocsp.pem") +ts.addSSLfile("ssl/server.ocsp.key") +ts.addSSLfile("ssl/ocsp_response.der") + +ts.Variables.ssl_port = 4443 +ts.Disk.remap_config.AddLine( + 'map https://example.com:4443 http://127.0.0.1:{0}'.format(server.Variables.Port) +) + +ts.Disk.ssl_multicert_config.AddLine( + 'dest_ip=* ssl_cert_name=server.ocsp.pem ssl_key_name=server.ocsp.key ssl_ocsp_name=ocsp_response.der' +) + +# Case 1, global config policy=permissive properties=signature +# override for foo.com policy=enforced properties=all +ts.Disk.records_config.update({ + 'proxy.config.ssl.server.cert.path': '{0}'.format(ts.Variables.SSLDir), + 'proxy.config.ssl.server.private_key.path': '{0}'.format(ts.Variables.SSLDir), + 'proxy.config.ssl.server.cert_chain.filename': 'ca.ocsp.pem', + # enable prefetched OCSP responses + 'proxy.config.ssl.ocsp.response.path': '{0}'.format(ts.Variables.SSLDir), + 'proxy.config.ssl.ocsp.enabled': 1, + # enable ssl port + 'proxy.config.http.server_ports': '{0} {1}:proto=http2;http:ssl'.format(ts.Variables.port, ts.Variables.ssl_port), + 'proxy.config.ssl.server.cipher_suite': 'ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA384:AES128-GCM-SHA256:AES256-GCM-SHA384:ECDHE-RSA-RC4-SHA:ECDHE-RSA-AES128-SHA:ECDHE-RSA-AES256-SHA:RC4-SHA:RC4-MD5:AES128-SHA:AES256-SHA:DES-CBC3-SHA!SRP:!DSS:!PSK:!aNULL:!eNULL:!SSLv2', + 'proxy.config.exec_thread.autoconfig.scale': 1.0 +}) + + +tr = Test.AddTestRun("Check OCSP response using curl") +tr.Processes.Default.StartBefore(server) +tr.Processes.Default.StartBefore(Test.Processes.ts, ready=When.PortOpen(ts.Variables.ssl_port)) +tr.StillRunningAfter = server +tr.StillRunningAfter = ts +tr.Processes.Default.Command = "curl -v --cacert {0} --cert-status -H \"host:example.com\" https://127.0.0.1:{1}".format(os.path.join(ts.Variables.SSLDir, "ca.ocsp.pem"), ts.Variables.ssl_port) +tr.ReturnCode = 0 diff --git a/tests/gold_tests/tls/tls_ticket.test.py b/tests/gold_tests/tls/tls_ticket.test.py index 9a2b43b1252..29c9b34406a 100644 --- a/tests/gold_tests/tls/tls_ticket.test.py +++ b/tests/gold_tests/tls/tls_ticket.test.py @@ -66,6 +66,7 @@ 'proxy.config.http.server_ports': '{0}:proto=http2;http:ssl'.format(ts.Variables.ssl_port), 'proxy.config.ssl.client.verify.server': 0, 'proxy.config.ssl.server.cipher_suite': 'ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA384:AES128-GCM-SHA256:AES256-GCM-SHA384:ECDHE-RSA-RC4-SHA:ECDHE-RSA-AES128-SHA:ECDHE-RSA-AES256-SHA:RC4-SHA:RC4-MD5:AES128-SHA:AES256-SHA:DES-CBC3-SHA!SRP:!DSS:!PSK:!aNULL:!eNULL:!SSLv2', + 'proxy.config.exec_thread.autoconfig.scale': 1.0, 'proxy.config.ssl.server.session_ticket.enable': '1', 'proxy.config.ssl.server.ticket_key.filename': '../../file.ticket' }) @@ -76,21 +77,20 @@ 'proxy.config.ssl.client.verify.server': 0, 'proxy.config.ssl.server.cipher_suite': 'ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA384:AES128-GCM-SHA256:AES256-GCM-SHA384:ECDHE-RSA-RC4-SHA:ECDHE-RSA-AES128-SHA:ECDHE-RSA-AES256-SHA:RC4-SHA:RC4-MD5:AES128-SHA:AES256-SHA:DES-CBC3-SHA!SRP:!DSS:!PSK:!aNULL:!eNULL:!SSLv2', 'proxy.config.ssl.server.session_ticket.enable': '1', + 'proxy.config.exec_thread.autoconfig.scale': 1.0, 'proxy.config.ssl.server.ticket_key.filename': '../../file.ticket' }) tr = Test.AddTestRun("Create ticket") tr.Setup.Copy('file.ticket') -tr.Command = 'echo -e "GET / HTTP/1.0\r\n" | openssl s_client -connect 127.0.0.1:{0} -sess_out ticket.out'.format(ts.Variables.ssl_port) +tr.Command = 'echo -e "GET / HTTP/1.0\r\n" | openssl s_client -tls1_2 -connect 127.0.0.1:{0} -sess_out ticket.out'.format(ts.Variables.ssl_port) tr.ReturnCode = 0 # time delay as proxy.config.http.wait_for_cache could be broken tr.Processes.Default.StartBefore(server) tr.Processes.Default.StartBefore(Test.Processes.ts, ready=When.PortOpen(ts.Variables.ssl_port)) path1 = tr.Processes.Default.Streams.stdout.AbsPath tr.StillRunningAfter = server -tr.Processes.Default.TimeOut = 5 -tr.TimeOut = 5 # Pull out session created in tr to test for session id in tr2 def checkSession(ev) : @@ -120,9 +120,8 @@ def checkSession(ev) : tr2 = Test.AddTestRun("Test ticket") tr2.Setup.Copy('file.ticket') -tr2.Command = 'echo -e "GET / HTTP/1.0\r\n" | openssl s_client -connect 127.0.0.1:{0} -sess_in ticket.out'.format(ts2.Variables.ssl_port) +tr2.Command = 'echo -e "GET / HTTP/1.0\r\n" | openssl s_client -tls1_2 -connect 127.0.0.1:{0} -sess_in ticket.out'.format(ts2.Variables.ssl_port) tr2.Processes.Default.StartBefore(Test.Processes.ts2, ready=When.PortOpen(ts2.Variables.ssl_port)) tr2.ReturnCode = 0 path2 = tr2.Processes.Default.Streams.stdout.AbsPath -tr2.Processes.Default.TimeOut = 5 tr2.Processes.Default.Streams.All.Content = Testers.Lambda(checkSession) diff --git a/tests/gold_tests/tls/tls_tunnel.test.py b/tests/gold_tests/tls/tls_tunnel.test.py new file mode 100644 index 00000000000..36dc0125c53 --- /dev/null +++ b/tests/gold_tests/tls/tls_tunnel.test.py @@ -0,0 +1,191 @@ +''' +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import subprocess +Test.Summary = ''' +Test tunneling based on SNI +''' + +# need Curl +Test.SkipUnless( + Condition.HasProgram("curl", "Curl need to be installed on system for this test to work") +) + +# Define default ATS +ts = Test.MakeATSProcess("ts", command="traffic_manager", select_ports=False) +server_foo = Test.MakeOriginServer("server_foo", ssl=True) +server_bar = Test.MakeOriginServer("server_bar", ssl=True) +server2 = Test.MakeOriginServer("server2") + +request_foo_header = {"headers": "GET / HTTP/1.1\r\nHost: foo.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +request_bar_header = {"headers": "GET / HTTP/1.1\r\nHost: bar.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +response_foo_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": "foo ok"} +response_bar_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": "bar ok"} +server_foo.addResponse("sessionlog.json", request_foo_header, response_foo_header) +server_bar.addResponse("sessionlog.json", request_bar_header, response_bar_header) + +# add ssl materials like key, certificates for the server +ts.addSSLfile("ssl/signed-foo.pem") +ts.addSSLfile("ssl/signed-foo.key") +ts.addSSLfile("ssl/signed-bar.pem") +ts.addSSLfile("ssl/signed-bar.key") +ts.addSSLfile("ssl/server.pem") +ts.addSSLfile("ssl/server.key") +ts.addSSLfile("ssl/signer.pem") +ts.addSSLfile("ssl/signer.key") + +ts.Variables.ssl_port = 4443 + +# Need no remap rules. Everything should be proccessed by ssl_server_name + +# Make sure the TS server certs are different from the origin certs +ts.Disk.ssl_multicert_config.AddLine( + 'dest_ip=* ssl_cert_name=signed-foo.pem ssl_key_name=signed-foo.key' +) + +# Case 1, global config policy=permissive properties=signature +# override for foo.com policy=enforced properties=all +ts.Disk.records_config.update({ + 'proxy.config.ssl.server.cert.path': '{0}'.format(ts.Variables.SSLDir), + 'proxy.config.ssl.server.private_key.path': '{0}'.format(ts.Variables.SSLDir), + # enable ssl port + 'proxy.config.http.server_ports': '{0} {1}:proto=http2;http:ssl'.format(ts.Variables.port, ts.Variables.ssl_port), + 'proxy.config.http.connect_ports': '{0} {1} {2}'.format(ts.Variables.ssl_port,server_foo.Variables.SSL_Port,server_bar.Variables.SSL_Port), + 'proxy.config.ssl.server.cipher_suite': 'ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA384:AES128-GCM-SHA256:AES256-GCM-SHA384:ECDHE-RSA-RC4-SHA:ECDHE-RSA-AES128-SHA:ECDHE-RSA-AES256-SHA:RC4-SHA:RC4-MD5:AES128-SHA:AES256-SHA:DES-CBC3-SHA!SRP:!DSS:!PSK:!aNULL:!eNULL:!SSLv2', + 'proxy.config.ssl.client.CA.cert.path': '{0}'.format(ts.Variables.SSLDir), + 'proxy.config.ssl.client.CA.cert.filename': 'signer.pem', + 'proxy.config.exec_thread.autoconfig.scale': 1.0, + 'proxy.config.url_remap.pristine_host_hdr': 1 +}) + +# foo.com should not terminate. Just tunnel to server_foo +# bar.com should terminate. +# empty SNI should tunnel to server_bar +ts.Disk.ssl_server_name_yaml.AddLines([ + '- fqdn: foo.com', + " tunnel_route: localhost:{0}".format(server_foo.Variables.SSL_Port), + "- fqdn: bob.*.com", + " tunnel_route: localhost:{0}".format(server_foo.Variables.SSL_Port), + "- fqdn: ''", # No SNI sent + " tunnel_route: localhost:{0}".format(server_bar.Variables.SSL_Port) +]) + +tr = Test.AddTestRun("foo.com Tunnel-test") +tr.Processes.Default.StartBefore(server_foo) +tr.Processes.Default.StartBefore(server_bar) +tr.Processes.Default.StartBefore(Test.Processes.ts, ready=When.PortOpen(ts.Variables.ssl_port)) +tr.Processes.Default.Command = "curl -v --resolve 'foo.com:{0}:127.0.0.1' -k https://foo.com:{0}".format(ts.Variables.ssl_port) +tr.ReturnCode = 0 +tr.StillRunningAfter = ts +tr.Processes.Default.Streams.All += Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") +tr.Processes.Default.Streams.All += Testers.ExcludesExpression("Not Found on Accelerato", "Should not try to remap on Traffic Server") +tr.Processes.Default.Streams.All += Testers.ExcludesExpression("CN=foo.com", "Should not TLS terminate on Traffic Server") +tr.Processes.Default.Streams.All += Testers.ContainsExpression("HTTP/1.1 200 OK", "Should get a successful response") +tr.Processes.Default.Streams.All += Testers.ExcludesExpression("ATS", "Do not terminate on Traffic Server") +tr.Processes.Default.Streams.All += Testers.ContainsExpression("foo ok", "Should get a response from foo") + +tr = Test.AddTestRun("bob.bar.com Tunnel-test") +tr.Processes.Default.Command = "curl -v --resolve 'bob.bar.com:{0}:127.0.0.1' -k https://bob.bar.com:{0}".format(ts.Variables.ssl_port) +tr.ReturnCode = 0 +tr.StillRunningAfter = ts +tr.Processes.Default.Streams.All += Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") +tr.Processes.Default.Streams.All += Testers.ExcludesExpression("Not Found on Accelerato", "Should not try to remap on Traffic Server") +tr.Processes.Default.Streams.All += Testers.ExcludesExpression("CN=foo.com", "Should not TLS terminate on Traffic Server") +tr.Processes.Default.Streams.All += Testers.ContainsExpression("HTTP/1.1 200 OK", "Should get a successful response") +tr.Processes.Default.Streams.All += Testers.ExcludesExpression("ATS", "Do not terminate on Traffic Server") +tr.Processes.Default.Streams.All += Testers.ContainsExpression("foo ok", "Should get a response from foo") + +tr = Test.AddTestRun("bar.com no Tunnel-test") +tr.Processes.Default.Command = "curl -v --resolve 'bar.com:{0}:127.0.0.1' -k https://bar.com:{0}".format(ts.Variables.ssl_port) +tr.ReturnCode = 0 +tr.StillRunningAfter = ts +tr.Processes.Default.Streams.All += Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") +tr.Processes.Default.Streams.All += Testers.ContainsExpression("Not Found on Accelerato", "Terminates on on Traffic Server") +tr.Processes.Default.Streams.All += Testers.ContainsExpression("ATS", "Terminate on Traffic Server") + +tr = Test.AddTestRun("no SNI Tunnel-test") +tr.Processes.Default.Command = "curl -v -k https://127.0.0.1:{0}".format(ts.Variables.ssl_port) +tr.ReturnCode = 0 +tr.StillRunningAfter = ts +tr.Processes.Default.Streams.All += Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") +tr.Processes.Default.Streams.All += Testers.ExcludesExpression("Not Found on Accelerato", "Should not try to remap on Traffic Server") +tr.Processes.Default.Streams.All += Testers.ContainsExpression("HTTP/1.1 200 OK", "Should get a successful response") +tr.Processes.Default.Streams.All += Testers.ExcludesExpression("ATS", "Do not terminate on Traffic Server") +tr.Processes.Default.Streams.All += Testers.ContainsExpression("bar ok", "Should get a response from bar") + +# Update ssl_server_name file and reload +tr = Test.AddTestRun("Update config files") +# Update the SNI config +snipath = ts.Disk.ssl_server_name_yaml.AbsPath +recordspath = ts.Disk.records_config.AbsPath +tr.Disk.File(snipath, id = "ssl_server_name_yaml", typename="ats:config"), +tr.Disk.ssl_server_name_yaml.AddLines([ + '- fqdn: bar.com', + " tunnel_route: localhost:{0}".format(server_bar.Variables.SSL_Port), +]) +tr.StillRunningAfter = ts +tr.StillRunningAfter = server_foo +tr.StillRunningAfter = server_bar +tr.Processes.Default.Env = ts.Env +tr.Processes.Default.Command = 'echo Updated configs' +tr.Processes.Default.ReturnCode = 0 + +trreload = Test.AddTestRun("Reload config") +trreload.StillRunningAfter = ts +trreload.StillRunningAfter = server_foo +trreload.StillRunningAfter = server_bar +trreload.Processes.Default.Command = 'traffic_ctl config reload' +# Need to copy over the environment so traffic_ctl knows where to find the unix domain socket +trreload.Processes.Default.Env = ts.Env +trreload.Processes.Default.ReturnCode = 0 + +# Parking this as a ready tester on a meaningless process +# Stall the test runs until the ssl_server_name reload has completed +# At that point the new ssl_server_name settings are ready to go +def ssl_server_name_reload_done(tsenv): + def done_reload(process, hasRunFor, **kw): + cmd = "grep 'ssl_server_name.yaml finished loading' {0} | wc -l > {1}/test.out".format(ts.Disk.diags_log.Name, Test.RunDirectory) + retval = subprocess.run(cmd, shell=True, env=tsenv) + if retval.returncode == 0: + cmd ="if [ -f {0}/test.out -a \"`cat {0}/test.out`\" = \"2\" ] ; then true; else false; fi".format(Test.RunDirectory) + retval = subprocess.run(cmd, shell = True, env=tsenv) + return retval.returncode == 0 + + return done_reload + +# Should termimate on traffic_server (not tunnel) +tr = Test.AddTestRun("foo.com no Tunnel-test") +tr.StillRunningAfter = ts +# Wait for the reload to complete by running the ssl_server_name_reload_done test +tr.Processes.Default.StartBefore(server2, ready=ssl_server_name_reload_done(ts.Env)) +tr.Processes.Default.Command = "curl -v --resolve 'foo.com:{0}:127.0.0.1' -k https://foo.com:{0}".format(ts.Variables.ssl_port) +tr.Processes.Default.Streams.All += Testers.ContainsExpression("Not Found on Accelerato", "Terminates on on Traffic Server") +tr.Processes.Default.Streams.All += Testers.ContainsExpression("ATS", "Terminate on Traffic Server") +tr.Processes.Default.Streams.All += Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") + +# Should tunnel to server_bar +tr = Test.AddTestRun("bar.com Tunnel-test") +tr.Processes.Default.Command = "curl -v --resolve 'bar.com:{0}:127.0.0.1' -k https://bar.com:{0}".format(ts.Variables.ssl_port) +tr.ReturnCode = 0 +tr.StillRunningAfter = ts +tr.Processes.Default.Streams.All += Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") +tr.Processes.Default.Streams.All += Testers.ExcludesExpression("Not Found on Accelerato", "Terminates on on Traffic Server") +tr.Processes.Default.Streams.All += Testers.ExcludesExpression("ATS", "Terminate on Traffic Server") +tr.Processes.Default.Streams.All += Testers.ContainsExpression("bar ok", "Should get a response from bar") + diff --git a/tests/gold_tests/tls/tls_tunnel_forward.test.py b/tests/gold_tests/tls/tls_tunnel_forward.test.py new file mode 100644 index 00000000000..f8068e7800d --- /dev/null +++ b/tests/gold_tests/tls/tls_tunnel_forward.test.py @@ -0,0 +1,125 @@ +''' +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +Test.Summary = ''' +Test tunneling and forwarding based on SNI +''' + +# need Curl +Test.SkipUnless( + Condition.HasProgram("curl", "Curl need to be installed on system for this test to work") +) + +# Define default ATS +ts = Test.MakeATSProcess("ts", select_ports=False) +server_foo = Test.MakeOriginServer("server_foo", ssl=True) +server_bar = Test.MakeOriginServer("server_bar", ssl=False) +server_random = Test.MakeOriginServer("server_random", ssl=False) + +request_foo_header = {"headers": "GET / HTTP/1.1\r\nHost: foo.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +request_bar_header = {"headers": "GET / HTTP/1.1\r\nHost: bar.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +request_random_header = {"headers": "GET / HTTP/1.1\r\nHost: random.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +response_foo_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": "ok foo"} +response_bar_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": "ok bar"} +response_random_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": "ok random"} +server_foo.addResponse("sessionlog_foo.json", request_foo_header, response_foo_header) +server_bar.addResponse("sessionlog_bar.json", request_bar_header, response_bar_header) +server_random.addResponse("sessionlog_random.json", request_random_header, response_random_header) + +# add ssl materials like key, certificates for the server +ts.addSSLfile("ssl/signed-foo.pem") +ts.addSSLfile("ssl/signed-foo.key") +ts.addSSLfile("ssl/signed-bar.pem") +ts.addSSLfile("ssl/signed-bar.key") +ts.addSSLfile("ssl/server.pem") +ts.addSSLfile("ssl/server.key") +ts.addSSLfile("ssl/signer.pem") +ts.addSSLfile("ssl/signer.key") + +ts.Variables.ssl_port = 4443 + +# Need no remap rules. Everything should be proccessed by ssl_server_name + +# Make sure the TS server certs are different from the origin certs +ts.Disk.ssl_multicert_config.AddLine( + 'dest_ip=* ssl_cert_name=signed-foo.pem ssl_key_name=signed-foo.key' +) + +# Case 1, global config policy=permissive properties=signature +# override for foo.com policy=enforced properties=all +ts.Disk.records_config.update({ + 'proxy.config.ssl.server.cert.path': '{0}'.format(ts.Variables.SSLDir), + 'proxy.config.ssl.server.private_key.path': '{0}'.format(ts.Variables.SSLDir), + # enable ssl port + 'proxy.config.http.server_ports': '{0} {1}:proto=http2;http:ssl'.format(ts.Variables.port, ts.Variables.ssl_port), + 'proxy.config.http.connect_ports': '{0} {1} {2} {3}'.format(ts.Variables.ssl_port, server_foo.Variables.SSL_Port, server_bar.Variables.Port, server_random.Variables.Port), + 'proxy.config.ssl.server.cipher_suite': 'ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA384:AES128-GCM-SHA256:AES256-GCM-SHA384:ECDHE-RSA-RC4-SHA:ECDHE-RSA-AES128-SHA:ECDHE-RSA-AES256-SHA:RC4-SHA:RC4-MD5:AES128-SHA:AES256-SHA:DES-CBC3-SHA!SRP:!DSS:!PSK:!aNULL:!eNULL:!SSLv2', + 'proxy.config.ssl.client.CA.cert.path': '{0}'.format(ts.Variables.SSLDir), + 'proxy.config.ssl.client.CA.cert.filename': 'signer.pem', + 'proxy.config.exec_thread.autoconfig.scale': 1.0, + 'proxy.config.url_remap.pristine_host_hdr': 1 +}) + +# foo.com should not terminate. Just tunnel to server_foo +# bar.com should terminate. Forward its tcp stream to server_bar +ts.Disk.ssl_server_name_yaml.AddLines([ + "- fqdn: 'foo.com'", + " tunnel_route: 'localhost:{0}'".format(server_foo.Variables.SSL_Port), + "- fqdn: 'bar.com'", + " forward_route: 'localhost:{0}'".format(server_bar.Variables.Port), + "- fqdn: ''", #default case + " forward_route: 'localhost:{0}'".format(server_random.Variables.Port), + ]) + +tr = Test.AddTestRun("Tunnel-test") +tr.Processes.Default.Command = "curl -v --resolve 'foo.com:{0}:127.0.0.1' -k https://foo.com:{0}".format(ts.Variables.ssl_port) +tr.ReturnCode = 0 +tr.Processes.Default.StartBefore(server_foo) +tr.Processes.Default.StartBefore(server_bar) +tr.Processes.Default.StartBefore(server_random) +tr.Processes.Default.StartBefore(Test.Processes.ts, ready=When.PortOpen(ts.Variables.ssl_port)) +tr.StillRunningAfter = ts +tr.Processes.Default.Streams.All += Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") +tr.Processes.Default.Streams.All += Testers.ExcludesExpression("Not Found on Accelerato", "Should not try to remap on Traffic Server") +tr.Processes.Default.Streams.All += Testers.ExcludesExpression("CN=foo.com", "Should not TLS terminate on Traffic Server") +tr.Processes.Default.Streams.All += Testers.ContainsExpression("HTTP/1.1 200 OK", "Should get a successful response") +tr.Processes.Default.Streams.All += Testers.ContainsExpression("ok foo", "Body is expected") + +tr2 = Test.AddTestRun("Forward-test") +tr2.Processes.Default.Command = "curl -v --http1.1 -H 'host:bar.com' --resolve 'bar.com:{0}:127.0.0.1' -k https://bar.com:{0}".format(ts.Variables.ssl_port) +tr2.ReturnCode = 0 +tr2.StillRunningAfter = server_bar +tr2.StillRunningAfter = ts +tr2.Processes.Default.Streams.All += Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") +tr2.Processes.Default.Streams.All += Testers.ExcludesExpression("Not Found on Accelerato", "Should not try to remap on Traffic Server") +tr2.Processes.Default.Streams.All += Testers.ContainsExpression("CN=foo.com", "Should TLS terminate on Traffic Server") +tr2.Processes.Default.Streams.All += Testers.ContainsExpression("HTTP/1.1 200 OK", "Should get a successful response") +tr2.Processes.Default.Streams.All += Testers.ContainsExpression("ok bar", "Body is expected") + +tr3 = Test.AddTestRun("no-sni-forward-test") +tr3.Processes.Default.Command = "curl --http1.1 -v -k -H 'host:random.com' https://127.0.0.1:{0}".format(ts.Variables.ssl_port) +tr3.ReturnCode = 0 +tr3.StillRunningAfter = server_random +tr3.StillRunningAfter = ts +tr3.Processes.Default.Streams.All += Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") +tr3.Processes.Default.Streams.All += Testers.ExcludesExpression("Not Found on Accelerato", "Should not try to remap on Traffic Server") +tr3.Processes.Default.Streams.All += Testers.ContainsExpression("CN=foo.com", "Should TLS terminate on Traffic Server") +tr3.Processes.Default.Streams.All += Testers.ContainsExpression("HTTP/1.1 200 OK", "Should get a successful response") +tr3.Processes.Default.Streams.All += Testers.ContainsExpression("ok random", "Body is expected") + diff --git a/tests/gold_tests/tls/tls_tunnel_plugin_rename.test.py b/tests/gold_tests/tls/tls_tunnel_plugin_rename.test.py new file mode 100644 index 00000000000..229c4af9c5e --- /dev/null +++ b/tests/gold_tests/tls/tls_tunnel_plugin_rename.test.py @@ -0,0 +1,100 @@ +''' +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +Test.Summary = ''' +Test tunneling based on SNI renaming +''' + +# need Curl +Test.SkipUnless( + Condition.HasProgram("curl", "Curl need to be installed on system for this test to work") +) + +# Define default ATS +ts = Test.MakeATSProcess("ts", select_ports=False) +server_bar = Test.MakeOriginServer("server_bar", ssl=True) +server_random = Test.MakeOriginServer("server_random", ssl=True) + +request_bar_header = {"headers": "GET / HTTP/1.1\r\nHost: bar.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +request_random_header = {"headers": "GET / HTTP/1.1\r\nHost: random.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +response_bar_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": "ok bar"} +response_random_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": "ok random"} +server_bar.addResponse("sessionlog_bar.json", request_bar_header, response_bar_header) +server_random.addResponse("sessionlog_random.json", request_random_header, response_random_header) + +Test.PreparePlugin(os.path.join(Test.Variables.AtsTestToolsDir, 'plugins', 'ssl_sni_rename_test.cc'), ts) + +# add ssl materials like key, certificates for the server +ts.addSSLfile("ssl/signed-foo.pem") +ts.addSSLfile("ssl/signed-foo.key") +ts.addSSLfile("ssl/signed-bar.pem") +ts.addSSLfile("ssl/signed-bar.key") +ts.addSSLfile("ssl/server.pem") +ts.addSSLfile("ssl/server.key") +ts.addSSLfile("ssl/signer.pem") +ts.addSSLfile("ssl/signer.key") + +ts.Variables.ssl_port = 4443 + +# Need no remap rules. Everything should be proccessed by ssl_server_name + +# Make sure the TS server certs are different from the origin certs +ts.Disk.ssl_multicert_config.AddLine( + 'dest_ip=* ssl_cert_name=signed-foo.pem ssl_key_name=signed-foo.key' +) + +# Case 1, global config policy=permissive properties=signature +# override for foo.com policy=enforced properties=all +ts.Disk.records_config.update({ + 'proxy.config.ssl.server.cert.path': '{0}'.format(ts.Variables.SSLDir), + 'proxy.config.ssl.server.private_key.path': '{0}'.format(ts.Variables.SSLDir), + # enable ssl port + 'proxy.config.http.server_ports': '{0} {1}:proto=http2;http:ssl'.format(ts.Variables.port, ts.Variables.ssl_port), + 'proxy.config.http.connect_ports': '{0} {1} {2}'.format(ts.Variables.ssl_port,server_bar.Variables.SSL_Port,server_random.Variables.SSL_Port), + 'proxy.config.ssl.server.cipher_suite': 'ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA384:AES128-GCM-SHA256:AES256-GCM-SHA384:ECDHE-RSA-RC4-SHA:ECDHE-RSA-AES128-SHA:ECDHE-RSA-AES256-SHA:RC4-SHA:RC4-MD5:AES128-SHA:AES256-SHA:DES-CBC3-SHA!SRP:!DSS:!PSK:!aNULL:!eNULL:!SSLv2', + 'proxy.config.ssl.client.CA.cert.path': '{0}'.format(ts.Variables.SSLDir), + 'proxy.config.ssl.client.CA.cert.filename': 'signer.pem', + 'proxy.config.exec_thread.autoconfig.scale': 1.0, + 'proxy.config.url_remap.pristine_host_hdr': 1 +}) + +# bar.com should terminate. +# empty should tunnel to server_random (should not happen) +# newname should tunnel to server_bar +ts.Disk.ssl_server_name_yaml.AddLines([ + "- fqdn: newname", + " tunnel_route: localhost:{0}".format(server_bar.Variables.SSL_Port), + "- fqdn: ''", #default case + " tunnel_route: localhost:{0}".format(server_random.Variables.SSL_Port), + ]) + +# Plugin should add "newname" to the empty sni and go to _bar instead of random.com +tr = Test.AddTestRun("no-sni-tunnel-test") +tr.Processes.Default.Command = "curl --http1.1 -v -k https://127.0.0.1:{0}".format(ts.Variables.ssl_port) +tr.ReturnCode = 0 +tr.Processes.Default.StartBefore(server_bar) +tr.Processes.Default.StartBefore(server_random) +tr.Processes.Default.StartBefore(Test.Processes.ts, ready=When.PortOpen(ts.Variables.ssl_port)) +tr.StillRunningAfter = server_random +tr.StillRunningAfter = ts +tr.Processes.Default.Streams.All += Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") +tr.Processes.Default.Streams.All += Testers.ExcludesExpression("Not Found on Accelerato", "Should not try to remap on Traffic Server") +tr.Processes.Default.Streams.All += Testers.ContainsExpression("HTTP/1.1 200 OK", "Should get a successful response") +tr.Processes.Default.Streams.All += Testers.ContainsExpression("ok bar", "Body is expected") + diff --git a/tests/gold_tests/tls/tls_verify.test.py b/tests/gold_tests/tls/tls_verify.test.py index 3a33e9eeb96..52571e6c5cc 100644 --- a/tests/gold_tests/tls/tls_verify.test.py +++ b/tests/gold_tests/tls/tls_verify.test.py @@ -30,6 +30,7 @@ ts = Test.MakeATSProcess("ts", select_ports=False) server_foo = Test.MakeOriginServer("server_foo", ssl=True, options = {"--key": "{0}/signed-foo.key".format(Test.RunDirectory), "--cert": "{0}/signed-foo.pem".format(Test.RunDirectory)}) server_bar = Test.MakeOriginServer("server_bar", ssl=True, options = {"--key": "{0}/signed-bar.key".format(Test.RunDirectory), "--cert": "{0}/signed-bar.pem".format(Test.RunDirectory)}) +server_wild = Test.MakeOriginServer("server_wild", ssl=True, options = {"--key": "{0}/wild.key".format(Test.RunDirectory), "--cert": "{0}/wild-signed.pem".format(Test.RunDirectory)}) server = Test.MakeOriginServer("server", ssl=True) request_foo_header = {"headers": "GET / HTTP/1.1\r\nHost: foo.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} @@ -41,6 +42,7 @@ server_foo.addResponse("sessionlog.json", request_bad_foo_header, response_header) server_bar.addResponse("sessionlog.json", request_bar_header, response_header) server_bar.addResponse("sessionlog.json", request_bad_bar_header, response_header) +server_wild.addResponse("sessionlog.json", request_bar_header, response_header) # add ssl materials like key, certificates for the server ts.addSSLfile("ssl/signed-foo.pem") @@ -51,18 +53,24 @@ ts.addSSLfile("ssl/server.key") ts.addSSLfile("ssl/signer.pem") ts.addSSLfile("ssl/signer.key") +ts.addSSLfile("ssl/wild.key") +ts.addSSLfile("ssl/wild-signed.pem") ts.Variables.ssl_port = 4443 ts.Disk.remap_config.AddLine( - 'map / https://127.0.0.1:{0}'.format(server.Variables.Port)) + 'map / https://127.0.0.1:{0}'.format(server.Variables.SSL_Port)) ts.Disk.remap_config.AddLine( - 'map https://foo.com/ https://127.0.0.1:{0}'.format(server_foo.Variables.Port)) + 'map https://foo.com/ https://127.0.0.1:{0}'.format(server_foo.Variables.SSL_Port)) ts.Disk.remap_config.AddLine( - 'map https://bad_foo.com/ https://127.0.0.1:{0}'.format(server_foo.Variables.Port)) + 'map https://bad_foo.com/ https://127.0.0.1:{0}'.format(server_foo.Variables.SSL_Port)) ts.Disk.remap_config.AddLine( - 'map https://bar.com/ https://127.0.0.1:{0}'.format(server_bar.Variables.Port)) + 'map https://bar.com/ https://127.0.0.1:{0}'.format(server_bar.Variables.SSL_Port)) ts.Disk.remap_config.AddLine( - 'map https://bad_bar.com/ https://127.0.0.1:{0}'.format(server_bar.Variables.Port)) + 'map https://bad_bar.com/ https://127.0.0.1:{0}'.format(server_bar.Variables.SSL_Port)) +ts.Disk.remap_config.AddLine( + 'map https://foo.wild.com/ https://127.0.0.1:{0}'.format(server_wild.Variables.SSL_Port)) +ts.Disk.remap_config.AddLine( + 'map https://foo_bar.wild.com/ https://127.0.0.1:{0}'.format(server_wild.Variables.SSL_Port)) ts.Disk.ssl_multicert_config.AddLine( 'dest_ip=* ssl_cert_name=server.pem ssl_key_name=server.key' @@ -71,8 +79,6 @@ # Case 1, global config policy=permissive properties=signature # override for foo.com policy=enforced properties=all ts.Disk.records_config.update({ - 'proxy.config.diags.debug.enabled': 1, - 'proxy.config.diags.debug.tags': 'http|ssl', 'proxy.config.ssl.server.cert.path': '{0}'.format(ts.Variables.SSLDir), 'proxy.config.ssl.server.private_key.path': '{0}'.format(ts.Variables.SSLDir), # enable ssl port @@ -83,56 +89,68 @@ 'proxy.config.ssl.client.verify.server.properties': 'SIGNATURE', 'proxy.config.ssl.client.CA.cert.path': '{0}'.format(ts.Variables.SSLDir), 'proxy.config.ssl.client.CA.cert.filename': 'signer.pem', + 'proxy.config.exec_thread.autoconfig.scale': 1.0, 'proxy.config.url_remap.pristine_host_hdr': 1 }) -ts.Disk.ssl_server_name_yaml.AddLine( - '- fqdn: bar.com') -ts.Disk.ssl_server_name_yaml.AddLine( - ' verify_server_policy: ENFORCED') -ts.Disk.ssl_server_name_yaml.AddLine( - ' verify_server_properties: ALL') -ts.Disk.ssl_server_name_yaml.AddLine( - '- fqdn: bad_bar.com') -ts.Disk.ssl_server_name_yaml.AddLine( - ' verify_server_policy: ENFORCED') -ts.Disk.ssl_server_name_yaml.AddLine( - ' verify_server_properties: ALL') +ts.Disk.ssl_server_name_yaml.AddLines([ + '- fqdn: bar.com', + ' verify_server_policy: ENFORCED', + ' verify_server_properties: ALL', + '- fqdn: "*.wild.com"', + ' verify_server_policy: ENFORCED', + ' verify_server_properties: ALL', + '- fqdn: bad_bar.com', + ' verify_server_policy: ENFORCED', + ' verify_server_properties: ALL' +]) tr = Test.AddTestRun("Permissive-Test") tr.Setup.Copy("ssl/signed-foo.key") tr.Setup.Copy("ssl/signed-foo.pem") tr.Setup.Copy("ssl/signed-bar.key") tr.Setup.Copy("ssl/signed-bar.pem") -tr.Processes.Default.Command = "curl -k -H \"host: foo.com\" https://127.0.0.1:{0}".format(ts.Variables.ssl_port) +tr.Setup.Copy("ssl/wild-signed.pem") +tr.Setup.Copy("ssl/wild.key") +tr.Processes.Default.Command = "curl -v -k -H \"host: foo.com\" https://127.0.0.1:{0}".format(ts.Variables.ssl_port) tr.ReturnCode = 0 # time delay as proxy.config.http.wait_for_cache could be broken tr.Processes.Default.StartBefore(server_foo) tr.Processes.Default.StartBefore(server_bar) tr.Processes.Default.StartBefore(server) +tr.Processes.Default.StartBefore(server_wild) tr.Processes.Default.StartBefore(Test.Processes.ts, ready=When.PortOpen(ts.Variables.ssl_port)) tr.StillRunningAfter = server tr.StillRunningAfter = ts -tr.Processes.Default.TimeOut = 5 tr.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") -tr.TimeOut = 5 tr2 = Test.AddTestRun("Override-enforcing-Test") -tr2.Processes.Default.Command = "curl -k -H \"host: bar.com\" https://127.0.0.1:{0}".format(ts.Variables.ssl_port) +tr2.Processes.Default.Command = "curl -v -k -H \"host: bar.com\" https://127.0.0.1:{0}".format(ts.Variables.ssl_port) tr2.ReturnCode = 0 tr2.StillRunningAfter = server -tr2.Processes.Default.TimeOut = 5 tr2.StillRunningAfter = ts tr2.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") -tr2.TimeOut = 5 tr3 = Test.AddTestRun("Override-enforcing-Test-fail-name-check") -tr3.Processes.Default.Command = "curl -k -H \"host: bad_bar.com\" https://127.0.0.1:{0}".format(ts.Variables.ssl_port) +tr3.Processes.Default.Command = "curl -v -k -H \"host: bad_bar.com\" https://127.0.0.1:{0}".format(ts.Variables.ssl_port) tr3.Processes.Default.Streams.stdout = Testers.ContainsExpression("Could Not Connect", "Curl attempt should have failed") tr3.ReturnCode = 0 tr3.StillRunningAfter = server tr3.StillRunningAfter = ts -tr3.Processes.Default.TimeOut = 5 + +tr4 = Test.AddTestRun("Exercise-wildcard-cert-name-check") +tr4.Processes.Default.Command = "curl -v -k -H \"host: foo.wild.com\" https://127.0.0.1:{0}".format(ts.Variables.ssl_port) +tr4.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") +tr4.ReturnCode = 0 +tr4.StillRunningAfter = server +tr4.StillRunningAfter = ts + +tr5 = Test.AddTestRun("Exercise-wildcard-cert-underscore-name-check") +tr5.Processes.Default.Command = "curl -v -k -H \"host: foo_bar.wild.com\" https://127.0.0.1:{0}".format(ts.Variables.ssl_port) +tr5.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") +tr5.ReturnCode = 0 +tr5.StillRunningAfter = server +tr5.StillRunningAfter = ts # Over riding the built in ERROR check since we expect tr3 to fail ts.Disk.diags_log.Content = Testers.ExcludesExpression("verification failed", "Make sure the signatures didn't fail") diff --git a/tests/gold_tests/tls/tls_verify2.test.py b/tests/gold_tests/tls/tls_verify2.test.py index 5958c76c7ba..197b3c88add 100644 --- a/tests/gold_tests/tls/tls_verify2.test.py +++ b/tests/gold_tests/tls/tls_verify2.test.py @@ -54,15 +54,15 @@ ts.Variables.ssl_port = 4443 ts.Disk.remap_config.AddLine( - 'map / https://127.0.0.1:{0}'.format(server.Variables.Port)) + 'map https://foo.com/ https://127.0.0.1:{0}'.format(server_foo.Variables.SSL_Port)) ts.Disk.remap_config.AddLine( - 'map https://foo.com/ https://127.0.0.1:{0}'.format(server_foo.Variables.Port)) + 'map https://bad_foo.com/ https://127.0.0.1:{0}'.format(server_foo.Variables.SSL_Port)) ts.Disk.remap_config.AddLine( - 'map https://bad_foo.com/ https://127.0.0.1:{0}'.format(server_foo.Variables.Port)) + 'map https://bar.com/ https://127.0.0.1:{0}'.format(server_bar.Variables.SSL_Port)) ts.Disk.remap_config.AddLine( - 'map https://bar.com/ https://127.0.0.1:{0}'.format(server_bar.Variables.Port)) + 'map https://bad_bar.com/ https://127.0.0.1:{0}'.format(server_bar.Variables.SSL_Port)) ts.Disk.remap_config.AddLine( - 'map https://bad_bar.com/ https://127.0.0.1:{0}'.format(server_bar.Variables.Port)) + 'map / https://127.0.0.1:{0}'.format(server.Variables.SSL_Port)) ts.Disk.ssl_multicert_config.AddLine( 'dest_ip=* ssl_cert_name=server.pem ssl_key_name=server.key' @@ -71,8 +71,6 @@ # Case 1, global config policy=permissive properties=signature # override for foo.com policy=enforced properties=all ts.Disk.records_config.update({ - 'proxy.config.diags.debug.enabled': 1, - 'proxy.config.diags.debug.tags': 'http|ssl', 'proxy.config.ssl.server.cert.path': '{0}'.format(ts.Variables.SSLDir), 'proxy.config.ssl.server.private_key.path': '{0}'.format(ts.Variables.SSLDir), # enable ssl port @@ -83,6 +81,7 @@ 'proxy.config.ssl.client.verify.server.properties': 'ALL', 'proxy.config.ssl.client.CA.cert.path': '{0}'.format(ts.Variables.SSLDir), 'proxy.config.ssl.client.CA.cert.filename': 'signer.pem', + 'proxy.config.exec_thread.autoconfig.scale': 1.0, 'proxy.config.url_remap.pristine_host_hdr': 1 }) @@ -116,25 +115,20 @@ tr.Processes.Default.StartBefore(Test.Processes.ts, ready=When.PortOpen(ts.Variables.ssl_port)) tr.StillRunningAfter = server tr.StillRunningAfter = ts -tr.Processes.Default.TimeOut = 5 tr.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") -tr.TimeOut = 5 tr2 = Test.AddTestRun("override-disabled") tr2.Processes.Default.Command = "curl -k -H \"host: random.com\" https://127.0.0.1:{0}".format(ts.Variables.ssl_port) tr2.ReturnCode = 0 tr2.StillRunningAfter = server -tr2.Processes.Default.TimeOut = 5 tr2.StillRunningAfter = ts tr2.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") -tr2.TimeOut = 5 tr3 = Test.AddTestRun("override-permissive") tr3.Processes.Default.Command = "curl -k -H \"host: bar.com\" https://127.0.0.1:{0}".format(ts.Variables.ssl_port) tr3.ReturnCode = 0 tr3.StillRunningAfter = server tr3.StillRunningAfter = ts -tr3.Processes.Default.TimeOut = 5 tr3.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") tr4 = Test.AddTestRun("override-permissive-bad-name") @@ -142,7 +136,6 @@ tr4.ReturnCode = 0 tr4.StillRunningAfter = server tr4.StillRunningAfter = ts -tr4.Processes.Default.TimeOut = 5 tr4.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") tr5 = Test.AddTestRun("default-enforce-bad-sig") @@ -151,7 +144,6 @@ tr5.Processes.Default.Streams.stdout = Testers.ContainsExpression("Could Not Connect", "Curl attempt should have failed") tr5.StillRunningAfter = server tr5.StillRunningAfter = ts -tr5.Processes.Default.TimeOut = 5 tr6 = Test.AddTestRun("default-enforce-fail") tr6.Processes.Default.Command = "curl -k -H \"host: bad_foo.com\" https://127.0.0.1:{0}".format(ts.Variables.ssl_port) @@ -159,17 +151,12 @@ tr6.Processes.Default.Streams.stdout = Testers.ContainsExpression("Could Not Connect", "Curl attempt should have failed") tr6.StillRunningAfter = server tr6.StillRunningAfter = ts -tr6.Processes.Default.TimeOut = 5 -# Over riding the built in ERROR check since we expect tr4 and tr7 to fail # No name checking for the sig-only permissive override for bad_bar -ts.Disk.diags_log.Content = Testers.ExcludesExpression("WARNING: SNI \(bad_bar.com\) not in certificate", "bad_bar name checked should be skipped.") +ts.Disk.diags_log.Content += Testers.ExcludesExpression("WARNING: SNI \(bad_bar.com\) not in certificate", "bad_bar name checked should be skipped.") ts.Disk.diags_log.Content = Testers.ExcludesExpression("WARNING: SNI \(foo.com\) not in certificate", "foo name checked should be skipped.") # No checking for the self-signed on random.com. No messages ts.Disk.diags_log.Content += Testers.ExcludesExpression("WARNING: Core server certificate verification failed for \(random.com\)", "signature check for random.com should be skipped") -ts.Disk.diags_log.Content += Testers.ExcludesExpression("ERROR: SSL connection failed for 'random.com'", "random.com should not fail") -ts.Disk.diags_log.Content += Testers.ContainsExpression("ERROR: SSL connection failed for 'random2.com'", "random2.com should fail") ts.Disk.diags_log.Content += Testers.ContainsExpression("WARNING: Core server certificate verification failed for \(random2.com\)", "signature check for random.com should fail'") ts.Disk.diags_log.Content += Testers.ContainsExpression("WARNING: SNI \(bad_foo.com\) not in certificate", "bad_foo name checked should be checked.") -ts.Disk.diags_log.Content += Testers.ContainsExpression("ERROR: SSL connection failed for 'bad_foo.com'", "bad_foo.com should fail") diff --git a/tests/gold_tests/tls/tls_verify3.test.py b/tests/gold_tests/tls/tls_verify3.test.py new file mode 100644 index 00000000000..3999f657788 --- /dev/null +++ b/tests/gold_tests/tls/tls_verify3.test.py @@ -0,0 +1,146 @@ +''' +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +Test.Summary = ''' +Test tls server certificate verification options +''' + +# need Curl +Test.SkipUnless( + Condition.HasProgram("curl", "Curl need to be installed on system for this test to work") +) + +# Define default ATS +ts = Test.MakeATSProcess("ts", select_ports=False) +server_foo = Test.MakeOriginServer("server_foo", ssl=True, options = {"--key": "{0}/signed-foo.key".format(Test.RunDirectory), "--cert": "{0}/signed-foo.pem".format(Test.RunDirectory)}) +server_bar = Test.MakeOriginServer("server_bar", ssl=True, options = {"--key": "{0}/signed-bar.key".format(Test.RunDirectory), "--cert": "{0}/signed-bar.pem".format(Test.RunDirectory)}) +server = Test.MakeOriginServer("server", ssl=True) + +request_foo_header = {"headers": "GET / HTTP/1.1\r\nHost: foo.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +request_bad_foo_header = {"headers": "GET / HTTP/1.1\r\nHost: badfoo.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +request_bar_header = {"headers": "GET / HTTP/1.1\r\nHost: bar.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +request_bad_bar_header = {"headers": "GET / HTTP/1.1\r\nHost: badbar.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +response_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +server_foo.addResponse("sessionlog.json", request_foo_header, response_header) +server_foo.addResponse("sessionlog.json", request_bad_foo_header, response_header) +server_bar.addResponse("sessionlog.json", request_bar_header, response_header) +server_bar.addResponse("sessionlog.json", request_bad_bar_header, response_header) + +# add ssl materials like key, certificates for the server +ts.addSSLfile("ssl/signed-foo.pem") +ts.addSSLfile("ssl/signed-foo.key") +ts.addSSLfile("ssl/signed-bar.pem") +ts.addSSLfile("ssl/signed-bar.key") +ts.addSSLfile("ssl/server.pem") +ts.addSSLfile("ssl/server.key") +ts.addSSLfile("ssl/signer.pem") +ts.addSSLfile("ssl/signer.key") + +ts.Variables.ssl_port = 4443 +ts.Disk.remap_config.AddLine( + 'map https://foo.com:{1}/ https://127.0.0.1:{0}'.format(server_foo.Variables.SSL_Port, ts.Variables.ssl_port)) +ts.Disk.remap_config.AddLine( + 'map https://bob.foo.com:{1}/ https://127.0.0.1:{0}'.format(server_foo.Variables.SSL_Port, ts.Variables.ssl_port)) +ts.Disk.remap_config.AddLine( + 'map https://bar.com:{1}/ https://127.0.0.1:{0}'.format(server_bar.Variables.SSL_Port, ts.Variables.ssl_port)) +ts.Disk.remap_config.AddLine( + 'map https://bob.bar.com:{1}/ https://127.0.0.1:{0}'.format(server_bar.Variables.SSL_Port,ts.Variables.ssl_port)) +ts.Disk.remap_config.AddLine( + 'map / https://127.0.0.1:{0}'.format(server.Variables.SSL_Port)) + +ts.Disk.ssl_multicert_config.AddLine( + 'dest_ip=* ssl_cert_name=server.pem ssl_key_name=server.key' +) + +# Case 1, global config policy=permissive properties=signature +# override for foo.com policy=enforced properties=all +ts.Disk.records_config.update({ + 'proxy.config.ssl.server.cert.path': '{0}'.format(ts.Variables.SSLDir), + 'proxy.config.ssl.server.private_key.path': '{0}'.format(ts.Variables.SSLDir), + # enable ssl port + 'proxy.config.http.server_ports': '{0} {1}:proto=http2;http:ssl'.format(ts.Variables.port, ts.Variables.ssl_port), + 'proxy.config.ssl.server.cipher_suite': 'ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA384:AES128-GCM-SHA256:AES256-GCM-SHA384:ECDHE-RSA-RC4-SHA:ECDHE-RSA-AES128-SHA:ECDHE-RSA-AES256-SHA:RC4-SHA:RC4-MD5:AES128-SHA:AES256-SHA:DES-CBC3-SHA!SRP:!DSS:!PSK:!aNULL:!eNULL:!SSLv2', + # set global policy + 'proxy.config.ssl.client.verify.server.policy': 'PERMISSIVE', + 'proxy.config.ssl.client.verify.server.properties': 'ALL', + 'proxy.config.ssl.client.CA.cert.path': '{0}'.format(ts.Variables.SSLDir), + 'proxy.config.ssl.client.CA.cert.filename': 'signer.pem', + 'proxy.config.exec_thread.autoconfig.scale': 1.0, + 'proxy.config.url_remap.pristine_host_hdr': 1 +}) + +ts.Disk.ssl_server_name_yaml.AddLines([ + '- fqdn: bob.bar.com', + ' verify_server_policy: ENFORCED', + ' verify_server_properties: ALL', + '- fqdn: bob.*.com', + ' verify_server_policy: ENFORCED', + ' verify_server_properties: SIGNATURE', + "- fqdn: '*bar.com'", + ' verify_server_policy: DISABLED', +]) + +tr = Test.AddTestRun("foo.com Permissive-Test") +tr.Setup.Copy("ssl/signed-foo.key") +tr.Setup.Copy("ssl/signed-foo.pem") +tr.Setup.Copy("ssl/signed-bar.key") +tr.Setup.Copy("ssl/signed-bar.pem") +tr.Processes.Default.Command = "curl -v -k --resolve 'foo.com:{0}:127.0.0.1' https://foo.com:{0}".format(ts.Variables.ssl_port) +tr.ReturnCode = 0 +tr.Processes.Default.StartBefore(Test.Processes.ts, ready=When.PortOpen(ts.Variables.ssl_port)) +tr.Processes.Default.StartBefore(server_foo) +tr.Processes.Default.StartBefore(server_bar) +tr.Processes.Default.StartBefore(server) +tr.StillRunningAfter = server +tr.StillRunningAfter = ts +tr.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") + +tr = Test.AddTestRun("my.foo.com Permissive-Test log failure") +tr.Processes.Default.Command = "curl -v -k --resolve 'my.foo.com:{0}:127.0.0.1' https://my.foo.com:{0}".format(ts.Variables.ssl_port) +tr.ReturnCode = 0 +tr.StillRunningAfter = server +tr.StillRunningAfter = ts +tr.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") + + +tr2 = Test.AddTestRun("bob.bar.com Override-enforcing-Test") +tr2.Processes.Default.Command = "curl -v -k --resolve 'bob.bar.com:{0}:127.0.0.1' https://bob.bar.com:{0}/".format(ts.Variables.ssl_port) +tr2.ReturnCode = 0 +tr2.StillRunningAfter = server +tr2.StillRunningAfter = ts +tr2.Processes.Default.Streams.stdout = Testers.ContainsExpression("Could Not Connect", "Curl attempt should have succeeded") + +tr3 = Test.AddTestRun("bob.foo.com override-enforcing-name-test") +tr3.Processes.Default.Command = "curl -v -k --resolve 'bob.foo.com:{0}:127.0.0.1' https://bob.foo.com:{0}/".format(ts.Variables.ssl_port) +tr3.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could Not Connect", "Curl attempt should not fail") +tr3.ReturnCode = 0 +tr3.StillRunningAfter = server +tr3.StillRunningAfter = ts + +tr3 = Test.AddTestRun("random.bar.com override-no-test") +tr3.Processes.Default.Command = "curl -v -k --resolve 'random.bar.com:{0}:127.0.0.1' https://random.bar.com:{0}".format(ts.Variables.ssl_port) +tr3.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could Not Connect", "Curl attempt should not fail") +tr3.ReturnCode = 0 +tr3.StillRunningAfter = server +tr3.StillRunningAfter = ts + + +# Over riding the built in ERROR check since we expect tr3 to fail +ts.Disk.diags_log.Content = Testers.ContainsExpression("WARNING: SNI \(bob.bar.com\) not in certificate", "Make sure bob.bar name checked failed.") +ts.Disk.diags_log.Content += Testers.ContainsExpression("WARNING: Core server certificate verification failed for \(my.foo.com\). Action=Continue", "Make sure default permissive action takes") diff --git a/tests/gold_tests/tls/tls_verify_base.test.py b/tests/gold_tests/tls/tls_verify_base.test.py new file mode 100644 index 00000000000..1b896ff6aac --- /dev/null +++ b/tests/gold_tests/tls/tls_verify_base.test.py @@ -0,0 +1,136 @@ +''' +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +Test.Summary = ''' +Test tls server certificate verification options +''' + +# need Curl +Test.SkipUnless( + Condition.HasProgram("curl", "Curl need to be installed on system for this test to work") +) + +# Define default ATS +ts = Test.MakeATSProcess("ts", select_ports=False) +server_foo = Test.MakeOriginServer("server_foo", ssl=True, options = {"--key": "{0}/signed-foo.key".format(Test.RunDirectory), "--cert": "{0}/signed-foo.pem".format(Test.RunDirectory)}) +server_bar = Test.MakeOriginServer("server_bar", ssl=True, options = {"--key": "{0}/signed-bar.key".format(Test.RunDirectory), "--cert": "{0}/signed-bar.pem".format(Test.RunDirectory)}) +server = Test.MakeOriginServer("server", ssl=True) + +request_foo_header = {"headers": "GET / HTTP/1.1\r\nHost: foo.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +request_bad_foo_header = {"headers": "GET / HTTP/1.1\r\nHost: badfoo.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +request_bar_header = {"headers": "GET / HTTP/1.1\r\nHost: bar.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +request_bad_bar_header = {"headers": "GET / HTTP/1.1\r\nHost: badbar.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +response_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +server_foo.addResponse("sessionlog.json", request_foo_header, response_header) +server_foo.addResponse("sessionlog.json", request_bad_foo_header, response_header) +server_bar.addResponse("sessionlog.json", request_bar_header, response_header) +server_bar.addResponse("sessionlog.json", request_bad_bar_header, response_header) + +# add ssl materials like key, certificates for the server +ts.addSSLfile("ssl/signed-foo.pem") +ts.addSSLfile("ssl/signed-foo.key") +ts.addSSLfile("ssl/signed-bar.pem") +ts.addSSLfile("ssl/signed-bar.key") +ts.addSSLfile("ssl/server.pem") +ts.addSSLfile("ssl/server.key") +ts.addSSLfile("ssl/signer.pem") +ts.addSSLfile("ssl/signer.key") + +ts.Variables.ssl_port = 4443 +ts.Disk.remap_config.AddLine( + 'map / https://127.0.0.1:{0}'.format(server.Variables.SSL_Port)) +ts.Disk.remap_config.AddLine( + 'map https://foo.com/ https://127.0.0.1:{0}'.format(server_foo.Variables.SSL_Port)) +ts.Disk.remap_config.AddLine( + 'map https://bad_foo.com/ https://127.0.0.1:{0}'.format(server_foo.Variables.SSL_Port)) +ts.Disk.remap_config.AddLine( + 'map https://bar.com/ https://127.0.0.1:{0}'.format(server_bar.Variables.SSL_Port)) +ts.Disk.remap_config.AddLine( + 'map https://bad_bar.com/ https://127.0.0.1:{0}'.format(server_bar.Variables.SSL_Port)) + +ts.Disk.ssl_multicert_config.AddLine( + 'dest_ip=* ssl_cert_name=server.pem ssl_key_name=server.key' +) + +# Case 1, global config policy=permissive properties=signature +# override for foo.com policy=enforced properties=all +ts.Disk.records_config.update({ + 'proxy.config.ssl.server.cert.path': '{0}'.format(ts.Variables.SSLDir), + 'proxy.config.ssl.server.private_key.path': '{0}'.format(ts.Variables.SSLDir), + # enable ssl port + 'proxy.config.http.server_ports': '{0} {1}:proto=http2;http:ssl'.format(ts.Variables.port, ts.Variables.ssl_port), + 'proxy.config.ssl.server.cipher_suite': 'ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA384:AES128-GCM-SHA256:AES256-GCM-SHA384:ECDHE-RSA-RC4-SHA:ECDHE-RSA-AES128-SHA:ECDHE-RSA-AES256-SHA:RC4-SHA:RC4-MD5:AES128-SHA:AES256-SHA:DES-CBC3-SHA!SRP:!DSS:!PSK:!aNULL:!eNULL:!SSLv2', + # set global policy + 'proxy.config.ssl.client.verify.server': 2, + 'proxy.config.ssl.client.CA.cert.path': '{0}'.format(ts.Variables.SSLDir), + 'proxy.config.ssl.client.CA.cert.filename': 'signer.pem', + 'proxy.config.url_remap.pristine_host_hdr': 1, + 'proxy.config.exec_thread.autoconfig.scale': 1.0, + 'proxy.config.ssl.client.sni_policy': 'host' +}) + +ts.Disk.ssl_server_name_yaml.AddLines([ + '- fqdn: bar.com', + ' verify_server_policy: ENFORCED', + ' verify_server_properties: ALL', + '- fqdn: bad_bar.com', + ' verify_server_policy: ENFORCED', + ' verify_server_properties: ALL' +]) + +tr = Test.AddTestRun("Permissive-Test") +tr.Setup.Copy("ssl/signed-foo.key") +tr.Setup.Copy("ssl/signed-foo.pem") +tr.Setup.Copy("ssl/signed-bar.key") +tr.Setup.Copy("ssl/signed-bar.pem") +tr.Processes.Default.Command = "curl -v -k -H \"host: foo.com\" https://127.0.0.1:{0}".format(ts.Variables.ssl_port) +tr.ReturnCode = 0 +tr.Processes.Default.StartBefore(server_foo) +tr.Processes.Default.StartBefore(server_bar) +tr.Processes.Default.StartBefore(server) +tr.Processes.Default.StartBefore(Test.Processes.ts, ready=When.PortOpen(ts.Variables.ssl_port)) +tr.StillRunningAfter = server +tr.StillRunningAfter = ts +tr.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") + +tr = Test.AddTestRun("Permissive-Test with logged failure") +tr.Processes.Default.Command = "curl -v -k -H \"host: random.com\" https://127.0.0.1:{0}".format(ts.Variables.ssl_port) +tr.ReturnCode = 0 +tr.StillRunningAfter = server +tr.StillRunningAfter = ts +tr.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") + + +tr2 = Test.AddTestRun("Override-enforcing-Test") +tr2.Processes.Default.Command = "curl -v -k -H \"host: bar.com\" https://127.0.0.1:{0}".format(ts.Variables.ssl_port) +tr2.ReturnCode = 0 +tr2.StillRunningAfter = server +tr2.StillRunningAfter = ts +tr2.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") + +tr3 = Test.AddTestRun("Override-enforcing-Test-fail-name-check") +tr3.Processes.Default.Command = "curl -v -k -H \"host: bad_bar.com\" https://127.0.0.1:{0}".format(ts.Variables.ssl_port) +tr3.Processes.Default.Streams.stdout = Testers.ContainsExpression("Could Not Connect", "Curl attempt should have failed") +tr3.ReturnCode = 0 +tr3.StillRunningAfter = server +tr3.StillRunningAfter = ts + +# Over riding the built in ERROR check since we expect tr3 to fail +ts.Disk.diags_log.Content = Testers.ContainsExpression("WARNING: SNI \(bad_bar.com\) not in certificate. Action=Terminate", "Make sure bad_bar name checked failed.") +ts.Disk.diags_log.Content += Testers.ContainsExpression("WARNING: SNI \(random.com\) not in certificate. Action=Continue ", "Permissive failure for random") diff --git a/tests/gold_tests/tls/tls_verify_ca_override.test.py b/tests/gold_tests/tls/tls_verify_ca_override.test.py new file mode 100644 index 00000000000..a2c5acf0047 --- /dev/null +++ b/tests/gold_tests/tls/tls_verify_ca_override.test.py @@ -0,0 +1,129 @@ +''' +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +Test.Summary = ''' +Test tls server certificate verification options. Exercise conf_remap for ca bundle +''' + +# need Curl +Test.SkipUnless( + Condition.HasProgram("curl", "Curl need to be installed on system for this test to work") +) + +# Define default ATS +ts = Test.MakeATSProcess("ts", select_ports=False) +server1 = Test.MakeOriginServer("server1", ssl=True, options = {"--key": "{0}/signed-foo.key".format(Test.RunDirectory), "--cert": "{0}/signed-foo.pem".format(Test.RunDirectory)}) +server2 = Test.MakeOriginServer("server2", ssl=True, options = {"--key": "{0}/signed-foo.key".format(Test.RunDirectory), "--cert": "{0}/signed2-foo.pem".format(Test.RunDirectory)}) + +request_foo_header = {"headers": "GET / HTTP/1.1\r\nHost: foo.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +request_bad_foo_header = {"headers": "GET / HTTP/1.1\r\nHost: bad_foo.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +request_bar_header = {"headers": "GET / HTTP/1.1\r\nHost: bar.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +request_bad_bar_header = {"headers": "GET / HTTP/1.1\r\nHost: bad_bar.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +response_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +server1.addResponse("sessionlog.json", request_foo_header, response_header) +server1.addResponse("sessionlog.json", request_bad_foo_header, response_header) +server2.addResponse("sessionlog.json", request_bar_header, response_header) +server2.addResponse("sessionlog.json", request_bad_bar_header, response_header) + +# add ssl materials like key, certificates for the server +ts.addSSLfile("ssl/signed-foo.pem") +ts.addSSLfile("ssl/signed2-foo.pem") +ts.addSSLfile("ssl/signed-foo.key") +ts.addSSLfile("ssl/server.pem") +ts.addSSLfile("ssl/server.key") +ts.addSSLfile("ssl/signer.pem") +ts.addSSLfile("ssl/signer.key") +ts.addSSLfile("ssl/signer2.pem") +ts.addSSLfile("ssl/signer2.key") + +ts.Variables.ssl_port = 4443 +ts.Disk.remap_config.AddLine( + 'map /case1 https://127.0.0.1:{0}/ @plugin=conf_remap.so @pparam=proxy.config.ssl.client.CA.cert.filename={1}/{2}'.format(server1.Variables.SSL_Port, ts.Variables.SSLDir, "signer.pem") +) +ts.Disk.remap_config.AddLine( + 'map /badcase1 https://127.0.0.1:{0}/ @plugin=conf_remap.so @pparam=proxy.config.ssl.client.CA.cert.filename={1}/{2}'.format(server1.Variables.SSL_Port, ts.Variables.SSLDir, "signer2.pem") +) +ts.Disk.remap_config.AddLine( + 'map /case2 https://127.0.0.1:{0}/ @plugin=conf_remap.so @pparam=proxy.config.ssl.client.CA.cert.filename={1}/{2}'.format(server2.Variables.SSL_Port, ts.Variables.SSLDir, "signer2.pem") +) +ts.Disk.remap_config.AddLine( + 'map /badcase2 https://127.0.0.1:{0}/ @plugin=conf_remap.so @pparam=proxy.config.ssl.client.CA.cert.filename={1}/{2}'.format(server2.Variables.SSL_Port, ts.Variables.SSLDir, "signer.pem") +) + +ts.Disk.ssl_multicert_config.AddLine( + 'dest_ip=* ssl_cert_name=server.pem ssl_key_name=server.key' +) + +# Case 1, global config policy=permissive properties=signature +# override for foo.com policy=enforced properties=all +ts.Disk.records_config.update({ + 'proxy.config.ssl.server.cert.path': '{0}'.format(ts.Variables.SSLDir), + 'proxy.config.ssl.server.private_key.path': '{0}'.format(ts.Variables.SSLDir), + # enable ssl port + 'proxy.config.http.server_ports': '{0} {1}:proto=http2;http:ssl'.format(ts.Variables.port, ts.Variables.ssl_port), + 'proxy.config.ssl.server.cipher_suite': 'ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA384:AES128-GCM-SHA256:AES256-GCM-SHA384:ECDHE-RSA-RC4-SHA:ECDHE-RSA-AES128-SHA:ECDHE-RSA-AES256-SHA:RC4-SHA:RC4-MD5:AES128-SHA:AES256-SHA:DES-CBC3-SHA!SRP:!DSS:!PSK:!aNULL:!eNULL:!SSLv2', + # set global policy + 'proxy.config.ssl.client.verify.server.policy': 'ENFORCED', + 'proxy.config.ssl.client.verify.server.properties': 'SIGNATURE', + 'proxy.config.ssl.client.CA.cert.path': '/tmp', + 'proxy.config.ssl.client.CA.cert.filename': '{0}/signer.pem'.format(ts.Variables.SSLDir), + 'proxy.config.exec_thread.autoconfig.scale': 1.0, + 'proxy.config.url_remap.pristine_host_hdr': 1 +}) + +# Should succeed +tr = Test.AddTestRun("Use corrcect ca bundle for server 1") +tr.Processes.Default.Command = 'curl -k -H \"host: foo.com\" http://127.0.0.1:{0}/case1'.format(ts.Variables.port) +tr.ReturnCode = 0 +tr.Setup.Copy("ssl/signed-foo.key") +tr.Setup.Copy("ssl/signed-foo.pem") +tr.Setup.Copy("ssl/signed2-foo.pem") +tr.Processes.Default.StartBefore(server1) +tr.Processes.Default.StartBefore(server2) +tr.Processes.Default.StartBefore(Test.Processes.ts, ready=When.PortOpen(ts.Variables.port)) +tr.StillRunningAfter = server1 +tr.StillRunningAfter = ts +# Should succed. No message +tr.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") + +tr2 = Test.AddTestRun("Use incorrect ca bundle for server 1") +tr2.Processes.Default.Command = "curl -k -H \"host: bar.com\" http://127.0.0.1:{0}/badcase1".format(ts.Variables.port) +tr2.ReturnCode = 0 +tr2.StillRunningAfter = server1 +tr2.StillRunningAfter = ts +# Should succeed, but will be message in log about name mismatch +tr2.Processes.Default.Streams.stdout = Testers.ContainsExpression("Could Not Connect", "Curl attempt should have succeeded") + +tr2 = Test.AddTestRun("Use currect ca bundle for server 2") +tr2.Processes.Default.Command = "curl -k -H \"host: random.com\" http://127.0.0.1:{0}/case2".format(ts.Variables.port) +tr2.ReturnCode = 0 +tr2.StillRunningAfter = server2 +tr2.StillRunningAfter = ts +# Should succeed, but will be message in log about signature +tr2.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") + +tr3 = Test.AddTestRun("User incorrect ca bundle for server 2") +tr3.Processes.Default.Command = "curl -k -H \"host: foo.com\" http://127.0.0.1:{0}/badcase2".format(ts.Variables.port) +tr3.ReturnCode = 0 +tr3.StillRunningAfter = server2 +tr3.StillRunningAfter = ts +# Should succeed. No error messages +tr3.Processes.Default.Streams.stdout = Testers.ContainsExpression("Could Not Connect", "Curl attempt should have succeeded") + + diff --git a/tests/gold_tests/tls/tls_verify_not_pristine.test.py b/tests/gold_tests/tls/tls_verify_not_pristine.test.py new file mode 100644 index 00000000000..e416d94d436 --- /dev/null +++ b/tests/gold_tests/tls/tls_verify_not_pristine.test.py @@ -0,0 +1,109 @@ +''' +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +Test.Summary = ''' +Test tls server certificate verification without a pristine host header +''' + +# need Curl +Test.SkipUnless( + Condition.HasProgram("curl", "Curl need to be installed on system for this test to work") +) + +# Define default ATS +ts = Test.MakeATSProcess("ts", select_ports=False) +server_foo = Test.MakeOriginServer("server_foo", ssl=True, options = {"--key": "{0}/signed-foo.key".format(Test.RunDirectory), "--cert": "{0}/signed-foo.pem".format(Test.RunDirectory)}) +server = Test.MakeOriginServer("server", ssl=True) +dns = Test.MakeDNServer("dns") + +request_foo_header = {"headers": "GET / HTTP/1.1\r\nHost: foo.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +request_bad_foo_header = {"headers": "GET / HTTP/1.1\r\nHost: badfoo.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +response_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +server_foo.addResponse("sessionlog.json", request_foo_header, response_header) +server_foo.addResponse("sessionlog.json", request_bad_foo_header, response_header) + +# add ssl materials like key, certificates for the server +ts.addSSLfile("ssl/signed-foo.pem") +ts.addSSLfile("ssl/signed-foo.key") +ts.addSSLfile("ssl/signed-bar.pem") +ts.addSSLfile("ssl/signed-bar.key") +ts.addSSLfile("ssl/server.pem") +ts.addSSLfile("ssl/server.key") +ts.addSSLfile("ssl/signer.pem") +ts.addSSLfile("ssl/signer.key") + +ts.Variables.ssl_port = 4443 +ts.Disk.remap_config.AddLine( + 'map https://bar.com:{0}/ https://foo.com:{1}'.format(ts.Variables.ssl_port, server_foo.Variables.SSL_Port)) +ts.Disk.remap_config.AddLine( + 'map https://foo.com:{0}/ https://bar.com:{1}'.format(ts.Variables.ssl_port, server_foo.Variables.SSL_Port)) + +ts.Disk.ssl_multicert_config.AddLine( + 'dest_ip=* ssl_cert_name=server.pem ssl_key_name=server.key' +) + +# Case 1, global config policy=permissive properties=signature +# override for foo.com policy=enforced properties=all +ts.Disk.records_config.update({ + 'proxy.config.ssl.server.cert.path': '{0}'.format(ts.Variables.SSLDir), + 'proxy.config.ssl.server.private_key.path': '{0}'.format(ts.Variables.SSLDir), + # enable ssl port + 'proxy.config.http.server_ports': '{0} {1}:proto=http2;http:ssl'.format(ts.Variables.port, ts.Variables.ssl_port), + 'proxy.config.ssl.server.cipher_suite': 'ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA384:AES128-GCM-SHA256:AES256-GCM-SHA384:ECDHE-RSA-RC4-SHA:ECDHE-RSA-AES128-SHA:ECDHE-RSA-AES256-SHA:RC4-SHA:RC4-MD5:AES128-SHA:AES256-SHA:DES-CBC3-SHA!SRP:!DSS:!PSK:!aNULL:!eNULL:!SSLv2', + # set global policy + 'proxy.config.ssl.client.verify.server.policy': 'ENFORCED', + 'proxy.config.ssl.client.verify.server.properties': 'ALL', + 'proxy.config.ssl.client.CA.cert.path': '{0}'.format(ts.Variables.SSLDir), + 'proxy.config.ssl.client.CA.cert.filename': 'signer.pem', + 'proxy.config.url_remap.pristine_host_hdr': 0, + 'proxy.config.dns.nameservers': '127.0.0.1:{0}'.format(dns.Variables.Port), + 'proxy.config.exec_thread.autoconfig.scale': 1.0, + 'proxy.config.dns.resolv_conf': 'NULL' +}) + +dns.addRecords(records={"foo.com.": ["127.0.0.1"]}) +dns.addRecords(records={"bar.com.": ["127.0.0.1"]}) + +# bar.com in. foo.com out. Should verify +tr = Test.AddTestRun("Enforced-Good-Test") +tr.Setup.Copy("ssl/signed-foo.key") +tr.Setup.Copy("ssl/signed-foo.pem") +tr.Setup.Copy("ssl/signed-bar.key") +tr.Setup.Copy("ssl/signed-bar.pem") +tr.Processes.Default.Command = "curl -v --resolve 'bar.com:{0}:127.0.0.1' -k https://bar.com:{0}".format(ts.Variables.ssl_port) +tr.ReturnCode = 0 +# time delay as proxy.config.http.wait_for_cache could be broken +tr.Processes.Default.StartBefore(server_foo) +tr.Processes.Default.StartBefore(dns) +tr.Processes.Default.StartBefore(Test.Processes.ts, ready=When.PortOpen(ts.Variables.ssl_port)) +tr.StillRunningAfter = server +tr.StillRunningAfter = ts +tr.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") + +# foo.com in. bar.com out. Should not verify +tr2 = Test.AddTestRun("Enforced-bad-test") +tr2.Processes.Default.Command = "curl -v -k --resolve 'foo.com:{0}:127.0.0.1' https://foo.com:{0}".format(ts.Variables.ssl_port) +tr2.ReturnCode = 0 +tr2.StillRunningAfter = server +tr2.StillRunningAfter = ts +tr2.Processes.Default.Streams.stdout = Testers.ContainsExpression("Could Not Connect", "Curl attempt should not have succeeded") + +# Over riding the built in ERROR check since we expect tr3 to fail +ts.Disk.diags_log.Content = Testers.ExcludesExpression("verification failed", "Make sure the signatures didn't fail") +ts.Disk.diags_log.Content += Testers.ContainsExpression("WARNING: SNI \(bar.com\) not in certificate", "Make sure bad_bar name checked failed.") diff --git a/tests/gold_tests/tls/tls_verify_override.test.py b/tests/gold_tests/tls/tls_verify_override.test.py new file mode 100644 index 00000000000..fe7147c8dd4 --- /dev/null +++ b/tests/gold_tests/tls/tls_verify_override.test.py @@ -0,0 +1,233 @@ +''' +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +Test.Summary = ''' +Test tls server certificate verification options. Exercise conf_remap +''' + +# need Curl +Test.SkipUnless( + Condition.HasProgram("curl", "Curl need to be installed on system for this test to work") +) + +# Define default ATS +ts = Test.MakeATSProcess("ts", select_ports=False) +server_foo = Test.MakeOriginServer("server_foo", ssl=True, options = {"--key": "{0}/signed-foo.key".format(Test.RunDirectory), "--cert": "{0}/signed-foo.pem".format(Test.RunDirectory)}) +server_bar = Test.MakeOriginServer("server_bar", ssl=True, options = {"--key": "{0}/signed-bar.key".format(Test.RunDirectory), "--cert": "{0}/signed-bar.pem".format(Test.RunDirectory)}) +server = Test.MakeOriginServer("server", ssl=True) + +dns = Test.MakeDNServer("dns") + +request_foo_header = {"headers": "GET / HTTP/1.1\r\nHost: foo.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +request_bad_foo_header = {"headers": "GET / HTTP/1.1\r\nHost: bad_foo.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +request_bar_header = {"headers": "GET / HTTP/1.1\r\nHost: bar.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +request_bad_bar_header = {"headers": "GET / HTTP/1.1\r\nHost: bad_bar.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +response_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +server_foo.addResponse("sessionlog.json", request_foo_header, response_header) +server_foo.addResponse("sessionlog.json", request_bad_foo_header, response_header) +server_bar.addResponse("sessionlog.json", request_bar_header, response_header) +server_bar.addResponse("sessionlog.json", request_bad_bar_header, response_header) + +# add ssl materials like key, certificates for the server +ts.addSSLfile("ssl/signed-foo.pem") +ts.addSSLfile("ssl/signed-foo.key") +ts.addSSLfile("ssl/signed-bar.pem") +ts.addSSLfile("ssl/signed-bar.key") +ts.addSSLfile("ssl/server.pem") +ts.addSSLfile("ssl/server.key") +ts.addSSLfile("ssl/signer.pem") +ts.addSSLfile("ssl/signer.key") + +ts.Variables.ssl_port = 4443 +ts.Disk.remap_config.AddLine( + 'map http://foo.com/basictobar https://bar.com:{0}'.format(server_bar.Variables.SSL_Port)) +ts.Disk.remap_config.AddLine( + 'map http://foo.com/basic https://foo.com:{0}'.format(server_foo.Variables.SSL_Port)) +ts.Disk.remap_config.AddLine( + 'map http://foo.com/override https://foo.com:{0} @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.policy=ENFORCED'.format(server_foo.Variables.SSL_Port)) +ts.Disk.remap_config.AddLine( + 'map http://bar.com/basic https://bar.com:{0}'.format(server_foo.Variables.SSL_Port)) +ts.Disk.remap_config.AddLine( + 'map http://bar.com/overridedisabled https://bar.com:{0} @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.policy=DISABLED'.format(server_foo.Variables.SSL_Port)) +ts.Disk.remap_config.AddLine( + 'map http://bar.com/overridesignature https://bar.com:{0} @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.properties=SIGNATURE @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.policy=ENFORCED'.format(server_foo.Variables.SSL_Port)) +ts.Disk.remap_config.AddLine( + 'map http://bar.com/overrideenforced https://bar.com:{0} @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.policy=ENFORCED'.format(server_foo.Variables.SSL_Port)) +ts.Disk.remap_config.AddLine( + 'map /basic https://random.com:{0}'.format(server.Variables.SSL_Port)) +ts.Disk.remap_config.AddLine( + 'map /overrideenforce https://127.0.0.1:{0} @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.policy=ENFORCED'.format(server.Variables.SSL_Port)) +ts.Disk.remap_config.AddLine( + 'map /overridename https://127.0.0.1:{0} @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.properties=NAME'.format(server.Variables.SSL_Port)) +ts.Disk.remap_config.AddLine( + 'map /snipolicyfooremap https://foo.com:{0} @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.properties=NAME @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.policy=ENFORCED @plugin=conf_remap.so @pparam=proxy.config.ssl.client.sni_policy=remap'.format(server_bar.Variables.SSL_Port)) +ts.Disk.remap_config.AddLine( + 'map /snipolicyfoohost https://foo.com:{0} @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.properties=NAME @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.policy=ENFORCED @plugin=conf_remap.so @pparam=proxy.config.ssl.client.sni_policy=host'.format(server_bar.Variables.SSL_Port)) +ts.Disk.remap_config.AddLine( + 'map /snipolicybarremap https://bar.com:{0} @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.properties=NAME @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.policy=ENFORCED @plugin=conf_remap.so @pparam=proxy.config.ssl.client.sni_policy=remap'.format(server_bar.Variables.SSL_Port)) +ts.Disk.remap_config.AddLine( + 'map /snipolicybarhost https://bar.com:{0} @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.properties=NAME @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.policy=ENFORCED @plugin=conf_remap.so @pparam=proxy.config.ssl.client.sni_policy=host'.format(server_bar.Variables.SSL_Port)) + +ts.Disk.ssl_multicert_config.AddLine( + 'dest_ip=* ssl_cert_name=server.pem ssl_key_name=server.key' +) + +# Case 1, global config policy=permissive properties=signature +# override for foo.com policy=enforced properties=all +ts.Disk.records_config.update({ + 'proxy.config.ssl.server.cert.path': '{0}'.format(ts.Variables.SSLDir), + 'proxy.config.ssl.server.private_key.path': '{0}'.format(ts.Variables.SSLDir), + # enable ssl port + 'proxy.config.http.server_ports': '{0} {1}:proto=http2;http:ssl'.format(ts.Variables.port, ts.Variables.ssl_port), + 'proxy.config.ssl.server.cipher_suite': 'ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA384:AES128-GCM-SHA256:AES256-GCM-SHA384:ECDHE-RSA-RC4-SHA:ECDHE-RSA-AES128-SHA:ECDHE-RSA-AES256-SHA:RC4-SHA:RC4-MD5:AES128-SHA:AES256-SHA:DES-CBC3-SHA!SRP:!DSS:!PSK:!aNULL:!eNULL:!SSLv2', + # set global policy + 'proxy.config.ssl.client.verify.server.policy': 'PERMISSIVE', + 'proxy.config.ssl.client.verify.server.properties': 'ALL', + 'proxy.config.ssl.client.CA.cert.path': '{0}'.format(ts.Variables.SSLDir), + 'proxy.config.ssl.client.CA.cert.filename': 'signer.pem', + 'proxy.config.url_remap.pristine_host_hdr': 1, + 'proxy.config.dns.nameservers': '127.0.0.1:{0}'.format(dns.Variables.Port), + 'proxy.config.dns.resolv_conf': 'NULL', + 'proxy.config.exec_thread.autoconfig.scale': 1.0, + 'proxy.config.ssl.client.sni_policy': 'remap' +}) + +dns.addRecords(records={"foo.com.": ["127.0.0.1"]}) +dns.addRecords(records={"bar.com.": ["127.0.0.1"]}) +dns.addRecords(records={"random.com.": ["127.0.0.1"]}) + +# Should succeed without message +tr = Test.AddTestRun("default-permissive-success") +tr.Setup.Copy("ssl/signed-foo.key") +tr.Setup.Copy("ssl/signed-foo.pem") +tr.Setup.Copy("ssl/signed-bar.key") +tr.Setup.Copy("ssl/signed-bar.pem") +tr.Processes.Default.Command = 'curl -k -H \"host: foo.com\" http://127.0.0.1:{0}/basic'.format(ts.Variables.port) +tr.ReturnCode = 0 +tr.Processes.Default.StartBefore(dns) +tr.Processes.Default.StartBefore(server_foo) +tr.Processes.Default.StartBefore(server_bar) +tr.Processes.Default.StartBefore(server) +tr.Processes.Default.StartBefore(Test.Processes.ts, ready=When.PortOpen(ts.Variables.port)) +tr.StillRunningAfter = server +tr.StillRunningAfter = ts +# Should succed. No message +tr.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") + +tr2 = Test.AddTestRun("default-permissive-fail") +tr2.Processes.Default.Command = "curl -k -H \"host: bar.com\" http://127.0.0.1:{0}/basic".format(ts.Variables.port) +tr2.ReturnCode = 0 +tr2.StillRunningAfter = server +tr2.StillRunningAfter = ts +# Should succeed, but will be message in log about name mismatch +tr2.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") + +tr2 = Test.AddTestRun("default-permissive-fail2") +tr2.Processes.Default.Command = "curl -k -H \"host: random.com\" http://127.0.0.1:{0}/basic".format(ts.Variables.port) +tr2.ReturnCode = 0 +tr2.StillRunningAfter = server +tr2.StillRunningAfter = ts +# Should succeed, but will be message in log about signature +tr2.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") + +tr3 = Test.AddTestRun("default-foo-to-bar") +tr3.Processes.Default.Command = "curl -k -v -H \"host: foo.com\" http://127.0.0.1:{0}/basictobar".format(ts.Variables.port) +tr3.ReturnCode = 0 +tr3.StillRunningAfter = server +tr3.StillRunningAfter = ts +# Should succeed. No error messages +tr3.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") + +tr3 = Test.AddTestRun("override-foo") +tr3.Processes.Default.Command = "curl -k -H \"host: foo.com\" http://127.0.0.1:{0}/override".format(ts.Variables.port) +tr3.ReturnCode = 0 +tr3.StillRunningAfter = server +tr3.StillRunningAfter = ts +# Should succeed. No error messages +tr3.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") + +tr4 = Test.AddTestRun("override-bar-disabled") +tr4.Processes.Default.Command = "curl -k -H \"host: bad_bar.com\" http://127.0.0.1:{0}/overridedisabled".format(ts.Variables.port) +tr4.ReturnCode = 0 +tr4.StillRunningAfter = server +tr4.StillRunningAfter = ts +# Succeed. No error messages +tr4.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") + +tr5 = Test.AddTestRun("override-bar-signature-enforced") +tr5.Processes.Default.Command = "curl -k -H \"host: bar.com\" http://127.0.0.1:{0}/overridesignature".format(ts.Variables.port) +tr5.ReturnCode = 0 +tr5.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") +tr5.StillRunningAfter = server +tr5.StillRunningAfter = ts + +tr6 = Test.AddTestRun("override-bar-enforced") +tr6.Processes.Default.Command = "curl -k -H \"host: bar.com\" http://127.0.0.1:{0}/overrideenforced".format(ts.Variables.port) +tr6.ReturnCode = 0 +# Should fail +tr6.Processes.Default.Streams.stdout = Testers.ContainsExpression("Could Not Connect", "Curl attempt should have failed") +tr6.StillRunningAfter = server +tr6.StillRunningAfter = ts + +# Should succeed +tr = Test.AddTestRun("foo-to-bar-sni-policy-remap") +tr.Processes.Default.Command = "curl -k -H \"host: foo.com\" http://127.0.0.1:{0}/snipolicybarremap".format(ts.Variables.port) +tr.ReturnCode = 0 +tr.StillRunningAfter = server +tr.StillRunningAfter = ts +tr.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could not connect", "Curl attempt should succeed") + +# Should fail +tr = Test.AddTestRun("foo-to-bar-sni-policy-host") +tr.Processes.Default.Command = "curl -k -H \"host: foo.com\" http://127.0.0.1:{0}/snipolicybarhost".format(ts.Variables.port) +tr.ReturnCode = 0 +tr.StillRunningAfter = server +tr.StillRunningAfter = ts +tr.Processes.Default.Streams.stdout = Testers.ContainsExpression("Could not connect", "Curl attempt should fail") + +# Should fail +tr = Test.AddTestRun("bar-to-foo-sni-policy-remap") +tr.Processes.Default.Command = "curl -k -H \"host: bar.com\" http://127.0.0.1:{0}/snipolicyfooremap".format(ts.Variables.port) +tr.ReturnCode = 0 +tr.StillRunningAfter = server +tr.StillRunningAfter = ts +tr.Processes.Default.Streams.stdout = Testers.ContainsExpression("Could not connect", "Curl attempt should fail") + +# Should succeed +tr = Test.AddTestRun("bar-to-foo-sni-policy-host") +tr.Processes.Default.Command = "curl -k -H \"host: bar.com\" http://127.0.0.1:{0}/snipolicyfoohost".format(ts.Variables.port) +tr.ReturnCode = 0 +tr.StillRunningAfter = server +tr.StillRunningAfter = ts +tr.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could not connect", "Curl attempt should succeed") + +# Over riding the built in ERROR check since we expect some cases to fail + +# checks on random.com should fail with message only +ts.Disk.diags_log.Content = Testers.ContainsExpression("WARNING: Core server certificate verification failed for \(random.com\). Action=Continue Error=self signed certificate server=random.com\(127.0.0.1\) depth=0", "Warning for self signed certificate") +# permissive failure for bar.com +ts.Disk.diags_log.Content += Testers.ContainsExpression("WARNING: SNI \(bar.com\) not in certificate. Action=Continue server=bar.com\(127.0.0.1\)", "Warning on missing name for bar.com") +# name check failure for random.com +ts.Disk.diags_log.Content += Testers.ContainsExpression("WARNING: SNI \(random.com\) not in certificate. Action=Continue server=random.com\(127.0.0.1\)", "Warning on missing name for randome.com") +# name check failure for bar.com +ts.Disk.diags_log.Content += Testers.ContainsExpression("WARNING: SNI \(bar.com\) not in certificate. Action=Terminate server=bar.com\(127.0.0.1\)", "Failure on missing name for bar.com") +# See if the explicitly set default sni_policy of remap works. +ts.Disk.diags_log.Content += Testers.ExcludesExpression("WARNING: SNI \(foo.com\) not in certificate. Action=Continue", "Warning on missing name for foo.com") + + diff --git a/tests/gold_tests/tls/tls_verify_override_base.test.py b/tests/gold_tests/tls/tls_verify_override_base.test.py new file mode 100644 index 00000000000..d14763411ea --- /dev/null +++ b/tests/gold_tests/tls/tls_verify_override_base.test.py @@ -0,0 +1,219 @@ +''' +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +Test.Summary = ''' +Test tls server certificate verification options. Exercise conf_remap +''' + +# need Curl +Test.SkipUnless( + Condition.HasProgram("curl", "Curl need to be installed on system for this test to work") +) + +# Define default ATS +ts = Test.MakeATSProcess("ts", select_ports=False) +server_foo = Test.MakeOriginServer("server_foo", ssl=True, options = {"--key": "{0}/signed-foo.key".format(Test.RunDirectory), "--cert": "{0}/signed-foo.pem".format(Test.RunDirectory)}) +server_bar = Test.MakeOriginServer("server_bar", ssl=True, options = {"--key": "{0}/signed-bar.key".format(Test.RunDirectory), "--cert": "{0}/signed-bar.pem".format(Test.RunDirectory)}) +server = Test.MakeOriginServer("server", ssl=True) + +dns = Test.MakeDNServer("dns") + +request_foo_header = {"headers": "GET / HTTP/1.1\r\nHost: foo.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +request_bad_foo_header = {"headers": "GET / HTTP/1.1\r\nHost: bad_foo.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +request_bar_header = {"headers": "GET / HTTP/1.1\r\nHost: bar.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +request_bad_bar_header = {"headers": "GET / HTTP/1.1\r\nHost: bad_bar.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +response_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +server_foo.addResponse("sessionlog.json", request_foo_header, response_header) +server_foo.addResponse("sessionlog.json", request_bad_foo_header, response_header) +server_bar.addResponse("sessionlog.json", request_bar_header, response_header) +server_bar.addResponse("sessionlog.json", request_bad_bar_header, response_header) + +# add ssl materials like key, certificates for the server +ts.addSSLfile("ssl/signed-foo.pem") +ts.addSSLfile("ssl/signed-foo.key") +ts.addSSLfile("ssl/signed-bar.pem") +ts.addSSLfile("ssl/signed-bar.key") +ts.addSSLfile("ssl/server.pem") +ts.addSSLfile("ssl/server.key") +ts.addSSLfile("ssl/signer.pem") +ts.addSSLfile("ssl/signer.key") + +ts.Variables.ssl_port = 4443 +ts.Disk.remap_config.AddLine( + 'map http://foo.com/basic https://foo.com:{0}'.format(server_foo.Variables.SSL_Port)) +ts.Disk.remap_config.AddLine( + 'map http://foo.com/override https://foo.com:{0} @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.policy=ENFORCED'.format(server_foo.Variables.SSL_Port)) +ts.Disk.remap_config.AddLine( + 'map http://bar.com/basic https://bar.com:{0}'.format(server_foo.Variables.SSL_Port)) +ts.Disk.remap_config.AddLine( + 'map http://bar.com/overridedisabled https://bar.com:{0} @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.policy=DISABLED'.format(server_foo.Variables.SSL_Port)) +ts.Disk.remap_config.AddLine( + 'map http://bar.com/overridesignature https://bar.com:{0} @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.properties=SIGNATURE @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.policy=ENFORCED'.format(server_foo.Variables.SSL_Port)) +ts.Disk.remap_config.AddLine( + 'map http://bar.com/overrideenforced https://bar.com:{0} @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.policy=ENFORCED'.format(server_foo.Variables.SSL_Port)) +ts.Disk.remap_config.AddLine( + 'map /basic https://127.0.0.1:{0}'.format(server.Variables.SSL_Port)) +ts.Disk.remap_config.AddLine( + 'map /overrideenforce https://127.0.0.1:{0} @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.policy=ENFORCED'.format(server.Variables.SSL_Port)) +ts.Disk.remap_config.AddLine( + 'map /overridename https://127.0.0.1:{0} @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.properties=NAME'.format(server.Variables.SSL_Port)) +ts.Disk.remap_config.AddLine( + 'map /snipolicyfooremap https://foo.com:{0} @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.properties=NAME @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.policy=ENFORCED @plugin=conf_remap.so @pparam=proxy.config.ssl.client.sni_policy=remap'.format(server_bar.Variables.SSL_Port)) +ts.Disk.remap_config.AddLine( + 'map /snipolicyfoohost https://foo.com:{0} @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.properties=NAME @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.policy=ENFORCED @plugin=conf_remap.so @pparam=proxy.config.ssl.client.sni_policy=host'.format(server_bar.Variables.SSL_Port)) +ts.Disk.remap_config.AddLine( + 'map /snipolicybarremap https://bar.com:{0} @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.properties=NAME @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.policy=ENFORCED @plugin=conf_remap.so @pparam=proxy.config.ssl.client.sni_policy=remap'.format(server_bar.Variables.SSL_Port)) +ts.Disk.remap_config.AddLine( + 'map /snipolicybarhost https://bar.com:{0} @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.properties=NAME @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.policy=ENFORCED @plugin=conf_remap.so @pparam=proxy.config.ssl.client.sni_policy=host'.format(server_bar.Variables.SSL_Port)) + +ts.Disk.ssl_multicert_config.AddLine( + 'dest_ip=* ssl_cert_name=server.pem ssl_key_name=server.key' +) + +# Case 1, global config policy=permissive properties=signature +# override for foo.com policy=enforced properties=all +ts.Disk.records_config.update({ + 'proxy.config.ssl.server.cert.path': '{0}'.format(ts.Variables.SSLDir), + 'proxy.config.ssl.server.private_key.path': '{0}'.format(ts.Variables.SSLDir), + # enable ssl port + 'proxy.config.http.server_ports': '{0} {1}:proto=http2;http:ssl'.format(ts.Variables.port, ts.Variables.ssl_port), + 'proxy.config.ssl.server.cipher_suite': 'ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA384:AES128-GCM-SHA256:AES256-GCM-SHA384:ECDHE-RSA-RC4-SHA:ECDHE-RSA-AES128-SHA:ECDHE-RSA-AES256-SHA:RC4-SHA:RC4-MD5:AES128-SHA:AES256-SHA:DES-CBC3-SHA!SRP:!DSS:!PSK:!aNULL:!eNULL:!SSLv2', + # set global policy + 'proxy.config.ssl.client.verify.server' : 2, + 'proxy.config.ssl.client.CA.cert.path': '{0}'.format(ts.Variables.SSLDir), + 'proxy.config.ssl.client.CA.cert.filename': 'signer.pem', + 'proxy.config.url_remap.pristine_host_hdr': 1, + 'proxy.config.exec_thread.autoconfig.scale': 1.0, + 'proxy.config.dns.nameservers': '127.0.0.1:{0}'.format(dns.Variables.Port), + 'proxy.config.dns.resolv_conf': 'NULL' +}) + +dns.addRecords(records={"foo.com.": ["127.0.0.1"]}) +dns.addRecords(records={"bar.com.": ["127.0.0.1"]}) + +# Should succeed without message +tr = Test.AddTestRun("default-permissive-success") +tr.Setup.Copy("ssl/signed-foo.key") +tr.Setup.Copy("ssl/signed-foo.pem") +tr.Setup.Copy("ssl/signed-bar.key") +tr.Setup.Copy("ssl/signed-bar.pem") +tr.Processes.Default.Command = 'curl -k -H \"host: foo.com\" http://127.0.0.1:{0}/basic'.format(ts.Variables.port) +tr.ReturnCode = 0 +tr.Processes.Default.StartBefore(dns) +tr.Processes.Default.StartBefore(server_foo) +tr.Processes.Default.StartBefore(server_bar) +tr.Processes.Default.StartBefore(server) +tr.Processes.Default.StartBefore(Test.Processes.ts, ready=When.PortOpen(ts.Variables.port)) +tr.StillRunningAfter = server +tr.StillRunningAfter = ts +# Should succed. No message +tr.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") + +tr2 = Test.AddTestRun("default-permissive-fail") +tr2.Processes.Default.Command = "curl -k -H \"host: bar.com\" http://127.0.0.1:{0}/basic".format(ts.Variables.port) +tr2.ReturnCode = 0 +tr2.StillRunningAfter = server +tr2.StillRunningAfter = ts +# Should succeed, but will be message in log about name mismatch +tr2.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") + +tr2 = Test.AddTestRun("default-permissive-fail2") +tr2.Processes.Default.Command = "curl -k -H \"host: random.com\" http://127.0.0.1:{0}/basic".format(ts.Variables.port) +tr2.ReturnCode = 0 +tr2.StillRunningAfter = server +tr2.StillRunningAfter = ts +# Should succeed, but will be message in log about signature +tr2.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") + +tr3 = Test.AddTestRun("override-foo") +tr3.Processes.Default.Command = "curl -k -H \"host: foo.com\" http://127.0.0.1:{0}/override".format(ts.Variables.port) +tr3.ReturnCode = 0 +tr3.StillRunningAfter = server +tr3.StillRunningAfter = ts +# Should succeed. No error messages +tr3.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") + +tr4 = Test.AddTestRun("override-bar-disabled") +tr4.Processes.Default.Command = "curl -k -H \"host: bad_bar.com\" http://127.0.0.1:{0}/overridedisabled".format(ts.Variables.port) +tr4.ReturnCode = 0 +tr4.StillRunningAfter = server +tr4.StillRunningAfter = ts +# Succeed. No error messages +tr4.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") + +tr5 = Test.AddTestRun("override-bar-signature-enforced") +tr5.Processes.Default.Command = "curl -k -H \"host: bar.com\" http://127.0.0.1:{0}/overridesignature".format(ts.Variables.port) +tr5.ReturnCode = 0 +tr5.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") +tr5.StillRunningAfter = server +tr5.StillRunningAfter = ts + +tr6 = Test.AddTestRun("override-bar-enforced") +tr6.Processes.Default.Command = "curl -k -H \"host: bar.com\" http://127.0.0.1:{0}/overrideenforced".format(ts.Variables.port) +tr6.ReturnCode = 0 +# Should fail +tr6.Processes.Default.Streams.stdout = Testers.ContainsExpression("Could Not Connect", "Curl attempt should have failed") +tr6.StillRunningAfter = server +tr6.StillRunningAfter = ts + +# Should succeed +tr = Test.AddTestRun("foo-to-bar-sni-policy-remap") +tr.Processes.Default.Command = "curl -k -H \"host: foo.com\" http://127.0.0.1:{0}/snipolicybarremap".format(ts.Variables.port) +tr.ReturnCode = 0 +tr.StillRunningAfter = server +tr.StillRunningAfter = ts +tr.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could not connect", "Curl attempt should succeed") + +# Should fail +tr = Test.AddTestRun("foo-to-bar-sni-policy-host") +tr.Processes.Default.Command = "curl -k -H \"host: foo.com\" http://127.0.0.1:{0}/snipolicybarhost".format(ts.Variables.port) +tr.ReturnCode = 0 +tr.StillRunningAfter = server +tr.StillRunningAfter = ts +tr.Processes.Default.Streams.stdout = Testers.ContainsExpression("Could not connect", "Curl attempt should fail") + +# Should fail +tr = Test.AddTestRun("bar-to-foo-sni-policy-remap") +tr.Processes.Default.Command = "curl -k -H \"host: bar.com\" http://127.0.0.1:{0}/snipolicyfooremap".format(ts.Variables.port) +tr.ReturnCode = 0 +tr.StillRunningAfter = server +tr.StillRunningAfter = ts +tr.Processes.Default.Streams.stdout = Testers.ContainsExpression("Could not connect", "Curl attempt should fail") + +# Should succeed +tr = Test.AddTestRun("bar-to-foo-sni-policy-host") +tr.Processes.Default.Command = "curl -k -H \"host: bar.com\" http://127.0.0.1:{0}/snipolicyfoohost".format(ts.Variables.port) +tr.ReturnCode = 0 +tr.StillRunningAfter = server +tr.StillRunningAfter = ts +tr.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could not connect", "Curl attempt should succeed") + + +# Over riding the built in ERROR check since we expect some cases to fail + +# checks on random.com should fail with message only +ts.Disk.diags_log.Content = Testers.ContainsExpression("WARNING: Core server certificate verification failed for \(random.com\). Action=Continue Error=self signed certificate server=127.0.0.1\(127.0.0.1\) depth=0", "Warning for self signed certificate") +# permissive failure for bar.com +ts.Disk.diags_log.Content += Testers.ContainsExpression("WARNING: SNI \(bar.com\) not in certificate. Action=Continue server=bar.com\(127.0.0.1\)", "Warning on missing name for bar.com") +# name check failure for random.com +ts.Disk.diags_log.Content += Testers.ContainsExpression("WARNING: SNI \(random.com\) not in certificate. Action=Continue server=127.0.0.1\(127.0.0.1\)", "Warning on missing name for randome.com") +# name check failure for bar.com +ts.Disk.diags_log.Content += Testers.ContainsExpression("WARNING: SNI \(bar.com\) not in certificate. Action=Terminate server=bar.com\(127.0.0.1\)", "Failure on missing name for bar.com") + + diff --git a/tests/gold_tests/tls_hooks/gold/client-hello-1.gold b/tests/gold_tests/tls_hooks/gold/client-hello-1.gold new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/gold_tests/tls_hooks/gold/ts-cert-1-im-2.gold b/tests/gold_tests/tls_hooks/gold/ts-cert-1-im-2.gold index a2ec9b73667..98c8ce588f6 100644 --- a/tests/gold_tests/tls_hooks/gold/ts-cert-1-im-2.gold +++ b/tests/gold_tests/tls_hooks/gold/ts-cert-1-im-2.gold @@ -1,4 +1,4 @@ -`` DIAG: (ssl_hook_test) Setup callbacks pa=0 sni=0 cert=1 cert_imm=2 pa_delay=0 +`` DIAG: (ssl_hook_test) Setup callbacks pa=0 client_hello=0 client_hello_imm=0 sni=0 cert=1 cert_imm=2 pa_delay=0 `` DIAG: (ssl_hook_test) Cert callback 0 ssl_vc=`` `` DIAG: (ssl_hook_test) Callback reenable ssl_vc=`` `` DIAG: (ssl_hook_test) Cert callback 0 ssl_vc=`` diff --git a/tests/gold_tests/tls_hooks/gold/ts-cert-1.gold b/tests/gold_tests/tls_hooks/gold/ts-cert-1.gold index 91f7b38c3d3..4a77e239a99 100644 --- a/tests/gold_tests/tls_hooks/gold/ts-cert-1.gold +++ b/tests/gold_tests/tls_hooks/gold/ts-cert-1.gold @@ -1,3 +1,3 @@ -`` DIAG: (ssl_hook_test) Setup callbacks pa=0 sni=0 cert=1 cert_imm=0 pa_delay=0 +`` DIAG: (ssl_hook_test) Setup callbacks pa=0 client_hello=0 client_hello_imm=0 sni=0 cert=1 cert_imm=0 pa_delay=0 `` DIAG: (ssl_hook_test) Cert callback 0 ssl_vc=`` `` DIAG: (ssl_hook_test) Callback reenable ssl_vc=`` diff --git a/tests/gold_tests/tls_hooks/gold/ts-cert-2.gold b/tests/gold_tests/tls_hooks/gold/ts-cert-2.gold index 355c5aefbfa..b56bd6d581b 100644 --- a/tests/gold_tests/tls_hooks/gold/ts-cert-2.gold +++ b/tests/gold_tests/tls_hooks/gold/ts-cert-2.gold @@ -1,4 +1,4 @@ -`` DIAG: (ssl_hook_test) Setup callbacks pa=0 sni=0 cert=2 cert_imm=0 pa_delay=0 +`` DIAG: (ssl_hook_test) Setup callbacks pa=0 client_hello=0 client_hello_imm=0 sni=0 cert=2 cert_imm=0 pa_delay=0 `` DIAG: (ssl_hook_test) Cert callback 0 ssl_vc=`` `` DIAG: (ssl_hook_test) Callback reenable ssl_vc=`` `` DIAG: (ssl_hook_test) Cert callback 1 ssl_vc=`` diff --git a/tests/gold_tests/tls_hooks/gold/ts-cert-im-1.gold b/tests/gold_tests/tls_hooks/gold/ts-cert-im-1.gold index c571bae3b87..11bb4e7fcd1 100644 --- a/tests/gold_tests/tls_hooks/gold/ts-cert-im-1.gold +++ b/tests/gold_tests/tls_hooks/gold/ts-cert-im-1.gold @@ -1,2 +1,2 @@ -`` DIAG: (ssl_hook_test) Setup callbacks pa=0 sni=0 cert=0 cert_imm=1 pa_delay=0 +`` DIAG: (ssl_hook_test) Setup callbacks pa=0 client_hello=0 client_hello_imm=0 sni=0 cert=0 cert_imm=1 pa_delay=0 `` DIAG: (ssl_hook_test) Cert callback 0 ssl_vc=`` diff --git a/tests/gold_tests/tls_hooks/gold/ts-client-hello-1.gold b/tests/gold_tests/tls_hooks/gold/ts-client-hello-1.gold new file mode 100644 index 00000000000..8f444f3c95b --- /dev/null +++ b/tests/gold_tests/tls_hooks/gold/ts-client-hello-1.gold @@ -0,0 +1,3 @@ +`` DIAG: (ssl_hook_test) Setup callbacks pa=0 client_hello=0 client_hello_imm=1 sni=0 cert=0 cert_imm=0 pa_delay=0 +`` DIAG: (ssl_hook_test) Client Hello callback 0 `` +`` diff --git a/tests/gold_tests/tls_hooks/gold/ts-client-hello-2.gold b/tests/gold_tests/tls_hooks/gold/ts-client-hello-2.gold new file mode 100644 index 00000000000..4489a55652f --- /dev/null +++ b/tests/gold_tests/tls_hooks/gold/ts-client-hello-2.gold @@ -0,0 +1,5 @@ +`` DIAG: (ssl_hook_test) Setup callbacks pa=0 client_hello=2 client_hello_imm=0 sni=0 cert=0 cert_imm=0 pa_delay=0 +`` DIAG: (ssl_hook_test) Client Hello callback 0 ssl_vc=`` +`` DIAG: (ssl_hook_test) Callback reenable ssl_vc=`` +`` DIAG: (ssl_hook_test) Client Hello callback 1 ssl_vc=`` +`` DIAG: (ssl_hook_test) Callback reenable ssl_vc=`` diff --git a/tests/gold_tests/tls_hooks/gold/ts-client-hello-delayed-1.gold b/tests/gold_tests/tls_hooks/gold/ts-client-hello-delayed-1.gold new file mode 100644 index 00000000000..4cea3ccd6f5 --- /dev/null +++ b/tests/gold_tests/tls_hooks/gold/ts-client-hello-delayed-1.gold @@ -0,0 +1,4 @@ +`` DIAG: (ssl_hook_test) Setup callbacks pa=0 client_hello=1 client_hello_imm=0 sni=0 cert=0 cert_imm=0 pa_delay=0 +`` DIAG: (ssl_hook_test) Client Hello callback 0 `` +`` DIAG: (ssl_hook_test) Callback reenable ssl_vc=`` +`` diff --git a/tests/gold_tests/tls_hooks/gold/ts-out-delay-start-2.gold b/tests/gold_tests/tls_hooks/gold/ts-out-delay-start-2.gold index 21c5d61073c..89ef1cc4573 100644 --- a/tests/gold_tests/tls_hooks/gold/ts-out-delay-start-2.gold +++ b/tests/gold_tests/tls_hooks/gold/ts-out-delay-start-2.gold @@ -1,4 +1,4 @@ -`` DIAG: (ssl_hook_test) Setup callbacks pa=0 sni=0 cert=0 cert_imm=0 pa_delay=0 +`` DIAG: (ssl_hook_test) Setup callbacks pa=0 client_hello=0 client_hello_imm=0 sni=0 cert=0 cert_imm=0 pa_delay=0 `` DIAG: (ssl_hook_test) Outbound delay start callback 0 `` `` DIAG: (ssl_hook_test) Callback reenable ssl_vc=`` `` DIAG: (ssl_hook_test) Outbound delay start callback 1 `` diff --git a/tests/gold_tests/tls_hooks/gold/ts-preaccept-1.gold b/tests/gold_tests/tls_hooks/gold/ts-preaccept-1.gold index c8278ea9866..a2abb9d50ca 100644 --- a/tests/gold_tests/tls_hooks/gold/ts-preaccept-1.gold +++ b/tests/gold_tests/tls_hooks/gold/ts-preaccept-1.gold @@ -1,3 +1,3 @@ -`` DIAG: (ssl_hook_test) Setup callbacks pa=1 sni=0 cert=0 cert_imm=0 pa_delay=0 +`` DIAG: (ssl_hook_test) Setup callbacks pa=1 client_hello=0 client_hello_imm=0 sni=0 cert=0 cert_imm=0 pa_delay=0 `` DIAG: (ssl_hook_test) Pre accept callback 0 `` - event is good `` diff --git a/tests/gold_tests/tls_hooks/gold/ts-preaccept-2.gold b/tests/gold_tests/tls_hooks/gold/ts-preaccept-2.gold index cfac682852b..f70e8724e8b 100644 --- a/tests/gold_tests/tls_hooks/gold/ts-preaccept-2.gold +++ b/tests/gold_tests/tls_hooks/gold/ts-preaccept-2.gold @@ -1,4 +1,4 @@ -`` DIAG: (ssl_hook_test) Setup callbacks pa=2 sni=0 cert=0 cert_imm=0 pa_delay=0 +`` DIAG: (ssl_hook_test) Setup callbacks pa=2 client_hello=0 client_hello_imm=0 sni=0 cert=0 cert_imm=0 pa_delay=0 `` DIAG: (ssl_hook_test) Pre accept callback 0 `` - event is good `` DIAG: (ssl_hook_test) Pre accept callback 1 `` - event is good `` diff --git a/tests/gold_tests/tls_hooks/gold/ts-preaccept-delayed-1-immdate-2.gold b/tests/gold_tests/tls_hooks/gold/ts-preaccept-delayed-1-immdate-2.gold index 16427c9b444..685ffb492cf 100644 --- a/tests/gold_tests/tls_hooks/gold/ts-preaccept-delayed-1-immdate-2.gold +++ b/tests/gold_tests/tls_hooks/gold/ts-preaccept-delayed-1-immdate-2.gold @@ -1,4 +1,4 @@ -``DIAG: (ssl_hook_test) Setup callbacks pa=2 sni=0 cert=0 cert_imm=0 pa_delay=1 +``DIAG: (ssl_hook_test) Setup callbacks pa=2 client_hello=0 client_hello_imm=0 sni=0 cert=0 cert_imm=0 pa_delay=1 ``DIAG: (ssl_hook_test) Pre accept callback 0 `` - event is good ``DIAG: (ssl_hook_test) Pre accept callback 1 `` - event is good ``DIAG: (ssl_hook_test) Pre accept delay callback 0 `` - event is good diff --git a/tests/gold_tests/tls_hooks/gold/ts-preaccept-delayed-1.gold b/tests/gold_tests/tls_hooks/gold/ts-preaccept-delayed-1.gold index 0b85e1786c6..3ebced31bcb 100644 --- a/tests/gold_tests/tls_hooks/gold/ts-preaccept-delayed-1.gold +++ b/tests/gold_tests/tls_hooks/gold/ts-preaccept-delayed-1.gold @@ -1,3 +1,3 @@ -`` DIAG: (ssl_hook_test) Setup callbacks pa=0 sni=0 cert=0 cert_imm=0 pa_delay=1 +`` DIAG: (ssl_hook_test) Setup callbacks pa=0 client_hello=0 client_hello_imm=0 sni=0 cert=0 cert_imm=0 pa_delay=1 `` DIAG: (ssl_hook_test) Pre accept delay callback 0 `` - event is good `` diff --git a/tests/gold_tests/tls_hooks/gold/ts-preaccept1-sni1-cert1.gold b/tests/gold_tests/tls_hooks/gold/ts-preaccept1-sni1-cert1.gold index 07ee131bc85..b028f5eea93 100644 --- a/tests/gold_tests/tls_hooks/gold/ts-preaccept1-sni1-cert1.gold +++ b/tests/gold_tests/tls_hooks/gold/ts-preaccept1-sni1-cert1.gold @@ -1,4 +1,4 @@ -`` DIAG: (ssl_hook_test) Setup callbacks pa=1 sni=1 cert=1 cert_imm=0 pa_delay=0 +`` DIAG: (ssl_hook_test) Setup callbacks pa=1 client_hello=0 client_hello_imm=0 sni=1 cert=1 cert_imm=0 pa_delay=0 `` DIAG: (ssl_hook_test) Pre accept callback 0 `` - event is good `` DIAG: (ssl_hook_test) SNI callback 0 `` `` DIAG: (ssl_hook_test) Cert callback 0 ssl_vc=`` diff --git a/tests/gold_tests/tls_hooks/gold/ts-sni-1.gold b/tests/gold_tests/tls_hooks/gold/ts-sni-1.gold index 4b7d335f90c..bda783ae230 100644 --- a/tests/gold_tests/tls_hooks/gold/ts-sni-1.gold +++ b/tests/gold_tests/tls_hooks/gold/ts-sni-1.gold @@ -1,3 +1,3 @@ -`` DIAG: (ssl_hook_test) Setup callbacks pa=0 sni=1 cert=0 cert_imm=0 pa_delay=0 +`` DIAG: (ssl_hook_test) Setup callbacks pa=0 client_hello=0 client_hello_imm=0 sni=1 cert=0 cert_imm=0 pa_delay=0 `` DIAG: (ssl_hook_test) SNI callback 0 `` `` diff --git a/tests/gold_tests/tls_hooks/gold/ts-sni-2.gold b/tests/gold_tests/tls_hooks/gold/ts-sni-2.gold index ecb2cb6cbef..7ca56f72920 100644 --- a/tests/gold_tests/tls_hooks/gold/ts-sni-2.gold +++ b/tests/gold_tests/tls_hooks/gold/ts-sni-2.gold @@ -1,4 +1,4 @@ -`` DIAG: (ssl_hook_test) Setup callbacks pa=0 sni=2 cert=0 cert_imm=0 pa_delay=0 +`` DIAG: (ssl_hook_test) Setup callbacks pa=0 client_hello=0 client_hello_imm=0 sni=2 cert=0 cert_imm=0 pa_delay=0 `` DIAG: (ssl_hook_test) SNI callback 0 `` `` DIAG: (ssl_hook_test) SNI callback 1 `` `` diff --git a/tests/gold_tests/tls_hooks/tls_hooks.test.py b/tests/gold_tests/tls_hooks/tls_hooks.test.py index deb644858fe..7b4f006d41b 100644 --- a/tests/gold_tests/tls_hooks/tls_hooks.test.py +++ b/tests/gold_tests/tls_hooks/tls_hooks.test.py @@ -45,6 +45,7 @@ # enable ssl port 'proxy.config.http.server_ports': '{0}:ssl'.format(ts.Variables.ssl_port), 'proxy.config.ssl.client.verify.server': 0, + 'proxy.config.ssl.TLSv1_3': 0, 'proxy.config.ssl.server.cipher_suite': 'ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA384:AES128-GCM-SHA256:AES256-GCM-SHA384:ECDHE-RSA-RC4-SHA:ECDHE-RSA-AES128-SHA:ECDHE-RSA-AES256-SHA:RC4-SHA:RC4-MD5:AES128-SHA:AES256-SHA:DES-CBC3-SHA!SRP:!DSS:!PSK:!aNULL:!eNULL:!SSLv2', }) @@ -63,9 +64,10 @@ tr.Processes.Default.StartBefore(Test.Processes.ts, ready=When.PortOpen(ts.Variables.ssl_port)) tr.StillRunningAfter = ts tr.StillRunningAfter = server -tr.Processes.Default.Command = 'curl -k -H \'host:example.com:{0}\' https://127.0.0.1:{0}'.format(ts.Variables.ssl_port) +tr.Processes.Default.Command = 'curl -v -k -H \'host:example.com:{0}\' https://127.0.0.1:{0}'.format(ts.Variables.ssl_port) tr.Processes.Default.ReturnCode = 0 tr.Processes.Default.Streams.stdout = "gold/preaccept-1.gold" +tr.Processes.Default.Streams.All = Testers.ExcludesExpression("TLSv1.3 (IN), TLS handshake, Finished (20):", "Should not negotiate a TLSv1.3 connection") ts.Streams.stderr = "gold/ts-preaccept-1.gold" diff --git a/tests/gold_tests/tls_hooks/tls_hooks13.test.py b/tests/gold_tests/tls_hooks/tls_hooks13.test.py index 9388221bce6..704a111cab3 100644 --- a/tests/gold_tests/tls_hooks/tls_hooks13.test.py +++ b/tests/gold_tests/tls_hooks/tls_hooks13.test.py @@ -53,7 +53,7 @@ ) ts.Disk.remap_config.AddLine( - 'map https://example.com:4443 https://127.0.0.1:{0}'.format(server.Variables.Port) + 'map https://example.com:4443 https://127.0.0.1:{0}'.format(server.Variables.SSL_Port) ) Test.PreparePlugin(os.path.join(Test.Variables.AtsTestToolsDir, 'plugins', 'ssl_hook_test.cc'), ts, '-out_start=1 -out_close=2') diff --git a/tests/gold_tests/tls_hooks/tls_hooks14.test.py b/tests/gold_tests/tls_hooks/tls_hooks14.test.py index b411f65c6fe..a8c8d3d6acf 100644 --- a/tests/gold_tests/tls_hooks/tls_hooks14.test.py +++ b/tests/gold_tests/tls_hooks/tls_hooks14.test.py @@ -53,7 +53,7 @@ ) ts.Disk.remap_config.AddLine( - 'map https://example.com:4443 https://127.0.0.1:{0}'.format(server.Variables.Port) + 'map https://example.com:4443 https://127.0.0.1:{0}'.format(server.Variables.SSL_Port) ) Test.PreparePlugin(os.path.join(Test.Variables.AtsTestToolsDir, 'plugins', 'ssl_hook_test.cc'), ts, '-out_start_delay=2') diff --git a/tests/gold_tests/tls_hooks/tls_hooks15.test.py b/tests/gold_tests/tls_hooks/tls_hooks15.test.py index ed6e687cda1..9194802f46b 100644 --- a/tests/gold_tests/tls_hooks/tls_hooks15.test.py +++ b/tests/gold_tests/tls_hooks/tls_hooks15.test.py @@ -53,7 +53,7 @@ ) ts.Disk.remap_config.AddLine( - 'map https://example.com:4443 https://127.0.0.1:{0}'.format(server.Variables.Port) + 'map https://example.com:4443 https://127.0.0.1:{0}'.format(server.Variables.SSL_Port) ) Test.PreparePlugin(os.path.join(Test.Variables.AtsTestToolsDir, 'plugins', 'ssl_hook_test.cc'), ts, '-close=2 -out_close=1') diff --git a/tests/gold_tests/tls_hooks/tls_hooks16.test.py b/tests/gold_tests/tls_hooks/tls_hooks16.test.py new file mode 100644 index 00000000000..877c32018d5 --- /dev/null +++ b/tests/gold_tests/tls_hooks/tls_hooks16.test.py @@ -0,0 +1,79 @@ +''' +Test single immediate client hello hook +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import re + +Test.Summary = ''' +Test different combinations of TLS handshake hooks to ensure they are applied consistently. +''' + +Test.SkipUnless( + Condition.HasProgram("grep", "grep needs to be installed on system for this test to work"), + Condition.HasOpenSSLVersion("1.1.1")) + +ts = Test.MakeATSProcess("ts", select_ports=False) +server = Test.MakeOriginServer("server") +request_header = {"headers": "GET / HTTP/1.1\r\nHost: www.example.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +# desired response form the origin server +response_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +server.addResponse("sessionlog.json", request_header, response_header) + +ts.addSSLfile("ssl/server.pem") +ts.addSSLfile("ssl/server.key") + +ts.Variables.ssl_port = 4443 +ts.Disk.records_config.update({ + 'proxy.config.diags.debug.enabled': 1, + 'proxy.config.diags.debug.tags': 'ssl_hook_test', + 'proxy.config.ssl.server.cert.path': '{0}'.format(ts.Variables.SSLDir), + 'proxy.config.ssl.server.private_key.path': '{0}'.format(ts.Variables.SSLDir), + # enable ssl port + 'proxy.config.http.server_ports': '{0}:ssl'.format(ts.Variables.ssl_port), + 'proxy.config.ssl.client.verify.server': 0, + 'proxy.config.ssl.server.cipher_suite': 'ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA384:AES128-GCM-SHA256:AES256-GCM-SHA384:ECDHE-RSA-RC4-SHA:ECDHE-RSA-AES128-SHA:ECDHE-RSA-AES256-SHA:RC4-SHA:RC4-MD5:AES128-SHA:AES256-SHA:DES-CBC3-SHA!SRP:!DSS:!PSK:!aNULL:!eNULL:!SSLv2', +}) + +ts.Disk.ssl_multicert_config.AddLine( + 'dest_ip=* ssl_cert_name=server.pem ssl_key_name=server.key' +) + +ts.Disk.remap_config.AddLine( + 'map https://example.com:4443 http://127.0.0.1:{0}'.format(server.Variables.Port) +) + +Test.PreparePlugin(os.path.join(Test.Variables.AtsTestToolsDir, 'plugins', 'ssl_hook_test.cc'), ts, '-client_hello_imm=1') + +tr = Test.AddTestRun("Test one immediate client hello hook") +tr.Processes.Default.StartBefore(server) +tr.Processes.Default.StartBefore(Test.Processes.ts, ready=When.PortOpen(ts.Variables.ssl_port)) +tr.StillRunningAfter = ts +tr.StillRunningAfter = server +tr.Processes.Default.Command = 'curl -k -H \'host:example.com:{0}\' https://127.0.0.1:{0}'.format(ts.Variables.ssl_port) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Streams.stdout = "gold/client-hello-1.gold" + +ts.Streams.stderr = "gold/ts-client-hello-1.gold" + +snistring = "Client Hello callback 0" +ts.Streams.All = Testers.ContainsExpression( + "\A(?:(?!{0}).)*{0}(?!.*{0}).*\Z".format(snistring), "Client Hello message appears only once", reflags=re.S | re.M) + +tr.Processes.Default.TimeOut = 5 +tr.TimeOut = 5 diff --git a/tests/gold_tests/tls_hooks/tls_hooks17.test.py b/tests/gold_tests/tls_hooks/tls_hooks17.test.py new file mode 100644 index 00000000000..327c36d7204 --- /dev/null +++ b/tests/gold_tests/tls_hooks/tls_hooks17.test.py @@ -0,0 +1,79 @@ +''' +Test single delayed client hello hook +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import re + +Test.Summary = ''' +Test different combinations of TLS handshake hooks to ensure they are applied consistently. +''' + +Test.SkipUnless( + Condition.HasProgram("grep", "grep needs to be installed on system for this test to work"), + Condition.HasOpenSSLVersion("1.1.1")) + +ts = Test.MakeATSProcess("ts", select_ports=False) +server = Test.MakeOriginServer("server") +request_header = {"headers": "GET / HTTP/1.1\r\nHost: www.example.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +# desired response form the origin server +response_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +server.addResponse("sessionlog.json", request_header, response_header) + +ts.addSSLfile("ssl/server.pem") +ts.addSSLfile("ssl/server.key") + +ts.Variables.ssl_port = 4443 +ts.Disk.records_config.update({ + 'proxy.config.diags.debug.enabled': 1, + 'proxy.config.diags.debug.tags': 'ssl_hook_test', + 'proxy.config.ssl.server.cert.path': '{0}'.format(ts.Variables.SSLDir), + 'proxy.config.ssl.server.private_key.path': '{0}'.format(ts.Variables.SSLDir), + # enable ssl port + 'proxy.config.http.server_ports': '{0}:ssl'.format(ts.Variables.ssl_port), + 'proxy.config.ssl.client.verify.server': 0, + 'proxy.config.ssl.server.cipher_suite': 'ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA384:AES128-GCM-SHA256:AES256-GCM-SHA384:ECDHE-RSA-RC4-SHA:ECDHE-RSA-AES128-SHA:ECDHE-RSA-AES256-SHA:RC4-SHA:RC4-MD5:AES128-SHA:AES256-SHA:DES-CBC3-SHA!SRP:!DSS:!PSK:!aNULL:!eNULL:!SSLv2', +}) + +ts.Disk.ssl_multicert_config.AddLine( + 'dest_ip=* ssl_cert_name=server.pem ssl_key_name=server.key' +) + +ts.Disk.remap_config.AddLine( + 'map https://example.com:4443 http://127.0.0.1:{0}'.format(server.Variables.Port) +) + +Test.PreparePlugin(os.path.join(Test.Variables.AtsTestToolsDir, 'plugins', 'ssl_hook_test.cc'), ts, '-client_hello=1') + +tr = Test.AddTestRun("Test one delayed client hello hook") +tr.Processes.Default.StartBefore(server) +tr.Processes.Default.StartBefore(Test.Processes.ts, ready=When.PortOpen(ts.Variables.ssl_port)) +tr.StillRunningAfter = ts +tr.StillRunningAfter = server +tr.Processes.Default.Command = 'curl -k -H \'host:example.com:{0}\' https://127.0.0.1:{0}'.format(ts.Variables.ssl_port) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Streams.stdout = "gold/client-hello-1.gold" + +ts.Streams.stderr = "gold/ts-client-hello-delayed-1.gold" + +snistring = "Client Hello callback 0" +ts.Streams.All = Testers.ContainsExpression( + "\A(?:(?!{0}).)*{0}(?!.*{0}).*\Z".format(snistring), "Client Hello message appears only once", reflags=re.S | re.M) + +tr.Processes.Default.TimeOut = 5 +tr.TimeOut = 5 diff --git a/tests/gold_tests/tls_hooks/tls_hooks18.test.py b/tests/gold_tests/tls_hooks/tls_hooks18.test.py new file mode 100644 index 00000000000..4ce45d12dba --- /dev/null +++ b/tests/gold_tests/tls_hooks/tls_hooks18.test.py @@ -0,0 +1,83 @@ +''' +Test two delayed client hello callbacks +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import re + +Test.Summary = ''' +Test different combinations of TLS handshake hooks to ensure they are applied consistently. +''' + +Test.SkipUnless( + Condition.HasProgram("grep", "grep needs to be installed on system for this test to work"), + Condition.HasOpenSSLVersion("1.1.1") + ) + +ts = Test.MakeATSProcess("ts", select_ports=False) +server = Test.MakeOriginServer("server") +request_header = {"headers": "GET / HTTP/1.1\r\nHost: www.example.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +# desired response form the origin server +response_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +server.addResponse("sessionlog.json", request_header, response_header) + +ts.addSSLfile("ssl/server.pem") +ts.addSSLfile("ssl/server.key") + +ts.Variables.ssl_port = 4443 +ts.Disk.records_config.update({ + 'proxy.config.diags.debug.enabled': 1, + 'proxy.config.diags.debug.tags': 'ssl_hook_test', + 'proxy.config.ssl.server.cert.path': '{0}'.format(ts.Variables.SSLDir), + 'proxy.config.ssl.server.private_key.path': '{0}'.format(ts.Variables.SSLDir), + # enable ssl port + 'proxy.config.http.server_ports': '{0}:ssl'.format(ts.Variables.ssl_port), + 'proxy.config.ssl.client.verify.server': 0, + 'proxy.config.ssl.server.cipher_suite': 'ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA384:AES128-GCM-SHA256:AES256-GCM-SHA384:ECDHE-RSA-RC4-SHA:ECDHE-RSA-AES128-SHA:ECDHE-RSA-AES256-SHA:RC4-SHA:RC4-MD5:AES128-SHA:AES256-SHA:DES-CBC3-SHA!SRP:!DSS:!PSK:!aNULL:!eNULL:!SSLv2', +}) + +ts.Disk.ssl_multicert_config.AddLine( + 'dest_ip=* ssl_cert_name=server.pem ssl_key_name=server.key' +) + +ts.Disk.remap_config.AddLine( + 'map https://example.com:4443 http://127.0.0.1:{0}'.format(server.Variables.Port) +) + +Test.PreparePlugin(os.path.join(Test.Variables.AtsTestToolsDir, 'plugins', 'ssl_hook_test.cc'), ts, '-client_hello=2') + +tr = Test.AddTestRun("Test two client hello hooks") +tr.Processes.Default.StartBefore(server) +tr.Processes.Default.StartBefore(Test.Processes.ts, ready=When.PortOpen(ts.Variables.ssl_port)) +tr.StillRunningAfter = ts +tr.StillRunningAfter = server +tr.Processes.Default.Command = 'curl -k -H \'host:example.com:{0}\' https://127.0.0.1:{0}'.format(ts.Variables.ssl_port) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Streams.stdout = "gold/preaccept-1.gold" + +ts.Streams.stderr = "gold/ts-client-hello-2.gold" + +certstring0 = "Client Hello callback 0" +certstring1 = "Client Hello callback 1" +ts.Streams.All = Testers.ContainsExpression( + "\A(?:(?!{0}).)*{0}(?!.*{0}).*\Z".format(certstring0), "Cert message appears only once", reflags=re.S | re.M) +ts.Streams.All = Testers.ContainsExpression( + "\A(?:(?!{0}).)*{0}(?!.*{0}).*\Z".format(certstring1), "Cert message appears only once", reflags=re.S | re.M) + +tr.Processes.Default.TimeOut = 5 +tr.TimeOut = 5 diff --git a/tests/tools/lib/result.py b/tests/tools/lib/result.py deleted file mode 100644 index 7322e65f050..00000000000 --- a/tests/tools/lib/result.py +++ /dev/null @@ -1,117 +0,0 @@ -#!/bin/env python3 -''' -''' -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import sys - - -class TermColors: - ''' Collection of colors for printing out to terminal ''' - HEADER = '\033[95m' - OKBLUE = '\033[94m' - OKGREEN = '\033[92m' - WARNING = '\033[93m' - FAIL = '\033[91m' - BOLD = '\033[1m' - UNDERLINE = '\033[4m' - ENDC = '\033[0m' - - -ignoredFields = {'age', 'set-cookie', 'server', 'date', 'last-modified', - 'via', 'expires', 'cache-control', 'vary', 'connection'} # all lower case - - -class Result(object): - ''' Result encapsulates the result of a single session replay ''' - - def __init__(self, test_name, expected_response, received_response, recv_resp_body=None): - ''' expected_response and received_response can be any datatype the caller wants as long as they are the same datatype ''' - self._test_name = test_name - self._expected_response = expected_response - self._received_response = received_response - self._received_response_body = recv_resp_body - - def getTestName(self): - return self._test_name - - def getResultBool(self): - return self._expected_response == self._received_response - - def getRespBody(self): - if self._received_response_body: - return self._received_response_body - else: - return "" - - def Compare(self, received_dict, expected_dict, src=None): - global ignoredFields - # print("RECEIVED") - # print(received_dict) - # print("RECIEVED CACHE CONTROL") - # print(received_dict['Cache-Control'.lower()]) - # print("EXPECTED") - # print(expected_dict) - try: - for key in received_dict: - # print(key) - if key.lower() in expected_dict and key.lower() not in ignoredFields: - # print("{0} ==? {1}".format(expected_dict[key.lower()],received_dict[key])) - if received_dict[key.lower()] != expected_dict[key.lower()]: - print("{0}Difference in the field \"{1}\": \n received:\n{2}\n expected:\n{3}{4}".format( - TermColors.FAIL, key, received_dict[key], expected_dict[key], TermColors.ENDC)) - return False - if key.lower() == 'content-length' and self._received_response_body: - if int(received_dict[key.lower()]) != len(self._received_response_body): - print("{0}Difference in received content length and actual body length \ncontent-length: {1} \nbody: {2}\nbody length: {3}{4}".format( - TermColors.FAIL, received_dict[key.lower()], self._received_response_body, len( - self._received_response_body, TermColors.ENDC) - )) - return False - - except: - e = sys.exc_info() - if src: - print("In {0}: ".format(src), end='') - print("Error in comparing key ", e, key, "expected", expected_dict[key.lower()], "received", received_dict[key]) - return False - return True - - def getResult(self, received_dict, expected_dict, colorize=False): - global ignoredFields - retval = False - ''' Return a nicely formatted result string with color if requested ''' - if self.getResultBool() and self.Compare(received_dict, expected_dict, self._test_name): - if colorize: - outstr = "{0}PASS{1}".format( - TermColors.OKGREEN, TermColors.ENDC) - - else: - outstr = "PASS" - - retval = True - - else: - if colorize: - outstr = "{0}FAIL{1}: expected {2}, received {3}, session file: {4}".format( - TermColors.FAIL, TermColors.ENDC, self._expected_response, self._received_response, self._test_name) - - else: - outstr = "FAIL: expected {0}, received {1}".format( - self._expected_response, self._received_response) - - return (retval, outstr) diff --git a/tests/tools/microDNS/uDNS.py b/tests/tools/microDNS/uDNS.py deleted file mode 100644 index 297cdcb714d..00000000000 --- a/tests/tools/microDNS/uDNS.py +++ /dev/null @@ -1,207 +0,0 @@ -# coding=utf-8 - -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import datetime -import sys -import time -import threading -import traceback -import socketserver -import argparse -import codecs -import json -from dnslib import * - -sys.path.append( - os.path.normpath( - os.path.join( - os.path.dirname(os.path.abspath(__file__)), - '..' - ) - ) -) - -import lib.IPConstants as IPConstants - -TTL = 60 * 5 # completely arbitrary TTL value -round_robin = False -default_records = list() -records = dict() - -class DomainName(str): - def __getattr__(self, item): - return DomainName(item + '.' + self) - - -class BaseRequestHandler(socketserver.BaseRequestHandler): - - def get_data(self): - raise NotImplementedError - - def send_data(self, data): - raise NotImplementedError - - def handle(self): - now = datetime.datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S.%f') - print("\n\n%s request %s (%s %s):" % (self.__class__.__name__[:3], now, self.client_address[0], - self.client_address[1])) - try: - data = self.get_data() - self.send_data(dns_response(data)) - except Exception: - traceback.print_exc(file=sys.stderr) - - -class TCPRequestHandler(BaseRequestHandler): - - def get_data(self): - data = self.request.recv(8192).strip() - sz = int(codecs.encode(data[:2], 'hex'), 16) - if sz < len(data) - 2: - raise Exception("Wrong size of TCP packet") - elif sz > len(data) - 2: - raise Exception("Too big TCP packet") - return data[2:] - - def send_data(self, data): - sz = codecs.decode(hex(len(data))[2:].zfill(4), 'hex') - return self.request.sendall(sz + data) - - -class UDPRequestHandler(BaseRequestHandler): - - def get_data(self): - return self.request[0].strip() - - def send_data(self, data): - return self.request[1].sendto(data, self.client_address) - - -def build_domain_mappings(path): - with open(path) as f: - zone_file = json.load(f) - - for domain in zone_file['mappings']: - for d in iter(domain.keys()): - # this loop only runs once, kind of a hack to access the only key in the dict - domain_name = DomainName(d) - print("Domain name:", domain_name) - # we can test using python's built-in ipaddress module, but this should suffice - records[domain_name] = [A(x) if ":" not in x else AAAA(x) for x in domain[domain_name]] - print(records[domain_name]) - - if 'otherwise' in zone_file: - default_records.extend([A(d) if ":" not in d else AAAA(d) for d in zone_file['otherwise']]) - - -def add_authoritative_records(reply, domain): - # ns1 and ns1 are hardcoded in, change if necessary - reply.add_auth(RR(rname=domain, rtype=QTYPE.NS, rclass=1, ttl=TTL, rdata=NS(domain.ns1))) - reply.add_auth(RR(rname=domain, rtype=QTYPE.NS, rclass=1, ttl=TTL, rdata=NS(domain.ns2))) - - -def dns_response(data): - ''' dns_response takes in the raw bytes from the socket and does all the logic behind what - RRs get returned as the response ''' - global default_records, records, TTL, round_robin - - request = DNSRecord.parse(data) - print(request) - - reply = DNSRecord(DNSHeader(id=request.header.id, qr=1, aa=1, ra=1), q=request.q) - qname = request.q.qname - qn = str(qname) - qtype = request.q.qtype - qt = QTYPE[qtype] - found_specific = False - - # first look for a specific mapping - for domain, rrs in records.items(): - if domain == qn or qn.endswith('.' + domain): - # we are the authoritative name server for this domain and all subdomains - for rdata in rrs: - # only include requested record types (ie. A, MX, etc) - rqt = rdata.__class__.__name__ - if qt in ['*', rqt]: - found_specific = True - reply.add_answer(RR(rname=qname, rtype=getattr(QTYPE, str(rqt)), rclass=1, ttl=TTL, rdata=rdata)) - - # rotate the A entries if round robin is on - if round_robin: - a_records = [x for x in rrs if type(x) == A] - records[domain] = a_records[1:] + a_records[:1] # rotate list - break - - # else if a specific mapping is not found, return default A-records - if not found_specific: - for a in default_records: - found_specific = True - reply.add_answer(RR(rname=qname, rtype=QTYPE.A, rclass=1, ttl=TTL, rdata=a)) - - if round_robin: - default_records = default_records[1:] + default_records[:1] - - if not found_specific: - reply.header.set_rcode(3) - - print("---- Reply: ----\n", reply) - return reply.pack() - - -if __name__ == '__main__': - # handle cmd line args - parser = argparse.ArgumentParser() - parser.add_argument("ip", type=str, help="Interface") - parser.add_argument("port", type=int, help="port uDNS should listen on") - parser.add_argument("zone_file", help="path to zone file") - parser.add_argument("--rr", action='store_true', - help='round robin load balances if multiple IP addresses are present for 1 domain') - args = parser.parse_args() - - if IPConstants.isIPv6(args.ip): - # *UDPServer derives from TCPServer, so setting one will affect the other - socketserver.TCPServer.address_family = socket.AF_INET6 - - # exit(1) - - if args.rr: - round_robin = True - build_domain_mappings(args.zone_file) - - ipaddr = IPConstants.getIP(args.ip) - - servers = [ - socketserver.ThreadingUDPServer((ipaddr, args.port), UDPRequestHandler), - socketserver.ThreadingTCPServer((ipaddr, args.port), TCPRequestHandler), - ] - - print("Starting DNS on address {0} port {1}...".format(ipaddr, args.port)) - for s in servers: - thread = threading.Thread(target=s.serve_forever) # that thread will start one more thread for each request - thread.daemon = True # exit the server thread when the main thread terminates - thread.start() - - try: - while 1: - time.sleep(1) - sys.stderr.flush() - sys.stdout.flush() - - except KeyboardInterrupt: - print("Got SigINT") - # pass - finally: - for s in servers: - s.shutdown() diff --git a/tests/tools/microServer/README.md b/tests/tools/microServer/README.md deleted file mode 100644 index a7681a378b0..00000000000 --- a/tests/tools/microServer/README.md +++ /dev/null @@ -1,49 +0,0 @@ -uWServer -======== - -uWServer is a mock HTTP server that takes predefined set of sessions for serving response to HTTP requests. Each session includes one or more transactions. A transaction is composed of an HTTP request and an HTTP response. -uWServer accepts session data in JSON fromat only. - - -Command: ----------------- - -`python3.5 uWServer.py --data-dir ` - -Options: ------------ - -To see the options please run `python3.5 uWServer.py --help` - -Session Definitions: --------------------- - -Example session: - -``` -{ - "encoding": "url_encoded", - "version": "0.2", - "txns": [ - { - "response": { - "headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", - "body": "", - "timestamp": "1469733493.993" - }, - "request": { - "headers": "GET / HTTP/1.1\r\nHost: www.example.test\r\n\r\n", - "body": "", - "timestamp": "1469733493.993" - }, - "uuid": "", - "timestamp": "" - } - ], - "timestamp": "1234567890.098" -} -``` - -Each session should be in its own file, and any number of files may be created to define sessions. - -The `response` map may include an `options` string, which is a comma-delimited list of options to be enabled. Currently the only option supported is `skipHooks`, which will ignore any hooks created for the matching request/response pair. See **Options**. diff --git a/tests/tools/microServer/uWServer.py b/tests/tools/microServer/uWServer.py deleted file mode 100644 index f2dfb383283..00000000000 --- a/tests/tools/microServer/uWServer.py +++ /dev/null @@ -1,734 +0,0 @@ -#!/bin/env python3 -''' -''' -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import string -import http.client -import cgi -import time -import sys -import json -import os -import threading -from ipaddress import ip_address -from http.server import BaseHTTPRequestHandler, HTTPServer -from socketserver import ThreadingMixIn, ForkingMixIn, BaseServer -from http import HTTPStatus -import argparse -import ssl -import socket -import importlib.util -import time -test_mode_enabled = True -lookup_key_ = "{PATH}" -__version__ = "1.1" - - -sys.path.append( - os.path.normpath( - os.path.join( - os.path.dirname(os.path.abspath(__file__)), - '..' - ) - ) -) - -import sessionvalidation.sessionvalidation as sv -import lib.IPConstants as IPConstants - - -SERVER_PORT = 5005 # default port -SERVER_DELAY = 0 # default delay -HTTP_VERSION = 'HTTP/1.1' -G_replay_dict = {} - -count = 0 - -# Simple class to hold lists of callbacks associated with a key. - - -class HookSet: - # Helper class to provide controlled access to the HookSet to the loading module. - class Registrar: - def __init__(self, hook_set): - self.hooks = hook_set - - def register(self, hook, cb): - self.hooks.register(hook, cb) - - def __init__(self): - self.hooks = {} - self.modules = [] - self.registrar = HookSet.Registrar(self) - # Define all the valid hooks here. - for item in ['ReadRequestHook']: - if isinstance(item, list): - hook = item[0] - label = item[1] - else: - hook = label = item - exec("HookSet.{} = '{}'".format(label, hook)) - exec("HookSet.Registrar.{} = '{}'".format(label, hook)) - self.hooks[hook] = [] - - def load(self, source): - try: - spec = importlib.util.spec_from_file_location('Observer', source) - mod = importlib.util.module_from_spec(spec) - mod.Hooks = self.registrar - spec.loader.exec_module(mod) - except ImportError: - print("Failed to import {}".format(source)) - else: - self.modules.append(mod) - - # Add a callback cb to the hook. - # Error if the hook isn't defined. - def register(self, hook, cb): - if hook in self.hooks: - self.hooks[hook].append(cb) - else: - raise ValueError("{} is not a valid hook name".format(hook)) - - # Invoke a hook. Pass on any additional arguments to the callback. - def invoke(self, hook, *args, **kwargs): - cb_list = self.hooks[hook] - if cb_list == None: - raise ValueError("{} is not a valid hook name to invoke".format(hook)) - else: - for cb in cb_list: - cb(*args, **kwargs) - - -class ThreadingServer(ThreadingMixIn, HTTPServer): - '''This class forces the creation of a new thread on each connection''' - - def __init__(self, local_addr, handler_class, options): - HTTPServer.__init__(self, local_addr, handler_class) - self.hook_set = HookSet() - if (options.load): - self.hook_set.load(options.load) - - -class ForkingServer(ForkingMixIn, HTTPServer): - '''This class forces the creation of a new process on each connection''' - pass - - -class SSLServer(ThreadingMixIn, HTTPServer): - def __init__(self, server_address, HandlerClass, options): - BaseServer.__init__(self, server_address, HandlerClass) - pwd = os.path.dirname(os.path.realpath(__file__)) - keys = os.path.join(pwd, options.key) - certs = os.path.join(pwd, options.cert) - clientCA = os.path.join(pwd, options.clientCA) - self.options = options - self.hook_set = HookSet() - - self.daemon_threads = True - self.protocol_version = 'HTTP/1.1' - - if options.load: - self.hook_set.load(options.load) - - print ("clientverify={0}".format(options.clientverify)) - - if options.clientverify: - self.socket = ssl.wrap_socket(socket.socket(self.address_family, self.socket_type), - keyfile=keys, certfile=certs, server_side=True, cert_reqs=ssl.CERT_REQUIRED, ca_certs=clientCA) - else: - self.socket = ssl.wrap_socket(socket.socket(self.address_family, self.socket_type), - keyfile=keys, certfile=certs, server_side=True) - - self.server_bind() - self.server_activate() - print("Port Configured for SSL communication") - - -class MyHandler(BaseHTTPRequestHandler): - def handleExpect100Continue(self, contentLength, chunked=False): - print("....expect", contentLength) - self.wfile.write(bytes('HTTP/1.1 100 Continue\r\n\r\n', 'UTF-8')) - if(not chunked): - message = self.rfile.read(contentLength) - else: - readChunks() - - def getLookupKey(self, requestline): - global lookup_key_ - kpath = "" - path = "" - print("Request={0} lookup_key={1}".format(requestline, lookup_key_)) - url_part = requestline.split(" ") - if url_part: - if url_part[1].startswith("http"): - path = url_part[1].split("/", 2)[2] - host_, path = path.split("/", 1) - else: - path = url_part[1].split("/", 1)[1] - argsList = [] - keyslist = lookup_key_.split("}") - for keystr in keyslist: - if keystr == '{PATH': - kpath = kpath + path - continue # do not include path in the list of header fields - if keystr == '{HOST': - kpath = kpath + host_ - continue - stringk = keystr.replace("{%", "") - argsList.append(stringk) - KeyList = [] - for argsL in argsList: - if len(argsL) > 0: - val = self.headers.get(argsL) - if val: - field_val, __ = cgi.parse_header(val) - else: - field_val = None - if field_val != None: - KeyList.append(field_val) - key = "".join(KeyList) + kpath - print("lookup key", key, len(key)) - - return key - - def parseRequestline(self, requestline): - testName = None - return testName - - def testMode(self, requestline): - print(requestline) - key = self.parseRequestline(requestline) - - self.send_response(200) - self.send_header('Connection', 'close') - self.end_headers() - - def get_response_code(self, header): - # this could totally go wrong - return int(header.split(' ')[1]) - - def generator(self): - yield 'micro' - yield 'server' - yield 'apache' - yield 'traffic' - yield 'server' - - def send_response(self, code, message=None): - ''' Override `send_response()`'s tacking on of server and date header lines. ''' - self.send_response_only(code, message) - - def createDummyBodywithLength(self, numberOfbytes): - if numberOfbytes == 0: - return None - body = 'a' - while numberOfbytes != 1: - body += 'b' - numberOfbytes -= 1 - return body - - def writeChunkedData(self): - for chunk in self.generator(): - response_string = bytes('%X\r\n%s\r\n' % (len(chunk), chunk), 'UTF-8') - self.wfile.write(response_string) - response_string = bytes('0\r\n\r\n', 'UTF-8') - self.wfile.write(response_string) - - def readChunks(self): - raw_data = b'' - raw_size = self.rfile.readline(65537) - size = str(raw_size, 'UTF-8').rstrip('\r\n') - # print("==========================================>",size) - size = int(size, 16) - while size > 0: - chunk = self.rfile.read(size + 2) # 2 for reading /r/n - raw_data += chunk - raw_size = self.rfile.readline(65537) - size = str(raw_size, 'UTF-8').rstrip('\r\n') - size = int(size, 16) - chunk = self.rfile.readline(65537) # read the extra blank newline \r\n after the last chunk - - def send_header(self, keyword, value): - """Send a MIME header to the headers buffer.""" - if self.request_version != 'HTTP/0.9': - if not hasattr(self, '_headers_buffer'): - self._headers_buffer = [] - self._headers_buffer.append( - ("%s: %s\r\n" % (keyword, value)).encode('UTF-8', 'strict')) # original code used latin-1.. seriously? - - if keyword.lower() == 'connection': - if value.lower() == 'close': - self.close_connection = True - elif value.lower() == 'keep-alive': - self.close_connection = False - - def parse_request(self): - """Parse a request (internal). - - The request should be stored in self.raw_requestline; the results - are in self.command, self.path, self.request_version and - self.headers. Any matching response is in self.response. - - Return True for success, False for failure; on failure, an - error is sent back. - - """ - - global count, test_mode_enabled, G_replay_dict - - self.command = None # set in case of error on the first line - self.request_version = version = self.default_request_version - self.close_connection = True - print("Raw request {0}".format(self.raw_requestline)) - requestline = str(self.raw_requestline, 'UTF-8') - requestline = requestline.rstrip('\r\n') - self.requestline = requestline - - # Examine the headers and look for a Connection directive. - try: - self.headers = http.client.parse_headers(self.rfile, - _class=self.MessageClass) - key = self.getLookupKey(self.requestline) - self.resp = G_replay_dict[key] if key in G_replay_dict else None - - if self.resp is None or 'skipHooks' not in self.resp.getOptions(): - self.server.hook_set.invoke(HookSet.ReadRequestHook, self.headers) - # read message body - if self.headers.get('Content-Length') != None: - bodysize = int(self.headers.get('Content-Length')) - #print("length of the body is",bodysize) - message = self.rfile.read(bodysize) - #print("message body",message) - elif self.headers.get('Transfer-Encoding', "") == 'chunked': - # print(self.headers) - self.readChunks() - except http.client.LineTooLong: - self.send_error( - HTTPStatus.BAD_REQUEST, - "Line too long") - return False - except http.client.HTTPException as err: - self.send_error( - HTTPStatus.REQUEST_HEADER_FIELDS_TOO_LARGE, - "Too many headers", - str(err) - ) - return False - - words = requestline.split() - if len(words) == 3: - command, path, version = words - if version[:5] != 'HTTP/': - self.send_error( - HTTPStatus.BAD_REQUEST, - "Bad request version (%r)" % version) - return False - try: - base_version_number = version.split('/', 1)[1] - version_number = base_version_number.split(".") - # RFC 2145 section 3.1 says there can be only one "." and - # - major and minor numbers MUST be treated as - # separate integers; - # - HTTP/2.4 is a lower version than HTTP/2.13, which in - # turn is lower than HTTP/12.3; - # - Leading zeros MUST be ignored by recipients. - if len(version_number) != 2: - raise ValueError - version_number = int(version_number[0]), int(version_number[1]) - except (ValueError, IndexError): - self.send_error( - HTTPStatus.BAD_REQUEST, - "Bad request version (%r)" % version) - return False - if version_number >= (1, 1) and self.protocol_version >= "HTTP/1.1": - self.close_connection = False - if version_number >= (2, 0): - self.send_error( - HTTPStatus.HTTP_VERSION_NOT_SUPPORTED, - "Invalid HTTP Version (%s)" % base_version_number) - return False - elif len(words) == 2: - command, path = words - self.close_connection = True - if command != 'GET': - self.send_error( - HTTPStatus.BAD_REQUEST, - "Bad HTTP/0.9 request type (%r)" % command) - return False - elif not words: - count += 1 - print("bla bla on 157 {0} => {1}".format(count, self.close_connection)) - return False - else: - self.send_error( - HTTPStatus.BAD_REQUEST, - "Bad request syntax (%r)" % requestline) - return False - self.command, self.path, self.request_version = command, path, version - - conntype = self.headers.get('Connection', "") - if conntype.lower() == 'close': - self.close_connection = True - elif (conntype.lower() == 'keep-alive' and - self.protocol_version >= "HTTP/1.1"): - self.close_connection = False - - return True - - def do_GET(self): - global G_replay_dict, test_mode_enabled - # skip time delay for the autest ready check - if test_mode_enabled and self.requestline != "GET /ruok HTTP/1.1": - time.sleep(time_delay) - - try: - response_string = None - chunkedResponse = False - if self.resp is None: - self.send_response(404) - self.send_header('Server', 'MicroServer') - self.send_header('Connection', 'close') - self.end_headers() - return - - else: - headers = self.resp.getHeaders().split('\r\n') - - # set status codes - status_code = self.get_response_code(headers[0]) - self.send_response(status_code) - - # set headers - for header in headers[1:]: # skip first one b/c it's response code - if header == '': - continue - elif 'Content-Length' in header: - if 'Access-Control' in header: # skipping Access-Control-Allow-Credentials, Access-Control-Allow-Origin, Content-Length - header_parts = header.split(':', 1) - header_field = str(header_parts[0].strip()) - header_field_val = str(header_parts[1].strip()) - self.send_header(header_field, header_field_val) - continue - lengthSTR = header.split(':')[1] - length = lengthSTR.strip(' ') - if test_mode_enabled: # the length of the body is given priority in test mode rather than the value in Content-Length. But in replay mode Content-Length gets the priority - if not (self.resp.getBody()): # Don't attach content-length yet if body is present in the response specified by tester - self.send_header('Content-Length', str(length)) - else: - self.send_header('Content-Length', str(length)) - response_string = self.createDummyBodywithLength(int(length)) - continue - if 'Transfer-Encoding' in header: - self.send_header('Transfer-Encoding', 'Chunked') - response_string = '%X\r\n%s\r\n' % (len('ats'), 'ats') - chunkedResponse = True - continue - - header_parts = header.split(':', 1) - header_field = str(header_parts[0].strip()) - header_field_val = str(header_parts[1].strip()) - self.send_header(header_field, header_field_val) - # End for - if test_mode_enabled: - if self.resp.getBody(): - length = len(bytes(self.resp.getBody(), 'UTF-8')) - response_string = self.resp.getBody() - self.send_header('Content-Length', str(length)) - self.end_headers() - - if (chunkedResponse): - self.writeChunkedData() - elif response_string != None and response_string != '': - self.wfile.write(bytes(response_string, 'UTF-8')) - except: - e = sys.exc_info() - print("Error", e, self.headers) - self.send_response(400) - self.send_header('Connection', 'close') - self.end_headers() - - def do_HEAD(self): - if self.resp is None: - self.send_response(404) - self.send_header('Connection', 'close') - self.end_headers() - return - - headers = self.resp.getHeaders().split('\r\n') - - # set status codes - status_code = self.get_response_code(headers[0]) - self.send_response(status_code) - - # set headers - for header in headers[1:]: # skip first one b/c it's response code - if header == '': - continue - elif 'Content-Length' in header: - self.send_header('Content-Length', '0') - continue - - header_parts = header.split(':', 1) - header_field = str(header_parts[0].strip()) - header_field_val = str(header_parts[1].strip()) - self.send_header(header_field, header_field_val) - - self.end_headers() - - def do_POST(self): - response_string = None - chunkedResponse = False - global test_mode_enabled - try: - - if self.resp is None: - self.send_response(404) - self.send_header('Connection', 'close') - self.end_headers() - return - else: - resp_headers = self.resp.getHeaders().split('\r\n') - # set status codes - status_code = self.get_response_code(resp_headers[0]) - #print("response code",status_code) - self.send_response(status_code) - #print("reposen is ",resp_headers) - # set headers - for header in resp_headers[1:]: # skip first one b/c it's response code - - if header == '': - continue - elif 'Content-Length' in header: - if 'Access-Control' in header: # skipping Access-Control-Allow-Credentials, Access-Control-Allow-Origin, Content-Length - header_parts = header.split(':', 1) - header_field = str(header_parts[0].strip()) - header_field_val = str(header_parts[1].strip()) - self.send_header(header_field, header_field_val) - continue - - lengthSTR = header.split(':')[1] - length = lengthSTR.strip(' ') - if test_mode_enabled: # the length of the body is given priority in test mode rather than the value in Content-Length. Otherwise, Content-Length gets the priority - if not (self.resp.getBody()): # Don't attach content-length yet if body is present in the response specified by tester - self.send_header('Content-Length', str(length)) - else: - self.send_header('Content-Length', str(length)) - response_string = self.createDummyBodywithLength(int(length)) - continue - if 'Transfer-Encoding' in header: - self.send_header('Transfer-Encoding', 'Chunked') - response_string = '%X\r\n%s\r\n' % (len('microserver'), 'microserver') - chunkedResponse = True - continue - - header_parts = header.split(':', 1) - header_field = str(header_parts[0].strip()) - header_field_val = str(header_parts[1].strip()) - #print("{0} === >{1}".format(header_field, header_field_val)) - self.send_header(header_field, header_field_val) - # End for loop - if test_mode_enabled: - if self.resp.getBody(): - length = len(bytes(self.resp.getBody(), 'UTF-8')) - response_string = self.resp.getBody() - self.send_header('Content-Length', str(length)) - self.end_headers() - - if (chunkedResponse): - self.writeChunkedData() - elif response_string != None and response_string != '': - self.wfile.write(bytes(response_string, 'UTF-8')) - except: - e = sys.exc_info() - print("Error", e, self.headers) - self.send_response(400) - self.send_header('Connection', 'close') - self.end_headers() - - -def populate_global_replay_dictionary(sessions): - ''' Populates the global dictionary of {uuid (string): reponse (Response object)} ''' - global G_replay_dict - for session in sessions: - for txn in session.getTransactionIter(): - G_replay_dict[txn._uuid] = txn.getResponse() - - print("size", len(G_replay_dict)) - -# tests will add responses to the dictionary where key is the testname - - -def addResponseHeader(key, response_header): - G_replay_dict[key] = response_header - - -def _path(exists, arg): - path = os.path.abspath(arg) - if not os.path.exists(path) and exists: - msg = '"{0}" is not a valid path'.format(path) - raise argparse.ArgumentTypeError(msg) - return path - - -def _bool(arg): - - opt_true_values = set(['y', 'yes', 'true', 't', '1', 'on', 'all']) - opt_false_values = set(['n', 'no', 'false', 'f', '0', 'off', 'none', None]) - - tmp = arg.lower() if arg is not None else None - if tmp in opt_true_values: - return True - elif tmp in opt_false_values: - return False - else: - msg = 'Invalid value Boolean value : "{0}"\n Valid options are {1}'.format(arg, - opt_true_values | opt_false_values) - raise ValueError(msg) - - -def _argparse_bool(arg): - try: - return _bool(arg) - except ValueError as ve: - raise argparse.ArgumentTypeError(ve) - - -def main(): - global test_mode_enabled - parser = argparse.ArgumentParser() - - parser.add_argument("--data-dir", "-d", - type=lambda x: _path(True, x), - required=True, - help="Directory with data file" - ) - - parser.add_argument("--ip_address", "-ip", - type=str, - default='INADDR_LOOPBACK', - help="IP address of the interface to serve on" - ) - - parser.add_argument("--port", "-p", - type=int, - default=SERVER_PORT, - help="Port to use") - - parser.add_argument("--delay", "-dy", - type=float, - default=SERVER_DELAY, - help="Response delay") - - parser.add_argument("--timeout", "-t", - type=float, - default=None, - help="socket time out in seconds") - - parser.add_argument('-V', '--version', action='version', version='%(prog)s {0}'.format(__version__)) - - parser.add_argument("--mode", "-m", - type=str, - default="test", - help="Mode of operation") - parser.add_argument("--ssl", "-ssl", - type=str, - default="False", - help="SSL port") - parser.add_argument("--key", "-k", - type=str, - default="ssl/server.pem", - help="key for ssl connnection") - parser.add_argument("--cert", "-cert", - type=str, - default="ssl/server.crt", - help="certificate") - parser.add_argument("--clientCA", - type=str, - default="", - help="CA for client certificates") - parser.add_argument("--clientverify", "-cverify", - type=_argparse_bool, - default=False, - help="verify client cert") - parser.add_argument("--load", - dest='load', - type=str, - default='', - help="A file which will install observers on hooks") - parser.add_argument("--lookupkey", - type=str, - default="{PATH}", - help="format string used as a key for response lookup: \ - example: \"{%%Host}{%%Server}{PATH}\", \"{HOST}{PATH}\", \"{PATH}\"\ - All the args preceded by %% are header fields in the request\ - The only two acceptable arguments which are not header fields are : fqdn (represented by HOST) and the url path (represented by PATH) in a request line.\ - Example: given a client request as << GET /some/resource/location HTTP/1.1\nHost: hahaha.com\n\n >>, if the user wishes the host field and the path to be used for the response lookup\ - then the required format will be {%%Host}{PATH}") - - args = parser.parse_args() - global time_delay - time_delay = args.delay - - # set up global dictionary of {uuid (string): response (Response object)} - s = sv.SessionValidator(args.data_dir) - populate_global_replay_dictionary(s.getSessionIter()) - print("Dropped {0} sessions for being malformed".format(len(s.getBadSessionList()))) - - # start server - try: - socket_timeout = args.timeout - test_mode_enabled = args.mode == "test" - global lookup_key_ - lookup_key_ = args.lookupkey - MyHandler.protocol_version = HTTP_VERSION - - if IPConstants.isIPv6(args.ip_address): - print("Server running on IPv6") - HTTPServer.address_family = socket.AF_INET6 - - if args.ssl == "True" or args.ssl == "true": - server = SSLServer((IPConstants.getIP(args.ip_address), args.port), MyHandler, args) - else: - server = ThreadingServer((IPConstants.getIP(args.ip_address), args.port), MyHandler, args) - - server.timeout = 5 - print("Started server on port {0}".format(args.port)) - server_thread = threading.Thread(target=server.serve_forever) - server_thread.daemon = True - server_thread.start() - - try: - while 1: - time.sleep(1) - sys.stderr.flush() - sys.stdout.flush() - except KeyboardInterrupt: - print("\n=== ^C received, shutting down microservers ===") - server.shutdown() - server_thread.join() - - except KeyboardInterrupt: - print("\n=== ^C received, shutting down httpserver ===") - server.socket.close() - # s_server.socket.close() - sys.exit(0) - - -if __name__ == '__main__': - main() diff --git a/tests/tools/microServer/ssl/server.crt b/tests/tools/microserver/ssl/server.crt similarity index 100% rename from tests/tools/microServer/ssl/server.crt rename to tests/tools/microserver/ssl/server.crt diff --git a/tests/tools/microserver/ssl/server.pem b/tests/tools/microserver/ssl/server.pem new file mode 100644 index 00000000000..58b9b9715b7 --- /dev/null +++ b/tests/tools/microserver/ssl/server.pem @@ -0,0 +1,32 @@ +-----BEGIN RSA PRIVATE KEY----- +MIICXQIBAAKBgQDWMHOiUF+ORmZjAxI8MWE9dblb7gQSJ36WCXlPFiFx6ynF+S1E +kXAYpIip5X0pzDUaIbLukxJUAAnOtMEO0PCgxJQUrEtRWh8wiJdbdQJF0Zs/9R+u +SUgb61f+mdTQvhqefBGx+xrpfAcgtcWiZuSA9Q3fvpDj5WOWSPWXBUuxywIDAQAB +AoGBAJPxRX2gjFAGWmQbU/YVmXfNH6navh8X/nx9sLeqrpE0AFeJI/ZPiqDKzMal +B43eSfNxwVi+ZxN0L1ICUbL9KKZvHs/QBxWLA1fGVAXrz7sRplEVvakPpTfHoEnv +sKaMWVKaK/S5WGbDhElb6zb/Lwo19DsIAPjGYqFvzFJBmobJAkEA9iSeTGkR9X26 +GywZoYrIMlRh34htOIRx1UUq88rFzdrCF21kQ4lhBIkX5OZMMy652i2gyak4OZTe +YewIv8jw9QJBAN7EQNHG8jPwXfVp91/fqxVQEfumuP2i6uiWWYQgZCmla2+0xcLZ +pMQ6sQEe10hhTrVnzHgAUVp50Ntn2jwBX78CQF09veGAI9d1Cxzj9cmmAvRd1r2Q +tp8kPOLnUsALXib+6WtqewLCdcf8DtsdClyRJMIraq85tRzK8fryKNZNzkkCQEgA +yS7FDj5JgCU15hZgFk1iPx3HCt44jZM2HaL+UUHAzRQjKxTLAl3G1rWVAWLMyQML +lORoveLvotl4HOruSsMCQQCAx9dV9JUSFoyc1CWILp/FgUH/se4cjQCThGO0DoQQ +vGTYmntY7j9WRJ9esQrjdD6Clw8zM/45GIBNwnXzqo7Z +-----END RSA PRIVATE KEY----- +-----BEGIN CERTIFICATE----- +MIICszCCAhwCCQD4jSkztmlO1TANBgkqhkiG9w0BAQsFADCBnTELMAkGA1UEBhMC +VVMxCzAJBgNVBAgTAklMMRIwEAYDVQQHEwlDaGFtcGFpZ24xDjAMBgNVBAoTBVlh +aG9vMQ0wCwYDVQQLEwRFZGdlMSgwJgYDVQQDEx9qdWljZXByb2R1Y2UuY29ycC5u +ZTEueWFob28uY29tMSQwIgYJKoZIhvcNAQkBFhVwZXJzaWEuYXppekB5YWhvby5j +b20wHhcNMTcwODI4MDM0NDQ1WhcNMjcwODI2MDM0NDQ1WjCBnTELMAkGA1UEBhMC +VVMxCzAJBgNVBAgTAklMMRIwEAYDVQQHEwlDaGFtcGFpZ24xDjAMBgNVBAoTBVlh +aG9vMQ0wCwYDVQQLEwRFZGdlMSgwJgYDVQQDEx9qdWljZXByb2R1Y2UuY29ycC5u +ZTEueWFob28uY29tMSQwIgYJKoZIhvcNAQkBFhVwZXJzaWEuYXppekB5YWhvby5j +b20wgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBANYwc6JQX45GZmMDEjwxYT11 +uVvuBBInfpYJeU8WIXHrKcX5LUSRcBikiKnlfSnMNRohsu6TElQACc60wQ7Q8KDE +lBSsS1FaHzCIl1t1AkXRmz/1H65JSBvrV/6Z1NC+Gp58EbH7Gul8ByC1xaJm5ID1 +Dd++kOPlY5ZI9ZcFS7HLAgMBAAEwDQYJKoZIhvcNAQELBQADgYEATX7975NdhIbJ +glda+sXI9a86GgOpiuKO+vKubRJQZA+UlPf2vHEONjC2+7Y1aZvZYaKYL74vxGky +zkgp6ANSPl45lqD632x0e1Z7vzW5TkqK1JB2/xH2WgDcQZmP0FuQHzVNs4GjghDr +HCp1+sQDhfPB4aLmLFeyN0TkhdH1N3M= +-----END CERTIFICATE----- diff --git a/tests/tools/plugins/cont_schedule.cc b/tests/tools/plugins/cont_schedule.cc new file mode 100644 index 00000000000..e678029f659 --- /dev/null +++ b/tests/tools/plugins/cont_schedule.cc @@ -0,0 +1,331 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include // for abort +#include // for debug + +// debug messages viewable by setting 'proxy.config.diags.debug.tags' +// in 'records.config' + +// debug messages during one-time initialization +static const char DEBUG_TAG_INIT[] = "TSContSchedule_test.init"; +static const char DEBUG_TAG_SCHD[] = "TSContSchedule_test.schedule"; +static const char DEBUG_TAG_HDL[] = "TSContSchedule_test.handler"; +static const char DEBUG_TAG_CHK[] = "TSContSchedule_test.check"; + +// plugin registration info +static char plugin_name[] = "TSContSchedule_test"; +static char vendor_name[] = "apache"; +static char support_email[] = "duke8253@apache.org"; + +static int test_flag = 0; + +static TSEventThread test_thread = nullptr; +static TSThread check_thread = nullptr; + +static int TSContSchedule_handler_1(TSCont contp, TSEvent event, void *edata); +static int TSContSchedule_handler_2(TSCont contp, TSEvent event, void *edata); +static int TSContScheduleOnPool_handler_1(TSCont contp, TSEvent event, void *edata); +static int TSContScheduleOnPool_handler_2(TSCont contp, TSEvent event, void *edata); +static int TSContScheduleOnThread_handler_1(TSCont contp, TSEvent event, void *edata); +static int TSContScheduleOnThread_handler_2(TSCont contp, TSEvent event, void *edata); +static int TSContThreadAffinity_handler(TSCont contp, TSEvent event, void *edata); + +static int +TSContSchedule_handler_1(TSCont contp, TSEvent event, void *edata) +{ + TSDebug(DEBUG_TAG_HDL, "TSContSchedule handler 1 thread [%p]", TSThreadSelf()); + if (test_thread == nullptr) { + test_thread = TSEventThreadSelf(); + + TSCont contp_new = TSContCreate(TSContSchedule_handler_2, TSMutexCreate()); + + if (contp_new == nullptr) { + TSDebug(DEBUG_TAG_HDL, "[%s] could not create continuation", plugin_name); + abort(); + } else { + TSDebug(DEBUG_TAG_HDL, "[%s] scheduling continuation", plugin_name); + TSContThreadAffinitySet(contp_new, test_thread); + TSContSchedule(contp_new, 0); + TSContSchedule(contp_new, 100); + } + } else if (check_thread == nullptr) { + TSDebug(DEBUG_TAG_CHK, "fail [schedule delay not applied]"); + } else { + if (check_thread != TSThreadSelf()) { + TSDebug(DEBUG_TAG_CHK, "pass [should not be the same thread]"); + } else { + TSDebug(DEBUG_TAG_CHK, "fail [on the same thread]"); + } + } + return 0; +} + +static int +TSContSchedule_handler_2(TSCont contp, TSEvent event, void *edata) +{ + TSDebug(DEBUG_TAG_HDL, "TSContSchedule handler 2 thread [%p]", TSThreadSelf()); + if (check_thread == nullptr) { + check_thread = TSThreadSelf(); + } else if (check_thread == TSThreadSelf()) { + TSDebug(DEBUG_TAG_CHK, "pass [should be the same thread]"); + } else { + TSDebug(DEBUG_TAG_CHK, "fail [not the same thread]"); + } + return 0; +} + +static int +TSContScheduleOnPool_handler_1(TSCont contp, TSEvent event, void *edata) +{ + TSDebug(DEBUG_TAG_HDL, "TSContScheduleOnPool handler 1 thread [%p]", TSThreadSelf()); + if (check_thread == nullptr) { + check_thread = TSThreadSelf(); + } else { + if (check_thread != TSThreadSelf()) { + TSDebug(DEBUG_TAG_CHK, "pass [should not be the same thread]"); + } else { + TSDebug(DEBUG_TAG_CHK, "fail [on the same thread]"); + } + check_thread = nullptr; + } + return 0; +} + +static int +TSContScheduleOnPool_handler_2(TSCont contp, TSEvent event, void *edata) +{ + TSDebug(DEBUG_TAG_HDL, "TSContScheduleOnPool handler 2 thread [%p]", TSThreadSelf()); + if (check_thread == nullptr) { + check_thread = TSThreadSelf(); + } else { + if (check_thread == TSThreadSelf()) { + TSDebug(DEBUG_TAG_CHK, "pass [should be the same thread]"); + } else { + TSDebug(DEBUG_TAG_CHK, "fail [not the same thread]"); + } + check_thread = TSThreadSelf(); + } + return 0; +} + +static int +TSContScheduleOnThread_handler_1(TSCont contp, TSEvent event, void *edata) +{ + TSDebug(DEBUG_TAG_HDL, "TSContScheduleOnThread handler 1 thread [%p]", TSThreadSelf()); + if (test_thread == nullptr) { + test_thread = TSEventThreadSelf(); + + TSCont contp_new = TSContCreate(TSContScheduleOnThread_handler_2, TSMutexCreate()); + + if (contp_new == nullptr) { + TSDebug(DEBUG_TAG_HDL, "[%s] could not create continuation", plugin_name); + abort(); + } else { + TSDebug(DEBUG_TAG_HDL, "[%s] scheduling continuation", plugin_name); + TSContScheduleOnThread(contp_new, 0, test_thread); + TSContScheduleOnThread(contp_new, 100, test_thread); + } + } else if (check_thread == nullptr) { + TSDebug(DEBUG_TAG_CHK, "fail [schedule delay not applied]"); + } else { + if (check_thread != TSThreadSelf()) { + TSDebug(DEBUG_TAG_CHK, "pass [should not be the same thread]"); + } else { + TSDebug(DEBUG_TAG_CHK, "fail [on the same thread]"); + } + } + return 0; +} + +static int +TSContScheduleOnThread_handler_2(TSCont contp, TSEvent event, void *edata) +{ + TSDebug(DEBUG_TAG_HDL, "TSContScheduleOnThread handler 2 thread [%p]", TSThreadSelf()); + if (check_thread == nullptr) { + check_thread = TSThreadSelf(); + } else if (check_thread == TSThreadSelf()) { + TSDebug(DEBUG_TAG_CHK, "pass [should be the same thread]"); + } else { + TSDebug(DEBUG_TAG_CHK, "fail [not the same thread]"); + } + return 0; +} + +static int +TSContThreadAffinity_handler(TSCont contp, TSEvent event, void *edata) +{ + TSDebug(DEBUG_TAG_HDL, "TSContThreadAffinity handler thread [%p]", TSThreadSelf()); + + test_thread = TSEventThreadSelf(); + + if (TSContThreadAffinityGet(contp) != nullptr) { + TSDebug(DEBUG_TAG_CHK, "pass [affinity thread is not null]"); + TSContThreadAffinityClear(contp); + if (TSContThreadAffinityGet(contp) == nullptr) { + TSDebug(DEBUG_TAG_CHK, "pass [affinity thread is cleared]"); + TSContThreadAffinitySet(contp, TSEventThreadSelf()); + if (TSContThreadAffinityGet(contp) == test_thread) { + TSDebug(DEBUG_TAG_CHK, "pass [affinity thread is set]"); + } else { + TSDebug(DEBUG_TAG_CHK, "fail [affinity thread is not set]"); + } + } else { + TSDebug(DEBUG_TAG_CHK, "fail [affinity thread is not cleared]"); + } + } else { + TSDebug(DEBUG_TAG_CHK, "fail [affinity thread is null]"); + } + + return 0; +} + +void +TSContSchedule_test() +{ + TSCont contp = TSContCreate(TSContSchedule_handler_1, TSMutexCreate()); + + if (contp == nullptr) { + TSDebug(DEBUG_TAG_SCHD, "[%s] could not create continuation", plugin_name); + abort(); + } else { + TSDebug(DEBUG_TAG_SCHD, "[%s] scheduling continuation", plugin_name); + TSContScheduleOnPool(contp, 0, TS_THREAD_POOL_NET); + TSContThreadAffinityClear(contp); + TSContScheduleOnPool(contp, 200, TS_THREAD_POOL_NET); + } +} + +void +TSContScheduleOnPool_test() +{ + TSCont contp_1 = TSContCreate(TSContScheduleOnPool_handler_1, TSMutexCreate()); + TSCont contp_2 = TSContCreate(TSContScheduleOnPool_handler_2, TSMutexCreate()); + + if (contp_1 == nullptr || contp_2 == nullptr) { + TSDebug(DEBUG_TAG_SCHD, "[%s] could not create continuation", plugin_name); + abort(); + } else { + TSDebug(DEBUG_TAG_SCHD, "[%s] scheduling continuation", plugin_name); + TSContScheduleOnPool(contp_1, 0, TS_THREAD_POOL_NET); + TSContThreadAffinityClear(contp_1); + TSContScheduleOnPool(contp_1, 100, TS_THREAD_POOL_NET); + TSContScheduleOnPool(contp_2, 200, TS_THREAD_POOL_TASK); + TSContThreadAffinityClear(contp_2); + TSContScheduleOnPool(contp_2, 300, TS_THREAD_POOL_TASK); + } +} + +void +TSContScheduleOnThread_test() +{ + TSCont contp = TSContCreate(TSContScheduleOnThread_handler_1, TSMutexCreate()); + + if (contp == nullptr) { + TSDebug(DEBUG_TAG_SCHD, "[%s] could not create continuation", plugin_name); + abort(); + } else { + TSDebug(DEBUG_TAG_SCHD, "[%s] scheduling continuation", plugin_name); + TSContScheduleOnPool(contp, 0, TS_THREAD_POOL_NET); + TSContThreadAffinityClear(contp); + TSContScheduleOnPool(contp, 200, TS_THREAD_POOL_NET); + } +} + +void +TSContThreadAffinity_test() +{ + TSCont contp = TSContCreate(TSContThreadAffinity_handler, TSMutexCreate()); + + if (contp == nullptr) { + TSDebug(DEBUG_TAG_SCHD, "[%s] could not create continuation", plugin_name); + abort(); + } else { + TSDebug(DEBUG_TAG_SCHD, "[%s] scheduling continuation", plugin_name); + TSContScheduleOnPool(contp, 0, TS_THREAD_POOL_NET); + } +} + +static int +LifecycleHookTracer(TSCont contp, TSEvent event, void *edata) +{ + if (event == TS_EVENT_LIFECYCLE_TASK_THREADS_READY) { + switch (test_flag) { + case 1: + TSContSchedule_test(); + break; + case 2: + TSContScheduleOnPool_test(); + break; + case 3: + TSContScheduleOnThread_test(); + break; + case 4: + TSContThreadAffinity_test(); + break; + default: + break; + } + } + return 0; +} + +void +TSPluginInit(int argc, const char *argv[]) +{ + if (argc == 1) { + TSDebug(DEBUG_TAG_INIT, "initializing plugin for testing TSContSchedule"); + test_flag = 1; + } else if (argc == 2) { + int len = strlen(argv[1]); + if (len == 4 && strncmp(argv[1], "pool", 4) == 0) { + TSDebug(DEBUG_TAG_INIT, "initializing plugin for testing TSContScheduleOnPool"); + test_flag = 2; + } else if (len == 6 && strncmp(argv[1], "thread", 6) == 0) { + TSDebug(DEBUG_TAG_INIT, "initializing plugin for testing TSContScheduleOnThread"); + test_flag = 3; + } else if (len == 8 && strncmp(argv[1], "affinity", 8) == 0) { + TSDebug(DEBUG_TAG_INIT, "initializing plugin for testing TSContThreadAffinity"); + test_flag = 4; + } else { + goto Lerror; + } + } else { + goto Lerror; + } + + TSPluginRegistrationInfo info; + + info.plugin_name = plugin_name; + info.vendor_name = vendor_name; + info.support_email = support_email; + + if (TSPluginRegister(&info) != TS_SUCCESS) { + TSDebug(DEBUG_TAG_INIT, "[%s] plugin registration failed", plugin_name); + abort(); + } + + TSLifecycleHookAdd(TS_LIFECYCLE_TASK_THREADS_READY_HOOK, TSContCreate(LifecycleHookTracer, TSMutexCreate())); + + return; + +Lerror: + TSDebug(DEBUG_TAG_INIT, "[%s] plugin invalid argument", plugin_name); + abort(); +} diff --git a/tests/tools/plugins/continuations_verify.cc b/tests/tools/plugins/continuations_verify.cc index d87fd51993c..079c0495b28 100644 --- a/tests/tools/plugins/continuations_verify.cc +++ b/tests/tools/plugins/continuations_verify.cc @@ -23,35 +23,46 @@ limitations under the License. */ -#define __STDC_FORMAT_MACROS 1 // for inttypes.h -#include // for PRIu64 -#include #include // for abort -#include // for NULL macro #include // for debug -// TODO Is LIFECYCLE_MSG enabled in 6.2.0, or 7.0.0, might require push -// with version rework - // debug messages viewable by setting 'proxy.config.diags.debug.tags' // in 'records.config' // debug messages during one-time initialization static const char DEBUG_TAG_INIT[] = "continuations_verify.init"; +static const char DEBUG_TAG_MSG[] = "continuations_verify.msg"; +static const char DEBUG_TAG_HOOK[] = "continuations_verify.hook"; // plugin registration info static char plugin_name[] = "continuations_verify"; -static char vendor_name[] = "Yahoo! Inc."; -static char support_email[] = "ats-devel@yahoo-inc.com"; - -static TSMutex order_mutex_1; // lock on global data -static TSMutex order_mutex_2; // lock on global data +static char vendor_name[] = "apache"; +static char support_email[] = "shinrich@apache.org"; // Statistics provided by the plugin static int stat_ssn_close_1 = 0; // number of TS_HTTP_SSN_CLOSE hooks caught by the first continuation static int stat_ssn_close_2 = 0; // number of TS_HTTP_SSN_CLOSE hooks caught by the second continuation static int stat_txn_close_1 = 0; // number of TS_HTTP_TXN_CLOSE hooks caught by the first continuation static int stat_txn_close_2 = 0; // number of TS_HTTP_TXN_CLOSE hooks caught by the second continuation +static int stat_test_done = 0; // Incremented when receiving a traffic_ctl message + +static int +handle_msg(TSCont contp, TSEvent event, void *edata) +{ + if (event == TS_EVENT_LIFECYCLE_MSG) { // External trigger, such as traffic_ctl + TSDebug(DEBUG_TAG_MSG, "event TS_EVENT_LIFECYCLE_MSG"); + // Send to a ET net thread just to be sure. + // Turns out the msg is sent to a task thread, but task + // threads do not get their thread local copy of the stats + // merged in. So externally, test.done was stuck at 0 without + // the Schedule to a NET thread + TSContScheduleOnPool(contp, 0, TS_THREAD_POOL_NET); + } else { + TSDebug(DEBUG_TAG_MSG, "event %d", event); + TSStatIntIncrement(stat_test_done, 1); + } + return 0; +} /** This function is called on every request and logs session and transaction @@ -64,6 +75,8 @@ handle_order_1(TSCont contp, TSEvent event, void *edata) TSHttpSsn ssnp; // session data TSHttpTxn txnp; // transaction data + TSDebug(DEBUG_TAG_HOOK, "order_1 event %d", event); + // Find the event that happened switch (event) { case TS_EVENT_HTTP_TXN_CLOSE: // End of transaction @@ -92,6 +105,8 @@ handle_order_2(TSCont contp, TSEvent event, void *edata) TSHttpSsn ssnp; // session data TSHttpTxn txnp; // transaction data + TSDebug(DEBUG_TAG_HOOK, "order_2 event %d", event); + // Find the event that happened switch (event) { case TS_EVENT_HTTP_TXN_CLOSE: // End of transaction @@ -142,14 +157,11 @@ TSPluginInit(int argc, const char *argv[]) TSError("[%s] Plugin registration failed. \n", plugin_name); } - order_mutex_1 = TSMutexCreate(); - TSCont contp_1; - order_mutex_2 = TSMutexCreate(); - TSCont contp_2; + TSCont contp_1 = TSContCreate(handle_order_1, TSMutexCreate()); + TSCont contp_2 = TSContCreate(handle_order_2, TSMutexCreate()); + TSCont contp = TSContCreate(handle_msg, TSMutexCreate()); - contp_1 = TSContCreate(handle_order_1, order_mutex_1); - contp_2 = TSContCreate(handle_order_2, order_mutex_2); - if (contp_1 == NULL || contp_2 == NULL) { + if (contp_1 == nullptr || contp_2 == nullptr || contp == nullptr) { // Continuation initialization failed. Unrecoverable, report and exit. TSError("[%s] could not create continuation", plugin_name); abort(); @@ -164,6 +176,8 @@ TSPluginInit(int argc, const char *argv[]) TSStatCreate("continuations_verify.txn.close.2", TS_RECORDDATATYPE_INT, TS_STAT_NON_PERSISTENT, TS_STAT_SYNC_SUM); stat_ssn_close_2 = TSStatCreate("continuations_verify.ssn.close.2", TS_RECORDDATATYPE_INT, TS_STAT_NON_PERSISTENT, TS_STAT_SYNC_SUM); + stat_test_done = + TSStatCreate("continuations_verify.test.done", TS_RECORDDATATYPE_INT, TS_STAT_NON_PERSISTENT, TS_STAT_SYNC_SUM); // Add all hooks. TSHttpHookAdd(TS_HTTP_TXN_CLOSE_HOOK, contp_1); @@ -171,5 +185,8 @@ TSPluginInit(int argc, const char *argv[]) TSHttpHookAdd(TS_HTTP_TXN_CLOSE_HOOK, contp_2); TSHttpHookAdd(TS_HTTP_SSN_CLOSE_HOOK, contp_2); + + // Respond to a traffic_ctl message + TSLifecycleHookAdd(TS_LIFECYCLE_MSG_HOOK, contp); } } diff --git a/tests/tools/plugins/ssl_hook_test.cc b/tests/tools/plugins/ssl_hook_test.cc index f7fde1c3915..2c9f02f57ea 100644 --- a/tests/tools/plugins/ssl_hook_test.cc +++ b/tests/tools/plugins/ssl_hook_test.cc @@ -73,7 +73,7 @@ CB_Pre_Accept_Delay(TSCont cont, TSEvent event, void *edata) TSContDataSet(cb, ssl_vc); // Schedule to reenable in a bit - TSContSchedule(cb, 2000, TS_THREAD_POOL_NET); + TSContScheduleOnPool(cb, 2000, TS_THREAD_POOL_NET); return TS_SUCCESS; } @@ -108,7 +108,7 @@ CB_out_start_delay(TSCont cont, TSEvent event, void *edata) TSContDataSet(cb, ssl_vc); // Schedule to reenable in a bit - TSContSchedule(cb, 2000, TS_THREAD_POOL_NET); + TSContScheduleOnPool(cb, 2000, TS_THREAD_POOL_NET); return TS_SUCCESS; } @@ -141,6 +141,38 @@ CB_out_close(TSCont cont, TSEvent event, void *edata) TSVConnReenable(ssl_vc); return TS_SUCCESS; } +int +CB_Client_Hello_Immediate(TSCont cont, TSEvent event, void *edata) +{ + TSVConn ssl_vc = reinterpret_cast(edata); + + int count = reinterpret_cast(TSContDataGet(cont)); + + TSDebug(PN, "Client Hello callback %d ssl_vc=%p", count, ssl_vc); + + // All done, reactivate things + TSVConnReenable(ssl_vc); + return TS_SUCCESS; +} + +int +CB_Client_Hello(TSCont cont, TSEvent event, void *edata) +{ + TSVConn ssl_vc = reinterpret_cast(edata); + + int count = reinterpret_cast(TSContDataGet(cont)); + + TSDebug(PN, "Client Hello callback %d ssl_vc=%p", count, ssl_vc); + + TSCont cb = TSContCreate(&ReenableSSL, TSMutexCreate()); + + TSContDataSet(cb, ssl_vc); + + // Schedule to reenable in a bit + TSContScheduleOnPool(cb, 2000, TS_THREAD_POOL_NET); + + return TS_SUCCESS; +} int CB_SNI(TSCont cont, TSEvent event, void *edata) @@ -183,15 +215,15 @@ CB_Cert(TSCont cont, TSEvent event, void *edata) TSContDataSet(cb, ssl_vc); // Schedule to reenable in a bit - TSContSchedule(cb, 2000, TS_THREAD_POOL_NET); + TSContScheduleOnPool(cb, 2000, TS_THREAD_POOL_NET); return TS_SUCCESS; } void -parse_callbacks(int argc, const char *argv[], int &preaccept_count, int &sni_count, int &cert_count, int &cert_count_immediate, - int &preaccept_count_delay, int &close_count, int &out_start_count, int &out_start_delay_count, - int &out_close_count) +parse_callbacks(int argc, const char *argv[], int &preaccept_count, int &client_hello_count, int &client_hello_count_immediate, + int &sni_count, int &cert_count, int &cert_count_immediate, int &preaccept_count_delay, int &close_count, + int &out_start_count, int &out_start_delay_count, int &out_close_count) { int i = 0; const char *ptr; @@ -215,6 +247,10 @@ parse_callbacks(int argc, const char *argv[], int &preaccept_count, int &sni_cou if (ptr) { if (strncmp(argv[i] + 1, "close", strlen("close")) == 0) { close_count = atoi(ptr + i); + } else if (strncmp(argv[i] + 1, "client_hello_imm", strlen("client_hello_imm")) == 0) { + client_hello_count_immediate = atoi(ptr + i); + } else if (strncmp(argv[i] + 1, "client_hello", strlen("client_hello")) == 0) { + client_hello_count = atoi(ptr + i); } else { cert_count = atoi(ptr + 1); } @@ -249,14 +285,15 @@ parse_callbacks(int argc, const char *argv[], int &preaccept_count, int &sni_cou } void -setup_callbacks(TSHttpTxn txn, int preaccept_count, int sni_count, int cert_count, int cert_count_immediate, - int preaccept_count_delay, int close_count, int out_start_count, int out_start_delay_count, int out_close_count) +setup_callbacks(TSHttpTxn txn, int preaccept_count, int client_hello_count, int client_hello_count_immediate, int sni_count, + int cert_count, int cert_count_immediate, int preaccept_count_delay, int close_count, int out_start_count, + int out_start_delay_count, int out_close_count) { TSCont cb = nullptr; // pre-accept callback continuation int i; - TSDebug(PN, "Setup callbacks pa=%d sni=%d cert=%d cert_imm=%d pa_delay=%d", preaccept_count, sni_count, cert_count, - cert_count_immediate, preaccept_count_delay); + TSDebug(PN, "Setup callbacks pa=%d client_hello=%d client_hello_imm=%d sni=%d cert=%d cert_imm=%d pa_delay=%d", preaccept_count, + client_hello_count, client_hello_count_immediate, sni_count, cert_count, cert_count_immediate, preaccept_count_delay); for (i = 0; i < preaccept_count; i++) { cb = TSContCreate(&CB_Pre_Accept, TSMutexCreate()); TSContDataSet(cb, (void *)(intptr_t)i); @@ -275,6 +312,24 @@ setup_callbacks(TSHttpTxn txn, int preaccept_count, int sni_count, int cert_coun TSHttpHookAdd(TS_VCONN_START_HOOK, cb); } } + for (i = 0; i < client_hello_count; i++) { + cb = TSContCreate(&CB_Client_Hello, TSMutexCreate()); + TSContDataSet(cb, (void *)(intptr_t)i); + if (txn) { + TSHttpTxnHookAdd(txn, TS_SSL_CLIENT_HELLO_HOOK, cb); + } else { + TSHttpHookAdd(TS_SSL_CLIENT_HELLO_HOOK, cb); + } + } + for (i = 0; i < client_hello_count_immediate; i++) { + cb = TSContCreate(&CB_Client_Hello_Immediate, TSMutexCreate()); + TSContDataSet(cb, (void *)(intptr_t)i); + if (txn) { + TSHttpTxnHookAdd(txn, TS_SSL_CLIENT_HELLO_HOOK, cb); + } else { + TSHttpHookAdd(TS_SSL_CLIENT_HELLO_HOOK, cb); + } + } for (i = 0; i < sni_count; i++) { cb = TSContCreate(&CB_SNI, TSMutexCreate()); TSContDataSet(cb, (void *)(intptr_t)i); @@ -355,18 +410,22 @@ TSPluginInit(int argc, const char *argv[]) TSError("[%s] Plugin registration failed", PN); } - int preaccept_count = 0; - int sni_count = 0; - int cert_count = 0; - int cert_count_immediate = 0; - int preaccept_count_delay = 0; - int close_count = 0; - int out_start_count = 0; - int out_start_delay_count = 0; - int out_close_count = 0; - parse_callbacks(argc, argv, preaccept_count, sni_count, cert_count, cert_count_immediate, preaccept_count_delay, close_count, - out_start_count, out_start_delay_count, out_close_count); - setup_callbacks(nullptr, preaccept_count, sni_count, cert_count, cert_count_immediate, preaccept_count_delay, close_count, - out_start_count, out_start_delay_count, out_close_count); + int preaccept_count = 0; + int client_hello_count = 0; + int client_hello_count_immediate = 0; + int sni_count = 0; + int cert_count = 0; + int cert_count_immediate = 0; + int preaccept_count_delay = 0; + int close_count = 0; + int out_start_count = 0; + int out_start_delay_count = 0; + int out_close_count = 0; + parse_callbacks(argc, argv, preaccept_count, client_hello_count, client_hello_count_immediate, sni_count, cert_count, + cert_count_immediate, preaccept_count_delay, close_count, out_start_count, out_start_delay_count, + out_close_count); + setup_callbacks(nullptr, preaccept_count, client_hello_count, client_hello_count_immediate, sni_count, cert_count, + cert_count_immediate, preaccept_count_delay, close_count, out_start_count, out_start_delay_count, + out_close_count); return; } diff --git a/tests/tools/plugins/ssl_sni_rename_test.cc b/tests/tools/plugins/ssl_sni_rename_test.cc new file mode 100644 index 00000000000..99a55ef4d74 --- /dev/null +++ b/tests/tools/plugins/ssl_sni_rename_test.cc @@ -0,0 +1,73 @@ +/** @file + + SSL Preaccept test plugin + Implements blind tunneling based on the client IP address + The client ip addresses are specified in the plugin's + config file as an array of IP addresses or IP address ranges under the + key "client-blind-tunnel" + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#include +#include +#include +#include +#include +#include +#include + +#define PN "ssl_rename_test" +#define PCP "[" PN " Plugin] " + +std::map bad_names; + +int +CB_server_rename(TSCont cont, TSEvent event, void *edata) +{ + TSVConn ssl_vc = reinterpret_cast(edata); + + TSSslConnection sslobj = TSVConnSSLConnectionGet(ssl_vc); + SSL *ssl = (SSL *)sslobj; + const char *sni_name = SSL_get_servername(ssl, TLSEXT_NAMETYPE_host_name); + if (!sni_name) { + SSL_set_tlsext_host_name(ssl, "newname"); + } + + // All done, reactivate things + TSVConnReenable(ssl_vc); + return TS_SUCCESS; +} + +// Called by ATS as our initialization point +void +TSPluginInit(int argc, const char *argv[]) +{ + TSPluginRegistrationInfo info; + info.plugin_name = const_cast("SSL rename test"); + info.vendor_name = const_cast("apache"); + info.support_email = const_cast("shinrich@apache.org"); + if (TSPluginRegister(&info) != TS_SUCCESS) { + TSError("[%s] Plugin registration failed", PN); + } + TSCont cb = TSContCreate(&CB_server_rename, TSMutexCreate()); + TSHttpHookAdd(TS_SSL_SERVERNAME_HOOK, cb); + + return; +} diff --git a/tests/tools/plugins/ssntxnorder_verify.cc b/tests/tools/plugins/ssntxnorder_verify.cc index e2889d8531e..fd0a2ce9749 100644 --- a/tests/tools/plugins/ssntxnorder_verify.cc +++ b/tests/tools/plugins/ssntxnorder_verify.cc @@ -22,19 +22,13 @@ See the License for the specific language governing permissions and limitations under the License. */ - -#define __STDC_FORMAT_MACROS 1 // for inttypes.h -#include // for PRIu64 #include -#include #include #include -#include // for abort -#include // for NULL macro -#include // for debug - -// TODO Is LIFECYCLE_MSG enabled in 6.2.0, or 7.0.0, might require push -// with version rework +#include // for abort +#include // for debug +#include // for PRIu64 +#include // debug messages viewable by setting 'proxy.config.diags.debug.tags' // in 'records.config' @@ -47,10 +41,8 @@ static const char DEBUG_TAG_HOOK[] = "ssntxnorder_verify.hook"; // plugin registration info static char plugin_name[] = "ssntxnorder_verify"; -static char vendor_name[] = "Yahoo! Inc."; -static char support_email[] = "ats-devel@yahoo-inc.com"; - -static TSMutex order_mutex; // lock on global data +static char vendor_name[] = "Apache"; +static char support_email[] = "shinrich@apache.org"; // List of started sessions, SSN_START seen, SSN_CLOSE not seen yet. static std::set started_ssns; @@ -84,6 +76,7 @@ static int stat_ssn_start = 0; // number of TS_HTTP_SSN_START hooks caught static int stat_txn_close = 0; // number of TS_HTTP_TXN_CLOSE hooks caught static int stat_txn_start = 0; // number of TS_HTTP_TXN_START hooks caught static int stat_err = 0; // number of inaccuracies encountered +static int stat_test_done = 0; // Set to 1 when the test is done // IPC information static char *ctl_tag = plugin_name; // name is a convenient identifier @@ -255,12 +248,18 @@ handle_order(TSCont contp, TSEvent event, void *edata) // Verify message is with the appropriate tag if (!strcmp(ctl_tag, msgp->tag) && strncmp(ctl_dump, reinterpret_cast(msgp->data), strlen(ctl_dump)) == 0) { dump_tables(); + } else { + TSContScheduleOnPool(contp, 0, TS_THREAD_POOL_NET); } break; } #endif + case TS_EVENT_IMMEDIATE: + TSStatIntIncrement(stat_test_done, 1); + break; + // Just release the lock for all other states and do nothing default: break; @@ -297,11 +296,8 @@ TSPluginInit(int argc, const char *argv[]) TSError("[%s] Plugin registration failed. \n", plugin_name); } - order_mutex = TSMutexCreate(); - TSCont contp; - - contp = TSContCreate(handle_order, order_mutex); - if (contp == NULL) { + TSCont contp = TSContCreate(handle_order, TSMutexCreate()); + if (contp == nullptr) { // Continuation initialization failed. Unrecoverable, report and exit. TSError("[%s] could not create continuation", plugin_name); abort(); @@ -313,6 +309,7 @@ TSPluginInit(int argc, const char *argv[]) stat_txn_start = TSStatCreate("ssntxnorder_verify.txn.start", TS_RECORDDATATYPE_INT, TS_STAT_NON_PERSISTENT, TS_STAT_SYNC_SUM); stat_txn_close = TSStatCreate("ssntxnorder_verify.txn.close", TS_RECORDDATATYPE_INT, TS_STAT_NON_PERSISTENT, TS_STAT_SYNC_SUM); stat_err = TSStatCreate("ssntxnorder_verify.err", TS_RECORDDATATYPE_INT, TS_STAT_NON_PERSISTENT, TS_STAT_SYNC_SUM); + stat_test_done = TSStatCreate("ssntxnorder_verify.test.done", TS_RECORDDATATYPE_INT, TS_STAT_NON_PERSISTENT, TS_STAT_SYNC_SUM); // Add all hooks. TSHttpHookAdd(TS_HTTP_SSN_START_HOOK, contp); diff --git a/tests/tools/plugins/test_hooks.cc b/tests/tools/plugins/test_hooks.cc index a369148d56f..67ff9bbda4e 100644 --- a/tests/tools/plugins/test_hooks.cc +++ b/tests/tools/plugins/test_hooks.cc @@ -44,6 +44,8 @@ char PIName[] = PINAME; // std::fstream logFile; +TSVConn activeVConn; + TSHttpSsn activeSsn; TSHttpTxn activeTxn; @@ -167,6 +169,40 @@ globalContFunc(TSCont, TSEvent event, void *eventData) TSDebug(PIName, "Global: event=%s(%d) eventData=%p", TSHttpEventNameLookup(event), event, eventData); switch (event) { + case TS_EVENT_VCONN_START: { + ALWAYS_ASSERT(!activeVConn) + + auto vConn = static_cast(eventData); + + activeVConn = vConn; + + logFile << "Global: ssl flag=" << TSVConnIsSsl(vConn) << std::endl; + + TSVConnReenable(vConn); + } break; + case TS_EVENT_SSL_CERT: + case TS_EVENT_SSL_SERVERNAME: { + auto vConn = static_cast(eventData); + + ALWAYS_ASSERT(vConn == activeVConn) + + logFile << "Global: ssl flag=" << TSVConnIsSsl(vConn) << std::endl; + + TSVConnReenable(vConn); + } break; + + case TS_EVENT_VCONN_CLOSE: { + auto vConn = static_cast(eventData); + + ALWAYS_ASSERT(vConn == activeVConn) + + logFile << "Global: ssl flag=" << TSVConnIsSsl(vConn) << std::endl; + + TSVConnReenable(vConn); + + activeVConn = nullptr; + } break; + case TS_EVENT_HTTP_SSN_START: { ALWAYS_ASSERT(!activeSsn) @@ -288,6 +324,21 @@ TSPluginInit(int argc, const char *argv[]) TSHttpHookAdd(TS_HTTP_SSN_CLOSE_HOOK, gCont); TSHttpHookAdd(TS_HTTP_TXN_START_HOOK, gCont); TSHttpHookAdd(TS_HTTP_TXN_CLOSE_HOOK, gCont); + TSHttpHookAdd(TS_SSL_CERT_HOOK, gCont); + TSHttpHookAdd(TS_SSL_SERVERNAME_HOOK, gCont); + // NOTE: as of January 2019 these two hooks are only triggered for TLS connections. It seems that, at trafficserver + // startup, spurious data on the TLS TCP port may cause trafficserver to attempt (and fail) to create a TLS + // connection. If this happens, it will result in TS_VCONN_START_HOOK being triggered, and then TS_VCONN_CLOSE_HOOK + // will be triggered when the connection closes due to failure. + // + TSHttpHookAdd(TS_VCONN_START_HOOK, gCont); + TSHttpHookAdd(TS_VCONN_CLOSE_HOOK, gCont); + + // TSHttpHookAdd(TS_SSL_SESSION_HOOK, gCont); -- Event is TS_EVENT_SSL_SESSION_NEW -- Event data is TSHttpSsn + // TSHttpHookAdd(TS_SSL_SERVER_VERIFY_HOOK, gCont); + // TSHttpHookAdd(TS_SSL_VERIFY_CLIENT_HOOK, gCont); + // TSHttpHookAdd(TS_VCONN_OUTBOUND_START_HOOK, gCont); + // TSHttpHookAdd(TS_VCONN_OUTBOUND_CLOSE_HOOK, gCont); sCont = TSContCreate(sessionContFunc, mtx); diff --git a/tests/tools/plugins/test_tsapi.cc b/tests/tools/plugins/test_tsapi.cc new file mode 100644 index 00000000000..986a904c751 --- /dev/null +++ b/tests/tools/plugins/test_tsapi.cc @@ -0,0 +1,197 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* +Regression testing code for TS API. Not comprehensive, hopefully will be built up over time. +*/ + +#include +#include +#include + +#include + +// TSReleaseAssert() doesn't seem to produce any logging output for a debug build, so do both kinds of assert. +// +#define ALWAYS_ASSERT(EXPR) \ + { \ + TSAssert(EXPR); \ + TSReleaseAssert(EXPR); \ + } + +namespace +{ +#define PINAME "test_tsapi" +char PIName[] = PINAME; + +// NOTE: It's important to flush this after writing so that a gold test using this plugin can examine the log before TS +// terminates. +// +std::fstream logFile; + +TSCont tCont, gCont; + +void +testsForReadReqHdrHook(TSHttpTxn txn) +{ + logFile << "TSHttpTxnEffectiveUrlStringGet(): "; + int urlLength; + char *urlStr = TSHttpTxnEffectiveUrlStringGet(txn, &urlLength); + if (!urlStr) { + logFile << "URL null" << std::endl; + } else if (0 == urlLength) { + logFile << "URL length zero" << std::endl; + } else if (0 > urlLength) { + logFile << "URL length negative" << std::endl; + } else { + logFile << std::string_view(urlStr, urlLength) << std::endl; + + TSfree(urlStr); + } +} + +int +transactionContFunc(TSCont, TSEvent event, void *eventData) +{ + logFile << "Transaction: event=" << TSHttpEventNameLookup(event) << std::endl; + + TSDebug(PIName, "Transaction: event=%s(%d) eventData=%p", TSHttpEventNameLookup(event), event, eventData); + + switch (event) { + case TS_EVENT_HTTP_READ_REQUEST_HDR: { + auto txn = static_cast(eventData); + + testsForReadReqHdrHook(txn); + + TSHttpTxnReenable(txn, TS_EVENT_HTTP_CONTINUE); + } break; + + default: { + ALWAYS_ASSERT(false) + } break; + + } // end switch + + return 0; +} + +int +globalContFunc(TSCont, TSEvent event, void *eventData) +{ + logFile << "Global: event=" << TSHttpEventNameLookup(event) << std::endl; + + TSDebug(PIName, "Global: event=%s(%d) eventData=%p", TSHttpEventNameLookup(event), event, eventData); + + switch (event) { + case TS_EVENT_HTTP_TXN_START: { + auto txn = static_cast(eventData); + + TSHttpTxnHookAdd(txn, TS_HTTP_READ_REQUEST_HDR_HOOK, tCont); + + TSHttpTxnReenable(txn, TS_EVENT_HTTP_CONTINUE); + } break; + + case TS_EVENT_HTTP_READ_REQUEST_HDR: { + auto txn = static_cast(eventData); + + testsForReadReqHdrHook(txn); + + TSHttpTxnReenable(txn, TS_EVENT_HTTP_CONTINUE); + } break; + + default: { + ALWAYS_ASSERT(false) + } break; + + } // end switch + + return 0; +} + +} // end anonymous namespace + +void +TSPluginInit(int argc, const char *argv[]) +{ + TSDebug(PIName, "TSPluginInit()"); + + TSPluginRegistrationInfo info; + + info.plugin_name = PIName; + info.vendor_name = "Apache Software Foundation"; + info.support_email = "dev@trafficserver.apache.org"; + + if (TSPluginRegister(&info) != TS_SUCCESS) { + TSError(PINAME ": Plugin registration failed"); + + return; + } + + const char *fileSpec = std::getenv("OUTPUT_FILE"); + + if (nullptr == fileSpec) { + TSError(PINAME ": Environment variable OUTPUT_FILE not found."); + + return; + } + + // Disable output buffering for logFile, so that explicit flushing is not necessary. + logFile.rdbuf()->pubsetbuf(nullptr, 0); + + logFile.open(fileSpec, std::ios::out); + if (!logFile.is_open()) { + TSError(PINAME ": could not open log file \"%s\"", fileSpec); + + return; + } + + // Mutex to protext the logFile object. + // + TSMutex mtx = TSMutexCreate(); + + gCont = TSContCreate(globalContFunc, mtx); + + TSHttpHookAdd(TS_HTTP_TXN_START_HOOK, gCont); + TSHttpHookAdd(TS_HTTP_READ_REQUEST_HDR_HOOK, gCont); + + tCont = TSContCreate(transactionContFunc, mtx); +} + +namespace +{ +class Cleanup +{ +public: + ~Cleanup() + { + // In practice it is not strictly necessary to destroy remaining continuations on program exit. + + if (tCont) { + TSContDestroy(tCont); + } + if (gCont) { + TSContDestroy(gCont); + } + } +}; + +// Do any needed cleanup for this source file at program termination time. +// +Cleanup cleanup; + +} // end anonymous namespace diff --git a/tests/tools/sessionvalidation/request.py b/tests/tools/sessionvalidation/request.py deleted file mode 100644 index 39598d7962b..00000000000 --- a/tests/tools/sessionvalidation/request.py +++ /dev/null @@ -1,48 +0,0 @@ -''' -''' -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import hashlib - - -class Request(object): - ''' Request encapsulates a single request from the UA ''' - - def getTimestamp(self): - return self._timestamp - - def getHeaders(self): - return self._headers - - def getBody(self): - return self._body - - def getHeaderMD5(self): - ''' Returns the MD5 hash of the headers - - This is used to do a unique mapping to a request/response transaction ''' - return hashlib.md5(self._headers.encode()).hexdigest() - - def __repr__(self): - # return str(self._timestamp) - return "".format( - str(self._timestamp), str(self._headers), str(self._body) - ) - - def __init__(self, timestamp, headers, body): - self._timestamp = timestamp - self._headers = headers - self._body = body diff --git a/tests/tools/sessionvalidation/response.py b/tests/tools/sessionvalidation/response.py deleted file mode 100644 index faa5f97255f..00000000000 --- a/tests/tools/sessionvalidation/response.py +++ /dev/null @@ -1,49 +0,0 @@ -''' -''' -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import re - - -class Response(object): - ''' Response encapsulates a single request from the UA ''' - - def getTimestamp(self): - return self._timestamp - - def getHeaders(self): - return self._headers - - def getBody(self): - return self._body - - def getOptions(self): - return self._options - - def __repr__(self): - return "".format( - self._timestamp, self._headers, self._body, self._options - ) - - def __init__(self, timestamp, headers, body, options_string): - self._timestamp = timestamp - self._headers = headers - self._body = body - if options_string: - self._options = re.compile(r'\s*,\s*').split(options_string) - else: - self._options = list() diff --git a/tests/tools/sessionvalidation/session.py b/tests/tools/sessionvalidation/session.py deleted file mode 100644 index e8bb0e2e463..00000000000 --- a/tests/tools/sessionvalidation/session.py +++ /dev/null @@ -1,45 +0,0 @@ -''' -''' -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import sessionvalidation.transaction as transaction - - -class Session(object): - ''' Session encapsulates a single user session ''' - - def getTransactionList(self): - ''' Returns a list of transaction objects ''' - return self._transaction_list - - def getTransactionIter(self): - ''' Returns an iterator of transaction objects ''' - return iter(self._transaction_list) - - def returnFirstTransaction(self): - return self._transaction_list[0] - - def __repr__(self): - return "".format( - self._filename, self._version, self._timestamp, self._encoding, repr(self._transaction_list) - ) - - def __init__(self, filename, version, timestamp, transaction_list, encoding=None): - self._filename = filename - self._version = version - self._timestamp = timestamp - self._encoding = encoding - self._transaction_list = transaction_list diff --git a/tests/tools/sessionvalidation/sessionvalidation.py b/tests/tools/sessionvalidation/sessionvalidation.py deleted file mode 100644 index 7ff97e6facf..00000000000 --- a/tests/tools/sessionvalidation/sessionvalidation.py +++ /dev/null @@ -1,259 +0,0 @@ -''' -''' -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import json -import os - -import sessionvalidation.session as session -import sessionvalidation.transaction as transaction -import sessionvalidation.request as request -import sessionvalidation.response as response - -# valid_HTTP_request_methods = ['GET', 'POST', 'HEAD'] -# custom_HTTP_request_methods = ['PULL'] # transaction monitor plugin for ATS may have custom methods -allowed_HTTP_request_methods = ['GET', 'POST', 'HEAD', 'PULL'] -G_CUSTOM_METHODS = False -G_VERBOSE_LOG = True - - -def _verbose_print(msg, verbose_on=False): - ''' Print msg if verbose_on is set to True or G_VERBOSE_LOG is set to True''' - if verbose_on or G_VERBOSE_LOG: - print(msg) - - -class SessionValidator(object): - ''' - SessionValidator parses, validates, and exports an API for a given set of JSON sessions generated from Apache Traffic Server - - SessionValidator is initialized with a path to a directory of JSON sessions. It then automatically parses and validates all the - session in the directory. After initialization, the user may use the provided API - - TODO : - Provide a list of guaranteed fields for each type of object (ie a Transaction has a request and a response, a request has ...) - ''' - - def parse(self): - ''' - Constructs Session objects from JSON files on disk and stores objects into _sessions - - All sessions missing required fields (ie. a session timestamp, a response for every request, etc) are - dropped and the filename is stored inside _bad_sessions - ''' - - log_filenames = [os.path.join(self._json_log_dir, f) for f in os.listdir( - self._json_log_dir) if os.path.isfile(os.path.join(self._json_log_dir, f))] - - for fname in log_filenames: - with open(fname) as f: - # first attempt to load the JSON - try: - sesh = json.load(f) - except: - self._bad_sessions.append(fname) - _verbose_print("Warning: JSON parse error on file={0}".format(fname)) - print("Warning: JSON parse error on file={0}".format(fname)) - continue - - # then attempt to extract all the required fields from the JSON - try: - session_timestamp = sesh['timestamp'] - session_version = sesh['version'] - session_txns = list() - for txn in sesh['txns']: - # create transaction Request object - txn_request = txn['request'] - - txn_request_body = '' - if 'body' in txn_request: - txn_request_body = txn_request['body'] - txn_request_obj = request.Request(txn_request['timestamp'], txn_request['headers'], txn_request_body) - # Create transaction Response object - txn_response = txn['response'] - txn_response_body = '' - if 'body' in txn_response: - txn_response_body = txn_response['body'] - txn_response_obj = response.Response(txn_response['timestamp'], txn_response['headers'], txn_response_body, - txn_response.get('options')) - - # create Transaction object - txn_obj = transaction.Transaction(txn_request_obj, txn_response_obj, txn['uuid']) - session_txns.append(txn_obj) - session_obj = session.Session(fname, session_version, session_timestamp, session_txns) - - except KeyError as e: - self._bad_sessions.append(fname) - print("Warning: parse error on key={0} for file={1}".format(e, fname)) - _verbose_print("Warning: parse error on key={0} for file={1}".format(e, fname)) - continue - - self._sessions.append(session_obj) - - def validate(self): - ''' Prunes out all the invalid Sessions in _sessions ''' - - good_sessions = list() - - for sesh in self._sessions: - if SessionValidator.validateSingleSession(sesh): - good_sessions.append(sesh) - else: - self._bad_sessions.append(sesh._filename) - - self._sessions = good_sessions - - @staticmethod - def validateSingleSession(sesh): - ''' Takes in a single Session object as input, returns whether or not the Session is valid ''' - - retval = True - - try: - # first validate fields - if not sesh._filename: - _verbose_print("bad session filename") - retval = False - elif not sesh._version: - _verbose_print("bad session version") - retval = False - elif float(sesh._timestamp) <= 0: - _verbose_print("bad session timestamp") - retval = False - elif not bool(sesh.getTransactionList()): - _verbose_print("session has no transaction list") - retval = False - - # validate Transactions now - for txn in sesh.getTransactionIter(): - if not SessionValidator.validateSingleTransaction(txn): - retval = False - - except ValueError as e: - _verbose_print("most likely an invalid session timestamp") - retval = False - - return retval - - @staticmethod - def validateSingleTransaction(txn): - ''' Takes in a single Transaction object as input, and returns whether or not the Transaction is valid ''' - - txn_req = txn.getRequest() - txn_resp = txn.getResponse() - retval = True - - #valid_HTTP_request_methods = ['GET', 'HEAD', 'POST', 'PUT', 'DELETE', 'TRACE', 'OPTIONS', 'CONNECT', 'PATCH'] - # we can later uncomment the previous line to support more HTTP methods - valid_HTTP_versions = ['HTTP/1.0', 'HTTP/1.1', 'HTTP/2.0'] - - try: - # validate request first - if not txn_req: - _verbose_print("no transaction request") - retval = False - elif txn_req.getBody() == None: - _verbose_print("transaction body is set to None") - retval = False - elif float(txn_req.getTimestamp()) <= 0: - _verbose_print("invalid transaction request timestamp") - retval = False - elif txn_req.getHeaders().split()[0] not in allowed_HTTP_request_methods: - _verbose_print("invalid HTTP method for transaction {0}".format(txn_req.getHeaders().split()[0])) - retval = False - elif not txn_req.getHeaders().endswith("\r\n\r\n"): - _verbose_print("transaction request headers didn't end with \\r\\n\\r\\n") - retval = False - elif txn_req.getHeaders().split()[2] not in valid_HTTP_versions: - _verbose_print("invalid HTTP version in request") - retval = False - - # if the Host header is not present and vaild we reject this transaction - found_host = False - for header in txn_req.getHeaders().split('\r\n'): - split_header = header.split(' ') - if split_header[0] == 'Host:': - found_host = True - host_header_no_space = len(split_header) == 1 - host_header_with_space = len(split_header) == 2 and split_header[1] == '' - if host_header_no_space or host_header_with_space: - found_host = False - if not found_host: - print("missing host", txn_req) - _verbose_print("transaction request Host header doesn't have specified host") - retval = False - - # now validate response - if not txn_resp: - _verbose_print("no transaction response") - retval = False - elif txn_resp.getBody() == None: - _verbose_print("transaction response body set to None") - retval = False - elif float(txn_resp.getTimestamp()) <= 0: - _verbose_print("invalid transaction response timestamp") - retval = False - elif txn_resp.getHeaders().split()[0] not in valid_HTTP_versions: - _verbose_print("invalid HTTP response header") - retval = False - elif not txn_resp.getHeaders().endswith("\r\n\r\n"): - _verbose_print("transaction response headers didn't end with \\r\\n\\r\\n") - retval = False - - # if any of the 3xx responses have bodies, then the must reject this transaction, since 3xx - # errors by definition can't have bodies - response_line = txn_resp.getHeaders().split('\r\n')[0] - response_code = response_line.split(' ')[1] - if response_code.startswith('3') and txn_resp.getBody(): - _verbose_print("transaction response was 3xx and had a body") - retval = False - - except ValueError as e: - _verbose_print("most likely an invalid transaction timestamp") - retval = False - - except IndexError as e: - _verbose_print("most likely a bad transaction header") - retval = False - - return retval - - def getSessionList(self): - ''' Returns the list of Session objects ''' - return self._sessions - - def getSessionIter(self): - ''' Returns an iterator of the Session objects ''' - return iter(self._sessions) - - def getBadSessionList(self): - ''' Returns a list of bad session filenames (list of strings) ''' - return self._bad_sessions - - def getBadSessionListIter(self): - ''' Returns an iterator of bad session filenames (iterator of strings) ''' - return iter(self._bad_sessions) - - def __init__(self, json_log_dir, allow_custom=False): - global valid_HTTP_request_methods - global G_CUSTOM_METHODS - G_CUSTOM_METHODS = allow_custom - self._json_log_dir = json_log_dir - self._bad_sessions = list() # list of filenames - self._sessions = list() # list of _good_ session objects - - self.parse() - self.validate() diff --git a/tests/tools/traffic-replay/NonSSL.py b/tests/tools/traffic-replay/NonSSL.py deleted file mode 100644 index 3d6b85b090d..00000000000 --- a/tests/tools/traffic-replay/NonSSL.py +++ /dev/null @@ -1,192 +0,0 @@ -#!/bin/env python3 -''' -''' -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import socket -import requests -import os -from threading import Thread -import sys -from multiprocessing import current_process -import sessionvalidation.sessionvalidation as sv -import lib.result as result -import extractHeader -import mainProcess -import json -import gzip -bSTOP = False - - -def createDummyBodywithLength(numberOfbytes): - if numberOfbytes <= 0: - return None - body = 'a' - while numberOfbytes != 1: - body += 'b' - numberOfbytes -= 1 - return body - - -def handleResponse(response, *args, **kwargs): - print(response.status_code) - # resp=args[0] - #expected_output_split = resp.getHeaders().split('\r\n')[ 0].split(' ', 2) - #expected_output = (int(expected_output_split[1]), str( expected_output_split[2])) - #r = result.Result(session_filename, expected_output[0], response.status_code) - # print(r.getResultString(colorize=True)) -# make sure len of the message body is greater than length - - -def gen(): - yield 'pforpersia,champaignurbana'.encode('utf-8') - yield 'there'.encode('utf-8') - - -def txn_replay(session_filename, txn, proxy, result_queue, request_session): - """ Replays a single transaction - :param request_session: has to be a valid requests session""" - req = txn.getRequest() - resp = txn.getResponse() - - # Construct HTTP request & fire it off - txn_req_headers = req.getHeaders() - txn_req_headers_dict = extractHeader.header_to_dict(txn_req_headers) - txn_req_headers_dict['Content-MD5'] = txn._uuid # used as unique identifier - if 'body' in txn_req_headers_dict: - del txn_req_headers_dict['body'] - - #print("Replaying session") - try: - # response = request_session.request(extractHeader.extract_txn_req_method(txn_req_headers), - # 'http://' + extractHeader.extract_host(txn_req_headers) + extractHeader.extract_GET_path(txn_req_headers), - # headers=txn_req_headers_dict,stream=False) # making stream=False raises contentdecoding exception? kill me - method = extractHeader.extract_txn_req_method(txn_req_headers) - response = None - body = None - content = None - if 'Transfer-Encoding' in txn_req_headers_dict: - # deleting the host key, since the STUPID post/get functions are going to add host field anyway, so there will be multiple host fields in the header - # This confuses the ATS and it returns 400 "Invalid HTTP request". I don't believe this - # BUT, this is not a problem if the data is not chunked encoded.. Strange, huh? - del txn_req_headers_dict['Host'] - if 'Content-Length' in txn_req_headers_dict: - #print("ewww !") - del txn_req_headers_dict['Content-Length'] - body = gen() - if 'Content-Length' in txn_req_headers_dict: - nBytes = int(txn_req_headers_dict['Content-Length']) - body = createDummyBodywithLength(nBytes) - #print("request session is",id(request_session)) - if method == 'GET': - r1 = request_session.request('GET', 'http://'+extractHeader.extract_host(txn_req_headers)+extractHeader.extract_GET_path( - txn_req_headers), headers=txn_req_headers_dict, data=body) - responseHeaders = r1.headers - responseContent = r1.content # byte array - - #print("len: {0} received {1}".format(responseHeaders['Content-Length'], responseContent)) - - elif method == 'POST': - r1 = request_session.request('POST', 'http://'+extractHeader.extract_host(txn_req_headers)+extractHeader.extract_GET_path( - txn_req_headers), headers=txn_req_headers_dict, data=body) - responseHeaders = r1.headers - responseContent = r1.content - - #print("len: {0} received {1}".format(responseHeaders['Content-Length'], responseContent)) - elif method == 'HEAD': - r1 = request_session.request('HEAD', 'http://'+extractHeader.extract_host(txn_req_headers)+extractHeader.extract_GET_path( - txn_req_headers), headers=txn_req_headers_dict, data=body) - responseHeaders = r1.headers - responseContent = r1.content - else: # EXPERIMENTAL - r1 = request_session.request(method, 'http://'+extractHeader.extract_host(txn_req_headers)+extractHeader.extract_GET_path( - txn_req_headers), headers=txn_req_headers_dict, data=body) - responseHeaders = r1.headers - responseContent = r1.content - - #gzip_file = gzip.GzipFile(fileobj=responseContent) - #shutil.copyfileobj(gzip_file, f) - - expected = extractHeader.responseHeader_to_dict(resp.getHeaders()) - # print("------------EXPECTED-----------") - # print(expected) - # print("------------RESP--------------") - # print(responseHeaders) - # print() - - if mainProcess.verbose: - expected_output_split = resp.getHeaders().split('\r\n')[0].split(' ', 2) - expected_output = (int(expected_output_split[1]), str(expected_output_split[2])) - r = result.Result(session_filename, expected_output[0], r1.status_code, responseContent) - b_res, res = r.getResult(responseHeaders, expected, colorize=True) - print(res) - - if not b_res: - print("Received response") - print(responseHeaders) - print("Expected response") - print(expected) - # result_queue.put(r) - except UnicodeEncodeError as e: - # these unicode errors are due to the interaction between Requests and our wiretrace data. - # TODO fix - print("UnicodeEncodeError exception") - - except requests.exceptions.ContentDecodingError as e: - print("ContentDecodingError", e) - except: - e = sys.exc_info() - print("ERROR in NonSSLReplay: ", e, response, session_filename) - - -def session_replay(input, proxy, result_queue): - global bSTOP - ''' Replay all transactions in session - - This entire session will be replayed in one requests.Session (so one socket / TCP connection)''' - # if timing_control: - # time.sleep(float(session._timestamp)) # allow other threads to run - while bSTOP == False: - for session in iter(input.get, 'STOP'): - # print(bSTOP) - if session == 'STOP': - print("Queue is empty") - bSTOP = True - break - with requests.Session() as request_session: - request_session.proxies = proxy - for txn in session.getTransactionIter(): - try: - txn_replay(session._filename, txn, proxy, result_queue, request_session) - except: - e = sys.exc_info() - print("ERROR in replaying: ", e, txn.getRequest().getHeaders()) - bSTOP = True - #print("Queue is empty") - input.put('STOP') - break - - -def client_replay(input, proxy, result_queue, nThread): - Threads = [] - for i in range(nThread): - t = Thread(target=session_replay, args=[input, proxy, result_queue]) - t.start() - Threads.append(t) - - for t1 in Threads: - t1.join() diff --git a/tests/tools/traffic-replay/RandomReplay.py b/tests/tools/traffic-replay/RandomReplay.py deleted file mode 100644 index f6bf8691ff4..00000000000 --- a/tests/tools/traffic-replay/RandomReplay.py +++ /dev/null @@ -1,91 +0,0 @@ -#!/bin/env python3 -''' -''' -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import socket -import requests -import os -from threading import Thread -import sys -from multiprocessing import current_process -import sessionvalidation.sessionvalidation as sv -from collections import deque -import collections -import lib.result as result -import extractHeader -import mainProcess -import json -import gzip -import NonSSL -import SSLReplay -import h2Replay -import itertools -import random -bSTOP = False - - -def session_replay(input, proxy, result_queue): - global bSTOP - ''' Replay all transactions in session - - This entire session will be replayed in one requests.Session (so one socket / TCP connection)''' - # if timing_control: - # time.sleep(float(session._timestamp)) # allow other threads to run - while bSTOP == False: - for session in iter(input.get, 'STOP'): - # print(bSTOP) - if session == 'STOP': - print("Queue is empty") - bSTOP = True - break - with requests.Session() as request_session: - request_session.proxies = proxy - for txn in session.getTransactionIter(): - type = random.randint(1, 1000) - try: - if type % 3 == 0: - NonSSL.txn_replay(session._filename, txn, proxy, result_queue, request_session) - elif type % 3 == 1: - SSLReplay.txn_replay(session._filename, txn, proxy, result_queue, request_session) - elif type % 3 == 2: - h2Replay.txn_replay(session._filename, txn, proxy, result_queue, request_session) - except: - e = sys.exc_info() - print("ERROR in replaying: ", e, txn.getRequest().getHeaders()) - bSTOP = True - #print("Queue is empty") - input.put('STOP') - break - - -def client_replay(input, proxy, result_queue, nThread): - Threads = [] - for i in range(nThread): - - t2 = Thread(target=SSLReplay.session_replay, args=[input, proxy, result_queue]) - t = Thread(target=NonSSL.session_replay, args=[input, proxy, result_queue]) - t1 = Thread(target=h2Replay.session_replay, args=[input, proxy, result_queue]) - t2.start() - t.start() - t1.start() - Threads.append(t) - Threads.append(t2) - Threads.append(t1) - - for t1 in Threads: - t1.join() diff --git a/tests/tools/traffic-replay/SSLReplay.py b/tests/tools/traffic-replay/SSLReplay.py deleted file mode 100644 index c75b6a5b014..00000000000 --- a/tests/tools/traffic-replay/SSLReplay.py +++ /dev/null @@ -1,233 +0,0 @@ -#!/bin/env python3 -''' -''' -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import http.client -import socket -import ssl -import pprint -# import gevent -import requests -import os -#import threading -import sys -from multiprocessing import current_process -import sessionvalidation.sessionvalidation as sv -import lib.result as result -import extractHeader -# from gevent import monkey, sleep -from threading import Thread -import mainProcess -import json -import extractHeader -import time -import Config -bSTOP = False - - -class ProxyHTTPSConnection(http.client.HTTPSConnection): - "This class allows communication via SSL." - - default_port = http.client.HTTPS_PORT - - # XXX Should key_file and cert_file be deprecated in favour of context? - - def __init__(self, host, port=None, key_file=None, cert_file=None, - timeout=socket._GLOBAL_DEFAULT_TIMEOUT, - source_address=None, *, context=None, - check_hostname=None, server_name=None): - # http.client.HTTPSConnection.__init__(self) - super().__init__(host, port, key_file, cert_file, timeout, source_address, context=context, check_hostname=check_hostname) - ''' - self.key_file = key_file - self.cert_file = cert_file - if context is None: - context = ssl._create_default_https_context() - will_verify = context.verify_mode != ssl.CERT_NONE - if check_hostname is None: - check_hostname = context.check_hostname - if check_hostname and not will_verify: - raise ValueError("check_hostname needs a SSL context with " - "either CERT_OPTIONAL or CERT_REQUIRED") - if key_file or cert_file: - context.load_cert_chain(cert_file, key_file) - self._context = context - self._check_hostname = check_hostname - ''' - self.server_name = server_name - - def connect(self): - "Connect to a host on a given (SSL) port." - http.client.HTTPConnection.connect(self) - - if self._tunnel_host: - server_hostname = self._tunnel_host - else: - server_hostname = self.server_name - self.sock = self._context.wrap_socket(self.sock, - do_handshake_on_connect=True, - server_side=False, - server_hostname=server_hostname) - if not self._context.check_hostname and self._check_hostname: - try: - ssl.match_hostname(self.sock.getpeercert(), server_hostname) - except Exception: - self.sock.shutdown(socket.SHUT_RDWR) - self.sock.close() - raise - - -def txn_replay(session_filename, txn, proxy, result_queue, request_session): - """ Replays a single transaction - :param request_session: has to be a valid requests session""" - req = txn.getRequest() - resp = txn.getResponse() - responseDict = {} - # Construct HTTP request & fire it off - txn_req_headers = req.getHeaders() - txn_req_headers_dict = extractHeader.header_to_dict(txn_req_headers) - txn_req_headers_dict['Content-MD5'] = txn._uuid # used as unique identifier - if 'body' in txn_req_headers_dict: - del txn_req_headers_dict['body'] - - #print("Replaying session") - try: - # response = request_session.request(extractHeader.extract_txn_req_method(txn_req_headers), - # 'http://' + extractHeader.extract_host(txn_req_headers) + extractHeader.extract_GET_path(txn_req_headers), - # headers=txn_req_headers_dict,stream=False) # making stream=False raises contentdecoding exception? kill me - method = extractHeader.extract_txn_req_method(txn_req_headers) - response = None - body = None - content = None - if 'Transfer-Encoding' in txn_req_headers_dict: - # deleting the host key, since the STUPID post/get functions are going to add host field anyway, so there will be multiple host fields in the header - # This confuses the ATS and it returns 400 "Invalid HTTP request". I don't believe this - # BUT, this is not a problem if the data is not chunked encoded.. Strange, huh? - del txn_req_headers_dict['Host'] - if 'Content-Length' in txn_req_headers_dict: - #print("ewww !") - del txn_req_headers_dict['Content-Length'] - body = gen() - if 'Content-Length' in txn_req_headers_dict: - nBytes = int(txn_req_headers_dict['Content-Length']) - body = createDummyBodywithLength(nBytes) - #print("request session is",id(request_session)) - - # NOTE: request_session here is actually python's HTTPSConnection, which is different from that in NonSSL, which uses the requests library -_- - if method == 'GET': - request_session.request('GET', extractHeader.extract_GET_path( - txn_req_headers), headers=txn_req_headers_dict, body=body) - r1 = request_session.getresponse() - responseContent = r1.read() # byte array - - elif method == 'POST': - request_session.request('POST', extractHeader.extract_GET_path( - txn_req_headers), headers=txn_req_headers_dict, body=body) - r1 = request_session.getresponse() - responseContent = r1.read() - - elif method == 'HEAD': - request_session.request('HEAD', extractHeader.extract_GET_path( - txn_req_headers), headers=txn_req_headers_dict, body=body) - r1 = request_session.getresponse() - responseContent = r1.read() - else: # EXPERIMENTAL - request_session.request(method, extractHeader.extract_GET_path( - txn_req_headers), headers=txn_req_headers_dict, body=body) - r1 = request_session.getresponse() - responseContent = r1.read() - - responseHeaders = extractHeader.responseHeaderTuple_to_dict(r1.getheaders()) - expected = extractHeader.responseHeader_to_dict(resp.getHeaders()) - # print("------------EXPECTED-----------") - # print(expected) - # print("------------RESP--------------") - # print(responseHeaders) - # print() - if mainProcess.verbose: - expected_output_split = resp.getHeaders().split('\r\n')[0].split(' ', 2) - expected_output = (int(expected_output_split[1]), str(expected_output_split[2])) - r = result.Result(session_filename, expected_output[0], r1.status, responseContent) - b_res, res = r.getResult(responseHeaders, expected, colorize=True) - print(res) - - if not res: - print("Received response") - print(responseHeaders) - print("Expected response") - print(expected) - # result_queue.put(r) - except UnicodeEncodeError as e: - # these unicode errors are due to the interaction between Requests and our wiretrace data. - # TODO fix - print("UnicodeEncodeError exception") - - except requests.exceptions.ContentDecodingError as e: - print("ContentDecodingError", e) - except: - e = sys.exc_info() - print("ERROR in SSLReplay: ", e, response, session_filename) - - -def client_replay(input, proxy, result_queue, nThread): - Threads = [] - for i in range(nThread): - t = Thread(target=session_replay, args=[input, proxy, result_queue]) - t.start() - Threads.append(t) - - for t1 in Threads: - t1.join() - - -def session_replay(input, proxy, result_queue): - ''' Replay all transactions in session - - This entire session will be replayed in one requests.Session (so one socket / TCP connection)''' - # if timing_control: - # time.sleep(float(session._timestamp)) # allow other threads to run - global bSTOP - sslSocks = [] - while bSTOP == False: - for session in iter(input.get, 'STOP'): - txn = session.returnFirstTransaction() - req = txn.getRequest() - # Construct HTTP request & fire it off - txn_req_headers = req.getHeaders() - txn_req_headers_dict = extractHeader.header_to_dict(txn_req_headers) - sc = ssl.SSLContext(protocol=ssl.PROTOCOL_SSLv23) - sc.load_cert_chain(Config.ca_certs, keyfile=Config.keyfile) - conn = ProxyHTTPSConnection(Config.proxy_host, Config.proxy_ssl_port, cert_file=Config.ca_certs, - key_file=Config.keyfile, context=sc, server_name=txn_req_headers_dict['Host']) - for txn in session.getTransactionIter(): - try: - # print(txn._uuid) - txn_replay(session._filename, txn, proxy, result_queue, conn) - except: - e = sys.exc_info() - print("ERROR in replaying: ", e, txn.getRequest().getHeaders()) - #sslSocket.bStop = False - - bSTOP = True - print("stopping now") - input.put('STOP') - break - - # time.sleep(0.5) - for sslSock in sslSocks: - sslSock.ssl_sock.close() diff --git a/tests/tools/traffic-replay/Scheduler.py b/tests/tools/traffic-replay/Scheduler.py deleted file mode 100644 index a1a4353d49a..00000000000 --- a/tests/tools/traffic-replay/Scheduler.py +++ /dev/null @@ -1,88 +0,0 @@ -#!/bin/env python3 -''' -''' -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import time -import random -import json -from multiprocessing import Process, Queue, current_process -import sessionvalidation.sessionvalidation as sv -import WorkerTask -import time - - -def LaunchWorkers(path, nProcess, proxy, replay_type, nThread): - ms1 = time.time() - s = sv.SessionValidator(path, allow_custom=True) - sessions = s.getSessionList() - sessions.sort(key=lambda session: session._timestamp) - Processes = [] - Qsize = 25000 # int (1.5 * len(sessions)/(nProcess)) - QList = [Queue(Qsize) for i in range(nProcess)] - print("Dropped {0} sessions for being malformed. Number of correct sessions {1}".format( - len(s.getBadSessionList()), len(sessions))) - print(range(nProcess)) - OutputQ = Queue() - #======================================== Pre-load queues - for session in sessions: - if replay_type == 'mixed': - if nProcess < 2: - raise ValueError("For mixed replay type, there should be at least 2 processes.") - # odd Qs for SSL sessions, even Qs for nonSSL sessions - num = random.randint(0, nProcess - 1) - - # get the first transaction in each session, which is indictive if session is over SSL or not - if "https" in session.returnFirstTransaction().getRequest().getHeaders(): - # spin until we get an odd number - while num & 1 == 0: - num = random.randint(0, nProcess - 1) - else: - # nonSSL sessions get put here into even Qs - while num & 1 == 1: - num = random.randint(0, nProcess - 1) - - QList[num].put(session) - else: - # if nProcess == 1: - # QList[0].put(session) - # else: - QList[random.randint(0, nProcess - 1)].put(session) - # if QList[0].qsize() > 10 : - # break - #=============================================== Launch Processes - # for i in range(nProcess): - # QList[i].put('STOP') - for i in range(nProcess): - QList[i].put('STOP') - - if replay_type == 'mixed': - if i & 1: # odd/SSL - p = Process(target=WorkerTask.worker, args=[QList[i], OutputQ, proxy, 'ssl', nThread]) - else: # even/nonSSL - p = Process(target=WorkerTask.worker, args=[QList[i], OutputQ, proxy, 'nossl', nThread]) - else: - p = Process(target=WorkerTask.worker, args=[QList[i], OutputQ, proxy, replay_type, nThread]) - - p.daemon = False - Processes.append(p) - p.start() - - for p in Processes: - p.join() - ms2 = time.time() - print("OK enough, it is time to exit, running time in seconds", (ms2 - ms1)) diff --git a/tests/tools/traffic-replay/WorkerTask.py b/tests/tools/traffic-replay/WorkerTask.py deleted file mode 100644 index 839e696665a..00000000000 --- a/tests/tools/traffic-replay/WorkerTask.py +++ /dev/null @@ -1,49 +0,0 @@ -#!/bin/env python3 -''' -''' -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import socket -import requests -import os -#import threading -import sys -from multiprocessing import current_process -import sessionvalidation.sessionvalidation as sv -import lib.result as result -import extractHeader -import NonSSL -import SSLReplay -import h2Replay -import RandomReplay - - -def worker(input, output, proxy, replay_type, nThread): - #progress_bar = Bar(" Replaying sessions {0}".format(current_process().name), max=input.qsize()) - #print("playing {0}=>{1}:{2}".format(current_process().name,session._timestamp,proxy)) - if replay_type == 'nossl': - NonSSL.client_replay(input, proxy, output, nThread) - elif replay_type == 'ssl': - SSLReplay.client_replay(input, proxy, output, nThread) - elif replay_type == 'h2': - h2Replay.client_replay(input, proxy, output, nThread) - elif replay_type == 'random': - RandomReplay.client_replay(input, proxy, output, nThread) - - # progress_bar.next() - # progress_bar.finish() - print("process{0} has exited".format(current_process().name)) diff --git a/tests/tools/traffic-replay/__main__.py b/tests/tools/traffic-replay/__main__.py deleted file mode 100644 index 4b64e5180ae..00000000000 --- a/tests/tools/traffic-replay/__main__.py +++ /dev/null @@ -1,44 +0,0 @@ -#!/bin/env python3 -''' -''' -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from __future__ import absolute_import, division, print_function -import mainProcess -import argparse -import Config - -if __name__ == '__main__': - - parser = argparse.ArgumentParser() - parser.add_argument("-type", action='store', dest='replay_type', - help="Replay type: ssl/random/h2/nossl/mixed (at least 2 processes needed for mixed)") - parser.add_argument("-log_dir", type=str, help="directory of JSON replay files") - parser.add_argument("-v", dest="verbose", help="verify response status code", action="store_true") - parser.add_argument("-host", help="proxy/host to send the requests to", default=Config.proxy_host) - parser.add_argument("-port", type=int, help=" The non secure port of ATS to send the request to", - default=Config.proxy_nonssl_port) - parser.add_argument("-s_port", type=int, help="secure port", default=Config.proxy_ssl_port) - parser.add_argument("-ca_cert", help="Certificate to present", default=Config.ca_certs) - parser.add_argument("-colorize", type=str, help="specify whether to use colorize the output", default='True') - - args = parser.parse_args() - - # Let 'er loose - #main(args.log_dir, args.hostname, int(args.port), args.threads, args.timing, args.verbose) - Config.colorize = True if args.colorize == 'True' else False - mainProcess.main(args.log_dir, args.replay_type, args.verbose, pHost=args.host, pNSSLport=args.port, pSSL=args.s_port) diff --git a/tests/tools/traffic-replay/extractHeader.py b/tests/tools/traffic-replay/extractHeader.py deleted file mode 100644 index a8151833b3d..00000000000 --- a/tests/tools/traffic-replay/extractHeader.py +++ /dev/null @@ -1,91 +0,0 @@ -#!/bin/env python3 -''' -''' -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import sessionvalidation.sessionvalidation as sv - - -def extract_txn_req_method(headers): - ''' Extracts the HTTP request method from the header in a string format ''' - line = (headers.split('\r\n'))[0] - return (line.split(' '))[0] - - -def extract_host(headers): - ''' Returns the host header from the given headers ''' - lines = headers.split('\r\n') - for line in lines: - if 'Host:' in line: - return line.split(' ')[1] - return "notfound" - -def responseHeaderTuple_to_dict(header): - header_dict = {} - - for key, val in header: - if key.lower() in header_dict: - header_dict[key.lower()] += ", {0}".format(val) - else: - header_dict[key.lower()] = val - - return header_dict - -def responseHeader_to_dict(header): - headerFields = header.split('\r\n', 1)[1] - fields = headerFields.split('\r\n') - header = [x for x in fields if (x != u'')] - headers = {} - for line in header: - split_here = line.find(":") - # append multiple headers into a single string - if line[:split_here].lower() in headers: - headers[line[:split_here].lower()] += ", {0}".format(line[(split_here + 1):].strip()) - else: - headers[line[:split_here].lower()] = line[(split_here + 1):].strip() - - return headers - - -def header_to_dict(header): - ''' Convert a HTTP header in string format to a python dictionary - Returns a dictionary of header values - ''' - header = header.split('\r\n') - header = [x for x in header if (x != u'')] - headers = {} - for line in header: - should_skip = False - - # we have to ignore the intital request line with the HTTP method in it - for method in sv.allowed_HTTP_request_methods: - if method in line: - should_skip = True - - if should_skip: # ignore initial request line - continue - - split_here = line.find(":") - headers[line[:split_here]] = line[(split_here + 1):].strip() - - return headers - - -def extract_GET_path(headers): - ''' Extracts the HTTP request URL from the header in a string format ''' - line = (headers.split('\r\n'))[0] - return (line.split(' '))[1] diff --git a/tests/tools/traffic-replay/h2Replay.py b/tests/tools/traffic-replay/h2Replay.py deleted file mode 100644 index 2026c74bf88..00000000000 --- a/tests/tools/traffic-replay/h2Replay.py +++ /dev/null @@ -1,331 +0,0 @@ -#!/bin/env python3 -''' -''' -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -from threading import Thread -import sys -from multiprocessing import current_process -import sessionvalidation.sessionvalidation as sv -import lib.result as result -import extractHeader -import mainProcess -import json -from hyper import HTTP20Connection -from hyper.tls import wrap_socket, H2_NPN_PROTOCOLS, H2C_PROTOCOL -from hyper.common.bufsocket import BufferedSocket -import hyper -import socket -import logging -import h2 -from h2.connection import H2Configuration -import threading -import Config - -log = logging.getLogger(__name__) -bSTOP = False -hyper.tls._context = hyper.tls.init_context() -hyper.tls._context.check_hostname = False -hyper.tls._context.verify_mode = hyper.compat.ssl.CERT_NONE - - -class _LockedObject(object): - """ - A wrapper class that hides a specific object behind a lock. - - The goal here is to provide a simple way to protect access to an object - that cannot safely be simultaneously accessed from multiple threads. The - intended use of this class is simple: take hold of it with a context - manager, which returns the protected object. - """ - - def __init__(self, obj): - self.lock = threading.RLock() - self._obj = obj - - def __enter__(self): - self.lock.acquire() - return self._obj - - def __exit__(self, _exc_type, _exc_val, _exc_tb): - self.lock.release() - - -class h2ATS(HTTP20Connection): - - def __init_state(self): - """ - Initializes the 'mutable state' portions of the HTTP/2 connection - object. - - This method exists to enable HTTP20Connection objects to be reused if - they're closed, by resetting the connection object to its basic state - whenever it ends up closed. Any situation that needs to recreate the - connection can call this method and it will be done. - - This is one of the only methods in hyper that is truly private, as - users should be strongly discouraged from messing about with connection - objects themselves. - """ - - config1 = H2Configuration( - client_side=True, - header_encoding='utf-8', - validate_outbound_headers=False, - validate_inbound_headers=False, - - ) - self._conn = _LockedObject(h2.connection.H2Connection(config=config1)) - - # Streams are stored in a dictionary keyed off their stream IDs. We - # also save the most recent one for easy access without having to walk - # the dictionary. - # - # We add a set of all streams that we or the remote party forcefully - # closed with RST_STREAM, to avoid encountering issues where frames - # were already in flight before the RST was processed. - # - # Finally, we add a set of streams that recently received data. When - # using multiple threads, this avoids reading on threads that have just - # acquired the I/O lock whose streams have already had their data read - # for them by prior threads. - self.streams = {} - self.recent_stream = None - self.next_stream_id = 1 - self.reset_streams = set() - self.recent_recv_streams = set() - - # The socket used to send data. - self._sock = None - - # Instantiate a window manager. - #self.window_manager = self.__wm_class(65535) - - return - - def __init__(self, host, **kwargs): - HTTP20Connection.__init__(self, host, **kwargs) - self.__init_state() - - def connect(self): - """ - Connect to the server specified when the object was created. This is a - no-op if we're already connected. - - Concurrency - ----------- - - This method is thread-safe. It may be called from multiple threads, and - is a noop for all threads apart from the first. - - :returns: Nothing. - - """ - #print("connecting to ATS") - with self._lock: - if self._sock is not None: - return - sni = self.host - if not self.proxy_host: - host = self.host - port = self.port - else: - host = self.proxy_host - port = self.proxy_port - - sock = socket.create_connection((host, port)) - - if self.secure: - #assert not self.proxy_host, "Proxy with HTTPS not supported." - sock, proto = wrap_socket(sock, sni, self.ssl_context, - force_proto=self.force_proto) - else: - proto = H2C_PROTOCOL - - log.debug("Selected NPN protocol: %s", proto) - assert proto in H2_NPN_PROTOCOLS or proto == H2C_PROTOCOL - - self._sock = BufferedSocket(sock, self.network_buffer_size) - - self._send_preamble() - - -def createDummyBodywithLength(numberOfbytes): - if numberOfbytes == 0: - return None - body = 'a' - while numberOfbytes != 1: - body += 'b' - numberOfbytes -= 1 - return body - - -def handleResponse(response, *args, **kwargs): - print(response.status_code) - # resp=args[0] - #expected_output_split = resp.getHeaders().split('\r\n')[ 0].split(' ', 2) - #expected_output = (int(expected_output_split[1]), str( expected_output_split[2])) - #r = result.Result(session_filename, expected_output[0], response.status_code) - # print(r.getResultString(colorize=True)) -# make sure len of the message body is greater than length - - -def gen(): - yield 'pforpersia,champaignurbana'.encode('utf-8') - yield 'there'.encode('utf-8') - - -def txn_replay(session_filename, txn, proxy, result_queue, h2conn, request_IDs): - """ Replays a single transaction - :param request_session: has to be a valid requests session""" - req = txn.getRequest() - resp = txn.getResponse() - # Construct HTTP request & fire it off - txn_req_headers = req.getHeaders() - txn_req_headers_dict = extractHeader.header_to_dict(txn_req_headers) - txn_req_headers_dict['Content-MD5'] = txn._uuid # used as unique identifier - if 'body' in txn_req_headers_dict: - del txn_req_headers_dict['body'] - responseID = -1 - #print("Replaying session") - try: - # response = request_session.request(extractHeader.extract_txn_req_method(txn_req_headers), - # 'http://' + extractHeader.extract_host(txn_req_headers) + extractHeader.extract_GET_path(txn_req_headers), - # headers=txn_req_headers_dict,stream=False) # making stream=False raises contentdecoding exception? kill me - method = extractHeader.extract_txn_req_method(txn_req_headers) - response = None - mbody = None - #txn_req_headers_dict['Host'] = "localhost" - if 'Transfer-Encoding' in txn_req_headers_dict: - # deleting the host key, since the STUPID post/get functions are going to add host field anyway, so there will be multiple host fields in the header - # This confuses the ATS and it returns 400 "Invalid HTTP request". I don't believe this - # BUT, this is not a problem if the data is not chunked encoded.. Strange, huh? - #del txn_req_headers_dict['Host'] - if 'Content-Length' in txn_req_headers_dict: - #print("ewww !") - del txn_req_headers_dict['Content-Length'] - mbody = gen() - if 'Content-Length' in txn_req_headers_dict: - nBytes = int(txn_req_headers_dict['Content-Length']) - mbody = createDummyBodywithLength(nBytes) - if 'Connection' in txn_req_headers_dict: - del txn_req_headers_dict['Connection'] - #str2 = extractHeader.extract_host(txn_req_headers)+ extractHeader.extract_GET_path(txn_req_headers) - # print(str2) - if method == 'GET': - responseID = h2conn.request('GET', url=extractHeader.extract_GET_path( - txn_req_headers), headers=txn_req_headers_dict, body=mbody) - # print("get response", responseID) - return responseID - # request_IDs.append(responseID) - #response = h2conn.get_response(id) - # print(response.headers) - # if 'Content-Length' in response.headers: - # content = response.read() - #print("len: {0} received {1}".format(response.headers['Content-Length'],content)) - - elif method == 'POST': - responseID = h2conn.request('POST', url=extractHeader.extract_GET_path( - txn_req_headers), headers=txn_req_headers_dict, body=mbody) - print("get response", responseID) - return responseID - - elif method == 'HEAD': - responseID = h2conn.request('HEAD', url=extractHeader.extract_GET_path(txn_req_headers), headers=txn_req_headers_dict) - print("get response", responseID) - return responseID - - except UnicodeEncodeError as e: - # these unicode errors are due to the interaction between Requests and our wiretrace data. - # TODO fix - print("UnicodeEncodeError exception") - - except: - e = sys.exc_info() - print("ERROR in requests: ", e, response, session_filename) - - -def session_replay(input, proxy, result_queue): - global bSTOP - ''' Replay all transactions in session - - This entire session will be replayed in one requests.Session (so one socket / TCP connection)''' - # if timing_control: - # time.sleep(float(session._timestamp)) # allow other threads to run - while bSTOP == False: - for session in iter(input.get, 'STOP'): - print(bSTOP) - if session == 'STOP': - print("Queue is empty") - bSTOP = True - break - txn = session.returnFirstTransaction() - req = txn.getRequest() - # Construct HTTP request & fire it off - txn_req_headers = req.getHeaders() - txn_req_headers_dict = extractHeader.header_to_dict(txn_req_headers) - with h2ATS(txn_req_headers_dict['Host'], secure=True, proxy_host=Config.proxy_host, proxy_port=Config.proxy_ssl_port) as h2conn: - request_IDs = [] - respList = [] - for txn in session.getTransactionIter(): - try: - ret = txn_replay(session._filename, txn, proxy, result_queue, h2conn, request_IDs) - respList.append(txn.getResponse()) - request_IDs.append(ret) - #print("txn return value is ",ret) - except: - e = sys.exc_info() - print("ERROR in replaying: ", e, txn.getRequest().getHeaders()) - for id in request_IDs: - expectedH = respList.pop(0) - # print("extracting",id) - response = h2conn.get_response(id) - #print("code {0}:{1}".format(response.status,response.headers)) - response_dict = {} - if mainProcess.verbose: - for field, value in response.headers.items(): - response_dict[field.decode('utf-8')] = value.decode('utf-8') - - expected_output_split = expectedH.getHeaders().split('\r\n')[0].split(' ', 2) - expected_output = (int(expected_output_split[1]), str(expected_output_split[2])) - r = result.Result("", expected_output[0], response.status, response.read()) - expected_Dict = extractHeader.responseHeader_to_dict(expectedH.getHeaders()) - b_res, res = r.getResult(response_dict, expected_Dict, colorize=Config.colorize) - print(res) - - if not b_res: - print("Received response") - print(response_dict) - print("Expected response") - print(expected_Dict) - - bSTOP = True - #print("Queue is empty") - input.put('STOP') - break - - -def client_replay(input, proxy, result_queue, nThread): - Threads = [] - for i in range(nThread): - t = Thread(target=session_replay, args=[input, proxy, result_queue]) - t.start() - Threads.append(t) - - for t1 in Threads: - t1.join() diff --git a/tests/tools/traffic-replay/mainProcess.py b/tests/tools/traffic-replay/mainProcess.py deleted file mode 100644 index bde8de4a5fb..00000000000 --- a/tests/tools/traffic-replay/mainProcess.py +++ /dev/null @@ -1,76 +0,0 @@ -#!/bin/env python3 -''' -''' -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -import sys -import json -import socket -import os -import threading -import time -import argparse -import subprocess -import shlex -from multiprocessing import Pool, Process -from collections import deque -#from progress.bar import Bar - -sys.path.append( - os.path.normpath( - os.path.join( - os.path.dirname(os.path.abspath(__file__)), - '..' - ) - ) -) - -import sessionvalidation.sessionvalidation as sv -import lib.result as result -import WorkerTask -import Scheduler -import Config -verbose = False - - -def check_for_ats(hostname, port): - ''' Checks to see if ATS is running on `hostname` and `port` - If not running, this function will terminate the script - ''' - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - result = sock.connect_ex((hostname, port)) - if result != 0: - # hostname:port is not being listened to - print('==========') - print('Error: Apache Traffic Server is not running on {0}:{1}'.format(hostname, port)) - print('Aborting') - print('==========') - sys.exit() -# Note: this function can't handle multi-line (ie wrapped line) headers -# Hopefully this isn't an issue because multi-line headers are deprecated now - - -def main(path, replay_type, Bverbose, pHost=Config.proxy_host, pNSSLport=Config.proxy_nonssl_port, pSSL=Config.proxy_ssl_port): - global verbose - verbose = Bverbose - check_for_ats(pHost, pNSSLport) - Config.proxy_host = pHost - Config.proxy_nonssl_port = pNSSLport - Config.proxy_ssl_port = pSSL - proxy = {"http": "http://{0}:{1}".format(Config.proxy_host, Config.proxy_nonssl_port)} - Scheduler.LaunchWorkers(path, Config.nProcess, proxy, replay_type, Config.nThread) diff --git a/tests/unit_tests/Makefile.am b/tests/unit_tests/Makefile.am index 84db12b2842..791c6005d2c 100644 --- a/tests/unit_tests/Makefile.am +++ b/tests/unit_tests/Makefile.am @@ -27,3 +27,4 @@ unit_tests_SOURCES = main.cpp clang-tidy-local: $(DIST_SOURCES) $(CXX_Clang_Tidy) + \ No newline at end of file diff --git a/tools/changelog.pl b/tools/changelog.pl index ba62f093bd6..0de170e193d 100755 --- a/tools/changelog.pl +++ b/tools/changelog.pl @@ -22,105 +22,96 @@ use WWW::Curl::Easy; use JSON; -my $owner = shift; -my $repo = shift; +my $owner = shift; +my $repo = shift; my $milestone = shift; -my $auth = shift; -my $url = "https://api.github.com"; +my $auth = shift; +my $url = "https://api.github.com"; sub rate_fail { - print STDERR "You have exceeded your rate limit. Try using an auth token.\n"; - exit 2; + print STDERR "You have exceeded your rate limit. Try using an auth token.\n"; + exit 2; } sub milestone_lookup { - my $curl = shift; - my $url = shift; - my $owner = shift; - my $repo = shift; - my $milestone_title = shift; - my $endpoint = "/repos/$owner/$repo/milestones"; - - my $resp_body; - - $curl->setopt(CURLOPT_WRITEDATA, \$resp_body); - $curl->setopt(CURLOPT_URL, $url . $endpoint); - - my $retcode = $curl->perform(); - if ($retcode == 0 && $curl->getinfo(CURLINFO_HTTP_CODE) == 200) - { - my $milestones = from_json($resp_body); - foreach my $milestone (@{ $milestones }) - { - if ($milestone->{title} eq $milestone_title) - { - return $milestone->{number}; - } + my $curl = shift; + my $url = shift; + my $owner = shift; + my $repo = shift; + my $milestone_title = shift; + my $endpoint = "/repos/$owner/$repo/milestones"; + + my $resp_body; + + $curl->setopt(CURLOPT_WRITEDATA, \$resp_body); + $curl->setopt(CURLOPT_URL, $url . $endpoint); + + my $retcode = $curl->perform(); + if ($retcode == 0 && $curl->getinfo(CURLINFO_HTTP_CODE) == 200) { + my $milestones = from_json($resp_body); + foreach my $milestone (@{$milestones}) { + if ($milestone->{title} eq $milestone_title) { + return $milestone->{number}; + } + } + } elsif ($retcode == 0 && $curl->getinfo(CURLINFO_HTTP_CODE) == 403) { + rate_fail(); } - } - elsif ($retcode == 0 && $curl->getinfo(CURLINFO_HTTP_CODE) == 403) - { - rate_fail(); - } - undef; + undef; } sub is_merged { - my $curl = shift; - my $url = shift; - my $owner = shift; - my $repo = shift; - my $issue_id = shift; - my $endpoint = "/repos/$owner/$repo/pulls/$issue_id/merge"; - - my $resp_body; - - $curl->setopt(CURLOPT_WRITEDATA, \$resp_body); - $curl->setopt(CURLOPT_URL, $url . $endpoint); - - my $retcode = $curl->perform(); - if ($retcode == 0 && $curl->getinfo(CURLINFO_HTTP_CODE) == 204) { - return 1; - } - elsif ($retcode == 0 && $curl->getinfo(CURLINFO_HTTP_CODE) == 403) - { - rate_fail(); - } - - undef; + my $curl = shift; + my $url = shift; + my $owner = shift; + my $repo = shift; + my $issue_id = shift; + my $endpoint = "/repos/$owner/$repo/pulls/$issue_id/merge"; + + my $resp_body; + + $curl->setopt(CURLOPT_WRITEDATA, \$resp_body); + $curl->setopt(CURLOPT_URL, $url . $endpoint); + + my $retcode = $curl->perform(); + if ($retcode == 0 && $curl->getinfo(CURLINFO_HTTP_CODE) == 204) { + return 1; + } elsif ($retcode == 0 && $curl->getinfo(CURLINFO_HTTP_CODE) == 403) { + rate_fail(); + } + + undef; } sub issue_search { - my $curl = shift; - my $url = shift; - my $owner = shift; - my $repo = shift; - my $milestone_id = shift; - my $page = shift; - my $endpoint = "/repos/$owner/$repo/issues"; - - my $params = "milestone=$milestone_id&state=closed&page=$page"; - - my $resp_body; - - $curl->setopt(CURLOPT_WRITEDATA, \$resp_body); - $curl->setopt(CURLOPT_URL, $url . $endpoint . '?' . $params); - - my $retcode = $curl->perform(); - if ($retcode == 0 && $curl->getinfo(CURLINFO_HTTP_CODE) == 200) { - return from_json($resp_body); - } - elsif ($retcode == 0 && $curl->getinfo(CURLINFO_HTTP_CODE) == 403) - { - rate_fail(); - } - - undef; + my $curl = shift; + my $url = shift; + my $owner = shift; + my $repo = shift; + my $milestone_id = shift; + my $page = shift; + my $endpoint = "/repos/$owner/$repo/issues"; + + my $params = "milestone=$milestone_id&state=closed&page=$page"; + + my $resp_body; + + $curl->setopt(CURLOPT_WRITEDATA, \$resp_body); + $curl->setopt(CURLOPT_URL, $url . $endpoint . '?' . $params); + + my $retcode = $curl->perform(); + if ($retcode == 0 && $curl->getinfo(CURLINFO_HTTP_CODE) == 200) { + return from_json($resp_body); + } elsif ($retcode == 0 && $curl->getinfo(CURLINFO_HTTP_CODE) == 403) { + rate_fail(); + } + + undef; } my $curl = WWW::Curl::Easy->new; @@ -128,17 +119,15 @@ sub issue_search #$curl->setopt(CURLOPT_VERBOSE, 1); $curl->setopt(CURLOPT_HTTPHEADER, ['Accept: application/vnd.github.v3+json', 'User-Agent: Awesome-Octocat-App']); -if (defined($auth)) -{ - $curl->setopt(CURLOPT_USERPWD, $auth); +if (defined($auth)) { + $curl->setopt(CURLOPT_USERPWD, $auth); } my $milestone_id = milestone_lookup($curl, $url, $owner, $repo, $milestone); -if (!defined($milestone_id)) -{ - print STDERR "Milestone not found!\n"; - exit 1; +if (!defined($milestone_id)) { + print STDERR "Milestone not found!\n"; + exit 1; } my $issues; @@ -148,39 +137,33 @@ sub issue_search print STDERR "Looking for issues from Milestone $milestone\n"; do { - print STDERR "Page $page\n"; - $issues = issue_search($curl, $url, $owner, $repo, $milestone_id, $page); - foreach my $issue (@{ $issues }) - { - if (defined($issue)) - { - print STDERR "Issue #" . $issue->{number} . " - " . $issue->{title} . " "; - - if (!exists($issue->{pull_request})) - { - print STDERR "not a PR.\n"; - next; - } - - if (!is_merged($curl, $url, $owner, $repo, $issue->{number})) - { - print STDERR "not merged.\n"; - next; - } - - print STDERR "added.\n"; - push @{ $changelog }, {number => $issue->{number}, title => $issue->{title}}; + print STDERR "Page $page\n"; + $issues = issue_search($curl, $url, $owner, $repo, $milestone_id, $page); + foreach my $issue (@{$issues}) { + if (defined($issue)) { + print STDERR "Issue #" . $issue->{number} . " - " . $issue->{title} . " "; + + if (!exists($issue->{pull_request})) { + print STDERR "not a PR.\n"; + next; + } + + if (!is_merged($curl, $url, $owner, $repo, $issue->{number})) { + print STDERR "not merged.\n"; + next; + } + + print STDERR "added.\n"; + push @{$changelog}, {number => $issue->{number}, title => $issue->{title}}; + } } - } - $page++; -} while (scalar @{ $issues }); + $page++; +} while (scalar @{$issues}); -if (defined($changelog)) -{ - print "Changes with Apache Traffic Server $milestone\n"; +if (defined($changelog)) { + print "Changes with Apache Traffic Server $milestone\n"; - foreach my $issue (sort {$a->{number} <=> $b->{number}} @{ $changelog }) - { - print " #$issue->{number} - $issue->{title}\n"; - } + foreach my $issue (sort {$a->{number} <=> $b->{number}} @{$changelog}) { + print " #$issue->{number} - $issue->{title}\n"; + } } diff --git a/tools/clang-format.sh b/tools/clang-format.sh index ff97232fd86..6c37d5df5eb 100755 --- a/tools/clang-format.sh +++ b/tools/clang-format.sh @@ -77,7 +77,7 @@ EOF echo "or alternatively, undefine the FORMAT environment variable" exit 1 else - for file in $(find $DIR -iname \*.[ch] -o -iname \*.cc); do + for file in $(find $DIR -iname \*.[ch] -o -iname \*.cc -o -iname \*.h.in); do echo $file ${FORMAT} -i $file done diff --git a/tools/code-make b/tools/code-make new file mode 100755 index 00000000000..281ab56e336 --- /dev/null +++ b/tools/code-make @@ -0,0 +1,68 @@ +#!/usr/bin/env perl + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This tool is a "replacement" for make, which wraps the make command around +# a filter that tries to normalize the output such that the filenames are relative +# to the build directory. This passes all arguments passed along to make itself, +# and you can override the "make" command using the MAKE environment variable. + +use strict; +use Term::ANSIColor; + +my %COLORS = ( + "_header" => "italic green", + "_line" => "cyan", + "_message" => "italic magenta", + "error" => "red", + "warning" => "yellow", + "note" => "blue" +); +my $CMD = $ENV{"MAKE"} || "make" . " 2>&1 " . join(" ", @ARGV); +my @DIRS = (); + +print colored("Running: $CMD\n", $COLORS{"_header"}); + +open my $cmd, '-|', $CMD; +while (my $line = <$cmd>) { + if ($line =~ /(error|warning|note):/) { + my $msg = $1; + my @parts = split(/:/, $line); + my $file = $parts[0]; + + if (substr($file, 0, 1) ne "/") { + $file =~ s/^[\.\/]+//; # Strip leading ./, ../, ../../, etc. + + # Lazy eval on this, assuming that we will not find errors typically... + if (!@DIRS) { + @DIRS = split(/\n/, `find . -type d | fgrep -v -e .deps -e .libs -e .git -e .vscode`); + } + + foreach (@DIRS) { + if (-f "$_/$file") { + $file = "$_/$file"; + last; + } + } + } + print colored("$file:$parts[1]:$parts[2]:", $COLORS{"_line"}); + print colored("$msg", $COLORS{"$msg"}); + print colored("$parts[4]\n", $COLORS{"_message"}); + } else { + print $line; + } +} diff --git a/tools/compare_records.pl b/tools/compare_records.pl index cf0046360b3..eb16cd4b17d 100755 --- a/tools/compare_records.pl +++ b/tools/compare_records.pl @@ -38,35 +38,39 @@ use warnings; use Getopt::Long; -my($file1, $file2, $in_files, $help); +my ($file1, $file2, $in_files, $help); my %file1_settings; my %file2_settings; my $diff_metrics; -usage() if (@ARGV < 1 or - !GetOptions( - 'f=s@' => \$in_files, - 'm' => \$diff_metrics, - 'help|?' => \$help) or - defined $help); +usage() + if ( + @ARGV < 1 + or !GetOptions( + 'f=s@' => \$in_files, + 'm' => \$diff_metrics, + 'help|?' => \$help + ) + or defined $help + ); # Input file is mandatory die "\nTwo input files must be specified to compare\n" - unless defined $in_files; + unless defined $in_files; # Print the usage sub usage { - print "Unknown option: @_\n" if (@_); - print "Provide 2 files to compare configs or metrics.\n"; - print "By default this tool will diff only configs,\n"; - print "to get diff of metrics pass -m flag\n\n"; - print "Usage: compare_records.pl -m -f -f \n"; - print " -m to diff the metrics\n"; - print " -h for help\n\n"; - print "where the files are generated with e.g.\n\n"; - print " \$ traffic_ctl config match .\n"; - exit; + print "Unknown option: @_\n" if (@_); + print "Provide 2 files to compare configs or metrics.\n"; + print "By default this tool will diff only configs,\n"; + print "to get diff of metrics pass -m flag\n\n"; + print "Usage: compare_records.pl -m -f -f \n"; + print " -m to diff the metrics\n"; + print " -h for help\n\n"; + print "where the files are generated with e.g.\n\n"; + print " \$ traffic_ctl config match .\n"; + exit; } my @file_list = @$in_files; @@ -76,89 +80,88 @@ sub usage # Open input files if (defined $in_file1) { - open $file1, $in_file1 or die "Could not open $in_file1: $!"; + open $file1, $in_file1 or die "Could not open $in_file1: $!"; } if (defined $in_file2) { - open $file2, $in_file2 or die "Could not open $in_file2: $!"; + open $file2, $in_file2 or die "Could not open $in_file2: $!"; } # Read input files while (my $setting = <$file1>) { - chomp $setting; - my($record, $value) = split(/:/, $setting); - if (defined $diff_metrics) { - # Obtain only metrics, excluding configs - if ($record !~ /proxy.config/) { - $file1_settings{$record} = $value; - } - } else { - # Obtain only configs - if ($record =~ /proxy.config/) { - $file1_settings{$record} = $value; + chomp $setting; + my ($record, $value) = split(/:/, $setting); + if (defined $diff_metrics) { + # Obtain only metrics, excluding configs + if ($record !~ /proxy.config/) { + $file1_settings{$record} = $value; + } + } else { + # Obtain only configs + if ($record =~ /proxy.config/) { + $file1_settings{$record} = $value; + } } - } } close $file1; while (my $setting = <$file2>) { - chomp $setting; - my($record, $value) = split(/:/, $setting); - if (defined $diff_metrics) { - # Obtain only metrics, excluding configs - if ($record !~ /proxy.config/) { - $file2_settings{$record} = $value; - } - } else { - # Obtain only configs - if ($record =~ /proxy.config/) { - $file2_settings{$record} = $value; + chomp $setting; + my ($record, $value) = split(/:/, $setting); + if (defined $diff_metrics) { + # Obtain only metrics, excluding configs + if ($record !~ /proxy.config/) { + $file2_settings{$record} = $value; + } + } else { + # Obtain only configs + if ($record =~ /proxy.config/) { + $file2_settings{$record} = $value; + } } - } } close $file2; # Subroutine to compare configs/metrics and obtain common and difference between them sub compare_configs_or_metrics { - my($records1, $records2, $file) = @_; - my %common_settings; - my %diff_settings; - my %settings1 = %$records1; - my %settings2 = %$records2; - - foreach my $record(sort keys %settings1) { - if ($settings2{$record}) { - $common_settings{$record} = $settings1{$record}; - } else { - $diff_settings{$record} = $settings1{$record}; + my ($records1, $records2, $file) = @_; + my %common_settings; + my %diff_settings; + my %settings1 = %$records1; + my %settings2 = %$records2; + + foreach my $record (sort keys %settings1) { + if ($settings2{$record}) { + $common_settings{$record} = $settings1{$record}; + } else { + $diff_settings{$record} = $settings1{$record}; + } } - } - print "####################################################################################\n"; - print "Configs/metrics found only in $file\n"; - print "####################################################################################\n"; - foreach my $key(sort keys %diff_settings) - { - print "$key\n"; - } - return (\%common_settings); + print "####################################################################################\n"; + print "Configs/metrics found only in $file\n"; + print "####################################################################################\n"; + foreach my $key (sort keys %diff_settings) { + print "$key\n"; + } + return (\%common_settings); } # Subroutine to obtain changes in default values among common configs/metrics sub compare_default_values { - my($records1, $records2) = @_; - my %settings1 = %$records1; - my %settings2 = %$records2; - - foreach my $record(sort keys %settings1) { - if (defined $settings1{$record} && $settings2{$record}) { - if ($settings1{$record} ne $settings2{$record}) { - # Values doesn't match - print "$record default value changed from $settings1{$record} -> $settings2{$record}\n"; - } + my ($records1, $records2) = @_; + my %settings1 = %$records1; + my %settings2 = %$records2; + + foreach my $record (sort keys %settings1) { + if (defined $settings1{$record} && $settings2{$record}) { + if ($settings1{$record} ne $settings2{$record}) { + # Values doesn't match + print "$record default value changed from $settings1{$record} -> $settings2{$record}\n"; + } + } } - } } # Obtain common configs/metrics between two files @@ -171,9 +174,8 @@ sub compare_default_values print "####################################################################################\n"; print "Common configs/metrics between $in_file1 and $in_file2\n"; print "####################################################################################\n"; -foreach my $key(sort keys %common2_settings) -{ - print "$key\n"; +foreach my $key (sort keys %common2_settings) { + print "$key\n"; } # Compare common configs/metrics and obtain changes in default values diff --git a/tools/compare_servers.pl b/tools/compare_servers.pl new file mode 100755 index 00000000000..a7bfaaa534b --- /dev/null +++ b/tools/compare_servers.pl @@ -0,0 +1,259 @@ +#!/usr/bin/perl +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +use strict; +use warnings; +use Getopt::Long; +use Data::Dumper; +use Net::hostent; +use Socket; +use LWP::UserAgent; +use Digest::SHA1; + +my $verbose = 0; + +#---------------------------------------------------------------------------- +sub usage() +{ + print STDERR "USAGE: compare_hosts.pl --verbose level --host1 testing_host --host2 valid_host --file url_file\n\n"; + print STDERR "\t--host1 The host running the newest version\n"; + print STDERR "\t--host2 The host running the older version\n"; + print STDERR "\t--file A file that contains a list of URLs\n"; + print STDERR "\t--verbose verbose level 1-3, 1 is the least verbose\n\n"; + print STDERR "Example:\n"; + print STDERR "\tcompare_hosts.pl --host1 new_ats --host2 old_ats --file top_1000_urls\n"; + exit 1; +} + +#---------------------------------------------------------------------------- +sub compareHeaderNames($$) +{ + my ($response1, $response2) = @_; + + my @names1 = $response1->header_field_names; + my @names2 = $response2->header_field_names; + + my %hash2; + $hash2{$_} = 1 for (@names2); + my %hash1; + $hash1{$_} = 1 for (@names1); + + my $return_val = 0; # header names match + + foreach my $name (@names1) { + if (!defined $hash2{$name}) { + print "\t\t- $name header not found on host2\n" if $verbose >= 2; + $return_val = 1; + } + } + + foreach my $name (@names2) { + if (!defined $hash1{$name}) { + print "\t\t- $name header not found on host1\n" if $verbose >= 2; + $return_val = 1; + } + } + + return $return_val; +} + +#---------------------------------------------------------------------------- +sub compareHeaderValues($$) +{ + my ($response1, $response2) = @_; + + my @test_headers = + qw(ETag Cache-Control Connection Accept-Ranges Server Content-Type Access-Control-Allow-Methods Access-Control-Allow-Origin Strict-Transport-Security); + my $return_val = 0; # header value match + + if ($verbose >= 3) { + foreach my $field ($response1->header_field_names) { + print "\t\t\t~ " . $field . ": " . $response1->header($field) . "\n"; + } + + print "\t\tHost2: \n"; + + foreach my $field ($response2->header_field_names) { + print "\t\t\t~ " . $field . ": " . $response2->header($field) . "\n"; + } + } + + # Test specific headers that are defined above + foreach my $field (@test_headers) { + my $value1 = $response1->header($field); + my $value2 = $response2->header($field); + + if (defined $value1 && defined $value2) { + if ($value1 ne $value2) { + print "\t\t- $field: $value1 ne $value2\n" if $verbose; + print "\t\t\t - Via host1: " . $response1->header('Via') . " host2: " . $response2->header('Via') . "\n" + if $verbose; + print "\t\t\t - Last-Modified host1: " + . $response1->header('Last-Modified') + . " host2: " + . $response2->header('Last-Modified') . "\n" + if $verbose; + if (defined $response2->header('Content-Encoding')) { + print "\t\t\t - Content-Encoding host1: " + . $response1->header('Content-Encoding') + . " host2: " + . $response2->header('Content-Encoding') . "\n"; + } else { + print "\t\t\t - Content-Encoding host1: " . $response1->header('Content-Encoding') . " host2: ''\n"; + } + $return_val = 1; + } else { + print "\t\t- $field: $value1 eq $value2\n" if $verbose >= 2; + } + } + } + return $return_val; +} + +#---------------------------------------------------------------------------- +{ + my %stats; + + $ENV{PERL_LWP_SSL_VERIFY_HOSTNAME} = '0'; + my ($host1, $host2, $file); + GetOptions( + "host1=s" => \$host1, + "host2=s" => \$host2, + "file=s" => \$file, + "verbose=f" => \$verbose + ) || die $!; + + usage() if (!defined $host1 || !defined $host2 || !defined $file); + + my $count = 0; + my $status_error = 0; + my $sha_error = 0; + my $header_names_mismatch = 0; + my $header_values_mismatch = 0; + + my $host1_addr = inet_ntoa(inet_aton($host1)); + my $host2_addr = inet_ntoa(inet_aton($host2)); + + print "Testing with host1: $host1 ($host1_addr) - host2: $host2 ($host2_addr)\n"; + print '-' x 78, "\n"; + + open(FILE, $file) || die $!; + + # Create a user agent object + my $ua1 = LWP::UserAgent->new(keep_alive => 100); + $ua1->agent("MyApp/0.1 "); + + # Create a user agent object + my $ua2 = LWP::UserAgent->new(keep_alive => 100); + $ua2->agent("MyApp/0.1 "); + + while (my $url = ) { + next if ($url =~ m|hc.l.yimg.com|); + chomp $url; + my $exit = 0; + + if ($url =~ m|(https?)://([^/]+)(.+)|) { + + my $scheme = $1; + my $host = $2; + my $path = $3; + + $count++; + print "Test $count - URL: $url\n"; + + my $port = 80; + $port = 443 if $scheme eq 'https'; + + my $request1 = HTTP::Request->new(GET => "${scheme}://${host1_addr}${path}"); + $request1->header('Host' => $host); + my $response1 = $ua1->request($request1); + + my $request2 = HTTP::Request->new(GET => "${scheme}://${host2_addr}${path}"); + $request2->header('Host' => $host); + $request2->header('Accept-Encoding' => 'deflate'); + my $response2 = $ua2->request($request2); + + print "\tStatus code for host1: " . $response1->code . " - host2: " . $response2->code . "\n" if $verbose; + + my $sha1 = Digest::SHA1->new; + $sha1->add($response1->content); + my $digest1 = $sha1->hexdigest; + open(FILE1, "> /tmp/tmp1"); + open(FILE2, "> /tmp/tmp2"); + print FILE1 $response1->content; + print FILE2 $response2->content; + close FILE1; + close FILE2; + #print $response1->content, "\n"; # for internal debugging + #print $response2->content, "\n"; # for internal debugging + + my $sha2 = Digest::SHA1->new; + $sha2->add($response2->content); + my $digest2 = $sha2->hexdigest; + + print "\tSHA hash for host1: $digest1 - host2: $digest2\n" if $verbose; + + # Build up stats + if ($response1->status_line eq $response2->status_line) { + + # Do the hashes + if ($digest1 eq $digest2) { + $stats{stat_line_match}->{$response1->code}->{sha_match}++; + print "\tResponse code: " . $response1->code . " - Status lines and SHA1 of response bodies match\n"; + } else { + $stats{stat_line_match}->{$response1->code}->{sha_nomatch}++; + print "\tResponse code: " . $response1->code . " - Status lines match SHA1 doesn't match\n"; + $sha_error++; + #$exit = 1 if $response1->code == 200; # for internal debugging + } + + # Compare the header field names + if (compareHeaderNames($response1, $response2) == 0) { + $stats{stat_line_match}->{$response1->code}->{field_names_match}++; + } else { + $stats{stat_line_match}->{$response1->code}->{field_names_nomatch}++; + $header_names_mismatch++; + } + + # Compare the values of the header fields + if (compareHeaderValues($response1, $response2) == 0) { + $stats{stat_line_match}->{$response1->code}->{field_values_match}++; + } else { + $stats{stat_line_match}->{$response1->code}->{field_values_nomatch}++; + $header_values_mismatch++; + } + } else { + $status_error++; + $stats{stat_line_nomatch}++; + print "\tERROR: status lines don't match\n"; + } + + last if $exit; + } + } + + print '-' x 78, "\n"; + print "SUMMARY:\n"; + print "URLs tested: $count\n"; + print "Status line mismatches: $status_error\n"; + print "SHA1 mismatches: $sha_error\n"; + print "Responses with header names mismatches: $header_names_mismatch\n"; + print "Responses with header values mismatches: $header_values_mismatch\n"; + print Dumper \%stats if $verbose; +} + diff --git a/tools/freelist_diff.pl b/tools/freelist_diff.pl index 8f15a93d970..4e9e92d68e6 100755 --- a/tools/freelist_diff.pl +++ b/tools/freelist_diff.pl @@ -16,21 +16,24 @@ # See the License for the specific language governing permissions and # limitations under the License. -sub usage { +sub usage +{ print "Usage: freelist_diff.pl dump1.txt dump2.txt\n"; } -sub int_meg { +sub int_meg +{ my $bytes = shift; - return $bytes / (1024*1024); + return $bytes / (1024 * 1024); } -sub load_file { +sub load_file +{ my $file = shift; my %data; open(DATA, $file) || return undef; - while() { + while () { my @items = split; chomp @items; @@ -49,11 +52,13 @@ sub load_file { while (my ($key, $value) = each(%{$data1})) { # before alloc [0], after alloc [1], before in-use [2], after in-use [3] - $diff{$key} = [ $value->[0], $data2->{$key}->[0], $value->[1], $data2->{$key}->[1], - # diff alloc [4], diff in-use [5] - $data2->{$key}->[0] - $value->[0], $data2->{$key}->[1] - $value->[1], - # type size [6] - $value->[2] ]; + $diff{$key} = [ + $value->[0], $data2->{$key}->[0], $value->[1], $data2->{$key}->[1], + # diff alloc [4], diff in-use [5] + $data2->{$key}->[0] - $value->[0], $data2->{$key}->[1] - $value->[1], + # type size [6] + $value->[2] + ]; } print "Sorted by in-use growth\n"; diff --git a/tools/git/pre-commit b/tools/git/pre-commit index 3aee8e7420d..748215b85a8 100755 --- a/tools/git/pre-commit +++ b/tools/git/pre-commit @@ -47,7 +47,7 @@ patch_file=$(mktemp -t clang-format.XXXXXXXXXX) trap "rm -f $patch_file" 0 1 2 3 5 15 # Loop over all files that are changed, and produce a diff file -git diff-index --cached --diff-filter=ACMR --name-only HEAD | while read file; do +git diff-index --cached --diff-filter=ACMR --name-only HEAD | grep -vE "lib/tsconfig|lib/yamlcpp" | while read file; do case "$file" in *.cc | *.c | *.h | *.h.in) ${FORMAT} "$file" | diff -u "$file" - >> "$patch_file" diff --git a/tools/http_load/http_load.c b/tools/http_load/http_load.c index 9268f7432c6..da8c74c6bff 100644 --- a/tools/http_load/http_load.c +++ b/tools/http_load/http_load.c @@ -2846,7 +2846,7 @@ close_connection(int cnum) url_num = connections[cnum].url_num; /* Only check to update got_bytes, byte count errors and/or checksums - if the request was succesful (i.e. no HTTP error). */ + if the request was successful (i.e. no HTTP error). */ if (connections[cnum].http_status >= 0 && connections[cnum].http_status < 400) { if (do_checksum) { if (!urls[url_num].got_checksum) { diff --git a/tools/http_load/merge_stats.pl b/tools/http_load/merge_stats.pl index 49e93a4f709..317460bb5f4 100644 --- a/tools/http_load/merge_stats.pl +++ b/tools/http_load/merge_stats.pl @@ -20,54 +20,57 @@ # See the License for the specific language governing permissions and # limitations under the License. -my $runs = 0; -my $fetches = 0; -my $conns = 0; -my $parallel = 0; -my $bytes = 0; -my $seconds = 0; -my $mean_bytes = 0; -my $fetches_sec = 0.0; -my $bytes_sec = 0.0; -my %msecs_connect = ( "mean" => 0.0, - "max" => 0.0, - "min" => 0.0 ); -my %msecs_response = ( "mean" => 0.0, - "max" => 0.0, - "min" => 0.0 ); - +my $runs = 0; +my $fetches = 0; +my $conns = 0; +my $parallel = 0; +my $bytes = 0; +my $seconds = 0; +my $mean_bytes = 0; +my $fetches_sec = 0.0; +my $bytes_sec = 0.0; +my %msecs_connect = ( + "mean" => 0.0, + "max" => 0.0, + "min" => 0.0 +); +my %msecs_response = ( + "mean" => 0.0, + "max" => 0.0, + "min" => 0.0 +); while (<>) { - my @c = split(); - if (/fetches on/) { - $fetches += $c[0]; - $conns += $c[3]; - $parallel += $c[5]; - $bytes += $c[8]; - $seconds += $c[11]; - $runs++; - } elsif (/mean bytes/) { - $mean_bytes += $c[0]; - } elsif (/fetches\/sec/) { - $fetches_sec += $c[0]; - $bytes_sec += $c[2]; - } elsif (/msecs\/connect/) { - $msecs_connect{"mean"} += $c[1]; - $msecs_connect{"max"} += $c[3]; - $msecs_connect{"min"} += $c[5]; - } elsif (/msecs\/first/) { - $msecs_response{"mean"} += $c[1]; - $msecs_response{"max"} += $c[3]; - $msecs_response{"min"} += $c[5]; - } + my @c = split(); + if (/fetches on/) { + $fetches += $c[0]; + $conns += $c[3]; + $parallel += $c[5]; + $bytes += $c[8]; + $seconds += $c[11]; + $runs++; + } elsif (/mean bytes/) { + $mean_bytes += $c[0]; + } elsif (/fetches\/sec/) { + $fetches_sec += $c[0]; + $bytes_sec += $c[2]; + } elsif (/msecs\/connect/) { + $msecs_connect{"mean"} += $c[1]; + $msecs_connect{"max"} += $c[3]; + $msecs_connect{"min"} += $c[5]; + } elsif (/msecs\/first/) { + $msecs_response{"mean"} += $c[1]; + $msecs_response{"max"} += $c[3]; + $msecs_response{"min"} += $c[5]; + } } print "Total runs: ", $runs, "\n"; -printf "%d fetches on %d conns, %d max parallell, %.5e bytes in %d seconds\n", +printf "%d fetches on %d conns, %d max parallel, %.5e bytes in %d seconds\n", $fetches, $conns, $parallel, $bytes, $seconds / $runs; -print $mean_bytes/$runs, " mean bytes/fetch\n"; -printf "%.2f fetches/sec, %.5e bytes/sec\n", $fetches_sec, $bytes_sec; -print "msecs/connect: ", $msecs_connect{"mean"}/$runs, " mean, ", - $msecs_connect{"max"}/$runs, " max, ", $msecs_connect{"min"}/$runs, " min\n"; -print "msecs/first-response: ", $msecs_response{"mean"}/$runs, " mean, ", - $msecs_response{"max"}/$runs, " max, ", $msecs_response{"min"}/$runs, " min\n"; +print $mean_bytes/ $runs, " mean bytes/fetch\n"; +printf "%.2f fetches/sec, %.5e bytes/sec\n", $fetches_sec, $bytes_sec; +print "msecs/connect: ", $msecs_connect{"mean"} / $runs, " mean, ", + $msecs_connect{"max"} / $runs, " max, ", $msecs_connect{"min"} / $runs, " min\n"; +print "msecs/first-response: ", $msecs_response{"mean"} / $runs, " mean, ", + $msecs_response{"max"} / $runs, " max, ", $msecs_response{"min"} / $runs, " min\n"; diff --git a/tools/jtest/README.zh.md b/tools/jtest/README.zh.md index c7d43f4e7cf..286ad9e8b7c 100644 --- a/tools/jtest/README.zh.md +++ b/tools/jtest/README.zh.md @@ -190,8 +190,8 @@ hash是jtest、ats里无处不在的，如何让hash互相影响，甚至测试h -N, --alternates int 0 Number of Alternates -e, --client_rate int 0 Clients Per Sec -o, --abort_retry_speed int 0 Abort/Retry Speed - - , --abort_retry_bytes int 0 Abort/Retry Threshhold (bytes) - - , --abort_retry_secs int 5 Abort/Retry Threshhold (secs) + - , --abort_retry_bytes int 0 Abort/Retry Threshold (bytes) + - , --abort_retry_secs int 5 Abort/Retry Threshold (secs) -W, --reload_rate dbl 0.000 Reload Rate * -D，用于生成url的随机数，如果有多个jtest并发运行，可以对这个随机的seed进行区分以控制cache的多小等 diff --git a/tools/jtest/jtest.cc b/tools/jtest/jtest.cc index 6f612aa1ff0..4253d04bbc7 100644 --- a/tools/jtest/jtest.cc +++ b/tools/jtest/jtest.cc @@ -92,7 +92,7 @@ #define MAX_BUFSIZE (65536 + 4096) // -// Contants +// Constants // #define MAXFDS 65536 #define HEADER_DONE -1 @@ -294,7 +294,7 @@ static const ArgumentDescription argument_descriptions[] = { {"alternates", 'N', "Number of Alternates", "I", &alternates, "JTEST_ALTERNATES", nullptr}, {"client_rate", 'e', "Clients Per Sec", "I", &client_rate, "JTEST_CLIENT_RATE", nullptr}, {"abort_retry_speed", 'o', "Abort/Retry Speed", "I", &abort_retry_speed, "JTEST_ABORT_RETRY_SPEED", nullptr}, - {"abort_retry_bytes", ' ', "Abort/Retry Threshhold (bytes)", "I", &abort_retry_bytes, "JTEST_ABORT_RETRY_THRESHHOLD_BYTES", + {"abort_retry_bytes", ' ', "Abort/Retry Threshold (bytes)", "I", &abort_retry_bytes, "JTEST_ABORT_RETRY_THRESHHOLD_BYTES", nullptr}, {"abort_retry_secs", ' ', "Abort/Retry Threshhold (secs)", "I", &abort_retry_secs, "JTEST_ABORT_RETRY_THRESHHOLD_SECS", nullptr}, {"reload_rate", 'W', "Reload Rate", "D", &reload_rate, "JTEST_RELOAD_RATE", nullptr}, @@ -1806,7 +1806,7 @@ poll_loop() if (ip >= POLL_GROUP_SIZE || i == last_fd) { int n = poll(pfd, ip, POLL_TIMEOUT); if (n > 0) { - for (int j = 0; j < ip; j++) { + for (int j = 0; j < n; j++) { if (pfd[j].revents & (POLLIN | POLLERR | POLLHUP | POLLNVAL)) { if (verbose) { printf("poll read %d %X\n", pfd[j].fd, pfd[j].revents); @@ -2523,7 +2523,7 @@ read_response(int sock) } if (check_content && !cl) { if (verbose || verbose_errors) { - printf("missiing Content-Length '%s'\n", fd[sock].base_url); + printf("missing Content-Length '%s'\n", fd[sock].base_url); } return read_response_error(sock); } @@ -4280,7 +4280,7 @@ ink_web_canonicalize_url(const char *base_url, const char *emb_url, char *dest_u } else { use_base_host = 1; - /* step 4 - if emb_path preceeded by slash, skip to 7 */ + /* step 4 - if emb_path preceded by slash, skip to 7 */ if (emb.leading_slash != 1) { /* step 5 */ diff --git a/tools/package/trafficserver.spec b/tools/package/trafficserver.spec index 86dc4d90d61..3df89b95c0a 100755 --- a/tools/package/trafficserver.spec +++ b/tools/package/trafficserver.spec @@ -34,12 +34,12 @@ URL: https://trafficserver.apache.org/ Source0: http://www.apache.org/dist/%{name}/%{name}-%{version}.tar.bz2 -BuildRequires: expat-devel hwloc-devel openssl-devel pcre-devel tcl-devel zlib-devel xz-devel +BuildRequires: expat-devel hwloc-devel openssl-devel pcre-devel zlib-devel xz-devel BuildRequires: libcurl-devel ncurses-devel BuildRequires: gcc gcc-c++ perl-ExtUtils-MakeMaker BuildRequires: libcap-devel -Requires: expat hwloc openssl pcre tcl zlib xz libcurl ncurses pkgconfig +Requires: expat hwloc openssl pcre zlib xz libcurl ncurses pkgconfig Requires: libcap # Can't seem to use libunwind on RHEL7 or older diff --git a/tools/slow_log_report.pl b/tools/slow_log_report.pl index 597b2f525f7..81fb52d47da 100755 --- a/tools/slow_log_report.pl +++ b/tools/slow_log_report.pl @@ -21,62 +21,72 @@ use warnings; #use Data::Dumper; -sub addStat($$$) { - my($stats, $key, $value) = @_; - #print "$key $value\n"; - $stats->{$key}->{total} = 0 if (! defined $stats->{$key}->{total}); - $stats->{$key}->{count} = 0 if (! defined $stats->{$key}->{count}); - return if (! ($value =~ m|^-?\d+\.?\d*$|)); - #print "$key\n"; - $stats->{$key}->{total} += $value if $value >= 0; - $stats->{$key}->{count}++ if $value >= 0; - push(@{$stats->{$key}->{values}}, $value) if $value >= 0; +sub addStat($$$) +{ + my ($stats, $key, $value) = @_; + #print "$key $value\n"; + $stats->{$key}->{total} = 0 if (!defined $stats->{$key}->{total}); + $stats->{$key}->{count} = 0 if (!defined $stats->{$key}->{count}); + return if (!($value =~ m|^-?\d+\.?\d*$|)); + #print "$key\n"; + $stats->{$key}->{total} += $value if $value >= 0; + $stats->{$key}->{count}++ if $value >= 0; + push(@{$stats->{$key}->{values}}, $value) if $value >= 0; } -sub displayStat($) { - my($stats) = @_; +sub displayStat($) +{ + my ($stats) = @_; + + printf("%25s %10s %10s %10s %10s %10s %10s %10s %10s\n", + 'key', 'total', 'count', 'mean', 'median', '95th', '99th', 'min', 'max'); + foreach my $key ( + 'ua_begin', 'ua_first_read', 'ua_read_header_done', 'cache_open_read_begin', + 'cache_open_read_end', 'dns_lookup_begin', 'dns_lookup_end', 'server_connect', + 'server_connect_end', 'server_first_read', 'server_read_header_done', 'server_close', + 'ua_close', 'sm_finish' + ) + { - printf("%25s %10s %10s %10s %10s %10s %10s %10s %10s\n", 'key', 'total', 'count', 'mean', 'median', '95th', '99th', 'min', 'max'); - foreach my $key ('ua_begin', 'ua_first_read', 'ua_read_header_done', 'cache_open_read_begin', 'cache_open_read_end', 'dns_lookup_begin', 'dns_lookup_end', 'server_connect', 'server_first_read', 'server_read_header_done', 'server_close', 'ua_close', 'sm_finish') { + my $count = $stats->{$key}->{count}; + my $total = $stats->{$key}->{total}; + if (!defined $stats->{$key}->{values}) { + next; + #print "$key\n"; + #die $key; + } + my @sorted = sort {$a <=> $b} @{$stats->{$key}->{values}}; + my $median = $sorted[int($count / 2)]; + my $p95th = $sorted[int($count * .95)]; + my $p99th = $sorted[int($count * .99)]; + my $min = $sorted[0]; + my $max = $sorted[$count - 1]; + my $mean = 0; + $mean = $total / $count if $count > 0; - my $count = $stats->{$key}->{count}; - my $total = $stats->{$key}->{total}; - if (!defined $stats->{$key}->{values}) { - next; - #print "$key\n"; - #die $key; + printf("%25s %10.2f %10.2f %10.2f %10.2f %10.2f %10.2f %10.2f %10.2f\n", + $key, $total, $count, $mean, $median, $p95th, $p99th, $min, $max); } - my @sorted = sort {$a <=> $b} @{$stats->{$key}->{values}}; - my $median = $sorted[int($count/2)]; - my $p95th = $sorted[int($count * .95)]; - my $p99th = $sorted[int($count * .99)]; - my $min = $sorted[0]; - my $max = $sorted[$count - 1]; - my $mean = 0; - $mean = $total / $count if $count > 0; - - printf("%25s %10.2f %10.2f %10.2f %10.2f %10.2f %10.2f %10.2f %10.2f\n", $key, $total, $count, $mean, $median, $p95th, $p99th, $min, $max); - } - print "NOTE: Times are in seconds\n"; + print "NOTE: Times are in seconds\n"; } { - my %stats; + my %stats; - while (<>) { - chomp; - s/unique id/unique_id/; - s/server state/server_state/; - s/client state/client_state/; - if (m|Slow Request: .+ (ua_begin: .+)|) { - my %data = split(/: | /, $1); - foreach my $key (keys %data) { - next if (!defined $data{$key}); - #print "$key $data{$key}\n"; - addStat(\%stats, $key, $data{$key}); - } + while (<>) { + chomp; + s/unique id/unique_id/; + s/server state/server_state/; + s/client state/client_state/; + if (m|Slow Request: .+ (ua_begin: .+)|) { + my %data = split(/: | /, $1); + foreach my $key (keys %data) { + next if (!defined $data{$key}); + #print "$key $data{$key}\n"; + addStat(\%stats, $key, $data{$key}); + } + } } - } - displayStat(\%stats); + displayStat(\%stats); } diff --git a/tools/traffic_primer b/tools/traffic_primer index 4c3dc4c0663..2d73c9d79ad 100755 --- a/tools/traffic_primer +++ b/tools/traffic_primer @@ -18,7 +18,7 @@ # Simple script to fetch a URL through one proxy, and then PUSH that response (headers # and body) to a set of hosts. The host:port defaults to localhost:80 for fetching -# the URL, but can be overriden with -h/-p. +# the URL, but can be overridden with -h/-p. # Default values for command line options @@ -72,7 +72,7 @@ push() { if [ $? -gt 0 ]; then echo -e "\tError: ${h}:${PORT} is down" else - echo -en "\tProcesing ${h}:${PORT}..." + echo -en "\tProcessing ${h}:${PORT}..." # PURGE the object first if we -f (force) it if [ $FORCE -gt 0 ]; then echo -n " purged..." diff --git a/tools/traffic_via.pl b/tools/traffic_via.pl deleted file mode 100755 index 6eee2a6cb13..00000000000 --- a/tools/traffic_via.pl +++ /dev/null @@ -1,277 +0,0 @@ -#!/usr/bin/env perl - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Script to decode via header -# -# @File traffic_via.pl -# @Author Meera Mosale Nataraja -# -# Usage: Use it in 2 ways -# 1. Pass Via Header with -s option \n"; -# traffic_via [-s viaheader]"; -# or -# 2. Pipe curl output -# curl -v -H "X-Debug: Via" http://ats_server:port 2>&1| ./traffic_via.pl -# - -use strict; -use warnings; -use Data::Dumper qw(Dumper); -use Getopt::Long; - -my $via_header; -my $help; - -#Proxy request header flags and titles -my @proxy_header_array = ( - { - "Request headers received from client:", - { - 'I' => "If Modified Since (IMS)", - 'C' => "cookie", - 'E' => "error in request", - 'S' => "simple request (not conditional)", - 'N' => "no-cache", - ' ' => "unknown?", - }, - }, - { - "Result of Traffic Server cache lookup for URL:", - { - 'A' => "in cache, not acceptable (a cache \"MISS\")", - 'H' => "in cache, fresh (a cache \"HIT\")", - 'S' => "in cache, stale (a cache \"MISS\")", - 'R' => "in cache, fresh Ram hit (a cache \"HIT\")", - 'M' => "miss (a cache \"MISS\")", - ' ' => "unknown?", - }, - }, - { - "Response information received from origin server:", - { - 'E' => "error in response", - ' ' => "no server connection needed", - 'S' => "served", - 'N'=> "not-modified", - } - }, - { - "Result of document write-to-cache:", - { - 'U' => "updated old cache copy", - 'D' => "cached copy deleted", - 'W' => "written into cache (new copy)", - ' ' => "no cache write performed", - }, - }, - { - "Proxy operation result:", - { - 'R' => "origin server revalidated", - ' ' => "unknown?", - 'S' => "served", - 'N' => "not-modified", - }, - }, - { - "Error codes (if any):", - { - 'A' => "authorization failure", - 'H' => "header syntax unacceptable", - 'C' => "connection to server failed", - 'T' => "connection timed out", - 'S' => "server related error", - 'D' => "dns failure", - 'N' => "no error", - 'F' => "request forbidden", - }, - }, - { - "Tunnel info:", - { - ' ' => "no tunneling", - 'U' => "tunneling because of url (url suggests dynamic content)", - 'M' => "tunneling due to a method (e.g. CONNECT)", - 'O' => "tunneling because cache is turned off", - 'F' => "tunneling due to a header field (such as presence of If-Range header)", - }, - }, - { - "Cache type:", - { - 'I' => "icp", - ' ' => "cache miss or no cache lookup", - 'C' => "cache", - }, - }, - { - "Cache lookup result:", - { - ' ' => "no cache lookup", - 'S' => "cache hit, but expired", - 'U' => "cache hit, but client forces revalidate (e.g. Pragma: no-cache)", - 'D' => "cache hit, but method forces revalidated (e.g. ftp, not anonymous)", - 'I' => "conditional miss (client sent conditional, fresh in cache, returned 412)", - 'H' => "cache hit", - 'M' => "cache miss (url not in cache)", - 'C' => "cache hit, but config forces revalidate", - 'N' => "conditional hit (client sent conditional, doc fresh in cache, returned 304)", - }, - }, - { - "ICP status:", - { - ' ' => "no icp", - 'S' => "connection opened successfully", - 'F' => "connection open failed", - }, - }, - { - "Parent proxy connection status:", - { - ' ' => "no parent proxy", - 'S' => "connection opened successfully", - 'F' => "connection open failed", - }, - - }, - { - "Origin server connection status:", - { - ' ' => "no server connection", - 'S' => "connection opened successfully", - 'F' => "connection open failed", - }, - }, -); - -##Print script usage -sub usage -{ - print "\nPass Via Header with -s option \n"; - print "Usage: traffic_via [-s viaheader]"; - print "\n or \n"; - print "\nPipe curl command output to this program"; - print "\nEg: curl xxxx | traffic_via\n"; - print "\n-h for help\n"; - exit; -} - -if (@ARGV == 0) { - #if passed through standard input - my @userinput = ; - my $via_string; - - for my $element (@userinput) { - #Pattern matching for Via - if ($element =~ /Via:(.*)\[(.*)\]/) { - #Search and grep via header - $via_string = $2; - chomp($via_string); - print "Via Header is [$via_string]"; - decode_via_header($via_string); - } - } -} else { - usage() if (!GetOptions('s=s' => \$via_header, - 'help|?' => \$help) or - defined $help); - - if (defined $via_header) { - #if passed through commandline dashed argument - print "Via Header is [$via_header]"; - decode_via_header($via_header); - } - -} - -#Subroutine to decode via header -sub decode_via_header { - my($header) = @_; - my $hdrLength; - my $newHeader; - - #Check via header syntax - if ($header =~ /([a-zA-Z: ]+)/) { - #Get via header length - $hdrLength = length($header); - - # Valid Via header length is 24 or 6. - # When Via header length is 24, it will have both proxy request header result and operational results. - if ($hdrLength == 24) { - #Split via header: proxy result and operational result - $newHeader = join('', split(':', $header)); - } elsif ($hdrLength == 6) { - $newHeader = $header; - } elsif ($hdrLength == 5) { - # When Via header length is 5, it might be missing last field. Fill it and decode header. - my $newHeader = "$header"." "; - } else { - # Invalid header size, come out. - print "\nInvalid VIA header. VIA header length should be 6 or 24 characters\n"; - return; - } - convert_header_to_array($newHeader); - } - - -} - -sub convert_header_to_array { - my ($viaHeader) = @_; - my @ResultArray; - #Convert string header into character array - while ($viaHeader =~ /(.)/g) { - #Only capital letters indicate flags - if ($1 !~ m/[a-z]+/) { - push(@ResultArray, $1); - } - } - print "\nVia Header details: \n"; - for (my $arrayIndex=0; $arrayIndex < scalar(@ResultArray); $arrayIndex++ ) { - get_via_header_flags(\@proxy_header_array, $arrayIndex, $ResultArray[$arrayIndex]); - } -} - -#Get values from header arrays -sub get_via_header_flags { - my ($arrayName, $inputIndex, $flag) = @_; - - my %flagValues; - my @flagKeys; - my %flags; - my @keys; - - my @array = @$arrayName; - - %flagValues = %{$array[$inputIndex]}; - @flagKeys = keys (%flagValues); - - foreach my $keyEntry ( @flagKeys ) { - printf ("%-55s", $keyEntry); - %flags = %{$flagValues{$keyEntry}}; - @keys = keys (%flags); - foreach my $key ( @keys ) { - if ($key =~ /$flag/) { - #print $flags{$key}; - printf("%s",$flags{$key}); - print "\n"; - } - } - } -} diff --git a/tools/traffic_via/test_traffic_via_pl b/tools/traffic_via/test_traffic_via_pl new file mode 100755 index 00000000000..ed4c6e1c715 --- /dev/null +++ b/tools/traffic_via/test_traffic_via_pl @@ -0,0 +1,47 @@ +#! /usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e # exit on error + +TMPDIR=${TMPDIR:-/tmp} +tmpfile=$(mktemp "$TMPDIR/via.XXXXXX") +if [ ! -z "$srcdir"]; then + srcdir=$(cd $srcdir && pwd) +else + srcdir=$(pwd) +fi + +# test command line option +echo "Testing the command line option:" +for f in $srcdir/tests/*; do + name=$(basename "$f") + echo "testing $name" + ./traffic_via.pl -s "$name" > "$tmpfile" 2>&1 || true + diff -u "$tmpfile" "$srcdir/tests/$name" +done + +# test stdin +echo -e "\nTesting stdin:" +for f in $srcdir/tests/\[*; do + name=$(basename "$f") + echo "testing $name" + echo "$name" | ./traffic_via.pl > "$tmpfile" 2>&1 || true + diff -u "$tmpfile" "$srcdir/tests/$name" +done + +rm -f $tmpfile diff --git a/tools/traffic_via/tests/[u c s f p eS;tNc p s ] b/tools/traffic_via/tests/[u c s f p eS;tNc p s ] new file mode 100644 index 00000000000..e662dc0e814 --- /dev/null +++ b/tools/traffic_via/tests/[u c s f p eS;tNc p s ] @@ -0,0 +1,13 @@ +Via header is [u c s f p eS;tNc p s ], Length is 22 +Via Header Details: +Request headers received from client :unknown +Result of Traffic Server cache lookup for URL :no cache lookup +Response information received from origin server :no server connection needed +Result of document write-to-cache: :no cache write performed +Proxy operation result :unknown +Error codes (if any) :server related error +Tunnel info :tunneling due to no forward +Cache Type :unknown +Cache Lookup Result :cache miss or no cache lookup +Parent proxy connection status :no parent proxy or unknown +Origin server connection status :no server connection needed diff --git a/tools/traffic_via/tests/[uIcRs f p eN;t cCHp s ] b/tools/traffic_via/tests/[uIcRs f p eN;t cCHp s ] new file mode 100644 index 00000000000..0755883b90f --- /dev/null +++ b/tools/traffic_via/tests/[uIcRs f p eN;t cCHp s ] @@ -0,0 +1,13 @@ +Via header is [uIcRs f p eN;t cCHp s ], Length is 22 +Via Header Details: +Request headers received from client :IMS +Result of Traffic Server cache lookup for URL :in cache, fresh Ram hit (a cache "HIT") +Response information received from origin server :no server connection needed +Result of document write-to-cache: :no cache write performed +Proxy operation result :unknown +Error codes (if any) :no error +Tunnel info :no tunneling +Cache Type :cache +Cache Lookup Result :cache hit +Parent proxy connection status :no parent proxy or unknown +Origin server connection status :no server connection needed diff --git a/tools/traffic_via/tests/[uIcRs f p eN;t cCNp s ] b/tools/traffic_via/tests/[uIcRs f p eN;t cCNp s ] new file mode 100644 index 00000000000..6ecfc64517a --- /dev/null +++ b/tools/traffic_via/tests/[uIcRs f p eN;t cCNp s ] @@ -0,0 +1,13 @@ +Via header is [uIcRs f p eN;t cCNp s ], Length is 22 +Via Header Details: +Request headers received from client :IMS +Result of Traffic Server cache lookup for URL :in cache, fresh Ram hit (a cache "HIT") +Response information received from origin server :no server connection needed +Result of document write-to-cache: :no cache write performed +Proxy operation result :unknown +Error codes (if any) :no error +Tunnel info :no tunneling +Cache Type :cache +Cache Lookup Result :conditional hit (client sent conditional, doc fresh in cache, returned 304) +Parent proxy connection status :no parent proxy or unknown +Origin server connection status :no server connection needed diff --git a/tools/traffic_via/tests/[uScMsSf pSeN;t cCMp sS] b/tools/traffic_via/tests/[uScMsSf pSeN;t cCMp sS] new file mode 100644 index 00000000000..6ee096a1889 --- /dev/null +++ b/tools/traffic_via/tests/[uScMsSf pSeN;t cCMp sS] @@ -0,0 +1,13 @@ +Via header is [uScMsSf pSeN;t cCMp sS], Length is 22 +Via Header Details: +Request headers received from client :simple request (not conditional) +Result of Traffic Server cache lookup for URL :miss (a cache "MISS") +Response information received from origin server :connection opened successfully +Result of document write-to-cache: :no cache write performed +Proxy operation result :served or connection opened successfully +Error codes (if any) :no error +Tunnel info :no tunneling +Cache Type :cache +Cache Lookup Result :cache miss (url not in cache) +Parent proxy connection status :no parent proxy or unknown +Origin server connection status :connection opened successfully diff --git a/tools/traffic_via/tests/[uScRs f p eN;t cCHp s ] b/tools/traffic_via/tests/[uScRs f p eN;t cCHp s ] new file mode 100644 index 00000000000..ea96fdb54fa --- /dev/null +++ b/tools/traffic_via/tests/[uScRs f p eN;t cCHp s ] @@ -0,0 +1,13 @@ +Via header is [uScRs f p eN;t cCHp s ], Length is 22 +Via Header Details: +Request headers received from client :simple request (not conditional) +Result of Traffic Server cache lookup for URL :in cache, fresh Ram hit (a cache "HIT") +Response information received from origin server :no server connection needed +Result of document write-to-cache: :no cache write performed +Proxy operation result :unknown +Error codes (if any) :no error +Tunnel info :no tunneling +Cache Type :cache +Cache Lookup Result :cache hit +Parent proxy connection status :no parent proxy or unknown +Origin server connection status :no server connection needed diff --git a/tools/traffic_via/tests/long rubbish via code2 b/tools/traffic_via/tests/long rubbish via code2 new file mode 100644 index 00000000000..470c5a1b560 --- /dev/null +++ b/tools/traffic_via/tests/long rubbish via code2 @@ -0,0 +1,15 @@ +Via header is [long rubbish via code2], Length is 22 +traffic_via: Invalid VIA header character: l +traffic_via: Invalid VIA header character: o +traffic_via: Invalid VIA header character: n +traffic_via: Invalid VIA header character: g +traffic_via: Invalid VIA header character: r +traffic_via: Invalid VIA header character: b +traffic_via: Invalid VIA header character: b +traffic_via: Invalid VIA header character: i +traffic_via: Invalid VIA header character: h +traffic_via: Invalid VIA header character: v +traffic_via: Invalid VIA header character: i +traffic_via: Invalid VIA header character: a +traffic_via: Invalid VIA header character: o +traffic_via: Invalid VIA header character: d diff --git a/tools/traffic_via/tests/rubbish b/tools/traffic_via/tests/rubbish new file mode 100644 index 00000000000..dbeeab0e892 --- /dev/null +++ b/tools/traffic_via/tests/rubbish @@ -0,0 +1,6 @@ +Via header is [rubbish], Length is 7 +traffic_via: Invalid VIA header character: r +traffic_via: Invalid VIA header character: b +traffic_via: Invalid VIA header character: b +traffic_via: Invalid VIA header character: i +traffic_via: Invalid VIA header character: h diff --git a/tools/traffic_via/tests/short b/tools/traffic_via/tests/short new file mode 100644 index 00000000000..384a48330cc --- /dev/null +++ b/tools/traffic_via/tests/short @@ -0,0 +1,5 @@ +Via header is [short], Length is 5 +traffic_via: Invalid VIA header character: h +traffic_via: Invalid VIA header character: o +traffic_via: Invalid VIA header character: r +traffic_via: Invalid VIA header character: t diff --git a/tools/traffic_via/traffic_via.pl b/tools/traffic_via/traffic_via.pl new file mode 100755 index 00000000000..a135364f75f --- /dev/null +++ b/tools/traffic_via/traffic_via.pl @@ -0,0 +1,318 @@ +#!/usr/bin/env perl + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Script to decode via header +# +# @File traffic_via.pl +# @Author Meera Mosale Nataraja +# +# Usage: Use it in 2 ways +# 1. Pass Via Header with -s option \n"; +# traffic_via [-s viaheader]"; +# or +# 2. Pipe curl output +# curl -v -H "X-Debug: Via" http://ats_server:port 2>&1| ./traffic_via.pl +# + +use strict; +use warnings; +use Data::Dumper qw(Dumper); +use Getopt::Long; + +my $via_header; +my $help; + +#Proxy request header flags and titles +my @proxy_header_array = ( + { + "Request headers received from client", { + 'I' => "IMS", + 'C' => "cookie", + 'E' => "error in request", + 'S' => "simple request (not conditional)", + 'N' => "no-cache", + ' ' => "unknown", + }, + }, { + "Result of Traffic Server cache lookup for URL", { + 'A' => "in cache, not acceptable (a cache \"MISS\")", + 'H' => "in cache, fresh (a cache \"HIT\")", + 'S' => "in cache, stale (a cache \"MISS\")", + 'R' => "in cache, fresh Ram hit (a cache \"HIT\")", + 'M' => "miss (a cache \"MISS\")", + ' ' => "no cache lookup", + }, + }, { + "Response information received from origin server", { + 'E' => "error in response", + ' ' => "no server connection needed", + 'S' => "connection opened successfully", + 'N' => "not-modified", + } + }, { + "Result of document write-to-cache:", { + 'U' => "updated old cache copy", + 'D' => "cached copy deleted", + 'W' => "written into cache (new copy)", + ' ' => "no cache write performed", + }, + }, { + "Proxy operation result", { + 'R' => "origin server revalidated", + ' ' => "unknown", + 'S' => "served or connection opened successfully", + 'N' => "not-modified", + }, + }, { + "Error codes (if any)", { + 'A' => "authorization failure", + 'H' => "header syntax unacceptable", + 'C' => "connection to server failed", + 'T' => "connection timed out", + 'S' => "server related error", + 'D' => "dns failure", + 'N' => "no error", + 'F' => "request forbidden", + }, + }, { + "Tunnel info", { + ' ' => "no tunneling", + 'U' => "tunneling because of url (url suggests dynamic content)", + 'M' => "tunneling due to a method (e.g. CONNECT)", + 'O' => "tunneling because cache is turned off", + 'F' => "tunneling due to a header field (such as presence of If-Range header)", + 'N' => "tunneling due to no forward", + }, + }, { + "Cache Type", { + ' ' => "unknown", + 'I' => "icp", + 'C' => "cache", + }, + }, { + "Cache Lookup Result", { + ' ' => "cache miss or no cache lookup", + 'S' => "cache hit, but expired", + 'U' => "cache hit, but client forces revalidate (e.g. Pragma: no-cache)", + 'D' => "cache hit, but method forces revalidated (e.g. ftp, not anonymous)", + 'I' => "conditional miss (client sent conditional, fresh in cache, returned 412)", + 'H' => "cache hit", + 'M' => "cache miss (url not in cache)", + 'C' => "cache miss (url not in cache)", + 'N' => "conditional hit (client sent conditional, doc fresh in cache, returned 304)", + }, + }, { + "Parent proxy connection status", { + ' ' => "no parent proxy or unknown", + 'S' => "connection opened successfully", + 'F' => "connection open failed", + }, + + }, { + "Origin server connection status", { + ' ' => "no server connection needed", + 'S' => "connection opened successfully", + 'F' => "connection open failed", + }, + }, +); + +##Print script usage +sub usage() +{ + print "\nPass Via Header with -s option \n"; + print "Usage: traffic_via [-s viaheader]"; + print "\n or \n"; + print "\nPipe curl command output to this program"; + print "\nEg: curl xxxx | traffic_via\n"; + print "\n-h for help\n"; + exit; +} + + + +#Subroutine to decode via header +sub decode_via_header($) +{ + my ($header) = @_; + my $hdrLength; + my $newHeader; + + #Check via header syntax + if ($header =~ /([a-zA-Z: ]+)/) { + #Get via header length + $hdrLength = length($header); + + # Valid Via header length is 22 or 6. + # When Via header length is 22, it will have both proxy request header result and operational results. + if ($hdrLength == 22) { + #Split via header: proxy result and operational result + $newHeader = join('', split(':', $header)); + } elsif ($hdrLength == 6) { + $newHeader = $header; + } elsif ($hdrLength == 5) { + # When Via header length is 5, it might be missing last field. Fill it and decode header. + $newHeader = $header . " "; + } else { + # Invalid header size, come out. + print "\nInvalid VIA header. VIA header length should be 6 or 22 characters\n"; + return; + } + + convert_header_to_array($newHeader); + } + +} + +sub convert_header_to_array($) +{ + my ($viaHeader) = @_; + my @ResultArray; + #Convert string header into character array + while ($viaHeader =~ /(.)/g) { + #Only capital letters indicate flags + if ($1 !~ m/[a-z;]+/) { + push(@ResultArray, $1); + } + } + print "Via Header Details:\n"; + for (my $arrayIndex = 0; $arrayIndex < scalar(@ResultArray); $arrayIndex++) { + get_via_header_flags(\@proxy_header_array, $arrayIndex, $ResultArray[$arrayIndex]); + } +} + +my %valid_keys = ('main' => { + 'u' => 1, + 'c' => 1, + 's' => 1, + 'f' => 1, + 'p' => 1, + 'e' => 1, + }, + 'detail' => { + 't' => 1, + 'c' => 1, + 'p' => 1, + 's' => 1, + } +); + +sub valid_char ($$) +{ + my($char, $hash) = @_; + + return exists $hash->{$char} +} + +sub validate_keys($) +{ + my($viaHeader) = @_; + my($main, $detail) = split(';', $viaHeader); + my $running_main = 1; + my $return_value_valid = 1; + + foreach my $group ($main, $detail) { + next if !defined $group; + + while ($group =~ /([a-z])/g) { + my $char = $1; + + my $valid = 0; + if ($running_main) { + $valid = valid_char($char, $valid_keys{main}); + } else { + $valid = valid_char($char, $valid_keys{detail}); + } + if (! $valid) { + print "traffic_via: Invalid VIA header character: $char\n"; + $return_value_valid = 0; + } + } + $running_main = 0; + } + return $return_value_valid; +} + +#Get values from header arrays +sub get_via_header_flags($$$) +{ + my ($arrayName, $inputIndex, $flag) = @_; + + my %flagValues; + my @flagKeys; + my %flags; + my @keys; + + my @array = @$arrayName; + + %flagValues = %{$array[$inputIndex]}; + @flagKeys = keys(%flagValues); + + foreach my $keyEntry (@flagKeys) { + printf("%-55s", $keyEntry); + %flags = %{$flagValues{$keyEntry}}; + @keys = keys(%flags); + foreach my $key (@keys) { + if ($key =~ /$flag/) { + #print $flags{$key}; + printf(":%s", $flags{$key}); + print "\n"; + } + } + } +} + +# main +{ + if (@ARGV == 0) { + #if passed through standard input + my @userinput = ; + + for my $element (@userinput) { + #Pattern matching for Via + if ($element =~ /Via:\s+\[(.+)\]/i || $element =~ /\[(.+)\]/ ) { + #Search and grep via header + my $via_string = $1; + chomp($via_string); + print "Via header is [$via_string], Length is ", length($via_string), "\n"; + last unless validate_keys($via_string); + decode_via_header($via_string); + } + } + } else { + usage() + if ( + !GetOptions( + 's=s' => \$via_header, + 'help|?' => \$help + ) + or defined $help + ); + + if (defined $via_header) { + if ($via_header =~ /Via:\s+\[(.+)\]/i || $via_header =~ /\[(.+)\]/ || $via_header =~ /(.+)/) { + #if passed through commandline dashed argument + my $via_string = $1; + print "Via header is [$via_string], Length is ", length($via_string), "\n"; + last unless validate_keys($via_string); + decode_via_header($via_string); + } + } + + } +} \ No newline at end of file diff --git a/tools/voluspa/.gitignore b/tools/voluspa/.gitignore new file mode 100644 index 00000000000..09a98f8e11d --- /dev/null +++ b/tools/voluspa/.gitignore @@ -0,0 +1,4 @@ +voluspa +voluspa-covered +/dist +/RPMS diff --git a/tools/voluspa/.gometalint.conf b/tools/voluspa/.gometalint.conf new file mode 100644 index 00000000000..c3719752a3d --- /dev/null +++ b/tools/voluspa/.gometalint.conf @@ -0,0 +1,5 @@ +{ + "$comment": "Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements; and to You under the Apache License, Version 2.0.", + "EnableAll": true, + "Disable": ["gosec", "gas", "golint", "vetshadow", "gocyclo"] +} diff --git a/tools/voluspa/Godeps/Godeps.json b/tools/voluspa/Godeps/Godeps.json new file mode 100644 index 00000000000..3ef1bdf7daf --- /dev/null +++ b/tools/voluspa/Godeps/Godeps.json @@ -0,0 +1,36 @@ +{ + "ImportPath": "github.com/apache/trafficserver/tools/voluspa", + "GoVersion": "go1.12", + "GodepVersion": "v80", + "Packages": [ + ".", + "./adapters", + "./adapters/confremap", + "./adapters/headerrewrite", + "./adapters/lua", + "./adapters/util", + "./cmd/voluspa", + "./internal/util/regex", + "./version" + ], + "Deps": [ + { + "ImportPath": "github.com/xeipuuv/gojsonpointer", + "Rev": "4e3ac2762d5f479393488629ee9370b50873b3a6" + }, + { + "ImportPath": "github.com/xeipuuv/gojsonreference", + "Rev": "bd5ef7bd5415a7ac448318e64f11a24cd21e594b" + }, + { + "ImportPath": "github.com/xeipuuv/gojsonschema", + "Comment": "v1.1.0-14-g354ad34", + "Rev": "354ad34c2300dd6e84920ac70b41487336219e43" + }, + { + "ImportPath": "gopkg.in/yaml.v2", + "Comment": "v2.2.2-1-g7b8349a", + "Rev": "7b8349ac747c6a24702b762d2c4fd9266cf4f1d6" + } + ] +} diff --git a/tools/voluspa/Godeps/Readme b/tools/voluspa/Godeps/Readme new file mode 100644 index 00000000000..4cdaa53d56d --- /dev/null +++ b/tools/voluspa/Godeps/Readme @@ -0,0 +1,5 @@ +This directory tree is generated automatically by godep. + +Please do not edit. + +See https://github.com/tools/godep for more information. diff --git a/tools/voluspa/LICENSE b/tools/voluspa/LICENSE new file mode 100644 index 00000000000..d6456956733 --- /dev/null +++ b/tools/voluspa/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/tools/voluspa/Makefile b/tools/voluspa/Makefile new file mode 100644 index 00000000000..c75fdd37967 --- /dev/null +++ b/tools/voluspa/Makefile @@ -0,0 +1,147 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +APP := voluspa +RELEASE ?= 0 +VERSION_BASE := 24A +BUILD_NUMBER ?= devel +VERSION := $(VERSION_BASE)$(BUILD_NUMBER) +GIT_VERSION := $(shell git describe --always --long) +PKG := github.com/apache/trafficserver/tools/voluspa +LDFLAGS := -ldflags "-w -X $(PKG).Version=$(VERSION) -X $(PKG).GitVersion=$(GIT_VERSION)" +DIST := dist +ARCHIVE := $(APP)-$(VERSION) +CURRENT_SCHEMA := schema_v1.json +DEST ?= . +GO ?= go + +COMMON_GO_FILES := *.go */*.go internal/util/*/*go */*/*.go + +GO_PKG := $(shell go list ./... | egrep -v 'vendor|cmd|other') +GO_PKG_DIRS := $(subst $(shell go list -e .),.,$(shell go list ./... | egrep -v 'vendor|other')) + +CMD_SOURCES := $(shell find cmd -name main.go) +DEV_TARGETS := $(patsubst cmd/%/main.go,%,$(CMD_SOURCES)) +DIST_TARGETS := voluspa +LINUX_TARGETS := $(patsubst %,dist/linux-x86_64/%,$(DIST_TARGETS)) +DARWIN_TARGETS := $(patsubst %,dist/darwin-x86_64/%,$(DIST_TARGETS)) + +all: default + +generate: + go generate ./... + +default: $(DEV_TARGETS) + +sanity_tests: $(APP) + @make -C tests test + +test: $(APP) sanity_tests + $(GO) test -cover $(GO_PKG_DIRS) + +lint: + -$(GO) vet ./... 2>&1 + -golint $(GO_PKG) | egrep -v 'vendor|unexport' + +metalint: + -gometalinter --config .gometalint.conf --deadline=60s -e 'vendor' ./... + +junit: + @mkdir -p .out/test-results/ # where rio junit support looks by default + $(GO) test -v $(GO_PKG_DIRS) 2>&1 | go-junit-report > .out/test-results/junit-report.xml + +cover: + # go get github.com/wadey/gocovmerge github.com/axw/gocov/... github.com/matm/gocov-html + # based on https://github.com/golang/go/issues/6909#issuecomment-233493644 + @echo Generating an overall coverage report + @rm -f *.coverprofile + go list -f '{{if or (len .TestGoFiles) (len .XTestGoFiles)}}go test -test.timeout=120s -covermode=count -coverprofile={{.Name}}_{{len .Imports}}_{{len .Deps}}.coverprofile -coverpkg ./... {{.ImportPath}}{{end}}' $(GO_PKG) | sh -x + gocovmerge `ls *.coverprofile` > cover.out + @rm -f *.coverprofile + @mkdir -p .out/coverage/ + gocov convert cover.out | gocov-html > .out/coverage/merged-coverage.html + @[[ $$(uname) == 'Darwin' ]] && open .out/coverage/merged-coverage.html || true + +voluspa-covered: cmd/voluspa/*.go $(COMMON_GO_FILES) + @# build a test version of the voluspa command with coverage enabled + @# e.g.: ./voluspa-covered -test.coverprofile cover.out .../*.conf + go test -o $(DEST)/$@ -c -tags testrunmain -coverpkg ./... $(PKG)/cmd/voluspa + +gocov: .out/coverage/coverage.html + +.out/coverage/coverage.json: voluspa-covered + @mkdir -p .out/coverage/ + gocov test $(GO_PKG) > .out/coverage/coverage.json + @make -C tests coverage + gocov convert tests/regress/profile*.out > tests/regress/system.json + gocov convert tests/other/profile*.out > tests/other/system.json + cat .out/coverage/coverage.json tests/other/system.json tests/regress/system.json | gocov-merge > .out/coverage/coverage.json + +.out/coverage/coverage.html: .out/coverage/coverage.json + gocov-html < .out/coverage/coverage.json > .out/coverage/coverage.html + +dist: artifactory_targets $(DIST)/artifactory/version.txt $(DIST)/artifactory/versions.sh + +clean: + @rm -rf $(DIST) out out_remap RPMS .dist .out + @rm -f $(DEV_TARGETS) $(TOOLS_TARGETS) $(OTHER_TARGETS) voluspa-covered + @make -C tests clean + +%: cmd/%/*.go $(COMMON_GO_FILES) + $(GO) build -o $(DEST)/$@ $(LDFLAGS) cmd/$@/*.go + +$(DIST)/linux-x86_64/%: + GOOS=linux GOARCH=amd64 $(GO) build -o $@ $(LDFLAGS) cmd/$(notdir $@)/*.go + +$(DIST)/darwin-x86_64/%: + GOOS=darwin GOARCH=amd64 $(GO) build -o $@ $(LDFLAGS) cmd/$(notdir $@)/*.go + +$(DIST)/freebsd-x86_64/%: + GOOS=freebsd GOARCH=amd64 $(GO) build -o $@ $(LDFLAGS) cmd/$(notdir $@)/*.go + +$(DIST)/artifactory: + @mkdir -p $@ + +$(DIST)/artifactory/$(APP)-$(VERSION)-darwin-x86_64: $(DIST)/artifactory dist/darwin-x86_64/voluspa + cp dist/darwin-x86_64/voluspa $@ + +$(DIST)/artifactory/$(APP)-$(VERSION)-linux-x86_64: $(DIST)/artifactory dist/linux-x86_64/voluspa + cp dist/linux-x86_64/voluspa $@ + +$(DIST)/artifactory/$(CURRENT_SCHEMA)-$(VERSION): $(DIST)/artifactory $(CURRENT_SCHEMA) + cp $(CURRENT_SCHEMA) $@ + +$(DIST)/artifactory/version.txt: + @mkdir -p dist/artifactory + echo $(VERSION) > $@ + +artifactory_targets: $(DIST)/artifactory/$(APP)-$(VERSION)-darwin-x86_64 $(DIST)/artifactory/$(APP)-$(VERSION)-linux-x86_64 $(DIST)/artifactory/$(CURRENT_SCHEMA)-$(VERSION) + +$(DIST)/artifactory/versions.sh: $(DIST)/artifactory/$(APP)-$(VERSION)-darwin-x86_64 $(DIST)/artifactory/$(APP)-$(VERSION)-linux-x86_64 + echo VOLUSPA_VERSION=$(VERSION) > $@ + echo >> $@ + sha1sum $(DIST)/artifactory/$(APP)-$(VERSION)-linux-x86_64 | awk 'BEGIN { FS = " " } ; { printf "linux_SHA1=%s\n", $$1 }' >> $@ + sha1sum $(DIST)/artifactory/$(APP)-$(VERSION)-darwin-x86_64 | awk 'BEGIN { FS = " " } ; { printf "darwin_SHA1=%s\n", $$1 }' >> $@ + sha1sum $(DIST)/artifactory/$(CURRENT_SCHEMA)-$(VERSION) | awk 'BEGIN { FS = " " } ; { printf "schema_SHA1=%s\n", $$1 }' >> $@ + +update-deps: + -grep ImportPath Godeps/Godeps.json | cut -f 4 -d \" | grep -v voluspa | xargs go get -u + @rm -rf Godeps vendor/* + godep save $(GO_PKG_DIRS) + +.PHONY: all clean test linux dist sanity_tests diff --git a/tools/voluspa/README.md b/tools/voluspa/README.md new file mode 100644 index 00000000000..f712ccc34bb --- /dev/null +++ b/tools/voluspa/README.md @@ -0,0 +1,47 @@ +# Voluspa + +### Apache Traffic Server Configuration Tool (experiment) + +A tool to codify the CDN feature set and make it easier and more reliable to setup new CDN properties. + +Configuration files are well-formed YAML files that express the desired behaviour. + + +## Usage +``` +$ ./voluspa -dest=ourconfigs myproperty.conf + +$ find ourconfigs -type f +ourconfigs/ssl_multicert.config_tenant +ourconfigs/myproperty/myproperty.config +ourconfigs/myproperty/hdrs.config +ourconfigs/ssl_multicert.config_default +ourconfigs/voluspa.sls +``` + +The voluspa configuration language is defined by a +[JSON Schema](https://spacetelescope.github.io/understanding-json-schema/index.html) +stored in the [schema file](schema_v1.json). + +Tool will generally error out if something is wrong (missing required fields) or weirdly configured options. +Warnings will be generated for unknown configuration options (but will attempt to still process the file). +There are a number of options to control the output and output location (see -help) + +By default, all configs get a receipt header and propstats enabled unless disabled via the appropriate option. + +## Extending + +### General process includes: +- adding the shared library name (sharedLibraryNames) +- configuration option name (see pluginConfigNameMap) +- weighting the plugin (for sorting) (pluginWeights) +- most importantly, the class/struct/file for the plugin +- a bit more complicated if it's a "compound plugin" (eg HeaderRewrite) + + +## Tests +There are a couple of go-based unit tests (no where near comprehensive) and there's what I would call sanity tests in tests/. The script "test.sh" will use the current build of the tool to regenerate the remap configs for a given yaml file; and do a diff between base/ and the new output. Generally, there should be no changes between runs. If there are expected changes, the baselines would need to be regenerated. The configurations tested range from very simple to something exercising all current configuration options. + + +## TODO +- error on mutually exclusive options (eg allow\_ip and deny\_methods) diff --git a/tools/voluspa/adapter_config.go b/tools/voluspa/adapter_config.go new file mode 100644 index 00000000000..1ee9c03b4c1 --- /dev/null +++ b/tools/voluspa/adapter_config.go @@ -0,0 +1,282 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package voluspa + +import ( + "bytes" + "fmt" + "sort" + "strings" +) + +type AdapterType string + +const ( + UnknownAdapter AdapterType = "unknown" + Receipt = "receipt" +) + +type ConfigLocation int + +const ( + UnknownLocation ConfigLocation = iota + ChildConfig + ParentConfig +) + +// validConfigrationOptions are options that are not plugin specific (which are managed by the plugin) +var validConfigurationOptions = map[string]bool{ + "parent_child": true, + Parent: true, + Child: true, +} + +type mappingRule struct { + Type AdapterType + Weight int + SubConfigFileName string + SubConfigContent *bytes.Buffer + ConfigContent *bytes.Buffer +} + +func newMappingRule(vp Adapter) *mappingRule { + return &mappingRule{Type: vp.Type(), Weight: getWeight(vp)} +} + +// hasContent returns true if there's configuration content, false otherwise +func (p *mappingRule) hasContent() bool { + return (p.SubConfigContent != nil && p.SubConfigContent.Len() > 0) || + (p.ConfigContent != nil && p.ConfigContent.Len() > 0) +} + +func (p *mappingRule) isCommandLineTemplated() bool { + return p.ConfigContent != nil && strings.Contains(p.ConfigContent.String(), "{%") +} + +// pluginWeights affects the ordering of the configuration output. +// Only add to this list if you have hard ordering requirements, otherwise just stick +// with the default weighting (5) +func getWeight(vp Adapter) int { + t, ok := vp.(Weighty) + if !ok { + return 5 + } + return t.Weight() +} + +// SubConfigFilename returns the subconfig filename given plugin and property name +func SubConfigFilename(p SubConfigAdapter, env *ConfigEnvironment) string { + extension := "config" + name := p.Name() + if strings.HasSuffix(p.Name(), ".lua") { + extension = "lua" + name = strings.TrimSuffix(name, ".lua") + } + if env.id == 1 { + if env.ConfigLocation == ParentConfig { + return fmt.Sprintf("%s_parent.%s", name, extension) + } + return fmt.Sprintf("%s.%s", name, extension) + } + + if env.ConfigLocation == ParentConfig { + return fmt.Sprintf("%s%d_parent.%s", name, env.id, extension) + } + return fmt.Sprintf("%s%d.%s", name, env.id, extension) +} + +// ConfigFileNameSpecification appends to buffer the sub config filename configuration specification +// nolint: interfacer +func ConfigFileNameSpecification(p Adapter, env *ConfigEnvironment, buffer *bytes.Buffer) (*bytes.Buffer, error) { + var configFileSwitch string + cfs, ok := p.(ConfigFileSwitch) + if ok { + configFileSwitch = cfs.ConfigFileSwitch() + } + + scp, ok := p.(SubConfigAdapter) + if ok { + fmt.Fprintf(buffer, " @pparam=%s%s/%s", configFileSwitch, strings.ToLower(env.Property), SubConfigFilename(scp, env)) + } + return buffer, nil +} + +func isParameterBasedAdapter(p Adapter) bool { + _, ok := p.(ParameterAdapter) + if ok { + return true + } + + _, ok = p.(RoleifiedParameterAdapter) + + return ok +} + +func ConfigContent(p Adapter, env *ConfigEnvironment) (*bytes.Buffer, error) { + + var buffer bytes.Buffer + + var roleEnabled bool + rep, ok := p.(RoleEnabled) + if ok { + roleEnabled = true + fmt.Fprintf(&buffer, "{%% if salt.pillar.get(\"%s\") %%}%s", rep.Role(), remapPrefixBuffer) + } + + sla, ok := p.(SharedLibraryAdapter) + if ok { + fmt.Fprintf(&buffer, "@plugin=%s", sla.SharedLibraryName()) + } + + if !isParameterBasedAdapter(p) { + return ConfigFileNameSpecification(p, env, &buffer) + } + + var err error + var subfer *bytes.Buffer + switch vpp := p.(type) { + case ParameterAdapter: + subfer, err = processParameterAdapter(p, env, vpp) + case RoleifiedParameterAdapter: + subfer, err = processRoleifiedParameterAdapter(vpp, env) + default: + return nil, fmt.Errorf("unhandled adapter %s", vpp) + } + + if err != nil { + return nil, err + } + + // if sub-buffer is nil, pass back nil + if subfer == nil { + return nil, nil + } + + buffer.Write(subfer.Bytes()) + + if roleEnabled { + fmt.Fprintf(&buffer, " \\{%% else %%}%s\\{%% endif %%}", remapPrefixBuffer) + fmt.Fprintln(&buffer, "") + } + + return &buffer, nil +} + +func processParameterAdapter(p Adapter, env *ConfigEnvironment, vpp ParameterAdapter) (*bytes.Buffer, error) { + var buffer bytes.Buffer + + // throw away all work if we get an error (which can be a signal not to include this plugin (see propstats)) + // TODO: signal with appropriate error (like ErrSkipConfig) + pparams, err := vpp.PParams(env) + if err != nil { + return &buffer, err + } + + raw := false + rpparams, ok := vpp.(RawParameterAdapter) + if ok { + raw = rpparams.Raw() + } + + if len(pparams) == 0 { + if p.PluginType() == CompoundAdapter { + return &buffer, nil + } + // no buffer to append? + return nil, nil + } + + if raw { + for _, val := range pparams { + fmt.Fprintf(&buffer, " %s", val) + } + } else { + for _, val := range pparams { + fmt.Fprintf(&buffer, " @pparam=%s", val) + } + } + + return &buffer, nil +} + +func processRoleifiedParameterAdapter(vpp RoleifiedParameterAdapter, env *ConfigEnvironment) (*bytes.Buffer, error) { + var buffer bytes.Buffer + pparams, err := vpp.PParams(env) + if err != nil { + return &buffer, err + } + + // extract non-Default role keys from parameter list + var roleKeys []string + for key := range pparams { + if key == DefaultRole { + continue + } + roleKeys = append(roleKeys, key) + } + sort.Strings(roleKeys) + + // 2 passes + // 1. shove the pparams and role-guards into a list + // 2. iterate over that listen, building up buffer with the \n and \\ and whitespace + + var args []string + pparam, found := pparams[DefaultRole] + if found { + for _, val := range pparam { + args = append(args, fmt.Sprintf("@pparam=%s", val)) + } + } + + for _, key := range roleKeys { + args = append(args, fmt.Sprintf("{%% if salt.pillar.get(\"%s\") %%}%s", key, remapPrefixBuffer)) + + for _, val := range pparams[key] { + args = append(args, fmt.Sprintf("@pparam=%s", val)) + } + args = append(args, fmt.Sprintf(" \\{%% else %%}%s\\{%% endif %%}", remapPrefixBuffer)) + } + + // peel off last arg for special handling + lastArg := args[len(args)-1] + args = args[:len(args)-1] + + for _, arg := range args { + fmt.Fprintf(&buffer, " %s", arg) + } + + // TODO: assuming last element is {% endif %} block + fmt.Fprintf(&buffer, "%s\n", lastArg) + + return &buffer, nil +} + +func SubConfigContent(p Adapter, env *ConfigEnvironment) (*bytes.Buffer, error) { + cba, ok := p.(ContentBasedAdapter) + if ok { + return cba.Content(env) + } + return nil, nil +} + +// IsValidConfigurationOption returns true if configuration key is valid +func IsValidConfigurationOption(option string) bool { + return validConfigurationOptions[option] +} diff --git a/tools/voluspa/adapters.go b/tools/voluspa/adapters.go new file mode 100644 index 00000000000..b705011a596 --- /dev/null +++ b/tools/voluspa/adapters.go @@ -0,0 +1,115 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package voluspa + +import "bytes" + +type PluginType int + +const ( + UnknownVoluspaAdapter PluginType = iota + GeneralAdapter + ActionAdapter + CompoundAdapter +) + +type ConfigEnvironment struct { + Property string + Options Options + RemapOptions RemapOptions + ConfigLocation ConfigLocation + id int +} + +func newConfigEnvironment(p *PropertyConfig, options Options, mappingID int, configLocation ConfigLocation) *ConfigEnvironment { + return &ConfigEnvironment{ + Property: p.Property, + Options: options, + ConfigLocation: configLocation, + id: mappingID, + } +} + +// Adapter needs to be implemented by all adapters +type Adapter interface { + PluginType() PluginType + Type() AdapterType + ConfigParameters() []string +} + +// ContentBasedAdapter should be implemented if a configuration is a blob of text (opposite of ParameterAdapter) +type ContentBasedAdapter interface { + Content(*ConfigEnvironment) (*bytes.Buffer, error) +} + +// SharedLibraryAdapter should be implemented if a configuration has an associated shared library +type SharedLibraryAdapter interface { + SharedLibraryName() string +} + +// CompoundTypeAdapter should be implemented if an adapter's output is shared with other like adapters +type CompoundTypeAdapter interface { + CompoundType() AdapterType +} + +// ParameterAdapter should be implemented if a configuration has @pparams +type ParameterAdapter interface { + PParams(*ConfigEnvironment) ([]string, error) +} + +// RawParameterAdapter should be implemented if a configuration has @pparams that should not be processed +type RawParameterAdapter interface { + Raw() bool +} + +// RolePParams is a map of role names to a list of PParam options +type RolePParams map[string][]string + +// RoleifiedParameterAdapter should be implemented in adapters +// that can vary their options based on host role/type +type RoleifiedParameterAdapter interface { + PParams(*ConfigEnvironment) (RolePParams, error) +} + +// SubConfigAdapter should be implemented if a sub config is involved with rule/plugin +type SubConfigAdapter interface { + Name() string +} + +// ConfigFileSwitch should be implemented if a sub config is specified with a plugin switch (eg --config=) +type ConfigFileSwitch interface { + ConfigFileSwitch() string +} + +// ParentChildAdapter should be implemented if plugin is conditionally placed based on parent/child context +type ParentChildAdapter interface { + ConfigLocations() []ConfigLocation +} + +// RoleEnabled should be implemented if command-line should be protected/enabled by a jinja role-block +type RoleEnabled interface { + Role() string +} + +// Weighty should be implemented if an Adapter's output needs to be placed in a certain +// order (in relation to other adapter's output) +type Weighty interface { + Weight() int +} diff --git a/tools/voluspa/adapters/access_approval.go b/tools/voluspa/adapters/access_approval.go new file mode 100644 index 00000000000..b49b000ecae --- /dev/null +++ b/tools/voluspa/adapters/access_approval.go @@ -0,0 +1,83 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package adapters + +import ( + "strings" + + "github.com/apache/trafficserver/tools/voluspa" + "github.com/apache/trafficserver/tools/voluspa/adapters/util" +) + +const ( + AccessApprovalAdapterParameter = "access_approval" +) + +func init() { + voluspa.AdaptersRegistry.AddAdapter(&AccessApprovalAdapter{}) +} + +type AccessApprovalAdapter struct { +} + +func (*AccessApprovalAdapter) Weight() int { + return 1 +} + +func (p *AccessApprovalAdapter) PluginType() voluspa.PluginType { + return voluspa.GeneralAdapter +} + +func (p *AccessApprovalAdapter) Type() voluspa.AdapterType { + return voluspa.AdapterType("access_approval") +} + +func (p *AccessApprovalAdapter) Name() string { + return "access_approval" +} + +func (p *AccessApprovalAdapter) ConfigParameters() []string { + return []string{AccessApprovalAdapterParameter} +} + +func (p *AccessApprovalAdapter) SharedLibraryName() string { + return "access_control.so" +} + +func (p *AccessApprovalAdapter) PParams(env *voluspa.ConfigEnvironment) ([]string, error) { + alt, err := env.RemapOptions.ValueByNameAsStringMapInterface(AccessApprovalAdapterParameter) + if err != nil { + return nil, util.FormatError(AccessApprovalAdapterParameter, err) + } + + val, ok := alt["raw"] + if !ok { + return nil, nil + } + + var pparams []string + pparams = append(pparams, strings.Split(val.(string), "\n")...) + + return pparams, nil +} + +func (p *AccessApprovalAdapter) Raw() bool { + return true +} diff --git a/tools/voluspa/adapters/allow_ip.go b/tools/voluspa/adapters/allow_ip.go new file mode 100644 index 00000000000..f7dbf5abdf3 --- /dev/null +++ b/tools/voluspa/adapters/allow_ip.go @@ -0,0 +1,86 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package adapters + +import ( + "bytes" + "fmt" + + "github.com/apache/trafficserver/tools/voluspa" + "github.com/apache/trafficserver/tools/voluspa/adapters/util" +) + +const ( + allowIPAdapterParameter = "allow_ip" + allowIPAdapterParameterParent = "allow_ip_parent" +) + +func init() { + voluspa.AdaptersRegistry.AddAdapter(&AllowIPAdapter{}) +} + +type AllowIPAdapter struct { +} + +func (*AllowIPAdapter) Weight() int { + return 120 +} +func (p *AllowIPAdapter) PluginType() voluspa.PluginType { + return voluspa.ActionAdapter +} + +func (p *AllowIPAdapter) Type() voluspa.AdapterType { + return voluspa.AdapterType("allow_ip") +} + +func (p *AllowIPAdapter) ConfigParameters() []string { + return []string{allowIPAdapterParameter, allowIPAdapterParameterParent} +} + +func (p *AllowIPAdapter) Content(env *voluspa.ConfigEnvironment) (*bytes.Buffer, error) { + switch env.ConfigLocation { + case voluspa.UnknownLocation, voluspa.ChildConfig: + return p.configContent(env, allowIPAdapterParameter) + case voluspa.ParentConfig: + return p.configContent(env, allowIPAdapterParameterParent) + } + return nil, fmt.Errorf("Unhandled configLocation %q", env.ConfigLocation) +} + +func (p *AllowIPAdapter) configContent(env *voluspa.ConfigEnvironment, parameterName string) (*bytes.Buffer, error) { + ips, err := env.RemapOptions.ValueByNameAsSlice(parameterName) + if err != nil { + return nil, nil + } + + if len(ips) == 0 { + return nil, util.FormatError(parameterName, fmt.Errorf("empty IP range speclist")) + } + + // TODO other validation on actual content + + content := &bytes.Buffer{} + for _, ip := range ips { + fmt.Fprintf(content, "@src_ip=%s ", ip) + } + content.WriteString("@action=allow") + + return content, nil +} diff --git a/tools/voluspa/adapters/allow_ip_test.go b/tools/voluspa/adapters/allow_ip_test.go new file mode 100644 index 00000000000..736d81b8938 --- /dev/null +++ b/tools/voluspa/adapters/allow_ip_test.go @@ -0,0 +1,85 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package adapters + +import ( + "testing" + + "github.com/apache/trafficserver/tools/voluspa" +) + +var ( + childIPRange = []string{"17.0.0.0-17.255.255.255", "10.0.0.0-10.255.255.255"} + parentIPRange = []string{"17.253.0.0-17.255.255.255", "10.253.0.0-10.255.255.255"} +) + +func TestIPAllowAdapter(t *testing.T) { + var plugin AllowIPAdapter + ro := &voluspa.RemapOptions{} + env := &voluspa.ConfigEnvironment{RemapOptions: *ro} + + (*ro)[allowIPAdapterParameter] = "" + out, err := plugin.Content(env) + if err == nil && out != nil { + t.Fatalf("Expected error for invalid configuration.") + } + + (*ro)[allowIPAdapterParameter] = []string{} + _, err = plugin.Content(env) + if err == nil { + t.Fatalf("Expected error for invalid configuration.") + } + + (*ro)[allowIPAdapterParameter] = childIPRange + out, err = plugin.Content(env) + if err != nil { + t.Fatalf("Unexpected error. err=%s", err) + } + + expected := "@src_ip=17.0.0.0-17.255.255.255 @src_ip=10.0.0.0-10.255.255.255 @action=allow" + if out.String() != expected { + t.Fatalf("Error: got '%s'; expected '%s'", out, expected) + } + + // "child" config. should still return child version + (*ro)[allowIPAdapterParameterParent] = parentIPRange + out, err = plugin.Content(env) + if err != nil { + t.Fatalf("Unexpected error. err=%s", err) + } + + expected = "@src_ip=17.0.0.0-17.255.255.255 @src_ip=10.0.0.0-10.255.255.255 @action=allow" + if out.String() != expected { + t.Fatalf("Error: got '%s'; expected '%s'", out, expected) + } + + ro = &voluspa.RemapOptions{} + env = &voluspa.ConfigEnvironment{RemapOptions: *ro, ConfigLocation: voluspa.ParentConfig} + (*ro)[allowIPAdapterParameterParent] = []string{"17.0.0.0-17.255.255.255", "10.0.0.0-10.255.255.255"} + out, err = plugin.Content(env) + if err != nil { + t.Fatalf("Unexpected error. err=%s", err) + } + + expected = "@src_ip=17.0.0.0-17.255.255.255 @src_ip=10.0.0.0-10.255.255.255 @action=allow" + if out.String() != expected { + t.Fatalf("Error: got '%s'; expected '%s'", out, expected) + } +} diff --git a/tools/voluspa/adapters/allow_methods.go b/tools/voluspa/adapters/allow_methods.go new file mode 100644 index 00000000000..02ff33a92f6 --- /dev/null +++ b/tools/voluspa/adapters/allow_methods.go @@ -0,0 +1,65 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package adapters + +import ( + "bytes" + "fmt" + + "github.com/apache/trafficserver/tools/voluspa" +) + +func init() { + voluspa.AdaptersRegistry.AddAdapter(&AllowMethodsAdapter{}) +} + +type AllowMethodsAdapter struct { +} + +func (p *AllowMethodsAdapter) Weight() int { + return 140 +} + +func (p *AllowMethodsAdapter) PluginType() voluspa.PluginType { + return voluspa.ActionAdapter +} + +func (p *AllowMethodsAdapter) Type() voluspa.AdapterType { + return voluspa.AdapterType("allow_methods") +} + +func (p *AllowMethodsAdapter) ConfigParameters() []string { + return []string{"allow_methods"} +} + +func (p *AllowMethodsAdapter) Content(env *voluspa.ConfigEnvironment) (*bytes.Buffer, error) { + methods, err := env.RemapOptions.ValueByNameAsSlice("allow_methods") + if err != nil { + return nil, err + } + + content := &bytes.Buffer{} + content.WriteString("@action=allow") + for _, method := range methods { + fmt.Fprintf(content, " @method=%s", method) + } + + return content, nil +} diff --git a/tools/voluspa/adapters/auth_proxy.go b/tools/voluspa/adapters/auth_proxy.go new file mode 100644 index 00000000000..3d9d4be4736 --- /dev/null +++ b/tools/voluspa/adapters/auth_proxy.go @@ -0,0 +1,67 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package adapters + +import ( + "errors" + "fmt" + + "github.com/apache/trafficserver/tools/voluspa" +) + +func init() { + voluspa.AdaptersRegistry.AddAdapter(&AuthProxyAdapter{}) +} + +type AuthProxyAdapter struct { +} + +func (p *AuthProxyAdapter) PluginType() voluspa.PluginType { + return voluspa.GeneralAdapter +} + +func (p *AuthProxyAdapter) Type() voluspa.AdapterType { + return voluspa.AdapterType("auth_proxy") +} + +func (p *AuthProxyAdapter) Name() string { + return "authproxy" +} + +func (p *AuthProxyAdapter) ConfigParameters() []string { + return []string{"authproxy"} +} + +func (p *AuthProxyAdapter) SharedLibraryName() string { + return "authproxy.so" +} + +func (p *AuthProxyAdapter) ConfigLocations() []voluspa.ConfigLocation { + return []voluspa.ConfigLocation{voluspa.ChildConfig} +} + +func (p *AuthProxyAdapter) PParams(env *voluspa.ConfigEnvironment) ([]string, error) { + value, err := env.RemapOptions.ValueByNameAsString(p.Name()) + if err != nil && len(value) == 0 { + return nil, errors.New("Need transform parameter") + } + + return []string{fmt.Sprintf("--auth-transform=%s", value)}, nil +} diff --git a/tools/voluspa/adapters/cache_promote.go b/tools/voluspa/adapters/cache_promote.go new file mode 100644 index 00000000000..16b38f05690 --- /dev/null +++ b/tools/voluspa/adapters/cache_promote.go @@ -0,0 +1,134 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package adapters + +import ( + "fmt" + + "github.com/apache/trafficserver/tools/voluspa" + "github.com/apache/trafficserver/tools/voluspa/adapters/util" +) + +const ( + CachePromoteAdapterParameter = "cache_promote" + CachePromoteAdapterPolicyParameter = "policy" +) + +func init() { + voluspa.AdaptersRegistry.AddAdapter(&CachePromoteAdapter{}) +} + +type CachePromoteAdapter struct { +} + +func (p *CachePromoteAdapter) PluginType() voluspa.PluginType { + return voluspa.GeneralAdapter +} + +func (p *CachePromoteAdapter) Type() voluspa.AdapterType { + return voluspa.AdapterType("cache_promote") +} + +func (p *CachePromoteAdapter) Name() string { + return "cache_promote" +} + +func (p *CachePromoteAdapter) SharedLibraryName() string { + return "cache_promote.so" +} + +func (p *CachePromoteAdapter) Role() string { + return "roles_trafficserver_cache_promote" +} + +func (p *CachePromoteAdapter) ConfigLocations() []voluspa.ConfigLocation { + return []voluspa.ConfigLocation{voluspa.ChildConfig} +} + +func (p *CachePromoteAdapter) ConfigParameters() []string { + return []string{CachePromoteAdapterParameter} +} + +func (p *CachePromoteAdapter) isValidPolicy(policy string) bool { + return policy == "lru" || policy == "chance" +} + +func (p *CachePromoteAdapter) PParams(env *voluspa.ConfigEnvironment) ([]string, error) { + + var alt map[string]interface{} + alt, err := env.RemapOptions.ValueByNameAsStringMapInterface(CachePromoteAdapterParameter) + if err != nil { + return nil, err + } + + if err = p.validateArgs(alt); err != nil { + return nil, err + } + + args, err := p.processArgs(alt) + if err != nil { + return nil, err + } + + return args, nil +} + +func (p *CachePromoteAdapter) validateArgs(alt map[string]interface{}) error { + val := util.ParamToValue(alt["policy"]) + if len(val) == 0 || !p.isValidPolicy(val) { + return util.FormatError(CachePromoteAdapterPolicyParameter, fmt.Errorf("invalid policy specified")) + } + + return nil +} + +func (p *CachePromoteAdapter) processArgs(alt map[string]interface{}) ([]string, error) { + var args []string + args = p.maybeAddArgs(alt["policy"], "policy", args) + args = p.maybeAddArgs(alt["sample"], "sample", args) + + policy := util.ParamToValue(alt["policy"]) + if policy != "lru" { + return args, nil + } + args = p.maybeAddArgs(alt["lru_hits"], "hits", args) + args = p.maybeAddArgs(alt["lru_buckets"], "buckets", args) + + return args, nil +} + +func (p *CachePromoteAdapter) maybeAddArgs(option interface{}, param string, args []string) []string { + value := util.ParamToValue(option) + switch param { + case "policy": + if p.isValidPolicy(value) { + return append(args, fmt.Sprintf("--%s=%s", param, value)) + } + case "sample": + if len(value) > 0 { + return append(args, fmt.Sprintf("--%s=%s%%", param, value)) + } + default: + if len(value) > 0 { + return append(args, fmt.Sprintf("--%s=%s", param, value)) + } + } + return args +} diff --git a/tools/voluspa/adapters/cache_promote_test.go b/tools/voluspa/adapters/cache_promote_test.go new file mode 100644 index 00000000000..99d3e322596 --- /dev/null +++ b/tools/voluspa/adapters/cache_promote_test.go @@ -0,0 +1,107 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package adapters + +import ( + "strings" + "testing" + + "github.com/apache/trafficserver/tools/voluspa" +) + +func TestCachePromoteAdapter(t *testing.T) { + var plugin CachePromoteAdapter + ro := &voluspa.RemapOptions{} + subOptions := make(map[interface{}]interface{}) + (*ro)["cache_promote"] = subOptions + + env := &voluspa.ConfigEnvironment{RemapOptions: *ro} + + _, err := plugin.PParams(env) + if err == nil { + t.Fatalf("Expected error for invalid configuration") + } + + subOptions["policy"] = "notapolicy" + _, err = plugin.PParams(env) + if err == nil { + t.Fatalf("Expected error for invalid configuration") + } + + subOptions["policy"] = "lru" + out, err := plugin.PParams(env) + if err != nil { + t.Fatalf("Unexpected error: %s", err) + } + + expected := "--policy=lru" + if out[0] != expected { + t.Fatalf("Error: got '%s'; expected '%s'", out[0], expected) + } + + subOptions["sample"] = 66 + out, err = plugin.PParams(env) + if err != nil { + t.Fatalf("Unexpected error: %s", err) + } + + joined := strings.Join(out, " ") + expected = "--policy=lru --sample=66%" + if joined != expected { + t.Fatalf("Error: got %q; expected %q", joined, expected) + } + + subOptions["lru_hits"] = 1000 + out, err = plugin.PParams(env) + if err != nil { + t.Fatalf("Unexpected error: %s", err) + } + + joined = strings.Join(out, " ") + expected = "--policy=lru --sample=66% --hits=1000" + if joined != expected { + t.Fatalf("Error: got '%s'; expected '%s'", joined, expected) + } + + subOptions["lru_buckets"] = 10000 + out, err = plugin.PParams(env) + if err != nil { + t.Fatalf("Unexpected error: %s", err) + } + + joined = strings.Join(out, " ") + expected = "--policy=lru --sample=66% --hits=1000 --buckets=10000" + if joined != expected { + t.Fatalf("Error: got %q; expected %q", joined, expected) + } + + // Non-LRU policy's should drop hits and buckets from command-line + subOptions["policy"] = "chance" + out, err = plugin.PParams(env) + if err != nil { + t.Fatalf("Unexpected error: %s", err) + } + + joined = strings.Join(out, " ") + expected = "--policy=chance --sample=66%" + if joined != expected { + t.Fatalf("Error: got %q; expected %q", joined, expected) + } +} diff --git a/tools/voluspa/adapters/cachekey.go b/tools/voluspa/adapters/cachekey.go new file mode 100644 index 00000000000..a349dcd5e19 --- /dev/null +++ b/tools/voluspa/adapters/cachekey.go @@ -0,0 +1,125 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package adapters + +import ( + "fmt" + "strconv" + "strings" + + "github.com/apache/trafficserver/tools/voluspa" +) + +const ( + CacheKeyAdapterParameter = "cachekey" +) + +func init() { + voluspa.AdaptersRegistry.AddAdapter(&CacheKeyAdapter{}) +} + +type CacheKeyAdapter struct { +} + +func (p *CacheKeyAdapter) PluginType() voluspa.PluginType { + return voluspa.GeneralAdapter +} + +func (p *CacheKeyAdapter) Type() voluspa.AdapterType { + return voluspa.AdapterType("cache_key") +} + +func (p *CacheKeyAdapter) SharedLibraryName() string { + return "cachekey.so" +} + +func (p *CacheKeyAdapter) ConfigParameters() []string { + return []string{CacheKeyAdapterParameter} +} + +func (p *CacheKeyAdapter) PParams(env *voluspa.ConfigEnvironment) ([]string, error) { + alt, err := env.RemapOptions.ValueByNameAsStringMapInterface(CacheKeyAdapterParameter) + if err != nil { + return nil, err + } + + args, err := p.processArgs(alt) + if err != nil { + return nil, err + } + + return args, nil +} + +func (p *CacheKeyAdapter) processArgs(alt map[string]interface{}) ([]string, error) { + var args []string + args = p.maybeAddArgs(alt["include_params"], "include-params", args) + args = p.maybeAddArgs(alt["include_cookies"], "include-cookies", args) + args = p.maybeAddArgs(alt["include_headers"], "include-headers", args) + args = p.maybeAddArgs(alt["exclude_params"], "exclude-params", args) + args = p.maybeAddArgs(alt["static_prefix"], "static-prefix", args) + args = p.maybeAddArgs(alt["sort_query"], "sort-params", args) + args = p.maybeAddArgs(alt["remove_all_params"], "remove-all-params", args) + args = p.maybeAddArgs(alt["regex_replace_path"], "capture-path", args) + args = p.maybeAddArgs(alt["regex_replace_path_uri"], "capture-path-uri", args) + args = p.maybeAddArgs(alt["capture_header"], "capture-header", args) + + return args, nil +} + +func (p *CacheKeyAdapter) paramToValue(i interface{}) string { + switch val := i.(type) { + case string: + return val + case int: + return strconv.Itoa(val) + case bool: + return strconv.FormatBool(val) + case []string: + return strings.Join(val, ",") + case []interface{}: + var vals []string + for _, v1 := range i.([]interface{}) { + vS := v1.(string) + vals = append(vals, vS) + } + return strings.Join(vals, ",") + default: + return "" + } +} + +func (p *CacheKeyAdapter) maybeAddArgs(option interface{}, param string, args []string) []string { + + // capture-header needs a separate "--capture-header=" flag for each one when there are multiple + if param == "capture-header" && option != nil { + for _, o := range option.([]interface{}) { + args = append(args, "--capture-header="+o.(string)) + } + return args + } + + value := p.paramToValue(option) + if len(value) > 0 { + return append(args, fmt.Sprintf("--%s=%s", param, value)) + } + + return args +} diff --git a/tools/voluspa/adapters/confremap/base_conf_remap.go b/tools/voluspa/adapters/confremap/base_conf_remap.go new file mode 100644 index 00000000000..4f5a61cac55 --- /dev/null +++ b/tools/voluspa/adapters/confremap/base_conf_remap.go @@ -0,0 +1,63 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package confremap + +import ( + "github.com/apache/trafficserver/tools/voluspa" +) + +func init() { + voluspa.AdaptersRegistry.AddAdapter(&BaseConfRemapAdapter{}) +} + +type BaseConfRemapAdapter struct { +} + +func (*BaseConfRemapAdapter) Weight() int { + return 17 +} + +func (p *BaseConfRemapAdapter) PluginType() voluspa.PluginType { + return voluspa.CompoundAdapter +} + +func (p *BaseConfRemapAdapter) Type() voluspa.AdapterType { + return adapterType +} + +func (p *BaseConfRemapAdapter) CompoundType() voluspa.AdapterType { + return adapterType +} + +func (p *BaseConfRemapAdapter) Name() string { + return "base_conf_remap" +} + +func (p *BaseConfRemapAdapter) SharedLibraryName() string { + return "conf_remap.so" +} + +func (p *BaseConfRemapAdapter) ConfigParameters() []string { + return []string{""} +} + +func (p *BaseConfRemapAdapter) PParams(env *voluspa.ConfigEnvironment) ([]string, error) { + return nil, nil +} diff --git a/tools/voluspa/adapters/confremap/cache_version.go b/tools/voluspa/adapters/confremap/cache_version.go new file mode 100644 index 00000000000..4f0802490ca --- /dev/null +++ b/tools/voluspa/adapters/confremap/cache_version.go @@ -0,0 +1,60 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package confremap + +import ( + "fmt" + + "github.com/apache/trafficserver/tools/voluspa" +) + +func init() { + voluspa.AdaptersRegistry.AddAdapter(&CacheVersionAdapter{}) +} + +type CacheVersionAdapter struct { +} + +func (s *CacheVersionAdapter) PluginType() voluspa.PluginType { + return voluspa.CompoundAdapter +} + +func (s *CacheVersionAdapter) Type() voluspa.AdapterType { + return voluspa.AdapterType("cache_version") +} + +func (s *CacheVersionAdapter) CompoundType() voluspa.AdapterType { + return adapterType +} + +func (s *CacheVersionAdapter) ConfigParameters() []string { + return []string{"cache_version"} +} + +func (s *CacheVersionAdapter) PParams(env *voluspa.ConfigEnvironment) ([]string, error) { + cacheVersion, err := env.RemapOptions.ValueByNameAsInt("cache_version") + if err != nil { + return nil, err + } + + return []string{ + fmt.Sprintf("proxy.config.http.cache.generation=%d", cacheVersion), + }, nil +} diff --git a/tools/voluspa/adapters/confremap/common.go b/tools/voluspa/adapters/confremap/common.go new file mode 100644 index 00000000000..6f64559ac12 --- /dev/null +++ b/tools/voluspa/adapters/confremap/common.go @@ -0,0 +1,24 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package confremap + +import "github.com/apache/trafficserver/tools/voluspa" + +var adapterType = voluspa.AdapterType("conf_remap") diff --git a/tools/voluspa/adapters/confremap/conf_remap.go b/tools/voluspa/adapters/confremap/conf_remap.go new file mode 100644 index 00000000000..a687141e5c2 --- /dev/null +++ b/tools/voluspa/adapters/confremap/conf_remap.go @@ -0,0 +1,107 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package confremap + +import ( + "fmt" + "regexp" + "strconv" + "strings" + + "github.com/apache/trafficserver/tools/voluspa" +) + +func init() { + voluspa.AdaptersRegistry.AddAdapter(&FreeformConfRemapAdapter{}) +} + +type FreeformConfRemapAdapter struct { +} + +func (*FreeformConfRemapAdapter) Weight() int { + return 17 +} + +func (p *FreeformConfRemapAdapter) PluginType() voluspa.PluginType { + return voluspa.CompoundAdapter +} + +func (p *FreeformConfRemapAdapter) Type() voluspa.AdapterType { + return voluspa.AdapterType("raw_conf_remap") +} + +func (p *FreeformConfRemapAdapter) CompoundType() voluspa.AdapterType { + return adapterType +} + +func (p *FreeformConfRemapAdapter) ConfigParameters() []string { + return []string{"conf_remap"} +} + +// CONFIG proxy.config.http.connect_attempts_timeout INT 600 +var confRegexp = regexp.MustCompile(`CONFIG (.*) ([A-Z]+) (.*)`) + +func (p *FreeformConfRemapAdapter) PParams(env *voluspa.ConfigEnvironment) ([]string, error) { + val, err := env.RemapOptions.ValueByNameAsString("conf_remap") + if err != nil { + return nil, err + } + + seen := make(map[string]bool) + + var params []string + + lines := strings.Split(val, "\n") + for _, line := range lines { + line = strings.TrimSpace(line) + if line == "" { + continue + } + + // eg CONFIG proxy.config.http.connect_attempts_timeout INT 600 + vals := confRegexp.FindStringSubmatch(line) + if vals == nil { + return nil, fmt.Errorf("%s is not properly formed", line) + } + + if _, ok := seen[line]; ok { + return nil, fmt.Errorf("Already seen key '%s'", vals[1]) + } + seen[line] = true + + switch vals[2] { + case "INT": + _, err := strconv.Atoi(vals[3]) + if err != nil { + return nil, fmt.Errorf("Value is not of appropriate type; in=%s type=%s line='%s'", vals[3], vals[2], line) + } + case "FLOAT": + _, err := strconv.ParseFloat(vals[3], 64) + if err != nil { + return nil, fmt.Errorf("Value is not of appropriate type; in=%s type=%s line='%s'", vals[3], vals[2], line) + } + case "STRING": + default: + return nil, fmt.Errorf("Type is unknown/unhandled; in=%s type=%s line='%s'", vals[3], vals[2], line) + } + params = append(params, fmt.Sprintf("%s=%s", vals[1], vals[3])) + } + return params, nil +} diff --git a/tools/voluspa/adapters/confremap/disable_cache.go b/tools/voluspa/adapters/confremap/disable_cache.go new file mode 100644 index 00000000000..06398e78476 --- /dev/null +++ b/tools/voluspa/adapters/confremap/disable_cache.go @@ -0,0 +1,51 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package confremap + +import ( + "github.com/apache/trafficserver/tools/voluspa" +) + +func init() { + voluspa.AdaptersRegistry.AddAdapter(&DisableCacheAdapter{}) +} + +type DisableCacheAdapter struct { +} + +func (p *DisableCacheAdapter) PluginType() voluspa.PluginType { + return voluspa.CompoundAdapter +} + +func (p *DisableCacheAdapter) Type() voluspa.AdapterType { + return voluspa.AdapterType("disable_cache") +} + +func (p *DisableCacheAdapter) CompoundType() voluspa.AdapterType { + return adapterType +} + +func (p *DisableCacheAdapter) ConfigParameters() []string { + return []string{"disable_cache"} +} + +func (p *DisableCacheAdapter) PParams(env *voluspa.ConfigEnvironment) ([]string, error) { + return []string{"proxy.config.http.cache.http=0"}, nil +} diff --git a/tools/voluspa/adapters/confremap/negative_caching.go b/tools/voluspa/adapters/confremap/negative_caching.go new file mode 100644 index 00000000000..9c9af2a1c49 --- /dev/null +++ b/tools/voluspa/adapters/confremap/negative_caching.go @@ -0,0 +1,68 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package confremap + +import ( + "fmt" + + "github.com/apache/trafficserver/tools/voluspa" + "github.com/apache/trafficserver/tools/voluspa/adapters/util" +) + +const ( + NegativeCachingAdapterParameterTTL = "negative_ttl" +) + +func init() { + voluspa.AdaptersRegistry.AddAdapter(&NegativeCachingAdapter{}) +} + +type NegativeCachingAdapter struct { +} + +func (p *NegativeCachingAdapter) PluginType() voluspa.PluginType { + return voluspa.CompoundAdapter +} + +func (p *NegativeCachingAdapter) Type() voluspa.AdapterType { + return voluspa.AdapterType("negative_caching") +} + +func (p *NegativeCachingAdapter) CompoundType() voluspa.AdapterType { + return adapterType +} + +func (p *NegativeCachingAdapter) ConfigParameters() []string { + return []string{NegativeCachingAdapterParameterTTL} +} + +func (p *NegativeCachingAdapter) PParams(env *voluspa.ConfigEnvironment) ([]string, error) { + duration, err := env.RemapOptions.ValueByNameAsString(NegativeCachingAdapterParameterTTL) + if err != nil { + return nil, err + } + + seconds, err := util.DurationToSeconds(duration) + if err != nil { + return nil, err + } + + return []string{fmt.Sprintf("proxy.config.http.negative_caching_lifetime=%s", seconds)}, nil +} diff --git a/tools/voluspa/adapters/confremap/preserve_host_header.go b/tools/voluspa/adapters/confremap/preserve_host_header.go new file mode 100644 index 00000000000..a2d35f0d7c3 --- /dev/null +++ b/tools/voluspa/adapters/confremap/preserve_host_header.go @@ -0,0 +1,82 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package confremap + +import ( + "fmt" + + "github.com/apache/trafficserver/tools/voluspa" + "github.com/apache/trafficserver/tools/voluspa/adapters/util" +) + +const ( + OriginHostHeaderAdapterParameter = "origin_host_header" +) + +func init() { + voluspa.AdaptersRegistry.AddAdapter(&PreserveHostHeaderAdapter{}) +} + +type PreserveHostHeaderAdapter struct { +} + +func (*PreserveHostHeaderAdapter) Weight() int { + return 25 +} + +func (p *PreserveHostHeaderAdapter) PluginType() voluspa.PluginType { + return voluspa.CompoundAdapter +} + +func (p *PreserveHostHeaderAdapter) Type() voluspa.AdapterType { + return voluspa.AdapterType("preserve_host_header") +} + +func (p *PreserveHostHeaderAdapter) CompoundType() voluspa.AdapterType { + return adapterType +} + +func (p *PreserveHostHeaderAdapter) ConfigLocations() []voluspa.ConfigLocation { + return []voluspa.ConfigLocation{voluspa.ParentConfig} +} + +func (p *PreserveHostHeaderAdapter) ConfigParameters() []string { + return []string{OriginHostHeaderAdapterParameter} +} + +func (p *PreserveHostHeaderAdapter) PParams(env *voluspa.ConfigEnvironment) ([]string, error) { + intval := 0 + if env.RemapOptions.HasOption(OriginHostHeaderAdapterParameter) { + ohhValue, err := env.RemapOptions.ValueByNameAsString(OriginHostHeaderAdapterParameter) + if err != nil { + return nil, util.FormatError(OriginHostHeaderAdapterParameter, err) + } + switch ohhValue { + case "alias": + intval = 1 + case "origin": + intval = 0 + default: + return nil, util.FormatError(OriginHostHeaderAdapterParameter, fmt.Errorf("unknown value %q", ohhValue)) + } + } + + return []string{fmt.Sprintf("proxy.config.url_remap.pristine_host_hdr=%d", intval)}, nil +} diff --git a/tools/voluspa/adapters/confremap/transaction_timeout.go b/tools/voluspa/adapters/confremap/transaction_timeout.go new file mode 100644 index 00000000000..db462cf17ae --- /dev/null +++ b/tools/voluspa/adapters/confremap/transaction_timeout.go @@ -0,0 +1,68 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package confremap + +import ( + "fmt" + + "github.com/apache/trafficserver/tools/voluspa" + "github.com/apache/trafficserver/tools/voluspa/adapters/util" +) + +func init() { + voluspa.AdaptersRegistry.AddAdapter(&TransactionTimeoutAdapter{}) +} + +type TransactionTimeoutAdapter struct { +} + +func (s *TransactionTimeoutAdapter) PluginType() voluspa.PluginType { + return voluspa.CompoundAdapter +} + +func (s *TransactionTimeoutAdapter) Type() voluspa.AdapterType { + return voluspa.AdapterType("transaction_timeout") +} + +func (s *TransactionTimeoutAdapter) CompoundType() voluspa.AdapterType { + return adapterType +} + +func (s *TransactionTimeoutAdapter) ConfigParameters() []string { + return []string{"transaction_timeout"} +} + +func (s *TransactionTimeoutAdapter) PParams(env *voluspa.ConfigEnvironment) ([]string, error) { + duration, err := env.RemapOptions.ValueByNameAsString("transaction_timeout") + if err != nil { + return nil, err + } + + seconds, err := util.DurationToSeconds(duration) + if err != nil { + return nil, err + } + + return []string{ + fmt.Sprintf("proxy.config.http.connect_attempts_timeout=%s", seconds), + fmt.Sprintf("proxy.config.http.transaction_no_activity_timeout_in=%s", seconds), + fmt.Sprintf("proxy.config.http.transaction_no_activity_timeout_out=%s", seconds), + }, nil +} diff --git a/tools/voluspa/adapters/deny_methods.go b/tools/voluspa/adapters/deny_methods.go new file mode 100644 index 00000000000..580447dd67f --- /dev/null +++ b/tools/voluspa/adapters/deny_methods.go @@ -0,0 +1,69 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package adapters + +import ( + "bytes" + "fmt" + + "github.com/apache/trafficserver/tools/voluspa" +) + +func init() { + voluspa.AdaptersRegistry.AddAdapter(&DenyMethodsAdapter{}) +} + +type DenyMethodsAdapter struct { +} + +func (*DenyMethodsAdapter) Weight() int { + return 150 +} + +func (p *DenyMethodsAdapter) PluginType() voluspa.PluginType { + return voluspa.ActionAdapter +} + +func (p *DenyMethodsAdapter) Type() voluspa.AdapterType { + return voluspa.AdapterType("deny_methods") +} + +func (p *DenyMethodsAdapter) ConfigLocations() []voluspa.ConfigLocation { + return []voluspa.ConfigLocation{voluspa.ChildConfig} +} + +func (p *DenyMethodsAdapter) ConfigParameters() []string { + return []string{"deny_methods"} +} + +func (p *DenyMethodsAdapter) Content(env *voluspa.ConfigEnvironment) (*bytes.Buffer, error) { + methods, err := env.RemapOptions.ValueByNameAsSlice("deny_methods") + if err != nil { + return nil, err + } + + content := &bytes.Buffer{} + content.WriteString("@action=deny") + for _, method := range methods { + fmt.Fprintf(content, " @method=%s", method) + } + + return content, nil +} diff --git a/tools/voluspa/adapters/escalate.go b/tools/voluspa/adapters/escalate.go new file mode 100644 index 00000000000..d4e0997d07a --- /dev/null +++ b/tools/voluspa/adapters/escalate.go @@ -0,0 +1,150 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package adapters + +import ( + "fmt" + "strconv" + "strings" + + "github.com/apache/trafficserver/tools/voluspa" +) + +const ( + EscalateAdapterFailoverParameter = "failover" +) + +func init() { + voluspa.AdaptersRegistry.AddAdapter(&EscalateAdapter{}) +} + +type EscalateAdapter struct { +} + +func (p *EscalateAdapter) PluginType() voluspa.PluginType { + return voluspa.GeneralAdapter +} + +func (p *EscalateAdapter) Type() voluspa.AdapterType { + return voluspa.AdapterType("escalate") +} + +func (p *EscalateAdapter) Name() string { + return "escalate" +} + +func (p *EscalateAdapter) SharedLibraryName() string { + return "escalate.so" +} + +func (p *EscalateAdapter) ConfigParameters() []string { + return []string{EscalateAdapterFailoverParameter} +} + +func (p *EscalateAdapter) ConfigLocations() []voluspa.ConfigLocation { + return []voluspa.ConfigLocation{voluspa.ParentConfig} +} + +func (p *EscalateAdapter) PParams(env *voluspa.ConfigEnvironment) ([]string, error) { + alt, err := env.RemapOptions.ValueByNameAsStringMapInterface(EscalateAdapterFailoverParameter) + if err != nil { + return nil, err + } + + if err = p.validateArgs(alt); err != nil { + return nil, err + } + + args, err := p.processArgs(alt) + if err != nil { + return nil, err + } + + return args, nil +} + +func (p *EscalateAdapter) validateArgs(alt map[string]interface{}) error { + domain := p.paramToValue(alt["domain"]) + statusCodes := p.paramToValue(alt["status_codes"]) + if len(domain) == 0 || len(statusCodes) == 0 { + return fmt.Errorf("%q and %q are required parameters", "domain", "status_codes") + } + + value := p.paramToValue(alt["host_header"]) + if len(value) > 0 && (value != "alias" && value != "origin") { + return fmt.Errorf("invalid 'host_header' option: %q", value) + } + + return nil +} + +func convertStatusCodes(in interface{}) []string { + switch val := in.(type) { + case []string: + return val + case []interface{}: + var vals []string + for _, v1 := range in.([]interface{}) { + vS := strconv.Itoa(v1.(int)) + vals = append(vals, vS) + } + return vals + } + return nil +} + +func (p *EscalateAdapter) processArgs(alt map[string]interface{}) ([]string, error) { + var args []string + + { + value := p.paramToValue(alt["host_header"]) + if len(value) > 0 && value == "origin" { + args = append(args, "--pristine") + } + } + + statusCodes := convertStatusCodes(alt["status_codes"]) + domain := p.paramToValue(alt["domain"]) + + args = append(args, fmt.Sprintf("%s:%s", strings.Join(statusCodes, ","), domain)) + + return args, nil +} + +func (p *EscalateAdapter) paramToValue(i interface{}) string { + switch val := i.(type) { + case string: + return val + case bool: + return strconv.FormatBool(val) + + case []string: + return strings.Join(val, ",") + case []interface{}: + var vals []string + for _, v1 := range i.([]interface{}) { + vS := strconv.Itoa(v1.(int)) + vals = append(vals, vS) + } + return strings.Join(vals, ",") + default: + return "" + } +} diff --git a/tools/voluspa/adapters/escalate_test.go b/tools/voluspa/adapters/escalate_test.go new file mode 100644 index 00000000000..83de0032c07 --- /dev/null +++ b/tools/voluspa/adapters/escalate_test.go @@ -0,0 +1,86 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package adapters + +import ( + "testing" + + "github.com/apache/trafficserver/tools/voluspa" +) + +func TestEscalateAdapter(t *testing.T) { + var plugin EscalateAdapter + ro := &voluspa.RemapOptions{} + + subOptions := make(map[interface{}]interface{}) + (*ro)[EscalateAdapterFailoverParameter] = subOptions + + env := &voluspa.ConfigEnvironment{RemapOptions: *ro} + + _, err := plugin.PParams(env) + if err == nil { + t.Fatalf("Expected error for invalid configuration.") + } + + subOptions["domain"] = "" + _, err = plugin.PParams(env) + if err == nil { + t.Fatalf("Expected error for invalid configuration.") + } + + subOptions["domain"] = "domain.example.com" + _, err = plugin.PParams(env) + if err == nil { + t.Fatalf("Expected error for invalid configuration.") + } + + subOptions["status_codes"] = []string{} + _, err = plugin.PParams(env) + if err == nil { + t.Fatalf("Expected error for invalid configuration.") + } + + subOptions["status_codes"] = []string{"401", "403"} + out, err := plugin.PParams(env) + if err != nil { + t.Fatalf("Unexpected error: %s", err) + } + + expected := "401,403:domain.example.com" + if out[0] != expected { + t.Fatalf("Error: got %q; expected %q", out[0], expected) + } + + subOptions["host_header"] = "origin" + out, err = plugin.PParams(env) + if err != nil { + t.Fatalf("Unexpected error: %s", err) + } + + expected = "--pristine" + if out[0] != expected { + t.Fatalf("Error: got %q; expected %q", out[0], expected) + } + + expected = "401,403:domain.example.com" + if out[1] != expected { + t.Fatalf("Error: got %q; expected %q", out[1], expected) + } +} diff --git a/tools/voluspa/adapters/gzip.go b/tools/voluspa/adapters/gzip.go new file mode 100644 index 00000000000..ee486c552f0 --- /dev/null +++ b/tools/voluspa/adapters/gzip.go @@ -0,0 +1,198 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package adapters + +import ( + "bytes" + "fmt" + "sort" + "strings" + + "github.com/apache/trafficserver/tools/voluspa" +) + +const ( + GzipAdapterParameterCompressibleContentType = "static_origin_compressible_content_type" + GzipAdapterParameter = "edge_compression" + GzipAdapterParameterEnabled = "enabled" + GzipAdapterParameterAlgorithms = "algorithms" + GzipAdapterParameterRemoveAcceptEncoding = "remove_accept_encoding" + GzipAdapterParameterFlush = "flush" + GzipAdapterParameterMinimumContentLength = "minimum_content_length" +) + +var ( + defaultCompressibleTypes = []string{ + "*font*", + "*javascript", + "*json", + "*ml;*", + "*mpegURL", + "*mpegurl", + "*otf", + "*ttf", + "*type", + "*xml", + "application/eot", + "application/pkix-crl", + "application/x-httpd-cgi", + "application/x-perl", + "image/vnd.microsoft.icon", + "image/x-icon", + "text/*", + } +) + +func init() { + voluspa.AdaptersRegistry.AddAdapter(&GzipAdapter{}) +} + +var ( + GzipDefaultAlgorithms = []string{"gzip"} +) + +type GzipAdapter struct { +} + +func (p *GzipAdapter) PluginType() voluspa.PluginType { + return voluspa.GeneralAdapter +} + +func (p *GzipAdapter) Type() voluspa.AdapterType { + return voluspa.AdapterType("gzip") +} + +func (p *GzipAdapter) Name() string { + return "gzip" +} + +func (p *GzipAdapter) SharedLibraryName() string { + return "gzip.so" +} + +func (p *GzipAdapter) ConfigLocations() []voluspa.ConfigLocation { + return []voluspa.ConfigLocation{voluspa.ParentConfig} +} + +func (p *GzipAdapter) ConfigParameters() []string { + return []string{GzipAdapterParameter} +} + +func (p *GzipAdapter) Content(env *voluspa.ConfigEnvironment) (*bytes.Buffer, error) { + content := &bytes.Buffer{} + + config, err := env.RemapOptions.ValueByNameAsStringMapInterface(GzipAdapterParameter) + if err != nil { + return nil, err + } + + { + value, hasOption := config[GzipAdapterParameterEnabled] + if hasOption { + bvalue, ok := value.(bool) + if !ok { + return nil, fmt.Errorf("invalid 'enabled' value (%s): %t", value, value) + } + + if !bvalue { + return content, nil + } + } + } + + var algorithms []string + avalue, hasOption := config[GzipAdapterParameterAlgorithms] + if hasOption { + sliceValue, ok := avalue.([]interface{}) + if !ok { + return nil, fmt.Errorf("invalid value for %q: (%s) %t", GzipAdapterParameterAlgorithms, avalue, avalue) + } + + if len(sliceValue) > 0 { + for _, a := range sliceValue { + value := a.(string) + switch value { + case "gzip": + fallthrough + case "br": + algorithms = append(algorithms, value) + default: + return nil, fmt.Errorf("invalid value for %q: (%s)", GzipAdapterParameterAlgorithms, value) + } + } + } + } else { + algorithms = GzipDefaultAlgorithms + } + sort.Sort(sort.StringSlice(algorithms)) + + var compressibleTypes []string + params, found := config[GzipAdapterParameterCompressibleContentType] + if found { + typesSlice, ok := params.([]interface{}) + if !ok { + return nil, fmt.Errorf("invalid value for %q: (%s) %t", GzipAdapterParameterCompressibleContentType, params, params) + } + + for _, a := range typesSlice { + compressibleTypes = append(compressibleTypes, a.(string)) + } + } else { + compressibleTypes = defaultCompressibleTypes + } + + sort.Sort(sort.StringSlice(compressibleTypes)) + + content.WriteString(`enabled true +cache false +`) + + value, hasOption := config[GzipAdapterParameterRemoveAcceptEncoding] + if hasOption { + on, ok := value.(bool) + if ok && on { + content.WriteString("remove-accept-encoding true\n") + } + } + + value, hasOption = config[GzipAdapterParameterFlush] + if hasOption { + on, ok := value.(bool) + if ok && on { + content.WriteString("flush true\n") + } + } + + value, hasOption = config[GzipAdapterParameterMinimumContentLength] + if hasOption { + minLen, ok := value.(int) + if ok && minLen > 0 { + content.WriteString(fmt.Sprintf("minimum-content-length %d\n", minLen)) + } + } + + content.WriteString(fmt.Sprintf("supported-algorithms %s\n", strings.Join(algorithms, ","))) + + for _, param := range compressibleTypes { + content.WriteString(fmt.Sprintf("compressible-content-type %s\n", param)) + } + + return content, nil +} diff --git a/tools/voluspa/adapters/headerrewrite/add_header.go b/tools/voluspa/adapters/headerrewrite/add_header.go new file mode 100644 index 00000000000..ec5fcce7f54 --- /dev/null +++ b/tools/voluspa/adapters/headerrewrite/add_header.go @@ -0,0 +1,75 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package headerrewrite + +import ( + "bytes" + "fmt" + + "github.com/apache/trafficserver/tools/voluspa" + "github.com/apache/trafficserver/tools/voluspa/adapters/util" +) + +const ( + AddHeaderAdapterParameter = "add_header" + AddHeaderAdapterOriginParameter = "add_header_origin" +) + +func init() { + voluspa.AdaptersRegistry.AddAdapter(&AddHeaderAdapter{}) +} + +type AddHeaderAdapter struct { +} + +func (p *AddHeaderAdapter) PluginType() voluspa.PluginType { + return voluspa.CompoundAdapter +} + +func (p *AddHeaderAdapter) Type() voluspa.AdapterType { + return voluspa.AdapterType("add_header") +} + +func (p *AddHeaderAdapter) CompoundType() voluspa.AdapterType { + return adapterType +} + +func (p *AddHeaderAdapter) ConfigParameters() []string { + return []string{AddHeaderAdapterParameter, AddHeaderAdapterOriginParameter} +} + +func (p *AddHeaderAdapter) Content(env *voluspa.ConfigEnvironment) (*bytes.Buffer, error) { + header, _ := env.RemapOptions.ValueByNameAsString(AddHeaderAdapterParameter) + + var buf bytes.Buffer + if len(header) > 0 { + buf.WriteString(util.FormatSimpleHeaderRewrite("READ_RESPONSE_HDR_HOOK", fmt.Sprintf("add-header %s", header))) + } + + header, _ = env.RemapOptions.ValueByNameAsString(AddHeaderAdapterOriginParameter) + if len(header) > 0 { + if buf.Len() > 0 { + buf.WriteByte('\n') + } + buf.WriteString(util.FormatSimpleHeaderRewrite("SEND_REQUEST_HDR_HOOK", fmt.Sprintf("add-header %s", header))) + } + + return &buf, nil +} diff --git a/tools/voluspa/adapters/headerrewrite/base_header_rewrite.go b/tools/voluspa/adapters/headerrewrite/base_header_rewrite.go new file mode 100644 index 00000000000..9c2daed2288 --- /dev/null +++ b/tools/voluspa/adapters/headerrewrite/base_header_rewrite.go @@ -0,0 +1,65 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package headerrewrite + +import ( + "bytes" + + "github.com/apache/trafficserver/tools/voluspa" +) + +func init() { + voluspa.AdaptersRegistry.AddAdapter(&BaseHeaderRewriteAdapter{}) +} + +type BaseHeaderRewriteAdapter struct { +} + +func (*BaseHeaderRewriteAdapter) Weight() int { + return 25 +} + +func (p *BaseHeaderRewriteAdapter) PluginType() voluspa.PluginType { + return voluspa.CompoundAdapter +} + +func (p *BaseHeaderRewriteAdapter) Type() voluspa.AdapterType { + return adapterType +} + +func (p *BaseHeaderRewriteAdapter) CompoundType() voluspa.AdapterType { + return adapterType +} + +func (p *BaseHeaderRewriteAdapter) Name() string { + return "hdrs" +} + +func (p *BaseHeaderRewriteAdapter) SharedLibraryName() string { + return "header_rewrite.so" +} + +func (p *BaseHeaderRewriteAdapter) ConfigParameters() []string { + return []string{""} +} + +func (p *BaseHeaderRewriteAdapter) Content(env *voluspa.ConfigEnvironment) (*bytes.Buffer, error) { + return &bytes.Buffer{}, nil +} diff --git a/tools/voluspa/adapters/headerrewrite/common.go b/tools/voluspa/adapters/headerrewrite/common.go new file mode 100644 index 00000000000..dd14765f974 --- /dev/null +++ b/tools/voluspa/adapters/headerrewrite/common.go @@ -0,0 +1,24 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package headerrewrite + +import "github.com/apache/trafficserver/tools/voluspa" + +var adapterType = voluspa.AdapterType("header_rewrite") diff --git a/tools/voluspa/adapters/headerrewrite/content_type_forge.go b/tools/voluspa/adapters/headerrewrite/content_type_forge.go new file mode 100644 index 00000000000..b92b1a94f00 --- /dev/null +++ b/tools/voluspa/adapters/headerrewrite/content_type_forge.go @@ -0,0 +1,88 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package headerrewrite + +import ( + "bytes" + "fmt" + "sort" + + "github.com/apache/trafficserver/tools/voluspa" + "github.com/apache/trafficserver/tools/voluspa/adapters/util" +) + +const ( + ContentTypeForgeAdapterParameter = "content_type_forge" +) + +func init() { + voluspa.AdaptersRegistry.AddAdapter(&ContentTypeForgeAdapter{}) +} + +type ContentTypeForgeAdapter struct { +} + +func (p *ContentTypeForgeAdapter) PluginType() voluspa.PluginType { + return voluspa.CompoundAdapter +} + +func (p *ContentTypeForgeAdapter) Type() voluspa.AdapterType { + return voluspa.AdapterType("content_type_forge") +} + +func (p *ContentTypeForgeAdapter) CompoundType() voluspa.AdapterType { + return adapterType +} + +func (p *ContentTypeForgeAdapter) Name() string { + return "" +} + +func (p *ContentTypeForgeAdapter) ConfigParameters() []string { + return []string{ContentTypeForgeAdapterParameter} +} + +func (p *ContentTypeForgeAdapter) Content(env *voluspa.ConfigEnvironment) (*bytes.Buffer, error) { + content := &bytes.Buffer{} + + contentTypes, err := env.RemapOptions.ValueByNameAsStringMapString(ContentTypeForgeAdapterParameter) + if err != nil { + return nil, util.FormatError(ContentTypeForgeAdapterParameter, err) + } + + var keys []string + for k := range contentTypes { + keys = append(keys, k) + } + sort.Strings(keys) + + for _, regex := range keys { + contentType := contentTypes[regex] + content.WriteString(fmt.Sprintf( + `cond %%{SEND_RESPONSE_HDR_HOOK} [AND] + cond %%{PATH} /%s/ + set-header Content-Type %s`, + regex, contentType, + )) + content.WriteString("\n") + } + + return content, nil +} diff --git a/tools/voluspa/adapters/headerrewrite/defaultttl.go b/tools/voluspa/adapters/headerrewrite/defaultttl.go new file mode 100644 index 00000000000..6cadc29d2cd --- /dev/null +++ b/tools/voluspa/adapters/headerrewrite/defaultttl.go @@ -0,0 +1,83 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package headerrewrite + +import ( + "bytes" + "fmt" + + "github.com/apache/trafficserver/tools/voluspa" + "github.com/apache/trafficserver/tools/voluspa/adapters/util" +) + +const ( + DefaultTTLAdapterParameter = "default_ttl" +) + +func init() { + voluspa.AdaptersRegistry.AddAdapter(&DefaultTTLAdapter{}) +} + +type DefaultTTLAdapter struct { +} + +func (p *DefaultTTLAdapter) PluginType() voluspa.PluginType { + return voluspa.CompoundAdapter +} + +func (p *DefaultTTLAdapter) Type() voluspa.AdapterType { + return voluspa.AdapterType("default_ttl") +} + +func (p *DefaultTTLAdapter) CompoundType() voluspa.AdapterType { + return adapterType +} + +func (p *DefaultTTLAdapter) Name() string { + return "" +} + +func (p *DefaultTTLAdapter) ConfigParameters() []string { + return []string{DefaultTTLAdapterParameter} +} + +func (p *DefaultTTLAdapter) Content(env *voluspa.ConfigEnvironment) (*bytes.Buffer, error) { + content := &bytes.Buffer{} + + cacheControl, err := env.RemapOptions.ValueByNameAsString(DefaultTTLAdapterParameter) + if err != nil { + return nil, util.FormatError(DefaultTTLAdapterParameter, err) + } + + // translate a simple duration to a max-age public directive + if maxAge, err := util.DurationToSeconds(cacheControl); err == nil { + cacheControl = fmt.Sprintf("max-age=%s, public", maxAge) + } + + content.WriteString(fmt.Sprintf( + `cond %%{READ_RESPONSE_HDR_HOOK} [AND] +cond %%{HEADER:Cache-Control} ="" [AND] +%s + set-header Cache-Control %q`, + conditionStatusCodeSuccess, cacheControl, + )) + + return content, nil +} diff --git a/tools/voluspa/adapters/headerrewrite/defaultttl_test.go b/tools/voluspa/adapters/headerrewrite/defaultttl_test.go new file mode 100644 index 00000000000..305f3d6a833 --- /dev/null +++ b/tools/voluspa/adapters/headerrewrite/defaultttl_test.go @@ -0,0 +1,97 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package headerrewrite + +import ( + "errors" + "fmt" + "strings" + "testing" + + "github.com/apache/trafficserver/tools/voluspa" +) + +func TestDefaultTTL(t *testing.T) { + + plugin := &DefaultTTLAdapter{} + + if plugin.Type() != "default_ttl" { + t.Errorf("Type() returned unexpected %q", plugin.Type()) + } + + if fmt.Sprintf("%v", plugin.ConfigParameters()) != "[default_ttl]" { + t.Errorf("ConfigParameters() returned unexpected %v", plugin.ConfigParameters()) + } + + for i, tt := range []struct { + param interface{} + expected string + err error + }{ + { + param: nil, + err: errors.New(`option "default_ttl": value for "default_ttl" is not a string: `), + }, + { + param: true, + err: errors.New(`option "default_ttl": value for "default_ttl" is not a string: true`), + }, + { + param: 42, + err: errors.New(`option "default_ttl": value for "default_ttl" is not a string: 42`), + }, + { + param: "5d", + expected: ` +cond %{READ_RESPONSE_HDR_HOOK} [AND] +cond %{HEADER:Cache-Control} ="" [AND] +cond %{STATUS} >199 [AND] +cond %{STATUS} <400 + set-header Cache-Control "max-age=432000, public"`, + }, + { + param: "no-cache, no-store, must-revalidate, stale-while-revalidate=30", + expected: ` +cond %{READ_RESPONSE_HDR_HOOK} [AND] +cond %{HEADER:Cache-Control} ="" [AND] +cond %{STATUS} >199 [AND] +cond %{STATUS} <400 + set-header Cache-Control "no-cache, no-store, must-revalidate, stale-while-revalidate=30"`, + }, + } { + + ro := &voluspa.RemapOptions{} + env := &voluspa.ConfigEnvironment{RemapOptions: *ro} + + t.Run(fmt.Sprintf("%d-%v", i, tt.param), func(t *testing.T) { + + (*ro)[DefaultTTLAdapterParameter] = tt.param + got, err := plugin.Content(env) + if fmt.Sprintf("%v", err) != fmt.Sprintf("%v", tt.err) { + t.Errorf("Content() error expected `%v` got: %v", tt.err, err) + return + } + expected := strings.TrimPrefix(tt.expected, "\n") + if err == nil && got.String() != expected { + t.Errorf("Content() buffer expected:\n%s\ngot:\n%s", expected, got.String()) + } + }) + } +} diff --git a/tools/voluspa/adapters/headerrewrite/dscp.go b/tools/voluspa/adapters/headerrewrite/dscp.go new file mode 100644 index 00000000000..94618995503 --- /dev/null +++ b/tools/voluspa/adapters/headerrewrite/dscp.go @@ -0,0 +1,92 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package headerrewrite + +import ( + "bytes" + "fmt" + + "github.com/apache/trafficserver/tools/voluspa" + "github.com/apache/trafficserver/tools/voluspa/adapters/util" +) + +const ( + DSCPAdapterParameterPriority = "priority" +) + +func init() { + voluspa.AdaptersRegistry.AddAdapter(&DSCPAdapter{}) +} + +type DSCPAdapter struct { +} + +func (p *DSCPAdapter) PluginType() voluspa.PluginType { + return voluspa.CompoundAdapter +} + +func (p *DSCPAdapter) Type() voluspa.AdapterType { + return voluspa.AdapterType("dscp") +} + +func (p *DSCPAdapter) CompoundType() voluspa.AdapterType { + return adapterType +} + +func (p *DSCPAdapter) ConfigParameters() []string { + return []string{DSCPAdapterParameterPriority} +} + +func (p *DSCPAdapter) ConfigLocations() []voluspa.ConfigLocation { + return []voluspa.ConfigLocation{voluspa.ChildConfig} +} + +func (p *DSCPAdapter) Content(env *voluspa.ConfigEnvironment) (*bytes.Buffer, error) { + content := &bytes.Buffer{} + value, err := env.RemapOptions.ValueByNameAsString(DSCPAdapterParameterPriority) + if err != nil { + return nil, err + } + + // http://www.tucny.com/Home/dscp-tos + var dscpvalue int + switch value { + case "background": + // Queue 1 - Low-Priority Data - CS1 + dscpvalue = 8 + case "foreground": + // Queue 0 - Best-Effort - AF31, AF32, AF33, AF11, AF12, AF13 + // + // AF11 = High-Throughput Data - iTunes Downloads (App Store, Songs, etc.) + dscpvalue = 10 + case "streamingaudio": + // AF31 = Multimedia Streaming - HTTP Live Streaming - Streaming Audio + dscpvalue = 26 + case "streamingvideo": + // AF32 = Multimedia Streaming - HTTP Live Streaming - Streaming Video + dscpvalue = 28 + default: + return nil, util.FormatError(DSCPAdapterParameterPriority, fmt.Errorf("Unhandled priority %q", value)) + } + + content.WriteString(util.FormatSimpleHeaderRewrite("REMAP_PSEUDO_HOOK", fmt.Sprintf("set-conn-dscp %d", dscpvalue))) + + return content, nil +} diff --git a/tools/voluspa/adapters/headerrewrite/forcettl.go b/tools/voluspa/adapters/headerrewrite/forcettl.go new file mode 100644 index 00000000000..c70b91db394 --- /dev/null +++ b/tools/voluspa/adapters/headerrewrite/forcettl.go @@ -0,0 +1,82 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package headerrewrite + +import ( + "bytes" + "fmt" + + "github.com/apache/trafficserver/tools/voluspa" + "github.com/apache/trafficserver/tools/voluspa/adapters/util" +) + +const ( + ForceTTLAdapterParameter = "force_ttl" +) + +func init() { + voluspa.AdaptersRegistry.AddAdapter(&ForceTTLAdapter{}) +} + +type ForceTTLAdapter struct { +} + +func (p *ForceTTLAdapter) PluginType() voluspa.PluginType { + return voluspa.CompoundAdapter +} + +func (p *ForceTTLAdapter) Type() voluspa.AdapterType { + return voluspa.AdapterType("force_ttl") +} + +func (p *ForceTTLAdapter) CompoundType() voluspa.AdapterType { + return adapterType +} + +func (p *ForceTTLAdapter) Name() string { + return "" +} + +func (p *ForceTTLAdapter) ConfigParameters() []string { + return []string{ForceTTLAdapterParameter} +} + +func (p *ForceTTLAdapter) Content(env *voluspa.ConfigEnvironment) (*bytes.Buffer, error) { + content := &bytes.Buffer{} + + cacheControl, err := env.RemapOptions.ValueByNameAsString(ForceTTLAdapterParameter) + if err != nil { + return nil, util.FormatError(ForceTTLAdapterParameter, err) + } + + // translate a simple duration to a max-age public directive + if maxAge, err := util.DurationToSeconds(cacheControl); err == nil { + cacheControl = fmt.Sprintf("max-age=%s, public", maxAge) + } + + content.WriteString(fmt.Sprintf( + `cond %%{READ_RESPONSE_HDR_HOOK} [AND] +%s + set-header Cache-Control %q`, + conditionStatusCodeSuccess, cacheControl, + )) + + return content, nil +} diff --git a/tools/voluspa/adapters/headerrewrite/forcettl_test.go b/tools/voluspa/adapters/headerrewrite/forcettl_test.go new file mode 100644 index 00000000000..3ae8bacdf63 --- /dev/null +++ b/tools/voluspa/adapters/headerrewrite/forcettl_test.go @@ -0,0 +1,95 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package headerrewrite + +import ( + "errors" + "fmt" + "strings" + "testing" + + "github.com/apache/trafficserver/tools/voluspa" +) + +func TestForceTTL(t *testing.T) { + + plugin := &ForceTTLAdapter{} + + if plugin.Type() != "force_ttl" { + t.Errorf("Type() returned unexpected %q", plugin.Type()) + } + + if fmt.Sprintf("%v", plugin.ConfigParameters()) != "[force_ttl]" { + t.Errorf("ConfigParameters() returned unexpected %v", plugin.ConfigParameters()) + } + + for i, tt := range []struct { + param interface{} + expected string + err error + }{ + { + param: nil, + err: errors.New(`option "force_ttl": value for "force_ttl" is not a string: `), + }, + { + param: true, + err: errors.New(`option "force_ttl": value for "force_ttl" is not a string: true`), + }, + { + param: 42, + err: errors.New(`option "force_ttl": value for "force_ttl" is not a string: 42`), + }, + { + param: "5d", + expected: ` +cond %{READ_RESPONSE_HDR_HOOK} [AND] +cond %{STATUS} >199 [AND] +cond %{STATUS} <400 + set-header Cache-Control "max-age=432000, public"`, + }, + { + param: "no-cache, no-store, must-revalidate, stale-while-revalidate=30", + expected: ` +cond %{READ_RESPONSE_HDR_HOOK} [AND] +cond %{STATUS} >199 [AND] +cond %{STATUS} <400 + set-header Cache-Control "no-cache, no-store, must-revalidate, stale-while-revalidate=30"`, + }, + } { + + ro := &voluspa.RemapOptions{} + env := &voluspa.ConfigEnvironment{RemapOptions: *ro} + + t.Run(fmt.Sprintf("%d-%v", i, tt.param), func(t *testing.T) { + + (*ro)[ForceTTLAdapterParameter] = tt.param + got, err := plugin.Content(env) + if fmt.Sprintf("%v", err) != fmt.Sprintf("%v", tt.err) { + t.Errorf("Content() error expected `%v` got: %v", tt.err, err) + return + } + expected := strings.TrimPrefix(tt.expected, "\n") + if err == nil && got.String() != expected { + t.Errorf("Content() buffer expected:\n%s\ngot:\n%s", expected, got.String()) + } + }) + } +} diff --git a/tools/voluspa/adapters/headerrewrite/header_rewrite.go b/tools/voluspa/adapters/headerrewrite/header_rewrite.go new file mode 100644 index 00000000000..8e77d83540d --- /dev/null +++ b/tools/voluspa/adapters/headerrewrite/header_rewrite.go @@ -0,0 +1,77 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package headerrewrite + +import ( + "bytes" + "fmt" + + "github.com/apache/trafficserver/tools/voluspa" + "github.com/apache/trafficserver/tools/voluspa/adapters/util" +) + +const ( + HeaderRewriteAdapterParameter = "header_rewrite" +) + +func init() { + voluspa.AdaptersRegistry.AddAdapter(&FreeformHeaderRewriteAdapter{}) +} + +type FreeformHeaderRewriteAdapter struct { +} + +func (*FreeformHeaderRewriteAdapter) Weight() int { + return 25 +} + +func (p *FreeformHeaderRewriteAdapter) PluginType() voluspa.PluginType { + return voluspa.CompoundAdapter +} + +func (p *FreeformHeaderRewriteAdapter) Type() voluspa.AdapterType { + return voluspa.AdapterType("raw_header_rewrite") +} + +func (p *FreeformHeaderRewriteAdapter) CompoundType() voluspa.AdapterType { + return adapterType +} + +func (p *FreeformHeaderRewriteAdapter) Name() string { + return "" +} + +func (p *FreeformHeaderRewriteAdapter) ConfigParameters() []string { + return []string{HeaderRewriteAdapterParameter} +} + +func (p *FreeformHeaderRewriteAdapter) SharedLibraryName() string { + return "header_rewrite.so" +} + +func (p *FreeformHeaderRewriteAdapter) Content(env *voluspa.ConfigEnvironment) (*bytes.Buffer, error) { + val, err := env.RemapOptions.ValueByNameAsString(HeaderRewriteAdapterParameter) + if err != nil { + return nil, util.FormatError(HeaderRewriteAdapterParameter, err) + } + + // Do nothing with the config, just pass through + return bytes.NewBufferString(fmt.Sprintf("%s\n", val)), nil +} diff --git a/tools/voluspa/adapters/headerrewrite/log_cookie_header.go b/tools/voluspa/adapters/headerrewrite/log_cookie_header.go new file mode 100644 index 00000000000..14043051c91 --- /dev/null +++ b/tools/voluspa/adapters/headerrewrite/log_cookie_header.go @@ -0,0 +1,107 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package headerrewrite + +import ( + "bytes" + "fmt" + "sort" + + "github.com/apache/trafficserver/tools/voluspa" +) + +const ( + logCookieDirective = "log_cookie" + logHeaderDirective = "log_header" +) + +func init() { + voluspa.AdaptersRegistry.AddAdapter(&LogAdapter{}) +} + +type LogAdapter struct { +} + +func (p *LogAdapter) PluginType() voluspa.PluginType { + return voluspa.CompoundAdapter +} + +func (p *LogAdapter) Type() voluspa.AdapterType { + return voluspa.AdapterType("log_field") +} + +func (p *LogAdapter) CompoundType() voluspa.AdapterType { + return adapterType +} + +func (p *LogAdapter) ConfigParameters() []string { + return []string{logCookieDirective, logHeaderDirective} +} + +func (p *LogAdapter) logCookieHeader(logFields map[string]string, content *bytes.Buffer, parameterName string, index int) (*bytes.Buffer, int) { + var logFieldNames []string + + for k := range logFields { + logFieldNames = append(logFieldNames, k) + } + sort.Strings(logFieldNames) + + content.WriteString(fmt.Sprintf( + `cond %%{REMAP_PSEUDO_HOOK}`, + )) + content.WriteString("\n") + + for _, logFieldName := range logFieldNames { + logFieldValue := logFields[logFieldName] + if parameterName == "log_cookie" { + content.WriteString(fmt.Sprintf( + ` set-header Bazinga%d %%{COOKIE:%s}`, + index, logFieldValue, + )) + index++ + } else if parameterName == "log_header" { + content.WriteString(fmt.Sprintf( + ` set-header Bazinga%d %%{HEADER:%s}`, + index, logFieldValue, + )) + index++ + } + content.WriteString("\n") + } + return content, index +} + +func (p *LogAdapter) Content(env *voluspa.ConfigEnvironment) (*bytes.Buffer, error) { + content := &bytes.Buffer{} + index := 1 + + // Log Cookies + logFields, _ := env.RemapOptions.ValueByNameAsStringMapString(logCookieDirective) + if len(logFields) > 0 { + content, index = p.logCookieHeader(logFields, content, logCookieDirective, index) + } + + // Log headers + logFields, _ = env.RemapOptions.ValueByNameAsStringMapString(logHeaderDirective) + if len(logFields) > 0 { + content, _ = p.logCookieHeader(logFields, content, logHeaderDirective, index) + } + return content, nil +} diff --git a/tools/voluspa/adapters/headerrewrite/log_type.go b/tools/voluspa/adapters/headerrewrite/log_type.go new file mode 100644 index 00000000000..caa085e64af --- /dev/null +++ b/tools/voluspa/adapters/headerrewrite/log_type.go @@ -0,0 +1,74 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package headerrewrite + +import ( + "bytes" + + "github.com/apache/trafficserver/tools/voluspa" + "github.com/apache/trafficserver/tools/voluspa/adapters/util" +) + +const ( + LogTypeAdapterParameter = "log_type" +) + +func init() { + voluspa.AdaptersRegistry.AddAdapter(&LogTypeAdapter{}) +} + +type LogTypeAdapter struct { +} + +func (p *LogTypeAdapter) PluginType() voluspa.PluginType { + return voluspa.CompoundAdapter +} + +func (p *LogTypeAdapter) Type() voluspa.AdapterType { + return voluspa.AdapterType("log_type") +} + +func (p *LogTypeAdapter) CompoundType() voluspa.AdapterType { + return adapterType +} + +func (p *LogTypeAdapter) Name() string { + return "" +} + +func (p *LogTypeAdapter) ConfigParameters() []string { + return []string{LogTypeAdapterParameter} +} + +func (p *LogTypeAdapter) Content(env *voluspa.ConfigEnvironment) (*bytes.Buffer, error) { + content := &bytes.Buffer{} + + val, err := env.RemapOptions.ValueByNameAsString(LogTypeAdapterParameter) + if err != nil { + return nil, err + } + + if val == "public" { + return nil, nil + } + content.WriteString(util.FormatSimpleHeaderRewrite("REMAP_PSEUDO_HOOK", `set-header @cdnlog "private"`)) + + return content, nil +} diff --git a/tools/voluspa/adapters/headerrewrite/proxy_cache_control.go b/tools/voluspa/adapters/headerrewrite/proxy_cache_control.go new file mode 100644 index 00000000000..9d2a36b48a0 --- /dev/null +++ b/tools/voluspa/adapters/headerrewrite/proxy_cache_control.go @@ -0,0 +1,77 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package headerrewrite + +import ( + "bytes" + "fmt" + + "github.com/apache/trafficserver/tools/voluspa" + "github.com/apache/trafficserver/tools/voluspa/adapters/util" +) + +const ( + ProxyCacheControlAdapterParameter = "proxy_cache_control" +) + +func init() { + voluspa.AdaptersRegistry.AddAdapter(&ProxyCacheControlAdapter{}) +} + +type ProxyCacheControlAdapter struct { +} + +func (p *ProxyCacheControlAdapter) PluginType() voluspa.PluginType { + return voluspa.CompoundAdapter +} + +func (p *ProxyCacheControlAdapter) Type() voluspa.AdapterType { + return voluspa.AdapterType("proxy_cache_control") +} + +func (p *ProxyCacheControlAdapter) CompoundType() voluspa.AdapterType { + return adapterType +} + +func (p *ProxyCacheControlAdapter) ConfigParameters() []string { + return []string{ProxyCacheControlAdapterParameter} +} + +func (p *ProxyCacheControlAdapter) Content(env *voluspa.ConfigEnvironment) (*bytes.Buffer, error) { + header, err := env.RemapOptions.ValueByNameAsString(ProxyCacheControlAdapterParameter) + if err != nil { + return nil, util.FormatError(ProxyCacheControlAdapterParameter, err) + } + + content := &bytes.Buffer{} + + content.WriteString(fmt.Sprintf( + `cond %%{READ_RESPONSE_HDR_HOOK} + set-header @Original-Cache-Control %%{HEADER:Cache-Control} + set-header Cache-Control %%{HEADER:%s} + +cond %%{SEND_RESPONSE_HDR_HOOK} + set-header Cache-Control %%{HEADER:@Original-Cache-Control} + rm-header Proxy-Cache-Control`, + header, + )) + + return content, nil +} diff --git a/tools/voluspa/adapters/headerrewrite/receipt.go b/tools/voluspa/adapters/headerrewrite/receipt.go new file mode 100644 index 00000000000..e077132bff4 --- /dev/null +++ b/tools/voluspa/adapters/headerrewrite/receipt.go @@ -0,0 +1,101 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package headerrewrite + +import ( + "bytes" + "fmt" + + "github.com/apache/trafficserver/tools/voluspa" + "github.com/apache/trafficserver/tools/voluspa/adapters/util" +) + +const ( + ReceiptsAdapterParameter = "receipts" + ReceiptsAdapterParameterEnabled = "enabled" + ReceiptsAdapterParameterName = "name" +) + +func init() { + voluspa.AdaptersRegistry.AddAdapter(&ReceiptAdapter{}) +} + +type ReceiptAdapter struct { +} + +func (*ReceiptAdapter) Weight() int { + return 25 +} + +func (p *ReceiptAdapter) PluginType() voluspa.PluginType { + return voluspa.CompoundAdapter +} + +func (p *ReceiptAdapter) Type() voluspa.AdapterType { + return voluspa.Receipt +} + +func (p *ReceiptAdapter) CompoundType() voluspa.AdapterType { + return adapterType +} + +func (p *ReceiptAdapter) Name() string { + return "" +} + +func (p *ReceiptAdapter) ConfigParameters() []string { + return []string{ReceiptsAdapterParameter} +} + +func (p *ReceiptAdapter) Content(env *voluspa.ConfigEnvironment) (*bytes.Buffer, error) { + content := &bytes.Buffer{} + + receiptsConfig, err := env.RemapOptions.ValueByNameAsStringMapInterface(ReceiptsAdapterParameter) + if err != nil { + return nil, err + } + + value, hasOption := receiptsConfig[ReceiptsAdapterParameterEnabled] + if hasOption { + bvalue, ok := value.(bool) + if !ok { + return nil, fmt.Errorf("invalid 'enabled' value (%s)", value) + } + + if !bvalue { + return content, nil + } + } + + receiptName := env.Property + value, hasOption = receiptsConfig[ReceiptsAdapterParameterName] + if hasOption { + strvalue, ok := value.(string) + if !ok { + return nil, fmt.Errorf("invalid 'name' value (%s)", value) + } + receiptName = strvalue + } + + receipt := fmt.Sprintf("%s{{hosttype}}", receiptName) + content.WriteString(util.FormatSimpleHeaderRewrite("REMAP_PSEUDO_HOOK", fmt.Sprintf(`set-header @ReceiptService "%s"`, receipt))) + + return content, nil +} diff --git a/tools/voluspa/adapters/headerrewrite/remove_header.go b/tools/voluspa/adapters/headerrewrite/remove_header.go new file mode 100644 index 00000000000..4843137fd16 --- /dev/null +++ b/tools/voluspa/adapters/headerrewrite/remove_header.go @@ -0,0 +1,74 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package headerrewrite + +import ( + "bytes" + "fmt" + + "github.com/apache/trafficserver/tools/voluspa" + "github.com/apache/trafficserver/tools/voluspa/adapters/util" +) + +const ( + RemoveHeaderAdapterParameter = "remove_header" + RemoveHeaderAdapterOriginParameter = "remove_header_origin" +) + +func init() { + voluspa.AdaptersRegistry.AddAdapter(&RemoveHeaderAdapter{}) +} + +type RemoveHeaderAdapter struct { +} + +func (p *RemoveHeaderAdapter) PluginType() voluspa.PluginType { + return voluspa.CompoundAdapter +} + +func (p *RemoveHeaderAdapter) Type() voluspa.AdapterType { + return voluspa.AdapterType("remove_header") +} + +func (p *RemoveHeaderAdapter) CompoundType() voluspa.AdapterType { + return adapterType +} + +func (p *RemoveHeaderAdapter) ConfigParameters() []string { + return []string{RemoveHeaderAdapterParameter, RemoveHeaderAdapterOriginParameter} +} + +func (p *RemoveHeaderAdapter) Content(env *voluspa.ConfigEnvironment) (*bytes.Buffer, error) { + header, _ := env.RemapOptions.ValueByNameAsString(RemoveHeaderAdapterParameter) + + var buf bytes.Buffer + if len(header) > 0 { + buf.WriteString(util.FormatSimpleHeaderRewrite("REMAP_PSEUDO_HOOK", fmt.Sprintf("rm-header %s", header))) + } + header, _ = env.RemapOptions.ValueByNameAsString(RemoveHeaderAdapterOriginParameter) + if len(header) > 0 { + if buf.Len() > 0 { + buf.WriteByte('\n') + } + buf.WriteString(util.FormatSimpleHeaderRewrite("SEND_REQUEST_HDR_HOOK", fmt.Sprintf("rm-header %s", header))) + } + + return &buf, nil +} diff --git a/tools/voluspa/adapters/headerrewrite/rewrite_utils.go b/tools/voluspa/adapters/headerrewrite/rewrite_utils.go new file mode 100644 index 00000000000..6634a62477b --- /dev/null +++ b/tools/voluspa/adapters/headerrewrite/rewrite_utils.go @@ -0,0 +1,23 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package headerrewrite + +const conditionStatusCodeSuccess = `cond %{STATUS} >199 [AND] +cond %{STATUS} <400` diff --git a/tools/voluspa/adapters/headerrewrite/set_header.go b/tools/voluspa/adapters/headerrewrite/set_header.go new file mode 100644 index 00000000000..011036773e0 --- /dev/null +++ b/tools/voluspa/adapters/headerrewrite/set_header.go @@ -0,0 +1,75 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package headerrewrite + +import ( + "bytes" + "fmt" + + "github.com/apache/trafficserver/tools/voluspa" + "github.com/apache/trafficserver/tools/voluspa/adapters/util" +) + +const ( + SetHeaderAdapterParameter = "set_header" + SetHeaderAdapterOriginParameter = "set_header_origin" +) + +func init() { + voluspa.AdaptersRegistry.AddAdapter(&SetHeaderAdapter{}) +} + +type SetHeaderAdapter struct { +} + +func (p *SetHeaderAdapter) PluginType() voluspa.PluginType { + return voluspa.CompoundAdapter +} + +func (p *SetHeaderAdapter) Type() voluspa.AdapterType { + return voluspa.AdapterType("set_header") +} + +func (p *SetHeaderAdapter) CompoundType() voluspa.AdapterType { + return adapterType +} + +func (p *SetHeaderAdapter) ConfigParameters() []string { + return []string{SetHeaderAdapterParameter, SetHeaderAdapterOriginParameter} +} + +func (p *SetHeaderAdapter) Content(env *voluspa.ConfigEnvironment) (*bytes.Buffer, error) { + header, _ := env.RemapOptions.ValueByNameAsString(SetHeaderAdapterParameter) + + var buf bytes.Buffer + if len(header) > 0 { + buf.WriteString(util.FormatSimpleHeaderRewrite("READ_RESPONSE_HDR_HOOK", fmt.Sprintf("set-header %s", header))) + } + + header, _ = env.RemapOptions.ValueByNameAsString(SetHeaderAdapterOriginParameter) + if len(header) > 0 { + if buf.Len() > 0 { + buf.WriteByte('\n') + } + buf.WriteString(util.FormatSimpleHeaderRewrite("SEND_REQUEST_HDR_HOOK", fmt.Sprintf("set-header %s", header))) + } + + return &buf, nil +} diff --git a/tools/voluspa/adapters/init.go b/tools/voluspa/adapters/init.go new file mode 100644 index 00000000000..c83ad48814d --- /dev/null +++ b/tools/voluspa/adapters/init.go @@ -0,0 +1,24 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package adapters + +import _ "github.com/apache/trafficserver/tools/voluspa/adapters/confremap" // for convenience +import _ "github.com/apache/trafficserver/tools/voluspa/adapters/headerrewrite" // for convenience +import _ "github.com/apache/trafficserver/tools/voluspa/adapters/lua" // for convenience diff --git a/tools/voluspa/adapters/lua/base_lua.go b/tools/voluspa/adapters/lua/base_lua.go new file mode 100644 index 00000000000..19fa496a63f --- /dev/null +++ b/tools/voluspa/adapters/lua/base_lua.go @@ -0,0 +1,65 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package lua + +import ( + "bytes" + + "github.com/apache/trafficserver/tools/voluspa" +) + +func init() { + voluspa.AdaptersRegistry.AddAdapter(&BaseLuaAdapter{}) +} + +type BaseLuaAdapter struct { +} + +func (*BaseLuaAdapter) Weight() int { + return 99 +} + +func (p *BaseLuaAdapter) PluginType() voluspa.PluginType { + return voluspa.CompoundAdapter +} + +func (p *BaseLuaAdapter) Type() voluspa.AdapterType { + return adapterType +} + +func (p *BaseLuaAdapter) CompoundType() voluspa.AdapterType { + return adapterType +} + +func (p *BaseLuaAdapter) Name() string { + return luaScriptName +} + +func (p *BaseLuaAdapter) SharedLibraryName() string { + return "tslua.so" +} + +func (p *BaseLuaAdapter) ConfigParameters() []string { + return []string{""} +} + +func (p *BaseLuaAdapter) Content(env *voluspa.ConfigEnvironment) (*bytes.Buffer, error) { + return &bytes.Buffer{}, nil +} diff --git a/tools/voluspa/adapters/lua/common.go b/tools/voluspa/adapters/lua/common.go new file mode 100644 index 00000000000..470aa3f67e4 --- /dev/null +++ b/tools/voluspa/adapters/lua/common.go @@ -0,0 +1,25 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package lua + +import "github.com/apache/trafficserver/tools/voluspa" + +var adapterType = voluspa.AdapterType("lua") +var luaScriptName = "youdidntsetluascriptname.lua" diff --git a/tools/voluspa/adapters/lua/echo_cors.go b/tools/voluspa/adapters/lua/echo_cors.go new file mode 100644 index 00000000000..b5e01e276f7 --- /dev/null +++ b/tools/voluspa/adapters/lua/echo_cors.go @@ -0,0 +1,83 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package lua + +import ( + "bytes" + "fmt" + + "github.com/apache/trafficserver/tools/voluspa" +) + +const ( + EchoCORSAdapterParameter = "echo_cors" +) + +func init() { + voluspa.AdaptersRegistry.AddAdapter(&EchoCORSAdapter{}) + + // Every standard script in adapters/lua has to set this + luaScriptName = "echo_cors.lua" +} + +type EchoCORSAdapter struct { +} + +func (p *EchoCORSAdapter) PluginType() voluspa.PluginType { + return voluspa.CompoundAdapter +} + +func (p *EchoCORSAdapter) Type() voluspa.AdapterType { + return voluspa.AdapterType("echo_cors") +} + +func (p *EchoCORSAdapter) CompoundType() voluspa.AdapterType { + return adapterType +} + +func (p *EchoCORSAdapter) ConfigParameters() []string { + return []string{EchoCORSAdapterParameter} +} + +func (p *EchoCORSAdapter) Content(env *voluspa.ConfigEnvironment) (*bytes.Buffer, error) { + content := &bytes.Buffer{} + + headerName, err := env.RemapOptions.ValueByNameAsString(EchoCORSAdapterParameter) + if err != nil { + return nil, err + } + + content.WriteString(fmt.Sprintf( + ` +function send_response() + ts.client_response.header['Access-Control-Allow-Origin'] = ts.ctx['origin'] + ts.client_response.header['Access-Control-Allow-Authentication'] = 'True' + return 0 +end + +function do_remap() + ts.ctx['origin'] = ts.client_request.header.%s + ts.hook(TS_LUA_HOOK_SEND_RESPONSE_HDR, send_response) + return 0 +end + `, headerName)) + + return content, nil +} diff --git a/tools/voluspa/adapters/lua/lua.go b/tools/voluspa/adapters/lua/lua.go new file mode 100644 index 00000000000..1fcf35c0254 --- /dev/null +++ b/tools/voluspa/adapters/lua/lua.go @@ -0,0 +1,78 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package lua + +import ( + "bytes" + "fmt" + + "github.com/apache/trafficserver/tools/voluspa" + "github.com/apache/trafficserver/tools/voluspa/adapters/util" +) + +const ( + LuaAdapterParameter = "lua" +) + +func init() { + voluspa.AdaptersRegistry.AddAdapter(&FreeformLuaAdapter{}) + luaScriptName = "inline.lua" +} + +type FreeformLuaAdapter struct { +} + +func (*FreeformLuaAdapter) Weight() int { + return 25 +} + +func (p *FreeformLuaAdapter) PluginType() voluspa.PluginType { + return voluspa.CompoundAdapter +} + +func (p *FreeformLuaAdapter) Type() voluspa.AdapterType { + return voluspa.AdapterType("raw_lua") +} + +func (p *FreeformLuaAdapter) CompoundType() voluspa.AdapterType { + return adapterType +} + +func (p *FreeformLuaAdapter) Name() string { + return "" +} + +func (p *FreeformLuaAdapter) ConfigParameters() []string { + return []string{LuaAdapterParameter} +} + +func (p *FreeformLuaAdapter) SharedLibraryName() string { + return "tslua.so" +} + +func (p *FreeformLuaAdapter) Content(env *voluspa.ConfigEnvironment) (*bytes.Buffer, error) { + val, err := env.RemapOptions.ValueByNameAsString(LuaAdapterParameter) + if err != nil { + return nil, util.FormatError(LuaAdapterParameter, err) + } + + // Do nothing with the config, just pass through + return bytes.NewBufferString(fmt.Sprintf("%s\n", val)), nil +} diff --git a/tools/voluspa/adapters/null.go b/tools/voluspa/adapters/null.go new file mode 100644 index 00000000000..898de95cf38 --- /dev/null +++ b/tools/voluspa/adapters/null.go @@ -0,0 +1,51 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package adapters + +import ( + "github.com/apache/trafficserver/tools/voluspa" +) + +func init() { + voluspa.AdaptersRegistry.AddAdapter(&NullAdapter{}) +} + +type NullAdapter struct { +} + +func (p *NullAdapter) PluginType() voluspa.PluginType { + return voluspa.GeneralAdapter +} + +func (p *NullAdapter) Type() voluspa.AdapterType { + return voluspa.AdapterType("null") +} + +func (p *NullAdapter) Name() string { + return "null" +} + +func (p *NullAdapter) ConfigParameters() []string { + return []string{"null"} +} + +func (p *NullAdapter) PParams(env *voluspa.ConfigEnvironment) ([]string, error) { + return nil, nil +} diff --git a/tools/voluspa/adapters/redirect.go b/tools/voluspa/adapters/redirect.go new file mode 100644 index 00000000000..55b0c089e54 --- /dev/null +++ b/tools/voluspa/adapters/redirect.go @@ -0,0 +1,87 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package adapters + +import ( + "bytes" + "fmt" + + "github.com/apache/trafficserver/tools/voluspa" +) + +func init() { + voluspa.AdaptersRegistry.AddAdapter(&RedirectAdapter{}) +} + +type RedirectAdapter struct { +} + +func (*RedirectAdapter) Weight() int { + return 30 +} + +func (p *RedirectAdapter) PluginType() voluspa.PluginType { + return voluspa.GeneralAdapter +} + +func (p *RedirectAdapter) Type() voluspa.AdapterType { + return voluspa.AdapterType("redirect") +} + +func (p *RedirectAdapter) Name() string { + return "redirect" +} + +func (p *RedirectAdapter) SharedLibraryName() string { + return regexRemapSharedLibraryName +} + +func (p *RedirectAdapter) ConfigParameters() []string { + return []string{"redirect"} +} + +func (p *RedirectAdapter) Content(env *voluspa.ConfigEnvironment) (*bytes.Buffer, error) { + content := &bytes.Buffer{} + urls, err := env.RemapOptions.ValueByNameAsStringMapInterface(p.Name()) + if err != nil { + return nil, err + } + + url, ok := urls["url"].(string) + if !ok { + return nil, fmt.Errorf("Expecting url as string and Redirect url is mandatory") + } + + // default status code to 302 if not provided + httpCode, ok := urls["http_code"].(int) + if !ok { + httpCode = 302 + } + + // configurable source regex pattern defaults to (.*) + src, ok := urls["src"] + if !ok { + src = "(.*)" + } + + content.WriteString(fmt.Sprintf("%s %s @status=%d", src, url, httpCode)) + + return content, nil +} diff --git a/tools/voluspa/adapters/regex_remap.go b/tools/voluspa/adapters/regex_remap.go new file mode 100644 index 00000000000..a4dbbb37310 --- /dev/null +++ b/tools/voluspa/adapters/regex_remap.go @@ -0,0 +1,76 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package adapters + +import ( + "bytes" + + "github.com/apache/trafficserver/tools/voluspa" + "github.com/apache/trafficserver/tools/voluspa/adapters/util" +) + +const ( + RegexRemapAdapterParameter = "regex_remap" + regexRemapSharedLibraryName = "regex_remap.so" +) + +func init() { + voluspa.AdaptersRegistry.AddAdapter(&RegexRemapAdapter{}) +} + +type RegexRemapAdapter struct { +} + +func (*RegexRemapAdapter) Weight() int { + return 15 +} + +func (p *RegexRemapAdapter) PluginType() voluspa.PluginType { + return voluspa.GeneralAdapter +} + +func (p *RegexRemapAdapter) Type() voluspa.AdapterType { + return voluspa.AdapterType("regex_remap") +} + +func (p *RegexRemapAdapter) ConfigLocations() []voluspa.ConfigLocation { + return []voluspa.ConfigLocation{voluspa.ParentConfig} +} + +func (p *RegexRemapAdapter) Name() string { + return "regex" +} + +func (p *RegexRemapAdapter) SharedLibraryName() string { + return regexRemapSharedLibraryName +} + +func (p *RegexRemapAdapter) ConfigParameters() []string { + return []string{RegexRemapAdapterParameter} +} + +func (p *RegexRemapAdapter) Content(env *voluspa.ConfigEnvironment) (*bytes.Buffer, error) { + regex, err := env.RemapOptions.ValueByNameAsString(RegexRemapAdapterParameter) + if err != nil { + return nil, util.FormatError(RegexRemapAdapterParameter, err) + } + + return bytes.NewBufferString(regex), nil +} diff --git a/tools/voluspa/adapters/s3_auth.go b/tools/voluspa/adapters/s3_auth.go new file mode 100644 index 00000000000..b9776164c0c --- /dev/null +++ b/tools/voluspa/adapters/s3_auth.go @@ -0,0 +1,139 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package adapters + +import ( + "strconv" + + "fmt" + "strings" + + "github.com/apache/trafficserver/tools/voluspa" +) + +const ( + S3AuthAdapterParameter = "s3" +) + +func init() { + voluspa.AdaptersRegistry.AddAdapter(&S3AuthAdapter{}) +} + +type S3AuthAdapter struct { +} + +func (*S3AuthAdapter) Weight() int { + return 16 +} + +func (p *S3AuthAdapter) PluginType() voluspa.PluginType { + return voluspa.GeneralAdapter +} + +func (p *S3AuthAdapter) Type() voluspa.AdapterType { + return voluspa.AdapterType("s3_auth") +} + +func (p *S3AuthAdapter) SharedLibraryName() string { + return "s3_auth.so" +} + +func (p *S3AuthAdapter) ConfigLocations() []voluspa.ConfigLocation { + return []voluspa.ConfigLocation{voluspa.ParentConfig} +} + +func (p *S3AuthAdapter) ConfigParameters() []string { + return []string{S3AuthAdapterParameter} +} + +func (p *S3AuthAdapter) PParams(env *voluspa.ConfigEnvironment) ([]string, error) { + var alt map[string]interface{} + alt, err := env.RemapOptions.ValueByNameAsStringMapInterface(S3AuthAdapterParameter) + if err != nil { + return nil, err + } + + args, err := p.processArgs(alt) + if err != nil { + return nil, err + } + + return args, nil +} + +func (p *S3AuthAdapter) processArgs(alt map[string]interface{}) ([]string, error) { + var args []string + args = p.maybeAddArgs(alt["path"], "config", args) + args = p.maybeAddArgs(alt["auth"], "config", args) + args = p.maybeAddArgs(alt["virtual_host"], "virtual_host", args) + args = p.maybeAddArgs(alt["version"], "version", args) + args = p.maybeAddArgs(alt["v4_include_headers"], "v4-include-headers", args) + args = p.maybeAddArgs(alt["v4_exclude_headers"], "v4-exclude-headers", args) + args = p.maybeAddArgs(alt["region_map"], "v4-region-map", args) + + // if this is v4 and there is no v4_exclude_headers defined, add a @pparam=--v4-exclude-headers=x-forwarded-for,forwarded,via,authorization + _, excludeHeadersDefined := alt["v4_exclude_headers"] + if alt["version"] == 4 && !excludeHeadersDefined { + args = p.maybeAddArgs("x-forwarded-for,forwarded,via,authorization", "v4-exclude-headers", args) + } + + return args, nil +} + +func (p *S3AuthAdapter) paramToValue(i interface{}) string { + switch val := i.(type) { + case string: + return val + case int: + return strconv.Itoa(val) + case bool: + return strconv.FormatBool(val) + case []string: + return strings.Join(val, ",") + case []interface{}: + var vals []string + for _, v1 := range i.([]interface{}) { + vS := v1.(string) + vals = append(vals, vS) + } + return strings.Join(vals, ",") + default: + return "" + } +} + +func (p *S3AuthAdapter) maybeAddArgs(option interface{}, param string, args []string) []string { + value := p.paramToValue(option) + switch param { + case "virtual_host": + if value == "true" { + return append(args, "--virtual_host") + } + case "config": + if len(value) > 0 { + return append(args, []string{"--config", value}...) + } + default: + if len(value) > 0 { + return append(args, fmt.Sprintf("--%s=%s", param, value)) + } + } + return args +} diff --git a/tools/voluspa/adapters/strip_query.go b/tools/voluspa/adapters/strip_query.go new file mode 100644 index 00000000000..57c9a4d41b9 --- /dev/null +++ b/tools/voluspa/adapters/strip_query.go @@ -0,0 +1,57 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package adapters + +import ( + "bytes" + + "github.com/apache/trafficserver/tools/voluspa" +) + +func init() { + voluspa.AdaptersRegistry.AddAdapter(&StripQueryAdapter{}) +} + +type StripQueryAdapter struct { +} + +func (p *StripQueryAdapter) PluginType() voluspa.PluginType { + return voluspa.GeneralAdapter +} + +func (p *StripQueryAdapter) Type() voluspa.AdapterType { + return voluspa.AdapterType("strip_query") +} + +func (p *StripQueryAdapter) Name() string { + return "strip_query" +} + +func (p *StripQueryAdapter) SharedLibraryName() string { + return regexRemapSharedLibraryName +} + +func (p *StripQueryAdapter) ConfigParameters() []string { + return []string{"strip_query"} +} + +func (p *StripQueryAdapter) Content(env *voluspa.ConfigEnvironment) (*bytes.Buffer, error) { + return bytes.NewBufferString(". $s://$t/$P\n"), nil +} diff --git a/tools/voluspa/adapters/util/ttl_utils.go b/tools/voluspa/adapters/util/ttl_utils.go new file mode 100644 index 00000000000..d18f0a83aa8 --- /dev/null +++ b/tools/voluspa/adapters/util/ttl_utils.go @@ -0,0 +1,62 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package util + +import ( + "fmt" + "regexp" + "strconv" + "time" +) + +var durationRE = regexp.MustCompile(`^(\d+)([a-z])$`) + +// DurationToSeconds takes a duration string and returns number of seconds +// required to be a single time unit (eg 7d or 24h or 3s) +// complex duration not supported(yet) (eg 7d24h3s) +func DurationToSeconds(value string) (string, error) { + if value == "0" { + return value, nil + } + + if !durationRE.MatchString(value) { + return "", fmt.Errorf("Bad duration \"%s\"", value) + } + + matches := durationRE.FindStringSubmatch(value) + count, err := strconv.ParseUint(matches[1], 10, 64) + if err != nil { + return "", fmt.Errorf("Bad duration \"%s\"", value) + } + + switch matches[2] { + case "d": + return strconv.FormatUint(count*60*60*24, 10), nil + case "w": + return strconv.FormatUint(count*60*60*24*7, 10), nil + + default: + duration, err := time.ParseDuration(value) + if err == nil { + return fmt.Sprintf("%d", int(duration.Seconds())), nil + } + return "", fmt.Errorf("Unhandled spec '%s' for '%s'. err=%s", matches[2], value, err) + } +} diff --git a/tools/voluspa/adapters/util/ttl_utils_test.go b/tools/voluspa/adapters/util/ttl_utils_test.go new file mode 100644 index 00000000000..d8baf8f178e --- /dev/null +++ b/tools/voluspa/adapters/util/ttl_utils_test.go @@ -0,0 +1,85 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package util + +import "testing" + +func TestDurationToSeconds(t *testing.T) { + in := "30d" + expected := "2592000" + val, err := DurationToSeconds(in) + + if err != nil || val != expected { + t.Fatalf("Got '%s' expected '%s' err=%s", val, expected, err) + } + + in = "0" + expected = "0" + val, err = DurationToSeconds(in) + + if err != nil || val != expected { + t.Fatalf("Got '%s' expected '%s' err=%s", val, expected, err) + } + + in = "7w" + expected = "4233600" + val, err = DurationToSeconds(in) + + if err != nil || val != expected { + t.Fatalf("Got '%s' expected '%s' err=%s", val, expected, err) + } + + in = "1h" + expected = "3600" + val, err = DurationToSeconds(in) + + if err != nil || val != expected { + t.Fatalf("Got '%s' expected '%s' err=%s", val, expected, err) + } + + in = "2h3s" + expected = "7203" + val, err = DurationToSeconds(in) + + if err == nil || val == expected { + t.Fatalf("Got '%s' expected '%s' err=%s", val, expected, err) + } + + in = "7r" + val, err = DurationToSeconds(in) + + if err == nil { + t.Fatalf("Expected err converting '%s' val=%s", in, val) + } + + in = "5" + val, err = DurationToSeconds(in) + + if err == nil { + t.Fatalf("Expected err converting '%s' val=%s", in, val) + } + + in = "7w2d3h" + val, err = DurationToSeconds(in) + + if err == nil { + t.Fatalf("Expected err converting '%s' val=%s", in, val) + } +} diff --git a/tools/voluspa/adapters/util/util.go b/tools/voluspa/adapters/util/util.go new file mode 100644 index 00000000000..20a21c81459 --- /dev/null +++ b/tools/voluspa/adapters/util/util.go @@ -0,0 +1,46 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package util + +import ( + "fmt" + "strconv" +) + +func FormatSimpleHeaderRewrite(condition, action string) string { + return fmt.Sprintf("cond %%{%s}\n %s", condition, action) +} + +func FormatError(option string, err error) error { + return fmt.Errorf("option %q: %s", option, err.Error()) +} + +func ParamToValue(i interface{}) string { + switch val := i.(type) { + case string: + return val + case int: + return strconv.Itoa(val) + case bool: + return strconv.FormatBool(val) + default: + return "" + } +} diff --git a/tools/voluspa/adapters/video_background_fetch.go b/tools/voluspa/adapters/video_background_fetch.go new file mode 100644 index 00000000000..23ccf69a2eb --- /dev/null +++ b/tools/voluspa/adapters/video_background_fetch.go @@ -0,0 +1,198 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package adapters + +import ( + "fmt" + "strings" + + "github.com/apache/trafficserver/tools/voluspa" + "github.com/apache/trafficserver/tools/voluspa/adapters/util" +) + +func init() { + voluspa.AdaptersRegistry.AddAdapter(&VideoBackgroundFetchAdapter{}) +} + +const ( + VideoBackgroundFetchDirective = "video_background_fetch" + + VideoBackgroundFetchAdapterFrontend = "frontend" + VideoBackgroundFetchAdapterBackend = "backend" + + VideoBackgroundFetchAdapterFetchCount = "fetch_count" + VideoBackgroundFetchAdapterFetchPolicy = "fetch_policy" + VideoBackgroundFetchAdapterFetchMax = "fetch_max" + VideoBackgroundFetchAdapterAPIHeader = "api_header" + VideoBackgroundFetchAdapterFetchPathPattern = "fetch_path_pattern" + VideoBackgroundFetchAdapterReplaceHost = "replace_host" + VideoBackgroundFetchAdapterNameSpace = "name_space" + VideoBackgroundFetchAdapterMetricsPrefix = "metrics_prefix" + VideoBackgroundFetchAdapterExactMatch = "exact_match" + VideoBackgroundFetchAdapterLogName = "log_name" +) + +type VideoBackgroundFetchAdapter struct { +} + +func (p *VideoBackgroundFetchAdapter) PluginType() voluspa.PluginType { + return voluspa.GeneralAdapter +} + +func (p *VideoBackgroundFetchAdapter) Type() voluspa.AdapterType { + return voluspa.AdapterType("video_background_fetch") +} + +func (p *VideoBackgroundFetchAdapter) ConfigLocations() []voluspa.ConfigLocation { + return []voluspa.ConfigLocation{voluspa.ChildConfig} +} + +func (p *VideoBackgroundFetchAdapter) ConfigParameters() []string { + return []string{VideoBackgroundFetchDirective} +} + +func (p *VideoBackgroundFetchAdapter) Name() string { + return "volcano" +} + +func (p *VideoBackgroundFetchAdapter) SharedLibraryName() string { + return "volcano.so" +} + +func (p *VideoBackgroundFetchAdapter) PParams(env *voluspa.ConfigEnvironment) (voluspa.RolePParams, error) { + + alt, err := env.RemapOptions.ValueByNameAsStringMapInterface(VideoBackgroundFetchDirective) + if err != nil { + return nil, util.FormatError(VideoBackgroundFetchDirective, err) + } + + // process default args + args, err := p.processArgs(alt) + if err != nil { + return nil, err + } + + results := voluspa.RolePParams{} + results[voluspa.DefaultRole] = args + + for _, v := range []string{VideoBackgroundFetchAdapterFrontend, VideoBackgroundFetchAdapterBackend} { + sub, found := alt[v] + if !found { + continue + } + + submap, ok := sub.(map[interface{}]interface{}) + if !ok { + continue + } + + convertedMap, err := convertMap(submap) + if err != nil { + return nil, err + } + + args, err := p.processArgs(convertedMap) + if err != nil { + return nil, err + } + + args = append(args, fmt.Sprintf("--front=%t", v == VideoBackgroundFetchAdapterFrontend)) + + // eg roles_video_fetch_backend or roles_video_fetch_frontend + role := fmt.Sprintf("roles_video_fetch_%s", v) + results[role] = args + } + + return results, nil +} + +// convertMap attempts to create a usable submap from the very generic map returned by RemapOptions +func convertMap(in map[interface{}]interface{}) (map[string]interface{}, error) { + results := make(map[string]interface{}) + for k, v := range in { + ck, ok := k.(string) + if !ok { + return nil, fmt.Errorf("key %+q is not string", k) + } + results[ck] = v + } + return results, nil +} + +func (p *VideoBackgroundFetchAdapter) processArgs(alt map[string]interface{}) ([]string, error) { + var args []string + + args = p.maybeAddArgs(alt[VideoBackgroundFetchAdapterFetchPolicy], "fetch-policy", args) + args = p.maybeAddArgs(alt[VideoBackgroundFetchAdapterFetchCount], "fetch-count", args) + args = p.maybeAddArgs(alt[VideoBackgroundFetchAdapterFetchMax], "fetch-max", args) + args = p.maybeAddArgs(alt[VideoBackgroundFetchAdapterAPIHeader], "api-header", args) + args = p.maybeAddArgs(alt[VideoBackgroundFetchAdapterFetchPathPattern], "fetch-path-pattern", args) + args = p.maybeAddArgs(alt[VideoBackgroundFetchAdapterReplaceHost], "replace-host", args) + args = p.maybeAddArgs(alt[VideoBackgroundFetchAdapterNameSpace], "name-space", args) + args = p.maybeAddArgs(alt[VideoBackgroundFetchAdapterMetricsPrefix], "metrics-prefix", args) + args = p.maybeAddArgs(alt[VideoBackgroundFetchAdapterExactMatch], "exact-match", args) + args = p.maybeAddArgs(alt[VideoBackgroundFetchAdapterLogName], "log-name", args) + + // from here on out, it's parsing fetch_path_pattern + if _, found := alt[VideoBackgroundFetchAdapterFetchPathPattern]; !found { + return args, nil + } + + var fetchPatterns []string + switch val := alt[VideoBackgroundFetchAdapterFetchPathPattern].(type) { + case []interface{}: + for _, ipattern := range val { + pattern, ok := ipattern.(string) + if !ok { + return nil, util.FormatError(VideoBackgroundFetchAdapterFetchPathPattern, fmt.Errorf("Expected a string")) + } + fetchPatterns = append(fetchPatterns, pattern) + } + case []string: + fetchPatterns = val + default: + return nil, util.FormatError(VideoBackgroundFetchAdapterFetchPathPattern, fmt.Errorf("Expected an array of strings")) + } + + for _, pattern := range fetchPatterns { + args = p.maybeAddArgs(pattern, "fetch-path-pattern", args) + } + + return args, nil +} + +func isValidFetchPolicy(policy string) bool { + return policy == "simple" || strings.Contains(policy, "lru") +} + +func (p *VideoBackgroundFetchAdapter) maybeAddArgs(option interface{}, param string, args []string) []string { + value := util.ParamToValue(option) + switch param { + case "fetch-policy": + if isValidFetchPolicy(value) { + return append(args, fmt.Sprintf("--%s=%s", param, value)) + } + default: + if len(value) > 0 { + return append(args, fmt.Sprintf("--%s=%s", param, value)) + } + } + return args +} diff --git a/tools/voluspa/adapters/video_background_fetch_test.go b/tools/voluspa/adapters/video_background_fetch_test.go new file mode 100644 index 00000000000..b9ecb68b04b --- /dev/null +++ b/tools/voluspa/adapters/video_background_fetch_test.go @@ -0,0 +1,103 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package adapters + +import ( + "sort" + "testing" + + "github.com/apache/trafficserver/tools/voluspa" +) + +func TestVideoBackgroundFetch(t *testing.T) { + var plugin VideoBackgroundFetchAdapter + + alt := make(map[interface{}]interface{}) + ro := voluspa.RemapOptions{} + env := &voluspa.ConfigEnvironment{ + Property: "testVideoBackgroundFetch", + RemapOptions: ro, + } + + alt[VideoBackgroundFetchAdapterFetchCount] = "100" + alt[VideoBackgroundFetchAdapterFetchMax] = "1000" + alt[VideoBackgroundFetchAdapterFetchPolicy] = "simple" + alt[VideoBackgroundFetchAdapterAPIHeader] = "header1" + alt[VideoBackgroundFetchAdapterFetchPathPattern] = []string{"(.*)"} + alt[VideoBackgroundFetchAdapterReplaceHost] = "host1" + alt[VideoBackgroundFetchAdapterNameSpace] = "ns1" + alt[VideoBackgroundFetchAdapterMetricsPrefix] = "prefix1" + alt[VideoBackgroundFetchAdapterExactMatch] = "true" + alt[VideoBackgroundFetchAdapterLogName] = "log1" + + ro["video_background_fetch"] = alt + + pparams, err := plugin.PParams(env) + if err != nil { + t.Fatalf("Failed to generate output for plugin. error=%v", err) + } + + out := pparams[voluspa.DefaultRole] + sort.Sort(sort.StringSlice(out)) + + if len(out) == 0 { + t.Fatalf("Failed to generate output for plugin. no output.") + } + + if expected := "--api-header=header1"; out[0] != expected { + t.Fatalf("Error: got '%s'; expected '%s'", out[0], expected) + } + + if expected := "--exact-match=true"; out[1] != expected { + t.Fatalf("Error: got '%s'; expected '%s'", out[1], expected) + } + + if expected := "--fetch-count=100"; out[2] != expected { + t.Fatalf("Error: got '%s'; expected '%s'", out[2], expected) + } + + if expected := "--fetch-max=1000"; out[3] != expected { + t.Fatalf("Error: got '%s'; expected '%s'", out[3], expected) + } + + if expected := "--fetch-path-pattern=(.*)"; out[4] != expected { + t.Fatalf("Error: got '%s'; expected '%s'", out[4], expected) + } + + if expected := "--fetch-policy=simple"; out[5] != expected { + t.Fatalf("Error: got '%s'; expected '%s'", out[5], expected) + } + + if expected := "--log-name=log1"; out[6] != expected { + t.Fatalf("Error: got '%s'; expected '%s'", out[6], expected) + } + + if expected := "--metrics-prefix=prefix1"; out[7] != expected { + t.Fatalf("Error: got '%s'; expected '%s'", out[7], expected) + } + + if expected := "--name-space=ns1"; out[8] != expected { + t.Fatalf("Error: got '%s'; expected '%s'", out[8], expected) + } + + if expected := "--replace-host=host1"; out[9] != expected { + t.Fatalf("Error: got '%s'; expected '%s'", out[9], expected) + } +} diff --git a/tools/voluspa/attributes.go b/tools/voluspa/attributes.go new file mode 100644 index 00000000000..826fec24258 --- /dev/null +++ b/tools/voluspa/attributes.go @@ -0,0 +1,56 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package voluspa + +import "sort" + +// GetCDNs returns a list of CDNs in current config set +func (v *Voluspa) GetCDNs() []string { + cdns := make(map[string]interface{}) + for _, parsedConfig := range v.parsedConfigs { + for cdn := range parsedConfig.cdn { + cdns[cdn] = nil + } + } + + var all []string + for k := range cdns { + all = append(all, k) + } + sort.Strings(all) + + return all +} + +// GetRoles returns a list of Roles in current config set +func (v *Voluspa) GetRoles() []string { + roles := make(map[string]interface{}) + for _, parsedConfig := range v.parsedConfigs { + roles[parsedConfig.role] = nil + } + + var all []string + for k := range roles { + all = append(all, k) + } + sort.Strings(all) + + return all +} diff --git a/tools/voluspa/cdncheck.go b/tools/voluspa/cdncheck.go new file mode 100644 index 00000000000..2501d03b35f --- /dev/null +++ b/tools/voluspa/cdncheck.go @@ -0,0 +1,81 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package voluspa + +import ( + "fmt" +) + +type CDNTest struct { + Type CheckType `json:"type,omitempty" yaml:"type,omitempty"` + DomainNames []string `json:"domain_names,omitempty" yaml:"domain_names,omitempty"` + URLs []string `json:"urls,omitempty" yaml:"urls,omitempty"` + HostTypes []HostType `json:"host_types,omitempty" yaml:"host_types,omitempty"` + Description string `json:"description,omitempty" yaml:"description,omitempty"` + Role string `json:"role,omitempty" yaml:"role,omitempty"` + Range string `json:"range,omitempty" yaml:"range,omitempty"` + Headers map[string]string `json:"headers,omitempty" yaml:"headers,omitempty"` + PurgeBefore bool `json:"purge_before,omitempty" yaml:"purge_before,omitempty"` + Insecure bool `json:"insecure" yaml:"insecure"` + Success *CDNCheckSuccessCriteria `json:"success,omitempty" yaml:"success,omitempty"` + Ciphers []string `json:"ciphers,omitempty" yaml:"ciphers,omitempty"` +} + +type CDNCheckSuccessCriteria struct { + StatusCode int `json:"status_code" yaml:"status_code"` + Headers map[string]string `json:"headers,omitempty" yaml:"headers,omitempty"` + SerialNumbers []string `json:"serial_numbers,omitempty" yaml:"serial_numbers,omitempty"` +} + +type HostType string + +const ( + UnknownHostType HostType = "unknown" + ParentHost HostType = Parent + ChildHost HostType = Child +) + +type CheckType string + +const ( + UnknownCheckType CheckType = "unknown" + HTTPCheckType CheckType = "http" + SSLCheckType CheckType = "ssl" +) + +func (h *HostType) UnmarshalYAML(unmarshal func(interface{}) error) error { + var in string + if err := unmarshal(&in); err != nil { + return err + } + + if len(in) == 0 { + return fmt.Errorf("empty host type") + } + + switch in { + case Parent, Child: + *h = HostType(in) + default: + return fmt.Errorf("unknown host type '%s'", in) + } + + return nil +} diff --git a/tools/voluspa/cmd/voluspa/main.go b/tools/voluspa/cmd/voluspa/main.go new file mode 100644 index 00000000000..ad047e38966 --- /dev/null +++ b/tools/voluspa/cmd/voluspa/main.go @@ -0,0 +1,172 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package main + +import ( + "flag" + "fmt" + "log" + "os" + "sort" + "strings" + + "github.com/apache/trafficserver/tools/voluspa" + _ "github.com/apache/trafficserver/tools/voluspa/adapters" +) + +var ( + configDest = flag.String("dest", "out", "destination directory for written out configs") + installLocation = flag.String("install", voluspa.DefaultTrafficserverConfigurationDir, "trafficserver configuration location") + defaultsLocation = flag.String("defaults", "", "location of default configurations") + schemaLocation = flag.String("schema-location", "", "location of schema (defaults to cwd)") + disableAdapters = flag.String("disable-adapters", "", "a comma separated list of adapters to disable") + + cdnList = flag.String("cdns", "", "a comma separated list of CDNs") + roleList = flag.String("roles", "", "a comma separated list of roles") + showVersion = flag.Bool("version", false, "Show version") + verbose = flag.Bool("verbose", false, "Verbose output") + skipSchemaValidation = flag.Bool("skip-schema-validation", false, "skip schema validation") + treatRolesAsCDN = flag.Bool("promote-roles-to-cdn", false, "promote roles to CDN") + ramDiskRole = flag.String("ramdisk-role", "roles_trafficserver_ramdisk", "The role that indicates this host has a ramdisk volume") + + skipValidation = flag.Bool("skip-validation", false, "skip validation") + validationOnly = flag.Bool("validate-only", false, "stop after validation") + strictMode = flag.Bool("strict", false, "enable strict validation") + dumpSchemaVersion = flag.Int("dump-schema", -1, "dump this version of the schema (0 = highest) to STDOUT and exit 0") +) + +func init() { + log.SetFlags(0) + + flag.Parse() +} + +var exit = func(code int) { + os.Exit(code) +} + +func main() { + if *showVersion { + fmt.Println("voluspa", voluspa.Version) + exit(0) + } + + v, err := voluspa.NewVoluspaWithOptions(&voluspa.Options{ + Verbose: *verbose, + Destination: *configDest, + SkipSchemaValidation: *skipSchemaValidation, + PromoteRolesToCDN: *treatRolesAsCDN, + DefaultsLocation: *defaultsLocation, + SchemaLocation: *schemaLocation, + RamDiskRole: *ramDiskRole, + }, *installLocation) + if err != nil { + log.Printf("%s", err) + exit(1) + } + + if *dumpSchemaVersion >= 0 { + schema, err := v.SchemaDefinition(*dumpSchemaVersion) + if err != nil { + log.Printf("%s", err) + exit(1) + } + fmt.Fprint(os.Stdout, string(schema)) + exit(0) + } + + if len(*disableAdapters) > 0 { + adapters := strings.Split(*disableAdapters, ",") + for _, adapter := range adapters { + if err = voluspa.AdaptersRegistry.RemoveAdapterByType(adapter); err != nil { + log.Fatalf("Could not disable adapter %q: %s", adapter, err) + } + } + } + + if len(flag.Args()) == 0 { + log.Printf("Must specify at least 1 config file") + exit(1) + } + + filenames := flag.Args() + sort.Strings(filenames) + + for _, filename := range filenames { + err = v.AddConfig(filename) + if err != nil { + log.Printf("%s", err) + exit(1) + } + } + + if !*skipValidation { + if err = v.Validate(*strictMode); err == nil { + goto validationDone + } + + fmt.Fprintf(os.Stderr, "voluspa: errors validating configs:\n") + + errs, ok := err.(voluspa.Errors) + if !ok { + fmt.Fprintln(os.Stderr, err) + exit(1) + } + + // sort and unique-ify the set of errors + // like errs.Error() but also uniques and has different format + seen := make(map[string]interface{}) + var errOut []string + for _, e := range errs { + if _, found := seen[e.Error()]; found { + continue + } + errOut = append(errOut, fmt.Sprintf(" - %s", e.Error())) + seen[e.Error()] = nil + } + + sort.Strings(errOut) + + for _, str := range errOut { + fmt.Fprintln(os.Stderr, str) + } + exit(1) + } + +validationDone: + if *validationOnly { + exit(0) + } + + var cdns []string + if len(*cdnList) > 0 { + cdns = strings.Split(*cdnList, ",") + } + + var roles []string + if len(*roleList) > 0 { + roles = strings.Split(*roleList, ",") + } + + if err = v.WriteAllFiles(cdns, roles); err != nil { + log.Printf("%s", err) + exit(1) + } +} diff --git a/tools/voluspa/cmd/voluspa/main_test.go b/tools/voluspa/cmd/voluspa/main_test.go new file mode 100644 index 00000000000..7093562c366 --- /dev/null +++ b/tools/voluspa/cmd/voluspa/main_test.go @@ -0,0 +1,33 @@ +// +build testrunmain + +package main + +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import ( + "testing" +) + +func TestRunMain(t *testing.T) { + exit = func(code int) { + // ignore exit code + } + main() +} diff --git a/tools/voluspa/cmd/voluspa/no_test.go b/tools/voluspa/cmd/voluspa/no_test.go new file mode 100644 index 00000000000..822b16bcbd4 --- /dev/null +++ b/tools/voluspa/cmd/voluspa/no_test.go @@ -0,0 +1,20 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package main diff --git a/tools/voluspa/config_generator.go b/tools/voluspa/config_generator.go new file mode 100644 index 00000000000..18c62c1d6eb --- /dev/null +++ b/tools/voluspa/config_generator.go @@ -0,0 +1,567 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package voluspa + +import ( + "bytes" + "fmt" + "net/url" + "sort" + "strings" +) + +// NewCustomerConfig creates a new CustomerConfig from a PropertyConfig +func NewCustomerConfig(p *PropertyConfig, configFilename string, options Options) (*CustomerConfig, error) { + config, err := newCustomerConfig(p) + if err != nil { + return nil, err + } + + if options.PromoteRolesToCDN && isPromotableRole(p.Role) { + parts := strings.SplitN(p.Role, "_", 2) + if len(parts) == 2 { + // If the only CDN is DefaultCDN, purge and use the role + if len(p.CDN) == 1 && p.CDN[0] == DefaultCDN { + config.cdn = make(map[string]interface{}) + } + + config.cdn[parts[1]] = nil + config.role = "" + } + } + + config.filename = configFilename + + configLocations := []ConfigLocation{UnknownLocation} + if p.ParentChild { + configLocations = []ConfigLocation{ChildConfig, ParentConfig} + } + + var mappings []*Mapping + + // mappings are like explicit, but smarter + // They're parent/child aware, no longer requiring different rulesets for each type. + for _, m := range p.Mappings { + + schemes := p.Schemes + if len(m.Schemes) > 0 { + schemes = m.schemes() + } + + for _, alias := range m.Alias { + for _, scheme := range schemes { + for _, configLocation := range configLocations { + + incoming := formatURL(scheme, alias) + originURL := formatURL(scheme, m.Origin) + + // for parent/child, use the first incoming alias as the origin URL. on the parent side, + // we map it as the inbound + if configLocation == ChildConfig && m.RemapOptions.isParentChildEnabled() { + originURL = formatURL(scheme, m.Alias[0]) + } + + if configLocation == ParentConfig && m.RemapOptions.isParentChildEnabled() { + if len(m.Alias) > 0 && alias != m.Alias[0] { + continue + } + } + + // clone separate versions of RemapOptions with the appropriate key set + ro := m.RemapOptions.Clone() + + // clear out manually set attributes + delete(ro, Parent) + delete(ro, Child) + + switch configLocation { + case ParentConfig: + ro[Parent] = true + case ChildConfig: + ro[Child] = true + } + + mapping := &Mapping{ + Origin: originURL, + Alias: []string{incoming}, + Group: DefaultGroup, + Schemes: []string{scheme}, + RegexMap: m.RegexMap, + RuleName: m.RuleName, + RemapOptions: ro, + id: m.id, + ParentOverride: m.ParentOverride, + } + + if configLocation == ChildConfig && m.RemapOptions.isParentChildEnabled() { + // only include the first alias of an alias set. as seen above, we only map + // through the first alias to the parent + if alias == m.Alias[0] { + if len(m.ParentDestDomain) > 0 { + mapping.ParentDestDomain = m.ParentDestDomain + } else { + mapping.ParentDestDomain = stripScheme(incoming) + } + } + } + + mappings = append(mappings, mapping) + } + } + } + } + + for _, m := range p.ExplicitMappings { + m.explicit = true + mappings = append(mappings, m) + } + + for _, m := range mappings { + + schemes := p.Schemes + if len(m.Schemes) > 0 { + schemes = m.schemes() + } + + for _, scheme := range schemes { + for _, incoming := range m.Alias { + + for _, configLocation := range configLocations { + if !forCurrentMatch(configLocation, &m.RemapOptions) { + continue + } + + env := newConfigEnvironment(p, options, m.id, configLocation) + + mrules, err := convertToMappingRules(m.RemapOptions, env, m.explicit) + if err != nil { + return nil, err + } + + if configLocation == ChildConfig && m.RemapOptions.isParentChildEnabled() { + config.parentDomains = append(config.parentDomains, m.ParentDomain()) + } + + incomingURL := formatURL(scheme, incoming) + originURL := formatURL(scheme, m.Origin) + + remap := Remap{ + IncomingURL: incomingURL, + OriginURL: originURL, + Group: m.Group, + Role: m.Role, + RegexMap: m.RegexMap, + mappingRules: mrules, + ConfigLocation: env.ConfigLocation, + sourceConfig: env, + scheme: scheme, + parentOverrides: m.ParentOverride, + parentDomain: m.ParentDomain(), + } + + config.Remaps = append(config.Remaps, remap) + } + } + } + } + + // For now, sort if the config has mappings. Later, introduce the change for explicit and others + if len(p.Mappings) > 0 { + sort.Stable(sort.Reverse(byIncomingURL(config.Remaps))) + } + + return config, nil +} + +func stripScheme(in string) string { + if !strings.Contains(in, "/") { + return in + } + parts := strings.Split(in, "/") + return parts[2] +} + +func forCurrentMatch(configLocation ConfigLocation, ro *RemapOptions) bool { + if configLocation == ChildConfig && ro.isParentConfig() { + return false + } + if configLocation == ParentConfig && ro.isChildConfig() { + return false + } + return true +} + +func (m *Mapping) schemes() []string { + if len(m.Schemes) > 0 { + return m.Schemes + } + + return []string{""} +} + +// formatURL will create URL from scheme and incoming, unless incoming already has scheme applied to it +func formatURL(scheme, incoming string) string { + if strings.HasPrefix(incoming, "/") { + return incoming + } + + if len(scheme) == 0 { + return incoming + } + + u, err := url.Parse(incoming) + if err == nil && strings.HasPrefix(u.Scheme, "http") { + return incoming + } + + // in the error case, it could be a regex URL that url.Parse barfs on. try a brute force method + if strings.HasPrefix(incoming, "http:") || strings.HasPrefix(incoming, "https:") { + return incoming + } + + return fmt.Sprintf("%s://%s", scheme, incoming) +} + +// isParentChildEnabled returns the value specified by "parent_child". defaults to true +func (options *RemapOptions) isParentChildEnabled() bool { + value, err := options.ValueByNameAsBool("parent_child") + if err == nil { + return value + } + return true +} + +// isParentConfig returns the value specified by "parent". defaults to false +func (options *RemapOptions) isParentConfig() bool { + value, err := options.ValueByNameAsBool(Parent) + if err == nil { + return value + } + return false +} + +// isChildConfig returns the value specified by "child". defaults to false +func (options *RemapOptions) isChildConfig() bool { + value, err := options.ValueByNameAsBool(Child) + if err == nil { + return value + } + return false +} + +// Iterate over all "rules". adding to array of *ATSPLugin +// If a general adapter, use general logic +// If an action, use action logic +// otherwise, assume a compound adapter +// - group by type +// Iterate over groups of compounds (calculated above) +// Append to adapter array +// Sort array by weights and return + +func convertToMappingRules(ro RemapOptions, env *ConfigEnvironment, explicit bool) ([]mappingRule, error) { + var adapters []mappingRule + + compounds := make(map[AdapterType][]AdapterType) + + // stable ordering + var options []string + for option := range ro { + options = append(options, option) + } + sort.Strings(options) + + var errs Errors + + seen := make(map[Adapter]interface{}) + for _, option := range options { + if option == "storage_volume" { // prevent warning on rule property that isn't associated with an adapter + continue + } + adapterType := AdaptersRegistry.adapterTypeByConfigName(option) + if adapterType == UnknownAdapter { + // Warn when there's garbage in + if !IsValidConfigurationOption(option) { + errs = append(errs, fmt.Errorf("unknown configuration option %q", option)) + } + continue + } + + vp := AdaptersRegistry.adapterForType(adapterType) + if vp == nil { + errs = append(errs, fmt.Errorf("unknown adapter %q", adapterType)) + continue + } + + // NOTE: even though an adapter can have multiple options associated with it, + // it's the first invocation that only matters. Once inside PParams, the adapter + // is responsible for determining any other usage of itself + if _, ok := seen[vp]; ok { + continue + } + seen[vp] = nil + + if env.ConfigLocation != UnknownLocation { + vpcp, ok := vp.(ParentChildAdapter) + matches := false + if ok { + for _, loc := range vpcp.ConfigLocations() { + if loc == env.ConfigLocation { + matches = true + } + } + + if !matches && explicit { + // ignore and let through + } else if !matches { + continue + } + } + } + + env.RemapOptions = ro + + switch vp.PluginType() { + case GeneralAdapter: + adapter, err := createAdapter(vp, env) + if err != nil { + errs = append(errs, err) + continue + } + + if adapter.hasContent() { + adapters = append(adapters, *adapter) + } + case ActionAdapter: + adapter, err := createAction(vp, env) + if err != nil { + errs = append(errs, err) + continue + } + if adapter.hasContent() { + adapters = append(adapters, *adapter) + } + case CompoundAdapter: + cta, ok := vp.(CompoundTypeAdapter) + if !ok { + errs = append(errs, fmt.Errorf("compound adapter does not implement CompoundTypeAdapter")) + continue + } + compounds[cta.CompoundType()] = append(compounds[cta.CompoundType()], adapterType) + } + } + + for pt, sp := range compounds { + + sort.Sort(receiptsFirst(sp)) + + adapter, err := createCompoundMappingRule(pt, ro, sp, env) + if err != nil { + errs = append(errs, err) + continue + } + + if adapter.hasContent() { + adapters = append(adapters, *adapter) + } + + } + + sort.Stable(byRuleWeight(adapters)) + + if len(errs) > 0 { + return nil, errs + } + + return adapters, nil +} + +func createCompoundMappingRule(compoundType AdapterType, ro RemapOptions, adapters []AdapterType, env *ConfigEnvironment) (*mappingRule, error) { + vp := AdaptersRegistry.adapterForType(compoundType) + if vp == nil { + return nil, fmt.Errorf("unknown AdapterType %q", compoundType) + } + + var errs Errors + var content bytes.Buffer + + adapter := baseCreateAdapter(vp, env) + + ccontent, err := ConfigContent(vp, env) + if err != nil { + errs = append(errs, err) + } + adapter.ConfigContent = ccontent + + ccontent, err = SubConfigContent(vp, env) + if err != nil { + errs = append(errs, err) + } + + if ccontent != nil { + content.Write(ccontent.Bytes()) + } + + var buffer bytes.Buffer + for _, adapterType := range adapters { + vps := AdaptersRegistry.adaptersForType(adapterType) + for _, vp = range vps { + if vp == nil { + return nil, fmt.Errorf("unknown AdapterType %s", adapterType) + } + + // filter out unrelated types + cta, ok := vp.(CompoundTypeAdapter) + if ok && cta.CompoundType() != compoundType { + continue + } + + vpp, ok := vp.(ParameterAdapter) + if ok { + pparams, err := vpp.PParams(env) + if err != nil { + errs = append(errs, err) + } + + for _, val := range pparams { + fmt.Fprintf(&buffer, " @pparam=%s", val) + } + } + + env.RemapOptions = ro + val, err := SubConfigContent(vp, env) + if err != nil { + errs = append(errs, err) + } + + if val != nil && val.Len() > 0 { + content.Write(val.Bytes()) + content.WriteString("\n") + } + } + } + + if len(errs) > 0 { + return nil, errs + } + + if buffer.Len() > 0 { + adapter.ConfigContent.WriteString(buffer.String()) + } else if content.Len() == 0 { + // if no subconfig content, then there's nothing to be output (at least at this time) + adapter.ConfigContent = &bytes.Buffer{} + } + + adapter.SubConfigContent = &content + + return adapter, nil +} + +func baseCreateAdapter(vp Adapter, env *ConfigEnvironment) *mappingRule { + adapter := newMappingRule(vp) + + scp, ok := vp.(SubConfigAdapter) + if ok { + adapter.SubConfigFileName = SubConfigFilename(scp, env) + } + + return adapter +} + +func createAdapter(vp Adapter, env *ConfigEnvironment) (*mappingRule, error) { + adapter := baseCreateAdapter(vp, env) + + content, err := ConfigContent(vp, env) + if err != nil { + return nil, err + } + adapter.ConfigContent = content + + content, err = SubConfigContent(vp, env) + if err != nil { + return nil, err + } + adapter.SubConfigContent = content + + return adapter, nil +} + +func createAction(vp Adapter, env *ConfigEnvironment) (*mappingRule, error) { + adapter := baseCreateAdapter(vp, env) + content, err := SubConfigContent(vp, env) + if err != nil { + return nil, err + } + adapter.SubConfigContent = content + adapter.ConfigContent = content + + return adapter, nil +} + +// receiptsFirst implements sort.Interface, sorting by AdapterType, bubbling Receipt to the top. +type receiptsFirst []AdapterType + +func (s receiptsFirst) Len() int { return len(s) } +func (s receiptsFirst) Swap(i, j int) { s[i], s[j] = s[j], s[i] } +func (s receiptsFirst) Less(i, j int) bool { return s[i] == Receipt } + +// byRuleWeight implements sort.Interface, sorting by a Rule's Weight. +type byRuleWeight []mappingRule + +func (s byRuleWeight) Len() int { return len(s) } +func (s byRuleWeight) Swap(i, j int) { s[i], s[j] = s[j], s[i] } +func (s byRuleWeight) Less(i, j int) bool { return s[i].Weight < s[j].Weight } + +// byIncomingURL will sort by incoming URL domain +// for the same scheme and inbound host: +// http will be put before https +// empty paths come last +// sorting alphabetically +// ignoring host-less and regex remaps +type byIncomingURL []Remap + +func (s byIncomingURL) Len() int { return len(s) } +func (s byIncomingURL) Swap(i, j int) { s[i], s[j] = s[j], s[i] } +func (s byIncomingURL) Less(i, j int) bool { + if s[i].RegexMap || s[j].RegexMap { + return false + } + + u1, _ := url.Parse(s[i].IncomingURL) + u2, _ := url.Parse(s[j].IncomingURL) + if u1 == nil || u2 == nil { + return len(s[i].IncomingURL) < len(s[j].IncomingURL) + } + + if len(u1.Host) == 0 || len(u2.Host) == 0 { + return false + } + + if u1.Host == u2.Host { + if u1.Scheme == u2.Scheme { + if len(u1.Path) == 0 || len(u2.Path) == 0 { + return u1.Path < u2.Path + } + return u1.Path > u2.Path + } + return !(u1.Scheme == "http" || u2.Scheme == "http") + } + + return u1.Path < u2.Path +} diff --git a/tools/voluspa/config_writer.go b/tools/voluspa/config_writer.go new file mode 100644 index 00000000000..73abed44011 --- /dev/null +++ b/tools/voluspa/config_writer.go @@ -0,0 +1,186 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package voluspa + +import ( + "bytes" + "fmt" + "io/ioutil" + "log" + "os" + "path/filepath" + "strings" +) + +type FilesystemWriter struct { + options *Options +} + +func NewFilesystemWriter(options *Options) (*FilesystemWriter, error) { + fw := &FilesystemWriter{options: options} + err := os.MkdirAll(fw.options.Destination, 0755) + if err != nil { + return nil, err + } + return fw, nil +} + +func (f *FilesystemWriter) WriteFiles(files []ManagedFile) error { + groups := make(map[string]interface{}) + for _, mf := range files { + groups[mf.Role] = nil + } + + for _, managedFile := range files { + if _, exists := groups[managedFile.Role]; managedFile.Role != "" && !exists { + continue + } + + filename := fmt.Sprintf("%s/%s", f.options.Destination, managedFile.Filename) + if f.options.Verbose { + log.Printf("Writing %s", filename) + } + + err := os.MkdirAll(filepath.Dir(filename), 0755) + if err != nil { + return err + } + + err = ioutil.WriteFile(filename, managedFile.Contents.Bytes(), 0644) + if err != nil { + return err + } + } + return nil +} + +type ConfigExpander struct { + options *Options +} + +func NewConfigExpander(options *Options) *ConfigExpander { + return &ConfigExpander{options: options} +} + +func (c *ConfigExpander) ExpandRemapConf(parsedConfig *CustomerConfig) ([]ManagedFile, error) { + var managedFiles []ManagedFile + + groups := make(map[string]bool) + for _, config := range parsedConfig.Remaps { + groups[config.Group] = true + } + + configLocations := []ConfigLocation{UnknownLocation} + if parsedConfig.parentChild { + configLocations = []ConfigLocation{ChildConfig, ParentConfig} + } + + for _, configLocation := range configLocations { + for group := range groups { + + var buf bytes.Buffer + buf.WriteString(generatedFileBanner) + + var hasContent bool + for _, config := range parsedConfig.Remaps { + if group != DefaultGroup && config.Group != group { + continue + } + + if configLocation != UnknownLocation && config.ConfigLocation != configLocation { + continue + } + + var roleEnabled bool + if len(config.Role) > 0 { + roleEnabled = true + buf.WriteString(fmt.Sprintf("{%% if salt.pillar.get(\"%s\") %%}\n\n", config.Role)) + } + + buf.WriteString(config.asRemapConf()) + buf.WriteString("\n") + + if roleEnabled { + buf.WriteString("{% endif %}\n\n") + } + + hasContent = true + } + + filename := fmt.Sprintf("%s/%s", strings.ToLower(parsedConfig.property), parsedConfig.remapConfigFilename(group, configLocation)) + + if !hasContent { + return nil, fmt.Errorf("Configuration for %s empty. Not writing %s", parsedConfig.property, filename) + } + + for cdn := range parsedConfig.cdn { + managedFiles = append(managedFiles, NewManagedFile(filename, filename, cdn, parsedConfig.role, parsedConfig.property, &buf, configLocation)) + } + } + } + + return managedFiles, nil +} + +func (c *ConfigExpander) ExpandSubConfigs(parsedConfig *CustomerConfig) ([]ManagedFile, error) { + var managedFiles []ManagedFile + + for _, remap := range parsedConfig.Remaps { + managedSubFiles, err := c.expandSubConfig(parsedConfig, remap) + if err != nil { + return nil, err + } + managedFiles = append(managedFiles, managedSubFiles...) + } + return managedFiles, nil +} + +func (c *ConfigExpander) expandSubConfig(parsedConfig *CustomerConfig, remap Remap) ([]ManagedFile, error) { + var managedFiles []ManagedFile + for i := range remap.mappingRules { + adapter := remap.mappingRules[i] + value := adapter.SubConfigContent + + if len(adapter.SubConfigFileName) == 0 { + continue + } + if value == nil || value.Len() == 0 { + continue + } + + var buf bytes.Buffer + buf.WriteString(generatedFileBanner) + buf.Write(value.Bytes()) + + // append newline if config does not end with one + if value.Bytes()[len(value.Bytes())-1] != '\n' { + buf.WriteByte('\n') + } + + // TODO new method off of PCC? + filename := fmt.Sprintf("%s/%s", strings.ToLower(parsedConfig.property), adapter.SubConfigFileName) + + for cdn := range parsedConfig.cdn { + managedFiles = append(managedFiles, NewManagedFile(filename, filename, cdn, parsedConfig.role, parsedConfig.property, &buf, remap.ConfigLocation)) + } + } + + return managedFiles, nil +} diff --git a/tools/voluspa/consts.go b/tools/voluspa/consts.go new file mode 100644 index 00000000000..d51a0f98c71 --- /dev/null +++ b/tools/voluspa/consts.go @@ -0,0 +1,30 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package voluspa + +const ( + // DefaultTrafficserverConfigurationDir is the default install dir for ATS + DefaultTrafficserverConfigurationDir = "/opt/ats/etc/trafficserver" + + // Child is the label for child hosts, configurations or attributes + Child = "child" + // Parent is the label for parent hosts, configurations, or attributes + Parent = "parent" +) diff --git a/tools/voluspa/customer_config.go b/tools/voluspa/customer_config.go new file mode 100644 index 00000000000..59960d10ae0 --- /dev/null +++ b/tools/voluspa/customer_config.go @@ -0,0 +1,197 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package voluspa + +import ( + "bytes" + "fmt" + "strings" +) + +const ( + // DefaultGroup is the default group name + DefaultGroup = "all" + + DefaultRole = "default" + + remapPrefixBuffer = " " + remapMaxLineLength = 100 + + // generatedFileBanner is the text prepended to all config files + generatedFileBanner = "# Code generated by Voluspa. DO NOT EDIT.\n\n" +) + +// NOTE CustomerConfig and PropertyConfig are closely related. +// Every time I think I want to embed one in the other (or share a common struct), +// I'm reminded that the YAML parser does not handle embedded structs. + +type CustomerConfig struct { + owner string + reference string + lifecycle LifecycleState + property string + role string + filename string + sslCertNames []string + qps int + parentChild bool + parentDomains []string + schemes []string + cdn map[string]interface{} + Remaps []Remap + HAProxyVIPs []HAProxyVIP +} + +type Remap struct { + OriginURL string + IncomingURL string + Group string + Role string + scheme string + RegexMap bool + ConfigLocation ConfigLocation + mappingRules []mappingRule + sourceConfig *ConfigEnvironment + parentOverrides []ParentOverride + parentDomain string +} + +// newCustomerConfig creates a new CustomerConfig from a PropertyConfig +func newCustomerConfig(p *PropertyConfig) (*CustomerConfig, error) { + cc := &CustomerConfig{ + property: p.Property, + sslCertNames: p.SSLCertNames, + parentChild: p.ParentChild, + lifecycle: p.Lifecycle, + qps: p.QPS, + owner: p.Owner, + reference: p.Reference, + role: p.Role, + cdn: make(map[string]interface{}), + schemes: p.Schemes, + HAProxyVIPs: p.HAProxyVIPs, + } + for _, cdn := range p.CDN { + cc.cdn[cdn] = nil + } + return cc, nil +} + +func (c Remap) hasContent() bool { + hasContent := false + for i := range c.mappingRules { + adapter := c.mappingRules[i] + + if adapter.hasContent() { + hasContent = true + } + } + return hasContent +} + +func splitLine(value, prefix string) string { + if len(value) < remapMaxLineLength { + return value + } + + if strings.Contains(value, "{%") { + return value + } + + words := strings.Split(value, " ") + if len(words) < 2 { + return value + } + + out := bytes.Buffer{} + out.WriteString(fmt.Sprintf("%s %s", words[0], words[1])) + + for i := 2; i < len(words); i++ { + if words[i-1] == "@pparam=--config" { + out.WriteString(fmt.Sprintf(" %s", words[i])) + continue + } + out.WriteString(fmt.Sprintf(" \\\n%s %s", prefix, words[i])) + } + return out.String() + +} + +func (c Remap) asRemapConf() string { + var buf bytes.Buffer + if c.RegexMap { + buf.WriteString(fmt.Sprintf("regex_map")) + } else { + buf.WriteString(fmt.Sprintf("map")) + } + buf.WriteString(fmt.Sprintf(" %s \\\n %s", c.IncomingURL, c.OriginURL)) + + if !c.hasContent() { + buf.WriteString("\n") + return buf.String() + } + + buf.WriteString(" \\\n") + + for i := range c.mappingRules { + adapter := c.mappingRules[i] + + if !adapter.hasContent() { + continue + } + + prefixBuffer := remapPrefixBuffer + content := adapter.ConfigContent.String() + if adapter.isCommandLineTemplated() && strings.HasPrefix(content, "{%") { + prefixBuffer = "" + } + + buf.WriteString(fmt.Sprintf("%s%s", prefixBuffer, splitLine(content, prefixBuffer))) + if adapter.isCommandLineTemplated() { + continue + } + + if i < len(c.mappingRules)-1 && c.mappingRules[i+1].hasContent() { + buf.WriteString(" \\") + } + buf.WriteString("\n") + } + + return buf.String() +} + +func (c *CustomerConfig) remapConfigFilename(group string, configLocation ConfigLocation) string { + return configFilename(strings.ToLower(c.property), group, configLocation) +} + +func configFilename(baseFilename, group string, configLocation ConfigLocation) string { + if group == DefaultGroup || group == "" { + if configLocation == ParentConfig { + return fmt.Sprintf("%s_parent.config", baseFilename) + } + return fmt.Sprintf("%s.config", baseFilename) + } + + if configLocation == ParentConfig { + return fmt.Sprintf("%s_%s_parent.config", baseFilename, group) + } + + return fmt.Sprintf("%s_%s.config", baseFilename, group) +} diff --git a/tools/voluspa/doc.go b/tools/voluspa/doc.go new file mode 100644 index 00000000000..59005d41277 --- /dev/null +++ b/tools/voluspa/doc.go @@ -0,0 +1,100 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/* +Package voluspa generates ATS configuration files for a CDN from a set of +simple YAML files that express the desired behaviour of each of the hosted properties. + +For example, given this property specification: + + schema_version: 1 + owner: Web Group + rcpt: httpbin + + mappings: + - alias: + - httpbin.example.com + origin: https://httpbin.origin.example.com + schemes: [http, https] + + rules: + default: + priority: background + origin_host_header: origin + deny_methods: [ CONNECT ] + +Voluspa will generate the corresponding ATS configuration files in the +specified output directory. A `httpbin/httpbin.config` file: + + # Code generated by Voluspa. DO NOT EDIT. + map http://httpbin.example.com \ + https://https://httpbin.origin.example.com \ + @plugin=conf_remap.so @pparam=proxy.config.url_remap.pristine_host_hdr=0 \ + @plugin=header_rewrite.so @pparam=httpbin/hdrs.config \ + @plugin=propstats.so @pparam=--prefix=proxy.domain.httpbin \ + @action=deny @method=CONNECT + +and a `httpbin/hdrs.config` file: + + # Code generated by Voluspa. DO NOT EDIT. + cond %{REMAP_PSEUDO_HOOK} + set-header @ReceiptService "httpbin{{hosttype}}" + cond %{REMAP_PSEUDO_HOOK} + set-conn-dscp 8 + +The voluspa package enables fine control over the generation of configurations +and enables functionality to be extended via "adapters". + +The distribution includes a `voluspa` command line utility that loads a set of +YAML files and generates the ATS configuration files using code like this: + + import ( + "flag" + "fmt" + "log" + + "github.com/apache/trafficserver/tools/voluspa" + _ "github.com/apache/trafficserver/tools/voluspa/adapters" + ) + + v, err := voluspa.NewVoluspa() + if err != nil { + log.Fatalf("voluspa: %s", err) + } + + // Load the YAML config file for each property in the CDN + for _, filename := range flag.Args() { + err = v.AddConfig(filename) + if err != nil { + log.Fatalf("%s", err) // err includes the filename + } + } + + // Validate the set of configurations, individually and as a whole + if err = v.Validate(true); err != nil { + log.Fatalf("%s", err) + } + + // Generate the resulting ATS config files + if err = v.WriteAllATSFiles(); err != nil { + log.Fatalf("%s", err) + } + +*/ +package voluspa diff --git a/tools/voluspa/haproxy.go b/tools/voluspa/haproxy.go new file mode 100644 index 00000000000..798633e5e8f --- /dev/null +++ b/tools/voluspa/haproxy.go @@ -0,0 +1,81 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package voluspa + +import ( + "bytes" + "fmt" + "strings" +) + +// HAProxyVIP describes a haproxy VIP +type HAProxyVIP struct { + Port int + RIPs []RIP +} + +// RIP is the real IP and Port for a HAProxyVIP +type RIP struct { + IP string + Port int +} + +type HAProxyConfigGenerator struct { +} + +func (h *HAProxyConfigGenerator) Do(parsedConfigs []*CustomerConfig, merged bool) ([]ManagedFile, error) { + var results []ManagedFile + for _, parsedConfig := range parsedConfigs { + if len(parsedConfig.HAProxyVIPs) == 0 { + continue + } + + var buf bytes.Buffer + buf.WriteString(generatedFileBanner) + + for _, hapConfig := range parsedConfig.HAProxyVIPs { + + buf.WriteString(fmt.Sprintf("frontend %s 127.0.0.1:%d\n", parsedConfig.property, hapConfig.Port)) + buf.WriteString(" default_backend app\n") + buf.WriteString(" mode http\n") + buf.WriteString("\n") + buf.WriteString(fmt.Sprintf("backend %s\n", parsedConfig.property)) + buf.WriteString(" mode http\n") + buf.WriteString(" balance roundrobin\n") + buf.WriteString(" option httpchk GET / HTTP/1.1\\r\\nHost:localhost\n") + + for i, rip := range hapConfig.RIPs { + buf.WriteString(fmt.Sprintf(" server %s%03d %s:%d check\n", parsedConfig.property, i+1, rip.IP, rip.Port)) + } + + filename := fmt.Sprintf("%s/%s.hap.config", strings.ToLower(parsedConfig.property), strings.ToLower(parsedConfig.property)) + + results = append(results, ManagedFile{ + Filename: filename, + Role: "", + Property: parsedConfig.property, + Contents: &buf, + ConfigType: UnknownLocation, + }) + } + } + + return results, nil +} diff --git a/tools/voluspa/hosting_config.go b/tools/voluspa/hosting_config.go new file mode 100644 index 00000000000..315dfe00c1e --- /dev/null +++ b/tools/voluspa/hosting_config.go @@ -0,0 +1,213 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package voluspa + +import ( + "bytes" + "fmt" + "net/url" + "sort" + "strings" +) + +type HostingConfigurator struct { + options *Options +} + +const ( + ramVolString string = "{{ ramdisk_volume }}" + defaultVolString string = "{{ default_volumes }}" + + HostingConfigDefaultFilename = "hosting.config_default" +) + +type StorageType int + +const ( + ramVolume StorageType = iota + diskVolume +) + +// hostingHostnameMap maps hostnames to StorageType +type hostingHostnameMap map[string]StorageType + +type hostingHostname struct { +} + +func newHostingHostname(hn string) *hostingHostname { + return &hostingHostname{} +} + +func newHostingConfigurator(options *Options) *HostingConfigurator { + return &HostingConfigurator{ + options: options, + } +} + +func (h *HostingConfigurator) Do(parsedConfigs []*CustomerConfig, merge bool) ([]ManagedFile, error) { + if merge { + return h.get(parsedConfigs, NoCDN) + } + + // otherwise, group properties by CDN and generate hosting.config for each CDN separately + grouped := groupCustomerConfigsByCDN(parsedConfigs) + + var managedFiles []ManagedFile + for cdn, configs := range grouped { + files, err := h.get(configs, cdn) + if err != nil { + return nil, err + } + managedFiles = append(managedFiles, files...) + } + + return managedFiles, nil + +} + +func (h *HostingConfigurator) get(parsedConfigs []*CustomerConfig, cdn string) ([]ManagedFile, error) { + + hostingHostnames := make(map[string]hostingHostnameMap) + for _, parsedConfig := range parsedConfigs { + role := parsedConfig.role + if hostingHostnames[role] == nil { + hostingHostnames[role] = make(hostingHostnameMap) + } + for _, remap := range parsedConfig.Remaps { + destURL := remap.OriginURL + if remap.sourceConfig.RemapOptions.HasOptionSet("origin_host_header") { + value, err := remap.sourceConfig.RemapOptions.ValueByNameAsString("origin_host_header") + if err != nil { + return nil, err + } + if value == "alias" { + destURL = remap.IncomingURL + } + } + u, err := url.Parse(destURL) + if err != nil { + return nil, err + } + storageConfig := "disk_volume" // default + if remap.sourceConfig.RemapOptions.HasOptionSet("storage_volume") { + value, err := remap.sourceConfig.RemapOptions.ValueByNameAsString("storage_volume") + if err != nil { + return nil, err + } + storageConfig = value + } + if storageConfig == "ramdisk_volume" { + hostingHostnames[role][u.Host] = ramVolume + } else { + hostingHostnames[role][u.Host] = diskVolume + } + } + } + + return h.ExpandConfigTemplate(hostingHostnames, cdn) +} + +func (h *HostingConfigurator) ExpandConfigTemplate(hostingHostnamesByRole map[string]hostingHostnameMap, cdn string) ([]ManagedFile, error) { + seen := make(map[string]bool) + + var buf bytes.Buffer + buf.WriteString(generatedFileBanner) + buf.WriteString(fmt.Sprintf("{%% if salt.pillar.get(\"%s\") %%}\n", h.options.RamDiskRole)) + + var roles []string + for k := range hostingHostnamesByRole { + roles = append(roles, k) + } + sort.Strings(roles) + + for _, role := range roles { + hostingHostnames := hostingHostnamesByRole[role] + + var hostnames []string + for hostname := range hostingHostnames { + hostnames = append(hostnames, hostname) + } + + if len(hostnames) == 0 { + continue + } + + sort.Strings(hostnames) + + hasRamVolumeHosts := false + for _, hostname := range hostnames { + storage := hostingHostnames[hostname] + if storage == ramVolume { + hasRamVolumeHosts = true + break + } + } + + if !hasRamVolumeHosts { + continue + } + + buf.WriteString(h.startRoleGuard(role)) + + for _, hostname := range hostnames { + if _, exists := seen[hostname]; exists { + continue + } + + storage := hostingHostnames[hostname] + if storage == ramVolume { + seen[hostname] = true + buf.WriteString(fmt.Sprintf("hostname=%s %s\n", hostname, ramVolString)) + } + } + buf.WriteString(h.endRoleGuard(role)) + } + + buf.WriteString(fmt.Sprintf("hostname=* %s\n", defaultVolString)) + + buf.WriteString("{% else %}\n# hosting.config disabled on this host\n{% endif %}\n") + + filename := HostingConfigDefaultFilename + if len(cdn) > 0 && (cdn != DefaultCDN && cdn != NoCDN) { + filename = fmt.Sprintf("hosting.config_%s", cdn) + } + + return []ManagedFile{NewManagedFile(filename, "hosting.config", cdn, "", "", &buf, UnknownLocation)}, nil + +} + +func (h *HostingConfigurator) startRoleGuard(role string) string { + if len(role) == 0 { + return "" + } + + roleName := role + if !strings.HasPrefix(role, "roles_") { + roleName = fmt.Sprintf("roles_%s", role) + } + return fmt.Sprintf("{%% if salt.pillar.get('%s') %%}\n", roleName) +} + +func (h *HostingConfigurator) endRoleGuard(role string) string { + if len(role) == 0 { + return "" + } + return "{% endif %}\n\n" +} diff --git a/tools/voluspa/internal/util/regex/regex.go b/tools/voluspa/internal/util/regex/regex.go new file mode 100644 index 00000000000..f5717b40687 --- /dev/null +++ b/tools/voluspa/internal/util/regex/regex.go @@ -0,0 +1,24 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package regex + +type Validator interface { + IsValid(in string) (bool, error) +} diff --git a/tools/voluspa/internal/util/regex/regex_test.go b/tools/voluspa/internal/util/regex/regex_test.go new file mode 100644 index 00000000000..ad438a75b92 --- /dev/null +++ b/tools/voluspa/internal/util/regex/regex_test.go @@ -0,0 +1,45 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package regex + +import ( + "testing" +) + +func TestGoLangRegex(t *testing.T) { + testCases := []struct { + u string + e bool + }{ + {u: "asdf", e: true}, + {u: `^[-_A-Za-z0-9\.]+$`, e: true}, + {u: "", e: true}, + {u: " ^/([^/]*-eu-dub-[^/]*)/(.*) https://s3-eu-west-1.amazonaws.com$0", e: true}, + {u: "[", e: false}, + } + + glr := &GoLangRegex{} + for _, tc := range testCases { + e, err := glr.IsValid(tc.u) + if e != tc.e { + t.Errorf("%q: expected %t got %t: %s", tc.u, tc.e, e, err) + } + } +} diff --git a/tools/voluspa/internal/util/regex/vregex.go b/tools/voluspa/internal/util/regex/vregex.go new file mode 100644 index 00000000000..cde553967f2 --- /dev/null +++ b/tools/voluspa/internal/util/regex/vregex.go @@ -0,0 +1,33 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package regex + +import ( + "regexp" +) + +// GoLangRegex implements regex.Validator interface using go's builtin regex methods +type GoLangRegex struct { +} + +func (e *GoLangRegex) IsValid(in string) (bool, error) { + _, err := regexp.Compile(in) + return err == nil, err +} diff --git a/tools/voluspa/lifecycle.go b/tools/voluspa/lifecycle.go new file mode 100644 index 00000000000..c59186feaa3 --- /dev/null +++ b/tools/voluspa/lifecycle.go @@ -0,0 +1,29 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package voluspa + +type LifecycleState string + +const ( + UnknownLifecycleState LifecycleState = "" + Onboarding LifecycleState = "onboarding" + Live LifecycleState = "live" + Retired LifecycleState = "retired" +) diff --git a/tools/voluspa/managed_file.go b/tools/voluspa/managed_file.go new file mode 100644 index 00000000000..48971622998 --- /dev/null +++ b/tools/voluspa/managed_file.go @@ -0,0 +1,51 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package voluspa + +import ( + "bytes" + "fmt" +) + +type ManagedFile struct { + Filename string + RemoteFilename string + Role string + CDN string + Property string + Contents *bytes.Buffer + ConfigType ConfigLocation +} + +func NewManagedFile(filename, remoteFilename, cdn, role, property string, contents *bytes.Buffer, configType ConfigLocation) ManagedFile { + return ManagedFile{ + Filename: filename, + RemoteFilename: remoteFilename, + Role: role, + CDN: cdn, + Property: property, + Contents: contents, + ConfigType: configType, + } +} + +func (mf ManagedFile) String() string { + return fmt.Sprintf("Filename=%s RFN=%s Role=%s CDN=%s Property=%s Size=%d Type=%d", mf.Filename, mf.RemoteFilename, mf.Role, mf.CDN, mf.Property, mf.Contents.Len(), mf.ConfigType) +} diff --git a/tools/voluspa/options.go b/tools/voluspa/options.go new file mode 100644 index 00000000000..6888ede3903 --- /dev/null +++ b/tools/voluspa/options.go @@ -0,0 +1,30 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package voluspa + +type Options struct { + Verbose bool + SkipSchemaValidation bool + PromoteRolesToCDN bool + Destination string + DefaultsLocation string + SchemaLocation string + RamDiskRole string +} diff --git a/tools/voluspa/parent_config.go b/tools/voluspa/parent_config.go new file mode 100644 index 00000000000..5b825d12080 --- /dev/null +++ b/tools/voluspa/parent_config.go @@ -0,0 +1,393 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package voluspa + +import ( + "bytes" + "fmt" + "net/url" + "sort" + "strings" +) + +// parentDomainMap maps domain names to parentDomain struct +type parentDomainMap map[string]*parentDomain + +type ParentConfigurator struct { + options *Options +} + +type Override struct { + schemes map[string]bool + primaryList []string + secondaryList []string + HTTPPort int + HTTPSPort int + strategy string + goDirect bool + ignoreQueryString bool + overrideDomain string +} + +type parentDomain struct { + originalDomain string + schemes map[string]bool + ignoreQueryString bool + OverridesByRole map[string]Override +} + +func newParentDomain(domain string) *parentDomain { + return &parentDomain{originalDomain: domain, schemes: make(map[string]bool), OverridesByRole: make(map[string]Override)} +} + +func newParentConfigurator(options *Options) *ParentConfigurator { + return &ParentConfigurator{ + options: options, + } +} + +func (p *ParentConfigurator) Do(parsedConfigs []*CustomerConfig, merge bool) ([]ManagedFile, error) { + if merge { + return p.get(parsedConfigs, NoCDN) + } + + // otherwise, group properties by CDN and generate parent.config for each CDN separately + grouped := groupCustomerConfigsByCDN(parsedConfigs) + + var managedFiles []ManagedFile + for cdn, configs := range grouped { + files, err := p.get(configs, cdn) + if err != nil { + return nil, err + } + managedFiles = append(managedFiles, files...) + } + + return managedFiles, nil + +} + +func (p *ParentConfigurator) get(parsedConfigs []*CustomerConfig, cdn string) ([]ManagedFile, error) { + // a map of roles to parentDomainMaps + parentDomains := make(map[string]parentDomainMap) + for _, parsedConfig := range parsedConfigs { + + role := parsedConfig.role + if _, exists := parentDomains[role]; !exists { + parentDomains[role] = make(parentDomainMap) + } + + for _, domain := range parsedConfig.parentDomains { + if parsedConfig.lifecycle == Retired { + continue + } + + for _, remap := range parsedConfig.Remaps { + if remap.ConfigLocation == ParentConfig { + continue + } + + pd, ok := parentDomains[role][domain] + if !ok { + pd = newParentDomain(domain) + parentDomains[role][domain] = pd + } + + // extract schemes from toplevel or from remap rules + if len(remap.scheme) > 0 { + pd.schemes[remap.scheme] = true + } else { + for _, scheme := range parsedConfig.schemes { + pd.schemes[scheme] = true + } + } + + scheme, err := extractScheme(remap.IncomingURL) + if err != nil { + return nil, fmt.Errorf("could not extract scheme from %q: %s", remap.IncomingURL, err) + } + + if len(scheme) > 0 { + pd.schemes[scheme] = true + } + + if len(remap.parentOverrides) > 0 && remap.parentDomain == pd.originalDomain { + for _, o := range remap.parentOverrides { + ignore := o.IgnoreQuerystring + if hasRemoveAllParamsOption(remap.sourceConfig.RemapOptions) { + ignore = true + } + pd.OverridesByRole[o.Role] = Override{ + primaryList: o.PrimaryList, + secondaryList: o.SecondaryList, + HTTPPort: o.HTTPPort, + HTTPSPort: o.HTTPSPort, + ignoreQueryString: ignore, + strategy: o.Strategy, + goDirect: o.GoDirect, + overrideDomain: o.DestDomain, + } + } + } + + if remap.sourceConfig.RemapOptions == nil { + continue + } + + // We obey if it was set for any remap + if hasRemoveAllParamsOption(remap.sourceConfig.RemapOptions) || hasIgnoreQuery(remap.sourceConfig.RemapOptions) { + pd.ignoreQueryString = true + } + + } + } + } + + return p.ExpandConfigTemplate(parentDomains, cdn) +} + +func extractScheme(in string) (string, error) { + parsedURL, err := url.Parse(in) + if err == nil { + return parsedURL.Scheme, nil + } + + if strings.HasPrefix(in, "https:") { + return "https", nil + } + + if strings.HasPrefix(in, "http:") { + return "http", nil + } + + return "", nil +} + +func hasRemoveAllParamsOption(remapOptions RemapOptions) bool { + vals, err := remapOptions.ValueByNameAsStringMapInterface("cachekey") + if err != nil { + return false + } + + val, ok := vals["remove_all_params"] + if !ok { + return false + } + + optionValue, ok := val.(bool) + if !ok { + return false + } + + return optionValue +} + +func hasIgnoreQuery(remapOptions RemapOptions) bool { + vals, err := remapOptions.ValueByNameAsStringMapInterface("cachekey") + if err != nil { + return false + } + + val, ok := vals["parent_selection"] + if !ok { + return false + } + + optionValue, ok := val.(string) + if !ok { + return false + } + return optionValue == "ignore_query" +} + +func (p *ParentConfigurator) ExpandConfigTemplate(parentDomains map[string]parentDomainMap, cdn string) ([]ManagedFile, error) { + var files []ManagedFile + for role := range parentDomains { + parentDomains, ok := parentDomains[role] + if !ok { + return nil, fmt.Errorf("role %q not found", role) + } + + mf, err := p.expandConfigTemplate(parentDomains, cdn, role) + if err != nil { + return nil, err + } + + if mf == nil { + continue + } + + files = append(files, *mf) + + // ATS wants a file on parent hosts. match name that's used in salt + filename := "parent_empty.config" + if len(role) > 0 { + filename = fmt.Sprintf("parent_empty.config_%s", role) + } + + emptyParentConfig := NewManagedFile(filename, "parent.config", "", role, "", intentionallyEmptyMessage, ParentConfig) + files = append(files, emptyParentConfig) + } + + return files, nil +} + +// DomainNames allows for implementing the sort interface; regular string sort is not good enough +// we want the longest match to win in the case of bar.com and foo.bar.com. +type DomainNames []string + +// var _ sort.Interface = DomainNames{} + +func (d DomainNames) Len() int { + return len(d) +} +func (d DomainNames) Swap(i, j int) { + d[i], d[j] = d[j], d[i] +} + +func (d DomainNames) Less(i, j int) bool { + + if len(strings.Split(d[i], ".")) != len(strings.Split(d[j], ".")) { + return len(strings.Split(d[i], ".")) > len(strings.Split(d[j], ".")) + } + if strings.Compare(d[i], d[j]) < 0 { + return true + } + return false +} + +var intentionallyEmptyMessage = bytes.NewBufferString("# This file is intentionally left blank\n") + +func (p *ParentConfigurator) expandConfigTemplate(parentDomains parentDomainMap, cdn, role string) (*ManagedFile, error) { + seen := make(map[string]bool) + + var buf bytes.Buffer + buf.WriteString("\n") + + buf.WriteString(generatedFileBanner) + var domains DomainNames + for _, pd := range parentDomains { + domains = append(domains, pd.originalDomain) + } + sort.Sort(domains) + + for _, domain := range domains { + + if _, exists := seen[domain]; exists { + continue + } + + pd := parentDomains[domain] + + var schemes []string + for scheme := range pd.schemes { + schemes = append(schemes, scheme) + } + sort.Strings(schemes) + + for _, scheme := range schemes { + if len(pd.OverridesByRole) > 0 { + + // sort roles to make diffs more predictable + roles := make([]string, 0) + for role := range pd.OverridesByRole { + roles = append(roles, role) + } + sort.Strings(roles) + + for _, role := range roles { + override := pd.OverridesByRole[role] + port := override.HTTPPort + if scheme == "https" { + port = override.HTTPSPort + } + pstr := fmt.Sprintf(`parent="{{ %s_parents }}" {{ secondary_%s_parents }}`, scheme, scheme) // the default + if len(override.primaryList) > 0 { + primaryString := "parent=\"" + secondaryString := "secondary_parent=\"" + for _, h := range override.primaryList { + primaryString += fmt.Sprintf("%s:%d,", h, port) + } + primaryString = strings.TrimSuffix(primaryString, ",") + primaryString += "\"" + if len(override.secondaryList) > 0 { + for _, h := range override.secondaryList { + secondaryString += fmt.Sprintf("%s:%d,", h, port) + } + secondaryString = strings.TrimSuffix(secondaryString, ",") + secondaryString += "\"" + } else { + secondaryString = "" + } + pstr = fmt.Sprintf("%s %s", primaryString, secondaryString) + } + if role != "" { + buf.WriteString(fmt.Sprintf("\n{%% if salt.pillar.get(\"%s\") %%}\n", role)) + } + strategy := override.strategy + if strategy == "" { + strategy = "consistent_hash" + } + direct := override.goDirect // false is default? + finalDomain := domain + if override.overrideDomain != "" { + finalDomain = override.overrideDomain + } + buf.WriteString(fmt.Sprintf(`dest_domain=%s scheme=%s %s round_robin=%s go_direct=%t`, finalDomain, scheme, pstr, strategy, direct)) + if override.ignoreQueryString { + buf.WriteString(" qstring=ignore") + } + buf.WriteString("\n") + if role != "" { + buf.WriteString("{% endif %}\n\n") + } + } + } else { + // This is the "legacy way" with parent_dest_domwain at the mapping level NOTE DEPRECATED + buf.WriteString(fmt.Sprintf( + `dest_domain=%s scheme=%s parent="{{ %s_parents }}" {{ secondary_%s_parents }} round_robin=consistent_hash go_direct=false`, domain, scheme, scheme, scheme)) + if pd.ignoreQueryString { + buf.WriteString(" qstring=ignore") + } + buf.WriteString("\n") + } + } + seen[domain] = true + } + + buf.WriteString("\n") + + if len(seen) == 0 { + return nil, nil + } + + filename := "parent.config_default" + if len(role) > 0 { + filename = fmt.Sprintf("parent.config_%s", role) + } + + if cdn != DefaultCDN && cdn != NoCDN { + filename = fmt.Sprintf("parent.config_%s", cdn) + } + + mf := NewManagedFile(filename, "parent.config", cdn, role, "", &buf, ChildConfig) + + return &mf, nil +} diff --git a/tools/voluspa/parent_config_test.go b/tools/voluspa/parent_config_test.go new file mode 100644 index 00000000000..b4558690cfd --- /dev/null +++ b/tools/voluspa/parent_config_test.go @@ -0,0 +1,47 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package voluspa + +import ( + "testing" +) + +func TestExtractScheme(t *testing.T) { + testCases := []struct { + in string + out string + }{ + {"https://store-(fred|123|test).domain.com/", "https"}, + {"http://store-(bob|test).domain.com/", "http"}, + {"http://www.domain.com/", "http"}, + {"www.domain.com/", ""}, + } + for i, tc := range testCases { + scheme, err := extractScheme(tc.in) + if err != nil { + t.Errorf("test #%d: unexpected error: %s", i, err) + continue + } + + if scheme != tc.out { + t.Errorf("test #%d: Sorted order expected %q got %q", i, tc.out, scheme) + } + } +} diff --git a/tools/voluspa/property_config.go b/tools/voluspa/property_config.go new file mode 100644 index 00000000000..fb327b6382f --- /dev/null +++ b/tools/voluspa/property_config.go @@ -0,0 +1,326 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package voluspa + +import ( + "fmt" + "io/ioutil" + "net/url" + "os" + "regexp" + "sort" + "unicode" + + yaml "gopkg.in/yaml.v2" +) + +// PropertyConfig represents the raw parsed structure from YAML +type PropertyConfig struct { + Owner string `json:"owner,omitempty" yaml:",omitempty"` + Reference string `json:"reference,omitempty" yaml:",omitempty"` + Lifecycle LifecycleState `json:"lifecycle,omitempty" yaml:",omitempty"` + Role string `json:"role,omitempty" yaml:",omitempty"` + Property string `json:"rcpt,omitempty" yaml:"rcpt,omitempty"` + SchemaVersion string `json:"schema_version,omitempty" yaml:"schema_version,omitempty"` + SSLCertNames []string `json:"ssl_cert_names,omitempty" yaml:"ssl_cert_names,omitempty"` + Schemes []string `json:"schemes,omitempty" yaml:",omitempty"` + CDN []string `json:"cdn,omitempty" yaml:",omitempty"` + Org string `json:"org,omitempty" yaml:"org,omitempty"` + Rules map[string]RemapOptions + ExplicitMappings []*Mapping `json:"explicit,omitempty" yaml:"explicit,omitempty"` + Mappings []*Mapping `json:"mappings,omitempty" yaml:"mappings,omitempty"` + ParentChild bool `json:"parent_child,omitempty" yaml:"parent_child,omitempty"` + QPS int `json:"qps" yaml:"qps"` + HAProxyVIPs []HAProxyVIP `json:"ha_proxy,omitempty" yaml:"ha_proxy,omitempty"` + Tests map[string]CDNTest `json:"tests,omitempty" yaml:"tests,omitempty"` + Receiptsd *ReceiptsdConfig `json:"receiptsd,omitempty" yaml:"receiptsd,omitempty"` +} + +type Mapping struct { + Origin string `json:"origin,omitempty" yaml:"origin,omitempty"` + Group string `json:"group,omitempty" yaml:"group,omitempty"` + Role string `json:"role,omitempty" yaml:"role,omitempty"` + Alias []string `json:"alias,omitempty" yaml:"alias,omitempty"` + Schemes []string `json:"schemes,omitempty" yaml:"schemes,omitempty"` + + RuleName string `json:"rule" yaml:"rule"` + ParentDestDomain string `json:"parent_dest_domain,omitempty" yaml:"parent_dest_domain,omitempty"` + ParentOverride []ParentOverride `json:"parent_override,omitempty" yaml:"parent_override,omitempty"` + RegexMap bool `json:"regex_map" yaml:"regex_map"` + RemapOptions RemapOptions `json:",omitempty" yaml:",omitempty"` + explicit bool + id int +} + +type ParentOverride struct { + Role string `json:"role,omitempty" yaml:"role,omitempty"` + DestDomain string `json:"dest_domain,omitempty" yaml:"dest_domain"` + PrimaryList []string `json:"primary_list,omitempty" yaml:"primary_list,omitempty"` + SecondaryList []string `json:"secondary_list,omitempty" yaml:"secondary_list,omitempty"` + Strategy string `json:"strategy,omitempty" yaml:"strategy,omitempty"` + IgnoreQuerystring bool `json:"ignore_querystring,omitempty" yaml:"ignore_querystring,omitempty"` + HTTPPort int `json:"http_port,omitempty" yaml:"http_port,omitempty"` + HTTPSPort int `json:"https_port,omitempty" yaml:"https_port,omitempty"` + GoDirect bool `json:"go_direct,omitempty" yaml:"go_direct,omitempty"` +} + +func (m *Mapping) ParentDomain() string { + + // a po without a role but with a domain + for _, po := range m.ParentOverride { + if po.Role == "" && len(po.DestDomain) > 0 { + return po.DestDomain + } + } + + // Deprecated + if len(m.ParentDestDomain) > 0 { + return m.ParentDestDomain + } + // the domain is not overriden, old style or new style, return the origin domain + parentURL, err := url.Parse(m.Origin) + if err == nil && len(parentURL.Scheme) > 0 { + return parentURL.Host + } + + return m.Origin +} + +const ( + NoCDN = "" + NoRole = "" + DefaultCDN = "edge" + DefaultScheme = "http" + DefaultRulesetName = "default" +) + +var propertyNameRE = regexp.MustCompile(`^[-_A-Za-z0-9\.]+$`) + +func isValidPropertyName(in string) bool { + return propertyNameRE.MatchString(in) +} + +// Setup validates and sets up a PropertyConfig +func (p *PropertyConfig) Setup() error { + + if len(p.Schemes) > 0 { + for _, scheme := range p.Schemes { + if !isValidScheme(scheme) { + return fmt.Errorf("invalid/unsupported scheme %q", scheme) + } + } + } + + // rcpt is the public-facing name of the field, Property is the internal name + if len(p.Property) == 0 { + return fmt.Errorf("rcpt is required") + } + + if !isValidPropertyName(p.Property) { + return fmt.Errorf("property name can only contain alphanumeric characters") + } + + // assign an id for each rule name + // sort keys/path for stable ids + matchIds := make(map[string]int) + { + var keys []string + for k := range p.Rules { + keys = append(keys, k) + } + sort.Strings(keys) + + i := 1 + for _, k := range keys { + matchIds[k] = i + i++ + } + } + + for _, m := range p.Mappings { + if m.RuleName == "" { + if _, ok := p.Rules["default"]; ok { + m.RuleName = "default" + } + } + + if m.Group == "" { + m.Group = DefaultGroup + } + + m.id = matchIds[m.RuleName] + + if ruleset, ok := p.Rules[m.RuleName]; ok { + m.RemapOptions = ruleset + } else { + return fmt.Errorf("invalid ruleset name: %q for origin %q", m.RuleName, m.Origin) + } + + if len(m.Alias) == 0 { + return fmt.Errorf("no alias specified for origin %q", m.Origin) + } + if hasLoop(m) { + return fmt.Errorf("alias and origin %q will loop", m.Origin) + } + } + + for _, m := range p.ExplicitMappings { + if m.RuleName == "" { + if _, ok := p.Rules[DefaultRulesetName]; ok { + m.RuleName = DefaultRulesetName + } + } + + if m.Group == "" { + m.Group = DefaultGroup + } + + m.id = matchIds[m.RuleName] + + if ruleset, ok := p.Rules[m.RuleName]; ok { + m.RemapOptions = ruleset + } else { + return fmt.Errorf("invalid ruleset name: %q for origin %q", m.RuleName, m.Origin) + } + + if len(m.Alias) == 0 { + return fmt.Errorf("no alias specified for origin %q", m.Origin) + } + } + + return nil +} + +func hasLoop(m *Mapping) bool { + for _, scheme := range m.schemes() { + for _, incoming := range m.Alias { + incomingURL := formatURL(scheme, incoming) + originURL := formatURL(scheme, m.Origin) + if incomingURL == originURL { + return true + } + } + } + return false +} + +func (p *PropertyConfig) loadDefaultConfig(defaultsLocation, filename string) error { + var fullPath string + if len(defaultsLocation) > 0 { + fullPath = fmt.Sprintf("%s/%s", defaultsLocation, filename) + } else { + fullPath = filename + } + if _, err := os.Stat(fullPath); os.IsNotExist(err) { + return nil + } + + cfgContents, err := ioutil.ReadFile(fullPath) + if err != nil { + return err + } + + return p.parseConfig(cfgContents) +} + +func isASCII(in string) (bool, rune) { + for _, c := range in { + if c > unicode.MaxASCII { + return false, c + } + } + return true, 0 +} + +func (p *PropertyConfig) parseConfig(cfgContents []byte) error { + valid, char := isASCII(string(cfgContents)) + if !valid { + return fmt.Errorf("non-ASCII characters not supported: invalid rune %q", string(char)) + } + + return yaml.Unmarshal(cfgContents, &p) +} + +func (p *PropertyConfig) mergeConfig(c *PropertyConfig) { + overrides, ok := c.Rules[DefaultRulesetName] + if !ok { + return + } + + for _, ruleset := range p.Rules { + if ruleset == nil { + continue + } + for k, v := range overrides { + if _, exists := ruleset[k]; !exists { + ruleset[k] = v + } + } + } +} + +func newPropertyConfig() *PropertyConfig { + return &PropertyConfig{ + Rules: make(map[string]RemapOptions), + Schemes: []string{DefaultScheme}, + CDN: []string{DefaultCDN}, + Lifecycle: Live, + } +} + +// NewPropertyConfig creates a new PropertyConfig from a buffer containing a YAML file +func NewPropertyConfig(buffer []byte) (*PropertyConfig, error) { + return NewPropertyConfigWithDefaults(buffer, "") +} + +// NewPropertyConfigWithDefaults creates a new PropertyConfig from a buffer containing a YAML file +func NewPropertyConfigWithDefaults(buffer []byte, defaultsLocation string) (*PropertyConfig, error) { + cfg := newPropertyConfig() + + if err := cfg.parseConfig(buffer); err != nil { + return nil, fmt.Errorf("Could not load YAML file. err=%s", err) + } + + return inflatePropertyConfig(cfg, defaultsLocation) +} + +func inflatePropertyConfig(cfg *PropertyConfig, defaultsLocation string) (*PropertyConfig, error) { + dcfg := newPropertyConfig() + + if err := dcfg.loadDefaultConfig(defaultsLocation, "default.conf"); err != nil { + return nil, fmt.Errorf("Could not load %q: %s", "defaults/default.conf", err) + } + + cfg.mergeConfig(dcfg) + + if cfg.ParentChild { + dpCfg := newPropertyConfig() + + if err := dpCfg.loadDefaultConfig(defaultsLocation, "default_parent.conf"); err != nil { + return nil, fmt.Errorf("Could not load %q: %s", "defaults/default_parent.conf", err) + } + cfg.mergeConfig(dpCfg) + } + + if err := cfg.Setup(); err != nil { + return nil, err + } + return cfg, nil +} diff --git a/tools/voluspa/property_remaps.go b/tools/voluspa/property_remaps.go new file mode 100644 index 00000000000..19685ab70a7 --- /dev/null +++ b/tools/voluspa/property_remaps.go @@ -0,0 +1,168 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package voluspa + +import ( + "bytes" + "fmt" + "strings" +) + +type PropertyRemapGenerator struct { + options *Options +} + +func newPropertyRemapGenerator(options *Options) *PropertyRemapGenerator { + return &PropertyRemapGenerator{options: options} +} + +func (p *PropertyRemapGenerator) Do(parsedConfigs []*CustomerConfig, merge bool) ([]ManagedFile, error) { + if len(parsedConfigs) == 0 { + return nil, ErrMinimumConfigsNotMet + } + + var managedFiles []ManagedFile + for _, parsedConfig := range parsedConfigs { + if parsedConfig.lifecycle == Retired { + continue + } + expanded, err := p.expandRemapConf(parsedConfig) + if err != nil { + return nil, fmt.Errorf("Could not write remap config for property %q: %s", parsedConfig.property, err) + } + + managedFiles = append(managedFiles, expanded...) + + expanded, err = p.expandSubConfigs(parsedConfig) + if err != nil { + return nil, fmt.Errorf("Could not write subconfigs for property %q: %s", parsedConfig.property, err) + } + + managedFiles = append(managedFiles, expanded...) + } + + return managedFiles, nil +} + +func (p *PropertyRemapGenerator) expandRemapConf(parsedConfig *CustomerConfig) ([]ManagedFile, error) { + var managedFiles []ManagedFile + + groups := make(map[string]bool) + for _, config := range parsedConfig.Remaps { + groups[config.Group] = true + } + + configLocations := []ConfigLocation{UnknownLocation} + if parsedConfig.parentChild { + configLocations = []ConfigLocation{ChildConfig, ParentConfig} + } + + for _, configLocation := range configLocations { + for group := range groups { + + var buf bytes.Buffer + buf.WriteString(generatedFileBanner) + + var hasContent bool + for _, config := range parsedConfig.Remaps { + if group != DefaultGroup && config.Group != group { + continue + } + + if configLocation != UnknownLocation && config.ConfigLocation != configLocation { + continue + } + + var roleEnabled bool + if len(config.Role) > 0 { + roleEnabled = true + buf.WriteString(fmt.Sprintf("{%% if salt.pillar.get(\"%s\") %%}\n\n", config.Role)) + } + + buf.WriteString(config.asRemapConf()) + buf.WriteString("\n") + + if roleEnabled { + buf.WriteString("{% endif %}\n\n") + } + + hasContent = true + } + + filename := fmt.Sprintf("%s/%s", strings.ToLower(parsedConfig.property), parsedConfig.remapConfigFilename(group, configLocation)) + + if !hasContent { + return nil, fmt.Errorf("Configuration for %s empty. Not writing %s", parsedConfig.property, filename) + } + + for cdn := range parsedConfig.cdn { + managedFiles = append(managedFiles, NewManagedFile(filename, filename, cdn, parsedConfig.role, parsedConfig.property, &buf, configLocation)) + } + } + } + + return managedFiles, nil +} + +func (p *PropertyRemapGenerator) expandSubConfigs(parsedConfig *CustomerConfig) ([]ManagedFile, error) { + var managedFiles []ManagedFile + + for _, remap := range parsedConfig.Remaps { + managedSubFiles, err := p.expandSubConfig(parsedConfig, remap) + if err != nil { + return nil, err + } + managedFiles = append(managedFiles, managedSubFiles...) + } + return managedFiles, nil +} + +func (p *PropertyRemapGenerator) expandSubConfig(parsedConfig *CustomerConfig, remap Remap) ([]ManagedFile, error) { + var managedFiles []ManagedFile + for i := range remap.mappingRules { + adapter := remap.mappingRules[i] + value := adapter.SubConfigContent + + if len(adapter.SubConfigFileName) == 0 { + continue + } + if value == nil || value.Len() == 0 { + continue + } + + var buf bytes.Buffer + buf.WriteString(generatedFileBanner) + buf.Write(value.Bytes()) + + // append newline if config does not end with one + if value.Bytes()[len(value.Bytes())-1] != '\n' { + buf.WriteByte('\n') + } + + // TODO new method off of PCC? + filename := fmt.Sprintf("%s/%s", strings.ToLower(parsedConfig.property), adapter.SubConfigFileName) + + for cdn := range parsedConfig.cdn { + managedFiles = append(managedFiles, NewManagedFile(filename, filename, cdn, parsedConfig.role, parsedConfig.property, &buf, remap.ConfigLocation)) + } + } + + return managedFiles, nil +} diff --git a/tools/voluspa/receiptsd.go b/tools/voluspa/receiptsd.go new file mode 100644 index 00000000000..6ee8434bae8 --- /dev/null +++ b/tools/voluspa/receiptsd.go @@ -0,0 +1,32 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package voluspa + +// ReceiptsdConfig is the config for receiptsd managed by voluspa +type ReceiptsdConfig struct { + IgnoredExtensions []string `json:"ignored_extensions,omitempty" yaml:"ignored_extensions"` + + Sampling *struct { + Posters Posters `json:"posters,omitempty" yaml:"posters"` + } `json:"sampling,omitempty"` +} + +// Posters is a key/value map of poster name to % to post +type Posters map[string]string diff --git a/tools/voluspa/registry.go b/tools/voluspa/registry.go new file mode 100644 index 00000000000..9bbcbf5a54e --- /dev/null +++ b/tools/voluspa/registry.go @@ -0,0 +1,132 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package voluspa + +import ( + "fmt" + "log" +) + +var AdaptersRegistry *AdapterRegistry + +func init() { + AdaptersRegistry = NewAdapterRegistry() +} + +// AdapterRegistry is the central registry for all plugin config plugins +type AdapterRegistry struct { + adapters map[AdapterType]Adapter + compoundAdapters map[AdapterType][]Adapter + configNameAdapterMap map[string]AdapterType + fileNameMap map[string]AdapterType +} + +func NewAdapterRegistry() *AdapterRegistry { + return &AdapterRegistry{ + adapters: make(map[AdapterType]Adapter), + compoundAdapters: make(map[AdapterType][]Adapter), + configNameAdapterMap: make(map[string]AdapterType), + fileNameMap: make(map[string]AdapterType), + } +} + +func (r *AdapterRegistry) AddAdapter(p Adapter) { + if p.PluginType() == CompoundAdapter { + r.compoundAdapters[p.Type()] = append(r.compoundAdapters[p.Type()], p) + } else { + r.adapters[p.Type()] = p + } + + for _, v := range p.ConfigParameters() { + if v == "" { + continue + } + if previous, ok := r.configNameAdapterMap[v]; !ok { + r.configNameAdapterMap[v] = p.Type() + } else { + log.Printf("Duplicate config parameter: %s for %s. Previous AdapterType=%s", v, p.Type(), previous) + } + } + + scp, ok := p.(SubConfigAdapter) + if !ok { + return + } + if scp.Name() == "" { + return + } + if previous, ok := r.fileNameMap[scp.Name()]; !ok { + r.fileNameMap[scp.Name()] = p.Type() + } else { + log.Printf("Duplicate subconfig filename: %s for %s. Previous AdapterType=%s", p.Type(), scp.Name(), previous) + } +} + +func (r *AdapterRegistry) adapterForType(t AdapterType) Adapter { + vp := r.adapters[t] + if vp != nil { + return vp + } + + plugins := r.compoundAdapters[t] + if len(plugins) > 0 { + return plugins[0] + } + return nil +} + +func (r *AdapterRegistry) RemoveAdapterByType(adapterType string) error { + at := AdapterType(adapterType) + + var adapter Adapter + + nullAdapter, found := r.adapters[AdapterType("null")] + if !found { + return fmt.Errorf("%q adapter not found", "null") + } + + adapter, found = r.adapters[at] + if found { + r.adapters[at] = nullAdapter + } + + if !found { + return fmt.Errorf("unknown adapter type %q", adapterType) + } + + // override keywords + for _, v := range adapter.ConfigParameters() { + r.configNameAdapterMap[v] = nullAdapter.Type() + } + + return nil +} + +func (r *AdapterRegistry) adaptersForType(t AdapterType) []Adapter { + return r.compoundAdapters[t] +} + +func (r *AdapterRegistry) adapterTypeByConfigName(key string) AdapterType { + v, ok := r.configNameAdapterMap[key] + if !ok { + return UnknownAdapter + } + return v +} diff --git a/tools/voluspa/remap_options.go b/tools/voluspa/remap_options.go new file mode 100644 index 00000000000..e61d0c269f7 --- /dev/null +++ b/tools/voluspa/remap_options.go @@ -0,0 +1,173 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package voluspa + +import ( + "encoding/json" + "fmt" + "strconv" +) + +// RemapOptions key/value pairs from yaml configuration +type RemapOptions map[string]interface{} + +func (r RemapOptions) HasOptionSet(name string) bool { + var isSet bool + switch v := r[name].(type) { + case string: + isSet = len(v) > 0 + case bool: + isSet = v + default: + return false + } + return isSet +} + +func (r RemapOptions) Clone() RemapOptions { + ro := RemapOptions{} + for k, v := range r { + ro[k] = v + } + return ro +} + +func (r RemapOptions) HasOption(name string) bool { + _, ok := r[name] + return ok +} + +func (r RemapOptions) AddOption(name string, val interface{}) bool { + r[name] = val + return true +} + +func (r RemapOptions) RemoveOption(name string) bool { + delete(r, name) + return true +} + +func (r RemapOptions) ValueByNameAsBool(name string) (bool, error) { + v, ok := r[name].(bool) + if !ok { + return false, fmt.Errorf("value for %q is not a boolean: %v", name, r[name]) + } + return v, nil +} + +func (r RemapOptions) ValueByNameAsString(name string) (string, error) { + v, ok := r[name].(string) + if !ok { + return "", fmt.Errorf("value for %q is not a string: %v", name, r[name]) + } + return v, nil +} + +func (r RemapOptions) ValueByNameAsStringMapString(name string) (map[string]string, error) { + v, ok := r[name].(map[interface{}]interface{}) + if !ok { + return nil, fmt.Errorf("value for %q is of unexpected type, \"%T\". expected a map", name, r[name]) + } + + newv := make(map[string]string) + for k, v := range v { + newv[k.(string)] = v.(string) + } + return newv, nil +} + +func (r RemapOptions) ValueByNameAsInt(key string) (int, error) { + v, ok := r[key].(int) + if !ok { + return 0, fmt.Errorf("value for %q is not a integer: %v", key, r[key]) + } + return v, nil +} + +func (r RemapOptions) ValueByNameAsSlice(key string) ([]string, error) { + switch t := r[key].(type) { + case []interface{}: + vs := make([]string, len(t)) + for i, d := range t { + var val string + switch d := d.(type) { + case string: + val = d + case int: + val = strconv.Itoa(d) + } + + vs[i] = val + } + return vs, nil + case []string: + vs := make([]string, len(t)) + copy(vs, t) + + return vs, nil + } + return []string{}, fmt.Errorf("field %s is unhandled type: %T", key, r[key]) +} + +type remapOptionsJSON map[string]interface{} + +func (r RemapOptions) makeRemapOptionsJSONStruct() (remapOptionsJSON, error) { + rmo := remapOptionsJSON{} + for k, v := range r { + im, ok := r[k].(map[interface{}]interface{}) + if ok { + m := make(map[string]interface{}) + for ik, iv := range im { + sik, ok := ik.(string) + if !ok { + return nil, fmt.Errorf("non-string for key in map %+v", im) + } + m[sik] = iv + } + rmo[k] = m + continue + } + rmo[k] = v + } + return rmo, nil +} + +func (r RemapOptions) MarshalJSON() ([]byte, error) { + roj, err := r.makeRemapOptionsJSONStruct() + if err != nil { + return nil, err + } + return json.Marshal(roj) +} + +func (r RemapOptions) ValueByNameAsStringMapInterface(name string) (map[string]interface{}, error) { + v, ok := r[name].(map[interface{}]interface{}) + if !ok { + return nil, fmt.Errorf("value is of unexpected type. name=%s val=%v\n r=%+v\nt=%T", name, r[name], r, v) + } + + newv := make(map[string]interface{}) + for k, v := range v { + if v != nil { + newv[k.(string)] = v.(interface{}) + } + } + return newv, nil +} diff --git a/tools/voluspa/remap_options_test.go b/tools/voluspa/remap_options_test.go new file mode 100644 index 00000000000..faccbaf0d13 --- /dev/null +++ b/tools/voluspa/remap_options_test.go @@ -0,0 +1,66 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package voluspa + +import "testing" + +func TestRemapOptions_ValueByNameAsBool(t *testing.T) { + ro := &RemapOptions{} + for _, v := range []bool{true, false} { + (*ro)["key"] = v + + out, err := ro.ValueByNameAsBool("key") + if err != nil || out != v { + t.Fatalf("Expected value '%t', got '%t'", v, out) + } + } + + _, err := ro.ValueByNameAsBool("non-existent-key") + if err == nil { + t.Fatalf("Expected err getting value by non-existent key") + } + + (*ro)["key2"] = "stringval" + _, err = ro.ValueByNameAsBool("key2") + if err == nil { + t.Fatalf("Expected err getting boolean value for key") + } +} + +func TestRemapOptions_ValueByNameAsString(t *testing.T) { + ro := &RemapOptions{} + (*ro)["key"] = "value" + + out, err := ro.ValueByNameAsString("key") + if err != nil || out != "value" { + t.Fatalf("Expected value '%s', got '%s'", (*ro)["key"], out) + } + + _, err = ro.ValueByNameAsString("non-existent-key") + if err == nil { + t.Fatalf("Expected err getting value by non-existent key") + } + + (*ro)["key2"] = false + _, err = ro.ValueByNameAsString("key2") + if err == nil { + t.Fatalf("Expected err getting string value for key") + } +} diff --git a/tools/voluspa/sample.conf-template b/tools/voluspa/sample.conf-template new file mode 100644 index 00000000000..7685dd4bfba --- /dev/null +++ b/tools/voluspa/sample.conf-template @@ -0,0 +1,178 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +schema_version: '1.0' +owner: Owner Name +reference: reference number here +rcpt: MyReceipt +parent_child: true # use parent/child hierarchy +qps: 2 # Approximate queries per second to order rules +role: roles_edge_child # If this should be applied to specific salt role. +lifecycle: onboarding|live|retired #live: in production; onboarding: various things may not yet be working; retired: No longer active (no need for new ssl certs, etc); +cdn: [edge, internal] +ssl_cert_names: # a list of SSL cert filenames. used to populate ssl_multicert.config + - /secret/mysecret1.domain.com + - /secret/mysecret2.domain.com + +mappings: + - alias: + - mydomain1.com # can have multiple. If no scheme, uses from schemes: + origin: http://myorigin.domain.com + schemes: [http, https] # applied to alias and origin (if not specified) + rule: default #not needed if "default" + +rules: + default: # named set of rules to apply to this target/origin + parent_child: false # child remap goes direct to origin. (Overrides metadata's parent_child for this ruleset) + strip_query: true # strips the query string completely -- the origin won't see it. + force_ttl: 7d # Always set Cache-Control: max-age: $time; overwrites if origin had set. + default_ttl: 7d # If the Origin doesn't set a Cache-Control, set it to Cache-Control: max-age: $time + negative_ttl: 10s # amount of time to cache http errors (404, 502, 504, etc) + transaction_timeout: 10m # Sets transaction timeouts (CDN default is 30s) + content_type_forge: # Change the content type based on a series of regexes (Need this line too) + # 'regex': content-type. urls matching regex will have have the Content-type header set to content-type + '.*\.json$': application/json + '.*\.css$': text/css + edge_compression: # if the origin does not surpport compression, CDN should do the compression instead + enabled: true + algorithms: [gzip] # only gzip for now + static_origin_compressible_content_type: + - application/json #compress only these mime types -- not the defaults + header_rewrite: |- # raw values to put into header_rewrite configration + cond %{SEND_REQUEST_HDR_HOOK} + set-header Authorization "Basic 123412341234" + deny_methods: [ CONNECT, POST, PUT, DELETE ] #Reject these HTTP verbs + priority: background | foreground | streamingaudio | streamingvideo # QOS priority (DSCP bits) + set_header: 'X-Client-Protocol "%"'# Set this header; overwrites if origin had set. + add_header: 'X-Client-Protocol "%"' # add this header; + remove_header: "X-Client-Protocol" # remove this header; + add_header_origin: 'X-Client-Protocol "%"' # Send this header to the origin + set_header_origin: 'X-Client-Protocol "%"' # Send this header to the origin; overwrites if client had set. + proxy_cache_control: "MyProxyHeader" # Use this header's value for Cache-Control. + disable_cache: true # turn off the cache for this rule + regex_remap: "^/([^/]+)/(.*) https://$1.s3.amazonaws.com/$2" # use regex to match the inbound path (only) and substition for origin. Can add optional http status code (like @status=301). Can also use "|-" format + conf_remap: |- # raw values to override specific ATS settings. + CONFIG proxy.config.http.connect_attempts_timeout INT 600 + CONFIG proxy.config.http.negative_caching_enabled FLOAT .050 + allow_ip: [10.0.0.0-10.0.255.255, 10.13.0.0-10.13.255.255] # Only accept requests from these IP ranges + log_cookie: # log specific cookies for this property. Logger-cookie-Name1 will become the key name in logger + Logger-cookie-Name1: NameOfCookie1 + Logger-cookie-Name2: NameOfCookie2 + log_header: # log specific header for this property. Logger-header-Name1 will become the key name in logger + Logger-header-Name1: NameOfHeader1 + Logger-header-Name2: NameOfHeader2 + log_type: private|public # Private logs no personal info (no client IP, url, referalurl ...); Public is default + cachekey: #modify the cache key + include_params: ["param1", "param2"] # only use these query parameters in the cachekey + exclude_params: ["param1", "param2"] # ignore these query parameters in the cachekey + sort_query: true # sort the query parameters for the cachekey + remove_all_params: true # Don't use the query string as part of the cachekey. + include_headers: ["Origin", "Content-Type"] # include specific headers in the cachekey + include_cookies: ["Cookie1", "Cookie2"] # include specific cookies in the cachekey + static_prefix: SomeString # specified value will be added to the cachekey + regex_replace_path: '/.*/filename.html/' # replace cachekey URI path with regex capture + cache_promote: # cache promotion plugin options -- only cache the popular objects + policy: lru | chance + sample: 25 + lru_hits: 10 + lru_buckets: 10000 + failover: # escalate plugin options -- when object is unavailable, try another origin + domain: domain.com + status_codes: [401, 403, 404, 407, 410, 500, 501, 502, 503, 504, 505] + host_header: alias|origin # What hostname should we use when contacting the failover? (alias or origin) +# Flags to disable options automatically enabled: + receipts: + enabled: false # don't set a rcpt field in logs + name: override_name # override receipt name on ruleset + propstats: false # don't send stats to epic for this remap rule + origin_host_header: alias| origin # origin=Send the origin's Host: header to the origin. alias=send the inbound Host: header to origin (default)" +# Property Specific options: + s3: #Use Amazon S3 authentication + path: "path/to/config" # Origin is s3; eg certs/s3.config + version: 2|4 # which version of Amazon AWS signature algorithm to use; 2 or 4 with default of 2 + virtual_host: true # populates Host: header with S3 virtual host + authproxy: range | head # Send Range/Head request to secondary origin to authenticate request. Will need 2nd remap rule to catch these requests. +# Video background fetch plugin options + video_background_fetch: + fetch_count: 10 + fetch_max: 1000 + api_header: header1 + replace_host: host1 + name_space: ns1 + metrics_prefix: prefix1 + exact_match: true + log_name: log1 + frontend: + fetch_path_pattern: '/(.*-)(\d+)(.*)/$1{$2+2}$3/' + fetch_policy: lru:10000000 + backend: + fetch_policy: simple + +tests: +# Status code checks + health: # arbitrary name for test + urls: # urls to test + - http://mydomain1.com/show/health + - https://mydomain1.com/show/health + success: # rules defining a successful test + status_code: 200 + +# Header checks + parent_cache_key_and_etag: + description: run only on hosts in trafficserver_parent salt role + role: roles_uat # specify role of host to run test against + insecure: true # Default False. Ignore ssl errors thus allowing insecure connection + range: 0-499 # specifies the first 500 bytes + purge_before: true # Default False. Purge url before running a test + headers: # request headers + X-Debug: X-Cache-Key + host_types: # valid values: parent|child + - parent + urls: + - http://mydomain1.com/test.txt + success: + status_code: 200 + headers: + X-Cache-Key: /mydomain1.com:80/test.txt + ETag: \"d36f8f9425c4a8000ad9c4a97185aca5\" +# domain names for SSL tests + ssl_tests: + domain_names: + - working.domain.com + + cert_serial_number_check: + description: devimages has a pinned cert + type: ssl + domain_names: + - devimages-cdn.domain.com + success: + serial_numbers: + - ABC123 + +# log settings +receiptsd: # configure receipts (logs) sent + sampling: + posters: + global: 10% # only send 10% of logs to main instance + adcdownload: 100% # also send 100% of logs to the download instance + ignored_extensions: # don't send any logs for files with these extensions + - .dist + - .dist.gz + - .pkm + +# vim: set et sw=2 ts=2 ft=yaml : diff --git a/tools/voluspa/schema_v1.json b/tools/voluspa/schema_v1.json new file mode 100644 index 00000000000..2abbcc0235d --- /dev/null +++ b/tools/voluspa/schema_v1.json @@ -0,0 +1,1126 @@ +{ + "$schema": "http://json-schema.org/schema#", + "$comment": "Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements; and to You under the Apache License, Version 2.0.", + "type": "object", + "additionalProperties": false, + "required": [ + "rcpt", + "schema_version", + "owner" + ], + "definitions": { + "schemes": { + "type": "array", + "uniqueItems": true, + "items": { + "type": "string", + "enum": [ + "http", + "https" + ], + "minItems": 1 + }, + "description": "http|https applied to alias and origin (if not specified there)" + }, + "role": { + "type": "string", + "enum": [ + "roles_edge_child" + ], + "description": "If this should be applied to specific set of machines." + }, + "parent_child": { + "type": "boolean", + "description": "Use parent/child hierarchy" + }, + "alias": { + "type": "array", + "items": { + "type": "string" + }, + "minItems": 1, + "description": "Inbound URI. Can have multiple. If there is no scheme, uses from schemes:. Any extra path and query parameters are passed along at end of origin." + }, + "origin": { + "type": "string", + "description": "Outbound URL. If there is no scheme, uses it from schemes:. Extra path and query parameters after aliases are passed along at the end." + }, + "propstats": { + "type": "boolean", + "description": "Keep metrics on this property or not." + }, + "fetch_policy": { + "type": "string", + "description": "simple or lru (with lru size). eg lru:10000000" + }, + "mapping": { + "type": "object", + "additionalProperties": false, + "required": [ + "alias", + "origin" + ], + "items": { + "type": "object" + }, + "properties": { + "comment": { + "type": "string" + }, + "alias": { + "$ref": "#/definitions/alias" + }, + "origin":{ + "$ref": "#/definitions/origin" + }, + "group": { + "type": "string", + "enum": [ + "group1", + "group2", + "group3", + "group4" + ], + "description": "Used to override resolver." + }, + "rule": { + "type": "string", + "description": "Which ruleset to apply to this mapping set" + }, + "parent_dest_domain": { + "type": "string", + "description": "May need to tell Voluspa what domain to use when finding the parent. Note: this option is being deprecated! Use parent_override instead!" + }, + "regex_map": { + "type": "boolean", + "description": "if there is a regular expression in the hostname of this uri." + }, + "role": { + "$ref": "#/definitions/role" + }, + "schemes": { + "$ref": "#/definitions/schemes" + }, + "parent_override": { + "description": "List of overrides for parent.config.", + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "properties": { + "comment": { + "type": "string" + }, + "role": { + "$ref": "#/definitions/role" + }, + "dest_domain": { + "type": "string", + "description": "What destination domain when finding the parent." + }, + "strategy": { + "type": "string", + "enum": [ + "true_round_robin", + "strict_round_robin", + "no_round_robin", + "consistent_hash", + "latched" + ], + "default": "consistent_hash", + "description": "Routing strategy." + }, + "ignore_querystring": { + "type": "boolean", + "default": false, + "description": "When set to true the query string will not be included in the hash to select the parent." + }, + "http_port": { + "type": "integer", + "description": "Port to use for http scheme." + }, + "https_port": { + "type": "integer", + "description": "Port to use for https scheme." + }, + "go_direct": { + "type":"boolean", + "default": false, + "description": "When set to true, requests bypass parent hierarchies and go directly to the origin server." + }, + "primary_list": { + "type":"array", + "minItems": 1, + "items": { + "anyOf": [ + { + "type": "string", + "format": "hostname" + }, + { + "type": "string", + "format":"ipv4" + }, + { + "type": "string", + "format":"ipv6" + } + ] + }, + "description": "Array of hostnames (or IP adderesses) of the primary parents." + }, + "secondary_list": { + "type":"array", + "minItems": 1, + "items": { + "anyOf": [ + { + "type": "string", + "format": "hostname" + }, + { + "type": "string", + "format":"ipv4" + }, + { + "type": "string", + "format":"ipv6" + } + ] + }, + "description": "Array of hostnames (or IP adderesses) of the secondary parents." + } + } + } + } + } + } + }, + "properties": { + "comment": { + "type": "string" + }, + "schema_version": { + "oneOf": [ + { "type": "string", "pattern": "^1(\\.0)?$" }, + { "type": "integer", "minimum": 1, "maximum": 1 } + ], + "description": "Declares the version of the Voluspa schema this configuration is defined in terms of." + }, + "alias": { + "$ref": "#/definitions/alias" + }, + "cdn": { + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { + "type": "string", + "enum": [ + "edge", + "internal" + ] + }, + "description": "Which machines to target." + }, + "lifecycle": { + "type": "string", + "enum": [ + "onboarding", + "live", + "retired" + ], + "description": "live: in production; onboarding: various things may not yet be working; retired: No longer active (no need for new ssl certs, etc);" + }, + "org": { + "type": "string", + "enum": [ + "IST", + "InternetServices", + "SoftwareEngineering" + ], + "description": "Used for billing the correct org. Metadata only, not used for cache configuration." + }, + "owner": { + "type": "string", + "description": "What group should we email/page if there are problems?" + }, + "propstats": { + "$ref": "#/definitions/propstats" + }, + "rcpt": { + "type": "string", + "description": "A unique property name." + }, + "reference": { + "type": "string", + "description": "Where to find more information about this property, such as a URL." + }, + "role": { + "$ref": "#/definitions/role" + }, + "rules": { + "type": "object", + "additionalProperties": false, + "patternProperties": { + ".*": { + "type": "object", + "additionalProperties": false, + "properties": { + "comment": { + "type": "string" + }, + "set_header": { + "type": "string", + "description": "Set this header; overwrites if origin had set." + }, + "cachekey": { + "type": "object", + "additionalProperties": false, + "items": { + "type": "string" + }, + "description": "Defines how the cachekey is composed. Changing any of these parameters is likely to effectively clear the cache.", + "properties": { + "comment": { + "type": "string" + }, + "remove_all_params": { + "type": "boolean", + "description": "Don't use the query string as part of the cachekey or parent_selection." + }, + "include_headers": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Include specific headers from the request in the cachekey." + }, + "regex_replace_path": { + "type": "string", + "examples": [ + "/.*/probe-info-ok.html/" + ], + "description": "Replace cachekey URI path with regex capture. Format is PCRE pattern or PCRE pattern + replacement in format '///'" + }, + "regex_replace_path_uri": { + "type": "string", + "examples": [ + ["/http[s]?:\\/\\/[^\\/]*(.*\\.html|.*).*/$1/"] + ], + "description": "Replace cachekey full URI with regex capture. Format is PCRE pattern or PCRE pattern + replacement in format '///'" + }, + "include_params": { + "type": "array", + "items": { + "type": "string" + }, + "examples": [ + ["queryparam1", "queryparam2"] + ], + "description": "Only use these query parameters in the cachekey." + }, + "static_prefix": { + "type": "string", + "description": "Specified value will start the cachekey." + }, + "include_cookies": { + "type": "array", + "uniqueItems": true, + "items": { + "type": "string" + }, + "description": "Include these specific cookies in the cachekey." + }, + "exclude_params": { + "type": "array", + "uniqueItems": true, + "items": { + "type": "string" + }, + "examples": [ + ["queryparam1", "queryparam2"] + ], + "description": "Ignore these query parameters in the cachekey." + }, + "sort_query": { + "type": "boolean", + "description": "Sort the query parameters for the cachekey." + }, + "parent_selection": { + "type": "string", + "enum": [ + "ignore_query" + ], + "description": "ignore_query: Ignore the query string when choosing the parent. Note: this option is being deprecated! Use parent_override in stead!" + }, + "capture_header": { + "type": "array", + "uniqueItems": true, + "items": { + "type": "string" + }, + "examples": [ + [ "Authorization:/AWS\\s([^:]+).*/clientID:$1/", "Authorization:/BYT\\s([^,]+).*/clientID:$1/" ] + ], + "description": "Add specific parts of headers to the cachekey via a regex capture. Format is PCRE pattern or PCRE pattern + replacement in format '///'" + } + } + }, + "allow_ip": { + "type": "array", + "uniqueItems": true, + "items": { + "type": "string" + }, + "description": "Only accept requests from these IPs and IP ranges.", + "examples": [ + [ "127.0.0.1", "::1" ], + [ "10.0.0.0-10.255.255.255", "127.0.0.1", "::1" ] + ] + }, + "allow_ip_parent": { + "type": "array", + "uniqueItems": true, + "items": { + "type": "string" + }, + "description": "Only accept requests from these IPs and IP ranges (applies to parent hosts).", + "examples": [ + [ "10.0.0.0-10.255.255.255", "10.127.0.0-10.127.255.255" ] + ] + + }, + "header_rewrite": { + "type": "string", + "description": "Raw text to put into ATS header_rewrite configuration." + }, + "add_header": { + "type": "string", + "description": "Add this header to the response.", + "examples": [ "Strict-Transport-Security \"max-age=31536000; includeSubDomains\"" ] + }, + "add_header_origin": { + "type": "string", + "description": "Add this header to the origin request.", + "examples": [ "X-CLIENT-IP \"%\"" ] + }, + "origin_host_header": { + "type": "string", + "enum": [ + "origin", + "alias" + ], + "description": "What Host: to send to the origin. origin=Send the origin's Host; alias=send the inbound Host: (default)" + }, + "content_type_forge": { + "type": "object", + "patternProperties": { + ".*": { + "type": "string", + "description": "'URL regex': content-type. URLs matching regex will have have the response Content-Type set to content-type" + } + }, + "description": "Change the Content-Type based on a series of URL path regexes", + "examples": [ "\".*\\.json$\": \"application/json\"" ] + }, + "conf_remap": { + "type": "string", + "description": "Raw text to put into ATS conf_remap configuration." + }, + "cache_promote": { + "type": "object", + "additionalProperties": false, + "description": "Configuration for the cache_promote plugin. Selectively cache objects. For really large working sets.", + "properties": { + "comment": { + "type": "string" + }, + "policy": { + "type": "string", + "enum": [ + "lru", + "chance" + ], + "description": "`chance`: randomly choose to cache the item (based on sample number); `lru`: Track most recently used URLs, and cache those with more than lru_hits." + }, + "sample": { + "type": "number", + "minimum": 1, + "maximum": 100, + "description": "The percentage sampling rate for the request to be considered, for either policy." + }, + "lru_hits": { + "type": "integer", + "description": "In `lru` policy, how often the object is fetched before promotion." + }, + "lru_buckets": { + "type": "integer", + "description": "In `lru` policy, how many objects are tracked." + } + } + }, + "propstats": { + "$ref": "#/definitions/propstats" + }, + "regex_remap": { + "type": "string", + "description": "Use regex to match the inbound path (only) and substition for origin. Can add optional http status code (like @status=301). Can also use '|-' format", + "examples": [ "^/([^/]+)/([^\\?]*)\\??.* https://$1.s3.amazonaws.com/$2" ] + }, + "negative_ttl": { + "type": "string", + "description": "Amount of time to cache http errors (404, 502, 504, etc). Default 10s.", + "examples": [ "10s" ] + }, + "priority": { + "type": "string", + "enum": [ + "background", + "foreground", + "streamingaudio", + "streamingvideo" + ], + "description": "QOS priority (DSCP bits)" + }, + "default_ttl": { + "type": "string", + "description": "If the Origin doesn't set a Cache-Control, set it to this value. A number followed by a letter (like 17s, 5m, 3h, 2d, 1w) is a shortcut for 'max-age=N, public'", + "examples": [ "10s", "60m", "12h", "no-cache, no-store, must-revalidate, stale-while-revalidate=30" ], + "pattern": "^([0-9]+[smhdw])$|^((no-cache|no-store|no-transform|must-revalidate|public|private|proxy-revalidate|immutable|(max-age=|s-maxage=|stale-while-revalidate=|stale-if-error=)[0-9]+) *(, *|$))+$" + }, + "set_header_origin": { + "type": "string", + "description": "Send this header to the origin; overwrites if client had set." + }, + "authproxy": { + "type": "string", + "emum": [ + "range", + "head" + ], + "description": "Send Range/Head request to secondary origin to authenticate request. Will need 2nd remap rule to catch these requests." + }, + "parent_child": { + "$ref": "#/definitions/parent_child" + }, + "parent": { + "type": "boolean" + }, + "child": { + "type": "boolean" + }, + "redirect": { + "type": "object", + "additionalProperties": false, + "description": "Configure the Regex Remap Plugin. See https://docs.trafficserver.apache.org/en/7.1.x/admin-guide/plugins/regex_remap.en.html", + "properties": { + "comment": { + "type": "string" + }, + "src": { + "type": "string", + "description": "regex src pattern, defaults to `(.*)`" + }, + "url": { + "type": "string", + "description": "regex URL to redirect to, may include substitution variables" + }, + "http_code": { + "type": "integer", + "minimum": 100, + "maximum": 999, + "description": "HTTP status code to return, defaults to 302" + } + }, + "examples": [ + { "url": "https://example.com$0", "http_code": 410 }, + { "src": "http://old.(.*).z.com", "url": "http://new.$1.z.com" } + ] + }, + "video_background_fetch": { + "type": "object", + "description": "Prefetch sequential URLs for improved caching. Usually for HLS video.", + "additionalProperties": false, + "properties": { + "comment": { + "type": "string" + }, + "log_name": { + "type": "string", + "description": "Rcpt to log these requests." + }, + "fetch_count": { + "type": "integer", + "description": "How many URLs ahead to fetch." + }, + "replace_host": { + "type": "string", + "description": "What Host: header to use in pre-fetch." + }, + "exact_match": { + "type": "boolean" + }, + "api_header": { + "type": "string", + "description": "What header to use to signal this is a prefetch request." + }, + "name_space": { + "type": "string", + "description": "Name Space to segregate LRU count." + }, + "metrics_prefix": { + "type": "string", + "description": "Prefix for prefetch metrics." + }, + "fetch_policy": { + "$ref": "#/definitions/fetch_policy" + }, + "frontend": { + "type": "object", + "properties": { + "comment": { + "type": "string" + }, + "fetch_policy": { + "$ref": "#/definitions/fetch_policy" + }, + "fetch_path_pattern": { + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { + "type": "string" + }, + "description": "Regex to find the portion of the url that can be incremented." + } + } + }, + "fetch_max": { + "type": "integer" + }, + "backend": { + "type": "object", + "properties": { + "comment": { + "type": "string" + }, + "fetch_policy": { + "type": "string", + "description": "simple or lru (with lru size). eg lru:10000000", + "pattern": "^(simple|lru:[0-9]+)$" + } + } + } + } + }, + "log_header": { + "type": "object", + "properties": {}, + "description": "Log specific request headers with the specified keyword names.", + "examples": [ { "name-in-log": "name-of-header", "cacheControlValue": "Cache-Control", "playbackSessionID": "X-PlayBack-Session-Id" } ] + }, + "proxy_cache_control": { + "type": "string", + "description": "Use this header's value for Cache-Control.", + "examples": [ "Proxy-Cache-Control" ] + }, + "failover": { + "type": "object", + "description": "When object is unavailable, try a different origin.", + "additionalProperties": false, + "properties": { + "comment": { + "type": "string" + }, + "domain": { + "type": "string", + "description": "When object is unavailable, try what origin?" + }, + "status_codes": { + "type": "array", + "uniqueItems": true, + "items": { + "type": "integer", + "minimum": 100, + "maximum": 999 + }, + "description": "What HTTP status codes indicate we should try another origin?" + }, + "host_header": { + "type": "string", + "enum": [ + "origin", + "alias" + ], + "description": "What hostname should we use when contacting the failover? (alias or origin)" + } + } + }, + "remove_header": { + "type": "string", + "description": "Remove this header." + }, + "remove_header_origin": { + "type": "string", + "description": "Remove this header in origin request." + }, + "log_cookie": { + "type": "object", + "properties": {}, + "description": "Log specific request cookies with the specified keyword names.", + "examples": [ { "name-in-log": "name-of-cookie", "session": "s", "auth": "ADCDownloadAuth" } ] + }, + "force_ttl": { + "type": "string", + "description": "Always set Cache-Control to this value; overwrites if origin had set. A number followed by a letter (like 17s, 5m, 3h, 2d, 1w) is a shortcut for 'max-age=N, public'", + "pattern": "^([0-9]+[smhdw])$|^((no-cache|no-store|no-transform|must-revalidate|public|private|proxy-revalidate|immutable|(max-age=|s-maxage=|stale-while-revalidate=|stale-if-error=)[0-9]+) *(, *|$))+$" + }, + "receipts": { + "type": "object", + "description": "Control receipt (eg log) production.", + "additionalProperties": false, + "properties": { + "comment": { + "type": "string" + }, + "enabled": { + "type": "boolean", + "description": "Should we send receipts (eg logs) for this property?" + }, + "name": { + "type": "string", + "description": "Should we use a different rcpt name for this ruleset?" + } + } + }, + "allow_methods": { + "type": "array", + "uniqueItems": true, + "items": { + "type": "string", + "enum": [ + "GET", + "HEAD", + "POST", + "PUT", + "DELETE", + "CONNECT", + "OPTIONS", + "TRACE", + "PATCH", + "PURGE" + ] + }, + "description": "Accept only these HTTP verbs" + }, + "deny_methods": { + "type": "array", + "uniqueItems": true, + "items": { + "type": "string", + "enum": [ + "GET", + "HEAD", + "POST", + "PUT", + "DELETE", + "CONNECT", + "OPTIONS", + "TRACE", + "PATCH" + ] + }, + "description": "Reject these HTTP verbs" + }, + "strip_query": { + "type": "boolean", + "description": "Strips the query string completely -- the origin won't see it." + }, + "transaction_timeout": { + "type": "string", + "description": "Sets transaction timeouts (CDN default is 60s)" + }, + "s3": { + "type": "object", + "additionalProperties": false, + "description": "Use Amazon S3 authentication when contacting origin.", + "required": [ + "path" + ], + "properties": { + "comment": { + "type": "string" + }, + "path": { + "type": "string", + "description": "Path to the s3 configuration containing the shared secret." + }, + "virtual_host": { + "type": "boolean", + "description": "Populate Host: header with S3 virtual host." + }, + "version": { + "type": "integer", + "enum": [ + 2, + 4 + ], + "description": "Which version of Amazon AWS signature algorithm to use." + }, + "v4_include_headers": { + "type": "array", + "items": { + "type": "string" + } + }, + "v4_exclude_headers": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Don't use these headers in the authorization signature.", + "examples": [ { "v4_exclude_headers": ["x-forwarded-for","forwarded","via","authorization"] } ] + }, + "region_map": { + "type": "string", + "description": "Path to a region map config file." + } + } + }, + "edge_compression": { + "type": "object", + "additionalProperties": false, + "description": "", + "properties": { + "comment": { + "type": "string" + }, + "enabled": { + "type": "boolean", + "description": "Should we enable compression?" + }, + "algorithms": { + "type": "array", + "uniqueItems": true, + "items": { + "type": "string", + "enum": [ + "gzip", + "br" + ], + "description": "Specify compression algorithm(s) to use." + } + }, + "static_origin_compressible_content_type": { + "type": "array", + "uniqueItems": true, + "items": { + "type": "string" + }, + "description": "Compression for specified mime-types.", + "examples": [ [ "application/x-mpegURL" ] ] + }, + "flush": { + "type": "boolean" + }, + "remove_accept_encoding": { + "type": "boolean", + "description": "Removes Accept-Encoding header in origin requests." + }, + "minimum_content_length": { + "type": "integer", + "description": "Minimum content length for compression to be enabled (in bytes)" + } + } + }, + "alt_svc": { + "type": "object", + "additionalProperties": false, + "properties": { + "comment": { + "type": "string" + }, + "authority": { + "type": "string" + }, + "protocol": { + "type": "string" + }, + "ma": { + "type": "integer" + }, + "health": { + "type": "string" + }, + "parameters": { + "type": "array", + "items": { + "type": "string" + } + } + } + }, + "disable_cache": { + "type": "boolean", + "description": "Turn off the cache for this rule." + }, + "log_type": { + "type": "string", + "enum": [ + "private", + "public" + ], + "description": "Private if we should not log privacy sensitive information, like the URL, referral URL, client IP, etc. Default is public." + }, + "access_approval": { + "type": "object", + "additionalProperties": false, + "properties": { + "comment": { + "type": "string" + }, + "raw": { + "type": "string" + } + } + }, + "storage_volume": { + "type": "string", + "description": "Selects which volume will be used for caching (disk_volume or ramdisk_volume); when this is omitted the default used is disk_volume.", + "enum": [ + "disk_volume", + "ramdisk_volume" + ] + }, + "lua": { + "type": "string", + "description": "Raw text to run as Lua remap script." + }, + "echo_cors": { + "type": "string", + "description": "Header to echo back as Access-Control-Allow-Origin in response." + }, + "cache_version": { + "type": "integer", + "description": "To clear the cache for this rule, increment the integer (or add this rule with value 1)." + } + } + } + } + }, + "schemes": { + "$ref": "#/definitions/schemes" + }, + "mappings": { + "type": "array", + "items": { + "$ref": "#/definitions/mapping" + } + }, + "explicit": { + "type": "array", + "items": { + "$ref": "#/definitions/mapping" + }, + "uniqueItems": true + }, + "ha_proxy": { + "type": "array", + "additionalProperties": false, + "items": { + "type": "object", + "required": [ + "port", + "rips" + ] + }, + "properties": { + "comment": { + "type": "string" + }, + "port": { + "type": "integer" + }, + "rips": { + "type": "array", + "items": { + "type": "object", + "required": [ + "ip", + "port" + ] + }, + "properties": { + "ip": { + "type": "string" + }, + "port": { + "type": "integer" + } + } + } + } + }, + "parent_child": { + "$ref": "#/definitions/parent_child" + }, + "qps": { + "type": "integer", + "minimum": 1, + "description": "Approximate queries per second, used to order rules for efficiency." + }, + "receiptsd": { + "type": "object", + "properties": { + "comment": { + "type": "string" + }, + "ignored_extensions": { + "type": "array", + "items": { + "type": "string" + } + }, + "sampling": { + "type": "object", + "properties": { + "comment": { + "type": "string" + }, + "posters": { + "type": "object", + "description": "Name of the program posting the logs. Current values are Global (for iTunes splunk), adcdownload, and adcdownload-uat (for posting to WWDR's prod/uat)", + "examples": [ "global: 1%" ], + "patternProperties": { + ".*": { + "type": "string", + "pattern": "^\\d+%$" + } + } + } + }, + "additionalProperties": false + } + } + }, + "ssl_cert_names": { + "type": "array", + "uniqueItems": true, + "items": { + "type": "string", + "pattern": "^([\\w\\-\\.,:/]+)$" + }, + "description": "A list of SSL cert filenames. used to populate ssl_multicert.config." + }, + "tests": { + "type": "object", + "additionalProperties": false, + "patternProperties": { + ".*": { + "type": "object", + "properties": { + "comment": { + "type": "string" + }, + "description": { + "type": [ + "integer", + "string" + ] + }, + "domain_names": { + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { + "type": "string" + }, + "description": "Domain names for SSL tests" + }, + "headers": { + "type": "object", + "patternProperties": { + ".*": { + "type": "string" + } + }, + "description": "Headers to go into the request." + }, + "host_types": { + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { + "type": "string", + "enum": [ + "parent", + "child" + ] + }, + "description": "Which kind of hosts should the check run on?" + }, + "insecure": { + "type": "boolean", + "description": "If curl should ignore certificate warnings." + }, + "purge_before": { + "type": "boolean", + "description": "If we should run a PURGE command before the test." + }, + "range": { + "type": "string", + "description": "Use a range request -- eg, 0-499 will ask for the 1st 500 bytes." + }, + "role": { + "type": "string", + "description": "Specify role of host to run test against." + }, + "success": { + "type": "object", + "additionalProperties": false, + "properties": { + "comment": { + "type": "string" + }, + "headers": { + "type": "object", + "patternProperties": { + ".*": { + "type": "string" + } + }, + "description": "Headers and their values that should be returned" + }, + "serial_numbers": { + "type": "array", + "uniqueItems": true, + "minItems": 1, + "items": { + "type": "string" + }, + "description": "Serial Number of cert or Intermediate." + }, + "status_code": { + "type": "integer", + "minimum": 100, + "maximum": 999, + "description": "HTTP status code expected in the response." + } + } + }, + "urls": { + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { + "type": "string" + }, + "description": "URLs to test" + } + } + } + } + } + } +} diff --git a/tools/voluspa/specs/voluspa.spec b/tools/voluspa/specs/voluspa.spec new file mode 100644 index 00000000000..75c85f6a24a --- /dev/null +++ b/tools/voluspa/specs/voluspa.spec @@ -0,0 +1,53 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +%{!?_release: %define _release 0} +%{!?_version: %define _version UNKNOWN} +%{!?commit0: %define commit0 UNKNOWN} + +Name: voluspa +Version: %{_version} +Release: 1 +Summary: Voluspa +License: Apache Software License 2.0 (AL2) +Group: Applications/System +URL: https://github.com/apache/trafficserver/tools/voluspa + +%description +CDN -> ATS Configuration Tool + +Version: %{commit0} + +%prep + +%build + +%check + +%install +rm -rf $RPM_BUILD_ROOT +mkdir -p $RPM_BUILD_ROOT/usr/bin +cp -p %{_topdir}/dist/linux/voluspa $RPM_BUILD_ROOT/usr/bin + +%files +%defattr(-, root, root, -) +/usr/bin/voluspa + +%clean + +%changelog diff --git a/tools/voluspa/ssl_multicert.go b/tools/voluspa/ssl_multicert.go new file mode 100644 index 00000000000..cb4e491244d --- /dev/null +++ b/tools/voluspa/ssl_multicert.go @@ -0,0 +1,199 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package voluspa + +import ( + "bytes" + "fmt" + "sort" + "strings" +) + +var ( + // defaultCertNames is a set of cert names to be specified as the default + defaultCertNames = map[string]interface{}{ + "/secret/edgecdn:images.domain.com": nil, + "/secret/images.domain.com.rsa": nil, + "/secret/images.domain.com.rsa,/secret/images.domain.com.ecdsa": nil, + } + + SSLMulticertDefaultFilename = "ssl_multicert.config_default" + SSLMulticertFilename = "ssl_multicert.config" +) + +type sslCert struct { + filename string + comment string +} + +type SSLMulticertConfigurator struct { + options *Options +} + +func newSSLMulticertConfigurator(options *Options) *SSLMulticertConfigurator { + return &SSLMulticertConfigurator{options: options} +} + +func initCertsMap(parsedConfigs []*CustomerConfig) map[string]map[string]sslCert { + // a map of roles to an array of sslCerts + certs := make(map[string]map[string]sslCert) + + sort.Stable(sort.Reverse(byQPSAndName(parsedConfigs))) + + for _, parsedConfig := range parsedConfigs { + if len(parsedConfig.sslCertNames) == 0 { + continue + } + if parsedConfig.lifecycle == Retired { + continue + } + + role := parsedConfig.role + + if _, exists := certs[role]; !exists { + certs[role] = make(map[string]sslCert) + } + + sort.Strings(parsedConfig.sslCertNames) + + for _, sslCertName := range parsedConfig.sslCertNames { + certs[role][sslCertName] = sslCert{ + filename: sslCertName, + comment: parsedConfig.reference, + } + } + } + + return certs +} + +func (s *SSLMulticertConfigurator) Do(parsedConfigs []*CustomerConfig, merge bool) ([]ManagedFile, error) { + if merge { + return s.get(parsedConfigs, NoCDN) + } + + // otherwise, group properties by CDN and generate ssl_multicert.config for each CDN + grouped := groupCustomerConfigsByCDN(parsedConfigs) + var managedFiles []ManagedFile + for cdn, configs := range grouped { + files, err := s.get(configs, cdn) + if err != nil { + return nil, err + } + managedFiles = append(managedFiles, files...) + } + + return managedFiles, nil +} + +func (s *SSLMulticertConfigurator) get(configs []*CustomerConfig, cdn string) ([]ManagedFile, error) { + certs := initCertsMap(configs) + if len(certs) == 0 { + return nil, nil + } + + var buf bytes.Buffer + buf.WriteString(generatedFileBanner) + var roles []string + for role := range certs { + roles = append(roles, role) + } + sort.Strings(roles) + + for _, role := range roles { + err := s.expandConfigTemplate(certs, role, &buf) + if err != nil { + return nil, err + } + } + + fileName := SSLMulticertDefaultFilename + if len(cdn) > 0 && (cdn != DefaultCDN && cdn != NoCDN) { + fileName = fmt.Sprintf("ssl_multicert.config_%s", cdn) + } + + return []ManagedFile{ + NewManagedFile(fileName, SSLMulticertFilename, cdn, "", "", &buf, UnknownLocation), + }, nil +} + +func (s *SSLMulticertConfigurator) startRoleGuard(role string) string { + if len(role) == 0 { + if s.options.PromoteRolesToCDN { + return "" + } + + return "{% if not salt.pillar.get('roles_uat') %}\n\n" + } + + roleName := role + if !strings.HasPrefix(role, "roles_") { + roleName = fmt.Sprintf("roles_%s", role) + } + return fmt.Sprintf("{%% if salt.pillar.get('%s') %%}\n\n", roleName) +} + +func (s *SSLMulticertConfigurator) endRoleGuard(role string) string { + if len(role) == 0 && s.options.PromoteRolesToCDN { + return "" + } + return "{% endif %}\n\n" +} + +func (s *SSLMulticertConfigurator) expandConfigTemplate(certsMap map[string]map[string]sslCert, role string, buf *bytes.Buffer) error { + certs, ok := certsMap[role] + if !ok { + return fmt.Errorf("role %q not found", role) + } + + if len(certs) == 0 { + return nil + } + + var certFilenames []string + for certFilename := range certs { + certFilenames = append(certFilenames, certFilename) + } + sort.Strings(certFilenames) + + buf.WriteString(s.startRoleGuard(role)) + + var sawDefaultCert bool + for _, certFilename := range certFilenames { + cert := certs[certFilename] + if len(cert.comment) > 0 { + buf.WriteString(fmt.Sprintf("# %s\n", cert.comment)) + } + if _, found := defaultCertNames[certFilename]; found { + if sawDefaultCert { + return fmt.Errorf("more than one default cert found for role") + } + + buf.WriteString("dest_ip=* ") + sawDefaultCert = true + } + + buf.WriteString(fmt.Sprintf("ssl_cert_name=%s\n\n", certFilename)) + } + + buf.WriteString(s.endRoleGuard(role)) + + return nil +} diff --git a/tools/voluspa/tests/.gitignore b/tools/voluspa/tests/.gitignore new file mode 100644 index 00000000000..b98b0e32b34 --- /dev/null +++ b/tools/voluspa/tests/.gitignore @@ -0,0 +1,5 @@ +*/test +*/*/test +/out +*/out +*.out diff --git a/tools/voluspa/tests/Makefile b/tools/voluspa/tests/Makefile new file mode 100644 index 00000000000..dbe8b164d72 --- /dev/null +++ b/tools/voluspa/tests/Makefile @@ -0,0 +1,53 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +test: regress other custom + +clean: + @rm -f */system.json + @rm -f */profile_*.out + +COVERAGE_OPTIONS=-test.run ^TestRunMain$$ -test.coverprofile=system.out + +coverage: + cd regress && OPTIONS="$(COVERAGE_OPTIONS)" VOLUSPA=voluspa-covered ./test.sh + cd other && OPTIONS="$(COVERAGE_OPTIONS)" VOLUSPA=voluspa-covered ./test.sh + if [ -x custom ]; then \ + cd custom && OPTIONS="$(COVERAGE_OPTIONS)" VOLUSPA=voluspa-covered ./test.sh; \ + fi; + +regress: + cd regress && ./test.sh + +other: + cd other && ./test.sh + +custom: + if [ -x custom ]; then \ + cd custom && ./test.sh; \ + fi; + +regen_baseline: + cd regress && ./regen_baseline.sh + cd other && ./regen_baseline.sh + if [ -x custom ]; then \ + cd custom && ./regen_baseline.sh; \ + fi; + +.PHONY: regress other custom +.NOTPARALLEL: diff --git a/tools/voluspa/tests/README.md b/tools/voluspa/tests/README.md new file mode 100644 index 00000000000..98c5420ac46 --- /dev/null +++ b/tools/voluspa/tests/README.md @@ -0,0 +1,16 @@ +# Tests + +This directory contains a series of sanity tests. These are simple tests comparing the output of a known good build/configuration to the latest locally built version of **voluspa**. + +The script "test.sh" will use the current build of the tool to regenerate the remap configs for a given yaml file; and do a diff between base/ and the new output. Generally, there should be no changes between runs. If there are expected changes, the baselines would need to be regenerated. The configurations tested range from very simple to something exercising all current configuration options. + +## Current Tests + +* simple/ + * Basic functionality +* all/ + * Every current configuration option exercising combinations of config types (normal and explicit) +* order/ + * Covers both ordering of properties by QPS and grouping via "role" +* parentchild/ + * A baseline of current parent/child config generation diff --git a/tools/voluspa/tests/other/all_bad/.gitignore b/tools/voluspa/tests/other/all_bad/.gitignore new file mode 100644 index 00000000000..9daeafb9864 --- /dev/null +++ b/tools/voluspa/tests/other/all_bad/.gitignore @@ -0,0 +1 @@ +test diff --git a/tools/voluspa/tests/other/all_bad/base/stderr.txt b/tools/voluspa/tests/other/all_bad/base/stderr.txt new file mode 100644 index 00000000000..dd3c698ba51 --- /dev/null +++ b/tools/voluspa/tests/other/all_bad/base/stderr.txt @@ -0,0 +1,14 @@ +voluspa: errors validating configs: + - alias http://(.*.myalias.com not unique (all_bad/simple2.conf) + - alias http://inbound.testing.example.com not unique (all_bad/simple2.conf) + - alias https://inbound.testing.example.com not unique (all_bad/simple2.conf) + - invalid regex URL: error parsing regexp: missing closing ): `http://(.*.myalias.com` (all_bad/simple2.conf) + - invalid regex: error parsing regexp: unexpected ): `# If Cookie : ADCDownloadAuth is missing then +# 302 to developer.example.com/unauthorized/ +cond %{COOKIE:Tasty} ="" [AND] +cond %{CLIENT-IP} ^127\.0\.0\.1|10\..*)/ [NOT] + set-redirect 302 https://inbound.example.com/unauthorized/ [L] +# If Origin serves a 000, 500 501 502 503 then +# 302 to inboud.example.com/unauthorized/` (all_bad/simple2.conf) + - property All not unique (all_bad/simple.conf) + - property All not unique (all_bad/simple2.conf) diff --git a/tools/voluspa/tests/other/all_bad/duplicated_property_name.conf b/tools/voluspa/tests/other/all_bad/duplicated_property_name.conf new file mode 100644 index 00000000000..3536d7c9fb9 --- /dev/null +++ b/tools/voluspa/tests/other/all_bad/duplicated_property_name.conf @@ -0,0 +1,13 @@ +schema_version: 1 +owner: nobody +rcpt: All +schemes: [https, http] + +mappings: + - alias: + - dupe.testing.example.com + origin: https://dupe.example.com + +rules: + default: + deny_methods: [ POST, PUT, DELETE ] diff --git a/tools/voluspa/tests/other/all_bad/simple.conf b/tools/voluspa/tests/other/all_bad/simple.conf new file mode 100644 index 00000000000..9da5d2e60af --- /dev/null +++ b/tools/voluspa/tests/other/all_bad/simple.conf @@ -0,0 +1,13 @@ +schema_version: '1.0' +owner: nobody +rcpt: All +schemes: [https, http] + +mappings: + - alias: + - inbound.testing.example.com + origin: https://origin.example.com + +rules: + default: + deny_methods: [ POST, PUT, DELETE ] diff --git a/tools/voluspa/tests/other/all_bad/simple2.conf b/tools/voluspa/tests/other/all_bad/simple2.conf new file mode 100644 index 00000000000..1fe7f8150b1 --- /dev/null +++ b/tools/voluspa/tests/other/all_bad/simple2.conf @@ -0,0 +1,25 @@ +schema_version: 1 +owner: nobody +rcpt: All +schemes: [https, http] + +mappings: + - alias: + - inbound.testing.example.com + origin: https://origin.example.com + - alias: + - http://(.*.myalias.com + origin: http://origin3.example.com + regex_map: true + +rules: + default: + deny_methods: [ POST, PUT, DELETE ] + header_rewrite: |- + # If Cookie : ADCDownloadAuth is missing then + # 302 to developer.example.com/unauthorized/ + cond %{COOKIE:Tasty} ="" [AND] + cond %{CLIENT-IP} ^127\.0\.0\.1|10\..*)/ [NOT] + set-redirect 302 https://inbound.example.com/unauthorized/ [L] + # If Origin serves a 000, 500 501 502 503 then + # 302 to inboud.example.com/unauthorized/ diff --git a/tools/voluspa/tests/other/cachekey_bad/base/stderr.txt b/tools/voluspa/tests/other/cachekey_bad/base/stderr.txt new file mode 100644 index 00000000000..9a5cc8b30cc --- /dev/null +++ b/tools/voluspa/tests/other/cachekey_bad/base/stderr.txt @@ -0,0 +1,3 @@ +voluspa: errors validating configs: + - invalid regex: error parsing regexp: missing closing ): `BYT\s([^,]+.*` (cachekey_bad/test.conf) + - invalid regex: unexpected / (cachekey_bad/test.conf) diff --git a/tools/voluspa/tests/other/cachekey_bad/test.conf b/tools/voluspa/tests/other/cachekey_bad/test.conf new file mode 100644 index 00000000000..3f9e41961c2 --- /dev/null +++ b/tools/voluspa/tests/other/cachekey_bad/test.conf @@ -0,0 +1,28 @@ +schema_version: 1 +owner: "" +reference: your reference notes here +rcpt: All +ssl_cert_names: + - /secrets/mysecret.txt + +mappings: + - origin: origin7.example.com + alias: + - myalias7.example.com + - myalias7.example.com/ran + - myalias7.example.com/bob + - /path/test.txt + +rules: + default: + cachekey: + include_params: ["param1", "param2"] + exclude_params: ["param1", "param2"] + include_headers: ["header1", "header2"] + include_cookies: ["cookie1", "cookie2"] + sort_query: true + remove_all_params: true + static_prefix: "SomeString" + regex_replace_path: '/.*/probe-info-ok.html/' + regex_replace_path_uri: '/http[s]?:\//[^\/]*.*\.html|.*).*/$1/' + capture_header: [ '/Authorization:AWS\s([^:]+).*/clientID:$1//', '/Authorization:BYT\s([^,]+.*/clientID:$1/' ] diff --git a/tools/voluspa/tests/other/duped/base/stderr.txt b/tools/voluspa/tests/other/duped/base/stderr.txt new file mode 100644 index 00000000000..fa0270f952c --- /dev/null +++ b/tools/voluspa/tests/other/duped/base/stderr.txt @@ -0,0 +1,2 @@ +voluspa: errors validating configs: + - alias http://inbound.example.com not unique for rule role "roles_edge_child" (duped/test.conf) diff --git a/tools/voluspa/tests/other/duped/test.conf b/tools/voluspa/tests/other/duped/test.conf new file mode 100644 index 00000000000..b06e3c1418d --- /dev/null +++ b/tools/voluspa/tests/other/duped/test.conf @@ -0,0 +1,27 @@ +schema_version: 1 +owner: owner +rcpt: streamer +parent_child: true + +rules: + default_child: + priority: streamingvideo + + default_child2: + priority: background + +explicit: + - alias: + - http://inbound.example.com + origin: http://origin.example.com + rule: default_child + role: roles_edge_child + + # NOTE: Duping above on purpose to trigger duplicate alias + - alias: + - http://inbound.example.com + origin: http://origin2.example.com + rule: default_child2 + role: roles_edge_child + +# vim: set et sw=2 ts=2 ft=yaml : diff --git a/tools/voluspa/tests/other/properties.sh b/tools/voluspa/tests/other/properties.sh new file mode 100644 index 00000000000..d903ee41f53 --- /dev/null +++ b/tools/voluspa/tests/other/properties.sh @@ -0,0 +1,19 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +CUST=(all_bad cachekey_bad duped unicode scheme) diff --git a/tests/tools/sessionvalidation/badsession.py b/tools/voluspa/tests/other/regen_baseline.sh old mode 100644 new mode 100755 similarity index 59% rename from tests/tools/sessionvalidation/badsession.py rename to tools/voluspa/tests/other/regen_baseline.sh index 7f55de276f0..96880d7fe4e --- a/tests/tools/sessionvalidation/badsession.py +++ b/tools/voluspa/tests/other/regen_baseline.sh @@ -1,5 +1,5 @@ -''' -''' +#!/bin/bash +# # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -15,21 +15,23 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +# +. properties.sh -class BadSession(object): - ''' - Session encapsulates a single BAD user session. Bad meaning that for some reason the session is invalid. +if [ $# -gt 0 ]; then + CUST=($1) +fi - _filename is the filename of the bad JSON session - _reason is a string with some kind of explanation on why the session was bad - ''' +CMD=../../voluspa +SCHEMA=../.. - def __repr__(self): - return "".format( - self._filename, self._reason - ) +for i in ${CUST[@]}; do + DEST=./${i}/base + rm -rf $DEST + mkdir $DEST + ./$CMD --schema-location $SCHEMA --dest $DEST ${i}/*.conf 2> $DEST/stderr.txt +done - def __init__(self, filename, reason): - self._filename = filename - self._reason = reason +# Ignore failures from voluspa +exit 0 diff --git a/tools/voluspa/tests/other/scheme/base/stderr.txt b/tools/voluspa/tests/other/scheme/base/stderr.txt new file mode 100644 index 00000000000..7195ca43c90 --- /dev/null +++ b/tools/voluspa/tests/other/scheme/base/stderr.txt @@ -0,0 +1,2 @@ +problem loading "scheme/test.conf": + - mappings.0.schemes.0: mappings.0.schemes.0 must be one of the following: "http", "https" diff --git a/tools/voluspa/tests/other/scheme/test.conf b/tools/voluspa/tests/other/scheme/test.conf new file mode 100644 index 00000000000..779a72a7174 --- /dev/null +++ b/tools/voluspa/tests/other/scheme/test.conf @@ -0,0 +1,9 @@ +schema_version: 1 +rcpt: seed +owner: "nobody" + +mappings: + - alias: + - inbound.example.com + origin: https://origin.example.com + schemes: [httpZs, http] diff --git a/tools/voluspa/tests/other/test.sh b/tools/voluspa/tests/other/test.sh new file mode 100755 index 00000000000..eca4975f2cf --- /dev/null +++ b/tools/voluspa/tests/other/test.sh @@ -0,0 +1,47 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +. properties.sh + +if [ $# -gt 0 ]; then + CUST=($1) +fi + +# this expects 'make voluspa' to be run (make sanity_tests or make test does this) +if [ -z "$VOLUSPA" ]; then + VOLUSPA="voluspa" +fi + +CMD=../../$DEST/$VOLUSPA +SCHEMA=../.. + +for i in ${CUST[@]}; do + DEST=./${i}/test + rm -rf $DEST + mkdir $DEST + ./$CMD --schema-location $SCHEMA --dest $DEST ${i}/*.conf 2> $DEST/stderr.txt + if [ "$VOLUSPA" == "voluspa" ]; then + diff -r ${i}/base $DEST + if [ $? != 0 ]; then + exit 1 + fi + else + mv system.out profile_${i}.out + fi +done diff --git a/tools/voluspa/tests/other/unicode/base/stderr.txt b/tools/voluspa/tests/other/unicode/base/stderr.txt new file mode 100644 index 00000000000..0327c36679d --- /dev/null +++ b/tools/voluspa/tests/other/unicode/base/stderr.txt @@ -0,0 +1 @@ +problem loading "unicode/httpbin.conf": Could not load YAML file. err=non-ASCII characters not supported: invalid rune "🍎" diff --git a/tools/voluspa/tests/other/unicode/httpbin.conf b/tools/voluspa/tests/other/unicode/httpbin.conf new file mode 100644 index 00000000000..c630cc59e0e --- /dev/null +++ b/tools/voluspa/tests/other/unicode/httpbin.conf @@ -0,0 +1,16 @@ +schema_version: 1 +owner: Property Owner +reference: For testing ATS behavior 🍎 +rcpt: httpbin +schemes: [http, https] + +mappings: + - alias: + - httpbin.example.com + origin: https://httpbin.edge.example.com + +rules: + default: + priority: background + origin_host_header: alias + deny_methods: [ CONNECT ] diff --git a/tools/voluspa/tests/regress/all/base/all/all.config b/tools/voluspa/tests/regress/all/base/all/all.config new file mode 100644 index 00000000000..fea885654ab --- /dev/null +++ b/tools/voluspa/tests/regress/all/base/all/all.config @@ -0,0 +1,196 @@ +# Code generated by Voluspa. DO NOT EDIT. + +map http://myalias7.example.com/bob \ + http://origin7.example.com + +map http://myalias7.example.com/ran \ + http://origin7.example.com + +map http://myalias7.example.com \ + http://origin7.example.com + +map /path/test.txt \ + http://origin7.example.com + +map http://redirect.example.com \ + http://origin7.example.com \ + @plugin=regex_remap.so @pparam=all/redirect9.config + +map https://myalias1.example.com \ + https://origin2.example.com + +map https://myalias2.example.com \ + https://origin2.example.com + +regex_map http://(.*).example.com \ + http://origin3.example.com \ + @plugin=access_control.so @pparam=--symmetric-keys-map=hmac_keys.txt \ + @pparam=--check-cookie=TokenCookie \ + @pparam=--extract-subject-to-header=@TokenSubject \ + @pparam=--extract-tokenid-to-header=@TokenId \ + @pparam=--extract-status-to-header=@TokenStatus \ + @pparam=--token-response-header=TokenRespHdr \ +{% if salt.pillar.get("roles_trafficserver_cache_promote") %} @plugin=cache_promote.so @pparam=--policy=lru @pparam=--hits=5 @pparam=--buckets=100000 \{% else %} \{% endif %} + @plugin=cachekey.so @pparam=--include-params=param1,param2 \ + @pparam=--include-cookies=cookie1,cookie2 \ + @pparam=--include-headers=header1,header2 \ + @pparam=--exclude-params=param1,param2 \ + @pparam=--static-prefix=SomeString \ + @pparam=--sort-params=true \ + @pparam=--remove-all-params=true \ + @pparam=--capture-path=/.*/probe-info-ok.html/ \ + @pparam=--capture-path-uri=/http[s]?:\/\/[^\/]*(.*\.html|.*).*/$1/ \ + @pparam=--capture-header=Authorization:/AWS\s([^:]+).*/clientID:$1/ \ + @pparam=--capture-header=Authorization:/BYT\s([^,]+).*/clientID:$1/ \ + @plugin=gzip.so @pparam=all/gzip2.config \ + @plugin=escalate.so @pparam=--pristine \ + @pparam=401,403,404,407,410,500,501,502,503,504,505:assets-origin.subdomain.example.com \ + @plugin=regex_remap.so @pparam=all/strip_query2.config \ + @plugin=volcano.so @pparam=--fetch-count=10 @pparam=--fetch-max=1000 @pparam=--api-header=header1 @pparam=--replace-host=host1 @pparam=--name-space=ns1 @pparam=--metrics-prefix=prefix1 @pparam=--exact-match=true @pparam=--log-name=log1 {% if salt.pillar.get("roles_video_fetch_backend") %} @pparam=--fetch-policy=simple @pparam=--front=false \{% else %} \{% endif %} {% if salt.pillar.get("roles_video_fetch_frontend") %} @pparam=--fetch-policy=lru:123 @pparam=--fetch-path-pattern=/(.*-)(\d+)(.*)/$1{$2+2}$3/ @pparam=--fetch-path-pattern=/(.*)(\d{3})(_.*\.jpg.*)/$1{$2+1:3}$3/ @pparam=--front=true \{% else %} \{% endif %} + @plugin=regex_remap.so @pparam=all/regex2.config \ + @plugin=s3_auth.so @pparam=--config @pparam=certs/s3.config \ + @pparam=--virtual_host \ + @pparam=--version=4 \ + @pparam=--v4-include-headers=header1,header2 \ + @pparam=--v4-exclude-headers=header3,header4 \ + @pparam=--v4-region-map=region_map_test.config \ + @plugin=conf_remap.so @pparam=proxy.config.http.cache.generation=3 \ + @pparam=proxy.config.http.connect_attempts_timeout=600 \ + @pparam=proxy.config.http.transaction_no_activity_timeout_in=600 \ + @pparam=proxy.config.http.transaction_no_activity_timeout_out=600 \ + @pparam=proxy.config.http.cache.http=0 \ + @pparam=proxy.config.http.negative_caching_lifetime=3600 \ + @pparam=proxy.config.url_remap.pristine_host_hdr=0 \ + @pparam=proxy.config.http.connect_attempts_timeout=600 \ + @pparam=proxy.config.http.transaction_no_activity_timeout_in=600 \ + @pparam=proxy.config.http.transaction_no_activity_timeout_out=600 \ + @plugin=header_rewrite.so @pparam=all/hdrs2.config \ + @src_ip=10.153.0.0-10.153.255.255 @src_ip=10.179.0.0-10.179.255.255 @action=allow \ + @action=allow @method=PURGE \ + @action=deny @method=POST @method=PUT @method=DELETE + +map http://myalias4.example.com \ + http://origin4.example.com \ + @plugin=access_control.so @pparam=--symmetric-keys-map=hmac_keys.txt \ + @pparam=--check-cookie=TokenCookie \ + @pparam=--extract-subject-to-header=@TokenSubject \ + @pparam=--extract-tokenid-to-header=@TokenId \ + @pparam=--extract-status-to-header=@TokenStatus \ + @pparam=--token-response-header=TokenRespHdr \ +{% if salt.pillar.get("roles_trafficserver_cache_promote") %} @plugin=cache_promote.so @pparam=--policy=lru @pparam=--hits=5 @pparam=--buckets=100000 \{% else %} \{% endif %} + @plugin=cachekey.so @pparam=--include-params=param1,param2 \ + @pparam=--include-cookies=cookie1,cookie2 \ + @pparam=--include-headers=header1,header2 \ + @pparam=--exclude-params=param1,param2 \ + @pparam=--static-prefix=SomeString \ + @pparam=--sort-params=true \ + @pparam=--remove-all-params=true \ + @pparam=--capture-path=/.*/probe-info-ok.html/ \ + @pparam=--capture-path-uri=/http[s]?:\/\/[^\/]*(.*\.html|.*).*/$1/ \ + @pparam=--capture-header=Authorization:/AWS\s([^:]+).*/clientID:$1/ \ + @pparam=--capture-header=Authorization:/BYT\s([^,]+).*/clientID:$1/ \ + @plugin=gzip.so @pparam=all/gzip2.config \ + @plugin=escalate.so @pparam=--pristine \ + @pparam=401,403,404,407,410,500,501,502,503,504,505:assets-origin.subdomain.example.com \ + @plugin=regex_remap.so @pparam=all/strip_query2.config \ + @plugin=volcano.so @pparam=--fetch-count=10 @pparam=--fetch-max=1000 @pparam=--api-header=header1 @pparam=--replace-host=host1 @pparam=--name-space=ns1 @pparam=--metrics-prefix=prefix1 @pparam=--exact-match=true @pparam=--log-name=log1 {% if salt.pillar.get("roles_video_fetch_backend") %} @pparam=--fetch-policy=simple @pparam=--front=false \{% else %} \{% endif %} {% if salt.pillar.get("roles_video_fetch_frontend") %} @pparam=--fetch-policy=lru:123 @pparam=--fetch-path-pattern=/(.*-)(\d+)(.*)/$1{$2+2}$3/ @pparam=--fetch-path-pattern=/(.*)(\d{3})(_.*\.jpg.*)/$1{$2+1:3}$3/ @pparam=--front=true \{% else %} \{% endif %} + @plugin=regex_remap.so @pparam=all/regex2.config \ + @plugin=s3_auth.so @pparam=--config @pparam=certs/s3.config \ + @pparam=--virtual_host \ + @pparam=--version=4 \ + @pparam=--v4-include-headers=header1,header2 \ + @pparam=--v4-exclude-headers=header3,header4 \ + @pparam=--v4-region-map=region_map_test.config \ + @plugin=conf_remap.so @pparam=proxy.config.http.cache.generation=3 \ + @pparam=proxy.config.http.connect_attempts_timeout=600 \ + @pparam=proxy.config.http.transaction_no_activity_timeout_in=600 \ + @pparam=proxy.config.http.transaction_no_activity_timeout_out=600 \ + @pparam=proxy.config.http.cache.http=0 \ + @pparam=proxy.config.http.negative_caching_lifetime=3600 \ + @pparam=proxy.config.url_remap.pristine_host_hdr=0 \ + @pparam=proxy.config.http.connect_attempts_timeout=600 \ + @pparam=proxy.config.http.transaction_no_activity_timeout_in=600 \ + @pparam=proxy.config.http.transaction_no_activity_timeout_out=600 \ + @plugin=header_rewrite.so @pparam=all/hdrs2.config \ + @src_ip=10.153.0.0-10.153.255.255 @src_ip=10.179.0.0-10.179.255.255 @action=allow \ + @action=allow @method=PURGE \ + @action=deny @method=POST @method=PUT @method=DELETE + +map https://myalias4.example.com \ + https://origin4.example.com \ + @plugin=access_control.so @pparam=--symmetric-keys-map=hmac_keys.txt \ + @pparam=--check-cookie=TokenCookie \ + @pparam=--extract-subject-to-header=@TokenSubject \ + @pparam=--extract-tokenid-to-header=@TokenId \ + @pparam=--extract-status-to-header=@TokenStatus \ + @pparam=--token-response-header=TokenRespHdr \ +{% if salt.pillar.get("roles_trafficserver_cache_promote") %} @plugin=cache_promote.so @pparam=--policy=lru @pparam=--hits=5 @pparam=--buckets=100000 \{% else %} \{% endif %} + @plugin=cachekey.so @pparam=--include-params=param1,param2 \ + @pparam=--include-cookies=cookie1,cookie2 \ + @pparam=--include-headers=header1,header2 \ + @pparam=--exclude-params=param1,param2 \ + @pparam=--static-prefix=SomeString \ + @pparam=--sort-params=true \ + @pparam=--remove-all-params=true \ + @pparam=--capture-path=/.*/probe-info-ok.html/ \ + @pparam=--capture-path-uri=/http[s]?:\/\/[^\/]*(.*\.html|.*).*/$1/ \ + @pparam=--capture-header=Authorization:/AWS\s([^:]+).*/clientID:$1/ \ + @pparam=--capture-header=Authorization:/BYT\s([^,]+).*/clientID:$1/ \ + @plugin=gzip.so @pparam=all/gzip2.config \ + @plugin=escalate.so @pparam=--pristine \ + @pparam=401,403,404,407,410,500,501,502,503,504,505:assets-origin.subdomain.example.com \ + @plugin=regex_remap.so @pparam=all/strip_query2.config \ + @plugin=volcano.so @pparam=--fetch-count=10 @pparam=--fetch-max=1000 @pparam=--api-header=header1 @pparam=--replace-host=host1 @pparam=--name-space=ns1 @pparam=--metrics-prefix=prefix1 @pparam=--exact-match=true @pparam=--log-name=log1 {% if salt.pillar.get("roles_video_fetch_backend") %} @pparam=--fetch-policy=simple @pparam=--front=false \{% else %} \{% endif %} {% if salt.pillar.get("roles_video_fetch_frontend") %} @pparam=--fetch-policy=lru:123 @pparam=--fetch-path-pattern=/(.*-)(\d+)(.*)/$1{$2+2}$3/ @pparam=--fetch-path-pattern=/(.*)(\d{3})(_.*\.jpg.*)/$1{$2+1:3}$3/ @pparam=--front=true \{% else %} \{% endif %} + @plugin=regex_remap.so @pparam=all/regex2.config \ + @plugin=s3_auth.so @pparam=--config @pparam=certs/s3.config \ + @pparam=--virtual_host \ + @pparam=--version=4 \ + @pparam=--v4-include-headers=header1,header2 \ + @pparam=--v4-exclude-headers=header3,header4 \ + @pparam=--v4-region-map=region_map_test.config \ + @plugin=conf_remap.so @pparam=proxy.config.http.cache.generation=3 \ + @pparam=proxy.config.http.connect_attempts_timeout=600 \ + @pparam=proxy.config.http.transaction_no_activity_timeout_in=600 \ + @pparam=proxy.config.http.transaction_no_activity_timeout_out=600 \ + @pparam=proxy.config.http.cache.http=0 \ + @pparam=proxy.config.http.negative_caching_lifetime=3600 \ + @pparam=proxy.config.url_remap.pristine_host_hdr=0 \ + @pparam=proxy.config.http.connect_attempts_timeout=600 \ + @pparam=proxy.config.http.transaction_no_activity_timeout_in=600 \ + @pparam=proxy.config.http.transaction_no_activity_timeout_out=600 \ + @plugin=header_rewrite.so @pparam=all/hdrs2.config \ + @src_ip=10.153.0.0-10.153.255.255 @src_ip=10.179.0.0-10.179.255.255 @action=allow \ + @action=allow @method=PURGE \ + @action=deny @method=POST @method=PUT @method=DELETE + +map http://myalias5.example.com \ + http://origin4.example.com \ + @plugin=header_rewrite.so @pparam=all/hdrs6.config + +{% if salt.pillar.get("roles_edge_child") %} + +map http://myalias6.example.com \ + http://origin6.example.com \ + @plugin=volcano.so {% if salt.pillar.get("roles_video_fetch_backend") %} @pparam=--fetch-policy=simple @pparam=--front=false \{% else %} \{% endif %} {% if salt.pillar.get("roles_video_fetch_frontend") %} @pparam=--fetch-policy=lru:123 @pparam=--front=true \{% else %} \{% endif %} + +{% endif %} + +map http://myalias8.example.com \ + http://origin8.example.com \ + @plugin=header_rewrite.so @pparam=all/hdrs10.config + +map http://myalias9.example.com \ + http://origin9.example.com \ + @plugin=s3_auth.so @pparam=--config @pparam=certs/s3.config \ + @pparam=--virtual_host \ + @pparam=--version=4 \ + @pparam=--v4-region-map=region_map_test.config \ + @pparam=--v4-exclude-headers=x-forwarded-for,forwarded,via,authorization + +map http://myaliasa.example.com \ + http://origina.example.com \ + @plugin=s3_auth.so @pparam=--config @pparam=certs/s3.config \ + @pparam=--virtual_host \ + @pparam=--version=2 \ + @pparam=--v4-region-map=region_map_test.config + diff --git a/tools/voluspa/tests/regress/all/base/all/gzip2.config b/tools/voluspa/tests/regress/all/base/all/gzip2.config new file mode 100644 index 00000000000..6dc3e065479 --- /dev/null +++ b/tools/voluspa/tests/regress/all/base/all/gzip2.config @@ -0,0 +1,10 @@ +# Code generated by Voluspa. DO NOT EDIT. + +enabled true +cache false +remove-accept-encoding true +flush true +minimum-content-length 532 +supported-algorithms br,gzip +compressible-content-type *javascript* +compressible-content-type text/* diff --git a/tools/voluspa/tests/regress/all/base/all/hdrs10.config b/tools/voluspa/tests/regress/all/base/all/hdrs10.config new file mode 100644 index 00000000000..7ae70a9452b --- /dev/null +++ b/tools/voluspa/tests/regress/all/base/all/hdrs10.config @@ -0,0 +1,4 @@ +# Code generated by Voluspa. DO NOT EDIT. + +cond %{REMAP_PSEUDO_HOOK} + set-header @cdnlog "private" diff --git a/tools/voluspa/tests/regress/all/base/all/hdrs2.config b/tools/voluspa/tests/regress/all/base/all/hdrs2.config new file mode 100644 index 00000000000..1f5c961bd93 --- /dev/null +++ b/tools/voluspa/tests/regress/all/base/all/hdrs2.config @@ -0,0 +1,52 @@ +# Code generated by Voluspa. DO NOT EDIT. + +cond %{READ_RESPONSE_HDR_HOOK} + add-header X-Client-Protocol % +cond %{SEND_REQUEST_HDR_HOOK} + add-header X-TestOriginHeader "1" +cond %{SEND_RESPONSE_HDR_HOOK} [AND] + cond %{PATH} /.*\.crl$/ + set-header Content-Type application/pkix-crl +cond %{SEND_RESPONSE_HDR_HOOK} [AND] + cond %{PATH} /.*\.css$/ + set-header Content-Type text/css +cond %{SEND_RESPONSE_HDR_HOOK} [AND] + cond %{PATH} /.*\.json$/ + set-header Content-Type application/json + +cond %{READ_RESPONSE_HDR_HOOK} [AND] +cond %{HEADER:Cache-Control} ="" [AND] +cond %{STATUS} >199 [AND] +cond %{STATUS} <400 + set-header Cache-Control "max-age=3600, public" +cond %{READ_RESPONSE_HDR_HOOK} [AND] +cond %{STATUS} >199 [AND] +cond %{STATUS} <400 + set-header Cache-Control "max-age=0, public" +cond %{SEND_REQUEST_HDR_HOOK} + add-header Authorization "Basic 123412341234" [L] + +cond %{REMAP_PSEUDO_HOOK} + set-header Bazinga1 %{COOKIE:NameOfCookie1} + set-header Bazinga2 %{COOKIE:NameOfCookie2} +cond %{REMAP_PSEUDO_HOOK} + set-header Bazinga3 %{HEADER:NameOfHeader1} + set-header Bazinga4 %{HEADER:NameOfHeader2} + +cond %{REMAP_PSEUDO_HOOK} + set-conn-dscp 8 +cond %{READ_RESPONSE_HDR_HOOK} + set-header @Original-Cache-Control %{HEADER:Cache-Control} + set-header Cache-Control %{HEADER:MyProxyHeader} + +cond %{SEND_RESPONSE_HDR_HOOK} + set-header Cache-Control %{HEADER:@Original-Cache-Control} + rm-header Proxy-Cache-Control +cond %{REMAP_PSEUDO_HOOK} + rm-header Proxy +cond %{SEND_REQUEST_HDR_HOOK} + rm-header Proxy +cond %{READ_RESPONSE_HDR_HOOK} + set-header X-Client-Protocol % +cond %{SEND_REQUEST_HDR_HOOK} + set-header X-TestOriginHeader "1" diff --git a/tools/voluspa/tests/regress/all/base/all/hdrs6.config b/tools/voluspa/tests/regress/all/base/all/hdrs6.config new file mode 100644 index 00000000000..d1606fafb7e --- /dev/null +++ b/tools/voluspa/tests/regress/all/base/all/hdrs6.config @@ -0,0 +1,4 @@ +# Code generated by Voluspa. DO NOT EDIT. + +cond %{REMAP_PSEUDO_HOOK} + set-header @ReceiptService "override{{hosttype}}" diff --git a/tools/voluspa/tests/regress/all/base/all/redirect9.config b/tools/voluspa/tests/regress/all/base/all/redirect9.config new file mode 100644 index 00000000000..866f0304a09 --- /dev/null +++ b/tools/voluspa/tests/regress/all/base/all/redirect9.config @@ -0,0 +1,3 @@ +# Code generated by Voluspa. DO NOT EDIT. + +(.*) https://example.com/filenotfound @status=302 diff --git a/tools/voluspa/tests/regress/all/base/all/regex2.config b/tools/voluspa/tests/regress/all/base/all/regex2.config new file mode 100644 index 00000000000..9889c40e6ce --- /dev/null +++ b/tools/voluspa/tests/regress/all/base/all/regex2.config @@ -0,0 +1,3 @@ +# Code generated by Voluspa. DO NOT EDIT. + +^/([^/]+)/(.*) https://$1.s3.amazonaws.com/$2 diff --git a/tools/voluspa/tests/regress/all/base/all/strip_query2.config b/tools/voluspa/tests/regress/all/base/all/strip_query2.config new file mode 100644 index 00000000000..7b0312493d4 --- /dev/null +++ b/tools/voluspa/tests/regress/all/base/all/strip_query2.config @@ -0,0 +1,3 @@ +# Code generated by Voluspa. DO NOT EDIT. + +. $s://$t/$P diff --git a/tools/voluspa/tests/regress/all/base/hosting.config_default b/tools/voluspa/tests/regress/all/base/hosting.config_default new file mode 100644 index 00000000000..4dd9b032a1c --- /dev/null +++ b/tools/voluspa/tests/regress/all/base/hosting.config_default @@ -0,0 +1,7 @@ +# Code generated by Voluspa. DO NOT EDIT. + +{% if salt.pillar.get("roles_trafficserver_ramdisk") %} +hostname=* {{ default_volumes }} +{% else %} +# hosting.config disabled on this host +{% endif %} diff --git a/tools/voluspa/tests/regress/all/base/ssl_multicert.config_default b/tools/voluspa/tests/regress/all/base/ssl_multicert.config_default new file mode 100644 index 00000000000..4fad5702530 --- /dev/null +++ b/tools/voluspa/tests/regress/all/base/ssl_multicert.config_default @@ -0,0 +1,9 @@ +# Code generated by Voluspa. DO NOT EDIT. + +{% if not salt.pillar.get('roles_uat') %} + +# your reference notes here +ssl_cert_name=/secrets/mysecret.txt + +{% endif %} + diff --git a/tools/voluspa/tests/regress/all/test.conf b/tools/voluspa/tests/regress/all/test.conf new file mode 100644 index 00000000000..71461394018 --- /dev/null +++ b/tools/voluspa/tests/regress/all/test.conf @@ -0,0 +1,239 @@ +schema_version: 1 +owner: "" +reference: your reference notes here +rcpt: All +ssl_cert_names: + - /secrets/mysecret.txt + +rules: + default: + add_header: "X-Client-Protocol %" + add_header_origin: X-TestOriginHeader "1" + set_header: "X-Client-Protocol %" + set_header_origin: X-TestOriginHeader "1" + remove_header: "Proxy" + remove_header_origin: "Proxy" + allow_ip: [10.153.0.0-10.153.255.255, 10.179.0.0-10.179.255.255] + allow_ip_parent: [10.253.0.0-10.253.255.255, 10.254.0.0-10.254.255.255] + strip_query: true + force_ttl: 0s + proxy_cache_control: "MyProxyHeader" + origin_host_header: origin + disable_cache: true + deny_methods: [ POST, PUT, DELETE ] + allow_methods: [ PURGE ] + regex_remap: "^/([^/]+)/(.*) https://$1.s3.amazonaws.com/$2" + s3: + path: certs/s3.config + virtual_host: true + version: 4 + v4_include_headers: ["header1", "header2"] + v4_exclude_headers: ["header3", "header4"] + region_map: region_map_test.config + video_background_fetch: + fetch_count: 10 + fetch_max: 1000 + api_header: header1 + replace_host: host1 + name_space: ns1 + metrics_prefix: prefix1 + exact_match: true + log_name: log1 + frontend: + fetch_path_pattern: + - '/(.*-)(\d+)(.*)/$1{$2+2}$3/' + - /(.*)(\d{3})(_.*\.jpg.*)/$1{$2+1:3}$3/ + fetch_policy: lru:123 + backend: + fetch_policy: simple + + cache_promote: + policy: lru + lru_hits: 5 + lru_buckets: 100000 + + cachekey: + include_params: ["param1", "param2"] + exclude_params: ["param1", "param2"] + include_headers: ["header1", "header2"] + include_cookies: ["cookie1", "cookie2"] + sort_query: true + remove_all_params: true + static_prefix: "SomeString" + regex_replace_path: '/.*/probe-info-ok.html/' + regex_replace_path_uri: '/http[s]?:\/\/[^\/]*(.*\.html|.*).*/$1/' + capture_header: [ 'Authorization:/AWS\s([^:]+).*/clientID:$1/', 'Authorization:/BYT\s([^,]+).*/clientID:$1/' ] + failover: + domain: assets-origin.subdomain.example.com + status_codes: [401, 403, 404, 407, 410, 500, 501, 502, 503, 504, 505] + host_header: origin + priority: background + default_ttl: 60m + content_type_forge: + '.*\.json$': application/json + '.*\.css$': text/css + '.*\.crl$': application/pkix-crl + conf_remap: "CONFIG proxy.config.http.connect_attempts_timeout INT 600\n + CONFIG proxy.config.http.transaction_no_activity_timeout_in INT 600\n + CONFIG proxy.config.http.transaction_no_activity_timeout_out INT 600" + header_rewrite: "cond %{SEND_REQUEST_HDR_HOOK}\n + add-header Authorization \"Basic 123412341234\" [L]" + log_cookie: + Logger-cookie-Name1: NameOfCookie1 + Logger-cookie-Name2: NameOfCookie2 + log_header: + Logger-header-Name1: NameOfHeader1 + Logger-header-Name2: NameOfHeader2 + negative_ttl: 60m + transaction_timeout: 10m + edge_compression: + enabled: true + flush: true + remove_accept_encoding: true + minimum_content_length: 532 + algorithms: [gzip, br] + static_origin_compressible_content_type: + - "text/*" + - "*javascript*" + access_approval: + raw: |- + @pparam=--symmetric-keys-map=hmac_keys.txt + @pparam=--check-cookie=TokenCookie + @pparam=--extract-subject-to-header=@TokenSubject + @pparam=--extract-tokenid-to-header=@TokenId + @pparam=--extract-status-to-header=@TokenStatus + @pparam=--token-response-header=TokenRespHdr + cache_version: 3 + + no_receipts: + receipts: + enabled: false + + override_receipt: + receipts: + name: override + + # Redirect takes one redirect url and status code. + # If status code is empty, it takes default value of 302. + redirect_this: + redirect: + url: https://example.com/filenotfound + + preserve_host_header: + origin_host_header: alias + receipts: + enabled: false + priority: foreground + s3: + path: /opt/ats/etc/trafficserver/certs/s3.config + virtual_host: false + + escalate_pristine: + failover: + domain: assets-origin.subdomain.example.com + status_codes: [401, 403, 404, 407, 410, 500, 501, 502, 503, 504, 505] + host_header: origin + + streamingaudio: + priority: streamingaudio + + streamingvideo: + priority: streamingvideo + + roles_within_roles: + video_background_fetch: + frontend: + fetch_policy: lru:123 + backend: + fetch_policy: simple + + cache_promote: + cache_promote: + policy: lru + lru_hits: 5 + lru_buckets: 100000 + + rlog_type: + log_type: private + + storage_volume_ram: + storage_volume: ramdisk_volume + + storage_volume_disk: + storage_volume: disk_volume + + raw_lua: + lua: |- + function do_remap() + local to_host = ts.remap.get_to_url_host() + ts.debug(to_host) + return 0 + end + + echo_cors_test: + echo_cors: "Origin" + + s3_v4: + s3: + path: certs/s3.config + virtual_host: true + version: 4 + region_map: region_map_test.config + + + s3_v2: + s3: + path: certs/s3.config + virtual_host: true + version: 2 + region_map: region_map_test.config + + +explicit: + - origin: https://origin2.example.com + alias: + - https://myalias1.example.com + - https://myalias2.example.com + rule: no_receipts + - origin: http://origin3.example.com + alias: + - http://(.*).example.com + regex_map: true + - origin: origin4.example.com + alias: + - myalias4.example.com + schemes: [http, https] + - origin: origin4.example.com + alias: + - myalias5.example.com + rule: override_receipt + - origin: origin6.example.com + alias: + - myalias6.example.com + role: roles_edge_child + rule: roles_within_roles + - origin: origin8.example.com + alias: + - myalias8.example.com + rule: rlog_type + - origin: origin9.example.com + alias: + - myalias9.example.com + rule: s3_v4 + - origin: origina.example.com + alias: + - myaliasa.example.com + rule: s3_v2 + +mappings: + - origin: origin7.example.com + alias: + - myalias7.example.com + - myalias7.example.com/ran + - myalias7.example.com/bob + - /path/test.txt + rule: no_receipts + - origin: origin7.example.com + alias: + - redirect.example.com + rule: redirect_this diff --git a/tests/tools/sessionvalidation/__init__.py b/tools/voluspa/tests/regress/properties.sh similarity index 98% rename from tests/tools/sessionvalidation/__init__.py rename to tools/voluspa/tests/regress/properties.sh index bcbf6855425..f5c3cc28307 100644 --- a/tests/tools/sessionvalidation/__init__.py +++ b/tools/voluspa/tests/regress/properties.sh @@ -1,5 +1,4 @@ -''' -''' +# # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -15,3 +14,5 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +# +CUST=(all) diff --git a/tools/voluspa/tests/regress/regen_baseline.sh b/tools/voluspa/tests/regress/regen_baseline.sh new file mode 100755 index 00000000000..3733ceb352f --- /dev/null +++ b/tools/voluspa/tests/regress/regen_baseline.sh @@ -0,0 +1,36 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +set +e + +. properties.sh + +if [ $# -gt 0 ]; then + CUST=($1) +fi + +CMD=../../voluspa +SCHEMA=../.. + +for i in ${CUST[@]}; do + DEST=./${i}/base + rm -rf $DEST + mkdir $DEST + ./$CMD --schema-location $SCHEMA --dest $DEST ${i}/*.conf +done diff --git a/tests/tools/sessionvalidation/transaction.py b/tools/voluspa/tests/regress/test.sh old mode 100644 new mode 100755 similarity index 55% rename from tests/tools/sessionvalidation/transaction.py rename to tools/voluspa/tests/regress/test.sh index 19950abea69..5e6b66890ea --- a/tests/tools/sessionvalidation/transaction.py +++ b/tools/voluspa/tests/regress/test.sh @@ -1,5 +1,5 @@ -''' -''' +#!/bin/bash +# # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -15,26 +15,38 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +# -import sessionvalidation.request as request -import sessionvalidation.response as response +. properties.sh +if [ $# -gt 0 ]; then + CUST=($1) +fi -class Transaction(object): - ''' Tranaction encapsulates a single UA transaction ''' +# this expects 'make voluspa' to be run (make sanity_tests or make test does this) +if [ -z "$VOLUSPA" ]; then + VOLUSPA="voluspa" +fi - def getRequest(self): - return self._request +CMD=../../$DEST/$VOLUSPA +SCHEMA=../.. - def getResponse(self): - return self._response +for i in ${CUST[@]}; do + DEST=./${i}/test + rm -rf $DEST + mkdir $DEST + ./$CMD --schema-location $SCHEMA --dest $DEST ${i}/*.conf > /dev/null + if [ "$VOLUSPA" == "voluspa" ]; then + if [ $? != 0 ]; then + exit 1 + fi - def __repr__(self): - return "".format( - self._uuid, self._request, self._response - ) + diff -x .gitignore -x \*.swp -r ${i}/base $DEST + if [ $? != 0 ]; then + exit 1 + fi + else + mv system.out profile_${i}.out + fi +done - def __init__(self, request, response, uuid): - self._request = request - self._response = response - self._uuid = uuid diff --git a/tools/voluspa/toplevel_remaps.go b/tools/voluspa/toplevel_remaps.go new file mode 100644 index 00000000000..102d5621662 --- /dev/null +++ b/tools/voluspa/toplevel_remaps.go @@ -0,0 +1,197 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package voluspa + +import ( + "bytes" + "fmt" + "sort" + "strings" +) + +type TopLevelRemapGenerator struct { + options *Options +} + +func newTopLevelRemapGenerator(options *Options) *TopLevelRemapGenerator { + return &TopLevelRemapGenerator{options: options} +} + +func (tlrg *TopLevelRemapGenerator) formatDescriptionComment(config *CustomerConfig) string { + var domain string + if strings.HasPrefix(config.Remaps[0].IncomingURL, "http") { + parts := strings.Split(config.Remaps[0].IncomingURL, "/") + if len(parts) >= 2 { + domain = parts[2] + } + } + + var buf bytes.Buffer + buf.WriteString("############################################################################\n") + buf.WriteString(fmt.Sprintf("# Host: %s\n", domain)) + buf.WriteString(fmt.Sprintf("# Owner: %s\n", config.owner)) + if len(config.reference) > 0 { + buf.WriteString(fmt.Sprintf("# Reference: %s\n", config.reference)) + } + buf.WriteString("############################################################################\n") + return buf.String() +} + +func (tlrg *TopLevelRemapGenerator) filenameForRole(role, cdn, parentOrChild string) string { + var buf bytes.Buffer + if cdn == DefaultCDN || cdn == NoCDN { + buf.WriteString("remap_") + } else { + buf.WriteString(fmt.Sprintf("%s_remap_", cdn)) + } + + if len(role) > 0 { + buf.WriteString(fmt.Sprintf("%s_", role)) + } + + buf.WriteString(fmt.Sprintf("%s_include.config", parentOrChild)) + + return buf.String() +} + +// groupedRemaps returns true if a Group has been explicitly set +func groupedRemaps(remaps []Remap) bool { + for _, remap := range remaps { + if remap.Group != DefaultGroup { + return true + } + } + return false +} + +func (tlrg *TopLevelRemapGenerator) Do(parsedConfigs []*CustomerConfig, merge bool) ([]ManagedFile, error) { + if merge { + return tlrg.get(parsedConfigs, NoRole, NoCDN) + } + + if len(parsedConfigs) == 0 { + return nil, ErrMinimumConfigsNotMet + } + + // Generating the include file is next to useless for 1 file + if len(parsedConfigs) == 1 { + return nil, nil + } + + // group properties by CDN and role and generate includeable files for each role/parent/child combo + grouped := make(map[string]map[string][]*CustomerConfig) + + for _, config := range parsedConfigs { + if config.lifecycle == Retired { + continue + } + + for cdn := range config.cdn { + + if _, exists := grouped[cdn]; !exists { + grouped[cdn] = make(map[string][]*CustomerConfig) + } + + role := config.role + grouped[cdn][role] = append(grouped[cdn][role], config) + } + } + var managedFiles []ManagedFile + for cdn, cdnGrouped := range grouped { + for role, configs := range cdnGrouped { + sort.Stable(sort.Reverse(byQPSAndName(configs))) + + remaps, err := tlrg.get(configs, role, cdn) + if err != nil { + return nil, err + } + managedFiles = append(managedFiles, remaps...) + } + } + + return managedFiles, nil + +} + +func (tlrg *TopLevelRemapGenerator) get(configs []*CustomerConfig, role, cdn string) ([]ManagedFile, error) { + var managedFiles []ManagedFile + + for _, parentOrChild := range []string{Child, Parent} { + + var buf bytes.Buffer + buf.WriteString("# START voluspa-generated config\n") + + if len(role) == 0 { + buf.WriteString("\n") + } else { + buf.WriteString(fmt.Sprintf("# ROLE: %s\n\n", role)) + } + + for _, parsedConfig := range configs { + if _, exists := parsedConfig.cdn[cdn]; !exists && cdn != NoCDN { + continue + } + + // skip grouped remap sets as the output cannot be used + // without knowledge beyond the scope of voluspa + // (feature used by cdn-icloud-content-uat) + if groupedRemaps(parsedConfig.Remaps) { + continue + } + + property := strings.ToLower(parsedConfig.property) + buf.WriteString(tlrg.formatDescriptionComment(parsedConfig)) + + filename := property + + if parentOrChild == Child || !parsedConfig.parentChild { + buf.WriteString(fmt.Sprintf(".include %s/%s.config\n", property, filename)) + } else { + buf.WriteString(fmt.Sprintf(".include %s/%s_parent.config\n", property, filename)) + } + buf.WriteString("\n") + } + buf.WriteString("# END voluspa-generated config\n\n") + + fileName := tlrg.filenameForRole(role, cdn, parentOrChild) + + var remoteFilename string + if len(role) > 0 { + remoteFilename = fmt.Sprintf("remap_voluspa_%s.config", role) + } else { + remoteFilename = "remap_voluspa.config" + } + + managedFiles = append(managedFiles, NewManagedFile(fileName, remoteFilename, cdn, role, "", &buf, tlrg.configType(parentOrChild))) + } + + return managedFiles, nil +} + +func (tlrg *TopLevelRemapGenerator) configType(val string) ConfigLocation { + switch val { + case Child: + return ChildConfig + case Parent: + return ParentConfig + default: + panic(fmt.Errorf("Unhandled configType '%s'", val)) + } +} diff --git a/tools/voluspa/util.go b/tools/voluspa/util.go new file mode 100644 index 00000000000..4d5e984c04f --- /dev/null +++ b/tools/voluspa/util.go @@ -0,0 +1,32 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package voluspa + +// groupCustomerConfigsByCDN groups CustomerConfigs by CDN +func groupCustomerConfigsByCDN(parsedConfigs []*CustomerConfig) map[string][]*CustomerConfig { + // group properties by CDN and generate parent.config for each CDN + grouped := make(map[string][]*CustomerConfig) + for _, config := range parsedConfigs { + for cdn := range config.cdn { + grouped[cdn] = append(grouped[cdn], config) + } + } + return grouped +} diff --git a/tools/voluspa/validate.go b/tools/voluspa/validate.go new file mode 100644 index 00000000000..eebb1e189b8 --- /dev/null +++ b/tools/voluspa/validate.go @@ -0,0 +1,312 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package voluspa + +import ( + "errors" + "fmt" + "net" + "net/url" + "sort" + "strings" + + "github.com/apache/trafficserver/tools/voluspa/internal/util/regex" +) + +type Errors []error + +func (err Errors) Error() string { + if len(err) == 0 { + return "Unknown" + } + + var errOut []string + for _, e := range err { + errOut = append(errOut, e.Error()) + } + sort.Strings(errOut) + + return strings.Join(errOut, "\n") +} + +func (err *Errors) Add(newErr error) { + *err = append(*err, newErr) +} + +func (err *Errors) HasErrors() bool { + return len(*err) > 0 +} + +func isValidScheme(scheme string) bool { + return scheme == "http" || scheme == "https" +} + +func (v *Voluspa) ensureValidSchemes() Errors { + var errs Errors + for _, config := range v.parsedConfigs { + for _, scheme := range config.schemes { + if !isValidScheme(scheme) { + errs.Add(fmt.Errorf("property %q contains invalid scheme: %s (%s)", config.property, scheme, config.filename)) + } + } + + for _, remap := range config.Remaps { + if len(remap.scheme) > 0 && !isValidScheme(remap.scheme) { + errs.Add(fmt.Errorf("property %q contains invalid scheme: %s (%s)", config.property, remap.scheme, config.filename)) + } + + } + } + if len(errs) > 0 { + return errs + } + return nil +} + +func (v *Voluspa) ensurePropertyNameUniqueness() Errors { + var errs Errors + + properties := make(map[string]interface{}) + + for _, config := range v.parsedConfigs { + if _, exists := properties[config.property]; exists { + errs.Add(fmt.Errorf("property %s not unique (%s)", config.property, config.filename)) + } else { + properties[config.property] = true + } + } + + if len(errs) > 0 { + return errs + } + return nil +} + +func (v *Voluspa) ensureAliasUniqueness() Errors { + var errs Errors + // map of cdn to role to configlocation to alias + aliases := make(map[string]map[string]map[ConfigLocation]map[string]interface{}) + + // Initialize map/struct + for _, config := range v.parsedConfigs { + for cdn := range config.cdn { + if _, exists := aliases[cdn]; !exists { + aliases[cdn] = make(map[string]map[ConfigLocation]map[string]interface{}) + } + if _, exists := aliases[cdn][config.role]; !exists { + aliases[cdn][config.role] = make(map[ConfigLocation]map[string]interface{}) + for _, location := range []ConfigLocation{UnknownLocation, ChildConfig, ParentConfig} { + aliases[cdn][config.role][location] = make(map[string]interface{}) + } + } + } + } + + for _, config := range v.parsedConfigs { + for cdn := range config.cdn { + for _, remap := range config.Remaps { + alias := remap.IncomingURL + + // aliases are unique based on the aliases itself + the group it's part of and the role + aliasKey := alias + remap.Group + remap.Role + + if _, exists := aliases[cdn][config.role][remap.ConfigLocation][aliasKey]; exists { + var err error + switch { + case len(config.role) > 0: + err = fmt.Errorf("alias %s not unique for role %q (%s)", alias, config.role, config.filename) + case len(remap.Role) > 0: + err = fmt.Errorf("alias %s not unique for rule role %q (%s)", alias, remap.Role, config.filename) + default: + err = fmt.Errorf("alias %s not unique (%s)", alias, config.filename) + } + errs.Add(err) + } else { + aliases[cdn][config.role][remap.ConfigLocation][aliasKey] = nil + } + } + } + } + + if len(errs) > 0 { + return errs + } + return nil +} + +func (v *Voluspa) ensureHostsResolve() Errors { + var errs Errors + + for _, config := range v.parsedConfigs { + for _, remap := range config.Remaps { + if remap.RegexMap { + continue + } + + url, err := url.Parse(remap.OriginURL) + if err != nil { + errs.Add(fmt.Errorf("%s invalid url; %s (%s)", remap.OriginURL, err, config.filename)) + } + if err = isResolvable(url.Host); err != nil { + errs = append(errs, fmt.Errorf("%s unresolvable host (%s)", url.Host, config.filename)) + } + } + } + + if len(errs) > 0 { + return errs + } + return nil +} + +func isResolvable(host string) error { + if strings.Contains(host, ":") { + host = strings.Split(host, ":")[0] + } + + _, err := net.LookupHost(host) + return err +} + +// See https://docs.trafficserver.apache.org/en/latest/admin-guide/plugins/cachekey.en.html#capture-definition +// This is either a regex or a "/" + regex_with_capture + "/" + regex_replace + "/" +func validateCacheKeyCapture(input string) error { + + validator := ®ex.GoLangRegex{} + + regexes := make([]string, 0) + currentRe := "" + prev := ' ' + + // loop over all the characters + for _, c := range input { + if c == '/' && prev != '\\' { // find an unescaped / - it is the boundary + if currentRe != "" { + regexes = append(regexes, currentRe) + } + currentRe = "" + prev = c + continue + } + currentRe += string(c) + prev = c + } + + // if nothing was pushed, there was no / at all, so push the whole string and a dummy + if currentRe != "" { + regexes = append(regexes, currentRe) + regexes = append(regexes, "dummy") + } + + // too many unescaped /'s + if len(regexes) != 2 { + return errors.New("unexpected /") + } + + // check the regexes + for _, re := range regexes { + _, err := validator.IsValid(re) + if err != nil { + return err + } + } + return nil +} + +func (v *Voluspa) validateRegexOptions() Errors { + validator := ®ex.GoLangRegex{} + + var errs Errors + for _, config := range v.parsedConfigs { + for _, remap := range config.Remaps { + if remap.RegexMap { + _, err := validator.IsValid(remap.IncomingURL) + if err != nil { + errs = append(errs, fmt.Errorf("invalid regex URL: %s (%s)", err, config.filename)) + } + } + + for k := range remap.sourceConfig.RemapOptions { + switch k { + case "header_rewrite", // HANDLE header_rewrite separately (split and look for Cond) + "regex_remap", + "asset_token_include", + "asset_token_exclude": + + v, err := remap.sourceConfig.RemapOptions.ValueByNameAsString(k) + if err != nil { + errs = append(errs, fmt.Errorf("%s invalid field. err=%s (%s)", k, err, config.filename)) + continue + } + + _, err = validator.IsValid(v) + if err != nil { + errs = append(errs, fmt.Errorf("invalid regex: %s (%s)", err, config.filename)) + } + + case "content_type_forge": + forgeMap, err := remap.sourceConfig.RemapOptions.ValueByNameAsStringMapString(k) + if err != nil { + errs = append(errs, fmt.Errorf("%s invalid field. err=%s (%s)", k, err, config.filename)) + continue + } + + for mapKey := range forgeMap { + _, err := validator.IsValid(mapKey) + if err != nil { + errs = append(errs, fmt.Errorf("invalid regex: %s (%s)", err, config.filename)) + } + } + case "cachekey": + cachekeyMap, err := remap.sourceConfig.RemapOptions.ValueByNameAsStringMapInterface(k) + if err != nil { + errs = append(errs, fmt.Errorf("%s invalid field. err=%s (%s)", k, err, config.filename)) + continue + } + + for mapKey, mapValue := range cachekeyMap { + if mapKey == "regex_replace_path" || mapKey == "regex_replace_path_uri" || mapKey == "capture_path" { + err := validateCacheKeyCapture(mapValue.(string)) + if err != nil { + errs = append(errs, fmt.Errorf("invalid regex: %s (%s)", err, config.filename)) + } + } + if mapKey == "capture_header" { + for _, re := range mapValue.([]interface{}) { + regexParts := strings.Split(re.(string), ":")[1:] + regexString := strings.Join(regexParts, ":") + err := validateCacheKeyCapture(regexString) + if err != nil { + errs = append(errs, fmt.Errorf("invalid regex: %s (%s)", err, config.filename)) + } + + } + } + } + } + } + } + } + + if len(errs) > 0 { + return errs + } + return nil +} diff --git a/tools/voluspa/vendor/.gitignore b/tools/voluspa/vendor/.gitignore new file mode 100644 index 00000000000..7fa54591b3a --- /dev/null +++ b/tools/voluspa/vendor/.gitignore @@ -0,0 +1,9 @@ +github.com/hpcloud/tail/vendor +.travis* +README* +CONTRIB* +*sh +*pl +*conf +Gopkg.* +*yaml diff --git a/tools/voluspa/vendor/github.com/xeipuuv/gojsonpointer/LICENSE-APACHE-2.0.txt b/tools/voluspa/vendor/github.com/xeipuuv/gojsonpointer/LICENSE-APACHE-2.0.txt new file mode 100644 index 00000000000..55ede8a42cc --- /dev/null +++ b/tools/voluspa/vendor/github.com/xeipuuv/gojsonpointer/LICENSE-APACHE-2.0.txt @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2015 xeipuuv + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/tools/voluspa/vendor/github.com/xeipuuv/gojsonpointer/pointer.go b/tools/voluspa/vendor/github.com/xeipuuv/gojsonpointer/pointer.go new file mode 100644 index 00000000000..7faf5d7f943 --- /dev/null +++ b/tools/voluspa/vendor/github.com/xeipuuv/gojsonpointer/pointer.go @@ -0,0 +1,211 @@ +// Copyright 2015 xeipuuv ( https://github.com/xeipuuv ) +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// author xeipuuv +// author-github https://github.com/xeipuuv +// author-mail xeipuuv@gmail.com +// +// repository-name gojsonpointer +// repository-desc An implementation of JSON Pointer - Go language +// +// description Main and unique file. +// +// created 25-02-2013 + +package gojsonpointer + +import ( + "errors" + "fmt" + "reflect" + "strconv" + "strings" +) + +const ( + const_empty_pointer = `` + const_pointer_separator = `/` + + const_invalid_start = `JSON pointer must be empty or start with a "` + const_pointer_separator + `"` +) + +type implStruct struct { + mode string // "SET" or "GET" + + inDocument interface{} + + setInValue interface{} + + getOutNode interface{} + getOutKind reflect.Kind + outError error +} + +type JsonPointer struct { + referenceTokens []string +} + +// NewJsonPointer parses the given string JSON pointer and returns an object +func NewJsonPointer(jsonPointerString string) (p JsonPointer, err error) { + + // Pointer to the root of the document + if len(jsonPointerString) == 0 { + // Keep referenceTokens nil + return + } + if jsonPointerString[0] != '/' { + return p, errors.New(const_invalid_start) + } + + p.referenceTokens = strings.Split(jsonPointerString[1:], const_pointer_separator) + return +} + +// Uses the pointer to retrieve a value from a JSON document +func (p *JsonPointer) Get(document interface{}) (interface{}, reflect.Kind, error) { + + is := &implStruct{mode: "GET", inDocument: document} + p.implementation(is) + return is.getOutNode, is.getOutKind, is.outError + +} + +// Uses the pointer to update a value from a JSON document +func (p *JsonPointer) Set(document interface{}, value interface{}) (interface{}, error) { + + is := &implStruct{mode: "SET", inDocument: document, setInValue: value} + p.implementation(is) + return document, is.outError + +} + +// Uses the pointer to delete a value from a JSON document +func (p *JsonPointer) Delete(document interface{}) (interface{}, error) { + is := &implStruct{mode: "DEL", inDocument: document} + p.implementation(is) + return document, is.outError +} + +// Both Get and Set functions use the same implementation to avoid code duplication +func (p *JsonPointer) implementation(i *implStruct) { + + kind := reflect.Invalid + + // Full document when empty + if len(p.referenceTokens) == 0 { + i.getOutNode = i.inDocument + i.outError = nil + i.getOutKind = kind + i.outError = nil + return + } + + node := i.inDocument + + previousNodes := make([]interface{}, len(p.referenceTokens)) + previousTokens := make([]string, len(p.referenceTokens)) + + for ti, token := range p.referenceTokens { + + isLastToken := ti == len(p.referenceTokens)-1 + previousNodes[ti] = node + previousTokens[ti] = token + + switch v := node.(type) { + + case map[string]interface{}: + decodedToken := decodeReferenceToken(token) + if _, ok := v[decodedToken]; ok { + node = v[decodedToken] + if isLastToken && i.mode == "SET" { + v[decodedToken] = i.setInValue + } else if isLastToken && i.mode =="DEL" { + delete(v,decodedToken) + } + } else if (isLastToken && i.mode == "SET") { + v[decodedToken] = i.setInValue + } else { + i.outError = fmt.Errorf("Object has no key '%s'", decodedToken) + i.getOutKind = reflect.Map + i.getOutNode = nil + return + } + + case []interface{}: + tokenIndex, err := strconv.Atoi(token) + if err != nil { + i.outError = fmt.Errorf("Invalid array index '%s'", token) + i.getOutKind = reflect.Slice + i.getOutNode = nil + return + } + if tokenIndex < 0 || tokenIndex >= len(v) { + i.outError = fmt.Errorf("Out of bound array[0,%d] index '%d'", len(v), tokenIndex) + i.getOutKind = reflect.Slice + i.getOutNode = nil + return + } + + node = v[tokenIndex] + if isLastToken && i.mode == "SET" { + v[tokenIndex] = i.setInValue + } else if isLastToken && i.mode =="DEL" { + v[tokenIndex] = v[len(v)-1] + v[len(v)-1] = nil + v = v[:len(v)-1] + previousNodes[ti-1].(map[string]interface{})[previousTokens[ti-1]] = v + } + + default: + i.outError = fmt.Errorf("Invalid token reference '%s'", token) + i.getOutKind = reflect.ValueOf(node).Kind() + i.getOutNode = nil + return + } + + } + + i.getOutNode = node + i.getOutKind = reflect.ValueOf(node).Kind() + i.outError = nil +} + +// Pointer to string representation function +func (p *JsonPointer) String() string { + + if len(p.referenceTokens) == 0 { + return const_empty_pointer + } + + pointerString := const_pointer_separator + strings.Join(p.referenceTokens, const_pointer_separator) + + return pointerString +} + +// Specific JSON pointer encoding here +// ~0 => ~ +// ~1 => / +// ... and vice versa + +func decodeReferenceToken(token string) string { + step1 := strings.Replace(token, `~1`, `/`, -1) + step2 := strings.Replace(step1, `~0`, `~`, -1) + return step2 +} + +func encodeReferenceToken(token string) string { + step1 := strings.Replace(token, `~`, `~0`, -1) + step2 := strings.Replace(step1, `/`, `~1`, -1) + return step2 +} diff --git a/tools/voluspa/vendor/github.com/xeipuuv/gojsonreference/LICENSE-APACHE-2.0.txt b/tools/voluspa/vendor/github.com/xeipuuv/gojsonreference/LICENSE-APACHE-2.0.txt new file mode 100644 index 00000000000..55ede8a42cc --- /dev/null +++ b/tools/voluspa/vendor/github.com/xeipuuv/gojsonreference/LICENSE-APACHE-2.0.txt @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2015 xeipuuv + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/tools/voluspa/vendor/github.com/xeipuuv/gojsonreference/reference.go b/tools/voluspa/vendor/github.com/xeipuuv/gojsonreference/reference.go new file mode 100644 index 00000000000..6457291301d --- /dev/null +++ b/tools/voluspa/vendor/github.com/xeipuuv/gojsonreference/reference.go @@ -0,0 +1,147 @@ +// Copyright 2015 xeipuuv ( https://github.com/xeipuuv ) +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// author xeipuuv +// author-github https://github.com/xeipuuv +// author-mail xeipuuv@gmail.com +// +// repository-name gojsonreference +// repository-desc An implementation of JSON Reference - Go language +// +// description Main and unique file. +// +// created 26-02-2013 + +package gojsonreference + +import ( + "errors" + "net/url" + "path/filepath" + "runtime" + "strings" + + "github.com/xeipuuv/gojsonpointer" +) + +const ( + const_fragment_char = `#` +) + +func NewJsonReference(jsonReferenceString string) (JsonReference, error) { + + var r JsonReference + err := r.parse(jsonReferenceString) + return r, err + +} + +type JsonReference struct { + referenceUrl *url.URL + referencePointer gojsonpointer.JsonPointer + + HasFullUrl bool + HasUrlPathOnly bool + HasFragmentOnly bool + HasFileScheme bool + HasFullFilePath bool +} + +func (r *JsonReference) GetUrl() *url.URL { + return r.referenceUrl +} + +func (r *JsonReference) GetPointer() *gojsonpointer.JsonPointer { + return &r.referencePointer +} + +func (r *JsonReference) String() string { + + if r.referenceUrl != nil { + return r.referenceUrl.String() + } + + if r.HasFragmentOnly { + return const_fragment_char + r.referencePointer.String() + } + + return r.referencePointer.String() +} + +func (r *JsonReference) IsCanonical() bool { + return (r.HasFileScheme && r.HasFullFilePath) || (!r.HasFileScheme && r.HasFullUrl) +} + +// "Constructor", parses the given string JSON reference +func (r *JsonReference) parse(jsonReferenceString string) (err error) { + + r.referenceUrl, err = url.Parse(jsonReferenceString) + if err != nil { + return + } + refUrl := r.referenceUrl + + if refUrl.Scheme != "" && refUrl.Host != "" { + r.HasFullUrl = true + } else { + if refUrl.Path != "" { + r.HasUrlPathOnly = true + } else if refUrl.RawQuery == "" && refUrl.Fragment != "" { + r.HasFragmentOnly = true + } + } + + r.HasFileScheme = refUrl.Scheme == "file" + if runtime.GOOS == "windows" { + // on Windows, a file URL may have an extra leading slash, and if it + // doesn't then its first component will be treated as the host by the + // Go runtime + if refUrl.Host == "" && strings.HasPrefix(refUrl.Path, "/") { + r.HasFullFilePath = filepath.IsAbs(refUrl.Path[1:]) + } else { + r.HasFullFilePath = filepath.IsAbs(refUrl.Host + refUrl.Path) + } + } else { + r.HasFullFilePath = filepath.IsAbs(refUrl.Path) + } + + // invalid json-pointer error means url has no json-pointer fragment. simply ignore error + r.referencePointer, _ = gojsonpointer.NewJsonPointer(refUrl.Fragment) + + return +} + +// Creates a new reference from a parent and a child +// If the child cannot inherit from the parent, an error is returned +func (r *JsonReference) Inherits(child JsonReference) (*JsonReference, error) { + if child.GetUrl() == nil { + return nil, errors.New("childUrl is nil!") + } + + if r.GetUrl() == nil { + return nil, errors.New("parentUrl is nil!") + } + + // Get a copy of the parent url to make sure we do not modify the original. + // URL reference resolving fails if the fragment of the child is empty, but the parent's is not. + // The fragment of the child must be used, so the fragment of the parent is manually removed. + parentUrl := *r.GetUrl() + parentUrl.Fragment = "" + + ref, err := NewJsonReference(parentUrl.ResolveReference(child.GetUrl()).String()) + if err != nil { + return nil, err + } + return &ref, err +} diff --git a/tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/.gitignore b/tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/.gitignore new file mode 100644 index 00000000000..68e993ce3e0 --- /dev/null +++ b/tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/.gitignore @@ -0,0 +1,3 @@ +*.sw[nop] +*.iml +.vscode/ diff --git a/tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/LICENSE-APACHE-2.0.txt b/tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/LICENSE-APACHE-2.0.txt new file mode 100644 index 00000000000..55ede8a42cc --- /dev/null +++ b/tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/LICENSE-APACHE-2.0.txt @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2015 xeipuuv + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/errors.go b/tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/errors.go new file mode 100644 index 00000000000..2f326a2228b --- /dev/null +++ b/tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/errors.go @@ -0,0 +1,324 @@ +package gojsonschema + +import ( + "bytes" + "sync" + "text/template" +) + +var errorTemplates errorTemplate = errorTemplate{template.New("errors-new"), sync.RWMutex{}} + +// template.Template is not thread-safe for writing, so some locking is done +// sync.RWMutex is used for efficiently locking when new templates are created +type errorTemplate struct { + *template.Template + sync.RWMutex +} + +type ( + // RequiredError. ErrorDetails: property string + RequiredError struct { + ResultErrorFields + } + + // InvalidTypeError. ErrorDetails: expected, given + InvalidTypeError struct { + ResultErrorFields + } + + // NumberAnyOfError. ErrorDetails: - + NumberAnyOfError struct { + ResultErrorFields + } + + // NumberOneOfError. ErrorDetails: - + NumberOneOfError struct { + ResultErrorFields + } + + // NumberAllOfError. ErrorDetails: - + NumberAllOfError struct { + ResultErrorFields + } + + // NumberNotError. ErrorDetails: - + NumberNotError struct { + ResultErrorFields + } + + // MissingDependencyError. ErrorDetails: dependency + MissingDependencyError struct { + ResultErrorFields + } + + // InternalError. ErrorDetails: error + InternalError struct { + ResultErrorFields + } + + // ConstError. ErrorDetails: allowed + ConstError struct { + ResultErrorFields + } + + // EnumError. ErrorDetails: allowed + EnumError struct { + ResultErrorFields + } + + // ArrayNoAdditionalItemsError. ErrorDetails: - + ArrayNoAdditionalItemsError struct { + ResultErrorFields + } + + // ArrayMinItemsError. ErrorDetails: min + ArrayMinItemsError struct { + ResultErrorFields + } + + // ArrayMaxItemsError. ErrorDetails: max + ArrayMaxItemsError struct { + ResultErrorFields + } + + // ItemsMustBeUniqueError. ErrorDetails: type + ItemsMustBeUniqueError struct { + ResultErrorFields + } + + // ArrayContainsError. ErrorDetails: + ArrayContainsError struct { + ResultErrorFields + } + + // ArrayMinPropertiesError. ErrorDetails: min + ArrayMinPropertiesError struct { + ResultErrorFields + } + + // ArrayMaxPropertiesError. ErrorDetails: max + ArrayMaxPropertiesError struct { + ResultErrorFields + } + + // AdditionalPropertyNotAllowedError. ErrorDetails: property + AdditionalPropertyNotAllowedError struct { + ResultErrorFields + } + + // InvalidPropertyPatternError. ErrorDetails: property, pattern + InvalidPropertyPatternError struct { + ResultErrorFields + } + + // InvalidPopertyNameError. ErrorDetails: property + InvalidPropertyNameError struct { + ResultErrorFields + } + + // StringLengthGTEError. ErrorDetails: min + StringLengthGTEError struct { + ResultErrorFields + } + + // StringLengthLTEError. ErrorDetails: max + StringLengthLTEError struct { + ResultErrorFields + } + + // DoesNotMatchPatternError. ErrorDetails: pattern + DoesNotMatchPatternError struct { + ResultErrorFields + } + + // DoesNotMatchFormatError. ErrorDetails: format + DoesNotMatchFormatError struct { + ResultErrorFields + } + + // MultipleOfError. ErrorDetails: multiple + MultipleOfError struct { + ResultErrorFields + } + + // NumberGTEError. ErrorDetails: min + NumberGTEError struct { + ResultErrorFields + } + + // NumberGTError. ErrorDetails: min + NumberGTError struct { + ResultErrorFields + } + + // NumberLTEError. ErrorDetails: max + NumberLTEError struct { + ResultErrorFields + } + + // NumberLTError. ErrorDetails: max + NumberLTError struct { + ResultErrorFields + } + + // ConditionThenError. ErrorDetails: - + ConditionThenError struct { + ResultErrorFields + } + + // ConditionElseError. ErrorDetails: - + ConditionElseError struct { + ResultErrorFields + } +) + +// newError takes a ResultError type and sets the type, context, description, details, value, and field +func newError(err ResultError, context *JsonContext, value interface{}, locale locale, details ErrorDetails) { + var t string + var d string + switch err.(type) { + case *RequiredError: + t = "required" + d = locale.Required() + case *InvalidTypeError: + t = "invalid_type" + d = locale.InvalidType() + case *NumberAnyOfError: + t = "number_any_of" + d = locale.NumberAnyOf() + case *NumberOneOfError: + t = "number_one_of" + d = locale.NumberOneOf() + case *NumberAllOfError: + t = "number_all_of" + d = locale.NumberAllOf() + case *NumberNotError: + t = "number_not" + d = locale.NumberNot() + case *MissingDependencyError: + t = "missing_dependency" + d = locale.MissingDependency() + case *InternalError: + t = "internal" + d = locale.Internal() + case *ConstError: + t = "const" + d = locale.Const() + case *EnumError: + t = "enum" + d = locale.Enum() + case *ArrayNoAdditionalItemsError: + t = "array_no_additional_items" + d = locale.ArrayNoAdditionalItems() + case *ArrayMinItemsError: + t = "array_min_items" + d = locale.ArrayMinItems() + case *ArrayMaxItemsError: + t = "array_max_items" + d = locale.ArrayMaxItems() + case *ItemsMustBeUniqueError: + t = "unique" + d = locale.Unique() + case *ArrayContainsError: + t = "contains" + d = locale.ArrayContains() + case *ArrayMinPropertiesError: + t = "array_min_properties" + d = locale.ArrayMinProperties() + case *ArrayMaxPropertiesError: + t = "array_max_properties" + d = locale.ArrayMaxProperties() + case *AdditionalPropertyNotAllowedError: + t = "additional_property_not_allowed" + d = locale.AdditionalPropertyNotAllowed() + case *InvalidPropertyPatternError: + t = "invalid_property_pattern" + d = locale.InvalidPropertyPattern() + case *InvalidPropertyNameError: + t = "invalid_property_name" + d = locale.InvalidPropertyName() + case *StringLengthGTEError: + t = "string_gte" + d = locale.StringGTE() + case *StringLengthLTEError: + t = "string_lte" + d = locale.StringLTE() + case *DoesNotMatchPatternError: + t = "pattern" + d = locale.DoesNotMatchPattern() + case *DoesNotMatchFormatError: + t = "format" + d = locale.DoesNotMatchFormat() + case *MultipleOfError: + t = "multiple_of" + d = locale.MultipleOf() + case *NumberGTEError: + t = "number_gte" + d = locale.NumberGTE() + case *NumberGTError: + t = "number_gt" + d = locale.NumberGT() + case *NumberLTEError: + t = "number_lte" + d = locale.NumberLTE() + case *NumberLTError: + t = "number_lt" + d = locale.NumberLT() + case *ConditionThenError: + t = "condition_then" + d = locale.ConditionThen() + case *ConditionElseError: + t = "condition_else" + d = locale.ConditionElse() + } + + err.SetType(t) + err.SetContext(context) + err.SetValue(value) + err.SetDetails(details) + err.SetDescriptionFormat(d) + details["field"] = err.Field() + + if _, exists := details["context"]; !exists && context != nil { + details["context"] = context.String() + } + + err.SetDescription(formatErrorDescription(err.DescriptionFormat(), details)) +} + +// formatErrorDescription takes a string in the default text/template +// format and converts it to a string with replacements. The fields come +// from the ErrorDetails struct and vary for each type of error. +func formatErrorDescription(s string, details ErrorDetails) string { + + var tpl *template.Template + var descrAsBuffer bytes.Buffer + var err error + + errorTemplates.RLock() + tpl = errorTemplates.Lookup(s) + errorTemplates.RUnlock() + + if tpl == nil { + errorTemplates.Lock() + tpl = errorTemplates.New(s) + + if ErrorTemplateFuncs != nil { + tpl.Funcs(ErrorTemplateFuncs) + } + + tpl, err = tpl.Parse(s) + errorTemplates.Unlock() + + if err != nil { + return err.Error() + } + } + + err = tpl.Execute(&descrAsBuffer, details) + if err != nil { + return err.Error() + } + + return descrAsBuffer.String() +} diff --git a/tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/format_checkers.go b/tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/format_checkers.go new file mode 100644 index 00000000000..26217fca128 --- /dev/null +++ b/tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/format_checkers.go @@ -0,0 +1,343 @@ +package gojsonschema + +import ( + "net" + "net/mail" + "net/url" + "regexp" + "strings" + "sync" + "time" +) + +type ( + // FormatChecker is the interface all formatters added to FormatCheckerChain must implement + FormatChecker interface { + IsFormat(input interface{}) bool + } + + // FormatCheckerChain holds the formatters + FormatCheckerChain struct { + formatters map[string]FormatChecker + } + + // EmailFormatter verifies email address formats + EmailFormatChecker struct{} + + // IPV4FormatChecker verifies IP addresses in the ipv4 format + IPV4FormatChecker struct{} + + // IPV6FormatChecker verifies IP addresses in the ipv6 format + IPV6FormatChecker struct{} + + // DateTimeFormatChecker verifies date/time formats per RFC3339 5.6 + // + // Valid formats: + // Partial Time: HH:MM:SS + // Full Date: YYYY-MM-DD + // Full Time: HH:MM:SSZ-07:00 + // Date Time: YYYY-MM-DDTHH:MM:SSZ-0700 + // + // Where + // YYYY = 4DIGIT year + // MM = 2DIGIT month ; 01-12 + // DD = 2DIGIT day-month ; 01-28, 01-29, 01-30, 01-31 based on month/year + // HH = 2DIGIT hour ; 00-23 + // MM = 2DIGIT ; 00-59 + // SS = 2DIGIT ; 00-58, 00-60 based on leap second rules + // T = Literal + // Z = Literal + // + // Note: Nanoseconds are also suported in all formats + // + // http://tools.ietf.org/html/rfc3339#section-5.6 + DateTimeFormatChecker struct{} + + DateFormatChecker struct{} + + TimeFormatChecker struct{} + + // URIFormatChecker validates a URI with a valid Scheme per RFC3986 + URIFormatChecker struct{} + + // URIReferenceFormatChecker validates a URI or relative-reference per RFC3986 + URIReferenceFormatChecker struct{} + + // URITemplateFormatChecker validates a URI template per RFC6570 + URITemplateFormatChecker struct{} + + // HostnameFormatChecker validates a hostname is in the correct format + HostnameFormatChecker struct{} + + // UUIDFormatChecker validates a UUID is in the correct format + UUIDFormatChecker struct{} + + // RegexFormatChecker validates a regex is in the correct format + RegexFormatChecker struct{} + + // JSONPointerFormatChecker validates a JSON Pointer per RFC6901 + JSONPointerFormatChecker struct{} + + // RelativeJSONPointerFormatChecker validates a relative JSON Pointer is in the correct format + RelativeJSONPointerFormatChecker struct{} +) + +var ( + // Formatters holds the valid formatters, and is a public variable + // so library users can add custom formatters + FormatCheckers = FormatCheckerChain{ + formatters: map[string]FormatChecker{ + "date": DateFormatChecker{}, + "time": TimeFormatChecker{}, + "date-time": DateTimeFormatChecker{}, + "hostname": HostnameFormatChecker{}, + "email": EmailFormatChecker{}, + "idn-email": EmailFormatChecker{}, + "ipv4": IPV4FormatChecker{}, + "ipv6": IPV6FormatChecker{}, + "uri": URIFormatChecker{}, + "uri-reference": URIReferenceFormatChecker{}, + "iri": URIFormatChecker{}, + "iri-reference": URIReferenceFormatChecker{}, + "uri-template": URITemplateFormatChecker{}, + "uuid": UUIDFormatChecker{}, + "regex": RegexFormatChecker{}, + "json-pointer": JSONPointerFormatChecker{}, + "relative-json-pointer": RelativeJSONPointerFormatChecker{}, + }, + } + + // Regex credit: https://www.socketloop.com/tutorials/golang-validate-hostname + rxHostname = regexp.MustCompile(`^([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]{0,61}[a-zA-Z0-9])(\.([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]{0,61}[a-zA-Z0-9]))*$`) + + // Use a regex to make sure curly brackets are balanced properly after validating it as a AURI + rxURITemplate = regexp.MustCompile("^([^{]*({[^}]*})?)*$") + + rxUUID = regexp.MustCompile("^[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}$") + + rxJSONPointer = regexp.MustCompile("^(?:/(?:[^~/]|~0|~1)*)*$") + + rxRelJSONPointer = regexp.MustCompile("^(?:0|[1-9][0-9]*)(?:#|(?:/(?:[^~/]|~0|~1)*)*)$") + + lock = new(sync.Mutex) +) + +// Add adds a FormatChecker to the FormatCheckerChain +// The name used will be the value used for the format key in your json schema +func (c *FormatCheckerChain) Add(name string, f FormatChecker) *FormatCheckerChain { + lock.Lock() + c.formatters[name] = f + lock.Unlock() + + return c +} + +// Remove deletes a FormatChecker from the FormatCheckerChain (if it exists) +func (c *FormatCheckerChain) Remove(name string) *FormatCheckerChain { + lock.Lock() + delete(c.formatters, name) + lock.Unlock() + + return c +} + +// Has checks to see if the FormatCheckerChain holds a FormatChecker with the given name +func (c *FormatCheckerChain) Has(name string) bool { + lock.Lock() + _, ok := c.formatters[name] + lock.Unlock() + + return ok +} + +// IsFormat will check an input against a FormatChecker with the given name +// to see if it is the correct format +func (c *FormatCheckerChain) IsFormat(name string, input interface{}) bool { + f, ok := c.formatters[name] + + if !ok { + return false + } + + return f.IsFormat(input) +} + +func (f EmailFormatChecker) IsFormat(input interface{}) bool { + + asString, ok := input.(string) + if ok == false { + return false + } + + _, err := mail.ParseAddress(asString) + + return err == nil +} + +// Credit: https://github.com/asaskevich/govalidator +func (f IPV4FormatChecker) IsFormat(input interface{}) bool { + + asString, ok := input.(string) + if ok == false { + return false + } + + ip := net.ParseIP(asString) + return ip != nil && strings.Contains(asString, ".") +} + +// Credit: https://github.com/asaskevich/govalidator +func (f IPV6FormatChecker) IsFormat(input interface{}) bool { + + asString, ok := input.(string) + if ok == false { + return false + } + + ip := net.ParseIP(asString) + return ip != nil && strings.Contains(asString, ":") +} + +func (f DateTimeFormatChecker) IsFormat(input interface{}) bool { + + asString, ok := input.(string) + if ok == false { + return false + } + + formats := []string{ + "15:04:05", + "15:04:05Z07:00", + "2006-01-02", + time.RFC3339, + time.RFC3339Nano, + } + + for _, format := range formats { + if _, err := time.Parse(format, asString); err == nil { + return true + } + } + + return false +} + +func (f DateFormatChecker) IsFormat(input interface{}) bool { + asString, ok := input.(string) + if ok == false { + return false + } + _, err := time.Parse("2006-01-02", asString) + return err == nil +} + +func (f TimeFormatChecker) IsFormat(input interface{}) bool { + asString, ok := input.(string) + if ok == false { + return false + } + + if _, err := time.Parse("15:04:05Z07:00", asString); err == nil { + return true + } + + _, err := time.Parse("15:04:05", asString) + return err == nil +} + +func (f URIFormatChecker) IsFormat(input interface{}) bool { + + asString, ok := input.(string) + if ok == false { + return false + } + + u, err := url.Parse(asString) + + if err != nil || u.Scheme == "" { + return false + } + + return !strings.Contains(asString, `\`) +} + +func (f URIReferenceFormatChecker) IsFormat(input interface{}) bool { + + asString, ok := input.(string) + if ok == false { + return false + } + + _, err := url.Parse(asString) + return err == nil && !strings.Contains(asString, `\`) +} + +func (f URITemplateFormatChecker) IsFormat(input interface{}) bool { + asString, ok := input.(string) + if ok == false { + return false + } + + u, err := url.Parse(asString) + if err != nil || strings.Contains(asString, `\`) { + return false + } + + return rxURITemplate.MatchString(u.Path) +} + +func (f HostnameFormatChecker) IsFormat(input interface{}) bool { + + asString, ok := input.(string) + if ok == false { + return false + } + + return rxHostname.MatchString(asString) && len(asString) < 256 +} + +func (f UUIDFormatChecker) IsFormat(input interface{}) bool { + + asString, ok := input.(string) + if ok == false { + return false + } + + return rxUUID.MatchString(asString) +} + +// IsFormat implements FormatChecker interface. +func (f RegexFormatChecker) IsFormat(input interface{}) bool { + + asString, ok := input.(string) + if ok == false { + return false + } + + if asString == "" { + return true + } + _, err := regexp.Compile(asString) + if err != nil { + return false + } + return true +} + +func (f JSONPointerFormatChecker) IsFormat(input interface{}) bool { + asString, ok := input.(string) + if ok == false { + return false + } + + return rxJSONPointer.MatchString(asString) +} + +func (f RelativeJSONPointerFormatChecker) IsFormat(input interface{}) bool { + asString, ok := input.(string) + if ok == false { + return false + } + + return rxRelJSONPointer.MatchString(asString) +} diff --git a/tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/internalLog.go b/tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/internalLog.go new file mode 100644 index 00000000000..4ef7a8d03e7 --- /dev/null +++ b/tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/internalLog.go @@ -0,0 +1,37 @@ +// Copyright 2015 xeipuuv ( https://github.com/xeipuuv ) +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// author xeipuuv +// author-github https://github.com/xeipuuv +// author-mail xeipuuv@gmail.com +// +// repository-name gojsonschema +// repository-desc An implementation of JSON Schema, based on IETF's draft v4 - Go language. +// +// description Very simple log wrapper. +// Used for debugging/testing purposes. +// +// created 01-01-2015 + +package gojsonschema + +import ( + "log" +) + +const internalLogEnabled = false + +func internalLog(format string, v ...interface{}) { + log.Printf(format, v...) +} diff --git a/tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/jsonContext.go b/tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/jsonContext.go new file mode 100644 index 00000000000..f40668a74c4 --- /dev/null +++ b/tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/jsonContext.go @@ -0,0 +1,72 @@ +// Copyright 2013 MongoDB, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// author tolsen +// author-github https://github.com/tolsen +// +// repository-name gojsonschema +// repository-desc An implementation of JSON Schema, based on IETF's draft v4 - Go language. +// +// description Implements a persistent (immutable w/ shared structure) singly-linked list of strings for the purpose of storing a json context +// +// created 04-09-2013 + +package gojsonschema + +import "bytes" + +// JsonContext implements a persistent linked-list of strings +type JsonContext struct { + head string + tail *JsonContext +} + +func NewJsonContext(head string, tail *JsonContext) *JsonContext { + return &JsonContext{head, tail} +} + +// String displays the context in reverse. +// This plays well with the data structure's persistent nature with +// Cons and a json document's tree structure. +func (c *JsonContext) String(del ...string) string { + byteArr := make([]byte, 0, c.stringLen()) + buf := bytes.NewBuffer(byteArr) + c.writeStringToBuffer(buf, del) + + return buf.String() +} + +func (c *JsonContext) stringLen() int { + length := 0 + if c.tail != nil { + length = c.tail.stringLen() + 1 // add 1 for "." + } + + length += len(c.head) + return length +} + +func (c *JsonContext) writeStringToBuffer(buf *bytes.Buffer, del []string) { + if c.tail != nil { + c.tail.writeStringToBuffer(buf, del) + + if len(del) > 0 { + buf.WriteString(del[0]) + } else { + buf.WriteString(".") + } + } + + buf.WriteString(c.head) +} diff --git a/tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/jsonLoader.go b/tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/jsonLoader.go new file mode 100644 index 00000000000..cbe46649218 --- /dev/null +++ b/tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/jsonLoader.go @@ -0,0 +1,362 @@ +// Copyright 2015 xeipuuv ( https://github.com/xeipuuv ) +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// author xeipuuv +// author-github https://github.com/xeipuuv +// author-mail xeipuuv@gmail.com +// +// repository-name gojsonschema +// repository-desc An implementation of JSON Schema, based on IETF's draft v4 - Go language. +// +// description Different strategies to load JSON files. +// Includes References (file and HTTP), JSON strings and Go types. +// +// created 01-02-2015 + +package gojsonschema + +import ( + "bytes" + "encoding/json" + "errors" + "io" + "io/ioutil" + "net/http" + "os" + "path/filepath" + "runtime" + "strings" + + "github.com/xeipuuv/gojsonreference" +) + +var osFS = osFileSystem(os.Open) + +// JSON loader interface + +type JSONLoader interface { + JsonSource() interface{} + LoadJSON() (interface{}, error) + JsonReference() (gojsonreference.JsonReference, error) + LoaderFactory() JSONLoaderFactory +} + +type JSONLoaderFactory interface { + New(source string) JSONLoader +} + +type DefaultJSONLoaderFactory struct { +} + +type FileSystemJSONLoaderFactory struct { + fs http.FileSystem +} + +func (d DefaultJSONLoaderFactory) New(source string) JSONLoader { + return &jsonReferenceLoader{ + fs: osFS, + source: source, + } +} + +func (f FileSystemJSONLoaderFactory) New(source string) JSONLoader { + return &jsonReferenceLoader{ + fs: f.fs, + source: source, + } +} + +// osFileSystem is a functional wrapper for os.Open that implements http.FileSystem. +type osFileSystem func(string) (*os.File, error) + +func (o osFileSystem) Open(name string) (http.File, error) { + return o(name) +} + +// JSON Reference loader +// references are used to load JSONs from files and HTTP + +type jsonReferenceLoader struct { + fs http.FileSystem + source string +} + +func (l *jsonReferenceLoader) JsonSource() interface{} { + return l.source +} + +func (l *jsonReferenceLoader) JsonReference() (gojsonreference.JsonReference, error) { + return gojsonreference.NewJsonReference(l.JsonSource().(string)) +} + +func (l *jsonReferenceLoader) LoaderFactory() JSONLoaderFactory { + return &FileSystemJSONLoaderFactory{ + fs: l.fs, + } +} + +// NewReferenceLoader returns a JSON reference loader using the given source and the local OS file system. +func NewReferenceLoader(source string) JSONLoader { + return &jsonReferenceLoader{ + fs: osFS, + source: source, + } +} + +// NewReferenceLoaderFileSystem returns a JSON reference loader using the given source and file system. +func NewReferenceLoaderFileSystem(source string, fs http.FileSystem) JSONLoader { + return &jsonReferenceLoader{ + fs: fs, + source: source, + } +} + +func (l *jsonReferenceLoader) LoadJSON() (interface{}, error) { + + var err error + + reference, err := gojsonreference.NewJsonReference(l.JsonSource().(string)) + if err != nil { + return nil, err + } + + refToUrl := reference + refToUrl.GetUrl().Fragment = "" + + var document interface{} + + if reference.HasFileScheme { + + filename := strings.TrimPrefix(refToUrl.String(), "file://") + if runtime.GOOS == "windows" { + // on Windows, a file URL may have an extra leading slash, use slashes + // instead of backslashes, and have spaces escaped + filename = strings.TrimPrefix(filename, "/") + filename = filepath.FromSlash(filename) + } + + document, err = l.loadFromFile(filename) + if err != nil { + return nil, err + } + + } else { + + document, err = l.loadFromHTTP(refToUrl.String()) + if err != nil { + return nil, err + } + + } + + return document, nil + +} + +func (l *jsonReferenceLoader) loadFromHTTP(address string) (interface{}, error) { + + resp, err := http.Get(address) + if err != nil { + return nil, err + } + + // must return HTTP Status 200 OK + if resp.StatusCode != http.StatusOK { + return nil, errors.New(formatErrorDescription(Locale.HttpBadStatus(), ErrorDetails{"status": resp.Status})) + } + + bodyBuff, err := ioutil.ReadAll(resp.Body) + if err != nil { + return nil, err + } + + return decodeJsonUsingNumber(bytes.NewReader(bodyBuff)) + +} + +func (l *jsonReferenceLoader) loadFromFile(path string) (interface{}, error) { + f, err := l.fs.Open(path) + if err != nil { + return nil, err + } + defer f.Close() + + bodyBuff, err := ioutil.ReadAll(f) + if err != nil { + return nil, err + } + + return decodeJsonUsingNumber(bytes.NewReader(bodyBuff)) + +} + +// JSON string loader + +type jsonStringLoader struct { + source string +} + +func (l *jsonStringLoader) JsonSource() interface{} { + return l.source +} + +func (l *jsonStringLoader) JsonReference() (gojsonreference.JsonReference, error) { + return gojsonreference.NewJsonReference("#") +} + +func (l *jsonStringLoader) LoaderFactory() JSONLoaderFactory { + return &DefaultJSONLoaderFactory{} +} + +func NewStringLoader(source string) JSONLoader { + return &jsonStringLoader{source: source} +} + +func (l *jsonStringLoader) LoadJSON() (interface{}, error) { + + return decodeJsonUsingNumber(strings.NewReader(l.JsonSource().(string))) + +} + +// JSON bytes loader + +type jsonBytesLoader struct { + source []byte +} + +func (l *jsonBytesLoader) JsonSource() interface{} { + return l.source +} + +func (l *jsonBytesLoader) JsonReference() (gojsonreference.JsonReference, error) { + return gojsonreference.NewJsonReference("#") +} + +func (l *jsonBytesLoader) LoaderFactory() JSONLoaderFactory { + return &DefaultJSONLoaderFactory{} +} + +func NewBytesLoader(source []byte) JSONLoader { + return &jsonBytesLoader{source: source} +} + +func (l *jsonBytesLoader) LoadJSON() (interface{}, error) { + return decodeJsonUsingNumber(bytes.NewReader(l.JsonSource().([]byte))) +} + +// JSON Go (types) loader +// used to load JSONs from the code as maps, interface{}, structs ... + +type jsonGoLoader struct { + source interface{} +} + +func (l *jsonGoLoader) JsonSource() interface{} { + return l.source +} + +func (l *jsonGoLoader) JsonReference() (gojsonreference.JsonReference, error) { + return gojsonreference.NewJsonReference("#") +} + +func (l *jsonGoLoader) LoaderFactory() JSONLoaderFactory { + return &DefaultJSONLoaderFactory{} +} + +func NewGoLoader(source interface{}) JSONLoader { + return &jsonGoLoader{source: source} +} + +func (l *jsonGoLoader) LoadJSON() (interface{}, error) { + + // convert it to a compliant JSON first to avoid types "mismatches" + + jsonBytes, err := json.Marshal(l.JsonSource()) + if err != nil { + return nil, err + } + + return decodeJsonUsingNumber(bytes.NewReader(jsonBytes)) + +} + +type jsonIOLoader struct { + buf *bytes.Buffer +} + +func NewReaderLoader(source io.Reader) (JSONLoader, io.Reader) { + buf := &bytes.Buffer{} + return &jsonIOLoader{buf: buf}, io.TeeReader(source, buf) +} + +func NewWriterLoader(source io.Writer) (JSONLoader, io.Writer) { + buf := &bytes.Buffer{} + return &jsonIOLoader{buf: buf}, io.MultiWriter(source, buf) +} + +func (l *jsonIOLoader) JsonSource() interface{} { + return l.buf.String() +} + +func (l *jsonIOLoader) LoadJSON() (interface{}, error) { + return decodeJsonUsingNumber(l.buf) +} + +func (l *jsonIOLoader) JsonReference() (gojsonreference.JsonReference, error) { + return gojsonreference.NewJsonReference("#") +} + +func (l *jsonIOLoader) LoaderFactory() JSONLoaderFactory { + return &DefaultJSONLoaderFactory{} +} + +// JSON raw loader +// In case the JSON is already marshalled to interface{} use this loader +// This is used for testing as otherwise there is no guarantee the JSON is marshalled +// "properly" by using https://golang.org/pkg/encoding/json/#Decoder.UseNumber +type jsonRawLoader struct { + source interface{} +} + +func NewRawLoader(source interface{}) *jsonRawLoader { + return &jsonRawLoader{source: source} +} +func (l *jsonRawLoader) JsonSource() interface{} { + return l.source +} +func (l *jsonRawLoader) LoadJSON() (interface{}, error) { + return l.source, nil +} +func (l *jsonRawLoader) JsonReference() (gojsonreference.JsonReference, error) { + return gojsonreference.NewJsonReference("#") +} +func (l *jsonRawLoader) LoaderFactory() JSONLoaderFactory { + return &DefaultJSONLoaderFactory{} +} + +func decodeJsonUsingNumber(r io.Reader) (interface{}, error) { + + var document interface{} + + decoder := json.NewDecoder(r) + decoder.UseNumber() + + err := decoder.Decode(&document) + if err != nil { + return nil, err + } + + return document, nil + +} diff --git a/tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/locales.go b/tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/locales.go new file mode 100644 index 00000000000..7bd9a9dce59 --- /dev/null +++ b/tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/locales.go @@ -0,0 +1,313 @@ +// Copyright 2015 xeipuuv ( https://github.com/xeipuuv ) +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// author xeipuuv +// author-github https://github.com/xeipuuv +// author-mail xeipuuv@gmail.com +// +// repository-name gojsonschema +// repository-desc An implementation of JSON Schema, based on IETF's draft v4 - Go language. +// +// description Contains const string and messages. +// +// created 01-01-2015 + +package gojsonschema + +type ( + // locale is an interface for defining custom error strings + locale interface { + Required() string + InvalidType() string + NumberAnyOf() string + NumberOneOf() string + NumberAllOf() string + NumberNot() string + MissingDependency() string + Internal() string + Const() string + Enum() string + ArrayNotEnoughItems() string + ArrayNoAdditionalItems() string + ArrayMinItems() string + ArrayMaxItems() string + Unique() string + ArrayContains() string + ArrayMinProperties() string + ArrayMaxProperties() string + AdditionalPropertyNotAllowed() string + InvalidPropertyPattern() string + InvalidPropertyName() string + StringGTE() string + StringLTE() string + DoesNotMatchPattern() string + DoesNotMatchFormat() string + MultipleOf() string + NumberGTE() string + NumberGT() string + NumberLTE() string + NumberLT() string + + // Schema validations + RegexPattern() string + GreaterThanZero() string + MustBeOfA() string + MustBeOfAn() string + CannotBeUsedWithout() string + CannotBeGT() string + MustBeOfType() string + MustBeValidRegex() string + MustBeValidFormat() string + MustBeGTEZero() string + KeyCannotBeGreaterThan() string + KeyItemsMustBeOfType() string + KeyItemsMustBeUnique() string + ReferenceMustBeCanonical() string + NotAValidType() string + Duplicated() string + HttpBadStatus() string + ParseError() string + + ConditionThen() string + ConditionElse() string + + // ErrorFormat + ErrorFormat() string + } + + // DefaultLocale is the default locale for this package + DefaultLocale struct{} +) + +func (l DefaultLocale) Required() string { + return `{{.property}} is required` +} + +func (l DefaultLocale) InvalidType() string { + return `Invalid type. Expected: {{.expected}}, given: {{.given}}` +} + +func (l DefaultLocale) NumberAnyOf() string { + return `Must validate at least one schema (anyOf)` +} + +func (l DefaultLocale) NumberOneOf() string { + return `Must validate one and only one schema (oneOf)` +} + +func (l DefaultLocale) NumberAllOf() string { + return `Must validate all the schemas (allOf)` +} + +func (l DefaultLocale) NumberNot() string { + return `Must not validate the schema (not)` +} + +func (l DefaultLocale) MissingDependency() string { + return `Has a dependency on {{.dependency}}` +} + +func (l DefaultLocale) Internal() string { + return `Internal Error {{.error}}` +} + +func (l DefaultLocale) Const() string { + return `{{.field}} does not match: {{.allowed}}` +} + +func (l DefaultLocale) Enum() string { + return `{{.field}} must be one of the following: {{.allowed}}` +} + +func (l DefaultLocale) ArrayNoAdditionalItems() string { + return `No additional items allowed on array` +} + +func (l DefaultLocale) ArrayNotEnoughItems() string { + return `Not enough items on array to match positional list of schema` +} + +func (l DefaultLocale) ArrayMinItems() string { + return `Array must have at least {{.min}} items` +} + +func (l DefaultLocale) ArrayMaxItems() string { + return `Array must have at most {{.max}} items` +} + +func (l DefaultLocale) Unique() string { + return `{{.type}} items must be unique` +} + +func (l DefaultLocale) ArrayContains() string { + return `At least one of the items must match` +} + +func (l DefaultLocale) ArrayMinProperties() string { + return `Must have at least {{.min}} properties` +} + +func (l DefaultLocale) ArrayMaxProperties() string { + return `Must have at most {{.max}} properties` +} + +func (l DefaultLocale) AdditionalPropertyNotAllowed() string { + return `Additional property {{.property}} is not allowed` +} + +func (l DefaultLocale) InvalidPropertyPattern() string { + return `Property "{{.property}}" does not match pattern {{.pattern}}` +} + +func (l DefaultLocale) InvalidPropertyName() string { + return `Property name of "{{.property}}" does not match` +} + +func (l DefaultLocale) StringGTE() string { + return `String length must be greater than or equal to {{.min}}` +} + +func (l DefaultLocale) StringLTE() string { + return `String length must be less than or equal to {{.max}}` +} + +func (l DefaultLocale) DoesNotMatchPattern() string { + return `Does not match pattern '{{.pattern}}'` +} + +func (l DefaultLocale) DoesNotMatchFormat() string { + return `Does not match format '{{.format}}'` +} + +func (l DefaultLocale) MultipleOf() string { + return `Must be a multiple of {{.multiple}}` +} + +func (l DefaultLocale) NumberGTE() string { + return `Must be greater than or equal to {{.min}}` +} + +func (l DefaultLocale) NumberGT() string { + return `Must be greater than {{.min}}` +} + +func (l DefaultLocale) NumberLTE() string { + return `Must be less than or equal to {{.max}}` +} + +func (l DefaultLocale) NumberLT() string { + return `Must be less than {{.max}}` +} + +// Schema validators +func (l DefaultLocale) RegexPattern() string { + return `Invalid regex pattern '{{.pattern}}'` +} + +func (l DefaultLocale) GreaterThanZero() string { + return `{{.number}} must be strictly greater than 0` +} + +func (l DefaultLocale) MustBeOfA() string { + return `{{.x}} must be of a {{.y}}` +} + +func (l DefaultLocale) MustBeOfAn() string { + return `{{.x}} must be of an {{.y}}` +} + +func (l DefaultLocale) CannotBeUsedWithout() string { + return `{{.x}} cannot be used without {{.y}}` +} + +func (l DefaultLocale) CannotBeGT() string { + return `{{.x}} cannot be greater than {{.y}}` +} + +func (l DefaultLocale) MustBeOfType() string { + return `{{.key}} must be of type {{.type}}` +} + +func (l DefaultLocale) MustBeValidRegex() string { + return `{{.key}} must be a valid regex` +} + +func (l DefaultLocale) MustBeValidFormat() string { + return `{{.key}} must be a valid format {{.given}}` +} + +func (l DefaultLocale) MustBeGTEZero() string { + return `{{.key}} must be greater than or equal to 0` +} + +func (l DefaultLocale) KeyCannotBeGreaterThan() string { + return `{{.key}} cannot be greater than {{.y}}` +} + +func (l DefaultLocale) KeyItemsMustBeOfType() string { + return `{{.key}} items must be {{.type}}` +} + +func (l DefaultLocale) KeyItemsMustBeUnique() string { + return `{{.key}} items must be unique` +} + +func (l DefaultLocale) ReferenceMustBeCanonical() string { + return `Reference {{.reference}} must be canonical` +} + +func (l DefaultLocale) NotAValidType() string { + return `has a primitive type that is NOT VALID -- given: {{.given}} Expected valid values are:{{.expected}}` +} + +func (l DefaultLocale) Duplicated() string { + return `{{.type}} type is duplicated` +} + +func (l DefaultLocale) HttpBadStatus() string { + return `Could not read schema from HTTP, response status is {{.status}}` +} + +// Replacement options: field, description, context, value +func (l DefaultLocale) ErrorFormat() string { + return `{{.field}}: {{.description}}` +} + +//Parse error +func (l DefaultLocale) ParseError() string { + return `Expected: {{.expected}}, given: Invalid JSON` +} + +//If/Else +func (l DefaultLocale) ConditionThen() string { + return `Must validate "then" as "if" was valid` +} + +func (l DefaultLocale) ConditionElse() string { + return `Must validate "else" as "if" was not valid` +} + +const ( + STRING_NUMBER = "number" + STRING_ARRAY_OF_STRINGS = "array of strings" + STRING_ARRAY_OF_SCHEMAS = "array of schemas" + STRING_SCHEMA = "valid schema" + STRING_SCHEMA_OR_ARRAY_OF_STRINGS = "schema or array of strings" + STRING_PROPERTIES = "properties" + STRING_DEPENDENCY = "dependency" + STRING_PROPERTY = "property" + STRING_UNDEFINED = "undefined" + STRING_CONTEXT_ROOT = "(root)" + STRING_ROOT_SCHEMA_PROPERTY = "(root)" +) diff --git a/tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/result.go b/tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/result.go new file mode 100644 index 00000000000..7896f312990 --- /dev/null +++ b/tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/result.go @@ -0,0 +1,195 @@ +// Copyright 2015 xeipuuv ( https://github.com/xeipuuv ) +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// author xeipuuv +// author-github https://github.com/xeipuuv +// author-mail xeipuuv@gmail.com +// +// repository-name gojsonschema +// repository-desc An implementation of JSON Schema, based on IETF's draft v4 - Go language. +// +// description Result and ResultError implementations. +// +// created 01-01-2015 + +package gojsonschema + +import ( + "fmt" + "strings" +) + +type ( + // ErrorDetails is a map of details specific to each error. + // While the values will vary, every error will contain a "field" value + ErrorDetails map[string]interface{} + + // ResultError is the interface that library errors must implement + ResultError interface { + Field() string + SetType(string) + Type() string + SetContext(*JsonContext) + Context() *JsonContext + SetDescription(string) + Description() string + SetDescriptionFormat(string) + DescriptionFormat() string + SetValue(interface{}) + Value() interface{} + SetDetails(ErrorDetails) + Details() ErrorDetails + String() string + } + + // ResultErrorFields holds the fields for each ResultError implementation. + // ResultErrorFields implements the ResultError interface, so custom errors + // can be defined by just embedding this type + ResultErrorFields struct { + errorType string // A string with the type of error (i.e. invalid_type) + context *JsonContext // Tree like notation of the part that failed the validation. ex (root).a.b ... + description string // A human readable error message + descriptionFormat string // A format for human readable error message + value interface{} // Value given by the JSON file that is the source of the error + details ErrorDetails + } + + Result struct { + errors []ResultError + // Scores how well the validation matched. Useful in generating + // better error messages for anyOf and oneOf. + score int + } +) + +// Field outputs the field name without the root context +// i.e. firstName or person.firstName instead of (root).firstName or (root).person.firstName +func (v *ResultErrorFields) Field() string { + if p, ok := v.Details()["property"]; ok { + if str, isString := p.(string); isString { + return str + } + } + + return strings.TrimPrefix(v.context.String(), STRING_ROOT_SCHEMA_PROPERTY+".") +} + +func (v *ResultErrorFields) SetType(errorType string) { + v.errorType = errorType +} + +func (v *ResultErrorFields) Type() string { + return v.errorType +} + +func (v *ResultErrorFields) SetContext(context *JsonContext) { + v.context = context +} + +func (v *ResultErrorFields) Context() *JsonContext { + return v.context +} + +func (v *ResultErrorFields) SetDescription(description string) { + v.description = description +} + +func (v *ResultErrorFields) Description() string { + return v.description +} + +func (v *ResultErrorFields) SetDescriptionFormat(descriptionFormat string) { + v.descriptionFormat = descriptionFormat +} + +func (v *ResultErrorFields) DescriptionFormat() string { + return v.descriptionFormat +} + +func (v *ResultErrorFields) SetValue(value interface{}) { + v.value = value +} + +func (v *ResultErrorFields) Value() interface{} { + return v.value +} + +func (v *ResultErrorFields) SetDetails(details ErrorDetails) { + v.details = details +} + +func (v *ResultErrorFields) Details() ErrorDetails { + return v.details +} + +func (v ResultErrorFields) String() string { + // as a fallback, the value is displayed go style + valueString := fmt.Sprintf("%v", v.value) + + // marshal the go value value to json + if v.value == nil { + valueString = TYPE_NULL + } else { + if vs, err := marshalToJsonString(v.value); err == nil { + if vs == nil { + valueString = TYPE_NULL + } else { + valueString = *vs + } + } + } + + return formatErrorDescription(Locale.ErrorFormat(), ErrorDetails{ + "context": v.context.String(), + "description": v.description, + "value": valueString, + "field": v.Field(), + }) +} + +func (v *Result) Valid() bool { + return len(v.errors) == 0 +} + +func (v *Result) Errors() []ResultError { + return v.errors +} + +// Add a fully filled error to the error set +// SetDescription() will be called with the result of the parsed err.DescriptionFormat() +func (v *Result) AddError(err ResultError, details ErrorDetails) { + if _, exists := details["context"]; !exists && err.Context() != nil { + details["context"] = err.Context().String() + } + + err.SetDescription(formatErrorDescription(err.DescriptionFormat(), details)) + + v.errors = append(v.errors, err) +} + +func (v *Result) addInternalError(err ResultError, context *JsonContext, value interface{}, details ErrorDetails) { + newError(err, context, value, Locale, details) + v.errors = append(v.errors, err) + v.score -= 2 // results in a net -1 when added to the +1 we get at the end of the validation function +} + +// Used to copy errors from a sub-schema to the main one +func (v *Result) mergeErrors(otherResult *Result) { + v.errors = append(v.errors, otherResult.Errors()...) + v.score += otherResult.score +} + +func (v *Result) incrementScore() { + v.score++ +} diff --git a/tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/schema.go b/tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/schema.go new file mode 100644 index 00000000000..4ae3c62047a --- /dev/null +++ b/tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/schema.go @@ -0,0 +1,1000 @@ +// Copyright 2015 xeipuuv ( https://github.com/xeipuuv ) +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// author xeipuuv +// author-github https://github.com/xeipuuv +// author-mail xeipuuv@gmail.com +// +// repository-name gojsonschema +// repository-desc An implementation of JSON Schema, based on IETF's draft v4 - Go language. +// +// description Defines Schema, the main entry to every subSchema. +// Contains the parsing logic and error checking. +// +// created 26-02-2013 + +package gojsonschema + +import ( + "errors" + "math/big" + "reflect" + "regexp" + "text/template" + + "github.com/xeipuuv/gojsonreference" +) + +var ( + // Locale is the default locale to use + // Library users can overwrite with their own implementation + Locale locale = DefaultLocale{} + + // ErrorTemplateFuncs allows you to define custom template funcs for use in localization. + ErrorTemplateFuncs template.FuncMap +) + +func NewSchema(l JSONLoader) (*Schema, error) { + return NewSchemaLoader().Compile(l) +} + +type Schema struct { + documentReference gojsonreference.JsonReference + rootSchema *subSchema + pool *schemaPool + referencePool *schemaReferencePool +} + +func (d *Schema) parse(document interface{}) error { + d.rootSchema = &subSchema{property: STRING_ROOT_SCHEMA_PROPERTY} + return d.parseSchema(document, d.rootSchema) +} + +func (d *Schema) SetRootSchemaName(name string) { + d.rootSchema.property = name +} + +// Parses a subSchema +// +// Pretty long function ( sorry :) )... but pretty straight forward, repetitive and boring +// Not much magic involved here, most of the job is to validate the key names and their values, +// then the values are copied into subSchema struct +// +func (d *Schema) parseSchema(documentNode interface{}, currentSchema *subSchema) error { + + // As of draft 6 "true" is equivalent to an empty schema "{}" and false equals "{"not":{}}" + if isKind(documentNode, reflect.Bool) { + b := documentNode.(bool) + if b { + documentNode = map[string]interface{}{} + } else { + documentNode = map[string]interface{}{"not": true} + } + } + + if !isKind(documentNode, reflect.Map) { + return errors.New(formatErrorDescription( + Locale.ParseError(), + ErrorDetails{ + "expected": STRING_SCHEMA, + }, + )) + } + + m := documentNode.(map[string]interface{}) + + if currentSchema.parent == nil { + currentSchema.ref = &d.documentReference + currentSchema.id = &d.documentReference + + if existsMapKey(m, KEY_SCHEMA) && false { + if !isKind(m[KEY_SCHEMA], reflect.String) { + return errors.New(formatErrorDescription( + Locale.InvalidType(), + ErrorDetails{ + "expected": TYPE_STRING, + "given": KEY_SCHEMA, + }, + )) + } + } + } + + if currentSchema.id == nil && currentSchema.parent != nil { + currentSchema.id = currentSchema.parent.id + } + + // In draft 6 the id keyword was renamed to $id + // Use the old id by default + keyID := KEY_ID_NEW + if existsMapKey(m, KEY_ID) { + keyID = KEY_ID + } + if existsMapKey(m, keyID) && !isKind(m[keyID], reflect.String) { + return errors.New(formatErrorDescription( + Locale.InvalidType(), + ErrorDetails{ + "expected": TYPE_STRING, + "given": keyID, + }, + )) + } + if k, ok := m[keyID].(string); ok { + jsonReference, err := gojsonreference.NewJsonReference(k) + if err != nil { + return err + } + if currentSchema == d.rootSchema { + currentSchema.id = &jsonReference + } else { + ref, err := currentSchema.parent.id.Inherits(jsonReference) + if err != nil { + return err + } + currentSchema.id = ref + } + } + + // definitions + if existsMapKey(m, KEY_DEFINITIONS) { + if isKind(m[KEY_DEFINITIONS], reflect.Map, reflect.Bool) { + for _, dv := range m[KEY_DEFINITIONS].(map[string]interface{}) { + if isKind(dv, reflect.Map, reflect.Bool) { + + newSchema := &subSchema{property: KEY_DEFINITIONS, parent: currentSchema} + + err := d.parseSchema(dv, newSchema) + + if err != nil { + return err + } + } else { + return errors.New(formatErrorDescription( + Locale.InvalidType(), + ErrorDetails{ + "expected": STRING_ARRAY_OF_SCHEMAS, + "given": KEY_DEFINITIONS, + }, + )) + } + } + } else { + return errors.New(formatErrorDescription( + Locale.InvalidType(), + ErrorDetails{ + "expected": STRING_ARRAY_OF_SCHEMAS, + "given": KEY_DEFINITIONS, + }, + )) + } + + } + + // title + if existsMapKey(m, KEY_TITLE) && !isKind(m[KEY_TITLE], reflect.String) { + return errors.New(formatErrorDescription( + Locale.InvalidType(), + ErrorDetails{ + "expected": TYPE_STRING, + "given": KEY_TITLE, + }, + )) + } + if k, ok := m[KEY_TITLE].(string); ok { + currentSchema.title = &k + } + + // description + if existsMapKey(m, KEY_DESCRIPTION) && !isKind(m[KEY_DESCRIPTION], reflect.String) { + return errors.New(formatErrorDescription( + Locale.InvalidType(), + ErrorDetails{ + "expected": TYPE_STRING, + "given": KEY_DESCRIPTION, + }, + )) + } + if k, ok := m[KEY_DESCRIPTION].(string); ok { + currentSchema.description = &k + } + + // $ref + if existsMapKey(m, KEY_REF) && !isKind(m[KEY_REF], reflect.String) { + return errors.New(formatErrorDescription( + Locale.InvalidType(), + ErrorDetails{ + "expected": TYPE_STRING, + "given": KEY_REF, + }, + )) + } + + if k, ok := m[KEY_REF].(string); ok { + + jsonReference, err := gojsonreference.NewJsonReference(k) + if err != nil { + return err + } + + currentSchema.ref = &jsonReference + + if sch, ok := d.referencePool.Get(currentSchema.ref.String()); ok { + currentSchema.refSchema = sch + } else { + err := d.parseReference(documentNode, currentSchema) + + if err != nil { + return err + } + + return nil + } + } + + // type + if existsMapKey(m, KEY_TYPE) { + if isKind(m[KEY_TYPE], reflect.String) { + if k, ok := m[KEY_TYPE].(string); ok { + err := currentSchema.types.Add(k) + if err != nil { + return err + } + } + } else { + if isKind(m[KEY_TYPE], reflect.Slice) { + arrayOfTypes := m[KEY_TYPE].([]interface{}) + for _, typeInArray := range arrayOfTypes { + if reflect.ValueOf(typeInArray).Kind() != reflect.String { + return errors.New(formatErrorDescription( + Locale.InvalidType(), + ErrorDetails{ + "expected": TYPE_STRING + "/" + STRING_ARRAY_OF_STRINGS, + "given": KEY_TYPE, + }, + )) + } else { + currentSchema.types.Add(typeInArray.(string)) + } + } + + } else { + return errors.New(formatErrorDescription( + Locale.InvalidType(), + ErrorDetails{ + "expected": TYPE_STRING + "/" + STRING_ARRAY_OF_STRINGS, + "given": KEY_TYPE, + }, + )) + } + } + } + + // properties + if existsMapKey(m, KEY_PROPERTIES) { + err := d.parseProperties(m[KEY_PROPERTIES], currentSchema) + if err != nil { + return err + } + } + + // additionalProperties + if existsMapKey(m, KEY_ADDITIONAL_PROPERTIES) { + if isKind(m[KEY_ADDITIONAL_PROPERTIES], reflect.Bool) { + currentSchema.additionalProperties = m[KEY_ADDITIONAL_PROPERTIES].(bool) + } else if isKind(m[KEY_ADDITIONAL_PROPERTIES], reflect.Map) { + newSchema := &subSchema{property: KEY_ADDITIONAL_PROPERTIES, parent: currentSchema, ref: currentSchema.ref} + currentSchema.additionalProperties = newSchema + err := d.parseSchema(m[KEY_ADDITIONAL_PROPERTIES], newSchema) + if err != nil { + return errors.New(err.Error()) + } + } else { + return errors.New(formatErrorDescription( + Locale.InvalidType(), + ErrorDetails{ + "expected": TYPE_BOOLEAN + "/" + STRING_SCHEMA, + "given": KEY_ADDITIONAL_PROPERTIES, + }, + )) + } + } + + // patternProperties + if existsMapKey(m, KEY_PATTERN_PROPERTIES) { + if isKind(m[KEY_PATTERN_PROPERTIES], reflect.Map) { + patternPropertiesMap := m[KEY_PATTERN_PROPERTIES].(map[string]interface{}) + if len(patternPropertiesMap) > 0 { + currentSchema.patternProperties = make(map[string]*subSchema) + for k, v := range patternPropertiesMap { + _, err := regexp.MatchString(k, "") + if err != nil { + return errors.New(formatErrorDescription( + Locale.RegexPattern(), + ErrorDetails{"pattern": k}, + )) + } + newSchema := &subSchema{property: k, parent: currentSchema, ref: currentSchema.ref} + err = d.parseSchema(v, newSchema) + if err != nil { + return errors.New(err.Error()) + } + currentSchema.patternProperties[k] = newSchema + } + } + } else { + return errors.New(formatErrorDescription( + Locale.InvalidType(), + ErrorDetails{ + "expected": STRING_SCHEMA, + "given": KEY_PATTERN_PROPERTIES, + }, + )) + } + } + + // propertyNames + if existsMapKey(m, KEY_PROPERTY_NAMES) { + if isKind(m[KEY_PROPERTY_NAMES], reflect.Map, reflect.Bool) { + newSchema := &subSchema{property: KEY_PROPERTY_NAMES, parent: currentSchema, ref: currentSchema.ref} + currentSchema.propertyNames = newSchema + err := d.parseSchema(m[KEY_PROPERTY_NAMES], newSchema) + if err != nil { + return err + } + } else { + return errors.New(formatErrorDescription( + Locale.InvalidType(), + ErrorDetails{ + "expected": STRING_SCHEMA, + "given": KEY_PATTERN_PROPERTIES, + }, + )) + } + } + + // dependencies + if existsMapKey(m, KEY_DEPENDENCIES) { + err := d.parseDependencies(m[KEY_DEPENDENCIES], currentSchema) + if err != nil { + return err + } + } + + // items + if existsMapKey(m, KEY_ITEMS) { + if isKind(m[KEY_ITEMS], reflect.Slice) { + for _, itemElement := range m[KEY_ITEMS].([]interface{}) { + if isKind(itemElement, reflect.Map, reflect.Bool) { + newSchema := &subSchema{parent: currentSchema, property: KEY_ITEMS} + newSchema.ref = currentSchema.ref + currentSchema.AddItemsChild(newSchema) + err := d.parseSchema(itemElement, newSchema) + if err != nil { + return err + } + } else { + return errors.New(formatErrorDescription( + Locale.InvalidType(), + ErrorDetails{ + "expected": STRING_SCHEMA + "/" + STRING_ARRAY_OF_SCHEMAS, + "given": KEY_ITEMS, + }, + )) + } + currentSchema.itemsChildrenIsSingleSchema = false + } + } else if isKind(m[KEY_ITEMS], reflect.Map, reflect.Bool) { + newSchema := &subSchema{parent: currentSchema, property: KEY_ITEMS} + newSchema.ref = currentSchema.ref + currentSchema.AddItemsChild(newSchema) + err := d.parseSchema(m[KEY_ITEMS], newSchema) + if err != nil { + return err + } + currentSchema.itemsChildrenIsSingleSchema = true + } else { + return errors.New(formatErrorDescription( + Locale.InvalidType(), + ErrorDetails{ + "expected": STRING_SCHEMA + "/" + STRING_ARRAY_OF_SCHEMAS, + "given": KEY_ITEMS, + }, + )) + } + } + + // additionalItems + if existsMapKey(m, KEY_ADDITIONAL_ITEMS) { + if isKind(m[KEY_ADDITIONAL_ITEMS], reflect.Bool) { + currentSchema.additionalItems = m[KEY_ADDITIONAL_ITEMS].(bool) + } else if isKind(m[KEY_ADDITIONAL_ITEMS], reflect.Map) { + newSchema := &subSchema{property: KEY_ADDITIONAL_ITEMS, parent: currentSchema, ref: currentSchema.ref} + currentSchema.additionalItems = newSchema + err := d.parseSchema(m[KEY_ADDITIONAL_ITEMS], newSchema) + if err != nil { + return errors.New(err.Error()) + } + } else { + return errors.New(formatErrorDescription( + Locale.InvalidType(), + ErrorDetails{ + "expected": TYPE_BOOLEAN + "/" + STRING_SCHEMA, + "given": KEY_ADDITIONAL_ITEMS, + }, + )) + } + } + + // validation : number / integer + + if existsMapKey(m, KEY_MULTIPLE_OF) { + multipleOfValue := mustBeNumber(m[KEY_MULTIPLE_OF]) + if multipleOfValue == nil { + return errors.New(formatErrorDescription( + Locale.InvalidType(), + ErrorDetails{ + "expected": STRING_NUMBER, + "given": KEY_MULTIPLE_OF, + }, + )) + } + if multipleOfValue.Cmp(big.NewFloat(0)) <= 0 { + return errors.New(formatErrorDescription( + Locale.GreaterThanZero(), + ErrorDetails{"number": KEY_MULTIPLE_OF}, + )) + } + currentSchema.multipleOf = multipleOfValue + } + + if existsMapKey(m, KEY_MINIMUM) { + minimumValue := mustBeNumber(m[KEY_MINIMUM]) + if minimumValue == nil { + return errors.New(formatErrorDescription( + Locale.MustBeOfA(), + ErrorDetails{"x": KEY_MINIMUM, "y": STRING_NUMBER}, + )) + } + currentSchema.minimum = minimumValue + } + + if existsMapKey(m, KEY_EXCLUSIVE_MINIMUM) { + if isKind(m[KEY_EXCLUSIVE_MINIMUM], reflect.Bool) { + if currentSchema.minimum == nil { + return errors.New(formatErrorDescription( + Locale.CannotBeUsedWithout(), + ErrorDetails{"x": KEY_EXCLUSIVE_MINIMUM, "y": KEY_MINIMUM}, + )) + } + exclusiveMinimumValue := m[KEY_EXCLUSIVE_MINIMUM].(bool) + currentSchema.exclusiveMinimum = exclusiveMinimumValue + } else if isJsonNumber(m[KEY_EXCLUSIVE_MINIMUM]) { + minimumValue := mustBeNumber(m[KEY_EXCLUSIVE_MINIMUM]) + + currentSchema.minimum = minimumValue + currentSchema.exclusiveMinimum = true + } else { + return errors.New(formatErrorDescription( + Locale.InvalidType(), + ErrorDetails{"expected": TYPE_BOOLEAN + ", " + TYPE_NUMBER, "given": KEY_EXCLUSIVE_MINIMUM}, + )) + } + } + + if existsMapKey(m, KEY_MAXIMUM) { + maximumValue := mustBeNumber(m[KEY_MAXIMUM]) + if maximumValue == nil { + return errors.New(formatErrorDescription( + Locale.MustBeOfA(), + ErrorDetails{"x": KEY_MAXIMUM, "y": STRING_NUMBER}, + )) + } + currentSchema.maximum = maximumValue + } + + if existsMapKey(m, KEY_EXCLUSIVE_MAXIMUM) { + if isKind(m[KEY_EXCLUSIVE_MAXIMUM], reflect.Bool) { + if currentSchema.maximum == nil { + return errors.New(formatErrorDescription( + Locale.CannotBeUsedWithout(), + ErrorDetails{"x": KEY_EXCLUSIVE_MAXIMUM, "y": KEY_MAXIMUM}, + )) + } + exclusiveMaximumValue := m[KEY_EXCLUSIVE_MAXIMUM].(bool) + currentSchema.exclusiveMaximum = exclusiveMaximumValue + } else if isJsonNumber(m[KEY_EXCLUSIVE_MAXIMUM]) { + maximumValue := mustBeNumber(m[KEY_EXCLUSIVE_MAXIMUM]) + + currentSchema.maximum = maximumValue + currentSchema.exclusiveMaximum = true + } else { + return errors.New(formatErrorDescription( + Locale.InvalidType(), + ErrorDetails{"expected": TYPE_BOOLEAN + ", " + TYPE_NUMBER, "given": KEY_EXCLUSIVE_MAXIMUM}, + )) + } + } + + if currentSchema.minimum != nil && currentSchema.maximum != nil { + if currentSchema.minimum.Cmp(currentSchema.maximum) == 1 { + return errors.New(formatErrorDescription( + Locale.CannotBeGT(), + ErrorDetails{"x": KEY_MINIMUM, "y": KEY_MAXIMUM}, + )) + } + } + + // validation : string + + if existsMapKey(m, KEY_MIN_LENGTH) { + minLengthIntegerValue := mustBeInteger(m[KEY_MIN_LENGTH]) + if minLengthIntegerValue == nil { + return errors.New(formatErrorDescription( + Locale.MustBeOfAn(), + ErrorDetails{"x": KEY_MIN_LENGTH, "y": TYPE_INTEGER}, + )) + } + if *minLengthIntegerValue < 0 { + return errors.New(formatErrorDescription( + Locale.MustBeGTEZero(), + ErrorDetails{"key": KEY_MIN_LENGTH}, + )) + } + currentSchema.minLength = minLengthIntegerValue + } + + if existsMapKey(m, KEY_MAX_LENGTH) { + maxLengthIntegerValue := mustBeInteger(m[KEY_MAX_LENGTH]) + if maxLengthIntegerValue == nil { + return errors.New(formatErrorDescription( + Locale.MustBeOfAn(), + ErrorDetails{"x": KEY_MAX_LENGTH, "y": TYPE_INTEGER}, + )) + } + if *maxLengthIntegerValue < 0 { + return errors.New(formatErrorDescription( + Locale.MustBeGTEZero(), + ErrorDetails{"key": KEY_MAX_LENGTH}, + )) + } + currentSchema.maxLength = maxLengthIntegerValue + } + + if currentSchema.minLength != nil && currentSchema.maxLength != nil { + if *currentSchema.minLength > *currentSchema.maxLength { + return errors.New(formatErrorDescription( + Locale.CannotBeGT(), + ErrorDetails{"x": KEY_MIN_LENGTH, "y": KEY_MAX_LENGTH}, + )) + } + } + + if existsMapKey(m, KEY_PATTERN) { + if isKind(m[KEY_PATTERN], reflect.String) { + regexpObject, err := regexp.Compile(m[KEY_PATTERN].(string)) + if err != nil { + return errors.New(formatErrorDescription( + Locale.MustBeValidRegex(), + ErrorDetails{"key": KEY_PATTERN}, + )) + } + currentSchema.pattern = regexpObject + } else { + return errors.New(formatErrorDescription( + Locale.MustBeOfA(), + ErrorDetails{"x": KEY_PATTERN, "y": TYPE_STRING}, + )) + } + } + + if existsMapKey(m, KEY_FORMAT) { + formatString, ok := m[KEY_FORMAT].(string) + if ok && FormatCheckers.Has(formatString) { + currentSchema.format = formatString + } + } + + // validation : object + + if existsMapKey(m, KEY_MIN_PROPERTIES) { + minPropertiesIntegerValue := mustBeInteger(m[KEY_MIN_PROPERTIES]) + if minPropertiesIntegerValue == nil { + return errors.New(formatErrorDescription( + Locale.MustBeOfAn(), + ErrorDetails{"x": KEY_MIN_PROPERTIES, "y": TYPE_INTEGER}, + )) + } + if *minPropertiesIntegerValue < 0 { + return errors.New(formatErrorDescription( + Locale.MustBeGTEZero(), + ErrorDetails{"key": KEY_MIN_PROPERTIES}, + )) + } + currentSchema.minProperties = minPropertiesIntegerValue + } + + if existsMapKey(m, KEY_MAX_PROPERTIES) { + maxPropertiesIntegerValue := mustBeInteger(m[KEY_MAX_PROPERTIES]) + if maxPropertiesIntegerValue == nil { + return errors.New(formatErrorDescription( + Locale.MustBeOfAn(), + ErrorDetails{"x": KEY_MAX_PROPERTIES, "y": TYPE_INTEGER}, + )) + } + if *maxPropertiesIntegerValue < 0 { + return errors.New(formatErrorDescription( + Locale.MustBeGTEZero(), + ErrorDetails{"key": KEY_MAX_PROPERTIES}, + )) + } + currentSchema.maxProperties = maxPropertiesIntegerValue + } + + if currentSchema.minProperties != nil && currentSchema.maxProperties != nil { + if *currentSchema.minProperties > *currentSchema.maxProperties { + return errors.New(formatErrorDescription( + Locale.KeyCannotBeGreaterThan(), + ErrorDetails{"key": KEY_MIN_PROPERTIES, "y": KEY_MAX_PROPERTIES}, + )) + } + } + + if existsMapKey(m, KEY_REQUIRED) { + if isKind(m[KEY_REQUIRED], reflect.Slice) { + requiredValues := m[KEY_REQUIRED].([]interface{}) + for _, requiredValue := range requiredValues { + if isKind(requiredValue, reflect.String) { + err := currentSchema.AddRequired(requiredValue.(string)) + if err != nil { + return err + } + } else { + return errors.New(formatErrorDescription( + Locale.KeyItemsMustBeOfType(), + ErrorDetails{"key": KEY_REQUIRED, "type": TYPE_STRING}, + )) + } + } + } else { + return errors.New(formatErrorDescription( + Locale.MustBeOfAn(), + ErrorDetails{"x": KEY_REQUIRED, "y": TYPE_ARRAY}, + )) + } + } + + // validation : array + + if existsMapKey(m, KEY_MIN_ITEMS) { + minItemsIntegerValue := mustBeInteger(m[KEY_MIN_ITEMS]) + if minItemsIntegerValue == nil { + return errors.New(formatErrorDescription( + Locale.MustBeOfAn(), + ErrorDetails{"x": KEY_MIN_ITEMS, "y": TYPE_INTEGER}, + )) + } + if *minItemsIntegerValue < 0 { + return errors.New(formatErrorDescription( + Locale.MustBeGTEZero(), + ErrorDetails{"key": KEY_MIN_ITEMS}, + )) + } + currentSchema.minItems = minItemsIntegerValue + } + + if existsMapKey(m, KEY_MAX_ITEMS) { + maxItemsIntegerValue := mustBeInteger(m[KEY_MAX_ITEMS]) + if maxItemsIntegerValue == nil { + return errors.New(formatErrorDescription( + Locale.MustBeOfAn(), + ErrorDetails{"x": KEY_MAX_ITEMS, "y": TYPE_INTEGER}, + )) + } + if *maxItemsIntegerValue < 0 { + return errors.New(formatErrorDescription( + Locale.MustBeGTEZero(), + ErrorDetails{"key": KEY_MAX_ITEMS}, + )) + } + currentSchema.maxItems = maxItemsIntegerValue + } + + if existsMapKey(m, KEY_UNIQUE_ITEMS) { + if isKind(m[KEY_UNIQUE_ITEMS], reflect.Bool) { + currentSchema.uniqueItems = m[KEY_UNIQUE_ITEMS].(bool) + } else { + return errors.New(formatErrorDescription( + Locale.MustBeOfA(), + ErrorDetails{"x": KEY_UNIQUE_ITEMS, "y": TYPE_BOOLEAN}, + )) + } + } + + if existsMapKey(m, KEY_CONTAINS) { + newSchema := &subSchema{property: KEY_CONTAINS, parent: currentSchema, ref: currentSchema.ref} + currentSchema.contains = newSchema + err := d.parseSchema(m[KEY_CONTAINS], newSchema) + if err != nil { + return err + } + } + + // validation : all + + if existsMapKey(m, KEY_CONST) { + err := currentSchema.AddConst(m[KEY_CONST]) + if err != nil { + return err + } + } + + if existsMapKey(m, KEY_ENUM) { + if isKind(m[KEY_ENUM], reflect.Slice) { + for _, v := range m[KEY_ENUM].([]interface{}) { + err := currentSchema.AddEnum(v) + if err != nil { + return err + } + } + } else { + return errors.New(formatErrorDescription( + Locale.MustBeOfAn(), + ErrorDetails{"x": KEY_ENUM, "y": TYPE_ARRAY}, + )) + } + } + + // validation : subSchema + + if existsMapKey(m, KEY_ONE_OF) { + if isKind(m[KEY_ONE_OF], reflect.Slice) { + for _, v := range m[KEY_ONE_OF].([]interface{}) { + newSchema := &subSchema{property: KEY_ONE_OF, parent: currentSchema, ref: currentSchema.ref} + currentSchema.AddOneOf(newSchema) + err := d.parseSchema(v, newSchema) + if err != nil { + return err + } + } + } else { + return errors.New(formatErrorDescription( + Locale.MustBeOfAn(), + ErrorDetails{"x": KEY_ONE_OF, "y": TYPE_ARRAY}, + )) + } + } + + if existsMapKey(m, KEY_ANY_OF) { + if isKind(m[KEY_ANY_OF], reflect.Slice) { + for _, v := range m[KEY_ANY_OF].([]interface{}) { + newSchema := &subSchema{property: KEY_ANY_OF, parent: currentSchema, ref: currentSchema.ref} + currentSchema.AddAnyOf(newSchema) + err := d.parseSchema(v, newSchema) + if err != nil { + return err + } + } + } else { + return errors.New(formatErrorDescription( + Locale.MustBeOfAn(), + ErrorDetails{"x": KEY_ANY_OF, "y": TYPE_ARRAY}, + )) + } + } + + if existsMapKey(m, KEY_ALL_OF) { + if isKind(m[KEY_ALL_OF], reflect.Slice) { + for _, v := range m[KEY_ALL_OF].([]interface{}) { + newSchema := &subSchema{property: KEY_ALL_OF, parent: currentSchema, ref: currentSchema.ref} + currentSchema.AddAllOf(newSchema) + err := d.parseSchema(v, newSchema) + if err != nil { + return err + } + } + } else { + return errors.New(formatErrorDescription( + Locale.MustBeOfAn(), + ErrorDetails{"x": KEY_ANY_OF, "y": TYPE_ARRAY}, + )) + } + } + + if existsMapKey(m, KEY_NOT) { + if isKind(m[KEY_NOT], reflect.Map, reflect.Bool) { + newSchema := &subSchema{property: KEY_NOT, parent: currentSchema, ref: currentSchema.ref} + currentSchema.SetNot(newSchema) + err := d.parseSchema(m[KEY_NOT], newSchema) + if err != nil { + return err + } + } else { + return errors.New(formatErrorDescription( + Locale.MustBeOfAn(), + ErrorDetails{"x": KEY_NOT, "y": TYPE_OBJECT}, + )) + } + } + + if existsMapKey(m, KEY_IF) { + if isKind(m[KEY_IF], reflect.Map, reflect.Bool) { + newSchema := &subSchema{property: KEY_IF, parent: currentSchema, ref: currentSchema.ref} + currentSchema.SetIf(newSchema) + err := d.parseSchema(m[KEY_IF], newSchema) + if err != nil { + return err + } + } else { + return errors.New(formatErrorDescription( + Locale.MustBeOfAn(), + ErrorDetails{"x": KEY_IF, "y": TYPE_OBJECT}, + )) + } + } + + if existsMapKey(m, KEY_THEN) { + if isKind(m[KEY_THEN], reflect.Map, reflect.Bool) { + newSchema := &subSchema{property: KEY_THEN, parent: currentSchema, ref: currentSchema.ref} + currentSchema.SetThen(newSchema) + err := d.parseSchema(m[KEY_THEN], newSchema) + if err != nil { + return err + } + } else { + return errors.New(formatErrorDescription( + Locale.MustBeOfAn(), + ErrorDetails{"x": KEY_THEN, "y": TYPE_OBJECT}, + )) + } + } + + if existsMapKey(m, KEY_ELSE) { + if isKind(m[KEY_ELSE], reflect.Map, reflect.Bool) { + newSchema := &subSchema{property: KEY_ELSE, parent: currentSchema, ref: currentSchema.ref} + currentSchema.SetElse(newSchema) + err := d.parseSchema(m[KEY_ELSE], newSchema) + if err != nil { + return err + } + } else { + return errors.New(formatErrorDescription( + Locale.MustBeOfAn(), + ErrorDetails{"x": KEY_ELSE, "y": TYPE_OBJECT}, + )) + } + } + + return nil +} + +func (d *Schema) parseReference(documentNode interface{}, currentSchema *subSchema) error { + var ( + refdDocumentNode interface{} + dsp *schemaPoolDocument + err error + ) + + newSchema := &subSchema{property: KEY_REF, parent: currentSchema, ref: currentSchema.ref} + + d.referencePool.Add(currentSchema.ref.String(), newSchema) + + dsp, err = d.pool.GetDocument(*currentSchema.ref) + if err != nil { + return err + } + newSchema.id = currentSchema.ref + + refdDocumentNode = dsp.Document + + if err != nil { + return err + } + + if !isKind(refdDocumentNode, reflect.Map, reflect.Bool) { + return errors.New(formatErrorDescription( + Locale.MustBeOfType(), + ErrorDetails{"key": STRING_SCHEMA, "type": TYPE_OBJECT}, + )) + } + + err = d.parseSchema(refdDocumentNode, newSchema) + if err != nil { + return err + } + + currentSchema.refSchema = newSchema + + return nil + +} + +func (d *Schema) parseProperties(documentNode interface{}, currentSchema *subSchema) error { + + if !isKind(documentNode, reflect.Map) { + return errors.New(formatErrorDescription( + Locale.MustBeOfType(), + ErrorDetails{"key": STRING_PROPERTIES, "type": TYPE_OBJECT}, + )) + } + + m := documentNode.(map[string]interface{}) + for k := range m { + schemaProperty := k + newSchema := &subSchema{property: schemaProperty, parent: currentSchema, ref: currentSchema.ref} + currentSchema.AddPropertiesChild(newSchema) + err := d.parseSchema(m[k], newSchema) + if err != nil { + return err + } + } + + return nil +} + +func (d *Schema) parseDependencies(documentNode interface{}, currentSchema *subSchema) error { + + if !isKind(documentNode, reflect.Map) { + return errors.New(formatErrorDescription( + Locale.MustBeOfType(), + ErrorDetails{"key": KEY_DEPENDENCIES, "type": TYPE_OBJECT}, + )) + } + + m := documentNode.(map[string]interface{}) + currentSchema.dependencies = make(map[string]interface{}) + + for k := range m { + switch reflect.ValueOf(m[k]).Kind() { + + case reflect.Slice: + values := m[k].([]interface{}) + var valuesToRegister []string + + for _, value := range values { + if !isKind(value, reflect.String) { + return errors.New(formatErrorDescription( + Locale.MustBeOfType(), + ErrorDetails{ + "key": STRING_DEPENDENCY, + "type": STRING_SCHEMA_OR_ARRAY_OF_STRINGS, + }, + )) + } else { + valuesToRegister = append(valuesToRegister, value.(string)) + } + currentSchema.dependencies[k] = valuesToRegister + } + + case reflect.Map, reflect.Bool: + depSchema := &subSchema{property: k, parent: currentSchema, ref: currentSchema.ref} + err := d.parseSchema(m[k], depSchema) + if err != nil { + return err + } + currentSchema.dependencies[k] = depSchema + + default: + return errors.New(formatErrorDescription( + Locale.MustBeOfType(), + ErrorDetails{ + "key": STRING_DEPENDENCY, + "type": STRING_SCHEMA_OR_ARRAY_OF_STRINGS, + }, + )) + } + + } + + return nil +} diff --git a/tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/schemaLoader.go b/tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/schemaLoader.go new file mode 100644 index 00000000000..09d3a37e255 --- /dev/null +++ b/tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/schemaLoader.go @@ -0,0 +1,102 @@ +package gojsonschema + +import ( + "github.com/xeipuuv/gojsonreference" +) + +type SchemaLoader struct { + pool *schemaPool +} + +func NewSchemaLoader() *SchemaLoader { + + ps := &SchemaLoader{ + pool: &schemaPool{ + schemaPoolDocuments: make(map[string]*schemaPoolDocument), + }, + } + + return ps +} + +// AddSchemas adds an arbritrary amount of schemas to the schema cache. As this function does not require +// an explicit URL, every schema should contain an $id, so that it can be referenced by the main schema +func (sl *SchemaLoader) AddSchemas(loaders ...JSONLoader) error { + emptyRef, _ := gojsonreference.NewJsonReference("") + + for _, loader := range loaders { + doc, err := loader.LoadJSON() + if err != nil { + return err + } + // Directly use the Recursive function, so that it get only added to the schema pool by $id + // and not by the ref of the document as it's empty + if err = sl.pool.parseReferencesRecursive(doc, emptyRef); err != nil { + return err + } + } + + return nil +} + +//AddSchema adds a schema under the provided URL to the schema cache +func (sl *SchemaLoader) AddSchema(url string, loader JSONLoader) error { + + ref, err := gojsonreference.NewJsonReference(url) + + if err != nil { + return err + } + + doc, err := loader.LoadJSON() + + if err != nil { + return err + } + + return sl.pool.ParseReferences(doc, ref) +} + +func (sl *SchemaLoader) Compile(rootSchema JSONLoader) (*Schema, error) { + + ref, err := rootSchema.JsonReference() + + if err != nil { + return nil, err + } + + d := Schema{} + d.pool = sl.pool + d.pool.jsonLoaderFactory = rootSchema.LoaderFactory() + d.documentReference = ref + d.referencePool = newSchemaReferencePool() + + var doc interface{} + if ref.String() != "" { + // Get document from schema pool + spd, err := d.pool.GetDocument(d.documentReference) + if err != nil { + return nil, err + } + doc = spd.Document + } else { + // Load JSON directly + doc, err = rootSchema.LoadJSON() + if err != nil { + return nil, err + } + // References need only be parsed if loading JSON directly + // as pool.GetDocument already does this for us if loading by reference + err = d.pool.ParseReferences(doc, ref) + if err != nil { + return nil, err + } + } + + err = d.parse(doc) + if err != nil { + return nil, err + } + + return &d, nil +} diff --git a/tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/schemaPool.go b/tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/schemaPool.go new file mode 100644 index 00000000000..c3de8e5ef3f --- /dev/null +++ b/tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/schemaPool.go @@ -0,0 +1,201 @@ +// Copyright 2015 xeipuuv ( https://github.com/xeipuuv ) +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// author xeipuuv +// author-github https://github.com/xeipuuv +// author-mail xeipuuv@gmail.com +// +// repository-name gojsonschema +// repository-desc An implementation of JSON Schema, based on IETF's draft v4 - Go language. +// +// description Defines resources pooling. +// Eases referencing and avoids downloading the same resource twice. +// +// created 26-02-2013 + +package gojsonschema + +import ( + "errors" + "fmt" + "reflect" + + "github.com/xeipuuv/gojsonreference" +) + +type schemaPoolDocument struct { + Document interface{} +} + +type schemaPool struct { + schemaPoolDocuments map[string]*schemaPoolDocument + jsonLoaderFactory JSONLoaderFactory +} + +func newSchemaPool(f JSONLoaderFactory) *schemaPool { + + p := &schemaPool{} + p.schemaPoolDocuments = make(map[string]*schemaPoolDocument) + p.jsonLoaderFactory = f + + return p +} + +func (p *schemaPool) ParseReferences(document interface{}, ref gojsonreference.JsonReference) error { + // Only the root document should be added to the schema pool + if _, ok := p.schemaPoolDocuments[ref.String()]; ok { + return fmt.Errorf("Reference already exists: \"%s\"", ref.String()) + } + err := p.parseReferencesRecursive(document, ref) + p.schemaPoolDocuments[ref.String()] = &schemaPoolDocument{Document: document} + return err +} + +func (p *schemaPool) parseReferencesRecursive(document interface{}, ref gojsonreference.JsonReference) error { + // parseReferencesRecursive parses a JSON document and resolves all $id and $ref references. + // For $ref references it takes into account the $id scope it is in and replaces + // the reference by the absolute resolved reference + + // When encountering errors it fails silently. Error handling is done when the schema + // is syntactically parsed and any error encountered here should also come up there. + switch m := document.(type) { + case []interface{}: + for _, v := range m { + p.parseReferencesRecursive(v, ref) + } + case map[string]interface{}: + localRef := &ref + + keyID := KEY_ID_NEW + if existsMapKey(m, KEY_ID) { + keyID = KEY_ID + } + if existsMapKey(m, keyID) && isKind(m[keyID], reflect.String) { + jsonReference, err := gojsonreference.NewJsonReference(m[keyID].(string)) + if err == nil { + localRef, err = ref.Inherits(jsonReference) + if err == nil { + if _, ok := p.schemaPoolDocuments[localRef.String()]; ok { + return fmt.Errorf("Reference already exists: \"%s\"", localRef.String()) + } + p.schemaPoolDocuments[localRef.String()] = &schemaPoolDocument{Document: document} + } + } + } + + if existsMapKey(m, KEY_REF) && isKind(m[KEY_REF], reflect.String) { + jsonReference, err := gojsonreference.NewJsonReference(m[KEY_REF].(string)) + if err == nil { + absoluteRef, err := localRef.Inherits(jsonReference) + if err == nil { + m[KEY_REF] = absoluteRef.String() + } + } + } + + for k, v := range m { + // const and enums should be interpreted literally, so ignore them + if k == KEY_CONST || k == KEY_ENUM { + continue + } + // Something like a property or a dependency is not a valid schema, as it might describe properties named "$ref", "$id" or "const", etc + // Therefore don't treat it like a schema. + if k == KEY_PROPERTIES || k == KEY_DEPENDENCIES || k == KEY_PATTERN_PROPERTIES { + if child, ok := v.(map[string]interface{}); ok { + for _, v := range child { + p.parseReferencesRecursive(v, *localRef) + } + } + } else { + p.parseReferencesRecursive(v, *localRef) + } + } + } + return nil +} + +func (p *schemaPool) GetDocument(reference gojsonreference.JsonReference) (*schemaPoolDocument, error) { + + var ( + spd *schemaPoolDocument + ok bool + err error + ) + + if internalLogEnabled { + internalLog("Get Document ( %s )", reference.String()) + } + + // Create a deep copy, so we can remove the fragment part later on without altering the original + refToUrl, _ := gojsonreference.NewJsonReference(reference.String()) + + // First check if the given fragment is a location independent identifier + // http://json-schema.org/latest/json-schema-core.html#rfc.section.8.2.3 + + if spd, ok = p.schemaPoolDocuments[refToUrl.String()]; ok { + if internalLogEnabled { + internalLog(" From pool") + } + return spd, nil + } + + // If the given reference is not a location independent identifier, + // strip the fragment and look for a document with it's base URI + + refToUrl.GetUrl().Fragment = "" + + if cachedSpd, ok := p.schemaPoolDocuments[refToUrl.String()]; ok { + document, _, err := reference.GetPointer().Get(cachedSpd.Document) + + if err != nil { + return nil, err + } + + if internalLogEnabled { + internalLog(" From pool") + } + + spd = &schemaPoolDocument{Document: document} + p.schemaPoolDocuments[reference.String()] = spd + + return spd, nil + } + + // It is not possible to load anything remotely that is not canonical... + if !reference.IsCanonical() { + return nil, errors.New(formatErrorDescription( + Locale.ReferenceMustBeCanonical(), + ErrorDetails{"reference": reference.String()}, + )) + } + + jsonReferenceLoader := p.jsonLoaderFactory.New(reference.String()) + document, err := jsonReferenceLoader.LoadJSON() + + if err != nil { + return nil, err + } + + // add the whole document to the pool for potential re-use + p.ParseReferences(document, refToUrl) + + // resolve the potential fragment and also cache it + document, _, err = reference.GetPointer().Get(document) + + if err != nil { + return nil, err + } + + return &schemaPoolDocument{Document: document}, nil +} diff --git a/tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/schemaReferencePool.go b/tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/schemaReferencePool.go new file mode 100644 index 00000000000..6e5e1b5cdb3 --- /dev/null +++ b/tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/schemaReferencePool.go @@ -0,0 +1,68 @@ +// Copyright 2015 xeipuuv ( https://github.com/xeipuuv ) +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// author xeipuuv +// author-github https://github.com/xeipuuv +// author-mail xeipuuv@gmail.com +// +// repository-name gojsonschema +// repository-desc An implementation of JSON Schema, based on IETF's draft v4 - Go language. +// +// description Pool of referenced schemas. +// +// created 25-06-2013 + +package gojsonschema + +import ( + "fmt" +) + +type schemaReferencePool struct { + documents map[string]*subSchema +} + +func newSchemaReferencePool() *schemaReferencePool { + + p := &schemaReferencePool{} + p.documents = make(map[string]*subSchema) + + return p +} + +func (p *schemaReferencePool) Get(ref string) (r *subSchema, o bool) { + + if internalLogEnabled { + internalLog(fmt.Sprintf("Schema Reference ( %s )", ref)) + } + + if sch, ok := p.documents[ref]; ok { + if internalLogEnabled { + internalLog(fmt.Sprintf(" From pool")) + } + return sch, true + } + + return nil, false +} + +func (p *schemaReferencePool) Add(ref string, sch *subSchema) { + + if internalLogEnabled { + internalLog(fmt.Sprintf("Add Schema Reference %s to pool", ref)) + } + if _, ok := p.documents[ref]; !ok { + p.documents[ref] = sch + } +} diff --git a/tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/schemaType.go b/tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/schemaType.go new file mode 100644 index 00000000000..36b447a2915 --- /dev/null +++ b/tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/schemaType.go @@ -0,0 +1,83 @@ +// Copyright 2015 xeipuuv ( https://github.com/xeipuuv ) +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// author xeipuuv +// author-github https://github.com/xeipuuv +// author-mail xeipuuv@gmail.com +// +// repository-name gojsonschema +// repository-desc An implementation of JSON Schema, based on IETF's draft v4 - Go language. +// +// description Helper structure to handle schema types, and the combination of them. +// +// created 28-02-2013 + +package gojsonschema + +import ( + "errors" + "fmt" + "strings" +) + +type jsonSchemaType struct { + types []string +} + +// Is the schema typed ? that is containing at least one type +// When not typed, the schema does not need any type validation +func (t *jsonSchemaType) IsTyped() bool { + return len(t.types) > 0 +} + +func (t *jsonSchemaType) Add(etype string) error { + + if !isStringInSlice(JSON_TYPES, etype) { + return errors.New(formatErrorDescription(Locale.NotAValidType(), ErrorDetails{"given": "/" + etype + "/", "expected": JSON_TYPES})) + } + + if t.Contains(etype) { + return errors.New(formatErrorDescription(Locale.Duplicated(), ErrorDetails{"type": etype})) + } + + t.types = append(t.types, etype) + + return nil +} + +func (t *jsonSchemaType) Contains(etype string) bool { + + for _, v := range t.types { + if v == etype { + return true + } + } + + return false +} + +func (t *jsonSchemaType) String() string { + + if len(t.types) == 0 { + return STRING_UNDEFINED // should never happen + } + + // Displayed as a list [type1,type2,...] + if len(t.types) > 1 { + return fmt.Sprintf("[%s]", strings.Join(t.types, ",")) + } + + // Only one type: name only + return t.types[0] +} diff --git a/tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/subSchema.go b/tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/subSchema.go new file mode 100644 index 00000000000..ea792a111a2 --- /dev/null +++ b/tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/subSchema.go @@ -0,0 +1,255 @@ +// Copyright 2015 xeipuuv ( https://github.com/xeipuuv ) +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// author xeipuuv +// author-github https://github.com/xeipuuv +// author-mail xeipuuv@gmail.com +// +// repository-name gojsonschema +// repository-desc An implementation of JSON Schema, based on IETF's draft v4 - Go language. +// +// description Defines the structure of a sub-subSchema. +// A sub-subSchema can contain other sub-schemas. +// +// created 27-02-2013 + +package gojsonschema + +import ( + "errors" + "math/big" + "regexp" + "strings" + + "github.com/xeipuuv/gojsonreference" +) + +const ( + KEY_SCHEMA = "$schema" + KEY_ID = "id" + KEY_ID_NEW = "$id" + KEY_REF = "$ref" + KEY_TITLE = "title" + KEY_DESCRIPTION = "description" + KEY_TYPE = "type" + KEY_ITEMS = "items" + KEY_ADDITIONAL_ITEMS = "additionalItems" + KEY_PROPERTIES = "properties" + KEY_PATTERN_PROPERTIES = "patternProperties" + KEY_ADDITIONAL_PROPERTIES = "additionalProperties" + KEY_PROPERTY_NAMES = "propertyNames" + KEY_DEFINITIONS = "definitions" + KEY_MULTIPLE_OF = "multipleOf" + KEY_MINIMUM = "minimum" + KEY_MAXIMUM = "maximum" + KEY_EXCLUSIVE_MINIMUM = "exclusiveMinimum" + KEY_EXCLUSIVE_MAXIMUM = "exclusiveMaximum" + KEY_MIN_LENGTH = "minLength" + KEY_MAX_LENGTH = "maxLength" + KEY_PATTERN = "pattern" + KEY_FORMAT = "format" + KEY_MIN_PROPERTIES = "minProperties" + KEY_MAX_PROPERTIES = "maxProperties" + KEY_DEPENDENCIES = "dependencies" + KEY_REQUIRED = "required" + KEY_MIN_ITEMS = "minItems" + KEY_MAX_ITEMS = "maxItems" + KEY_UNIQUE_ITEMS = "uniqueItems" + KEY_CONTAINS = "contains" + KEY_CONST = "const" + KEY_ENUM = "enum" + KEY_ONE_OF = "oneOf" + KEY_ANY_OF = "anyOf" + KEY_ALL_OF = "allOf" + KEY_NOT = "not" + KEY_IF = "if" + KEY_THEN = "then" + KEY_ELSE = "else" +) + +type subSchema struct { + + // basic subSchema meta properties + id *gojsonreference.JsonReference + title *string + description *string + + property string + + // Types associated with the subSchema + types jsonSchemaType + + // Reference url + ref *gojsonreference.JsonReference + // Schema referenced + refSchema *subSchema + + // hierarchy + parent *subSchema + itemsChildren []*subSchema + itemsChildrenIsSingleSchema bool + propertiesChildren []*subSchema + + // validation : number / integer + multipleOf *big.Float + maximum *big.Float + exclusiveMaximum bool + minimum *big.Float + exclusiveMinimum bool + + // validation : string + minLength *int + maxLength *int + pattern *regexp.Regexp + format string + + // validation : object + minProperties *int + maxProperties *int + required []string + + dependencies map[string]interface{} + additionalProperties interface{} + patternProperties map[string]*subSchema + propertyNames *subSchema + + // validation : array + minItems *int + maxItems *int + uniqueItems bool + contains *subSchema + + additionalItems interface{} + + // validation : all + _const *string //const is a golang keyword + enum []string + + // validation : subSchema + oneOf []*subSchema + anyOf []*subSchema + allOf []*subSchema + not *subSchema + _if *subSchema // if/else are golang keywords + _then *subSchema + _else *subSchema +} + +func (s *subSchema) AddConst(i interface{}) error { + + is, err := marshalWithoutNumber(i) + if err != nil { + return err + } + s._const = is + return nil +} + +func (s *subSchema) AddEnum(i interface{}) error { + + is, err := marshalWithoutNumber(i) + if err != nil { + return err + } + + if isStringInSlice(s.enum, *is) { + return errors.New(formatErrorDescription( + Locale.KeyItemsMustBeUnique(), + ErrorDetails{"key": KEY_ENUM}, + )) + } + + s.enum = append(s.enum, *is) + + return nil +} + +func (s *subSchema) ContainsEnum(i interface{}) (bool, error) { + + is, err := marshalWithoutNumber(i) + if err != nil { + return false, err + } + + return isStringInSlice(s.enum, *is), nil +} + +func (s *subSchema) AddOneOf(subSchema *subSchema) { + s.oneOf = append(s.oneOf, subSchema) +} + +func (s *subSchema) AddAllOf(subSchema *subSchema) { + s.allOf = append(s.allOf, subSchema) +} + +func (s *subSchema) AddAnyOf(subSchema *subSchema) { + s.anyOf = append(s.anyOf, subSchema) +} + +func (s *subSchema) SetNot(subSchema *subSchema) { + s.not = subSchema +} + +func (s *subSchema) SetIf(subSchema *subSchema) { + s._if = subSchema +} + +func (s *subSchema) SetThen(subSchema *subSchema) { + s._then = subSchema +} + +func (s *subSchema) SetElse(subSchema *subSchema) { + s._else = subSchema +} + +func (s *subSchema) AddRequired(value string) error { + + if isStringInSlice(s.required, value) { + return errors.New(formatErrorDescription( + Locale.KeyItemsMustBeUnique(), + ErrorDetails{"key": KEY_REQUIRED}, + )) + } + + s.required = append(s.required, value) + + return nil +} + +func (s *subSchema) AddItemsChild(child *subSchema) { + s.itemsChildren = append(s.itemsChildren, child) +} + +func (s *subSchema) AddPropertiesChild(child *subSchema) { + s.propertiesChildren = append(s.propertiesChildren, child) +} + +func (s *subSchema) PatternPropertiesString() string { + + if s.patternProperties == nil || len(s.patternProperties) == 0 { + return STRING_UNDEFINED // should never happen + } + + patternPropertiesKeySlice := []string{} + for pk := range s.patternProperties { + patternPropertiesKeySlice = append(patternPropertiesKeySlice, `"`+pk+`"`) + } + + if len(patternPropertiesKeySlice) == 1 { + return patternPropertiesKeySlice[0] + } + + return "[" + strings.Join(patternPropertiesKeySlice, ",") + "]" + +} diff --git a/tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/types.go b/tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/types.go new file mode 100644 index 00000000000..952d22ef65e --- /dev/null +++ b/tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/types.go @@ -0,0 +1,58 @@ +// Copyright 2015 xeipuuv ( https://github.com/xeipuuv ) +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// author xeipuuv +// author-github https://github.com/xeipuuv +// author-mail xeipuuv@gmail.com +// +// repository-name gojsonschema +// repository-desc An implementation of JSON Schema, based on IETF's draft v4 - Go language. +// +// description Contains const types for schema and JSON. +// +// created 28-02-2013 + +package gojsonschema + +const ( + TYPE_ARRAY = `array` + TYPE_BOOLEAN = `boolean` + TYPE_INTEGER = `integer` + TYPE_NUMBER = `number` + TYPE_NULL = `null` + TYPE_OBJECT = `object` + TYPE_STRING = `string` +) + +var JSON_TYPES []string +var SCHEMA_TYPES []string + +func init() { + JSON_TYPES = []string{ + TYPE_ARRAY, + TYPE_BOOLEAN, + TYPE_INTEGER, + TYPE_NUMBER, + TYPE_NULL, + TYPE_OBJECT, + TYPE_STRING} + + SCHEMA_TYPES = []string{ + TYPE_ARRAY, + TYPE_BOOLEAN, + TYPE_INTEGER, + TYPE_NUMBER, + TYPE_OBJECT, + TYPE_STRING} +} diff --git a/tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/utils.go b/tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/utils.go new file mode 100644 index 00000000000..cc3100a78dd --- /dev/null +++ b/tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/utils.go @@ -0,0 +1,226 @@ +// Copyright 2015 xeipuuv ( https://github.com/xeipuuv ) +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// author xeipuuv +// author-github https://github.com/xeipuuv +// author-mail xeipuuv@gmail.com +// +// repository-name gojsonschema +// repository-desc An implementation of JSON Schema, based on IETF's draft v4 - Go language. +// +// description Various utility functions. +// +// created 26-02-2013 + +package gojsonschema + +import ( + "encoding/json" + "fmt" + "math" + "math/big" + "reflect" +) + +func isKind(what interface{}, kinds ...reflect.Kind) bool { + target := what + if isJsonNumber(what) { + // JSON Numbers are strings! + target = *mustBeNumber(what) + } + targetKind := reflect.ValueOf(target).Kind() + for _, kind := range kinds { + if targetKind == kind { + return true + } + } + return false +} + +func existsMapKey(m map[string]interface{}, k string) bool { + _, ok := m[k] + return ok +} + +func isStringInSlice(s []string, what string) bool { + for i := range s { + if s[i] == what { + return true + } + } + return false +} + +func marshalToJsonString(value interface{}) (*string, error) { + + mBytes, err := json.Marshal(value) + if err != nil { + return nil, err + } + + sBytes := string(mBytes) + return &sBytes, nil +} + +func marshalWithoutNumber(value interface{}) (*string, error) { + + // The JSON is decoded using https://golang.org/pkg/encoding/json/#Decoder.UseNumber + // This means the numbers are internally still represented as strings and therefore 1.00 is unequal to 1 + // One way to eliminate these differences is to decode and encode the JSON one more time without Decoder.UseNumber + // so that these differences in representation are removed + + jsonString, err := marshalToJsonString(value) + if err != nil { + return nil, err + } + + var document interface{} + + err = json.Unmarshal([]byte(*jsonString), &document) + if err != nil { + return nil, err + } + + return marshalToJsonString(document) +} + +func isJsonNumber(what interface{}) bool { + + switch what.(type) { + + case json.Number: + return true + } + + return false +} + +func checkJsonInteger(what interface{}) (isInt bool) { + + jsonNumber := what.(json.Number) + + bigFloat, isValidNumber := new(big.Float).SetString(string(jsonNumber)) + + return isValidNumber && bigFloat.IsInt() + +} + +// same as ECMA Number.MAX_SAFE_INTEGER and Number.MIN_SAFE_INTEGER +const ( + max_json_float = float64(1<<53 - 1) // 9007199254740991.0 2^53 - 1 + min_json_float = -float64(1<<53 - 1) //-9007199254740991.0 -2^53 - 1 +) + +func isFloat64AnInteger(f float64) bool { + + if math.IsNaN(f) || math.IsInf(f, 0) || f < min_json_float || f > max_json_float { + return false + } + + return f == float64(int64(f)) || f == float64(uint64(f)) +} + +func mustBeInteger(what interface{}) *int { + + if isJsonNumber(what) { + + number := what.(json.Number) + + isInt := checkJsonInteger(number) + + if isInt { + + int64Value, err := number.Int64() + if err != nil { + return nil + } + + int32Value := int(int64Value) + return &int32Value + + } else { + return nil + } + + } + + return nil +} + +func mustBeNumber(what interface{}) *big.Float { + + if isJsonNumber(what) { + number := what.(json.Number) + float64Value, success := new(big.Float).SetString(string(number)) + if success { + return float64Value + } else { + return nil + } + + } + + return nil + +} + +// formats a number so that it is displayed as the smallest string possible +func resultErrorFormatJsonNumber(n json.Number) string { + + if int64Value, err := n.Int64(); err == nil { + return fmt.Sprintf("%d", int64Value) + } + + float64Value, _ := n.Float64() + + return fmt.Sprintf("%g", float64Value) +} + +// formats a number so that it is displayed as the smallest string possible +func resultErrorFormatNumber(n float64) string { + + if isFloat64AnInteger(n) { + return fmt.Sprintf("%d", int64(n)) + } + + return fmt.Sprintf("%g", n) +} + +func convertDocumentNode(val interface{}) interface{} { + + if lval, ok := val.([]interface{}); ok { + + res := []interface{}{} + for _, v := range lval { + res = append(res, convertDocumentNode(v)) + } + + return res + + } + + if mval, ok := val.(map[interface{}]interface{}); ok { + + res := map[string]interface{}{} + + for k, v := range mval { + res[k.(string)] = convertDocumentNode(v) + } + + return res + + } + + return val +} diff --git a/tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/validation.go b/tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/validation.go new file mode 100644 index 00000000000..2f1e7df36c1 --- /dev/null +++ b/tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/validation.go @@ -0,0 +1,928 @@ +// Copyright 2015 xeipuuv ( https://github.com/xeipuuv ) +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// author xeipuuv +// author-github https://github.com/xeipuuv +// author-mail xeipuuv@gmail.com +// +// repository-name gojsonschema +// repository-desc An implementation of JSON Schema, based on IETF's draft v4 - Go language. +// +// description Extends Schema and subSchema, implements the validation phase. +// +// created 28-02-2013 + +package gojsonschema + +import ( + "encoding/json" + "math/big" + "reflect" + "regexp" + "strconv" + "strings" + "unicode/utf8" +) + +func Validate(ls JSONLoader, ld JSONLoader) (*Result, error) { + + var err error + + // load schema + + schema, err := NewSchema(ls) + if err != nil { + return nil, err + } + + // begine validation + + return schema.Validate(ld) + +} + +func (v *Schema) Validate(l JSONLoader) (*Result, error) { + + // load document + + root, err := l.LoadJSON() + if err != nil { + return nil, err + } + + return v.validateDocument(root), nil +} + +func (v *Schema) validateDocument(root interface{}) *Result { + // begin validation + + result := &Result{} + context := NewJsonContext(STRING_CONTEXT_ROOT, nil) + v.rootSchema.validateRecursive(v.rootSchema, root, result, context) + + return result +} + +func (v *subSchema) subValidateWithContext(document interface{}, context *JsonContext) *Result { + result := &Result{} + v.validateRecursive(v, document, result, context) + return result +} + +// Walker function to validate the json recursively against the subSchema +func (v *subSchema) validateRecursive(currentSubSchema *subSchema, currentNode interface{}, result *Result, context *JsonContext) { + + if internalLogEnabled { + internalLog("validateRecursive %s", context.String()) + internalLog(" %v", currentNode) + } + + // Handle referenced schemas, returns directly when a $ref is found + if currentSubSchema.refSchema != nil { + v.validateRecursive(currentSubSchema.refSchema, currentNode, result, context) + return + } + + // Check for null value + if currentNode == nil { + if currentSubSchema.types.IsTyped() && !currentSubSchema.types.Contains(TYPE_NULL) { + result.addInternalError( + new(InvalidTypeError), + context, + currentNode, + ErrorDetails{ + "expected": currentSubSchema.types.String(), + "given": TYPE_NULL, + }, + ) + return + } + + currentSubSchema.validateSchema(currentSubSchema, currentNode, result, context) + v.validateCommon(currentSubSchema, currentNode, result, context) + + } else { // Not a null value + + if isJsonNumber(currentNode) { + + value := currentNode.(json.Number) + + isInt := checkJsonInteger(value) + + validType := currentSubSchema.types.Contains(TYPE_NUMBER) || (isInt && currentSubSchema.types.Contains(TYPE_INTEGER)) + + if currentSubSchema.types.IsTyped() && !validType { + + givenType := TYPE_INTEGER + if !isInt { + givenType = TYPE_NUMBER + } + + result.addInternalError( + new(InvalidTypeError), + context, + currentNode, + ErrorDetails{ + "expected": currentSubSchema.types.String(), + "given": givenType, + }, + ) + return + } + + currentSubSchema.validateSchema(currentSubSchema, value, result, context) + v.validateNumber(currentSubSchema, value, result, context) + v.validateCommon(currentSubSchema, value, result, context) + v.validateString(currentSubSchema, value, result, context) + + } else { + + rValue := reflect.ValueOf(currentNode) + rKind := rValue.Kind() + + switch rKind { + + // Slice => JSON array + + case reflect.Slice: + + if currentSubSchema.types.IsTyped() && !currentSubSchema.types.Contains(TYPE_ARRAY) { + result.addInternalError( + new(InvalidTypeError), + context, + currentNode, + ErrorDetails{ + "expected": currentSubSchema.types.String(), + "given": TYPE_ARRAY, + }, + ) + return + } + + castCurrentNode := currentNode.([]interface{}) + + currentSubSchema.validateSchema(currentSubSchema, castCurrentNode, result, context) + + v.validateArray(currentSubSchema, castCurrentNode, result, context) + v.validateCommon(currentSubSchema, castCurrentNode, result, context) + + // Map => JSON object + + case reflect.Map: + if currentSubSchema.types.IsTyped() && !currentSubSchema.types.Contains(TYPE_OBJECT) { + result.addInternalError( + new(InvalidTypeError), + context, + currentNode, + ErrorDetails{ + "expected": currentSubSchema.types.String(), + "given": TYPE_OBJECT, + }, + ) + return + } + + castCurrentNode, ok := currentNode.(map[string]interface{}) + if !ok { + castCurrentNode = convertDocumentNode(currentNode).(map[string]interface{}) + } + + currentSubSchema.validateSchema(currentSubSchema, castCurrentNode, result, context) + + v.validateObject(currentSubSchema, castCurrentNode, result, context) + v.validateCommon(currentSubSchema, castCurrentNode, result, context) + + for _, pSchema := range currentSubSchema.propertiesChildren { + nextNode, ok := castCurrentNode[pSchema.property] + if ok { + subContext := NewJsonContext(pSchema.property, context) + v.validateRecursive(pSchema, nextNode, result, subContext) + } + } + + // Simple JSON values : string, number, boolean + + case reflect.Bool: + + if currentSubSchema.types.IsTyped() && !currentSubSchema.types.Contains(TYPE_BOOLEAN) { + result.addInternalError( + new(InvalidTypeError), + context, + currentNode, + ErrorDetails{ + "expected": currentSubSchema.types.String(), + "given": TYPE_BOOLEAN, + }, + ) + return + } + + value := currentNode.(bool) + + currentSubSchema.validateSchema(currentSubSchema, value, result, context) + v.validateNumber(currentSubSchema, value, result, context) + v.validateCommon(currentSubSchema, value, result, context) + v.validateString(currentSubSchema, value, result, context) + + case reflect.String: + + if currentSubSchema.types.IsTyped() && !currentSubSchema.types.Contains(TYPE_STRING) { + result.addInternalError( + new(InvalidTypeError), + context, + currentNode, + ErrorDetails{ + "expected": currentSubSchema.types.String(), + "given": TYPE_STRING, + }, + ) + return + } + + value := currentNode.(string) + + currentSubSchema.validateSchema(currentSubSchema, value, result, context) + v.validateNumber(currentSubSchema, value, result, context) + v.validateCommon(currentSubSchema, value, result, context) + v.validateString(currentSubSchema, value, result, context) + + } + + } + + } + + result.incrementScore() +} + +// Different kinds of validation there, subSchema / common / array / object / string... +func (v *subSchema) validateSchema(currentSubSchema *subSchema, currentNode interface{}, result *Result, context *JsonContext) { + + if internalLogEnabled { + internalLog("validateSchema %s", context.String()) + internalLog(" %v", currentNode) + } + + if len(currentSubSchema.anyOf) > 0 { + + validatedAnyOf := false + var bestValidationResult *Result + + for _, anyOfSchema := range currentSubSchema.anyOf { + if !validatedAnyOf { + validationResult := anyOfSchema.subValidateWithContext(currentNode, context) + validatedAnyOf = validationResult.Valid() + + if !validatedAnyOf && (bestValidationResult == nil || validationResult.score > bestValidationResult.score) { + bestValidationResult = validationResult + } + } + } + if !validatedAnyOf { + + result.addInternalError(new(NumberAnyOfError), context, currentNode, ErrorDetails{}) + + if bestValidationResult != nil { + // add error messages of closest matching subSchema as + // that's probably the one the user was trying to match + result.mergeErrors(bestValidationResult) + } + } + } + + if len(currentSubSchema.oneOf) > 0 { + + nbValidated := 0 + var bestValidationResult *Result + + for _, oneOfSchema := range currentSubSchema.oneOf { + validationResult := oneOfSchema.subValidateWithContext(currentNode, context) + if validationResult.Valid() { + nbValidated++ + } else if nbValidated == 0 && (bestValidationResult == nil || validationResult.score > bestValidationResult.score) { + bestValidationResult = validationResult + } + } + + if nbValidated != 1 { + + result.addInternalError(new(NumberOneOfError), context, currentNode, ErrorDetails{}) + + if nbValidated == 0 { + // add error messages of closest matching subSchema as + // that's probably the one the user was trying to match + result.mergeErrors(bestValidationResult) + } + } + + } + + if len(currentSubSchema.allOf) > 0 { + nbValidated := 0 + + for _, allOfSchema := range currentSubSchema.allOf { + validationResult := allOfSchema.subValidateWithContext(currentNode, context) + if validationResult.Valid() { + nbValidated++ + } + result.mergeErrors(validationResult) + } + + if nbValidated != len(currentSubSchema.allOf) { + result.addInternalError(new(NumberAllOfError), context, currentNode, ErrorDetails{}) + } + } + + if currentSubSchema.not != nil { + validationResult := currentSubSchema.not.subValidateWithContext(currentNode, context) + if validationResult.Valid() { + result.addInternalError(new(NumberNotError), context, currentNode, ErrorDetails{}) + } + } + + if currentSubSchema.dependencies != nil && len(currentSubSchema.dependencies) > 0 { + if isKind(currentNode, reflect.Map) { + for elementKey := range currentNode.(map[string]interface{}) { + if dependency, ok := currentSubSchema.dependencies[elementKey]; ok { + switch dependency := dependency.(type) { + + case []string: + for _, dependOnKey := range dependency { + if _, dependencyResolved := currentNode.(map[string]interface{})[dependOnKey]; !dependencyResolved { + result.addInternalError( + new(MissingDependencyError), + context, + currentNode, + ErrorDetails{"dependency": dependOnKey}, + ) + } + } + + case *subSchema: + dependency.validateRecursive(dependency, currentNode, result, context) + } + } + } + } + } + + if currentSubSchema._if != nil { + validationResultIf := currentSubSchema._if.subValidateWithContext(currentNode, context) + if currentSubSchema._then != nil && validationResultIf.Valid() { + validationResultThen := currentSubSchema._then.subValidateWithContext(currentNode, context) + if !validationResultThen.Valid() { + result.addInternalError(new(ConditionThenError), context, currentNode, ErrorDetails{}) + result.mergeErrors(validationResultThen) + } + } + if currentSubSchema._else != nil && !validationResultIf.Valid() { + validationResultElse := currentSubSchema._else.subValidateWithContext(currentNode, context) + if !validationResultElse.Valid() { + result.addInternalError(new(ConditionElseError), context, currentNode, ErrorDetails{}) + result.mergeErrors(validationResultElse) + } + } + } + + result.incrementScore() +} + +func (v *subSchema) validateCommon(currentSubSchema *subSchema, value interface{}, result *Result, context *JsonContext) { + + if internalLogEnabled { + internalLog("validateCommon %s", context.String()) + internalLog(" %v", value) + } + + // const: + if currentSubSchema._const != nil { + vString, err := marshalWithoutNumber(value) + if err != nil { + result.addInternalError(new(InternalError), context, value, ErrorDetails{"error": err}) + } + if *vString != *currentSubSchema._const { + result.addInternalError(new(ConstError), + context, + value, + ErrorDetails{ + "allowed": *currentSubSchema._const, + }, + ) + } + } + + // enum: + if len(currentSubSchema.enum) > 0 { + has, err := currentSubSchema.ContainsEnum(value) + if err != nil { + result.addInternalError(new(InternalError), context, value, ErrorDetails{"error": err}) + } + if !has { + result.addInternalError( + new(EnumError), + context, + value, + ErrorDetails{ + "allowed": strings.Join(currentSubSchema.enum, ", "), + }, + ) + } + } + + result.incrementScore() +} + +func (v *subSchema) validateArray(currentSubSchema *subSchema, value []interface{}, result *Result, context *JsonContext) { + + if internalLogEnabled { + internalLog("validateArray %s", context.String()) + internalLog(" %v", value) + } + + nbValues := len(value) + + // TODO explain + if currentSubSchema.itemsChildrenIsSingleSchema { + for i := range value { + subContext := NewJsonContext(strconv.Itoa(i), context) + validationResult := currentSubSchema.itemsChildren[0].subValidateWithContext(value[i], subContext) + result.mergeErrors(validationResult) + } + } else { + if currentSubSchema.itemsChildren != nil && len(currentSubSchema.itemsChildren) > 0 { + + nbItems := len(currentSubSchema.itemsChildren) + + // while we have both schemas and values, check them against each other + for i := 0; i != nbItems && i != nbValues; i++ { + subContext := NewJsonContext(strconv.Itoa(i), context) + validationResult := currentSubSchema.itemsChildren[i].subValidateWithContext(value[i], subContext) + result.mergeErrors(validationResult) + } + + if nbItems < nbValues { + // we have less schemas than elements in the instance array, + // but that might be ok if "additionalItems" is specified. + + switch currentSubSchema.additionalItems.(type) { + case bool: + if !currentSubSchema.additionalItems.(bool) { + result.addInternalError(new(ArrayNoAdditionalItemsError), context, value, ErrorDetails{}) + } + case *subSchema: + additionalItemSchema := currentSubSchema.additionalItems.(*subSchema) + for i := nbItems; i != nbValues; i++ { + subContext := NewJsonContext(strconv.Itoa(i), context) + validationResult := additionalItemSchema.subValidateWithContext(value[i], subContext) + result.mergeErrors(validationResult) + } + } + } + } + } + + // minItems & maxItems + if currentSubSchema.minItems != nil { + if nbValues < int(*currentSubSchema.minItems) { + result.addInternalError( + new(ArrayMinItemsError), + context, + value, + ErrorDetails{"min": *currentSubSchema.minItems}, + ) + } + } + if currentSubSchema.maxItems != nil { + if nbValues > int(*currentSubSchema.maxItems) { + result.addInternalError( + new(ArrayMaxItemsError), + context, + value, + ErrorDetails{"max": *currentSubSchema.maxItems}, + ) + } + } + + // uniqueItems: + if currentSubSchema.uniqueItems { + var stringifiedItems []string + for _, v := range value { + vString, err := marshalWithoutNumber(v) + if err != nil { + result.addInternalError(new(InternalError), context, value, ErrorDetails{"err": err}) + } + if isStringInSlice(stringifiedItems, *vString) { + result.addInternalError( + new(ItemsMustBeUniqueError), + context, + value, + ErrorDetails{"type": TYPE_ARRAY}, + ) + } + stringifiedItems = append(stringifiedItems, *vString) + } + } + + // contains: + + if currentSubSchema.contains != nil { + validatedOne := false + var bestValidationResult *Result + + for i, v := range value { + subContext := NewJsonContext(strconv.Itoa(i), context) + + validationResult := currentSubSchema.contains.subValidateWithContext(v, subContext) + if validationResult.Valid() { + validatedOne = true + break + } else { + if bestValidationResult == nil || validationResult.score > bestValidationResult.score { + bestValidationResult = validationResult + } + } + } + if !validatedOne { + result.addInternalError( + new(ArrayContainsError), + context, + value, + ErrorDetails{}, + ) + if bestValidationResult != nil { + result.mergeErrors(bestValidationResult) + } + } + } + + result.incrementScore() +} + +func (v *subSchema) validateObject(currentSubSchema *subSchema, value map[string]interface{}, result *Result, context *JsonContext) { + + if internalLogEnabled { + internalLog("validateObject %s", context.String()) + internalLog(" %v", value) + } + + // minProperties & maxProperties: + if currentSubSchema.minProperties != nil { + if len(value) < int(*currentSubSchema.minProperties) { + result.addInternalError( + new(ArrayMinPropertiesError), + context, + value, + ErrorDetails{"min": *currentSubSchema.minProperties}, + ) + } + } + if currentSubSchema.maxProperties != nil { + if len(value) > int(*currentSubSchema.maxProperties) { + result.addInternalError( + new(ArrayMaxPropertiesError), + context, + value, + ErrorDetails{"max": *currentSubSchema.maxProperties}, + ) + } + } + + // required: + for _, requiredProperty := range currentSubSchema.required { + _, ok := value[requiredProperty] + if ok { + result.incrementScore() + } else { + result.addInternalError( + new(RequiredError), + context, + value, + ErrorDetails{"property": requiredProperty}, + ) + } + } + + // additionalProperty & patternProperty: + if currentSubSchema.additionalProperties != nil { + + switch currentSubSchema.additionalProperties.(type) { + case bool: + + if !currentSubSchema.additionalProperties.(bool) { + + for pk := range value { + + found := false + for _, spValue := range currentSubSchema.propertiesChildren { + if pk == spValue.property { + found = true + } + } + + pp_has, pp_match := v.validatePatternProperty(currentSubSchema, pk, value[pk], result, context) + + if found { + + if pp_has && !pp_match { + result.addInternalError( + new(AdditionalPropertyNotAllowedError), + context, + value[pk], + ErrorDetails{"property": pk}, + ) + } + + } else { + + if !pp_has || !pp_match { + result.addInternalError( + new(AdditionalPropertyNotAllowedError), + context, + value[pk], + ErrorDetails{"property": pk}, + ) + } + + } + } + } + + case *subSchema: + + additionalPropertiesSchema := currentSubSchema.additionalProperties.(*subSchema) + for pk := range value { + + found := false + for _, spValue := range currentSubSchema.propertiesChildren { + if pk == spValue.property { + found = true + } + } + + pp_has, pp_match := v.validatePatternProperty(currentSubSchema, pk, value[pk], result, context) + + if found { + + if pp_has && !pp_match { + validationResult := additionalPropertiesSchema.subValidateWithContext(value[pk], context) + result.mergeErrors(validationResult) + } + + } else { + + if !pp_has || !pp_match { + validationResult := additionalPropertiesSchema.subValidateWithContext(value[pk], context) + result.mergeErrors(validationResult) + } + + } + + } + } + } else { + + for pk := range value { + + pp_has, pp_match := v.validatePatternProperty(currentSubSchema, pk, value[pk], result, context) + + if pp_has && !pp_match { + + result.addInternalError( + new(InvalidPropertyPatternError), + context, + value[pk], + ErrorDetails{ + "property": pk, + "pattern": currentSubSchema.PatternPropertiesString(), + }, + ) + } + + } + } + + // propertyNames: + if currentSubSchema.propertyNames != nil { + for pk := range value { + validationResult := currentSubSchema.propertyNames.subValidateWithContext(pk, context) + if !validationResult.Valid() { + result.addInternalError(new(InvalidPropertyNameError), + context, + value, ErrorDetails{ + "property": pk, + }) + result.mergeErrors(validationResult) + } + } + } + + result.incrementScore() +} + +func (v *subSchema) validatePatternProperty(currentSubSchema *subSchema, key string, value interface{}, result *Result, context *JsonContext) (has bool, matched bool) { + + if internalLogEnabled { + internalLog("validatePatternProperty %s", context.String()) + internalLog(" %s %v", key, value) + } + + has = false + + validatedkey := false + + for pk, pv := range currentSubSchema.patternProperties { + if matches, _ := regexp.MatchString(pk, key); matches { + has = true + subContext := NewJsonContext(key, context) + validationResult := pv.subValidateWithContext(value, subContext) + result.mergeErrors(validationResult) + validatedkey = true + } + } + + if !validatedkey { + return has, false + } + + result.incrementScore() + + return has, true +} + +func (v *subSchema) validateString(currentSubSchema *subSchema, value interface{}, result *Result, context *JsonContext) { + + // Ignore JSON numbers + if isJsonNumber(value) { + return + } + + // Ignore non strings + if !isKind(value, reflect.String) { + return + } + + if internalLogEnabled { + internalLog("validateString %s", context.String()) + internalLog(" %v", value) + } + + stringValue := value.(string) + + // minLength & maxLength: + if currentSubSchema.minLength != nil { + if utf8.RuneCount([]byte(stringValue)) < int(*currentSubSchema.minLength) { + result.addInternalError( + new(StringLengthGTEError), + context, + value, + ErrorDetails{"min": *currentSubSchema.minLength}, + ) + } + } + if currentSubSchema.maxLength != nil { + if utf8.RuneCount([]byte(stringValue)) > int(*currentSubSchema.maxLength) { + result.addInternalError( + new(StringLengthLTEError), + context, + value, + ErrorDetails{"max": *currentSubSchema.maxLength}, + ) + } + } + + // pattern: + if currentSubSchema.pattern != nil { + if !currentSubSchema.pattern.MatchString(stringValue) { + result.addInternalError( + new(DoesNotMatchPatternError), + context, + value, + ErrorDetails{"pattern": currentSubSchema.pattern}, + ) + + } + } + + // format + if currentSubSchema.format != "" { + if !FormatCheckers.IsFormat(currentSubSchema.format, stringValue) { + result.addInternalError( + new(DoesNotMatchFormatError), + context, + value, + ErrorDetails{"format": currentSubSchema.format}, + ) + } + } + + result.incrementScore() +} + +func (v *subSchema) validateNumber(currentSubSchema *subSchema, value interface{}, result *Result, context *JsonContext) { + + // Ignore non numbers + if !isJsonNumber(value) { + return + } + + if internalLogEnabled { + internalLog("validateNumber %s", context.String()) + internalLog(" %v", value) + } + + number := value.(json.Number) + float64Value, _ := new(big.Float).SetString(string(number)) + + // multipleOf: + if currentSubSchema.multipleOf != nil { + + if q := new(big.Float).Quo(float64Value, currentSubSchema.multipleOf); !q.IsInt() { + result.addInternalError( + new(MultipleOfError), + context, + resultErrorFormatJsonNumber(number), + ErrorDetails{"multiple": currentSubSchema.multipleOf}, + ) + } + } + + //maximum & exclusiveMaximum: + if currentSubSchema.maximum != nil { + if currentSubSchema.exclusiveMaximum { + if float64Value.Cmp(currentSubSchema.maximum) >= 0 { + result.addInternalError( + new(NumberLTError), + context, + resultErrorFormatJsonNumber(number), + ErrorDetails{ + "max": currentSubSchema.maximum, + }, + ) + } + } else { + if float64Value.Cmp(currentSubSchema.maximum) == 1 { + result.addInternalError( + new(NumberLTEError), + context, + resultErrorFormatJsonNumber(number), + ErrorDetails{ + "max": currentSubSchema.maximum, + }, + ) + } + } + } + + //minimum & exclusiveMinimum: + if currentSubSchema.minimum != nil { + if currentSubSchema.exclusiveMinimum { + if float64Value.Cmp(currentSubSchema.minimum) <= 0 { + // if float64Value <= *currentSubSchema.minimum { + result.addInternalError( + new(NumberGTError), + context, + resultErrorFormatJsonNumber(number), + ErrorDetails{ + "min": currentSubSchema.minimum, + }, + ) + } + } else { + if float64Value.Cmp(currentSubSchema.minimum) == -1 { + result.addInternalError( + new(NumberGTEError), + context, + resultErrorFormatJsonNumber(number), + ErrorDetails{ + "min": currentSubSchema.minimum, + }, + ) + } + } + } + + // format + if currentSubSchema.format != "" { + if !FormatCheckers.IsFormat(currentSubSchema.format, float64Value) { + result.addInternalError( + new(DoesNotMatchFormatError), + context, + value, + ErrorDetails{"format": currentSubSchema.format}, + ) + } + } + + result.incrementScore() +} diff --git a/tools/voluspa/vendor/gopkg.in/yaml.v2/LICENSE b/tools/voluspa/vendor/gopkg.in/yaml.v2/LICENSE new file mode 100644 index 00000000000..8dada3edaf5 --- /dev/null +++ b/tools/voluspa/vendor/gopkg.in/yaml.v2/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/tools/voluspa/vendor/gopkg.in/yaml.v2/LICENSE.libyaml b/tools/voluspa/vendor/gopkg.in/yaml.v2/LICENSE.libyaml new file mode 100644 index 00000000000..8da58fbf6f8 --- /dev/null +++ b/tools/voluspa/vendor/gopkg.in/yaml.v2/LICENSE.libyaml @@ -0,0 +1,31 @@ +The following files were ported to Go from C files of libyaml, and thus +are still covered by their original copyright and license: + + apic.go + emitterc.go + parserc.go + readerc.go + scannerc.go + writerc.go + yamlh.go + yamlprivateh.go + +Copyright (c) 2006 Kirill Simonov + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/tools/voluspa/vendor/gopkg.in/yaml.v2/apic.go b/tools/voluspa/vendor/gopkg.in/yaml.v2/apic.go new file mode 100644 index 00000000000..1f7e87e6727 --- /dev/null +++ b/tools/voluspa/vendor/gopkg.in/yaml.v2/apic.go @@ -0,0 +1,739 @@ +package yaml + +import ( + "io" +) + +func yaml_insert_token(parser *yaml_parser_t, pos int, token *yaml_token_t) { + //fmt.Println("yaml_insert_token", "pos:", pos, "typ:", token.typ, "head:", parser.tokens_head, "len:", len(parser.tokens)) + + // Check if we can move the queue at the beginning of the buffer. + if parser.tokens_head > 0 && len(parser.tokens) == cap(parser.tokens) { + if parser.tokens_head != len(parser.tokens) { + copy(parser.tokens, parser.tokens[parser.tokens_head:]) + } + parser.tokens = parser.tokens[:len(parser.tokens)-parser.tokens_head] + parser.tokens_head = 0 + } + parser.tokens = append(parser.tokens, *token) + if pos < 0 { + return + } + copy(parser.tokens[parser.tokens_head+pos+1:], parser.tokens[parser.tokens_head+pos:]) + parser.tokens[parser.tokens_head+pos] = *token +} + +// Create a new parser object. +func yaml_parser_initialize(parser *yaml_parser_t) bool { + *parser = yaml_parser_t{ + raw_buffer: make([]byte, 0, input_raw_buffer_size), + buffer: make([]byte, 0, input_buffer_size), + } + return true +} + +// Destroy a parser object. +func yaml_parser_delete(parser *yaml_parser_t) { + *parser = yaml_parser_t{} +} + +// String read handler. +func yaml_string_read_handler(parser *yaml_parser_t, buffer []byte) (n int, err error) { + if parser.input_pos == len(parser.input) { + return 0, io.EOF + } + n = copy(buffer, parser.input[parser.input_pos:]) + parser.input_pos += n + return n, nil +} + +// Reader read handler. +func yaml_reader_read_handler(parser *yaml_parser_t, buffer []byte) (n int, err error) { + return parser.input_reader.Read(buffer) +} + +// Set a string input. +func yaml_parser_set_input_string(parser *yaml_parser_t, input []byte) { + if parser.read_handler != nil { + panic("must set the input source only once") + } + parser.read_handler = yaml_string_read_handler + parser.input = input + parser.input_pos = 0 +} + +// Set a file input. +func yaml_parser_set_input_reader(parser *yaml_parser_t, r io.Reader) { + if parser.read_handler != nil { + panic("must set the input source only once") + } + parser.read_handler = yaml_reader_read_handler + parser.input_reader = r +} + +// Set the source encoding. +func yaml_parser_set_encoding(parser *yaml_parser_t, encoding yaml_encoding_t) { + if parser.encoding != yaml_ANY_ENCODING { + panic("must set the encoding only once") + } + parser.encoding = encoding +} + +// Create a new emitter object. +func yaml_emitter_initialize(emitter *yaml_emitter_t) { + *emitter = yaml_emitter_t{ + buffer: make([]byte, output_buffer_size), + raw_buffer: make([]byte, 0, output_raw_buffer_size), + states: make([]yaml_emitter_state_t, 0, initial_stack_size), + events: make([]yaml_event_t, 0, initial_queue_size), + } +} + +// Destroy an emitter object. +func yaml_emitter_delete(emitter *yaml_emitter_t) { + *emitter = yaml_emitter_t{} +} + +// String write handler. +func yaml_string_write_handler(emitter *yaml_emitter_t, buffer []byte) error { + *emitter.output_buffer = append(*emitter.output_buffer, buffer...) + return nil +} + +// yaml_writer_write_handler uses emitter.output_writer to write the +// emitted text. +func yaml_writer_write_handler(emitter *yaml_emitter_t, buffer []byte) error { + _, err := emitter.output_writer.Write(buffer) + return err +} + +// Set a string output. +func yaml_emitter_set_output_string(emitter *yaml_emitter_t, output_buffer *[]byte) { + if emitter.write_handler != nil { + panic("must set the output target only once") + } + emitter.write_handler = yaml_string_write_handler + emitter.output_buffer = output_buffer +} + +// Set a file output. +func yaml_emitter_set_output_writer(emitter *yaml_emitter_t, w io.Writer) { + if emitter.write_handler != nil { + panic("must set the output target only once") + } + emitter.write_handler = yaml_writer_write_handler + emitter.output_writer = w +} + +// Set the output encoding. +func yaml_emitter_set_encoding(emitter *yaml_emitter_t, encoding yaml_encoding_t) { + if emitter.encoding != yaml_ANY_ENCODING { + panic("must set the output encoding only once") + } + emitter.encoding = encoding +} + +// Set the canonical output style. +func yaml_emitter_set_canonical(emitter *yaml_emitter_t, canonical bool) { + emitter.canonical = canonical +} + +//// Set the indentation increment. +func yaml_emitter_set_indent(emitter *yaml_emitter_t, indent int) { + if indent < 2 || indent > 9 { + indent = 2 + } + emitter.best_indent = indent +} + +// Set the preferred line width. +func yaml_emitter_set_width(emitter *yaml_emitter_t, width int) { + if width < 0 { + width = -1 + } + emitter.best_width = width +} + +// Set if unescaped non-ASCII characters are allowed. +func yaml_emitter_set_unicode(emitter *yaml_emitter_t, unicode bool) { + emitter.unicode = unicode +} + +// Set the preferred line break character. +func yaml_emitter_set_break(emitter *yaml_emitter_t, line_break yaml_break_t) { + emitter.line_break = line_break +} + +///* +// * Destroy a token object. +// */ +// +//YAML_DECLARE(void) +//yaml_token_delete(yaml_token_t *token) +//{ +// assert(token); // Non-NULL token object expected. +// +// switch (token.type) +// { +// case YAML_TAG_DIRECTIVE_TOKEN: +// yaml_free(token.data.tag_directive.handle); +// yaml_free(token.data.tag_directive.prefix); +// break; +// +// case YAML_ALIAS_TOKEN: +// yaml_free(token.data.alias.value); +// break; +// +// case YAML_ANCHOR_TOKEN: +// yaml_free(token.data.anchor.value); +// break; +// +// case YAML_TAG_TOKEN: +// yaml_free(token.data.tag.handle); +// yaml_free(token.data.tag.suffix); +// break; +// +// case YAML_SCALAR_TOKEN: +// yaml_free(token.data.scalar.value); +// break; +// +// default: +// break; +// } +// +// memset(token, 0, sizeof(yaml_token_t)); +//} +// +///* +// * Check if a string is a valid UTF-8 sequence. +// * +// * Check 'reader.c' for more details on UTF-8 encoding. +// */ +// +//static int +//yaml_check_utf8(yaml_char_t *start, size_t length) +//{ +// yaml_char_t *end = start+length; +// yaml_char_t *pointer = start; +// +// while (pointer < end) { +// unsigned char octet; +// unsigned int width; +// unsigned int value; +// size_t k; +// +// octet = pointer[0]; +// width = (octet & 0x80) == 0x00 ? 1 : +// (octet & 0xE0) == 0xC0 ? 2 : +// (octet & 0xF0) == 0xE0 ? 3 : +// (octet & 0xF8) == 0xF0 ? 4 : 0; +// value = (octet & 0x80) == 0x00 ? octet & 0x7F : +// (octet & 0xE0) == 0xC0 ? octet & 0x1F : +// (octet & 0xF0) == 0xE0 ? octet & 0x0F : +// (octet & 0xF8) == 0xF0 ? octet & 0x07 : 0; +// if (!width) return 0; +// if (pointer+width > end) return 0; +// for (k = 1; k < width; k ++) { +// octet = pointer[k]; +// if ((octet & 0xC0) != 0x80) return 0; +// value = (value << 6) + (octet & 0x3F); +// } +// if (!((width == 1) || +// (width == 2 && value >= 0x80) || +// (width == 3 && value >= 0x800) || +// (width == 4 && value >= 0x10000))) return 0; +// +// pointer += width; +// } +// +// return 1; +//} +// + +// Create STREAM-START. +func yaml_stream_start_event_initialize(event *yaml_event_t, encoding yaml_encoding_t) { + *event = yaml_event_t{ + typ: yaml_STREAM_START_EVENT, + encoding: encoding, + } +} + +// Create STREAM-END. +func yaml_stream_end_event_initialize(event *yaml_event_t) { + *event = yaml_event_t{ + typ: yaml_STREAM_END_EVENT, + } +} + +// Create DOCUMENT-START. +func yaml_document_start_event_initialize( + event *yaml_event_t, + version_directive *yaml_version_directive_t, + tag_directives []yaml_tag_directive_t, + implicit bool, +) { + *event = yaml_event_t{ + typ: yaml_DOCUMENT_START_EVENT, + version_directive: version_directive, + tag_directives: tag_directives, + implicit: implicit, + } +} + +// Create DOCUMENT-END. +func yaml_document_end_event_initialize(event *yaml_event_t, implicit bool) { + *event = yaml_event_t{ + typ: yaml_DOCUMENT_END_EVENT, + implicit: implicit, + } +} + +///* +// * Create ALIAS. +// */ +// +//YAML_DECLARE(int) +//yaml_alias_event_initialize(event *yaml_event_t, anchor *yaml_char_t) +//{ +// mark yaml_mark_t = { 0, 0, 0 } +// anchor_copy *yaml_char_t = NULL +// +// assert(event) // Non-NULL event object is expected. +// assert(anchor) // Non-NULL anchor is expected. +// +// if (!yaml_check_utf8(anchor, strlen((char *)anchor))) return 0 +// +// anchor_copy = yaml_strdup(anchor) +// if (!anchor_copy) +// return 0 +// +// ALIAS_EVENT_INIT(*event, anchor_copy, mark, mark) +// +// return 1 +//} + +// Create SCALAR. +func yaml_scalar_event_initialize(event *yaml_event_t, anchor, tag, value []byte, plain_implicit, quoted_implicit bool, style yaml_scalar_style_t) bool { + *event = yaml_event_t{ + typ: yaml_SCALAR_EVENT, + anchor: anchor, + tag: tag, + value: value, + implicit: plain_implicit, + quoted_implicit: quoted_implicit, + style: yaml_style_t(style), + } + return true +} + +// Create SEQUENCE-START. +func yaml_sequence_start_event_initialize(event *yaml_event_t, anchor, tag []byte, implicit bool, style yaml_sequence_style_t) bool { + *event = yaml_event_t{ + typ: yaml_SEQUENCE_START_EVENT, + anchor: anchor, + tag: tag, + implicit: implicit, + style: yaml_style_t(style), + } + return true +} + +// Create SEQUENCE-END. +func yaml_sequence_end_event_initialize(event *yaml_event_t) bool { + *event = yaml_event_t{ + typ: yaml_SEQUENCE_END_EVENT, + } + return true +} + +// Create MAPPING-START. +func yaml_mapping_start_event_initialize(event *yaml_event_t, anchor, tag []byte, implicit bool, style yaml_mapping_style_t) { + *event = yaml_event_t{ + typ: yaml_MAPPING_START_EVENT, + anchor: anchor, + tag: tag, + implicit: implicit, + style: yaml_style_t(style), + } +} + +// Create MAPPING-END. +func yaml_mapping_end_event_initialize(event *yaml_event_t) { + *event = yaml_event_t{ + typ: yaml_MAPPING_END_EVENT, + } +} + +// Destroy an event object. +func yaml_event_delete(event *yaml_event_t) { + *event = yaml_event_t{} +} + +///* +// * Create a document object. +// */ +// +//YAML_DECLARE(int) +//yaml_document_initialize(document *yaml_document_t, +// version_directive *yaml_version_directive_t, +// tag_directives_start *yaml_tag_directive_t, +// tag_directives_end *yaml_tag_directive_t, +// start_implicit int, end_implicit int) +//{ +// struct { +// error yaml_error_type_t +// } context +// struct { +// start *yaml_node_t +// end *yaml_node_t +// top *yaml_node_t +// } nodes = { NULL, NULL, NULL } +// version_directive_copy *yaml_version_directive_t = NULL +// struct { +// start *yaml_tag_directive_t +// end *yaml_tag_directive_t +// top *yaml_tag_directive_t +// } tag_directives_copy = { NULL, NULL, NULL } +// value yaml_tag_directive_t = { NULL, NULL } +// mark yaml_mark_t = { 0, 0, 0 } +// +// assert(document) // Non-NULL document object is expected. +// assert((tag_directives_start && tag_directives_end) || +// (tag_directives_start == tag_directives_end)) +// // Valid tag directives are expected. +// +// if (!STACK_INIT(&context, nodes, INITIAL_STACK_SIZE)) goto error +// +// if (version_directive) { +// version_directive_copy = yaml_malloc(sizeof(yaml_version_directive_t)) +// if (!version_directive_copy) goto error +// version_directive_copy.major = version_directive.major +// version_directive_copy.minor = version_directive.minor +// } +// +// if (tag_directives_start != tag_directives_end) { +// tag_directive *yaml_tag_directive_t +// if (!STACK_INIT(&context, tag_directives_copy, INITIAL_STACK_SIZE)) +// goto error +// for (tag_directive = tag_directives_start +// tag_directive != tag_directives_end; tag_directive ++) { +// assert(tag_directive.handle) +// assert(tag_directive.prefix) +// if (!yaml_check_utf8(tag_directive.handle, +// strlen((char *)tag_directive.handle))) +// goto error +// if (!yaml_check_utf8(tag_directive.prefix, +// strlen((char *)tag_directive.prefix))) +// goto error +// value.handle = yaml_strdup(tag_directive.handle) +// value.prefix = yaml_strdup(tag_directive.prefix) +// if (!value.handle || !value.prefix) goto error +// if (!PUSH(&context, tag_directives_copy, value)) +// goto error +// value.handle = NULL +// value.prefix = NULL +// } +// } +// +// DOCUMENT_INIT(*document, nodes.start, nodes.end, version_directive_copy, +// tag_directives_copy.start, tag_directives_copy.top, +// start_implicit, end_implicit, mark, mark) +// +// return 1 +// +//error: +// STACK_DEL(&context, nodes) +// yaml_free(version_directive_copy) +// while (!STACK_EMPTY(&context, tag_directives_copy)) { +// value yaml_tag_directive_t = POP(&context, tag_directives_copy) +// yaml_free(value.handle) +// yaml_free(value.prefix) +// } +// STACK_DEL(&context, tag_directives_copy) +// yaml_free(value.handle) +// yaml_free(value.prefix) +// +// return 0 +//} +// +///* +// * Destroy a document object. +// */ +// +//YAML_DECLARE(void) +//yaml_document_delete(document *yaml_document_t) +//{ +// struct { +// error yaml_error_type_t +// } context +// tag_directive *yaml_tag_directive_t +// +// context.error = YAML_NO_ERROR // Eliminate a compiler warning. +// +// assert(document) // Non-NULL document object is expected. +// +// while (!STACK_EMPTY(&context, document.nodes)) { +// node yaml_node_t = POP(&context, document.nodes) +// yaml_free(node.tag) +// switch (node.type) { +// case YAML_SCALAR_NODE: +// yaml_free(node.data.scalar.value) +// break +// case YAML_SEQUENCE_NODE: +// STACK_DEL(&context, node.data.sequence.items) +// break +// case YAML_MAPPING_NODE: +// STACK_DEL(&context, node.data.mapping.pairs) +// break +// default: +// assert(0) // Should not happen. +// } +// } +// STACK_DEL(&context, document.nodes) +// +// yaml_free(document.version_directive) +// for (tag_directive = document.tag_directives.start +// tag_directive != document.tag_directives.end +// tag_directive++) { +// yaml_free(tag_directive.handle) +// yaml_free(tag_directive.prefix) +// } +// yaml_free(document.tag_directives.start) +// +// memset(document, 0, sizeof(yaml_document_t)) +//} +// +///** +// * Get a document node. +// */ +// +//YAML_DECLARE(yaml_node_t *) +//yaml_document_get_node(document *yaml_document_t, index int) +//{ +// assert(document) // Non-NULL document object is expected. +// +// if (index > 0 && document.nodes.start + index <= document.nodes.top) { +// return document.nodes.start + index - 1 +// } +// return NULL +//} +// +///** +// * Get the root object. +// */ +// +//YAML_DECLARE(yaml_node_t *) +//yaml_document_get_root_node(document *yaml_document_t) +//{ +// assert(document) // Non-NULL document object is expected. +// +// if (document.nodes.top != document.nodes.start) { +// return document.nodes.start +// } +// return NULL +//} +// +///* +// * Add a scalar node to a document. +// */ +// +//YAML_DECLARE(int) +//yaml_document_add_scalar(document *yaml_document_t, +// tag *yaml_char_t, value *yaml_char_t, length int, +// style yaml_scalar_style_t) +//{ +// struct { +// error yaml_error_type_t +// } context +// mark yaml_mark_t = { 0, 0, 0 } +// tag_copy *yaml_char_t = NULL +// value_copy *yaml_char_t = NULL +// node yaml_node_t +// +// assert(document) // Non-NULL document object is expected. +// assert(value) // Non-NULL value is expected. +// +// if (!tag) { +// tag = (yaml_char_t *)YAML_DEFAULT_SCALAR_TAG +// } +// +// if (!yaml_check_utf8(tag, strlen((char *)tag))) goto error +// tag_copy = yaml_strdup(tag) +// if (!tag_copy) goto error +// +// if (length < 0) { +// length = strlen((char *)value) +// } +// +// if (!yaml_check_utf8(value, length)) goto error +// value_copy = yaml_malloc(length+1) +// if (!value_copy) goto error +// memcpy(value_copy, value, length) +// value_copy[length] = '\0' +// +// SCALAR_NODE_INIT(node, tag_copy, value_copy, length, style, mark, mark) +// if (!PUSH(&context, document.nodes, node)) goto error +// +// return document.nodes.top - document.nodes.start +// +//error: +// yaml_free(tag_copy) +// yaml_free(value_copy) +// +// return 0 +//} +// +///* +// * Add a sequence node to a document. +// */ +// +//YAML_DECLARE(int) +//yaml_document_add_sequence(document *yaml_document_t, +// tag *yaml_char_t, style yaml_sequence_style_t) +//{ +// struct { +// error yaml_error_type_t +// } context +// mark yaml_mark_t = { 0, 0, 0 } +// tag_copy *yaml_char_t = NULL +// struct { +// start *yaml_node_item_t +// end *yaml_node_item_t +// top *yaml_node_item_t +// } items = { NULL, NULL, NULL } +// node yaml_node_t +// +// assert(document) // Non-NULL document object is expected. +// +// if (!tag) { +// tag = (yaml_char_t *)YAML_DEFAULT_SEQUENCE_TAG +// } +// +// if (!yaml_check_utf8(tag, strlen((char *)tag))) goto error +// tag_copy = yaml_strdup(tag) +// if (!tag_copy) goto error +// +// if (!STACK_INIT(&context, items, INITIAL_STACK_SIZE)) goto error +// +// SEQUENCE_NODE_INIT(node, tag_copy, items.start, items.end, +// style, mark, mark) +// if (!PUSH(&context, document.nodes, node)) goto error +// +// return document.nodes.top - document.nodes.start +// +//error: +// STACK_DEL(&context, items) +// yaml_free(tag_copy) +// +// return 0 +//} +// +///* +// * Add a mapping node to a document. +// */ +// +//YAML_DECLARE(int) +//yaml_document_add_mapping(document *yaml_document_t, +// tag *yaml_char_t, style yaml_mapping_style_t) +//{ +// struct { +// error yaml_error_type_t +// } context +// mark yaml_mark_t = { 0, 0, 0 } +// tag_copy *yaml_char_t = NULL +// struct { +// start *yaml_node_pair_t +// end *yaml_node_pair_t +// top *yaml_node_pair_t +// } pairs = { NULL, NULL, NULL } +// node yaml_node_t +// +// assert(document) // Non-NULL document object is expected. +// +// if (!tag) { +// tag = (yaml_char_t *)YAML_DEFAULT_MAPPING_TAG +// } +// +// if (!yaml_check_utf8(tag, strlen((char *)tag))) goto error +// tag_copy = yaml_strdup(tag) +// if (!tag_copy) goto error +// +// if (!STACK_INIT(&context, pairs, INITIAL_STACK_SIZE)) goto error +// +// MAPPING_NODE_INIT(node, tag_copy, pairs.start, pairs.end, +// style, mark, mark) +// if (!PUSH(&context, document.nodes, node)) goto error +// +// return document.nodes.top - document.nodes.start +// +//error: +// STACK_DEL(&context, pairs) +// yaml_free(tag_copy) +// +// return 0 +//} +// +///* +// * Append an item to a sequence node. +// */ +// +//YAML_DECLARE(int) +//yaml_document_append_sequence_item(document *yaml_document_t, +// sequence int, item int) +//{ +// struct { +// error yaml_error_type_t +// } context +// +// assert(document) // Non-NULL document is required. +// assert(sequence > 0 +// && document.nodes.start + sequence <= document.nodes.top) +// // Valid sequence id is required. +// assert(document.nodes.start[sequence-1].type == YAML_SEQUENCE_NODE) +// // A sequence node is required. +// assert(item > 0 && document.nodes.start + item <= document.nodes.top) +// // Valid item id is required. +// +// if (!PUSH(&context, +// document.nodes.start[sequence-1].data.sequence.items, item)) +// return 0 +// +// return 1 +//} +// +///* +// * Append a pair of a key and a value to a mapping node. +// */ +// +//YAML_DECLARE(int) +//yaml_document_append_mapping_pair(document *yaml_document_t, +// mapping int, key int, value int) +//{ +// struct { +// error yaml_error_type_t +// } context +// +// pair yaml_node_pair_t +// +// assert(document) // Non-NULL document is required. +// assert(mapping > 0 +// && document.nodes.start + mapping <= document.nodes.top) +// // Valid mapping id is required. +// assert(document.nodes.start[mapping-1].type == YAML_MAPPING_NODE) +// // A mapping node is required. +// assert(key > 0 && document.nodes.start + key <= document.nodes.top) +// // Valid key id is required. +// assert(value > 0 && document.nodes.start + value <= document.nodes.top) +// // Valid value id is required. +// +// pair.key = key +// pair.value = value +// +// if (!PUSH(&context, +// document.nodes.start[mapping-1].data.mapping.pairs, pair)) +// return 0 +// +// return 1 +//} +// +// diff --git a/tools/voluspa/vendor/gopkg.in/yaml.v2/decode.go b/tools/voluspa/vendor/gopkg.in/yaml.v2/decode.go new file mode 100644 index 00000000000..e4e56e28e0e --- /dev/null +++ b/tools/voluspa/vendor/gopkg.in/yaml.v2/decode.go @@ -0,0 +1,775 @@ +package yaml + +import ( + "encoding" + "encoding/base64" + "fmt" + "io" + "math" + "reflect" + "strconv" + "time" +) + +const ( + documentNode = 1 << iota + mappingNode + sequenceNode + scalarNode + aliasNode +) + +type node struct { + kind int + line, column int + tag string + // For an alias node, alias holds the resolved alias. + alias *node + value string + implicit bool + children []*node + anchors map[string]*node +} + +// ---------------------------------------------------------------------------- +// Parser, produces a node tree out of a libyaml event stream. + +type parser struct { + parser yaml_parser_t + event yaml_event_t + doc *node + doneInit bool +} + +func newParser(b []byte) *parser { + p := parser{} + if !yaml_parser_initialize(&p.parser) { + panic("failed to initialize YAML emitter") + } + if len(b) == 0 { + b = []byte{'\n'} + } + yaml_parser_set_input_string(&p.parser, b) + return &p +} + +func newParserFromReader(r io.Reader) *parser { + p := parser{} + if !yaml_parser_initialize(&p.parser) { + panic("failed to initialize YAML emitter") + } + yaml_parser_set_input_reader(&p.parser, r) + return &p +} + +func (p *parser) init() { + if p.doneInit { + return + } + p.expect(yaml_STREAM_START_EVENT) + p.doneInit = true +} + +func (p *parser) destroy() { + if p.event.typ != yaml_NO_EVENT { + yaml_event_delete(&p.event) + } + yaml_parser_delete(&p.parser) +} + +// expect consumes an event from the event stream and +// checks that it's of the expected type. +func (p *parser) expect(e yaml_event_type_t) { + if p.event.typ == yaml_NO_EVENT { + if !yaml_parser_parse(&p.parser, &p.event) { + p.fail() + } + } + if p.event.typ == yaml_STREAM_END_EVENT { + failf("attempted to go past the end of stream; corrupted value?") + } + if p.event.typ != e { + p.parser.problem = fmt.Sprintf("expected %s event but got %s", e, p.event.typ) + p.fail() + } + yaml_event_delete(&p.event) + p.event.typ = yaml_NO_EVENT +} + +// peek peeks at the next event in the event stream, +// puts the results into p.event and returns the event type. +func (p *parser) peek() yaml_event_type_t { + if p.event.typ != yaml_NO_EVENT { + return p.event.typ + } + if !yaml_parser_parse(&p.parser, &p.event) { + p.fail() + } + return p.event.typ +} + +func (p *parser) fail() { + var where string + var line int + if p.parser.problem_mark.line != 0 { + line = p.parser.problem_mark.line + // Scanner errors don't iterate line before returning error + if p.parser.error == yaml_SCANNER_ERROR { + line++ + } + } else if p.parser.context_mark.line != 0 { + line = p.parser.context_mark.line + } + if line != 0 { + where = "line " + strconv.Itoa(line) + ": " + } + var msg string + if len(p.parser.problem) > 0 { + msg = p.parser.problem + } else { + msg = "unknown problem parsing YAML content" + } + failf("%s%s", where, msg) +} + +func (p *parser) anchor(n *node, anchor []byte) { + if anchor != nil { + p.doc.anchors[string(anchor)] = n + } +} + +func (p *parser) parse() *node { + p.init() + switch p.peek() { + case yaml_SCALAR_EVENT: + return p.scalar() + case yaml_ALIAS_EVENT: + return p.alias() + case yaml_MAPPING_START_EVENT: + return p.mapping() + case yaml_SEQUENCE_START_EVENT: + return p.sequence() + case yaml_DOCUMENT_START_EVENT: + return p.document() + case yaml_STREAM_END_EVENT: + // Happens when attempting to decode an empty buffer. + return nil + default: + panic("attempted to parse unknown event: " + p.event.typ.String()) + } +} + +func (p *parser) node(kind int) *node { + return &node{ + kind: kind, + line: p.event.start_mark.line, + column: p.event.start_mark.column, + } +} + +func (p *parser) document() *node { + n := p.node(documentNode) + n.anchors = make(map[string]*node) + p.doc = n + p.expect(yaml_DOCUMENT_START_EVENT) + n.children = append(n.children, p.parse()) + p.expect(yaml_DOCUMENT_END_EVENT) + return n +} + +func (p *parser) alias() *node { + n := p.node(aliasNode) + n.value = string(p.event.anchor) + n.alias = p.doc.anchors[n.value] + if n.alias == nil { + failf("unknown anchor '%s' referenced", n.value) + } + p.expect(yaml_ALIAS_EVENT) + return n +} + +func (p *parser) scalar() *node { + n := p.node(scalarNode) + n.value = string(p.event.value) + n.tag = string(p.event.tag) + n.implicit = p.event.implicit + p.anchor(n, p.event.anchor) + p.expect(yaml_SCALAR_EVENT) + return n +} + +func (p *parser) sequence() *node { + n := p.node(sequenceNode) + p.anchor(n, p.event.anchor) + p.expect(yaml_SEQUENCE_START_EVENT) + for p.peek() != yaml_SEQUENCE_END_EVENT { + n.children = append(n.children, p.parse()) + } + p.expect(yaml_SEQUENCE_END_EVENT) + return n +} + +func (p *parser) mapping() *node { + n := p.node(mappingNode) + p.anchor(n, p.event.anchor) + p.expect(yaml_MAPPING_START_EVENT) + for p.peek() != yaml_MAPPING_END_EVENT { + n.children = append(n.children, p.parse(), p.parse()) + } + p.expect(yaml_MAPPING_END_EVENT) + return n +} + +// ---------------------------------------------------------------------------- +// Decoder, unmarshals a node into a provided value. + +type decoder struct { + doc *node + aliases map[*node]bool + mapType reflect.Type + terrors []string + strict bool +} + +var ( + mapItemType = reflect.TypeOf(MapItem{}) + durationType = reflect.TypeOf(time.Duration(0)) + defaultMapType = reflect.TypeOf(map[interface{}]interface{}{}) + ifaceType = defaultMapType.Elem() + timeType = reflect.TypeOf(time.Time{}) + ptrTimeType = reflect.TypeOf(&time.Time{}) +) + +func newDecoder(strict bool) *decoder { + d := &decoder{mapType: defaultMapType, strict: strict} + d.aliases = make(map[*node]bool) + return d +} + +func (d *decoder) terror(n *node, tag string, out reflect.Value) { + if n.tag != "" { + tag = n.tag + } + value := n.value + if tag != yaml_SEQ_TAG && tag != yaml_MAP_TAG { + if len(value) > 10 { + value = " `" + value[:7] + "...`" + } else { + value = " `" + value + "`" + } + } + d.terrors = append(d.terrors, fmt.Sprintf("line %d: cannot unmarshal %s%s into %s", n.line+1, shortTag(tag), value, out.Type())) +} + +func (d *decoder) callUnmarshaler(n *node, u Unmarshaler) (good bool) { + terrlen := len(d.terrors) + err := u.UnmarshalYAML(func(v interface{}) (err error) { + defer handleErr(&err) + d.unmarshal(n, reflect.ValueOf(v)) + if len(d.terrors) > terrlen { + issues := d.terrors[terrlen:] + d.terrors = d.terrors[:terrlen] + return &TypeError{issues} + } + return nil + }) + if e, ok := err.(*TypeError); ok { + d.terrors = append(d.terrors, e.Errors...) + return false + } + if err != nil { + fail(err) + } + return true +} + +// d.prepare initializes and dereferences pointers and calls UnmarshalYAML +// if a value is found to implement it. +// It returns the initialized and dereferenced out value, whether +// unmarshalling was already done by UnmarshalYAML, and if so whether +// its types unmarshalled appropriately. +// +// If n holds a null value, prepare returns before doing anything. +func (d *decoder) prepare(n *node, out reflect.Value) (newout reflect.Value, unmarshaled, good bool) { + if n.tag == yaml_NULL_TAG || n.kind == scalarNode && n.tag == "" && (n.value == "null" || n.value == "~" || n.value == "" && n.implicit) { + return out, false, false + } + again := true + for again { + again = false + if out.Kind() == reflect.Ptr { + if out.IsNil() { + out.Set(reflect.New(out.Type().Elem())) + } + out = out.Elem() + again = true + } + if out.CanAddr() { + if u, ok := out.Addr().Interface().(Unmarshaler); ok { + good = d.callUnmarshaler(n, u) + return out, true, good + } + } + } + return out, false, false +} + +func (d *decoder) unmarshal(n *node, out reflect.Value) (good bool) { + switch n.kind { + case documentNode: + return d.document(n, out) + case aliasNode: + return d.alias(n, out) + } + out, unmarshaled, good := d.prepare(n, out) + if unmarshaled { + return good + } + switch n.kind { + case scalarNode: + good = d.scalar(n, out) + case mappingNode: + good = d.mapping(n, out) + case sequenceNode: + good = d.sequence(n, out) + default: + panic("internal error: unknown node kind: " + strconv.Itoa(n.kind)) + } + return good +} + +func (d *decoder) document(n *node, out reflect.Value) (good bool) { + if len(n.children) == 1 { + d.doc = n + d.unmarshal(n.children[0], out) + return true + } + return false +} + +func (d *decoder) alias(n *node, out reflect.Value) (good bool) { + if d.aliases[n] { + // TODO this could actually be allowed in some circumstances. + failf("anchor '%s' value contains itself", n.value) + } + d.aliases[n] = true + good = d.unmarshal(n.alias, out) + delete(d.aliases, n) + return good +} + +var zeroValue reflect.Value + +func resetMap(out reflect.Value) { + for _, k := range out.MapKeys() { + out.SetMapIndex(k, zeroValue) + } +} + +func (d *decoder) scalar(n *node, out reflect.Value) bool { + var tag string + var resolved interface{} + if n.tag == "" && !n.implicit { + tag = yaml_STR_TAG + resolved = n.value + } else { + tag, resolved = resolve(n.tag, n.value) + if tag == yaml_BINARY_TAG { + data, err := base64.StdEncoding.DecodeString(resolved.(string)) + if err != nil { + failf("!!binary value contains invalid base64 data") + } + resolved = string(data) + } + } + if resolved == nil { + if out.Kind() == reflect.Map && !out.CanAddr() { + resetMap(out) + } else { + out.Set(reflect.Zero(out.Type())) + } + return true + } + if resolvedv := reflect.ValueOf(resolved); out.Type() == resolvedv.Type() { + // We've resolved to exactly the type we want, so use that. + out.Set(resolvedv) + return true + } + // Perhaps we can use the value as a TextUnmarshaler to + // set its value. + if out.CanAddr() { + u, ok := out.Addr().Interface().(encoding.TextUnmarshaler) + if ok { + var text []byte + if tag == yaml_BINARY_TAG { + text = []byte(resolved.(string)) + } else { + // We let any value be unmarshaled into TextUnmarshaler. + // That might be more lax than we'd like, but the + // TextUnmarshaler itself should bowl out any dubious values. + text = []byte(n.value) + } + err := u.UnmarshalText(text) + if err != nil { + fail(err) + } + return true + } + } + switch out.Kind() { + case reflect.String: + if tag == yaml_BINARY_TAG { + out.SetString(resolved.(string)) + return true + } + if resolved != nil { + out.SetString(n.value) + return true + } + case reflect.Interface: + if resolved == nil { + out.Set(reflect.Zero(out.Type())) + } else if tag == yaml_TIMESTAMP_TAG { + // It looks like a timestamp but for backward compatibility + // reasons we set it as a string, so that code that unmarshals + // timestamp-like values into interface{} will continue to + // see a string and not a time.Time. + // TODO(v3) Drop this. + out.Set(reflect.ValueOf(n.value)) + } else { + out.Set(reflect.ValueOf(resolved)) + } + return true + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + switch resolved := resolved.(type) { + case int: + if !out.OverflowInt(int64(resolved)) { + out.SetInt(int64(resolved)) + return true + } + case int64: + if !out.OverflowInt(resolved) { + out.SetInt(resolved) + return true + } + case uint64: + if resolved <= math.MaxInt64 && !out.OverflowInt(int64(resolved)) { + out.SetInt(int64(resolved)) + return true + } + case float64: + if resolved <= math.MaxInt64 && !out.OverflowInt(int64(resolved)) { + out.SetInt(int64(resolved)) + return true + } + case string: + if out.Type() == durationType { + d, err := time.ParseDuration(resolved) + if err == nil { + out.SetInt(int64(d)) + return true + } + } + } + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: + switch resolved := resolved.(type) { + case int: + if resolved >= 0 && !out.OverflowUint(uint64(resolved)) { + out.SetUint(uint64(resolved)) + return true + } + case int64: + if resolved >= 0 && !out.OverflowUint(uint64(resolved)) { + out.SetUint(uint64(resolved)) + return true + } + case uint64: + if !out.OverflowUint(uint64(resolved)) { + out.SetUint(uint64(resolved)) + return true + } + case float64: + if resolved <= math.MaxUint64 && !out.OverflowUint(uint64(resolved)) { + out.SetUint(uint64(resolved)) + return true + } + } + case reflect.Bool: + switch resolved := resolved.(type) { + case bool: + out.SetBool(resolved) + return true + } + case reflect.Float32, reflect.Float64: + switch resolved := resolved.(type) { + case int: + out.SetFloat(float64(resolved)) + return true + case int64: + out.SetFloat(float64(resolved)) + return true + case uint64: + out.SetFloat(float64(resolved)) + return true + case float64: + out.SetFloat(resolved) + return true + } + case reflect.Struct: + if resolvedv := reflect.ValueOf(resolved); out.Type() == resolvedv.Type() { + out.Set(resolvedv) + return true + } + case reflect.Ptr: + if out.Type().Elem() == reflect.TypeOf(resolved) { + // TODO DOes this make sense? When is out a Ptr except when decoding a nil value? + elem := reflect.New(out.Type().Elem()) + elem.Elem().Set(reflect.ValueOf(resolved)) + out.Set(elem) + return true + } + } + d.terror(n, tag, out) + return false +} + +func settableValueOf(i interface{}) reflect.Value { + v := reflect.ValueOf(i) + sv := reflect.New(v.Type()).Elem() + sv.Set(v) + return sv +} + +func (d *decoder) sequence(n *node, out reflect.Value) (good bool) { + l := len(n.children) + + var iface reflect.Value + switch out.Kind() { + case reflect.Slice: + out.Set(reflect.MakeSlice(out.Type(), l, l)) + case reflect.Array: + if l != out.Len() { + failf("invalid array: want %d elements but got %d", out.Len(), l) + } + case reflect.Interface: + // No type hints. Will have to use a generic sequence. + iface = out + out = settableValueOf(make([]interface{}, l)) + default: + d.terror(n, yaml_SEQ_TAG, out) + return false + } + et := out.Type().Elem() + + j := 0 + for i := 0; i < l; i++ { + e := reflect.New(et).Elem() + if ok := d.unmarshal(n.children[i], e); ok { + out.Index(j).Set(e) + j++ + } + } + if out.Kind() != reflect.Array { + out.Set(out.Slice(0, j)) + } + if iface.IsValid() { + iface.Set(out) + } + return true +} + +func (d *decoder) mapping(n *node, out reflect.Value) (good bool) { + switch out.Kind() { + case reflect.Struct: + return d.mappingStruct(n, out) + case reflect.Slice: + return d.mappingSlice(n, out) + case reflect.Map: + // okay + case reflect.Interface: + if d.mapType.Kind() == reflect.Map { + iface := out + out = reflect.MakeMap(d.mapType) + iface.Set(out) + } else { + slicev := reflect.New(d.mapType).Elem() + if !d.mappingSlice(n, slicev) { + return false + } + out.Set(slicev) + return true + } + default: + d.terror(n, yaml_MAP_TAG, out) + return false + } + outt := out.Type() + kt := outt.Key() + et := outt.Elem() + + mapType := d.mapType + if outt.Key() == ifaceType && outt.Elem() == ifaceType { + d.mapType = outt + } + + if out.IsNil() { + out.Set(reflect.MakeMap(outt)) + } + l := len(n.children) + for i := 0; i < l; i += 2 { + if isMerge(n.children[i]) { + d.merge(n.children[i+1], out) + continue + } + k := reflect.New(kt).Elem() + if d.unmarshal(n.children[i], k) { + kkind := k.Kind() + if kkind == reflect.Interface { + kkind = k.Elem().Kind() + } + if kkind == reflect.Map || kkind == reflect.Slice { + failf("invalid map key: %#v", k.Interface()) + } + e := reflect.New(et).Elem() + if d.unmarshal(n.children[i+1], e) { + d.setMapIndex(n.children[i+1], out, k, e) + } + } + } + d.mapType = mapType + return true +} + +func (d *decoder) setMapIndex(n *node, out, k, v reflect.Value) { + if d.strict && out.MapIndex(k) != zeroValue { + d.terrors = append(d.terrors, fmt.Sprintf("line %d: key %#v already set in map", n.line+1, k.Interface())) + return + } + out.SetMapIndex(k, v) +} + +func (d *decoder) mappingSlice(n *node, out reflect.Value) (good bool) { + outt := out.Type() + if outt.Elem() != mapItemType { + d.terror(n, yaml_MAP_TAG, out) + return false + } + + mapType := d.mapType + d.mapType = outt + + var slice []MapItem + var l = len(n.children) + for i := 0; i < l; i += 2 { + if isMerge(n.children[i]) { + d.merge(n.children[i+1], out) + continue + } + item := MapItem{} + k := reflect.ValueOf(&item.Key).Elem() + if d.unmarshal(n.children[i], k) { + v := reflect.ValueOf(&item.Value).Elem() + if d.unmarshal(n.children[i+1], v) { + slice = append(slice, item) + } + } + } + out.Set(reflect.ValueOf(slice)) + d.mapType = mapType + return true +} + +func (d *decoder) mappingStruct(n *node, out reflect.Value) (good bool) { + sinfo, err := getStructInfo(out.Type()) + if err != nil { + panic(err) + } + name := settableValueOf("") + l := len(n.children) + + var inlineMap reflect.Value + var elemType reflect.Type + if sinfo.InlineMap != -1 { + inlineMap = out.Field(sinfo.InlineMap) + inlineMap.Set(reflect.New(inlineMap.Type()).Elem()) + elemType = inlineMap.Type().Elem() + } + + var doneFields []bool + if d.strict { + doneFields = make([]bool, len(sinfo.FieldsList)) + } + for i := 0; i < l; i += 2 { + ni := n.children[i] + if isMerge(ni) { + d.merge(n.children[i+1], out) + continue + } + if !d.unmarshal(ni, name) { + continue + } + if info, ok := sinfo.FieldsMap[name.String()]; ok { + if d.strict { + if doneFields[info.Id] { + d.terrors = append(d.terrors, fmt.Sprintf("line %d: field %s already set in type %s", ni.line+1, name.String(), out.Type())) + continue + } + doneFields[info.Id] = true + } + var field reflect.Value + if info.Inline == nil { + field = out.Field(info.Num) + } else { + field = out.FieldByIndex(info.Inline) + } + d.unmarshal(n.children[i+1], field) + } else if sinfo.InlineMap != -1 { + if inlineMap.IsNil() { + inlineMap.Set(reflect.MakeMap(inlineMap.Type())) + } + value := reflect.New(elemType).Elem() + d.unmarshal(n.children[i+1], value) + d.setMapIndex(n.children[i+1], inlineMap, name, value) + } else if d.strict { + d.terrors = append(d.terrors, fmt.Sprintf("line %d: field %s not found in type %s", ni.line+1, name.String(), out.Type())) + } + } + return true +} + +func failWantMap() { + failf("map merge requires map or sequence of maps as the value") +} + +func (d *decoder) merge(n *node, out reflect.Value) { + switch n.kind { + case mappingNode: + d.unmarshal(n, out) + case aliasNode: + an, ok := d.doc.anchors[n.value] + if ok && an.kind != mappingNode { + failWantMap() + } + d.unmarshal(n, out) + case sequenceNode: + // Step backwards as earlier nodes take precedence. + for i := len(n.children) - 1; i >= 0; i-- { + ni := n.children[i] + if ni.kind == aliasNode { + an, ok := d.doc.anchors[ni.value] + if ok && an.kind != mappingNode { + failWantMap() + } + } else if ni.kind != mappingNode { + failWantMap() + } + d.unmarshal(ni, out) + } + default: + failWantMap() + } +} + +func isMerge(n *node) bool { + return n.kind == scalarNode && n.value == "<<" && (n.implicit == true || n.tag == yaml_MERGE_TAG) +} diff --git a/tools/voluspa/vendor/gopkg.in/yaml.v2/emitterc.go b/tools/voluspa/vendor/gopkg.in/yaml.v2/emitterc.go new file mode 100644 index 00000000000..a1c2cc52627 --- /dev/null +++ b/tools/voluspa/vendor/gopkg.in/yaml.v2/emitterc.go @@ -0,0 +1,1685 @@ +package yaml + +import ( + "bytes" + "fmt" +) + +// Flush the buffer if needed. +func flush(emitter *yaml_emitter_t) bool { + if emitter.buffer_pos+5 >= len(emitter.buffer) { + return yaml_emitter_flush(emitter) + } + return true +} + +// Put a character to the output buffer. +func put(emitter *yaml_emitter_t, value byte) bool { + if emitter.buffer_pos+5 >= len(emitter.buffer) && !yaml_emitter_flush(emitter) { + return false + } + emitter.buffer[emitter.buffer_pos] = value + emitter.buffer_pos++ + emitter.column++ + return true +} + +// Put a line break to the output buffer. +func put_break(emitter *yaml_emitter_t) bool { + if emitter.buffer_pos+5 >= len(emitter.buffer) && !yaml_emitter_flush(emitter) { + return false + } + switch emitter.line_break { + case yaml_CR_BREAK: + emitter.buffer[emitter.buffer_pos] = '\r' + emitter.buffer_pos += 1 + case yaml_LN_BREAK: + emitter.buffer[emitter.buffer_pos] = '\n' + emitter.buffer_pos += 1 + case yaml_CRLN_BREAK: + emitter.buffer[emitter.buffer_pos+0] = '\r' + emitter.buffer[emitter.buffer_pos+1] = '\n' + emitter.buffer_pos += 2 + default: + panic("unknown line break setting") + } + emitter.column = 0 + emitter.line++ + return true +} + +// Copy a character from a string into buffer. +func write(emitter *yaml_emitter_t, s []byte, i *int) bool { + if emitter.buffer_pos+5 >= len(emitter.buffer) && !yaml_emitter_flush(emitter) { + return false + } + p := emitter.buffer_pos + w := width(s[*i]) + switch w { + case 4: + emitter.buffer[p+3] = s[*i+3] + fallthrough + case 3: + emitter.buffer[p+2] = s[*i+2] + fallthrough + case 2: + emitter.buffer[p+1] = s[*i+1] + fallthrough + case 1: + emitter.buffer[p+0] = s[*i+0] + default: + panic("unknown character width") + } + emitter.column++ + emitter.buffer_pos += w + *i += w + return true +} + +// Write a whole string into buffer. +func write_all(emitter *yaml_emitter_t, s []byte) bool { + for i := 0; i < len(s); { + if !write(emitter, s, &i) { + return false + } + } + return true +} + +// Copy a line break character from a string into buffer. +func write_break(emitter *yaml_emitter_t, s []byte, i *int) bool { + if s[*i] == '\n' { + if !put_break(emitter) { + return false + } + *i++ + } else { + if !write(emitter, s, i) { + return false + } + emitter.column = 0 + emitter.line++ + } + return true +} + +// Set an emitter error and return false. +func yaml_emitter_set_emitter_error(emitter *yaml_emitter_t, problem string) bool { + emitter.error = yaml_EMITTER_ERROR + emitter.problem = problem + return false +} + +// Emit an event. +func yaml_emitter_emit(emitter *yaml_emitter_t, event *yaml_event_t) bool { + emitter.events = append(emitter.events, *event) + for !yaml_emitter_need_more_events(emitter) { + event := &emitter.events[emitter.events_head] + if !yaml_emitter_analyze_event(emitter, event) { + return false + } + if !yaml_emitter_state_machine(emitter, event) { + return false + } + yaml_event_delete(event) + emitter.events_head++ + } + return true +} + +// Check if we need to accumulate more events before emitting. +// +// We accumulate extra +// - 1 event for DOCUMENT-START +// - 2 events for SEQUENCE-START +// - 3 events for MAPPING-START +// +func yaml_emitter_need_more_events(emitter *yaml_emitter_t) bool { + if emitter.events_head == len(emitter.events) { + return true + } + var accumulate int + switch emitter.events[emitter.events_head].typ { + case yaml_DOCUMENT_START_EVENT: + accumulate = 1 + break + case yaml_SEQUENCE_START_EVENT: + accumulate = 2 + break + case yaml_MAPPING_START_EVENT: + accumulate = 3 + break + default: + return false + } + if len(emitter.events)-emitter.events_head > accumulate { + return false + } + var level int + for i := emitter.events_head; i < len(emitter.events); i++ { + switch emitter.events[i].typ { + case yaml_STREAM_START_EVENT, yaml_DOCUMENT_START_EVENT, yaml_SEQUENCE_START_EVENT, yaml_MAPPING_START_EVENT: + level++ + case yaml_STREAM_END_EVENT, yaml_DOCUMENT_END_EVENT, yaml_SEQUENCE_END_EVENT, yaml_MAPPING_END_EVENT: + level-- + } + if level == 0 { + return false + } + } + return true +} + +// Append a directive to the directives stack. +func yaml_emitter_append_tag_directive(emitter *yaml_emitter_t, value *yaml_tag_directive_t, allow_duplicates bool) bool { + for i := 0; i < len(emitter.tag_directives); i++ { + if bytes.Equal(value.handle, emitter.tag_directives[i].handle) { + if allow_duplicates { + return true + } + return yaml_emitter_set_emitter_error(emitter, "duplicate %TAG directive") + } + } + + // [Go] Do we actually need to copy this given garbage collection + // and the lack of deallocating destructors? + tag_copy := yaml_tag_directive_t{ + handle: make([]byte, len(value.handle)), + prefix: make([]byte, len(value.prefix)), + } + copy(tag_copy.handle, value.handle) + copy(tag_copy.prefix, value.prefix) + emitter.tag_directives = append(emitter.tag_directives, tag_copy) + return true +} + +// Increase the indentation level. +func yaml_emitter_increase_indent(emitter *yaml_emitter_t, flow, indentless bool) bool { + emitter.indents = append(emitter.indents, emitter.indent) + if emitter.indent < 0 { + if flow { + emitter.indent = emitter.best_indent + } else { + emitter.indent = 0 + } + } else if !indentless { + emitter.indent += emitter.best_indent + } + return true +} + +// State dispatcher. +func yaml_emitter_state_machine(emitter *yaml_emitter_t, event *yaml_event_t) bool { + switch emitter.state { + default: + case yaml_EMIT_STREAM_START_STATE: + return yaml_emitter_emit_stream_start(emitter, event) + + case yaml_EMIT_FIRST_DOCUMENT_START_STATE: + return yaml_emitter_emit_document_start(emitter, event, true) + + case yaml_EMIT_DOCUMENT_START_STATE: + return yaml_emitter_emit_document_start(emitter, event, false) + + case yaml_EMIT_DOCUMENT_CONTENT_STATE: + return yaml_emitter_emit_document_content(emitter, event) + + case yaml_EMIT_DOCUMENT_END_STATE: + return yaml_emitter_emit_document_end(emitter, event) + + case yaml_EMIT_FLOW_SEQUENCE_FIRST_ITEM_STATE: + return yaml_emitter_emit_flow_sequence_item(emitter, event, true) + + case yaml_EMIT_FLOW_SEQUENCE_ITEM_STATE: + return yaml_emitter_emit_flow_sequence_item(emitter, event, false) + + case yaml_EMIT_FLOW_MAPPING_FIRST_KEY_STATE: + return yaml_emitter_emit_flow_mapping_key(emitter, event, true) + + case yaml_EMIT_FLOW_MAPPING_KEY_STATE: + return yaml_emitter_emit_flow_mapping_key(emitter, event, false) + + case yaml_EMIT_FLOW_MAPPING_SIMPLE_VALUE_STATE: + return yaml_emitter_emit_flow_mapping_value(emitter, event, true) + + case yaml_EMIT_FLOW_MAPPING_VALUE_STATE: + return yaml_emitter_emit_flow_mapping_value(emitter, event, false) + + case yaml_EMIT_BLOCK_SEQUENCE_FIRST_ITEM_STATE: + return yaml_emitter_emit_block_sequence_item(emitter, event, true) + + case yaml_EMIT_BLOCK_SEQUENCE_ITEM_STATE: + return yaml_emitter_emit_block_sequence_item(emitter, event, false) + + case yaml_EMIT_BLOCK_MAPPING_FIRST_KEY_STATE: + return yaml_emitter_emit_block_mapping_key(emitter, event, true) + + case yaml_EMIT_BLOCK_MAPPING_KEY_STATE: + return yaml_emitter_emit_block_mapping_key(emitter, event, false) + + case yaml_EMIT_BLOCK_MAPPING_SIMPLE_VALUE_STATE: + return yaml_emitter_emit_block_mapping_value(emitter, event, true) + + case yaml_EMIT_BLOCK_MAPPING_VALUE_STATE: + return yaml_emitter_emit_block_mapping_value(emitter, event, false) + + case yaml_EMIT_END_STATE: + return yaml_emitter_set_emitter_error(emitter, "expected nothing after STREAM-END") + } + panic("invalid emitter state") +} + +// Expect STREAM-START. +func yaml_emitter_emit_stream_start(emitter *yaml_emitter_t, event *yaml_event_t) bool { + if event.typ != yaml_STREAM_START_EVENT { + return yaml_emitter_set_emitter_error(emitter, "expected STREAM-START") + } + if emitter.encoding == yaml_ANY_ENCODING { + emitter.encoding = event.encoding + if emitter.encoding == yaml_ANY_ENCODING { + emitter.encoding = yaml_UTF8_ENCODING + } + } + if emitter.best_indent < 2 || emitter.best_indent > 9 { + emitter.best_indent = 2 + } + if emitter.best_width >= 0 && emitter.best_width <= emitter.best_indent*2 { + emitter.best_width = 80 + } + if emitter.best_width < 0 { + emitter.best_width = 1<<31 - 1 + } + if emitter.line_break == yaml_ANY_BREAK { + emitter.line_break = yaml_LN_BREAK + } + + emitter.indent = -1 + emitter.line = 0 + emitter.column = 0 + emitter.whitespace = true + emitter.indention = true + + if emitter.encoding != yaml_UTF8_ENCODING { + if !yaml_emitter_write_bom(emitter) { + return false + } + } + emitter.state = yaml_EMIT_FIRST_DOCUMENT_START_STATE + return true +} + +// Expect DOCUMENT-START or STREAM-END. +func yaml_emitter_emit_document_start(emitter *yaml_emitter_t, event *yaml_event_t, first bool) bool { + + if event.typ == yaml_DOCUMENT_START_EVENT { + + if event.version_directive != nil { + if !yaml_emitter_analyze_version_directive(emitter, event.version_directive) { + return false + } + } + + for i := 0; i < len(event.tag_directives); i++ { + tag_directive := &event.tag_directives[i] + if !yaml_emitter_analyze_tag_directive(emitter, tag_directive) { + return false + } + if !yaml_emitter_append_tag_directive(emitter, tag_directive, false) { + return false + } + } + + for i := 0; i < len(default_tag_directives); i++ { + tag_directive := &default_tag_directives[i] + if !yaml_emitter_append_tag_directive(emitter, tag_directive, true) { + return false + } + } + + implicit := event.implicit + if !first || emitter.canonical { + implicit = false + } + + if emitter.open_ended && (event.version_directive != nil || len(event.tag_directives) > 0) { + if !yaml_emitter_write_indicator(emitter, []byte("..."), true, false, false) { + return false + } + if !yaml_emitter_write_indent(emitter) { + return false + } + } + + if event.version_directive != nil { + implicit = false + if !yaml_emitter_write_indicator(emitter, []byte("%YAML"), true, false, false) { + return false + } + if !yaml_emitter_write_indicator(emitter, []byte("1.1"), true, false, false) { + return false + } + if !yaml_emitter_write_indent(emitter) { + return false + } + } + + if len(event.tag_directives) > 0 { + implicit = false + for i := 0; i < len(event.tag_directives); i++ { + tag_directive := &event.tag_directives[i] + if !yaml_emitter_write_indicator(emitter, []byte("%TAG"), true, false, false) { + return false + } + if !yaml_emitter_write_tag_handle(emitter, tag_directive.handle) { + return false + } + if !yaml_emitter_write_tag_content(emitter, tag_directive.prefix, true) { + return false + } + if !yaml_emitter_write_indent(emitter) { + return false + } + } + } + + if yaml_emitter_check_empty_document(emitter) { + implicit = false + } + if !implicit { + if !yaml_emitter_write_indent(emitter) { + return false + } + if !yaml_emitter_write_indicator(emitter, []byte("---"), true, false, false) { + return false + } + if emitter.canonical { + if !yaml_emitter_write_indent(emitter) { + return false + } + } + } + + emitter.state = yaml_EMIT_DOCUMENT_CONTENT_STATE + return true + } + + if event.typ == yaml_STREAM_END_EVENT { + if emitter.open_ended { + if !yaml_emitter_write_indicator(emitter, []byte("..."), true, false, false) { + return false + } + if !yaml_emitter_write_indent(emitter) { + return false + } + } + if !yaml_emitter_flush(emitter) { + return false + } + emitter.state = yaml_EMIT_END_STATE + return true + } + + return yaml_emitter_set_emitter_error(emitter, "expected DOCUMENT-START or STREAM-END") +} + +// Expect the root node. +func yaml_emitter_emit_document_content(emitter *yaml_emitter_t, event *yaml_event_t) bool { + emitter.states = append(emitter.states, yaml_EMIT_DOCUMENT_END_STATE) + return yaml_emitter_emit_node(emitter, event, true, false, false, false) +} + +// Expect DOCUMENT-END. +func yaml_emitter_emit_document_end(emitter *yaml_emitter_t, event *yaml_event_t) bool { + if event.typ != yaml_DOCUMENT_END_EVENT { + return yaml_emitter_set_emitter_error(emitter, "expected DOCUMENT-END") + } + if !yaml_emitter_write_indent(emitter) { + return false + } + if !event.implicit { + // [Go] Allocate the slice elsewhere. + if !yaml_emitter_write_indicator(emitter, []byte("..."), true, false, false) { + return false + } + if !yaml_emitter_write_indent(emitter) { + return false + } + } + if !yaml_emitter_flush(emitter) { + return false + } + emitter.state = yaml_EMIT_DOCUMENT_START_STATE + emitter.tag_directives = emitter.tag_directives[:0] + return true +} + +// Expect a flow item node. +func yaml_emitter_emit_flow_sequence_item(emitter *yaml_emitter_t, event *yaml_event_t, first bool) bool { + if first { + if !yaml_emitter_write_indicator(emitter, []byte{'['}, true, true, false) { + return false + } + if !yaml_emitter_increase_indent(emitter, true, false) { + return false + } + emitter.flow_level++ + } + + if event.typ == yaml_SEQUENCE_END_EVENT { + emitter.flow_level-- + emitter.indent = emitter.indents[len(emitter.indents)-1] + emitter.indents = emitter.indents[:len(emitter.indents)-1] + if emitter.canonical && !first { + if !yaml_emitter_write_indicator(emitter, []byte{','}, false, false, false) { + return false + } + if !yaml_emitter_write_indent(emitter) { + return false + } + } + if !yaml_emitter_write_indicator(emitter, []byte{']'}, false, false, false) { + return false + } + emitter.state = emitter.states[len(emitter.states)-1] + emitter.states = emitter.states[:len(emitter.states)-1] + + return true + } + + if !first { + if !yaml_emitter_write_indicator(emitter, []byte{','}, false, false, false) { + return false + } + } + + if emitter.canonical || emitter.column > emitter.best_width { + if !yaml_emitter_write_indent(emitter) { + return false + } + } + emitter.states = append(emitter.states, yaml_EMIT_FLOW_SEQUENCE_ITEM_STATE) + return yaml_emitter_emit_node(emitter, event, false, true, false, false) +} + +// Expect a flow key node. +func yaml_emitter_emit_flow_mapping_key(emitter *yaml_emitter_t, event *yaml_event_t, first bool) bool { + if first { + if !yaml_emitter_write_indicator(emitter, []byte{'{'}, true, true, false) { + return false + } + if !yaml_emitter_increase_indent(emitter, true, false) { + return false + } + emitter.flow_level++ + } + + if event.typ == yaml_MAPPING_END_EVENT { + emitter.flow_level-- + emitter.indent = emitter.indents[len(emitter.indents)-1] + emitter.indents = emitter.indents[:len(emitter.indents)-1] + if emitter.canonical && !first { + if !yaml_emitter_write_indicator(emitter, []byte{','}, false, false, false) { + return false + } + if !yaml_emitter_write_indent(emitter) { + return false + } + } + if !yaml_emitter_write_indicator(emitter, []byte{'}'}, false, false, false) { + return false + } + emitter.state = emitter.states[len(emitter.states)-1] + emitter.states = emitter.states[:len(emitter.states)-1] + return true + } + + if !first { + if !yaml_emitter_write_indicator(emitter, []byte{','}, false, false, false) { + return false + } + } + if emitter.canonical || emitter.column > emitter.best_width { + if !yaml_emitter_write_indent(emitter) { + return false + } + } + + if !emitter.canonical && yaml_emitter_check_simple_key(emitter) { + emitter.states = append(emitter.states, yaml_EMIT_FLOW_MAPPING_SIMPLE_VALUE_STATE) + return yaml_emitter_emit_node(emitter, event, false, false, true, true) + } + if !yaml_emitter_write_indicator(emitter, []byte{'?'}, true, false, false) { + return false + } + emitter.states = append(emitter.states, yaml_EMIT_FLOW_MAPPING_VALUE_STATE) + return yaml_emitter_emit_node(emitter, event, false, false, true, false) +} + +// Expect a flow value node. +func yaml_emitter_emit_flow_mapping_value(emitter *yaml_emitter_t, event *yaml_event_t, simple bool) bool { + if simple { + if !yaml_emitter_write_indicator(emitter, []byte{':'}, false, false, false) { + return false + } + } else { + if emitter.canonical || emitter.column > emitter.best_width { + if !yaml_emitter_write_indent(emitter) { + return false + } + } + if !yaml_emitter_write_indicator(emitter, []byte{':'}, true, false, false) { + return false + } + } + emitter.states = append(emitter.states, yaml_EMIT_FLOW_MAPPING_KEY_STATE) + return yaml_emitter_emit_node(emitter, event, false, false, true, false) +} + +// Expect a block item node. +func yaml_emitter_emit_block_sequence_item(emitter *yaml_emitter_t, event *yaml_event_t, first bool) bool { + if first { + if !yaml_emitter_increase_indent(emitter, false, emitter.mapping_context && !emitter.indention) { + return false + } + } + if event.typ == yaml_SEQUENCE_END_EVENT { + emitter.indent = emitter.indents[len(emitter.indents)-1] + emitter.indents = emitter.indents[:len(emitter.indents)-1] + emitter.state = emitter.states[len(emitter.states)-1] + emitter.states = emitter.states[:len(emitter.states)-1] + return true + } + if !yaml_emitter_write_indent(emitter) { + return false + } + if !yaml_emitter_write_indicator(emitter, []byte{'-'}, true, false, true) { + return false + } + emitter.states = append(emitter.states, yaml_EMIT_BLOCK_SEQUENCE_ITEM_STATE) + return yaml_emitter_emit_node(emitter, event, false, true, false, false) +} + +// Expect a block key node. +func yaml_emitter_emit_block_mapping_key(emitter *yaml_emitter_t, event *yaml_event_t, first bool) bool { + if first { + if !yaml_emitter_increase_indent(emitter, false, false) { + return false + } + } + if event.typ == yaml_MAPPING_END_EVENT { + emitter.indent = emitter.indents[len(emitter.indents)-1] + emitter.indents = emitter.indents[:len(emitter.indents)-1] + emitter.state = emitter.states[len(emitter.states)-1] + emitter.states = emitter.states[:len(emitter.states)-1] + return true + } + if !yaml_emitter_write_indent(emitter) { + return false + } + if yaml_emitter_check_simple_key(emitter) { + emitter.states = append(emitter.states, yaml_EMIT_BLOCK_MAPPING_SIMPLE_VALUE_STATE) + return yaml_emitter_emit_node(emitter, event, false, false, true, true) + } + if !yaml_emitter_write_indicator(emitter, []byte{'?'}, true, false, true) { + return false + } + emitter.states = append(emitter.states, yaml_EMIT_BLOCK_MAPPING_VALUE_STATE) + return yaml_emitter_emit_node(emitter, event, false, false, true, false) +} + +// Expect a block value node. +func yaml_emitter_emit_block_mapping_value(emitter *yaml_emitter_t, event *yaml_event_t, simple bool) bool { + if simple { + if !yaml_emitter_write_indicator(emitter, []byte{':'}, false, false, false) { + return false + } + } else { + if !yaml_emitter_write_indent(emitter) { + return false + } + if !yaml_emitter_write_indicator(emitter, []byte{':'}, true, false, true) { + return false + } + } + emitter.states = append(emitter.states, yaml_EMIT_BLOCK_MAPPING_KEY_STATE) + return yaml_emitter_emit_node(emitter, event, false, false, true, false) +} + +// Expect a node. +func yaml_emitter_emit_node(emitter *yaml_emitter_t, event *yaml_event_t, + root bool, sequence bool, mapping bool, simple_key bool) bool { + + emitter.root_context = root + emitter.sequence_context = sequence + emitter.mapping_context = mapping + emitter.simple_key_context = simple_key + + switch event.typ { + case yaml_ALIAS_EVENT: + return yaml_emitter_emit_alias(emitter, event) + case yaml_SCALAR_EVENT: + return yaml_emitter_emit_scalar(emitter, event) + case yaml_SEQUENCE_START_EVENT: + return yaml_emitter_emit_sequence_start(emitter, event) + case yaml_MAPPING_START_EVENT: + return yaml_emitter_emit_mapping_start(emitter, event) + default: + return yaml_emitter_set_emitter_error(emitter, + fmt.Sprintf("expected SCALAR, SEQUENCE-START, MAPPING-START, or ALIAS, but got %v", event.typ)) + } +} + +// Expect ALIAS. +func yaml_emitter_emit_alias(emitter *yaml_emitter_t, event *yaml_event_t) bool { + if !yaml_emitter_process_anchor(emitter) { + return false + } + emitter.state = emitter.states[len(emitter.states)-1] + emitter.states = emitter.states[:len(emitter.states)-1] + return true +} + +// Expect SCALAR. +func yaml_emitter_emit_scalar(emitter *yaml_emitter_t, event *yaml_event_t) bool { + if !yaml_emitter_select_scalar_style(emitter, event) { + return false + } + if !yaml_emitter_process_anchor(emitter) { + return false + } + if !yaml_emitter_process_tag(emitter) { + return false + } + if !yaml_emitter_increase_indent(emitter, true, false) { + return false + } + if !yaml_emitter_process_scalar(emitter) { + return false + } + emitter.indent = emitter.indents[len(emitter.indents)-1] + emitter.indents = emitter.indents[:len(emitter.indents)-1] + emitter.state = emitter.states[len(emitter.states)-1] + emitter.states = emitter.states[:len(emitter.states)-1] + return true +} + +// Expect SEQUENCE-START. +func yaml_emitter_emit_sequence_start(emitter *yaml_emitter_t, event *yaml_event_t) bool { + if !yaml_emitter_process_anchor(emitter) { + return false + } + if !yaml_emitter_process_tag(emitter) { + return false + } + if emitter.flow_level > 0 || emitter.canonical || event.sequence_style() == yaml_FLOW_SEQUENCE_STYLE || + yaml_emitter_check_empty_sequence(emitter) { + emitter.state = yaml_EMIT_FLOW_SEQUENCE_FIRST_ITEM_STATE + } else { + emitter.state = yaml_EMIT_BLOCK_SEQUENCE_FIRST_ITEM_STATE + } + return true +} + +// Expect MAPPING-START. +func yaml_emitter_emit_mapping_start(emitter *yaml_emitter_t, event *yaml_event_t) bool { + if !yaml_emitter_process_anchor(emitter) { + return false + } + if !yaml_emitter_process_tag(emitter) { + return false + } + if emitter.flow_level > 0 || emitter.canonical || event.mapping_style() == yaml_FLOW_MAPPING_STYLE || + yaml_emitter_check_empty_mapping(emitter) { + emitter.state = yaml_EMIT_FLOW_MAPPING_FIRST_KEY_STATE + } else { + emitter.state = yaml_EMIT_BLOCK_MAPPING_FIRST_KEY_STATE + } + return true +} + +// Check if the document content is an empty scalar. +func yaml_emitter_check_empty_document(emitter *yaml_emitter_t) bool { + return false // [Go] Huh? +} + +// Check if the next events represent an empty sequence. +func yaml_emitter_check_empty_sequence(emitter *yaml_emitter_t) bool { + if len(emitter.events)-emitter.events_head < 2 { + return false + } + return emitter.events[emitter.events_head].typ == yaml_SEQUENCE_START_EVENT && + emitter.events[emitter.events_head+1].typ == yaml_SEQUENCE_END_EVENT +} + +// Check if the next events represent an empty mapping. +func yaml_emitter_check_empty_mapping(emitter *yaml_emitter_t) bool { + if len(emitter.events)-emitter.events_head < 2 { + return false + } + return emitter.events[emitter.events_head].typ == yaml_MAPPING_START_EVENT && + emitter.events[emitter.events_head+1].typ == yaml_MAPPING_END_EVENT +} + +// Check if the next node can be expressed as a simple key. +func yaml_emitter_check_simple_key(emitter *yaml_emitter_t) bool { + length := 0 + switch emitter.events[emitter.events_head].typ { + case yaml_ALIAS_EVENT: + length += len(emitter.anchor_data.anchor) + case yaml_SCALAR_EVENT: + if emitter.scalar_data.multiline { + return false + } + length += len(emitter.anchor_data.anchor) + + len(emitter.tag_data.handle) + + len(emitter.tag_data.suffix) + + len(emitter.scalar_data.value) + case yaml_SEQUENCE_START_EVENT: + if !yaml_emitter_check_empty_sequence(emitter) { + return false + } + length += len(emitter.anchor_data.anchor) + + len(emitter.tag_data.handle) + + len(emitter.tag_data.suffix) + case yaml_MAPPING_START_EVENT: + if !yaml_emitter_check_empty_mapping(emitter) { + return false + } + length += len(emitter.anchor_data.anchor) + + len(emitter.tag_data.handle) + + len(emitter.tag_data.suffix) + default: + return false + } + return length <= 128 +} + +// Determine an acceptable scalar style. +func yaml_emitter_select_scalar_style(emitter *yaml_emitter_t, event *yaml_event_t) bool { + + no_tag := len(emitter.tag_data.handle) == 0 && len(emitter.tag_data.suffix) == 0 + if no_tag && !event.implicit && !event.quoted_implicit { + return yaml_emitter_set_emitter_error(emitter, "neither tag nor implicit flags are specified") + } + + style := event.scalar_style() + if style == yaml_ANY_SCALAR_STYLE { + style = yaml_PLAIN_SCALAR_STYLE + } + if emitter.canonical { + style = yaml_DOUBLE_QUOTED_SCALAR_STYLE + } + if emitter.simple_key_context && emitter.scalar_data.multiline { + style = yaml_DOUBLE_QUOTED_SCALAR_STYLE + } + + if style == yaml_PLAIN_SCALAR_STYLE { + if emitter.flow_level > 0 && !emitter.scalar_data.flow_plain_allowed || + emitter.flow_level == 0 && !emitter.scalar_data.block_plain_allowed { + style = yaml_SINGLE_QUOTED_SCALAR_STYLE + } + if len(emitter.scalar_data.value) == 0 && (emitter.flow_level > 0 || emitter.simple_key_context) { + style = yaml_SINGLE_QUOTED_SCALAR_STYLE + } + if no_tag && !event.implicit { + style = yaml_SINGLE_QUOTED_SCALAR_STYLE + } + } + if style == yaml_SINGLE_QUOTED_SCALAR_STYLE { + if !emitter.scalar_data.single_quoted_allowed { + style = yaml_DOUBLE_QUOTED_SCALAR_STYLE + } + } + if style == yaml_LITERAL_SCALAR_STYLE || style == yaml_FOLDED_SCALAR_STYLE { + if !emitter.scalar_data.block_allowed || emitter.flow_level > 0 || emitter.simple_key_context { + style = yaml_DOUBLE_QUOTED_SCALAR_STYLE + } + } + + if no_tag && !event.quoted_implicit && style != yaml_PLAIN_SCALAR_STYLE { + emitter.tag_data.handle = []byte{'!'} + } + emitter.scalar_data.style = style + return true +} + +// Write an anchor. +func yaml_emitter_process_anchor(emitter *yaml_emitter_t) bool { + if emitter.anchor_data.anchor == nil { + return true + } + c := []byte{'&'} + if emitter.anchor_data.alias { + c[0] = '*' + } + if !yaml_emitter_write_indicator(emitter, c, true, false, false) { + return false + } + return yaml_emitter_write_anchor(emitter, emitter.anchor_data.anchor) +} + +// Write a tag. +func yaml_emitter_process_tag(emitter *yaml_emitter_t) bool { + if len(emitter.tag_data.handle) == 0 && len(emitter.tag_data.suffix) == 0 { + return true + } + if len(emitter.tag_data.handle) > 0 { + if !yaml_emitter_write_tag_handle(emitter, emitter.tag_data.handle) { + return false + } + if len(emitter.tag_data.suffix) > 0 { + if !yaml_emitter_write_tag_content(emitter, emitter.tag_data.suffix, false) { + return false + } + } + } else { + // [Go] Allocate these slices elsewhere. + if !yaml_emitter_write_indicator(emitter, []byte("!<"), true, false, false) { + return false + } + if !yaml_emitter_write_tag_content(emitter, emitter.tag_data.suffix, false) { + return false + } + if !yaml_emitter_write_indicator(emitter, []byte{'>'}, false, false, false) { + return false + } + } + return true +} + +// Write a scalar. +func yaml_emitter_process_scalar(emitter *yaml_emitter_t) bool { + switch emitter.scalar_data.style { + case yaml_PLAIN_SCALAR_STYLE: + return yaml_emitter_write_plain_scalar(emitter, emitter.scalar_data.value, !emitter.simple_key_context) + + case yaml_SINGLE_QUOTED_SCALAR_STYLE: + return yaml_emitter_write_single_quoted_scalar(emitter, emitter.scalar_data.value, !emitter.simple_key_context) + + case yaml_DOUBLE_QUOTED_SCALAR_STYLE: + return yaml_emitter_write_double_quoted_scalar(emitter, emitter.scalar_data.value, !emitter.simple_key_context) + + case yaml_LITERAL_SCALAR_STYLE: + return yaml_emitter_write_literal_scalar(emitter, emitter.scalar_data.value) + + case yaml_FOLDED_SCALAR_STYLE: + return yaml_emitter_write_folded_scalar(emitter, emitter.scalar_data.value) + } + panic("unknown scalar style") +} + +// Check if a %YAML directive is valid. +func yaml_emitter_analyze_version_directive(emitter *yaml_emitter_t, version_directive *yaml_version_directive_t) bool { + if version_directive.major != 1 || version_directive.minor != 1 { + return yaml_emitter_set_emitter_error(emitter, "incompatible %YAML directive") + } + return true +} + +// Check if a %TAG directive is valid. +func yaml_emitter_analyze_tag_directive(emitter *yaml_emitter_t, tag_directive *yaml_tag_directive_t) bool { + handle := tag_directive.handle + prefix := tag_directive.prefix + if len(handle) == 0 { + return yaml_emitter_set_emitter_error(emitter, "tag handle must not be empty") + } + if handle[0] != '!' { + return yaml_emitter_set_emitter_error(emitter, "tag handle must start with '!'") + } + if handle[len(handle)-1] != '!' { + return yaml_emitter_set_emitter_error(emitter, "tag handle must end with '!'") + } + for i := 1; i < len(handle)-1; i += width(handle[i]) { + if !is_alpha(handle, i) { + return yaml_emitter_set_emitter_error(emitter, "tag handle must contain alphanumerical characters only") + } + } + if len(prefix) == 0 { + return yaml_emitter_set_emitter_error(emitter, "tag prefix must not be empty") + } + return true +} + +// Check if an anchor is valid. +func yaml_emitter_analyze_anchor(emitter *yaml_emitter_t, anchor []byte, alias bool) bool { + if len(anchor) == 0 { + problem := "anchor value must not be empty" + if alias { + problem = "alias value must not be empty" + } + return yaml_emitter_set_emitter_error(emitter, problem) + } + for i := 0; i < len(anchor); i += width(anchor[i]) { + if !is_alpha(anchor, i) { + problem := "anchor value must contain alphanumerical characters only" + if alias { + problem = "alias value must contain alphanumerical characters only" + } + return yaml_emitter_set_emitter_error(emitter, problem) + } + } + emitter.anchor_data.anchor = anchor + emitter.anchor_data.alias = alias + return true +} + +// Check if a tag is valid. +func yaml_emitter_analyze_tag(emitter *yaml_emitter_t, tag []byte) bool { + if len(tag) == 0 { + return yaml_emitter_set_emitter_error(emitter, "tag value must not be empty") + } + for i := 0; i < len(emitter.tag_directives); i++ { + tag_directive := &emitter.tag_directives[i] + if bytes.HasPrefix(tag, tag_directive.prefix) { + emitter.tag_data.handle = tag_directive.handle + emitter.tag_data.suffix = tag[len(tag_directive.prefix):] + return true + } + } + emitter.tag_data.suffix = tag + return true +} + +// Check if a scalar is valid. +func yaml_emitter_analyze_scalar(emitter *yaml_emitter_t, value []byte) bool { + var ( + block_indicators = false + flow_indicators = false + line_breaks = false + special_characters = false + + leading_space = false + leading_break = false + trailing_space = false + trailing_break = false + break_space = false + space_break = false + + preceded_by_whitespace = false + followed_by_whitespace = false + previous_space = false + previous_break = false + ) + + emitter.scalar_data.value = value + + if len(value) == 0 { + emitter.scalar_data.multiline = false + emitter.scalar_data.flow_plain_allowed = false + emitter.scalar_data.block_plain_allowed = true + emitter.scalar_data.single_quoted_allowed = true + emitter.scalar_data.block_allowed = false + return true + } + + if len(value) >= 3 && ((value[0] == '-' && value[1] == '-' && value[2] == '-') || (value[0] == '.' && value[1] == '.' && value[2] == '.')) { + block_indicators = true + flow_indicators = true + } + + preceded_by_whitespace = true + for i, w := 0, 0; i < len(value); i += w { + w = width(value[i]) + followed_by_whitespace = i+w >= len(value) || is_blank(value, i+w) + + if i == 0 { + switch value[i] { + case '#', ',', '[', ']', '{', '}', '&', '*', '!', '|', '>', '\'', '"', '%', '@', '`': + flow_indicators = true + block_indicators = true + case '?', ':': + flow_indicators = true + if followed_by_whitespace { + block_indicators = true + } + case '-': + if followed_by_whitespace { + flow_indicators = true + block_indicators = true + } + } + } else { + switch value[i] { + case ',', '?', '[', ']', '{', '}': + flow_indicators = true + case ':': + flow_indicators = true + if followed_by_whitespace { + block_indicators = true + } + case '#': + if preceded_by_whitespace { + flow_indicators = true + block_indicators = true + } + } + } + + if !is_printable(value, i) || !is_ascii(value, i) && !emitter.unicode { + special_characters = true + } + if is_space(value, i) { + if i == 0 { + leading_space = true + } + if i+width(value[i]) == len(value) { + trailing_space = true + } + if previous_break { + break_space = true + } + previous_space = true + previous_break = false + } else if is_break(value, i) { + line_breaks = true + if i == 0 { + leading_break = true + } + if i+width(value[i]) == len(value) { + trailing_break = true + } + if previous_space { + space_break = true + } + previous_space = false + previous_break = true + } else { + previous_space = false + previous_break = false + } + + // [Go]: Why 'z'? Couldn't be the end of the string as that's the loop condition. + preceded_by_whitespace = is_blankz(value, i) + } + + emitter.scalar_data.multiline = line_breaks + emitter.scalar_data.flow_plain_allowed = true + emitter.scalar_data.block_plain_allowed = true + emitter.scalar_data.single_quoted_allowed = true + emitter.scalar_data.block_allowed = true + + if leading_space || leading_break || trailing_space || trailing_break { + emitter.scalar_data.flow_plain_allowed = false + emitter.scalar_data.block_plain_allowed = false + } + if trailing_space { + emitter.scalar_data.block_allowed = false + } + if break_space { + emitter.scalar_data.flow_plain_allowed = false + emitter.scalar_data.block_plain_allowed = false + emitter.scalar_data.single_quoted_allowed = false + } + if space_break || special_characters { + emitter.scalar_data.flow_plain_allowed = false + emitter.scalar_data.block_plain_allowed = false + emitter.scalar_data.single_quoted_allowed = false + emitter.scalar_data.block_allowed = false + } + if line_breaks { + emitter.scalar_data.flow_plain_allowed = false + emitter.scalar_data.block_plain_allowed = false + } + if flow_indicators { + emitter.scalar_data.flow_plain_allowed = false + } + if block_indicators { + emitter.scalar_data.block_plain_allowed = false + } + return true +} + +// Check if the event data is valid. +func yaml_emitter_analyze_event(emitter *yaml_emitter_t, event *yaml_event_t) bool { + + emitter.anchor_data.anchor = nil + emitter.tag_data.handle = nil + emitter.tag_data.suffix = nil + emitter.scalar_data.value = nil + + switch event.typ { + case yaml_ALIAS_EVENT: + if !yaml_emitter_analyze_anchor(emitter, event.anchor, true) { + return false + } + + case yaml_SCALAR_EVENT: + if len(event.anchor) > 0 { + if !yaml_emitter_analyze_anchor(emitter, event.anchor, false) { + return false + } + } + if len(event.tag) > 0 && (emitter.canonical || (!event.implicit && !event.quoted_implicit)) { + if !yaml_emitter_analyze_tag(emitter, event.tag) { + return false + } + } + if !yaml_emitter_analyze_scalar(emitter, event.value) { + return false + } + + case yaml_SEQUENCE_START_EVENT: + if len(event.anchor) > 0 { + if !yaml_emitter_analyze_anchor(emitter, event.anchor, false) { + return false + } + } + if len(event.tag) > 0 && (emitter.canonical || !event.implicit) { + if !yaml_emitter_analyze_tag(emitter, event.tag) { + return false + } + } + + case yaml_MAPPING_START_EVENT: + if len(event.anchor) > 0 { + if !yaml_emitter_analyze_anchor(emitter, event.anchor, false) { + return false + } + } + if len(event.tag) > 0 && (emitter.canonical || !event.implicit) { + if !yaml_emitter_analyze_tag(emitter, event.tag) { + return false + } + } + } + return true +} + +// Write the BOM character. +func yaml_emitter_write_bom(emitter *yaml_emitter_t) bool { + if !flush(emitter) { + return false + } + pos := emitter.buffer_pos + emitter.buffer[pos+0] = '\xEF' + emitter.buffer[pos+1] = '\xBB' + emitter.buffer[pos+2] = '\xBF' + emitter.buffer_pos += 3 + return true +} + +func yaml_emitter_write_indent(emitter *yaml_emitter_t) bool { + indent := emitter.indent + if indent < 0 { + indent = 0 + } + if !emitter.indention || emitter.column > indent || (emitter.column == indent && !emitter.whitespace) { + if !put_break(emitter) { + return false + } + } + for emitter.column < indent { + if !put(emitter, ' ') { + return false + } + } + emitter.whitespace = true + emitter.indention = true + return true +} + +func yaml_emitter_write_indicator(emitter *yaml_emitter_t, indicator []byte, need_whitespace, is_whitespace, is_indention bool) bool { + if need_whitespace && !emitter.whitespace { + if !put(emitter, ' ') { + return false + } + } + if !write_all(emitter, indicator) { + return false + } + emitter.whitespace = is_whitespace + emitter.indention = (emitter.indention && is_indention) + emitter.open_ended = false + return true +} + +func yaml_emitter_write_anchor(emitter *yaml_emitter_t, value []byte) bool { + if !write_all(emitter, value) { + return false + } + emitter.whitespace = false + emitter.indention = false + return true +} + +func yaml_emitter_write_tag_handle(emitter *yaml_emitter_t, value []byte) bool { + if !emitter.whitespace { + if !put(emitter, ' ') { + return false + } + } + if !write_all(emitter, value) { + return false + } + emitter.whitespace = false + emitter.indention = false + return true +} + +func yaml_emitter_write_tag_content(emitter *yaml_emitter_t, value []byte, need_whitespace bool) bool { + if need_whitespace && !emitter.whitespace { + if !put(emitter, ' ') { + return false + } + } + for i := 0; i < len(value); { + var must_write bool + switch value[i] { + case ';', '/', '?', ':', '@', '&', '=', '+', '$', ',', '_', '.', '~', '*', '\'', '(', ')', '[', ']': + must_write = true + default: + must_write = is_alpha(value, i) + } + if must_write { + if !write(emitter, value, &i) { + return false + } + } else { + w := width(value[i]) + for k := 0; k < w; k++ { + octet := value[i] + i++ + if !put(emitter, '%') { + return false + } + + c := octet >> 4 + if c < 10 { + c += '0' + } else { + c += 'A' - 10 + } + if !put(emitter, c) { + return false + } + + c = octet & 0x0f + if c < 10 { + c += '0' + } else { + c += 'A' - 10 + } + if !put(emitter, c) { + return false + } + } + } + } + emitter.whitespace = false + emitter.indention = false + return true +} + +func yaml_emitter_write_plain_scalar(emitter *yaml_emitter_t, value []byte, allow_breaks bool) bool { + if !emitter.whitespace { + if !put(emitter, ' ') { + return false + } + } + + spaces := false + breaks := false + for i := 0; i < len(value); { + if is_space(value, i) { + if allow_breaks && !spaces && emitter.column > emitter.best_width && !is_space(value, i+1) { + if !yaml_emitter_write_indent(emitter) { + return false + } + i += width(value[i]) + } else { + if !write(emitter, value, &i) { + return false + } + } + spaces = true + } else if is_break(value, i) { + if !breaks && value[i] == '\n' { + if !put_break(emitter) { + return false + } + } + if !write_break(emitter, value, &i) { + return false + } + emitter.indention = true + breaks = true + } else { + if breaks { + if !yaml_emitter_write_indent(emitter) { + return false + } + } + if !write(emitter, value, &i) { + return false + } + emitter.indention = false + spaces = false + breaks = false + } + } + + emitter.whitespace = false + emitter.indention = false + if emitter.root_context { + emitter.open_ended = true + } + + return true +} + +func yaml_emitter_write_single_quoted_scalar(emitter *yaml_emitter_t, value []byte, allow_breaks bool) bool { + + if !yaml_emitter_write_indicator(emitter, []byte{'\''}, true, false, false) { + return false + } + + spaces := false + breaks := false + for i := 0; i < len(value); { + if is_space(value, i) { + if allow_breaks && !spaces && emitter.column > emitter.best_width && i > 0 && i < len(value)-1 && !is_space(value, i+1) { + if !yaml_emitter_write_indent(emitter) { + return false + } + i += width(value[i]) + } else { + if !write(emitter, value, &i) { + return false + } + } + spaces = true + } else if is_break(value, i) { + if !breaks && value[i] == '\n' { + if !put_break(emitter) { + return false + } + } + if !write_break(emitter, value, &i) { + return false + } + emitter.indention = true + breaks = true + } else { + if breaks { + if !yaml_emitter_write_indent(emitter) { + return false + } + } + if value[i] == '\'' { + if !put(emitter, '\'') { + return false + } + } + if !write(emitter, value, &i) { + return false + } + emitter.indention = false + spaces = false + breaks = false + } + } + if !yaml_emitter_write_indicator(emitter, []byte{'\''}, false, false, false) { + return false + } + emitter.whitespace = false + emitter.indention = false + return true +} + +func yaml_emitter_write_double_quoted_scalar(emitter *yaml_emitter_t, value []byte, allow_breaks bool) bool { + spaces := false + if !yaml_emitter_write_indicator(emitter, []byte{'"'}, true, false, false) { + return false + } + + for i := 0; i < len(value); { + if !is_printable(value, i) || (!emitter.unicode && !is_ascii(value, i)) || + is_bom(value, i) || is_break(value, i) || + value[i] == '"' || value[i] == '\\' { + + octet := value[i] + + var w int + var v rune + switch { + case octet&0x80 == 0x00: + w, v = 1, rune(octet&0x7F) + case octet&0xE0 == 0xC0: + w, v = 2, rune(octet&0x1F) + case octet&0xF0 == 0xE0: + w, v = 3, rune(octet&0x0F) + case octet&0xF8 == 0xF0: + w, v = 4, rune(octet&0x07) + } + for k := 1; k < w; k++ { + octet = value[i+k] + v = (v << 6) + (rune(octet) & 0x3F) + } + i += w + + if !put(emitter, '\\') { + return false + } + + var ok bool + switch v { + case 0x00: + ok = put(emitter, '0') + case 0x07: + ok = put(emitter, 'a') + case 0x08: + ok = put(emitter, 'b') + case 0x09: + ok = put(emitter, 't') + case 0x0A: + ok = put(emitter, 'n') + case 0x0b: + ok = put(emitter, 'v') + case 0x0c: + ok = put(emitter, 'f') + case 0x0d: + ok = put(emitter, 'r') + case 0x1b: + ok = put(emitter, 'e') + case 0x22: + ok = put(emitter, '"') + case 0x5c: + ok = put(emitter, '\\') + case 0x85: + ok = put(emitter, 'N') + case 0xA0: + ok = put(emitter, '_') + case 0x2028: + ok = put(emitter, 'L') + case 0x2029: + ok = put(emitter, 'P') + default: + if v <= 0xFF { + ok = put(emitter, 'x') + w = 2 + } else if v <= 0xFFFF { + ok = put(emitter, 'u') + w = 4 + } else { + ok = put(emitter, 'U') + w = 8 + } + for k := (w - 1) * 4; ok && k >= 0; k -= 4 { + digit := byte((v >> uint(k)) & 0x0F) + if digit < 10 { + ok = put(emitter, digit+'0') + } else { + ok = put(emitter, digit+'A'-10) + } + } + } + if !ok { + return false + } + spaces = false + } else if is_space(value, i) { + if allow_breaks && !spaces && emitter.column > emitter.best_width && i > 0 && i < len(value)-1 { + if !yaml_emitter_write_indent(emitter) { + return false + } + if is_space(value, i+1) { + if !put(emitter, '\\') { + return false + } + } + i += width(value[i]) + } else if !write(emitter, value, &i) { + return false + } + spaces = true + } else { + if !write(emitter, value, &i) { + return false + } + spaces = false + } + } + if !yaml_emitter_write_indicator(emitter, []byte{'"'}, false, false, false) { + return false + } + emitter.whitespace = false + emitter.indention = false + return true +} + +func yaml_emitter_write_block_scalar_hints(emitter *yaml_emitter_t, value []byte) bool { + if is_space(value, 0) || is_break(value, 0) { + indent_hint := []byte{'0' + byte(emitter.best_indent)} + if !yaml_emitter_write_indicator(emitter, indent_hint, false, false, false) { + return false + } + } + + emitter.open_ended = false + + var chomp_hint [1]byte + if len(value) == 0 { + chomp_hint[0] = '-' + } else { + i := len(value) - 1 + for value[i]&0xC0 == 0x80 { + i-- + } + if !is_break(value, i) { + chomp_hint[0] = '-' + } else if i == 0 { + chomp_hint[0] = '+' + emitter.open_ended = true + } else { + i-- + for value[i]&0xC0 == 0x80 { + i-- + } + if is_break(value, i) { + chomp_hint[0] = '+' + emitter.open_ended = true + } + } + } + if chomp_hint[0] != 0 { + if !yaml_emitter_write_indicator(emitter, chomp_hint[:], false, false, false) { + return false + } + } + return true +} + +func yaml_emitter_write_literal_scalar(emitter *yaml_emitter_t, value []byte) bool { + if !yaml_emitter_write_indicator(emitter, []byte{'|'}, true, false, false) { + return false + } + if !yaml_emitter_write_block_scalar_hints(emitter, value) { + return false + } + if !put_break(emitter) { + return false + } + emitter.indention = true + emitter.whitespace = true + breaks := true + for i := 0; i < len(value); { + if is_break(value, i) { + if !write_break(emitter, value, &i) { + return false + } + emitter.indention = true + breaks = true + } else { + if breaks { + if !yaml_emitter_write_indent(emitter) { + return false + } + } + if !write(emitter, value, &i) { + return false + } + emitter.indention = false + breaks = false + } + } + + return true +} + +func yaml_emitter_write_folded_scalar(emitter *yaml_emitter_t, value []byte) bool { + if !yaml_emitter_write_indicator(emitter, []byte{'>'}, true, false, false) { + return false + } + if !yaml_emitter_write_block_scalar_hints(emitter, value) { + return false + } + + if !put_break(emitter) { + return false + } + emitter.indention = true + emitter.whitespace = true + + breaks := true + leading_spaces := true + for i := 0; i < len(value); { + if is_break(value, i) { + if !breaks && !leading_spaces && value[i] == '\n' { + k := 0 + for is_break(value, k) { + k += width(value[k]) + } + if !is_blankz(value, k) { + if !put_break(emitter) { + return false + } + } + } + if !write_break(emitter, value, &i) { + return false + } + emitter.indention = true + breaks = true + } else { + if breaks { + if !yaml_emitter_write_indent(emitter) { + return false + } + leading_spaces = is_blank(value, i) + } + if !breaks && is_space(value, i) && !is_space(value, i+1) && emitter.column > emitter.best_width { + if !yaml_emitter_write_indent(emitter) { + return false + } + i += width(value[i]) + } else { + if !write(emitter, value, &i) { + return false + } + } + emitter.indention = false + breaks = false + } + } + return true +} diff --git a/tools/voluspa/vendor/gopkg.in/yaml.v2/encode.go b/tools/voluspa/vendor/gopkg.in/yaml.v2/encode.go new file mode 100644 index 00000000000..0ee738e11b6 --- /dev/null +++ b/tools/voluspa/vendor/gopkg.in/yaml.v2/encode.go @@ -0,0 +1,390 @@ +package yaml + +import ( + "encoding" + "fmt" + "io" + "reflect" + "regexp" + "sort" + "strconv" + "strings" + "time" + "unicode/utf8" +) + +// jsonNumber is the interface of the encoding/json.Number datatype. +// Repeating the interface here avoids a dependency on encoding/json, and also +// supports other libraries like jsoniter, which use a similar datatype with +// the same interface. Detecting this interface is useful when dealing with +// structures containing json.Number, which is a string under the hood. The +// encoder should prefer the use of Int64(), Float64() and string(), in that +// order, when encoding this type. +type jsonNumber interface { + Float64() (float64, error) + Int64() (int64, error) + String() string +} + +type encoder struct { + emitter yaml_emitter_t + event yaml_event_t + out []byte + flow bool + // doneInit holds whether the initial stream_start_event has been + // emitted. + doneInit bool +} + +func newEncoder() *encoder { + e := &encoder{} + yaml_emitter_initialize(&e.emitter) + yaml_emitter_set_output_string(&e.emitter, &e.out) + yaml_emitter_set_unicode(&e.emitter, true) + return e +} + +func newEncoderWithWriter(w io.Writer) *encoder { + e := &encoder{} + yaml_emitter_initialize(&e.emitter) + yaml_emitter_set_output_writer(&e.emitter, w) + yaml_emitter_set_unicode(&e.emitter, true) + return e +} + +func (e *encoder) init() { + if e.doneInit { + return + } + yaml_stream_start_event_initialize(&e.event, yaml_UTF8_ENCODING) + e.emit() + e.doneInit = true +} + +func (e *encoder) finish() { + e.emitter.open_ended = false + yaml_stream_end_event_initialize(&e.event) + e.emit() +} + +func (e *encoder) destroy() { + yaml_emitter_delete(&e.emitter) +} + +func (e *encoder) emit() { + // This will internally delete the e.event value. + e.must(yaml_emitter_emit(&e.emitter, &e.event)) +} + +func (e *encoder) must(ok bool) { + if !ok { + msg := e.emitter.problem + if msg == "" { + msg = "unknown problem generating YAML content" + } + failf("%s", msg) + } +} + +func (e *encoder) marshalDoc(tag string, in reflect.Value) { + e.init() + yaml_document_start_event_initialize(&e.event, nil, nil, true) + e.emit() + e.marshal(tag, in) + yaml_document_end_event_initialize(&e.event, true) + e.emit() +} + +func (e *encoder) marshal(tag string, in reflect.Value) { + if !in.IsValid() || in.Kind() == reflect.Ptr && in.IsNil() { + e.nilv() + return + } + iface := in.Interface() + switch m := iface.(type) { + case jsonNumber: + integer, err := m.Int64() + if err == nil { + // In this case the json.Number is a valid int64 + in = reflect.ValueOf(integer) + break + } + float, err := m.Float64() + if err == nil { + // In this case the json.Number is a valid float64 + in = reflect.ValueOf(float) + break + } + // fallback case - no number could be obtained + in = reflect.ValueOf(m.String()) + case time.Time, *time.Time: + // Although time.Time implements TextMarshaler, + // we don't want to treat it as a string for YAML + // purposes because YAML has special support for + // timestamps. + case Marshaler: + v, err := m.MarshalYAML() + if err != nil { + fail(err) + } + if v == nil { + e.nilv() + return + } + in = reflect.ValueOf(v) + case encoding.TextMarshaler: + text, err := m.MarshalText() + if err != nil { + fail(err) + } + in = reflect.ValueOf(string(text)) + case nil: + e.nilv() + return + } + switch in.Kind() { + case reflect.Interface: + e.marshal(tag, in.Elem()) + case reflect.Map: + e.mapv(tag, in) + case reflect.Ptr: + if in.Type() == ptrTimeType { + e.timev(tag, in.Elem()) + } else { + e.marshal(tag, in.Elem()) + } + case reflect.Struct: + if in.Type() == timeType { + e.timev(tag, in) + } else { + e.structv(tag, in) + } + case reflect.Slice, reflect.Array: + if in.Type().Elem() == mapItemType { + e.itemsv(tag, in) + } else { + e.slicev(tag, in) + } + case reflect.String: + e.stringv(tag, in) + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + if in.Type() == durationType { + e.stringv(tag, reflect.ValueOf(iface.(time.Duration).String())) + } else { + e.intv(tag, in) + } + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: + e.uintv(tag, in) + case reflect.Float32, reflect.Float64: + e.floatv(tag, in) + case reflect.Bool: + e.boolv(tag, in) + default: + panic("cannot marshal type: " + in.Type().String()) + } +} + +func (e *encoder) mapv(tag string, in reflect.Value) { + e.mappingv(tag, func() { + keys := keyList(in.MapKeys()) + sort.Sort(keys) + for _, k := range keys { + e.marshal("", k) + e.marshal("", in.MapIndex(k)) + } + }) +} + +func (e *encoder) itemsv(tag string, in reflect.Value) { + e.mappingv(tag, func() { + slice := in.Convert(reflect.TypeOf([]MapItem{})).Interface().([]MapItem) + for _, item := range slice { + e.marshal("", reflect.ValueOf(item.Key)) + e.marshal("", reflect.ValueOf(item.Value)) + } + }) +} + +func (e *encoder) structv(tag string, in reflect.Value) { + sinfo, err := getStructInfo(in.Type()) + if err != nil { + panic(err) + } + e.mappingv(tag, func() { + for _, info := range sinfo.FieldsList { + var value reflect.Value + if info.Inline == nil { + value = in.Field(info.Num) + } else { + value = in.FieldByIndex(info.Inline) + } + if info.OmitEmpty && isZero(value) { + continue + } + e.marshal("", reflect.ValueOf(info.Key)) + e.flow = info.Flow + e.marshal("", value) + } + if sinfo.InlineMap >= 0 { + m := in.Field(sinfo.InlineMap) + if m.Len() > 0 { + e.flow = false + keys := keyList(m.MapKeys()) + sort.Sort(keys) + for _, k := range keys { + if _, found := sinfo.FieldsMap[k.String()]; found { + panic(fmt.Sprintf("Can't have key %q in inlined map; conflicts with struct field", k.String())) + } + e.marshal("", k) + e.flow = false + e.marshal("", m.MapIndex(k)) + } + } + } + }) +} + +func (e *encoder) mappingv(tag string, f func()) { + implicit := tag == "" + style := yaml_BLOCK_MAPPING_STYLE + if e.flow { + e.flow = false + style = yaml_FLOW_MAPPING_STYLE + } + yaml_mapping_start_event_initialize(&e.event, nil, []byte(tag), implicit, style) + e.emit() + f() + yaml_mapping_end_event_initialize(&e.event) + e.emit() +} + +func (e *encoder) slicev(tag string, in reflect.Value) { + implicit := tag == "" + style := yaml_BLOCK_SEQUENCE_STYLE + if e.flow { + e.flow = false + style = yaml_FLOW_SEQUENCE_STYLE + } + e.must(yaml_sequence_start_event_initialize(&e.event, nil, []byte(tag), implicit, style)) + e.emit() + n := in.Len() + for i := 0; i < n; i++ { + e.marshal("", in.Index(i)) + } + e.must(yaml_sequence_end_event_initialize(&e.event)) + e.emit() +} + +// isBase60 returns whether s is in base 60 notation as defined in YAML 1.1. +// +// The base 60 float notation in YAML 1.1 is a terrible idea and is unsupported +// in YAML 1.2 and by this package, but these should be marshalled quoted for +// the time being for compatibility with other parsers. +func isBase60Float(s string) (result bool) { + // Fast path. + if s == "" { + return false + } + c := s[0] + if !(c == '+' || c == '-' || c >= '0' && c <= '9') || strings.IndexByte(s, ':') < 0 { + return false + } + // Do the full match. + return base60float.MatchString(s) +} + +// From http://yaml.org/type/float.html, except the regular expression there +// is bogus. In practice parsers do not enforce the "\.[0-9_]*" suffix. +var base60float = regexp.MustCompile(`^[-+]?[0-9][0-9_]*(?::[0-5]?[0-9])+(?:\.[0-9_]*)?$`) + +func (e *encoder) stringv(tag string, in reflect.Value) { + var style yaml_scalar_style_t + s := in.String() + canUsePlain := true + switch { + case !utf8.ValidString(s): + if tag == yaml_BINARY_TAG { + failf("explicitly tagged !!binary data must be base64-encoded") + } + if tag != "" { + failf("cannot marshal invalid UTF-8 data as %s", shortTag(tag)) + } + // It can't be encoded directly as YAML so use a binary tag + // and encode it as base64. + tag = yaml_BINARY_TAG + s = encodeBase64(s) + case tag == "": + // Check to see if it would resolve to a specific + // tag when encoded unquoted. If it doesn't, + // there's no need to quote it. + rtag, _ := resolve("", s) + canUsePlain = rtag == yaml_STR_TAG && !isBase60Float(s) + } + // Note: it's possible for user code to emit invalid YAML + // if they explicitly specify a tag and a string containing + // text that's incompatible with that tag. + switch { + case strings.Contains(s, "\n"): + style = yaml_LITERAL_SCALAR_STYLE + case canUsePlain: + style = yaml_PLAIN_SCALAR_STYLE + default: + style = yaml_DOUBLE_QUOTED_SCALAR_STYLE + } + e.emitScalar(s, "", tag, style) +} + +func (e *encoder) boolv(tag string, in reflect.Value) { + var s string + if in.Bool() { + s = "true" + } else { + s = "false" + } + e.emitScalar(s, "", tag, yaml_PLAIN_SCALAR_STYLE) +} + +func (e *encoder) intv(tag string, in reflect.Value) { + s := strconv.FormatInt(in.Int(), 10) + e.emitScalar(s, "", tag, yaml_PLAIN_SCALAR_STYLE) +} + +func (e *encoder) uintv(tag string, in reflect.Value) { + s := strconv.FormatUint(in.Uint(), 10) + e.emitScalar(s, "", tag, yaml_PLAIN_SCALAR_STYLE) +} + +func (e *encoder) timev(tag string, in reflect.Value) { + t := in.Interface().(time.Time) + s := t.Format(time.RFC3339Nano) + e.emitScalar(s, "", tag, yaml_PLAIN_SCALAR_STYLE) +} + +func (e *encoder) floatv(tag string, in reflect.Value) { + // Issue #352: When formatting, use the precision of the underlying value + precision := 64 + if in.Kind() == reflect.Float32 { + precision = 32 + } + + s := strconv.FormatFloat(in.Float(), 'g', -1, precision) + switch s { + case "+Inf": + s = ".inf" + case "-Inf": + s = "-.inf" + case "NaN": + s = ".nan" + } + e.emitScalar(s, "", tag, yaml_PLAIN_SCALAR_STYLE) +} + +func (e *encoder) nilv() { + e.emitScalar("null", "", "", yaml_PLAIN_SCALAR_STYLE) +} + +func (e *encoder) emitScalar(value, anchor, tag string, style yaml_scalar_style_t) { + implicit := tag == "" + e.must(yaml_scalar_event_initialize(&e.event, []byte(anchor), []byte(tag), []byte(value), implicit, implicit, style)) + e.emit() +} diff --git a/tools/voluspa/vendor/gopkg.in/yaml.v2/parserc.go b/tools/voluspa/vendor/gopkg.in/yaml.v2/parserc.go new file mode 100644 index 00000000000..81d05dfe573 --- /dev/null +++ b/tools/voluspa/vendor/gopkg.in/yaml.v2/parserc.go @@ -0,0 +1,1095 @@ +package yaml + +import ( + "bytes" +) + +// The parser implements the following grammar: +// +// stream ::= STREAM-START implicit_document? explicit_document* STREAM-END +// implicit_document ::= block_node DOCUMENT-END* +// explicit_document ::= DIRECTIVE* DOCUMENT-START block_node? DOCUMENT-END* +// block_node_or_indentless_sequence ::= +// ALIAS +// | properties (block_content | indentless_block_sequence)? +// | block_content +// | indentless_block_sequence +// block_node ::= ALIAS +// | properties block_content? +// | block_content +// flow_node ::= ALIAS +// | properties flow_content? +// | flow_content +// properties ::= TAG ANCHOR? | ANCHOR TAG? +// block_content ::= block_collection | flow_collection | SCALAR +// flow_content ::= flow_collection | SCALAR +// block_collection ::= block_sequence | block_mapping +// flow_collection ::= flow_sequence | flow_mapping +// block_sequence ::= BLOCK-SEQUENCE-START (BLOCK-ENTRY block_node?)* BLOCK-END +// indentless_sequence ::= (BLOCK-ENTRY block_node?)+ +// block_mapping ::= BLOCK-MAPPING_START +// ((KEY block_node_or_indentless_sequence?)? +// (VALUE block_node_or_indentless_sequence?)?)* +// BLOCK-END +// flow_sequence ::= FLOW-SEQUENCE-START +// (flow_sequence_entry FLOW-ENTRY)* +// flow_sequence_entry? +// FLOW-SEQUENCE-END +// flow_sequence_entry ::= flow_node | KEY flow_node? (VALUE flow_node?)? +// flow_mapping ::= FLOW-MAPPING-START +// (flow_mapping_entry FLOW-ENTRY)* +// flow_mapping_entry? +// FLOW-MAPPING-END +// flow_mapping_entry ::= flow_node | KEY flow_node? (VALUE flow_node?)? + +// Peek the next token in the token queue. +func peek_token(parser *yaml_parser_t) *yaml_token_t { + if parser.token_available || yaml_parser_fetch_more_tokens(parser) { + return &parser.tokens[parser.tokens_head] + } + return nil +} + +// Remove the next token from the queue (must be called after peek_token). +func skip_token(parser *yaml_parser_t) { + parser.token_available = false + parser.tokens_parsed++ + parser.stream_end_produced = parser.tokens[parser.tokens_head].typ == yaml_STREAM_END_TOKEN + parser.tokens_head++ +} + +// Get the next event. +func yaml_parser_parse(parser *yaml_parser_t, event *yaml_event_t) bool { + // Erase the event object. + *event = yaml_event_t{} + + // No events after the end of the stream or error. + if parser.stream_end_produced || parser.error != yaml_NO_ERROR || parser.state == yaml_PARSE_END_STATE { + return true + } + + // Generate the next event. + return yaml_parser_state_machine(parser, event) +} + +// Set parser error. +func yaml_parser_set_parser_error(parser *yaml_parser_t, problem string, problem_mark yaml_mark_t) bool { + parser.error = yaml_PARSER_ERROR + parser.problem = problem + parser.problem_mark = problem_mark + return false +} + +func yaml_parser_set_parser_error_context(parser *yaml_parser_t, context string, context_mark yaml_mark_t, problem string, problem_mark yaml_mark_t) bool { + parser.error = yaml_PARSER_ERROR + parser.context = context + parser.context_mark = context_mark + parser.problem = problem + parser.problem_mark = problem_mark + return false +} + +// State dispatcher. +func yaml_parser_state_machine(parser *yaml_parser_t, event *yaml_event_t) bool { + //trace("yaml_parser_state_machine", "state:", parser.state.String()) + + switch parser.state { + case yaml_PARSE_STREAM_START_STATE: + return yaml_parser_parse_stream_start(parser, event) + + case yaml_PARSE_IMPLICIT_DOCUMENT_START_STATE: + return yaml_parser_parse_document_start(parser, event, true) + + case yaml_PARSE_DOCUMENT_START_STATE: + return yaml_parser_parse_document_start(parser, event, false) + + case yaml_PARSE_DOCUMENT_CONTENT_STATE: + return yaml_parser_parse_document_content(parser, event) + + case yaml_PARSE_DOCUMENT_END_STATE: + return yaml_parser_parse_document_end(parser, event) + + case yaml_PARSE_BLOCK_NODE_STATE: + return yaml_parser_parse_node(parser, event, true, false) + + case yaml_PARSE_BLOCK_NODE_OR_INDENTLESS_SEQUENCE_STATE: + return yaml_parser_parse_node(parser, event, true, true) + + case yaml_PARSE_FLOW_NODE_STATE: + return yaml_parser_parse_node(parser, event, false, false) + + case yaml_PARSE_BLOCK_SEQUENCE_FIRST_ENTRY_STATE: + return yaml_parser_parse_block_sequence_entry(parser, event, true) + + case yaml_PARSE_BLOCK_SEQUENCE_ENTRY_STATE: + return yaml_parser_parse_block_sequence_entry(parser, event, false) + + case yaml_PARSE_INDENTLESS_SEQUENCE_ENTRY_STATE: + return yaml_parser_parse_indentless_sequence_entry(parser, event) + + case yaml_PARSE_BLOCK_MAPPING_FIRST_KEY_STATE: + return yaml_parser_parse_block_mapping_key(parser, event, true) + + case yaml_PARSE_BLOCK_MAPPING_KEY_STATE: + return yaml_parser_parse_block_mapping_key(parser, event, false) + + case yaml_PARSE_BLOCK_MAPPING_VALUE_STATE: + return yaml_parser_parse_block_mapping_value(parser, event) + + case yaml_PARSE_FLOW_SEQUENCE_FIRST_ENTRY_STATE: + return yaml_parser_parse_flow_sequence_entry(parser, event, true) + + case yaml_PARSE_FLOW_SEQUENCE_ENTRY_STATE: + return yaml_parser_parse_flow_sequence_entry(parser, event, false) + + case yaml_PARSE_FLOW_SEQUENCE_ENTRY_MAPPING_KEY_STATE: + return yaml_parser_parse_flow_sequence_entry_mapping_key(parser, event) + + case yaml_PARSE_FLOW_SEQUENCE_ENTRY_MAPPING_VALUE_STATE: + return yaml_parser_parse_flow_sequence_entry_mapping_value(parser, event) + + case yaml_PARSE_FLOW_SEQUENCE_ENTRY_MAPPING_END_STATE: + return yaml_parser_parse_flow_sequence_entry_mapping_end(parser, event) + + case yaml_PARSE_FLOW_MAPPING_FIRST_KEY_STATE: + return yaml_parser_parse_flow_mapping_key(parser, event, true) + + case yaml_PARSE_FLOW_MAPPING_KEY_STATE: + return yaml_parser_parse_flow_mapping_key(parser, event, false) + + case yaml_PARSE_FLOW_MAPPING_VALUE_STATE: + return yaml_parser_parse_flow_mapping_value(parser, event, false) + + case yaml_PARSE_FLOW_MAPPING_EMPTY_VALUE_STATE: + return yaml_parser_parse_flow_mapping_value(parser, event, true) + + default: + panic("invalid parser state") + } +} + +// Parse the production: +// stream ::= STREAM-START implicit_document? explicit_document* STREAM-END +// ************ +func yaml_parser_parse_stream_start(parser *yaml_parser_t, event *yaml_event_t) bool { + token := peek_token(parser) + if token == nil { + return false + } + if token.typ != yaml_STREAM_START_TOKEN { + return yaml_parser_set_parser_error(parser, "did not find expected ", token.start_mark) + } + parser.state = yaml_PARSE_IMPLICIT_DOCUMENT_START_STATE + *event = yaml_event_t{ + typ: yaml_STREAM_START_EVENT, + start_mark: token.start_mark, + end_mark: token.end_mark, + encoding: token.encoding, + } + skip_token(parser) + return true +} + +// Parse the productions: +// implicit_document ::= block_node DOCUMENT-END* +// * +// explicit_document ::= DIRECTIVE* DOCUMENT-START block_node? DOCUMENT-END* +// ************************* +func yaml_parser_parse_document_start(parser *yaml_parser_t, event *yaml_event_t, implicit bool) bool { + + token := peek_token(parser) + if token == nil { + return false + } + + // Parse extra document end indicators. + if !implicit { + for token.typ == yaml_DOCUMENT_END_TOKEN { + skip_token(parser) + token = peek_token(parser) + if token == nil { + return false + } + } + } + + if implicit && token.typ != yaml_VERSION_DIRECTIVE_TOKEN && + token.typ != yaml_TAG_DIRECTIVE_TOKEN && + token.typ != yaml_DOCUMENT_START_TOKEN && + token.typ != yaml_STREAM_END_TOKEN { + // Parse an implicit document. + if !yaml_parser_process_directives(parser, nil, nil) { + return false + } + parser.states = append(parser.states, yaml_PARSE_DOCUMENT_END_STATE) + parser.state = yaml_PARSE_BLOCK_NODE_STATE + + *event = yaml_event_t{ + typ: yaml_DOCUMENT_START_EVENT, + start_mark: token.start_mark, + end_mark: token.end_mark, + } + + } else if token.typ != yaml_STREAM_END_TOKEN { + // Parse an explicit document. + var version_directive *yaml_version_directive_t + var tag_directives []yaml_tag_directive_t + start_mark := token.start_mark + if !yaml_parser_process_directives(parser, &version_directive, &tag_directives) { + return false + } + token = peek_token(parser) + if token == nil { + return false + } + if token.typ != yaml_DOCUMENT_START_TOKEN { + yaml_parser_set_parser_error(parser, + "did not find expected ", token.start_mark) + return false + } + parser.states = append(parser.states, yaml_PARSE_DOCUMENT_END_STATE) + parser.state = yaml_PARSE_DOCUMENT_CONTENT_STATE + end_mark := token.end_mark + + *event = yaml_event_t{ + typ: yaml_DOCUMENT_START_EVENT, + start_mark: start_mark, + end_mark: end_mark, + version_directive: version_directive, + tag_directives: tag_directives, + implicit: false, + } + skip_token(parser) + + } else { + // Parse the stream end. + parser.state = yaml_PARSE_END_STATE + *event = yaml_event_t{ + typ: yaml_STREAM_END_EVENT, + start_mark: token.start_mark, + end_mark: token.end_mark, + } + skip_token(parser) + } + + return true +} + +// Parse the productions: +// explicit_document ::= DIRECTIVE* DOCUMENT-START block_node? DOCUMENT-END* +// *********** +// +func yaml_parser_parse_document_content(parser *yaml_parser_t, event *yaml_event_t) bool { + token := peek_token(parser) + if token == nil { + return false + } + if token.typ == yaml_VERSION_DIRECTIVE_TOKEN || + token.typ == yaml_TAG_DIRECTIVE_TOKEN || + token.typ == yaml_DOCUMENT_START_TOKEN || + token.typ == yaml_DOCUMENT_END_TOKEN || + token.typ == yaml_STREAM_END_TOKEN { + parser.state = parser.states[len(parser.states)-1] + parser.states = parser.states[:len(parser.states)-1] + return yaml_parser_process_empty_scalar(parser, event, + token.start_mark) + } + return yaml_parser_parse_node(parser, event, true, false) +} + +// Parse the productions: +// implicit_document ::= block_node DOCUMENT-END* +// ************* +// explicit_document ::= DIRECTIVE* DOCUMENT-START block_node? DOCUMENT-END* +// +func yaml_parser_parse_document_end(parser *yaml_parser_t, event *yaml_event_t) bool { + token := peek_token(parser) + if token == nil { + return false + } + + start_mark := token.start_mark + end_mark := token.start_mark + + implicit := true + if token.typ == yaml_DOCUMENT_END_TOKEN { + end_mark = token.end_mark + skip_token(parser) + implicit = false + } + + parser.tag_directives = parser.tag_directives[:0] + + parser.state = yaml_PARSE_DOCUMENT_START_STATE + *event = yaml_event_t{ + typ: yaml_DOCUMENT_END_EVENT, + start_mark: start_mark, + end_mark: end_mark, + implicit: implicit, + } + return true +} + +// Parse the productions: +// block_node_or_indentless_sequence ::= +// ALIAS +// ***** +// | properties (block_content | indentless_block_sequence)? +// ********** * +// | block_content | indentless_block_sequence +// * +// block_node ::= ALIAS +// ***** +// | properties block_content? +// ********** * +// | block_content +// * +// flow_node ::= ALIAS +// ***** +// | properties flow_content? +// ********** * +// | flow_content +// * +// properties ::= TAG ANCHOR? | ANCHOR TAG? +// ************************* +// block_content ::= block_collection | flow_collection | SCALAR +// ****** +// flow_content ::= flow_collection | SCALAR +// ****** +func yaml_parser_parse_node(parser *yaml_parser_t, event *yaml_event_t, block, indentless_sequence bool) bool { + //defer trace("yaml_parser_parse_node", "block:", block, "indentless_sequence:", indentless_sequence)() + + token := peek_token(parser) + if token == nil { + return false + } + + if token.typ == yaml_ALIAS_TOKEN { + parser.state = parser.states[len(parser.states)-1] + parser.states = parser.states[:len(parser.states)-1] + *event = yaml_event_t{ + typ: yaml_ALIAS_EVENT, + start_mark: token.start_mark, + end_mark: token.end_mark, + anchor: token.value, + } + skip_token(parser) + return true + } + + start_mark := token.start_mark + end_mark := token.start_mark + + var tag_token bool + var tag_handle, tag_suffix, anchor []byte + var tag_mark yaml_mark_t + if token.typ == yaml_ANCHOR_TOKEN { + anchor = token.value + start_mark = token.start_mark + end_mark = token.end_mark + skip_token(parser) + token = peek_token(parser) + if token == nil { + return false + } + if token.typ == yaml_TAG_TOKEN { + tag_token = true + tag_handle = token.value + tag_suffix = token.suffix + tag_mark = token.start_mark + end_mark = token.end_mark + skip_token(parser) + token = peek_token(parser) + if token == nil { + return false + } + } + } else if token.typ == yaml_TAG_TOKEN { + tag_token = true + tag_handle = token.value + tag_suffix = token.suffix + start_mark = token.start_mark + tag_mark = token.start_mark + end_mark = token.end_mark + skip_token(parser) + token = peek_token(parser) + if token == nil { + return false + } + if token.typ == yaml_ANCHOR_TOKEN { + anchor = token.value + end_mark = token.end_mark + skip_token(parser) + token = peek_token(parser) + if token == nil { + return false + } + } + } + + var tag []byte + if tag_token { + if len(tag_handle) == 0 { + tag = tag_suffix + tag_suffix = nil + } else { + for i := range parser.tag_directives { + if bytes.Equal(parser.tag_directives[i].handle, tag_handle) { + tag = append([]byte(nil), parser.tag_directives[i].prefix...) + tag = append(tag, tag_suffix...) + break + } + } + if len(tag) == 0 { + yaml_parser_set_parser_error_context(parser, + "while parsing a node", start_mark, + "found undefined tag handle", tag_mark) + return false + } + } + } + + implicit := len(tag) == 0 + if indentless_sequence && token.typ == yaml_BLOCK_ENTRY_TOKEN { + end_mark = token.end_mark + parser.state = yaml_PARSE_INDENTLESS_SEQUENCE_ENTRY_STATE + *event = yaml_event_t{ + typ: yaml_SEQUENCE_START_EVENT, + start_mark: start_mark, + end_mark: end_mark, + anchor: anchor, + tag: tag, + implicit: implicit, + style: yaml_style_t(yaml_BLOCK_SEQUENCE_STYLE), + } + return true + } + if token.typ == yaml_SCALAR_TOKEN { + var plain_implicit, quoted_implicit bool + end_mark = token.end_mark + if (len(tag) == 0 && token.style == yaml_PLAIN_SCALAR_STYLE) || (len(tag) == 1 && tag[0] == '!') { + plain_implicit = true + } else if len(tag) == 0 { + quoted_implicit = true + } + parser.state = parser.states[len(parser.states)-1] + parser.states = parser.states[:len(parser.states)-1] + + *event = yaml_event_t{ + typ: yaml_SCALAR_EVENT, + start_mark: start_mark, + end_mark: end_mark, + anchor: anchor, + tag: tag, + value: token.value, + implicit: plain_implicit, + quoted_implicit: quoted_implicit, + style: yaml_style_t(token.style), + } + skip_token(parser) + return true + } + if token.typ == yaml_FLOW_SEQUENCE_START_TOKEN { + // [Go] Some of the events below can be merged as they differ only on style. + end_mark = token.end_mark + parser.state = yaml_PARSE_FLOW_SEQUENCE_FIRST_ENTRY_STATE + *event = yaml_event_t{ + typ: yaml_SEQUENCE_START_EVENT, + start_mark: start_mark, + end_mark: end_mark, + anchor: anchor, + tag: tag, + implicit: implicit, + style: yaml_style_t(yaml_FLOW_SEQUENCE_STYLE), + } + return true + } + if token.typ == yaml_FLOW_MAPPING_START_TOKEN { + end_mark = token.end_mark + parser.state = yaml_PARSE_FLOW_MAPPING_FIRST_KEY_STATE + *event = yaml_event_t{ + typ: yaml_MAPPING_START_EVENT, + start_mark: start_mark, + end_mark: end_mark, + anchor: anchor, + tag: tag, + implicit: implicit, + style: yaml_style_t(yaml_FLOW_MAPPING_STYLE), + } + return true + } + if block && token.typ == yaml_BLOCK_SEQUENCE_START_TOKEN { + end_mark = token.end_mark + parser.state = yaml_PARSE_BLOCK_SEQUENCE_FIRST_ENTRY_STATE + *event = yaml_event_t{ + typ: yaml_SEQUENCE_START_EVENT, + start_mark: start_mark, + end_mark: end_mark, + anchor: anchor, + tag: tag, + implicit: implicit, + style: yaml_style_t(yaml_BLOCK_SEQUENCE_STYLE), + } + return true + } + if block && token.typ == yaml_BLOCK_MAPPING_START_TOKEN { + end_mark = token.end_mark + parser.state = yaml_PARSE_BLOCK_MAPPING_FIRST_KEY_STATE + *event = yaml_event_t{ + typ: yaml_MAPPING_START_EVENT, + start_mark: start_mark, + end_mark: end_mark, + anchor: anchor, + tag: tag, + implicit: implicit, + style: yaml_style_t(yaml_BLOCK_MAPPING_STYLE), + } + return true + } + if len(anchor) > 0 || len(tag) > 0 { + parser.state = parser.states[len(parser.states)-1] + parser.states = parser.states[:len(parser.states)-1] + + *event = yaml_event_t{ + typ: yaml_SCALAR_EVENT, + start_mark: start_mark, + end_mark: end_mark, + anchor: anchor, + tag: tag, + implicit: implicit, + quoted_implicit: false, + style: yaml_style_t(yaml_PLAIN_SCALAR_STYLE), + } + return true + } + + context := "while parsing a flow node" + if block { + context = "while parsing a block node" + } + yaml_parser_set_parser_error_context(parser, context, start_mark, + "did not find expected node content", token.start_mark) + return false +} + +// Parse the productions: +// block_sequence ::= BLOCK-SEQUENCE-START (BLOCK-ENTRY block_node?)* BLOCK-END +// ******************** *********** * ********* +// +func yaml_parser_parse_block_sequence_entry(parser *yaml_parser_t, event *yaml_event_t, first bool) bool { + if first { + token := peek_token(parser) + parser.marks = append(parser.marks, token.start_mark) + skip_token(parser) + } + + token := peek_token(parser) + if token == nil { + return false + } + + if token.typ == yaml_BLOCK_ENTRY_TOKEN { + mark := token.end_mark + skip_token(parser) + token = peek_token(parser) + if token == nil { + return false + } + if token.typ != yaml_BLOCK_ENTRY_TOKEN && token.typ != yaml_BLOCK_END_TOKEN { + parser.states = append(parser.states, yaml_PARSE_BLOCK_SEQUENCE_ENTRY_STATE) + return yaml_parser_parse_node(parser, event, true, false) + } else { + parser.state = yaml_PARSE_BLOCK_SEQUENCE_ENTRY_STATE + return yaml_parser_process_empty_scalar(parser, event, mark) + } + } + if token.typ == yaml_BLOCK_END_TOKEN { + parser.state = parser.states[len(parser.states)-1] + parser.states = parser.states[:len(parser.states)-1] + parser.marks = parser.marks[:len(parser.marks)-1] + + *event = yaml_event_t{ + typ: yaml_SEQUENCE_END_EVENT, + start_mark: token.start_mark, + end_mark: token.end_mark, + } + + skip_token(parser) + return true + } + + context_mark := parser.marks[len(parser.marks)-1] + parser.marks = parser.marks[:len(parser.marks)-1] + return yaml_parser_set_parser_error_context(parser, + "while parsing a block collection", context_mark, + "did not find expected '-' indicator", token.start_mark) +} + +// Parse the productions: +// indentless_sequence ::= (BLOCK-ENTRY block_node?)+ +// *********** * +func yaml_parser_parse_indentless_sequence_entry(parser *yaml_parser_t, event *yaml_event_t) bool { + token := peek_token(parser) + if token == nil { + return false + } + + if token.typ == yaml_BLOCK_ENTRY_TOKEN { + mark := token.end_mark + skip_token(parser) + token = peek_token(parser) + if token == nil { + return false + } + if token.typ != yaml_BLOCK_ENTRY_TOKEN && + token.typ != yaml_KEY_TOKEN && + token.typ != yaml_VALUE_TOKEN && + token.typ != yaml_BLOCK_END_TOKEN { + parser.states = append(parser.states, yaml_PARSE_INDENTLESS_SEQUENCE_ENTRY_STATE) + return yaml_parser_parse_node(parser, event, true, false) + } + parser.state = yaml_PARSE_INDENTLESS_SEQUENCE_ENTRY_STATE + return yaml_parser_process_empty_scalar(parser, event, mark) + } + parser.state = parser.states[len(parser.states)-1] + parser.states = parser.states[:len(parser.states)-1] + + *event = yaml_event_t{ + typ: yaml_SEQUENCE_END_EVENT, + start_mark: token.start_mark, + end_mark: token.start_mark, // [Go] Shouldn't this be token.end_mark? + } + return true +} + +// Parse the productions: +// block_mapping ::= BLOCK-MAPPING_START +// ******************* +// ((KEY block_node_or_indentless_sequence?)? +// *** * +// (VALUE block_node_or_indentless_sequence?)?)* +// +// BLOCK-END +// ********* +// +func yaml_parser_parse_block_mapping_key(parser *yaml_parser_t, event *yaml_event_t, first bool) bool { + if first { + token := peek_token(parser) + parser.marks = append(parser.marks, token.start_mark) + skip_token(parser) + } + + token := peek_token(parser) + if token == nil { + return false + } + + if token.typ == yaml_KEY_TOKEN { + mark := token.end_mark + skip_token(parser) + token = peek_token(parser) + if token == nil { + return false + } + if token.typ != yaml_KEY_TOKEN && + token.typ != yaml_VALUE_TOKEN && + token.typ != yaml_BLOCK_END_TOKEN { + parser.states = append(parser.states, yaml_PARSE_BLOCK_MAPPING_VALUE_STATE) + return yaml_parser_parse_node(parser, event, true, true) + } else { + parser.state = yaml_PARSE_BLOCK_MAPPING_VALUE_STATE + return yaml_parser_process_empty_scalar(parser, event, mark) + } + } else if token.typ == yaml_BLOCK_END_TOKEN { + parser.state = parser.states[len(parser.states)-1] + parser.states = parser.states[:len(parser.states)-1] + parser.marks = parser.marks[:len(parser.marks)-1] + *event = yaml_event_t{ + typ: yaml_MAPPING_END_EVENT, + start_mark: token.start_mark, + end_mark: token.end_mark, + } + skip_token(parser) + return true + } + + context_mark := parser.marks[len(parser.marks)-1] + parser.marks = parser.marks[:len(parser.marks)-1] + return yaml_parser_set_parser_error_context(parser, + "while parsing a block mapping", context_mark, + "did not find expected key", token.start_mark) +} + +// Parse the productions: +// block_mapping ::= BLOCK-MAPPING_START +// +// ((KEY block_node_or_indentless_sequence?)? +// +// (VALUE block_node_or_indentless_sequence?)?)* +// ***** * +// BLOCK-END +// +// +func yaml_parser_parse_block_mapping_value(parser *yaml_parser_t, event *yaml_event_t) bool { + token := peek_token(parser) + if token == nil { + return false + } + if token.typ == yaml_VALUE_TOKEN { + mark := token.end_mark + skip_token(parser) + token = peek_token(parser) + if token == nil { + return false + } + if token.typ != yaml_KEY_TOKEN && + token.typ != yaml_VALUE_TOKEN && + token.typ != yaml_BLOCK_END_TOKEN { + parser.states = append(parser.states, yaml_PARSE_BLOCK_MAPPING_KEY_STATE) + return yaml_parser_parse_node(parser, event, true, true) + } + parser.state = yaml_PARSE_BLOCK_MAPPING_KEY_STATE + return yaml_parser_process_empty_scalar(parser, event, mark) + } + parser.state = yaml_PARSE_BLOCK_MAPPING_KEY_STATE + return yaml_parser_process_empty_scalar(parser, event, token.start_mark) +} + +// Parse the productions: +// flow_sequence ::= FLOW-SEQUENCE-START +// ******************* +// (flow_sequence_entry FLOW-ENTRY)* +// * ********** +// flow_sequence_entry? +// * +// FLOW-SEQUENCE-END +// ***************** +// flow_sequence_entry ::= flow_node | KEY flow_node? (VALUE flow_node?)? +// * +// +func yaml_parser_parse_flow_sequence_entry(parser *yaml_parser_t, event *yaml_event_t, first bool) bool { + if first { + token := peek_token(parser) + parser.marks = append(parser.marks, token.start_mark) + skip_token(parser) + } + token := peek_token(parser) + if token == nil { + return false + } + if token.typ != yaml_FLOW_SEQUENCE_END_TOKEN { + if !first { + if token.typ == yaml_FLOW_ENTRY_TOKEN { + skip_token(parser) + token = peek_token(parser) + if token == nil { + return false + } + } else { + context_mark := parser.marks[len(parser.marks)-1] + parser.marks = parser.marks[:len(parser.marks)-1] + return yaml_parser_set_parser_error_context(parser, + "while parsing a flow sequence", context_mark, + "did not find expected ',' or ']'", token.start_mark) + } + } + + if token.typ == yaml_KEY_TOKEN { + parser.state = yaml_PARSE_FLOW_SEQUENCE_ENTRY_MAPPING_KEY_STATE + *event = yaml_event_t{ + typ: yaml_MAPPING_START_EVENT, + start_mark: token.start_mark, + end_mark: token.end_mark, + implicit: true, + style: yaml_style_t(yaml_FLOW_MAPPING_STYLE), + } + skip_token(parser) + return true + } else if token.typ != yaml_FLOW_SEQUENCE_END_TOKEN { + parser.states = append(parser.states, yaml_PARSE_FLOW_SEQUENCE_ENTRY_STATE) + return yaml_parser_parse_node(parser, event, false, false) + } + } + + parser.state = parser.states[len(parser.states)-1] + parser.states = parser.states[:len(parser.states)-1] + parser.marks = parser.marks[:len(parser.marks)-1] + + *event = yaml_event_t{ + typ: yaml_SEQUENCE_END_EVENT, + start_mark: token.start_mark, + end_mark: token.end_mark, + } + + skip_token(parser) + return true +} + +// +// Parse the productions: +// flow_sequence_entry ::= flow_node | KEY flow_node? (VALUE flow_node?)? +// *** * +// +func yaml_parser_parse_flow_sequence_entry_mapping_key(parser *yaml_parser_t, event *yaml_event_t) bool { + token := peek_token(parser) + if token == nil { + return false + } + if token.typ != yaml_VALUE_TOKEN && + token.typ != yaml_FLOW_ENTRY_TOKEN && + token.typ != yaml_FLOW_SEQUENCE_END_TOKEN { + parser.states = append(parser.states, yaml_PARSE_FLOW_SEQUENCE_ENTRY_MAPPING_VALUE_STATE) + return yaml_parser_parse_node(parser, event, false, false) + } + mark := token.end_mark + skip_token(parser) + parser.state = yaml_PARSE_FLOW_SEQUENCE_ENTRY_MAPPING_VALUE_STATE + return yaml_parser_process_empty_scalar(parser, event, mark) +} + +// Parse the productions: +// flow_sequence_entry ::= flow_node | KEY flow_node? (VALUE flow_node?)? +// ***** * +// +func yaml_parser_parse_flow_sequence_entry_mapping_value(parser *yaml_parser_t, event *yaml_event_t) bool { + token := peek_token(parser) + if token == nil { + return false + } + if token.typ == yaml_VALUE_TOKEN { + skip_token(parser) + token := peek_token(parser) + if token == nil { + return false + } + if token.typ != yaml_FLOW_ENTRY_TOKEN && token.typ != yaml_FLOW_SEQUENCE_END_TOKEN { + parser.states = append(parser.states, yaml_PARSE_FLOW_SEQUENCE_ENTRY_MAPPING_END_STATE) + return yaml_parser_parse_node(parser, event, false, false) + } + } + parser.state = yaml_PARSE_FLOW_SEQUENCE_ENTRY_MAPPING_END_STATE + return yaml_parser_process_empty_scalar(parser, event, token.start_mark) +} + +// Parse the productions: +// flow_sequence_entry ::= flow_node | KEY flow_node? (VALUE flow_node?)? +// * +// +func yaml_parser_parse_flow_sequence_entry_mapping_end(parser *yaml_parser_t, event *yaml_event_t) bool { + token := peek_token(parser) + if token == nil { + return false + } + parser.state = yaml_PARSE_FLOW_SEQUENCE_ENTRY_STATE + *event = yaml_event_t{ + typ: yaml_MAPPING_END_EVENT, + start_mark: token.start_mark, + end_mark: token.start_mark, // [Go] Shouldn't this be end_mark? + } + return true +} + +// Parse the productions: +// flow_mapping ::= FLOW-MAPPING-START +// ****************** +// (flow_mapping_entry FLOW-ENTRY)* +// * ********** +// flow_mapping_entry? +// ****************** +// FLOW-MAPPING-END +// **************** +// flow_mapping_entry ::= flow_node | KEY flow_node? (VALUE flow_node?)? +// * *** * +// +func yaml_parser_parse_flow_mapping_key(parser *yaml_parser_t, event *yaml_event_t, first bool) bool { + if first { + token := peek_token(parser) + parser.marks = append(parser.marks, token.start_mark) + skip_token(parser) + } + + token := peek_token(parser) + if token == nil { + return false + } + + if token.typ != yaml_FLOW_MAPPING_END_TOKEN { + if !first { + if token.typ == yaml_FLOW_ENTRY_TOKEN { + skip_token(parser) + token = peek_token(parser) + if token == nil { + return false + } + } else { + context_mark := parser.marks[len(parser.marks)-1] + parser.marks = parser.marks[:len(parser.marks)-1] + return yaml_parser_set_parser_error_context(parser, + "while parsing a flow mapping", context_mark, + "did not find expected ',' or '}'", token.start_mark) + } + } + + if token.typ == yaml_KEY_TOKEN { + skip_token(parser) + token = peek_token(parser) + if token == nil { + return false + } + if token.typ != yaml_VALUE_TOKEN && + token.typ != yaml_FLOW_ENTRY_TOKEN && + token.typ != yaml_FLOW_MAPPING_END_TOKEN { + parser.states = append(parser.states, yaml_PARSE_FLOW_MAPPING_VALUE_STATE) + return yaml_parser_parse_node(parser, event, false, false) + } else { + parser.state = yaml_PARSE_FLOW_MAPPING_VALUE_STATE + return yaml_parser_process_empty_scalar(parser, event, token.start_mark) + } + } else if token.typ != yaml_FLOW_MAPPING_END_TOKEN { + parser.states = append(parser.states, yaml_PARSE_FLOW_MAPPING_EMPTY_VALUE_STATE) + return yaml_parser_parse_node(parser, event, false, false) + } + } + + parser.state = parser.states[len(parser.states)-1] + parser.states = parser.states[:len(parser.states)-1] + parser.marks = parser.marks[:len(parser.marks)-1] + *event = yaml_event_t{ + typ: yaml_MAPPING_END_EVENT, + start_mark: token.start_mark, + end_mark: token.end_mark, + } + skip_token(parser) + return true +} + +// Parse the productions: +// flow_mapping_entry ::= flow_node | KEY flow_node? (VALUE flow_node?)? +// * ***** * +// +func yaml_parser_parse_flow_mapping_value(parser *yaml_parser_t, event *yaml_event_t, empty bool) bool { + token := peek_token(parser) + if token == nil { + return false + } + if empty { + parser.state = yaml_PARSE_FLOW_MAPPING_KEY_STATE + return yaml_parser_process_empty_scalar(parser, event, token.start_mark) + } + if token.typ == yaml_VALUE_TOKEN { + skip_token(parser) + token = peek_token(parser) + if token == nil { + return false + } + if token.typ != yaml_FLOW_ENTRY_TOKEN && token.typ != yaml_FLOW_MAPPING_END_TOKEN { + parser.states = append(parser.states, yaml_PARSE_FLOW_MAPPING_KEY_STATE) + return yaml_parser_parse_node(parser, event, false, false) + } + } + parser.state = yaml_PARSE_FLOW_MAPPING_KEY_STATE + return yaml_parser_process_empty_scalar(parser, event, token.start_mark) +} + +// Generate an empty scalar event. +func yaml_parser_process_empty_scalar(parser *yaml_parser_t, event *yaml_event_t, mark yaml_mark_t) bool { + *event = yaml_event_t{ + typ: yaml_SCALAR_EVENT, + start_mark: mark, + end_mark: mark, + value: nil, // Empty + implicit: true, + style: yaml_style_t(yaml_PLAIN_SCALAR_STYLE), + } + return true +} + +var default_tag_directives = []yaml_tag_directive_t{ + {[]byte("!"), []byte("!")}, + {[]byte("!!"), []byte("tag:yaml.org,2002:")}, +} + +// Parse directives. +func yaml_parser_process_directives(parser *yaml_parser_t, + version_directive_ref **yaml_version_directive_t, + tag_directives_ref *[]yaml_tag_directive_t) bool { + + var version_directive *yaml_version_directive_t + var tag_directives []yaml_tag_directive_t + + token := peek_token(parser) + if token == nil { + return false + } + + for token.typ == yaml_VERSION_DIRECTIVE_TOKEN || token.typ == yaml_TAG_DIRECTIVE_TOKEN { + if token.typ == yaml_VERSION_DIRECTIVE_TOKEN { + if version_directive != nil { + yaml_parser_set_parser_error(parser, + "found duplicate %YAML directive", token.start_mark) + return false + } + if token.major != 1 || token.minor != 1 { + yaml_parser_set_parser_error(parser, + "found incompatible YAML document", token.start_mark) + return false + } + version_directive = &yaml_version_directive_t{ + major: token.major, + minor: token.minor, + } + } else if token.typ == yaml_TAG_DIRECTIVE_TOKEN { + value := yaml_tag_directive_t{ + handle: token.value, + prefix: token.prefix, + } + if !yaml_parser_append_tag_directive(parser, value, false, token.start_mark) { + return false + } + tag_directives = append(tag_directives, value) + } + + skip_token(parser) + token = peek_token(parser) + if token == nil { + return false + } + } + + for i := range default_tag_directives { + if !yaml_parser_append_tag_directive(parser, default_tag_directives[i], true, token.start_mark) { + return false + } + } + + if version_directive_ref != nil { + *version_directive_ref = version_directive + } + if tag_directives_ref != nil { + *tag_directives_ref = tag_directives + } + return true +} + +// Append a tag directive to the directives stack. +func yaml_parser_append_tag_directive(parser *yaml_parser_t, value yaml_tag_directive_t, allow_duplicates bool, mark yaml_mark_t) bool { + for i := range parser.tag_directives { + if bytes.Equal(value.handle, parser.tag_directives[i].handle) { + if allow_duplicates { + return true + } + return yaml_parser_set_parser_error(parser, "found duplicate %TAG directive", mark) + } + } + + // [Go] I suspect the copy is unnecessary. This was likely done + // because there was no way to track ownership of the data. + value_copy := yaml_tag_directive_t{ + handle: make([]byte, len(value.handle)), + prefix: make([]byte, len(value.prefix)), + } + copy(value_copy.handle, value.handle) + copy(value_copy.prefix, value.prefix) + parser.tag_directives = append(parser.tag_directives, value_copy) + return true +} diff --git a/tools/voluspa/vendor/gopkg.in/yaml.v2/readerc.go b/tools/voluspa/vendor/gopkg.in/yaml.v2/readerc.go new file mode 100644 index 00000000000..7c1f5fac3db --- /dev/null +++ b/tools/voluspa/vendor/gopkg.in/yaml.v2/readerc.go @@ -0,0 +1,412 @@ +package yaml + +import ( + "io" +) + +// Set the reader error and return 0. +func yaml_parser_set_reader_error(parser *yaml_parser_t, problem string, offset int, value int) bool { + parser.error = yaml_READER_ERROR + parser.problem = problem + parser.problem_offset = offset + parser.problem_value = value + return false +} + +// Byte order marks. +const ( + bom_UTF8 = "\xef\xbb\xbf" + bom_UTF16LE = "\xff\xfe" + bom_UTF16BE = "\xfe\xff" +) + +// Determine the input stream encoding by checking the BOM symbol. If no BOM is +// found, the UTF-8 encoding is assumed. Return 1 on success, 0 on failure. +func yaml_parser_determine_encoding(parser *yaml_parser_t) bool { + // Ensure that we had enough bytes in the raw buffer. + for !parser.eof && len(parser.raw_buffer)-parser.raw_buffer_pos < 3 { + if !yaml_parser_update_raw_buffer(parser) { + return false + } + } + + // Determine the encoding. + buf := parser.raw_buffer + pos := parser.raw_buffer_pos + avail := len(buf) - pos + if avail >= 2 && buf[pos] == bom_UTF16LE[0] && buf[pos+1] == bom_UTF16LE[1] { + parser.encoding = yaml_UTF16LE_ENCODING + parser.raw_buffer_pos += 2 + parser.offset += 2 + } else if avail >= 2 && buf[pos] == bom_UTF16BE[0] && buf[pos+1] == bom_UTF16BE[1] { + parser.encoding = yaml_UTF16BE_ENCODING + parser.raw_buffer_pos += 2 + parser.offset += 2 + } else if avail >= 3 && buf[pos] == bom_UTF8[0] && buf[pos+1] == bom_UTF8[1] && buf[pos+2] == bom_UTF8[2] { + parser.encoding = yaml_UTF8_ENCODING + parser.raw_buffer_pos += 3 + parser.offset += 3 + } else { + parser.encoding = yaml_UTF8_ENCODING + } + return true +} + +// Update the raw buffer. +func yaml_parser_update_raw_buffer(parser *yaml_parser_t) bool { + size_read := 0 + + // Return if the raw buffer is full. + if parser.raw_buffer_pos == 0 && len(parser.raw_buffer) == cap(parser.raw_buffer) { + return true + } + + // Return on EOF. + if parser.eof { + return true + } + + // Move the remaining bytes in the raw buffer to the beginning. + if parser.raw_buffer_pos > 0 && parser.raw_buffer_pos < len(parser.raw_buffer) { + copy(parser.raw_buffer, parser.raw_buffer[parser.raw_buffer_pos:]) + } + parser.raw_buffer = parser.raw_buffer[:len(parser.raw_buffer)-parser.raw_buffer_pos] + parser.raw_buffer_pos = 0 + + // Call the read handler to fill the buffer. + size_read, err := parser.read_handler(parser, parser.raw_buffer[len(parser.raw_buffer):cap(parser.raw_buffer)]) + parser.raw_buffer = parser.raw_buffer[:len(parser.raw_buffer)+size_read] + if err == io.EOF { + parser.eof = true + } else if err != nil { + return yaml_parser_set_reader_error(parser, "input error: "+err.Error(), parser.offset, -1) + } + return true +} + +// Ensure that the buffer contains at least `length` characters. +// Return true on success, false on failure. +// +// The length is supposed to be significantly less that the buffer size. +func yaml_parser_update_buffer(parser *yaml_parser_t, length int) bool { + if parser.read_handler == nil { + panic("read handler must be set") + } + + // [Go] This function was changed to guarantee the requested length size at EOF. + // The fact we need to do this is pretty awful, but the description above implies + // for that to be the case, and there are tests + + // If the EOF flag is set and the raw buffer is empty, do nothing. + if parser.eof && parser.raw_buffer_pos == len(parser.raw_buffer) { + // [Go] ACTUALLY! Read the documentation of this function above. + // This is just broken. To return true, we need to have the + // given length in the buffer. Not doing that means every single + // check that calls this function to make sure the buffer has a + // given length is Go) panicking; or C) accessing invalid memory. + //return true + } + + // Return if the buffer contains enough characters. + if parser.unread >= length { + return true + } + + // Determine the input encoding if it is not known yet. + if parser.encoding == yaml_ANY_ENCODING { + if !yaml_parser_determine_encoding(parser) { + return false + } + } + + // Move the unread characters to the beginning of the buffer. + buffer_len := len(parser.buffer) + if parser.buffer_pos > 0 && parser.buffer_pos < buffer_len { + copy(parser.buffer, parser.buffer[parser.buffer_pos:]) + buffer_len -= parser.buffer_pos + parser.buffer_pos = 0 + } else if parser.buffer_pos == buffer_len { + buffer_len = 0 + parser.buffer_pos = 0 + } + + // Open the whole buffer for writing, and cut it before returning. + parser.buffer = parser.buffer[:cap(parser.buffer)] + + // Fill the buffer until it has enough characters. + first := true + for parser.unread < length { + + // Fill the raw buffer if necessary. + if !first || parser.raw_buffer_pos == len(parser.raw_buffer) { + if !yaml_parser_update_raw_buffer(parser) { + parser.buffer = parser.buffer[:buffer_len] + return false + } + } + first = false + + // Decode the raw buffer. + inner: + for parser.raw_buffer_pos != len(parser.raw_buffer) { + var value rune + var width int + + raw_unread := len(parser.raw_buffer) - parser.raw_buffer_pos + + // Decode the next character. + switch parser.encoding { + case yaml_UTF8_ENCODING: + // Decode a UTF-8 character. Check RFC 3629 + // (http://www.ietf.org/rfc/rfc3629.txt) for more details. + // + // The following table (taken from the RFC) is used for + // decoding. + // + // Char. number range | UTF-8 octet sequence + // (hexadecimal) | (binary) + // --------------------+------------------------------------ + // 0000 0000-0000 007F | 0xxxxxxx + // 0000 0080-0000 07FF | 110xxxxx 10xxxxxx + // 0000 0800-0000 FFFF | 1110xxxx 10xxxxxx 10xxxxxx + // 0001 0000-0010 FFFF | 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx + // + // Additionally, the characters in the range 0xD800-0xDFFF + // are prohibited as they are reserved for use with UTF-16 + // surrogate pairs. + + // Determine the length of the UTF-8 sequence. + octet := parser.raw_buffer[parser.raw_buffer_pos] + switch { + case octet&0x80 == 0x00: + width = 1 + case octet&0xE0 == 0xC0: + width = 2 + case octet&0xF0 == 0xE0: + width = 3 + case octet&0xF8 == 0xF0: + width = 4 + default: + // The leading octet is invalid. + return yaml_parser_set_reader_error(parser, + "invalid leading UTF-8 octet", + parser.offset, int(octet)) + } + + // Check if the raw buffer contains an incomplete character. + if width > raw_unread { + if parser.eof { + return yaml_parser_set_reader_error(parser, + "incomplete UTF-8 octet sequence", + parser.offset, -1) + } + break inner + } + + // Decode the leading octet. + switch { + case octet&0x80 == 0x00: + value = rune(octet & 0x7F) + case octet&0xE0 == 0xC0: + value = rune(octet & 0x1F) + case octet&0xF0 == 0xE0: + value = rune(octet & 0x0F) + case octet&0xF8 == 0xF0: + value = rune(octet & 0x07) + default: + value = 0 + } + + // Check and decode the trailing octets. + for k := 1; k < width; k++ { + octet = parser.raw_buffer[parser.raw_buffer_pos+k] + + // Check if the octet is valid. + if (octet & 0xC0) != 0x80 { + return yaml_parser_set_reader_error(parser, + "invalid trailing UTF-8 octet", + parser.offset+k, int(octet)) + } + + // Decode the octet. + value = (value << 6) + rune(octet&0x3F) + } + + // Check the length of the sequence against the value. + switch { + case width == 1: + case width == 2 && value >= 0x80: + case width == 3 && value >= 0x800: + case width == 4 && value >= 0x10000: + default: + return yaml_parser_set_reader_error(parser, + "invalid length of a UTF-8 sequence", + parser.offset, -1) + } + + // Check the range of the value. + if value >= 0xD800 && value <= 0xDFFF || value > 0x10FFFF { + return yaml_parser_set_reader_error(parser, + "invalid Unicode character", + parser.offset, int(value)) + } + + case yaml_UTF16LE_ENCODING, yaml_UTF16BE_ENCODING: + var low, high int + if parser.encoding == yaml_UTF16LE_ENCODING { + low, high = 0, 1 + } else { + low, high = 1, 0 + } + + // The UTF-16 encoding is not as simple as one might + // naively think. Check RFC 2781 + // (http://www.ietf.org/rfc/rfc2781.txt). + // + // Normally, two subsequent bytes describe a Unicode + // character. However a special technique (called a + // surrogate pair) is used for specifying character + // values larger than 0xFFFF. + // + // A surrogate pair consists of two pseudo-characters: + // high surrogate area (0xD800-0xDBFF) + // low surrogate area (0xDC00-0xDFFF) + // + // The following formulas are used for decoding + // and encoding characters using surrogate pairs: + // + // U = U' + 0x10000 (0x01 00 00 <= U <= 0x10 FF FF) + // U' = yyyyyyyyyyxxxxxxxxxx (0 <= U' <= 0x0F FF FF) + // W1 = 110110yyyyyyyyyy + // W2 = 110111xxxxxxxxxx + // + // where U is the character value, W1 is the high surrogate + // area, W2 is the low surrogate area. + + // Check for incomplete UTF-16 character. + if raw_unread < 2 { + if parser.eof { + return yaml_parser_set_reader_error(parser, + "incomplete UTF-16 character", + parser.offset, -1) + } + break inner + } + + // Get the character. + value = rune(parser.raw_buffer[parser.raw_buffer_pos+low]) + + (rune(parser.raw_buffer[parser.raw_buffer_pos+high]) << 8) + + // Check for unexpected low surrogate area. + if value&0xFC00 == 0xDC00 { + return yaml_parser_set_reader_error(parser, + "unexpected low surrogate area", + parser.offset, int(value)) + } + + // Check for a high surrogate area. + if value&0xFC00 == 0xD800 { + width = 4 + + // Check for incomplete surrogate pair. + if raw_unread < 4 { + if parser.eof { + return yaml_parser_set_reader_error(parser, + "incomplete UTF-16 surrogate pair", + parser.offset, -1) + } + break inner + } + + // Get the next character. + value2 := rune(parser.raw_buffer[parser.raw_buffer_pos+low+2]) + + (rune(parser.raw_buffer[parser.raw_buffer_pos+high+2]) << 8) + + // Check for a low surrogate area. + if value2&0xFC00 != 0xDC00 { + return yaml_parser_set_reader_error(parser, + "expected low surrogate area", + parser.offset+2, int(value2)) + } + + // Generate the value of the surrogate pair. + value = 0x10000 + ((value & 0x3FF) << 10) + (value2 & 0x3FF) + } else { + width = 2 + } + + default: + panic("impossible") + } + + // Check if the character is in the allowed range: + // #x9 | #xA | #xD | [#x20-#x7E] (8 bit) + // | #x85 | [#xA0-#xD7FF] | [#xE000-#xFFFD] (16 bit) + // | [#x10000-#x10FFFF] (32 bit) + switch { + case value == 0x09: + case value == 0x0A: + case value == 0x0D: + case value >= 0x20 && value <= 0x7E: + case value == 0x85: + case value >= 0xA0 && value <= 0xD7FF: + case value >= 0xE000 && value <= 0xFFFD: + case value >= 0x10000 && value <= 0x10FFFF: + default: + return yaml_parser_set_reader_error(parser, + "control characters are not allowed", + parser.offset, int(value)) + } + + // Move the raw pointers. + parser.raw_buffer_pos += width + parser.offset += width + + // Finally put the character into the buffer. + if value <= 0x7F { + // 0000 0000-0000 007F . 0xxxxxxx + parser.buffer[buffer_len+0] = byte(value) + buffer_len += 1 + } else if value <= 0x7FF { + // 0000 0080-0000 07FF . 110xxxxx 10xxxxxx + parser.buffer[buffer_len+0] = byte(0xC0 + (value >> 6)) + parser.buffer[buffer_len+1] = byte(0x80 + (value & 0x3F)) + buffer_len += 2 + } else if value <= 0xFFFF { + // 0000 0800-0000 FFFF . 1110xxxx 10xxxxxx 10xxxxxx + parser.buffer[buffer_len+0] = byte(0xE0 + (value >> 12)) + parser.buffer[buffer_len+1] = byte(0x80 + ((value >> 6) & 0x3F)) + parser.buffer[buffer_len+2] = byte(0x80 + (value & 0x3F)) + buffer_len += 3 + } else { + // 0001 0000-0010 FFFF . 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx + parser.buffer[buffer_len+0] = byte(0xF0 + (value >> 18)) + parser.buffer[buffer_len+1] = byte(0x80 + ((value >> 12) & 0x3F)) + parser.buffer[buffer_len+2] = byte(0x80 + ((value >> 6) & 0x3F)) + parser.buffer[buffer_len+3] = byte(0x80 + (value & 0x3F)) + buffer_len += 4 + } + + parser.unread++ + } + + // On EOF, put NUL into the buffer and return. + if parser.eof { + parser.buffer[buffer_len] = 0 + buffer_len++ + parser.unread++ + break + } + } + // [Go] Read the documentation of this function above. To return true, + // we need to have the given length in the buffer. Not doing that means + // every single check that calls this function to make sure the buffer + // has a given length is Go) panicking; or C) accessing invalid memory. + // This happens here due to the EOF above breaking early. + for buffer_len < length { + parser.buffer[buffer_len] = 0 + buffer_len++ + } + parser.buffer = parser.buffer[:buffer_len] + return true +} diff --git a/tools/voluspa/vendor/gopkg.in/yaml.v2/resolve.go b/tools/voluspa/vendor/gopkg.in/yaml.v2/resolve.go new file mode 100644 index 00000000000..4120e0c9160 --- /dev/null +++ b/tools/voluspa/vendor/gopkg.in/yaml.v2/resolve.go @@ -0,0 +1,258 @@ +package yaml + +import ( + "encoding/base64" + "math" + "regexp" + "strconv" + "strings" + "time" +) + +type resolveMapItem struct { + value interface{} + tag string +} + +var resolveTable = make([]byte, 256) +var resolveMap = make(map[string]resolveMapItem) + +func init() { + t := resolveTable + t[int('+')] = 'S' // Sign + t[int('-')] = 'S' + for _, c := range "0123456789" { + t[int(c)] = 'D' // Digit + } + for _, c := range "yYnNtTfFoO~" { + t[int(c)] = 'M' // In map + } + t[int('.')] = '.' // Float (potentially in map) + + var resolveMapList = []struct { + v interface{} + tag string + l []string + }{ + {true, yaml_BOOL_TAG, []string{"y", "Y", "yes", "Yes", "YES"}}, + {true, yaml_BOOL_TAG, []string{"true", "True", "TRUE"}}, + {true, yaml_BOOL_TAG, []string{"on", "On", "ON"}}, + {false, yaml_BOOL_TAG, []string{"n", "N", "no", "No", "NO"}}, + {false, yaml_BOOL_TAG, []string{"false", "False", "FALSE"}}, + {false, yaml_BOOL_TAG, []string{"off", "Off", "OFF"}}, + {nil, yaml_NULL_TAG, []string{"", "~", "null", "Null", "NULL"}}, + {math.NaN(), yaml_FLOAT_TAG, []string{".nan", ".NaN", ".NAN"}}, + {math.Inf(+1), yaml_FLOAT_TAG, []string{".inf", ".Inf", ".INF"}}, + {math.Inf(+1), yaml_FLOAT_TAG, []string{"+.inf", "+.Inf", "+.INF"}}, + {math.Inf(-1), yaml_FLOAT_TAG, []string{"-.inf", "-.Inf", "-.INF"}}, + {"<<", yaml_MERGE_TAG, []string{"<<"}}, + } + + m := resolveMap + for _, item := range resolveMapList { + for _, s := range item.l { + m[s] = resolveMapItem{item.v, item.tag} + } + } +} + +const longTagPrefix = "tag:yaml.org,2002:" + +func shortTag(tag string) string { + // TODO This can easily be made faster and produce less garbage. + if strings.HasPrefix(tag, longTagPrefix) { + return "!!" + tag[len(longTagPrefix):] + } + return tag +} + +func longTag(tag string) string { + if strings.HasPrefix(tag, "!!") { + return longTagPrefix + tag[2:] + } + return tag +} + +func resolvableTag(tag string) bool { + switch tag { + case "", yaml_STR_TAG, yaml_BOOL_TAG, yaml_INT_TAG, yaml_FLOAT_TAG, yaml_NULL_TAG, yaml_TIMESTAMP_TAG: + return true + } + return false +} + +var yamlStyleFloat = regexp.MustCompile(`^[-+]?(\.[0-9]+|[0-9]+(\.[0-9]*)?)([eE][-+]?[0-9]+)?$`) + +func resolve(tag string, in string) (rtag string, out interface{}) { + if !resolvableTag(tag) { + return tag, in + } + + defer func() { + switch tag { + case "", rtag, yaml_STR_TAG, yaml_BINARY_TAG: + return + case yaml_FLOAT_TAG: + if rtag == yaml_INT_TAG { + switch v := out.(type) { + case int64: + rtag = yaml_FLOAT_TAG + out = float64(v) + return + case int: + rtag = yaml_FLOAT_TAG + out = float64(v) + return + } + } + } + failf("cannot decode %s `%s` as a %s", shortTag(rtag), in, shortTag(tag)) + }() + + // Any data is accepted as a !!str or !!binary. + // Otherwise, the prefix is enough of a hint about what it might be. + hint := byte('N') + if in != "" { + hint = resolveTable[in[0]] + } + if hint != 0 && tag != yaml_STR_TAG && tag != yaml_BINARY_TAG { + // Handle things we can lookup in a map. + if item, ok := resolveMap[in]; ok { + return item.tag, item.value + } + + // Base 60 floats are a bad idea, were dropped in YAML 1.2, and + // are purposefully unsupported here. They're still quoted on + // the way out for compatibility with other parser, though. + + switch hint { + case 'M': + // We've already checked the map above. + + case '.': + // Not in the map, so maybe a normal float. + floatv, err := strconv.ParseFloat(in, 64) + if err == nil { + return yaml_FLOAT_TAG, floatv + } + + case 'D', 'S': + // Int, float, or timestamp. + // Only try values as a timestamp if the value is unquoted or there's an explicit + // !!timestamp tag. + if tag == "" || tag == yaml_TIMESTAMP_TAG { + t, ok := parseTimestamp(in) + if ok { + return yaml_TIMESTAMP_TAG, t + } + } + + plain := strings.Replace(in, "_", "", -1) + intv, err := strconv.ParseInt(plain, 0, 64) + if err == nil { + if intv == int64(int(intv)) { + return yaml_INT_TAG, int(intv) + } else { + return yaml_INT_TAG, intv + } + } + uintv, err := strconv.ParseUint(plain, 0, 64) + if err == nil { + return yaml_INT_TAG, uintv + } + if yamlStyleFloat.MatchString(plain) { + floatv, err := strconv.ParseFloat(plain, 64) + if err == nil { + return yaml_FLOAT_TAG, floatv + } + } + if strings.HasPrefix(plain, "0b") { + intv, err := strconv.ParseInt(plain[2:], 2, 64) + if err == nil { + if intv == int64(int(intv)) { + return yaml_INT_TAG, int(intv) + } else { + return yaml_INT_TAG, intv + } + } + uintv, err := strconv.ParseUint(plain[2:], 2, 64) + if err == nil { + return yaml_INT_TAG, uintv + } + } else if strings.HasPrefix(plain, "-0b") { + intv, err := strconv.ParseInt("-" + plain[3:], 2, 64) + if err == nil { + if true || intv == int64(int(intv)) { + return yaml_INT_TAG, int(intv) + } else { + return yaml_INT_TAG, intv + } + } + } + default: + panic("resolveTable item not yet handled: " + string(rune(hint)) + " (with " + in + ")") + } + } + return yaml_STR_TAG, in +} + +// encodeBase64 encodes s as base64 that is broken up into multiple lines +// as appropriate for the resulting length. +func encodeBase64(s string) string { + const lineLen = 70 + encLen := base64.StdEncoding.EncodedLen(len(s)) + lines := encLen/lineLen + 1 + buf := make([]byte, encLen*2+lines) + in := buf[0:encLen] + out := buf[encLen:] + base64.StdEncoding.Encode(in, []byte(s)) + k := 0 + for i := 0; i < len(in); i += lineLen { + j := i + lineLen + if j > len(in) { + j = len(in) + } + k += copy(out[k:], in[i:j]) + if lines > 1 { + out[k] = '\n' + k++ + } + } + return string(out[:k]) +} + +// This is a subset of the formats allowed by the regular expression +// defined at http://yaml.org/type/timestamp.html. +var allowedTimestampFormats = []string{ + "2006-1-2T15:4:5.999999999Z07:00", // RCF3339Nano with short date fields. + "2006-1-2t15:4:5.999999999Z07:00", // RFC3339Nano with short date fields and lower-case "t". + "2006-1-2 15:4:5.999999999", // space separated with no time zone + "2006-1-2", // date only + // Notable exception: time.Parse cannot handle: "2001-12-14 21:59:43.10 -5" + // from the set of examples. +} + +// parseTimestamp parses s as a timestamp string and +// returns the timestamp and reports whether it succeeded. +// Timestamp formats are defined at http://yaml.org/type/timestamp.html +func parseTimestamp(s string) (time.Time, bool) { + // TODO write code to check all the formats supported by + // http://yaml.org/type/timestamp.html instead of using time.Parse. + + // Quick check: all date formats start with YYYY-. + i := 0 + for ; i < len(s); i++ { + if c := s[i]; c < '0' || c > '9' { + break + } + } + if i != 4 || i == len(s) || s[i] != '-' { + return time.Time{}, false + } + for _, format := range allowedTimestampFormats { + if t, err := time.Parse(format, s); err == nil { + return t, true + } + } + return time.Time{}, false +} diff --git a/tools/voluspa/vendor/gopkg.in/yaml.v2/scannerc.go b/tools/voluspa/vendor/gopkg.in/yaml.v2/scannerc.go new file mode 100644 index 00000000000..077fd1dd2d4 --- /dev/null +++ b/tools/voluspa/vendor/gopkg.in/yaml.v2/scannerc.go @@ -0,0 +1,2696 @@ +package yaml + +import ( + "bytes" + "fmt" +) + +// Introduction +// ************ +// +// The following notes assume that you are familiar with the YAML specification +// (http://yaml.org/spec/1.2/spec.html). We mostly follow it, although in +// some cases we are less restrictive that it requires. +// +// The process of transforming a YAML stream into a sequence of events is +// divided on two steps: Scanning and Parsing. +// +// The Scanner transforms the input stream into a sequence of tokens, while the +// parser transform the sequence of tokens produced by the Scanner into a +// sequence of parsing events. +// +// The Scanner is rather clever and complicated. The Parser, on the contrary, +// is a straightforward implementation of a recursive-descendant parser (or, +// LL(1) parser, as it is usually called). +// +// Actually there are two issues of Scanning that might be called "clever", the +// rest is quite straightforward. The issues are "block collection start" and +// "simple keys". Both issues are explained below in details. +// +// Here the Scanning step is explained and implemented. We start with the list +// of all the tokens produced by the Scanner together with short descriptions. +// +// Now, tokens: +// +// STREAM-START(encoding) # The stream start. +// STREAM-END # The stream end. +// VERSION-DIRECTIVE(major,minor) # The '%YAML' directive. +// TAG-DIRECTIVE(handle,prefix) # The '%TAG' directive. +// DOCUMENT-START # '---' +// DOCUMENT-END # '...' +// BLOCK-SEQUENCE-START # Indentation increase denoting a block +// BLOCK-MAPPING-START # sequence or a block mapping. +// BLOCK-END # Indentation decrease. +// FLOW-SEQUENCE-START # '[' +// FLOW-SEQUENCE-END # ']' +// BLOCK-SEQUENCE-START # '{' +// BLOCK-SEQUENCE-END # '}' +// BLOCK-ENTRY # '-' +// FLOW-ENTRY # ',' +// KEY # '?' or nothing (simple keys). +// VALUE # ':' +// ALIAS(anchor) # '*anchor' +// ANCHOR(anchor) # '&anchor' +// TAG(handle,suffix) # '!handle!suffix' +// SCALAR(value,style) # A scalar. +// +// The following two tokens are "virtual" tokens denoting the beginning and the +// end of the stream: +// +// STREAM-START(encoding) +// STREAM-END +// +// We pass the information about the input stream encoding with the +// STREAM-START token. +// +// The next two tokens are responsible for tags: +// +// VERSION-DIRECTIVE(major,minor) +// TAG-DIRECTIVE(handle,prefix) +// +// Example: +// +// %YAML 1.1 +// %TAG ! !foo +// %TAG !yaml! tag:yaml.org,2002: +// --- +// +// The correspoding sequence of tokens: +// +// STREAM-START(utf-8) +// VERSION-DIRECTIVE(1,1) +// TAG-DIRECTIVE("!","!foo") +// TAG-DIRECTIVE("!yaml","tag:yaml.org,2002:") +// DOCUMENT-START +// STREAM-END +// +// Note that the VERSION-DIRECTIVE and TAG-DIRECTIVE tokens occupy a whole +// line. +// +// The document start and end indicators are represented by: +// +// DOCUMENT-START +// DOCUMENT-END +// +// Note that if a YAML stream contains an implicit document (without '---' +// and '...' indicators), no DOCUMENT-START and DOCUMENT-END tokens will be +// produced. +// +// In the following examples, we present whole documents together with the +// produced tokens. +// +// 1. An implicit document: +// +// 'a scalar' +// +// Tokens: +// +// STREAM-START(utf-8) +// SCALAR("a scalar",single-quoted) +// STREAM-END +// +// 2. An explicit document: +// +// --- +// 'a scalar' +// ... +// +// Tokens: +// +// STREAM-START(utf-8) +// DOCUMENT-START +// SCALAR("a scalar",single-quoted) +// DOCUMENT-END +// STREAM-END +// +// 3. Several documents in a stream: +// +// 'a scalar' +// --- +// 'another scalar' +// --- +// 'yet another scalar' +// +// Tokens: +// +// STREAM-START(utf-8) +// SCALAR("a scalar",single-quoted) +// DOCUMENT-START +// SCALAR("another scalar",single-quoted) +// DOCUMENT-START +// SCALAR("yet another scalar",single-quoted) +// STREAM-END +// +// We have already introduced the SCALAR token above. The following tokens are +// used to describe aliases, anchors, tag, and scalars: +// +// ALIAS(anchor) +// ANCHOR(anchor) +// TAG(handle,suffix) +// SCALAR(value,style) +// +// The following series of examples illustrate the usage of these tokens: +// +// 1. A recursive sequence: +// +// &A [ *A ] +// +// Tokens: +// +// STREAM-START(utf-8) +// ANCHOR("A") +// FLOW-SEQUENCE-START +// ALIAS("A") +// FLOW-SEQUENCE-END +// STREAM-END +// +// 2. A tagged scalar: +// +// !!float "3.14" # A good approximation. +// +// Tokens: +// +// STREAM-START(utf-8) +// TAG("!!","float") +// SCALAR("3.14",double-quoted) +// STREAM-END +// +// 3. Various scalar styles: +// +// --- # Implicit empty plain scalars do not produce tokens. +// --- a plain scalar +// --- 'a single-quoted scalar' +// --- "a double-quoted scalar" +// --- |- +// a literal scalar +// --- >- +// a folded +// scalar +// +// Tokens: +// +// STREAM-START(utf-8) +// DOCUMENT-START +// DOCUMENT-START +// SCALAR("a plain scalar",plain) +// DOCUMENT-START +// SCALAR("a single-quoted scalar",single-quoted) +// DOCUMENT-START +// SCALAR("a double-quoted scalar",double-quoted) +// DOCUMENT-START +// SCALAR("a literal scalar",literal) +// DOCUMENT-START +// SCALAR("a folded scalar",folded) +// STREAM-END +// +// Now it's time to review collection-related tokens. We will start with +// flow collections: +// +// FLOW-SEQUENCE-START +// FLOW-SEQUENCE-END +// FLOW-MAPPING-START +// FLOW-MAPPING-END +// FLOW-ENTRY +// KEY +// VALUE +// +// The tokens FLOW-SEQUENCE-START, FLOW-SEQUENCE-END, FLOW-MAPPING-START, and +// FLOW-MAPPING-END represent the indicators '[', ']', '{', and '}' +// correspondingly. FLOW-ENTRY represent the ',' indicator. Finally the +// indicators '?' and ':', which are used for denoting mapping keys and values, +// are represented by the KEY and VALUE tokens. +// +// The following examples show flow collections: +// +// 1. A flow sequence: +// +// [item 1, item 2, item 3] +// +// Tokens: +// +// STREAM-START(utf-8) +// FLOW-SEQUENCE-START +// SCALAR("item 1",plain) +// FLOW-ENTRY +// SCALAR("item 2",plain) +// FLOW-ENTRY +// SCALAR("item 3",plain) +// FLOW-SEQUENCE-END +// STREAM-END +// +// 2. A flow mapping: +// +// { +// a simple key: a value, # Note that the KEY token is produced. +// ? a complex key: another value, +// } +// +// Tokens: +// +// STREAM-START(utf-8) +// FLOW-MAPPING-START +// KEY +// SCALAR("a simple key",plain) +// VALUE +// SCALAR("a value",plain) +// FLOW-ENTRY +// KEY +// SCALAR("a complex key",plain) +// VALUE +// SCALAR("another value",plain) +// FLOW-ENTRY +// FLOW-MAPPING-END +// STREAM-END +// +// A simple key is a key which is not denoted by the '?' indicator. Note that +// the Scanner still produce the KEY token whenever it encounters a simple key. +// +// For scanning block collections, the following tokens are used (note that we +// repeat KEY and VALUE here): +// +// BLOCK-SEQUENCE-START +// BLOCK-MAPPING-START +// BLOCK-END +// BLOCK-ENTRY +// KEY +// VALUE +// +// The tokens BLOCK-SEQUENCE-START and BLOCK-MAPPING-START denote indentation +// increase that precedes a block collection (cf. the INDENT token in Python). +// The token BLOCK-END denote indentation decrease that ends a block collection +// (cf. the DEDENT token in Python). However YAML has some syntax pecularities +// that makes detections of these tokens more complex. +// +// The tokens BLOCK-ENTRY, KEY, and VALUE are used to represent the indicators +// '-', '?', and ':' correspondingly. +// +// The following examples show how the tokens BLOCK-SEQUENCE-START, +// BLOCK-MAPPING-START, and BLOCK-END are emitted by the Scanner: +// +// 1. Block sequences: +// +// - item 1 +// - item 2 +// - +// - item 3.1 +// - item 3.2 +// - +// key 1: value 1 +// key 2: value 2 +// +// Tokens: +// +// STREAM-START(utf-8) +// BLOCK-SEQUENCE-START +// BLOCK-ENTRY +// SCALAR("item 1",plain) +// BLOCK-ENTRY +// SCALAR("item 2",plain) +// BLOCK-ENTRY +// BLOCK-SEQUENCE-START +// BLOCK-ENTRY +// SCALAR("item 3.1",plain) +// BLOCK-ENTRY +// SCALAR("item 3.2",plain) +// BLOCK-END +// BLOCK-ENTRY +// BLOCK-MAPPING-START +// KEY +// SCALAR("key 1",plain) +// VALUE +// SCALAR("value 1",plain) +// KEY +// SCALAR("key 2",plain) +// VALUE +// SCALAR("value 2",plain) +// BLOCK-END +// BLOCK-END +// STREAM-END +// +// 2. Block mappings: +// +// a simple key: a value # The KEY token is produced here. +// ? a complex key +// : another value +// a mapping: +// key 1: value 1 +// key 2: value 2 +// a sequence: +// - item 1 +// - item 2 +// +// Tokens: +// +// STREAM-START(utf-8) +// BLOCK-MAPPING-START +// KEY +// SCALAR("a simple key",plain) +// VALUE +// SCALAR("a value",plain) +// KEY +// SCALAR("a complex key",plain) +// VALUE +// SCALAR("another value",plain) +// KEY +// SCALAR("a mapping",plain) +// BLOCK-MAPPING-START +// KEY +// SCALAR("key 1",plain) +// VALUE +// SCALAR("value 1",plain) +// KEY +// SCALAR("key 2",plain) +// VALUE +// SCALAR("value 2",plain) +// BLOCK-END +// KEY +// SCALAR("a sequence",plain) +// VALUE +// BLOCK-SEQUENCE-START +// BLOCK-ENTRY +// SCALAR("item 1",plain) +// BLOCK-ENTRY +// SCALAR("item 2",plain) +// BLOCK-END +// BLOCK-END +// STREAM-END +// +// YAML does not always require to start a new block collection from a new +// line. If the current line contains only '-', '?', and ':' indicators, a new +// block collection may start at the current line. The following examples +// illustrate this case: +// +// 1. Collections in a sequence: +// +// - - item 1 +// - item 2 +// - key 1: value 1 +// key 2: value 2 +// - ? complex key +// : complex value +// +// Tokens: +// +// STREAM-START(utf-8) +// BLOCK-SEQUENCE-START +// BLOCK-ENTRY +// BLOCK-SEQUENCE-START +// BLOCK-ENTRY +// SCALAR("item 1",plain) +// BLOCK-ENTRY +// SCALAR("item 2",plain) +// BLOCK-END +// BLOCK-ENTRY +// BLOCK-MAPPING-START +// KEY +// SCALAR("key 1",plain) +// VALUE +// SCALAR("value 1",plain) +// KEY +// SCALAR("key 2",plain) +// VALUE +// SCALAR("value 2",plain) +// BLOCK-END +// BLOCK-ENTRY +// BLOCK-MAPPING-START +// KEY +// SCALAR("complex key") +// VALUE +// SCALAR("complex value") +// BLOCK-END +// BLOCK-END +// STREAM-END +// +// 2. Collections in a mapping: +// +// ? a sequence +// : - item 1 +// - item 2 +// ? a mapping +// : key 1: value 1 +// key 2: value 2 +// +// Tokens: +// +// STREAM-START(utf-8) +// BLOCK-MAPPING-START +// KEY +// SCALAR("a sequence",plain) +// VALUE +// BLOCK-SEQUENCE-START +// BLOCK-ENTRY +// SCALAR("item 1",plain) +// BLOCK-ENTRY +// SCALAR("item 2",plain) +// BLOCK-END +// KEY +// SCALAR("a mapping",plain) +// VALUE +// BLOCK-MAPPING-START +// KEY +// SCALAR("key 1",plain) +// VALUE +// SCALAR("value 1",plain) +// KEY +// SCALAR("key 2",plain) +// VALUE +// SCALAR("value 2",plain) +// BLOCK-END +// BLOCK-END +// STREAM-END +// +// YAML also permits non-indented sequences if they are included into a block +// mapping. In this case, the token BLOCK-SEQUENCE-START is not produced: +// +// key: +// - item 1 # BLOCK-SEQUENCE-START is NOT produced here. +// - item 2 +// +// Tokens: +// +// STREAM-START(utf-8) +// BLOCK-MAPPING-START +// KEY +// SCALAR("key",plain) +// VALUE +// BLOCK-ENTRY +// SCALAR("item 1",plain) +// BLOCK-ENTRY +// SCALAR("item 2",plain) +// BLOCK-END +// + +// Ensure that the buffer contains the required number of characters. +// Return true on success, false on failure (reader error or memory error). +func cache(parser *yaml_parser_t, length int) bool { + // [Go] This was inlined: !cache(A, B) -> unread < B && !update(A, B) + return parser.unread >= length || yaml_parser_update_buffer(parser, length) +} + +// Advance the buffer pointer. +func skip(parser *yaml_parser_t) { + parser.mark.index++ + parser.mark.column++ + parser.unread-- + parser.buffer_pos += width(parser.buffer[parser.buffer_pos]) +} + +func skip_line(parser *yaml_parser_t) { + if is_crlf(parser.buffer, parser.buffer_pos) { + parser.mark.index += 2 + parser.mark.column = 0 + parser.mark.line++ + parser.unread -= 2 + parser.buffer_pos += 2 + } else if is_break(parser.buffer, parser.buffer_pos) { + parser.mark.index++ + parser.mark.column = 0 + parser.mark.line++ + parser.unread-- + parser.buffer_pos += width(parser.buffer[parser.buffer_pos]) + } +} + +// Copy a character to a string buffer and advance pointers. +func read(parser *yaml_parser_t, s []byte) []byte { + w := width(parser.buffer[parser.buffer_pos]) + if w == 0 { + panic("invalid character sequence") + } + if len(s) == 0 { + s = make([]byte, 0, 32) + } + if w == 1 && len(s)+w <= cap(s) { + s = s[:len(s)+1] + s[len(s)-1] = parser.buffer[parser.buffer_pos] + parser.buffer_pos++ + } else { + s = append(s, parser.buffer[parser.buffer_pos:parser.buffer_pos+w]...) + parser.buffer_pos += w + } + parser.mark.index++ + parser.mark.column++ + parser.unread-- + return s +} + +// Copy a line break character to a string buffer and advance pointers. +func read_line(parser *yaml_parser_t, s []byte) []byte { + buf := parser.buffer + pos := parser.buffer_pos + switch { + case buf[pos] == '\r' && buf[pos+1] == '\n': + // CR LF . LF + s = append(s, '\n') + parser.buffer_pos += 2 + parser.mark.index++ + parser.unread-- + case buf[pos] == '\r' || buf[pos] == '\n': + // CR|LF . LF + s = append(s, '\n') + parser.buffer_pos += 1 + case buf[pos] == '\xC2' && buf[pos+1] == '\x85': + // NEL . LF + s = append(s, '\n') + parser.buffer_pos += 2 + case buf[pos] == '\xE2' && buf[pos+1] == '\x80' && (buf[pos+2] == '\xA8' || buf[pos+2] == '\xA9'): + // LS|PS . LS|PS + s = append(s, buf[parser.buffer_pos:pos+3]...) + parser.buffer_pos += 3 + default: + return s + } + parser.mark.index++ + parser.mark.column = 0 + parser.mark.line++ + parser.unread-- + return s +} + +// Get the next token. +func yaml_parser_scan(parser *yaml_parser_t, token *yaml_token_t) bool { + // Erase the token object. + *token = yaml_token_t{} // [Go] Is this necessary? + + // No tokens after STREAM-END or error. + if parser.stream_end_produced || parser.error != yaml_NO_ERROR { + return true + } + + // Ensure that the tokens queue contains enough tokens. + if !parser.token_available { + if !yaml_parser_fetch_more_tokens(parser) { + return false + } + } + + // Fetch the next token from the queue. + *token = parser.tokens[parser.tokens_head] + parser.tokens_head++ + parser.tokens_parsed++ + parser.token_available = false + + if token.typ == yaml_STREAM_END_TOKEN { + parser.stream_end_produced = true + } + return true +} + +// Set the scanner error and return false. +func yaml_parser_set_scanner_error(parser *yaml_parser_t, context string, context_mark yaml_mark_t, problem string) bool { + parser.error = yaml_SCANNER_ERROR + parser.context = context + parser.context_mark = context_mark + parser.problem = problem + parser.problem_mark = parser.mark + return false +} + +func yaml_parser_set_scanner_tag_error(parser *yaml_parser_t, directive bool, context_mark yaml_mark_t, problem string) bool { + context := "while parsing a tag" + if directive { + context = "while parsing a %TAG directive" + } + return yaml_parser_set_scanner_error(parser, context, context_mark, problem) +} + +func trace(args ...interface{}) func() { + pargs := append([]interface{}{"+++"}, args...) + fmt.Println(pargs...) + pargs = append([]interface{}{"---"}, args...) + return func() { fmt.Println(pargs...) } +} + +// Ensure that the tokens queue contains at least one token which can be +// returned to the Parser. +func yaml_parser_fetch_more_tokens(parser *yaml_parser_t) bool { + // While we need more tokens to fetch, do it. + for { + // Check if we really need to fetch more tokens. + need_more_tokens := false + + if parser.tokens_head == len(parser.tokens) { + // Queue is empty. + need_more_tokens = true + } else { + // Check if any potential simple key may occupy the head position. + if !yaml_parser_stale_simple_keys(parser) { + return false + } + + for i := range parser.simple_keys { + simple_key := &parser.simple_keys[i] + if simple_key.possible && simple_key.token_number == parser.tokens_parsed { + need_more_tokens = true + break + } + } + } + + // We are finished. + if !need_more_tokens { + break + } + // Fetch the next token. + if !yaml_parser_fetch_next_token(parser) { + return false + } + } + + parser.token_available = true + return true +} + +// The dispatcher for token fetchers. +func yaml_parser_fetch_next_token(parser *yaml_parser_t) bool { + // Ensure that the buffer is initialized. + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + + // Check if we just started scanning. Fetch STREAM-START then. + if !parser.stream_start_produced { + return yaml_parser_fetch_stream_start(parser) + } + + // Eat whitespaces and comments until we reach the next token. + if !yaml_parser_scan_to_next_token(parser) { + return false + } + + // Remove obsolete potential simple keys. + if !yaml_parser_stale_simple_keys(parser) { + return false + } + + // Check the indentation level against the current column. + if !yaml_parser_unroll_indent(parser, parser.mark.column) { + return false + } + + // Ensure that the buffer contains at least 4 characters. 4 is the length + // of the longest indicators ('--- ' and '... '). + if parser.unread < 4 && !yaml_parser_update_buffer(parser, 4) { + return false + } + + // Is it the end of the stream? + if is_z(parser.buffer, parser.buffer_pos) { + return yaml_parser_fetch_stream_end(parser) + } + + // Is it a directive? + if parser.mark.column == 0 && parser.buffer[parser.buffer_pos] == '%' { + return yaml_parser_fetch_directive(parser) + } + + buf := parser.buffer + pos := parser.buffer_pos + + // Is it the document start indicator? + if parser.mark.column == 0 && buf[pos] == '-' && buf[pos+1] == '-' && buf[pos+2] == '-' && is_blankz(buf, pos+3) { + return yaml_parser_fetch_document_indicator(parser, yaml_DOCUMENT_START_TOKEN) + } + + // Is it the document end indicator? + if parser.mark.column == 0 && buf[pos] == '.' && buf[pos+1] == '.' && buf[pos+2] == '.' && is_blankz(buf, pos+3) { + return yaml_parser_fetch_document_indicator(parser, yaml_DOCUMENT_END_TOKEN) + } + + // Is it the flow sequence start indicator? + if buf[pos] == '[' { + return yaml_parser_fetch_flow_collection_start(parser, yaml_FLOW_SEQUENCE_START_TOKEN) + } + + // Is it the flow mapping start indicator? + if parser.buffer[parser.buffer_pos] == '{' { + return yaml_parser_fetch_flow_collection_start(parser, yaml_FLOW_MAPPING_START_TOKEN) + } + + // Is it the flow sequence end indicator? + if parser.buffer[parser.buffer_pos] == ']' { + return yaml_parser_fetch_flow_collection_end(parser, + yaml_FLOW_SEQUENCE_END_TOKEN) + } + + // Is it the flow mapping end indicator? + if parser.buffer[parser.buffer_pos] == '}' { + return yaml_parser_fetch_flow_collection_end(parser, + yaml_FLOW_MAPPING_END_TOKEN) + } + + // Is it the flow entry indicator? + if parser.buffer[parser.buffer_pos] == ',' { + return yaml_parser_fetch_flow_entry(parser) + } + + // Is it the block entry indicator? + if parser.buffer[parser.buffer_pos] == '-' && is_blankz(parser.buffer, parser.buffer_pos+1) { + return yaml_parser_fetch_block_entry(parser) + } + + // Is it the key indicator? + if parser.buffer[parser.buffer_pos] == '?' && (parser.flow_level > 0 || is_blankz(parser.buffer, parser.buffer_pos+1)) { + return yaml_parser_fetch_key(parser) + } + + // Is it the value indicator? + if parser.buffer[parser.buffer_pos] == ':' && (parser.flow_level > 0 || is_blankz(parser.buffer, parser.buffer_pos+1)) { + return yaml_parser_fetch_value(parser) + } + + // Is it an alias? + if parser.buffer[parser.buffer_pos] == '*' { + return yaml_parser_fetch_anchor(parser, yaml_ALIAS_TOKEN) + } + + // Is it an anchor? + if parser.buffer[parser.buffer_pos] == '&' { + return yaml_parser_fetch_anchor(parser, yaml_ANCHOR_TOKEN) + } + + // Is it a tag? + if parser.buffer[parser.buffer_pos] == '!' { + return yaml_parser_fetch_tag(parser) + } + + // Is it a literal scalar? + if parser.buffer[parser.buffer_pos] == '|' && parser.flow_level == 0 { + return yaml_parser_fetch_block_scalar(parser, true) + } + + // Is it a folded scalar? + if parser.buffer[parser.buffer_pos] == '>' && parser.flow_level == 0 { + return yaml_parser_fetch_block_scalar(parser, false) + } + + // Is it a single-quoted scalar? + if parser.buffer[parser.buffer_pos] == '\'' { + return yaml_parser_fetch_flow_scalar(parser, true) + } + + // Is it a double-quoted scalar? + if parser.buffer[parser.buffer_pos] == '"' { + return yaml_parser_fetch_flow_scalar(parser, false) + } + + // Is it a plain scalar? + // + // A plain scalar may start with any non-blank characters except + // + // '-', '?', ':', ',', '[', ']', '{', '}', + // '#', '&', '*', '!', '|', '>', '\'', '\"', + // '%', '@', '`'. + // + // In the block context (and, for the '-' indicator, in the flow context + // too), it may also start with the characters + // + // '-', '?', ':' + // + // if it is followed by a non-space character. + // + // The last rule is more restrictive than the specification requires. + // [Go] Make this logic more reasonable. + //switch parser.buffer[parser.buffer_pos] { + //case '-', '?', ':', ',', '?', '-', ',', ':', ']', '[', '}', '{', '&', '#', '!', '*', '>', '|', '"', '\'', '@', '%', '-', '`': + //} + if !(is_blankz(parser.buffer, parser.buffer_pos) || parser.buffer[parser.buffer_pos] == '-' || + parser.buffer[parser.buffer_pos] == '?' || parser.buffer[parser.buffer_pos] == ':' || + parser.buffer[parser.buffer_pos] == ',' || parser.buffer[parser.buffer_pos] == '[' || + parser.buffer[parser.buffer_pos] == ']' || parser.buffer[parser.buffer_pos] == '{' || + parser.buffer[parser.buffer_pos] == '}' || parser.buffer[parser.buffer_pos] == '#' || + parser.buffer[parser.buffer_pos] == '&' || parser.buffer[parser.buffer_pos] == '*' || + parser.buffer[parser.buffer_pos] == '!' || parser.buffer[parser.buffer_pos] == '|' || + parser.buffer[parser.buffer_pos] == '>' || parser.buffer[parser.buffer_pos] == '\'' || + parser.buffer[parser.buffer_pos] == '"' || parser.buffer[parser.buffer_pos] == '%' || + parser.buffer[parser.buffer_pos] == '@' || parser.buffer[parser.buffer_pos] == '`') || + (parser.buffer[parser.buffer_pos] == '-' && !is_blank(parser.buffer, parser.buffer_pos+1)) || + (parser.flow_level == 0 && + (parser.buffer[parser.buffer_pos] == '?' || parser.buffer[parser.buffer_pos] == ':') && + !is_blankz(parser.buffer, parser.buffer_pos+1)) { + return yaml_parser_fetch_plain_scalar(parser) + } + + // If we don't determine the token type so far, it is an error. + return yaml_parser_set_scanner_error(parser, + "while scanning for the next token", parser.mark, + "found character that cannot start any token") +} + +// Check the list of potential simple keys and remove the positions that +// cannot contain simple keys anymore. +func yaml_parser_stale_simple_keys(parser *yaml_parser_t) bool { + // Check for a potential simple key for each flow level. + for i := range parser.simple_keys { + simple_key := &parser.simple_keys[i] + + // The specification requires that a simple key + // + // - is limited to a single line, + // - is shorter than 1024 characters. + if simple_key.possible && (simple_key.mark.line < parser.mark.line || simple_key.mark.index+1024 < parser.mark.index) { + + // Check if the potential simple key to be removed is required. + if simple_key.required { + return yaml_parser_set_scanner_error(parser, + "while scanning a simple key", simple_key.mark, + "could not find expected ':'") + } + simple_key.possible = false + } + } + return true +} + +// Check if a simple key may start at the current position and add it if +// needed. +func yaml_parser_save_simple_key(parser *yaml_parser_t) bool { + // A simple key is required at the current position if the scanner is in + // the block context and the current column coincides with the indentation + // level. + + required := parser.flow_level == 0 && parser.indent == parser.mark.column + + // + // If the current position may start a simple key, save it. + // + if parser.simple_key_allowed { + simple_key := yaml_simple_key_t{ + possible: true, + required: required, + token_number: parser.tokens_parsed + (len(parser.tokens) - parser.tokens_head), + } + simple_key.mark = parser.mark + + if !yaml_parser_remove_simple_key(parser) { + return false + } + parser.simple_keys[len(parser.simple_keys)-1] = simple_key + } + return true +} + +// Remove a potential simple key at the current flow level. +func yaml_parser_remove_simple_key(parser *yaml_parser_t) bool { + i := len(parser.simple_keys) - 1 + if parser.simple_keys[i].possible { + // If the key is required, it is an error. + if parser.simple_keys[i].required { + return yaml_parser_set_scanner_error(parser, + "while scanning a simple key", parser.simple_keys[i].mark, + "could not find expected ':'") + } + } + // Remove the key from the stack. + parser.simple_keys[i].possible = false + return true +} + +// Increase the flow level and resize the simple key list if needed. +func yaml_parser_increase_flow_level(parser *yaml_parser_t) bool { + // Reset the simple key on the next level. + parser.simple_keys = append(parser.simple_keys, yaml_simple_key_t{}) + + // Increase the flow level. + parser.flow_level++ + return true +} + +// Decrease the flow level. +func yaml_parser_decrease_flow_level(parser *yaml_parser_t) bool { + if parser.flow_level > 0 { + parser.flow_level-- + parser.simple_keys = parser.simple_keys[:len(parser.simple_keys)-1] + } + return true +} + +// Push the current indentation level to the stack and set the new level +// the current column is greater than the indentation level. In this case, +// append or insert the specified token into the token queue. +func yaml_parser_roll_indent(parser *yaml_parser_t, column, number int, typ yaml_token_type_t, mark yaml_mark_t) bool { + // In the flow context, do nothing. + if parser.flow_level > 0 { + return true + } + + if parser.indent < column { + // Push the current indentation level to the stack and set the new + // indentation level. + parser.indents = append(parser.indents, parser.indent) + parser.indent = column + + // Create a token and insert it into the queue. + token := yaml_token_t{ + typ: typ, + start_mark: mark, + end_mark: mark, + } + if number > -1 { + number -= parser.tokens_parsed + } + yaml_insert_token(parser, number, &token) + } + return true +} + +// Pop indentation levels from the indents stack until the current level +// becomes less or equal to the column. For each indentation level, append +// the BLOCK-END token. +func yaml_parser_unroll_indent(parser *yaml_parser_t, column int) bool { + // In the flow context, do nothing. + if parser.flow_level > 0 { + return true + } + + // Loop through the indentation levels in the stack. + for parser.indent > column { + // Create a token and append it to the queue. + token := yaml_token_t{ + typ: yaml_BLOCK_END_TOKEN, + start_mark: parser.mark, + end_mark: parser.mark, + } + yaml_insert_token(parser, -1, &token) + + // Pop the indentation level. + parser.indent = parser.indents[len(parser.indents)-1] + parser.indents = parser.indents[:len(parser.indents)-1] + } + return true +} + +// Initialize the scanner and produce the STREAM-START token. +func yaml_parser_fetch_stream_start(parser *yaml_parser_t) bool { + + // Set the initial indentation. + parser.indent = -1 + + // Initialize the simple key stack. + parser.simple_keys = append(parser.simple_keys, yaml_simple_key_t{}) + + // A simple key is allowed at the beginning of the stream. + parser.simple_key_allowed = true + + // We have started. + parser.stream_start_produced = true + + // Create the STREAM-START token and append it to the queue. + token := yaml_token_t{ + typ: yaml_STREAM_START_TOKEN, + start_mark: parser.mark, + end_mark: parser.mark, + encoding: parser.encoding, + } + yaml_insert_token(parser, -1, &token) + return true +} + +// Produce the STREAM-END token and shut down the scanner. +func yaml_parser_fetch_stream_end(parser *yaml_parser_t) bool { + + // Force new line. + if parser.mark.column != 0 { + parser.mark.column = 0 + parser.mark.line++ + } + + // Reset the indentation level. + if !yaml_parser_unroll_indent(parser, -1) { + return false + } + + // Reset simple keys. + if !yaml_parser_remove_simple_key(parser) { + return false + } + + parser.simple_key_allowed = false + + // Create the STREAM-END token and append it to the queue. + token := yaml_token_t{ + typ: yaml_STREAM_END_TOKEN, + start_mark: parser.mark, + end_mark: parser.mark, + } + yaml_insert_token(parser, -1, &token) + return true +} + +// Produce a VERSION-DIRECTIVE or TAG-DIRECTIVE token. +func yaml_parser_fetch_directive(parser *yaml_parser_t) bool { + // Reset the indentation level. + if !yaml_parser_unroll_indent(parser, -1) { + return false + } + + // Reset simple keys. + if !yaml_parser_remove_simple_key(parser) { + return false + } + + parser.simple_key_allowed = false + + // Create the YAML-DIRECTIVE or TAG-DIRECTIVE token. + token := yaml_token_t{} + if !yaml_parser_scan_directive(parser, &token) { + return false + } + // Append the token to the queue. + yaml_insert_token(parser, -1, &token) + return true +} + +// Produce the DOCUMENT-START or DOCUMENT-END token. +func yaml_parser_fetch_document_indicator(parser *yaml_parser_t, typ yaml_token_type_t) bool { + // Reset the indentation level. + if !yaml_parser_unroll_indent(parser, -1) { + return false + } + + // Reset simple keys. + if !yaml_parser_remove_simple_key(parser) { + return false + } + + parser.simple_key_allowed = false + + // Consume the token. + start_mark := parser.mark + + skip(parser) + skip(parser) + skip(parser) + + end_mark := parser.mark + + // Create the DOCUMENT-START or DOCUMENT-END token. + token := yaml_token_t{ + typ: typ, + start_mark: start_mark, + end_mark: end_mark, + } + // Append the token to the queue. + yaml_insert_token(parser, -1, &token) + return true +} + +// Produce the FLOW-SEQUENCE-START or FLOW-MAPPING-START token. +func yaml_parser_fetch_flow_collection_start(parser *yaml_parser_t, typ yaml_token_type_t) bool { + // The indicators '[' and '{' may start a simple key. + if !yaml_parser_save_simple_key(parser) { + return false + } + + // Increase the flow level. + if !yaml_parser_increase_flow_level(parser) { + return false + } + + // A simple key may follow the indicators '[' and '{'. + parser.simple_key_allowed = true + + // Consume the token. + start_mark := parser.mark + skip(parser) + end_mark := parser.mark + + // Create the FLOW-SEQUENCE-START of FLOW-MAPPING-START token. + token := yaml_token_t{ + typ: typ, + start_mark: start_mark, + end_mark: end_mark, + } + // Append the token to the queue. + yaml_insert_token(parser, -1, &token) + return true +} + +// Produce the FLOW-SEQUENCE-END or FLOW-MAPPING-END token. +func yaml_parser_fetch_flow_collection_end(parser *yaml_parser_t, typ yaml_token_type_t) bool { + // Reset any potential simple key on the current flow level. + if !yaml_parser_remove_simple_key(parser) { + return false + } + + // Decrease the flow level. + if !yaml_parser_decrease_flow_level(parser) { + return false + } + + // No simple keys after the indicators ']' and '}'. + parser.simple_key_allowed = false + + // Consume the token. + + start_mark := parser.mark + skip(parser) + end_mark := parser.mark + + // Create the FLOW-SEQUENCE-END of FLOW-MAPPING-END token. + token := yaml_token_t{ + typ: typ, + start_mark: start_mark, + end_mark: end_mark, + } + // Append the token to the queue. + yaml_insert_token(parser, -1, &token) + return true +} + +// Produce the FLOW-ENTRY token. +func yaml_parser_fetch_flow_entry(parser *yaml_parser_t) bool { + // Reset any potential simple keys on the current flow level. + if !yaml_parser_remove_simple_key(parser) { + return false + } + + // Simple keys are allowed after ','. + parser.simple_key_allowed = true + + // Consume the token. + start_mark := parser.mark + skip(parser) + end_mark := parser.mark + + // Create the FLOW-ENTRY token and append it to the queue. + token := yaml_token_t{ + typ: yaml_FLOW_ENTRY_TOKEN, + start_mark: start_mark, + end_mark: end_mark, + } + yaml_insert_token(parser, -1, &token) + return true +} + +// Produce the BLOCK-ENTRY token. +func yaml_parser_fetch_block_entry(parser *yaml_parser_t) bool { + // Check if the scanner is in the block context. + if parser.flow_level == 0 { + // Check if we are allowed to start a new entry. + if !parser.simple_key_allowed { + return yaml_parser_set_scanner_error(parser, "", parser.mark, + "block sequence entries are not allowed in this context") + } + // Add the BLOCK-SEQUENCE-START token if needed. + if !yaml_parser_roll_indent(parser, parser.mark.column, -1, yaml_BLOCK_SEQUENCE_START_TOKEN, parser.mark) { + return false + } + } else { + // It is an error for the '-' indicator to occur in the flow context, + // but we let the Parser detect and report about it because the Parser + // is able to point to the context. + } + + // Reset any potential simple keys on the current flow level. + if !yaml_parser_remove_simple_key(parser) { + return false + } + + // Simple keys are allowed after '-'. + parser.simple_key_allowed = true + + // Consume the token. + start_mark := parser.mark + skip(parser) + end_mark := parser.mark + + // Create the BLOCK-ENTRY token and append it to the queue. + token := yaml_token_t{ + typ: yaml_BLOCK_ENTRY_TOKEN, + start_mark: start_mark, + end_mark: end_mark, + } + yaml_insert_token(parser, -1, &token) + return true +} + +// Produce the KEY token. +func yaml_parser_fetch_key(parser *yaml_parser_t) bool { + + // In the block context, additional checks are required. + if parser.flow_level == 0 { + // Check if we are allowed to start a new key (not nessesary simple). + if !parser.simple_key_allowed { + return yaml_parser_set_scanner_error(parser, "", parser.mark, + "mapping keys are not allowed in this context") + } + // Add the BLOCK-MAPPING-START token if needed. + if !yaml_parser_roll_indent(parser, parser.mark.column, -1, yaml_BLOCK_MAPPING_START_TOKEN, parser.mark) { + return false + } + } + + // Reset any potential simple keys on the current flow level. + if !yaml_parser_remove_simple_key(parser) { + return false + } + + // Simple keys are allowed after '?' in the block context. + parser.simple_key_allowed = parser.flow_level == 0 + + // Consume the token. + start_mark := parser.mark + skip(parser) + end_mark := parser.mark + + // Create the KEY token and append it to the queue. + token := yaml_token_t{ + typ: yaml_KEY_TOKEN, + start_mark: start_mark, + end_mark: end_mark, + } + yaml_insert_token(parser, -1, &token) + return true +} + +// Produce the VALUE token. +func yaml_parser_fetch_value(parser *yaml_parser_t) bool { + + simple_key := &parser.simple_keys[len(parser.simple_keys)-1] + + // Have we found a simple key? + if simple_key.possible { + // Create the KEY token and insert it into the queue. + token := yaml_token_t{ + typ: yaml_KEY_TOKEN, + start_mark: simple_key.mark, + end_mark: simple_key.mark, + } + yaml_insert_token(parser, simple_key.token_number-parser.tokens_parsed, &token) + + // In the block context, we may need to add the BLOCK-MAPPING-START token. + if !yaml_parser_roll_indent(parser, simple_key.mark.column, + simple_key.token_number, + yaml_BLOCK_MAPPING_START_TOKEN, simple_key.mark) { + return false + } + + // Remove the simple key. + simple_key.possible = false + + // A simple key cannot follow another simple key. + parser.simple_key_allowed = false + + } else { + // The ':' indicator follows a complex key. + + // In the block context, extra checks are required. + if parser.flow_level == 0 { + + // Check if we are allowed to start a complex value. + if !parser.simple_key_allowed { + return yaml_parser_set_scanner_error(parser, "", parser.mark, + "mapping values are not allowed in this context") + } + + // Add the BLOCK-MAPPING-START token if needed. + if !yaml_parser_roll_indent(parser, parser.mark.column, -1, yaml_BLOCK_MAPPING_START_TOKEN, parser.mark) { + return false + } + } + + // Simple keys after ':' are allowed in the block context. + parser.simple_key_allowed = parser.flow_level == 0 + } + + // Consume the token. + start_mark := parser.mark + skip(parser) + end_mark := parser.mark + + // Create the VALUE token and append it to the queue. + token := yaml_token_t{ + typ: yaml_VALUE_TOKEN, + start_mark: start_mark, + end_mark: end_mark, + } + yaml_insert_token(parser, -1, &token) + return true +} + +// Produce the ALIAS or ANCHOR token. +func yaml_parser_fetch_anchor(parser *yaml_parser_t, typ yaml_token_type_t) bool { + // An anchor or an alias could be a simple key. + if !yaml_parser_save_simple_key(parser) { + return false + } + + // A simple key cannot follow an anchor or an alias. + parser.simple_key_allowed = false + + // Create the ALIAS or ANCHOR token and append it to the queue. + var token yaml_token_t + if !yaml_parser_scan_anchor(parser, &token, typ) { + return false + } + yaml_insert_token(parser, -1, &token) + return true +} + +// Produce the TAG token. +func yaml_parser_fetch_tag(parser *yaml_parser_t) bool { + // A tag could be a simple key. + if !yaml_parser_save_simple_key(parser) { + return false + } + + // A simple key cannot follow a tag. + parser.simple_key_allowed = false + + // Create the TAG token and append it to the queue. + var token yaml_token_t + if !yaml_parser_scan_tag(parser, &token) { + return false + } + yaml_insert_token(parser, -1, &token) + return true +} + +// Produce the SCALAR(...,literal) or SCALAR(...,folded) tokens. +func yaml_parser_fetch_block_scalar(parser *yaml_parser_t, literal bool) bool { + // Remove any potential simple keys. + if !yaml_parser_remove_simple_key(parser) { + return false + } + + // A simple key may follow a block scalar. + parser.simple_key_allowed = true + + // Create the SCALAR token and append it to the queue. + var token yaml_token_t + if !yaml_parser_scan_block_scalar(parser, &token, literal) { + return false + } + yaml_insert_token(parser, -1, &token) + return true +} + +// Produce the SCALAR(...,single-quoted) or SCALAR(...,double-quoted) tokens. +func yaml_parser_fetch_flow_scalar(parser *yaml_parser_t, single bool) bool { + // A plain scalar could be a simple key. + if !yaml_parser_save_simple_key(parser) { + return false + } + + // A simple key cannot follow a flow scalar. + parser.simple_key_allowed = false + + // Create the SCALAR token and append it to the queue. + var token yaml_token_t + if !yaml_parser_scan_flow_scalar(parser, &token, single) { + return false + } + yaml_insert_token(parser, -1, &token) + return true +} + +// Produce the SCALAR(...,plain) token. +func yaml_parser_fetch_plain_scalar(parser *yaml_parser_t) bool { + // A plain scalar could be a simple key. + if !yaml_parser_save_simple_key(parser) { + return false + } + + // A simple key cannot follow a flow scalar. + parser.simple_key_allowed = false + + // Create the SCALAR token and append it to the queue. + var token yaml_token_t + if !yaml_parser_scan_plain_scalar(parser, &token) { + return false + } + yaml_insert_token(parser, -1, &token) + return true +} + +// Eat whitespaces and comments until the next token is found. +func yaml_parser_scan_to_next_token(parser *yaml_parser_t) bool { + + // Until the next token is not found. + for { + // Allow the BOM mark to start a line. + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + if parser.mark.column == 0 && is_bom(parser.buffer, parser.buffer_pos) { + skip(parser) + } + + // Eat whitespaces. + // Tabs are allowed: + // - in the flow context + // - in the block context, but not at the beginning of the line or + // after '-', '?', or ':' (complex value). + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + + for parser.buffer[parser.buffer_pos] == ' ' || ((parser.flow_level > 0 || !parser.simple_key_allowed) && parser.buffer[parser.buffer_pos] == '\t') { + skip(parser) + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + } + + // Eat a comment until a line break. + if parser.buffer[parser.buffer_pos] == '#' { + for !is_breakz(parser.buffer, parser.buffer_pos) { + skip(parser) + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + } + } + + // If it is a line break, eat it. + if is_break(parser.buffer, parser.buffer_pos) { + if parser.unread < 2 && !yaml_parser_update_buffer(parser, 2) { + return false + } + skip_line(parser) + + // In the block context, a new line may start a simple key. + if parser.flow_level == 0 { + parser.simple_key_allowed = true + } + } else { + break // We have found a token. + } + } + + return true +} + +// Scan a YAML-DIRECTIVE or TAG-DIRECTIVE token. +// +// Scope: +// %YAML 1.1 # a comment \n +// ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +// %TAG !yaml! tag:yaml.org,2002: \n +// ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +// +func yaml_parser_scan_directive(parser *yaml_parser_t, token *yaml_token_t) bool { + // Eat '%'. + start_mark := parser.mark + skip(parser) + + // Scan the directive name. + var name []byte + if !yaml_parser_scan_directive_name(parser, start_mark, &name) { + return false + } + + // Is it a YAML directive? + if bytes.Equal(name, []byte("YAML")) { + // Scan the VERSION directive value. + var major, minor int8 + if !yaml_parser_scan_version_directive_value(parser, start_mark, &major, &minor) { + return false + } + end_mark := parser.mark + + // Create a VERSION-DIRECTIVE token. + *token = yaml_token_t{ + typ: yaml_VERSION_DIRECTIVE_TOKEN, + start_mark: start_mark, + end_mark: end_mark, + major: major, + minor: minor, + } + + // Is it a TAG directive? + } else if bytes.Equal(name, []byte("TAG")) { + // Scan the TAG directive value. + var handle, prefix []byte + if !yaml_parser_scan_tag_directive_value(parser, start_mark, &handle, &prefix) { + return false + } + end_mark := parser.mark + + // Create a TAG-DIRECTIVE token. + *token = yaml_token_t{ + typ: yaml_TAG_DIRECTIVE_TOKEN, + start_mark: start_mark, + end_mark: end_mark, + value: handle, + prefix: prefix, + } + + // Unknown directive. + } else { + yaml_parser_set_scanner_error(parser, "while scanning a directive", + start_mark, "found unknown directive name") + return false + } + + // Eat the rest of the line including any comments. + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + + for is_blank(parser.buffer, parser.buffer_pos) { + skip(parser) + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + } + + if parser.buffer[parser.buffer_pos] == '#' { + for !is_breakz(parser.buffer, parser.buffer_pos) { + skip(parser) + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + } + } + + // Check if we are at the end of the line. + if !is_breakz(parser.buffer, parser.buffer_pos) { + yaml_parser_set_scanner_error(parser, "while scanning a directive", + start_mark, "did not find expected comment or line break") + return false + } + + // Eat a line break. + if is_break(parser.buffer, parser.buffer_pos) { + if parser.unread < 2 && !yaml_parser_update_buffer(parser, 2) { + return false + } + skip_line(parser) + } + + return true +} + +// Scan the directive name. +// +// Scope: +// %YAML 1.1 # a comment \n +// ^^^^ +// %TAG !yaml! tag:yaml.org,2002: \n +// ^^^ +// +func yaml_parser_scan_directive_name(parser *yaml_parser_t, start_mark yaml_mark_t, name *[]byte) bool { + // Consume the directive name. + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + + var s []byte + for is_alpha(parser.buffer, parser.buffer_pos) { + s = read(parser, s) + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + } + + // Check if the name is empty. + if len(s) == 0 { + yaml_parser_set_scanner_error(parser, "while scanning a directive", + start_mark, "could not find expected directive name") + return false + } + + // Check for an blank character after the name. + if !is_blankz(parser.buffer, parser.buffer_pos) { + yaml_parser_set_scanner_error(parser, "while scanning a directive", + start_mark, "found unexpected non-alphabetical character") + return false + } + *name = s + return true +} + +// Scan the value of VERSION-DIRECTIVE. +// +// Scope: +// %YAML 1.1 # a comment \n +// ^^^^^^ +func yaml_parser_scan_version_directive_value(parser *yaml_parser_t, start_mark yaml_mark_t, major, minor *int8) bool { + // Eat whitespaces. + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + for is_blank(parser.buffer, parser.buffer_pos) { + skip(parser) + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + } + + // Consume the major version number. + if !yaml_parser_scan_version_directive_number(parser, start_mark, major) { + return false + } + + // Eat '.'. + if parser.buffer[parser.buffer_pos] != '.' { + return yaml_parser_set_scanner_error(parser, "while scanning a %YAML directive", + start_mark, "did not find expected digit or '.' character") + } + + skip(parser) + + // Consume the minor version number. + if !yaml_parser_scan_version_directive_number(parser, start_mark, minor) { + return false + } + return true +} + +const max_number_length = 2 + +// Scan the version number of VERSION-DIRECTIVE. +// +// Scope: +// %YAML 1.1 # a comment \n +// ^ +// %YAML 1.1 # a comment \n +// ^ +func yaml_parser_scan_version_directive_number(parser *yaml_parser_t, start_mark yaml_mark_t, number *int8) bool { + + // Repeat while the next character is digit. + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + var value, length int8 + for is_digit(parser.buffer, parser.buffer_pos) { + // Check if the number is too long. + length++ + if length > max_number_length { + return yaml_parser_set_scanner_error(parser, "while scanning a %YAML directive", + start_mark, "found extremely long version number") + } + value = value*10 + int8(as_digit(parser.buffer, parser.buffer_pos)) + skip(parser) + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + } + + // Check if the number was present. + if length == 0 { + return yaml_parser_set_scanner_error(parser, "while scanning a %YAML directive", + start_mark, "did not find expected version number") + } + *number = value + return true +} + +// Scan the value of a TAG-DIRECTIVE token. +// +// Scope: +// %TAG !yaml! tag:yaml.org,2002: \n +// ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +// +func yaml_parser_scan_tag_directive_value(parser *yaml_parser_t, start_mark yaml_mark_t, handle, prefix *[]byte) bool { + var handle_value, prefix_value []byte + + // Eat whitespaces. + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + + for is_blank(parser.buffer, parser.buffer_pos) { + skip(parser) + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + } + + // Scan a handle. + if !yaml_parser_scan_tag_handle(parser, true, start_mark, &handle_value) { + return false + } + + // Expect a whitespace. + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + if !is_blank(parser.buffer, parser.buffer_pos) { + yaml_parser_set_scanner_error(parser, "while scanning a %TAG directive", + start_mark, "did not find expected whitespace") + return false + } + + // Eat whitespaces. + for is_blank(parser.buffer, parser.buffer_pos) { + skip(parser) + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + } + + // Scan a prefix. + if !yaml_parser_scan_tag_uri(parser, true, nil, start_mark, &prefix_value) { + return false + } + + // Expect a whitespace or line break. + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + if !is_blankz(parser.buffer, parser.buffer_pos) { + yaml_parser_set_scanner_error(parser, "while scanning a %TAG directive", + start_mark, "did not find expected whitespace or line break") + return false + } + + *handle = handle_value + *prefix = prefix_value + return true +} + +func yaml_parser_scan_anchor(parser *yaml_parser_t, token *yaml_token_t, typ yaml_token_type_t) bool { + var s []byte + + // Eat the indicator character. + start_mark := parser.mark + skip(parser) + + // Consume the value. + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + + for is_alpha(parser.buffer, parser.buffer_pos) { + s = read(parser, s) + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + } + + end_mark := parser.mark + + /* + * Check if length of the anchor is greater than 0 and it is followed by + * a whitespace character or one of the indicators: + * + * '?', ':', ',', ']', '}', '%', '@', '`'. + */ + + if len(s) == 0 || + !(is_blankz(parser.buffer, parser.buffer_pos) || parser.buffer[parser.buffer_pos] == '?' || + parser.buffer[parser.buffer_pos] == ':' || parser.buffer[parser.buffer_pos] == ',' || + parser.buffer[parser.buffer_pos] == ']' || parser.buffer[parser.buffer_pos] == '}' || + parser.buffer[parser.buffer_pos] == '%' || parser.buffer[parser.buffer_pos] == '@' || + parser.buffer[parser.buffer_pos] == '`') { + context := "while scanning an alias" + if typ == yaml_ANCHOR_TOKEN { + context = "while scanning an anchor" + } + yaml_parser_set_scanner_error(parser, context, start_mark, + "did not find expected alphabetic or numeric character") + return false + } + + // Create a token. + *token = yaml_token_t{ + typ: typ, + start_mark: start_mark, + end_mark: end_mark, + value: s, + } + + return true +} + +/* + * Scan a TAG token. + */ + +func yaml_parser_scan_tag(parser *yaml_parser_t, token *yaml_token_t) bool { + var handle, suffix []byte + + start_mark := parser.mark + + // Check if the tag is in the canonical form. + if parser.unread < 2 && !yaml_parser_update_buffer(parser, 2) { + return false + } + + if parser.buffer[parser.buffer_pos+1] == '<' { + // Keep the handle as '' + + // Eat '!<' + skip(parser) + skip(parser) + + // Consume the tag value. + if !yaml_parser_scan_tag_uri(parser, false, nil, start_mark, &suffix) { + return false + } + + // Check for '>' and eat it. + if parser.buffer[parser.buffer_pos] != '>' { + yaml_parser_set_scanner_error(parser, "while scanning a tag", + start_mark, "did not find the expected '>'") + return false + } + + skip(parser) + } else { + // The tag has either the '!suffix' or the '!handle!suffix' form. + + // First, try to scan a handle. + if !yaml_parser_scan_tag_handle(parser, false, start_mark, &handle) { + return false + } + + // Check if it is, indeed, handle. + if handle[0] == '!' && len(handle) > 1 && handle[len(handle)-1] == '!' { + // Scan the suffix now. + if !yaml_parser_scan_tag_uri(parser, false, nil, start_mark, &suffix) { + return false + } + } else { + // It wasn't a handle after all. Scan the rest of the tag. + if !yaml_parser_scan_tag_uri(parser, false, handle, start_mark, &suffix) { + return false + } + + // Set the handle to '!'. + handle = []byte{'!'} + + // A special case: the '!' tag. Set the handle to '' and the + // suffix to '!'. + if len(suffix) == 0 { + handle, suffix = suffix, handle + } + } + } + + // Check the character which ends the tag. + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + if !is_blankz(parser.buffer, parser.buffer_pos) { + yaml_parser_set_scanner_error(parser, "while scanning a tag", + start_mark, "did not find expected whitespace or line break") + return false + } + + end_mark := parser.mark + + // Create a token. + *token = yaml_token_t{ + typ: yaml_TAG_TOKEN, + start_mark: start_mark, + end_mark: end_mark, + value: handle, + suffix: suffix, + } + return true +} + +// Scan a tag handle. +func yaml_parser_scan_tag_handle(parser *yaml_parser_t, directive bool, start_mark yaml_mark_t, handle *[]byte) bool { + // Check the initial '!' character. + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + if parser.buffer[parser.buffer_pos] != '!' { + yaml_parser_set_scanner_tag_error(parser, directive, + start_mark, "did not find expected '!'") + return false + } + + var s []byte + + // Copy the '!' character. + s = read(parser, s) + + // Copy all subsequent alphabetical and numerical characters. + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + for is_alpha(parser.buffer, parser.buffer_pos) { + s = read(parser, s) + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + } + + // Check if the trailing character is '!' and copy it. + if parser.buffer[parser.buffer_pos] == '!' { + s = read(parser, s) + } else { + // It's either the '!' tag or not really a tag handle. If it's a %TAG + // directive, it's an error. If it's a tag token, it must be a part of URI. + if directive && string(s) != "!" { + yaml_parser_set_scanner_tag_error(parser, directive, + start_mark, "did not find expected '!'") + return false + } + } + + *handle = s + return true +} + +// Scan a tag. +func yaml_parser_scan_tag_uri(parser *yaml_parser_t, directive bool, head []byte, start_mark yaml_mark_t, uri *[]byte) bool { + //size_t length = head ? strlen((char *)head) : 0 + var s []byte + hasTag := len(head) > 0 + + // Copy the head if needed. + // + // Note that we don't copy the leading '!' character. + if len(head) > 1 { + s = append(s, head[1:]...) + } + + // Scan the tag. + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + + // The set of characters that may appear in URI is as follows: + // + // '0'-'9', 'A'-'Z', 'a'-'z', '_', '-', ';', '/', '?', ':', '@', '&', + // '=', '+', '$', ',', '.', '!', '~', '*', '\'', '(', ')', '[', ']', + // '%'. + // [Go] Convert this into more reasonable logic. + for is_alpha(parser.buffer, parser.buffer_pos) || parser.buffer[parser.buffer_pos] == ';' || + parser.buffer[parser.buffer_pos] == '/' || parser.buffer[parser.buffer_pos] == '?' || + parser.buffer[parser.buffer_pos] == ':' || parser.buffer[parser.buffer_pos] == '@' || + parser.buffer[parser.buffer_pos] == '&' || parser.buffer[parser.buffer_pos] == '=' || + parser.buffer[parser.buffer_pos] == '+' || parser.buffer[parser.buffer_pos] == '$' || + parser.buffer[parser.buffer_pos] == ',' || parser.buffer[parser.buffer_pos] == '.' || + parser.buffer[parser.buffer_pos] == '!' || parser.buffer[parser.buffer_pos] == '~' || + parser.buffer[parser.buffer_pos] == '*' || parser.buffer[parser.buffer_pos] == '\'' || + parser.buffer[parser.buffer_pos] == '(' || parser.buffer[parser.buffer_pos] == ')' || + parser.buffer[parser.buffer_pos] == '[' || parser.buffer[parser.buffer_pos] == ']' || + parser.buffer[parser.buffer_pos] == '%' { + // Check if it is a URI-escape sequence. + if parser.buffer[parser.buffer_pos] == '%' { + if !yaml_parser_scan_uri_escapes(parser, directive, start_mark, &s) { + return false + } + } else { + s = read(parser, s) + } + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + hasTag = true + } + + if !hasTag { + yaml_parser_set_scanner_tag_error(parser, directive, + start_mark, "did not find expected tag URI") + return false + } + *uri = s + return true +} + +// Decode an URI-escape sequence corresponding to a single UTF-8 character. +func yaml_parser_scan_uri_escapes(parser *yaml_parser_t, directive bool, start_mark yaml_mark_t, s *[]byte) bool { + + // Decode the required number of characters. + w := 1024 + for w > 0 { + // Check for a URI-escaped octet. + if parser.unread < 3 && !yaml_parser_update_buffer(parser, 3) { + return false + } + + if !(parser.buffer[parser.buffer_pos] == '%' && + is_hex(parser.buffer, parser.buffer_pos+1) && + is_hex(parser.buffer, parser.buffer_pos+2)) { + return yaml_parser_set_scanner_tag_error(parser, directive, + start_mark, "did not find URI escaped octet") + } + + // Get the octet. + octet := byte((as_hex(parser.buffer, parser.buffer_pos+1) << 4) + as_hex(parser.buffer, parser.buffer_pos+2)) + + // If it is the leading octet, determine the length of the UTF-8 sequence. + if w == 1024 { + w = width(octet) + if w == 0 { + return yaml_parser_set_scanner_tag_error(parser, directive, + start_mark, "found an incorrect leading UTF-8 octet") + } + } else { + // Check if the trailing octet is correct. + if octet&0xC0 != 0x80 { + return yaml_parser_set_scanner_tag_error(parser, directive, + start_mark, "found an incorrect trailing UTF-8 octet") + } + } + + // Copy the octet and move the pointers. + *s = append(*s, octet) + skip(parser) + skip(parser) + skip(parser) + w-- + } + return true +} + +// Scan a block scalar. +func yaml_parser_scan_block_scalar(parser *yaml_parser_t, token *yaml_token_t, literal bool) bool { + // Eat the indicator '|' or '>'. + start_mark := parser.mark + skip(parser) + + // Scan the additional block scalar indicators. + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + + // Check for a chomping indicator. + var chomping, increment int + if parser.buffer[parser.buffer_pos] == '+' || parser.buffer[parser.buffer_pos] == '-' { + // Set the chomping method and eat the indicator. + if parser.buffer[parser.buffer_pos] == '+' { + chomping = +1 + } else { + chomping = -1 + } + skip(parser) + + // Check for an indentation indicator. + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + if is_digit(parser.buffer, parser.buffer_pos) { + // Check that the indentation is greater than 0. + if parser.buffer[parser.buffer_pos] == '0' { + yaml_parser_set_scanner_error(parser, "while scanning a block scalar", + start_mark, "found an indentation indicator equal to 0") + return false + } + + // Get the indentation level and eat the indicator. + increment = as_digit(parser.buffer, parser.buffer_pos) + skip(parser) + } + + } else if is_digit(parser.buffer, parser.buffer_pos) { + // Do the same as above, but in the opposite order. + + if parser.buffer[parser.buffer_pos] == '0' { + yaml_parser_set_scanner_error(parser, "while scanning a block scalar", + start_mark, "found an indentation indicator equal to 0") + return false + } + increment = as_digit(parser.buffer, parser.buffer_pos) + skip(parser) + + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + if parser.buffer[parser.buffer_pos] == '+' || parser.buffer[parser.buffer_pos] == '-' { + if parser.buffer[parser.buffer_pos] == '+' { + chomping = +1 + } else { + chomping = -1 + } + skip(parser) + } + } + + // Eat whitespaces and comments to the end of the line. + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + for is_blank(parser.buffer, parser.buffer_pos) { + skip(parser) + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + } + if parser.buffer[parser.buffer_pos] == '#' { + for !is_breakz(parser.buffer, parser.buffer_pos) { + skip(parser) + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + } + } + + // Check if we are at the end of the line. + if !is_breakz(parser.buffer, parser.buffer_pos) { + yaml_parser_set_scanner_error(parser, "while scanning a block scalar", + start_mark, "did not find expected comment or line break") + return false + } + + // Eat a line break. + if is_break(parser.buffer, parser.buffer_pos) { + if parser.unread < 2 && !yaml_parser_update_buffer(parser, 2) { + return false + } + skip_line(parser) + } + + end_mark := parser.mark + + // Set the indentation level if it was specified. + var indent int + if increment > 0 { + if parser.indent >= 0 { + indent = parser.indent + increment + } else { + indent = increment + } + } + + // Scan the leading line breaks and determine the indentation level if needed. + var s, leading_break, trailing_breaks []byte + if !yaml_parser_scan_block_scalar_breaks(parser, &indent, &trailing_breaks, start_mark, &end_mark) { + return false + } + + // Scan the block scalar content. + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + var leading_blank, trailing_blank bool + for parser.mark.column == indent && !is_z(parser.buffer, parser.buffer_pos) { + // We are at the beginning of a non-empty line. + + // Is it a trailing whitespace? + trailing_blank = is_blank(parser.buffer, parser.buffer_pos) + + // Check if we need to fold the leading line break. + if !literal && !leading_blank && !trailing_blank && len(leading_break) > 0 && leading_break[0] == '\n' { + // Do we need to join the lines by space? + if len(trailing_breaks) == 0 { + s = append(s, ' ') + } + } else { + s = append(s, leading_break...) + } + leading_break = leading_break[:0] + + // Append the remaining line breaks. + s = append(s, trailing_breaks...) + trailing_breaks = trailing_breaks[:0] + + // Is it a leading whitespace? + leading_blank = is_blank(parser.buffer, parser.buffer_pos) + + // Consume the current line. + for !is_breakz(parser.buffer, parser.buffer_pos) { + s = read(parser, s) + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + } + + // Consume the line break. + if parser.unread < 2 && !yaml_parser_update_buffer(parser, 2) { + return false + } + + leading_break = read_line(parser, leading_break) + + // Eat the following indentation spaces and line breaks. + if !yaml_parser_scan_block_scalar_breaks(parser, &indent, &trailing_breaks, start_mark, &end_mark) { + return false + } + } + + // Chomp the tail. + if chomping != -1 { + s = append(s, leading_break...) + } + if chomping == 1 { + s = append(s, trailing_breaks...) + } + + // Create a token. + *token = yaml_token_t{ + typ: yaml_SCALAR_TOKEN, + start_mark: start_mark, + end_mark: end_mark, + value: s, + style: yaml_LITERAL_SCALAR_STYLE, + } + if !literal { + token.style = yaml_FOLDED_SCALAR_STYLE + } + return true +} + +// Scan indentation spaces and line breaks for a block scalar. Determine the +// indentation level if needed. +func yaml_parser_scan_block_scalar_breaks(parser *yaml_parser_t, indent *int, breaks *[]byte, start_mark yaml_mark_t, end_mark *yaml_mark_t) bool { + *end_mark = parser.mark + + // Eat the indentation spaces and line breaks. + max_indent := 0 + for { + // Eat the indentation spaces. + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + for (*indent == 0 || parser.mark.column < *indent) && is_space(parser.buffer, parser.buffer_pos) { + skip(parser) + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + } + if parser.mark.column > max_indent { + max_indent = parser.mark.column + } + + // Check for a tab character messing the indentation. + if (*indent == 0 || parser.mark.column < *indent) && is_tab(parser.buffer, parser.buffer_pos) { + return yaml_parser_set_scanner_error(parser, "while scanning a block scalar", + start_mark, "found a tab character where an indentation space is expected") + } + + // Have we found a non-empty line? + if !is_break(parser.buffer, parser.buffer_pos) { + break + } + + // Consume the line break. + if parser.unread < 2 && !yaml_parser_update_buffer(parser, 2) { + return false + } + // [Go] Should really be returning breaks instead. + *breaks = read_line(parser, *breaks) + *end_mark = parser.mark + } + + // Determine the indentation level if needed. + if *indent == 0 { + *indent = max_indent + if *indent < parser.indent+1 { + *indent = parser.indent + 1 + } + if *indent < 1 { + *indent = 1 + } + } + return true +} + +// Scan a quoted scalar. +func yaml_parser_scan_flow_scalar(parser *yaml_parser_t, token *yaml_token_t, single bool) bool { + // Eat the left quote. + start_mark := parser.mark + skip(parser) + + // Consume the content of the quoted scalar. + var s, leading_break, trailing_breaks, whitespaces []byte + for { + // Check that there are no document indicators at the beginning of the line. + if parser.unread < 4 && !yaml_parser_update_buffer(parser, 4) { + return false + } + + if parser.mark.column == 0 && + ((parser.buffer[parser.buffer_pos+0] == '-' && + parser.buffer[parser.buffer_pos+1] == '-' && + parser.buffer[parser.buffer_pos+2] == '-') || + (parser.buffer[parser.buffer_pos+0] == '.' && + parser.buffer[parser.buffer_pos+1] == '.' && + parser.buffer[parser.buffer_pos+2] == '.')) && + is_blankz(parser.buffer, parser.buffer_pos+3) { + yaml_parser_set_scanner_error(parser, "while scanning a quoted scalar", + start_mark, "found unexpected document indicator") + return false + } + + // Check for EOF. + if is_z(parser.buffer, parser.buffer_pos) { + yaml_parser_set_scanner_error(parser, "while scanning a quoted scalar", + start_mark, "found unexpected end of stream") + return false + } + + // Consume non-blank characters. + leading_blanks := false + for !is_blankz(parser.buffer, parser.buffer_pos) { + if single && parser.buffer[parser.buffer_pos] == '\'' && parser.buffer[parser.buffer_pos+1] == '\'' { + // Is is an escaped single quote. + s = append(s, '\'') + skip(parser) + skip(parser) + + } else if single && parser.buffer[parser.buffer_pos] == '\'' { + // It is a right single quote. + break + } else if !single && parser.buffer[parser.buffer_pos] == '"' { + // It is a right double quote. + break + + } else if !single && parser.buffer[parser.buffer_pos] == '\\' && is_break(parser.buffer, parser.buffer_pos+1) { + // It is an escaped line break. + if parser.unread < 3 && !yaml_parser_update_buffer(parser, 3) { + return false + } + skip(parser) + skip_line(parser) + leading_blanks = true + break + + } else if !single && parser.buffer[parser.buffer_pos] == '\\' { + // It is an escape sequence. + code_length := 0 + + // Check the escape character. + switch parser.buffer[parser.buffer_pos+1] { + case '0': + s = append(s, 0) + case 'a': + s = append(s, '\x07') + case 'b': + s = append(s, '\x08') + case 't', '\t': + s = append(s, '\x09') + case 'n': + s = append(s, '\x0A') + case 'v': + s = append(s, '\x0B') + case 'f': + s = append(s, '\x0C') + case 'r': + s = append(s, '\x0D') + case 'e': + s = append(s, '\x1B') + case ' ': + s = append(s, '\x20') + case '"': + s = append(s, '"') + case '\'': + s = append(s, '\'') + case '\\': + s = append(s, '\\') + case 'N': // NEL (#x85) + s = append(s, '\xC2') + s = append(s, '\x85') + case '_': // #xA0 + s = append(s, '\xC2') + s = append(s, '\xA0') + case 'L': // LS (#x2028) + s = append(s, '\xE2') + s = append(s, '\x80') + s = append(s, '\xA8') + case 'P': // PS (#x2029) + s = append(s, '\xE2') + s = append(s, '\x80') + s = append(s, '\xA9') + case 'x': + code_length = 2 + case 'u': + code_length = 4 + case 'U': + code_length = 8 + default: + yaml_parser_set_scanner_error(parser, "while parsing a quoted scalar", + start_mark, "found unknown escape character") + return false + } + + skip(parser) + skip(parser) + + // Consume an arbitrary escape code. + if code_length > 0 { + var value int + + // Scan the character value. + if parser.unread < code_length && !yaml_parser_update_buffer(parser, code_length) { + return false + } + for k := 0; k < code_length; k++ { + if !is_hex(parser.buffer, parser.buffer_pos+k) { + yaml_parser_set_scanner_error(parser, "while parsing a quoted scalar", + start_mark, "did not find expected hexdecimal number") + return false + } + value = (value << 4) + as_hex(parser.buffer, parser.buffer_pos+k) + } + + // Check the value and write the character. + if (value >= 0xD800 && value <= 0xDFFF) || value > 0x10FFFF { + yaml_parser_set_scanner_error(parser, "while parsing a quoted scalar", + start_mark, "found invalid Unicode character escape code") + return false + } + if value <= 0x7F { + s = append(s, byte(value)) + } else if value <= 0x7FF { + s = append(s, byte(0xC0+(value>>6))) + s = append(s, byte(0x80+(value&0x3F))) + } else if value <= 0xFFFF { + s = append(s, byte(0xE0+(value>>12))) + s = append(s, byte(0x80+((value>>6)&0x3F))) + s = append(s, byte(0x80+(value&0x3F))) + } else { + s = append(s, byte(0xF0+(value>>18))) + s = append(s, byte(0x80+((value>>12)&0x3F))) + s = append(s, byte(0x80+((value>>6)&0x3F))) + s = append(s, byte(0x80+(value&0x3F))) + } + + // Advance the pointer. + for k := 0; k < code_length; k++ { + skip(parser) + } + } + } else { + // It is a non-escaped non-blank character. + s = read(parser, s) + } + if parser.unread < 2 && !yaml_parser_update_buffer(parser, 2) { + return false + } + } + + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + + // Check if we are at the end of the scalar. + if single { + if parser.buffer[parser.buffer_pos] == '\'' { + break + } + } else { + if parser.buffer[parser.buffer_pos] == '"' { + break + } + } + + // Consume blank characters. + for is_blank(parser.buffer, parser.buffer_pos) || is_break(parser.buffer, parser.buffer_pos) { + if is_blank(parser.buffer, parser.buffer_pos) { + // Consume a space or a tab character. + if !leading_blanks { + whitespaces = read(parser, whitespaces) + } else { + skip(parser) + } + } else { + if parser.unread < 2 && !yaml_parser_update_buffer(parser, 2) { + return false + } + + // Check if it is a first line break. + if !leading_blanks { + whitespaces = whitespaces[:0] + leading_break = read_line(parser, leading_break) + leading_blanks = true + } else { + trailing_breaks = read_line(parser, trailing_breaks) + } + } + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + } + + // Join the whitespaces or fold line breaks. + if leading_blanks { + // Do we need to fold line breaks? + if len(leading_break) > 0 && leading_break[0] == '\n' { + if len(trailing_breaks) == 0 { + s = append(s, ' ') + } else { + s = append(s, trailing_breaks...) + } + } else { + s = append(s, leading_break...) + s = append(s, trailing_breaks...) + } + trailing_breaks = trailing_breaks[:0] + leading_break = leading_break[:0] + } else { + s = append(s, whitespaces...) + whitespaces = whitespaces[:0] + } + } + + // Eat the right quote. + skip(parser) + end_mark := parser.mark + + // Create a token. + *token = yaml_token_t{ + typ: yaml_SCALAR_TOKEN, + start_mark: start_mark, + end_mark: end_mark, + value: s, + style: yaml_SINGLE_QUOTED_SCALAR_STYLE, + } + if !single { + token.style = yaml_DOUBLE_QUOTED_SCALAR_STYLE + } + return true +} + +// Scan a plain scalar. +func yaml_parser_scan_plain_scalar(parser *yaml_parser_t, token *yaml_token_t) bool { + + var s, leading_break, trailing_breaks, whitespaces []byte + var leading_blanks bool + var indent = parser.indent + 1 + + start_mark := parser.mark + end_mark := parser.mark + + // Consume the content of the plain scalar. + for { + // Check for a document indicator. + if parser.unread < 4 && !yaml_parser_update_buffer(parser, 4) { + return false + } + if parser.mark.column == 0 && + ((parser.buffer[parser.buffer_pos+0] == '-' && + parser.buffer[parser.buffer_pos+1] == '-' && + parser.buffer[parser.buffer_pos+2] == '-') || + (parser.buffer[parser.buffer_pos+0] == '.' && + parser.buffer[parser.buffer_pos+1] == '.' && + parser.buffer[parser.buffer_pos+2] == '.')) && + is_blankz(parser.buffer, parser.buffer_pos+3) { + break + } + + // Check for a comment. + if parser.buffer[parser.buffer_pos] == '#' { + break + } + + // Consume non-blank characters. + for !is_blankz(parser.buffer, parser.buffer_pos) { + + // Check for indicators that may end a plain scalar. + if (parser.buffer[parser.buffer_pos] == ':' && is_blankz(parser.buffer, parser.buffer_pos+1)) || + (parser.flow_level > 0 && + (parser.buffer[parser.buffer_pos] == ',' || + parser.buffer[parser.buffer_pos] == '?' || parser.buffer[parser.buffer_pos] == '[' || + parser.buffer[parser.buffer_pos] == ']' || parser.buffer[parser.buffer_pos] == '{' || + parser.buffer[parser.buffer_pos] == '}')) { + break + } + + // Check if we need to join whitespaces and breaks. + if leading_blanks || len(whitespaces) > 0 { + if leading_blanks { + // Do we need to fold line breaks? + if leading_break[0] == '\n' { + if len(trailing_breaks) == 0 { + s = append(s, ' ') + } else { + s = append(s, trailing_breaks...) + } + } else { + s = append(s, leading_break...) + s = append(s, trailing_breaks...) + } + trailing_breaks = trailing_breaks[:0] + leading_break = leading_break[:0] + leading_blanks = false + } else { + s = append(s, whitespaces...) + whitespaces = whitespaces[:0] + } + } + + // Copy the character. + s = read(parser, s) + + end_mark = parser.mark + if parser.unread < 2 && !yaml_parser_update_buffer(parser, 2) { + return false + } + } + + // Is it the end? + if !(is_blank(parser.buffer, parser.buffer_pos) || is_break(parser.buffer, parser.buffer_pos)) { + break + } + + // Consume blank characters. + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + + for is_blank(parser.buffer, parser.buffer_pos) || is_break(parser.buffer, parser.buffer_pos) { + if is_blank(parser.buffer, parser.buffer_pos) { + + // Check for tab characters that abuse indentation. + if leading_blanks && parser.mark.column < indent && is_tab(parser.buffer, parser.buffer_pos) { + yaml_parser_set_scanner_error(parser, "while scanning a plain scalar", + start_mark, "found a tab character that violates indentation") + return false + } + + // Consume a space or a tab character. + if !leading_blanks { + whitespaces = read(parser, whitespaces) + } else { + skip(parser) + } + } else { + if parser.unread < 2 && !yaml_parser_update_buffer(parser, 2) { + return false + } + + // Check if it is a first line break. + if !leading_blanks { + whitespaces = whitespaces[:0] + leading_break = read_line(parser, leading_break) + leading_blanks = true + } else { + trailing_breaks = read_line(parser, trailing_breaks) + } + } + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + } + + // Check indentation level. + if parser.flow_level == 0 && parser.mark.column < indent { + break + } + } + + // Create a token. + *token = yaml_token_t{ + typ: yaml_SCALAR_TOKEN, + start_mark: start_mark, + end_mark: end_mark, + value: s, + style: yaml_PLAIN_SCALAR_STYLE, + } + + // Note that we change the 'simple_key_allowed' flag. + if leading_blanks { + parser.simple_key_allowed = true + } + return true +} diff --git a/tools/voluspa/vendor/gopkg.in/yaml.v2/sorter.go b/tools/voluspa/vendor/gopkg.in/yaml.v2/sorter.go new file mode 100644 index 00000000000..4c45e660a8f --- /dev/null +++ b/tools/voluspa/vendor/gopkg.in/yaml.v2/sorter.go @@ -0,0 +1,113 @@ +package yaml + +import ( + "reflect" + "unicode" +) + +type keyList []reflect.Value + +func (l keyList) Len() int { return len(l) } +func (l keyList) Swap(i, j int) { l[i], l[j] = l[j], l[i] } +func (l keyList) Less(i, j int) bool { + a := l[i] + b := l[j] + ak := a.Kind() + bk := b.Kind() + for (ak == reflect.Interface || ak == reflect.Ptr) && !a.IsNil() { + a = a.Elem() + ak = a.Kind() + } + for (bk == reflect.Interface || bk == reflect.Ptr) && !b.IsNil() { + b = b.Elem() + bk = b.Kind() + } + af, aok := keyFloat(a) + bf, bok := keyFloat(b) + if aok && bok { + if af != bf { + return af < bf + } + if ak != bk { + return ak < bk + } + return numLess(a, b) + } + if ak != reflect.String || bk != reflect.String { + return ak < bk + } + ar, br := []rune(a.String()), []rune(b.String()) + for i := 0; i < len(ar) && i < len(br); i++ { + if ar[i] == br[i] { + continue + } + al := unicode.IsLetter(ar[i]) + bl := unicode.IsLetter(br[i]) + if al && bl { + return ar[i] < br[i] + } + if al || bl { + return bl + } + var ai, bi int + var an, bn int64 + if ar[i] == '0' || br[i] == '0' { + for j := i-1; j >= 0 && unicode.IsDigit(ar[j]); j-- { + if ar[j] != '0' { + an = 1 + bn = 1 + break + } + } + } + for ai = i; ai < len(ar) && unicode.IsDigit(ar[ai]); ai++ { + an = an*10 + int64(ar[ai]-'0') + } + for bi = i; bi < len(br) && unicode.IsDigit(br[bi]); bi++ { + bn = bn*10 + int64(br[bi]-'0') + } + if an != bn { + return an < bn + } + if ai != bi { + return ai < bi + } + return ar[i] < br[i] + } + return len(ar) < len(br) +} + +// keyFloat returns a float value for v if it is a number/bool +// and whether it is a number/bool or not. +func keyFloat(v reflect.Value) (f float64, ok bool) { + switch v.Kind() { + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + return float64(v.Int()), true + case reflect.Float32, reflect.Float64: + return v.Float(), true + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: + return float64(v.Uint()), true + case reflect.Bool: + if v.Bool() { + return 1, true + } + return 0, true + } + return 0, false +} + +// numLess returns whether a < b. +// a and b must necessarily have the same kind. +func numLess(a, b reflect.Value) bool { + switch a.Kind() { + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + return a.Int() < b.Int() + case reflect.Float32, reflect.Float64: + return a.Float() < b.Float() + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: + return a.Uint() < b.Uint() + case reflect.Bool: + return !a.Bool() && b.Bool() + } + panic("not a number") +} diff --git a/tools/voluspa/vendor/gopkg.in/yaml.v2/writerc.go b/tools/voluspa/vendor/gopkg.in/yaml.v2/writerc.go new file mode 100644 index 00000000000..a2dde608cb7 --- /dev/null +++ b/tools/voluspa/vendor/gopkg.in/yaml.v2/writerc.go @@ -0,0 +1,26 @@ +package yaml + +// Set the writer error and return false. +func yaml_emitter_set_writer_error(emitter *yaml_emitter_t, problem string) bool { + emitter.error = yaml_WRITER_ERROR + emitter.problem = problem + return false +} + +// Flush the output buffer. +func yaml_emitter_flush(emitter *yaml_emitter_t) bool { + if emitter.write_handler == nil { + panic("write handler not set") + } + + // Check if the buffer is empty. + if emitter.buffer_pos == 0 { + return true + } + + if err := emitter.write_handler(emitter, emitter.buffer[:emitter.buffer_pos]); err != nil { + return yaml_emitter_set_writer_error(emitter, "write error: "+err.Error()) + } + emitter.buffer_pos = 0 + return true +} diff --git a/tools/voluspa/vendor/gopkg.in/yaml.v2/yaml.go b/tools/voluspa/vendor/gopkg.in/yaml.v2/yaml.go new file mode 100644 index 00000000000..de85aa4cdb7 --- /dev/null +++ b/tools/voluspa/vendor/gopkg.in/yaml.v2/yaml.go @@ -0,0 +1,466 @@ +// Package yaml implements YAML support for the Go language. +// +// Source code and other details for the project are available at GitHub: +// +// https://github.com/go-yaml/yaml +// +package yaml + +import ( + "errors" + "fmt" + "io" + "reflect" + "strings" + "sync" +) + +// MapSlice encodes and decodes as a YAML map. +// The order of keys is preserved when encoding and decoding. +type MapSlice []MapItem + +// MapItem is an item in a MapSlice. +type MapItem struct { + Key, Value interface{} +} + +// The Unmarshaler interface may be implemented by types to customize their +// behavior when being unmarshaled from a YAML document. The UnmarshalYAML +// method receives a function that may be called to unmarshal the original +// YAML value into a field or variable. It is safe to call the unmarshal +// function parameter more than once if necessary. +type Unmarshaler interface { + UnmarshalYAML(unmarshal func(interface{}) error) error +} + +// The Marshaler interface may be implemented by types to customize their +// behavior when being marshaled into a YAML document. The returned value +// is marshaled in place of the original value implementing Marshaler. +// +// If an error is returned by MarshalYAML, the marshaling procedure stops +// and returns with the provided error. +type Marshaler interface { + MarshalYAML() (interface{}, error) +} + +// Unmarshal decodes the first document found within the in byte slice +// and assigns decoded values into the out value. +// +// Maps and pointers (to a struct, string, int, etc) are accepted as out +// values. If an internal pointer within a struct is not initialized, +// the yaml package will initialize it if necessary for unmarshalling +// the provided data. The out parameter must not be nil. +// +// The type of the decoded values should be compatible with the respective +// values in out. If one or more values cannot be decoded due to a type +// mismatches, decoding continues partially until the end of the YAML +// content, and a *yaml.TypeError is returned with details for all +// missed values. +// +// Struct fields are only unmarshalled if they are exported (have an +// upper case first letter), and are unmarshalled using the field name +// lowercased as the default key. Custom keys may be defined via the +// "yaml" name in the field tag: the content preceding the first comma +// is used as the key, and the following comma-separated options are +// used to tweak the marshalling process (see Marshal). +// Conflicting names result in a runtime error. +// +// For example: +// +// type T struct { +// F int `yaml:"a,omitempty"` +// B int +// } +// var t T +// yaml.Unmarshal([]byte("a: 1\nb: 2"), &t) +// +// See the documentation of Marshal for the format of tags and a list of +// supported tag options. +// +func Unmarshal(in []byte, out interface{}) (err error) { + return unmarshal(in, out, false) +} + +// UnmarshalStrict is like Unmarshal except that any fields that are found +// in the data that do not have corresponding struct members, or mapping +// keys that are duplicates, will result in +// an error. +func UnmarshalStrict(in []byte, out interface{}) (err error) { + return unmarshal(in, out, true) +} + +// A Decorder reads and decodes YAML values from an input stream. +type Decoder struct { + strict bool + parser *parser +} + +// NewDecoder returns a new decoder that reads from r. +// +// The decoder introduces its own buffering and may read +// data from r beyond the YAML values requested. +func NewDecoder(r io.Reader) *Decoder { + return &Decoder{ + parser: newParserFromReader(r), + } +} + +// SetStrict sets whether strict decoding behaviour is enabled when +// decoding items in the data (see UnmarshalStrict). By default, decoding is not strict. +func (dec *Decoder) SetStrict(strict bool) { + dec.strict = strict +} + +// Decode reads the next YAML-encoded value from its input +// and stores it in the value pointed to by v. +// +// See the documentation for Unmarshal for details about the +// conversion of YAML into a Go value. +func (dec *Decoder) Decode(v interface{}) (err error) { + d := newDecoder(dec.strict) + defer handleErr(&err) + node := dec.parser.parse() + if node == nil { + return io.EOF + } + out := reflect.ValueOf(v) + if out.Kind() == reflect.Ptr && !out.IsNil() { + out = out.Elem() + } + d.unmarshal(node, out) + if len(d.terrors) > 0 { + return &TypeError{d.terrors} + } + return nil +} + +func unmarshal(in []byte, out interface{}, strict bool) (err error) { + defer handleErr(&err) + d := newDecoder(strict) + p := newParser(in) + defer p.destroy() + node := p.parse() + if node != nil { + v := reflect.ValueOf(out) + if v.Kind() == reflect.Ptr && !v.IsNil() { + v = v.Elem() + } + d.unmarshal(node, v) + } + if len(d.terrors) > 0 { + return &TypeError{d.terrors} + } + return nil +} + +// Marshal serializes the value provided into a YAML document. The structure +// of the generated document will reflect the structure of the value itself. +// Maps and pointers (to struct, string, int, etc) are accepted as the in value. +// +// Struct fields are only marshalled if they are exported (have an upper case +// first letter), and are marshalled using the field name lowercased as the +// default key. Custom keys may be defined via the "yaml" name in the field +// tag: the content preceding the first comma is used as the key, and the +// following comma-separated options are used to tweak the marshalling process. +// Conflicting names result in a runtime error. +// +// The field tag format accepted is: +// +// `(...) yaml:"[][,[,]]" (...)` +// +// The following flags are currently supported: +// +// omitempty Only include the field if it's not set to the zero +// value for the type or to empty slices or maps. +// Zero valued structs will be omitted if all their public +// fields are zero, unless they implement an IsZero +// method (see the IsZeroer interface type), in which +// case the field will be included if that method returns true. +// +// flow Marshal using a flow style (useful for structs, +// sequences and maps). +// +// inline Inline the field, which must be a struct or a map, +// causing all of its fields or keys to be processed as if +// they were part of the outer struct. For maps, keys must +// not conflict with the yaml keys of other struct fields. +// +// In addition, if the key is "-", the field is ignored. +// +// For example: +// +// type T struct { +// F int `yaml:"a,omitempty"` +// B int +// } +// yaml.Marshal(&T{B: 2}) // Returns "b: 2\n" +// yaml.Marshal(&T{F: 1}} // Returns "a: 1\nb: 0\n" +// +func Marshal(in interface{}) (out []byte, err error) { + defer handleErr(&err) + e := newEncoder() + defer e.destroy() + e.marshalDoc("", reflect.ValueOf(in)) + e.finish() + out = e.out + return +} + +// An Encoder writes YAML values to an output stream. +type Encoder struct { + encoder *encoder +} + +// NewEncoder returns a new encoder that writes to w. +// The Encoder should be closed after use to flush all data +// to w. +func NewEncoder(w io.Writer) *Encoder { + return &Encoder{ + encoder: newEncoderWithWriter(w), + } +} + +// Encode writes the YAML encoding of v to the stream. +// If multiple items are encoded to the stream, the +// second and subsequent document will be preceded +// with a "---" document separator, but the first will not. +// +// See the documentation for Marshal for details about the conversion of Go +// values to YAML. +func (e *Encoder) Encode(v interface{}) (err error) { + defer handleErr(&err) + e.encoder.marshalDoc("", reflect.ValueOf(v)) + return nil +} + +// Close closes the encoder by writing any remaining data. +// It does not write a stream terminating string "...". +func (e *Encoder) Close() (err error) { + defer handleErr(&err) + e.encoder.finish() + return nil +} + +func handleErr(err *error) { + if v := recover(); v != nil { + if e, ok := v.(yamlError); ok { + *err = e.err + } else { + panic(v) + } + } +} + +type yamlError struct { + err error +} + +func fail(err error) { + panic(yamlError{err}) +} + +func failf(format string, args ...interface{}) { + panic(yamlError{fmt.Errorf("yaml: "+format, args...)}) +} + +// A TypeError is returned by Unmarshal when one or more fields in +// the YAML document cannot be properly decoded into the requested +// types. When this error is returned, the value is still +// unmarshaled partially. +type TypeError struct { + Errors []string +} + +func (e *TypeError) Error() string { + return fmt.Sprintf("yaml: unmarshal errors:\n %s", strings.Join(e.Errors, "\n ")) +} + +// -------------------------------------------------------------------------- +// Maintain a mapping of keys to structure field indexes + +// The code in this section was copied from mgo/bson. + +// structInfo holds details for the serialization of fields of +// a given struct. +type structInfo struct { + FieldsMap map[string]fieldInfo + FieldsList []fieldInfo + + // InlineMap is the number of the field in the struct that + // contains an ,inline map, or -1 if there's none. + InlineMap int +} + +type fieldInfo struct { + Key string + Num int + OmitEmpty bool + Flow bool + // Id holds the unique field identifier, so we can cheaply + // check for field duplicates without maintaining an extra map. + Id int + + // Inline holds the field index if the field is part of an inlined struct. + Inline []int +} + +var structMap = make(map[reflect.Type]*structInfo) +var fieldMapMutex sync.RWMutex + +func getStructInfo(st reflect.Type) (*structInfo, error) { + fieldMapMutex.RLock() + sinfo, found := structMap[st] + fieldMapMutex.RUnlock() + if found { + return sinfo, nil + } + + n := st.NumField() + fieldsMap := make(map[string]fieldInfo) + fieldsList := make([]fieldInfo, 0, n) + inlineMap := -1 + for i := 0; i != n; i++ { + field := st.Field(i) + if field.PkgPath != "" && !field.Anonymous { + continue // Private field + } + + info := fieldInfo{Num: i} + + tag := field.Tag.Get("yaml") + if tag == "" && strings.Index(string(field.Tag), ":") < 0 { + tag = string(field.Tag) + } + if tag == "-" { + continue + } + + inline := false + fields := strings.Split(tag, ",") + if len(fields) > 1 { + for _, flag := range fields[1:] { + switch flag { + case "omitempty": + info.OmitEmpty = true + case "flow": + info.Flow = true + case "inline": + inline = true + default: + return nil, errors.New(fmt.Sprintf("Unsupported flag %q in tag %q of type %s", flag, tag, st)) + } + } + tag = fields[0] + } + + if inline { + switch field.Type.Kind() { + case reflect.Map: + if inlineMap >= 0 { + return nil, errors.New("Multiple ,inline maps in struct " + st.String()) + } + if field.Type.Key() != reflect.TypeOf("") { + return nil, errors.New("Option ,inline needs a map with string keys in struct " + st.String()) + } + inlineMap = info.Num + case reflect.Struct: + sinfo, err := getStructInfo(field.Type) + if err != nil { + return nil, err + } + for _, finfo := range sinfo.FieldsList { + if _, found := fieldsMap[finfo.Key]; found { + msg := "Duplicated key '" + finfo.Key + "' in struct " + st.String() + return nil, errors.New(msg) + } + if finfo.Inline == nil { + finfo.Inline = []int{i, finfo.Num} + } else { + finfo.Inline = append([]int{i}, finfo.Inline...) + } + finfo.Id = len(fieldsList) + fieldsMap[finfo.Key] = finfo + fieldsList = append(fieldsList, finfo) + } + default: + //return nil, errors.New("Option ,inline needs a struct value or map field") + return nil, errors.New("Option ,inline needs a struct value field") + } + continue + } + + if tag != "" { + info.Key = tag + } else { + info.Key = strings.ToLower(field.Name) + } + + if _, found = fieldsMap[info.Key]; found { + msg := "Duplicated key '" + info.Key + "' in struct " + st.String() + return nil, errors.New(msg) + } + + info.Id = len(fieldsList) + fieldsList = append(fieldsList, info) + fieldsMap[info.Key] = info + } + + sinfo = &structInfo{ + FieldsMap: fieldsMap, + FieldsList: fieldsList, + InlineMap: inlineMap, + } + + fieldMapMutex.Lock() + structMap[st] = sinfo + fieldMapMutex.Unlock() + return sinfo, nil +} + +// IsZeroer is used to check whether an object is zero to +// determine whether it should be omitted when marshaling +// with the omitempty flag. One notable implementation +// is time.Time. +type IsZeroer interface { + IsZero() bool +} + +func isZero(v reflect.Value) bool { + kind := v.Kind() + if z, ok := v.Interface().(IsZeroer); ok { + if (kind == reflect.Ptr || kind == reflect.Interface) && v.IsNil() { + return true + } + return z.IsZero() + } + switch kind { + case reflect.String: + return len(v.String()) == 0 + case reflect.Interface, reflect.Ptr: + return v.IsNil() + case reflect.Slice: + return v.Len() == 0 + case reflect.Map: + return v.Len() == 0 + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + return v.Int() == 0 + case reflect.Float32, reflect.Float64: + return v.Float() == 0 + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: + return v.Uint() == 0 + case reflect.Bool: + return !v.Bool() + case reflect.Struct: + vt := v.Type() + for i := v.NumField() - 1; i >= 0; i-- { + if vt.Field(i).PkgPath != "" { + continue // Private field + } + if !isZero(v.Field(i)) { + return false + } + } + return true + } + return false +} diff --git a/tools/voluspa/vendor/gopkg.in/yaml.v2/yamlh.go b/tools/voluspa/vendor/gopkg.in/yaml.v2/yamlh.go new file mode 100644 index 00000000000..e25cee563be --- /dev/null +++ b/tools/voluspa/vendor/gopkg.in/yaml.v2/yamlh.go @@ -0,0 +1,738 @@ +package yaml + +import ( + "fmt" + "io" +) + +// The version directive data. +type yaml_version_directive_t struct { + major int8 // The major version number. + minor int8 // The minor version number. +} + +// The tag directive data. +type yaml_tag_directive_t struct { + handle []byte // The tag handle. + prefix []byte // The tag prefix. +} + +type yaml_encoding_t int + +// The stream encoding. +const ( + // Let the parser choose the encoding. + yaml_ANY_ENCODING yaml_encoding_t = iota + + yaml_UTF8_ENCODING // The default UTF-8 encoding. + yaml_UTF16LE_ENCODING // The UTF-16-LE encoding with BOM. + yaml_UTF16BE_ENCODING // The UTF-16-BE encoding with BOM. +) + +type yaml_break_t int + +// Line break types. +const ( + // Let the parser choose the break type. + yaml_ANY_BREAK yaml_break_t = iota + + yaml_CR_BREAK // Use CR for line breaks (Mac style). + yaml_LN_BREAK // Use LN for line breaks (Unix style). + yaml_CRLN_BREAK // Use CR LN for line breaks (DOS style). +) + +type yaml_error_type_t int + +// Many bad things could happen with the parser and emitter. +const ( + // No error is produced. + yaml_NO_ERROR yaml_error_type_t = iota + + yaml_MEMORY_ERROR // Cannot allocate or reallocate a block of memory. + yaml_READER_ERROR // Cannot read or decode the input stream. + yaml_SCANNER_ERROR // Cannot scan the input stream. + yaml_PARSER_ERROR // Cannot parse the input stream. + yaml_COMPOSER_ERROR // Cannot compose a YAML document. + yaml_WRITER_ERROR // Cannot write to the output stream. + yaml_EMITTER_ERROR // Cannot emit a YAML stream. +) + +// The pointer position. +type yaml_mark_t struct { + index int // The position index. + line int // The position line. + column int // The position column. +} + +// Node Styles + +type yaml_style_t int8 + +type yaml_scalar_style_t yaml_style_t + +// Scalar styles. +const ( + // Let the emitter choose the style. + yaml_ANY_SCALAR_STYLE yaml_scalar_style_t = iota + + yaml_PLAIN_SCALAR_STYLE // The plain scalar style. + yaml_SINGLE_QUOTED_SCALAR_STYLE // The single-quoted scalar style. + yaml_DOUBLE_QUOTED_SCALAR_STYLE // The double-quoted scalar style. + yaml_LITERAL_SCALAR_STYLE // The literal scalar style. + yaml_FOLDED_SCALAR_STYLE // The folded scalar style. +) + +type yaml_sequence_style_t yaml_style_t + +// Sequence styles. +const ( + // Let the emitter choose the style. + yaml_ANY_SEQUENCE_STYLE yaml_sequence_style_t = iota + + yaml_BLOCK_SEQUENCE_STYLE // The block sequence style. + yaml_FLOW_SEQUENCE_STYLE // The flow sequence style. +) + +type yaml_mapping_style_t yaml_style_t + +// Mapping styles. +const ( + // Let the emitter choose the style. + yaml_ANY_MAPPING_STYLE yaml_mapping_style_t = iota + + yaml_BLOCK_MAPPING_STYLE // The block mapping style. + yaml_FLOW_MAPPING_STYLE // The flow mapping style. +) + +// Tokens + +type yaml_token_type_t int + +// Token types. +const ( + // An empty token. + yaml_NO_TOKEN yaml_token_type_t = iota + + yaml_STREAM_START_TOKEN // A STREAM-START token. + yaml_STREAM_END_TOKEN // A STREAM-END token. + + yaml_VERSION_DIRECTIVE_TOKEN // A VERSION-DIRECTIVE token. + yaml_TAG_DIRECTIVE_TOKEN // A TAG-DIRECTIVE token. + yaml_DOCUMENT_START_TOKEN // A DOCUMENT-START token. + yaml_DOCUMENT_END_TOKEN // A DOCUMENT-END token. + + yaml_BLOCK_SEQUENCE_START_TOKEN // A BLOCK-SEQUENCE-START token. + yaml_BLOCK_MAPPING_START_TOKEN // A BLOCK-SEQUENCE-END token. + yaml_BLOCK_END_TOKEN // A BLOCK-END token. + + yaml_FLOW_SEQUENCE_START_TOKEN // A FLOW-SEQUENCE-START token. + yaml_FLOW_SEQUENCE_END_TOKEN // A FLOW-SEQUENCE-END token. + yaml_FLOW_MAPPING_START_TOKEN // A FLOW-MAPPING-START token. + yaml_FLOW_MAPPING_END_TOKEN // A FLOW-MAPPING-END token. + + yaml_BLOCK_ENTRY_TOKEN // A BLOCK-ENTRY token. + yaml_FLOW_ENTRY_TOKEN // A FLOW-ENTRY token. + yaml_KEY_TOKEN // A KEY token. + yaml_VALUE_TOKEN // A VALUE token. + + yaml_ALIAS_TOKEN // An ALIAS token. + yaml_ANCHOR_TOKEN // An ANCHOR token. + yaml_TAG_TOKEN // A TAG token. + yaml_SCALAR_TOKEN // A SCALAR token. +) + +func (tt yaml_token_type_t) String() string { + switch tt { + case yaml_NO_TOKEN: + return "yaml_NO_TOKEN" + case yaml_STREAM_START_TOKEN: + return "yaml_STREAM_START_TOKEN" + case yaml_STREAM_END_TOKEN: + return "yaml_STREAM_END_TOKEN" + case yaml_VERSION_DIRECTIVE_TOKEN: + return "yaml_VERSION_DIRECTIVE_TOKEN" + case yaml_TAG_DIRECTIVE_TOKEN: + return "yaml_TAG_DIRECTIVE_TOKEN" + case yaml_DOCUMENT_START_TOKEN: + return "yaml_DOCUMENT_START_TOKEN" + case yaml_DOCUMENT_END_TOKEN: + return "yaml_DOCUMENT_END_TOKEN" + case yaml_BLOCK_SEQUENCE_START_TOKEN: + return "yaml_BLOCK_SEQUENCE_START_TOKEN" + case yaml_BLOCK_MAPPING_START_TOKEN: + return "yaml_BLOCK_MAPPING_START_TOKEN" + case yaml_BLOCK_END_TOKEN: + return "yaml_BLOCK_END_TOKEN" + case yaml_FLOW_SEQUENCE_START_TOKEN: + return "yaml_FLOW_SEQUENCE_START_TOKEN" + case yaml_FLOW_SEQUENCE_END_TOKEN: + return "yaml_FLOW_SEQUENCE_END_TOKEN" + case yaml_FLOW_MAPPING_START_TOKEN: + return "yaml_FLOW_MAPPING_START_TOKEN" + case yaml_FLOW_MAPPING_END_TOKEN: + return "yaml_FLOW_MAPPING_END_TOKEN" + case yaml_BLOCK_ENTRY_TOKEN: + return "yaml_BLOCK_ENTRY_TOKEN" + case yaml_FLOW_ENTRY_TOKEN: + return "yaml_FLOW_ENTRY_TOKEN" + case yaml_KEY_TOKEN: + return "yaml_KEY_TOKEN" + case yaml_VALUE_TOKEN: + return "yaml_VALUE_TOKEN" + case yaml_ALIAS_TOKEN: + return "yaml_ALIAS_TOKEN" + case yaml_ANCHOR_TOKEN: + return "yaml_ANCHOR_TOKEN" + case yaml_TAG_TOKEN: + return "yaml_TAG_TOKEN" + case yaml_SCALAR_TOKEN: + return "yaml_SCALAR_TOKEN" + } + return "" +} + +// The token structure. +type yaml_token_t struct { + // The token type. + typ yaml_token_type_t + + // The start/end of the token. + start_mark, end_mark yaml_mark_t + + // The stream encoding (for yaml_STREAM_START_TOKEN). + encoding yaml_encoding_t + + // The alias/anchor/scalar value or tag/tag directive handle + // (for yaml_ALIAS_TOKEN, yaml_ANCHOR_TOKEN, yaml_SCALAR_TOKEN, yaml_TAG_TOKEN, yaml_TAG_DIRECTIVE_TOKEN). + value []byte + + // The tag suffix (for yaml_TAG_TOKEN). + suffix []byte + + // The tag directive prefix (for yaml_TAG_DIRECTIVE_TOKEN). + prefix []byte + + // The scalar style (for yaml_SCALAR_TOKEN). + style yaml_scalar_style_t + + // The version directive major/minor (for yaml_VERSION_DIRECTIVE_TOKEN). + major, minor int8 +} + +// Events + +type yaml_event_type_t int8 + +// Event types. +const ( + // An empty event. + yaml_NO_EVENT yaml_event_type_t = iota + + yaml_STREAM_START_EVENT // A STREAM-START event. + yaml_STREAM_END_EVENT // A STREAM-END event. + yaml_DOCUMENT_START_EVENT // A DOCUMENT-START event. + yaml_DOCUMENT_END_EVENT // A DOCUMENT-END event. + yaml_ALIAS_EVENT // An ALIAS event. + yaml_SCALAR_EVENT // A SCALAR event. + yaml_SEQUENCE_START_EVENT // A SEQUENCE-START event. + yaml_SEQUENCE_END_EVENT // A SEQUENCE-END event. + yaml_MAPPING_START_EVENT // A MAPPING-START event. + yaml_MAPPING_END_EVENT // A MAPPING-END event. +) + +var eventStrings = []string{ + yaml_NO_EVENT: "none", + yaml_STREAM_START_EVENT: "stream start", + yaml_STREAM_END_EVENT: "stream end", + yaml_DOCUMENT_START_EVENT: "document start", + yaml_DOCUMENT_END_EVENT: "document end", + yaml_ALIAS_EVENT: "alias", + yaml_SCALAR_EVENT: "scalar", + yaml_SEQUENCE_START_EVENT: "sequence start", + yaml_SEQUENCE_END_EVENT: "sequence end", + yaml_MAPPING_START_EVENT: "mapping start", + yaml_MAPPING_END_EVENT: "mapping end", +} + +func (e yaml_event_type_t) String() string { + if e < 0 || int(e) >= len(eventStrings) { + return fmt.Sprintf("unknown event %d", e) + } + return eventStrings[e] +} + +// The event structure. +type yaml_event_t struct { + + // The event type. + typ yaml_event_type_t + + // The start and end of the event. + start_mark, end_mark yaml_mark_t + + // The document encoding (for yaml_STREAM_START_EVENT). + encoding yaml_encoding_t + + // The version directive (for yaml_DOCUMENT_START_EVENT). + version_directive *yaml_version_directive_t + + // The list of tag directives (for yaml_DOCUMENT_START_EVENT). + tag_directives []yaml_tag_directive_t + + // The anchor (for yaml_SCALAR_EVENT, yaml_SEQUENCE_START_EVENT, yaml_MAPPING_START_EVENT, yaml_ALIAS_EVENT). + anchor []byte + + // The tag (for yaml_SCALAR_EVENT, yaml_SEQUENCE_START_EVENT, yaml_MAPPING_START_EVENT). + tag []byte + + // The scalar value (for yaml_SCALAR_EVENT). + value []byte + + // Is the document start/end indicator implicit, or the tag optional? + // (for yaml_DOCUMENT_START_EVENT, yaml_DOCUMENT_END_EVENT, yaml_SEQUENCE_START_EVENT, yaml_MAPPING_START_EVENT, yaml_SCALAR_EVENT). + implicit bool + + // Is the tag optional for any non-plain style? (for yaml_SCALAR_EVENT). + quoted_implicit bool + + // The style (for yaml_SCALAR_EVENT, yaml_SEQUENCE_START_EVENT, yaml_MAPPING_START_EVENT). + style yaml_style_t +} + +func (e *yaml_event_t) scalar_style() yaml_scalar_style_t { return yaml_scalar_style_t(e.style) } +func (e *yaml_event_t) sequence_style() yaml_sequence_style_t { return yaml_sequence_style_t(e.style) } +func (e *yaml_event_t) mapping_style() yaml_mapping_style_t { return yaml_mapping_style_t(e.style) } + +// Nodes + +const ( + yaml_NULL_TAG = "tag:yaml.org,2002:null" // The tag !!null with the only possible value: null. + yaml_BOOL_TAG = "tag:yaml.org,2002:bool" // The tag !!bool with the values: true and false. + yaml_STR_TAG = "tag:yaml.org,2002:str" // The tag !!str for string values. + yaml_INT_TAG = "tag:yaml.org,2002:int" // The tag !!int for integer values. + yaml_FLOAT_TAG = "tag:yaml.org,2002:float" // The tag !!float for float values. + yaml_TIMESTAMP_TAG = "tag:yaml.org,2002:timestamp" // The tag !!timestamp for date and time values. + + yaml_SEQ_TAG = "tag:yaml.org,2002:seq" // The tag !!seq is used to denote sequences. + yaml_MAP_TAG = "tag:yaml.org,2002:map" // The tag !!map is used to denote mapping. + + // Not in original libyaml. + yaml_BINARY_TAG = "tag:yaml.org,2002:binary" + yaml_MERGE_TAG = "tag:yaml.org,2002:merge" + + yaml_DEFAULT_SCALAR_TAG = yaml_STR_TAG // The default scalar tag is !!str. + yaml_DEFAULT_SEQUENCE_TAG = yaml_SEQ_TAG // The default sequence tag is !!seq. + yaml_DEFAULT_MAPPING_TAG = yaml_MAP_TAG // The default mapping tag is !!map. +) + +type yaml_node_type_t int + +// Node types. +const ( + // An empty node. + yaml_NO_NODE yaml_node_type_t = iota + + yaml_SCALAR_NODE // A scalar node. + yaml_SEQUENCE_NODE // A sequence node. + yaml_MAPPING_NODE // A mapping node. +) + +// An element of a sequence node. +type yaml_node_item_t int + +// An element of a mapping node. +type yaml_node_pair_t struct { + key int // The key of the element. + value int // The value of the element. +} + +// The node structure. +type yaml_node_t struct { + typ yaml_node_type_t // The node type. + tag []byte // The node tag. + + // The node data. + + // The scalar parameters (for yaml_SCALAR_NODE). + scalar struct { + value []byte // The scalar value. + length int // The length of the scalar value. + style yaml_scalar_style_t // The scalar style. + } + + // The sequence parameters (for YAML_SEQUENCE_NODE). + sequence struct { + items_data []yaml_node_item_t // The stack of sequence items. + style yaml_sequence_style_t // The sequence style. + } + + // The mapping parameters (for yaml_MAPPING_NODE). + mapping struct { + pairs_data []yaml_node_pair_t // The stack of mapping pairs (key, value). + pairs_start *yaml_node_pair_t // The beginning of the stack. + pairs_end *yaml_node_pair_t // The end of the stack. + pairs_top *yaml_node_pair_t // The top of the stack. + style yaml_mapping_style_t // The mapping style. + } + + start_mark yaml_mark_t // The beginning of the node. + end_mark yaml_mark_t // The end of the node. + +} + +// The document structure. +type yaml_document_t struct { + + // The document nodes. + nodes []yaml_node_t + + // The version directive. + version_directive *yaml_version_directive_t + + // The list of tag directives. + tag_directives_data []yaml_tag_directive_t + tag_directives_start int // The beginning of the tag directives list. + tag_directives_end int // The end of the tag directives list. + + start_implicit int // Is the document start indicator implicit? + end_implicit int // Is the document end indicator implicit? + + // The start/end of the document. + start_mark, end_mark yaml_mark_t +} + +// The prototype of a read handler. +// +// The read handler is called when the parser needs to read more bytes from the +// source. The handler should write not more than size bytes to the buffer. +// The number of written bytes should be set to the size_read variable. +// +// [in,out] data A pointer to an application data specified by +// yaml_parser_set_input(). +// [out] buffer The buffer to write the data from the source. +// [in] size The size of the buffer. +// [out] size_read The actual number of bytes read from the source. +// +// On success, the handler should return 1. If the handler failed, +// the returned value should be 0. On EOF, the handler should set the +// size_read to 0 and return 1. +type yaml_read_handler_t func(parser *yaml_parser_t, buffer []byte) (n int, err error) + +// This structure holds information about a potential simple key. +type yaml_simple_key_t struct { + possible bool // Is a simple key possible? + required bool // Is a simple key required? + token_number int // The number of the token. + mark yaml_mark_t // The position mark. +} + +// The states of the parser. +type yaml_parser_state_t int + +const ( + yaml_PARSE_STREAM_START_STATE yaml_parser_state_t = iota + + yaml_PARSE_IMPLICIT_DOCUMENT_START_STATE // Expect the beginning of an implicit document. + yaml_PARSE_DOCUMENT_START_STATE // Expect DOCUMENT-START. + yaml_PARSE_DOCUMENT_CONTENT_STATE // Expect the content of a document. + yaml_PARSE_DOCUMENT_END_STATE // Expect DOCUMENT-END. + yaml_PARSE_BLOCK_NODE_STATE // Expect a block node. + yaml_PARSE_BLOCK_NODE_OR_INDENTLESS_SEQUENCE_STATE // Expect a block node or indentless sequence. + yaml_PARSE_FLOW_NODE_STATE // Expect a flow node. + yaml_PARSE_BLOCK_SEQUENCE_FIRST_ENTRY_STATE // Expect the first entry of a block sequence. + yaml_PARSE_BLOCK_SEQUENCE_ENTRY_STATE // Expect an entry of a block sequence. + yaml_PARSE_INDENTLESS_SEQUENCE_ENTRY_STATE // Expect an entry of an indentless sequence. + yaml_PARSE_BLOCK_MAPPING_FIRST_KEY_STATE // Expect the first key of a block mapping. + yaml_PARSE_BLOCK_MAPPING_KEY_STATE // Expect a block mapping key. + yaml_PARSE_BLOCK_MAPPING_VALUE_STATE // Expect a block mapping value. + yaml_PARSE_FLOW_SEQUENCE_FIRST_ENTRY_STATE // Expect the first entry of a flow sequence. + yaml_PARSE_FLOW_SEQUENCE_ENTRY_STATE // Expect an entry of a flow sequence. + yaml_PARSE_FLOW_SEQUENCE_ENTRY_MAPPING_KEY_STATE // Expect a key of an ordered mapping. + yaml_PARSE_FLOW_SEQUENCE_ENTRY_MAPPING_VALUE_STATE // Expect a value of an ordered mapping. + yaml_PARSE_FLOW_SEQUENCE_ENTRY_MAPPING_END_STATE // Expect the and of an ordered mapping entry. + yaml_PARSE_FLOW_MAPPING_FIRST_KEY_STATE // Expect the first key of a flow mapping. + yaml_PARSE_FLOW_MAPPING_KEY_STATE // Expect a key of a flow mapping. + yaml_PARSE_FLOW_MAPPING_VALUE_STATE // Expect a value of a flow mapping. + yaml_PARSE_FLOW_MAPPING_EMPTY_VALUE_STATE // Expect an empty value of a flow mapping. + yaml_PARSE_END_STATE // Expect nothing. +) + +func (ps yaml_parser_state_t) String() string { + switch ps { + case yaml_PARSE_STREAM_START_STATE: + return "yaml_PARSE_STREAM_START_STATE" + case yaml_PARSE_IMPLICIT_DOCUMENT_START_STATE: + return "yaml_PARSE_IMPLICIT_DOCUMENT_START_STATE" + case yaml_PARSE_DOCUMENT_START_STATE: + return "yaml_PARSE_DOCUMENT_START_STATE" + case yaml_PARSE_DOCUMENT_CONTENT_STATE: + return "yaml_PARSE_DOCUMENT_CONTENT_STATE" + case yaml_PARSE_DOCUMENT_END_STATE: + return "yaml_PARSE_DOCUMENT_END_STATE" + case yaml_PARSE_BLOCK_NODE_STATE: + return "yaml_PARSE_BLOCK_NODE_STATE" + case yaml_PARSE_BLOCK_NODE_OR_INDENTLESS_SEQUENCE_STATE: + return "yaml_PARSE_BLOCK_NODE_OR_INDENTLESS_SEQUENCE_STATE" + case yaml_PARSE_FLOW_NODE_STATE: + return "yaml_PARSE_FLOW_NODE_STATE" + case yaml_PARSE_BLOCK_SEQUENCE_FIRST_ENTRY_STATE: + return "yaml_PARSE_BLOCK_SEQUENCE_FIRST_ENTRY_STATE" + case yaml_PARSE_BLOCK_SEQUENCE_ENTRY_STATE: + return "yaml_PARSE_BLOCK_SEQUENCE_ENTRY_STATE" + case yaml_PARSE_INDENTLESS_SEQUENCE_ENTRY_STATE: + return "yaml_PARSE_INDENTLESS_SEQUENCE_ENTRY_STATE" + case yaml_PARSE_BLOCK_MAPPING_FIRST_KEY_STATE: + return "yaml_PARSE_BLOCK_MAPPING_FIRST_KEY_STATE" + case yaml_PARSE_BLOCK_MAPPING_KEY_STATE: + return "yaml_PARSE_BLOCK_MAPPING_KEY_STATE" + case yaml_PARSE_BLOCK_MAPPING_VALUE_STATE: + return "yaml_PARSE_BLOCK_MAPPING_VALUE_STATE" + case yaml_PARSE_FLOW_SEQUENCE_FIRST_ENTRY_STATE: + return "yaml_PARSE_FLOW_SEQUENCE_FIRST_ENTRY_STATE" + case yaml_PARSE_FLOW_SEQUENCE_ENTRY_STATE: + return "yaml_PARSE_FLOW_SEQUENCE_ENTRY_STATE" + case yaml_PARSE_FLOW_SEQUENCE_ENTRY_MAPPING_KEY_STATE: + return "yaml_PARSE_FLOW_SEQUENCE_ENTRY_MAPPING_KEY_STATE" + case yaml_PARSE_FLOW_SEQUENCE_ENTRY_MAPPING_VALUE_STATE: + return "yaml_PARSE_FLOW_SEQUENCE_ENTRY_MAPPING_VALUE_STATE" + case yaml_PARSE_FLOW_SEQUENCE_ENTRY_MAPPING_END_STATE: + return "yaml_PARSE_FLOW_SEQUENCE_ENTRY_MAPPING_END_STATE" + case yaml_PARSE_FLOW_MAPPING_FIRST_KEY_STATE: + return "yaml_PARSE_FLOW_MAPPING_FIRST_KEY_STATE" + case yaml_PARSE_FLOW_MAPPING_KEY_STATE: + return "yaml_PARSE_FLOW_MAPPING_KEY_STATE" + case yaml_PARSE_FLOW_MAPPING_VALUE_STATE: + return "yaml_PARSE_FLOW_MAPPING_VALUE_STATE" + case yaml_PARSE_FLOW_MAPPING_EMPTY_VALUE_STATE: + return "yaml_PARSE_FLOW_MAPPING_EMPTY_VALUE_STATE" + case yaml_PARSE_END_STATE: + return "yaml_PARSE_END_STATE" + } + return "" +} + +// This structure holds aliases data. +type yaml_alias_data_t struct { + anchor []byte // The anchor. + index int // The node id. + mark yaml_mark_t // The anchor mark. +} + +// The parser structure. +// +// All members are internal. Manage the structure using the +// yaml_parser_ family of functions. +type yaml_parser_t struct { + + // Error handling + + error yaml_error_type_t // Error type. + + problem string // Error description. + + // The byte about which the problem occurred. + problem_offset int + problem_value int + problem_mark yaml_mark_t + + // The error context. + context string + context_mark yaml_mark_t + + // Reader stuff + + read_handler yaml_read_handler_t // Read handler. + + input_reader io.Reader // File input data. + input []byte // String input data. + input_pos int + + eof bool // EOF flag + + buffer []byte // The working buffer. + buffer_pos int // The current position of the buffer. + + unread int // The number of unread characters in the buffer. + + raw_buffer []byte // The raw buffer. + raw_buffer_pos int // The current position of the buffer. + + encoding yaml_encoding_t // The input encoding. + + offset int // The offset of the current position (in bytes). + mark yaml_mark_t // The mark of the current position. + + // Scanner stuff + + stream_start_produced bool // Have we started to scan the input stream? + stream_end_produced bool // Have we reached the end of the input stream? + + flow_level int // The number of unclosed '[' and '{' indicators. + + tokens []yaml_token_t // The tokens queue. + tokens_head int // The head of the tokens queue. + tokens_parsed int // The number of tokens fetched from the queue. + token_available bool // Does the tokens queue contain a token ready for dequeueing. + + indent int // The current indentation level. + indents []int // The indentation levels stack. + + simple_key_allowed bool // May a simple key occur at the current position? + simple_keys []yaml_simple_key_t // The stack of simple keys. + + // Parser stuff + + state yaml_parser_state_t // The current parser state. + states []yaml_parser_state_t // The parser states stack. + marks []yaml_mark_t // The stack of marks. + tag_directives []yaml_tag_directive_t // The list of TAG directives. + + // Dumper stuff + + aliases []yaml_alias_data_t // The alias data. + + document *yaml_document_t // The currently parsed document. +} + +// Emitter Definitions + +// The prototype of a write handler. +// +// The write handler is called when the emitter needs to flush the accumulated +// characters to the output. The handler should write @a size bytes of the +// @a buffer to the output. +// +// @param[in,out] data A pointer to an application data specified by +// yaml_emitter_set_output(). +// @param[in] buffer The buffer with bytes to be written. +// @param[in] size The size of the buffer. +// +// @returns On success, the handler should return @c 1. If the handler failed, +// the returned value should be @c 0. +// +type yaml_write_handler_t func(emitter *yaml_emitter_t, buffer []byte) error + +type yaml_emitter_state_t int + +// The emitter states. +const ( + // Expect STREAM-START. + yaml_EMIT_STREAM_START_STATE yaml_emitter_state_t = iota + + yaml_EMIT_FIRST_DOCUMENT_START_STATE // Expect the first DOCUMENT-START or STREAM-END. + yaml_EMIT_DOCUMENT_START_STATE // Expect DOCUMENT-START or STREAM-END. + yaml_EMIT_DOCUMENT_CONTENT_STATE // Expect the content of a document. + yaml_EMIT_DOCUMENT_END_STATE // Expect DOCUMENT-END. + yaml_EMIT_FLOW_SEQUENCE_FIRST_ITEM_STATE // Expect the first item of a flow sequence. + yaml_EMIT_FLOW_SEQUENCE_ITEM_STATE // Expect an item of a flow sequence. + yaml_EMIT_FLOW_MAPPING_FIRST_KEY_STATE // Expect the first key of a flow mapping. + yaml_EMIT_FLOW_MAPPING_KEY_STATE // Expect a key of a flow mapping. + yaml_EMIT_FLOW_MAPPING_SIMPLE_VALUE_STATE // Expect a value for a simple key of a flow mapping. + yaml_EMIT_FLOW_MAPPING_VALUE_STATE // Expect a value of a flow mapping. + yaml_EMIT_BLOCK_SEQUENCE_FIRST_ITEM_STATE // Expect the first item of a block sequence. + yaml_EMIT_BLOCK_SEQUENCE_ITEM_STATE // Expect an item of a block sequence. + yaml_EMIT_BLOCK_MAPPING_FIRST_KEY_STATE // Expect the first key of a block mapping. + yaml_EMIT_BLOCK_MAPPING_KEY_STATE // Expect the key of a block mapping. + yaml_EMIT_BLOCK_MAPPING_SIMPLE_VALUE_STATE // Expect a value for a simple key of a block mapping. + yaml_EMIT_BLOCK_MAPPING_VALUE_STATE // Expect a value of a block mapping. + yaml_EMIT_END_STATE // Expect nothing. +) + +// The emitter structure. +// +// All members are internal. Manage the structure using the @c yaml_emitter_ +// family of functions. +type yaml_emitter_t struct { + + // Error handling + + error yaml_error_type_t // Error type. + problem string // Error description. + + // Writer stuff + + write_handler yaml_write_handler_t // Write handler. + + output_buffer *[]byte // String output data. + output_writer io.Writer // File output data. + + buffer []byte // The working buffer. + buffer_pos int // The current position of the buffer. + + raw_buffer []byte // The raw buffer. + raw_buffer_pos int // The current position of the buffer. + + encoding yaml_encoding_t // The stream encoding. + + // Emitter stuff + + canonical bool // If the output is in the canonical style? + best_indent int // The number of indentation spaces. + best_width int // The preferred width of the output lines. + unicode bool // Allow unescaped non-ASCII characters? + line_break yaml_break_t // The preferred line break. + + state yaml_emitter_state_t // The current emitter state. + states []yaml_emitter_state_t // The stack of states. + + events []yaml_event_t // The event queue. + events_head int // The head of the event queue. + + indents []int // The stack of indentation levels. + + tag_directives []yaml_tag_directive_t // The list of tag directives. + + indent int // The current indentation level. + + flow_level int // The current flow level. + + root_context bool // Is it the document root context? + sequence_context bool // Is it a sequence context? + mapping_context bool // Is it a mapping context? + simple_key_context bool // Is it a simple mapping key context? + + line int // The current line. + column int // The current column. + whitespace bool // If the last character was a whitespace? + indention bool // If the last character was an indentation character (' ', '-', '?', ':')? + open_ended bool // If an explicit document end is required? + + // Anchor analysis. + anchor_data struct { + anchor []byte // The anchor value. + alias bool // Is it an alias? + } + + // Tag analysis. + tag_data struct { + handle []byte // The tag handle. + suffix []byte // The tag suffix. + } + + // Scalar analysis. + scalar_data struct { + value []byte // The scalar value. + multiline bool // Does the scalar contain line breaks? + flow_plain_allowed bool // Can the scalar be expessed in the flow plain style? + block_plain_allowed bool // Can the scalar be expressed in the block plain style? + single_quoted_allowed bool // Can the scalar be expressed in the single quoted style? + block_allowed bool // Can the scalar be expressed in the literal or folded styles? + style yaml_scalar_style_t // The output style. + } + + // Dumper stuff + + opened bool // If the stream was already opened? + closed bool // If the stream was already closed? + + // The information associated with the document nodes. + anchors *struct { + references int // The number of references. + anchor int // The anchor id. + serialized bool // If the node has been emitted? + } + + last_anchor_id int // The last assigned anchor id. + + document *yaml_document_t // The currently emitted document. +} diff --git a/tools/voluspa/vendor/gopkg.in/yaml.v2/yamlprivateh.go b/tools/voluspa/vendor/gopkg.in/yaml.v2/yamlprivateh.go new file mode 100644 index 00000000000..8110ce3c37a --- /dev/null +++ b/tools/voluspa/vendor/gopkg.in/yaml.v2/yamlprivateh.go @@ -0,0 +1,173 @@ +package yaml + +const ( + // The size of the input raw buffer. + input_raw_buffer_size = 512 + + // The size of the input buffer. + // It should be possible to decode the whole raw buffer. + input_buffer_size = input_raw_buffer_size * 3 + + // The size of the output buffer. + output_buffer_size = 128 + + // The size of the output raw buffer. + // It should be possible to encode the whole output buffer. + output_raw_buffer_size = (output_buffer_size*2 + 2) + + // The size of other stacks and queues. + initial_stack_size = 16 + initial_queue_size = 16 + initial_string_size = 16 +) + +// Check if the character at the specified position is an alphabetical +// character, a digit, '_', or '-'. +func is_alpha(b []byte, i int) bool { + return b[i] >= '0' && b[i] <= '9' || b[i] >= 'A' && b[i] <= 'Z' || b[i] >= 'a' && b[i] <= 'z' || b[i] == '_' || b[i] == '-' +} + +// Check if the character at the specified position is a digit. +func is_digit(b []byte, i int) bool { + return b[i] >= '0' && b[i] <= '9' +} + +// Get the value of a digit. +func as_digit(b []byte, i int) int { + return int(b[i]) - '0' +} + +// Check if the character at the specified position is a hex-digit. +func is_hex(b []byte, i int) bool { + return b[i] >= '0' && b[i] <= '9' || b[i] >= 'A' && b[i] <= 'F' || b[i] >= 'a' && b[i] <= 'f' +} + +// Get the value of a hex-digit. +func as_hex(b []byte, i int) int { + bi := b[i] + if bi >= 'A' && bi <= 'F' { + return int(bi) - 'A' + 10 + } + if bi >= 'a' && bi <= 'f' { + return int(bi) - 'a' + 10 + } + return int(bi) - '0' +} + +// Check if the character is ASCII. +func is_ascii(b []byte, i int) bool { + return b[i] <= 0x7F +} + +// Check if the character at the start of the buffer can be printed unescaped. +func is_printable(b []byte, i int) bool { + return ((b[i] == 0x0A) || // . == #x0A + (b[i] >= 0x20 && b[i] <= 0x7E) || // #x20 <= . <= #x7E + (b[i] == 0xC2 && b[i+1] >= 0xA0) || // #0xA0 <= . <= #xD7FF + (b[i] > 0xC2 && b[i] < 0xED) || + (b[i] == 0xED && b[i+1] < 0xA0) || + (b[i] == 0xEE) || + (b[i] == 0xEF && // #xE000 <= . <= #xFFFD + !(b[i+1] == 0xBB && b[i+2] == 0xBF) && // && . != #xFEFF + !(b[i+1] == 0xBF && (b[i+2] == 0xBE || b[i+2] == 0xBF)))) +} + +// Check if the character at the specified position is NUL. +func is_z(b []byte, i int) bool { + return b[i] == 0x00 +} + +// Check if the beginning of the buffer is a BOM. +func is_bom(b []byte, i int) bool { + return b[0] == 0xEF && b[1] == 0xBB && b[2] == 0xBF +} + +// Check if the character at the specified position is space. +func is_space(b []byte, i int) bool { + return b[i] == ' ' +} + +// Check if the character at the specified position is tab. +func is_tab(b []byte, i int) bool { + return b[i] == '\t' +} + +// Check if the character at the specified position is blank (space or tab). +func is_blank(b []byte, i int) bool { + //return is_space(b, i) || is_tab(b, i) + return b[i] == ' ' || b[i] == '\t' +} + +// Check if the character at the specified position is a line break. +func is_break(b []byte, i int) bool { + return (b[i] == '\r' || // CR (#xD) + b[i] == '\n' || // LF (#xA) + b[i] == 0xC2 && b[i+1] == 0x85 || // NEL (#x85) + b[i] == 0xE2 && b[i+1] == 0x80 && b[i+2] == 0xA8 || // LS (#x2028) + b[i] == 0xE2 && b[i+1] == 0x80 && b[i+2] == 0xA9) // PS (#x2029) +} + +func is_crlf(b []byte, i int) bool { + return b[i] == '\r' && b[i+1] == '\n' +} + +// Check if the character is a line break or NUL. +func is_breakz(b []byte, i int) bool { + //return is_break(b, i) || is_z(b, i) + return ( // is_break: + b[i] == '\r' || // CR (#xD) + b[i] == '\n' || // LF (#xA) + b[i] == 0xC2 && b[i+1] == 0x85 || // NEL (#x85) + b[i] == 0xE2 && b[i+1] == 0x80 && b[i+2] == 0xA8 || // LS (#x2028) + b[i] == 0xE2 && b[i+1] == 0x80 && b[i+2] == 0xA9 || // PS (#x2029) + // is_z: + b[i] == 0) +} + +// Check if the character is a line break, space, or NUL. +func is_spacez(b []byte, i int) bool { + //return is_space(b, i) || is_breakz(b, i) + return ( // is_space: + b[i] == ' ' || + // is_breakz: + b[i] == '\r' || // CR (#xD) + b[i] == '\n' || // LF (#xA) + b[i] == 0xC2 && b[i+1] == 0x85 || // NEL (#x85) + b[i] == 0xE2 && b[i+1] == 0x80 && b[i+2] == 0xA8 || // LS (#x2028) + b[i] == 0xE2 && b[i+1] == 0x80 && b[i+2] == 0xA9 || // PS (#x2029) + b[i] == 0) +} + +// Check if the character is a line break, space, tab, or NUL. +func is_blankz(b []byte, i int) bool { + //return is_blank(b, i) || is_breakz(b, i) + return ( // is_blank: + b[i] == ' ' || b[i] == '\t' || + // is_breakz: + b[i] == '\r' || // CR (#xD) + b[i] == '\n' || // LF (#xA) + b[i] == 0xC2 && b[i+1] == 0x85 || // NEL (#x85) + b[i] == 0xE2 && b[i+1] == 0x80 && b[i+2] == 0xA8 || // LS (#x2028) + b[i] == 0xE2 && b[i+1] == 0x80 && b[i+2] == 0xA9 || // PS (#x2029) + b[i] == 0) +} + +// Determine the width of the character. +func width(b byte) int { + // Don't replace these by a switch without first + // confirming that it is being inlined. + if b&0x80 == 0x00 { + return 1 + } + if b&0xE0 == 0xC0 { + return 2 + } + if b&0xF0 == 0xE0 { + return 3 + } + if b&0xF8 == 0xF0 { + return 4 + } + return 0 + +} diff --git a/tools/voluspa/version.go b/tools/voluspa/version.go new file mode 100644 index 00000000000..97ea06e872e --- /dev/null +++ b/tools/voluspa/version.go @@ -0,0 +1,28 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package voluspa + +var ( + // Version is the build version + Version = "dev" + + // GitVersion is the Git revision hash + GitVersion = "dev" +) diff --git a/tools/voluspa/version/version.go b/tools/voluspa/version/version.go new file mode 100644 index 00000000000..cc71dc4f65d --- /dev/null +++ b/tools/voluspa/version/version.go @@ -0,0 +1,39 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package version + +import "expvar" + +var ( + // Version is the build version + Version = "dev" + + // GitVersion is the Git revision hash + GitVersion = "dev" +) + +func init() { + if len(GitVersion) > 0 { + Version = Version + "/" + GitVersion + } + + v := expvar.NewString("version") + v.Set(Version) +} diff --git a/tools/voluspa/version/version_test.go b/tools/voluspa/version/version_test.go new file mode 100644 index 00000000000..701d5109d2d --- /dev/null +++ b/tools/voluspa/version/version_test.go @@ -0,0 +1,20 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package version diff --git a/tools/voluspa/voluspa.go b/tools/voluspa/voluspa.go new file mode 100644 index 00000000000..2de67d2118f --- /dev/null +++ b/tools/voluspa/voluspa.go @@ -0,0 +1,330 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package voluspa + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "io/ioutil" + "sort" + "strconv" + "strings" + + "github.com/xeipuuv/gojsonschema" + yaml "gopkg.in/yaml.v2" +) + +var ( + ErrMinimumConfigsNotMet = errors.New("Need at least 1 parsed config file") +) + +type Voluspa struct { + parsedConfigs []*CustomerConfig + + installLocation string + options *Options + defaultSchemaVersion int +} + +func NewVoluspa() (*Voluspa, error) { + return NewVoluspaWithOptions(&Options{}, DefaultTrafficserverConfigurationDir) +} + +func NewVoluspaWithOptions(options *Options, installLocation string) (*Voluspa, error) { + return &Voluspa{ + installLocation: installLocation, + options: options, + defaultSchemaVersion: 1, + }, nil +} + +func (v *Voluspa) SchemaDefaultVersion() int { + return v.defaultSchemaVersion +} + +func (v *Voluspa) SchemaDefinition(version int) ([]byte, error) { + if version == 0 { + version = v.SchemaDefaultVersion() + } + + var filename string + if len(v.options.SchemaLocation) > 0 { + filename = fmt.Sprintf("%s/schema_v%d.json", v.options.SchemaLocation, version) + } else { + filename = fmt.Sprintf("schema_v%d.json", version) + } + + return ioutil.ReadFile(filename) +} + +func (v *Voluspa) validateSchema(contents []byte) error { + var ycfg interface{} + if err := yaml.Unmarshal(contents, &ycfg); err != nil { + return err + } + + ycfg = convert(ycfg) + + jcfg, err := json.Marshal(ycfg) + if err != nil { + return err + } + + schemaVersion := v.SchemaDefaultVersion() // TODO extract schema_version from ycfg + schemaContent, err := v.SchemaDefinition(schemaVersion) + if err != nil { + return err + } + + schema, err := gojsonschema.NewSchema(gojsonschema.NewBytesLoader(schemaContent)) + if err != nil { + return fmt.Errorf("error loading the version %d schema definition: %v", schemaVersion, err) + } + + result, err := schema.Validate(gojsonschema.NewBytesLoader(jcfg)) + if err != nil { + return err + } + + if result.Valid() { + return nil + } + + var buf bytes.Buffer + for _, desc := range result.Errors() { + buf.WriteString(fmt.Sprintf(" - %s\n", desc)) + } + + return fmt.Errorf("\n%s", buf.String()) +} + +func convert(i interface{}) interface{} { + switch x := i.(type) { + case map[interface{}]interface{}: + m2 := map[string]interface{}{} + for k, v := range x { + var k2 string + switch k := k.(type) { + case int: + k2 = strconv.Itoa(k) + default: + k2 = k.(string) + } + m2[k2] = convert(v) + } + return m2 + case []interface{}: + for i, v := range x { + x[i] = convert(v) + } + } + return i +} + +// AddConfigByBytes takes a []byte buffer containing a Voluspa YAML config file and a filename and processes it +func (v *Voluspa) AddConfigByBytes(buffer []byte, filename string) error { + if !v.options.SkipSchemaValidation { + if err := v.validateSchema(buffer); err != nil { + return wrapError(filename, err) + } + } + + cfg, err := NewPropertyConfigWithDefaults(buffer, v.options.DefaultsLocation) + if err != nil { + return wrapError(filename, err) + } + + cc, err := NewCustomerConfig(cfg, filename, *v.options) + if err != nil { + return wrapError(filename, err) + } + + v.parsedConfigs = append(v.parsedConfigs, cc) + + return nil +} + +func (v *Voluspa) AddConfig(filename string) error { + buffer, err := ioutil.ReadFile(filename) + if err != nil { + return err + } + return v.AddConfigByBytes(buffer, filename) +} + +func (v *Voluspa) filterConfigsByCDNAndRole(cdns, roles []string) []*CustomerConfig { + if len(cdns) == 0 { + return v.parsedConfigs + } + + cdnMap := make(map[string]interface{}) + for k := range cdns { + cdnMap[cdns[k]] = nil + } + + roleMap := make(map[string]interface{}) + for k := range roles { + roleMap[roles[k]] = nil + } + + var grouped []*CustomerConfig + for _, config := range v.parsedConfigs { + if config.lifecycle == Retired { + continue + } + + if !v.options.PromoteRolesToCDN && isPromotableRole(config.role) { + continue + } + + includeCDN := false + for cdn := range config.cdn { + if _, exists := cdnMap[cdn]; exists { + includeCDN = true + break + } + } + + var includeRole bool + if len(config.role) > 0 { + _, includeRole = roleMap[config.role] + } else { + includeRole = true + } + + if includeCDN && includeRole { + grouped = append(grouped, config) + } + } + + sort.Stable(sort.Reverse(byQPSAndName(grouped))) + + return grouped +} + +// Validate will run all validation methods and return an error if any issues were found +// strict will enable extra checks that are known to fail with reasonable sets of configuration files +func (v *Voluspa) Validate(strict bool) error { + var errs Errors + errs = append(errs, v.ensurePropertyNameUniqueness()...) + errs = append(errs, v.ensureAliasUniqueness()...) + errs = append(errs, v.validateRegexOptions()...) + errs = append(errs, v.ensureValidSchemes()...) + + if strict { + errs = append(errs, v.ensureHostsResolve()...) + } + + if len(errs) > 0 { + return errs + } + + return nil +} + +// WriteAllFiles will write out all files, filter by passed in cdns and roles +func (v *Voluspa) WriteAllFiles(cdns, roles []string) error { + allFiles, err := v.GetManagedFilesByCDNAndRole(cdns, roles) + if err != nil { + return err + } + + writer, err := NewFilesystemWriter(v.options) + if err != nil { + return err + } + + return writer.WriteFiles(allFiles) +} + +// PropertyCount returns the number of properties +func (v *Voluspa) PropertyCount() int { + return len(v.parsedConfigs) +} + +type configGenerator interface { + // Do processes parsedConfigs, optionally merging, returning a slice of ManagedFiles + Do(parsedConfigs []*CustomerConfig, merge bool) ([]ManagedFile, error) +} + +func getGenerators(options *Options) []configGenerator { + var cg []configGenerator + cg = append(cg, newPropertyRemapGenerator(options)) + cg = append(cg, newTopLevelRemapGenerator(options)) + cg = append(cg, newParentConfigurator(options)) + cg = append(cg, newSSLMulticertConfigurator(options)) + cg = append(cg, &HAProxyConfigGenerator{}) + cg = append(cg, newHostingConfigurator(options)) + return cg +} + +func (v *Voluspa) GetManagedFiles(cdns []string) ([]ManagedFile, error) { + return v.GetManagedFilesByCDNAndRole(cdns, nil) +} + +func (v *Voluspa) GetManagedFilesByCDNAndRole(cdns, roles []string) ([]ManagedFile, error) { + cc := v.filterConfigsByCDNAndRole(cdns, roles) + merged := false + if len(cdns) > 0 { + merged = true + } + + var allFiles []ManagedFile + for _, g := range getGenerators(v.options) { + managedFiles, err := g.Do(cc, merged) + if err != nil { + return nil, err + } + allFiles = append(allFiles, managedFiles...) + } + return allFiles, nil +} + +func wrapError(filename string, err error) error { + _, ok := err.(Errors) + if !ok { + return fmt.Errorf("problem loading %q: %s", filename, err) + } + return fmt.Errorf("problem loading %q:\n%s", filename, err) +} + +// byQPSAndName implements sort.Interface, sorting by a CustomerConfig by qps/name. +type byQPSAndName []*CustomerConfig + +func (s byQPSAndName) Len() int { return len(s) } +func (s byQPSAndName) Swap(i, j int) { s[i], s[j] = s[j], s[i] } +func (s byQPSAndName) Less(i, j int) bool { + if s[i].qps < s[j].qps { + return true + } + if s[i].qps > s[j].qps { + return false + } + + // normalize the property names for sorting + return strings.ToLower(s[i].property) > strings.ToLower(s[j].property) +} + +// isPromotableRole returns true of role is promotable +func isPromotableRole(role string) bool { + return strings.HasPrefix(role, "roles_") +}