diff --git a/.clang-format b/.clang-format index a70897438e1..cd564fb109c 100644 --- a/.clang-format +++ b/.clang-format @@ -2,42 +2,48 @@ Language: Cpp AccessModifierOffset: -2 AlignAfterOpenBracket: Align +AlignConsecutiveMacros: false AlignConsecutiveAssignments: true AlignConsecutiveDeclarations: false AlignEscapedNewlines: Left AlignOperands: true AlignTrailingComments: true +AllowAllArgumentsOnNextLine: true +AllowAllConstructorInitializersOnNextLine: true AllowAllParametersOfDeclarationOnNextLine: false -AllowShortBlocksOnASingleLine: false +AllowShortBlocksOnASingleLine: Never AllowShortCaseLabelsOnASingleLine: false AllowShortFunctionsOnASingleLine: Inline -AllowShortIfStatementsOnASingleLine: false +AllowShortLambdasOnASingleLine: All +AllowShortIfStatementsOnASingleLine: Never AllowShortLoopsOnASingleLine: false AlwaysBreakAfterDefinitionReturnType: All AlwaysBreakAfterReturnType: AllDefinitions AlwaysBreakBeforeMultilineStrings: false -AlwaysBreakTemplateDeclarations: false +AlwaysBreakTemplateDeclarations: MultiLine BinPackArguments: true BinPackParameters: true BraceWrapping: + AfterCaseLabel: false AfterClass: true - AfterControlStatement: true - AfterEnum: true + AfterControlStatement: false + AfterEnum: false AfterFunction: true AfterNamespace: true AfterObjCDeclaration: false - AfterStruct: true - AfterUnion: true - AfterExternBlock: true + AfterStruct: false + AfterUnion: false + AfterExternBlock: false BeforeCatch: false BeforeElse: false IndentBraces: false - SplitEmptyFunction: false - SplitEmptyRecord: false - SplitEmptyNamespace: false + SplitEmptyFunction: true + SplitEmptyRecord: true + SplitEmptyNamespace: true BreakBeforeBinaryOperators: None BreakBeforeBraces: Linux BreakBeforeInheritanceComma: false +BreakInheritanceList: BeforeColon BreakBeforeTernaryOperators: false BreakConstructorInitializersBeforeComma: false BreakConstructorInitializers: BeforeColon @@ -50,6 +56,7 @@ ConstructorInitializerAllOnOneLineOrOnePerLine: true ConstructorInitializerIndentWidth: 2 ContinuationIndentWidth: 2 Cpp11BracedListStyle: true +DeriveLineEnding: true DerivePointerAlignment: false DisableFormat: false ExperimentalAutoDetectBinPacking: false @@ -65,12 +72,17 @@ IncludeBlocks: Preserve IncludeCategories: - Regex: '^"(llvm|llvm-c|clang|clang-c)/' Priority: 2 + SortPriority: 0 - Regex: '^(<|"(gtest|isl|json)/)' Priority: 3 + SortPriority: 0 - Regex: '.*' Priority: 1 + SortPriority: 0 IncludeIsMainRegex: '$' +IncludeIsMainSourceRegex: '' IndentCaseLabels: false +IndentGotoLabels: true IndentPPDirectives: None IndentWidth: 2 IndentWrappedFunctionNames: false @@ -81,6 +93,7 @@ MacroBlockBegin: '' MacroBlockEnd: '' MaxEmptyLinesToKeep: 1 NamespaceIndentation: Inner +ObjCBinPackProtocolList: Auto ObjCBlockIndentWidth: 2 ObjCSpaceAfterProperty: true ObjCSpaceBeforeProtocolList: false @@ -89,29 +102,38 @@ PenaltyBreakBeforeFirstCallParameter: 19 PenaltyBreakComment: 30000 PenaltyBreakFirstLessLess: 120 PenaltyBreakString: 1000 +PenaltyBreakTemplateDeclaration: 10 PenaltyExcessCharacter: 1000000 PenaltyReturnTypeOnItsOwnLine: 200 PointerAlignment: Right -RawStringFormats: - - Delimiter: pb - Language: TextProto - BasedOnStyle: Mozilla ReflowComments: true SortIncludes: false SortUsingDeclarations: false SpaceAfterCStyleCast: false +SpaceAfterLogicalNot: false SpaceAfterTemplateKeyword: true SpaceBeforeAssignmentOperators: true +SpaceBeforeCpp11BracedList: false +SpaceBeforeCtorInitializerColon: true +SpaceBeforeInheritanceColon: true SpaceBeforeParens: ControlStatements +SpaceBeforeRangeBasedForLoopColon: true +SpaceInEmptyBlock: false SpaceInEmptyParentheses: false SpacesBeforeTrailingComments: 1 SpacesInAngles: false +SpacesInConditionalStatement: false SpacesInContainerLiterals: true SpacesInCStyleCastParentheses: false SpacesInParentheses: false SpacesInSquareBrackets: false -Standard: Cpp11 +SpaceBeforeSquareBrackets: false +Standard: Latest +StatementMacros: + - Q_UNUSED + - QT_REQUIRE_VERSION TabWidth: 8 +UseCRLF: false UseTab: Never ... diff --git a/.gitignore b/.gitignore index c683ae3728b..493a99cca11 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,6 @@ *.o +*.c_o +*.cc_o *.a *.so *.pyc @@ -41,7 +43,6 @@ Makefile-pl config.cache config.status config.nice -config.h configs/records.config.default configs/storage.config.default @@ -93,32 +94,10 @@ lib/perl/lib/Apache/TS.pm iocore/net/test_certlookup iocore/net/test_UDPNet -iocore/net/quic/test_QUICAckFrameCreator -iocore/net/quic/test_QUICAddrVerifyState -iocore/net/quic/test_QUICAltConnectionManager -iocore/net/quic/test_QUICFlowController -iocore/net/quic/test_QUICFrame -iocore/net/quic/test_QUICFrameDispatcher -iocore/net/quic/test_QUICFrameRetransmitter -iocore/net/quic/test_QUICHandshake -iocore/net/quic/test_QUICHandshakeProtocol -iocore/net/quic/test_QUICIncomingFrameBuffer -iocore/net/quic/test_QUICInvariants -iocore/net/quic/test_QUICKeyGenerator -iocore/net/quic/test_QUICLossDetector -iocore/net/quic/test_QUICPacket -iocore/net/quic/test_QUICPacketHeaderProtector -iocore/net/quic/test_QUICPacketFactory -iocore/net/quic/test_QUICStream -iocore/net/quic/test_QUICStreamManager -iocore/net/quic/test_QUICStreamState -iocore/net/quic/test_QUICTransportParameters -iocore/net/quic/test_QUICType -iocore/net/quic/test_QUICTypeUtil -iocore/net/quic/test_QUICVersionNegotiator +iocore/net/quic/test_QUIC* iocore/aio/test_AIO -iocore/eventsystem/test_Buffer -iocore/eventsystem/test_Event +iocore/eventsystem/test_IOBuffer +iocore/eventsystem/test_EventSystem iocore/eventsystem/test_MIOBufferWriter iocore/hostdb/test_RefCountCache @@ -128,7 +107,14 @@ proxy/hdrs/test_proxy_hdrs proxy/hdrs/test_hdr_heap proxy/hdrs/test_Huffmancode proxy/hdrs/test_XPACK +proxy/http/remap/test_NextHopRoundRobin +proxy/http/remap/test_NextHopStrategyFactory +proxy/http/remap/test_PluginDso +proxy/http/remap/test_PluginFactory +proxy/http/remap/test_RemapPluginInfo proxy/http/test_proxy_http +proxy/http/remap/test_* +proxy/http2/test_libhttp2 proxy/http2/test_Http2DependencyTree proxy/http2/test_Http2FrequencyCounter proxy/http2/test_HPACK @@ -137,6 +123,7 @@ proxy/http3/test_libhttp3 proxy/http3/test_qpack proxy/logging/test_LogUtils proxy/logging/test_LogUtils2 +proxy/logging/test_RolledLogDeleter plugins/header_rewrite/header_rewrite_test plugins/experimental/cookie_remap/test_cookiejar @@ -144,6 +131,7 @@ plugins/experimental/esi/*_test plugins/experimental/slice/test_* plugins/experimental/sslheaders/test_sslheaders plugins/s3_auth/test_s3auth +plugins/experimental/traffic_dump/test_* plugins/esi/docnode_test plugins/esi/gzip_test @@ -182,6 +170,7 @@ rc/trafficserver.service .svn/ .vscode/ +target tsxs @@ -205,6 +194,8 @@ RELEASE # autest tests/env-test/ tests/Pipfile.lock +tests/gold_tests/chunked_encoding/smuggle-client +tests/gold_tests/tls/ssl-post iocore/cache/test_* iocore/cache/test/var/trafficserver/cache.db diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 00000000000..32b63427155 --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,43 @@ +{ + "version": "0.2.0", + "configurations": [ + { + "name": "(gdb) traffic_server", + "type": "cppdbg", + "request": "launch", + "program": "${workspaceFolder}/src/traffic_server/.libs/traffic_server", + "args": [], + "stopAtEntry": false, + "cwd": "${workspaceFolder}", + "environment": [], + "externalConsole": false, + "MIMode": "gdb", + "setupCommands": [ + { + "description": "Enable pretty-printing for gdb", + "text": "-enable-pretty-printing", + "ignoreFailures": true + } + ] + }, + { + "name": "(gdb) traffic_manager", + "type": "cppdbg", + "request": "launch", + "program": "${workspaceFolder}/src/traffic_manager/.libs/traffic_manager", + "args": [], + "stopAtEntry": false, + "cwd": "${workspaceFolder}", + "environment": [], + "externalConsole": false, + "MIMode": "gdb", + "setupCommands": [ + { + "description": "Enable pretty-printing for gdb", + "text": "-enable-pretty-printing", + "ignoreFailures": true + } + ] + } + ] +} diff --git a/.vscode/tasks.json b/.vscode/tasks.json new file mode 100644 index 00000000000..20264fce5e1 --- /dev/null +++ b/.vscode/tasks.json @@ -0,0 +1,64 @@ +{ + "version": "2.0.0", + "tasks": [ + { + "label": "Autoreconf", + "type": "shell", + "command": "autoreconf -if", + "problemMatcher": [] + }, + { + "label": "Configure", + "type": "shell", + "command": "./configure --prefix=${workspaceFolder}/target --enable-ccache --enable-experimental-plugins --enable-example-plugins --enable-test-tools --enable-debug --enable-werror ${env:ATS_VSCODE_CONFIGURE}", + "dependsOrder": "sequence", + "dependsOn": ["Autoreconf"], + "problemMatcher": [] + }, + { + "label": "Build", + "type": "shell", + "command": "make -j 16", + "problemMatcher": [ + "$gcc" + ], + "group": { + "kind": "build", + "isDefault": true + } + }, + { + "label": "Install", + "type": "shell", + "command": "make -j 16 install", + "problemMatcher": [ + "$gcc" + ] + }, + { + "label": "Test", + "type": "shell", + "command": "make -j 8 test", + "problemMatcher": [ + "$gcc" + ], + "group": { + "kind": "test", + "isDefault": true + } + }, + { + "label": "Full Build", + "dependsOrder": "sequence", + "dependsOn": ["Configure", "Build", "Install"], + "problemMatcher": [ + "$gcc" + ] + } + { + "label": "Dump Enviroment Variables", + "command": "env", + "problemMatcher": [] + } + ] +} diff --git a/CHANGELOG-9.0.0 b/CHANGELOG-9.0.0 new file mode 100644 index 00000000000..5fcb12bc7f0 --- /dev/null +++ b/CHANGELOG-9.0.0 @@ -0,0 +1,1103 @@ +Changes with Apache Traffic Server 9.0.0 + #3676 - IOBufferChain: initial version + #3693 - register stats in traffic_server if running in standalone mode + #3724 - Unify include paths for headers between core, internal plugins, and external plugins. + #3768 - Autest: Test revalidating cached objects + #3777 - Bump version on master to v9.0.0 + #3819 - Upstream max conn upgrade + #3851 - For atscppapi::AsyncTimer, pass the thread pool for execution to the constructor, … + #3900 - ts/Extendible and AcidPtr classes + #3901 - MemArena: Add make method to construct objects in the arena. + #3907 - IntrusiveDList: Refreshed for C++ eleventy, added const_iterator. + #3921 - Bug fix in microServer's threading and reorganized some code + #3925 - Enforce sphinx>=1.7.5 when building docs + #3930 - Remove proxy.config.config_dir from records.config + #3941 - Fix RecConfigReadPluginDir and clean up RecCore + #3951 - Fixed broken sphinx version check on MacOS + #3955 - IntrusiveHashMap: Refresh TSHashTable for C++ eleventy. + #3957 - Add generic "guard" class (PostScript) for exception and early function return safety. + #3958 - PROXY Protocol transformed to Forwarded HTTP header + #3963 - TextView: More unit tests about separators and tokens. + #3964 - Fixes spelling + #3965 - Testing: Convert test_History.cc to Catch + #3966 - Add BWF support for SourceLocation. + #3967 - BufferWriter: Add print overloads to FixedBufferWriter + #3968 - Test: Convert test_Regex.cc to Catch. + #3992 - TextView: More unit tests. + #3993 - Test: Convert test_Ptr.cc to Catch. + #4003 - Fix another crash on shutdown + #4008 - Add support for 'fwd' value to X-Debug header, and move to later hook any deletion of X-Debug header from client request. + #4020 - Add logic to clean up vios on HttpSM shutdown + #4029 - Replace TSHashTable in server session management with IntrusiveHashMap + #4033 - Removes remnants of dprintf support + #4039 - Enable ECDH explicitly only if OpenSSL version is v1.0.2 or lower + #4044 - Adjust tests to work with more versions of curl. + #4046 - Collapses LogAccess and LogAccessHttp into LogAccess + #4047 - Improve log entry for http connect result errors + #4049 - Removes carriage returns from MT_hashtable.h + #4050 - Optimize: make NetAccept::init_accept_loop has the logic similar to NetAccept::init_accept_per_thread + #4063 - BWF: Add "FirstOf" for better handling of printing alternates for null strings. + #4064 - Runroot: Update test to support different layout + #4067 - Bug fix for timeout test + #4075 - General code cleanup + #4077 - Skip timeout test + #4089 - reenable timeout and change microServer timeout logic + #4095 - Optimize: Assign nh->mutex to new NetVC first and switch to new mutex within UnixNetVC::acceptEvent + #4098 - adding a Mutex to the continuation + #4107 - Fixes MacOS linker issue with release build + #4113 - Disables openclose_h2 test. + #4115 - TextView: Add overload for strcasecmp for string_view & TextView. + #4125 - ssl_session_reuse plugin + #4129 - Update IPAllow infrastructure. + #4133 - Disables the double test for inconsistent execution + #4140 - Meta: Add conditional compilation case meta structures. + #4145 - Adds configurable behavior on redirect to loopback + #4146 - Cleanup: remove unused and unimplemented functions from ink_string.h + #4148 - Cleanup: Remove #define INT_TO_BOOL, use local lambda instead. + #4149 - Clang-format: Redo clang-format error that slipped through. + #4168 - YAML: Convert ip_allow.config + #4175 - TextView: Better support for std::string assignment. + #4176 - TextView: Add support for use as "Source" in STL containers. + #4177 - IntrusiveDList: Add ptr_ref_cast to make inheritance easier + #4184 - Update to changelog generation tool to not require milestone to be closed + #4185 - Amend Layer 4 routing documentation + #4186 - clang-format: Another one that slipped through. + #4187 - Plugins: Cleanup up dependencies on core headers - sslheaders + #4190 - Plugins: Cleanup up dependencies on core headers - authproxy + #4192 - Add exit code checks in clang-format.sh + #4193 - G++ 8.1.1: Fix complaint about lack of storage definitiona for class static constant + #4194 - Remove ssl_cert_loader. Certifier is more complete version. + #4198 - Cleanup, and adds support for new luajit option + #4206 - ts_file: Simple sketch of std::filesystem for internal use. + #4220 - IntrusiveHashMap: Inserts preserve order for equal keys, as with std::multimap + #4221 - IntrusiveHashMap: Fix for find false positive in some cases. + #4226 - HttpSessionManager: Fix potential infinite loop problem. + #4240 - Inherited Extendible + #4249 - Test: Convert test_PriorityQueue.cc to Catch + #4257 - Update base version of AuTest to 1.6.0 + #4260 - Removes logging of secrets + #4261 - Adds a few missing packages needed by autest + #4265 - CMakeLists.txt: Change to be generic and not need updates so often. + #4268 - Test: Convert test_X509HostnameValidator.cc to Catch + #4280 - Fix clang format problems + #4284 - Fixed ts_file test for out of tree builds + #4285 - Change atscppapi::TransformationPlugin to use atscppapi::Continuation instead of TSCont. + #4289 - Do not follow redirects after cache hit + #4290 - Remove unnecessary line in src/Makefile.am + #4305 - Generally random code cleanup + #4306 - Convert traffic_cache_tool to use ArgParser + #4309 - Orders k-v pairs to avoid making too many files + #4310 - Add warning of body_factory templates are not loaded due to missing dot file + #4331 - Make build failures also mark the CA builds as failed + #4332 - Fixes build issues related to Clang Analyzer only + #4334 - Converts ink_autoconf.h to #pragma once + #4337 - Fix Segmentation fault in ShowCache::handleCacheEvent #4328 + #4341 - clang-format + #4346 - TCL: remove tcl depencency from HostLookUp and refresh for C++11 + #4349 - Prevent segment violation in TSContDestroy() when it is called during destruction of statically-allocated objects + #4350 - Fix for ASAN buffer overrun for IpMap::contains in HttpTransact::OSDNSLookup + #4351 - Add some feeble amount of encapsulation to the Thread class. + #4357 - TCL: Remove TCL dependency from iocore + #4360 - Fix mysql_remap plugin build with MySQL 8 + #4363 - CMake: Add lib/records, unit_tests. + #4368 - s3_auth_v4: update default region map + #4374 - TCL: remove TCL dependency from RecHttp + #4377 - Add hooks for outbound TLS start and close. + #4384 - Fix gzip(compress) plugin not linked correctly with zlib + #4387 - Replace the overridable lookup with an unordered_map + #4397 - Remove deprecated HTTP/2 metrics + #4398 - Removes the old expander feature of header_rewrite + #4402 - Convert traffic_cache_tool to use ts_file + #4406 - ts_file: Add 'view' method to get a view of the path. + #4411 - Split current client transactions metrics into HTTP/1.1 and HTTP/2 + #4414 - Cleaning up TLS server verify options + #4424 - YAML: LibSWOC IntrusiveDList update. + #4426 - YAML: Move ts_meta to external C++ library. + #4427 - Add vconn reenable event + #4430 - TCL: Rmove TCL dependency from UrlRewrite + #4431 - TCL: All ink_hash_table in proxy converted to use STL + #4433 - YAML: LibSWOC upgrade for TextView. + #4434 - YAML: move MemSpan to external C++ library. + #4448 - YAML: TextView upgrade revisited. + #4449 - YAML: Inline BufferWriter formatting functions. + #4450 - Improve BufferWriter logic in HttpTransactHeaders.cc + #4453 - TCL: Code relocation in HttpCompact and HttpBodyFactory for TCL removal + #4454 - CacheTool: Cleanup TS_INLINE. + #4457 - Clean up client certificate specification and add tests + #4459 - Move the test-only plugins test_hooks and test_cppapi to tests/tools/plugins. + #4473 - Cleanup: Replace DynArray with STL vector + #4474 - clang-format + #4478 - Do not attempt clang-analyzer builds on 7.1.x branch + #4486 - Reload records client certs + #4487 - Cleanup: replace multiple HashMap with STL unordered_map + #4496 - Log deletion rewrite + #4497 - Fix chained server cert verify + #4500 - Augment cryptic diag message when log line fails to print + #4501 - Refresh mgmt signal logic to remove possible memory issues. + #4502 - Fix for #4413 - avoid ASAN crash on use after free for file name. + #4512 - Use references for indexes of range-based for loops when this avoids expensive copies. + #4522 - Rewrite TunnelHashMap in modern standard + #4524 - Convert ctx_store in SSLCertLookUp to use STL vector + #4526 - Atomic: Convert ProxyConfig to use std::atomic. + #4531 - Rewrite url after running all remap plugins + #4532 - Remove explicit free of ref-counted object. + #4536 - Minor C++ cleanup for background_fetch + #4542 - TCL: Replace RawHashTable with STL and refactor HttpBodyFactory + #4544 - Updated Dockerfile with some additional dependencies + #4545 - Fix tunnel_route action in ssl_server_name and various test fixes for Fedora 28 and 29 + #4546 - Test: fix failed clang link for test_proxy_http. + #4553 - hostdb: Replace the last TSHashTable with IntrusiveHashMap. + #4555 - Test: fix test_Ptr.cc to deal with clang not liking "p1 = p1". + #4564 - Fix duplicate keys in python dict + #4566 - Include additional names of events for debug output. + #4567 - Cleanup: Update Version support to be actual C++ code. + #4570 - MIME: Add string_view based overload for value_get. + #4573 - access_control: reduced some log errs to debug + #4576 - Add Session and SSL hooks to test_hooks Au test. + #4579 - traffic_ctl: Refactor and convert traffic_ctl to use ArgParser + #4580 - Make double_h2 reliably wait for stats to be ready + #4585 - Runroot: make runroot_handler support both ink_args and ArgParser + #4591 - An example of our preferred indentation style + #4596 - Add TS_EVENT_AIO_DONE, remove TS_AIO_EVENT_DONE + #4602 - New ScheduleOn APIs + #4606 - TLS Bridge: Fix remap for UA connection, tweak docs for remap support. + #4607 - adding TSHttpTxnRedoCacheLookup + #4609 - BWF: Change timestamp global name to be consistent with logging timestamps + #4613 - Runroot: Update verify command and refactor + #4616 - [WEBP] Transform images when a client accepts image/webp. + #4618 - Add case were origin returns 404 to x_remap gold test. + #4627 - Add test and fix regression with disable_h2 option + #4629 - Use LoopTailHandler in UDPNetHandler to support signalActivity + #4630 - Generalize the wild-card fqdn's for user_agent attributes in ssl_server_name + #4641 - Issue #4637: Clean up / unify hex conversions. + #4642 - Issue #4637 - Clean up / unify hex conversions + #4645 - Cleanup: Remove Map.h including HashMap, Vec and TSHashTable + #4646 - TCL: Remove TCL from our code base + #4657 - TLS Bridge: Fix error where connect failure lead to a silent timeout. + #4662 - Adds a reasonable rip-grep config file + #4663 - Add back in the option to conf_remap the verify_server settings. + #4666 - Allow empty fqdn in ssl_server_name for unset SNI case. + #4667 - Add forward_route in ssl_server_name + #4668 - Compare servers script for comparing responses from new and old versions of ATS + #4669 - Cleanup: This is just a stylistic cleanup of PR 4531. + #4673 - Avoid the auto-reschedule with dispatchEvent + #4674 - Cleanup: change RemapPlugins::run_single_remap to return bool. + #4675 - TLS Bridge: Add "--file" option to configuration. + #4677 - Remove the ssl wire trace feature. + #4680 - Change serverName member back to const char * to avoid crash. + #4690 - Remove unnecessary TSHandleMLocRelease + #4692 - BWF: Clean up diags log entry header construction. + #4696 - Updates the CI build scripts, adds a new build for HTTP cache-tests + #4698 - Fixes some cache-tests, and make check shouldn't run on build fail + #4703 - Correct the statements within ink_assert and ink_release_assert + #4704 - task threads ready lifecycle hook + #4705 - Eliminate remaining hack around copy CTORs + #4707 - Remove the APIHooks::invoke() function. + #4712 - Make client cert overridable + #4715 - Optimize: Avoid meaningless lock operations + #4718 - BWF: Fix handling of null char * - avoid string_view constructor. + #4719 - Add config reload to tls_tunnel test. + #4720 - Fix disable freelist options + #4721 - Optimize: Do not signal EThreads which are not blocked on cond_timedwait + #4722 - Add control for how outbound SNI is selected. + #4724 - Add shutdown lifecycle hook + #4727 - Remove unused Http2Stream destructor + #4730 - Moved AtomicBit into its own file. + #4734 - Event num ordering + #4738 - Fix typo in HdrHeap + #4746 - Optimize: tighten the logic of the PluginVC::process_read/write_side() + #4748 - Tries to enforce .git installation of pre-commit + #4749 - Add more information about event data to 'hook add' API function documentation. + #4760 - Updated STATUS with all known releases + #4764 - Adds logging around various config file loads + #4765 - Cleanup: Make _next_round_robin uint64_t + #4767 - Fixes the hook install conditional, Makefile.am is finicky... + #4779 - TLS Bridge: Fix regex compile issue with loading configurations from file. + #4782 - TLS Bridge: Cleanup, fix TSError for more consistency. + #4784 - Remove TSHttpTxnRedirectRequest appears experimental and unused. + #4785 - Bug in traffic_ctl, formatting output for config changes + #4791 - Adds some missing packages to the Dockerfile + #4796 - Autest test extension modification using opensourced testing tools + #4798 - Regex: Clean up constructors. + #4803 - i4637: A tiny bit of cleanup. + #4804 - Regex: update to use string_view. + #4805 - JA3 fingerprint and documentation + #4806 - Add Log fields that dump all MIME headers in a message. + #4816 - Fix potential unterminated string in logging in state machine. + #4818 - Add log method overload for string_view to LogObject + #4819 - Add clang-format build target to CMake editor file. + #4827 - MIME: Update MIMEField::name_get and MIMEField::value_get to return string_view + #4828 - Removes proxy.config.http.parse.allow_non_http + #4829 - Improves on the ripgrep .rc file + #4831 - Create an autest for the regex_revalidate plugin + #4832 - TextView: add overflow checking to svto_radix. + #4833 - Add test cases to exercise the verify.server* defaults. + #4834 - Change src/tscpp/api/TransformationPlugin.cc to avoid error for -Werror=subobject-linkage . + #4835 - TLS Bridge: Fix possible race condition. + #4836 - Correctly deal with the ssl.client.sni_policy if not set via conf_remap + #4837 - Fix potential instability in cacheIMSRange + #4841 - Do not call dns_result repeatedly for a valid dns result. + #4846 - Fix initialization style. + #4847 - Increase the per test case timeout from 5 to 10 seconds + #4849 - Allow client_cert and client_key to be specified by relative path + #4850 - Cleans up memcached_remap plugin README and code + #4851 - Add IOBufferReader::block_read_view method. + #4855 - Turns off HostDB disk sync by default + #4856 - Changes the return codes to be RecErrT + #4858 - Removes the echo from installing pre-commit + #4866 - Remove start_HttpProxyServerBackDoor. + #4870 - Cleanup: Remove "hooks_on" and associated machinery. + #4878 - Fix a segfault on mac, because of new ScheduleOn APIs + #4879 - Clean up StrHeapDesc constructor. + #4880 - Remove class StrTest, which is not used. + #4886 - Avoid reschedule HostDBContinuation::iterateEvent if there are no more buckets + #4888 - Adjust tests to make them more resilient for different curl versions. + #4889 - Add tsapi Au test. Initially just testing TSHttpTxnEffectiveUrlStringGet() function. + #4895 - URI Signing Strips token from upstream if configured and string buffers are dynamically allocated + #4901 - Update overridable config conversion logic. + #4902 - Adjust follow redirect logic to address a crash and avoid arguments shadowing + #4903 - Adjust connection timeout for TLS + #4909 - Update plugin API type conversions to support enums automatically. + #4914 - DFA: Update DFA to use string_view. + #4916 - Optimize: Keep cont->mutex locked, during probe the bucket by hash object within HostDBProcessor::getby + #4917 - Update HdrUtils to be C++17. + #4919 - Shutdown hook should grab the lock if there is one + #4923 - Makes the master triggers works with the 9.x Jenkins Tab name + #4926 - Make mutex lock routines deal with null mutex and general clean up. + #4927 - Cleanup: Convert HTTPHdr::length_get to real method. + #4928 - Pull Age out of the gold test + #4931 - For tls tests look at logs until config has been reloaded + #4933 - Plugin Traffic Dump: new feature for setting limit on disk usage. + #4935 - Use the appropriate length field for uintmax_t + #4940 - ats_scoped_str: c++17 cleanup. + #4941 - Add a case to the tls_client_cert2 to exercise one file with cert and key + #4942 - Load client key event if only the certificate file is specified. + #4948 - MIMEScanner: Make MIMEScanner a class, not a POD with free functions. + #4952 - Cleanup: Tweak for loops to be more consistently C++17 in style. + #4953 - Scalar: add generic rounding. + #4954 - Hdrheap more modernization + #4955 - Marks the YAML exceptions, which gives line number / pos info + #4956 - Fix a ssl handshake crash during shutdown + #4957 - Remove error-prone mirror enum in code handling TS API SSL hooks. + #4961 - Optimize: rewrite getbyname_imm and getSRVbyname_imm as wrappers for getby + #4962 - Remove unnecessary storing of redirect_url in redirect_info + #4964 - Rewrite URL before all remap plugins run + #4965 - HdrHeap: Remove pointless code and misleading comment. + #4966 - Some tidying up of the global namespace. + #4972 - Fix tls_client_verify curl command missing certificate + #4974 - Fix tls_check_cert_selection test + #4975 - URL: remove undefined function declaration. + #4976 - Fix tls_forward_nonhttp test requirement + #4979 - Ignore config.cache file + #4986 - Added TS_SSL_CLIENT_HELLO_HOOK and docs + #5000 - Add Perltidy configuration and build target + #5007 - Build fix: Fix build issue for -O3 on CI. + #5008 - Build: Fix array bounds error under -O3. + #5014 - ink_inet: Fix family string printing, add IpAddr::isAnyAddr. + #5015 - RecHttp.cc unit tests. + #5018 - RecHttp: Convert protocol session pool to use views and MemArena. + #5021 - Frees up disk_vols when volume creation failed + #5029 - Removing Lazy Buf Alloc from H1 Server session + #5030 - IpMap: Add move constructor. + #5032 - Cleanup loading certs from ssl_multicert.config + #5033 - Cleanup: use string_view for ssl_multicert.config field tags + #5035 - Overload ProcessManager::signalManager for string_view + #5038 - Cleanup: remove duplicated SSL_CTX_set_tlsext_status_cb calls for OCSP Stapling + #5039 - Add BufferWriter formatting support to IpMap + #5041 - Cleanup: Set SSL_OP_NO_TICKET in SSLInitServerContext() + #5044 - Fixed memory leaking introduced by 4873 + #5046 - Cleanup: Separate SSLStats and SSLDiags from SSLUtils + #5050 - Set thread affinity using type info + #5051 - Restructured SSL client context mapping to 2-level + #5053 - Fixed compiler error with std::string_view + #5054 - Cleanup: Separate TLS SessionTicket from SSLUtils + #5059 - Add Assertion when fragment_size is large than MAX_FRAG_SIZE + #5060 - IP support: Make IpAddr constexpr constructible, define min/max addresses in IpMap + #5065 - RemapPluginInfo Refresh. + #5068 - HdrHeap default size unit test fix. + #5069 - url_sig: fixed unit-test for remapped url. + #5070 - Ignore test_librecords + #5074 - Move minimum OpenSSL version to 1.0.2 + #5077 - Unify plugins hook dispatch + #5079 - Fix directives for checking TS_HAS_TESTS + #5082 - Add support for the lua and numeric log roll values + #5086 - Remove extra args to bwprint in SSLConfigParams::getCTX() + #5088 - MIME: Fix line_is_real false positive. + #5089 - Slight adjust to the thread affinity logic + #5091 - Override delete in Extendible + #5113 - Fix #5094: Fix use after free in test_IntrusiveHashMap.cc + #5114 - Fix #5093: new/delete mismatch in test_IntrusivePtr.cc + #5115 - Remove OS dependency in test_BufferWriterFormat.cc + #5120 - Ensure queued HostDB requests get rescheduled back to the original thread + #5123 - Fix SessionProtocolNameRegistry lookup + #5126 - -H "xdebug: probe" injects trace of headers into response body + #5128 - Change url_mapping::toUrl to url_mapping::toURL + #5129 - Fixed memory leaks in test_IntrusiveHashMap + #5132 - Fix for() loop, correctly calculate the value of seg_in_use within Vol::dir_check() + #5136 - Fixes some places where refactoring was not complete + #5138 - Run clang-format against the same files on both make clang-foramt and git pre-hook + #5139 - Add virtual destructor to SSLMultiCertConfigLoader. + #5141 - Need to pop messages from queue + #5142 - UrlRewrite: separate constructor and configuration loading for testability + #5144 - Fix AcidPtr test + #5150 - Augment wildcard_match to allow underscore in domain name + #5151 - Updates to the STATUS file + #5152 - Test acid ptr persistence + #5153 - Separate P_SSLUtil and P_SSLClientUtils includes + #5157 - Set the block's m_water_level in Arena:free even if the block is not the last block + #5167 - Fix Makefile.am to run clang-tidy + #5172 - Run clang-tidy one by one + #5174 - Adds Cache test suits + #5180 - Regex name checks on ssl_server_name should be anchored. + #5181 - Ignore check programs + #5182 - Ran clang-tidy + #5183 - Do not run clang-tidy on lib/yamlcpp/ + #5184 - traffic_via.pl: Fixed bugs, added tests, and make the output match more like traffic_via + #5185 - The response header of CONNECT should not have content-length or chun… + #5186 - Ran clang-tidy with modernize-use-default-member-init + #5187 - Use pthread_cancel to terminate plugin threads + #5192 - Exclude library headers from clang-tidy + #5193 - Suppress false positive of clang-tidy on macOS + #5194 - Check return value + #5201 - Replace EThread::has_event_loop with EThread::tt == REGULAR + #5204 - Moved unit test suppression file and updated it + #5208 - Adds a little wrapper script to build inside vscode + #5211 - Check return value of open + #5212 - Cache:ttl-in-cache should always override never-cache + #5214 - Ran CPP check on a few files while prodding around + #5215 - Fix compile problems with PR-5211 (hiredis autoconf) + #5216 - Fix the hiredis autoconf for the default case + #5222 - Fix mysql-remap compilation error + #5228 - Implement prefetched OCSP stapling responses + #5239 - Adds basic version feature for traffic_layout info + #5240 - Setting the correct directory for test_Cache + #5247 - Fixed cache test, using updated shutdown + #5255 - Remove unused variable in cache test + #5258 - Adds cache alternate update tests + #5259 - Fixed cache RWW test crash + #5261 - Fixed pthread mutex init issue with cache test + #5262 - JA3 fingerprint: Corrected usage of OpenSSL API for ec and ecpf list retrieval. + #5264 - Fix reason tag of traffic_ctl host + #5265 - Fix IntrusiveHashMap active bucket list corruption issue during expansion. + #5266 - Ignore unsupported HTTP/2 settings parameters + #5269 - Cleanup: Use internal linkage for functions which are only needed in SSLUtils.cc + #5274 - Fix HostDBReverseTest sa_family initialization. + #5277 - Tries to use linux specific tcpinfo fields + #5278 - Fixes the Brotli build issues + #5281 - Change Au test condition for minimal OpenSSL version to use traffic_layout info --versions + #5282 - Plugin reload + #5290 - MIMEScanner: Only clear scanner line buffer if at MIME_PARSE_BEFORE state + #5292 - Fix a build error in xdebug on macos + #5302 - Correct config name for proxy.config.dns.connection_mode + #5306 - Removes priorities for AIOs, thanks oknet + #5307 - correctly handle return value 0 from recv() + #5308 - cppcheck: Changed from C casting to C++ casting + #5309 - Removes non-existent include dir reference + #5310 - Cppcheck fix for iocore/dns + #5311 - cppcheck: change to C++ style pointer casting + #5312 - cppcheck: Minimize variable scopes and use different names to not shadow others + #5313 - cppcheck: Reduce the scope of the variable 'out_buf' + #5314 - cppcheck fixes for iocore/hostdb + #5315 - cppcheck: Fixed various issues with SSL files + #5317 - cppcheck: Fix various issues of Http2DependencyTree + #5318 - cppcheck: Fixes various issues under proxy/http/remap + #5320 - Added user defined conversion operator in ConstBuffer for string_view. + #5321 - cppcheck: Use initialization list instead of assigning in constructor body + #5322 - cppcheck: fixes issues found for tests in proxy/http + #5323 - cppcheck: fixes issues found for plugins/compress + #5324 - cppcheck: Change to C++ style pointer casting + #5325 - cppcheck: fix issue found in BufferWriterFormat.cc + #5326 - cppcheck: fixes issue found in proxy/IPAllow.cc + #5327 - cppcheck: Fixes issue found in DiagsConfig.cc + #5328 - cppcheck: fixes issues found for plugins/authproxy + #5330 - cppcheck: Fix various issues of Http2ConnectionState + #5333 - cppcheck: Reduces variable scope for files in mgmt/... + #5334 - cppcheck: fixes issues in ink_uuid.h + #5335 - cppcheck: Change to C++ style pointer casting in Thread.cc + #5336 - cppcheck: Fixes issues found in async_http_fetch_streaming + #5337 - cppcheck: Reduce the scope of the variable 'netvc' + #5338 - cppcheck: fixes issues found in proxy/logging + #5339 - cppcheck: fixes issues found for plugins/background_fetch + #5340 - cppcheck: fixes issues found in example/thread_pool + #5344 - cppcheck: Remove an unused private function + #5345 - cppcheck: minimize variable scopes + #5346 - cppcheck: fixes issues found in example/protocol + #5347 - cppcheck: fixes issues fround in example/remap + #5349 - cppcheck: Fix issues in P_UDPx.h + #5351 - cppcheck: fixes issues in example/ + #5353 - Remove log collation + #5355 - Revert TS-374 to avoid deadlocks + #5362 - Fix 3939 collision + #5365 - cppcheck: fix comparison issues found in plugins + #5366 - Adds assert, albeit not needed, makes CA happy + #5368 - Don't use the object after tests finishes and object is deleted + #5370 - Fixed clang-analyzer issues in cookie_remap + #5373 - Clang Analyzer: Fix IpMap.cc false positives. + #5374 - cppcheck: Fix various issues in proxy/http2/ + #5375 - cppcheck: Fix issues found in I_IOBuffer.h + #5376 - cppcheck: Fix various issues found in iocore/eventsystem + #5380 - Fixes spelling in src + #5381 - fixes spelling in include + #5382 - Fixes spelling in iocore + #5383 - Remove commented out includes + #5384 - Removes unreferenced, unused bits of code + #5386 - Remove Cache v23 support + #5387 - Fixes typos in various documentation files + #5388 - De-tabifies REVIEWERS + #5390 - Fixes logging after collation removal + #5392 - ssl_session_resuse: operator for redis endpoint compare functor must be const + #5396 - Rewrite the assert statement to avoid Socks Proxy triggering the assertions + #5398 - Load the Socks configuration after the host state persistence data is loaded + #5399 - Removes code related to removed configuration + #5400 - Fixes spelling in mgmt + #5401 - Fixes spelling in miscellaneous files + #5402 - Spelling fixes in plugins + #5403 - Fixes the number of traversal events + #5404 - Tries to make builds on older clang happy + #5411 - Update README.md + #5413 - TS C++ API: add member function TSSslConnection InterceptPlugin::getSslConnection() + #5414 - Add API and fix logic for TS_SSL_VERIFY_*_HOOK. + #5415 - MemSpan: Update to templated style, fix gcc9 compile error. + #5416 - Move setsockopt from UnixNetProcessor to Server::setup_fd_for_listen + #5420 - Adds update header only test case + #5421 - Check nullptr before call netvc->do_io_close in SSLNextProtocolAccept::mainEvent + #5423 - This fixes parsing where the [ ] section gets merged into values + #5425 - Updates the VSCode include dirs + #5431 - New APIs: Server/ClientCertUpdate + #5436 - Additional places to propagate the per-client-ip debug tags. + #5438 - Add wait_for_cache to make all_headers test more resilient + #5439 - Removes unused TSConfig usage in ssl_sni example + #5440 - Removes unused TSConfig usage in ssl_sni_whitelist example + #5441 - Improve test resilience by waiting for TS ports to be ready. + #5442 - Removes unused TSConfig usage in verify_cert example + #5443 - Address intermittent failures of all_headers Au test (see Issue # 5437). + #5445 - Rewrite SocksProxy based on states + #5446 - Fixes use-after-free in PVCTestDriver::start_tests + #5447 - Fixed nullptr check in cookie remap + #5448 - Fixed syntax issue with clang + #5450 - TLSv1.3 0-RTT support + #5451 - Fix FREELIST macros for AArch64 + #5452 - Clang Analyzer: Fix IpMap.cc false positives. #2 + #5455 - Convert ssl_preaccept plugin to use command-line arguments + #5458 - Updates the Docker image with latest dependencies + #5460 - Fixed clang-analyzer issue calling schedule_imm with the continuation possibly being a nullptr + #5463 - clang-analyzer: Fixed err value being uninitialized + #5464 - clang-analyzer: Fix API test logic and fixed clang-analyzer issue + #5466 - Set HTTP version on server side header conversion + #5467 - Add be32toh and htobe32 for macOS + #5470 - Changes traffic_wccp to use YAML as configuration format + #5472 - gcc9: Added default assignment operator + #5475 - Cleanup alarms and signals + #5477 - Removes proxy.config.http.server_tcp_init_cwnd + #5478 - Fixes some cache-tests build issues + #5480 - Add HKDF wrapper + #5490 - clang-analyzer: Fix uninitialized variable in make_to_lower_case. + #5492 - clang-analyzer: fix bogus use after free with Ptr in LogFieldInt constructor. + #5495 - Correct the clear range when cache restart + #5500 - Added Docker support for Fedora 29 and Fedora 30 + #5507 - AcidPtr: Tweaks tests to be a bit more robust. + #5508 - Grab lock before invoke + #5509 - clang analzyer - fix false positive in Vol::handle_recover_from_data. + #5510 - TextView: clean up on strcasecmp, strcmp, and memcmp. + #5518 - Adds a missing dependency, bump autest version + #5520 - Cleanup of autest prog checks, and indentation + #5521 - Check for OpenSSSL v1.1.1, for now, for this test + #5524 - clang analyzer: Fix false positive "use after free" in IpMap.cc + #5525 - clang analyzer: suppress nullptr derefence report in mime_hdr_sanity_check + #5526 - Remove f_callback_on_accept + #5528 - Allow number of settings per H2 session to be configurable + #5529 - clang analyzer: Suppress nullptr dereference error in SSLNetVConnection::update_rbio + #5530 - Removes disable_configuration_modification, making it always on + #5533 - Move logging config under toplevel YAML tag 'logging' + #5536 - More fixes and cleanup of CI scripts + #5537 - Fixed an ownership issue in autest + #5543 - Added loop detection via code and squid logging code + #5544 - Add an ignore_self_detect flag to parent.config so that the local cache host may be a member of a peering cache group. + #5547 - Make sure autest has all minimal config files + #5551 - Fix test to not break on custom layouts + #5552 - Add options to sort to reduce os differences in all_headers test + #5554 - Updates Dockerfile, and adds a comment to autest bootstrap + #5555 - Ran clang-tidy with google-readability-casting + #5557 - Removes empty config load warning for YAML-based configuration files + #5558 - Removed headers that don't exist in the dir to fix clang-tidy + #5559 - Move sni config (formerly ssl_server_name) under toplevel YAML tag 'sni' + #5560 - Added pipenv config script for AuTest + #5561 - Check DH_get_2048_256() should inclue + #5565 - Add cstdio in TextBuffer for vsnprintf + #5568 - Proxy txn cleanup + #5569 - New API: TSSslClientContextsNamesGet and TSSslClientContextFindByName + #5570 - Clean up: Remove proxy.config.http.parent_proxy_routing_enable variable + #5574 - Fix ParentSelection regression/unit tests. + #5575 - Fix order a little bit, based on F30 availability + #5579 - JA3 plugin: Fix ja3 hooks for openssl 1.0.2 + #5582 - Preserve ticket key data specified by TSSslTicketKeyUpdate. + #5586 - Ignore Pipfile.lock file + #5588 - Update UDPNet for QUIC + #5591 - Replace ProxyTransaction::get_parent()/set_parent() with get_proxy_ssn()/set_proxy_ssn() + #5592 - Fix a type mismatch in client_context_dump plugin + #5597 - Added CurlHeader tester to test curl output + #5598 - Use type info to assign an affinity thread + #5599 - Renames ssl_server_name.yaml to sni.yaml + #5600 - Use IPPROTO_TCP instead of SOL_TCP for macOS compatibility + #5603 - Eliminates most of the Rollback code, and things interacting with it + #5606 - Turns off TLS v1.0 and TLS v1.1 by default + #5608 - Removes the remaining code and configurations for SSL3 + #5609 - Added cert_reporting_tool plugin based off example/client_context_dump + #5610 - Removes the explicit Vary configurations and code + #5615 - Separate stubs for libinknet from test_I_UDPNet.cc + #5616 - Updates README to reflect current source tree layout + #5617 - Moves Errata to tscore, removes everything else from tsconfig + #5623 - This RSB is no longer used at all, so remove + #5625 - Re-enable the disjoint-wait-for-cache Au test + #5627 - Elevate privs to load TLS Session Ticket Key file + #5628 - Allows for resizing librecords via command line option + #5631 - Removes checks for curl that snuck back into the tests + #5632 - Fix up remaining references to ssl_server_name.yaml + #5635 - Add AUTest using h2spec + #5637 - TextView: Fix bug in rtrim_if when entire view is cleared. + #5639 - update example directory structure and add examples for lua plugin + #5643 - Cleanup debug log in mime_hdr_describe + #5645 - Fix default logging.yaml with new format. + #5647 - Replaces Emergency() with Error() when parsing these records.config values + #5648 - Add metrics to track SSLv3 and TLS versions + #5649 - TS API - Adds the TSHttpTxnNextHopNameGet() function. + #5651 - TS autest extension can now auto select both ssl and nonssl port + #5653 - cookie_remap plugin Au test case changes for compatibility with PR 4964. + #5654 - Fix the number of net_connections_currently_open_stat error increase + #5656 - Reenable redirect_actions Au test as it is working now. + #5658 - New API: TSEmergency and TSFatal + #5659 - Fix options processing for ja3_fingerprint plugin + #5660 - Auto port selection for more autests + #5664 - Update MT_Hashtable interface to use smart pointer + #5666 - Program to test if multiple URLs can be cached and generates a report on the cache headers + #5667 - Combohandler: Set response as private if one of the origin responses is private + #5668 - TSIOBufferReaderCopy: copy data from a reader to a fixed buffer. + #5669 - Added end pointer to ink_atoi64 and used when parsing cache size + #5670 - Convert HdrHeap regression test into unit test using Catch + #5673 - Add ats_unique_buf + #5674 - This fixes state machine looping when using upstream connection throttling with parent selection + #5678 - More Autest ports cleanup + #5683 - Cleanup example directory + #5690 - Fix race condition in test + #5698 - Remove unused LibBulkIO + #5699 - Remove unused header file ink_inout.h + #5703 - Fix indents in HttpTunnel.cc made by unifdef LAZY_BUF_ALLOC + #5704 - Add dest addr information to incoming UDPPacket objects + #5705 - Make TSVConnInacitivityTimeoutCancel work as expected. + #5706 - Add optional normalization of scheme and host to lower case letters in effective URLs. + #5707 - JA3: append to the last dub if X-JA3-Sig/X-JA3-RAW exist + #5711 - Fix client transaction stats + #5714 - Add a required library to "Getting Started" docs + #5715 - Sets macOS luajit linker flags only when luajit detected + #5716 - Fixes memory leak in traffic_crashlog + #5717 - Change default proxy name to be "traffic_server", not the build machine. + #5722 - In test_hooks Au test case, add work-around for flakeyness of VCONN start/close events. + #5724 - Enable logging of the Elliptic Curve used to communicate with the client + #5726 - Expose client request SSL stats via API & Lua plugin + #5728 - Remove header_rewrite conditions deprecated in previous versions + #5731 - Use un-deprecated records for SSL server verification + #5732 - Remove proxy.config.http.cache.allow_empty_doc + #5733 - Fix typos in log.gold file for tsapi Au test case in merged PR 5706. + #5736 - Adds voluspa - a configuration generator + #5738 - Plugins ready for Promotion (as discussed) + #5739 - Removes the various plugins as agreed upon in Beijing + #5740 - Fixes spelling in lib/records + #5745 - Cleanup debug logs around SSLWriteBuffer + #5746 - Remove unnecesary function name on debug logs in SSLNetVConnection + #5748 - cppcheck: fixed uninitialized variable and scoping in healthchecks + #5749 - cppcheck: fixed leak and scoping in ts::file::copy + #5750 - Preserve the raw log fields when wiping using case insensitive contains and update container log fields + #5751 - Add soft limit for HTTP Request URI and Header field length. + #5752 - fixed datatype in example plugin + #5753 - Add QUIC draft-20 support + #5755 - Initialize EventIO + #5762 - Limit resources used by regex_remap to prevent crashes on stack overflow + #5765 - Avoid a clang warning when all the defines are set + #5767 - Issue 3654 addr based loop detection + #5769 - Simplify h2 enable disable + #5770 - IPAllow: change top level key to "ip_allow". + #5771 - Fixes linker changes for luajit on macOS + #5772 - Remove ssl_error stats that aren't really errors. + #5774 - Updating the default cipher-suite lists for the 9.x release. + #5778 - Issue 5604 - fix memory leaks for http parser. + #5786 - Mark debug logs of polling as verbose + #5789 - Add protection against null pointer access + #5792 - Rename remaining references to ip_allow.config to ip_allow.yaml + #5803 - Make TS_NULL_MLOC a valid C compile-time constant. + #5808 - Remove unused assignment to satisfy clang-analyzer + #5809 - Address possible use after free issue in HttpVCTable::remove_entry + #5813 - Fixes broken links to documentation + #5815 - Updates links to trafficserver.apache.org to https + #5819 - make check race condition fix + #5828 - Make client_context_dump test resilient to dump order changes + #5829 - fix bugs from log filter feature + #5830 - Make proxy.config.http.per_server.min_keep_alive_connections overridable + #5831 - Fix QUIC build + #5834 - Ran clang-tidy before 9.0.x branching + #5839 - Fix inactivity timeout on QUIC + #5841 - Cleanup: unifdef WRITE_AND_TRANSFER + #5847 - Cleanup: Remove unused empty files + #5852 - Replaced ProxyTxn::outbound vars with accept::options + #5853 - correct the size of DNS buffers + #5856 - Fixes 'traffic_ctl server restart' to restart + #5857 - Check for nullptr when locking + #5863 - pipe buffer size for log.pipe should be configurable + #5864 - ProxyTxn refactor: moved host_res_style to t_state.dns_info + #5867 - PR#5867: Explain how to use open_con(). + #5868 - Update HttpTransact.cc + #5869 - ProxyTxn: removed restart_immediate + #5870 - Fix bad limit in poll loop. jtest -c1 now works again. + #5879 - Weak mutex locking macros + #5880 - ProxyTxn Refactor move code to cc + #5885 - Add the ability to static link ASAN, TSAN and LSAN + #5892 - ProxySsn renaming member vars + #5902 - Ran clang-format + #5905 - Update Server IP in Transaction when attaching a session from the pool + #5906 - Cleanup: unifdef TRACK_BUFFER_USER + #5907 - Fix macOS build + #5908 - ProxySession cleanup: moving inline functions to .h + #5917 - Make compress plugin normalization of Accept-Encoding header compatible with normalization in core TS. + #5919 - Rearrange config reload notifications + #5922 - Make code buildable with BoringSSL + #5926 - Fix AuTest for HTTP/2 using httpbin + #5937 - Update TSVConnSslConnectionGet name to match other TSVConnSsl* APIs + #5939 - Remove hard coded filename in error message + #5949 - Fix TSHttpTxnReenable to schedule back to original SM thread + #5951 - Add Base64 encoding/decoding to encryption/decryption. + #5952 - Fix code to avoid HostDBContinuation use after free + #5955 - Fix debug output for global_user_agent_header. + #5956 - Turn on certificate verification, in permissive mode for now + #5959 - Clear api set bit to avoid crash in following redirect. + #5962 - Removed hardcoded sni.yaml configuration filename in logs + #5964 - Fix a build issue on enabling FIPS + #5970 - "Plugin (un)used" post reload notification + #5977 - YAML config: output erroneous keyword and line number + #5978 - Removed hardcoded logging.yaml filename in logs + #5979 - Revert "Optimize: If failed on migrateToCurrentThread, put the server… + #5980 - Fix use-after-free problem related to logging headers + #5985 - Add filename to error message in header_rewrite + #5986 - Remove hard-coded ssl_multicert.config log reference + #5987 - Removing traffic_cop reference in admin guide introduction + #5992 - Cache SSL EC explicitly + #6000 - Add QUIC draft-23 support + #6001 - New TS maintenance commands: verify_global_plugin and verify_remap_plugin + #6006 - Rewrote remap_stats plugin to use C++ + #6007 - Graduate TSHttpTxnServerPush to the stable API interface + #6008 - Do not schedule stuff during shutdown + #6011 - clang-analyzer: Add a null check + #6013 - Add basic SystemTap markers support + #6016 - Fix host type in host matcher. + #6024 - Make proxy.config.http.request_buffer_enabled configurable and bug fix + #6025 - Remove remnants of obsolete remap thread. + #6026 - Remove obsolete 4-2-0-fixup + #6031 - listen on all the net threads + #6032 - Restore the MIOBufferWriter unit tests. + #6037 - Promote 'Enable_Config_Var' from HttpConnectionCount to HttpConfig. + #6049 - set host name in TLS extension for SNI Host check in service side with sni policy verify_with_name_source. + #6053 - Enhance Connection Collapse in ATS core + #6062 - Adding connection close headers to regex_remap test. + #6066 - Fixing session sharing with IP matching. + #6069 - Fix some long lines and reduntant plugin_config SO additions. + #6070 - Updated API header and ssl_session_reuse for new TSSslSessionInsert changes + #6071 - Premature DSO unload with "suicidal" continuations + #6074 - Allow txn handler to be set from ssn on same hook + #6075 - Disable tests using exceptions in MIOBufferWriter UT. + #6076 - Issue 4635: Address pipe reuse after configuration reload issues + #6078 - Cleanup: Ignore checkprograms of remap + #6084 - Improving the messaging around the use of TSSslSessionGetBuffer + #6085 - Only decrement log_stat_log_files_open_stat when the file is closed. + #6089 - Make If-Range date compare to be exact match + #6094 - Issue #4294: Handle return value of SSL_write() properly + #6095 - For remap_stats, removes configure time dependency on search.h + #6096 - Fixing log cleanup candidate selection and adding a test for it. + #6099 - doc + unittest TSRemap(Init|NewInstance) failures + #6103 - Use enqueue_local when scheduling on the same thread + #6104 - Cleanup the eventloop + #6106 - Removes proxy.config.cache.storage_filename + #6116 - Updated to clang-format v9.0.0 + #6120 - Fix null pointer dereference reported by clang-analyzer + #6125 - Add Metrics to track SSL Handshake attempts + #6126 - Fix the thread ready check + #6128 - Move websocket upgrade later in state machine + #6129 - Adding an autest for traffic_dump. + #6131 - Remove never implemented HttpTransact::service_transaction_in_proxy_only_mode + #6132 - Normalize loopback usage in session_match autest + #6134 - Removes the records.config.shadow alternative config file + #6137 - Make MIOBufferWriter unit tests work when compiled without -DDEBUG. + #6138 - tests: Adds autest for WebSocket mappings + #6139 - tests: fixes various python code issues found by pyflakes + #6140 - Remove using namespace std + #6141 - Update docs for SSL Handshake stats + #6144 - Cleans up some of the filenames mess + #6147 - Reverse debug specific check. + #6148 - Disable the most expensive "make check" tests by default + #6149 - For per-transaction config override, crossing the const-correctness event horizon. + #6155 - Remove unused functions of IOBuffer + #6159 - Detect bogus jemalloc version + #6160 - Fixes misc. spelling and whitespace + #6163 - Update yaml-cpp to 0.6.3 + #6166 - Make sure time is consistent between calculations + #6167 - Fixed gcc7 issue with yaml-cpp 0.6.3 + #6169 - Fixes cppcheck issues for cookie_remap plugin + #6170 - Add test to catch regressions in sni and override configs + #6173 - Remove HttpTransact::build_upgrade_response + #6174 - Remove useless UDP function + #6175 - Removes the ssn_close hook, it can never work + #6176 - Fixing rolled log cleanup parsing for .log. files. + #6180 - RBTree - fix potential nullptr dereference + #6181 - remap_stats: Fix BufferWriter usage error. + #6185 - Upgrade Catch.hpp to v2.11 + #6186 - Fix compile warnings in Catch checks for TextView + #6190 - Fix ssl_session_reuse to compile on macOS and FreeBSD + #6193 - Fixes various crashers loading/reloading parent.config + #6197 - Fix problems with "Probe" option for X-Debug MIME header field. + #6198 - Move logging before session could be freed + #6200 - Cleanup: Remove useless UDPConnection function + #6201 - ssl_session_reuse optimization to check if diags is on + #6203 - Fix the relative path for template_sets_dir to be install directory + #6204 - Lua plugin fix: Account for null in output from TSUrlPercentDecode. + #6206 - Fix the strategies.yaml documentation. + #6207 - Add autopep8 & pyflakes in dev-packages for AuTest + #6208 - Fixes spelling in strategies.yaml docs + #6210 - Cleanup trailing whitespaces, and YAML formatting + #6212 - Cleanup trailing whitespaces, and YAML formatting + #6214 - Run dos2unix on all files in tree + #6216 - This fixes next hop unit tests that segfault due to missing unit test yaml files. + #6217 - cache_range_requests plugin optional support for special IMS header + #6218 - Fix stringstream crash during shutdown + #6219 - Fixed next hop tests for out of tree builds + #6222 - For combo_handler plugin, add an optional whitelist of allowed values for Content-Type. + #6224 - Fixed build issue with clang5 and Extendible + #6225 - Add mechanism to enforce SNI policy + #6226 - Fix sni.yaml fqdn to match complete name string + #6227 - Remove never implemented regex descriptions + #6231 - Adjust the refcounts to avoid Mutex leak + #6233 - auto delete rolled log file fixes + #6235 - Fix out of bound array access in ssl_session_reuse plugin (#6235) + #6236 - remap_stats: restore handling of remap/hostname to remove memory leak + #6240 - Adds strategies.yaml to install target + #6242 - Fixes sphinx build warning with the strategies.yaml document in the admin-guide + #6244 - Copy the Client SNI Server Name out of the openssl SSL object. + #6251 - Some tweaks to reloading-plugins.en.rst + #6260 - Fix TS_USE_DIAGS usage for --disable-diags option + #6263 - TCL: cleanup in HostLookup.cc, make sure keys are stable. + #6265 - LGTM: Add header guards + #6266 - LGTM: Fix wrong type of arguments to formatting function + #6268 - Add tests to exercise H2 and chunked encoding + #6270 - LGTM: fix a comparison that is always false. + #6275 - Clear all pointers in API Hooks clear function. + #6278 - Add new log field to output server name sent by client in TLS handshake. + #6279 - Update the admin-guide hierachical caching and remap.config documentation. + #6280 - Assure no SM survives plugin factory deactivation. + #6281 - Perf: replace dynamic_cast with static_cast in this_thread() + #6283 - Adjust debug tag for ssl_sni_whitelist plugin to match plugin + #6284 - Removes the remaining references to TSQA + #6287 - Introduce NetEvent to split UnixNetVConnection and NetHandler + #6291 - Add extension ip.test.ext to Au Test, with Test method to allocate extra TCP ports. + #6293 - Add invalid config warning when cache open write fail and read retry are inconsistent + #6297 - Add links to RWW sections + #6303 - Fixes a case where the NextHop consistent hash ring may not be completely searched. + #6315 - Fix closed flag from #6287 + #6324 - Fixes clang-format issues + #6325 - Fixes typo in TLS Bridge illustration + #6329 - Fixed build issues on macOS after 0-RTT merge + #6333 - XDebug: Always append to the X- header, rather than prepend + #6339 - Fixes an issue where NextHopSelectionStrategy did not implement a parent check. + #6347 - Fix localstatedir and runtimedir for Debian layout + #6358 - Add header guard - issue #6357 + #6359 - Cleanup: Remove unused accessors of HttpVCTableEntry + #6362 - Change localtime/gmtime usages to use the threadsafe versions with local storage + #6363 - Removing always true/false comparisons + #6369 - Improve config_describe logs to print a better output format. + #6370 - Change default matching for connection value from ip to both + #6371 - Fixing shadowed variables, both global and local + #6372 - Change alloca usage to malloc/free + #6373 - Change bitfields to be unsigned explicitly + #6374 - Add exception to throw since there is no context and could cause a crash + #6375 - Export YAML CPP library headers for plugin use. + #6377 - Remove dependencies on include/tscore + #6388 - Replace python with python3 in AuTest + #6391 - Skip unnecessary HostDB update on a fall back to serving stale record + #6396 - Avoid cross-thread mutex conflicts + #6397 - Ensure that extra data beyond the chunked body is not tunneled + #6406 - Auto port select slow_post test + #6410 - Change gold files to be less restrictive + #6411 - Updating the autest version pin to 1.7.4. + #6414 - Traffic Dump: fix client request target parsing + #6421 - Add config option to enable/disable dynamic plugin reload feature + #6424 - SSL: Introduce proxy.config.ssl.server.session_ticket.number + #6431 - #6428 Clear stale captive_action during Cache read retry - WIP (in test) + #6432 - SSL: Introduce proxy.config.ssl.server.prioritize_chacha + #6435 - Fixed chunked_encoding test to work with OpenSSL 1.0.2 + #6436 - SSLNetVConnection, fixed/removed assert when running debug build + #6440 - Cleaned up smuggle-client + #6441 - Cleaned up ssl-post + #6446 - Avoid a weird name collision between HRW and tscore + #6454 - traffic_dump post_process.py + #6457 - Fix port selection for ssl ipv6 + #6459 - Add more flexible error handling when open a config file. + #6460 - Make traffic_manager be flexible when opening config files. + #6461 - Reduce minimum config files needed for golden_tests. + #6462 - x-remap ignoring age in gold file + #6463 - [CPPAPI] Provide access to TSRemapRequestInfo in RemapPlugins. + #6465 - Skipping log_retention.test.py because it is flaky in CI + #6468 - Refactor and generalize the User Arg concept, add global + #6469 - Delay cleanup when cache write continues after early client response + #6471 - Removes some things deprecated from older versions + #6474 - Remove --read_core option + #6475 - Remove noisy mutex warning + #6477 - traffic_dump: Fixing content:size collection. + #6480 - Remove some outdated files. + #6483 - Rework server side SSL_CTX creation to better handle dual_cert mismatches + #6486 - Issue 3546: Add "overridable" to the configuration variable description. + #6487 - Fix session reuse plugin shutdown crashes and cleanups + #6488 - Bikeshedding some code structures for reloadable plugins config + #6490 - Fix a compile warning + #6492 - Add null check to fix error tunnel crash + #6493 - Make all_headers test more resilient to timing + #6500 - traffic_dump: Make the client-request node gathered in a global hook + #6501 - Remove method that does nothing. + #6502 - Change Proxy Header Regression tests into Catch unit tests. + #6508 - Include start line of HTTP messages in xdebug probe output. + #6513 - Moves hosting.config finished loading message outside of parsing loop + #6516 - Fix SDK_API_TSSslServerContextCreate + #6517 - Make traffic_ctl limp along for now with large records.snap + #6519 - Adds support for configure option --enable-yaml-headers + #6523 - Removes noisy log statement from xdebug + #6524 - Require 1.1.1 as minimum openssl lib version for tls_check_dual_cert_selection Au test. + #6530 - Convert Mime and URL unit tests in proxy/hdrs to Catch. + #6532 - Add FetchSM support to dechunking in non-streaming mode with new TS API TSFetchFlagSet + #6538 - Adds partial_blind_route sni action + #6542 - Rework stats over http so that it supports both a config file and the original path parameter + #6550 - Fixes some tls autests on macOS + #6555 - Enable logging autests on macOS, clarify why Linux is required + #6556 - Removes copypasta curl text from tests and removes checks for curl + #6558 - Support body factory template suppression for internal requests + #6565 - Fixed build issue with Ubuntu 16 debug + #6566 - Add more options to session_sharing.match + #6567 - Moved printing the incoming headers for debug before remapping + #6569 - AuTest for server_push_preload plugin + #6571 - Lua plugin number of states configuration and stats printing + #6573 - SSL: Always renew TLS Session Tickets iff TLSv1.3 is being used + #6576 - Ensure TSContSchedule API family are called from an EThread. + #6577 - When using TSContSchedule() and TSContScheduleAPI() set the calling thread as the thread affinity when not already set + #6578 - Fix migrate use after free + #6586 - traffic_dump: don't dump cookies from the wire + #6590 - Skip compressible content type check with null strings. + #6591 - Cleanup: fix a inline function style + #6605 - Add support for a simple_server_retry_responses list + #6606 - Making client session id unique across HTTP/1 and 2 sessions + #6607 - Return TSFetchSM from TSFetchUrl so TSFetchFlagSet can set fetch flags + #6608 - Fix typo in system stats, change loadavg 10min to be 15min + #6613 - SNI - Tunnel: Add support to match captured group from fqdn into tunnel_route + #6615 - Updated ink_rwlock to be a real reader writer lock + #6616 - Adding a log pipe buffer size configuration test + #6617 - Remove configure option --max-api-stats which does not do anything. S… + #6618 - Fix missing virtual destructor for PluginUserArgsMixin. + #6628 - Use default rwlock attributes on initialize + #6632 - Fixes a bug where the nexthop markNextHop method to mark a host down is not called. + #6635 - Optimize HTTPHdr conversion of HTTP/1.1 to HTTP/2 + #6642 - Fixes crash loading combined(cert+key) certs + #6645 - Traffic Dump: Adding an SNI filtering option. + #6650 - Extendible asan simple + #6655 - Fix origin scheme selection with partial-blind addition + #6656 - Check sni against SSL object + #6658 - Update TSStatFindName to check that sync callback is set on the stat + #6662 - Fixes memory leak loading certs + #6663 - Fixes memory leak during log configuration + #6664 - cache_range_requests: remove unnecessary Last-Modified header from tests + #6677 - Format to match perferred if/else formatting for sh scripts + #6678 - Removes commented out code from example + #6682 - Lua Plugin - Extend the crypto API with SHA-256 and HMAC functions. + #6686 - Fixes to hostDB to avoid event and memory leaks + #6690 - Remove tls_versions from host sni policy check + #6694 - Normalizes function names to match hook names in intercept plugins + #6697 - Increase the default max_record_entries to match the original value + #6699 - Log whether client certs were exchanged in TLS handshake + #6700 - traffic_dump: add nullptr check for sni string + #6701 - slice plugin: add --include-regex, --exclude-regex parameters + #6704 - Removes ATS version from gold files + #6712 - Commenting EventIO methods + #6714 - Add Access log fields for ProxyProtocol Context + #6717 - Fixup .gitignores to match repo reality + #6718 - gcc10: fixed warning about returning local variable in int64_to_str() + #6723 - Document ip_allow in sni.yaml + #6727 - traffic_dump: add tls information to dump. + #6730 - Add HttpTransact::get_max_age and TSHttpTxnGetMaxAge + #6731 - Fix g++ 10 compile errors. + #6733 - Various python fixes + #6734 - Update expired test certificates for cert_update + #6735 - Remove unused index for SSL application specific data + #6736 - Generalize callbacks for TLS session resumption + #6737 - Removes refcounting from compress and s3_auth plugins + #6740 - Cleans up doubled words in documentation + #6742 - gcc10: fixed clearing an object of non-trivial type for wccp + #6743 - traffic_dump: refactor to make transactions written atomically + #6746 - Add an optional ramcache setting to volume.config to disable it + #6754 - Enforce Active Connection limits + #6755 - Add metrics to track default inactivity timed out connections + #6757 - ASAN: Fixed one definition rule violation + #6758 - Fix set manipulation in dual cert resolution + #6759 - Promote netvc to ProxySession + #6760 - Do not fail multicert load if line does not create entry + #6768 - clang-analyzer: Fix dead nested assignment issues + #6769 - ip_resolve - Make config variable overridable + #6770 - Weird characters in debug message + #6772 - Ensure inactivity timeout is not set when passed in timeout value is 0 + #6774 - Fixed `AddressSanitizer: odr-violation` + #6781 - 1. Set a non-zero default value for TLS Client Handshake Timeout + #6789 - Adding HTTP Status code 451 for Unavailable For Legal Reasons (RFC 7725) + #6794 - Remove whitespace from header field name in http response due to RFC7230:3.2.4 (#6793) + #6797 - Adding HTTP status 451 in apidefs as well (See PR#6789) + #6798 - clang-analyzer: uninitialized va_list + #6800 - Add TXN_CLOSE hook to CPPAPI TransactionPlugin + #6803 - fixes issue 6765, memory leak in unit test + #6805 - Fix session pool to add and fetch to beginning of hash chain instead of end + #6807 - Add le32toh and htole32 for macOS + #6808 - Open UDP ports on traffic_manager if ports are configured for QUIC + #6809 - Fix assert when client aborts during backfill + #6812 - Fix missing virtual destructor in TLSSessionResumptionSupport. + #6813 - Adding a basic ip_allow test. + #6816 - Fixes remaining memory leaks with nexthop strategy unit tests + #6817 - lua plugin: fix for incorrectly injecting global plugin stats + #6818 - Add CSV output as an option format for stats_over_http + #6819 - regex_remap: Adjust regex recursion limit down due to crashes in testing + #6822 - change overridable var type for proxy.config.http.server_session_sharing.match from int to string + #6824 - Fix test certs in client_context_dump + #6826 - Return null when do_io_write called on closed stream + #6827 - Make chunked encoding test more resilient + #6829 - traffic_dump: debug_tag and lock improvements + #6834 - Drastically improve generator.so performance for /nocache + #6835 - Change AM_LDFLAGS to be an addition, not an overwrite, in the plugin makefile + #6837 - Generalize KA check logic + #6839 - Issue 6838 Fixing the comparison in waited_enough (drain functionality) + #6843 - Schedule Transform on the same thread as the continuation + #6845 - LGTM: add header guard + #6856 - Traffic Dump: Add server response HTTP version + #6861 - Fix a crash on TLS resumption + #6866 - Protect against nullptr access during SSL Callback + #6868 - avoid dynamic_cast for non_internal requests + #6869 - Customize Max IOBuffer Size + #6872 - Track thread changes during origin connect and cache open write + #6873 - Metrics for origin close + #6880 - Add new API / TSPluginDSOReloadEnable to override DSO remap dynamic reload. + #6884 - microserver error handling: SSLError check and debug. + #6888 - AuTest: port selection improvements. + #6889 - Improve client_vc tracking + #6891 - Make h2spec test more resiliant by extending timeout + #6892 - Fix compiler issue with ICC 19.1 + #6896 - Fix dual_cert_select test to run with older openssl binary + #6898 - set sni_name with remapped origin name if sni_policy is not the default value + #6903 - Disable max_connections_active_in default and fix close logic + #6906 - Make QPACK.h self-contained + #6910 - Fix support for openssl async engine + #6915 - Make compress Au test less flakey. + #6916 - Ensure read_avail is set for the first non-empty block + #6917 - Disable lua_stats autest until we can reliably wait for stats + #6918 - example: Move to blocklists and allowlists + #6920 - Removes SSLNetVConnection::sslContextSet + #6923 - Fixed bug in the calculation of the header block fragment length + #6925 - Prevent stale netvc access on SSL Callbacks + #6929 - build: Require OCSP headers for OCSP-enablement + #6933 - Load combined file with bogus key path + #6937 - Prevent use-after-free of TransactionPlugin + #6940 - plugins: Move to blocklists and allowlists + #6941 - Move to blocklists and allowlists + #6942 - Remove dup code in QUICMultiCertConfigLoader + #6949 - Set the default thread count factor to 1x the number of logical cores + #6950 - Prevent buffer overflow during log filter actions + #6953 - Assert on valid boundaries for UserArgTable access + #6954 - Assert non-zero HdrHeap object size + #6957 - Fix leak in early data + #6968 - RateLimiting and Connection Config changes + #6969 - Update docs for some DNS config settings + #6977 - Preserve cert name through ssl vc migration + #6980 - Add maxmind acl plugin + #6981 - Update autest to version 1.8.0 + #6984 - Fix out of source tree builds for QUIC + #6994 - Adds null check + #7000 - Add QUIC draft-27 support + #7004 - Fixed core when sending back a redirect and having an invalid server response + #7005 - Added tasks and launch files for vscode, to configure, build and debug + #7007 - Updates to thread scale factor + #7008 - slice plugin: fix throttle not work + #7012 - Remove incorrect assert in inactivity timeout handling + #7014 - Add memory_profile plugin + #7022 - Fix up autest - Should fix autest on 9.0.x + #7031 - Fix code to eliminate warning and enable feature + #7034 - Move to denylists and allowlists + #7035 - Add a null check to avoid crashing + #7036 - Add virtual destructor to QUICTPConfig. + #7040 - Remove deprecated verify.server for 9.0 + #7046 - Fixes spelling H3-related code + #7048 - Fixes spelling in docs + #7050 - Fixed CLIENT-URL to use the pristine client URL + #7056 - Fix proxy.process.http.current_client_connections + #7057 - Quote out lists of servers and domains in splitdns.config example + #7059 - Fix a crash on active timeout on QUIC connections + #7060 - Document proxy.config.http.cache.post_method. + #7061 - Don't make an error on receiving retransmitted handshake data + #7062 - Signal WRITE_COMPLETE regardless of transmission progress + #7065 - Updating to AuTest 1.8.1. + #7069 - Move the direct self loop check later to HttpSM::do_http_server_open + #7071 - Adding autopep8 as a pre-commit hook. + #7074 - Add TS_USE_QUIC to traffic_layout info + #7080 - Fix a crash on connection migration to the advertised preferred address + #7081 - Updated release notes for 9.0.0 to have new QUIC 27 version + #7083 - Destroy threads after job done + #7084 - const-ify quic/http3 code + #7085 - Fixes build warnings in maxmind_acl + #7086 - Doc: fix no_content_length status code + #7087 - Fix eval_ip compile - missing const. + #7089 - Converts files to #pragma once + #7091 - Adds description for ssl_ticket_number in ssl_multicert docs + #7095 - Refresh proxy protocol diagram + #7098 - Remove more deadcode + #7104 - Don't send image/webp responses from cache to broswers that don't support it + #7110 - Add QUIC draft-29 support + #7113 - Updating our autest suite to require Python3.6 + #7117 - Fix #7116, skip the insertion of the same continuation to pending dns + #7118 - Backing out my update of our jenkin's autest file. + #7119 - URL::parse fixes for empty paths + #7120 - Check VIO availability before checking whether the VIO has data + #7121 - Don't return QUIC frame if the size exceeds maximum frame size + #7122 - Make tls_hooks tests more likely to pass particularly for 9.0.x branch + #7123 - Accept NAT rebinding on a QUIC connection + #7124 - Update and run the autopep8 make target + #7125 - SSLConfig mem leak fix + #7126 - Replaces "smart" quotes with ASCII equivalents + #7128 - Protect TSActionCancel from null INKContInternal actions + #7129 - Comment out a wrong assertion in QUIC Loss Detection logic + #7131 - Don't make an error on duplicated RETIRE_CONNECTION frames + #7134 - Do not lose original inactivity timeout on disable + #7135 - Replace ACTION_RESULT_NONE with nullptr + #7137 - Removes duplicated listing of files in same Makefile target + #7138 - Remove useless shortopt + #7140 - Fixes garbled logs when using % log tag + #7143 - Deprecate cqhv field + #7144 - Fix typo in cache docs + #7145 - Check VIO availability before acquiring a lock for it + #7147 - slice: check if vio is still valid before calling TSVIODone* on shutdown + #7148 - Fix stale pointer due to SSL config reload + #7155 - PluginFactory - Remove unused code. + #7157 - rc: fixes systemd unit file stopping + #7159 - MicroDNS Extension: handle different 'default' types + #7160 - Fix memory leaks in multiplexer plugin + #7161 - Traffic Dump documentation for post_process.py + #7162 - Removes references to non-existent function handle_conditional_headers + #7165 - Build test C/C++ files with Automake. (#6945) + #7166 - Fix #7164 Changing Warning to Debug and creating a stat + #7168 - Fix #7167, make pep8 failure + #7169 - AuTest: Properly handle experimental plugins. (#6971) + #7172 - Fix leaks in BaseLogFiles.cc + #7173 - Adds new plugin: statichit + #7178 - AuTest: Reuse venv if it exists already + #7181 - TS_API for Note,Status,Warning,Alert,Fatal + #7183 - Emits log when OCSP fails to connect to server + #7186 - autopep8: avoid running on non-committed files. + #7193 - Make custom xdebug HTTP header name available to other plugins. + #7199 - Rename ambiguous log variable + #7202 - Strip whitespaces after field-name and before the colon in headers from the origin + #7210 - Docs cleanup + #7213 - Follow redirection responses when refreshing stale cache objects. + #7215 - Log config reload: use new config for initialization diff --git a/CMakeLists.txt b/CMakeLists.txt index b464fcbea67..918ff24e1c5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -63,7 +63,7 @@ endmacro(CPP_LIB) macro(CPP_ADD_SOURCES target path) file(GLOB cpp_add_src_files ${path}/*.h ${path}/*.cc) - target_sources(${target} PUBLIC ${cpp_add_src_files}) + target_sources(${target} PRIVATE ${cpp_add_src_files}) endmacro(CPP_ADD_SOURCES) CC_EXEC(traffic_cache_tool src/traffic_cache_tool) @@ -74,6 +74,7 @@ CC_EXEC(traffic_logcat src/traffic_logcat) CC_EXEC(traffic_logstats src/traffic_logstats) CC_EXEC(traffic_manager src/traffic_manager) CC_EXEC(traffic_server src/traffic_server) +target_sources(traffic_server PRIVATE src/shared/overridable_txn_vars.cc) CC_EXEC(traffic_top src/traffic_top) CC_EXEC(traffic_via src/traffic_via) CC_EXEC(traffic_wccp src/traffic_wccp) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 9c208874686..b2f5f5703bd 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -16,7 +16,7 @@ New Issues process replacing old Jira 1. If there is an issue/feature, an existing Jira Ticket, and no code, then create a Github _Issue_. Copy the relevant information into the Github _Issue_ and request the Jira Ticket to be closed. Hopefully this use case - won’t happen very often. + won't happen very often. 2. If there is an issue/feature and no code, then create a Github _Issue_. When there is code later, create a Github Pull Request and reference the @@ -58,10 +58,13 @@ are a few simple rules to follow: 8. Make sure you run **clang-format** before making the _PR_. This is easiest done with e.g. "make clang-format", which works on OSX and Linux. -9. When making backports, make sure you mark the _PR_ for the appropriate +9. Make sure you run **autopep8** before making the _PR_. This is easiest + done with e.g. "make autopep8". + +10. When making backports, make sure you mark the _PR_ for the appropriate Github branch (e.g. **6.2.x**). -10. If you are making backports to an LTS branch, remember that the job of +11. If you are making backports to an LTS branch, remember that the job of merging such a _PR_ is the duty of the release manager. diff --git a/CRUFT.txt b/CRUFT.txt deleted file mode 100644 index 5d4a54a62fe..00000000000 --- a/CRUFT.txt +++ /dev/null @@ -1,44 +0,0 @@ -CRUFT -***** - -This file is designated for tracking cruft in our code. That is, code paths -or ugly hacks that were put into place for a reason: Getting stuff to work -but which could either been done better, or which time will obsolete. - -Examples are crude workarounds for broken compilers, libraries, OSes, or -hardware, or or massive ``#ifdef`` clusters, that are better abstracted away -into autoconf and convenience wrappers. - -If you are new to the project this is a good place to look for explanations -of why things are the way they are, or help us fix things and make the code -easier to read and maintain. - - -Store.cc -======== -``iocore/cache/Store.cc`` contains three different ``#ifdef`` clusters with -near identical code. Often you will read the same confused comment in three -places, see for instance: http://issues.apache.org/jira/browse/TS-1707 - - -Java -==== - -we should get rid of: ./example/protocol/test/*.java because, as zwoop says -"friends don't let friends write code in Java". - -Plugins -======= - -``geoip_acl`` should be a "helper plugin:, or a library/API which other -plugins can use. - - -Configuration & Defaults -======================== - -Right now our server doesn't work without a reasonably filled records.config -There are varying opinions on how this could or should be fixed, however one -issue that arrises is that default configuration options are often set ad-hoc -where ever they are needed as #defines. This should instead be consolidated -into a single place. diff --git a/LAYOUT b/LAYOUT deleted file mode 100644 index a6fb1d3f0f9..00000000000 --- a/LAYOUT +++ /dev/null @@ -1,32 +0,0 @@ -The Traffic Server 3.x Default LAYOUT --------------------------------- - -` ........................... Top-Level Traffic Server Directory -| -| -|-- bin ..................... Binaries -|-- etc ..................... Configuration files -| `-- trafficserver -| |-- body_factory -| | `-- default -| |-- internal -| `-- snapshots -|-- include -| `-- ts -|-- lib -| `-- trafficserver -|-- libexec -| `-- trafficserver -|-- share -| `-- trafficserver -| |-- configure -| | `-- helper -| |-- images -| |-- include -| |-- monitor -| `-- mrtg -`-- var - |-- log - | `-- trafficserver ... Log files - `-- trafficserver ....... Runtime data - diff --git a/Makefile.am b/Makefile.am index 889be94abc9..756b97c6387 100644 --- a/Makefile.am +++ b/Makefile.am @@ -27,7 +27,7 @@ export CCACHE_BASEDIR # and mgmt, hence we have to build proxy/hdrs first. # depends on the generates ts/ts.h include file. -SUBDIRS = src/tscpp/util lib src/tscore iocore proxy mgmt src plugins tools example rc configs include +SUBDIRS = src/tscpp/util lib src/tscore iocore proxy mgmt src plugins tools example rc configs include tests if BUILD_DOCS SUBDIRS += doc include @@ -104,13 +104,15 @@ if BUILD_DOCS @cd doc && $(MAKE) $(AM_MAKEFLAGS) install-man endif -TESTS = tools/check-unused-dependencies +if OS_LINUX + TESTS = tools/check-unused-dependencies +endif rat: java -jar $(top_srcdir)/ci/apache-rat-0.13-SNAPSHOT.jar -E $(top_srcdir)/ci/rat-regex.txt -d $(top_srcdir) autopep8: - @autopep8 -i -r $(top_srcdir) + @$(top_srcdir)/tools/autopep8.sh $(top_srcdir) # # These are rules to make clang-format easy and fast to run. Run it with e.g. diff --git a/REVIEWERS b/REVIEWERS deleted file mode 100644 index 628abc22baa..00000000000 --- a/REVIEWERS +++ /dev/null @@ -1,115 +0,0 @@ -These are Committers who have expressed general interest in the listed parts -of the Traffic Server code to the extent that they will review or at least read -over changes in those parts and should review major changes. - -All changes should have a jira ticket: - https://issues.apache.org/jira/browse/TS - -Big changes should be discussed on one or more of: - mailing list: dev@trafficserver.apache.org - IRC: #traffic-server irc.freenode.net - -Major or controversial changes should be have wiki page and may require a vote: - http://cwiki.apache.org/confluence/display/TS/Traffic+Server - -Committers: add modules as needed and any qualifications after your e-mail address -==================================================================================== - -all/general interest - jplevyak@apache.org - bcall@apache.org - briang@apache.org - mturk@apache.org - zwoop@apache.org - jim@apache.org - ericb@apache.org - manjesh@apache.org -lib/ts - jplevyak@apache.org - zwoop@apache.org - bcall@apache.org - georgep@apache.org - ericb@apache.org -lib/records - bcall@apache.org - georgep@apache.org - zwoop@apache.org -Event System/Buffering/VIO/VConnection - jplevyak@apache.org - bcall@apache.org - briang@apache.org - georgep@apache.org -Network I/O - jplevyak@apache.org - not including SSL or UDP except where it intersects the other code - bcall@apache.org - georgep@apache.org - briang@apache.org - jim@apache.org - zwoop@apache.org -Raw Cache and AIO - jplevyak@apache.org - bcall@apache.org - georgep@apache.org -HTTP Caching - bcall@apache.org - jim@apache.org - zwoop@apache.org -Block-Cache - jplevyak@apache.org - georgep@apache.org - jim@apache.org - zym@apache.org -DNS/HostDB - jplevyak@apache.org - zwoop@apache.org - bcall@apache.org - georgep@apache.org - ericb@apache.org -FastIO -Docs - dianes@apache.org -Examples/Plugins - zwoop@apache.org - bcall@apache.org - briang@apache.org - ericb@apache.org -HDRs - bcall@apache.org - zwoop@apache.org - briang@apache.org -HTTP - bcall@apache.org - zwoop@apache.org - briang@apache.org - jim@apache.org - ericb@apache.org -Remap - zwoop@apache.org - bcall@apache.org - jim@apache.org - ericb@apache.org -API - zwoop@apache.org - briang@apache.org - bcall@apache.org - georgep@apache.org - ericb@apache.org -Config - zwoop@apache.org - bcall@apache.org - georgep@apache.org -Logging - georgep@apache.org -MGMT - georgep@apache.org -Stats - bcall@apache.org - georgep@apache.org - briang@apache.org - zwoop@apache.org -Build System - zwoop@apache.org - bcall@apache.org - georgep@apache.org - mturk@apache.org - jim@apache.org diff --git a/build/common.m4 b/build/common.m4 index 6f5148a9aac..76c49eb4d16 100644 --- a/build/common.m4 +++ b/build/common.m4 @@ -323,7 +323,7 @@ AC_DEFUN([TS_LAYOUT], [ eval "val=\"\$$var\"" case $val in *+) - val=`echo $val | sed -e 's;\+$;;'` + val=`echo $val | sed -e 's;[\+]$;;'` eval "$var=\"\$val\"" autosuffix=yes ;; @@ -350,7 +350,7 @@ AC_DEFUN([TS_LAYOUT], [ eval "val=\"\$$var\"" case $val in *+) - val=`echo $val | sed -e 's;\+$;;'` + val=`echo $val | sed -e 's;[\+]$;;'` eval "$var=\"\$val\"" autosuffix=yes ;; diff --git a/build/crypto.m4 b/build/crypto.m4 index 6361955abbe..331be0616a5 100644 --- a/build/crypto.m4 +++ b/build/crypto.m4 @@ -259,10 +259,13 @@ AC_DEFUN([TS_CHECK_CRYPTO_OCSP], [ _ocsp_saved_LIBS=$LIBS TS_ADDTO(LIBS, [$OPENSSL_LIBS]) - AC_CHECK_HEADERS(openssl/ocsp.h) - AC_CHECK_FUNCS(OCSP_sendreq_new OCSP_REQ_CTX_add1_header OCSP_REQ_CTX_set1_req, [enable_tls_ocsp=yes], [enable_tls_ocsp=no]) + AC_CHECK_HEADERS(openssl/ocsp.h, [ocsp_have_headers=1], [enable_tls_ocsp=no]) - LIBS=$_ocsp_saved_LIBS + if test "$ocsp_have_headers" == "1"; then + AC_CHECK_FUNCS(OCSP_sendreq_new OCSP_REQ_CTX_add1_header OCSP_REQ_CTX_set1_req, [enable_tls_ocsp=yes], [enable_tls_ocsp=no]) + + LIBS=$_ocsp_saved_LIBS + fi AC_MSG_CHECKING(whether OCSP is supported) AC_MSG_RESULT([$enable_tls_ocsp]) @@ -287,3 +290,31 @@ AC_DEFUN([TS_CHECK_CRYPTO_SET_CIPHERSUITES], [ TS_ARG_ENABLE_VAR([use], [tls-set-ciphersuites]) AC_SUBST(use_tls_set_ciphersuites) ]) + +dnl +dnl Since OpenSSL 1.1.1 +dnl +AC_DEFUN([TS_CHECK_EARLY_DATA], [ + _set_ciphersuites_saved_LIBS=$LIBS + + TS_ADDTO(LIBS, [$OPENSSL_LIBS]) + AC_CHECK_HEADERS(openssl/ssl.h) + AC_CHECK_FUNCS( + SSL_set_max_early_data, + [ + has_tls_early_data=1 + early_data_check=yes + ], + [ + has_tls_early_data=0 + early_data_check=no + ] + ) + + LIBS=$_set_ciphersuites_saved_LIBS + + AC_MSG_CHECKING([for OpenSSL early data support]) + AC_MSG_RESULT([$early_data_check]) + + AC_SUBST(has_tls_early_data) +]) diff --git a/build/jemalloc.m4 b/build/jemalloc.m4 index 48b2cdc006d..6b707cdfd0a 100644 --- a/build/jemalloc.m4 +++ b/build/jemalloc.m4 @@ -68,7 +68,18 @@ if test "$enable_jemalloc" != "no"; then AC_CHECK_HEADERS(jemalloc/jemalloc.h, [jemalloc_have_headers=1]) fi if test "$jemalloc_have_headers" != "0"; then - jemalloch=1 + AC_RUN_IFELSE([ + AC_LANG_PROGRAM( + [#include ], + [ + #if (JEMALLOC_VERSION_MAJOR == 0) + exit(1); + #endif + ] + )], + [jemalloch=1], + [AC_MSG_ERROR(jemalloc has bogus version)] + ) else CPPFLAGS=$saved_cppflags LDFLAGS=$saved_ldflags diff --git a/build/plugins.mk b/build/plugins.mk index 86d0a66a681..4556fc22e2d 100644 --- a/build/plugins.mk +++ b/build/plugins.mk @@ -22,7 +22,7 @@ TS_PLUGIN_LD_FLAGS = \ -module \ -shared \ -avoid-version \ - -export-symbols-regex '^(TSRemapInit|TSRemapDone|TSRemapDoRemap|TSRemapNewInstance|TSRemapDeleteInstance|TSRemapOSResponse|TSPluginInit)$$' + -export-symbols-regex '^(TSRemapInit|TSRemapDone|TSRemapDoRemap|TSRemapNewInstance|TSRemapDeleteInstance|TSRemapOSResponse|TSPluginInit|TSRemapPreConfigReload|TSRemapPostConfigReload)$$' TS_PLUGIN_CPPFLAGS = \ -I$(abs_top_builddir)/proxy/api \ diff --git a/build/yaml-cpp.m4 b/build/yaml-cpp.m4 index 67c5b1397f7..596ca190386 100644 --- a/build/yaml-cpp.m4 +++ b/build/yaml-cpp.m4 @@ -84,3 +84,14 @@ AC_SUBST([YAMLCPP_LIBS]) AC_SUBST([YAMLCPP_LDFLAGS]) ]) + +dnl TS_CHECK_YAML_HEADERS_EXPORT: check if we want to export yaml-cpp headers from trafficserver. default: not exported +AC_DEFUN([TS_CHECK_YAML_HEADERS_EXPORT], [ +AC_MSG_CHECKING([whether to export yaml-cpp headers]) +AC_ARG_ENABLE([yaml-headers], + [AS_HELP_STRING([--enable-yaml-headers],[Export yaml-cpp headers])], + [], + [enable_yaml_headers=no] +) +AC_MSG_RESULT([$enable_yaml_headers]) +]) diff --git a/ci/jenkins/bin/autest.sh b/ci/jenkins/bin/autest.sh index de0855e8fb2..7df0372b1e5 100755 --- a/ci/jenkins/bin/autest.sh +++ b/ci/jenkins/bin/autest.sh @@ -18,7 +18,28 @@ set +x -INSTALL="${WORKSPACE}/${BUILD_NUMBER}/install" +cd src + +if [ ! -z "$ghprbActualCommit" ]; then + git diff ${ghprbActualCommit}^...${ghprbActualCommit} --name-only | egrep -E '^(build|iocore|proxy|tests|include|mgmt|plugins|proxy|src)/' > /dev/null + if [ $? = 1 ]; then + echo "No relevant files changed, skipping run" + exit 0 + fi +fi + +# Set default encoding UTF-8 for AuTest +export LC_ALL=en_US.UTF-8 +export LANG=en_US.UTF-8 +echo "LC_ALL: $LC_ALL" +echo "LANG: $LANG" + +# Check python version & encoding +python3 --version +echo "python default encoding: " +python3 -c "import sys; print(sys.getdefaultencoding())" + +INSTALL="${ATS_BUILD_BASEDIR}/install" URL="https://ci.trafficserver.apache.org/autest" JOB_ID=${ghprbPullId:-${ATS_BRANCH:-master}} AUSB="ausb-${JOB_ID}.${BUILD_NUMBER}" @@ -29,13 +50,23 @@ CCACHE="" WERROR="" DEBUG="" WCCP="" +LUAJIT="" +QUIC="" +CURL="" +AUTEST_DEBUG="" +AUTEST_VERBOSE="" + [ "1" == "$enable_ccache" ] && CCACHE="--enable-ccache" [ "1" == "$enable_werror" ] && WERROR="--enable-werror" [ "1" == "$enable_debug" ] && DEBUG="--enable-debug" [ "1" == "$enable_wccp" ] && WCCP="--enable-wccp" +[ "1" == "$enable_luajit" ] && LUAJIT="--enable-luajit" +[ "1" == "$enable_quic" ] && QUIC="--with-openssl=/opt/openssl-quic" +[ "1" == "$disable_curl" ] && CURL="--disable-curl" +[ "1" == "$enable_autest_debug" ] && AUTEST_DEBUG="--debug" +[ "1" == "$enable_autest_verbose" ] && AUTEST_VERBOSE="--verbose" mkdir -p ${INSTALL} -cd src # The tests directory must exist (i.e. for older branches we don't run this) [ -d tests ] || exit 0 @@ -44,6 +75,9 @@ echo "CCACHE: $CCACHE" echo "WERROR: $WERROR" echo "DEBUG: $DEBUG" echo "WCCP: $WCCP" +echo "LUAJIT: $LUAJIT" +echo "QUIC: $QUIC" +echo "CURL: $CURL" # Restore verbose shell output set -x @@ -56,8 +90,11 @@ autoreconf -if --enable-example-plugins \ ${CCACHE} \ ${WCCP} \ + ${LUAJIT} \ + ${QUIC} \ ${WERROR} \ - ${DEBUG} + ${DEBUG} \ + ${CURL} # Build and run regressions ${ATS_MAKE} -j4 && ${ATS_MAKE} install @@ -68,11 +105,7 @@ set +x echo -n "=======>>>> Started on " date -AUTEST="/usr/bin/autest" -[ ! -x ${AUTEST} ] && AUTEST="/usr/local/bin/autest" -set -x - -${AUTEST} -D ./tests/gold_tests --sandbox "$SANDBOX" --ats-bin "${INSTALL}/bin" +./tests/autest.sh --sandbox "$SANDBOX" --ats-bin "${INSTALL}/bin" $AUTEST_DEBUG $AUTEST_VERBOSE status=$? set +x diff --git a/ci/jenkins/bin/clang-format.sh b/ci/jenkins/bin/clang-format.sh index e6d1bc2f197..44e7c971932 100755 --- a/ci/jenkins/bin/clang-format.sh +++ b/ci/jenkins/bin/clang-format.sh @@ -23,6 +23,9 @@ autoreconf -if && ./configure ${ATS_MAKE} -j clang-format [ "0" != "$?" ] && exit -1 +${ATS_MAKE} autopep8 +[ "0" != "$?" ] && exit 1 + git diff --exit-code [ "0" != "$?" ] && exit -1 diff --git a/config.layout b/config.layout index 39183008c4c..da63e895a9d 100644 --- a/config.layout +++ b/config.layout @@ -192,8 +192,8 @@ docdir: ${prefix}/share/doc+ installbuilddir: ${prefix}/share/trafficserver/build includedir: ${prefix}/include - localstatedir: /var/run - runtimedir: /var/run+ + localstatedir: /run + runtimedir: /run+ logdir: /var/log+ cachedir: /var/cache+ diff --git a/configs/Makefile.am b/configs/Makefile.am index 0f60e0cdfff..409dbc0c161 100644 --- a/configs/Makefile.am +++ b/configs/Makefile.am @@ -34,10 +34,11 @@ dist_sysconf_DATA = \ parent.config.default \ plugin.config.default \ remap.config.default \ + sni.yaml.default \ socks.config.default \ splitdns.config.default \ ssl_multicert.config.default \ - sni.yaml.default \ + strategies.yaml.default \ volume.config.default install-exec-hook: diff --git a/configs/body_factory/default/Makefile.am b/configs/body_factory/default/Makefile.am index 953037ccfc8..a24d2e290dc 100644 --- a/configs/body_factory/default/Makefile.am +++ b/configs/body_factory/default/Makefile.am @@ -45,5 +45,5 @@ dist_bodyfactory_DATA = \ timeout\#inactivity \ transcoding\#unsupported \ urlrouting\#no_mapping \ - request\#uri_len_too_long + request\#uri_len_too_long diff --git a/configs/records.config.default.in b/configs/records.config.default.in index 61028b0501a..278139f5ffb 100644 --- a/configs/records.config.default.in +++ b/configs/records.config.default.in @@ -9,7 +9,7 @@ # https://docs.trafficserver.apache.org/records.config#thread-variables ############################################################################## CONFIG proxy.config.exec_thread.autoconfig INT 1 -CONFIG proxy.config.exec_thread.autoconfig.scale FLOAT 1.5 +CONFIG proxy.config.exec_thread.autoconfig.scale FLOAT 1.0 CONFIG proxy.config.exec_thread.limit INT 2 CONFIG proxy.config.accept_threads INT 1 CONFIG proxy.config.task_threads INT 2 @@ -118,7 +118,7 @@ CONFIG proxy.config.http.cache.heuristic_lm_factor FLOAT 0.10 ############################################################################## CONFIG proxy.config.net.connections_throttle INT 30000 CONFIG proxy.config.net.max_connections_in INT 30000 -CONFIG proxy.config.net.max_connections_active_in INT 10000 +CONFIG proxy.config.net.max_requests_in INT 0 ############################################################################## # RAM and disk cache configurations. Docs: @@ -163,10 +163,9 @@ CONFIG proxy.config.reverse_proxy.enabled INT 1 # https://docs.trafficserver.apache.org/records.config#client-related-configuration # https://docs.trafficserver.apache.org/en/latest/admin-guide/files/ssl_multicert.config.en.html ############################################################################## -CONFIG proxy.config.ssl.client.verify.server.policy STRING DISABLED +CONFIG proxy.config.ssl.client.verify.server.policy STRING PERMISSIVE CONFIG proxy.config.ssl.client.verify.server.properties STRING ALL CONFIG proxy.config.ssl.client.CA.cert.filename STRING NULL -CONFIG proxy.config.ssl.server.cipher_suite STRING ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:DHE-DSS-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-DSS-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES128-SHA:DHE-RSA-AES256-SHA256:DHE-DSS-AES256-SHA256:DHE-RSA-AES128-SHA256:DHE-DSS-AES128-SHA256:DHE-RSA-AES256-SHA:DHE-DSS-AES256-SHA:DHE-RSA-AES128-SHA:DHE-DSS-AES128-SHA:!aNULL:!eNULL:!EXPORT:!DES:!RC4:!MD5:!PSK:!aECDH:!EDH-DSS-DES-CBC3-SHA:!EDH-RSA-DES-CBC3-SHA:!KRB5-DES-CBC3-SHA ############################################################################## # Debugging. Docs: diff --git a/configs/strategies.yaml.default b/configs/strategies.yaml.default new file mode 100644 index 00000000000..c857fc1967d --- /dev/null +++ b/configs/strategies.yaml.default @@ -0,0 +1,132 @@ +# +# strategies.yaml +# +# Documentation: +# https://docs.trafficserver.apache.org/en/latest/admin-guide/files/strategies.yaml.en.html +# +# The purpose of this file is to specify the strategies available for +# use in locating upstream caches for use to satisfy requests +# +# This is a YAML formatted file to define hosts, groups of hosts and next hop strategies that +# may be used by remap +# +# There are three top-level YAML name spaces: 'hosts', 'groups', and 'strategies'. +# 'hosts' is a YAML list of host's definitions and is used when defining 'groups' YAML +# references are supported. +# 'groups' is a YAML list that aggregates a group of hosts together and serves as the +# equivalent to the rings used in parent.config. You may define upto five groups in a +# config, see MAX_GROUPS. +# 'strategies' is a YAML list of strategy definitions. +# +# Files may be broken up into several different files. The main file loaded by the Next Hop +# Strategy factory is this file, strategies.yaml. You may move the 'hosts' and 'groups' +# definitions into separate files and then include them in this file using: +# +# '#include path_to_hosts_and_groups_file' +# +# It is even possible to put individual strategies into separate file. The Next Hop +# strategy factory concatenates all included files together in a single YAML document at +# each point where it sees an '#include file_name'. When using this feature you just need to +# ensure that the final concatenation is a valid YAML document with 'hosts', 'groups' +# and 'strategies' in this given order. +# +# +# This example YAML document shows a complete definition in a single strategies.yaml +# file. There are other example unit test files in the source tree showing examples of +# using '#include' and different formats available for use, proxy/http/remap/unit-tests/ +# See the documentation which describes each parameter in detail. +# +# Example: +# +# +# hosts: +# - &p1 +# host: p1.foo.com +# hash_string: slsklslsk # optional hash string that replaces the hostname in consistent hashing. +# protocol: +# - scheme: http +# port: 80 +# health_check_url: http://192.168.1.1:80 +# - scheme: https +# port: 443 +# health_check_url: https://192.168.1.1:443 +# - &p2 +# host: p2.foo.com +# protocol: +# - scheme: http +# port: 80 +# health_check_url: http://192.168.1.2:80 +# - scheme: https +# port: 443 +# health_check_url: https://192.168.1.2:443 +# - &s1 +# host: s1.foo.com +# hash_string: slsklslsk # optional hash string that replaces the hostname in consistent hashing. +# protocol: +# - scheme: http +# port: 80 +# health_check_url: http://192.168.2.1:80 +# - scheme: https +# port: 443 +# health_check_url: https://192.168.2.1:443 +# - &s2 +# host: s2.foo.com +# protocol: +# - scheme: http +# port: 80 +# health_check_url: http://192.168.2.2:80 +# - scheme: https +# port: 443 +# health_check_url: https://192.168.2.2:443 +# groups: +# - &g1 +# - <<: *p1 +# weight: 0.5 +# - <<: *p2 +# weight: 0.5 +# - &g2 +# - <<: *s1 +# weight: 2.0 +# - <<: *s2 +# weight: 1.0 +# strategies: +# - strategy: 'mid-tier-north' +# policy: rr_ip # Selection strategy policy: Enum of 'consistent_hash' or 'first_live' or 'rr_strict' or 'rr_ip' or 'latched' +# hash_key: hostname # optional key to use for Hashing. Enum of 'url' or 'uri' or 'hostname' or 'path' or 'path+query' or 'cache_key' or 'path+fragment' +# go_direct: true # transactions may routed directly to the origin true/false default is true. +# parent_is_proxy: false # next hop hosts are origin servers when set to 'false', defaults to true and indicates next hop hosts are ats cache's. +# groups: # groups of hosts, these groups are used as rings in consistent hash and arrays of host groups for round_robin. +# - *g1 +# - *g2 +# scheme: http +# failover: +# max_simple_retries: 2 # default is 1, indicates the maximum number of simple retries for the listed response codes. +# ring_mode: exhaust_ring # enumerated as exhaust_ring or alternate_ring +# #1) in 'exhaust_ring' mode all the servers in a ring are exhausted before failing over to secondary ring +# #2) in 'alternate_ring' mode causes the failover to another server in secondary ring. +# response_codes: # defines the responses codes for failover in exhaust_ring mode +# - 404 +# - 502 +# - 503 +# health_check: # specifies the list of health checks that should be considered for failover. A list of enums: 'passive' or 'active' +# - passive +# - active +# - strategy: 'mid-tier-south' +# policy: latched +# hash_key: uri +# go_direct: false +# parent_is_proxy: false # next hop hosts are origin servers +# groups: +# - *g1 +# - *g2 +# scheme: https +# failover: +# max_simple_retries: 2 +# ring_mode: alternate_ring +# response_codes: +# - 404 +# - 502 +# - 503 +# health_check: +# - passive +# - active diff --git a/configure.ac b/configure.ac index c77b2841a27..806451f4741 100644 --- a/configure.ac +++ b/configure.ac @@ -283,6 +283,20 @@ AC_MSG_RESULT([$enable_tests]) TS_ARG_ENABLE_VAR([has], [tests]) AM_CONDITIONAL([BUILD_TESTS], [test 0 -ne $has_tests]) +# +# Build expensive unit tests ? +# + +AC_MSG_CHECKING([whether to enable expensive unit tests]) +AC_ARG_ENABLE([expensive-tests], + [AS_HELP_STRING([--enable-expensive-tests],[turn on expensive unit tests])], + [], + [enable_expensive_tests=no] +) +AC_MSG_RESULT([$enable_expensive_tests]) +TS_ARG_ENABLE_VAR([has], [expensive_tests]) +AM_CONDITIONAL([EXPENSIVE_TESTS], [test 0 -ne $has_expensive_tests]) + # # Build documentation? # @@ -458,16 +472,6 @@ AC_ARG_ENABLE([tproxy], ) AC_MSG_RESULT([$enable_tproxy]) -# -# Configure how many stats to allocate for plugins. Default is 512. -# -AC_ARG_WITH([max-api-stats], - [AS_HELP_STRING([--with-max-api-stats],[max number of plugin stats [default=512]])], - [max_api_stats=$withval], - [max_api_stats=512] -) -AC_SUBST(max_api_stats) - # # Max host name length that we deal with in URLs. # @@ -724,6 +728,7 @@ case $host_os in esac TS_ADDTO(AM_CPPFLAGS, [-D$host_os_def]) +AM_CONDITIONAL([OS_LINUX], [test "x$host_os_def" = "xlinux"]) dnl AM_PROG_AR is not always available, but it doesn't seem to be needed in older versions. ifdef([AM_PROG_AR], @@ -974,26 +979,97 @@ fi # Flags for ASAN if test "x${enable_asan}" = "xyes"; then - if test "x${enable_tsan}" = "xyes"; then + if test "x${enable_tsan}" = "xyes" -o "x${enable_tsan}" = "xstatic"; then AC_ERROR([Cannot have ASAN and TSAN options at the same time, pick one]) fi TS_ADDTO(AM_CFLAGS, [-fno-omit-frame-pointer -fsanitize=address]) TS_ADDTO(AM_CXXFLAGS, [-fno-omit-frame-pointer -fsanitize=address]) +elif test "x${enable_asan}" = "xstatic"; then + if test "x${enable_tsan}" = "xyes" -o "x${enable_tsan}" = "xstatic"; then + AC_ERROR([Cannot have ASAN and TSAN options at the same time, pick one]) + fi + asan_CXXFLAGS="$CXXFLAGS" + CXXFLAGS="$CXXFLAGS -fno-omit-frame-pointer -fsanitize=address -static-libasan" + AC_LANG_PUSH(C++) + AC_MSG_CHECKING([static ASAN library is available]) + AC_LINK_IFELSE( + [AC_LANG_PROGRAM([#include ], [])], + [AC_MSG_RESULT([yes])], + [ + AC_MSG_RESULT([no]) + AC_ERROR([Cannot find static ASAN library.]) + ] + ) + AC_LANG_POP + CXXFLAGS="$asan_CXXFLAGS" + TS_ADDTO(AM_CFLAGS, [-fno-omit-frame-pointer -fsanitize=address -static-libasan]) + TS_ADDTO(AM_CXXFLAGS, [-fno-omit-frame-pointer -fsanitize=address -static-libasan]) fi # Flags for LSAN stand-alone mode if test "x${enable_lsan}" = "xyes"; then - if test "x${enable_asan}" = "xyes"; then + if test "x${enable_asan}" = "xyes" -o "x${enable_asan}" = "xstatic"; then AC_ERROR([ASAN already specified, --enable-lsan is meant only for lsan stand-alone mode]) fi + if test "x${enable_tsan}" = "xyes" -o "x${enable_tsan}" = "xstatic"; then + AC_ERROR([Cannot have LSAN and TSAN options at the same time, pick one]) + fi TS_ADDTO(AM_CFLAGS, [-fno-omit-frame-pointer -fsanitize=leak]) TS_ADDTO(AM_CXXFLAGS, [-fno-omit-frame-pointer -fsanitize=leak]) +elif test "x${enable_lsan}" = "xstatic"; then + if test "x${enable_asan}" = "xyes" -o "x${enable_asan}" = "xstatic"; then + AC_ERROR([ASAN already specified, --enable-lsan is meant only for lsan stand-alone mode]) + fi + if test "x${enable_tsan}" = "xyes" -o "x${enable_tsan}" = "xstatic"; then + AC_ERROR([Cannot have LSAN and TSAN options at the same time, pick one]) + fi + AC_CHECK_LIB(lsan, _init, [lsan_have_libs=yes], [lsan_have_libs=no]) + if test "x${lsan_have_libs}" == "xno"; then + AC_ERROR([Cannot find LSAN static library]) + fi + lsan_CXXFLAGS="$CXXFLAGS" + CXXFLAGS="$CXXFLAGS -fno-omit-frame-pointer -fsanitize=leak -static-liblsan" + AC_LANG_PUSH(C++) + AC_MSG_CHECKING([static LSAN library is available]) + AC_LINK_IFELSE( + [AC_LANG_PROGRAM([#include ], [])], + [AC_MSG_RESULT([yes])], + [ + AC_MSG_RESULT([no]) + AC_ERROR([Cannot find static LSAN library.]) + ] + ) + AC_LANG_POP + CXXFLAGS="$lsan_CXXFLAGS" + TS_ADDTO(AM_CFLAGS, [-fno-omit-frame-pointer -fsanitize=leak -static-liblsan]) + TS_ADDTO(AM_CXXFLAGS, [-fno-omit-frame-pointer -fsanitize=leak -static-liblsan]) fi # Flags for TSAN if test "x${enable_tsan}" = "xyes"; then TS_ADDTO(AM_CFLAGS, [-fsanitize=thread]) TS_ADDTO(AM_CXXFLAGS, [-fsanitize=thread]) +elif test "x${enable_tsan}" = "xstatic"; then + AC_CHECK_LIB(tsan, _init, [tsan_have_libs=yes], [tsan_have_libs=no]) + if test "x${tsan_have_libs}" == "xno"; then + AC_ERROR([Cannot find TSAN static library]) + fi + tsan_CXXFLAGS="$CXXFLAGS" + CXXFLAGS="$CXXFLAGS -fsanitize=thread -static-libtsan" + AC_LANG_PUSH(C++) + AC_MSG_CHECKING([static TSAN library is available]) + AC_LINK_IFELSE( + [AC_LANG_PROGRAM([#include ], [])], + [AC_MSG_RESULT([yes])], + [ + AC_MSG_RESULT([no]) + AC_ERROR([Cannot find static TSAN library.]) + ] + ) + AC_LANG_POP + CXXFLAGS="$tsan_CXXFLAGS" + TS_ADDTO(AM_CFLAGS, [-fsanitize=thread -static-libtsan]) + TS_ADDTO(AM_CXXFLAGS, [-fsanitize=thread -static-libtsan]) fi # Checks for pointer size. @@ -1185,18 +1261,31 @@ enable_quic=no AC_MSG_CHECKING([whether APIs for QUIC are available]) AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include ]], [[ - #ifdef OPENSSL_IS_BORINGSSL SSL_QUIC_METHOD var; + ]]) + ], + [ + AC_MSG_RESULT([yes]) + enable_quic=yes + _quic_saved_LIBS=$LIBS + TS_ADDTO(LIBS, [$OPENSSL_LIBS]) + AC_CHECK_FUNCS(SSL_set_quic_early_data_enabled) + LIBS=$_quic_saved_LIBS + ], + [AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include ]], + [[ + #ifdef SSL_MODE_QUIC_HACK #else - #ifndef SSL_MODE_QUIC_HACK # error no hack for quic #endif - #endif ]]) ], - [AC_MSG_RESULT([yes]); enable_quic=yes], + [AC_MSG_RESULT([yes]); enable_quic=yes; enable_quic_old_api=yes], [AC_MSG_RESULT([no])]) + ]) + AM_CONDITIONAL([ENABLE_QUIC], [test "x$enable_quic" = "xyes"]) +AM_CONDITIONAL([ENABLE_QUIC_OLD_API], [test "x$enable_quic_old_api" = "xyes"]) TS_ARG_ENABLE_VAR([use], [quic]) AC_SUBST(use_quic) @@ -1206,6 +1295,9 @@ TS_CHECK_CRYPTO_OCSP # Check for SSL_CTX_set_ciphersuites call TS_CHECK_CRYPTO_SET_CIPHERSUITES +# Check for openssl early data support +TS_CHECK_EARLY_DATA + saved_LIBS="$LIBS" TS_ADDTO([LIBS], ["$OPENSSL_LIBS"]) @@ -1343,6 +1435,9 @@ AC_SUBST([LIBJANSSON]) TS_CHECK_YAML_CPP AM_CONDITIONAL([BUILD_YAML_CPP], [test x"$has_yaml_cpp" = x"no"]) +TS_CHECK_YAML_HEADERS_EXPORT +AM_CONDITIONAL([EXPORT_YAML_HEADERS], [test x"$enable_yaml_headers" = x"yes"]) + # Check for optional hiredis library TS_CHECK_HIREDIS @@ -1603,6 +1698,21 @@ AC_CHECK_HEADERS([GeoIP.h], [ ]) ]) +# +# Check for libmaxmind. This is the maxmind v2 API where GeoIP is the legacy +# v1 dat file based API +# +AC_CHECK_HEADERS([maxminddb.h], [ + AC_CHECK_LIB([maxminddb], [MMDB_open], [ + AC_SUBST([MAXMINDDB_LIBS], ["-lmaxminddb"]) + AC_SUBST(has_maxmind, 1) + ], [ + AC_SUBST([MAXMINDDB_LIBS], [""]) + AC_SUBST(has_maxmind, 0) + ]) +]) + +AM_CONDITIONAL([BUILD_MAXMIND_ACL_PLUGIN], [test "x${has_maxmind}" = "x1" ]) # Right now, the healthcheck plugins requires inotify_init (and friends) AM_CONDITIONAL([BUILD_HEALTHCHECK_PLUGIN], [ test "$ac_cv_func_inotify_init" = "yes" ]) @@ -1999,6 +2109,25 @@ AC_CHECK_TYPE([struct tcp_info], ]] ) +AC_MSG_CHECKING([whether to include systemtap tracing support]) +AC_ARG_ENABLE([systemtap], + [AS_HELP_STRING([--enable-systemtap], + [Enable inclusion of systemtap trace support])], + [ENABLE_SYSTEMTAP="${enableval}"], [ENABLE_SYSTEMTAP='no']) +AM_CONDITIONAL([ENABLE_SYSTEMTAP], [test x$ENABLE_SYSTEMTAP = xyes]) +AC_MSG_RESULT(${ENABLE_SYSTEMTAP}) + +if test "x${ENABLE_SYSTEMTAP}" = xyes; then + AC_CHECK_PROGS(DTRACE, dtrace) + if test -z "$DTRACE"; then + AC_MSG_ERROR([dtrace not found]) + fi + AC_CHECK_HEADER([sys/sdt.h], [SDT_H_FOUND='yes'], + [SDT_H_FOUND='no'; + AC_MSG_ERROR([systemtap support needs sys/sdt.h header])]) + AC_DEFINE([HAVE_SYSTEMTAP], [1], [Define to 1 if using probes.]) +fi + # See if we can build the remap_stats plugin AS_IF([test "x$enable_experimental_plugins" = "xyes"], [ @@ -2009,8 +2138,6 @@ AS_IF([test "x$enable_experimental_plugins" = "xyes"], AC_CHECK_FUNCS([hcreate_r hsearch_r]) ]) ]) -AM_CONDITIONAL([BUILD_REMAP_STATS_PLUGIN], - [ test "x$enable_experimental_plugins" = "xyes" -a "x$ac_cv_header_search_h" = "xyes" -a "x$ac_cv_type_struct_hsearch_data" = "xyes" -a "x$ac_cv_func_hcreate_r" = "xyes" -a "x$ac_cv_func_hsearch_r" = "xyes" ]) AC_ARG_WITH([default-stack-size], [AS_HELP_STRING([--with-default-stack-size],[specify the default stack size in bytes [default=1048576]])], @@ -2117,6 +2244,7 @@ AC_CONFIG_FILES([ tools/trafficserver.pc tools/tsxs tests/unit_tests/Makefile + tests/Makefile ]) # ----------------------------------------------------------------------------- diff --git a/contrib/openssl/README.md b/contrib/openssl/README.md index 428f9a664c0..faa89c73299 100644 --- a/contrib/openssl/README.md +++ b/contrib/openssl/README.md @@ -5,6 +5,6 @@ It should be built as follows. It must be build against openssl 1.1 or better f gcc -fPIC -shared -g -o async-test.so -I -L -lssl -lcrypto -lpthread async_engine.c -load_engine.cnf is an example openssl config file that can be passed to Traffic Server via the proxy.config.ssl.engine_cnf_file setting. +load_engine.cnf is an example openssl config file that can be passed to Traffic Server via the proxy.config.ssl.engine.conf_file setting. It describes which crypto engines should be loaded and how they should be used. In the case of our async-test crypto engine it will be used for RSA operations diff --git a/contrib/openssl/async_engine.c b/contrib/openssl/async_engine.c index facffe0b406..7c3604d606a 100644 --- a/contrib/openssl/async_engine.c +++ b/contrib/openssl/async_engine.c @@ -72,6 +72,15 @@ static int async_rsa_finish(RSA *rsa); static RSA_METHOD *async_rsa_method = NULL; +EVP_PKEY *async_load_privkey(ENGINE *e, const char *s_key_id, UI_METHOD *ui_method, void *callback_data) +{ + printf("Loading key %s\n", s_key_id); + FILE *f = fopen(s_key_id, "r"); + EVP_PKEY *key = PEM_read_PrivateKey(f, NULL, NULL, NULL); + fclose(f); + return key; +} + static int bind_async(ENGINE *e) { /* Setup RSA_METHOD */ @@ -96,7 +105,8 @@ static int bind_async(ENGINE *e) || !ENGINE_set_RSA(e, async_rsa_method) || !ENGINE_set_destroy_function(e, async_destroy) || !ENGINE_set_init_function(e, engine_async_init) - || !ENGINE_set_finish_function(e, async_finish)) { + || !ENGINE_set_finish_function(e, async_finish) + || !ENGINE_set_load_privkey_function(e, async_load_privkey)) { fprintf(stderr, "Failed to initialize\n"); return 0; } @@ -176,14 +186,17 @@ static void async_pause_job(void) { OSSL_ASYNC_FD *writefd; char buf = DUMMY_CHAR; - if ((job = ASYNC_get_current_job()) == NULL) + if ((job = ASYNC_get_current_job()) == NULL) { + printf("No job\n"); return; + } waitctx = ASYNC_get_wait_ctx(job); if (ASYNC_WAIT_CTX_get_fd(waitctx, engine_id, &pipefds[0], (void **)&writefd)) { - pipefds[1] = *writefd; + printf("Existing wait ctx\n"); + return; } else { writefd = (OSSL_ASYNC_FD *)OPENSSL_malloc(sizeof(*writefd)); if (writefd == NULL) @@ -194,6 +207,8 @@ static void async_pause_job(void) { } *writefd = pipefds[1]; + printf("New wait ctx %d %d\n", pipefds[0], pipefds[1]); + if(!ASYNC_WAIT_CTX_set_wait_fd(waitctx, engine_id, pipefds[0], writefd, wait_cleanup)) { wait_cleanup(waitctx, engine_id, pipefds[0], writefd); @@ -213,9 +228,11 @@ void * delay_method(void *arg) { int signal_fd = (intptr_t)arg; - sleep(5); - uint64_t buf = 1; + sleep(2); + char buf = DUMMY_CHAR; write(signal_fd, &buf, sizeof(buf)); + printf("Send signal to %d\n", signal_fd); + return NULL; } @@ -223,25 +240,27 @@ void spawn_delay_thread() { pthread_t thread_id; - OSSL_ASYNC_FD signal_fd; - OSSL_ASYNC_FD pipefds[2] = {0, 0}; ASYNC_JOB *job; - if ((job = ASYNC_get_current_job()) == NULL) + if ((job = ASYNC_get_current_job()) == NULL) { + printf("Spawn no job\n"); return; + } ASYNC_WAIT_CTX *waitctx = ASYNC_get_wait_ctx(job); size_t numfds; - if (ASYNC_WAIT_CTX_get_all_fds(waitctx, &signal_fd, &numfds) && numfds > 0) { + if (ASYNC_WAIT_CTX_get_all_fds(waitctx, NULL, &numfds) && numfds > 0) { + printf("Spawn, wait_ctx exists. Go away, something else is using this job\n"); } else { + OSSL_ASYNC_FD signal_fd; OSSL_ASYNC_FD pipefds[2] = {0,0}; OSSL_ASYNC_FD *writefd = OPENSSL_malloc(sizeof(*writefd)); pipe(pipefds); signal_fd = *writefd = pipefds[1]; ASYNC_WAIT_CTX_set_wait_fd(waitctx, engine_id, pipefds[0], writefd, wait_cleanup); + printf("Spawn, create wait_ctx %d %d\n", pipefds[0], pipefds[1]); + pthread_create(&thread_id, NULL, delay_method, (void *)((intptr_t)signal_fd)); } - - pthread_create(&thread_id, NULL, delay_method, (void *)((intptr_t)signal_fd)); } @@ -264,7 +283,7 @@ static int async_pub_dec(int flen, const unsigned char *from, static int async_rsa_priv_enc(int flen, const unsigned char *from, unsigned char *to, RSA *rsa, int padding) { - //printf("async_priv_enc\n"); + printf("async_priv_enc\n"); spawn_delay_thread(); async_pause_job(); return RSA_meth_get_priv_enc(RSA_PKCS1_OpenSSL()) @@ -274,7 +293,7 @@ static int async_rsa_priv_enc(int flen, const unsigned char *from, static int async_rsa_priv_dec(int flen, const unsigned char *from, unsigned char *to, RSA *rsa, int padding) { - //printf("async_priv_dec\n"); + printf("async_priv_dec\n"); spawn_delay_thread(); async_pause_job(); return RSA_meth_get_priv_dec(RSA_PKCS1_OpenSSL()) diff --git a/contrib/python/compare_RecordsConfigcc.py b/contrib/python/compare_RecordsConfigcc.py old mode 100644 new mode 100755 index 90835c97fb5..a2d9b948562 --- a/contrib/python/compare_RecordsConfigcc.py +++ b/contrib/python/compare_RecordsConfigcc.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file @@ -23,8 +23,8 @@ try: src_dir = sys.argv[1] except IndexError: - print "Usage: %s [trafficserver_source_dir]" % sys.argv[0] - print "Compares values in RecordsConfig.cc with the default records.config file" + print("Usage: %s [trafficserver_source_dir]" % sys.argv[0]) + print("Compares values in RecordsConfig.cc with the default records.config file") sys.exit(1) # We expect these keys to differ between files, so ignore them @@ -52,12 +52,12 @@ m = cc_re.search(line) if m: value = m.group(3) - value = string.lstrip(value, '"') - value = string.rstrip(value, '"') + value = value.lstrip('"') + value = value.rstrip('"') rc_cc[m.group(1)] = (m.group(2), value) # Process records.config.default.in -with open("%s/proxy/config/records.config.default.in" % src_dir) as fh: +with open("%s/configs/records.config.default.in" % src_dir) as fh: in_re = re.compile(r'(?:CONFIG|LOCAL) (\S+)\s+(\S+)\s+(\S+)') for line in fh: m = in_re.match(line) @@ -77,37 +77,37 @@ # Compare the two # If a value is in RecordsConfig.cc and not records.config.default.in, it is # ignored right now. -print "# Comparing RecordsConfig.cc -> records.config.default.in" +print("# Comparing RecordsConfig.cc -> records.config.default.in") for key in rc_in: if key in ignore_keys: continue if key not in rc_cc: - print "%s missing -> %s" % (key, "%s %s" % rc_in[key]) + print("%s missing -> %s" % (key, "%s %s" % rc_in[key])) continue if rc_cc[key] != rc_in[key]: - print "%s : %s -> %s" % (key, "%s %s" % rc_cc[key], "%s %s" % rc_in[key]) + print("%s : %s -> %s" % (key, "%s %s" % rc_cc[key], "%s %s" % rc_in[key])) # Search for undocumented variables ... missing = [k for k in rc_cc if k not in rc_doc] if len(missing) > 0: - print - print "Undocumented configuration variables:" + print() + print("Undocumented configuration variables:") for m in sorted(missing): - print "\t%s %s" % (m, "%s %s" % rc_cc[m]) + print("\t%s %s" % (m, "%s %s" % rc_cc[m])) # Search for incorrectly documented default values ... defaults = [k for k in rc_cc if k in rc_doc and rc_cc[k] != rc_doc[k]] if len(defaults) > 0: - print - print "Incorrectly documented defaults:" + print() + print("Incorrectly documented defaults:") for d in sorted(defaults): - print "\t%s %s -> %s" % (d, "%s %s" % rc_cc[d], "%s %s" % rc_doc[d]) + print("\t%s %s -> %s" % (d, "%s %s" % rc_cc[d], "%s %s" % rc_doc[d])) # Search for stale documentation ... stale = [k for k in rc_doc if k not in rc_cc] if (len(stale) > 0): - print - print "Stale documentation:" + print() + print("Stale documentation:") for s in sorted(stale): - print "\t%s" % (s) + print("\t%s" % (s)) diff --git a/contrib/python/compare_records_config.py b/contrib/python/compare_records_config.py old mode 100644 new mode 100755 index acde26ae0e1..bd610a9b5d1 --- a/contrib/python/compare_records_config.py +++ b/contrib/python/compare_records_config.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file @@ -61,7 +61,7 @@ def compare_settings(old, new): # Skip predefined values continue if old[key] != new[key]: - print "%s %s -> %s" % (key, old[key], new[key]) + print("%s %s -> %s" % (key, old[key], new[key])) if __name__ == '__main__': diff --git a/doc/.gitignore b/doc/.gitignore index 21ee9ce6bb5..2a5b506f2c5 100644 --- a/doc/.gitignore +++ b/doc/.gitignore @@ -1,4 +1,3 @@ uml/images ext/local-config.py -ext/plantuml* Pipfile.lock diff --git a/doc/.tx/config b/doc/.tx/config index 250615d8904..76f08a00f79 100644 --- a/doc/.tx/config +++ b/doc/.tx/config @@ -1927,34 +1927,34 @@ file_filter = locale//LC_MESSAGES/developer-guide/plugins/example-plugins/ source_file = _build/locale/pot/developer-guide/plugins/example-plugins/basic-authorization/working-with-http-headers.en.pot source_lang = en -[apache-traffic-server-6x.developer-guide--plugins--example-plugins--blacklist--accessing-the-transaction-being-processed_en] -file_filter = locale//LC_MESSAGES/developer-guide/plugins/example-plugins/blacklist/accessing-the-transaction-being-processed.en.po -source_file = _build/locale/pot/developer-guide/plugins/example-plugins/blacklist/accessing-the-transaction-being-processed.en.pot +[apache-traffic-server-6x.developer-guide--plugins--example-plugins--denylist--accessing-the-transaction-being-processed_en] +file_filter = locale//LC_MESSAGES/developer-guide/plugins/example-plugins/denylist/accessing-the-transaction-being-processed.en.po +source_file = _build/locale/pot/developer-guide/plugins/example-plugins/denylist/accessing-the-transaction-being-processed.en.pot source_lang = en -[apache-traffic-server-6x.developer-guide--plugins--example-plugins--blacklist--index_en] -file_filter = locale//LC_MESSAGES/developer-guide/plugins/example-plugins/blacklist/index.en.po -source_file = _build/locale/pot/developer-guide/plugins/example-plugins/blacklist/index.en.pot +[apache-traffic-server-6x.developer-guide--plugins--example-plugins--denylist--index_en] +file_filter = locale//LC_MESSAGES/developer-guide/plugins/example-plugins/denylist/index.en.po +source_file = _build/locale/pot/developer-guide/plugins/example-plugins/denylist/index.en.pot source_lang = en -[apache-traffic-server-6x.developer-guide--plugins--example-plugins--blacklist--setting-a-global-hook_en] -file_filter = locale//LC_MESSAGES/developer-guide/plugins/example-plugins/blacklist/setting-a-global-hook.en.po -source_file = _build/locale/pot/developer-guide/plugins/example-plugins/blacklist/setting-a-global-hook.en.pot +[apache-traffic-server-6x.developer-guide--plugins--example-plugins--denylist--setting-a-global-hook_en] +file_filter = locale//LC_MESSAGES/developer-guide/plugins/example-plugins/denylist/setting-a-global-hook.en.po +source_file = _build/locale/pot/developer-guide/plugins/example-plugins/denylist/setting-a-global-hook.en.pot source_lang = en -[apache-traffic-server-6x.developer-guide--plugins--example-plugins--blacklist--setting-up-a-transaction-hook_en] -file_filter = locale//LC_MESSAGES/developer-guide/plugins/example-plugins/blacklist/setting-up-a-transaction-hook.en.po -source_file = _build/locale/pot/developer-guide/plugins/example-plugins/blacklist/setting-up-a-transaction-hook.en.pot +[apache-traffic-server-6x.developer-guide--plugins--example-plugins--denylist--setting-up-a-transaction-hook_en] +file_filter = locale//LC_MESSAGES/developer-guide/plugins/example-plugins/denylist/setting-up-a-transaction-hook.en.po +source_file = _build/locale/pot/developer-guide/plugins/example-plugins/denylist/setting-up-a-transaction-hook.en.pot source_lang = en -[apache-traffic-server-6x.developer-guide--plugins--example-plugins--blacklist--source-code_en] -file_filter = locale//LC_MESSAGES/developer-guide/plugins/example-plugins/blacklist/source-code.en.po -source_file = _build/locale/pot/developer-guide/plugins/example-plugins/blacklist/source-code.en.pot +[apache-traffic-server-6x.developer-guide--plugins--example-plugins--denylist--source-code_en] +file_filter = locale//LC_MESSAGES/developer-guide/plugins/example-plugins/denylist/source-code.en.po +source_file = _build/locale/pot/developer-guide/plugins/example-plugins/denylist/source-code.en.pot source_lang = en -[apache-traffic-server-6x.developer-guide--plugins--example-plugins--blacklist--working-with-http-header-functions_en] -file_filter = locale//LC_MESSAGES/developer-guide/plugins/example-plugins/blacklist/working-with-http-header-functions.en.po -source_file = _build/locale/pot/developer-guide/plugins/example-plugins/blacklist/working-with-http-header-functions.en.pot +[apache-traffic-server-6x.developer-guide--plugins--example-plugins--denylist--working-with-http-header-functions_en] +file_filter = locale//LC_MESSAGES/developer-guide/plugins/example-plugins/denylist/working-with-http-header-functions.en.po +source_file = _build/locale/pot/developer-guide/plugins/example-plugins/denylist/working-with-http-header-functions.en.pot source_lang = en [apache-traffic-server-6x.developer-guide--plugins--example-plugins--query-remap--example-query-remap_en] diff --git a/doc/Doxyfile b/doc/Doxyfile index 39aca551484..083f1d00ac7 100644 --- a/doc/Doxyfile +++ b/doc/Doxyfile @@ -646,7 +646,7 @@ SHOW_NAMESPACES = YES # The FILE_VERSION_FILTER tag can be used to specify a program or script that # doxygen should invoke to get the current version for each file (typically from # the version control system). Doxygen will invoke the program by executing (via -# popen()) the command command input-file, where command is the value of the +# popen()) the command input-file, where command is the value of the # FILE_VERSION_FILTER tag, and input-file is the name of an input file provided # by doxygen. Whatever the program writes to standard output is used as the file # version. For an example see the documentation. diff --git a/doc/admin-guide/configuration/hierarchical-caching.en.rst b/doc/admin-guide/configuration/hierarchical-caching.en.rst index 81faf2f80e3..ecfb5c7c3ff 100644 --- a/doc/admin-guide/configuration/hierarchical-caching.en.rst +++ b/doc/admin-guide/configuration/hierarchical-caching.en.rst @@ -72,7 +72,7 @@ is stale or expired). Parent caching If the request is a cache miss on the parent, then the parent retrieves the -content from the origin server (or from another cache, depending on the parent’s +content from the origin server (or from another cache, depending on the parent's configuration). The parent caches the content and then sends a copy to Traffic Server (its child), where it is cached and served to the client. @@ -106,6 +106,11 @@ host name. This is true regardless of pristine host headers (:ts:cv:`proxy.config.url_remap.pristine_host_hdr`) being enabled or not. The parent node will receive the translated request (and thus needs to be configured to accept it). +:file:`remap.config` now also allows an alternative configuration that supports all the +**Parent Selection** policies and failover mentioned here using a new :file:`remap.config` tag, +**@strategy**. This eliminates the need for a second lookup against the remapped host name +required when using :file:`parent.config`. See using NextHop strategies with :file:`remap.config`, +:doc:`../../admin-guide/files/strategies.yaml.en` Example ~~~~~~~ @@ -165,7 +170,11 @@ the configuration adjustments detailed below. cache. #. Edit :file:`parent.config` to set parent proxy rules which will specify the - parent cache to which you want missed requests to be forwarded. + parent cache to which you want missed requests to be forwarded. Or as an + alternative to :file:`parent.config`, edit :file:`strategies.yaml` to + specify next hop parent proxy rules and hosts. Then in :file:`remap.config` + use the **@strategy** tag to select the next hop parent proxy rules. See + :doc:`../../admin-guide/files/strategies.yaml.en` The following example configures Traffic Server to route all requests containing the regular expression ``politics`` and the path diff --git a/doc/admin-guide/configuration/proxy-protocol.en.rst b/doc/admin-guide/configuration/proxy-protocol.en.rst index c857de9bf31..8df27d1eed0 100644 --- a/doc/admin-guide/configuration/proxy-protocol.en.rst +++ b/doc/admin-guide/configuration/proxy-protocol.en.rst @@ -31,7 +31,7 @@ TLS connections. .. note:: - The current version only supports transforming client IP from PROXY Version 1 + The current version only supports transforming client IP from PROXY Version 1 header to the Forwarded: header. In the current implementation, the client IP address in the PROXY protocol header @@ -44,12 +44,12 @@ Proxy Protocol on a port. Once enabled, all incoming requests must be prefaced with the PROXY v1 header. Any request not preface by this header will be dropped. -As a security measure, an optional whitelist of trusted IP addresses may be -configured with :ts:cv:`proxy.config.http.proxy_protocol_whitelist`. +As a security measure, an optional list of trusted IP addresses may be +configured with :ts:cv:`proxy.config.http.proxy_protocol_allowlist`. .. important:: - If the whitelist is configured, requests will only be accepted from these + If the allowlist is configured, requests will only be accepted from these IP addresses and must be prefaced with the PROXY v1 header. See :ts:cv:`proxy.config.http.insert_forwarded` for configuration information. diff --git a/doc/admin-guide/configuration/redirecting-http-requests.en.rst b/doc/admin-guide/configuration/redirecting-http-requests.en.rst index 0bc8d8c197f..defb77c44da 100644 --- a/doc/admin-guide/configuration/redirecting-http-requests.en.rst +++ b/doc/admin-guide/configuration/redirecting-http-requests.en.rst @@ -116,7 +116,7 @@ server. For additional information, see `HTTP Reverse Proxy`_. .. note:: - To avoid a DNS conflict, the origin server’s hostname and its advertised + To avoid a DNS conflict, the origin server's hostname and its advertised hostname must not be the same. HTTP Reverse Proxy @@ -138,7 +138,7 @@ The figure above demonstrates the following steps: 1. A client browser sends an HTTP request addressed to a host called ``www.host.com`` on port 80. Traffic Server receives the request - because it is acting as the origin server (the origin server’s + because it is acting as the origin server (the origin server's advertised hostname resolves to Traffic Server). 2. Traffic Server locates a map rule in the :file:`remap.config` file and diff --git a/doc/admin-guide/configuration/transparent-proxy/build.en.rst b/doc/admin-guide/configuration/transparent-proxy/build.en.rst index 10337a3a7b7..0f9dbf2f405 100644 --- a/doc/admin-guide/configuration/transparent-proxy/build.en.rst +++ b/doc/admin-guide/configuration/transparent-proxy/build.en.rst @@ -37,7 +37,7 @@ may need to twiddle the ``configure`` options. Enable TPROXY support, which is the Linux kernel feature used for transparency. This should be present in the base installation, there is no package associated with it. \* ``auto`` Do automatic checks - for the the TPROXY header file (``linux/in.h``) and enable TPROXY + for the TPROXY header file (``linux/in.h``) and enable TPROXY support if the ``IP_TRANSPARENT`` definition is present. This is the default if this option is not specified or ``value`` is omitted. \* ``no`` Do not check for TPROXY support, disable support for it. \* diff --git a/doc/admin-guide/configuration/transparent-proxy/wccp-service-config.en.rst b/doc/admin-guide/configuration/transparent-proxy/wccp-service-config.en.rst index cd841b5b561..42186cba1e2 100644 --- a/doc/admin-guide/configuration/transparent-proxy/wccp-service-config.en.rst +++ b/doc/admin-guide/configuration/transparent-proxy/wccp-service-config.en.rst @@ -59,7 +59,7 @@ Service group attributes include * id - The security group ID. It must match the service group ID that has been defined on the associated WCCP router. This is the true service group identifier from the WCCP perspective. -* type – This defines the type of service group either “STANDARD” or “DYNAMIC”. There is one standard defined service group, HTTP with the id of 0. The 4/2001 RFC indicates that id’s 0-50 are reserved for well known service groups. But more recent 8/2012 RFC indicates that values 0 through 254 are valid service id’s for dynamic services. To avoid differences with older WCCP routers, you probably want to avoid dynamic service ID’s 0 through 50. +* type – This defines the type of service group either "STANDARD" or "DYNAMIC". There is one standard defined service group, HTTP with the id of 0. The 4/2001 RFC indicates that id's 0-50 are reserved for well known service groups. But more recent 8/2012 RFC indicates that values 0 through 254 are valid service id's for dynamic services. To avoid differences with older WCCP routers, you probably want to avoid dynamic service ID's 0 through 50. * priority – This is a value from 0 to 255. The higher number is a higher priority. Well known (STANDARD) services are set to a value of 240. If there are multiple service groups that could match a given packet, the higher priority service group is applied. RFC For example, you have service group 100 defined for packets with destination port 80, and service group 101 defined for packets with source port 1024. For a packet with destination port set to 80 and source port set to 1024, the priorities of the service groups would need to be compared to determine which service group applies. @@ -80,5 +80,5 @@ Service group attributes include * routers – This is the list of router addresses the WCCP client communicates with. The WCCP protocols allows for multiple WCCP routers to be involved in a service group. The multiple router scenario has at most been lightly tested in the Traffic Server implementation. -* proc-name – This attribute is only used by traffic_wccp. It is not used in the traffic_server WCCP support. This is the path to a process’ PID file. The service group is advertised to the WCCP router if the process identified in the PID file is currently operational. +* proc-name – This attribute is only used by traffic_wccp. It is not used in the traffic_server WCCP support. This is the path to a process' PID file. The service group is advertised to the WCCP router if the process identified in the PID file is currently operational. diff --git a/doc/admin-guide/files/cache.config.en.rst b/doc/admin-guide/files/cache.config.en.rst index 4662992315f..c75fcaf72f9 100644 --- a/doc/admin-guide/files/cache.config.en.rst +++ b/doc/admin-guide/files/cache.config.en.rst @@ -198,7 +198,7 @@ specifiers of the rule in question. .. _cache-config-format-revalidate: ``revalidate`` - For objects that are in cache, overrides the the amount of time the object(s) + For objects that are in cache, overrides the amount of time the object(s) are to be considered fresh. Use the same time formats as ``pin-in-cache``. .. _cache-config-format-ttl-in-cache: diff --git a/doc/admin-guide/files/index.en.rst b/doc/admin-guide/files/index.en.rst index caeb584d139..179c68e1866 100644 --- a/doc/admin-guide/files/index.en.rst +++ b/doc/admin-guide/files/index.en.rst @@ -37,6 +37,7 @@ Configuration Files ssl_multicert.config.en sni.yaml.en storage.config.en + strategies.yaml.en volume.config.en :doc:`cache.config.en` @@ -80,5 +81,8 @@ Configuration Files :doc:`storage.config.en` Configures all storage devices and paths to be used for the |TS| cache. +:doc:`strategies.yaml.en` + Configures NextHop strategies used with `remap.config` + :doc:`volume.config.en` Defines cache space usage by individual protocols. diff --git a/doc/admin-guide/files/ip_allow.yaml.en.rst b/doc/admin-guide/files/ip_allow.yaml.en.rst index e8e98ba2879..4c4553cb112 100644 --- a/doc/admin-guide/files/ip_allow.yaml.en.rst +++ b/doc/admin-guide/files/ip_allow.yaml.en.rst @@ -21,7 +21,7 @@ ip_allow.yaml .. configfile:: ip_allow.yaml The :file:`ip_allow.yaml` file controls client access to |TS| and |TS| connections to upstream servers. -This control is specified rules. Each rule has +This control is specified via rules. Each rule has: * A direction (inbound or out). * A range of IP address to which the rule applies. @@ -69,7 +69,7 @@ Format Each rule is a mapping. The YAML data must have a top level key of "ip_allow" and its value must be a mapping or a sequence of mappings, each of those being one rule. -The keys in a rule are +The keys in a rule are: ``apply`` This is where the rule is applied, either ``in`` or ``out``. Inbound application means @@ -85,7 +85,7 @@ The keys in a rule are ``methods`` This is optional. If not present, the rule action applies to all methods. If present, the rule - action is applied connections using those methods and its opposite to all other connections. The + action is applied to connections using those methods and its opposite to all other connections. The keyword "ALL" means all methods, making the specification of any other method redundant. All methods comparisons are case insensitive. This is an optional key. @@ -97,14 +97,14 @@ allowed to have a range that contains addresses from different IP address famili * A CIDR based value, e.g. "10.1.0.0/16", which is a range containing exactly the specified network. A rule must have the ``apply``, ``ip_addrs``, and ``action`` keys. Rules match based on -IP addresses only, and are then applied to all matching sessions. If the rule is an ``allow`` rule, +IP addresses only and are then applied to all matching sessions. If the rule is an ``allow`` rule, the specified methods are allowed and all other methods are denied. If the rule is a ``deny`` rule, the specified methods are denied and all other methods are allowed. For example, from the default configuration, the rule for ``127.0.0.1`` is ``allow`` with all methods. Therefore an inbound connection from the loopback address (127.0.0.1) is allowed to use any method. The general IPv4 rule, covering all IPv4 address, is a ``deny`` rule and therefore when it -matches the methods "PURGE", "PUSH", and "DELETE" are denied, any other method is allowed. +matches the methods "PURGE", "PUSH", and "DELETE", these methods are denied and any other method is allowed. The rules are matched in order, by IP address, therefore the general IPv4 rule does not apply to the loopback address because the latter is matched first. @@ -114,6 +114,15 @@ inbound connections are denied and therefore if there is no rule that matches, t denied. Outbound rules allow by default, so the absence of rules in the default configuration enables all methods for all outbound connections. +.. note:: + + Be aware that ip_allow rules will not, and indeed cannot, be applied to TLS + connections which are tunneled via ``tunnel_route`` to the upstream target. + Such connections are not decrypted and thus are not processed by |TS|. This + applies as well to TLS connections which are forwarded via ``forward_route`` + since, while those are decrypted, they are not processed by |TS|. For + details, see :ref:`sni-routing` and :file:`sni.yaml`. + Examples ======== diff --git a/doc/admin-guide/files/logging.yaml.en.rst b/doc/admin-guide/files/logging.yaml.en.rst index f4d69373f2f..d7e076add49 100644 --- a/doc/admin-guide/files/logging.yaml.en.rst +++ b/doc/admin-guide/files/logging.yaml.en.rst @@ -325,7 +325,7 @@ common fields: formats: - name: minimalfmt - format: '% : % : %' + format: '% , % , %' The following is an example of a format that uses aggregate operators to produce a summary log: @@ -334,7 +334,7 @@ produce a summary log: formats: - name: summaryfmt - format: '% : % : %' + format: '%:%:%' interval: 10 The following is an example of a filter that will cause only REFRESH_HIT events diff --git a/doc/admin-guide/files/parent.config.en.rst b/doc/admin-guide/files/parent.config.en.rst index b47bf4267bb..39d33e36543 100644 --- a/doc/admin-guide/files/parent.config.en.rst +++ b/doc/admin-guide/files/parent.config.en.rst @@ -124,7 +124,7 @@ The following list shows the possible actions and their allowed values. .. _parent-config-format-parent: -``parent`` `(hostname or IP address):port[|weight][,another host]` +``parent`` `(hostname or IP address):port[|weight][&hash name][,another host]` An ordered list of parent servers, separated by commas or semicolons. If the request cannot be handled by the last parent server in the list, then it will be routed to the origin server. @@ -141,7 +141,7 @@ The following list shows the possible actions and their allowed values. parent="p1.x.com:8080|2.0, 192.168.0.3:80|3.0, 192.168.0.4:80|5.0" - If ``round_robin`` is set to ``consistent_hash``, you may add a ``unique hash string`` + If ``round_robin`` is set to ``consistent_hash``, you may add a unique ``hash string`` following the ``weight`` for each parent. The ``hash string`` must start with ``&`` and is used to build both the primary and secondary rings using the ``hash string`` for each parent instead of the parents ``hostname`` or ``ip address``. This can be @@ -185,7 +185,13 @@ The following list shows the possible actions and their allowed values. - If the chosen parent is marked down then another parent will be chosen from the ``secondary_parent`` list. The ``secondary_parent`` list will be exhausted before attempting - to choose another parent in the ``parent`` list. + to choose another parent in the ``parent`` list. This depends + on taking a parent down from a particular EDGE using traffic_ctl + like ``traffic_ctl host down sample.server.com``. This will be + useful during maintenance window or as a debugging aid when a + user wants to take down specific parents. Taking parents down + using ``traffic_ctl`` will cause the EDGE to ignore those parent + immediately from parent selection logic. - If the chosen parent is unavailable but not marked down then another parent will be chosen from the ``parent`` list. The @@ -207,7 +213,8 @@ The following list shows the possible actions and their allowed values. .. _parent-config-format-parent-retry: ``parent_retry`` - - ``simple_retry`` - If the parent origin server returns a 404 response on a request + - ``simple_retry`` - If the parent returns a 404 response or if the response matches + a list of http 4xx responses defined in ``simple_server_retry_responses`` on a request a new parent is selected and the request is retried. The number of retries is controlled by ``max_simple_retries`` which is set to 1 by default. - ``unavailable_server_retry`` - If the parent returns a 503 response or if the response matches @@ -216,6 +223,13 @@ The following list shows the possible actions and their allowed values. retries is controlled by ``max_unavailable_server_retries`` which is set to 1 by default. - ``both`` - This enables both ``simple_retry`` and ``unavailable_server_retry`` as described above. +.. _parent-config-format-simple-server-retry-responses: + +``simple_server_retry_responses`` + If ``parent_retry`` is set to either ``simple_retry`` or ``both``, this parameter is a comma separated list of + http 4xx response codes that will invoke the ``simple_retry`` described in the ``parent_retry`` section. By + default, ``simple_server_retry_responses`` is set to 404. + .. _parent-config-format-unavailable-server-retry-responses: ``unavailable_server_retry_responses`` diff --git a/doc/admin-guide/files/records.config.en.rst b/doc/admin-guide/files/records.config.en.rst index 05bdbd2f2ec..96ba3bc6cd7 100644 --- a/doc/admin-guide/files/records.config.en.rst +++ b/doc/admin-guide/files/records.config.en.rst @@ -193,7 +193,7 @@ System Variables .. ts:cv:: CONFIG proxy.config.proxy_binary_opts STRING -M - The command-line options for starting |TS|. + The :ref:`command-line options ` for starting |TS|. .. ts:cv:: CONFIG proxy.config.manager_binary STRING traffic_manager @@ -273,30 +273,6 @@ System Variables order of auto-deletion (if enabled). A default value of 0 means auto-deletion will try to keep output logs as much as possible. See :doc:`../logging/rotation.en` for guidance. -.. ts:cv:: CONFIG proxy.config.output.logfile.rolling_max_count INT 0 - :reloadable: - - Specifies the maximum count of rolled output logs to keep. This value will be used by the - auto-deletion (if enabled) to trim the number of rolled log files every time the log is rolled. - A default value of 0 means auto-deletion will not try to limit the number of output logs. - See :doc:`../logging/rotation.en` for an use-case for this option. - -.. ts:cv:: CONFIG proxy.config.output.logfile.rolling_allow_empty INT 0 - :reloadable: - - While rolling default behavior is to rename, close and re-open the log file *only* when/if there is - something to log to the log file. This option opens a new log file right after rolling even if there - is nothing to log (i.e. nothing to be logged due to lack of requests to the server) - which may lead to 0-sized log files while rolling. See :doc:`../logging/rotation.en` for an use-case - for this option. - - ===== ====================================================================== - Value Description - ===== ====================================================================== - ``0`` No empty log files created and rolled if there was nothing to log - ``1`` Allow empty log files to be created and rolled even if there was nothing to log - ===== ====================================================================== - Thread Variables ---------------- @@ -306,22 +282,36 @@ Thread Variables When enabled (the default, ``1``), |TS| scales threads according to the available CPU cores. See the config option below. -.. ts:cv:: CONFIG proxy.config.exec_thread.autoconfig.scale FLOAT 1.5 +.. ts:cv:: CONFIG proxy.config.exec_thread.autoconfig.scale FLOAT 1.0 Factor by which |TS| scales the number of threads. The multiplier is usually the number of available CPU cores. By default this is scaling factor is - ``1.5``. + ``1.0``. .. ts:cv:: CONFIG proxy.config.exec_thread.limit INT 2 The number of threads |TS| will create if `proxy.config.exec_thread.autoconfig` is set to ``0``, otherwise this option is ignored. +.. ts:cv:: CONFIG proxy.config.exec_thread.listen INT 0 + + If enabled (``1``) all the exec_threads listen for incoming connections. `proxy.config.accept_threads` + should be disabled to enable this variable. + .. ts:cv:: CONFIG proxy.config.accept_threads INT 1 The number of accept threads. If disabled (``0``), then accepts will be done in each of the worker threads. + ==================== ====================== ===================== + accept_threads exec_thread.listen Effect + ==================== ====================== ===================== + ``0`` ``0`` All worker threads accept new connections and share listen fd. + ``1`` ``0`` New connections are accepted on a dedicated accept thread and distributed to worker threads in round robin fashion. + ``0`` ``1`` All worker threads listen on the same port using SO_REUSEPORT. Each thread has its own listen fd and new connections are accepted on all the threads. + ==================== ====================== ===================== + + By default, `proxy.config.accept_threads` is set to 1 and `proxy.config.exec_thread.listen` is set to 0. .. ts:cv:: CONFIG proxy.config.thread.default.stacksize INT 1048576 Default thread stack size, in bytes, for all threads (default is 1 MB). @@ -346,7 +336,7 @@ Thread Variables .. ts:cv:: CONFIG proxy.config.system.file_max_pct FLOAT 0.9 - Set the maximum number of file handles for the traffic_server process as a percentage of the the fs.file-max proc value in Linux. The default is 90%. + Set the maximum number of file handles for the traffic_server process as a percentage of the fs.file-max proc value in Linux. The default is 90%. .. ts:cv:: CONFIG proxy.config.crash_log_helper STRING traffic_crashlog @@ -391,8 +381,11 @@ Thread Variables :reloadable: The shutdown timeout(in seconds) to apply when stopping Traffic - Server, in which ATS can initiate graceful shutdowns. It only supports - HTTP/2 graceful shutdown for now. Stopping |TS| here means sending + Server, in which ATS can initiate graceful shutdowns. In order + to effect graceful shutdown, the value specified should be greater + than 0. Value of 0 will not effect an abrupt shutdown. Abrupt + shutdowns can be achieved with out specifying --drain; + (traffic_ctl server stop /restart). Stopping |TS| here means sending `traffic_server` a signal either by `bin/trafficserver stop` or `kill`. .. ts:cv:: CONFIG proxy.config.thread.max_heartbeat_mseconds INT 60 @@ -419,6 +412,23 @@ Network handled. This should be tuned according to your memory size, and expected work load. If this is set to 0, the throttling logic is disabled. +.. ts:cv:: CONFIG proxy.config.net.max_connections_in INT 30000 + + The total number of client requests that |TS| can handle simultaneously. + This should be tuned according to your memory size, and expected work load + (network, cpu etc). This limit includes both idle (keep alive) connections + and active requests that |TS| can handle at any given instant. The delta + between `proxy.config.net.max_connections_in` and `proxy.config.net.max_requests_in` + is the amount of maximum idle (keepalive) connections |TS| will maintain. + +.. ts:cv:: CONFIG proxy.config.net.max_requests_in INT 0 + + The total number of concurrent requests or active client connections + that the |TS| can handle simultaneously. This should be tuned according + to your memory size, and expected work load (network, cpu etc). When + set to 0, active request tracking is disabled and max requests has no + separate limit and the total connections follow `proxy.config.net.connections_throttle` + .. ts:cv:: CONFIG proxy.config.net.default_inactivity_timeout INT 86400 :reloadable: @@ -527,10 +537,10 @@ Local Manager process as, which also has the effect of setting ownership of configuration and log files. - As of version 2.1.1 if the user_id is prefixed with pound character (``#``) + If the user_id is prefixed with pound character (``#``), the remainder of the string is considered to be a `numeric user identifier `_. - If the value is set to ``#-1`` |TS| will not change the user during startup. + If the value is set to ``#-1``, |TS| will not change the user during startup. .. important:: @@ -923,32 +933,29 @@ mptcp :overridable: Enable and set the ability to re-use server connections across client - connections. The valid values are: - - ======== =================================================================== - Value Description - ======== =================================================================== - ``none`` Do not match and do not re-use server sessions. If using this in - :ref:`ts-overridable-config` (like the :ref:`admin-plugins-conf-remap`), - use the integer ``0`` instead. - ``both`` Re-use server sessions, if *both* the IP address and fully qualified - domain name match. If using this in :ref:`ts-overridable-config` (like - the :ref:`admin-plugins-conf-remap`), use the integer ``1`` instead. - ``ip`` Re-use server sessions, checking only that the IP address and port - of the origin server matches. If using this in - :ref:`ts-overridable-config` (like the :ref:`admin-plugins-conf-remap`), - use the integer ``2`` instead. - ``host`` Re-use server sessions, checking only that the fully qualified - domain name matches. If using this in :ref:`ts-overridable-config` - (like the :ref:`admin-plugins-conf-remap`), use the integer ``3`` instead. - ======== =================================================================== - - It is strongly recommended to use either ``none`` or ``both`` for this value - unless you have a specific need for the other settings. The most common - reason is virtual hosts that share an IP address in which case performance - can be enhanced if those sessions can be re-used. However, not all web - servers support requests for different virtual hosts on the same connection - so use with caution. + connections. Multiple values can be specified when separated by commas with no white spaces. Valid values are: + + ============= =================================================================== + Value Description + ============= =================================================================== + ``none`` Do not match and do not re-use server sessions. + ``ip`` Re-use server sessions, checking only that the IP address and port + of the origin server matches. + ``host`` Re-use server sessions, checking that the fully qualified + domain name matches. In addition, if the session uses TLS, it also + checks that the current transaction's host header value matchs the session's SNI. + ``both`` Equivalent to ``host,ip``. + ``hostonly`` Check that the fully qualified domain name matches. + ``sni`` Check that the SNI of the session matches the SNI that would be used to + create a new session. Only applicable for TLS sessions. + ``cert`` Check that the certificate file name used for the server session matches the + certificate file name that would be used for the new server session. Only + applicable for TLS sessions. + ============= =================================================================== + + The setting must contain at least one of ``ip``, ``host``, ``hostonly`` or ``both`` + for session reuse to operate. The other values may be used for greater control + with TLS session reuse. .. note:: @@ -1091,11 +1098,10 @@ mptcp .. ts:cv:: CONFIG proxy.config.http.request_buffer_enabled INT 0 :overridable: - This is a configuration value that is overridable but not configurable. This is most likely an - implementation error. - This enables buffering the content for incoming ``POST`` requests. If enabled no outbound connection is made until the entire ``POST`` request has been buffered. + If enabled, `proxy.config.http.post_copy_size` needs to be set to the maximum of the post body + size allowed, otherwise, the post would fail. .. ts:cv:: CONFIG proxy.config.http.request_line_max_size INT 65535 @@ -1228,8 +1234,8 @@ HTTP Connection Timeouts :overridable: Specifies how long |TS| keeps connections to clients open for a - subsequent request after a transaction ends. A value of ``0`` will disable - the no activity timeout. + subsequent request after a transaction ends. A value of ``0`` will set + `proxy.config.net.default_inactivity_timeout` as the timeout. See :ref:`admin-performance-timeouts` for more discussion on |TS| timeouts. @@ -1238,8 +1244,8 @@ HTTP Connection Timeouts :overridable: Specifies how long |TS| keeps connections to origin servers open - for a subsequent transfer of data after a transaction ends. A value of - ``0`` will disable the no activity timeout. + for a subsequent transfer of data after a transaction ends. A value of ``0`` will + set `proxy.config.net.default_inactivity_timeout` as the timeout. See :ref:`admin-performance-timeouts` for more discussion on |TS| timeouts. @@ -1332,6 +1338,8 @@ HTTP Redirection .. note:: When :ts:cv:`proxy.config.http.number_of_redirections` is set to a positive value and |TS| has previously cached a 3XX Redirect response, the cached response will continue to be refreshed and returned until the response is no longer in the cache. + .. note:: In previous versions proxy.config.http.redirection_enabled had to be set to 1 before this setting was evaluated. Now setting :ts:cv:`proxy.config.http.number_of_redirections` to a value greater than zero is sufficient to cause |TS| to follow redirects. + .. ts:cv:: CONFIG proxy.config.http.redirect_host_no_port INT 1 :reloadable: @@ -1528,8 +1536,8 @@ Origin Server Connect Attempts :reloadable: :overridable: - The timeout value (in seconds) for time to first byte for an origin server - connection. + The timeout value (in seconds) for time to set up a connection to the origin. After the connection is established the value of + ``proxy.config.http.transaction_no_activity_timeout_out`` is used to established timeouts on the data over the connection. See :ref:`admin-performance-timeouts` for more discussion on |TS| timeouts. @@ -1747,9 +1755,9 @@ Proxy User Variables is prohibited by RFC 7239. Currently, for the ``host`` parameter to provide the original host from the incoming client request, `proxy.config.url_remap.pristine_host_hdr`_ must be enabled. -.. ts:cv:: CONFIG proxy.config.http.proxy_protocol_whitelist STRING `````` +.. ts:cv:: CONFIG proxy.config.http.proxy_protocol_allowlist STRING `````` - This defines a whitelist of server IPs that are trusted to provide + This defines a allowlist of server IPs that are trusted to provide connections with Proxy Protocol information. This is a comma delimited list of IP addresses. Addressed may be listed individually, in a range separated by a dash or by using CIDR notation. @@ -1829,6 +1837,26 @@ Security ``2`` Similar to 0, except return a 416 error code and no response body. ===== ====================================================================== +.. ts:cv:: CONFIG proxy.config.http.host_sni_policy INT 2 + + This option controls how host header and SNI name mismatches are handled. Mismatches + may result in SNI-based policies defined in :file:`sni.yaml` being avoided. For example, ``foo.com`` + may be the fqdn value in :file:`sni.yaml` which defines that client certificates are required. + The user could specify ``bar.com`` as the SNI to avoid the policy requiring the client certificate + but specify ``foo.com`` as the HTTP host header to still access the same object. + + Therefore, if a host header would have triggered a SNI policy, it is possible that the user is + trying to bypass a SNI policy if the host header and SNI values do not match. + + If this setting is 0, no checking is performed. If this setting is 1 or 2, the host header and SNI values + are compared if the host header value would have triggered a SNI policy. If there is a mismatch and the value + is 1, a warning is generated but the transaction is allowed to proceed. If the value is 2 and there is a + mismatch, a warning is generated and a status 403 is returned. + + You can override this global setting on a per domain basis in the :file:`sni.yaml` file using the :ref:`host_sni_policy attribute` action. + + Currently, only the verify_client policy is checked for host name and SNI matching. + Cache Control ============= @@ -1877,7 +1905,7 @@ Cache Control Forces the use of a specific hardware sector size, e.g. 4096, for all disks. - SSDs and "advanced format” drives claim a sector size of 512; however, it is safe to force a higher + SSDs and "advanced format" drives claim a sector size of 512; however, it is safe to force a higher size than the hardware supports natively as we count atomicity in 512 byte increments. 4096-sized drives formatted for Windows will have partitions aligned on 63 512-byte sector boundaries, @@ -1901,6 +1929,12 @@ Cache Control Enables (``1``) or disables (``0``) caching of HTTP requests. +.. ts:cv:: CONFIG proxy.config.http.cache.post_method INT 0 + :reloadable: + :overridable: + + Enables (``1``) or disables (``0``) caching of HTTP POST requests. + .. ts:cv:: CONFIG proxy.config.http.cache.generation INT -1 :reloadable: :overridable: @@ -2028,6 +2062,20 @@ Cache Control The maximum age allowed for a stale response before it cannot be cached. +.. ts:cv:: CONFIG proxy.config.http.cache.guaranteed_min_lifetime INT 0 + :reloadable: + :overridable: + + Establishes a guaranteed minimum lifetime boundary for object freshness. + Setting this to ``0`` (default) disables the feature. + +.. ts:cv:: CONFIG proxy.config.http.cache.guaranteed_max_lifetime INT 31536000 + :reloadable: + :overridable: + + Establishes a guaranteed maximum lifetime boundary for object freshness. + Setting this to ``0`` disables the feature. + .. ts:cv:: CONFIG proxy.config.http.cache.range.lookup INT 1 :overridable: @@ -2249,24 +2297,6 @@ Heuristic Expiration The aging factor for freshness computations. |TS| stores an object for this percentage of the time that elapsed since it last changed. -.. ts:cv:: CONFIG proxy.config.http.cache.guaranteed_min_lifetime INT 0 - :reloadable: - :overridable: - - Establishes a guaranteed minimum lifetime boundary for freshness heuristics. - When heuristics are used, and the :ts:cv:`proxy.config.http.cache.heuristic_lm_factor` - aging factor is applied, the final minimum age calculated will never be - lower than the value in this variable. - -.. ts:cv:: CONFIG proxy.config.http.cache.guaranteed_max_lifetime INT 31536000 - :reloadable: - :overridable: - - Establishes a guaranteed maximum lifetime boundary for freshness heuristics. - When heuristics are used, and the :ts:cv:`proxy.config.http.cache.heuristic_lm_factor` - aging factor is applied, the final maximum age calculated will never be - higher than the value in this variable. - Dynamic Content & Content Negotiation ===================================== @@ -2288,6 +2318,9 @@ Dynamic Content & Content Negotiation The number of times to attempt a cache open write upon failure to get a write lock. + This config is ignored when :ts:cv:`proxy.config.http.cache.open_write_fail_action` is + set to ``5``. + .. ts:cv:: CONFIG proxy.config.http.cache.open_write_fail_action INT 0 :reloadable: :overridable: @@ -2310,6 +2343,12 @@ Dynamic Content & Content Negotiation :ts:cv:`proxy.config.http.cache.max_stale_age`. Otherwise, go to origin server. ``4`` Return a ``502`` error on either a cache miss or on a revalidation. + ``5`` Retry Cache Read on a Cache Write Lock failure. This option together + with :ts:cv:`proxy.config.cache.enable_read_while_writer` configuration + allows to collapse concurrent requests without a need for any plugin. + Make sure to configure the :ref:`admin-config-read-while-writer` feature + correctly. Note that this option may result in CACHE_LOOKUP_COMPLETE HOOK + being called back more than once. ===== ====================================================================== Customizable User Response Pages @@ -2361,7 +2400,7 @@ Customizable User Response Pages ===== ====================================================================== ``0`` Never suppress generated response pages. ``1`` Always suppress generated response pages. - ``2`` Suppress response pages only for intercepted traffic. + ``2`` Suppress response pages only for internal traffic. ===== ====================================================================== .. ts:cv:: CONFIG proxy.config.http_ui_enabled INT 0 @@ -2437,7 +2476,8 @@ DNS Allows one to specify which ``resolv.conf`` file to use for finding resolvers. While the format of this file must be the same as the standard ``resolv.conf`` file, this option allows an administrator to manage the set of resolvers in an external configuration file, - without affecting how the rest of the operating system uses DNS. + without affecting how the rest of the operating system uses DNS. Note that this setting works in conjunction with + :ts:cv:`proxy.config.dns.nameservers`, with its settings appended to the ``resolv.conf`` contents. .. ts:cv:: CONFIG proxy.config.dns.round_robin_nameservers INT 1 :reloadable: @@ -2447,7 +2487,10 @@ DNS .. ts:cv:: CONFIG proxy.config.dns.nameservers STRING NULL :reloadable: - The DNS servers. + The DNS servers. Note that this does not override :ts:cv:`proxy.config.dns.resolv_conf`. + That is, the contents of the file listed in :ts:cv:`proxy.config.dns.resolv_conf` will + be appended to the list of nameservers specified here. To prevent this, a bogus file + can be listed there. .. ts:cv:: CONFIG proxy.config.srv_enabled INT 0 :reloadable: @@ -2490,6 +2533,18 @@ DNS ``2`` TCP_ONLY: |TS| always talks to nameservers over TCP. ===== ====================================================================== +.. ts:cv:: CONFIG proxy.config.dns.max_dns_in_flight INT 2048 + + Maximum inflight DNS queries made by |TS| at any given instant + +.. ts:cv:: CONFIG proxy.config.dns.lookup_timeout INT 20 + + Time to wait for a DNS response in seconds. + +.. ts:cv:: CONFIG proxy.config.dns.retries INT 5 + + Maximum number of retries made by |TS| on a given DNS query + HostDB ====== @@ -2620,6 +2675,7 @@ HostDB of partitions .. ts:cv:: CONFIG proxy.config.hostdb.ip_resolve STRING NULL + :overridable: Set the host resolution style. @@ -2826,6 +2882,30 @@ Logging Configuration order of auto-deletion (if enabled). A default value of 0 means auto-deletion will try to keep logs as much as possible. This value can be and should be overridden in logging.yaml. See :doc:`../logging/rotation.en` for guidance. +.. ts:cv:: CONFIG proxy.config.log.rolling_max_count INT 0 + :reloadable: + + Specifies the maximum count of rolled output logs to keep. This value will be used by the + auto-deletion (if enabled) to trim the number of rolled log files every time the log is rolled. + A default value of 0 means auto-deletion will not try to limit the number of output logs. + See :doc:`../logging/rotation.en` for an use-case for this option. + +.. ts:cv:: CONFIG proxy.config.log.rolling_allow_empty INT 0 + :reloadable: + + While rolling default behavior is to rename, close and re-open the log file *only* when/if there is + something to log to the log file. This option opens a new log file right after rolling even if there + is nothing to log (i.e. nothing to be logged due to lack of requests to the server) + which may lead to 0-sized log files while rolling. See :doc:`../logging/rotation.en` for an use-case + for this option. + + ===== ====================================================================== + Value Description + ===== ====================================================================== + ``0`` No empty log files created and rolled if there was nothing to log + ``1`` Allow empty log files to be created and rolled even if there was nothing to log + ===== ====================================================================== + .. ts:cv:: CONFIG proxy.config.log.auto_delete_rolled_files INT 1 :reloadable: @@ -2882,11 +2962,49 @@ Logging Configuration .. ts:cv:: CONFIG proxy.config.log.config.filename STRING logging.yaml :reloadable: + :deprecated: This configuration value specifies the path to the :file:`logging.yaml` configuration file. If this is a relative path, |TS| loads it relative to the ``SYSCONFDIR`` directory. +.. ts:cv:: CONFIG proxy.config.log.max_line_size INT 9216 + :units: bytes + + This controls the maximum line length for ``ASCII`` formatted log entries. + This applies to ``ASCII_PIPE`` and ``ASCII`` file logs, *unless* + :ts:cv:`proxy.config.log.ascii_buffer_size` is also specified and the value + of ``ascii_buffer_size`` is larger than ``max_line_size``: in that case, + ``max_line_size`` only applies to ``ASCII_PIPE`` logs while + ``ascii_buffer_size`` will apply to ``ASCII`` (non-pipe) log files. + +.. ts:cv:: CONFIG proxy.config.log.ascii_buffer_size INT 36864 + :units: bytes + + This controls the maximum line length for ``ASCII`` formatted log entries + that are non-pipe log files. If this value is smaller than + :ts:cv:`proxy.config.log.max_line_size`, then the latter will be used for + both ``ASCII`` and ``ASCII_PIPE`` log files. If both ``max_line_size`` and + ``ascii_buffer_size`` are set, then ``max_line_size`` will be used for + ``ASCII_PIPE`` logs while ``ascii_buffer_size`` will be used for ``ASCII`` + (non-pipe) log files. This all might seem complicated, but just keep in + mind that the intention of ``ascii_buffer_size`` is to simply provide a way + for the user to configure different ``ASCII`` and ``ASCII_PIPE`` maximum + line lengths. + +.. ts:cv:: CONFIG proxy.config.log.log_buffer_size INT 9216 + :reloadable: + :units: bytes + + This is an orthogonal mechanism from :ts:cv:`proxy.config.log.max_line_size` + and :ts:cv:`proxy.config.log.ascii_buffer_size` for limiting line length + size by constraining the log entry buffer to a particular amount of memory. + Unlike the above two configurations, ``log_buffer_size`` applies to both + binary and ``ASCII`` log file entries. For ``ASCII`` log files, if a maximum + log size is set via both the above mechanisms and by ``log_buffer_size``, + then the smaller of the two configurations will be applied to the line + length. + Diagnostic Logging Configuration ================================ @@ -3033,6 +3151,7 @@ URL Remap Rules =============== .. ts:cv:: CONFIG proxy.config.url_remap.filename STRING remap.config + :deprecated: Sets the name of the :file:`remap.config` file. @@ -3064,14 +3183,15 @@ SSL Termination formatting cipher_suite string, see `OpenSSL Ciphers `_. - The current default, included in the ``records.config.default`` example - configuration is: + The current default is: - ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:DHE-DSS-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-DSS-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES128-SHA:DHE-RSA-AES256-SHA256:DHE-DSS-AES256-SHA256:DHE-RSA-AES128-SHA256:DHE-DSS-AES128-SHA256:DHE-RSA-AES256-SHA:DHE-DSS-AES256-SHA:DHE-RSA-AES128-SHA:DHE-DSS-AES128-SHA:AES256-GCM-SHA384:AES128-GCM-SHA256:AES256-SHA256:AES128-SHA256:AES256-SHA:AES128-SHA:DES-CBC3-SHA:!aNULL:!eNULL:!EXPORT:!DES:!RC4:!MD5:!PSK:!aECDH:!EDH-DSS-DES-CBC3-SHA:!EDH-RSA-DES-CBC3-SHA:!KRB5-DES-CBC3-SHA + ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-CCM:ECDHE-ECDSA-AES128-CCM:ECDHE-ECDSA-AES256-CCM8:ECDHE-ECDSA-AES128-CCM8:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES256-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-RSA-AES256-SHA384:ECDHE-RSA-AES128-SHA256:DHE-RSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-CCM8:DHE-RSA-AES128-CCM8:DHE-RSA-AES256-CCM:DHE-RSA-AES128-CCM:DHE-RSA-AES256-SHA256:DHE-RSA-AES128-SHA256:AES256-GCM-SHA384:AES128-GCM-SHA256:AES256-CCM8:AES128-CCM8:AES256-CCM:AES128-CCM:AES256-SHA256:AES128-SHA2 .. ts:cv:: CONFIG proxy.config.ssl.client.cipher_suite STRING - Configures the cipher_suite which |TS| will use for SSL connections to origin or next hop. + Configures the cipher_suite which |TS| will use for SSL connections to origin or next hop. This currently defaults to: + + ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:DHE-DSS-AES256-GCM-SHA384:DHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-CCM8:ECDHE-ECDSA-AES256-CCM:DHE-RSA-AES256-CCM8:DHE-RSA-AES256-CCM:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-ARIA256-GCM-SHA384:ECDHE-ARIA256-GCM-SHA384:DHE-DSS-ARIA256-GCM-SHA384:DHE-RSA-ARIA256-GCM-SHA384:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:DHE-RSA-AES256-SHA256:DHE-DSS-AES256-SHA256:ECDHE-ECDSA-CAMELLIA256-SHA384:ECDHE-RSA-CAMELLIA256-SHA384:DHE-RSA-CAMELLIA256-SHA256:DHE-DSS-CAMELLIA256-SHA256:RSA-PSK-AES256-GCM-SHA384:RSA-PSK-CHACHA20-POLY1305:RSA-PSK-ARIA256-GCM-SHA384:AES256-GCM-SHA384:AES256-CCM8:AES256-CCM:ARIA256-GCM-SHA384:AES256-SHA256:CAMELLIA256-SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:DHE-DSS-AES128-GCM-SHA256:DHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-CCM8:ECDHE-ECDSA-AES128-CCM:DHE-RSA-AES128-CCM8:DHE-RSA-AES128-CCM:ECDHE-ECDSA-ARIA128-GCM-SHA256:ECDHE-ARIA128-GCM-SHA256:DHE-DSS-ARIA128-GCM-SHA256:DHE-RSA-ARIA128-GCM-SHA256:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA256:DHE-DSS-AES128-SHA256:ECDHE-ECDSA-CAMELLIA128-SHA256:ECDHE-RSA-CAMELLIA128-SHA256:DHE-RSA-CAMELLIA128-SHA256:DHE-DSS-CAMELLIA128-SHA256:RSA-PSK-AES128-GCM-SHA256:RSA-PSK-ARIA128-GCM-SHA256:AES128-GCM-SHA256:AES128-CCM8:AES128-CCM:ARIA128-GCM-SHA256:AES128-SHA256:CAMELLIA128-SHA256 .. ts:cv:: CONFIG proxy.config.ssl.server.TLSv1_3.cipher_suites STRING @@ -3080,9 +3200,9 @@ SSL Termination connections. For the list of algorithms and instructions, see The ``-ciphersuites`` section of `OpenSSL Ciphers `_. - The current default value with OpenSSL is: + The current default value is: - TLS_AES_256_GCM_SHA384:TLS_CHACHA20_POLY1305_SHA256:TLS_AES_128_GCM_SHA256 + TLS_AES_256_GCM_SHA384:TLS_AES_128_GCM_SHA256:TLS_CHACHA20_POLY1305_SHA256 This configuration works with OpenSSL v1.1.1 and above. @@ -3091,12 +3211,23 @@ SSL Termination By default (``1``) |TS| will use the server's cipher suites preferences instead of the client preferences. By disabling it (``0``) |TS| will use client's cipher suites preferences. +.. ts:cv:: CONFIG proxy.config.ssl.server.prioritize_chacha INT 0 + + By enabling it (``1``) |TS| will temporarily reprioritize ChaCha20-Poly1305 ciphers to the top of the + server cipher list if a ChaCha20-Poly1305 cipher is at the top of the client cipher list. + + This configuration works with OpenSSL v1.1.1 and above. + .. ts:cv:: CONFIG proxy.config.ssl.client.TLSv1_3.cipher_suites STRING Configures the cipher_suites which |TS| will use for TLSv1.3 connections to origin or next hop. This configuration works with OpenSSL v1.1.1 and above. + The current default is: + + TLS_AES_256_GCM_SHA384:TLS_AES_128_GCM_SHA256:TLS_CHACHA20_POLY1305_SHA256 + .. ts:cv:: CONFIG proxy.config.ssl.server.groups_list STRING Configures the list of supported groups provided by OpenSSL which @@ -3123,11 +3254,18 @@ SSL Termination .. ts:cv:: CONFIG proxy.config.ssl.TLSv1 INT 0 - Enables (``1``) or disables (``0``) TLSv1.0. + Enables (``1``) or disables (``0``) TLSv1.0. If not specified, disabled by default. .. ts:cv:: CONFIG proxy.config.ssl.TLSv1_1 INT 0 - Enables (``1``) or disables (``0``) TLS v1.1. If not specified, enabled by default. [Requires OpenSSL v1.0.1 and higher] + Enables (``1``) or disables (``0``) TLS v1.1. If not specified, disabled by default. [Requires OpenSSL v1.0.1 and higher] + +.. note:: + In order to enable TLS v1 or v1.1, additional ciphers must be added to proxy.config.ssl.client.cipher_suite. For + example this list would restore the SHA1 (insecure!) cipher suites suitable for these deprecated TLS versions: + + ECDHE-ECDSA-AES256-SHA:ECDHE-RSA-AES256-SHA:DHE-RSA-AES256-SHA:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES128-SHA:DHE-RSA-AES128-SHA:AES256-SHA:AES128-SHA + .. ts:cv:: CONFIG proxy.config.ssl.TLSv1_2 INT 1 @@ -3158,6 +3296,7 @@ SSL Termination .. ts:cv:: CONFIG proxy.config.ssl.server.multicert.filename STRING ssl_multicert.config + :deprecated: The location of the :file:`ssl_multicert.config` file, relative to the |TS| configuration directory. In the following @@ -3226,6 +3365,7 @@ SSL Termination file is changed with new tickets, use :option:`traffic_ctl config reload` to begin using them. .. ts:cv:: CONFIG proxy.config.ssl.servername.filename STRING sni.yaml + :deprecated: The filename of the :file:`sni.yaml` configuration file. If relative, it is relative to the configuration directory. @@ -3304,6 +3444,21 @@ SSL Termination Set to 1 to enable Traffic Server to process TLS tickets for TLS session resumption. +.. ts:cv:: CONFIG proxy.config.ssl.server.session_ticket.number INT 2 + + This configuration control the number of TLSv1.3 session tickets that are issued. + Take into account that setting the value to 0 will disable session caching for TLSv1.3 + connections. + + Lowering this setting to ``1`` can be interesting when ``proxy.config.ssl.session_cache`` is enabled because + otherwise for every new TLSv1.3 connection two session IDs will be inserted in the session cache. + On the other hand, if ``proxy.config.ssl.session_cache`` is disabled, using the default value is recommended. + In those scenarios, increasing the number of tickets could be potentially beneficial for clients performing + multiple requests over concurrent TLS connections as per RFC 8446 clients SHOULDN'T reuse TLS Tickets. + + For more information see https://www.openssl.org/docs/man1.1.1/man3/SSL_CTX_set_num_tickets.html + [Requires OpenSSL v1.1.1 and higher] + .. ts:cv:: CONFIG proxy.config.ssl.hsts_max_age INT -1 :overridable: @@ -3347,7 +3502,7 @@ SSL Termination Client-Related Configuration ---------------------------- -.. ts:cv:: CONFIG proxy.config.ssl.client.verify.server.policy STRING DISABLED +.. ts:cv:: CONFIG proxy.config.ssl.client.verify.server.policy STRING PERMISSIVE :reloadable: :overridable: @@ -3384,22 +3539,6 @@ Client-Related Configuration :code:`ALL` Check both the signature and the name. -.. ts:cv:: CONFIG proxy.config.ssl.client.verify.server INT 0 - :reloadable: - :deprecated: - - This setting has been deprecated and :ts:cv:`proxy.config.ssl.client.verify.server.policy` and - :ts:cv:`proxy.config.ssl.client.verify.server.properties` should be used instead. - - Configures |TS| to verify the origin server certificate - with the Certificate Authority (CA). This configuration takes a value between 0 to 2. - - You can override this global setting on a per domain basis in the :file:`sni.yaml` file using the :ref:`verify_origin_server attribute`. - - :0: Server Certificate will not be verified - :1: Certificate will be verified and the connection will not be established if verification fail - :2: The provided certificate will be verified and the connection will be established - .. ts:cv:: CONFIG proxy.config.ssl.client.cert.filename STRING NULL :reloadable: :overridable: @@ -3445,7 +3584,21 @@ Client-Related Configuration Indicate how the SNI value for the TLS connection to the origin is selected. By default it is `host` which means the host header field value is used for the SNI. If `remap` is specified, the - remapped origin name is used for the SNI value. + remapped origin name is used for the SNI value. If `verify_with_name_source` is specified, the + SNI will be the host header value and the name to check in the server certificate will be the + remap header value. + We have two names that could be used in the transaction host header and the SNI value to the + origin. These could be the host header from the client or the remap host name. Unless you have + pristine host header enabled, these are likely the same values. + If sni_policy = host, both the sni and the host header to origin will be the same. + If sni_policy = remap, the sni value with be the remap host name and the host header will be the + host header from the client. + In addition, We may want to set the SNI and host headers the same (makes some common web servers + happy), but the certificate served by the origin may have a name that corresponds to the remap + name. So instead of using the SNI name for the name check, we may want to use the remap name. + So if sni_policy = verify_with_name_source, the sni will be the host header value and the name to + check in the server certificate will be the remap header value. + .. ts:cv:: CONFIG proxy.config.ssl.client.TLSv1 INT 0 @@ -3478,6 +3631,24 @@ Client-Related Configuration engines. This setting assumes an absolute path. An example config file is at :ts:git:`contrib/openssl/load_engine.cnf`. +TLS v1.3 0-RTT Configuration +---------------------------- + +.. note:: + TLS v1.3 must be enabled in order to utilize 0-RTT early data. + +.. ts:cv:: CONFIG proxy.config.ssl.server.max_early_data INT 0 + + Specifies the maximum amount of early data in bytes that is permitted to be sent on a single connection. + + The minimum value that enables early data, and the suggested value for this option are both 16384 (16KB). + + Setting to ``0`` effectively disables 0-RTT. + +.. ts:cv:: CONFIG proxy.config.ssl.server.allow_early_data_params INT 0 + + Set to ``1`` to allow HTTP parameters on early data requests. + OCSP Stapling Configuration =========================== @@ -3547,7 +3718,7 @@ HTTP/2 Configuration :ts:cv:`proxy.config.http2.min_concurrent_streams_in`. To disable, set to zero (``0``). -.. ts:cv:: CONFIG proxy.config.http2.initial_window_size_in INT 1048576 +.. ts:cv:: CONFIG proxy.config.http2.initial_window_size_in INT 65535 :reloadable: The initial window size for inbound connections. @@ -3562,7 +3733,15 @@ HTTP/2 Configuration :reloadable: The maximum size of the header compression table used to decode header - blocks. + blocks. This value will be advertised as SETTINGS_HEADER_TABLE_SIZE. + +.. ts:cv:: CONFIG proxy.config.http2.header_table_size_limit INT 65536 + :reloadable: + + The maximum size of the header compression table ATS actually use when ATS + encodes headers. Setting 0 means ATS doesn't insert headers into HPACK + Dynamic Table, however, headers still can be encoded as indexable + representations. The upper limit is 65536. .. ts:cv:: CONFIG proxy.config.http2.max_header_list_size INT 131072 :reloadable: @@ -3658,7 +3837,9 @@ HTTP/2 Configuration Specifies how many number of PRIORITY frames |TS| receives for a minute at maximum. Clients exceeded this limit will be immediately disconnected with an error - code of ENHANCE_YOUR_CALM. + code of ENHANCE_YOUR_CALM. If this is set to 0, the limit logic is disabled. + This limit only will be enforced if :ts:cv:`proxy.config.http2.stream_priority_enabled` + is set to 1. .. ts:cv:: CONFIG proxy.config.http2.min_avg_window_update FLOAT 2560.0 :reloadable: @@ -3667,7 +3848,6 @@ HTTP/2 Configuration Clients that send smaller window increments lower than this limit will be immediately disconnected with an error code of ENHANCE_YOUR_CALM. - HTTP/3 Configuration ==================== @@ -3679,6 +3859,12 @@ QUIC Configuration All configurations for QUIC are still experimental and may be changed or removed in the future without prior notice. +.. ts:cv:: CONFIG proxy.config.quic.qlog_dir STRING NULL + :reloadable: + + The qlog is enabled when this configuration is not NULL. And will dump + the qlog to this dir. + .. ts:cv:: CONFIG proxy.config.quic.instance_id INT 0 :reloadable: @@ -3878,13 +4064,13 @@ removed in the future without prior notice. This is just for debugging. Do not change it from the default value unless you really understand what this is. -.. ts:cv:: CONFIG proxy.config.quic.congestion_control.initial_window_scale INT 10 +.. ts:cv:: CONFIG proxy.config.quic.congestion_control.initial_window INT 12000 :reloadable: This is just for debugging. Do not change it from the default value unless you really understand what this is. -.. ts:cv:: CONFIG proxy.config.quic.congestion_control.minimum_window_scale INT 2 +.. ts:cv:: CONFIG proxy.config.quic.congestion_control.minimum_window INT 2400 :reloadable: This is just for debugging. Do not change it from the default value unless @@ -3909,14 +4095,10 @@ Plug-in Configuration Specifies the location of |TS| plugins. -.. ts:cv:: CONFIG proxy.config.remap.num_remap_threads INT 0 +.. ts:cv:: CONFIG proxy.config.plugin.dynamic_reload_mode INT 1 - When this variable is set to ``0``, plugin remap callbacks are - executed in line on network threads. If remap processing takes - significant time, this can be cause additional request latency. - Setting this variable to causes remap processing to take place - on a dedicated thread pool, freeing the network threads to service - additional requests. + Enables (``1``) or disables (``0``) the dynamic reload feature for remap + plugins (`remap.config`). Global plugins (`plugin.config`) do not have dynamic reload feature yet. SOCKS Processor =============== @@ -3930,6 +4112,7 @@ SOCKS Processor Specifies the SOCKS version (``4``) or (``5``) .. ts:cv:: CONFIG proxy.config.socks.socks_config_file STRING socks.config + :deprecated: The socks.config file allows you to specify ranges of IP addresses that will not be relayed to the SOCKS server. It can also be used @@ -4178,11 +4361,57 @@ Sockets For more information on the implications of enabling huge pages, see `Wikipedia _`. +.. ts:cv:: CONFIG proxy.config.dump_mem_info_frequency INT 0 + :reloadable: + + Enable . When enabled makes Traffic Server dump IO Buffer memory information + to ``traffic.out`` at ```` (intervals are in seconds). A zero value implies it is + disabled + +.. ts:cv:: CONFIG proxy.config.res_track_memory INT 0 + + When enabled makes Traffic Server track memory usage (allocations and releases). This + information is dumped to ``traffic.out`` when the user sends a SIGUSR1 signal or + periodically when :ts:cv:`proxy.config.dump_mem_info_frequency` is enabled. + + ===== ====================================================================== + Value Description + ===== ====================================================================== + ``0`` Memory tracking Disabled + ``1`` Tracks IO Buffer Memory allocations and releases + ``2`` Tracks IO Buffer Memory and OpenSSL Memory allocations and releases + ===== ====================================================================== + .. ts:cv:: CONFIG proxy.config.allocator.dontdump_iobuffers INT 1 - Enable (1) the exclusion of IO buffers from core files when ATS crashes on supported - platforms. (Currently only linux). IO buffers are allocated with the MADV_DONTDUMP - with madvise() on linux platforms that support MADV_DONTDUMP. Enabled by default. + Enable (1) the exclusion of IO buffers from core files when ATS crashes on supported + platforms. (Currently only linux). IO buffers are allocated with the MADV_DONTDUMP + with madvise() on linux platforms that support MADV_DONTDUMP. Enabled by default. + +.. ts:cv:: CONFIG proxy.config.ssl.misc.io.max_buffer_index INT 8 + + Configures the max IOBuffer Block index used for various SSL Operations + such as Handshake or Protocol Probe. Default value is 8 which maps to a 32K buffer + +.. ts:cv:: CONFIG proxy.config.hostdb.io.max_buffer_index INT 8 + + Configures the max IOBuffer Block index used for storing HostDB records. + Default value is 8 which maps to a 32K buffer + +.. ts:cv:: CONFIG proxy.config.payload.io.max_buffer_index INT 8 + + Configures the max IOBuffer Block index used for storing request payload buffer + for a POST request. Default value is 8 which maps to a 32K buffer + +.. ts:cv:: CONFIG proxy.config.msg.io.max_buffer_index INT 8 + + Configures the max IOBuffer Block index used for storing miscellaneous transactional + buffers such as error response body. Default value is 8 which maps to a 32K buffer + +.. ts:cv:: CONFIG proxy.config.log.io.max_buffer_index INT 8 + + Configures the max IOBuffer Block index used for storing an access log entry. + Default value is 8 which maps to a 32K buffer .. ts:cv:: CONFIG proxy.config.http.enabled INT 1 diff --git a/doc/admin-guide/files/remap.config.en.rst b/doc/admin-guide/files/remap.config.en.rst index a4b2f83b596..5abcdf7aefe 100644 --- a/doc/admin-guide/files/remap.config.en.rst +++ b/doc/admin-guide/files/remap.config.en.rst @@ -480,6 +480,13 @@ mapping rules. (It is activated before any mappings and is never deactivated.) The filter `local_only` will only be applied to the second mapping. +NextHop Selection Strategies +============================ + +You may configure Nexthop or Parent hierarchical caching rules by remap using the +**@strategy** tag. See :doc:`../configuration/hierarchical-caching.en` and :doc:`strategies.yaml.en` +for configuration details and examples. + Including Additional Remap Files ================================ diff --git a/doc/admin-guide/files/sni.yaml.en.rst b/doc/admin-guide/files/sni.yaml.en.rst index 8556b7ac433..f4846b97f71 100644 --- a/doc/admin-guide/files/sni.yaml.en.rst +++ b/doc/admin-guide/files/sni.yaml.en.rst @@ -30,10 +30,10 @@ connections. The configuration is driven by the SNI values provided by the inbou file consists of a set of configuration items, each identified by an SNI value (``fqdn``). When an inbound TLS connection is made, the SNI value from the TLS negotiation is matched against the items specified by this file and if there is a match, the values specified in that item override -the defaults. This is done during the inbound connection processing and be some outbound properties +the defaults. This is done during the inbound connection processing; some outbound properties can be overridden again later, such as via :file:`remap.config` or plugins. -By default this is named :file:`sni.yaml`. The file can be changed by setting +By default this is named :file:`sni.yaml`. The filename can be changed by setting :ts:cv:`proxy.config.ssl.servername.filename`. This file is loaded on start up and by :option:`traffic_ctl config reload` if the file has been modified since process start. @@ -42,15 +42,20 @@ Each table is a set of key / value pairs that create a configuration item. This wildcard entries. To apply an SNI based setting on all the server names with a common upper level domain name, the user needs to enter the fqdn in the configuration with a ``*.`` followed by the common domain name. (``*.yahoo.com`` for example). -.. _override-verify-origin-server: .. _override-verify-server-policy: .. _override-verify-server-properties: +.. _override-host-sni-policy: -========================= ============================================================================== +========================= ======================================================================================== Key Meaning -========================= ============================================================================== +========================= ======================================================================================== fqdn Fully Qualified Domain Name. This item is used if the SNI value matches this. +ip_allow Specify a list of client IP address, subnets, or ranges what are allowed to complete + the connection. This list is comma separated. IPv4 and IPv6 addresses can be specified. + Here is an example list: 192.168.1.0/24,192.168.10.1-4. This would allow connections + from clients in the 19.168.1.0 network or in the range from 192.168.10.1 to 192.168.1.4. + verify_server_policy One of the values :code:`DISABLED`, :code:`PERMISSIVE`, or :code:`ENFORCED`. By default this is :ts:cv:`proxy.config.ssl.client.verify.server.policy`. @@ -61,10 +66,6 @@ verify_server_properties One of the values :code:`NONE`, :code:`SIGNATURE`, :co By default this is :ts:cv:`proxy.config.ssl.client.verify.server.properties`. This controls what Traffic Server checks when evaluating the origin certificate. -verify_origin_server Deprecated. Use verify_server_policy and verify_server_properties instead. - One of the values :code:`NONE`, :code:`MODERATE`, or :code:`STRICT`. - By default this is :ts:cv:`proxy.config.ssl.client.verify.server`. - verify_client One of the values :code:`NONE`, :code:`MODERATE`, or :code:`STRICT`. If ``NONE`` is specified, |TS| requests no certificate. If ``MODERATE`` is specified |TS| will verify a certificate that is presented by the client, but it will not @@ -73,15 +74,20 @@ verify_client One of the values :code:`NONE`, :code:`MODERATE`, or : By default this is :ts:cv:`proxy.config.ssl.client.certification_level`. -valid_tls_versions_in This specifies the list of TLS protocols that will be offered to user agents during - the TLS negotiation. This replaces the global settings in :ts:cv:`proxy.config.ssl.TLSv1`, - :ts:cv:`proxy.config.ssl.TLSv1_1`, :ts:cv:`proxy.config.ssl.TLSv1_2`, - and :ts:cv:`proxy.config.ssl.TLSv1_3`. The potential values are TLSv1, TLSv1_1, TLSv1_2, and - TLSv1_3. You must list all protocols that |TS| should offer to the client when using - this key. This key is only valid for openssl 1.1.0 and later. Older versions of openssl do not - provide a hook early enough to update the SSL object. It is a syntax error for |TS| built - against earlier versions. +host_sni_policy One of the values :code:`DISABLED`, :code:`PERMISSIVE`, or :code:`ENFORCED`. + + If not specified, the value of :ts:cv:`proxy.config.http.host_sni_policy` is used. + This controls how policy impacting mismatches between host header and SNI values are + dealt with. +valid_tls_versions_in This specifies the list of TLS protocols that will be offered to user agents during + the TLS negotiation. This replaces the global settings in + :ts:cv:`proxy.config.ssl.TLSv1`, :ts:cv:`proxy.config.ssl.TLSv1_1`, + :ts:cv:`proxy.config.ssl.TLSv1_2`, and :ts:cv:`proxy.config.ssl.TLSv1_3`. The potential + values are TLSv1, TLSv1_1, TLSv1_2, and TLSv1_3. You must list all protocols that |TS| + should offer to the client when using this key. This key is only valid for openssl + 1.1.0 and later. Older versions of openssl do not provide a hook early enough to update + the SSL object. It is a syntax error for |TS| built against earlier versions. client_cert The file containing the client certificate to use for the outbound connection. @@ -97,13 +103,15 @@ client_key The file containing the client private key that corres |TS| tries to use a private key in client_cert. Otherwise, :ts:cv:`proxy.config.ssl.client.private_key.filename` is used. -http2 Indicates whether the H2 protocol should be added to or removed from the +http2 Indicates whether the H2 protocol should be added to or removed from the protocol negotiation list. The valid values are :code:`on` or :code:`off`. disable_h2 Deprecated for the more general h2 setting. Setting disable_h2 to :code:`true` is the same as setting http2 to :code:`on`. tunnel_route Destination as an FQDN and port, separated by a colon ``:``. + Match group number can be specified by ``$N`` where N should refer to a specified group + in the FQDN, ``tunnel_route: $1.domain``. This will forward all traffic to the specified destination without first terminating the incoming TLS connection. @@ -113,7 +121,14 @@ forward_route Destination as an FQDN and port, separated by a colon This is similar to tunnel_route, but it terminates the TLS connection and forwards the decrypted traffic. |TS| will not interpret the decrypted data, so the contents do not need to be HTTP. -========================= ============================================================================== + +partial_blind_route Destination as an FQDN and port, separated by a colon ``:``. + + This is similar to forward_route in that |TS| terminates the incoming TLS connection. + In addition partial_blind_route creates a new TLS connection to the specified origin. + It does not interpret the decrypted data before passing it to the origin TLS + connection, so the contents do not need to be HTTP. +========================= ======================================================================================== Client verification, via ``verify_client``, corresponds to setting :ts:cv:`proxy.config.ssl.client.certification_level` for this connection as noted below. @@ -175,13 +190,24 @@ Disable HTTP/2 for ``no-http2.example.com``. - fqdn: no-http2.example.com http2: off -Require client certificate verification for ``example.com`` and any server name ending with ``.yahoo.com``. Therefore, client request for a server name ending with yahoo.com (e.g., def.yahoo.com, abc.yahoo.com etc.) will cause |TS| require and verify the client certificate. By contrast, |TS| will allow a client certificate to be provided for ``example.com`` and if it is, |TS| will require the certificate to be valid. +Require client certificate verification for ``foo.com`` and any server name ending with ``.yahoo.com``. Therefore, client +request for a server name ending with yahoo.com (e.g., def.yahoo.com, abc.yahoo.com etc.) will cause |TS| require and verify +the client certificate. + +For ``foo.com``, if the user agent sets the host header to foo.com but the SNI to some other value, |TS| will warn about the +mismatch but continue to process the request. Mismatches for the other domains will cause |TS| to warn and return 403. + +|TS| will allow a client certificate to be provided for ``example.com`` and if it is, |TS| will require the +certificate to be valid. .. code-block:: yaml sni: - fqdn: example.com verify_client: MODERATE + - fqdn: 'foo.com' + verify_client: STRICT + host_sni_policy: PERMISSIVE - fqdn: '*.yahoo.com' verify_client: STRICT @@ -195,6 +221,21 @@ client certificate. verify_server_policy: DISABLED verify_client: STRICT +Use FQDN captured group to match in ``tunnel_route``. + +.. code-block:: yaml + + sni: + - fqdn: '*.foo.com' + tunnel_route: '$1.myfoo' + - fqdn: '*.bar.*.com' + tunnel_route: '$2.some.$1.yahoo' + +FQDN ``some.foo.com`` will match and the captured string will be replaced in the ``tunnel_route`` which will end up being +``some.myfoo``. +Second part is using multiple groups, having ``bob.bar.example.com`` as FQDN, ``tunnel_route`` will end up being +``bar.some.bob.yahoo``. + See Also ======== diff --git a/doc/admin-guide/files/splitdns.config.en.rst b/doc/admin-guide/files/splitdns.config.en.rst index 72859c5a59e..3f5154eebee 100644 --- a/doc/admin-guide/files/splitdns.config.en.rst +++ b/doc/admin-guide/files/splitdns.config.en.rst @@ -105,7 +105,7 @@ Examples Consider the following DNS server selection specifications: :: - dest_domain=internal.company.com named=255.255.255.255:212 255.255.255.254 def_domain=company.com search_list=company.com company1.com + dest_domain=internal.company.com named="255.255.255.255:212 255.255.255.254" def_domain=company.com search_list="company.com company1.com" dest_domain=!internal.company.com named=255.255.255.253 Now consider the following two requests: :: diff --git a/doc/admin-guide/files/ssl_multicert.config.en.rst b/doc/admin-guide/files/ssl_multicert.config.en.rst index 54a391d3d50..d78bc75fc16 100644 --- a/doc/admin-guide/files/ssl_multicert.config.en.rst +++ b/doc/admin-guide/files/ssl_multicert.config.en.rst @@ -106,8 +106,9 @@ ssl_ticket_enabled=1|0 (optional) OpenSSL should be upgraded to version 0.9.8f or higher. This option must be set to `0` to disable session ticket support. -ticket_key_name=FILENAME (optional) [**REMOVED in 7.1.x and 8.0**] - Ticket key should be set in records.config via :ts:cv:`proxy.config.ssl.server.ticket_key.filename` +ssl_ticket_number=INTEGER (optional) + Specifies the number of TLSv1.3 session tickets that are issued. + This defaults to 2 (the OpenSSL default) ssl_key_dialog=builtin|"exec:/path/to/program [args]" (optional) Method used to provide a pass phrase for encrypted private keys. If the diff --git a/doc/admin-guide/files/strategies.yaml.en.rst b/doc/admin-guide/files/strategies.yaml.en.rst new file mode 100644 index 00000000000..8df48839487 --- /dev/null +++ b/doc/admin-guide/files/strategies.yaml.en.rst @@ -0,0 +1,235 @@ +.. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +=============== +strategies.yaml +=============== + +.. configfile:: strategies.yaml + +.. include:: ../../common.defs + +The :file:`strategies.yaml` file identifies the next hop proxies used in an +cache hierarchy and the algorithms used to select the next hop proxy. Use +this file to perform the following configuration: + +- Set up next hop cache hierarchies, with multiple parents and parent + failover + +Traffic Server uses the :file:`strategies.yaml` file only when one or more +remap lines in remap.config specifies the use of a strategy with the @strategy tag. +remap.config Example:: + + map http://www.foo.com http://www.bar.com @strategy='mid-tier-north' + +After you modify the :file:`strategies.yaml` file, run the :option:`traffic_ctl config reload` +command to apply your changes. + +Format +====== + +The `strategies.yaml` is a YAML document with three top level namespaces: **hosts**, **groups** +and **strategies**. These name spaces may be in separate files. When in separate files, use +**#include filename** in the `strategies.yaml` so that they are concatenated by the strategy +factory into a single YAML document in the order, **hosts**, **groups**, and the **strategies**. + +Alternatively if the config parameter `proxy.config.url_remap.strategies.filename` refers to +a directory, the NextHopStrategyFactory will alphanumerically concatenate all files in that directory that end in `.yaml` by name into a single document stream for parsing. The final document must be a valid `YAML` document with single `strategies` node and optionally a single `hosts` and `groups` node. Any **#include filename** strings are ignored when reading `.yaml` files in a directory. + +Hosts definitions +================= + +The **hosts** definitions is a **YAML** list of hosts. This list is **optional** but if not used, the +**groups** list **must** include complete definitions for hosts. See the **group** examples below. + +In the example below, **hosts** is a **YAML** list of hosts. Each host entry uses a **YAML** anchor, +**&p1** and **&p2** that may be used elsewhere in the **YAML** document to refer to hosts **p1** and **p2**. + +- **host**: the host value is a hostname string +- **protocol**: a list of schemes, ports, and health check urls for the host. +- **healthcheck**: health check information with the **url** used to check + the hosts health by some external health check agent. + +Example:: + + hosts: + - &p1 + host: p1.foo.com + protocol: + - scheme: http + port: 80 + health_check_url: http://192.168.1.1:80 + - scheme: https + port: 443 + health_check_url: https://192.168.1.1:443 + - &p2 + host: p2.foo.com + protocol: + - scheme: http + port: 80 + health_check_url: http://192.168.1.2:80 + +Groups definitions +================== + +The **groups** definitions is a **YAML** list of host groups. host groups are used as the primary and secondary groups used by nexthop to choose hosts from. The first group is the **primary** group next hop chooses hosts from. The remaining groups are used failover. The **strategies** **policy** specifies how the groups are used. + +Below are examples of group definitions. The first example is using **YAML** anchors and references. +When using **references**, the complete **YAML** document must include the **anchors** portion of the document first. + +The second example shows a complete **groups** definition without the use of a **hosts** name space and it's **YAML** anchors. + +The field definitions in the examples below are defined in the **hosts** section. + +Example using **YAML** anchors and references:: + + groups: + - &g1 + - <<: *p1 + weight: 1.5 + - <<: *p2 + weight: 0.5 + - &g2 + - <<: *p3 + weight: 0.5 + - <<: *p4 + weight: 1.5 + +Explicitly defined Example, no **YAML** references:: + + groups: + - &g1 + - p1 + host: p1.foo.com + protocol: + - scheme: http + port: 80 + health_check_url: http://192.168.1.1:80 + - scheme: https + port: 443 + health_check_url: https://192.168.1.1:443 + weight: 0.5 + - p2 + host: p2.foo.com + protocol: + - scheme: http + port: 80 + health_check_url: http://192.168.1.2:80 + - scheme: https + port: 443 + health_check_url: https://192.168.1.2:443 + weight: 0.5 + - &g2 + - p3 + host: p3.foo.com + protocol: + - scheme: http + port: 80 + health_check_url: http://192.168.1.3:80 + - scheme: https + port: 443 + health_check_url: https://192.168.1.3:443 + weight: 0.5 + - p4 + host: p4.foo.com + protocol: + - scheme: http + port: 80 + health_check_url: http://192.168.1.4:80 + - scheme: https + port: 443 + health_check_url: https://192.168.1.4:443 + weight: 0.5 + +Strategies definitions +====================== + +The **strategies** namespace defines a **YAML** list of strategies that may be applied to a **remap** +entry using the **@strategy** tag in remap.config. + +Each **strategy** in the list may using the following parameters:: + +- **strategy**: The value is the name of the strategy. +- **policy**: The algorithm the **strategy** uses to select hosts. Currently one of the following: + + #. **rr_ip**: round robin selection using the modulus of the client IP + #. **rr_strict**: strict round robin over the list of hosts in the primary group. + #. **first_live**: always selects the first host in the primary group. Other hosts are selected when the first host fails. + #. **latched**: Same as **first_live** but primary selection sticks to whatever host was used by a previous transaction. + #. **consistent_hash**: hosts are selected using a **hash_key**. + +- **hash_key**: The hashing key used by the **consistent_hash** policy. If not specified, defaults to **path** which is the + same policy used in the **parent.config** implementation. Use one of: + + #. **hostname**: Creates a hash using the **hostname** in the request URL. + #. **path**: (**default**) Creates a hash over the path portion of the request URL. + #. **path+query**: Same as **path** but adds the **query string** in the request URL. + #. **path+fragment**: Same as **path** but adds the fragment portion of the URL. + #. **cache_key**: Uses the hash key from the **cachekey** plugin. defaults to **path** if the **cachekey** plugin is not configured on the **remap**. + #. **url**: Creates a hash from the entire request url. + +- **go_direct** - A boolean value indicating whether a transaction may bypass proxies and go direct to the origin. Defaults to **true** +- **parent_is_proxy**: A boolean value which indicates if the groups of hosts are proxy caches or origins. **true** (default) means all the hosts used in the remap are |TS| caches. **false** means the hosts are origins that the next hop strategies may use for load balancing and/or failover. +- **scheme** Indicates which scheme the strategy supports, *http* or *https* + - **failover**: A map of **failover** information. + - **max_simple_retries**: Part of the **failover** map and is an integer value of the maximum number of retries for a **simple retry** on the list of indicated response codes. **simple retry** is used to retry an upstream request using another upstream server if the response received on from the original upstream request matches any of the response codes configured for this strategy in the **failover** map. If no failover response codes are configured, no **simple retry** is attempted. + + - **ring_mode**: Part of the **failover** map. The host ring selection mode. Use either **exhaust_ring** or **alternate_ring** + + #. **exhaust_ring**: when a host normally selected by the policy fails, another host is selected from the same group. A new group is not selected until all hosts on the previous group have been exhausted + #. **alternate_ring**: retry hosts are selected from groups in an alternating group fashion. + + - **response_codes**: Part of the **failover** map. This is a list of **http** response codes that may be used for **simple retry**. + - **health_check**: Part of the **failover** map. A list of health checks. **passive** is the default and means that the state machine marks down **hosts** when a transaction timeout or connection error is detected. **passive** is always used by the next hop strategies. **active** means that some external process may actively health check the hosts using the defined **health check url** and mark them down using **traffic_ctl**. + + +Example: +:: + + #include unit-tests/hosts.yaml + # + strategies: + - strategy: 'strategy-1' + policy: consistent_hash + hash_key: cache_key + go_direct: false + groups: + - *g1 + - *g2 + scheme http + failover: + ring_mode: exhaust_ring + response_codes: + - 404 + - 503 + health_check: + - passive + - strategy: 'strategy-2' + policy: rr_strict + hash_key: cache_key + go_direct: true + groups: + - *g1 + - *g2 + scheme http + failover: + ring_mode: exhaust_ring + response_codes: + - 404 + - 503 + health_check: + - passive diff --git a/doc/admin-guide/files/volume.config.en.rst b/doc/admin-guide/files/volume.config.en.rst index 7fdfc8e3381..7aa0426d3f5 100644 --- a/doc/admin-guide/files/volume.config.en.rst +++ b/doc/admin-guide/files/volume.config.en.rst @@ -57,14 +57,54 @@ volumes without deleting and clearing the existing volumes. Changing this file to add, remove or modify volumes effectively invalidates the cache. + +Optional ramcache setting +------------------------- + +You can also add an option ``ramcache=true/false`` to the volume configuration +line. True is the default setting and so not needed unless you want to explicitly +set it. Setting ``ramcache=false`` will disable the ramcache that normally +sits in front of a volume. This may be desirable if you are using something like +ramdisks, to avoid wasting RAM and cpu time on double caching objects. + + +Exclusive spans and volume sizes +================================ + +In the following sample configuration 2 spans `/dev/disk1` and `/dev/disk2` are defined +in :file:`storage.config`, where span `/dev/disk2` is assigned to `volume 3` exclusively +(`volume 3` is forced to an "exclusive" span `/dev/disk2`). +In :file:`volume.config` there are 3 volumes defined, where `volume 1` and `volume 2` +occupy span `/dev/disk1` taking each 50% of its space and `volume 3` takes 100% of span +`/dev/disk2` exclusively. + +storage.config:: + + /dev/disk1 + /dev/disk2 volume=3 # <- exclusive span + +volume.config:: + + volume=1 scheme=http size=50% + volume=2 scheme=http size=50% + volume=3 scheme=http size=512 # <- volume forced to a specific exclusive span + +It is important to note that when percentages are used to specify volume sizes +and "exclusive" spans are assigned (forced) to a particular volume (in this case `volume 3`), +the "exclusive" spans (in this case `/dev/disk2`) are excluded from the total cache +space when the "non-forced" volumes sizes are calculated (in this case `volume 1` and `volume 2`). + + Examples ======== The following example partitions the cache across 5 volumes to decreasing -single-lock pressure for a machine with few drives.:: +single-lock pressure for a machine with few drives. The last volume being +an example of one that might be composed of purely ramdisks so that the +ramcache has been disabled.:: volume=1 scheme=http size=20% volume=2 scheme=http size=20% volume=3 scheme=http size=20% volume=4 scheme=http size=20% - volume=5 scheme=http size=20% + volume=5 scheme=http size=20% ramcache=false diff --git a/doc/admin-guide/introduction.en.rst b/doc/admin-guide/introduction.en.rst index 06effa3c9ae..25d5c79f3ba 100644 --- a/doc/admin-guide/introduction.en.rst +++ b/doc/admin-guide/introduction.en.rst @@ -28,7 +28,7 @@ to and from all parts of the world. Information is free, abundant, and accessible. Unfortunately, global data networking can also be a nightmare for IT professionals as they struggle with overloaded servers and congested networks. It can be challenging to consistently and -reliably accommodate society’s growing data demands. +reliably accommodate society's growing data demands. |TS| is a high-performance web proxy cache that improves network efficiency and performance by caching frequently-accessed @@ -58,10 +58,10 @@ content as those requests travel to the destined web server (origin server). If |TS| contains the requested content, then it serves the content directly. If the requested content is not available from cache, then |TS| acts as a proxy: it obtains the content -from the origin server on the user’s behalf and also keeps a copy to +from the origin server on the user's behalf and also keeps a copy to satisfy future requests. -|TS| provides explicit proxy caching, in which the user’s +|TS| provides explicit proxy caching, in which the user's client software must be configured to send requests directly to Traffic Server. Explicit proxy caching is described in the :ref:`explicit-proxy-caching` chapter. @@ -75,7 +75,7 @@ section. ----------------------- As a reverse proxy, |TS| is configured to be the origin server -to which the user is trying to connect (typically, the origin server’s +to which the user is trying to connect (typically, the origin server's advertised hostname resolves to |TS|, which acts as the real origin server). The reverse proxy feature is also called server acceleration. Reverse proxy is described in more detail in @@ -165,15 +165,15 @@ bindings in memory, DNS traffic is reduced. |TS| Processes -------------- -|TS| contains three processes that work together to serve +|TS| contains two processes that work together to serve requests and manage, control, and monitor the health of the system. -- The :program:`traffic_server` process is the transaction processing engine +#. The :program:`traffic_server` process is the transaction processing engine of |TS|. It is responsible for accepting connections, processing protocol requests, and serving documents from the cache or origin server. -- The :program:`traffic_manager` process is the command and control facility +#. The :program:`traffic_manager` process is the command and control facility of the |TS|, responsible for launching, monitoring, and reconfiguring the :program:`traffic_server` process. The :program:`traffic_manager` process is also responsible for the proxy autoconfiguration port, the @@ -187,14 +187,6 @@ requests and manage, control, and monitor the health of the system. first-served order. This connection queueing shields users from any server restart downtime. -The figure below illustrates the |TS| processes. - -.. figure:: ../static/images/admin/process.jpg - :align: center - :alt: Illustration of the |TS| Processes - - Illustration of the |TS| Processes - Administration Tools -------------------- diff --git a/doc/admin-guide/layer-4-routing.en.rst b/doc/admin-guide/layer-4-routing.en.rst index 188eafa89e5..b6528973a2d 100644 --- a/doc/admin-guide/layer-4-routing.en.rst +++ b/doc/admin-guide/layer-4-routing.en.rst @@ -61,7 +61,10 @@ the option of interest is ``tunnel_route``. If this is set then |TS| synthesizes request to itself with the ``tunnel_route`` host and port as the upstream. That is, the inbound connection is treated as if the user agent had sent a ``CONNECT`` to the upstream and forwards the "`CLIENT HELLO -`__" to it. +`__" to it. In addition to the method appearing +as ``CONNECT``, be aware that logs printing the URL via the ``<%cquc>`` field format will show the +scheme in the URL as ``tunnel``. The scheme as printed via ``<%cqus>``, however, will show the +scheme used in the original client request. Example ------- @@ -108,12 +111,10 @@ The :file:`sni.yaml` contents would be - tunnel_route: app-server-56:4443 fqdn: service-2.example.com -In addition to this, in the :file:`records.config` file, edit the following variables: +In addition to this, in the :file:`records.config` file, edit ``connect_ports`` like so: - :ts:cv:`proxy.config.http.connect_ports`: ``443 4443`` to allow |TS| to connect to the destination port - - :ts:cv:`proxy.config.url_remap.remap_required`: ``0`` to permit |TS| to process requests - for hosts not explicitly configured in the remap rules The sequence of network activity for a Client connecting to ``service-2`` is diff --git a/doc/admin-guide/logging/destinations.en.rst b/doc/admin-guide/logging/destinations.en.rst index 5b70913ad64..03b6f75fb91 100644 --- a/doc/admin-guide/logging/destinations.en.rst +++ b/doc/admin-guide/logging/destinations.en.rst @@ -109,6 +109,9 @@ pipe, only full records are dropped. Output to named pipes is always, as the mode's name implies, in ASCII format. There is no option for logging binary format log data to a named pipe. +For ASCII pipes there exists an option to set the ``pipe_buffer_size`` in +the YAML config. + .. _admin-logging-ascii-v-binary: Deciding Between ASCII or Binary Output diff --git a/doc/admin-guide/logging/examples.en.rst b/doc/admin-guide/logging/examples.en.rst index 4cca17e9d38..9d200923161 100644 --- a/doc/admin-guide/logging/examples.en.rst +++ b/doc/admin-guide/logging/examples.en.rst @@ -183,7 +183,7 @@ No. Field Description of milliseconds between the time the client established the connection with |TS| and the time |TS| sent the last byte of the response back to the client. -3 chi The IP address of the client’s host machine. +3 chi The IP address of the client's host machine. 4 crc/pssc The cache result code; how the cache responded to the request: ``HIT``, ``MISS``, and so on. Cache result codes are described in :ref:`admin-logging-cache-results`. The @@ -247,7 +247,7 @@ to clients: formats: - name: mysummary - format: '% : % : %' + format: '%:%:%' interval: 10 Dual Output to Compact Binary Logs and ASCII Pipes @@ -293,12 +293,32 @@ for them to a UNIX pipe that the alerting software can constantly read from. accept: cqup MATCH "/nightmare/scenario/dont/touch" logs: - - mode: pipe + - mode: ascii_pipe format: canaryformat filters: - canaryfilter filename: alerting_canaries +Configuring ASCII Pipe Buffer Size +================================== + +This example mirrors the one above but also sets a ```pipe_buffer_size``` of +1024 * 1024 for the pipe. This can be set on a per-pipe basis but is only +available on Linux (later than 2.6.35). If this field is not set, the pipe buffer +will default to the OS default size. + +.. code:: yaml + + logs: + - mode: ascii_pipe + format: canaryformat + filters: + - canaryfilter + filename: alerting_canaries + pipe_buffer_size: 1048576 + + + Summarizing Origin Responses by Hour ==================================== @@ -313,12 +333,13 @@ the request to clients during that hour. logging: formats: - name: originrepformat - format: '% % %' + format: '% % %' interval: 3600 filters: - name: originfilter - reject: crc CONTAINS "HIT" + action: reject + condition: crc CONTAINS "HIT" logs: - mode: ascii diff --git a/doc/admin-guide/logging/formatting.en.rst b/doc/admin-guide/logging/formatting.en.rst index dc416d314b3..583e999a923 100644 --- a/doc/admin-guide/logging/formatting.en.rst +++ b/doc/admin-guide/logging/formatting.en.rst @@ -151,6 +151,9 @@ Cache Details .. _chm: .. _cwr: .. _cwtr: +.. _crra: +.. _cwra: +.. _cccs: These log fields reveal details of the |TS| proxy interaction with its own cache while attempting to service incoming client requests. @@ -173,6 +176,13 @@ cwr Proxy Cache Cache Write Result. Specifies the result of attempting to (``WL_MISS``), write interrupted (``INTR``), error while writing (``ERR``), or cache write successful (``FIN``). cwt Proxy Cache Cache Write Transform Result. +crra Proxy Cache Cache read retry attempts to read the object from cache. +cwra Proxy Cache Cache write retry attempts to write a fresh or updated + object to cache. +cccs Proxy Cache Cache collapsed connection success; + -1: collapsing was attempted but failed, request went upstream + 0: collapsing was unnecessary + 1: attempted to collapse and got a cache hit on subsequent read attempts ===== ============== ========================================================== .. _admin-logging-fields-txn: @@ -184,6 +194,8 @@ Connections and Transactions .. _sstc: .. _ccid: .. _ctid: +.. _ctpw: +.. _ctpd: The following log fields are used to list various details of connections and transactions between |TS| proxies and origin servers. @@ -203,6 +215,10 @@ ctid Client Request Client Transaction ID, a non-negative number for a transact which is different for all currently-active transactions on the same client connection. For client HTTP/2 transactions, this value is the stream ID for the transaction. +ctpw Client Request Client Transaction Priority Weight, the priority weight for the + underlying HTTP/2 protocol. +ctpd Client Request Client Transaction Priority Dependence, the transaction ID that + the current transaction depends on for HTTP/2 priority logic. ===== ============== ================================================================== .. _admin-logging-fields-content-type: @@ -325,7 +341,7 @@ cssh ecssh ============== =================== It is also possible to log all of the headers in a transaction message with a -single field. For each original original field, there is a variant which ends in +single field. For each original field, there is a variant which ends in ``ah`` rather than ``h``, as shown here: ============== =================== @@ -489,6 +505,13 @@ shi Origin Server IP address resolved via DNS by |TS| for the origin server. shn Origin Server Host name of the origin server. nhi Origin Server Destination IP address of next hop nhp Origin Server Destination port of next hop +ppv Proxy Protocol Proxy Protocol Version used (if any) between the Load Balancer + Version and |TS| +pps Proxy Protocol Source IP received via Proxy Protocol context from the LB to + Source IP the |TS| +ppd Proxy Protocol Destination IP received via Proxy Protocol context from the LB + Dest IP to the |TS| + ===== ============== ========================================================== .. note:: @@ -541,7 +564,8 @@ were in effect for a given event. ===== ===================== =================================================== Field Source Description ===== ===================== =================================================== -cqhv Client Request Client request HTTP version. +cqhv Client Request Client request HTTP version. Deprecated since 9.0. + Use ``cqpv`` instead. cqpv Client Request Client request protocol and version. csshv Cached Proxy Response Origin server's HTTP version from cached version of the document in |TS| proxy cache. @@ -573,31 +597,44 @@ cqtx Client Request The full HTTP client request text, minus headers, e.g. SSL / Encryption ~~~~~~~~~~~~~~~~ +.. _cssn: +.. _cscert: .. _cqssl: .. _cqssr: .. _cqssv: .. _cqssc: .. _cqssu: .. _pqssl: +.. _pscert: Fields which expose the use, or lack thereof, of specific SSL and encryption features. -===== ============== ========================================================== -Field Source Description -===== ============== ========================================================== -cqssl Client Request SSL client request status indicates if this client - connection is over SSL. -cqssr Client Request SSL session ticket reused status; indicates if the current - request hit the SSL session ticket and avoided a full SSL - handshake. -cqssv Client Request SSL version used to communicate with the client. -cqssc Client Request SSL Cipher used by |TS| to communicate with the client. -cqssu Client Request SSL Elliptic Curve used by |TS| to communicate with the - client when using an ECDHE cipher. -pqssl Proxy Request Indicates whether the connection from |TS| to the origin - was over SSL or not. -===== ============== ========================================================== +====== ============== ========================================================== +Field Source Description +====== ============== ========================================================== +cssn Client TLS SNI server name in client Hello message in TLS handshake. + Hello If no server name present in Hello, or the transaction + was not over TLS (over TCP), this field will contain + ``-``. +cscert Client Request 1 if |TS| requested certificate from client during TLS + handshake. 0 otherwise. +cqssl Client Request SSL client request status indicates if this client + connection is over SSL. +cqssr Client Request SSL session ticket reused status; indicates if the current + request hit the SSL session ticket and avoided a full SSL + handshake. +cqssv Client Request SSL version used to communicate with the client. +cqssc Client Request SSL Cipher used by |TS| to communicate with the client. +cqssu Client Request SSL Elliptic Curve used by |TS| to communicate with the + client when using an ECDHE cipher. +pqssl Proxy Request Indicates whether the connection from |TS| to the origin + was over SSL or not. +pscert Proxy Request 1 if origin requested certificate from |TS| during TLS + handshake but no client certificate was defined. 2 if origin + requested certificate from |TS| during TLS handshake and a + client certificate was defined. 0 otherwise. +====== ============== ========================================================== .. _admin-logging-fields-status: @@ -786,6 +823,17 @@ cquup Client Request Canonical (prior to remapping) path component from the cquuh Client Request Unmapped URL host from the client request. ===== ============== ========================================================== +Line Length +=========== + +The maximum line size for a log entry can be configured via the following +parameters, the details of which are documented in the linked +:file:`records.config` descriptions: + +- :ts:cv:`proxy.config.log.max_line_size` +- :ts:cv:`proxy.config.log.ascii_buffer_size` +- :ts:cv:`proxy.config.log.log_buffer_size` + Log Field Slicing ================= diff --git a/doc/admin-guide/logging/rotation.en.rst b/doc/admin-guide/logging/rotation.en.rst index 6dad554ba17..e0b93c2cd1d 100644 --- a/doc/admin-guide/logging/rotation.en.rst +++ b/doc/admin-guide/logging/rotation.en.rst @@ -255,13 +255,13 @@ the maximum number of rolled log files, and forcing |TS| to roll even when there Let us say we wanted the oldest log entry to be kept on the box to be no older than 2-hour old. -Set :ts:cv:`proxy.config.output.logfile.rolling_interval_sec` (yaml: `rolling_interval_sec`) to 3600 (1h) +Set :ts:cv:`proxy.config.log.rolling_interval_sec` (yaml: `rolling_interval_sec`) to 3600 (1h) which will lead to rolling every 1h. -Set :ts:cv:`proxy.config.output.logfile.rolling_max_count` (yaml: `rolling_max_count`) to 1 +Set :ts:cv:`proxy.config.log.rolling_max_count` (yaml: `rolling_max_count`) to 1 which will lead to keeping only one rolled log file at any moment (rolled will be trimmed on every roll). -Set :ts:cv:`proxy.config.output.logfile.rolling_allow_empty` (yaml: `rolling_allow_empty`) to 1 (default: 0) +Set :ts:cv:`proxy.config.log.rolling_allow_empty` (yaml: `rolling_allow_empty`) to 1 (default: 0) which will allow logs to be open and rolled even if there was nothing to be logged during the previous period (i.e. no requests to |TS|). diff --git a/doc/admin-guide/logging/understanding.en.rst b/doc/admin-guide/logging/understanding.en.rst index 6cad223de5e..b789315068a 100644 --- a/doc/admin-guide/logging/understanding.en.rst +++ b/doc/admin-guide/logging/understanding.en.rst @@ -151,7 +151,7 @@ aforementioned aggregate functions and the specification of an interval, as so: formats: - name: mysummary - format: '% : %' + format: '% , %' interval: n The interval itself is given with *n* as the number of seconds for each period diff --git a/doc/admin-guide/monitoring/error-messages.en.rst b/doc/admin-guide/monitoring/error-messages.en.rst index 5e2400c5c32..94225695423 100644 --- a/doc/admin-guide/monitoring/error-messages.en.rst +++ b/doc/admin-guide/monitoring/error-messages.en.rst @@ -177,7 +177,7 @@ with corresponding HTTP response codes and customizable files. ``timeout#inactivity`` ``Content Length Required`` - ``400`` + ``411`` Could not process this request because ``Content-Length`` was not specified. ``request#no_content_length`` diff --git a/doc/admin-guide/monitoring/statistics/core/bandwidth.en.rst b/doc/admin-guide/monitoring/statistics/core/bandwidth.en.rst index 0b5113b8ce8..73f99665b70 100644 --- a/doc/admin-guide/monitoring/statistics/core/bandwidth.en.rst +++ b/doc/admin-guide/monitoring/statistics/core/bandwidth.en.rst @@ -22,7 +22,6 @@ Bandwidth and Transfer ********************** -.. ts:stat:: global proxy.process.http.throttled_proxy_only integer .. ts:stat:: global proxy.process.http.user_agent_request_document_total_size integer :type: counter :units: bytes diff --git a/doc/admin-guide/monitoring/statistics/core/http-connection.en.rst b/doc/admin-guide/monitoring/statistics/core/http-connection.en.rst index 411881fa104..42c680e1f4e 100644 --- a/doc/admin-guide/monitoring/statistics/core/http-connection.en.rst +++ b/doc/admin-guide/monitoring/statistics/core/http-connection.en.rst @@ -204,3 +204,45 @@ HTTP/2 Represents the total number of closed HTTP/2 connections with high error rate which is configured by :ts:cv:`proxy.config.http2.stream_error_rate_threshold`. + +.. ts:stat:: global proxy.process.http2.max_settings_per_frame_exceeded integer + :type: counter + + Represents the total number of closed HTTP/2 connections for exceeding the + maximum allowed number of settings per frame limit which is configured by + :ts:cv:`proxy.config.http2.max_settings_per_frame`. + +.. ts:stat:: global proxy.process.http2.max_settings_per_minute_exceeded integer + :type: counter + + Represents the total number of closed HTTP/2 connections for exceeding the + maximum allowed number of settings per minute limit which is configured by + :ts:cv:`proxy.config.http2.max_settings_per_minute`. + +.. ts:stat:: global proxy.process.http2.max_settings_frames_per_minute_exceeded integer + :type: counter + + Represents the total number of closed HTTP/2 connections for exceeding the + maximum allowed number of settings frames per minute limit which is configured by + :ts:cv:`proxy.config.http2.max_settings_frames_per_minute`. + +.. ts:stat:: global proxy.process.http2.max_ping_frames_per_minute_exceeded integer + :type: counter + + Represents the total number of closed HTTP/2 connections for exceeding the + maximum allowed number of ping frames per minute limit which is configured by + :ts:cv:`proxy.config.http2.max_ping_frames_per_minute`. + +.. ts:stat:: global proxy.process.http2.max_priority_frames_per_minute_exceeded integer + :type: counter + + Represents the total number of closed HTTP/2 connections for exceeding the + maximum allowed number of priority frames per minute limit which is configured by + :ts:cv:`proxy.config.http2.max_priority_frames_per_minute`. + +.. ts:stat:: global proxy.process.http2.insufficient_avg_window_update integer + :type: counter + + Represents the total number of closed HTTP/2 connections for not reaching the + minimum average window increment limit which is configured by + :ts:cv:`proxy.config.http2.min_avg_window_update`. diff --git a/doc/admin-guide/monitoring/statistics/core/network-io.en.rst b/doc/admin-guide/monitoring/statistics/core/network-io.en.rst index 5393086495a..5ea7687457b 100644 --- a/doc/admin-guide/monitoring/statistics/core/network-io.en.rst +++ b/doc/admin-guide/monitoring/statistics/core/network-io.en.rst @@ -60,7 +60,23 @@ Network I/O .. ts:stat:: global proxy.process.net.connections_currently_open integer :type: counter +.. ts:stat:: global proxy.process.net.connections_throttled_in integer + :type: counter + +.. ts:stat:: global proxy.process.net.connections_throttled_out integer + :type: counter + +.. ts:stat:: global proxy.process.net.max.requests_throttled_in integer + :type: counter + .. ts:stat:: global proxy.process.net.default_inactivity_timeout_applied integer + The total number of connections that had no transaction or connection level timer running on them and + had to fallback to the catch-all 'default_inactivity_timeout' + :type: counter +.. ts:stat:: global proxy.process.net.default_inactivity_timeout_count integer + The total number of connections that were cleaned up due to 'default_inactivity_timeout' + :type: counter + .. ts:stat:: global proxy.process.net.dynamic_keep_alive_timeout_in_count integer .. ts:stat:: global proxy.process.net.dynamic_keep_alive_timeout_in_total integer .. ts:stat:: global proxy.process.net.inactivity_cop_lock_acquire_failure integer diff --git a/doc/admin-guide/monitoring/statistics/core/origin.en.rst b/doc/admin-guide/monitoring/statistics/core/origin.en.rst index 2b325c4ab6d..b58ff3d5a25 100644 --- a/doc/admin-guide/monitoring/statistics/core/origin.en.rst +++ b/doc/admin-guide/monitoring/statistics/core/origin.en.rst @@ -78,4 +78,63 @@ Origin Server :type: derivative :units: bytes +.. ts:stat:: global proxy.process.http.origin_shutdown.pool_lock_contention integer + :type counter + :units bytes + +.. ts:stat:: global proxy.process.http.origin_shutdown.migration_failure integer + :type counter + :units bytes + +.. ts:stat:: global proxy.process.http.origin_shutdown.tunnel_server integer + :type counter + :units bytes + +.. ts:stat:: global proxy.process.http.origin_shutdown.tunnel_server_no_keep_alive integer + :type counter + :units bytes + +.. ts:stat:: global proxy.process.http.origin_shutdown.tunnel_server_eos integer + :type counter + :units bytes + +.. ts:stat:: global proxy.process.http.origin_shutdown.tunnel_server_plugin_tunnel integer + :type counter + :units bytes + +.. ts:stat:: global proxy.process.http.origin_shutdown.tunnel_transform_read integer + :type counter + :units bytes + +.. ts:stat:: global proxy.process.http.origin_shutdown.release_no_sharing integer + :type counter + :units bytes + +.. ts:stat:: global proxy.process.http.origin_shutdown.release_no_keep_alive integer + :type counter + :units bytes + +.. ts:stat:: global proxy.process.http.origin_shutdown.release_invalid_response integer + :type counter + :units bytes + +.. ts:stat:: global proxy.process.http.origin_shutdown.release_invalid_request integer + :type counter + :units bytes + +.. ts:stat:: global proxy.process.http.origin_shutdown.release_modified integer + :type counter + :units bytes + +.. ts:stat:: global proxy.process.http.origin_shutdown.release_misc integer + :type counter + :units bytes + +.. ts:stat:: global proxy.process.http.origin_shutdown.cleanup_entry integer + :type counter + :units bytes + +.. ts:stat:: global proxy.process.http.origin_shutdown.tunnel_abort integer + :type counter + :units bytes diff --git a/doc/admin-guide/monitoring/statistics/core/ssl.en.rst b/doc/admin-guide/monitoring/statistics/core/ssl.en.rst index 34e35a96b75..3b6ee48dbba 100644 --- a/doc/admin-guide/monitoring/statistics/core/ssl.en.rst +++ b/doc/admin-guide/monitoring/statistics/core/ssl.en.rst @@ -108,10 +108,28 @@ SSL/TLS The total amount of time spent performing SSL/TLS handshakes for new sessions since statistics collection began. -.. ts:stat:: global proxy.process.ssl.total_success_handshake_count integer +.. ts:stat:: global proxy.process.ssl.total_attempts_handshake_count_in integer :type: counter - The total number of SSL/TLS handshakes successfully performed since + The total number of inbound SSL/TLS handshake attempts received since + statistics collection began. + +.. ts:stat:: global proxy.process.ssl.total_success_handshake_count_in integer + :type: counter + + The total number of inbound SSL/TLS handshakes successfully performed since + statistics collection began. + +.. ts:stat:: global proxy.process.ssl.total_attempts_handshake_count_out integer + :type: counter + + The total number of outbound SSL/TLS handshake attempts made since + statistics collection began. + +.. ts:stat:: global proxy.process.ssl.total_success_handshake_count_out integer + :type: counter + + The total number of outbound SSL/TLS handshakes successfully performed since statistics collection began. .. ts:stat:: global proxy.process.ssl.total_ticket_keys_renewed integer diff --git a/doc/admin-guide/performance/index.en.rst b/doc/admin-guide/performance/index.en.rst index 64520ee03e7..623cdaef294 100644 --- a/doc/admin-guide/performance/index.en.rst +++ b/doc/admin-guide/performance/index.en.rst @@ -182,7 +182,7 @@ CPU and Thread Optimization Thread Scaling ~~~~~~~~~~~~~~ -By default, |TS| creates 1.5 threads per CPU core on the host system. This may +By default, |TS| creates one thread per CPU core on the host system. This may be adjusted with the following settings in :file:`records.config`: * :ts:cv:`proxy.config.exec_thread.autoconfig` @@ -327,7 +327,9 @@ Origin Connection Timeouts Origin server connection timeouts are configured with :ts:cv:`proxy.config.http.connect_attempts_timeout`, which is applied both to the initial connection as well as any retries attempted, should an attempt timeout. The timeout applies from the moment |TS| begins the -connection attempt until the origin returns the first byte. +connection attempt until the origin fully establishes a connection (the connection is ready to write). +After the connection is established the value of +:ts:cv:`proxy.config.http.transaction_no_activity_timeout_out` is used to established timeouts on the data over the connection. In the case where you wish to have a different (generally longer) timeout for ``POST`` and ``PUT`` connections to an origin server, you may also adjust diff --git a/doc/admin-guide/plugins/access_control.en.rst b/doc/admin-guide/plugins/access_control.en.rst index 2aefafbf83c..8fec83be080 100644 --- a/doc/admin-guide/plugins/access_control.en.rst +++ b/doc/admin-guide/plugins/access_control.en.rst @@ -237,7 +237,7 @@ Query-Param-Style Named Claim format * ``iat`` for `issued at time`_, `optional` * ``tid`` for `token id`_, `optional` * ``ver`` for `version`_, `optional`, defaults to ``ver=1`` if not specified. - * ``scope`` for `scope`_, `optional`, ignored by by the current version of the plugin, still not finalized (more applications and their use cases need to be studied to finalize the format) + * ``scope`` for `scope`_, `optional`, ignored by the current version of the plugin, still not finalized (more applications and their use cases need to be studied to finalize the format) * ``kid`` for `key id`_, `required` (tokens to be always signed) * ``st`` for `signature type`_, `optional` (default would be ``SHA-256`` if not specified) * ``md`` for `message digest`_ - this claim is `required` and expected to be always the last claim. @@ -379,16 +379,16 @@ Configuration files * Format the ``access_control.log`` -.. code-block:: bash - - access_control_format = format { - Format = '% sub=%<{@TokenSubject}cqh> tid=%<{@TokenId}cqh> status=%<{@TokenStatus}cqh> cache=%<{x-cache}psh> key=%<{x-cache-key}psh>' } - - log.ascii { - Filename = 'access_control', - Format = access_control_format - } +.. code-block:: yaml + logging: + formats: + - format: '% sub=%<{@TokenSubject}cqh> tid=%<{@TokenId}cqh> status=%<{@TokenStatus}cqh> cache=%<{x-cache}psh> key=%<{x-cache-key}psh>' + name: access_control_format + logs: + - filename: access_control + format: access_control_format + mode: ascii * X-Debug plugin added to ``plugin.config`` @@ -446,7 +446,7 @@ Now let us send the same request again and since the object is in cache we get ` x-cache: hit-fresh -The previous activity should result in the following log (as defined in ``logging.config``) +The previous activity should result in the following log (as defined in ``logging.yaml``) .. code-block:: bash diff --git a/doc/admin-guide/plugins/cache_promote.en.rst b/doc/admin-guide/plugins/cache_promote.en.rst index aa831f1d45b..ea882fb8dce 100644 --- a/doc/admin-guide/plugins/cache_promote.en.rst +++ b/doc/admin-guide/plugins/cache_promote.en.rst @@ -41,6 +41,13 @@ are available: If :option:`--policy` is set to ``lru`` the following options are also available: +.. option:: --label + + An optional label for this LRU, to allow sharing an LRU across multiple remap + rules. Note: In order for an LRU to be used by multiple remap rules, not only + must the label match, both the :option:`--hits` and :option:`--buckets` + options must be identical. + .. option:: --hits The minimum number of hits before promotion. @@ -49,6 +56,21 @@ If :option:`--policy` is set to ``lru`` the following options are also available The size (number of entries) of the LRU. +.. option:: --stats-enable-with-id + + Enables collecting statistics. The option requires an argument, the + remap-identifier. The remap-identifier is a string that is concatenated + to the stat name. The following stats are collected. + +* **plugin.cache_promote.${remap-identifier}.cache_hits** - Cache hit total, available for all policies. +* **plugin.cache_promote.${remap-identifier}.freelist_size** - Size of the freelist when using the LRU policy. +* **plugin.cache_promote.${remap-identifier}.lru_size** - Size of the LRU when using the LRU policy. +* **plugin.cache_promote.${remap-identifier}.lru_hit** - LRU hit count when using the LRU policy. +* **plugin.cache_promote.${remap-identifier}.lru_miss** - LRU miss count when using the LRU policy. +* **plugin.cache_promote.${remap-identifier}.lru_vacated** - count of LRU entries removed to make room for a new request. +* **plugin.cache_promote.${remap-identifier}.promoted** - count requests promoted, available in all policies. +* **plugin.cache_promote.${remap-identifier}.total_requests** - count of all requests. + These two options combined with your usage patterns will control how likely a URL is to become promoted to enter the cache. diff --git a/doc/admin-guide/plugins/cache_range_requests.en.rst b/doc/admin-guide/plugins/cache_range_requests.en.rst new file mode 100644 index 00000000000..6d11d21121a --- /dev/null +++ b/doc/admin-guide/plugins/cache_range_requests.en.rst @@ -0,0 +1,191 @@ +.. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + + +.. include:: ../../common.defs + +.. _admin-plugins-cache-range-requests: + + +Cache Range Requests Plugin +*************************** + +Description +=========== + +Most origin servers support HTTP/1.1 range requests (rfc 7233). +ATS internally handles range request caching in one of 2 ways: + +* Don't cache range requests. +* Only server range requests from a wholly cached object. + +This plugin allows you to remap individual range requests so that they +are stored as individual objects in the ATS cache when subsequent range +requests are likely to use the same range. This spreads range requests +over multiple stripes thereby reducing I/O wait and system load averages. + +:program:`cache_range_requests` reads the range request header byte range +value and then creates a new ``cache key URL`` using the original request +url with the range value appended to it. The range header is removed +where appropriate from the requests and the origin server response code +is changed from a 206 to a 200 to insure that the object is written to +cache using the new cache key url. The response code sent to the client +will be changed back to a 206 and all requests to the origin server will +contain the range header so that the correct response is received. + +The :program:`cache_range_requests` plugin by itself has no logic to +efficiently manage overlapping ranges. It is best to use this plugin +in conjunction with a smart client that only requests predetermined +non overlapping cache ranges (request blocking) or as a helper for the +:program:`slice` plugin. + +Only requests which contain the ``Range: =`` GET header +will be served by the :program:`cache_range_requests` plugin. + +If/when ATS implements partial object caching this plugin will +become deprecated. + +*NOTE* Given a multi range request the :program:`cache_range_requests` +only processes the first range and ignores the rest. + +How to run the plugin +===================== + +The plugin can run as a global plugin (a single global instance configured +using :file:`plugin.config`) or as per-remap plugin (a separate instance +configured per remap rule in :file:`remap.config`). + +Global instance +--------------- + +.. code:: + + $ cat plugin.config + cache_range_request.so + + +Per-remap instance +------------------ + +.. code:: + + $cat remap.config + map http://www.example.com http://www.origin.com \ + @plugin=cache_range_requests.so + + +If both global and per-remap instance are used the per-remap configuration +would take precedence (per-remap configuration would be applied and the +global configuration ignored). + +Plugin options +============== + + +Parent Selection as Cache Key +----------------------------- + +.. option:: --ps-cachekey +.. option:: -p + +Without this option parent selection is based solely on the hash of a +URL Path a URL is requested from the same upstream parent cache listed +in parent.config + + +With this option parent selection is based on the full ``cache key URL`` +which includes information about the partial content range. In this mode, +all requests (include partial content) will use consistent hashing method +for parent selection. + + +X-CRR-IMS header support +------------------------ + +.. option:: --consider-ims +.. option:: -c + +To support slice plugin self healing an option to force revalidation +after cache lookup complete was added. This option is triggered by a +special header: + +.. code:: + + X-CRR-IMS: Tue, 19 Nov 2019 13:26:45 GMT + +When this header is provided and a `cache hit fresh` is encoutered the +``Date`` header of the object in cache is compared to this header date +value. If the cache date is *less* than this IMS date then the object +is marked as STALE and an appropriate If-Modified-Since or If-Match +request along with this X-CRR-IMS header is passed up to the parent. + +In order for this to properly work in a CDN each cache in the +chain *SHOULD* also contain a remap rule with the +:program:`cache_range_requests` plugin with this option set. + +Don't modify the Cache Key +-------------------------- + +.. option:: --no-modify-cachekey +.. option:: -n + +With each transaction TSCacheUrlSet may only be called once. When +using the `cache_range_requests` plugin in conjunction with the +`cachekey` plugin the option `--include-headers=Range` should be +added as a `cachekey` parameter with this option. Configuring this +incorrectly *WILL* result in cache poisoning. + +.. code:: + + map http://ats/ http://parent/ \ + @plugin=cachekey.so @pparam=--include-headers=Range \ + @plugin=cache_range_requests.so @pparam=--no-modify-cachekey + +*Without this `cache_range_requests` plugin option* + +*IF* the TSCacheUrlSet call in cache_range_requests fails, an error is +generated in the logs and the cache_range_requests plugin will disable +transaction caching in order to avoid cache poisoning. + +Configuration examples +====================== + +Global plugin +------------- + +.. code:: + + cache_range_requests.so --ps-cachekey --consider-ims --no-modify-cachekey + +or + +.. code:: + + cache_range_requests.so -p -c -n + +Remap plugin +------------ + +.. code:: + + map http://ats http://parent @plugin=cache_range_requests.so @pparam=--ps-cachekey @pparam=--consider-ims @pparam=--no-modify-cachekey + +or + +.. code:: + + map http://ats http://parent @plugin=cache_range_requests.so @pparam=-p @pparam=-c @pparam=-n diff --git a/doc/admin-guide/plugins/cachekey.en.rst b/doc/admin-guide/plugins/cachekey.en.rst index 5342d8b52b8..406256f5cf1 100644 --- a/doc/admin-guide/plugins/cachekey.en.rst +++ b/doc/admin-guide/plugins/cachekey.en.rst @@ -21,22 +21,24 @@ .. _admin-plugins-cachekey: -Cache Key Manipulation Plugin -***************************** +Cache Key and Parent Selection URL Manipulation Plugin +****************************************************** Description =========== -This plugin allows some common cache key manipulations based on various HTTP request components. It can +This plugin allows some common `cache key` or `parent selection URL` manipulations based on various HTTP request components. +Although `cache key` is used everywhere in this document, the same manipulations can be applied to `parent selection URL` +by switching `key type`_. The plugin can * sort query parameters to prevent query parameter reordering being a cache miss -* ignore specific query parameters from the cache key by name or regular expression -* ignore all query parameters from the cache key -* only use specific query parameters in the cache key by name or regular expression +* ignore specific query parameters from the `cache key` by name or regular expression +* ignore all query parameters from the `cache key` +* only use specific query parameters in the `cache key` by name or regular expression * include headers or cookies by name * capture values from the ``User-Agent`` header. * classify request using ``User-Agent`` and a list of regular expressions -* capture and replace strings from the URI and include them in the cache key +* capture and replace strings from the URI and include them in the `cache key` * do more - please find more examples below. URI type @@ -46,6 +48,17 @@ The plugin manipulates the ``remap`` URI (value set during URI remap) by default * ``--uri-type=[remap|pristine]`` (default: ``remap``) +Key type +======== + +The plugin manipulates the `cache key` by default. If `parent selection URL` manipulation is needed the following option can be used: + +* ``--key-type=`` (default: ``cache_key``) - list of ``cache_key`` or ``parent_selection_url``, if multiple ``--key-type`` options are specified then all values are combined together. + +An instance of this plugin can be used for applying manipulations to `cache key`, `parent selection URL` or both depending on the need. See `simultaneous cache key and parent selection URL manipulation`_ +for examples of how to apply the **same** set of manipulations to both targets with a single plugin instance or applying **different** sets of manipulations to each target using separate plugin instances. + + Cache key structure and related plugin parameters ================================================= @@ -59,11 +72,11 @@ Cache key structure and related plugin parameters │ (default) | (optional) │ (optional) │ (optional) │ (default) │ (default) │ └─────────────┴──────────────┴──────────────┴──────────────┴─────────────┴─────────────┘ -* The cache key set by the cachekey plugin can be considered as divided into several sections. +* The `cache key` set by the cachekey plugin can be considered as divided into several sections. * Every section is manipulated separately by the related plugin parameters (more info in each section description below). -* "User-Agent", "Headers" and "Cookies" sections are optional and will be missing from the cache key if no related plugin parameters are used. +* "User-Agent", "Headers" and "Cookies" sections are optional and will be missing from the `cache key` if no related plugin parameters are used. * "Prefix", "Path" and "Query" sections always have default values even if no related plugin parameters are used. -* All cachekey plugin parameters are optional and if missing some of the cache key sections will be missing (the optional sections) or their values will be left to their defaults. +* All cachekey plugin parameters are optional and if missing some of the `cache key` sections will be missing (the optional sections) or their values will be left to their defaults. "Prefix" section ^^^^^^^^^^^^^^^^ @@ -71,19 +84,29 @@ Cache key structure and related plugin parameters :: Optional components | ┌─────────────────┬──────────────────┬──────────────────────┐ - (included in this order) | │ --static-prefix | --capture-prefix │ --capture-prefix-uri │ + (included in this order) | │ --static-prefix │ --capture-prefix │ --capture-prefix-uri │ | ├─────────────────┴──────────────────┴──────────────────────┤ - Default values if no | │ /host/port | + Default values if no | │ /host/port or scheme://host:port (see the table below) │ optional components | └───────────────────────────────────────────────────────────┘ configured | -* ``--static-prefix=`` (default: empty string) - if specified and not an empty string the ```` will be added to the cache key. -* ``--capture-prefix=`` (default: empty string) - if specified and not empty then strings are captured from ``host:port`` based on the ```` and are added to the cache key. -* ``--capture-prefix-uri=`` (default: empty string) - if specified and not empty then strings are captured from the entire URI based on the ```` and are added to the cache key. -* If any of the "Prefix" related plugin parameters are used together in the plugin configuration they are added to the cache key in the order shown in the diagram. -* ``--remove-prefix=`` (default: empty string) - if specified and not an empty string the ```` will be added to the `cache key`. +* ``--capture-prefix=`` (default: empty string) - if specified and not empty then strings are captured from ``host:port`` based on the ```` and are added to the `cache key`. +* ``--capture-prefix-uri=`` (default: empty string) - if specified and not empty then strings are captured from the entire URI based on the ```` and are added to the `cache key`. +* If any of the "Prefix" related plugin parameters are used together in the plugin configuration they are added to the `cache key` in the order shown in the diagram. +* ``--remove-prefix=:`` (default: empty string) - loads a regex patterns list from a file ````, the patterns are matched against the ``User-Agent`` header and if matched ```` is added it to the key. - * ``--ua-blacklist=:`` (default: empty string) - loads a regex patterns list from a file ````, the patterns are matched against the ``User-Agent`` header and if **not** matched ```` is added it to the key. - * Multiple ``--ua-whitelist`` and ``--ua-blacklist`` can be used and the result will be defined by their order in the plugin configuration. + * ``--ua-allowlist=:`` (default: empty string) - loads a regex patterns list from a file ````, the patterns are matched against the ``User-Agent`` header and if matched ```` is added it to the key. + * ``--ua-denylist=:`` (default: empty string) - loads a regex patterns list from a file ````, the patterns are matched against the ``User-Agent`` header and if **not** matched ```` is added it to the key. + * Multiple ``--ua-allowlist`` and ``--ua-denylist`` can be used and the result will be defined by their order in the plugin configuration. * ``User-Agent`` regex capturing and replacement - * ``--ua-capture=`` (default: empty string) - if specified and not empty then strings are captured from the ``User-Agent`` header based on ```` (see below) and are added to the cache key. -* If any ``User-Agent`` classification and regex capturing and replacement plugin parameters are used together they are added to the cache key in the order shown in the diagram. + * ``--ua-capture=`` (default: empty string) - if specified and not empty then strings are captured from the ``User-Agent`` header based on ```` (see below) and are added to the `cache key`. +* If any ``User-Agent`` classification and regex capturing and replacement plugin parameters are used together they are added to the `cache key` in the order shown in the diagram. "Headers" section ^^^^^^^^^^^^^^^^^ @@ -112,14 +135,14 @@ Cache key structure and related plugin parameters Optional components | ┌───────────────────┬───────────────────┐ | │ --include-headers │ --capture-header │ - | ├───────────────────────────────────────┤ - Default values if no | │ (empty) | (empty) | + | ├───────────────────┼───────────────────┤ + Default values if no | │ (empty) │ (empty) │ optional components | └───────────────────┴───────────────────┘ configured | -* ``--include-headers`` (default: empty list) - comma separated list of headers to be added to the cache key. The list of headers defined by ``--include-headers`` are always sorted before adding them to the cache key. +* ``--include-headers`` (default: empty list) - comma separated list of headers to be added to the `cache key`. The list of headers defined by ``--include-headers`` are always sorted before adding them to the `cache key`. -* ``--capture-header=:`` (default: empty) - captures elements from header using and adds them to the cache key. +* ``--capture-header=:`` (default: empty) - captures elements from header using and adds them to the `cache key`. "Cookies" section ^^^^^^^^^^^^^^^^^ @@ -129,11 +152,11 @@ Cache key structure and related plugin parameters Optional components | ┌───────────────────┐ | │ --include-cookies │ | ├───────────────────┤ - Default values if no | │ (empty) | + Default values if no | │ (empty) │ optional components | └───────────────────┘ configured | -* ``--include-cookies`` (default: empty list) - comma separated list of cookies to be added to the cache key. The list of cookies defined by ``--include-cookies`` are always sorted before adding them to the cache key. +* ``--include-cookies`` (default: empty list) - comma separated list of cookies to be added to the `cache key`. The list of cookies defined by ``--include-cookies`` are always sorted before adding them to the `cache key`. "Path" section ^^^^^^^^^^^^^^ @@ -141,25 +164,25 @@ Cache key structure and related plugin parameters :: Optional components | ┌────────────────────┬────────────────┐ - (included in this order) | │ --capture-path-uri | --capture-path │ + (included in this order) | │ --capture-path-uri │ --capture-path │ | ├────────────────────┴────────────────┤ - Default values if no | │ URI path | + Default values if no | │ URI path │ optional components | └─────────────────────────────────────┘ configured | -* if no path related plugin parameters are used, the URI path string is included in the cache key. -* ``--capture-path=`` (default: empty string) - if specified and not empty then strings are captured from URI path based on the ```` and are added to the cache key. -* ``--capture-path-uri=`` (default: empty string) - if specified and not empty then strings are captured from the entire URI based on the ```` and are added to the cache key. +* if no path related plugin parameters are used, the URI path string is included in the `cache key`. +* ``--capture-path=`` (default: empty string) - if specified and not empty then strings are captured from URI path based on the ```` and are added to the `cache key`. +* ``--capture-path-uri=`` (default: empty string) - if specified and not empty then strings are captured from the entire URI based on the ```` and are added to the `cache key`. * ``--remove-path=`` can be in the following formats - * ```` - ```` defines regex capturing groups, up to 10 captured strings based on ```` will be added to the cache key. - * ``///`` - ```` defines regex capturing groups, ```` defines a pattern where the captured strings referenced with ``$0`` ... ``$9`` will be substituted and the result will be added to the cache key. + * ```` - ```` defines regex capturing groups, up to 10 captured strings based on ```` will be added to the `cache key`. + * ``///`` - ```` defines regex capturing groups, ```` defines a pattern where the captured strings referenced with ``$0`` ... ``$9`` will be substituted and the result will be added to the `cache key`. Cache key elements separator ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -* ``--separator=`` - the cache key is constructed by extracting elements from HTTP URI and headers or by using the UA classifiers and they are appended during the key construction and separated by ``/`` (by default). This options allows to override the default separator to any string (including an empty string). +* ``--separator=`` - the `cache key` is constructed by extracting elements from HTTP URI and headers or by using the UA classifiers and they are appended during the key construction and separated by ``/`` (by default). This options allows to override the default separator to any string (including an empty string). How to run the plugin @@ -241,7 +264,7 @@ Traffic server configuration :: $ cat etc/trafficserver/remap.config map http://www.example.com http://www.origin.com \ @plugin=cachekey.so \ - @pparam=--ua-whitelist=popular:popular_agents.config \ + @pparam=--ua-allowlist=popular:popular_agents.config \ @pparam=--ua-capture=(Mozilla\/[^\s]*).* \ @pparam=--include-headers=H1,H2 \ @pparam=--include-cookies=C1,C2 \ @@ -291,13 +314,13 @@ HTTP request :: * Connection #0 to host 127.0.0.1 left intact * Closing connection #0 -The response header ``X-Cache-Key`` header contains the cache key: :: +The response header ``X-Cache-Key`` header contains the `cache key`: :: /www.example.com/80/popular/Mozilla/5.0/H1:v1/H2:v2/C1=v1;C2=v2/path/to/data?a=1&b=2&c=3 -The ``xdebug.so`` plugin and ``X-Debug`` request header are used just to demonstrate basic cache key troubleshooting. +The ``xdebug.so`` plugin and ``X-Debug`` request header are used just to demonstrate basic `cache key` troubleshooting. -If we add ``--static-prefix=nice_custom_prefix`` to the remap rule then the cache key would look like the following: :: +If we add ``--static-prefix=nice_custom_prefix`` to the remap rule then the `cache key` would look like the following: :: /nice_custom_prefix/popular/Mozilla/5.0/H1:v1/H2:v2/C1=v1;C2=v2/path/to/data?a=1&b=2&c=3 @@ -309,52 +332,52 @@ URI query parameters Ignore the query string (all query parameters) """""""""""""""""""""""""""""""""""""""""""""" -The following added to the remap rule will ignore the query, removing it from the cache key. :: +The following added to the remap rule will ignore the query, removing it from the `cache key`. :: @plugin=cachekey.so @pparam=--remove-all-params=true Cache key normalization by sorting the query parameters """"""""""""""""""""""""""""""""""""""""""""""""""""""" -The following will normalize the cache key by sorting the query parameters. :: +The following will normalize the `cache key` by sorting the query parameters. :: @plugin=cachekey.so @pparam=--sort-params=true -If the URI has the following query string ``c=1&a=1&b=2&x=1&k=1&u=1&y=1`` the cache key will use ``a=1&b=2&c=1&k=1&u=1&x=1&y=1`` +If the URI has the following query string ``c=1&a=1&b=2&x=1&k=1&u=1&y=1`` the `cache key` will use ``a=1&b=2&c=1&k=1&u=1&x=1&y=1`` Ignore (exclude) certain query parameters """"""""""""""""""""""""""""""""""""""""" -The following will make sure query parameters `a` and `b` will **not** be used when constructing the cache key. :: +The following will make sure query parameters `a` and `b` will **not** be used when constructing the `cache key`. :: @plugin=cachekey.so @pparam=--exclude-params=a,b -If the URI has the following query string ``c=1&a=1&b=2&x=1&k=1&u=1&y=1`` the cache key will use ``c=1&x=1&k=1&u=1&y=1`` +If the URI has the following query string ``c=1&a=1&b=2&x=1&k=1&u=1&y=1`` the `cache key` will use ``c=1&x=1&k=1&u=1&y=1`` Ignore (exclude) certain query parameters from the cache key by using regular expression (PCRE) """"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" -The following will make sure query parameters ``a`` and ``b`` will **not** be used when constructing the cache key. :: +The following will make sure query parameters ``a`` and ``b`` will **not** be used when constructing the `cache key`. :: @plugin=cachekey.so @pparam=--exclude-match-params=(a|b) -If the URI has the following query string ``c=1&a=1&b=2&x=1&k=1&u=1&y=1`` the cache key will use ``c=1&x=1&k=1&u=1&y=1`` +If the URI has the following query string ``c=1&a=1&b=2&x=1&k=1&u=1&y=1`` the `cache key` will use ``c=1&x=1&k=1&u=1&y=1`` Include only certain query parameters """"""""""""""""""""""""""""""""""""" -The following will make sure only query parameters `a` and `c` will be used when constructing the cache key and the rest will be ignored. :: +The following will make sure only query parameters `a` and `c` will be used when constructing the `cache key` and the rest will be ignored. :: @plugin=cachekey.so @pparam=--include-params=a,c -If the URI has the following query string ``c=1&a=1&b=2&x=1&k=1&u=1&y=1`` the cache key will use ``c=1&a=1`` +If the URI has the following query string ``c=1&a=1&b=2&x=1&k=1&u=1&y=1`` the `cache key` will use ``c=1&a=1`` Include only certain query parameters by using regular expression (PCRE) """""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" -The following will make sure only query parameters ``a`` and ``c`` will be used when constructing the cache key and the rest will be ignored. :: +The following will make sure only query parameters ``a`` and ``c`` will be used when constructing the `cache key` and the rest will be ignored. :: @plugin=cachekey.so @pparam=--include-match-params=(a|c) -If the URI has the following query string ``c=1&a=1&b=2&x=1&k=1&u=1&y=1`` the cache key will use ``c=1&a=1`` +If the URI has the following query string ``c=1&a=1&b=2&x=1&k=1&u=1&y=1`` the `cache key` will use ``c=1&a=1`` -White-list + black-list certain parameters using multiple parameters in the same remap rule. +Include and exclude certain parameters using multiple parameters in the same remap rule. """""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" If the plugin is used with the following plugin parameters in the remap rule: :: @@ -365,9 +388,9 @@ If the plugin is used with the following plugin parameters in the remap rule: :: @pparam=--include-params=y,c \ @pparam=--include-params=x,b -and if the URI has the following query string ``c=1&a=1&b=2&x=1&k=1&u=1&y=1`` the cache key will use ``c=1&b=1`` +and if the URI has the following query string ``c=1&a=1&b=2&x=1&k=1&u=1&y=1`` the `cache key` will use ``c=1&b=1`` -White-list + black-list certain parameters using multiple parameters in the same remap rule and regular expressions (PCRE). +Include and exclude certain parameters using multiple parameters in the same remap rule and regular expressions (PCRE). """"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" If the plugin is used with the following plugin parameters in the remap rule: :: @@ -378,7 +401,7 @@ If the plugin is used with the following plugin parameters in the remap rule: :: @pparam=--include-match-params=(y|c) \ @pparam=--include-match-params=(x|b) -and if the URI has the following query string ``c=1&a=1&b=2&x=1&k=1&u=1&y=1`` the cache key will use ``c=1&b=1`` +and if the URI has the following query string ``c=1&a=1&b=2&x=1&k=1&u=1&y=1`` the `cache key` will use ``c=1&b=1`` Mixing --include-params, --exclude-params, --include-match-param and --exclude-match-param """""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" @@ -391,18 +414,18 @@ If the plugin is used with the following plugin parameters in the remap rule: :: @pparam=--include-params=y,c \ @pparam=--include-match-params=(x|b) -and if the URI has the following query string ``c=1&a=1&b=2&x=1&k=1&u=1&y=1`` the cache key will use ``c=1&b=1`` +and if the URI has the following query string ``c=1&a=1&b=2&x=1&k=1&u=1&y=1`` the `cache key` will use ``c=1&b=1`` HTTP Headers ^^^^^^^^^^^^ Include certain headers in the cache key """""""""""""""""""""""""""""""""""""""" -The following headers ``HeaderA`` and ``HeaderB`` will be used when constructing the cache key and the rest will be ignored. :: +The following headers ``HeaderA`` and ``HeaderB`` will be used when constructing the `cache key` and the rest will be ignored. :: @plugin=cachekey.so @pparam=--include-headers=HeaderA,HeaderB -The following would capture from the ``Authorization`` header and will add the captured element to the cache key :: +The following would capture from the ``Authorization`` header and will add the captured element to the `cache key` :: @plugin=cachekey.so \ @pparam=--capture-header=Authorization:/AWS\s(?[^:]+).*/clientID:$1/ @@ -412,7 +435,7 @@ If the request looks like the following:: http://example-cdn.com/path/file Authorization: AWS MKIARYMOG51PT0DLD:DLiWQ2lyS49H4Zyx34kW0URtg6s= -Cache key would be set to:: +The `cache key` would be set to:: /example-cdn.com/80/clientID:MKIARYMOG51PTCKQ0DLD/path/file @@ -423,7 +446,7 @@ HTTP Cookies Include certain cookies in the cache key """""""""""""""""""""""""""""""""""""""" -The following headers ``CookieA`` and ``CookieB`` will be used when constructing the cache key and the rest will be ignored. :: +The following headers ``CookieA`` and ``CookieB`` will be used when constructing the `cache key` and the rest will be ignored. :: @plugin=cachekey.so @pparam=--include-headers=CookieA,CookieB @@ -437,7 +460,7 @@ If the plugin is used with the following plugin parameter in the remap rule. :: @plugin=cachekey.so @pparam=--static-prefix=static_prefix -the cache key will be prefixed with ``/static_prefix`` instead of ``host:port`` when ``--static-prefix`` is not used. +the `cache key` will be prefixed with ``/static_prefix`` instead of ``host:port`` when ``--static-prefix`` is not used. Capturing from the host:port and adding it to the prefix section """""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" @@ -446,7 +469,7 @@ If the plugin is used with the following plugin parameter in the remap rule. :: @plugin=cachekey.so \ @pparam=--capture-prefix=(test_prefix).*:([^\s\/$]*) -the cache key will be prefixed with ``/test_prefix/80`` instead of ``test_prefix_371.example.com:80`` when ``--capture-prefix`` is not used. +the `cache key` will be prefixed with ``/test_prefix/80`` instead of ``test_prefix_371.example.com:80`` when ``--capture-prefix`` is not used. Capturing from the entire URI and adding it to the prefix section """"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" @@ -459,7 +482,7 @@ and if the request URI is the following :: http://test_prefix_123.example.com/path/to/object?a=1&b=2&c=3 -the the cache key will be prefixed with ``/test_prefix_object`` instead of ``test_prefix_123.example.com:80`` when ``--capture-prefix-uri`` is not used. +the `cache key` will be prefixed with ``/test_prefix_object`` instead of ``test_prefix_123.example.com:80`` when ``--capture-prefix-uri`` is not used. Combining prefix plugin parameters, i.e. --static-prefix and --capture-prefix """"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" @@ -469,7 +492,7 @@ If the plugin is used with the following plugin parameters in the remap rule. :: @pparam=--capture-prefix=(test_prefix).*:([^\s\/$]*) \ @pparam=--static-prefix=static_prefix -the cache key will be prefixed with ``/static_prefix/test_prefix/80`` instead of ``test_prefix_371.example.com:80`` when either ``--capture-prefix`` nor ``--static-prefix`` are used. +the `cache key` will be prefixed with ``/static_prefix/test_prefix/80`` instead of ``test_prefix_371.example.com:80`` when either ``--capture-prefix`` nor ``--static-prefix`` are used. Path, capture and replace from the path or entire URI @@ -487,7 +510,7 @@ and the request URI is the following :: http://test_path_123.example.com/path/to/object?a=1&b=2&c=3 -then the cache key will have ``/const_path_object`` in the path section of the cache key instead of ``/path/to/object`` when either ``--capture-path`` nor ``--capture-path-uri`` are used. +then the `cache key` will have ``/const_path_object`` in the path section of the `cache key` instead of ``/path/to/object`` when either ``--capture-path`` nor ``--capture-path-uri`` are used. Capture and replace groups from whole URI for the "Path" section @@ -502,7 +525,7 @@ and the request URI is the following :: http://test_path_123.example.com/path/to/object?a=1&b=2&c=3 -the the cache key will have ``/test_path_object`` in the path section of the cache key instead of ``/path/to/object`` when either ``--capture-path`` nor ``--capture-path-uri`` are used. +the `cache key` will have ``/test_path_object`` in the path section of the `cache key` instead of ``/path/to/object`` when either ``--capture-path`` nor ``--capture-path-uri`` are used. Combining path plugin parameters --capture-path and --capture-path-uri @@ -518,7 +541,7 @@ and the request URI is the following :: http://test_path_123.example.com/path/to/object?a=1&b=2&c=3 -the the cache key will have ``/test_path_object/const_path_object`` in the path section of the cache key instead of ``/path/to/object`` when either ``--capture-path`` nor ``--capture-path-uri`` are used. +the `cache key` will have ``/test_path_object/const_path_object`` in the path section of the `cache key` instead of ``/path/to/object`` when either ``--capture-path`` nor ``--capture-path-uri`` are used. User-Agent capturing, replacement and classification ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -547,12 +570,12 @@ If the plugin is used with the following plugin parameter:: then ``Mozilla/5.0_AppleWebKit/537.75.14`` will be used when constructing the key. -User-Agent white-list classifier +User-Agent allow-list classifier """""""""""""""""""""""""""""""" If the plugin is used with the following plugin parameter:: @plugin=cachekey.so \ - @pparam=--ua-whitelist=browser:browser_agents.config + @pparam=--ua-allowlist=browser:browser_agents.config and if ``browser_agents.config`` contains: :: @@ -562,12 +585,12 @@ and if ``browser_agents.config`` contains: :: then ``browser`` will be used when constructing the key. -User-Agent black-list classifier +User-Agent deny-list classifier """""""""""""""""""""""""""""""" If the plugin is used with the following plugin parameter:: @plugin=cachekey.so \ - @pparam=--ua-blacklist=browser:tool_agents.config + @pparam=--ua-denylist=browser:tool_agents.config and if ``tool_agents.config`` contains: :: @@ -580,16 +603,16 @@ then ``browser`` will be used when constructing the key. Cacheurl plugin to cachekey plugin migration ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -The plugin `cachekey` was not meant to replace the cacheurl plugin in terms of having exactly the same cache key strings generated. It just allows the operator to extract elements from the HTTP URI in the same way the `cacheurl` does (through a regular expression, please see `` above). +The plugin `cachekey` was not meant to replace the cacheurl plugin in terms of having exactly the same `cache key` strings generated. It just allows the operator to extract elements from the HTTP URI in the same way the `cacheurl` does (through a regular expression, please see `` above). -The following examples demonstrate different ways to achieve `cacheurl` compatibility on a cache key string level in order to avoid invalidation of the cache. +The following examples demonstrate different ways to achieve `cacheurl` compatibility on a `cache key` string level in order to avoid invalidation of the cache. The operator could use `--capture-path-uri`, `--capture-path`, `--capture-prefix-uri`, `--capture-prefix` to capture elements from the URI, path and authority elements. -By using `--separator=` the operator could override the default separator to an empty string `--separator=` and thus make sure there are no cache key element separators. +By using `--separator=` the operator could override the default separator to an empty string `--separator=` and thus make sure there are no `cache key` element separators. -Example 1: Let us say we have a capture definition used in `cacheurl`. Now by using `--capture-prefix-uri` one could extract elements through the same capture definition used with `cacheurl`, remove the cache key element separator `--separator=` and by using `--capture-path-uri` could remove the URI path and by using `--remove-all-params=true` could remove the query string:: +Example 1: Let us say we have a capture definition used in `cacheurl`. Now by using `--capture-prefix-uri` one could extract elements through the same capture definition used with `cacheurl`, remove the `cache key` element separator `--separator=` and by using `--capture-path-uri` could remove the URI path and by using `--remove-all-params=true` could remove the query string:: @plugin=cachekey.so \ @pparam=--capture-prefix-uri=/.*/$0/ \ @@ -597,7 +620,7 @@ Example 1: Let us say we have a capture definition used in `cacheurl`. Now by us @pparam=--remove-all-params=true \ @pparam=--separator= -Example 2: A more efficient way would be achieved by using `--capture-prefix-uri` to capture from the URI, remove the cache key element separator `--separator=` and by using `--remove-path` to remove the URI path and `--remove-all-params=true` to remove the query string:: +Example 2: A more efficient way would be achieved by using `--capture-prefix-uri` to capture from the URI, remove the `cache key` element separator `--separator=` and by using `--remove-path` to remove the URI path and `--remove-all-params=true` to remove the query string:: @plugin=cachekey.so \ @pparam=--capture-prefix-uri=/.*/$0/ \ @@ -605,7 +628,7 @@ Example 2: A more efficient way would be achieved by using `--capture-prefix-uri @pparam=--remove-all-params=true \ @pparam=--separator= -Example 3: Same result as the above but this time by using `--capture-path-uri` to capture from the URI, remove the cache key element separator `--separator=` and by using `--remove-prefix` to remove the URI authority elements and by using `--remove-all-params=true` to remove the query string:: +Example 3: Same result as the above but this time by using `--capture-path-uri` to capture from the URI, remove the `cache key` element separator `--separator=` and by using `--remove-prefix` to remove the URI authority elements and by using `--remove-all-params=true` to remove the query string:: @plugin=cachekey.so \ @pparam=--capture-path-uri=/(.*)/$0/ \ @@ -620,3 +643,41 @@ Example 4: Let us say that we would like to capture from URI in similar to `cach @pparam=--remove-path=true \ @pparam=--sort-params=true \ @pparam=--separator= + + +Simultaneous cache key and parent selection URL manipulation +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The following is an example of how to manipulate both `cache key` and `parent selection URL` in the same remap rule. +For this purpose two separate instances are loaded for that remap rule: + +:: + + @plugin=cachekey.so \ + @pparam=--key-type=parent_selection_url \ + @pparam=--static-prefix=this://goes.to/parent/selection/url \ + @pparam=--canonical-prefix=true \ + @plugin=cachekey.so \ + @pparam=--key-type=cache_key \ + @pparam=--static-prefix=this://goes.to/cache/key \ + @pparam=--canonical-prefix=true + +In the example above the first instance of the plugin sets the prefix to the parent selection URI and +the second instance of the plugin sets the prefix to the cache key. + +The **same** string manipulations can be applied to both cache key and parent selection url more concisely without chaining cachekey plugin instances by specifying multiple target types `--key-type`. + +Instead of:: + + @plugin=cachekey.so \ + @pparam=--key-type=parent_selection_url \ + @pparam=--remove-all-params=true + @plugin=cachekey.so \ + @pparam=--key-type=cache_key \ + @pparam=--remove-all-params=true + +one could write:: + + @plugin=cachekey.so \ + @pparam=--key-type=parent_selection_url,cache_key \ + @pparam=--remove-all-params=true diff --git a/doc/admin-guide/plugins/collapsed_forwarding.en.rst b/doc/admin-guide/plugins/collapsed_forwarding.en.rst index f479da32f78..8f4024acc15 100644 --- a/doc/admin-guide/plugins/collapsed_forwarding.en.rst +++ b/doc/admin-guide/plugins/collapsed_forwarding.en.rst @@ -79,14 +79,14 @@ on the error by using an internal redirect follow back to itself, essentially blocking the request until a response arrives, at which point, relies on read-while-writer feature to start downloading the object to all waiting clients. The following config parameters are assumed to be set for this -plugin to work:: +plugin to work: -:ts:cv:`proxy.config.http.cache.open_write_fail_action` 1 -:ts:cv:`proxy.config.cache.enable_read_while_writer` 1 -:ts:cv:`proxy.config.http.number_of_redirections` 10 -:ts:cv:`proxy.config.http.redirect_use_orig_cache_key` 1 -:ts:cv:`proxy.config.http.background_fill_active_timeout` 0 -:ts:cv:`proxy.config.http.background_fill_completed_threshold` 0 +- :ts:cv:`proxy.config.http.cache.open_write_fail_action` ``1`` +- :ts:cv:`proxy.config.cache.enable_read_while_writer` ``1`` +- :ts:cv:`proxy.config.http.number_of_redirections` ``10`` +- :ts:cv:`proxy.config.http.redirect_use_orig_cache_key` ``1`` +- :ts:cv:`proxy.config.http.background_fill_active_timeout` ``0`` +- :ts:cv:`proxy.config.http.background_fill_completed_threshold` ``0`` Additionally, given that collapsed forwarding works based on cache write lock failure detection, the plugin requires cache to be enabled and ready. @@ -115,12 +115,12 @@ Read-While-Writer (RWW), Stale-While-Revalidate (SWR) etc each very effective dealing with a majority of the use cases that can result in the Thundering herd problem. -For a large scale Video Streaming scenario, there’s a combination of a +For a large scale Video Streaming scenario, there's a combination of a large number of revalidations (e.g. media playlists) and cache misses -(e.g. media segments) that occur for the same file. Traffic Server’s +(e.g. media segments) that occur for the same file. Traffic Server's RWW works great in collapsing the concurrent requests in such a scenario, -however, as described in ``_admin-configuration-reducing-origin-requests``, -Traffic Server’s implementation of RWW has a significant limitation, which +however, as described in :ref:`admin-configuration-reducing-origin-requests`, +Traffic Server's implementation of RWW has a significant limitation, which restricts its ability to invoke RWW only when the response headers are already received. This means that any number of concurrent requests for the same file that are received before the response headers arrive are @@ -143,7 +143,7 @@ the Other hand, if the lookup is successful, meaning, the dirent exists for the generated cache key, Traffic Server tries to obtain a read lock on the cache object to be able to serve it from the cache. If the read lock is not successful (possibly, due to the fact that -the object’s being written to at that same instant and the response +the object's being written to at that same instant and the response headers are not in the cache yet), Traffic Server then moves to the next step of trying to obtain an exclusive write lock. If the write lock is already held exclusively by another request (transaction), the @@ -185,6 +185,4 @@ retries, allowing to be able to initiate RWW, whenever the response headers are received for the request that was allowed to go to the Origin. -More details are available at - -https://docs.trafficserver.apache.org/en/6.0.x/admin/http-proxy-caching.en.html#reducing-origin-server-requests-avoiding-the-thundering-herd +More details are available at :ref:`admin-configuration-reducing-origin-requests` \ No newline at end of file diff --git a/doc/admin-guide/plugins/combo_handler.en.rst b/doc/admin-guide/plugins/combo_handler.en.rst index 528704d30e1..926c5188da9 100644 --- a/doc/admin-guide/plugins/combo_handler.en.rst +++ b/doc/admin-guide/plugins/combo_handler.en.rst @@ -47,6 +47,16 @@ The arguments in the :file:`plugin.config` line in order represent 2. The name of the key used for signature verification (disabled by default) +3. A colon separated list of headers which, if present on at least one response, will be + added to the combo response. + +4. The path of a config file with allowed content types of objects to be combined, one per + line, without parameters. (Blank lines and comments starting with "#" are ignored.) + Parameters in the Content-Type field value will be ignored when + checking if they appear in the allowed types. If the path does not start with "/", the + config file must be located in the ATS config directory. By default, all content types + are allowed, but if this file is specified, it must contain at least one content type. + A "-" can be supplied as a value for any of these arguments to request default value be applied. diff --git a/doc/admin-guide/plugins/compress.en.rst b/doc/admin-guide/plugins/compress.en.rst index a355d33cdf8..41cf6a6278d 100644 --- a/doc/admin-guide/plugins/compress.en.rst +++ b/doc/admin-guide/plugins/compress.en.rst @@ -76,6 +76,8 @@ With no further options, this will enable the following default behavior: * Disable flush (flush compressed content to client). +* Only objects greater than 1Kb will be compressed + Alternatively, a configuration may be specified (shown here using the sample configuration provided with the plugin's source):: @@ -216,6 +218,25 @@ might create a configuration with the following options:: [bar.example.com] enabled false + # A reasonable list of content-types that are compressible + compressible-content-type text/* + compressible-content-type *font* + compressible-content-type *javascript + compressible-content-type *json + compressible-content-type *ml;* + compressible-content-type *mpegURL + compressible-content-type *mpegurl + compressible-content-type *otf + compressible-content-type *ttf + compressible-content-type *type + compressible-content-type *xml + compressible-content-type application/eot + compressible-content-type application/pkix-crl + compressible-content-type application/x-httpd-cgi + compressible-content-type application/x-perl + compressible-content-type image/vnd.microsoft.icon + compressible-content-type image/x-icon + Assuming the above options are in a file at ``/etc/trafficserver/compress.config`` the plugin would be enabled for |TS| in :file:`plugin.config` as:: diff --git a/doc/admin-guide/plugins/esi.en.rst b/doc/admin-guide/plugins/esi.en.rst index 0a8b521cfd4..48c3f8147fc 100644 --- a/doc/admin-guide/plugins/esi.en.rst +++ b/doc/admin-guide/plugins/esi.en.rst @@ -98,14 +98,14 @@ And inside handler.conf you can provide the list of cookie name that is allowed. :: - whitelistCookie A - whitelistCookie LOGIN + allowlistCookie A + allowlistCookie LOGIN We can also allow all cookie for HTTP_COOKIE variable by using a wildcard character. e.g. :: - whitelistCookie * + allowlistCookie * 4. We need a mapping for origin server response that contains the ESI markup. Assume that the ATS server is abc.com. And your origin server is xyz.com and the response containing ESI markup is http://xyz.com/esi.php. We will need the following line in /usr/local/etc/trafficserver/remap.config @@ -152,8 +152,8 @@ Useful Note 1. You can provide proper cache control header and the ESI response and ESI include response can be cached separately. It is extremely useful for rendering page with multiple modules. The page layout can be a ESI response with multiple - ESI include include, each for different module. The page layout ESI response can be cached and each individual ESI - include can also be cached with different duration. + ESI includes, each for a different module. The page layout ESI response can be cached and each individual ESI + included can also be cached with a different duration. 2. You should run the plugin without using "packed node support" because it is not fully tested. diff --git a/doc/admin-guide/plugins/geoip_acl.en.rst b/doc/admin-guide/plugins/geoip_acl.en.rst index 13d33683a7f..9444163a2a7 100644 --- a/doc/admin-guide/plugins/geoip_acl.en.rst +++ b/doc/admin-guide/plugins/geoip_acl.en.rst @@ -22,7 +22,7 @@ GeoIP ACLs Plugin This is a simple ATS plugin for denying (or allowing) requests based on the source IP geo-location. Currently only the Maxmind APIs are -supported, but we'd be happy to other other (open) APIs if you let us +supported, but we'd be happy to other (open) APIs if you let us know. This plugin comes with the standard distribution of Apache Traffic Server, and should be installed as part of the normal build process. diff --git a/doc/admin-guide/plugins/header_rewrite.en.rst b/doc/admin-guide/plugins/header_rewrite.en.rst index e5951e7f56a..0ca0e3b6a06 100644 --- a/doc/admin-guide/plugins/header_rewrite.en.rst +++ b/doc/admin-guide/plugins/header_rewrite.en.rst @@ -193,7 +193,7 @@ COOKIE cond %{COOKIE:} -Value of of the cookie ````. This does not expose or match against a +Value of the cookie ````. This does not expose or match against a cookie's expiration, the domain(s) on which it is valid, whether it is protocol restricted, or any of the other metadata; simply the current value of the cookie as presented by the client. @@ -562,7 +562,7 @@ be specified multiple times, such as ``Set-Cookie``, but for headers which may only be specified once you may prefer to use `set-header`_ instead. The header's ```` may be specified as a literal string, or it may take -advantage of :ref:`header-rewrite-concatenations` to calculate a dynamic value +advantage of `String concatenations`_ to calculate a dynamic value for the header. counter @@ -675,9 +675,8 @@ set-header Replaces the value of header ```` with ````, creating the header if necessary. -The header's ```` may be specified according to `Header Values`_ or take -advantage of :ref:`header-rewrite-concatenations` to calculate a dynamic value -for the header. +The header's ```` may be a literal string, or take advantage of +`String concatenations`_ to calculate a dynamic value for the header. set-redirect ~~~~~~~~~~~~ @@ -689,7 +688,7 @@ When invoked, sends a redirect response to the client, with HTTP status ````, and a new location of ````. If the ``QSA`` flag is enabled, the original query string will be preserved and added to the new location automatically. This operator supports -:ref:`header-rewrite-concatenations` for ````. +`String concatenations`_ for ````. set-status ~~~~~~~~~~ @@ -760,8 +759,6 @@ L Last rule, do not continue. QSA Append the results of the rule to the query string. ====== ======================================================================== -.. _header-rewrite-concatenations: - String concatenations --------------------- @@ -769,7 +766,8 @@ You can concatenate values using strings, condition values and variable expansio add-header CustomHeader "Hello from %{IP:SERVER}:%{INBOUND:LOCAL-PORT}" -String concatenation is not yet supported in condition testing. +This is the new, generic form of setting header values. Unfortunately, string +concatenation is not yet supported in conditionals. Note: In versions prior to ATS v9.0.0, an alternative string expansion was available. those expansions are no longer available, but the following table can help migrations: @@ -786,24 +784,6 @@ Old expansion variable Condition variable to use with concatenations % %{CLIENT-URL:PATH} ======================== ========================================================================== -Header Values -------------- - -Setting a header with a value can take the following formats: - -- Any `condition `_ which extracts a value from the request. - -- ``$N``, where 0 <= N <= 9, from matching groups in a regular expression. - -- A string (which can contain the numbered matches from a regular expression as - described above). - -- Null. - -Supplying no value for a header for certain operators can lead to an effective -no-op. In particular, `add-header`_ and `set-header`_ will simply short-circuit -if no value has been supplied for the named header. - URL Parts --------- @@ -822,7 +802,9 @@ Part Description HOST Full hostname. PATH URL substring beginning with (but not including) the first ``/`` after - the hostname up to, but not including, the query string. + the hostname up to, but not including, the query string. **Note**: previous + versions of ATS had a `%{PATH}` directive, this will no longer work. Instead, + you want to use `%{CLIENT-URL:PATH}`. PORT Port number. @@ -878,7 +860,7 @@ one forces the beginning of a new ruleset. node[shape=record]; Client[height=4, label="{ Client|{|} }"]; - ATS[height=4, fontsize=10,label="{ {{Global:\nREAD_REQUEST_PRE_REMAP_HOOK|Global:\nREAD_REQUEST_HDR_HOOK\nRemap rule:\nREMAP_PSEUDO_HOOK}|SEND_RESPONSE_HDR_HOOK}|ATS |{SEND_REQUEST_HDR_HOOK|READ_RESPONSE_HDR_HOOK} }",xlabel="ATS"]; + ATS[height=4, fontsize=10,label="{ {{Global:\nREAD_REQUEST_HDR_HOOK\nREAD_REQUEST_PRE_REMAP_HOOK|Remap rule:\nREMAP_PSEUDO_HOOK}|SEND_RESPONSE_HDR_HOOK}|ATS |{SEND_REQUEST_HDR_HOOK|READ_RESPONSE_HDR_HOOK} }",xlabel="ATS"]; Origin[height=4, label="{ {|}|Origin }"]; Client:p1 -> ATS:clientside0 [ label = "Request" ]; @@ -1117,7 +1099,7 @@ Add Cache Control Headers Based on Origin Path ---------------------------------------------- This rule adds cache control headers to CDN responses based matching the origin -path. One provides a max age and the other provides a “no-cache” statement to +path. One provides a max age and the other provides a "no-cache" statement to two different file paths. :: cond %{SEND_RESPONSE_HDR_HOOK} diff --git a/doc/admin-guide/plugins/index.en.rst b/doc/admin-guide/plugins/index.en.rst index e2ff09a4842..9e06acabba7 100644 --- a/doc/admin-guide/plugins/index.en.rst +++ b/doc/admin-guide/plugins/index.en.rst @@ -50,6 +50,7 @@ Plugins that are considered stable are installed by default in |TS| releases. Background Fetch Cache Key Manipulation Cache Promotion Policies + Cache Range Requests Combo Handler Configuration Remap Cookie Remap @@ -63,7 +64,6 @@ Plugins that are considered stable are installed by default in |TS| releases. Regex Remap Regex Revalidate Remap Purge - Slice Stats over HTTP TCPInfo XDebug @@ -77,12 +77,15 @@ Plugins that are considered stable are installed by default in |TS| releases. :doc:`Background Fetch ` Proactively fetch content from Origin in a way that it will fill the object into cache. -:doc:`Cache Key Manipulation ` - Allows some common cache key manipulations based on various HTTP request elements. +:doc:`Cache Key and Parent Selection URL Manipulation ` + Allows some common cache key or parent selection URL manipulations based on various HTTP request elements. :doc:`Cache Promotion Policies ` Allows for control over which assets should be written to cache, or not. +:doc:`Cache Range Requests ` + Cache ranges by adding the range request header to the cache key. + :doc:`Combo Handler ` Provides an intelligent way to combine multiple URLs into a single URL, and have Apache Traffic Server combine the components into one response. @@ -152,14 +155,16 @@ directory of the |TS| source tree. Experimental plugins can be compiled by passi Header Frequency Hook Trace JA3 Fingerprint + Maxmind ACL Memcache + Memory Profile Metalink Money Trace MP4 Multiplexer MySQL Remap Signed URLs - Slicer + Slice SSL Headers SSL Session Reuse System Statistics @@ -192,6 +197,9 @@ directory of the |TS| source tree. Experimental plugins can be compiled by passi :doc:`JA3 Fingerprint ` Calculates JA3 Fingerprints for incoming SSL traffic. +:doc:`MaxMind ACL ` + ACL based on the maxmind geo databases (GeoIP2 mmdb and libmaxminddb) + :doc:`Memcache ` Implements the memcache protocol for cache contents. @@ -223,10 +231,10 @@ directory of the |TS| source tree. Experimental plugins can be compiled by passi :doc:`Signed URLs ` Adds support for verifying URL signatures for incoming requests to either deny or redirect access. -:doc:`Slicer ` +:doc:`Slice ` Slice full file or range based requests into deterministic chunks, allowing large files to be - spread across multiple cache stripes. Allows range requests to be satisfied by stitching these - chunks together. + spread across multiple cache stripes. Allows arbitrary range requests to be satisfied by stitching + these chunks together. :doc:`SSL Session Reuse ` Coordinates Session ID and ticket based TLS session resumption between a group of ATS machines. diff --git a/doc/admin-guide/plugins/ja3_fingerprint.en.rst b/doc/admin-guide/plugins/ja3_fingerprint.en.rst index 931b9366674..62bf31608df 100644 --- a/doc/admin-guide/plugins/ja3_fingerprint.en.rst +++ b/doc/admin-guide/plugins/ja3_fingerprint.en.rst @@ -36,7 +36,7 @@ effective way to detect malicious clients even when superficial details are modi JA3 is available `here `__. The calculated JA3 fingerprints are then appended to upstream request in the field ``X-JA3-Sig`` -(to be processed at upstream). If multiple duplicates exist for the field name, it will append to the last +(to be processed at upstream). If multiple duplicates exist for the field name, it will append to the last occurrence; if none exists, it will add such a field to the headers. The signatures can also be logged locally. Plugin Configuration @@ -72,4 +72,3 @@ API changes with regard to opaque structures. There is a potential issue with very old TLS clients which can cause a crash in the plugin. This is due to a `bug in OpenSSL `__ which should be fixed in a future release. - diff --git a/doc/admin-guide/plugins/lua.en.rst b/doc/admin-guide/plugins/lua.en.rst index 465ee4ca3f4..de48ec3fbaa 100644 --- a/doc/admin-guide/plugins/lua.en.rst +++ b/doc/admin-guide/plugins/lua.en.rst @@ -128,6 +128,60 @@ If it is used as remap plugin, we can write the following in remap.config to def map http://a.tbcdn.cn/ http://inner.tbcdn.cn/ @plugin=/XXX/tslua.so @pparam=--states=64 @pparam=/XXX/test_hdr.lua +The maximum number of allowed states is set to 256 which is also the +default states value. The default value can be globally changed by +adding a configuration option to records.config. + +:: + + CONFIG proxy.config.plugin.lua.max_states INT 64 + +Any per plugin --states value overrides this default value but must be less than or equal to this value. This setting is not reloadable since it must be applied when all the lua states are first initialized. + +Profiling +========= + +The lua module collects runtime statistics about the lua states, for remap +and global instances. Per state stats are constantly maintained and are +made available through a lifecycle hook. These may be accessed through: + +:: + + traffic_ctl plugin msg ts_lua stats_print + +Sample output: + +:: + + [Feb 5 19:00:15.072] ts_lua (remap) id: 0 gc_kb: 2508 gc_kb_max: 3491 threads: 417 threads_max: 438 + [Feb 5 19:00:15.072] ts_lua (remap) id: 1 gc_kb: 1896 gc_kb_max: 3646 threads: 417 threads_max: 446 + [Feb 5 19:00:15.072] ts_lua (remap) id: 2 gc_kb: 3376 gc_kb_max: 3740 threads: 417 threads_max: 442 + +Max values may be reset at any time by running: + +:: + + traffic_ctl plugin msg ts_lua stats_reset + + +Summary statistics are aggregated every 5s and are available as metrics. + +:: + + traffic_ctl metric match lua + +Sample output: + +:: + + plugin.lua.global.states 8 + plugin.lua.remap.gc_bytes_min 4804608 + plugin.lua.remap.gc_bytes_mean 5552537 + plugin.lua.remap.gc_bytes_max 5779456 + plugin.lua.remap.threads_min 31 + plugin.lua.remap.threads_mean 44 + plugin.lua.remap.threads_max 146 + TS API for Lua ============== @@ -904,7 +958,7 @@ Here is an example: ts.debug(url_port) end -Then Then ``GET /liuyurou.txt HTTP/1.1\r\nHost: 192.168.231.129:8080\r\n...`` will yield the output: +Then ``GET /liuyurou.txt HTTP/1.1\r\nHost: 192.168.231.129:8080\r\n...`` will yield the output: ``8080`` @@ -957,6 +1011,86 @@ ts.client_request.set_url_scheme server request, and we should return TS_LUA_REMAP_DID_REMAP(_STOP) in do_remap. +:ref:`TOP ` + +ts.client_request.get_ssl_reused +----------------------------------------------- +**syntax:** *ts.client_request.get_ssl_reused()* + +**context:** do_remap/do_os_response or do_global_* or later + +**description**: This function can be used to know if the SSL session has been reused (1) or not (0) + +Here is an example: + +:: + + function do_global_read_request() + ssl_reused = ts.client_request.get_ssl_reused() + ts.debug(ssl_reused) -- 0 + end + + +:ref:`TOP ` + +ts.client_request.get_ssl_protocol +----------------------------------------------- +**syntax:** *ts.client_request.get_ssl_protocol()* + +**context:** do_remap/do_os_response or do_global_* or later + +**description**: This function can be used to get the SSL protocol used to communicate with the client + +Here is an example: + +:: + + function do_global_read_request() + ssl_protocol = ts.client_request.get_ssl_protocol() + ts.debug(ssl_protocol) -- TLSv1.2 + end + + +:ref:`TOP ` + +ts.client_request.get_ssl_cipher +----------------------------------------------- +**syntax:** *ts.client_request.get_ssl_cipher()* + +**context:** do_remap/do_os_response or do_global_* or later + +**description**: This function can be used to get the SSL cipher used to communicate with the client + +Here is an example: + +:: + + function do_global_read_request() + ssl_cipher = ts.client_request.get_ssl_cipher() + ts.debug(ssl_cipher) -- ECDHE-ECDSA-AES256-GCM-SHA384 + end + + +:ref:`TOP ` + +ts.client_request.get_ssl_curve +----------------------------------------------- +**syntax:** *ts.client_request.get_ssl_curve()* + +**context:** do_remap/do_os_response or do_global_* or later + +**description**: This function can be used to get the SSL Elliptic curve used to communicate with the client + +Here is an example: + +:: + + function do_global_read_request() + ssl_curve = ts.client_request.get_ssl_curve() + ts.debug(ssl_curve) -- X25519 + end + + :ref:`TOP ` ts.http.set_cache_url @@ -1624,6 +1758,125 @@ Here is an example: `TOP <#ts-lua-plugin>`_ +ts.sha256 +--------- +**syntax:** *digest = ts.sha256(str)* + +**context:** global + +**description:** Returns the hexadecimal representation of the SHA-256 digest of the ``str`` argument. + +Here is an example: + +:: + + function do_remap() + uri = ts.client_request.get_uri() + print(uri) + print(ts.sha256(uri)) + end + + +`TOP <#ts-lua-plugin>`_ + +ts.sha256_bin +------------- +**syntax:** *digest = ts.sha256_bin(str)* + +**context:** global + +**description:** Returns the binary form of the SHA-256 digest of the ``str`` argument. + +Here is an example: + +:: + + function do_remap() + uri = ts.client_request.get_uri() + bin = ts.sha256_bin(uri) + end + + +`TOP <#ts-lua-plugin>`_ + +ts.hmac_md5 +----------- +**syntax:** *digest = ts.hmac_md5(key, str)* + +**context:** global + +**description:** Returns the hexadecimal representation of the HMAC of the ``str`` argument. + +The message digest function used is MD5. + +The key value used is contained in the ``key`` argument. This should be a hexadecimal representation of the key value. + +Here is an example: + +:: + + function do_remap() + key = "012345" + uri = ts.client_request.get_uri() + print(uri) + print(ts.hmac_md5(key, uri)) + end + + +`TOP <#ts-lua-plugin>`_ + +ts.hmac_sha1 +------------ +**syntax:** *digest = ts.hmac_sha1(key, str)* + +**context:** global + +**description:** Returns the hexadecimal representation of the HMAC of the ``str`` argument. + +The message digest function used is SHA-1. + +The key value used is contained in the ``key`` argument. This should be a hexadecimal representation of the key value. + +Here is an example: + +:: + + function do_remap() + key = "012345" + uri = ts.client_request.get_uri() + print(uri) + print(ts.hmac_sha1(key, uri)) + end + + +`TOP <#ts-lua-plugin>`_ + +ts.hmac_sha256 +-------------- +**syntax:** *digest = ts.hmac_sha256(key, str)* + +**context:** global + +**description:** Returns the hexadecimal representation of the HMAC of the ``str`` argument. + +The message digest function used is SHA-256. + +The key value used is contained in the ``key`` argument. This should be a hexadecimal representation of the key value. + +Here is an example: + +:: + + function do_remap() + key = "012345" + uri = ts.client_request.get_uri() + print(uri) + print(ts.hmac_sha256(key, uri)) + end + + +`TOP <#ts-lua-plugin>`_ + ts.server_request.server_addr.get_ip ------------------------------------ **syntax:** *ts.server_request.server_addr.get_ip()* @@ -1823,6 +2076,83 @@ ts.server_request.set_url_scheme :ref:`TOP ` +ts.server_request.get_method +---------------------------- +**syntax:** *ts.server_request.get_method()* + +**context:** function @ TS_LUA_HOOK_SEND_REQUEST_HDR hook point or later + +**description:** This function can be used to retrieve the current server request's method name. String like "GET" or "POST" is returned. + +Here is an example: + +:: + + function send_request() + local method = ts.server_request.get_method() + ts.debug(method) + end + + function do_remap() + ts.hook(TS_LUA_HOOK_SEND_REQUEST_HDR, send_request) + return 0 + end + +Then ``HEAD /`` will yield the output: + +``HEAD`` + +:ref:`TOP ` + +ts.server_request.set_method +---------------------------- +**syntax:** *ts.server_request.set_method()* + +**context:** function @ TS_LUA_HOOK_SEND_REQUEST_HDR hook point or later + +**description:** This function can be used to override the current server request's method with METHOD_NAME. + +:: + ts.server_request.set_method('HEAD') + +:ref:`TOP ` + +ts.server_request_get_version +------------------------------ +**syntax:** *ver = ts.server_request.get_version()* + +**context:** function @ TS_LUA_HOOK_SEND_REQUEST_HDR hook point or later. + +**description:** Return the http version string of the server request. + +Current possible values are 1.0, 1.1, and 0.9. :: + + function send_request() + local version = ts.server_request.get_version() + ts.debug(version) + end + + function do_remap() + ts.hook(TS_LUA_HOOK_SEND_REQUEST_HDR, send_request) + return 0 + end + +:ref:`TOP ` + +ts.server_request.set_version +------------------------------ +**syntax:** *ts.server_request.set_version(VERSION_STR)* + +**context:** function @ TS_LUA_HOOK_READ_RESPONSE_HDR hook point + +**description:** Set the http version of the server request with the VERSION_STR + +:: + + ts.server_request.set_version('1.0') + +:ref:`TOP ` + ts.server_response.get_status ----------------------------- **syntax:** *status = ts.server_response.get_status()* diff --git a/doc/admin-guide/plugins/maxmind_acl.en.rst b/doc/admin-guide/plugins/maxmind_acl.en.rst new file mode 100644 index 00000000000..6fa1a8d94d3 --- /dev/null +++ b/doc/admin-guide/plugins/maxmind_acl.en.rst @@ -0,0 +1,80 @@ +.. _admin-plugins-maxmind-acl: + +MaxMind ACL Plugin +****************** + +.. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +This remap plugin provides allow and deny functionality based on the libmaxminddb +library and GeoIP2 databases (mmdb format). It requires libmaxminddb to run +and the associated development headers in order to build. You can find a sample +mmdb-lite database on the maxmind website or provide your own. You must provide a database +for any usages and specify it in the configuration file as shown below. + +Configuration +============= + +The plugin takes a single pparam which is the location of the configuration yaml +file. This can either be relative to the ATS configuration directory or an absolute path :: + + map http://example.com/music http://music.example.com @plugin=maxmind_acl.so @pparam=maxmind.yaml + +An example configuration :: + + maxmind: + database: GeoIP2-City.mmdb + html: deny.html + allow: + country: + - US + ip: + - 127.0.0.1 + - 192.168.10.0/20 + deny: + country: + - DE + ip: + - 127.0.0.1 + regex: + - [US, ".*\\.txt"] # Because these get parsed you must escape the escape of the ``.`` in order to have it be escaped in the regex, resulting in ".*\.txt" + - [US, ".*\\.mp3"] + +In order to load an updated configuration while ATS is running you will have to touch or modify the remap.config file in order to initiate a plugin reload to pull in any changes. + +Rules +===== + +You can mix and match the allow rules and deny rules, however deny rules will always take precedence so in the above case ``127.0.0.1`` would be denied. +The IP rules can take either single IPs or cidr formatted rules. It will also accept IPv6 IP and ranges. + +The regex portion can be added to both the allow and deny sections for creating allowable or deniable regexes. Each regex takes a country code first and a regex second. +In the above example all requests from the US would be allowed except for those on ``txt`` and ``mp3`` files. More rules should be added as pairs, not as additions to existing lists. + +Currently the only rules available are ``country``, ``ip``, and ``regex``, though more can easily be added if needed. Each config file does require a top level +``maxmind`` entry as well as a ``database`` entry for the IP lookups. You can supply a separate database for each remap used in case you use custom +ones and have specific needs per remap. + +One other thing to note. You can reverse the logic of the plugin, so that it will default to always allowing if you do not supply any ``allow`` rules. +In the case you supply no allow rules all connections will be allowed through except those that fall in to any of the deny rule lists. In the above example +the rule of denying ``DE`` would be a noop because there are allow rules set, so by default everything is blocked unless it is explicitly in an allow rule. +However in this case the regexes would still apply since they are based on an allowable country. + +Optional +======== + +There is an optional ``html`` field which takes a html file that will be used as the body of the response for any denied requests if you wish to use a custom one. diff --git a/doc/admin-guide/plugins/memory_profile.en.rst b/doc/admin-guide/plugins/memory_profile.en.rst new file mode 100644 index 00000000000..254ec8d2df5 --- /dev/null +++ b/doc/admin-guide/plugins/memory_profile.en.rst @@ -0,0 +1,106 @@ +Memory_profile Plugin +********************* + +.. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + + +This plugin listens for plugin messages and invokes jemalloc control +operations. + +Installation +============ + +Add the following line to :file:`plugin.config`:: + + memory_profile.so + +In addition, |TS| must be able to read jemalloc configuration +information either through the ``JEMALLOC_CONF`` environment variable +or via the string sym linked to ``/etc/malloc.conf``. + +For example, if the string below is in ``JEMALLOC_CONF`` or in the sym link string, it +enables profiling and indicates that the memory dump prefix is ``/tmp/jeprof``.:: + + prof:true,prof_prefix:/tmp/jeprof + +Details on configuration jemalloc options at ``. +Changes to the configuration in ``JEMALLOC_CONF`` or ``/etc/malloc.conf`` require a process +restart to pick up. + +Plugin Messages +=============== + +The plugin responds to the following messages sent via traffic_ctl. + +Message Action +========== =================================================================================== +activate Start jemalloc profiling. Useful if prof_active:false was in the configure string. + +deactivate Stop jemalloc profiling. + +dump If profiling is enabled and active, it will generate a profile dump file. + +stats Print jemalloc statistics in traffic.out + +The command below sends the stats message to the plugin causing the current statistics to be written to traffic.out:: + + traffic_ctl plugin msg memory_profile stats + +Example Usage +============= + +If your run time configuration string is:: + + prof:true,prof_prefix:/tmp/jeprof:prof_active:false + +|TS| has started without profile sampling started. Perhaps you didn't want to profile the start up phase of |TS|. To start +you need to send the activate message to the plugin:: + + traffic_ctl plugin msg memory_profile activate + +If your run time configuration string does not indicate that the profiling is not started (e.g. the prof_active field is missing or set to true), you do not +need to send the activate message. + +After waiting sometime for |TS| to gather some memory allocation data, you can send the dump message:: + + traffic_ctl plugin msg memory_profile dump + +This will cause a file containing information about the current state of the |TS| memory allocation to be dumped in a file prefixed +by the value of prof_prefix. In this example, it would be something like ``/tmp/jeprof.1234.0.m0.heap``, where 1234 is the process id +and 0 is a running counter indicating how many dumps have been performed on this process. Each dump is independent of the others +and records the current stat of allocations since the profiling was activated. The dump file can be processed by jeprof +to get text output or graphs. Details of how to use jeprof are in the man pages or ``. + +You may want to send the dump message periodically to analyze how the |TS| memory allocation changes over time. This periodic dump can also be achieved by setting the +``lg_prof_interval`` option in the run time configuration string. + +If the profiling is taking a significant amount of processing time and affecting |TS| performance, send the deactivate message to turn off profiling.:: + + traffic_ctl plugin msg memory_profile deactivate + +Send the stats message to cause detailed jemalloc stats to be printed in traffic.out. These stats represent activity since the start of the |TS| process.:: + + traffic_ctl plugin msg memory_profile stats + +Limitations +=========== + +Currently the plugin only functions for systems compiled against jemalloc. +Perhaps in the future, it can be augmented to interact with other memory +allocation systems. + diff --git a/doc/admin-guide/plugins/prefetch.en.rst b/doc/admin-guide/plugins/prefetch.en.rst index b95839239c9..5a94da6128a 100644 --- a/doc/admin-guide/plugins/prefetch.en.rst +++ b/doc/admin-guide/plugins/prefetch.en.rst @@ -88,7 +88,7 @@ It is worth mentioning that a small percentage of the requests did not follow a * All POPs were seeded periodically except for POP #1 and the plugin was deployed in the following order: POP #0, #1, #2, #3 and then to the rest at once. * POP #0 was the first plugin deployment and was used to tune its configuration for better results. -* POP #1 was a "testing ground" for the “worst case” (no seeding at all, imperfect conditions like low traffic and poorer connectivity to origin) and relying only on the Prefetch plugin. +* POP #1 was a "testing ground" for the "worst case" (no seeding at all, imperfect conditions like low traffic and poorer connectivity to origin) and relying only on the Prefetch plugin. * POP #2 and POP #3 experienced seeding problems (at times it reached ~60%, not shown here). @@ -153,7 +153,7 @@ If the "incoming" URL is :: http://example.com/path/file-104.mov?a=a&b=b -the the following URLs will be requested to be prefetched :: +the following URLs will be requested to be prefetched :: http://example-seed.com/path/file-106.mov?a=a&b=b http://example-seed.com/path/file-108.mov?a=a&b=b @@ -203,7 +203,7 @@ compromises: requests which would result in using the same **cache key** are not considered as separate requests (which could bloat/dilute the LRU cache if not normalized) -* **Check if the the fetch request is unique**. A ``simple`` prefetching policy is +* **Check if the fetch request is unique**. A ``simple`` prefetching policy is always used to make sure prefetches for the same object (same cache key) are never triggered simultaneously. * **Check if already cached**. Before triggering the prefetch request to the @@ -231,7 +231,7 @@ Plugin parameters * ``--api-header`` - the header used by the plugin internally, also used to mark a prefetch request to the next tier in dual-tier usage. * ``--fetch-policy`` - fetch policy - ``simple`` - this policy just makes sure there are no same concurrent prefetches triggered (default and always used in combination with any other policy) - - ``lru:n`` - this policy uses LRU to identify “hot” objects and triggers prefetch if the object is not found. `n` is the size of the LRU + - ``lru:n`` - this policy uses LRU to identify "hot" objects and triggers prefetch if the object is not found. `n` is the size of the LRU * ``--fetch-count`` - how many objects to be prefetched. * ``--fetch-path-pattern`` - regex/capture pattern that would transform the **incoming** into the **next object** path. * ``--fetch-max`` - maximum concurrent fetches allowed, this would allow to throttle the prefetch activity if necessary diff --git a/doc/admin-guide/plugins/slice.en.rst b/doc/admin-guide/plugins/slice.en.rst index 06ae34e564c..70bd2555e3e 100644 --- a/doc/admin-guide/plugins/slice.en.rst +++ b/doc/admin-guide/plugins/slice.en.rst @@ -5,9 +5,9 @@ to you under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - + http://www.apache.org/licenses/LICENSE-2.0 - + Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY @@ -69,17 +69,36 @@ The slice plugin supports the following options:: Suffix k,m,g supported Limited to 32k and 32m inclusive. - --test-blockbytes= (optional) + --blockbytes-test= (optional) Suffix k,m,g supported -t for short. Limited to any positive number. Ignored if --blockbytes provided. + --remap-host= (optional) + Uses effective url with given hostname for remapping. + Requires setting up an intermediate loopback remap rule. + -r for short + --pace-errorlog= (optional) Limit stitching error logs to every 'n' second(s) + -p for short --disable-errorlog (optional) Disable writing block stitch errors to the error log. + -d for short + + --exclude-regex= (optional) + If provided, only slice what matches. + If not provided will always slice + Cannot be used with --include-regex + -e for short + + --include-regex= (optional) + If provided, only slice what matches. + If not provided will always slice + Cannot be used with --exclude-regex + -i for short Examples:: @@ -95,10 +114,10 @@ Byte suffix examples:: slice.so -b 512k slice.so --blockbytes=32m -For testing and extreme purposes the parameter ``test-blockbytes`` may +For testing and extreme purposes the parameter ``blockbytes-test`` may be used instead which is unchecked:: - slice.so --test-blockbytes=1G + slice.so --blockbytes-test=1G slice.so -t 13 Because the slice plugin is susceptible to errors during block stitching @@ -114,6 +133,16 @@ After modifying :file:`remap.config`, restart or reload traffic server (sudo traffic_ctl config reload) or (sudo traffic_ctl server restart) to activate the new configuration values. +Don't slice txt files:: + + slice.so --exclude-regex=\\.txt + slice.so -e \\.txt + +Slice only mp4 files:: + + slice.so --include-regex=\\.mp4 + slice.so -i \\.mp4 + Debug Options ------------- @@ -128,9 +157,11 @@ Under normal logging these slice block errors tend to show up as:: crc value ERR_READ_ERROR By default more detailed stitching errors are written to ``diags.log``. -An example is as follows:: +Examples are as follows:: -[Apr 19 20:26:13.639] [ET_NET 17] ERROR: [slice] 1555705573.639 reason="Non 206 internal block response" uri="http://localhost:18080/%7Ep.tex/%7Es.50M/%7Eui.20000/" uas="curl/7.29.0" req_range="bytes=1000000-" norm_range="bytes 1000000-52428799/52428800" etag_exp="%221603934496%22" lm_exp="Fri, 19 Apr 2019 18:53:20 GMT" blk_range="21000000-21999999" status_got="400" cr_got="" etag_got="" lm_got="" cc="no-store" via="" +ERROR: [slice.cc: 288] logSliceError(): 1555705573.639 reason="Non 206 internal block response" uri="http://ats_ep/someasset.mp4" uas="curl" req_range="bytes=1000000-" norm_range="bytes 1000000-52428799/52428800" etag_exp="%221603934496%22" lm_exp="Fri, 19 Apr 2019 18:53:20 GMT" blk_range="21000000-21999999" status_got="206" cr_got="" etag_got="%221603934496%22" lm_got="" cc="no-store" via="" + +ERROR: [server.cc: 288] logSliceError(): 1572370000.219 reason="Mismatch block Etag" uri="http://ats_ep/someasset.mp4" uas="curl" req_range="bytes=1092779033-1096299354" norm_range="bytes 1092779033-1096299354/2147483648" etag_exp="%223719843648%22" lm_exp="Tue, 29 Oct 2019 14:40:00 GMT" blk_range="1095000000-1095999999" status_got="206" cr_got="bytes 1095000000-1095999999/2147483648" etag_got="%223719853648%22" lm_got="Tue, 29 Oct 2019 17:26:40 GMT" cc="max-age=10000" via="" Whether or how often these detailed log entries are written are configurable plugin options. @@ -175,7 +206,7 @@ block header and sends its bytes to the client until the client request is satisfied. Any extra bytes at the end of the last block are consumed by -the the Slice plugin to allow cache_range_requests to finish +the Slice plugin to allow cache_range_requests to finish the block fetch to ensure the block is cached. Important Notes @@ -204,14 +235,43 @@ by cache_range_requests. The parent will trim those requests to account for the asset Content-Length so only the appropriate number of bytes are actually transferred and cached. -Current Limitations +Effective URL remap =================== -By restoring the pristine Url the plugin as it works today reuses the -same remap rule for each slice block. This is wasteful in that it reruns +By default the plugin restores the Pristine Url which reuses the same +remap rule for each slice block. This is wasteful in that it reruns the previous remap rules, and those remap rules must be smart enough to -check for the existence of any headers they may have created the -first time they have were visited. +check for the existence of any headers they may have created the first +time they have were visited. + +To get around this the '--remap-host=' or '-r ' option may +be used. This requires an intermediate loopback remap to be defined which +handles each slice block request. + +This works well with any remap rules that use the url_sig or uri_signing +plugins. As the client remap rule is not caching any plugins that +manipulate the cache key would need to go into the loopback to parent +remap rule. + +NOTE: Requests NOT handled by the slice plugin (ie: HEAD requests) are +handled as with a typical remap rule. GET requests intercepted by the +slice plugin are virtually reissued into ATS and are proxied through +another remap rule which must contain the ``cache_range_requests`` plugin + +Examples:: + + map http://ats/ http://parent/ @plugin=slice.so @pparam=--remap-host=loopback + map http://loopback/ http://parent/ @plugin=cache_range_requests.so + +Alternatively:: + + map http://ats/ http://parent/ @plugin=slice.so @pparam=-r @pparam=loopback + map http://loopback/ http://parent/ @plugin=cache_range_requests.so + +Current Limitations +=================== Since the Slice plugin is written as an intercept handler it loses the -ability to use normal state machine hooks and transaction states. +ability to use normal state machine hooks and transaction states. This +functionality is handled by using the ``cache_range_requests`` plugin +to interact with ATS. diff --git a/doc/admin-guide/plugins/ssl_session_reuse.en.rst b/doc/admin-guide/plugins/ssl_session_reuse.en.rst index c253ff9c7d9..c0e48aebe6b 100644 --- a/doc/admin-guide/plugins/ssl_session_reuse.en.rst +++ b/doc/admin-guide/plugins/ssl_session_reuse.en.rst @@ -22,7 +22,7 @@ SSL Session Reuse Plugin ************************ -This plugin coordinates session state data between ATS instances running in a group. This should +This plugin coordinates session state data between ATS instances running in a group. This should improve TLS session reuse (both ticket and ID based) for a set of machines fronted by some form of layer 4 connection load balancer. @@ -31,14 +31,14 @@ How It Works The plugin coordinates TLS session reuse for both Session ID based resumption and ticket based resumption. For Session ID base resumption in uses the ATS SSL Session Cache for the local store of TLS sessions. It uses -Redis to communication new sessions with its peers. When a new session is seen by an ATS instances it +Redis to communication new sessions with its peers. When a new session is seen by an ATS instances it publishes an encrypted copy of the session state to the local Redis channel. When a new session is received on the Redis channel, the plugin stores that session state into its local ATS SSL session cache. Once the -session state is in the local ATS SSL session cache it is available to the openssl library for future TLS +session state is in the local ATS SSL session cache it is available to the openssl library for future TLS handshakes. For the ticket based session resumption, the plugin implements logic to decide on a Session Ticket Encryption Key (STEK) -master. The master will periodically create a new STEK key and use the Redis channel to publish the new STEK key +master. The master will periodically create a new STEK key and use the Redis channel to publish the new STEK key to the other ATS boxes in the group. When the plugin starts up, it will publish a Redis message requesting the master to resend the STEK key. The plugin uses the TSSslTicketKeyUpdate call to update ATS with the last two STEK's it has received. @@ -48,7 +48,7 @@ reuse must have access to that preshared key. Building ======== -This plugin uses Redis for communication. The hiredis client development library must be installed +This plugin uses Redis for communication. The hiredis client development library must be installed for this plugin to build. It can be installed in the standard system location or the install location can be specified by the --with-hiredis argument to configure. @@ -59,8 +59,8 @@ Deploying ========= The SSL Session Reuse plugin relies on Redis for communication. To deploy build your own redis server or use a standard rpm -package. It must be installed on at least one box in the ATS group. We have it installed on two boxes in a failover -scenario. The SSL Session Reuse configuration file describes how to communicate with the redis servers. +package. It must be installed on at least one box in the ATS group. We have it installed on two boxes in a failover +scenario. The SSL Session Reuse configuration file describes how to communicate with the redis servers. * :ts:cv:`proxy.config.ssl.session_cache` should be set to 2 to enable the ATS implementation of session cache * :ts:cv:`proxy.config.ssl.session_cache.size` and :ts:cv:`proxy.config.ssl.session_cache.num_buckets` may need to be adjusted to ensure good hash table performance for your workload. For example, we needed to increase the number of buckets to avoid long hash chains. @@ -70,7 +70,7 @@ scenario. The SSL Session Reuse configuration file describes how to communicate Config File =========== -SSL Session Reuse is a global plugin. Its configuration file is given as a argument to the plugin. +SSL Session Reuse is a global plugin. Its configuration file is given as a argument to the plugin. * redis.RedisEndpoints - This is a comma separated list of Redis servers to connect to. The description of the redis server may include a port * redis.RedisConnectTimeout - Timeout on the redis connect attempt in milliseconds. @@ -91,4 +91,3 @@ Example Config File =================== .. literalinclude:: ../../../plugins/experimental/ssl_session_reuse/example_config.config - diff --git a/doc/admin-guide/plugins/stats_over_http.en.rst b/doc/admin-guide/plugins/stats_over_http.en.rst index 34326cdfb3b..d09a4a0b878 100644 --- a/doc/admin-guide/plugins/stats_over_http.en.rst +++ b/doc/admin-guide/plugins/stats_over_http.en.rst @@ -23,8 +23,9 @@ Stats Over HTTP Plugin ********************** This plugin implements an HTTP interface to all Traffic Server statistics. The -metrics returned are in a JSON format, for easy processing. This plugin is now -part of the standard ATS build process, and should be available after install. +metrics returned are in a JSON format by default, for easy processing. You can +also output the stats in CSV format as well. This plugin is now part of the +standard ATS build process, and should be available after install. Enabling Stats Over HTTP ======================== @@ -67,3 +68,39 @@ and the URL would then be e.g.:: This is weak security at best, since the secret could possibly leak if you are careless and send it over clear text. + +Config File Usage +================= + +stats_over_http.so also accepts a configuration file taken as a parameter + +The plugin first checks if the parameter that was passed in is a file that exists, if so +it uses that as a config file, otherwise if a parameter exists it assumes that it is meant +to be used a path value (as if you were not using a config file) + +You can add comments to the config file, starting with a `#` value + +Other options you can specify: + +.. option:: path= + +This sets the path value for stats + +.. option:: allow_ip= + +A comma separated list of IPv4 addresses allowed to access the endpoint + +.. option:: allow_ip6= + +A comma separated list of IPv6 addresses allowed to access the endpoint + +Output Format +============= + +By default stats_over_http.so will output all the stats in JSON format. However +if you wish to have it in CSV format you can do so by passing an ``Accept`` header: + +.. option:: Accept: text/csv + +In either case the ``Content-Type`` header returned by stats_over_http.so will reflect +the content that has been returned, either ``text/json`` or ``text/csv``. diff --git a/doc/admin-guide/plugins/tcpinfo.en.rst b/doc/admin-guide/plugins/tcpinfo.en.rst index 28ca98758d7..6c806e8b9ee 100644 --- a/doc/admin-guide/plugins/tcpinfo.en.rst +++ b/doc/admin-guide/plugins/tcpinfo.en.rst @@ -45,7 +45,6 @@ The following options may be specified in :file:`plugin.config`: Event Name Triggered when ============== =============================================== send_resp_hdr The server begins sending an HTTP response. - ssn_close The TCP connection closes. ssn_start A new TCP connection is accepted. txn_close A HTTP transaction is completed. txn_start A HTTP transaction is initiated. diff --git a/doc/admin-guide/plugins/traffic_dump.en.rst b/doc/admin-guide/plugins/traffic_dump.en.rst index 2d012f11855..9b213438fda 100644 --- a/doc/admin-guide/plugins/traffic_dump.en.rst +++ b/doc/admin-guide/plugins/traffic_dump.en.rst @@ -27,37 +27,60 @@ Traffic Dump Plugin Description =========== -``Traffic Dump`` captures session traffic and writes to a replay file :ts:git:`tests/tools/traffic-replay/replay_schema.json` for each captured session. It then can be used to replay traffic for testing purpose. +``Traffic Dump`` captures traffic for a set of sampled sessions and records this traffic to replay files according to the :ts:git:`tests/tools/traffic-replay/replay_schema.json` schema. These replay files can be used to replay traffic for testing purposes. Further, since the traffic is written decrypted, they can be used to conveniently analyze HTTP traffic going through ATS. Plugin Configuration ==================== .. program:: traffic_dump.so -* ``Traffic Dump`` is a global plugin and is configured via :file:`plugin.config`. +``traffic_dump.so`` + ``Traffic Dump`` is a global plugin and is configured via :file:`plugin.config`. + .. option:: --logdir - (`required`) - specifies the directory for writing all dump files. If path is relative, it is relative to the Traffic Server directory. The plugin will use first three characters of client ip to create subdirs in an attempt to spread dumps evenly and avoid too many files in a single directory. + (`required`) - Specifies the directory for writing all dump files. If the path is relative it is considered relative to the Traffic Server directory. The plugin will use the first three characters of the client's IP to create subdirs in an attempt to spread dumps evenly and avoid too many files in a single directory. .. option:: --sample - (`required`) - specifies the sampling ratio N. Traffic Dump will capture every one out of N sessions. This ratio can also be changed via traffic_ctl without restarting ATS. + (`required`) - Specifies the sampling ratio N. Traffic Dump will capture every one out of N sessions. This ratio can also be changed via ``traffic_ctl`` without restarting ATS. .. option:: --limit - (`required`) - specifies the max disk usage N bytes(approximate). Traffic Dump will stop capturing new sessions once disk usage exceeds this limit. + (`required`) - Specifies the max disk usage to N bytes (approximate). Traffic Dump will stop capturing new sessions once disk usage exceeds this limit. + + .. option:: --sensitive-fields + + (`optional`) - A comma separated list of HTTP case-insensitive field names whose values are considered sensitive information. Traffic Dump will not dump the incoming field values for any of these fields but will instead dump a generic value for them of the same length as the original. If this option is not used, a default list of "Cookie,Set-Cookie" is used. Providing this option overwrites that default list with whatever values the user provides. Pass a quoted empty string as the argument to specify that no fields are sensitive, + + .. option:: --sni-filter + + (`optional`) - An SNI with which to filter sessions. Only HTTPS sessions with the provided SNI will be dumped. The sample option will apply a sampling rate to these filtered sessions. Thus, with a sample value of 2, 1/2 of all sessions with the specified SNI will be dumped. + +``traffic_ctl`` + ``Traffic Dump`` can be dynamically configured via ``traffic_ctl``. -* ``traffic_ctl`` command. - ``traffic_ctl plugin msg traffic_dump.sample N`` - changes the sampling ratio N as mentioned above. - ``traffic_ctl plugin msg traffic_dump.reset`` - resets the disk usage counter. - ``traffic_ctl plugin msg traffic_dump.limit N`` - changes the max disk usage. + * ``traffic_ctl plugin msg traffic_dump.sample N`` - changes the sampling ratio N as mentioned above. + * ``traffic_ctl plugin msg traffic_dump.reset`` - resets the disk usage counter. + * ``traffic_ctl plugin msg traffic_dump.limit N`` - changes the max disk usage to N bytes as mentioned above. Replay Format ============= -This format contains traffic data including: -* Each session and transactions in the session. -* Timestamps. -* The four headers (ua request, proxy request, origin server response, proxy response). -* The protocol stack for the user agent. -* The transaction count for the outbound session. -* The content block sizes. -* See schema here: :ts:git:`tests/tools/traffic-replay/replay_schema.json` +This replay files contain the following information: + +* Each sampled session and all the transactions for those sessions. +* Timestamps of each transaction. +* The four sets of HTTP message headers (user agent request, proxy request, origin server response, proxy response). +* The protocol stacks for the user agent and the origin server connections. +* The number of body bytes for each message. + +For details, see the schema: :ts:git:`tests/tools/lib/replay_schema.json` + +Post Processing +=============== +Traffic Dump comes with a post processing script located at :ts:git:`plugins/experimental/traffic_dump/post_process.py`. This filters out incomplete sessions and transactions and merges a specifiable amount of sessions from a client into single replay files. It also optionally formats the single-line JSON files into a human readable format. It takes the following arguments: + +* The first positional argument is the input directory containing the replay files captured by traffic_dump as described above. +* The second positional argument is the output directory containing the processed replay files. +* ``-n`` is an optional argument specifying how many sessions will be attempted to be merged into single replay files. The default is 10. + +There are other options. Use ``--help`` for a description of these. diff --git a/doc/admin-guide/plugins/xdebug.en.rst b/doc/admin-guide/plugins/xdebug.en.rst index bb5f376122a..126028d873d 100644 --- a/doc/admin-guide/plugins/xdebug.en.rst +++ b/doc/admin-guide/plugins/xdebug.en.rst @@ -78,7 +78,7 @@ X-Cache-Key key is being used. X-Cache - The ``X-Cache`` header contains the results of any cache lookup. + The ``X-Cache`` header contains the results of any cache lookups. ========== =========== Value Description @@ -90,6 +90,10 @@ X-Cache skipped The cache lookup was skipped. ========== =========== + If a request goes through multiple proxies, each one appends its X-Cache header content + the end of the existing X-Cache header. This is the same order as for the + ``Via`` header. + X-Cache-Generation The cache generation ID for this transaction, as specified by the :ts:cv:`proxy.config.http.cache.generation` configuration variable. @@ -109,3 +113,9 @@ X-Transaction-ID X-Remap If the URL was remapped for a request, this header gives the *to* and *from* field from the line in remap.config that caused the URL to be remapped. + +X-ParentSelection-Key + The ``X-ParentSelection-Key`` header contains the URL that is used to + determine parent selection for an object in the Traffic Server. This + header is particularly useful if a custom parent selection key is + being used. diff --git a/doc/admin-guide/storage/index.en.rst b/doc/admin-guide/storage/index.en.rst index 50e1e63fbd1..68bcd81f12c 100644 --- a/doc/admin-guide/storage/index.en.rst +++ b/doc/admin-guide/storage/index.en.rst @@ -137,6 +137,15 @@ To change the RAM cache size: 1GB or more, then restart with the :program:`trafficserver` command (refer to :ref:`start-traffic-server`). +Disabling the RAM Cache +----------------------- + +It is possible to disable the RAM cache. If you have configured your +storage using the :file:`volume.config` you can add an optional directive +of ``ramcache=false`` to whichever volumes you wish to have it disabled on. +This may be desirable for volumes composed of storage like RAM disks where +you may want to avoid double RAM caching. + Changing Cache Capacity ======================= @@ -307,7 +316,7 @@ Pushing an Object into the Cache ================================ Traffic Server accepts the custom HTTP request method ``PUSH`` to put an object -into the the cache. If the object is successfully written to the cache, then +into the cache. If the object is successfully written to the cache, then Traffic Server responds with a ``200 OK`` HTTP message; otherwise a ``400 Malformed Pushed Response Header`` message is returned. @@ -357,7 +366,7 @@ To access the Cache Inspector utility: map http://yourhost.com/myCI/ http://{cache} @action=allow @src_ip=172.28.56.1-172.28.56.254 #. Reload the Traffic Server configuration by running :option:`traffic_ctl config reload`. -#. Open your web browser and go to the the following URL:: +#. Open your web browser and go to the following URL:: http://yourhost/myCI/ diff --git a/doc/appendices/command-line/traffic_server.en.rst b/doc/appendices/command-line/traffic_server.en.rst index 7a2dae2f403..48b3e47a457 100644 --- a/doc/appendices/command-line/traffic_server.en.rst +++ b/doc/appendices/command-line/traffic_server.en.rst @@ -30,29 +30,68 @@ Options .. program:: traffic_server -.. option:: -n COUNT, --net_threads COUNT - -.. option:: -U COUNT, --udp_threads COUNT - .. option:: -a, --accepts_thread .. option:: -b, --accept_till_done -.. option:: -p PORT, --httpport PORT +.. option:: -B TAGS, --action_tags TAGS + +.. option:: -C 'CMD [ARGS]', --command 'CMD [ARGS]' + +Run a |TS| maintenance command. These commands perform various administrative +actions or queries against |TS|. Note that some commands (such as ``help`` and +``verify_global_plugin``) take an argument. To invoke the command and its +argument, surround the ``CMD`` and its argument in quotes. For instance, to +request help for the ``verify_global_plugin`` command, format your command like +so:: + + traffic_server -C "help verify_global_plugin" + +The following commands are supported: + +list + List the sizes of the host database and cache index as well as the storage + available to the cache. +check + Check the cache for inconsistencies or corruption. ``check`` does not make + any changes to the data stored in the cache. ``check`` requires a scan of + the contents of the cache and may take a long time for large caches. +clear + Clear the entire cache, both the document and the host database caches. All + data in the cache is lost and the cache is reconfigured based on the current + description of database sizes and available storage. +clear_cache + Clear the document cache. All documents in the cache are lost and the cache + is reconfigured based on the current description of database sizes and + available storage. +clear_hostdb + Clear the entire host database cache. All host name resolution information + is lost. +verify_config + Load the config and verify |TS| comes up correctly. +verify_global_plugin PLUGIN_SO_FILE + Load a global plugin's shared object file and verify it meets minimal global + plugin API requirements. +verify_remap_plugin PLUGIN_SO_FILE + Load a remap plugin's shared object file and verify it meets minimal remap + plugin API requirements. +help [CMD] + Obtain a short description of a command. For example, ``'help clear'`` + prints a description of the ``clear`` maintenance command. If no argument + is passed to ``help`` then a list of the supported maintenance commands are + printed along with a brief description of each. .. option:: -f, --disable_freelist -In order to improve performance, :program:`traffic_server` caches -commonly used data structures in a set of free object lists. This -option disables these caches, causing :program:`traffic_server` to -use :manpage:`malloc(3)` for every allocation. Though this option -should not commonly be needed, it may be beneficial in memory-constrained -environments or where the working set is highly variable. +In order to improve performance, :program:`traffic_server` caches commonly used data structures in a +set of free object lists. This option disables these caches, causing :program:`traffic_server` to +use :manpage:`malloc(3)` for every allocation. Though this option should not commonly be needed, it +may be beneficial in memory-constrained environments or where the working set is highly variable. .. option:: -F, --disable_pfreelist -Disable free list in ProxyAllocator which were left out by the -f -option. This option includes the functionality of :option:`-f`. +Disable free list in ProxyAllocator which were left out by the -f option. This option includes the +functionality of :option:`-f`. .. option:: -R LEVEL, --regression LEVEL @@ -65,30 +104,36 @@ the available tests. .. option:: -T TAGS, --debug_tags TAGS -.. option:: -B TAGS, --action_tags TAGS - .. option:: -i COUNT, --interval COUNT +.. option:: -m COUNT, --maxRecords + +The maximum number of entries in metrics and configuration variables. The default is 1600, which is +also the minimum. This may need to be increased if running plugins that create metrics. + .. option:: -M, --remote_management -.. option:: -C CMD, --command CMD +Indicates the process should expect to be managed by :ref:`traffic_manager`. This option should not +be used except by that process. + +.. option:: -n COUNT, --net_threads COUNT .. option:: -k, --clear_hostdb .. option:: -K, --clear_cache -.. option:: -c CORE, --read_core CORE - .. option:: --accept_mss MSS .. option:: -t MSECS, --poll_timeout MSECS -.. option:: -m RECORDS, --maxRecords RECORDS - .. option:: -h, --help Print usage information and exit. +.. option:: -p PORT, --httpport PORT + +.. option:: -U COUNT, --udp_threads COUNT + .. option:: -V, --version Print version information and exit. diff --git a/doc/appendices/glossary.en.rst b/doc/appendices/glossary.en.rst index 02318aeddc9..85dc49916c9 100644 --- a/doc/appendices/glossary.en.rst +++ b/doc/appendices/glossary.en.rst @@ -26,7 +26,7 @@ Glossary :sorted: continuation - A callable object that contains state. These are are mechanism used by + A callable object that contains state. This is a mechanism used by |TS| to implement callbacks and continued computations. Continued computations are critical to efficient processing of traffic because by avoiding any blocking operations that wait on external events. In any diff --git a/doc/checkvers.py b/doc/checkvers.py index e16e7e4d3e9..6f1addc5746 100644 --- a/doc/checkvers.py +++ b/doc/checkvers.py @@ -28,7 +28,7 @@ # Check whether we have the required version of sphinx. if options.checkvers: - min_sphinx_version_info = (1,7,5) + min_sphinx_version_info = (1, 7, 5) min_sphinx_version = '.'.join([str(x) for x in min_sphinx_version_info]) print('checking for sphinx version >= {0}... '.format(min_sphinx_version), end="") @@ -65,5 +65,5 @@ import sphinxcontrib.plantuml print('yes') except Exception as e: - print(e); + print(e) sys.exit(1) diff --git a/doc/conf.py b/doc/conf.py index eefc2d4622d..599e9d332d9 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -27,6 +27,14 @@ # All configuration values have a default; values that are commented out # serve to show the default. +from sphinx.writers import manpage +from docutils.transforms import frontmatter +from docutils.utils import unescape +from docutils.utils import punctuation_chars +from docutils.parsers.rst import states +from docutils import nodes +import re +from manpages import man_pages import sys import os from datetime import date @@ -39,7 +47,6 @@ sys.path.insert(0, os.path.abspath('ext')) sys.path.insert(0, os.path.abspath('.')) -from manpages import man_pages # -- General configuration ----------------------------------------------------- @@ -58,7 +65,7 @@ # Contains values that are dependent on configure.ac. LOCAL_CONFIG = 'ext/local-config.py' -with open(LOCAL_CONFIG) as f : +with open(LOCAL_CONFIG) as f: exec(compile(f.read(), LOCAL_CONFIG, 'exec')) if version_info >= (1, 4): @@ -96,10 +103,9 @@ # work identically when building with Autotools (e.g. $ make html) # and without (e.g. on Read the Docs) -import re contents = open('../configure.ac').read() -match = re.compile('m4_define\(\[TS_VERSION_S],\[(.*?)]\)').search(contents) +match = re.compile(r'm4_define\(\[TS_VERSION_S],\[(.*?)]\)').search(contents) # The full version, including alpha/beta/rc tags. release = match.group(1) @@ -134,7 +140,7 @@ import sphinx_rtd_theme html_theme = 'sphinx_rtd_theme' html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] - except: + except BaseException: pass # End of HACK @@ -169,22 +175,15 @@ #modindex_common_prefix = [] nitpicky = True -nitpick_ignore = [ ('c:type', 'int64_t') - , ('c:type', 'bool') - , ('c:type', 'sockaddr') - , ('cpp:identifier', 'T') # template arg - , ('cpp:identifier', 'F') # template arg - , ('cpp:identifier', 'Args') # variadic template arg - , ('cpp:identifier', 'Rest') # variadic template arg - ] +nitpick_ignore = [('c:type', 'int64_t'), ('c:type', 'bool'), ('c:type', 'sockaddr'), ('cpp:identifier', 'T') # template arg + , ('cpp:identifier', 'F') # template arg + , ('cpp:identifier', 'Args') # variadic template arg + , ('cpp:identifier', 'Rest') # variadic template arg + ] # Autolink issue references. # See Customizing the Parser in the docutils.parsers.rst module. -from docutils import nodes -from docutils.parsers.rst import states -from docutils.utils import punctuation_chars -from docutils.utils import unescape # Customize parser.inliner in the only way that Sphinx supports. # docutils.parsers.rst.Parser takes an instance of states.Inliner or a @@ -337,13 +336,13 @@ def issue_reference(self, match, lineno): latex_elements = { # The paper size ('letterpaper' or 'a4paper'). - #'papersize': 'letterpaper', + # 'papersize': 'letterpaper', # The font size ('10pt', '11pt' or '12pt'). - #'pointsize': '10pt', + # 'pointsize': '10pt', # Additional stuff for the LaTeX preamble. - #'preamble': '', + # 'preamble': '', } if 'latex_a4' in tags: @@ -390,8 +389,6 @@ def issue_reference(self, match, lineno): # documents and includes the same brief description in both the HTML # and manual page outputs. -from docutils.transforms import frontmatter -from sphinx.writers import manpage # Override ManualPageWriter and ManualPageTranslator in the only way # that Sphinx supports @@ -507,4 +504,4 @@ def __init__(self, builder, *args, **kwds): # Enabling marking bit fields as 'bitfield_N`. # Currently parameterized fields don't work. When they do, we should change to # 'bitfield(N)'. -cpp_id_attributes = [ 'bitfield_1', 'bitfield_3', 'bitfield_24' ] +cpp_id_attributes = ['bitfield_1', 'bitfield_3', 'bitfield_24'] diff --git a/doc/developer-guide/api/functions/TSAPI.en.rst b/doc/developer-guide/api/functions/TSAPI.en.rst index 1a9da2a25ef..e8c06fd4cc5 100644 --- a/doc/developer-guide/api/functions/TSAPI.en.rst +++ b/doc/developer-guide/api/functions/TSAPI.en.rst @@ -28,8 +28,10 @@ Introduction to the Apache Traffic Server API. Synopsis ======== -`#include ` -`#include ` +.. code-block:: cpp + + #include + #include Description =========== @@ -57,11 +59,11 @@ type. Possible uses for plugins include the following: -* HTTP processing plugins can filter, blacklist, authorize users or transform content. +* HTTP processing plugins can filter, denylist, authorize users or transform content. * Protocol plugins can enable Traffic Server to proxy-cache new protocol content. -* A blacklisting plugin denies attempts to access web sites that are off-limits. +* A denylisting plugin denies attempts to access web sites that are off-limits. * Append transform plugins add data to HTTP response content. diff --git a/doc/developer-guide/api/functions/TSAcceptor.en.rst b/doc/developer-guide/api/functions/TSAcceptor.en.rst index 35bc21e9902..b6ebb3c2bad 100644 --- a/doc/developer-guide/api/functions/TSAcceptor.en.rst +++ b/doc/developer-guide/api/functions/TSAcceptor.en.rst @@ -26,7 +26,9 @@ Traffic Server API's related to Accept objects Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: TSAcceptor TSAcceptorGet(TSVConn sslp) .. function:: TSAcceptor TSAcceptorGetbyID(int id) diff --git a/doc/developer-guide/api/functions/TSActionCancel.en.rst b/doc/developer-guide/api/functions/TSActionCancel.en.rst index c15afae8961..0626f748ef8 100644 --- a/doc/developer-guide/api/functions/TSActionCancel.en.rst +++ b/doc/developer-guide/api/functions/TSActionCancel.en.rst @@ -24,7 +24,9 @@ TSActionCancel Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: void TSActionCancel(TSAction actionp) diff --git a/doc/developer-guide/api/functions/TSActionDone.en.rst b/doc/developer-guide/api/functions/TSActionDone.en.rst index d7190ccf56d..0990fc7d226 100644 --- a/doc/developer-guide/api/functions/TSActionDone.en.rst +++ b/doc/developer-guide/api/functions/TSActionDone.en.rst @@ -24,7 +24,9 @@ TSActionDone Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: int TSActionDone(TSAction actionp) diff --git a/doc/developer-guide/api/functions/TSCacheRead.en.rst b/doc/developer-guide/api/functions/TSCacheRead.en.rst index 6642d246898..e64dbdde426 100644 --- a/doc/developer-guide/api/functions/TSCacheRead.en.rst +++ b/doc/developer-guide/api/functions/TSCacheRead.en.rst @@ -24,7 +24,9 @@ TSCacheRead Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: TSAction TSCacheRead(TSCont contp, TSCacheKey key) diff --git a/doc/developer-guide/api/functions/TSCacheRemove.en.rst b/doc/developer-guide/api/functions/TSCacheRemove.en.rst index f5647d59a32..29fe5cf7ea7 100644 --- a/doc/developer-guide/api/functions/TSCacheRemove.en.rst +++ b/doc/developer-guide/api/functions/TSCacheRemove.en.rst @@ -24,7 +24,9 @@ TSCacheRemove Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: TSAction TSCacheRemove(TSCont contp, TSCacheKey key) diff --git a/doc/developer-guide/api/functions/TSCacheWrite.en.rst b/doc/developer-guide/api/functions/TSCacheWrite.en.rst index e00c8414d9e..92e270d8eeb 100644 --- a/doc/developer-guide/api/functions/TSCacheWrite.en.rst +++ b/doc/developer-guide/api/functions/TSCacheWrite.en.rst @@ -24,7 +24,9 @@ TSCacheWrite Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: TSAction TSCacheWrite(TSCont contp, TSCacheKey key) diff --git a/doc/developer-guide/api/functions/TSClientProtocolStack.en.rst b/doc/developer-guide/api/functions/TSClientProtocolStack.en.rst index 246b47d9bf0..aaff54e1c19 100644 --- a/doc/developer-guide/api/functions/TSClientProtocolStack.en.rst +++ b/doc/developer-guide/api/functions/TSClientProtocolStack.en.rst @@ -25,7 +25,9 @@ TSClientProtocolStack Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: TSReturnCode TSHttpTxnClientProtocolStackGet(TSHttpTxn txnp, int n, char const** result, int* actual) diff --git a/doc/developer-guide/api/functions/TSConfig.en.rst b/doc/developer-guide/api/functions/TSConfig.en.rst index 0002853c488..73041fe43e4 100644 --- a/doc/developer-guide/api/functions/TSConfig.en.rst +++ b/doc/developer-guide/api/functions/TSConfig.en.rst @@ -24,7 +24,9 @@ TSConfig Functions Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. type:: void (*TSConfigDestroyFunc)(void*) .. function:: unsigned int TSConfigSet(unsigned int id, void * data, TSConfigDestroyFunc funcp) diff --git a/doc/developer-guide/api/functions/TSContCall.en.rst b/doc/developer-guide/api/functions/TSContCall.en.rst index 8f99332ac27..d567fd47421 100644 --- a/doc/developer-guide/api/functions/TSContCall.en.rst +++ b/doc/developer-guide/api/functions/TSContCall.en.rst @@ -24,7 +24,9 @@ TSContCall Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: int TSContCall(TSCont contp, TSEvent event, void * edata) @@ -49,11 +51,11 @@ If there is a mutex associated with :arg:`contp`, :func:`TSContCall` assumes tha value returned by the handler in :arg:`contp`. If :arg:`contp` has a mutex, the plugin must acquire the lock on the mutex for :arg:`contp` before calling -:func:`TSContCall`. See :func:`TSContMutexGet` and :func:`TSMutexLockTry` for mechanisms for doing this. +:func:`TSContCall`. See :func:`TSContMutexGet` and :func:`TSMutexLockTry` for mechanisms for doing this. The most common case is the code called by :func:`TSContCall` must complete before further code is executed at the call site. An alternative approach to handling the locking directly would be to split the call site -into two continuations, one of which is signalled (possibly via :func:`TSContCall`) from the original +into two continuations, one of which is signalled (possibly via :func:`TSContCall`) from the original :func:`TSContCall` target. Note mutexes returned by :func:`TSMutexCreate` are recursive mutexes, therefore if the lock is @@ -67,4 +69,3 @@ shareable so that the same mutex can be used for multiple continuations.:: In this example case, :code:`cont1` can assume the lock for :code:`cont2` is held. This should be considered carefully because for the same reason any thread protection between the continuations is removed. This works well for tightly coupled continuations that always operate in a fixed sequence. - diff --git a/doc/developer-guide/api/functions/TSContCreate.en.rst b/doc/developer-guide/api/functions/TSContCreate.en.rst index 8a02884e12f..774343239e4 100644 --- a/doc/developer-guide/api/functions/TSContCreate.en.rst +++ b/doc/developer-guide/api/functions/TSContCreate.en.rst @@ -24,7 +24,9 @@ TSContCreate Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: TSCont TSContCreate(TSEventFunc funcp, TSMutex mutexp) diff --git a/doc/developer-guide/api/functions/TSContDataGet.en.rst b/doc/developer-guide/api/functions/TSContDataGet.en.rst index 79c6eb3e4e2..00ca8147f6f 100644 --- a/doc/developer-guide/api/functions/TSContDataGet.en.rst +++ b/doc/developer-guide/api/functions/TSContDataGet.en.rst @@ -24,7 +24,9 @@ TSContDataGet Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: void* TSContDataGet(TSCont contp) diff --git a/doc/developer-guide/api/functions/TSContDataSet.en.rst b/doc/developer-guide/api/functions/TSContDataSet.en.rst index 349e724a975..4e00c31592c 100644 --- a/doc/developer-guide/api/functions/TSContDataSet.en.rst +++ b/doc/developer-guide/api/functions/TSContDataSet.en.rst @@ -24,7 +24,9 @@ TSContDataSet Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: void TSContDataSet(TSCont contp, void * data) diff --git a/doc/developer-guide/api/functions/TSContDestroy.en.rst b/doc/developer-guide/api/functions/TSContDestroy.en.rst index 53b6e1e8ce7..d973fcc3959 100644 --- a/doc/developer-guide/api/functions/TSContDestroy.en.rst +++ b/doc/developer-guide/api/functions/TSContDestroy.en.rst @@ -24,7 +24,9 @@ TSContDestroy Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: void TSContDestroy(TSCont contp) diff --git a/doc/developer-guide/api/functions/TSContMutexGet.en.rst b/doc/developer-guide/api/functions/TSContMutexGet.en.rst index fe2b8f3eddd..4986b08eba3 100644 --- a/doc/developer-guide/api/functions/TSContMutexGet.en.rst +++ b/doc/developer-guide/api/functions/TSContMutexGet.en.rst @@ -24,7 +24,9 @@ TSContMutexGet Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: TSMutex TSContMutexGet(TSCont contp) diff --git a/doc/developer-guide/api/functions/TSContSchedule.en.rst b/doc/developer-guide/api/functions/TSContSchedule.en.rst index dea45b01004..4dbb9f6a090 100644 --- a/doc/developer-guide/api/functions/TSContSchedule.en.rst +++ b/doc/developer-guide/api/functions/TSContSchedule.en.rst @@ -24,7 +24,9 @@ TSContSchedule Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: TSAction TSContSchedule(TSCont contp, TSHRTime timeout) @@ -42,8 +44,15 @@ another thread this can be problematic to be correctly timed. The return value c :func:`TSActionDone` to see if the continuation ran before the return, which is possible if :arg:`timeout` is `0`. Returns ``nullptr`` if thread affinity was cleared. +TSContSchedule() or TSContScheduleEvery() will default to set the thread affinity to the calling thread +when no affinity is already set for example, using :func:`TSContThreadAffinitySet` + +Note that the TSContSchedule() family of API shall only be called from an ATS EThread. +Calling it from raw non-EThreads can result in unpredictable behavior. + See Also ======== +:doc:`TSContScheduleEvery.en` :doc:`TSContScheduleOnPool.en` :doc:`TSContScheduleOnThread.en` diff --git a/doc/developer-guide/api/functions/TSContScheduleEvery.en.rst b/doc/developer-guide/api/functions/TSContScheduleEvery.en.rst new file mode 100644 index 00000000000..bc61392cac1 --- /dev/null +++ b/doc/developer-guide/api/functions/TSContScheduleEvery.en.rst @@ -0,0 +1,58 @@ +.. Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed + with this work for additional information regarding copyright + ownership. The ASF licenses this file to you under the Apache + License, Version 2.0 (the "License"); you may not use this file + except in compliance with the License. You may obtain a copy of + the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied. See the License for the specific language governing + permissions and limitations under the License. + +.. include:: ../../../common.defs + +.. default-domain:: c + +TSContScheduleEvery +******************* + +Synopsis +======== + +.. code-block:: cpp + + #include + +.. function:: TSAction TSContScheduleEvery(TSCont contp, TSHRTime every) + +Description +=========== + +Schedules :arg:`contp` to periodically run every :arg:`delay` milliseconds in the future. +This is approximate. The delay will be at least :arg:`delay` but possibly more. +Resolutions finer than roughly 5 milliseconds will not be effective. :arg:`contp` is +required to have a mutex, which is provided to :func:`TSContCreate`. + +The return value can be used to cancel the scheduled event via :func:`TSActionCancel`. This is +effective until the continuation :arg:`contp` is being dispatched. However, if it is scheduled on +another thread this can be problematic to be correctly timed. The return value can be checked with +:func:`TSActionDone` to see if the continuation ran before the return, which is possible if +:arg:`timeout` is `0`. Returns ``nullptr`` if thread affinity was cleared. + +TSContSchedule() or TSContScheduleEvery() will default to set the thread affinity to the calling thread +when no affinity is already set for example, using :func:`TSContThreadAffinitySet` + +Note that the TSContSchedule() family of API shall only be called from an ATS EThread. +Calling it from raw non-EThreads can result in unpredictable behavior. + +See Also +======== + +:doc:`TSContSchedule.en` +:doc:`TSContScheduleOnPool.en` +:doc:`TSContScheduleOnThread.en` diff --git a/doc/developer-guide/api/functions/TSContScheduleOnPool.en.rst b/doc/developer-guide/api/functions/TSContScheduleOnPool.en.rst index f35ddb95fa9..1517f87a0b2 100644 --- a/doc/developer-guide/api/functions/TSContScheduleOnPool.en.rst +++ b/doc/developer-guide/api/functions/TSContScheduleOnPool.en.rst @@ -24,7 +24,9 @@ TSContScheduleOnPool Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: TSAction TSContScheduleOnPool(TSCont contp, TSHRTime timeout, TSThreadPool tp) @@ -42,11 +44,8 @@ Pool Properties =========================== ======================================================================================= ``TS_THREAD_POOL_NET`` Transaction processing threads. Continuations on these threads must not block. ``TS_THREAD_POOL_TASK`` Background threads. Continuations can perform blocking operations. -``TS_THREAD_POOL_SSL`` *DEPRECATED* - these are no longer used as of ATS 6. ``TS_THREAD_POOL_DNS`` DNS request processing. May not exist depending on configuration. Not recommended. -``TS_THREAD_POOL_REMAP`` *DEPRECATED* - these are no longer used. -``TS_THREAD_POOL_CLUSTER`` *DEPRECATED* - these are no longer used as of ATS 7. -``TS_THREAD_POOL_UDP`` *DEPRECATED* +``TS_THREAD_POOL_UDP`` UDP processing. =========================== ======================================================================================= In practice, any choice except ``TS_THREAD_POOL_NET`` or ``TS_THREAD_POOL_TASK`` is strongly not @@ -55,6 +54,9 @@ called and continuations that use them have the same restrictions. ``TS_THREAD_P are threads that exist to perform long or blocking actions, although sufficiently long operation can impact system performance by blocking other continuations on the threads. +Note that the TSContSchedule() family of API shall only be called from an ATS EThread. +Calling it from raw non-EThreads can result in unpredictable behavior. + Example Scenarios ================= @@ -87,7 +89,7 @@ schedules "contp" on thread "A", and assigns thread "A" as the affinity thread f The reason behind this choice is that we are trying to keep things simple such that lock contention problems happens less. And for the most part, there is no point of scheduling the same thing on several different threads of the same type, because there is no parallelism between them (a thread will have to -wait for the previous thread to finish, either because locking or the nature of the job it’s handling is +wait for the previous thread to finish, either because locking or the nature of the job it's handling is serialized since its on the same continuation). Scenario 3 (has thread affinity info, different types of threads) @@ -104,7 +106,7 @@ be the affinity thread for "contp" already, the system will NOT overwrite that i Most of the time, a continuation will be scheduled on one type of threads, and rarely gets scheduled on a different type. But when that happens, we want it to return to the thread it was previously on, so it -won’t have any lock contention problems. And that’s also why "thread_affinity" is not a hashmap of thread +won't have any lock contention problems. And that's also why "thread_affinity" is not a hashmap of thread types and thread pointers. Scenario 4 (has thread affinity info, same types of threads) @@ -123,4 +125,5 @@ See Also ======== :doc:`TSContSchedule.en` +:doc:`TSContScheduleEvery.en` :doc:`TSContScheduleOnThread.en` diff --git a/doc/developer-guide/api/functions/TSContScheduleOnThread.en.rst b/doc/developer-guide/api/functions/TSContScheduleOnThread.en.rst index fed6313c4e8..0f91166b9c0 100644 --- a/doc/developer-guide/api/functions/TSContScheduleOnThread.en.rst +++ b/doc/developer-guide/api/functions/TSContScheduleOnThread.en.rst @@ -24,7 +24,9 @@ TSContScheduleOnThread Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: TSAction TSContScheduleOnThread(TSCont contp, TSHRTime timeout, TSEventThread ethread) @@ -33,8 +35,12 @@ Description Mostly the same as :func:`TSContSchedule`. Schedules :arg:`contp` on :arg:`ethread`. +Note that the TSContSchedule() family of API shall only be called from an ATS EThread. +Calling it from raw non-EThreads can result in unpredictable behavior. + See Also ======== :doc:`TSContSchedule.en` +:doc:`TSContScheduleEvery.en` :doc:`TSContScheduleOnPool.en` diff --git a/doc/developer-guide/api/functions/TSContThreadAffinityClear.en.rst b/doc/developer-guide/api/functions/TSContThreadAffinityClear.en.rst index ef17ae0cddc..b5d1e325473 100644 --- a/doc/developer-guide/api/functions/TSContThreadAffinityClear.en.rst +++ b/doc/developer-guide/api/functions/TSContThreadAffinityClear.en.rst @@ -24,7 +24,9 @@ TSContThreadAffinityClear Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: void TSContThreadAffinityClear(TSCont contp) diff --git a/doc/developer-guide/api/functions/TSContThreadAffinityGet.en.rst b/doc/developer-guide/api/functions/TSContThreadAffinityGet.en.rst index 2bc4955ad19..322240395bb 100644 --- a/doc/developer-guide/api/functions/TSContThreadAffinityGet.en.rst +++ b/doc/developer-guide/api/functions/TSContThreadAffinityGet.en.rst @@ -24,7 +24,9 @@ TSContThreadAffinityGet Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: TSEventThread TSContThreadAffinityGet(TSCont contp) diff --git a/doc/developer-guide/api/functions/TSContThreadAffinitySet.en.rst b/doc/developer-guide/api/functions/TSContThreadAffinitySet.en.rst index 5b355130af8..e78979c78ed 100644 --- a/doc/developer-guide/api/functions/TSContThreadAffinitySet.en.rst +++ b/doc/developer-guide/api/functions/TSContThreadAffinitySet.en.rst @@ -24,7 +24,9 @@ TSContThreadAffinitySet Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: TSReturnCode TSContThreadAffinitySet(TSCont contp, TSEventThread ethread) @@ -35,6 +37,9 @@ Set the thread affinity of continuation :arg:`contp` to :arg:`ethread`. Future c :func:`TSContSchedule`, and :func:`TSContScheduleOnPool` that has the same type as :arg:`ethread` will schedule the continuation on :arg:`ethread`, rather than an arbitrary thread of that type. +:func:`TSContSchedule` and :func:``TSContScheduleEvery` will default the affinity to calling thread +when invoked without explicitly setting the thread affinity. + Return Values ============= diff --git a/doc/developer-guide/api/functions/TSDebug.en.rst b/doc/developer-guide/api/functions/TSDebug.en.rst index 110131bbf63..3273dbd91d2 100644 --- a/doc/developer-guide/api/functions/TSDebug.en.rst +++ b/doc/developer-guide/api/functions/TSDebug.en.rst @@ -26,7 +26,9 @@ Traffic Server Debugging APIs. Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: void TSError(const char * format, ...) .. function:: void TSFatal(const char * format, ...) diff --git a/doc/developer-guide/api/functions/TSEventThreadSelf.en.rst b/doc/developer-guide/api/functions/TSEventThreadSelf.en.rst index 0c7728aac6b..badb24a33ea 100644 --- a/doc/developer-guide/api/functions/TSEventThreadSelf.en.rst +++ b/doc/developer-guide/api/functions/TSEventThreadSelf.en.rst @@ -24,7 +24,9 @@ TSEventThreadSelf Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: TSEventThread TSEventThreadSelf(void) diff --git a/doc/developer-guide/api/functions/TSFetchCreate.en.rst b/doc/developer-guide/api/functions/TSFetchCreate.en.rst new file mode 100644 index 00000000000..4dc4924632f --- /dev/null +++ b/doc/developer-guide/api/functions/TSFetchCreate.en.rst @@ -0,0 +1,55 @@ +.. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +.. include:: ../../../common.defs + +.. default-domain:: c + +TSFetchCreate +************* + +Traffic Server asynchronous Fetch API. + +Synopsis +======== + +.. code-block:: cpp + + #include + +.. function:: void TSFetchPages(TSFetchUrlParams_t *) +.. function:: TSFetchSM TSFetchUrl(const char *, int, sockaddr const *, TSCont, TSFetchWakeUpOptions, TSFetchEvent) +.. function:: void TSFetchFlagSet(TSFetchSM, int) +.. function:: TSFetchSM TSFetchCreate(TSCont, const char *, const char *, const char *, struct sockaddr const *, int) +.. function:: void TSFetchHeaderAdd(TSFetchSM, const char *, int, const char *, int) +.. function:: void TSFetchWriteData(TSFetchSM, const void *, size_t) +.. function:: ssize_t TSFetchReadData(TSFetchSM, void *, size_t) +.. function:: void TSFetchLaunch(TSFetchSM) +.. function:: void TSFetchDestroy(TSFetchSM) +.. function:: void TSFetchUserDataSet(TSFetchSM, void *) +.. function:: void* TSFetchUserDataGet(TSFetchSM) +.. function:: TSMBuffer TSFetchRespHdrMBufGet(TSFetchSM) +.. function:: TSMLoc TSFetchRespHdrMLocGet(TSFetchSM) + +Description +=========== + +Traffic Server provides a number of routines for fetching resources asynchronously. +These API are useful to support a number of use cases that may involve sideways +calls, while handling the client request. Some typical examples include centralized +rate limiting framework, database lookups for login/authentication, refreshing configs +in the background asynchronously, ESI etc. diff --git a/doc/developer-guide/api/functions/TSHostLookup.en.rst b/doc/developer-guide/api/functions/TSHostLookup.en.rst index dc350ad8f52..d4accda71d3 100644 --- a/doc/developer-guide/api/functions/TSHostLookup.en.rst +++ b/doc/developer-guide/api/functions/TSHostLookup.en.rst @@ -24,7 +24,9 @@ TSHostLookup Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: TSAction TSHostLookup(TSCont contp, const char * hostname, size_t namelen) diff --git a/doc/developer-guide/api/functions/TSHostLookupResultAddrGet.en.rst b/doc/developer-guide/api/functions/TSHostLookupResultAddrGet.en.rst index daff4e8ccfd..35522837e89 100644 --- a/doc/developer-guide/api/functions/TSHostLookupResultAddrGet.en.rst +++ b/doc/developer-guide/api/functions/TSHostLookupResultAddrGet.en.rst @@ -24,7 +24,9 @@ TSHostLookupResultAddrGet Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: sockaddr const* TSHostLookupResultAddrGet(TSHostLookupResult lookup_result) diff --git a/doc/developer-guide/api/functions/TSHttpArgs.en.rst b/doc/developer-guide/api/functions/TSHttpArgs.en.rst index d1526807d66..a564eb2d4f9 100644 --- a/doc/developer-guide/api/functions/TSHttpArgs.en.rst +++ b/doc/developer-guide/api/functions/TSHttpArgs.en.rst @@ -17,13 +17,22 @@ .. include:: ../../../common.defs .. default-domain:: c + TSHttpArgs ********** Synopsis ======== -`#include ` +.. note:: + + This set of API is obsoleted as of ATS v9.0.0, and will be removed with ATS v10.0.0! + For details of the new APIs, see :ref:`tsuserargs`. + + +.. code-block:: cpp + + #include .. function:: TSReturnCode TSHttpTxnArgIndexReserve(const char * name, const char * description, int * arg_idx) .. function:: TSReturnCode TSHttpTxnArgIndexNameLookup(const char * name, int * arg__idx, const char ** description) @@ -33,8 +42,8 @@ Synopsis .. function:: TSReturnCode TSHttpSsnArgIndexLookup(int arg_idx, const char ** name, const char ** description) .. function:: void TSHttpTxnArgSet(TSHttpTxn txnp, int arg_idx, void * arg) .. function:: void * TSHttpTxnArgGet(TSHttpTxn txnp, int arg_idx) -.. function:: void TSHttpSsnArgSet(TSHttpTxn txnp, int arg_idx, void * arg) -.. function:: void * TSHttpSsnArgGet(TSHttpTxn txnp, int arg_idx) +.. function:: void TSHttpSsnArgSet(TSHttpSsn ssnp, int arg_idx, void * arg) +.. function:: void * TSHttpSsnArgGet(TSHttpSsn ssnp, int arg_idx) Description =========== diff --git a/doc/developer-guide/api/functions/TSHttpConnect.en.rst b/doc/developer-guide/api/functions/TSHttpConnect.en.rst index 958346ea853..547ec5079fa 100644 --- a/doc/developer-guide/api/functions/TSHttpConnect.en.rst +++ b/doc/developer-guide/api/functions/TSHttpConnect.en.rst @@ -24,7 +24,9 @@ TSHttpConnect Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: TSVConn TSHttpConnect(sockaddr const * addr) diff --git a/doc/developer-guide/api/functions/TSHttpConnectWithPluginId.en.rst b/doc/developer-guide/api/functions/TSHttpConnectWithPluginId.en.rst index 0f6186f5ec9..fd3b7cc1d96 100644 --- a/doc/developer-guide/api/functions/TSHttpConnectWithPluginId.en.rst +++ b/doc/developer-guide/api/functions/TSHttpConnectWithPluginId.en.rst @@ -29,7 +29,9 @@ as if it came from a client. Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: TSVConn TSHttpConnectWithPluginId(sockaddr const * addr, char const * tag, int64_t id) diff --git a/doc/developer-guide/api/functions/TSHttpHdrClone.en.rst b/doc/developer-guide/api/functions/TSHttpHdrClone.en.rst index ab7ee2ab576..de88b09b35a 100644 --- a/doc/developer-guide/api/functions/TSHttpHdrClone.en.rst +++ b/doc/developer-guide/api/functions/TSHttpHdrClone.en.rst @@ -24,7 +24,9 @@ TSHttpHdrClone Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: TSReturnCode TSHttpHdrClone(TSMBuffer dest_bufp, TSMBuffer src_bufp, TSMLoc src_hdr, TSMLoc * locp) diff --git a/doc/developer-guide/api/functions/TSHttpHdrCopy.en.rst b/doc/developer-guide/api/functions/TSHttpHdrCopy.en.rst index feff3e2dd00..88e597d2844 100644 --- a/doc/developer-guide/api/functions/TSHttpHdrCopy.en.rst +++ b/doc/developer-guide/api/functions/TSHttpHdrCopy.en.rst @@ -28,7 +28,9 @@ Copies the contents of the HTTP header located at :arg:`src_loc` within Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: TSReturnCode TSHttpHdrCopy(TSMBuffer dest_bufp, TSMLoc dest_offset, TSMBuffer src_bufp, TSMLoc src_offset) diff --git a/doc/developer-guide/api/functions/TSHttpHdrCreate.en.rst b/doc/developer-guide/api/functions/TSHttpHdrCreate.en.rst index a4a7a8226e9..a478a66dab8 100644 --- a/doc/developer-guide/api/functions/TSHttpHdrCreate.en.rst +++ b/doc/developer-guide/api/functions/TSHttpHdrCreate.en.rst @@ -24,7 +24,9 @@ TSHttpHdrCreate Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: TSMLoc TSHttpHdrCreate(TSMBuffer bufp) diff --git a/doc/developer-guide/api/functions/TSHttpHdrDestroy.en.rst b/doc/developer-guide/api/functions/TSHttpHdrDestroy.en.rst index 2bf8254d1ff..a56b2b6d454 100644 --- a/doc/developer-guide/api/functions/TSHttpHdrDestroy.en.rst +++ b/doc/developer-guide/api/functions/TSHttpHdrDestroy.en.rst @@ -24,7 +24,9 @@ TSHttpHdrDestroy Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: void TSHttpHdrDestroy(TSMBuffer bufp, TSMLoc offset) diff --git a/doc/developer-guide/api/functions/TSHttpHdrHostGet.en.rst b/doc/developer-guide/api/functions/TSHttpHdrHostGet.en.rst index 710834f2965..420b17307e9 100644 --- a/doc/developer-guide/api/functions/TSHttpHdrHostGet.en.rst +++ b/doc/developer-guide/api/functions/TSHttpHdrHostGet.en.rst @@ -24,7 +24,9 @@ TSHttpHdrHostGet Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: const char* TSHttpHdrHostGet(TSMBuffer bufp, TSMLoc offset, int * length) diff --git a/doc/developer-guide/api/functions/TSHttpHdrLengthGet.en.rst b/doc/developer-guide/api/functions/TSHttpHdrLengthGet.en.rst index 495ef1a991c..12e4a43870b 100644 --- a/doc/developer-guide/api/functions/TSHttpHdrLengthGet.en.rst +++ b/doc/developer-guide/api/functions/TSHttpHdrLengthGet.en.rst @@ -24,7 +24,9 @@ TSHttpHdrLengthGet Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: int TSHttpHdrLengthGet(TSMBuffer bufp, TSMLoc mloc) diff --git a/doc/developer-guide/api/functions/TSHttpHdrMethodGet.en.rst b/doc/developer-guide/api/functions/TSHttpHdrMethodGet.en.rst index f3eaec8c17c..eda364a4d2f 100644 --- a/doc/developer-guide/api/functions/TSHttpHdrMethodGet.en.rst +++ b/doc/developer-guide/api/functions/TSHttpHdrMethodGet.en.rst @@ -24,7 +24,9 @@ TSHttpHdrMethodGet Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: const char* TSHttpHdrMethodGet(TSMBuffer bufp, TSMLoc offset, int * length) diff --git a/doc/developer-guide/api/functions/TSHttpHdrMethodSet.en.rst b/doc/developer-guide/api/functions/TSHttpHdrMethodSet.en.rst index bded2e11616..1ec1973226e 100644 --- a/doc/developer-guide/api/functions/TSHttpHdrMethodSet.en.rst +++ b/doc/developer-guide/api/functions/TSHttpHdrMethodSet.en.rst @@ -24,7 +24,9 @@ TSHttpHdrMethodSet Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: TSReturnCode TSHttpHdrMethodSet(TSMBuffer bufp, TSMLoc offset, const char * value, int length) diff --git a/doc/developer-guide/api/functions/TSHttpHdrPrint.en.rst b/doc/developer-guide/api/functions/TSHttpHdrPrint.en.rst index 1f6c1d75392..51b7b998914 100644 --- a/doc/developer-guide/api/functions/TSHttpHdrPrint.en.rst +++ b/doc/developer-guide/api/functions/TSHttpHdrPrint.en.rst @@ -24,7 +24,9 @@ TSHttpHdrPrint Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: void TSHttpHdrPrint(TSMBuffer bufp, TSMLoc offset, TSIOBuffer iobufp) diff --git a/doc/developer-guide/api/functions/TSHttpHdrReasonGet.en.rst b/doc/developer-guide/api/functions/TSHttpHdrReasonGet.en.rst index b3565197710..2a49d5fcc58 100644 --- a/doc/developer-guide/api/functions/TSHttpHdrReasonGet.en.rst +++ b/doc/developer-guide/api/functions/TSHttpHdrReasonGet.en.rst @@ -24,7 +24,9 @@ TSHttpHdrReasonGet Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: const char* TSHttpHdrReasonGet(TSMBuffer bufp, TSMLoc offset, int * length) diff --git a/doc/developer-guide/api/functions/TSHttpHdrReasonLookup.en.rst b/doc/developer-guide/api/functions/TSHttpHdrReasonLookup.en.rst index 9b85b15e0bb..4d97f2e319a 100644 --- a/doc/developer-guide/api/functions/TSHttpHdrReasonLookup.en.rst +++ b/doc/developer-guide/api/functions/TSHttpHdrReasonLookup.en.rst @@ -21,7 +21,9 @@ TSHttpHdrReasonLookup Synopsis -------- -`#include ` +.. code-block:: cpp + + #include .. c:function:: const char* TSHttpHdrReasonLookup(TSHttpStatus status) diff --git a/doc/developer-guide/api/functions/TSHttpHdrReasonSet.en.rst b/doc/developer-guide/api/functions/TSHttpHdrReasonSet.en.rst index a1b15249e8d..3f35d99bb77 100644 --- a/doc/developer-guide/api/functions/TSHttpHdrReasonSet.en.rst +++ b/doc/developer-guide/api/functions/TSHttpHdrReasonSet.en.rst @@ -24,7 +24,9 @@ TSHttpHdrReasonSet Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: TSReturnCode TSHttpHdrReasonSet(TSMBuffer bufp, TSMLoc offset, const char * value, int length) diff --git a/doc/developer-guide/api/functions/TSHttpHdrStatusGet.en.rst b/doc/developer-guide/api/functions/TSHttpHdrStatusGet.en.rst index 7d2e31dbfbe..5a6c9e499f7 100644 --- a/doc/developer-guide/api/functions/TSHttpHdrStatusGet.en.rst +++ b/doc/developer-guide/api/functions/TSHttpHdrStatusGet.en.rst @@ -24,7 +24,9 @@ TSHttpHdrStatusGet Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: TSHttpStatus TSHttpHdrStatusGet(TSMBuffer bufp, TSMLoc offset) diff --git a/doc/developer-guide/api/functions/TSHttpHdrStatusSet.en.rst b/doc/developer-guide/api/functions/TSHttpHdrStatusSet.en.rst index bccc5e4bfeb..5dbb3a70d58 100644 --- a/doc/developer-guide/api/functions/TSHttpHdrStatusSet.en.rst +++ b/doc/developer-guide/api/functions/TSHttpHdrStatusSet.en.rst @@ -24,7 +24,9 @@ TSHttpHdrStatusSet Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: TSReturnCode TSHttpHdrStatusSet(TSMBuffer bufp, TSMLoc offset, TSHttpStatus status) diff --git a/doc/developer-guide/api/functions/TSHttpHdrTypeGet.en.rst b/doc/developer-guide/api/functions/TSHttpHdrTypeGet.en.rst index c0adeeb5c3a..4c3ce6769fb 100644 --- a/doc/developer-guide/api/functions/TSHttpHdrTypeGet.en.rst +++ b/doc/developer-guide/api/functions/TSHttpHdrTypeGet.en.rst @@ -24,7 +24,9 @@ TSHttpHdrTypeGet Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: TSHttpType TSHttpHdrTypeGet(TSMBuffer bufp, TSMLoc offset) diff --git a/doc/developer-guide/api/functions/TSHttpHdrTypeSet.en.rst b/doc/developer-guide/api/functions/TSHttpHdrTypeSet.en.rst index edfada47594..302af2adede 100644 --- a/doc/developer-guide/api/functions/TSHttpHdrTypeSet.en.rst +++ b/doc/developer-guide/api/functions/TSHttpHdrTypeSet.en.rst @@ -24,7 +24,9 @@ TSHttpHdrTypeSet Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: TSReturnCode TSHttpHdrTypeSet(TSMBuffer bufp, TSMLoc offset, TSHttpType type) diff --git a/doc/developer-guide/api/functions/TSHttpHdrUrlGet.en.rst b/doc/developer-guide/api/functions/TSHttpHdrUrlGet.en.rst index 555ede1715c..f474ce4981d 100644 --- a/doc/developer-guide/api/functions/TSHttpHdrUrlGet.en.rst +++ b/doc/developer-guide/api/functions/TSHttpHdrUrlGet.en.rst @@ -24,7 +24,9 @@ TSHttpHdrUrlGet Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: TSReturnCode TSHttpHdrUrlGet(TSMBuffer bufp, TSMLoc offset, TSMLoc * locp) diff --git a/doc/developer-guide/api/functions/TSHttpHdrUrlSet.en.rst b/doc/developer-guide/api/functions/TSHttpHdrUrlSet.en.rst index 5a10097f81f..6b64f3b6268 100644 --- a/doc/developer-guide/api/functions/TSHttpHdrUrlSet.en.rst +++ b/doc/developer-guide/api/functions/TSHttpHdrUrlSet.en.rst @@ -24,7 +24,9 @@ TSHttpHdrUrlSet Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: TSReturnCode TSHttpHdrUrlSet(TSMBuffer bufp, TSMLoc offset, TSMLoc url) diff --git a/doc/developer-guide/api/functions/TSHttpHdrVersionGet.en.rst b/doc/developer-guide/api/functions/TSHttpHdrVersionGet.en.rst index 6dfceada1fb..861a7822d0e 100644 --- a/doc/developer-guide/api/functions/TSHttpHdrVersionGet.en.rst +++ b/doc/developer-guide/api/functions/TSHttpHdrVersionGet.en.rst @@ -24,7 +24,9 @@ TSHttpHdrVersionGet Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: int TSHttpHdrVersionGet(TSMBuffer bufp, TSMLoc offset) diff --git a/doc/developer-guide/api/functions/TSHttpHdrVersionSet.en.rst b/doc/developer-guide/api/functions/TSHttpHdrVersionSet.en.rst index a63cbd83a73..5abc6db7904 100644 --- a/doc/developer-guide/api/functions/TSHttpHdrVersionSet.en.rst +++ b/doc/developer-guide/api/functions/TSHttpHdrVersionSet.en.rst @@ -24,7 +24,9 @@ TSHttpHdrVersionSet Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: TSReturnCode TSHttpHdrVersionSet(TSMBuffer bufp, TSMLoc offset, int ver) diff --git a/doc/developer-guide/api/functions/TSHttpHookAdd.en.rst b/doc/developer-guide/api/functions/TSHttpHookAdd.en.rst index 6443ae10b75..9536a3f88d1 100644 --- a/doc/developer-guide/api/functions/TSHttpHookAdd.en.rst +++ b/doc/developer-guide/api/functions/TSHttpHookAdd.en.rst @@ -27,7 +27,9 @@ Intercept Traffic Server events. Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: void TSHttpHookAdd(TSHttpHookID id, TSCont contp) .. function:: void TSHttpSsnHookAdd(TSHttpSsn ssnp, TSHttpHookID id, TSCont contp) diff --git a/doc/developer-guide/api/functions/TSHttpOverridableConfig.en.rst b/doc/developer-guide/api/functions/TSHttpOverridableConfig.en.rst index a3c063ba3c6..fae74b57b9e 100644 --- a/doc/developer-guide/api/functions/TSHttpOverridableConfig.en.rst +++ b/doc/developer-guide/api/functions/TSHttpOverridableConfig.en.rst @@ -27,7 +27,9 @@ TSHttpOverridableConfig Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: TSReturnCode TSHttpTxnConfigIntSet(TSHttpTxn txnp, TSOverridableConfigKey key, TSMgmtInt value) @@ -170,7 +172,6 @@ TSOverridableConfigKey Value Configuratio :c:macro:`TS_CONFIG_SRV_ENABLED` :ts:cv:`proxy.config.srv_enabled` :c:macro:`TS_CONFIG_SSL_CLIENT_CERT_FILENAME` :ts:cv:`proxy.config.ssl.client.cert.filename` :c:macro:`TS_CONFIG_SSL_CERT_FILEPATH` :ts:cv:`proxy.config.ssl.client.cert.path` -:c:macro:`TS_CONFIG_SSL_CLIENT_VERIFY_SERVER` :ts:cv:`proxy.config.ssl.client.verify.server` :c:macro:`TS_CONFIG_SSL_CLIENT_VERIFY_SERVER_PROPERTIES` :ts:cv:`proxy.config.ssl.client.verify.server.properties` :c:macro:`TS_CONFIG_SSL_CLIENT_VERIFY_SERVER_POLICY` :ts:cv:`proxy.config.ssl.client.verify.server.policy` :c:macro:`TS_CONFIG_SSL_CLIENT_SNI_POLICY` :ts:cv:`proxy.config.ssl.client.sni_policy` @@ -179,11 +180,10 @@ TSOverridableConfigKey Value Configuratio :c:macro:`TS_CONFIG_URL_REMAP_PRISTINE_HOST_HDR` :ts:cv:`proxy.config.url_remap.pristine_host_hdr` :c:macro:`TS_CONFIG_WEBSOCKET_ACTIVE_TIMEOUT` :ts:cv:`proxy.config.websocket.active_timeout` :c:macro:`TS_CONFIG_WEBSOCKET_NO_ACTIVITY_TIMEOUT` :ts:cv:`proxy.config.websocket.no_activity_timeout` -:c:macro:`TS_CONFIG_SSL_CLIENT_VERIFY_SERVER_POLICY` :ts:cv:`proxy.config.ssl.client.verify.server.policy` -:c:macro:`TS_CONFIG_SSL_CLIENT_VERIFY_SERVER_PROPERTIES` :ts:cv:`proxy.config.ssl.client.verify.server.properties` :c:macro:`TS_CONFIG_SSL_CLIENT_CERT_FILENAME` :ts:cv:`proxy.config.ssl.client.cert.filename` :c:macro:`TS_CONFIG_SSL_CLIENT_PRIVATE_KEY_FILENAME` :ts:cv:`proxy.config.ssl.client.private_key.filename` :c:macro:`TS_CONFIG_SSL_CLIENT_CA_CERT_FILENAME` :ts:cv:`proxy.config.ssl.client.CA.cert.filename` +:c:macro:`TS_CONFIG_HTTP_HOST_RESOLUTION_PREFERENCE` :ts:cv:`proxy.config.hostdb.ip_resolve` ================================================================== ==================================================================== Examples diff --git a/doc/developer-guide/api/functions/TSHttpParserCreate.en.rst b/doc/developer-guide/api/functions/TSHttpParserCreate.en.rst index e8974ac6f50..d03d1fb37ac 100644 --- a/doc/developer-guide/api/functions/TSHttpParserCreate.en.rst +++ b/doc/developer-guide/api/functions/TSHttpParserCreate.en.rst @@ -27,7 +27,9 @@ Parse HTTP headers from memory buffers. Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: TSHttpParser TSHttpParserCreate(void) .. function:: void TSHttpParserClear(TSHttpParser parser) diff --git a/doc/developer-guide/api/functions/TSHttpSsnClientFdGet.en.rst b/doc/developer-guide/api/functions/TSHttpSsnClientFdGet.en.rst index 8ecf094eba2..1134ad4822d 100644 --- a/doc/developer-guide/api/functions/TSHttpSsnClientFdGet.en.rst +++ b/doc/developer-guide/api/functions/TSHttpSsnClientFdGet.en.rst @@ -21,7 +21,9 @@ TSHttpSsnClientFdGet Synopsis -------- -`#include ` +.. code-block:: cpp + + #include .. c:function:: TSReturnCode TSHttpSsnClientFdGet(TSHttpTxn txnp, int *fdp) diff --git a/doc/developer-guide/api/functions/TSHttpSsnIdGet.en.rst b/doc/developer-guide/api/functions/TSHttpSsnIdGet.en.rst index a9b30aef44d..23ee8b2bf34 100644 --- a/doc/developer-guide/api/functions/TSHttpSsnIdGet.en.rst +++ b/doc/developer-guide/api/functions/TSHttpSsnIdGet.en.rst @@ -24,7 +24,9 @@ Returns the unique identifier for client session. Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: int64_t TSHttpSsnIdGet(TSHttpSsn ssnp) diff --git a/doc/developer-guide/api/functions/TSHttpSsnReenable.en.rst b/doc/developer-guide/api/functions/TSHttpSsnReenable.en.rst index 9dd0002d2ff..450f5ce554d 100644 --- a/doc/developer-guide/api/functions/TSHttpSsnReenable.en.rst +++ b/doc/developer-guide/api/functions/TSHttpSsnReenable.en.rst @@ -24,7 +24,9 @@ TSHttpSsnReenable Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: void TSHttpSsnReenable(TSHttpSsn ssnp, TSEvent event) diff --git a/doc/developer-guide/api/functions/TSHttpTxnAborted.en.rst b/doc/developer-guide/api/functions/TSHttpTxnAborted.en.rst index d0638c51adf..afa0658a4a7 100644 --- a/doc/developer-guide/api/functions/TSHttpTxnAborted.en.rst +++ b/doc/developer-guide/api/functions/TSHttpTxnAborted.en.rst @@ -24,7 +24,9 @@ TSHttpTxnAborted Synopsis -------- -`#include ` +.. code-block:: cpp + + #include .. c:function:: TSReturnCode TSHttpTxnAborted(TSHttpTxn txnp) @@ -39,4 +41,4 @@ Return values ------------- The API returns :c:data:`TS_SUCCESS`, if the requested transaction is aborted, -:c:data:`TS_ERROR` otherwise. \ No newline at end of file +:c:data:`TS_ERROR` otherwise. diff --git a/doc/developer-guide/api/functions/TSHttpTxnCacheLookupStatusGet.en.rst b/doc/developer-guide/api/functions/TSHttpTxnCacheLookupStatusGet.en.rst index d9d1b6fd44d..04a33bac3a6 100644 --- a/doc/developer-guide/api/functions/TSHttpTxnCacheLookupStatusGet.en.rst +++ b/doc/developer-guide/api/functions/TSHttpTxnCacheLookupStatusGet.en.rst @@ -24,7 +24,9 @@ TSHttpTxnCacheLookupStatusGet Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: TSReturnCode TSHttpTxnCacheLookupStatusGet(TSHttpTxn txnp, int * lookup_status) diff --git a/doc/developer-guide/api/functions/TSHttpTxnCacheLookupUrlGet.en.rst b/doc/developer-guide/api/functions/TSHttpTxnCacheLookupUrlGet.en.rst index 7f6545941ae..4b5acc346db 100644 --- a/doc/developer-guide/api/functions/TSHttpTxnCacheLookupUrlGet.en.rst +++ b/doc/developer-guide/api/functions/TSHttpTxnCacheLookupUrlGet.en.rst @@ -24,7 +24,9 @@ TSHttpTxnCacheLookupUrlGet Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: TSReturnCode TSHttpTxnCacheLookupUrlGet(TSHttpTxn txnp, TSMBuffer bufp, TSMLoc offset) @@ -41,7 +43,9 @@ TSHttpTxnCacheLookupUrlSet Synopsis -------- -`#include ` +.. code-block:: cpp + + #include .. c:function:: TSReturnCode TSHttpTxnCacheLookupUrlSet(TSHttpTxn txnp, TSMBuffer bufp, TSMLoc offset) diff --git a/doc/developer-guide/api/functions/TSHttpTxnCachedReqGet.en.rst b/doc/developer-guide/api/functions/TSHttpTxnCachedReqGet.en.rst index 7fb6117b95c..215f6f73f93 100644 --- a/doc/developer-guide/api/functions/TSHttpTxnCachedReqGet.en.rst +++ b/doc/developer-guide/api/functions/TSHttpTxnCachedReqGet.en.rst @@ -24,7 +24,9 @@ TSHttpTxnCachedReqGet Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: TSReturnCode TSHttpTxnCachedReqGet(TSHttpTxn txnp, TSMBuffer * bufp, TSMLoc * offset) diff --git a/doc/developer-guide/api/functions/TSHttpTxnCachedRespGet.en.rst b/doc/developer-guide/api/functions/TSHttpTxnCachedRespGet.en.rst index 15823039ce3..48257a4d446 100644 --- a/doc/developer-guide/api/functions/TSHttpTxnCachedRespGet.en.rst +++ b/doc/developer-guide/api/functions/TSHttpTxnCachedRespGet.en.rst @@ -24,7 +24,9 @@ TSHttpTxnCachedRespGet Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: TSReturnCode TSHttpTxnCachedRespGet(TSHttpTxn txnp, TSMBuffer * bufp, TSMLoc * offset) diff --git a/doc/developer-guide/api/functions/TSHttpTxnClientFdGet.en.rst b/doc/developer-guide/api/functions/TSHttpTxnClientFdGet.en.rst index 4c61160696e..5a4f68239ce 100644 --- a/doc/developer-guide/api/functions/TSHttpTxnClientFdGet.en.rst +++ b/doc/developer-guide/api/functions/TSHttpTxnClientFdGet.en.rst @@ -21,7 +21,9 @@ TSHttpTxnClientFdGet Synopsis -------- -`#include ` +.. code-block:: cpp + + #include .. c:function:: TSReturnCode TSHttpTxnClientFdGet(TSHttpTxn txnp, int *fdp) diff --git a/doc/developer-guide/api/functions/TSHttpTxnClientPacketDscpSet.en.rst b/doc/developer-guide/api/functions/TSHttpTxnClientPacketDscpSet.en.rst index 0370735f463..47fc7ab3ca3 100644 --- a/doc/developer-guide/api/functions/TSHttpTxnClientPacketDscpSet.en.rst +++ b/doc/developer-guide/api/functions/TSHttpTxnClientPacketDscpSet.en.rst @@ -23,7 +23,9 @@ TSHttpTxnClientPacketDscpSet Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: TSReturnCode TSHttpTxnClientPacketDscpSet(TSHttpTxn txnp, int dscp) diff --git a/doc/developer-guide/api/functions/TSHttpTxnClientPacketMarkSet.en.rst b/doc/developer-guide/api/functions/TSHttpTxnClientPacketMarkSet.en.rst index 437f528b036..1f3cac5f72d 100644 --- a/doc/developer-guide/api/functions/TSHttpTxnClientPacketMarkSet.en.rst +++ b/doc/developer-guide/api/functions/TSHttpTxnClientPacketMarkSet.en.rst @@ -23,7 +23,9 @@ TSHttpTxnClientPacketMarkSet Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: TSReturnCode TSHttpTxnClientPacketMarkSet(TSHttpTxn txnp, int mark) diff --git a/doc/developer-guide/api/functions/TSHttpTxnClientPacketTosSet.en.rst b/doc/developer-guide/api/functions/TSHttpTxnClientPacketTosSet.en.rst index f1b694b9167..47b6dce243a 100644 --- a/doc/developer-guide/api/functions/TSHttpTxnClientPacketTosSet.en.rst +++ b/doc/developer-guide/api/functions/TSHttpTxnClientPacketTosSet.en.rst @@ -23,7 +23,9 @@ TSHttpTxnClientPacketTosSet Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: TSReturnCode TSHttpTxnClientPacketTosSet(TSHttpTxn txnp, int tos) diff --git a/doc/developer-guide/api/functions/TSHttpTxnClientReqGet.en.rst b/doc/developer-guide/api/functions/TSHttpTxnClientReqGet.en.rst index 1d59ab13970..b421e35a848 100644 --- a/doc/developer-guide/api/functions/TSHttpTxnClientReqGet.en.rst +++ b/doc/developer-guide/api/functions/TSHttpTxnClientReqGet.en.rst @@ -24,7 +24,9 @@ TSHttpTxnClientReqGet Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: TSReturnCode TSHttpTxnClientReqGet(TSHttpTxn txnp, TSMBuffer * bufp, TSMLoc * offset) diff --git a/doc/developer-guide/api/functions/TSHttpTxnClientRespGet.en.rst b/doc/developer-guide/api/functions/TSHttpTxnClientRespGet.en.rst index cd93d94b3e6..fb5a72c0773 100644 --- a/doc/developer-guide/api/functions/TSHttpTxnClientRespGet.en.rst +++ b/doc/developer-guide/api/functions/TSHttpTxnClientRespGet.en.rst @@ -24,7 +24,9 @@ TSHttpTxnClientRespGet Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: TSReturnCode TSHttpTxnClientRespGet(TSHttpTxn txnp, TSMBuffer * bufp, TSMLoc * offset) diff --git a/doc/developer-guide/api/functions/TSHttpTxnErrorBodySet.en.rst b/doc/developer-guide/api/functions/TSHttpTxnErrorBodySet.en.rst index 52fc4c01c71..082ba18d7b1 100644 --- a/doc/developer-guide/api/functions/TSHttpTxnErrorBodySet.en.rst +++ b/doc/developer-guide/api/functions/TSHttpTxnErrorBodySet.en.rst @@ -26,7 +26,9 @@ Sets an error type body to a transaction. Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: void TSHttpTxnErrorBodySet(TSHttpTxn txnp, char * buf, size_t buflength, char * mimetype) diff --git a/doc/developer-guide/api/functions/TSHttpTxnIncomingAddrGet.en.rst b/doc/developer-guide/api/functions/TSHttpTxnIncomingAddrGet.en.rst index cd691c13464..773b523be84 100644 --- a/doc/developer-guide/api/functions/TSHttpTxnIncomingAddrGet.en.rst +++ b/doc/developer-guide/api/functions/TSHttpTxnIncomingAddrGet.en.rst @@ -24,7 +24,9 @@ Get the incoming proxy address on which a client's connection is accepted. Synopsis -------- -`#include ` +.. code-block:: cpp + + #include .. c:function:: sockaddr const* TSHttpTxnIncomingAddrGet(TSHttpTxn txnp) diff --git a/doc/developer-guide/api/functions/TSHttpTxnInfoIntGet.en.rst b/doc/developer-guide/api/functions/TSHttpTxnInfoIntGet.en.rst index a8d2eeaa882..f2899660c65 100644 --- a/doc/developer-guide/api/functions/TSHttpTxnInfoIntGet.en.rst +++ b/doc/developer-guide/api/functions/TSHttpTxnInfoIntGet.en.rst @@ -21,7 +21,9 @@ TSHttpTxnInfoIntGet Synopsis -------- -`#include ` +.. code-block:: cpp + + #include .. c:function:: TSReturnCode TSHttpTxnInfoIntGet(TSHttpTxn txnp, TSHttpTxnInfoKey key, TSMgmtInt * value) diff --git a/doc/developer-guide/api/functions/TSHttpTxnIntercept.en.rst b/doc/developer-guide/api/functions/TSHttpTxnIntercept.en.rst index 52db4ab8ee6..43160b18f7b 100644 --- a/doc/developer-guide/api/functions/TSHttpTxnIntercept.en.rst +++ b/doc/developer-guide/api/functions/TSHttpTxnIntercept.en.rst @@ -27,7 +27,9 @@ origin server. Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: void TSHttpTxnIntercept(TSCont contp, TSHttpTxn txnp) diff --git a/doc/developer-guide/api/functions/TSHttpTxnIsCacheable.en.rst b/doc/developer-guide/api/functions/TSHttpTxnIsCacheable.en.rst index 58f40e467a9..bfb61cf9eaa 100644 --- a/doc/developer-guide/api/functions/TSHttpTxnIsCacheable.en.rst +++ b/doc/developer-guide/api/functions/TSHttpTxnIsCacheable.en.rst @@ -21,7 +21,9 @@ TSHttpTxnIsCacheable Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: TSReturnCode TSHttpTxnIsCacheable(TSHttpTxn txnp, TSMBuffer request, TSMBuffer response) diff --git a/doc/developer-guide/api/functions/TSHttpTxnIsInternal.en.rst b/doc/developer-guide/api/functions/TSHttpTxnIsInternal.en.rst index d0804bbc538..a3dd6757d51 100644 --- a/doc/developer-guide/api/functions/TSHttpTxnIsInternal.en.rst +++ b/doc/developer-guide/api/functions/TSHttpTxnIsInternal.en.rst @@ -24,7 +24,9 @@ Test whether a request is internally-generated. Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: int TSHttpTxnIsInternal(TSHttpTxn txnp) .. function:: int TSHttpSsnIsInternal(TSHttpSsn ssnp) diff --git a/doc/developer-guide/api/functions/TSHttpTxnIsWebsocket.en.rst b/doc/developer-guide/api/functions/TSHttpTxnIsWebsocket.en.rst index c60dc9dbf72..e73f84556b0 100644 --- a/doc/developer-guide/api/functions/TSHttpTxnIsWebsocket.en.rst +++ b/doc/developer-guide/api/functions/TSHttpTxnIsWebsocket.en.rst @@ -24,7 +24,9 @@ Test whether a request is attempting to initiate Websocket connection. Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: int TSHttpTxnIsWebsocket(TSHttpTxn txnp) diff --git a/doc/developer-guide/api/functions/TSHttpTxnMilestoneGet.en.rst b/doc/developer-guide/api/functions/TSHttpTxnMilestoneGet.en.rst index 1c0ee733053..2cebf100115 100644 --- a/doc/developer-guide/api/functions/TSHttpTxnMilestoneGet.en.rst +++ b/doc/developer-guide/api/functions/TSHttpTxnMilestoneGet.en.rst @@ -27,7 +27,9 @@ Get a specified :arg:`milestone` timer value for the current transaction. Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: TSReturnCode TSHttpTxnMilestoneGet(TSHttpTxn txnp, TSMilestonesType milestone, TSHRTime * time) @@ -82,7 +84,7 @@ is successful. .. macro:: TS_MILESTONE_SERVER_FIRST_CONNECT - First time origin server connect attempted or shared shared session attached. + First time origin server connect attempted or shared session attached. .. macro:: TS_MILESTONE_SERVER_CONNECT diff --git a/doc/developer-guide/api/functions/TSHttpTxnNextHopAddrGet.en.rst b/doc/developer-guide/api/functions/TSHttpTxnNextHopAddrGet.en.rst index 366acdb7379..34440ccc5bf 100644 --- a/doc/developer-guide/api/functions/TSHttpTxnNextHopAddrGet.en.rst +++ b/doc/developer-guide/api/functions/TSHttpTxnNextHopAddrGet.en.rst @@ -24,7 +24,9 @@ TSHttpTxnNextHopAddrGet Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: sockaddr const * TSHttpTxnNextHopAddrGet(TSHttpTxn txnp) diff --git a/doc/developer-guide/api/functions/TSHttpTxnNextHopNameGet.en.rst b/doc/developer-guide/api/functions/TSHttpTxnNextHopNameGet.en.rst index b1feeb7ebc0..a918da067d6 100644 --- a/doc/developer-guide/api/functions/TSHttpTxnNextHopNameGet.en.rst +++ b/doc/developer-guide/api/functions/TSHttpTxnNextHopNameGet.en.rst @@ -24,7 +24,9 @@ TSHttpTxnNextHopNameGet Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: const char *TSHttpTxnNextHopNameGet(TSHttpTxn txnp) diff --git a/doc/developer-guide/api/functions/TSHttpTxnOutgoingAddrGet.en.rst b/doc/developer-guide/api/functions/TSHttpTxnOutgoingAddrGet.en.rst index c20ed298f82..4f1b99ad336 100644 --- a/doc/developer-guide/api/functions/TSHttpTxnOutgoingAddrGet.en.rst +++ b/doc/developer-guide/api/functions/TSHttpTxnOutgoingAddrGet.en.rst @@ -26,7 +26,9 @@ Get or set the local IP address for outbound connections. Synopsis -------- -`#include ` +.. code-block:: cpp + + #include .. c:function:: sockaddr const* TSHttpTxnOutgoingAddrGet(TSHttpTxn txnp) .. c:function:: TSReturnCode TSHttpTxnOutgoingAddrSet(TSHttpTxn txnp, sockaddr const* addr) diff --git a/doc/developer-guide/api/functions/TSHttpTxnParentProxySet.en.rst b/doc/developer-guide/api/functions/TSHttpTxnParentProxySet.en.rst index 8da767d5a41..33e976999ff 100644 --- a/doc/developer-guide/api/functions/TSHttpTxnParentProxySet.en.rst +++ b/doc/developer-guide/api/functions/TSHttpTxnParentProxySet.en.rst @@ -26,7 +26,9 @@ Sets the parent proxy :arg:`hostname` and :arg:`port`. Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: void TSHttpTxnParentProxySet(TSHttpTxn txnp, const char * hostname, int port) diff --git a/doc/developer-guide/api/functions/TSHttpTxnParentSelectionUrlGet.en.rst b/doc/developer-guide/api/functions/TSHttpTxnParentSelectionUrlGet.en.rst index 405423407b2..5c48e2b2973 100644 --- a/doc/developer-guide/api/functions/TSHttpTxnParentSelectionUrlGet.en.rst +++ b/doc/developer-guide/api/functions/TSHttpTxnParentSelectionUrlGet.en.rst @@ -27,7 +27,9 @@ Traffic Server Parent Selection consistent hash URL manipulation API. Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: TSReturnCode TSHttpTxnParentSelectionUrlSet(TSHttpTxn txnp, TSMBuffer bufp, TSMLoc offset) .. function:: TSReturnCode TSHttpTxnParentSelectionUrlGet(TSHttpTxn txnp, TSMBuffer bufp, TSMLoc offset) diff --git a/doc/developer-guide/api/functions/TSHttpTxnPluginTagGet.en.rst b/doc/developer-guide/api/functions/TSHttpTxnPluginTagGet.en.rst index 4bf895b51f4..5286c277879 100644 --- a/doc/developer-guide/api/functions/TSHttpTxnPluginTagGet.en.rst +++ b/doc/developer-guide/api/functions/TSHttpTxnPluginTagGet.en.rst @@ -24,7 +24,9 @@ Fetch the tag of the plugin that created this transaction. Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: const char * TSHttpTxnPluginTagGet(TSHttpTxn txnp) diff --git a/doc/developer-guide/api/types/TSServerSessionSharingMatchType.en.rst b/doc/developer-guide/api/functions/TSHttpTxnRedoCacheLookup.en.rst similarity index 58% rename from doc/developer-guide/api/types/TSServerSessionSharingMatchType.en.rst rename to doc/developer-guide/api/functions/TSHttpTxnRedoCacheLookup.en.rst index 6b2c36941d9..ee3da4210c8 100644 --- a/doc/developer-guide/api/types/TSServerSessionSharingMatchType.en.rst +++ b/doc/developer-guide/api/functions/TSHttpTxnRedoCacheLookup.en.rst @@ -16,29 +16,35 @@ .. include:: ../../../common.defs -TSServerSessionSharingMatchType -******************************* +.. default-domain:: c + +TSHttpTxnRedoCacheLookup +******************* Synopsis ======== -`#include ` +.. code-block:: cpp -.. c:type:: TSServerSessionSharingMatchType + #include -Enum typedef. +.. function:: TSReturnCode TSHttpTxnRedoCacheLookup(TSHttpTxn txnp, const char *url, int length) -Enumeration Members -=================== +Description +=========== -.. c:member:: TSServerSessionSharingMatchType TS_SERVER_SESSION_SHARING_MATCH_NONE +Perform a cache lookup with a different url. +This function rewinds the state machine to perform the new cache lookup. The cache_info for the new +url must have been initialized before calling this function. -.. c:member:: TSServerSessionSharingMatchType TS_SERVER_SESSION_SHARING_MATCH_BOTH +If the length argument is -1, this function will take the length from the url argument. -.. c:member:: TSServerSessionSharingMatchType TS_SERVER_SESSION_SHARING_MATCH_IP +Notes +===== -.. c:member:: TSServerSessionSharingMatchType TS_SERVER_SESSION_SHARING_MATCH_HOST +This API may be changed in the future version since it is experimental. -Description -=========== +See Also +======== +:manpage:`TSAPI(3ts)`, diff --git a/doc/developer-guide/api/functions/TSHttpTxnReenable.en.rst b/doc/developer-guide/api/functions/TSHttpTxnReenable.en.rst index 4fc364aae10..808226f6983 100644 --- a/doc/developer-guide/api/functions/TSHttpTxnReenable.en.rst +++ b/doc/developer-guide/api/functions/TSHttpTxnReenable.en.rst @@ -27,7 +27,9 @@ processing the current hook. Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: void TSHttpTxnReenable(TSHttpTxn txnp, TSEvent event) diff --git a/doc/developer-guide/api/functions/TSHttpTxnServerAddrGet.en.rst b/doc/developer-guide/api/functions/TSHttpTxnServerAddrGet.en.rst index 800634e8e58..e5f327a39de 100644 --- a/doc/developer-guide/api/functions/TSHttpTxnServerAddrGet.en.rst +++ b/doc/developer-guide/api/functions/TSHttpTxnServerAddrGet.en.rst @@ -24,7 +24,9 @@ TSHttpTxnServerAddrGet Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: sockaddr const* TSHttpTxnServerAddrGet(TSHttpTxn txnp) diff --git a/doc/developer-guide/api/functions/TSHttpTxnServerAddrSet.en.rst b/doc/developer-guide/api/functions/TSHttpTxnServerAddrSet.en.rst index 5100513e1a2..1633f3f0b21 100644 --- a/doc/developer-guide/api/functions/TSHttpTxnServerAddrSet.en.rst +++ b/doc/developer-guide/api/functions/TSHttpTxnServerAddrSet.en.rst @@ -24,7 +24,9 @@ TSHttpTxnServerAddrSet Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: TSReturnCode TSHttpTxnServerAddrSet(TSHttpTxn txnp, struct sockaddr const* addr) diff --git a/doc/developer-guide/api/functions/TSHttpTxnServerFdGet.en.rst b/doc/developer-guide/api/functions/TSHttpTxnServerFdGet.en.rst index 1b0f6e0baa6..10254dcbed5 100644 --- a/doc/developer-guide/api/functions/TSHttpTxnServerFdGet.en.rst +++ b/doc/developer-guide/api/functions/TSHttpTxnServerFdGet.en.rst @@ -21,7 +21,9 @@ TSHttpTxnServerFdGet Synopsis -------- -`#include ` +.. code-block:: cpp + + #include .. c:function:: TSReturnCode TSHttpTxnServerFdGet(TSHttpTxn txnp, int *fdp) diff --git a/doc/developer-guide/api/functions/TSHttpTxnServerIntercept.en.rst b/doc/developer-guide/api/functions/TSHttpTxnServerIntercept.en.rst index bc70ecc5883..50c7d8839ab 100644 --- a/doc/developer-guide/api/functions/TSHttpTxnServerIntercept.en.rst +++ b/doc/developer-guide/api/functions/TSHttpTxnServerIntercept.en.rst @@ -27,7 +27,9 @@ Intercept origin server requests. Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: void TSHttpTxnServerIntercept(TSCont contp, TSHttpTxn txnp) diff --git a/doc/developer-guide/api/functions/TSHttpTxnServerPacketDscpSet.en.rst b/doc/developer-guide/api/functions/TSHttpTxnServerPacketDscpSet.en.rst index 1fc10085f9f..0ff43285b53 100644 --- a/doc/developer-guide/api/functions/TSHttpTxnServerPacketDscpSet.en.rst +++ b/doc/developer-guide/api/functions/TSHttpTxnServerPacketDscpSet.en.rst @@ -25,7 +25,9 @@ Change packet DSCP for the server side connection. Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: TSReturnCode TSHttpTxnServerPacketDscpSet(TSHttpTxn txnp, int dscp) diff --git a/doc/developer-guide/api/functions/TSHttpTxnServerPacketMarkSet.en.rst b/doc/developer-guide/api/functions/TSHttpTxnServerPacketMarkSet.en.rst index af31dd84ed3..0fb6c1d5c28 100644 --- a/doc/developer-guide/api/functions/TSHttpTxnServerPacketMarkSet.en.rst +++ b/doc/developer-guide/api/functions/TSHttpTxnServerPacketMarkSet.en.rst @@ -25,7 +25,9 @@ Change packet firewall mark for the server side connection. Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: TSReturnCode TSHttpTxnServerPacketMarkSet(TSHttpTxn txnp, int mark) diff --git a/doc/developer-guide/api/functions/TSHttpTxnServerPacketTosSet.en.rst b/doc/developer-guide/api/functions/TSHttpTxnServerPacketTosSet.en.rst index c412be1d08f..04ef74c3c5e 100644 --- a/doc/developer-guide/api/functions/TSHttpTxnServerPacketTosSet.en.rst +++ b/doc/developer-guide/api/functions/TSHttpTxnServerPacketTosSet.en.rst @@ -25,7 +25,9 @@ Change packet TOS for the server side connection. Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: TSReturnCode TSHttpTxnServerPacketTosSet(TSHttpTxn txnp, int tos) diff --git a/doc/developer-guide/api/functions/TSHttpTxnServerPush.en.rst b/doc/developer-guide/api/functions/TSHttpTxnServerPush.en.rst index ca37c3122bc..0710a07faa9 100644 --- a/doc/developer-guide/api/functions/TSHttpTxnServerPush.en.rst +++ b/doc/developer-guide/api/functions/TSHttpTxnServerPush.en.rst @@ -24,9 +24,11 @@ TSHttpTxnServerPush Synopsis ======== -`#include ` +.. code-block:: cpp -.. function:: void TSHttpTxnServerPush(TSHttpTxn txnp, const char *url, int url_len) + #include + +.. function:: TSReturnCode TSHttpTxnServerPush(TSHttpTxn txnp, const char *url, int url_len) Description =========== @@ -38,11 +40,8 @@ is not disabled by the client. You can call this API without checking whether Server Push is available on the transaction and it does nothing if Server Push is not available. - -Notes -===== - -This API may be changed in the future version since it is experimental. +This API returns an error if the URL to push is not valid, the client has Server Push disabled, +or there is an error creating the H/2 PUSH_PROMISE frame. See Also ======== diff --git a/doc/developer-guide/api/functions/TSHttpTxnServerReqGet.en.rst b/doc/developer-guide/api/functions/TSHttpTxnServerReqGet.en.rst index f9cb31e6a85..bed19dc5b6c 100644 --- a/doc/developer-guide/api/functions/TSHttpTxnServerReqGet.en.rst +++ b/doc/developer-guide/api/functions/TSHttpTxnServerReqGet.en.rst @@ -24,7 +24,9 @@ TSHttpTxnServerReqGet Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: TSReturnCode TSHttpTxnServerReqGet(TSHttpTxn txnp, TSMBuffer * bufp, TSMLoc * obj) diff --git a/doc/developer-guide/api/functions/TSHttpTxnServerRespGet.en.rst b/doc/developer-guide/api/functions/TSHttpTxnServerRespGet.en.rst index 3113297b871..0665ba868b4 100644 --- a/doc/developer-guide/api/functions/TSHttpTxnServerRespGet.en.rst +++ b/doc/developer-guide/api/functions/TSHttpTxnServerRespGet.en.rst @@ -24,7 +24,9 @@ TSHttpTxnServerRespGet Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: TSReturnCode TSHttpTxnServerRespGet(TSHttpTxn txnp, TSMBuffer * bufp, TSMLoc * offset) diff --git a/doc/developer-guide/api/functions/TSHttpTxnSsnGet.en.rst b/doc/developer-guide/api/functions/TSHttpTxnSsnGet.en.rst index 9c0b8ea1dce..f963f1ef453 100644 --- a/doc/developer-guide/api/functions/TSHttpTxnSsnGet.en.rst +++ b/doc/developer-guide/api/functions/TSHttpTxnSsnGet.en.rst @@ -24,7 +24,9 @@ TSHttpTxnSsnGet Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: TSHttpSsn TSHttpTxnSsnGet(TSHttpTxn txnp) diff --git a/doc/developer-guide/api/functions/TSHttpTxnTransformRespGet.en.rst b/doc/developer-guide/api/functions/TSHttpTxnTransformRespGet.en.rst index f27fc1eba34..bd3922d2db1 100644 --- a/doc/developer-guide/api/functions/TSHttpTxnTransformRespGet.en.rst +++ b/doc/developer-guide/api/functions/TSHttpTxnTransformRespGet.en.rst @@ -24,7 +24,9 @@ TSHttpTxnTransformRespGet Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: TSReturnCode TSHttpTxnTransformRespGet(TSHttpTxn txnp, TSMBuffer * bufp, TSMLoc * offset) diff --git a/doc/developer-guide/api/functions/TSHttpTxnTransformedRespCache.en.rst b/doc/developer-guide/api/functions/TSHttpTxnTransformedRespCache.en.rst index a84077f75bf..3864b67af9f 100644 --- a/doc/developer-guide/api/functions/TSHttpTxnTransformedRespCache.en.rst +++ b/doc/developer-guide/api/functions/TSHttpTxnTransformedRespCache.en.rst @@ -24,7 +24,9 @@ TSHttpTxnTransformedRespCache Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: void TSHttpTxnTransformedRespCache(TSHttpTxn txnp, int on) diff --git a/doc/developer-guide/api/functions/TSHttpTxnUntransformedRespCache.en.rst b/doc/developer-guide/api/functions/TSHttpTxnUntransformedRespCache.en.rst index 33e77db4028..5a1047d33dc 100644 --- a/doc/developer-guide/api/functions/TSHttpTxnUntransformedRespCache.en.rst +++ b/doc/developer-guide/api/functions/TSHttpTxnUntransformedRespCache.en.rst @@ -24,7 +24,9 @@ TSHttpTxnUntransformedRespCache Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: void TSHttpTxnUntransformedRespCache(TSHttpTxn txnp, int on) diff --git a/doc/developer-guide/api/functions/TSIOBufferBlockReadStart.en.rst b/doc/developer-guide/api/functions/TSIOBufferBlockReadStart.en.rst index 1208e4eac11..bf0122f34fb 100644 --- a/doc/developer-guide/api/functions/TSIOBufferBlockReadStart.en.rst +++ b/doc/developer-guide/api/functions/TSIOBufferBlockReadStart.en.rst @@ -24,7 +24,9 @@ TSIOBufferBlockReadStart Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: const char * TSIOBufferBlockReadStart(TSIOBufferBlock blockp, TSIOBufferReader readerp, int64_t * avail) diff --git a/doc/developer-guide/api/functions/TSIOBufferCopy.en.rst b/doc/developer-guide/api/functions/TSIOBufferCopy.en.rst index 81f70a00df5..a46d8a9124a 100644 --- a/doc/developer-guide/api/functions/TSIOBufferCopy.en.rst +++ b/doc/developer-guide/api/functions/TSIOBufferCopy.en.rst @@ -24,7 +24,9 @@ TSIOBufferCopy Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: int64_t TSIOBufferCopy(TSIOBuffer bufp, TSIOBufferReader readerp, int64_t length, int64_t offset) diff --git a/doc/developer-guide/api/functions/TSIOBufferCreate.en.rst b/doc/developer-guide/api/functions/TSIOBufferCreate.en.rst index ee7086316df..55fe62cd361 100644 --- a/doc/developer-guide/api/functions/TSIOBufferCreate.en.rst +++ b/doc/developer-guide/api/functions/TSIOBufferCreate.en.rst @@ -27,7 +27,9 @@ Traffic Server IO buffer API. Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: TSIOBuffer TSIOBufferCreate(void) .. function:: TSIOBuffer TSIOBufferSizedCreate(TSIOBufferSizeIndex index) diff --git a/doc/developer-guide/api/functions/TSIOBufferReader.en.rst b/doc/developer-guide/api/functions/TSIOBufferReader.en.rst index 031eb9591b4..d06e8fbc35a 100644 --- a/doc/developer-guide/api/functions/TSIOBufferReader.en.rst +++ b/doc/developer-guide/api/functions/TSIOBufferReader.en.rst @@ -25,7 +25,10 @@ Traffic Server IO buffer reader API. Synopsis ======== -`#include ` + +.. code-block:: cpp + + #include .. function:: TSIOBufferReader TSIOBufferReaderAlloc(TSIOBuffer bufp) .. function:: TSIOBufferReader TSIOBufferReaderClone(TSIOBufferReader readerp) @@ -33,17 +36,7 @@ Synopsis .. function:: void TSIOBufferReaderConsume(TSIOBufferReader readerp, int64_t nbytes) .. function:: TSIOBufferBlock TSIOBufferReaderStart(TSIOBufferReader readerp) .. function:: int64_t TSIOBufferReaderAvail(TSIOBufferReader readerp) -.. function:: bool TSIOBufferReaderIsAvailAtLeast(TSIOBufferReader, int64_t nbytes) .. function:: int64_t TSIOBufferReaderCopy(TSIOBufferReader reader, void * buf, int64_t length) -.. function:: bool TSIOBufferReaderIterate(TSIOBufferReader reader, TSIOBufferBlockFunc* func, void* context) - -.. type:: TSIOBufferBlockFunc - - ``bool (*TSIOBufferBlockFunc)(void const* data, int64_t nbytes, void* context)`` - - :arg:`data` is the data in the :type:`TSIOBufferBlock` and is :arg:`nbytes` long. :arg:`context` is - opaque data provided to the API call along with this function and passed on to the function. This - function should return ``true`` to continue iteration or ``false`` to terminate iteration. Description =========== @@ -76,7 +69,7 @@ time. Reader allocation is fast and cheap until this maximum is reached at which This also effectively consumes (see :func:`TSIOBufferReaderConsume`) all data for :arg:`reader`. :func:`TSIOBufferReaderConsume` consume data from :arg:`reader`. - This advances the position of :arg:`reader` in its IO buffer by the the smaller of :arg:`nbytes` + This advances the position of :arg:`reader` in its IO buffer by the smaller of :arg:`nbytes` and the maximum available in the buffer. This is required to release the buffer memory - when data has been consumed by all readers, it is discarded. @@ -88,34 +81,12 @@ time. Reader allocation is fast and cheap until this maximum is reached at which :func:`TSIOBufferReaderAvail` returns the number of bytes available. The bytes available is the amount of data that could be read from :arg:`reader`. -:func:`TSIOBufferReaderIsAvailAtLeast` - check amount of data available. - This function returns :code:`true` if the available number of bytes for :arg:`reader` is at least - :arg:`nbytes`, :code:`false` if not. This can be much more efficient than - :func:`TSIOBufferReaderAvail` because the latter must walk all the IO buffer blocks in the IO - buffer. This function returns as soon as the return value can be determined. In particular a - value of ``1`` for :arg:`nbytes` means only the first buffer block will be checked making the - call very fast. - :func:`TSIOBufferReaderCopy` copies data from :arg:`reader` into :arg:`buff`. This copies data from the IO buffer for :arg:`reader` to the target buffer :arg:`bufp`. The amount of data read in this fashion is the smaller of the amount of data available in the IO buffer for :arg:`reader` and the size of the target buffer (:arg:`length`). The number of bytes copied is returned. -:func:`TSIOBufferReaderIterate` iterate over the blocks for :arg:`reader`. - For each block :arg:`func` is called with with the data for the block and :arg:`context`. The - :arg:`context` is an opaque type to this function and is passed unchanged to :arg:`func`. It is - intended to be used as context for :arg:`func`. If :arg:`func` returns ``false`` the iteration - terminates. If :arg:`func` returns true the block is consumed. The return value for - :func:`TSIOBufferReaderIterate` is the return value from the last call to :arg:`func`. - -.. note:: - - If it would be a problem for the iteration to consume the data (especially in cases where - :code:`false` might be returned) the reader can be cloned via :func:`TSIOBufferReaderClone` to - keep the data in the IO buffer and available. If not needed the reader can be destroyed or - if needed the original reader can be destroyed and replaced by the clone. - .. note:: Destroying a :type:`TSIOBuffer` will de-allocate and destroy all readers for that buffer. diff --git a/doc/developer-guide/api/functions/TSInstallDirGet.en.rst b/doc/developer-guide/api/functions/TSInstallDirGet.en.rst index e9097a3ca02..023bbc5e7ea 100644 --- a/doc/developer-guide/api/functions/TSInstallDirGet.en.rst +++ b/doc/developer-guide/api/functions/TSInstallDirGet.en.rst @@ -27,7 +27,9 @@ Return Traffic Server installation directories. Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: const char * TSInstallDirGet(void) .. function:: const char * TSConfigDirGet(void) diff --git a/doc/developer-guide/api/functions/TSIpStringToAddr.en.rst b/doc/developer-guide/api/functions/TSIpStringToAddr.en.rst index 298705a976d..2b74ed1d16e 100644 --- a/doc/developer-guide/api/functions/TSIpStringToAddr.en.rst +++ b/doc/developer-guide/api/functions/TSIpStringToAddr.en.rst @@ -24,7 +24,9 @@ TSIpStringToAddr Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: TSReturnCode TSIpStringToAddr(const char * str, int str_len, sockaddr* addr) diff --git a/doc/developer-guide/api/functions/TSLifecycleHookAdd.en.rst b/doc/developer-guide/api/functions/TSLifecycleHookAdd.en.rst index 0d150dc7b66..e3913c1a615 100644 --- a/doc/developer-guide/api/functions/TSLifecycleHookAdd.en.rst +++ b/doc/developer-guide/api/functions/TSLifecycleHookAdd.en.rst @@ -26,7 +26,9 @@ TSLifecycleHookAdd Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: void TSLifecycleHookAdd(TSLifecycleHookID id, TSCont contp) diff --git a/doc/developer-guide/api/functions/TSMBufferCreate.en.rst b/doc/developer-guide/api/functions/TSMBufferCreate.en.rst index 4b7e3c7ed77..cf6c38b6500 100644 --- a/doc/developer-guide/api/functions/TSMBufferCreate.en.rst +++ b/doc/developer-guide/api/functions/TSMBufferCreate.en.rst @@ -25,7 +25,9 @@ Traffic Server marshall buffer API. Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: TSMBuffer TSMBufferCreate(void) .. function:: TSReturnCode TSMBufferDestroy(TSMBuffer bufp) diff --git a/doc/developer-guide/api/functions/TSMgmtCounterGet.en.rst b/doc/developer-guide/api/functions/TSMgmtCounterGet.en.rst index 860a603eb4c..31641dd5783 100644 --- a/doc/developer-guide/api/functions/TSMgmtCounterGet.en.rst +++ b/doc/developer-guide/api/functions/TSMgmtCounterGet.en.rst @@ -24,7 +24,9 @@ TSMgmtCounterGet Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: TSReturnCode TSMgmtCounterGet(const char * var_name, TSMgmtCounter * result) diff --git a/doc/developer-guide/api/functions/TSMgmtFloatGet.en.rst b/doc/developer-guide/api/functions/TSMgmtFloatGet.en.rst index 651f16eca69..77cee428786 100644 --- a/doc/developer-guide/api/functions/TSMgmtFloatGet.en.rst +++ b/doc/developer-guide/api/functions/TSMgmtFloatGet.en.rst @@ -24,7 +24,9 @@ TSMgmtFloatGet Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: TSReturnCode TSMgmtFloatGet(const char * var_name, TSMgmtFloat * result) diff --git a/doc/developer-guide/api/functions/TSMgmtIntGet.en.rst b/doc/developer-guide/api/functions/TSMgmtIntGet.en.rst index e7f6c082d80..3c920ebe069 100644 --- a/doc/developer-guide/api/functions/TSMgmtIntGet.en.rst +++ b/doc/developer-guide/api/functions/TSMgmtIntGet.en.rst @@ -24,7 +24,9 @@ TSMgmtIntGet Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: TSReturnCode TSMgmtIntGet(const char * var_name, TSMgmtInt * result) diff --git a/doc/developer-guide/api/functions/TSMgmtSourceGet.en.rst b/doc/developer-guide/api/functions/TSMgmtSourceGet.en.rst index 593d3dd6dfe..6839b5044c7 100644 --- a/doc/developer-guide/api/functions/TSMgmtSourceGet.en.rst +++ b/doc/developer-guide/api/functions/TSMgmtSourceGet.en.rst @@ -24,7 +24,9 @@ TSMgmtSourceGet Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: TSReturnCode TSMgmtSourceGet(const char * var_name, TSMgmtSource * result) diff --git a/doc/developer-guide/api/functions/TSMgmtStringGet.en.rst b/doc/developer-guide/api/functions/TSMgmtStringGet.en.rst index 0fe3e088738..c507f1f1d14 100644 --- a/doc/developer-guide/api/functions/TSMgmtStringGet.en.rst +++ b/doc/developer-guide/api/functions/TSMgmtStringGet.en.rst @@ -24,7 +24,9 @@ TSMgmtStringGet Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: TSReturnCode TSMgmtStringGet(const char * var_name, TSMgmtString * result) diff --git a/doc/developer-guide/api/functions/TSMgmtUpdateRegister.en.rst b/doc/developer-guide/api/functions/TSMgmtUpdateRegister.en.rst index d9f2f9d6f70..b841466bae1 100644 --- a/doc/developer-guide/api/functions/TSMgmtUpdateRegister.en.rst +++ b/doc/developer-guide/api/functions/TSMgmtUpdateRegister.en.rst @@ -24,7 +24,9 @@ TSMgmtUpdateRegister Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: void TSMgmtUpdateRegister(TSCont contp, const char * plugin_name) diff --git a/doc/developer-guide/api/functions/TSMimeHdrClone.en.rst b/doc/developer-guide/api/functions/TSMimeHdrClone.en.rst index b7cc1ada9fb..ef41671b2e4 100644 --- a/doc/developer-guide/api/functions/TSMimeHdrClone.en.rst +++ b/doc/developer-guide/api/functions/TSMimeHdrClone.en.rst @@ -24,7 +24,9 @@ TSMimeHdrClone Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: TSReturnCode TSMimeHdrClone(TSMBuffer dest_bufp, TSMBuffer src_bufp, TSMLoc src_hdr, TSMLoc * locp) diff --git a/doc/developer-guide/api/functions/TSMimeHdrCopy.en.rst b/doc/developer-guide/api/functions/TSMimeHdrCopy.en.rst index ea43b688efa..21a1c25778f 100644 --- a/doc/developer-guide/api/functions/TSMimeHdrCopy.en.rst +++ b/doc/developer-guide/api/functions/TSMimeHdrCopy.en.rst @@ -24,7 +24,9 @@ TSMimeHdrCopy Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: TSReturnCode TSMimeHdrCopy(TSMBuffer dest_bufp, TSMLoc dest_offset, TSMBuffer src_bufp, TSMLoc src_offset) diff --git a/doc/developer-guide/api/functions/TSMimeHdrCreate.en.rst b/doc/developer-guide/api/functions/TSMimeHdrCreate.en.rst index d8a7c08f75b..52928a3af4d 100644 --- a/doc/developer-guide/api/functions/TSMimeHdrCreate.en.rst +++ b/doc/developer-guide/api/functions/TSMimeHdrCreate.en.rst @@ -24,7 +24,9 @@ TSMimeHdrCreate Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: TSReturnCode TSMimeHdrCreate(TSMBuffer bufp, TSMLoc * locp) diff --git a/doc/developer-guide/api/functions/TSMimeHdrDestroy.en.rst b/doc/developer-guide/api/functions/TSMimeHdrDestroy.en.rst index 532c27fc92b..412cfcfc9fe 100644 --- a/doc/developer-guide/api/functions/TSMimeHdrDestroy.en.rst +++ b/doc/developer-guide/api/functions/TSMimeHdrDestroy.en.rst @@ -24,7 +24,9 @@ TSMimeHdrDestroy Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: TSReturnCode TSMimeHdrDestroy(TSMBuffer bufp, TSMLoc offset) diff --git a/doc/developer-guide/api/functions/TSMimeHdrFieldAppend.en.rst b/doc/developer-guide/api/functions/TSMimeHdrFieldAppend.en.rst index 7690fa8b5c6..0cee4dfd455 100644 --- a/doc/developer-guide/api/functions/TSMimeHdrFieldAppend.en.rst +++ b/doc/developer-guide/api/functions/TSMimeHdrFieldAppend.en.rst @@ -24,7 +24,9 @@ TSMimeHdrFieldAppend Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: TSReturnCode TSMimeHdrFieldAppend(TSMBuffer bufp, TSMLoc hdr, TSMLoc field) diff --git a/doc/developer-guide/api/functions/TSMimeHdrFieldClone.en.rst b/doc/developer-guide/api/functions/TSMimeHdrFieldClone.en.rst index 5f712cd742c..2e9b0de4a80 100644 --- a/doc/developer-guide/api/functions/TSMimeHdrFieldClone.en.rst +++ b/doc/developer-guide/api/functions/TSMimeHdrFieldClone.en.rst @@ -24,7 +24,9 @@ TSMimeHdrFieldClone Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: TSReturnCode TSMimeHdrFieldClone(TSMBuffer dest_bufp, TSMLoc dest_hdr, TSMBuffer src_bufp, TSMLoc src_hdr, TSMLoc src_field, TSMLoc * locp) diff --git a/doc/developer-guide/api/functions/TSMimeHdrFieldCopy.en.rst b/doc/developer-guide/api/functions/TSMimeHdrFieldCopy.en.rst index 8f7390e3c72..f6efc1a9789 100644 --- a/doc/developer-guide/api/functions/TSMimeHdrFieldCopy.en.rst +++ b/doc/developer-guide/api/functions/TSMimeHdrFieldCopy.en.rst @@ -24,7 +24,9 @@ TSMimeHdrFieldCopy Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: TSReturnCode TSMimeHdrFieldCopy(TSMBuffer dest_bufp, TSMLoc dest_hdr, TSMLoc dest_field, TSMBuffer src_bufp, TSMLoc src_hdr, TSMLoc src_field) diff --git a/doc/developer-guide/api/functions/TSMimeHdrFieldCopyValues.en.rst b/doc/developer-guide/api/functions/TSMimeHdrFieldCopyValues.en.rst index a7e6ef6870a..e973cfb4121 100644 --- a/doc/developer-guide/api/functions/TSMimeHdrFieldCopyValues.en.rst +++ b/doc/developer-guide/api/functions/TSMimeHdrFieldCopyValues.en.rst @@ -24,7 +24,9 @@ TSMimeHdrFieldCopyValues Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: TSReturnCode TSMimeHdrFieldCopyValues(TSMBuffer dest_bufp, TSMLoc dest_hdr, TSMLoc dest_field, TSMBuffer src_bufp, TSMLoc src_hdr, TSMLoc src_field) diff --git a/doc/developer-guide/api/functions/TSMimeHdrFieldCreate.en.rst b/doc/developer-guide/api/functions/TSMimeHdrFieldCreate.en.rst index 80cf6351882..76803ee3a7c 100644 --- a/doc/developer-guide/api/functions/TSMimeHdrFieldCreate.en.rst +++ b/doc/developer-guide/api/functions/TSMimeHdrFieldCreate.en.rst @@ -24,7 +24,9 @@ TSMimeHdrFieldCreate Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: TSReturnCode TSMimeHdrFieldCreate(TSMBuffer bufp, TSMLoc hdr, TSMLoc * out) .. function:: TSReturnCode TSMimeHdrFieldCreateNamed(TSMBuffer bufp, TSMLoc hdr, const char * name, int name_len, TSMLoc * out) diff --git a/doc/developer-guide/api/functions/TSMimeHdrFieldDestroy.en.rst b/doc/developer-guide/api/functions/TSMimeHdrFieldDestroy.en.rst index f14d01dad15..9858e14f861 100644 --- a/doc/developer-guide/api/functions/TSMimeHdrFieldDestroy.en.rst +++ b/doc/developer-guide/api/functions/TSMimeHdrFieldDestroy.en.rst @@ -24,7 +24,9 @@ TSMimeHdrFieldDestroy Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: TSReturnCode TSMimeHdrFieldDestroy(TSMBuffer bufp, TSMLoc hdr, TSMLoc field) diff --git a/doc/developer-guide/api/functions/TSMimeHdrFieldFind.en.rst b/doc/developer-guide/api/functions/TSMimeHdrFieldFind.en.rst index 3e26efb5e2f..149e07bb262 100644 --- a/doc/developer-guide/api/functions/TSMimeHdrFieldFind.en.rst +++ b/doc/developer-guide/api/functions/TSMimeHdrFieldFind.en.rst @@ -24,7 +24,9 @@ TSMimeHdrFieldFind Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: TSMLoc TSMimeHdrFieldFind(TSMBuffer bufp, TSMLoc hdr, const char * name, int length) diff --git a/doc/developer-guide/api/functions/TSMimeHdrFieldGet.en.rst b/doc/developer-guide/api/functions/TSMimeHdrFieldGet.en.rst index 27cbdc38c80..f9f03c6c1b6 100644 --- a/doc/developer-guide/api/functions/TSMimeHdrFieldGet.en.rst +++ b/doc/developer-guide/api/functions/TSMimeHdrFieldGet.en.rst @@ -24,7 +24,9 @@ TSMimeHdrFieldGet Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: TSMLoc TSMimeHdrFieldGet(TSMBuffer bufp, TSMLoc hdr, int idx) diff --git a/doc/developer-guide/api/functions/TSMimeHdrFieldLengthGet.en.rst b/doc/developer-guide/api/functions/TSMimeHdrFieldLengthGet.en.rst index 0b066f9a68a..901ce3d43f1 100644 --- a/doc/developer-guide/api/functions/TSMimeHdrFieldLengthGet.en.rst +++ b/doc/developer-guide/api/functions/TSMimeHdrFieldLengthGet.en.rst @@ -24,7 +24,9 @@ TSMimeHdrFieldLengthGet Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: int TSMimeHdrFieldLengthGet(TSMBuffer bufp, TSMLoc hdr, TSMLoc field) diff --git a/doc/developer-guide/api/functions/TSMimeHdrFieldNameGet.en.rst b/doc/developer-guide/api/functions/TSMimeHdrFieldNameGet.en.rst index 20a370b528e..72d4cb92974 100644 --- a/doc/developer-guide/api/functions/TSMimeHdrFieldNameGet.en.rst +++ b/doc/developer-guide/api/functions/TSMimeHdrFieldNameGet.en.rst @@ -24,7 +24,9 @@ TSMimeHdrFieldNameGet Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: const char * TSMimeHdrFieldNameGet(TSMBuffer bufp, TSMLoc hdr, TSMLoc field, int * length) diff --git a/doc/developer-guide/api/functions/TSMimeHdrFieldNameSet.en.rst b/doc/developer-guide/api/functions/TSMimeHdrFieldNameSet.en.rst index d013f3ee796..a0d033c2bb1 100644 --- a/doc/developer-guide/api/functions/TSMimeHdrFieldNameSet.en.rst +++ b/doc/developer-guide/api/functions/TSMimeHdrFieldNameSet.en.rst @@ -24,7 +24,9 @@ TSMimeHdrFieldNameSet Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: TSReturnCode TSMimeHdrFieldNameSet(TSMBuffer bufp, TSMLoc hdr, TSMLoc field, const char * name, int length) diff --git a/doc/developer-guide/api/functions/TSMimeHdrFieldNext.en.rst b/doc/developer-guide/api/functions/TSMimeHdrFieldNext.en.rst index 83e535ff7be..69dd763ea2d 100644 --- a/doc/developer-guide/api/functions/TSMimeHdrFieldNext.en.rst +++ b/doc/developer-guide/api/functions/TSMimeHdrFieldNext.en.rst @@ -24,7 +24,9 @@ TSMimeHdrFieldNext Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: TSMLoc TSMimeHdrFieldNext(TSMBuffer bufp, TSMLoc hdr, TSMLoc field) diff --git a/doc/developer-guide/api/functions/TSMimeHdrFieldNextDup.en.rst b/doc/developer-guide/api/functions/TSMimeHdrFieldNextDup.en.rst index 099bf165400..a4877981cc4 100644 --- a/doc/developer-guide/api/functions/TSMimeHdrFieldNextDup.en.rst +++ b/doc/developer-guide/api/functions/TSMimeHdrFieldNextDup.en.rst @@ -24,7 +24,9 @@ TSMimeHdrFieldNextDup Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: TSMLoc TSMimeHdrFieldNextDup(TSMBuffer bufp, TSMLoc hdr, TSMLoc field) diff --git a/doc/developer-guide/api/functions/TSMimeHdrFieldRemove.en.rst b/doc/developer-guide/api/functions/TSMimeHdrFieldRemove.en.rst index b35741e80bd..07fe987f2bc 100644 --- a/doc/developer-guide/api/functions/TSMimeHdrFieldRemove.en.rst +++ b/doc/developer-guide/api/functions/TSMimeHdrFieldRemove.en.rst @@ -24,7 +24,9 @@ TSMimeHdrFieldRemove Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: TSReturnCode TSMimeHdrFieldRemove(TSMBuffer bufp, TSMLoc hdr, TSMLoc field) diff --git a/doc/developer-guide/api/functions/TSMimeHdrFieldValueAppend.en.rst b/doc/developer-guide/api/functions/TSMimeHdrFieldValueAppend.en.rst index 21b5edf8ac3..0596663ba6a 100644 --- a/doc/developer-guide/api/functions/TSMimeHdrFieldValueAppend.en.rst +++ b/doc/developer-guide/api/functions/TSMimeHdrFieldValueAppend.en.rst @@ -24,7 +24,9 @@ TSMimeHdrFieldValueAppend Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: TSReturnCode TSMimeHdrFieldValueAppend(TSMBuffer bufp, TSMLoc hdr, TSMLoc field, int idx, const char * value, int length) diff --git a/doc/developer-guide/api/functions/TSMimeHdrFieldValueDateInsert.en.rst b/doc/developer-guide/api/functions/TSMimeHdrFieldValueDateInsert.en.rst index f80d6218543..4a9789ef9b3 100644 --- a/doc/developer-guide/api/functions/TSMimeHdrFieldValueDateInsert.en.rst +++ b/doc/developer-guide/api/functions/TSMimeHdrFieldValueDateInsert.en.rst @@ -24,7 +24,9 @@ TSMimeHdrFieldValueDateInsert Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: TSReturnCode TSMimeHdrFieldValueDateInsert(TSMBuffer bufp, TSMLoc hdr, TSMLoc field, time_t value) diff --git a/doc/developer-guide/api/functions/TSMimeHdrFieldValueDateSet.en.rst b/doc/developer-guide/api/functions/TSMimeHdrFieldValueDateSet.en.rst index c8efee44aba..2ed5086c37d 100644 --- a/doc/developer-guide/api/functions/TSMimeHdrFieldValueDateSet.en.rst +++ b/doc/developer-guide/api/functions/TSMimeHdrFieldValueDateSet.en.rst @@ -24,7 +24,9 @@ TSMimeHdrFieldValueDateSet Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: TSReturnCode TSMimeHdrFieldValueDateSet(TSMBuffer bufp, TSMLoc hdr, TSMLoc field, time_t value) diff --git a/doc/developer-guide/api/functions/TSMimeHdrFieldValueIntSet.en.rst b/doc/developer-guide/api/functions/TSMimeHdrFieldValueIntSet.en.rst index 3df3feda17e..034ccc06c8e 100644 --- a/doc/developer-guide/api/functions/TSMimeHdrFieldValueIntSet.en.rst +++ b/doc/developer-guide/api/functions/TSMimeHdrFieldValueIntSet.en.rst @@ -24,7 +24,9 @@ TSMimeHdrFieldValueIntSet Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: TSReturnCode TSMimeHdrFieldValueIntSet(TSMBuffer bufp, TSMLoc hdr, TSMLoc field, int idx, int value) diff --git a/doc/developer-guide/api/functions/TSMimeHdrFieldValueStringGet.en.rst b/doc/developer-guide/api/functions/TSMimeHdrFieldValueStringGet.en.rst index 73237531d75..2a2beb71566 100644 --- a/doc/developer-guide/api/functions/TSMimeHdrFieldValueStringGet.en.rst +++ b/doc/developer-guide/api/functions/TSMimeHdrFieldValueStringGet.en.rst @@ -26,7 +26,9 @@ Get HTTP MIME header values. Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: const char * TSMimeHdrFieldValueStringGet(TSMBuffer bufp, TSMLoc hdr, TSMLoc field, int idx, int * value_len_ptr) .. function:: int TSMimeHdrFieldValueIntGet(TSMBuffer bufp, TSMLoc hdr, TSMLoc field, int idx) diff --git a/doc/developer-guide/api/functions/TSMimeHdrFieldValueStringInsert.en.rst b/doc/developer-guide/api/functions/TSMimeHdrFieldValueStringInsert.en.rst index f679995c780..fcf8e1f9041 100644 --- a/doc/developer-guide/api/functions/TSMimeHdrFieldValueStringInsert.en.rst +++ b/doc/developer-guide/api/functions/TSMimeHdrFieldValueStringInsert.en.rst @@ -24,7 +24,9 @@ TSMimeHdrFieldValueStringInsert Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: TSReturnCode TSMimeHdrFieldValueStringInsert(TSMBuffer bufp, TSMLoc hdr, TSMLoc field, int idx, const char * value, int length) diff --git a/doc/developer-guide/api/functions/TSMimeHdrFieldValueStringSet.en.rst b/doc/developer-guide/api/functions/TSMimeHdrFieldValueStringSet.en.rst index a53a7d503ef..285f3317d66 100644 --- a/doc/developer-guide/api/functions/TSMimeHdrFieldValueStringSet.en.rst +++ b/doc/developer-guide/api/functions/TSMimeHdrFieldValueStringSet.en.rst @@ -24,7 +24,9 @@ TSMimeHdrFieldValueStringSet Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: TSReturnCode TSMimeHdrFieldValueStringSet(TSMBuffer bufp, TSMLoc hdr, TSMLoc field, int idx, const char * value, int length) diff --git a/doc/developer-guide/api/functions/TSMimeHdrFieldValueUintInsert.en.rst b/doc/developer-guide/api/functions/TSMimeHdrFieldValueUintInsert.en.rst index d7eb5589cfe..5a7ef0852b0 100644 --- a/doc/developer-guide/api/functions/TSMimeHdrFieldValueUintInsert.en.rst +++ b/doc/developer-guide/api/functions/TSMimeHdrFieldValueUintInsert.en.rst @@ -24,7 +24,9 @@ TSMimeHdrFieldValueUintInsert Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: TSReturnCode TSMimeHdrFieldValueUintInsert(TSMBuffer bufp, TSMLoc hdr, TSMLoc field, int idx, unsigned int value) diff --git a/doc/developer-guide/api/functions/TSMimeHdrFieldValueUintSet.en.rst b/doc/developer-guide/api/functions/TSMimeHdrFieldValueUintSet.en.rst index bc8d667a80e..484b17b6c9f 100644 --- a/doc/developer-guide/api/functions/TSMimeHdrFieldValueUintSet.en.rst +++ b/doc/developer-guide/api/functions/TSMimeHdrFieldValueUintSet.en.rst @@ -24,7 +24,9 @@ TSMimeHdrFieldValueUintSet Synopsis ========= -`#include ` +.. code-block:: cpp + + #include .. function:: TSReturnCode TSMimeHdrFieldValueUintSet(TSMBuffer bufp, TSMLoc hdr, TSMLoc field, int idx, unsigned int value) diff --git a/doc/developer-guide/api/functions/TSMimeHdrFieldValuesClear.en.rst b/doc/developer-guide/api/functions/TSMimeHdrFieldValuesClear.en.rst index 0e8eefcb146..7285599f196 100644 --- a/doc/developer-guide/api/functions/TSMimeHdrFieldValuesClear.en.rst +++ b/doc/developer-guide/api/functions/TSMimeHdrFieldValuesClear.en.rst @@ -24,7 +24,9 @@ TSMimeHdrFieldValuesClear Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: TSReturnCode TSMimeHdrFieldValuesClear(TSMBuffer bufp, TSMLoc hdr, TSMLoc field) diff --git a/doc/developer-guide/api/functions/TSMimeHdrFieldValuesCount.en.rst b/doc/developer-guide/api/functions/TSMimeHdrFieldValuesCount.en.rst index ccadb1fb8ba..d0701f3cccb 100644 --- a/doc/developer-guide/api/functions/TSMimeHdrFieldValuesCount.en.rst +++ b/doc/developer-guide/api/functions/TSMimeHdrFieldValuesCount.en.rst @@ -24,7 +24,9 @@ TSMimeHdrFieldValuesCount Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: int TSMimeHdrFieldValuesCount(TSMBuffer bufp, TSMLoc hdr, TSMLoc field) diff --git a/doc/developer-guide/api/functions/TSMimeHdrFieldsClear.en.rst b/doc/developer-guide/api/functions/TSMimeHdrFieldsClear.en.rst index 2b43cf7fdb6..1ff98449fe3 100644 --- a/doc/developer-guide/api/functions/TSMimeHdrFieldsClear.en.rst +++ b/doc/developer-guide/api/functions/TSMimeHdrFieldsClear.en.rst @@ -24,7 +24,9 @@ TSMimeHdrFieldsClear Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: TSReturnCode TSMimeHdrFieldsClear(TSMBuffer bufp, TSMLoc offset) diff --git a/doc/developer-guide/api/functions/TSMimeHdrFieldsCount.en.rst b/doc/developer-guide/api/functions/TSMimeHdrFieldsCount.en.rst index d841e73e663..cc293a39c6c 100644 --- a/doc/developer-guide/api/functions/TSMimeHdrFieldsCount.en.rst +++ b/doc/developer-guide/api/functions/TSMimeHdrFieldsCount.en.rst @@ -24,7 +24,9 @@ TSMimeHdrFieldsCount Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: int TSMimeHdrFieldsCount(TSMBuffer bufp, TSMLoc offset) diff --git a/doc/developer-guide/api/functions/TSMimeHdrLengthGet.en.rst b/doc/developer-guide/api/functions/TSMimeHdrLengthGet.en.rst index 5f499004817..b8bca00e844 100644 --- a/doc/developer-guide/api/functions/TSMimeHdrLengthGet.en.rst +++ b/doc/developer-guide/api/functions/TSMimeHdrLengthGet.en.rst @@ -24,7 +24,9 @@ TSMimeHdrLengthGet Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: int TSMimeHdrLengthGet(TSMBuffer bufp, TSMLoc offset) diff --git a/doc/developer-guide/api/functions/TSMimeHdrParse.en.rst b/doc/developer-guide/api/functions/TSMimeHdrParse.en.rst index 49e4bd083fa..2bcbcbd38e5 100644 --- a/doc/developer-guide/api/functions/TSMimeHdrParse.en.rst +++ b/doc/developer-guide/api/functions/TSMimeHdrParse.en.rst @@ -24,7 +24,9 @@ TSMimeHdrParse Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: TSParseResult TSMimeHdrParse(TSMimeParser parser, TSMBuffer bufp, TSMLoc offset, const char ** start, const char * end) diff --git a/doc/developer-guide/api/functions/TSMimeHdrPrint.en.rst b/doc/developer-guide/api/functions/TSMimeHdrPrint.en.rst index 38625e2619b..4cf5af99977 100644 --- a/doc/developer-guide/api/functions/TSMimeHdrPrint.en.rst +++ b/doc/developer-guide/api/functions/TSMimeHdrPrint.en.rst @@ -24,7 +24,9 @@ TSMimeHdrPrint Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: void TSMimeHdrPrint(TSMBuffer bufp, TSMLoc offset, TSIOBuffer iobufp) diff --git a/doc/developer-guide/api/functions/TSMimeParserClear.en.rst b/doc/developer-guide/api/functions/TSMimeParserClear.en.rst index 957f1706842..2d8383f0526 100644 --- a/doc/developer-guide/api/functions/TSMimeParserClear.en.rst +++ b/doc/developer-guide/api/functions/TSMimeParserClear.en.rst @@ -24,7 +24,9 @@ TSMimeParserClear Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: void TSMimeParserClear(TSMimeParser parser) diff --git a/doc/developer-guide/api/functions/TSMimeParserCreate.en.rst b/doc/developer-guide/api/functions/TSMimeParserCreate.en.rst index 8382efdc564..af50929e131 100644 --- a/doc/developer-guide/api/functions/TSMimeParserCreate.en.rst +++ b/doc/developer-guide/api/functions/TSMimeParserCreate.en.rst @@ -24,7 +24,9 @@ TSMimeParserCreate Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: TSMimeParser TSMimeParserCreate(void) diff --git a/doc/developer-guide/api/functions/TSMimeParserDestroy.en.rst b/doc/developer-guide/api/functions/TSMimeParserDestroy.en.rst index e0367be2929..8bcb98df6c0 100644 --- a/doc/developer-guide/api/functions/TSMimeParserDestroy.en.rst +++ b/doc/developer-guide/api/functions/TSMimeParserDestroy.en.rst @@ -24,7 +24,9 @@ TSMimeParserDestroy Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: void TSMimeParserDestroy(TSMimeParser parser) diff --git a/doc/developer-guide/api/functions/TSMutexCreate.en.rst b/doc/developer-guide/api/functions/TSMutexCreate.en.rst index 0bf24ebc0b7..5a6a25e9a2a 100644 --- a/doc/developer-guide/api/functions/TSMutexCreate.en.rst +++ b/doc/developer-guide/api/functions/TSMutexCreate.en.rst @@ -24,7 +24,9 @@ TSMutexCreate Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: TSMutex TSMutexCreate(void) diff --git a/doc/developer-guide/api/functions/TSMutexDestroy.en.rst b/doc/developer-guide/api/functions/TSMutexDestroy.en.rst index ab978dba2c0..ca841085046 100644 --- a/doc/developer-guide/api/functions/TSMutexDestroy.en.rst +++ b/doc/developer-guide/api/functions/TSMutexDestroy.en.rst @@ -24,7 +24,9 @@ TSMutexDestroy Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: void TSMutexDestroy(TSMutex mutexp) diff --git a/doc/developer-guide/api/functions/TSMutexLock.en.rst b/doc/developer-guide/api/functions/TSMutexLock.en.rst index 6c9539092e3..11558f2dbae 100644 --- a/doc/developer-guide/api/functions/TSMutexLock.en.rst +++ b/doc/developer-guide/api/functions/TSMutexLock.en.rst @@ -24,7 +24,9 @@ TSMutexLock Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: void TSMutexLock(TSMutex mutexp) diff --git a/doc/developer-guide/api/functions/TSMutexLockTry.en.rst b/doc/developer-guide/api/functions/TSMutexLockTry.en.rst index ed5b56cdf43..8f810b7d870 100644 --- a/doc/developer-guide/api/functions/TSMutexLockTry.en.rst +++ b/doc/developer-guide/api/functions/TSMutexLockTry.en.rst @@ -24,7 +24,9 @@ TSMutexLockTry Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: TSReturnCode TSMutexLockTry(TSMutex mutexp) diff --git a/doc/developer-guide/api/functions/TSMutexUnlock.en.rst b/doc/developer-guide/api/functions/TSMutexUnlock.en.rst index 887a12c76a0..37a96a99f23 100644 --- a/doc/developer-guide/api/functions/TSMutexUnlock.en.rst +++ b/doc/developer-guide/api/functions/TSMutexUnlock.en.rst @@ -24,7 +24,9 @@ TSMutexUnlock Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: void TSMutexUnlock(TSMutex mutexp) diff --git a/doc/developer-guide/api/functions/TSNetAccept.en.rst b/doc/developer-guide/api/functions/TSNetAccept.en.rst index bc3646daafd..e7bf427befe 100644 --- a/doc/developer-guide/api/functions/TSNetAccept.en.rst +++ b/doc/developer-guide/api/functions/TSNetAccept.en.rst @@ -24,7 +24,9 @@ TSNetAccept Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: TSAction TSNetAccept(TSCont contp, int port, int domain, int accept_threads) diff --git a/doc/developer-guide/api/functions/TSNetAcceptNamedProtocol.en.rst b/doc/developer-guide/api/functions/TSNetAcceptNamedProtocol.en.rst index 1edaed44d6d..4d7eefbb9a2 100644 --- a/doc/developer-guide/api/functions/TSNetAcceptNamedProtocol.en.rst +++ b/doc/developer-guide/api/functions/TSNetAcceptNamedProtocol.en.rst @@ -26,7 +26,9 @@ Listen on all SSL ports for connections for the specified protocol name. Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: TSReturnCode TSNetAcceptNamedProtocol(TSCont contp, const char * protocol) diff --git a/doc/developer-guide/api/functions/TSNetConnect.en.rst b/doc/developer-guide/api/functions/TSNetConnect.en.rst index 0dea9482757..f661e0533b3 100644 --- a/doc/developer-guide/api/functions/TSNetConnect.en.rst +++ b/doc/developer-guide/api/functions/TSNetConnect.en.rst @@ -24,7 +24,9 @@ TSNetConnect Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: TSAction TSNetConnect(TSCont contp, sockaddr const * addr) diff --git a/doc/developer-guide/api/functions/TSNetInvokingGet.en.rst b/doc/developer-guide/api/functions/TSNetInvokingGet.en.rst index 876196e4099..0941369e80b 100644 --- a/doc/developer-guide/api/functions/TSNetInvokingGet.en.rst +++ b/doc/developer-guide/api/functions/TSNetInvokingGet.en.rst @@ -24,7 +24,9 @@ TSNetInvokingContGet Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: TSCont TSNetInvokingContGet(TSVConn conn) diff --git a/doc/developer-guide/api/functions/TSPluginDSOReloadEnable.en.rst b/doc/developer-guide/api/functions/TSPluginDSOReloadEnable.en.rst new file mode 100644 index 00000000000..c0877a534ae --- /dev/null +++ b/doc/developer-guide/api/functions/TSPluginDSOReloadEnable.en.rst @@ -0,0 +1,51 @@ +.. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +.. include:: ../../../common.defs + +.. default-domain:: c + +TSPluginDSOReloadEnable +************************* + +Control whether this plugin will take part in the remap dynamic reload process (remap.config) + +Synopsis +======== + +.. code-block:: cpp + + #include + +.. function:: TSReturnCode TSPluginDSOReloadEnable(int enabled) + +Description +=========== + +This function provides the ability to enable/disable programmatically the plugin +dynamic reloading when the same Dynamic Shared Object (DSO) is also used as a remap plugin. +This overrides :ts:cv:`proxy.config.plugin.dynamic_reload_mode`. + +.. warning:: This function should be called from within :func:`TSPluginInit` + +The function will return :type:`TS_ERROR` in any of the following cases: + - The function was not called from within :func:`TSPluginInit` + - TS is unable to get the canonical path from the plugin's path. + +See Also +======== + +:manpage:`TSAPI(3ts)` diff --git a/doc/developer-guide/api/functions/TSPluginInit.en.rst b/doc/developer-guide/api/functions/TSPluginInit.en.rst index 232bb79fbe3..6e3c4503599 100644 --- a/doc/developer-guide/api/functions/TSPluginInit.en.rst +++ b/doc/developer-guide/api/functions/TSPluginInit.en.rst @@ -26,7 +26,9 @@ Traffic Server plugin loading and registration. Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: void TSPluginInit(int argc, const char* argv[]) .. function:: TSReturnCode TSPluginRegister(TSPluginRegistrationInfo* plugin_info) diff --git a/doc/developer-guide/api/functions/TSRecords.en.rst b/doc/developer-guide/api/functions/TSRecords.en.rst index 37441395e10..b52cb39d4b1 100644 --- a/doc/developer-guide/api/functions/TSRecords.en.rst +++ b/doc/developer-guide/api/functions/TSRecords.en.rst @@ -26,7 +26,9 @@ Traffic Server Records Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: TSReturnCode TSMgmtStringCreate(TSRecordType rec_type, const char* name, \ const TSMgmtString data_default, TSRecordUpdateType update_type, \ diff --git a/doc/developer-guide/api/functions/TSRemap.en.rst b/doc/developer-guide/api/functions/TSRemap.en.rst index 38d06d871eb..1d7e52ac022 100644 --- a/doc/developer-guide/api/functions/TSRemap.en.rst +++ b/doc/developer-guide/api/functions/TSRemap.en.rst @@ -26,14 +26,17 @@ Traffic Server remap plugin entry points. Synopsis ======== -`#include ` -`#include ` +.. code-block:: cpp -.. function:: TSReturnCode TSRemapInit(TSRemapInterface * api_info, char * errbuf, int errbuf_size) -.. function:: void TSRemapConfigReload(void) + #include + #include + +.. function:: TSReturnCode TSRemapInit(TSRemapInterface * api_info, char * errbuff, int errbuff_size) +.. function:: void TSRemapPreConfigReload(void) +.. function:: void TSRemapPostConfigReload(TSReturnCode reloadStatus) .. function:: void TSRemapDone(void) .. function:: TSRemapStatus TSRemapDoRemap(void * ih, TSHttpTxn rh, TSRemapRequestInfo * rri) -.. function:: TSReturnCode TSRemapNewInstance(int argc, char * argv[], void ** ih, char * errbuf, int errbuf_size) +.. function:: TSReturnCode TSRemapNewInstance(int argc, char * argv[], void ** ih, char * errbuff, int errbuff_size) .. function:: void TSRemapDeleteInstance(void * ) .. function:: void TSRemapOSResponse(void * ih, TSHttpTxn rh, int os_response_type) @@ -50,8 +53,8 @@ route the transaction through your plugin. Multiple remap plugins can be specified for a single remap rule, resulting in a remap plugin chain where each plugin is given an opportunity to examine the HTTP transaction. -:func:`TSRemapInit` is a required entry point. This function will be called -once when Traffic Server loads the plugin. If the optional :func:`TSRemapDone` +:func:`TSRemapInit` is a required entry point. This function will be called once when Traffic Server +loads the plugin. If the optional :func:`TSRemapDone` entry point is available, Traffic Server will call then when unloading the remap plugin. @@ -65,15 +68,24 @@ any data or continuations associated with that instance. entry point. In this function, the remap plugin may examine and modify the HTTP transaction. -:func:`TSRemapConfigReload` is called once for every remap plugin just before the -remap configuration file (:file:`remap.config`) is reloaded. This is an optional -entry point, which takes no arguments and has no return value. +:func:`TSRemapPreConfigReload` is called *before* the parsing of a new remap configuration starts +to notify plugins of the coming configuration reload. It is called on all already loaded plugins, +invoked by current and all previous still used configurations. This is an optional entry point. + +:func:`TSRemapPostConfigReload` is called to indicate the end of the new remap configuration +load. It is called on the newly and previously loaded plugins, invoked by the new, current and +previous still used configurations. It also indicates whether the configuration reload was successful +by passing :macro:`TSREMAP_CONFIG_RELOAD_FAILURE` in case of failure and to notify the plugins if they +are going to be part of the new configuration by passing :macro:`TSREMAP_CONFIG_RELOAD_SUCCESS_PLUGIN_USED` +or :macro:`TSREMAP_CONFIG_RELOAD_SUCCESS_PLUGIN_UNUSED`. This is an optional entry point. Generally speaking, calls to these functions are mutually exclusive. The exception is for functions which take an HTTP transaction as a parameter. Calls to these transaction-specific functions for different transactions are not necessarily mutually exclusive of each other. +For further information, see :ref:`developer-plugins-remap`. + Types ===== @@ -103,6 +115,20 @@ Types The remapping attempt in general failed and the transaction should fail with an error return to the user agent. +.. type:: TSRemapReloadStatus + + .. macro:: TSREMAP_CONFIG_RELOAD_FAILURE + + Notify the plugin that configuration parsing failed. + + .. macro:: TSREMAP_CONFIG_RELOAD_SUCCESS_PLUGIN_USED + + Configuration parsing succeeded and plugin was used by the new configuration. + + .. macro:: TSREMAP_CONFIG_RELOAD_SUCCESS_PLUGIN_UNUSED + + Configuration parsing succeeded but plugin was NOT used by the new configuration. + Return Values ============= diff --git a/doc/developer-guide/api/functions/TSRemapFromToUrlGet.en.rst b/doc/developer-guide/api/functions/TSRemapFromToUrlGet.en.rst index 75c23e9ed16..6fd5e666327 100644 --- a/doc/developer-guide/api/functions/TSRemapFromToUrlGet.en.rst +++ b/doc/developer-guide/api/functions/TSRemapFromToUrlGet.en.rst @@ -24,7 +24,9 @@ TSRemapFrom/ToUrlGet Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: TSReturnCode TSRemapFromUrlGet(TSHttpTxn txnp, TSMLoc * urlLocp) .. function:: TSReturnCode TSRemapToUrlGet(TSHttpTxn txnp, TSMLoc * urlLocp) diff --git a/doc/developer-guide/api/functions/TSSslClientCertUpdate.en.rst b/doc/developer-guide/api/functions/TSSslClientCertUpdate.en.rst index 62fa469eaf2..f1abe938c25 100644 --- a/doc/developer-guide/api/functions/TSSslClientCertUpdate.en.rst +++ b/doc/developer-guide/api/functions/TSSslClientCertUpdate.en.rst @@ -26,7 +26,9 @@ Traffic Server TLS client cert update Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: TSReturnCode TSSslClientCertUpdate(const char *cert_path, const char *key_path) diff --git a/doc/developer-guide/api/functions/TSSslClientContext.en.rst b/doc/developer-guide/api/functions/TSSslClientContext.en.rst index f8d4db417e0..ca14c17b11c 100644 --- a/doc/developer-guide/api/functions/TSSslClientContext.en.rst +++ b/doc/developer-guide/api/functions/TSSslClientContext.en.rst @@ -23,7 +23,9 @@ TSSslClientContext Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: TSReturnCode TSSslClientContextsNamesGet(int n, const char **result, int *actual) diff --git a/doc/developer-guide/api/functions/TSSslContext.en.rst b/doc/developer-guide/api/functions/TSSslContext.en.rst index 415e644adcd..e98606c150c 100644 --- a/doc/developer-guide/api/functions/TSSslContext.en.rst +++ b/doc/developer-guide/api/functions/TSSslContext.en.rst @@ -26,7 +26,9 @@ Traffic Server TLS server context. Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: TSSslContext TSSslContextFindByName(const char * name) diff --git a/doc/developer-guide/api/functions/TSSslServerCertUpdate.en.rst b/doc/developer-guide/api/functions/TSSslServerCertUpdate.en.rst index 44acf21105c..6c31ab39802 100644 --- a/doc/developer-guide/api/functions/TSSslServerCertUpdate.en.rst +++ b/doc/developer-guide/api/functions/TSSslServerCertUpdate.en.rst @@ -26,7 +26,9 @@ Traffic Server TLS server cert update Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: TSReturnCode TSSslServerCertUpdate(const char *cert_path, const char *key_path) diff --git a/doc/developer-guide/api/functions/TSSslServerContextCreate.en.rst b/doc/developer-guide/api/functions/TSSslServerContextCreate.en.rst index 4cae0645917..097044c33af 100644 --- a/doc/developer-guide/api/functions/TSSslServerContextCreate.en.rst +++ b/doc/developer-guide/api/functions/TSSslServerContextCreate.en.rst @@ -26,7 +26,9 @@ Traffic Server TLS server context creation. Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: TSSslContext TSSslServerContextCreate(TSSslX509 *cert, char *certname) .. function:: void TSSslContextDestroy(TSSslContext ctx) diff --git a/doc/developer-guide/api/functions/TSSslSession.en.rst b/doc/developer-guide/api/functions/TSSslSession.en.rst index d0870a1f4c0..7b265879652 100644 --- a/doc/developer-guide/api/functions/TSSslSession.en.rst +++ b/doc/developer-guide/api/functions/TSSslSession.en.rst @@ -24,7 +24,9 @@ TSSslSession Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: TSSslSession TSSslSessionGet(const TSSslSessionID * sessionid) .. function:: int TSSslSessionGetBuffer(const TSSslSessionID * sessionid, char * buffer, int * len_ptr) @@ -53,6 +55,10 @@ returns 0. :func:`TSSslSessionGetBuffer` returns the session information serialized in a buffer that can be shared between processes. When the function is called len_ptr should point to the amount of space available in the buffer parameter. The function returns the amount of data really needed to encode the session. len_ptr is updated with the amount of data actually stored in the buffer. +:func:`TSSslSessionGetBuffer` will not overrun the provided buffer, but the caller should ensure that the data's size was not larger +than the buffer by comparing the returned value with the value of len_ptr. If the returned value is larger than the buffer size, +then the session data did not fit in the buffer and the session data stored in the buffer output variable should not be used. + :func:`TSSslSessionInsert` inserts the session specified by the addSession parameter into the ATS session cache under the sessionid key. If there is already an entry in the cache for the session id key, it is first removed before the new entry is added. diff --git a/doc/developer-guide/api/functions/TSStat.en.rst b/doc/developer-guide/api/functions/TSStat.en.rst index 64b539f2bbc..2988bc17cef 100644 --- a/doc/developer-guide/api/functions/TSStat.en.rst +++ b/doc/developer-guide/api/functions/TSStat.en.rst @@ -28,7 +28,9 @@ in contrast to processing log files. Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: int TSStatCreate(const char * name, TSRecordDataType type, TSStatPersistence persistence, TSStatSync sync_style) .. function:: TSReturnCode TSStatFindName(const char * name, int * idx_ptr) diff --git a/doc/developer-guide/api/functions/TSTextLogObjectCreate.en.rst b/doc/developer-guide/api/functions/TSTextLogObjectCreate.en.rst index 10990248134..063bd8a4b07 100644 --- a/doc/developer-guide/api/functions/TSTextLogObjectCreate.en.rst +++ b/doc/developer-guide/api/functions/TSTextLogObjectCreate.en.rst @@ -27,7 +27,9 @@ Traffic Server text logging API. Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: TSReturnCode TSTextLogObjectCreate(const char * filename, int mode, TSTextLogObject * new_log_obj) .. function:: TSReturnCode TSTextLogObjectWrite(TSTextLogObject the_object, const char * format, ...) diff --git a/doc/developer-guide/api/functions/TSThreadCreate.en.rst b/doc/developer-guide/api/functions/TSThreadCreate.en.rst index 9d1c5be16ba..b0cc86cf71a 100644 --- a/doc/developer-guide/api/functions/TSThreadCreate.en.rst +++ b/doc/developer-guide/api/functions/TSThreadCreate.en.rst @@ -24,7 +24,9 @@ TSThreadCreate Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: TSThread TSThreadCreate(TSThreadFunc func, void * data) diff --git a/doc/developer-guide/api/functions/TSThreadDestroy.en.rst b/doc/developer-guide/api/functions/TSThreadDestroy.en.rst index 46df4be89b9..91ffc4a3261 100644 --- a/doc/developer-guide/api/functions/TSThreadDestroy.en.rst +++ b/doc/developer-guide/api/functions/TSThreadDestroy.en.rst @@ -24,7 +24,9 @@ TSThreadDestroy Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: void TSThreadDestroy(TSThread thread) diff --git a/doc/developer-guide/api/functions/TSThreadInit.en.rst b/doc/developer-guide/api/functions/TSThreadInit.en.rst index 0c3590f5308..4d1f88b0fd4 100644 --- a/doc/developer-guide/api/functions/TSThreadInit.en.rst +++ b/doc/developer-guide/api/functions/TSThreadInit.en.rst @@ -24,7 +24,9 @@ TSThreadInit Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: TSThread TSThreadInit(void) diff --git a/doc/developer-guide/api/functions/TSThreadSelf.en.rst b/doc/developer-guide/api/functions/TSThreadSelf.en.rst index b0d059013de..eedfbddd7f5 100644 --- a/doc/developer-guide/api/functions/TSThreadSelf.en.rst +++ b/doc/developer-guide/api/functions/TSThreadSelf.en.rst @@ -24,7 +24,9 @@ TSThreadSelf Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: TSThread TSThreadSelf(void) diff --git a/doc/developer-guide/api/functions/TSTrafficServerVersionGet.en.rst b/doc/developer-guide/api/functions/TSTrafficServerVersionGet.en.rst index 076ab1e7294..1d98ab992fd 100644 --- a/doc/developer-guide/api/functions/TSTrafficServerVersionGet.en.rst +++ b/doc/developer-guide/api/functions/TSTrafficServerVersionGet.en.rst @@ -26,7 +26,9 @@ Return Traffic Server version information. Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: const char * TSTrafficServerVersionGet(void) .. function:: int TSTrafficServerVersionGetMajor(void) diff --git a/doc/developer-guide/api/functions/TSTransformCreate.en.rst b/doc/developer-guide/api/functions/TSTransformCreate.en.rst index 4beea2ead2b..6735ca8586a 100644 --- a/doc/developer-guide/api/functions/TSTransformCreate.en.rst +++ b/doc/developer-guide/api/functions/TSTransformCreate.en.rst @@ -24,7 +24,9 @@ TSTransformCreate Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: TSVConn TSTransformCreate(TSEventFunc event_funcp, TSHttpTxn txnp) diff --git a/doc/developer-guide/api/functions/TSTransformOutputVConnGet.en.rst b/doc/developer-guide/api/functions/TSTransformOutputVConnGet.en.rst index 236b9b238cd..a272315c98a 100644 --- a/doc/developer-guide/api/functions/TSTransformOutputVConnGet.en.rst +++ b/doc/developer-guide/api/functions/TSTransformOutputVConnGet.en.rst @@ -24,7 +24,9 @@ TSTransformOutputVConnGet Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: TSVConn TSTransformOutputVConnGet(TSVConn connp) diff --git a/doc/developer-guide/api/functions/TSTypes.en.rst b/doc/developer-guide/api/functions/TSTypes.en.rst index 9ffc3185ad5..ef9098796d1 100644 --- a/doc/developer-guide/api/functions/TSTypes.en.rst +++ b/doc/developer-guide/api/functions/TSTypes.en.rst @@ -28,8 +28,10 @@ TSAPI Types Synopsis ======== -`#include ` -`#include ` +.. code-block:: cpp + + #include + #include Description =========== @@ -100,7 +102,7 @@ more widely. Those are described on this page. .. type:: TSMBuffer - Internally, data for a transaction is stored in one more more :term:`header heap`\s. These are + Internally, data for a transaction is stored in one or more :term:`header heap`\s. These are storage local to the transaction, and generally each HTTP header is stored in a separate one. This type is a handle to a header heap, and is provided or required by functions that locate HTTP header related data. @@ -143,8 +145,48 @@ more widely. Those are described on this page. .. type:: TSRemapInterface + Data passed to a remap plugin via :func:`TSRemapInit`. + + .. member:: unsigned long size + + The size of the structure in bytes, including this member. + + .. member:: unsigned long tsremap_version + + The API version of the C API. The lower 16 bits are the minor version, and the upper bits + the major version. + .. type:: TSRemapRequestInfo + Data passed to a remap plugin during the invocation of a remap rule. + + .. member:: TSMBuffer requestBufp + + The client request. All of the other :type:`TSMLoc` values use this as the base buffer. + + .. member:: TSMLoc requestHdrp + + The client request. + + .. member:: TSMLoc mapFromUrl + + The match URL in the remap rule. + + .. member:: TSMLoc mapToUrl + + The target URL in the remap rule. + + .. member:: TSMLoc requestUrl + + The current request URL. The remap rule and plugins listed earlier in the remap rule can modify this + from the client request URL. Remap plugins are expected to modify this value to perform the + remapping of the request. Note this is the same :code:`TSMLoc` as would be obtained by + calling :func:`TSHttpTxnClientReqGet`. + + .. member:: int redirect + + Flag for using the remapped URL as an explicit redirection. This can be set by the remap plugin. + .. type:: TSSslX509 This type represents the :code:`X509` object created from an SSL certificate. @@ -175,6 +217,10 @@ more widely. Those are described on this page. .. type:: TSThreadFunc +.. type:: TSUserArgType + + An enum for the supported types of user arguments. + .. type:: TSUuidVersion A version value for at :type:`TSUuid`. diff --git a/doc/developer-guide/api/functions/TSUrlCreate.en.rst b/doc/developer-guide/api/functions/TSUrlCreate.en.rst index 6b096d0f51b..7d5a027d67e 100644 --- a/doc/developer-guide/api/functions/TSUrlCreate.en.rst +++ b/doc/developer-guide/api/functions/TSUrlCreate.en.rst @@ -27,7 +27,9 @@ Traffic Server URL object construction API. Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: TSReturnCode TSUrlCreate(TSMBuffer bufp, TSMLoc * locp) .. function:: TSReturnCode TSUrlClone(TSMBuffer dest_bufp, TSMBuffer src_bufp, TSMLoc src_url, TSMLoc * locp) diff --git a/doc/developer-guide/api/functions/TSUrlFtpTypeGet.en.rst b/doc/developer-guide/api/functions/TSUrlFtpTypeGet.en.rst index ad363f963c4..ed4edd6f1b5 100644 --- a/doc/developer-guide/api/functions/TSUrlFtpTypeGet.en.rst +++ b/doc/developer-guide/api/functions/TSUrlFtpTypeGet.en.rst @@ -24,7 +24,9 @@ TSUrlFtpTypeGet Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: int TSUrlFtpTypeGet(TSMBuffer bufp, TSMLoc offset) diff --git a/doc/developer-guide/api/functions/TSUrlFtpTypeSet.en.rst b/doc/developer-guide/api/functions/TSUrlFtpTypeSet.en.rst index f6e99c0b972..fbf2316cb12 100644 --- a/doc/developer-guide/api/functions/TSUrlFtpTypeSet.en.rst +++ b/doc/developer-guide/api/functions/TSUrlFtpTypeSet.en.rst @@ -24,7 +24,9 @@ TSUrlFtpTypeSet Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: TSReturnCode TSUrlFtpTypeSet(TSMBuffer bufp, TSMLoc offset, int type) diff --git a/doc/developer-guide/api/functions/TSUrlHostGet.en.rst b/doc/developer-guide/api/functions/TSUrlHostGet.en.rst index d5d2db116e0..bb071765054 100644 --- a/doc/developer-guide/api/functions/TSUrlHostGet.en.rst +++ b/doc/developer-guide/api/functions/TSUrlHostGet.en.rst @@ -27,7 +27,9 @@ Traffic Server URL component retrieval API. Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: const char * TSUrlHostGet(TSMBuffer bufp, TSMLoc offset, int * length) .. function:: const char * TSUrlSchemeGet(TSMBuffer bufp, TSMLoc offset, int * length) @@ -49,7 +51,7 @@ and retrieve or modify parts of URLs, such as their host, port or scheme information. :func:`TSUrlSchemeGet`, :func:`TSUrlUserGet`, :func:`TSUrlPasswordGet`, -:func:`TSUrlHostGet`, :func:`TSUrlHttpParamsGet`, :func:`TSUrlHttpQueryGet` +:func:`TSUrlHostGet`, :func:`TSUrlPathGet`, :func:`TSUrlHttpParamsGet`, :func:`TSUrlHttpQueryGet` and :func:`TSUrlHttpFragmentGet` each retrieve an internal pointer to the specified portion of the URL from the marshall buffer :arg:`bufp`. The length of the returned string is placed in :arg:`length` and a pointer to the URL diff --git a/doc/developer-guide/api/functions/TSUrlHostSet.en.rst b/doc/developer-guide/api/functions/TSUrlHostSet.en.rst index 4ace0b61356..0b89f8e6e12 100644 --- a/doc/developer-guide/api/functions/TSUrlHostSet.en.rst +++ b/doc/developer-guide/api/functions/TSUrlHostSet.en.rst @@ -27,7 +27,9 @@ Traffic Server URL component manipulation API. Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: TSReturnCode TSUrlHostSet(TSMBuffer bufp, TSMLoc offset, const char * value, int length) .. function:: TSReturnCode TSUrlSchemeSet(TSMBuffer bufp, TSMLoc offset, const char * value, int length) diff --git a/doc/developer-guide/api/functions/TSUrlPercentEncode.en.rst b/doc/developer-guide/api/functions/TSUrlPercentEncode.en.rst index f5f13b2916b..d342368d452 100644 --- a/doc/developer-guide/api/functions/TSUrlPercentEncode.en.rst +++ b/doc/developer-guide/api/functions/TSUrlPercentEncode.en.rst @@ -27,7 +27,9 @@ Traffic Server URL percent encoding API. Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: TSReturnCode TSUrlPercentEncode(TSMBuffer bufp, TSMLoc offset, char * dst, size_t dst_size, size_t * length, const unsigned char * map) .. function:: TSReturnCode TSStringPercentEncode(const char * str, int str_len, char * dst, size_t dst_size, size_t * length, const unsigned char * map) @@ -49,16 +51,28 @@ failed. :func:`TSStringPercentEncode` is similar but operates on a string. If the optional :arg:`map` parameter is provided (not :literal:`NULL`) , it should be a map of characters to encode. -:func:`TSStringPercentDecode` perform percent-decoding of the string in the -:arg:`str` buffer, writing to the :arg:`dst` buffer. The source and -destination can be the same, in which case they overwrite. The decoded string -is always guaranteed to be no longer than the source string. +:func:`TSStringPercentDecode` perform percent-decoding of the string in the :arg:`str` buffer, +writing to the :arg:`dst` buffer. The source and destination can be the same, in which case the +decoded string is written on top of the source string. The decoded string is guaranteed to be +no longer than the source string, but will include a terminating null which, if there are no +escapes, makes the destination one longer than the source. In practice this means the destination +length needs to be bumped up by one to account for the null, and a string can't be decoded in place +if it's not already null terminated with the length of the destination including the null, but the +length of the source *not* including the null. E.g. :: + + static char const ORIGINAL[] = "A string without escapes, but null terminated"; + char * source = TSstrdup(ORIGINAL); // make it writeable. + size_t length; // return value. + // sizeof(ORIGINAL) includes the null, so don't include that in the input. + static const size_t N_CHARS = sizeof(ORIGINAL) - 1; + TSReturnCode result = TSUrlPercentDecode(source, N_CHARS, source, N_CHARS + 1, &length); + ink_assert(length == N_CHARS); Return Values ============= -All these APIs returns a :type:`TSReturnCode`, indicating success -(:data:`TS_SUCCESS`) or failure (:data:`TS_ERROR`) of the operation. +All these APIs returns a :type:`TSReturnCode`, indicating success (:data:`TS_SUCCESS`) or failure +(:data:`TS_ERROR`) of the operation. See Also ======== diff --git a/doc/developer-guide/api/functions/TSUrlStringGet.en.rst b/doc/developer-guide/api/functions/TSUrlStringGet.en.rst index 506a5e8d237..1e58cf6c2fc 100644 --- a/doc/developer-guide/api/functions/TSUrlStringGet.en.rst +++ b/doc/developer-guide/api/functions/TSUrlStringGet.en.rst @@ -27,7 +27,9 @@ Traffic Server URL string representations API. Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: char * TSUrlStringGet(TSMBuffer bufp, TSMLoc offset, int * length) .. function:: char * TSHttpTxnEffectiveUrlStringGet(TSHttpTxn txn, int * length) diff --git a/doc/developer-guide/api/functions/TSUserArgs.en.rst b/doc/developer-guide/api/functions/TSUserArgs.en.rst new file mode 100644 index 00000000000..5dc020885ca --- /dev/null +++ b/doc/developer-guide/api/functions/TSUserArgs.en.rst @@ -0,0 +1,112 @@ +.. Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed + with this work for additional information regarding copyright + ownership. The ASF licenses this file to you under the Apache + License, Version 2.0 (the "License"); you may not use this file + except in compliance with the License. You may obtain a copy of + the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied. See the License for the specific language governing + permissions and limitations under the License. + +.. include:: ../../../common.defs +.. default-domain:: c + +.. _tsuserargs: + +TSUserArgs +********** + +Synopsis +======== + +.. code-block:: cpp + + #include + + typedef enum { + TS_USER_ARGS_TXN, ///< Transaction based. + TS_USER_ARGS_SSN, ///< Session based + TS_USER_ARGS_VCONN, ///< VConnection based + TS_USER_ARGS_GLB, ///< Global based + TS_USER_ARGS_COUNT ///< Fake enum, # of valid entries. + } TSUserArgType; + +.. function:: TSReturnCode TSUserArgIndexReserve(TSUserArgType type, const char *name, const char *description, int *arg_idx) +.. function:: TSReturnCode TSUserArgIndexNameLookup(TSUserArgType type, const char *name, int *arg_idx, const char **description) +.. function:: TSReturnCode TSUserArgIndexLookup(TSUserArgType type, int arg_idx, const char **name, const char **description) +.. function:: void TSUserArgSet(void *data, int arg_idx, void *arg) +.. function:: void *TSUserArgGet(void *data, int arg_idx) + +Description +=========== + +|TS| sessions, transactions, virtual connections and globally provide a fixed array of void pointers +that can be used by plugins to store information. This can be used to avoid creating a per session or +transaction continuations to hold data, or to communicate between plugins as the values in the array +are visible to any plugin which can access the session or transaction. The array values are opaque +to |TS| and it will not dereference nor release them. Plugins are responsible for cleaning up any +resources pointed to by the values or, if the values are simply values, there is no need for the plugin +to remove them after the session or transaction has completed. + +To avoid collisions between plugins a plugin should first *reserve* an index in the array. A +plugin can reserve a slot of a particular type by calling :func:`TSUserArgIndexReserve`. The arguments are: + +:arg:`type` + The type for which the plugin intend to reserve a slot. See :code:`TSUserArgType` above. + +:arg:`name` + An identifying name for the plugin that reserved the index. Required. + +:arg:`description` + An optional description of the use of the arg. This can be :code:`nullptr`. + +:arg:`arg_idx` + A pointer to an :code:`int`. If an index is successfully reserved, the :code:`int` pointed at by this is + set to the reserved index. It is not modified if the call is unsuccessful. + +The functions return :code:`TS_SUCCESS` if an index was reserved, +:code:`TS_ERROR` if not (most likely because all of the indices have already been reserved). +Generally this will be a file or library scope global which is set at plugin initialization. This +function is used in the example remap plugin :ts:git:`example/plugins/c-api/remap/remap.cc`. The index is stored +in the plugin global :code:`arg_index`. Transaction and session plugin argument indices are reserved +independently. + +To look up the owner of a reserved index use :func:`TSUserArgIndexNameLookup`, with the appropriate type. +If :arg:`name` is found as an owner, the function returns :code:`TS_SUCCESS` and :arg:`arg_index` is +updated with the index reserved under that name. If :arg:`description` is not :code:`NULL` then +the character pointer to which it points will be updated to point at the description for that +reserved index. This enables communication between plugins where plugin "A" reserves an index under +a well known name and plugin "B" locates the index by looking it up under that name. + +The owner of a reserved index can be found with :func:`TSUserArgIndexLookup`. If +:arg:`arg_index` is reserved then the function returns :code:`TS_SUCCESS` and the pointers referred +to by :arg:`name` and :arg:`description` are updated. :arg:`name` must point at a valid character +pointer but :arg:`description` can be :code:`NULL` in which case it is ignored. + +Manipulating the array is simple. :func:`TSUserArgSet` sets the array slot at :arg:`arg_idx`, for the +particular type based on the provide data pointer. The values can be retrieved with the value from +:func:`TSUserArgGet`. Values that have not been set are :code:`NULL`. Note that both the setter and the getter are +context sensitive, based on the type (or value) of the data pointer: + + ============== ======================================================================= + data type Semantics + ============== ======================================================================= + ``TSHttpTxn`` The implicit context is for a transaction (``TS_USER_ARGS_TXN``) + ``TSHttpSsn`` The implicit context is for a transaction (``TS_USER_ARGS_SSN``) + ``TSVConn`` The implicit context is for a transaction (``TS_USER_ARGS_VCONN``) + ``nullptr`` The implicit context is global (``TS_USER_ARGS_GLB``) + ============== ======================================================================= + +Note that neither :func:`TSUserArgSet` nor :func:`TSUserArgGet` has any type safety on the :arg:`data` +parameters, being a ``void*`` pointer. + + +.. note:: Session arguments persist for the entire session, which means potentially across all transactions in that session. + +.. note:: Following arg index reservations is conventional, it is not enforced. diff --git a/doc/developer-guide/api/functions/TSUuidCreate.en.rst b/doc/developer-guide/api/functions/TSUuidCreate.en.rst index 7def6f38813..4e3a68dfd9d 100644 --- a/doc/developer-guide/api/functions/TSUuidCreate.en.rst +++ b/doc/developer-guide/api/functions/TSUuidCreate.en.rst @@ -26,7 +26,9 @@ Traffic Server UUID construction APIs. Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: TSUuid TSUuidCreate(void) .. function:: TSReturnCode TSUuidInitialize(TSUuid uuid, TSUuidVersion v) diff --git a/doc/developer-guide/api/functions/TSVConn.en.rst b/doc/developer-guide/api/functions/TSVConn.en.rst index e3f625b1eb7..e73b7872e72 100644 --- a/doc/developer-guide/api/functions/TSVConn.en.rst +++ b/doc/developer-guide/api/functions/TSVConn.en.rst @@ -26,7 +26,9 @@ Traffic Server APIs to get :type:`TSVConn` from :type:`TSHttpSsn` object Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: TSVConn TSHttpSsnClientVConnGet(TSHttpSsn ssnp) .. function:: TSVConn TSHttpSsnServerVConnGet(TSHttpSsn ssnp) diff --git a/doc/developer-guide/api/functions/TSVConnAbort.en.rst b/doc/developer-guide/api/functions/TSVConnAbort.en.rst index bb2d528984e..41f98b76af1 100644 --- a/doc/developer-guide/api/functions/TSVConnAbort.en.rst +++ b/doc/developer-guide/api/functions/TSVConnAbort.en.rst @@ -24,7 +24,9 @@ TSVConnAbort Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: void TSVConnAbort(TSVConn connp, int error) diff --git a/doc/developer-guide/api/functions/TSVConnArgs.en.rst b/doc/developer-guide/api/functions/TSVConnArgs.en.rst index 21036694a03..be575296b9d 100644 --- a/doc/developer-guide/api/functions/TSVConnArgs.en.rst +++ b/doc/developer-guide/api/functions/TSVConnArgs.en.rst @@ -17,15 +17,21 @@ .. include:: ../../../common.defs .. default-domain:: c -.. _TSVConnArgs: - TSVConnArgs ************ Synopsis ======== -`#include ` +.. note:: + + This set of API is obsoleted as of ATS v9.0.0, and will be removed with ATS v10.0.0! + For details of the new APIs, see :ref:`tsuserargs`. + + +.. code-block:: cpp + + #include .. function:: TSReturnCode TSVConnArgIndexReserve(const char * name, const char * description, int * arg_idx) .. function:: TSReturnCode TSVConnArgIndexNameLookup(const char * name, int * arg_idx, const char ** description) diff --git a/doc/developer-guide/api/functions/TSVConnCacheObjectSizeGet.en.rst b/doc/developer-guide/api/functions/TSVConnCacheObjectSizeGet.en.rst index 9411a97302b..c75780e25d4 100644 --- a/doc/developer-guide/api/functions/TSVConnCacheObjectSizeGet.en.rst +++ b/doc/developer-guide/api/functions/TSVConnCacheObjectSizeGet.en.rst @@ -24,7 +24,9 @@ TSVConnCacheObjectSizeGet Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: int64_t TSVConnCacheObjectSizeGet(TSVConn connp) diff --git a/doc/developer-guide/api/functions/TSVConnClose.en.rst b/doc/developer-guide/api/functions/TSVConnClose.en.rst index a0edc3efb3b..bbd67d9379a 100644 --- a/doc/developer-guide/api/functions/TSVConnClose.en.rst +++ b/doc/developer-guide/api/functions/TSVConnClose.en.rst @@ -24,7 +24,9 @@ TSVConnClose Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: void TSVConnClose(TSVConn connp) diff --git a/doc/developer-guide/api/functions/TSVConnClosedGet.en.rst b/doc/developer-guide/api/functions/TSVConnClosedGet.en.rst index ec102777018..e3608b1d49d 100644 --- a/doc/developer-guide/api/functions/TSVConnClosedGet.en.rst +++ b/doc/developer-guide/api/functions/TSVConnClosedGet.en.rst @@ -24,7 +24,9 @@ TSVConnClosedGet Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: int TSVConnClosedGet(TSVConn connp) diff --git a/doc/developer-guide/api/functions/TSVConnCreate.en.rst b/doc/developer-guide/api/functions/TSVConnCreate.en.rst index 1e0cd50865b..35d9eec84ad 100644 --- a/doc/developer-guide/api/functions/TSVConnCreate.en.rst +++ b/doc/developer-guide/api/functions/TSVConnCreate.en.rst @@ -24,7 +24,9 @@ TSVConnCreate Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: TSCont TSVConnCreate(TSEventFunc funcp, TSMutex mutexp) diff --git a/doc/developer-guide/api/functions/TSVConnFdCreate.en.rst b/doc/developer-guide/api/functions/TSVConnFdCreate.en.rst index 48e9aec4392..03fccfa2afd 100644 --- a/doc/developer-guide/api/functions/TSVConnFdCreate.en.rst +++ b/doc/developer-guide/api/functions/TSVConnFdCreate.en.rst @@ -26,7 +26,9 @@ Create a TSVConn from a socket. Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: TSVConn TSVConnFdCreate(int fd) diff --git a/doc/developer-guide/api/functions/TSVConnIsSsl.en.rst b/doc/developer-guide/api/functions/TSVConnIsSsl.en.rst index ba4bbf0cf18..94443bc1a38 100644 --- a/doc/developer-guide/api/functions/TSVConnIsSsl.en.rst +++ b/doc/developer-guide/api/functions/TSVConnIsSsl.en.rst @@ -24,7 +24,9 @@ TSVConnIsSsl Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: int TSVConnIsSsl(TSVConn svc) diff --git a/doc/developer-guide/api/functions/TSVConnProtocol.en.rst b/doc/developer-guide/api/functions/TSVConnProtocol.en.rst index 49d1b88d47d..7bcf994c06b 100644 --- a/doc/developer-guide/api/functions/TSVConnProtocol.en.rst +++ b/doc/developer-guide/api/functions/TSVConnProtocol.en.rst @@ -24,7 +24,9 @@ TSVConnProtocolEnable/Disable Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: TSReturnCode TSVConnProtocolEnable(TSVConn vconn, const char* protocol) .. function:: TSReturnCode TSVConnProtocolDisable(TSVConn vconn, const char* protocol) diff --git a/doc/developer-guide/api/functions/TSVConnRead.en.rst b/doc/developer-guide/api/functions/TSVConnRead.en.rst index 7334de244e5..965b8bc8946 100644 --- a/doc/developer-guide/api/functions/TSVConnRead.en.rst +++ b/doc/developer-guide/api/functions/TSVConnRead.en.rst @@ -24,7 +24,9 @@ TSVConnRead Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: TSVIO TSVConnRead(TSVConn connp, TSCont contp, TSIOBuffer bufp, int64_t nbytes) diff --git a/doc/developer-guide/api/functions/TSVConnReadVIOGet.en.rst b/doc/developer-guide/api/functions/TSVConnReadVIOGet.en.rst index 33369bb49e2..dfbafd4d9e3 100644 --- a/doc/developer-guide/api/functions/TSVConnReadVIOGet.en.rst +++ b/doc/developer-guide/api/functions/TSVConnReadVIOGet.en.rst @@ -24,7 +24,9 @@ TSVConnReadVIOGet Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: TSVIO TSVConnReadVIOGet(TSVConn connp) diff --git a/doc/developer-guide/api/functions/TSVConnReenable.en.rst b/doc/developer-guide/api/functions/TSVConnReenable.en.rst index a7075fc8e7e..e77bb038fc3 100644 --- a/doc/developer-guide/api/functions/TSVConnReenable.en.rst +++ b/doc/developer-guide/api/functions/TSVConnReenable.en.rst @@ -24,7 +24,9 @@ TSVConnReenable Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: void TSVConnReenable(TSVConn svc) @@ -66,8 +68,6 @@ An extended version of TSVConnEnable that allows the plugin to return a status t the core logic. If all goes well this is TS_EVENT_CONTINUE. However, if the plugin wants to stop the processing it can set the event to TS_EVENT_ERROR. -For example, in the case of the TS_SSL_VERIFY_SERVER_HOOK, the plugin make decide the -origin certificate is bad. By calling TSVonnReenable with TS_EVENT_ERROR, the +For example, in the case of the TS_SSL_VERIFY_SERVER_HOOK, the plugin make decide the +origin certificate is bad. By calling TSVonnReenable with TS_EVENT_ERROR, the certificate check will error and the TLS handshake will fail. - - diff --git a/doc/developer-guide/api/functions/TSVConnShutdown.en.rst b/doc/developer-guide/api/functions/TSVConnShutdown.en.rst index 25b05800bbd..4e36e6bf6d3 100644 --- a/doc/developer-guide/api/functions/TSVConnShutdown.en.rst +++ b/doc/developer-guide/api/functions/TSVConnShutdown.en.rst @@ -24,7 +24,9 @@ TSVConnShutdown Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: void TSVConnShutdown(TSVConn connp, int read, int write) diff --git a/doc/developer-guide/api/functions/TSVConnSslConnectionGet.en.rst b/doc/developer-guide/api/functions/TSVConnSslConnectionGet.en.rst index e4c1c490350..5cb3b60b90c 100644 --- a/doc/developer-guide/api/functions/TSVConnSslConnectionGet.en.rst +++ b/doc/developer-guide/api/functions/TSVConnSslConnectionGet.en.rst @@ -24,7 +24,9 @@ TSVConnSslConnectionGet Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: TSSslConnection TSVConnSslConnectionGet(TSVConn svc) diff --git a/doc/developer-guide/api/functions/TSVConnSslVerifyCTXGet.en.rst b/doc/developer-guide/api/functions/TSVConnSslVerifyCTXGet.en.rst index c35a25c5658..5a90ba24a27 100644 --- a/doc/developer-guide/api/functions/TSVConnSslVerifyCTXGet.en.rst +++ b/doc/developer-guide/api/functions/TSVConnSslVerifyCTXGet.en.rst @@ -24,14 +24,16 @@ TSVConnSslVerifyCTXGet Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: TSSslVerifyCTX TSVConnSslVerifyCTXGet(TSVConn svc) Description =========== -Get the TSSslVerifyCTX object that corresponds to the certificates being verified for the SSL connection +Get the TSSslVerifyCTX object that corresponds to the certificates being verified for the SSL connection corresponding to :arg:`svc`. This value is only meaningful during the peer certificate verification callbacks, specifically during callbacks diff --git a/doc/developer-guide/api/functions/TSVConnTunnel.en.rst b/doc/developer-guide/api/functions/TSVConnTunnel.en.rst index e27b8113760..cfbb8a18d94 100644 --- a/doc/developer-guide/api/functions/TSVConnTunnel.en.rst +++ b/doc/developer-guide/api/functions/TSVConnTunnel.en.rst @@ -24,7 +24,9 @@ TSVConnTunnel Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: TSReturnCode TSVConnTunnel(TSVConn svc) diff --git a/doc/developer-guide/api/functions/TSVConnWrite.en.rst b/doc/developer-guide/api/functions/TSVConnWrite.en.rst index 10b0fa611cb..acce4d8edc2 100644 --- a/doc/developer-guide/api/functions/TSVConnWrite.en.rst +++ b/doc/developer-guide/api/functions/TSVConnWrite.en.rst @@ -24,7 +24,9 @@ TSVConnWrite Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: TSVIO TSVConnWrite(TSVConn connp, TSCont contp, TSIOBufferReader readerp, int64_t nbytes) diff --git a/doc/developer-guide/api/functions/TSVConnWriteVIOGet.en.rst b/doc/developer-guide/api/functions/TSVConnWriteVIOGet.en.rst index 802812a1f5e..25061f2c81d 100644 --- a/doc/developer-guide/api/functions/TSVConnWriteVIOGet.en.rst +++ b/doc/developer-guide/api/functions/TSVConnWriteVIOGet.en.rst @@ -24,7 +24,9 @@ TSVConnWriteVIOGet Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: TSVIO TSVConnWriteVIOGet(TSVConn connp) diff --git a/doc/developer-guide/api/functions/TSVIOBufferGet.en.rst b/doc/developer-guide/api/functions/TSVIOBufferGet.en.rst index bf90c841496..0c6db9c7c7b 100644 --- a/doc/developer-guide/api/functions/TSVIOBufferGet.en.rst +++ b/doc/developer-guide/api/functions/TSVIOBufferGet.en.rst @@ -24,7 +24,9 @@ TSVIOBufferGet Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: TSIOBuffer TSVIOBufferGet(TSVIO viop) diff --git a/doc/developer-guide/api/functions/TSVIOContGet.en.rst b/doc/developer-guide/api/functions/TSVIOContGet.en.rst index 418eeeb5b37..37dd977530f 100644 --- a/doc/developer-guide/api/functions/TSVIOContGet.en.rst +++ b/doc/developer-guide/api/functions/TSVIOContGet.en.rst @@ -24,7 +24,9 @@ TSVIOContGet Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: TSCont TSVIOContGet(TSVIO viop) diff --git a/doc/developer-guide/api/functions/TSVIOMutexGet.en.rst b/doc/developer-guide/api/functions/TSVIOMutexGet.en.rst index 160cc1bdcb3..0b28ef2e6db 100644 --- a/doc/developer-guide/api/functions/TSVIOMutexGet.en.rst +++ b/doc/developer-guide/api/functions/TSVIOMutexGet.en.rst @@ -24,7 +24,9 @@ TSVIOMutexGet Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: TSMutex TSVIOMutexGet(TSVIO viop) diff --git a/doc/developer-guide/api/functions/TSVIONBytesGet.en.rst b/doc/developer-guide/api/functions/TSVIONBytesGet.en.rst index 242deb83a0a..6e15b46afbb 100644 --- a/doc/developer-guide/api/functions/TSVIONBytesGet.en.rst +++ b/doc/developer-guide/api/functions/TSVIONBytesGet.en.rst @@ -24,7 +24,9 @@ TSVIONBytesGet Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: int64_t TSVIONBytesGet(TSVIO viop) diff --git a/doc/developer-guide/api/functions/TSVIONBytesSet.en.rst b/doc/developer-guide/api/functions/TSVIONBytesSet.en.rst index 27a1bfe01dc..bd4ec83bfd6 100644 --- a/doc/developer-guide/api/functions/TSVIONBytesSet.en.rst +++ b/doc/developer-guide/api/functions/TSVIONBytesSet.en.rst @@ -24,7 +24,9 @@ TSVIONBytesSet Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: void TSVIONBytesSet(TSVIO viop, int64_t nbytes) diff --git a/doc/developer-guide/api/functions/TSVIONDoneGet.en.rst b/doc/developer-guide/api/functions/TSVIONDoneGet.en.rst index 79f7edfb7a1..fccb2776763 100644 --- a/doc/developer-guide/api/functions/TSVIONDoneGet.en.rst +++ b/doc/developer-guide/api/functions/TSVIONDoneGet.en.rst @@ -24,7 +24,9 @@ TSVIONDoneGet Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: int64_t TSVIONDoneGet(TSVIO viop) diff --git a/doc/developer-guide/api/functions/TSVIONDoneSet.en.rst b/doc/developer-guide/api/functions/TSVIONDoneSet.en.rst index f4483798d07..0580c7cb401 100644 --- a/doc/developer-guide/api/functions/TSVIONDoneSet.en.rst +++ b/doc/developer-guide/api/functions/TSVIONDoneSet.en.rst @@ -24,7 +24,9 @@ TSVIONDoneSet Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: void TSVIONDoneSet(TSVIO viop, int64_t ndone) diff --git a/doc/developer-guide/api/functions/TSVIONTodoGet.en.rst b/doc/developer-guide/api/functions/TSVIONTodoGet.en.rst index 91c6acd597c..249c5483843 100644 --- a/doc/developer-guide/api/functions/TSVIONTodoGet.en.rst +++ b/doc/developer-guide/api/functions/TSVIONTodoGet.en.rst @@ -24,7 +24,9 @@ TSVIONTodoGet Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: int64_t TSVIONTodoGet(TSVIO viop) diff --git a/doc/developer-guide/api/functions/TSVIOReaderGet.en.rst b/doc/developer-guide/api/functions/TSVIOReaderGet.en.rst index 559a6e6f941..22f1782fbc6 100644 --- a/doc/developer-guide/api/functions/TSVIOReaderGet.en.rst +++ b/doc/developer-guide/api/functions/TSVIOReaderGet.en.rst @@ -24,7 +24,9 @@ TSVIOReaderGet Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: TSIOBufferReader TSVIOReaderGet(TSVIO viop) diff --git a/doc/developer-guide/api/functions/TSVIOReenable.en.rst b/doc/developer-guide/api/functions/TSVIOReenable.en.rst index 7103f882296..c1f5d8ead0e 100644 --- a/doc/developer-guide/api/functions/TSVIOReenable.en.rst +++ b/doc/developer-guide/api/functions/TSVIOReenable.en.rst @@ -24,7 +24,9 @@ TSVIOReenable Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: void TSVIOReenable(TSVIO viop) diff --git a/doc/developer-guide/api/functions/TSVIOVConnGet.en.rst b/doc/developer-guide/api/functions/TSVIOVConnGet.en.rst index 21b27ec53d2..781304e32be 100644 --- a/doc/developer-guide/api/functions/TSVIOVConnGet.en.rst +++ b/doc/developer-guide/api/functions/TSVIOVConnGet.en.rst @@ -24,7 +24,9 @@ TSVIOVConnGet Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: TSVConn TSVIOVConnGet(TSVIO viop) diff --git a/doc/developer-guide/api/functions/TSfclose.en.rst b/doc/developer-guide/api/functions/TSfclose.en.rst index 47b3f857b10..ac49f90def5 100644 --- a/doc/developer-guide/api/functions/TSfclose.en.rst +++ b/doc/developer-guide/api/functions/TSfclose.en.rst @@ -24,7 +24,9 @@ TSfclose Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: void TSfclose(TSFile filep) diff --git a/doc/developer-guide/api/functions/TSfflush.en.rst b/doc/developer-guide/api/functions/TSfflush.en.rst index 0f73c130a02..68ff2310703 100644 --- a/doc/developer-guide/api/functions/TSfflush.en.rst +++ b/doc/developer-guide/api/functions/TSfflush.en.rst @@ -24,7 +24,9 @@ TSfflush Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: void TSfflush(TSFile filep) diff --git a/doc/developer-guide/api/functions/TSfgets.en.rst b/doc/developer-guide/api/functions/TSfgets.en.rst index 1a55ff05f8f..eaf0143e05f 100644 --- a/doc/developer-guide/api/functions/TSfgets.en.rst +++ b/doc/developer-guide/api/functions/TSfgets.en.rst @@ -24,7 +24,9 @@ TSfgets Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: char* TSfgets(TSFile filep, char * buf, size_t length) diff --git a/doc/developer-guide/api/functions/TSfopen.en.rst b/doc/developer-guide/api/functions/TSfopen.en.rst index 52e3f176048..7e45be7afc0 100644 --- a/doc/developer-guide/api/functions/TSfopen.en.rst +++ b/doc/developer-guide/api/functions/TSfopen.en.rst @@ -24,7 +24,9 @@ TSfopen Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: TSFile TSfopen(const char * filename, const char * mode) diff --git a/doc/developer-guide/api/functions/TSfread.en.rst b/doc/developer-guide/api/functions/TSfread.en.rst index 52b14211a09..1a87e24c8e8 100644 --- a/doc/developer-guide/api/functions/TSfread.en.rst +++ b/doc/developer-guide/api/functions/TSfread.en.rst @@ -24,7 +24,9 @@ TSfread Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: ssize_t TSfread(TSFile filep, void * buf, size_t length) diff --git a/doc/developer-guide/api/functions/TSfwrite.en.rst b/doc/developer-guide/api/functions/TSfwrite.en.rst index 3b064ace9db..f716b7e1e89 100644 --- a/doc/developer-guide/api/functions/TSfwrite.en.rst +++ b/doc/developer-guide/api/functions/TSfwrite.en.rst @@ -24,7 +24,9 @@ TSfwrite Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: ssize_t TSfwrite(TSFile filep, const void * buf, size_t length) diff --git a/doc/developer-guide/api/functions/TSmalloc.en.rst b/doc/developer-guide/api/functions/TSmalloc.en.rst index 79bc249504d..cbaf27568c5 100644 --- a/doc/developer-guide/api/functions/TSmalloc.en.rst +++ b/doc/developer-guide/api/functions/TSmalloc.en.rst @@ -27,7 +27,9 @@ Traffic Server memory allocation API. Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. function:: void * TSmalloc(size_t size) .. function:: void * TSrealloc(void * ptr , size_t size) diff --git a/doc/developer-guide/api/types/TSCacheDataType.en.rst b/doc/developer-guide/api/types/TSCacheDataType.en.rst index 39dfdd72f89..15f0f5746ec 100644 --- a/doc/developer-guide/api/types/TSCacheDataType.en.rst +++ b/doc/developer-guide/api/types/TSCacheDataType.en.rst @@ -22,7 +22,9 @@ TSCacheDataType Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. c:type:: TSCacheDataType diff --git a/doc/developer-guide/api/types/TSCacheError.en.rst b/doc/developer-guide/api/types/TSCacheError.en.rst index 3d1e22a7885..f7b83447fc7 100644 --- a/doc/developer-guide/api/types/TSCacheError.en.rst +++ b/doc/developer-guide/api/types/TSCacheError.en.rst @@ -22,7 +22,9 @@ TSCacheError Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. c:type:: TSCacheError diff --git a/doc/developer-guide/api/types/TSCacheLookupResult.en.rst b/doc/developer-guide/api/types/TSCacheLookupResult.en.rst index b6d364d26c4..92862a29a73 100644 --- a/doc/developer-guide/api/types/TSCacheLookupResult.en.rst +++ b/doc/developer-guide/api/types/TSCacheLookupResult.en.rst @@ -22,7 +22,9 @@ TSCacheLookupResult Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. c:type:: TSCacheLookupResult diff --git a/doc/developer-guide/api/types/TSCacheScanResult.en.rst b/doc/developer-guide/api/types/TSCacheScanResult.en.rst index 1e29c5688ba..302a1e09d8f 100644 --- a/doc/developer-guide/api/types/TSCacheScanResult.en.rst +++ b/doc/developer-guide/api/types/TSCacheScanResult.en.rst @@ -22,7 +22,9 @@ TSCacheScanResult Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. c:type:: TSCacheScanResult diff --git a/doc/developer-guide/api/types/TSEvent.en.rst b/doc/developer-guide/api/types/TSEvent.en.rst index 8261cfcef1d..b557a27a2fc 100644 --- a/doc/developer-guide/api/types/TSEvent.en.rst +++ b/doc/developer-guide/api/types/TSEvent.en.rst @@ -23,7 +23,9 @@ TSEvent Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. c:type:: TSEvent diff --git a/doc/developer-guide/api/types/TSFetchWakeUpOptions.en.rst b/doc/developer-guide/api/types/TSFetchWakeUpOptions.en.rst index b2e14cdb19e..17e78197530 100644 --- a/doc/developer-guide/api/types/TSFetchWakeUpOptions.en.rst +++ b/doc/developer-guide/api/types/TSFetchWakeUpOptions.en.rst @@ -22,7 +22,9 @@ TSFetchWakeUpOptions Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. c:type:: TSFetchWakeUpOptions diff --git a/doc/developer-guide/api/types/TSHttpHookID.en.rst b/doc/developer-guide/api/types/TSHttpHookID.en.rst index 663759581fc..02715b2b48a 100644 --- a/doc/developer-guide/api/types/TSHttpHookID.en.rst +++ b/doc/developer-guide/api/types/TSHttpHookID.en.rst @@ -24,7 +24,9 @@ TSHttpHookID Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. c:type:: TSHttpHookID diff --git a/doc/developer-guide/api/types/TSHttpStatus.en.rst b/doc/developer-guide/api/types/TSHttpStatus.en.rst index f68904967b9..7c4a524a8b5 100644 --- a/doc/developer-guide/api/types/TSHttpStatus.en.rst +++ b/doc/developer-guide/api/types/TSHttpStatus.en.rst @@ -24,7 +24,9 @@ TSHttpStatus Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. c:type:: TSHttpStatus diff --git a/doc/developer-guide/api/types/TSHttpType.en.rst b/doc/developer-guide/api/types/TSHttpType.en.rst index c8ddc9d9b10..f067cbee547 100644 --- a/doc/developer-guide/api/types/TSHttpType.en.rst +++ b/doc/developer-guide/api/types/TSHttpType.en.rst @@ -22,7 +22,9 @@ TSHttpType Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. c:type:: TSHttpType diff --git a/doc/developer-guide/api/types/TSIOBuffersSizeIndex.en.rst b/doc/developer-guide/api/types/TSIOBuffersSizeIndex.en.rst index 4b67491a46d..901045fd036 100644 --- a/doc/developer-guide/api/types/TSIOBuffersSizeIndex.en.rst +++ b/doc/developer-guide/api/types/TSIOBuffersSizeIndex.en.rst @@ -22,7 +22,9 @@ TSIOBuffersSizeIndex Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. c:type:: TSIOBuffersSizeIndex diff --git a/doc/developer-guide/api/types/TSLookingUpType.en.rst b/doc/developer-guide/api/types/TSLookingUpType.en.rst index 51ee7d298c9..d71c9a76d02 100644 --- a/doc/developer-guide/api/types/TSLookingUpType.en.rst +++ b/doc/developer-guide/api/types/TSLookingUpType.en.rst @@ -22,7 +22,9 @@ TSLookingUpType Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. c:type:: TSLookingUpType diff --git a/doc/developer-guide/api/types/TSOverridableConfigKey.en.rst b/doc/developer-guide/api/types/TSOverridableConfigKey.en.rst index 7eeb349b748..a17c791a15c 100644 --- a/doc/developer-guide/api/types/TSOverridableConfigKey.en.rst +++ b/doc/developer-guide/api/types/TSOverridableConfigKey.en.rst @@ -22,7 +22,9 @@ TSOverridableConfigKey Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. c:type:: TSOverridableConfigKey @@ -151,6 +153,7 @@ Enumeration Members .. c:macro:: TS_CONFIG_SSL_CLIENT_SNI_POLICY .. c:macro:: TS_CONFIG_SSL_CLIENT_PRIVATE_KEY_FILENAME .. c:macro:: TS_CONFIG_SSL_CLIENT_CA_CERT_FILENAME + .. c:macro:: TS_CONFIG_HTTP_HOST_RESOLUTION_PREFERENCE Description diff --git a/doc/developer-guide/api/types/TSParseResult.en.rst b/doc/developer-guide/api/types/TSParseResult.en.rst index 5fb321d4439..8777709c72d 100644 --- a/doc/developer-guide/api/types/TSParseResult.en.rst +++ b/doc/developer-guide/api/types/TSParseResult.en.rst @@ -22,7 +22,9 @@ TSParseResult Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. c:type:: TSParseResult diff --git a/doc/developer-guide/api/types/TSRecordAccessType.en.rst b/doc/developer-guide/api/types/TSRecordAccessType.en.rst index 84f71e19653..a6105ce8c67 100644 --- a/doc/developer-guide/api/types/TSRecordAccessType.en.rst +++ b/doc/developer-guide/api/types/TSRecordAccessType.en.rst @@ -22,7 +22,9 @@ TSRecordAccessType Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. c:type:: TSRecordAccessType diff --git a/doc/developer-guide/api/types/TSRecordCheckType.en.rst b/doc/developer-guide/api/types/TSRecordCheckType.en.rst index a6fc51f6e05..ad3967bb0b4 100644 --- a/doc/developer-guide/api/types/TSRecordCheckType.en.rst +++ b/doc/developer-guide/api/types/TSRecordCheckType.en.rst @@ -22,7 +22,9 @@ TSRecordCheckType Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. c:type:: TSRecordCheckType diff --git a/doc/developer-guide/api/types/TSRecordDataType.en.rst b/doc/developer-guide/api/types/TSRecordDataType.en.rst index 6ea96108aca..f2f09b1caa2 100644 --- a/doc/developer-guide/api/types/TSRecordDataType.en.rst +++ b/doc/developer-guide/api/types/TSRecordDataType.en.rst @@ -22,7 +22,9 @@ TSRecordDataType Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. c:type:: TSRecordDataType diff --git a/doc/developer-guide/api/types/TSRecordModeType.en.rst b/doc/developer-guide/api/types/TSRecordModeType.en.rst index cdfa6c32c4d..4e8c3e4fded 100644 --- a/doc/developer-guide/api/types/TSRecordModeType.en.rst +++ b/doc/developer-guide/api/types/TSRecordModeType.en.rst @@ -22,7 +22,9 @@ TSRecordModeType Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. c:type:: TSRecordModeType diff --git a/doc/developer-guide/api/types/TSRecordPersistType.en.rst b/doc/developer-guide/api/types/TSRecordPersistType.en.rst index 448e832dac6..076d7ff1760 100644 --- a/doc/developer-guide/api/types/TSRecordPersistType.en.rst +++ b/doc/developer-guide/api/types/TSRecordPersistType.en.rst @@ -22,7 +22,9 @@ TSRecordPersistType Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. c:type:: TSRecordPersistType diff --git a/doc/developer-guide/api/types/TSRecordType.en.rst b/doc/developer-guide/api/types/TSRecordType.en.rst index bb8e0247a33..cb72926b5ed 100644 --- a/doc/developer-guide/api/types/TSRecordType.en.rst +++ b/doc/developer-guide/api/types/TSRecordType.en.rst @@ -22,7 +22,9 @@ TSRecordType Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. c:type:: TSRecordType diff --git a/doc/developer-guide/api/types/TSRecordUpdateType.en.rst b/doc/developer-guide/api/types/TSRecordUpdateType.en.rst index 53ce764a403..af6c1c2d23a 100644 --- a/doc/developer-guide/api/types/TSRecordUpdateType.en.rst +++ b/doc/developer-guide/api/types/TSRecordUpdateType.en.rst @@ -22,7 +22,9 @@ TSRecordUpdateType Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. c:type:: TSRecordUpdateType diff --git a/doc/developer-guide/api/types/TSReturnCode.en.rst b/doc/developer-guide/api/types/TSReturnCode.en.rst index 5885fc3eae0..ac37ab5e7f8 100644 --- a/doc/developer-guide/api/types/TSReturnCode.en.rst +++ b/doc/developer-guide/api/types/TSReturnCode.en.rst @@ -22,7 +22,9 @@ TSReturnCode Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. c:type:: TSReturnCode diff --git a/doc/developer-guide/api/types/TSSDKVersion.en.rst b/doc/developer-guide/api/types/TSSDKVersion.en.rst index 2fa3b0e698b..323e911a383 100644 --- a/doc/developer-guide/api/types/TSSDKVersion.en.rst +++ b/doc/developer-guide/api/types/TSSDKVersion.en.rst @@ -22,7 +22,9 @@ TSSDKVersion Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. c:type:: TSSDKVersion diff --git a/doc/developer-guide/api/types/TSServerSessionSharingPoolType.en.rst b/doc/developer-guide/api/types/TSServerSessionSharingPoolType.en.rst deleted file mode 100644 index e21b88e38a3..00000000000 --- a/doc/developer-guide/api/types/TSServerSessionSharingPoolType.en.rst +++ /dev/null @@ -1,40 +0,0 @@ -.. Licensed to the Apache Software Foundation (ASF) under one or more - contributor license agreements. See the NOTICE file distributed - with this work for additional information regarding copyright - ownership. The ASF licenses this file to you under the Apache - License, Version 2.0 (the "License"); you may not use this file - except in compliance with the License. You may obtain a copy of - the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied. See the License for the specific language governing - permissions and limitations under the License. - -.. include:: ../../../common.defs - -TSServerSessionSharingPoolType -****************************** - -Synopsis -======== - -`#include ` - -.. c:type:: TSServerSessionSharingPoolType - -Enum typedef. - -Enumeration Members -=================== - -.. c:member:: TSServerSessionSharingPoolType TS_SERVER_SESSION_SHARING_POOL_GLOBAL - -.. c:member:: TSServerSessionSharingPoolType TS_SERVER_SESSION_SHARING_POOL_THREAD - -Description -=========== - diff --git a/doc/developer-guide/api/types/TSServerState.en.rst b/doc/developer-guide/api/types/TSServerState.en.rst index 6af8837df64..adf25901617 100644 --- a/doc/developer-guide/api/types/TSServerState.en.rst +++ b/doc/developer-guide/api/types/TSServerState.en.rst @@ -22,7 +22,9 @@ TSServerState Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. c:type:: TSServerState diff --git a/doc/developer-guide/api/types/TSSslSession.en.rst b/doc/developer-guide/api/types/TSSslSession.en.rst index 60a184f0778..74e6e0a9a46 100644 --- a/doc/developer-guide/api/types/TSSslSession.en.rst +++ b/doc/developer-guide/api/types/TSSslSession.en.rst @@ -24,7 +24,9 @@ TSSslSession Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. type:: TSSslSessionID diff --git a/doc/developer-guide/api/types/TSStatPeristence.en.rst b/doc/developer-guide/api/types/TSStatPeristence.en.rst index a8a75151c4b..7d383ab712c 100644 --- a/doc/developer-guide/api/types/TSStatPeristence.en.rst +++ b/doc/developer-guide/api/types/TSStatPeristence.en.rst @@ -22,7 +22,9 @@ TSStatPersistence Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. c:type:: TSStatPersistence @@ -42,4 +44,4 @@ Enumeration Members Description =========== -The level of persistence for a statistic value. \ No newline at end of file +The level of persistence for a statistic value. diff --git a/doc/developer-guide/api/types/TSStatSync.en.rst b/doc/developer-guide/api/types/TSStatSync.en.rst index 9b4b51e2839..028944388e5 100644 --- a/doc/developer-guide/api/types/TSStatSync.en.rst +++ b/doc/developer-guide/api/types/TSStatSync.en.rst @@ -22,7 +22,9 @@ TSStatSync Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. c:type:: TSStatSync @@ -33,11 +35,11 @@ Enumeration Members .. c:member:: TSStatSync TS_STAT_SYNC_SUM - Values should add be summed. + This stat sync type should be used for gauge metrics (i.e can increase or decrease with time). It may be manipulated using TSStatIntIncrement, TSStatIntDecrement, TSStatIntSet. E.g for counting number of available origin-servers or number of active threads. .. c:member:: TSStatSync TS_STAT_SYNC_COUNT - Values should be added together. + This stat sync type should be used for counter metrics (i.e it should only increase with time). It should only be manipulated using TSStatIntIncrement. E.g for tracking call counts or uptime. .. c:member:: TSStatSync TS_STAT_SYNC_AVG @@ -50,4 +52,4 @@ Enumeration Members Description =========== -The level of persistence for a statistic value. \ No newline at end of file +The level of persistence for a statistic value. diff --git a/doc/developer-guide/api/types/TSThreadPool.en.rst b/doc/developer-guide/api/types/TSThreadPool.en.rst index 298a8505b15..d9fb943e37e 100644 --- a/doc/developer-guide/api/types/TSThreadPool.en.rst +++ b/doc/developer-guide/api/types/TSThreadPool.en.rst @@ -22,7 +22,9 @@ TSThreadPool Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. c:type:: TSThreadPool @@ -35,16 +37,10 @@ Enumeration Members .. c:member:: TSThreadPool TS_THREAD_POOL_TASK -.. c:member:: TSThreadPool TS_THREAD_POOL_SSL - .. c:member:: TSThreadPool TS_THREAD_POOL_DNS -.. c:member:: TSThreadPool TS_THREAD_POOL_REMAP - -.. c:member:: TSThreadPool TS_THREAD_POOL_CLUSTER .. c:member:: TSThreadPool TS_THREAD_POOL_UDP Description =========== - diff --git a/doc/developer-guide/api/types/TSUuid.en.rst b/doc/developer-guide/api/types/TSUuid.en.rst index abf5044558f..a810bd5cdf9 100644 --- a/doc/developer-guide/api/types/TSUuid.en.rst +++ b/doc/developer-guide/api/types/TSUuid.en.rst @@ -22,7 +22,9 @@ TSUuid Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. c:type:: TSUuid diff --git a/doc/developer-guide/api/types/TSVConnCloseFlags.en.rst b/doc/developer-guide/api/types/TSVConnCloseFlags.en.rst index fff34731f01..7045a7af32d 100644 --- a/doc/developer-guide/api/types/TSVConnCloseFlags.en.rst +++ b/doc/developer-guide/api/types/TSVConnCloseFlags.en.rst @@ -22,7 +22,9 @@ TSVConnCloseFlags Synopsis ======== -`#include ` +.. code-block:: cpp + + #include .. c:type:: TSVConnCloseFlags diff --git a/doc/developer-guide/cache-architecture/architecture.en.rst b/doc/developer-guide/cache-architecture/architecture.en.rst index 0b03e7e1973..2bc34635424 100644 --- a/doc/developer-guide/cache-architecture/architecture.en.rst +++ b/doc/developer-guide/cache-architecture/architecture.en.rst @@ -95,8 +95,8 @@ assigned to a stripe (and in turn to a cache volume) automatically based on a hash of the URI used to retrieve the object from the :term:`origin server`. It is possible to configure this to a limited extent in :file:`hosting.config`, which supports content from specific hosts or domain to be stored on specific -cache volumes. As of version 4.0.1 it is also possible to control which cache -spans (and hence, which cache stripes) are contained in a specific cache volume. +cache volumes. It's also possible to control which cache spans (and hence, +which cache stripes) are contained in a specific cache volume. The layout and structure of the cache spans, the cache volumes, and the cache stripes that compose them are derived entirely from :file:`storage.config` and @@ -109,7 +109,7 @@ Span Structure Each cache span is marked at the front with a span header of type :class:`DiskHeader`. Each span is divided in to *span blocks*. These can be thought of similarly to normal disk partitions, marking -out blocks of storage. Span blocks can v ary in size, subject only to being a multiple of the +out blocks of storage. Span blocks can vary in size, subject only to being a multiple of the *volume block size* which is currently 128MB and, of course, being no larger than the span. The relationship between a span block and a cache stripe is the same as between a disk partition and a file system. A cache stripe is structured data contained in a span block and always occupies the @@ -990,7 +990,7 @@ evacuated. There are two types of evacuations: *reader based* and *forced*. The ``EvacuationBlock`` has a reader count to track this. If the reader count is -zero, then it is a forced evacuation and the the target, if it exists, will be +zero, then it is a forced evacuation and the target, if it exists, will be evacuated when the write cursor gets close. If the reader value is non-zero then it is a count of entities that are currently expecting to be able to read the object. Readers increment the count when they require read access to the diff --git a/doc/developer-guide/cache-architecture/data-structures.en.rst b/doc/developer-guide/cache-architecture/data-structures.en.rst index 3974eb7ac9a..d360c1d2e2c 100644 --- a/doc/developer-guide/cache-architecture/data-structures.en.rst +++ b/doc/developer-guide/cache-architecture/data-structures.en.rst @@ -160,7 +160,7 @@ Data Structures .. member:: DLL evacuate - Array of of :class:`EvacuationBlock` buckets. This is sized so there + Array of :class:`EvacuationBlock` buckets. This is sized so there is one bucket for every evacuation span. .. member:: off_t len diff --git a/doc/developer-guide/client-session-architecture.en.rst b/doc/developer-guide/client-session-architecture.en.rst index fb32a81a0ef..22e0c0f174e 100644 --- a/doc/developer-guide/client-session-architecture.en.rst +++ b/doc/developer-guide/client-session-architecture.en.rst @@ -80,7 +80,7 @@ When the Http1Transaction object is instantiated via :code:`ProxyTransaction::ne new HttpSM object, initializes it, and calls :code:`HttpSM::attach_client_session()` to associate the Http1Transaction object with the new HttpSM. -The ProxyTransaction object refers to the HttpSM via the current_reader member variable. The HttpSM object +The ProxyTransaction object refers to the HttpSM via the _sm member variable. The HttpSM object refers to ProxyTransaction via the ua_session member variable (session in the member name is historical because the HttpSM used to refer directly to the ClientSession object). @@ -93,7 +93,7 @@ HTTP/2 Objects This diagram shows the relationships between objects created as part of a HTTP/2 session. It is very similar to the HTTP/1.x case. The Http2ClientSession object interacts with the NetVC. The Http2Stream object creates -a HttpSM object object when :code:`ProxyClient::new_transaction()` is called. +a HttpSM object when :code:`ProxyClient::new_transaction()` is called. One difference is that the Http/2 protocol allows for multiple simultaneous transactions, so the Http2ClientSession object must be able to manage multiple streams. From the HttpSM perspective it is interacting with a diff --git a/doc/developer-guide/config-vars.en.rst b/doc/developer-guide/config-vars.en.rst index 11fd9c8c17f..a234a0e5322 100644 --- a/doc/developer-guide/config-vars.en.rst +++ b/doc/developer-guide/config-vars.en.rst @@ -33,11 +33,15 @@ .. |InkAPI.cc| replace:: ``InkAPI.cc`` -.. _InkAPI.cc: https://github.com/apache/trafficserver/blob/master/proxy/api/InkAPI.cc +.. _InkAPI.cc: https://github.com/apache/trafficserver/blob/master/src/traffic_server/InkAPI.cc .. |InkAPITest.cc| replace:: ``InkAPITest.cc`` -.. _InkAPITest.cc: https://github.com/apache/trafficserver/blob/master/proxy/api/InkAPITest.cc +.. _InkAPITest.cc: https://github.com/apache/trafficserver/blob/master/src/traffic_server/InkAPITest.cc + +.. |overridable_txn_vars.cc| replace:: ``overridable_txn_vars.cc`` + +.. _overridable_txn_vars.cc: https://github.com/apache/trafficserver/blob/master/src/shared/overridable_txn_vars.cc .. |ts_lua_http_config.c| replace:: ``ts_lua_http_config.c`` @@ -308,13 +312,10 @@ required for generic access: #. Add a value to the ``TSOverridableConfigKey`` enumeration in |apidefs.h.in|_. -#. Augment the ``TSHttpTxnConfigFind`` function to return this enumeration value - when given the name of the configuration variable. Be sure to count the - characters very carefully. +#. Augment ``Overridable_Map`` in |overridable_txn_vars.cc|_ to include configuration variable. -#. Augment the ``_conf_to_memberp`` function in |InkAPI.cc|_ to return a pointer - to the appropriate member of ``OverridableHttpConfigParams`` and set the type - if not a byte value. +#. Update the function ``_conf_to_memberp`` in |InkAPI.cc|_ to have a case for the enumeration value + in ``TSOverridableConfigKey``. #. Update the testing logic in |InkAPITest.cc|_ by adding the string name of the configuration variable to the ``SDK_Overridable_Configs`` array. diff --git a/doc/developer-guide/core-architecture/heap.en.rst b/doc/developer-guide/core-architecture/heap.en.rst index 31bee10df8f..2f0140a123b 100644 --- a/doc/developer-guide/core-architecture/heap.en.rst +++ b/doc/developer-guide/core-architecture/heap.en.rst @@ -150,7 +150,7 @@ Implementation String Coalescence ------------------ -String heaps do do not maintain lists of internal free space. Strings that are released are left in +String heaps do not maintain lists of internal free space. Strings that are released are left in place, creating dead space in the heap. For this reason it can become necessary to do a garbage collection operation on the writeable string heap in the header heap by calling :func:`HdrHeap::coalesce_str_heaps`. This is done when diff --git a/doc/developer-guide/debugging/debug-builds.en.rst b/doc/developer-guide/debugging/debug-builds.en.rst index 53ad22c9769..099312bf7e9 100644 --- a/doc/developer-guide/debugging/debug-builds.en.rst +++ b/doc/developer-guide/debugging/debug-builds.en.rst @@ -41,3 +41,22 @@ Debugging Tips: - Use assertions in your plugin (:c:func:`TSAssert` and :c:func:`TSReleaseAssert`). +SystemTap and DTrace support +**************************** + +Traffic Server can be instrumented with **SystemTap** on Linux systems, or +**DTrace** on \*BSDs and macOS. In order to use such tools, Traffic Server needs +to be built with ``-g``, or the debug symbols need to be installed. On Debian +systems, install the ``trafficserver-dbgsym`` package to install the debug +symbols. + +In addition to the normal probe points that can be used with SystemTap and +DTrace, such as function calls and specific statements, Traffic Server does +provide SDT markers at various interesting code paths. + +Pass the ``--enable-systemtap`` flag to ``./configure`` in order to build +Traffic Server with dtrace style markers (SDT). On Traffic Server builds with +SDT markers enabled, you can list the available markers with ``stap -L +'process("/path/to/traffic_server").mark("*")``. + +See the `SystemTap documentation `_ and the `DTrace guide `_ for more information. diff --git a/doc/developer-guide/debugging/memory-leaks.en.rst b/doc/developer-guide/debugging/memory-leaks.en.rst index 64a8b784847..23d451626ba 100644 --- a/doc/developer-guide/debugging/memory-leaks.en.rst +++ b/doc/developer-guide/debugging/memory-leaks.en.rst @@ -33,3 +33,19 @@ related to memory - you can use memory dump information. Enable This causes Traffic Server to dump memory information to ``traffic.out`` at ```` (intervals are in seconds). A zero value means that it is disabled. + +:: + + CONFIG proxy.config.res_track_memory INT + + When enabled makes Traffic Server track memory usage (allocations and releases). This + information is dumped to ``traffic.out`` when the user sends a SIGUSR1 signal or + periodically when :ts:cv:`proxy.config.dump_mem_info_frequency` is enabled. + + ===== ====================================================================== + Value Description + ===== ====================================================================== + ``0`` Memory tracking Disabled + ``1`` Tracks IO Buffer Memory allocations and releases + ``2`` Tracks IO Buffer Memory and OpenSSL Memory allocations and releases + ===== ====================================================================== diff --git a/doc/developer-guide/design-documents/reloading-plugins.en.rst b/doc/developer-guide/design-documents/reloading-plugins.en.rst index 23f96e04b0b..14e7aec95ae 100644 --- a/doc/developer-guide/design-documents/reloading-plugins.en.rst +++ b/doc/developer-guide/design-documents/reloading-plugins.en.rst @@ -22,26 +22,28 @@ Reloading Plugins ***************** -Reloading plugin allows new versions of a plugin code to be loaded and executed and old versions to be unloaded without +The reload plugin mechanism allows new versions of plugin code to be loaded and executed and old versions to be unloaded without restarting the |TS| process. -Plugins are Dynamic Shared Objects (DSO), new versions of the plugins are currently loaded by using a |TS| +Plugins are Dynamic Shared Objects (DSO). New versions of the plugins are currently loaded by using a |TS| configuration reload, i.e.:: traffic_ctl config reload -Although this feature should be transparent to there plugin developers, the following are some design considerations -and implementation details. +This feature is enabled by default. It can be turned off by setting the configuration variable :ts:cv:`proxy.config.plugin.dynamic_reload_mode` to ``0`` in :file:`records.config`. When the feature is turned off, once ATS is started one will be able load only one version of a plugin, and re-loading the same plugin would do nothing. + +Although plugin reloading should be transparent to plugin developers, the following are some design considerations +and implementation details for this feature. Design Considerations ===================== -1. The mechanism of the plugin reload should be transparent to the plugin developers, plugin developers should be +1. The mechanism of the plugin reload should be transparent to the plugin developers. Plugin developers should be concerned only with properly initializing and cleaning up after the plugin and its instances. -2. With the current |TS| implementation new version plugin (re)load is only triggered by a configuration - (re)load hence naturally the configuration should be always coupled with the set of plugins it loaded. +2. With the current |TS| implementation, new version plugin (re)load is only triggered by a configuration + (re)load. Hence the configuration should always be coupled with the set of plugins it loaded. 3. Due to its asynchronous nature, |TS| should allow running different newer and older versions of the same plugin at the same time. @@ -50,7 +52,7 @@ Design Considerations 5. Running different versions of the configuration and plugin versions at the same time requires maintaining a "current active set" to be used by new transactions, new continuations, etc. and also multiple "previous inactive" sets as well. -6. The result of the plugin reloading should be consistent across operating systems, file systems, dynamic loader +6. The result of the plugin reloading should be consistent across operating systems, file systems, and dynamic loader implementations. @@ -61,10 +63,10 @@ Currently only loading of "remap" plugins (`remap.config`) is supported but (re) Consistent (re)loading behavior ------------------------------- -The following are some of the problems noticed during the initial experimentation: +The following are some of the problems noticed during initial experimentation: a. There is an internal reference counting of the DSOs implemented inside the dynamic loader. - If an older version of the plugin DSO is still loaded then loading of a newer version of the DSO by using + If an older version of the plugin DSO is still loaded, then loading of a newer version of the DSO by using the same filename does not load the new version. b. If the filename used by the dynamic loader reference counting contains symbolic links the results are not @@ -79,22 +81,22 @@ The following possible solutions were considered: good results but it was not available on all supported platforms -A less efficient but more reliable solution was chosen - DSO files are temporarily copied to and consequently -loaded from a runtime location and the copies is kept until plugin is unloaded. +A less efficient but more reliable solution was chosen: DSO files are temporarily copied to and consequently +loaded from a runtime location and each copy is kept until the corresponding plugin is unloaded. -Each configuration / plugin reload would use a different runtime location, ``ATSUuid`` is used to create unique +Each configuration / plugin reload would use a different runtime location with ``ATSUuid`` being used to create unique runtime directories. Reference counting against DSOs ------------------------------- -During the initial analysis a common sense solution was considered - to add instrumentation around handling +During the initial analysis a common sense solution was considered: add instrumentation around handling of registered hooks in order to unload plugins safely. This would be more involved and not sufficient since hooks are not the only mechanism that relies on the plugin DSO being loaded. This design / implementation proposes a different approach. -Plugin code can be called from HTTP state machine (1) while handling HTTP transactions or (2) while calling +Plugin code can be called from the HTTP state machine (1) while handling HTTP transactions or (2) while calling event handling functions of continuations created by the plugin code. The plugin reload mechanism should guarantee that all necessary plugin DSOs are still loaded when those calls are performed. @@ -103,7 +105,6 @@ Those continuations are created by :c:func:`TSContCreate` and :c:func:`TSVConnCr could be used for registering hooks (i.e. registered by :c:func:`TSHttpHookAdd`) or for scheduling events in the future (i.e. :c:func:`TSContScheduleOnPool`). - Registering hooks always requires creating continuations from inside the plugin code and a separate instrumentation around handling of hooks is not necessary. @@ -117,38 +118,38 @@ Plugin context -------------- Reference counting and managing different configuration and plugin sets require the continuation creation and -destruction to know in which plugin context they are running. +destruction logic to know in which plugin context they are running. -Traffic server API change was considered for ``TSCreateCont``, ``TSVConnCreate`` and ``TSDestroyCont`` but +A |TS| API change was considered for ``TSCreateCont``, ``TSVConnCreate``, and ``TSDestroyCont``, but it was decided to keep things hidden from the plugin developer by using thread local plugin context which would be set/switched accordingly before executing the plugin code. -The continuations created by the plugin will have a context member added to them which will be used by -the reference counting and when continuations are destroyed or handle events. +The continuations created by the plugin will have a context member added to them which will be used for +reference counting, when continuations are destroyed, and to handle events. -TSHttpArgs +TSUserArgs ---------- -|TS| sessions and transactions provide a fixed array of void pointers that can be used by plugins -to store information. To avoid collisions between plugins a plugin should first *reserve* an index in the array. +|TS| sessions, transactions, virtual connections and globally provide a fixed array of void pointers that +can be used by plugins to store information. To avoid collisions between plugins a plugin should first +*reserve* an index in the array. -Since :c:func:`TSHttpTxnArgIndexReserve` and :c:func:`TSHttpSsnArgIndexReserve` are meant to be called during plugin -initialization we could end up "leaking" indices during plugins reload. -Hence it is necessary to make sure only one index is allocated per "plugin identifying name", current -:c:func:`TSHttpTxnArgIndexNameLookup` and :c:func:`TSHttpTxnArgIndexNameLookup` implementation assumes 1-1 -index-to-name relationship as well. +Since :c:func:`TSUserArgIndexReserve` is meant to be called during plugin initialization we could end up +"leaking" indices during plugin reload. Hence it is necessary to make sure only one index is allocated per +"plugin identifying name", current :c:func:`TSUserArgIndexNameLookup` and +:c:func:`TSUserArgIndexLookup` implementation assumes 1-1 index-to-name relationship as well. PluginFactory ------------- -`PluginFactory` - creates and holds all plugin instances corresponding to a single configuration (re)load. +`PluginFactory` creates and holds all plugin instances corresponding to a single configuration (re)load. -#. Instantiates and initializes 'remap' plugins, eventually signals plugin unload/destruction, makes sure each plugin +#. Instantiates and initializes 'remap' plugins, eventually signals plugin unload/destruction, and makes sure each plugin version is loaded only once per configuration (re)load by maintaining a list of DSOs already loaded. -#. Initializes, keeps track of all resulting plugin instances and eventually signals each instance destruction. +#. Initializes and keeps track of all resulting plugin instances and eventually signals each instance destruction. #. Handles multiple plugin search paths. @@ -160,7 +161,7 @@ PluginFactory RemapPluginInfo --------------- -`RemapPluginInfo` - a class representing a 'remap' plugin, derived from `PluginDso`, and handling 'remap' plugin specific +`RemapPluginInfo` is a class representing a 'remap' plugin. It is derived from `PluginDso`. It is responsible for handling 'remap' plugin specific initialization and destruction and also sets up the right plugin context when its methods are called. @@ -168,7 +169,7 @@ initialization and destruction and also sets up the right plugin context when it PluginDso --------- -`PluginDso` - a class performing the actual DSO loading and unloading and all related initialization and +`PluginDso` is a class performing the actual DSO loading and unloading and all related initialization and cleanup plus related error handling. Its functionality is modularized into a separate class in hopes to be reused by 'global' plugins in the future. @@ -176,3 +177,7 @@ be reused by 'global' plugins in the future. To make sure plugins are still loaded while their code is still in use there is reference counting done around ``PluginDso`` which inherits ``RefCountObj`` and implements ``acquire()`` and ``release()`` methods which are called by ``TSCreateCont``, ``TSVConnCreate`` and ``TSDestroyCont``. + +Other notes +----------- +When this feature for dynamic plugin reload is turned on (:ts:cv:`proxy.config.plugin.dynamic_reload_mode` is set to ``1``), there is one pitfall users should be aware of. Since "global" plugins do not support this feature while the "remap" plugins do, if a plugin is used as a global plugin as well as a remap plugin, there will be two different copies of the plugin loaded in memory with no state shared between them. diff --git a/doc/developer-guide/internal-libraries/Extendible.en.rst b/doc/developer-guide/internal-libraries/Extendible.en.rst index 26e03923657..8bb595e16bb 100644 --- a/doc/developer-guide/internal-libraries/Extendible.en.rst +++ b/doc/developer-guide/internal-libraries/Extendible.en.rst @@ -80,18 +80,19 @@ the type's constructor, destructor, and serializer. And to avoid corruption, the } -When an derived class is instantiated, :func:`template<> alloc()` will allocate a block of memory for the derived class and all added -fields. The only memory overhead per instance is an uint16 used as a offset to the start of the extendible block. +When an derived class is instantiated, :func:`template<> create()` will allocate a block of memory for the derived class and all added +fields. The only memory overhead per instance is an uint16 used as a offset to the start of the extendible block. Then the constructor of the class +is called, followed by the constructors of each extendible field. .. code-block:: cpp ExtendibleExample* alloc_example() { - return ext::alloc(); + return ext::create(); } Memory Layout ------------- -One block of memory is allocated per |Extendible|, which included all member variables and extended fields. +One block of memory is allocated per |Extendible|, which include all member variables and extended fields. Within the block, memory is arranged in the following order: #. Derived members (+padding align next field) @@ -129,8 +130,8 @@ which simplifies the code using it. Also this provides compile errors for common } PluginFunc() { - Food *banana = ext::alloc(); - Car *camry = ext::alloc(); + Food *banana = ext::create(); + Car *camry = ext::create(); // Common user errors @@ -140,11 +141,11 @@ which simplifies the code using it. Also this provides compile errors for common float speed = ext::get(banana,fld_max_speed); // ^^^^^^^^^^^^^ - // Compile error: Cannot downcast banana to type Extendible + // Compile error: Cannot convert banana to type Extendible float weight = ext::get(camry,fld_food_weight); // ^^^^^^^^^^^^^^^ - // Compile error: Cannot downcast camry to type Extendible, even though Car and Food each have a 'weight' field, the FieldId is strongly typed. + // Compile error: Cannot convert camry to type Extendible, even though Car and Food each have a 'weight' field, the FieldId is strongly typed. } @@ -157,20 +158,20 @@ which simplifies the code using it. Also this provides compile errors for common Fruit.schema.addField(fld_has_seeds, "has_seeds"); - Fruit mango = ext::alloc(); + Fruit mango = ext::create(); - ext::set(mango, fld_has_seeds) = true; // downcasts mango to Extendible - ext::set(mango, fld_food_weight) = 2; // downcasts mango to Extendible + ext::set(mango, fld_has_seeds) = true; // converts mango to Extendible + ext::set(mango, fld_food_weight) = 2; // converts mango to Extendible ext::set(mango, fld_max_speed) = 9; // ^^^^^^^^^^^^^ - // Compile error: Cannot downcast mango to type Extendible + // Compile error: Cannot convert mango to type Extendible Inheritance ----------- Unfortunately it is non-trivial handle multiple |Extendible| super types in the same inheritance tree. - :func:`template<> alloc()` handles allocation and initialization of the entire `Derived` class, but it is dependent on each class defining :code:`using super_type = *some_super_class*;` so that it recurse through the classes. + :func:`template<> create()` handles allocation and initialization of the entire `Derived` class, but it is dependent on each class defining :code:`using super_type = *some_super_class*;` so that it recurse through the classes. .. code-block:: cpp @@ -191,7 +192,7 @@ Inheritance ext::FieldId> ext_a_1; ext::FieldId ext_c_1; - C &x = *(ext::alloc()); + C &x = *(ext::create()); ext::viewFormat(x); :func:`viewFormat` prints a diagram of the position and size of bytes used within the allocated @@ -235,8 +236,9 @@ Namespace `ext` one schema instance per |Extendible| to define contained |FieldDesc| -.. function:: template Extendible* alloc() +.. function:: template Extendible* create() + To be used in place of `new Derived_t()`. Allocate a block of memory. Construct the base data. Recursively construct and initialize `Derived_t::super_type` and its |Extendible| classes. diff --git a/doc/developer-guide/internal-libraries/MemSpan.en.rst b/doc/developer-guide/internal-libraries/MemSpan.en.rst index a9b6775e2ac..e832c80bb37 100644 --- a/doc/developer-guide/internal-libraries/MemSpan.en.rst +++ b/doc/developer-guide/internal-libraries/MemSpan.en.rst @@ -25,7 +25,9 @@ MemSpan Synopsis ======== -:code:`#include ` +.. code-block:: cpp + + #include :class:`MemSpan` is a view on a contiguous section of writeable memory. A view does not own the memory and neither allocates nor de-allocates. The memory in the view is always owned by some other container diff --git a/doc/developer-guide/internal-libraries/TextView.en.rst b/doc/developer-guide/internal-libraries/TextView.en.rst index da4849d7f62..0951ef65691 100644 --- a/doc/developer-guide/internal-libraries/TextView.en.rst +++ b/doc/developer-guide/internal-libraries/TextView.en.rst @@ -25,7 +25,9 @@ TextView Synopsis ======== -:code:`#include ` +.. code-block:: cpp + + #include ` .. class:: TextView @@ -149,8 +151,8 @@ History The first attempt at this functionality was in the TSConfig library in the :code:`ts::Buffer` and :code:`ts::ConstBuffer` classes. Originally intended just as raw memory views, :code:`ts::ConstBuffer` in particular was repeated enhanced to provide better support for strings. -The header was eventually moved from :literal:`lib/tsconfig` to :literal:`lib/ts` and was used in in -various part of the |TS| core. +The header was eventually moved from :literal:`lib/tsconfig` to :literal:`lib/ts` and was used in +various parts of the |TS| core. There was then a proposal to make these classes available to plugin writers as they proved handy in the core. A suggested alternative was `Boost.StringRef diff --git a/doc/developer-guide/internal-libraries/scalar.en.rst b/doc/developer-guide/internal-libraries/scalar.en.rst index f1f4cca4a9e..185fe06f3d6 100644 --- a/doc/developer-guide/internal-libraries/scalar.en.rst +++ b/doc/developer-guide/internal-libraries/scalar.en.rst @@ -255,10 +255,10 @@ the library user. The increment / decrement and compound assignment operators we similar to pointer arithmetic to be unsurprising in this context. This was further influenced by the fact that, in general, these operators are useless in the value context. E.g. if a scalar has a scale greater than 1 (the common case) then increment and decrement of the value is always a null -operation. Once those operators are used on the count is is least surprising that the compound +operation. Once those operators are used on the count is least surprising that the compound operators act in the same way. The next step, to arithmetic operators, is not so clear and so those require explicit scale indicators, such as :code:`round_down` or explicit constructors. It was a design goal to avoid, as much as possible, the requirement that the library user keep track of the scale of specific variables. This has proved very useful in practice, but at the same time when -doing arithmetic is is almost always the case that either the values are both scalars (making the +doing arithmetic is almost always the case that either the values are both scalars (making the arithmetic unambiguous) or the scale of the literal is known (e.g., "add 6 kilobytes"). diff --git a/doc/developer-guide/introduction/index.en.rst b/doc/developer-guide/introduction/index.en.rst index 19c43814efb..0cfdc140db2 100644 --- a/doc/developer-guide/introduction/index.en.rst +++ b/doc/developer-guide/introduction/index.en.rst @@ -53,7 +53,7 @@ Below is a section-by-section breakdown of this guide: How to compile and load plugins. Walks through a simple "hello world" example; explains how to initialize and register plugins. Basic structures that all plugins use: events, continuations, and how to hook on to |TS| - processes. Detailed explication of a sample blacklisting plugin. + processes. Detailed explication of a sample denylisting plugin. :ref:`developer-plugins-examples-query-remap` This chapter demonstrates on a practical example how you can @@ -61,7 +61,7 @@ Below is a section-by-section breakdown of this guide: :ref:`developer-plugins-header-based-examples` Detailed explanation about writing plugins that work on HTTP - headers; discusses sample blacklisting and basic authorization + headers; discusses sample denylisting and basic authorization plugins. :ref:`developer-plugins-http-transformations` diff --git a/doc/developer-guide/layout/runroot.en.rst b/doc/developer-guide/layout/runroot.en.rst index 7b9c049d169..07c73bf9391 100644 --- a/doc/developer-guide/layout/runroot.en.rst +++ b/doc/developer-guide/layout/runroot.en.rst @@ -49,7 +49,7 @@ Work flow: #. Look in current directory and look up N (default 4) directories for ``runroot.yaml`` #. Look in executable directory and look up N directories for ``runroot.yaml``. #. ``$TS_ROOT`` Environment Variable -#. Compiler defaults in layout class +#. Compiler defaults in layout class Right now, the following programs are integrated with the runroot logic: **traffic_server**, **traffic_manager**, **traffic_ctl**, **traffic_layout**, **traffic_crashlog**, **traffic_logcat**, **traffic_logstat**. diff --git a/doc/developer-guide/logging-architecture/architecture.en.rst b/doc/developer-guide/logging-architecture/architecture.en.rst index c444aa290f1..641908b96f9 100644 --- a/doc/developer-guide/logging-architecture/architecture.en.rst +++ b/doc/developer-guide/logging-architecture/architecture.en.rst @@ -101,11 +101,11 @@ LogBuffer --------- The ``LogBuffer`` class is designed to provide a thread-safe mechanism -to buffer/store log entries before they’re flushed. To reduce system call +to buffer/store log entries before they're flushed. To reduce system call overhead, ``LogBuffer``\ s are designed to avoid heavy-weight mutexes in favor of using lightweight atomics built on top of compare-and-swap operations. When a caller wants to write into a ``LogBuffer``, the -caller “checks out” a segment of the buffer to write into. ``LogBuffer`` +caller "checks out" a segment of the buffer to write into. ``LogBuffer`` makes sure that no two callers are served overlapping segments. To illustrate this point, consider this diagram of a buffer: @@ -136,9 +136,9 @@ illustrate this point, consider this diagram of a buffer: | | +--------------------------------+ -In this manner, since no two threads are writing in the other’s segment, +In this manner, since no two threads are writing in the other's segment, we avoid race conditions during the actual logging. This also makes -LogBuffer’s critical section extremely small. In fact, the only time we +LogBuffer's critical section extremely small. In fact, the only time we need to enter a critical section is when we do the book keeping to keep track of which segments are checked out. Despite this, it's not unusual to see between 5% and 20% of total processor time spent inside ``LogBuffer`` diff --git a/doc/developer-guide/plugins/example-plugins/blacklist/accessing-the-transaction-being-processed.en.rst b/doc/developer-guide/plugins/example-plugins/denylist/accessing-the-transaction-being-processed.en.rst similarity index 91% rename from doc/developer-guide/plugins/example-plugins/blacklist/accessing-the-transaction-being-processed.en.rst rename to doc/developer-guide/plugins/example-plugins/denylist/accessing-the-transaction-being-processed.en.rst index 46d2a54a41d..9742418be6f 100644 --- a/doc/developer-guide/plugins/example-plugins/blacklist/accessing-the-transaction-being-processed.en.rst +++ b/doc/developer-guide/plugins/example-plugins/denylist/accessing-the-transaction-being-processed.en.rst @@ -17,7 +17,7 @@ .. include:: ../../../../common.defs -.. _developer-plugins-blacklist-access-process-txn: +.. _developer-plugins-denylist-access-process-txn: Accessing the Transaction Being Processed ***************************************** @@ -38,12 +38,12 @@ The key here is that if the event is an HTTP transaction event, then the data passed to the continuation's handler is of type ``TSHttpTxn`` (a data type that represents HTTP transactions). Your plugin can then do things with the transaction. Here's how it looks in the code for the -Blacklist plugin's handler: +Denylist plugin's handler: .. code-block:: c static int - blacklist_plugin (TSCont contp, TSEvent event, void *edata) + denylist_plugin (TSCont contp, TSEvent event, void *edata) { TSHttpTxn txnp = (TSHttpTxn) edata; switch (event) { @@ -60,5 +60,5 @@ Blacklist plugin's handler: } For example: when the origin server DNS lookup event is sent, -``blacklist_plugin`` can call ``handle_dns``\ and pass ``txnp`` as an +``denylist_plugin`` can call ``handle_dns``\ and pass ``txnp`` as an argument. diff --git a/doc/developer-guide/plugins/example-plugins/blacklist/index.en.rst b/doc/developer-guide/plugins/example-plugins/denylist/index.en.rst similarity index 75% rename from doc/developer-guide/plugins/example-plugins/blacklist/index.en.rst rename to doc/developer-guide/plugins/example-plugins/denylist/index.en.rst index b88c3f288fc..e45ed39f100 100644 --- a/doc/developer-guide/plugins/example-plugins/blacklist/index.en.rst +++ b/doc/developer-guide/plugins/example-plugins/denylist/index.en.rst @@ -17,21 +17,21 @@ .. include:: ../../../../common.defs -.. _developer-plugins-examples-blacklist: +.. _developer-plugins-examples-denylist: -Blacklist Plugin +Denylist Plugin **************** -The sample blacklisting plugin included in the Traffic Server SDK is -``blacklist_1.c``. This plugin checks every incoming HTTP client request -against a list of blacklisted web sites. If the client requests a -blacklisted site, then the plugin returns an ``Access forbidden`` +The sample denylisting plugin included in the Traffic Server SDK is +``denylist_1.c``. This plugin checks every incoming HTTP client request +against a list of listed web sites. If the client requests a +listed site, then the plugin returns an ``Access forbidden`` message to the client. -The flow of HTTP processing with the blacklist plugin is illustrated in -the figure titled :ref:`BlackListPlugin`. +The flow of HTTP processing with the denylist plugin is illustrated in +the figure titled :ref:`DenyListPlugin`. This example also contains a simple configuration management interface. -It can read a list of blacklisted sites from a file (``blacklist.txt``) +It can read a list of sites from a file (``denylist.txt``) that can be updated by a Traffic Server administrator. When the configuration file is updated, Traffic Server sends an event to the plugin that wakes it up to do some work. @@ -52,7 +52,7 @@ Traffic Server has a multi-threaded design, race conditions can occur if several threads try to access the same continuation's data. Here is how the static parent continuation is created in -``blacklist_1.c``: +``denylist_1.c``: .. code-block:: c @@ -62,20 +62,20 @@ Here is how the static parent continuation is created in // ... TSCont contp; - contp = TSContCreate (blacklist_plugin, NULL); + contp = TSContCreate (denylist_plugin, NULL); // ... } -The handler function for the plugin is ``blacklist_plugin``, and the +The handler function for the plugin is ``denylist_plugin``, and the mutex is null. The continuation handler function's job is to handle the -events that are sent to it; accordingly, the ``blacklist_plugin`` +events that are sent to it; accordingly, the ``denylist_plugin`` routine consists of a switch statement that covers each of the events that might be sent to it: .. code-block:: c static int - blacklist_plugin (TSCont contp, TSEvent event, void *edata) + denylist_plugin (TSCont contp, TSEvent event, void *edata) { TSHttpTxn txnp = (TSHttpTxn) edata; switch (event) { @@ -86,7 +86,7 @@ that might be sent to it: handle_response (txnp); return 0; default: - TSDebug ("blacklist_plugin", "This event was unexpected: %d", ); + TSDebug ("denylist_plugin", "This event was unexpected: %d", ); break; } return 0; @@ -94,10 +94,10 @@ that might be sent to it: When you write handler functions, you have to anticipate any events that might be sent to the handler by hooks or by other functions. In the -Blacklist plugin, ``TS_EVENT_OS_DNS`` is sent because of the global hook +Denylist plugin, ``TS_EVENT_OS_DNS`` is sent because of the global hook established in ``TSPluginInit``, ``TS_EVENT_HTTP_SEND_RESPONSE_HDR`` is sent because the plugin contains a transaction hook -(see :ref:`developer-plugins-examples-blacklist-txn-hook`). +(see :ref:`developer-plugins-examples-denylist-txn-hook`). It is good practice to have a default case in your switch statements. .. toctree:: diff --git a/doc/developer-guide/plugins/example-plugins/blacklist/setting-a-global-hook.en.rst b/doc/developer-guide/plugins/example-plugins/denylist/setting-a-global-hook.en.rst similarity index 86% rename from doc/developer-guide/plugins/example-plugins/blacklist/setting-a-global-hook.en.rst rename to doc/developer-guide/plugins/example-plugins/denylist/setting-a-global-hook.en.rst index e471b1c5d2d..d095dcd2b19 100644 --- a/doc/developer-guide/plugins/example-plugins/blacklist/setting-a-global-hook.en.rst +++ b/doc/developer-guide/plugins/example-plugins/denylist/setting-a-global-hook.en.rst @@ -23,7 +23,7 @@ Setting a Global Hook Global hooks are always added in ``TSPluginInit`` using ``TSHttpHookAdd``. The two arguments of ``TSHttpHookAdd`` are the hook ID and the continuation to call when processing the event corresponding -to the hook. In ``blacklist_1.c``, the global hook is added as follows: +to the hook. In ``denylist_1.c``, the global hook is added as follows: .. code-block:: c @@ -32,7 +32,7 @@ to the hook. In ``blacklist_1.c``, the global hook is added as follows: Above, ``TS_HTTP_OS_DNS_HOOK`` is the ID for the origin server DNS lookup hook and ``contp`` is the parent continuation created earlier. -This means that the Blacklist plugin is called at every origin server -DNS lookup. When it is called, the handler functio ``blacklist_plugin`` +This means that the Denylist plugin is called at every origin server +DNS lookup. When it is called, the handler functio ``denylist_plugin`` receives ``TS_EVENT_HTTP_OS_DNS`` and calls ``handle_dns`` to see if the request is forbidden. diff --git a/doc/developer-guide/plugins/example-plugins/blacklist/setting-up-a-transaction-hook.en.rst b/doc/developer-guide/plugins/example-plugins/denylist/setting-up-a-transaction-hook.en.rst similarity index 82% rename from doc/developer-guide/plugins/example-plugins/blacklist/setting-up-a-transaction-hook.en.rst rename to doc/developer-guide/plugins/example-plugins/denylist/setting-up-a-transaction-hook.en.rst index 7f89d31c054..10ad8511d34 100644 --- a/doc/developer-guide/plugins/example-plugins/blacklist/setting-up-a-transaction-hook.en.rst +++ b/doc/developer-guide/plugins/example-plugins/denylist/setting-up-a-transaction-hook.en.rst @@ -17,16 +17,16 @@ .. include:: ../../../../common.defs -.. _developer-plugins-examples-blacklist-txn-hook: +.. _developer-plugins-examples-denylist-txn-hook: Setting Up a Transaction Hook ***************************** -The Blacklist plugin sends "access forbidden" messages to clients if -their requests are directed to blacklisted hosts. Therefore, the plugin +The Denylist plugin sends "access forbidden" messages to clients if +their requests are directed to listed hosts. Therefore, the plugin needs a transaction hook so it will be called back when Traffic Server's HTTP state machine reaches the "send response header" event. In the -Blacklist plugin's ``handle_dns`` routine, the transaction hook is added +Denylist plugin's ``handle_dns`` routine, the transaction hook is added as follows: .. code-block:: c @@ -34,7 +34,7 @@ as follows: TSMutexLock (sites_mutex); for (i = 0; i < nsites; i++) { if (strncmp (host, sites[i], host_length) == 0) { - printf ("blacklisting site: %s\n", sites[i]); + printf ("denylisting site: %s\n", sites[i]); TSHttpTxnHookAdd (txnp, TS_HTTP_SEND_RESPONSE_HDR_HOOK, contp); @@ -50,10 +50,10 @@ as follows: TSHttpTxnReenable (txnp, TS_EVENT_HTTP_CONTINUE); This code fragment shows some interesting features. The plugin is -comparing the requested site to the list of blacklisted sites. While the -plugin is using the blacklist, it must acquire the mutex lock for the -blacklist to prevent configuration changes in the middle of a -blacklisting operation. If the requested site is blacklisted, then the +comparing the requested site to the list of listed sites. While the +plugin is using the denylist, it must acquire the mutex lock for the +denylist to prevent configuration changes in the middle of a +denylisting operation. If the requested site is listed, then the following things happen: #. A transaction hook is added with ``TSHttpTxnHookAdd``; the plugin is @@ -66,7 +66,7 @@ following things happen: ``TS_EVENT_HTTP_ERROR`` as its event argument. Reenabling with an error event tells the HTTP state machine to stop the transaction and jump to the "send response header" state. Notice that if the - requested site is not blacklisted, then the transaction is reenabled + requested site is not listed, then the transaction is reenabled with the ``TS_EVENT_HTTP_CONTINUE`` event. #. The string and ``TSMLoc`` data stored in the marshal buffer ``bufp`` is diff --git a/doc/developer-guide/plugins/example-plugins/blacklist/source-code.en.rst b/doc/developer-guide/plugins/example-plugins/denylist/source-code.en.rst similarity index 72% rename from doc/developer-guide/plugins/example-plugins/blacklist/source-code.en.rst rename to doc/developer-guide/plugins/example-plugins/denylist/source-code.en.rst index 0be18042423..b6c828de348 100644 --- a/doc/developer-guide/plugins/example-plugins/blacklist/source-code.en.rst +++ b/doc/developer-guide/plugins/example-plugins/denylist/source-code.en.rst @@ -17,20 +17,20 @@ .. include:: ../../../../common.defs -.. _developer-plugins-examples-blacklist-code: +.. _developer-plugins-examples-denylist-code: Sample Source Code ****************** -.. _blacklist-1.c: +.. _denylist-1.c: -blacklist_1.c +denylist_1.c ------------- -The sample blacklisting plugin included in the Traffic Server SDK is -``blacklist_1.c``. This plugin checks every incoming HTTP client request -against a list of blacklisted web sites. If the client requests a -blacklisted site, then the plugin returns an ``Access forbidden`` +The sample denylisting plugin included in the Traffic Server SDK is +``denylist_1.c``. This plugin checks every incoming HTTP client request +against a list of web sites. If the client requests a +listed site, then the plugin returns an ``Access forbidden`` message to the client. This plugin illustrates: @@ -43,5 +43,5 @@ This plugin illustrates: - How to use the plugin configuration management interface -.. literalinclude:: ../../../../../example/plugins/c-api/blacklist_1/blacklist_1.c +.. literalinclude:: ../../../../../example/plugins/c-api/denylist_1/denylist_1.c :language: c diff --git a/doc/developer-guide/plugins/example-plugins/blacklist/working-with-http-header-functions.en.rst b/doc/developer-guide/plugins/example-plugins/denylist/working-with-http-header-functions.en.rst similarity index 86% rename from doc/developer-guide/plugins/example-plugins/blacklist/working-with-http-header-functions.en.rst rename to doc/developer-guide/plugins/example-plugins/denylist/working-with-http-header-functions.en.rst index 5d394606306..baa678f1da5 100644 --- a/doc/developer-guide/plugins/example-plugins/blacklist/working-with-http-header-functions.en.rst +++ b/doc/developer-guide/plugins/example-plugins/denylist/working-with-http-header-functions.en.rst @@ -17,12 +17,12 @@ .. include:: ../../../../common.defs -.. _developer-plugins-examples-blacklist-http-header-functions: +.. _developer-plugins-examples-denylist-http-header-functions: Working with HTTP Header Functions ********************************** -The Blacklist plugin examines the host header in every client +The Denylist plugin examines the host header in every client transaction. This is done in the ``handle_dns`` routine, using ``TSHttpTxnClientReqGet``, ``TSHttpHdrUrlGet``, and ``TSUrlHostGet``. @@ -39,19 +39,19 @@ transaction. This is done in the ``handle_dns`` routine, using int host_length; if (TSHttpTxnClientReqGet(txnp, &bufp, &hdr_loc) != TS_SUCCESS) { - TSError("[blacklist] Couldn't retrieve client request header"); + TSError("[denylist] Couldn't retrieve client request header"); goto done; } if (TSHttpHdrUrlGet(bufp, hdr_loc, &url_loc) != TS_SUCCESS) { - TSError("[blacklist] Couldn't retrieve request url"); + TSError("[denylist] Couldn't retrieve request url"); TSHandleMLocRelease(bufp, TS_NULL_MLOC, hdr_loc); goto done; } host = TSUrlHostGet(bufp, url_loc, &host_length); if (!host) { - TSError("[blacklist] couldn't retrieve request hostname"); + TSError("[denylist] couldn't retrieve request hostname"); TSHandleMLocRelease(bufp, hdr_loc, url_loc); TSHandleMLocRelease(bufp, TS_NULL_MLOC, hdr_loc); goto done; diff --git a/doc/developer-guide/plugins/example-plugins/index.en.rst b/doc/developer-guide/plugins/example-plugins/index.en.rst index 7da4a35ee69..f9b740d3201 100644 --- a/doc/developer-guide/plugins/example-plugins/index.en.rst +++ b/doc/developer-guide/plugins/example-plugins/index.en.rst @@ -26,7 +26,7 @@ Example Plugins :maxdepth: 2 basic-authorization/index.en - blacklist/index.en + denylist/index.en query_remap/index.en tls_bridge.en @@ -47,7 +47,7 @@ understand the following topics: - Working with HTTP header functions -The two sample plugins discussed in this chapter are ``blacklist_1.c`` +The two sample plugins discussed in this chapter are ``denylist_1.c`` and ``basic_auth.c``. To build and install the example plugins use :: ./configure --enable-example-plugins diff --git a/doc/developer-guide/plugins/getting-started/index.en.rst b/doc/developer-guide/plugins/getting-started/index.en.rst index b4a482feddf..f300673e0cc 100644 --- a/doc/developer-guide/plugins/getting-started/index.en.rst +++ b/doc/developer-guide/plugins/getting-started/index.en.rst @@ -93,7 +93,7 @@ Possible Uses for Plugins Possible uses for plugins include the following: -- HTTP processing: plugins can filter, blacklist, authorize users, +- HTTP processing: plugins can filter, deny access, authorize users, transform content - Protocol support: plugins can enable Traffic Server to proxy-cache @@ -101,7 +101,7 @@ Possible uses for plugins include the following: Some examples of plugins include: -- **Blacklisting plugin**: denies attempts to access web sites that are +- **Denylisting plugin**: denies attempts to access web sites that are off-limits. - **Append transform plugin**: adds text to HTTP response content. @@ -151,9 +151,9 @@ You can find basic examples for many plugins in the SDK sample code: - ``basic_auth.c`` performs basic HTTP proxy authorization. -- ``blacklist_1.c`` reads blacklisted servers from a configuration file +- ``denylist_1.c`` reads a list of servers from a configuration file and denies client access to these servers. This plugin is explained - in :ref:`developer-plugins-examples-blacklist`. + in :ref:`developer-plugins-examples-denylist`. Plugin Loading -------------- diff --git a/doc/developer-guide/plugins/hooks-and-transactions/adding-hooks.en.rst b/doc/developer-guide/plugins/hooks-and-transactions/adding-hooks.en.rst index e6174314723..aca6268bd57 100644 --- a/doc/developer-guide/plugins/hooks-and-transactions/adding-hooks.en.rst +++ b/doc/developer-guide/plugins/hooks-and-transactions/adding-hooks.en.rst @@ -33,7 +33,7 @@ There are several ways to add hooks to your plugin. - **Transaction hooks** Transaction hooks can be used to call plugins back for a specific HTTP transaction. You cannot add transaction hooks in ``TSPluginInit``; you first need a handle to a transaction. - See :ref:`developer-plugins-blacklist-access-process-txn`. + See :ref:`developer-plugins-denylist-access-process-txn`. - **Transformation hooks** Transformation hooks are a special case of transaction hooks. See @@ -80,7 +80,7 @@ values for ``TSHttpHookID`` are: Called immediately after the HTTP state machine has completed a DNS lookup of the origin server. The HTTP state machine will know the origin server's IP address at this point, which is useful for - performing both authentication and blacklisting. Corresponds to the + performing both authentication and denylisting. Corresponds to the event ``TS_EVENT_HTTP_OS_DNS``. ``TS_HTTP_POST_REMAP_HOOK`` diff --git a/doc/developer-guide/plugins/hooks-and-transactions/http-sessions.en.rst b/doc/developer-guide/plugins/hooks-and-transactions/http-sessions.en.rst index 26da265ebdb..d99498937ad 100644 --- a/doc/developer-guide/plugins/hooks-and-transactions/http-sessions.en.rst +++ b/doc/developer-guide/plugins/hooks-and-transactions/http-sessions.en.rst @@ -40,7 +40,10 @@ The HTTP session hooks are: - ``TS_HTTP_SSN_CLOSE_HOOK`` Called when an HTTP session ends (a session ends when the client connection is closed). This hook must be - added as a global hook. + added as a global hook. The relative order of invocation between the + ``TS_VCONN_CLOSE_HOOK`` and ``TS_HTTP_SSN_CLOSE_HOOK`` is undefined. In + most cases the ``TS_VCONN_CLOSE_HOOK`` will execute first, but that is + not guaranteed. Use the session hooks to get a handle to a session (an ``TSHttpSsn`` object). If you want your plugin to be called back for each transaction diff --git a/doc/developer-guide/plugins/hooks-and-transactions/http-transactions.en.rst b/doc/developer-guide/plugins/hooks-and-transactions/http-transactions.en.rst index f58bf3cc626..1c6aae2b250 100644 --- a/doc/developer-guide/plugins/hooks-and-transactions/http-transactions.en.rst +++ b/doc/developer-guide/plugins/hooks-and-transactions/http-transactions.en.rst @@ -28,7 +28,7 @@ transactions. As described in the section on HTTP sessions, an **HTTP transaction** is an object defined for the lifetime of a single request from a client and -the corresponding response from Traffic Server. The **``TSHttpTxn``** +the corresponding response from Traffic Server. The ``TSHttpTxn`` structure is the main handle given to a plugin for manipulating a transaction's internal state. Additionally, an HTTP transaction has a reference back to the HTTP session that created it. diff --git a/doc/developer-guide/plugins/http-headers/trafficserver-http-header-system.en.rst b/doc/developer-guide/plugins/http-headers/trafficserver-http-header-system.en.rst index 6af66bcc633..938f24c850a 100644 --- a/doc/developer-guide/plugins/http-headers/trafficserver-http-header-system.en.rst +++ b/doc/developer-guide/plugins/http-headers/trafficserver-http-header-system.en.rst @@ -42,7 +42,7 @@ passed into the common ``str*()`` routines Values returned from a marshall buffer can be ``NULL``, which means the field or object requested does not exist. -For example (from the ``blacklist_1`` sample) +For example (from the ``denylist_1`` sample) .. code-block:: c diff --git a/doc/developer-guide/plugins/index.en.rst b/doc/developer-guide/plugins/index.en.rst index 35ba0ff2626..4f528436110 100644 --- a/doc/developer-guide/plugins/index.en.rst +++ b/doc/developer-guide/plugins/index.en.rst @@ -37,6 +37,7 @@ Plugin Development io/index.en http-headers/index.en http-transformations/index.en + remap-plugins.en new-protocol-plugins.en plugin-interfaces.en adding-statistics.en diff --git a/doc/developer-guide/plugins/introduction.en.rst b/doc/developer-guide/plugins/introduction.en.rst index 5a5469bae12..1a17043fdf5 100644 --- a/doc/developer-guide/plugins/introduction.en.rst +++ b/doc/developer-guide/plugins/introduction.en.rst @@ -90,7 +90,7 @@ Below are some guidelines for creating a plugin: These examples are discussed in the following chapters. #. Determine where your plugin needs to hook on to Traffic Server's HTTP - processing (view the :ref:`http-txn-state-diagram`. + processing (view the :ref:`http-txn-state-diagram`). #. Read :ref:`developer-plugins-header-based-examples` to learn the basics of writing plugins: creating continuations and setting up hooks. If you @@ -101,7 +101,7 @@ Below are some guidelines for creating a plugin: then read about the details of those APIs in this manual's reference chapters. -#. Compile and load your plugin (see :ref:`developer-plugins-getting-started`. +#. Compile and load your plugin (see :ref:`developer-plugins-getting-started`). #. Depending on your plugin's functionality, you might start testing it by issuing requests by hand and checking for the desired behavior in @@ -173,7 +173,7 @@ them into activity. A plugin may consist of just one static continuation that is called whenever certain events happen. Examples of such plugins include -``blacklist_1.c``, ``basic_auth.c``, and ``redirect_1.c``. +``denylist_1.c``, ``basic_auth.c``, and ``redirect_1.c``. Alternatively, a plugin might dynamically create other continuations as needed. Transform plugins are built in this manner: a static parent continuation checks all transactions to see if any are transformable; @@ -266,35 +266,35 @@ reflects the Traffic Server state that was *just completed*. For example, the "OS DNS lookup" hook wakes up a plugin right *after* the origin server DNS lookup. For a plugin that requires the IP address of the requested origin server, this hook is the right one to use. The -Blacklist plugin works in this manner, as shown in the :ref:`BlackListPlugin` +Denylist plugin works in this manner, as shown in the :ref:`DenyListPlugin` diagram below. -**Blacklist Plugin** +**Denylist Plugin** -.. _BlackListPlugin: +.. _DenyListPlugin: -.. figure:: /static/images/sdk/blacklist75.jpg - :alt: Blacklist Plugin +.. figure:: /static/images/sdk/denylist.jpg + :alt: Denylist Plugin - Blacklist Plugin + Denylist Plugin -Traffic Server calls the Blacklist plugin right after the origin server +Traffic Server calls the Denylist plugin right after the origin server DNS lookup. The plugin checks the requested host against a list of -blacklisted servers; if the request is allowed, then the transaction -proceeds. If the host is forbidden, then the Blacklist plugin sends the +denylisted servers; if the request is allowed, then the transaction +proceeds. If the host is forbidden, then the Denylist plugin sends the transaction into an error state. When the HTTP state machine gets to the -"send reply header" state, it then calls the Blacklist plugin to provide +"send reply header" state, it then calls the Denylist plugin to provide the error message that's sent to the client. Types of Hooks ^^^^^^^^^^^^^^ -The Blacklist plugin's hook to the origin server DNS lookup state is a *global +The Denylist plugin's hook to the origin server DNS lookup state is a *global hook*, meaning that the plugin is called every time there's an HTTP transaction with a DNS lookup event. The plugin's hook to the send reply header state is a *transaction hook*, meaning that this hook is only invoked for specified -transactions (in the :ref:`developer-plugins-examples-blacklist` example, it's -only used for requests to blacklisted servers). Several examples of setting up +transactions (in the :ref:`developer-plugins-examples-denylist` example, it's +only used for requests to denylisted servers). Several examples of setting up hooks are provided in :ref:`developer-plugins-header-based-examples` and :ref:`developer-plugins-http-transformations`. diff --git a/doc/developer-guide/plugins/io/cache-api.en.rst b/doc/developer-guide/plugins/io/cache-api.en.rst index fc2f659274e..ce7c8143beb 100644 --- a/doc/developer-guide/plugins/io/cache-api.en.rst +++ b/doc/developer-guide/plugins/io/cache-api.en.rst @@ -115,7 +115,7 @@ Example In the example below, suppose there is a cache hit and the cache returns a vconnection that enables you to read the document from cache. To do this, you need to prepare a buffer (``cache_bufp``) to hold the -document; meanwhile, use ``TSVConnCachedObjectSizeGet`` to find out the +document; meanwhile, use ``TSVConnCacheObjectSizeGet`` to find out the actual size of the document (``content_length``). Then, issue ``TSVConnRead`` to read the document with the total data length required as ``content_length``. Assume the following data: @@ -133,7 +133,7 @@ In the ``TS_CACHE_OPEN_READ`` handler: .. code-block:: c cache_vconnp = (TSVConn) data; - TSVConnCachedObjectSizeGet (cache_vconnp, &content_length); + content_length = TSVConnCacheObjectSizeGet (cache_vconnp); cache_vio = TSVConnRead (cache_vconn, contp, cache_bufp, content_length); In the ``TS_EVENT_VCONN_READ_READY`` handler: diff --git a/doc/developer-guide/plugins/mutexes.en.rst b/doc/developer-guide/plugins/mutexes.en.rst index 6c63c0d053a..ae59e41ea98 100644 --- a/doc/developer-guide/plugins/mutexes.en.rst +++ b/doc/developer-guide/plugins/mutexes.en.rst @@ -71,12 +71,12 @@ by other continuations). Locking Global Data =================== -The :ref:`blacklist-1.c` sample plugin implements a mutex that locks global -data. The blacklist plugin reads its blacklisted sites from a +The :ref:`denylist-1.c` sample plugin implements a mutex that locks global +data. The denylist plugin reads sites to be denied from a configuration file; file read operations are protected by a mutex -created in :c:func:`TSPluginInit`. The :ref:`blacklist-1.c` code uses +created in :c:func:`TSPluginInit`. The :ref:`denylist-1.c` code uses :c:func:`TSMutexLockTry` instead of :c:func:`TSMutexLock`. For more detailed -information, see the :ref:`blacklist-1.c` code; +information, see the :ref:`denylist-1.c` code; start by looking at the :c:func:`TSPluginInit` function. General guidelines for locking shared data are as follows: @@ -368,7 +368,7 @@ continuation created in ``txn_handler``: TSCont newCont; .... newCont = TSContCreate (newCont_handler, NULL); - //It's not necessary to create a new mutex for newCont. + // It's not necessary to create a new mutex for newCont. ... diff --git a/doc/developer-guide/plugins/plugin-management/logging-api.en.rst b/doc/developer-guide/plugins/plugin-management/logging-api.en.rst index 0ce902b2985..013d39e6f81 100644 --- a/doc/developer-guide/plugins/plugin-management/logging-api.en.rst +++ b/doc/developer-guide/plugins/plugin-management/logging-api.en.rst @@ -58,8 +58,8 @@ The logging API enables you to: :c:func:`TSTextLogObjectDestroy` The steps below show how the logging API is used in the -``blacklist_1.c`` sample plugin. For the complete source code, see the -:ref:`developer-plugins-examples-blacklist-code` section. +``denylist_1.c`` sample plugin. For the complete source code, see the +:ref:`developer-plugins-examples-denylist-code` section. #. A new log file is defined as a global variable. @@ -71,10 +71,10 @@ The steps below show how the logging API is used in the .. code-block:: c - TSReturnCode error = TSTextLogObjectCreate("blacklist", + TSReturnCode error = TSTextLogObjectCreate("denylist", TS_LOG_MODE_ADD_TIMESTAMP, &log); - The new log is named ``blacklist.log``. Each entry written to the log + The new log is named ``denylist.log``. Each entry written to the log will have a timestamp. The ``NULL`` argument specifies that the new log does not have a log header. The error argument stores the result of the log creation; if the log is created successfully, then an @@ -86,11 +86,11 @@ The steps below show how the logging API is used in the .. code-block:: c if (error != TS_SUCCESS) { - printf("Blacklist plugin: error %d while creating log\n", error); + printf("denylist plugin: error %d while creating log\n", error); } -#. The :ref:`developer-plugins-examples-blacklist` matches the host portion of - the URL (in each client request) with a list of blacklisted sites (stored in +#. The :ref:`developer-plugins-examples-denylist` matches the host portion of + the URL (in each client request) with a list of denylisted sites (stored in the array ``sites[]``): .. code-block:: c @@ -101,23 +101,23 @@ The steps below show how the logging API is used in the } } - If the host matches one of the blacklisted - sites (such as ``sites[i]``), then the plugin writes a blacklist - entry to ``blacklist.log``: + If the host matches one of the denylisted + sites (such as ``sites[i]``), then the plugin writes a denylist + entry to ``denylist.log``: .. code-block:: c - if (log) { TSTextLogObjectWrite(log, "blacklisting site: %s", + if (log) { TSTextLogObjectWrite(log, "denylisting site: %s", sites[i]); The format of the log entry is as follows: :: - blacklisting site: sites[i] + denylisting site: sites[i] The log is not flushed or - destroyed in the ``blacklist_1`` plugin - it lives for the life of + destroyed in the ``denylist_1`` plugin - it lives for the life of the plugin. diff --git a/doc/developer-guide/plugins/remap-plugins.en.rst b/doc/developer-guide/plugins/remap-plugins.en.rst new file mode 100644 index 00000000000..2bab2883520 --- /dev/null +++ b/doc/developer-guide/plugins/remap-plugins.en.rst @@ -0,0 +1,183 @@ +.. Licensed to the Apache Software Foundation (ASF) under one or more contributor license + agreements. See the NOTICE file distributed with this work for additional information regarding + copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with the License. You may obtain + a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software distributed under the License + is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + or implied. See the License for the specific language governing permissions and limitations + under the License. + +.. include:: ../../common.defs +.. default-domain:: c +.. _developer-plugins-remap: + +Remap Plugins +************* + +Remap plugins are called during remap (URL rewriting). The particular plugins and the order is +determined by the remap rule. The :file:`remap.config` file can contain explicit references to +remap plugins in a rule and the presence of such a reference in a rule causes the plugin to be +invoked when the rule is matched. For example, a rule such as + +.. code-block:: text + + map http://example.one/ http://example.two/ @plugin=example.so @pparam=first_arg @pparm=second_arg + +will, if matched, cause the plugin "example.so" to be called with parameters `http://example.one/`, +`http://example.two/`, "first_arg" and "second_arg". Please keep in mind that "from" URL and "to" URL +will be converted to their canonical view. + +A key difference between global and remap plugins is reconfiguration and reloading. If +:file:`remap.config` is reloaded, then all remap plugins are reconfigured based on the new version +of the file. Global plugins need to handle their own configuration reloading, if any. + +In addition, as of |TS| 9.0, remap plugins can be reloaded during runtime. During a +:file:`remap.config` reload, if the plugin image file has changed, a new one is loaded and used. + +All of the externally invoked functions must be declared as :code:`extern "C"` in order to be +correctly located by the Traffic Server core. This is already done if :ts:git:`include/ts/remap.h` +is included, otherwise it must be done explicitly. + +Initialization +============== + +If any rule uses a plugin, the remap configuration loading will load the dynamic library and then +call :func:`TSRemapInit`. The plugin must return :macro:`TS_SUCCESS` or the configuration loading +will fail. If there is an error during the invocation of this function a C string style message +should be placed in :arg:`errbuff`, taking note of the maximum size of the buffer passed in +:arg:`errbuff_size`. The message is checked if the function returns a value other than +:macro:`TS_SUCCESS`. + +If :func:`TSRemapInit` returns :macro:`TS_ERROR` then the remap configuration loading +is aborted immediately. + +This function should perform any plugin global initialization, such as setting up static data +tables. It only be called immediately after the dynamic library is loaded from disk. + +Configuration +============= + +For each plugin invocation in a remap rule, :func:`TSRemapNewInstance` is called. + +The parameters :arg:`argc`, :arg:`argv` specify an array of arguments to the invocation instance in +the standard way. :arg:`argc` is the number of arguments present and :arg:`argv` is an array of +pointers, each of which points to a plugin parameter. The number of valid elements is :arg:`argc`. +Note these pointers are valid only for the duration of the function call. If any part of them need +persistent storage, that must be provided by the plugin. + +:arg:`ih` is for invocation instance storage data. This initially points at a :code:`nullptr`. If +that pointer is updated the new value will be preserved and passed back to the plugin in later +callbacks. This enables it to serve to identify which particular rule was matched and provide +context data. The standard use is to allocate a class instance, store rule relevant context data in +that instance, and update :arg:`ih` to point at the instance. The most common data is that derived +from the invocation arguments passed in :arg:`argc`, :arg:`argv`. + +:arg:`errbuff` and :arg:`errbuff_size` specify a writeable buffer used to report errors. Error +messages must be C strings and must fit in the buffer, including the terminating null. + +In essence, :func:`TSRemapNewInstance` is called to create an invocation instance for the plugin to +store rule local data. If the plugin is invoked multiples time on a rule, this will be called +multiple times for the rule, once for each invocation. Only the value store in :arg:`ih` will be +available when the rule is actually matched. In particular the plugin arguments will not be +available. + +Calls to :func:`TSRemapNewInstance` are guaranteed to be serialized. All calls to +:func:`TSRemapNewInstance` for a given new configuration are guaranteed to be called and +completed before any calls to :func:`TSRemapDoRemap`. + +If there is an error then the callback should return :macro:`TS_ERROR` and fill in the +:arg:`errbuff` with a C-string describing the error. Otherwise the function must return +:macro:`TS_SUCCESS`. + +If :func:`TSRemapNewInstance` returns :macro:`TS_ERROR` then the remap configuration loading +is aborted immediately. + + +Configuration reload notifications +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Most of the plugins is assumed to use per-plugin-instance data-structures when reloading their +configs and only a few of them that wish to optimize performance or deal with the complexities +of using a per-plugin DSO "global" data-structures would use plugin configuration reload +notifications like :func:`TSRemapPreConfigReload` and :func:`TSRemapPostConfigReload`. + +Instead of trying to foresee the needs or the expectations of each use-case, a more "open-ended" +and straight-forward design was chosen for the configuration reload notifications. +The notifications are broadcast to all loaded plugins at the moments before and after +the reload attempt, regardless of whether a plugin is part of the new configuration or not. + +:func:`TSRemapPreConfigReload` is called *before* the parsing of a new remap configuration starts +to notify plugins of the coming configuration reload. It is called on all already loaded plugins, +invoked by current and all previous still used configurations. This is an optional entry point. + +:func:`TSRemapPostConfigReload` is called to indicate the end of the new remap configuration +load. It is called on the newly and previously loaded plugins, invoked by the new, current and +previous still used configurations. It also indicates whether the configuration reload was successful +by passing :macro:`TSREMAP_CONFIG_RELOAD_FAILURE` in case of failure and to notify the plugins if they +are going to be part of the new configuration by passing :macro:`TSREMAP_CONFIG_RELOAD_SUCCESS_PLUGIN_USED` +or :macro:`TSREMAP_CONFIG_RELOAD_SUCCESS_PLUGIN_UNUSED`. This is an optional entry point. + +These calls are called per *plugin*, not per invocation of the plugin in :file:`remap.config` +and only will be called if the plugin was instantiated by at least one configuration loaded +after the traffic server started and at least one configuration using it is still loaded. + +:func:`TSRemapPreConfigReload` will be called serially for all loaded plugins +before any call to :func:`TSRemapNewInstance` during parsing of the new configuration. + +:func:`TSRemapPostConfigReload` will be called serially for all plugins after +all calls to :func:`TSRemapNewInstance` during parsing of the new configuration. + +The intention of these callbacks can be demonstrated with the following use-case. +A plugin could use :func:`TSRemapPreConfigReload` as a signal to drop (or allocate) temporary +per plugin data structures. These structures can be created (or updated) as needed +when a plugin invocation instance is loaded (:func:`TSRemapNewInstance` is called). +Then it could be used in subsequent invocation instances loading. After the configuration +reload is done :func:`TSRemapPostConfigReload` could be used to confirm the data +structures update if reload was successful, recover / clean-up after a failed +reload attempt, or if so wishes to ignore the notification if plugin is not part +of the new configuration.. + + +Runtime +======= + +At runtime, if a remap rule is matched, the plugin is invoked by calling :func:`TSRemapDoRemap`. +This function is responsible for performing the plugin operation for the transaction. + +:arg:`ih` is the same value set in :func:`TSRemapNewInstance` for the invocation instance. This is +not examined or checked by the core. :arg:`rh` is the transaction for which the rule matched. +:arg:`rri` is information about the rule and the transaction. + +The callback is required to return a :type:`TSRemapStatus` indicating whether it performed a remap. +This is used for verifying a request was remapped if remapping is required. This can also be used +to prevent further remapping, although this should be used with caution. + +Calls to :func:`TSRemapDoRemap` are not serialized, they can be concurrent, even for the same +invocation instance. However, the callbacks for a single rule for a single transaction are +serialized in the order the plugins are invoked in the rule. + +No calls to :func:`TSRemapDoRemap` will occur before :func:`TSRemapPostConfigReload` for +all plugin instances invoked by the new configuration. + +The old configurations, if any, are still active during the calls to :func:`TSRemapPreConfigReload` +and :func:`TSRemapPreConfigReload` and therefore calls to :func:`TSRemapDoRemap` may occur +concurrently with those functions. + + +Cleanup +======= + +When a new :file:`remap.config` is loaded successfully, the prior configuration is cleaned up. For +each call to :func:`TSRemapNewInstance` a corresponding call to :func:`TSRemapDeleteInstance` is +called. The only argument is the invocation instance handle, originally provided by the plugin to +:func:`TSRemapNewInstance`. This is expected to suffice for the plugin to clean up any rule specific +data. Calls to :func:`TSRemapDeleteInstance` will be serial for all plugin invocations in a +remap configuration. + +If no rule uses a plugin, it may be unloaded. In that case :func:`TSRemapDone` is called. This is +an optional entry point, a plugin is not required to provide it. It corresponds to :func:`TSRemapInit`. +It is called once per plugin, not per plugin invocation. diff --git a/doc/developer-guide/testing/blackbox-testing.en.rst b/doc/developer-guide/testing/blackbox-testing.en.rst index eba82401cfa..8f1e67d6332 100644 --- a/doc/developer-guide/testing/blackbox-testing.en.rst +++ b/doc/developer-guide/testing/blackbox-testing.en.rst @@ -26,7 +26,7 @@ for functional testing. The current layout is: :: - gold_tests/ - contains all the TSQA v4 based tests that run on the Reusable Gold Testing System (AuTest) + gold_tests/ - contains all the tests that run on the Reusable Gold Testing System (AuTest) tools/ - contains programs used to help with testing. Scripts @@ -115,13 +115,13 @@ A number of file objects are also defined to help test TrafficServer. Files that - diags.log - records.config - cache.config -- congestion.config - hosting.config - ip_allow.yaml - logging.yaml - parent.config - plugin.config - remap.config +- sni.yaml - socks.config - splitdns.config - ssl_multicert.config @@ -252,6 +252,7 @@ Condition Testing - TS_HAS_IP_TOS - TS_USE_HWLOC - TS_USE_SET_RBIO + - TS_USE_QUIC - TS_USE_LINUX_NATIVE_AIO - TS_HAS_SO_PEERCRED - TS_USE_REMOTE_UNWINDING diff --git a/doc/ext/doxygen.py b/doc/ext/doxygen.py index 7ffdc004ab3..6d896693907 100644 --- a/doc/ext/doxygen.py +++ b/doc/ext/doxygen.py @@ -44,7 +44,7 @@ def escape(name): Partial reimplementation in Python of Doxygen escapeCharsInString() """ - return name.replace('_', '__').replace(':', '_1').replace('/', '_2').replace('<', '_3').replace('>', '_4').replace('*', '_5').replace('&', '_6').replace('|', '_7').replace('.', '_8').replace('!', '_9').replace(',', '_00').replace(' ', '_01').replace('{', '_02').replace('}', '_03').replace('?', '_04').replace('^', '_05').replace('%', '_06').replace('(', '_07').replace(')', '_08').replace('+', '_09').replace('=', '_0A').replace('$', '_0B').replace('\\', '_0C') + return name.replace('_', '__').replace(':', '_1').replace('/', '_2').replace('<', '_3').replace('>', '_4').replace('*', '_5').replace('&', '_6').replace('|', '_7').replace('.', '_8').replace('!', '_9').replace(',', '_00').replace(' ', '_01').replace('{', '_02').replace('}', '_03').replace('?', '_04').replace('^', '_05').replace('%', '_06').replace('(', '_07').replace(')', '_08').replace('+', '_09').replace('=', '_0A').replace('$', '_0B').replace('\\', '_0C') # nopep8 class doctree_resolved: @@ -66,7 +66,9 @@ def __init__(self, app, doctree, docname): # Style the links raw = nodes.raw( - '', '', format='html') + '', + '', + format='html') doctree.insert(0, raw) def traverse(self, node, owner): @@ -108,7 +110,8 @@ def traverse(self, node, owner): cache[filename] = etree.parse('xml/' + filename) # An enumvalue has no location - memberdef, = cache[filename].xpath('descendant::compounddef[compoundname[text() = $name]]', name=name) or cache[filename].xpath( + memberdef, = cache[filename].xpath( + 'descendant::compounddef[compoundname[text() = $name]]', name=name) or cache[filename].xpath( 'descendant::memberdef[name[text() = $name] | enumvalue[name[text() = $name]]]', name=name) # Append the link after the object's signature. diff --git a/doc/ext/traffic-server.py b/doc/ext/traffic-server.py index 0fb2044d158..20336945ad8 100644 --- a/doc/ext/traffic-server.py +++ b/doc/ext/traffic-server.py @@ -47,6 +47,7 @@ def is_string_type(s): def is_string_type(s): return isinstance(s, str) + class TSConfVar(std.Target): """ Description of a traffic server configuration variable. @@ -116,7 +117,7 @@ def run(self): title.set_class(self.options.get('class')) # This has to be a distinct node before the title. if nested then # the browser will scroll forward to just past the title. - anchor = nodes.target('', '', names=[cv_name]) + nodes.target('', '', names=[cv_name]) # Second (optional) arg is 'msgNode' - no idea what I should pass for that # or if it even matters, although I now think it should not be used. self.state.document.note_explicit_target(title) @@ -168,7 +169,19 @@ def metrictypes(typename): def metricunits(unitname): - return directives.choice(unitname.lower(), ('ratio', 'percent', 'kbits', 'mbits', 'bytes', 'kbytes', 'mbytes', 'nanoseconds', 'microseconds', 'milliseconds', 'seconds')) + return directives.choice( + unitname.lower(), + ('ratio', + 'percent', + 'kbits', + 'mbits', + 'bytes', + 'kbytes', + 'mbytes', + 'nanoseconds', + 'microseconds', + 'milliseconds', + 'seconds')) class TSStat(std.Target): @@ -240,7 +253,7 @@ def run(self): # This has to be a distinct node before the title. if nested then # the browser will scroll forward to just past the title. - anchor = nodes.target('', '', names=[stat_name]) + nodes.target('', '', names=[stat_name]) # Second (optional) arg is 'msgNode' - no idea what I should pass for that # or if it even matters, although I now think it should not be used. self.state.document.note_explicit_target(title) diff --git a/doc/index.rst b/doc/index.rst index 36ac32cf381..0495c31436f 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -27,6 +27,7 @@ Apache Traffic Server Manual preface/index.en getting-started/index.en + release-notes/index.en admin-guide/index.en developer-guide/index.en appendices/index.en diff --git a/doc/locale/ja/LC_MESSAGES/admin-guide/configuration/cache-basics.en.po b/doc/locale/ja/LC_MESSAGES/admin-guide/configuration/cache-basics.en.po index c69382af9ff..e0ca0c4a184 100644 --- a/doc/locale/ja/LC_MESSAGES/admin-guide/configuration/cache-basics.en.po +++ b/doc/locale/ja/LC_MESSAGES/admin-guide/configuration/cache-basics.en.po @@ -1666,7 +1666,7 @@ msgid "" msgstr "" "最後に :ts:cv:`proxy.config.http.cache.fuzz.min_time` は小さい TTL と大きい " "TTL で再確認の確率を評価する時間が異なることを許容します。TTL の小さなオブ" -"ジェクトは ``fuzz.min_time`` 付近で “再確認のサイコロを転がす” ことを始めま" +"ジェクトは ``fuzz.min_time`` 付近で "再確認のサイコロを転がす" ことを始めま" "す。一方、大きな TTL のオブジェクトは ``fuzz.time`` 付近で始めるでしょう。" #: ../../../admin-guide/configuration/cache-basics.en.rst:849 diff --git a/doc/locale/ja/LC_MESSAGES/admin-guide/configuration/hierachical-caching.en.po b/doc/locale/ja/LC_MESSAGES/admin-guide/configuration/hierachical-caching.en.po index a69cfc2788f..06745a4aa2d 100644 --- a/doc/locale/ja/LC_MESSAGES/admin-guide/configuration/hierachical-caching.en.po +++ b/doc/locale/ja/LC_MESSAGES/admin-guide/configuration/hierachical-caching.en.po @@ -59,7 +59,7 @@ msgstr "" msgid "" "If the request is a cache miss on the parent, then the parent retrieves the " "content from the origin server (or from another cache, depending on the " -"parent’s configuration). The parent caches the content and then sends a " +"parent's configuration). The parent caches the content and then sends a " "copy to Traffic Server (its child), where it is cached and served to the " "client." msgstr "" diff --git a/doc/locale/ja/LC_MESSAGES/admin-guide/configuration/redirecting-http-requests.en.po b/doc/locale/ja/LC_MESSAGES/admin-guide/configuration/redirecting-http-requests.en.po index b561f2ef5ae..68f4d998437 100644 --- a/doc/locale/ja/LC_MESSAGES/admin-guide/configuration/redirecting-http-requests.en.po +++ b/doc/locale/ja/LC_MESSAGES/admin-guide/configuration/redirecting-http-requests.en.po @@ -89,7 +89,7 @@ msgstr "" msgid "" "A client browser sends an HTTP request addressed to a host called ``www." "host.com`` on port 80. Traffic Server receives the request because it is " -"acting as the origin server (the origin server’s advertised hostname " +"acting as the origin server (the origin server's advertised hostname " "resolves to Traffic Server)." msgstr "" "クライアントブラウザが ``www.host.com`` の 80 番ポートに HTTP リクエストを送" @@ -399,7 +399,7 @@ msgstr "URL のスキームが同じであること。" #: ../../admin-guide/configuration/redirecting-http-requests.en.rst:119 msgid "" -"To avoid a DNS conflict, the origin server’s hostname and its advertised " +"To avoid a DNS conflict, the origin server's hostname and its advertised " "hostname must not be the same." msgstr "" "DNS の衝突を避けるため、オリジンサーバーのホスト名とその広告されたホスト名は" diff --git a/doc/locale/ja/LC_MESSAGES/admin-guide/configuration/transparent-proxy/wccp-service-config.en.po b/doc/locale/ja/LC_MESSAGES/admin-guide/configuration/transparent-proxy/wccp-service-config.en.po index c5fa5a58d97..5ad2c750947 100644 --- a/doc/locale/ja/LC_MESSAGES/admin-guide/configuration/transparent-proxy/wccp-service-config.en.po +++ b/doc/locale/ja/LC_MESSAGES/admin-guide/configuration/transparent-proxy/wccp-service-config.en.po @@ -111,12 +111,12 @@ msgstr "" #: ../../../admin-guide/configuration/transparent-proxy/wccp-service-config.en.rst:62 msgid "" -"type – This defines the type of service group either “STANDARD” or " -"“DYNAMIC”. There is one standard defined service group, HTTP with the id " -"of 0. The 4/2001 RFC indicates that id’s 0-50 are reserved for well known " +"type – This defines the type of service group either "STANDARD" or " +""DYNAMIC". There is one standard defined service group, HTTP with the id " +"of 0. The 4/2001 RFC indicates that id's 0-50 are reserved for well known " "service groups. But more recent 8/2012 RFC indicates that values 0 through " -"254 are valid service id’s for dynamic services. To avoid differences with " -"older WCCP routers, you probably want to avoid dynamic service ID’s 0 " +"254 are valid service id's for dynamic services. To avoid differences with " +"older WCCP routers, you probably want to avoid dynamic service ID's 0 " "through 50." msgstr "" @@ -213,7 +213,7 @@ msgstr "" #: ../../../admin-guide/configuration/transparent-proxy/wccp-service-config.en.rst:83 msgid "" "proc-name – This attribute is only used by traffic_wccp. It is not used in " -"the traffic_server WCCP support. This is the path to a process’ PID file. " +"the traffic_server WCCP support. This is the path to a process' PID file. " "The service group is advertised to the WCCP router if the process " "identified in the PID file is currently operational." msgstr "" diff --git a/doc/locale/ja/LC_MESSAGES/admin-guide/files/records.config.en.po b/doc/locale/ja/LC_MESSAGES/admin-guide/files/records.config.en.po index 92f70b9d891..e5c6a48972a 100644 --- a/doc/locale/ja/LC_MESSAGES/admin-guide/files/records.config.en.po +++ b/doc/locale/ja/LC_MESSAGES/admin-guide/files/records.config.en.po @@ -2750,7 +2750,7 @@ msgstr "" #: ../../../admin-guide/files/records.config.en.rst:1664 msgid "" -"SSDs and \"advanced format” drives claim a sector size of 512; however, it " +"SSDs and \"advanced format" drives claim a sector size of 512; however, it " "is safe to force a higher size than the hardware supports natively as we " "count atomicity in 512 byte increments." msgstr "" diff --git a/doc/locale/ja/LC_MESSAGES/admin-guide/introduction.en.po b/doc/locale/ja/LC_MESSAGES/admin-guide/introduction.en.po index 6a0a120ffac..b3e94d80b47 100644 --- a/doc/locale/ja/LC_MESSAGES/admin-guide/introduction.en.po +++ b/doc/locale/ja/LC_MESSAGES/admin-guide/introduction.en.po @@ -39,7 +39,7 @@ msgid "" "accessible. Unfortunately, global data networking can also be a nightmare " "for IT professionals as they struggle with overloaded servers and congested " "networks. It can be challenging to consistently and reliably accommodate " -"society’s growing data demands." +"society's growing data demands." msgstr "" "グローバルなデータネットワークの利用は日常生活の一部となりました。インター" "ネットユーザーは日常生活の基盤の上で数１０億ものドキュメントやペタバイトの" @@ -111,7 +111,7 @@ msgid "" "Traffic Server contains the requested content, then it serves the content " "directly. If the requested content is not available from cache, then " "Traffic Server acts as a proxy: it obtains the content from the origin " -"server on the user’s behalf and also keeps a copy to satisfy future " +"server on the user's behalf and also keeps a copy to satisfy future " "requests." msgstr "" "ウェブプロキシーキャッシュとして Traffic Server はウェブコンテンツへのユー" @@ -124,7 +124,7 @@ msgstr "" #: ../../../admin-guide/introduction.en.rst:65 msgid "" -"Traffic Server provides explicit proxy caching, in which the user’s client " +"Traffic Server provides explicit proxy caching, in which the user's client " "software must be configured to send requests directly to Traffic Server. " "Explicit proxy caching is described in the :ref:`explicit-proxy-caching` " "chapter." @@ -152,7 +152,7 @@ msgstr "リバースプロキシーとしての Traffic Server" #: ../../../admin-guide/introduction.en.rst:78 msgid "" "As a reverse proxy, Traffic Server is configured to be the origin server to " -"which the user is trying to connect (typically, the origin server’s " +"which the user is trying to connect (typically, the origin server's " "advertised hostname resolves to Traffic Server, which acts as the real " "origin server). The reverse proxy feature is also called server " "acceleration. Reverse proxy is described in more detail in :ref:`reverse-" diff --git a/doc/locale/ja/LC_MESSAGES/admin-guide/monitoring/error-messages.en.po b/doc/locale/ja/LC_MESSAGES/admin-guide/monitoring/error-messages.en.po index 0000fb2ca0f..7c223fc8c70 100644 --- a/doc/locale/ja/LC_MESSAGES/admin-guide/monitoring/error-messages.en.po +++ b/doc/locale/ja/LC_MESSAGES/admin-guide/monitoring/error-messages.en.po @@ -405,10 +405,10 @@ msgstr "" #: ../../../admin-guide/monitoring/error-messages.en.rst:185 msgid "" -"``400`` Could not process this request because ``Content-Length`` was not " +"``411`` Could not process this request because ``Content-Length`` was not " "specified. ``request#no_content_length``" msgstr "" -"``400`` Could not process this request because ``Content-Length`` was not " +"``411`` Could not process this request because ``Content-Length`` was not " "specified. (``Content-Length`` が指定されなかったためリクエストを処理できま" "せんでした。) ``request#no_content_length``" diff --git a/doc/locale/ja/LC_MESSAGES/admin-guide/monitoring/logging/log-formats.en.po b/doc/locale/ja/LC_MESSAGES/admin-guide/monitoring/logging/log-formats.en.po index 354fb9e0e1e..34c1569f59a 100644 --- a/doc/locale/ja/LC_MESSAGES/admin-guide/monitoring/logging/log-formats.en.po +++ b/doc/locale/ja/LC_MESSAGES/admin-guide/monitoring/logging/log-formats.en.po @@ -273,7 +273,7 @@ msgid "chi" msgstr "chi" #: ../../../admin-guide/monitoring/logging/log-formats.en.rst:124 -msgid "The IP address of the client’s host machine." +msgid "The IP address of the client's host machine." msgstr "クライアントのホストマシンの IP アドレス" #: ../../../admin-guide/monitoring/logging/log-formats.en.rst:125 diff --git a/doc/locale/ja/LC_MESSAGES/admin-guide/monitoring/logging/summary-logs.en.po b/doc/locale/ja/LC_MESSAGES/admin-guide/monitoring/logging/summary-logs.en.po index 4019d6a7112..f3056d47b5c 100644 --- a/doc/locale/ja/LC_MESSAGES/admin-guide/monitoring/logging/summary-logs.en.po +++ b/doc/locale/ja/LC_MESSAGES/admin-guide/monitoring/logging/summary-logs.en.po @@ -59,7 +59,7 @@ msgid "``SUM``" msgstr "" #: ../../../admin-guide/monitoring/logging/summary-logs.en.rst:38 -msgid "``AVERAGE``" +msgid "``AVG``" msgstr "" #: ../../../admin-guide/monitoring/logging/summary-logs.en.rst:39 @@ -82,7 +82,7 @@ msgstr "" #: ../../../admin-guide/monitoring/logging/summary-logs.en.rst:52 msgid "" "Where ``operator`` is one of the five aggregate operators (``COUNT``, " -"``SUM``, ``AVERAGE``, ``FIRST``, ``LAST``); ``field`` is the logging field " +"``SUM``, ``AVG``, ``FIRST``, ``LAST``); ``field`` is the logging field " "you want to aggregate; and ``n`` is the interval (in seconds) between " "summary log entries." msgstr "" diff --git a/doc/locale/ja/LC_MESSAGES/admin-guide/performance/index.en.po b/doc/locale/ja/LC_MESSAGES/admin-guide/performance/index.en.po index 6cb4bad1ff6..3e7de8b13a9 100644 --- a/doc/locale/ja/LC_MESSAGES/admin-guide/performance/index.en.po +++ b/doc/locale/ja/LC_MESSAGES/admin-guide/performance/index.en.po @@ -231,7 +231,7 @@ msgstr "" #: ../../../admin-guide/performance/index.en.rst:185 msgid "" -"By default, |TS| creates 1.5 threads per CPU core on the host system. This " +"By default, |TS| creates one thread per CPU core on the host system. This " "may be adjusted with the following settings in :file:`records.config`:" msgstr "" diff --git a/doc/locale/ja/LC_MESSAGES/admin-guide/plugins/collapsed_forwarding.en.po b/doc/locale/ja/LC_MESSAGES/admin-guide/plugins/collapsed_forwarding.en.po index 176bf2d6935..3484a8c80b1 100644 --- a/doc/locale/ja/LC_MESSAGES/admin-guide/plugins/collapsed_forwarding.en.po +++ b/doc/locale/ja/LC_MESSAGES/admin-guide/plugins/collapsed_forwarding.en.po @@ -149,12 +149,12 @@ msgstr "" #: ../../../admin-guide/plugins/collapsed_forwarding.en.rst:113 msgid "" -"For a large scale Video Streaming scenario, there’s a combination of a " +"For a large scale Video Streaming scenario, there's a combination of a " "large number of revalidations (e.g. media playlists) and cache misses (e.g. " -"media segments) that occur for the same file. Traffic Server’s RWW works " +"media segments) that occur for the same file. Traffic Server's RWW works " "great in collapsing the concurrent requests in such a scenario, however, as " "described in ``_admin-configuration-reducing-origin-requests``, Traffic " -"Server’s implementation of RWW has a significant limitation, which " +"Server's implementation of RWW has a significant limitation, which " "restricts its ability to invoke RWW only when the response headers are " "already received. This means that any number of concurrent requests for the " "same file that are received before the response headers arrive are leaked " @@ -182,7 +182,7 @@ msgid "" "lookup is successful, meaning, the dirent exists for the generated cache " "key, Traffic Server tries to obtain a read lock on the cache object to be " "able to serve it from the cache. If the read lock is not successful " -"(possibly, due to the fact that the object’s being written to at that same " +"(possibly, due to the fact that the object's being written to at that same " "instant and the response headers are not in the cache yet), Traffic Server " "then moves to the next step of trying to obtain an exclusive write lock. If " "the write lock is already held exclusively by another request " diff --git a/doc/locale/ja/LC_MESSAGES/admin-guide/plugins/header_rewrite.en.po b/doc/locale/ja/LC_MESSAGES/admin-guide/plugins/header_rewrite.en.po index e2722c190c2..24d35375cfd 100644 --- a/doc/locale/ja/LC_MESSAGES/admin-guide/plugins/header_rewrite.en.po +++ b/doc/locale/ja/LC_MESSAGES/admin-guide/plugins/header_rewrite.en.po @@ -1424,7 +1424,7 @@ msgstr "" #: ../../../admin-guide/plugins/header_rewrite.en.rst:1019 msgid "" "This rule adds cache control headers to CDN responses based matching the " -"origin path. One provides a max age and the other provides a “no-cache” " +"origin path. One provides a max age and the other provides a "no-cache" " "statement to two different file paths.::" msgstr "" diff --git a/doc/locale/ja/LC_MESSAGES/admin-guide/plugins/tcpinfo.en.po b/doc/locale/ja/LC_MESSAGES/admin-guide/plugins/tcpinfo.en.po index 0b180032772..88ae2c51245 100644 --- a/doc/locale/ja/LC_MESSAGES/admin-guide/plugins/tcpinfo.en.po +++ b/doc/locale/ja/LC_MESSAGES/admin-guide/plugins/tcpinfo.en.po @@ -232,10 +232,6 @@ msgstr "" msgid "snd_ssthresh" msgstr "" -#: ../../admin-guide/plugins/tcpinfo.en.rst:48 -msgid "ssn_close" -msgstr "" - #: ../../admin-guide/plugins/tcpinfo.en.rst:69 #: ../../admin-guide/plugins/tcpinfo.en.rst:81 msgid "timestamp" diff --git a/doc/locale/ja/LC_MESSAGES/admin-guide/transparent-proxy/wccp-service-config.en.po b/doc/locale/ja/LC_MESSAGES/admin-guide/transparent-proxy/wccp-service-config.en.po index ca14fef2ac1..5601d170e11 100644 --- a/doc/locale/ja/LC_MESSAGES/admin-guide/transparent-proxy/wccp-service-config.en.po +++ b/doc/locale/ja/LC_MESSAGES/admin-guide/transparent-proxy/wccp-service-config.en.po @@ -106,12 +106,12 @@ msgstr "" #: ../../admin-guide/transparent-proxy/wccp-service-config.en.rst:53 msgid "" -"type – This defines the type of service group either “STANDARD” or " -"“DYNAMIC”. There is one standard defined service group, HTTP with the id " -"of 0. The 4/2001 RFC indicates that id’s 0-50 are reserved for well known " +"type – This defines the type of service group either "STANDARD" or " +""DYNAMIC". There is one standard defined service group, HTTP with the id " +"of 0. The 4/2001 RFC indicates that id's 0-50 are reserved for well known " "service groups. But more recent 8/2012 RFC indicates that values 0 through " -"254 are valid service id’s for dynamic services. To avoid differences with " -"older WCCP routers, you probably want to avoid dynamic service ID’s 0 " +"254 are valid service id's for dynamic services. To avoid differences with " +"older WCCP routers, you probably want to avoid dynamic service ID's 0 " "through 50." msgstr "" @@ -208,7 +208,7 @@ msgstr "" #: ../../admin-guide/transparent-proxy/wccp-service-config.en.rst:74 msgid "" "proc-name – This attribute is only used by traffic_wccp. It is not used in " -"the traffic_server WCCP support. This is the path to a process’ PID file. " +"the traffic_server WCCP support. This is the path to a process' PID file. " "The service group is advertised to the WCCP router if the process " "identified in the PID file is currently operational." msgstr "" diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/plugins/io/cache-api.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/plugins/io/cache-api.en.po index 555960c2bc5..5dd172ef779 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/plugins/io/cache-api.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/plugins/io/cache-api.en.po @@ -151,7 +151,7 @@ msgid "" "In the example below, suppose there is a cache hit and the cache returns a " "vconnection that enables you to read the document from cache. To do this, " "you need to prepare a buffer (``cache_bufp``) to hold the document; " -"meanwhile, use ``TSVConnCachedObjectSizeGet`` to find out the actual size " +"meanwhile, use ``TSVConnCacheObjectSizeGet`` to find out the actual size " "of the document (``content_length``). Then, issue ``TSVConnRead`` to read " "the document with the total data length required as ``content_length``. " "Assume the following data:" diff --git a/doc/release-notes/index.en.rst b/doc/release-notes/index.en.rst new file mode 100644 index 00000000000..1a770aca200 --- /dev/null +++ b/doc/release-notes/index.en.rst @@ -0,0 +1,27 @@ +.. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +.. include:: ../common.defs + +Release Notes +************* + +.. toctree:: + :maxdepth: 1 + + whats-new.en + upgrading.en diff --git a/doc/release-notes/upgrading.en.rst b/doc/release-notes/upgrading.en.rst new file mode 100644 index 00000000000..a46bef5fbed --- /dev/null +++ b/doc/release-notes/upgrading.en.rst @@ -0,0 +1,153 @@ +.. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +.. include:: ../common.defs + +.. _upgrading: + +Upgrading to ATS v9.x +====================== + +.. toctree:: + :maxdepth: 1 + +Remapping +--------- + +One of the biggest changes in ATS v9.0.0 is that URL rewrites, as specified in a :file:`remap.config` rule, now always happens +**before** all plugins are executed. This can have significant impact on behavior, since plugins might now see a different URL than +they did in prior versions. In particular, plugins modifying the cache key could have serious problems (see the section below for +details). + +YAML +---- + +We are moving configurations over to YAML, and thus far, the following configurations are now fully migrated over to YAML: + +* :file:`logging.yaml` (*was* `logging.config` or `logging.lua`) +* :file:`ip_allow.yaml` (*was* `ip_allow.config`) + +In addition, a new file for TLS handhsake negotiation configuration is added: + +* :file:`sni.yaml` (this was for a while named ssl_server_name.config in Github) + +New records.config settings +---------------------------- + +These are the changes that are most likely to cause problems during an upgrade. Take special care making sure you have updated your +configurations accordingly. + +Connection management +~~~~~~~~~~~~~~~~~~~~~ + +The old settings for origin connection management included the following settings: + +* `proxy.config.http.origin_max_connections` +* `proxy.config.http.origin_max_connections_queue` +* `proxy.config.http.origin_min_keep_alive_connections` + +These are all gone, and replaced with the following set of configurations: + +* :ts:cv:`proxy.config.http.per_server.connection.max` (overridable) +* :ts:cv:`proxy.config.http.per_server.connection.match` (overridable) +* :ts:cv:`proxy.config.http.per_server.connection.alert_delay` +* :ts:cv:`proxy.config.http.per_server.connection.queue_size` +* :ts:cv:`proxy.config.http.per_server.connection.queue_delay` +* :ts:cv:`proxy.config.http.per_server.connection.min` + +Removed records.config settings +------------------------------- + +The following settings are simply gone, and have no purpose: + +* `proxy.config.config_dir` (see PROXY_CONFIG_CONFIG_DIR environment variable) +* `proxy.config.cache.storage_filename` (see next section as well) + +Deprecated records.config settings +---------------------------------- + +The following configurations still exist, and functions, but are considered deprecated and will be removed in a future release. We +**strongly** encourage you to avoid using any of these: + + * :ts:cv:`proxy.config.socks.socks_config_file` + * :ts:cv:`proxy.config.log.config.filename` + * :ts:cv:`proxy.config.url_remap.filename` + * :ts:cv:`proxy.config.ssl.server.multicert.filename` + * :ts:cv:`proxy.config.ssl.servername.filename` + * ``proxy.config.http.parent_proxy.file`` + * ``proxy.config.cache.control.filename`` + * ``proxy.config.cache.ip_allow.filename`` + * ``proxy.config.cache.hosting_filename`` + * ``proxy.config.cache.volume_filename`` + * ``proxy.config.dns.splitdns.filename`` + +Deprecated or Removed Features +------------------------------ + +The following features, configurations and plugins are either removed or deprecated in this version of ATS. Deprecated features +should be avoided, with the expectation that they will be removed in the next major release of ATS. + + +API Changes +----------- + +Our APIs are guaranteed to be compatible within major versions, but we do make changes for each new major release. + +Removed APIs +~~~~~~~~~~~~ + +* ``TSHttpTxnRedirectRequest()`` + +Renamed or modified APIs +~~~~~~~~~~~~~~~~~~~~~~~~ + +* ``TSVConnSSLConnectionGet()`` is renamed to be :c:func:`TSVConnSslConnectionGet` + +* ``TSHttpTxnServerPush()`` now returns a :c:type:`TSReturnCode` + + +Cache +----- + +The cache in this releases of ATS is compatible with previous versions of ATS. You would not expect to lose your cache, or have to +reinitialize the cache when upgrading. + +However, due to changes in how remap plugins are processed, your cache key *might* change. In versions to v9.0.0, the first plugin +in a remap rule would get the pristine URL, and subsequent plugins would get the remapped URL. As of v9.0.0, **all** plugins now +receive the remapped URL. If you are using a plugin that modifies the cache key, e.g. :ref:`admin-plugins-cachekey`, if it was +evaluated first in a remap rule, the behavior (input) changes, and therefore, cache keys can change! + +The old ``v23`` cache is no longer supported, which means caches created with ATS v2.x will no longer be possible to load with ATS +v9.0.0 or later. We feel that this is an unlikely scenario, but if you do run into this, clearing the cache is required. + +Plugins +------- + +The following plugins have changes that might require you to change +configurations. + +header_rewrite +~~~~~~~~~~~~~~ + +* The `%{PATH}` directive is now removed, and instead you want to use `%{CLIENT-URL:PATH}`. This was done to unify the behavior of + these operators, rather than having this one-off directive. + +Platform specific +----------------- + +Solaris is no longer a supported platform, but the code is still there. However, it's unlikely to work, and unless someone takes on +ownership of this Platform, it will be removed from the source in ATS v10.0.0. For more details, see issue #5553. diff --git a/doc/release-notes/whats-new.en.rst b/doc/release-notes/whats-new.en.rst new file mode 100644 index 00000000000..a16f1efc462 --- /dev/null +++ b/doc/release-notes/whats-new.en.rst @@ -0,0 +1,100 @@ +.. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +.. include:: ../common.defs + +.. _whats_new: + +What's New in ATS v9.x +======================= + +This version of ATS includes over commits, from pull requests. A total of contributors have participated in this +development cycle. + +.. toctree:: + :maxdepth: 1 + +New Features +------------ + +This version of ATS has a number of new features (details below), but we're particularly excited about the following features: + +* Experimental QUIC support (draft 27). +* TLS v1.3 0RTT support. + +PROXY protocol +~~~~~~~~~~~~~~ + +ATS now supports the `PROXY `_ protocol, on the inbound side. The +incoming PROXY data gets transformed into the ``Forwarded`` header. + +Developer features +~~~~~~~~~~~~~~~~~~ + +* Add support for dtrace style markers (SDT) and include a few markers at locations of interest to users of SystemTap, dtrace, and + gdb. See :ref:`developer-debug-builds`. + +Command line tools +~~~~~~~~~~~~~~~~~~ + +* The :program:`traffic_server` program now has two new maintenance commands: ``verify_global_plugin`` and ``verify_remap_plugin``. + These commands load a plugin's shared object file and verify it meets minimal global or remap plugin API requirements. + +Incompatible records.config settings +------------------------------------ + +These are the changes that are most likely to cause problems during an upgrade. +Take special care making sure you have updated your configurations accordingly. + +Connection management +~~~~~~~~~~~~~~~~~~~~~ + +The old settings for origin connection management included the following settings: + +* `proxy.config.http.origin_max_connections` +* `proxy.config.http.origin_max_connections_queue` +* `proxy.config.http.origin_min_keep_alive_connections` + +These are all gone, and replaced with the following set of configurations: + +* :ts:cv:`proxy.config.http.per_server.connection.max` (overridable) +* :ts:cv:`proxy.config.http.per_server.connection.match` (overridable) +* :ts:cv:`proxy.config.http.per_server.connection.alert_delay` +* :ts:cv:`proxy.config.http.per_server.connection.queue_size` +* :ts:cv:`proxy.config.http.per_server.connection.queue_delay` +* :ts:cv:`proxy.config.http.per_server.connection.min` + +Logging and Metrics +------------------- + +Plugins +------- + +xdebug +~~~~~~ + +* A new directive, `fwd=` to control the number of hops the header is forwarded for. + +Plugin APIs +----------- + +The API for server push is promoted to stable, and modified to return an error code, to be consistent with other similar APIs. The +new prototype is: + +.. code-block:: c + + .. c:function:: TSReturnCode TSHttpTxnServerPush(TSHttpTxn txnp, const char *url, int url_len); diff --git a/doc/static/images/admin/process.jpg b/doc/static/images/admin/process.jpg deleted file mode 100644 index f683ac7e553..00000000000 Binary files a/doc/static/images/admin/process.jpg and /dev/null differ diff --git a/doc/static/images/admin/proxy-protocol.png b/doc/static/images/admin/proxy-protocol.png index e69de29bb2d..ee38975c5d1 100644 Binary files a/doc/static/images/admin/proxy-protocol.png and b/doc/static/images/admin/proxy-protocol.png differ diff --git a/doc/static/images/sdk/blacklist75.jpg b/doc/static/images/sdk/denylist.jpg similarity index 77% rename from doc/static/images/sdk/blacklist75.jpg rename to doc/static/images/sdk/denylist.jpg index 880c32cb356..c2a990ea328 100644 Binary files a/doc/static/images/sdk/blacklist75.jpg and b/doc/static/images/sdk/denylist.jpg differ diff --git a/doc/uml/Makefile.am b/doc/uml/Makefile.am index 2a9fa761056..050d91e848e 100644 --- a/doc/uml/Makefile.am +++ b/doc/uml/Makefile.am @@ -24,7 +24,7 @@ PLANTUML_JAR := $(shell ../ext/plantuml_fetch.sh | tail -1) all-am: jar-check $(images) endif -# These are handle when the make target comes from the the base doc makefile. +# These are handle when the make target comes from the base doc makefile. # In cases the .SVG files are built. html-local: all-am diff --git a/example/plugins/c-api/Makefile.am b/example/plugins/c-api/Makefile.am index b63f81e675e..81a89b3a26b 100644 --- a/example/plugins/c-api/Makefile.am +++ b/example/plugins/c-api/Makefile.am @@ -26,13 +26,13 @@ example_Plugins = \ add_header.la \ append_transform.la \ basic_auth.la \ - blacklist_0.la \ - blacklist_1.la \ bnull_transform.la \ cert_update.la \ request_buffer.la \ cache_scan.la \ client_context_dump.la \ + denylist_0.la \ + denylist_1.la \ file_1.la \ hello.la \ intercept.la \ @@ -54,7 +54,7 @@ example_Plugins = \ server_transform.la \ session_hooks.la \ ssl_preaccept.la \ - ssl_sni_whitelist.la \ + ssl_sni_allowlist.la \ ssl_sni.la \ statistic.la \ thread_1.la \ @@ -71,8 +71,8 @@ endif add_header_la_SOURCES = add_header/add_header.c append_transform_la_SOURCES = append_transform/append_transform.c basic_auth_la_SOURCES = basic_auth/basic_auth.c -blacklist_0_la_SOURCES = blacklist_0/blacklist_0.c -blacklist_1_la_SOURCES = blacklist_1/blacklist_1.c +denylist_0_la_SOURCES = denylist_0/denylist_0.c +denylist_1_la_SOURCES = denylist_1/denylist_1.c bnull_transform_la_SOURCES = bnull_transform/bnull_transform.c cert_update_la_SOURCES = cert_update/cert_update.cc request_buffer_la_SOURCES = request_buffer/request_buffer.c @@ -98,7 +98,7 @@ server_push_la_SOURCES = server_push/server_push.c server_transform_la_SOURCES = server_transform/server_transform.c ssl_preaccept_la_SOURCES = ssl_preaccept/ssl_preaccept.cc ssl_sni_la_SOURCES = ssl_sni/ssl_sni.cc -ssl_sni_whitelist_la_SOURCES = ssl_sni_whitelist/ssl_sni_whitelist.cc +ssl_sni_allowlist_la_SOURCES = ssl_sni_allowlist/ssl_sni_allowlist.cc disable_http2_la_SOURCES = disable_http2/disable_http2.cc verify_cert_la_SOURCES = verify_cert/verify_cert.cc statistic_la_SOURCES = statistic/statistic.cc diff --git a/example/plugins/c-api/blacklist_1/readme.txt b/example/plugins/c-api/blacklist_1/readme.txt deleted file mode 100644 index 0e7cf271eb9..00000000000 --- a/example/plugins/c-api/blacklist_1/readme.txt +++ /dev/null @@ -1,17 +0,0 @@ -How to run the blacklist plugin -=============================== - -1. Modify blacklist.cgi to specify the location of perl and traffic server. -2. Copy blacklist.cgi, blacklist_1.so, PoweredByInktomi.gif to the directory - specified by the variable proxy.config.plugin.plugin_dir. -3. Modify plugin.config to load the blacklist plugin. - - - -About the blacklist plugin -========================== - -The blacklist plugin allows Traffic Server to compare all incoming request -origin servers with a blacklisted set of web servers. If the requested origin -server is blacklisted, Traffic Server sends the client a message saying that -access is denied. diff --git a/example/plugins/c-api/cache_scan/cache_scan.cc b/example/plugins/c-api/cache_scan/cache_scan.cc index dfdf4436100..4e0d9a952e3 100644 --- a/example/plugins/c-api/cache_scan/cache_scan.cc +++ b/example/plugins/c-api/cache_scan/cache_scan.cc @@ -287,7 +287,6 @@ handle_io(TSCont contp, TSEvent event, void * /* edata ATS_UNUSED */) cstate->write_pending = false; // the cache scan handler should call vio reenable when there is // available data - // TSVIOReenable(cstate->write_vio); return 0; } break; case TS_EVENT_VCONN_WRITE_COMPLETE: { @@ -491,7 +490,7 @@ cache_print_plugin(TSCont contp, TSEvent event, void *edata) //---------------------------------------------------------------------------- void -TSPluginInit(int /* argc ATS_UNUSED */, const char * /* argv ATS_UNUSED */ []) +TSPluginInit(int /* argc ATS_UNUSED */, const char * /* argv ATS_UNUSED */[]) { TSPluginRegistrationInfo info; diff --git a/example/plugins/c-api/client_context_dump/client_context_dump.cc b/example/plugins/c-api/client_context_dump/client_context_dump.cc index 36b20ddc4ce..034b556cec3 100644 --- a/example/plugins/c-api/client_context_dump/client_context_dump.cc +++ b/example/plugins/c-api/client_context_dump/client_context_dump.cc @@ -116,7 +116,7 @@ dump_context(const char *ca_path, const char *ck_path) // Serial number int64_t sn = 0; -#if OPENSSL_VERSION_NUMBER >= 0x010100000 +#if !defined(OPENSSL_IS_BORINGSSL) && (OPENSSL_VERSION_NUMBER >= 0x010100000) ASN1_INTEGER_get_int64(&sn, serial); #else sn = ASN1_INTEGER_get(serial); diff --git a/example/plugins/c-api/blacklist_0/blacklist_0.c b/example/plugins/c-api/denylist_0/denylist_0.c similarity index 93% rename from example/plugins/c-api/blacklist_0/blacklist_0.c rename to example/plugins/c-api/denylist_0/denylist_0.c index 5ca5179e1fb..67a4796f414 100644 --- a/example/plugins/c-api/blacklist_0/blacklist_0.c +++ b/example/plugins/c-api/denylist_0/denylist_0.c @@ -22,8 +22,8 @@ */ /* - * blacklist_0.c: - * original version of blacklist-1, now used for internal testing + * denylist_0.c: + * original version of denylist-1, now used for internal testing * * * Usage: @@ -34,7 +34,7 @@ #include #include -#define PLUGIN_NAME "blacklist_0" +#define PLUGIN_NAME "denylist_0" static char **sites; static int nsites; @@ -69,7 +69,7 @@ handle_dns(TSHttpTxn txnp, TSCont contp) } for (i = 0; i < nsites; i++) { if (strncmp(host, sites[i], host_length) == 0) { - printf("blacklisting site: %s\n", sites[i]); + printf("denylisting site: %s\n", sites[i]); TSHttpTxnHookAdd(txnp, TS_HTTP_SEND_RESPONSE_HDR_HOOK, contp); TSHandleMLocRelease(bufp, hdr_loc, url_loc); TSHandleMLocRelease(bufp, TS_NULL_MLOC, url_loc); @@ -130,7 +130,7 @@ handle_response(TSHttpTxn txnp) } static int -blacklist_plugin(TSCont contp, TSEvent event, void *edata) +denylist_plugin(TSCont contp, TSEvent event, void *edata) { TSHttpTxn txnp = (TSHttpTxn)edata; @@ -168,6 +168,6 @@ TSPluginInit(int argc, const char *argv[]) sites[i] = TSstrdup(argv[i + 1]); } - TSHttpHookAdd(TS_HTTP_OS_DNS_HOOK, TSContCreate(blacklist_plugin, NULL)); + TSHttpHookAdd(TS_HTTP_OS_DNS_HOOK, TSContCreate(denylist_plugin, NULL)); } } diff --git a/example/plugins/c-api/blacklist_1/blacklist.txt b/example/plugins/c-api/denylist_1/denylist.txt similarity index 100% rename from example/plugins/c-api/blacklist_1/blacklist.txt rename to example/plugins/c-api/denylist_1/denylist.txt diff --git a/example/plugins/c-api/blacklist_1/blacklist_1.c b/example/plugins/c-api/denylist_1/denylist_1.c similarity index 89% rename from example/plugins/c-api/blacklist_1/blacklist_1.c rename to example/plugins/c-api/denylist_1/denylist_1.c index 7b08186cb7d..0cbcec2e928 100644 --- a/example/plugins/c-api/blacklist_1/blacklist_1.c +++ b/example/plugins/c-api/denylist_1/denylist_1.c @@ -1,6 +1,6 @@ /** @file - An example plugin that denies client access to blacklisted sites (blacklist.txt). + An example plugin that denies client access to specified sites (denylist.txt). @section license License @@ -27,7 +27,7 @@ #include "ts/ts.h" #include "tscore/ink_defs.h" -#define PLUGIN_NAME "blacklist_1" +#define PLUGIN_NAME "denylist_1" #define MAX_NSITES 500 #define RETRY_TIME 10 @@ -44,7 +44,7 @@ typedef struct contp_data { enum calling_func { HANDLE_DNS, HANDLE_RESPONSE, - READ_BLACKLIST, + READ_BLOCKLIST, } cf; TSHttpTxn txnp; @@ -95,7 +95,7 @@ handle_dns(TSHttpTxn txnp, TSCont contp) } /* We need to lock the sites_mutex as that is the mutex that is - protecting the global list of all blacklisted sites. */ + protecting the global list of all denylisted sites. */ if (TSMutexLockTry(sites_mutex) != TS_SUCCESS) { TSDebug(PLUGIN_NAME, "Unable to get lock. Will retry after some time"); TSHandleMLocRelease(bufp, hdr_loc, url_loc); @@ -107,9 +107,9 @@ handle_dns(TSHttpTxn txnp, TSCont contp) for (i = 0; i < nsites; i++) { if (strncmp(host, sites[i], host_length) == 0) { if (log) { - TSTextLogObjectWrite(log, "blacklisting site: %s", sites[i]); + TSTextLogObjectWrite(log, "denylisting site: %s", sites[i]); } else { - TSDebug(PLUGIN_NAME, "blacklisting site: %s", sites[i]); + TSDebug(PLUGIN_NAME, "denylisting site: %s", sites[i]); } TSHttpTxnHookAdd(txnp, TS_HTTP_SEND_RESPONSE_HDR_HOOK, contp); TSHandleMLocRelease(bufp, hdr_loc, url_loc); @@ -174,13 +174,13 @@ handle_response(TSHttpTxn txnp, TSCont contp ATS_UNUSED) } static void -read_blacklist(TSCont contp) +read_denylist(TSCont contp) { - char blacklist_file[1024]; + char denylist_file[1024]; TSFile file; - sprintf(blacklist_file, "%s/blacklist.txt", TSPluginDirGet()); - file = TSfopen(blacklist_file, "r"); + sprintf(denylist_file, "%s/denylist.txt", TSPluginDirGet()); + file = TSfopen(denylist_file, "r"); nsites = 0; /* If the Mutex lock is not successful try again in RETRY_TIME */ @@ -215,7 +215,7 @@ read_blacklist(TSCont contp) TSfclose(file); } else { - TSError("[%s] Unable to open %s", PLUGIN_NAME, blacklist_file); + TSError("[%s] Unable to open %s", PLUGIN_NAME, denylist_file); TSError("[%s] All sites will be allowed", PLUGIN_NAME); } @@ -223,7 +223,7 @@ read_blacklist(TSCont contp) } static int -blacklist_plugin(TSCont contp, TSEvent event, void *edata) +denylist_plugin(TSCont contp, TSEvent event, void *edata) { TSHttpTxn txnp; cdata *cd; @@ -276,7 +276,7 @@ blacklist_plugin(TSCont contp, TSEvent event, void *edata) break; } } else { - read_blacklist(contp); + read_denylist(contp); return 0; } default: @@ -291,7 +291,7 @@ handle_txn_start(TSCont contp ATS_UNUSED, TSHttpTxn txnp) TSCont txn_contp; cdata *cd; - txn_contp = TSContCreate((TSEventFunc)blacklist_plugin, TSMutexCreate()); + txn_contp = TSContCreate((TSEventFunc)denylist_plugin, TSMutexCreate()); /* create the data that'll be associated with the continuation */ cd = (cdata *)TSmalloc(sizeof(cdata)); TSContDataSet(txn_contp, cd); @@ -319,8 +319,8 @@ TSPluginInit(int argc ATS_UNUSED, const char *argv[] ATS_UNUSED) TSError("[%s] Plugin registration failed", PLUGIN_NAME); } - /* create an TSTextLogObject to log blacklisted requests to */ - error = TSTextLogObjectCreate("blacklist", TS_LOG_MODE_ADD_TIMESTAMP, &log); + /* create an TSTextLogObject to log denied requests to */ + error = TSTextLogObjectCreate("denylist", TS_LOG_MODE_ADD_TIMESTAMP, &log); if (!log || error == TS_ERROR) { TSDebug(PLUGIN_NAME, "error while creating log"); } @@ -332,8 +332,8 @@ TSPluginInit(int argc ATS_UNUSED, const char *argv[] ATS_UNUSED) sites[i] = NULL; } - global_contp = TSContCreate(blacklist_plugin, sites_mutex); - read_blacklist(global_contp); + global_contp = TSContCreate(denylist_plugin, sites_mutex); + read_denylist(global_contp); /*TSHttpHookAdd (TS_HTTP_OS_DNS_HOOK, contp); */ TSHttpHookAdd(TS_HTTP_TXN_START_HOOK, global_contp); diff --git a/example/plugins/c-api/denylist_1/readme.txt b/example/plugins/c-api/denylist_1/readme.txt new file mode 100644 index 00000000000..70a75fdc686 --- /dev/null +++ b/example/plugins/c-api/denylist_1/readme.txt @@ -0,0 +1,17 @@ +How to run the denylist plugin +=============================== + +1. Modify denylist.cgi to specify the location of perl and traffic server. +2. Copy denylist.cgi, denylist_1.so, PoweredByInktomi.gif to the directory + specified by the variable proxy.config.plugin.plugin_dir. +3. Modify plugin.config to load the denylist plugin. + + + +About the denylist plugin +========================== + +The denylist plugin allows Traffic Server to compare all incoming request +origin servers with a deny-listed set of web servers. If the requested origin +server is listed, Traffic Server sends the client a message saying that +access is denied. diff --git a/example/plugins/c-api/disable_http2/disable_http2.cc b/example/plugins/c-api/disable_http2/disable_http2.cc index 0169bc4f05c..036c991a1b1 100644 --- a/example/plugins/c-api/disable_http2/disable_http2.cc +++ b/example/plugins/c-api/disable_http2/disable_http2.cc @@ -42,7 +42,7 @@ int CB_SNI(TSCont contp, TSEvent, void *cb_data) { auto vc = static_cast(cb_data); - TSSslConnection ssl_conn = TSVConnSSLConnectionGet(vc); + TSSslConnection ssl_conn = TSVConnSslConnectionGet(vc); auto *ssl = reinterpret_cast(ssl_conn); char const *sni = SSL_get_servername(ssl, TLSEXT_NAMETYPE_host_name); if (sni) { diff --git a/example/plugins/c-api/intercept/intercept.cc b/example/plugins/c-api/intercept/intercept.cc index c793e19b96f..03bf86da29c 100644 --- a/example/plugins/c-api/intercept/intercept.cc +++ b/example/plugins/c-api/intercept/intercept.cc @@ -1,6 +1,6 @@ /** @file - an example hello world plugin + an example intercept plugin @section license License @@ -40,8 +40,7 @@ // This plugin intercepts all cache misses and proxies them to a separate server // that is assumed to be running on localhost:60000. The plugin does no HTTP // processing at all, it simply shuffles data until the client closes the -// request. The TSQA test test-server-intercept exercises this plugin. You can -// enable extensive logging with the "intercept" diagnostic tag. +// request. You can enable extensive logging with the "intercept" diagnostic tag. #define PLUGIN_NAME "intercept" #define PORT 60000 @@ -61,7 +60,7 @@ static TSCont TxnHook; static TSCont InterceptHook; -static int InterceptInterceptionHook(TSCont contp, TSEvent event, void *edata); +static int InterceptInterceptHook(TSCont contp, TSEvent event, void *edata); static int InterceptTxnHook(TSCont contp, TSEvent event, void *edata); // We are going to stream data between Traffic Server and an @@ -267,7 +266,7 @@ InterceptTransferData(InterceptIO *from, InterceptIO *to) // starts with TS_EVENT_NET_ACCEPT, and then continues with // TSVConn events. static int -InterceptInterceptionHook(TSCont contp, TSEvent event, void *edata) +InterceptInterceptHook(TSCont contp, TSEvent event, void *edata) { argument_type arg(edata); @@ -515,7 +514,7 @@ InterceptTxnHook(TSCont contp, TSEvent event, void *edata) switch (event) { case TS_EVENT_HTTP_CACHE_LOOKUP_COMPLETE: { if (InterceptShouldInterceptRequest(arg.txn)) { - TSCont c = InterceptContCreate(InterceptInterceptionHook, TSMutexCreate(), arg.txn); + TSCont c = InterceptContCreate(InterceptInterceptHook, TSMutexCreate(), arg.txn); VDEBUG("intercepting origin server request for txn=%p, cont=%p", arg.txn, c); TSHttpTxnServerIntercept(c, arg.txn); @@ -534,7 +533,7 @@ InterceptTxnHook(TSCont contp, TSEvent event, void *edata) } void -TSPluginInit(int /* argc */, const char * /* argv */ []) +TSPluginInit(int /* argc */, const char * /* argv */[]) { TSPluginRegistrationInfo info; @@ -549,9 +548,9 @@ TSPluginInit(int /* argc */, const char * /* argv */ []) // XXX accept hostname and port arguments TxnHook = InterceptContCreate(InterceptTxnHook, nullptr, nullptr); - InterceptHook = InterceptContCreate(InterceptInterceptionHook, nullptr, nullptr); + InterceptHook = InterceptContCreate(InterceptInterceptHook, nullptr, nullptr); // Wait until after the cache lookup to decide whether to - // intercept a request. For cache hits we will never intercept. + // intercept a request. For cache hits, we will never intercept. TSHttpHookAdd(TS_HTTP_CACHE_LOOKUP_COMPLETE_HOOK, TxnHook); } diff --git a/example/plugins/c-api/null_transform/null_transform.c b/example/plugins/c-api/null_transform/null_transform.c index b350c3f076f..992841aa2f9 100644 --- a/example/plugins/c-api/null_transform/null_transform.c +++ b/example/plugins/c-api/null_transform/null_transform.c @@ -92,9 +92,7 @@ handle_transform(TSCont contp) data->output_buffer = TSIOBufferCreate(); data->output_reader = TSIOBufferReaderAlloc(data->output_buffer); TSDebug(PLUGIN_NAME, "\tWriting %" PRId64 " bytes on VConn", TSVIONBytesGet(input_vio)); - // data->output_vio = TSVConnWrite(output_conn, contp, data->output_reader, INT32_MAX); data->output_vio = TSVConnWrite(output_conn, contp, data->output_reader, INT64_MAX); - // data->output_vio = TSVConnWrite(output_conn, contp, data->output_reader, TSVIONBytesGet(input_vio)); TSContDataSet(contp, data); } diff --git a/example/plugins/c-api/passthru/passthru.cc b/example/plugins/c-api/passthru/passthru.cc index 8912e2961be..887dac000ca 100644 --- a/example/plugins/c-api/passthru/passthru.cc +++ b/example/plugins/c-api/passthru/passthru.cc @@ -315,7 +315,7 @@ PassthruListen() } void -TSPluginInit(int /* argc */, const char * /* argv */ []) +TSPluginInit(int /* argc */, const char * /* argv */[]) { TSPluginRegistrationInfo info = {PLUGIN_NAME, "Apache Software Foundation", "dev@trafficserver.apache.org"}; diff --git a/example/plugins/c-api/remap/remap.cc b/example/plugins/c-api/remap/remap.cc index 2f437407b56..f46991eda49 100644 --- a/example/plugins/c-api/remap/remap.cc +++ b/example/plugins/c-api/remap/remap.cc @@ -267,9 +267,9 @@ TSRemapDoRemap(void *ih, TSHttpTxn rh, TSRemapRequestInfo *rri) } // How to store plugin private arguments inside Traffic Server request processing block. - if (TSHttpTxnArgIndexReserve("remap_example", "Example remap plugin", &arg_index) == TS_SUCCESS) { + if (TSUserArgIndexReserve(TS_USER_ARGS_TXN, "remap_example", "Example remap plugin", &arg_index) == TS_SUCCESS) { TSDebug(PLUGIN_NAME, "Save processing counter %" PRIu64 " inside request processing block\n", _processing_counter); - TSHttpTxnArgSet(rh, arg_index, (void *)_processing_counter); // save counter + TSUserArgSet(rh, arg_index, (void *)_processing_counter); // save counter } // How to cancel request processing and return error message to the client // We will do it every other request @@ -321,7 +321,7 @@ TSRemapDoRemap(void *ih, TSHttpTxn rh, TSRemapRequestInfo *rri) void TSRemapOSResponse(void *ih ATS_UNUSED, TSHttpTxn rh, int os_response_type) { - void *data = TSHttpTxnArgGet(rh, arg_index); // read counter (we store it in TSRemapDoRemap function call) + void *data = TSUserArgGet(rh, arg_index); // read counter (we store it in TSRemapDoRemap function call) int request_id = data ? static_cast(data)[0] : -1; TSDebug(PLUGIN_NAME, "Read processing counter %d from request processing block\n", request_id); diff --git a/example/plugins/c-api/server_push/server_push.c b/example/plugins/c-api/server_push/server_push.c index 41203f532db..c921269aa1b 100644 --- a/example/plugins/c-api/server_push/server_push.c +++ b/example/plugins/c-api/server_push/server_push.c @@ -36,7 +36,6 @@ #include #include "ts/ts.h" -#include "ts/experimental.h" #include "tscore/ink_defs.h" const char *PLUGIN_NAME = "server_push"; diff --git a/example/plugins/c-api/server_transform/server_transform.c b/example/plugins/c-api/server_transform/server_transform.c index 67f26937443..c4f29bdf6e1 100644 --- a/example/plugins/c-api/server_transform/server_transform.c +++ b/example/plugins/c-api/server_transform/server_transform.c @@ -421,7 +421,6 @@ transform_read_status_event(TSCont contp, TransformData *data, TSEvent event, vo buf_ptr = (char *)buf_ptr + read_ndone; } } - // data->content_length = ntohl(data->content_length); return transform_read(contp, data); } return transform_bypass(contp, data); diff --git a/example/plugins/c-api/ssl_sni/ssl_sni.cc b/example/plugins/c-api/ssl_sni/ssl_sni.cc index f835edc3423..a0b676be081 100644 --- a/example/plugins/c-api/ssl_sni/ssl_sni.cc +++ b/example/plugins/c-api/ssl_sni/ssl_sni.cc @@ -49,7 +49,7 @@ int CB_servername(TSCont /* contp */, TSEvent /* event */, void *edata) { TSVConn ssl_vc = reinterpret_cast(edata); - TSSslConnection sslobj = TSVConnSSLConnectionGet(ssl_vc); + TSSslConnection sslobj = TSVConnSslConnectionGet(ssl_vc); SSL *ssl = reinterpret_cast(sslobj); const char *servername = SSL_get_servername(ssl, TLSEXT_NAMETYPE_host_name); if (servername != nullptr) { diff --git a/example/plugins/c-api/ssl_sni_whitelist/ssl_sni_whitelist.cc b/example/plugins/c-api/ssl_sni_allowlist/ssl_sni_allowlist.cc similarity index 88% rename from example/plugins/c-api/ssl_sni_whitelist/ssl_sni_whitelist.cc rename to example/plugins/c-api/ssl_sni_allowlist/ssl_sni_allowlist.cc index f0217b06c3d..b2e969c384a 100644 --- a/example/plugins/c-api/ssl_sni_whitelist/ssl_sni_whitelist.cc +++ b/example/plugins/c-api/ssl_sni_allowlist/ssl_sni_allowlist.cc @@ -1,6 +1,6 @@ /** @file - SSL SNI white list plugin + SSL SNI allow list plugin If the server name and IP address are not in the ssl_multicert.config go ahead and blind tunnel it. @@ -31,16 +31,16 @@ #include -#define PLUGIN_NAME "ssl_sni_whitelist" +#define PLUGIN_NAME "ssl_sni_allowlist" #define PCP "[" PLUGIN_NAME "] " namespace { int -CB_servername_whitelist(TSCont /* contp */, TSEvent /* event */, void *edata) +CB_servername_allowlist(TSCont /* contp */, TSEvent /* event */, void *edata) { TSVConn ssl_vc = reinterpret_cast(edata); - TSSslConnection sslobj = TSVConnSSLConnectionGet(ssl_vc); + TSSslConnection sslobj = TSVConnSslConnectionGet(ssl_vc); SSL *ssl = reinterpret_cast(sslobj); const char *servername = SSL_get_servername(ssl, TLSEXT_NAMETYPE_host_name); @@ -58,7 +58,7 @@ CB_servername_whitelist(TSCont /* contp */, TSEvent /* event */, void *edata) } } if (do_blind_tunnel) { - TSDebug("skh", "SNI callback: do blind tunnel for %s", servername); + TSDebug(PLUGIN_NAME, "SNI callback: do blind tunnel for %s", servername); TSVConnTunnel(ssl_vc); return TS_SUCCESS; // Don't re-enable so we interrupt processing } @@ -84,7 +84,7 @@ TSPluginInit(int argc, const char *argv[]) TSError(PCP "registration failed"); } else if (TSTrafficServerVersionGetMajor() < 2) { TSError(PCP "requires Traffic Server 2.0 or later"); - } else if (nullptr == (cb_sni = TSContCreate(&CB_servername_whitelist, TSMutexCreate()))) { + } else if (nullptr == (cb_sni = TSContCreate(&CB_servername_allowlist, TSMutexCreate()))) { TSError(PCP "Failed to create SNI callback"); } else { TSHttpHookAdd(TS_SSL_CERT_HOOK, cb_sni); diff --git a/example/plugins/c-api/statistic/statistic.cc b/example/plugins/c-api/statistic/statistic.cc index 09ee0477bda..f76f736a922 100644 --- a/example/plugins/c-api/statistic/statistic.cc +++ b/example/plugins/c-api/statistic/statistic.cc @@ -36,7 +36,7 @@ #define PLUGIN_NAME "statistics" void -TSPluginInit(int /* argc */, const char * /* argv */ []) +TSPluginInit(int /* argc */, const char * /* argv */[]) { TSPluginRegistrationInfo info; diff --git a/example/plugins/c-api/vconn_args/vconn_args.cc b/example/plugins/c-api/vconn_args/vconn_args.cc index 2185dbe1f01..18620ac8a8c 100644 --- a/example/plugins/c-api/vconn_args/vconn_args.cc +++ b/example/plugins/c-api/vconn_args/vconn_args.cc @@ -37,10 +37,10 @@ vconn_arg_handler(TSCont contp, TSEvent event, void *edata) case TS_EVENT_VCONN_START: { // Testing set argument int idx = 0; - while (TSVConnArgIndexReserve(PLUGIN_NAME, "test", &idx) == TS_SUCCESS) { + while (TSUserArgIndexReserve(TS_USER_ARGS_VCONN, PLUGIN_NAME, "test", &idx) == TS_SUCCESS) { char *buf = static_cast(TSmalloc(64)); snprintf(buf, 64, "Test Arg Idx %d", idx); - TSVConnArgSet(ssl_vc, idx, (void *)buf); + TSUserArgSet(ssl_vc, idx, (void *)buf); TSDebug(PLUGIN_NAME, "Successfully reserve and set arg #%d", idx); } last_arg = idx; @@ -52,7 +52,7 @@ vconn_arg_handler(TSCont contp, TSEvent event, void *edata) while (idx <= last_arg) { const char *name = nullptr; const char *desc = nullptr; - if (TSVConnArgIndexLookup(idx, &name, &desc) == TS_SUCCESS) { + if (TSUserArgIndexLookup(TS_USER_ARGS_VCONN, idx, &name, &desc) == TS_SUCCESS) { TSDebug(PLUGIN_NAME, "Successful lookup for arg #%d: [%s] [%s]", idx, name, desc); } else { TSDebug(PLUGIN_NAME, "Failed lookup for arg #%d", idx); @@ -65,7 +65,7 @@ vconn_arg_handler(TSCont contp, TSEvent event, void *edata) // Testing argget and delete int idx = 0; while (idx <= last_arg) { - char *buf = static_cast(TSVConnArgGet(ssl_vc, idx)); + char *buf = static_cast(TSUserArgGet(ssl_vc, idx)); if (buf) { TSDebug(PLUGIN_NAME, "Successfully retrieve vconn arg #%d: %s", idx, buf); TSfree(buf); diff --git a/example/plugins/c-api/verify_cert/verify_cert.cc b/example/plugins/c-api/verify_cert/verify_cert.cc index e245fdb4436..4682379c09f 100644 --- a/example/plugins/c-api/verify_cert/verify_cert.cc +++ b/example/plugins/c-api/verify_cert/verify_cert.cc @@ -62,7 +62,7 @@ int CB_clientcert(TSCont /* contp */, TSEvent /* event */, void *edata) { TSVConn ssl_vc = reinterpret_cast(edata); - TSSslConnection sslobj = TSVConnSSLConnectionGet(ssl_vc); + TSSslConnection sslobj = TSVConnSslConnectionGet(ssl_vc); SSL *ssl = reinterpret_cast(sslobj); X509 *cert = SSL_get_peer_certificate(ssl); TSDebug(PLUGIN_NAME, "plugin verify_cert verifying client certificate"); diff --git a/example/plugins/cpp-api/async_http_fetch_streaming/AsyncHttpFetchStreaming.cc b/example/plugins/cpp-api/async_http_fetch_streaming/AsyncHttpFetchStreaming.cc index f953ab08259..d7f4f27062e 100644 --- a/example/plugins/cpp-api/async_http_fetch_streaming/AsyncHttpFetchStreaming.cc +++ b/example/plugins/cpp-api/async_http_fetch_streaming/AsyncHttpFetchStreaming.cc @@ -80,7 +80,7 @@ class InterceptInstaller : public GlobalPlugin }; void -TSPluginInit(int /* argc ATS_UNUSED */, const char * /* argv ATS_UNUSED */ []) +TSPluginInit(int /* argc ATS_UNUSED */, const char * /* argv ATS_UNUSED */[]) { if (!RegisterGlobalPlugin("CPP_Example_AsyncHttpFetchStreaming", "apache", "dev@trafficserver.apache.org")) { return; diff --git a/example/plugins/cpp-api/boom/boom.cc b/example/plugins/cpp-api/boom/boom.cc index 8db6059625b..a312e7f2043 100644 --- a/example/plugins/cpp-api/boom/boom.cc +++ b/example/plugins/cpp-api/boom/boom.cc @@ -84,9 +84,6 @@ const std::string DEFAULT_ERROR_FILE = "default"; // default.html will be search // Default error response TBD when the default response will be used const std::string DEFAULT_ERROR_RESPONSE = "

This page will be back soon

"; -// Default HTTP status code to use after booming -// const int DEFAULT_BOOM_HTTP_STATUS_CODE = 200; - // Default HTTP status string to use after booming const std::string DEFAULT_BOOM_HTTP_STATUS = "OK (BOOM)"; diff --git a/example/plugins/cpp-api/intercept/intercept.cc b/example/plugins/cpp-api/intercept/intercept.cc index da615cb077b..332f1047fd5 100644 --- a/example/plugins/cpp-api/intercept/intercept.cc +++ b/example/plugins/cpp-api/intercept/intercept.cc @@ -58,7 +58,7 @@ class InterceptInstaller : public GlobalPlugin }; void -TSPluginInit(int /* argc ATS_UNUSED */, const char * /* argv ATS_UNUSED */ []) +TSPluginInit(int /* argc ATS_UNUSED */, const char * /* argv ATS_UNUSED */[]) { if (!RegisterGlobalPlugin("CPP_Example_Intercept", "apache", "dev@trafficserver.apache.org")) { return; diff --git a/example/plugins/cpp-api/logger_example/LoggerExample.cc b/example/plugins/cpp-api/logger_example/LoggerExample.cc index f6b1bfe3e74..9e9561ab766 100644 --- a/example/plugins/cpp-api/logger_example/LoggerExample.cc +++ b/example/plugins/cpp-api/logger_example/LoggerExample.cc @@ -33,7 +33,7 @@ using std::string; namespace { -Logger log; +Logger logger; GlobalPlugin *plugin; } // namespace @@ -61,7 +61,7 @@ class GlobalHookPlugin : public GlobalPlugin void handleReadRequestHeadersPostRemap(Transaction &transaction) override { - LOG_DEBUG(log, + LOG_DEBUG(logger, "handleReadRequestHeadersPostRemap.\n" "\tRequest URL: %s\n" "\tRequest Path: %s\n" @@ -74,23 +74,23 @@ class GlobalHookPlugin : public GlobalPlugin // Next, to demonstrate how you can change logging levels: if (transaction.getClientRequest().getUrl().getPath() == "change_log_level") { if (transaction.getClientRequest().getUrl().getQuery().find("level=debug") != string::npos) { - log.setLogLevel(Logger::LOG_LEVEL_DEBUG); - LOG_DEBUG(log, "Changed log level to DEBUG"); + logger.setLogLevel(Logger::LOG_LEVEL_DEBUG); + LOG_DEBUG(logger, "Changed log level to DEBUG"); } else if (transaction.getClientRequest().getUrl().getQuery().find("level=info") != string::npos) { - log.setLogLevel(Logger::LOG_LEVEL_INFO); - LOG_INFO(log, "Changed log level to INFO"); + logger.setLogLevel(Logger::LOG_LEVEL_INFO); + LOG_INFO(logger, "Changed log level to INFO"); } else if (transaction.getClientRequest().getUrl().getQuery().find("level=error") != string::npos) { - log.setLogLevel(Logger::LOG_LEVEL_ERROR); - LOG_ERROR(log, "Changed log level to ERROR"); + logger.setLogLevel(Logger::LOG_LEVEL_ERROR); + LOG_ERROR(logger, "Changed log level to ERROR"); } } // One drawback to using the Traffic Server Text Loggers is that you're limited in the size of the log // lines, this limit is now set at 8kb for atscppapi, but this limit might be removed in the future. - LOG_INFO(log, "This message will be dropped (see error.log) because it's just too big: %s", big_buffer_14kb_); + LOG_INFO(logger, "This message will be dropped (see error.log) because it's just too big: %s", big_buffer_14kb_); // This should work though: - LOG_INFO(log, "%s", big_buffer_6kb_); + LOG_INFO(logger, "%s", big_buffer_6kb_); transaction.resume(); } @@ -119,24 +119,24 @@ TSPluginInit(int argc ATSCPPAPI_UNUSED, const char *argv[] ATSCPPAPI_UNUSED) // The fifth argument is to enable log rolling, this is enabled by default. // The sixth argument is the frequency in which we will roll the logs, 300 seconds is very low, // the default for this argument is 3600. - log.init("logger_example", true, true, Logger::LOG_LEVEL_DEBUG, true, 300); + logger.init("logger_example", true, true, Logger::LOG_LEVEL_DEBUG, true, 300); // Now that we've initialized a logger we can do all kinds of fun things on it: - log.setRollingEnabled(true); // already done via log.init, just an example. - log.setRollingIntervalSeconds(300); // already done via log.init + logger.setRollingEnabled(true); // already done via log.init, just an example. + logger.setRollingIntervalSeconds(300); // already done via log.init // You have two ways to log to a logger, you can log directly on the object itself: - log.logInfo("Hello World from: %s", argv[0]); + logger.logInfo("Hello World from: %s", argv[0]); // Alternatively you can take advantage of the super helper macros for logging // that will include the file, function, and line number automatically as part // of the log message: - LOG_INFO(log, "Hello World with more info from: %s", argv[0]); + LOG_INFO(logger, "Hello World with more info from: %s", argv[0]); // This will hurt performance, but it's an option that's always available to you // to force flush the logs. Otherwise TrafficServer will flush the logs around // once every second. You should really avoid flushing the log unless it's really necessary. - log.flush(); + logger.flush(); plugin = new GlobalHookPlugin(); } diff --git a/example/plugins/cpp-api/websocket/WSBuffer.cc b/example/plugins/cpp-api/websocket/WSBuffer.cc index a53ac6750a0..dce462dfa43 100644 --- a/example/plugins/cpp-api/websocket/WSBuffer.cc +++ b/example/plugins/cpp-api/websocket/WSBuffer.cc @@ -113,7 +113,7 @@ WSBuffer::read_buffered_message(std::string &message, int &code) if (avail < 4 + mask_len) { // 2 + 2 + length bytes + mask. return false; } - msg_len = ntohs(*(uint16_t *)(ws_buf_.data() + 2)); + msg_len = ntohs(*reinterpret_cast(ws_buf_.data() + 2)); pos = 4; } else if (msg_len == WS_64BIT_LEN) { if (avail < 10 + mask_len) { // 2 + 8 length bytes + mask. diff --git a/example/plugins/lua-api/connect_geoip.lua b/example/plugins/lua-api/connect_geoip.lua index a1af55baa44..7e734970fa9 100644 --- a/example/plugins/lua-api/connect_geoip.lua +++ b/example/plugins/lua-api/connect_geoip.lua @@ -14,7 +14,6 @@ -- See the License for the specific language governing permissions and -- limitations under the License. - -- This example depends on "luajit-geoip". -- It illustrates how to connect to GeoIP and use it to look up country of an IP address. -- It can be used in plugin.config with the lua plugin. @@ -22,15 +21,15 @@ -- Setup Instructions -- 1) install GeoIP - 1.6.12 -- 2) install GeoIP legacy country database - https://dev.maxmind.com/geoip/legacy/install/country/ --- 3) install luajit-geoip (https://github.com/leafo/luajit-geoip) --- or just copy geoip/init.lua from the repo to /usr/local/share/lua/5.1/geoip/init.lua --- 4) You may need to make change so luajit-geoip does ffi.load() on /usr/local/lib/libGeoIP.so +-- 3) install luajit-geoip (https://github.com/leafo/luajit-geoip) +-- or just copy geoip/init.lua from the repo to /usr/local/share/lua/5.1/geoip/init.lua +-- 4) You may need to make change so luajit-geoip does ffi.load() on /usr/local/lib/libGeoIP.so -ts.add_package_path('/usr/local/share/lua/5.1/?.lua') +ts.add_package_path("/usr/local/share/lua/5.1/?.lua") -local geoip = require 'geoip' +local geoip = require "geoip" function do_global_send_response() local res = geoip.lookup_addr("8.8.8.8") - ts.client_response.header['X-Country'] = res.country_code + ts.client_response.header["X-Country"] = res.country_code end diff --git a/example/plugins/lua-api/connect_redis.lua b/example/plugins/lua-api/connect_redis.lua index 8d34f15c2f9..dddb2af3dce 100644 --- a/example/plugins/lua-api/connect_redis.lua +++ b/example/plugins/lua-api/connect_redis.lua @@ -14,10 +14,9 @@ -- See the License for the specific language governing permissions and -- limitations under the License. - -- This example depends on "redis-lua" 2.0.4 - https://github.com/nrk/redis-lua -- And redis-lua depends on LuaSocket v3.0-rc1 - https://github.com/diegonehab/luasocket --- It illustrates how to connect to redis and retrieve a key value. +-- It illustrates how to connect to redis and retrieve a key value. -- It can be used in plugin.config with the lua plugin. -- unix domain socket has better performance and so we should set up local redis to use that @@ -31,19 +30,19 @@ -- 6. sudo -u nobody redis-server /etc/redis/redis.conf -- 7. sudo -u nobody redis-cli -s /var/run/redis/redis.sock set mykey helloworld -ts.add_package_cpath('/usr/local/lib/lua/5.1/socket/?.so;/usr/local/lib/lua/5.1/mime/?.so') -ts.add_package_path('/usr/local/share/lua/5.1/?.lua;/usr/local/share/lua/5.1/socket/?.lua') +ts.add_package_cpath("/usr/local/lib/lua/5.1/socket/?.so;/usr/local/lib/lua/5.1/mime/?.so") +ts.add_package_path("/usr/local/share/lua/5.1/?.lua;/usr/local/share/lua/5.1/socket/?.lua") -local redis = require 'redis' +local redis = require "redis" -- not connecting to redis default port -- local client = redis.connect('127.0.0.1', 6379) -- connecting to unix domain socket -local client = redis.connect('unix:///var/run/redis/redis.sock') +local client = redis.connect("unix:///var/run/redis/redis.sock") function do_global_send_response() local response = client:ping() - local value = client:get('mykey') - ts.client_response.header['X-Redis-Ping'] = tostring(response) - ts.client_response.header['X-Redis-MyKey'] = value + local value = client:get("mykey") + ts.client_response.header["X-Redis-Ping"] = tostring(response) + ts.client_response.header["X-Redis-MyKey"] = value end diff --git a/example/plugins/lua-api/sorted_query_params.lua b/example/plugins/lua-api/sorted_query_params.lua index e4a45e955b3..ec6f24748bc 100644 --- a/example/plugins/lua-api/sorted_query_params.lua +++ b/example/plugins/lua-api/sorted_query_params.lua @@ -14,40 +14,44 @@ -- See the License for the specific language governing permissions and -- limitations under the License. - -- This script is for sorting query parameters on incoming requests before doing cache lookup -- so we can get better cache hit ratio -- It can be used in remap.config for a remap rule with the lua plugin. -function pairsByKeys (t, f) +function pairsByKeys(t, f) local a = {} - for n in pairs(t) do table.insert(a, n) end + for n in pairs(t) do + table.insert(a, n) + end table.sort(a, f) - local i = 0 -- iterator variable - local iter = function () -- iterator function + local i = 0 -- iterator variable + local iter = function() + -- iterator function i = i + 1 - if a[i] == nil then return nil - else return a[i], t[a[i]] + if a[i] == nil then + return nil + else + return a[i], t[a[i]] end end return iter end function do_remap() - t = {} - s = ts.client_request.get_uri_args() or '' + t = {} + s = ts.client_request.get_uri_args() or "" -- Original String i = 1 for k, v in string.gmatch(s, "([0-9a-zA-Z-_]+)=([0-9a-zA-Z-_]+)") do t[k] = v end - output = '' + output = "" for name, line in pairsByKeys(t) do - output = output .. '&' .. name .. '=' .. line + output = output .. "&" .. name .. "=" .. line end output = string.sub(output, 2) - -- Modified String + -- Modified String ts.client_request.set_uri_args(output) return 0 -end +end diff --git a/example/plugins/lua-api/uncompress.lua b/example/plugins/lua-api/uncompress.lua index c80ff228ab8..788e5c985e3 100644 --- a/example/plugins/lua-api/uncompress.lua +++ b/example/plugins/lua-api/uncompress.lua @@ -14,39 +14,38 @@ -- See the License for the specific language governing permissions and -- limitations under the License. - --- This example depends on "lua-zlib". +-- This example depends on "lua-zlib". -- It uncompresses a gzipped content body and prints it out in debug log. -- It can be added in remap.config for a remap rule with the lua plugin. -- Setup Instructions -- 1) install lua-zlib - v1.2 -ts.add_package_cpath('/usr/lib/lua/5.1/?.so') +ts.add_package_cpath("/usr/lib/lua/5.1/?.so") local zlib = require "zlib" function upper_transform(data, eos) - ts.ctx['text'] = ts.ctx['text'] .. data - - if eos ==1 then - local stream = zlib.inflate() - local inflated, eof, bytes_in, bytes_out = stream(ts.ctx['text']) - if (eof == true) then - ts.debug("==== eof ====") - end - ts.debug("==== bytes_in: "..(bytes_in or '')) - ts.debug("==== bytes_out:"..(bytes_out or '')) - ts.debug("==== uncompressed data begin ===") - ts.debug(inflated or 'no data') - ts.debug("==== uncompressed data end ===") - end + ts.ctx["text"] = ts.ctx["text"] .. data - return string.upper(data), eos + if eos == 1 then + local stream = zlib.inflate() + local inflated, eof, bytes_in, bytes_out = stream(ts.ctx["text"]) + if (eof == true) then + ts.debug("==== eof ====") + end + ts.debug("==== bytes_in: " .. (bytes_in or "")) + ts.debug("==== bytes_out:" .. (bytes_out or "")) + ts.debug("==== uncompressed data begin ===") + ts.debug(inflated or "no data") + ts.debug("==== uncompressed data end ===") + end + + return string.upper(data), eos end function do_remap() - ts.hook(TS_LUA_RESPONSE_TRANSFORM, upper_transform) - ts.ctx['text'] = '' - return 0 + ts.hook(TS_LUA_RESPONSE_TRANSFORM, upper_transform) + ts.ctx["text"] = "" + return 0 end diff --git a/include/shared/overridable_txn_vars.h b/include/shared/overridable_txn_vars.h new file mode 100644 index 00000000000..4b9111cf360 --- /dev/null +++ b/include/shared/overridable_txn_vars.h @@ -0,0 +1,35 @@ +/** @file + + Map of transaction overridable configuration variables and names. + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#pragma once + +#include +#include "HTTP.h" +#include "HttpConnectionCount.h" + +namespace ts +{ +/// Map of configuration variable name to enum and type for all transaction overridable variables. +extern const std::unordered_map> + Overridable_Txn_Vars; +} // namespace ts diff --git a/include/ts/apidefs.h.in b/include/ts/apidefs.h.in index a33cc192d78..57073351c10 100644 --- a/include/ts/apidefs.h.in +++ b/include/ts/apidefs.h.in @@ -196,6 +196,7 @@ typedef enum { TS_HTTP_STATUS_PRECONDITION_REQUIRED = 428, TS_HTTP_STATUS_TOO_MANY_REQUESTS = 429, TS_HTTP_STATUS_REQUEST_HEADER_FIELDS_TOO_LARGE = 431, + TS_HTTP_STATUS_UNAVAILABLE_FOR_LEGAL_REASONS = 451, TS_HTTP_STATUS_INTERNAL_SERVER_ERROR = 500, TS_HTTP_STATUS_NOT_IMPLEMENTED = 501, TS_HTTP_STATUS_BAD_GATEWAY = 502, @@ -599,22 +600,6 @@ typedef enum { #ifndef _HTTP_PROXY_API_ENUMS_H_ #define _HTTP_PROXY_API_ENUMS_H_ -/// Server session sharing values - match -/// Must be identical to definition in HttpProxyAPIEnums.h -typedef enum { - TS_SERVER_SESSION_SHARING_MATCH_NONE, - TS_SERVER_SESSION_SHARING_MATCH_BOTH, - TS_SERVER_SESSION_SHARING_MATCH_IP, - TS_SERVER_SESSION_SHARING_MATCH_HOST -} TSServerSessionSharingMatchType; - -/// Server session sharing values - pool -/// Must be identical to definition in HttpProxyAPIEnums.h -typedef enum { - TS_SERVER_SESSION_SHARING_POOL_GLOBAL, - TS_SERVER_SESSION_SHARING_POOL_THREAD, -} TSServerSessionSharingPoolType; - /// Values for per server outbound connection tracking group definition. /// See proxy.config.http.per_server.match typedef enum { @@ -815,6 +800,7 @@ typedef enum { TS_CONFIG_SSL_CLIENT_SNI_POLICY, TS_CONFIG_SSL_CLIENT_PRIVATE_KEY_FILENAME, TS_CONFIG_SSL_CLIENT_CA_CERT_FILENAME, + TS_CONFIG_HTTP_HOST_RESOLUTION_PREFERENCE, TS_CONFIG_LAST_ENTRY } TSOverridableConfigKey; @@ -824,9 +810,7 @@ typedef enum { TS_THREAD_POOL_NET, TS_THREAD_POOL_TASK, /* unlikely you should use these */ - TS_THREAD_POOL_SSL, TS_THREAD_POOL_DNS, - TS_THREAD_POOL_REMAP, TS_THREAD_POOL_UDP } TSThreadPool; @@ -880,6 +864,15 @@ typedef enum { TS_MGMT_SOURCE_ENV ///< Process environment variable. } TSMgmtSource; +/// The User Arg type, used for Txn/Ssn/VConn user argument slots +typedef enum { + TS_USER_ARGS_TXN, ///< Transaction based. + TS_USER_ARGS_SSN, ///< Session based + TS_USER_ARGS_VCONN, ///< VConnection based + TS_USER_ARGS_GLB, ///< Global based + TS_USER_ARGS_COUNT ///< Fake enum, # of valid entries. +} TSUserArgType; + typedef struct tsapi_file *TSFile; typedef struct tsapi_mloc *TSMLoc; @@ -915,6 +908,8 @@ typedef struct tsapi_hostlookupresult *TSHostLookupResult; typedef struct tsapi_aiocallback *TSAIOCallback; typedef struct tsapi_net_accept *TSAcceptor; +typedef struct tsapi_fetchsm *TSFetchSM; + typedef void *(*TSThreadFunc)(void *data); typedef int (*TSEventFunc)(TSCont contp, TSEvent event, void *edata); typedef void (*TSConfigDestroyFunc)(void *data); @@ -1237,7 +1232,9 @@ extern tsapi const char *const TS_ALPN_PROTOCOL_HTTP_1_0; extern tsapi const char *const TS_ALPN_PROTOCOL_HTTP_1_1; extern tsapi const char *const TS_ALPN_PROTOCOL_HTTP_2_0; extern tsapi const char *const TS_ALPN_PROTOCOL_HTTP_3; +extern tsapi const char *const TS_ALPN_PROTOCOL_HTTP_3_D27; extern tsapi const char *const TS_ALPN_PROTOCOL_HTTP_QUIC; +extern tsapi const char *const TS_ALPN_PROTOCOL_HTTP_QUIC_D27; extern tsapi int TS_ALPN_PROTOCOL_INDEX_HTTP_0_9; extern tsapi int TS_ALPN_PROTOCOL_INDEX_HTTP_1_0; diff --git a/include/ts/experimental.h b/include/ts/experimental.h index fa8e7ead89c..80f80b82a55 100644 --- a/include/ts/experimental.h +++ b/include/ts/experimental.h @@ -50,8 +50,6 @@ typedef enum { TS_FETCH_FLAGS_NOT_INTERNAL_REQUEST = 1 << 4 // Allow this fetch to be created as a non-internal request. } TSFetchFlags; -typedef struct tsapi_fetchsm *TSFetchSM; - /* Forward declaration of in_addr, any user of these APIs should probably include net/netinet.h or whatever is appropriate on the platform. */ struct in_addr; @@ -223,8 +221,6 @@ tsapi TSReturnCode TSHttpTxnUpdateCachedObject(TSHttpTxn txnp); ****************************************************************************/ tsapi int TSHttpTxnLookingUpTypeGet(TSHttpTxn txnp); -tsapi void TSHttpTxnServerPush(TSHttpTxn txnp, const char *url, int url_len); - /* ip addr parsing */ tsapi TSReturnCode TSIpStringToAddr(const char *str, size_t str_len, struct sockaddr *addr); @@ -360,6 +356,16 @@ tsapi TSReturnCode TSMgmtConfigIntSet(const char *var_name, TSMgmtInt value); tsapi TSFetchSM TSFetchCreate(TSCont contp, const char *method, const char *url, const char *version, struct sockaddr const *client_addr, int flags); +/* + * Set fetch flags to FetchSM Context + * + * @param fetch_sm: returned value of TSFetchCreate(). + * @param flags: can be bitwise OR of several TSFetchFlags. + * + * return void + */ +tsapi void TSFetchFlagSet(TSFetchSM fetch_sm, int flags); + /* * Create FetchSM, this API will enable stream IO automatically. * diff --git a/include/ts/remap.h b/include/ts/remap.h index cc263fdd4ea..1e804b936a6 100644 --- a/include/ts/remap.h +++ b/include/ts/remap.h @@ -75,6 +75,16 @@ typedef enum { TSREMAP_ERROR = -1 /* Some error, that should generate an error page */ } TSRemapStatus; +/* Status code passed to the plugin by TSRemapPostConfigReload() signaling + * (1) if the configuration reload was successful and + * (2) if (1) is successful show if the plugin was part of the new configuration */ +typedef enum { + TSREMAP_CONFIG_RELOAD_FAILURE = 0, /* notify the plugin that configuration parsing failed */ + TSREMAP_CONFIG_RELOAD_SUCCESS_PLUGIN_USED = 1, /* configuration parsing succeeded and plugin was used by the new configuration */ + TSREMAP_CONFIG_RELOAD_SUCCESS_PLUGIN_UNUSED = + 2 /* configuration parsing succeeded but plugin was NOT used by the new configuration */ +} TSRemapReloadStatus; + /* ---------------------------------------------------------------------------------- These are the entry points a plugin can implement. Note that TSRemapInit() and TSRemapDoRemap() are both required. @@ -88,13 +98,27 @@ typedef enum { */ tsapi TSReturnCode TSRemapInit(TSRemapInterface *api_info, char *errbuf, int errbuf_size); -/* This gets called every time remap.config is reloaded. This is complementary - to TSRemapInit() which gets called when the plugin is first loaded. You can - not fail, or cause reload to stop here, it's merely a notification. +/* This gets called every time before remap.config is reloaded. This is complementary + to TSRemapInit() which gets called when the plugin is first loaded. + It is guaranteed to be called before TSRemapInit() and TSRemapNewInstance(). + It cannot fail, or cause reload to stop here, it's merely a notification. + Optional function. + Params: none + Return: none +*/ +tsapi void TSRemapPreConfigReload(void); + +/* This gets called every time afterremap.config is reloaded. This is complementary + to TSRemapInit() which gets called when the plugin is first loaded. + It is guaranteed to be called after TSRemapInit() and TSRemapNewInstance(). + It cannot fail, or cause reload to stop here, it's merely a notification that + the (re)load is done and provide a status of its success or failure.. Optional function. + Params: reloadStatus - TS_SUCCESS - (re)load was successful, + TS_ERROR - (re)load failed. Return: none */ -tsapi void TSRemapConfigReload(void); +tsapi void TSRemapPostConfigReload(TSRemapReloadStatus reloadStatus); /* Remap new request Mandatory interface function. diff --git a/include/ts/sdt.h b/include/ts/sdt.h new file mode 100644 index 00000000000..68f8d926af8 --- /dev/null +++ b/include/ts/sdt.h @@ -0,0 +1,40 @@ +/** @file + + A brief file description + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#pragma once + +#ifdef HAVE_SYSTEMTAP + +#include + +#define ATS_PROBE(probe) DTRACE_PROBE(trafficserver, probe) +#define ATS_PROBE1(probe, param1) DTRACE_PROBE1(trafficserver, probe, param1) +#define ATS_PROBE2(probe, param1, param2) DTRACE_PROBE2(trafficserver, probe, param1, param2) + +#else + +#define ATS_PROBE(...) +#define ATS_PROBE1(...) +#define ATS_PROBE2(...) + +#endif diff --git a/include/ts/ts.h b/include/ts/ts.h index 71a663eb5da..a44827c3b54 100644 --- a/include/ts/ts.h +++ b/include/ts/ts.h @@ -162,6 +162,20 @@ int TSTrafficServerVersionGetPatch(void); */ tsapi TSReturnCode TSPluginRegister(const TSPluginRegistrationInfo *plugin_info); +/** + This function provides the ability to enable/disable programmatically + the plugin dynamic reloading when the same Dynamic Shared Object (DSO) + is also used as a remap plugin. This overrides `proxy.config.plugin.dynamic_reload_mode` + configuration variable. + + @param enabled boolean flag. 0/false will disable the reload on the caller plugin. + @return TS_ERROR if the function is not called from within TSPluginInit or if TS is + unable to get the canonical path from the plugin's path. TS_SUCCESS otherwise. + + @note This function should be called from within TSPluginInit + */ +tsapi TSReturnCode TSPluginDSOReloadEnable(int enabled); + /* -------------------------------------------------------------------------- Files */ /** @@ -384,8 +398,8 @@ tsapi int TSUrlLengthGet(TSMBuffer bufp, TSMLoc offset); string in the parameter length. This is the same length that TSUrlLengthGet() returns. The returned string is allocated by a call to TSmalloc(). It should be freed by a call to TSfree(). - The length parameter must present, providing storage for the URL - string length value. + The length parameter must be present, providing storage for the + URL string length value. Note: To get the effective URL from a request, use the alternative TSHttpTxnEffectiveUrlStringGet or TSHttpHdrEffectiveUrlBufGet APIs. @@ -1233,7 +1247,7 @@ tsapi void TSVConnReenableEx(TSVConn sslvcp, TSEvent event); /* Set the connection to go into blind tunnel mode */ tsapi TSReturnCode TSVConnTunnel(TSVConn sslp); /* Return the SSL object associated with the connection */ -tsapi TSSslConnection TSVConnSSLConnectionGet(TSVConn sslp); +tsapi TSSslConnection TSVConnSslConnectionGet(TSVConn sslp); /* Return the intermediate X509StoreCTX object that references the certificate being validated */ tsapi TSSslVerifyCTX TSVConnSslVerifyCTXGet(TSVConn sslp); /* Fetch a SSL context from the global lookup table */ @@ -1262,7 +1276,7 @@ TSReturnCode TSVConnProtocolEnable(TSVConn connp, const char *protocol_name); tsapi int TSVConnIsSsl(TSVConn sslp); tsapi TSSslSession TSSslSessionGet(const TSSslSessionID *session_id); tsapi int TSSslSessionGetBuffer(const TSSslSessionID *session_id, char *buffer, int *len_ptr); -tsapi TSReturnCode TSSslSessionInsert(const TSSslSessionID *session_id, TSSslSession add_session); +tsapi TSReturnCode TSSslSessionInsert(const TSSslSessionID *session_id, TSSslSession add_session, TSSslConnection ssl_conn); tsapi TSReturnCode TSSslSessionRemove(const TSSslSessionID *session_id); /* -------------------------------------------------------------------------- @@ -1548,25 +1562,32 @@ tsapi void TSHttpTxnTransformedRespCache(TSHttpTxn txnp, int on); tsapi void TSHttpTxnReenable(TSHttpTxn txnp, TSEvent event); tsapi TSReturnCode TSHttpCacheReenable(TSCacheTxn txnp, const TSEvent event, const void *data, const uint64_t size); -tsapi void TSHttpTxnArgSet(TSHttpTxn txnp, int arg_idx, void *arg); -tsapi void *TSHttpTxnArgGet(TSHttpTxn txnp, int arg_idx); -tsapi void TSHttpSsnArgSet(TSHttpSsn ssnp, int arg_idx, void *arg); -tsapi void *TSHttpSsnArgGet(TSHttpSsn ssnp, int arg_idx); -tsapi void TSVConnArgSet(TSVConn connp, int arg_idx, void *arg); -tsapi void *TSVConnArgGet(TSVConn connp, int arg_idx); - -/* The reserve API should only be use in TSAPI plugins, during plugin initialization! */ -/* The lookup methods can be used anytime, but are best used during initialization as well, +/* The reserve API should only be use in TSAPI plugins, during plugin initialization! + The lookup methods can be used anytime, but are best used during initialization as well, or at least "cache" the results for best performance. */ -tsapi TSReturnCode TSHttpTxnArgIndexReserve(const char *name, const char *description, int *arg_idx); -tsapi TSReturnCode TSHttpTxnArgIndexNameLookup(const char *name, int *arg_idx, const char **description); -tsapi TSReturnCode TSHttpTxnArgIndexLookup(int arg_idx, const char **name, const char **description); -tsapi TSReturnCode TSHttpSsnArgIndexReserve(const char *name, const char *description, int *arg_idx); -tsapi TSReturnCode TSHttpSsnArgIndexNameLookup(const char *name, int *arg_idx, const char **description); -tsapi TSReturnCode TSHttpSsnArgIndexLookup(int arg_idx, const char **name, const char **description); -tsapi TSReturnCode TSVConnArgIndexReserve(const char *name, const char *description, int *arg_idx); -tsapi TSReturnCode TSVConnArgIndexNameLookup(const char *name, int *arg_idx, const char **description); -tsapi TSReturnCode TSVConnArgIndexLookup(int arg_idx, const char **name, const char **description); +tsapi TSReturnCode TSUserArgIndexReserve(TSUserArgType type, const char *name, const char *description, int *arg_idx); +tsapi TSReturnCode TSUserArgIndexNameLookup(TSUserArgType type, const char *name, int *arg_idx, const char **description); +tsapi TSReturnCode TSUserArgIndexLookup(TSUserArgType type, int arg_idx, const char **name, const char **description); +tsapi void TSUserArgSet(void *data, int arg_idx, void *arg); +tsapi void *TSUserArgGet(void *data, int arg_idx); + +/* These are deprecated as of v9.0.0, and will be removed in v10.0.0 */ +tsapi TS_DEPRECATED void TSHttpTxnArgSet(TSHttpTxn txnp, int arg_idx, void *arg); +tsapi TS_DEPRECATED void *TSHttpTxnArgGet(TSHttpTxn txnp, int arg_idx); +tsapi TS_DEPRECATED void TSHttpSsnArgSet(TSHttpSsn ssnp, int arg_idx, void *arg); +tsapi TS_DEPRECATED void *TSHttpSsnArgGet(TSHttpSsn ssnp, int arg_idx); +tsapi TS_DEPRECATED void TSVConnArgSet(TSVConn connp, int arg_idx, void *arg); +tsapi TS_DEPRECATED void *TSVConnArgGet(TSVConn connp, int arg_idx); + +tsapi TS_DEPRECATED TSReturnCode TSHttpTxnArgIndexReserve(const char *name, const char *description, int *arg_idx); +tsapi TS_DEPRECATED TSReturnCode TSHttpTxnArgIndexNameLookup(const char *name, int *arg_idx, const char **description); +tsapi TS_DEPRECATED TSReturnCode TSHttpTxnArgIndexLookup(int arg_idx, const char **name, const char **description); +tsapi TS_DEPRECATED TSReturnCode TSHttpSsnArgIndexReserve(const char *name, const char *description, int *arg_idx); +tsapi TS_DEPRECATED TSReturnCode TSHttpSsnArgIndexNameLookup(const char *name, int *arg_idx, const char **description); +tsapi TS_DEPRECATED TSReturnCode TSHttpSsnArgIndexLookup(int arg_idx, const char **name, const char **description); +tsapi TS_DEPRECATED TSReturnCode TSVConnArgIndexReserve(const char *name, const char *description, int *arg_idx); +tsapi TS_DEPRECATED TSReturnCode TSVConnArgIndexNameLookup(const char *name, int *arg_idx, const char **description); +tsapi TS_DEPRECATED TSReturnCode TSVConnArgIndexLookup(int arg_idx, const char **name, const char **description); tsapi void TSHttpTxnStatusSet(TSHttpTxn txnp, TSHttpStatus status); tsapi TSHttpStatus TSHttpTxnStatusGet(TSHttpTxn txnp); @@ -1715,8 +1736,8 @@ tsapi TSVConn TSHttpConnect(struct sockaddr const *addr); */ tsapi TSVConn TSHttpConnectTransparent(struct sockaddr const *client_addr, struct sockaddr const *server_addr); -tsapi void TSFetchUrl(const char *request, int request_len, struct sockaddr const *addr, TSCont contp, - TSFetchWakeUpOptions callback_options, TSFetchEvent event); +tsapi TSFetchSM TSFetchUrl(const char *request, int request_len, struct sockaddr const *addr, TSCont contp, + TSFetchWakeUpOptions callback_options, TSFetchEvent event); tsapi void TSFetchPages(TSFetchUrlParams_t *params); /* Check if HTTP State machine is internal or not */ @@ -2296,6 +2317,37 @@ tsapi int TSHttpTxnServerRespHdrBytesGet(TSHttpTxn txnp); tsapi int64_t TSHttpTxnServerRespBodyBytesGet(TSHttpTxn txnp); tsapi int TSHttpTxnClientRespHdrBytesGet(TSHttpTxn txnp); tsapi int64_t TSHttpTxnClientRespBodyBytesGet(TSHttpTxn txnp); +tsapi int TSVConnIsSslReused(TSVConn sslp); + +/** + Return the current (if set) SSL Cipher. This is still owned by the + core, and must not be free'd. + + @param sslp The connection pointer + + @return the SSL Cipher +*/ +tsapi const char *TSVConnSslCipherGet(TSVConn sslp); + +/** + Return the current (if set) SSL Protocol. This is still owned by the + core, and must not be free'd. + + @param sslp The connection pointer + + @return the SSL Protocol +*/ +tsapi const char *TSVConnSslProtocolGet(TSVConn sslp); + +/** + Return the current (if set) SSL Curve. This is still owned by the + core, and must not be free'd. + + @param txnp the transaction pointer + + @return the SSL Curve +*/ +tsapi const char *TSVConnSslCurveGet(TSVConn sslp); /* NetVC timeout APIs. */ tsapi void TSVConnInactivityTimeoutSet(TSVConn connp, TSHRTime timeout); @@ -2323,21 +2375,7 @@ tsapi TSReturnCode TSHttpTxnConfigStringGet(TSHttpTxn txnp, TSOverridableConfigK tsapi TSReturnCode TSHttpTxnConfigFind(const char *name, int length, TSOverridableConfigKey *conf, TSRecordDataType *type); /** - This API informs the core to try to follow redirections (e.g. 301 responses. - The new URL would be provided in the standard Location header. - - This is deprecated as of v8.0.0. You should instead rely on using the overridable - proxy.config.http.number_of_redirections setting. - - @param txnp the transaction pointer - @param on turn this on or off (0 or 1) - - @return @c TS_SUCCESS if it succeeded -*/ -tsapi TS_DEPRECATED TSReturnCode TSHttpTxnFollowRedirect(TSHttpTxn txnp, int on); - -/** - This is a generalization of the TSHttpTxnFollowRedirect(), but gives finer + This is a generalization of the old TSHttpTxnFollowRedirect(), but gives finer control over the behavior. Instead of using the Location: header for the new destination, this API takes the new URL as a parameter. Calling this API transfers the ownership of the URL from the plugin to the core, so you must @@ -2430,6 +2468,18 @@ tsapi TSReturnCode TSHttpTxnMilestoneGet(TSHttpTxn txnp, TSMilestonesType milest */ tsapi int TSHttpTxnIsCacheable(TSHttpTxn txnp, TSMBuffer request, TSMBuffer response); +/** + Get the maximum age in seconds as indicated by the origin server. + This would typically be used in TS_HTTP_READ_RESPONSE_HDR_HOOK, when you have + the server response ready. + + @param txnp the transaction pointer + @param response the server response header. If NULL, use the transactions origin response. + + @return the age in seconds if specified by Cache-Control, -1 otherwise +*/ +tsapi int TSHttpTxnGetMaxAge(TSHttpTxn txnp, TSMBuffer response); + /** Return a string representation for a TSServerState value. This is useful for plugin debugging. @@ -2487,9 +2537,9 @@ tsapi const char *TSHttpSsnClientProtocolStackContains(TSHttpSsn ssnp, char cons tsapi const char *TSNormalizedProtocolTag(char const *tag); tsapi const char *TSRegisterProtocolTag(char const *tag); -// If, for the given transaction, the URL has been remapped, this function puts the memory location of the "from" URL object in the -// variable pointed to by urlLocp, and returns TS_SUCCESS. (The URL object will be within memory allocated to the transaction -// object.) Otherwise, the function returns TS_ERROR. +// If, for the given transaction, the URL has been remapped, this function puts the memory location of the "from" URL object in +// the variable pointed to by urlLocp, and returns TS_SUCCESS. (The URL object will be within memory allocated to the +// transaction object.) Otherwise, the function returns TS_ERROR. // tsapi TSReturnCode TSRemapFromUrlGet(TSHttpTxn txnp, TSMLoc *urlLocp); @@ -2504,6 +2554,15 @@ tsapi TSReturnCode TSRemapToUrlGet(TSHttpTxn txnp, TSMLoc *urlLocp); */ tsapi TSIOBufferReader TSHttpTxnPostBufferReaderGet(TSHttpTxn txnp); +/** + * Initiate an HTTP/2 Server Push preload request. + * Use this api to register a URL that you want to preload with HTTP/2 Server Push. + * + * @param url the URL string to preload. + * @param url_len the length of the URL string. + */ +tsapi TSReturnCode TSHttpTxnServerPush(TSHttpTxn txnp, const char *url, int url_len); + #ifdef __cplusplus } #endif /* __cplusplus */ diff --git a/include/tscore/ArgParser.h b/include/tscore/ArgParser.h index 45b0e416252..a7b4b7a4f0a 100644 --- a/include/tscore/ArgParser.h +++ b/include/tscore/ArgParser.h @@ -48,7 +48,11 @@ class ArgumentData // bool to check if certain command/option is called operator bool() const noexcept { return _is_called; } // index accessing [] - std::string const &operator[](int x) const { return _values.at(x); } + std::string const & + operator[](int x) const + { + return _values.at(x); + } // return the Environment variable std::string const &env() const noexcept; // iterator for arguments diff --git a/include/tscore/BaseLogFile.h b/include/tscore/BaseLogFile.h index 38f6a496f04..aad473586d1 100644 --- a/include/tscore/BaseLogFile.h +++ b/include/tscore/BaseLogFile.h @@ -177,7 +177,7 @@ class BaseLogFile static bool rolled_logfile(char *path); static bool exists(const char *pathname); int open_file(int perm = -1); - void close_file(); + int close_file(); void change_name(const char *new_name); void display(FILE *fd = stdout); const char * diff --git a/include/tscore/BufferWriter.h b/include/tscore/BufferWriter.h index 3e693a30b2e..103af0b62b8 100644 --- a/include/tscore/BufferWriter.h +++ b/include/tscore/BufferWriter.h @@ -529,9 +529,7 @@ namespace bw_fmt /// @a TUPLE. Due to language limitations it cannot be done directly. The formatters can be /// accessed via standard array access in contrast to templated tuple access. The actual array is /// static and therefore at run time the only operation is loading the address of the array. - template - ArgFormatterSignature * - Get_Arg_Formatter_Array(std::index_sequence) + template ArgFormatterSignature *Get_Arg_Formatter_Array(std::index_sequence) { static ArgFormatterSignature fa[sizeof...(N)] = {&bw_fmt::Arg_Formatter...}; return fa; diff --git a/include/tscore/CryptoHash.h b/include/tscore/CryptoHash.h index 27077adad59..767cb7b64fe 100644 --- a/include/tscore/CryptoHash.h +++ b/include/tscore/CryptoHash.h @@ -84,7 +84,11 @@ union CryptoHash { } /// Access 64 bit slice. - uint64_t operator[](int i) const { return u64[i]; } + uint64_t + operator[](int i) const + { + return u64[i]; + } /// Access 64 bit slice. /// @note Identical to @ operator[] but included for symmetry. uint64_t diff --git a/include/tscore/Diags.h b/include/tscore/Diags.h index f9e8a2d5881..23c669637a6 100644 --- a/include/tscore/Diags.h +++ b/include/tscore/Diags.h @@ -111,7 +111,7 @@ struct DiagsConfigState { class Diags { public: - Diags(const char *prefix_string, const char *base_debug_tags, const char *base_action_tags, BaseLogFile *_diags_log, + Diags(std::string_view prefix_string, const char *base_debug_tags, const char *base_action_tags, BaseLogFile *_diags_log, int diags_log_perm = -1, int output_log_perm = -1); virtual ~Diags(); @@ -230,7 +230,7 @@ class Diags IpAddr debug_client_ip; private: - const char *prefix_str; + const std::string prefix_str; mutable ink_mutex tag_table_lock; // prevents reconfig/read races DFA *activated_tags[2]; // 1 table for debug, 1 for action @@ -312,7 +312,7 @@ extern inkcoreapi Diags *diags; #define AlertV(fmt, ap) DiagsErrorV(DL_Alert, fmt, ap) #define EmergencyV(fmt, ap) DiagsErrorV(DL_Emergency, fmt, ap) -#ifdef TS_USE_DIAGS +#if TS_USE_DIAGS #define Diag(tag, ...) \ do { \ diff --git a/include/tscore/EnumDescriptor.h b/include/tscore/EnumDescriptor.h index 485a5c3db59..f1c4b5075a4 100644 --- a/include/tscore/EnumDescriptor.h +++ b/include/tscore/EnumDescriptor.h @@ -19,6 +19,8 @@ limitations under the License. */ +#pragma once + #include #include diff --git a/include/tscore/Errata.h b/include/tscore/Errata.h index 6bc5e3a3441..b28a291c837 100644 --- a/include/tscore/Errata.h +++ b/include/tscore/Errata.h @@ -336,7 +336,7 @@ class Errata int indent, ///< Additional indention per line for messages. int shift, ///< Additional @a indent for nested @c Errata. char const *lead ///< Leading text for nested @c Errata. - ) const; + ) const; /// Simple formatted output to fixed sized buffer. /// @return Number of characters written to @a buffer. size_t write(char *buffer, ///< Output buffer. @@ -345,7 +345,7 @@ class Errata int indent, ///< Additional indention per line for messages. int shift, ///< Additional @a indent for nested @c Errata. char const *lead ///< Leading text for nested @c Errata. - ) const; + ) const; protected: /// Construct from implementation pointer. diff --git a/include/tscore/Extendible.h b/include/tscore/Extendible.h index 6e2d30672e8..2ac826ffeb5 100644 --- a/include/tscore/Extendible.h +++ b/include/tscore/Extendible.h @@ -50,6 +50,13 @@ #include "tscore/ink_memory.h" #include "tscore/ink_defs.h" +////////////////////////////////////////// +/// SUPPORT MACRO +#define DEF_EXT_NEW_DEL(cls) \ + void *operator new(size_t sz) { return ats_malloc(ext::sizeOf()); } \ + void *operator new(size_t sz, void *ptr) { return ptr; } \ + void operator delete(void *ptr) { free(ptr); } + ////////////////////////////////////////// /// HELPER CLASSES @@ -141,9 +148,11 @@ namespace details // internal stuff { public: std::unordered_map fields; ///< defined elements of the blob by name - size_t alloc_size = 0; ///< bytes to allocate for fields - uint8_t alloc_align = 1; ///< alignment of block - std::atomic_uint instance_count = {0}; ///< the number of Extendible instances in use. + size_t alloc_size = 0; ///< bytes to allocate for fields + uint8_t alloc_align = 1; ///< alignment of block + std::atomic cnt_constructed = {0}; ///< the number of Extendible created. + std::atomic cnt_fld_constructed = {0}; ///< the number of Extendible that constructed fields. + std::atomic cnt_destructed = {0}; ///< the number of Extendible destroyed. public: Schema() {} @@ -163,7 +172,7 @@ namespace details // internal stuff /// ext::Extendible allows code (and Plugins) to declare member-like variables during system init. /* - * This class uses a special allocator (ext::alloc) to extend the memory allocated to store run-time static + * This class uses a special allocator (ext::create) to extend the memory allocated to store run-time static * variables, which are registered by plugins during system init. The API is in a functional style to support * multiple inheritance of Extendible classes. This is templated so static variables are instanced per Derived * type, because we need to have different field schema per type. @@ -199,10 +208,10 @@ template class Extendible protected: Extendible(); - // use ext::alloc() exclusively for allocation and initialization + // use ext::create() exclusively for allocation and initialization /** destruct all fields */ - ~Extendible() { schema.callDestructor(uintptr_t(this) + ext_loc); } + ~Extendible(); private: /** construct all fields */ @@ -214,7 +223,7 @@ template class Extendible return uintptr_t(this) + ext_loc; } - template friend T *alloc(); + template friend T *create(); template friend uintptr_t details::initRecurseSuper(T &, uintptr_t); template friend FieldPtr details::FieldPtrGet(Extendible const &, details::FieldDesc const &); template friend std::string viewFormat(T const &, uintptr_t, int); @@ -575,6 +584,17 @@ sizeOf(size_t size = sizeof(Derived_t)) template Extendible::Extendible() { ink_assert(ext::details::areFieldsFinalized()); + // don't call callConstructor until the derived class is fully constructed. + ++schema.cnt_constructed; +} + +template Extendible::~Extendible() +{ + // assert callConstructors was called. + ink_assert(ext_loc); + schema.callDestructor(uintptr_t(this) + ext_loc); + ++schema.cnt_destructed; + ink_assert(schema.cnt_destructed <= schema.cnt_fld_constructed); } /// tell this extendible where it's memory offset start is. Added to support inheriting from extendible classes @@ -584,8 +604,9 @@ Extendible::initFields(uintptr_t start_ptr) { ink_assert(ext_loc == 0); start_ptr = ROUNDUP(start_ptr, schema.alloc_align); // pad the previous struct, so that our fields are memaligned correctly - ext_loc = uint16_t(start_ptr - uintptr_t(this)); // store the offset to be used by ext::get and ext::set - ink_assert(ext_loc < 256); + ink_assert(start_ptr - uintptr_t(this) < UINT16_MAX); + ext_loc = uint16_t(start_ptr - uintptr_t(this)); // store the offset to be used by ext::get and ext::set + ink_assert(ext_loc > 0); schema.callConstructor(start_ptr); // construct all fields return start_ptr + schema.alloc_size; // return the end of the extendible data } @@ -608,23 +629,26 @@ namespace details } return tail_ptr; } + } // namespace details // allocate and initialize an extendible data structure -template +template Derived_t * -alloc() +create(Args &&... args) { - static_assert(std::is_base_of, Derived_t>::value); + // don't instantiate until all Fields are finalized. ink_assert(ext::details::areFieldsFinalized()); - // calculate the memory needed + // calculate the memory needed for the class and all Extendible blocks const size_t type_size = ext::sizeOf(); - // alloc a block of memory - Derived_t *ptr = (Derived_t *)ats_memalign(alignof(Derived_t), type_size); - // Extendible_t *ptr = (Extendible_t *)::operator new(type_size); // alloc a block of memory + + // alloc one block of memory + Derived_t *ptr = static_cast(ats_memalign(alignof(Derived_t), type_size)); + // construct (recursively super-to-sub class) - new (ptr) Derived_t(); + new (ptr) Derived_t(std::forward(args)...); + // define extendible blocks start offsets (recursively super-to-sub class) details::initRecurseSuper(*ptr, uintptr_t(ptr) + sizeof(Derived_t)); return ptr; @@ -720,7 +744,10 @@ serialize(std::ostream &os, T const &t) for (const auto &kv : schema.fields) { name_width = max(name_width, kv.first.length()); } - for (const auto &[fname, field] : schema.fields) { + // TODO: clang-5 didn't like the use of a range based for here, change later + for (auto it = schema.fields.begin(); it != schema.fields.end(); ++it) { + auto &fname = it->first; + auto &field = it->second; ink_assert(field.serializer); os << setw(indent) << "" << setw(name_width) << right << fname << ": "; field.serializer(os, details::FieldPtrGet(t, field)); diff --git a/include/tscore/Filenames.h b/include/tscore/Filenames.h new file mode 100644 index 00000000000..6fe0eb8f460 --- /dev/null +++ b/include/tscore/Filenames.h @@ -0,0 +1,50 @@ +/** @file + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#pragma once + +namespace ts +{ +namespace filename +{ + constexpr const char *STORAGE = "storage.config"; + constexpr const char *RECORDS = "records.config"; + constexpr const char *VOLUME = "volume.config"; + constexpr const char *PLUGIN = "plugin.config"; + + // These still need to have their corrensponding records.config settings remove + constexpr const char *LOGGING = "logging.yaml"; + constexpr const char *CACHE = "cache.config"; + constexpr const char *IP_ALLOW = "ip_allow.yaml"; + constexpr const char *HOSTING = "hosting.config"; + constexpr const char *SOCKS = "socks.config"; + constexpr const char *PARENT = "parent.config"; + constexpr const char *REMAP = "remap.config"; + constexpr const char *SSL_MULTICERT = "ssl_multicert.config"; + constexpr const char *SPLITDNS = "splitdns.config"; + constexpr const char *SNI = "sni.yaml"; + + /////////////////////////////////////////////////////////////////// + // Various other file names + constexpr const char *RECORDS_STATS = "records.snap"; + +} // namespace filename +} // namespace ts diff --git a/include/tscore/History.h b/include/tscore/History.h index e413ddaef62..de8feb7c98f 100644 --- a/include/tscore/History.h +++ b/include/tscore/History.h @@ -67,7 +67,11 @@ template class History return history_pos > Count ? Count : history_pos; } - const HistoryEntry &operator[](unsigned int i) const { return history[i]; } + const HistoryEntry & + operator[](unsigned int i) const + { + return history[i]; + } private: HistoryEntry history[Count]; diff --git a/include/tscore/IntrusiveHashMap.h b/include/tscore/IntrusiveHashMap.h index 8a64a7b0980..f8dd340aff2 100644 --- a/include/tscore/IntrusiveHashMap.h +++ b/include/tscore/IntrusiveHashMap.h @@ -539,16 +539,6 @@ IntrusiveHashMap::insert(value_type *v) if (spot != bucket->_v) { mixed_p = true; // found some other key, it's going to be mixed. } - if (spot != limit) { - // If an equal key was found, walk past those to insert at the upper end of the range. - do { - spot = H::next_ptr(spot); - } while (spot != limit && H::equal(key, H::key_of(spot))); - if (spot != limit) { // something not equal past last equivalent, it's going to be mixed. - mixed_p = true; - } - } - _list.insert_before(spot, v); if (spot == bucket->_v) { // added before the bucket start, update the start. bucket->_v = v; diff --git a/include/tscore/IntrusivePtr.h b/include/tscore/IntrusivePtr.h index dcf54aef068..bfe63a3aee0 100644 --- a/include/tscore/IntrusivePtr.h +++ b/include/tscore/IntrusivePtr.h @@ -472,13 +472,17 @@ IntrusivePtr::operator=(IntrusivePtr &&that) return *this; } -template T *IntrusivePtr::operator->() const +template +T * +IntrusivePtr::operator->() const { IntrusivePtrPolicy::dereferenceCheck(m_obj); return m_obj; } -template T &IntrusivePtr::operator*() const +template +T & +IntrusivePtr::operator*() const { IntrusivePtrPolicy::dereferenceCheck(m_obj); return *m_obj; diff --git a/include/tscore/IpMap.h b/include/tscore/IpMap.h index 50a9fbe2434..4268afa9cd4 100644 --- a/include/tscore/IpMap.h +++ b/include/tscore/IpMap.h @@ -306,7 +306,7 @@ class IpMap */ bool contains(sockaddr const *target, ///< Search target (network order). void **ptr = nullptr ///< Client data return. - ) const; + ) const; /** Test for membership. @@ -318,7 +318,7 @@ class IpMap */ bool contains(in_addr_t target, ///< Search target (network order). void **ptr = nullptr ///< Client data return. - ) const; + ) const; /** Test for membership. @@ -330,7 +330,7 @@ class IpMap */ bool contains(IpEndpoint const *target, ///< Search target (network order). void **ptr = nullptr ///< Client data return. - ) const; + ) const; /** Test for membership. @@ -342,7 +342,7 @@ class IpMap */ bool contains(IpAddr const &target, ///< Search target (network order). void **ptr = nullptr ///< Client data return. - ) const; + ) const; /** Remove all addresses from the map. @@ -482,12 +482,14 @@ IpMap::iterator::operator==(iterator const &that) const return _tree == that._tree && _node == that._node; } -inline IpMap::iterator::reference IpMap::iterator::operator*() const +inline IpMap::iterator::reference +IpMap::iterator::operator*() const { return *_node; } -inline IpMap::iterator::pointer IpMap::iterator::operator->() const +inline IpMap::iterator::pointer +IpMap::iterator::operator->() const { return _node; } diff --git a/include/tscore/JeAllocator.h b/include/tscore/JeAllocator.h index 7d1b678fdcc..24e94d9022e 100644 --- a/include/tscore/JeAllocator.h +++ b/include/tscore/JeAllocator.h @@ -26,6 +26,9 @@ #if TS_HAS_JEMALLOC #include +#if (JEMALLOC_VERSION_MAJOR == 0) +#error jemalloc has bogus version +#endif #if (JEMALLOC_VERSION_MAJOR == 5) && defined(MADV_DONTDUMP) #define JEMALLOC_NODUMP_ALLOCATOR_SUPPORTED 1 #endif /* MADV_DONTDUMP */ diff --git a/include/tscore/PluginUserArgs.h b/include/tscore/PluginUserArgs.h new file mode 100644 index 00000000000..057e37c1b3a --- /dev/null +++ b/include/tscore/PluginUserArgs.h @@ -0,0 +1,75 @@ +/** @file + + Base class and implementation details for the User Args features. + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#pragma once + +#include +#include "ts/apidefs.h" +#include "tscore/ink_assert.h" +#include "tscore/PluginUserArgs.h" + +static constexpr std::array MAX_USER_ARGS = {{ + 16, /* max number of user arguments for TXN */ + 8, /* max number of user arguments for SSN */ + 4, /* max number of user arguments for VCONN */ + 128 /* max number of user arguments for GLB */ +}}; + +/** + This is a mixin class (sort of), implementing the appropriate APIs and data storage for + a particular user arg table. Used by VConn / Ssn / Txn user arg data. +*/ +class PluginUserArgsMixin +{ +public: + virtual ~PluginUserArgsMixin() = default; + virtual void *get_user_arg(size_t ix) const = 0; + virtual void set_user_arg(size_t ix, void *arg) = 0; +}; + +template class PluginUserArgs : public virtual PluginUserArgsMixin +{ +public: + void * + get_user_arg(size_t ix) const + { + ink_release_assert(ix < user_args.size()); + return this->user_args[ix]; + }; + + void + set_user_arg(size_t ix, void *arg) + { + ink_release_assert(ix < user_args.size()); + user_args[ix] = arg; + }; + + void + clear() + { + user_args.fill(nullptr); + } + +private: + std::array user_args{{nullptr}}; +}; diff --git a/include/tscore/Ptr.h b/include/tscore/Ptr.h index 3a79e01b9dd..3e934770612 100644 --- a/include/tscore/Ptr.h +++ b/include/tscore/Ptr.h @@ -105,8 +105,16 @@ template class Ptr Ptr &operator=(const Ptr &); Ptr &operator=(T *); - T *operator->() const { return (m_ptr); } - T &operator*() const { return (*m_ptr); } + T * + operator->() const + { + return (m_ptr); + } + T & + operator*() const + { + return (*m_ptr); + } // Making this explicit avoids unwanted conversions. See https://en.wikibooks.org/wiki/More_C%2B%2B_Idioms/Safe_bool . explicit operator bool() const { return m_ptr != nullptr; } diff --git a/include/tscore/RbTree.h b/include/tscore/RbTree.h index dcb52f58ec6..446e1ed8a6e 100644 --- a/include/tscore/RbTree.h +++ b/include/tscore/RbTree.h @@ -52,7 +52,7 @@ namespace detail /// @return The child in the direction @a d if it exists, /// @c NULL if not. self *getChild(Direction d //!< The direction of the desired child - ) const; + ) const; /** Determine which child a node is @return @c LEFT if @a n is the left child, @@ -61,7 +61,7 @@ namespace detail */ Direction getChildDirection(self *const &n //!< The presumed child node - ) const + ) const { return (n == _left) ? LEFT : (n == _right) ? RIGHT : NONE; } diff --git a/include/tscore/Regression.h b/include/tscore/Regression.h index e34a497df4e..8ebbe8c515a 100644 --- a/include/tscore/Regression.h +++ b/include/tscore/Regression.h @@ -54,8 +54,6 @@ #define REGRESSION_TEST_QUICK 1 #define REGRESSION_TEST_NIGHTLY 2 #define REGRESSION_TEST_EXTENDED 3 -// use only for testing TS error handling! -#define REGRESSION_TEST_FATAL 4 // regression options #define REGRESSION_OPT_EXCLUSIVE (1 << 0) diff --git a/include/tscore/Scalar.h b/include/tscore/Scalar.h index 327a178fc02..8de58331417 100644 --- a/include/tscore/Scalar.h +++ b/include/tscore/Scalar.h @@ -826,27 +826,39 @@ Scalar::operator*=(C n) -> self & return *this; } -template Scalar operator*(Scalar const &lhs, C n) +template +Scalar +operator*(Scalar const &lhs, C n) { return Scalar(lhs) *= n; } -template Scalar operator*(C n, Scalar const &rhs) +template +Scalar +operator*(C n, Scalar const &rhs) { return Scalar(rhs) *= n; } -template Scalar operator*(Scalar const &lhs, int n) +template +Scalar +operator*(Scalar const &lhs, int n) { return Scalar(lhs) *= n; } -template Scalar operator*(int n, Scalar const &rhs) +template +Scalar +operator*(int n, Scalar const &rhs) { return Scalar(rhs) *= n; } -template Scalar operator*(Scalar const &lhs, int n) +template +Scalar +operator*(Scalar const &lhs, int n) { return Scalar(lhs) *= n; } -template Scalar operator*(int n, Scalar const &rhs) +template +Scalar +operator*(int n, Scalar const &rhs) { return Scalar(rhs) *= n; } diff --git a/include/tscore/TsBuffer.h b/include/tscore/TsBuffer.h index e2ad2f7f16a..a823641472f 100644 --- a/include/tscore/TsBuffer.h +++ b/include/tscore/TsBuffer.h @@ -323,7 +323,8 @@ Buffer::operator==(ConstBuffer const &that) const { return _size == that._size && _ptr == that._ptr; } -inline bool Buffer::operator!() const +inline bool +Buffer::operator!() const { return !(_ptr && _size); } @@ -331,7 +332,8 @@ inline Buffer::operator pseudo_bool() const { return _ptr && _size ? &self::operator! : nullptr; } -inline char Buffer::operator*() const +inline char +Buffer::operator*() const { return *_ptr; } @@ -407,7 +409,8 @@ ConstBuffer::operator==(Buffer const &that) const { return _size == that._size && 0 == memcmp(_ptr, that._ptr, _size); } -inline bool ConstBuffer::operator!() const +inline bool +ConstBuffer::operator!() const { return !(_ptr && _size); } @@ -415,7 +418,8 @@ inline ConstBuffer::operator pseudo_bool() const { return _ptr && _size ? &self::operator! : nullptr; } -inline char ConstBuffer::operator*() const +inline char +ConstBuffer::operator*() const { return *_ptr; } @@ -438,7 +442,8 @@ ConstBuffer::data() const { return _ptr; } -inline char ConstBuffer::operator[](int n) const +inline char +ConstBuffer::operator[](int n) const { return _ptr[n]; } diff --git a/include/tscore/ink_config.h.in b/include/tscore/ink_config.h.in index e994ea0a38c..3ff2acbcf2d 100644 --- a/include/tscore/ink_config.h.in +++ b/include/tscore/ink_config.h.in @@ -78,6 +78,7 @@ #define TS_USE_LINUX_NATIVE_AIO @use_linux_native_aio@ #define TS_USE_REMOTE_UNWINDING @use_remote_unwinding@ #define TS_USE_TLS_OCSP @use_tls_ocsp@ +#define TS_HAS_TLS_EARLY_DATA @has_tls_early_data@ #define TS_HAS_SO_PEERCRED @has_so_peercred@ diff --git a/include/tscore/ink_endian.h b/include/tscore/ink_endian.h index 866518b11da..54823bffbe2 100644 --- a/include/tscore/ink_endian.h +++ b/include/tscore/ink_endian.h @@ -58,4 +58,14 @@ htobe32(uint32_t x) { return OSSwapHostToBigInt32(x); } +inline uint32_t +le32toh(uint32_t x) +{ + return OSSwapLittleToHostInt32(x); +} +inline uint32_t +htole32(uint32_t x) +{ + return OSSwapHostToLittleInt32(x); +} #endif diff --git a/include/tscore/ink_inet.h b/include/tscore/ink_inet.h index 70cd47ca78e..8c9ccafe46c 100644 --- a/include/tscore/ink_inet.h +++ b/include/tscore/ink_inet.h @@ -61,6 +61,8 @@ extern const std::string_view IP_PROTO_TAG_HTTP_1_1; extern const std::string_view IP_PROTO_TAG_HTTP_2_0; extern const std::string_view IP_PROTO_TAG_HTTP_QUIC; extern const std::string_view IP_PROTO_TAG_HTTP_3; +extern const std::string_view IP_PROTO_TAG_HTTP_QUIC_D27; +extern const std::string_view IP_PROTO_TAG_HTTP_3_D27; struct IpAddr; // forward declare. @@ -1214,7 +1216,7 @@ struct IpAddr { */ char *toString(char *dest, ///< [out] Destination string buffer. size_t len ///< [in] Size of buffer. - ) const; + ) const; /// Equality. bool diff --git a/include/tscore/ink_memory.h b/include/tscore/ink_memory.h index 713cb02e3f6..9f772e57e08 100644 --- a/include/tscore/ink_memory.h +++ b/include/tscore/ink_memory.h @@ -591,7 +591,11 @@ class ats_scoped_obj : public ats_scoped_resource() const { return *this; } + T * + operator->() const + { + return *this; + } }; /** Combine two strings as file paths. diff --git a/include/tscore/ink_platform.h b/include/tscore/ink_platform.h index 6bef79d7855..922b474c57c 100644 --- a/include/tscore/ink_platform.h +++ b/include/tscore/ink_platform.h @@ -157,9 +157,11 @@ typedef unsigned int in_addr_t; #include #endif +#if defined(darwin) || defined(freebsd) #ifdef HAVE_SYS_SYSCTL_H #include #endif +#endif #ifdef HAVE_SYS_SYSTEMINFO_H #include diff --git a/include/tscore/ink_queue.h b/include/tscore/ink_queue.h index 5c6cc742459..24f860594fb 100644 --- a/include/tscore/ink_queue.h +++ b/include/tscore/ink_queue.h @@ -145,8 +145,8 @@ union head_p { */ /* Detect which shift is implemented by the simple expression ((~0 >> 1) < 0): * - * If the shift is `logical’ the highest order bit of the left side of the comparison is 0 so the result is positive. - * If the shift is `arithmetic’ the highest order bit of the left side is 1 so the result is negative. + * If the shift is 'logical' the highest order bit of the left side of the comparison is 0 so the result is positive. + * If the shift is 'arithmetic' the highest order bit of the left side is 1 so the result is negative. */ #if ((~0 >> 1) < 0) /* the shift is `arithmetic' */ diff --git a/include/tscore/ink_resolver.h b/include/tscore/ink_resolver.h index b9af473f772..394e121824e 100644 --- a/include/tscore/ink_resolver.h +++ b/include/tscore/ink_resolver.h @@ -182,11 +182,9 @@ enum HostResStyle { extern const char *const HOST_RES_STYLE_STRING[]; /// Calculate the effective resolution preferences. -extern HostResStyle ats_host_res_from(int family, ///< Connection family - HostResPreferenceOrder ///< Preference ordering. +extern HostResStyle ats_host_res_from(int family, ///< Connection family + HostResPreferenceOrder const & ///< Preference ordering. ); -/// Calculate the host resolution style to force a family match to @a addr. -extern HostResStyle ats_host_res_match(sockaddr const *addr); /** Parse a host resolution configuration string. */ @@ -194,6 +192,18 @@ extern void parse_host_res_preference(const char *value, ///< [in] Con HostResPreferenceOrder order /// [out] Order to update. ); +/// Configure the preference order to hold only what's from the client address. +/// @addr[in] client's address. +/// @order[out] Order to update +extern void ats_force_order_by_family(sockaddr const *addr, HostResPreferenceOrder order); + +// Domain resolution priority for origin. +struct HostResData { + HostResPreferenceOrder order; + // keep the configuration value to satisfy the API(TSHttpTxnConfigStringSet) + char *conf_value{nullptr}; +}; + #ifndef NS_GET16 #define NS_GET16(s, cp) \ do { \ diff --git a/include/tscore/ink_rwlock.h b/include/tscore/ink_rwlock.h index 5b3dcb36982..c2938e3240a 100644 --- a/include/tscore/ink_rwlock.h +++ b/include/tscore/ink_rwlock.h @@ -29,23 +29,37 @@ #pragma once -#include "tscore/ink_mutex.h" -#include "tscore/ink_thread.h" - -#define RW_MAGIC 0x19283746 - -struct ink_rwlock { - ink_mutex rw_mutex; /* basic lock on this struct */ - ink_cond rw_condreaders; /* for reader threads waiting */ - ink_cond rw_condwriters; /* for writer threads waiting */ - int rw_magic; /* for error checking */ - int rw_nwaitreaders; /* the number waiting */ - int rw_nwaitwriters; /* the number waiting */ - int rw_refcount; -}; - -int ink_rwlock_init(ink_rwlock *rw); -int ink_rwlock_destroy(ink_rwlock *rw); -int ink_rwlock_rdlock(ink_rwlock *rw); -int ink_rwlock_wrlock(ink_rwlock *rw); -int ink_rwlock_unlock(ink_rwlock *rw); +#include "tscore/ink_error.h" +#include + +typedef pthread_rwlock_t ink_rwlock; + +void ink_rwlock_init(ink_rwlock *rw); +void ink_rwlock_destroy(ink_rwlock *rw); + +static inline void +ink_rwlock_rdlock(ink_rwlock *rw) +{ + int error = pthread_rwlock_rdlock(rw); + if (unlikely(error != 0)) { + ink_abort("pthread_rwlock_rdlock(%p) failed: %s (%d)", rw, strerror(error), error); + } +} + +static inline void +ink_rwlock_wrlock(ink_rwlock *rw) +{ + int error = pthread_rwlock_wrlock(rw); + if (unlikely(error != 0)) { + ink_abort("pthread_rwlock_wrlock(%p) failed: %s (%d)", rw, strerror(error), error); + } +} + +static inline void +ink_rwlock_unlock(ink_rwlock *rw) +{ + int error = pthread_rwlock_unlock(rw); + if (unlikely(error != 0)) { + ink_abort("pthread_rwlock_unlock(%p) failed: %s (%d)", rw, strerror(error), error); + } +} diff --git a/include/tscore/ts_file.h b/include/tscore/ts_file.h index e9bfd8b8010..c4389e948f6 100644 --- a/include/tscore/ts_file.h +++ b/include/tscore/ts_file.h @@ -178,6 +178,18 @@ namespace file // Returns return the canonicalized absolute pathname path canonical(const path &p, std::error_code &ec); + // Return the filename derived from path p. + // + // This is made to match the std::filesystem::path::filename behavior: + // https://en.cppreference.com/w/cpp/filesystem/path/filename + // + // Examples: + // given "/foo/bar.txt", this returns "bar.txt" + // given "/foo/bar", this returns "bar" + // given "/foo/bar/", this returns "" + // given "/", this returns "" + path filename(const path &p); + // Checks if the file/directory exists bool exists(const path &p); diff --git a/include/tscpp/api/HttpStatus.h b/include/tscpp/api/HttpStatus.h index 1fb3e15efe0..89f6dd8f945 100644 --- a/include/tscpp/api/HttpStatus.h +++ b/include/tscpp/api/HttpStatus.h @@ -82,6 +82,7 @@ enum HttpStatus { HTTP_STATUS_PRECONDITION_REQUIRED = 428, HTTP_STATUS_TOO_MANY_REQUESTS = 429, HTTP_STATUS_REQUEST_HEADER_FIELDS_TOO_LARGE = 431, + HTTP_STATUS_UNAVAILABLE_FOR_LEGAL_REASONS = 451, HTTP_STATUS_INTERNAL_SERVER_ERROR = 500, HTTP_STATUS_NOT_IMPLEMENTED = 501, diff --git a/include/tscpp/api/Plugin.h b/include/tscpp/api/Plugin.h index 2f57352cbe0..f37b8050e17 100644 --- a/include/tscpp/api/Plugin.h +++ b/include/tscpp/api/Plugin.h @@ -58,7 +58,8 @@ class Plugin : noncopyable HOOK_READ_REQUEST_HEADERS, /**< This hook will be fired after the request is read. */ HOOK_READ_CACHE_HEADERS, /**< This hook will be fired after the CACHE hdrs. */ HOOK_CACHE_LOOKUP_COMPLETE, /**< This hook will be fired after cache lookup complete. */ - HOOK_SELECT_ALT /**< This hook will be fired after select alt. */ + HOOK_TXN_CLOSE, /**< This hook will be fired after send response headers, only for TransactionPlugins::registerHook()!. */ + HOOK_SELECT_ALT /**< This hook will be fired after select alt. */ }; /** @@ -142,6 +143,15 @@ class Plugin : noncopyable transaction.resume(); }; + /** + * This method must be implemented when you hook HOOK_TXN_CLOSE + */ + virtual void + handleTxnClose(Transaction &transaction) + { + transaction.resume(); + }; + /** * This method must be implemented when you hook HOOK_SELECT_ALT */ diff --git a/include/tscpp/api/RemapPlugin.h b/include/tscpp/api/RemapPlugin.h index 44e550ead8d..e69cbb6ee42 100644 --- a/include/tscpp/api/RemapPlugin.h +++ b/include/tscpp/api/RemapPlugin.h @@ -25,6 +25,7 @@ #include "tscpp/api/Transaction.h" #include "tscpp/api/Url.h" #include "tscpp/api/utils.h" +#include "ts/remap.h" namespace atscppapi { @@ -49,6 +50,25 @@ class RemapPlugin RESULT_DID_REMAP_STOP, }; + /** + * Invoked when a request matches the remap.config line - it gives you access to the remap information + * if you want to have more control over how the remap happens. + * + * @param transaction Transaction + * @param rri The remap information in the remap.config line. + * + * @return Result of the remap - will dictate further processing by the system. + */ + virtual Result + remapTransaction(Transaction &transaction, TSRemapRequestInfo *rri) + { + Url map_from_url(rri->requestBufp, rri->mapFromUrl), map_to_url(rri->requestBufp, rri->mapToUrl); + bool redirect = false; + RemapPlugin::Result result = doRemap(map_from_url, map_to_url, transaction, redirect); + rri->redirect = redirect ? 1 : 0; + return result; + } + /** * Invoked when a request matches the remap.config line - implementation should perform the * remap. The client's URL is in the transaction and that's where it should be modified. diff --git a/include/tscpp/api/TransactionPlugin.h b/include/tscpp/api/TransactionPlugin.h index b34fba01f8b..cc3567d4e02 100644 --- a/include/tscpp/api/TransactionPlugin.h +++ b/include/tscpp/api/TransactionPlugin.h @@ -93,6 +93,10 @@ class TransactionPlugin : public Plugin * see HookType and Plugin for the correspond HookTypes and callback methods. If you fail to implement the * callback, a default implementation will be used that will only resume the Transaction. * + * \note For automatic destruction, you must either register dynamically allocated instances of + * classes derived from this class with the the corresponding Transaction object (using + * Transaction::addPlugin()), or register HOOK_TXN_CLOSE (but not both). + * * @param HookType the type of hook you wish to register * @see HookType * @see Plugin @@ -114,6 +118,8 @@ class TransactionPlugin : public Plugin */ std::shared_ptr getMutex(); + std::shared_ptr getMutex(TSHttpTxn); + private: TransactionPluginState *state_; /**< The internal state for a TransactionPlugin */ friend class utils::internal; diff --git a/include/tscpp/api/Url.h b/include/tscpp/api/Url.h index cf4b85914a0..c01b8da6439 100644 --- a/include/tscpp/api/Url.h +++ b/include/tscpp/api/Url.h @@ -124,14 +124,6 @@ class Url : noncopyable */ void setPort(const uint16_t); - /** - * This method allows you to reset the url, this will force the Url to fully re-read all cached values. - * If this method is used on a Detached Requests' Url object it will completely destroy the values. - * - * \note This method should rarely be used. - */ - void reset(); - private: bool isInitialized() const; void init(void *hdr_buf, void *url_loc); diff --git a/include/tscpp/util/IntrusiveDList.h b/include/tscpp/util/IntrusiveDList.h index e94fda3b550..decbda9c194 100644 --- a/include/tscpp/util/IntrusiveDList.h +++ b/include/tscpp/util/IntrusiveDList.h @@ -493,12 +493,16 @@ ts::IntrusiveDList::iterator::operator--(int) -> self_type return tmp; } -template auto ts::IntrusiveDList::const_iterator::operator-> () const -> value_type * +template +auto +ts::IntrusiveDList::const_iterator::operator->() const -> value_type * { return _v; } -template auto ts::IntrusiveDList::iterator::operator-> () const -> value_type * +template +auto +ts::IntrusiveDList::iterator::operator->() const -> value_type * { return super_type::_v; } @@ -508,12 +512,16 @@ template ts::IntrusiveDList::const_iterator::operator value_type return _v; } -template auto ts::IntrusiveDList::const_iterator::operator*() const -> value_type & +template +auto +ts::IntrusiveDList::const_iterator::operator*() const -> value_type & { return *_v; } -template auto ts::IntrusiveDList::iterator::operator*() const -> value_type & +template +auto +ts::IntrusiveDList::iterator::operator*() const -> value_type & { return *super_type::_v; } diff --git a/include/tscpp/util/MemSpan.h b/include/tscpp/util/MemSpan.h index 1c757b3de1d..e34862d5e2d 100644 --- a/include/tscpp/util/MemSpan.h +++ b/include/tscpp/util/MemSpan.h @@ -603,7 +603,9 @@ MemSpan::operator!=(self_type const &that) const return !(*this == that); } -template bool MemSpan::operator!() const +template +bool +MemSpan::operator!() const { return _count == 0; } @@ -641,7 +643,9 @@ MemSpan::end() const return _ptr + _count; } -template T &MemSpan::operator[](size_t idx) const +template +T & +MemSpan::operator[](size_t idx) const { return _ptr[idx]; } @@ -769,7 +773,8 @@ MemSpan::operator!=(self_type const &that) const return !(*this == that); } -inline bool MemSpan::operator!() const +inline bool +MemSpan::operator!() const { return _size == 0; } diff --git a/include/tscpp/util/TextView.h b/include/tscpp/util/TextView.h index 894b1666cf6..bf0f8ae7009 100644 --- a/include/tscpp/util/TextView.h +++ b/include/tscpp/util/TextView.h @@ -624,12 +624,14 @@ TextView::clear() return *this; } -inline char TextView::operator*() const +inline char +TextView::operator*() const { return this->empty() ? 0 : *(this->data()); } -inline bool TextView::operator!() const +inline bool +TextView::operator!() const { return this->empty(); } diff --git a/include/wccp/Wccp.h b/include/wccp/Wccp.h index 0e1d26571fa..468ebbeade4 100644 --- a/include/wccp/Wccp.h +++ b/include/wccp/Wccp.h @@ -187,7 +187,7 @@ class ServiceGroup : public ServiceConstants /// Get a port value. uint16_t getPort(int idx ///< Index of target port. - ) const; + ) const; /// Set a port value. self &setPort(int idx, ///< Index of port. uint16_t port ///< Value for port. diff --git a/iocore/aio/AIO.cc b/iocore/aio/AIO.cc index 07eb8ae7439..df8e27bea8f 100644 --- a/iocore/aio/AIO.cc +++ b/iocore/aio/AIO.cc @@ -192,8 +192,13 @@ struct AIOThreadInfo : public Continuation { (void)event; (void)e; #if TS_USE_HWLOC +#if HWLOC_API_VERSION >= 0x20000 + hwloc_set_membind(ink_get_topology(), hwloc_topology_get_topology_nodeset(ink_get_topology()), HWLOC_MEMBIND_INTERLEAVE, + HWLOC_MEMBIND_THREAD | HWLOC_MEMBIND_BYNODESET); +#else hwloc_set_membind_nodeset(ink_get_topology(), hwloc_topology_get_topology_nodeset(ink_get_topology()), HWLOC_MEMBIND_INTERLEAVE, HWLOC_MEMBIND_THREAD); +#endif #endif aio_thread_main(this); delete this; @@ -475,9 +480,9 @@ aio_thread_main(void *arg) SCOPED_MUTEX_LOCK(lock, op->mutex, thr_info->mutex->thread_holding); op->handleEvent(EVENT_NONE, nullptr); } else if (op->thread == AIO_CALLBACK_THREAD_ANY) { - eventProcessor.schedule_imm_signal(op); + eventProcessor.schedule_imm(op); } else { - op->thread->schedule_imm_signal(op); + op->thread->schedule_imm(op); } ink_mutex_acquire(&my_aio_req->aio_mutex); } while (true); diff --git a/iocore/cache/Cache.cc b/iocore/cache/Cache.cc index ee7effe417b..f84f50f0899 100644 --- a/iocore/cache/Cache.cc +++ b/iocore/cache/Cache.cc @@ -28,12 +28,12 @@ #include "StatPages.h" #include "tscore/I_Layout.h" +#include "tscore/Filenames.h" #include "HttpTransactCache.h" #include "HttpSM.h" #include "HttpCacheSM.h" #include "InkAPIInternal.h" -#include "P_CacheBC.h" #include "tscore/hugepages.h" @@ -82,9 +82,6 @@ int cache_config_read_while_writer = 0; int cache_config_mutex_retry_delay = 2; int cache_read_while_writer_retry_delay = 50; int cache_config_read_while_writer_max_retries = 10; -/// Fix up a specific known problem with the 4.2.0 release. -/// Not used for stripes with a cache version later than 4.2.0. -int cache_config_compatibility_4_2_0_fixup = 1; // Globals @@ -914,12 +911,14 @@ CacheProcessor::cacheInitialized() Debug("cache_init", "CacheProcessor::cacheInitialized - cache_config_ram_cache_size == AUTO_SIZE_RAM_CACHE"); for (i = 0; i < gnvol; i++) { vol = gvol[i]; - gvol[i]->ram_cache->init(vol->dirlen() * DEFAULT_RAM_CACHE_MULTIPLIER, vol); - ram_cache_bytes += gvol[i]->dirlen(); - Debug("cache_init", "CacheProcessor::cacheInitialized - ram_cache_bytes = %" PRId64 " = %" PRId64 "Mb", ram_cache_bytes, - ram_cache_bytes / (1024 * 1024)); - CACHE_VOL_SUM_DYN_STAT(cache_ram_cache_bytes_total_stat, (int64_t)gvol[i]->dirlen()); + if (gvol[i]->cache_vol->ramcache_enabled) { + gvol[i]->ram_cache->init(vol->dirlen() * DEFAULT_RAM_CACHE_MULTIPLIER, vol); + ram_cache_bytes += gvol[i]->dirlen(); + Debug("cache_init", "CacheProcessor::cacheInitialized - ram_cache_bytes = %" PRId64 " = %" PRId64 "Mb", ram_cache_bytes, + ram_cache_bytes / (1024 * 1024)); + CACHE_VOL_SUM_DYN_STAT(cache_ram_cache_bytes_total_stat, (int64_t)gvol[i]->dirlen()); + } vol_total_cache_bytes = gvol[i]->len - gvol[i]->dirlen(); total_cache_bytes += vol_total_cache_bytes; Debug("cache_init", "CacheProcessor::cacheInitialized - total_cache_bytes = %" PRId64 " = %" PRId64 "Mb", @@ -958,14 +957,14 @@ CacheProcessor::cacheInitialized() for (i = 0; i < gnvol; i++) { vol = gvol[i]; double factor; - if (gvol[i]->cache == theCache) { + if (gvol[i]->cache == theCache && gvol[i]->cache_vol->ramcache_enabled) { ink_assert(gvol[i]->cache != nullptr); factor = static_cast(static_cast(gvol[i]->len >> STORE_BLOCK_SHIFT)) / theCache->cache_size; Debug("cache_init", "CacheProcessor::cacheInitialized - factor = %f", factor); gvol[i]->ram_cache->init(static_cast(http_ram_cache_size * factor), vol); ram_cache_bytes += static_cast(http_ram_cache_size * factor); CACHE_VOL_SUM_DYN_STAT(cache_ram_cache_bytes_total_stat, (int64_t)(http_ram_cache_size * factor)); - } else { + } else if (gvol[i]->cache_vol->ramcache_enabled) { ink_release_assert(!"Unexpected non-HTTP cache volume"); } Debug("cache_init", "CacheProcessor::cacheInitialized[%d] - ram_cache_bytes = %" PRId64 " = %" PRId64 "Mb", i, @@ -2151,92 +2150,6 @@ unmarshal_helper(Doc *doc, Ptr &buf, int &okay) } } -/** Upgrade a marshalled fragment buffer to the current version. - - @internal I looked at doing this in place (rather than a copy & modify) but - - The in place logic would be even worse than this mess - - It wouldn't save you that much, since you end up doing inserts early in the buffer. - Without extreme care in the logic it could end up doing more copying than - the simpler copy & modify. - - @internal This logic presumes the existence of some slack at the end of the buffer, which - is usually the case (because lots of rounding is done). If there isn't enough slack then - this fails and we have a cache miss. The assumption that this is sufficiently rare that - code simplicity takes precedence should be checked at some point. - */ -static bool -upgrade_doc_version(Ptr &buf) -{ - // Type definition is close enough to use for initial checking. - cache_bc::Doc_v23 *doc = reinterpret_cast(buf->data()); - bool zret = true; - - if (DOC_MAGIC == doc->magic) { - if (0 == doc->hlen) { - Debug("cache_bc", "Doc %p without header, no upgrade needed.", doc); - } else if (CACHE_FRAG_TYPE_HTTP_V23 == doc->doc_type) { - cache_bc::HTTPCacheAlt_v21 *alt = reinterpret_cast(doc->hdr()); - if (alt && alt->is_unmarshalled_format()) { - Ptr d_buf(ioDataAllocator.alloc()); - Doc *d_doc; - char *src; - char *dst; - char *hdr_limit = doc->data(); - HTTPInfo::FragOffset *frags = - reinterpret_cast(static_cast(buf->data()) + sizeof(cache_bc::Doc_v23)); - int frag_count = doc->_flen / sizeof(HTTPInfo::FragOffset); - size_t n = 0; - size_t content_size = doc->data_len(); - - Debug("cache_bc", "Doc %p is 3.2", doc); - - // Use the same buffer size, fail if no fit. - d_buf->alloc(buf->_size_index, buf->_mem_type); // Duplicate. - d_doc = reinterpret_cast(d_buf->data()); - n = d_buf->block_size(); - - src = buf->data(); - dst = d_buf->data(); - memcpy(dst, src, sizeof(Doc)); - src += sizeof(Doc) + doc->_flen; - dst += sizeof(Doc); - n -= sizeof(Doc); - - // We copy the fragment table iff there is a fragment table and there is only one alternate. - if (frag_count > 0 && cache_bc::HTTPInfo_v21::marshalled_length(src) > doc->hlen) { - frag_count = 0; // inhibit fragment table insertion. - } - - while (zret && src < hdr_limit) { - zret = cache_bc::HTTPInfo_v21::copy_and_upgrade_unmarshalled_to_v23(dst, src, n, frag_count, frags); - } - if (zret && content_size <= n) { - memcpy(dst, src, content_size); // content - // Must update new Doc::len and Doc::hlen - // dst points at the first byte of the content, or one past the last byte of the alt header. - d_doc->len = (dst - reinterpret_cast(d_doc)) + content_size; - d_doc->hlen = (dst - reinterpret_cast(d_doc)) - sizeof(Doc); - buf = d_buf; // replace original buffer with new buffer. - } else { - zret = false; - } - } - Doc *n_doc = reinterpret_cast(buf->data()); // access as current version. - // For now the base header size is the same. If that changes we'll need to handle the v22/23 case here - // as with the v21 and shift the content down to accommodate the bigger header. - ink_assert(sizeof(*n_doc) == sizeof(*doc)); - - n_doc->doc_type = CACHE_FRAG_TYPE_HTTP; // We converted so adjust doc_type. - // Set these to zero for debugging - they'll be updated to the current values if/when this is - // put in the aggregation buffer. - n_doc->v_major = 0; - n_doc->v_minor = 0; - n_doc->unused = 0; // force to zero to make future use easier. - } - } - return zret; -} - // [amc] I think this is where all disk reads from cache funnel through here. int CacheVC::handleReadDone(int event, Event *e) @@ -2269,37 +2182,12 @@ CacheVC::handleReadDone(int event, Event *e) ink_assert(vol->mutex->nthread_holding < 1000); ink_assert(doc->magic == DOC_MAGIC); - /* We've read the raw data from disk, time to deserialize it. We have to account for a variety of formats that - may be present. - - As of cache version 24 we changed the @a doc_type to indicate a format change in the header which includes - version data inside the header. Prior to that we must use heuristics to deduce the actual format. For this reason - we send that header type off for special processing. Otherwise we can use the in object version to determine - the format. - - All of this processing must produce a serialized header that is compliant with the current version. This includes - updating the doc_type. - */ - - if (doc->doc_type == CACHE_FRAG_TYPE_HTTP_V23) { - if (upgrade_doc_version(buf)) { - doc = reinterpret_cast(buf->data()); // buf may be a new copy - } else { - Debug("cache_bc", "Upgrade of fragment failed - disk %s - doc id = %" PRIx64 ":%" PRIx64 "", vol->hash_text.get(), - read_key->slice64(0), read_key->slice64(1)); - doc->magic = DOC_CORRUPT; - // Should really trash the directory entry for this, as it's never going to work in the future. - // Or does that happen later anyway? - goto Ldone; - } - } else if (doc->doc_type == CACHE_FRAG_TYPE_HTTP) { // handle any version updates based on the object version - if (ts::VersionNumber(doc->v_major, doc->v_minor) > CACHE_DB_VERSION) { - // future version, count as corrupted - doc->magic = DOC_CORRUPT; - Debug("cache_bc", "Object is future version %d:%d - disk %s - doc id = %" PRIx64 ":%" PRIx64 "", doc->v_major, doc->v_minor, - vol->hash_text.get(), read_key->slice64(0), read_key->slice64(1)); - goto Ldone; - } + if (ts::VersionNumber(doc->v_major, doc->v_minor) > CACHE_DB_VERSION) { + // future version, count as corrupted + doc->magic = DOC_CORRUPT; + Debug("cache_bc", "Object is future version %d:%d - disk %s - doc id = %" PRIx64 ":%" PRIx64 "", doc->v_major, doc->v_minor, + vol->hash_text.get(), read_key->slice64(0), read_key->slice64(1)); + goto Ldone; } #ifdef VERIFY_JTEST_DATA @@ -2644,6 +2532,8 @@ cplist_init() } } +static int fillExclusiveDisks(CacheVol *cp); + void cplist_update() { @@ -2655,7 +2545,8 @@ cplist_update() for (config_vol = config_volumes.cp_queue.head; config_vol; config_vol = config_vol->link.next) { if (config_vol->number == cp->vol_number) { if (cp->scheme == config_vol->scheme) { - config_vol->cachep = cp; + cp->ramcache_enabled = config_vol->ramcache_enabled; + config_vol->cachep = cp; } else { /* delete this volume from all the disks */ int d_no; @@ -2699,6 +2590,42 @@ cplist_update() cp = cp->link.next; } } + + // Look for (exclusive) spans forced to a specific volume but not yet referenced by any volumes in cp_list, + // if found then create a new volume. This also makes sure new exclusive disk volumes are created first + // before any other new volumes to assure proper span free space calculation and proper volume block distribution. + for (config_vol = config_volumes.cp_queue.head; config_vol; config_vol = config_vol->link.next) { + if (nullptr == config_vol->cachep) { + // Find out if this is a forced volume assigned exclusively to a span which was cleared (hence not referenced in cp_list). + // Note: non-exclusive cleared spans are not handled here, only the "exclusive" + bool forced_volume = false; + for (int d_no = 0; d_no < gndisks; d_no++) { + if (gdisks[d_no]->forced_volume_num == config_vol->number) { + forced_volume = true; + } + } + + if (forced_volume) { + CacheVol *new_cp = new CacheVol(); + if (nullptr != new_cp) { + new_cp->disk_vols = static_castdisk_vols)>(ats_malloc(gndisks * sizeof(DiskVol *))); + if (nullptr != new_cp->disk_vols) { + memset(new_cp->disk_vols, 0, gndisks * sizeof(DiskVol *)); + new_cp->vol_number = config_vol->number; + new_cp->scheme = config_vol->scheme; + config_vol->cachep = new_cp; + fillExclusiveDisks(config_vol->cachep); + cp_list.enqueue(new_cp); + } else { + delete new_cp; + } + } + } + } else { + // Fill if this is exclusive disk. + fillExclusiveDisks(config_vol->cachep); + } + } } static int @@ -2712,20 +2639,32 @@ fillExclusiveDisks(CacheVol *cp) if (gdisks[i]->forced_volume_num != volume_number) { continue; } - /* The user had created several volumes before - clear the disk - and create one volume for http */ + + /* OK, this should be an "exclusive" disk (span). */ + diskCount++; + + /* There should be a single "forced" volume and no other volumes should exist on this "exclusive" disk (span) */ + bool found_nonforced_volumes = false; for (int j = 0; j < static_cast(gdisks[i]->header->num_volumes); j++) { if (volume_number != gdisks[i]->disk_vols[j]->vol_number) { - Note("Clearing Disk: %s", gdisks[i]->path); - gdisks[i]->delete_all_volumes(); + found_nonforced_volumes = true; break; } } - diskCount++; + + if (found_nonforced_volumes) { + /* The user had created several volumes before - clear the disk and create one volume for http */ + Note("Clearing Disk: %s", gdisks[i]->path); + gdisks[i]->delete_all_volumes(); + } else if (1 == gdisks[i]->header->num_volumes) { + /* "Forced" volumes take the whole disk (span) hence nothing more to do for this span. */ + continue; + } + + /* Now, volumes have been either deleted or did not exist to begin with so we need to create them. */ int64_t size_diff = gdisks[i]->num_usable_blocks; DiskVolBlock *dpb; - do { dpb = gdisks[i]->create_volume(volume_number, size_diff, cp->scheme); if (dpb) { @@ -2741,6 +2680,8 @@ fillExclusiveDisks(CacheVol *cp) } } while ((size_diff > 0)); } + + /* Report back the number of disks (spans) that were assigned to volume specified by volume_number. */ return diskCount; } @@ -2805,7 +2746,11 @@ cplist_reconfigure() /* sum up the total space available on all the disks. round down the space to 128 megabytes */ for (int i = 0; i < gndisks; i++) { - tot_space_in_blks += (gdisks[i]->num_usable_blocks / blocks_per_vol) * blocks_per_vol; + // Exclude exclusive disks (with forced volumes) from the following total space calculation, + // in such a way forced volumes will not impact volume percentage calculations. + if (-1 == gdisks[i]->forced_volume_num) { + tot_space_in_blks += (gdisks[i]->num_usable_blocks / blocks_per_vol) * blocks_per_vol; + } } double percent_remaining = 100.00; @@ -2816,33 +2761,46 @@ cplist_reconfigure() Warning("no volumes created"); return -1; } - int64_t space_in_blks = static_cast(((config_vol->percent / percent_remaining)) * tot_space_in_blks); + + // Find if the volume is forced and if it is then calculate the total forced volume size. + // Forced volumes take the whole span (disk) also sum all disk space this volume is forced to. + int64_t tot_forced_space_in_blks = 0; + for (int i = 0; i < gndisks; i++) { + if (config_vol->number == gdisks[i]->forced_volume_num) { + tot_forced_space_in_blks += (gdisks[i]->num_usable_blocks / blocks_per_vol) * blocks_per_vol; + } + } + + int64_t space_in_blks = 0; + if (0 == tot_forced_space_in_blks) { + // Calculate the space as percentage of total space in blocks. + space_in_blks = static_cast(((config_vol->percent / percent_remaining)) * tot_space_in_blks); + } else { + // Forced volumes take all disk space, so no percentage calculations here. + space_in_blks = tot_forced_space_in_blks; + } space_in_blks = space_in_blks >> (20 - STORE_BLOCK_SHIFT); /* round down to 128 megabyte multiple */ space_in_blks = (space_in_blks >> 7) << 7; config_vol->size = space_in_blks; - tot_space_in_blks -= space_in_blks << (20 - STORE_BLOCK_SHIFT); - percent_remaining -= (config_vol->size < 128) ? 0 : config_vol->percent; + + if (0 == tot_forced_space_in_blks) { + tot_space_in_blks -= space_in_blks << (20 - STORE_BLOCK_SHIFT); + percent_remaining -= (config_vol->size < 128) ? 0 : config_vol->percent; + } } if (config_vol->size < 128) { Warning("the size of volume %d (%" PRId64 ") is less than the minimum required volume size %d", config_vol->number, (int64_t)config_vol->size, 128); Warning("volume %d is not created", config_vol->number); } - Debug("cache_hosting", "Volume: %d Size: %" PRId64, config_vol->number, (int64_t)config_vol->size); + Debug("cache_hosting", "Volume: %d Size: %" PRId64 " Ramcache: %d", config_vol->number, (int64_t)config_vol->size, + config_vol->ramcache_enabled); } cplist_update(); - /* go through volume config and grow and create volumes */ - - for (config_vol = config_volumes.cp_queue.head; config_vol; config_vol = config_vol->link.next) { - // if volume is given exclusive disks, fill here and continue - if (!config_vol->cachep) { - continue; - } - fillExclusiveDisks(config_vol->cachep); - } + /* go through volume config and grow and create volumes */ for (config_vol = config_volumes.cp_queue.head; config_vol; config_vol = config_vol->link.next) { size = config_vol->size; if (size < 128) { @@ -3227,8 +3185,6 @@ ink_cache_init(ts::ModuleVersion v) cache_config_target_fragment_size = DEFAULT_TARGET_FRAGMENT_SIZE; } - REC_EstablishStaticConfigInt32(cache_config_compatibility_4_2_0_fixup, "proxy.config.cache.http.compatibility.4-2-0-fixup"); - REC_EstablishStaticConfigInt32(cache_config_max_disk_errors, "proxy.config.cache.max_disk_errors"); Debug("cache_init", "proxy.config.cache.max_disk_errors = %d", cache_config_max_disk_errors); @@ -3252,14 +3208,14 @@ ink_cache_init(ts::ModuleVersion v) Result result = theCacheStore.read_config(); if (result.failed()) { - Fatal("Failed to read cache storage configuration: %s", result.message()); + Fatal("Failed to read cache configuration %s: %s", ts::filename::STORAGE, result.message()); } } //---------------------------------------------------------------------------- Action * -CacheProcessor::open_read(Continuation *cont, const HttpCacheKey *key, CacheHTTPHdr *request, OverridableHttpConfigParams *params, - time_t pin_in_cache, CacheFragType type) +CacheProcessor::open_read(Continuation *cont, const HttpCacheKey *key, CacheHTTPHdr *request, + const OverridableHttpConfigParams *params, time_t pin_in_cache, CacheFragType type) { return caches[type]->open_read(cont, &key->hash, request, params, type, key->hostname, key->hostlen); } @@ -3299,109 +3255,3 @@ CacheProcessor::find_by_path(const char *path, int len) return nullptr; } - -// ---------------------------- - -namespace cache_bc -{ -static size_t const HTTP_ALT_MARSHAL_SIZE = HdrHeapMarshalBlocks{ts::round_up(sizeof(HTTPCacheAlt))}; // current size. -size_t -HTTPInfo_v21::marshalled_length(void *data) -{ - size_t zret = HdrHeapMarshalBlocks{ts::round_up(sizeof(HTTPCacheAlt_v21))}; - HTTPCacheAlt_v21 *alt = static_cast(data); - HdrHeap *hdr; - - hdr = reinterpret_cast(reinterpret_cast(alt) + reinterpret_cast(alt->m_request_hdr.m_heap)); - zret += HdrHeapMarshalBlocks{ts::round_up(hdr->unmarshal_size())}; - hdr = reinterpret_cast(reinterpret_cast(alt) + reinterpret_cast(alt->m_response_hdr.m_heap)); - zret += HdrHeapMarshalBlocks{ts::round_up(hdr->unmarshal_size())}; - return zret; -} - -// Copy an unmarshalled instance from @a src to @a dst. -// @a src is presumed to be Cache version 21 and the result -// is Cache version 23. @a length is the buffer available in @a dst. -// @return @c false if something went wrong (e.g., data overrun). -bool -HTTPInfo_v21::copy_and_upgrade_unmarshalled_to_v23(char *&dst, char *&src, size_t &length, int n_frags, FragOffset *frag_offsets) -{ - // Offsets of the data after the new stuff. - static const size_t OLD_OFFSET = offsetof(HTTPCacheAlt_v21, m_ext_buffer); - static const size_t NEW_OFFSET = offsetof(HTTPCacheAlt_v23, m_ext_buffer); - - HTTPCacheAlt_v21 *s_alt = reinterpret_cast(src); - HTTPCacheAlt_v23 *d_alt = reinterpret_cast(dst); - HdrHeap_v23 *s_hdr; - HdrHeap_v23 *d_hdr; - size_t hdr_size; - - if (length < HTTP_ALT_MARSHAL_SIZE) { - return false; // Absolutely no hope in this case. - } - - memcpy(dst, src, OLD_OFFSET); // initially same data - // Now data that's now after extra - memcpy(static_cast(dst) + NEW_OFFSET, static_cast(src) + OLD_OFFSET, sizeof(HTTPCacheAlt_v21) - OLD_OFFSET); - dst += HTTP_ALT_MARSHAL_SIZE; // move past fixed data. - length -= HTTP_ALT_MARSHAL_SIZE; - - // Extra data is fragment table - set that if we have it. - if (n_frags) { - static size_t const IFT_SIZE = HTTPCacheAlt_v23::N_INTEGRAL_FRAG_OFFSETS * sizeof(FragOffset); - size_t ift_actual = std::min(n_frags, HTTPCacheAlt_v23::N_INTEGRAL_FRAG_OFFSETS) * sizeof(FragOffset); - - if (length < (HTTP_ALT_MARSHAL_SIZE + n_frags * sizeof(FragOffset) - IFT_SIZE)) { - return false; // can't place fragment table. - } - - d_alt->m_frag_offset_count = n_frags; - d_alt->m_frag_offsets = reinterpret_cast(dst - reinterpret_cast(d_alt)); - - memcpy(d_alt->m_integral_frag_offsets, frag_offsets, ift_actual); - n_frags -= HTTPCacheAlt_v23::N_INTEGRAL_FRAG_OFFSETS; - if (n_frags > 0) { - size_t k = sizeof(FragOffset) * n_frags; - memcpy(dst, frag_offsets + IFT_SIZE, k); - dst += k; - length -= k; - } else if (n_frags < 0) { - memset(dst + ift_actual, 0, IFT_SIZE - ift_actual); - } - } else { - d_alt->m_frag_offset_count = 0; - d_alt->m_frag_offsets = nullptr; - ink_zero(d_alt->m_integral_frag_offsets); - } - - // Copy over the headers, tweaking the swizzled pointers. - s_hdr = - reinterpret_cast(reinterpret_cast(s_alt) + reinterpret_cast(s_alt->m_request_hdr.m_heap)); - d_hdr = reinterpret_cast(dst); - hdr_size = HdrHeapMarshalBlocks{ts::round_up(s_hdr->unmarshal_size())}; - if (hdr_size > length) { - return false; - } - memcpy(static_cast(d_hdr), s_hdr, hdr_size); - d_alt->m_request_hdr.m_heap = reinterpret_cast(reinterpret_cast(d_hdr) - reinterpret_cast(d_alt)); - dst += hdr_size; - length -= hdr_size; - - s_hdr = - reinterpret_cast(reinterpret_cast(s_alt) + reinterpret_cast(s_alt->m_response_hdr.m_heap)); - d_hdr = reinterpret_cast(dst); - hdr_size = HdrHeapMarshalBlocks{ts::round_up(s_hdr->unmarshal_size())}; - if (hdr_size > length) { - return false; - } - memcpy(static_cast(d_hdr), s_hdr, hdr_size); - d_alt->m_response_hdr.m_heap = reinterpret_cast(reinterpret_cast(d_hdr) - reinterpret_cast(d_alt)); - dst += hdr_size; - length -= hdr_size; - - src = reinterpret_cast(s_hdr) + hdr_size; - - return true; -} - -} // namespace cache_bc diff --git a/iocore/cache/CacheDir.cc b/iocore/cache/CacheDir.cc index a0cedc45f5e..04b6288e05d 100644 --- a/iocore/cache/CacheDir.cc +++ b/iocore/cache/CacheDir.cc @@ -1191,7 +1191,8 @@ compare_ushort(void const *a, void const *b) // Check // -int Vol::dir_check(bool /* fix ATS_UNUSED */) // TODO: we should eliminate this parameter ? +int +Vol::dir_check(bool /* fix ATS_UNUSED */) // TODO: we should eliminate this parameter ? { static int const SEGMENT_HISTOGRAM_WIDTH = 16; int hist[SEGMENT_HISTOGRAM_WIDTH + 1] = {0}; diff --git a/iocore/cache/CacheHosting.cc b/iocore/cache/CacheHosting.cc index 98e75073611..f8080bce52b 100644 --- a/iocore/cache/CacheHosting.cc +++ b/iocore/cache/CacheHosting.cc @@ -26,6 +26,8 @@ #include "tscore/HostLookup.h" #include "tscore/Tokenizer.h" #include "tscore/Regression.h" +#include "tscore/Filenames.h" +#include "tscore/ts_file.h" extern int gndisks; @@ -244,7 +246,7 @@ int fstat_wrapper(int fd, struct stat *s); int CacheHostTable::BuildTableFromString(const char *config_file_path, char *file_buf) { - Note("hosting.config loading ..."); + Note("%s loading ...", ts::filename::HOSTING); // Table build locals Tokenizer bufTok("\n"); @@ -325,7 +327,7 @@ CacheHostTable::BuildTableFromString(const char *config_file_path, char *file_bu if (gen_host_rec.Init(type)) { Warning("Problems encountered while initializing the Generic Volume"); } - Note("hosting.config finished loading"); + Note("%s finished loading", ts::filename::HOSTING); return 0; } @@ -373,10 +375,10 @@ CacheHostTable::BuildTableFromString(const char *config_file_path, char *file_bu last = current; current = current->next; ats_free(last); - - Note("hosting.config finished loading"); } + Note("%s finished loading", ts::filename::HOSTING); + if (!generic_rec_initd) { const char *cache_type = (type == CACHE_HTTP_TYPE) ? "http" : "mixt"; RecSignalWarning(REC_SIGNAL_CONFIG_ERROR, @@ -395,20 +397,22 @@ CacheHostTable::BuildTableFromString(const char *config_file_path, char *file_bu int CacheHostTable::BuildTable(const char *config_file_path) { - char *file_buf; - int ret; - - file_buf = readIntoBuffer(config_file_path, matcher_name, nullptr); + std::error_code ec; + std::string content{ts::file::load(ts::file::path{config_file_path}, ec)}; - if (file_buf == nullptr) { - Warning("Cannot read the config file: %s", config_file_path); - gen_host_rec.Init(type); - return 0; + if (ec) { + switch (ec.value()) { + case ENOENT: + Warning("Cannot open the config file: %s - %s", config_file_path, strerror(ec.value())); + break; + default: + Error("%s failed to load: %s", config_file_path, strerror(ec.value())); + gen_host_rec.Init(type); + return 0; + } } - ret = BuildTableFromString(config_file_path, file_buf); - ats_free(file_buf); - return ret; + return BuildTableFromString(config_file_path, content.data()); } int @@ -591,23 +595,27 @@ void ConfigVolumes::read_config_file() { ats_scoped_str config_path; - char *file_buf; config_path = RecConfigReadConfigPath("proxy.config.cache.volume_filename"); ink_release_assert(config_path); - Note("volume.config loading ..."); + Note("%s loading ...", ts::filename::VOLUME); - file_buf = readIntoBuffer(config_path, "[CacheVolition]", nullptr); - if (file_buf == nullptr) { - Error("volume.config failed to load"); - Warning("Cannot read the config file: %s", (const char *)config_path); - return; - } + std::error_code ec; + std::string content{ts::file::load(ts::file::path{config_path}, ec)}; - BuildListFromString(config_path, file_buf); - ats_free(file_buf); + if (ec) { + switch (ec.value()) { + case ENOENT: + Warning("Cannot open the config file: %s - %s", (const char *)config_path, strerror(ec.value())); + break; + default: + Error("%s failed to load: %s", (const char *)config_path, strerror(ec.value())); + return; + } + } + BuildListFromString(config_path, content.data()); Note("volume.config finished loading"); return; @@ -642,12 +650,13 @@ ConfigVolumes::BuildListFromString(char *config_file_path, char *file_buf) line_num++; char *end; - char *line_end = nullptr; - const char *err = nullptr; - int volume_number = 0; - CacheType scheme = CACHE_NONE_TYPE; - int size = 0; - int in_percent = 0; + char *line_end = nullptr; + const char *err = nullptr; + int volume_number = 0; + CacheType scheme = CACHE_NONE_TYPE; + int size = 0; + int in_percent = 0; + bool ramcache_enabled = true; while (true) { // skip all blank spaces at beginning of line @@ -735,6 +744,18 @@ ConfigVolumes::BuildListFromString(char *config_file_path, char *file_buf) } else { in_percent = 0; } + } else if (strcasecmp(tmp, "ramcache") == 0) { // match ramcache + tmp += 9; + if (!strcasecmp(tmp, "false")) { + tmp += 5; + ramcache_enabled = false; + } else if (!strcasecmp(tmp, "true")) { + tmp += 4; + ramcache_enabled = true; + } else { + err = "Unexpected end of line"; + break; + } } // ends here @@ -757,9 +778,10 @@ ConfigVolumes::BuildListFromString(char *config_file_path, char *file_buf) } else { configp->in_percent = false; } - configp->scheme = scheme; - configp->size = size; - configp->cachep = nullptr; + configp->scheme = scheme; + configp->size = size; + configp->cachep = nullptr; + configp->ramcache_enabled = ramcache_enabled; cp_queue.enqueue(configp); num_volumes++; if (scheme == CACHE_HTTP_TYPE) { @@ -767,7 +789,8 @@ ConfigVolumes::BuildListFromString(char *config_file_path, char *file_buf) } else { ink_release_assert(!"Unexpected non-HTTP cache volume"); } - Debug("cache_hosting", "added volume=%d, scheme=%d, size=%d percent=%d", volume_number, scheme, size, in_percent); + Debug("cache_hosting", "added volume=%d, scheme=%d, size=%d percent=%d, ramcache enabled=%d", volume_number, scheme, size, + in_percent, ramcache_enabled); } tmp = bufTok.iterNext(&i_state); @@ -867,7 +890,7 @@ create_config(RegressionTest *t, int num) } // make sure we have at least 1280 M bytes - if (total_space<(10 << 27)>> STORE_BLOCK_SHIFT) { + if (total_space < ((10 << 27) >> STORE_BLOCK_SHIFT)) { rprintf(t, "Not enough space for 10 volume\n"); return 0; } diff --git a/iocore/cache/CacheLink.cc b/iocore/cache/CacheLink.cc index f8ae328f581..d3055f4e881 100644 --- a/iocore/cache/CacheLink.cc +++ b/iocore/cache/CacheLink.cc @@ -42,7 +42,7 @@ Cache::link(Continuation *cont, const CacheKey *from, const CacheKey *to, CacheF c->buf = new_IOBufferData(BUFFER_SIZE_INDEX_512); #ifdef DEBUG - Doc *doc = (Doc *)c->buf->data(); + Doc *doc = reinterpret_cast(c->buf->data()); memcpy(doc->data(), to, sizeof(*to)); // doublecheck #endif diff --git a/iocore/cache/CachePages.cc b/iocore/cache/CachePages.cc index 5f2930b7cb8..651499ce32c 100644 --- a/iocore/cache/CachePages.cc +++ b/iocore/cache/CachePages.cc @@ -427,7 +427,7 @@ ShowCache::handleCacheEvent(int event, Event *e) case VC_EVENT_READ_READY: if (!cvio) { - buffer = new_empty_MIOBuffer(); + buffer = new_empty_MIOBuffer(BUFFER_SIZE_INDEX_32K); buffer_reader = buffer->alloc_reader(); content_length = cache_vc->get_object_size(); cvio = cache_vc->do_io_read(this, content_length, buffer); diff --git a/iocore/cache/CacheRead.cc b/iocore/cache/CacheRead.cc index ee8f31cd51c..6c89987a3e1 100644 --- a/iocore/cache/CacheRead.cc +++ b/iocore/cache/CacheRead.cc @@ -25,8 +25,6 @@ #include "HttpCacheSM.h" //Added to get the scope of HttpCacheSM object. -extern int cache_config_compatibility_4_2_0_fixup; - Action * Cache::open_read(Continuation *cont, const CacheKey *key, CacheFragType type, const char *hostname, int host_len) { @@ -93,7 +91,7 @@ Cache::open_read(Continuation *cont, const CacheKey *key, CacheFragType type, co } Action * -Cache::open_read(Continuation *cont, const CacheKey *key, CacheHTTPHdr *request, OverridableHttpConfigParams *params, +Cache::open_read(Continuation *cont, const CacheKey *key, CacheHTTPHdr *request, const OverridableHttpConfigParams *params, CacheFragType type, const char *hostname, int host_len) { if (!CacheProcessor::IsCacheReady(type)) { @@ -719,8 +717,46 @@ CacheVC::openReadMain(int /* event ATS_UNUSED */, Event * /* e ATS_UNUSED */) bytes = doc->len - doc_pos; if (is_debug_tag_set("cache_seek")) { char target_key_str[CRYPTO_HEX_SIZE]; - key.toHexStr(target_key_str); - Debug("cache_seek", "Read # %d @ %" PRId64 "/%d for %" PRId64, fragment, doc_pos, doc->len, bytes); + Debug("cache_seek", "Read # %d @ %" PRId64 "/%d for %" PRId64 " %s", fragment, doc_pos, doc->len, bytes, + key.toHexStr(target_key_str)); + } + + // This shouldn't happen for HTTP assets but it does + // occasionally in production. This is a temporary fix + // to clean up broken objects until the root cause can + // be found. It must be the case that either the fragment + // offsets are incorrect or a fragment table isn't being + // created when it should be. + if (frag_type == CACHE_FRAG_TYPE_HTTP && bytes < 0) { + char xt[CRYPTO_HEX_SIZE]; + char yt[CRYPTO_HEX_SIZE]; + + int url_length = 0; + char const *url_text = nullptr; + if (request.valid()) { + url_text = request.url_get()->string_get_ref(&url_length); + } + + int64_t prev_frag_size = 0; + if (fragment && frags) { + prev_frag_size = static_cast(frags[fragment - 1]); + } + + Warning("cache_seek range request bug: read %s targ %s - %s frag # %d (prev_frag %" PRId64 ") @ %" PRId64 "/%d for %" PRId64 + " tot %" PRId64 " url '%.*s'", + doc->key.toHexStr(xt), key.toHexStr(yt), f.single_fragment ? "single" : "multi", fragment, prev_frag_size, doc_pos, + doc->len, bytes, doc->total_len, url_length, url_text); + + doc->magic = DOC_CORRUPT; + + CACHE_TRY_LOCK(lock, vol->mutex, mutex->thread_holding); + if (!lock.is_locked()) { + SET_HANDLER(&CacheVC::openReadDirDelete); + VC_SCHED_LOCK_RETRY(); + } + + dir_delete(&earliest_key, vol, &earliest_dir); + goto Lerror; } } if (ntodo <= 0) { @@ -1220,3 +1256,18 @@ CacheVC::openReadStartHead(int event, Event *e) SET_HANDLER(&CacheVC::openReadStartEarliest); return openReadStartEarliest(event, e); } + +/* + Handle a directory delete event in case of some detected corruption. +*/ +int +CacheVC::openReadDirDelete(int event, Event *e) +{ + MUTEX_TRY_LOCK(lock, vol->mutex, mutex->thread_holding); + if (!lock.is_locked()) { + VC_SCHED_LOCK_RETRY(); + } + + dir_delete(&earliest_key, vol, &earliest_dir); + return calluser(VC_EVENT_ERROR); +} diff --git a/iocore/cache/CacheTest.cc b/iocore/cache/CacheTest.cc index fac78b90011..d4f1b0cd015 100644 --- a/iocore/cache/CacheTest.cc +++ b/iocore/cache/CacheTest.cc @@ -28,8 +28,6 @@ #include #include -using namespace std; - CacheTestSM::CacheTestSM(RegressionTest *t, const char *name) : RegressionSM(t), cache_test_name(name) { SET_HANDLER(&CacheTestSM::event_handler); @@ -97,7 +95,7 @@ CacheTestSM::event_handler(int event, void *data) cancel_timeout(); cache_action = nullptr; cache_vc = static_cast(data); - buffer = new_empty_MIOBuffer(); + buffer = new_empty_MIOBuffer(BUFFER_SIZE_INDEX_32K); buffer_reader = buffer->alloc_reader(); if (open_read_callout() < 0) { goto Lclose_error_next; @@ -131,7 +129,7 @@ CacheTestSM::event_handler(int event, void *data) cancel_timeout(); cache_action = nullptr; cache_vc = static_cast(data); - buffer = new_empty_MIOBuffer(); + buffer = new_empty_MIOBuffer(BUFFER_SIZE_INDEX_32K); buffer_reader = buffer->alloc_reader(); if (open_write_callout() < 0) { goto Lclose_error_next; @@ -333,13 +331,14 @@ EXCLUSIVE_REGRESSION_TEST(cache)(RegressionTest *t, int /* atype ATS_UNUSED */, remove_fail_test.expect_event = CACHE_EVENT_REMOVE_FAILED; rand_CacheKey(&remove_fail_test.key, thread->mutex); - CACHE_SM(t, replace_write_test, - { cacheProcessor.open_write(this, &key, CACHE_FRAG_TYPE_NONE, 100, CACHE_WRITE_OPT_SYNC); } int open_write_callout() { - header.serial = 10; - cache_vc->set_header(&header, sizeof(header)); - cvio = cache_vc->do_io_write(this, nbytes, buffer_reader); - return 1; - }); + CACHE_SM( + t, replace_write_test, + { cacheProcessor.open_write(this, &key, CACHE_FRAG_TYPE_NONE, 100, CACHE_WRITE_OPT_SYNC); } int open_write_callout() { + header.serial = 10; + cache_vc->set_header(&header, sizeof(header)); + cvio = cache_vc->do_io_write(this, nbytes, buffer_reader); + return 1; + }); replace_write_test.expect_initial_event = CACHE_EVENT_OPEN_WRITE; replace_write_test.expect_event = VC_EVENT_WRITE_COMPLETE; replace_write_test.nbytes = 100; @@ -365,16 +364,17 @@ EXCLUSIVE_REGRESSION_TEST(cache)(RegressionTest *t, int /* atype ATS_UNUSED */, replace_test.key = replace_write_test.key; replace_test.content_salt = 1; - CACHE_SM(t, replace_read_test, { cacheProcessor.open_read(this, &key); } int open_read_callout() { - CacheTestHeader *h = nullptr; - int hlen = 0; - if (cache_vc->get_header((void **)&h, &hlen) < 0) - return -1; - if (h->serial != 11) - return -1; - cvio = cache_vc->do_io_read(this, nbytes, buffer); - return 1; - }); + CACHE_SM( + t, replace_read_test, { cacheProcessor.open_read(this, &key); } int open_read_callout() { + CacheTestHeader *h = nullptr; + int hlen = 0; + if (cache_vc->get_header((void **)&h, &hlen) < 0) + return -1; + if (h->serial != 11) + return -1; + cvio = cache_vc->do_io_read(this, nbytes, buffer); + return 1; + }); replace_read_test.expect_initial_event = CACHE_EVENT_OPEN_READ; replace_read_test.expect_event = VC_EVENT_READ_COMPLETE; replace_read_test.nbytes = 100; @@ -387,10 +387,11 @@ EXCLUSIVE_REGRESSION_TEST(cache)(RegressionTest *t, int /* atype ATS_UNUSED */, large_write_test.nbytes = 10000000; rand_CacheKey(&large_write_test.key, thread->mutex); - CACHE_SM(t, pread_test, { cacheProcessor.open_read(this, &key); } int open_read_callout() { - cvio = cache_vc->do_io_pread(this, nbytes, buffer, 7000000); - return 1; - }); + CACHE_SM( + t, pread_test, { cacheProcessor.open_read(this, &key); } int open_read_callout() { + cvio = cache_vc->do_io_pread(this, nbytes, buffer, 7000000); + return 1; + }); pread_test.expect_initial_event = CACHE_EVENT_OPEN_READ; pread_test.expect_event = VC_EVENT_READ_COMPLETE; pread_test.nbytes = 100; @@ -550,7 +551,7 @@ test_RamCache(RegressionTest *t, RamCache *cache, const char *name, int64_t cach bool pass = true; CacheKey key; Vol *vol = theCache->key_to_vol(&key, "example.com", sizeof("example.com") - 1); - vector> data; + std::vector> data; cache->init(cache_size, vol); diff --git a/iocore/cache/CacheWrite.cc b/iocore/cache/CacheWrite.cc index b41edcbdeb0..cff181864d3 100644 --- a/iocore/cache/CacheWrite.cc +++ b/iocore/cache/CacheWrite.cc @@ -105,7 +105,9 @@ CacheVC::updateVector(int /* event ATS_UNUSED */, Event * /* e ATS_UNUSED */) called iff the update is a header only update so the fragment data should remain valid. */ - if (alternate_index >= 0) { + // If we are not in header only updating case. Don't copy fragments. + if (alternate_index >= 0 && + ((total_len == 0 && alternate.get_frag_offset_count() == 0) && !(f.allow_empty_doc && this->vio.nbytes == 0))) { alternate.copy_frag_offsets_from(write_vector->get(alternate_index)); } alternate_index = write_vector->insert(&alternate, alternate_index); @@ -358,7 +360,7 @@ Vol::aggWriteDone(int event, Event *e) CacheVC *c = nullptr; while ((c = sync.dequeue())) { if (UINT_WRAP_LTE(c->write_serial + 2, header->write_serial)) { - eventProcessor.schedule_imm_signal(c, ET_CALL, AIO_EVENT_DONE); + eventProcessor.schedule_imm(c, ET_CALL, AIO_EVENT_DONE); } else { sync.push(c); // put it back on the front break; @@ -1026,7 +1028,7 @@ Vol::aggWrite(int event, void * /* e ATS_UNUSED */) ink_assert(false); while ((c = agg.dequeue())) { agg_todo_size -= c->agg_len; - eventProcessor.schedule_imm_signal(c, ET_CALL, AIO_EVENT_DONE); + eventProcessor.schedule_imm(c, ET_CALL, AIO_EVENT_DONE); } return EVENT_CONT; } @@ -1090,7 +1092,7 @@ Vol::aggWrite(int event, void * /* e ATS_UNUSED */) if (event == EVENT_CALL && c->mutex->thread_holding == mutex->thread_holding) { ret = EVENT_RETURN; } else { - eventProcessor.schedule_imm_signal(c, ET_CALL, AIO_EVENT_DONE); + eventProcessor.schedule_imm(c, ET_CALL, AIO_EVENT_DONE); } } return ret; diff --git a/iocore/cache/I_Cache.h b/iocore/cache/I_Cache.h index b0fd5441ef4..d38716a45b0 100644 --- a/iocore/cache/I_Cache.h +++ b/iocore/cache/I_Cache.h @@ -86,7 +86,7 @@ struct CacheProcessor : public Processor { Action *scan(Continuation *cont, char *hostname = nullptr, int host_len = 0, int KB_per_second = SCAN_KB_PER_SECOND); Action *lookup(Continuation *cont, const HttpCacheKey *key, CacheFragType frag_type = CACHE_FRAG_TYPE_HTTP); inkcoreapi Action *open_read(Continuation *cont, const HttpCacheKey *key, CacheHTTPHdr *request, - OverridableHttpConfigParams *params, time_t pin_in_cache = (time_t)0, + const OverridableHttpConfigParams *params, time_t pin_in_cache = (time_t)0, CacheFragType frag_type = CACHE_FRAG_TYPE_HTTP); Action *open_write(Continuation *cont, int expected_size, const HttpCacheKey *key, CacheHTTPHdr *request, CacheHTTPInfo *old_info, time_t pin_in_cache = (time_t)0, CacheFragType frag_type = CACHE_FRAG_TYPE_HTTP); diff --git a/iocore/cache/I_CacheDefs.h b/iocore/cache/I_CacheDefs.h index cdd39bb2199..b8dfd5c3bd6 100644 --- a/iocore/cache/I_CacheDefs.h +++ b/iocore/cache/I_CacheDefs.h @@ -113,11 +113,11 @@ enum CacheDataType { }; enum CacheFragType { - CACHE_FRAG_TYPE_NONE, - CACHE_FRAG_TYPE_HTTP_V23, ///< DB version 23 or prior. - CACHE_FRAG_TYPE_RTSP, ///< Should be removed once Cache Toolkit is implemented. - CACHE_FRAG_TYPE_HTTP, - NUM_CACHE_FRAG_TYPES + CACHE_FRAG_TYPE_NONE = 0, + CACHE_FRAG_TYPE_UNUSED_1 = 1, //. Because of the history we need to occupy a space + CACHE_FRAG_TYPE_RTSP = 2, ///< Should be removed once Cache Toolkit is implemented. + CACHE_FRAG_TYPE_HTTP = 3, + NUM_CACHE_FRAG_TYPES = 4 }; typedef CryptoHash CacheKey; diff --git a/iocore/cache/I_Store.h b/iocore/cache/I_Store.h index 73a2b6d8e65..ae44bf84aca 100644 --- a/iocore/cache/I_Store.h +++ b/iocore/cache/I_Store.h @@ -62,7 +62,11 @@ struct span_diskid_t { return id[0] == rhs.id[0] && id[1] == rhs.id[1]; } - int64_t &operator[](unsigned i) { return id[i]; } + int64_t & + operator[](unsigned i) + { + return id[i]; + } }; // diff --git a/iocore/cache/Makefile.am b/iocore/cache/Makefile.am index 3c528880f28..307f05bf7c4 100644 --- a/iocore/cache/Makefile.am +++ b/iocore/cache/Makefile.am @@ -116,6 +116,7 @@ test_LDADD = \ @YAMLCPP_LIBS@ \ -lm +if EXPENSIVE_TESTS check_PROGRAMS = \ test_Cache \ test_RWW \ @@ -128,6 +129,7 @@ check_PROGRAMS = \ test_Update_L_to_S \ test_Update_S_to_L \ test_Update_header +endif test_main_SOURCES = \ ./test/main.cc \ diff --git a/iocore/cache/P_CacheArray.h b/iocore/cache/P_CacheArray.h index bc26179c84c..b18c7075df2 100644 --- a/iocore/cache/P_CacheArray.h +++ b/iocore/cache/P_CacheArray.h @@ -81,12 +81,16 @@ template TS_INLINE CacheArray::operator const T *() const return data; } -template TS_INLINE CacheArray::operator T *() +template +TS_INLINE +CacheArray::operator T *() { return data; } -template TS_INLINE T &CacheArray::operator[](int idx) +template +TS_INLINE T & +CacheArray::operator[](int idx) { return data[idx]; } diff --git a/iocore/cache/P_CacheBC.h b/iocore/cache/P_CacheBC.h deleted file mode 100644 index ac4d9255826..00000000000 --- a/iocore/cache/P_CacheBC.h +++ /dev/null @@ -1,137 +0,0 @@ -/** @file - - Backwards compatibility support for the cache. - - @section license License - - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - */ - -#pragma once - -namespace cache_bc -{ -/* This looks kind of dumb, but I think it's useful. We import external structure - dependencies in to this namespace so we can at least (1) notice them and - (2) change them if the current structure changes. -*/ - -typedef HTTPHdr HTTPHdr_v21; -typedef HdrHeap HdrHeap_v23; -typedef CryptoHash CryptoHash_v23; -typedef HTTPCacheAlt HTTPCacheAlt_v23; - -/** Cache backwards compatibility structure - the fragment table. - This is copied from @c HTTPCacheAlt in @c HTTP.h. -*/ -struct HTTPCacheFragmentTable { - /// # of fragment offsets in this alternate. - /// @note This is one less than the number of fragments. - int m_frag_offset_count; - /// Type of offset for a fragment. - typedef uint64_t FragOffset; - /// Table of fragment offsets. - /// @note The offsets are forward looking so that frag[0] is the - /// first byte past the end of fragment 0 which is also the first - /// byte of fragment 1. For this reason there is no fragment offset - /// for the last fragment. - FragOffset *m_frag_offsets; - /// # of fragment offsets built in to object. - static int const N_INTEGRAL_FRAG_OFFSETS = 4; - /// Integral fragment offset table. - FragOffset m_integral_frag_offsets[N_INTEGRAL_FRAG_OFFSETS]; -}; - -// From before moving the fragment table to the alternate. -struct HTTPCacheAlt_v21 { - uint32_t m_magic; - - int32_t m_writeable; - int32_t m_unmarshal_len; - - int32_t m_id; - int32_t m_rid; - - int32_t m_object_key[4]; - int32_t m_object_size[2]; - - HTTPHdr_v21 m_request_hdr; - HTTPHdr_v21 m_response_hdr; - - time_t m_request_sent_time; - time_t m_response_received_time; - - RefCountObj *m_ext_buffer; - - // The following methods were added for BC support. - // Checks itself to verify that it is unmarshalled and v21 format. - bool - is_unmarshalled_format() const - { - return CACHE_ALT_MAGIC_MARSHALED == m_magic && reinterpret_cast(m_request_hdr.m_heap) == sizeof(*this); - } -}; - -/// Really just a namespace, doesn't depend on any of the members. -struct HTTPInfo_v21 { - typedef uint64_t FragOffset; - /// Version upgrade methods - /// @a src , @a dst , and @a n are updated upon return. - /// @a n is the space in @a dst remaining. - /// @return @c false if something went wrong. - static bool copy_and_upgrade_unmarshalled_to_v23(char *&dst, char *&src, size_t &length, int n_frags, FragOffset *frag_offsets); - /// The size of the marshalled data of a marshalled alternate header. - static size_t marshalled_length(void *data); -}; - -/// Pre version 24. -struct Doc_v23 { - uint32_t magic; // DOC_MAGIC - uint32_t len; // length of this segment (including hlen, flen & sizeof(Doc), unrounded) - uint64_t total_len; // total length of document - CryptoHash_v23 first_key; ///< first key in object. - CryptoHash_v23 key; ///< Key for this doc. - uint32_t hlen; ///< Length of this header. - uint32_t doc_type : 8; ///< Doc type - indicates the format of this structure and its content. - uint32_t _flen : 24; ///< Fragment table length. - uint32_t sync_serial; - uint32_t write_serial; - uint32_t pinned; // pinned until - uint32_t checksum; - - char *hdr(); - char *data(); - size_t data_len(); -}; - -char * -Doc_v23::data() -{ - return reinterpret_cast(this) + sizeof(Doc_v23) + _flen + hlen; -} -size_t -Doc_v23::data_len() -{ - return len - sizeof(Doc_v23) - hlen; -} -char * -Doc_v23::hdr() -{ - return reinterpret_cast(this) + sizeof(Doc_v23); -} - -} // namespace cache_bc diff --git a/iocore/cache/P_CacheHosting.h b/iocore/cache/P_CacheHosting.h index 90822e2c44b..ba4af9b20a1 100644 --- a/iocore/cache/P_CacheHosting.h +++ b/iocore/cache/P_CacheHosting.h @@ -171,6 +171,7 @@ struct ConfigVol { CacheType scheme; off_t size; bool in_percent; + bool ramcache_enabled; int percent; CacheVol *cachep; LINK(ConfigVol, link); diff --git a/iocore/cache/P_CacheInternal.h b/iocore/cache/P_CacheInternal.h index 1a61b7194c0..3e05f2cdcf9 100644 --- a/iocore/cache/P_CacheInternal.h +++ b/iocore/cache/P_CacheInternal.h @@ -326,6 +326,7 @@ struct CacheVC : public CacheVConnection { int openReadFromWriterMain(int event, Event *e); int openReadFromWriterFailure(int event, Event *); int openReadChooseWriter(int event, Event *e); + int openReadDirDelete(int event, Event *e); int openWriteCloseDir(int event, Event *e); int openWriteCloseHeadDone(int event, Event *e); @@ -441,7 +442,7 @@ struct CacheVC : public CacheVConnection { CacheFragType frag_type; CacheHTTPInfo *info; CacheHTTPInfoVector *write_vector; - OverridableHttpConfigParams *params; + const OverridableHttpConfigParams *params; int header_len; // for communicating with agg_copy int frag_len; // for communicating with agg_copy uint32_t write_len; // for communicating with agg_copy @@ -988,7 +989,7 @@ struct Cache { const char *hostname = nullptr, int host_len = 0); Action *scan(Continuation *cont, const char *hostname = nullptr, int host_len = 0, int KB_per_second = 2500); - Action *open_read(Continuation *cont, const CacheKey *key, CacheHTTPHdr *request, OverridableHttpConfigParams *params, + Action *open_read(Continuation *cont, const CacheKey *key, CacheHTTPHdr *request, const OverridableHttpConfigParams *params, CacheFragType type, const char *hostname, int host_len); Action *open_write(Continuation *cont, const CacheKey *key, CacheHTTPInfo *old_info, time_t pin_in_cache = (time_t)0, const CacheKey *key1 = nullptr, CacheFragType type = CACHE_FRAG_TYPE_HTTP, const char *hostname = nullptr, diff --git a/iocore/cache/P_CacheTest.h b/iocore/cache/P_CacheTest.h index ba9814e72ae..47f0cfd3bbe 100644 --- a/iocore/cache/P_CacheTest.h +++ b/iocore/cache/P_CacheTest.h @@ -44,7 +44,7 @@ struct PinnedDocTable : public Continuation { int remove(CacheKey *key); int cleanup(int event, Event *e); - PinnedDocTable() : Continuation(new_ProxyMutex()) { memset(bucket, 0, sizeof(Queue) * PINNED_DOC_TABLE_SIZE); } + PinnedDocTable() : Continuation(new_ProxyMutex()) { ink_zero(bucket); } }; struct CacheTestHost { diff --git a/iocore/cache/P_CacheVol.h b/iocore/cache/P_CacheVol.h index 5b3254ea7bf..a51d245e585 100644 --- a/iocore/cache/P_CacheVol.h +++ b/iocore/cache/P_CacheVol.h @@ -279,12 +279,13 @@ struct AIO_Callback_handler : public Continuation { }; struct CacheVol { - int vol_number = -1; - int scheme = 0; - off_t size = 0; - int num_vols = 0; - Vol **vols = nullptr; - DiskVol **disk_vols = nullptr; + int vol_number = -1; + int scheme = 0; + off_t size = 0; + int num_vols = 0; + bool ramcache_enabled = true; + Vol **vols = nullptr; + DiskVol **disk_vols = nullptr; LINK(CacheVol, link); // per volume stats RecRawStatBlock *vol_rsb = nullptr; diff --git a/iocore/cache/Store.cc b/iocore/cache/Store.cc index c5f009faea7..22592fdb726 100644 --- a/iocore/cache/Store.cc +++ b/iocore/cache/Store.cc @@ -24,6 +24,7 @@ #include "tscore/ink_platform.h" #include "P_Cache.h" #include "tscore/I_Layout.h" +#include "tscore/Filenames.h" #include "tscore/ink_file.h" #include "tscore/Tokenizer.h" #include "tscore/SimpleTokenizer.h" @@ -36,7 +37,6 @@ // // Store // - const char Store::VOLUME_KEY[] = "volume"; const char Store::HASH_BASE_STRING_KEY[] = "id"; @@ -325,13 +325,13 @@ Store::read_config() Span *sd = nullptr, *cur = nullptr; Span *ns; ats_scoped_fd fd; - ats_scoped_str storage_path(RecConfigReadConfigPath("proxy.config.cache.storage_filename", "storage.config")); + ats_scoped_str storage_path(RecConfigReadConfigPath(nullptr, ts::filename::STORAGE)); - Note("storage.config loading ..."); + Note("%s loading ...", ts::filename::STORAGE); Debug("cache_init", "Store::read_config, fd = -1, \"%s\"", (const char *)storage_path); fd = ::open(storage_path, O_RDONLY); if (fd < 0) { - Error("storage.config failed to load"); + Error("%s failed to load", ts::filename::STORAGE); return Result::failure("open %s: %s", (const char *)storage_path, strerror(errno)); } @@ -372,7 +372,7 @@ Store::read_config() const char *end; if ((size = ink_atoi64(e, &end)) <= 0 || *end != '\0') { delete sd; - Error("storage.config failed to load"); + Error("%s failed to load", ts::filename::STORAGE); return Result::failure("failed to parse size '%s'", e); } } else if (0 == strncasecmp(HASH_BASE_STRING_KEY, e, sizeof(HASH_BASE_STRING_KEY) - 1)) { @@ -390,7 +390,7 @@ Store::read_config() } if (!*e || !ParseRules::is_digit(*e) || 0 >= (volume_num = ink_atoi(e))) { delete sd; - Error("storage.config failed to load"); + Error("%s failed to load", ts::filename::STORAGE); return Result::failure("failed to parse volume number '%s'", e); } } @@ -442,7 +442,7 @@ Store::read_config() sd = nullptr; // these are all used. sort(); - Note("storage.config finished loading"); + Note("%s finished loading", ts::filename::STORAGE); return Result::ok(); } diff --git a/iocore/cache/test/main.cc b/iocore/cache/test/main.cc index 6e199600ea0..7c57a0cc638 100644 --- a/iocore/cache/test/main.cc +++ b/iocore/cache/test/main.cc @@ -42,7 +42,7 @@ struct EventProcessorListener : Catch::TestEventListenerBase { testRunStarting(Catch::TestRunInfo const &testRunInfo) override { BaseLogFile *base_log_file = new BaseLogFile("stderr"); - diags = new Diags(testRunInfo.name.c_str(), "*" /* tags */, "" /* actions */, base_log_file); + diags = new Diags(testRunInfo.name, "*" /* tags */, "" /* actions */, base_log_file); diags->activate_taglist("cache.*|agg.*|locks", DiagsTagType_Debug); diags->config.enabled[DiagsTagType_Debug] = true; diags->show_location = SHOW_LOCATION_DEBUG; diff --git a/iocore/cache/test/test_Update_L_to_S.cc b/iocore/cache/test/test_Update_L_to_S.cc index 7ace4acc774..9b4ea8e75fb 100644 --- a/iocore/cache/test/test_Update_L_to_S.cc +++ b/iocore/cache/test/test_Update_L_to_S.cc @@ -51,6 +51,7 @@ class CacheUpdateReadAgain : public CacheTestHandler switch (event) { case CACHE_EVENT_OPEN_READ: base->do_io_read(); + REQUIRE(base->vc->alternate.get_frag_offset_count() < 5); break; case VC_EVENT_READ_READY: base->reenable(); diff --git a/iocore/cache/test/test_Update_S_to_L.cc b/iocore/cache/test/test_Update_S_to_L.cc index def645d217c..5b0f6d16faa 100644 --- a/iocore/cache/test/test_Update_S_to_L.cc +++ b/iocore/cache/test/test_Update_S_to_L.cc @@ -51,6 +51,7 @@ class CacheUpdateReadAgain : public CacheTestHandler switch (event) { case CACHE_EVENT_OPEN_READ: base->do_io_read(); + REQUIRE(base->vc->alternate.get_frag_offset_count() > 8); break; case VC_EVENT_READ_READY: base->reenable(); diff --git a/iocore/dns/DNS.cc b/iocore/dns/DNS.cc index 7a7df3b6a36..d33c4dc28a5 100644 --- a/iocore/dns/DNS.cc +++ b/iocore/dns/DNS.cc @@ -468,6 +468,15 @@ DNSHandler::open_cons(sockaddr const *target, bool failed, int icon) Open (and close) connections as necessary and also assures that the epoll fd struct is properly updated. + target == nullptr : + open connection to DNSHandler::ip. + generally, the icon should be 0 if target == nullptr. + + target != nullptr and icon == 0 : + open connection to target, and the target is assigned to DNSHandler::ip. + + target != nullptr and icon > 0 : + open connection to target. */ void DNSHandler::open_con(sockaddr const *target, bool failed, int icon, bool over_tcp) @@ -475,6 +484,8 @@ DNSHandler::open_con(sockaddr const *target, bool failed, int icon, bool over_tc ip_port_text_buffer ip_text; PollDescriptor *pd = get_PollDescriptor(dnsProcessor.thread); + ink_assert(target != &ip.sa); + if (!icon && target) { ats_ip_copy(&ip, target); } else if (!target) { @@ -549,6 +560,12 @@ DNSHandler::startEvent(int /* event ATS_UNUSED */, Event *e) dns_handler_initialized = 1; SET_HANDLER(&DNSHandler::mainEvent); if (dns_ns_rr) { + /* Round Robin mode: + * Establish a connection to each DNS server to make it a connection pool. + * For each DNS Request, a connection is picked up from the pool by round robin method. + * + * The first DNS server is assigned to DNSHandler::ip within open_con() function. + */ int max_nscount = m_res->nscount; if (max_nscount > MAX_NAMED) { max_nscount = MAX_NAMED; @@ -565,6 +582,18 @@ DNSHandler::startEvent(int /* event ATS_UNUSED */, Event *e) } dns_ns_rr_init_down = 0; } else { + /* Primary - Secondary mode: + * Establish a connection to the Primary DNS server. + * It always send DNS requests to the Primary DNS server. + * If the Primary DNS server dies, + * - it will attempt to send DNS requests to the secondary DNS server until the Primary DNS server is back. + * - and keep to detect the health of the Primary DNS server. + * If DNSHandler::recv_dns() got a valid DNS response from the Primary DNS server, + * - it means that the Primary DNS server returns. + * - it send all DNS requests to the Primary DNS server. + * + * The first DNS server is the Primary DNS server, and it is assigned to DNSHandler::ip within validate_ip() function. + */ open_cons(nullptr); // use current target address. n_con = 1; } @@ -587,8 +616,8 @@ DNSHandler::startEvent_sdns(int /* event ATS_UNUSED */, Event *e) this->validate_ip(); SET_HANDLER(&DNSHandler::mainEvent); - open_cons(&ip.sa, false, n_con); - ++n_con; // TODO should n_con be zeroed? + open_cons(nullptr, false, 0); + n_con = 1; return EVENT_CONT; } @@ -597,7 +626,7 @@ static inline int _ink_res_mkquery(ink_res_state res, char *qname, int qtype, unsigned char *buffer, bool over_tcp = false) { int offset = over_tcp ? tcp_data_length_offset : 0; - int r = ink_res_mkquery(res, QUERY, qname, C_IN, qtype, nullptr, 0, nullptr, buffer + offset, MAX_DNS_PACKET_LEN); + int r = ink_res_mkquery(res, QUERY, qname, C_IN, qtype, nullptr, 0, nullptr, buffer + offset, MAX_DNS_REQUEST_LEN - offset); if (over_tcp) { NS_PUT16(r, buffer); } @@ -629,7 +658,7 @@ DNSHandler::retry_named(int ndx, ink_hrtime t, bool reopen) } bool over_tcp = dns_conn_mode == DNS_CONN_MODE::TCP_ONLY; int con_fd = over_tcp ? tcpcon[ndx].fd : udpcon[ndx].fd; - unsigned char buffer[MAX_DNS_PACKET_LEN]; + unsigned char buffer[MAX_DNS_REQUEST_LEN]; Debug("dns", "trying to resolve '%s' from DNS connection, ndx %d", try_server_names[try_servers], ndx); int r = _ink_res_mkquery(m_res, try_server_names[try_servers], T_A, buffer, over_tcp); try_servers = (try_servers + 1) % countof(try_server_names); @@ -647,10 +676,10 @@ DNSHandler::try_primary_named(bool reopen) if (reopen && ((t - last_primary_reopen) > DNS_PRIMARY_REOPEN_PERIOD)) { Debug("dns", "try_primary_named: reopening primary DNS connection"); last_primary_reopen = t; - open_cons(&ip.sa, true, 0); + open_cons(nullptr, true, 0); } if ((t - last_primary_retry) > DNS_PRIMARY_RETRY_PERIOD) { - unsigned char buffer[MAX_DNS_PACKET_LEN]; + unsigned char buffer[MAX_DNS_REQUEST_LEN]; bool over_tcp = dns_conn_mode == DNS_CONN_MODE::TCP_ONLY; int con_fd = over_tcp ? tcpcon[0].fd : udpcon[0].fd; last_primary_retry = t; @@ -824,7 +853,7 @@ DNSHandler::recv_dns(int /* event ATS_UNUSED */, Event * /* e ATS_UNUSED */) goto Lerror; } dnsc->tcp_data.total_length = ntohs(dnsc->tcp_data.total_length); - if (res != sizeof(dnsc->tcp_data.total_length) || dnsc->tcp_data.total_length > MAX_DNS_PACKET_LEN) { + if (res != sizeof(dnsc->tcp_data.total_length)) { goto Lerror; } } @@ -852,7 +881,7 @@ DNSHandler::recv_dns(int /* event ATS_UNUSED */, Event * /* e ATS_UNUSED */) hostent_cache = dnsBufAllocator.alloc(); } - res = socketManager.recvfrom(dnsc->fd, hostent_cache->buf, MAX_DNS_PACKET_LEN, 0, &from_ip.sa, &from_length); + res = socketManager.recvfrom(dnsc->fd, hostent_cache->buf, MAX_DNS_RESPONSE_LEN, 0, &from_ip.sa, &from_length); Debug("dns", "DNSHandler::recv_dns res = [%d]", res); if (res == -EAGAIN) { break; @@ -1092,7 +1121,7 @@ static bool write_dns_event(DNSHandler *h, DNSEntry *e, bool over_tcp) { ProxyMutex *mutex = h->mutex.get(); - unsigned char buffer[MAX_DNS_PACKET_LEN]; + unsigned char buffer[MAX_DNS_REQUEST_LEN]; int offset = over_tcp ? tcp_data_length_offset : 0; HEADER *header = reinterpret_cast
(buffer + offset); int r = 0; @@ -1408,7 +1437,7 @@ DNSEntry::post(DNSHandler *h, HostEnt *ent) } else { mutex = action.mutex; SET_HANDLER(&DNSEntry::postOneEvent); - submit_thread->schedule_imm_signal(this); + submit_thread->schedule_imm(this); } return 0; } diff --git a/iocore/dns/DNSConnection.cc b/iocore/dns/DNSConnection.cc index 546215641a9..1f2dab651c2 100644 --- a/iocore/dns/DNSConnection.cc +++ b/iocore/dns/DNSConnection.cc @@ -157,7 +157,7 @@ DNSConnection::connect(sockaddr const *addr, Options const &opt) p = static_cast((p % (LAST_RANDOM_PORT - FIRST_RANDOM_PORT)) + FIRST_RANDOM_PORT); ats_ip_port_cast(&bind_addr.sa) = htons(p); // stuff port in sockaddr. Debug("dns", "random port = %s", ats_ip_nptop(&bind_addr.sa, b, sizeof b)); - if ((res = socketManager.ink_bind(fd, &bind_addr.sa, bind_size, Proto)) < 0) { + if (socketManager.ink_bind(fd, &bind_addr.sa, bind_size, Proto) < 0) { continue; } goto Lok; diff --git a/iocore/dns/I_DNSProcessor.h b/iocore/dns/I_DNSProcessor.h index a0ec65115eb..83f4bd230b1 100644 --- a/iocore/dns/I_DNSProcessor.h +++ b/iocore/dns/I_DNSProcessor.h @@ -28,11 +28,12 @@ const int DOMAIN_SERVICE_PORT = NAMESERVER_PORT; const int DEFAULT_DOMAIN_NAME_SERVER = 0; -const int MAX_DNS_PACKET_LEN = 8192; -const int DNS_RR_MAX_COUNT = (MAX_DNS_PACKET_LEN - HFIXEDSZ + RRFIXEDSZ - 1) / RRFIXEDSZ; -const int DNS_MAX_ALIASES = DNS_RR_MAX_COUNT; -const int DNS_MAX_ADDRS = DNS_RR_MAX_COUNT; -const int DNS_HOSTBUF_SIZE = MAX_DNS_PACKET_LEN; +const int MAX_DNS_REQUEST_LEN = NS_PACKETSZ; +const int MAX_DNS_RESPONSE_LEN = 65536; +const int DNS_RR_MAX_COUNT = (MAX_DNS_RESPONSE_LEN - HFIXEDSZ + RRFIXEDSZ - 1) / RRFIXEDSZ; +const int DNS_MAX_ALIASES = DNS_RR_MAX_COUNT; +const int DNS_MAX_ADDRS = DNS_RR_MAX_COUNT; +const int DNS_HOSTBUF_SIZE = MAX_DNS_RESPONSE_LEN; /** All buffering required to handle a DNS receipt. For asynchronous DNS, @@ -41,10 +42,10 @@ const int DNS_HOSTBUF_SIZE = MAX_DNS_PACKET_LEN; */ struct HostEnt : RefCountObj { - struct hostent ent = {.h_name = nullptr, .h_aliases = nullptr, .h_addrtype = 0, .h_length = 0, .h_addr_list = nullptr}; - uint32_t ttl = 0; - int packet_size = 0; - char buf[MAX_DNS_PACKET_LEN] = {0}; + struct hostent ent = {.h_name = nullptr, .h_aliases = nullptr, .h_addrtype = 0, .h_length = 0, .h_addr_list = nullptr}; + uint32_t ttl = 0; + int packet_size = 0; + char buf[MAX_DNS_RESPONSE_LEN] = {0}; u_char *host_aliases[DNS_MAX_ALIASES] = {nullptr}; u_char *h_addr_ptrs[DNS_MAX_ADDRS + 1] = {nullptr}; u_char hostbuf[DNS_HOSTBUF_SIZE] = {0}; diff --git a/iocore/dns/P_SplitDNSProcessor.h b/iocore/dns/P_SplitDNSProcessor.h index 41d0fb92172..316e2eba6de 100644 --- a/iocore/dns/P_SplitDNSProcessor.h +++ b/iocore/dns/P_SplitDNSProcessor.h @@ -72,16 +72,9 @@ struct SplitDNSResult { ------------ */ DNSResultType r = DNS_SRVR_UNDEFINED; - DNSServer *get_dns_record(); - int get_dns_srvr_count(); - - /* ------------ - private - ------------ */ int m_line_number = 0; SplitDNSRecord *m_rec = nullptr; - bool m_wrap_around = false; }; /* -------------------------------------------------------------- diff --git a/iocore/dns/SplitDNS.cc b/iocore/dns/SplitDNS.cc index 827204b7e32..49f5a99ee5c 100644 --- a/iocore/dns/SplitDNS.cc +++ b/iocore/dns/SplitDNS.cc @@ -30,6 +30,7 @@ #include "tscore/ink_platform.h" #include "tscore/Tokenizer.h" +#include "tscore/Filenames.h" #ifdef SPLIT_DNS #include @@ -130,7 +131,7 @@ SplitDNSConfig::reconfigure() return; } - Note("splitdns.config loading ..."); + Note("%s loading ...", ts::filename::SPLITDNS); SplitDNS *params = new SplitDNS; @@ -138,8 +139,7 @@ SplitDNSConfig::reconfigure() params->m_DNSSrvrTable = new DNS_table("proxy.config.dns.splitdns.filename", modulePrefix, &sdns_dest_tags); if (nullptr == params->m_DNSSrvrTable || (0 == params->m_DNSSrvrTable->getEntryCount())) { - Error("splitdns.config failed to load"); - Warning("No NAMEDs provided! Disabling SplitDNS"); + Warning("Failed to load %s - No NAMEDs provided! Disabling SplitDNS", ts::filename::SPLITDNS); gsplit_dns_enabled = 0; delete params; return; @@ -159,7 +159,7 @@ SplitDNSConfig::reconfigure() SplitDNSConfig::print(); } - Note("splitdns.config finished loading"); + Note("%s finished loading", ts::filename::SPLITDNS); } /* -------------------------------------------------------------- @@ -219,7 +219,6 @@ SplitDNS::findServer(RequestData *rdata, SplitDNSResult *result) result->m_rec = nullptr; result->m_line_number = 0xffffffff; - result->m_wrap_around = false; /* --------------------------- the 'alleged' fast path ... @@ -476,7 +475,7 @@ SplitDNSRecord::Init(matcher_line *line_info) } if (!ats_is_ip(&m_servers.x_server_ip[0].sa)) { - return Result::failure("%s No server specified in splitdns.config at line %d", modulePrefix, line_num); + return Result::failure("%s No server specified in %s at line %d", modulePrefix, ts::filename::SPLITDNS, line_num); } DNSHandler *dnsH = new DNSHandler; @@ -505,7 +504,7 @@ SplitDNSRecord::Init(matcher_line *line_info) if (line_info->num_el > 0) { const char *tmp = ProcessModifiers(line_info); if (tmp != nullptr) { - return Result::failure("%s %s at line %d in splitdns.config", modulePrefix, tmp, line_num); + return Result::failure("%s %s at line %d in %s", modulePrefix, tmp, line_num, ts::filename::SPLITDNS); } } diff --git a/iocore/eventsystem/EventSystem.cc b/iocore/eventsystem/EventSystem.cc index f5d5dc03ae3..14971779d51 100644 --- a/iocore/eventsystem/EventSystem.cc +++ b/iocore/eventsystem/EventSystem.cc @@ -34,8 +34,7 @@ void ink_event_system_init(ts::ModuleVersion v) { ink_release_assert(v.check(EVENT_SYSTEM_MODULE_INTERNAL_VERSION)); - int config_max_iobuffer_size = DEFAULT_MAX_BUFFER_SIZE; - int iobuffer_advice = 0; + int iobuffer_advice = 0; // For backwards compatibility make sure to allow thread_freelist_size // This needs to change in 6.0 @@ -43,16 +42,6 @@ ink_event_system_init(ts::ModuleVersion v) REC_EstablishStaticConfigInt32(thread_freelist_low_watermark, "proxy.config.allocator.thread_freelist_low_watermark"); - REC_ReadConfigInteger(config_max_iobuffer_size, "proxy.config.io.max_buffer_size"); - - max_iobuffer_size = buffer_size_to_index(config_max_iobuffer_size, DEFAULT_BUFFER_SIZES - 1); - if (default_small_iobuffer_size > max_iobuffer_size) { - default_small_iobuffer_size = max_iobuffer_size; - } - if (default_large_iobuffer_size > max_iobuffer_size) { - default_large_iobuffer_size = max_iobuffer_size; - } - #ifdef MADV_DONTDUMP // This should only exist on Linux 3.4 and higher. RecBool dont_dump_enabled = true; RecGetRecordBool("proxy.config.allocator.dontdump_iobuffers", &dont_dump_enabled, false); diff --git a/iocore/eventsystem/IOBuffer.cc b/iocore/eventsystem/IOBuffer.cc index e1a255a27d5..61f71f56086 100644 --- a/iocore/eventsystem/IOBuffer.cc +++ b/iocore/eventsystem/IOBuffer.cc @@ -26,7 +26,6 @@ **************************************************************************/ #include "tscore/ink_defs.h" -#include "I_MIOBufferWriter.h" #include "P_EventSystem.h" // @@ -60,26 +59,9 @@ init_buffer_allocators(int iobuffer_advice) } } -int64_t -MIOBuffer::remove_append(IOBufferReader *r) -{ - int64_t l = 0; - while (r->block) { - Ptr b = r->block; - r->block = r->block->next; - b->_start += r->start_offset; - if (b->start() >= b->end()) { - r->start_offset = -r->start_offset; - continue; - } - r->start_offset = 0; - l += b->read_avail(); - append_block(b.get()); - } - r->mbuf->_writer = nullptr; - return l; -} - +// +// MIOBuffer +// int64_t MIOBuffer::write(const void *abuf, int64_t alen) { @@ -134,7 +116,7 @@ MIOBuffer::write(IOBufferBlock const *b, int64_t alen, int64_t offset) continue; } int64_t bytes; - if (len < 0 || len >= max_bytes) { + if (len >= max_bytes) { bytes = max_bytes; } else { bytes = len; @@ -151,28 +133,29 @@ MIOBuffer::write(IOBufferBlock const *b, int64_t alen, int64_t offset) return alen - len; } -int64_t -MIOBuffer::puts(char *s, int64_t len) +bool +MIOBuffer::is_max_read_avail_more_than(int64_t size) { - char *pc = end(); - char *pb = s; - while (pc < buf_end()) { - if (len-- <= 0) { - return -1; - } - if (!*pb || *pb == '\n') { - int64_t n = static_cast(pb - s); - memcpy(end(), s, n + 1); // Up to and including '\n' - end()[n + 1] = 0; - fill(n + 1); - return n + 1; + bool no_reader = true; + for (auto &reader : this->readers) { + if (reader.allocated()) { + if (reader.is_read_avail_more_than(size)) { + return true; + } + no_reader = false; } - pc++; - pb++; } - return 0; + + if (no_reader && this->_writer) { + return (this->_writer->read_avail() > size); + } + + return false; } +// +// IOBufferReader +// int64_t IOBufferReader::read(void *ab, int64_t len) { @@ -262,6 +245,9 @@ IOBufferReader::memcpy(void *ap, int64_t len, int64_t offset) return p; } +// +// IOBufferChain +// int64_t IOBufferChain::write(IOBufferBlock *blocks, int64_t length, int64_t offset) { @@ -348,76 +334,3 @@ IOBufferChain::consume(int64_t size) } return zret; } - -//-- MIOBufferWriter -MIOBufferWriter & -MIOBufferWriter::write(const void *data_, size_t length) -{ - const char *data = static_cast(data_); - - while (length) { - IOBufferBlock *iobbPtr = _miob->first_write_block(); - - if (!iobbPtr) { - addBlock(); - - iobbPtr = _miob->first_write_block(); - - ink_assert(iobbPtr); - } - - size_t writeSize = iobbPtr->write_avail(); - - if (length < writeSize) { - writeSize = length; - } - - std::memcpy(iobbPtr->end(), data, writeSize); - iobbPtr->fill(writeSize); - - data += writeSize; - length -= writeSize; - - _numWritten += writeSize; - } - - return *this; -} - -std::ostream & -MIOBufferWriter::operator>>(std::ostream &stream) const -{ - IOBufferReader *r = _miob->alloc_reader(); - if (r) { - IOBufferBlock *b; - while (nullptr != (b = r->get_current_block())) { - auto n = b->read_avail(); - stream.write(b->start(), n); - r->consume(n); - } - _miob->dealloc_reader(r); - } - return stream; -} - -ssize_t -MIOBufferWriter::operator>>(int fd) const -{ - ssize_t zret = 0; - IOBufferReader *reader = _miob->alloc_reader(); - if (reader) { - IOBufferBlock *b; - while (nullptr != (b = reader->get_current_block())) { - auto n = b->read_avail(); - auto r = ::write(fd, b->start(), n); - if (r <= 0) { - break; - } else { - reader->consume(r); - zret += r; - } - } - _miob->dealloc_reader(reader); - } - return zret; -} diff --git a/iocore/eventsystem/I_Action.h b/iocore/eventsystem/I_Action.h index edd9159d5ce..a0b500e82df 100644 --- a/iocore/eventsystem/I_Action.h +++ b/iocore/eventsystem/I_Action.h @@ -193,10 +193,8 @@ class Action virtual ~Action() {} }; -#define ACTION_RESULT_NONE MAKE_ACTION_RESULT(0) #define ACTION_RESULT_DONE MAKE_ACTION_RESULT(1) #define ACTION_IO_ERROR MAKE_ACTION_RESULT(2) -#define ACTION_RESULT_INLINE MAKE_ACTION_RESULT(3) // Use these classes by // #define ACTION_RESULT_HOST_DB_OFFLINE diff --git a/iocore/eventsystem/I_EThread.h b/iocore/eventsystem/I_EThread.h index c70fb04050f..45afc7309d9 100644 --- a/iocore/eventsystem/I_EThread.h +++ b/iocore/eventsystem/I_EThread.h @@ -126,7 +126,6 @@ class EThread : public Thread */ Event *schedule_imm(Continuation *c, int callback_event = EVENT_IMMEDIATE, void *cookie = nullptr); - Event *schedule_imm_signal(Continuation *c, int callback_event = EVENT_IMMEDIATE, void *cookie = nullptr); /** Schedules the continuation on this EThread to receive an event @@ -300,7 +299,7 @@ class EThread : public Thread EThread &operator=(const EThread &) = delete; ~EThread() override; - Event *schedule(Event *e, bool fast_signal = false); + Event *schedule(Event *e); /** Block of memory to allocate thread specific data e.g. stat system arrays. */ char thread_private[PER_THREAD_DATA]; @@ -314,9 +313,6 @@ class EThread : public Thread ProtectedQueue EventQueueExternal; PriorityEventQueue EventQueue; - EThread **ethreads_to_be_signalled = nullptr; - int n_ethreads_to_be_signalled = 0; - static constexpr int NO_ETHREAD_ID = -1; int id = NO_ETHREAD_ID; unsigned int event_types = 0; diff --git a/iocore/eventsystem/I_EventProcessor.h b/iocore/eventsystem/I_EventProcessor.h index 761f9ca85c7..782137a558e 100644 --- a/iocore/eventsystem/I_EventProcessor.h +++ b/iocore/eventsystem/I_EventProcessor.h @@ -151,11 +151,7 @@ class EventProcessor : public Processor */ Event *schedule_imm(Continuation *c, EventType event_type = ET_CALL, int callback_event = EVENT_IMMEDIATE, void *cookie = nullptr); - /* - provides the same functionality as schedule_imm and also signals the thread immediately - */ - Event *schedule_imm_signal(Continuation *c, EventType event_type = ET_CALL, int callback_event = EVENT_IMMEDIATE, - void *cookie = nullptr); + /** Schedules the continuation on a specific thread group to receive an event at the given timeout. Requests the EventProcessor to schedule @@ -325,11 +321,13 @@ class EventProcessor : public Processor */ int n_ethreads = 0; + bool has_tg_started(int etype); + /*------------------------------------------------------*\ | Unix & non NT Interface | \*------------------------------------------------------*/ - Event *schedule(Event *e, EventType etype, bool fast_signal = false); + Event *schedule(Event *e, EventType etype); EThread *assign_thread(EventType etype); EThread *assign_affinity_by_type(Continuation *cont, EventType etype); @@ -410,3 +408,5 @@ class EventProcessor : public Processor }; extern inkcoreapi class EventProcessor eventProcessor; + +void thread_started(EThread *); diff --git a/iocore/eventsystem/I_IOBuffer.h b/iocore/eventsystem/I_IOBuffer.h index 404b41d029e..6440749d4ac 100644 --- a/iocore/eventsystem/I_IOBuffer.h +++ b/iocore/eventsystem/I_IOBuffer.h @@ -52,21 +52,10 @@ class MIOBuffer; class IOBufferReader; class VIO; -inkcoreapi extern int64_t max_iobuffer_size; -extern int64_t default_small_iobuffer_size; -extern int64_t default_large_iobuffer_size; // matched to size of OS buffers - -#if !defined(TRACK_BUFFER_USER) -#define TRACK_BUFFER_USER 1 -#endif - enum AllocType { NO_ALLOC, - FAST_ALLOCATED, - XMALLOCED, MEMALIGNED, DEFAULT_ALLOC, - CONSTANT, }; #define DEFAULT_BUFFER_NUMBER 128 @@ -143,14 +132,6 @@ void init_buffer_allocators(int iobuffer_advice); NO_ALLOC - - FAST_ALLOCATED - - - - XMALLOCED - - MEMALIGNED @@ -159,10 +140,6 @@ void init_buffer_allocators(int iobuffer_advice); DEFAULT_ALLOC - - CONSTANT - - */ @@ -246,9 +223,7 @@ class IOBufferData : public RefCountObj */ char *_data = nullptr; -#ifdef TRACK_BUFFER_USER const char *_location = nullptr; -#endif /** Constructor. Initializes state for a IOBufferData object. Do not use @@ -438,7 +413,7 @@ class IOBufferBlock : public RefCountObj section in MIOBuffer. */ - void alloc(int64_t i = default_large_iobuffer_size); + void alloc(int64_t i); /** Clear the IOBufferData this IOBufferBlock handles. Clears this @@ -462,11 +437,6 @@ class IOBufferBlock : public RefCountObj */ void set(IOBufferData *d, int64_t len = 0, int64_t offset = 0); void set_internal(void *b, int64_t len, int64_t asize_index); - void realloc_set_internal(void *b, int64_t buf_size, int64_t asize_index); - void realloc(void *b, int64_t buf_size); - void realloc(int64_t i); - void realloc_xmalloc(void *b, int64_t buf_size); - void realloc_xmalloc(int64_t buf_size); /** Frees the IOBufferBlock object and its underlying memory. @@ -481,9 +451,7 @@ class IOBufferBlock : public RefCountObj char *_end = nullptr; char *_buf_end = nullptr; -#ifdef TRACK_BUFFER_USER const char *_location = nullptr; -#endif /** The underlying reference to the allocated memory. A reference to a @@ -950,13 +918,16 @@ class MIOBuffer void append_block(int64_t asize_index); /** - Adds new block to the end of block list using the block size for - the buffer specified when the buffer was allocated. - + Adds a new block to the end of the block list. Note that this does nothing when the next block of the current writer exists. + The block size is the same as specified size when the buffer was allocated. */ void add_block(); /** + Deprecated + + TODO: remove this function. Because ats_xmalloc() doesn't exist anymore. + Adds by reference len bytes of data pointed to by b to the end of the buffer. b MUST be a pointer to the beginning of block allocated from the ats_xmalloc() routine. The data will be deallocated @@ -1022,8 +993,6 @@ class MIOBuffer */ int64_t write(IOBufferChain const *chain, int64_t len = INT64_MAX, int64_t offset = 0); - int64_t remove_append(IOBufferReader *); - /** Returns a pointer to the first writable block on the block chain. Returns nullptr if there are not currently any writable blocks on the @@ -1098,16 +1067,6 @@ class MIOBuffer */ int64_t block_size(); - /** - Returns the default data block size for this buffer. - - */ - int64_t - total_size() - { - return block_size(); - } - /** Returns true if amount of the data outstanding on the buffer exceeds the watermark. @@ -1116,7 +1075,7 @@ class MIOBuffer bool high_water() { - return max_read_avail() > water_mark; + return is_max_read_avail_more_than(this->water_mark); } /** @@ -1141,7 +1100,6 @@ class MIOBuffer { return current_write_avail() <= water_mark; } - void set_size_index(int64_t size); /** Allocates a new IOBuffer reader and sets it's its 'accessor' field @@ -1186,22 +1144,30 @@ class MIOBuffer void dealloc_all_readers(); void set(void *b, int64_t len); - void set_xmalloced(void *b, int64_t len); - void alloc(int64_t i = default_large_iobuffer_size); - void alloc_xmalloc(int64_t buf_size); + void alloc(int64_t i); void append_block_internal(IOBufferBlock *b); int64_t write(IOBufferBlock const *b, int64_t len, int64_t offset); - int64_t puts(char *buf, int64_t len); // internal interface - bool - empty() - { - return !_writer; - } + /** + Get the maximum amount of available data across all of the readers. + If there're no allocated reader, return available data size of current writer. + + This calls IOBufferReader::read_avail() and it could be expensive when it has a ton of IOBufferBlock. + The `is_max_read_avail(int64_t size)` is preferred if possible. + + @return maximum amount of available data + */ int64_t max_read_avail(); + /** + Check if there is more than @a size bytes available to read. + + @return @c true if more than @a size byte are available. + */ + bool is_max_read_avail_more_than(int64_t size); + int max_block_count(); void check_add_block(); @@ -1245,27 +1211,6 @@ class MIOBuffer water_mark = 0; } - void - realloc(int64_t i) - { - _writer->realloc(i); - } - void - realloc(void *b, int64_t buf_size) - { - _writer->realloc(b, buf_size); - } - void - realloc_xmalloc(void *b, int64_t buf_size) - { - _writer->realloc_xmalloc(b, buf_size); - } - void - realloc_xmalloc(int64_t buf_size) - { - _writer->realloc_xmalloc(buf_size); - } - int64_t size_index; /** @@ -1281,9 +1226,7 @@ class MIOBuffer Ptr _writer; IOBufferReader readers[MAX_MIOBUFFER_READERS]; -#ifdef TRACK_BUFFER_USER const char *_location = nullptr; -#endif MIOBuffer(void *b, int64_t bufsize, int64_t aWater_mark); // cppcheck-suppress noExplicitConstructor; allow implicit conversion @@ -1298,13 +1241,13 @@ class MIOBuffer */ struct MIOBufferAccessor { IOBufferReader * - reader() + reader() const { return entry; } MIOBuffer * - writer() + writer() const { return mbuf; } @@ -1315,12 +1258,6 @@ struct MIOBufferAccessor { return mbuf->block_size(); } - int64_t - total_size() const - { - return block_size(); - } - void reader_for(IOBufferReader *abuf); void reader_for(MIOBuffer *abuf); void writer_for(MIOBuffer *abuf); @@ -1349,13 +1286,8 @@ struct MIOBufferAccessor { IOBufferReader *entry = nullptr; }; -extern MIOBuffer *new_MIOBuffer_internal( -#ifdef TRACK_BUFFER_USER - const char *loc, -#endif - int64_t size_index = default_large_iobuffer_size); +extern MIOBuffer *new_MIOBuffer_internal(const char *loc, int64_t size_index); -#ifdef TRACK_BUFFER_USER class MIOBuffer_tracker { const char *loc; @@ -1363,20 +1295,14 @@ class MIOBuffer_tracker public: explicit MIOBuffer_tracker(const char *_loc) : loc(_loc) {} MIOBuffer * - operator()(int64_t size_index = default_large_iobuffer_size) + operator()(int64_t size_index) { return new_MIOBuffer_internal(loc, size_index); } }; -#endif -extern MIOBuffer *new_empty_MIOBuffer_internal( -#ifdef TRACK_BUFFER_USER - const char *loc, -#endif - int64_t size_index = default_large_iobuffer_size); +extern MIOBuffer *new_empty_MIOBuffer_internal(const char *loc, int64_t size_index); -#ifdef TRACK_BUFFER_USER class Empty_MIOBuffer_tracker { const char *loc; @@ -1384,37 +1310,22 @@ class Empty_MIOBuffer_tracker public: explicit Empty_MIOBuffer_tracker(const char *_loc) : loc(_loc) {} MIOBuffer * - operator()(int64_t size_index = default_large_iobuffer_size) + operator()(int64_t size_index) { return new_empty_MIOBuffer_internal(loc, size_index); } }; -#endif /// MIOBuffer allocator/deallocator -#ifdef TRACK_BUFFER_USER #define new_MIOBuffer MIOBuffer_tracker(RES_PATH("memory/IOBuffer/")) #define new_empty_MIOBuffer Empty_MIOBuffer_tracker(RES_PATH("memory/IOBuffer/")) -#else -#define new_MIOBuffer new_MIOBuffer_internal -#define new_empty_MIOBuffer new_empty_MIOBuffer_internal -#endif extern void free_MIOBuffer(MIOBuffer *mio); ////////////////////////////////////////////////////////////////////// -extern IOBufferBlock *new_IOBufferBlock_internal( -#ifdef TRACK_BUFFER_USER - const char *loc -#endif -); +extern IOBufferBlock *new_IOBufferBlock_internal(const char *loc); -extern IOBufferBlock *new_IOBufferBlock_internal( -#ifdef TRACK_BUFFER_USER - const char *loc, -#endif - IOBufferData *d, int64_t len = 0, int64_t offset = 0); +extern IOBufferBlock *new_IOBufferBlock_internal(const char *loc, IOBufferData *d, int64_t len = 0, int64_t offset = 0); -#ifdef TRACK_BUFFER_USER class IOBufferBlock_tracker { const char *loc; @@ -1432,35 +1343,15 @@ class IOBufferBlock_tracker return new_IOBufferBlock_internal(loc, d.get(), len, offset); } }; -#endif /// IOBufferBlock allocator -#ifdef TRACK_BUFFER_USER #define new_IOBufferBlock IOBufferBlock_tracker(RES_PATH("memory/IOBuffer/")) -#else -#define new_IOBufferBlock new_IOBufferBlock_internal -#endif //////////////////////////////////////////////////////////// -extern IOBufferData *new_IOBufferData_internal( -#ifdef TRACK_BUFFER_USER - const char *location, -#endif - int64_t size_index = default_large_iobuffer_size, AllocType type = DEFAULT_ALLOC); - -extern IOBufferData *new_xmalloc_IOBufferData_internal( -#ifdef TRACK_BUFFER_USER - const char *location, -#endif - void *b, int64_t size); +extern IOBufferData *new_IOBufferData_internal(const char *location, int64_t size_index, AllocType type = DEFAULT_ALLOC); -extern IOBufferData *new_constant_IOBufferData_internal( -#ifdef TRACK_BUFFER_USER - const char *locaction, -#endif - void *b, int64_t size); +extern IOBufferData *new_xmalloc_IOBufferData_internal(const char *location, void *b, int64_t size); -#ifdef TRACK_BUFFER_USER class IOBufferData_tracker { const char *loc; @@ -1468,24 +1359,17 @@ class IOBufferData_tracker public: explicit IOBufferData_tracker(const char *_loc) : loc(_loc) {} IOBufferData * - operator()(int64_t size_index = default_large_iobuffer_size, AllocType type = DEFAULT_ALLOC) + operator()(int64_t size_index, AllocType type = DEFAULT_ALLOC) { return new_IOBufferData_internal(loc, size_index, type); } }; -#endif -#ifdef TRACK_BUFFER_USER +// TODO: remove new_xmalloc_IOBufferData. Because ats_xmalloc() doesn't exist anymore. #define new_IOBufferData IOBufferData_tracker(RES_PATH("memory/IOBuffer/")) #define new_xmalloc_IOBufferData(b, size) new_xmalloc_IOBufferData_internal(RES_PATH("memory/IOBuffer/"), (b), (size)) -#define new_constant_IOBufferData(b, size) new_constant_IOBufferData_internal(RES_PATH("memory/IOBuffer/"), (b), (size)) -#else -#define new_IOBufferData new_IOBufferData_internal -#define new_xmalloc_IOBufferData new_xmalloc_IOBufferData_internal -#define new_constant_IOBufferData new_constant_IOBufferData_internal -#endif -extern int64_t iobuffer_size_to_index(int64_t size, int64_t max = max_iobuffer_size); +extern int64_t iobuffer_size_to_index(int64_t size, int64_t max); extern int64_t index_to_buffer_size(int64_t idx); /** Clone a IOBufferBlock chain. Used to snarf a IOBufferBlock chain @@ -1580,12 +1464,14 @@ IOBufferChain::const_iterator::operator!=(self_type const &that) const return _b != that._b; } -inline IOBufferChain::const_iterator::value_type &IOBufferChain::const_iterator::operator*() const +inline IOBufferChain::const_iterator::value_type & +IOBufferChain::const_iterator::operator*() const { return *_b; } -inline IOBufferChain::const_iterator::value_type *IOBufferChain::const_iterator::operator->() const +inline IOBufferChain::const_iterator::value_type * +IOBufferChain::const_iterator::operator->() const { return _b; } @@ -1605,12 +1491,14 @@ IOBufferChain::const_iterator::operator++(int) return pre; } -inline IOBufferChain::iterator::value_type &IOBufferChain::iterator::operator*() const +inline IOBufferChain::iterator::value_type & +IOBufferChain::iterator::operator*() const { return *_b; } -inline IOBufferChain::iterator::value_type *IOBufferChain::iterator::operator->() const +inline IOBufferChain::iterator::value_type * +IOBufferChain::iterator::operator->() const { return _b; } diff --git a/iocore/eventsystem/I_Lock.h b/iocore/eventsystem/I_Lock.h index b5bef2c40e0..c5b8373bdd5 100644 --- a/iocore/eventsystem/I_Lock.h +++ b/iocore/eventsystem/I_Lock.h @@ -48,13 +48,20 @@ @param _t The current EThread executing your code. */ + +// A weak version of the SCOPED_MUTEX_LOCK macro, allows the mutex to be a nullptr. #ifdef DEBUG -#define SCOPED_MUTEX_LOCK(_l, _m, _t) MutexLock _l(MakeSourceLocation(), nullptr, _m, _t) -#else -#define SCOPED_MUTEX_LOCK(_l, _m, _t) MutexLock _l(_m, _t) +#define WEAK_SCOPED_MUTEX_LOCK(_l, _m, _t) WeakMutexLock _l(MakeSourceLocation(), (char *)nullptr, _m, _t); +#else // DEBUG +#define WEAK_SCOPED_MUTEX_LOCK(_l, _m, _t) WeakMutexLock _l(_m, _t); #endif // DEBUG #ifdef DEBUG +#define SCOPED_MUTEX_LOCK(_l, _m, _t) MutexLock _l(MakeSourceLocation(), (char *)nullptr, _m, _t) +#else // DEBUG +#define SCOPED_MUTEX_LOCK(_l, _m, _t) MutexLock _l(_m, _t) +#endif // DEBUG + /** Attempts to acquire the lock to the ProxyMutex. @@ -68,8 +75,15 @@ @param _t The current EThread executing your code. */ -#define MUTEX_TRY_LOCK(_l, _m, _t) MutexTryLock _l(MakeSourceLocation(), (char *)nullptr, _m, _t) +#ifdef DEBUG +#define WEAK_MUTEX_TRY_LOCK(_l, _m, _t) WeakMutexTryLock _l(MakeSourceLocation(), (char *)nullptr, _m, _t); +#else // DEBUG +#define WEAK_MUTEX_TRY_LOCK(_l, _m, _t) WeakMutexTryLock _l(_m, _t); +#endif // DEBUG + +#ifdef DEBUG +#define MUTEX_TRY_LOCK(_l, _m, _t) MutexTryLock _l(MakeSourceLocation(), (char *)nullptr, _m, _t) #else // DEBUG #define MUTEX_TRY_LOCK(_l, _m, _t) MutexTryLock _l(_m, _t) #endif // DEBUG @@ -242,7 +256,7 @@ Mutex_trylock( #ifdef DEBUG const SourceLocation &location, const char *ahandler, #endif - Ptr &m, EThread *t) + ProxyMutex *m, EThread *t) { ink_assert(t != nullptr); ink_assert(t == reinterpret_cast(this_thread())); @@ -281,12 +295,26 @@ Mutex_trylock( return true; } +inline bool +Mutex_trylock( +#ifdef DEBUG + const SourceLocation &location, const char *ahandler, +#endif + Ptr &m, EThread *t) +{ + return Mutex_trylock( +#ifdef DEBUG + location, ahandler, +#endif + m.get(), t); +} + inline int Mutex_lock( #ifdef DEBUG const SourceLocation &location, const char *ahandler, #endif - Ptr &m, EThread *t) + ProxyMutex *m, EThread *t) { ink_assert(t != nullptr); if (m->thread_holding != t) { @@ -313,8 +341,22 @@ Mutex_lock( return true; } +inline int +Mutex_lock( +#ifdef DEBUG + const SourceLocation &location, const char *ahandler, +#endif + Ptr &m, EThread *t) +{ + return Mutex_lock( +#ifdef DEBUG + location, ahandler, +#endif + m.get(), t); +} + inline void -Mutex_unlock(Ptr &m, EThread *t) +Mutex_unlock(ProxyMutex *m, EThread *t) { if (m->nthread_holding) { ink_assert(t == m->thread_holding); @@ -337,6 +379,47 @@ Mutex_unlock(Ptr &m, EThread *t) } } +inline void +Mutex_unlock(Ptr &m, EThread *t) +{ + Mutex_unlock(m.get(), t); +} + +class WeakMutexLock +{ +private: + Ptr m; + bool locked_p; + +public: + WeakMutexLock( +#ifdef DEBUG + const SourceLocation &location, const char *ahandler, +#endif // DEBUG + Ptr &am, EThread *t) + : m(am), locked_p(true) + { + if (m.get()) { + Mutex_lock( +#ifdef DEBUG + location, ahandler, +#endif // DEBUG + m, t); + } + } + + void + release() + { + if (locked_p && m.get()) { + Mutex_unlock(m, m->thread_holding); + } + locked_p = false; + } + + ~WeakMutexLock() { this->release(); } +}; + /** Scoped lock class for ProxyMutex */ class MutexLock @@ -374,21 +457,21 @@ class MutexLock /** Scoped try lock class for ProxyMutex */ -class MutexTryLock +class WeakMutexTryLock { private: Ptr m; bool lock_acquired; public: - MutexTryLock( + WeakMutexTryLock( #ifdef DEBUG const SourceLocation &location, const char *ahandler, #endif // DEBUG Ptr &am, EThread *t) : m(am) { - if (am) { + if (m.get()) { lock_acquired = Mutex_trylock( #ifdef DEBUG location, ahandler, @@ -399,11 +482,12 @@ class MutexTryLock } } - ~MutexTryLock() + ~WeakMutexTryLock() { if (lock_acquired && m.get()) { Mutex_unlock(m, m->thread_holding); } + lock_acquired = false; } /** Spin till lock is acquired @@ -439,6 +523,67 @@ class MutexTryLock } }; +/** Scoped try lock class for ProxyMutex + */ +class MutexTryLock +{ +private: + Ptr m; + bool lock_acquired; + +public: + MutexTryLock( +#ifdef DEBUG + const SourceLocation &location, const char *ahandler, +#endif // DEBUG + Ptr &am, EThread *t) + : m(am) + { + lock_acquired = Mutex_trylock( +#ifdef DEBUG + location, ahandler, +#endif // DEBUG + m, t); + } + + ~MutexTryLock() + { + if (lock_acquired) { + Mutex_unlock(m, m->thread_holding); + } + } + + /** Spin till lock is acquired + */ + void + acquire(EThread *t) + { + MUTEX_TAKE_LOCK(m, t); + lock_acquired = true; + } + + void + release() + { + if (lock_acquired) { + Mutex_unlock(m, m->thread_holding); + } + lock_acquired = false; + } + + bool + is_locked() const + { + return lock_acquired; + } + + const ProxyMutex * + get_mutex() const + { + return m.get(); + } +}; + inline void ProxyMutex::free() { diff --git a/iocore/eventsystem/I_MIOBufferWriter.h b/iocore/eventsystem/I_MIOBufferWriter.h index 856881626a6..08e29a75524 100644 --- a/iocore/eventsystem/I_MIOBufferWriter.h +++ b/iocore/eventsystem/I_MIOBufferWriter.h @@ -27,6 +27,12 @@ #include #include "tscore/ink_assert.h" + +#if defined(UNIT_TEST_BUFFER_WRITER) +#undef ink_assert +#define ink_assert ink_release_assert +#endif + #include "tscore/BufferWriter.h" #if !defined(UNIT_TEST_BUFFER_WRITER) diff --git a/iocore/eventsystem/I_ProtectedQueue.h b/iocore/eventsystem/I_ProtectedQueue.h index 7742fac61fb..8bd0eebbaa1 100644 --- a/iocore/eventsystem/I_ProtectedQueue.h +++ b/iocore/eventsystem/I_ProtectedQueue.h @@ -37,7 +37,7 @@ #include "tscore/ink_platform.h" #include "I_Event.h" struct ProtectedQueue { - void enqueue(Event *e, bool fast_signal = false); + void enqueue(Event *e); void signal(); int try_signal(); // Use non blocking lock and if acquired, signal void enqueue_local(Event *e); // Safe when called from the same thread @@ -54,5 +54,3 @@ struct ProtectedQueue { ProtectedQueue(); }; - -void flush_signals(EThread *t); diff --git a/iocore/eventsystem/I_Thread.h b/iocore/eventsystem/I_Thread.h index 116463fbc18..2ab995a83ed 100644 --- a/iocore/eventsystem/I_Thread.h +++ b/iocore/eventsystem/I_Thread.h @@ -123,11 +123,9 @@ class Thread ProxyAllocator http2ClientSessionAllocator; ProxyAllocator http2StreamAllocator; ProxyAllocator quicClientSessionAllocator; - ProxyAllocator quicHandshakeAllocator; ProxyAllocator quicBidiStreamAllocator; ProxyAllocator quicSendStreamAllocator; ProxyAllocator quicReceiveStreamAllocator; - ProxyAllocator quicStreamManagerAllocator; ProxyAllocator httpServerSessionAllocator; ProxyAllocator hdrHeapAllocator; ProxyAllocator strHeapAllocator; diff --git a/iocore/eventsystem/I_VConnection.h b/iocore/eventsystem/I_VConnection.h index 80a7218f08e..e739fc6039d 100644 --- a/iocore/eventsystem/I_VConnection.h +++ b/iocore/eventsystem/I_VConnection.h @@ -25,16 +25,13 @@ #pragma once #include "tscore/ink_platform.h" +#include "tscore/PluginUserArgs.h" #include "I_EventSystem.h" #if !defined(I_VIO_h) #error "include I_VIO.h" #endif -#include - -static constexpr int TS_VCONN_MAX_USER_ARG = 4; - // // Data Types // @@ -378,38 +375,7 @@ class VConnection : public Continuation int lerrno; }; -/** - Subclass of VConnection to provide support for user arguments - - Inherited by DummyVConnection (down to INKContInternal) and NetVConnection -*/ -class AnnotatedVConnection : public VConnection -{ - using self_type = AnnotatedVConnection; - using super_type = VConnection; - -public: - explicit AnnotatedVConnection(ProxyMutex *aMutex) : super_type(aMutex){}; - explicit AnnotatedVConnection(Ptr &aMutex) : super_type(aMutex){}; - - void * - get_user_arg(unsigned ix) const - { - ink_assert(ix < user_args.size()); - return this->user_args[ix]; - }; - void - set_user_arg(unsigned ix, void *arg) - { - ink_assert(ix < user_args.size()); - user_args[ix] = arg; - }; - -protected: - std::array user_args{{nullptr}}; -}; - -struct DummyVConnection : public AnnotatedVConnection { +struct DummyVConnection : public VConnection, public PluginUserArgs { VIO * do_io_write(Continuation * /* c ATS_UNUSED */, int64_t /* nbytes ATS_UNUSED */, IOBufferReader * /* buf ATS_UNUSED */, bool /* owner ATS_UNUSED */) override @@ -440,5 +406,5 @@ struct DummyVConnection : public AnnotatedVConnection { "cannot use default implementation"); } - explicit DummyVConnection(ProxyMutex *m) : AnnotatedVConnection(m) {} + explicit DummyVConnection(ProxyMutex *m) : VConnection(m) {} }; diff --git a/iocore/eventsystem/I_VIO.h b/iocore/eventsystem/I_VIO.h index b80522035c8..d2b2b10568f 100644 --- a/iocore/eventsystem/I_VIO.h +++ b/iocore/eventsystem/I_VIO.h @@ -25,17 +25,12 @@ #pragma once #define I_VIO_h -#include "tscore/ink_platform.h" -#include "I_EventSystem.h" #if !defined(I_IOBuffer_h) #error "include I_IOBuffer.h" ----include I_IOBuffer.h #endif -#include "tscore/ink_apidefs.h" - class Continuation; + +class Continuation; class VConnection; -class IOVConnection; -class MIOBuffer; class ProxyMutex; /** @@ -73,9 +68,12 @@ class ProxyMutex; class VIO { public: + explicit VIO(int aop); + VIO(); ~VIO() {} + /** Interface for the VConnection that owns this handle. */ - Continuation *get_continuation(); + Continuation *get_continuation() const; void set_continuation(Continuation *cont); /** @@ -95,15 +93,15 @@ class VIO @return The number of bytes to be processed by the operation. */ - int64_t ntodo(); + int64_t ntodo() const; ///////////////////// // buffer settings // ///////////////////// void set_writer(MIOBuffer *writer); void set_reader(IOBufferReader *reader); - MIOBuffer *get_writer(); - IOBufferReader *get_reader(); + MIOBuffer *get_writer() const; + IOBufferReader *get_reader() const; /** Reenable the IO operation. @@ -140,10 +138,7 @@ class VIO inkcoreapi void reenable_re(); void disable(); - bool is_disabled(); - - explicit VIO(int aop); - VIO(); + bool is_disabled() const; enum { NONE = 0, @@ -160,7 +155,6 @@ class VIO STAT, }; -public: /** Continuation to callback. @@ -225,5 +219,3 @@ class VIO private: bool _disabled = false; }; - -#include "I_VConnection.h" diff --git a/iocore/eventsystem/MIOBufferWriter.cc b/iocore/eventsystem/MIOBufferWriter.cc new file mode 100644 index 00000000000..67f6108952e --- /dev/null +++ b/iocore/eventsystem/MIOBufferWriter.cc @@ -0,0 +1,124 @@ +/** @file + + A brief file description + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +/************************************************************************** + MIOBufferWriter.cc + +**************************************************************************/ + +#include "I_MIOBufferWriter.h" + +// +// MIOBufferWriter +// +MIOBufferWriter & +MIOBufferWriter::write(const void *data_, size_t length) +{ + const char *data = static_cast(data_); + + while (length) { + IOBufferBlock *iobbPtr = _miob->first_write_block(); + + if (!iobbPtr) { + addBlock(); + + iobbPtr = _miob->first_write_block(); + + ink_assert(iobbPtr); + } + + size_t writeSize = iobbPtr->write_avail(); + + if (length < writeSize) { + writeSize = length; + } + + std::memcpy(iobbPtr->end(), data, writeSize); + iobbPtr->fill(writeSize); + + data += writeSize; + length -= writeSize; + + _numWritten += writeSize; + } + + return *this; +} + +#if defined(UNIT_TEST_BUFFER_WRITER) + +// Dummys just for linkage (never called). + +std::ostream & +MIOBufferWriter::operator>>(std::ostream &stream) const +{ + return stream; +} + +ssize_t +MIOBufferWriter::operator>>(int fd) const +{ + return 0; +} + +#else + +std::ostream & +MIOBufferWriter::operator>>(std::ostream &stream) const +{ + IOBufferReader *r = _miob->alloc_reader(); + if (r) { + IOBufferBlock *b; + while (nullptr != (b = r->get_current_block())) { + auto n = b->read_avail(); + stream.write(b->start(), n); + r->consume(n); + } + _miob->dealloc_reader(r); + } + return stream; +} + +ssize_t +MIOBufferWriter::operator>>(int fd) const +{ + ssize_t zret = 0; + IOBufferReader *reader = _miob->alloc_reader(); + if (reader) { + IOBufferBlock *b; + while (nullptr != (b = reader->get_current_block())) { + auto n = b->read_avail(); + auto r = ::write(fd, b->start(), n); + if (r <= 0) { + break; + } else { + reader->consume(r); + zret += r; + } + } + _miob->dealloc_reader(reader); + } + return zret; +} + +#endif // defined(UNIT_TEST_BUFFER_WRITER) diff --git a/iocore/eventsystem/Makefile.am b/iocore/eventsystem/Makefile.am index 369bf55bb23..3421eb40167 100644 --- a/iocore/eventsystem/Makefile.am +++ b/iocore/eventsystem/Makefile.am @@ -47,6 +47,7 @@ libinkevent_a_SOURCES = \ I_VIO.h \ Inline.cc \ Lock.cc \ + MIOBufferWriter.cc \ PQ-List.cc \ P_EventSystem.h \ P_Freer.h \ @@ -69,7 +70,8 @@ libinkevent_a_SOURCES = \ UnixEvent.cc \ UnixEventProcessor.cc -check_PROGRAMS = test_Buffer test_Event \ +check_PROGRAMS = test_IOBuffer \ + test_EventSystem \ test_MIOBufferWriter test_LD_FLAGS = \ @@ -85,6 +87,7 @@ test_CPP_FLAGS = \ -I$(abs_top_srcdir)/proxy/logging \ -I$(abs_top_srcdir)/mgmt \ -I$(abs_top_srcdir)/mgmt/utils \ + -I$(abs_top_srcdir)/tests/include \ @OPENSSL_INCLUDES@ test_LD_ADD = \ @@ -92,31 +95,24 @@ test_LD_ADD = \ $(top_builddir)/lib/records/librecords_p.a \ $(top_builddir)/mgmt/libmgmt_p.la \ $(top_builddir)/iocore/eventsystem/libinkevent.a \ - $(top_builddir)/src/tscore/libtscore.la $(top_builddir)/src/tscpp/util/libtscpputil.la \ + $(top_builddir)/src/tscore/libtscore.la \ + $(top_builddir)/src/tscpp/util/libtscpputil.la \ $(top_builddir)/proxy/shared/libUglyLogStubs.a \ @HWLOC_LIBS@ -test_Buffer_SOURCES = \ - test_Buffer.cc - -test_Event_SOURCES = \ - test_Event.cc - -test_Buffer_CPPFLAGS = $(test_CPP_FLAGS) -test_Event_CPPFLAGS = $(test_CPP_FLAGS) - -test_Buffer_LDFLAGS = $(test_LD_FLAGS) -test_Event_LDFLAGS = $(test_LD_FLAGS) - -test_Buffer_LDADD = $(test_LD_ADD) -test_Event_LDADD = $(test_LD_ADD) +test_EventSystem_SOURCES = unit_tests/test_EventSystem.cc +test_EventSystem_CPPFLAGS = $(test_CPP_FLAGS) +test_EventSystem_LDFLAGS = $(test_LD_FLAGS) +test_EventSystem_LDADD = $(test_LD_ADD) +test_IOBuffer_SOURCES = unit_tests/test_IOBuffer.cc +test_IOBuffer_CPPFLAGS = $(test_CPP_FLAGS) +test_IOBuffer_LDFLAGS = $(test_LD_FLAGS) +test_IOBuffer_LDADD = $(test_LD_ADD) test_MIOBufferWriter_SOURCES = unit_tests/test_MIOBufferWriter.cc - -test_MIOBufferWriter_CPPFLAGS = $(test_CPP_FLAGS) -I$(abs_top_srcdir)/tests/include +test_MIOBufferWriter_CPPFLAGS = $(test_CPP_FLAGS) test_MIOBufferWriter_LDFLAGS = $(test_LD_FLAGS) -test_MIOBufferWriter_LDADD = $(test_LD_ADD) include $(top_srcdir)/build/tidy.mk diff --git a/iocore/eventsystem/P_IOBuffer.h b/iocore/eventsystem/P_IOBuffer.h index c13202d0259..41e73f079ec 100644 --- a/iocore/eventsystem/P_IOBuffer.h +++ b/iocore/eventsystem/P_IOBuffer.h @@ -36,7 +36,7 @@ // ////////////////////////////////////////////////////////////// TS_INLINE int64_t -buffer_size_to_index(int64_t size, int64_t max = max_iobuffer_size) +buffer_size_to_index(int64_t size, int64_t max) { int64_t r = max; @@ -146,7 +146,6 @@ iobufferblock_skip(IOBufferBlock *b, int64_t *poffset, int64_t *plen, int64_t wr return b; } -#ifdef TRACK_BUFFER_USER TS_INLINE void iobuffer_mem_inc(const char *_loc, int64_t _size_index) { @@ -179,7 +178,6 @@ iobuffer_mem_dec(const char *_loc, int64_t _size_index) } ResourceTracker::increment(_loc, -index_to_buffer_size(_size_index)); } -#endif ////////////////////////////////////////////////////////////////// // @@ -199,76 +197,28 @@ IOBufferData::block_size() } TS_INLINE IOBufferData * -new_IOBufferData_internal( -#ifdef TRACK_BUFFER_USER - const char *location, -#endif - void *b, int64_t size, int64_t asize_index) +new_IOBufferData_internal(const char *location, void *b, int64_t size, int64_t asize_index) { (void)size; IOBufferData *d = THREAD_ALLOC(ioDataAllocator, this_thread()); d->_size_index = asize_index; ink_assert(BUFFER_SIZE_INDEX_IS_CONSTANT(asize_index) || size <= d->block_size()); -#ifdef TRACK_BUFFER_USER d->_location = location; -#endif - d->_data = (char *)b; + d->_data = (char *)b; return d; } TS_INLINE IOBufferData * -new_constant_IOBufferData_internal( -#ifdef TRACK_BUFFER_USER - const char *loc, -#endif - void *b, int64_t size) +new_xmalloc_IOBufferData_internal(const char *location, void *b, int64_t size) { - return new_IOBufferData_internal( -#ifdef TRACK_BUFFER_USER - loc, -#endif - b, size, BUFFER_SIZE_INDEX_FOR_CONSTANT_SIZE(size)); + return new_IOBufferData_internal(location, b, size, BUFFER_SIZE_INDEX_FOR_XMALLOC_SIZE(size)); } TS_INLINE IOBufferData * -new_xmalloc_IOBufferData_internal( -#ifdef TRACK_BUFFER_USER - const char *location, -#endif - void *b, int64_t size) -{ - return new_IOBufferData_internal( -#ifdef TRACK_BUFFER_USER - location, -#endif - b, size, BUFFER_SIZE_INDEX_FOR_XMALLOC_SIZE(size)); -} - -TS_INLINE IOBufferData * -new_IOBufferData_internal( -#ifdef TRACK_BUFFER_USER - const char *location, -#endif - void *b, int64_t size) -{ - return new_IOBufferData_internal( -#ifdef TRACK_BUFFER_USER - location, -#endif - b, size, iobuffer_size_to_index(size)); -} - -TS_INLINE IOBufferData * -new_IOBufferData_internal( -#ifdef TRACK_BUFFER_USER - const char *loc, -#endif - int64_t size_index, AllocType type) +new_IOBufferData_internal(const char *loc, int64_t size_index, AllocType type) { IOBufferData *d = THREAD_ALLOC(ioDataAllocator, this_thread()); -#ifdef TRACK_BUFFER_USER - d->_location = loc; -#endif + d->_location = loc; d->alloc(size_index, type); return d; } @@ -285,9 +235,7 @@ IOBufferData::alloc(int64_t size_index, AllocType type) } _size_index = size_index; _mem_type = type; -#ifdef TRACK_BUFFER_USER iobuffer_mem_inc(_location, size_index); -#endif switch (type) { case MEMALIGNED: if (BUFFER_SIZE_INDEX_IS_FAST_ALLOCATED(size_index)) { @@ -313,9 +261,7 @@ IOBufferData::alloc(int64_t size_index, AllocType type) TS_INLINE void IOBufferData::dealloc() { -#ifdef TRACK_BUFFER_USER iobuffer_mem_dec(_location, _size_index); -#endif switch (_mem_type) { case MEMALIGNED: if (BUFFER_SIZE_INDEX_IS_FAST_ALLOCATED(_size_index)) { @@ -352,30 +298,18 @@ IOBufferData::free() // ////////////////////////////////////////////////////////////////// TS_INLINE IOBufferBlock * -new_IOBufferBlock_internal( -#ifdef TRACK_BUFFER_USER - const char *location -#endif -) +new_IOBufferBlock_internal(const char *location) { IOBufferBlock *b = THREAD_ALLOC(ioBlockAllocator, this_thread()); -#ifdef TRACK_BUFFER_USER - b->_location = location; -#endif + b->_location = location; return b; } TS_INLINE IOBufferBlock * -new_IOBufferBlock_internal( -#ifdef TRACK_BUFFER_USER - const char *location, -#endif - IOBufferData *d, int64_t len, int64_t offset) +new_IOBufferBlock_internal(const char *location, IOBufferData *d, int64_t len, int64_t offset) { IOBufferBlock *b = THREAD_ALLOC(ioBlockAllocator, this_thread()); -#ifdef TRACK_BUFFER_USER - b->_location = location; -#endif + b->_location = location; b->set(d, len, offset); return b; } @@ -411,11 +345,7 @@ TS_INLINE void IOBufferBlock::alloc(int64_t i) { ink_assert(BUFFER_SIZE_ALLOCATED(i)); -#ifdef TRACK_BUFFER_USER data = new_IOBufferData_internal(_location, i); -#else - data = new_IOBufferData_internal(i); -#endif reset(); } @@ -448,18 +378,12 @@ IOBufferBlock::clear() TS_INLINE IOBufferBlock * IOBufferBlock::clone() const { -#ifdef TRACK_BUFFER_USER IOBufferBlock *b = new_IOBufferBlock_internal(_location); -#else - IOBufferBlock *b = new_IOBufferBlock_internal(); -#endif - b->data = data; - b->_start = _start; - b->_end = _end; - b->_buf_end = _end; -#ifdef TRACK_BUFFER_USER - b->_location = _location; -#endif + b->data = data; + b->_start = _start; + b->_end = _end; + b->_buf_end = _end; + b->_location = _location; return b; } @@ -479,15 +403,9 @@ IOBufferBlock::free() TS_INLINE void IOBufferBlock::set_internal(void *b, int64_t len, int64_t asize_index) { -#ifdef TRACK_BUFFER_USER - data = new_IOBufferData_internal(_location, BUFFER_SIZE_NOT_ALLOCATED); -#else - data = new_IOBufferData_internal(BUFFER_SIZE_NOT_ALLOCATED); -#endif + data = new_IOBufferData_internal(_location, BUFFER_SIZE_NOT_ALLOCATED); data->_data = (char *)b; -#ifdef TRACK_BUFFER_USER iobuffer_mem_inc(_location, asize_index); -#endif data->_size_index = asize_index; reset(); _end = _start + len; @@ -502,46 +420,6 @@ IOBufferBlock::set(IOBufferData *d, int64_t len, int64_t offset) _buf_end = buf() + d->block_size(); } -TS_INLINE void -IOBufferBlock::realloc_set_internal(void *b, int64_t buf_size, int64_t asize_index) -{ - int64_t data_size = size(); - memcpy(b, _start, size()); - dealloc(); - set_internal(b, buf_size, asize_index); - _end = _start + data_size; -} - -TS_INLINE void -IOBufferBlock::realloc(void *b, int64_t buf_size) -{ - realloc_set_internal(b, buf_size, BUFFER_SIZE_NOT_ALLOCATED); -} - -TS_INLINE void -IOBufferBlock::realloc_xmalloc(void *b, int64_t buf_size) -{ - realloc_set_internal(b, buf_size, -buf_size); -} - -TS_INLINE void -IOBufferBlock::realloc_xmalloc(int64_t buf_size) -{ - realloc_set_internal(ats_malloc(buf_size), buf_size, -buf_size); -} - -TS_INLINE void -IOBufferBlock::realloc(int64_t i) -{ - if ((i == data->_size_index) || (i >= (int64_t)countof(ioBufAllocator))) { - return; - } - - ink_release_assert(i > data->_size_index && i != BUFFER_SIZE_NOT_ALLOCATED); - void *b = ioBufAllocator[i].alloc_void(); - realloc_set_internal(b, BUFFER_SIZE_FOR_INDEX(i), i); -} - ////////////////////////////////////////////////////////////////// // // class IOBufferReader -- @@ -693,7 +571,8 @@ IOBufferReader::consume(int64_t n) } } -TS_INLINE char &IOBufferReader::operator[](int64_t i) +TS_INLINE char & +IOBufferReader::operator[](int64_t i) { static char default_ret = '\0'; // This is just to avoid compiler warnings... IOBufferBlock *b = block.get(); @@ -754,9 +633,7 @@ inkcoreapi extern ClassAllocator ioAllocator; TS_INLINE MIOBuffer::MIOBuffer(void *b, int64_t bufsize, int64_t aWater_mark) { -#ifdef TRACK_BUFFER_USER _location = nullptr; -#endif set(b, bufsize); water_mark = aWater_mark; size_index = BUFFER_SIZE_NOT_ALLOCATED; @@ -768,9 +645,7 @@ MIOBuffer::MIOBuffer(int64_t default_size_index) { clear(); size_index = default_size_index; -#ifdef TRACK_BUFFER_USER - _location = nullptr; -#endif + _location = nullptr; return; } @@ -778,9 +653,7 @@ TS_INLINE MIOBuffer::MIOBuffer() { clear(); -#ifdef TRACK_BUFFER_USER _location = nullptr; -#endif return; } @@ -792,16 +665,10 @@ MIOBuffer::~MIOBuffer() } TS_INLINE MIOBuffer * -new_MIOBuffer_internal( -#ifdef TRACK_BUFFER_USER - const char *location, -#endif - int64_t size_index) +new_MIOBuffer_internal(const char *location, int64_t size_index) { MIOBuffer *b = THREAD_ALLOC(ioAllocator, this_thread()); -#ifdef TRACK_BUFFER_USER b->_location = location; -#endif b->alloc(size_index); b->water_mark = 0; return b; @@ -816,18 +683,12 @@ free_MIOBuffer(MIOBuffer *mio) } TS_INLINE MIOBuffer * -new_empty_MIOBuffer_internal( -#ifdef TRACK_BUFFER_USER - const char *location, -#endif - int64_t size_index) +new_empty_MIOBuffer_internal(const char *location, int64_t size_index) { MIOBuffer *b = THREAD_ALLOC(ioAllocator, this_thread()); b->size_index = size_index; b->water_mark = 0; -#ifdef TRACK_BUFFER_USER - b->_location = location; -#endif + b->_location = location; return b; } @@ -974,11 +835,7 @@ TS_INLINE void MIOBuffer::append_block(int64_t asize_index) { ink_assert(BUFFER_SIZE_ALLOCATED(asize_index)); -#ifdef TRACK_BUFFER_USER IOBufferBlock *b = new_IOBufferBlock_internal(_location); -#else - IOBufferBlock *b = new_IOBufferBlock_internal(); -#endif b->alloc(asize_index); append_block_internal(b); return; @@ -987,7 +844,9 @@ MIOBuffer::append_block(int64_t asize_index) TS_INLINE void MIOBuffer::add_block() { - append_block(size_index); + if (this->_writer == nullptr || this->_writer->next == nullptr) { + append_block(size_index); + } } TS_INLINE void @@ -1095,35 +954,15 @@ MIOBuffer::max_read_avail() TS_INLINE void MIOBuffer::set(void *b, int64_t len) { -#ifdef TRACK_BUFFER_USER _writer = new_IOBufferBlock_internal(_location); -#else - _writer = new_IOBufferBlock_internal(); -#endif _writer->set_internal(b, len, BUFFER_SIZE_INDEX_FOR_CONSTANT_SIZE(len)); init_readers(); } -TS_INLINE void -MIOBuffer::set_xmalloced(void *b, int64_t len) -{ -#ifdef TRACK_BUFFER_USER - _writer = new_IOBufferBlock_internal(_location); -#else - _writer = new_IOBufferBlock_internal(); -#endif - _writer->set_internal(b, len, BUFFER_SIZE_INDEX_FOR_XMALLOC_SIZE(len)); - init_readers(); -} - TS_INLINE void MIOBuffer::append_xmalloced(void *b, int64_t len) { -#ifdef TRACK_BUFFER_USER IOBufferBlock *x = new_IOBufferBlock_internal(_location); -#else - IOBufferBlock *x = new_IOBufferBlock_internal(); -#endif x->set_internal(b, len, BUFFER_SIZE_INDEX_FOR_XMALLOC_SIZE(len)); append_block_internal(x); } @@ -1131,11 +970,7 @@ MIOBuffer::append_xmalloced(void *b, int64_t len) TS_INLINE void MIOBuffer::append_fast_allocated(void *b, int64_t len, int64_t fast_size_index) { -#ifdef TRACK_BUFFER_USER IOBufferBlock *x = new_IOBufferBlock_internal(_location); -#else - IOBufferBlock *x = new_IOBufferBlock_internal(); -#endif x->set_internal(b, len, fast_size_index); append_block_internal(x); } @@ -1143,23 +978,12 @@ MIOBuffer::append_fast_allocated(void *b, int64_t len, int64_t fast_size_index) TS_INLINE void MIOBuffer::alloc(int64_t i) { -#ifdef TRACK_BUFFER_USER _writer = new_IOBufferBlock_internal(_location); -#else - _writer = new_IOBufferBlock_internal(); -#endif _writer->alloc(i); size_index = i; init_readers(); } -TS_INLINE void -MIOBuffer::alloc_xmalloc(int64_t buf_size) -{ - char *b = (char *)ats_malloc(buf_size); - set_xmalloced(b, buf_size); -} - TS_INLINE void MIOBuffer::dealloc_reader(IOBufferReader *e) { @@ -1193,12 +1017,6 @@ MIOBuffer::dealloc_all_readers() } } -TS_INLINE void -MIOBuffer::set_size_index(int64_t size) -{ - size_index = iobuffer_size_to_index(size); -} - TS_INLINE void MIOBufferAccessor::reader_for(MIOBuffer *abuf) { diff --git a/iocore/eventsystem/P_UnixEThread.h b/iocore/eventsystem/P_UnixEThread.h index d597d480c82..520ffdf9015 100644 --- a/iocore/eventsystem/P_UnixEThread.h +++ b/iocore/eventsystem/P_UnixEThread.h @@ -33,7 +33,7 @@ #include "I_EThread.h" #include "I_EventProcessor.h" -const int DELAY_FOR_RETRY = HRTIME_MSECONDS(10); +const ink_hrtime DELAY_FOR_RETRY = HRTIME_MSECONDS(10); TS_INLINE Event * EThread::schedule_imm(Continuation *cont, int callback_event, void *cookie) @@ -44,15 +44,6 @@ EThread::schedule_imm(Continuation *cont, int callback_event, void *cookie) return schedule(e->init(cont, 0, 0)); } -TS_INLINE Event * -EThread::schedule_imm_signal(Continuation *cont, int callback_event, void *cookie) -{ - Event *e = ::eventAllocator.alloc(); - e->callback_event = callback_event; - e->cookie = cookie; - return schedule(e->init(cont, 0, 0), true); -} - TS_INLINE Event * EThread::schedule_at(Continuation *cont, ink_hrtime t, int callback_event, void *cookie) { @@ -85,7 +76,7 @@ EThread::schedule_every(Continuation *cont, ink_hrtime t, int callback_event, vo } TS_INLINE Event * -EThread::schedule(Event *e, bool fast_signal) +EThread::schedule(Event *e) { e->ethread = this; ink_assert(tt == REGULAR); @@ -100,7 +91,13 @@ EThread::schedule(Event *e, bool fast_signal) // The continuation that gets scheduled later is not always the // client VC, it can be HttpCacheSM etc. so save the flags e->continuation->control_flags.set_flags(get_cont_flags().get_flags()); - EventQueueExternal.enqueue(e, fast_signal); + + if (e->ethread == this_ethread()) { + EventQueueExternal.enqueue_local(e); + } else { + EventQueueExternal.enqueue(e); + } + return e; } @@ -138,9 +135,9 @@ EThread::schedule_every_local(Continuation *cont, ink_hrtime t, int callback_eve e->callback_event = callback_event; e->cookie = cookie; if (t < 0) { - return schedule(e->init(cont, t, t)); + return schedule_local(e->init(cont, t, t)); } else { - return schedule(e->init(cont, get_hrtime() + t, t)); + return schedule_local(e->init(cont, get_hrtime() + t, t)); } } @@ -186,7 +183,7 @@ EThread::schedule_spawn(Continuation *c, int ev, void *cookie) TS_INLINE EThread * this_ethread() { - return dynamic_cast(this_thread()); + return static_cast(this_thread()); } TS_INLINE EThread * diff --git a/iocore/eventsystem/P_UnixEventProcessor.h b/iocore/eventsystem/P_UnixEventProcessor.h index 8ebf1f95d7d..a8ba4f475a1 100644 --- a/iocore/eventsystem/P_UnixEventProcessor.h +++ b/iocore/eventsystem/P_UnixEventProcessor.h @@ -23,6 +23,8 @@ #pragma once +#include + #include "tscore/ink_align.h" #include "I_EventProcessor.h" @@ -87,47 +89,41 @@ EventProcessor::assign_affinity_by_type(Continuation *cont, EventType etype) } TS_INLINE Event * -EventProcessor::schedule(Event *e, EventType etype, bool fast_signal) +EventProcessor::schedule(Event *e, EventType etype) { ink_assert(etype < MAX_EVENT_TYPES); - EThread *ethread = e->continuation->getThreadAffinity(); - if (ethread != nullptr && ethread->is_event_type(etype)) { - e->ethread = ethread; + if (TSSystemState::is_event_system_shut_down()) { + return nullptr; + } + + EThread *affinity_thread = e->continuation->getThreadAffinity(); + EThread *curr_thread = this_ethread(); + if (affinity_thread != nullptr && affinity_thread->is_event_type(etype)) { + e->ethread = affinity_thread; } else { - ethread = this_ethread(); // Is the current thread eligible? - if (ethread != nullptr && ethread->is_event_type(etype)) { - e->ethread = ethread; + if (curr_thread != nullptr && curr_thread->is_event_type(etype)) { + e->ethread = curr_thread; } else { e->ethread = assign_thread(etype); } - if (e->continuation->getThreadAffinity() == nullptr) { + if (affinity_thread == nullptr) { e->continuation->setThreadAffinity(e->ethread); } } if (e->continuation->mutex) { e->mutex = e->continuation->mutex; - } else { - e->mutex = e->continuation->mutex = e->ethread->mutex; } - e->ethread->EventQueueExternal.enqueue(e, fast_signal); - return e; -} -TS_INLINE Event * -EventProcessor::schedule_imm_signal(Continuation *cont, EventType et, int callback_event, void *cookie) -{ - Event *e = eventAllocator.alloc(); + if (curr_thread != nullptr && e->ethread == curr_thread) { + e->ethread->EventQueueExternal.enqueue_local(e); + } else { + e->ethread->EventQueueExternal.enqueue(e); + } - ink_assert(et < MAX_EVENT_TYPES); -#ifdef ENABLE_TIME_TRACE - e->start_time = Thread::get_hrtime(); -#endif - e->callback_event = callback_event; - e->cookie = cookie; - return schedule(e->init(cont, 0, 0), et, true); + return e; } TS_INLINE Event * diff --git a/iocore/eventsystem/P_VIO.h b/iocore/eventsystem/P_VIO.h index 4eebf0698ca..b934e910f77 100644 --- a/iocore/eventsystem/P_VIO.h +++ b/iocore/eventsystem/P_VIO.h @@ -27,44 +27,45 @@ TS_INLINE VIO::VIO(int aop) : op(aop), buffer(), mutex(nullptr) {} -///////////////////////////////////////////////////////////// -// -// VIO::VIO() -// -///////////////////////////////////////////////////////////// TS_INLINE VIO::VIO() : buffer(), mutex(nullptr) {} TS_INLINE Continuation * -VIO::get_continuation() +VIO::get_continuation() const { return cont; } + TS_INLINE void VIO::set_writer(MIOBuffer *writer) { buffer.writer_for(writer); } + TS_INLINE void VIO::set_reader(IOBufferReader *reader) { buffer.reader_for(reader); } + TS_INLINE MIOBuffer * -VIO::get_writer() +VIO::get_writer() const { return buffer.writer(); } + TS_INLINE IOBufferReader * -VIO::get_reader() +VIO::get_reader() const { return (buffer.reader()); } + TS_INLINE int64_t -VIO::ntodo() +VIO::ntodo() const { return nbytes - ndone; } + TS_INLINE void VIO::done() { @@ -75,11 +76,6 @@ VIO::done() } } -///////////////////////////////////////////////////////////// -// -// VIO::set_continuation() -// -///////////////////////////////////////////////////////////// TS_INLINE void VIO::set_continuation(Continuation *acont) { @@ -96,11 +92,6 @@ VIO::set_continuation(Continuation *acont) return; } -///////////////////////////////////////////////////////////// -// -// VIO::reenable() -// -///////////////////////////////////////////////////////////// TS_INLINE void VIO::reenable() { @@ -110,11 +101,6 @@ VIO::reenable() } } -///////////////////////////////////////////////////////////// -// -// VIO::reenable_re() -// -///////////////////////////////////////////////////////////// TS_INLINE void VIO::reenable_re() { @@ -131,7 +117,7 @@ VIO::disable() } TS_INLINE bool -VIO::is_disabled() +VIO::is_disabled() const { return this->_disabled; } diff --git a/iocore/eventsystem/ProtectedQueue.cc b/iocore/eventsystem/ProtectedQueue.cc index e8741c79a89..d7936c63ee0 100644 --- a/iocore/eventsystem/ProtectedQueue.cc +++ b/iocore/eventsystem/ProtectedQueue.cc @@ -44,7 +44,7 @@ extern ClassAllocator eventAllocator; void -ProtectedQueue::enqueue(Event *e, bool fast_signal) +ProtectedQueue::enqueue(Event *e) { ink_assert(!e->in_the_prot_queue && !e->in_the_priority_queue); EThread *e_ethread = e->ethread; @@ -61,25 +61,6 @@ ProtectedQueue::enqueue(Event *e, bool fast_signal) } } -void -flush_signals(EThread *thr) -{ - ink_assert(this_ethread() == thr); - int n = thr->n_ethreads_to_be_signalled; - if (n > eventProcessor.n_ethreads) { - n = eventProcessor.n_ethreads; // MAX - } - int i; - - for (i = 0; i < n; i++) { - if (thr->ethreads_to_be_signalled[i]) { - thr->ethreads_to_be_signalled[i]->tail_cb->signalActivity(); - thr->ethreads_to_be_signalled[i] = nullptr; - } - } - thr->n_ethreads_to_be_signalled = 0; -} - void ProtectedQueue::dequeue_timed(ink_hrtime cur_time, ink_hrtime timeout, bool sleep) { @@ -120,7 +101,7 @@ ProtectedQueue::wait(ink_hrtime timeout) * - And then the Event Thread goes to sleep and waits for the wakeup signal of `EThread::might_have_data`, * - The `EThread::lock` will be locked again when the Event Thread wakes up. */ - if (INK_ATOMICLIST_EMPTY(al)) { + if (INK_ATOMICLIST_EMPTY(al) && localQueue.empty()) { timespec ts = ink_hrtime_to_timespec(timeout); ink_cond_timedwait(&might_have_data, &lock, &ts); } diff --git a/iocore/eventsystem/UnixEThread.cc b/iocore/eventsystem/UnixEThread.cc index dc44a83f6f4..06c9943af78 100644 --- a/iocore/eventsystem/UnixEThread.cc +++ b/iocore/eventsystem/UnixEThread.cc @@ -34,6 +34,8 @@ #include #endif +#include + struct AIOCallback; #define NO_HEARTBEAT -1 @@ -56,8 +58,6 @@ EThread::EThread() EThread::EThread(ThreadType att, int anid) : id(anid), tt(att) { - ethreads_to_be_signalled = static_cast(ats_malloc(MAX_EVENT_THREADS * sizeof(EThread *))); - memset(ethreads_to_be_signalled, 0, MAX_EVENT_THREADS * sizeof(EThread *)); memset(thread_private, 0, PER_THREAD_DATA); #if HAVE_EVENTFD evfd = eventfd(0, EFD_NONBLOCK | EFD_CLOEXEC); @@ -92,15 +92,7 @@ EThread::EThread(ThreadType att, Event *e) : tt(att), start_event(e) // Provide a destructor so that SDK functions which create and destroy // threads won't have to deal with EThread memory deallocation. -EThread::~EThread() -{ - if (n_ethreads_to_be_signalled > 0) { - flush_signals(this); - } - ats_free(ethreads_to_be_signalled); - // TODO: This can't be deleted .... - // delete[]l1_hash; -} +EThread::~EThread() {} bool EThread::is_event_type(EventType et) @@ -118,7 +110,7 @@ void EThread::process_event(Event *e, int calling_code) { ink_assert((!e->in_the_prot_queue && !e->in_the_priority_queue)); - MUTEX_TRY_LOCK(lock, e->mutex, this); + WEAK_MUTEX_TRY_LOCK(lock, e->mutex, this); if (!lock.is_locked()) { e->timeout_at = cur_time + DELAY_FOR_RETRY; EventQueueExternal.enqueue_local(e); @@ -128,7 +120,6 @@ EThread::process_event(Event *e, int calling_code) return; } Continuation *c_temp = e->continuation; - // Make sure that the continuation is locked before calling the handler // Restore the client IP debugging flags set_cont_flags(e->continuation->control_flags); @@ -142,11 +133,7 @@ EThread::process_event(Event *e, int calling_code) if (e->period < 0) { e->timeout_at = e->period; } else { - this->get_hrtime_updated(); - e->timeout_at = cur_time + e->period; - if (e->timeout_at < cur_time) { - e->timeout_at = cur_time; - } + e->timeout_at = Thread::get_hrtime_updated() + e->period; } EventQueueExternal.enqueue_local(e); } @@ -238,7 +225,7 @@ EThread::execute_regular() do { done_one = false; // execute all the eligible internal events - EventQueue.check_ready(cur_time, this); + EventQueue.check_ready(loop_start_time, this); while ((e = EventQueue.dequeue_ready(cur_time))) { ink_assert(e); ink_assert(e->timeout_at > 0); @@ -264,20 +251,22 @@ EThread::execute_regular() next_time = EventQueue.earliest_timeout(); ink_hrtime sleep_time = next_time - Thread::get_hrtime_updated(); if (sleep_time > 0) { - sleep_time = std::min(sleep_time, HRTIME_MSECONDS(thread_max_heartbeat_mseconds)); + if (EventQueueExternal.localQueue.empty()) { + sleep_time = std::min(sleep_time, HRTIME_MSECONDS(thread_max_heartbeat_mseconds)); + } else { + // Because of a missed lock, Timed-Event and Negative-Event have been pushed into localQueue for retry in awhile. + // Therefore, we have to set the limitation of sleep time in order to handle the next retry in time. + sleep_time = std::min(sleep_time, DELAY_FOR_RETRY); + } ++(current_metric->_wait); } else { sleep_time = 0; } - if (n_ethreads_to_be_signalled) { - flush_signals(this); - } - tail_cb->waitForActivity(sleep_time); // loop cleanup - loop_finish_time = this->get_hrtime_updated(); + loop_finish_time = Thread::get_hrtime_updated(); delta = loop_finish_time - loop_start_time; // This can happen due to time of day adjustments (which apparently happen quite frequently). I diff --git a/iocore/eventsystem/UnixEventProcessor.cc b/iocore/eventsystem/UnixEventProcessor.cc index 2182c34841d..c3a30fe494c 100644 --- a/iocore/eventsystem/UnixEventProcessor.cc +++ b/iocore/eventsystem/UnixEventProcessor.cc @@ -193,7 +193,7 @@ ThreadAffinityInitializer::set_affinity(int, Event *) hwloc_obj_t obj = hwloc_get_obj_by_type(ink_get_topology(), obj_type, t->id % obj_count); #if HWLOC_API_VERSION >= 0x00010100 int cpu_mask_len = hwloc_bitmap_snprintf(nullptr, 0, obj->cpuset) + 1; - char *cpu_mask = (char *)alloca(cpu_mask_len); + char *cpu_mask = static_cast(alloca(cpu_mask_len)); hwloc_bitmap_snprintf(cpu_mask, cpu_mask_len, obj->cpuset); Debug("iocore_thread", "EThread: %p %s: %d CPU Mask: %s\n", t, obj_name, obj->logical_index, cpu_mask); #else @@ -230,7 +230,11 @@ ThreadAffinityInitializer::alloc_numa_stack(EThread *t, size_t stacksize) if (mem_policy != HWLOC_MEMBIND_DEFAULT) { // Let's temporarily set the memory binding to our destination NUMA node +#if HWLOC_API_VERSION >= 0x20000 + hwloc_set_membind(ink_get_topology(), nodeset, mem_policy, HWLOC_MEMBIND_THREAD | HWLOC_MEMBIND_BYNODESET); +#else hwloc_set_membind_nodeset(ink_get_topology(), nodeset, mem_policy, HWLOC_MEMBIND_THREAD); +#endif } // Alloc our stack @@ -238,8 +242,13 @@ ThreadAffinityInitializer::alloc_numa_stack(EThread *t, size_t stacksize) if (mem_policy != HWLOC_MEMBIND_DEFAULT) { // Now let's set it back to default for this thread. +#if HWLOC_API_VERSION >= 0x20000 + hwloc_set_membind(ink_get_topology(), hwloc_topology_get_topology_nodeset(ink_get_topology()), HWLOC_MEMBIND_DEFAULT, + HWLOC_MEMBIND_THREAD | HWLOC_MEMBIND_BYNODESET); +#else hwloc_set_membind_nodeset(ink_get_topology(), hwloc_topology_get_topology_nodeset(ink_get_topology()), HWLOC_MEMBIND_DEFAULT, HWLOC_MEMBIND_THREAD); +#endif } hwloc_bitmap_free(nodeset); @@ -371,6 +380,7 @@ EventProcessor::spawn_event_threads(EventType ev_type, int n_threads, size_t sta } tg->_count = n_threads; n_ethreads += n_threads; + schedule_spawn(&thread_started, ev_type); // Separate loop to avoid race conditions between spawn events and updating the thread table for // the group. Some thread set up depends on knowing the total number of threads but that can't be @@ -394,10 +404,7 @@ EventProcessor::initThreadState(EThread *t) { // Run all thread type initialization continuations that match the event types for this thread. for (int i = 0; i < MAX_EVENT_TYPES; ++i) { - if (t->is_event_type(i)) { // that event type done here, roll thread start events of that type. - if (++thread_group[i]._started == thread_group[i]._count && thread_group[i]._afterStartCallback != nullptr) { - thread_group[i]._afterStartCallback(); - } + if (t->is_event_type(i)) { // To avoid race conditions on the event in the spawn queue, create a local one to actually send. // Use the spawn queue event as a read only model. Event *nev = eventAllocator.alloc(); @@ -491,3 +498,24 @@ EventProcessor::spawn_thread(Continuation *cont, const char *thr_name, size_t st return e; } + +bool +EventProcessor::has_tg_started(int etype) +{ + return thread_group[etype]._started == thread_group[etype]._count; +} + +void +thread_started(EThread *t) +{ + // Find what type of thread this is, and increment the "_started" counter of that thread type. + for (int i = 0; i < MAX_EVENT_TYPES; ++i) { + if (t->is_event_type(i)) { + if (++eventProcessor.thread_group[i]._started == eventProcessor.thread_group[i]._count && + eventProcessor.thread_group[i]._afterStartCallback != nullptr) { + eventProcessor.thread_group[i]._afterStartCallback(); + } + break; + } + } +} diff --git a/iocore/eventsystem/test_Buffer.cc b/iocore/eventsystem/test_Buffer.cc deleted file mode 100644 index ccef1036864..00000000000 --- a/iocore/eventsystem/test_Buffer.cc +++ /dev/null @@ -1,64 +0,0 @@ -/** @file - - A brief file description - - @section license License - - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - */ - -#include "I_EventSystem.h" -#include "tscore/I_Layout.h" -#include "tscore/ink_string.h" - -#include "diags.i" - -#define TEST_TIME_SECOND 60 -#define TEST_THREADS 2 - -int -main(int /* argc ATS_UNUSED */, const char * /* argv ATS_UNUSED */ []) -{ - RecModeT mode_type = RECM_STAND_ALONE; - - Layout::create(); - init_diags("", nullptr); - RecProcessInit(mode_type); - - ink_event_system_init(EVENT_SYSTEM_MODULE_PUBLIC_VERSION); - eventProcessor.start(TEST_THREADS); - - Thread *main_thread = new EThread; - main_thread->set_specific(); - - for (unsigned i = 0; i < 100; ++i) { - MIOBuffer *b1 = new_MIOBuffer(default_large_iobuffer_size); - IOBufferReader *b1reader ATS_UNUSED = b1->alloc_reader(); - b1->fill(b1->write_avail()); - - MIOBuffer *b2 = new_MIOBuffer(default_large_iobuffer_size); - IOBufferReader *b2reader ATS_UNUSED = b2->alloc_reader(); - b2->fill(b2->write_avail()); - - // b1->write(b2reader, 2*1024); - - free_MIOBuffer(b2); - free_MIOBuffer(b1); - } - - exit(0); -} diff --git a/iocore/eventsystem/test_Event.cc b/iocore/eventsystem/test_Event.cc deleted file mode 100644 index 78c47bb3624..00000000000 --- a/iocore/eventsystem/test_Event.cc +++ /dev/null @@ -1,83 +0,0 @@ -/** @file - - A brief file description - - @section license License - - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - */ - -#include "I_EventSystem.h" -#include "tscore/I_Layout.h" -#include "tscore/TSSystemState.h" - -#include "diags.i" - -#define TEST_TIME_SECOND 60 -#define TEST_THREADS 2 - -static int count; - -struct alarm_printer : public Continuation { - alarm_printer(ProxyMutex *m) : Continuation(m) { SET_HANDLER(&alarm_printer::dummy_function); } - int - dummy_function(int /* event ATS_UNUSED */, Event * /* e ATS_UNUSED */) - { - ink_atomic_increment((int *)&count, 1); - printf("Count = %d\n", count); - return 0; - } -}; -struct process_killer : public Continuation { - process_killer(ProxyMutex *m) : Continuation(m) { SET_HANDLER(&process_killer::kill_function); } - int - kill_function(int /* event ATS_UNUSED */, Event * /* e ATS_UNUSED */) - { - printf("Count is %d \n", count); - if (count <= 0) { - exit(1); - } - if (count > TEST_TIME_SECOND * TEST_THREADS) { - exit(1); - } - exit(0); - return 0; - } -}; - -int -main(int /* argc ATS_UNUSED */, const char * /* argv ATS_UNUSED */ []) -{ - RecModeT mode_type = RECM_STAND_ALONE; - count = 0; - - Layout::create(); - init_diags("", nullptr); - RecProcessInit(mode_type); - - ink_event_system_init(EVENT_SYSTEM_MODULE_PUBLIC_VERSION); - eventProcessor.start(TEST_THREADS, 1048576); // Hardcoded stacksize at 1MB - - alarm_printer *alrm = new alarm_printer(new_ProxyMutex()); - process_killer *killer = new process_killer(new_ProxyMutex()); - eventProcessor.schedule_in(killer, HRTIME_SECONDS(10)); - eventProcessor.schedule_every(alrm, HRTIME_SECONDS(1)); - while (!TSSystemState::is_event_system_shut_down()) { - sleep(1); - } - return 0; -} diff --git a/iocore/eventsystem/unit_tests/test_EventSystem.cc b/iocore/eventsystem/unit_tests/test_EventSystem.cc new file mode 100644 index 00000000000..75faad2718d --- /dev/null +++ b/iocore/eventsystem/unit_tests/test_EventSystem.cc @@ -0,0 +1,101 @@ +/** @file + + Catch based unit tests for EventSystem + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#define CATCH_CONFIG_MAIN +#include "catch.hpp" + +#include "I_EventSystem.h" +#include "tscore/I_Layout.h" +#include "tscore/TSSystemState.h" + +#include "diags.i" + +#define TEST_TIME_SECOND 60 +#define TEST_THREADS 2 + +TEST_CASE("EventSystem", "[iocore]") +{ + static int count; + + struct alarm_printer : public Continuation { + alarm_printer(ProxyMutex *m) : Continuation(m) { SET_HANDLER(&alarm_printer::dummy_function); } + + int + dummy_function(int /* event ATS_UNUSED */, Event * /* e ATS_UNUSED */) + { + ink_atomic_increment((int *)&count, 1); + + EThread *e = this_ethread(); + printf("thread=%d (%p) count = %d\n", e->id, e, count); + + return 0; + } + }; + + struct process_killer : public Continuation { + process_killer(ProxyMutex *m) : Continuation(m) { SET_HANDLER(&process_killer::kill_function); } + + int + kill_function(int /* event ATS_UNUSED */, Event * /* e ATS_UNUSED */) + { + EThread *e = this_ethread(); + printf("thread=%d (%p) count is %d\n", e->id, e, count); + + REQUIRE(count > 0); + REQUIRE(count <= TEST_TIME_SECOND * TEST_THREADS); + + TSSystemState::shut_down_event_system(); + + return 0; + } + }; + + alarm_printer *alrm = new alarm_printer(new_ProxyMutex()); + process_killer *killer = new process_killer(new_ProxyMutex()); + eventProcessor.schedule_in(killer, HRTIME_SECONDS(10)); + eventProcessor.schedule_every(alrm, HRTIME_SECONDS(1)); + + while (!TSSystemState::is_event_system_shut_down()) { + sleep(1); + } +} + +struct EventProcessorListener : Catch::TestEventListenerBase { + using TestEventListenerBase::TestEventListenerBase; + + void + testRunStarting(Catch::TestRunInfo const &testRunInfo) override + { + Layout::create(); + init_diags("", nullptr); + RecProcessInit(RECM_STAND_ALONE); + + ink_event_system_init(EVENT_SYSTEM_MODULE_PUBLIC_VERSION); + eventProcessor.start(TEST_THREADS, 1048576); // Hardcoded stacksize at 1MB + + EThread *main_thread = new EThread; + main_thread->set_specific(); + } +}; + +CATCH_REGISTER_LISTENER(EventProcessorListener); diff --git a/iocore/eventsystem/unit_tests/test_IOBuffer.cc b/iocore/eventsystem/unit_tests/test_IOBuffer.cc new file mode 100644 index 00000000000..1c2c407212c --- /dev/null +++ b/iocore/eventsystem/unit_tests/test_IOBuffer.cc @@ -0,0 +1,353 @@ +/** @file + + Catch based unit tests for IOBuffer + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#define CATCH_CONFIG_MAIN +#include "catch.hpp" + +#include "tscore/I_Layout.h" + +#include "I_EventSystem.h" +#include "RecordsConfig.h" + +#include "diags.i" + +#define TEST_THREADS 1 + +TEST_CASE("MIOBuffer", "[iocore]") +{ + SECTION("new_MIOBuffer 100 times") + { + int64_t read_avail_len1 = 0; + int64_t read_avail_len2 = 0; + + for (unsigned i = 0; i < 100; ++i) { + MIOBuffer *b1 = new_MIOBuffer(BUFFER_SIZE_INDEX_512); + int64_t len1 = b1->write_avail(); + IOBufferReader *b1reader = b1->alloc_reader(); + b1->fill(len1); + read_avail_len1 += b1reader->read_avail(); + + MIOBuffer *b2 = new_MIOBuffer(BUFFER_SIZE_INDEX_4K); + int64_t len2 = b2->write_avail(); + IOBufferReader *b2reader = b2->alloc_reader(); + b2->fill(len2); + read_avail_len2 += b2reader->read_avail(); + + free_MIOBuffer(b2); + free_MIOBuffer(b1); + } + + CHECK(read_avail_len1 == 100 * BUFFER_SIZE_FOR_INDEX(BUFFER_SIZE_INDEX_512)); + CHECK(read_avail_len2 == 100 * BUFFER_SIZE_FOR_INDEX(BUFFER_SIZE_INDEX_4K)); + } + + SECTION("write") + { + MIOBuffer *miob = new_MIOBuffer(BUFFER_SIZE_INDEX_4K); + IOBufferReader *miob_r = miob->alloc_reader(); + const IOBufferBlock *block = miob->first_write_block(); + + SECTION("initial state") + { + CHECK(miob->size_index == BUFFER_SIZE_INDEX_4K); + CHECK(miob->water_mark == 0); + CHECK(miob->first_write_block() != nullptr); + CHECK(miob->block_size() == 4096); + CHECK(miob->block_write_avail() == 4096); + CHECK(miob->current_write_avail() == 4096); + CHECK(miob->write_avail() == 4096); + + CHECK(miob->max_read_avail() == 0); + CHECK(miob_r->read_avail() == 0); + } + + SECTION("write(const void *rbuf, int64_t nbytes)") + { + SECTION("1K") + { + uint8_t buf[1024]; + memset(buf, 0xAA, sizeof(buf)); + + int64_t written = miob->write(buf, sizeof(buf)); + + REQUIRE(written == sizeof(buf)); + + CHECK(miob->block_size() == 4096); + CHECK(miob->block_write_avail() == 3072); + CHECK(miob->current_write_avail() == 3072); + CHECK(miob->write_avail() == 3072); + + CHECK(miob->first_write_block() == block); + + CHECK(miob->max_read_avail() == sizeof(buf)); + CHECK(miob_r->read_avail() == sizeof(buf)); + } + + SECTION("4K") + { + uint8_t buf[4096]; + memset(buf, 0xAA, sizeof(buf)); + + int64_t written = miob->write(buf, sizeof(buf)); + + REQUIRE(written == sizeof(buf)); + + CHECK(miob->block_size() == 4096); + CHECK(miob->block_write_avail() == 0); + CHECK(miob->current_write_avail() == 0); + CHECK(miob->write_avail() == 0); + + CHECK(miob->first_write_block() == block); + + CHECK(miob->max_read_avail() == sizeof(buf)); + CHECK(miob_r->read_avail() == sizeof(buf)); + } + + SECTION("5K") + { + uint8_t buf[5120]; + memset(buf, 0xAA, sizeof(buf)); + + int64_t written = miob->write(buf, sizeof(buf)); + + REQUIRE(written == sizeof(buf)); + + CHECK(miob->block_size() == 4096); + CHECK(miob->block_write_avail() == 3072); + CHECK(miob->current_write_avail() == 3072); + CHECK(miob->write_avail() == 3072); + + CHECK(miob->first_write_block() != block); + + CHECK(miob->max_read_avail() == sizeof(buf)); + CHECK(miob_r->read_avail() == sizeof(buf)); + } + + SECTION("8K") + { + uint8_t buf[8192]; + memset(buf, 0xAA, sizeof(buf)); + + int64_t written = miob->write(buf, sizeof(buf)); + + REQUIRE(written == sizeof(buf)); + + CHECK(miob->block_size() == 4096); + CHECK(miob->block_write_avail() == 0); + CHECK(miob->current_write_avail() == 0); + CHECK(miob->write_avail() == 0); + + CHECK(miob->first_write_block() != block); + + CHECK(miob->max_read_avail() == sizeof(buf)); + CHECK(miob_r->read_avail() == sizeof(buf)); + } + } + + free_MIOBuffer(miob); + } + + SECTION("write_avail") + { + MIOBuffer *miob = new_MIOBuffer(BUFFER_SIZE_INDEX_4K); + IOBufferReader *miob_r = miob->alloc_reader(); + uint8_t buf[8192]; + memset(buf, 0xAA, sizeof(buf)); + + // initial state + CHECK(miob->block_size() == 4096); + CHECK(miob->current_write_avail() == 4096); + CHECK(miob->write_avail() == 4096); + + SECTION("water_mark == 0 (default)") + { + REQUIRE(miob->water_mark == 0); + + // fill half of the current buffer + miob->write(buf, 2048); + CHECK(miob->max_read_avail() == 2048); + CHECK(miob->current_write_avail() == 2048); + CHECK(miob->high_water() == true); + CHECK(miob->current_low_water() == false); + CHECK(miob->write_avail() == 2048); ///< should have no side effect + + // fill all of the current buffer + miob->write(buf, 2048); + CHECK(miob->max_read_avail() == 4096); + CHECK(miob->current_write_avail() == 0); + CHECK(miob->high_water() == true); + CHECK(miob->current_low_water() == true); + CHECK(miob->write_avail() == 0); ///< should have no side effect + + // consume half of the data + miob_r->consume(2048); + CHECK(miob->max_read_avail() == 2048); + CHECK(miob->current_write_avail() == 0); + CHECK(miob->high_water() == true); + CHECK(miob->current_low_water() == true); + CHECK(miob->write_avail() == 0); ///< should have no side effect + + // consume all of the data + miob_r->consume(2048); + CHECK(miob->max_read_avail() == 0); + CHECK(miob->current_write_avail() == 0); + CHECK(miob->high_water() == false); + CHECK(miob->current_low_water() == true); + CHECK(miob->write_avail() == 4096); ///< should have a side effect: add a new block + + CHECK(miob->max_read_avail() == 0); + CHECK(miob->current_write_avail() == 4096); + CHECK(miob->high_water() == false); + CHECK(miob->current_low_water() == false); + CHECK(miob->write_avail() == 4096); ///< should have no side effect + } + + SECTION("water_mark == half of block size") + { + miob->water_mark = 2048; + REQUIRE(miob->water_mark * 2 == miob->block_size()); + + // fill half of the current buffer + miob->write(buf, 2048); + CHECK(miob->max_read_avail() == 2048); + CHECK(miob->current_write_avail() == 2048); + CHECK(miob->high_water() == false); + CHECK(miob->current_low_water() == true); + CHECK(miob->write_avail() == 6144); ///< should have a side effect: add a new block + + CHECK(miob->max_read_avail() == 2048); + CHECK(miob->current_write_avail() == 6144); + CHECK(miob->high_water() == false); + CHECK(miob->current_low_water() == false); + CHECK(miob->write_avail() == 6144); ///< should have no side effect + + // fill all of the current buffer + miob->write(buf, 6144); + CHECK(miob->max_read_avail() == 8192); + CHECK(miob->current_write_avail() == 0); + CHECK(miob->high_water() == true); + CHECK(miob->current_low_water() == true); + CHECK(miob->write_avail() == 0); ///< should have no side effect + + // consume half of the data + miob_r->consume(4096); + CHECK(miob->max_read_avail() == 4096); + CHECK(miob->current_write_avail() == 0); + CHECK(miob->high_water() == true); + CHECK(miob->current_low_water() == true); + CHECK(miob->write_avail() == 0); ///< should have no side effect + + // consume all of the data + miob_r->consume(4096); + CHECK(miob->max_read_avail() == 0); + CHECK(miob->current_write_avail() == 0); + CHECK(miob->high_water() == false); + CHECK(miob->current_low_water() == true); + CHECK(miob->write_avail() == 4096); ///< should have a side effect: add a new block + + CHECK(miob->max_read_avail() == 0); + CHECK(miob->current_write_avail() == 4096); + CHECK(miob->high_water() == false); + CHECK(miob->current_low_water() == false); + CHECK(miob->write_avail() == 4096); ///< should have no side effect + } + + SECTION("water_mark == block_size()") + { + miob->water_mark = 4096; + REQUIRE(miob->water_mark == miob->block_size()); + + // fill half of the current buffer + miob->write(buf, 2048); + CHECK(miob->max_read_avail() == 2048); + CHECK(miob->current_write_avail() == 2048); + CHECK(miob->high_water() == false); + CHECK(miob->current_low_water() == true); + CHECK(miob->write_avail() == 6144); ///< should have a side effect: add a new block + + CHECK(miob->max_read_avail() == 2048); + CHECK(miob->current_write_avail() == 6144); + CHECK(miob->high_water() == false); + CHECK(miob->current_low_water() == false); + CHECK(miob->write_avail() == 6144); ///< should have no side effect + + // fill all of the current buffer + miob->write(buf, 6144); + CHECK(miob->max_read_avail() == 8192); + CHECK(miob->current_write_avail() == 0); + CHECK(miob->high_water() == true); + CHECK(miob->current_low_water() == true); + CHECK(miob->write_avail() == 0); ///< should have no side effect + + // consume half of the data + miob_r->consume(4096); + CHECK(miob->max_read_avail() == 4096); + CHECK(miob->current_write_avail() == 0); + CHECK(miob->high_water() == false); + CHECK(miob->current_low_water() == true); + CHECK(miob->write_avail() == 4096); ///< should have a side effect: add a new block + IOBufferBlock *tail = miob->_writer->next.get(); + CHECK(tail != nullptr); + + CHECK(miob->max_read_avail() == 4096); + CHECK(miob->current_write_avail() == 4096); + CHECK(miob->high_water() == false); + CHECK(miob->current_low_water() == true); + CHECK(miob->write_avail() == 4096); ///< should have no side effect + CHECK(tail == miob->_writer->next.get()); ///< the tail block should not be changed + + // consume all of the data + miob_r->consume(4096); + CHECK(miob->max_read_avail() == 0); + CHECK(miob->current_write_avail() == 4096); + CHECK(miob->high_water() == false); + CHECK(miob->current_low_water() == true); + CHECK(miob->write_avail() == 4096); ///< should have no side effect + CHECK(tail == miob->_writer->next.get()); ///< the tail block should not be changed + } + + free_MIOBuffer(miob); + } +} + +struct EventProcessorListener : Catch::TestEventListenerBase { + using TestEventListenerBase::TestEventListenerBase; + + void + testRunStarting(Catch::TestRunInfo const &testRunInfo) override + { + Layout::create(); + init_diags("", nullptr); + RecProcessInit(RECM_STAND_ALONE); + + LibRecordsConfigInit(); + + ink_event_system_init(EVENT_SYSTEM_MODULE_PUBLIC_VERSION); + eventProcessor.start(TEST_THREADS); + + EThread *main_thread = new EThread; + main_thread->set_specific(); + } +}; + +CATCH_REGISTER_LISTENER(EventProcessorListener); diff --git a/iocore/eventsystem/unit_tests/test_MIOBufferWriter.cc b/iocore/eventsystem/unit_tests/test_MIOBufferWriter.cc index c75d59e478a..34799466641 100644 --- a/iocore/eventsystem/unit_tests/test_MIOBufferWriter.cc +++ b/iocore/eventsystem/unit_tests/test_MIOBufferWriter.cc @@ -21,38 +21,94 @@ limitations under the License. */ -#define CATCH_CONFIG_RUNNER +#define CATCH_CONFIG_MAIN #include "catch.hpp" #include -#include -#include "I_EventSystem.h" -#include "tscore/I_Layout.h" +struct IOBufferBlock { + std::int64_t write_avail(); -#include "diags.i" + char *end(); + + void fill(int64_t); +}; + +struct MIOBuffer { + IOBufferBlock *first_write_block(); + + void add_block(); +}; + +#define UNIT_TEST_BUFFER_WRITER #include "I_MIOBufferWriter.h" +#include "MIOBufferWriter.cc" + +IOBufferBlock iobb[1]; +int iobbIdx{0}; + +const int BlockSize = 11 * 11; +char block[BlockSize]; +int blockUsed{0}; + +std::int64_t +IOBufferBlock::write_avail() +{ + REQUIRE(this == (iobb + iobbIdx)); + return BlockSize - blockUsed; +} + +char * +IOBufferBlock::end() +{ + REQUIRE(this == (iobb + iobbIdx)); + return block + blockUsed; +} -int -main(int argc, char *argv[]) +void +IOBufferBlock::fill(int64_t len) { - // global setup... - Layout::create(); - init_diags("", nullptr); - RecProcessInit(RECM_STAND_ALONE); + static std::uint8_t dataCheck; + + REQUIRE(this == (iobb + iobbIdx)); + + while (len-- and (blockUsed < BlockSize)) { + REQUIRE(block[blockUsed] == static_cast(dataCheck)); + + ++blockUsed; + + dataCheck += 7; + } + + REQUIRE(len == -1); +} - ink_event_system_init(EVENT_SYSTEM_MODULE_PUBLIC_VERSION); - eventProcessor.start(2); +MIOBuffer theMIOBuffer; - Thread *main_thread = new EThread; - main_thread->set_specific(); +IOBufferBlock * +MIOBuffer::first_write_block() +{ + REQUIRE(this == &theMIOBuffer); - std::cout << "Pre-Catch" << std::endl; - int result = Catch::Session().run(argc, argv); + REQUIRE(blockUsed <= BlockSize); - // global clean-up... + if (blockUsed == BlockSize) { + return nullptr; + } - exit(result); + return iobb + iobbIdx; +} + +void +MIOBuffer::add_block() +{ + REQUIRE(this == &theMIOBuffer); + + REQUIRE(blockUsed == BlockSize); + + blockUsed = 0; + + ++iobbIdx; } std::string @@ -70,12 +126,75 @@ genData(int numBytes) return s; } +void +writeOnce(MIOBufferWriter &bw, std::size_t len) +{ + static bool toggle; + + std::string s{genData(len)}; + + if (len == 1) { + bw.write(s[0]); + + } else if (toggle) { + std::size_t cap{bw.auxBufferCapacity()}; + + if (cap >= len) { + memcpy(bw.auxBuffer(), s.data(), len); + bw.fill(len); + + } else { + memcpy(bw.auxBuffer(), s.data(), cap); + bw.fill(cap); + bw.write(s.data() + cap, len - cap); + } + } else { + bw.write(s.data(), len); + } + + toggle = !toggle; + + REQUIRE(bw.auxBufferCapacity() <= BlockSize); +} + class InkAssertExcept { }; TEST_CASE("MIOBufferWriter", "[MIOBW]") { - MIOBuffer *theMIOBuffer = new_MIOBuffer(default_large_iobuffer_size); - MIOBufferWriter bw(theMIOBuffer); + MIOBufferWriter bw(&theMIOBuffer); + + REQUIRE(bw.auxBufferCapacity() == BlockSize); + + writeOnce(bw, 0); + writeOnce(bw, 1); + writeOnce(bw, 1); + writeOnce(bw, 1); + writeOnce(bw, 10); + writeOnce(bw, 1000); + writeOnce(bw, 1); + writeOnce(bw, 0); + writeOnce(bw, 1); + writeOnce(bw, 2000); + writeOnce(bw, 69); + writeOnce(bw, 666); + + for (int i = 0; i < 3000; i += 13) { + writeOnce(bw, i); + } + + writeOnce(bw, 0); + writeOnce(bw, 1); + + REQUIRE(bw.extent() == ((iobbIdx * BlockSize) + blockUsed)); + + REQUIRE_THROWS_AS(bw.fill(bw.auxBufferCapacity() + 1), InkAssertExcept); + REQUIRE_THROWS_AS(bw.data(), InkAssertExcept); +} + +void +_ink_assert(const char *a, const char *f, int l) +{ + throw InkAssertExcept(); } diff --git a/iocore/hostdb/HostDB.cc b/iocore/hostdb/HostDB.cc index 27a5b1c98fd..5101dc8c91f 100644 --- a/iocore/hostdb/HostDB.cc +++ b/iocore/hostdb/HostDB.cc @@ -43,7 +43,6 @@ HostDBContinuation::Options const HostDBContinuation::DEFAULT_OPTIONS; int hostdb_enable = true; int hostdb_migrate_on_demand = true; int hostdb_lookup_timeout = 30; -int hostdb_insert_timeout = 160; int hostdb_re_dns_on_reload = false; int hostdb_ttl_mode = TTL_OBEY; unsigned int hostdb_round_robin_max_count = 16; @@ -63,6 +62,7 @@ int hostdb_max_count = DEFAULT_HOST_DB_SIZE; char hostdb_hostfile_path[PATH_NAME_MAX] = ""; int hostdb_sync_frequency = 0; int hostdb_disable_reverse_lookup = 0; +int hostdb_max_iobuf_index = BUFFER_SIZE_INDEX_32K; ClassAllocator hostDBContAllocator("hostDBContAllocator"); @@ -326,6 +326,8 @@ HostDBCache::start(int flags) // how often to sync hostdb to disk REC_EstablishStaticConfigInt32(hostdb_sync_frequency, "proxy.config.cache.hostdb.sync_frequency"); + REC_EstablishStaticConfigInt32(hostdb_max_iobuf_index, "proxy.config.hostdb.io.max_buffer_index"); + if (hostdb_max_size == 0) { Fatal("proxy.config.hostdb.max_size must be a non-zero number"); } @@ -994,7 +996,7 @@ HostDBContinuation::removeEvent(int /* event ATS_UNUSED */, Event *e) if (cont) { proxy_mutex = cont->mutex; } - MUTEX_TRY_LOCK(lock, proxy_mutex, e->ethread); + WEAK_MUTEX_TRY_LOCK(lock, proxy_mutex, e->ethread); if (!lock.is_locked()) { e->schedule_in(HOST_DB_RETRY_PERIOD); return EVENT_CONT; @@ -1175,10 +1177,12 @@ HostDBContinuation::dnsEvent(int event, HostEnt *e) timeout = nullptr; } EThread *thread = mutex->thread_holding; - if (event == EVENT_INTERVAL) { + if (event != DNS_EVENT_LOOKUP) { + // This was an event_interval or an event_immediate + // Either we timed out, or remove_trigger_pending gave up on us if (!action.continuation) { // give up on insert, it has been too long - remove_trigger_pending_dns(); + hostDB.pending_dns_for_hash(hash.hash).remove(this); hostdb_cont_free(this); return EVENT_DONE; } @@ -1195,8 +1199,6 @@ HostDBContinuation::dnsEvent(int event, HostEnt *e) action.continuation->handleEvent(EVENT_HOST_DB_LOOKUP, nullptr); } action = nullptr; - // do not exit yet, wait to see if we can insert into DB - timeout = thread->schedule_in(this, HRTIME_SECONDS(hostdb_insert_timeout)); return EVENT_DONE; } else { bool failed = !e || !e->good; @@ -1288,9 +1290,11 @@ HostDBContinuation::dnsEvent(int event, HostEnt *e) // If the DNS lookup failed (errors such as SERVFAIL, etc.) but we have an old record // which is okay with being served stale-- lets continue to serve the stale record as long as // the record is willing to be served. + bool serve_stale = false; if (failed && old_r && old_r->serve_stale_but_revalidate()) { r->free(); - r = old_r.get(); + r = old_r.get(); + serve_stale = true; } else if (is_byname()) { if (first_record) { ip_addr_set(tip, af, first_record); @@ -1400,7 +1404,11 @@ HostDBContinuation::dnsEvent(int event, HostEnt *e) ink_assert(!r || !r->round_robin || !r->reverse_dns); ink_assert(failed || !r->round_robin || r->app.rr.offset); - hostDB.refcountcache->put(hash.hash.fold(), r, allocSize, r->expiry_time()); + if (!serve_stale) { + hostDB.refcountcache->put(hash.hash.fold(), r, allocSize, r->expiry_time()); + } else { + Warning("Fallback to serving stale record, skip re-update of hostdb for %s", aname); + } // try to callback the user // @@ -1413,39 +1421,40 @@ HostDBContinuation::dnsEvent(int event, HostEnt *e) return EVENT_CONT; } - // We have seen cases were the action.mutex != action.continuation.mutex. + // We have seen cases were the action.mutex != action.continuation.mutex. However, it seems that case + // is likely a memory corruption... Thus the introduction of the assert. // Since reply_to_cont will call the handler on the action.continuation, it is important that we hold // that mutex. bool need_to_reschedule = true; MUTEX_TRY_LOCK(lock, action.mutex, thread); if (lock.is_locked()) { - need_to_reschedule = !action.cancelled; if (!action.cancelled) { if (action.continuation->mutex) { - MUTEX_TRY_LOCK(lock2, action.continuation->mutex, thread); - if (lock2.is_locked()) { - reply_to_cont(action.continuation, r, is_srv()); - need_to_reschedule = false; - } - } else { - reply_to_cont(action.continuation, r, is_srv()); - need_to_reschedule = false; + ink_release_assert(action.continuation->mutex == action.mutex); } + reply_to_cont(action.continuation, r, is_srv()); } + need_to_reschedule = false; } + if (need_to_reschedule) { - remove_trigger_pending_dns(); SET_HANDLER((HostDBContHandler)&HostDBContinuation::probeEvent); - thread->schedule_in(this, HOST_DB_RETRY_PERIOD); + // Will reschedule on affinity thread or current thread + timeout = eventProcessor.schedule_in(this, HOST_DB_RETRY_PERIOD); return EVENT_CONT; } } + + // Clean ourselves up + hostDB.pending_dns_for_hash(hash.hash).remove(this); + // wake up everyone else who is waiting remove_trigger_pending_dns(); - // all done - // hostdb_cont_free(this); + + // all done, or at least scheduled to be all done + // return EVENT_DONE; } } @@ -1516,12 +1525,17 @@ HostDBContinuation::probeEvent(int /* event ATS_UNUSED */, Event *e) ink_assert(!link.prev && !link.next); EThread *t = e ? e->ethread : this_ethread(); + if (timeout) { + timeout->cancel(this); + timeout = nullptr; + } + MUTEX_TRY_LOCK(lock, action.mutex, t); // Separating lock checks here to make sure things don't break // when we check if the action is cancelled. if (!lock.is_locked()) { - mutex->thread_holding->schedule_in(this, HOST_DB_RETRY_PERIOD); + timeout = mutex->thread_holding->schedule_in(this, HOST_DB_RETRY_PERIOD); return EVENT_CONT; } @@ -1530,13 +1544,8 @@ HostDBContinuation::probeEvent(int /* event ATS_UNUSED */, Event *e) return EVENT_DONE; } - // Go ahead and grab the continuation mutex or just grab the action mutex again of there is no continuation mutex - MUTEX_TRY_LOCK(lock2, (action.continuation && action.continuation->mutex) ? action.continuation->mutex : action.mutex, t); - // Don't continue unless we have both mutexes - if (!lock2.is_locked()) { - mutex->thread_holding->schedule_in(this, HOST_DB_RETRY_PERIOD); - return EVENT_CONT; - } + // If the action.continuation->mutex != action.mutex, we have a use after free/realloc + ink_release_assert(!action.continuation || action.continuation->mutex == action.mutex); if (!hostdb_enable || (!*hash.host_name && !hash.ip.isValid())) { if (action.continuation) { @@ -1556,7 +1565,7 @@ HostDBContinuation::probeEvent(int /* event ATS_UNUSED */, Event *e) } if (action.continuation && r) { - reply_to_cont(action.continuation, r.get()); + reply_to_cont(action.continuation, r.get(), is_srv()); } // If it succeeds or it was a remote probe, we are done @@ -1577,6 +1586,11 @@ HostDBContinuation::set_check_pending_dns() { Queue &q = hostDB.pending_dns_for_hash(hash.hash); this->setThreadAffinity(this_ethread()); + if (q.in(this)) { + HOSTDB_INCREMENT_DYN_STAT(hostdb_insert_duplicate_to_pending_dns_stat); + Debug("hostdb", "Skip the insertion of the same continuation to pending dns"); + return false; + } HostDBContinuation *c = q.head; for (; c; c = static_cast(c->link.next)) { if (hash.hash == c->hash.hash) { @@ -1612,7 +1626,10 @@ HostDBContinuation::remove_trigger_pending_dns() if (!affinity_thread || affinity_thread == thread) { c->handleEvent(EVENT_IMMEDIATE, nullptr); } else { - eventProcessor.schedule_imm(c); + if (c->timeout) { + c->timeout->cancel(); + } + c->timeout = eventProcessor.schedule_imm(c); } } } @@ -2144,6 +2161,9 @@ ink_hostdb_init(ts::ModuleVersion v) RecRegisterRawStat(hostdb_rsb, RECT_PROCESS, "proxy.process.hostdb.re_dns_on_reload", RECD_INT, RECP_PERSISTENT, (int)hostdb_re_dns_on_reload_stat, RecRawStatSyncSum); + RecRegisterRawStat(hostdb_rsb, RECT_PROCESS, "proxy.process.hostdb.insert_duplicate_to_pending_dns", RECD_INT, RECP_PERSISTENT, + (int)hostdb_insert_duplicate_to_pending_dns_stat, RecRawStatSyncSum); + ts_host_res_global_init(); } @@ -2209,7 +2229,7 @@ ParseHostLine(Ptr &map, char *l) } void -ParseHostFile(const char *path, unsigned int hostdb_hostfile_check_interval) +ParseHostFile(const char *path, unsigned int hostdb_hostfile_check_interval_parse) { Ptr parsed_hosts_file_ptr; @@ -2229,7 +2249,7 @@ ParseHostFile(const char *path, unsigned int hostdb_hostfile_check_interval) int64_t size = info.st_size + 1; parsed_hosts_file_ptr = new RefCountedHostsFileMap; - parsed_hosts_file_ptr->next_sync_time = ink_time() + hostdb_hostfile_check_interval; + parsed_hosts_file_ptr->next_sync_time = ink_time() + hostdb_hostfile_check_interval_parse; parsed_hosts_file_ptr->HostFileText = static_cast(ats_malloc(size)); if (parsed_hosts_file_ptr->HostFileText) { char *base = parsed_hosts_file_ptr->HostFileText; diff --git a/iocore/hostdb/I_HostDBProcessor.h b/iocore/hostdb/I_HostDBProcessor.h index 3d15faf8423..fbf068a9ac3 100644 --- a/iocore/hostdb/I_HostDBProcessor.h +++ b/iocore/hostdb/I_HostDBProcessor.h @@ -62,6 +62,8 @@ extern unsigned int hostdb_ip_fail_timeout_interval; extern unsigned int hostdb_serve_stale_but_revalidate; extern unsigned int hostdb_round_robin_max_count; +extern int hostdb_max_iobuf_index; + static inline unsigned int makeHostHash(const char *string) { @@ -146,7 +148,7 @@ struct HostDBInfo : public RefCountObj { alloc(int size = 0) { size += sizeof(HostDBInfo); - int iobuffer_index = iobuffer_size_to_index(size); + int iobuffer_index = iobuffer_size_to_index(size, hostdb_max_iobuf_index); ink_release_assert(iobuffer_index >= 0); void *ptr = ioBufAllocator[iobuffer_index].alloc_void(); memset(ptr, 0, size); diff --git a/iocore/hostdb/P_HostDBProcessor.h b/iocore/hostdb/P_HostDBProcessor.h index 9739bf6a6eb..65c41a98f36 100644 --- a/iocore/hostdb/P_HostDBProcessor.h +++ b/iocore/hostdb/P_HostDBProcessor.h @@ -37,7 +37,6 @@ extern int hostdb_enable; extern int hostdb_migrate_on_demand; extern int hostdb_lookup_timeout; -extern int hostdb_insert_timeout; extern int hostdb_re_dns_on_reload; // 0 = obey, 1 = ignore, 2 = min(X,ttl), 3 = max(X,ttl) @@ -141,6 +140,7 @@ enum HostDB_Stats { hostdb_ttl_stat, // D average TTL hostdb_ttl_expires_stat, // D == TTL Expires hostdb_re_dns_on_reload_stat, + hostdb_insert_duplicate_to_pending_dns_stat, HostDB_Stat_Count }; diff --git a/iocore/net/BIO_fastopen.cc b/iocore/net/BIO_fastopen.cc index 78af8fab3c7..014d03a1ae9 100644 --- a/iocore/net/BIO_fastopen.cc +++ b/iocore/net/BIO_fastopen.cc @@ -159,7 +159,7 @@ fastopen_bread(BIO *bio, char *out, int outsz) if (err < 0) { errno = -err; if (BIO_sock_non_fatal_error(errno)) { - BIO_set_retry_write(bio); + BIO_set_retry_read(bio); } } diff --git a/iocore/net/Connection.cc b/iocore/net/Connection.cc index da7f530c9b4..4f5395ce6f8 100644 --- a/iocore/net/Connection.cc +++ b/iocore/net/Connection.cc @@ -136,7 +136,8 @@ add_http_filter(int fd ATS_UNUSED) int Server::setup_fd_for_listen(bool non_blocking, const NetProcessor::AcceptOptions &opt) { - int res = 0; + int res = 0; + int listen_per_thread = 0; ink_assert(fd != NO_FD); @@ -191,7 +192,7 @@ Server::setup_fd_for_listen(bool non_blocking, const NetProcessor::AcceptOptions } } - if ((res = safe_fcntl(fd, F_SETFD, FD_CLOEXEC)) < 0) { + if (safe_fcntl(fd, F_SETFD, FD_CLOEXEC) < 0) { goto Lerror; } @@ -200,33 +201,44 @@ Server::setup_fd_for_listen(bool non_blocking, const NetProcessor::AcceptOptions l.l_onoff = 0; l.l_linger = 0; if ((opt.sockopt_flags & NetVCOptions::SOCK_OPT_LINGER_ON) && - (res = safe_setsockopt(fd, SOL_SOCKET, SO_LINGER, reinterpret_cast(&l), sizeof(l))) < 0) { + safe_setsockopt(fd, SOL_SOCKET, SO_LINGER, reinterpret_cast(&l), sizeof(l)) < 0) { goto Lerror; } } - if (ats_is_ip6(&addr) && (res = safe_setsockopt(fd, IPPROTO_IPV6, IPV6_V6ONLY, SOCKOPT_ON, sizeof(int))) < 0) { + if (ats_is_ip6(&addr) && safe_setsockopt(fd, IPPROTO_IPV6, IPV6_V6ONLY, SOCKOPT_ON, sizeof(int)) < 0) { goto Lerror; } - if ((res = safe_setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, SOCKOPT_ON, sizeof(int))) < 0) { + if (safe_setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, SOCKOPT_ON, sizeof(int)) < 0) { goto Lerror; } + REC_ReadConfigInteger(listen_per_thread, "proxy.config.exec_thread.listen"); + if (listen_per_thread == 1) { + if (safe_setsockopt(fd, SOL_SOCKET, SO_REUSEPORT, SOCKOPT_ON, sizeof(int)) < 0) { + goto Lerror; + } +#ifdef SO_REUSEPORT_LB + if (safe_setsockopt(fd, SOL_SOCKET, SO_REUSEPORT_LB, SOCKOPT_ON, sizeof(int)) < 0) { + goto Lerror; + } +#endif + } if ((opt.sockopt_flags & NetVCOptions::SOCK_OPT_NO_DELAY) && - (res = safe_setsockopt(fd, IPPROTO_TCP, TCP_NODELAY, SOCKOPT_ON, sizeof(int))) < 0) { + safe_setsockopt(fd, IPPROTO_TCP, TCP_NODELAY, SOCKOPT_ON, sizeof(int)) < 0) { goto Lerror; } // enables 2 hour inactivity probes, also may fix IRIX FIN_WAIT_2 leak if ((opt.sockopt_flags & NetVCOptions::SOCK_OPT_KEEP_ALIVE) && - (res = safe_setsockopt(fd, SOL_SOCKET, SO_KEEPALIVE, SOCKOPT_ON, sizeof(int))) < 0) { + safe_setsockopt(fd, SOL_SOCKET, SO_KEEPALIVE, SOCKOPT_ON, sizeof(int)) < 0) { goto Lerror; } #ifdef TCP_FASTOPEN if ((opt.sockopt_flags & NetVCOptions::SOCK_OPT_TCP_FAST_OPEN) && - (res = safe_setsockopt(fd, IPPROTO_TCP, TCP_FASTOPEN, (char *)&opt.tfo_queue_length, sizeof(int)))) { + safe_setsockopt(fd, IPPROTO_TCP, TCP_FASTOPEN, (char *)&opt.tfo_queue_length, sizeof(int))) { goto Lerror; } #endif @@ -248,8 +260,7 @@ Server::setup_fd_for_listen(bool non_blocking, const NetProcessor::AcceptOptions #if defined(TCP_MAXSEG) if (NetProcessor::accept_mss > 0) { - if ((res = safe_setsockopt(fd, IPPROTO_TCP, TCP_MAXSEG, reinterpret_cast(&NetProcessor::accept_mss), sizeof(int))) < - 0) { + if (safe_setsockopt(fd, IPPROTO_TCP, TCP_MAXSEG, reinterpret_cast(&NetProcessor::accept_mss), sizeof(int)) < 0) { goto Lerror; } } @@ -258,7 +269,7 @@ Server::setup_fd_for_listen(bool non_blocking, const NetProcessor::AcceptOptions #ifdef TCP_DEFER_ACCEPT // set tcp defer accept timeout if it is configured, this will not trigger an accept until there is // data on the socket ready to be read - if (opt.defer_accept > 0 && (res = setsockopt(fd, IPPROTO_TCP, TCP_DEFER_ACCEPT, &opt.defer_accept, sizeof(int))) < 0) { + if (opt.defer_accept > 0 && setsockopt(fd, IPPROTO_TCP, TCP_DEFER_ACCEPT, &opt.defer_accept, sizeof(int)) < 0) { // FIXME: should we go to the error // goto error; Error("[Server::listen] Defer accept is configured but set failed: %d", errno); @@ -266,7 +277,7 @@ Server::setup_fd_for_listen(bool non_blocking, const NetProcessor::AcceptOptions #endif if (non_blocking) { - if ((res = safe_nonblocking(fd)) < 0) { + if (safe_nonblocking(fd) < 0) { goto Lerror; } } diff --git a/iocore/net/I_NetVConnection.h b/iocore/net/I_NetVConnection.h index df0667f2abc..ca7c3ab97f3 100644 --- a/iocore/net/I_NetVConnection.h +++ b/iocore/net/I_NetVConnection.h @@ -192,10 +192,14 @@ struct NetVCOptions { */ ats_scoped_str ssl_servername; + /** Server host name from client's request to use for SNI data on an outbound connection. + */ + ats_scoped_str sni_hostname; + /** * Client certificate to use in response to OS's certificate request */ - const char *ssl_client_cert_name = nullptr; + ats_scoped_str ssl_client_cert_name; /* * File containing private key matching certificate */ @@ -209,6 +213,8 @@ struct NetVCOptions { */ const char *ssl_client_ca_cert_path = nullptr; + bool tls_upstream = false; + /// Reset all values to defaults. /** @@ -247,6 +253,17 @@ struct NetVCOptions { return *this; } + self & + set_ssl_client_cert_name(const char *name) + { + if (name) { + ssl_client_cert_name = ats_strdup(name); + } else { + ssl_client_cert_name = nullptr; + } + return *this; + } + self & set_ssl_servername(const char *name) { @@ -258,6 +275,20 @@ struct NetVCOptions { return *this; } + self & + set_sni_hostname(const char *name, size_t len) + { + IpEndpoint ip; + + // Literal IPv4 and IPv6 addresses are not permitted in "HostName".(rfc6066#section-3) + if (name && len && ats_ip_pton(std::string_view(name, len), &ip) != 0) { + sni_hostname = ats_strndup(name, len); + } else { + sni_hostname = nullptr; + } + return *this; + } + self & operator=(self const &that) { @@ -272,8 +303,10 @@ struct NetVCOptions { * memcpy removes the extra reference to that's copy of the string * Removing the release will eventually cause a double free crash */ - sni_servername = nullptr; // release any current name. - ssl_servername = nullptr; + sni_servername = nullptr; // release any current name. + ssl_servername = nullptr; + sni_hostname = nullptr; + ssl_client_cert_name = nullptr; memcpy(static_cast(this), &that, sizeof(self)); if (that.sni_servername) { sni_servername.release(); // otherwise we'll free the source string. @@ -283,6 +316,14 @@ struct NetVCOptions { ssl_servername.release(); // otherwise we'll free the source string. this->ssl_servername = ats_strdup(that.ssl_servername); } + if (that.sni_hostname) { + sni_hostname.release(); // otherwise we'll free the source string. + this->sni_hostname = ats_strdup(that.sni_hostname); + } + if (that.ssl_client_cert_name) { + this->ssl_client_cert_name.release(); // otherwise we'll free the source string. + this->ssl_client_cert_name = ats_strdup(that.ssl_client_cert_name); + } } return *this; } @@ -309,7 +350,7 @@ struct NetVCOptions { stream IO to be done based on a single read or write call. */ -class NetVConnection : public AnnotatedVConnection +class NetVConnection : public VConnection, public PluginUserArgs { public: // How many bytes have been queued to the OS for sending by haven't been sent yet @@ -433,6 +474,15 @@ class NetVConnection : public AnnotatedVConnection */ virtual Action *send_OOB(Continuation *cont, char *buf, int len); + /** + Return the server name that is appropriate for the network VC type + */ + virtual const char * + get_server_name() const + { + return nullptr; + } + /** Cancels a scheduled send_OOB. Part of the message could have been sent already. Not callbacks to the cont are made after @@ -505,7 +555,9 @@ class NetVConnection : public AnnotatedVConnection is currently active. See section on timeout semantics above. */ - virtual void set_inactivity_timeout(ink_hrtime timeout_in) = 0; + virtual void set_inactivity_timeout(ink_hrtime timeout_in) = 0; + virtual void set_default_inactivity_timeout(ink_hrtime timeout_in) = 0; + virtual bool is_default_inactivity_timeout() = 0; /** Clears the active timeout. No active timeouts will be sent until @@ -605,6 +657,34 @@ class NetVConnection : public AnnotatedVConnection return netvc_context; } + /** + * Returns true if the network protocol + * supports a client provided SNI value + */ + virtual bool + support_sni() const + { + return false; + } + + virtual const char * + get_sni_servername() const + { + return nullptr; + } + + virtual bool + peer_provided_cert() const + { + return false; + } + + virtual int + provided_cert() const + { + return 0; + } + /** Structure holding user options. */ NetVCOptions options; @@ -846,7 +926,7 @@ class NetVConnection : public AnnotatedVConnection NetVConnectionContext_t netvc_context = NET_VCONNECTION_UNSET; }; -inline NetVConnection::NetVConnection() : AnnotatedVConnection(nullptr) +inline NetVConnection::NetVConnection() : VConnection(nullptr) { ink_zero(local_addr); diff --git a/iocore/net/I_UDPConnection.h b/iocore/net/I_UDPConnection.h index d06920f52de..5156eb8ea7d 100644 --- a/iocore/net/I_UDPConnection.h +++ b/iocore/net/I_UDPConnection.h @@ -49,7 +49,7 @@ class UDPConnection : public Continuation SOCKET getFd(); void setBinding(struct sockaddr const *); void setBinding(const IpAddr &, in_port_t); - inkcoreapi int getBinding(struct sockaddr *); + inkcoreapi bool getBinding(struct sockaddr *); void destroy(); int shouldDestroy(); @@ -73,7 +73,7 @@ class UDPConnection : public Continuation
cont->handleEvent(NET_EVENT_DATAGRAM_READ_READY, Queue<UDPPacketInternal> *) on incoming packets. - @return Action* Always returns ACTION_RESULT_NONE. Can't be + @return Action* Always returns nullptr. Can't be cancelled via this Action. @param c continuation to be called back */ @@ -86,8 +86,6 @@ class UDPConnection : public Continuation int getPortNum(); int GetSendGenerationNumber(); // const - void SetLastSentPktTSSeqNum(int64_t sentSeqNum); - int64_t cancel(); void setContinuation(Continuation *c); /** diff --git a/iocore/net/I_UDPNet.h b/iocore/net/I_UDPNet.h index 846c3a78e07..289a87f4ec1 100644 --- a/iocore/net/I_UDPNet.h +++ b/iocore/net/I_UDPNet.h @@ -63,6 +63,7 @@ class UDPNetProcessor : public Processor @param c Continuation that is called back with newly created socket. @param addr Address to bind (includes port) + @param fd File descriptor to use (if exists) @param send_bufsize (optional) Socket buffer size for sending. Limits how much outstanding data to OS before it is able to send to the NIC. @@ -71,7 +72,7 @@ class UDPNetProcessor : public Processor @return Action* Always returns ACTION_RESULT_DONE if socket was created successfully, or ACTION_IO_ERROR if not. */ - inkcoreapi Action *UDPBind(Continuation *c, sockaddr const *addr, int send_bufsize = 0, int recv_bufsize = 0); + inkcoreapi Action *UDPBind(Continuation *c, sockaddr const *addr, int fd = -1, int send_bufsize = 0, int recv_bufsize = 0); // Regarding sendto_re, sendmsg_re, recvfrom_re: // * You may be called back on 'c' with completion or error status. diff --git a/iocore/net/Makefile.am b/iocore/net/Makefile.am index 2ec6f4e668c..6af18e982cf 100644 --- a/iocore/net/Makefile.am +++ b/iocore/net/Makefile.am @@ -86,7 +86,7 @@ test_UDPNet_SOURCES = \ test_I_UDPNet.cc libinknet_a_SOURCES = \ - ALPNSupport.cc \ + ALPNSupport.cc \ BIO_fastopen.cc \ BIO_fastopen.h \ Connection.cc \ @@ -103,7 +103,7 @@ libinknet_a_SOURCES = \ YamlSNIConfig.cc \ Net.cc \ NetVConnection.cc \ - P_ALPNSupport.h \ + P_ALPNSupport.h \ P_SNIActionPerformer.h \ P_CompletionUtil.h \ P_Connection.h \ @@ -122,7 +122,6 @@ libinknet_a_SOURCES = \ P_SSLUtils.h \ P_SSLClientUtils.h \ P_OCSPStapling.h \ - P_Socks.h \ P_UDPConnection.h \ P_UDPIOEvent.h \ P_UDPNet.h \ @@ -153,7 +152,7 @@ libinknet_a_SOURCES = \ SSLSessionTicket.cc \ SSLUtils.cc \ OCSPStapling.cc \ - Socks.cc \ + TLSSessionResumptionSupport.cc \ UDPIOEvent.cc \ UnixConnection.cc \ UnixNet.cc \ diff --git a/iocore/net/Net.cc b/iocore/net/Net.cc index c8af271e5fc..f0cac5b4767 100644 --- a/iocore/net/Net.cc +++ b/iocore/net/Net.cc @@ -65,10 +65,15 @@ configure_net() REC_ReadConfigStringAlloc(ccp, "proxy.config.net.tcp_congestion_control_in"); if (ccp && *ccp != '\0') { net_ccp_in = ccp; + } else { + ats_free(ccp); } + REC_ReadConfigStringAlloc(ccp, "proxy.config.net.tcp_congestion_control_out"); if (ccp && *ccp != '\0') { net_ccp_out = ccp; + } else { + ats_free(ccp); } } @@ -97,7 +102,8 @@ register_net_stats() const std::pair non_persistent[] = { {"proxy.process.net.accepts_currently_open", net_accepts_currently_open_stat}, {"proxy.process.net.connections_currently_open", net_connections_currently_open_stat}, - {"proxy.process.net.default_inactivity_timeout_applied", default_inactivity_timeout_stat}, + {"proxy.process.net.default_inactivity_timeout_applied", default_inactivity_timeout_applied_stat}, + {"proxy.process.net.default_inactivity_timeout_count", default_inactivity_timeout_count_stat}, {"proxy.process.net.dynamic_keep_alive_timeout_in_count", keep_alive_queue_timeout_count_stat}, {"proxy.process.net.dynamic_keep_alive_timeout_in_total", keep_alive_queue_timeout_total_stat}, {"proxy.process.socks.connections_currently_open", socks_connections_currently_open_stat}, @@ -125,7 +131,8 @@ register_net_stats() NET_CLEAR_DYN_STAT(socks_connections_currently_open_stat); NET_CLEAR_DYN_STAT(keep_alive_queue_timeout_total_stat); NET_CLEAR_DYN_STAT(keep_alive_queue_timeout_count_stat); - NET_CLEAR_DYN_STAT(default_inactivity_timeout_stat); + NET_CLEAR_DYN_STAT(default_inactivity_timeout_count_stat); + NET_CLEAR_DYN_STAT(default_inactivity_timeout_applied_stat); RecRegisterRawStat(net_rsb, RECT_PROCESS, "proxy.process.tcp.total_accepts", RECD_INT, RECP_NON_PERSISTENT, static_cast(net_tcp_accept_stat), RecRawStatSyncSum); @@ -135,6 +142,8 @@ register_net_stats() (int)net_connections_throttled_in_stat, RecRawStatSyncSum); RecRegisterRawStat(net_rsb, RECT_PROCESS, "proxy.process.net.connections_throttled_out", RECD_INT, RECP_PERSISTENT, (int)net_connections_throttled_out_stat, RecRawStatSyncSum); + RecRegisterRawStat(net_rsb, RECT_PROCESS, "proxy.process.net.max.requests_throttled_in", RECD_INT, RECP_PERSISTENT, + (int)net_requests_max_throttled_in_stat, RecRawStatSyncSum); } void diff --git a/iocore/net/NetEvent.h b/iocore/net/NetEvent.h new file mode 100644 index 00000000000..94e96fedda9 --- /dev/null +++ b/iocore/net/NetEvent.h @@ -0,0 +1,96 @@ +/** @file + + A brief file description + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#pragma once + +#include "I_EventSystem.h" + +class NetHandler; + +// this class is used to NetHandler to hide some detail of NetEvent. +// To combine the `UDPConenction` and `NetEvent`. NetHandler should +// callback to net_read_io or net_write_io when net event happen. +class NetEvent +{ +public: + NetEvent() = default; + virtual ~NetEvent() {} + virtual void net_read_io(NetHandler *nh, EThread *lthread) = 0; + virtual void net_write_io(NetHandler *nh, EThread *lthread) = 0; + virtual void free(EThread *t) = 0; + + // since we want this class to be independent from VConnection, Continutaion. There should be + // a pure virtual function which connect sub class and NetHandler. + virtual int callback(int event = CONTINUATION_EVENT_NONE, void *data = nullptr) = 0; + + // Duplicate with `NetVConnection::set_inactivity_timeout` + // TODO: more abstraction. + virtual void set_inactivity_timeout(ink_hrtime timeout_in) = 0; + virtual void set_default_inactivity_timeout(ink_hrtime timeout_in) = 0; + virtual bool is_default_inactivity_timeout() = 0; + + // get this vc's thread + virtual EThread *get_thread() = 0; + + // Close when EventIO close; + virtual int close() = 0; + + // get fd + virtual int get_fd() = 0; + virtual Ptr &get_mutex() = 0; + virtual ContFlags &get_control_flags() = 0; + + EventIO ep{}; + NetState read{}; + NetState write{}; + + int closed = 0; + NetHandler *nh = nullptr; + + ink_hrtime inactivity_timeout_in = 0; + ink_hrtime active_timeout_in = 0; + ink_hrtime next_inactivity_timeout_at = 0; + ink_hrtime next_activity_timeout_at = 0; + ink_hrtime submit_time = 0; + + bool default_inactivity_timeout = false; + + LINK(NetEvent, open_link); + LINK(NetEvent, cop_link); + LINKM(NetEvent, read, ready_link) + SLINKM(NetEvent, read, enable_link) + LINKM(NetEvent, write, ready_link) + SLINKM(NetEvent, write, enable_link) + LINK(NetEvent, keep_alive_queue_link); + LINK(NetEvent, active_queue_link); + + union { + unsigned int flags = 0; +#define NET_VC_SHUTDOWN_READ 1 +#define NET_VC_SHUTDOWN_WRITE 2 + struct { + unsigned int got_local_addr : 1; + unsigned int shutdown : 2; + } f; + }; +}; diff --git a/iocore/net/NetVCTest.cc b/iocore/net/NetVCTest.cc index 5d3eaa99e25..59a066b8b0f 100644 --- a/iocore/net/NetVCTest.cc +++ b/iocore/net/NetVCTest.cc @@ -138,8 +138,8 @@ NetVCTest::start_test() test_vc->set_inactivity_timeout(HRTIME_SECONDS(timeout)); test_vc->set_active_timeout(HRTIME_SECONDS(timeout + 5)); - read_buffer = new_MIOBuffer(); - write_buffer = new_MIOBuffer(); + read_buffer = new_MIOBuffer(BUFFER_SIZE_INDEX_32K); + write_buffer = new_MIOBuffer(BUFFER_SIZE_INDEX_32K); reader_for_rbuf = read_buffer->alloc_reader(); reader_for_wbuf = write_buffer->alloc_reader(); diff --git a/iocore/net/OCSPStapling.cc b/iocore/net/OCSPStapling.cc index 55e2b7e9393..7c39d2ba64b 100644 --- a/iocore/net/OCSPStapling.cc +++ b/iocore/net/OCSPStapling.cc @@ -74,7 +74,7 @@ certinfo_map_free(void * /*parent*/, void *ptr, CRYPTO_EX_DATA * /*ad*/, int /*i ink_mutex_destroy(&iter->second->stapling_mutex); OPENSSL_free(iter->second); } - free(map); + delete map; } static int ssl_stapling_index = -1; @@ -374,10 +374,12 @@ query_responder(BIO *b, char *host, char *path, OCSP_REQUEST *req, int req_timeo OCSP_REQ_CTX_free(ctx); - if (rv) { + if (rv == 1) { return resp; } + Error("failed to connect to OCSP server; host=%s path=%s", host, path); + return nullptr; } @@ -396,7 +398,7 @@ process_responder(OCSP_REQUEST *req, char *host, char *path, char *port, int req BIO_set_nbio(cbio, 1); if (BIO_do_connect(cbio) <= 0 && !BIO_should_retry(cbio)) { - Debug("ssl_ocsp", "process_responder: failed to connect to OCSP response server. host=%s port=%s path=%s", host, port, path); + Debug("ssl_ocsp", "process_responder: failed to connect to OCSP server; host=%s port=%s path=%s", host, port, path); goto end; } resp = query_responder(cbio, host, path, req, req_timeout); @@ -415,16 +417,18 @@ stapling_refresh_response(certinfo *cinf, OCSP_RESPONSE **prsp) OCSP_REQUEST *req = nullptr; OCSP_CERTID *id = nullptr; char *host = nullptr, *port = nullptr, *path = nullptr; - int ssl_flag = 0; - int req_timeout = -1; + int ssl_flag = 0; + int response_status = 0; - Debug("ssl_ocsp", "stapling_refresh_response: querying responder"); *prsp = nullptr; if (!OCSP_parse_url(cinf->uri, &host, &port, &path, &ssl_flag)) { + Debug("ssl_ocsp", "stapling_refresh_response: OCSP_parse_url failed; uri=%s", cinf->uri); goto err; } + Debug("ssl_ocsp", "stapling_refresh_response: querying responder; host=%s port=%s path=%s", host, port, path); + req = OCSP_REQUEST_new(); if (!req) { goto err; @@ -437,19 +441,18 @@ stapling_refresh_response(certinfo *cinf, OCSP_RESPONSE **prsp) goto err; } - req_timeout = SSLConfigParams::ssl_ocsp_request_timeout; - *prsp = process_responder(req, host, path, port, req_timeout); - + *prsp = process_responder(req, host, path, port, SSLConfigParams::ssl_ocsp_request_timeout); if (*prsp == nullptr) { goto done; } - if (OCSP_response_status(*prsp) == OCSP_RESPONSE_STATUS_SUCCESSFUL) { + response_status = OCSP_response_status(*prsp); + if (response_status == OCSP_RESPONSE_STATUS_SUCCESSFUL) { Debug("ssl_ocsp", "stapling_refresh_response: query response received"); stapling_check_response(cinf, *prsp); } else { - // TODO: We should log the actual openssl error - Error("stapling_refresh_response: responder error"); + Error("stapling_refresh_response: responder response error; host=%s port=%s path=%s response_status=%d", host, port, path, + response_status); } if (!stapling_cache_response(*prsp, cinf)) { diff --git a/iocore/net/P_Net.h b/iocore/net/P_Net.h index 15a55bb8bb6..b3027ce155a 100644 --- a/iocore/net/P_Net.h +++ b/iocore/net/P_Net.h @@ -51,12 +51,14 @@ enum Net_Stats { inactivity_cop_lock_acquire_failure_stat, keep_alive_queue_timeout_total_stat, keep_alive_queue_timeout_count_stat, - default_inactivity_timeout_stat, + default_inactivity_timeout_applied_stat, + default_inactivity_timeout_count_stat, net_fastopen_attempts_stat, net_fastopen_successes_stat, net_tcp_accept_stat, net_connections_throttled_in_stat, net_connections_throttled_out_stat, + net_requests_max_throttled_in_stat, Net_Stat_Count }; @@ -109,13 +111,6 @@ extern RecRawStatBlock *net_rsb; #include "P_SSLNetAccept.h" #include "P_SSLCertLookup.h" -#if TS_USE_QUIC == 1 -#include "P_QUICNetVConnection.h" -#include "P_QUICNetProcessor.h" -#include "P_QUICPacketHandler.h" -#include "P_QUICNet.h" -#endif - static constexpr ts::ModuleVersion NET_SYSTEM_MODULE_INTERNAL_VERSION(NET_SYSTEM_MODULE_PUBLIC_VERSION, ts::ModuleVersion::PRIVATE); // For very verbose iocore debugging. diff --git a/iocore/net/P_NetAccept.h b/iocore/net/P_NetAccept.h index aa136b0fb7a..db7ceb6ce88 100644 --- a/iocore/net/P_NetAccept.h +++ b/iocore/net/P_NetAccept.h @@ -107,6 +107,7 @@ struct NetAccept : public Continuation { virtual int acceptEvent(int event, void *e); virtual int acceptFastEvent(int event, void *e); + virtual int accept_per_thread(int event, void *e); int acceptLoopEvent(int event, Event *e); void cancel(); diff --git a/iocore/net/P_QUICNet.h b/iocore/net/P_QUICNet.h index 802bf6ed6e6..09468bc03a4 100644 --- a/iocore/net/P_QUICNet.h +++ b/iocore/net/P_QUICNet.h @@ -21,14 +21,16 @@ limitations under the License. */ -#ifndef __P_QUICNET_H__ -#define __P_QUICNET_H__ +#pragma once #include #include "tscore/ink_platform.h" #include "P_Net.h" +#include "quic/QUICTypes.h" +#include "P_QUICNetProcessor.h" +#include "P_QUICNetVConnection.h" class NetHandler; typedef int (NetHandler::*NetContHandler)(int, void *); @@ -74,4 +76,3 @@ get_QUICPollCont(EThread *t) } extern ClassAllocator quicPollEventAllocator; -#endif diff --git a/iocore/net/P_QUICNetProcessor.h b/iocore/net/P_QUICNetProcessor.h index bb3e8576b74..68e722814d5 100644 --- a/iocore/net/P_QUICNetProcessor.h +++ b/iocore/net/P_QUICNetProcessor.h @@ -42,6 +42,7 @@ #include "quic/QUICConnectionTable.h" class UnixNetVConnection; +class QUICResetTokenTable; struct NetAccept; ////////////////////////////////////////////////////////////////// @@ -73,6 +74,7 @@ class QUICNetProcessor : public UnixNetProcessor QUICNetProcessor &operator=(const QUICNetProcessor &); QUICConnectionTable *_ctable = nullptr; + QUICResetTokenTable *_rtable = nullptr; }; extern QUICNetProcessor quic_NetProcessor; diff --git a/iocore/net/P_QUICNetVConnection.h b/iocore/net/P_QUICNetVConnection.h index 925a4fed057..f04f32da6c8 100644 --- a/iocore/net/P_QUICNetVConnection.h +++ b/iocore/net/P_QUICNetVConnection.h @@ -37,31 +37,39 @@ #include "P_UnixNetVConnection.h" #include "P_UnixNet.h" #include "P_UDPNet.h" +#include "P_ALPNSupport.h" +#include "TLSSessionResumptionSupport.h" #include "tscore/ink_apidefs.h" #include "tscore/List.h" #include "quic/QUICConfig.h" #include "quic/QUICConnection.h" #include "quic/QUICConnectionTable.h" +#include "quic/QUICResetTokenTable.h" #include "quic/QUICVersionNegotiator.h" #include "quic/QUICPacket.h" #include "quic/QUICPacketFactory.h" #include "quic/QUICFrame.h" #include "quic/QUICFrameDispatcher.h" -#include "quic/QUICHandshake.h" #include "quic/QUICApplication.h" #include "quic/QUICStream.h" #include "quic/QUICHandshakeProtocol.h" #include "quic/QUICAckFrameCreator.h" #include "quic/QUICPinger.h" +#include "quic/QUICPadder.h" #include "quic/QUICLossDetector.h" #include "quic/QUICStreamManager.h" #include "quic/QUICAltConnectionManager.h" #include "quic/QUICPathValidator.h" +#include "quic/QUICPathManager.h" #include "quic/QUICApplicationMap.h" #include "quic/QUICPacketReceiveQueue.h" +#include "quic/QUICPacketHeaderProtector.h" #include "quic/QUICAddrVerifyState.h" #include "quic/QUICPacketProtectionKeyInfo.h" +#include "quic/QUICContext.h" +#include "quic/QUICTokenCreator.h" +#include "quic/qlog/QLogListener.h" // Size of connection ids for debug log : e.g. aaaaaaaa-bbbbbbbb\0 static constexpr size_t MAX_CIDS_SIZE = 8 + 1 + 8 + 1; @@ -76,6 +84,7 @@ static constexpr size_t MAX_CIDS_SIZE = 8 + 1 + 8 + 1; class QUICPacketHandler; class QUICLossDetector; +class QUICHandshake; class SSLNextProtocolSet; @@ -129,22 +138,27 @@ class SSLNextProtocolSet; **/ class QUICNetVConnection : public UnixNetVConnection, public QUICConnection, - public QUICFrameGenerator, public RefCountObj, - public ALPNSupport + public ALPNSupport, + public TLSSessionResumptionSupport { using super = UnixNetVConnection; ///< Parent type. public: QUICNetVConnection(); ~QUICNetVConnection(); - void init(QUICConnectionId peer_cid, QUICConnectionId original_cid, UDPConnection *, QUICPacketHandler *); - void init(QUICConnectionId peer_cid, QUICConnectionId original_cid, QUICConnectionId first_cid, UDPConnection *, - QUICPacketHandler *, QUICConnectionTable *ctable); + void init(QUICVersion version, QUICConnectionId peer_cid, QUICConnectionId original_cid, UDPConnection *, QUICPacketHandler *, + QUICResetTokenTable *rtable); + void init(QUICVersion version, QUICConnectionId peer_cid, QUICConnectionId original_cid, QUICConnectionId first_cid, + QUICConnectionId retry_cid, UDPConnection *, QUICPacketHandler *, QUICResetTokenTable *rtable, + QUICConnectionTable *ctable); // accept new conn_id int acceptEvent(int event, Event *e); + // NetVConnection + void set_local_addr() override; + // UnixNetVConnection void reenable(VIO *vio) override; VIO *do_io_read(Continuation *c, int64_t nbytes, MIOBuffer *buf) override; @@ -177,7 +191,8 @@ class QUICNetVConnection : public UnixNetVConnection, // QUICConnection QUICStreamManager *stream_manager() override; - void close(QUICConnectionErrorUPtr error) override; + void close_quic_connection(QUICConnectionErrorUPtr error) override; + void reset_quic_connection() override; void handle_received_packet(UDPPacket *packet) override; void ping() override; @@ -185,23 +200,25 @@ class QUICNetVConnection : public UnixNetVConnection, QUICConnectionId peer_connection_id() const override; QUICConnectionId original_connection_id() const override; QUICConnectionId first_connection_id() const override; + QUICConnectionId retry_source_connection_id() const override; + QUICConnectionId initial_source_connection_id() const override; QUICConnectionId connection_id() const override; std::string_view cids() const override; const QUICFiveTuple five_tuple() const override; uint32_t pmtu() const override; NetVConnectionContext_t direction() const override; + QUICVersion negotiated_version() const override; std::string_view negotiated_application_name() const override; bool is_closed() const override; + bool is_at_anti_amplification_limit() const override; + bool is_address_validation_completed() const override; + bool is_handshake_completed() const override; + bool has_keys_for(QUICPacketNumberSpace space) const override; // QUICConnection (QUICFrameHandler) std::vector interests() override; QUICConnectionErrorUPtr handle_frame(QUICEncryptionLevel level, const QUICFrame &frame) override; - // QUICFrameGenerator - bool will_generate_frame(QUICEncryptionLevel level, ink_hrtime timestamp) override; - QUICFrame *generate_frame(uint8_t *buf, QUICEncryptionLevel level, uint64_t connection_credit, uint16_t maximum_frame_size, - ink_hrtime timestamp) override; - int in_closed_queue = 0; bool shouldDestroy(); @@ -209,29 +226,34 @@ class QUICNetVConnection : public UnixNetVConnection, LINK(QUICNetVConnection, closed_link); SLINK(QUICNetVConnection, closed_alink); +protected: + const IpEndpoint &_getLocalEndpoint() override; + private: std::random_device _rnd; QUICConfig::scoped_config _quic_config; - QUICConnectionId _peer_quic_connection_id; // dst cid in local - QUICConnectionId _peer_old_quic_connection_id; // dst previous cid in local - QUICConnectionId _original_quic_connection_id; // dst cid of initial packet from client - QUICConnectionId _first_quic_connection_id; // dst cid of initial packet from client that doesn't have retry token - QUICConnectionId _quic_connection_id; // src cid in local + QUICConnectionId _peer_quic_connection_id; // dst cid in local + QUICConnectionId _peer_old_quic_connection_id; // dst previous cid in local + QUICConnectionId _original_quic_connection_id; // dst cid of initial packet from client + QUICConnectionId _first_quic_connection_id; // dst cid of initial packet from client that doesn't have retry token + QUICConnectionId _retry_source_connection_id; // src cid used for sending Retry packet + QUICConnectionId _initial_source_connection_id; // src cid used for Initial packet + QUICConnectionId _quic_connection_id; // src cid in local QUICFiveTuple _five_tuple; bool _connection_migration_initiated = false; char _cids_data[MAX_CIDS_SIZE] = {0}; std::string_view _cids; + QUICVersion _initial_version; UDPConnection *_udp_con = nullptr; QUICPacketProtectionKeyInfo _pp_key_info; QUICPacketHandler *_packet_handler = nullptr; QUICPacketFactory _packet_factory; QUICFrameFactory _frame_factory; QUICAckFrameManager _ack_frame_manager; - QUICPinger _pinger; QUICPacketHeaderProtector _ph_protector; QUICRTTMeasure _rtt_measure; QUICApplicationMap *_application_map = nullptr; @@ -240,6 +262,8 @@ class QUICNetVConnection : public UnixNetVConnection, // TODO: use custom allocator and make them std::unique_ptr or std::shared_ptr // or make them just member variables. + QUICPinger *_pinger = nullptr; + QUICPadder *_padder = nullptr; QUICHandshake *_handshake_handler = nullptr; QUICHandshakeProtocol *_hs_protocol = nullptr; QUICLossDetector *_loss_detector = nullptr; @@ -248,11 +272,14 @@ class QUICNetVConnection : public UnixNetVConnection, QUICCongestionController *_congestion_controller = nullptr; QUICRemoteFlowController *_remote_flow_controller = nullptr; QUICLocalFlowController *_local_flow_controller = nullptr; + QUICResetTokenTable *_rtable = nullptr; QUICConnectionTable *_ctable = nullptr; QUICAltConnectionManager *_alt_con_manager = nullptr; QUICPathValidator *_path_validator = nullptr; + QUICPathManager *_path_manager = nullptr; + QUICTokenCreator *_token_creator = nullptr; - std::vector _frame_generators; + QUICFrameGeneratorManager _frame_generators; QUICPacketReceiveQueue _packet_recv_queue = {this->_packet_factory, this->_ph_protector}; @@ -261,6 +288,8 @@ class QUICNetVConnection : public UnixNetVConnection, uint32_t _state_closing_recv_packet_window = 1; uint64_t _flow_control_buffer_size = 1024; + void _init_submodules(); + void _schedule_packet_write_ready(bool delay = false); void _unschedule_packet_write_ready(); void _close_packet_write_ready(Event *data); @@ -276,11 +305,6 @@ class QUICNetVConnection : public UnixNetVConnection, void _close_closed_event(Event *data); Event *_closed_event = nullptr; - void _schedule_path_validation_timeout(ink_hrtime interval); - void _unschedule_path_validation_timeout(); - void _close_path_validation_timeout(Event *data); - Event *_path_validation_timeout = nullptr; - void _schedule_ack_manager_periodic(ink_hrtime interval); void _unschedule_ack_manager_periodic(); Event *_ack_manager_periodic = nullptr; @@ -293,24 +317,24 @@ class QUICNetVConnection : public UnixNetVConnection, uint64_t _maximum_stream_frame_data_size(); Ptr _store_frame(Ptr parent_block, size_t &size_added, uint64_t &max_frame_size, QUICFrame &frame, - std::vector &frames); - QUICPacketUPtr _packetize_frames(QUICEncryptionLevel level, uint64_t max_packet_size, std::vector &frames); + std::vector &frames); + QUICPacketUPtr _packetize_frames(uint8_t *packet_buf, QUICEncryptionLevel level, uint64_t max_packet_size, + std::vector &frames); void _packetize_closing_frame(); - Ptr _generate_padding_frame(size_t frame_size); - QUICPacketUPtr _build_packet(QUICEncryptionLevel level, Ptr parent_block, bool retransmittable, bool probing, - bool crypto); + QUICPacketUPtr _build_packet(uint8_t *packet_buf, QUICEncryptionLevel level, const Ptr &parent_block, + bool retransmittable, bool probing, bool crypto); - QUICConnectionErrorUPtr _recv_and_ack(const QUICPacket &packet, bool *has_non_probing_frame = nullptr); + QUICConnectionErrorUPtr _recv_and_ack(const QUICPacketR &packet, bool *has_non_probing_frame = nullptr); QUICConnectionErrorUPtr _state_handshake_process_packet(const QUICPacket &packet); - QUICConnectionErrorUPtr _state_handshake_process_version_negotiation_packet(const QUICPacket &packet); - QUICConnectionErrorUPtr _state_handshake_process_initial_packet(const QUICPacket &packet); - QUICConnectionErrorUPtr _state_handshake_process_retry_packet(const QUICPacket &packet); - QUICConnectionErrorUPtr _state_handshake_process_handshake_packet(const QUICPacket &packet); - QUICConnectionErrorUPtr _state_handshake_process_zero_rtt_protected_packet(const QUICPacket &packet); + QUICConnectionErrorUPtr _state_handshake_process_version_negotiation_packet(const QUICVersionNegotiationPacketR &packet); + QUICConnectionErrorUPtr _state_handshake_process_initial_packet(const QUICInitialPacketR &packet); + QUICConnectionErrorUPtr _state_handshake_process_retry_packet(const QUICRetryPacketR &packet); + QUICConnectionErrorUPtr _state_handshake_process_handshake_packet(const QUICHandshakePacketR &packet); + QUICConnectionErrorUPtr _state_handshake_process_zero_rtt_protected_packet(const QUICZeroRttPacketR &packet); QUICConnectionErrorUPtr _state_connection_established_receive_packet(); - QUICConnectionErrorUPtr _state_connection_established_process_protected_packet(const QUICPacket &packet); - QUICConnectionErrorUPtr _state_connection_established_migrate_connection(const QUICPacket &packet); + QUICConnectionErrorUPtr _state_connection_established_process_protected_packet(const QUICShortHeaderPacketR &packet); + QUICConnectionErrorUPtr _state_connection_established_migrate_connection(const QUICPacketR &packet); QUICConnectionErrorUPtr _state_connection_established_initiate_connection_migration(); QUICConnectionErrorUPtr _state_closing_receive_packet(); QUICConnectionErrorUPtr _state_draining_receive_packet(); @@ -323,10 +347,12 @@ class QUICNetVConnection : public UnixNetVConnection, void _init_flow_control_params(const std::shared_ptr &local_tp, const std::shared_ptr &remote_tp); void _handle_error(QUICConnectionErrorUPtr error); - QUICPacketUPtr _dequeue_recv_packet(QUICPacketCreationResult &result); - void _validate_new_path(); + QUICPacketUPtr _dequeue_recv_packet(uint8_t *packet_buf, QUICPacketCreationResult &result); + void _validate_new_path(const QUICPath &path); + bool _handshake_completed = false; int _complete_handshake_if_possible(); + void _switch_to_handshake_state(); void _switch_to_established_state(); void _switch_to_closing_state(QUICConnectionErrorUPtr error); @@ -337,8 +363,8 @@ class QUICNetVConnection : public UnixNetVConnection, void _start_application(); void _handle_periodic_ack_event(); - void _handle_path_validation_timeout(Event *data); void _handle_idle_timeout(); + void _handle_active_timeout(); QUICConnectionErrorUPtr _handle_frame(const QUICNewConnectionIdFrame &frame); @@ -347,24 +373,24 @@ class QUICNetVConnection : public UnixNetVConnection, void _update_local_cid(const QUICConnectionId &new_cid); void _rerandomize_original_cid(); - QUICHandshakeProtocol *_setup_handshake_protocol(shared_SSL_CTX ctx); + QUICHandshakeProtocol *_setup_handshake_protocol(const shared_SSL_CTX &ctx); QUICPacketUPtr _the_final_packet = QUICPacketFactory::create_null_packet(); + uint8_t _final_packet_buf[QUICPacket::MAX_INSTANCE_SIZE]; QUICStatelessResetToken _reset_token; - ats_unique_buf _av_token = {nullptr}; - size_t _av_token_len = 0; - bool _is_resumption_token_sent = false; + ats_unique_buf _av_token = {nullptr}; + size_t _av_token_len = 0; uint64_t _stream_frames_sent = 0; + uint32_t _seq_num = 0; // TODO: Source addresses verification through an address validation token - bool _has_ack_eliciting_packet_out = true; + QUICAddrVerifyState _verified_state; - QUICAddrVerifyState _verfied_state; + std::unique_ptr _context; - // QUICFrameGenerator - void _on_frame_lost(QUICFrameInformationUPtr &info) override; + std::shared_ptr _qlog; }; typedef int (QUICNetVConnection::*QUICNetVConnHandler)(int, void *); diff --git a/iocore/net/P_QUICPacketHandler.h b/iocore/net/P_QUICPacketHandler.h index 27a5235d911..9428b89a4ba 100644 --- a/iocore/net/P_QUICPacketHandler.h +++ b/iocore/net/P_QUICPacketHandler.h @@ -28,6 +28,7 @@ #include "P_NetAccept.h" #include "quic/QUICTypes.h" #include "quic/QUICConnectionTable.h" +#include "quic/QUICResetTokenTable.h" class QUICClosedConCollector; class QUICNetVConnection; @@ -37,11 +38,11 @@ class QUICPacketHeaderProtector; class QUICPacketHandler { public: - QUICPacketHandler(); + QUICPacketHandler(QUICResetTokenTable &rtable); ~QUICPacketHandler(); void send_packet(const QUICPacket &packet, QUICNetVConnection *vc, const QUICPacketHeaderProtector &pn_protector); - void send_packet(QUICNetVConnection *vc, Ptr udp_payload); + void send_packet(QUICNetVConnection *vc, const Ptr &udp_payload); void close_connection(QUICNetVConnection *conn); @@ -49,25 +50,28 @@ class QUICPacketHandler void _send_packet(const QUICPacket &packet, UDPConnection *udp_con, IpEndpoint &addr, uint32_t pmtu, const QUICPacketHeaderProtector *ph_protector, int dcil); void _send_packet(UDPConnection *udp_con, IpEndpoint &addr, Ptr udp_payload); + QUICConnection *_check_stateless_reset(const uint8_t *buf, size_t buf_len); // FIXME Remove this - // QUICPacketHandler could be a continuation, but NetAccept is a contination too. + // QUICPacketHandler could be a continuation, but NetAccept is a continuation too. virtual Continuation *_get_continuation() = 0; Event *_collector_event = nullptr; QUICClosedConCollector *_closed_con_collector = nullptr; virtual void _recv_packet(int event, UDPPacket *udpPacket) = 0; + + QUICResetTokenTable &_rtable; }; /* - * @class QUICPacketHanderIn + * @class QUICPacketHandlerIn * @brief QUIC Packet Handler for incoming connections */ class QUICPacketHandlerIn : public NetAccept, public QUICPacketHandler { public: - QUICPacketHandlerIn(const NetProcessor::AcceptOptions &opt, QUICConnectionTable &ctable); + QUICPacketHandlerIn(const NetProcessor::AcceptOptions &opt, QUICConnectionTable &ctable, QUICResetTokenTable &rtable); ~QUICPacketHandlerIn(); // NetAccept @@ -83,19 +87,23 @@ class QUICPacketHandlerIn : public NetAccept, public QUICPacketHandler private: void _recv_packet(int event, UDPPacket *udp_packet) override; int _stateless_retry(const uint8_t *buf, uint64_t buf_len, UDPConnection *connection, IpEndpoint from, QUICConnectionId dcid, - QUICConnectionId scid, QUICConnectionId *original_cid); + QUICConnectionId scid, QUICConnectionId *original_cid, QUICConnectionId *retry_cid, QUICVersion version); + bool _send_stateless_reset(QUICConnectionId dcid, uint32_t instance_id, UDPConnection *udp_con, IpEndpoint &addr, + size_t maximum_size); + void _send_invalid_token_error(const uint8_t *initial_packet, uint64_t initial_packet_len, UDPConnection *connection, + IpEndpoint from); QUICConnectionTable &_ctable; }; /* - * @class QUICPacketHanderOut + * @class QUICPacketHandlerOut * @brief QUIC Packet Handler for outgoing connections */ class QUICPacketHandlerOut : public Continuation, public QUICPacketHandler { public: - QUICPacketHandlerOut(); + QUICPacketHandlerOut(QUICResetTokenTable &rtable); ~QUICPacketHandlerOut(){}; void init(QUICNetVConnection *vc); diff --git a/iocore/net/P_SNIActionPerformer.h b/iocore/net/P_SNIActionPerformer.h index 5e9f352ccd6..2f9bd01265d 100644 --- a/iocore/net/P_SNIActionPerformer.h +++ b/iocore/net/P_SNIActionPerformer.h @@ -38,7 +38,30 @@ class ActionItem { public: - virtual int SNIAction(Continuation *cont) const = 0; + /** + * Context should contain extra data needed to be passed to the actual SNIAction. + */ + struct Context { + /** + * if any, fqdn_wildcard_captured_groups will hold the captured groups from the `fqdn` + * match which will be used to construct the tunnel destination. + */ + std::optional> _fqdn_wildcard_captured_groups; + }; + + virtual int SNIAction(Continuation *cont, const Context &ctx) const = 0; + + /** + This method tests whether this action would have been triggered by a + particuarly SNI value and IP address combination. This is run after the + TLS exchange finished to see if the client used an SNI name different from + the host name to avoid SNI-based policy + */ + virtual bool + TestClientSNIAction(const char *servername, const IpEndpoint &ep, int &policy) const + { + return false; + } virtual ~ActionItem(){}; }; @@ -49,7 +72,7 @@ class ControlH2 : public ActionItem ~ControlH2() override {} int - SNIAction(Continuation *cont) const override + SNIAction(Continuation *cont, const Context &ctx) const override { auto ssl_vc = dynamic_cast(cont); if (ssl_vc) { @@ -69,21 +92,102 @@ class ControlH2 : public ActionItem class TunnelDestination : public ActionItem { public: - TunnelDestination(const std::string_view &dest, bool decrypt) : destination(dest), tunnel_decrypt(decrypt) {} + TunnelDestination(const std::string_view &dest, bool decrypt, bool tls_upstream) + : destination(dest), tunnel_decrypt(decrypt), tls_upstream(tls_upstream) + { + need_fix = (destination.find_first_of('$') != std::string::npos); + } ~TunnelDestination() override {} int - SNIAction(Continuation *cont) const override + SNIAction(Continuation *cont, const Context &ctx) const override { // Set the netvc option? SSLNetVConnection *ssl_netvc = dynamic_cast(cont); if (ssl_netvc) { - ssl_netvc->set_tunnel_destination(destination, tunnel_decrypt); + // If needed, we will try to amend the tunnel destination. + if (ctx._fqdn_wildcard_captured_groups && need_fix) { + const auto &fixed_dst = replace_match_groups(destination, *ctx._fqdn_wildcard_captured_groups); + ssl_netvc->set_tunnel_destination(fixed_dst, tunnel_decrypt, tls_upstream); + Debug("TunnelDestination", "Destination now is [%s], configured [%s]", fixed_dst.c_str(), destination.c_str()); + } else { + ssl_netvc->set_tunnel_destination(destination, tunnel_decrypt, tls_upstream); + } } return SSL_TLSEXT_ERR_OK; } + +private: + bool + is_number(const std::string &s) const + { + return !s.empty() && + std::find_if(std::begin(s), std::end(s), [](std::string::value_type c) { return !std::isdigit(c); }) == std::end(s); + } + + /** + * `tunnel_route` may contain matching groups ie: `$1` which needs to be replaced by the corresponding + * captured group from the `fqdn`, this function will replace them using proper group string. Matching + * groups could be at any order. + */ + std::string + replace_match_groups(const std::string &dst, const std::vector &groups) const + { + if (dst.empty() || groups.empty()) { + return dst; + } + std::string real_dst; + std::string::size_type pos{0}; + + const auto end = std::end(dst); + // We need to split the tunnel string and place each corresponding match on the + // configured one, so we need to first, get the match, then get the match number + // making sure that it does exist in the captured group. + for (auto c = std::begin(dst); c != end; c++, pos++) { + if (*c == '$') { + // find the next '.' so we can get the group number. + const auto dot = dst.find('.', pos); + std::string::size_type to = std::string::npos; + if (dot != std::string::npos) { + to = dot - (pos + 1); + } else { + // It may not have a dot, which could be because it's the last part. In that case + // we should check for the port separator. + if (const auto port = dst.find(':', pos); port != std::string::npos) { + to = (port - pos) - 1; + } + } + const auto &number_str = dst.substr(pos + 1, to); + if (!is_number(number_str)) { + // it may be some issue on the configured string, place the char and keep going. + real_dst += *c; + continue; + } + const std::size_t group_index = std::stoi(number_str); + if ((group_index - 1) < groups.size()) { + // place the captured group. + real_dst += groups[group_index - 1]; + // if it was the last match, then ... + if (dot == std::string::npos && to == std::string::npos) { + // that's it. + break; + } + pos += number_str.size() + 1; + std::advance(c, number_str.size() + 1); + } + // If there is no match for a specific group, then we keep the `$#` as defined in the string. + } + real_dst += *c; + } + + return real_dst; + } + std::string destination; - bool tunnel_decrypt = false; + + bool tunnel_decrypt; + bool need_fix; + bool tls_upstream; }; class VerifyClient : public ActionItem @@ -95,13 +199,43 @@ class VerifyClient : public ActionItem VerifyClient(uint8_t param) : mode(param) {} ~VerifyClient() override {} int - SNIAction(Continuation *cont) const override + SNIAction(Continuation *cont, const Context &ctx) const override { auto ssl_vc = dynamic_cast(cont); Debug("ssl_sni", "action verify param %d", this->mode); setClientCertLevel(ssl_vc->ssl, this->mode); return SSL_TLSEXT_ERR_OK; } + bool + TestClientSNIAction(const char *servername, const IpEndpoint &ep, int &policy) const override + { + // This action is triggered by a SNI if it was set + return true; + } +}; + +class HostSniPolicy : public ActionItem +{ + uint8_t policy; + +public: + HostSniPolicy(const char *param) : policy(atoi(param)) {} + HostSniPolicy(uint8_t param) : policy(param) {} + ~HostSniPolicy() override {} + int + SNIAction(Continuation *cont, const Context &ctx) const override + { + // On action this doesn't do anything + return SSL_TLSEXT_ERR_OK; + } + bool + TestClientSNIAction(const char *servername, const IpEndpoint &ep, int &in_policy) const override + { + // Update the policy when testing + in_policy = this->policy; + // But this action didn't really trigger during the action phase + return false; + } }; class TLSValidProtocols : public ActionItem @@ -118,7 +252,7 @@ class TLSValidProtocols : public ActionItem TLSValidProtocols() : protocol_mask(max_mask) {} TLSValidProtocols(unsigned long protocols) : unset(false), protocol_mask(protocols) {} int - SNIAction(Continuation *cont) const override + SNIAction(Continuation *cont, const Context &ctx) const override { if (!unset) { auto ssl_vc = dynamic_cast(cont); @@ -158,7 +292,7 @@ class SNI_IpAllow : public ActionItem } // end function SNI_IpAllow int - SNIAction(Continuation *cont) const override + SNIAction(Continuation *cont, const Context &ctx) const override { // i.e, ip filtering is not required if (ip_map.count() == 0) { @@ -178,4 +312,13 @@ class SNI_IpAllow : public ActionItem return SSL_TLSEXT_ERR_ALERT_FATAL; } } + bool + TestClientSNIAction(const char *servrername, const IpEndpoint &ep, int &policy) const override + { + bool retval = false; + if (ip_map.contains(ep)) { + retval = true; + } + return retval; + } }; diff --git a/iocore/net/P_SSLCertLookup.h b/iocore/net/P_SSLCertLookup.h index b36daf86ddf..7ee0f2ab54c 100644 --- a/iocore/net/P_SSLCertLookup.h +++ b/iocore/net/P_SSLCertLookup.h @@ -46,9 +46,11 @@ struct SSLMultiCertConfigParams { SSLMultiCertConfigParams() : opt(SSLCertContextOption::OPT_NONE) { REC_ReadConfigInt32(session_ticket_enabled, "proxy.config.ssl.server.session_ticket.enable"); + REC_ReadConfigInt32(session_ticket_number, "proxy.config.ssl.server.session_ticket.number"); } int session_ticket_enabled; ///< session ticket enabled + int session_ticket_number; ///< amount of session tickets to issue for new TLSv1.3 connections ats_scoped_str addr; ///< IPv[64] address to match ats_scoped_str cert; ///< certificate ats_scoped_str first_cert; ///< the first certificate name when multiple cert files are in 'ssl_cert_name' @@ -98,7 +100,7 @@ struct SSLCertContext { { } SSLCertContext(shared_SSL_CTX sc, shared_SSLMultiCertConfigParams u) - : ctx_mutex(), ctx(sc), opt(u->opt), userconfig(nullptr), keyblock(nullptr) + : ctx_mutex(), ctx(sc), opt(u->opt), userconfig(u), keyblock(nullptr) { } SSLCertContext(shared_SSL_CTX sc, shared_SSLMultiCertConfigParams u, shared_ssl_ticket_key_block kb) diff --git a/iocore/net/P_SSLConfig.h b/iocore/net/P_SSLConfig.h index 54a43dd2b4d..382cd079ee2 100644 --- a/iocore/net/P_SSLConfig.h +++ b/iocore/net/P_SSLConfig.h @@ -40,6 +40,8 @@ #include "SSLSessionCache.h" #include "YamlSNIConfig.h" +#include "P_SSLUtils.h" + struct SSLCertLookup; struct ssl_ticket_key_block; @@ -91,6 +93,7 @@ struct SSLConfigParams : public ConfigInfo { char *clientCACertPath; YamlSNIConfig::Policy verifyServerPolicy; YamlSNIConfig::Property verifyServerProperties; + bool tls_server_connection; int client_verify_depth; long ssl_ctx_options; long ssl_client_ctx_options; @@ -100,7 +103,12 @@ struct SSLConfigParams : public ConfigInfo { char *server_groups_list; char *client_groups_list; + static uint32_t server_max_early_data; + static uint32_t server_recv_max_early_data; + static bool server_allow_early_data_params; + static int ssl_maxrecord; + static int ssl_misc_max_iobuffer_size_index; static bool ssl_allow_client_renegotiation; static bool ssl_ocsp_enabled; diff --git a/iocore/net/P_SSLNetVConnection.h b/iocore/net/P_SSLNetVConnection.h index 0240ee7e908..c059e75b5f1 100644 --- a/iocore/net/P_SSLNetVConnection.h +++ b/iocore/net/P_SSLNetVConnection.h @@ -31,9 +31,13 @@ ****************************************************************************/ #pragma once +#include + #include "tscore/ink_platform.h" #include "ts/apidefs.h" #include +#include +#include #include #include @@ -43,6 +47,9 @@ #include "P_UnixNetVConnection.h" #include "P_UnixNet.h" #include "P_ALPNSupport.h" +#include "TLSSessionResumptionSupport.h" +#include "P_SSLUtils.h" +#include "P_SSLConfig.h" // These are included here because older OpenSSL libraries don't have them. // Don't copy these defines, or use their values directly, they are merely @@ -86,7 +93,7 @@ enum SSLHandshakeStatus { SSL_HANDSHAKE_ONGOING, SSL_HANDSHAKE_DONE, SSL_HANDSHA // A VConnection for a network socket. // ////////////////////////////////////////////////////////////////// -class SSLNetVConnection : public UnixNetVConnection, public ALPNSupport +class SSLNetVConnection : public UnixNetVConnection, public ALPNSupport, public TLSSessionResumptionSupport { typedef UnixNetVConnection super; ///< Parent type. @@ -124,18 +131,6 @@ class SSLNetVConnection : public UnixNetVConnection, public ALPNSupport sslHandshakeStatus = state; } - void - setSSLSessionCacheHit(bool state) - { - sslSessionCacheHit = state; - } - - bool - getSSLSessionCacheHit() const - { - return sslSessionCacheHit; - } - int sslServerHandShakeEvent(int &err); int sslClientHandShakeEvent(int &err); void net_read_io(NetHandler *nh, EThread *lthread) override; @@ -183,16 +178,12 @@ class SSLNetVConnection : public UnixNetVConnection, public ALPNSupport /// Reenable the VC after a pre-accept or SNI hook is called. virtual void reenable(NetHandler *nh, int event = TS_EVENT_CONTINUE); - /// Set the SSL context. - /// @note This must be called after the SSL endpoint has been created. - virtual bool sslContextSet(void *ctx); - int64_t read_raw_data(); void initialize_handshake_buffers() { - this->handShakeBuffer = new_MIOBuffer(); + this->handShakeBuffer = new_MIOBuffer(SSLConfigParams::ssl_misc_max_iobuffer_size_index); this->handShakeReader = this->handShakeBuffer->alloc_reader(); this->handShakeHolder = this->handShakeReader->clone(); this->handShakeBioStored = 0; @@ -308,13 +299,23 @@ class SSLNetVConnection : public UnixNetVConnection, public ALPNSupport if (!ssl) { return nullptr; } - - int curve_nid = SSL_get_shared_curve(ssl, 0); - - if (curve_nid == NID_undef) { + ssl_curve_id curve; + if (getSSLSessionCacheHit()) { + curve = getSSLCurveNID(); + } else { + curve = SSLGetCurveNID(ssl); + } +#ifndef OPENSSL_IS_BORINGSSL + if (curve == NID_undef) { return nullptr; } - return OBJ_nid2sn(curve_nid); + return OBJ_nid2sn(curve); +#else + if (curve == 0) { + return nullptr; + } + return SSL_get_curve_name(curve); +#endif } bool @@ -335,7 +336,7 @@ class SSLNetVConnection : public UnixNetVConnection, public ALPNSupport return tunnel_port; } - /* Returns true if this vc was configured for forward_route + /* Returns true if this vc was configured for forward_route or partial_blind_route */ bool decrypt_tunnel() @@ -343,8 +344,16 @@ class SSLNetVConnection : public UnixNetVConnection, public ALPNSupport return has_tunnel_destination() && tunnel_decrypt; } + /* Returns true if this vc was configured partial_blind_route + */ + bool + upstream_tls() + { + return has_tunnel_destination() && tls_upstream; + } + void - set_tunnel_destination(const std::string_view &destination, bool decrypt) + set_tunnel_destination(const std::string_view &destination, bool decrypt, bool upstream_tls) { auto pos = destination.find(":"); if (nullptr != tunnel_host) { @@ -358,6 +367,7 @@ class SSLNetVConnection : public UnixNetVConnection, public ALPNSupport tunnel_host = ats_strndup(destination.data(), destination.length()); } tunnel_decrypt = decrypt; + tls_upstream = upstream_tls; } int populate_protocol(std::string_view *results, int n) const override; @@ -375,11 +385,22 @@ class SSLNetVConnection : public UnixNetVConnection, public ALPNSupport ink_hrtime sslHandshakeEndTime = 0; ink_hrtime sslLastWriteTime = 0; int64_t sslTotalBytesSent = 0; - // The serverName is either a pointer to the name fetched from the - // SSL object or the empty string. Therefore, we do not allocate - // extra memory for this value. If plugins in the future can set the - // serverName value, this strategy will have to change. - const char *serverName = nullptr; + + // The serverName is either a pointer to the (null-terminated) name fetched from the + // SSL object or the empty string. + const char * + get_server_name() const override + { + return _serverName.get() ? _serverName.get() : ""; + } + + void set_server_name(std::string_view name); + + bool + support_sni() const override + { + return true; + } /// Set by asynchronous hooks to request a specific operation. SslVConnOp hookOpRequested = SSL_HOOK_OP_DEFAULT; @@ -391,6 +412,12 @@ class SSLNetVConnection : public UnixNetVConnection, public ALPNSupport bool protocol_mask_set = false; unsigned long protocol_mask; + // early data related stuff + bool early_data_finish = false; + MIOBuffer *early_data_buf = nullptr; + IOBufferReader *early_data_reader = nullptr; + int64_t read_from_early_data = 0; + // Only applies during the VERIFY certificate hooks (client and server side) // Means to give the plugin access to the data structure passed in during the underlying // openssl callback so the plugin can make more detailed decisions about the @@ -406,6 +433,47 @@ class SSLNetVConnection : public UnixNetVConnection, public ALPNSupport verify_cert = ctx; } + const char * + get_sni_servername() const override + { + return SSL_get_servername(this->ssl, TLSEXT_NAMETYPE_host_name); + } + + bool + peer_provided_cert() const override + { + X509 *cert = SSL_get_peer_certificate(this->ssl); + if (cert != nullptr) { + X509_free(cert); + return true; + } else { + return false; + } + } + + int + provided_cert() const override + { + if (this->get_context() == NET_VCONNECTION_OUT) { + return this->sent_cert; + } else { + return 1; + } + } + + void + set_sent_cert(int send_the_cert) + { + sent_cert = send_the_cert; + } + +protected: + const IpEndpoint & + _getLocalEndpoint() override + { + return local_addr; + } + private: std::string_view map_tls_protocol_to_tag(const char *proto_string) const; bool update_rbio(bool move_to_socket); @@ -413,7 +481,6 @@ class SSLNetVConnection : public UnixNetVConnection, public ALPNSupport enum SSLHandshakeStatus sslHandshakeStatus = SSL_HANDSHAKE_ONGOING; bool sslClientRenegotiationAbort = false; - bool sslSessionCacheHit = false; MIOBuffer *handShakeBuffer = nullptr; IOBufferReader *handShakeHolder = nullptr; IOBufferReader *handShakeReader = nullptr; @@ -421,6 +488,8 @@ class SSLNetVConnection : public UnixNetVConnection, public ALPNSupport bool transparentPassThrough = false; + int sent_cert = 0; + /// The current hook. /// @note For @C SSL_HOOKS_INVOKE, this is the hook to invoke. class APIHook *curHook = nullptr; @@ -445,7 +514,11 @@ class SSLNetVConnection : public UnixNetVConnection, public ALPNSupport char *tunnel_host = nullptr; in_port_t tunnel_port = 0; bool tunnel_decrypt = false; + bool tls_upstream = false; X509_STORE_CTX *verify_cert = nullptr; + + // Null-terminated string, or nullptr if there is no SNI server name. + std::unique_ptr _serverName; }; typedef int (SSLNetVConnection::*SSLNetVConnHandler)(int, void *); diff --git a/iocore/net/P_SSLSNI.h b/iocore/net/P_SSLSNI.h index 4cf71ff2a02..6e83c2f04cc 100644 --- a/iocore/net/P_SSLSNI.h +++ b/iocore/net/P_SSLSNI.h @@ -45,6 +45,7 @@ struct NextHopProperty { std::string client_key_file; // full path to client key file for lookup YamlSNIConfig::Policy verifyServerPolicy = YamlSNIConfig::Policy::UNSET; // whether to verify the next hop YamlSNIConfig::Property verifyServerProperties = YamlSNIConfig::Property::UNSET; // what to verify on the next hop + bool tls_upstream = false; // whether the upstream connection should be TLS SSL_CTX *ctx = nullptr; // ctx generated off the certificate to present to this server NextHopProperty() {} @@ -66,7 +67,7 @@ struct namedElement { } pos = 0; while ((pos = name.find('*', pos)) != std::string::npos) { - name.replace(pos, 1, ".{0,}"); + name.replace(pos, 1, "(.{0,})"); } Debug("ssl_sni", "Regexed fqdn=%s", name.c_str()); setRegexName(name); @@ -111,7 +112,7 @@ struct SNIConfigParams : public ConfigInfo { void cleanup(); int Initialize(); void loadSNIConfig(); - const actionVector *get(const std::string &servername) const; + std::pair get(const std::string &servername) const; }; struct SNIConfig { @@ -122,6 +123,8 @@ struct SNIConfig { typedef ConfigProcessor::scoped_config scoped_config; + static bool TestClientAction(const char *servername, const IpEndpoint &ep, int &enforcement_policy); + private: static int configid; }; diff --git a/iocore/net/P_SSLUtils.h b/iocore/net/P_SSLUtils.h index 19f6bc1008d..46553dd28c2 100644 --- a/iocore/net/P_SSLUtils.h +++ b/iocore/net/P_SSLUtils.h @@ -34,39 +34,63 @@ #include "records/I_RecCore.h" #include "P_SSLCertLookup.h" +#include +#include + struct SSLConfigParams; class SSLNetVConnection; typedef int ssl_error_t; +#ifndef OPENSSL_IS_BORING +typedef int ssl_curve_id; +#else +typedef uint16_t ssl_curve_id; +#endif + +// Return the SSL Curve ID associated to the specified SSL connection +ssl_curve_id SSLGetCurveNID(SSL *ssl); + /** @brief Load SSL certificates from ssl_multicert.config and setup SSLCertLookup for SSLCertificateConfig */ class SSLMultiCertConfigLoader { public: + struct CertLoadData { + std::vector cert_names_list, key_list, ca_list, ocsp_list; + }; SSLMultiCertConfigLoader(const SSLConfigParams *p) : _params(p) {} virtual ~SSLMultiCertConfigLoader(){}; bool load(SSLCertLookup *lookup); virtual SSL_CTX *default_server_ssl_ctx(); - virtual SSL_CTX *init_server_ssl_ctx(std::vector &certList, const SSLMultiCertConfigParams *sslMultCertSettings); - - static bool load_certs(SSL_CTX *ctx, std::vector &certList, const SSLConfigParams *params, - const SSLMultiCertConfigParams *ssl_multi_cert_params); + virtual SSL_CTX *init_server_ssl_ctx(CertLoadData const &data, const SSLMultiCertConfigParams *sslMultCertSettings, + std::set &names); + + static bool load_certs(SSL_CTX *ctx, CertLoadData const &data, const SSLConfigParams *params, + const SSLMultiCertConfigParams *sslMultCertSettings); + bool load_certs_and_cross_reference_names(std::vector &cert_list, CertLoadData &data, const SSLConfigParams *params, + const SSLMultiCertConfigParams *sslMultCertSettings, + std::set &common_names, + std::unordered_map> &unique_names); static bool set_session_id_context(SSL_CTX *ctx, const SSLConfigParams *params, const SSLMultiCertConfigParams *sslMultCertSettings); - static bool index_certificate(SSLCertLookup *lookup, SSLCertContext const &cc, X509 *cert, const char *certname); + static bool index_certificate(SSLCertLookup *lookup, SSLCertContext const &cc, const char *sni_name); static int check_server_cert_now(X509 *cert, const char *certname); static void clear_pw_references(SSL_CTX *ssl_ctx); protected: const SSLConfigParams *_params; + bool _store_single_ssl_ctx(SSLCertLookup *lookup, const shared_SSLMultiCertConfigParams &sslMultCertSettings, shared_SSL_CTX ctx, + std::set &names); + private: - virtual SSL_CTX *_store_ssl_ctx(SSLCertLookup *lookup, const shared_SSLMultiCertConfigParams ssl_multi_cert_params); + virtual const char *_debug_tag() const; + bool _store_ssl_ctx(SSLCertLookup *lookup, const shared_SSLMultiCertConfigParams &ssl_multi_cert_params); virtual void _set_handshake_callbacks(SSL_CTX *ctx); }; diff --git a/iocore/net/P_UDPConnection.h b/iocore/net/P_UDPConnection.h index ef69b31f68b..10e57c2d953 100644 --- a/iocore/net/P_UDPConnection.h +++ b/iocore/net/P_UDPConnection.h @@ -35,19 +35,17 @@ class UDPConnectionInternal : public UDPConnection { public: - UDPConnectionInternal(); + UDPConnectionInternal() = default; ~UDPConnectionInternal() override; Continuation *continuation = nullptr; - int recvActive = 0; // interested in receiving int refcount = 0; // public for assertion - SOCKET fd; - IpEndpoint binding; - int binding_valid = 0; - int tobedestroyed = 0; - int sendGenerationNum; - int64_t lastSentPktTSSeqNum; + SOCKET fd = -1; + IpEndpoint binding{}; + bool binding_valid = false; + int tobedestroyed = 0; + int sendGenerationNum = 0; // this is for doing packet scheduling: we keep two values so that we can // implement cancel. The first value tracks the startTime of the last @@ -55,20 +53,10 @@ class UDPConnectionInternal : public UDPConnection // startTime of the last packet when we are doing scheduling; whenever the // associated continuation cancels a packet, we rest lastPktStartTime to be // the same as the lastSentPktStartTime. - uint64_t lastSentPktStartTime; - uint64_t lastPktStartTime; + uint64_t lastSentPktStartTime = 0; + uint64_t lastPktStartTime = 0; }; -TS_INLINE -UDPConnectionInternal::UDPConnectionInternal() : fd(-1) -{ - sendGenerationNum = 0; - lastSentPktTSSeqNum = -1; - lastSentPktStartTime = 0; - lastPktStartTime = 0; - memset(&binding, 0, sizeof binding); -} - TS_INLINE UDPConnectionInternal::~UDPConnectionInternal() { @@ -87,7 +75,7 @@ UDPConnection::setBinding(struct sockaddr const *s) { UDPConnectionInternal *p = static_cast(this); ats_ip_copy(&p->binding, s); - p->binding_valid = 1; + p->binding_valid = true; } TS_INLINE void @@ -97,10 +85,10 @@ UDPConnection::setBinding(IpAddr const &ip, in_port_t port) IpEndpoint addr; addr.assign(ip, htons(port)); ats_ip_copy(&p->binding, addr); - p->binding_valid = 1; + p->binding_valid = true; } -TS_INLINE int +TS_INLINE bool UDPConnection::getBinding(struct sockaddr *s) { UDPConnectionInternal *p = static_cast(this); @@ -144,22 +132,6 @@ UDPConnection::getPortNum() return ats_ip_port_host_order(&static_cast(this)->binding); } -TS_INLINE int64_t -UDPConnection::cancel() -{ - UDPConnectionInternal *p = static_cast(this); - - p->sendGenerationNum++; - p->lastPktStartTime = p->lastSentPktStartTime; - return p->lastSentPktTSSeqNum; -} - -TS_INLINE void -UDPConnection::SetLastSentPktTSSeqNum(int64_t sentSeqNum) -{ - static_cast(this)->lastSentPktTSSeqNum = sentSeqNum; -} - TS_INLINE void UDPConnection::setContinuation(Continuation *c) { diff --git a/iocore/net/P_UDPPacket.h b/iocore/net/P_UDPPacket.h index d6473aa068f..a9f015b9af3 100644 --- a/iocore/net/P_UDPPacket.h +++ b/iocore/net/P_UDPPacket.h @@ -38,8 +38,6 @@ class UDPPacketInternal : public UDPPacket UDPPacketInternal(); ~UDPPacketInternal() override; - void append_block_internal(IOBufferBlock *block); - void free() override; SLINK(UDPPacketInternal, alink); // atomic link @@ -163,47 +161,6 @@ UDPPacket::getConnection() return static_cast(this)->conn; } -TS_INLINE UDPPacket * -new_UDPPacket(struct sockaddr const *to, ink_hrtime when, char *buf, int len) -{ - UDPPacketInternal *p = udpPacketAllocator.alloc(); - - p->in_the_priority_queue = 0; - p->in_heap = 0; - p->delivery_time = when; - ats_ip_copy(&p->to, to); - - if (buf) { - IOBufferBlock *body = new_IOBufferBlock(); - body->alloc(iobuffer_size_to_index(len)); - memcpy(body->end(), buf, len); - body->fill(len); - p->append_block(body); - } - - return p; -} - -TS_INLINE UDPPacket * -new_UDPPacket(struct sockaddr const *to, ink_hrtime when, IOBufferBlock *buf, int len) -{ - (void)len; - UDPPacketInternal *p = udpPacketAllocator.alloc(); - - p->in_the_priority_queue = 0; - p->in_heap = 0; - p->delivery_time = when; - ats_ip_copy(&p->to, to); - - while (buf) { - IOBufferBlock *body = buf->clone(); - p->append_block(body); - buf = buf->next.get(); - } - - return p; -} - TS_INLINE UDPPacket * new_UDPPacket(struct sockaddr const *to, ink_hrtime when, Ptr &buf) { @@ -218,32 +175,6 @@ new_UDPPacket(struct sockaddr const *to, ink_hrtime when, Ptr &bu return p; } -TS_INLINE UDPPacket * -new_UDPPacket(ink_hrtime when, Ptr buf) -{ - return new_UDPPacket(nullptr, when, buf); -} - -TS_INLINE UDPPacket * -new_incoming_UDPPacket(struct sockaddr *from, struct sockaddr *to, char *buf, int len) -{ - UDPPacketInternal *p = udpPacketAllocator.alloc(); - - p->in_the_priority_queue = 0; - p->in_heap = 0; - p->delivery_time = 0; - ats_ip_copy(&p->from, from); - ats_ip_copy(&p->to, to); - - IOBufferBlock *body = new_IOBufferBlock(); - body->alloc(iobuffer_size_to_index(len)); - memcpy(body->end(), buf, len); - body->fill(len); - p->append_block(body); - - return p; -} - TS_INLINE UDPPacket * new_incoming_UDPPacket(struct sockaddr *from, struct sockaddr *to, Ptr &block) { diff --git a/iocore/net/P_UnixNet.h b/iocore/net/P_UnixNet.h index a29cd7f4437..cd872194aa7 100644 --- a/iocore/net/P_UnixNet.h +++ b/iocore/net/P_UnixNet.h @@ -38,6 +38,9 @@ #define EVENTIO_ASYNC_SIGNAL 5 #if TS_USE_EPOLL +#ifndef EPOLLEXCLUSIVE +#define EPOLLEXCLUSIVE 0 +#endif #ifdef USE_EDGE_TRIGGER_EPOLL #define USE_EDGE_TRIGGER 1 #define EVENTIO_READ (EPOLLIN | EPOLLET) @@ -72,37 +75,62 @@ struct PollDescriptor; typedef PollDescriptor *EventLoop; -class UnixNetVConnection; +class NetEvent; class UnixUDPConnection; struct DNSConnection; struct NetAccept; + +/// Unified API for setting and clearing kernel and epoll events. struct EventIO { - int fd = -1; + int fd = -1; ///< file descriptor, often a system port #if TS_USE_KQUEUE || TS_USE_EPOLL && !defined(USE_EDGE_TRIGGER) || TS_USE_PORT - int events = 0; + int events = 0; ///< a bit mask of enabled events #endif - EventLoop event_loop = nullptr; - bool syscall = true; - int type = 0; + EventLoop event_loop = nullptr; ///< the assigned event loop + bool syscall = true; ///< if false, disable all functionality (for QUIC) + int type = 0; ///< class identifier of union data. union { Continuation *c; - UnixNetVConnection *vc; + NetEvent *ne; DNSConnection *dnscon; NetAccept *na; UnixUDPConnection *uc; - } data; + } data; ///< a kind of continuation + + /** The start methods all logically Setup a class to be called + when a file descriptor is available for read or write. + The type of the classes vary. Generally the file descriptor + is pulled from the class, but there is one option that lets + the file descriptor be expressed directly. + @param l the event loop + @param events a mask of flags (for details `man epoll_ctl`) + @return int the number of events created, -1 is error + */ int start(EventLoop l, DNSConnection *vc, int events); int start(EventLoop l, NetAccept *vc, int events); - int start(EventLoop l, UnixNetVConnection *vc, int events); + int start(EventLoop l, NetEvent *ne, int events); int start(EventLoop l, UnixUDPConnection *vc, int events); - int start(EventLoop l, int fd, Continuation *c, int events); - // Change the existing events by adding modify(EVENTIO_READ) - // or removing modify(-EVENTIO_READ), for level triggered I/O + int start(EventLoop l, int fd, NetEvent *ne, int events); + int start_common(EventLoop l, int fd, int events); + + /** Alter the events that will trigger the continuation, for level triggered I/O. + @param events add with positive mask(+EVENTIO_READ), or remove with negative mask (-EVENTIO_READ) + @return int the number of events created, -1 is error + */ int modify(int events); - // Refresh the existing events (i.e. KQUEUE EV_CLEAR), for edge triggered I/O + + /** Refresh the existing events (i.e. KQUEUE EV_CLEAR), for edge triggered I/O + @param events mask of events + @return int the number of events created, -1 is error + */ int refresh(int events); + + /// Remove the kernal or epoll event. Returns 0 on success. int stop(); + + /// Remove the epoll event and close the connection. Returns 0 on success. int close(); + EventIO() { data.c = nullptr; } }; @@ -115,7 +143,7 @@ struct EventIO { #include "P_UnixPollDescriptor.h" #include -class UnixNetVConnection; +class NetEvent; class NetHandler; typedef int (NetHandler::*NetContHandler)(int, void *); typedef unsigned int uint32; @@ -168,9 +196,9 @@ struct PollCont : public Continuation { }; /** - NetHandler is the processor of NetVC for the Net sub-system. The NetHandler + NetHandler is the processor of NetEvent for the Net sub-system. The NetHandler is the core component of the Net sub-system. Once started, it is responsible - for polling socket fds and perform the I/O tasks in NetVC. + for polling socket fds and perform the I/O tasks in NetEvent. The NetHandler is executed periodically to perform read/write tasks for NetVConnection. The NetHandler::mainNetEvent() should be viewed as a part of @@ -178,7 +206,7 @@ struct PollCont : public Continuation { By get_NetHandler(this_ethread()), you can get the NetHandler object that runs inside the current EThread and then @c startIO / @c stopIO which - assign/release a NetVC to/from NetHandler. Before you call these functions, + assign/release a NetEvent to/from NetHandler. Before you call these functions, holding the mutex of this NetHandler is required. The NetVConnection provides a set of do_io functions through which you can @@ -189,10 +217,11 @@ struct PollCont : public Continuation { Multi-thread scheduler: The NetHandler should be viewed as multi-threaded schedulers which process - NetVCs from their queues. The NetVC can be made of NetProcessor (allocate_vc) - either by directly adding a NetVC to the queue (NetHandler::startIO), or more + NetEvents from their queues. If vc wants to be managed by NetHandler, the vc + should be derived from NetEvent. The vc can be made of NetProcessor (allocate_vc) + either by directly adding a NetEvent to the queue (NetHandler::startIO), or more conveniently, calling a method service call (NetProcessor::connect_re) which - synthesizes the NetVC and places it in the queue. + synthesizes the NetEvent and places it in the queue. Callback event codes: @@ -208,14 +237,14 @@ struct PollCont : public Continuation { NetVConnection allocation policy: - NetVCs are allocated by the NetProcessor and deallocated by NetHandler. - A state machine may access the returned, non-recurring NetVC / VIO until - it is closed by do_io_close. For recurring NetVC, the NetVC may be - accessed until it is closed. Once the NetVC is closed, it's the + VCs are allocated by the NetProcessor and deallocated by NetHandler. + A state machine may access the returned, non-recurring NetEvent / VIO until + it is closed by do_io_close. For recurring NetEvent, the NetEvent may be + accessed until it is closed. Once the NetEvent is closed, it's the NetHandler's responsibility to deallocate it. Before assign to NetHandler or after release from NetHandler, it's the - NetVC's responsibility to deallocate itself. + NetEvent's responsibility to deallocate itself. */ @@ -233,21 +262,21 @@ class NetHandler : public Continuation, public EThread::LoopTailHandler // If we don't get rid of @a trigger_event we should remove @a thread. EThread *thread = nullptr; Event *trigger_event = nullptr; - QueM(UnixNetVConnection, NetState, read, ready_link) read_ready_list; - QueM(UnixNetVConnection, NetState, write, ready_link) write_ready_list; - Que(UnixNetVConnection, link) open_list; - DList(UnixNetVConnection, cop_link) cop_list; - ASLLM(UnixNetVConnection, NetState, read, enable_link) read_enable_list; - ASLLM(UnixNetVConnection, NetState, write, enable_link) write_enable_list; - Que(UnixNetVConnection, keep_alive_queue_link) keep_alive_queue; + QueM(NetEvent, NetState, read, ready_link) read_ready_list; + QueM(NetEvent, NetState, write, ready_link) write_ready_list; + Que(NetEvent, open_link) open_list; + DList(NetEvent, cop_link) cop_list; + ASLLM(NetEvent, NetState, read, enable_link) read_enable_list; + ASLLM(NetEvent, NetState, write, enable_link) write_enable_list; + Que(NetEvent, keep_alive_queue_link) keep_alive_queue; uint32_t keep_alive_queue_size = 0; - Que(UnixNetVConnection, active_queue_link) active_queue; + Que(NetEvent, active_queue_link) active_queue; uint32_t active_queue_size = 0; /// configuration settings for managing the active and keep-alive queues struct Config { uint32_t max_connections_in = 0; - uint32_t max_connections_active_in = 0; + uint32_t max_requests_in = 0; uint32_t inactive_threshold_in = 0; uint32_t transaction_no_activity_timeout_in = 0; uint32_t keep_alive_no_activity_timeout_in = 0; @@ -259,7 +288,11 @@ class NetHandler : public Continuation, public EThread::LoopTailHandler Making it a method means the knowledge of which member is the first one is localized to this struct, not scattered about. */ - uint32_t &operator[](int n) { return *(&max_connections_in + n); } + uint32_t & + operator[](int n) + { + return *(&max_connections_in + n); + } }; /** Static global config, set and updated per process. @@ -272,8 +305,8 @@ class NetHandler : public Continuation, public EThread::LoopTailHandler Config config; ///< Per thread copy of the @c global_config // Active and keep alive queue values that depend on other configuration values. // These are never updated directly, they are computed from other config values. - uint32_t max_connections_per_thread_in = 0; - uint32_t max_connections_active_per_thread_in = 0; + uint32_t max_connections_per_thread_in = 0; + uint32_t max_requests_per_thread_in = 0; /// Number of configuration items in @c Config. static constexpr int CONFIG_ITEM_COUNT = sizeof(Config) / sizeof(uint32_t); /// Which members of @c Config the per thread values depend on. @@ -289,11 +322,11 @@ class NetHandler : public Continuation, public EThread::LoopTailHandler void process_enabled_list(); void process_ready_list(); void manage_keep_alive_queue(); - bool manage_active_queue(bool ignore_queue_size); - void add_to_keep_alive_queue(UnixNetVConnection *vc); - void remove_from_keep_alive_queue(UnixNetVConnection *vc); - bool add_to_active_queue(UnixNetVConnection *vc); - void remove_from_active_queue(UnixNetVConnection *vc); + bool manage_active_queue(NetEvent *ne, bool ignore_queue_size); + void add_to_keep_alive_queue(NetEvent *ne); + void remove_from_keep_alive_queue(NetEvent *ne); + bool add_to_active_queue(NetEvent *ne); + void remove_from_active_queue(NetEvent *ne); /// Per process initialization logic. static void init_for_process(); @@ -301,58 +334,57 @@ class NetHandler : public Continuation, public EThread::LoopTailHandler void configure_per_thread_values(); /** - Start to handle read & write event on a UnixNetVConnection. - Initial the socket fd of netvc for polling system. + Start to handle read & write event on a NetEvent. + Initial the socket fd of ne for polling system. Only be called when holding the mutex of this NetHandler. - @param netvc UnixNetVConnection to be managed by this NetHandler. - @return 0 on success, netvc->nh set to this NetHandler. + @param ne NetEvent to be managed by this NetHandler. + @return 0 on success, ne->nh set to this NetHandler. -ERRNO on failure. */ - int startIO(UnixNetVConnection *netvc); + int startIO(NetEvent *ne); /** - Stop to handle read & write event on a UnixNetVConnection. - Remove the socket fd of netvc from polling system. - Only be called when holding the mutex of this NetHandler and must call stopCop(netvc) first. + Stop to handle read & write event on a NetEvent. + Remove the socket fd of ne from polling system. + Only be called when holding the mutex of this NetHandler and must call stopCop(ne) first. - @param netvc UnixNetVConnection to be released. - @return netvc->nh set to nullptr. + @param ne NetEvent to be released. + @return ne->nh set to nullptr. */ - void stopIO(UnixNetVConnection *netvc); + void stopIO(NetEvent *ne); /** - Start to handle active timeout and inactivity timeout on a UnixNetVConnection. - Put the netvc into open_list. All NetVCs in the open_list is checked for timeout by InactivityCop. - Only be called when holding the mutex of this NetHandler and must call startIO(netvc) first. + Start to handle active timeout and inactivity timeout on a NetEvent. + Put the ne into open_list. All NetEvents in the open_list is checked for timeout by InactivityCop. + Only be called when holding the mutex of this NetHandler and must call startIO(ne) first. - @param netvc UnixNetVConnection to be managed by InactivityCop + @param ne NetEvent to be managed by InactivityCop */ - void startCop(UnixNetVConnection *netvc); + void startCop(NetEvent *ne); /** - Stop to handle active timeout and inactivity on a UnixNetVConnection. - Remove the netvc from open_list and cop_list. - Also remove the netvc from keep_alive_queue and active_queue if its context is IN. + Stop to handle active timeout and inactivity on a NetEvent. + Remove the ne from open_list and cop_list. + Also remove the ne from keep_alive_queue and active_queue if its context is IN. Only be called when holding the mutex of this NetHandler. - @param netvc UnixNetVConnection to be released. + @param ne NetEvent to be released. */ - void stopCop(UnixNetVConnection *netvc); + void stopCop(NetEvent *ne); // Signal the epoll_wait to terminate. void signalActivity() override; /** - Release a netvc and free it. + Release a ne and free it. - @param netvc UnixNetVConnection to be detached. + @param ne NetEvent to be detached. */ - void free_netvc(UnixNetVConnection *netvc); + void free_netevent(NetEvent *ne); NetHandler(); private: - void _close_vc(UnixNetVConnection *vc, ink_hrtime now, int &handle_event, int &closed, int &total_idle_time, - int &total_idle_count); + void _close_ne(NetEvent *ne, ink_hrtime now, int &handle_event, int &closed, int &total_idle_time, int &total_idle_count); /// Static method used as the callback for runtime configuration updates. static int update_nethandler_config(const char *name, RecDataT, RecData data, void *); @@ -503,57 +535,82 @@ check_transient_accept_error(int res) } } -// -// Disable a UnixNetVConnection -// +/** Disable reading on the NetEvent @a ne. + @param nh Nethandler that owns @a ne. + @param ne The @c NetEvent to modify. + + - If write is already disable, also disable the inactivity timeout. + - clear read enabled flag. + - Remove the @c epoll READ flag. + - Take @a ne out of the read ready list. +*/ static inline void -read_disable(NetHandler *nh, UnixNetVConnection *vc) +read_disable(NetHandler *nh, NetEvent *ne) { - if (!vc->write.enabled) { - vc->set_inactivity_timeout(0); - Debug("socket", "read_disable updating inactivity_at %" PRId64 ", NetVC=%p", vc->next_inactivity_timeout_at, vc); + if (!ne->write.enabled) { + // Clear the next scheduled inactivity time, but don't clear inactivity_timeout_in, + // so the current timeout is used when the NetEvent is reenabled and not the default inactivity timeout + ne->next_inactivity_timeout_at = 0; + Debug("socket", "read_disable updating inactivity_at %" PRId64 ", NetEvent=%p", ne->next_inactivity_timeout_at, ne); } - vc->read.enabled = 0; - nh->read_ready_list.remove(vc); - vc->ep.modify(-EVENTIO_READ); + ne->read.enabled = 0; + nh->read_ready_list.remove(ne); + ne->ep.modify(-EVENTIO_READ); } +/** Disable writing on the NetEvent @a ne. + @param nh Nethandler that owns @a ne. + @param ne The @c NetEvent to modify. + + - If read is already disable, also disable the inactivity timeout. + - clear write enabled flag. + - Remove the @c epoll WRITE flag. + - Take @a ne out of the write ready list. +*/ static inline void -write_disable(NetHandler *nh, UnixNetVConnection *vc) +write_disable(NetHandler *nh, NetEvent *ne) { - if (!vc->read.enabled) { - vc->set_inactivity_timeout(0); - Debug("socket", "write_disable updating inactivity_at %" PRId64 ", NetVC=%p", vc->next_inactivity_timeout_at, vc); + if (!ne->read.enabled) { + // Clear the next scheduled inactivity time, but don't clear inactivity_timeout_in, + // so the current timeout is used when the NetEvent is reenabled and not the default inactivity timeout + ne->next_inactivity_timeout_at = 0; + Debug("socket", "write_disable updating inactivity_at %" PRId64 ", NetEvent=%p", ne->next_inactivity_timeout_at, ne); } - vc->write.enabled = 0; - nh->write_ready_list.remove(vc); - vc->ep.modify(-EVENTIO_WRITE); + ne->write.enabled = 0; + nh->write_ready_list.remove(ne); + ne->ep.modify(-EVENTIO_WRITE); } TS_INLINE int EventIO::start(EventLoop l, DNSConnection *vc, int events) { - type = EVENTIO_DNS_CONNECTION; - return start(l, vc->fd, (Continuation *)vc, events); + type = EVENTIO_DNS_CONNECTION; + data.dnscon = vc; + return start_common(l, vc->fd, events); } TS_INLINE int EventIO::start(EventLoop l, NetAccept *vc, int events) { - type = EVENTIO_NETACCEPT; - return start(l, vc->server.fd, (Continuation *)vc, events); + type = EVENTIO_NETACCEPT; + data.na = vc; + return start_common(l, vc->server.fd, events); } TS_INLINE int -EventIO::start(EventLoop l, UnixNetVConnection *vc, int events) +EventIO::start(EventLoop l, NetEvent *ne, int events) { - type = EVENTIO_READWRITE_VC; - return start(l, vc->con.fd, (Continuation *)vc, events); + type = EVENTIO_READWRITE_VC; + data.ne = ne; + return start_common(l, ne->get_fd(), events); } + TS_INLINE int EventIO::start(EventLoop l, UnixUDPConnection *vc, int events) { - type = EVENTIO_UDP_CONNECTION; - return start(l, vc->fd, (Continuation *)vc, events); + type = EVENTIO_UDP_CONNECTION; + data.uc = vc; + return start_common(l, vc->fd, events); } + TS_INLINE int EventIO::close() { @@ -573,26 +630,32 @@ EventIO::close() return data.na->server.close(); break; case EVENTIO_READWRITE_VC: - return data.vc->con.close(); + return data.ne->close(); break; } return -1; } TS_INLINE int -EventIO::start(EventLoop l, int afd, Continuation *c, int e) +EventIO::start(EventLoop l, int afd, NetEvent *ne, int e) +{ + data.ne = ne; + return start_common(l, afd, e); +} + +TS_INLINE int +EventIO::start_common(EventLoop l, int afd, int e) { if (!this->syscall) { return 0; } - data.c = c; fd = afd; event_loop = l; #if TS_USE_EPOLL struct epoll_event ev; memset(&ev, 0, sizeof(ev)); - ev.events = e; + ev.events = e | EPOLLEXCLUSIVE; ev.data.ptr = this; #ifndef USE_EDGE_TRIGGER events = e; @@ -772,14 +835,14 @@ EventIO::stop() } TS_INLINE int -NetHandler::startIO(UnixNetVConnection *netvc) +NetHandler::startIO(NetEvent *ne) { ink_assert(this->mutex->thread_holding == this_ethread()); - ink_assert(netvc->thread == this_ethread()); + ink_assert(ne->get_thread() == this_ethread()); int res = 0; PollDescriptor *pd = get_PollDescriptor(this->thread); - if (netvc->ep.start(pd, netvc, EVENTIO_READ | EVENTIO_WRITE) < 0) { + if (ne->ep.start(pd, ne, EVENTIO_READ | EVENTIO_WRITE) < 0) { res = errno; // EEXIST should be ok, though it should have been cleared before we got back here if (errno != EEXIST) { @@ -788,51 +851,51 @@ NetHandler::startIO(UnixNetVConnection *netvc) } } - if (netvc->read.triggered == 1) { - read_ready_list.enqueue(netvc); + if (ne->read.triggered == 1) { + read_ready_list.enqueue(ne); } - netvc->nh = this; + ne->nh = this; return res; } TS_INLINE void -NetHandler::stopIO(UnixNetVConnection *netvc) +NetHandler::stopIO(NetEvent *ne) { - ink_release_assert(netvc->nh == this); + ink_release_assert(ne->nh == this); - netvc->ep.stop(); + ne->ep.stop(); - read_ready_list.remove(netvc); - write_ready_list.remove(netvc); - if (netvc->read.in_enabled_list) { - read_enable_list.remove(netvc); - netvc->read.in_enabled_list = 0; + read_ready_list.remove(ne); + write_ready_list.remove(ne); + if (ne->read.in_enabled_list) { + read_enable_list.remove(ne); + ne->read.in_enabled_list = 0; } - if (netvc->write.in_enabled_list) { - write_enable_list.remove(netvc); - netvc->write.in_enabled_list = 0; + if (ne->write.in_enabled_list) { + write_enable_list.remove(ne); + ne->write.in_enabled_list = 0; } - netvc->nh = nullptr; + ne->nh = nullptr; } TS_INLINE void -NetHandler::startCop(UnixNetVConnection *netvc) +NetHandler::startCop(NetEvent *ne) { ink_assert(this->mutex->thread_holding == this_ethread()); - ink_release_assert(netvc->nh == this); - ink_assert(!open_list.in(netvc)); + ink_release_assert(ne->nh == this); + ink_assert(!open_list.in(ne)); - open_list.enqueue(netvc); + open_list.enqueue(ne); } TS_INLINE void -NetHandler::stopCop(UnixNetVConnection *netvc) +NetHandler::stopCop(NetEvent *ne) { - ink_release_assert(netvc->nh == this); + ink_release_assert(ne->nh == this); - open_list.remove(netvc); - cop_list.remove(netvc); - remove_from_keep_alive_queue(netvc); - remove_from_active_queue(netvc); + open_list.remove(ne); + cop_list.remove(ne); + remove_from_keep_alive_queue(ne); + remove_from_active_queue(ne); } diff --git a/iocore/net/P_UnixNetState.h b/iocore/net/P_UnixNetState.h index 3a2265c92c9..46fddbc7164 100644 --- a/iocore/net/P_UnixNetState.h +++ b/iocore/net/P_UnixNetState.h @@ -40,13 +40,13 @@ #include "I_VIO.h" class Event; -class UnixNetVConnection; +class NetEvent; struct NetState { int enabled = 0; VIO vio; - Link ready_link; - SLink enable_link; + Link ready_link; + SLink enable_link; int in_enabled_list = 0; int triggered = 0; diff --git a/iocore/net/P_UnixNetVConnection.h b/iocore/net/P_UnixNetVConnection.h index 128f5723e2a..cd941782932 100644 --- a/iocore/net/P_UnixNetVConnection.h +++ b/iocore/net/P_UnixNetVConnection.h @@ -36,6 +36,7 @@ #include "P_UnixNetState.h" #include "P_Connection.h" #include "P_NetAccept.h" +#include "NetEvent.h" class UnixNetVConnection; class NetHandler; @@ -68,9 +69,9 @@ NetVCOptions::reset() sni_servername = nullptr; ssl_servername = nullptr; + sni_hostname = nullptr; ssl_client_cert_name = nullptr; ssl_client_private_key_name = nullptr; - ssl_client_ca_cert_name = nullptr; } inline void @@ -103,7 +104,7 @@ struct OOB_callback : public Continuation { enum tcp_congestion_control_t { CLIENT_SIDE, SERVER_SIDE }; -class UnixNetVConnection : public NetVConnection +class UnixNetVConnection : public NetVConnection, public NetEvent { public: int64_t outstanding() override; @@ -137,6 +138,12 @@ class UnixNetVConnection : public NetVConnection return false; } + const char * + get_server_name() const override + { + return nullptr; + } + void do_io_close(int lerrno = -1) override; void do_io_shutdown(ShutdownHowTo_t howto) override; @@ -154,6 +161,8 @@ class UnixNetVConnection : public NetVConnection //////////////////////////////////////////////////////////// virtual void set_active_timeout(ink_hrtime timeout_in) override; virtual void set_inactivity_timeout(ink_hrtime timeout_in) override; + virtual void set_default_inactivity_timeout(ink_hrtime timeout_in) override; + virtual bool is_default_inactivity_timeout() override; virtual void cancel_active_timeout() override; virtual void cancel_inactivity_timeout() override; void set_action(Continuation *c) override; @@ -215,7 +224,45 @@ class UnixNetVConnection : public NetVConnection return false; } - virtual void net_read_io(NetHandler *nh, EThread *lthread); + // NetEvent + virtual void net_read_io(NetHandler *nh, EThread *lthread) override; + virtual void net_write_io(NetHandler *nh, EThread *lthread) override; + virtual void free(EThread *t) override; + virtual int + close() override + { + return this->con.close(); + } + virtual int + get_fd() override + { + return this->con.fd; + } + + virtual EThread * + get_thread() override + { + return this->thread; + } + + virtual int + callback(int event = CONTINUATION_EVENT_NONE, void *data = nullptr) override + { + return this->handleEvent(event, data); + } + + virtual Ptr & + get_mutex() override + { + return this->mutex; + } + + virtual ContFlags & + get_control_flags() override + { + return this->control_flags; + } + virtual int64_t load_buffer_and_write(int64_t towrite, MIOBufferAccessor &buf, int64_t &total_written, int &needs); void readDisable(NetHandler *nh); void readSignalError(NetHandler *nh, int err); @@ -232,40 +279,11 @@ class UnixNetVConnection : public NetVConnection UnixNetVConnection *migrateToCurrentThread(Continuation *c, EThread *t); Action action_; - int closed = 0; - NetState read; - NetState write; - - LINK(UnixNetVConnection, cop_link); - LINKM(UnixNetVConnection, read, ready_link) - SLINKM(UnixNetVConnection, read, enable_link) - LINKM(UnixNetVConnection, write, ready_link) - SLINKM(UnixNetVConnection, write, enable_link) - LINK(UnixNetVConnection, keep_alive_queue_link); - LINK(UnixNetVConnection, active_queue_link); - - ink_hrtime inactivity_timeout_in = 0; - ink_hrtime active_timeout_in = 0; - ink_hrtime next_inactivity_timeout_at = 0; - ink_hrtime next_activity_timeout_at = 0; - - EventIO ep; - NetHandler *nh = nullptr; - unsigned int id = 0; - union { - unsigned int flags; -#define NET_VC_SHUTDOWN_READ 1 -#define NET_VC_SHUTDOWN_WRITE 2 - struct { - unsigned int got_local_addr : 1; - unsigned int shutdown : 2; - } f; - }; + unsigned int id = 0; Connection con; int recursion = 0; - ink_hrtime submit_time = 0; OOB_callback *oob_ptr = nullptr; bool from_accept_thread = false; NetAccept *accept_object = nullptr; @@ -286,7 +304,6 @@ class UnixNetVConnection : public NetVConnection */ virtual int populate(Connection &con, Continuation *c, void *arg); virtual void clear(); - virtual void free(EThread *t); ink_hrtime get_inactivity_timeout() override; ink_hrtime get_active_timeout() override; diff --git a/iocore/net/P_UnixUDPConnection.h b/iocore/net/P_UnixUDPConnection.h index 9e3c2f2a804..62b47f6dd92 100644 --- a/iocore/net/P_UnixUDPConnection.h +++ b/iocore/net/P_UnixUDPConnection.h @@ -97,9 +97,8 @@ UDPConnection::recv(Continuation *c) // register callback interest. p->continuation = c; ink_assert(c != nullptr); - mutex = c->mutex; - p->recvActive = 1; - return ACTION_RESULT_NONE; + mutex = c->mutex; + return nullptr; } TS_INLINE UDPConnection * diff --git a/iocore/net/ProxyProtocol.h b/iocore/net/ProxyProtocol.h index 607dcdfbd30..f5961af4e6a 100644 --- a/iocore/net/ProxyProtocol.h +++ b/iocore/net/ProxyProtocol.h @@ -23,8 +23,7 @@ limitations under the License. */ -#ifndef ProxyProtocol_H_ -#define ProxyProtocol_H_ +#pragma once #include "tscore/ink_defs.h" #include "tscore/ink_memory.h" @@ -51,5 +50,3 @@ const size_t PROXY_V2_CONNECTION_HEADER_LEN_MIN = 16; const size_t PROXY_V1_CONNECTION_HEADER_LEN_MAX = 108; const size_t PROXY_V2_CONNECTION_HEADER_LEN_MAX = 16; - -#endif /* ProxyProtocol_H_ */ diff --git a/iocore/net/QUICMultiCertConfigLoader.cc b/iocore/net/QUICMultiCertConfigLoader.cc index 7e37f3427dc..e2285f25f3b 100644 --- a/iocore/net/QUICMultiCertConfigLoader.cc +++ b/iocore/net/QUICMultiCertConfigLoader.cc @@ -29,6 +29,7 @@ #include "QUICConfig.h" #include "QUICConnection.h" #include "QUICTypes.h" +#include "tscore/Filenames.h" // #include "QUICGlobals.h" #define QUICConfDebug(fmt, ...) Debug("quic_conf", fmt, ##__VA_ARGS__) @@ -79,7 +80,8 @@ QUICMultiCertConfigLoader::default_server_ssl_ctx() } SSL_CTX * -QUICMultiCertConfigLoader::init_server_ssl_ctx(std::vector &cert_list, const SSLMultiCertConfigParams *multi_cert_params) +QUICMultiCertConfigLoader::init_server_ssl_ctx(SSLMultiCertConfigLoader::CertLoadData const &data, + const SSLMultiCertConfigParams *multi_cert_params, std::set &names) { const SSLConfigParams *params = this->_params; @@ -91,7 +93,7 @@ QUICMultiCertConfigLoader::init_server_ssl_ctx(std::vector &cert_list, c } if (multi_cert_params->cert) { - if (!SSLMultiCertConfigLoader::load_certs(ctx, cert_list, params, multi_cert_params)) { + if (!SSLMultiCertConfigLoader::load_certs(ctx, data, params, multi_cert_params)) { goto fail; } } @@ -128,7 +130,7 @@ QUICMultiCertConfigLoader::init_server_ssl_ctx(std::vector &cert_list, c #if TS_USE_TLS_SET_CIPHERSUITES if (params->server_tls13_cipher_suites != nullptr) { if (!SSL_CTX_set_ciphersuites(ctx, params->server_tls13_cipher_suites)) { - Error("invalid tls server cipher suites in records.config"); + Error("invalid tls server cipher suites in %s", ts::filename::RECORDS); goto fail; } } @@ -141,7 +143,7 @@ QUICMultiCertConfigLoader::init_server_ssl_ctx(std::vector &cert_list, c #else if (!SSL_CTX_set1_curves_list(ctx, params->server_groups_list)) { #endif - Error("invalid groups list for server in records.config"); + Error("invalid groups list for server in %s", ts::filename::RECORDS); goto fail; } } @@ -151,26 +153,6 @@ QUICMultiCertConfigLoader::init_server_ssl_ctx(std::vector &cert_list, c SSL_CTX_set_alpn_select_cb(ctx, QUICMultiCertConfigLoader::ssl_select_next_protocol, nullptr); -#if TS_USE_TLS_OCSP - if (SSLConfigParams::ssl_ocsp_enabled) { - QUICConfDebug("SSL OCSP Stapling is enabled"); - SSL_CTX_set_tlsext_status_cb(ctx, ssl_callback_ocsp_stapling); - const char *cert_name = multi_cert_params ? multi_cert_params->cert.get() : nullptr; - - for (auto cert : cert_list) { - if (!ssl_stapling_init_cert(ctx, cert, cert_name, nullptr)) { - Warning("failed to configure SSL_CTX for OCSP Stapling info for certificate at %s", cert_name); - } - } - } else { - QUICConfDebug("SSL OCSP Stapling is disabled"); - } -#else - if (SSLConfigParams::ssl_ocsp_enabled) { - Warning("failed to enable SSL OCSP Stapling; this version of OpenSSL does not support it"); - } -#endif /* TS_USE_TLS_OCSP */ - if (SSLConfigParams::init_ssl_ctx_cb) { SSLConfigParams::init_ssl_ctx_cb(ctx, true); } @@ -179,87 +161,9 @@ QUICMultiCertConfigLoader::init_server_ssl_ctx(std::vector &cert_list, c fail: SSLReleaseContext(ctx); - for (auto cert : cert_list) { - X509_free(cert); - } - return nullptr; } -SSL_CTX * -QUICMultiCertConfigLoader::_store_ssl_ctx(SSLCertLookup *lookup, const shared_SSLMultiCertConfigParams multi_cert_params) -{ - std::vector cert_list; - shared_SSL_CTX ctx(this->init_server_ssl_ctx(cert_list, multi_cert_params.get()), SSL_CTX_free); - shared_ssl_ticket_key_block keyblock = nullptr; - bool inserted = false; - - if (!ctx || !multi_cert_params) { - lookup->is_valid = false; - return nullptr; - } - - const char *certname = multi_cert_params->cert.get(); - for (auto cert : cert_list) { - if (0 > SSLMultiCertConfigLoader::check_server_cert_now(cert, certname)) { - /* At this point, we know cert is bad, and we've already printed a - descriptive reason as to why cert is bad to the log file */ - QUICConfDebug("Marking certificate as NOT VALID: %s", certname); - lookup->is_valid = false; - } - } - - // Index this certificate by the specified IP(v6) address. If the address is "*", make it the default context. - if (multi_cert_params->addr) { - if (strcmp(multi_cert_params->addr, "*") == 0) { - if (lookup->insert(multi_cert_params->addr, SSLCertContext(ctx, multi_cert_params, keyblock)) >= 0) { - inserted = true; - lookup->ssl_default = ctx; - this->_set_handshake_callbacks(ctx.get()); - } - } else { - IpEndpoint ep; - - if (ats_ip_pton(multi_cert_params->addr, &ep) == 0) { - QUICConfDebug("mapping '%s' to certificate %s", (const char *)multi_cert_params->addr, (const char *)certname); - if (lookup->insert(ep, SSLCertContext(ctx, multi_cert_params, keyblock)) >= 0) { - inserted = true; - } - } else { - Error("'%s' is not a valid IPv4 or IPv6 address", (const char *)multi_cert_params->addr); - lookup->is_valid = false; - } - } - } - - // Insert additional mappings. Note that this maps multiple keys to the same value, so when - // this code is updated to reconfigure the SSL certificates, it will need some sort of - // refcounting or alternate way of avoiding double frees. - QUICConfDebug("importing SNI names from %s", (const char *)certname); - for (auto cert : cert_list) { - if (SSLMultiCertConfigLoader::index_certificate(lookup, SSLCertContext(ctx, multi_cert_params), cert, certname)) { - inserted = true; - } - } - - if (inserted) { - if (SSLConfigParams::init_ssl_ctx_cb) { - SSLConfigParams::init_ssl_ctx_cb(ctx.get(), true); - } - } - - if (!inserted) { - SSLReleaseContext(ctx.get()); - ctx = nullptr; - } - - for (auto &i : cert_list) { - X509_free(i); - } - - return ctx.get(); -} - void QUICMultiCertConfigLoader::_set_handshake_callbacks(SSL_CTX *ssl_ctx) { @@ -342,3 +246,9 @@ QUICMultiCertConfigLoader::ssl_cert_cb(SSL *ssl, void * /*arg*/) return 1; } + +const char * +QUICMultiCertConfigLoader::_debug_tag() const +{ + return "quic"; +} diff --git a/iocore/net/QUICMultiCertConfigLoader.h b/iocore/net/QUICMultiCertConfigLoader.h index f29bda6ff25..01f8e0bebdd 100644 --- a/iocore/net/QUICMultiCertConfigLoader.h +++ b/iocore/net/QUICMultiCertConfigLoader.h @@ -46,10 +46,12 @@ class QUICMultiCertConfigLoader : public SSLMultiCertConfigLoader QUICMultiCertConfigLoader(const SSLConfigParams *p) : SSLMultiCertConfigLoader(p) {} virtual SSL_CTX *default_server_ssl_ctx() override; - virtual SSL_CTX *init_server_ssl_ctx(std::vector &cert_list, const SSLMultiCertConfigParams *multi_cert_params) override; + // override; + SSL_CTX *init_server_ssl_ctx(SSLMultiCertConfigLoader::CertLoadData const &data, + const SSLMultiCertConfigParams *sslMultCertSettings, std::set &names) override; private: - virtual SSL_CTX *_store_ssl_ctx(SSLCertLookup *lookup, const shared_SSLMultiCertConfigParams multi_cert_params) override; + const char *_debug_tag() const override; virtual void _set_handshake_callbacks(SSL_CTX *ssl_ctx) override; static int ssl_select_next_protocol(SSL *ssl, const unsigned char **out, unsigned char *outlen, const unsigned char *in, unsigned inlen, void *); diff --git a/iocore/net/QUICNet.cc b/iocore/net/QUICNet.cc index d3df166be11..31d33d9ea96 100644 --- a/iocore/net/QUICNet.cc +++ b/iocore/net/QUICNet.cc @@ -22,6 +22,7 @@ */ #include "P_Net.h" +#include "P_QUICNet.h" #include "quic/QUICEvents.h" ClassAllocator quicPollEventAllocator("quicPollEvent"); @@ -65,10 +66,10 @@ QUICPollCont::_process_long_header_packet(QUICPollEvent *e, NetHandler *nh) UDPPacketInternal *p = e->packet; // FIXME: VC is nullptr ? QUICNetVConnection *vc = static_cast(e->con); - uint8_t *buf = (uint8_t *)p->getIOBlockChain()->buf(); + uint8_t *buf = reinterpret_cast(p->getIOBlockChain()->buf()); QUICPacketType ptype; - QUICPacketLongHeader::type(ptype, buf, 1); + QUICLongHeaderPacketR::type(ptype, buf, 1); if (ptype == QUICPacketType::INITIAL && !vc->read.triggered) { SCOPED_MUTEX_LOCK(lock, vc->mutex, this_ethread()); vc->read.triggered = 1; @@ -146,7 +147,7 @@ QUICPollCont::pollEvent(int, Event *) } while ((e = result.pop())) { - buf = (uint8_t *)e->packet->getIOBlockChain()->buf(); + buf = reinterpret_cast(e->packet->getIOBlockChain()->buf()); if (QUICInvariants::is_long_header(buf)) { // Long Header Packet with Connection ID, has a valid type value. this->_process_long_header_packet(e, nh); @@ -165,7 +166,7 @@ initialize_thread_for_quic_net(EThread *thread) NetHandler *nh = get_NetHandler(thread); QUICPollCont *quicpc = get_QUICPollCont(thread); - new ((ink_dummy_for_new *)quicpc) QUICPollCont(thread->mutex, nh); + new (reinterpret_cast(quicpc)) QUICPollCont(thread->mutex, nh); thread->schedule_every(quicpc, -HRTIME_MSECONDS(UDP_PERIOD)); } diff --git a/iocore/net/QUICNetProcessor.cc b/iocore/net/QUICNetProcessor.cc index c14e000230c..5bdaa2b8dc7 100644 --- a/iocore/net/QUICNetProcessor.cc +++ b/iocore/net/QUICNetProcessor.cc @@ -25,9 +25,13 @@ #include "P_Net.h" #include "records/I_RecHttp.h" +#include "P_QUICNetProcessor.h" +#include "P_QUICNet.h" +#include "P_QUICPacketHandler.h" #include "QUICGlobals.h" #include "QUICConfig.h" #include "QUICMultiCertConfigLoader.h" +#include "QUICResetTokenTable.h" // // Global Data @@ -39,7 +43,7 @@ QUICNetProcessor::QUICNetProcessor() {} QUICNetProcessor::~QUICNetProcessor() { - // TODO: clear all values before destory the table. + // TODO: clear all values before destroy the table. delete this->_ctable; } @@ -76,8 +80,9 @@ QUICNetProcessor::createNetAccept(const NetProcessor::AcceptOptions &opt) if (this->_ctable == nullptr) { QUICConfig::scoped_config params; this->_ctable = new QUICConnectionTable(params->connection_table_size()); + this->_rtable = new QUICResetTokenTable(); } - return (NetAccept *)new QUICPacketHandlerIn(opt, *this->_ctable); + return (NetAccept *)new QUICPacketHandlerIn(opt, *this->_ctable, *this->_rtable); } NetVConnection * @@ -125,7 +130,8 @@ QUICNetProcessor::connect_re(Continuation *cont, sockaddr const *remote_addr, Ne UnixUDPConnection *con = new UnixUDPConnection(fd); Debug("quic_ps", "con=%p fd=%d", con, fd); - QUICPacketHandlerOut *packet_handler = new QUICPacketHandlerOut(); + this->_rtable = new QUICResetTokenTable(); + QUICPacketHandlerOut *packet_handler = new QUICPacketHandlerOut(*this->_rtable); if (opt->local_ip.isValid()) { con->setBinding(opt->local_ip, opt->local_port); } @@ -144,7 +150,7 @@ QUICNetProcessor::connect_re(Continuation *cont, sockaddr const *remote_addr, Ne QUICConnectionId client_dst_cid; client_dst_cid.randomize(); // vc->init set handler of vc `QUICNetVConnection::startEvent` - vc->init(client_dst_cid, client_dst_cid, con, packet_handler); + vc->init(QUIC_SUPPORTED_VERSIONS[0], client_dst_cid, client_dst_cid, con, packet_handler, this->_rtable); packet_handler->init(vc); // Connection ID will be changed @@ -215,7 +221,7 @@ QUICNetProcessor::main_accept(Continuation *cont, SOCKET fd, AcceptOptions const na->init_accept(); SCOPED_MUTEX_LOCK(lock, na->mutex, this_ethread()); - udpNet.UDPBind((Continuation *)na, &na->server.accept_addr.sa, 1048576, 1048576); + udpNet.UDPBind((Continuation *)na, &na->server.accept_addr.sa, fd, 1048576, 1048576); return na->action_.get(); } diff --git a/iocore/net/QUICNetVConnection.cc b/iocore/net/QUICNetVConnection.cc index 7070727e72b..9b2b33d2225 100644 --- a/iocore/net/QUICNetVConnection.cc +++ b/iocore/net/QUICNetVConnection.cc @@ -27,6 +27,8 @@ #include "records/I_RecHttp.h" #include "tscore/Diags.h" +#include "P_QUICNetVConnection.h" +#include "P_QUICPacketHandler.h" #include "P_Net.h" #include "InkAPIInternal.h" // Added to include the quic_hook definitions #include "Log.h" @@ -35,22 +37,29 @@ #include "QUICMultiCertConfigLoader.h" #include "QUICTLS.h" +#include "QUICNewRenoCongestionController.h" + #include "QUICStats.h" #include "QUICGlobals.h" #include "QUICDebugNames.h" #include "QUICEvents.h" +#include "QUICHandshake.h" #include "QUICConfig.h" #include "QUICIntUtil.h" using namespace std::literals; static constexpr std::string_view QUIC_DEBUG_TAG = "quic_net"sv; +static constexpr uint16_t QUANTUM_TEST_ID = 3127; +static constexpr uint8_t QUANTUM_TEST_VALUE[1200] = {'Q'}; + #define QUICConDebug(fmt, ...) Debug(QUIC_DEBUG_TAG.data(), "[%s] " fmt, this->cids().data(), ##__VA_ARGS__) #define QUICConVDebug(fmt, ...) Debug("v_quic_net", "[%s] " fmt, this->cids().data(), ##__VA_ARGS__) #define QUICConVVVDebug(fmt, ...) Debug("vvv_quic_net", "[%s] " fmt, this->cids().data(), ##__VA_ARGS__) #define QUICFCDebug(fmt, ...) Debug("quic_flow_ctrl", "[%s] " fmt, this->cids().data(), ##__VA_ARGS__) +#define QUICFCVDebug(fmt, ...) Debug("v_quic_flow_ctrl", "[%s] " fmt, this->cids().data(), ##__VA_ARGS__) #define QUICError(fmt, ...) \ Debug("quic_net", "[%s] " fmt, this->cids().data(), ##__VA_ARGS__); \ @@ -64,7 +73,7 @@ static constexpr uint32_t MINIMUM_INITIAL_PACKET_SIZE = 1200; static constexpr ink_hrtime WRITE_READY_INTERVAL = HRTIME_MSECONDS(2); static constexpr uint32_t PACKET_PER_EVENT = 256; static constexpr uint32_t MAX_CONSECUTIVE_STREAMS = 8; ///< Interrupt sending STREAM frames to send ACK frame -static constexpr uint32_t MIN_PKT_PAYLOAD_LEN = 3; ///< Minimum payload length for sampling for header protection +// static constexpr uint32_t MIN_PKT_PAYLOAD_LEN = 3; ///< Minimum payload length for sampling for header protection static constexpr uint32_t STATE_CLOSING_MAX_SEND_PKT_NUM = 8; ///< Max number of sending packets which contain a closing frame. static constexpr uint32_t STATE_CLOSING_MAX_RECV_PKT_WIND = 1 << STATE_CLOSING_MAX_SEND_PKT_NUM; @@ -178,81 +187,43 @@ class QUICTPConfigQCP : public QUICTPConfig } } -private: - const QUICConfigParams *_params; - NetVConnectionContext_t _ctx; -}; - -class QUICCCConfigQCP : public QUICCCConfig -{ -public: - QUICCCConfigQCP(const QUICConfigParams *params) : _params(params) {} - - uint32_t - max_datagram_size() const override - { - return this->_params->cc_max_datagram_size(); - } - - uint32_t - initial_window() const override + uint8_t + active_cid_limit() const override { - return this->_params->cc_initial_window(); + if (this->_ctx == NET_VCONNECTION_IN) { + return this->_params->active_cid_limit_in(); + } else { + return this->_params->active_cid_limit_out(); + } } - uint32_t - minimum_window() const override + bool + disable_active_migration() const override { - return this->_params->cc_minimum_window(); + if (this->_ctx == NET_VCONNECTION_IN) { + return this->_params->disable_active_migration(); + } else { + return false; + } } - float - loss_reduction_factor() const override + std::unordered_map> + additional_tp() const override { - return this->_params->cc_loss_reduction_factor(); + return this->_additional_tp; } - uint32_t - persistent_congestion_threshold() const override + void + add_tp(uint16_t id, const uint8_t *value, uint16_t length) { - return this->_params->cc_persistent_congestion_threshold(); + this->_additional_tp.emplace(std::piecewise_construct, std::forward_as_tuple(id), std::forward_as_tuple(value, length)); } private: const QUICConfigParams *_params; -}; - -class QUICLDConfigQCP : public QUICLDConfig -{ -public: - QUICLDConfigQCP(const QUICConfigParams *params) : _params(params) {} - - uint32_t - packet_threshold() const override - { - return this->_params->ld_packet_threshold(); - } - - float - time_threshold() const override - { - return this->_params->ld_time_threshold(); - } - - ink_hrtime - granularity() const override - { - return this->_params->ld_granularity(); - } - - ink_hrtime - initial_rtt() const override - { - return this->_params->ld_initial_rtt(); - } + NetVConnectionContext_t _ctx; -private: - const QUICConfigParams *_params; + std::unordered_map> _additional_tp; }; QUICNetVConnection::QUICNetVConnection() : _packet_factory(this->_pp_key_info), _ph_protector(this->_pp_key_info) {} @@ -263,63 +234,86 @@ QUICNetVConnection::~QUICNetVConnection() this->_unschedule_packet_write_ready(); this->_unschedule_closing_timeout(); this->_unschedule_closed_event(); - this->_unschedule_path_validation_timeout(); } // XXX This might be called on ET_UDP thread // Initialize QUICNetVC for out going connection (NET_VCONNECTION_OUT) void -QUICNetVConnection::init(QUICConnectionId peer_cid, QUICConnectionId original_cid, UDPConnection *udp_con, - QUICPacketHandler *packet_handler) +QUICNetVConnection::init(QUICVersion version, QUICConnectionId peer_cid, QUICConnectionId original_cid, UDPConnection *udp_con, + QUICPacketHandler *packet_handler, QUICResetTokenTable *rtable) { SET_HANDLER((NetVConnHandler)&QUICNetVConnection::startEvent); + this->_initial_version = version; this->_udp_con = udp_con; this->_packet_handler = packet_handler; this->_peer_quic_connection_id = peer_cid; this->_original_quic_connection_id = original_cid; + this->_rtable = rtable; this->_quic_connection_id.randomize(); + this->_initial_source_connection_id = this->_quic_connection_id; this->_update_cids(); if (is_debug_tag_set(QUIC_DEBUG_TAG.data())) { - char dcid_hex_str[QUICConnectionId::MAX_HEX_STR_LENGTH]; - char scid_hex_str[QUICConnectionId::MAX_HEX_STR_LENGTH]; - this->_peer_quic_connection_id.hex(dcid_hex_str, QUICConnectionId::MAX_HEX_STR_LENGTH); - this->_quic_connection_id.hex(scid_hex_str, QUICConnectionId::MAX_HEX_STR_LENGTH); - - QUICConDebug("dcid=%s scid=%s", dcid_hex_str, scid_hex_str); + QUICConDebug("dcid=%s scid=%s", this->_peer_quic_connection_id.hex().c_str(), this->_quic_connection_id.hex().c_str()); } + + this->_init_submodules(); } // Initialize QUICNetVC for in coming connection (NET_VCONNECTION_IN) void -QUICNetVConnection::init(QUICConnectionId peer_cid, QUICConnectionId original_cid, QUICConnectionId first_cid, - UDPConnection *udp_con, QUICPacketHandler *packet_handler, QUICConnectionTable *ctable) +QUICNetVConnection::init(QUICVersion version, QUICConnectionId peer_cid, QUICConnectionId original_cid, QUICConnectionId first_cid, + QUICConnectionId retry_cid, UDPConnection *udp_con, QUICPacketHandler *packet_handler, + QUICResetTokenTable *rtable, QUICConnectionTable *ctable) { SET_HANDLER((NetVConnHandler)&QUICNetVConnection::acceptEvent); + this->_initial_version = version; this->_udp_con = udp_con; this->_packet_handler = packet_handler; this->_peer_quic_connection_id = peer_cid; this->_original_quic_connection_id = original_cid; this->_first_quic_connection_id = first_cid; + this->_retry_source_connection_id = retry_cid; this->_quic_connection_id.randomize(); + this->_initial_source_connection_id = this->_quic_connection_id; if (ctable) { this->_ctable = ctable; this->_ctable->insert(this->_quic_connection_id, this); this->_ctable->insert(this->_original_quic_connection_id, this); } + this->_rtable = rtable; this->_update_cids(); if (is_debug_tag_set(QUIC_DEBUG_TAG.data())) { - char dcid_hex_str[QUICConnectionId::MAX_HEX_STR_LENGTH]; - char scid_hex_str[QUICConnectionId::MAX_HEX_STR_LENGTH]; - this->_peer_quic_connection_id.hex(dcid_hex_str, QUICConnectionId::MAX_HEX_STR_LENGTH); - this->_quic_connection_id.hex(scid_hex_str, QUICConnectionId::MAX_HEX_STR_LENGTH); - - QUICConDebug("dcid=%s scid=%s", dcid_hex_str, scid_hex_str); + QUICConDebug("dcid=%s scid=%s", this->_peer_quic_connection_id.hex().c_str(), this->_quic_connection_id.hex().c_str()); } + + this->_init_submodules(); +} + +void +QUICNetVConnection::_init_submodules() +{ + this->_pinger = new QUICPinger(); + this->_padder = new QUICPadder(this->netvc_context); + this->_path_validator = new QUICPathValidator(*this, [this](bool succeeded) { + if (succeeded) { + this->_alt_con_manager->drop_cid(this->_peer_old_quic_connection_id); + // FIXME This is a kind of workaround for connection migration. + // This PING make peer to send an ACK frame so that ATS can detect packet loss. + // It would be better if QUICLossDetector could detect the loss in another way. + this->ping(); + } + }); + this->_path_manager = new QUICPathManagerImpl(*this, *this->_path_validator); + this->_context = std::make_unique(&this->_rtt_measure, this, &this->_pp_key_info, this->_path_manager); + this->_congestion_controller = new QUICNewRenoCongestionController(*_context); + this->_rtt_measure.init(this->_context->ld_config()); + this->_loss_detector = + new QUICLossDetector(*_context, this->_congestion_controller, &this->_rtt_measure, this->_pinger, this->_padder); } bool @@ -419,64 +413,80 @@ void QUICNetVConnection::start() { ink_release_assert(this->thread != nullptr); - this->_five_tuple.update(this->local_addr, this->remote_addr, SOCK_DGRAM); + QUICPath trusted_path = {{}, {}}; // Version 0x00000001 uses stream 0 for cryptographic handshake with TLS 1.3, but newer version may not if (this->direction() == NET_VCONNECTION_IN) { QUICCertConfig::scoped_config server_cert; this->_pp_key_info.set_context(QUICPacketProtectionKeyInfo::Context::SERVER); this->_ack_frame_manager.set_ack_delay_exponent(this->_quic_config->ack_delay_exponent_in()); - this->_reset_token = QUICStatelessResetToken(this->_quic_connection_id, this->_quic_config->instance_id()); - this->_hs_protocol = this->_setup_handshake_protocol(server_cert->ssl_default); - this->_handshake_handler = - new QUICHandshake(this, this->_hs_protocol, this->_reset_token, this->_quic_config->stateless_retry()); + this->_reset_token = QUICStatelessResetToken(this->_quic_connection_id, this->_quic_config->instance_id()); + this->_hs_protocol = this->_setup_handshake_protocol(server_cert->ssl_default); + this->_handshake_handler = new QUICHandshake(this->_initial_version, this, this->_hs_protocol, this->_reset_token, + this->_quic_config->stateless_retry()); this->_ack_frame_manager.set_max_ack_delay(this->_quic_config->max_ack_delay_in()); this->_schedule_ack_manager_periodic(this->_quic_config->max_ack_delay_in()); } else { + trusted_path = {this->local_addr, this->remote_addr}; QUICTPConfigQCP tp_config(this->_quic_config, NET_VCONNECTION_OUT); + if (this->_quic_config->quantum_readiness_test_enabled_out()) { + tp_config.add_tp(QUANTUM_TEST_ID, QUANTUM_TEST_VALUE, sizeof(QUANTUM_TEST_VALUE)); + } this->_pp_key_info.set_context(QUICPacketProtectionKeyInfo::Context::CLIENT); this->_ack_frame_manager.set_ack_delay_exponent(this->_quic_config->ack_delay_exponent_out()); this->_hs_protocol = this->_setup_handshake_protocol(this->_quic_config->client_ssl_ctx()); - this->_handshake_handler = new QUICHandshake(this, this->_hs_protocol); + this->_handshake_handler = new QUICHandshake(this->_initial_version, this, this->_hs_protocol); this->_handshake_handler->start(tp_config, &this->_packet_factory, this->_quic_config->vn_exercise_enabled()); this->_handshake_handler->do_handshake(); this->_ack_frame_manager.set_max_ack_delay(this->_quic_config->max_ack_delay_out()); this->_schedule_ack_manager_periodic(this->_quic_config->max_ack_delay_out()); } + this->_path_manager->set_trusted_path(trusted_path); this->_application_map = new QUICApplicationMap(); this->_frame_dispatcher = new QUICFrameDispatcher(this); // Create frame handlers - QUICCCConfigQCP cc_config(this->_quic_config); - QUICLDConfigQCP ld_config(this->_quic_config); - this->_rtt_measure.init(ld_config); - this->_congestion_controller = new QUICCongestionController(this->_rtt_measure, this, cc_config); - this->_loss_detector = new QUICLossDetector(this, this->_congestion_controller, &this->_rtt_measure, ld_config); this->_frame_dispatcher->add_handler(this->_loss_detector); this->_remote_flow_controller = new QUICRemoteConnectionFlowController(UINT64_MAX); this->_local_flow_controller = new QUICLocalConnectionFlowController(&this->_rtt_measure, UINT64_MAX); - this->_path_validator = new QUICPathValidator(); - this->_stream_manager = new QUICStreamManager(this, &this->_rtt_measure, this->_application_map); + this->_stream_manager = new QUICStreamManager(this->_context.get(), this->_application_map); + this->_token_creator = new QUICTokenCreator(this->_context.get()); + + static constexpr int QUIC_STREAM_MANAGER_WEIGHT = QUICFrameGeneratorWeight::AFTER_DATA - 1; + static constexpr int QUIC_PINGER_WEIGHT = QUICFrameGeneratorWeight::LATE + 1; + static constexpr int QUIC_PADDER_WEIGHT = QUICFrameGeneratorWeight::LATE + 2; // Register frame generators - this->_frame_generators.push_back(this->_handshake_handler); // CRYPTO - this->_frame_generators.push_back(this->_path_validator); // PATH_CHALLENGE, PATH_RESPOSNE - this->_frame_generators.push_back(this->_local_flow_controller); // MAX_DATA - this->_frame_generators.push_back(this->_remote_flow_controller); // DATA_BLOCKED - this->_frame_generators.push_back(this); // NEW_TOKEN - this->_frame_generators.push_back(this->_stream_manager); // STREAM, MAX_STREAM_DATA, STREAM_DATA_BLOCKED - this->_frame_generators.push_back(&this->_ack_frame_manager); // ACK - this->_frame_generators.push_back(&this->_pinger); // PING + this->_frame_generators.add_generator(*this->_handshake_handler, QUICFrameGeneratorWeight::EARLY); // CRYPTO + this->_frame_generators.add_generator(*this->_path_validator, QUICFrameGeneratorWeight::EARLY); // PATH_CHALLENGE, PATH_RESPOSNE + this->_frame_generators.add_generator(*this->_local_flow_controller, QUICFrameGeneratorWeight::BEFORE_DATA); // MAX_DATA + this->_frame_generators.add_generator(*this->_remote_flow_controller, QUICFrameGeneratorWeight::BEFORE_DATA); // DATA_BLOCKED + this->_frame_generators.add_generator(*this->_token_creator, QUICFrameGeneratorWeight::BEFORE_DATA); // NEW_TOKEN + this->_frame_generators.add_generator(*this->_stream_manager, + QUIC_STREAM_MANAGER_WEIGHT); // STREAM, MAX_STREAM_DATA, STREAM_DATA_BLOCKED + this->_frame_generators.add_generator(this->_ack_frame_manager, QUICFrameGeneratorWeight::BEFORE_DATA); // ACK + + this->_frame_generators.add_generator(*this->_pinger, QUIC_PINGER_WEIGHT); // PING + // Warning: padder should be tail of the frame generators + this->_frame_generators.add_generator(*this->_padder, QUIC_PADDER_WEIGHT); // PADDING // Register frame handlers this->_frame_dispatcher->add_handler(this); this->_frame_dispatcher->add_handler(this->_stream_manager); this->_frame_dispatcher->add_handler(this->_path_validator); this->_frame_dispatcher->add_handler(this->_handshake_handler); + + // regist qlog + if (this->_context->config()->qlog_dir() != nullptr) { + this->_qlog = std::make_unique(*this->_context, this->_original_quic_connection_id.hex()); + this->_qlog->last_trace().set_vantage_point( + {"ats", QLog::Trace::VantagePointType::server, QLog::Trace::VantagePointType::server}); + this->_context->regist_callback(this->_qlog); + } } void @@ -503,6 +513,7 @@ QUICNetVConnection::free(EThread *t) super::clear(); */ + this->_context->trigger(QUICContext::CallbackEvent::CONNECTION_CLOSE); ALPNSupport::clear(); this->_packet_handler->close_connection(this); } @@ -532,7 +543,7 @@ void QUICNetVConnection::destroy(EThread *t) { QUICConDebug("Destroy connection"); - /* TODO: Uncmment these blocks after refactoring read / write process + /* TODO: Uncomment these blocks after refactoring read / write process if (from_accept_thread) { quicNetVCAllocator.free(this); } else { @@ -541,6 +552,13 @@ QUICNetVConnection::destroy(EThread *t) */ } +void +QUICNetVConnection::set_local_addr() +{ + int local_sa_size = sizeof(local_addr); + ATS_UNUSED_RETURN(safe_getsockname(this->_udp_con->getFd(), &local_addr.sa, &local_sa_size)); +} + void QUICNetVConnection::reenable(VIO *vio) { @@ -566,6 +584,9 @@ QUICNetVConnection::connectUp(EThread *t, int fd) // FIXME: complete do_io_xxxx instead this->read.enabled = 1; + this->set_local_addr(); + this->remote_addr = con.addr; + this->start(); // start QUIC handshake @@ -592,6 +613,18 @@ QUICNetVConnection::first_connection_id() const return this->_first_quic_connection_id; } +QUICConnectionId +QUICNetVConnection::retry_source_connection_id() const +{ + return this->_retry_source_connection_id; +} + +QUICConnectionId +QUICNetVConnection::initial_source_connection_id() const +{ + return this->_initial_source_connection_id; +} + QUICConnectionId QUICNetVConnection::connection_id() const { @@ -667,16 +700,17 @@ void QUICNetVConnection::handle_received_packet(UDPPacket *packet) { this->_packet_recv_queue.enqueue(packet); + this->_loss_detector->on_datagram_received(); } void QUICNetVConnection::ping() { - this->_pinger.request(QUICEncryptionLevel::ONE_RTT); + this->_pinger->request(QUICEncryptionLevel::ONE_RTT); } void -QUICNetVConnection::close(QUICConnectionErrorUPtr error) +QUICNetVConnection::close_quic_connection(QUICConnectionErrorUPtr error) { if (this->handler == reinterpret_cast(&QUICNetVConnection::state_connection_closed) || this->handler == reinterpret_cast(&QUICNetVConnection::state_connection_closing)) { @@ -686,6 +720,24 @@ QUICNetVConnection::close(QUICConnectionErrorUPtr error) } } +void +QUICNetVConnection::reset_quic_connection() +{ + this->_switch_to_close_state(); + + QUICStatelessResetToken token(this->connection_id(), this->_quic_config->instance_id()); + auto packet = QUICPacketFactory::create_stateless_reset_packet(token, 128); + if (packet) { + Ptr udp_payload(new_IOBufferBlock()); + udp_payload->alloc(iobuffer_size_to_index(128, BUFFER_SIZE_INDEX_32K)); + uint8_t *buf = reinterpret_cast(udp_payload->end()); + size_t len = 0; + packet->store(buf, &len); + udp_payload->fill(len); + this->_packet_handler->send_packet(this, udp_payload); + } +} + std::vector QUICNetVConnection::interests() { @@ -776,14 +828,15 @@ QUICNetVConnection::state_handshake(int event, Event *data) QUICPacketCreationResult result; net_activity(this, this_ethread()); do { - QUICPacketUPtr packet = this->_dequeue_recv_packet(result); + uint8_t packet_buf[QUICPacket::MAX_INSTANCE_SIZE]; + QUICPacketUPtr packet = this->_dequeue_recv_packet(packet_buf, result); if (result == QUICPacketCreationResult::NOT_READY) { error = nullptr; } else if (result == QUICPacketCreationResult::FAILED) { // Don't make this error, and discard the packet. // Because: // - Attacker can terminate connections - // - It could be just an errora on lower layer + // - It could be just an error on lower layer error = nullptr; } else if (result == QUICPacketCreationResult::SUCCESS || result == QUICPacketCreationResult::UNSUPPORTED) { error = this->_state_handshake_process_packet(*packet); @@ -798,6 +851,9 @@ QUICNetVConnection::state_handshake(int event, Event *data) } while (error == nullptr && (result == QUICPacketCreationResult::SUCCESS || result == QUICPacketCreationResult::IGNORED)); break; } + case QUIC_EVENT_STATELESS_RESET: + this->_switch_to_draining_state(std::make_unique(QUICTransErrorCode::NO_ERROR, "Stateless Reset")); + break; case QUIC_EVENT_ACK_PERIODIC: this->_handle_periodic_ack_event(); break; @@ -808,13 +864,14 @@ QUICNetVConnection::state_handshake(int event, Event *data) // Reschedule WRITE_READY this->_schedule_packet_write_ready(true); break; - case QUIC_EVENT_PATH_VALIDATION_TIMEOUT: - this->_handle_path_validation_timeout(data); - break; case VC_EVENT_INACTIVITY_TIMEOUT: // Start Immediate Close because of Idle Timeout this->_handle_idle_timeout(); break; + case VC_EVENT_ACTIVE_TIMEOUT: + // Start Immediate Close + this->_handle_active_timeout(); + break; default: QUICConDebug("Unexpected event: %s (%d)", QUICDebugNames::quic_event(event), event); } @@ -844,13 +901,17 @@ QUICNetVConnection::state_connection_established(int event, Event *data) // Reschedule WRITE_READY this->_schedule_packet_write_ready(true); break; - case QUIC_EVENT_PATH_VALIDATION_TIMEOUT: - this->_handle_path_validation_timeout(data); + case QUIC_EVENT_STATELESS_RESET: + this->_switch_to_draining_state(std::make_unique(QUICTransErrorCode::NO_ERROR, "Stateless Reset")); break; case VC_EVENT_INACTIVITY_TIMEOUT: // Start Immediate Close because of Idle Timeout this->_handle_idle_timeout(); break; + case VC_EVENT_ACTIVE_TIMEOUT: + // Start Immediate Close + this->_handle_active_timeout(); + break; default: QUICConDebug("Unexpected event: %s (%d)", QUICDebugNames::quic_event(event), event); } @@ -877,13 +938,15 @@ QUICNetVConnection::state_connection_closing(int event, Event *data) this->_close_packet_write_ready(data); this->_state_closing_send_packet(); break; - case QUIC_EVENT_PATH_VALIDATION_TIMEOUT: - this->_handle_path_validation_timeout(data); - break; case QUIC_EVENT_CLOSING_TIMEOUT: this->_close_closing_timeout(data); this->_switch_to_close_state(); break; + case QUIC_EVENT_STATELESS_RESET: + break; + case VC_EVENT_ACTIVE_TIMEOUT: + // Do nothing because closing is in progress + break; case QUIC_EVENT_ACK_PERIODIC: default: QUICConDebug("Unexpected event: %s (%d)", QUICDebugNames::quic_event(event), event); @@ -908,13 +971,15 @@ QUICNetVConnection::state_connection_draining(int event, Event *data) // This should be the only difference between this and closing_state. this->_close_packet_write_ready(data); break; - case QUIC_EVENT_PATH_VALIDATION_TIMEOUT: - this->_handle_path_validation_timeout(data); - break; case QUIC_EVENT_CLOSING_TIMEOUT: this->_close_closing_timeout(data); this->_switch_to_close_state(); break; + case QUIC_EVENT_STATELESS_RESET: + break; + case VC_EVENT_ACTIVE_TIMEOUT: + // Do nothing because closing is in progress + break; case QUIC_EVENT_ACK_PERIODIC: default: QUICConDebug("Unexpected event: %s (%d)", QUICDebugNames::quic_event(event), event); @@ -933,7 +998,6 @@ QUICNetVConnection::state_connection_closed(int event, Event *data) this->_unschedule_ack_manager_periodic(); this->_unschedule_packet_write_ready(); this->_unschedule_closing_timeout(); - this->_unschedule_path_validation_timeout(); this->_close_closed_event(data); this->next_inactivity_timeout_at = 0; this->next_activity_timeout_at = 0; @@ -949,7 +1013,7 @@ QUICNetVConnection::state_connection_closed(int event, Event *data) // FIXME I'm not sure whether we can block here, but it's needed to not crash. SCOPED_MUTEX_LOCK(lock, this->nh->mutex, this_ethread()); if (this->nh) { - this->nh->free_netvc(this); + this->nh->free_netevent(this); } else { this->free(this->mutex->thread_holding); } @@ -1026,7 +1090,7 @@ QUICNetVConnection::select_next_protocol(SSL *ssl, const unsigned char **out, un if (this->getNPN(&npnptr, &npnsize)) { // SSL_select_next_proto chooses the first server-offered protocol that appears in the clients protocol set, ie. the // server selects the protocol. This is a n^2 search, so it's preferable to keep the protocol set short. - if (SSL_select_next_proto((unsigned char **)out, outlen, npnptr, npnsize, in, inlen) == OPENSSL_NPN_NEGOTIATED) { + if (SSL_select_next_proto(const_cast(out), outlen, npnptr, npnsize, in, inlen) == OPENSSL_NPN_NEGOTIATED) { Debug("ssl", "selected ALPN protocol %.*s", (int)(*outlen), *out); return SSL_TLSEXT_ERR_OK; } @@ -1043,6 +1107,44 @@ QUICNetVConnection::is_closed() const return this->handler == reinterpret_cast(&QUICNetVConnection::state_connection_closed); } +bool +QUICNetVConnection::is_at_anti_amplification_limit() const +{ + return !this->_verified_state.is_verified() || this->_verified_state.windows() == 0; +} + +bool +QUICNetVConnection::is_address_validation_completed() const +{ + return this->_verified_state.is_verified(); +} + +bool +QUICNetVConnection::is_handshake_completed() const +{ + return this->_handshake_completed; +} + +bool +QUICNetVConnection::has_keys_for(QUICPacketNumberSpace space) const +{ + switch (space) { + case QUICPacketNumberSpace::INITIAL: + return this->_pp_key_info.is_decryption_key_available(QUICKeyPhase::INITIAL) && + this->_pp_key_info.is_encryption_key_available(QUICKeyPhase::INITIAL); + case QUICPacketNumberSpace::HANDSHAKE: + return this->_pp_key_info.is_decryption_key_available(QUICKeyPhase::HANDSHAKE) && + this->_pp_key_info.is_encryption_key_available(QUICKeyPhase::HANDSHAKE); + case QUICPacketNumberSpace::APPLICATION_DATA: + return (this->_pp_key_info.is_decryption_key_available(QUICKeyPhase::PHASE_0) && + this->_pp_key_info.is_encryption_key_available(QUICKeyPhase::PHASE_0)) || + (this->_pp_key_info.is_decryption_key_available(QUICKeyPhase::PHASE_1) && + this->_pp_key_info.is_encryption_key_available(QUICKeyPhase::PHASE_1)); + default: + return false; + } +} + QUICPacketNumber QUICNetVConnection::_largest_acked_packet_number(QUICEncryptionLevel level) const { @@ -1051,6 +1153,12 @@ QUICNetVConnection::_largest_acked_packet_number(QUICEncryptionLevel level) cons return this->_loss_detector->largest_acked_packet_number(index); } +QUICVersion +QUICNetVConnection::negotiated_version() const +{ + return this->_handshake_handler->negotiated_version(); +} + std::string_view QUICNetVConnection::negotiated_application_name() const { @@ -1068,28 +1176,29 @@ QUICNetVConnection::_state_handshake_process_packet(const QUICPacket &packet) QUICConnectionErrorUPtr error = nullptr; switch (packet.type()) { case QUICPacketType::VERSION_NEGOTIATION: - error = this->_state_handshake_process_version_negotiation_packet(packet); + error = this->_state_handshake_process_version_negotiation_packet(static_cast(packet)); break; case QUICPacketType::INITIAL: - error = this->_state_handshake_process_initial_packet(packet); + error = this->_state_handshake_process_initial_packet(static_cast(packet)); break; case QUICPacketType::RETRY: - error = this->_state_handshake_process_retry_packet(packet); + error = this->_state_handshake_process_retry_packet(static_cast(packet)); break; case QUICPacketType::HANDSHAKE: - error = this->_state_handshake_process_handshake_packet(packet); + error = this->_state_handshake_process_handshake_packet(static_cast(packet)); if (this->_pp_key_info.is_decryption_key_available(QUICKeyPhase::INITIAL) && this->netvc_context == NET_VCONNECTION_IN) { this->_pp_key_info.drop_keys(QUICKeyPhase::INITIAL); + this->_loss_detector->on_packet_number_space_discarded(QUICPacketNumberSpace::INITIAL); this->_minimum_encryption_level = QUICEncryptionLevel::HANDSHAKE; } break; case QUICPacketType::ZERO_RTT_PROTECTED: - error = this->_state_handshake_process_zero_rtt_protected_packet(packet); + error = this->_state_handshake_process_zero_rtt_protected_packet(static_cast(packet)); break; case QUICPacketType::PROTECTED: - default: QUICConDebug("Ignore %s(%" PRIu8 ") packet", QUICDebugNames::packet_type(packet.type()), static_cast(packet.type())); - + break; + default: error = std::make_unique(QUICTransErrorCode::INTERNAL_ERROR); break; } @@ -1097,7 +1206,7 @@ QUICNetVConnection::_state_handshake_process_packet(const QUICPacket &packet) } QUICConnectionErrorUPtr -QUICNetVConnection::_state_handshake_process_version_negotiation_packet(const QUICPacket &packet) +QUICNetVConnection::_state_handshake_process_version_negotiation_packet(const QUICVersionNegotiationPacketR &packet) { QUICConnectionErrorUPtr error = nullptr; @@ -1126,7 +1235,7 @@ QUICNetVConnection::_state_handshake_process_version_negotiation_packet(const QU } QUICConnectionErrorUPtr -QUICNetVConnection::_state_handshake_process_initial_packet(const QUICPacket &packet) +QUICNetVConnection::_state_handshake_process_initial_packet(const QUICInitialPacketR &packet) { // QUIC packet could be smaller than MINIMUM_INITIAL_PACKET_SIZE when coalescing packets // if (packet->size() < MINIMUM_INITIAL_PACKET_SIZE) { @@ -1139,26 +1248,43 @@ QUICNetVConnection::_state_handshake_process_initial_packet(const QUICPacket &pa // Start handshake if (this->netvc_context == NET_VCONNECTION_IN) { + this->local_addr.sa = packet.to().sa; + this->remote_addr.sa = packet.from().sa; + this->_five_tuple.update(this->local_addr, this->remote_addr, SOCK_DGRAM); + if (!this->_alt_con_manager) { this->_alt_con_manager = - new QUICAltConnectionManager(this, *this->_ctable, this->_peer_quic_connection_id, this->_quic_config->instance_id(), - this->_quic_config->num_alt_connection_ids(), this->_quic_config->preferred_address_ipv4(), - this->_quic_config->preferred_address_ipv6()); - this->_frame_generators.push_back(this->_alt_con_manager); + new QUICAltConnectionManager(this, *this->_ctable, *this->_rtable, this->_peer_quic_connection_id, + this->_quic_config->instance_id(), this->_quic_config->active_cid_limit_in(), + this->_quic_config->preferred_address_ipv4(), this->_quic_config->preferred_address_ipv6()); + this->_frame_generators.add_generator(*this->_alt_con_manager, QUICFrameGeneratorWeight::EARLY); this->_frame_dispatcher->add_handler(this->_alt_con_manager); } QUICTPConfigQCP tp_config(this->_quic_config, NET_VCONNECTION_IN); + if (this->_quic_config->quantum_readiness_test_enabled_in()) { + tp_config.add_tp(QUANTUM_TEST_ID, QUANTUM_TEST_VALUE, sizeof(QUANTUM_TEST_VALUE)); + } error = this->_handshake_handler->start(tp_config, packet, &this->_packet_factory, this->_alt_con_manager->preferred_address()); // If version negotiation was failed and VERSION NEGOTIATION packet was sent, nothing to do. if (this->_handshake_handler->is_version_negotiated()) { error = this->_recv_and_ack(packet); - if (error == nullptr && !this->_handshake_handler->has_remote_tp()) { + if (error == nullptr && this->_handshake_handler->is_completed() && !this->_handshake_handler->has_remote_tp()) { error = std::make_unique(QUICTransErrorCode::TRANSPORT_PARAMETER_ERROR); } } } else { + if (!this->_alt_con_manager) { + this->_alt_con_manager = + new QUICAltConnectionManager(this, *this->_ctable, *this->_rtable, this->_peer_quic_connection_id, + this->_quic_config->instance_id(), this->_quic_config->active_cid_limit_out()); + this->_frame_generators.add_generator(*this->_alt_con_manager, QUICFrameGeneratorWeight::BEFORE_DATA); + this->_frame_dispatcher->add_handler(this->_alt_con_manager); + } + + this->_handshake_handler->update(packet); + // on client side, _handshake_handler is already started. Just process packet like _state_handshake_process_handshake_packet() error = this->_recv_and_ack(packet); } @@ -1170,7 +1296,7 @@ QUICNetVConnection::_state_handshake_process_initial_packet(const QUICPacket &pa This doesn't call this->_recv_and_ack(), because RETRY packet doesn't have any frames. */ QUICConnectionErrorUPtr -QUICNetVConnection::_state_handshake_process_retry_packet(const QUICPacket &packet) +QUICNetVConnection::_state_handshake_process_retry_packet(const QUICRetryPacketR &packet) { ink_assert(this->netvc_context == NET_VCONNECTION_OUT); @@ -1179,13 +1305,25 @@ QUICNetVConnection::_state_handshake_process_retry_packet(const QUICPacket &pack return nullptr; } + // Check Integrity Tag + if (!packet.has_valid_tag(this->_original_quic_connection_id)) { + // Discard the packet + QUICConDebug("Ignore RETRY packet - integrity tag is not valid"); + return nullptr; + } else { + QUICConDebug("Integrity tag is valid"); + } + // TODO: move packet->payload to _av_token - this->_av_token_len = packet.payload_length(); + this->_av_token_len = packet.token().length(); this->_av_token = ats_unique_malloc(this->_av_token_len); - memcpy(this->_av_token.get(), packet.payload(), this->_av_token_len); + memcpy(this->_av_token.get(), packet.token().buf(), this->_av_token_len); + + this->_padder->set_av_token_len(this->_av_token_len); // discard all transport state this->_handshake_handler->reset(); + this->_handshake_handler->update(packet); this->_packet_factory.reset(); this->_loss_detector->reset(); @@ -1194,7 +1332,7 @@ QUICNetVConnection::_state_handshake_process_retry_packet(const QUICPacket &pack // Initialize Key Materials with peer CID. Because peer CID is DCID of (second) INITIAL packet from client which reply to RETRY // packet from server - this->_hs_protocol->initialize_key_materials(this->_peer_quic_connection_id); + this->_hs_protocol->initialize_key_materials(this->_peer_quic_connection_id, packet.version()); // start handshake over this->_handshake_handler->do_handshake(); @@ -1204,18 +1342,18 @@ QUICNetVConnection::_state_handshake_process_retry_packet(const QUICPacket &pack } QUICConnectionErrorUPtr -QUICNetVConnection::_state_handshake_process_handshake_packet(const QUICPacket &packet) +QUICNetVConnection::_state_handshake_process_handshake_packet(const QUICHandshakePacketR &packet) { // Source address is verified by receiving any message from the client encrypted using the // Handshake keys. - if (this->netvc_context == NET_VCONNECTION_IN && !this->_verfied_state.is_verified()) { - this->_verfied_state.set_addr_verifed(); + if (this->netvc_context == NET_VCONNECTION_IN && !this->_verified_state.is_verified()) { + this->_verified_state.set_addr_verifed(); } return this->_recv_and_ack(packet); } QUICConnectionErrorUPtr -QUICNetVConnection::_state_handshake_process_zero_rtt_protected_packet(const QUICPacket &packet) +QUICNetVConnection::_state_handshake_process_zero_rtt_protected_packet(const QUICZeroRttPacketR &packet) { this->_stream_manager->init_flow_control_params(this->_handshake_handler->local_transport_parameters(), this->_handshake_handler->remote_transport_parameters()); @@ -1224,7 +1362,7 @@ QUICNetVConnection::_state_handshake_process_zero_rtt_protected_packet(const QUI } QUICConnectionErrorUPtr -QUICNetVConnection::_state_connection_established_process_protected_packet(const QUICPacket &packet) +QUICNetVConnection::_state_connection_established_process_protected_packet(const QUICShortHeaderPacketR &packet) { QUICConnectionErrorUPtr error = nullptr; bool has_non_probing_frame = false; @@ -1234,16 +1372,10 @@ QUICNetVConnection::_state_connection_established_process_protected_packet(const return error; } - // Migrate connection if required - // FIXME Connection migration will be initiated when a peer sent non-probing frames. - // We need to two or more paths because we need to respond to probing packets on a new path and also need to send other frames - // on the old path until they initiate migration. - // if (packet.destination_cid() == this->_quic_connection_id && has_non_probing_frame) { + // Migrate connection if needed if (this->_alt_con_manager != nullptr) { - if (packet.destination_cid() != this->_quic_connection_id || !ats_ip_addr_port_eq(packet.from(), this->remote_addr)) { - if (!has_non_probing_frame) { - QUICConDebug("FIXME: Connection migration has been initiated without non-probing frames"); - } + if (has_non_probing_frame && + (packet.destination_cid() != this->_quic_connection_id || !ats_ip_addr_port_eq(packet.from(), this->remote_addr))) { error = this->_state_connection_established_migrate_connection(packet); if (error != nullptr) { return error; @@ -1251,7 +1383,7 @@ QUICNetVConnection::_state_connection_established_process_protected_packet(const } } - // For Connection Migration excercise + // For Connection Migration exercise if (this->netvc_context == NET_VCONNECTION_OUT && this->_quic_config->cm_exercise_enabled()) { this->_state_connection_established_initiate_connection_migration(); } @@ -1268,12 +1400,13 @@ QUICNetVConnection::_state_connection_established_receive_packet() // Receive a QUIC packet net_activity(this, this_ethread()); do { - QUICPacketUPtr packet = this->_dequeue_recv_packet(result); + uint8_t packet_buf[QUICPacket::MAX_INSTANCE_SIZE]; + QUICPacketUPtr packet = this->_dequeue_recv_packet(packet_buf, result); if (result == QUICPacketCreationResult::FAILED) { // Don't make this error, and discard the packet. // Because: // - Attacker can terminate connections - // - It could be just an errora on lower layer + // - It could be just an error on lower layer continue; } else if (result == QUICPacketCreationResult::NO_PACKET) { return error; @@ -1286,13 +1419,13 @@ QUICNetVConnection::_state_connection_established_receive_packet() // Process the packet switch (packet->type()) { case QUICPacketType::PROTECTED: - error = this->_state_connection_established_process_protected_packet(*packet); + error = this->_state_connection_established_process_protected_packet(static_cast(*packet)); break; case QUICPacketType::INITIAL: case QUICPacketType::HANDSHAKE: case QUICPacketType::ZERO_RTT_PROTECTED: // Pass packet to _recv_and_ack to send ack to the packet. Stream data will be discarded by offset mismatch. - error = this->_recv_and_ack(*packet); + error = this->_recv_and_ack(static_cast(*packet)); break; default: QUICConDebug("Unknown packet type: %s(%" PRIu8 ")", QUICDebugNames::packet_type(packet->type()), @@ -1311,14 +1444,15 @@ QUICNetVConnection::_state_closing_receive_packet() { while (this->_packet_recv_queue.size() > 0) { QUICPacketCreationResult result; - QUICPacketUPtr packet = this->_dequeue_recv_packet(result); + uint8_t packet_buf[QUICPacket::MAX_INSTANCE_SIZE]; + QUICPacketUPtr packet = this->_dequeue_recv_packet(packet_buf, result); if (result == QUICPacketCreationResult::SUCCESS) { switch (packet->type()) { case QUICPacketType::VERSION_NEGOTIATION: // Ignore VN packets on closing state break; default: - this->_recv_and_ack(*packet); + this->_recv_and_ack(static_cast(*packet)); break; } } @@ -1342,9 +1476,10 @@ QUICNetVConnection::_state_draining_receive_packet() { while (this->_packet_recv_queue.size() > 0) { QUICPacketCreationResult result; - QUICPacketUPtr packet = this->_dequeue_recv_packet(result); + uint8_t packet_buf[QUICPacket::MAX_INSTANCE_SIZE]; + QUICPacketUPtr packet = this->_dequeue_recv_packet(packet_buf, result); if (result == QUICPacketCreationResult::SUCCESS) { - this->_recv_and_ack(*packet); + this->_recv_and_ack(static_cast(*packet)); // Do NOT schedule WRITE_READY event from this point. // An endpoint in the draining state MUST NOT send any packets. } @@ -1357,7 +1492,7 @@ QUICNetVConnection::_state_draining_receive_packet() * 1. Check congestion window * 2. Allocate buffer for UDP Payload * 3. Generate QUIC Packet - * 4. Store data to the paylaod + * 4. Store data to the payload * 5. Send UDP Packet */ QUICConnectionErrorUPtr @@ -1366,7 +1501,7 @@ QUICNetVConnection::_state_common_send_packet() uint32_t packet_count = 0; uint32_t error = 0; while (error == 0 && packet_count < PACKET_PER_EVENT) { - uint32_t window = this->_congestion_controller->open_window(); + uint32_t window = this->_congestion_controller->credit(); if (window == 0) { break; @@ -1374,29 +1509,33 @@ QUICNetVConnection::_state_common_send_packet() Ptr udp_payload(new_IOBufferBlock()); uint32_t udp_payload_len = std::min(window, this->_pmtu); - udp_payload->alloc(iobuffer_size_to_index(udp_payload_len)); + udp_payload->alloc(iobuffer_size_to_index(udp_payload_len, BUFFER_SIZE_INDEX_32K)); uint32_t written = 0; for (int i = static_cast(this->_minimum_encryption_level); i <= static_cast(QUICEncryptionLevel::ONE_RTT); ++i) { + uint8_t packet_buf[QUICPacket::MAX_INSTANCE_SIZE]; + auto level = QUIC_ENCRYPTION_LEVELS[i]; if (level == QUICEncryptionLevel::ONE_RTT && !this->_handshake_handler->is_completed()) { continue; } uint32_t max_packet_size = udp_payload_len - written; - if (this->netvc_context == NET_VCONNECTION_IN && !this->_verfied_state.is_verified()) { - max_packet_size = std::min(max_packet_size, this->_verfied_state.windows()); + if (this->netvc_context == NET_VCONNECTION_IN && !this->_verified_state.is_verified()) { + max_packet_size = std::min(max_packet_size, this->_verified_state.windows()); } - QUICPacketInfoUPtr packet_info = std::make_unique(); - QUICPacketUPtr packet = this->_packetize_frames(level, max_packet_size, packet_info->frames); + QUICSentPacketInfoUPtr packet_info = std::make_unique(); + QUICPacketUPtr packet = this->_packetize_frames(packet_buf, level, max_packet_size, packet_info->frames); if (packet) { - packet_info->packet_number = packet->packet_number(); - packet_info->time_sent = Thread::get_hrtime(); - packet_info->ack_eliciting = packet->is_ack_eliciting(); - packet_info->is_crypto_packet = packet->is_crypto_packet(); - packet_info->in_flight = true; + // trigger callback + this->_context->trigger(QUICContext::CallbackEvent::PACKET_SEND, packet.get()); + + packet_info->packet_number = packet->packet_number(); + packet_info->time_sent = Thread::get_hrtime(); + packet_info->ack_eliciting = packet->is_ack_eliciting(); + packet_info->in_flight = true; if (packet_info->ack_eliciting) { packet_info->sent_bytes = packet->size(); } else { @@ -1405,9 +1544,9 @@ QUICNetVConnection::_state_common_send_packet() packet_info->type = packet->type(); packet_info->pn_space = QUICTypeUtil::pn_space(level); - if (this->netvc_context == NET_VCONNECTION_IN && !this->_verfied_state.is_verified()) { - QUICConDebug("send to unverified window: %u", this->_verfied_state.windows()); - this->_verfied_state.consume(packet->size()); + if (this->netvc_context == NET_VCONNECTION_IN && !this->_verified_state.is_verified()) { + QUICConDebug("send to unverified window: %u", this->_verified_state.windows()); + this->_verified_state.consume(packet->size()); } // TODO: do not write two QUIC Short Header Packets @@ -1418,14 +1557,17 @@ QUICNetVConnection::_state_common_send_packet() written += len; int dcil = (this->_peer_quic_connection_id == QUICConnectionId::ZERO()) ? 0 : this->_peer_quic_connection_id.length(); - this->_ph_protector.protect(buf, len, dcil); + if (!this->_ph_protector.protect(buf, len, dcil)) { + ink_assert(!"failed to protect buffer"); + } - QUICConDebug("[TX] %s packet #%" PRIu64 " size=%zu", QUICDebugNames::packet_type(packet->type()), packet->packet_number(), - len); + QUICConVDebug("[TX] %s packet #%" PRIu64 " size=%zu", QUICDebugNames::packet_type(packet->type()), packet->packet_number(), + len); if (this->_pp_key_info.is_encryption_key_available(QUICKeyPhase::INITIAL) && packet->type() == QUICPacketType::HANDSHAKE && this->netvc_context == NET_VCONNECTION_OUT) { this->_pp_key_info.drop_keys(QUICKeyPhase::INITIAL); + this->_loss_detector->on_packet_number_space_discarded(QUICPacketNumberSpace::INITIAL); this->_minimum_encryption_level = QUICEncryptionLevel::HANDSHAKE; } @@ -1443,6 +1585,9 @@ QUICNetVConnection::_state_common_send_packet() } if (packet_count) { + this->_context->trigger(QUICContext::CallbackEvent::METRICS_UPDATE, this->_congestion_controller->congestion_window(), + this->_congestion_controller->bytes_in_flight(), this->_congestion_controller->current_ssthresh()); + QUIC_INCREMENT_DYN_STAT_EX(QUICStats::total_packets_sent_stat, packet_count); net_activity(this, this_ethread()); } @@ -1471,15 +1616,13 @@ QUICNetVConnection::_state_closing_send_packet() Ptr QUICNetVConnection::_store_frame(Ptr parent_block, size_t &size_added, uint64_t &max_frame_size, QUICFrame &frame, - std::vector &frames) + std::vector &frames) { Ptr new_block = frame.to_io_buffer_block(max_frame_size); - size_added = 0; - Ptr tmp = new_block; - while (tmp) { + size_added = 0; + for (Ptr tmp = new_block; tmp; tmp = tmp->next) { size_added += tmp->size(); - tmp = tmp->next; } if (parent_block == nullptr) { @@ -1496,7 +1639,7 @@ QUICNetVConnection::_store_frame(Ptr parent_block, size_t &size_a if (is_debug_tag_set(QUIC_DEBUG_TAG.data())) { char msg[1024]; frame.debug_msg(msg, sizeof(msg)); - QUICConDebug("[TX] | %s", msg); + QUICConVDebug("[TX] | %s", msg); } frames.emplace_back(frame.id(), frame.generated_by()); @@ -1507,25 +1650,10 @@ QUICNetVConnection::_store_frame(Ptr parent_block, size_t &size_a return parent_block; } -// FIXME QUICNetVConnection should not know the actual type value of PADDING frame -Ptr -QUICNetVConnection::_generate_padding_frame(size_t frame_size) -{ - Ptr block = make_ptr(new_IOBufferBlock()); - block->alloc(iobuffer_size_to_index(frame_size)); - memset(block->start(), 0, frame_size); - block->fill(frame_size); - - ink_assert(frame_size == static_cast(block->size())); - - return block; -} - QUICPacketUPtr -QUICNetVConnection::_packetize_frames(QUICEncryptionLevel level, uint64_t max_packet_size, std::vector &frames) +QUICNetVConnection::_packetize_frames(uint8_t *packet_buf, QUICEncryptionLevel level, uint64_t max_packet_size, + std::vector &frames) { - ink_hrtime timestamp = Thread::get_hrtime(); - QUICPacketUPtr packet = QUICPacketFactory::create_null_packet(); if (max_packet_size <= MAX_PACKET_OVERHEAD) { return packet; @@ -1543,39 +1671,45 @@ QUICNetVConnection::_packetize_frames(QUICEncryptionLevel level, uint64_t max_pa size_t len = 0; Ptr first_block = make_ptr(new_IOBufferBlock()); Ptr last_block = first_block; - first_block->alloc(iobuffer_size_to_index(0)); + first_block->alloc(iobuffer_size_to_index(0, BUFFER_SIZE_INDEX_32K)); first_block->fill(0); - if (!this->_has_ack_eliciting_packet_out) { - // Sent too much ack_only packet. At this moment we need to packetize a ping frame - // to force peer send ack frame. - this->_pinger.request(level); - } - + uint32_t seq_num = this->_seq_num++; size_t size_added = 0; bool ack_eliciting = false; bool crypto = false; uint8_t frame_instance_buffer[QUICFrame::MAX_INSTANCE_SIZE]; // This is for a frame instance but not serialized frame data QUICFrame *frame = nullptr; - for (auto g : this->_frame_generators) { - while (g->will_generate_frame(level, timestamp)) { + for (auto g : this->_frame_generators.generators()) { + // a non-ack_eliciting packet is ready, but we can not send continuous two ack_eliciting packets. + while (g->will_generate_frame(level, len, ack_eliciting, seq_num)) { // FIXME will_generate_frame should receive more parameters so we don't need extra checks - if (g == this->_remote_flow_controller && !this->_stream_manager->will_generate_frame(level, timestamp)) { - break; - } - if (g == this->_stream_manager && this->_path_validator->is_validating()) { + if (g == this->_remote_flow_controller && !this->_stream_manager->will_generate_frame(level, len, ack_eliciting, seq_num)) { break; } // Common block - frame = g->generate_frame(frame_instance_buffer, level, this->_remote_flow_controller->credit(), max_frame_size, timestamp); + frame = + g->generate_frame(frame_instance_buffer, level, this->_remote_flow_controller->credit(), max_frame_size, len, seq_num); if (frame) { + this->_context->trigger(QUICContext::CallbackEvent::FRAME_PACKETIZE, *frame); + // Some frame types must not be sent on Initial and Handshake packets + switch (auto t = frame->type(); level) { + case QUICEncryptionLevel::INITIAL: + case QUICEncryptionLevel::HANDSHAKE: + ink_assert(t == QUICFrameType::CRYPTO || t == QUICFrameType::ACK || t == QUICFrameType::PADDING || + t == QUICFrameType::CONNECTION_CLOSE || t == QUICFrameType::PING); + break; + default: + break; + } + ++frame_count; probing |= frame->is_probing_frame(); if (frame->is_flow_controlled()) { int ret = this->_remote_flow_controller->update(this->_stream_manager->total_offset_sent()); - QUICFCDebug("[REMOTE] %" PRIu64 "/%" PRIu64, this->_remote_flow_controller->current_offset(), - this->_remote_flow_controller->current_limit()); + QUICFCVDebug("[REMOTE] %" PRIu64 "/%" PRIu64, this->_remote_flow_controller->current_offset(), + this->_remote_flow_controller->current_limit()); ink_assert(ret == 0); } last_block = this->_store_frame(last_block, size_added, max_frame_size, *frame, frames); @@ -1588,13 +1722,11 @@ QUICNetVConnection::_packetize_frames(QUICEncryptionLevel level, uint64_t max_pa } } - if (!ack_eliciting && frame->type() != QUICFrameType::ACK) { + if (!ack_eliciting && frame->ack_eliciting()) { ack_eliciting = true; - this->_pinger.cancel(level); } - if (frame->type() == QUICFrameType::CRYPTO && - (level == QUICEncryptionLevel::INITIAL || level == QUICEncryptionLevel::HANDSHAKE)) { + if (frame->type() == QUICFrameType::CRYPTO && frame->ack_eliciting()) { crypto = true; } @@ -1608,31 +1740,8 @@ QUICNetVConnection::_packetize_frames(QUICEncryptionLevel level, uint64_t max_pa // Schedule a packet if (len != 0) { - if (level == QUICEncryptionLevel::INITIAL && this->netvc_context == NET_VCONNECTION_OUT) { - // Pad with PADDING frames - uint64_t min_size = this->_minimum_quic_packet_size(); - if (this->_av_token) { - min_size = min_size - this->_av_token_len; - } - min_size = std::min(min_size, max_packet_size); - - if (min_size > len) { - Ptr pad_block = _generate_padding_frame(min_size - len); - last_block->next = pad_block; - len += pad_block->size(); - } - } else { - // Pad with PADDING frames to make sure payload length satisfy the minimum length for sampling for header protection - if (MIN_PKT_PAYLOAD_LEN > len) { - Ptr pad_block = _generate_padding_frame(MIN_PKT_PAYLOAD_LEN - len); - last_block->next = pad_block; - len += pad_block->size(); - } - } - // Packet is retransmittable if it's not ack only packet - packet = this->_build_packet(level, first_block, ack_eliciting, probing, crypto); - this->_has_ack_eliciting_packet_out = ack_eliciting; + packet = this->_build_packet(packet_buf, level, first_block, ack_eliciting, probing, crypto); } return packet; @@ -1655,23 +1764,31 @@ QUICNetVConnection::_packetize_closing_frame() size_t size_added = 0; uint64_t max_frame_size = static_cast(max_size); - std::vector frames; - Ptr parent_block; - parent_block = nullptr; - parent_block = this->_store_frame(parent_block, size_added, max_frame_size, *frame, frames); + std::vector frames; + Ptr first_block = make_ptr(new_IOBufferBlock()); + Ptr last_block = first_block; + first_block->alloc(iobuffer_size_to_index(0, BUFFER_SIZE_INDEX_32K)); + first_block->fill(0); + last_block = this->_store_frame(last_block, size_added, max_frame_size, *frame, frames); QUICEncryptionLevel level = this->_hs_protocol->current_encryption_level(); ink_assert(level != QUICEncryptionLevel::ZERO_RTT); - this->_the_final_packet = this->_build_packet(level, parent_block, true, false, false); + this->_the_final_packet = this->_build_packet(this->_final_packet_buf, level, first_block, true, false, false); } QUICConnectionErrorUPtr -QUICNetVConnection::_recv_and_ack(const QUICPacket &packet, bool *has_non_probing_frame) +QUICNetVConnection::_recv_and_ack(const QUICPacketR &packet, bool *has_non_probing_frame) { ink_assert(packet.type() != QUICPacketType::RETRY); - const uint8_t *payload = packet.payload(); uint16_t size = packet.payload_length(); + ats_unique_buf payload_ubuf = ats_unique_malloc(size); + uint8_t *payload = payload_ubuf.get(); + size_t copied_len = 0; + for (auto b = packet.payload_block(); b; b = b->next) { + memcpy(payload + copied_len, b->start(), b->size()); + copied_len += b->size(); + } QUICPacketNumber packet_num = packet.packet_number(); QUICEncryptionLevel level = QUICTypeUtil::encryption_level(packet.type()); @@ -1683,23 +1800,26 @@ QUICNetVConnection::_recv_and_ack(const QUICPacket &packet, bool *has_non_probin *has_non_probing_frame = false; } - error = this->_frame_dispatcher->receive_frames(level, payload, size, ack_only, is_flow_controlled, has_non_probing_frame); + error = this->_frame_dispatcher->receive_frames(*this->_context, level, payload, size, ack_only, is_flow_controlled, + has_non_probing_frame, static_cast(&packet)); + this->_context->trigger(QUICContext::CallbackEvent::PACKET_RECV, &packet); + if (error != nullptr) { return error; } if (is_flow_controlled) { int ret = this->_local_flow_controller->update(this->_stream_manager->total_offset_received()); - QUICFCDebug("[LOCAL] %" PRIu64 "/%" PRIu64, this->_local_flow_controller->current_offset(), - this->_local_flow_controller->current_limit()); + QUICFCVDebug("[LOCAL] %" PRIu64 "/%" PRIu64, this->_local_flow_controller->current_offset(), + this->_local_flow_controller->current_limit()); if (ret != 0) { return std::make_unique(QUICTransErrorCode::FLOW_CONTROL_ERROR); } this->_local_flow_controller->forward_limit(this->_stream_manager->total_reordered_bytes() + this->_flow_control_buffer_size); - QUICFCDebug("[LOCAL] %" PRIu64 "/%" PRIu64, this->_local_flow_controller->current_offset(), - this->_local_flow_controller->current_limit()); + QUICFCVDebug("[LOCAL] %" PRIu64 "/%" PRIu64, this->_local_flow_controller->current_offset(), + this->_local_flow_controller->current_limit()); } this->_ack_frame_manager.update(level, packet_num, size, ack_only); @@ -1708,21 +1828,15 @@ QUICNetVConnection::_recv_and_ack(const QUICPacket &packet, bool *has_non_probin } QUICPacketUPtr -QUICNetVConnection::_build_packet(QUICEncryptionLevel level, Ptr parent_block, bool ack_eliciting, bool probing, - bool crypto) +QUICNetVConnection::_build_packet(uint8_t *packet_buf, QUICEncryptionLevel level, const Ptr &parent_block, + bool ack_eliciting, bool probing, bool crypto) { QUICPacketType type = QUICTypeUtil::packet_type(level); QUICPacketUPtr packet = QUICPacketFactory::create_null_packet(); - // FIXME Pass parent_block to create_x_packet - // No need to make a unique buf here - ats_unique_buf buf = ats_unique_malloc(2048); - uint8_t *raw_buf = buf.get(); - size_t len = 0; - while (parent_block) { - memcpy(raw_buf + len, parent_block->start(), parent_block->size()); - len += parent_block->size(); - parent_block = parent_block->next; + size_t len = 0; + for (Ptr tmp = parent_block; tmp; tmp = tmp->next) { + len += tmp->size(); } switch (type) { @@ -1732,7 +1846,7 @@ QUICNetVConnection::_build_packet(QUICEncryptionLevel level, Ptr size_t token_len = 0; if (this->netvc_context == NET_VCONNECTION_OUT) { - // TODO: Add a case of using token which is advertized by NEW_TOKEN frame + // TODO: Add a case of using token which is advertised by NEW_TOKEN frame if (this->_av_token) { token = ats_unique_malloc(this->_av_token_len); token_len = this->_av_token_len; @@ -1743,26 +1857,26 @@ QUICNetVConnection::_build_packet(QUICEncryptionLevel level, Ptr } packet = this->_packet_factory.create_initial_packet( - dcid, this->_quic_connection_id, this->_largest_acked_packet_number(QUICEncryptionLevel::INITIAL), std::move(buf), len, - ack_eliciting, probing, crypto, std::move(token), token_len); + packet_buf, dcid, this->_quic_connection_id, this->_largest_acked_packet_number(QUICEncryptionLevel::INITIAL), parent_block, + len, ack_eliciting, probing, crypto, std::move(token), token_len); break; } case QUICPacketType::HANDSHAKE: { - packet = this->_packet_factory.create_handshake_packet(this->_peer_quic_connection_id, this->_quic_connection_id, + packet = this->_packet_factory.create_handshake_packet(packet_buf, this->_peer_quic_connection_id, this->_quic_connection_id, this->_largest_acked_packet_number(QUICEncryptionLevel::HANDSHAKE), - std::move(buf), len, ack_eliciting, probing, crypto); + parent_block, len, ack_eliciting, probing, crypto); break; } case QUICPacketType::ZERO_RTT_PROTECTED: { - packet = this->_packet_factory.create_zero_rtt_packet(this->_original_quic_connection_id, this->_quic_connection_id, + packet = this->_packet_factory.create_zero_rtt_packet(packet_buf, this->_original_quic_connection_id, this->_quic_connection_id, this->_largest_acked_packet_number(QUICEncryptionLevel::ZERO_RTT), - std::move(buf), len, ack_eliciting, probing); + parent_block, len, ack_eliciting, probing); break; } case QUICPacketType::PROTECTED: { - packet = this->_packet_factory.create_protected_packet(this->_peer_quic_connection_id, - this->_largest_acked_packet_number(QUICEncryptionLevel::ONE_RTT), - std::move(buf), len, ack_eliciting, probing); + packet = this->_packet_factory.create_short_header_packet(packet_buf, this->_peer_quic_connection_id, + this->_largest_acked_packet_number(QUICEncryptionLevel::ONE_RTT), + parent_block, len, ack_eliciting, probing); break; } default: @@ -1805,29 +1919,31 @@ QUICNetVConnection::_handle_error(QUICConnectionErrorUPtr error) static_cast(error->cls), QUICDebugNames::error_code(error->code), error->code); // Connection Error - this->close(std::move(error)); + this->close_quic_connection(std::move(error)); } QUICPacketUPtr -QUICNetVConnection::_dequeue_recv_packet(QUICPacketCreationResult &result) +QUICNetVConnection::_dequeue_recv_packet(uint8_t *packet_buf, QUICPacketCreationResult &result) { - QUICPacketUPtr packet = this->_packet_recv_queue.dequeue(result); + QUICPacketUPtr packet = this->_packet_recv_queue.dequeue(packet_buf, result); if (result == QUICPacketCreationResult::SUCCESS) { if (this->direction() == NET_VCONNECTION_OUT) { // Reset CID if a server sent back a new CID - // FIXME This should happen only once - QUICConnectionId src_cid = packet->source_cid(); - // FIXME src connection id could be zero ? if so, check packet header type. - if (src_cid != QUICConnectionId::ZERO()) { - if (this->_peer_quic_connection_id != src_cid) { - this->_update_peer_cid(src_cid); + // FIXME This should happen only once - it should probably be controlled by PathManager + if (packet->type() != QUICPacketType::PROTECTED && (packet->type() != QUICPacketType::RETRY || !this->_av_token)) { + QUICConnectionId src_cid = static_cast(*packet).source_cid(); + // FIXME src connection id could be zero ? if so, check packet header type. + if (src_cid != QUICConnectionId::ZERO()) { + if (this->_peer_quic_connection_id != src_cid) { + this->_update_peer_cid(src_cid); + } } } } - if (!this->_verfied_state.is_verified()) { - this->_verfied_state.fill(packet->size()); + if (!this->_verified_state.is_verified()) { + this->_verified_state.fill(packet->size()); } } @@ -1845,11 +1961,15 @@ QUICNetVConnection::_dequeue_recv_packet(QUICPacketCreationResult &result) QUICConDebug("Unsupported version"); break; case QUICPacketCreationResult::SUCCESS: - if (packet->type() == QUICPacketType::VERSION_NEGOTIATION) { - QUICConDebug("[RX] %s packet size=%u", QUICDebugNames::packet_type(packet->type()), packet->size()); - } else { - QUICConDebug("[RX] %s packet #%" PRIu64 " size=%u header_len=%u payload_len=%u", QUICDebugNames::packet_type(packet->type()), - packet->packet_number(), packet->size(), packet->header_size(), packet->payload_length()); + switch (packet->type()) { + case QUICPacketType::VERSION_NEGOTIATION: + case QUICPacketType::RETRY: + QUICConVDebug("[RX] %s packet size=%u", QUICDebugNames::packet_type(packet->type()), packet->size()); + break; + default: + QUICConVDebug("[RX] %s packet #%" PRIu64 " size=%u header_len=%u payload_len=%u", QUICDebugNames::packet_type(packet->type()), + packet->packet_number(), packet->size(), packet->header_size(), packet->payload_length()); + break; } break; default: @@ -1976,41 +2096,27 @@ QUICNetVConnection::_complete_handshake_if_possible() this->_init_flow_control_params(this->_handshake_handler->local_transport_parameters(), this->_handshake_handler->remote_transport_parameters()); - // PN space doesn't matter but seems like this is the way to pick the LossDetector for 0-RTT and Short packet uint64_t ack_delay_exponent = this->_handshake_handler->remote_transport_parameters()->getAsUInt(QUICTransportParameterId::ACK_DELAY_EXPONENT); this->_loss_detector->update_ack_delay_exponent(ack_delay_exponent); - this->_start_application(); + uint64_t max_ack_delay = + this->_handshake_handler->remote_transport_parameters()->getAsUInt(QUICTransportParameterId::MAX_ACK_DELAY); + this->_rtt_measure.set_max_ack_delay(max_ack_delay); - return 0; -} - -void -QUICNetVConnection::_schedule_path_validation_timeout(ink_hrtime interval) -{ - if (!this->_path_validation_timeout) { - QUICConDebug("Schedule %s event in %" PRIu64 "ms", QUICDebugNames::quic_event(QUIC_EVENT_PATH_VALIDATION_TIMEOUT), - interval / HRTIME_MSECOND); - this->_path_validation_timeout = this->thread->schedule_in_local(this, interval, QUIC_EVENT_PATH_VALIDATION_TIMEOUT); + const uint8_t *reset_token; + uint16_t reset_token_len; + reset_token = this->_handshake_handler->remote_transport_parameters()->getAsBytes(QUICTransportParameterId::STATELESS_RESET_TOKEN, + reset_token_len); + if (reset_token) { + this->_rtable->insert({reset_token}, this); } -} -void -QUICNetVConnection::_unschedule_path_validation_timeout() -{ - if (this->_path_validation_timeout) { - QUICConDebug("Unschedule %s event", QUICDebugNames::quic_event(QUIC_EVENT_PATH_VALIDATION_TIMEOUT)); - this->_path_validation_timeout->cancel(); - this->_path_validation_timeout = nullptr; - } -} + this->_start_application(); -void -QUICNetVConnection::_close_path_validation_timeout(Event *data) -{ - ink_assert(this->_path_validation_timeout == data); - this->_path_validation_timeout = nullptr; + this->_handshake_completed = true; + + return 0; } void @@ -2029,7 +2135,7 @@ QUICNetVConnection::_start_application() if (netvc_context == NET_VCONNECTION_IN) { if (!this->setSelectedProtocol(app_name, app_name_len)) { - this->_handle_error(std::make_unique(QUICTransErrorCode::VERSION_NEGOTIATION_ERROR)); + this->_handle_error(std::make_unique(QUICTransErrorCode::PROTOCOL_VIOLATION)); } else { this->endpoint()->handleEvent(NET_EVENT_ACCEPT, this); } @@ -2055,15 +2161,26 @@ QUICNetVConnection::_switch_to_established_state() SET_HANDLER((NetVConnHandler)&QUICNetVConnection::state_connection_established); + std::shared_ptr remote_tp = this->_handshake_handler->remote_transport_parameters(); + std::shared_ptr local_tp = this->_handshake_handler->local_transport_parameters(); + + uint64_t active_cid_limit = remote_tp->getAsUInt(QUICTransportParameterId::ACTIVE_CONNECTION_ID_LIMIT); + if (active_cid_limit) { + this->_alt_con_manager->set_remote_active_cid_limit(active_cid_limit); + } + + this->set_inactivity_timeout(HRTIME_MSECONDS(std::min(remote_tp->getAsUInt(QUICTransportParameterId::MAX_IDLE_TIMEOUT), + local_tp->getAsUInt(QUICTransportParameterId::MAX_IDLE_TIMEOUT)))); + if (this->direction() == NET_VCONNECTION_OUT) { - std::shared_ptr remote_tp = this->_handshake_handler->remote_transport_parameters(); - const uint8_t *pref_addr_buf; uint16_t len; - pref_addr_buf = remote_tp->getAsBytes(QUICTransportParameterId::PREFERRED_ADDRESS, len); - this->_alt_con_manager = new QUICAltConnectionManager(this, *this->_ctable, this->_peer_quic_connection_id, - this->_quic_config->instance_id(), 1, {pref_addr_buf, len}); - this->_frame_generators.push_back(this->_alt_con_manager); - this->_frame_dispatcher->add_handler(this->_alt_con_manager); + const uint8_t *pref_addr_buf = remote_tp->getAsBytes(QUICTransportParameterId::PREFERRED_ADDRESS, len); + if (pref_addr_buf) { + this->_alt_con_manager->set_remote_preferred_address({pref_addr_buf, len}); + } + } else { + QUICPath trusted_path = {this->local_addr, this->remote_addr}; + this->_path_manager->set_trusted_path(trusted_path); } } else { // Illegal state change @@ -2135,7 +2252,6 @@ void QUICNetVConnection::_switch_to_close_state() { this->_unschedule_closing_timeout(); - this->_unschedule_path_validation_timeout(); if (this->_complete_handshake_if_possible() != 0) { QUICConDebug("Switching state without handshake completion"); @@ -2155,13 +2271,18 @@ QUICNetVConnection::_handle_idle_timeout() } void -QUICNetVConnection::_validate_new_path() +QUICNetVConnection::_handle_active_timeout() +{ + this->close_quic_connection(std::make_unique(QUICTransErrorCode::NO_ERROR, "Active Timeout")); +} + +void +QUICNetVConnection::_validate_new_path(const QUICPath &path) { - this->_path_validator->validate(); // Not sure how long we should wait. The spec says just "enough time". // Use the same time amount as the closing timeout. ink_hrtime rto = this->_rtt_measure.current_pto_period(); - this->_schedule_path_validation_timeout(3 * rto); + this->_path_manager->open_new_path(path, 3 * rto); } void @@ -2177,12 +2298,7 @@ void QUICNetVConnection::_update_peer_cid(const QUICConnectionId &new_cid) { if (is_debug_tag_set(QUIC_DEBUG_TAG.data())) { - char old_cid_str[QUICConnectionId::MAX_HEX_STR_LENGTH]; - char new_cid_str[QUICConnectionId::MAX_HEX_STR_LENGTH]; - this->_peer_quic_connection_id.hex(old_cid_str, QUICConnectionId::MAX_HEX_STR_LENGTH); - new_cid.hex(new_cid_str, QUICConnectionId::MAX_HEX_STR_LENGTH); - - QUICConDebug("dcid: %s -> %s", old_cid_str, new_cid_str); + QUICConDebug("update peer dcid: %s -> %s", this->_peer_quic_connection_id.hex().c_str(), new_cid.hex().c_str()); } this->_peer_old_quic_connection_id = this->_peer_quic_connection_id; @@ -2194,12 +2310,7 @@ void QUICNetVConnection::_update_local_cid(const QUICConnectionId &new_cid) { if (is_debug_tag_set(QUIC_DEBUG_TAG.data())) { - char old_cid_str[QUICConnectionId::MAX_HEX_STR_LENGTH]; - char new_cid_str[QUICConnectionId::MAX_HEX_STR_LENGTH]; - this->_quic_connection_id.hex(old_cid_str, QUICConnectionId::MAX_HEX_STR_LENGTH); - new_cid.hex(new_cid_str, QUICConnectionId::MAX_HEX_STR_LENGTH); - - QUICConDebug("scid: %s -> %s", old_cid_str, new_cid_str); + QUICConDebug("update local dcid: %s -> %s", this->_quic_connection_id.hex().c_str(), new_cid.hex().c_str()); } this->_quic_connection_id = new_cid; @@ -2213,29 +2324,25 @@ QUICNetVConnection::_rerandomize_original_cid() this->_original_quic_connection_id.randomize(); if (is_debug_tag_set(QUIC_DEBUG_TAG.data())) { - char old_cid_str[QUICConnectionId::MAX_HEX_STR_LENGTH]; - char new_cid_str[QUICConnectionId::MAX_HEX_STR_LENGTH]; - tmp.hex(old_cid_str, QUICConnectionId::MAX_HEX_STR_LENGTH); - this->_original_quic_connection_id.hex(new_cid_str, QUICConnectionId::MAX_HEX_STR_LENGTH); - - QUICConDebug("original cid: %s -> %s", old_cid_str, new_cid_str); + QUICConDebug("original cid: %s -> %s", tmp.hex().c_str(), this->_original_quic_connection_id.hex().c_str()); } } QUICHandshakeProtocol * -QUICNetVConnection::_setup_handshake_protocol(shared_SSL_CTX ctx) +QUICNetVConnection::_setup_handshake_protocol(const shared_SSL_CTX &ctx) { // Initialize handshake protocol specific stuff // For QUICv1 TLS is the only option QUICTLS *tls = new QUICTLS(this->_pp_key_info, ctx.get(), this->direction(), this->options, this->_quic_config->client_session_file(), this->_quic_config->client_keylog_file()); SSL_set_ex_data(tls->ssl_handle(), QUIC::ssl_quic_qc_index, static_cast(this)); + TLSSessionResumptionSupport::bind(tls->ssl_handle(), this); return tls; } QUICConnectionErrorUPtr -QUICNetVConnection::_state_connection_established_migrate_connection(const QUICPacket &p) +QUICNetVConnection::_state_connection_established_migrate_connection(const QUICPacketR &p) { ink_assert(this->_handshake_handler->is_completed()); @@ -2243,26 +2350,31 @@ QUICNetVConnection::_state_connection_established_migrate_connection(const QUICP QUICConnectionId dcid = p.destination_cid(); if (this->netvc_context == NET_VCONNECTION_IN) { - if (!this->_alt_con_manager->is_ready_to_migrate()) { - // TODO: Should endpoint send connection error when remote endpoint doesn't send NEW_CONNECTION_ID frames before initiating - // connection migration ? - QUICConDebug("Ignore connection migration - remote endpoint initiated CM before sending NEW_CONNECTION_ID frames"); - return error; - } QUICConDebug("Connection migration is initiated by remote"); } if (this->connection_id() == dcid) { + QUICConDebug("Maybe NAT rebinding"); // On client side (NET_VCONNECTION_OUT), nothing to do any more if (this->netvc_context == NET_VCONNECTION_IN) { Connection con; con.setRemote(&(p.from().sa)); this->con.move(con); this->set_remote_addr(); - this->_udp_con = p.udp_con(); - this->_validate_new_path(); + this->_udp_con = p.udp_con(); + this->_packet_handler = static_cast( + static_cast(static_cast(this->_udp_con)->continuation)); + + QUICPath new_path = {p.to(), p.from()}; + this->_validate_new_path(new_path); } } else { + QUICConDebug("Different CID"); + if (!this->_alt_con_manager->is_ready_to_migrate()) { + QUICConDebug("Ignore connection migration - remote endpoint initiated CM before sending NEW_CONNECTION_ID frames"); + return error; + } + if (this->_alt_con_manager->migrate_to(dcid, this->_reset_token)) { // DCID of received packet is local cid this->_update_local_cid(dcid); @@ -2273,15 +2385,17 @@ QUICNetVConnection::_state_connection_established_migrate_connection(const QUICP con.setRemote(&(p.from().sa)); this->con.move(con); this->set_remote_addr(); - this->_udp_con = p.udp_con(); + this->_udp_con = p.udp_con(); + this->_packet_handler = static_cast( + static_cast(static_cast(this->_udp_con)->continuation)); this->_update_peer_cid(this->_alt_con_manager->migrate_to_alt_cid()); - this->_validate_new_path(); + + QUICPath new_path = {p.to(), p.from()}; + this->_validate_new_path(new_path); } } else { - char dcid_str[QUICConnectionId::MAX_HEX_STR_LENGTH]; - dcid.hex(dcid_str, QUICConnectionId::MAX_HEX_STR_LENGTH); - QUICConDebug("Connection migration failed cid=%s", dcid_str); + QUICConDebug("Connection migration failed cid=%s", dcid.hex().c_str()); } } @@ -2289,7 +2403,7 @@ QUICNetVConnection::_state_connection_established_migrate_connection(const QUICP } /** - * Connection Migration Excercise from client + * Connection Migration Exercise from client */ QUICConnectionErrorUPtr QUICNetVConnection::_state_connection_established_initiate_connection_migration() @@ -2298,13 +2412,12 @@ QUICNetVConnection::_state_connection_established_initiate_connection_migration( ink_assert(this->netvc_context == NET_VCONNECTION_OUT); QUICConnectionErrorUPtr error = nullptr; - ink_hrtime timestamp = Thread::get_hrtime(); std::shared_ptr remote_tp = this->_handshake_handler->remote_transport_parameters(); - if (this->_connection_migration_initiated || remote_tp->contains(QUICTransportParameterId::DISABLE_MIGRATION) || + if (this->_connection_migration_initiated || remote_tp->contains(QUICTransportParameterId::DISABLE_ACTIVE_MIGRATION) || !this->_alt_con_manager->is_ready_to_migrate() || - this->_alt_con_manager->will_generate_frame(QUICEncryptionLevel::ONE_RTT, timestamp)) { + this->_alt_con_manager->will_generate_frame(QUICEncryptionLevel::ONE_RTT, 0, true, this->_seq_num++)) { return error; } @@ -2313,7 +2426,9 @@ QUICNetVConnection::_state_connection_established_initiate_connection_migration( this->_update_peer_cid(this->_alt_con_manager->migrate_to_alt_cid()); - this->_validate_new_path(); + auto current_path = this->_path_manager->get_verified_path(); + QUICPath new_path = {current_path.local_ep(), current_path.remote_ep()}; + this->_validate_new_path(new_path); return error; } @@ -2321,10 +2436,9 @@ QUICNetVConnection::_state_connection_established_initiate_connection_migration( void QUICNetVConnection::_handle_periodic_ack_event() { - ink_hrtime timestamp = Thread::get_hrtime(); - bool need_schedule = false; + bool need_schedule = false; for (int i = static_cast(this->_minimum_encryption_level); i <= static_cast(QUICEncryptionLevel::ONE_RTT); ++i) { - if (this->_ack_frame_manager.will_generate_frame(QUIC_ENCRYPTION_LEVELS[i], timestamp)) { + if (this->_ack_frame_manager.will_generate_frame(QUIC_ENCRYPTION_LEVELS[i], 0, true, this->_seq_num++)) { need_schedule = true; break; } @@ -2337,67 +2451,8 @@ QUICNetVConnection::_handle_periodic_ack_event() } } -void -QUICNetVConnection::_handle_path_validation_timeout(Event *data) -{ - this->_close_path_validation_timeout(data); - if (this->_path_validator->is_validated()) { - QUICConDebug("Path validated"); - this->_alt_con_manager->drop_cid(this->_peer_old_quic_connection_id); - // FIXME This is a kind of workaround for connection migration. - // This PING make peer to send an ACK frame so that ATS can detect packet loss. - // It would be better if QUICLossDetector could detect the loss in another way. - this->ping(); - } else { - QUICConDebug("Path validation failed"); - this->_switch_to_close_state(); - } -} - -// QUICFrameGenerator -bool -QUICNetVConnection::will_generate_frame(QUICEncryptionLevel level, ink_hrtime timestamp) -{ - if (!this->_is_level_matched(level)) { - return false; - } - - return !this->_is_resumption_token_sent; -} - -QUICFrame * -QUICNetVConnection::generate_frame(uint8_t *buf, QUICEncryptionLevel level, uint64_t connection_credit, uint16_t maximum_frame_size, - ink_hrtime timestamp) -{ - QUICFrame *frame = nullptr; - - if (!this->_is_level_matched(level)) { - return frame; - } - - if (this->_is_resumption_token_sent) { - return frame; - } - - if (this->direction() == NET_VCONNECTION_IN) { - // TODO Make expiration period configurable - QUICResumptionToken token(this->get_remote_endpoint(), this->connection_id(), Thread::get_hrtime() + HRTIME_HOURS(24)); - frame = QUICFrameFactory::create_new_token_frame(buf, token, this->_issue_frame_id(), this); - if (frame) { - if (frame->size() < maximum_frame_size) { - this->_is_resumption_token_sent = true; - } else { - // Cancel generating frame - frame = nullptr; - } - } - } - - return frame; -} - -void -QUICNetVConnection::_on_frame_lost(QUICFrameInformationUPtr &info) +const IpEndpoint & +QUICNetVConnection::_getLocalEndpoint() { - this->_is_resumption_token_sent = false; + return local_addr; } diff --git a/iocore/net/QUICPacketHandler.cc b/iocore/net/QUICPacketHandler.cc index fc906b2d366..4a69065ce3e 100644 --- a/iocore/net/QUICPacketHandler.cc +++ b/iocore/net/QUICPacketHandler.cc @@ -22,6 +22,9 @@ #include "tscore/ink_config.h" #include "P_Net.h" +#include "P_QUICPacketHandler.h" +#include "P_QUICNetProcessor.h" +#include "P_QUICNet.h" #include "P_QUICClosedConCollector.h" #include "QUICGlobals.h" @@ -29,21 +32,27 @@ #include "QUICPacket.h" #include "QUICDebugNames.h" #include "QUICEvents.h" +#include "QUICResetTokenTable.h" -static constexpr int LONG_HDR_OFFSET_CONNECTION_ID = 6; -static constexpr char debug_tag[] = "quic_sec"; +#include "QUICMultiCertConfigLoader.h" +#include "QUICTLS.h" + +static constexpr char debug_tag[] = "quic_sec"; +static constexpr char v_debug_tag[] = "v_quic_sec"; #define QUICDebug(fmt, ...) Debug(debug_tag, fmt, ##__VA_ARGS__) -#define QUICDebugQC(qc, fmt, ...) Debug(debug_tag, "[%s] " fmt, qc->cids().data(), ##__VA_ARGS__) +#define QUICQCDebug(qc, fmt, ...) Debug(debug_tag, "[%s] " fmt, qc->cids().data(), ##__VA_ARGS__) // ["local dcid" - "local scid"] -#define QUICDebugDS(dcid, scid, fmt, ...) \ +#define QUICPHDebug(dcid, scid, fmt, ...) \ Debug(debug_tag, "[%08" PRIx32 "-%08" PRIx32 "] " fmt, dcid.h32(), scid.h32(), ##__VA_ARGS__) +#define QUICVPHDebug(dcid, scid, fmt, ...) \ + Debug(v_debug_tag, "[%08" PRIx32 "-%08" PRIx32 "] " fmt, dcid.h32(), scid.h32(), ##__VA_ARGS__) // // QUICPacketHandler // -QUICPacketHandler::QUICPacketHandler() +QUICPacketHandler::QUICPacketHandler(QUICResetTokenTable &rtable) : _rtable(rtable) { this->_closed_con_collector = new QUICClosedConCollector; this->_closed_con_collector->mutex = new_ProxyMutex(); @@ -77,7 +86,7 @@ QUICPacketHandler::_send_packet(const QUICPacket &packet, UDPConnection *udp_con { size_t udp_len; Ptr udp_payload(new_IOBufferBlock()); - udp_payload->alloc(iobuffer_size_to_index(pmtu)); + udp_payload->alloc(iobuffer_size_to_index(pmtu, BUFFER_SIZE_INDEX_32K)); packet.store(reinterpret_cast(udp_payload->end()), &udp_len); udp_payload->fill(udp_len); @@ -93,7 +102,7 @@ QUICPacketHandler::_send_packet(UDPConnection *udp_con, IpEndpoint &addr, PtrgetPortNum(), buf_len); + QUICVPHDebug(dcid, scid, "send %s packet to %s from port %u size=%" PRId64, (QUICInvariants::is_long_header(buf) ? "LH" : "SH"), + ats_ip_nptop(&addr, ipb, sizeof(ipb)), udp_con->getPortNum(), buf_len); } udp_con->send(this->_get_continuation(), udp_packet); get_UDPNetHandler(static_cast(udp_con)->ethread)->signalActivity(); } +QUICConnection * +QUICPacketHandler::_check_stateless_reset(const uint8_t *buf, size_t buf_len) +{ + return this->_rtable.lookup({buf + (buf_len - 16)}); +} + // // QUICPacketHandlerIn // -QUICPacketHandlerIn::QUICPacketHandlerIn(const NetProcessor::AcceptOptions &opt, QUICConnectionTable &ctable) - : NetAccept(opt), QUICPacketHandler(), _ctable(ctable) +QUICPacketHandlerIn::QUICPacketHandlerIn(const NetProcessor::AcceptOptions &opt, QUICConnectionTable &ctable, + QUICResetTokenTable &rtable) + : NetAccept(opt), QUICPacketHandler(rtable), _ctable(ctable) { this->mutex = new_ProxyMutex(); // create Connection Table @@ -142,7 +158,7 @@ NetAccept * QUICPacketHandlerIn::clone() const { NetAccept *na; - na = new QUICPacketHandlerIn(opt, this->_ctable); + na = new QUICPacketHandlerIn(opt, this->_ctable, this->_rtable); *na = *this; return na; } @@ -164,7 +180,7 @@ QUICPacketHandlerIn::acceptEvent(int event, void *data) this->_collector_event = this_ethread()->schedule_every(this->_closed_con_collector, HRTIME_MSECONDS(100)); } - Queue *queue = (Queue *)data; + Queue *queue = static_cast *>(data); UDPPacket *packet_r; while ((packet_r = queue->dequeue())) { this->_recv_packet(event, packet_r); @@ -178,7 +194,7 @@ QUICPacketHandlerIn::acceptEvent(int event, void *data) if (((long)data) == -ECONNABORTED) { } - ink_abort("QUIC accept received fatal error: errno = %d", -((int)(intptr_t)data)); + ink_abort("QUIC accept received fatal error: errno = %d", -(static_cast((intptr_t)data))); return EVENT_CONT; return 0; } @@ -202,6 +218,7 @@ QUICPacketHandlerIn::_recv_packet(int event, UDPPacket *udp_packet) IOBufferBlock *block = udp_packet->getIOBlockChain(); const uint8_t *buf = reinterpret_cast(block->buf()); uint64_t buf_len = block->size(); + QUICVersion version; if (buf_len == 0) { QUICDebug("Ignore packet - payload is too small"); @@ -225,25 +242,24 @@ QUICPacketHandlerIn::_recv_packet(int event, UDPPacket *udp_packet) return; } - if (is_debug_tag_set(debug_tag)) { + if (is_debug_tag_set(v_debug_tag)) { ip_port_text_buffer ipb_from; ip_port_text_buffer ipb_to; - QUICDebugDS(scid, dcid, "recv LH packet from %s to %s size=%" PRId64, - ats_ip_nptop(&udp_packet->from.sa, ipb_from, sizeof(ipb_from)), - ats_ip_nptop(&udp_packet->to.sa, ipb_to, sizeof(ipb_to)), udp_packet->getPktLength()); + QUICVPHDebug(scid, dcid, "recv LH packet from %s to %s size=%" PRId64, + ats_ip_nptop(&udp_packet->from.sa, ipb_from, sizeof(ipb_from)), + ats_ip_nptop(&udp_packet->to.sa, ipb_to, sizeof(ipb_to)), udp_packet->getPktLength()); } - QUICVersion v; - if (unlikely(!QUICInvariants::version(v, buf, buf_len))) { + if (unlikely(!QUICInvariants::version(version, buf, buf_len))) { QUICDebug("Ignore packet - payload is too small"); udp_packet->free(); return; } - if (!QUICInvariants::is_version_negotiation(v) && !QUICTypeUtil::is_supported_version(v)) { - QUICDebugDS(scid, dcid, "Unsupported version: 0x%x", v); + if (!QUICInvariants::is_version_negotiation(version) && !QUICTypeUtil::is_supported_version(version)) { + QUICPHDebug(scid, dcid, "Unsupported version: 0x%x", version); - QUICPacketUPtr vn = QUICPacketFactory::create_version_negotiation_packet(scid, dcid); + QUICPacketUPtr vn = QUICPacketFactory::create_version_negotiation_packet(scid, dcid, version); this->_send_packet(*vn, udp_packet->getConnection(), udp_packet->from, 1200, nullptr, 0); udp_packet->free(); return; @@ -255,7 +271,7 @@ QUICPacketHandlerIn::_recv_packet(int event, UDPPacket *udp_packet) } QUICPacketType type = QUICPacketType::UNINITIALIZED; - QUICPacketLongHeader::type(type, buf, buf_len); + QUICLongHeaderPacketR::type(type, buf, buf_len); if (type == QUICPacketType::INITIAL) { // [draft-18] 7.2. // When an Initial packet is sent by a client which has not previously received a Retry packet from the server, it populates @@ -268,12 +284,12 @@ QUICPacketHandlerIn::_recv_packet(int event, UDPPacket *udp_packet) } } else { // TODO: lookup DCID by 5-tuple when ATS omits SCID - if (is_debug_tag_set(debug_tag)) { + if (is_debug_tag_set(v_debug_tag)) { ip_port_text_buffer ipb_from; ip_port_text_buffer ipb_to; - QUICDebugDS(scid, dcid, "recv SH packet from %s to %s size=%" PRId64, - ats_ip_nptop(&udp_packet->from.sa, ipb_from, sizeof(ipb_from)), - ats_ip_nptop(&udp_packet->to.sa, ipb_to, sizeof(ipb_to)), udp_packet->getPktLength()); + QUICVPHDebug(scid, dcid, "recv SH packet from %s to %s size=%" PRId64, + ats_ip_nptop(&udp_packet->from.sa, ipb_from, sizeof(ipb_from)), + ats_ip_nptop(&udp_packet->to.sa, ipb_to, sizeof(ipb_to)), udp_packet->getPktLength()); } } @@ -282,9 +298,11 @@ QUICPacketHandlerIn::_recv_packet(int event, UDPPacket *udp_packet) // Server Stateless Retry QUICConfig::scoped_config params; - QUICConnectionId cid_in_retry_token = QUICConnectionId::ZERO(); + QUICConnectionId ocid_in_retry_token = QUICConnectionId::ZERO(); + QUICConnectionId rcid_in_retry_token = QUICConnectionId::ZERO(); if (!vc && params->stateless_retry() && QUICInvariants::is_long_header(buf)) { - int ret = this->_stateless_retry(buf, buf_len, udp_packet->getConnection(), udp_packet->from, dcid, scid, &cid_in_retry_token); + int ret = this->_stateless_retry(buf, buf_len, udp_packet->getConnection(), udp_packet->from, dcid, scid, &ocid_in_retry_token, + &rcid_in_retry_token, version); if (ret < 0) { udp_packet->free(); return; @@ -294,22 +312,33 @@ QUICPacketHandlerIn::_recv_packet(int event, UDPPacket *udp_packet) // [draft-12] 6.1.2. Server Packet Handling // Servers MUST drop incoming packets under all other circumstances. They SHOULD send a Stateless Reset (Section 6.10.4) if a // connection ID is present in the header. - if ((!vc && !QUICInvariants::is_long_header(buf)) || (vc && vc->in_closed_queue)) { - if (is_debug_tag_set(debug_tag)) { - char dcid_str[QUICConnectionId::MAX_HEX_STR_LENGTH]; - dcid.hex(dcid_str, QUICConnectionId::MAX_HEX_STR_LENGTH); - - if (!vc && !QUICInvariants::is_long_header(buf)) { - QUICDebugDS(scid, dcid, "sent Stateless Reset : connection not found, dcid=%s", dcid_str); - } else if (vc && vc->in_closed_queue) { - QUICDebugDS(scid, dcid, "sent Stateless Reset : connection is already closed, dcid=%s", dcid_str); - } + if (!vc && !QUICInvariants::is_long_header(buf)) { + auto connection = static_cast(this->_check_stateless_reset(buf, buf_len)); + if (connection) { + QUICDebug("Stateless Reset has been received"); + connection->thread->schedule_imm(connection, QUIC_EVENT_STATELESS_RESET); + return; } - QUICStatelessResetToken token(dcid, params->instance_id()); - auto packet = QUICPacketFactory::create_stateless_reset_packet(dcid, token); - this->_send_packet(*packet, udp_packet->getConnection(), udp_packet->from, 1200, nullptr, 0); + bool sent = + this->_send_stateless_reset(dcid, params->instance_id(), udp_packet->getConnection(), udp_packet->from, buf_len - 1); udp_packet->free(); + + if (is_debug_tag_set(debug_tag) && sent) { + QUICPHDebug(scid, dcid, "sent Stateless Reset : connection not found, dcid=%s", dcid.hex().c_str()); + } + + return; + + } else if (vc && vc->in_closed_queue) { + bool sent = + this->_send_stateless_reset(dcid, params->instance_id(), udp_packet->getConnection(), udp_packet->from, buf_len - 1); + udp_packet->free(); + + if (is_debug_tag_set(debug_tag) && sent) { + QUICPHDebug(scid, dcid, "sent Stateless Reset : connection is already closed, dcid=%s", dcid.hex().c_str()); + } + return; } @@ -324,13 +353,12 @@ QUICPacketHandlerIn::_recv_packet(int event, UDPPacket *udp_packet) QUICConnectionId peer_cid = scid; if (is_debug_tag_set("quic_sec")) { - char client_dcid_hex_str[QUICConnectionId::MAX_HEX_STR_LENGTH]; - original_cid.hex(client_dcid_hex_str, QUICConnectionId::MAX_HEX_STR_LENGTH); - QUICDebugDS(peer_cid, original_cid, "client initial dcid=%s", client_dcid_hex_str); + QUICPHDebug(peer_cid, original_cid, "client initial dcid=%s", original_cid.hex().c_str()); } vc = static_cast(getNetProcessor()->allocate_vc(nullptr)); - vc->init(peer_cid, original_cid, cid_in_retry_token, udp_packet->getConnection(), this, &this->_ctable); + vc->init(version, peer_cid, original_cid, ocid_in_retry_token, rcid_in_retry_token, udp_packet->getConnection(), this, + &this->_rtable, &this->_ctable); vc->id = net_next_connection_number(); vc->con.move(con); vc->submit_time = Thread::get_hrtime(); @@ -364,50 +392,59 @@ QUICPacketHandler::send_packet(const QUICPacket &packet, QUICNetVConnection *vc, } void -QUICPacketHandler::send_packet(QUICNetVConnection *vc, Ptr udp_payload) +QUICPacketHandler::send_packet(QUICNetVConnection *vc, const Ptr &udp_payload) { this->_send_packet(vc->get_udp_con(), vc->con.addr, udp_payload); } int QUICPacketHandlerIn::_stateless_retry(const uint8_t *buf, uint64_t buf_len, UDPConnection *connection, IpEndpoint from, - QUICConnectionId dcid, QUICConnectionId scid, QUICConnectionId *original_cid) + QUICConnectionId dcid, QUICConnectionId scid, QUICConnectionId *original_cid, + QUICConnectionId *retry_cid, QUICVersion version) { QUICPacketType type = QUICPacketType::UNINITIALIZED; - QUICPacketLongHeader::type(type, buf, buf_len); + QUICPacketR::type(type, buf, buf_len); if (type != QUICPacketType::INITIAL) { return 1; } // TODO: refine packet parsers in here, QUICPacketLongHeader, and QUICPacketReceiveQueue - size_t token_length = 0; - uint8_t token_length_field_len = 0; - if (!QUICPacketLongHeader::token_length(token_length, &token_length_field_len, buf, buf_len)) { + size_t token_length = 0; + uint8_t token_length_field_len = 0; + size_t token_length_field_offset = 0; + if (!QUICInitialPacketR::token_length(token_length, token_length_field_len, token_length_field_offset, buf, buf_len)) { return -1; } if (token_length == 0) { - QUICRetryToken token(from, dcid); QUICConnectionId local_cid; local_cid.randomize(); - QUICPacketUPtr retry_packet = QUICPacketFactory::create_retry_packet(scid, local_cid, dcid, token); + QUICRetryToken token(from, dcid, local_cid); + QUICPacketUPtr retry_packet = QUICPacketFactory::create_retry_packet(version, scid, local_cid, token); + QUICDebug("[TX] %s packet ODCID=%" PRIx64 " RCID=%" PRIx64 " token_length=%u token=%02x%02x%02x%02x...", + QUICDebugNames::packet_type(retry_packet->type()), static_cast(token.original_dcid()), + static_cast(token.scid()), token.length(), token.buf()[0], token.buf()[1], token.buf()[2], token.buf()[3]); this->_send_packet(*retry_packet, connection, from, 1200, nullptr, 0); return -2; } else { - uint8_t dcil, scil; - QUICPacketLongHeader::dcil(dcil, buf, buf_len); - QUICPacketLongHeader::scil(scil, buf, buf_len); - const uint8_t *token = buf + LONG_HDR_OFFSET_CONNECTION_ID + dcil + scil + token_length_field_len; - - if (QUICAddressValidationToken::type(token) == QUICAddressValidationToken::Type::RETRY) { - QUICRetryToken token1(token, token_length); - if (token1.is_valid(from)) { - *original_cid = token1.original_dcid(); + size_t token_offset = token_length_field_offset + token_length_field_len; + + if (QUICAddressValidationToken::type(buf + token_offset) == QUICAddressValidationToken::Type::RETRY) { + QUICRetryToken token(buf + token_offset, token_length); + if (token.is_valid(from)) { + *original_cid = token.original_dcid(); + *retry_cid = token.scid(); + QUICDebug("Retry Token is valid. ODCID=%" PRIx64 " RCID=%" PRIx64, static_cast(*original_cid), + static_cast(*retry_cid)); return 0; } else { + QUICDebug("Retry token is invalid: ODCID=%" PRIx64 " RCID=%" PRIx64 " token_length=%u token=%02x%02x%02x%02x...", + static_cast(token.original_dcid()), static_cast(*retry_cid), token.length(), token.buf()[0], + token.buf()[1], token.buf()[2], token.buf()[3]); + this->_send_invalid_token_error(buf, buf_len, connection, from); return -3; } } else { @@ -419,10 +456,63 @@ QUICPacketHandlerIn::_stateless_retry(const uint8_t *buf, uint64_t buf_len, UDPC return 0; } +bool +QUICPacketHandlerIn::_send_stateless_reset(QUICConnectionId dcid, uint32_t instance_id, UDPConnection *udp_con, IpEndpoint &addr, + size_t maximum_size) +{ + QUICStatelessResetToken token(dcid, instance_id); + auto packet = QUICPacketFactory::create_stateless_reset_packet(token, maximum_size); + if (packet) { + this->_send_packet(*packet, udp_con, addr, 1200, nullptr, 0); + return true; + } + return false; +} + +void +QUICPacketHandlerIn::_send_invalid_token_error(const uint8_t *initial_packet, uint64_t initial_packet_len, + UDPConnection *connection, IpEndpoint from) +{ + QUICConnectionId scid_in_initial; + QUICConnectionId dcid_in_initial; + QUICInvariants::scid(scid_in_initial, initial_packet, initial_packet_len); + QUICInvariants::dcid(dcid_in_initial, initial_packet, initial_packet_len); + QUICVersion version_in_initial; + QUICLongHeaderPacketR::version(version_in_initial, initial_packet, initial_packet_len); + + // Create CONNECTION_CLOSE frame + auto error = std::make_unique(QUICTransErrorCode::INVALID_TOKEN); + uint8_t frame_buf[QUICFrame::MAX_INSTANCE_SIZE]; + QUICFrame *frame = QUICFrameFactory::create_connection_close_frame(frame_buf, *error); + Ptr block = frame->to_io_buffer_block(1200); + size_t block_len = 0; + for (Ptr tmp = block; tmp; tmp = tmp->next) { + block_len += tmp->size(); + } + frame->~QUICFrame(); + + // Prepare for packet protection + QUICPacketProtectionKeyInfo ppki; + ppki.set_context(QUICPacketProtectionKeyInfo::Context::SERVER); + QUICPacketFactory pf(ppki); + QUICPacketHeaderProtector php(ppki); + QUICCertConfig::scoped_config server_cert; + QUICTLS tls(ppki, server_cert->ssl_default.get(), NET_VCONNECTION_IN, {}, "", ""); + tls.initialize_key_materials(dcid_in_initial, version_in_initial); + + // Create INITIAL packet + QUICConnectionId scid; + scid.randomize(); + uint8_t packet_buf[QUICPacket::MAX_INSTANCE_SIZE]; + QUICPacketUPtr cc_packet = pf.create_initial_packet(packet_buf, scid_in_initial, scid, 0, block, block_len, false, false, true); + + this->_send_packet(*cc_packet, connection, from, 0, &php, scid_in_initial); +} + // // QUICPacketHandlerOut // -QUICPacketHandlerOut::QUICPacketHandlerOut() : Continuation(new_ProxyMutex()), QUICPacketHandler() +QUICPacketHandlerOut::QUICPacketHandlerOut(QUICResetTokenTable &rtable) : Continuation(new_ProxyMutex()), QUICPacketHandler(rtable) { SET_HANDLER(&QUICPacketHandlerOut::event_handler); } @@ -442,7 +532,7 @@ QUICPacketHandlerOut::event_handler(int event, Event *data) return EVENT_CONT; } case NET_EVENT_DATAGRAM_READ_READY: { - Queue *queue = (Queue *)data; + Queue *queue = reinterpret_cast *>(data); UDPPacket *packet_r; while ((packet_r = queue->dequeue())) { this->_recv_packet(event, packet_r); @@ -467,17 +557,36 @@ QUICPacketHandlerOut::_get_continuation() void QUICPacketHandlerOut::_recv_packet(int event, UDPPacket *udp_packet) { - if (is_debug_tag_set(debug_tag)) { - IOBufferBlock *block = udp_packet->getIOBlockChain(); - const uint8_t *buf = reinterpret_cast(block->buf()); + IOBufferBlock *block = udp_packet->getIOBlockChain(); + const uint8_t *buf = reinterpret_cast(block->buf()); + uint64_t buf_len = block->size(); + if (is_debug_tag_set(debug_tag)) { ip_port_text_buffer ipb_from; ip_port_text_buffer ipb_to; - QUICDebugQC(this->_vc, "recv %s packet from %s to %s size=%" PRId64, (QUICInvariants::is_long_header(buf) ? "LH" : "SH"), + QUICQCDebug(this->_vc, "recv %s packet from %s to %s size=%" PRId64, (QUICInvariants::is_long_header(buf) ? "LH" : "SH"), ats_ip_nptop(&udp_packet->from.sa, ipb_from, sizeof(ipb_from)), ats_ip_nptop(&udp_packet->to.sa, ipb_to, sizeof(ipb_to)), udp_packet->getPktLength()); } + QUICConnectionId dcid; + if (!QUICInvariants::dcid(dcid, buf, buf_len)) { + QUICDebug("Ignore packet - payload is too small"); + udp_packet->free(); + return; + } + + if (!QUICInvariants::is_long_header(buf) && dcid != this->_vc->connection_id()) { + auto connection = static_cast(this->_check_stateless_reset(buf, buf_len)); + if (connection) { + if (connection->connection_id() == this->_vc->connection_id()) { + QUICDebug("Stateless Reset has been received"); + this->_vc->thread->schedule_imm(this->_vc, QUIC_EVENT_STATELESS_RESET); + } + return; + } + } + this->_vc->handle_received_packet(udp_packet); eventProcessor.schedule_imm(this->_vc, ET_CALL, QUIC_EVENT_PACKET_READ_READY, nullptr); } diff --git a/iocore/net/SSLClientUtils.cc b/iocore/net/SSLClientUtils.cc index 59f2bce646d..012db75b624 100644 --- a/iocore/net/SSLClientUtils.cc +++ b/iocore/net/SSLClientUtils.cc @@ -22,6 +22,7 @@ #include "tscore/ink_config.h" #include "records/I_RecHttp.h" #include "tscore/ink_platform.h" +#include "tscore/Filenames.h" #include "tscore/X509HostnameValidator.h" #include "P_Net.h" @@ -52,7 +53,7 @@ verify_callback(int signature_ok, X509_STORE_CTX *ctx) // No enforcing, go away if (netvc == nullptr) { // No netvc, very bad. Go away. Things are not good. - Warning("Netvc gone by in verify_callback"); + SSLDebug("WARN, Netvc gone by in verify_callback"); return false; } else if (netvc->options.verifyServerPolicy == YamlSNIConfig::Policy::DISABLED) { return true; // Tell them that all is well @@ -137,6 +138,20 @@ verify_callback(int signature_ok, X509_STORE_CTX *ctx) return true; } +static int +ssl_client_cert_callback(SSL *ssl, void * /*arg*/) +{ + SSLNetVConnection *netvc = SSLNetVCAccess(ssl); + SSL_CTX *ctx = SSL_get_SSL_CTX(ssl); + if (ctx) { + // Do not need to free either the cert or the ssl_ctx + // both are internal pointers + X509 *cert = SSL_CTX_get0_certificate(ctx); + netvc->set_sent_cert(cert != nullptr ? 2 : 1); + } + return 1; +} + SSL_CTX * SSLInitClientContext(const SSLConfigParams *params) { @@ -158,7 +173,7 @@ SSLInitClientContext(const SSLConfigParams *params) SSL_CTX_set_options(client_ctx, params->ssl_client_ctx_options); if (params->client_cipherSuite != nullptr) { if (!SSL_CTX_set_cipher_list(client_ctx, params->client_cipherSuite)) { - SSLError("invalid client cipher suite in records.config"); + SSLError("invalid client cipher suite in %s", ts::filename::RECORDS); goto fail; } } @@ -166,7 +181,7 @@ SSLInitClientContext(const SSLConfigParams *params) #if TS_USE_TLS_SET_CIPHERSUITES if (params->client_tls13_cipher_suites != nullptr) { if (!SSL_CTX_set_ciphersuites(client_ctx, params->client_tls13_cipher_suites)) { - SSLError("invalid tls client cipher suites in records.config"); + SSLError("invalid tls client cipher suites in %s", ts::filename::RECORDS); goto fail; } } @@ -179,7 +194,7 @@ SSLInitClientContext(const SSLConfigParams *params) #else if (!SSL_CTX_set1_curves_list(client_ctx, params->client_groups_list)) { #endif - SSLError("invalid groups list for client in records.config"); + SSLError("invalid groups list for client in %s", ts::filename::RECORDS); goto fail; } } @@ -191,6 +206,8 @@ SSLInitClientContext(const SSLConfigParams *params) SSLConfigParams::init_ssl_ctx_cb(client_ctx, false); } + SSL_CTX_set_cert_cb(client_ctx, ssl_client_cert_callback, nullptr); + return client_ctx; fail: diff --git a/iocore/net/SSLConfig.cc b/iocore/net/SSLConfig.cc index c4492e85a3d..7c058819a91 100644 --- a/iocore/net/SSLConfig.cc +++ b/iocore/net/SSLConfig.cc @@ -41,7 +41,6 @@ #include "HttpConfig.h" #include "P_Net.h" -#include "P_SSLUtils.h" #include "P_SSLClientUtils.h" #include "P_SSLCertLookup.h" #include "SSLDiags.h" @@ -53,6 +52,7 @@ int SSLConfig::configid = 0; int SSLCertificateConfig::configid = 0; int SSLTicketKeyConfig::configid = 0; int SSLConfigParams::ssl_maxrecord = 0; +int SSLConfigParams::ssl_misc_max_iobuffer_size_index = 8; bool SSLConfigParams::ssl_allow_client_renegotiation = false; bool SSLConfigParams::ssl_ocsp_enabled = false; int SSLConfigParams::ssl_ocsp_cache_timeout = 3600; @@ -66,6 +66,11 @@ init_ssl_ctx_func SSLConfigParams::init_ssl_ctx_cb = nullptr; load_ssl_file_func SSLConfigParams::load_ssl_file_cb = nullptr; IpMap *SSLConfigParams::proxy_protocol_ipmap = nullptr; +const uint32_t EARLY_DATA_DEFAULT_SIZE = 16384; +uint32_t SSLConfigParams::server_max_early_data = 0; +uint32_t SSLConfigParams::server_recv_max_early_data = EARLY_DATA_DEFAULT_SIZE; +bool SSLConfigParams::server_allow_early_data_params = false; + int SSLConfigParams::async_handshake_enabled = 0; char *SSLConfigParams::engine_conf_file = nullptr; @@ -249,6 +254,13 @@ SSLConfigParams::initialize() } #endif +#ifdef SSL_OP_PRIORITIZE_CHACHA + REC_ReadConfigInteger(option, "proxy.config.ssl.server.prioritize_chacha"); + if (option) { + ssl_ctx_options |= SSL_OP_PRIORITIZE_CHACHA; + } +#endif + #ifdef SSL_OP_NO_COMPRESSION ssl_ctx_options |= SSL_OP_NO_COMPRESSION; ssl_client_ctx_options |= SSL_OP_NO_COMPRESSION; @@ -278,6 +290,13 @@ SSLConfigParams::initialize() ssl_client_ctx_options |= SSL_OP_NO_SESSION_RESUMPTION_ON_RENEGOTIATION; #endif + REC_ReadConfigInteger(server_max_early_data, "proxy.config.ssl.server.max_early_data"); + REC_ReadConfigInt32(server_allow_early_data_params, "proxy.config.ssl.server.allow_early_data_params"); + + // According to OpenSSL the default value is 16384, + // we keep it unless "server_max_early_data" is higher. + server_recv_max_early_data = std::max(server_max_early_data, EARLY_DATA_DEFAULT_SIZE); + REC_ReadConfigStringAlloc(serverCertChainFilename, "proxy.config.ssl.server.cert_chain.filename"); REC_ReadConfigStringAlloc(serverCertRelativePath, "proxy.config.ssl.server.cert.path"); set_paths_helper(serverCertRelativePath, nullptr, &serverCertPathOnly, nullptr); @@ -333,82 +352,36 @@ SSLConfigParams::initialize() // ++++++++++++++++++++++++ Client part ++++++++++++++++++++ client_verify_depth = 7; - // remove before 9.0.0 release - // Backwards compatibility if proxy.config.ssl.client.verify.server is explicitly set - RecSourceT source = REC_SOURCE_DEFAULT; - bool set_backwards_compatible = false; - if (RecGetRecordSource("proxy.config.ssl.client.verify.server", &source, false) == REC_ERR_OKAY) { - if (source != REC_SOURCE_DEFAULT && source != REC_SOURCE_NULL) { - int8_t verifyServer = 0; - REC_EstablishStaticConfigByte(verifyServer, "proxy.config.ssl.client.verify.server"); - verifyServerProperties = YamlSNIConfig::Property::ALL_MASK; - switch (verifyServer) { - case 0: - verifyServerPolicy = YamlSNIConfig::Policy::DISABLED; - set_backwards_compatible = true; - break; - case 1: - verifyServerPolicy = YamlSNIConfig::Policy::ENFORCED; - set_backwards_compatible = true; - break; - case 2: - verifyServerPolicy = YamlSNIConfig::Policy::PERMISSIVE; - set_backwards_compatible = true; - break; - } - } - } - - bool policy_default = true; - bool properties_default = true; - if (!set_backwards_compatible) { - policy_default = properties_default = false; - } else { // Only check for non-defaults if we have a backwards compatible situation - if (RecGetRecordSource("proxy.config.ssl.client.verify.server.policy", &source, false) == REC_ERR_OKAY && - source != REC_SOURCE_DEFAULT && source != REC_SOURCE_NULL) { - policy_default = false; - } - if (RecGetRecordSource("proxy.config.ssl.client.verify.server.properties", &source, false) == REC_ERR_OKAY && - source != REC_SOURCE_DEFAULT && source != REC_SOURCE_NULL) { - properties_default = false; - } - } - - if (!set_backwards_compatible || !policy_default) { - char *verify_server = nullptr; - REC_ReadConfigStringAlloc(verify_server, "proxy.config.ssl.client.verify.server.policy"); - if (strcmp(verify_server, "DISABLED") == 0) { - verifyServerPolicy = YamlSNIConfig::Policy::DISABLED; - } else if (strcmp(verify_server, "PERMISSIVE") == 0) { - verifyServerPolicy = YamlSNIConfig::Policy::PERMISSIVE; - } else if (strcmp(verify_server, "ENFORCED") == 0) { - verifyServerPolicy = YamlSNIConfig::Policy::ENFORCED; - } else { - Warning("%s is invalid for proxy.config.ssl.client.verify.server.policy. Should be one of DISABLED, PERMISSIVE, or ENFORCED", - verify_server); - verifyServerPolicy = YamlSNIConfig::Policy::DISABLED; - } - ats_free(verify_server); + char *verify_server = nullptr; + REC_ReadConfigStringAlloc(verify_server, "proxy.config.ssl.client.verify.server.policy"); + if (strcmp(verify_server, "DISABLED") == 0) { + verifyServerPolicy = YamlSNIConfig::Policy::DISABLED; + } else if (strcmp(verify_server, "PERMISSIVE") == 0) { + verifyServerPolicy = YamlSNIConfig::Policy::PERMISSIVE; + } else if (strcmp(verify_server, "ENFORCED") == 0) { + verifyServerPolicy = YamlSNIConfig::Policy::ENFORCED; + } else { + Warning("%s is invalid for proxy.config.ssl.client.verify.server.policy. Should be one of DISABLED, PERMISSIVE, or ENFORCED", + verify_server); + verifyServerPolicy = YamlSNIConfig::Policy::DISABLED; } - - if (!set_backwards_compatible || !properties_default) { - char *verify_server = nullptr; - REC_ReadConfigStringAlloc(verify_server, "proxy.config.ssl.client.verify.server.properties"); - if (strcmp(verify_server, "SIGNATURE") == 0) { - verifyServerProperties = YamlSNIConfig::Property::SIGNATURE_MASK; - } else if (strcmp(verify_server, "NAME") == 0) { - verifyServerProperties = YamlSNIConfig::Property::NAME_MASK; - } else if (strcmp(verify_server, "ALL") == 0) { - verifyServerProperties = YamlSNIConfig::Property::ALL_MASK; - } else if (strcmp(verify_server, "NONE") == 0) { - verifyServerProperties = YamlSNIConfig::Property::NONE; - } else { - Warning("%s is invalid for proxy.config.ssl.client.verify.server.properties. Should be one of SIGNATURE, NAME, or ALL", - verify_server); - verifyServerProperties = YamlSNIConfig::Property::NONE; - } - ats_free(verify_server); + ats_free(verify_server); + + REC_ReadConfigStringAlloc(verify_server, "proxy.config.ssl.client.verify.server.properties"); + if (strcmp(verify_server, "SIGNATURE") == 0) { + verifyServerProperties = YamlSNIConfig::Property::SIGNATURE_MASK; + } else if (strcmp(verify_server, "NAME") == 0) { + verifyServerProperties = YamlSNIConfig::Property::NAME_MASK; + } else if (strcmp(verify_server, "ALL") == 0) { + verifyServerProperties = YamlSNIConfig::Property::ALL_MASK; + } else if (strcmp(verify_server, "NONE") == 0) { + verifyServerProperties = YamlSNIConfig::Property::NONE; + } else { + Warning("%s is invalid for proxy.config.ssl.client.verify.server.properties. Should be one of SIGNATURE, NAME, or ALL", + verify_server); + verifyServerProperties = YamlSNIConfig::Property::NONE; } + ats_free(verify_server); ssl_client_cert_filename = nullptr; ssl_client_cert_path = nullptr; @@ -434,6 +407,8 @@ SSLConfigParams::initialize() REC_ReadConfigInt32(ssl_allow_client_renegotiation, "proxy.config.ssl.allow_client_renegotiation"); + REC_ReadConfigInt32(ssl_misc_max_iobuffer_size_index, "proxy.config.ssl.misc.io.max_buffer_index"); + // Enable client regardless of config file settings as remap file // can cause HTTP layer to connect using SSL. But only if SSL // initialization hasn't failed already. @@ -530,9 +505,9 @@ SSLCertificateConfig::reconfigure() } if (retStatus) { - Note("ssl_multicert.config finished loading"); + Note("%s finished loading", params->configFilePath); } else { - Error("ssl_multicert.config failed to load"); + Error("%s failed to load", params->configFilePath); } return retStatus; diff --git a/iocore/net/SSLNetProcessor.cc b/iocore/net/SSLNetProcessor.cc index fd855084135..0707cc3645c 100644 --- a/iocore/net/SSLNetProcessor.cc +++ b/iocore/net/SSLNetProcessor.cc @@ -78,7 +78,10 @@ SSLNetProcessor::start(int, size_t stacksize) #if TS_USE_TLS_OCSP if (SSLConfigParams::ssl_ocsp_enabled) { // Call the update initially to get things populated + Note("Initial OCSP refresh started"); ocsp_update(); + Note("Initial OCSP refresh finished"); + EventType ET_OCSP = eventProcessor.spawn_event_threads("ET_OCSP", 1, stacksize); eventProcessor.schedule_every(new OCSPContinuation(), HRTIME_SECONDS(SSLConfigParams::ssl_ocsp_update_period), ET_OCSP); } diff --git a/iocore/net/SSLNetVConnection.cc b/iocore/net/SSLNetVConnection.cc index 112cca4e43f..59f6f279a33 100644 --- a/iocore/net/SSLNetVConnection.cc +++ b/iocore/net/SSLNetVConnection.cc @@ -34,6 +34,7 @@ #include "P_Net.h" #include "P_SSLUtils.h" +#include "P_SSLNextProtocolSet.h" #include "P_SSLConfig.h" #include "P_SSLClientUtils.h" #include "P_SSLSNI.h" @@ -44,6 +45,7 @@ #include #include +#include using namespace std::literals; @@ -175,9 +177,42 @@ make_ssl_connection(SSL_CTX *ctx, SSLNetVConnection *netvc) BIO *wbio = BIO_new_fd(netvc->get_socket(), BIO_NOCLOSE); BIO_set_mem_eof_return(wbio, -1); SSL_set_bio(ssl, rbio, wbio); + +#if TS_HAS_TLS_EARLY_DATA + // Must disable OpenSSL's internal anti-replay if external cache is used with + // 0-rtt, otherwise session reuse will be broken. The freshness check described + // in https://tools.ietf.org/html/rfc8446#section-8.3 is still performed. But we + // still need to implement something to try to prevent replay atacks. + // + // We are now also disabling this when using OpenSSL's internal cache, since we + // are calling "ssl_accept" non-blocking, it seems to be confusing the anti-replay + // mechanism and causing session resumption to fail. + SSLConfig::scoped_config params; + if (SSL_version(ssl) >= TLS1_3_VERSION && params->server_max_early_data > 0) { + bool ret1 = false; + bool ret2 = false; + if ((ret1 = SSL_set_max_early_data(ssl, params->server_max_early_data)) == 1) { + Debug("ssl_early_data", "SSL_set_max_early_data: success"); + } else { + Debug("ssl_early_data", "SSL_set_max_early_data: failed"); + } + + if ((ret2 = SSL_set_recv_max_early_data(ssl, params->server_recv_max_early_data)) == 1) { + Debug("ssl_early_data", "SSL_set_recv_max_early_data: success"); + } else { + Debug("ssl_early_data", "SSL_set_recv_max_early_data: failed"); + } + + if (ret1 && ret2) { + Debug("ssl_early_data", "Must disable anti-replay if 0-rtt is enabled."); + SSL_set_options(ssl, SSL_OP_NO_ANTI_REPLAY); + } + } +#endif } SSLNetVCAttach(ssl, netvc); + TLSSessionResumptionSupport::bind(ssl, netvc); } return ssl; @@ -375,7 +410,7 @@ SSLNetVConnection::read_raw_data() if (this->get_is_proxy_protocol()) { Debug("proxyprotocol", "proxy protocol is enabled on this port"); if (pp_ipmap->count() > 0) { - Debug("proxyprotocol", "proxy protocol has a configured whitelist of trusted IPs - checking"); + Debug("proxyprotocol", "proxy protocol has a configured allowlist of trusted IPs - checking"); // At this point, using get_remote_addr() will return the ip of the // proxy source IP, not the Proxy Protocol client ip. Since we are @@ -383,17 +418,17 @@ SSLNetVConnection::read_raw_data() // what we want now. void *payload = nullptr; if (!pp_ipmap->contains(get_remote_addr(), &payload)) { - Debug("proxyprotocol", "proxy protocol src IP is NOT in the configured whitelist of trusted IPs - " + Debug("proxyprotocol", "proxy protocol src IP is NOT in the configured allowlist of trusted IPs - " "closing connection"); r = -ENOTCONN; // Need a quick close/exit here to refuse the connection!!!!!!!!! goto proxy_protocol_bypass; } else { char new_host[INET6_ADDRSTRLEN]; - Debug("proxyprotocol", "Source IP [%s] is in the trusted whitelist for proxy protocol", + Debug("proxyprotocol", "Source IP [%s] is in the trusted allowlist for proxy protocol", ats_ip_ntop(this->get_remote_addr(), new_host, sizeof(new_host))); } } else { - Debug("proxyprotocol", "proxy protocol DOES NOT have a configured whitelist of trusted IPs but " + Debug("proxyprotocol", "proxy protocol DOES NOT have a configured allowlist of trusted IPs but " "proxy protocol is enabled on this port - processing all connections"); } @@ -444,7 +479,7 @@ bool SSLNetVConnection::update_rbio(bool move_to_socket) { bool retval = false; - if (BIO_eof(SSL_get_rbio(this->ssl))) { + if (BIO_eof(SSL_get_rbio(this->ssl)) && this->handShakeReader != nullptr) { this->handShakeReader->consume(this->handShakeBioStored); this->handShakeBioStored = 0; // Load up the next block if present @@ -567,7 +602,6 @@ SSLNetVConnection::net_read_io(NetHandler *nh, EThread *lthread) this->read.triggered = 0; readSignalError(nh, err); } else if (ret == SSL_HANDSHAKE_WANT_READ || ret == SSL_HANDSHAKE_WANT_ACCEPT) { - ink_assert(this->handShakeReader != nullptr); if (SSLConfigParams::ssl_handshake_timeout_in > 0) { double handshake_time = (static_cast(Thread::get_hrtime() - sslHandshakeBeginTime) / 1000000000); Debug("ssl", "ssl handshake for vc %p, took %.3f seconds, configured handshake_timer: %d", this, handshake_time, @@ -581,7 +615,7 @@ SSLNetVConnection::net_read_io(NetHandler *nh, EThread *lthread) } } // move over to the socket if we haven't already - if (this->handShakeBuffer) { + if (this->handShakeBuffer != nullptr) { read.triggered = update_rbio(true); } else { read.triggered = 0; @@ -874,6 +908,11 @@ SSLNetVConnection::do_io_close(int lerrno) // Send the close-notify int ret = SSL_shutdown(ssl); Debug("ssl-shutdown", "SSL_shutdown %s", (ret) ? "success" : "failed"); + } else { + // Request a quiet shutdown to OpenSSL + SSL_set_quiet_shutdown(ssl, 1); + SSL_set_shutdown(ssl, SSL_RECEIVED_SHUTDOWN | SSL_SENT_SHUTDOWN); + Debug("ssl-shutdown", "Enable quiet shutdown"); } } } @@ -884,18 +923,20 @@ SSLNetVConnection::do_io_close(int lerrno) void SSLNetVConnection::clear() { + _serverName.reset(); + if (ssl != nullptr) { SSL_free(ssl); ssl = nullptr; } ALPNSupport::clear(); + TLSSessionResumptionSupport::clear(); sslHandshakeStatus = SSL_HANDSHAKE_ONGOING; sslHandshakeBeginTime = 0; sslLastWriteTime = 0; sslTotalBytesSent = 0; sslClientRenegotiationAbort = false; - sslSessionCacheHit = false; curHook = nullptr; hookOpRequested = SSL_HOOK_OP_DEFAULT; @@ -918,6 +959,17 @@ SSLNetVConnection::free(EThread *t) ats_free(tunnel_host); + if (early_data_reader != nullptr) { + early_data_reader->dealloc(); + } + + if (early_data_buf != nullptr) { + free_MIOBuffer(early_data_buf); + } + + early_data_reader = nullptr; + early_data_buf = nullptr; + clear(); SET_CONTINUATION_HANDLER(this, (SSLNetVConnHandler)&SSLNetVConnection::startEvent); ink_assert(con.fd == NO_FD); @@ -1016,7 +1068,7 @@ SSLNetVConnection::sslStartHandShake(int event, int &err) // First Look to see if there are override parameters if (options.ssl_client_cert_name) { - std::string certFilePath = Layout::get()->relative_to(params->clientCertPathOnly, options.ssl_client_cert_name); + std::string certFilePath = Layout::get()->relative_to(params->clientCertPathOnly, options.ssl_client_cert_name.get()); std::string keyFilePath; if (options.ssl_client_private_key_name) { keyFilePath = Layout::get()->relative_to(params->clientKeyPathOnly, options.ssl_client_private_key_name); @@ -1073,11 +1125,12 @@ SSLNetVConnection::sslStartHandShake(int event, int &err) SSL_set_verify(this->ssl, SSL_VERIFY_PEER, verify_callback); - if (this->options.sni_servername) { - if (SSL_set_tlsext_host_name(this->ssl, this->options.sni_servername)) { - Debug("ssl", "using SNI name '%s' for client handshake", this->options.sni_servername.get()); + ats_scoped_str &tlsext_host_name = this->options.sni_hostname ? this->options.sni_hostname : this->options.sni_servername; + if (tlsext_host_name) { + if (SSL_set_tlsext_host_name(this->ssl, tlsext_host_name)) { + Debug("ssl", "using SNI name '%s' for client handshake", tlsext_host_name.get()); } else { - Debug("ssl.error", "failed to set SNI name '%s' for client handshake", this->options.sni_servername.get()); + Debug("ssl.error", "failed to set SNI name '%s' for client handshake", tlsext_host_name.get()); SSL_INCREMENT_DYN_STAT(ssl_sni_name_set_failure); } } @@ -1102,6 +1155,7 @@ SSLNetVConnection::sslServerHandShakeEvent(int &err) // Go do the preaccept hooks if (sslHandshakeHookState == HANDSHAKE_HOOKS_PRE) { + SSL_INCREMENT_DYN_STAT(ssl_total_attempts_handshake_count_in_stat); if (!curHook) { Debug("ssl", "Initialize preaccept curHook from NULL"); curHook = ssl_hooks->get(TSSslHookInternalID(TS_VCONN_START_HOOK)); @@ -1179,18 +1233,22 @@ SSLNetVConnection::sslServerHandShakeEvent(int &err) #if TS_USE_TLS_ASYNC if (ssl_error == SSL_ERROR_WANT_ASYNC) { size_t numfds; - OSSL_ASYNC_FD waitfd; + OSSL_ASYNC_FD *waitfds; // Set up the epoll entry for the signalling - if (SSL_get_all_async_fds(ssl, &waitfd, &numfds) && numfds > 0) { - // Temporarily disable regular net - read_disable(nh, this); - this->ep.stop(); // Modify used in read_disable doesn't work for edge triggered epol - // Have to have the read NetState enabled because we are using it for the signal vc - read.enabled = true; - write_disable(nh, this); - PollDescriptor *pd = get_PollDescriptor(this_ethread()); - this->ep.start(pd, waitfd, this, EVENTIO_READ); - this->ep.type = EVENTIO_READWRITE_VC; + if (SSL_get_all_async_fds(ssl, nullptr, &numfds) && numfds > 0) { + // Allocate space for the waitfd on the stack, should only be one most all of the time + waitfds = reinterpret_cast(alloca(sizeof(OSSL_ASYNC_FD) * numfds)); + if (SSL_get_all_async_fds(ssl, waitfds, &numfds) && numfds > 0) { + // Temporarily disable regular net + this->read.triggered = false; + this->write.triggered = false; + this->ep.stop(); // Modify used in read_disable doesn't work for edge triggered epol + // Have to have the read NetState enabled because we are using it for the signal vc + read.enabled = true; + PollDescriptor *pd = get_PollDescriptor(this_ethread()); + this->ep.start(pd, waitfds[0], static_cast(this), EVENTIO_READ); + this->ep.type = EVENTIO_READWRITE_VC; + } } } else if (SSLConfigParams::async_handshake_enabled) { // Clean up the epoll entry for signalling @@ -1346,6 +1404,7 @@ SSLNetVConnection::sslClientHandShakeEvent(int &err) // Go do the preaccept hooks if (sslHandshakeHookState == HANDSHAKE_HOOKS_OUTBOUND_PRE) { + SSL_INCREMENT_DYN_STAT(ssl_total_attempts_handshake_count_out_stat); if (!curHook) { Debug("ssl", "Initialize outbound connect curHook from NULL"); curHook = ssl_hooks->get(TSSslHookInternalID(TS_VCONN_OUTBOUND_START_HOOK)); @@ -1450,7 +1509,7 @@ SSLNetVConnection::advertise_next_protocol(SSL *ssl, const unsigned char **out, { SSLNetVConnection *netvc = SSLNetVCAccess(ssl); - ink_release_assert(netvc != nullptr); + ink_release_assert(netvc && netvc->ssl == ssl); if (netvc->getNPN(out, outlen)) { // Successful return tells OpenSSL to advertise. @@ -1467,7 +1526,7 @@ SSLNetVConnection::select_next_protocol(SSL *ssl, const unsigned char **out, uns { SSLNetVConnection *netvc = SSLNetVCAccess(ssl); - ink_release_assert(netvc != nullptr); + ink_release_assert(netvc && netvc->ssl == ssl); const unsigned char *npnptr = nullptr; unsigned int npnsize = 0; if (netvc->getNPN(&npnptr, &npnsize)) { @@ -1596,18 +1655,6 @@ SSLNetVConnection::reenable(NetHandler *nh, int event) this->readReschedule(nh); } -bool -SSLNetVConnection::sslContextSet(void *ctx) -{ - bool zret = true; - if (ssl) { - SSL_set_SSL_CTX(ssl, static_cast(ctx)); - } else { - zret = false; - } - return zret; -} - bool SSLNetVConnection::callHooks(TSEvent eventId) { @@ -1747,6 +1794,7 @@ SSLNetVConnection::callHooks(TSEvent eventId) } if (curHook != nullptr) { + WEAK_SCOPED_MUTEX_LOCK(lock, curHook->m_cont->mutex, this_ethread()); curHook->invoke(eventId, this); reenabled = (this->sslHandshakeHookState != HANDSHAKE_HOOKS_CERT_INVOKE && this->sslHandshakeHookState != HANDSHAKE_HOOKS_PRE_INVOKE && @@ -1770,6 +1818,7 @@ SSLNetVConnection::populate(Connection &con, Continuation *c, void *arg) sslHandshakeStatus = SSL_HANDSHAKE_DONE; SSLNetVCAttach(this->ssl, this); + TLSSessionResumptionSupport::bind(this->ssl, this); return EVENT_DONE; } @@ -1859,3 +1908,14 @@ SSLNetVConnection::protocol_contains(std::string_view prefix) const } return retval; } + +void +SSLNetVConnection::set_server_name(std::string_view name) +{ + if (name.size()) { + char *n = new char[name.size() + 1]; + std::memcpy(n, name.data(), name.size()); + n[name.size()] = '\0'; + _serverName.reset(n); + } +} diff --git a/iocore/net/SSLNextProtocolAccept.cc b/iocore/net/SSLNextProtocolAccept.cc index 942bf0270c8..dc21c4905c4 100644 --- a/iocore/net/SSLNextProtocolAccept.cc +++ b/iocore/net/SSLNextProtocolAccept.cc @@ -166,7 +166,10 @@ SSLNextProtocolAccept::enableProtocols(const SessionProtocolSet &protos) } SSLNextProtocolAccept::SSLNextProtocolAccept(Continuation *ep, bool transparent_passthrough) - : SessionAccept(nullptr), buffer(new_empty_MIOBuffer()), endpoint(ep), transparent_passthrough(transparent_passthrough) + : SessionAccept(nullptr), + buffer(new_empty_MIOBuffer(SSLConfigParams::ssl_misc_max_iobuffer_size_index)), + endpoint(ep), + transparent_passthrough(transparent_passthrough) { SET_HANDLER(&SSLNextProtocolAccept::mainEvent); } diff --git a/iocore/net/SSLSNIConfig.cc b/iocore/net/SSLSNIConfig.cc index 32b88260664..a77ed90b489 100644 --- a/iocore/net/SSLSNIConfig.cc +++ b/iocore/net/SSLSNIConfig.cc @@ -40,6 +40,7 @@ #include static ConfigUpdateHandler *sniConfigUpdate; +static constexpr int OVECSIZE{30}; const NextHopProperty * SNIConfigParams::getPropertyConfig(const std::string &servername) const @@ -70,11 +71,14 @@ SNIConfigParams::loadSNIConfig() if (item.verify_client_level != 255) { ai->actions.push_back(std::make_unique(item.verify_client_level)); } + if (item.host_sni_policy != 255) { + ai->actions.push_back(std::make_unique(item.host_sni_policy)); + } if (!item.protocol_unset) { ai->actions.push_back(std::make_unique(item.protocol_mask)); } if (item.tunnel_destination.length() > 0) { - ai->actions.push_back(std::make_unique(item.tunnel_destination, item.tunnel_decrypt)); + ai->actions.push_back(std::make_unique(item.tunnel_destination, item.tunnel_decrypt, item.tls_upstream)); } ai->actions.push_back(std::make_unique(item.ip_allow, item.fqdn)); @@ -98,6 +102,7 @@ SNIConfigParams::loadSNIConfig() nps->setGlobName(item.fqdn); nps->prop.verifyServerPolicy = item.verify_server_policy; nps->prop.verifyServerProperties = item.verify_server_properties; + nps->prop.tls_upstream = item.tls_upstream; } // end for } @@ -105,17 +110,47 @@ int SNIConfig::configid = 0; /*definition of member functions of SNIConfigParams*/ SNIConfigParams::SNIConfigParams() {} -const actionVector * +std::pair SNIConfigParams::get(const std::string &servername) const { + int ovector[OVECSIZE]; + ActionItem::Context context; + for (const auto &retval : sni_action_list) { - if (retval.match == nullptr && servername.length() == 0) { - return &retval.actions; - } else if (pcre_exec(retval.match, nullptr, servername.c_str(), servername.length(), 0, 0, nullptr, 0) >= 0) { - return &retval.actions; + int length = servername.length(); + if (retval.match == nullptr && length == 0) { + return {&retval.actions, context}; + } else if (auto offset = pcre_exec(retval.match, nullptr, servername.c_str(), length, 0, 0, ovector, OVECSIZE); offset >= 0) { + if (offset == 1) { + // first pair identify the portion of the subject string matched by the entire pattern + if (ovector[0] == 0 && ovector[1] == length) { + // full match + return {&retval.actions, context}; + } else { + continue; + } + } + // If contains groups + if (offset == 0) { + // reset to max if too many. + offset = OVECSIZE / 3; + } + + const char *psubStrMatchStr = nullptr; + std::vector groups; + for (int strnum = 1; strnum < offset; strnum++) { + pcre_get_substring(servername.c_str(), ovector, offset, strnum, &(psubStrMatchStr)); + groups.emplace_back(psubStrMatchStr); + } + context._fqdn_wildcard_captured_groups = std::move(groups); + if (psubStrMatchStr) { + pcre_free_substring(psubStrMatchStr); + } + + return {&retval.actions, context}; } } - return nullptr; + return {nullptr, context}; } int @@ -123,11 +158,11 @@ SNIConfigParams::Initialize() { sni_filename = ats_stringdup(RecConfigReadConfigPath("proxy.config.ssl.servername.filename")); - Note("sni.yaml loading ..."); + Note("%s loading ...", sni_filename); struct stat sbuf; if (stat(sni_filename, &sbuf) == -1 && errno == ENOENT) { - Note("sni.yaml failed to load"); + Note("%s failed to load", sni_filename); Warning("Loading SNI configuration - filename: %s doesn't exist", sni_filename); return 1; } @@ -136,12 +171,12 @@ SNIConfigParams::Initialize() if (!zret.isOK()) { std::stringstream errMsg; errMsg << zret; - Error("sni.yaml failed to load: %s", errMsg.str().c_str()); + Error("%s failed to load: %s", sni_filename, errMsg.str().c_str()); return 1; } loadSNIConfig(); - Note("sni.yaml finished loading"); + Note("%s finished loading", sni_filename); return 0; } @@ -180,3 +215,23 @@ SNIConfig::release(SNIConfigParams *params) { configProcessor.release(configid, params); } + +// See if any of the client-side actions would trigger for this combination of servername and +// client IP +// host_sni_policy is an in/out paramter. It starts with the global policy from the records.config +// setting proxy.config.http.host_sni_policy and is possibly overridden if the sni policy +// contains a host_sni_policy entry +bool +SNIConfig::TestClientAction(const char *servername, const IpEndpoint &ep, int &host_sni_policy) +{ + bool retval = false; + SNIConfig::scoped_config params; + + const auto &actions = params->get(servername); + if (actions.first) { + for (auto &&item : *actions.first) { + retval |= item->TestClientSNIAction(servername, ep, host_sni_policy); + } + } + return retval; +} diff --git a/iocore/net/SSLSessionCache.cc b/iocore/net/SSLSessionCache.cc index 9ba73bc56fb..17fb174bfd6 100644 --- a/iocore/net/SSLSessionCache.cc +++ b/iocore/net/SSLSessionCache.cc @@ -60,7 +60,7 @@ SSLSessionCache::getSessionBuffer(const SSLSessionID &sid, char *buffer, int &le } bool -SSLSessionCache::getSession(const SSLSessionID &sid, SSL_SESSION **sess) const +SSLSessionCache::getSession(const SSLSessionID &sid, SSL_SESSION **sess, ssl_session_cache_exdata **data) const { uint64_t hash = sid.hash(); uint64_t target_bucket = hash % nbuckets; @@ -73,7 +73,7 @@ SSLSessionCache::getSession(const SSLSessionID &sid, SSL_SESSION **sess) const target_bucket, bucket, buf, hash); } - return bucket->getSession(sid, sess); + return bucket->getSession(sid, sess, data); } void @@ -97,7 +97,7 @@ SSLSessionCache::removeSession(const SSLSessionID &sid) } void -SSLSessionCache::insertSession(const SSLSessionID &sid, SSL_SESSION *sess) +SSLSessionCache::insertSession(const SSLSessionID &sid, SSL_SESSION *sess, SSL *ssl) { uint64_t hash = sid.hash(); uint64_t target_bucket = hash % nbuckets; @@ -110,11 +110,11 @@ SSLSessionCache::insertSession(const SSLSessionID &sid, SSL_SESSION *sess) target_bucket, bucket, buf, hash); } - bucket->insertSession(sid, sess); + bucket->insertSession(sid, sess, ssl); } void -SSLSessionBucket::insertSession(const SSLSessionID &id, SSL_SESSION *sess) +SSLSessionBucket::insertSession(const SSLSessionID &id, SSL_SESSION *sess, SSL *ssl) { size_t len = i2d_SSL_SESSION(sess, nullptr); // make sure we're not going to need more than SSL_MAX_SESSION_SIZE bytes /* do not cache a session that's too big. */ @@ -142,6 +142,9 @@ SSLSessionBucket::insertSession(const SSLSessionID &id, SSL_SESSION *sess) PRINT_BUCKET("insertSession before") if (queue.size >= static_cast(SSLConfigParams::session_cache_max_bucket_size)) { + if (ssl_rsb) { + SSL_INCREMENT_DYN_STAT(ssl_session_cache_eviction); + } removeOldestSession(); } @@ -155,12 +158,19 @@ SSLSessionBucket::insertSession(const SSLSessionID &id, SSL_SESSION *sess) } Ptr buf; - buf = new_IOBufferData(buffer_size_to_index(len, MAX_BUFFER_SIZE_INDEX), MEMALIGNED); + Ptr buf_exdata; + size_t len_exdata = sizeof(ssl_session_cache_exdata); + buf = new_IOBufferData(buffer_size_to_index(len, MAX_BUFFER_SIZE_INDEX), MEMALIGNED); ink_release_assert(static_cast(buf->block_size()) >= len); unsigned char *loc = reinterpret_cast(buf->data()); i2d_SSL_SESSION(sess, &loc); + buf_exdata = new_IOBufferData(buffer_size_to_index(len, MAX_BUFFER_SIZE_INDEX), MEMALIGNED); + ink_release_assert(static_cast(buf_exdata->block_size()) >= len_exdata); + ssl_session_cache_exdata *exdata = reinterpret_cast(buf_exdata->data()); + // This could be moved to a function in charge of populating exdata + exdata->curve = (ssl == nullptr) ? 0 : SSLGetCurveNID(ssl); - ats_scoped_obj ssl_session(new SSLSession(id, buf, len)); + ats_scoped_obj ssl_session(new SSLSession(id, buf, len, buf_exdata)); /* do the actual insert */ queue.enqueue(ssl_session.release()); @@ -204,7 +214,7 @@ SSLSessionBucket::getSessionBuffer(const SSLSessionID &id, char *buffer, int &le } bool -SSLSessionBucket::getSession(const SSLSessionID &id, SSL_SESSION **sess) +SSLSessionBucket::getSession(const SSLSessionID &id, SSL_SESSION **sess, ssl_session_cache_exdata **data) { char buf[id.len * 2 + 1]; buf[0] = '\0'; // just to be safe. @@ -234,6 +244,10 @@ SSLSessionBucket::getSession(const SSLSessionID &id, SSL_SESSION **sess) if (node->session_id == id) { const unsigned char *loc = reinterpret_cast(node->asn1_data->data()); *sess = d2i_SSL_SESSION(nullptr, &loc, node->len_asn1_data); + if (data != nullptr) { + ssl_session_cache_exdata *exdata = reinterpret_cast(node->extra_data->data()); + *data = exdata; + } return true; } diff --git a/iocore/net/SSLSessionCache.h b/iocore/net/SSLSessionCache.h index a32809c3c6a..44ba12d764c 100644 --- a/iocore/net/SSLSessionCache.h +++ b/iocore/net/SSLSessionCache.h @@ -32,6 +32,10 @@ #define SSL_MAX_SESSION_SIZE 256 +struct ssl_session_cache_exdata { + ssl_curve_id curve = 0; +}; + struct SSLSessionID : public TSSslSessionID { SSLSessionID(const unsigned char *s, size_t l) { @@ -115,9 +119,10 @@ class SSLSession SSLSessionID session_id; Ptr asn1_data; /* this is the ASN1 representation of the SSL_CTX */ size_t len_asn1_data; + Ptr extra_data; - SSLSession(const SSLSessionID &id, const Ptr &ssl_asn1_data, size_t len_asn1) - : session_id(id), asn1_data(ssl_asn1_data), len_asn1_data(len_asn1) + SSLSession(const SSLSessionID &id, const Ptr &ssl_asn1_data, size_t len_asn1, Ptr &exdata) + : session_id(id), asn1_data(ssl_asn1_data), len_asn1_data(len_asn1), extra_data(exdata) { } @@ -129,8 +134,8 @@ class SSLSessionBucket public: SSLSessionBucket(); ~SSLSessionBucket(); - void insertSession(const SSLSessionID &, SSL_SESSION *ctx); - bool getSession(const SSLSessionID &, SSL_SESSION **ctx); + void insertSession(const SSLSessionID &, SSL_SESSION *ctx, SSL *ssl); + bool getSession(const SSLSessionID &, SSL_SESSION **ctx, ssl_session_cache_exdata **data); int getSessionBuffer(const SSLSessionID &, char *buffer, int &len); void removeSession(const SSLSessionID &); @@ -146,9 +151,9 @@ class SSLSessionBucket class SSLSessionCache { public: - bool getSession(const SSLSessionID &sid, SSL_SESSION **sess) const; + bool getSession(const SSLSessionID &sid, SSL_SESSION **sess, ssl_session_cache_exdata **data) const; int getSessionBuffer(const SSLSessionID &sid, char *buffer, int &len) const; - void insertSession(const SSLSessionID &sid, SSL_SESSION *sess); + void insertSession(const SSLSessionID &sid, SSL_SESSION *sess, SSL *ssl); void removeSession(const SSLSessionID &sid); SSLSessionCache(); ~SSLSessionCache(); diff --git a/iocore/net/SSLSessionTicket.cc b/iocore/net/SSLSessionTicket.cc index 5610764fc30..2e7d3973d42 100644 --- a/iocore/net/SSLSessionTicket.cc +++ b/iocore/net/SSLSessionTicket.cc @@ -25,20 +25,8 @@ #if TS_HAVE_OPENSSL_SESSION_TICKETS -#include "tscore/MatcherUtils.h" - -#include "P_Net.h" -#include "SSLStats.h" -#include "P_SSLConfig.h" - -// Remove this when drop OpenSSL 1.0.2 support -#ifndef evp_md_func -#ifdef OPENSSL_NO_SHA256 -#define evp_md_func EVP_sha1() -#else -#define evp_md_func EVP_sha256() -#endif -#endif +#include "P_SSLCertLookup.h" +#include "TLSSessionResumptionSupport.h" void ssl_session_ticket_free(void * /*parent*/, void *ptr, CRYPTO_EX_DATA * /*ad*/, int /*idx*/, long /*argl*/, void * /*argp*/) @@ -55,62 +43,18 @@ int ssl_callback_session_ticket(SSL *ssl, unsigned char *keyname, unsigned char *iv, EVP_CIPHER_CTX *cipher_ctx, HMAC_CTX *hctx, int enc) { - SSLCertificateConfig::scoped_config lookup; - SSLTicketKeyConfig::scoped_config params; - SSLNetVConnection &netvc = *SSLNetVCAccess(ssl); + TLSSessionResumptionSupport *srs = TLSSessionResumptionSupport::getInstance(ssl); - // Get the IP address to look up the keyblock - IpEndpoint ip; - int namelen = sizeof(ip); - SSLCertContext *cc = nullptr; - if (0 == safe_getsockname(netvc.get_socket(), &ip.sa, &namelen)) { - cc = lookup->find(ip); - } - ssl_ticket_key_block *keyblock = nullptr; - if (cc == nullptr || cc->keyblock == nullptr) { - // Try the default - keyblock = params->default_global_keyblock; + if (srs) { + return srs->processSessionTicket(ssl, keyname, iv, cipher_ctx, hctx, enc); } else { - keyblock = cc->keyblock.get(); - } - ink_release_assert(keyblock != nullptr && keyblock->num_keys > 0); - - if (enc == 1) { - const ssl_ticket_key_t &most_recent_key = keyblock->keys[0]; - memcpy(keyname, most_recent_key.key_name, sizeof(most_recent_key.key_name)); - RAND_bytes(iv, EVP_MAX_IV_LENGTH); - EVP_EncryptInit_ex(cipher_ctx, EVP_aes_128_cbc(), nullptr, most_recent_key.aes_key, iv); - HMAC_Init_ex(hctx, most_recent_key.hmac_secret, sizeof(most_recent_key.hmac_secret), evp_md_func, nullptr); + // We could implement a default behavior that would have been done if this callback was not registred, but it's not necessary at + // the moment because TLSSessionResumptionSupport is alawys available when the callback is registerd. + ink_assert(!"srs should be available"); - Debug("ssl", "create ticket for a new session."); - SSL_INCREMENT_DYN_STAT(ssl_total_tickets_created_stat); - return 1; - } else if (enc == 0) { - for (unsigned i = 0; i < keyblock->num_keys; ++i) { - if (memcmp(keyname, keyblock->keys[i].key_name, sizeof(keyblock->keys[i].key_name)) == 0) { - EVP_DecryptInit_ex(cipher_ctx, EVP_aes_128_cbc(), nullptr, keyblock->keys[i].aes_key, iv); - HMAC_Init_ex(hctx, keyblock->keys[i].hmac_secret, sizeof(keyblock->keys[i].hmac_secret), evp_md_func, nullptr); - - Debug("ssl", "verify the ticket for an existing session."); - // Increase the total number of decrypted tickets. - SSL_INCREMENT_DYN_STAT(ssl_total_tickets_verified_stat); - - if (i != 0) { // The number of tickets decrypted with "older" keys. - SSL_INCREMENT_DYN_STAT(ssl_total_tickets_verified_old_key_stat); - } - - netvc.setSSLSessionCacheHit(true); - // When we decrypt with an "older" key, encrypt the ticket again with the most recent key. - return (i == 0) ? 1 : 2; - } - } - - Debug("ssl", "keyname is not consistent."); - SSL_INCREMENT_DYN_STAT(ssl_total_tickets_not_found_stat); - return 0; + // For now, make it an error (this would cause handshake failure) + return -1; } - - return -1; } #endif /* TS_HAVE_OPENSSL_SESSION_TICKETS */ diff --git a/iocore/net/SSLStats.cc b/iocore/net/SSLStats.cc index 9ab09eda41a..f9d53045753 100644 --- a/iocore/net/SSLStats.cc +++ b/iocore/net/SSLStats.cc @@ -132,8 +132,12 @@ SSLInitializeStatistics() // SSL handshake time RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.total_handshake_time", RECD_COUNTER, RECP_PERSISTENT, (int)ssl_total_handshake_time_stat, RecRawStatSyncSum); + RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.total_attempts_handshake_count_in", RECD_COUNTER, RECP_PERSISTENT, + (int)ssl_total_attempts_handshake_count_in_stat, RecRawStatSyncCount); RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.total_success_handshake_count_in", RECD_COUNTER, RECP_PERSISTENT, (int)ssl_total_success_handshake_count_in_stat, RecRawStatSyncCount); + RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.total_attempts_handshake_count_out", RECD_COUNTER, RECP_PERSISTENT, + (int)ssl_total_attempts_handshake_count_out_stat, RecRawStatSyncCount); RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.total_success_handshake_count_out", RECD_COUNTER, RECP_PERSISTENT, (int)ssl_total_success_handshake_count_out_stat, RecRawStatSyncCount); @@ -169,7 +173,7 @@ SSLInitializeStatistics() RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.ssl_session_cache_lock_contention", RECD_COUNTER, RECP_PERSISTENT, (int)ssl_session_cache_lock_contention, RecRawStatSyncCount); - /* Track dynamic record size */ + // Track dynamic record size RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.default_record_size_count", RECD_COUNTER, RECP_PERSISTENT, (int)ssl_total_dyn_def_tls_record_count, RecRawStatSyncSum); RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.max_record_size_count", RECD_COUNTER, RECP_PERSISTENT, @@ -177,7 +181,7 @@ SSLInitializeStatistics() RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.redo_record_size_count", RECD_COUNTER, RECP_PERSISTENT, (int)ssl_total_dyn_redo_tls_record_count, RecRawStatSyncCount); - /* error stats */ + // error stats RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.ssl_error_syscall", RECD_COUNTER, RECP_PERSISTENT, (int)ssl_error_syscall, RecRawStatSyncCount); RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.ssl_error_read_eos", RECD_COUNTER, RECP_PERSISTENT, @@ -187,7 +191,7 @@ SSLInitializeStatistics() RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.ssl_sni_name_set_failure", RECD_COUNTER, RECP_PERSISTENT, (int)ssl_sni_name_set_failure, RecRawStatSyncCount); - /* ocsp stapling stats */ + // ocsp stapling stats RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.ssl_ocsp_revoked_cert_stat", RECD_COUNTER, RECP_PERSISTENT, (int)ssl_ocsp_revoked_cert_stat, RecRawStatSyncCount); RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.ssl_ocsp_unknown_cert_stat", RECD_COUNTER, RECP_PERSISTENT, @@ -197,7 +201,7 @@ SSLInitializeStatistics() RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.ssl_ocsp_refresh_cert_failure", RECD_INT, RECP_PERSISTENT, (int)ssl_ocsp_refresh_cert_failure_stat, RecRawStatSyncCount); - /* SSL Version stats */ + // SSL Version stats RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.ssl_total_sslv3", RECD_COUNTER, RECP_PERSISTENT, (int)ssl_total_sslv3, RecRawStatSyncCount); RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.ssl_total_tlsv1", RECD_COUNTER, RECP_PERSISTENT, @@ -209,6 +213,10 @@ SSLInitializeStatistics() RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.ssl_total_tlsv13", RECD_COUNTER, RECP_PERSISTENT, (int)ssl_total_tlsv13, RecRawStatSyncCount); + // TLSv1.3 0-RTT stats + RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.early_data_received", RECD_INT, RECP_PERSISTENT, + (int)ssl_early_data_received_count, RecRawStatSyncCount); + // Get and register the SSL cipher stats. Note that we are using the default SSL context to obtain // the cipher list. This means that the set of ciphers is fixed by the build configuration and not // filtered by proxy.config.ssl.server.cipher_suite. This keeps the set of cipher suites stable across diff --git a/iocore/net/SSLStats.h b/iocore/net/SSLStats.h index 2ad0ce0b87e..202aa15d563 100644 --- a/iocore/net/SSLStats.h +++ b/iocore/net/SSLStats.h @@ -68,6 +68,7 @@ enum SSL_Stats { ssl_user_agent_session_miss_stat, ssl_user_agent_session_timeout_stat, ssl_total_handshake_time_stat, + ssl_total_attempts_handshake_count_in_stat, ssl_total_success_handshake_count_in_stat, ssl_total_tickets_created_stat, ssl_total_tickets_verified_stat, @@ -83,12 +84,14 @@ enum SSL_Stats { ssl_session_cache_eviction, ssl_session_cache_lock_contention, ssl_session_cache_new_session, + ssl_early_data_received_count, // how many times we received early data /* error stats */ ssl_error_syscall, ssl_error_read_eos, ssl_error_ssl, ssl_sni_name_set_failure, + ssl_total_attempts_handshake_count_out_stat, ssl_total_success_handshake_count_out_stat, /* ocsp stapling stats */ diff --git a/iocore/net/SSLUtils.cc b/iocore/net/SSLUtils.cc index d25b69219d1..b3ee80686c9 100644 --- a/iocore/net/SSLUtils.cc +++ b/iocore/net/SSLUtils.cc @@ -27,7 +27,9 @@ #include "tscore/I_Layout.h" #include "tscore/ink_cap.h" #include "tscore/ink_mutex.h" +#include "tscore/Filenames.h" #include "records/I_RecHttp.h" +#include "tscore/ts_file.h" #include "P_Net.h" #include "InkAPIInternal.h" @@ -58,6 +60,7 @@ #include #include #include +#include #if HAVE_OPENSSL_TS_H #include @@ -74,6 +77,7 @@ static constexpr std::string_view SSL_CA_TAG("ssl_ca_name"sv); static constexpr std::string_view SSL_ACTION_TAG("action"sv); static constexpr std::string_view SSL_ACTION_TUNNEL_TAG("tunnel"sv); static constexpr std::string_view SSL_SESSION_TICKET_ENABLED("ssl_ticket_enabled"sv); +static constexpr std::string_view SSL_SESSION_TICKET_NUMBER("ssl_ticket_number"sv); static constexpr std::string_view SSL_KEY_DIALOG("ssl_key_dialog"sv); static constexpr std::string_view SSL_SERVERNAME("dest_fqdn"sv); static constexpr char SSL_CERT_SEPARATE_DELIM = ','; @@ -88,10 +92,6 @@ static constexpr char SSL_CERT_SEPARATE_DELIM = ','; SSLSessionCache *session_cache; // declared extern in P_SSLConfig.h -#if TS_HAVE_OPENSSL_SESSION_TICKETS -static int ssl_session_ticket_index = -1; -#endif - static int ssl_vc_index = -1; static ink_mutex *mutex_buf = nullptr; @@ -170,14 +170,6 @@ SSL_CTX_add_extra_chain_cert_file(SSL_CTX *ctx, const char *chainfile) return SSL_CTX_add_extra_chain_cert_bio(ctx, bio); } -static bool -ssl_session_timed_out(SSL_SESSION *session) -{ - return SSL_SESSION_get_timeout(session) < (time(nullptr) - SSL_SESSION_get_time(session)); -} - -static void ssl_rm_cached_session(SSL_CTX *ctx, SSL_SESSION *sess); - static SSL_SESSION * #if OPENSSL_VERSION_NUMBER < 0x10100000L || defined(LIBRESSL_VERSION_NUMBER) ssl_get_cached_session(SSL *ssl, unsigned char *id, int len, int *copy) @@ -185,43 +177,14 @@ ssl_get_cached_session(SSL *ssl, unsigned char *id, int len, int *copy) ssl_get_cached_session(SSL *ssl, const unsigned char *id, int len, int *copy) #endif { - SSLSessionID sid(id, len); + TLSSessionResumptionSupport *srs = TLSSessionResumptionSupport::getInstance(ssl); - *copy = 0; - if (diags->tag_activated("ssl.session_cache")) { - char printable_buf[(len * 2) + 1]; - sid.toString(printable_buf, sizeof(printable_buf)); - Debug("ssl.session_cache.get", "ssl_get_cached_session cached session '%s' context %p", printable_buf, SSL_get_SSL_CTX(ssl)); + ink_assert(srs); + if (srs) { + return srs->getSession(ssl, id, len, copy); } - APIHook *hook = ssl_hooks->get(TSSslHookInternalID(TS_SSL_SESSION_HOOK)); - while (hook) { - hook->invoke(TS_EVENT_SSL_SESSION_GET, &sid); - hook = hook->m_link.next; - } - - SSL_SESSION *session = nullptr; - if (session_cache->getSession(sid, &session)) { - ink_assert(session); - - // Double check the timeout - if (ssl_session_timed_out(session)) { - SSL_INCREMENT_DYN_STAT(ssl_session_cache_miss); -// Due to bug in openssl, the timeout is checked, but only removed -// from the openssl built-in hash table. The external remove cb is not called -#if 0 // This is currently eliminated, since it breaks things in odd ways (see TS-3710) - ssl_rm_cached_session(SSL_get_SSL_CTX(ssl), session); -#endif - session = nullptr; - } else { - SSLNetVConnection *netvc = SSLNetVCAccess(ssl); - SSL_INCREMENT_DYN_STAT(ssl_session_cache_hit); - netvc->setSSLSessionCacheHit(true); - } - } else { - SSL_INCREMENT_DYN_STAT(ssl_session_cache_miss); - } - return session; + return nullptr; } static int @@ -229,6 +192,7 @@ ssl_new_cached_session(SSL *ssl, SSL_SESSION *sess) { unsigned int len = 0; const unsigned char *id = SSL_SESSION_get_id(sess, &len); + SSLSessionID sid(id, len); if (diags->tag_activated("ssl.session_cache")) { @@ -239,7 +203,7 @@ ssl_new_cached_session(SSL *ssl, SSL_SESSION *sess) } SSL_INCREMENT_DYN_STAT(ssl_session_cache_new_session); - session_cache->insertSession(sid, sess); + session_cache->insertSession(sid, sess, ssl); // Call hook after new session is created APIHook *hook = ssl_hooks->get(TSSslHookInternalID(TS_SSL_SESSION_HOOK)); @@ -286,6 +250,12 @@ set_context_cert(SSL *ssl) bool found = true; int retval = 1; + if (!netvc || netvc->ssl != ssl) { + Debug("ssl.error", "set_context_cert call back on stale netvc"); + retval = 0; // Error + goto done; + } + Debug("ssl", "set_context_cert ssl=%p server=%s handshake_complete=%d", ssl, servername, netvc->getSSLHandShakeComplete()); // catch the client renegotiation early on @@ -353,6 +323,11 @@ ssl_verify_client_callback(int preverify_ok, X509_STORE_CTX *ctx) auto *ssl = static_cast(X509_STORE_CTX_get_ex_data(ctx, SSL_get_ex_data_X509_STORE_CTX_idx())); SSLNetVConnection *netvc = SSLNetVCAccess(ssl); + if (!netvc || netvc->ssl != ssl) { + Debug("ssl.error", "ssl_verify_client_callback call back on stale netvc"); + return false; + } + netvc->set_verify_cert(ctx); netvc->callHooks(TS_EVENT_SSL_VERIFY_CLIENT); netvc->set_verify_cert(nullptr); @@ -369,12 +344,11 @@ static int PerformAction(Continuation *cont, const char *servername) { SNIConfig::scoped_config params; - const actionVector *actionvec = params->get(servername); - if (!actionvec) { + if (const auto &actions = params->get(servername); !actions.first) { Debug("ssl_sni", "%s not available in the map", servername); } else { - for (auto &&item : *actionvec) { - auto ret = item->SNIAction(cont); + for (auto &&item : *actions.first) { + auto ret = item->SNIAction(cont, actions.second); if (ret != SSL_TLSEXT_ERR_OK) { return ret; } @@ -392,6 +366,12 @@ ssl_client_hello_callback(SSL *s, int *al, void *arg) const char *servername = nullptr; const unsigned char *p; size_t remaining, len; + + if (!netvc || netvc->ssl != s) { + Debug("ssl.error", "ssl_client_hello_callback call back on stale netvc"); + return SSL_CLIENT_HELLO_ERROR; + } + // Parse the server name if the get extension call succeeds and there are more than 2 bytes to parse if (SSL_client_hello_get0_ext(s, TLSEXT_TYPE_server_name, &p, &remaining) && remaining > 2) { // Parse to get to the name, originally from test/handshake_helper.c in openssl tree @@ -404,22 +384,23 @@ ssl_client_hello_callback(SSL *s, int *al, void *arg) * The list in practice only has a single element, so we only consider * the first one. */ - if (remaining != 0 && *p++ == TLSEXT_NAMETYPE_host_name) { + if (*p++ == TLSEXT_NAMETYPE_host_name) { remaining--; /* Now we can finally pull out the byte array with the actual hostname. */ if (remaining > 2) { len = *(p++) << 8; len += *(p++); if (len + 2 <= remaining) { - remaining = len; servername = reinterpret_cast(p); } } } } } - netvc->serverName = servername ? servername : ""; - int ret = PerformAction(netvc, netvc->serverName); + if (servername) { + netvc->set_server_name(std::string_view(servername, len)); + } + int ret = PerformAction(netvc, netvc->get_server_name()); if (ret != SSL_TLSEXT_ERR_OK) { return SSL_CLIENT_HELLO_ERROR; } @@ -450,6 +431,11 @@ ssl_cert_callback(SSL *ssl, void * /*arg*/) bool reenabled; int retval = 1; + if (!netvc || netvc->ssl != ssl) { + Debug("ssl.error", "ssl_cert_callback call back on stale netvc"); + return 0; + } + // If we are in tunnel mode, don't select a cert. Pause! if (HttpProxyPort::TRANSPORT_BLIND_TUNNEL == netvc->attributes) { return -1; // Pause @@ -483,16 +469,22 @@ static int ssl_servername_callback(SSL *ssl, int * /* ad */, void * /*arg*/) { SSLNetVConnection *netvc = SSLNetVCAccess(ssl); + + if (!netvc || netvc->ssl != ssl) { + Debug("ssl.error", "ssl_servername_callback call back on stale netvc"); + return SSL_TLSEXT_ERR_ALERT_FATAL; + } + netvc->callHooks(TS_EVENT_SSL_SERVERNAME); - netvc->serverName = SSL_get_servername(ssl, TLSEXT_NAMETYPE_host_name); - if (nullptr == netvc->serverName) { - netvc->serverName = ""; + const char *name = SSL_get_servername(ssl, TLSEXT_NAMETYPE_host_name); + if (name) { + netvc->set_server_name(name); } #if !TS_USE_HELLO_CB // Only call the SNI actions here if not already performed in the HELLO_CB - int ret = PerformAction(netvc, netvc->serverName); + int ret = PerformAction(netvc, netvc->get_server_name()); if (ret != SSL_TLSEXT_ERR_OK) { return SSL_TLSEXT_ERR_ALERT_FATAL; } @@ -904,13 +896,6 @@ SSLInitializeLibrary() CRYPTO_set_dynlock_destroy_callback(ssl_dyn_destroy_callback); } -#ifdef TS_HAVE_OPENSSL_SESSION_TICKETS - ssl_session_ticket_index = SSL_CTX_get_ex_new_index(0, nullptr, nullptr, nullptr, ssl_session_ticket_free); - if (ssl_session_ticket_index == -1) { - SSLError("failed to create session ticket index"); - } -#endif - #if TS_USE_TLS_OCSP ssl_stapling_ex_init(); #endif /* TS_USE_TLS_OCSP */ @@ -919,6 +904,8 @@ SSLInitializeLibrary() // the SSLNetVConnection to the SSL session. ssl_vc_index = SSL_get_ex_new_index(0, (void *)"NetVC index", nullptr, nullptr, nullptr); + TLSSessionResumptionSupport::initialize(); + open_ssl_initialized = true; } @@ -934,7 +921,12 @@ SSLPrivateKeyHandler(SSL_CTX *ctx, const SSLConfigParams *params, const std::str #ifndef OPENSSL_IS_BORINGSSL ENGINE *e = ENGINE_get_default_RSA(); if (e != nullptr) { - const char *argkey = (keyPath == nullptr || keyPath[0] == '\0') ? completeServerCertPath.c_str() : keyPath; + ats_scoped_str argkey; + if (keyPath == nullptr || keyPath[0] == '\0') { + argkey = completeServerCertPath.c_str(); + } else { + argkey = Layout::get()->relative_to(params->serverKeyPathOnly, keyPath); + } if (!SSL_CTX_use_PrivateKey(ctx, ENGINE_load_private_key(e, argkey, nullptr, nullptr))) { SSLError("failed to load server private key from engine"); } @@ -942,7 +934,7 @@ SSLPrivateKeyHandler(SSL_CTX *ctx, const SSLConfigParams *params, const std::str ENGINE *e = nullptr; if (false) { #endif - } else if (!keyPath) { + } else if (!keyPath || keyPath[0] == '\0') { // assume private key is contained in cert obtained from multicert file. if (!SSL_CTX_use_PrivateKey_file(ctx, completeServerCertPath.c_str(), SSL_FILETYPE_PEM)) { SSLError("failed to load server private key from %s", completeServerCertPath.c_str()); @@ -958,7 +950,7 @@ SSLPrivateKeyHandler(SSL_CTX *ctx, const SSLConfigParams *params, const std::str SSLConfigParams::load_ssl_file_cb(completeServerKeyPath); } } else { - SSLError("empty SSL private key path in records.config"); + SSLError("empty SSL private key path in %s", ts::filename::RECORDS); return false; } @@ -1038,63 +1030,15 @@ asn1_strdup(ASN1_STRING *s) @static */ bool -SSLMultiCertConfigLoader::index_certificate(SSLCertLookup *lookup, SSLCertContext const &cc, X509 *cert, const char *certname) +SSLMultiCertConfigLoader::index_certificate(SSLCertLookup *lookup, SSLCertContext const &cc, const char *sni_name) { - X509_NAME *subject = nullptr; - bool inserted = false; + bool inserted = false; - if (nullptr == cert) { - Error("Failed to load certificate %s", certname); - lookup->is_valid = false; - return false; + Debug("ssl", "mapping '%s'", sni_name); + if (lookup->insert(sni_name, cc) >= 0) { + inserted = true; } - // Insert a key for the subject CN. - subject = X509_get_subject_name(cert); - ats_scoped_str subj_name; - if (subject) { - int pos = -1; - for (;;) { - pos = X509_NAME_get_index_by_NID(subject, NID_commonName, pos); - if (pos == -1) { - break; - } - - X509_NAME_ENTRY *e = X509_NAME_get_entry(subject, pos); - ASN1_STRING *cn = X509_NAME_ENTRY_get_data(e); - subj_name = asn1_strdup(cn); - - Debug("ssl", "mapping '%s' to certificate %s", (const char *)subj_name, certname); - if (lookup->insert(subj_name, cc) >= 0) { - inserted = true; - } - } - } - -#if HAVE_OPENSSL_TS_H - // Traverse the subjectAltNames (if any) and insert additional keys for the SSL context. - GENERAL_NAMES *names = static_cast(X509_get_ext_d2i(cert, NID_subject_alt_name, nullptr, nullptr)); - if (names) { - unsigned count = sk_GENERAL_NAME_num(names); - for (unsigned i = 0; i < count; ++i) { - GENERAL_NAME *name; - - name = sk_GENERAL_NAME_value(names, i); - if (name->type == GEN_DNS) { - ats_scoped_str dns(asn1_strdup(name->d.dNSName)); - // only try to insert if the alternate name is not the main name - if (subj_name == nullptr || strcmp(dns, subj_name) != 0) { - Debug("ssl", "mapping '%s' to certificates %s", (const char *)dns, certname); - if (lookup->insert(dns, cc) >= 0) { - inserted = true; - } - } - } - } - - GENERAL_NAMES_free(names); - } -#endif // HAVE_OPENSSL_TS_H return inserted; } @@ -1104,9 +1048,15 @@ SSLMultiCertConfigLoader::index_certificate(SSLCertLookup *lookup, SSLCertContex static void ssl_callback_info(const SSL *ssl, int where, int ret) { - Debug("ssl", "ssl_callback_info ssl: %p where: %d ret: %d State: %s", ssl, where, ret, SSL_state_string_long(ssl)); + Debug("ssl", "ssl_callback_info ssl: %p, where: %d, ret: %d, State: %s", ssl, where, ret, SSL_state_string_long(ssl)); + SSLNetVConnection *netvc = SSLNetVCAccess(ssl); + if (!netvc || netvc->ssl != ssl) { + Debug("ssl.error", "ssl_callback_info call back on stale netvc"); + return; + } + if ((where & SSL_CB_ACCEPT_LOOP) && netvc->getSSLHandShakeComplete() == true && SSLConfigParams::ssl_allow_client_renegotiation == false) { int state = SSL_get_state(ssl); @@ -1126,6 +1076,13 @@ ssl_callback_info(const SSL *ssl, int where, int ret) if (state == TLS_ST_SR_CLNT_HELLO) { #endif #endif +#endif +#ifdef TLS1_3_VERSION + // TLSv1.3 has no renegotiation. + if (SSL_version(ssl) >= TLS1_3_VERSION) { + Debug("ssl", "TLSv1.3 has no renegotiation."); + return; + } #endif netvc->setSSLClientRenegotiationAbort(true); Debug("ssl", "ssl_callback_info trying to renegotiate from the client"); @@ -1189,7 +1146,8 @@ setClientCertLevel(SSL *ssl, uint8_t certLevel) This is public function because of used by SSLCreateServerContext. */ SSL_CTX * -SSLMultiCertConfigLoader::init_server_ssl_ctx(std::vector &cert_list, const SSLMultiCertConfigParams *sslMultCertSettings) +SSLMultiCertConfigLoader::init_server_ssl_ctx(CertLoadData const &data, const SSLMultiCertConfigParams *sslMultCertSettings, + std::set &names) { const SSLConfigParams *params = this->_params; @@ -1232,7 +1190,6 @@ SSLMultiCertConfigLoader::init_server_ssl_ctx(std::vector &cert_list, co SSL_CTX_sess_set_get_cb(ctx, ssl_get_cached_session); SSL_CTX_set_session_cache_mode(ctx, SSL_SESS_CACHE_SERVER | SSL_SESS_CACHE_NO_INTERNAL | additional_cache_flags); - break; } } @@ -1274,7 +1231,7 @@ SSLMultiCertConfigLoader::init_server_ssl_ctx(std::vector &cert_list, co } if (sslMultCertSettings->cert) { - if (!SSLMultiCertConfigLoader::load_certs(ctx, cert_list, params, sslMultCertSettings)) { + if (!SSLMultiCertConfigLoader::load_certs(ctx, data, params, sslMultCertSettings)) { goto fail; } } @@ -1305,6 +1262,12 @@ SSLMultiCertConfigLoader::init_server_ssl_ctx(std::vector &cert_list, co SSL_CTX_set_options(ctx, SSL_OP_NO_TICKET); Debug("ssl", "ssl session ticket is disabled"); } +#endif +#if defined(TLS1_3_VERSION) && !defined(LIBRESSL_VERSION_NUMBER) && !defined(OPENSSL_IS_BORINGSSL) + if (!(params->ssl_ctx_options & SSL_OP_NO_TLSv1_3)) { + SSL_CTX_set_num_tickets(ctx, sslMultCertSettings->session_ticket_number); + Debug("ssl", "ssl session ticket number set to %d", sslMultCertSettings->session_ticket_number); + } #endif } @@ -1324,7 +1287,7 @@ SSLMultiCertConfigLoader::init_server_ssl_ctx(std::vector &cert_list, co } else { // disable client cert support server_verify_client = SSL_VERIFY_NONE; - Error("illegal client certification level %d in records.config", server_verify_client); + Error("illegal client certification level %d in %s", server_verify_client, ts::filename::RECORDS); } SSL_CTX_set_verify(ctx, server_verify_client, ssl_verify_client_callback); SSL_CTX_set_verify_depth(ctx, params->verify_depth); // might want to make configurable at some point. @@ -1336,7 +1299,7 @@ SSLMultiCertConfigLoader::init_server_ssl_ctx(std::vector &cert_list, co if (params->cipherSuite != nullptr) { if (!SSL_CTX_set_cipher_list(ctx, params->cipherSuite)) { - SSLError("invalid cipher suite in records.config"); + SSLError("invalid cipher suite in %s", ts::filename::RECORDS); goto fail; } } @@ -1344,7 +1307,7 @@ SSLMultiCertConfigLoader::init_server_ssl_ctx(std::vector &cert_list, co #if TS_USE_TLS_SET_CIPHERSUITES if (params->server_tls13_cipher_suites != nullptr) { if (!SSL_CTX_set_ciphersuites(ctx, params->server_tls13_cipher_suites)) { - SSLError("invalid tls server cipher suites in records.config"); + SSLError("invalid tls server cipher suites in %s", ts::filename::RECORDS); goto fail; } } @@ -1357,7 +1320,7 @@ SSLMultiCertConfigLoader::init_server_ssl_ctx(std::vector &cert_list, co #else if (!SSL_CTX_set1_curves_list(ctx, params->server_groups_list)) { #endif - SSLError("invalid groups list for server in records.config"); + SSLError("invalid groups list for server in %s", ts::filename::RECORDS); goto fail; } } @@ -1386,10 +1349,6 @@ SSLMultiCertConfigLoader::init_server_ssl_ctx(std::vector &cert_list, co fail: SSLMultiCertConfigLoader::clear_pw_references(ctx); SSLReleaseContext(ctx); - for (auto cert : cert_list) { - X509_free(cert); - } - return nullptr; } @@ -1398,29 +1357,34 @@ SSLCreateServerContext(const SSLConfigParams *params, const SSLMultiCertConfigPa const char *key_path) { SSLMultiCertConfigLoader loader(params); - std::vector cert_list; std::unique_ptr ctx(nullptr, &SSL_CTX_free); - ctx.reset(loader.init_server_ssl_ctx(cert_list, sslMultiCertSettings)); - ink_assert(cert_list.empty()); - - if (cert_path) { + std::vector cert_list; + std::set common_names; + std::unordered_map> unique_names; + SSLMultiCertConfigLoader::CertLoadData data; + if (loader.load_certs_and_cross_reference_names(cert_list, data, params, sslMultiCertSettings, common_names, unique_names)) { + ctx.reset(loader.init_server_ssl_ctx(data, sslMultiCertSettings, common_names)); + } + for (auto &i : cert_list) { + X509_free(i); + } + if (ctx && cert_path) { if (!SSL_CTX_use_certificate_file(ctx.get(), cert_path, SSL_FILETYPE_PEM)) { SSLError("SSLCreateServerContext(): failed to load server certificate."); - return nullptr; - } - if (!key_path || key_path[0] == '\0') { + ctx = nullptr; + } else if (!key_path || key_path[0] == '\0') { key_path = cert_path; } - if (!SSL_CTX_use_PrivateKey_file(ctx.get(), key_path, SSL_FILETYPE_PEM)) { - SSLError("SSLCreateServerContext(): failed to load server private key."); - return nullptr; - } - if (!SSL_CTX_check_private_key(ctx.get())) { - SSLError("SSLCreateServerContext(): server private key does not match server certificate."); - return nullptr; + if (ctx) { + if (!SSL_CTX_use_PrivateKey_file(ctx.get(), key_path, SSL_FILETYPE_PEM)) { + SSLError("SSLCreateServerContext(): failed to load server private key."); + ctx = nullptr; + } else if (!SSL_CTX_check_private_key(ctx.get())) { + SSLError("SSLCreateServerContext(): server private key does not match server certificate."); + ctx = nullptr; + } } } - return ctx.release(); } @@ -1428,29 +1392,73 @@ SSLCreateServerContext(const SSLConfigParams *params, const SSLMultiCertConfigPa Insert SSLCertContext (SSL_CTX ans options) into SSLCertLookup with key. Do NOT call SSL_CTX_set_* functions from here. SSL_CTX should be set up by SSLMultiCertConfigLoader::init_server_ssl_ctx(). */ -SSL_CTX * -SSLMultiCertConfigLoader::_store_ssl_ctx(SSLCertLookup *lookup, const shared_SSLMultiCertConfigParams sslMultCertSettings) +bool +SSLMultiCertConfigLoader::_store_ssl_ctx(SSLCertLookup *lookup, const shared_SSLMultiCertConfigParams &sslMultCertSettings) { + bool retval = true; std::vector cert_list; - shared_ssl_ticket_key_block keyblock = nullptr; - bool inserted = false; - shared_SSL_CTX ctx(this->init_server_ssl_ctx(cert_list, sslMultCertSettings.get()), SSL_CTX_free); + std::set common_names; + std::unordered_map> unique_names; + SSLMultiCertConfigLoader::CertLoadData data; - if (!ctx || !sslMultCertSettings) { - lookup->is_valid = false; - return nullptr; - } + const SSLConfigParams *params = this->_params; + + this->load_certs_and_cross_reference_names(cert_list, data, params, sslMultCertSettings.get(), common_names, unique_names); - const char *certname = sslMultCertSettings->cert.get(); + int i = 0; for (auto cert : cert_list) { - if (0 > SSLMultiCertConfigLoader::check_server_cert_now(cert, certname)) { + const char *current_cert_name = data.cert_names_list[i].c_str(); + if (0 > SSLMultiCertConfigLoader::check_server_cert_now(cert, current_cert_name)) { /* At this point, we know cert is bad, and we've already printed a descriptive reason as to why cert is bad to the log file */ - Debug("ssl", "Marking certificate as NOT VALID: %s", certname); + Debug(this->_debug_tag(), "Marking certificate as NOT VALID: %s", current_cert_name); lookup->is_valid = false; } + i++; + } + + shared_SSL_CTX ctx(this->init_server_ssl_ctx(data, sslMultCertSettings.get(), common_names), SSL_CTX_free); + + if (!ctx || !sslMultCertSettings || !this->_store_single_ssl_ctx(lookup, sslMultCertSettings, ctx, common_names)) { + retval = false; + std::string names; + for (auto name : data.cert_names_list) { + names.append(name); + names.append(" "); + } + Warning("(%s) Failed to insert SSL_CTX for certificate %s entries for names already made", this->_debug_tag(), names.c_str()); + } + + for (auto iter = unique_names.begin(); retval && iter != unique_names.end(); ++iter) { + size_t i = iter->first; + + SSLMultiCertConfigLoader::CertLoadData single_data; + single_data.cert_names_list.push_back(data.cert_names_list[i]); + if (i < data.key_list.size()) { + single_data.key_list.push_back(data.key_list[i]); + } + single_data.ca_list.push_back(i < data.ca_list.size() ? data.ca_list[i] : ""); + single_data.ocsp_list.push_back(i < data.ocsp_list.size() ? data.ocsp_list[i] : ""); + + shared_SSL_CTX unique_ctx(this->init_server_ssl_ctx(single_data, sslMultCertSettings.get(), iter->second), SSL_CTX_free); + if (!unique_ctx || !this->_store_single_ssl_ctx(lookup, sslMultCertSettings, unique_ctx, iter->second)) { + retval = false; + } } + for (auto &i : cert_list) { + X509_free(i); + } + + return retval; +} + +bool +SSLMultiCertConfigLoader::_store_single_ssl_ctx(SSLCertLookup *lookup, const shared_SSLMultiCertConfigParams &sslMultCertSettings, + shared_SSL_CTX ctx, std::set &names) +{ + bool inserted = false; + shared_ssl_ticket_key_block keyblock = nullptr; // Load the session ticket key if session tickets are not disabled if (sslMultCertSettings->session_ticket_enabled != 0) { keyblock = shared_ssl_ticket_key_block(ssl_context_enable_tickets(ctx.get(), nullptr), ticket_block_free); @@ -1468,7 +1476,6 @@ SSLMultiCertConfigLoader::_store_ssl_ctx(SSLCertLookup *lookup, const shared_SSL IpEndpoint ep; if (ats_ip_pton(sslMultCertSettings->addr, &ep) == 0) { - Debug("ssl", "mapping '%s' to certificate %s", (const char *)sslMultCertSettings->addr, (const char *)certname); if (lookup->insert(ep, SSLCertContext(ctx, sslMultCertSettings, keyblock)) >= 0) { inserted = true; } @@ -1482,9 +1489,8 @@ SSLMultiCertConfigLoader::_store_ssl_ctx(SSLCertLookup *lookup, const shared_SSL // Insert additional mappings. Note that this maps multiple keys to the same value, so when // this code is updated to reconfigure the SSL certificates, it will need some sort of // refcounting or alternate way of avoiding double frees. - Debug("ssl", "importing SNI names from %s", (const char *)certname); - for (auto cert : cert_list) { - if (SSLMultiCertConfigLoader::index_certificate(lookup, SSLCertContext(ctx, sslMultCertSettings), cert, certname)) { + for (auto sni_name : names) { + if (SSLMultiCertConfigLoader::index_certificate(lookup, SSLCertContext(ctx, sslMultCertSettings), sni_name.c_str())) { inserted = true; } } @@ -1499,10 +1505,6 @@ SSLMultiCertConfigLoader::_store_ssl_ctx(SSLCertLookup *lookup, const shared_SSL ctx = nullptr; } - for (auto &i : cert_list) { - X509_free(i); - } - return ctx.get(); } @@ -1544,6 +1546,10 @@ ssl_extract_certificate(const matcher_line *line_info, SSLMultiCertConfigParams sslMultCertSettings->session_ticket_enabled = atoi(value); } + if (strcasecmp(label, SSL_SESSION_TICKET_NUMBER) == 0) { + sslMultCertSettings->session_ticket_number = atoi(value); + } + if (strcasecmp(label, SSL_KEY_DIALOG) == 0) { sslMultCertSettings->dialog = ats_strdup(value); } @@ -1578,23 +1584,26 @@ SSLMultiCertConfigLoader::load(SSLCertLookup *lookup) { const SSLConfigParams *params = this->_params; - char *tok_state = nullptr; - char *line = nullptr; - ats_scoped_str file_buf; + char *tok_state = nullptr; + char *line = nullptr; unsigned line_num = 0; matcher_line line_info; const matcher_tags sslCertTags = {nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, false}; - Note("ssl_multicert.config loading ..."); + Note("%s loading ...", ts::filename::SSL_MULTICERT); - if (params->configFilePath) { - file_buf = readIntoBuffer(params->configFilePath, __func__, nullptr); - } - - if (!file_buf) { - Error("failed to read SSL certificate configuration from %s", params->configFilePath); - return false; + std::error_code ec; + std::string content{ts::file::load(ts::file::path{params->configFilePath}, ec)}; + if (ec) { + switch (ec.value()) { + case ENOENT: + Warning("Cannot open SSL certificate configuration from %s - %s", params->configFilePath, strerror(ec.value())); + break; + default: + Error("Failed to read SSL certificate configuration from %s - %s", params->configFilePath, strerror(ec.value())); + return false; + } } // Optionally elevate/allow file access to read root-only @@ -1603,7 +1612,7 @@ SSLMultiCertConfigLoader::load(SSLCertLookup *lookup) REC_ReadConfigInteger(elevate_setting, "proxy.config.ssl.cert.load_elevated"); ElevateAccess elevate_access(elevate_setting ? ElevateAccess::FILE_PRIVILEGE : 0); - line = tokLine(file_buf, &tok_state); + line = tokLine(content.data(), &tok_state); while (line != nullptr) { line_num++; @@ -1642,7 +1651,7 @@ SSLMultiCertConfigLoader::load(SSLCertLookup *lookup) if (lookup->ssl_default == nullptr) { shared_SSLMultiCertConfigParams sslMultiCertSettings(new SSLMultiCertConfigParams); sslMultiCertSettings->addr = ats_strdup("*"); - if (this->_store_ssl_ctx(lookup, sslMultiCertSettings) == nullptr) { + if (!this->_store_ssl_ctx(lookup, sslMultiCertSettings)) { Error("failed set default context"); return false; } @@ -1691,7 +1700,26 @@ SSLWriteBuffer(SSL *ssl, const void *buf, int64_t nbytes, int64_t &nwritten) return SSL_ERROR_NONE; } ERR_clear_error(); - int ret = SSL_write(ssl, buf, static_cast(nbytes)); + + int ret; +#if TS_HAS_TLS_EARLY_DATA + if (SSL_version(ssl) >= TLS1_3_VERSION) { + if (SSL_is_init_finished(ssl)) { + ret = SSL_write(ssl, buf, static_cast(nbytes)); + } else { + size_t nwrite; + ret = SSL_write_early_data(ssl, buf, static_cast(nbytes), &nwrite); + if (ret == 1) { + ret = nwrite; + } + } + } else { + ret = SSL_write(ssl, buf, static_cast(nbytes)); + } +#else + ret = SSL_write(ssl, buf, static_cast(nbytes)); +#endif + if (ret > 0) { nwritten = ret; BIO *bio = SSL_get_wbio(ssl); @@ -1702,10 +1730,10 @@ SSLWriteBuffer(SSL *ssl, const void *buf, int64_t nbytes, int64_t &nwritten) } int ssl_error = SSL_get_error(ssl, ret); if (ssl_error == SSL_ERROR_SSL && is_debug_tag_set("ssl.error.write")) { - char buf[512]; + char tempbuf[512]; unsigned long e = ERR_peek_last_error(); - ERR_error_string_n(e, buf, sizeof(buf)); - Debug("ssl.error.write", "SSL write returned %d, ssl_error=%d, ERR_get_error=%ld (%s)", ret, ssl_error, e, buf); + ERR_error_string_n(e, tempbuf, sizeof(tempbuf)); + Debug("ssl.error.write", "SSL write returned %d, ssl_error=%d, ERR_get_error=%ld (%s)", ret, ssl_error, e, tempbuf); } return ssl_error; } @@ -1719,6 +1747,63 @@ SSLReadBuffer(SSL *ssl, void *buf, int64_t nbytes, int64_t &nread) return SSL_ERROR_NONE; } ERR_clear_error(); + +#if TS_HAS_TLS_EARLY_DATA + if (SSL_version(ssl) >= TLS1_3_VERSION) { + SSLNetVConnection *netvc = SSLNetVCAccess(ssl); + + int64_t early_data_len = 0; + if (netvc->early_data_reader != nullptr) { + early_data_len = netvc->early_data_reader->read_avail(); + } + + if (early_data_len > 0) { + Debug("ssl_early_data", "Reading from early data buffer."); + netvc->read_from_early_data += netvc->early_data_reader->read(buf, nbytes < early_data_len ? nbytes : early_data_len); + + if (nbytes < early_data_len) { + nread = nbytes; + } else { + nread = early_data_len; + } + + return SSL_ERROR_NONE; + } + + if (SSLConfigParams::server_max_early_data > 0 && !netvc->early_data_finish) { + Debug("ssl_early_data", "More early data to read."); + ssl_error_t ssl_error = SSL_ERROR_NONE; + size_t read_bytes = 0; + + int ret = SSL_read_early_data(ssl, buf, static_cast(nbytes), &read_bytes); + + if (ret == SSL_READ_EARLY_DATA_ERROR) { + Debug("ssl_early_data", "SSL_READ_EARLY_DATA_ERROR"); + ssl_error = SSL_get_error(ssl, ret); + Debug("ssl_early_data", "Error reading early data: %s", ERR_error_string(ERR_get_error(), nullptr)); + } else { + if ((nread = read_bytes) > 0) { + netvc->read_from_early_data += read_bytes; + SSL_INCREMENT_DYN_STAT(ssl_early_data_received_count); + if (is_debug_tag_set("ssl_early_data_show_received")) { + std::string early_data_str(reinterpret_cast(buf), nread); + Debug("ssl_early_data_show_received", "Early data buffer: \n%s", early_data_str.c_str()); + } + } + + if (ret == SSL_READ_EARLY_DATA_FINISH) { + netvc->early_data_finish = true; + Debug("ssl_early_data", "SSL_READ_EARLY_DATA_FINISH: size = %" PRId64, nread); + } else { + Debug("ssl_early_data", "SSL_READ_EARLY_DATA_SUCCESS: size = %" PRId64, nread); + } + } + + return ssl_error; + } + } +#endif + int ret = SSL_read(ssl, buf, static_cast(nbytes)); if (ret > 0) { nread = ret; @@ -1726,10 +1811,10 @@ SSLReadBuffer(SSL *ssl, void *buf, int64_t nbytes, int64_t &nread) } int ssl_error = SSL_get_error(ssl, ret); if (ssl_error == SSL_ERROR_SSL && is_debug_tag_set("ssl.error.read")) { - char buf[512]; + char tempbuf[512]; unsigned long e = ERR_peek_last_error(); - ERR_error_string_n(e, buf, sizeof(buf)); - Debug("ssl.error.read", "SSL read returned %d, ssl_error=%d, ERR_get_error=%ld (%s)", ret, ssl_error, e, buf); + ERR_error_string_n(e, tempbuf, sizeof(tempbuf)); + Debug("ssl.error.read", "SSL read returned %d, ssl_error=%d, ERR_get_error=%ld (%s)", ret, ssl_error, e, tempbuf); } return ssl_error; @@ -1739,11 +1824,68 @@ ssl_error_t SSLAccept(SSL *ssl) { ERR_clear_error(); - int ret = SSL_accept(ssl); + + int ret = 0; + int ssl_error = SSL_ERROR_NONE; + +#if TS_HAS_TLS_EARLY_DATA + SSLNetVConnection *netvc = SSLNetVCAccess(ssl); + + if (SSLConfigParams::server_max_early_data > 0 && !netvc->early_data_finish) { + size_t nread; + + while (true) { + IOBufferBlock *block = new_IOBufferBlock(); + block->alloc(BUFFER_SIZE_INDEX_16K); + ret = SSL_read_early_data(ssl, block->buf(), index_to_buffer_size(BUFFER_SIZE_INDEX_16K), &nread); + + if (ret == SSL_READ_EARLY_DATA_ERROR) { + Debug("ssl_early_data", "SSL_READ_EARLY_DATA_ERROR"); + block->free(); + break; + } else { + if (nread > 0) { + if (netvc->early_data_buf == nullptr) { + netvc->early_data_buf = new_MIOBuffer(BUFFER_SIZE_INDEX_16K); + netvc->early_data_reader = netvc->early_data_buf->alloc_reader(); + } + block->fill(nread); + netvc->early_data_buf->append_block(block); + SSL_INCREMENT_DYN_STAT(ssl_early_data_received_count); + + if (is_debug_tag_set("ssl_early_data_show_received")) { + std::string early_data_str(reinterpret_cast(block->buf()), nread); + Debug("ssl_early_data_show_received", "Early data buffer: \n%s", early_data_str.c_str()); + } + } else { + block->free(); + } + + if (ret == SSL_READ_EARLY_DATA_FINISH) { + netvc->early_data_finish = true; + Debug("ssl_early_data", "SSL_READ_EARLY_DATA_FINISH: size = %lu", nread); + + if (netvc->early_data_reader == nullptr || netvc->early_data_reader->read_avail() == 0) { + Debug("ssl_early_data", "no data in early data buffer"); + ERR_clear_error(); + ret = SSL_accept(ssl); + } + break; + } + Debug("ssl_early_data", "SSL_READ_EARLY_DATA_SUCCESS: size = %lu", nread); + } + } + } else { + ret = SSL_accept(ssl); + } +#else + ret = SSL_accept(ssl); +#endif + if (ret > 0) { return SSL_ERROR_NONE; } - int ssl_error = SSL_get_error(ssl, ret); + ssl_error = SSL_get_error(ssl, ret); if (ssl_error == SSL_ERROR_SSL && is_debug_tag_set("ssl.error.accept")) { char buf[512]; unsigned long e = ERR_peek_last_error(); @@ -1774,22 +1916,36 @@ SSLConnect(SSL *ssl) } /** - Load certificates to SSL_CTX - @static + * Process the config to pull out the list of file names, and process the certs to get the list + * of subject and sni names. Thanks to dual cert configurations, there may be mulitple files of each type. + * If some names are not in all the listed certs they are listed in the uniqe_names map, keyed by the index + * of the including certificate */ bool -SSLMultiCertConfigLoader::load_certs(SSL_CTX *ctx, std::vector &cert_list, const SSLConfigParams *params, - const SSLMultiCertConfigParams *sslMultCertSettings) +SSLMultiCertConfigLoader::load_certs_and_cross_reference_names(std::vector &cert_list, + SSLMultiCertConfigLoader::CertLoadData &data, + const SSLConfigParams *params, + const SSLMultiCertConfigParams *sslMultCertSettings, + std::set &common_names, + std::unordered_map> &unique_names) { - SimpleTokenizer cert_tok((const char *)sslMultCertSettings->cert, SSL_CERT_SEPARATE_DELIM); - SimpleTokenizer key_tok((sslMultCertSettings->key ? (const char *)sslMultCertSettings->key : ""), SSL_CERT_SEPARATE_DELIM); + SimpleTokenizer cert_tok(sslMultCertSettings && sslMultCertSettings->cert ? (const char *)sslMultCertSettings->cert : "", + SSL_CERT_SEPARATE_DELIM); - if (sslMultCertSettings->key && cert_tok.getNumTokensRemaining() != key_tok.getNumTokensRemaining()) { + SimpleTokenizer key_tok(SSL_CERT_SEPARATE_DELIM); + if (sslMultCertSettings && sslMultCertSettings->key) { + key_tok.setString((const char *)sslMultCertSettings->key); + } else { + key_tok.setString(""); + } + + if (sslMultCertSettings && sslMultCertSettings->key && cert_tok.getNumTokensRemaining() != key_tok.getNumTokensRemaining()) { Error("the number of certificates in ssl_cert_name and ssl_key_name doesn't match"); return false; } + SimpleTokenizer ca_tok("", SSL_CERT_SEPARATE_DELIM); - if (sslMultCertSettings->ca) { + if (sslMultCertSettings && sslMultCertSettings->ca) { ca_tok.setString(sslMultCertSettings->ca); if (cert_tok.getNumTokensRemaining() != ca_tok.getNumTokensRemaining()) { Error("the number of certificates in ssl_cert_name and ssl_ca_name doesn't match"); @@ -1797,29 +1953,154 @@ SSLMultiCertConfigLoader::load_certs(SSL_CTX *ctx, std::vector &cert_lis } } -#if TS_USE_TLS_OCSP - if (SSLConfigParams::ssl_ocsp_enabled) { - Debug("ssl", "SSL OCSP Stapling is enabled"); - SSL_CTX_set_tlsext_status_cb(ctx, ssl_callback_ocsp_stapling); - } else { - Debug("ssl", "SSL OCSP Stapling is disabled"); - } SimpleTokenizer ocsp_tok("", SSL_CERT_SEPARATE_DELIM); - if (sslMultCertSettings->ocsp_response) { + if (sslMultCertSettings && sslMultCertSettings->ocsp_response) { ocsp_tok.setString(sslMultCertSettings->ocsp_response); if (cert_tok.getNumTokensRemaining() != ocsp_tok.getNumTokensRemaining()) { Error("the number of certificates in ssl_cert_name and ssl_ocsp_name doesn't match"); return false; } } + + for (const char *keyname = key_tok.getNext(); keyname; keyname = key_tok.getNext()) { + data.key_list.push_back(keyname); + } + + for (const char *caname = ca_tok.getNext(); caname; caname = ca_tok.getNext()) { + data.ca_list.push_back(caname); + } + + for (const char *ocspname = ocsp_tok.getNext(); ocspname; ocspname = ocsp_tok.getNext()) { + data.ocsp_list.push_back(ocspname); + } + + bool first_pass = true; + int cert_index = 0; + for (const char *certname = cert_tok.getNext(); certname; certname = cert_tok.getNext()) { + data.cert_names_list.push_back(certname); + std::string completeServerCertPath = Layout::relative_to(params->serverCertPathOnly, certname); + scoped_BIO bio(BIO_new_file(completeServerCertPath.c_str(), "r")); + X509 *cert = nullptr; + if (bio) { + cert = PEM_read_bio_X509(bio.get(), nullptr, nullptr, nullptr); + } + if (!bio || !cert) { + SSLError("failed to load certificate chain from %s", completeServerCertPath.c_str()); + return false; + } + + cert_list.push_back(cert); + if (SSLConfigParams::load_ssl_file_cb) { + SSLConfigParams::load_ssl_file_cb(completeServerCertPath.c_str()); + } + + std::set name_set; + // Grub through the names in the certs + X509_NAME *subject = nullptr; + + // Insert a key for the subject CN. + subject = X509_get_subject_name(cert); + ats_scoped_str subj_name; + if (subject) { + int pos = -1; + for (;;) { + pos = X509_NAME_get_index_by_NID(subject, NID_commonName, pos); + if (pos == -1) { + break; + } + + X509_NAME_ENTRY *e = X509_NAME_get_entry(subject, pos); + ASN1_STRING *cn = X509_NAME_ENTRY_get_data(e); + subj_name = asn1_strdup(cn); + + Debug("ssl", "subj '%s' in certificate %s %p", (const char *)subj_name, certname, cert); + name_set.insert(subj_name.get()); + } + } + + // Traverse the subjectAltNames (if any) and insert additional keys for the SSL context. + GENERAL_NAMES *names = static_cast(X509_get_ext_d2i(cert, NID_subject_alt_name, nullptr, nullptr)); + if (names) { + unsigned count = sk_GENERAL_NAME_num(names); + for (unsigned i = 0; i < count; ++i) { + GENERAL_NAME *name; + + name = sk_GENERAL_NAME_value(names, i); + if (name->type == GEN_DNS) { + ats_scoped_str dns(asn1_strdup(name->d.dNSName)); + name_set.insert(dns.get()); + } + } + sk_GENERAL_NAME_pop_free(names, GENERAL_NAME_free); + } + + if (first_pass) { + first_pass = false; + common_names = name_set; + } else { + // Check that all elements in common_names are in name_set + auto common_iter = common_names.begin(); + while (common_iter != common_names.end()) { + auto iter = name_set.find(*common_iter); + if (iter == name_set.end()) { + // Common_name not in new set, move name to unique set + auto iter = unique_names.find(cert_index - 1); + if (iter == unique_names.end()) { + std::set new_set; + new_set.insert(*common_iter); + unique_names.insert({cert_index - 1, new_set}); + } else { + iter->second.insert(*common_iter); + } + auto erase_iter = common_iter; + ++common_iter; + common_names.erase(erase_iter); + } else { + // New name already in common set, go ahead and remove it from further consideration + name_set.erase(iter); + ++common_iter; + } + } + // Anything still in name_set was not in common_names + for (auto name : name_set) { + auto iter = unique_names.find(cert_index); + if (iter == unique_names.end()) { + std::set new_set; + new_set.insert(name); + unique_names.insert({cert_index, new_set}); + } else { + iter->second.insert(name); + } + } + } + cert_index++; + } + return true; +} + +/** + Load certificates to SSL_CTX + @static + */ +bool +SSLMultiCertConfigLoader::load_certs(SSL_CTX *ctx, SSLMultiCertConfigLoader::CertLoadData const &data, + const SSLConfigParams *params, const SSLMultiCertConfigParams *sslMultCertSettings) +{ +#if TS_USE_TLS_OCSP + if (SSLConfigParams::ssl_ocsp_enabled) { + Debug("ssl", "SSL OCSP Stapling is enabled"); + SSL_CTX_set_tlsext_status_cb(ctx, ssl_callback_ocsp_stapling); + } else { + Debug("ssl", "SSL OCSP Stapling is disabled"); + } #else if (SSLConfigParams::ssl_ocsp_enabled) { Warning("failed to enable SSL OCSP Stapling; this version of OpenSSL does not support it"); } #endif /* TS_USE_TLS_OCSP */ - for (const char *certname = cert_tok.getNext(); certname; certname = cert_tok.getNext()) { - std::string completeServerCertPath = Layout::relative_to(params->serverCertPathOnly, certname); + for (size_t i = 0; i < data.cert_names_list.size(); i++) { + std::string completeServerCertPath = Layout::relative_to(params->serverCertPathOnly, data.cert_names_list[i]); scoped_BIO bio(BIO_new_file(completeServerCertPath.c_str(), "r")); X509 *cert = nullptr; if (bio) { @@ -1838,12 +2119,11 @@ SSLMultiCertConfigLoader::load_certs(SSL_CTX *ctx, std::vector &cert_lis // Load up any additional chain certificates SSL_CTX_add_extra_chain_cert_bio(ctx, bio); - const char *keyPath = key_tok.getNext(); + const char *keyPath = i < data.key_list.size() ? data.key_list[i].c_str() : nullptr; if (!SSLPrivateKeyHandler(ctx, params, completeServerCertPath, keyPath)) { return false; } - cert_list.push_back(cert); if (SSLConfigParams::load_ssl_file_cb) { SSLConfigParams::load_ssl_file_cb(completeServerCertPath.c_str()); } @@ -1866,7 +2146,7 @@ SSLMultiCertConfigLoader::load_certs(SSL_CTX *ctx, std::vector &cert_lis // Now, load any additional certificate chains specified in this entry. if (sslMultCertSettings->ca) { - const char *ca_name = ca_tok.getNext(); + const char *ca_name = data.ca_list[i].c_str(); if (ca_name != nullptr) { ats_scoped_str completeServerCertChainPath(Layout::relative_to(params->serverCertPathOnly, ca_name)); if (!SSL_CTX_add_extra_chain_cert_file(ctx, completeServerCertChainPath)) { @@ -1881,18 +2161,19 @@ SSLMultiCertConfigLoader::load_certs(SSL_CTX *ctx, std::vector &cert_lis #if TS_USE_TLS_OCSP if (SSLConfigParams::ssl_ocsp_enabled) { if (sslMultCertSettings->ocsp_response) { - const char *ocsp_response_name = ocsp_tok.getNext(); + const char *ocsp_response_name = data.ocsp_list[i].c_str(); ats_scoped_str completeOCSPResponsePath(Layout::relative_to(params->ssl_ocsp_response_path_only, ocsp_response_name)); - if (!ssl_stapling_init_cert(ctx, cert, certname, (const char *)completeOCSPResponsePath)) { - Warning("failed to configure SSL_CTX for OCSP Stapling info for certificate at %s", certname); + if (!ssl_stapling_init_cert(ctx, cert, data.cert_names_list[i].c_str(), (const char *)completeOCSPResponsePath)) { + Warning("failed to configure SSL_CTX for OCSP Stapling info for certificate at %s", data.cert_names_list[i].c_str()); } } else { - if (!ssl_stapling_init_cert(ctx, cert, certname, nullptr)) { - Warning("failed to configure SSL_CTX for OCSP Stapling info for certificate at %s", certname); + if (!ssl_stapling_init_cert(ctx, cert, data.cert_names_list[i].c_str(), nullptr)) { + Warning("failed to configure SSL_CTX for OCSP Stapling info for certificate at %s", data.cert_names_list[i].c_str()); } } } #endif /* TS_USE_TLS_OCSP */ + X509_free(cert); } return true; @@ -1913,12 +2194,8 @@ SSLMultiCertConfigLoader::set_session_id_context(SSL_CTX *ctx, const SSLConfigPa const char *setting_cert = sslMultCertSettings ? sslMultCertSettings->cert.get() : nullptr; bool result = false; - // Set the list of CA's to send to client if we ask for a client certificate if (params->serverCACertFilename) { ca_list = SSL_load_client_CA_file(params->serverCACertFilename); - if (ca_list) { - SSL_CTX_set_client_CA_list(ctx, ca_list); - } } if (EVP_DigestInit_ex(digest, evp_md_func, nullptr) == 0) { @@ -1945,6 +2222,9 @@ SSLMultiCertConfigLoader::set_session_id_context(SSL_CTX *ctx, const SSLConfigPa goto fail; } } + + // Set the list of CA's to send to client if we ask for a client certificate + SSL_CTX_set_client_CA_list(ctx, ca_list); } if (EVP_DigestFinal_ex(digest, hash_buf, &hash_len) == 0) { @@ -1965,6 +2245,12 @@ SSLMultiCertConfigLoader::set_session_id_context(SSL_CTX *ctx, const SSLConfigPa return result; } +const char * +SSLMultiCertConfigLoader::_debug_tag() const +{ + return "ssl"; +} + /** Clear password in SSL_CTX @static @@ -1975,3 +2261,13 @@ SSLMultiCertConfigLoader::clear_pw_references(SSL_CTX *ssl_ctx) SSL_CTX_set_default_passwd_cb(ssl_ctx, nullptr); SSL_CTX_set_default_passwd_cb_userdata(ssl_ctx, nullptr); } + +ssl_curve_id +SSLGetCurveNID(SSL *ssl) +{ +#ifndef OPENSSL_IS_BORINGSSL + return SSL_get_shared_curve(ssl, 0); +#else + return SSL_get_curve_id(ssl); +#endif +} diff --git a/iocore/net/Socks.cc b/iocore/net/Socks.cc index 7771abff56b..dbc2a02835a 100644 --- a/iocore/net/Socks.cc +++ b/iocore/net/Socks.cc @@ -44,7 +44,7 @@ void SocksEntry::init(Ptr &m, SocksNetVC *vc, unsigned char socks_support, unsigned char ver) { mutex = m; - buf = new_MIOBuffer(); + buf = new_MIOBuffer(BUFFER_SIZE_INDEX_32K); reader = buf->alloc_reader(); socks_cmd = socks_support; @@ -514,7 +514,6 @@ loadSocksConfiguration(socks_conf_struct *socks_conf_stuff) } #ifdef SOCKS_WITH_TS tmp = Load_IpMap_From_File(&socks_conf_stuff->ip_map, socks_config_fd, "no_socks"); - // tmp = socks_conf_stuff->ip_range.read_table_from_file(socks_config_fd, "no_socks"); if (tmp) { Error("SOCKS Config: Error while reading ip_range: %s.", tmp); diff --git a/iocore/net/TLSSessionResumptionSupport.cc b/iocore/net/TLSSessionResumptionSupport.cc new file mode 100644 index 00000000000..8935c990705 --- /dev/null +++ b/iocore/net/TLSSessionResumptionSupport.cc @@ -0,0 +1,232 @@ +/** @file + + TLSSessionResumptionSupport.cc provides implmentations for + TLSSessionResumptionSupport methods + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +// Check if the ticket_key callback #define is available, and if so, enable session tickets. + +#include "TLSSessionResumptionSupport.h" + +#ifdef SSL_CTX_set_tlsext_ticket_key_cb +#define TS_HAVE_OPENSSL_SESSION_TICKETS 1 +#endif + +#ifdef TS_HAVE_OPENSSL_SESSION_TICKETS + +#include "P_SSLConfig.h" +#include "SSLStats.h" +#include +#include "InkAPIInternal.h" + +// Remove this when drop OpenSSL 1.0.2 support +#ifndef evp_md_func +#ifdef OPENSSL_NO_SHA256 +#define evp_md_func EVP_sha1() +#else +#define evp_md_func EVP_sha256() +#endif +#endif + +int TLSSessionResumptionSupport::_ex_data_index = -1; + +static bool +is_ssl_session_timed_out(SSL_SESSION *session) +{ + return SSL_SESSION_get_timeout(session) < (time(nullptr) - SSL_SESSION_get_time(session)); +} + +void +TLSSessionResumptionSupport::initialize() +{ + ink_assert(_ex_data_index == -1); + if (_ex_data_index == -1) { + _ex_data_index = SSL_get_ex_new_index(0, (void *)"TLSSessionResumptionSupport index", nullptr, nullptr, nullptr); + } +} + +TLSSessionResumptionSupport * +TLSSessionResumptionSupport::getInstance(SSL *ssl) +{ + return static_cast(SSL_get_ex_data(ssl, _ex_data_index)); +} + +void +TLSSessionResumptionSupport::bind(SSL *ssl, TLSSessionResumptionSupport *srs) +{ + SSL_set_ex_data(ssl, _ex_data_index, srs); +} + +int +TLSSessionResumptionSupport::processSessionTicket(SSL *ssl, unsigned char *keyname, unsigned char *iv, EVP_CIPHER_CTX *cipher_ctx, + HMAC_CTX *hctx, int enc) +{ + SSLConfig::scoped_config config; + SSLCertificateConfig::scoped_config lookup; + SSLTicketKeyConfig::scoped_config params; + + // Get the IP address to look up the keyblock + const IpEndpoint &ip = this->_getLocalEndpoint(); + SSLCertContext *cc = lookup->find(ip); + ssl_ticket_key_block *keyblock = nullptr; + if (cc == nullptr || cc->keyblock == nullptr) { + // Try the default + keyblock = params->default_global_keyblock; + } else { + keyblock = cc->keyblock.get(); + } + ink_release_assert(keyblock != nullptr && keyblock->num_keys > 0); + + if (enc == 1) { + return this->_setSessionInformation(keyblock, ssl, keyname, iv, cipher_ctx, hctx); + } else if (enc == 0) { + return this->_getSessionInformation(keyblock, ssl, keyname, iv, cipher_ctx, hctx); + } + + return -1; +} + +bool +TLSSessionResumptionSupport::getSSLSessionCacheHit() const +{ + return this->_sslSessionCacheHit; +} + +ssl_curve_id +TLSSessionResumptionSupport::getSSLCurveNID() const +{ + return this->_sslCurveNID; +} + +SSL_SESSION * +TLSSessionResumptionSupport::getSession(SSL *ssl, const unsigned char *id, int len, int *copy) +{ + SSLSessionID sid(id, len); + + *copy = 0; + if (diags->tag_activated("ssl.session_cache")) { + char printable_buf[(len * 2) + 1]; + sid.toString(printable_buf, sizeof(printable_buf)); + Debug("ssl.session_cache.get", "ssl_get_cached_session cached session '%s' context %p", printable_buf, SSL_get_SSL_CTX(ssl)); + } + + APIHook *hook = ssl_hooks->get(TSSslHookInternalID(TS_SSL_SESSION_HOOK)); + while (hook) { + hook->invoke(TS_EVENT_SSL_SESSION_GET, &sid); + hook = hook->m_link.next; + } + + SSL_SESSION *session = nullptr; + ssl_session_cache_exdata *exdata = nullptr; + if (session_cache->getSession(sid, &session, &exdata)) { + ink_assert(session); + ink_assert(exdata); + + // Double check the timeout + if (is_ssl_session_timed_out(session)) { + SSL_INCREMENT_DYN_STAT(ssl_session_cache_miss); +// Due to bug in openssl, the timeout is checked, but only removed +// from the openssl built-in hash table. The external remove cb is not called +#if 0 // This is currently eliminated, since it breaks things in odd ways (see TS-3710) + ssl_rm_cached_session(SSL_get_SSL_CTX(ssl), session); +#endif + session = nullptr; + } else { + SSL_INCREMENT_DYN_STAT(ssl_session_cache_hit); + this->_setSSLSessionCacheHit(true); + this->_setSSLCurveNID(exdata->curve); + } + } else { + SSL_INCREMENT_DYN_STAT(ssl_session_cache_miss); + } + return session; +} + +void +TLSSessionResumptionSupport::clear() +{ + this->_sslSessionCacheHit = false; +} + +int +TLSSessionResumptionSupport::_setSessionInformation(ssl_ticket_key_block *keyblock, SSL *ssl, unsigned char *keyname, + unsigned char *iv, EVP_CIPHER_CTX *cipher_ctx, HMAC_CTX *hctx) +{ + const ssl_ticket_key_t &most_recent_key = keyblock->keys[0]; + memcpy(keyname, most_recent_key.key_name, sizeof(most_recent_key.key_name)); + RAND_bytes(iv, EVP_MAX_IV_LENGTH); + EVP_EncryptInit_ex(cipher_ctx, EVP_aes_128_cbc(), nullptr, most_recent_key.aes_key, iv); + HMAC_Init_ex(hctx, most_recent_key.hmac_secret, sizeof(most_recent_key.hmac_secret), evp_md_func, nullptr); + + Debug("ssl_session_ticket", "create ticket for a new session."); + SSL_INCREMENT_DYN_STAT(ssl_total_tickets_created_stat); + return 1; +} + +int +TLSSessionResumptionSupport::_getSessionInformation(ssl_ticket_key_block *keyblock, SSL *ssl, unsigned char *keyname, + unsigned char *iv, EVP_CIPHER_CTX *cipher_ctx, HMAC_CTX *hctx) +{ + for (unsigned i = 0; i < keyblock->num_keys; ++i) { + if (memcmp(keyname, keyblock->keys[i].key_name, sizeof(keyblock->keys[i].key_name)) == 0) { + EVP_DecryptInit_ex(cipher_ctx, EVP_aes_128_cbc(), nullptr, keyblock->keys[i].aes_key, iv); + HMAC_Init_ex(hctx, keyblock->keys[i].hmac_secret, sizeof(keyblock->keys[i].hmac_secret), evp_md_func, nullptr); + + Debug("ssl_session_ticket", "verify the ticket for an existing session."); + // Increase the total number of decrypted tickets. + SSL_INCREMENT_DYN_STAT(ssl_total_tickets_verified_stat); + + if (i != 0) { // The number of tickets decrypted with "older" keys. + SSL_INCREMENT_DYN_STAT(ssl_total_tickets_verified_old_key_stat); + } + + this->_setSSLSessionCacheHit(true); + +#ifdef TLS1_3_VERSION + if (SSL_version(ssl) >= TLS1_3_VERSION) { + Debug("ssl_session_ticket", "make sure tickets are only used once."); + return 2; + } +#endif + + // When we decrypt with an "older" key, encrypt the ticket again with the most recent key. + return (i == 0) ? 1 : 2; + } + } + + Debug("ssl_session_ticket", "keyname is not consistent."); + SSL_INCREMENT_DYN_STAT(ssl_total_tickets_not_found_stat); + return 0; +} + +void +TLSSessionResumptionSupport::_setSSLSessionCacheHit(bool state) +{ + this->_sslSessionCacheHit = state; +} + +void +TLSSessionResumptionSupport::_setSSLCurveNID(ssl_curve_id curve_nid) +{ + this->_sslCurveNID = curve_nid; +} + +#endif /* TS_HAVE_OPENSSL_SESSION_TICKETS */ diff --git a/iocore/net/TLSSessionResumptionSupport.h b/iocore/net/TLSSessionResumptionSupport.h new file mode 100644 index 00000000000..466618b9e9b --- /dev/null +++ b/iocore/net/TLSSessionResumptionSupport.h @@ -0,0 +1,66 @@ +/** @file + + TLSSessionResumptionSupport implements common methods and members to + support TLS Ssssion Resumption + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#pragma once + +#include + +#include "tscore/ink_inet.h" +#include "P_SSLCertLookup.h" +#include "P_SSLUtils.h" + +class TLSSessionResumptionSupport +{ +public: + virtual ~TLSSessionResumptionSupport() = default; + + static void initialize(); + static TLSSessionResumptionSupport *getInstance(SSL *ssl); + static void bind(SSL *ssl, TLSSessionResumptionSupport *srs); + + int processSessionTicket(SSL *ssl, unsigned char *keyname, unsigned char *iv, EVP_CIPHER_CTX *cipher_ctx, HMAC_CTX *hctx, + int enc); + bool getSSLSessionCacheHit() const; + ssl_curve_id getSSLCurveNID() const; + + SSL_SESSION *getSession(SSL *ssl, const unsigned char *id, int len, int *copy); + +protected: + void clear(); + virtual const IpEndpoint &_getLocalEndpoint() = 0; + +private: + static int _ex_data_index; + + bool _sslSessionCacheHit = false; + int _sslCurveNID = NID_undef; + + int _setSessionInformation(ssl_ticket_key_block *keyblock, SSL *ssl, unsigned char *keyname, unsigned char *iv, + EVP_CIPHER_CTX *cipher_ctx, HMAC_CTX *hctx); + int _getSessionInformation(ssl_ticket_key_block *keyblock, SSL *ssl, unsigned char *keyname, unsigned char *iv, + EVP_CIPHER_CTX *cipher_ctx, HMAC_CTX *hctx); + + void _setSSLSessionCacheHit(bool state); + void _setSSLCurveNID(ssl_curve_id curve_nid); +}; diff --git a/iocore/net/UnixNet.cc b/iocore/net/UnixNet.cc index c2eded5e1cf..3e15b972459 100644 --- a/iocore/net/UnixNet.cc +++ b/iocore/net/UnixNet.cc @@ -41,7 +41,7 @@ extern "C" void fd_reify(struct ev_loop *); // INKqa10496 // One Inactivity cop runs on each thread once every second and -// loops through the list of NetVCs and calls the timeouts +// loops through the list of NetEvents and calls the timeouts class InactivityCop : public Continuation { public: @@ -54,52 +54,64 @@ class InactivityCop : public Continuation NetHandler &nh = *get_NetHandler(this_ethread()); Debug("inactivity_cop_check", "Checking inactivity on Thread-ID #%d", this_ethread()->id); - // The rest NetVCs in cop_list which are not triggered between InactivityCop runs. + // The rest NetEvents in cop_list which are not triggered between InactivityCop runs. // Use pop() to catch any closes caused by callbacks. - while (UnixNetVConnection *vc = nh.cop_list.pop()) { + while (NetEvent *ne = nh.cop_list.pop()) { // If we cannot get the lock don't stop just keep cleaning - MUTEX_TRY_LOCK(lock, vc->mutex, this_ethread()); + MUTEX_TRY_LOCK(lock, ne->get_mutex(), this_ethread()); if (!lock.is_locked()) { NET_INCREMENT_DYN_STAT(inactivity_cop_lock_acquire_failure_stat); continue; } - if (vc->closed) { - nh.free_netvc(vc); + if (ne->closed) { + nh.free_netevent(ne); continue; } - if (vc->next_inactivity_timeout_at && vc->next_inactivity_timeout_at < now) { - if (nh.keep_alive_queue.in(vc)) { + // set a default inactivity timeout if one is not set + if (ne->next_inactivity_timeout_at == 0 && nh.config.default_inactivity_timeout > 0) { + Debug("inactivity_cop", "vc: %p inactivity timeout not set, setting a default of %d", ne, + nh.config.default_inactivity_timeout); + ne->set_default_inactivity_timeout(HRTIME_SECONDS(nh.config.default_inactivity_timeout)); + NET_INCREMENT_DYN_STAT(default_inactivity_timeout_applied_stat); + } + + if (ne->next_inactivity_timeout_at && ne->next_inactivity_timeout_at < now) { + if (ne->is_default_inactivity_timeout()) { + // track the connections that timed out due to default inactivity + NET_INCREMENT_DYN_STAT(default_inactivity_timeout_count_stat); + } + if (nh.keep_alive_queue.in(ne)) { // only stat if the connection is in keep-alive, there can be other inactivity timeouts - ink_hrtime diff = (now - (vc->next_inactivity_timeout_at - vc->inactivity_timeout_in)) / HRTIME_SECOND; + ink_hrtime diff = (now - (ne->next_inactivity_timeout_at - ne->inactivity_timeout_in)) / HRTIME_SECOND; NET_SUM_DYN_STAT(keep_alive_queue_timeout_total_stat, diff); NET_INCREMENT_DYN_STAT(keep_alive_queue_timeout_count_stat); } - Debug("inactivity_cop_verbose", "vc: %p now: %" PRId64 " timeout at: %" PRId64 " timeout in: %" PRId64, vc, - ink_hrtime_to_sec(now), vc->next_inactivity_timeout_at, vc->inactivity_timeout_in); - vc->handleEvent(VC_EVENT_INACTIVITY_TIMEOUT, e); - } else if (vc->next_activity_timeout_at && vc->next_activity_timeout_at < now) { - Debug("inactivity_cop_verbose", "active vc: %p now: %" PRId64 " timeout at: %" PRId64 " timeout in: %" PRId64, vc, - ink_hrtime_to_sec(now), vc->next_activity_timeout_at, vc->active_timeout_in); - vc->handleEvent(VC_EVENT_ACTIVE_TIMEOUT, e); + Debug("inactivity_cop_verbose", "ne: %p now: %" PRId64 " timeout at: %" PRId64 " timeout in: %" PRId64, ne, + ink_hrtime_to_sec(now), ne->next_inactivity_timeout_at, ne->inactivity_timeout_in); + ne->callback(VC_EVENT_INACTIVITY_TIMEOUT, e); + } else if (ne->next_activity_timeout_at && ne->next_activity_timeout_at < now) { + Debug("inactivity_cop_verbose", "active ne: %p now: %" PRId64 " timeout at: %" PRId64 " timeout in: %" PRId64, ne, + ink_hrtime_to_sec(now), ne->next_activity_timeout_at, ne->active_timeout_in); + ne->callback(VC_EVENT_ACTIVE_TIMEOUT, e); } } // The cop_list is empty now. // Let's reload the cop_list from open_list again. - forl_LL(UnixNetVConnection, vc, nh.open_list) + forl_LL(NetEvent, ne, nh.open_list) { - if (vc->thread == this_ethread()) { - nh.cop_list.push(vc); + if (ne->get_thread() == this_ethread()) { + nh.cop_list.push(ne); } } - // NetHandler will remove NetVC from cop_list if it is triggered. - // As the NetHandler runs, the number of NetVCs in the cop_list is decreasing. + // NetHandler will remove NetEvent from cop_list if it is triggered. + // As the NetHandler runs, the number of NetEvents in the cop_list is decreasing. // NetHandler runs 100 times maximum between InactivityCop runs. - // Therefore we don't have to check all the NetVCs as much as open_list. + // Therefore we don't have to check all the NetEvents as much as open_list. // Cleanup the active and keep-alive queues periodically - nh.manage_active_queue(true); // close any connections over the active timeout + nh.manage_active_queue(nullptr, true); // close any connections over the active timeout nh.manage_keep_alive_queue(); return 0; @@ -264,9 +276,9 @@ NetHandler::update_nethandler_config(const char *str, RecDataT, RecData data, vo if (name == "proxy.config.net.max_connections_in"sv) { updated_member = &NetHandler::global_config.max_connections_in; Debug("net_queue", "proxy.config.net.max_connections_in updated to %" PRId64, data.rec_int); - } else if (name == "proxy.config.net.max_active_connections_in"sv) { - updated_member = &NetHandler::global_config.max_connections_active_in; - Debug("net_queue", "proxy.config.net.max_active_connections_in updated to %" PRId64, data.rec_int); + } else if (name == "proxy.config.net.max_requests_in"sv) { + updated_member = &NetHandler::global_config.max_requests_in; + Debug("net_queue", "proxy.config.net.max_requests_in updated to %" PRId64, data.rec_int); } else if (name == "proxy.config.net.inactive_threshold_in"sv) { updated_member = &NetHandler::global_config.inactive_threshold_in; Debug("net_queue", "proxy.config.net.inactive_threshold_in updated to %" PRId64, data.rec_int); @@ -309,21 +321,21 @@ NetHandler::init_for_process() { // read configuration values and setup callbacks for when they change REC_ReadConfigInt32(global_config.max_connections_in, "proxy.config.net.max_connections_in"); - REC_ReadConfigInt32(global_config.max_connections_active_in, "proxy.config.net.max_connections_active_in"); + REC_ReadConfigInt32(global_config.max_requests_in, "proxy.config.net.max_requests_in"); REC_ReadConfigInt32(global_config.inactive_threshold_in, "proxy.config.net.inactive_threshold_in"); REC_ReadConfigInt32(global_config.transaction_no_activity_timeout_in, "proxy.config.net.transaction_no_activity_timeout_in"); REC_ReadConfigInt32(global_config.keep_alive_no_activity_timeout_in, "proxy.config.net.keep_alive_no_activity_timeout_in"); REC_ReadConfigInt32(global_config.default_inactivity_timeout, "proxy.config.net.default_inactivity_timeout"); RecRegisterConfigUpdateCb("proxy.config.net.max_connections_in", update_nethandler_config, nullptr); - RecRegisterConfigUpdateCb("proxy.config.net.max_active_connections_in", update_nethandler_config, nullptr); + RecRegisterConfigUpdateCb("proxy.config.net.max_requests_in", update_nethandler_config, nullptr); RecRegisterConfigUpdateCb("proxy.config.net.inactive_threshold_in", update_nethandler_config, nullptr); RecRegisterConfigUpdateCb("proxy.config.net.transaction_no_activity_timeout_in", update_nethandler_config, nullptr); RecRegisterConfigUpdateCb("proxy.config.net.keep_alive_no_activity_timeout_in", update_nethandler_config, nullptr); RecRegisterConfigUpdateCb("proxy.config.net.default_inactivity_timeout", update_nethandler_config, nullptr); Debug("net_queue", "proxy.config.net.max_connections_in updated to %d", global_config.max_connections_in); - Debug("net_queue", "proxy.config.net.max_active_connections_in updated to %d", global_config.max_connections_active_in); + Debug("net_queue", "proxy.config.net.max_requests_in updated to %d", global_config.max_requests_in); Debug("net_queue", "proxy.config.net.inactive_threshold_in updated to %d", global_config.inactive_threshold_in); Debug("net_queue", "proxy.config.net.transaction_no_activity_timeout_in updated to %d", global_config.transaction_no_activity_timeout_in); @@ -333,23 +345,23 @@ NetHandler::init_for_process() } // -// Function used to release a UnixNetVConnection and free it. +// Function used to release a NetEvent and free it. // void -NetHandler::free_netvc(UnixNetVConnection *netvc) +NetHandler::free_netevent(NetEvent *ne) { EThread *t = this->thread; ink_assert(t == this_ethread()); - ink_release_assert(netvc->thread == t); - ink_release_assert(netvc->nh == this); - - // Release netvc from InactivityCop - stopCop(netvc); - // Release netvc from NetHandler - stopIO(netvc); - // Clear and deallocate netvc - netvc->free(t); + ink_release_assert(ne->get_thread() == t); + ink_release_assert(ne->nh == this); + + // Release ne from InactivityCop + stopCop(ne); + // Release ne from NetHandler + stopIO(ne); + // Clear and deallocate ne + ne->free(t); } // @@ -358,25 +370,25 @@ NetHandler::free_netvc(UnixNetVConnection *netvc) void NetHandler::process_enabled_list() { - UnixNetVConnection *vc = nullptr; - - SListM(UnixNetVConnection, NetState, read, enable_link) rq(read_enable_list.popall()); - while ((vc = rq.pop())) { - vc->ep.modify(EVENTIO_READ); - vc->ep.refresh(EVENTIO_READ); - vc->read.in_enabled_list = 0; - if ((vc->read.enabled && vc->read.triggered) || vc->closed) { - read_ready_list.in_or_enqueue(vc); + NetEvent *ne = nullptr; + + SListM(NetEvent, NetState, read, enable_link) rq(read_enable_list.popall()); + while ((ne = rq.pop())) { + ne->ep.modify(EVENTIO_READ); + ne->ep.refresh(EVENTIO_READ); + ne->read.in_enabled_list = 0; + if ((ne->read.enabled && ne->read.triggered) || ne->closed) { + read_ready_list.in_or_enqueue(ne); } } - SListM(UnixNetVConnection, NetState, write, enable_link) wq(write_enable_list.popall()); - while ((vc = wq.pop())) { - vc->ep.modify(EVENTIO_WRITE); - vc->ep.refresh(EVENTIO_WRITE); - vc->write.in_enabled_list = 0; - if ((vc->write.enabled && vc->write.triggered) || vc->closed) { - write_ready_list.in_or_enqueue(vc); + SListM(NetEvent, NetState, write, enable_link) wq(write_enable_list.popall()); + while ((ne = wq.pop())) { + ne->ep.modify(EVENTIO_WRITE); + ne->ep.refresh(EVENTIO_WRITE); + ne->write.in_enabled_list = 0; + if ((ne->write.enabled && ne->write.triggered) || ne->closed) { + write_ready_list.in_or_enqueue(ne); } } } @@ -387,63 +399,63 @@ NetHandler::process_enabled_list() void NetHandler::process_ready_list() { - UnixNetVConnection *vc = nullptr; + NetEvent *ne = nullptr; #if defined(USE_EDGE_TRIGGER) - // UnixNetVConnection * - while ((vc = read_ready_list.dequeue())) { + // NetEvent * + while ((ne = read_ready_list.dequeue())) { // Initialize the thread-local continuation flags - set_cont_flags(vc->control_flags); - if (vc->closed) { - free_netvc(vc); - } else if (vc->read.enabled && vc->read.triggered) { - vc->net_read_io(this, this->thread); - } else if (!vc->read.enabled) { - read_ready_list.remove(vc); + set_cont_flags(ne->get_control_flags()); + if (ne->closed) { + free_netevent(ne); + } else if (ne->read.enabled && ne->read.triggered) { + ne->net_read_io(this, this->thread); + } else if (!ne->read.enabled) { + read_ready_list.remove(ne); #if defined(solaris) - if (vc->read.triggered && vc->write.enabled) { - vc->ep.modify(-EVENTIO_READ); - vc->ep.refresh(EVENTIO_WRITE); - vc->writeReschedule(this); + if (ne->read.triggered && ne->write.enabled) { + ne->ep.modify(-EVENTIO_READ); + ne->ep.refresh(EVENTIO_WRITE); + ne->writeReschedule(this); } #endif } } - while ((vc = write_ready_list.dequeue())) { - set_cont_flags(vc->control_flags); - if (vc->closed) { - free_netvc(vc); - } else if (vc->write.enabled && vc->write.triggered) { - write_to_net(this, vc, this->thread); - } else if (!vc->write.enabled) { - write_ready_list.remove(vc); + while ((ne = write_ready_list.dequeue())) { + set_cont_flags(ne->get_control_flags()); + if (ne->closed) { + free_netevent(ne); + } else if (ne->write.enabled && ne->write.triggered) { + ne->net_write_io(this, this->thread); + } else if (!ne->write.enabled) { + write_ready_list.remove(ne); #if defined(solaris) - if (vc->write.triggered && vc->read.enabled) { - vc->ep.modify(-EVENTIO_WRITE); - vc->ep.refresh(EVENTIO_READ); - vc->readReschedule(this); + if (ne->write.triggered && ne->read.enabled) { + ne->ep.modify(-EVENTIO_WRITE); + ne->ep.refresh(EVENTIO_READ); + ne->readReschedule(this); } #endif } } #else /* !USE_EDGE_TRIGGER */ - while ((vc = read_ready_list.dequeue())) { - set_cont_flags(vc->control_flags); - if (vc->closed) - free_netvc(vc); - else if (vc->read.enabled && vc->read.triggered) - vc->net_read_io(this, this->thread); - else if (!vc->read.enabled) - vc->ep.modify(-EVENTIO_READ); + while ((ne = read_ready_list.dequeue())) { + set_cont_flags(ne->get_control_flags()); + if (ne->closed) + free_netevent(ne); + else if (ne->read.enabled && ne->read.triggered) + ne->net_read_io(this, this->thread); + else if (!ne->read.enabled) + ne->ep.modify(-EVENTIO_READ); } - while ((vc = write_ready_list.dequeue())) { - set_cont_flags(vc->control_flags); - if (vc->closed) - free_netvc(vc); - else if (vc->write.enabled && vc->write.triggered) - write_to_net(this, vc, this->thread); - else if (!vc->write.enabled) - vc->ep.modify(-EVENTIO_WRITE); + while ((ne = write_ready_list.dequeue())) { + set_cont_flags(ne->get_control_flags()); + if (ne->closed) + free_netevent(ne); + else if (ne->write.enabled && ne->write.triggered) + write_to_net(this, ne, this->thread); + else if (!ne->write.enabled) + ne->ep.modify(-EVENTIO_WRITE); } #endif /* !USE_EDGE_TRIGGER */ } @@ -482,35 +494,35 @@ NetHandler::waitForActivity(ink_hrtime timeout) p->do_poll(timeout); // Get & Process polling result - PollDescriptor *pd = get_PollDescriptor(this->thread); - UnixNetVConnection *vc = nullptr; + PollDescriptor *pd = get_PollDescriptor(this->thread); + NetEvent *ne = nullptr; for (int x = 0; x < pd->result; x++) { epd = static_cast get_ev_data(pd, x); if (epd->type == EVENTIO_READWRITE_VC) { - vc = epd->data.vc; - // Remove triggered NetVC from cop_list because it won't be timeout before next InactivityCop runs. - if (cop_list.in(vc)) { - cop_list.remove(vc); + ne = epd->data.ne; + // Remove triggered NetEvent from cop_list because it won't be timeout before next InactivityCop runs. + if (cop_list.in(ne)) { + cop_list.remove(ne); } if (get_ev_events(pd, x) & (EVENTIO_READ | EVENTIO_ERROR)) { - vc->read.triggered = 1; - if (!read_ready_list.in(vc)) { - read_ready_list.enqueue(vc); + ne->read.triggered = 1; + if (!read_ready_list.in(ne)) { + read_ready_list.enqueue(ne); } else if (get_ev_events(pd, x) & EVENTIO_ERROR) { // check for unhandled epoll events that should be handled Debug("iocore_net_main", "Unhandled epoll event on read: 0x%04x read.enabled=%d closed=%d read.netready_queue=%d", - get_ev_events(pd, x), vc->read.enabled, vc->closed, read_ready_list.in(vc)); + get_ev_events(pd, x), ne->read.enabled, ne->closed, read_ready_list.in(ne)); } } - vc = epd->data.vc; + ne = epd->data.ne; if (get_ev_events(pd, x) & (EVENTIO_WRITE | EVENTIO_ERROR)) { - vc->write.triggered = 1; - if (!write_ready_list.in(vc)) { - write_ready_list.enqueue(vc); + ne->write.triggered = 1; + if (!write_ready_list.in(ne)) { + write_ready_list.enqueue(ne); } else if (get_ev_events(pd, x) & EVENTIO_ERROR) { // check for unhandled epoll events that should be handled Debug("iocore_net_main", "Unhandled epoll event on write: 0x%04x write.enabled=%d closed=%d write.netready_queue=%d", - get_ev_events(pd, x), vc->write.enabled, vc->closed, write_ready_list.in(vc)); + get_ev_events(pd, x), ne->write.enabled, ne->closed, write_ready_list.in(ne)); } } else if (!(get_ev_events(pd, x) & EVENTIO_READ)) { Debug("iocore_net_main", "Unhandled epoll event: 0x%04x", get_ev_events(pd, x)); @@ -524,6 +536,8 @@ NetHandler::waitForActivity(ink_hrtime timeout) } } else if (epd->type == EVENTIO_ASYNC_SIGNAL) { net_signal_hook_callback(this->thread); + } else if (epd->type == EVENTIO_NETACCEPT) { + this->thread->schedule_imm(epd->data.c); } ev_next_event(pd, x); } @@ -551,40 +565,49 @@ NetHandler::signalActivity() } bool -NetHandler::manage_active_queue(bool ignore_queue_size = false) +NetHandler::manage_active_queue(NetEvent *enabling_ne, bool ignore_queue_size = false) { const int total_connections_in = active_queue_size + keep_alive_queue_size; Debug("v_net_queue", - "max_connections_per_thread_in: %d max_connections_active_per_thread_in: %d total_connections_in: %d " + "max_connections_per_thread_in: %d max_requests_per_thread_in: %d total_connections_in: %d " "active_queue_size: %d keep_alive_queue_size: %d", - max_connections_per_thread_in, max_connections_active_per_thread_in, total_connections_in, active_queue_size, - keep_alive_queue_size); + max_connections_per_thread_in, max_requests_per_thread_in, total_connections_in, active_queue_size, keep_alive_queue_size); - if (ignore_queue_size == false && max_connections_active_per_thread_in > active_queue_size) { + if (!max_requests_per_thread_in) { + // active queue has no max + return true; + } + + if (ignore_queue_size == false && max_requests_per_thread_in > active_queue_size) { return true; } ink_hrtime now = Thread::get_hrtime(); // loop over the non-active connections and try to close them - UnixNetVConnection *vc = active_queue.head; - UnixNetVConnection *vc_next = nullptr; - int closed = 0; - int handle_event = 0; - int total_idle_time = 0; - int total_idle_count = 0; - for (; vc != nullptr; vc = vc_next) { - vc_next = vc->active_queue_link.next; - if ((vc->inactivity_timeout_in && vc->next_inactivity_timeout_at <= now) || - (vc->active_timeout_in && vc->next_activity_timeout_at <= now)) { - _close_vc(vc, now, handle_event, closed, total_idle_time, total_idle_count); + NetEvent *ne = active_queue.head; + NetEvent *ne_next = nullptr; + int closed = 0; + int handle_event = 0; + int total_idle_time = 0; + int total_idle_count = 0; + for (; ne != nullptr; ne = ne_next) { + ne_next = ne->active_queue_link.next; + // It seems dangerous closing the current ne at this point + // Let the activity_cop deal with it + if (ne == enabling_ne) { + continue; + } + if ((ne->inactivity_timeout_in && ne->next_inactivity_timeout_at <= now) || + (ne->active_timeout_in && ne->next_activity_timeout_at <= now)) { + _close_ne(ne, now, handle_event, closed, total_idle_time, total_idle_count); } - if (ignore_queue_size == false && max_connections_active_per_thread_in > active_queue_size) { + if (ignore_queue_size == false && max_requests_per_thread_in > active_queue_size) { return true; } } - if (max_connections_active_per_thread_in > active_queue_size) { + if (max_requests_per_thread_in > active_queue_size) { return true; } @@ -595,12 +618,11 @@ void NetHandler::configure_per_thread_values() { // figure out the number of threads and calculate the number of connections per thread - int threads = eventProcessor.thread_group[ET_NET]._count; - max_connections_per_thread_in = config.max_connections_in / threads; - max_connections_active_per_thread_in = config.max_connections_active_in / threads; + int threads = eventProcessor.thread_group[ET_NET]._count; + max_connections_per_thread_in = config.max_connections_in / threads; + max_requests_per_thread_in = config.max_requests_in / threads; Debug("net_queue", "max_connections_per_thread_in updated to %d threads: %d", max_connections_per_thread_in, threads); - Debug("net_queue", "max_connections_active_per_thread_in updated to %d threads: %d", max_connections_active_per_thread_in, - threads); + Debug("net_queue", "max_requests_per_thread_in updated to %d threads: %d", max_requests_per_thread_in, threads); } void @@ -617,14 +639,14 @@ NetHandler::manage_keep_alive_queue() } // loop over the non-active connections and try to close them - UnixNetVConnection *vc_next = nullptr; - int closed = 0; - int handle_event = 0; - int total_idle_time = 0; - int total_idle_count = 0; - for (UnixNetVConnection *vc = keep_alive_queue.head; vc != nullptr; vc = vc_next) { - vc_next = vc->keep_alive_queue_link.next; - _close_vc(vc, now, handle_event, closed, total_idle_time, total_idle_count); + NetEvent *ne_next = nullptr; + int closed = 0; + int handle_event = 0; + int total_idle_time = 0; + int total_idle_count = 0; + for (NetEvent *ne = keep_alive_queue.head; ne != nullptr; ne = ne_next) { + ne_next = ne->keep_alive_queue_link.next; + _close_ne(ne, now, handle_event, closed, total_idle_time, total_idle_count); total_connections_in = active_queue_size + keep_alive_queue_size; if (total_connections_in <= max_connections_per_thread_in) { @@ -640,40 +662,39 @@ NetHandler::manage_keep_alive_queue() } void -NetHandler::_close_vc(UnixNetVConnection *vc, ink_hrtime now, int &handle_event, int &closed, int &total_idle_time, - int &total_idle_count) +NetHandler::_close_ne(NetEvent *ne, ink_hrtime now, int &handle_event, int &closed, int &total_idle_time, int &total_idle_count) { - if (vc->thread != this_ethread()) { + if (ne->get_thread() != this_ethread()) { return; } - MUTEX_TRY_LOCK(lock, vc->mutex, this_ethread()); + MUTEX_TRY_LOCK(lock, ne->get_mutex(), this_ethread()); if (!lock.is_locked()) { return; } - ink_hrtime diff = (now - (vc->next_inactivity_timeout_at - vc->inactivity_timeout_in)) / HRTIME_SECOND; + ink_hrtime diff = (now - (ne->next_inactivity_timeout_at - ne->inactivity_timeout_in)) / HRTIME_SECOND; if (diff > 0) { total_idle_time += diff; ++total_idle_count; NET_SUM_DYN_STAT(keep_alive_queue_timeout_total_stat, diff); NET_INCREMENT_DYN_STAT(keep_alive_queue_timeout_count_stat); } - Debug("net_queue", "closing connection NetVC=%p idle: %u now: %" PRId64 " at: %" PRId64 " in: %" PRId64 " diff: %" PRId64, vc, - keep_alive_queue_size, ink_hrtime_to_sec(now), ink_hrtime_to_sec(vc->next_inactivity_timeout_at), - ink_hrtime_to_sec(vc->inactivity_timeout_in), diff); - if (vc->closed) { - free_netvc(vc); + Debug("net_queue", "closing connection NetEvent=%p idle: %u now: %" PRId64 " at: %" PRId64 " in: %" PRId64 " diff: %" PRId64, ne, + keep_alive_queue_size, ink_hrtime_to_sec(now), ink_hrtime_to_sec(ne->next_inactivity_timeout_at), + ink_hrtime_to_sec(ne->inactivity_timeout_in), diff); + if (ne->closed) { + free_netevent(ne); ++closed; } else { - vc->next_inactivity_timeout_at = now; + ne->next_inactivity_timeout_at = now; // create a dummy event Event event; event.ethread = this_ethread(); - if (vc->inactivity_timeout_in && vc->next_inactivity_timeout_at <= now) { - if (vc->handleEvent(VC_EVENT_INACTIVITY_TIMEOUT, &event) == EVENT_DONE) { + if (ne->inactivity_timeout_in && ne->next_inactivity_timeout_at <= now) { + if (ne->callback(VC_EVENT_INACTIVITY_TIMEOUT, &event) == EVENT_DONE) { ++handle_event; } - } else if (vc->active_timeout_in && vc->next_activity_timeout_at <= now) { - if (vc->handleEvent(VC_EVENT_ACTIVE_TIMEOUT, &event) == EVENT_DONE) { + } else if (ne->active_timeout_in && ne->next_activity_timeout_at <= now) { + if (ne->callback(VC_EVENT_ACTIVE_TIMEOUT, &event) == EVENT_DONE) { ++handle_event; } } @@ -681,72 +702,78 @@ NetHandler::_close_vc(UnixNetVConnection *vc, ink_hrtime now, int &handle_event, } void -NetHandler::add_to_keep_alive_queue(UnixNetVConnection *vc) +NetHandler::add_to_keep_alive_queue(NetEvent *ne) { - Debug("net_queue", "NetVC: %p", vc); + Debug("net_queue", "NetEvent: %p", ne); ink_assert(mutex->thread_holding == this_ethread()); - if (keep_alive_queue.in(vc)) { + if (keep_alive_queue.in(ne)) { // already in the keep-alive queue, move the head - keep_alive_queue.remove(vc); + keep_alive_queue.remove(ne); } else { // in the active queue or no queue, new to this queue - remove_from_active_queue(vc); + remove_from_active_queue(ne); ++keep_alive_queue_size; } - keep_alive_queue.enqueue(vc); + keep_alive_queue.enqueue(ne); // if keep-alive queue is over size then close connections manage_keep_alive_queue(); } void -NetHandler::remove_from_keep_alive_queue(UnixNetVConnection *vc) +NetHandler::remove_from_keep_alive_queue(NetEvent *ne) { - Debug("net_queue", "NetVC: %p", vc); + Debug("net_queue", "NetEvent: %p", ne); ink_assert(mutex->thread_holding == this_ethread()); - if (keep_alive_queue.in(vc)) { - keep_alive_queue.remove(vc); + if (keep_alive_queue.in(ne)) { + keep_alive_queue.remove(ne); --keep_alive_queue_size; } } bool -NetHandler::add_to_active_queue(UnixNetVConnection *vc) +NetHandler::add_to_active_queue(NetEvent *ne) { - Debug("net_queue", "NetVC: %p", vc); + Debug("net_queue", "NetEvent: %p", ne); Debug("net_queue", "max_connections_per_thread_in: %d active_queue_size: %d keep_alive_queue_size: %d", max_connections_per_thread_in, active_queue_size, keep_alive_queue_size); ink_assert(mutex->thread_holding == this_ethread()); + bool active_queue_full = false; + // if active queue is over size then close inactive connections - if (manage_active_queue() == false) { - // there is no room left in the queue - return false; + if (manage_active_queue(ne) == false) { + active_queue_full = true; } - if (active_queue.in(vc)) { + if (active_queue.in(ne)) { // already in the active queue, move the head - active_queue.remove(vc); + active_queue.remove(ne); } else { + if (active_queue_full) { + // there is no room left in the queue + NET_SUM_DYN_STAT(net_requests_max_throttled_in_stat, 1); + return false; + } // in the keep-alive queue or no queue, new to this queue - remove_from_keep_alive_queue(vc); + remove_from_keep_alive_queue(ne); ++active_queue_size; } - active_queue.enqueue(vc); + active_queue.enqueue(ne); return true; } void -NetHandler::remove_from_active_queue(UnixNetVConnection *vc) +NetHandler::remove_from_active_queue(NetEvent *ne) { - Debug("net_queue", "NetVC: %p", vc); + Debug("net_queue", "NetEvent: %p", ne); ink_assert(mutex->thread_holding == this_ethread()); - if (active_queue.in(vc)) { - active_queue.remove(vc); + if (active_queue.in(ne)) { + active_queue.remove(ne); --active_queue_size; } } diff --git a/iocore/net/UnixNetAccept.cc b/iocore/net/UnixNetAccept.cc index 3a452f62f14..89c1c9646f5 100644 --- a/iocore/net/UnixNetAccept.cc +++ b/iocore/net/UnixNetAccept.cc @@ -201,15 +201,17 @@ NetAccept::init_accept(EThread *t) t->schedule_every(this, period); } -void -NetAccept::init_accept_per_thread() +int +NetAccept::accept_per_thread(int event, void *ep) { - int i, n; + int listen_per_thread = 0; + REC_ReadConfigInteger(listen_per_thread, "proxy.config.exec_thread.listen"); - ink_assert(opt.etype >= 0); - - if (do_listen(NON_BLOCKING)) { - return; + if (listen_per_thread == 1) { + if (do_listen(NON_BLOCKING)) { + Fatal("[NetAccept::accept_per_thread]:error listenting on ports"); + return -1; + } } if (accept_fn == net_accept) { @@ -217,21 +219,38 @@ NetAccept::init_accept_per_thread() } else { SET_HANDLER((NetAcceptHandler)&NetAccept::acceptEvent); } + PollDescriptor *pd = get_PollDescriptor(this_ethread()); + if (this->ep.start(pd, this, EVENTIO_READ) < 0) { + Fatal("[NetAccept::accept_per_thread]:error starting EventIO"); + return -1; + } + return 0; +} - period = -HRTIME_MSECONDS(net_accept_period); - n = eventProcessor.thread_group[opt.etype]._count; +void +NetAccept::init_accept_per_thread() +{ + int i, n; + int listen_per_thread = 0; - for (i = 0; i < n; i++) { - NetAccept *a = (i < n - 1) ? clone() : this; - EThread *t = eventProcessor.thread_group[opt.etype]._thread[i]; - PollDescriptor *pd = get_PollDescriptor(t); + ink_assert(opt.etype >= 0); + REC_ReadConfigInteger(listen_per_thread, "proxy.config.exec_thread.listen"); - if (a->ep.start(pd, a, EVENTIO_READ) < 0) { - Warning("[NetAccept::init_accept_per_thread]:error starting EventIO"); + if (listen_per_thread == 0) { + if (do_listen(NON_BLOCKING)) { + Fatal("[NetAccept::accept_per_thread]:error listenting on ports"); + return; } + } - a->mutex = get_NetHandler(t)->mutex; - t->schedule_every(a, period); + SET_HANDLER((NetAcceptHandler)&NetAccept::accept_per_thread); + n = eventProcessor.thread_group[opt.etype]._count; + + for (i = 0; i < n; i++) { + NetAccept *a = (i < n - 1) ? clone() : this; + EThread *t = eventProcessor.thread_group[opt.etype]._thread[i]; + a->mutex = get_NetHandler(t)->mutex; + t->schedule_imm(a); } } @@ -337,11 +356,11 @@ NetAccept::do_blocking_accept(EThread *t) #endif SET_CONTINUATION_HANDLER(vc, (NetVConnHandler)&UnixNetVConnection::acceptEvent); - EThread *t = eventProcessor.assign_thread(opt.etype); - NetHandler *h = get_NetHandler(t); + EThread *localt = eventProcessor.assign_thread(opt.etype); + NetHandler *h = get_NetHandler(localt); // Assign NetHandler->mutex to NetVC vc->mutex = h->mutex; - t->schedule_imm_signal(vc); + localt->schedule_imm(vc); } while (loop); return 1; @@ -401,16 +420,18 @@ NetAccept::acceptFastEvent(int event, void *ep) int loop = accept_till_done; do { - if (!opt.backdoor && check_net_throttle(ACCEPT)) { - ifd = NO_FD; - return EVENT_CONT; - } - socklen_t sz = sizeof(con.addr); int fd = socketManager.accept4(server.fd, &con.addr.sa, &sz, SOCK_NONBLOCK | SOCK_CLOEXEC); con.fd = fd; if (likely(fd >= 0)) { + // check for throttle + if (!opt.backdoor && check_net_throttle(ACCEPT)) { + // close the connection as we are in throttle state + con.close(); + NET_SUM_DYN_STAT(net_connections_throttled_in_stat, 1); + continue; + } Debug("iocore_net", "accepted a new socket: %d", fd); NET_SUM_GLOBAL_DYN_STAT(net_tcp_accept_stat, 1); if (opt.send_bufsize > 0) { @@ -440,6 +461,7 @@ NetAccept::acceptFastEvent(int event, void *ep) } // check return value from accept() if (res < 0) { + Debug("iocore_net", "received : %s", strerror(errno)); res = -errno; if (res == -EAGAIN || res == -ECONNABORTED #if defined(linux) @@ -458,9 +480,7 @@ NetAccept::acceptFastEvent(int event, void *ep) } vc = (UnixNetVConnection *)this->getNetProcessor()->allocate_vc(e->ethread); - if (!vc) { - goto Ldone; - } + ink_release_assert(vc); NET_SUM_GLOBAL_DYN_STAT(net_connections_currently_open_stat, 1); vc->id = net_next_connection_number(); diff --git a/iocore/net/UnixNetPages.cc b/iocore/net/UnixNetPages.cc index bd8d64002bd..702e2ebb440 100644 --- a/iocore/net/UnixNetPages.cc +++ b/iocore/net/UnixNetPages.cc @@ -61,10 +61,11 @@ struct ShowNet : public ShowCont { } ink_hrtime now = Thread::get_hrtime(); - forl_LL(UnixNetVConnection, vc, nh->open_list) + forl_LL(NetEvent, ne, nh->open_list) { + auto vc = dynamic_cast(ne); // uint16_t port = ats_ip_port_host_order(&addr.sa); - if (ats_is_ip(&addr) && !ats_ip_addr_port_eq(&addr.sa, vc->get_remote_addr())) { + if (vc == nullptr || (ats_is_ip(&addr) && !ats_ip_addr_port_eq(&addr.sa, vc->get_remote_addr()))) { continue; } // if (port && port != ats_ip_port_host_order(&vc->server_addr.sa) && port != vc->accept_port) @@ -158,7 +159,12 @@ struct ShowNet : public ShowCont { CHECK_SHOW(show("

Thread: %d

\n", ithread)); CHECK_SHOW(show("\n")); int connections = 0; - forl_LL(UnixNetVConnection, vc, nh->open_list) connections++; + forl_LL(NetEvent, ne, nh->open_list) + { + if (dynamic_cast(ne) != nullptr) { + ++connections; + } + } CHECK_SHOW(show("\n", "Connections", connections)); // CHECK_SHOW(show("\n", "Last Poll Size", pollDescriptor->nfds)); CHECK_SHOW(show("\n", "Last Poll Ready", pollDescriptor->result)); diff --git a/iocore/net/UnixNetProcessor.cc b/iocore/net/UnixNetProcessor.cc index 11bf503187d..5d8c71aa717 100644 --- a/iocore/net/UnixNetProcessor.cc +++ b/iocore/net/UnixNetProcessor.cc @@ -92,6 +92,7 @@ UnixNetProcessor::accept_internal(Continuation *cont, int fd, AcceptOptions cons ProxyMutex *mutex = this_ethread()->mutex.get(); int accept_threads = opt.accept_threads; // might be changed. IpEndpoint accept_ip; // local binding address. + int listen_per_thread = 0; NetAccept *na = createNetAccept(opt); na->id = ink_atomic_increment(&net_accept_number, 1); @@ -101,6 +102,10 @@ UnixNetProcessor::accept_internal(Continuation *cont, int fd, AcceptOptions cons if (opt.accept_threads < 0) { REC_ReadConfigInteger(accept_threads, "proxy.config.accept_threads"); } + REC_ReadConfigInteger(listen_per_thread, "proxy.config.exec_thread.listen"); + if (accept_threads > 0 && listen_per_thread > 0) { + Fatal("Please disable accept_threads or exec_threads.listen"); + } NET_INCREMENT_DYN_STAT(net_accepts_currently_open_stat); @@ -137,7 +142,7 @@ UnixNetProcessor::accept_internal(Continuation *cont, int fd, AcceptOptions cons na->action_->server = &na->server; if (opt.frequent_accept) { // true - if (accept_threads > 0) { + if (accept_threads > 0 && listen_per_thread == 0) { na->init_accept_loop(); } else { na->init_accept_per_thread(); @@ -176,7 +181,7 @@ Action * UnixNetProcessor::connect_re_internal(Continuation *cont, sockaddr const *target, NetVCOptions *opt) { if (TSSystemState::is_event_system_shut_down()) { - return ACTION_RESULT_NONE; + return nullptr; } EThread *t = eventProcessor.assign_affinity_by_type(cont, opt->etype); UnixNetVConnection *vc = (UnixNetVConnection *)this->allocate_vc(t); diff --git a/iocore/net/UnixNetVConnection.cc b/iocore/net/UnixNetVConnection.cc index e4e8cf67d70..c1c608b9836 100644 --- a/iocore/net/UnixNetVConnection.cc +++ b/iocore/net/UnixNetVConnection.cc @@ -88,6 +88,7 @@ read_signal_and_update(int event, UnixNetVConnection *vc) case VC_EVENT_ACTIVE_TIMEOUT: case VC_EVENT_INACTIVITY_TIMEOUT: Debug("inactivity_cop", "event %d: null read.vio cont, closing vc %p", event, vc); + Warning("read: Closing orphaned vc %p", vc); vc->closed = 1; break; default: @@ -99,7 +100,7 @@ read_signal_and_update(int event, UnixNetVConnection *vc) if (!--vc->recursion && vc->closed) { /* BZ 31932 */ ink_assert(vc->thread == this_ethread()); - vc->nh->free_netvc(vc); + vc->nh->free_netevent(vc); return EVENT_DONE; } else { return EVENT_CONT; @@ -119,6 +120,7 @@ write_signal_and_update(int event, UnixNetVConnection *vc) case VC_EVENT_ACTIVE_TIMEOUT: case VC_EVENT_INACTIVITY_TIMEOUT: Debug("inactivity_cop", "event %d: null write.vio cont, closing vc %p", event, vc); + Warning("write: Closing orphaned vc %p", vc); vc->closed = 1; break; default: @@ -130,7 +132,7 @@ write_signal_and_update(int event, UnixNetVConnection *vc) if (!--vc->recursion && vc->closed) { /* BZ 31932 */ ink_assert(vc->thread == this_ethread()); - vc->nh->free_netvc(vc); + vc->nh->free_netevent(vc); return EVENT_DONE; } else { return EVENT_CONT; @@ -197,7 +199,7 @@ read_from_net(NetHandler *nh, UnixNetVConnection *vc, EThread *thread) // global session pool case. If so, the closed flag should be stable once we get the // s->vio.mutex (the global session pool mutex). if (vc->closed) { - vc->nh->free_netvc(vc); + vc->nh->free_netevent(vc); return; } // if it is not enabled. @@ -288,8 +290,9 @@ read_from_net(NetHandler *nh, UnixNetVConnection *vc, EThread *thread) // Add data to buffer and signal continuation. buf.writer()->fill(r); #ifdef DEBUG - if (buf.writer()->write_avail() <= 0) + if (buf.writer()->write_avail() <= 0) { Debug("iocore_net", "read_from_net, read buffer full"); + } #endif s->vio.ndone += r; net_activity(vc, thread); @@ -657,7 +660,7 @@ UnixNetVConnection::do_io_close(int alerrno /* = -1 */) if (close_inline) { if (nh) { - nh->free_netvc(this); + nh->free_netevent(this); } else { free(t); } @@ -866,8 +869,7 @@ UnixNetVConnection::reenable_re(VIO *vio) } } -UnixNetVConnection::UnixNetVConnection() : flags(0) - +UnixNetVConnection::UnixNetVConnection() { SET_HANDLER((NetVConnHandler)&UnixNetVConnection::startEvent); } @@ -891,6 +893,12 @@ UnixNetVConnection::net_read_io(NetHandler *nh, EThread *lthread) read_from_net(nh, this, lthread); } +void +UnixNetVConnection::net_write_io(NetHandler *nh, EThread *lthread) +{ + write_to_net(nh, this, lthread); +} + // This code was pulled out of write_to_net so // I could overwrite it for the SSL implementation // (SSL read does not support overlapped i/o) @@ -908,7 +916,7 @@ UnixNetVConnection::load_buffer_and_write(int64_t towrite, MIOBufferAccessor &bu try_to_write = 0; while (niov < NET_MAX_IOV) { - int64_t wavail = towrite - total_written; + int64_t wavail = towrite - total_written - try_to_write; int64_t len = tmp_reader->block_read_avail(); // Check if we have done this block. @@ -1121,6 +1129,11 @@ UnixNetVConnection::mainEvent(int event, Event *e) Event **signal_timeout = &t; switch (event) { + // Treating immediate as inactivity timeout for any + // deprecated remaining immediates. The previous code was using EVENT_INTERVAL + // and EVENT_IMMEDIATE to distinguish active and inactive timeouts. + // There appears to be some stray EVENT_IMMEDIATEs floating around + case EVENT_IMMEDIATE: case VC_EVENT_INACTIVITY_TIMEOUT: signal_event = VC_EVENT_INACTIVITY_TIMEOUT; signal_timeout_at = &next_inactivity_timeout_at; @@ -1139,7 +1152,7 @@ UnixNetVConnection::mainEvent(int event, Event *e) writer_cont = write.vio.cont; if (closed) { - nh->free_netvc(this); + nh->free_netevent(this); return EVENT_DONE; } @@ -1176,7 +1189,6 @@ UnixNetVConnection::populate(Connection &con_in, Continuation *c, void *arg) } if (h->startIO(this) < 0) { - con_in.move(this->con); Debug("iocore_net", "populate : Failed to add to epoll list"); return EVENT_ERROR; } @@ -1348,12 +1360,23 @@ TS_INLINE void UnixNetVConnection::set_inactivity_timeout(ink_hrtime timeout_in) { Debug("socket", "Set inactive timeout=%" PRId64 ", for NetVC=%p", timeout_in, this); - if (timeout_in == 0) { - // set default inactivity timeout - timeout_in = HRTIME_SECONDS(nh->config.default_inactivity_timeout); - } inactivity_timeout_in = timeout_in; - next_inactivity_timeout_at = Thread::get_hrtime() + inactivity_timeout_in; + next_inactivity_timeout_at = (timeout_in > 0) ? Thread::get_hrtime() + inactivity_timeout_in : 0; +} + +TS_INLINE void +UnixNetVConnection::set_default_inactivity_timeout(ink_hrtime timeout_in) +{ + Debug("socket", "Set default inactive timeout=%" PRId64 ", for NetVC=%p", timeout_in, this); + inactivity_timeout_in = 0; + default_inactivity_timeout = true; + next_inactivity_timeout_at = Thread::get_hrtime() + timeout_in; +} + +TS_INLINE bool +UnixNetVConnection::is_default_inactivity_timeout() +{ + return (default_inactivity_timeout && inactivity_timeout_in == 0); } /* @@ -1370,71 +1393,50 @@ UnixNetVConnection::migrateToCurrentThread(Continuation *cont, EThread *t) return this; } - // Lock the NetHandler first in order to put the new NetVC into NetHandler and InactivityCop. - // It is safe and no performance issue to get the mutex lock for a NetHandler of current ethread. - SCOPED_MUTEX_LOCK(lock, client_nh->mutex, t); - - // Try to get the mutex lock for NetHandler of this NetVC - MUTEX_TRY_LOCK(lock_src, this->nh->mutex, t); - if (lock_src.is_locked()) { - // Detach this NetVC from original NetHandler & InactivityCop. - this->nh->stopCop(this); - this->nh->stopIO(this); - // Put this NetVC into current NetHandler & InactivityCop. - this->thread = t; - client_nh->startIO(this); - client_nh->startCop(this); - // Move this NetVC to current EThread Successfully. - return this; - } - - // Failed to get the mutex lock for original NetHandler. - // Try to migrate it by create a new NetVC and then move con.fd and ssl ctx. + Connection hold_con; + hold_con.move(this->con); SSLNetVConnection *sslvc = dynamic_cast(this); - SSL *save_ssl = (sslvc) ? sslvc->ssl : nullptr; - UnixNetVConnection *ret_vc = nullptr; + SSL *save_ssl = nullptr; + if (sslvc) { + save_ssl = sslvc->ssl; + SSLNetVCDetach(sslvc->ssl); + sslvc->ssl = nullptr; + } + + // Do_io_close will signal the VC to be freed on the original thread + // Since we moved the con context, the fd will not be closed + // Go ahead and remove the fd from the original thread's epoll structure, so it is not + // processed on two threads simultaneously + this->ep.stop(); // Create new VC: + UnixNetVConnection *netvc = nullptr; if (save_ssl) { - SSLNetVConnection *sslvc = static_cast(sslNetProcessor.allocate_vc(t)); - if (sslvc->populate(this->con, cont, save_ssl) != EVENT_DONE) { - sslvc->free(t); - sslvc = nullptr; - ret_vc = this; + sslvc = static_cast(sslNetProcessor.allocate_vc(t)); + if (sslvc->populate(hold_con, cont, save_ssl) != EVENT_DONE) { + sslvc->do_io_close(); + sslvc = nullptr; } else { + // Update the SSL fields sslvc->set_context(get_context()); - ret_vc = dynamic_cast(sslvc); } + netvc = sslvc; } else { - UnixNetVConnection *netvc = static_cast(netProcessor.allocate_vc(t)); - if (netvc->populate(this->con, cont, save_ssl) != EVENT_DONE) { - netvc->free(t); - netvc = nullptr; - ret_vc = this; + netvc = static_cast(netProcessor.allocate_vc(t)); + if (netvc->populate(hold_con, cont, save_ssl) != EVENT_DONE) { + netvc->do_io_close(); + netvc = nullptr; } else { netvc->set_context(get_context()); - ret_vc = netvc; } } - - // clear con.fd and ssl ctx from this NetVC since a new NetVC is created. - if (ret_vc != this) { - if (save_ssl) { - SSLNetVCDetach(sslvc->ssl); - sslvc->ssl = nullptr; - } - ink_assert(this->con.fd == NO_FD); - - // Do_io_close will signal the VC to be freed on the original thread - // Since we moved the con context, the fd will not be closed - // Go ahead and remove the fd from the original thread's epoll structure, so it is not - // processed on two threads simultaneously - this->ep.stop(); - this->do_io_close(); + if (netvc) { + netvc->options = this->options; } - - return ret_vc; + // Do not mark this closed until the end so it does not get freed by the other thread too soon + this->do_io_close(); + return netvc; } void diff --git a/iocore/net/UnixUDPConnection.cc b/iocore/net/UnixUDPConnection.cc index 4d1eaf370bf..57c8814f2c7 100644 --- a/iocore/net/UnixUDPConnection.cc +++ b/iocore/net/UnixUDPConnection.cc @@ -121,7 +121,7 @@ UDPConnection::send(Continuation *c, UDPPacket *xp) if (shouldDestroy()) { ink_assert(!"freeing packet sent on dead connection"); p->free(); - return ACTION_RESULT_NONE; + return nullptr; } ink_assert(mutex == c->mutex); @@ -132,7 +132,7 @@ UDPConnection::send(Continuation *c, UDPPacket *xp) mutex = c->mutex; p->reqGenerationNum = conn->sendGenerationNum; get_UDPNetHandler(conn->ethread)->udpOutQueue.send(p); - return ACTION_RESULT_NONE; + return nullptr; } void diff --git a/iocore/net/UnixUDPNet.cc b/iocore/net/UnixUDPNet.cc index d2d3a7157f6..6b82104df4b 100644 --- a/iocore/net/UnixUDPNet.cc +++ b/iocore/net/UnixUDPNet.cc @@ -647,7 +647,7 @@ UDPNetProcessor::CreateUDPSocket(int *resfd, sockaddr const *remote_addr, Action } fd = res; - if ((res = safe_fcntl(fd, F_SETFL, O_NONBLOCK)) < 0) { + if (safe_fcntl(fd, F_SETFL, O_NONBLOCK) < 0) { goto HardError; } @@ -666,12 +666,12 @@ UDPNetProcessor::CreateUDPSocket(int *resfd, sockaddr const *remote_addr, Action bool succeeded = false; int enable = 1; #ifdef IP_PKTINFO - if ((res = safe_setsockopt(fd, IPPROTO_IP, IP_PKTINFO, reinterpret_cast(&enable), sizeof(enable))) == 0) { + if (safe_setsockopt(fd, IPPROTO_IP, IP_PKTINFO, reinterpret_cast(&enable), sizeof(enable)) == 0) { succeeded = true; } #endif #ifdef IP_RECVDSTADDR - if ((res = safe_setsockopt(fd, IPPROTO_IP, IP_RECVDSTADDR, reinterpret_cast(&enable), sizeof(enable))) == 0) { + if (safe_setsockopt(fd, IPPROTO_IP, IP_RECVDSTADDR, reinterpret_cast(&enable), sizeof(enable)) == 0) { succeeded = true; } #endif @@ -683,12 +683,12 @@ UDPNetProcessor::CreateUDPSocket(int *resfd, sockaddr const *remote_addr, Action bool succeeded = false; int enable = 1; #ifdef IPV6_PKTINFO - if ((res = safe_setsockopt(fd, IPPROTO_IPV6, IPV6_PKTINFO, reinterpret_cast(&enable), sizeof(enable))) == 0) { + if (safe_setsockopt(fd, IPPROTO_IPV6, IPV6_PKTINFO, reinterpret_cast(&enable), sizeof(enable)) == 0) { succeeded = true; } #endif #ifdef IPV6_RECVPKTINFO - if ((res = safe_setsockopt(fd, IPPROTO_IPV6, IPV6_RECVPKTINFO, reinterpret_cast(&enable), sizeof(enable))) == 0) { + if (safe_setsockopt(fd, IPPROTO_IPV6, IPV6_RECVPKTINFO, reinterpret_cast(&enable), sizeof(enable)) == 0) { succeeded = true; } #endif @@ -705,7 +705,7 @@ UDPNetProcessor::CreateUDPSocket(int *resfd, sockaddr const *remote_addr, Action goto SoftError; } - if ((res = safe_getsockname(fd, &local_addr.sa, &local_addr_len)) < 0) { + if (safe_getsockname(fd, &local_addr.sa, &local_addr_len) < 0) { Debug("udpnet", "CreateUdpsocket: getsockname didn't work"); goto HardError; } @@ -735,21 +735,25 @@ UDPNetProcessor::CreateUDPSocket(int *resfd, sockaddr const *remote_addr, Action } Action * -UDPNetProcessor::UDPBind(Continuation *cont, sockaddr const *addr, int send_bufsize, int recv_bufsize) +UDPNetProcessor::UDPBind(Continuation *cont, sockaddr const *addr, int fd, int send_bufsize, int recv_bufsize) { int res = 0; - int fd = -1; UnixUDPConnection *n = nullptr; IpEndpoint myaddr; int myaddr_len = sizeof(myaddr); PollCont *pc = nullptr; PollDescriptor *pd = nullptr; + bool need_bind = true; - if ((res = socketManager.socket(addr->sa_family, SOCK_DGRAM, 0)) < 0) { - goto Lerror; + if (fd == -1) { + if ((res = socketManager.socket(addr->sa_family, SOCK_DGRAM, 0)) < 0) { + goto Lerror; + } + fd = res; + } else { + need_bind = false; } - fd = res; - if ((res = fcntl(fd, F_SETFL, O_NONBLOCK) < 0)) { + if (fcntl(fd, F_SETFL, O_NONBLOCK) < 0) { goto Lerror; } @@ -757,12 +761,12 @@ UDPNetProcessor::UDPBind(Continuation *cont, sockaddr const *addr, int send_bufs bool succeeded = false; int enable = 1; #ifdef IP_PKTINFO - if ((res = safe_setsockopt(fd, IPPROTO_IP, IP_PKTINFO, reinterpret_cast(&enable), sizeof(enable))) == 0) { + if (safe_setsockopt(fd, IPPROTO_IP, IP_PKTINFO, reinterpret_cast(&enable), sizeof(enable)) == 0) { succeeded = true; } #endif #ifdef IP_RECVDSTADDR - if ((res = safe_setsockopt(fd, IPPROTO_IP, IP_RECVDSTADDR, reinterpret_cast(&enable), sizeof(enable))) == 0) { + if (safe_setsockopt(fd, IPPROTO_IP, IP_RECVDSTADDR, reinterpret_cast(&enable), sizeof(enable)) == 0) { succeeded = true; } #endif @@ -774,12 +778,12 @@ UDPNetProcessor::UDPBind(Continuation *cont, sockaddr const *addr, int send_bufs bool succeeded = false; int enable = 1; #ifdef IPV6_PKTINFO - if ((res = safe_setsockopt(fd, IPPROTO_IPV6, IPV6_PKTINFO, reinterpret_cast(&enable), sizeof(enable))) == 0) { + if (safe_setsockopt(fd, IPPROTO_IPV6, IPV6_PKTINFO, reinterpret_cast(&enable), sizeof(enable)) == 0) { succeeded = true; } #endif #ifdef IPV6_RECVPKTINFO - if ((res = safe_setsockopt(fd, IPPROTO_IPV6, IPV6_RECVPKTINFO, reinterpret_cast(&enable), sizeof(enable))) == 0) { + if (safe_setsockopt(fd, IPPROTO_IPV6, IPV6_RECVPKTINFO, reinterpret_cast(&enable), sizeof(enable)) == 0) { succeeded = true; } #endif @@ -793,17 +797,17 @@ UDPNetProcessor::UDPBind(Continuation *cont, sockaddr const *addr, int send_bufs if (ats_is_ip_multicast(addr)) { int enable_reuseaddr = 1; - if ((res = safe_setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, reinterpret_cast(&enable_reuseaddr), - sizeof(enable_reuseaddr)) < 0)) { + if (safe_setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, reinterpret_cast(&enable_reuseaddr), sizeof(enable_reuseaddr)) < 0) { goto Lerror; } } - if (ats_is_ip6(addr) && (res = safe_setsockopt(fd, IPPROTO_IPV6, IPV6_V6ONLY, SOCKOPT_ON, sizeof(int))) < 0) { + if (need_bind && ats_is_ip6(addr) && safe_setsockopt(fd, IPPROTO_IPV6, IPV6_V6ONLY, SOCKOPT_ON, sizeof(int)) < 0) { goto Lerror; } - if ((res = socketManager.ink_bind(fd, addr, ats_ip_size(addr))) < 0) { + if (need_bind && (socketManager.ink_bind(fd, addr, ats_ip_size(addr)) < 0)) { + Debug("udpnet", "ink_bind failed"); goto Lerror; } @@ -817,7 +821,7 @@ UDPNetProcessor::UDPBind(Continuation *cont, sockaddr const *addr, int send_bufs Debug("udpnet", "set_dnsbuf_size(%d) failed", send_bufsize); } } - if ((res = safe_getsockname(fd, &myaddr.sa, &myaddr_len)) < 0) { + if (safe_getsockname(fd, &myaddr.sa, &myaddr_len) < 0) { goto Lerror; } n = new UnixUDPConnection(fd); diff --git a/iocore/net/YamlSNIConfig.cc b/iocore/net/YamlSNIConfig.cc index 7f909ff03aa..22b0fb66ce0 100644 --- a/iocore/net/YamlSNIConfig.cc +++ b/iocore/net/YamlSNIConfig.cc @@ -43,7 +43,7 @@ YamlSNIConfig::loader(const char *cfgFilename) } if (!config["sni"]) { - return ts::Errata::Message(1, 1, "malformed ssl_server_name.yaml file; expected a toplevel 'sni' node"); + return ts::Errata::Message(1, 1, "expected a toplevel 'sni' node"); } config = config["sni"]; @@ -98,18 +98,17 @@ std::set valid_sni_config_keys = {TS_fqdn, TS_verify_client, TS_tunnel_route, TS_forward_route, - TS_verify_origin_server, + TS_partial_blind_route, TS_verify_server_policy, TS_verify_server_properties, TS_client_cert, TS_client_key, TS_http2, - TS_ip_allow + TS_ip_allow, #if TS_USE_HELLO_CB - , - TS_valid_tls_versions_in + TS_valid_tls_versions_in, #endif -}; + TS_host_sni_policy}; namespace YAML { @@ -117,10 +116,10 @@ template <> struct convert { static bool decode(const Node &node, YamlSNIConfig::Item &item) { - for (auto &&item : node) { + for (const auto &elem : node) { if (std::none_of(valid_sni_config_keys.begin(), valid_sni_config_keys.end(), - [&item](const std::string &s) { return s == item.first.as(); })) { - throw YAML::ParserException(item.Mark(), "unsupported key " + item.first.as()); + [&elem](const std::string &s) { return s == elem.first.as(); })) { + throw YAML::ParserException(elem.first.Mark(), "unsupported key " + elem.first.as()); } } @@ -146,31 +145,24 @@ template <> struct convert { item.verify_client_level = static_cast(level); } + if (node[TS_host_sni_policy]) { + auto value = node[TS_host_sni_policy].as(); + int policy = POLICY_DESCRIPTOR.get(value); + item.host_sni_policy = static_cast(policy); + } + if (node[TS_tunnel_route]) { item.tunnel_destination = node[TS_tunnel_route].as(); item.tunnel_decrypt = false; + item.tls_upstream = false; } else if (node[TS_forward_route]) { item.tunnel_destination = node[TS_forward_route].as(); item.tunnel_decrypt = true; - } - - // remove before 9.0.0 release - // backwards compatibility - if (node[TS_verify_origin_server]) { - auto value = node[TS_verify_origin_server].as(); - YamlSNIConfig::Level level = static_cast(LEVEL_DESCRIPTOR.get(value)); - item.verify_server_properties = YamlSNIConfig::Property::ALL_MASK; - switch (level) { - case YamlSNIConfig::Level::NONE: - item.verify_server_policy = YamlSNIConfig::Policy::DISABLED; - break; - case YamlSNIConfig::Level::MODERATE: - item.verify_server_policy = YamlSNIConfig::Policy::PERMISSIVE; - break; - case YamlSNIConfig::Level::STRICT: - item.verify_server_policy = YamlSNIConfig::Policy::ENFORCED; - break; - } + item.tls_upstream = false; + } else if (node[TS_partial_blind_route]) { + item.tunnel_destination = node[TS_partial_blind_route].as(); + item.tunnel_decrypt = true; + item.tls_upstream = true; } if (node[TS_verify_server_policy]) { diff --git a/iocore/net/YamlSNIConfig.h b/iocore/net/YamlSNIConfig.h index d528388a10a..a94fc2bb658 100644 --- a/iocore/net/YamlSNIConfig.h +++ b/iocore/net/YamlSNIConfig.h @@ -33,6 +33,7 @@ TSDECL(disable_h2); TSDECL(verify_client); TSDECL(tunnel_route); TSDECL(forward_route); +TSDECL(partial_blind_route); TSDECL(verify_server_policy); TSDECL(verify_server_properties); TSDECL(verify_origin_server); @@ -41,6 +42,7 @@ TSDECL(client_key); TSDECL(ip_allow); TSDECL(valid_tls_versions_in); TSDECL(http2); +TSDECL(host_sni_policy); #undef TSDECL const int start = 0; @@ -50,10 +52,12 @@ struct YamlSNIConfig { verify_client, tunnel_route, // blind tunnel action forward_route, // decrypt data and then blind tunnel action + partial_blind_route, // decrypt data; partial blind routing verify_server_policy, // this applies to server side vc only verify_server_properties, // this applies to server side vc only client_cert, - h2 // this applies to client side only + h2, // this applies to client side only + host_sni_policy // Applies to client side only }; enum class Level { NONE = 0, MODERATE, STRICT }; enum class Policy : uint8_t { DISABLED = 0, PERMISSIVE, ENFORCED, UNSET }; @@ -67,8 +71,10 @@ struct YamlSNIConfig { std::string fqdn; std::optional offer_h2; // Has no value by default, so do not initialize! uint8_t verify_client_level = 255; + uint8_t host_sni_policy = 255; std::string tunnel_destination; bool tunnel_decrypt = false; + bool tls_upstream = false; Policy verify_server_policy = Policy::UNSET; Property verify_server_properties = Property::UNSET; std::string client_cert; diff --git a/iocore/net/quic/Makefile.am b/iocore/net/quic/Makefile.am index e13d1de2a93..da29c128cf1 100644 --- a/iocore/net/quic/Makefile.am +++ b/iocore/net/quic/Makefile.am @@ -18,6 +18,7 @@ AM_CPPFLAGS += \ $(iocore_include_dirs) \ + -I$(abs_top_srcdir)/include \ -I$(abs_top_srcdir)/lib \ -I$(abs_top_srcdir)/lib/records \ -I$(abs_top_srcdir)/proxy \ @@ -29,7 +30,7 @@ AM_CPPFLAGS += \ -I$(abs_top_srcdir)/mgmt \ -I$(abs_top_srcdir)/mgmt/utils \ $(TS_INCLUDES) \ - @OPENSSL_INCLUDES@ + @OPENSSL_INCLUDES@ @YAMLCPP_INCLUDES@ noinst_LIBRARIES = libquic.a @@ -39,11 +40,20 @@ QUICPPProtector_impl = QUICPacketPayloadProtector_boringssl.cc QUICTLS_impl = QUICTLS_boringssl.cc QUICKeyGenerator_impl = QUICKeyGenerator_boringssl.cc else +if ENABLE_QUIC_OLD_API +QUICPHProtector_impl = QUICPacketHeaderProtector_legacy.cc +QUICPPProtector_impl = QUICPacketPayloadProtector_legacy.cc +QUICTLS_impl = QUICTLS_legacy.cc +QUICKeyGenerator_impl = QUICKeyGenerator_legacy.cc +else QUICPHProtector_impl = QUICPacketHeaderProtector_openssl.cc QUICPPProtector_impl = QUICPacketPayloadProtector_openssl.cc QUICTLS_impl = QUICTLS_openssl.cc QUICKeyGenerator_impl = QUICKeyGenerator_openssl.cc endif +endif + +QLog_impl = qlog/QLogEvent.cc qlog/QLogFrame.cc qlog/QLog.cc libquic_a_SOURCES = \ QUICGlobals.cc \ @@ -56,7 +66,7 @@ libquic_a_SOURCES = \ QUICVersionNegotiator.cc \ QUICLossDetector.cc \ QUICStreamManager.cc \ - QUICCongestionController.cc \ + QUICNewRenoCongestionController.cc \ QUICFlowController.cc \ QUICStreamState.cc \ QUICStream.cc \ @@ -68,7 +78,6 @@ libquic_a_SOURCES = \ QUICPacketProtectionKeyInfo.cc \ QUICTLS.cc \ $(QUICTLS_impl) \ - QUICKeyGenerator.cc \ $(QUICKeyGenerator_impl) \ QUICKeyGenerator.cc \ QUICHKDF.cc \ @@ -82,15 +91,22 @@ libquic_a_SOURCES = \ QUICApplicationMap.cc \ QUICIncomingFrameBuffer.cc \ QUICPacketReceiveQueue.cc \ + QUICPathManager.cc \ QUICPathValidator.cc \ QUICPinger.cc \ + QUICRetryIntegrityTag.cc \ + QUICResetTokenTable.cc \ QUICFrameGenerator.cc \ QUICFrameRetransmitter.cc \ QUICAddrVerifyState.cc \ QUICBidirectionalStream.cc \ QUICCryptoStream.cc \ QUICUnidirectionalStream.cc \ - QUICStreamFactory.cc + QUICStreamFactory.cc \ + QUICPadder.cc \ + QUICContext.cc \ + QUICTokenCreator.cc \ + $(QLog_impl) # # Check Programs @@ -109,6 +125,7 @@ check_PROGRAMS = \ test_QUICPacket \ test_QUICPacketHeaderProtector \ test_QUICPacketFactory \ + test_QUICPathValidator \ test_QUICStream \ test_QUICStreamManager \ test_QUICStreamState \ @@ -117,13 +134,14 @@ check_PROGRAMS = \ test_QUICTypeUtil \ test_QUICVersionNegotiator \ test_QUICFrameRetransmitter \ - test_QUICAddrVerifyState + test_QUICAddrVerifyState \ + test_QUICPinger TESTS = $(check_PROGRAMS) test_CPPFLAGS = \ $(AM_CPPFLAGS) \ - -I$(abs_top_srcdir)/tests/include + -I$(abs_top_srcdir)/tests/include -O0 test_LDADD = \ libquic.a \ @@ -226,6 +244,13 @@ test_QUICPacketFactory_SOURCES = \ $(test_main_SOURCES) \ ./test/test_QUICPacketFactory.cc +test_QUICPathValidator_CPPFLAGS = $(test_CPPFLAGS) +test_QUICPathValidator_LDFLAGS = @AM_LDFLAGS@ +test_QUICPathValidator_LDADD = $(test_LDADD) +test_QUICPathValidator_SOURCES = \ + $(test_main_SOURCES) \ + ./test/test_QUICPathValidator.cc + test_QUICPacketHeaderProtector_CPPFLAGS = $(test_CPPFLAGS) test_QUICPacketHeaderProtector_LDFLAGS = @AM_LDFLAGS@ test_QUICPacketHeaderProtector_LDADD = $(test_LDADD) @@ -296,6 +321,13 @@ test_QUICAddrVerifyState_SOURCES = \ $(test_main_SOURCES) \ ./test/test_QUICAddrVerifyState.cc +test_QUICPinger_CPPFLAGS = $(test_CPPFLAGS) +test_QUICPinger_LDFLAGS = @AM_LDFLAGS@ +test_QUICPinger_LDADD = $(test_LDADD) +test_QUICPinger_SOURCES = \ + $(test_main_SOURCES) \ + ./test/test_QUICPinger.cc + # # clang-tidy # diff --git a/iocore/net/quic/Mock.h b/iocore/net/quic/Mock.h index d4692b05d43..c0d28a3fa6d 100644 --- a/iocore/net/quic/Mock.h +++ b/iocore/net/quic/Mock.h @@ -28,15 +28,227 @@ #include "QUICApplication.h" #include "QUICStreamManager.h" #include "QUICLossDetector.h" +#include "QUICPacketProtectionKeyInfo.h" +#include "QUICPinger.h" +#include "QUICPadder.h" #include "QUICEvents.h" +#include "QUICPacketProtectionKeyInfo.h" +#include "QUICPinger.h" +#include "QUICPadder.h" +#include "QUICHandshakeProtocol.h" + +class MockQUICContext; using namespace std::literals; -std::string_view negotiated_application_name_sv = "h3-20"sv; +std::string_view negotiated_application_name_sv = "h3-29"sv; + +class MockQUICLDConfig : public QUICLDConfig +{ + uint32_t + packet_threshold() const + { + return 3; + } + + float + time_threshold() const + { + return 1.25; + } + + ink_hrtime + granularity() const + { + return HRTIME_MSECONDS(1); + } + + ink_hrtime + initial_rtt() const + { + return HRTIME_MSECONDS(100); + } +}; + +class MockQUICCCConfig : public QUICCCConfig +{ + uint32_t + max_datagram_size() const + { + return 1200; + } + + uint32_t + initial_window() const + { + return 10; + } + + uint32_t + minimum_window() const + { + return 2; + } + + float + loss_reduction_factor() const + { + return 0.5; + } + + uint32_t + persistent_congestion_threshold() const + { + return 2; + } +}; + +class MockQUICPathManager : public QUICPathManager +{ +public: + virtual ~MockQUICPathManager() {} + virtual const QUICPath & + get_current_path() + { + return _path; + } + virtual const QUICPath & + get_verified_path() + { + return _path; + } + virtual void + open_new_path(const QUICPath &path, ink_hrtime timeout_in) + { + return; + } + virtual void + set_trusted_path(const QUICPath &path) + { + return; + } + +private: + QUICPath _path = {{}, {}}; +}; + +class MockQUICConnectionInfoProvider : public QUICConnectionInfoProvider +{ + QUICConnectionId + connection_id() const override + { + return {reinterpret_cast("\x00"), 1}; + } + + QUICConnectionId + peer_connection_id() const override + { + return {reinterpret_cast("\x00"), 1}; + } + + QUICConnectionId + original_connection_id() const override + { + return {reinterpret_cast("\x00"), 1}; + } + + QUICConnectionId + first_connection_id() const override + { + return {reinterpret_cast("\x00"), 1}; + } + + QUICConnectionId + retry_source_connection_id() const override + { + return {reinterpret_cast("\x00"), 1}; + } + + QUICConnectionId + initial_source_connection_id() const override + { + return {reinterpret_cast("\x00"), 1}; + } + + const QUICFiveTuple + five_tuple() const override + { + return QUICFiveTuple(); + } + + std::string_view + cids() const override + { + using namespace std::literals; + return std::string_view("00000000-00000000"sv); + } + + uint32_t + pmtu() const override + { + return 1280; + } + + NetVConnectionContext_t + direction() const override + { + return NET_VCONNECTION_OUT; + } + + int + select_next_protocol(SSL *ssl, const unsigned char **out, unsigned char *outlen, const unsigned char *in, + unsigned inlen) const override + { + return SSL_TLSEXT_ERR_OK; + } + + bool + is_closed() const override + { + return false; + } + + bool + is_at_anti_amplification_limit() const override + { + return false; + } + + bool + is_address_validation_completed() const override + { + return true; + } + + bool + is_handshake_completed() const override + { + return true; + } + + bool + has_keys_for(QUICPacketNumberSpace space) const override + { + return true; + } + + QUICVersion + negotiated_version() const override + { + return QUIC_SUPPORTED_VERSIONS[0]; + } + + std::string_view + negotiated_application_name() const override + { + return negotiated_application_name_sv; + } +}; class MockQUICStreamManager : public QUICStreamManager { public: - MockQUICStreamManager() : QUICStreamManager() {} + MockQUICStreamManager(QUICContext *context) : QUICStreamManager(context, nullptr) {} + // Override virtual QUICConnectionErrorUPtr handle_frame(QUICEncryptionLevel level, const QUICFrame &f) override @@ -174,6 +386,18 @@ class MockQUICConnection : public QUICConnection return {reinterpret_cast("\x00"), 1}; } + QUICConnectionId + retry_source_connection_id() const override + { + return {reinterpret_cast("\x00"), 1}; + } + + QUICConnectionId + initial_source_connection_id() const override + { + return {reinterpret_cast("\x00"), 1}; + } + const QUICFiveTuple five_tuple() const override { @@ -215,7 +439,12 @@ class MockQUICConnection : public QUICConnection } void - close(QUICConnectionErrorUPtr error) override + close_quic_connection(QUICConnectionErrorUPtr error) override + { + } + + void + reset_quic_connection() override { } @@ -228,7 +457,7 @@ class MockQUICConnection : public QUICConnection QUICStreamManager * stream_manager() override { - return &_stream_manager; + return nullptr; } bool @@ -237,6 +466,30 @@ class MockQUICConnection : public QUICConnection return false; } + bool + is_at_anti_amplification_limit() const override + { + return false; + } + + bool + is_address_validation_completed() const override + { + return true; + } + + bool + is_handshake_completed() const override + { + return true; + } + + bool + has_keys_for(QUICPacketNumberSpace space) const override + { + return true; + } + void handle_received_packet(UDPPacket *) override { @@ -247,6 +500,12 @@ class MockQUICConnection : public QUICConnection { } + QUICVersion + negotiated_version() const override + { + return QUIC_SUPPORTED_VERSIONS[0]; + } + std::string_view negotiated_application_name() const override { @@ -265,97 +524,67 @@ class MockQUICConnection : public QUICConnection Ptr _mutex; int _totalFrameCount = 0; int _frameCount[256] = {0}; - MockQUICStreamManager _stream_manager; QUICTransportParametersInEncryptedExtensions dummy_transport_parameters(); NetVConnectionContext_t _direction; }; -class MockQUICConnectionInfoProvider : public QUICConnectionInfoProvider +class MockQUICCongestionController : public QUICCongestionController { - QUICConnectionId - connection_id() const override - { - return {reinterpret_cast("\x00"), 1}; - } - - QUICConnectionId - peer_connection_id() const override +public: + MockQUICCongestionController() {} + // Override + virtual void + on_packets_lost(const std::map &packets) override { - return {reinterpret_cast("\x00"), 1}; + for (auto &p : packets) { + lost_packets.insert(p.first); + } } - QUICConnectionId - original_connection_id() const override + virtual void + on_packet_sent(size_t bytes_sent) override { - return {reinterpret_cast("\x00"), 1}; } - - QUICConnectionId - first_connection_id() const override + virtual void + on_packets_acked(const std::vector &packets) override { - return {reinterpret_cast("\x00"), 1}; } - - const QUICFiveTuple - five_tuple() const override + void + on_packet_number_space_discarded(size_t bytes_in_flight) override { - return QUICFiveTuple(); } - - std::string_view - cids() const override + virtual void + process_ecn(const QUICAckFrame &ack_frame, QUICPacketNumberSpace pn_space, ink_hrtime largest_acked_time_sent) override { - using namespace std::literals; - return std::string_view("00000000-00000000"sv); } - - uint32_t - pmtu() const override + virtual void + add_extra_credit() override { - return 1280; } - - NetVConnectionContext_t - direction() const override + virtual void + reset() override { - return NET_VCONNECTION_OUT; } - - int - select_next_protocol(SSL *ssl, const unsigned char **out, unsigned char *outlen, const unsigned char *in, - unsigned inlen) const override + virtual uint32_t + credit() const override { - return SSL_TLSEXT_ERR_OK; - } - - bool - is_closed() const override - { - return false; + return 0; } - - std::string_view - negotiated_application_name() const override + virtual uint32_t + bytes_in_flight() const override { - return negotiated_application_name_sv; + return 0; } -}; - -class MockQUICCongestionController : public QUICCongestionController -{ -public: - MockQUICCongestionController(QUICConnectionInfoProvider *info, const QUICCCConfig &cc_config) - : QUICCongestionController(this->_rtt_measure, info, cc_config) + virtual uint32_t + congestion_window() const override { + return 0; } - // Override - virtual void - on_packets_lost(const std::map &packets) override + virtual uint32_t + current_ssthresh() const override { - for (auto &p : packets) { - lost_packets.insert(p.first); - } + return 0; } // for Test @@ -388,27 +617,118 @@ class MockQUICCongestionController : public QUICCongestionController private: int _totalFrameCount = 0; int _frameCount[256] = {0}; +}; +class MockQUICPacketProtectionKeyInfo : public QUICPacketProtectionKeyInfo +{ +public: + const EVP_CIPHER * + get_cipher(QUICKeyPhase phase) const override + { + return EVP_aes_128_gcm(); + } + + size_t + get_tag_len(QUICKeyPhase phase) const override + { + return EVP_GCM_TLS_TAG_LEN; + } + + const size_t *encryption_iv_len(QUICKeyPhase) const override + { + static size_t dummy = 12; + return &dummy; + } +}; + +class MockQUICContext : public QUICContext +{ +public: + MockQUICContext() : QUICContext() + { + _info = std::make_unique(); + _key_info = std::make_unique(); + _path_manager = std::make_unique(); + _ld_config = std::make_unique(); + _cc_config = std::make_unique(); + } + + virtual QUICConnectionInfoProvider * + connection_info() const override + { + return _info.get(); + } + virtual QUICConfig::scoped_config + config() const override + { + return _config; + } + virtual QUICRTTProvider * + rtt_provider() const override + { + return const_cast(&_rtt_measure); + } + + virtual QUICPacketProtectionKeyInfo * + key_info() const override + { + return _key_info.get(); + } + + virtual QUICLDConfig & + ld_config() const override + { + return *_ld_config; + } + + virtual QUICCCConfig & + cc_config() const override + { + return *_cc_config; + } + + virtual QUICPathManager * + path_manager() const override + { + return _path_manager.get(); + } + +private: + QUICConfig::scoped_config _config; QUICRTTMeasure _rtt_measure; + std::unique_ptr _info; + std::unique_ptr _key_info; + std::unique_ptr _ld_config; + std::unique_ptr _cc_config; + std::unique_ptr _path_manager; }; class MockQUICLossDetector : public QUICLossDetector { public: - MockQUICLossDetector(QUICConnectionInfoProvider *info, QUICCongestionController *cc, QUICRTTMeasure *rtt_measure, - const QUICLDConfig &ld_config) - : QUICLossDetector(info, cc, rtt_measure, ld_config) + MockQUICLossDetector(MockQUICContext &context) + : QUICLossDetector(context, &_cc, &_rtt_measure, &this->_pinger, &this->_padder), + _padder(NetVConnectionContext_t::NET_VCONNECTION_UNSET) { } + void rcv_frame(std::shared_ptr) {} + void - rcv_frame(std::shared_ptr) + on_packet_sent(QUICPacketUPtr packet) { } void - on_packet_sent(QUICPacketUPtr packet) + on_packet_number_space_discarded(QUICPacketNumberSpace pn_space) { + this->_cc.on_packet_number_space_discarded(0); } + +private: + QUICPinger _pinger; + QUICPadder _padder; + QUICRTTMeasure _rtt_measure; + MockQUICCongestionController _cc; }; class MockQUICApplication : public QUICApplication @@ -436,26 +756,56 @@ class MockQUICApplication : public QUICApplication } }; -class MockQUICPacketProtectionKeyInfo : public QUICPacketProtectionKeyInfo +class MockQUICPacketR : public QUICPacketR { public: - const EVP_CIPHER * - get_cipher(QUICKeyPhase phase) const override + MockQUICPacketR() : QUICPacketR(nullptr, {}, {}) {} + + QUICPacketType + type() const override { - return EVP_aes_128_gcm(); + return QUICPacketType::PROTECTED; } - size_t - get_tag_len(QUICKeyPhase phase) const override + QUICConnectionId + destination_cid() const override { - return EVP_GCM_TLS_TAG_LEN; + return QUICConnectionId::ZERO(); } - const size_t *encryption_iv_len(QUICKeyPhase) const override + QUICPacketNumber + packet_number() const override { - static size_t dummy = 12; - return &dummy; + return 0; + } + + const IpEndpoint & + from() const override + { + return this->_from; } + + const IpEndpoint & + to() const override + { + return this->_to; + } + + void + set_to(const IpEndpoint ep) + { + this->_to = ep; + } + + void + set_from(const IpEndpoint ep) + { + this->_from = ep; + } + +private: + IpEndpoint _to; + IpEndpoint _from; }; class MockQUICHandshakeProtocol : public QUICHandshakeProtocol @@ -464,7 +814,7 @@ class MockQUICHandshakeProtocol : public QUICHandshakeProtocol MockQUICHandshakeProtocol(QUICPacketProtectionKeyInfo &pp_key_info) : QUICHandshakeProtocol(pp_key_info) {} int - handshake(QUICHandshakeMsgs *out, const QUICHandshakeMsgs *in) override + handshake(QUICHandshakeMsgs **out, const QUICHandshakeMsgs *in) override { return true; } @@ -487,7 +837,7 @@ class MockQUICHandshakeProtocol : public QUICHandshakeProtocol } int - initialize_key_materials(QUICConnectionId cid) override + initialize_key_materials(QUICConnectionId cid, QUICVersion version) override { return 0; } @@ -646,14 +996,14 @@ class MockQUICFrameGenerator : public QUICFrameGenerator { public: bool - will_generate_frame(QUICEncryptionLevel level, ink_hrtime timestamp) override + will_generate_frame(QUICEncryptionLevel level, size_t connection_credit, bool ack_eliciting, uint32_t seq_num) override { return true; } QUICFrame * generate_frame(uint8_t *buf, QUICEncryptionLevel level, uint64_t connection_credit, uint16_t maximum_frame_size, - ink_hrtime timestamp) override + size_t current_packet_size, uint32_t seq_num) override { QUICFrame *frame = QUICFrameFactory::create_ping_frame(buf, 0, this); QUICFrameInformationUPtr info = QUICFrameInformationUPtr(quicFrameInformationAllocator.alloc()); @@ -670,63 +1020,3 @@ class MockQUICFrameGenerator : public QUICFrameGenerator lost_frame_count++; } }; - -class MockQUICLDConfig : public QUICLDConfig -{ - uint32_t - packet_threshold() const - { - return 3; - } - - float - time_threshold() const - { - return 1.25; - } - - ink_hrtime - granularity() const - { - return HRTIME_MSECONDS(1); - } - - ink_hrtime - initial_rtt() const - { - return HRTIME_MSECONDS(100); - } -}; - -class MockQUICCCConfig : public QUICCCConfig -{ - uint32_t - max_datagram_size() const - { - return 1200; - } - - uint32_t - initial_window() const - { - return 10; - } - - uint32_t - minimum_window() const - { - return 2; - } - - float - loss_reduction_factor() const - { - return 0.5; - } - - uint32_t - persistent_congestion_threshold() const - { - return 2; - } -}; diff --git a/iocore/net/quic/QUICAckFrameCreator.cc b/iocore/net/quic/QUICAckFrameCreator.cc index 27923b0f499..bee2b76bc9d 100644 --- a/iocore/net/quic/QUICAckFrameCreator.cc +++ b/iocore/net/quic/QUICAckFrameCreator.cc @@ -28,7 +28,7 @@ QUICAckFrameManager::QUICAckFrameManager() { - for (auto i = 0; i < kPacketNumberSpace; i++) { + for (auto i = 0; i < QUIC_N_PACKET_SPACES; i++) { this->_ack_creator[i] = std::make_unique(static_cast(i), this); } } @@ -61,7 +61,7 @@ QUICAckFrameManager::update(QUICEncryptionLevel level, QUICPacketNumber packet_n */ QUICFrame * QUICAckFrameManager::generate_frame(uint8_t *buf, QUICEncryptionLevel level, uint64_t /* connection_credit */, - uint16_t maximum_frame_size, ink_hrtime timestamp) + uint16_t maximum_frame_size, size_t current_packet_size, uint32_t seq_num) { QUICAckFrame *ack_frame = nullptr; @@ -87,7 +87,8 @@ QUICAckFrameManager::generate_frame(uint8_t *buf, QUICEncryptionLevel level, uin } bool -QUICAckFrameManager::will_generate_frame(QUICEncryptionLevel level, ink_hrtime timestamp) +QUICAckFrameManager::will_generate_frame(QUICEncryptionLevel level, size_t current_packet_size, bool ack_eliciting, + uint32_t seq_num) { // No ACK frame on ZERO_RTT level if (!this->_is_level_matched(level) || level == QUICEncryptionLevel::ZERO_RTT) { @@ -131,7 +132,7 @@ QUICAckFrameManager::ack_delay_exponent() const void QUICAckFrameManager::set_max_ack_delay(uint16_t delay) { - for (auto i = 0; i < kPacketNumberSpace; i++) { + for (auto i = 0; i < QUIC_N_PACKET_SPACES; i++) { this->_ack_creator[i]->set_max_ack_delay(delay); } } @@ -181,21 +182,18 @@ QUICAckFrameManager::QUICAckFrameCreator::push_back(QUICPacketNumber packet_numb this->_latest_packet_received_time = Thread::get_hrtime(); } - // unorder packet should send ack immediately to accellerate the recovery + // unorder packet should send ack immediately to accelerate the recovery if (this->_expect_next != packet_number) { this->_should_send = true; } - // every 2 full-packet should send a ack frame like tcp - this->_size_unsend += size; - // FIXME: this size should be fixed with PMTU - if (this->_size_unsend > 2 * 1480) { - this->_size_unsend = 0; + // every 2 ack-eliciting packet should send a ack frame + if (!ack_only && ++this->_ack_eliciting_count % 2 == 0) { this->_should_send = true; } // can not delay handshake packet - if ((this->_pn_space == QUICPacketNumberSpace::Initial || this->_pn_space == QUICPacketNumberSpace::Handshake) && !ack_only) { + if ((this->_pn_space == QUICPacketNumberSpace::INITIAL || this->_pn_space == QUICPacketNumberSpace::HANDSHAKE) && !ack_only) { this->_should_send = true; } @@ -211,7 +209,7 @@ QUICAckFrameManager::QUICAckFrameCreator::push_back(QUICPacketNumber packet_numb } size_t -QUICAckFrameManager::QUICAckFrameCreator::size() +QUICAckFrameManager::QUICAckFrameCreator::size() const { return this->_packet_numbers.size(); } @@ -223,19 +221,19 @@ QUICAckFrameManager::QUICAckFrameCreator::clear() this->_largest_ack_number = 0; this->_largest_ack_received_time = 0; this->_latest_packet_received_time = 0; - this->_size_unsend = 0; + this->_ack_eliciting_count = 0; this->_should_send = false; this->_available = false; } QUICPacketNumber -QUICAckFrameManager::QUICAckFrameCreator::largest_ack_number() +QUICAckFrameManager::QUICAckFrameCreator::largest_ack_number() const { return this->_largest_ack_number; } ink_hrtime -QUICAckFrameManager::QUICAckFrameCreator::largest_ack_received_time() +QUICAckFrameManager::QUICAckFrameCreator::largest_ack_received_time() const { return this->_largest_ack_received_time; } @@ -337,7 +335,7 @@ QUICAckFrameManager::QUICAckFrameCreator::_calculate_delay() ink_hrtime now = Thread::get_hrtime(); uint64_t delay = (now - this->_largest_ack_received_time) / 1000; uint8_t ack_delay_exponent = 3; - if (this->_pn_space != QUICPacketNumberSpace::Initial && this->_pn_space != QUICPacketNumberSpace::Handshake) { + if (this->_pn_space != QUICPacketNumberSpace::INITIAL && this->_pn_space != QUICPacketNumberSpace::HANDSHAKE) { ack_delay_exponent = this->_ack_manager->ack_delay_exponent(); } return delay >> ack_delay_exponent; diff --git a/iocore/net/quic/QUICAckFrameCreator.h b/iocore/net/quic/QUICAckFrameCreator.h index 08b50d470bc..55e20f254a9 100644 --- a/iocore/net/quic/QUICAckFrameCreator.h +++ b/iocore/net/quic/QUICAckFrameCreator.h @@ -44,7 +44,7 @@ class QUICAckFrameManager : public QUICFrameGenerator ~QUICAckFrameCreator(); void push_back(QUICPacketNumber packet_number, size_t size, bool ack_only); - size_t size(); + size_t size() const; void clear(); void sort(); void forget(QUICPacketNumber largest_acknowledged); @@ -58,8 +58,8 @@ class QUICAckFrameManager : public QUICFrameGenerator // refresh state when frame lost void refresh_state(); - QUICPacketNumber largest_ack_number(); - ink_hrtime largest_ack_received_time(); + QUICPacketNumber largest_ack_number() const; + ink_hrtime largest_ack_received_time() const; private: uint64_t _calculate_delay(); @@ -69,7 +69,7 @@ class QUICAckFrameManager : public QUICFrameGenerator bool _available = false; // packet_number has data to sent bool _should_send = false; // ack frame should be sent immediately bool _has_new_data = false; // new data after last sent - size_t _size_unsend = 0; + uint32_t _ack_eliciting_count = 0; // every two ack-eliciting packet should send ack immediately uint16_t _max_ack_delay = 25; QUICPacketNumber _largest_ack_number = 0; QUICPacketNumber _expect_next = 0; @@ -78,7 +78,7 @@ class QUICAckFrameManager : public QUICFrameGenerator QUICAckFrameManager *_ack_manager = nullptr; - QUICPacketNumberSpace _pn_space = QUICPacketNumberSpace::Initial; + QUICPacketNumberSpace _pn_space = QUICPacketNumberSpace::INITIAL; }; static constexpr int MAXIMUM_PACKET_COUNT = 256; @@ -100,13 +100,13 @@ class QUICAckFrameManager : public QUICFrameGenerator /* * Returns true only if should send ack. */ - bool will_generate_frame(QUICEncryptionLevel level, ink_hrtime timestamp) override; + bool will_generate_frame(QUICEncryptionLevel level, size_t current_packet_size, bool ack_eliciting, uint32_t seq_num) override; /* * Calls create directly. */ QUICFrame *generate_frame(uint8_t *buf, QUICEncryptionLevel level, uint64_t connection_credit, uint16_t maximum_frame_size, - ink_hrtime timestamp) override; + size_t current_packet_size, uint32_t seq_num) override; QUICFrameId issue_frame_id(); uint8_t ack_delay_exponent() const; diff --git a/iocore/net/quic/QUICAddrVerifyState.cc b/iocore/net/quic/QUICAddrVerifyState.cc index 67281b592d6..6db501961c7 100644 --- a/iocore/net/quic/QUICAddrVerifyState.cc +++ b/iocore/net/quic/QUICAddrVerifyState.cc @@ -48,7 +48,7 @@ QUICAddrVerifyState::consume(uint32_t windows) } uint32_t -QUICAddrVerifyState::windows() +QUICAddrVerifyState::windows() const { return this->_windows; } diff --git a/iocore/net/quic/QUICAddrVerifyState.h b/iocore/net/quic/QUICAddrVerifyState.h index d6d6c7c4e42..6b454e4e22c 100644 --- a/iocore/net/quic/QUICAddrVerifyState.h +++ b/iocore/net/quic/QUICAddrVerifyState.h @@ -34,7 +34,7 @@ class QUICAddrVerifyState void fill(uint32_t windows); void consume(uint32_t windows); void set_addr_verifed(); - uint32_t windows(); + uint32_t windows() const; bool is_verified() const; private: diff --git a/iocore/net/quic/QUICAltConnectionManager.cc b/iocore/net/quic/QUICAltConnectionManager.cc index faca19e8743..770f5f9f73a 100644 --- a/iocore/net/quic/QUICAltConnectionManager.cc +++ b/iocore/net/quic/QUICAltConnectionManager.cc @@ -21,52 +21,62 @@ limitations under the License. */ +#include "algorithm" +#include "tscore/ink_assert.h" +#include "tscore/ink_defs.h" #include "QUICAltConnectionManager.h" #include "QUICConnectionTable.h" +#include "QUICResetTokenTable.h" static constexpr char V_DEBUG_TAG[] = "v_quic_alt_con"; #define QUICACMVDebug(fmt, ...) Debug(V_DEBUG_TAG, "[%s] " fmt, this->_qc->cids().data(), ##__VA_ARGS__) -QUICAltConnectionManager::QUICAltConnectionManager(QUICConnection *qc, QUICConnectionTable &ctable, +QUICAltConnectionManager::QUICAltConnectionManager(QUICConnection *qc, QUICConnectionTable &ctable, QUICResetTokenTable &rtable, const QUICConnectionId &peer_initial_cid, uint32_t instance_id, - uint8_t num_alt_con, const QUICPreferredAddress &preferred_address) - : _qc(qc), _ctable(ctable), _instance_id(instance_id), _nids(num_alt_con) + uint8_t local_active_cid_limit, const IpEndpoint *preferred_endpoint_ipv4, + const IpEndpoint *preferred_endpoint_ipv6) + : _qc(qc), _ctable(ctable), _rtable(rtable), _instance_id(instance_id), _local_active_cid_limit(local_active_cid_limit) { // Sequence number of the initial CID is 0 this->_alt_quic_connection_ids_remote.push_back({0, peer_initial_cid, {}, {true}}); + this->_alt_quic_connection_ids_local[0].seq_num = 0; + this->_alt_quic_connection_ids_local[0].advertised = true; - // Sequence number of the preferred address is 1 if available - if (preferred_address.is_available()) { - this->_alt_quic_connection_ids_remote.push_back({1, preferred_address.cid(), preferred_address.token(), {false}}); - } - - this->_alt_quic_connection_ids_local = static_cast(ats_malloc(sizeof(AltConnectionInfo) * this->_nids)); -} + if ((preferred_endpoint_ipv4 && !ats_ip_addr_port_eq(*preferred_endpoint_ipv4, qc->five_tuple().source())) || + (preferred_endpoint_ipv6 && !ats_ip_addr_port_eq(*preferred_endpoint_ipv6, qc->five_tuple().source()))) { + this->_alt_quic_connection_ids_local[1] = this->_generate_next_alt_con_info(); + // This alt cid will be advertised via Transport Parameter, so no need to advertise it via NCID frame + this->_alt_quic_connection_ids_local[1].advertised = true; -QUICAltConnectionManager::QUICAltConnectionManager(QUICConnection *qc, QUICConnectionTable &ctable, - const QUICConnectionId &peer_initial_cid, uint32_t instance_id, - uint8_t num_alt_con, const IpEndpoint *preferred_endpoint_ipv4, - const IpEndpoint *preferred_endpoint_ipv6) - : _qc(qc), _ctable(ctable), _instance_id(instance_id), _nids(num_alt_con) -{ - // Sequence number of the initial CID is 0 - this->_alt_quic_connection_ids_remote.push_back({0, peer_initial_cid, {}, {true}}); + IpEndpoint empty_endpoint_ipv4; + IpEndpoint empty_endpoint_ipv6; + empty_endpoint_ipv4.sa.sa_family = AF_UNSPEC; + empty_endpoint_ipv6.sa.sa_family = AF_UNSPEC; + if (preferred_endpoint_ipv4 == nullptr) { + preferred_endpoint_ipv4 = &empty_endpoint_ipv4; + } + if (preferred_endpoint_ipv6 == nullptr) { + preferred_endpoint_ipv6 = &empty_endpoint_ipv6; + } - this->_alt_quic_connection_ids_local = static_cast(ats_malloc(sizeof(AltConnectionInfo) * this->_nids)); - this->_init_alt_connection_ids(preferred_endpoint_ipv4, preferred_endpoint_ipv6); + // FIXME Check nullptr dereference + this->_local_preferred_address = + new QUICPreferredAddress(*preferred_endpoint_ipv4, *preferred_endpoint_ipv6, this->_alt_quic_connection_ids_local[1].id, + this->_alt_quic_connection_ids_local[1].token); + } } QUICAltConnectionManager::~QUICAltConnectionManager() { ats_free(this->_alt_quic_connection_ids_local); - delete this->_preferred_address; + delete this->_local_preferred_address; } const QUICPreferredAddress * QUICAltConnectionManager::preferred_address() const { - return this->_preferred_address; + return this->_local_preferred_address; } std::vector @@ -109,63 +119,30 @@ QUICAltConnectionManager::_generate_next_alt_con_info() } if (is_debug_tag_set(V_DEBUG_TAG)) { - char new_cid_str[QUICConnectionId::MAX_HEX_STR_LENGTH]; - conn_id.hex(new_cid_str, QUICConnectionId::MAX_HEX_STR_LENGTH); - QUICACMVDebug("alt-cid=%s", new_cid_str); + QUICACMVDebug("alt-cid=%s", conn_id.hex().c_str()); } return aci; } void -QUICAltConnectionManager::_init_alt_connection_ids(const IpEndpoint *preferred_endpoint_ipv4, - const IpEndpoint *preferred_endpoint_ipv6) +QUICAltConnectionManager::_init_alt_connection_ids() { - if (preferred_endpoint_ipv4 || preferred_endpoint_ipv6) { - this->_alt_quic_connection_ids_local[0] = this->_generate_next_alt_con_info(); - // This alt cid will be advertised via Transport Parameter - this->_alt_quic_connection_ids_local[0].advertised = true; - - IpEndpoint empty_endpoint_ipv4; - IpEndpoint empty_endpoint_ipv6; - empty_endpoint_ipv4.sa.sa_family = AF_UNSPEC; - empty_endpoint_ipv6.sa.sa_family = AF_UNSPEC; - if (preferred_endpoint_ipv4 == nullptr) { - preferred_endpoint_ipv4 = &empty_endpoint_ipv4; - } - if (preferred_endpoint_ipv6 == nullptr) { - preferred_endpoint_ipv6 = &empty_endpoint_ipv6; - } - - // FIXME Check nullptr dereference - this->_preferred_address = - new QUICPreferredAddress(*preferred_endpoint_ipv4, *preferred_endpoint_ipv6, this->_alt_quic_connection_ids_local[0].id, - this->_alt_quic_connection_ids_local[0].token); - } - - for (int i = (preferred_endpoint_ipv4 || preferred_endpoint_ipv6 ? 1 : 0); i < this->_nids; ++i) { + for (int i = this->_alt_quic_connection_id_seq_num + 1; i < this->_remote_active_cid_limit; ++i) { this->_alt_quic_connection_ids_local[i] = this->_generate_next_alt_con_info(); } this->_need_advertise = true; } -bool +void QUICAltConnectionManager::_update_alt_connection_id(uint64_t chosen_seq_num) { - for (int i = 0; i < this->_nids; ++i) { - if (_alt_quic_connection_ids_local[i].seq_num == chosen_seq_num) { - _alt_quic_connection_ids_local[i] = this->_generate_next_alt_con_info(); - this->_need_advertise = true; - return true; + for (int i = 0; i < this->_remote_active_cid_limit; ++i) { + if (this->_alt_quic_connection_ids_local[i].seq_num == chosen_seq_num) { + this->_alt_quic_connection_ids_local[i] = this->_generate_next_alt_con_info(); + this->_need_advertise = true; } } - - // Seq 0 is special so it's not in the array - if (chosen_seq_num == 0) { - return true; - } - - return false; } QUICConnectionErrorUPtr @@ -177,8 +154,22 @@ QUICAltConnectionManager::_register_remote_connection_id(const QUICNewConnection error = std::make_unique(QUICTransErrorCode::PROTOCOL_VIOLATION, "received zero-length cid", QUICFrameType::NEW_CONNECTION_ID); } else { - this->_alt_quic_connection_ids_remote.push_back( - {frame.sequence(), frame.connection_id(), frame.stateless_reset_token(), {false}}); + int unused = 0; + for (auto &&x : this->_alt_quic_connection_ids_remote) { + if (x.seq_num == frame.sequence()) { + return error; + } + if (x.used == false && x.seq_num != 1) { + ++unused; + } + } + if (unused > this->_local_active_cid_limit) { + error = std::make_unique(QUICTransErrorCode::CONNECTION_ID_LIMIT_ERROR, "received too many alt CIDs", + QUICFrameType::NEW_CONNECTION_ID); + } else { + this->_alt_quic_connection_ids_remote.push_back( + {frame.sequence(), frame.connection_id(), frame.stateless_reset_token(), {false}}); + } } return error; @@ -189,9 +180,11 @@ QUICAltConnectionManager::_retire_remote_connection_id(const QUICRetireConnectio { QUICConnectionErrorUPtr error = nullptr; - if (!this->_update_alt_connection_id(frame.seq_num())) { + if (frame.seq_num() > this->_alt_quic_connection_id_seq_num) { error = std::make_unique(QUICTransErrorCode::PROTOCOL_VIOLATION, "received unused sequence number", QUICFrameType::RETIRE_CONNECTION_ID); + } else { + this->_update_alt_connection_id(frame.seq_num()); } return error; } @@ -214,15 +207,12 @@ QUICAltConnectionManager::is_ready_to_migrate() const QUICConnectionId QUICAltConnectionManager::migrate_to_alt_cid() { - if (this->_qc->direction() == NET_VCONNECTION_OUT) { - this->_init_alt_connection_ids(); - } - for (auto &info : this->_alt_quic_connection_ids_remote) { if (info.used) { continue; } info.used = true; + this->_rtable.insert(info.token, this->_qc); return info.id; } @@ -233,7 +223,14 @@ QUICAltConnectionManager::migrate_to_alt_cid() bool QUICAltConnectionManager::migrate_to(const QUICConnectionId &cid, QUICStatelessResetToken &new_reset_token) { - for (unsigned int i = 0; i < this->_nids; ++i) { + if (this->_local_preferred_address) { + if (cid == this->_local_preferred_address->cid()) { + new_reset_token = this->_local_preferred_address->token(); + return true; + } + } + + for (int i = 0; i < this->_remote_active_cid_limit; ++i) { AltConnectionInfo &info = this->_alt_quic_connection_ids_local[i]; if (info.id == cid) { // Migrate connection @@ -249,8 +246,9 @@ QUICAltConnectionManager::drop_cid(const QUICConnectionId &cid) { for (auto it = this->_alt_quic_connection_ids_remote.begin(); it != this->_alt_quic_connection_ids_remote.end(); ++it) { if (it->id == cid) { - QUICACMVDebug("Dropping advertized CID %" PRIx32 " seq# %" PRIu64, it->id.h32(), it->seq_num); + QUICACMVDebug("Dropping advertised CID %" PRIx32 " seq# %" PRIu64, it->id.h32(), it->seq_num); this->_retired_seq_nums.push(it->seq_num); + this->_rtable.erase(it->token); this->_alt_quic_connection_ids_remote.erase(it); return; } @@ -260,13 +258,33 @@ QUICAltConnectionManager::drop_cid(const QUICConnectionId &cid) void QUICAltConnectionManager::invalidate_alt_connections() { - for (unsigned int i = 0; i < this->_nids; ++i) { + int n = this->_remote_active_cid_limit + ((this->_local_preferred_address == nullptr) ? 1 : 0); + + for (int i = 0; i < n; ++i) { this->_ctable.erase(this->_alt_quic_connection_ids_local[i].id, this->_qc); } } +void +QUICAltConnectionManager::set_remote_preferred_address(const QUICPreferredAddress &preferred_address) +{ + ink_assert(preferred_address.is_available()); + + // Sequence number of the preferred address is 1 if available + this->_alt_quic_connection_ids_remote.push_back({1, preferred_address.cid(), preferred_address.token(), {false}}); +} + +void +QUICAltConnectionManager::set_remote_active_cid_limit(uint8_t active_cid_limit) +{ + this->_remote_active_cid_limit = + std::min(static_cast(active_cid_limit), countof(this->_alt_quic_connection_ids_local)); + this->_init_alt_connection_ids(); +} + bool -QUICAltConnectionManager::will_generate_frame(QUICEncryptionLevel level, ink_hrtime timestamp) +QUICAltConnectionManager::will_generate_frame(QUICEncryptionLevel level, size_t current_packet_size, bool ack_eliciting, + uint32_t seq_num) { if (!this->_is_level_matched(level)) { return false; @@ -280,7 +298,7 @@ QUICAltConnectionManager::will_generate_frame(QUICEncryptionLevel level, ink_hrt */ QUICFrame * QUICAltConnectionManager::generate_frame(uint8_t *buf, QUICEncryptionLevel level, uint64_t /* connection_credit */, - uint16_t maximum_frame_size, ink_hrtime timestamp) + uint16_t maximum_frame_size, size_t current_packet_size, uint32_t seq_num) { QUICFrame *frame = nullptr; if (!this->_is_level_matched(level)) { @@ -288,10 +306,10 @@ QUICAltConnectionManager::generate_frame(uint8_t *buf, QUICEncryptionLevel level } if (this->_need_advertise) { - int count = this->_nids; - for (int i = 0; i < count; ++i) { + for (int i = 0; i < this->_remote_active_cid_limit; ++i) { if (!this->_alt_quic_connection_ids_local[i].advertised) { - frame = QUICFrameFactory::create_new_connection_id_frame(buf, this->_alt_quic_connection_ids_local[i].seq_num, + // FIXME Should send a sequence number for retire_prior_to. Sending 0 for now. + frame = QUICFrameFactory::create_new_connection_id_frame(buf, this->_alt_quic_connection_ids_local[i].seq_num, 0, this->_alt_quic_connection_ids_local[i].id, this->_alt_quic_connection_ids_local[i].token); @@ -310,12 +328,11 @@ QUICAltConnectionManager::generate_frame(uint8_t *buf, QUICEncryptionLevel level } if (!this->_retired_seq_nums.empty()) { - if (auto s = this->_retired_seq_nums.front()) { - frame = QUICFrameFactory::create_retire_connection_id_frame(buf, s); - this->_records_retire_connection_id_frame(level, static_cast(*frame)); - this->_retired_seq_nums.pop(); - return frame; - } + auto s = this->_retired_seq_nums.front(); + frame = QUICFrameFactory::create_retire_connection_id_frame(buf, s); + this->_records_retire_connection_id_frame(level, static_cast(*frame)); + this->_retired_seq_nums.pop(); + return frame; } return frame; @@ -351,7 +368,7 @@ QUICAltConnectionManager::_on_frame_lost(QUICFrameInformationUPtr &info) switch (info->type) { case QUICFrameType::NEW_CONNECTION_ID: { AltConnectionInfo *frame_info = reinterpret_cast(info->data); - for (int i = 0; i < this->_nids; ++i) { + for (int i = 0; i < this->_remote_active_cid_limit; ++i) { if (this->_alt_quic_connection_ids_local[i].seq_num == frame_info->seq_num) { ink_assert(this->_alt_quic_connection_ids_local[i].advertised); this->_alt_quic_connection_ids_local[i].advertised = false; diff --git a/iocore/net/quic/QUICAltConnectionManager.h b/iocore/net/quic/QUICAltConnectionManager.h index 501ab99a28c..ade5f8942d9 100644 --- a/iocore/net/quic/QUICAltConnectionManager.h +++ b/iocore/net/quic/QUICAltConnectionManager.h @@ -31,20 +31,14 @@ #include "QUICConnection.h" class QUICConnectionTable; +class QUICResetTokenTable; class QUICAltConnectionManager : public QUICFrameHandler, public QUICFrameGenerator { public: - /** - * Constructor for clients - */ - QUICAltConnectionManager(QUICConnection *qc, QUICConnectionTable &ctable, const QUICConnectionId &peer_initial_cid, - uint32_t instance_id, uint8_t num_alt_con, const QUICPreferredAddress &preferred_address); - /** - * Constructor for servers - */ - QUICAltConnectionManager(QUICConnection *qc, QUICConnectionTable &ctable, const QUICConnectionId &peer_initial_cid, - uint32_t instance_id, uint8_t num_alt_con, const IpEndpoint *preferred_endpoint_ipv4 = nullptr, + QUICAltConnectionManager(QUICConnection *qc, QUICConnectionTable &ctable, QUICResetTokenTable &rtable, + const QUICConnectionId &peer_initial_cid, uint32_t instance_id, uint8_t active_cid_limit, + const IpEndpoint *preferred_endpoint_ipv4 = nullptr, const IpEndpoint *preferred_endpoint_ipv6 = nullptr); ~QUICAltConnectionManager(); @@ -62,12 +56,15 @@ class QUICAltConnectionManager : public QUICFrameHandler, public QUICFrameGenera /** * Migrate to new CID * - * cid need to match with one of alt CID that AltConnnectionManager prepared. + * cid need to match with one of alt CID that AltConnectionManager prepared. */ bool migrate_to(const QUICConnectionId &cid, QUICStatelessResetToken &new_reset_token); void drop_cid(const QUICConnectionId &cid); + void set_remote_preferred_address(const QUICPreferredAddress &preferred_address); + void set_remote_active_cid_limit(uint8_t active_cid_limit); + /** * Invalidate all CIDs prepared */ @@ -83,9 +80,9 @@ class QUICAltConnectionManager : public QUICFrameHandler, public QUICFrameGenera virtual QUICConnectionErrorUPtr handle_frame(QUICEncryptionLevel level, const QUICFrame &frame) override; // QUICFrameGenerator - bool will_generate_frame(QUICEncryptionLevel level, ink_hrtime timestamp) override; + bool will_generate_frame(QUICEncryptionLevel level, size_t current_packet_size, bool ack_eliciting, uint32_t seq_num) override; QUICFrame *generate_frame(uint8_t *buf, QUICEncryptionLevel level, uint64_t connection_credit, uint16_t maximum_frame_size, - ink_hrtime timestamp) override; + size_t current_packet_size, uint32_t seq_num) override; private: struct AltConnectionInfo { @@ -100,19 +97,20 @@ class QUICAltConnectionManager : public QUICFrameHandler, public QUICFrameGenera QUICConnection *_qc = nullptr; QUICConnectionTable &_ctable; - AltConnectionInfo *_alt_quic_connection_ids_local; + QUICResetTokenTable &_rtable; + AltConnectionInfo _alt_quic_connection_ids_local[8]; // 8 is perhaps enough std::vector _alt_quic_connection_ids_remote; std::queue _retired_seq_nums; - uint32_t _instance_id = 0; - uint8_t _nids = 1; - uint64_t _alt_quic_connection_id_seq_num = 0; - bool _need_advertise = false; - QUICPreferredAddress *_preferred_address = nullptr; + uint32_t _instance_id = 0; + uint8_t _local_active_cid_limit = 0; + uint8_t _remote_active_cid_limit = 0; + uint64_t _alt_quic_connection_id_seq_num = 0; + bool _need_advertise = false; + QUICPreferredAddress *_local_preferred_address = nullptr; AltConnectionInfo _generate_next_alt_con_info(); - void _init_alt_connection_ids(const IpEndpoint *preferred_endpoint_ipv4 = nullptr, - const IpEndpoint *preferred_endpoint_ipv6 = nullptr); - bool _update_alt_connection_id(uint64_t chosen_seq_num); + void _init_alt_connection_ids(); + void _update_alt_connection_id(uint64_t chosen_seq_num); void _records_new_connection_id_frame(QUICEncryptionLevel level, const QUICNewConnectionIdFrame &frame); void _records_retire_connection_id_frame(QUICEncryptionLevel, const QUICRetireConnectionIdFrame &frame); diff --git a/iocore/net/quic/QUICApplication.cc b/iocore/net/quic/QUICApplication.cc index 54ad68e1137..a5bfd26541f 100644 --- a/iocore/net/quic/QUICApplication.cc +++ b/iocore/net/quic/QUICApplication.cc @@ -115,7 +115,7 @@ QUICStreamIO::consume(int64_t len) } bool -QUICStreamIO::is_read_done() +QUICStreamIO::is_read_done() const { return this->_read_vio->ntodo() == 0; } diff --git a/iocore/net/quic/QUICApplication.h b/iocore/net/quic/QUICApplication.h index dd6d52489bd..c4fb16fd50d 100644 --- a/iocore/net/quic/QUICApplication.h +++ b/iocore/net/quic/QUICApplication.h @@ -46,7 +46,7 @@ class QUICStreamIO int64_t read(uint8_t *buf, int64_t len); int64_t peek(uint8_t *buf, int64_t len); void consume(int64_t len); - bool is_read_done(); + bool is_read_done() const; virtual void read_reenable(); int64_t write(const uint8_t *buf, int64_t len); diff --git a/iocore/net/quic/QUICBidirectionalStream.cc b/iocore/net/quic/QUICBidirectionalStream.cc index bbd98d9652a..4203ec0eeb6 100644 --- a/iocore/net/quic/QUICBidirectionalStream.cc +++ b/iocore/net/quic/QUICBidirectionalStream.cc @@ -161,7 +161,7 @@ QUICBidirectionalStream::is_cancelled() const /** * @brief Receive STREAM frame * @detail When receive STREAM frame, reorder frames and write to buffer of read_vio. - * If the reordering or writting operation is heavy, split out them to read function, + * If the reordering or writing operation is heavy, split out them to read function, * which is called by application via do_io_read() or reenable(). */ QUICConnectionErrorUPtr @@ -352,18 +352,25 @@ QUICBidirectionalStream::reenable(VIO *vio) } bool -QUICBidirectionalStream::will_generate_frame(QUICEncryptionLevel level, ink_hrtime timestamp) +QUICBidirectionalStream::will_generate_frame(QUICEncryptionLevel level, size_t current_packet_size, bool ack_eliciting, + uint32_t seq_num) { - return this->_local_flow_controller.will_generate_frame(level, timestamp) || !this->is_retransmited_frame_queue_empty() || - this->_write_vio.get_reader()->is_read_avail_more_than(0); + if (this->_local_flow_controller.will_generate_frame(level, current_packet_size, ack_eliciting, seq_num)) { + return true; + } + if (!this->is_retransmited_frame_queue_empty()) { + return true; + } + if (this->_write_vio.op != VIO::NONE && this->_write_vio.get_reader()->is_read_avail_more_than(0)) { + return true; + }; + return false; } QUICFrame * QUICBidirectionalStream::generate_frame(uint8_t *buf, QUICEncryptionLevel level, uint64_t connection_credit, - uint16_t maximum_frame_size, ink_hrtime timestamp) + uint16_t maximum_frame_size, size_t current_packet_size, uint32_t seq_num) { - SCOPED_MUTEX_LOCK(lock, this->_write_vio.mutex, this_ethread()); - QUICFrame *frame = this->create_retransmitted_frame(buf, level, maximum_frame_size, this->_issue_frame_id(), this); if (frame != nullptr) { ink_assert(frame->type() == QUICFrameType::STREAM); @@ -374,6 +381,10 @@ QUICBidirectionalStream::generate_frame(uint8_t *buf, QUICEncryptionLevel level, // RESET_STREAM if (this->_reset_reason && !this->_is_reset_sent) { frame = QUICFrameFactory::create_rst_stream_frame(buf, *this->_reset_reason, this->_issue_frame_id(), this); + if (frame->size() > maximum_frame_size) { + frame->~QUICFrame(); + return nullptr; + } this->_records_rst_stream_frame(level, *static_cast(frame)); this->_state.update_with_sending_frame(*frame); this->_is_reset_sent = true; @@ -384,6 +395,10 @@ QUICBidirectionalStream::generate_frame(uint8_t *buf, QUICEncryptionLevel level, if (this->_stop_sending_reason && !this->_is_stop_sending_sent) { frame = QUICFrameFactory::create_stop_sending_frame(buf, this->id(), this->_stop_sending_reason->code, this->_issue_frame_id(), this); + if (frame->size() > maximum_frame_size) { + frame->~QUICFrame(); + return nullptr; + } this->_records_stop_sending_frame(level, *static_cast(frame)); this->_state.update_with_sending_frame(*frame); this->_is_stop_sending_sent = true; @@ -391,96 +406,98 @@ QUICBidirectionalStream::generate_frame(uint8_t *buf, QUICEncryptionLevel level, } // MAX_STREAM_DATA - frame = this->_local_flow_controller.generate_frame(buf, level, UINT16_MAX, maximum_frame_size, timestamp); + frame = this->_local_flow_controller.generate_frame(buf, level, UINT16_MAX, maximum_frame_size, current_packet_size, seq_num); if (frame) { + // maximum_frame_size should be checked in QUICFlowController return frame; } - if (!this->_state.is_allowed_to_send(QUICFrameType::STREAM)) { - return frame; - } - - uint64_t maximum_data_size = 0; - if (maximum_frame_size <= MAX_STREAM_FRAME_OVERHEAD) { - return frame; - } - maximum_data_size = maximum_frame_size - MAX_STREAM_FRAME_OVERHEAD; - - bool pure_fin = false; - bool fin = false; - if ((this->_write_vio.nbytes != 0 || this->_write_vio.nbytes != INT64_MAX) && - this->_write_vio.nbytes == static_cast(this->_send_offset)) { - // Pure FIN stream should be sent regardless status of remote flow controller, because the length is zero. - pure_fin = true; - fin = true; - } + if (this->_write_vio.op != VIO::NONE && this->_state.is_allowed_to_send(QUICFrameType::STREAM)) { + SCOPED_MUTEX_LOCK(lock, this->_write_vio.mutex, this_ethread()); - uint64_t len = 0; - IOBufferReader *reader = this->_write_vio.get_reader(); - if (!pure_fin) { - uint64_t data_len = reader->block_read_avail(); - if (data_len == 0) { + uint64_t maximum_data_size = 0; + if (maximum_frame_size <= MAX_STREAM_FRAME_OVERHEAD) { return frame; } - - // Check Connection/Stream level credit only if the generating STREAM frame is not pure fin - uint64_t stream_credit = this->_remote_flow_controller.credit(); - if (stream_credit == 0) { - // STREAM_DATA_BLOCKED - frame = this->_remote_flow_controller.generate_frame(buf, level, UINT16_MAX, maximum_frame_size, timestamp); - return frame; + maximum_data_size = maximum_frame_size - MAX_STREAM_FRAME_OVERHEAD; + + bool pure_fin = false; + bool fin = false; + if ((this->_write_vio.nbytes != 0 || this->_write_vio.nbytes != INT64_MAX) && + this->_write_vio.nbytes == static_cast(this->_send_offset)) { + // Pure FIN stream should be sent regardless status of remote flow controller, because the length is zero. + pure_fin = true; + fin = true; } - if (connection_credit == 0) { - // BLOCKED - BLOCKED frame will be sent by connection level remote flow controller - return frame; + uint64_t len = 0; + IOBufferReader *reader = this->_write_vio.get_reader(); + if (!pure_fin) { + uint64_t data_len = reader->block_read_avail(); + if (data_len == 0) { + return frame; + } + + // Check Connection/Stream level credit only if the generating STREAM frame is not pure fin + uint64_t stream_credit = this->_remote_flow_controller.credit(); + if (stream_credit == 0) { + // STREAM_DATA_BLOCKED + frame = + this->_remote_flow_controller.generate_frame(buf, level, UINT16_MAX, maximum_frame_size, current_packet_size, seq_num); + return frame; + } + + if (connection_credit == 0) { + // BLOCKED - BLOCKED frame will be sent by connection level remote flow controller + return frame; + } + + len = std::min(data_len, std::min(maximum_data_size, std::min(stream_credit, connection_credit))); + + // data_len, maximum_data_size, stream_credit and connection_credit are already checked they're larger than 0 + ink_assert(len != 0); + + if (this->_write_vio.nbytes == static_cast(this->_send_offset + len)) { + fin = true; + } } - len = std::min(data_len, std::min(maximum_data_size, std::min(stream_credit, connection_credit))); - - // data_len, maximum_data_size, stream_credit and connection_credit are already checked they're larger than 0 - ink_assert(len != 0); - - if (this->_write_vio.nbytes == static_cast(this->_send_offset + len)) { - fin = true; + Ptr block = make_ptr(reader->get_current_block()->clone()); + block->consume(reader->start_offset); + block->_end = std::min(block->start() + len, block->_buf_end); + ink_assert(static_cast(block->read_avail()) == len); + + // STREAM - Pure FIN or data length is lager than 0 + // FIXME has_length_flag and has_offset_flag should be configurable + frame = QUICFrameFactory::create_stream_frame(buf, block, this->_id, this->_send_offset, fin, true, true, + this->_issue_frame_id(), this); + if (!this->_state.is_allowed_to_send(*frame)) { + QUICStreamDebug("Canceled sending %s frame due to the stream state", QUICDebugNames::frame_type(frame->type())); + return frame; } - } - Ptr block = make_ptr(reader->get_current_block()->clone()); - block->consume(reader->start_offset); - block->_end = std::min(block->start() + len, block->_buf_end); - ink_assert(static_cast(block->read_avail()) == len); - - // STREAM - Pure FIN or data length is lager than 0 - // FIXME has_length_flag and has_offset_flag should be configurable - frame = QUICFrameFactory::create_stream_frame(buf, block, this->_id, this->_send_offset, fin, true, true, this->_issue_frame_id(), - this); - if (!this->_state.is_allowed_to_send(*frame)) { - QUICStreamDebug("Canceled sending %s frame due to the stream state", QUICDebugNames::frame_type(frame->type())); - return frame; - } + if (!pure_fin) { + int ret = this->_remote_flow_controller.update(this->_send_offset + len); + // We cannot cancel sending the frame after updating the flow controller - if (!pure_fin) { - int ret = this->_remote_flow_controller.update(this->_send_offset + len); - // We cannot cancel sending the frame after updating the flow controller + // Calling update always success, because len is always less than stream_credit + ink_assert(ret == 0); - // Calling update always success, because len is always less than stream_credit - ink_assert(ret == 0); + QUICVStreamFCDebug("[REMOTE] %" PRIu64 "/%" PRIu64, this->_remote_flow_controller.current_offset(), + this->_remote_flow_controller.current_limit()); + if (this->_remote_flow_controller.current_offset() == this->_remote_flow_controller.current_limit()) { + QUICStreamDebug("Flow Controller will block sending a STREAM frame"); + } - QUICStreamFCDebug("[REMOTE] %" PRIu64 "/%" PRIu64, this->_remote_flow_controller.current_offset(), - this->_remote_flow_controller.current_limit()); - if (this->_remote_flow_controller.current_offset() == this->_remote_flow_controller.current_limit()) { - QUICStreamDebug("Flow Controller will block sending a STREAM frame"); + reader->consume(len); + this->_send_offset += len; + this->_write_vio.ndone += len; } + this->_records_stream_frame(level, *static_cast(frame)); - reader->consume(len); - this->_send_offset += len; - this->_write_vio.ndone += len; + this->_signal_write_event(); + this->_state.update_with_sending_frame(*frame); } - this->_records_stream_frame(level, *static_cast(frame)); - - this->_signal_write_event(); - this->_state.update_with_sending_frame(*frame); return frame; } diff --git a/iocore/net/quic/QUICBidirectionalStream.h b/iocore/net/quic/QUICBidirectionalStream.h index fcbd5d7140b..d64f899bd6c 100644 --- a/iocore/net/quic/QUICBidirectionalStream.h +++ b/iocore/net/quic/QUICBidirectionalStream.h @@ -44,9 +44,9 @@ class QUICBidirectionalStream : public QUICStreamVConnection, public QUICTransfe int state_stream_closed(int event, void *data); // QUICFrameGenerator - bool will_generate_frame(QUICEncryptionLevel level, ink_hrtime timestamp) override; + bool will_generate_frame(QUICEncryptionLevel level, size_t current_packet_size, bool ack_eliciting, uint32_t seq_num) override; QUICFrame *generate_frame(uint8_t *buf, QUICEncryptionLevel level, uint64_t connection_credit, uint16_t maximum_frame_size, - ink_hrtime timestamp) override; + size_t current_packet_size, uint32_t seq_num) override; virtual QUICConnectionErrorUPtr recv(const QUICStreamFrame &frame) override; virtual QUICConnectionErrorUPtr recv(const QUICMaxStreamDataFrame &frame) override; diff --git a/iocore/net/quic/QUICConfig.cc b/iocore/net/quic/QUICConfig.cc index 600cef8e75f..aec4f137bc1 100644 --- a/iocore/net/quic/QUICConfig.cc +++ b/iocore/net/quic/QUICConfig.cc @@ -44,23 +44,24 @@ quic_new_ssl_ctx() SSL_CTX_set_max_proto_version(ssl_ctx, TLS1_3_VERSION); #ifndef OPENSSL_IS_BORINGSSL - // FIXME: OpenSSL (1.1.1-alpha) enable this option by default. But this shoule be removed when OpenSSL disable this by default. + // FIXME: OpenSSL (1.1.1-alpha) enable this option by default. But this should be removed when OpenSSL disable this by default. SSL_CTX_clear_options(ssl_ctx, SSL_OP_ENABLE_MIDDLEBOX_COMPAT); SSL_CTX_set_max_early_data(ssl_ctx, UINT32_C(0xFFFFFFFF)); - SSL_CTX_add_custom_ext(ssl_ctx, QUICTransportParametersHandler::TRANSPORT_PARAMETER_ID, - SSL_EXT_TLS_ONLY | SSL_EXT_CLIENT_HELLO | SSL_EXT_TLS1_3_ENCRYPTED_EXTENSIONS, - &QUICTransportParametersHandler::add, &QUICTransportParametersHandler::free, nullptr, - &QUICTransportParametersHandler::parse, nullptr); #else - // QUIC Transport Parameters are accesible with SSL_set_quic_transport_params and SSL_get_peer_quic_transport_params + // QUIC Transport Parameters are accessible with SSL_set_quic_transport_params and SSL_get_peer_quic_transport_params #endif #ifdef SSL_MODE_QUIC_HACK // tatsuhiro-t's custom OpenSSL for QUIC draft-13 // https://github.com/tatsuhiro-t/openssl/tree/quic-draft-13 SSL_CTX_set_mode(ssl_ctx, SSL_MODE_QUIC_HACK); + SSL_CTX_add_custom_ext(ssl_ctx, QUICTransportParametersHandler::TRANSPORT_PARAMETER_ID, + SSL_EXT_TLS_ONLY | SSL_EXT_CLIENT_HELLO | SSL_EXT_TLS1_3_ENCRYPTED_EXTENSIONS, + &QUICTransportParametersHandler::add, &QUICTransportParametersHandler::free, nullptr, + &QUICTransportParametersHandler::parse, nullptr); + #endif return ssl_ctx; @@ -117,15 +118,19 @@ QUICConfigParams::initialize() { REC_EstablishStaticConfigInt32U(this->_instance_id, "proxy.config.quic.instance_id"); REC_EstablishStaticConfigInt32(this->_connection_table_size, "proxy.config.quic.connection_table.size"); - REC_EstablishStaticConfigInt32U(this->_num_alt_connection_ids, "proxy.config.quic.num_alt_connection_ids"); REC_EstablishStaticConfigInt32U(this->_stateless_retry, "proxy.config.quic.server.stateless_retry_enabled"); REC_EstablishStaticConfigInt32U(this->_vn_exercise_enabled, "proxy.config.quic.client.vn_exercise_enabled"); REC_EstablishStaticConfigInt32U(this->_cm_exercise_enabled, "proxy.config.quic.client.cm_exercise_enabled"); + REC_EstablishStaticConfigInt32U(this->_quantum_readiness_test_enabled_out, + "proxy.config.quic.client.quantum_readiness_test_enabled"); + REC_EstablishStaticConfigInt32U(this->_quantum_readiness_test_enabled_in, + "proxy.config.quic.server.quantum_readiness_test_enabled"); REC_ReadConfigStringAlloc(this->_server_supported_groups, "proxy.config.quic.server.supported_groups"); REC_ReadConfigStringAlloc(this->_client_supported_groups, "proxy.config.quic.client.supported_groups"); REC_ReadConfigStringAlloc(this->_client_session_file, "proxy.config.quic.client.session_file"); REC_ReadConfigStringAlloc(this->_client_keylog_file, "proxy.config.quic.client.keylog_file"); + REC_ReadConfigStringAlloc(this->_qlog_dir, "proxy.config.quic.qlog_dir"); // Transport Parameters REC_EstablishStaticConfigInt32U(this->_no_activity_timeout_in, "proxy.config.quic.no_activity_timeout_in"); @@ -158,6 +163,9 @@ QUICConfigParams::initialize() REC_EstablishStaticConfigInt32U(this->_ack_delay_exponent_out, "proxy.config.quic.ack_delay_exponent_out"); REC_EstablishStaticConfigInt32U(this->_max_ack_delay_in, "proxy.config.quic.max_ack_delay_in"); REC_EstablishStaticConfigInt32U(this->_max_ack_delay_out, "proxy.config.quic.max_ack_delay_out"); + REC_EstablishStaticConfigInt32U(this->_active_cid_limit_in, "proxy.config.quic.active_cid_limit_in"); + REC_EstablishStaticConfigInt32U(this->_active_cid_limit_out, "proxy.config.quic.active_cid_limit_out"); + REC_EstablishStaticConfigInt32U(this->_disable_active_migration, "proxy.config.quic.disable_active_migration"); // Loss Detection REC_EstablishStaticConfigInt32U(this->_ld_packet_threshold, "proxy.config.quic.loss_detection.packet_threshold"); @@ -171,9 +179,8 @@ QUICConfigParams::initialize() this->_ld_initial_rtt = HRTIME_MSECONDS(timeout); // Congestion Control - REC_EstablishStaticConfigInt32U(this->_cc_max_datagram_size, "proxy.config.quic.congestion_control.max_datagram_size"); - REC_EstablishStaticConfigInt32U(this->_cc_initial_window_scale, "proxy.config.quic.congestion_control.initial_window_scale"); - REC_EstablishStaticConfigInt32U(this->_cc_minimum_window_scale, "proxy.config.quic.congestion_control.minimum_window_scale"); + REC_EstablishStaticConfigInt32U(this->_cc_initial_window, "proxy.config.quic.congestion_control.initial_window"); + REC_EstablishStaticConfigInt32U(this->_cc_minimum_window, "proxy.config.quic.congestion_control.minimum_window"); REC_EstablishStaticConfigFloat(this->_cc_loss_reduction_factor, "proxy.config.quic.congestion_control.loss_reduction_factor"); REC_EstablishStaticConfigInt32U(this->_cc_persistent_congestion_threshold, "proxy.config.quic.congestion_control.persistent_congestion_threshold"); @@ -225,12 +232,6 @@ QUICConfigParams::connection_table_size() return _connection_table_size; } -uint32_t -QUICConfigParams::num_alt_connection_ids() const -{ - return this->_num_alt_connection_ids; -} - uint32_t QUICConfigParams::stateless_retry() const { @@ -249,6 +250,18 @@ QUICConfigParams::cm_exercise_enabled() const return this->_cm_exercise_enabled; } +uint32_t +QUICConfigParams::quantum_readiness_test_enabled_in() const +{ + return this->_quantum_readiness_test_enabled_in; +} + +uint32_t +QUICConfigParams::quantum_readiness_test_enabled_out() const +{ + return this->_quantum_readiness_test_enabled_out; +} + uint32_t QUICConfigParams::initial_max_data_in() const { @@ -345,6 +358,24 @@ QUICConfigParams::max_ack_delay_out() const return this->_max_ack_delay_out; } +uint8_t +QUICConfigParams::active_cid_limit_in() const +{ + return this->_active_cid_limit_in; +} + +uint8_t +QUICConfigParams::active_cid_limit_out() const +{ + return this->_active_cid_limit_out; +} + +bool +QUICConfigParams::disable_active_migration() const +{ + return this->_disable_active_migration; +} + const char * QUICConfigParams::server_supported_groups() const { @@ -387,27 +418,16 @@ QUICConfigParams::ld_initial_rtt() const return _ld_initial_rtt; } -uint32_t -QUICConfigParams::cc_max_datagram_size() const -{ - return _cc_max_datagram_size; -} - uint32_t QUICConfigParams::cc_initial_window() const { - // kInitialWindow: Default limit on the initial amount of data in - // flight, in bytes. Taken from [RFC6928]. The RECOMMENDED value is - // the minimum of 10 * kMaxDatagramSize and max(2* kMaxDatagramSize, - // 14600)). - return std::min(_cc_initial_window_scale * _cc_max_datagram_size, - std::max(2 * _cc_max_datagram_size, static_cast(14600))); + return _cc_initial_window; } uint32_t QUICConfigParams::cc_minimum_window() const { - return _cc_minimum_window_scale * _cc_max_datagram_size; + return _cc_minimum_window; } float @@ -440,6 +460,12 @@ QUICConfigParams::client_keylog_file() const return this->_client_keylog_file; } +const char * +QUICConfigParams::qlog_dir() const +{ + return this->_qlog_dir; +} + // // QUICConfig // diff --git a/iocore/net/quic/QUICConfig.h b/iocore/net/quic/QUICConfig.h index 36ff7af9554..0ce045a97de 100644 --- a/iocore/net/quic/QUICConfig.h +++ b/iocore/net/quic/QUICConfig.h @@ -37,15 +37,17 @@ class QUICConfigParams : public ConfigInfo void initialize(); uint32_t instance_id() const; - uint32_t num_alt_connection_ids() const; uint32_t stateless_retry() const; uint32_t vn_exercise_enabled() const; uint32_t cm_exercise_enabled() const; + uint32_t quantum_readiness_test_enabled_in() const; + uint32_t quantum_readiness_test_enabled_out() const; const char *server_supported_groups() const; const char *client_supported_groups() const; const char *client_session_file() const; const char *client_keylog_file() const; + const char *qlog_dir() const; shared_SSL_CTX client_ssl_ctx() const; @@ -70,6 +72,9 @@ class QUICConfigParams : public ConfigInfo uint8_t ack_delay_exponent_out() const; uint8_t max_ack_delay_in() const; uint8_t max_ack_delay_out() const; + uint8_t active_cid_limit_in() const; + uint8_t active_cid_limit_out() const; + bool disable_active_migration() const; // Loss Detection uint32_t ld_packet_threshold() const; @@ -92,16 +97,18 @@ class QUICConfigParams : public ConfigInfo // TODO: make configurable static const uint8_t _scid_len = 18; //< Length of Source Connection ID - uint32_t _instance_id = 0; - uint32_t _num_alt_connection_ids = 0; - uint32_t _stateless_retry = 0; - uint32_t _vn_exercise_enabled = 0; - uint32_t _cm_exercise_enabled = 0; + uint32_t _instance_id = 0; + uint32_t _stateless_retry = 0; + uint32_t _vn_exercise_enabled = 0; + uint32_t _cm_exercise_enabled = 0; + uint32_t _quantum_readiness_test_enabled_in = 0; + uint32_t _quantum_readiness_test_enabled_out = 0; char *_server_supported_groups = nullptr; char *_client_supported_groups = nullptr; char *_client_session_file = nullptr; char *_client_keylog_file = nullptr; + char *_qlog_dir = nullptr; shared_SSL_CTX _client_ssl_ctx = nullptr; @@ -128,19 +135,21 @@ class QUICConfigParams : public ConfigInfo uint32_t _ack_delay_exponent_out = 0; uint32_t _max_ack_delay_in = 0; uint32_t _max_ack_delay_out = 0; + uint32_t _active_cid_limit_in = 0; + uint32_t _active_cid_limit_out = 0; + uint32_t _disable_active_migration = 0; // [draft-17 recovery] 6.4.1. Constants of interest uint32_t _ld_packet_threshold = 3; float _ld_time_threshold = 1.25; ink_hrtime _ld_granularity = HRTIME_MSECONDS(1); - ink_hrtime _ld_initial_rtt = HRTIME_MSECONDS(100); + ink_hrtime _ld_initial_rtt = HRTIME_MSECONDS(500); // [draft-11 recovery] 4.7.1. Constants of interest - uint32_t _cc_max_datagram_size = 1200; - uint32_t _cc_initial_window_scale = 10; // Actual initial window size is this value multiplied by the _cc_default_mss - uint32_t _cc_minimum_window_scale = 2; // Actual minimum window size is this value multiplied by the _cc_default_mss + uint32_t _cc_initial_window = 1200 * 10; + uint32_t _cc_minimum_window = 1200 * 2; float _cc_loss_reduction_factor = 0.5; - uint32_t _cc_persistent_congestion_threshold = 2; + uint32_t _cc_persistent_congestion_threshold = 3; }; class QUICConfig diff --git a/iocore/net/quic/QUICCongestionController.cc b/iocore/net/quic/QUICCongestionController.cc deleted file mode 100644 index 79d8fd8df18..00000000000 --- a/iocore/net/quic/QUICCongestionController.cc +++ /dev/null @@ -1,242 +0,0 @@ -/** @file - * - * A brief file description - * - * @section license License - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include - -#define QUICCCDebug(fmt, ...) \ - Debug("quic_cc", \ - "[%s] " \ - "window: %" PRIu32 " bytes: %" PRIu32 " ssthresh: %" PRIu32 " " fmt, \ - this->_info->cids().data(), this->_congestion_window, this->_bytes_in_flight, this->_ssthresh, ##__VA_ARGS__) - -#define QUICCCError(fmt, ...) \ - Error("quic_cc", \ - "[%s] " \ - "window: %" PRIu32 " bytes: %" PRIu32 " ssthresh: %" PRIu32 " " fmt, \ - this->_info->cids().data(), this->_congestion_window, this->_bytes_in_flight, this->_ssthresh, ##__VA_ARGS__) - -QUICCongestionController::QUICCongestionController(const QUICRTTProvider &rtt_provider, QUICConnectionInfoProvider *info, - const QUICCCConfig &cc_config) - : _cc_mutex(new_ProxyMutex()), _info(info), _rtt_provider(rtt_provider) -{ - this->_k_max_datagram_size = cc_config.max_datagram_size(); - this->_k_initial_window = cc_config.initial_window(); - this->_k_minimum_window = cc_config.minimum_window(); - this->_k_loss_reduction_factor = cc_config.loss_reduction_factor(); - this->_k_persistent_congestion_threshold = cc_config.persistent_congestion_threshold(); - - this->reset(); -} - -void -QUICCongestionController::on_packet_sent(size_t bytes_sent) -{ - SCOPED_MUTEX_LOCK(lock, this->_cc_mutex, this_ethread()); - this->_bytes_in_flight += bytes_sent; -} - -bool -QUICCongestionController::_in_recovery(ink_hrtime sent_time) -{ - return sent_time <= this->_recovery_start_time; -} - -bool -QUICCongestionController::is_app_limited() -{ - // FIXME : don't known how does app worked here - return false; -} - -void -QUICCongestionController::on_packet_acked(const QUICPacketInfo &acked_packet) -{ - // Remove from bytes_in_flight. - SCOPED_MUTEX_LOCK(lock, this->_cc_mutex, this_ethread()); - this->_bytes_in_flight -= acked_packet.sent_bytes; - if (this->_in_recovery(acked_packet.time_sent)) { - // Do not increase congestion window in recovery period. - return; - } - - if (this->is_app_limited()) { - // Do not increase congestion_window if application - // limited. - return; - } - - if (this->_congestion_window < this->_ssthresh) { - // Slow start. - this->_congestion_window += acked_packet.sent_bytes; - QUICCCDebug("slow start window chaged"); - } else { - // Congestion avoidance. - this->_congestion_window += this->_k_max_datagram_size * acked_packet.sent_bytes / this->_congestion_window; - QUICCCDebug("Congestion avoidance window changed"); - } -} - -// addtional code -// the original one is: -// CongestionEvent(sent_time): -void -QUICCongestionController::_congestion_event(ink_hrtime sent_time) -{ - // Start a new congestion event if the sent time is larger - // than the start time of the previous recovery epoch. - if (!this->_in_recovery(sent_time)) { - this->_recovery_start_time = Thread::get_hrtime(); - this->_congestion_window *= this->_k_loss_reduction_factor; - this->_congestion_window = std::max(this->_congestion_window, this->_k_minimum_window); - this->_ssthresh = this->_congestion_window; - } -} - -// additional code -// the original one is: -// ProcessECN(ack): -void -QUICCongestionController::process_ecn(const QUICPacketInfo &acked_largest_packet, const QUICAckFrame::EcnSection *ecn_section) -{ - // If the ECN-CE counter reported by the peer has increased, - // this could be a new congestion event. - if (ecn_section->ecn_ce_count() > this->_ecn_ce_counter) { - this->_ecn_ce_counter = ecn_section->ecn_ce_count(); - // Start a new congestion event if the last acknowledged - // packet was sent after the start of the previous - // recovery epoch. - this->_congestion_event(acked_largest_packet.time_sent); - } -} - -bool -QUICCongestionController::_in_persistent_congestion(const std::map &lost_packets, - QUICPacketInfo *largest_lost_packet) -{ - ink_hrtime period = this->_rtt_provider.congestion_period(this->_k_persistent_congestion_threshold); - // Determine if all packets in the window before the - // newest lost packet, including the edges, are marked - // lost - return this->_in_window_lost(lost_packets, largest_lost_packet, period); -} - -// additional code -// the original one is: -// OnPacketsLost(lost_packets): -void -QUICCongestionController::on_packets_lost(const std::map &lost_packets) -{ - if (lost_packets.empty()) { - return; - } - - SCOPED_MUTEX_LOCK(lock, this->_cc_mutex, this_ethread()); - // Remove lost packets from bytes_in_flight. - for (auto &lost_packet : lost_packets) { - this->_bytes_in_flight -= lost_packet.second->sent_bytes; - } - QUICPacketInfo *largest_lost_packet = lost_packets.rbegin()->second; - // Start a new recovery epoch if the lost packet is larger - // than the end of the previous recovery epoch. - this->_congestion_event(largest_lost_packet->time_sent); - - // Collapse congestion window if persistent congestion - if (this->_in_persistent_congestion(lost_packets, largest_lost_packet)) { - this->_congestion_window = this->_k_minimum_window; - } -} - -bool -QUICCongestionController::check_credit() const -{ - if (this->_bytes_in_flight >= this->_congestion_window) { - QUICCCDebug("Congestion control pending"); - } - - return this->_bytes_in_flight < this->_congestion_window; -} - -uint32_t -QUICCongestionController::open_window() const -{ - if (this->check_credit()) { - return this->_congestion_window - this->_bytes_in_flight; - } else { - return 0; - } -} - -uint32_t -QUICCongestionController::bytes_in_flight() const -{ - return this->_bytes_in_flight; -} - -uint32_t -QUICCongestionController::congestion_window() const -{ - return this->_congestion_window; -} - -uint32_t -QUICCongestionController::current_ssthresh() const -{ - return this->_ssthresh; -} - -// [draft-17 recovery] 7.9.3. Initialization -void -QUICCongestionController::reset() -{ - SCOPED_MUTEX_LOCK(lock, this->_cc_mutex, this_ethread()); - - this->_bytes_in_flight = 0; - this->_congestion_window = this->_k_initial_window; - this->_recovery_start_time = 0; - this->_ssthresh = UINT32_MAX; -} - -bool -QUICCongestionController::_in_window_lost(const std::map &lost_packets, - QUICPacketInfo *largest_lost_packet, ink_hrtime period) const -{ - // check whether packets are continuous. return true if all continuous packets are in period - QUICPacketNumber next_expected = UINT64_MAX; - for (auto &it : lost_packets) { - if (it.second->time_sent >= largest_lost_packet->time_sent - period) { - if (next_expected == UINT64_MAX) { - next_expected = it.second->packet_number + 1; - continue; - } - - if (next_expected != it.second->packet_number) { - return false; - } - - next_expected = it.second->packet_number + 1; - } - } - - return next_expected == UINT64_MAX ? false : true; -} diff --git a/iocore/net/quic/QUICCongestionController.h b/iocore/net/quic/QUICCongestionController.h new file mode 100644 index 00000000000..7417ac873ad --- /dev/null +++ b/iocore/net/quic/QUICCongestionController.h @@ -0,0 +1,56 @@ +/** @file + * + * A brief file description + * + * @section license License + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "QUICFrame.h" + +class QUICCongestionController +{ +public: + enum class State : uint8_t { + RECOVERY, + CONGESTION_AVOIDANCE, + SLOW_START, + APPLICATION_LIMITED, + }; + + virtual ~QUICCongestionController() {} + // Appendix B. Congestion Control Pseudocode + virtual void on_packet_sent(size_t bytes_sent) = 0; + virtual void on_packets_acked(const std::vector &packets) = 0; + virtual void process_ecn(const QUICAckFrame &ack, QUICPacketNumberSpace pn_space, ink_hrtime largest_acked_packet_time_sent) = 0; + virtual void on_packets_lost(const std::map &packets) = 0; + // The function signature is different from the pseudo code because LD takes care of most of the processes + virtual void on_packet_number_space_discarded(size_t bytes_in_flight) = 0; + + // These are additional and not on the spec + virtual void add_extra_credit() = 0; + virtual void reset() = 0; + virtual uint32_t credit() const = 0; + virtual uint32_t bytes_in_flight() const = 0; + + // Debug + virtual uint32_t congestion_window() const = 0; + virtual uint32_t current_ssthresh() const = 0; +}; diff --git a/iocore/net/quic/QUICConnection.h b/iocore/net/quic/QUICConnection.h index d2c3d56b78a..02cbbb04db9 100644 --- a/iocore/net/quic/QUICConnection.h +++ b/iocore/net/quic/QUICConnection.h @@ -34,26 +34,44 @@ class UDPPacket; class QUICConnectionInfoProvider { public: + virtual ~QUICConnectionInfoProvider() {} virtual QUICConnectionId peer_connection_id() const = 0; virtual QUICConnectionId original_connection_id() const = 0; - virtual QUICConnectionId first_connection_id() const = 0; - virtual QUICConnectionId connection_id() const = 0; - virtual std::string_view cids() const = 0; - virtual const QUICFiveTuple five_tuple() const = 0; + /** + * This is S1 on 7.3.Authenticating Connection IDs + */ + virtual QUICConnectionId first_connection_id() const = 0; + /** + * This is S2 on 7.3.Authenticating Connection IDs + */ + virtual QUICConnectionId retry_source_connection_id() const = 0; + /** + * This is C1 or S3 on 7.3.Authenticating Connection IDs + */ + virtual QUICConnectionId initial_source_connection_id() const = 0; + virtual QUICConnectionId connection_id() const = 0; + virtual std::string_view cids() const = 0; + virtual const QUICFiveTuple five_tuple() const = 0; virtual uint32_t pmtu() const = 0; virtual NetVConnectionContext_t direction() const = 0; virtual int select_next_protocol(SSL *ssl, const unsigned char **out, unsigned char *outlen, const unsigned char *in, unsigned inlen) const = 0; virtual bool is_closed() const = 0; + virtual bool is_at_anti_amplification_limit() const = 0; + virtual bool is_address_validation_completed() const = 0; + virtual bool is_handshake_completed() const = 0; + virtual bool has_keys_for(QUICPacketNumberSpace space) const = 0; + virtual QUICVersion negotiated_version() const = 0; virtual std::string_view negotiated_application_name() const = 0; }; class QUICConnection : public QUICFrameHandler, public QUICConnectionInfoProvider { public: - virtual QUICStreamManager *stream_manager() = 0; - virtual void close(QUICConnectionErrorUPtr error) = 0; - virtual void handle_received_packet(UDPPacket *packeet) = 0; - virtual void ping() = 0; + virtual QUICStreamManager *stream_manager() = 0; + virtual void close_quic_connection(QUICConnectionErrorUPtr error) = 0; + virtual void reset_quic_connection() = 0; + virtual void handle_received_packet(UDPPacket *packet) = 0; + virtual void ping() = 0; }; diff --git a/iocore/net/quic/QUICContext.cc b/iocore/net/quic/QUICContext.cc new file mode 100644 index 00000000000..b065f56102a --- /dev/null +++ b/iocore/net/quic/QUICContext.cc @@ -0,0 +1,148 @@ +/** @file + + A brief file description + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#include "QUICContext.h" +#include "QUICConnection.h" +#include "QUICLossDetector.h" +#include "QUICPacketProtectionKeyInfo.h" + +class QUICCCConfigQCP : public QUICCCConfig +{ +public: + virtual ~QUICCCConfigQCP() {} + QUICCCConfigQCP(const QUICConfigParams *params) : _params(params) {} + + uint32_t + initial_window() const override + { + return this->_params->cc_initial_window(); + } + + uint32_t + minimum_window() const override + { + return this->_params->cc_minimum_window(); + } + + float + loss_reduction_factor() const override + { + return this->_params->cc_loss_reduction_factor(); + } + + uint32_t + persistent_congestion_threshold() const override + { + return this->_params->cc_persistent_congestion_threshold(); + } + +private: + const QUICConfigParams *_params; +}; + +class QUICLDConfigQCP : public QUICLDConfig +{ +public: + virtual ~QUICLDConfigQCP() {} + QUICLDConfigQCP(const QUICConfigParams *params) : _params(params) {} + + uint32_t + packet_threshold() const override + { + return this->_params->ld_packet_threshold(); + } + + float + time_threshold() const override + { + return this->_params->ld_time_threshold(); + } + + ink_hrtime + granularity() const override + { + return this->_params->ld_granularity(); + } + + ink_hrtime + initial_rtt() const override + { + return this->_params->ld_initial_rtt(); + } + +private: + const QUICConfigParams *_params; +}; + +QUICContext::QUICContext(QUICRTTProvider *rtt, QUICConnectionInfoProvider *info, QUICPacketProtectionKeyInfoProvider *key_info, + QUICPathManager *path_manager) + : _key_info(key_info), + _connection_info(info), + _rtt_provider(rtt), + _path_manager(path_manager), + _ld_config(std::make_unique(_config)), + _cc_config(std::make_unique(_config)) +{ +} + +QUICConnectionInfoProvider * +QUICContext::connection_info() const +{ + return _connection_info; +} + +QUICConfig::scoped_config +QUICContext::config() const +{ + return _config; +} + +QUICPacketProtectionKeyInfoProvider * +QUICContext::key_info() const +{ + return _key_info; +} + +QUICRTTProvider * +QUICContext::rtt_provider() const +{ + return _rtt_provider; +} + +QUICLDConfig & +QUICContext::ld_config() const +{ + return *_ld_config; +} + +QUICCCConfig & +QUICContext::cc_config() const +{ + return *_cc_config; +} + +QUICPathManager * +QUICContext::path_manager() const +{ + return _path_manager; +} diff --git a/iocore/net/quic/QUICContext.h b/iocore/net/quic/QUICContext.h new file mode 100644 index 00000000000..ab2dae9a694 --- /dev/null +++ b/iocore/net/quic/QUICContext.h @@ -0,0 +1,195 @@ +/** @file + + A brief file description + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#pragma once + +#include "QUICTypes.h" +#include "QUICConnection.h" +#include "QUICConfig.h" +#include "QUICEvents.h" +#include "QUICCongestionController.h" + +class QUICRTTProvider; +class QUICCongestionController; +class QUICPacketProtectionKeyInfoProvider; +class QUICPathManager; +class QUICPacketR; +class QUICPacket; + +class QUICNetVConnection; +struct QUICPacketInfo; + +// this class is a connection between the callbacks. it should do something +// TODO: it should do something +class QUICCallbackContext +{ +}; + +class QUICCallback +{ +public: + virtual ~QUICCallback() {} + + // callback on connection close event + virtual void connection_close_callback(QUICCallbackContext &){}; + // callback on packet send event + virtual void packet_send_callback(QUICCallbackContext &, const QUICPacket &p){}; + // callback on packet lost event + virtual void packet_lost_callback(QUICCallbackContext &, const QUICSentPacketInfo &p){}; + // callback on packet receive event + virtual void packet_recv_callback(QUICCallbackContext &, const QUICPacket &p){}; + // callback on packet acked event + virtual void cc_metrics_update_callback(QUICCallbackContext &, uint64_t congestion_window, uint64_t bytes_in_flight, + uint64_t sshresh){}; + // callback on packet receive event + virtual void frame_packetize_callback(QUICCallbackContext &, const QUICFrame &p){}; + // callback on packet receive event + virtual void frame_recv_callback(QUICCallbackContext &, const QUICFrame &p){}; + // callback on packet receive event + virtual void congestion_state_updated_callback(QUICCallbackContext &, QUICCongestionController::State p){}; +}; + +class QUICContext +{ +public: + QUICContext(QUICRTTProvider *rtt, QUICConnectionInfoProvider *info, QUICPacketProtectionKeyInfoProvider *key_info, + QUICPathManager *path_manager); + + virtual ~QUICContext(){}; + virtual QUICConnectionInfoProvider *connection_info() const; + virtual QUICConfig::scoped_config config() const; + virtual QUICLDConfig &ld_config() const; + virtual QUICPacketProtectionKeyInfoProvider *key_info() const; + virtual QUICCCConfig &cc_config() const; + virtual QUICRTTProvider *rtt_provider() const; + virtual QUICPathManager *path_manager() const; + + // regist a callback which will be called when specified event happen. + void + regist_callback(std::shared_ptr cbs) + { + this->_callbacks.push_back(cbs); + } + + enum class CallbackEvent : uint8_t { + PACKET_LOST, + PACKET_SEND, + FRAME_PACKETIZE, + PACKET_RECV, + FRAME_RECV, + METRICS_UPDATE, + CONNECTION_CLOSE, + CONGESTION_STATE_CHANGED, + }; + + // FIXME stupid trigger should be fix in more smart way. + void + trigger(CallbackEvent e, const QUICPacket *p = nullptr) + { + QUICCallbackContext ctx; + switch (e) { + case CallbackEvent::PACKET_RECV: + for (auto &&it : this->_callbacks) { + it->packet_recv_callback(ctx, *p); + } + break; + case CallbackEvent::PACKET_SEND: + for (auto &&it : this->_callbacks) { + it->packet_send_callback(ctx, *p); + } + break; + case CallbackEvent::CONNECTION_CLOSE: + for (auto &&it : this->_callbacks) { + it->connection_close_callback(ctx); + } + break; + default: + break; + } + } + + void + trigger(CallbackEvent e, const QUICSentPacketInfo &p) + { + QUICCallbackContext ctx; + for (auto &&it : this->_callbacks) { + it->packet_lost_callback(ctx, p); + } + } + + void + trigger(CallbackEvent e, uint64_t congestion_window, uint64_t bytes_in_flight, uint64_t sshresh) + { + QUICCallbackContext ctx; + for (auto &&it : this->_callbacks) { + it->cc_metrics_update_callback(ctx, congestion_window, bytes_in_flight, sshresh); + } + } + + void + trigger(CallbackEvent e, QUICCongestionController::State state) + { + QUICCallbackContext ctx; + for (auto &&it : this->_callbacks) { + it->congestion_state_updated_callback(ctx, state); + } + } + + void + trigger(CallbackEvent e, const QUICFrame &frame) + { + QUICCallbackContext ctx; + switch (e) { + case CallbackEvent::FRAME_PACKETIZE: + + for (auto &&it : this->_callbacks) { + it->frame_packetize_callback(ctx, frame); + } + break; + + case CallbackEvent::FRAME_RECV: + for (auto &&it : this->_callbacks) { + it->frame_recv_callback(ctx, frame); + } + break; + default: + break; + } + } + +protected: + // For Mock + QUICContext() {} + +private: + QUICConfig::scoped_config _config; + QUICPacketProtectionKeyInfoProvider *_key_info = nullptr; + QUICConnectionInfoProvider *_connection_info = nullptr; + QUICRTTProvider *_rtt_provider = nullptr; + QUICPathManager *_path_manager = nullptr; + + std::unique_ptr _ld_config = nullptr; + std::unique_ptr _cc_config = nullptr; + + std::vector> _callbacks; +}; diff --git a/iocore/net/quic/QUICCryptoStream.cc b/iocore/net/quic/QUICCryptoStream.cc index f3b702cc487..ba3afa277b7 100644 --- a/iocore/net/quic/QUICCryptoStream.cc +++ b/iocore/net/quic/QUICCryptoStream.cc @@ -102,7 +102,7 @@ QUICCryptoStream::write(const uint8_t *buf, int64_t len) } bool -QUICCryptoStream::will_generate_frame(QUICEncryptionLevel level, ink_hrtime timestamp) +QUICCryptoStream::will_generate_frame(QUICEncryptionLevel level, size_t current_packet_size, bool ack_eliciting, uint32_t seq_num) { return this->_write_buffer_reader->is_read_avail_more_than(0) || !this->is_retransmited_frame_queue_empty(); } @@ -112,7 +112,7 @@ QUICCryptoStream::will_generate_frame(QUICEncryptionLevel level, ink_hrtime time */ QUICFrame * QUICCryptoStream::generate_frame(uint8_t *buf, QUICEncryptionLevel level, uint64_t /* connection_credit */, - uint16_t maximum_frame_size, ink_hrtime timestamp) + uint16_t maximum_frame_size, size_t current_packet_size, uint32_t seq_num) { QUICConnectionErrorUPtr error = nullptr; diff --git a/iocore/net/quic/QUICCryptoStream.h b/iocore/net/quic/QUICCryptoStream.h index 6da3b2fcd2b..42f99db742e 100644 --- a/iocore/net/quic/QUICCryptoStream.h +++ b/iocore/net/quic/QUICCryptoStream.h @@ -53,9 +53,9 @@ class QUICCryptoStream : public QUICStream int64_t write(const uint8_t *buf, int64_t len); // QUICFrameGenerator - bool will_generate_frame(QUICEncryptionLevel level, ink_hrtime timestamp) override; + bool will_generate_frame(QUICEncryptionLevel level, size_t current_packet_size, bool ack_eliciting, uint32_t seq_num) override; QUICFrame *generate_frame(uint8_t *buf, QUICEncryptionLevel level, uint64_t connection_credit, uint16_t maximum_frame_size, - ink_hrtime timestamp) override; + size_t current_packet_size, uint32_t seq_num) override; private: void _on_frame_acked(QUICFrameInformationUPtr &info) override; diff --git a/iocore/net/quic/QUICDebugNames.cc b/iocore/net/quic/QUICDebugNames.cc index b70d95975bf..79d2797f688 100644 --- a/iocore/net/quic/QUICDebugNames.cc +++ b/iocore/net/quic/QUICDebugNames.cc @@ -90,6 +90,8 @@ QUICDebugNames::frame_type(QUICFrameType type) return "RETIRE_CONNECTION_ID"; case QUICFrameType::NEW_TOKEN: return "NEW_TOKEN"; + case QUICFrameType::HANDSHAKE_DONE: + return "HANDSHAKE_DONE"; case QUICFrameType::UNKNOWN: default: return "UNKNOWN"; @@ -119,24 +121,30 @@ QUICDebugNames::error_code(uint16_t code) return "NO_ERROR"; case static_cast(QUICTransErrorCode::INTERNAL_ERROR): return "INTERNAL_ERROR"; + case static_cast(QUICTransErrorCode::CONNECTION_REFUSED): + return "CONNECTION_REFUSED"; case static_cast(QUICTransErrorCode::FLOW_CONTROL_ERROR): return "FLOW_CONTROL_ERROR"; - case static_cast(QUICTransErrorCode::STREAM_ID_ERROR): - return "STREAM_ID_ERROR"; + case static_cast(QUICTransErrorCode::STREAM_LIMIT_ERROR): + return "STREAM_LIMIT_ERROR"; case static_cast(QUICTransErrorCode::STREAM_STATE_ERROR): return "STREAM_STATE_ERROR"; - case static_cast(QUICTransErrorCode::FINAL_OFFSET_ERROR): - return "FINAL_OFFSET_ERROR"; + case static_cast(QUICTransErrorCode::FINAL_SIZE_ERROR): + return "FINAL_SIZE_ERROR"; case static_cast(QUICTransErrorCode::FRAME_ENCODING_ERROR): return "FRAME_ENCODING_ERROR"; case static_cast(QUICTransErrorCode::TRANSPORT_PARAMETER_ERROR): return "TRANSPORT_PARAMETER_ERROR"; - case static_cast(QUICTransErrorCode::VERSION_NEGOTIATION_ERROR): - return "VERSION_NEGOTIATION_ERROR"; + case static_cast(QUICTransErrorCode::CONNECTION_ID_LIMIT_ERROR): + return "CONNECTION_ID_LIMIT_ERROR"; case static_cast(QUICTransErrorCode::PROTOCOL_VIOLATION): return "PROTOCOL_VIOLATION"; - case static_cast(QUICTransErrorCode::INVALID_MIGRATION): - return "INVALID_MIGRATION"; + case static_cast(QUICTransErrorCode::INVALID_TOKEN): + return "INVALID_TOKEN"; + case static_cast(QUICTransErrorCode::APPLICATION_ERROR): + return "APPLICATION_ERROR"; + case static_cast(QUICTransErrorCode::CRYPTO_BUFFER_EXCEEDED): + return "CRYPTO_BUFFER_EXCEEDED"; default: if (0x0100 <= code && code <= 0x01FF) { return "CRYPTO_ERROR"; @@ -181,28 +189,34 @@ QUICDebugNames::transport_parameter_id(QUICTransportParameterId id) return "INITIAL_MAX_DATA"; case QUICTransportParameterId::INITIAL_MAX_STREAMS_BIDI: return "INITIAL_MAX_STREAMS_BIDI"; - case QUICTransportParameterId::IDLE_TIMEOUT: - return "IDLE_TIMEOUT"; + case QUICTransportParameterId::MAX_IDLE_TIMEOUT: + return "MAX_IDLE_TIMEOUT"; case QUICTransportParameterId::PREFERRED_ADDRESS: return "PREFERRED_ADDRESS"; - case QUICTransportParameterId::MAX_PACKET_SIZE: - return "MAX_PACKET_SIZE"; + case QUICTransportParameterId::MAX_UDP_PAYLOAD_SIZE: + return "MAX_UDP_PAYLOAD_SIZE"; case QUICTransportParameterId::STATELESS_RESET_TOKEN: return "STATELESS_RESET_TOKEN"; case QUICTransportParameterId::ACK_DELAY_EXPONENT: return "ACK_DELAY_EXPONENT"; case QUICTransportParameterId::INITIAL_MAX_STREAMS_UNI: return "INITIAL_MAX_STREAMS_UNI"; - case QUICTransportParameterId::DISABLE_MIGRATION: - return "DISABLE_MIGRATION"; + case QUICTransportParameterId::DISABLE_ACTIVE_MIGRATION: + return "DISABLE_ACTIVE_MIGRATION"; case QUICTransportParameterId::INITIAL_MAX_STREAM_DATA_BIDI_REMOTE: return "INITIAL_MAX_STREAM_DATA_BIDI_REMOTE"; case QUICTransportParameterId::INITIAL_MAX_STREAM_DATA_UNI: return "INITIAL_MAX_STREAM_DATA_UNI"; case QUICTransportParameterId::MAX_ACK_DELAY: return "INITIAL_MAX_ACK_DELAY"; - case QUICTransportParameterId::ORIGINAL_CONNECTION_ID: - return "INITIAL_ORIGINAL_CONNECTION_ID"; + case QUICTransportParameterId::ORIGINAL_DESTINATION_CONNECTION_ID: + return "INITIAL_ORIGINAL_DESTINATION_CONNECTION_ID"; + case QUICTransportParameterId::ACTIVE_CONNECTION_ID_LIMIT: + return "ACTIVE_CONNECTION_ID_LIMIT"; + case QUICTransportParameterId::INITIAL_SOURCE_CONNECTION_ID: + return "INITIAL_SOURCE_CONNECTION_ID"; + case QUICTransportParameterId::RETRY_SOURCE_CONNECTION_ID: + return "RETRY_SOURCE_CONNECTION_ID"; default: return "UNKNOWN"; } @@ -317,12 +331,12 @@ const char * QUICDebugNames::pn_space(QUICPacketNumberSpace pn_space) { switch (pn_space) { - case QUICPacketNumberSpace::Initial: - return "QUICPacketNumberSpace::Initial"; - case QUICPacketNumberSpace::Handshake: - return "QUICPacketNumberSpace::Handshake"; - case QUICPacketNumberSpace::ApplicationData: - return "QUICPacketNumberSpace::ApplicationData"; + case QUICPacketNumberSpace::INITIAL: + return "INITIAL"; + case QUICPacketNumberSpace::HANDSHAKE: + return "HANDSHAKE"; + case QUICPacketNumberSpace::APPLICATION_DATA: + return "APPLCIATION_DATA"; default: return "UNKNOWN"; } diff --git a/iocore/net/quic/QUICEvents.h b/iocore/net/quic/QUICEvents.h index c758249ee93..50362e899a8 100644 --- a/iocore/net/quic/QUICEvents.h +++ b/iocore/net/quic/QUICEvents.h @@ -35,4 +35,5 @@ enum { QUIC_EVENT_ACK_PERIODIC, QUIC_EVENT_SHUTDOWN, QUIC_EVENT_LD_SHUTDOWN, + QUIC_EVENT_STATELESS_RESET, }; diff --git a/iocore/net/quic/QUICFlowController.cc b/iocore/net/quic/QUICFlowController.cc index 7c2e3db9e0b..647e4c48582 100644 --- a/iocore/net/quic/QUICFlowController.cc +++ b/iocore/net/quic/QUICFlowController.cc @@ -37,7 +37,7 @@ QUICRateAnalyzer::update(QUICOffset offset) } uint64_t -QUICRateAnalyzer::expect_recv_bytes(ink_hrtime time) +QUICRateAnalyzer::expect_recv_bytes(ink_hrtime time) const { return static_cast(time * this->_rate); } @@ -79,7 +79,7 @@ QUICFlowController::update(QUICOffset offset) void QUICFlowController::forward_limit(QUICOffset limit) { - // MAX_(STREAM_)DATA might be unorderd due to delay + // MAX_(STREAM_)DATA might be unordered due to delay // Just ignore if the size was smaller than the last one if (this->_limit > limit) { return; @@ -96,7 +96,7 @@ QUICFlowController::set_limit(QUICOffset limit) // For RemoteFlowController, caller of this function should also check QUICStreamManager::will_generate_frame() bool -QUICFlowController::will_generate_frame(QUICEncryptionLevel level, ink_hrtime timestamp) +QUICFlowController::will_generate_frame(QUICEncryptionLevel level, size_t current_packet_size, bool ack_eliciting, uint32_t seq_num) { if (!this->_is_level_matched(level)) { return false; @@ -110,7 +110,7 @@ QUICFlowController::will_generate_frame(QUICEncryptionLevel level, ink_hrtime ti */ QUICFrame * QUICFlowController::generate_frame(uint8_t *buf, QUICEncryptionLevel level, uint64_t /* connection_credit */, - uint16_t maximum_frame_size, ink_hrtime timestamp) + uint16_t maximum_frame_size, size_t current_packet_size, uint32_t seq_num) { QUICFrame *frame = nullptr; @@ -120,8 +120,18 @@ QUICFlowController::generate_frame(uint8_t *buf, QUICEncryptionLevel level, uint if (this->_should_create_frame) { frame = this->_create_frame(buf); - if (frame && frame->size() <= maximum_frame_size) { - this->_should_create_frame = false; + if (frame) { + if (frame->size() <= maximum_frame_size) { + this->_should_create_frame = false; + QUICFrameInformationUPtr info = QUICFrameInformationUPtr(quicFrameInformationAllocator.alloc()); + info->type = frame->type(); + info->level = QUICEncryptionLevel::NONE; + *(reinterpret_cast(info->data)) = this->_limit; + this->_records_frame(frame->id(), std::move(info)); + } else { + frame->~QUICFrame(); + frame = nullptr; + } } } @@ -223,48 +233,23 @@ QUICLocalFlowController::_need_to_forward_limit() QUICFrame * QUICRemoteConnectionFlowController::_create_frame(uint8_t *buf) { - auto frame = QUICFrameFactory::create_data_blocked_frame(buf, this->_offset, this->_issue_frame_id(), this); - QUICFrameInformationUPtr info = QUICFrameInformationUPtr(quicFrameInformationAllocator.alloc()); - info->type = frame->type(); - info->level = QUICEncryptionLevel::NONE; - *(reinterpret_cast(info->data)) = this->_offset; - this->_records_frame(frame->id(), std::move(info)); - return frame; + return QUICFrameFactory::create_data_blocked_frame(buf, this->_offset, this->_issue_frame_id(), this); } QUICFrame * QUICLocalConnectionFlowController::_create_frame(uint8_t *buf) { - auto frame = QUICFrameFactory::create_max_data_frame(buf, this->_limit, this->_issue_frame_id(), this); - QUICFrameInformationUPtr info = QUICFrameInformationUPtr(quicFrameInformationAllocator.alloc()); - info->type = frame->type(); - info->level = QUICEncryptionLevel::NONE; - *(reinterpret_cast(info->data)) = this->_limit; - this->_records_frame(frame->id(), std::move(info)); - return frame; + return QUICFrameFactory::create_max_data_frame(buf, this->_limit, this->_issue_frame_id(), this); } QUICFrame * QUICRemoteStreamFlowController::_create_frame(uint8_t *buf) { - auto frame = - QUICFrameFactory::create_stream_data_blocked_frame(buf, this->_stream_id, this->_offset, this->_issue_frame_id(), this); - QUICFrameInformationUPtr info = QUICFrameInformationUPtr(quicFrameInformationAllocator.alloc()); - info->type = frame->type(); - info->level = QUICEncryptionLevel::NONE; - *(reinterpret_cast(info->data)) = this->_offset; - this->_records_frame(frame->id(), std::move(info)); - return frame; + return QUICFrameFactory::create_stream_data_blocked_frame(buf, this->_stream_id, this->_offset, this->_issue_frame_id(), this); } QUICFrame * QUICLocalStreamFlowController::_create_frame(uint8_t *buf) { - auto frame = QUICFrameFactory::create_max_stream_data_frame(buf, this->_stream_id, this->_limit, this->_issue_frame_id(), this); - QUICFrameInformationUPtr info = QUICFrameInformationUPtr(quicFrameInformationAllocator.alloc()); - info->type = frame->type(); - info->level = QUICEncryptionLevel::NONE; - *(reinterpret_cast(info->data)) = this->_limit; - this->_records_frame(frame->id(), std::move(info)); - return frame; + return QUICFrameFactory::create_max_stream_data_frame(buf, this->_stream_id, this->_limit, this->_issue_frame_id(), this); } diff --git a/iocore/net/quic/QUICFlowController.h b/iocore/net/quic/QUICFlowController.h index c3cb6371139..3ff183827e6 100644 --- a/iocore/net/quic/QUICFlowController.h +++ b/iocore/net/quic/QUICFlowController.h @@ -33,7 +33,7 @@ class QUICRateAnalyzer { public: void update(QUICOffset offset); - uint64_t expect_recv_bytes(ink_hrtime time); + uint64_t expect_recv_bytes(ink_hrtime time) const; private: double _rate = 0.0; @@ -60,9 +60,9 @@ class QUICFlowController : public QUICFrameGenerator virtual void set_limit(QUICOffset limit); // QUICFrameGenerator - bool will_generate_frame(QUICEncryptionLevel level, ink_hrtime timestamp) override; + bool will_generate_frame(QUICEncryptionLevel level, size_t current_packet_size, bool ack_eliciting, uint32_t seq_num) override; QUICFrame *generate_frame(uint8_t *buf, QUICEncryptionLevel level, uint64_t connection_credit, uint16_t maximum_frame_size, - ink_hrtime timestamp) override; + size_t current_packet_size, uint32_t seq_num) override; protected: QUICFlowController(uint64_t initial_limit) : _limit(initial_limit) {} diff --git a/iocore/net/quic/QUICFrame.cc b/iocore/net/quic/QUICFrame.cc index 903c4e96679..35c14b68390 100644 --- a/iocore/net/quic/QUICFrame.cc +++ b/iocore/net/quic/QUICFrame.cc @@ -33,7 +33,7 @@ #define LEFT_SPACE(pos) ((size_t)(buf + len - pos)) #define FRAME_SIZE(pos) (pos - buf) -// the pos will auto move forward . return true if the data vaild +// the pos will auto move forward . return true if the data valid static bool read_varint(uint8_t *&pos, size_t len, uint64_t &field, size_t &field_len) { @@ -46,7 +46,7 @@ read_varint(uint8_t *&pos, size_t len, uint64_t &field, size_t &field_len) return false; } - field = QUICIntUtil::read_QUICVariableInt(pos); + field = QUICIntUtil::read_QUICVariableInt(pos, len); pos += field_len; return true; } @@ -58,6 +58,20 @@ QUICFrame::type() const return QUICFrameType::UNKNOWN; } +bool +QUICFrame::ack_eliciting() const +{ + auto type = this->type(); + + return type != QUICFrameType::PADDING && type != QUICFrameType::ACK && type != QUICFrameType::CONNECTION_CLOSE; +} + +const QUICPacketR * +QUICFrame::packet() const +{ + return this->_packet; +} + bool QUICFrame::is_probing_frame() const { @@ -98,29 +112,13 @@ QUICFrame::type(const uint8_t *buf) buf[0] < static_cast(QUICFrameType::NEW_CONNECTION_ID)) { return QUICFrameType::STREAMS_BLOCKED; } else if (static_cast(QUICFrameType::CONNECTION_CLOSE) <= buf[0] && - buf[0] < static_cast(QUICFrameType::UNKNOWN)) { + buf[0] < static_cast(QUICFrameType::HANDSHAKE_DONE)) { return QUICFrameType::CONNECTION_CLOSE; } else { return static_cast(buf[0]); } } -Ptr -QUICFrame::to_io_buffer_block(size_t limit) const -{ - // FIXME Each classes should override this and drop store(). - // This just wraps store() for now. - - Ptr block = make_ptr(new_IOBufferBlock()); - block->alloc(iobuffer_size_to_index(limit)); - - size_t written_len = 0; - this->store(reinterpret_cast(block->start()), &written_len, limit); - block->fill(written_len); - - return block; -} - int QUICFrame::debug_msg(char *msg, size_t msg_len) const { @@ -149,9 +147,9 @@ QUICStreamFrame::QUICStreamFrame(Ptr &block, QUICStreamId stream_ { } -QUICStreamFrame::QUICStreamFrame(const uint8_t *buf, size_t len) +QUICStreamFrame::QUICStreamFrame(const uint8_t *buf, size_t len, const QUICPacketR *packet) : QUICFrame(0, nullptr, packet) { - this->parse(buf, len); + this->parse(buf, len, packet); } QUICStreamFrame::QUICStreamFrame(const QUICStreamFrame &o) @@ -166,10 +164,11 @@ QUICStreamFrame::QUICStreamFrame(const QUICStreamFrame &o) } void -QUICStreamFrame::parse(const uint8_t *buf, size_t len) +QUICStreamFrame::parse(const uint8_t *buf, size_t len, const QUICPacketR *packet) { ink_assert(len >= 1); this->_reset(); + this->_packet = packet; uint8_t *pos = const_cast(buf); this->_has_offset_field = (buf[0] & 0x04) != 0; // "O" of "0b00010OLF" @@ -200,7 +199,7 @@ QUICStreamFrame::parse(const uint8_t *buf, size_t len) this->_valid = true; this->_block = make_ptr(new_IOBufferBlock()); - this->_block->alloc(); + this->_block->alloc(BUFFER_SIZE_INDEX_32K); ink_assert(static_cast(this->_block->write_avail()) > data_len); memcpy(this->_block->start(), pos, data_len); this->_block->fill(data_len); @@ -261,12 +260,6 @@ QUICStreamFrame::is_flow_controlled() const return true; } -size_t -QUICStreamFrame::store(uint8_t *buf, size_t *len, size_t limit) const -{ - return this->store(buf, len, limit, true); -} - int QUICStreamFrame::debug_msg(char *msg, size_t msg_len) const { @@ -286,7 +279,7 @@ QUICStreamFrame::to_io_buffer_block(size_t limit) const // Create header block size_t written_len = 0; header = make_ptr(new_IOBufferBlock()); - header->alloc(iobuffer_size_to_index(MAX_HEADER_SIZE)); + header->alloc(iobuffer_size_to_index(MAX_HEADER_SIZE, BUFFER_SIZE_INDEX_32K)); this->_store_header(reinterpret_cast(header->start()), &written_len, true); header->fill(written_len); @@ -333,13 +326,6 @@ QUICStreamFrame::_store_header(uint8_t *buf, size_t *len, bool include_length_fi return *len; } -size_t -QUICStreamFrame::store(uint8_t *buf, size_t *len, size_t limit, bool include_length_field) const -{ - ink_assert(!"Call to_io_buffer_block() instead"); - return 0; -} - QUICStreamId QUICStreamFrame::stream_id() const { @@ -406,9 +392,9 @@ QUICCryptoFrame::QUICCryptoFrame(Ptr &block, QUICOffset offset, Q { } -QUICCryptoFrame::QUICCryptoFrame(const uint8_t *buf, size_t len) +QUICCryptoFrame::QUICCryptoFrame(const uint8_t *buf, size_t len, const QUICPacketR *packet) : QUICFrame(0, nullptr, packet) { - this->parse(buf, len); + this->parse(buf, len, packet); } QUICCryptoFrame::QUICCryptoFrame(const QUICCryptoFrame &o) @@ -417,11 +403,12 @@ QUICCryptoFrame::QUICCryptoFrame(const QUICCryptoFrame &o) } void -QUICCryptoFrame::parse(const uint8_t *buf, size_t len) +QUICCryptoFrame::parse(const uint8_t *buf, size_t len, const QUICPacketR *packet) { ink_assert(len >= 1); this->_reset(); - uint8_t *pos = const_cast(buf) + 1; + this->_packet = packet; + uint8_t *pos = const_cast(buf) + 1; size_t field_len = 0; if (!read_varint(pos, LEFT_SPACE(pos), this->_offset, field_len)) { @@ -439,7 +426,7 @@ QUICCryptoFrame::parse(const uint8_t *buf, size_t len) this->_valid = true; this->_block = make_ptr(new_IOBufferBlock()); - this->_block->alloc(); + this->_block->alloc(BUFFER_SIZE_INDEX_32K); ink_assert(static_cast(this->_block->write_avail()) > data_len); memcpy(this->_block->start(), pos, data_len); this->_block->fill(data_len); @@ -488,14 +475,34 @@ QUICCryptoFrame::debug_msg(char *msg, size_t msg_len) const this->data_length()); } -size_t -QUICCryptoFrame::store(uint8_t *buf, size_t *len, size_t limit) const +Ptr +QUICCryptoFrame::to_io_buffer_block(size_t limit) const { + Ptr header; + if (limit < this->size()) { - return 0; + return header; } - // Frame Type + // Create header block + size_t written_len = 0; + header = make_ptr(new_IOBufferBlock()); + header->alloc(iobuffer_size_to_index(MAX_HEADER_SIZE, BUFFER_SIZE_INDEX_32K)); + this->_store_header(reinterpret_cast(header->start()), &written_len); + header->fill(written_len); + + // Append payload block to a chain + ink_assert(written_len + this->data_length() <= limit); + header->next = this->data(); + + // Return the chain + return header; +} + +size_t +QUICCryptoFrame::_store_header(uint8_t *buf, size_t *len) const +{ + // Type buf[0] = static_cast(QUICFrameType::CRYPTO); *len = 1; @@ -509,10 +516,6 @@ QUICCryptoFrame::store(uint8_t *buf, size_t *len, size_t limit) const QUICIntUtil::write_QUICVariableInt(this->data_length(), buf + *len, &n); *len += n; - // Crypto Data (*) - memcpy(buf + *len, this->data()->start(), this->data_length()); - *len += this->data_length(); - return *len; } @@ -538,18 +541,35 @@ QUICCryptoFrame::data() const // ACK frame // -QUICAckFrame::QUICAckFrame(const uint8_t *buf, size_t len) +std::set +QUICAckFrame::ranges() const { - this->parse(buf, len); + std::set numbers; + QUICPacketNumber x = this->largest_acknowledged(); + numbers.insert({x, static_cast(x) - this->ack_block_section()->first_ack_block()}); + x -= this->ack_block_section()->first_ack_block() + 1; + for (auto &&block : *(this->ack_block_section())) { + x -= block.gap() + 1; + numbers.insert({x, static_cast(x) - block.length()}); + x -= block.length() + 1; + } + + return numbers; +} + +QUICAckFrame::QUICAckFrame(const uint8_t *buf, size_t len, const QUICPacketR *packet) : QUICFrame(0, nullptr, packet) +{ + this->parse(buf, len, packet); } void -QUICAckFrame::parse(const uint8_t *buf, size_t len) +QUICAckFrame::parse(const uint8_t *buf, size_t len, const QUICPacketR *packet) { ink_assert(len >= 1); this->_reset(); - uint8_t *pos = const_cast(buf) + 1; - bool has_ecn = (buf[0] == static_cast(QUICFrameType::ACK_WITH_ECN)); + this->_packet = packet; + uint8_t *pos = const_cast(buf) + 1; + bool has_ecn = (buf[0] == static_cast(QUICFrameType::ACK_WITH_ECN)); size_t field_len = 0; if (!read_varint(pos, LEFT_SPACE(pos), this->_largest_acknowledged, field_len)) { @@ -666,33 +686,43 @@ QUICAckFrame::size() const return pre_len; } -size_t -QUICAckFrame::store(uint8_t *buf, size_t *len, size_t limit) const +Ptr +QUICAckFrame::to_io_buffer_block(size_t limit) const { + Ptr block; + size_t n = 0; + if (limit < this->size()) { - return 0; + return block; } - uint8_t *p = buf; - size_t n; - *p = static_cast(QUICFrameType::ACK); - ++p; + size_t written_len = 0; + block = make_ptr(new_IOBufferBlock()); + block->alloc(iobuffer_size_to_index(1 + 24, BUFFER_SIZE_INDEX_32K)); + uint8_t *block_start = reinterpret_cast(block->start()); - QUICIntUtil::write_QUICVariableInt(this->_largest_acknowledged, p, &n); - p += n; - QUICIntUtil::write_QUICVariableInt(this->_ack_delay, p, &n); - p += n; - QUICIntUtil::write_QUICVariableInt(this->ack_block_count(), p, &n); - p += n; + // Type + block_start[0] = static_cast(QUICFrameType::ACK); + n += 1; - ink_assert(limit >= static_cast(p - buf)); - limit -= (p - buf); - this->_ack_block_section->store(p, &n, limit); - p += n; + // Largest Acknowledged (i) + QUICIntUtil::write_QUICVariableInt(this->_largest_acknowledged, block_start + n, &written_len); + n += written_len; - *len = p - buf; + // Ack Delay (i) + QUICIntUtil::write_QUICVariableInt(this->_ack_delay, block_start + n, &written_len); + n += written_len; - return *len; + // Ack Range Count (i) + QUICIntUtil::write_QUICVariableInt(this->ack_block_count(), block_start + n, &written_len); + n += written_len; + + block->fill(n); + + // First Ack Range (i) + Ack Ranges (*) + block->next = this->_ack_block_section->to_io_buffer_block(limit - n); + + return block; } int @@ -829,29 +859,33 @@ QUICAckFrame::AckBlockSection::size() const return n; } -size_t -QUICAckFrame::AckBlockSection::store(uint8_t *buf, size_t *len, size_t limit) const +Ptr +QUICAckFrame::AckBlockSection::to_io_buffer_block(size_t limit) const { + Ptr block; + size_t n = 0; + if (limit < this->size()) { - return 0; + return block; } - size_t n; - uint8_t *p = buf; + size_t written_len = 0; + block = make_ptr(new_IOBufferBlock()); + block->alloc(iobuffer_size_to_index(limit, BUFFER_SIZE_INDEX_32K)); + uint8_t *block_start = reinterpret_cast(block->start()); - QUICIntUtil::write_QUICVariableInt(this->_first_ack_block, p, &n); - p += n; + QUICIntUtil::write_QUICVariableInt(this->_first_ack_block, block_start + n, &written_len); + n += written_len; for (auto &&block : *this) { - QUICIntUtil::write_QUICVariableInt(block.gap(), p, &n); - p += n; - QUICIntUtil::write_QUICVariableInt(block.length(), p, &n); - p += n; + QUICIntUtil::write_QUICVariableInt(block.gap(), block_start + n, &written_len); + n += written_len; + QUICIntUtil::write_QUICVariableInt(block.length(), block_start + n, &written_len); + n += written_len; } - *len = p - buf; - - return *len; + block->fill(n); + return block; } uint64_t @@ -861,7 +895,7 @@ QUICAckFrame::AckBlockSection::first_ack_block() const } void -QUICAckFrame::AckBlockSection::add_ack_block(AckBlock block) +QUICAckFrame::AckBlockSection::add_ack_block(const AckBlock &block) { this->_ack_blocks.push_back(block); } @@ -979,30 +1013,35 @@ QUICRstStreamFrame::QUICRstStreamFrame(QUICStreamId stream_id, QUICAppErrorCode { } -QUICRstStreamFrame::QUICRstStreamFrame(const uint8_t *buf, size_t len) +QUICRstStreamFrame::QUICRstStreamFrame(const uint8_t *buf, size_t len, const QUICPacketR *packet) : QUICFrame(0, nullptr, packet) { - this->parse(buf, len); + this->parse(buf, len, packet); } void -QUICRstStreamFrame::parse(const uint8_t *buf, size_t len) +QUICRstStreamFrame::parse(const uint8_t *buf, size_t len, const QUICPacketR *packet) { ink_assert(len >= 1); this->_reset(); - uint8_t *pos = 1 + const_cast(buf); + this->_packet = packet; + uint8_t *pos = 1 + const_cast(buf); size_t field_len = 0; + + // Stream ID (i) if (!read_varint(pos, LEFT_SPACE(pos), this->_stream_id, field_len)) { return; } - if (LEFT_SPACE(pos) < 2) { + // Error Code (i) + if (LEFT_SPACE(pos) < 1) { + return; + } + if (!read_varint(pos, LEFT_SPACE(pos), this->_error_code, field_len)) { return; } - this->_error_code = QUICIntUtil::read_nbytes_as_uint(pos, 2); - pos += 2; - + // Final Offset (i) if (!read_varint(pos, LEFT_SPACE(pos), this->_final_offset, field_len)) { return; } @@ -1037,36 +1076,49 @@ QUICRstStreamFrame::size() const return this->_size; } - return 1 + QUICVariableInt::size(this->_stream_id) + sizeof(QUICAppErrorCode) + QUICVariableInt::size(this->_final_offset); + return 1 + QUICVariableInt::size(this->_stream_id) + QUICVariableInt::size(this->_error_code) + + QUICVariableInt::size(this->_final_offset); } -size_t -QUICRstStreamFrame::store(uint8_t *buf, size_t *len, size_t limit) const +Ptr +QUICRstStreamFrame::to_io_buffer_block(size_t limit) const { + Ptr block; + size_t n = 0; + if (limit < this->size()) { - return 0; + return block; } - size_t n; - uint8_t *p = buf; - *p = static_cast(QUICFrameType::RESET_STREAM); - ++p; - QUICTypeUtil::write_QUICStreamId(this->_stream_id, p, &n); - p += n; - QUICTypeUtil::write_QUICAppErrorCode(this->_error_code, p, &n); - p += n; - QUICTypeUtil::write_QUICOffset(this->_final_offset, p, &n); - p += n; - - *len = p - buf; + size_t written_len = 0; + block = make_ptr(new_IOBufferBlock()); + block->alloc(iobuffer_size_to_index(1 + 24, BUFFER_SIZE_INDEX_32K)); + uint8_t *block_start = reinterpret_cast(block->start()); - return *len; + // Type + block_start[0] = static_cast(QUICFrameType::RESET_STREAM); + n += 1; + + // Stream ID (i) + QUICTypeUtil::write_QUICStreamId(this->_stream_id, block_start + n, &written_len); + n += written_len; + + // Application Error Code (i) + QUICTypeUtil::write_QUICAppErrorCode(this->_error_code, block_start + n, &written_len); + n += written_len; + + // Final Size (i) + QUICTypeUtil::write_QUICOffset(this->_final_offset, block_start + n, &written_len); + n += written_len; + + block->fill(n); + return block; } int QUICRstStreamFrame::debug_msg(char *msg, size_t msg_len) const { - return snprintf(msg, msg_len, "RESET_STREAM size=%zu stream_id=%" PRIu64 " code=0x%" PRIx16, this->size(), this->stream_id(), + return snprintf(msg, msg_len, "RESET_STREAM size=%zu stream_id=%" PRIu64 " code=0x%" PRIx64, this->size(), this->stream_id(), this->error_code()); } @@ -1092,17 +1144,18 @@ QUICRstStreamFrame::final_offset() const // PING frame // -QUICPingFrame::QUICPingFrame(const uint8_t *buf, size_t len) +QUICPingFrame::QUICPingFrame(const uint8_t *buf, size_t len, const QUICPacketR *packet) : QUICFrame(0, nullptr, packet) { - this->parse(buf, len); + this->parse(buf, len, packet); } void -QUICPingFrame::parse(const uint8_t *buf, size_t len) +QUICPingFrame::parse(const uint8_t *buf, size_t len, const QUICPacketR *packet) { this->_reset(); - this->_valid = true; - this->_size = 1; + this->_packet = packet; + this->_valid = true; + this->_size = 1; } QUICFrameType @@ -1117,33 +1170,52 @@ QUICPingFrame::size() const return 1; } -size_t -QUICPingFrame::store(uint8_t *buf, size_t *len, size_t limit) const +Ptr +QUICPingFrame::to_io_buffer_block(size_t limit) const { + Ptr block; + size_t n = 0; + if (limit < this->size()) { - return 0; + return block; } - *len = this->size(); - buf[0] = static_cast(QUICFrameType::PING); - return *len; + block = make_ptr(new_IOBufferBlock()); + block->alloc(iobuffer_size_to_index(this->size(), BUFFER_SIZE_INDEX_32K)); + uint8_t *block_start = reinterpret_cast(block->start()); + + // Type + block_start[0] = static_cast(QUICFrameType::PING); + n += 1; + + block->fill(n); + return block; } // // PADDING frame // -QUICPaddingFrame::QUICPaddingFrame(const uint8_t *buf, size_t len) +QUICPaddingFrame::QUICPaddingFrame(const uint8_t *buf, size_t len, const QUICPacketR *packet) : QUICFrame(0, nullptr, packet) { - this->parse(buf, len); + this->parse(buf, len, packet); } void -QUICPaddingFrame::parse(const uint8_t *buf, size_t len) +QUICPaddingFrame::parse(const uint8_t *buf, size_t len, const QUICPacketR *packet) { ink_assert(len >= 1); this->_reset(); - this->_valid = true; - this->_size = 1; + this->_packet = packet; + this->_size = 0; + this->_valid = true; + // find out how many padding frames in this buf + for (size_t i = 0; i < len; i++) { + if (*(buf + i) == static_cast(QUICFrameType::PADDING)) { + ++this->_size; + } else { + break; + } + } } QUICFrameType @@ -1155,7 +1227,7 @@ QUICPaddingFrame::type() const size_t QUICPaddingFrame::size() const { - return 1; + return this->_size; } bool @@ -1164,22 +1236,31 @@ QUICPaddingFrame::is_probing_frame() const return true; } -size_t -QUICPaddingFrame::store(uint8_t *buf, size_t *len, size_t limit) const +Ptr +QUICPaddingFrame::to_io_buffer_block(size_t limit) const { + Ptr block; + size_t n = 0; + if (limit < this->size()) { - return 0; + return block; } - buf[0] = static_cast(QUICFrameType::PADDING); - *len = 1; - return *len; + block = make_ptr(new_IOBufferBlock()); + block->alloc(iobuffer_size_to_index(this->_size, BUFFER_SIZE_INDEX_32K)); + uint8_t *block_start = reinterpret_cast(block->start()); + + memset(block_start, 0, this->_size); + n = this->_size; + + block->fill(n); + return block; } // // CONNECTION_CLOSE frame // -QUICConnectionCloseFrame::QUICConnectionCloseFrame(uint16_t error_code, QUICFrameType frame_type, uint64_t reason_phrase_length, +QUICConnectionCloseFrame::QUICConnectionCloseFrame(uint64_t error_code, QUICFrameType frame_type, uint64_t reason_phrase_length, const char *reason_phrase, QUICFrameId id, QUICFrameGenerator *owner) : QUICFrame(id, owner), _type(0x1c), @@ -1190,7 +1271,7 @@ QUICConnectionCloseFrame::QUICConnectionCloseFrame(uint16_t error_code, QUICFram { } -QUICConnectionCloseFrame::QUICConnectionCloseFrame(uint16_t error_code, uint64_t reason_phrase_length, const char *reason_phrase, +QUICConnectionCloseFrame::QUICConnectionCloseFrame(uint64_t error_code, uint64_t reason_phrase_length, const char *reason_phrase, QUICFrameId id, QUICFrameGenerator *owner) : QUICFrame(id, owner), _type(0x1d), @@ -1200,9 +1281,10 @@ QUICConnectionCloseFrame::QUICConnectionCloseFrame(uint16_t error_code, uint64_t { } -QUICConnectionCloseFrame::QUICConnectionCloseFrame(const uint8_t *buf, size_t len) +QUICConnectionCloseFrame::QUICConnectionCloseFrame(const uint8_t *buf, size_t len, const QUICPacketR *packet) + : QUICFrame(0, nullptr, packet) { - this->parse(buf, len); + this->parse(buf, len, packet); } void @@ -1220,28 +1302,29 @@ QUICConnectionCloseFrame::_reset() } void -QUICConnectionCloseFrame::parse(const uint8_t *buf, size_t len) +QUICConnectionCloseFrame::parse(const uint8_t *buf, size_t len, const QUICPacketR *packet) { ink_assert(len >= 1); this->_reset(); - this->_type = buf[0]; - uint8_t *pos = const_cast(buf) + 1; - - if (LEFT_SPACE(pos) < 2) { - return; - } - - this->_error_code = QUICIntUtil::read_nbytes_as_uint(pos, 2); - pos += 2; + this->_packet = packet; + this->_type = buf[0]; + uint8_t *pos = const_cast(buf) + 1; size_t field_len = 0; uint64_t field = 0; + // Error Code (i) + if (LEFT_SPACE(pos) < 1) { + return; + } + read_varint(pos, LEFT_SPACE(pos), field, field_len); + this->_error_code = field; + if (this->_type == 0x1c) { + // Frame Type (i) if (!read_varint(pos, LEFT_SPACE(pos), field, field_len)) { return; } - this->_frame_type = static_cast(field); /** @@ -1255,16 +1338,21 @@ QUICConnectionCloseFrame::parse(const uint8_t *buf, size_t len) } } + // Reason Phrase Length (i) + if (LEFT_SPACE(pos) < 1) { + return; + } if (!read_varint(pos, LEFT_SPACE(pos), this->_reason_phrase_length, field_len)) { return; } + // Reason Phrase if (LEFT_SPACE(pos) < this->_reason_phrase_length) { return; } - - this->_valid = true; this->_reason_phrase = reinterpret_cast(pos); + + this->_valid = true; pos += this->_reason_phrase_length; this->_size = FRAME_SIZE(pos); } @@ -1282,7 +1370,7 @@ QUICConnectionCloseFrame::size() const return this->_size; } - return 1 + sizeof(QUICTransErrorCode) + QUICVariableInt::size(sizeof(QUICFrameType)) + + return 1 + QUICVariableInt::size(sizeof(QUICTransErrorCode)) + QUICVariableInt::size(sizeof(QUICFrameType)) + QUICVariableInt::size(this->_reason_phrase_length) + this->_reason_phrase_length; } @@ -1292,42 +1380,58 @@ QUICConnectionCloseFrame::size() const PADDING frame in Frame Type field means frame type that triggered the error is unknown. When `_frame_type` is QUICFrameType::UNKNOWN, it's converted to QUICFrameType::PADDING (0x0). */ -size_t -QUICConnectionCloseFrame::store(uint8_t *buf, size_t *len, size_t limit) const +Ptr +QUICConnectionCloseFrame::to_io_buffer_block(size_t limit) const { + Ptr first_block; + size_t n = 0; + if (limit < this->size()) { - return 0; + return first_block; } - size_t n; - uint8_t *p = buf; - *p = this->_type; - ++p; + // Create a block for Error Code(i) and Frame Type(i) + size_t written_len = 0; + first_block = make_ptr(new_IOBufferBlock()); + first_block->alloc(iobuffer_size_to_index(1 + 24, BUFFER_SIZE_INDEX_32K)); + uint8_t *block_start = reinterpret_cast(first_block->start()); + + // Type + block_start[0] = this->_type; + n += 1; - // Error Code (16) - QUICTypeUtil::write_QUICTransErrorCode(this->_error_code, p, &n); - p += n; + // Error Code (i) + QUICIntUtil::write_QUICVariableInt(this->_error_code, block_start + n, &written_len); + n += written_len; // Frame Type (i) QUICFrameType frame_type = this->_frame_type; if (frame_type == QUICFrameType::UNKNOWN) { frame_type = QUICFrameType::PADDING; } - *p = static_cast(frame_type); - ++p; + QUICIntUtil::write_QUICVariableInt(static_cast(frame_type), block_start + n, &written_len); + n += written_len; // Reason Phrase Length (i) - QUICIntUtil::write_QUICVariableInt(this->_reason_phrase_length, p, &n); - p += n; + QUICIntUtil::write_QUICVariableInt(this->_reason_phrase_length, block_start + n, &written_len); + n += written_len; + + first_block->fill(n); - // Reason Phrase (*) - if (this->_reason_phrase_length > 0) { - memcpy(p, this->_reason_phrase, this->_reason_phrase_length); - p += this->_reason_phrase_length; + // Create a block for reason if necessary + if (this->_reason_phrase_length != 0) { + // Reason Phrase (*) + Ptr reason_block = make_ptr(new_IOBufferBlock()); + reason_block->alloc(iobuffer_size_to_index(this->_reason_phrase_length, BUFFER_SIZE_INDEX_32K)); + memcpy(reinterpret_cast(reason_block->start()), this->_reason_phrase, this->_reason_phrase_length); + reason_block->fill(this->_reason_phrase_length); + + // Append reason block to the first block + first_block->next = reason_block; } - *len = p - buf; - return *len; + // Return the chain + return first_block; } int @@ -1400,17 +1504,18 @@ QUICMaxDataFrame::_reset() this->_size = 0; } -QUICMaxDataFrame::QUICMaxDataFrame(const uint8_t *buf, size_t len) +QUICMaxDataFrame::QUICMaxDataFrame(const uint8_t *buf, size_t len, const QUICPacketR *packet) : QUICFrame(0, nullptr, packet) { - this->parse(buf, len); + this->parse(buf, len, packet); } void -QUICMaxDataFrame::parse(const uint8_t *buf, size_t len) +QUICMaxDataFrame::parse(const uint8_t *buf, size_t len, const QUICPacketR *packet) { ink_assert(len >= 1); this->_reset(); - uint8_t *pos = 1 + const_cast(buf); + this->_packet = packet; + uint8_t *pos = 1 + const_cast(buf); size_t field_len = 0; if (!read_varint(pos, LEFT_SPACE(pos), this->_maximum_data, field_len)) { @@ -1437,22 +1542,31 @@ QUICMaxDataFrame::size() const return sizeof(QUICFrameType) + QUICVariableInt::size(this->_maximum_data); } -size_t -QUICMaxDataFrame::store(uint8_t *buf, size_t *len, size_t limit) const +Ptr +QUICMaxDataFrame::to_io_buffer_block(size_t limit) const { + Ptr block; + size_t n = 0; + if (limit < this->size()) { - return 0; + return block; } - size_t n; - uint8_t *p = buf; - *p = static_cast(QUICFrameType::MAX_DATA); - ++p; - QUICTypeUtil::write_QUICMaxData(this->_maximum_data, p, &n); - p += n; + size_t written_len = 0; + block = make_ptr(new_IOBufferBlock()); + block->alloc(iobuffer_size_to_index(1 + sizeof(size_t), BUFFER_SIZE_INDEX_32K)); + uint8_t *block_start = reinterpret_cast(block->start()); - *len = p - buf; - return *len; + // Type + block_start[0] = static_cast(QUICFrameType::MAX_DATA); + n += 1; + + // Maximum Data (i) + QUICTypeUtil::write_QUICMaxData(this->_maximum_data, block_start + n, &written_len); + n += written_len; + + block->fill(n); + return block; } int @@ -1490,17 +1604,19 @@ QUICMaxStreamDataFrame::_reset() this->_size = 0; } -QUICMaxStreamDataFrame::QUICMaxStreamDataFrame(const uint8_t *buf, size_t len) +QUICMaxStreamDataFrame::QUICMaxStreamDataFrame(const uint8_t *buf, size_t len, const QUICPacketR *packet) + : QUICFrame(0, nullptr, packet) { - this->parse(buf, len); + this->parse(buf, len, packet); } void -QUICMaxStreamDataFrame::parse(const uint8_t *buf, size_t len) +QUICMaxStreamDataFrame::parse(const uint8_t *buf, size_t len, const QUICPacketR *packet) { ink_assert(len >= 1); this->_reset(); - uint8_t *pos = const_cast(buf) + 1; + this->_packet = packet; + uint8_t *pos = const_cast(buf) + 1; size_t field_len = 0; if (!read_varint(pos, LEFT_SPACE(pos), this->_stream_id, field_len)) { @@ -1531,23 +1647,35 @@ QUICMaxStreamDataFrame::size() const return sizeof(QUICFrameType) + QUICVariableInt::size(this->_maximum_stream_data) + QUICVariableInt::size(this->_stream_id); } -size_t -QUICMaxStreamDataFrame::store(uint8_t *buf, size_t *len, size_t limit) const +Ptr +QUICMaxStreamDataFrame::to_io_buffer_block(size_t limit) const { + Ptr block; + size_t n = 0; + if (limit < this->size()) { - return 0; + return block; } - size_t n; - uint8_t *p = buf; - *p = static_cast(QUICFrameType::MAX_STREAM_DATA); - ++p; - QUICTypeUtil::write_QUICStreamId(this->_stream_id, p, &n); - p += n; - QUICTypeUtil::write_QUICMaxData(this->_maximum_stream_data, p, &n); - p += n; - - *len = p - buf; - return *len; + + size_t written_len = 0; + block = make_ptr(new_IOBufferBlock()); + block->alloc(iobuffer_size_to_index(1 + sizeof(uint64_t) + sizeof(size_t), BUFFER_SIZE_INDEX_32K)); + uint8_t *block_start = reinterpret_cast(block->start()); + + // Type + block_start[0] = static_cast(QUICFrameType::MAX_STREAM_DATA); + n += 1; + + // Stream ID (i) + QUICTypeUtil::write_QUICStreamId(this->_stream_id, block_start + n, &written_len); + n += written_len; + + // Maximum Stream Data (i) + QUICTypeUtil::write_QUICMaxData(this->_maximum_stream_data, block_start + n, &written_len); + n += written_len; + + block->fill(n); + return block; } int @@ -1589,17 +1717,18 @@ QUICMaxStreamsFrame::_reset() this->_size = 0; } -QUICMaxStreamsFrame::QUICMaxStreamsFrame(const uint8_t *buf, size_t len) +QUICMaxStreamsFrame::QUICMaxStreamsFrame(const uint8_t *buf, size_t len, const QUICPacketR *packet) : QUICFrame(0, nullptr, packet) { - this->parse(buf, len); + this->parse(buf, len, packet); } void -QUICMaxStreamsFrame::parse(const uint8_t *buf, size_t len) +QUICMaxStreamsFrame::parse(const uint8_t *buf, size_t len, const QUICPacketR *packet) { ink_assert(len >= 1); this->_reset(); - uint8_t *pos = const_cast(buf) + 1; + this->_packet = packet; + uint8_t *pos = const_cast(buf) + 1; size_t field_len = 0; if (!read_varint(pos, LEFT_SPACE(pos), this->_maximum_streams, field_len)) { @@ -1626,22 +1755,31 @@ QUICMaxStreamsFrame::size() const return sizeof(QUICFrameType) + QUICVariableInt::size(this->_maximum_streams); } -size_t -QUICMaxStreamsFrame::store(uint8_t *buf, size_t *len, size_t limit) const +Ptr +QUICMaxStreamsFrame::to_io_buffer_block(size_t limit) const { + Ptr block; + size_t n = 0; + if (limit < this->size()) { - return 0; + return block; } - size_t n; - uint8_t *p = buf; - *p = static_cast(QUICFrameType::MAX_STREAMS); - ++p; - QUICTypeUtil::write_QUICStreamId(this->_maximum_streams, p, &n); - p += n; + size_t written_len = 0; + block = make_ptr(new_IOBufferBlock()); + block->alloc(iobuffer_size_to_index(1 + sizeof(size_t), BUFFER_SIZE_INDEX_32K)); + uint8_t *block_start = reinterpret_cast(block->start()); - *len = p - buf; - return *len; + // Type + block_start[0] = static_cast(QUICFrameType::MAX_STREAMS); + n += 1; + + // Maximum Streams (i) + QUICTypeUtil::write_QUICStreamId(this->_maximum_streams, block_start + n, &written_len); + n += written_len; + + block->fill(n); + return block; } uint64_t @@ -1653,9 +1791,10 @@ QUICMaxStreamsFrame::maximum_streams() const // // DATA_BLOCKED frame // -QUICDataBlockedFrame::QUICDataBlockedFrame(const uint8_t *buf, size_t len) +QUICDataBlockedFrame::QUICDataBlockedFrame(const uint8_t *buf, size_t len, const QUICPacketR *packet) + : QUICFrame(0, nullptr, packet) { - this->parse(buf, len); + this->parse(buf, len, packet); } void @@ -1670,11 +1809,12 @@ QUICDataBlockedFrame::_reset() } void -QUICDataBlockedFrame::parse(const uint8_t *buf, size_t len) +QUICDataBlockedFrame::parse(const uint8_t *buf, size_t len, const QUICPacketR *packet) { ink_assert(len >= 1); this->_reset(); - uint8_t *pos = const_cast(buf) + 1; + this->_packet = packet; + uint8_t *pos = const_cast(buf) + 1; size_t field_len = 0; if (!read_varint(pos, LEFT_SPACE(pos), this->_offset, field_len)) { @@ -1707,24 +1847,31 @@ QUICDataBlockedFrame::size() const return sizeof(QUICFrameType) + QUICVariableInt::size(this->offset()); } -size_t -QUICDataBlockedFrame::store(uint8_t *buf, size_t *len, size_t limit) const +Ptr +QUICDataBlockedFrame::to_io_buffer_block(size_t limit) const { + Ptr block; + size_t n = 0; + if (limit < this->size()) { - return 0; + return block; } - size_t n; - uint8_t *p = buf; + size_t written_len = 0; + block = make_ptr(new_IOBufferBlock()); + block->alloc(iobuffer_size_to_index(1 + sizeof(size_t), BUFFER_SIZE_INDEX_32K)); + uint8_t *block_start = reinterpret_cast(block->start()); - *p = static_cast(QUICFrameType::DATA_BLOCKED); - ++p; - QUICTypeUtil::write_QUICOffset(this->_offset, p, &n); - p += n; + // Type + block_start[0] = static_cast(QUICFrameType::DATA_BLOCKED); + n += 1; - *len = p - buf; + // Data Limit (i) + QUICTypeUtil::write_QUICOffset(this->_offset, block_start + n, &written_len); + n += written_len; - return *len; + block->fill(n); + return block; } QUICOffset @@ -1736,9 +1883,10 @@ QUICDataBlockedFrame::offset() const // // STREAM_DATA_BLOCKED frame // -QUICStreamDataBlockedFrame::QUICStreamDataBlockedFrame(const uint8_t *buf, size_t len) +QUICStreamDataBlockedFrame::QUICStreamDataBlockedFrame(const uint8_t *buf, size_t len, const QUICPacketR *packet) + : QUICFrame(0, nullptr, packet) { - this->parse(buf, len); + this->parse(buf, len, packet); } void @@ -1754,11 +1902,12 @@ QUICStreamDataBlockedFrame::_reset() } void -QUICStreamDataBlockedFrame::parse(const uint8_t *buf, size_t len) +QUICStreamDataBlockedFrame::parse(const uint8_t *buf, size_t len, const QUICPacketR *packet) { ink_assert(len >= 1); this->_reset(); - uint8_t *pos = const_cast(buf) + 1; + this->_packet = packet; + uint8_t *pos = const_cast(buf) + 1; size_t field_len = 0; if (!read_varint(pos, LEFT_SPACE(pos), this->_stream_id, field_len)) { @@ -1796,25 +1945,35 @@ QUICStreamDataBlockedFrame::size() const return sizeof(QUICFrameType) + QUICVariableInt::size(this->_offset) + QUICVariableInt::size(this->_stream_id); } -size_t -QUICStreamDataBlockedFrame::store(uint8_t *buf, size_t *len, size_t limit) const +Ptr +QUICStreamDataBlockedFrame::to_io_buffer_block(size_t limit) const { + Ptr block; + size_t n = 0; + if (limit < this->size()) { - return 0; + return block; } - size_t n; - uint8_t *p = buf; - *p = static_cast(QUICFrameType::STREAM_DATA_BLOCKED); - ++p; - QUICTypeUtil::write_QUICStreamId(this->_stream_id, p, &n); - p += n; - QUICTypeUtil::write_QUICOffset(this->_offset, p, &n); - p += n; + size_t written_len = 0; + block = make_ptr(new_IOBufferBlock()); + block->alloc(iobuffer_size_to_index(1 + sizeof(size_t), BUFFER_SIZE_INDEX_32K)); + uint8_t *block_start = reinterpret_cast(block->start()); - *len = p - buf; + // Type + block_start[0] = static_cast(QUICFrameType::STREAM_DATA_BLOCKED); + n += 1; - return *len; + // Stream ID (i) + QUICTypeUtil::write_QUICStreamId(this->_stream_id, block_start + n, &written_len); + n += written_len; + + // Data Limit (i) + QUICTypeUtil::write_QUICOffset(this->_offset, block_start + n, &written_len); + n += written_len; + + block->fill(n); + return block; } QUICStreamId @@ -1832,9 +1991,10 @@ QUICStreamDataBlockedFrame::offset() const // // STREAMS_BLOCKED frame // -QUICStreamIdBlockedFrame::QUICStreamIdBlockedFrame(const uint8_t *buf, size_t len) +QUICStreamIdBlockedFrame::QUICStreamIdBlockedFrame(const uint8_t *buf, size_t len, const QUICPacketR *packet) + : QUICFrame(0, nullptr, packet) { - this->parse(buf, len); + this->parse(buf, len, packet); } void @@ -1849,11 +2009,12 @@ QUICStreamIdBlockedFrame::_reset() } void -QUICStreamIdBlockedFrame::parse(const uint8_t *buf, size_t len) +QUICStreamIdBlockedFrame::parse(const uint8_t *buf, size_t len, const QUICPacketR *packet) { ink_assert(len >= 1); this->_reset(); - uint8_t *pos = const_cast(buf) + 1; + this->_packet = packet; + uint8_t *pos = const_cast(buf) + 1; size_t field_len = 0; if (!read_varint(pos, LEFT_SPACE(pos), this->_stream_id, field_len)) { @@ -1880,23 +2041,31 @@ QUICStreamIdBlockedFrame::size() const return sizeof(QUICFrameType) + QUICVariableInt::size(this->_stream_id); } -size_t -QUICStreamIdBlockedFrame::store(uint8_t *buf, size_t *len, size_t limit) const +Ptr +QUICStreamIdBlockedFrame::to_io_buffer_block(size_t limit) const { + Ptr block; + size_t n = 0; + if (limit < this->size()) { - return 0; + return block; } - size_t n; - uint8_t *p = buf; + size_t written_len = 0; + block = make_ptr(new_IOBufferBlock()); + block->alloc(iobuffer_size_to_index(1 + sizeof(size_t), BUFFER_SIZE_INDEX_32K)); + uint8_t *block_start = reinterpret_cast(block->start()); - *p = static_cast(QUICFrameType::STREAMS_BLOCKED); - ++p; - QUICTypeUtil::write_QUICStreamId(this->_stream_id, p, &n); - p += n; + // Type + block_start[0] = static_cast(QUICFrameType::STREAMS_BLOCKED); + n += 1; - *len = p - buf; - return *len; + // Stream Limit (i) + QUICTypeUtil::write_QUICStreamId(this->_stream_id, block_start + n, &written_len); + n += written_len; + + block->fill(n); + return block; } QUICStreamId @@ -1908,16 +2077,18 @@ QUICStreamIdBlockedFrame::stream_id() const // // NEW_CONNECTION_ID frame // -QUICNewConnectionIdFrame::QUICNewConnectionIdFrame(const uint8_t *buf, size_t len) +QUICNewConnectionIdFrame::QUICNewConnectionIdFrame(const uint8_t *buf, size_t len, const QUICPacketR *packet) + : QUICFrame(0, nullptr, packet) { - this->parse(buf, len); + this->parse(buf, len, packet); } void QUICNewConnectionIdFrame::_reset() { - this->_sequence = 0; - this->_connection_id = QUICConnectionId::ZERO(); + this->_sequence = 0; + this->_retire_prior_to = 0; + this->_connection_id = QUICConnectionId::ZERO(); this->_owner = nullptr; this->_id = 0; @@ -1926,38 +2097,49 @@ QUICNewConnectionIdFrame::_reset() } void -QUICNewConnectionIdFrame::parse(const uint8_t *buf, size_t len) +QUICNewConnectionIdFrame::parse(const uint8_t *buf, size_t len, const QUICPacketR *packet) { ink_assert(len >= 1); this->_reset(); - uint8_t *pos = const_cast(buf) + 1; + this->_packet = packet; + uint8_t *pos = const_cast(buf) + 1; + // Sequence Number (i) size_t field_len = 0; if (!read_varint(pos, LEFT_SPACE(pos), this->_sequence, field_len)) { return; } + // Retire Prior To (i) if (LEFT_SPACE(pos) < 1) { return; } + if (!read_varint(pos, LEFT_SPACE(pos), this->_retire_prior_to, field_len)) { + return; + } + // Length (8) + if (LEFT_SPACE(pos) < 1) { + return; + } size_t cid_len = *pos; pos += 1; + // Connection ID (8..160) if (LEFT_SPACE(pos) < cid_len) { return; } - this->_connection_id = QUICTypeUtil::read_QUICConnectionId(pos, cid_len); pos += cid_len; - if (LEFT_SPACE(pos) < 16) { + // Stateless Reset Token (128) + if (LEFT_SPACE(pos) < QUICStatelessResetToken::LEN) { return; } this->_stateless_reset_token = QUICStatelessResetToken(pos); this->_valid = true; - this->_size = FRAME_SIZE(pos) + 16; + this->_size = FRAME_SIZE(pos) + QUICStatelessResetToken::LEN; } QUICFrameType @@ -1973,40 +2155,62 @@ QUICNewConnectionIdFrame::size() const return this->_size; } - return sizeof(QUICFrameType) + QUICVariableInt::size(this->_sequence) + 1 + this->_connection_id.length() + 16; + return sizeof(QUICFrameType) + QUICVariableInt::size(this->_sequence) + QUICVariableInt::size(this->_retire_prior_to) + 1 + + this->_connection_id.length() + QUICStatelessResetToken::LEN; } -size_t -QUICNewConnectionIdFrame::store(uint8_t *buf, size_t *len, size_t limit) const +Ptr +QUICNewConnectionIdFrame::to_io_buffer_block(size_t limit) const { + Ptr block; + size_t n = 0; + if (limit < this->size()) { - return 0; + return block; } - size_t n; - uint8_t *p = buf; - *p = static_cast(QUICFrameType::NEW_CONNECTION_ID); - ++p; - QUICIntUtil::write_QUICVariableInt(this->_sequence, p, &n); - p += n; - *p = this->_connection_id.length(); - p += 1; - QUICTypeUtil::write_QUICConnectionId(this->_connection_id, p, &n); - p += n; - memcpy(p, this->_stateless_reset_token.buf(), QUICStatelessResetToken::LEN); - p += QUICStatelessResetToken::LEN; - - *len = p - buf; - return *len; + size_t written_len = 0; + block = make_ptr(new_IOBufferBlock()); + block->alloc(iobuffer_size_to_index(1 + sizeof(uint64_t) + sizeof(uint64_t) + 1 + QUICConnectionId::MAX_LENGTH + + QUICStatelessResetToken::LEN, + BUFFER_SIZE_INDEX_32K)); + uint8_t *block_start = reinterpret_cast(block->start()); + + // Type + block_start[0] = static_cast(QUICFrameType::NEW_CONNECTION_ID); + n += 1; + + // Sequence Number (i) + QUICIntUtil::write_QUICVariableInt(this->_sequence, block_start + n, &written_len); + n += written_len; + + // Retire Prior To (i) + QUICIntUtil::write_QUICVariableInt(this->_retire_prior_to, block_start + n, &written_len); + n += written_len; + + // Length (8) + *(block_start + n) = this->_connection_id.length(); + n += 1; + + // Connection ID (8..160) + QUICTypeUtil::write_QUICConnectionId(this->_connection_id, block_start + n, &written_len); + n += written_len; + + // Stateless Reset Token (128) + memcpy(block_start + n, this->_stateless_reset_token.buf(), QUICStatelessResetToken::LEN); + n += QUICStatelessResetToken::LEN; + + block->fill(n); + return block; } int QUICNewConnectionIdFrame::debug_msg(char *msg, size_t msg_len) const { - char cid_str[QUICConnectionId::MAX_HEX_STR_LENGTH]; - this->connection_id().hex(cid_str, QUICConnectionId::MAX_HEX_STR_LENGTH); - - return snprintf(msg, msg_len, "NEW_CONNECTION_ID size=%zu seq=%" PRIu64 " cid=0x%s", this->size(), this->sequence(), cid_str); + return snprintf(msg, msg_len, "NEW_CONNECTION_ID size=%zu seq=%" PRIu64 " rpt=%" PRIu64 " cid=0x%s srt=%02x%02x%02x%02x", + this->size(), this->sequence(), this->retire_prior_to(), this->connection_id().hex().c_str(), + this->stateless_reset_token().buf()[0], this->stateless_reset_token().buf()[1], + this->stateless_reset_token().buf()[2], this->stateless_reset_token().buf()[3]); } uint64_t @@ -2015,6 +2219,12 @@ QUICNewConnectionIdFrame::sequence() const return this->_sequence; } +uint64_t +QUICNewConnectionIdFrame::retire_prior_to() const +{ + return this->_retire_prior_to; +} + QUICConnectionId QUICNewConnectionIdFrame::connection_id() const { @@ -2049,30 +2259,36 @@ QUICStopSendingFrame::_reset() this->_size = 0; } -QUICStopSendingFrame::QUICStopSendingFrame(const uint8_t *buf, size_t len) +QUICStopSendingFrame::QUICStopSendingFrame(const uint8_t *buf, size_t len, const QUICPacketR *packet) + : QUICFrame(0, nullptr, packet) { - this->parse(buf, len); + this->parse(buf, len, packet); } void -QUICStopSendingFrame::parse(const uint8_t *buf, size_t len) +QUICStopSendingFrame::parse(const uint8_t *buf, size_t len, const QUICPacketR *packet) { ink_assert(len >= 1); this->_reset(); - uint8_t *pos = const_cast(buf) + 1; + this->_packet = packet; + uint8_t *pos = const_cast(buf) + 1; + // Stream ID (i) size_t field_len = 0; if (!read_varint(pos, LEFT_SPACE(pos), this->_stream_id, field_len)) { return; } - if (LEFT_SPACE(pos) < 2) { + // Error Code (i) + if (LEFT_SPACE(pos) < 1) { + return; + } + if (!read_varint(pos, LEFT_SPACE(pos), this->_error_code, field_len)) { return; } - this->_error_code = static_cast(QUICIntUtil::read_nbytes_as_uint(pos, 2)); - this->_valid = true; - this->_size = FRAME_SIZE(pos) + 2; + this->_valid = true; + this->_size = FRAME_SIZE(pos); } QUICFrameType @@ -2088,27 +2304,38 @@ QUICStopSendingFrame::size() const return this->_size; } - return sizeof(QUICFrameType) + QUICVariableInt::size(this->_stream_id) + sizeof(QUICAppErrorCode); + return sizeof(QUICFrameType) + QUICVariableInt::size(this->_stream_id) + QUICVariableInt::size(sizeof(QUICAppErrorCode)); } -size_t -QUICStopSendingFrame::store(uint8_t *buf, size_t *len, size_t limit) const +Ptr +QUICStopSendingFrame::to_io_buffer_block(size_t limit) const { + Ptr block; + size_t n = 0; + if (limit < this->size()) { - return 0; + return block; } - size_t n; - uint8_t *p = buf; - *p = static_cast(QUICFrameType::STOP_SENDING); - ++p; - QUICTypeUtil::write_QUICStreamId(this->_stream_id, p, &n); - p += n; - QUICTypeUtil::write_QUICAppErrorCode(this->_error_code, p, &n); - p += n; - - *len = p - buf; - return *len; + size_t written_len = 0; + block = make_ptr(new_IOBufferBlock()); + block->alloc(iobuffer_size_to_index(1 + 24, BUFFER_SIZE_INDEX_32K)); + uint8_t *block_start = reinterpret_cast(block->start()); + + // Type + block_start[0] = static_cast(QUICFrameType::STOP_SENDING); + n += 1; + + // Stream ID (i) + QUICTypeUtil::write_QUICStreamId(this->_stream_id, block_start + n, &written_len); + n += written_len; + + // Application Error Code (i) + QUICTypeUtil::write_QUICAppErrorCode(this->_error_code, block_start + n, &written_len); + n += written_len; + + block->fill(n); + return block; } QUICAppErrorCode @@ -2126,9 +2353,10 @@ QUICStopSendingFrame::stream_id() const // // PATH_CHALLENGE frame // -QUICPathChallengeFrame::QUICPathChallengeFrame(const uint8_t *buf, size_t len) +QUICPathChallengeFrame::QUICPathChallengeFrame(const uint8_t *buf, size_t len, const QUICPacketR *packet) + : QUICFrame(0, nullptr, packet) { - this->parse(buf, len); + this->parse(buf, len, packet); } void @@ -2142,11 +2370,12 @@ QUICPathChallengeFrame::_reset() } void -QUICPathChallengeFrame::parse(const uint8_t *buf, size_t len) +QUICPathChallengeFrame::parse(const uint8_t *buf, size_t len, const QUICPacketR *packet) { ink_assert(len >= 1); this->_reset(); - uint8_t *pos = const_cast(buf) + 1; + this->_packet = packet; + uint8_t *pos = const_cast(buf) + 1; if (LEFT_SPACE(pos) < QUICPathChallengeFrame::DATA_LEN) { return; @@ -2180,19 +2409,38 @@ QUICPathChallengeFrame::is_probing_frame() const return true; } -size_t -QUICPathChallengeFrame::store(uint8_t *buf, size_t *len, size_t limit) const +Ptr +QUICPathChallengeFrame::to_io_buffer_block(size_t limit) const { + Ptr block; + size_t n = 0; + if (limit < this->size()) { - return 0; + return block; } - *len = this->size(); + block = make_ptr(new_IOBufferBlock()); + block->alloc(iobuffer_size_to_index(1 + QUICPathChallengeFrame::DATA_LEN, BUFFER_SIZE_INDEX_32K)); + uint8_t *block_start = reinterpret_cast(block->start()); - buf[0] = static_cast(QUICFrameType::PATH_CHALLENGE); - memcpy(buf + 1, this->data(), QUICPathChallengeFrame::DATA_LEN); + // Type + block_start[0] = static_cast(QUICFrameType::PATH_CHALLENGE); + n += 1; - return *len; + // Data (64) + memcpy(block_start + n, this->data(), QUICPathChallengeFrame::DATA_LEN); + n += QUICPathChallengeFrame::DATA_LEN; + + block->fill(n); + return block; +} + +int +QUICPathChallengeFrame::debug_msg(char *msg, size_t msg_len) const +{ + auto data = this->data(); + return snprintf(msg, msg_len, "PATH_CHALLENGE size=%zu data=0x%02x%02x%02x%02x%02x%02x%02x%02x", this->size(), data[0], data[1], + data[2], data[3], data[4], data[5], data[6], data[7]); } const uint8_t * @@ -2204,9 +2452,10 @@ QUICPathChallengeFrame::data() const // // PATH_RESPONSE frame // -QUICPathResponseFrame::QUICPathResponseFrame(const uint8_t *buf, size_t len) +QUICPathResponseFrame::QUICPathResponseFrame(const uint8_t *buf, size_t len, const QUICPacketR *packet) + : QUICFrame(0, nullptr, packet) { - this->parse(buf, len); + this->parse(buf, len, packet); } void @@ -2219,12 +2468,39 @@ QUICPathResponseFrame::_reset() this->_size = 0; } +Ptr +QUICPathResponseFrame::to_io_buffer_block(size_t limit) const +{ + Ptr block; + size_t n = 0; + + if (limit < this->size()) { + return block; + } + + block = make_ptr(new_IOBufferBlock()); + block->alloc(iobuffer_size_to_index(1 + QUICPathResponseFrame::DATA_LEN, BUFFER_SIZE_INDEX_32K)); + uint8_t *block_start = reinterpret_cast(block->start()); + + // Type + block_start[0] = static_cast(QUICFrameType::PATH_RESPONSE); + n += 1; + + // Data (64) + memcpy(block_start + n, this->data(), QUICPathChallengeFrame::DATA_LEN); + n += QUICPathChallengeFrame::DATA_LEN; + + block->fill(n); + return block; +} + void -QUICPathResponseFrame::parse(const uint8_t *buf, size_t len) +QUICPathResponseFrame::parse(const uint8_t *buf, size_t len, const QUICPacketR *packet) { ink_assert(len >= 1); this->_reset(); - uint8_t *pos = const_cast(buf) + 1; + this->_packet = packet; + uint8_t *pos = const_cast(buf) + 1; if (LEFT_SPACE(pos) < QUICPathChallengeFrame::DATA_LEN) { return; @@ -2254,19 +2530,12 @@ QUICPathResponseFrame::is_probing_frame() const return true; } -size_t -QUICPathResponseFrame::store(uint8_t *buf, size_t *len, size_t limit) const +int +QUICPathResponseFrame::debug_msg(char *msg, size_t msg_len) const { - if (limit < this->size()) { - return 0; - } - - *len = this->size(); - - buf[0] = static_cast(QUICFrameType::PATH_RESPONSE); - memcpy(buf + 1, this->data(), QUICPathResponseFrame::DATA_LEN); - - return *len; + auto data = this->data(); + return snprintf(msg, msg_len, "PATH_RESPONSE size=%zu data=0x%02x%02x%02x%02x%02x%02x%02x%02x", this->size(), data[0], data[1], + data[2], data[3], data[4], data[5], data[6], data[7]); } const uint8_t * @@ -2278,9 +2547,9 @@ QUICPathResponseFrame::data() const // // QUICNewTokenFrame // -QUICNewTokenFrame::QUICNewTokenFrame(const uint8_t *buf, size_t len) +QUICNewTokenFrame::QUICNewTokenFrame(const uint8_t *buf, size_t len, const QUICPacketR *packet) : QUICFrame(0, nullptr, packet) { - this->parse(buf, len); + this->parse(buf, len, packet); } void @@ -2296,11 +2565,12 @@ QUICNewTokenFrame::_reset() } void -QUICNewTokenFrame::parse(const uint8_t *buf, size_t len) +QUICNewTokenFrame::parse(const uint8_t *buf, size_t len, const QUICPacketR *packet) { ink_assert(len >= 1); this->_reset(); - uint8_t *pos = const_cast(buf) + 1; + this->_packet = packet; + uint8_t *pos = const_cast(buf) + 1; size_t field_len = 0; if (!read_varint(pos, LEFT_SPACE(pos), this->_token_length, field_len)) { @@ -2333,30 +2603,35 @@ QUICNewTokenFrame::size() const return 1 + QUICVariableInt::size(this->_token_length) + this->token_length(); } -size_t -QUICNewTokenFrame::store(uint8_t *buf, size_t *len, size_t limit) const +Ptr +QUICNewTokenFrame::to_io_buffer_block(size_t limit) const { + Ptr block; + size_t n = 0; + if (limit < this->size()) { - return 0; + return block; } - uint8_t *p = buf; + size_t written_len = 0; + block = make_ptr(new_IOBufferBlock()); + block->alloc(iobuffer_size_to_index(1 + 24, BUFFER_SIZE_INDEX_32K)); + uint8_t *block_start = reinterpret_cast(block->start()); - // Type (i) - *p = static_cast(QUICFrameType::NEW_TOKEN); - ++p; + // Type + block_start[0] = static_cast(QUICFrameType::NEW_TOKEN); + n += 1; // Token Length (i) - size_t n; - QUICIntUtil::write_QUICVariableInt(this->_token_length, p, &n); - p += n; + QUICIntUtil::write_QUICVariableInt(this->_token_length, block_start + n, &written_len); + n += written_len; // Token (*) - memcpy(p, this->token(), this->token_length()); - p += this->token_length(); + memcpy(block_start + n, this->token(), this->token_length()); + n += this->token_length(); - *len = p - buf; - return *len; + block->fill(n); + return block; } uint64_t @@ -2374,9 +2649,10 @@ QUICNewTokenFrame::token() const // // RETIRE_CONNECTION_ID frame // -QUICRetireConnectionIdFrame::QUICRetireConnectionIdFrame(const uint8_t *buf, size_t len) +QUICRetireConnectionIdFrame::QUICRetireConnectionIdFrame(const uint8_t *buf, size_t len, const QUICPacketR *packet) + : QUICFrame(0, nullptr, packet) { - this->parse(buf, len); + this->parse(buf, len, packet); } void @@ -2392,11 +2668,12 @@ QUICRetireConnectionIdFrame::_reset() } void -QUICRetireConnectionIdFrame::parse(const uint8_t *buf, size_t len) +QUICRetireConnectionIdFrame::parse(const uint8_t *buf, size_t len, const QUICPacketR *packet) { ink_assert(len >= 1); this->_reset(); - uint8_t *pos = const_cast(buf) + 1; + this->_packet = packet; + uint8_t *pos = const_cast(buf) + 1; size_t field_len = 0; if (!read_varint(pos, LEFT_SPACE(pos), this->_seq_num, field_len)) { @@ -2423,23 +2700,31 @@ QUICRetireConnectionIdFrame::size() const return sizeof(QUICFrameType) + QUICVariableInt::size(this->_seq_num); } -size_t -QUICRetireConnectionIdFrame::store(uint8_t *buf, size_t *len, size_t limit) const +Ptr +QUICRetireConnectionIdFrame::to_io_buffer_block(size_t limit) const { + Ptr block; + size_t n = 0; + if (limit < this->size()) { - return 0; + return block; } - size_t n; - uint8_t *p = buf; - *p = static_cast(QUICFrameType::RETIRE_CONNECTION_ID); - ++p; - QUICIntUtil::write_QUICVariableInt(this->_seq_num, p, &n); - p += n; + size_t written_len = 0; + block = make_ptr(new_IOBufferBlock()); + block->alloc(iobuffer_size_to_index(1 + sizeof(uint64_t), BUFFER_SIZE_INDEX_32K)); + uint8_t *block_start = reinterpret_cast(block->start()); - *len = p - buf; + // Type + block_start[0] = static_cast(QUICFrameType::RETIRE_CONNECTION_ID); + n += 1; - return *len; + // Sequence Number (i) + QUICIntUtil::write_QUICVariableInt(this->_seq_num, block_start + n, &written_len); + n += written_len; + + block->fill(n); + return block; } int @@ -2454,6 +2739,59 @@ QUICRetireConnectionIdFrame::seq_num() const return this->_seq_num; } +// +// HANDSHAKE_DONE frame +// + +QUICHandshakeDoneFrame::QUICHandshakeDoneFrame(const uint8_t *buf, size_t len, const QUICPacketR *packet) + : QUICFrame(0, nullptr, packet) +{ + this->parse(buf, len, packet); +} + +void +QUICHandshakeDoneFrame::parse(const uint8_t *buf, size_t len, const QUICPacketR *packet) +{ + this->_reset(); + this->_packet = packet; + this->_valid = true; + this->_size = 1; +} + +QUICFrameType +QUICHandshakeDoneFrame::type() const +{ + return QUICFrameType::HANDSHAKE_DONE; +} + +size_t +QUICHandshakeDoneFrame::size() const +{ + return 1; +} + +Ptr +QUICHandshakeDoneFrame::to_io_buffer_block(size_t limit) const +{ + Ptr block; + size_t n = 0; + + if (limit < this->size()) { + return block; + } + + block = make_ptr(new_IOBufferBlock()); + block->alloc(iobuffer_size_to_index(this->size(), BUFFER_SIZE_INDEX_32K)); + uint8_t *block_start = reinterpret_cast(block->start()); + + // Type + block_start[0] = static_cast(QUICFrameType::HANDSHAKE_DONE); + n += 1; + + block->fill(n); + return block; +} + // // UNKNOWN // @@ -2470,15 +2808,17 @@ QUICUnknownFrame::size() const return 0; } -size_t -QUICUnknownFrame::store(uint8_t *buf, size_t *len, size_t limit) const +Ptr +QUICUnknownFrame::to_io_buffer_block(size_t limit) const { - return 0; + Ptr block; + return block; } void -QUICUnknownFrame::parse(const uint8_t *buf, size_t len) +QUICUnknownFrame::parse(const uint8_t *buf, size_t len, const QUICPacketR *packet) { + this->_packet = packet; } int @@ -2492,65 +2832,68 @@ QUICUnknownFrame::debug_msg(char *msg, size_t msg_len) const // QUICFrame * -QUICFrameFactory::create(uint8_t *buf, const uint8_t *src, size_t len) +QUICFrameFactory::create(uint8_t *buf, const uint8_t *src, size_t len, const QUICPacketR *packet) { switch (QUICFrame::type(src)) { case QUICFrameType::STREAM: - new (buf) QUICStreamFrame(src, len); + new (buf) QUICStreamFrame(src, len, packet); return reinterpret_cast(buf); case QUICFrameType::CRYPTO: - new (buf) QUICCryptoFrame(src, len); + new (buf) QUICCryptoFrame(src, len, packet); return reinterpret_cast(buf); case QUICFrameType::ACK: - new (buf) QUICAckFrame(src, len); + new (buf) QUICAckFrame(src, len, packet); return reinterpret_cast(buf); case QUICFrameType::PADDING: - new (buf) QUICPaddingFrame(src, len); + new (buf) QUICPaddingFrame(src, len, packet); return reinterpret_cast(buf); case QUICFrameType::RESET_STREAM: - new (buf) QUICRstStreamFrame(src, len); + new (buf) QUICRstStreamFrame(src, len, packet); return reinterpret_cast(buf); case QUICFrameType::CONNECTION_CLOSE: - new (buf) QUICConnectionCloseFrame(src, len); + new (buf) QUICConnectionCloseFrame(src, len, packet); return reinterpret_cast(buf); case QUICFrameType::MAX_DATA: - new (buf) QUICMaxDataFrame(src, len); + new (buf) QUICMaxDataFrame(src, len, packet); return reinterpret_cast(buf); case QUICFrameType::MAX_STREAM_DATA: - new (buf) QUICMaxStreamDataFrame(src, len); + new (buf) QUICMaxStreamDataFrame(src, len, packet); return reinterpret_cast(buf); case QUICFrameType::MAX_STREAMS: - new (buf) QUICMaxStreamsFrame(src, len); + new (buf) QUICMaxStreamsFrame(src, len, packet); return reinterpret_cast(buf); case QUICFrameType::PING: - new (buf) QUICPingFrame(src, len); + new (buf) QUICPingFrame(src, len, packet); return reinterpret_cast(buf); case QUICFrameType::DATA_BLOCKED: - new (buf) QUICDataBlockedFrame(src, len); + new (buf) QUICDataBlockedFrame(src, len, packet); return reinterpret_cast(buf); case QUICFrameType::STREAM_DATA_BLOCKED: - new (buf) QUICStreamDataBlockedFrame(src, len); + new (buf) QUICStreamDataBlockedFrame(src, len, packet); return reinterpret_cast(buf); case QUICFrameType::STREAMS_BLOCKED: - new (buf) QUICStreamIdBlockedFrame(src, len); + new (buf) QUICStreamIdBlockedFrame(src, len, packet); return reinterpret_cast(buf); case QUICFrameType::NEW_CONNECTION_ID: - new (buf) QUICNewConnectionIdFrame(src, len); + new (buf) QUICNewConnectionIdFrame(src, len, packet); return reinterpret_cast(buf); case QUICFrameType::STOP_SENDING: - new (buf) QUICStopSendingFrame(src, len); + new (buf) QUICStopSendingFrame(src, len, packet); return reinterpret_cast(buf); case QUICFrameType::PATH_CHALLENGE: - new (buf) QUICPathChallengeFrame(src, len); + new (buf) QUICPathChallengeFrame(src, len, packet); return reinterpret_cast(buf); case QUICFrameType::PATH_RESPONSE: - new (buf) QUICPathResponseFrame(src, len); + new (buf) QUICPathResponseFrame(src, len, packet); return reinterpret_cast(buf); case QUICFrameType::NEW_TOKEN: - new (buf) QUICNewTokenFrame(src, len); + new (buf) QUICNewTokenFrame(src, len, packet); return reinterpret_cast(buf); case QUICFrameType::RETIRE_CONNECTION_ID: - new (buf) QUICRetireConnectionIdFrame(src, len); + new (buf) QUICRetireConnectionIdFrame(src, len, packet); + return reinterpret_cast(buf); + case QUICFrameType::HANDSHAKE_DONE: + new (buf) QUICHandshakeDoneFrame(src, len, packet); return reinterpret_cast(buf); default: // Unknown frame @@ -2560,7 +2903,7 @@ QUICFrameFactory::create(uint8_t *buf, const uint8_t *src, size_t len) } const QUICFrame & -QUICFrameFactory::fast_create(const uint8_t *buf, size_t len) +QUICFrameFactory::fast_create(const uint8_t *buf, size_t len, const QUICPacketR *packet) { if (QUICFrame::type(buf) == QUICFrameType::UNKNOWN) { return this->_unknown_frame; @@ -2570,12 +2913,12 @@ QUICFrameFactory::fast_create(const uint8_t *buf, size_t len) QUICFrame *frame = this->_reusable_frames[type_index]; if (frame == nullptr) { - frame = QUICFrameFactory::create(this->_buf_for_fast_create + (type_index * QUICFrame::MAX_INSTANCE_SIZE), buf, len); + frame = QUICFrameFactory::create(this->_buf_for_fast_create + (type_index * QUICFrame::MAX_INSTANCE_SIZE), buf, len, packet); if (frame != nullptr) { this->_reusable_frames[static_cast(QUICFrame::type(buf))] = frame; } } else { - frame->parse(buf, len); + frame->parse(buf, len, packet); } return *frame; @@ -2617,7 +2960,8 @@ QUICFrameFactory::create_connection_close_frame(uint8_t *buf, uint16_t error_cod } QUICConnectionCloseFrame * -QUICFrameFactory::create_connection_close_frame(uint8_t *buf, QUICConnectionError &error, QUICFrameId id, QUICFrameGenerator *owner) +QUICFrameFactory::create_connection_close_frame(uint8_t *buf, const QUICConnectionError &error, QUICFrameId id, + QUICFrameGenerator *owner) { ink_assert(error.cls == QUICErrorClass::TRANSPORT); if (error.msg) { @@ -2656,6 +3000,13 @@ QUICFrameFactory::create_ping_frame(uint8_t *buf, QUICFrameId id, QUICFrameGener return reinterpret_cast(buf); } +QUICPaddingFrame * +QUICFrameFactory::create_padding_frame(uint8_t *buf, size_t size, QUICFrameId id, QUICFrameGenerator *owner) +{ + new (buf) QUICPaddingFrame(size); + return reinterpret_cast(buf); +} + QUICPathChallengeFrame * QUICFrameFactory::create_path_challenge_frame(uint8_t *buf, const uint8_t *data, QUICFrameId id, QUICFrameGenerator *owner) { @@ -2721,11 +3072,11 @@ QUICFrameFactory::create_stop_sending_frame(uint8_t *buf, QUICStreamId stream_id } QUICNewConnectionIdFrame * -QUICFrameFactory::create_new_connection_id_frame(uint8_t *buf, uint32_t sequence, QUICConnectionId connectoin_id, - QUICStatelessResetToken stateless_reset_token, QUICFrameId id, - QUICFrameGenerator *owner) +QUICFrameFactory::create_new_connection_id_frame(uint8_t *buf, uint64_t sequence, uint64_t retire_prior_to, + QUICConnectionId connection_id, QUICStatelessResetToken stateless_reset_token, + QUICFrameId id, QUICFrameGenerator *owner) { - new (buf) QUICNewConnectionIdFrame(sequence, connectoin_id, stateless_reset_token, id, owner); + new (buf) QUICNewConnectionIdFrame(sequence, retire_prior_to, connection_id, stateless_reset_token, id, owner); return reinterpret_cast(buf); } @@ -2747,14 +3098,9 @@ QUICFrameFactory::create_retire_connection_id_frame(uint8_t *buf, uint64_t seq_n return reinterpret_cast(buf); } -QUICFrameId -QUICFrameInfo::id() const -{ - return this->_id; -} - -QUICFrameGenerator * -QUICFrameInfo::generated_by() const +QUICHandshakeDoneFrame * +QUICFrameFactory::create_handshake_done_frame(uint8_t *buf, QUICFrameId id, QUICFrameGenerator *owner) { - return this->_generator; + new (buf) QUICHandshakeDoneFrame(id, owner); + return reinterpret_cast(buf); } diff --git a/iocore/net/quic/QUICFrame.h b/iocore/net/quic/QUICFrame.h index 20f1bfe035d..93cd692fc7c 100644 --- a/iocore/net/quic/QUICFrame.h +++ b/iocore/net/quic/QUICFrame.h @@ -30,17 +30,16 @@ #include "I_IOBuffer.h" #include #include +#include #include "QUICTypes.h" class QUICFrame; class QUICStreamFrame; class QUICCryptoFrame; -class QUICPacket; +class QUICPacketR; class QUICFrameGenerator; -using QUICFrameId = uint64_t; - class QUICFrame { public: @@ -55,21 +54,26 @@ class QUICFrame virtual size_t size() const = 0; virtual bool is_probing_frame() const; virtual bool is_flow_controlled() const; - virtual size_t store(uint8_t *buf, size_t *len, size_t limit) const = 0; - virtual Ptr to_io_buffer_block(size_t limit) const; + virtual Ptr to_io_buffer_block(size_t limit) const = 0; virtual int debug_msg(char *msg, size_t msg_len) const; - virtual void parse(const uint8_t *buf, size_t len){}; + virtual void parse(const uint8_t *buf, size_t len, const QUICPacketR *packet){}; virtual QUICFrameGenerator *generated_by(); bool valid() const; + bool ack_eliciting() const; + const QUICPacketR *packet() const; LINK(QUICFrame, link); protected: virtual void _reset(){}; - QUICFrame(QUICFrameId id = 0, QUICFrameGenerator *owner = nullptr) : _id(id), _owner(owner) {} + QUICFrame(QUICFrameId id = 0, QUICFrameGenerator *owner = nullptr, const QUICPacketR *packet = nullptr) + : _id(id), _owner(owner), _packet(packet) + { + } size_t _size = 0; bool _valid = false; QUICFrameId _id = 0; QUICFrameGenerator *_owner = nullptr; + const QUICPacketR *_packet = nullptr; }; // @@ -80,7 +84,7 @@ class QUICStreamFrame : public QUICFrame { public: QUICStreamFrame(QUICFrameId id = 0, QUICFrameGenerator *owner = nullptr) : QUICFrame(id, owner) {} - QUICStreamFrame(const uint8_t *buf, size_t len); + QUICStreamFrame(const uint8_t *buf, size_t len, const QUICPacketR *packet = nullptr); QUICStreamFrame(Ptr &block, QUICStreamId streamid, QUICOffset offset, bool last = false, bool has_offset_field = true, bool has_length_field = true, QUICFrameId id = 0, QUICFrameGenerator *owner = nullptr); @@ -89,12 +93,10 @@ class QUICStreamFrame : public QUICFrame virtual QUICFrameType type() const override; virtual size_t size() const override; virtual bool is_flow_controlled() const override; - virtual size_t store(uint8_t *buf, size_t *len, size_t limit) const override; virtual Ptr to_io_buffer_block(size_t limit) const override; virtual int debug_msg(char *msg, size_t msg_len) const override; - virtual void parse(const uint8_t *buf, size_t len) override; + virtual void parse(const uint8_t *buf, size_t len, const QUICPacketR *packet) override; - size_t store(uint8_t *buf, size_t *len, size_t limit, bool include_length_field) const; QUICStreamId stream_id() const; QUICOffset offset() const; IOBufferBlock *data() const; @@ -128,15 +130,15 @@ class QUICCryptoFrame : public QUICFrame { public: QUICCryptoFrame(QUICFrameId id = 0, QUICFrameGenerator *owner = nullptr) : QUICFrame(id, owner) {} - QUICCryptoFrame(const uint8_t *buf, size_t len); + QUICCryptoFrame(const uint8_t *buf, size_t len, const QUICPacketR *packet = nullptr); QUICCryptoFrame(Ptr &block, QUICOffset offset, QUICFrameId id = 0, QUICFrameGenerator *owner = nullptr); QUICCryptoFrame(const QUICCryptoFrame &o); virtual QUICFrameType type() const override; virtual size_t size() const override; - virtual size_t store(uint8_t *buf, size_t *len, size_t limit) const override; + virtual Ptr to_io_buffer_block(size_t limit) const override; virtual int debug_msg(char *msg, size_t msg_len) const override; - virtual void parse(const uint8_t *buf, size_t len) override; + virtual void parse(const uint8_t *buf, size_t len, const QUICPacketR *packet) override; QUICOffset offset() const; uint64_t data_length() const; @@ -145,8 +147,12 @@ class QUICCryptoFrame : public QUICFrame LINK(QUICCryptoFrame, link); private: + static constexpr uint8_t MAX_HEADER_SIZE = 16; + virtual void _reset() override; + size_t _store_header(uint8_t *buf, size_t *len) const; + QUICOffset _offset = 0; Ptr _block; }; @@ -203,8 +209,16 @@ class QUICAckFrame : public QUICFrame public: const_iterator(uint8_t index, const std::vector *ack_blocks); - const QUICAckFrame::AckBlock &operator*() const { return this->_current_block; }; - const QUICAckFrame::AckBlock *operator->() const { return &this->_current_block; }; + const QUICAckFrame::AckBlock & + operator*() const + { + return this->_current_block; + }; + const QUICAckFrame::AckBlock * + operator->() const + { + return &this->_current_block; + }; const QUICAckFrame::AckBlock &operator++(); const bool operator!=(const const_iterator &ite) const; const bool operator==(const const_iterator &ite) const; @@ -218,9 +232,9 @@ class QUICAckFrame : public QUICFrame AckBlockSection(uint64_t first_ack_block) : _first_ack_block(first_ack_block) {} uint8_t count() const; size_t size() const; - size_t store(uint8_t *buf, size_t *len, size_t limit) const; + Ptr to_io_buffer_block(size_t limit) const; uint64_t first_ack_block() const; - void add_ack_block(const AckBlock block); + void add_ack_block(const AckBlock &block); const_iterator begin() const; const_iterator end() const; bool has_protected() const; @@ -250,18 +264,19 @@ class QUICAckFrame : public QUICFrame }; QUICAckFrame(QUICFrameId id = 0) : QUICFrame(id) {} - QUICAckFrame(const uint8_t *buf, size_t len); + QUICAckFrame(const uint8_t *buf, size_t len, const QUICPacketR *packet = nullptr); QUICAckFrame(QUICPacketNumber largest_acknowledged, uint64_t ack_delay, uint64_t first_ack_block, QUICFrameId id = 0, QUICFrameGenerator *owner = nullptr); + std::set ranges() const; - // There's no reasont restrict copy, but we need to write the copy constructor. Otherwise it will crash on destruct. + // There's no reason to restrict copy, but we need to write the copy constructor. Otherwise it will crash on destruct. QUICAckFrame(const QUICAckFrame &) = delete; virtual ~QUICAckFrame(); virtual QUICFrameType type() const override; virtual size_t size() const override; - virtual size_t store(uint8_t *buf, size_t *len, size_t limit) const override; - virtual void parse(const uint8_t *buf, size_t len) override; + virtual Ptr to_io_buffer_block(size_t limit) const override; + virtual void parse(const uint8_t *buf, size_t len, const QUICPacketR *packet) override; virtual int debug_msg(char *msg, size_t msg_len) const override; QUICPacketNumber largest_acknowledged() const; @@ -289,15 +304,15 @@ class QUICRstStreamFrame : public QUICFrame { public: QUICRstStreamFrame(QUICFrameId id = 0) : QUICFrame(id) {} - QUICRstStreamFrame(const uint8_t *buf, size_t len); + QUICRstStreamFrame(const uint8_t *buf, size_t len, const QUICPacketR *packet = nullptr); QUICRstStreamFrame(QUICStreamId stream_id, QUICAppErrorCode error_code, QUICOffset final_offset, QUICFrameId id = 0, QUICFrameGenerator *owner = nullptr); virtual QUICFrameType type() const override; virtual size_t size() const override; - virtual size_t store(uint8_t *buf, size_t *len, size_t limit) const override; + virtual Ptr to_io_buffer_block(size_t limit) const override; virtual int debug_msg(char *msg, size_t msg_len) const override; - virtual void parse(const uint8_t *buf, size_t len) override; + virtual void parse(const uint8_t *buf, size_t len, const QUICPacketR *packet) override; QUICStreamId stream_id() const; QUICAppErrorCode error_code() const; @@ -319,11 +334,11 @@ class QUICPingFrame : public QUICFrame { public: QUICPingFrame(QUICFrameId id = 0, QUICFrameGenerator *owner = nullptr) : QUICFrame(id, owner) {} - QUICPingFrame(const uint8_t *buf, size_t len); + QUICPingFrame(const uint8_t *buf, size_t len, const QUICPacketR *packet = nullptr); virtual QUICFrameType type() const override; virtual size_t size() const override; - virtual size_t store(uint8_t *buf, size_t *len, size_t limit) const override; - virtual void parse(const uint8_t *buf, size_t len) override; + virtual Ptr to_io_buffer_block(size_t limit) const override; + virtual void parse(const uint8_t *buf, size_t len, const QUICPacketR *packet) override; private: }; @@ -335,13 +350,18 @@ class QUICPingFrame : public QUICFrame class QUICPaddingFrame : public QUICFrame { public: - QUICPaddingFrame() {} - QUICPaddingFrame(const uint8_t *buf, size_t len); + QUICPaddingFrame(size_t size) : _size(size) {} + QUICPaddingFrame(const uint8_t *buf, size_t len, const QUICPacketR *packet = nullptr); virtual QUICFrameType type() const override; virtual size_t size() const override; virtual bool is_probing_frame() const override; - virtual size_t store(uint8_t *buf, size_t *len, size_t limit) const override; - virtual void parse(const uint8_t *buf, size_t len) override; + virtual Ptr to_io_buffer_block(size_t limit) const override; + virtual void parse(const uint8_t *buf, size_t len, const QUICPacketR *packet) override; + +private: + // padding frame is a resident of padding frames + // size indicate how many padding frames in this QUICPaddingFrame + size_t _size = 0; }; // @@ -352,18 +372,18 @@ class QUICConnectionCloseFrame : public QUICFrame { public: QUICConnectionCloseFrame(QUICFrameId id = 0, QUICFrameGenerator *owner = nullptr) : QUICFrame(id, owner) {} - QUICConnectionCloseFrame(const uint8_t *buf, size_t len); + QUICConnectionCloseFrame(const uint8_t *buf, size_t len, const QUICPacketR *packet = nullptr); // Constructor for transport error codes - QUICConnectionCloseFrame(uint16_t error_code, QUICFrameType frame_type, uint64_t reason_phrase_length, const char *reason_phrase, + QUICConnectionCloseFrame(uint64_t error_code, QUICFrameType frame_type, uint64_t reason_phrase_length, const char *reason_phrase, QUICFrameId id = 0, QUICFrameGenerator *owner = nullptr); // Constructor for application protocol error codes - QUICConnectionCloseFrame(uint16_t error_code, uint64_t reason_phrase_length, const char *reason_phrase, QUICFrameId id = 0, + QUICConnectionCloseFrame(uint64_t error_code, uint64_t reason_phrase_length, const char *reason_phrase, QUICFrameId id = 0, QUICFrameGenerator *owner = nullptr); virtual QUICFrameType type() const override; virtual size_t size() const override; - virtual size_t store(uint8_t *buf, size_t *len, size_t limit) const override; + virtual Ptr to_io_buffer_block(size_t limit) const override; virtual int debug_msg(char *msg, size_t msg_len) const override; - virtual void parse(const uint8_t *buf, size_t len) override; + virtual void parse(const uint8_t *buf, size_t len, const QUICPacketR *packet) override; uint16_t error_code() const; QUICFrameType frame_type() const; @@ -374,7 +394,7 @@ class QUICConnectionCloseFrame : public QUICFrame virtual void _reset() override; uint8_t _type = 0; - uint16_t _error_code; + uint64_t _error_code; QUICFrameType _frame_type = QUICFrameType::UNKNOWN; uint64_t _reason_phrase_length = 0; const char *_reason_phrase = nullptr; @@ -388,13 +408,13 @@ class QUICMaxDataFrame : public QUICFrame { public: QUICMaxDataFrame(QUICFrameId id = 0, QUICFrameGenerator *owner = nullptr) : QUICFrame(id, owner) {} - QUICMaxDataFrame(const uint8_t *buf, size_t len); + QUICMaxDataFrame(const uint8_t *buf, size_t len, const QUICPacketR *packet = nullptr); QUICMaxDataFrame(uint64_t maximum_data, QUICFrameId id = 0, QUICFrameGenerator *owner = nullptr); virtual QUICFrameType type() const override; virtual size_t size() const override; - virtual size_t store(uint8_t *buf, size_t *len, size_t limit) const override; + virtual Ptr to_io_buffer_block(size_t limit) const override; virtual int debug_msg(char *msg, size_t msg_len) const override; - virtual void parse(const uint8_t *buf, size_t len) override; + virtual void parse(const uint8_t *buf, size_t len, const QUICPacketR *packet) override; uint64_t maximum_data() const; @@ -412,13 +432,13 @@ class QUICMaxStreamDataFrame : public QUICFrame { public: QUICMaxStreamDataFrame(QUICFrameId id = 0, QUICFrameGenerator *owner = nullptr) : QUICFrame(id, owner) {} - QUICMaxStreamDataFrame(const uint8_t *buf, size_t len); + QUICMaxStreamDataFrame(const uint8_t *buf, size_t len, const QUICPacketR *packet = nullptr); QUICMaxStreamDataFrame(QUICStreamId stream_id, uint64_t maximum_stream_data, QUICFrameId id = 0, QUICFrameGenerator *owner = nullptr); virtual QUICFrameType type() const override; virtual size_t size() const override; - virtual void parse(const uint8_t *buf, size_t len) override; - virtual size_t store(uint8_t *buf, size_t *len, size_t limit) const override; + virtual void parse(const uint8_t *buf, size_t len, const QUICPacketR *packet) override; + virtual Ptr to_io_buffer_block(size_t limit) const override; virtual int debug_msg(char *msg, size_t msg_len) const override; QUICStreamId stream_id() const; @@ -439,12 +459,12 @@ class QUICMaxStreamsFrame : public QUICFrame { public: QUICMaxStreamsFrame(QUICFrameId id = 0, QUICFrameGenerator *owner = nullptr) : QUICFrame(id, owner) {} - QUICMaxStreamsFrame(const uint8_t *buf, size_t len); + QUICMaxStreamsFrame(const uint8_t *buf, size_t len, const QUICPacketR *packet = nullptr); QUICMaxStreamsFrame(QUICStreamId maximum_streams, QUICFrameId id = 0, QUICFrameGenerator *owner = nullptr); virtual QUICFrameType type() const override; virtual size_t size() const override; - virtual size_t store(uint8_t *buf, size_t *len, size_t limit) const override; - virtual void parse(const uint8_t *buf, size_t len) override; + virtual Ptr to_io_buffer_block(size_t limit) const override; + virtual void parse(const uint8_t *buf, size_t len, const QUICPacketR *packet) override; uint64_t maximum_streams() const; private: @@ -460,15 +480,15 @@ class QUICDataBlockedFrame : public QUICFrame { public: QUICDataBlockedFrame(QUICFrameId id = 0, QUICFrameGenerator *owner = nullptr) : QUICFrame(id, owner) {} - QUICDataBlockedFrame(const uint8_t *buf, size_t len); + QUICDataBlockedFrame(const uint8_t *buf, size_t len, const QUICPacketR *packet = nullptr); QUICDataBlockedFrame(QUICOffset offset, QUICFrameId id = 0, QUICFrameGenerator *owner = nullptr) : QUICFrame(id, owner), _offset(offset){}; virtual QUICFrameType type() const override; virtual size_t size() const override; - virtual void parse(const uint8_t *buf, size_t len) override; + virtual void parse(const uint8_t *buf, size_t len, const QUICPacketR *packet) override; virtual int debug_msg(char *msg, size_t msg_len) const override; - virtual size_t store(uint8_t *buf, size_t *len, size_t limit) const override; + virtual Ptr to_io_buffer_block(size_t limit) const override; QUICOffset offset() const; @@ -486,14 +506,14 @@ class QUICStreamDataBlockedFrame : public QUICFrame { public: QUICStreamDataBlockedFrame(QUICFrameId id = 0, QUICFrameGenerator *owner = nullptr) : QUICFrame(id, owner) {} - QUICStreamDataBlockedFrame(const uint8_t *buf, size_t len); + QUICStreamDataBlockedFrame(const uint8_t *buf, size_t len, const QUICPacketR *packet = nullptr); QUICStreamDataBlockedFrame(QUICStreamId s, QUICOffset o, QUICFrameId id = 0, QUICFrameGenerator *owner = nullptr) : QUICFrame(id, owner), _stream_id(s), _offset(o){}; virtual QUICFrameType type() const override; virtual size_t size() const override; - virtual size_t store(uint8_t *buf, size_t *len, size_t limit) const override; - virtual void parse(const uint8_t *buf, size_t len) override; + virtual Ptr to_io_buffer_block(size_t limit) const override; + virtual void parse(const uint8_t *buf, size_t len, const QUICPacketR *packet) override; virtual int debug_msg(char *msg, size_t msg_len) const override; QUICStreamId stream_id() const; @@ -513,15 +533,15 @@ class QUICStreamIdBlockedFrame : public QUICFrame { public: QUICStreamIdBlockedFrame(QUICFrameId id = 0, QUICFrameGenerator *owner = nullptr) : QUICFrame(id, owner) {} - QUICStreamIdBlockedFrame(const uint8_t *buf, size_t len); + QUICStreamIdBlockedFrame(const uint8_t *buf, size_t len, const QUICPacketR *packet = nullptr); QUICStreamIdBlockedFrame(QUICStreamId s, QUICFrameId id = 0, QUICFrameGenerator *owner = nullptr) : QUICFrame(id, owner), _stream_id(s) { } virtual QUICFrameType type() const override; virtual size_t size() const override; - virtual size_t store(uint8_t *buf, size_t *len, size_t limit) const override; - virtual void parse(const uint8_t *buf, size_t len) override; + virtual Ptr to_io_buffer_block(size_t limit) const override; + virtual void parse(const uint8_t *buf, size_t len, const QUICPacketR *packet) override; QUICStreamId stream_id() const; @@ -539,18 +559,19 @@ class QUICNewConnectionIdFrame : public QUICFrame { public: QUICNewConnectionIdFrame(QUICFrameId id = 0, QUICFrameGenerator *owner = nullptr) : QUICFrame(id, owner) {} - QUICNewConnectionIdFrame(const uint8_t *buf, size_t len); - QUICNewConnectionIdFrame(uint64_t seq, const QUICConnectionId &cid, QUICStatelessResetToken token, QUICFrameId id = 0, - QUICFrameGenerator *owner = nullptr) - : QUICFrame(id, owner), _sequence(seq), _connection_id(cid), _stateless_reset_token(token){}; + QUICNewConnectionIdFrame(const uint8_t *buf, size_t len, const QUICPacketR *packet = nullptr); + QUICNewConnectionIdFrame(uint64_t seq, uint64_t ret, const QUICConnectionId &cid, QUICStatelessResetToken token, + QUICFrameId id = 0, QUICFrameGenerator *owner = nullptr) + : QUICFrame(id, owner), _sequence(seq), _retire_prior_to(ret), _connection_id(cid), _stateless_reset_token(token){}; virtual QUICFrameType type() const override; virtual size_t size() const override; - virtual size_t store(uint8_t *buf, size_t *len, size_t limit) const override; - virtual void parse(const uint8_t *buf, size_t len) override; + virtual Ptr to_io_buffer_block(size_t limit) const override; + virtual void parse(const uint8_t *buf, size_t len, const QUICPacketR *packet) override; virtual int debug_msg(char *msg, size_t msg_len) const override; uint64_t sequence() const; + uint64_t retire_prior_to() const; QUICConnectionId connection_id() const; QUICStatelessResetToken stateless_reset_token() const; @@ -558,6 +579,7 @@ class QUICNewConnectionIdFrame : public QUICFrame virtual void _reset() override; uint64_t _sequence = 0; + uint64_t _retire_prior_to = 0; QUICConnectionId _connection_id = QUICConnectionId::ZERO(); QUICStatelessResetToken _stateless_reset_token; }; @@ -570,14 +592,14 @@ class QUICStopSendingFrame : public QUICFrame { public: QUICStopSendingFrame(QUICFrameId id = 0, QUICFrameGenerator *owner = nullptr) : QUICFrame(id, owner) {} - QUICStopSendingFrame(const uint8_t *buf, size_t len); + QUICStopSendingFrame(const uint8_t *buf, size_t len, const QUICPacketR *packet = nullptr); QUICStopSendingFrame(QUICStreamId stream_id, QUICAppErrorCode error_code, QUICFrameId id = 0, QUICFrameGenerator *owner = nullptr); virtual QUICFrameType type() const override; virtual size_t size() const override; - virtual void parse(const uint8_t *buf, size_t len) override; - virtual size_t store(uint8_t *buf, size_t *len, size_t limit) const override; + virtual void parse(const uint8_t *buf, size_t len, const QUICPacketR *packet) override; + virtual Ptr to_io_buffer_block(size_t limit) const override; QUICStreamId stream_id() const; QUICAppErrorCode error_code() const; @@ -598,7 +620,7 @@ class QUICPathChallengeFrame : public QUICFrame public: static constexpr uint8_t DATA_LEN = 8; QUICPathChallengeFrame(QUICFrameId id = 0, QUICFrameGenerator *owner = nullptr) : QUICFrame(id, owner) {} - QUICPathChallengeFrame(const uint8_t *buf, size_t len); + QUICPathChallengeFrame(const uint8_t *buf, size_t len, const QUICPacketR *packet = nullptr); QUICPathChallengeFrame(ats_unique_buf data, QUICFrameId id = 0, QUICFrameGenerator *owner = nullptr) : QUICFrame(id, owner), _data(std::move(data)) { @@ -606,8 +628,9 @@ class QUICPathChallengeFrame : public QUICFrame virtual QUICFrameType type() const override; virtual size_t size() const override; virtual bool is_probing_frame() const override; - virtual void parse(const uint8_t *buf, size_t len) override; - virtual size_t store(uint8_t *buf, size_t *len, size_t limit) const override; + virtual void parse(const uint8_t *buf, size_t len, const QUICPacketR *packet) override; + virtual Ptr to_io_buffer_block(size_t limit) const override; + virtual int debug_msg(char *msg, size_t msg_len) const override; const uint8_t *data() const; @@ -626,7 +649,7 @@ class QUICPathResponseFrame : public QUICFrame public: static constexpr uint8_t DATA_LEN = 8; QUICPathResponseFrame(QUICFrameId id = 0, QUICFrameGenerator *owner = nullptr) : QUICFrame(id, owner) {} - QUICPathResponseFrame(const uint8_t *buf, size_t len); + QUICPathResponseFrame(const uint8_t *buf, size_t len, const QUICPacketR *packet = nullptr); QUICPathResponseFrame(ats_unique_buf data, QUICFrameId id = 0, QUICFrameGenerator *owner = nullptr) : QUICFrame(id, owner), _data(std::move(data)) { @@ -634,8 +657,9 @@ class QUICPathResponseFrame : public QUICFrame virtual QUICFrameType type() const override; virtual size_t size() const override; virtual bool is_probing_frame() const override; - virtual void parse(const uint8_t *buf, size_t len) override; - virtual size_t store(uint8_t *buf, size_t *len, size_t limit) const override; + virtual void parse(const uint8_t *buf, size_t len, const QUICPacketR *packet) override; + virtual Ptr to_io_buffer_block(size_t limit) const override; + virtual int debug_msg(char *msg, size_t msg_len) const override; const uint8_t *data() const; @@ -653,15 +677,15 @@ class QUICNewTokenFrame : public QUICFrame { public: QUICNewTokenFrame(QUICFrameId id = 0, QUICFrameGenerator *owner = nullptr) : QUICFrame(id, owner) {} - QUICNewTokenFrame(const uint8_t *buf, size_t len); + QUICNewTokenFrame(const uint8_t *buf, size_t len, const QUICPacketR *packet = nullptr); QUICNewTokenFrame(ats_unique_buf token, size_t token_length, QUICFrameId id = 0, QUICFrameGenerator *owner = nullptr) : QUICFrame(id, owner), _token_length(token_length), _token(std::move(token)) { } virtual QUICFrameType type() const override; virtual size_t size() const override; - virtual size_t store(uint8_t *buf, size_t *len, size_t limit) const override; - virtual void parse(const uint8_t *buf, size_t len) override; + virtual Ptr to_io_buffer_block(size_t limit) const override; + virtual void parse(const uint8_t *buf, size_t len, const QUICPacketR *packet) override; uint64_t token_length() const; const uint8_t *token() const; @@ -681,15 +705,15 @@ class QUICRetireConnectionIdFrame : public QUICFrame { public: QUICRetireConnectionIdFrame(QUICFrameId id = 0, QUICFrameGenerator *owner = nullptr) : QUICFrame(id, owner) {} - QUICRetireConnectionIdFrame(const uint8_t *buf, size_t len); + QUICRetireConnectionIdFrame(const uint8_t *buf, size_t len, const QUICPacketR *packet = nullptr); QUICRetireConnectionIdFrame(uint64_t seq_num, QUICFrameId id = 0, QUICFrameGenerator *owner = nullptr) : QUICFrame(id, owner), _seq_num(seq_num) { } virtual QUICFrameType type() const override; virtual size_t size() const override; - virtual size_t store(uint8_t *buf, size_t *len, size_t limit) const override; - virtual void parse(const uint8_t *buf, size_t len) override; + virtual Ptr to_io_buffer_block(size_t limit) const override; + virtual void parse(const uint8_t *buf, size_t len, const QUICPacketR *packet) override; virtual int debug_msg(char *msg, size_t msg_len) const override; uint64_t seq_num() const; @@ -700,16 +724,32 @@ class QUICRetireConnectionIdFrame : public QUICFrame uint64_t _seq_num = 0; }; +// +// HANDSHAKE_DONE +// + +class QUICHandshakeDoneFrame : public QUICFrame +{ +public: + QUICHandshakeDoneFrame(QUICFrameId id = 0, QUICFrameGenerator *owner = nullptr) : QUICFrame(id, owner) {} + QUICHandshakeDoneFrame(const uint8_t *buf, size_t len, const QUICPacketR *packet = nullptr); + virtual QUICFrameType type() const override; + virtual size_t size() const override; + virtual Ptr to_io_buffer_block(size_t limit) const override; + virtual void parse(const uint8_t *buf, size_t len, const QUICPacketR *packet) override; +}; + // // UNKNOWN // class QUICUnknownFrame : public QUICFrame { +public: QUICFrameType type() const override; size_t size() const override; - size_t store(uint8_t *buf, size_t *len, size_t limit) const override; - void parse(const uint8_t *buf, size_t len) override; + virtual Ptr to_io_buffer_block(size_t limit) const override; + void parse(const uint8_t *buf, size_t len, const QUICPacketR *packet) override; int debug_msg(char *msg, size_t msg_len) const override; }; @@ -722,13 +762,13 @@ class QUICFrameFactory /* * This is used for creating a QUICFrame object based on received data. */ - static QUICFrame *create(uint8_t *buf, const uint8_t *src, size_t len); + static QUICFrame *create(uint8_t *buf, const uint8_t *src, size_t len, const QUICPacketR *packet); /* * This works almost the same as create() but it reuses created objects for performance. * If you create a frame object which has the same frame type that you created before, the object will be reset by new data. */ - const QUICFrame &fast_create(const uint8_t *buf, size_t len); + const QUICFrame &fast_create(const uint8_t *buf, size_t len, const QUICPacketR *packet); /* * Creates a STREAM frame. @@ -760,7 +800,7 @@ class QUICFrameFactory const char *reason_phrase = nullptr, QUICFrameId id = 0, QUICFrameGenerator *owner = nullptr); - static QUICConnectionCloseFrame *create_connection_close_frame(uint8_t *buf, QUICConnectionError &error, QUICFrameId id = 0, + static QUICConnectionCloseFrame *create_connection_close_frame(uint8_t *buf, const QUICConnectionError &error, QUICFrameId id = 0, QUICFrameGenerator *owner = nullptr); /* @@ -833,7 +873,8 @@ class QUICFrameFactory /* * Creates a NEW_CONNECTION_ID frame. */ - static QUICNewConnectionIdFrame *create_new_connection_id_frame(uint8_t *buf, uint32_t sequence, QUICConnectionId connectoin_id, + static QUICNewConnectionIdFrame *create_new_connection_id_frame(uint8_t *buf, uint64_t sequence, uint64_t retire_prior_to, + QUICConnectionId connection_id, QUICStatelessResetToken stateless_reset_token, QUICFrameId id = 0, QUICFrameGenerator *owner = nullptr); @@ -849,21 +890,19 @@ class QUICFrameFactory static QUICRetireConnectionIdFrame *create_retire_connection_id_frame(uint8_t *buf, uint64_t seq_num, QUICFrameId id = 0, QUICFrameGenerator *owner = nullptr); + /* + * Creates a PADDING frame + */ + static QUICPaddingFrame *create_padding_frame(uint8_t *buf, size_t size, QUICFrameId id = 0, QUICFrameGenerator *owner = nullptr); + + /* + * Creates a HANDSHAKE_DONE frame + */ + static QUICHandshakeDoneFrame *create_handshake_done_frame(uint8_t *buf, QUICFrameId id = 0, QUICFrameGenerator *owner = nullptr); + private: // FIXME Actual number of frame types is several but some of the values are not sequential. QUICFrame *_reusable_frames[256] = {nullptr}; uint8_t _buf_for_fast_create[256 * QUICFrame::MAX_INSTANCE_SIZE]; QUICUnknownFrame _unknown_frame; }; - -class QUICFrameInfo -{ -public: - QUICFrameInfo(QUICFrameId id, QUICFrameGenerator *generator) : _id(id), _generator(generator) {} - QUICFrameId id() const; - QUICFrameGenerator *generated_by() const; - -private: - QUICFrameId _id = 0; - QUICFrameGenerator *_generator; -}; diff --git a/iocore/net/quic/QUICFrameDispatcher.cc b/iocore/net/quic/QUICFrameDispatcher.cc index c2f3d1f7757..131bc4001ae 100644 --- a/iocore/net/quic/QUICFrameDispatcher.cc +++ b/iocore/net/quic/QUICFrameDispatcher.cc @@ -24,9 +24,11 @@ #include "QUICFrameDispatcher.h" #include "QUICDebugNames.h" -static constexpr char tag[] = "quic_net"; +static constexpr char tag[] = "quic_net"; +static constexpr char v_tag[] = "v_quic_net"; #define QUICDebug(fmt, ...) Debug(tag, "[%s] " fmt, this->_info->cids().data(), ##__VA_ARGS__) +#define QUICVDebug(fmt, ...) Debug(v_tag, "[%s] " fmt, this->_info->cids().data(), ##__VA_ARGS__) // // Frame Dispatcher @@ -42,8 +44,9 @@ QUICFrameDispatcher::add_handler(QUICFrameHandler *handler) } QUICConnectionErrorUPtr -QUICFrameDispatcher::receive_frames(QUICEncryptionLevel level, const uint8_t *payload, uint16_t size, bool &ack_only, - bool &is_flow_controlled, bool *has_non_probing_frame) +QUICFrameDispatcher::receive_frames(QUICContext &ctx, QUICEncryptionLevel level, const uint8_t *payload, uint16_t size, + bool &ack_only, bool &is_flow_controlled, bool *has_non_probing_frame, + const QUICPacketR *packet) { uint16_t cursor = 0; ack_only = true; @@ -51,7 +54,7 @@ QUICFrameDispatcher::receive_frames(QUICEncryptionLevel level, const uint8_t *pa QUICConnectionErrorUPtr error = nullptr; while (cursor < size) { - const QUICFrame &frame = this->_frame_factory.fast_create(payload + cursor, size - cursor); + const QUICFrame &frame = this->_frame_factory.fast_create(payload + cursor, size - cursor, packet); if (frame.type() == QUICFrameType::UNKNOWN) { QUICDebug("Failed to create a frame (%u bytes skipped)", size - cursor); break; @@ -63,14 +66,16 @@ QUICFrameDispatcher::receive_frames(QUICEncryptionLevel level, const uint8_t *pa QUICFrameType type = frame.type(); + ctx.trigger(QUICContext::CallbackEvent::FRAME_RECV, frame); + if (type == QUICFrameType::STREAM) { is_flow_controlled = true; } - if (is_debug_tag_set(tag) && type != QUICFrameType::PADDING) { + if (is_debug_tag_set(v_tag) && type != QUICFrameType::PADDING) { char msg[1024]; frame.debug_msg(msg, sizeof(msg)); - QUICDebug("[RX] | %s", msg); + QUICVDebug("[RX] | %s", msg); } if (type != QUICFrameType::PADDING && type != QUICFrameType::ACK) { diff --git a/iocore/net/quic/QUICFrameDispatcher.h b/iocore/net/quic/QUICFrameDispatcher.h index 79d71316c41..435103badc2 100644 --- a/iocore/net/quic/QUICFrameDispatcher.h +++ b/iocore/net/quic/QUICFrameDispatcher.h @@ -28,14 +28,16 @@ #include "QUICConnection.h" #include "QUICFrame.h" #include "QUICFrameHandler.h" +#include "QUICContext.h" class QUICFrameDispatcher { public: QUICFrameDispatcher(QUICConnectionInfoProvider *info); - QUICConnectionErrorUPtr receive_frames(QUICEncryptionLevel level, const uint8_t *payload, uint16_t size, - bool &should_send_ackbool, bool &is_flow_controlled, bool *has_non_probing_frame); + QUICConnectionErrorUPtr receive_frames(QUICContext &context, QUICEncryptionLevel level, const uint8_t *payload, uint16_t size, + bool &should_send_ackbool, bool &is_flow_controlled, bool *has_non_probing_frame, + const QUICPacketR *packet); void add_handler(QUICFrameHandler *handler); diff --git a/iocore/net/quic/QUICFrameGenerator.cc b/iocore/net/quic/QUICFrameGenerator.cc index 3dce9665d4a..908374e67fb 100644 --- a/iocore/net/quic/QUICFrameGenerator.cc +++ b/iocore/net/quic/QUICFrameGenerator.cc @@ -23,6 +23,7 @@ #include "QUICFrameGenerator.h" +// QUICFrameGenerator void QUICFrameGenerator::_records_frame(QUICFrameId id, QUICFrameInformationUPtr info) { @@ -58,3 +59,31 @@ QUICFrameGenerator::on_frame_lost(QUICFrameId id) this->_info.erase(it); } } + +void +QUICFrameGeneratorManager::add_generator(QUICFrameGenerator &generator, int weight) +{ + auto it = this->_inline_vector.begin(); + for (; it != this->_inline_vector.end(); ++it) { + if (it->first >= weight) { + break; + } + } + this->_inline_vector.emplace(it, weight, &generator); +} + +const std::vector & +QUICFrameGeneratorManager::generators() +{ + // Because we don't remove generators. So The size changed means new generators is coming + if (!this->_generators.empty() && this->_generators.size() == this->_inline_vector.size()) { + return this->_generators; + } + + this->_generators.clear(); + for (auto it : this->_inline_vector) { + this->_generators.emplace_back(it.second); + } + + return this->_generators; +} diff --git a/iocore/net/quic/QUICFrameGenerator.h b/iocore/net/quic/QUICFrameGenerator.h index 2298de597d0..f415ff7aaf5 100644 --- a/iocore/net/quic/QUICFrameGenerator.h +++ b/iocore/net/quic/QUICFrameGenerator.h @@ -30,14 +30,14 @@ class QUICFrameGenerator { public: virtual ~QUICFrameGenerator(){}; - virtual bool will_generate_frame(QUICEncryptionLevel level, ink_hrtime timestamp) = 0; + virtual bool will_generate_frame(QUICEncryptionLevel level, size_t current_packet_size, bool ack_eliciting, uint32_t seq_num) = 0; /* * This function constructs an instance of QUICFrame on buf. * It returns a pointer for the frame if it succeeded, and returns nullptr if it failed. */ virtual QUICFrame *generate_frame(uint8_t *buf, QUICEncryptionLevel level, uint64_t connection_credit, - uint16_t maximum_frame_size, ink_hrtime timestamp) = 0; + uint16_t maximum_frame_size, size_t current_packet_size, uint32_t seq_num) = 0; void on_frame_acked(QUICFrameId id); void on_frame_lost(QUICFrameId id); @@ -67,3 +67,55 @@ class QUICFrameGenerator QUICEncryptionLevel _encryption_level_filter = QUICEncryptionLevel::ONE_RTT; std::map _info; }; + +// only generate one frame per loop +class QUICFrameOnceGenerator : public QUICFrameGenerator +{ +public: + bool + will_generate_frame(QUICEncryptionLevel level, size_t current_packet_size, bool ack_eliciting, uint32_t seq_num) override + { + if (this->_seq_num == seq_num) { + return false; + } + + this->_seq_num = seq_num; + return this->_will_generate_frame(level, current_packet_size, ack_eliciting); + } + + QUICFrame * + generate_frame(uint8_t *buf, QUICEncryptionLevel level, uint64_t connection_credit, uint16_t maximum_frame_size, + size_t current_packet_size, uint32_t seq_num) override + { + this->_seq_num = seq_num; + return this->_generate_frame(buf, level, connection_credit, maximum_frame_size, current_packet_size); + } + +protected: + virtual bool _will_generate_frame(QUICEncryptionLevel level, size_t current_packet_size, bool ack_eliciting) = 0; + virtual QUICFrame *_generate_frame(uint8_t *buf, QUICEncryptionLevel level, uint64_t connection_credit, + uint16_t maximum_frame_size, size_t current_packet_size) = 0; + +private: + uint32_t _seq_num = UINT32_MAX; +}; + +enum QUICFrameGeneratorWeight { + EARLY = 100, + BEFORE_DATA = 200, + AFTER_DATA = 300, + LATE = 400, +}; + +class QUICFrameGeneratorManager +{ +public: + void add_generator(QUICFrameGenerator &generator, int weight); + const std::vector &generators(); + +private: + using QUICActiveFrameGenerator = std::pair; + + std::vector _generators; + std::vector _inline_vector; +}; diff --git a/iocore/net/quic/QUICFrameRetransmitter.h b/iocore/net/quic/QUICFrameRetransmitter.h index 9f45dd5b959..64f1f9536af 100644 --- a/iocore/net/quic/QUICFrameRetransmitter.h +++ b/iocore/net/quic/QUICFrameRetransmitter.h @@ -36,7 +36,7 @@ struct QUICFrameInformation { }; struct QUICFrameInformationDeleter { - void operator()(QUICFrameInformation *p); + void operator()(QUICFrameInformation *p) const; }; using QUICFrameInformationUPtr = std::unique_ptr; @@ -90,7 +90,7 @@ class QUICFrameRetransmitter extern ClassAllocator quicFrameInformationAllocator; inline void -QUICFrameInformationDeleter::operator()(QUICFrameInformation *p) +QUICFrameInformationDeleter::operator()(QUICFrameInformation *p) const { quicFrameInformationAllocator.free(p); } diff --git a/iocore/net/quic/QUICGlobals.cc b/iocore/net/quic/QUICGlobals.cc index 14761aa5d8d..9cef369b684 100644 --- a/iocore/net/quic/QUICGlobals.cc +++ b/iocore/net/quic/QUICGlobals.cc @@ -89,7 +89,7 @@ QUIC::_register_stats() { quic_rsb = RecAllocateRawStatBlock(static_cast(QUICStats::count)); - // Transfered packet counts + // Transferred packet counts RecRegisterRawStat(quic_rsb, RECT_PROCESS, "proxy.process.quic.total_packets_sent", RECD_INT, RECP_PERSISTENT, static_cast(QUICStats::total_packets_sent_stat), RecRawStatSyncSum); // RecRegisterRawStat(quic_rsb, RECT_PROCESS, "proxy.process.quic.total_packets_retransmitted", RECD_INT, RECP_PERSISTENT, diff --git a/iocore/net/quic/QUICHandshake.cc b/iocore/net/quic/QUICHandshake.cc index 6042d26a67c..f64f2221315 100644 --- a/iocore/net/quic/QUICHandshake.cc +++ b/iocore/net/quic/QUICHandshake.cc @@ -27,6 +27,7 @@ #include "QUICEvents.h" #include "QUICGlobals.h" +#include "QUICHandshakeProtocol.h" #include "QUICPacketFactory.h" #include "QUICVersionNegotiator.h" #include "QUICConfig.h" @@ -80,19 +81,21 @@ } static constexpr int UDP_MAXIMUM_PAYLOAD_SIZE = 65527; -// TODO: fix size -static constexpr int MAX_HANDSHAKE_MSG_LEN = 65527; -QUICHandshake::QUICHandshake(QUICConnection *qc, QUICHandshakeProtocol *hsp) : QUICHandshake(qc, hsp, {}, false) {} +QUICHandshake::QUICHandshake(QUICVersion version, QUICConnection *qc, QUICHandshakeProtocol *hsp) + : QUICHandshake(version, qc, hsp, {}, false) +{ +} -QUICHandshake::QUICHandshake(QUICConnection *qc, QUICHandshakeProtocol *hsp, QUICStatelessResetToken token, bool stateless_retry) +QUICHandshake::QUICHandshake(QUICVersion version, QUICConnection *qc, QUICHandshakeProtocol *hsp, QUICStatelessResetToken token, + bool stateless_retry) : _qc(qc), _hs_protocol(hsp), _version_negotiator(new QUICVersionNegotiator()), _reset_token(token), _stateless_retry(stateless_retry) { - this->_hs_protocol->initialize_key_materials(this->_qc->original_connection_id()); + this->_hs_protocol->initialize_key_materials(this->_qc->original_connection_id(), version); if (this->_qc->direction() == NET_VCONNECTION_OUT) { this->_client_initial = true; @@ -107,19 +110,19 @@ QUICHandshake::~QUICHandshake() QUICConnectionErrorUPtr QUICHandshake::start(const QUICTPConfig &tp_config, QUICPacketFactory *packet_factory, bool vn_exercise_enabled) { - QUICVersion initital_version = QUIC_SUPPORTED_VERSIONS[0]; + QUICVersion initial_version = QUIC_SUPPORTED_VERSIONS[0]; if (vn_exercise_enabled) { - initital_version = QUIC_EXERCISE_VERSION; + initial_version = QUIC_EXERCISE_VERSION1; } this->_load_local_client_transport_parameters(tp_config); - packet_factory->set_version(initital_version); + packet_factory->set_version(initial_version); return nullptr; } QUICConnectionErrorUPtr -QUICHandshake::start(const QUICTPConfig &tp_config, const QUICPacket &initial_packet, QUICPacketFactory *packet_factory, +QUICHandshake::start(const QUICTPConfig &tp_config, const QUICInitialPacketR &initial_packet, QUICPacketFactory *packet_factory, const QUICPreferredAddress *pref_addr) { // Negotiate version @@ -133,17 +136,18 @@ QUICHandshake::start(const QUICTPConfig &tp_config, const QUICPacket &initial_pa this->_load_local_server_transport_parameters(tp_config, pref_addr); packet_factory->set_version(this->_version_negotiator->negotiated_version()); } else { - ink_assert(!"Unsupported version initial packet should be droped QUICPakcetHandler"); + ink_assert(!"Unsupported version initial packet should be dropped QUICPacketHandler"); } } else { return std::make_unique(QUICTransErrorCode::PROTOCOL_VIOLATION); } + this->_initial_source_cid_received = initial_packet.source_cid(); } return nullptr; } QUICConnectionErrorUPtr -QUICHandshake::negotiate_version(const QUICPacket &vn, QUICPacketFactory *packet_factory) +QUICHandshake::negotiate_version(const QUICVersionNegotiationPacketR &vn, QUICPacketFactory *packet_factory) { // Client side only ink_assert(this->_qc->direction() == NET_VCONNECTION_OUT); @@ -166,7 +170,7 @@ QUICHandshake::negotiate_version(const QUICPacket &vn, QUICPacketFactory *packet packet_factory->set_version(version); } else { QUICHSDebug("Version negotiation failed"); - return std::make_unique(QUICTransErrorCode::VERSION_NEGOTIATION_ERROR); + return std::make_unique(QUICTransErrorCode::PROTOCOL_VIOLATION); } return nullptr; @@ -185,6 +189,16 @@ QUICHandshake::is_completed() const return this->_hs_protocol->is_handshake_finished(); } +bool +QUICHandshake::is_confirmed() const +{ + if (this->_qc->direction() == NET_VCONNECTION_IN) { + return this->is_completed(); + } else { + return this->_is_handshake_done_received; + } +} + bool QUICHandshake::is_stateless_retry_enabled() const { @@ -198,20 +212,20 @@ QUICHandshake::has_remote_tp() const } QUICVersion -QUICHandshake::negotiated_version() +QUICHandshake::negotiated_version() const { return this->_version_negotiator->negotiated_version(); } // Similar to SSLNetVConnection::getSSLCipherSuite() const char * -QUICHandshake::negotiated_cipher_suite() +QUICHandshake::negotiated_cipher_suite() const { return this->_hs_protocol->negotiated_cipher_suite(); } void -QUICHandshake::negotiated_application_name(const uint8_t **name, unsigned int *len) +QUICHandshake::negotiated_application_name(const uint8_t **name, unsigned int *len) const { this->_hs_protocol->negotiated_application_name(name, len); } @@ -244,6 +258,17 @@ QUICHandshake::_check_remote_transport_parameters(std::shared_ptrnegotiated_version() == QUIC_SUPPORTED_VERSIONS[0]) { // draft-28 + uint16_t cid_buf_len; + const uint8_t *cid_buf = tp->getAsBytes(QUICTransportParameterId::INITIAL_SOURCE_CONNECTION_ID, cid_buf_len); + QUICConnectionId cid_in_tp(cid_buf, cid_buf_len); + if (cid_in_tp != this->_initial_source_cid_received) { + this->_abort_handshake(QUICTransErrorCode::PROTOCOL_VIOLATION); + return false; + } + } + this->_remote_transport_parameters = tp; return true; @@ -259,6 +284,26 @@ QUICHandshake::_check_remote_transport_parameters(std::shared_ptrnegotiated_version() == QUIC_SUPPORTED_VERSIONS[0]) { // draft-28 + uint16_t cid_buf_len; + const uint8_t *cid_buf = tp->getAsBytes(QUICTransportParameterId::INITIAL_SOURCE_CONNECTION_ID, cid_buf_len); + QUICConnectionId cid_in_tp(cid_buf, cid_buf_len); + if (cid_in_tp != this->_initial_source_cid_received) { + this->_abort_handshake(QUICTransErrorCode::PROTOCOL_VIOLATION); + return false; + } + + if (!this->_retry_source_cid_received.is_zero()) { + cid_buf = tp->getAsBytes(QUICTransportParameterId::RETRY_SOURCE_CONNECTION_ID, cid_buf_len); + QUICConnectionId cid_in_tp(cid_buf, cid_buf_len); + if (cid_in_tp != this->_retry_source_cid_received) { + this->_abort_handshake(QUICTransErrorCode::PROTOCOL_VIOLATION); + return false; + } + } + } + this->_remote_transport_parameters = tp; return true; @@ -293,11 +338,24 @@ QUICHandshake::reset() } } +void +QUICHandshake::update(const QUICInitialPacketR &packet) +{ + this->_initial_source_cid_received = packet.source_cid(); +} + +void +QUICHandshake::update(const QUICRetryPacketR &packet) +{ + this->_retry_source_cid_received = packet.source_cid(); +} + std::vector QUICHandshake::interests() { return { QUICFrameType::CRYPTO, + QUICFrameType::HANDSHAKE_DONE, }; } @@ -308,8 +366,15 @@ QUICHandshake::handle_frame(QUICEncryptionLevel level, const QUICFrame &frame) switch (frame.type()) { case QUICFrameType::CRYPTO: error = this->_crypto_streams[static_cast(level)].recv(static_cast(frame)); - if (error != nullptr) { - return error; + if (error == nullptr) { + error = this->do_handshake(); + } + break; + case QUICFrameType::HANDSHAKE_DONE: + if (this->_qc->direction() == NET_VCONNECTION_IN) { + error = std::make_unique(QUICTransErrorCode::PROTOCOL_VIOLATION); + } else { + this->_is_handshake_done_received = true; } break; default: @@ -318,28 +383,44 @@ QUICHandshake::handle_frame(QUICEncryptionLevel level, const QUICFrame &frame) break; } - return this->do_handshake(); + return error; } bool -QUICHandshake::will_generate_frame(QUICEncryptionLevel level, ink_hrtime timestamp) +QUICHandshake::will_generate_frame(QUICEncryptionLevel level, size_t current_packet_size, bool ack_eliciting, uint32_t seq_num) { if (!this->_is_level_matched(level)) { return false; } - return this->_crypto_streams[static_cast(level)].will_generate_frame(level, timestamp); + return (this->_qc->direction() == NET_VCONNECTION_IN && !this->_is_handshake_done_sent) || + this->_crypto_streams[static_cast(level)].will_generate_frame(level, current_packet_size, ack_eliciting, seq_num); } QUICFrame * QUICHandshake::generate_frame(uint8_t *buf, QUICEncryptionLevel level, uint64_t connection_credit, uint16_t maximum_frame_size, - ink_hrtime timestamp) + size_t current_packet_size, uint32_t seq_num) { QUICFrame *frame = nullptr; if (this->_is_level_matched(level)) { - frame = - this->_crypto_streams[static_cast(level)].generate_frame(buf, level, connection_credit, maximum_frame_size, timestamp); + // CRYPTO + frame = this->_crypto_streams[static_cast(level)].generate_frame(buf, level, connection_credit, maximum_frame_size, + current_packet_size, seq_num); + if (frame) { + return frame; + } + } + + if (level == QUICEncryptionLevel::ONE_RTT) { + // HANDSHAKE_DONE + if (!this->_is_handshake_done_sent && this->is_completed()) { + frame = QUICFrameFactory::create_handshake_done_frame(buf, this->_issue_frame_id(), this); + } + if (frame) { + this->_is_handshake_done_sent = true; + return frame; + } } return frame; @@ -351,10 +432,21 @@ QUICHandshake::_load_local_server_transport_parameters(const QUICTPConfig &tp_co QUICTransportParametersInEncryptedExtensions *tp = new QUICTransportParametersInEncryptedExtensions(); // MUSTs - tp->set(QUICTransportParameterId::IDLE_TIMEOUT, static_cast(tp_config.no_activity_timeout())); + tp->set(QUICTransportParameterId::MAX_IDLE_TIMEOUT, static_cast(tp_config.no_activity_timeout())); if (this->_stateless_retry) { - tp->set(QUICTransportParameterId::ORIGINAL_CONNECTION_ID, this->_qc->first_connection_id(), + tp->set(QUICTransportParameterId::ORIGINAL_DESTINATION_CONNECTION_ID, this->_qc->first_connection_id(), this->_qc->first_connection_id().length()); + tp->set(QUICTransportParameterId::RETRY_SOURCE_CONNECTION_ID, this->_qc->retry_source_connection_id(), + this->_qc->retry_source_connection_id().length()); + } else { + if (this->negotiated_version() == QUIC_SUPPORTED_VERSIONS[0]) { // draft-28 + tp->set(QUICTransportParameterId::ORIGINAL_DESTINATION_CONNECTION_ID, this->_qc->original_connection_id(), + this->_qc->original_connection_id().length()); + } + } + if (this->negotiated_version() == QUIC_SUPPORTED_VERSIONS[0]) { // draft-28 + tp->set(QUICTransportParameterId::INITIAL_SOURCE_CONNECTION_ID, this->_qc->initial_source_connection_id(), + this->_qc->initial_source_connection_id().length()); } // MAYs @@ -376,18 +468,32 @@ QUICHandshake::_load_local_server_transport_parameters(const QUICTPConfig &tp_co if (tp_config.initial_max_stream_data_uni() != 0) { tp->set(QUICTransportParameterId::INITIAL_MAX_STREAM_DATA_UNI, tp_config.initial_max_stream_data_uni()); } + if (tp_config.disable_active_migration()) { + tp->set(QUICTransportParameterId::DISABLE_ACTIVE_MIGRATION, nullptr, 0); + } if (pref_addr != nullptr) { uint8_t pref_addr_buf[QUICPreferredAddress::MAX_LEN]; uint16_t len; pref_addr->store(pref_addr_buf, len); tp->set(QUICTransportParameterId::PREFERRED_ADDRESS, pref_addr_buf, len); } + if (tp_config.active_cid_limit() != 0) { + tp->set(QUICTransportParameterId::ACTIVE_CONNECTION_ID_LIMIT, tp_config.active_cid_limit()); + } // MAYs (server) tp->set(QUICTransportParameterId::STATELESS_RESET_TOKEN, this->_reset_token.buf(), QUICStatelessResetToken::LEN); tp->set(QUICTransportParameterId::ACK_DELAY_EXPONENT, tp_config.ack_delay_exponent()); - tp->add_version(QUIC_SUPPORTED_VERSIONS[0]); + // Additional parameters + for (auto &¶m : tp_config.additional_tp()) { + tp->set(param.first, param.second.first, param.second.second); + } + + // Additional parameters + for (auto &¶m : tp_config.additional_tp()) { + tp->set(param.first, param.second.first, param.second.second); + } this->_local_transport_parameters = std::shared_ptr(tp); this->_hs_protocol->set_local_transport_parameters(this->_local_transport_parameters); @@ -399,7 +505,9 @@ QUICHandshake::_load_local_client_transport_parameters(const QUICTPConfig &tp_co QUICTransportParametersInClientHello *tp = new QUICTransportParametersInClientHello(); // MUSTs - tp->set(QUICTransportParameterId::IDLE_TIMEOUT, static_cast(tp_config.no_activity_timeout())); + tp->set(QUICTransportParameterId::MAX_IDLE_TIMEOUT, static_cast(tp_config.no_activity_timeout())); + tp->set(QUICTransportParameterId::INITIAL_SOURCE_CONNECTION_ID, this->_qc->initial_source_connection_id(), + this->_qc->initial_source_connection_id().length()); // MAYs if (tp_config.initial_max_data() != 0) { @@ -421,6 +529,14 @@ QUICHandshake::_load_local_client_transport_parameters(const QUICTPConfig &tp_co tp->set(QUICTransportParameterId::INITIAL_MAX_STREAM_DATA_UNI, tp_config.initial_max_stream_data_uni()); } tp->set(QUICTransportParameterId::ACK_DELAY_EXPONENT, tp_config.ack_delay_exponent()); + if (tp_config.active_cid_limit() != 0) { + tp->set(QUICTransportParameterId::ACTIVE_CONNECTION_ID_LIMIT, tp_config.active_cid_limit()); + } + + // Additional parameters + for (auto &¶m : tp_config.additional_tp()) { + tp->set(param.first, param.second.first, param.second.second); + } this->_local_transport_parameters = std::shared_ptr(tp); this->_hs_protocol->set_local_transport_parameters(std::unique_ptr(tp)); @@ -446,18 +562,13 @@ QUICHandshake::do_handshake() // TODO: check size if (bytes_avail > 0) { stream->read(in.buf + in.offsets[index], bytes_avail); - in.offsets[index] = bytes_avail; - in.offsets[4] += bytes_avail; } + in.offsets[index + 1] = in.offsets[index] + bytes_avail; } } - QUICHandshakeMsgs out; - uint8_t out_buf[MAX_HANDSHAKE_MSG_LEN] = {0}; - out.buf = out_buf; - out.max_buf_len = MAX_HANDSHAKE_MSG_LEN; - - int result = this->_hs_protocol->handshake(&out, &in); + QUICHandshakeMsgs *out = nullptr; + int result = this->_hs_protocol->handshake(&out, &in); if (this->_remote_transport_parameters == nullptr) { if (!this->check_remote_transport_parameters()) { result = 0; @@ -465,18 +576,25 @@ QUICHandshake::do_handshake() } if (result == 1) { - for (auto level : QUIC_ENCRYPTION_LEVELS) { - int index = static_cast(level); - QUICCryptoStream *stream = &this->_crypto_streams[index]; - size_t len = out.offsets[index + 1] - out.offsets[index]; - // TODO: check size - if (len > 0) { - stream->write(out.buf + out.offsets[index], len); + if (out) { + for (auto level : QUIC_ENCRYPTION_LEVELS) { + int index = static_cast(level); + QUICCryptoStream *stream = &this->_crypto_streams[index]; + size_t len = out->offsets[index + 1] - out->offsets[index]; + // TODO: check size + if (len > 0) { + stream->write(out->buf + out->offsets[index], len); + } } } - } else if (out.error_code != 0) { + } else { this->_hs_protocol->abort_handshake(); - error = std::make_unique(QUICErrorClass::TRANSPORT, out.error_code); + if (this->_hs_protocol->has_crypto_error()) { + error = std::make_unique(QUICErrorClass::TRANSPORT, this->_hs_protocol->crypto_error()); + } else { + error = std::make_unique(QUICErrorClass::TRANSPORT, + static_cast(QUICTransErrorCode::PROTOCOL_VIOLATION)); + } } return error; @@ -489,7 +607,7 @@ QUICHandshake::_abort_handshake(QUICTransErrorCode code) this->_hs_protocol->abort_handshake(); - this->_qc->close(QUICConnectionErrorUPtr(new QUICConnectionError(code))); + this->_qc->close_quic_connection(QUICConnectionErrorUPtr(new QUICConnectionError(code))); } /* @@ -508,3 +626,10 @@ QUICHandshake::_is_level_matched(QUICEncryptionLevel level) { return true; } + +void +QUICHandshake::_on_frame_lost(QUICFrameInformationUPtr &info) +{ + ink_assert(info->type == QUICFrameType::HANDSHAKE_DONE); + this->_is_handshake_done_sent = false; +} diff --git a/iocore/net/quic/QUICHandshake.h b/iocore/net/quic/QUICHandshake.h index 4dda103616a..4f55985badf 100644 --- a/iocore/net/quic/QUICHandshake.h +++ b/iocore/net/quic/QUICHandshake.h @@ -35,15 +35,17 @@ */ class QUICVersionNegotiator; class QUICPacketFactory; +class QUICHandshakeProtocol; class SSLNextProtocolSet; class QUICHandshake : public QUICFrameHandler, public QUICFrameGenerator { public: // Constructor for client side - QUICHandshake(QUICConnection *qc, QUICHandshakeProtocol *hsp); + QUICHandshake(QUICVersion version, QUICConnection *qc, QUICHandshakeProtocol *hsp); // Constructor for server side - QUICHandshake(QUICConnection *qc, QUICHandshakeProtocol *hsp, QUICStatelessResetToken token, bool stateless_retry); + QUICHandshake(QUICVersion version, QUICConnection *qc, QUICHandshakeProtocol *hsp, QUICStatelessResetToken token, + bool stateless_retry); ~QUICHandshake(); // QUICFrameHandler @@ -51,32 +53,35 @@ class QUICHandshake : public QUICFrameHandler, public QUICFrameGenerator virtual QUICConnectionErrorUPtr handle_frame(QUICEncryptionLevel level, const QUICFrame &frame) override; // QUICFrameGenerator - bool will_generate_frame(QUICEncryptionLevel level, ink_hrtime timestamp) override; + bool will_generate_frame(QUICEncryptionLevel level, size_t current_packet_size, bool ack_eliciting, uint32_t seq_num) override; QUICFrame *generate_frame(uint8_t *buf, QUICEncryptionLevel level, uint64_t connection_credit, uint16_t maximum_frame_size, - ink_hrtime timestamp) override; + size_t current_packet_size, uint32_t seq_num) override; // for client side QUICConnectionErrorUPtr start(const QUICTPConfig &tp_config, QUICPacketFactory *packet_factory, bool vn_exercise_enabled); - QUICConnectionErrorUPtr negotiate_version(const QUICPacket &packet, QUICPacketFactory *packet_factory); + QUICConnectionErrorUPtr negotiate_version(const QUICVersionNegotiationPacketR &packet, QUICPacketFactory *packet_factory); void reset(); + void update(const QUICInitialPacketR &initial_packet); + void update(const QUICRetryPacketR &retry_packet); // for server side - QUICConnectionErrorUPtr start(const QUICTPConfig &tp_config, const QUICPacket &initial_packet, QUICPacketFactory *packet_factory, - const QUICPreferredAddress *pref_addr); + QUICConnectionErrorUPtr start(const QUICTPConfig &tp_config, const QUICInitialPacketR &initial_packet, + QUICPacketFactory *packet_factory, const QUICPreferredAddress *pref_addr); QUICConnectionErrorUPtr do_handshake(); bool check_remote_transport_parameters(); // Getters - QUICVersion negotiated_version(); - const char *negotiated_cipher_suite(); - void negotiated_application_name(const uint8_t **name, unsigned int *len); + QUICVersion negotiated_version() const; + const char *negotiated_cipher_suite() const; + void negotiated_application_name(const uint8_t **name, unsigned int *len) const; std::shared_ptr local_transport_parameters(); std::shared_ptr remote_transport_parameters(); bool is_version_negotiated() const; bool is_completed() const; + bool is_confirmed() const; bool is_stateless_retry_enabled() const; bool has_remote_tp() const; @@ -89,8 +94,10 @@ class QUICHandshake : public QUICFrameHandler, public QUICFrameGenerator QUICVersionNegotiator *_version_negotiator = nullptr; QUICStatelessResetToken _reset_token; - bool _client_initial = false; - bool _stateless_retry = false; + bool _client_initial = false; + bool _stateless_retry = false; + QUICConnectionId _initial_source_cid_received = QUICConnectionId::ZERO(); + QUICConnectionId _retry_source_cid_received = QUICConnectionId::ZERO(); QUICCryptoStream _crypto_streams[4]; @@ -102,4 +109,10 @@ class QUICHandshake : public QUICFrameHandler, public QUICFrameGenerator std::shared_ptr _remote_transport_parameters = nullptr; void _abort_handshake(QUICTransErrorCode code); + + bool _is_handshake_done_sent = false; + bool _is_handshake_done_received = false; + + // QUICFrameGenerator + void _on_frame_lost(QUICFrameInformationUPtr &info) override; }; diff --git a/iocore/net/quic/QUICHandshakeProtocol.h b/iocore/net/quic/QUICHandshakeProtocol.h index fd4c68b65c1..8bc6e776186 100644 --- a/iocore/net/quic/QUICHandshakeProtocol.h +++ b/iocore/net/quic/QUICHandshakeProtocol.h @@ -34,7 +34,7 @@ struct QUICHandshakeMsgs { uint8_t *buf = nullptr; //< pointer to the buffer size_t max_buf_len = 0; //< size of buffer size_t offsets[5] = {0}; //< offset to the each encryption level - {initial, zero_rtt, handshake, one_rtt, total length} - uint16_t error_code = 0; //< CRYPTO_ERROR - TLS Alert Desciption + 0x100 + uint16_t error_code = 0; //< CRYPTO_ERROR - TLS Alert Description + 0x100 }; class QUICHandshakeProtocol @@ -43,11 +43,11 @@ class QUICHandshakeProtocol QUICHandshakeProtocol(QUICPacketProtectionKeyInfo &pp_key_info) : _pp_key_info(pp_key_info) {} virtual ~QUICHandshakeProtocol(){}; - virtual int handshake(QUICHandshakeMsgs *out, const QUICHandshakeMsgs *in) = 0; + virtual int handshake(QUICHandshakeMsgs **out, const QUICHandshakeMsgs *in) = 0; virtual void reset() = 0; virtual bool is_handshake_finished() const = 0; virtual bool is_ready_to_derive() const = 0; - virtual int initialize_key_materials(QUICConnectionId cid) = 0; + virtual int initialize_key_materials(QUICConnectionId cid, QUICVersion) = 0; virtual const char *negotiated_cipher_suite() const = 0; virtual void negotiated_application_name(const uint8_t **name, unsigned int *len) const = 0; @@ -58,6 +58,8 @@ class QUICHandshakeProtocol virtual QUICEncryptionLevel current_encryption_level() const = 0; virtual void abort_handshake() = 0; + virtual bool has_crypto_error() const = 0; + virtual uint64_t crypto_error() const = 0; protected: QUICPacketProtectionKeyInfo &_pp_key_info; diff --git a/iocore/net/quic/QUICIncomingFrameBuffer.cc b/iocore/net/quic/QUICIncomingFrameBuffer.cc index 154563f4278..373d0093123 100644 --- a/iocore/net/quic/QUICIncomingFrameBuffer.cc +++ b/iocore/net/quic/QUICIncomingFrameBuffer.cc @@ -113,7 +113,11 @@ QUICIncomingStreamFrameBuffer::insert(const QUICFrame *frame) this->_recv_offset = offset + len; this->_recv_buffer.push(stream_frame); } else { - this->_out_of_order_queue.insert(std::make_pair(offset, stream_frame)); + auto result = this->_out_of_order_queue.insert(std::make_pair(offset, stream_frame)); + if (!result.second) { + // Duplicate frame doesn't need to be inserted + delete stream_frame; + } } return nullptr; @@ -134,30 +138,30 @@ QUICIncomingStreamFrameBuffer::_check_and_set_fin_flag(QUICOffset offset, size_t // stream with fin flag {11.3. Stream Final Offset} // Once a final offset for a stream is known, it cannot change. // If a RESET_STREAM or STREAM frame causes the final offset to change for a stream, - // an endpoint SHOULD respond with a FINAL_OFFSET_ERROR error (see Section 12). + // an endpoint SHOULD respond with a FINAL_SIZE_ERROR error (see Section 12). // A receiver SHOULD treat receipt of data at or beyond the final offset as a - // FINAL_OFFSET_ERROR error, even after a stream is closed. + // FINAL_SIZE_ERROR error, even after a stream is closed. // {11.3. Stream Final Offset} // A receiver SHOULD treat receipt of data at or beyond the final offset as a - // FINAL_OFFSET_ERROR error, even after a stream is closed. + // FINAL_SIZE_ERROR error, even after a stream is closed. if (fin_flag) { if (this->_fin_offset != UINT64_MAX) { if (this->_fin_offset == offset + len) { // dup fin frame return nullptr; } - return std::make_unique(QUICTransErrorCode::FINAL_OFFSET_ERROR); + return std::make_unique(QUICTransErrorCode::FINAL_SIZE_ERROR); } this->_fin_offset = offset + len; if (this->_max_offset > this->_fin_offset) { - return std::make_unique(QUICTransErrorCode::FINAL_OFFSET_ERROR); + return std::make_unique(QUICTransErrorCode::FINAL_SIZE_ERROR); } } else if (this->_fin_offset != UINT64_MAX && this->_fin_offset <= offset) { - return std::make_unique(QUICTransErrorCode::FINAL_OFFSET_ERROR); + return std::make_unique(QUICTransErrorCode::FINAL_SIZE_ERROR); } this->_max_offset = std::max(offset + len, this->_max_offset); @@ -242,7 +246,11 @@ QUICIncomingCryptoFrameBuffer::insert(const QUICFrame *frame) this->_recv_offset = offset + len; this->_recv_buffer.push(crypto_frame); } else { - this->_out_of_order_queue.insert(std::make_pair(offset, crypto_frame)); + auto result = this->_out_of_order_queue.insert(std::make_pair(offset, crypto_frame)); + if (!result.second) { + // Duplicate frame doesn't need to be inserted + delete crypto_frame; + } } return nullptr; diff --git a/iocore/net/quic/QUICIntUtil.cc b/iocore/net/quic/QUICIntUtil.cc index e9840ca2d61..26159d74770 100644 --- a/iocore/net/quic/QUICIntUtil.cc +++ b/iocore/net/quic/QUICIntUtil.cc @@ -102,11 +102,11 @@ QUICVariableInt::decode(uint64_t &dst, size_t &len, const uint8_t *src, size_t s } uint64_t -QUICIntUtil::read_QUICVariableInt(const uint8_t *buf) +QUICIntUtil::read_QUICVariableInt(const uint8_t *buf, size_t buf_len) { uint64_t dst = 0; size_t len = 0; - QUICVariableInt::decode(dst, len, buf, 8); + QUICVariableInt::decode(dst, len, buf, buf_len); return dst; } diff --git a/iocore/net/quic/QUICIntUtil.h b/iocore/net/quic/QUICIntUtil.h index c259bca682a..5116304643a 100644 --- a/iocore/net/quic/QUICIntUtil.h +++ b/iocore/net/quic/QUICIntUtil.h @@ -38,7 +38,7 @@ class QUICVariableInt class QUICIntUtil { public: - static uint64_t read_QUICVariableInt(const uint8_t *buf); + static uint64_t read_QUICVariableInt(const uint8_t *buf, size_t buf_len); static void write_QUICVariableInt(uint64_t data, uint8_t *buf, size_t *len); static uint64_t read_nbytes_as_uint(const uint8_t *buf, uint8_t n); static void write_uint_as_nbytes(uint64_t value, uint8_t n, uint8_t *buf, size_t *len); diff --git a/iocore/net/quic/QUICKeyGenerator.cc b/iocore/net/quic/QUICKeyGenerator.cc index 05c1796f950..7ac32baf653 100644 --- a/iocore/net/quic/QUICKeyGenerator.cc +++ b/iocore/net/quic/QUICKeyGenerator.cc @@ -34,7 +34,10 @@ using namespace std::literals; constexpr static uint8_t QUIC_VERSION_1_SALT[] = { - 0xef, 0x4f, 0xb0, 0xab, 0xb4, 0x74, 0x70, 0xc4, 0x1b, 0xef, 0xcf, 0x80, 0x31, 0x33, 0x4f, 0xae, 0x48, 0x5e, 0x09, 0xa0, + 0xaf, 0xbf, 0xec, 0x28, 0x99, 0x93, 0xd2, 0x4c, 0x9e, 0x97, 0x86, 0xf1, 0x9c, 0x61, 0x11, 0xe0, 0x43, 0x90, 0xa8, 0x99, +}; +constexpr static uint8_t QUIC_VERSION_1_D27_SALT[] = { + 0xc3, 0xee, 0xf7, 0x12, 0xc7, 0x2e, 0xbb, 0x5a, 0x11, 0xa7, 0xd2, 0x43, 0x2b, 0xb4, 0x63, 0x65, 0xbe, 0xf9, 0xf5, 0x02, }; constexpr static std::string_view LABEL_FOR_CLIENT_INITIAL_SECRET("client in"sv); constexpr static std::string_view LABEL_FOR_SERVER_INITIAL_SECRET("server in"sv); @@ -43,17 +46,17 @@ constexpr static std::string_view LABEL_FOR_IV("quic iv"sv); constexpr static std::string_view LABEL_FOR_HP("quic hp"sv); void -QUICKeyGenerator::generate(uint8_t *hp_key, uint8_t *pp_key, uint8_t *iv, size_t *iv_len, QUICConnectionId cid) +QUICKeyGenerator::generate(QUICVersion version, uint8_t *hp_key, uint8_t *pp_key, uint8_t *iv, size_t *iv_len, QUICConnectionId cid) { - const QUIC_EVP_CIPHER *cipher = this->_get_cipher_for_initial(); - const EVP_MD *md = EVP_sha256(); + const EVP_CIPHER *cipher = this->_get_cipher_for_initial(); + const EVP_MD *md = EVP_sha256(); uint8_t secret[512]; size_t secret_len = sizeof(secret); QUICHKDF hkdf(md); switch (this->_ctx) { case Context::CLIENT: - this->_generate_initial_secret(secret, &secret_len, hkdf, cid, LABEL_FOR_CLIENT_INITIAL_SECRET.data(), + this->_generate_initial_secret(version, secret, &secret_len, hkdf, cid, LABEL_FOR_CLIENT_INITIAL_SECRET.data(), LABEL_FOR_CLIENT_INITIAL_SECRET.length(), EVP_MD_size(md)); if (is_debug_tag_set("vv_quic_crypto")) { uint8_t print_buf[1024 + 1]; @@ -63,7 +66,7 @@ QUICKeyGenerator::generate(uint8_t *hp_key, uint8_t *pp_key, uint8_t *iv, size_t break; case Context::SERVER: - this->_generate_initial_secret(secret, &secret_len, hkdf, cid, LABEL_FOR_SERVER_INITIAL_SECRET.data(), + this->_generate_initial_secret(version, secret, &secret_len, hkdf, cid, LABEL_FOR_SERVER_INITIAL_SECRET.data(), LABEL_FOR_SERVER_INITIAL_SECRET.length(), EVP_MD_size(md)); if (is_debug_tag_set("vv_quic_crypto")) { uint8_t print_buf[1024 + 1]; @@ -79,14 +82,14 @@ QUICKeyGenerator::generate(uint8_t *hp_key, uint8_t *pp_key, uint8_t *iv, size_t void QUICKeyGenerator::regenerate(uint8_t *hp_key, uint8_t *pp_key, uint8_t *iv, size_t *iv_len, const uint8_t *secret, - size_t secret_len, const QUIC_EVP_CIPHER *cipher, QUICHKDF &hkdf) + size_t secret_len, const EVP_CIPHER *cipher, QUICHKDF &hkdf) { this->_generate(hp_key, pp_key, iv, iv_len, hkdf, secret, secret_len, cipher); } int QUICKeyGenerator::_generate(uint8_t *hp_key, uint8_t *pp_key, uint8_t *iv, size_t *iv_len, QUICHKDF &hkdf, const uint8_t *secret, - size_t secret_len, const QUIC_EVP_CIPHER *cipher) + size_t secret_len, const EVP_CIPHER *cipher) { // Generate key, iv, and hp_key // key = HKDF-Expand-Label(S, "quic key", "", key_length) @@ -101,18 +104,33 @@ QUICKeyGenerator::_generate(uint8_t *hp_key, uint8_t *pp_key, uint8_t *iv, size_ } int -QUICKeyGenerator::_generate_initial_secret(uint8_t *out, size_t *out_len, QUICHKDF &hkdf, QUICConnectionId cid, const char *label, - size_t label_len, size_t length) +QUICKeyGenerator::_generate_initial_secret(QUICVersion version, uint8_t *out, size_t *out_len, QUICHKDF &hkdf, QUICConnectionId cid, + const char *label, size_t label_len, size_t length) { uint8_t client_connection_id[QUICConnectionId::MAX_LENGTH]; size_t cid_len = 0; uint8_t initial_secret[512]; size_t initial_secret_len = sizeof(initial_secret); + const uint8_t *salt; + size_t salt_len; // TODO: do not extract initial secret twice QUICTypeUtil::write_QUICConnectionId(cid, client_connection_id, &cid_len); - if (hkdf.extract(initial_secret, &initial_secret_len, QUIC_VERSION_1_SALT, sizeof(QUIC_VERSION_1_SALT), client_connection_id, - cid.length()) != 1) { + switch (version) { + case 0xff00001d: // Draft-29 + salt = QUIC_VERSION_1_SALT; + salt_len = sizeof(QUIC_VERSION_1_SALT); + break; + case 0xff00001b: // Draft-27 + salt = QUIC_VERSION_1_D27_SALT; + salt_len = sizeof(QUIC_VERSION_1_D27_SALT); + break; + default: + salt = QUIC_VERSION_1_SALT; + salt_len = sizeof(QUIC_VERSION_1_SALT); + break; + } + if (hkdf.extract(initial_secret, &initial_secret_len, salt, salt_len, client_connection_id, cid.length()) != 1) { return -1; } diff --git a/iocore/net/quic/QUICKeyGenerator.h b/iocore/net/quic/QUICKeyGenerator.h index a82ad43a2a3..db2924c7864 100644 --- a/iocore/net/quic/QUICKeyGenerator.h +++ b/iocore/net/quic/QUICKeyGenerator.h @@ -43,10 +43,10 @@ class QUICKeyGenerator * Generate keys for Initial encryption level * The keys for the remaining encryption level are derived by TLS stack with "quic " prefix */ - void generate(uint8_t *hp_key, uint8_t *pp_key, uint8_t *iv, size_t *iv_len, QUICConnectionId cid); + void generate(QUICVersion version, uint8_t *hp_key, uint8_t *pp_key, uint8_t *iv, size_t *iv_len, QUICConnectionId cid); void regenerate(uint8_t *hp_key, uint8_t *pp_key, uint8_t *iv, size_t *iv_len, const uint8_t *secret, size_t secret_len, - const QUIC_EVP_CIPHER *cipher, QUICHKDF &hkdf); + const EVP_CIPHER *cipher, QUICHKDF &hkdf); private: Context _ctx = Context::SERVER; @@ -55,15 +55,15 @@ class QUICKeyGenerator size_t _last_secret_len = 0; int _generate(uint8_t *hp_key, uint8_t *pp_key, uint8_t *iv, size_t *iv_len, QUICHKDF &hkdf, const uint8_t *secret, - size_t secret_len, const QUIC_EVP_CIPHER *cipher); - int _generate_initial_secret(uint8_t *out, size_t *out_len, QUICHKDF &hkdf, QUICConnectionId cid, const char *label, - size_t label_len, size_t length); + size_t secret_len, const EVP_CIPHER *cipher); + int _generate_initial_secret(QUICVersion version, uint8_t *out, size_t *out_len, QUICHKDF &hkdf, QUICConnectionId cid, + const char *label, size_t label_len, size_t length); int _generate_key(uint8_t *out, size_t *out_len, QUICHKDF &hkdf, const uint8_t *secret, size_t secret_len, size_t key_length) const; int _generate_iv(uint8_t *out, size_t *out_len, QUICHKDF &hkdf, const uint8_t *secret, size_t secret_len, size_t iv_length) const; int _generate_hp(uint8_t *out, size_t *out_len, QUICHKDF &hkdf, const uint8_t *secret, size_t secret_len, size_t hp_length) const; - size_t _get_key_len(const QUIC_EVP_CIPHER *cipher) const; - size_t _get_iv_len(const QUIC_EVP_CIPHER *cipher) const; - const QUIC_EVP_CIPHER *_get_cipher_for_initial() const; - const QUIC_EVP_CIPHER *_get_cipher_for_protected_packet(const SSL *ssl) const; + size_t _get_key_len(const EVP_CIPHER *cipher) const; + size_t _get_iv_len(const EVP_CIPHER *cipher) const; + const EVP_CIPHER *_get_cipher_for_initial() const; + const EVP_CIPHER *_get_cipher_for_protected_packet(const SSL *ssl) const; }; diff --git a/iocore/net/quic/QUICKeyGenerator_boringssl.cc b/iocore/net/quic/QUICKeyGenerator_boringssl.cc index e2204bbd3b0..ecc9cdfa838 100644 --- a/iocore/net/quic/QUICKeyGenerator_boringssl.cc +++ b/iocore/net/quic/QUICKeyGenerator_boringssl.cc @@ -25,40 +25,41 @@ #include size_t -QUICKeyGenerator::_get_key_len(const QUIC_EVP_CIPHER *cipher) const +QUICKeyGenerator::_get_key_len(const EVP_CIPHER *cipher) const { - return EVP_AEAD_key_length(cipher); + return EVP_CIPHER_key_length(cipher); } size_t -QUICKeyGenerator::_get_iv_len(const QUIC_EVP_CIPHER *cipher) const +QUICKeyGenerator::_get_iv_len(const EVP_CIPHER *cipher) const { - return EVP_AEAD_nonce_length(cipher); + return EVP_CIPHER_iv_length(cipher); } -const QUIC_EVP_CIPHER * +const EVP_CIPHER * QUICKeyGenerator::_get_cipher_for_initial() const { - return EVP_aead_aes_128_gcm(); + return EVP_aes_128_gcm(); } -const QUIC_EVP_CIPHER * +const EVP_CIPHER * QUICKeyGenerator::_get_cipher_for_protected_packet(const SSL *ssl) const { switch (SSL_CIPHER_get_id(SSL_get_current_cipher(ssl))) { case TLS1_CK_AES_128_GCM_SHA256: - return EVP_aead_aes_128_gcm(); + return EVP_aes_128_gcm(); case TLS1_CK_AES_256_GCM_SHA384: - return EVP_aead_aes_256_gcm(); + return EVP_aes_256_gcm(); case TLS1_CK_CHACHA20_POLY1305_SHA256: - return EVP_aead_chacha20_poly1305(); + return nullptr; + // return EVP_aead_chacha20_poly1305(); default: ink_assert(false); return nullptr; } } -// SSL_HANDSHAKE_MAC_SHA256, SSL_HANDSHAKE_MAC_SHA384 are defind in `ssl/internal.h` of BoringSSL +// SSL_HANDSHAKE_MAC_SHA256, SSL_HANDSHAKE_MAC_SHA384 are defined in `ssl/internal.h` of BoringSSL /* const EVP_MD * QUICKeyGenerator::get_handshake_digest(const SSL *ssl) diff --git a/iocore/net/quic/QUICKeyGenerator_legacy.cc b/iocore/net/quic/QUICKeyGenerator_legacy.cc new file mode 100644 index 00000000000..03b46e1cf1e --- /dev/null +++ b/iocore/net/quic/QUICKeyGenerator_legacy.cc @@ -0,0 +1,63 @@ +/** @file + * + * A key generator for QUIC connection (OpenSSL specific parts) + * + * @section license License + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "tscore/ink_assert.h" +#include "QUICKeyGenerator.h" + +#include + +size_t +QUICKeyGenerator::_get_key_len(const EVP_CIPHER *cipher) const +{ + return EVP_CIPHER_key_length(cipher); +} + +size_t +QUICKeyGenerator::_get_iv_len(const EVP_CIPHER *cipher) const +{ + return EVP_CIPHER_iv_length(cipher); +} + +const EVP_CIPHER * +QUICKeyGenerator::_get_cipher_for_initial() const +{ + return EVP_aes_128_gcm(); +} + +const EVP_CIPHER * +QUICKeyGenerator::_get_cipher_for_protected_packet(const SSL *ssl) const +{ + switch (SSL_CIPHER_get_id(SSL_get_current_cipher(ssl))) { + case TLS1_3_CK_AES_128_GCM_SHA256: + return EVP_aes_128_gcm(); + case TLS1_3_CK_AES_256_GCM_SHA384: + return EVP_aes_256_gcm(); + case TLS1_3_CK_CHACHA20_POLY1305_SHA256: + return EVP_chacha20_poly1305(); + case TLS1_3_CK_AES_128_CCM_SHA256: + case TLS1_3_CK_AES_128_CCM_8_SHA256: + return EVP_aes_128_ccm(); + default: + ink_assert(false); + return nullptr; + } +} diff --git a/iocore/net/quic/QUICKeyGenerator_openssl.cc b/iocore/net/quic/QUICKeyGenerator_openssl.cc index 5d9d0294e0f..f85f4758882 100644 --- a/iocore/net/quic/QUICKeyGenerator_openssl.cc +++ b/iocore/net/quic/QUICKeyGenerator_openssl.cc @@ -24,26 +24,25 @@ #include "QUICKeyGenerator.h" #include - size_t -QUICKeyGenerator::_get_key_len(const QUIC_EVP_CIPHER *cipher) const +QUICKeyGenerator::_get_key_len(const EVP_CIPHER *cipher) const { return EVP_CIPHER_key_length(cipher); } size_t -QUICKeyGenerator::_get_iv_len(const QUIC_EVP_CIPHER *cipher) const +QUICKeyGenerator::_get_iv_len(const EVP_CIPHER *cipher) const { return EVP_CIPHER_iv_length(cipher); } -const QUIC_EVP_CIPHER * +const EVP_CIPHER * QUICKeyGenerator::_get_cipher_for_initial() const { return EVP_aes_128_gcm(); } -const QUIC_EVP_CIPHER * +const EVP_CIPHER * QUICKeyGenerator::_get_cipher_for_protected_packet(const SSL *ssl) const { switch (SSL_CIPHER_get_id(SSL_get_current_cipher(ssl))) { @@ -61,3 +60,21 @@ QUICKeyGenerator::_get_cipher_for_protected_packet(const SSL *ssl) const return nullptr; } } + +// SSL_HANDSHAKE_MAC_SHA256, SSL_HANDSHAKE_MAC_SHA384 are defined in `ssl/internal.h` of BoringSSL +/* +const EVP_MD * +QUICKeyGenerator::get_handshake_digest(const SSL *ssl) +{ + switch (SSL_CIPHER_get_id(SSL_get_current_cipher(ssl))) { + case TLS1_CK_AES_128_GCM_SHA256: + case TLS1_CK_CHACHA20_POLY1305_SHA256: + return EVP_sha256(); + case TLS1_CK_AES_256_GCM_SHA384: + return EVP_sha384(); + default: + ink_assert(false); + return nullptr; + } +} +*/ diff --git a/iocore/net/quic/QUICLossDetector.cc b/iocore/net/quic/QUICLossDetector.cc index 7cd94c0f394..702f17ac75d 100644 --- a/iocore/net/quic/QUICLossDetector.cc +++ b/iocore/net/quic/QUICLossDetector.cc @@ -29,14 +29,20 @@ #include "QUICEvents.h" #include "QUICDebugNames.h" #include "QUICFrameGenerator.h" +#include "QUICPinger.h" +#include "QUICPadder.h" +#include "QUICPacketProtectionKeyInfo.h" -#define QUICLDDebug(fmt, ...) Debug("quic_loss_detector", "[%s] " fmt, this->_info->cids().data(), ##__VA_ARGS__) -#define QUICLDVDebug(fmt, ...) Debug("v_quic_loss_detector", "[%s] " fmt, this->_info->cids().data(), ##__VA_ARGS__) +#define QUICLDDebug(fmt, ...) \ + Debug("quic_loss_detector", "[%s] " fmt, this->_context.connection_info()->cids().data(), ##__VA_ARGS__) +#define QUICLDVDebug(fmt, ...) \ + Debug("v_quic_loss_detector", "[%s] " fmt, this->_context.connection_info()->cids().data(), ##__VA_ARGS__) -QUICLossDetector::QUICLossDetector(QUICConnectionInfoProvider *info, QUICCongestionController *cc, QUICRTTMeasure *rtt_measure, - const QUICLDConfig &ld_config) - : _info(info), _rtt_measure(rtt_measure), _cc(cc) +QUICLossDetector::QUICLossDetector(QUICContext &context, QUICCongestionController *cc, QUICRTTMeasure *rtt_measure, + QUICPinger *pinger, QUICPadder *padder) + : _rtt_measure(rtt_measure), _pinger(pinger), _padder(padder), _cc(cc), _context(context) { + auto &ld_config = _context.ld_config(); this->mutex = new_ProxyMutex(); this->_loss_detection_mutex = new_ProxyMutex(); @@ -55,7 +61,7 @@ QUICLossDetector::~QUICLossDetector() this->_loss_detection_timer = nullptr; } - for (auto i = 0; i < kPacketNumberSpace; i++) { + for (auto i = 0; i < QUIC_N_PACKET_SPACES; i++) { this->_sent_packets[i].clear(); } } @@ -112,64 +118,98 @@ QUICLossDetector::handle_frame(QUICEncryptionLevel level, const QUICFrame &frame } QUICPacketNumber -QUICLossDetector::largest_acked_packet_number(QUICPacketNumberSpace pn_space) +QUICLossDetector::largest_acked_packet_number(QUICPacketNumberSpace pn_space) const { int index = static_cast(pn_space); return this->_largest_acked_packet[index]; } void -QUICLossDetector::on_packet_sent(QUICPacketInfoUPtr packet_info, bool in_flight) +QUICLossDetector::on_packet_sent(QUICSentPacketInfoUPtr packet_info, bool in_flight) { + // ADDITIONAL CODE if (packet_info->type == QUICPacketType::VERSION_NEGOTIATION) { return; } + // END OF ADDITIONAL CODE SCOPED_MUTEX_LOCK(lock, this->_loss_detection_mutex, this_ethread()); QUICPacketNumber packet_number = packet_info->packet_number; bool ack_eliciting = packet_info->ack_eliciting; - bool is_crypto_packet = packet_info->is_crypto_packet; ink_hrtime now = packet_info->time_sent; size_t sent_bytes = packet_info->sent_bytes; + QUICPacketNumberSpace pn_space = packet_info->pn_space; + + QUICLDVDebug("%s packet sent : %" PRIu64 " bytes: %lu ack_eliciting: %d", QUICDebugNames::pn_space(packet_info->pn_space), + packet_number, sent_bytes, ack_eliciting); this->_add_to_sent_packet_list(packet_number, std::move(packet_info)); if (in_flight) { - if (is_crypto_packet) { - this->_time_of_last_sent_crypto_packet = now; - } - if (ack_eliciting) { - this->_time_of_last_sent_ack_eliciting_packet = now; + this->_time_of_last_ack_eliciting_packet[static_cast(pn_space)] = now; } this->_cc->on_packet_sent(sent_bytes); this->_set_loss_detection_timer(); } } +void +QUICLossDetector::on_datagram_received() +{ + if (this->_context.connection_info()->is_at_anti_amplification_limit()) { + this->_set_loss_detection_timer(); + } +} + +void +QUICLossDetector::on_packet_number_space_discarded(QUICPacketNumberSpace pn_space) +{ + ink_assert(pn_space != QUICPacketNumberSpace::APPLICATION_DATA); + size_t bytes_in_flight = 0; + for (auto it = this->_sent_packets[static_cast(pn_space)].begin(); + it != this->_sent_packets[static_cast(pn_space)].end();) { + auto ret = this->_remove_from_sent_packet_list(it, pn_space); + auto &pi = ret.first; + if (pi->in_flight) { + bytes_in_flight += pi->sent_bytes; + } + it = ret.second; + } + this->_cc->on_packet_number_space_discarded(bytes_in_flight); + // Reset the loss detection and PTO timer + this->_time_of_last_ack_eliciting_packet[static_cast(pn_space)] = 0; + this->_loss_time[static_cast(pn_space)] = 0; + this->_rtt_measure->set_pto_count(0); + this->_set_loss_detection_timer(); + QUICLDDebug("[%s] Packets have been discarded because keys for the space are discarded", QUICDebugNames::pn_space(pn_space)); +} + void QUICLossDetector::reset() { SCOPED_MUTEX_LOCK(lock, this->_loss_detection_mutex, this_ethread()); + + // A.4. Initialization if (this->_loss_detection_timer) { this->_loss_detection_timer->cancel(); this->_loss_detection_timer = nullptr; } - - this->_ack_eliciting_outstanding = 0; - this->_crypto_outstanding = 0; - - // [draft-17 recovery] 6.4.3. Initialization - this->_time_of_last_sent_ack_eliciting_packet = 0; - this->_time_of_last_sent_crypto_packet = 0; - for (auto i = 0; i < kPacketNumberSpace; i++) { - this->_largest_acked_packet[i] = 0; - this->_loss_time[i] = 0; + this->_rtt_measure->reset(); + for (auto i = 0; i < QUIC_N_PACKET_SPACES; i++) { + this->_largest_acked_packet[i] = UINT64_MAX; + this->_time_of_last_ack_eliciting_packet[i] = 0; + this->_loss_time[i] = 0; this->_sent_packets[i].clear(); + // ADDITIONAL CODE + this->_num_packets_in_flight[i] = 0; + // END OF ADDITIONAL CODE } - this->_rtt_measure->reset(); + // ADDITIONAL CODE + this->_ack_eliciting_outstanding = 0; + // END OF ADDITIONAL CODE } void @@ -178,75 +218,88 @@ QUICLossDetector::update_ack_delay_exponent(uint8_t ack_delay_exponent) this->_ack_delay_exponent = ack_delay_exponent; } +bool +QUICLossDetector::_include_ack_eliciting(const std::vector &acked_packets) const +{ + // Find out ack_elicting packet. + // FIXME: this loop is the same as _on_ack_received's loop it would better + // to combine it. + for (const auto &packet : acked_packets) { + if (packet->ack_eliciting) { + return true; + } + } + + return false; +} + void QUICLossDetector::_on_ack_received(const QUICAckFrame &ack_frame, QUICPacketNumberSpace pn_space) { SCOPED_MUTEX_LOCK(lock, this->_loss_detection_mutex, this_ethread()); - int index = static_cast(pn_space); - this->_largest_acked_packet[index] = std::max(this->_largest_acked_packet[index], ack_frame.largest_acknowledged()); + int index = static_cast(pn_space); + if (this->_largest_acked_packet[index] == UINT64_MAX) { + this->_largest_acked_packet[index] = ack_frame.largest_acknowledged(); + } else { + this->_largest_acked_packet[index] = std::max(this->_largest_acked_packet[index], ack_frame.largest_acknowledged()); + } + + auto newly_acked_packets = this->_detect_and_remove_acked_packets(ack_frame, pn_space); + if (newly_acked_packets.empty()) { + return; + } + // If the largest acknowledged is newly acked and // ack-eliciting, update the RTT. - auto pi = this->_sent_packets[index].find(ack_frame.largest_acknowledged()); - if (pi != this->_sent_packets[index].end() && pi->second->ack_eliciting) { - ink_hrtime latest_rtt = Thread::get_hrtime() - pi->second->time_sent; + const auto &largest_acked = newly_acked_packets[0]; + if (largest_acked->packet_number == ack_frame.largest_acknowledged() && this->_include_ack_eliciting(newly_acked_packets)) { + ink_hrtime latest_rtt = Thread::get_hrtime() - largest_acked->time_sent; // _latest_rtt is nanosecond but ack_frame.ack_delay is microsecond and scaled - ink_hrtime delay = HRTIME_USECONDS(ack_frame.ack_delay() << this->_ack_delay_exponent); - this->_rtt_measure->update_rtt(latest_rtt, delay); + ink_hrtime ack_delay = 0; + if (pn_space == QUICPacketNumberSpace::APPLICATION_DATA) { + ack_delay = HRTIME_USECONDS(ack_frame.ack_delay() << this->_ack_delay_exponent); + } + this->_rtt_measure->update_rtt(latest_rtt, ack_delay); } - QUICLDVDebug("[%s] Unacked packets %lu (retransmittable %u, includes %u handshake packets)", QUICDebugNames::pn_space(pn_space), - this->_sent_packets[index].size(), this->_ack_eliciting_outstanding.load(), this->_crypto_outstanding.load()); - // if (ACK frame contains ECN information): // ProcessECN(ack) - if (ack_frame.ecn_section() != nullptr && pi != this->_sent_packets[index].end()) { - this->_cc->process_ecn(*pi->second, ack_frame.ecn_section()); + if (ack_frame.ecn_section() != nullptr) { + this->_cc->process_ecn(ack_frame, pn_space, largest_acked->time_sent); } + // ADDITIONAL CODE // Find all newly acked packets. - bool newly_acked_packets = false; - for (auto &&range : this->_determine_newly_acked_packets(ack_frame)) { - for (auto ite = this->_sent_packets[index].begin(); ite != this->_sent_packets[index].end(); /* no increment here*/) { - auto tmp_ite = ite; - tmp_ite++; - if (range.contains(ite->first)) { - newly_acked_packets = true; - this->_on_packet_acked(*(ite->second)); - } - ite = tmp_ite; - } + for (const auto &info : newly_acked_packets) { + this->_on_packet_acked(*info); } + // END OF ADDITIONAL CODE - if (!newly_acked_packets) { - return; + auto lost_packets = this->_detect_and_remove_lost_packets(pn_space); + if (!lost_packets.empty()) { + this->_cc->on_packets_lost(lost_packets); } + this->_cc->on_packets_acked(newly_acked_packets); - QUICLDVDebug("[%s] Unacked packets %lu (retransmittable %u, includes %u handshake packets)", QUICDebugNames::pn_space(pn_space), - this->_sent_packets[index].size(), this->_ack_eliciting_outstanding.load(), this->_crypto_outstanding.load()); - - this->_detect_lost_packets(pn_space); - - this->_rtt_measure->set_crypto_count(0); - this->_rtt_measure->set_pto_count(0); - - QUICLDDebug("[%s] Unacked packets %lu (retransmittable %u, includes %u handshake packets)", QUICDebugNames::pn_space(pn_space), - this->_sent_packets[index].size(), this->_ack_eliciting_outstanding.load(), this->_crypto_outstanding.load()); + QUICLDVDebug("[%s] Newly acked:%lu Lost:%lu Unacked packets:%lu (%u ack eliciting)", QUICDebugNames::pn_space(pn_space), + newly_acked_packets.size(), lost_packets.size(), this->_sent_packets[index].size(), + this->_ack_eliciting_outstanding.load()); + if (this->_peer_completed_address_validation()) { + this->_rtt_measure->set_pto_count(0); + } this->_set_loss_detection_timer(); } void -QUICLossDetector::_on_packet_acked(const QUICPacketInfo &acked_packet) +QUICLossDetector::_on_packet_acked(const QUICSentPacketInfo &acked_packet) { SCOPED_MUTEX_LOCK(lock, this->_loss_detection_mutex, this_ethread()); - // QUICLDDebug("Packet number %" PRIu64 " has been acked", acked_packet_number); + QUICLDVDebug("[%s] Packet number %" PRIu64 " has been acked", QUICDebugNames::pn_space(acked_packet.pn_space), + acked_packet.packet_number); - if (acked_packet.ack_eliciting) { - this->_cc->on_packet_acked(acked_packet); - } - - for (const QUICFrameInfo &frame_info : acked_packet.frames) { + for (const QUICSentPacketInfo::FrameInfo &frame_info : acked_packet.frames) { auto reactor = frame_info.generated_by(); if (reactor == nullptr) { continue; @@ -254,17 +307,15 @@ QUICLossDetector::_on_packet_acked(const QUICPacketInfo &acked_packet) reactor->on_frame_acked(frame_info.id()); } - - this->_remove_from_sent_packet_list(acked_packet.packet_number, acked_packet.pn_space); } ink_hrtime -QUICLossDetector::_get_earliest_loss_time(QUICPacketNumberSpace &pn_space) +QUICLossDetector::_get_loss_time_and_space(QUICPacketNumberSpace &pn_space) { - ink_hrtime time = this->_loss_time[static_cast(QUICPacketNumberSpace::Initial)]; - pn_space = QUICPacketNumberSpace::Initial; - for (auto i = 1; i < kPacketNumberSpace; i++) { - if (this->_loss_time[i] != 0 && (time != 0 || this->_loss_time[i] < time)) { + ink_hrtime time = this->_loss_time[static_cast(QUICPacketNumberSpace::INITIAL)]; + pn_space = QUICPacketNumberSpace::INITIAL; + for (auto i = 1; i < QUIC_N_PACKET_SPACES; i++) { + if (time == 0 || this->_loss_time[i] < time) { time = this->_loss_time[i]; pn_space = static_cast(i); } @@ -273,101 +324,181 @@ QUICLossDetector::_get_earliest_loss_time(QUICPacketNumberSpace &pn_space) return time; } +ink_hrtime +QUICLossDetector::_get_pto_time_and_space(QUICPacketNumberSpace &space) +{ + ink_hrtime duration = + (this->_rtt_measure->smoothed_rtt() + std::max(4 * this->_rtt_measure->rttvar(), this->_rtt_measure->k_granularity())) * + (1 << this->_rtt_measure->pto_count()); + + // Arm PTO from now when there are no inflight packets. + if (this->_num_packets_in_flight[static_cast(QUICPacketNumberSpace::INITIAL)].load() == 0 && + this->_num_packets_in_flight[static_cast(QUICPacketNumberSpace::HANDSHAKE)].load() == 0 && + this->_num_packets_in_flight[static_cast(QUICPacketNumberSpace::APPLICATION_DATA)].load() == 0) { + ink_assert(!this->_peer_completed_address_validation()); + if (this->_context.connection_info()->has_keys_for(QUICPacketNumberSpace::HANDSHAKE)) { + space = QUICPacketNumberSpace::HANDSHAKE; + return Thread::get_hrtime() + duration; + } else { + space = QUICPacketNumberSpace::INITIAL; + return Thread::get_hrtime() + duration; + } + } + ink_hrtime pto_timeout = INT64_MAX; + QUICPacketNumberSpace pto_space = QUICPacketNumberSpace::INITIAL; + for (int i = 0; i < QUIC_N_PACKET_SPACES; ++i) { + if (this->_num_packets_in_flight[i].load() == 0) { + continue; + } + if (i == static_cast(QUICPacketNumberSpace::APPLICATION_DATA)) { + // Skip ApplicationData until handshake complete. + if (!this->_context.connection_info()->is_address_validation_completed()) { + space = pto_space; + return pto_timeout; + } + // Include max_ack_delay and backoff for ApplicationData. + // FIXME should be set by transport parameters + duration += this->_rtt_measure->max_ack_delay() * (1 << this->_rtt_measure->pto_count()); + } + + ink_hrtime t = this->_time_of_last_ack_eliciting_packet[i] + duration; + if (t < pto_timeout) { + pto_timeout = t; + pto_space = QUICPacketNumberSpace(i); + } + } + space = pto_space; + return pto_timeout; +} + +bool +QUICLossDetector::_peer_completed_address_validation() const +{ + return this->_context.connection_info()->is_address_validation_completed(); +} + void QUICLossDetector::_set_loss_detection_timer() { - // Don't arm the alarm if there are no packets with retransmittable data in flight. - // -- MODIFIED CODE -- - // In psuedocode, `bytes_in_flight` is used, but we're tracking "retransmittable data in flight" by `_ack_eliciting_outstanding` - if (this->_ack_eliciting_outstanding == 0) { - if (this->_loss_detection_timer) { - this->_loss_detection_alarm_at = 0; - this->_loss_detection_timer->cancel(); - this->_loss_detection_timer = nullptr; - QUICLDDebug("Loss detection alarm has been unset"); + std::function update_timer = [this](ink_hrtime time) { + this->_loss_detection_alarm_at = time; + if (!this->_loss_detection_timer) { + this->_loss_detection_timer = eventProcessor.schedule_every(this, HRTIME_MSECONDS(25)); } + }; + std::function cancel_timer = [this]() { + this->_loss_detection_alarm_at = 0; + this->_loss_detection_timer->cancel(); + this->_loss_detection_timer = nullptr; + }; + + QUICPacketNumberSpace pn_space; + ink_hrtime earliest_loss_time = this->_get_loss_time_and_space(pn_space); + if (earliest_loss_time != 0) { + update_timer(earliest_loss_time); + QUICLDDebug("[%s] time threshold loss detection timer: %" PRId64 "ms", QUICDebugNames::pn_space(pn_space), + (this->_loss_detection_alarm_at - Thread::get_hrtime()) / HRTIME_MSECOND); return; } - // -- END OF MODIFIED CODE -- - QUICPacketNumberSpace pn_space; - ink_hrtime loss_time = this->_get_earliest_loss_time(pn_space); - if (loss_time != 0) { - // Time threshold loss detection. - this->_loss_detection_alarm_at = loss_time; - QUICLDDebug("[%s] time threshold loss detection timer: %" PRId64, QUICDebugNames::pn_space(pn_space), - this->_loss_detection_alarm_at); - } else if (this->_crypto_outstanding) { - // Handshake retransmission alarm. - this->_loss_detection_alarm_at = this->_time_of_last_sent_crypto_packet + this->_rtt_measure->handshake_retransmit_timeout(); - QUICLDDebug("%s crypto packet alarm will be set: %" PRId64, QUICDebugNames::pn_space(pn_space), this->_loss_detection_alarm_at); - // -- ADDITIONAL CODE -- - // In psudocode returning here, but we don't do for scheduling _loss_detection_alarm event. - // -- END OF ADDITIONAL CODE -- - } else { - // PTO Duration - this->_loss_detection_alarm_at = this->_time_of_last_sent_ack_eliciting_packet + this->_rtt_measure->current_pto_period(); - QUICLDDebug("[%s] PTO timeout will be set: %" PRId64, QUICDebugNames::pn_space(pn_space), this->_loss_detection_alarm_at); + if (this->_context.connection_info()->is_at_anti_amplification_limit()) { + if (this->_loss_detection_timer) { + cancel_timer(); + QUICLDDebug("Loss detection alarm has been unset because of anti-amplification limit"); + return; + } } - if (!this->_loss_detection_timer) { - this->_loss_detection_timer = eventProcessor.schedule_every(this, HRTIME_MSECONDS(25)); + // Don't arm the alarm if there are no packets with retransmittable data in flight. + if (this->_ack_eliciting_outstanding == 0 && this->_peer_completed_address_validation()) { + if (this->_loss_detection_timer) { + cancel_timer(); + QUICLDDebug("Loss detection alarm has been unset because of no ack eliciting packets outstanding"); + } + return; } + + // PTO Duration + ink_hrtime timeout = this->_get_pto_time_and_space(pn_space); + update_timer(timeout); + QUICLDVDebug("[%s] PTO timeout has been set: %" PRId64 "ms", QUICDebugNames::pn_space(pn_space), + (timeout - this->_time_of_last_ack_eliciting_packet[static_cast(pn_space)]) / HRTIME_MSECOND); } void QUICLossDetector::_on_loss_detection_timeout() { QUICPacketNumberSpace pn_space; - ink_hrtime loss_time = this->_get_earliest_loss_time(pn_space); - if (loss_time != 0) { + ink_hrtime earliest_loss_time = this->_get_loss_time_and_space(pn_space); + if (earliest_loss_time != 0) { // Time threshold loss Detection - this->_detect_lost_packets(pn_space); - } else if (this->_crypto_outstanding) { - // Handshake retransmission alarm. - this->_retransmit_all_unacked_crypto_data(); - this->_rtt_measure->set_crypto_count(this->_rtt_measure->crypto_count() + 1); + auto lost_packets = this->_detect_and_remove_lost_packets(pn_space); + ink_assert(!lost_packets.empty()); + this->_cc->on_packets_lost(lost_packets); + this->_set_loss_detection_timer(); + return; + } + + if (this->_cc->bytes_in_flight() > 0) { + // PTO. Send new data if available, else retransmit old data. + // If neither is available, send a single PING frame. + QUICPacketNumberSpace pns; + this->_get_pto_time_and_space(pns); + this->_send_one_or_two_ack_eliciting_packet(pns); } else { - QUICLDVDebug("PTO"); - this->_send_two_packets(); - this->_rtt_measure->set_pto_count(this->_rtt_measure->pto_count() + 1); + // This assertion is on draft-29 but not correct + // Keep it as a comment for now to not add it back + // ink_assert(this->_is_client_without_one_rtt_key()); + + // Client sends an anti-deadlock packet: Initial is padded + // to earn more anti-amplification credit, + // a Handshake packet proves address ownership. + if (this->_context.key_info()->is_encryption_key_available(QUICKeyPhase::HANDSHAKE)) { + this->_send_one_ack_eliciting_handshake_packet(); + } else { + this->_send_one_ack_eliciting_padded_initial_packet(); + } } - QUICLDDebug("[%s] Unacked packets %lu (retransmittable %u, includes %u handshake packets)", QUICDebugNames::pn_space(pn_space), - this->_sent_packets[static_cast(pn_space)].size(), this->_ack_eliciting_outstanding.load(), - this->_crypto_outstanding.load()); + this->_rtt_measure->set_pto_count(this->_rtt_measure->pto_count() + 1); + this->_set_loss_detection_timer(); + + QUICLDDebug("[%s] Unacked packets %lu (ack_eliciting %u)", QUICDebugNames::pn_space(pn_space), + this->_sent_packets[static_cast(pn_space)].size(), this->_ack_eliciting_outstanding.load()); if (is_debug_tag_set("v_quic_loss_detector")) { for (auto i = 0; i < 3; i++) { for (auto &unacked : this->_sent_packets[i]) { - QUICLDVDebug("[%s] #%" PRIu64 " is_crypto=%i ack_eliciting=%i size=%zu %u %u", - QUICDebugNames::pn_space(static_cast(i)), unacked.first, - unacked.second->is_crypto_packet, unacked.second->ack_eliciting, unacked.second->sent_bytes, - this->_ack_eliciting_outstanding.load(), this->_crypto_outstanding.load()); + QUICLDVDebug("[%s] #%" PRIu64 " ack_eliciting=%i size=%zu %u", + QUICDebugNames::pn_space(static_cast(i)), unacked.first, unacked.second->ack_eliciting, + unacked.second->sent_bytes, this->_ack_eliciting_outstanding.load()); } } } - - this->_set_loss_detection_timer(); } -void -QUICLossDetector::_detect_lost_packets(QUICPacketNumberSpace pn_space) +std::map +QUICLossDetector::_detect_and_remove_lost_packets(QUICPacketNumberSpace pn_space) { SCOPED_MUTEX_LOCK(lock, this->_loss_detection_mutex, this_ethread()); + ink_assert(this->_largest_acked_packet[static_cast(pn_space)] != UINT64_MAX); + this->_loss_time[static_cast(pn_space)] = 0; + std::map lost_packets; ink_hrtime loss_delay = this->_k_time_threshold * std::max(this->_rtt_measure->latest_rtt(), this->_rtt_measure->smoothed_rtt()); - std::map lost_packets; + + // Minimum time of kGranularity before packets are deemed lost. + loss_delay = std::max(loss_delay, this->_rtt_measure->k_granularity()); // Packets sent before this time are deemed lost. ink_hrtime lost_send_time = Thread::get_hrtime() - loss_delay; // Packets with packet numbers before this are deemed lost. - QUICPacketNumber lost_pn = this->_largest_acked_packet[static_cast(pn_space)] - this->_k_packet_threshold; + // QUICPacketNumber lost_pn = this->_largest_acked_packet[static_cast(pn_space)] - this->_k_packet_threshold; for (auto it = this->_sent_packets[static_cast(pn_space)].begin(); - it != this->_sent_packets[static_cast(pn_space)].end(); ++it) { + it != this->_sent_packets[static_cast(pn_space)].end();) { if (it->first > this->_largest_acked_packet[static_cast(pn_space)]) { // the spec uses continue but we can break here because the _sent_packets is sorted by packet_number. break; @@ -376,8 +507,9 @@ QUICLossDetector::_detect_lost_packets(QUICPacketNumberSpace pn_space) auto &unacked = it->second; // Mark packet as lost, or set time when it should be marked. - if (unacked->time_sent < lost_send_time || unacked->packet_number < lost_pn) { - if (unacked->time_sent < lost_send_time) { + if (unacked->time_sent <= lost_send_time || + this->_largest_acked_packet[static_cast(pn_space)] >= unacked->packet_number + this->_k_packet_threshold) { + if (unacked->time_sent <= lost_send_time) { QUICLDDebug("[%s] Lost: time since sent is too long (#%" PRId64 " sent=%" PRId64 ", delay=%" PRId64 ", fraction=%lf, lrtt=%" PRId64 ", srtt=%" PRId64 ")", QUICDebugNames::pn_space(pn_space), it->first, unacked->time_sent, lost_send_time, this->_k_time_threshold, @@ -388,75 +520,87 @@ QUICLossDetector::_detect_lost_packets(QUICPacketNumberSpace pn_space) this->_k_packet_threshold); } - if (unacked->in_flight) { - lost_packets.insert({it->first, it->second.get()}); - } else if (this->_loss_time[static_cast(pn_space)] == 0) { + auto ret = this->_remove_from_sent_packet_list(it, pn_space); + auto pi = std::move(ret.first); + it = ret.second; + if (pi->in_flight) { + this->_context.trigger(QUICContext::CallbackEvent::PACKET_LOST, *pi); + lost_packets.emplace(pi->packet_number, std::move(pi)); + } + + } else { + if (this->_loss_time[static_cast(pn_space)] == 0) { this->_loss_time[static_cast(pn_space)] = unacked->time_sent + loss_delay; } else { this->_loss_time[static_cast(pn_space)] = std::min(this->_loss_time[static_cast(pn_space)], unacked->time_sent + loss_delay); } + ++it; } } - // Inform the congestion controller of lost packets and - // lets it decide whether to retransmit immediately. + // -- ADDITIONAL CODE -- + // Not sure how we can get feedback from congestion control and when we should retransmit the lost packets but we need to send + // them somewhere. + // I couldn't find the place so just send them here for now. if (!lost_packets.empty()) { - this->_cc->on_packets_lost(lost_packets); - for (auto lost_packet : lost_packets) { - // -- ADDITIONAL CODE -- - // Not sure how we can get feedback from congestion control and when we should retransmit the lost packets but we need to send - // them somewhere. - // I couldn't find the place so just send them here for now. + for (const auto &lost_packet : lost_packets) { this->_retransmit_lost_packet(*lost_packet.second); - // -- END OF ADDITIONAL CODE -- - // -- ADDITIONAL CODE -- - this->_remove_from_sent_packet_list(lost_packet.first, pn_space); - // -- END OF ADDITIONAL CODE -- } } + // -- END OF ADDITIONAL CODE -- + + return lost_packets; } // ===== Functions below are used on the spec but there're no pseudo code ===== void -QUICLossDetector::_retransmit_all_unacked_crypto_data() +QUICLossDetector::_send_packet(QUICEncryptionLevel level, bool padded) { - SCOPED_MUTEX_LOCK(lock, this->_loss_detection_mutex, this_ethread()); - for (auto i = 0; i < kPacketNumberSpace; i++) { - std::set retransmitted_crypto_packets; - std::map lost_packets; - for (auto &info : this->_sent_packets[i]) { - if (info.second->is_crypto_packet) { - retransmitted_crypto_packets.insert(info.first); - this->_retransmit_lost_packet(*info.second); - lost_packets.insert({info.first, info.second.get()}); - } - } - - this->_cc->on_packets_lost(lost_packets); - for (auto packet_number : retransmitted_crypto_packets) { - this->_remove_from_sent_packet_list(packet_number, static_cast(i)); - } + if (padded) { + this->_padder->request(level); + } else { + this->_pinger->request(level); } + this->_cc->add_extra_credit(); } void -QUICLossDetector::_send_two_packets() +QUICLossDetector::_send_one_or_two_ack_eliciting_packet(QUICPacketNumberSpace pn_space) { - SCOPED_MUTEX_LOCK(lock, this->_loss_detection_mutex, this_ethread()); - // TODO sent ping + this->_send_packet(QUICEncryptionLevel::ONE_RTT); + this->_send_packet(QUICEncryptionLevel::ONE_RTT); + ink_assert(this->_pinger->count(QUICEncryptionLevel::ONE_RTT) >= 2); + QUICLDDebug("[%s] send ping frame %" PRIu64, QUICDebugNames::encryption_level(QUICEncryptionLevel::ONE_RTT), + this->_pinger->count(QUICEncryptionLevel::ONE_RTT)); +} + +void +QUICLossDetector::_send_one_ack_eliciting_handshake_packet() +{ + this->_send_packet(QUICEncryptionLevel::HANDSHAKE); + QUICLDDebug("[%s] send handshake packet: ping count=%" PRIu64, QUICDebugNames::encryption_level(QUICEncryptionLevel::HANDSHAKE), + this->_pinger->count(QUICEncryptionLevel::HANDSHAKE)); +} + +void +QUICLossDetector::_send_one_ack_eliciting_padded_initial_packet() +{ + this->_send_packet(QUICEncryptionLevel::INITIAL, true); + QUICLDDebug("[%s] send PADDING frame: ping count=%" PRIu64, QUICDebugNames::encryption_level(QUICEncryptionLevel::INITIAL), + this->_pinger->count(QUICEncryptionLevel::INITIAL)); } // ===== Functions below are helper functions ===== void -QUICLossDetector::_retransmit_lost_packet(QUICPacketInfo &packet_info) +QUICLossDetector::_retransmit_lost_packet(const QUICSentPacketInfo &packet_info) { SCOPED_MUTEX_LOCK(lock, this->_loss_detection_mutex, this_ethread()); QUICLDDebug("Retransmit %s packet #%" PRIu64, QUICDebugNames::packet_type(packet_info.type), packet_info.packet_number); - for (QUICFrameInfo &frame_info : packet_info.frames) { + for (const QUICSentPacketInfo::FrameInfo &frame_info : packet_info.frames) { auto reactor = frame_info.generated_by(); if (reactor == nullptr) { continue; @@ -466,10 +610,13 @@ QUICLossDetector::_retransmit_lost_packet(QUICPacketInfo &packet_info) } } -std::set -QUICLossDetector::_determine_newly_acked_packets(const QUICAckFrame &ack_frame) +std::vector +QUICLossDetector::_detect_and_remove_acked_packets(const QUICAckFrame &ack_frame, QUICPacketNumberSpace pn_space) { + std::vector packets; std::set numbers; + int index = static_cast(pn_space); + QUICPacketNumber x = ack_frame.largest_acknowledged(); numbers.insert({x, static_cast(x) - ack_frame.ack_block_section()->first_ack_block()}); x -= ack_frame.ack_block_section()->first_ack_block() + 1; @@ -479,69 +626,77 @@ QUICLossDetector::_determine_newly_acked_packets(const QUICAckFrame &ack_frame) x -= block.length() + 1; } - return numbers; + for (auto &&range : numbers) { + for (auto ite = this->_sent_packets[index].begin(); ite != this->_sent_packets[index].end();) { + if (range.contains(ite->first)) { + auto ret = this->_remove_from_sent_packet_list(ite, pn_space); + packets.push_back(std::move(ret.first)); + ite = ret.second; + } else { + ++ite; + } + } + } + + return packets; } void -QUICLossDetector::_add_to_sent_packet_list(QUICPacketNumber packet_number, QUICPacketInfoUPtr packet_info) +QUICLossDetector::_add_to_sent_packet_list(QUICPacketNumber packet_number, QUICSentPacketInfoUPtr packet_info) { SCOPED_MUTEX_LOCK(lock, this->_loss_detection_mutex, this_ethread()); // Add to the list int index = static_cast(packet_info->pn_space); - this->_sent_packets[index].insert(std::pair(packet_number, std::move(packet_info))); + this->_sent_packets[index].insert(std::pair(packet_number, std::move(packet_info))); // Increment counters auto ite = this->_sent_packets[index].find(packet_number); if (ite != this->_sent_packets[index].end()) { - if (ite->second->is_crypto_packet) { - ++this->_crypto_outstanding; - ink_assert(this->_crypto_outstanding.load() > 0); - } if (ite->second->ack_eliciting) { ++this->_ack_eliciting_outstanding; ink_assert(this->_ack_eliciting_outstanding.load() > 0); } + if (ite->second->in_flight) { + ++this->_num_packets_in_flight[index]; + } } } -void -QUICLossDetector::_remove_from_sent_packet_list(QUICPacketNumber packet_number, QUICPacketNumberSpace pn_space) -{ - SCOPED_MUTEX_LOCK(lock, this->_loss_detection_mutex, this_ethread()); - - auto ite = this->_sent_packets[static_cast(pn_space)].find(packet_number); - this->_decrement_outstanding_counters(ite, pn_space); - this->_sent_packets[static_cast(pn_space)].erase(packet_number); -} - -std::map::iterator -QUICLossDetector::_remove_from_sent_packet_list(std::map::iterator it, +std::pair::iterator> +QUICLossDetector::_remove_from_sent_packet_list(std::map::iterator it, QUICPacketNumberSpace pn_space) { SCOPED_MUTEX_LOCK(lock, this->_loss_detection_mutex, this_ethread()); - this->_decrement_outstanding_counters(it, pn_space); - return this->_sent_packets[static_cast(pn_space)].erase(it); + this->_decrement_counters(it, pn_space); + auto pi = std::move(it->second); + return {std::move(pi), this->_sent_packets[static_cast(pn_space)].erase(it)}; } void -QUICLossDetector::_decrement_outstanding_counters(std::map::iterator it, - QUICPacketNumberSpace pn_space) +QUICLossDetector::_decrement_counters(std::map::iterator it, + QUICPacketNumberSpace pn_space) { if (it != this->_sent_packets[static_cast(pn_space)].end()) { - // Decrement counters - if (it->second->is_crypto_packet) { - ink_assert(this->_crypto_outstanding.load() > 0); - --this->_crypto_outstanding; - } if (it->second->ack_eliciting) { ink_assert(this->_ack_eliciting_outstanding.load() > 0); --this->_ack_eliciting_outstanding; } + --this->_num_packets_in_flight[static_cast(pn_space)]; } } +bool +QUICLossDetector::_is_client_without_one_rtt_key() const +{ + return this->_context.connection_info()->direction() == NET_VCONNECTION_OUT && + !((this->_context.key_info()->is_encryption_key_available(QUICKeyPhase::PHASE_1) && + this->_context.key_info()->is_decryption_key_available(QUICKeyPhase::PHASE_1)) || + (this->_context.key_info()->is_encryption_key_available(QUICKeyPhase::PHASE_0) && + this->_context.key_info()->is_decryption_key_available(QUICKeyPhase::PHASE_0))); +} + // // QUICRTTMeasure // @@ -566,30 +721,29 @@ QUICRTTMeasure::smoothed_rtt() const void QUICRTTMeasure::update_rtt(ink_hrtime latest_rtt, ink_hrtime ack_delay) { - // additional code this->_latest_rtt = latest_rtt; + if (this->_is_first_sample) { + this->_min_rtt = this->_latest_rtt; + this->_smoothed_rtt = this->_latest_rtt; + this->_rttvar = this->_latest_rtt / 2; + this->_is_first_sample = false; + return; + } + // min_rtt ignores ack delay. this->_min_rtt = std::min(this->_min_rtt, latest_rtt); // Limit ack_delay by max_ack_delay ack_delay = std::min(ack_delay, this->_max_ack_delay); // Adjust for ack delay if it's plausible. - if (this->_latest_rtt - this->_min_rtt > ack_delay) { - this->_latest_rtt -= ack_delay; - - // the newest spec has removed the max_ack_delay assignment. but we need to assign it in somewhere - // this code is from draft-19 - this->_max_ack_delay = std::max(ack_delay, this->_max_ack_delay); + auto adjusted_rtt = this->_latest_rtt; + if (adjusted_rtt > this->_min_rtt + ack_delay) { + adjusted_rtt -= ack_delay; } + // Based on {{RFC6298}}. - if (this->_smoothed_rtt == 0) { - this->_smoothed_rtt = latest_rtt; - this->_rttvar = latest_rtt / 2.0; - } else { - double rttvar_sample = ABS(this->_smoothed_rtt - latest_rtt); - this->_rttvar = 3.0 / 4.0 * this->_rttvar + 1.0 / 4.0 * rttvar_sample; - this->_smoothed_rtt = 7.0 / 8.0 * this->_smoothed_rtt + 1.0 / 8.0 * latest_rtt; - } + this->_rttvar = 3.0 / 4.0 * this->_rttvar + 1.0 / 4.0 * ABS(this->_smoothed_rtt - adjusted_rtt); + this->_smoothed_rtt = 7.0 / 8.0 * this->_smoothed_rtt + 1.0 / 8.0 * adjusted_rtt; } ink_hrtime @@ -607,35 +761,19 @@ ink_hrtime QUICRTTMeasure::congestion_period(uint32_t threshold) const { ink_hrtime pto = this->_smoothed_rtt + std::max(this->_rttvar * 4, this->_k_granularity); - return pto * (1 << (threshold - 1)); -} - -ink_hrtime -QUICRTTMeasure::handshake_retransmit_timeout() const -{ - // Handshake retransmission alarm. - ink_hrtime timeout = 0; - if (this->_smoothed_rtt == 0) { - timeout = 2 * this->_k_initial_rtt; - } else { - timeout = 2 * this->_smoothed_rtt; - } - timeout = std::max(timeout, this->_k_granularity); - timeout = timeout * (1 << this->_crypto_count); - - return timeout; + return pto * threshold; } void -QUICRTTMeasure::set_crypto_count(uint32_t count) +QUICRTTMeasure::set_pto_count(uint32_t count) { - this->_crypto_count = count; + this->_pto_count = count; } void -QUICRTTMeasure::set_pto_count(uint32_t count) +QUICRTTMeasure::set_max_ack_delay(ink_hrtime max_ack_delay) { - this->_pto_count = count; + this->_max_ack_delay = max_ack_delay; } ink_hrtime @@ -651,23 +789,30 @@ QUICRTTMeasure::latest_rtt() const } uint32_t -QUICRTTMeasure::crypto_count() const +QUICRTTMeasure::pto_count() const { - return this->_crypto_count; + return this->_pto_count; } -uint32_t -QUICRTTMeasure::pto_count() const +ink_hrtime +QUICRTTMeasure::max_ack_delay() const { - return this->_pto_count; + return this->_max_ack_delay; +} + +ink_hrtime +QUICRTTMeasure::k_granularity() const +{ + return this->_k_granularity; } void QUICRTTMeasure::reset() { - this->_crypto_count = 0; + // A.4. Initialization this->_pto_count = 0; - this->_smoothed_rtt = 0; - this->_rttvar = 0; - this->_min_rtt = INT64_MAX; + this->_latest_rtt = 0; + this->_smoothed_rtt = this->_k_initial_rtt; + this->_rttvar = this->_k_initial_rtt / 2.0; + this->_min_rtt = 0; } diff --git a/iocore/net/quic/QUICLossDetector.h b/iocore/net/quic/QUICLossDetector.h index 1b7a536a7c6..5ed1c228b7a 100644 --- a/iocore/net/quic/QUICLossDetector.h +++ b/iocore/net/quic/QUICLossDetector.h @@ -36,101 +36,33 @@ #include "QUICFrame.h" #include "QUICFrameHandler.h" #include "QUICConnection.h" +#include "QUICContext.h" +#include "QUICCongestionController.h" +class QUICPadder; +class QUICPinger; class QUICLossDetector; class QUICRTTMeasure; -struct QUICPacketInfo { - // 6.3.1. Sent Packet Fields - QUICPacketNumber packet_number; - ink_hrtime time_sent; - bool ack_eliciting; - bool is_crypto_packet; - bool in_flight; - size_t sent_bytes; - - // addition - QUICPacketType type; - std::vector frames; - QUICPacketNumberSpace pn_space; - // end -}; - -using QUICPacketInfoUPtr = std::unique_ptr; - -class QUICRTTProvider -{ -public: - virtual ink_hrtime smoothed_rtt() const = 0; - virtual ink_hrtime rttvar() const = 0; - virtual ink_hrtime latest_rtt() const = 0; - - virtual ink_hrtime congestion_period(uint32_t threshold) const = 0; -}; - -class QUICCongestionController -{ -public: - QUICCongestionController(const QUICRTTProvider &rtt_provider, QUICConnectionInfoProvider *info, const QUICCCConfig &cc_config); - virtual ~QUICCongestionController() {} - void on_packet_sent(size_t bytes_sent); - void on_packet_acked(const QUICPacketInfo &acked_packet); - virtual void on_packets_lost(const std::map &packets); - void on_retransmission_timeout_verified(); - void process_ecn(const QUICPacketInfo &acked_largest_packet, const QUICAckFrame::EcnSection *ecn_section); - bool check_credit() const; - uint32_t open_window() const; - void reset(); - bool is_app_limited(); - - // Debug - uint32_t bytes_in_flight() const; - uint32_t congestion_window() const; - uint32_t current_ssthresh() const; - -private: - Ptr _cc_mutex; - - void _congestion_event(ink_hrtime sent_time); - bool _in_persistent_congestion(const std::map &lost_packets, - QUICPacketInfo *largest_lost_packet); - bool _in_window_lost(const std::map &lost_packets, QUICPacketInfo *largest_lost_packet, - ink_hrtime period) const; - - // [draft-17 recovery] 7.9.1. Constants of interest - // Values will be loaded from records.config via QUICConfig at constructor - uint32_t _k_max_datagram_size = 0; - uint32_t _k_initial_window = 0; - uint32_t _k_minimum_window = 0; - float _k_loss_reduction_factor = 0.0; - uint32_t _k_persistent_congestion_threshold = 0; - - // [draft-17 recovery] 7.9.2. Variables of interest - uint32_t _ecn_ce_counter = 0; - uint32_t _bytes_in_flight = 0; - uint32_t _congestion_window = 0; - ink_hrtime _recovery_start_time = 0; - uint32_t _ssthresh = UINT32_MAX; - - QUICConnectionInfoProvider *_info = nullptr; - const QUICRTTProvider &_rtt_provider; - - bool _in_recovery(ink_hrtime sent_time); -}; - class QUICLossDetector : public Continuation, public QUICFrameHandler { public: - QUICLossDetector(QUICConnectionInfoProvider *info, QUICCongestionController *cc, QUICRTTMeasure *rtt_measure, - const QUICLDConfig &ld_config); + QUICLossDetector(QUICContext &context, QUICCongestionController *cc, QUICRTTMeasure *rtt_measure, QUICPinger *pinger, + QUICPadder *padder); ~QUICLossDetector(); int event_handler(int event, Event *edata); + // QUICFrameHandler interface std::vector interests() override; virtual QUICConnectionErrorUPtr handle_frame(QUICEncryptionLevel level, const QUICFrame &frame) override; - void on_packet_sent(QUICPacketInfoUPtr packet_info, bool in_flight = true); - QUICPacketNumber largest_acked_packet_number(QUICPacketNumberSpace pn_space); + + void on_packet_sent(QUICSentPacketInfoUPtr packet_info, bool in_flight = true); + void on_datagram_received(); + // OnPacketNumberSpaceDiscarded is on Congestion Control section but having it here makes more sense because most processes are + // for LD. + void on_packet_number_space_discarded(QUICPacketNumberSpace pn_space); + QUICPacketNumber largest_acked_packet_number(QUICPacketNumberSpace pn_space) const; void update_ack_delay_exponent(uint8_t ack_delay_exponent); void reset(); @@ -139,29 +71,32 @@ class QUICLossDetector : public Continuation, public QUICFrameHandler uint8_t _ack_delay_exponent = 3; - // [draft-17 recovery] 6.4.1. Constants of interest + // Recovery A.2. Constants of Interest // Values will be loaded from records.config via QUICConfig at constructor uint32_t _k_packet_threshold = 0; float _k_time_threshold = 0.0; + // kGranularity, kInitialRtt are defined in QUICRTTMeasure + QUICRTTMeasure *_rtt_measure = nullptr; + // kPacketNumberSpace is defined as QUICPacketNumberSpace on QUICTypes.h - // [draft-11 recovery] 3.5.2. Variables of interest + // Recovery A.3. Variables of interest // Keep the order as the same as the spec so that we can see the difference easily. - Action *_loss_detection_timer = nullptr; - ink_hrtime _time_of_last_sent_ack_eliciting_packet = 0; - ink_hrtime _time_of_last_sent_crypto_packet = 0; - ink_hrtime _loss_time[kPacketNumberSpace] = {0}; - QUICPacketNumber _largest_acked_packet[kPacketNumberSpace] = {0}; - std::map _sent_packets[kPacketNumberSpace]; + // latest_rtt, smoothed_rtt, rttvar, min_rtt and max_ack_delay are defined in QUICRttMeasure + Action *_loss_detection_timer = nullptr; + // pto_count is defined in QUICRttMeasure + ink_hrtime _time_of_last_ack_eliciting_packet[QUIC_N_PACKET_SPACES] = {0}; + QUICPacketNumber _largest_acked_packet[QUIC_N_PACKET_SPACES] = {0}; + ink_hrtime _loss_time[QUIC_N_PACKET_SPACES] = {0}; + std::map _sent_packets[QUIC_N_PACKET_SPACES]; // These are not defined on the spec but expected to be count // These counter have to be updated when inserting / erasing packets from _sent_packets with following functions. - std::atomic _crypto_outstanding; std::atomic _ack_eliciting_outstanding; - void _add_to_sent_packet_list(QUICPacketNumber packet_number, std::unique_ptr packet_info); - void _remove_from_sent_packet_list(QUICPacketNumber packet_number, QUICPacketNumberSpace pn_space); - std::map::iterator _remove_from_sent_packet_list( - std::map::iterator it, QUICPacketNumberSpace pn_space); - void _decrement_outstanding_counters(std::map::iterator it, QUICPacketNumberSpace pn_space); + std::atomic _num_packets_in_flight[QUIC_N_PACKET_SPACES]; + void _add_to_sent_packet_list(QUICPacketNumber packet_number, std::unique_ptr packet_info); + std::pair::iterator> _remove_from_sent_packet_list( + std::map::iterator it, QUICPacketNumberSpace pn_space); + void _decrement_counters(std::map::iterator it, QUICPacketNumberSpace pn_space); /* * Because this alarm will be reset on every packet transmission, to reduce number of events, @@ -170,29 +105,39 @@ class QUICLossDetector : public Continuation, public QUICFrameHandler ink_hrtime _loss_detection_alarm_at = 0; void _on_ack_received(const QUICAckFrame &ack_frame, QUICPacketNumberSpace pn_space); - void _on_packet_acked(const QUICPacketInfo &acked_packet); - void _detect_lost_packets(QUICPacketNumberSpace pn_space); + void _on_packet_acked(const QUICSentPacketInfo &acked_packet); + std::map _detect_and_remove_lost_packets(QUICPacketNumberSpace pn_space); void _set_loss_detection_timer(); void _on_loss_detection_timeout(); - void _retransmit_lost_packet(QUICPacketInfo &packet_info); + void _retransmit_lost_packet(const QUICSentPacketInfo &packet_info); - ink_hrtime _get_earliest_loss_time(QUICPacketNumberSpace &pn_space); + ink_hrtime _get_loss_time_and_space(QUICPacketNumberSpace &space); + ink_hrtime _get_pto_time_and_space(QUICPacketNumberSpace &space); + bool _peer_completed_address_validation() const; - std::set _determine_newly_acked_packets(const QUICAckFrame &ack_frame); + std::vector _detect_and_remove_acked_packets(const QUICAckFrame &ack_frame, + QUICPacketNumberSpace pn_space); + bool _include_ack_eliciting(const std::vector &acked_packets) const; - void _retransmit_all_unacked_crypto_data(); - void _send_one_packet(); - void _send_two_packets(); + void _send_one_or_two_ack_eliciting_packet(QUICPacketNumberSpace pn_space); + void _send_one_ack_eliciting_handshake_packet(); + void _send_one_ack_eliciting_padded_initial_packet(); - QUICConnectionInfoProvider *_info = nullptr; - QUICRTTMeasure *_rtt_measure = nullptr; - QUICCongestionController *_cc = nullptr; + void _send_packet(QUICEncryptionLevel level, bool padded = false); + + bool _is_client_without_one_rtt_key() const; + + QUICPinger *_pinger = nullptr; + QUICPadder *_padder = nullptr; + QUICCongestionController *_cc = nullptr; + + QUICContext &_context; }; class QUICRTTMeasure : public QUICRTTProvider { public: - // use `friend` so ld can acesss RTTMeasure. + // use `friend` so ld can access RTTMeasure. // friend QUICLossDetector; QUICRTTMeasure(const QUICLDConfig &ld_config); @@ -201,7 +146,6 @@ class QUICRTTMeasure : public QUICRTTProvider void init(const QUICLDConfig &ld_config); // period - ink_hrtime handshake_retransmit_timeout() const; ink_hrtime current_pto_period() const; ink_hrtime congestion_period(uint32_t threshold) const override; @@ -211,27 +155,29 @@ class QUICRTTMeasure : public QUICRTTProvider ink_hrtime latest_rtt() const override; uint32_t pto_count() const; - uint32_t crypto_count() const; + ink_hrtime max_ack_delay() const; - void set_crypto_count(uint32_t count); void set_pto_count(uint32_t count); + void set_max_ack_delay(ink_hrtime max_ack_delay); void update_rtt(ink_hrtime latest_rtt, ink_hrtime ack_delay); void reset(); + ink_hrtime k_granularity() const; + private: - // related to rtt calculate - uint32_t _crypto_count = 0; - uint32_t _pto_count = 0; - ink_hrtime _max_ack_delay = 0; + bool _is_first_sample = false; - // rtt vars + // A.3. Variables of interest ink_hrtime _latest_rtt = 0; ink_hrtime _smoothed_rtt = 0; ink_hrtime _rttvar = 0; ink_hrtime _min_rtt = INT64_MAX; + // FIXME should be set by transport parameters + ink_hrtime _max_ack_delay = HRTIME_MSECONDS(25); + uint32_t _pto_count = 0; - // config + // Recovery A.2. Constants of Interest ink_hrtime _k_granularity = 0; - ink_hrtime _k_initial_rtt = 0; + ink_hrtime _k_initial_rtt = HRTIME_MSECONDS(500); }; diff --git a/iocore/net/quic/QUICNewRenoCongestionController.cc b/iocore/net/quic/QUICNewRenoCongestionController.cc new file mode 100644 index 00000000000..7745670082d --- /dev/null +++ b/iocore/net/quic/QUICNewRenoCongestionController.cc @@ -0,0 +1,278 @@ +/** @file + * + * A brief file description + * + * @section license License + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include + +#define QUICCCDebug(fmt, ...) \ + Debug("quic_cc", \ + "[%s] " \ + "window:%" PRIu32 " in-flight:%" PRIu32 " ssthresh:%" PRIu32 " extra:%" PRIu32 " " fmt, \ + this->_context.connection_info()->cids().data(), this->_congestion_window, this->_bytes_in_flight, this->_ssthresh, \ + this->_extra_packets_count, ##__VA_ARGS__) +#define QUICCCVDebug(fmt, ...) \ + Debug("v_quic_cc", \ + "[%s] " \ + "window:%" PRIu32 " in-flight:%" PRIu32 " ssthresh:%" PRIu32 " extra:%" PRIu32 " " fmt, \ + this->_context.connection_info()->cids().data(), this->_congestion_window, this->_bytes_in_flight, this->_ssthresh, \ + this->_extra_packets_count, ##__VA_ARGS__) + +#define QUICCCError(fmt, ...) \ + Error("quic_cc", \ + "[%s] " \ + "window:%" PRIu32 " in-flight:%" PRIu32 " ssthresh:%" PRIu32 " extra:%" PRIu32 " " fmt, \ + this->_context.connection_info()->cids().data(), this->_congestion_window, this->_bytes_in_flight, this->_ssthresh, \ + this->_extra_packets_count, ##__VA_ARGS__) + +QUICNewRenoCongestionController::QUICNewRenoCongestionController(QUICContext &context) + : _cc_mutex(new_ProxyMutex()), _context(context) +{ + auto &cc_config = context.cc_config(); + this->_k_initial_window = cc_config.initial_window(); + this->_k_minimum_window = cc_config.minimum_window(); + this->_k_loss_reduction_factor = cc_config.loss_reduction_factor(); + this->_k_persistent_congestion_threshold = cc_config.persistent_congestion_threshold(); + + this->reset(); +} + +void +QUICNewRenoCongestionController::on_packet_sent(size_t bytes_sent) +{ + SCOPED_MUTEX_LOCK(lock, this->_cc_mutex, this_ethread()); + if (this->_extra_packets_count > 0) { + --this->_extra_packets_count; + } + + this->_bytes_in_flight += bytes_sent; +} + +bool +QUICNewRenoCongestionController::_in_congestion_recovery(ink_hrtime sent_time) const +{ + return sent_time <= this->_congestion_recovery_start_time; +} + +bool +QUICNewRenoCongestionController::_is_app_or_flow_control_limited() +{ + // FIXME : don't known how does app worked here + return false; +} + +void +QUICNewRenoCongestionController::_maybe_send_one_packet() +{ + // TODO Implement _maybe_send_one_packet +} + +bool +QUICNewRenoCongestionController::_are_all_packets_lost(const std::map &lost_packets, + const QUICSentPacketInfoUPtr &largest_lost_packet, ink_hrtime period) const +{ + // check whether packets are continuous. return true if all continuous packets are in period + QUICPacketNumber next_expected = UINT64_MAX; + for (auto &it : lost_packets) { + if (it.second->time_sent >= largest_lost_packet->time_sent - period) { + if (next_expected == UINT64_MAX) { + next_expected = it.second->packet_number + 1; + continue; + } + + if (next_expected != it.second->packet_number) { + return false; + } + + next_expected = it.second->packet_number + 1; + } + } + + return next_expected == UINT64_MAX ? false : true; +} + +void +QUICNewRenoCongestionController::_congestion_event(ink_hrtime sent_time) +{ + // Start a new congestion event if packet was sent after the + // start of the previous congestion recovery period. + if (!this->_in_congestion_recovery(sent_time)) { + this->_congestion_recovery_start_time = Thread::get_hrtime(); + this->_congestion_window *= this->_k_loss_reduction_factor; + this->_congestion_window = std::max(this->_congestion_window, this->_k_minimum_window); + this->_ssthresh = this->_congestion_window; + this->_context.trigger(QUICContext::CallbackEvent::CONGESTION_STATE_CHANGED, QUICCongestionController::State::RECOVERY); + this->_context.trigger(QUICContext::CallbackEvent::METRICS_UPDATE, this->_congestion_window, this->_bytes_in_flight, + this->_ssthresh); + // A packet can be sent to speed up loss recovery. + this->_maybe_send_one_packet(); + } +} + +void +QUICNewRenoCongestionController::process_ecn(const QUICAckFrame &ack_frame, QUICPacketNumberSpace pn_space, + ink_hrtime largest_acked_time_sent) +{ + // If the ECN-CE counter reported by the peer has increased, + // this could be a new congestion event. + if (ack_frame.ecn_section()->ecn_ce_count() > this->_ecn_ce_counters[static_cast(pn_space)]) { + this->_ecn_ce_counters[static_cast(pn_space)] = ack_frame.ecn_section()->ecn_ce_count(); + // Start a new congestion event if the last acknowledged + // packet was sent after the start of the previous + // recovery epoch. + this->_congestion_event(largest_acked_time_sent); + } +} + +bool +QUICNewRenoCongestionController::_in_persistent_congestion(const std::map &lost_packets, + const QUICSentPacketInfoUPtr &largest_lost_packet) +{ + ink_hrtime congestion_period = this->_context.rtt_provider()->congestion_period(this->_k_persistent_congestion_threshold); + // Determine if all packets in the time period before the + // largest newly lost packet, including the edges, are + // marked lost + return this->_are_all_packets_lost(lost_packets, largest_lost_packet, congestion_period); +} + +void +QUICNewRenoCongestionController::on_packets_acked(const std::vector &packets) +{ + SCOPED_MUTEX_LOCK(lock, this->_cc_mutex, this_ethread()); + + for (auto &packet : packets) { + // Remove from bytes_in_flight. + this->_bytes_in_flight -= packet->sent_bytes; + if (this->_in_congestion_recovery(packet->time_sent)) { + // Do not increase congestion window in recovery period. + continue; + } + if (this->_is_app_or_flow_control_limited()) { + // Do not increase congestion_window if application + // limited or flow control limited. + continue; + } + if (this->_congestion_window < this->_ssthresh) { + // Slow start. + this->_context.trigger(QUICContext::CallbackEvent::CONGESTION_STATE_CHANGED, QUICCongestionController::State::SLOW_START); + this->_congestion_window += packet->sent_bytes; + QUICCCVDebug("slow start window changed"); + continue; + } + // Congestion avoidance. + this->_context.trigger(QUICContext::CallbackEvent::CONGESTION_STATE_CHANGED, + QUICCongestionController::State::CONGESTION_AVOIDANCE); + this->_congestion_window += this->_max_datagram_size * static_cast(packet->sent_bytes) / this->_congestion_window; + QUICCCVDebug("Congestion avoidance window changed"); + } +} + +// additional code +// the original one is: +// OnPacketsLost(lost_packets): +void +QUICNewRenoCongestionController::on_packets_lost(const std::map &lost_packets) +{ + SCOPED_MUTEX_LOCK(lock, this->_cc_mutex, this_ethread()); + + // Remove lost packets from bytes_in_flight. + for (auto &lost_packet : lost_packets) { + this->_bytes_in_flight -= lost_packet.second->sent_bytes; + } + const auto &largest_lost_packet = lost_packets.rbegin()->second; + this->_congestion_event(largest_lost_packet->time_sent); + + // Collapse congestion window if persistent congestion + if (this->_in_persistent_congestion(lost_packets, largest_lost_packet)) { + this->_congestion_window = this->_k_minimum_window; + } +} + +void +QUICNewRenoCongestionController::on_packet_number_space_discarded(size_t bytes_in_flight) +{ + this->_bytes_in_flight -= bytes_in_flight; +} + +bool +QUICNewRenoCongestionController::_check_credit() const +{ + if (this->_bytes_in_flight >= this->_congestion_window) { + QUICCCDebug("Congestion control pending"); + } + + return this->_bytes_in_flight < this->_congestion_window; +} + +uint32_t +QUICNewRenoCongestionController::credit() const +{ + if (this->_extra_packets_count) { + return UINT32_MAX; + } + + if (this->_check_credit()) { + return this->_congestion_window - this->_bytes_in_flight; + } else { + return 0; + } +} + +uint32_t +QUICNewRenoCongestionController::bytes_in_flight() const +{ + return this->_bytes_in_flight; +} + +uint32_t +QUICNewRenoCongestionController::congestion_window() const +{ + return this->_congestion_window; +} + +uint32_t +QUICNewRenoCongestionController::current_ssthresh() const +{ + return this->_ssthresh; +} + +// [draft-17 recovery] 7.9.3. Initialization +void +QUICNewRenoCongestionController::reset() +{ + SCOPED_MUTEX_LOCK(lock, this->_cc_mutex, this_ethread()); + + this->_congestion_window = this->_k_initial_window; + this->_bytes_in_flight = 0; + this->_congestion_recovery_start_time = 0; + this->_ssthresh = UINT32_MAX; + for (int i = 0; i < QUIC_N_PACKET_SPACES; ++i) { + this->_ecn_ce_counters[i] = 0; + } +} + +void +QUICNewRenoCongestionController::add_extra_credit() +{ + ++this->_extra_packets_count; +} diff --git a/iocore/net/quic/QUICNewRenoCongestionController.h b/iocore/net/quic/QUICNewRenoCongestionController.h new file mode 100644 index 00000000000..43a870ccc53 --- /dev/null +++ b/iocore/net/quic/QUICNewRenoCongestionController.h @@ -0,0 +1,81 @@ +/** @file + * + * A brief file description + * + * @section license License + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "QUICTypes.h" +#include "QUICContext.h" +#include "QUICCongestionController.h" + +class QUICNewRenoCongestionController : public QUICCongestionController +{ +public: + QUICNewRenoCongestionController(QUICContext &context); + virtual ~QUICNewRenoCongestionController() {} + + void on_packet_sent(size_t bytes_sent) override; + void on_packets_acked(const std::vector &packets) override; + virtual void on_packets_lost(const std::map &packets) override; + void on_packet_number_space_discarded(size_t bytes_in_flight) override; + void process_ecn(const QUICAckFrame &ack, QUICPacketNumberSpace pn_space, ink_hrtime largest_acked_packet_time_sent) override; + uint32_t credit() const override; + void reset() override; + + // Debug + uint32_t bytes_in_flight() const override; + uint32_t congestion_window() const override; + uint32_t current_ssthresh() const override; + + void add_extra_credit() override; + +private: + Ptr _cc_mutex; + uint32_t _extra_packets_count = 0; + QUICContext &_context; + bool _check_credit() const; + + // Appendix B. Congestion Control Pseudocode + bool _in_congestion_recovery(ink_hrtime sent_time) const; + void _congestion_event(ink_hrtime sent_time); + bool _in_persistent_congestion(const std::map &lost_packets, + const QUICSentPacketInfoUPtr &largest_lost_packet); + bool _is_app_or_flow_control_limited(); + void _maybe_send_one_packet(); + bool _are_all_packets_lost(const std::map &lost_packets, + const QUICSentPacketInfoUPtr &largest_lost_packet, ink_hrtime period) const; + + // Recovery B.1. Constants of interest + // Values will be loaded from records.config via QUICConfig at constructor + uint32_t _k_initial_window = 0; + uint32_t _k_minimum_window = 0; + float _k_loss_reduction_factor = 0.0; + uint32_t _k_persistent_congestion_threshold = 0; + + // B.2. Variables of interest + uint32_t _max_datagram_size = 0; + uint32_t _ecn_ce_counters[QUIC_N_PACKET_SPACES] = {0}; + uint32_t _bytes_in_flight = 0; + uint32_t _congestion_window = 0; + ink_hrtime _congestion_recovery_start_time = 0; + uint32_t _ssthresh = UINT32_MAX; +}; diff --git a/iocore/net/quic/QUICPacket.cc b/iocore/net/quic/QUICPacket.cc index ec8ac37c6ba..5f5707dceed 100644 --- a/iocore/net/quic/QUICPacket.cc +++ b/iocore/net/quic/QUICPacket.cc @@ -23,249 +23,438 @@ #include "QUICPacket.h" +#include + #include #include #include "QUICIntUtil.h" #include "QUICDebugNames.h" +#include "QUICRetryIntegrityTag.h" using namespace std::literals; -static constexpr std::string_view tag = "quic_packet"sv; -static constexpr uint64_t aead_tag_len = 16; +static constexpr uint64_t aead_tag_len = 16; +static constexpr int LONG_HDR_OFFSET_CONNECTION_ID = 6; +static constexpr int LONG_HDR_OFFSET_VERSION = 1; #define QUICDebug(dcid, scid, fmt, ...) \ Debug(tag.data(), "[%08" PRIx32 "-%08" PRIx32 "] " fmt, dcid.h32(), scid.h32(), ##__VA_ARGS__); -ClassAllocator quicPacketAllocator("quicPacketAllocator"); -ClassAllocator quicPacketLongHeaderAllocator("quicPacketLongHeaderAllocator"); -ClassAllocator quicPacketShortHeaderAllocator("quicPacketShortHeaderAllocator"); - -static constexpr int LONG_HDR_OFFSET_CONNECTION_ID = 6; -static constexpr int LONG_HDR_OFFSET_VERSION = 1; - // -// QUICPacketHeader +// QUICPacket // -const uint8_t * -QUICPacketHeader::buf() -{ - if (this->_buf) { - return this->_buf.get(); - } else { - // TODO Reuse serialzied data if nothing has changed - this->store(this->_serialized, &this->_buf_len); - if (this->_buf_len > MAX_PACKET_HEADER_LEN) { - ink_assert(!"Serialized packet header is too long"); - } +QUICPacket::QUICPacket() {} - this->_buf_len += this->_payload_length; - return this->_serialized; - } -} +QUICPacket::QUICPacket(bool ack_eliciting, bool probing) : _is_ack_eliciting(ack_eliciting), _is_probing_packet(probing) {} -const IpEndpoint & -QUICPacketHeader::from() const +QUICPacket::~QUICPacket() {} + +QUICKeyPhase +QUICPacket::key_phase() const { - return this->_from; + ink_assert(!"This function should not be called"); + return QUICKeyPhase::INITIAL; } bool -QUICPacketHeader::is_crypto_packet() const +QUICPacket::is_ack_eliciting() const { - return false; + return this->_is_ack_eliciting; } -uint16_t -QUICPacketHeader::packet_size() const +bool +QUICPacket::is_probing_packet() const { - return this->_buf_len; + return this->_is_probing_packet; } -QUICPacketHeaderUPtr -QUICPacketHeader::load(const IpEndpoint from, ats_unique_buf buf, size_t len, QUICPacketNumber base) +uint16_t +QUICPacket::header_size() const { - QUICPacketHeaderUPtr header = QUICPacketHeaderUPtr(nullptr, &QUICPacketHeaderDeleter::delete_null_header); - if (QUICInvariants::is_long_header(buf.get())) { - QUICPacketLongHeader *long_header = quicPacketLongHeaderAllocator.alloc(); - new (long_header) QUICPacketLongHeader(from, std::move(buf), len, base); - header = QUICPacketHeaderUPtr(long_header, &QUICPacketHeaderDeleter::delete_long_header); - } else { - QUICPacketShortHeader *short_header = quicPacketShortHeaderAllocator.alloc(); - new (short_header) QUICPacketShortHeader(from, std::move(buf), len, base); - header = QUICPacketHeaderUPtr(short_header, &QUICPacketHeaderDeleter::delete_short_header); + uint16_t size = 0; + + for (auto b = this->header_block(); b; b = b->next) { + size += b->size(); } - return header; + + return size; } -QUICPacketHeaderUPtr -QUICPacketHeader::build(QUICPacketType type, QUICKeyPhase key_phase, QUICConnectionId destination_cid, QUICConnectionId source_cid, - QUICPacketNumber packet_number, QUICPacketNumber base_packet_number, QUICVersion version, bool crypto, - ats_unique_buf payload, size_t len) +uint16_t +QUICPacket::payload_length() const { - QUICPacketLongHeader *long_header = quicPacketLongHeaderAllocator.alloc(); - new (long_header) QUICPacketLongHeader(type, key_phase, destination_cid, source_cid, packet_number, base_packet_number, version, - crypto, std::move(payload), len); - return QUICPacketHeaderUPtr(long_header, &QUICPacketHeaderDeleter::delete_long_header); + uint16_t size = 0; + + for (auto b = this->payload_block(); b; b = b->next) { + size += b->size(); + } + + return size; } -QUICPacketHeaderUPtr -QUICPacketHeader::build(QUICPacketType type, QUICKeyPhase key_phase, QUICConnectionId destination_cid, QUICConnectionId source_cid, - QUICPacketNumber packet_number, QUICPacketNumber base_packet_number, QUICVersion version, bool crypto, - ats_unique_buf payload, size_t len, ats_unique_buf token, size_t token_len) +uint16_t +QUICPacket::size() const { - QUICPacketLongHeader *long_header = quicPacketLongHeaderAllocator.alloc(); - new (long_header) QUICPacketLongHeader(type, key_phase, destination_cid, source_cid, packet_number, base_packet_number, version, - crypto, std::move(payload), len, std::move(token), token_len); - return QUICPacketHeaderUPtr(long_header, &QUICPacketHeaderDeleter::delete_long_header); + return this->header_size() + this->payload_length(); } -QUICPacketHeaderUPtr -QUICPacketHeader::build(QUICPacketType type, QUICKeyPhase key_phase, QUICVersion version, QUICConnectionId destination_cid, - QUICConnectionId source_cid, QUICConnectionId original_dcid, ats_unique_buf retry_token, - size_t retry_token_len) +void +QUICPacket::store(uint8_t *buf, size_t *len) const { - QUICPacketLongHeader *long_header = quicPacketLongHeaderAllocator.alloc(); - new (long_header) QUICPacketLongHeader(type, key_phase, version, destination_cid, source_cid, original_dcid, - std::move(retry_token), retry_token_len); - return QUICPacketHeaderUPtr(long_header, &QUICPacketHeaderDeleter::delete_long_header); + size_t written = 0; + Ptr block; + + block = this->header_block(); + while (block) { + memcpy(buf + written, block->start(), block->size()); + written += block->size(); + block = block->next; + } + + block = this->payload_block(); + while (block) { + memcpy(buf + written, block->start(), block->size()); + written += block->size(); + block = block->next; + } + + *len = written; } -QUICPacketHeaderUPtr -QUICPacketHeader::build(QUICPacketType type, QUICKeyPhase key_phase, QUICPacketNumber packet_number, - QUICPacketNumber base_packet_number, ats_unique_buf payload, size_t len) +uint8_t +QUICPacket::calc_packet_number_len(QUICPacketNumber num, QUICPacketNumber base) { - QUICPacketShortHeader *short_header = quicPacketShortHeaderAllocator.alloc(); - new (short_header) QUICPacketShortHeader(type, key_phase, packet_number, base_packet_number, std::move(payload), len); - return QUICPacketHeaderUPtr(short_header, &QUICPacketHeaderDeleter::delete_short_header); + uint64_t d = (num - base) * 2; + uint8_t len = 0; + + if (d > 0xFFFFFF) { + len = 4; + } else if (d > 0xFFFF) { + len = 3; + } else if (d > 0xFF) { + len = 2; + } else { + len = 1; + } + + return len; } -QUICPacketHeaderUPtr -QUICPacketHeader::build(QUICPacketType type, QUICKeyPhase key_phase, QUICConnectionId connection_id, QUICPacketNumber packet_number, - QUICPacketNumber base_packet_number, ats_unique_buf payload, size_t len) +bool +QUICPacket::encode_packet_number(QUICPacketNumber &dst, QUICPacketNumber src, size_t len) { - QUICPacketShortHeader *short_header = quicPacketShortHeaderAllocator.alloc(); - new (short_header) - QUICPacketShortHeader(type, key_phase, connection_id, packet_number, base_packet_number, std::move(payload), len); - return QUICPacketHeaderUPtr(short_header, &QUICPacketHeaderDeleter::delete_short_header); + uint64_t mask = 0; + switch (len) { + case 1: + mask = 0xFF; + break; + case 2: + mask = 0xFFFF; + break; + case 3: + mask = 0xFFFFFF; + break; + case 4: + mask = 0xFFFFFFFF; + break; + default: + ink_assert(!"len must be 1, 2, or 4"); + return false; + } + dst = src & mask; + + return true; } -QUICPacketHeaderUPtr -QUICPacketHeader::clone() const +bool +QUICPacket::decode_packet_number(QUICPacketNumber &dst, QUICPacketNumber src, size_t len, QUICPacketNumber largest_acked) { - return QUICPacketHeaderUPtr(nullptr, &QUICPacketHeaderDeleter::delete_null_header); + ink_assert(len == 1 || len == 2 || len == 3 || len == 4); + + uint64_t maximum_diff = 0; + switch (len) { + case 1: + maximum_diff = 0x100; + break; + case 2: + maximum_diff = 0x10000; + break; + case 3: + maximum_diff = 0x1000000; + break; + case 4: + maximum_diff = 0x100000000; + break; + default: + ink_assert(!"len must be 1, 2, 3 or 4"); + } + QUICPacketNumber base = largest_acked & (~(maximum_diff - 1)); + QUICPacketNumber candidate1 = base + src; + QUICPacketNumber candidate2 = base + src + maximum_diff; + QUICPacketNumber expected = largest_acked + 1; + + if (((candidate1 > expected) ? (candidate1 - expected) : (expected - candidate1)) < + ((candidate2 > expected) ? (candidate2 - expected) : (expected - candidate2))) { + dst = candidate1; + } else { + dst = candidate2; + } + + return true; } // -// QUICPacketLongHeader +// QUICPacketR // -QUICPacketLongHeader::QUICPacketLongHeader(const IpEndpoint from, ats_unique_buf buf, size_t len, QUICPacketNumber base) - : QUICPacketHeader(from, std::move(buf), len, base) +QUICPacketR::QUICPacketR(UDPConnection *udp_con, IpEndpoint from, IpEndpoint to) : _udp_con(udp_con), _from(from), _to(to) {} + +UDPConnection * +QUICPacketR::udp_con() const { - this->_key_phase = QUICTypeUtil::key_phase(this->type()); - uint8_t *raw_buf = this->_buf.get(); + return this->_udp_con; +} - uint8_t dcil = 0; - uint8_t scil = 0; - QUICPacketLongHeader::dcil(dcil, raw_buf, len); - QUICPacketLongHeader::scil(scil, raw_buf, len); +const IpEndpoint & +QUICPacketR::from() const +{ + return this->_from; +} - size_t offset = LONG_HDR_OFFSET_CONNECTION_ID; - this->_destination_cid = {raw_buf + offset, dcil}; - offset += dcil; - this->_source_cid = {raw_buf + offset, scil}; - offset += scil; +const IpEndpoint & +QUICPacketR::to() const +{ + return this->_to; +} - if (this->type() != QUICPacketType::VERSION_NEGOTIATION) { - if (this->type() == QUICPacketType::RETRY) { - uint8_t odcil = (raw_buf[0] & 0x0f) + 3; - this->_original_dcid = {raw_buf + offset, odcil}; - offset += odcil; - } else { - if (this->type() == QUICPacketType::INITIAL) { - // Token Length Field - this->_token_len = QUICIntUtil::read_QUICVariableInt(raw_buf + offset); - offset += QUICVariableInt::size(raw_buf + offset); - // Token Field - this->_token_offset = offset; - offset += this->_token_len; - } +bool +QUICPacketR::type(QUICPacketType &type, const uint8_t *packet, size_t packet_len) +{ + if (packet_len < 1) { + return false; + } + + if (QUICInvariants::is_long_header(packet)) { + return QUICLongHeaderPacketR::type(type, packet, packet_len); + } else { + type = QUICPacketType::PROTECTED; + return true; + } +} + +bool +QUICPacketR::read_essential_info(Ptr block, QUICPacketType &type, QUICVersion &version, QUICConnectionId &dcid, + QUICConnectionId &scid, QUICPacketNumber &packet_number, QUICPacketNumber base_packet_number, + QUICKeyPhase &key_phase) +{ + uint8_t tmp[47 + 64]; + IOBufferReader reader; + reader.block = block; + int64_t len = std::min(static_cast(sizeof(tmp)), reader.read_avail()); - // Length Field - offset += QUICVariableInt::size(raw_buf + offset); + if (len < 10) { + return false; + } - // PN Field - int pn_len = QUICTypeUtil::read_QUICPacketNumberLen(raw_buf); - QUICPacket::decode_packet_number(this->_packet_number, QUICTypeUtil::read_QUICPacketNumber(raw_buf + offset, pn_len), pn_len, - this->_base_packet_number); - offset += pn_len; + reader.memcpy(tmp, 1, 0); + if (QUICInvariants::is_long_header(tmp)) { + reader.memcpy(tmp, len, 0); + type = static_cast((0x30 & tmp[0]) >> 4); + QUICInvariants::version(version, tmp, len); + if (version == 0x00) { + type = QUICPacketType::VERSION_NEGOTIATION; + } + if (!QUICInvariants::dcid(dcid, tmp, len) || !QUICInvariants::scid(scid, tmp, len)) { + return false; + } + if (type != QUICPacketType::RETRY) { + int packet_number_len = QUICTypeUtil::read_QUICPacketNumberLen(tmp); + size_t length_offset = 7 + dcid.length() + scid.length(); + if (length_offset >= static_cast(len)) { + return false; + } + uint64_t value; + size_t field_len; + QUICVariableInt::decode(value, field_len, tmp + length_offset); + switch (type) { + case QUICPacketType::INITIAL: + length_offset += field_len + value; + if (length_offset >= static_cast(len)) { + return false; + } + QUICVariableInt::decode(value, field_len, tmp + length_offset); + if (length_offset + field_len >= static_cast(len)) { + return false; + } + if (length_offset + field_len + packet_number_len > static_cast(len)) { + return false; + } + packet_number = QUICTypeUtil::read_QUICPacketNumber(tmp + length_offset + field_len, packet_number_len); + key_phase = QUICKeyPhase::INITIAL; + break; + case QUICPacketType::ZERO_RTT_PROTECTED: + if (length_offset + field_len + packet_number_len >= static_cast(len)) { + return false; + } + packet_number = QUICTypeUtil::read_QUICPacketNumber(tmp + length_offset + field_len, packet_number_len); + key_phase = QUICKeyPhase::ZERO_RTT; + break; + case QUICPacketType::HANDSHAKE: + if (length_offset + field_len + packet_number_len >= static_cast(len)) { + return false; + } + packet_number = QUICTypeUtil::read_QUICPacketNumber(tmp + length_offset + field_len, packet_number_len); + key_phase = QUICKeyPhase::INITIAL; + break; + case QUICPacketType::VERSION_NEGOTIATION: + break; + default: + break; + } + } else { + packet_number = 0; + } + } else { + len = std::min(static_cast(25), len); + reader.memcpy(tmp, len, 0); + type = QUICPacketType::PROTECTED; + QUICInvariants::dcid(dcid, tmp, len); + int packet_number_len = QUICTypeUtil::read_QUICPacketNumberLen(tmp); + if (tmp[0] & 0x04) { + key_phase = QUICKeyPhase::PHASE_1; + } else { + key_phase = QUICKeyPhase::PHASE_0; } + packet_number = QUICTypeUtil::read_QUICPacketNumber(tmp + 1 + dcid.length(), packet_number_len); } + return true; +} - this->_payload_offset = offset; - this->_payload_length = len - this->_payload_offset; +// +// QUICLongHeaderPacket +// +QUICLongHeaderPacket::QUICLongHeaderPacket(QUICVersion version, const QUICConnectionId &dcid, const QUICConnectionId &scid, + bool ack_eliciting, bool probing, bool crypto) + : QUICPacket(ack_eliciting, probing), _version(version), _dcid(dcid), _scid(scid), _is_crypto_packet(crypto) +{ } -QUICPacketLongHeader::QUICPacketLongHeader(QUICPacketType type, QUICKeyPhase key_phase, const QUICConnectionId &destination_cid, - const QUICConnectionId &source_cid, QUICPacketNumber packet_number, - QUICPacketNumber base_packet_number, QUICVersion version, bool crypto, - ats_unique_buf buf, size_t len, ats_unique_buf token, size_t token_len) - : QUICPacketHeader(type, packet_number, base_packet_number, true, version, std::move(buf), len, key_phase), - _destination_cid(destination_cid), - _source_cid(source_cid), - _token_len(token_len), - _token(std::move(token)), - _is_crypto_packet(crypto) +QUICConnectionId +QUICLongHeaderPacket::destination_cid() const { - if (this->_type == QUICPacketType::VERSION_NEGOTIATION) { - this->_buf_len = - LONG_HDR_OFFSET_CONNECTION_ID + this->_destination_cid.length() + this->_source_cid.length() + this->_payload_length; - } else { - this->buf(); - } + return this->_dcid; } -QUICPacketLongHeader::QUICPacketLongHeader(QUICPacketType type, QUICKeyPhase key_phase, QUICVersion version, - const QUICConnectionId &destination_cid, const QUICConnectionId &source_cid, - const QUICConnectionId &original_dcid, ats_unique_buf retry_token, - size_t retry_token_len) - : QUICPacketHeader(type, 0, 0, true, version, std::move(retry_token), retry_token_len, key_phase), - _destination_cid(destination_cid), - _source_cid(source_cid), - _original_dcid(original_dcid) +QUICConnectionId +QUICLongHeaderPacket::source_cid() const +{ + return this->_scid; +} +uint16_t +QUICLongHeaderPacket::payload_length() const { - // this->_buf_len will be set - this->buf(); + return this->_payload_length; } -QUICPacketType -QUICPacketLongHeader::type() const +QUICVersion +QUICLongHeaderPacket::version() const +{ + return this->_version; +} + +size_t +QUICLongHeaderPacket::_write_common_header(uint8_t *buf) const { - if (this->_buf) { - QUICPacketType type = QUICPacketType::UNINITIALIZED; - QUICPacketLongHeader::type(type, this->_buf.get(), this->_buf_len); - return type; + size_t n; + size_t len = 0; + + buf[0] = 0xC0; + buf[0] += static_cast(this->type()) << 4; + len += 1; + + QUICTypeUtil::write_QUICVersion(this->_version, buf + len, &n); + len += n; + + // DICD + if (this->_dcid != QUICConnectionId::ZERO()) { + // Len + buf[len] = this->_dcid.length(); + len += 1; + + // ID + QUICTypeUtil::write_QUICConnectionId(this->_dcid, buf + len, &n); + len += n; + } else { + buf[len] = 0; + len += 1; + } + + // SCID + if (this->_scid != QUICConnectionId::ZERO()) { + // Len + buf[len] = this->_scid.length(); + len += 1; + + // ID + QUICTypeUtil::write_QUICConnectionId(this->_scid, buf + len, &n); + len += n; } else { - return this->_type; + buf[len] = 0; + len += 1; } + + return len; } bool -QUICPacketLongHeader::is_crypto_packet() const +QUICLongHeaderPacket::is_crypto_packet() const { return this->_is_crypto_packet; } +// +// QUICLongHeaderPacketR +// +QUICLongHeaderPacketR::QUICLongHeaderPacketR(UDPConnection *udp_con, IpEndpoint from, IpEndpoint to, Ptr blocks) + : QUICPacketR(udp_con, from, to) +{ + IOBufferReader reader; + uint8_t data[47]; + + reader.block = blocks; + int64_t data_len = reader.read(data, sizeof(data)); + + QUICLongHeaderPacketR::version(this->_version, data, data_len); +} + +QUICVersion +QUICLongHeaderPacketR::version() const +{ + return this->_version; +} + +QUICConnectionId +QUICLongHeaderPacketR::source_cid() const +{ + return this->_scid; +} + +QUICConnectionId +QUICLongHeaderPacketR::destination_cid() const +{ + return this->_dcid; +} + bool -QUICPacketLongHeader::type(QUICPacketType &type, const uint8_t *packet, size_t packet_len) +QUICLongHeaderPacketR::type(QUICPacketType &type, const uint8_t *packet, size_t packet_len) { if (packet_len < 1) { return false; } QUICVersion version; - if (QUICPacketLongHeader::version(version, packet, packet_len) && version == 0x00) { + if (QUICLongHeaderPacketR::version(version, packet, packet_len) && version == 0x00) { type = QUICPacketType::VERSION_NEGOTIATION; } else { uint8_t raw_type = (packet[0] & 0x30) >> 4; @@ -275,7 +464,7 @@ QUICPacketLongHeader::type(QUICPacketType &type, const uint8_t *packet, size_t p } bool -QUICPacketLongHeader::version(QUICVersion &version, const uint8_t *packet, size_t packet_len) +QUICLongHeaderPacketR::version(QUICVersion &version, const uint8_t *packet, size_t packet_len) { if (packet_len < 5) { return false; @@ -286,700 +475,1379 @@ QUICPacketLongHeader::version(QUICVersion &version, const uint8_t *packet, size_ } bool -QUICPacketLongHeader::dcil(uint8_t &dcil, const uint8_t *packet, size_t packet_len) +QUICLongHeaderPacketR::key_phase(QUICKeyPhase &phase, const uint8_t *packet, size_t packet_len) { - if (QUICInvariants::dcil(dcil, packet, packet_len)) { - if (dcil != 0) { - dcil += 3; - } - return true; - } else { - return false; - } + QUICPacketType type = QUICPacketType::UNINITIALIZED; + QUICLongHeaderPacketR::type(type, packet, packet_len); + phase = QUICTypeUtil::key_phase(type); + return true; } bool -QUICPacketLongHeader::scil(uint8_t &scil, const uint8_t *packet, size_t packet_len) +QUICLongHeaderPacketR::length(size_t &length, uint8_t &length_field_len, size_t &length_field_offset, const uint8_t *packet, + size_t packet_len) { - if (QUICInvariants::scil(scil, packet, packet_len)) { - if (scil != 0) { - scil += 3; - } - return true; - } else { + // FIXME This is not great because each packet types have different formats. + // We should remove this function and have length() on each packet type classes instead. + + uint8_t dcil; + if (!QUICInvariants::dcil(dcil, packet, packet_len)) { return false; } -} - -bool -QUICPacketLongHeader::token_length(size_t &token_length, uint8_t *field_len, const uint8_t *packet, size_t packet_len) -{ - QUICPacketType type = QUICPacketType::UNINITIALIZED; - QUICPacketLongHeader::type(type, packet, packet_len); - if (type != QUICPacketType::INITIAL) { - token_length = 0; - if (field_len) { - *field_len = 0; - } - return true; + uint8_t scil; + if (!QUICInvariants::scil(scil, packet, packet_len)) { + return false; } - uint8_t dcil, scil; - QUICPacketLongHeader::dcil(dcil, packet, packet_len); - QUICPacketLongHeader::scil(scil, packet, packet_len); + length_field_offset = LONG_HDR_OFFSET_CONNECTION_ID + dcil + 1 + scil; - size_t offset = LONG_HDR_OFFSET_CONNECTION_ID + dcil + scil; - if (offset >= packet_len) { - return false; + QUICPacketType type = QUICPacketType::UNINITIALIZED; + QUICLongHeaderPacketR::type(type, packet, packet_len); + if (type == QUICPacketType::INITIAL) { + // Token Length (i) + Token (*) (for INITIAL packet) + size_t token_length = 0; + uint8_t token_length_field_len = 0; + size_t token_length_field_offset = 0; + if (!QUICInitialPacketR::token_length(token_length, token_length_field_len, token_length_field_offset, packet, packet_len)) { + return false; + } + length_field_offset += token_length_field_len + token_length; } - if (offset > packet_len) { + // Length (i) + if (length_field_offset >= packet_len) { return false; } - token_length = QUICIntUtil::read_QUICVariableInt(packet + offset); - if (field_len) { - *field_len = QUICVariableInt::size(packet + offset); - } + length_field_len = QUICVariableInt::size(packet + length_field_offset); + length = QUICIntUtil::read_QUICVariableInt(packet + length_field_offset, packet_len - length_field_offset); return true; } bool -QUICPacketLongHeader::length(size_t &length, uint8_t *field_len, const uint8_t *packet, size_t packet_len) +QUICLongHeaderPacketR::packet_length(size_t &packet_len, const uint8_t *buf, size_t buf_len) { - uint8_t dcil, scil; - QUICPacketLongHeader::dcil(dcil, packet, packet_len); - QUICPacketLongHeader::scil(scil, packet, packet_len); + size_t length; + uint8_t length_field_len; + size_t length_field_offset; - // Token Length (i) + Token (*) (for INITIAL packet) - size_t token_length = 0; - uint8_t token_length_field_len = 0; - if (!QUICPacketLongHeader::token_length(token_length, &token_length_field_len, packet, packet_len)) { + if (!QUICLongHeaderPacketR::length(length, length_field_len, length_field_offset, buf, buf_len)) { return false; } + packet_len = length + length_field_offset + length_field_len; - // Length (i) - size_t length_offset = LONG_HDR_OFFSET_CONNECTION_ID + dcil + scil + token_length_field_len + token_length; - if (length_offset >= packet_len) { + if (packet_len > buf_len) { return false; } - length = QUICIntUtil::read_QUICVariableInt(packet + length_offset); - if (field_len) { - *field_len = QUICVariableInt::size(packet + length_offset); - } + return true; } bool -QUICPacketLongHeader::packet_number_offset(uint8_t &pn_offset, const uint8_t *packet, size_t packet_len) +QUICLongHeaderPacketR::packet_number_offset(size_t &pn_offset, const uint8_t *packet, size_t packet_len) { - QUICPacketType type; - QUICPacketLongHeader::type(type, packet, packet_len); - - uint8_t dcil, scil; - size_t token_length; - uint8_t token_length_field_len; - size_t length; + size_t dummy; uint8_t length_field_len; - if (!QUICPacketLongHeader::dcil(dcil, packet, packet_len) || !QUICPacketLongHeader::scil(scil, packet, packet_len) || - !QUICPacketLongHeader::token_length(token_length, &token_length_field_len, packet, packet_len) || - !QUICPacketLongHeader::length(length, &length_field_len, packet, packet_len)) { + size_t length_field_offset; + + if (!QUICLongHeaderPacketR::length(dummy, length_field_len, length_field_offset, packet, packet_len)) { + return false; + } + pn_offset = length_field_offset + length_field_len; + + if (pn_offset >= packet_len) { return false; } - pn_offset = 6 + dcil + scil + token_length_field_len + token_length + length_field_len; return true; } -bool -QUICPacketLongHeader::key_phase(QUICKeyPhase &phase, const uint8_t *packet, size_t packet_len) +// +// QUICShortHeaderPacket +// +QUICShortHeaderPacket::QUICShortHeaderPacket(const QUICConnectionId &dcid, QUICPacketNumber packet_number, + QUICPacketNumber base_packet_number, QUICKeyPhase key_phase, bool ack_eliciting, + bool probing) + : QUICPacket(ack_eliciting, probing), _dcid(dcid), _packet_number(packet_number), _key_phase(key_phase) { - QUICPacketType type = QUICPacketType::UNINITIALIZED; - QUICPacketLongHeader::type(type, packet, packet_len); - phase = QUICTypeUtil::key_phase(type); - return true; + this->_packet_number_len = QUICPacket::calc_packet_number_len(packet_number, base_packet_number); } -QUICConnectionId -QUICPacketLongHeader::destination_cid() const +QUICPacketType +QUICShortHeaderPacket::type() const { - return this->_destination_cid; + return QUICPacketType::PROTECTED; } -QUICConnectionId -QUICPacketLongHeader::source_cid() const +QUICKeyPhase +QUICShortHeaderPacket::key_phase() const { - return this->_source_cid; + return this->_key_phase; } QUICConnectionId -QUICPacketLongHeader::original_dcid() const +QUICShortHeaderPacket::destination_cid() const { - return this->_original_dcid; + return this->_dcid; } QUICPacketNumber -QUICPacketLongHeader::packet_number() const +QUICShortHeaderPacket::packet_number() const { return this->_packet_number; } +uint16_t +QUICShortHeaderPacket::payload_length() const +{ + return this->_payload_length; +} + +Ptr +QUICShortHeaderPacket::header_block() const +{ + Ptr block; + size_t written_len = 0; + + block = make_ptr(new_IOBufferBlock()); + block->alloc(iobuffer_size_to_index(1 + QUICConnectionId::MAX_LENGTH + 4, BUFFER_SIZE_INDEX_32K)); + uint8_t *buf = reinterpret_cast(block->start()); + + size_t n; + buf[0] = 0x40; + + // Type + buf[0] = 0x40; + + // TODO Spin Bit + + // KeyPhase + if (this->_key_phase == QUICKeyPhase::PHASE_1) { + buf[0] |= 0x04; + } + + written_len += 1; + + // Destination Connection ID + if (this->_dcid != QUICConnectionId::ZERO()) { + QUICTypeUtil::write_QUICConnectionId(this->_dcid, buf + written_len, &n); + written_len += n; + } + + // Packet Number + QUICPacketNumber dst = 0; + size_t dst_len = this->_packet_number_len; + QUICPacket::encode_packet_number(dst, this->_packet_number, dst_len); + QUICTypeUtil::write_QUICPacketNumber(dst, dst_len, buf + written_len, &n); + written_len += n; + + // Packet Number Length + QUICTypeUtil::write_QUICPacketNumberLen(n, buf); + + block->fill(written_len); + + return block; +} + +Ptr +QUICShortHeaderPacket::payload_block() const +{ + return this->_payload_block; +} + +void +QUICShortHeaderPacket::attach_payload(Ptr payload, bool unprotected) +{ + this->_payload_block = payload; + this->_payload_length = 0; + Ptr tmp = payload; + while (tmp) { + this->_payload_length += tmp->size(); + tmp = tmp->next; + } + if (unprotected) { + this->_payload_length += aead_tag_len; + } +} + +// +// QUICShortHeaderPacketR +// +QUICShortHeaderPacketR::QUICShortHeaderPacketR(UDPConnection *udp_con, IpEndpoint from, IpEndpoint to, Ptr blocks, + QUICPacketNumber base_packet_number) + : QUICPacketR(udp_con, from, to) +{ + size_t len = 0; + for (auto b = blocks; b; b = b->next) { + len += b->size(); + } + + Ptr concatenated_block = make_ptr(new_IOBufferBlock()); + concatenated_block->alloc(iobuffer_size_to_index(len, BUFFER_SIZE_INDEX_32K)); + concatenated_block->fill(len); + + uint8_t *raw_buf = reinterpret_cast(concatenated_block->start()); + + size_t copied_len = 0; + for (auto b = blocks; b; b = b->next) { + memcpy(raw_buf + copied_len, b->start(), b->size()); + copied_len += b->size(); + } + + if (raw_buf[0] & 0x04) { + this->_key_phase = QUICKeyPhase::PHASE_1; + } else { + this->_key_phase = QUICKeyPhase::PHASE_0; + } + + QUICInvariants::dcid(this->_dcid, raw_buf, len); + + int offset = 1 + this->_dcid.length(); + this->_packet_number_len = QUICTypeUtil::read_QUICPacketNumberLen(raw_buf); + QUICPacketNumber src = QUICTypeUtil::read_QUICPacketNumber(raw_buf + offset, this->_packet_number_len); + QUICPacket::decode_packet_number(this->_packet_number, src, this->_packet_number_len, base_packet_number); + offset += this->_packet_number_len; + + this->_header_block = concatenated_block->clone(); + this->_header_block->_end = this->_header_block->_start + offset; + this->_header_block->next = nullptr; + this->_payload_block = concatenated_block->clone(); + this->_payload_block->_start = this->_payload_block->_start + offset; +} + +QUICPacketType +QUICShortHeaderPacketR::type() const +{ + return QUICPacketType::PROTECTED; +} + +QUICKeyPhase +QUICShortHeaderPacketR::key_phase() const +{ + return this->_key_phase; +} + +QUICPacketNumber +QUICShortHeaderPacketR::packet_number() const +{ + return this->_packet_number; +} + +QUICConnectionId +QUICShortHeaderPacketR::destination_cid() const +{ + return this->_dcid; +} + +Ptr +QUICShortHeaderPacketR::header_block() const +{ + return this->_header_block; +} + +Ptr +QUICShortHeaderPacketR::payload_block() const +{ + return this->_payload_block; +} + +void +QUICShortHeaderPacketR::attach_payload(Ptr payload, bool unprotected) +{ + this->_payload_block = payload; +} + bool -QUICPacketLongHeader::has_version() const +QUICShortHeaderPacketR::packet_number_offset(size_t &pn_offset, const uint8_t *packet, size_t packet_len, int dcil) { + pn_offset = 1 + dcil; return true; } -bool -QUICPacketLongHeader::is_valid() const +// +// QUICStatelessResetPacket +// +QUICStatelessResetPacket::QUICStatelessResetPacket(QUICStatelessResetToken token, size_t maximum_size) + : QUICPacket(), _token(token), _maximum_size(maximum_size) { - if (this->_buf && this->_buf_len != this->_payload_offset + this->_payload_length) { - QUICDebug(this->_source_cid, this->_destination_cid, - "Invalid packet: packet_size(%zu) should be header_size(%zu) + payload_size(%zu)", this->_buf_len, - this->_payload_offset, this->_payload_length); - Warning("Invalid packet: packet_size(%zu) should be header_size(%zu) + payload_size(%zu)", this->_buf_len, - this->_payload_offset, this->_payload_length); +} - return false; +QUICPacketType +QUICStatelessResetPacket::type() const +{ + return QUICPacketType::STATELESS_RESET; +} + +QUICConnectionId +QUICStatelessResetPacket::destination_cid() const +{ + ink_assert(!"You should not need DCID of Stateless Reset Packet"); + return QUICConnectionId::ZERO(); +} + +Ptr +QUICStatelessResetPacket::header_block() const +{ + // Required shortest length is 38 bits however less than 41 bytes in total indicates this is stateless reset. + constexpr uint8_t MIN_UNPREDICTABLE_FIELD_LEN = 5 + 20; + + std::random_device rnd; + + Ptr block; + size_t written_len = 0; + + size_t random_extra_length = rnd() & 0x07; // Extra 0 to 7 bytes + + if (MIN_UNPREDICTABLE_FIELD_LEN + random_extra_length > this->_maximum_size) { + return block; } - return true; + block = make_ptr(new_IOBufferBlock()); + block->alloc(iobuffer_size_to_index(MIN_UNPREDICTABLE_FIELD_LEN + random_extra_length, BUFFER_SIZE_INDEX_32K)); + uint8_t *buf = reinterpret_cast(block->start()); + + // Generate random octets + for (int i = 0; i < MIN_UNPREDICTABLE_FIELD_LEN; ++i) { + buf[i] = static_cast(rnd() & 0xFF); + } + buf[0] = (buf[0] | 0x40) & 0x7f; + written_len += MIN_UNPREDICTABLE_FIELD_LEN; + + block->fill(written_len); + + return block; +} + +Ptr +QUICStatelessResetPacket::payload_block() const +{ + Ptr block; + size_t written_len = 0; + + block = make_ptr(new_IOBufferBlock()); + block->alloc(iobuffer_size_to_index(QUICStatelessResetToken::LEN, BUFFER_SIZE_INDEX_32K)); + uint8_t *buf = reinterpret_cast(block->start()); + + memcpy(buf, this->_token.buf(), QUICStatelessResetToken::LEN); + written_len += QUICStatelessResetToken::LEN; + + block->fill(written_len); + + return block; +} + +QUICPacketNumber +QUICStatelessResetPacket::packet_number() const +{ + ink_assert(!"You should not need packet number of Stateless Reset Packet"); + return 0; +} + +QUICStatelessResetToken +QUICStatelessResetPacket::token() const +{ + return this->_token; +} + +// +// QUICStatelessResetPacketR +// +QUICStatelessResetPacketR::QUICStatelessResetPacketR(UDPConnection *udp_con, IpEndpoint from, IpEndpoint to, + Ptr blocks) + : QUICPacketR(udp_con, from, to) +{ +} + +QUICPacketType +QUICStatelessResetPacketR::type() const +{ + return QUICPacketType::STATELESS_RESET; +} + +QUICPacketNumber +QUICStatelessResetPacketR::packet_number() const +{ + ink_assert(!"You should not need packet number of Stateless Reset Packet"); + return 0; +} + +QUICConnectionId +QUICStatelessResetPacketR::destination_cid() const +{ + ink_assert(!"You should not need DCID of Stateless Reset Packet"); + return QUICConnectionId::ZERO(); +} + +// +// QUICVersionNegotiationPacket +// + +QUICVersionNegotiationPacket::QUICVersionNegotiationPacket(const QUICConnectionId &dcid, const QUICConnectionId &scid, + const QUICVersion versions[], int nversions, + QUICVersion version_in_initial) + : QUICLongHeaderPacket(0, dcid, scid, false, false, false), + _versions(versions), + _nversions(nversions), + _version_in_initial(version_in_initial) +{ +} + +QUICPacketType +QUICVersionNegotiationPacket::type() const +{ + return QUICPacketType::VERSION_NEGOTIATION; } QUICVersion -QUICPacketLongHeader::version() const +QUICVersionNegotiationPacket::version() const { - if (this->_buf) { - QUICVersion version = 0; - QUICPacketLongHeader::version(version, this->_buf.get(), this->_buf_len); - return version; - } else { - return this->_version; + return 0; +} + +QUICPacketNumber +QUICVersionNegotiationPacket::packet_number() const +{ + ink_assert(!"You should not need packet number of Version Negotiation Packet"); + return 0; +} + +uint16_t +QUICVersionNegotiationPacket::payload_length() const +{ + uint16_t size = 0; + + for (auto b = this->payload_block(); b; b = b->next) { + size += b->size(); } + + return size; } -const uint8_t * -QUICPacketLongHeader::payload() const +Ptr +QUICVersionNegotiationPacket::header_block() const { - if (this->_buf) { - uint8_t *raw = this->_buf.get(); - return raw + this->_payload_offset; + Ptr block; + size_t written_len = 0; + + block = make_ptr(new_IOBufferBlock()); + block->alloc(iobuffer_size_to_index(2048, BUFFER_SIZE_INDEX_32K)); + uint8_t *buf = reinterpret_cast(block->start()); + + // Common Long Header + written_len += this->_write_common_header(buf + written_len); + + // Overwrite the first byte + buf[0] = 0x80 | rand(); + + block->fill(written_len); + + return block; +} + +Ptr +QUICVersionNegotiationPacket::payload_block() const +{ + Ptr block; + uint8_t *buf; + size_t written_len = 0; + size_t n; + + block = make_ptr(new_IOBufferBlock()); + block->alloc(iobuffer_size_to_index(sizeof(QUICVersion) * (this->_nversions + 1), BUFFER_SIZE_INDEX_32K)); + buf = reinterpret_cast(block->start()); + + for (auto i = 0; i < this->_nversions; ++i) { + QUICTypeUtil::write_QUICVersion(*(this->_versions + i), buf + written_len, &n); + written_len += n; + } + + // [draft-18] 6.3. Using Reserved Versions + // To help ensure this, a server SHOULD include a reserved version (see Section 15) while generating a + // Version Negotiation packet. + QUICVersion exersice_version = QUIC_EXERCISE_VERSION1; + if (this->_version_in_initial == QUIC_EXERCISE_VERSION1) { + exersice_version = QUIC_EXERCISE_VERSION2; + } + QUICTypeUtil::write_QUICVersion(exersice_version, buf + written_len, &n); + written_len += n; + + block->fill(written_len); + + return block; +} + +const QUICVersion * +QUICVersionNegotiationPacket::versions() const +{ + return this->_versions; +} + +int +QUICVersionNegotiationPacket::nversions() const +{ + return this->_nversions; +} + +// +// QUICVersionNegotiationPacketR +// +QUICVersionNegotiationPacketR::QUICVersionNegotiationPacketR(UDPConnection *udp_con, IpEndpoint from, IpEndpoint to, + Ptr blocks) + : QUICLongHeaderPacketR(udp_con, from, to, blocks) +{ + size_t len = 0; + for (auto b = blocks; b; b = b->next) { + len += b->size(); + } + + Ptr concatenated_block = make_ptr(new_IOBufferBlock()); + concatenated_block->alloc(iobuffer_size_to_index(len, BUFFER_SIZE_INDEX_32K)); + concatenated_block->fill(len); + + uint8_t *raw_buf = reinterpret_cast(concatenated_block->start()); + + size_t copied_len = 0; + for (auto b = blocks; b; b = b->next) { + memcpy(raw_buf + copied_len, b->start(), b->size()); + copied_len += b->size(); + } + + uint8_t dcil = 0; + uint8_t scil = 0; + QUICInvariants::dcil(dcil, raw_buf, len); + QUICInvariants::scil(scil, raw_buf, len); + + size_t offset = LONG_HDR_OFFSET_CONNECTION_ID; + this->_dcid = {raw_buf + offset, dcil}; + offset += dcil + 1; + this->_scid = {raw_buf + offset, scil}; + offset += scil; + + this->_versions = raw_buf + offset; + this->_nversions = (len - offset) / sizeof(QUICVersion); + + this->_header_block = concatenated_block->clone(); + this->_header_block->_end = this->_header_block->_start + offset; + this->_header_block->next = nullptr; + this->_payload_block = concatenated_block->clone(); + this->_payload_block->_start = this->_payload_block->_start + offset; +} + +QUICPacketType +QUICVersionNegotiationPacketR::type() const +{ + return QUICPacketType::VERSION_NEGOTIATION; +} + +QUICPacketNumber +QUICVersionNegotiationPacketR::packet_number() const +{ + ink_assert(!"You should not need packet number of Version Negotiation Packet"); + return 0; +} + +QUICConnectionId +QUICVersionNegotiationPacketR::destination_cid() const +{ + return this->_dcid; +} + +Ptr +QUICVersionNegotiationPacketR::header_block() const +{ + return this->_header_block; +} + +Ptr +QUICVersionNegotiationPacketR::payload_block() const +{ + return this->_payload_block; +} + +const QUICVersion +QUICVersionNegotiationPacketR::supported_version(uint8_t index) const +{ + return QUICTypeUtil::read_QUICVersion(this->_versions + sizeof(QUICVersion) * index); +} + +int +QUICVersionNegotiationPacketR::nversions() const +{ + return this->_nversions; +} + +// +// QUICInitialPacket +// +QUICInitialPacket::QUICInitialPacket(QUICVersion version, const QUICConnectionId &dcid, const QUICConnectionId &scid, + size_t token_len, ats_unique_buf token, size_t length, QUICPacketNumber packet_number, + bool ack_eliciting, bool probing, bool crypto) + : QUICLongHeaderPacket(version, dcid, scid, ack_eliciting, probing, crypto), + _token_len(token_len), + _token(std::move(token)), + _packet_number(packet_number) +{ +} + +QUICPacketType +QUICInitialPacket::type() const +{ + return QUICPacketType::INITIAL; +} + +QUICKeyPhase +QUICInitialPacket::key_phase() const +{ + return QUICKeyPhase::INITIAL; +} + +QUICPacketNumber +QUICInitialPacket::packet_number() const +{ + return this->_packet_number; +} + +Ptr +QUICInitialPacket::header_block() const +{ + Ptr block; + size_t written_len = 0; + size_t n; + + block = make_ptr(new_IOBufferBlock()); + block->alloc(iobuffer_size_to_index(2048, BUFFER_SIZE_INDEX_32K)); + uint8_t *buf = reinterpret_cast(block->start()); + + // Common Long Header + written_len += this->_write_common_header(buf + written_len); + + // Token Length + QUICIntUtil::write_QUICVariableInt(this->_token_len, buf + written_len, &n); + written_len += n; + + // Token + memcpy(buf + written_len, this->_token.get(), this->_token_len); + written_len += this->_token_len; + + QUICPacketNumber pn = 0; + size_t pn_len = 4; + QUICPacket::encode_packet_number(pn, this->_packet_number, pn_len); + + if (pn > 0x7FFFFF) { + pn_len = 4; + } else if (pn > 0x7FFF) { + pn_len = 3; + } else if (pn > 0x7F) { + pn_len = 2; } else { - return this->_payload.get(); + pn_len = 1; } + + // PN Len + QUICTypeUtil::write_QUICPacketNumberLen(pn_len, buf); + + // Length + QUICIntUtil::write_QUICVariableInt(pn_len + this->_payload_length, buf + written_len, &n); + written_len += n; + + // PN Field + QUICTypeUtil::write_QUICPacketNumber(pn, pn_len, buf + written_len, &n); + written_len += n; + + block->fill(written_len); + + return block; } -uint16_t -QUICPacketHeader::payload_size() const +Ptr +QUICInitialPacket::payload_block() const { - return this->_payload_length; + return this->_payload_block; +} + +void +QUICInitialPacket::attach_payload(Ptr payload, bool unprotected) +{ + this->_payload_block = payload; + this->_payload_length = 0; + Ptr tmp = payload; + while (tmp) { + this->_payload_length += tmp->size(); + tmp = tmp->next; + } + if (unprotected) { + this->_payload_length += aead_tag_len; + } } -const uint8_t * -QUICPacketLongHeader::token() const +// +// QUICInitialPacketR +// +QUICInitialPacketR::QUICInitialPacketR(UDPConnection *udp_con, IpEndpoint from, IpEndpoint to, Ptr blocks, + QUICPacketNumber base_packet_number) + : QUICLongHeaderPacketR(udp_con, from, to, blocks) { - if (this->_buf) { - uint8_t *raw = this->_buf.get(); - return raw + this->_token_offset; + size_t len = 0; + for (auto b = blocks; b; b = b->next) { + len += b->size(); + } + + Ptr concatenated_block = make_ptr(new_IOBufferBlock()); + concatenated_block->alloc(iobuffer_size_to_index(len, BUFFER_SIZE_INDEX_32K)); + concatenated_block->fill(len); + + uint8_t *raw_buf = reinterpret_cast(concatenated_block->start()); + + size_t copied_len = 0; + for (auto b = blocks; b; b = b->next) { + memcpy(raw_buf + copied_len, b->start(), b->size()); + copied_len += b->size(); + } + + uint8_t dcil = 0; + uint8_t scil = 0; + QUICInvariants::dcil(dcil, raw_buf, len); + QUICInvariants::scil(scil, raw_buf, len); + + size_t offset = LONG_HDR_OFFSET_CONNECTION_ID; + this->_dcid = {raw_buf + offset, dcil}; + offset += dcil + 1; + this->_scid = {raw_buf + offset, scil}; + offset += scil; + + // Token Length Field + uint64_t token_len = QUICIntUtil::read_QUICVariableInt(raw_buf + offset, len - offset); + offset += QUICVariableInt::size(raw_buf + offset); + + // Token Field + if (token_len) { + this->_token = new QUICAddressValidationToken(raw_buf + offset, token_len); + offset += token_len; } else { - return this->_token.get(); + this->_token = new QUICAddressValidationToken(nullptr, 0); + } + + // Length Field + offset += QUICVariableInt::size(raw_buf + offset); + + // PN Field + int pn_len = QUICTypeUtil::read_QUICPacketNumberLen(raw_buf); + QUICPacket::decode_packet_number(this->_packet_number, QUICTypeUtil::read_QUICPacketNumber(raw_buf + offset, pn_len), pn_len, + base_packet_number); + offset += pn_len; + + this->_header_block = concatenated_block->clone(); + this->_header_block->_end = this->_header_block->_start + offset; + this->_header_block->next = nullptr; + this->_payload_block = concatenated_block->clone(); + this->_payload_block->_start = this->_payload_block->_start + offset; +} + +QUICInitialPacketR::~QUICInitialPacketR() +{ + delete this->_token; +} + +QUICPacketType +QUICInitialPacketR::type() const +{ + return QUICPacketType::INITIAL; +} + +QUICPacketNumber +QUICInitialPacketR::packet_number() const +{ + return this->_packet_number; +} + +QUICKeyPhase +QUICInitialPacketR::key_phase() const +{ + return QUICKeyPhase::INITIAL; +} + +Ptr +QUICInitialPacketR::header_block() const +{ + return this->_header_block; +} + +Ptr +QUICInitialPacketR::payload_block() const +{ + return this->_payload_block; +} + +void +QUICInitialPacketR::attach_payload(Ptr payload, bool unprotected) +{ + this->_payload_block = payload; +} + +const QUICAddressValidationToken & +QUICInitialPacketR::token() const +{ + return *(this->_token); +} + +bool +QUICInitialPacketR::token_length(size_t &token_length, uint8_t &field_len, size_t &token_length_filed_offset, const uint8_t *packet, + size_t packet_len) +{ + QUICPacketType type = QUICPacketType::UNINITIALIZED; + QUICPacketR::type(type, packet, packet_len); + + ink_assert(type == QUICPacketType::INITIAL); + + uint8_t dcil, scil; + QUICInvariants::dcil(dcil, packet, packet_len); + QUICInvariants::scil(scil, packet, packet_len); + + token_length_filed_offset = LONG_HDR_OFFSET_CONNECTION_ID + dcil + 1 + scil; + if (token_length_filed_offset >= packet_len) { + return false; } + + token_length = QUICIntUtil::read_QUICVariableInt(packet + token_length_filed_offset, packet_len - token_length_filed_offset); + field_len = QUICVariableInt::size(packet + token_length_filed_offset); + + return true; +} + +// +// QUICZeroRttPacket +// +QUICZeroRttPacket::QUICZeroRttPacket(QUICVersion version, const QUICConnectionId &dcid, const QUICConnectionId &scid, size_t length, + QUICPacketNumber packet_number, bool ack_eliciting, bool probing) + : QUICLongHeaderPacket(version, dcid, scid, ack_eliciting, probing, false), _packet_number(packet_number) +{ } -size_t -QUICPacketLongHeader::token_len() const +QUICPacketType +QUICZeroRttPacket::type() const { - return this->_token_len; + return QUICPacketType::ZERO_RTT_PROTECTED; } QUICKeyPhase -QUICPacketLongHeader::key_phase() const +QUICZeroRttPacket::key_phase() const { - return this->_key_phase; + return QUICKeyPhase::ZERO_RTT; } -uint16_t -QUICPacketLongHeader::size() const +QUICPacketNumber +QUICZeroRttPacket::packet_number() const { - return this->_buf_len - this->_payload_length; + return this->_packet_number; } -void -QUICPacketLongHeader::store(uint8_t *buf, size_t *len) const +Ptr +QUICZeroRttPacket::header_block() const { + Ptr block; + size_t written_len = 0; size_t n; - *len = 0; - buf[0] = 0xC0; - buf[0] += static_cast(this->_type) << 4; - if (this->_type == QUICPacketType::VERSION_NEGOTIATION) { - buf[0] |= rand(); - } - *len += 1; - QUICTypeUtil::write_QUICVersion(this->_version, buf + *len, &n); - *len += n; + block = make_ptr(new_IOBufferBlock()); + block->alloc(iobuffer_size_to_index(2048, BUFFER_SIZE_INDEX_32K)); + uint8_t *buf = reinterpret_cast(block->start()); - buf[*len] = this->_destination_cid == QUICConnectionId::ZERO() ? 0 : (this->_destination_cid.length() - 3) << 4; - buf[*len] += this->_source_cid == QUICConnectionId::ZERO() ? 0 : this->_source_cid.length() - 3; - *len += 1; + // Common Long Header + written_len += this->_write_common_header(buf + written_len); - if (this->_destination_cid != QUICConnectionId::ZERO()) { - QUICTypeUtil::write_QUICConnectionId(this->_destination_cid, buf + *len, &n); - *len += n; - } - if (this->_source_cid != QUICConnectionId::ZERO()) { - QUICTypeUtil::write_QUICConnectionId(this->_source_cid, buf + *len, &n); - *len += n; + QUICPacketNumber pn = 0; + size_t pn_len = 4; + QUICPacket::encode_packet_number(pn, this->_packet_number, pn_len); + + if (pn > 0x7FFFFF) { + pn_len = 4; + } else if (pn > 0x7FFF) { + pn_len = 3; + } else if (pn > 0x7F) { + pn_len = 2; + } else { + pn_len = 1; } - if (this->_type != QUICPacketType::VERSION_NEGOTIATION) { - if (this->_type == QUICPacketType::RETRY) { - // Original Destination Connection ID - if (this->_original_dcid != QUICConnectionId::ZERO()) { - QUICTypeUtil::write_QUICConnectionId(this->_original_dcid, buf + *len, &n); - *len += n; - } - // ODCIL - buf[0] |= this->_original_dcid.length() - 3; - } else { - if (this->_type == QUICPacketType::INITIAL) { - // Token Length Field - QUICIntUtil::write_QUICVariableInt(this->_token_len, buf + *len, &n); - *len += n; - - // Token Field - memcpy(buf + *len, this->token(), this->token_len()); - *len += this->token_len(); - } + // PN Len + QUICTypeUtil::write_QUICPacketNumberLen(pn_len, buf); - QUICPacketNumber pn = 0; - size_t pn_len = 4; - QUICPacket::encode_packet_number(pn, this->_packet_number, pn_len); - - if (pn > 0x7FFFFF) { - pn_len = 4; - } else if (pn > 0x7FFF) { - pn_len = 3; - } else if (pn > 0x7F) { - pn_len = 2; - } else { - pn_len = 1; - } + // Length + QUICIntUtil::write_QUICVariableInt(pn_len + this->_payload_length, buf + written_len, &n); + written_len += n; - if (this->_type != QUICPacketType::RETRY) { - // PN Len field - QUICTypeUtil::write_QUICPacketNumberLen(pn_len, buf); - } + // PN Field + QUICTypeUtil::write_QUICPacketNumber(pn, pn_len, buf + written_len, &n); + written_len += n; - // Length Field - QUICIntUtil::write_QUICVariableInt(pn_len + this->_payload_length + aead_tag_len, buf + *len, &n); - *len += n; + block->fill(written_len); - // PN Field - QUICTypeUtil::write_QUICPacketNumber(pn, pn_len, buf + *len, &n); - *len += n; - } + return block; +} - // Payload will be stored +Ptr +QUICZeroRttPacket::payload_block() const +{ + return this->_payload_block; +} + +void +QUICZeroRttPacket::attach_payload(Ptr payload, bool unprotected) +{ + this->_payload_block = payload; + this->_payload_length = 0; + Ptr tmp = payload; + while (tmp) { + this->_payload_length += tmp->size(); + tmp = tmp->next; + } + if (unprotected) { + this->_payload_length += aead_tag_len; } } // -// QUICPacketShortHeader +// QUICZeroRttPacketR // - -QUICPacketShortHeader::QUICPacketShortHeader(const IpEndpoint from, ats_unique_buf buf, size_t len, QUICPacketNumber base) - : QUICPacketHeader(from, std::move(buf), len, base) +QUICZeroRttPacketR::QUICZeroRttPacketR(UDPConnection *udp_con, IpEndpoint from, IpEndpoint to, Ptr blocks, + QUICPacketNumber base_packet_number) + : QUICLongHeaderPacketR(udp_con, from, to, blocks) { - QUICInvariants::dcid(this->_connection_id, this->_buf.get(), len); + size_t len = 0; + for (auto b = blocks; b; b = b->next) { + len += b->size(); + } + + Ptr concatenated_block = make_ptr(new_IOBufferBlock()); + concatenated_block->alloc(iobuffer_size_to_index(len, BUFFER_SIZE_INDEX_32K)); + concatenated_block->fill(len); + + uint8_t *raw_buf = reinterpret_cast(concatenated_block->start()); + + size_t copied_len = 0; + for (auto b = blocks; b; b = b->next) { + memcpy(raw_buf + copied_len, b->start(), b->size()); + copied_len += b->size(); + } + + uint8_t dcil = 0; + uint8_t scil = 0; + QUICInvariants::dcil(dcil, raw_buf, len); + QUICInvariants::scil(scil, raw_buf, len); + + size_t offset = LONG_HDR_OFFSET_CONNECTION_ID; + this->_dcid = {raw_buf + offset, dcil}; + offset += dcil + 1; + this->_scid = {raw_buf + offset, scil}; + offset += scil; - int offset = 1 + this->_connection_id.length(); - this->_packet_number_len = QUICTypeUtil::read_QUICPacketNumberLen(this->_buf.get()); - QUICPacketNumber src = QUICTypeUtil::read_QUICPacketNumber(this->_buf.get() + offset, this->_packet_number_len); - QUICPacket::decode_packet_number(this->_packet_number, src, this->_packet_number_len, this->_base_packet_number); - this->_payload_length = len - (1 + QUICConnectionId::SCID_LEN + this->_packet_number_len); + // Length Field + offset += QUICVariableInt::size(raw_buf + offset); + + // PN Field + int pn_len = QUICTypeUtil::read_QUICPacketNumberLen(raw_buf); + QUICPacket::decode_packet_number(this->_packet_number, QUICTypeUtil::read_QUICPacketNumber(raw_buf + offset, pn_len), pn_len, + base_packet_number); + offset += pn_len; + + this->_header_block = concatenated_block->clone(); + this->_header_block->_end = this->_header_block->_start + offset; + this->_header_block->next = nullptr; + this->_payload_block = concatenated_block->clone(); + this->_payload_block->_start = this->_payload_block->_start + offset; } -QUICPacketShortHeader::QUICPacketShortHeader(QUICPacketType type, QUICKeyPhase key_phase, QUICPacketNumber packet_number, - QUICPacketNumber base_packet_number, ats_unique_buf buf, size_t len) +QUICPacketType +QUICZeroRttPacketR::type() const { - this->_type = type; - this->_key_phase = key_phase; - this->_packet_number = packet_number; - this->_base_packet_number = base_packet_number; - this->_packet_number_len = QUICPacket::calc_packet_number_len(packet_number, base_packet_number); - this->_payload = std::move(buf); - this->_payload_length = len; + return QUICPacketType::ZERO_RTT_PROTECTED; } -QUICPacketShortHeader::QUICPacketShortHeader(QUICPacketType type, QUICKeyPhase key_phase, const QUICConnectionId &connection_id, - QUICPacketNumber packet_number, QUICPacketNumber base_packet_number, - ats_unique_buf buf, size_t len) +QUICPacketNumber +QUICZeroRttPacketR::packet_number() const { - this->_type = type; - this->_key_phase = key_phase; - this->_connection_id = connection_id; - this->_packet_number = packet_number; - this->_base_packet_number = base_packet_number; - this->_packet_number_len = QUICPacket::calc_packet_number_len(packet_number, base_packet_number); - this->_payload = std::move(buf); - this->_payload_length = len; + return this->_packet_number; } -QUICPacketType -QUICPacketShortHeader::type() const +QUICKeyPhase +QUICZeroRttPacketR::key_phase() const { - QUICKeyPhase key_phase = this->key_phase(); - - switch (key_phase) { - case QUICKeyPhase::PHASE_0: { - return QUICPacketType::PROTECTED; - } - case QUICKeyPhase::PHASE_1: { - return QUICPacketType::PROTECTED; - } - default: - return QUICPacketType::STATELESS_RESET; - } + return QUICKeyPhase::ZERO_RTT; } -QUICConnectionId -QUICPacketShortHeader::destination_cid() const +Ptr +QUICZeroRttPacketR::header_block() const { - if (this->_buf) { - QUICConnectionId dcid = QUICConnectionId::ZERO(); - QUICInvariants::dcid(dcid, this->_buf.get(), this->_buf_len); - return dcid; - } else { - return _connection_id; - } + return this->_header_block; } -QUICPacketNumber -QUICPacketShortHeader::packet_number() const +Ptr +QUICZeroRttPacketR::payload_block() const { - return this->_packet_number; + return this->_payload_block; } -bool -QUICPacketShortHeader::has_version() const +void +QUICZeroRttPacketR::attach_payload(Ptr payload, bool unprotected) { - return false; + this->_payload_block = payload; } -bool -QUICPacketShortHeader::is_valid() const +// +// QUICHandshakePacket +// +QUICHandshakePacket::QUICHandshakePacket(QUICVersion version, const QUICConnectionId &dcid, const QUICConnectionId &scid, + size_t length, QUICPacketNumber packet_number, bool ack_eliciting, bool probing, + bool crypto) + : QUICLongHeaderPacket(version, dcid, scid, ack_eliciting, probing, crypto), _packet_number(packet_number) { - return true; } -QUICVersion -QUICPacketShortHeader::version() const +QUICPacketType +QUICHandshakePacket::type() const { - return 0; + return QUICPacketType::HANDSHAKE; } -const uint8_t * -QUICPacketShortHeader::payload() const +QUICKeyPhase +QUICHandshakePacket::key_phase() const { - if (this->_buf) { - return this->_buf.get() + this->size(); - } else { - return this->_payload.get(); - } + return QUICKeyPhase::HANDSHAKE; } -QUICKeyPhase -QUICPacketShortHeader::key_phase() const +QUICPacketNumber +QUICHandshakePacket::packet_number() const { - if (this->_buf) { - QUICKeyPhase phase = QUICKeyPhase::INITIAL; - QUICPacketShortHeader::key_phase(phase, this->_buf.get(), this->_buf_len); - return phase; - } else { - return this->_key_phase; - } + return this->_packet_number; } -bool -QUICPacketShortHeader::key_phase(QUICKeyPhase &phase, const uint8_t *packet, size_t packet_len) +Ptr +QUICHandshakePacket::header_block() const { - if (packet_len < 1) { - return false; - } - if (packet[0] & 0x04) { - phase = QUICKeyPhase::PHASE_1; + Ptr block; + size_t written_len = 0; + size_t n; + + block = make_ptr(new_IOBufferBlock()); + block->alloc(iobuffer_size_to_index(2048, BUFFER_SIZE_INDEX_32K)); + uint8_t *buf = reinterpret_cast(block->start()); + + // Common Long Header + written_len += this->_write_common_header(buf + written_len); + + QUICPacketNumber pn = 0; + size_t pn_len = 4; + QUICPacket::encode_packet_number(pn, this->_packet_number, pn_len); + + if (pn > 0x7FFFFF) { + pn_len = 4; + } else if (pn > 0x7FFF) { + pn_len = 3; + } else if (pn > 0x7F) { + pn_len = 2; } else { - phase = QUICKeyPhase::PHASE_0; + pn_len = 1; } - return true; + + // PN Len + QUICTypeUtil::write_QUICPacketNumberLen(pn_len, buf); + + // Length + QUICIntUtil::write_QUICVariableInt(pn_len + this->_payload_length, buf + written_len, &n); + written_len += n; + + // PN Field + QUICTypeUtil::write_QUICPacketNumber(pn, pn_len, buf + written_len, &n); + written_len += n; + + block->fill(written_len); + + return block; } -bool -QUICPacketShortHeader::packet_number_offset(uint8_t &pn_offset, const uint8_t *packet, size_t packet_len, int dcil) +Ptr +QUICHandshakePacket::payload_block() const { - pn_offset = 1 + dcil; - return true; + return this->_payload_block; } -/** - * Header Length (doesn't include payload length) - */ -uint16_t -QUICPacketShortHeader::size() const -{ - uint16_t len = 1; - if (this->_connection_id != QUICConnectionId::ZERO()) { - len += this->_connection_id.length(); +void +QUICHandshakePacket::attach_payload(Ptr payload, bool unprotected) +{ + this->_payload_block = payload; + this->_payload_length = 0; + Ptr tmp = payload; + while (tmp) { + this->_payload_length += tmp->size(); + tmp = tmp->next; + } + if (unprotected) { + this->_payload_length += aead_tag_len; } - len += this->_packet_number_len; - - return len; } -void -QUICPacketShortHeader::store(uint8_t *buf, size_t *len) const +// +// QUICHandshakePacketR +// +QUICHandshakePacketR::QUICHandshakePacketR(UDPConnection *udp_con, IpEndpoint from, IpEndpoint to, Ptr blocks, + QUICPacketNumber base_packet_number) + : QUICLongHeaderPacketR(udp_con, from, to, blocks) { - size_t n; - *len = 0; - buf[0] = 0x40; - if (this->_key_phase == QUICKeyPhase::PHASE_1) { - buf[0] |= 0x04; + size_t len = 0; + for (auto b = blocks; b; b = b->next) { + len += b->size(); } - *len += 1; - if (this->_connection_id != QUICConnectionId::ZERO()) { - QUICTypeUtil::write_QUICConnectionId(this->_connection_id, buf + *len, &n); - *len += n; + Ptr concatenated_block = make_ptr(new_IOBufferBlock()); + concatenated_block->alloc(iobuffer_size_to_index(len, BUFFER_SIZE_INDEX_32K)); + concatenated_block->fill(len); + + uint8_t *raw_buf = reinterpret_cast(concatenated_block->start()); + + size_t copied_len = 0; + for (auto b = blocks; b; b = b->next) { + memcpy(raw_buf + copied_len, b->start(), b->size()); + copied_len += b->size(); } - QUICPacketNumber dst = 0; - size_t dst_len = this->_packet_number_len; - QUICPacket::encode_packet_number(dst, this->_packet_number, dst_len); - QUICTypeUtil::write_QUICPacketNumber(dst, dst_len, buf + *len, &n); - *len += n; + uint8_t dcil = 0; + uint8_t scil = 0; + QUICInvariants::dcil(dcil, raw_buf, len); + QUICInvariants::scil(scil, raw_buf, len); - QUICTypeUtil::write_QUICPacketNumberLen(n, buf); -} + size_t offset = LONG_HDR_OFFSET_CONNECTION_ID; + this->_dcid = {raw_buf + offset, dcil}; + offset += dcil + 1; + this->_scid = {raw_buf + offset, scil}; + offset += scil; -// -// QUICPacket -// + // Length Field + offset += QUICVariableInt::size(raw_buf + offset); -QUICPacket::QUICPacket() {} + // PN Field + int pn_len = QUICTypeUtil::read_QUICPacketNumberLen(raw_buf); + QUICPacket::decode_packet_number(this->_packet_number, QUICTypeUtil::read_QUICPacketNumber(raw_buf + offset, pn_len), pn_len, + base_packet_number); + offset += pn_len; -QUICPacket::QUICPacket(UDPConnection *udp_con, QUICPacketHeaderUPtr header, ats_unique_buf payload, size_t payload_len) - : _udp_con(udp_con), _header(std::move(header)), _payload(std::move(payload)), _payload_size(payload_len) -{ + this->_header_block = concatenated_block->clone(); + this->_header_block->_end = this->_header_block->_start + offset; + this->_header_block->next = nullptr; + this->_payload_block = concatenated_block->clone(); + this->_payload_block->_start = this->_payload_block->_start + offset; } -QUICPacket::QUICPacket(QUICPacketHeaderUPtr header, ats_unique_buf payload, size_t payload_len, bool ack_eliciting, bool probing) - : _header(std::move(header)), - _payload(std::move(payload)), - _payload_size(payload_len), - _is_ack_eliciting(ack_eliciting), - _is_probing_packet(probing) +QUICPacketType +QUICHandshakePacketR::type() const { + return QUICPacketType::HANDSHAKE; } -QUICPacket::~QUICPacket() +QUICKeyPhase +QUICHandshakePacketR::key_phase() const { - this->_header = nullptr; + return QUICKeyPhase::HANDSHAKE; } -const IpEndpoint & -QUICPacket::from() const +QUICPacketNumber +QUICHandshakePacketR::packet_number() const { - return this->_header->from(); + return this->_packet_number; } -UDPConnection * -QUICPacket::udp_con() const +Ptr +QUICHandshakePacketR::header_block() const { - return this->_udp_con; + return this->_header_block; } -/** - * When packet is "Short Header Packet", QUICPacket::type() will return 1-RTT Protected (key phase 0) - * or 1-RTT Protected (key phase 1) - */ -QUICPacketType -QUICPacket::type() const +Ptr +QUICHandshakePacketR::payload_block() const { - return this->_header->type(); + return this->_payload_block; } -QUICConnectionId -QUICPacket::destination_cid() const +void +QUICHandshakePacketR::attach_payload(Ptr payload, bool unprotected) { - return this->_header->destination_cid(); + this->_payload_block = payload; } -QUICConnectionId -QUICPacket::source_cid() const +// +// QUICRetryPacket +// +QUICRetryPacket::QUICRetryPacket(QUICVersion version, const QUICConnectionId &dcid, const QUICConnectionId &scid, + QUICRetryToken &token) + : QUICLongHeaderPacket(version, dcid, scid, false, false, false), _token(token) { - return this->_header->source_cid(); } -QUICPacketNumber -QUICPacket::packet_number() const +QUICPacketType +QUICRetryPacket::type() const { - return this->_header->packet_number(); + return QUICPacketType::RETRY; } -bool -QUICPacket::is_crypto_packet() const +QUICPacketNumber +QUICRetryPacket::packet_number() const { - return this->_header->is_crypto_packet(); + ink_assert(!"You should not need packet number of Retry Packet"); + return 0; } -const QUICPacketHeader & -QUICPacket::header() const +uint16_t +QUICRetryPacket::payload_length() const { - return *this->_header; -} + uint16_t size = 0; -const uint8_t * -QUICPacket::payload() const -{ - return this->_payload.get(); + for (auto b = this->payload_block(); b; b = b->next) { + size += b->size(); + } + + return size; } -QUICVersion -QUICPacket::version() const +Ptr +QUICRetryPacket::header_block() const { - return this->_header->version(); + Ptr block; + size_t written_len = 0; + + block = make_ptr(new_IOBufferBlock()); + block->alloc(iobuffer_size_to_index(2048, BUFFER_SIZE_INDEX_32K)); + uint8_t *buf = reinterpret_cast(block->start()); + + // Common Long Header + written_len += this->_write_common_header(buf + written_len); + + block->fill(written_len); + + return block; } -bool -QUICPacket::is_ack_eliciting() const +Ptr +QUICRetryPacket::payload_block() const { - return this->_is_ack_eliciting; + Ptr block; + uint8_t *buf; + size_t written_len = 0; + + block = make_ptr(new_IOBufferBlock()); + block->alloc(iobuffer_size_to_index(QUICConnectionId::MAX_LENGTH + this->_token.length() + QUICRetryIntegrityTag::LEN, + BUFFER_SIZE_INDEX_32K)); + buf = reinterpret_cast(block->start()); + + // Retry Token + memcpy(buf + written_len, this->_token.buf(), this->_token.length()); + written_len += this->_token.length(); + block->fill(written_len); + + // Retry Integrity Tag + QUICRetryIntegrityTag::compute(buf + written_len, this->version(), this->_token.original_dcid(), this->header_block(), block); + written_len += QUICRetryIntegrityTag::LEN; + block->fill(QUICRetryIntegrityTag::LEN); + + return block; } -bool -QUICPacket::is_probing_packet() const +const QUICRetryToken & +QUICRetryPacket::token() const { - return this->_is_probing_packet; + return this->_token; } -uint16_t -QUICPacket::size() const +// +// QUICRetryPacketR +// +QUICRetryPacketR::QUICRetryPacketR(UDPConnection *udp_con, IpEndpoint from, IpEndpoint to, Ptr blocks) + : QUICLongHeaderPacketR(udp_con, from, to, blocks) { - // This includes not only header size and payload size but also AEAD tag length - uint16_t size = this->_header->packet_size(); - if (size == 0) { - size = this->header_size() + this->payload_length(); + size_t len = 0; + for (auto b = blocks; b; b = b->next) { + len += b->size(); } - return size; + + Ptr concatenated_block = make_ptr(new_IOBufferBlock()); + concatenated_block->alloc(iobuffer_size_to_index(len, BUFFER_SIZE_INDEX_32K)); + concatenated_block->fill(len); + + uint8_t *raw_buf = reinterpret_cast(concatenated_block->start()); + + size_t copied_len = 0; + for (auto b = blocks; b; b = b->next) { + memcpy(raw_buf + copied_len, b->start(), b->size()); + copied_len += b->size(); + } + + uint8_t dcil = 0; + uint8_t scil = 0; + QUICInvariants::dcil(dcil, raw_buf, len); + QUICInvariants::scil(scil, raw_buf, len); + + size_t offset = LONG_HDR_OFFSET_CONNECTION_ID; + this->_dcid = {raw_buf + offset, dcil}; + offset += dcil + 1; + this->_scid = {raw_buf + offset, scil}; + offset += scil; + + // Retry Token field + this->_token = new QUICRetryToken(raw_buf + offset, len - offset - QUICRetryIntegrityTag::LEN); + offset += this->_token->length(); + + // Retry Integrity Tag + memcpy(this->_integrity_tag, raw_buf + offset, QUICRetryIntegrityTag::LEN); + + this->_header_block = concatenated_block->clone(); + this->_header_block->_end = this->_header_block->_start + offset; + this->_header_block->next = nullptr; + this->_payload_block = concatenated_block->clone(); + this->_payload_block->_start = this->_payload_block->_start + offset; + this->_payload_block_without_tag = this->_payload_block->clone(); + this->_payload_block_without_tag->_end = this->_payload_block_without_tag->_end - QUICRetryIntegrityTag::LEN; } -uint16_t -QUICPacket::header_size() const +QUICRetryPacketR::~QUICRetryPacketR() { - return this->_header->size(); + delete this->_token; } -uint16_t -QUICPacket::payload_length() const +Ptr +QUICRetryPacketR::header_block() const { - return this->_payload_size; + return this->_header_block; } -QUICKeyPhase -QUICPacket::key_phase() const +Ptr +QUICRetryPacketR::payload_block() const { - return this->_header->key_phase(); + return this->_payload_block; } -void -QUICPacket::store(uint8_t *buf, size_t *len) const +QUICPacketType +QUICRetryPacketR::type() const { - memcpy(buf, this->_header->buf(), this->_header->size()); - memcpy(buf + this->_header->size(), this->payload(), this->payload_length()); - *len = this->_header->size() + this->payload_length(); + return QUICPacketType::RETRY; } -uint8_t -QUICPacket::calc_packet_number_len(QUICPacketNumber num, QUICPacketNumber base) +QUICPacketNumber +QUICRetryPacketR::packet_number() const { - uint64_t d = (num - base) * 2; - uint8_t len = 0; - - if (d > 0xFFFFFF) { - len = 4; - } else if (d > 0xFFFF) { - len = 3; - } else if (d > 0xFF) { - len = 2; - } else { - len = 1; - } - - return len; + return 0; } -bool -QUICPacket::encode_packet_number(QUICPacketNumber &dst, QUICPacketNumber src, size_t len) +const QUICAddressValidationToken & +QUICRetryPacketR::token() const { - uint64_t mask = 0; - switch (len) { - case 1: - mask = 0xFF; - break; - case 2: - mask = 0xFFFF; - break; - case 3: - mask = 0xFFFFFF; - break; - case 4: - mask = 0xFFFFFFFF; - break; - default: - ink_assert(!"len must be 1, 2, or 4"); - return false; - } - dst = src & mask; - - return true; + return *(this->_token); } bool -QUICPacket::decode_packet_number(QUICPacketNumber &dst, QUICPacketNumber src, size_t len, QUICPacketNumber largest_acked) +QUICRetryPacketR::has_valid_tag(const QUICConnectionId &odcid) const { - ink_assert(len == 1 || len == 2 || len == 3 || len == 4); - - uint64_t maximum_diff = 0; - switch (len) { - case 1: - maximum_diff = 0x100; - break; - case 2: - maximum_diff = 0x10000; - break; - case 3: - maximum_diff = 0x1000000; - break; - case 4: - maximum_diff = 0x100000000; - break; - default: - ink_assert(!"len must be 1, 2, 3 or 4"); - } - QUICPacketNumber base = largest_acked & (~(maximum_diff - 1)); - QUICPacketNumber candidate1 = base + src; - QUICPacketNumber candidate2 = base + src + maximum_diff; - QUICPacketNumber expected = largest_acked + 1; + uint8_t tag_computed[QUICRetryIntegrityTag::LEN]; + QUICRetryIntegrityTag::compute(tag_computed, this->version(), odcid, this->_header_block, this->_payload_block_without_tag); - if (((candidate1 > expected) ? (candidate1 - expected) : (expected - candidate1)) < - ((candidate2 > expected) ? (candidate2 - expected) : (expected - candidate2))) { - dst = candidate1; - } else { - dst = candidate2; - } - - return true; + return memcmp(this->_integrity_tag, tag_computed, QUICRetryIntegrityTag::LEN) == 0; } diff --git a/iocore/net/quic/QUICPacket.h b/iocore/net/quic/QUICPacket.h index dd9c9bcdb90..e9d02127328 100644 --- a/iocore/net/quic/QUICPacket.h +++ b/iocore/net/quic/QUICPacket.h @@ -27,393 +27,527 @@ #include #include -#include "tscore/List.h" #include "I_IOBuffer.h" #include "QUICTypes.h" -#include "QUICHandshakeProtocol.h" -#include "QUICPacketHeaderProtector.h" -#include "QUICFrame.h" +#include "QUICRetryIntegrityTag.h" #define QUIC_FIELD_OFFSET_CONNECTION_ID 1 #define QUIC_FIELD_OFFSET_PACKET_NUMBER 4 #define QUIC_FIELD_OFFSET_PAYLOAD 5 class UDPConnection; -class QUICPacketHeader; -class QUICPacket; -class QUICPacketLongHeader; -class QUICPacketShortHeader; -extern ClassAllocator quicPacketAllocator; -extern ClassAllocator quicPacketLongHeaderAllocator; -extern ClassAllocator quicPacketShortHeaderAllocator; - -using QUICPacketHeaderDeleterFunc = void (*)(QUICPacketHeader *p); -using QUICPacketHeaderUPtr = std::unique_ptr; - -class QUICPacketHeader +class QUICPacket { public: - QUICPacketHeader(const IpEndpoint from, ats_unique_buf buf, size_t len, QUICPacketNumber base) - : _from(from), _buf(std::move(buf)), _buf_len(len), _base_packet_number(base) - { - } - ~QUICPacketHeader() {} - const uint8_t *buf(); + static constexpr int MAX_INSTANCE_SIZE = 1024; - virtual bool is_crypto_packet() const; + // Token field in Initial packet could be very long. + static constexpr size_t MAX_PACKET_HEADER_LEN = 256; - const IpEndpoint &from() const; + /** + * Creates a QUICPacket for sending packets + */ + QUICPacket(bool ack_eliciting, bool probing); - virtual QUICPacketType type() const = 0; + virtual ~QUICPacket(); - /* - * Returns a connection id - */ + virtual QUICPacketType type() const = 0; virtual QUICConnectionId destination_cid() const = 0; - virtual QUICConnectionId source_cid() const = 0; + virtual QUICPacketNumber packet_number() const = 0; + bool is_ack_eliciting() const; + bool is_probing_packet() const; - virtual QUICPacketNumber packet_number() const = 0; - virtual QUICVersion version() const = 0; + // TODO These two should be pure virtual + virtual Ptr + header_block() const + { + return Ptr(); + }; + virtual Ptr + payload_block() const + { + return Ptr(); + }; /* - * Returns a pointer for the payload + * Size of whole QUIC packet (header + payload + integrity check) */ - virtual const uint8_t *payload() const = 0; + virtual uint16_t size() const; /* - * Returns its payload size based on header length and buffer size that is specified to the constructo. + * Size of header */ - uint16_t payload_size() const; + virtual uint16_t header_size() const; /* - * Returns its header size + * Length of payload (payload + integrity check if exists) */ - virtual uint16_t size() const = 0; + virtual uint16_t payload_length() const; - /* - * Returns its packet size + /** + * Key phase */ - uint16_t packet_size() const; + virtual QUICKeyPhase key_phase() const; - /* - * Returns a key phase - */ - virtual QUICKeyPhase key_phase() const = 0; + // FIXME Remove this and use IOBufferBlock instead + void store(uint8_t *buf, size_t *len) const; - /* - * Stores serialized header - * - * The serialized data doesn't contain a payload part even if it was created with a buffer that contains payload data. - */ - virtual void store(uint8_t *buf, size_t *len) const = 0; + /***** STATIC MEMBERS *****/ - QUICPacketHeaderUPtr clone() const; + static uint8_t calc_packet_number_len(QUICPacketNumber num, QUICPacketNumber base); + static bool encode_packet_number(QUICPacketNumber &dst, QUICPacketNumber src, size_t len); + static bool decode_packet_number(QUICPacketNumber &dst, QUICPacketNumber src, size_t len, QUICPacketNumber largest_acked); - virtual bool has_version() const = 0; - virtual bool is_valid() const = 0; +protected: + QUICPacket(); - /***** STATIC members *****/ +private: + bool _is_ack_eliciting = false; + bool _is_probing_packet = false; +}; - /* - * Load data from a buffer and create a QUICPacketHeader - * - * This creates either a QUICPacketShortHeader or a QUICPacketLongHeader. +class QUICPacketR : public QUICPacket +{ +public: + /** + * Creates a QUICPacket for receiving packets */ - static QUICPacketHeaderUPtr load(const IpEndpoint from, ats_unique_buf buf, size_t len, QUICPacketNumber base); + QUICPacketR(UDPConnection *udp_con, IpEndpoint from, IpEndpoint to); - /* - * Build a QUICPacketHeader - * - * This creates a QUICPacketLongHeader. - */ - static QUICPacketHeaderUPtr build(QUICPacketType type, QUICKeyPhase key_phase, QUICConnectionId destination_cid, - QUICConnectionId source_cid, QUICPacketNumber packet_number, - QUICPacketNumber base_packet_number, QUICVersion version, bool crypto, ats_unique_buf payload, - size_t len); + virtual QUICPacketType type() const override = 0; - /* - * Build a QUICPacketHeader - * - * This creates a QUICPacketLongHeader for INITIAL packet - */ - static QUICPacketHeaderUPtr build(QUICPacketType type, QUICKeyPhase key_phase, QUICConnectionId destination_cid, - QUICConnectionId source_cid, QUICPacketNumber packet_number, - QUICPacketNumber base_packet_number, QUICVersion version, bool crypto, ats_unique_buf payload, - size_t len, ats_unique_buf token, size_t token_len); + UDPConnection *udp_con() const; + virtual const IpEndpoint &from() const; + virtual const IpEndpoint &to() const; - /* - * Build a QUICPacketHeader - * - * This creates a QUICPacketLongHeader for RETRY packet - */ - static QUICPacketHeaderUPtr build(QUICPacketType type, QUICKeyPhase key_phase, QUICVersion version, - QUICConnectionId destination_cid, QUICConnectionId source_cid, QUICConnectionId original_dcid, - ats_unique_buf retry_token, size_t retry_token_len); + static bool read_essential_info(Ptr block, QUICPacketType &type, QUICVersion &version, QUICConnectionId &dcid, + QUICConnectionId &scid, QUICPacketNumber &packet_number, QUICPacketNumber base_packet_number, + QUICKeyPhase &key_phase); + static bool type(QUICPacketType &type, const uint8_t *packet, size_t packet_len); - /* - * Build a QUICPacketHeader - * - * This creates a QUICPacketShortHeader that contains a ConnectionID. - */ - static QUICPacketHeaderUPtr build(QUICPacketType type, QUICKeyPhase key_phase, QUICPacketNumber packet_number, - QUICPacketNumber base_packet_number, ats_unique_buf payload, size_t len); +protected: + Ptr _header_block; + Ptr _payload_block; - /* - * Build a QUICPacketHeader - * - * This creates a QUICPacketShortHeader that doesn't contain a ConnectionID (Stateless Reset Packet). +private: + UDPConnection *_udp_con = nullptr; + const IpEndpoint _from = {}; + const IpEndpoint _to = {}; +}; + +using QUICPacketDeleterFunc = void (*)(QUICPacket *p); +using QUICPacketUPtr = std::unique_ptr; + +class QUICPacketDeleter +{ +public: + static void + delete_null_packet(QUICPacket *packet) + { + ink_assert(packet == nullptr); + } + + static void + delete_dont_free(QUICPacket *packet) + { + packet->~QUICPacket(); + } + + static void + delete_packet_new(QUICPacket *packet) + { + delete packet; + } +}; + +class QUICLongHeaderPacket : public QUICPacket +{ +public: + /** + * For sending packet */ - static QUICPacketHeaderUPtr build(QUICPacketType type, QUICKeyPhase key_phase, QUICConnectionId connection_id, - QUICPacketNumber packet_number, QUICPacketNumber base_packet_number, ats_unique_buf payload, - size_t len); + QUICLongHeaderPacket(QUICVersion version, const QUICConnectionId &dcid, const QUICConnectionId &scid, bool ack_eliciting, + bool probing, bool crypto); + + QUICConnectionId source_cid() const; + + QUICConnectionId destination_cid() const override; + uint16_t payload_length() const override; + virtual QUICVersion version() const; + virtual bool is_crypto_packet() const; protected: - QUICPacketHeader(){}; - QUICPacketHeader(QUICPacketType type, QUICPacketNumber packet_number, QUICPacketNumber base_packet_number, bool has_version, - QUICVersion version, ats_unique_buf payload, size_t payload_length, QUICKeyPhase key_phase) - : _payload(std::move(payload)), - _type(type), - _key_phase(key_phase), - _packet_number(packet_number), - _base_packet_number(base_packet_number), - _version(version), - _payload_length(payload_length), - _has_version(has_version){}; - // Token field in Initial packet could be very long. - static constexpr size_t MAX_PACKET_HEADER_LEN = 256; + size_t _write_common_header(uint8_t *buf) const; - const IpEndpoint _from = {}; - - // These two are used only if the instance was created with a buffer - ats_unique_buf _buf = {nullptr}; - size_t _buf_len = 0; - - // These are used only if the instance was created without a buffer - uint8_t _serialized[MAX_PACKET_HEADER_LEN]; - ats_unique_buf _payload = ats_unique_buf(nullptr); - QUICPacketType _type = QUICPacketType::UNINITIALIZED; - QUICKeyPhase _key_phase = QUICKeyPhase::INITIAL; - QUICConnectionId _connection_id = QUICConnectionId::ZERO(); - QUICPacketNumber _packet_number = 0; - QUICPacketNumber _base_packet_number = 0; - QUICVersion _version = 0; - size_t _payload_length = 0; - bool _has_version = false; + Ptr _payload_block; + size_t _payload_length = 0; + +private: + QUICVersion _version; + QUICConnectionId _dcid; + QUICConnectionId _scid; + + bool _is_crypto_packet; }; -class QUICPacketLongHeader : public QUICPacketHeader +class QUICLongHeaderPacketR : public QUICPacketR { public: - QUICPacketLongHeader() : QUICPacketHeader(){}; - virtual ~QUICPacketLongHeader(){}; - QUICPacketLongHeader(const IpEndpoint from, ats_unique_buf buf, size_t len, QUICPacketNumber base); - QUICPacketLongHeader(QUICPacketType type, QUICKeyPhase key_phase, const QUICConnectionId &destination_cid, - const QUICConnectionId &source_cid, QUICPacketNumber packet_number, QUICPacketNumber base_packet_number, - QUICVersion version, bool crypto, ats_unique_buf buf, size_t len, - ats_unique_buf token = ats_unique_buf(nullptr), size_t token_len = 0); - QUICPacketLongHeader(QUICPacketType type, QUICKeyPhase key_phase, QUICVersion version, const QUICConnectionId &destination_cid, - const QUICConnectionId &source_cid, const QUICConnectionId &original_dcid, ats_unique_buf retry_token, - size_t retry_token_len); + /** + * For receiving packet + */ + QUICLongHeaderPacketR(UDPConnection *udp_con, IpEndpoint from, IpEndpoint to, Ptr blocks); + + virtual ~QUICLongHeaderPacketR(){}; - QUICPacketType type() const override; QUICConnectionId destination_cid() const override; - QUICConnectionId source_cid() const override; - QUICConnectionId original_dcid() const; - QUICPacketNumber packet_number() const override; - bool has_version() const override; - bool is_valid() const override; - bool is_crypto_packet() const override; - QUICVersion version() const override; - const uint8_t *payload() const override; - const uint8_t *token() const; - size_t token_len() const; - QUICKeyPhase key_phase() const override; - uint16_t size() const override; - void store(uint8_t *buf, size_t *len) const override; + QUICConnectionId source_cid() const; + virtual QUICVersion version() const; static bool type(QUICPacketType &type, const uint8_t *packet, size_t packet_len); static bool version(QUICVersion &version, const uint8_t *packet, size_t packet_len); + static bool key_phase(QUICKeyPhase &key_phase, const uint8_t *packet, size_t packet_len); + static bool length(size_t &length, uint8_t &length_field_len, size_t &length_field_offset, const uint8_t *packet, + size_t packet_len); + static bool packet_length(size_t &packet_len, const uint8_t *buf, size_t buf_len); + static bool packet_number_offset(size_t &pn_offset, const uint8_t *packet, size_t packet_len); + +protected: + QUICVersion _version; + QUICConnectionId _scid; + QUICConnectionId _dcid; +}; + +class QUICShortHeaderPacket : public QUICPacket +{ +public: /** - * Unlike QUICInvariants::dcil(), this returns actual connection id length + * For sending packet */ - static bool dcil(uint8_t &dcil, const uint8_t *packet, size_t packet_len); + QUICShortHeaderPacket(const QUICConnectionId &dcid, QUICPacketNumber packet_number, QUICPacketNumber base_packet_number, + QUICKeyPhase key_phase, bool ack_eliciting, bool probing); + + QUICPacketType type() const override; + QUICKeyPhase key_phase() const override; + QUICPacketNumber packet_number() const override; + QUICConnectionId destination_cid() const override; + + uint16_t payload_length() const override; + Ptr header_block() const override; + Ptr payload_block() const override; + + void attach_payload(Ptr payload, bool unprotected); + +private: + QUICConnectionId _dcid; + QUICPacketNumber _packet_number; + QUICKeyPhase _key_phase; + int _packet_number_len; + + Ptr _payload_block; + size_t _payload_length; +}; + +class QUICShortHeaderPacketR : public QUICPacketR +{ +public: /** - * Unlike QUICInvariants::scil(), this returns actual connection id length + * For receiving packet */ - static bool scil(uint8_t &scil, const uint8_t *packet, size_t packet_len); - static bool token_length(size_t &token_length, uint8_t *field_len, const uint8_t *packet, size_t packet_len); - static bool length(size_t &length, uint8_t *field_len, const uint8_t *packet, size_t packet_len); - static bool key_phase(QUICKeyPhase &key_phase, const uint8_t *packet, size_t packet_len); - static bool packet_number_offset(uint8_t &pn_offset, const uint8_t *packet, size_t packet_len); + QUICShortHeaderPacketR(UDPConnection *udp_con, IpEndpoint from, IpEndpoint to, Ptr blocks, + QUICPacketNumber base_packet_number); + + void attach_payload(Ptr payload, bool unprotected); + + QUICPacketType type() const override; + QUICKeyPhase key_phase() const override; + QUICPacketNumber packet_number() const override; + QUICConnectionId destination_cid() const override; + + Ptr header_block() const override; + Ptr payload_block() const override; + + static bool packet_number_offset(size_t &pn_offset, const uint8_t *packet, size_t packet_len, int dcil); private: - QUICConnectionId _destination_cid = QUICConnectionId::ZERO(); - QUICConnectionId _source_cid = QUICConnectionId::ZERO(); - QUICConnectionId _original_dcid = QUICConnectionId::ZERO(); //< RETRY packet only - size_t _token_len = 0; //< INITIAL packet only - size_t _token_offset = 0; //< INITIAL packet only - ats_unique_buf _token = ats_unique_buf(nullptr); //< INITIAL packet only - size_t _payload_offset = 0; - bool _is_crypto_packet = false; + QUICKeyPhase _key_phase; + QUICPacketNumber _packet_number; + int _packet_number_len; + QUICConnectionId _dcid; }; -class QUICPacketShortHeader : public QUICPacketHeader +class QUICStatelessResetPacket : public QUICPacket { public: - QUICPacketShortHeader() : QUICPacketHeader(){}; - virtual ~QUICPacketShortHeader(){}; - QUICPacketShortHeader(const IpEndpoint from, ats_unique_buf buf, size_t len, QUICPacketNumber base); - QUICPacketShortHeader(QUICPacketType type, QUICKeyPhase key_phase, QUICPacketNumber packet_number, - QUICPacketNumber base_packet_number, ats_unique_buf buf, size_t len); - QUICPacketShortHeader(QUICPacketType type, QUICKeyPhase key_phase, const QUICConnectionId &connection_id, - QUICPacketNumber packet_number, QUICPacketNumber base_packet_number, ats_unique_buf buf, size_t len); + /** + * For sending packet + */ + QUICStatelessResetPacket(QUICStatelessResetToken token, size_t maximum_size); + QUICPacketType type() const override; + QUICPacketNumber packet_number() const override; QUICConnectionId destination_cid() const override; - QUICConnectionId - source_cid() const override - { - return QUICConnectionId::ZERO(); - } + + Ptr header_block() const override; + Ptr payload_block() const override; + + QUICStatelessResetToken token() const; + +private: + QUICStatelessResetToken _token; + size_t _maximum_size; +}; + +class QUICStatelessResetPacketR : public QUICPacketR +{ +public: + /** + * For receiving packet + */ + QUICStatelessResetPacketR(UDPConnection *udp_con, IpEndpoint from, IpEndpoint to, Ptr blocks); + + QUICPacketType type() const override; QUICPacketNumber packet_number() const override; - bool has_version() const override; - bool is_valid() const override; + QUICConnectionId destination_cid() const override; +}; + +class QUICVersionNegotiationPacket : public QUICLongHeaderPacket +{ +public: + /** + * For sending packet + */ + QUICVersionNegotiationPacket(const QUICConnectionId &dcid, const QUICConnectionId &scid, const QUICVersion versions[], + int nversions, QUICVersion version_in_initial); + + QUICPacketType type() const override; QUICVersion version() const override; - const uint8_t *payload() const override; - QUICKeyPhase key_phase() const override; - uint16_t size() const override; - void store(uint8_t *buf, size_t *len) const override; + QUICPacketNumber packet_number() const override; + uint16_t payload_length() const override; - static bool key_phase(QUICKeyPhase &key_phase, const uint8_t *packet, size_t packet_len); - static bool packet_number_offset(uint8_t &pn_offset, const uint8_t *packet, size_t packet_len, int dcil); + Ptr header_block() const override; + Ptr payload_block() const override; + + const QUICVersion *versions() const; + int nversions() const; private: - int _packet_number_len; + const QUICVersion *_versions; + int _nversions; + const QUICVersion _version_in_initial; }; -class QUICPacketHeaderDeleter +class QUICVersionNegotiationPacketR : public QUICLongHeaderPacketR { public: - static void - delete_null_header(QUICPacketHeader *header) - { - ink_assert(header == nullptr); - } + /** + * For receiving packet + */ + QUICVersionNegotiationPacketR(UDPConnection *udp_con, IpEndpoint from, IpEndpoint to, Ptr blocks); - static void - delete_long_header(QUICPacketHeader *header) - { - QUICPacketLongHeader *long_header = dynamic_cast(header); - ink_assert(long_header != nullptr); - long_header->~QUICPacketLongHeader(); - quicPacketLongHeaderAllocator.free(long_header); - } + QUICPacketType type() const override; + QUICPacketNumber packet_number() const override; + QUICConnectionId destination_cid() const override; - static void - delete_short_header(QUICPacketHeader *header) - { - QUICPacketShortHeader *short_header = dynamic_cast(header); - ink_assert(short_header != nullptr); - short_header->~QUICPacketShortHeader(); - quicPacketShortHeaderAllocator.free(short_header); - } + Ptr header_block() const override; + Ptr payload_block() const override; + + const QUICVersion supported_version(uint8_t index) const; + int nversions() const; + +private: + QUICConnectionId _dcid; + uint8_t *_versions; + int _nversions; }; -class QUICPacket +class QUICInitialPacket : public QUICLongHeaderPacket { public: - QUICPacket(); - - /* - * Creates a QUICPacket with a QUICPacketHeader and a buffer that contains payload - * - * This will be used for receiving packets. Therefore, it is expected that payload is already decrypted. - * However, QUICPacket class itself doesn't care about whether the payload is protected (encrypted) or not. + /** + * For sending packet */ - QUICPacket(UDPConnection *udp_con, QUICPacketHeaderUPtr header, ats_unique_buf payload, size_t payload_len); + QUICInitialPacket(QUICVersion version, const QUICConnectionId &dcid, const QUICConnectionId &scid, size_t token_len, + ats_unique_buf token, size_t length, QUICPacketNumber packet_number, bool ack_eliciting, bool probing, + bool crypto); - QUICPacket(QUICPacketHeaderUPtr header, ats_unique_buf payload, size_t payload_len, std::vector &frames); + QUICPacketType type() const override; + QUICPacketNumber packet_number() const override; + QUICKeyPhase key_phase() const override; - /* - * Creates a QUICPacket with a QUICPacketHeader, a buffer that contains payload and a flag that indicates whether the packet is - * ack_eliciting - * - * This will be used for sending packets. Therefore, it is expected that payload is already encrypted. - * However, QUICPacket class itself doesn't care about whether the payload is protected (encrypted) or not. + Ptr header_block() const override; + Ptr payload_block() const override; + void attach_payload(Ptr payload, bool unprotected); + +private: + size_t _token_len = 0; + ats_unique_buf _token = ats_unique_buf(nullptr); + QUICPacketNumber _packet_number; +}; + +class QUICInitialPacketR : public QUICLongHeaderPacketR +{ +public: + /** + * For receiving packet */ - QUICPacket(QUICPacketHeaderUPtr header, ats_unique_buf payload, size_t payload_len, bool ack_eliciting, bool probing); + QUICInitialPacketR(UDPConnection *udp_con, IpEndpoint from, IpEndpoint to, Ptr blocks, + QUICPacketNumber base_packet_number); + ~QUICInitialPacketR(); - QUICPacket(QUICPacketHeaderUPtr header, ats_unique_buf payload, size_t payload_len, bool ack_eliciting, bool probing, - std::vector &frames); + Ptr header_block() const override; + Ptr payload_block() const override; + void attach_payload(Ptr payload, bool unprotected); - ~QUICPacket(); + QUICPacketType type() const override; + QUICPacketNumber packet_number() const override; + QUICKeyPhase key_phase() const override; - UDPConnection *udp_con() const; - const IpEndpoint &from() const; - QUICPacketType type() const; - QUICConnectionId destination_cid() const; - QUICConnectionId source_cid() const; - QUICPacketNumber packet_number() const; - QUICVersion version() const; - const QUICPacketHeader &header() const; - const uint8_t *payload() const; - bool is_ack_eliciting() const; - bool is_crypto_packet() const; - bool is_probing_packet() const; + const QUICAddressValidationToken &token() const; - /* - * Size of whole QUIC packet (header + payload + integrity check) + static bool token_length(size_t &token_length, uint8_t &field_len, size_t &token_length_filed_offset, const uint8_t *packet, + size_t packet_len); + +protected: + Ptr _payload_block; + +private: + QUICPacketNumber _packet_number; + QUICAddressValidationToken *_token = nullptr; + + bool _parse(); +}; + +class QUICZeroRttPacket : public QUICLongHeaderPacket +{ +public: + /** + * For sending packet */ - uint16_t size() const; + QUICZeroRttPacket(QUICVersion version, const QUICConnectionId &dcid, const QUICConnectionId &scid, size_t length, + QUICPacketNumber packet_number, bool ack_eliciting, bool probing); - /* - * Size of header + QUICPacketType type() const override; + QUICPacketNumber packet_number() const override; + QUICKeyPhase key_phase() const override; + + Ptr header_block() const override; + Ptr payload_block() const override; + void attach_payload(Ptr payload, bool unprotected); + +private: + QUICPacketNumber _packet_number; +}; + +class QUICZeroRttPacketR : public QUICLongHeaderPacketR +{ +public: + /** + * For receiving packet */ - uint16_t header_size() const; + QUICZeroRttPacketR(UDPConnection *udp_con, IpEndpoint from, IpEndpoint to, Ptr blocks, + QUICPacketNumber base_packet_number); - /* - * Length of payload + Ptr header_block() const override; + Ptr payload_block() const override; + void attach_payload(Ptr payload, bool unprotected); + + QUICPacketType type() const override; + QUICPacketNumber packet_number() const override; + QUICKeyPhase key_phase() const override; + +private: + QUICPacketNumber _packet_number; +}; + +class QUICHandshakePacket : public QUICLongHeaderPacket +{ +public: + /** + * For sending packet */ - uint16_t payload_length() const; + QUICHandshakePacket(QUICVersion version, const QUICConnectionId &dcid, const QUICConnectionId &scid, size_t length, + QUICPacketNumber packet_number, bool ack_eliciting, bool probing, bool crypto); - void store(uint8_t *buf, size_t *len) const; - QUICKeyPhase key_phase() const; + QUICPacketType type() const override; + QUICKeyPhase key_phase() const override; + QUICPacketNumber packet_number() const override; - /***** STATIC MEMBERS *****/ + Ptr header_block() const override; + Ptr payload_block() const override; + void attach_payload(Ptr payload, bool unprotected); - static uint8_t calc_packet_number_len(QUICPacketNumber num, QUICPacketNumber base); - static bool encode_packet_number(QUICPacketNumber &dst, QUICPacketNumber src, size_t len); - static bool decode_packet_number(QUICPacketNumber &dst, QUICPacketNumber src, size_t len, QUICPacketNumber largest_acked); +private: + QUICPacketNumber _packet_number; +}; + +class QUICHandshakePacketR : public QUICLongHeaderPacketR +{ +public: + /** + * For receiving packet + */ + QUICHandshakePacketR(UDPConnection *udp_con, IpEndpoint from, IpEndpoint to, Ptr blocks, + QUICPacketNumber base_packet_number); + + Ptr header_block() const override; + Ptr payload_block() const override; + void attach_payload(Ptr payload, bool unprotected); - LINK(QUICPacket, link); + QUICPacketType type() const override; + QUICKeyPhase key_phase() const override; + QUICPacketNumber packet_number() const override; private: - UDPConnection *_udp_con = nullptr; - QUICPacketHeaderUPtr _header = QUICPacketHeaderUPtr(nullptr, &QUICPacketHeaderDeleter::delete_null_header); - ats_unique_buf _payload = ats_unique_buf(nullptr); - size_t _payload_size = 0; - bool _is_ack_eliciting = false; - bool _is_probing_packet = false; + QUICPacketNumber _packet_number; }; -using QUICPacketDeleterFunc = void (*)(QUICPacket *p); -using QUICPacketUPtr = std::unique_ptr; +class QUICRetryPacket : public QUICLongHeaderPacket +{ +public: + /** + * For sending packet + */ + QUICRetryPacket(QUICVersion version, const QUICConnectionId &dcid, const QUICConnectionId &scid, QUICRetryToken &token); -class QUICPacketDeleter + QUICPacketType type() const override; + QUICPacketNumber packet_number() const override; + uint16_t payload_length() const override; + + Ptr header_block() const override; + Ptr payload_block() const override; + + const QUICRetryToken &token() const; + +private: + QUICRetryToken _token; + + bool _compute_retry_integrity_tag(uint8_t *out, QUICConnectionId odcid, Ptr header, + Ptr payload) const; +}; + +class QUICRetryPacketR : public QUICLongHeaderPacketR { public: - // TODO Probably these methods should call destructor - static void - delete_null_packet(QUICPacket *packet) - { - ink_assert(packet == nullptr); - } + /** + * For receiving packet + */ + QUICRetryPacketR(UDPConnection *udp_con, IpEndpoint from, IpEndpoint to, Ptr blocks); + ~QUICRetryPacketR(); - static void - delete_packet(QUICPacket *packet) - { - packet->~QUICPacket(); - quicPacketAllocator.free(packet); - } + Ptr header_block() const override; + Ptr payload_block() const override; + + QUICPacketType type() const override; + QUICPacketNumber packet_number() const override; + + const QUICAddressValidationToken &token() const; + bool has_valid_tag(const QUICConnectionId &odcid) const; + +private: + QUICAddressValidationToken *_token = nullptr; + uint8_t _integrity_tag[QUICRetryIntegrityTag::LEN]; + Ptr _payload_block_without_tag; }; diff --git a/iocore/net/quic/QUICPacketFactory.cc b/iocore/net/quic/QUICPacketFactory.cc index 4c88c1b3151..2fcb1a29549 100644 --- a/iocore/net/quic/QUICPacketFactory.cc +++ b/iocore/net/quic/QUICPacketFactory.cc @@ -62,281 +62,256 @@ QUICPacketFactory::create_null_packet() } QUICPacketUPtr -QUICPacketFactory::create(UDPConnection *udp_con, IpEndpoint from, ats_unique_buf buf, size_t len, - QUICPacketNumber base_packet_number, QUICPacketCreationResult &result) +QUICPacketFactory::create(uint8_t *packet_buf, UDPConnection *udp_con, IpEndpoint from, IpEndpoint to, ats_unique_buf buf, + size_t len, QUICPacketNumber base_packet_number, QUICPacketCreationResult &result) { - size_t max_plain_txt_len = 2048; - ats_unique_buf plain_txt = ats_unique_malloc(max_plain_txt_len); - size_t plain_txt_len = 0; - - QUICPacketHeaderUPtr header = QUICPacketHeader::load(from, std::move(buf), len, base_packet_number); - - QUICConnectionId dcid = header->destination_cid(); - QUICConnectionId scid = header->source_cid(); - QUICVDebug(scid, dcid, "Decrypting %s packet #%" PRIu64 " using %s", QUICDebugNames::packet_type(header->type()), - header->packet_number(), QUICDebugNames::key_phase(header->key_phase())); - - if (header->has_version() && !QUICTypeUtil::is_supported_version(header->version())) { - if (header->type() == QUICPacketType::VERSION_NEGOTIATION) { - // version of VN packet is 0x00000000 - // This packet is unprotected. Just copy the payload - result = QUICPacketCreationResult::SUCCESS; - memcpy(plain_txt.get(), header->payload(), header->payload_size()); - plain_txt_len = header->payload_size(); - } else { - // We can't decrypt packets that have unknown versions - // What we can use is invariant field of Long Header - version, dcid, and scid - result = QUICPacketCreationResult::UNSUPPORTED; - } - } else { - Ptr plain = make_ptr(new_IOBufferBlock()); - Ptr protected_ibb = make_ptr(new_IOBufferBlock()); - protected_ibb->set_internal(reinterpret_cast(const_cast(header->payload())), header->payload_size(), - BUFFER_SIZE_NOT_ALLOCATED); - Ptr header_ibb = make_ptr(new_IOBufferBlock()); - header_ibb->set_internal(reinterpret_cast(const_cast(header->buf())), header->size(), - BUFFER_SIZE_NOT_ALLOCATED); - - switch (header->type()) { - case QUICPacketType::STATELESS_RESET: - case QUICPacketType::RETRY: - // These packets are unprotected. Just copy the payload - memcpy(plain_txt.get(), header->payload(), header->payload_size()); - plain_txt_len = header->payload_size(); - result = QUICPacketCreationResult::SUCCESS; - break; - case QUICPacketType::PROTECTED: - if (this->_pp_key_info.is_decryption_key_available(header->key_phase())) { - plain = this->_pp_protector.unprotect(header_ibb, protected_ibb, header->packet_number(), header->key_phase()); - if (plain != nullptr) { - memcpy(plain_txt.get(), plain->buf(), plain->size()); - plain_txt_len = plain->size(); - result = QUICPacketCreationResult::SUCCESS; - } else { - result = QUICPacketCreationResult::FAILED; - } + QUICPacket *packet = nullptr; + + // FIXME This is temporal. Receive IOBufferBlock from the caller. + Ptr whole_data = make_ptr(new_IOBufferBlock()); + whole_data->alloc(iobuffer_size_to_index(len, BUFFER_SIZE_INDEX_32K)); + memcpy(whole_data->start(), buf.get(), len); + whole_data->fill(len); + + QUICPacketType type; + QUICVersion version; + QUICConnectionId dcid; + QUICConnectionId scid; + QUICPacketNumber packet_number; + QUICKeyPhase key_phase; + + if (QUICPacketR::read_essential_info(whole_data, type, version, dcid, scid, packet_number, base_packet_number, key_phase)) { + QUICVDebug(scid, dcid, "Decrypting %s packet #%" PRIu64 " using %s", QUICDebugNames::packet_type(type), packet_number, + QUICDebugNames::key_phase(key_phase)); + + if (type != QUICPacketType::PROTECTED && !QUICTypeUtil::is_supported_version(version)) { + if (type == QUICPacketType::VERSION_NEGOTIATION) { + packet = new QUICVersionNegotiationPacketR(udp_con, from, to, whole_data); + result = QUICPacketCreationResult::SUCCESS; } else { - result = QUICPacketCreationResult::NOT_READY; + // We can't decrypt packets that have unknown versions + // What we can use is invariant field of Long Header - version, dcid, and scid + result = QUICPacketCreationResult::UNSUPPORTED; } - break; - case QUICPacketType::INITIAL: - if (this->_pp_key_info.is_decryption_key_available(QUICKeyPhase::INITIAL)) { - if (QUICTypeUtil::is_supported_version(header->version())) { - plain = this->_pp_protector.unprotect(header_ibb, protected_ibb, header->packet_number(), header->key_phase()); + } else { + Ptr plain; + switch (type) { + case QUICPacketType::STATELESS_RESET: + packet = new (packet_buf) QUICStatelessResetPacketR(udp_con, from, to, whole_data); + result = QUICPacketCreationResult::SUCCESS; + break; + case QUICPacketType::RETRY: + packet = new (packet_buf) QUICRetryPacketR(udp_con, from, to, whole_data); + result = QUICPacketCreationResult::SUCCESS; + break; + case QUICPacketType::PROTECTED: + packet = new (packet_buf) QUICShortHeaderPacketR(udp_con, from, to, whole_data, base_packet_number); + if (this->_pp_key_info.is_decryption_key_available(packet->key_phase())) { + plain = this->_pp_protector.unprotect(packet->header_block(), packet->payload_block(), packet->packet_number(), + packet->key_phase()); if (plain != nullptr) { - memcpy(plain_txt.get(), plain->buf(), plain->size()); - plain_txt_len = plain->size(); - result = QUICPacketCreationResult::SUCCESS; + static_cast(packet)->attach_payload(plain, true); + result = QUICPacketCreationResult::SUCCESS; } else { result = QUICPacketCreationResult::FAILED; } } else { - result = QUICPacketCreationResult::SUCCESS; + result = QUICPacketCreationResult::NOT_READY; } - } else { - result = QUICPacketCreationResult::IGNORED; - } - break; - case QUICPacketType::HANDSHAKE: - if (this->_pp_key_info.is_decryption_key_available(QUICKeyPhase::HANDSHAKE)) { - plain = this->_pp_protector.unprotect(header_ibb, protected_ibb, header->packet_number(), header->key_phase()); - if (plain != nullptr) { - memcpy(plain_txt.get(), plain->buf(), plain->size()); - plain_txt_len = plain->size(); - result = QUICPacketCreationResult::SUCCESS; + break; + case QUICPacketType::INITIAL: + packet = new (packet_buf) QUICInitialPacketR(udp_con, from, to, whole_data, base_packet_number); + if (this->_pp_key_info.is_decryption_key_available(QUICKeyPhase::INITIAL)) { + plain = this->_pp_protector.unprotect(packet->header_block(), packet->payload_block(), packet->packet_number(), + packet->key_phase()); + if (plain != nullptr) { + static_cast(packet)->attach_payload(plain, true); + result = QUICPacketCreationResult::SUCCESS; + } else { + result = QUICPacketCreationResult::FAILED; + } } else { - result = QUICPacketCreationResult::FAILED; + result = QUICPacketCreationResult::IGNORED; } - } else { - result = QUICPacketCreationResult::IGNORED; - } - break; - case QUICPacketType::ZERO_RTT_PROTECTED: - if (this->_pp_key_info.is_decryption_key_available(QUICKeyPhase::ZERO_RTT)) { - plain = this->_pp_protector.unprotect(header_ibb, protected_ibb, header->packet_number(), header->key_phase()); - if (plain != nullptr) { - memcpy(plain_txt.get(), plain->buf(), plain->size()); - plain_txt_len = plain->size(); - result = QUICPacketCreationResult::SUCCESS; + break; + case QUICPacketType::HANDSHAKE: + packet = new (packet_buf) QUICHandshakePacketR(udp_con, from, to, whole_data, base_packet_number); + if (this->_pp_key_info.is_decryption_key_available(QUICKeyPhase::HANDSHAKE)) { + plain = this->_pp_protector.unprotect(packet->header_block(), packet->payload_block(), packet->packet_number(), + packet->key_phase()); + if (plain != nullptr) { + static_cast(packet)->attach_payload(plain, true); + result = QUICPacketCreationResult::SUCCESS; + } else { + result = QUICPacketCreationResult::FAILED; + } } else { result = QUICPacketCreationResult::IGNORED; } - } else { - result = QUICPacketCreationResult::NOT_READY; + break; + case QUICPacketType::ZERO_RTT_PROTECTED: + packet = new (packet_buf) QUICZeroRttPacketR(udp_con, from, to, whole_data, base_packet_number); + if (this->_pp_key_info.is_decryption_key_available(QUICKeyPhase::ZERO_RTT)) { + plain = this->_pp_protector.unprotect(packet->header_block(), packet->payload_block(), packet->packet_number(), + packet->key_phase()); + if (plain != nullptr) { + static_cast(packet)->attach_payload(plain, true); + result = QUICPacketCreationResult::SUCCESS; + } else { + result = QUICPacketCreationResult::IGNORED; + } + } else { + result = QUICPacketCreationResult::NOT_READY; + } + break; + default: + result = QUICPacketCreationResult::FAILED; + break; } - break; - default: - result = QUICPacketCreationResult::FAILED; - break; } + } else { + Debug(tag.data(), "Failed to read essential field"); + uint8_t *buf = reinterpret_cast(whole_data->start()); + if (len > 16) { + Debug(tag_v.data(), "%02x %02x %02x %02x %02x %02x %02x %02x %02x %02x %02x %02x %02x %02x %02x %02x", buf[0], buf[1], buf[2], + buf[3], buf[4], buf[5], buf[6], buf[7], buf[8], buf[9], buf[10], buf[11], buf[12], buf[13], buf[14], buf[15]); + } + if (len > 32) { + Debug(tag_v.data(), "%02x %02x %02x %02x %02x %02x %02x %02x %02x %02x %02x %02x %02x %02x %02x %02x", buf[16 + 0], + buf[16 + 1], buf[16 + 2], buf[16 + 3], buf[16 + 4], buf[16 + 5], buf[16 + 6], buf[16 + 7], buf[16 + 8], buf[16 + 9], + buf[16 + 10], buf[16 + 11], buf[16 + 12], buf[16 + 13], buf[16 + 14], buf[16 + 15]); + } + if (len > 48) { + Debug(tag_v.data(), "%02x %02x %02x %02x %02x %02x %02x %02x %02x %02x %02x %02x %02x %02x %02x %02x", buf[32 + 0], + buf[32 + 1], buf[32 + 2], buf[32 + 3], buf[32 + 4], buf[32 + 5], buf[32 + 6], buf[32 + 7], buf[32 + 8], buf[32 + 9], + buf[32 + 10], buf[32 + 11], buf[32 + 12], buf[32 + 13], buf[32 + 14], buf[32 + 15]); + } + result = QUICPacketCreationResult::FAILED; } - QUICPacket *packet = nullptr; - if (result == QUICPacketCreationResult::SUCCESS || result == QUICPacketCreationResult::UNSUPPORTED) { - packet = quicPacketAllocator.alloc(); - new (packet) QUICPacket(udp_con, std::move(header), std::move(plain_txt), plain_txt_len); + if (result != QUICPacketCreationResult::SUCCESS && result != QUICPacketCreationResult::UNSUPPORTED) { + packet = nullptr; } - return QUICPacketUPtr(packet, &QUICPacketDeleter::delete_packet); + return QUICPacketUPtr(packet, &QUICPacketDeleter::delete_dont_free); } QUICPacketUPtr -QUICPacketFactory::create_version_negotiation_packet(QUICConnectionId dcid, QUICConnectionId scid) +QUICPacketFactory::create_version_negotiation_packet(QUICConnectionId dcid, QUICConnectionId scid, QUICVersion version_in_initial) { - size_t len = sizeof(QUICVersion) * (countof(QUIC_SUPPORTED_VERSIONS) + 1); - ats_unique_buf versions(reinterpret_cast(ats_malloc(len))); - uint8_t *p = versions.get(); - - size_t n; - for (auto v : QUIC_SUPPORTED_VERSIONS) { - QUICTypeUtil::write_QUICVersion(v, p, &n); - p += n; - } - - // [draft-18] 6.3. Using Reserved Versions - // To help ensure this, a server SHOULD include a reserved version (see Section 15) while generating a - // Version Negotiation packet. - QUICTypeUtil::write_QUICVersion(QUIC_EXERCISE_VERSION, p, &n); - p += n; - - ink_assert(len == static_cast(p - versions.get())); - // VN packet dosen't have packet number field and version field is always 0x00000000 - QUICPacketHeaderUPtr header = QUICPacketHeader::build(QUICPacketType::VERSION_NEGOTIATION, QUICKeyPhase::INITIAL, dcid, scid, - 0x00, 0x00, 0x00, false, std::move(versions), len); - - return QUICPacketFactory::_create_unprotected_packet(std::move(header)); + return QUICPacketUPtr( + new QUICVersionNegotiationPacket(dcid, scid, QUIC_SUPPORTED_VERSIONS, countof(QUIC_SUPPORTED_VERSIONS), version_in_initial), + &QUICPacketDeleter::delete_packet_new); } QUICPacketUPtr -QUICPacketFactory::create_initial_packet(QUICConnectionId destination_cid, QUICConnectionId source_cid, - QUICPacketNumber base_packet_number, ats_unique_buf payload, size_t len, - bool retransmittable, bool probing, bool crypto, ats_unique_buf token, size_t token_len) +QUICPacketFactory::create_initial_packet(uint8_t *packet_buf, QUICConnectionId destination_cid, QUICConnectionId source_cid, + QUICPacketNumber base_packet_number, Ptr payload, size_t length, + bool ack_eliciting, bool probing, bool crypto, ats_unique_buf token, size_t token_len) { QUICPacketNumberSpace index = QUICTypeUtil::pn_space(QUICEncryptionLevel::INITIAL); QUICPacketNumber pn = this->_packet_number_generator[static_cast(index)].next(); - QUICPacketHeaderUPtr header = - QUICPacketHeader::build(QUICPacketType::INITIAL, QUICKeyPhase::INITIAL, destination_cid, source_cid, pn, base_packet_number, - this->_version, crypto, std::move(payload), len, std::move(token), token_len); - return this->_create_encrypted_packet(std::move(header), retransmittable, probing); -} -QUICPacketUPtr -QUICPacketFactory::create_retry_packet(QUICConnectionId destination_cid, QUICConnectionId source_cid, - QUICConnectionId original_dcid, QUICRetryToken &token) -{ - ats_unique_buf payload = ats_unique_malloc(token.length()); - memcpy(payload.get(), token.buf(), token.length()); + QUICInitialPacket *packet = new (packet_buf) QUICInitialPacket(this->_version, destination_cid, source_cid, token_len, + std::move(token), length, pn, ack_eliciting, probing, crypto); - QUICPacketHeaderUPtr header = - QUICPacketHeader::build(QUICPacketType::RETRY, QUICKeyPhase::INITIAL, QUIC_SUPPORTED_VERSIONS[0], destination_cid, source_cid, - original_dcid, std::move(payload), token.length()); - return QUICPacketFactory::_create_unprotected_packet(std::move(header)); -} + packet->attach_payload(payload, true); // Attach a cleartext payload with extra headers + Ptr protected_payload = + this->_pp_protector.protect(packet->header_block(), packet->payload_block(), packet->packet_number(), packet->key_phase()); + if (protected_payload != nullptr) { + packet->attach_payload(protected_payload, false); // Replace its payload with the protected payload + } else { + QUICDebug(destination_cid, source_cid, "Failed to encrypt a packet"); + packet = nullptr; + } -QUICPacketUPtr -QUICPacketFactory::create_handshake_packet(QUICConnectionId destination_cid, QUICConnectionId source_cid, - QUICPacketNumber base_packet_number, ats_unique_buf payload, size_t len, - bool retransmittable, bool probing, bool crypto) -{ - QUICPacketNumberSpace index = QUICTypeUtil::pn_space(QUICEncryptionLevel::INITIAL); - QUICPacketNumber pn = this->_packet_number_generator[static_cast(index)].next(); - QUICPacketHeaderUPtr header = - QUICPacketHeader::build(QUICPacketType::HANDSHAKE, QUICKeyPhase::HANDSHAKE, destination_cid, source_cid, pn, base_packet_number, - this->_version, crypto, std::move(payload), len); - return this->_create_encrypted_packet(std::move(header), retransmittable, probing); + return QUICPacketUPtr(packet, &QUICPacketDeleter::delete_dont_free); } QUICPacketUPtr -QUICPacketFactory::create_zero_rtt_packet(QUICConnectionId destination_cid, QUICConnectionId source_cid, - QUICPacketNumber base_packet_number, ats_unique_buf payload, size_t len, - bool retransmittable, bool probing) +QUICPacketFactory::create_retry_packet(QUICVersion version, QUICConnectionId destination_cid, QUICConnectionId source_cid, + QUICRetryToken &token) { - QUICPacketNumberSpace index = QUICTypeUtil::pn_space(QUICEncryptionLevel::INITIAL); - QUICPacketNumber pn = this->_packet_number_generator[static_cast(index)].next(); - QUICPacketHeaderUPtr header = - QUICPacketHeader::build(QUICPacketType::ZERO_RTT_PROTECTED, QUICKeyPhase::ZERO_RTT, destination_cid, source_cid, pn, - base_packet_number, this->_version, false, std::move(payload), len); - return this->_create_encrypted_packet(std::move(header), retransmittable, probing); + return QUICPacketUPtr(new QUICRetryPacket(version, destination_cid, source_cid, token), &QUICPacketDeleter::delete_packet_new); } QUICPacketUPtr -QUICPacketFactory::create_protected_packet(QUICConnectionId connection_id, QUICPacketNumber base_packet_number, - ats_unique_buf payload, size_t len, bool retransmittable, bool probing) +QUICPacketFactory::create_handshake_packet(uint8_t *packet_buf, QUICConnectionId destination_cid, QUICConnectionId source_cid, + QUICPacketNumber base_packet_number, Ptr payload, size_t length, + bool ack_eliciting, bool probing, bool crypto) { - QUICPacketNumberSpace index = QUICTypeUtil::pn_space(QUICEncryptionLevel::INITIAL); + QUICPacketNumberSpace index = QUICTypeUtil::pn_space(QUICEncryptionLevel::HANDSHAKE); QUICPacketNumber pn = this->_packet_number_generator[static_cast(index)].next(); - // TODO Key phase should be picked up from QUICHandshakeProtocol, probably - QUICPacketHeaderUPtr header = QUICPacketHeader::build(QUICPacketType::PROTECTED, QUICKeyPhase::PHASE_0, connection_id, pn, - base_packet_number, std::move(payload), len); - return this->_create_encrypted_packet(std::move(header), retransmittable, probing); -} - -QUICPacketUPtr -QUICPacketFactory::create_stateless_reset_packet(QUICConnectionId connection_id, QUICStatelessResetToken stateless_reset_token) -{ - std::random_device rnd; - uint8_t random_packet_number = static_cast(rnd() & 0xFF); - size_t payload_len = static_cast((rnd() & 0xFF) | 16); // Mimimum length has to be 16 - ats_unique_buf payload = ats_unique_malloc(payload_len + 16); - uint8_t *naked_payload = payload.get(); + QUICHandshakePacket *packet = + new (packet_buf) QUICHandshakePacket(this->_version, destination_cid, source_cid, length, pn, ack_eliciting, probing, crypto); - // Generate random octets - for (int i = payload_len - 1; i >= 0; --i) { - naked_payload[i] = static_cast(rnd() & 0xFF); + packet->attach_payload(payload, true); // Attach a cleartext payload with extra headers + Ptr protected_payload = + this->_pp_protector.protect(packet->header_block(), packet->payload_block(), packet->packet_number(), packet->key_phase()); + if (protected_payload != nullptr) { + packet->attach_payload(protected_payload, false); // Replace its payload with the protected payload + } else { + QUICDebug(destination_cid, source_cid, "Failed to encrypt a packet"); + packet = nullptr; } - // Copy stateless reset token into payload - memcpy(naked_payload + payload_len - 16, stateless_reset_token.buf(), 16); - // KeyPhase won't be used - QUICPacketHeaderUPtr header = QUICPacketHeader::build(QUICPacketType::STATELESS_RESET, QUICKeyPhase::INITIAL, connection_id, - random_packet_number, 0, std::move(payload), payload_len); - return QUICPacketFactory::_create_unprotected_packet(std::move(header)); + return QUICPacketUPtr(packet, &QUICPacketDeleter::delete_dont_free); } QUICPacketUPtr -QUICPacketFactory::_create_unprotected_packet(QUICPacketHeaderUPtr header) +QUICPacketFactory::create_zero_rtt_packet(uint8_t *packet_buf, QUICConnectionId destination_cid, QUICConnectionId source_cid, + QUICPacketNumber base_packet_number, Ptr payload, size_t length, + bool ack_eliciting, bool probing) { - ats_unique_buf cleartext = ats_unique_malloc(2048); - size_t cleartext_len = header->payload_size(); + QUICPacketNumberSpace index = QUICTypeUtil::pn_space(QUICEncryptionLevel::ZERO_RTT); + QUICPacketNumber pn = this->_packet_number_generator[static_cast(index)].next(); + + QUICZeroRttPacket *packet = + new (packet_buf) QUICZeroRttPacket(this->_version, destination_cid, source_cid, length, pn, ack_eliciting, probing); - memcpy(cleartext.get(), header->payload(), cleartext_len); - QUICPacket *packet = quicPacketAllocator.alloc(); - new (packet) QUICPacket(std::move(header), std::move(cleartext), cleartext_len, false, false); + packet->attach_payload(payload, true); // Attach a cleartext payload with extra headers + Ptr protected_payload = + this->_pp_protector.protect(packet->header_block(), packet->payload_block(), packet->packet_number(), packet->key_phase()); + if (protected_payload != nullptr) { + packet->attach_payload(protected_payload, false); // Replace its payload with the protected payload + } else { + QUICDebug(destination_cid, source_cid, "Failed to encrypt a packet"); + packet = nullptr; + } - return QUICPacketUPtr(packet, &QUICPacketDeleter::delete_packet); + return QUICPacketUPtr(packet, &QUICPacketDeleter::delete_dont_free); } QUICPacketUPtr -QUICPacketFactory::_create_encrypted_packet(QUICPacketHeaderUPtr header, bool retransmittable, bool probing) +QUICPacketFactory::create_short_header_packet(uint8_t *packet_buf, QUICConnectionId destination_cid, + QUICPacketNumber base_packet_number, Ptr payload, size_t length, + bool ack_eliciting, bool probing) { - QUICConnectionId dcid = header->destination_cid(); - QUICConnectionId scid = header->source_cid(); - QUICVDebug(dcid, scid, "Encrypting %s packet #%" PRIu64 " using %s", QUICDebugNames::packet_type(header->type()), - header->packet_number(), QUICDebugNames::key_phase(header->key_phase())); - - QUICPacket *packet = nullptr; - - Ptr payload_ibb = make_ptr(new_IOBufferBlock()); - payload_ibb->set_internal(reinterpret_cast(const_cast(header->payload())), header->payload_size(), - BUFFER_SIZE_NOT_ALLOCATED); + QUICPacketNumberSpace index = QUICTypeUtil::pn_space(QUICEncryptionLevel::ONE_RTT); + QUICPacketNumber pn = this->_packet_number_generator[static_cast(index)].next(); - Ptr header_ibb = make_ptr(new_IOBufferBlock()); - header_ibb->set_internal(reinterpret_cast(const_cast(header->buf())), header->size(), - BUFFER_SIZE_NOT_ALLOCATED); + // TODO Key phase should be picked up from QUICHandshakeProtocol, probably + QUICShortHeaderPacket *packet = + new (packet_buf) QUICShortHeaderPacket(destination_cid, pn, base_packet_number, QUICKeyPhase::PHASE_0, ack_eliciting, probing); + packet->attach_payload(payload, true); // Attach a cleartext payload with extra headers Ptr protected_payload = - this->_pp_protector.protect(header_ibb, payload_ibb, header->packet_number(), header->key_phase()); + this->_pp_protector.protect(packet->header_block(), packet->payload_block(), packet->packet_number(), packet->key_phase()); if (protected_payload != nullptr) { - ats_unique_buf cipher_txt = ats_unique_malloc(protected_payload->size()); - memcpy(cipher_txt.get(), protected_payload->buf(), protected_payload->size()); - packet = quicPacketAllocator.alloc(); - new (packet) QUICPacket(std::move(header), std::move(cipher_txt), protected_payload->size(), retransmittable, probing); + packet->attach_payload(protected_payload, false); // Replace its payload with the protected payload } else { - QUICDebug(dcid, scid, "Failed to encrypt a packet"); + QUICDebug(destination_cid, QUICConnectionId::ZERO(), "Failed to encrypt a packet"); + packet = nullptr; } - return QUICPacketUPtr(packet, &QUICPacketDeleter::delete_packet); + return QUICPacketUPtr(packet, &QUICPacketDeleter::delete_dont_free); +} + +QUICPacketUPtr +QUICPacketFactory::create_stateless_reset_packet(QUICStatelessResetToken stateless_reset_token, size_t maximum_size) +{ + return QUICPacketUPtr(new QUICStatelessResetPacket(stateless_reset_token, maximum_size), &QUICPacketDeleter::delete_packet_new); } void @@ -355,7 +330,7 @@ QUICPacketFactory::is_ready_to_create_protected_packet() void QUICPacketFactory::reset() { - for (auto i = 0; i < kPacketNumberSpace; i++) { + for (auto i = 0; i < QUIC_N_PACKET_SPACES; i++) { this->_packet_number_generator[i].reset(); } } diff --git a/iocore/net/quic/QUICPacketFactory.h b/iocore/net/quic/QUICPacketFactory.h index a00639dfda4..2d59de66c9f 100644 --- a/iocore/net/quic/QUICPacketFactory.h +++ b/iocore/net/quic/QUICPacketFactory.h @@ -44,28 +44,29 @@ class QUICPacketFactory { public: static QUICPacketUPtr create_null_packet(); - static QUICPacketUPtr create_version_negotiation_packet(QUICConnectionId dcid, QUICConnectionId scid); - static QUICPacketUPtr create_stateless_reset_packet(QUICConnectionId connection_id, - QUICStatelessResetToken stateless_reset_token); - static QUICPacketUPtr create_retry_packet(QUICConnectionId destination_cid, QUICConnectionId source_cid, - QUICConnectionId original_dcid, QUICRetryToken &token); + static QUICPacketUPtr create_version_negotiation_packet(QUICConnectionId dcid, QUICConnectionId scid, + QUICVersion version_in_version); + static QUICPacketUPtr create_stateless_reset_packet(QUICStatelessResetToken stateless_reset_token, size_t maximum_size); + static QUICPacketUPtr create_retry_packet(QUICVersion version, QUICConnectionId destination_cid, QUICConnectionId source_cid, + QUICRetryToken &token); QUICPacketFactory(const QUICPacketProtectionKeyInfo &pp_key_info) : _pp_key_info(pp_key_info), _pp_protector(pp_key_info) {} - QUICPacketUPtr create(UDPConnection *udp_con, IpEndpoint from, ats_unique_buf buf, size_t len, + QUICPacketUPtr create(uint8_t *packet_buf, UDPConnection *udp_con, IpEndpoint from, IpEndpoint to, ats_unique_buf buf, size_t len, QUICPacketNumber base_packet_number, QUICPacketCreationResult &result); - QUICPacketUPtr create_initial_packet(QUICConnectionId destination_cid, QUICConnectionId source_cid, - QUICPacketNumber base_packet_number, ats_unique_buf payload, size_t len, bool ack_eliciting, - bool probing, bool crypto, ats_unique_buf token = ats_unique_buf(nullptr), - size_t token_len = 0); - QUICPacketUPtr create_handshake_packet(QUICConnectionId destination_cid, QUICConnectionId source_cid, - QUICPacketNumber base_packet_number, ats_unique_buf payload, size_t len, + QUICPacketUPtr create_initial_packet(uint8_t *packet_buf, QUICConnectionId destination_cid, QUICConnectionId source_cid, + QUICPacketNumber base_packet_number, Ptr payload, size_t length, + bool ack_eliciting, bool probing, bool crypto, + ats_unique_buf token = ats_unique_buf(nullptr), size_t token_len = 0); + QUICPacketUPtr create_handshake_packet(uint8_t *packet_buf, QUICConnectionId destination_cid, QUICConnectionId source_cid, + QUICPacketNumber base_packet_number, Ptr payload, size_t length, bool ack_eliciting, bool probing, bool crypto); - QUICPacketUPtr create_zero_rtt_packet(QUICConnectionId destination_cid, QUICConnectionId source_cid, - QUICPacketNumber base_packet_number, ats_unique_buf payload, size_t len, bool ack_eliciting, - bool probing); - QUICPacketUPtr create_protected_packet(QUICConnectionId connection_id, QUICPacketNumber base_packet_number, - ats_unique_buf payload, size_t len, bool ack_eliciting, bool probing); + QUICPacketUPtr create_zero_rtt_packet(uint8_t *packet_buf, QUICConnectionId destination_cid, QUICConnectionId source_cid, + QUICPacketNumber base_packet_number, Ptr payload, size_t length, + bool ack_eliciting, bool probing); + QUICPacketUPtr create_short_header_packet(uint8_t *packet_buf, QUICConnectionId destination_cid, + QUICPacketNumber base_packet_number, Ptr payload, size_t length, + bool ack_eliciting, bool probing); void set_version(QUICVersion negotiated_version); bool is_ready_to_create_protected_packet(); @@ -79,7 +80,4 @@ class QUICPacketFactory // Initial, 0/1-RTT, and Handshake QUICPacketNumberGenerator _packet_number_generator[3]; - - static QUICPacketUPtr _create_unprotected_packet(QUICPacketHeaderUPtr header); - QUICPacketUPtr _create_encrypted_packet(QUICPacketHeaderUPtr header, bool ack_eliciting, bool probing); }; diff --git a/iocore/net/quic/QUICPacketHeaderProtector.cc b/iocore/net/quic/QUICPacketHeaderProtector.cc index 49a29adb3ba..f6a71b4da6d 100644 --- a/iocore/net/quic/QUICPacketHeaderProtector.cc +++ b/iocore/net/quic/QUICPacketHeaderProtector.cc @@ -32,19 +32,15 @@ bool QUICPacketHeaderProtector::protect(uint8_t *unprotected_packet, size_t unprotected_packet_len, int dcil) const { // Do nothing if the packet is VN - if (QUICInvariants::is_long_header(unprotected_packet)) { - QUICVersion version; - QUICPacketLongHeader::version(version, unprotected_packet, unprotected_packet_len); - if (version == 0x0) { - return true; - } + QUICPacketType type; + QUICPacketR::type(type, unprotected_packet, unprotected_packet_len); + if (type == QUICPacketType::VERSION_NEGOTIATION) { + return true; } QUICKeyPhase phase; - QUICPacketType type; if (QUICInvariants::is_long_header(unprotected_packet)) { - QUICPacketLongHeader::key_phase(phase, unprotected_packet, unprotected_packet_len); - QUICPacketLongHeader::type(type, unprotected_packet, unprotected_packet_len); + QUICLongHeaderPacketR::key_phase(phase, unprotected_packet, unprotected_packet_len); } else { // This is a kind of hack. For short header we need to use the same key for header protection regardless of the key phase. phase = QUICKeyPhase::PHASE_0; @@ -89,24 +85,15 @@ bool QUICPacketHeaderProtector::unprotect(uint8_t *protected_packet, size_t protected_packet_len) const { // Do nothing if the packet is VN or RETRY - if (QUICInvariants::is_long_header(protected_packet)) { - QUICVersion version; - QUICPacketLongHeader::version(version, protected_packet, protected_packet_len); - if (version == 0x0) { - return true; - } - QUICPacketType type; - QUICPacketLongHeader::type(type, protected_packet, protected_packet_len); - if (type == QUICPacketType::RETRY) { - return true; - } + QUICPacketType type; + QUICPacketR::type(type, protected_packet, protected_packet_len); + if (type == QUICPacketType::VERSION_NEGOTIATION || type == QUICPacketType::RETRY) { + return true; } QUICKeyPhase phase; - QUICPacketType type; if (QUICInvariants::is_long_header(protected_packet)) { - QUICPacketLongHeader::key_phase(phase, protected_packet, protected_packet_len); - QUICPacketLongHeader::type(type, protected_packet, protected_packet_len); + QUICLongHeaderPacketR::key_phase(phase, protected_packet, protected_packet_len); } else { // This is a kind of hack. For short header we need to use the same key for header protection regardless of the key phase. phase = QUICKeyPhase::PHASE_0; @@ -152,25 +139,16 @@ QUICPacketHeaderProtector::_calc_sample_offset(uint8_t *sample_offset, const uin int dcil) const { if (QUICInvariants::is_long_header(protected_packet)) { - uint8_t dcil; - uint8_t scil; size_t dummy; uint8_t length_len; - QUICPacketLongHeader::dcil(dcil, protected_packet, protected_packet_len); - QUICPacketLongHeader::scil(scil, protected_packet, protected_packet_len); - QUICPacketLongHeader::length(dummy, &length_len, protected_packet, protected_packet_len); - *sample_offset = 6 + dcil + scil + length_len + 4; - - QUICPacketType type; - QUICPacketLongHeader::type(type, protected_packet, protected_packet_len); - if (type == QUICPacketType::INITIAL) { - size_t token_len; - uint8_t token_length_len; - QUICPacketLongHeader::token_length(token_len, &token_length_len, protected_packet, protected_packet_len); - *sample_offset += token_len + token_length_len; + size_t length_offset; + if (!QUICLongHeaderPacketR::length(dummy, length_len, length_offset, protected_packet, protected_packet_len)) { + return false; } + + *sample_offset = length_offset + length_len + 4; } else { - *sample_offset = 1 + dcil + 4; + *sample_offset = QUICInvariants::SH_DCID_OFFSET + dcil + 4; } return static_cast(*sample_offset + 16) <= protected_packet_len; @@ -179,15 +157,15 @@ QUICPacketHeaderProtector::_calc_sample_offset(uint8_t *sample_offset, const uin bool QUICPacketHeaderProtector::_unprotect(uint8_t *protected_packet, size_t protected_packet_len, const uint8_t *mask) const { - uint8_t pn_offset; + size_t pn_offset; // Unprotect packet number if (QUICInvariants::is_long_header(protected_packet)) { protected_packet[0] ^= mask[0] & 0x0f; - QUICPacketLongHeader::packet_number_offset(pn_offset, protected_packet, protected_packet_len); + QUICLongHeaderPacketR::packet_number_offset(pn_offset, protected_packet, protected_packet_len); } else { protected_packet[0] ^= mask[0] & 0x1f; - QUICPacketShortHeader::packet_number_offset(pn_offset, protected_packet, protected_packet_len, QUICConnectionId::SCID_LEN); + QUICShortHeaderPacketR::packet_number_offset(pn_offset, protected_packet, protected_packet_len, QUICConnectionId::SCID_LEN); } uint8_t pn_length = QUICTypeUtil::read_QUICPacketNumberLen(protected_packet); @@ -201,17 +179,17 @@ QUICPacketHeaderProtector::_unprotect(uint8_t *protected_packet, size_t protecte bool QUICPacketHeaderProtector::_protect(uint8_t *protected_packet, size_t protected_packet_len, const uint8_t *mask, int dcil) const { - uint8_t pn_offset; + size_t pn_offset; uint8_t pn_length = QUICTypeUtil::read_QUICPacketNumberLen(protected_packet); // Protect packet number if (QUICInvariants::is_long_header(protected_packet)) { protected_packet[0] ^= mask[0] & 0x0f; - QUICPacketLongHeader::packet_number_offset(pn_offset, protected_packet, protected_packet_len); + QUICLongHeaderPacketR::packet_number_offset(pn_offset, protected_packet, protected_packet_len); } else { protected_packet[0] ^= mask[0] & 0x1f; - QUICPacketShortHeader::packet_number_offset(pn_offset, protected_packet, protected_packet_len, dcil); + QUICShortHeaderPacketR::packet_number_offset(pn_offset, protected_packet, protected_packet_len, dcil); } for (int i = 0; i < pn_length; ++i) { diff --git a/iocore/net/quic/QUICPacketHeaderProtector_boringssl.cc b/iocore/net/quic/QUICPacketHeaderProtector_boringssl.cc index 54c539e6c59..63e71bf0105 100644 --- a/iocore/net/quic/QUICPacketHeaderProtector_boringssl.cc +++ b/iocore/net/quic/QUICPacketHeaderProtector_boringssl.cc @@ -23,9 +23,36 @@ #include "QUICPacketHeaderProtector.h" +#include "openssl/chacha.h" + bool QUICPacketHeaderProtector::_generate_mask(uint8_t *mask, const uint8_t *sample, const uint8_t *key, const EVP_CIPHER *cipher) const { - ink_assert(!"not implemented"); - return false; + static constexpr unsigned char FIVE_ZEROS[] = {0x00, 0x00, 0x00, 0x00, 0x00}; + + if (cipher == nullptr) { + uint32_t counter = htole32(*reinterpret_cast(&sample[0])); + CRYPTO_chacha_20(mask, FIVE_ZEROS, sizeof(FIVE_ZEROS), key, &sample[4], counter); + } else { + int len = 0; + EVP_CIPHER_CTX *ctx = EVP_CIPHER_CTX_new(); + if (!ctx) { + return false; + } + if (!EVP_EncryptInit_ex(ctx, cipher, nullptr, key, sample)) { + EVP_CIPHER_CTX_free(ctx); + return false; + } + if (!EVP_EncryptUpdate(ctx, mask, &len, sample, 16)) { + EVP_CIPHER_CTX_free(ctx); + return false; + } + if (!EVP_EncryptFinal_ex(ctx, mask + len, &len)) { + EVP_CIPHER_CTX_free(ctx); + return false; + } + EVP_CIPHER_CTX_free(ctx); + } + + return true; } diff --git a/iocore/net/quic/QUICPacketHeaderProtector_legacy.cc b/iocore/net/quic/QUICPacketHeaderProtector_legacy.cc new file mode 100644 index 00000000000..43bbba8e901 --- /dev/null +++ b/iocore/net/quic/QUICPacketHeaderProtector_legacy.cc @@ -0,0 +1,53 @@ +/** @file + * + * QUIC Packet Header Protector (OpenSSL specific code) + * + * @section license License + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "QUICPacketHeaderProtector.h" + +bool +QUICPacketHeaderProtector::_generate_mask(uint8_t *mask, const uint8_t *sample, const uint8_t *key, const EVP_CIPHER *cipher) const +{ + static constexpr unsigned char FIVE_ZEROS[] = {0x00, 0x00, 0x00, 0x00, 0x00}; + EVP_CIPHER_CTX *ctx = EVP_CIPHER_CTX_new(); + + if (!ctx || !EVP_EncryptInit_ex(ctx, cipher, nullptr, key, sample)) { + return false; + } + + int len = 0; + if (cipher == EVP_chacha20()) { + if (!EVP_EncryptUpdate(ctx, mask, &len, FIVE_ZEROS, sizeof(FIVE_ZEROS))) { + return false; + } + } else { + if (!EVP_EncryptUpdate(ctx, mask, &len, sample, 16)) { + return false; + } + } + if (!EVP_EncryptFinal_ex(ctx, mask + len, &len)) { + return false; + } + + EVP_CIPHER_CTX_free(ctx); + + return true; +} diff --git a/iocore/net/quic/QUICPacketHeaderProtector_openssl.cc b/iocore/net/quic/QUICPacketHeaderProtector_openssl.cc index 43bbba8e901..c7d65a19e11 100644 --- a/iocore/net/quic/QUICPacketHeaderProtector_openssl.cc +++ b/iocore/net/quic/QUICPacketHeaderProtector_openssl.cc @@ -29,21 +29,28 @@ QUICPacketHeaderProtector::_generate_mask(uint8_t *mask, const uint8_t *sample, static constexpr unsigned char FIVE_ZEROS[] = {0x00, 0x00, 0x00, 0x00, 0x00}; EVP_CIPHER_CTX *ctx = EVP_CIPHER_CTX_new(); - if (!ctx || !EVP_EncryptInit_ex(ctx, cipher, nullptr, key, sample)) { + if (!ctx) { + return false; + } + if (!EVP_EncryptInit_ex(ctx, cipher, nullptr, key, sample)) { + EVP_CIPHER_CTX_free(ctx); return false; } int len = 0; if (cipher == EVP_chacha20()) { if (!EVP_EncryptUpdate(ctx, mask, &len, FIVE_ZEROS, sizeof(FIVE_ZEROS))) { + EVP_CIPHER_CTX_free(ctx); return false; } } else { if (!EVP_EncryptUpdate(ctx, mask, &len, sample, 16)) { + EVP_CIPHER_CTX_free(ctx); return false; } } if (!EVP_EncryptFinal_ex(ctx, mask + len, &len)) { + EVP_CIPHER_CTX_free(ctx); return false; } diff --git a/iocore/net/quic/QUICPacketPayloadProtector.cc b/iocore/net/quic/QUICPacketPayloadProtector.cc index ca690577c2d..4b8291c8a39 100644 --- a/iocore/net/quic/QUICPacketPayloadProtector.cc +++ b/iocore/net/quic/QUICPacketPayloadProtector.cc @@ -46,12 +46,16 @@ QUICPacketPayloadProtector::protect(const Ptr unprotected_header, const EVP_CIPHER *cipher = this->_pp_key_info.get_cipher(phase); - protected_payload = make_ptr(new_IOBufferBlock()); - protected_payload->alloc(iobuffer_size_to_index(unprotected_payload->size() + tag_len)); + protected_payload = make_ptr(new_IOBufferBlock()); + size_t unprotected_payload_len = 0; + for (Ptr tmp = unprotected_payload; tmp; tmp = tmp->next) { + unprotected_payload_len += tmp->size(); + } + protected_payload->alloc(iobuffer_size_to_index(unprotected_payload_len + tag_len, BUFFER_SIZE_INDEX_32K)); size_t written_len = 0; if (!this->_protect(reinterpret_cast(protected_payload->start()), written_len, protected_payload->write_avail(), - unprotected_payload, pkt_num, reinterpret_cast(unprotected_header->buf()), + unprotected_payload, pkt_num, reinterpret_cast(unprotected_header->start()), unprotected_header->size(), key, iv, iv_len, cipher, tag_len)) { Debug(tag, "Failed to encrypt a packet #%" PRIu64 " with keys for %s", pkt_num, QUICDebugNames::key_phase(phase)); protected_payload = nullptr; @@ -80,13 +84,13 @@ QUICPacketPayloadProtector::unprotect(const Ptr unprotected_heade const EVP_CIPHER *cipher = this->_pp_key_info.get_cipher(phase); unprotected_payload = make_ptr(new_IOBufferBlock()); - unprotected_payload->alloc(iobuffer_size_to_index(protected_payload->size())); + unprotected_payload->alloc(iobuffer_size_to_index(protected_payload->size(), BUFFER_SIZE_INDEX_32K)); size_t written_len = 0; if (!this->_unprotect(reinterpret_cast(unprotected_payload->start()), written_len, unprotected_payload->write_avail(), - reinterpret_cast(protected_payload->buf()), protected_payload->size(), pkt_num, - reinterpret_cast(unprotected_header->buf()), unprotected_header->size(), key, iv, iv_len, cipher, - tag_len)) { + reinterpret_cast(protected_payload->start()), protected_payload->size(), pkt_num, + reinterpret_cast(unprotected_header->start()), unprotected_header->size(), key, iv, iv_len, + cipher, tag_len)) { Debug(tag, "Failed to decrypt a packet #%" PRIu64, pkt_num); unprotected_payload = nullptr; } else { @@ -122,3 +126,111 @@ QUICPacketPayloadProtector::_gen_nonce(uint8_t *nonce, size_t &nonce_len, uint64 nonce[iv_len - 8 + i] ^= p[i]; } } + +bool +QUICPacketPayloadProtector::_protect(uint8_t *cipher, size_t &cipher_len, size_t max_cipher_len, const Ptr plain, + uint64_t pkt_num, const uint8_t *ad, size_t ad_len, const uint8_t *key, const uint8_t *iv, + size_t iv_len, const EVP_CIPHER *aead, size_t tag_len) const +{ + EVP_CIPHER_CTX *aead_ctx; + int len; + uint8_t nonce[EVP_MAX_IV_LENGTH] = {0}; + size_t nonce_len = 0; + + this->_gen_nonce(nonce, nonce_len, pkt_num, iv, iv_len); + + if (!(aead_ctx = EVP_CIPHER_CTX_new())) { + return false; + } + if (!EVP_EncryptInit_ex(aead_ctx, aead, nullptr, nullptr, nullptr)) { + return false; + } + if (!EVP_CIPHER_CTX_ctrl(aead_ctx, EVP_CTRL_AEAD_SET_IVLEN, nonce_len, nullptr)) { + return false; + } + if (!EVP_EncryptInit_ex(aead_ctx, nullptr, nullptr, key, nonce)) { + return false; + } + if (!EVP_EncryptUpdate(aead_ctx, nullptr, &len, ad, ad_len)) { + return false; + } + + cipher_len = 0; + for (Ptr b = plain; b; b = b->next) { + if (!EVP_EncryptUpdate(aead_ctx, cipher + cipher_len, &len, reinterpret_cast(b->start()), b->size())) { + return false; + } + cipher_len += len; + } + + if (!EVP_EncryptFinal_ex(aead_ctx, cipher + cipher_len, &len)) { + return false; + } + cipher_len += len; + + if (max_cipher_len < cipher_len + tag_len) { + return false; + } + if (!EVP_CIPHER_CTX_ctrl(aead_ctx, EVP_CTRL_AEAD_GET_TAG, tag_len, cipher + cipher_len)) { + return false; + } + cipher_len += tag_len; + + EVP_CIPHER_CTX_free(aead_ctx); + + return true; +} + +bool +QUICPacketPayloadProtector::_unprotect(uint8_t *plain, size_t &plain_len, size_t max_plain_len, const uint8_t *cipher, + size_t cipher_len, uint64_t pkt_num, const uint8_t *ad, size_t ad_len, const uint8_t *key, + const uint8_t *iv, size_t iv_len, const EVP_CIPHER *aead, size_t tag_len) const +{ + EVP_CIPHER_CTX *aead_ctx; + int len; + uint8_t nonce[EVP_MAX_IV_LENGTH] = {0}; + size_t nonce_len = 0; + + this->_gen_nonce(nonce, nonce_len, pkt_num, iv, iv_len); + + if (!(aead_ctx = EVP_CIPHER_CTX_new())) { + return false; + } + if (!EVP_DecryptInit_ex(aead_ctx, aead, nullptr, nullptr, nullptr)) { + return false; + } + if (!EVP_CIPHER_CTX_ctrl(aead_ctx, EVP_CTRL_AEAD_SET_IVLEN, nonce_len, nullptr)) { + return false; + } + if (!EVP_DecryptInit_ex(aead_ctx, nullptr, nullptr, key, nonce)) { + return false; + } + if (!EVP_DecryptUpdate(aead_ctx, nullptr, &len, ad, ad_len)) { + return false; + } + + if (cipher_len < tag_len) { + return false; + } + cipher_len -= tag_len; + if (!EVP_DecryptUpdate(aead_ctx, plain, &len, cipher, cipher_len)) { + return false; + } + plain_len = len; + + if (!EVP_CIPHER_CTX_ctrl(aead_ctx, EVP_CTRL_AEAD_SET_TAG, tag_len, const_cast(cipher + cipher_len))) { + return false; + } + + int ret = EVP_DecryptFinal_ex(aead_ctx, plain + len, &len); + + EVP_CIPHER_CTX_free(aead_ctx); + + if (ret > 0) { + plain_len += len; + return true; + } else { + Debug(tag, "Failed to decrypt -- the first 4 bytes decrypted are %0x %0x %0x %0x", plain[0], plain[1], plain[2], plain[3]); + return false; + } +} diff --git a/iocore/net/quic/QUICPacketPayloadProtector_boringssl.cc b/iocore/net/quic/QUICPacketPayloadProtector_boringssl.cc index 56e88dd45fd..0352e513d13 100644 --- a/iocore/net/quic/QUICPacketPayloadProtector_boringssl.cc +++ b/iocore/net/quic/QUICPacketPayloadProtector_boringssl.cc @@ -25,24 +25,129 @@ #include "QUICPacketPayloadProtector.h" #include "tscore/Diags.h" -// static constexpr char tag[] = "quic_ppp"; +static constexpr char tag[] = "quic_ppp"; bool -QUICPacketPayloadProtector::_protect(uint8_t *protected_payload, size_t &protected_payload_len, size_t max_protecgted_payload_len, - const Ptr plain, uint64_t pkt_num, const uint8_t *ad, size_t ad_len, - const uint8_t *key, const uint8_t *iv, size_t iv_len, const EVP_CIPHER *cipher, - size_t tag_len) const +QUICPacketPayloadProtector::_protect(uint8_t *cipher, size_t &cipher_len, size_t max_cipher_len, const Ptr plain, + uint64_t pkt_num, const uint8_t *ad, size_t ad_len, const uint8_t *key, const uint8_t *iv, + size_t iv_len, const EVP_CIPHER *aead, size_t tag_len) const { - ink_assert(!"not implemented"); - return false; + EVP_CIPHER_CTX *aead_ctx; + int len; + uint8_t nonce[EVP_MAX_IV_LENGTH] = {0}; + size_t nonce_len = 0; + + this->_gen_nonce(nonce, nonce_len, pkt_num, iv, iv_len); + + if (!(aead_ctx = EVP_CIPHER_CTX_new())) { + return false; + } + if (!EVP_EncryptInit_ex(aead_ctx, aead, nullptr, nullptr, nullptr)) { + EVP_CIPHER_CTX_free(aead_ctx); + return false; + } + if (!EVP_CIPHER_CTX_ctrl(aead_ctx, EVP_CTRL_AEAD_SET_IVLEN, nonce_len, nullptr)) { + EVP_CIPHER_CTX_free(aead_ctx); + return false; + } + if (!EVP_EncryptInit_ex(aead_ctx, nullptr, nullptr, key, nonce)) { + EVP_CIPHER_CTX_free(aead_ctx); + return false; + } + if (!EVP_EncryptUpdate(aead_ctx, nullptr, &len, ad, ad_len)) { + EVP_CIPHER_CTX_free(aead_ctx); + return false; + } + + cipher_len = 0; + Ptr b = plain; + while (b) { + if (!EVP_EncryptUpdate(aead_ctx, cipher + cipher_len, &len, reinterpret_cast(b->buf()), b->size())) { + EVP_CIPHER_CTX_free(aead_ctx); + return false; + } + cipher_len += len; + b = b->next; + } + + if (!EVP_EncryptFinal_ex(aead_ctx, cipher + cipher_len, &len)) { + EVP_CIPHER_CTX_free(aead_ctx); + return false; + } + cipher_len += len; + + if (max_cipher_len < cipher_len + tag_len) { + EVP_CIPHER_CTX_free(aead_ctx); + return false; + } + if (!EVP_CIPHER_CTX_ctrl(aead_ctx, EVP_CTRL_AEAD_GET_TAG, tag_len, cipher + cipher_len)) { + EVP_CIPHER_CTX_free(aead_ctx); + return false; + } + cipher_len += tag_len; + + EVP_CIPHER_CTX_free(aead_ctx); + + return true; } bool -QUICPacketPayloadProtector::_unprotect(uint8_t *plain, size_t &plain_len, size_t max_plain_len, const uint8_t *protected_payload, - size_t protected_payload_len, uint64_t pkt_num, const uint8_t *ad, size_t ad_len, - const uint8_t *key, const uint8_t *iv, size_t iv_len, const EVP_CIPHER *cipher, - size_t tag_len) const +QUICPacketPayloadProtector::_unprotect(uint8_t *plain, size_t &plain_len, size_t max_plain_len, const uint8_t *cipher, + size_t cipher_len, uint64_t pkt_num, const uint8_t *ad, size_t ad_len, const uint8_t *key, + const uint8_t *iv, size_t iv_len, const EVP_CIPHER *aead, size_t tag_len) const { - ink_assert(!"not implemented"); - return false; + EVP_CIPHER_CTX *aead_ctx; + int len; + uint8_t nonce[EVP_MAX_IV_LENGTH] = {0}; + size_t nonce_len = 0; + + this->_gen_nonce(nonce, nonce_len, pkt_num, iv, iv_len); + + if (!(aead_ctx = EVP_CIPHER_CTX_new())) { + return false; + } + if (!EVP_DecryptInit_ex(aead_ctx, aead, nullptr, nullptr, nullptr)) { + EVP_CIPHER_CTX_free(aead_ctx); + return false; + } + if (!EVP_CIPHER_CTX_ctrl(aead_ctx, EVP_CTRL_AEAD_SET_IVLEN, nonce_len, nullptr)) { + EVP_CIPHER_CTX_free(aead_ctx); + return false; + } + if (!EVP_DecryptInit_ex(aead_ctx, nullptr, nullptr, key, nonce)) { + EVP_CIPHER_CTX_free(aead_ctx); + return false; + } + if (!EVP_DecryptUpdate(aead_ctx, nullptr, &len, ad, ad_len)) { + EVP_CIPHER_CTX_free(aead_ctx); + return false; + } + + if (cipher_len < tag_len) { + EVP_CIPHER_CTX_free(aead_ctx); + return false; + } + cipher_len -= tag_len; + if (!EVP_DecryptUpdate(aead_ctx, plain, &len, cipher, cipher_len)) { + EVP_CIPHER_CTX_free(aead_ctx); + return false; + } + plain_len = len; + + if (!EVP_CIPHER_CTX_ctrl(aead_ctx, EVP_CTRL_AEAD_SET_TAG, tag_len, const_cast(cipher + cipher_len))) { + EVP_CIPHER_CTX_free(aead_ctx); + return false; + } + + int ret = EVP_DecryptFinal_ex(aead_ctx, plain + len, &len); + + EVP_CIPHER_CTX_free(aead_ctx); + + if (ret > 0) { + plain_len += len; + return true; + } else { + Debug(tag, "Failed to decrypt -- the first 4 bytes decrypted are %0x %0x %0x %0x", plain[0], plain[1], plain[2], plain[3]); + return false; + } } diff --git a/iocore/net/quic/QUICPacketPayloadProtector_legacy.cc b/iocore/net/quic/QUICPacketPayloadProtector_legacy.cc new file mode 100644 index 00000000000..5fc9e2e6e11 --- /dev/null +++ b/iocore/net/quic/QUICPacketPayloadProtector_legacy.cc @@ -0,0 +1,136 @@ +/** @file + * + * QUIC Packet Payload Protector (OpenSSL specific code) + * + * @section license License + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "QUICPacketProtectionKeyInfo.h" +#include "QUICPacketPayloadProtector.h" +#include "tscore/Diags.h" + +static constexpr char tag[] = "quic_ppp"; + +bool +QUICPacketPayloadProtector::_protect(uint8_t *cipher, size_t &cipher_len, size_t max_cipher_len, const Ptr plain, + uint64_t pkt_num, const uint8_t *ad, size_t ad_len, const uint8_t *key, const uint8_t *iv, + size_t iv_len, const EVP_CIPHER *aead, size_t tag_len) const +{ + EVP_CIPHER_CTX *aead_ctx; + int len; + uint8_t nonce[EVP_MAX_IV_LENGTH] = {0}; + size_t nonce_len = 0; + + this->_gen_nonce(nonce, nonce_len, pkt_num, iv, iv_len); + + if (!(aead_ctx = EVP_CIPHER_CTX_new())) { + return false; + } + if (!EVP_EncryptInit_ex(aead_ctx, aead, nullptr, nullptr, nullptr)) { + return false; + } + if (!EVP_CIPHER_CTX_ctrl(aead_ctx, EVP_CTRL_AEAD_SET_IVLEN, nonce_len, nullptr)) { + return false; + } + if (!EVP_EncryptInit_ex(aead_ctx, nullptr, nullptr, key, nonce)) { + return false; + } + if (!EVP_EncryptUpdate(aead_ctx, nullptr, &len, ad, ad_len)) { + return false; + } + + cipher_len = 0; + for (Ptr b = plain; b; b = b->next) { + if (!EVP_EncryptUpdate(aead_ctx, cipher + cipher_len, &len, reinterpret_cast(b->start()), b->size())) { + return false; + } + cipher_len += len; + } + + if (!EVP_EncryptFinal_ex(aead_ctx, cipher + cipher_len, &len)) { + return false; + } + cipher_len += len; + + if (max_cipher_len < cipher_len + tag_len) { + return false; + } + if (!EVP_CIPHER_CTX_ctrl(aead_ctx, EVP_CTRL_AEAD_GET_TAG, tag_len, cipher + cipher_len)) { + return false; + } + cipher_len += tag_len; + + EVP_CIPHER_CTX_free(aead_ctx); + + return true; +} + +bool +QUICPacketPayloadProtector::_unprotect(uint8_t *plain, size_t &plain_len, size_t max_plain_len, const uint8_t *cipher, + size_t cipher_len, uint64_t pkt_num, const uint8_t *ad, size_t ad_len, const uint8_t *key, + const uint8_t *iv, size_t iv_len, const EVP_CIPHER *aead, size_t tag_len) const +{ + EVP_CIPHER_CTX *aead_ctx; + int len; + uint8_t nonce[EVP_MAX_IV_LENGTH] = {0}; + size_t nonce_len = 0; + + this->_gen_nonce(nonce, nonce_len, pkt_num, iv, iv_len); + + if (!(aead_ctx = EVP_CIPHER_CTX_new())) { + return false; + } + if (!EVP_DecryptInit_ex(aead_ctx, aead, nullptr, nullptr, nullptr)) { + return false; + } + if (!EVP_CIPHER_CTX_ctrl(aead_ctx, EVP_CTRL_AEAD_SET_IVLEN, nonce_len, nullptr)) { + return false; + } + if (!EVP_DecryptInit_ex(aead_ctx, nullptr, nullptr, key, nonce)) { + return false; + } + if (!EVP_DecryptUpdate(aead_ctx, nullptr, &len, ad, ad_len)) { + return false; + } + + if (cipher_len < tag_len) { + return false; + } + cipher_len -= tag_len; + if (!EVP_DecryptUpdate(aead_ctx, plain, &len, cipher, cipher_len)) { + return false; + } + plain_len = len; + + if (!EVP_CIPHER_CTX_ctrl(aead_ctx, EVP_CTRL_AEAD_SET_TAG, tag_len, const_cast(cipher + cipher_len))) { + return false; + } + + int ret = EVP_DecryptFinal_ex(aead_ctx, plain + len, &len); + + EVP_CIPHER_CTX_free(aead_ctx); + + if (ret > 0) { + plain_len += len; + return true; + } else { + Debug(tag, "Failed to decrypt -- the first 4 bytes decrypted are %0x %0x %0x %0x", plain[0], plain[1], plain[2], plain[3]); + return false; + } +} diff --git a/iocore/net/quic/QUICPacketPayloadProtector_openssl.cc b/iocore/net/quic/QUICPacketPayloadProtector_openssl.cc index b25f099051e..c900718417b 100644 --- a/iocore/net/quic/QUICPacketPayloadProtector_openssl.cc +++ b/iocore/net/quic/QUICPacketPayloadProtector_openssl.cc @@ -43,15 +43,19 @@ QUICPacketPayloadProtector::_protect(uint8_t *cipher, size_t &cipher_len, size_t return false; } if (!EVP_EncryptInit_ex(aead_ctx, aead, nullptr, nullptr, nullptr)) { + EVP_CIPHER_CTX_free(aead_ctx); return false; } if (!EVP_CIPHER_CTX_ctrl(aead_ctx, EVP_CTRL_AEAD_SET_IVLEN, nonce_len, nullptr)) { + EVP_CIPHER_CTX_free(aead_ctx); return false; } if (!EVP_EncryptInit_ex(aead_ctx, nullptr, nullptr, key, nonce)) { + EVP_CIPHER_CTX_free(aead_ctx); return false; } if (!EVP_EncryptUpdate(aead_ctx, nullptr, &len, ad, ad_len)) { + EVP_CIPHER_CTX_free(aead_ctx); return false; } @@ -59,6 +63,7 @@ QUICPacketPayloadProtector::_protect(uint8_t *cipher, size_t &cipher_len, size_t Ptr b = plain; while (b) { if (!EVP_EncryptUpdate(aead_ctx, cipher + cipher_len, &len, reinterpret_cast(b->buf()), b->size())) { + EVP_CIPHER_CTX_free(aead_ctx); return false; } cipher_len += len; @@ -66,14 +71,17 @@ QUICPacketPayloadProtector::_protect(uint8_t *cipher, size_t &cipher_len, size_t } if (!EVP_EncryptFinal_ex(aead_ctx, cipher + cipher_len, &len)) { + EVP_CIPHER_CTX_free(aead_ctx); return false; } cipher_len += len; if (max_cipher_len < cipher_len + tag_len) { + EVP_CIPHER_CTX_free(aead_ctx); return false; } if (!EVP_CIPHER_CTX_ctrl(aead_ctx, EVP_CTRL_AEAD_GET_TAG, tag_len, cipher + cipher_len)) { + EVP_CIPHER_CTX_free(aead_ctx); return false; } cipher_len += tag_len; @@ -99,28 +107,35 @@ QUICPacketPayloadProtector::_unprotect(uint8_t *plain, size_t &plain_len, size_t return false; } if (!EVP_DecryptInit_ex(aead_ctx, aead, nullptr, nullptr, nullptr)) { + EVP_CIPHER_CTX_free(aead_ctx); return false; } if (!EVP_CIPHER_CTX_ctrl(aead_ctx, EVP_CTRL_AEAD_SET_IVLEN, nonce_len, nullptr)) { + EVP_CIPHER_CTX_free(aead_ctx); return false; } if (!EVP_DecryptInit_ex(aead_ctx, nullptr, nullptr, key, nonce)) { + EVP_CIPHER_CTX_free(aead_ctx); return false; } if (!EVP_DecryptUpdate(aead_ctx, nullptr, &len, ad, ad_len)) { + EVP_CIPHER_CTX_free(aead_ctx); return false; } if (cipher_len < tag_len) { + EVP_CIPHER_CTX_free(aead_ctx); return false; } cipher_len -= tag_len; if (!EVP_DecryptUpdate(aead_ctx, plain, &len, cipher, cipher_len)) { + EVP_CIPHER_CTX_free(aead_ctx); return false; } plain_len = len; if (!EVP_CIPHER_CTX_ctrl(aead_ctx, EVP_CTRL_AEAD_SET_TAG, tag_len, const_cast(cipher + cipher_len))) { + EVP_CIPHER_CTX_free(aead_ctx); return false; } diff --git a/iocore/net/quic/QUICPacketProtectionKeyInfo.h b/iocore/net/quic/QUICPacketProtectionKeyInfo.h index 32a38d41383..71d077d8b10 100644 --- a/iocore/net/quic/QUICPacketProtectionKeyInfo.h +++ b/iocore/net/quic/QUICPacketProtectionKeyInfo.h @@ -26,11 +26,43 @@ #include "QUICTypes.h" #include "QUICKeyGenerator.h" -class QUICPacketProtectionKeyInfo +class QUICPacketProtectionKeyInfoProvider +{ +public: + virtual ~QUICPacketProtectionKeyInfoProvider() {} + // Payload Protection (common) + virtual const EVP_CIPHER *get_cipher(QUICKeyPhase phase) const = 0; + virtual size_t get_tag_len(QUICKeyPhase phase) const = 0; + + // Payload Protection (encryption) + virtual bool is_encryption_key_available(QUICKeyPhase phase) const = 0; + virtual const uint8_t *encryption_key(QUICKeyPhase phase) const = 0; + virtual size_t encryption_key_len(QUICKeyPhase phase) const = 0; + virtual const uint8_t *encryption_iv(QUICKeyPhase phase) const = 0; + virtual const size_t *encryption_iv_len(QUICKeyPhase phase) const = 0; + + // Payload Protection (decryption) + virtual bool is_decryption_key_available(QUICKeyPhase phase) const = 0; + virtual const uint8_t *decryption_key(QUICKeyPhase phase) const = 0; + virtual size_t decryption_key_len(QUICKeyPhase phase) const = 0; + virtual const uint8_t *decryption_iv(QUICKeyPhase phase) const = 0; + virtual const size_t *decryption_iv_len(QUICKeyPhase phase) const = 0; + + // Header Protection + virtual const EVP_CIPHER *get_cipher_for_hp(QUICKeyPhase phase) const = 0; + virtual const uint8_t *encryption_key_for_hp(QUICKeyPhase phase) const = 0; + virtual size_t encryption_key_for_hp_len(QUICKeyPhase phase) const = 0; + virtual const uint8_t *decryption_key_for_hp(QUICKeyPhase phase) const = 0; + virtual size_t decryption_key_for_hp_len(QUICKeyPhase phase) const = 0; +}; + +class QUICPacketProtectionKeyInfo : public QUICPacketProtectionKeyInfoProvider { public: enum class Context { SERVER, CLIENT }; + virtual ~QUICPacketProtectionKeyInfo() {} + // FIXME This should be passed to the constructor but NetVC cannot pass it because it has set_context too. void set_context(Context ctx); @@ -38,58 +70,58 @@ class QUICPacketProtectionKeyInfo // Payload Protection (common) - virtual const EVP_CIPHER *get_cipher(QUICKeyPhase phase) const; - virtual size_t get_tag_len(QUICKeyPhase phase) const; + virtual const EVP_CIPHER *get_cipher(QUICKeyPhase phase) const override; + virtual size_t get_tag_len(QUICKeyPhase phase) const override; virtual void set_cipher_initial(const EVP_CIPHER *cipher); virtual void set_cipher(const EVP_CIPHER *cipher, size_t tag_len); // Payload Protection (encryption) - virtual bool is_encryption_key_available(QUICKeyPhase phase) const; + virtual bool is_encryption_key_available(QUICKeyPhase phase) const override; virtual void set_encryption_key_available(QUICKeyPhase phase); - virtual const uint8_t *encryption_key(QUICKeyPhase phase) const; + virtual const uint8_t *encryption_key(QUICKeyPhase phase) const override; virtual uint8_t *encryption_key(QUICKeyPhase phase); - virtual size_t encryption_key_len(QUICKeyPhase phase) const; + virtual size_t encryption_key_len(QUICKeyPhase phase) const override; - virtual const uint8_t *encryption_iv(QUICKeyPhase phase) const; + virtual const uint8_t *encryption_iv(QUICKeyPhase phase) const override; virtual uint8_t *encryption_iv(QUICKeyPhase phase); - virtual const size_t *encryption_iv_len(QUICKeyPhase phase) const; + virtual const size_t *encryption_iv_len(QUICKeyPhase phase) const override; virtual size_t *encryption_iv_len(QUICKeyPhase phase); // Payload Protection (decryption) - virtual bool is_decryption_key_available(QUICKeyPhase phase) const; + virtual bool is_decryption_key_available(QUICKeyPhase phase) const override; virtual void set_decryption_key_available(QUICKeyPhase phase); - virtual const uint8_t *decryption_key(QUICKeyPhase phase) const; + virtual const uint8_t *decryption_key(QUICKeyPhase phase) const override; virtual uint8_t *decryption_key(QUICKeyPhase phase); - virtual size_t decryption_key_len(QUICKeyPhase phase) const; + virtual size_t decryption_key_len(QUICKeyPhase phase) const override; - virtual const uint8_t *decryption_iv(QUICKeyPhase phase) const; + virtual const uint8_t *decryption_iv(QUICKeyPhase phase) const override; virtual uint8_t *decryption_iv(QUICKeyPhase phase); - virtual const size_t *decryption_iv_len(QUICKeyPhase phase) const; + virtual const size_t *decryption_iv_len(QUICKeyPhase phase) const override; virtual size_t *decryption_iv_len(QUICKeyPhase phase); // Header Protection - virtual const EVP_CIPHER *get_cipher_for_hp(QUICKeyPhase phase) const; + virtual const EVP_CIPHER *get_cipher_for_hp(QUICKeyPhase phase) const override; virtual void set_cipher_for_hp_initial(const EVP_CIPHER *cipher); virtual void set_cipher_for_hp(const EVP_CIPHER *cipher); - virtual const uint8_t *encryption_key_for_hp(QUICKeyPhase phase) const; + virtual const uint8_t *encryption_key_for_hp(QUICKeyPhase phase) const override; virtual uint8_t *encryption_key_for_hp(QUICKeyPhase phase); - virtual size_t encryption_key_for_hp_len(QUICKeyPhase phase) const; + virtual size_t encryption_key_for_hp_len(QUICKeyPhase phase) const override; - virtual const uint8_t *decryption_key_for_hp(QUICKeyPhase phase) const; + virtual const uint8_t *decryption_key_for_hp(QUICKeyPhase phase) const override; virtual uint8_t *decryption_key_for_hp(QUICKeyPhase phase); - virtual size_t decryption_key_for_hp_len(QUICKeyPhase phase) const; + virtual size_t decryption_key_for_hp_len(QUICKeyPhase phase) const override; private: Context _ctx = Context::SERVER; diff --git a/iocore/net/quic/QUICPacketReceiveQueue.cc b/iocore/net/quic/QUICPacketReceiveQueue.cc index 405c29533de..a2aee1bd726 100644 --- a/iocore/net/quic/QUICPacketReceiveQueue.cc +++ b/iocore/net/quic/QUICPacketReceiveQueue.cc @@ -22,46 +22,17 @@ */ #include "QUICPacketReceiveQueue.h" +#include "QUICPacketHeaderProtector.h" #include "QUICPacketFactory.h" #include "QUICIntUtil.h" -// FIXME: workaround for coalescing packets -static constexpr int LONG_HDR_OFFSET_CONNECTION_ID = 6; - static bool is_vn(QUICVersion v) { return v == 0x0; } -static bool -long_hdr_pkt_len(size_t &pkt_len, uint8_t *buf, size_t len) -{ - uint8_t dcil, scil; - QUICPacketLongHeader::dcil(dcil, buf, len); - QUICPacketLongHeader::scil(scil, buf, len); - - size_t offset = LONG_HDR_OFFSET_CONNECTION_ID + dcil + scil; - - // token_length and token_length_field_len should be 0 except INITIAL packet - size_t token_length = 0; - uint8_t token_length_field_len = 0; - if (!QUICPacketLongHeader::token_length(token_length, &token_length_field_len, buf, len)) { - return false; - } - - size_t length = 0; - uint8_t length_field_len = 0; - if (!QUICPacketLongHeader::length(length, &length_field_len, buf, len)) { - return false; - } - - pkt_len = offset + token_length + token_length_field_len + length_field_len + length; - - return true; -} - QUICPacketReceiveQueue::QUICPacketReceiveQueue(QUICPacketFactory &packet_factory, QUICPacketHeaderProtector &ph_protector) : _packet_factory(packet_factory), _ph_protector(ph_protector) { @@ -74,7 +45,7 @@ QUICPacketReceiveQueue::enqueue(UDPPacket *packet) } QUICPacketUPtr -QUICPacketReceiveQueue::dequeue(QUICPacketCreationResult &result) +QUICPacketReceiveQueue::dequeue(uint8_t *packet_buf, QUICPacketCreationResult &result) { QUICPacketUPtr quic_packet = QUICPacketFactory::create_null_packet(); UDPPacket *udp_packet = nullptr; @@ -91,12 +62,13 @@ QUICPacketReceiveQueue::dequeue(QUICPacketCreationResult &result) // Create a QUIC packet this->_udp_con = udp_packet->getConnection(); this->_from = udp_packet->from; + this->_to = udp_packet->to; this->_payload_len = udp_packet->getPktLength(); this->_payload = ats_unique_malloc(this->_payload_len); IOBufferBlock *b = udp_packet->getIOBlockChain(); size_t written = 0; while (b) { - memcpy(this->_payload.get() + written, b->buf(), b->read_avail()); + memcpy(this->_payload.get() + written, b->start(), b->read_avail()); written += b->read_avail(); b = b->next.get(); } @@ -112,7 +84,7 @@ QUICPacketReceiveQueue::dequeue(QUICPacketCreationResult &result) if (QUICInvariants::is_long_header(buf)) { QUICVersion version; - QUICPacketLongHeader::version(version, buf, remaining_len); + QUICLongHeaderPacketR::version(version, buf, remaining_len); if (is_vn(version)) { pkt_len = remaining_len; type = QUICPacketType::VERSION_NEGOTIATION; @@ -120,17 +92,17 @@ QUICPacketReceiveQueue::dequeue(QUICPacketCreationResult &result) result = QUICPacketCreationResult::UNSUPPORTED; pkt_len = remaining_len; } else { - QUICPacketLongHeader::type(type, this->_payload.get() + this->_offset, remaining_len); + QUICLongHeaderPacketR::type(type, this->_payload.get() + this->_offset, remaining_len); if (type == QUICPacketType::RETRY) { pkt_len = remaining_len; } else { - if (!long_hdr_pkt_len(pkt_len, this->_payload.get() + this->_offset, remaining_len)) { + if (!QUICLongHeaderPacketR::packet_length(pkt_len, this->_payload.get() + this->_offset, remaining_len)) { + // This should not happen basically. Ignore rest of data on current packet. this->_payload.release(); this->_payload = nullptr; this->_payload_len = 0; this->_offset = 0; - - result = QUICPacketCreationResult::IGNORED; + result = QUICPacketCreationResult::IGNORED; return quic_packet; } @@ -177,7 +149,7 @@ QUICPacketReceiveQueue::dequeue(QUICPacketCreationResult &result) } if (this->_ph_protector.unprotect(pkt.get(), pkt_len)) { - quic_packet = this->_packet_factory.create(this->_udp_con, this->_from, std::move(pkt), pkt_len, + quic_packet = this->_packet_factory.create(packet_buf, this->_udp_con, this->_from, this->_to, std::move(pkt), pkt_len, this->_largest_received_packet_number, result); } else { // ZERO_RTT might be rejected @@ -204,7 +176,8 @@ QUICPacketReceiveQueue::dequeue(QUICPacketCreationResult &result) // do nothing - if the packet is unsupported version, we don't know packet number break; default: - if (quic_packet && quic_packet->packet_number() > this->_largest_received_packet_number) { + if (quic_packet && quic_packet->type() != QUICPacketType::VERSION_NEGOTIATION && + quic_packet->packet_number() > this->_largest_received_packet_number) { this->_largest_received_packet_number = quic_packet->packet_number(); } } @@ -213,7 +186,7 @@ QUICPacketReceiveQueue::dequeue(QUICPacketCreationResult &result) } uint32_t -QUICPacketReceiveQueue::size() +QUICPacketReceiveQueue::size() const { return this->_queue.size; } diff --git a/iocore/net/quic/QUICPacketReceiveQueue.h b/iocore/net/quic/QUICPacketReceiveQueue.h index 5027df37b9a..3ebded31de2 100644 --- a/iocore/net/quic/QUICPacketReceiveQueue.h +++ b/iocore/net/quic/QUICPacketReceiveQueue.h @@ -29,6 +29,7 @@ #include "QUICPacket.h" class QUICPacketFactory; +class QUICPacketHeaderProtector; class QUICPacketReceiveQueue { @@ -36,8 +37,8 @@ class QUICPacketReceiveQueue QUICPacketReceiveQueue(QUICPacketFactory &packet_factory, QUICPacketHeaderProtector &ph_protector); void enqueue(UDPPacket *packet); - QUICPacketUPtr dequeue(QUICPacketCreationResult &result); - uint32_t size(); + QUICPacketUPtr dequeue(uint8_t *packet_buf, QUICPacketCreationResult &result); + uint32_t size() const; void reset(); private: @@ -51,4 +52,5 @@ class QUICPacketReceiveQueue size_t _offset = 0; UDPConnection *_udp_con; IpEndpoint _from; + IpEndpoint _to; }; diff --git a/iocore/net/quic/QUICPadder.cc b/iocore/net/quic/QUICPadder.cc new file mode 100644 index 00000000000..0ae913ebb9a --- /dev/null +++ b/iocore/net/quic/QUICPadder.cc @@ -0,0 +1,109 @@ +/** @file + + A brief file description + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#include "QUICIntUtil.h" +#include "QUICPadder.h" + +static constexpr uint32_t MINIMUM_INITIAL_PACKET_SIZE = 1200; +static constexpr uint32_t MIN_PKT_PAYLOAD_LEN = 3; ///< Minimum payload length for sampling for header protection + +void +QUICPadder::request(QUICEncryptionLevel level) +{ + SCOPED_MUTEX_LOCK(lock, this->_mutex, this_ethread()); + ++this->_need_to_fire[static_cast(level)]; +} + +void +QUICPadder::cancel(QUICEncryptionLevel level) +{ + SCOPED_MUTEX_LOCK(lock, this->_mutex, this_ethread()); + this->_need_to_fire[static_cast(level)] = 0; +} + +uint64_t +QUICPadder::count(QUICEncryptionLevel level) +{ + SCOPED_MUTEX_LOCK(lock, this->_mutex, this_ethread()); + return this->_need_to_fire[static_cast(level)]; +} + +bool +QUICPadder::_will_generate_frame(QUICEncryptionLevel level, size_t current_packet_size, bool ack_eliciting) +{ + SCOPED_MUTEX_LOCK(lock, this->_mutex, this_ethread()); + // no extre padding packet + if (current_packet_size == 0 && this->_need_to_fire[static_cast(level)] == 0) { + return false; + } + + // every packets need to be padded + return true; +} + +QUICFrame * +QUICPadder::_generate_frame(uint8_t *buf, QUICEncryptionLevel level, uint64_t connection_credit, uint16_t maximum_frame_size, + size_t current_packet_size) +{ + SCOPED_MUTEX_LOCK(lock, this->_mutex, this_ethread()); + QUICFrame *frame = nullptr; + + uint64_t min_size = 0; + if (level == QUICEncryptionLevel::INITIAL && this->_context == NET_VCONNECTION_OUT) { + min_size = this->_minimum_quic_packet_size(); + if (this->_av_token_len && min_size > (QUICVariableInt::size(this->_av_token_len) + this->_av_token_len)) { + min_size -= (QUICVariableInt::size(this->_av_token_len) + this->_av_token_len); + } + } else { + min_size = MIN_PKT_PAYLOAD_LEN; + } + + if (min_size > current_packet_size) { // ignore if we don't need to pad. + frame = QUICFrameFactory::create_padding_frame( + buf, std::min(min_size - current_packet_size, static_cast(maximum_frame_size))); + } + + this->_need_to_fire[static_cast(level)] = 0; + return frame; +} + +uint32_t +QUICPadder::_minimum_quic_packet_size() +{ + SCOPED_MUTEX_LOCK(lock, this->_mutex, this_ethread()); + if (this->_context == NET_VCONNECTION_OUT) { + // FIXME Only the first packet need to be 1200 bytes at least + return MINIMUM_INITIAL_PACKET_SIZE; + } else { + // FIXME This size should be configurable and should have some randomness + // This is just for providing protection against packet analysis for protected packets + return 32 + (this->_rnd() & 0x3f); // 32 to 96 + } +} + +void +QUICPadder::set_av_token_len(uint32_t len) +{ + SCOPED_MUTEX_LOCK(lock, this->_mutex, this_ethread()); + this->_av_token_len = len; +} diff --git a/iocore/net/quic/QUICPadder.h b/iocore/net/quic/QUICPadder.h new file mode 100644 index 00000000000..bba8dfc631b --- /dev/null +++ b/iocore/net/quic/QUICPadder.h @@ -0,0 +1,57 @@ +/** @file + * + * A brief file description + * + * @section license License + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "QUICTypes.h" +#include "QUICFrameHandler.h" +#include "QUICFrameGenerator.h" + +#include "I_Lock.h" + +class QUICPadder : public QUICFrameOnceGenerator +{ +public: + QUICPadder(NetVConnectionContext_t context) : _mutex(new_ProxyMutex()), _context(context) {} + + void request(QUICEncryptionLevel level); + void cancel(QUICEncryptionLevel level); + uint64_t count(QUICEncryptionLevel level); + void set_av_token_len(uint32_t len); + +private: + uint32_t _minimum_quic_packet_size(); + + // QUICFrameGenerator + bool _will_generate_frame(QUICEncryptionLevel level, size_t current_packet_size, bool ack_eliciting) override; + QUICFrame *_generate_frame(uint8_t *buf, QUICEncryptionLevel level, uint64_t connection_credit, uint16_t maximum_frame_size, + size_t current_packet_size) override; + + Ptr _mutex; + std::random_device _rnd; + + uint32_t _av_token_len = 0; + // Initial, 0/1-RTT, and Handshake + uint64_t _need_to_fire[4] = {0}; + NetVConnectionContext_t _context = NET_VCONNECTION_OUT; +}; diff --git a/iocore/net/quic/QUICPathManager.cc b/iocore/net/quic/QUICPathManager.cc new file mode 100644 index 00000000000..c0adbdecc5b --- /dev/null +++ b/iocore/net/quic/QUICPathManager.cc @@ -0,0 +1,84 @@ +/** @file + * + * A brief file description + * + * @section license License + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "QUICPathManager.h" +#include "QUICPathValidator.h" +#include "QUICConnection.h" + +#define QUICDebug(fmt, ...) Debug("quic_path", "[%s] " fmt, this->_cinfo.cids().data(), ##__VA_ARGS__) + +void +QUICPathManagerImpl::open_new_path(const QUICPath &path, ink_hrtime timeout_in) +{ + if (this->_verify_timeout_at == 0) { + // Overwrite _previous_path only if _current_path is verified + // _previous_path should always have a verified path if available + this->_previous_path = this->_current_path; + } + this->_current_path = path; + this->_path_validator.validate(path); + this->_verify_timeout_at = Thread::get_hrtime() + timeout_in; +} + +void +QUICPathManagerImpl::set_trusted_path(const QUICPath &path) +{ + this->_current_path = path; + this->_previous_path = path; +} + +void +QUICPathManagerImpl::_check_verify_timeout() +{ + if (this->_verify_timeout_at != 0) { + if (this->_path_validator.is_validated(this->_current_path)) { + // Address validation succeeded + this->_verify_timeout_at = 0; + this->_previous_path = {{}, {}}; + } else if (this->_verify_timeout_at < Thread::get_hrtime()) { + // Address validation failed + QUICDebug("Switching back to the previous path"); + this->_current_path = this->_previous_path; + this->_verify_timeout_at = 0; + this->_previous_path = {{}, {}}; + } + } +} + +const QUICPath & +QUICPathManagerImpl::get_current_path() +{ + this->_check_verify_timeout(); + return this->_current_path; +} + +const QUICPath & +QUICPathManagerImpl::get_verified_path() +{ + this->_check_verify_timeout(); + if (this->_verify_timeout_at != 0) { + return this->_previous_path; + } else { + return this->_current_path; + } +} diff --git a/iocore/net/quic/QUICPathManager.h b/iocore/net/quic/QUICPathManager.h new file mode 100644 index 00000000000..1d4542a629b --- /dev/null +++ b/iocore/net/quic/QUICPathManager.h @@ -0,0 +1,63 @@ +/** @file + * + * A brief file description + * + * @section license License + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "QUICTypes.h" + +class QUICConnectionInfoProvider; +class QUICPathValidator; + +class QUICPathManager +{ +public: + virtual ~QUICPathManager() {} + virtual const QUICPath &get_current_path() = 0; + virtual const QUICPath &get_verified_path() = 0; + virtual void open_new_path(const QUICPath &path, ink_hrtime timeout_in) = 0; + virtual void set_trusted_path(const QUICPath &path) = 0; +}; + +class QUICPathManagerImpl : public QUICPathManager +{ +public: + QUICPathManagerImpl(const QUICConnectionInfoProvider &info, QUICPathValidator &path_validator) + : _cinfo(info), _path_validator(path_validator) + { + } + + const QUICPath &get_current_path() override; + const QUICPath &get_verified_path() override; + void open_new_path(const QUICPath &path, ink_hrtime timeout_in) override; + void set_trusted_path(const QUICPath &path) override; + +private: + const QUICConnectionInfoProvider &_cinfo; + QUICPathValidator &_path_validator; + + ink_hrtime _verify_timeout_at = 0; + QUICPath _current_path = {{}, {}}; + QUICPath _previous_path = {{}, {}}; + + void _check_verify_timeout(); +}; diff --git a/iocore/net/quic/QUICPathValidator.cc b/iocore/net/quic/QUICPathValidator.cc index 40a23904886..e12022c75b4 100644 --- a/iocore/net/quic/QUICPathValidator.cc +++ b/iocore/net/quic/QUICPathValidator.cc @@ -23,32 +23,85 @@ #include #include "QUICPathValidator.h" +#include "QUICPacket.h" + +#define QUICDebug(fmt, ...) Debug("quic_path", "[%s] " fmt, this->_cinfo.cids().data(), ##__VA_ARGS__) bool -QUICPathValidator::is_validating() +QUICPathValidator::ValidationJob::has_more_challenges() const { - return this->_state == ValidationState::VALIDATING; + return this->_has_outgoing_challenge; +} + +const uint8_t * +QUICPathValidator::ValidationJob::get_next_challenge() const +{ + if (this->_has_outgoing_challenge) { + return this->_outgoing_challenge + ((this->_has_outgoing_challenge - 1) * 8); + } else { + return nullptr; + } +} + +void +QUICPathValidator::ValidationJob::consume_challenge() +{ + --(this->_has_outgoing_challenge); } bool -QUICPathValidator::is_validated() +QUICPathValidator::is_validating(const QUICPath &path) const { - return this->_state == ValidationState::VALIDATED; + if (auto j = this->_jobs.find(path); j != this->_jobs.end()) { + return j->second.is_validating(); + } else { + return false; + } +} + +bool +QUICPathValidator::is_validated(const QUICPath &path) const +{ + if (auto j = this->_jobs.find(path); j != this->_jobs.end()) { + return j->second.is_validated(); + } else { + return false; + } } void -QUICPathValidator::validate() +QUICPathValidator::validate(const QUICPath &path) { - if (this->_state == ValidationState::VALIDATING) { + if (this->_jobs.find(path) != this->_jobs.end()) { // Do nothing } else { - this->_state = ValidationState::VALIDATING; - this->_generate_challenge(); + auto result = this->_jobs.emplace(std::piecewise_construct, std::forward_as_tuple(path), std::forward_as_tuple()); + result.first->second.start(); + // QUICDebug("Validating a path between %s and %s", path.local_ep(), path.remote_ep()); } } +bool +QUICPathValidator::ValidationJob::is_validating() const +{ + return this->_state == ValidationState::VALIDATING; +} + +bool +QUICPathValidator::ValidationJob::is_validated() const +{ + return this->_state == ValidationState::VALIDATED; +} + +void +QUICPathValidator::ValidationJob::start() +{ + this->_state = ValidationState::VALIDATING; + this->_generate_challenge(); +} + void -QUICPathValidator::_generate_challenge() +QUICPathValidator::ValidationJob::_generate_challenge() { size_t seed = std::chrono::duration_cast(std::chrono::high_resolution_clock::now().time_since_epoch()).count(); @@ -60,29 +113,17 @@ QUICPathValidator::_generate_challenge() this->_has_outgoing_challenge = 3; } -void -QUICPathValidator::_generate_response(const QUICPathChallengeFrame &frame) -{ - memcpy(this->_incoming_challenge, frame.data(), QUICPathChallengeFrame::DATA_LEN); - this->_has_outgoing_response = true; -} - -QUICConnectionErrorUPtr -QUICPathValidator::_validate_response(const QUICPathResponseFrame &frame) +bool +QUICPathValidator::ValidationJob::validate_response(const uint8_t *data) { - QUICConnectionErrorUPtr error = std::make_unique(QUICTransErrorCode::PROTOCOL_VIOLATION); - for (int i = 0; i < 3; ++i) { - if (memcmp(this->_outgoing_challenge + (QUICPathChallengeFrame::DATA_LEN * i), frame.data(), - QUICPathChallengeFrame::DATA_LEN) == 0) { + if (memcmp(this->_outgoing_challenge + (QUICPathChallengeFrame::DATA_LEN * i), data, QUICPathChallengeFrame::DATA_LEN) == 0) { this->_state = ValidationState::VALIDATED; this->_has_outgoing_challenge = 0; - error = nullptr; - break; + return true; } } - - return error; + return false; } // @@ -101,10 +142,21 @@ QUICPathValidator::handle_frame(QUICEncryptionLevel level, const QUICFrame &fram switch (frame.type()) { case QUICFrameType::PATH_CHALLENGE: - this->_generate_response(static_cast(frame)); + this->_incoming_challenges.emplace(this->_incoming_challenges.begin(), + static_cast(frame).data()); break; case QUICFrameType::PATH_RESPONSE: - error = this->_validate_response(static_cast(frame)); + if (auto item = this->_jobs.find({frame.packet()->to(), frame.packet()->from()}); item != this->_jobs.end()) { + if (item->second.validate_response(static_cast(frame).data())) { + QUICDebug("validation succeeded"); + this->_on_validation_callback(true); + } else { + QUICDebug("validation failed"); + this->_on_validation_callback(false); + } + } else { + error = std::make_unique(QUICTransErrorCode::PROTOCOL_VIOLATION); + } break; default: ink_assert(!"Can't happen"); @@ -117,17 +169,30 @@ QUICPathValidator::handle_frame(QUICEncryptionLevel level, const QUICFrame &fram // QUICFrameGenerator // bool -QUICPathValidator::will_generate_frame(QUICEncryptionLevel level, ink_hrtime timestamp) +QUICPathValidator::will_generate_frame(QUICEncryptionLevel level, size_t current_packet_size, bool ack_eliciting, uint32_t seq_num) { if (!this->_is_level_matched(level)) { return false; } - if (this->_last_sent_at == timestamp) { + if (this->_latest_seq_num == seq_num) { return false; } - return (this->_has_outgoing_challenge || this->_has_outgoing_response); + // Check challenges + for (auto &&item : this->_jobs) { + auto &j = item.second; + if (!j.is_validating() && !j.is_validated()) { + j.start(); + return true; + } + if (j.has_more_challenges()) { + return true; + } + } + + // Check responses + return !this->_incoming_challenges.empty(); } /** @@ -135,7 +200,7 @@ QUICPathValidator::will_generate_frame(QUICEncryptionLevel level, ink_hrtime tim */ QUICFrame * QUICPathValidator::generate_frame(uint8_t *buf, QUICEncryptionLevel level, uint64_t /* connection_credit */, - uint16_t maximum_frame_size, ink_hrtime timestamp) + uint16_t maximum_frame_size, size_t current_packet_size, uint32_t seq_num) { QUICFrame *frame = nullptr; @@ -143,27 +208,31 @@ QUICPathValidator::generate_frame(uint8_t *buf, QUICEncryptionLevel level, uint6 return frame; } - if (this->_has_outgoing_response) { - frame = QUICFrameFactory::create_path_response_frame(buf, this->_incoming_challenge); + if (!this->_incoming_challenges.empty()) { + frame = QUICFrameFactory::create_path_response_frame(buf, this->_incoming_challenges.back()); if (frame && frame->size() > maximum_frame_size) { // Cancel generating frame frame = nullptr; } else { - this->_has_outgoing_response = false; + this->_incoming_challenges.pop_back(); } - } else if (this->_has_outgoing_challenge) { - frame = QUICFrameFactory::create_path_challenge_frame( - buf, this->_outgoing_challenge + (QUICPathChallengeFrame::DATA_LEN * (this->_has_outgoing_challenge - 1))); - if (frame && frame->size() > maximum_frame_size) { - // Cancel generating frame - frame = nullptr; - } else { - --this->_has_outgoing_challenge; - ink_assert(this->_has_outgoing_challenge >= 0); + } else { + for (auto &&item : this->_jobs) { + auto &j = item.second; + if (j.has_more_challenges()) { + frame = QUICFrameFactory::create_path_challenge_frame(buf, j.get_next_challenge()); + if (frame && frame->size() > maximum_frame_size) { + // Cancel generating frame + frame = nullptr; + } else { + j.consume_challenge(); + } + break; + } } } - this->_last_sent_at = timestamp; + this->_latest_seq_num = seq_num; return frame; } diff --git a/iocore/net/quic/QUICPathValidator.h b/iocore/net/quic/QUICPathValidator.h index 47a2975d108..96f5f4f64c3 100644 --- a/iocore/net/quic/QUICPathValidator.h +++ b/iocore/net/quic/QUICPathValidator.h @@ -24,26 +24,31 @@ #pragma once #include +#include #include "QUICTypes.h" #include "QUICFrameHandler.h" #include "QUICFrameGenerator.h" +#include "QUICConnection.h" class QUICPathValidator : public QUICFrameHandler, public QUICFrameGenerator { public: - QUICPathValidator() {} - bool is_validating(); - bool is_validated(); - void validate(); + QUICPathValidator(const QUICConnectionInfoProvider &info, std::function callback) + : _cinfo(info), _on_validation_callback(callback) + { + } + bool is_validating(const QUICPath &path) const; + bool is_validated(const QUICPath &path) const; + void validate(const QUICPath &path); // QUICFrameHandler std::vector interests() override; QUICConnectionErrorUPtr handle_frame(QUICEncryptionLevel level, const QUICFrame &frame) override; - // QUICFrameGeneratro - bool will_generate_frame(QUICEncryptionLevel level, ink_hrtime timestamp) override; + // QUICFrameGenerator + bool will_generate_frame(QUICEncryptionLevel level, size_t current_packet_size, bool ack_eliciting, uint32_t seq_num) override; QUICFrame *generate_frame(uint8_t *buf, QUICEncryptionLevel level, uint64_t connection_credit, uint16_t maximum_frame_size, - ink_hrtime timestamp) override; + size_t current_packet_size, uint32_t seq_num) override; private: enum class ValidationState : int { @@ -51,14 +56,33 @@ class QUICPathValidator : public QUICFrameHandler, public QUICFrameGenerator VALIDATING, VALIDATED, }; - ValidationState _state = ValidationState::NOT_VALIDATED; - int _has_outgoing_challenge = 0; - bool _has_outgoing_response = false; - ink_hrtime _last_sent_at = 0; - uint8_t _incoming_challenge[QUICPathChallengeFrame::DATA_LEN]; - uint8_t _outgoing_challenge[QUICPathChallengeFrame::DATA_LEN * 3]; - void _generate_challenge(); - void _generate_response(const QUICPathChallengeFrame &frame); - QUICConnectionErrorUPtr _validate_response(const QUICPathResponseFrame &frame); + class ValidationJob + { + public: + ValidationJob(){}; + ~ValidationJob(){}; + bool is_validating() const; + bool is_validated() const; + void start(); + bool validate_response(const uint8_t *data); + bool has_more_challenges() const; + const uint8_t *get_next_challenge() const; + void consume_challenge(); + + private: + ValidationState _state = ValidationState::NOT_VALIDATED; + uint8_t _outgoing_challenge[QUICPathChallengeFrame::DATA_LEN * 3]; + int _has_outgoing_challenge = 0; + + void _generate_challenge(); + }; + + const QUICConnectionInfoProvider &_cinfo; + std::unordered_map _jobs; + + std::function _on_validation_callback; + + uint32_t _latest_seq_num = 0; + std::vector _incoming_challenges; }; diff --git a/iocore/net/quic/QUICPinger.cc b/iocore/net/quic/QUICPinger.cc index e6f69c69d5d..2514c028210 100644 --- a/iocore/net/quic/QUICPinger.cc +++ b/iocore/net/quic/QUICPinger.cc @@ -26,52 +26,72 @@ void QUICPinger::request(QUICEncryptionLevel level) { - if (!this->_is_level_matched(level)) { - return; - } + SCOPED_MUTEX_LOCK(lock, this->_mutex, this_ethread()); + ink_assert(level != QUICEncryptionLevel::NONE); ++this->_need_to_fire[static_cast(level)]; } void QUICPinger::cancel(QUICEncryptionLevel level) { - if (!this->_is_level_matched(level)) { - return; - } - + SCOPED_MUTEX_LOCK(lock, this->_mutex, this_ethread()); + ink_assert(level != QUICEncryptionLevel::NONE); if (this->_need_to_fire[static_cast(level)] > 0) { --this->_need_to_fire[static_cast(level)]; } } bool -QUICPinger::will_generate_frame(QUICEncryptionLevel level, ink_hrtime timestamp) +QUICPinger::_will_generate_frame(QUICEncryptionLevel level, size_t current_packet_size, bool ack_eliciting) { - if (!this->_is_level_matched(level)) { + SCOPED_MUTEX_LOCK(lock, this->_mutex, this_ethread()); + + // PING Frame is meaningless for ack_eliciting packet. Cancel it. + if (ack_eliciting) { + this->_ack_eliciting_packet_out = true; + this->cancel(level); return false; } - return this->_need_to_fire[static_cast(QUICTypeUtil::pn_space(level))] > 0; + ink_assert(level != QUICEncryptionLevel::NONE); + if (this->_ack_eliciting_packet_out == false && !ack_eliciting && current_packet_size > 0 && + this->_need_to_fire[static_cast(level)] == 0) { + // force to send an PING Frame + this->request(level); + } + + // only update `_ack_eliciting_packet_out` when we has something to send. + if (current_packet_size) { + this->_ack_eliciting_packet_out = ack_eliciting; + } + return this->_need_to_fire[static_cast(level)] > 0; } /** * @param connection_credit This is not used. Because PING frame is not flow-controlled */ QUICFrame * -QUICPinger::generate_frame(uint8_t *buf, QUICEncryptionLevel level, uint64_t /* connection_credit */, uint16_t maximum_frame_size, - ink_hrtime timestamp) +QUICPinger::_generate_frame(uint8_t *buf, QUICEncryptionLevel level, uint64_t /* connection_credit */, uint16_t maximum_frame_size, + size_t current_packet_size) { + SCOPED_MUTEX_LOCK(lock, this->_mutex, this_ethread()); QUICFrame *frame = nullptr; - if (!this->_is_level_matched(level)) { - return frame; - } - + ink_assert(level != QUICEncryptionLevel::NONE); if (this->_need_to_fire[static_cast(level)] > 0 && maximum_frame_size > 0) { // don't care ping frame lost or acked - frame = QUICFrameFactory::create_ping_frame(buf, 0, nullptr); - this->_need_to_fire[static_cast(level)] = 0; + frame = QUICFrameFactory::create_ping_frame(buf, 0, nullptr); + --this->_need_to_fire[static_cast(level)]; + this->_ack_eliciting_packet_out = true; } return frame; } + +uint64_t +QUICPinger::count(QUICEncryptionLevel level) +{ + SCOPED_MUTEX_LOCK(lock, this->_mutex, this_ethread()); + ink_assert(level != QUICEncryptionLevel::NONE); + return this->_need_to_fire[static_cast(level)]; +} diff --git a/iocore/net/quic/QUICPinger.h b/iocore/net/quic/QUICPinger.h index efa59a375d9..0ea9b58f8d2 100644 --- a/iocore/net/quic/QUICPinger.h +++ b/iocore/net/quic/QUICPinger.h @@ -28,20 +28,25 @@ #include "QUICFrameHandler.h" #include "QUICFrameGenerator.h" -class QUICPinger : public QUICFrameGenerator +#include "I_Lock.h" + +class QUICPinger : public QUICFrameOnceGenerator { public: - QUICPinger() {} + QUICPinger() : _mutex(new_ProxyMutex()) {} void request(QUICEncryptionLevel level); void cancel(QUICEncryptionLevel level); + uint64_t count(QUICEncryptionLevel level); +private: // QUICFrameGenerator - bool will_generate_frame(QUICEncryptionLevel level, ink_hrtime timestamp) override; - QUICFrame *generate_frame(uint8_t *buf, QUICEncryptionLevel level, uint64_t connection_credit, uint16_t maximum_frame_size, - ink_hrtime timestamp) override; + bool _will_generate_frame(QUICEncryptionLevel level, size_t current_packet_size, bool ack_eliciting) override; + QUICFrame *_generate_frame(uint8_t *buf, QUICEncryptionLevel level, uint64_t connection_credit, uint16_t maximum_frame_size, + size_t current_packet_size) override; -private: - // Initial, 0/1-RTT, and Handshake - uint64_t _need_to_fire[4] = {0}; + bool _ack_eliciting_packet_out = false; + + Ptr _mutex; + uint64_t _need_to_fire[4] = {0}; // Initial, 0RTT, HANDSHAKE and 1RTT }; diff --git a/iocore/net/quic/QUICResetTokenTable.cc b/iocore/net/quic/QUICResetTokenTable.cc new file mode 100644 index 00000000000..0da753f976f --- /dev/null +++ b/iocore/net/quic/QUICResetTokenTable.cc @@ -0,0 +1,53 @@ +/** @file + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#include "tscore/Diags.h" +#include "QUICResetTokenTable.h" +#include "QUICConnection.h" + +void +QUICResetTokenTable::insert(const QUICStatelessResetToken token, QUICConnection *connection) +{ + Debug("quic_reset_token_table", "Token:%02x%02x%02x%02x... CID:%08" PRIx32 "...", token.buf()[0], token.buf()[1], token.buf()[2], + token.buf()[3], connection->connection_id().h32()); + this->_map.emplace(token, connection); +} + +QUICConnection * +QUICResetTokenTable::lookup(QUICStatelessResetToken token) +{ + Debug("quic_reset_token_table", "Token:%02x%02x%02x%02x...", token.buf()[0], token.buf()[1], token.buf()[2], token.buf()[3]); + auto result = this->_map.find(token); + if (result != this->_map.end()) { + Debug("quic_reset_token_table", "CID:%08" PRIx32 "...", result->second->connection_id().h32()); + return result->second; + } else { + Debug("quic_reset_token_table", "not fouund"); + return nullptr; + } +} + +void +QUICResetTokenTable::erase(const QUICStatelessResetToken token) +{ + Debug("quic_reset_token_table", "Token:%02x%02x%02x%02x...", token.buf()[0], token.buf()[1], token.buf()[2], token.buf()[3]); + this->_map.erase(token); +} diff --git a/iocore/net/quic/QUICResetTokenTable.h b/iocore/net/quic/QUICResetTokenTable.h new file mode 100644 index 00000000000..559b2647582 --- /dev/null +++ b/iocore/net/quic/QUICResetTokenTable.h @@ -0,0 +1,46 @@ +/** @file + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#pragma once + +#include "QUICTypes.h" + +class QUICConnection; + +class QUICResetTokenTable +{ +public: + void insert(const QUICStatelessResetToken token, QUICConnection *connection); + QUICConnection *lookup(const QUICStatelessResetToken token); + void erase(const QUICStatelessResetToken token); + +private: + class TokenHasher + { + public: + uint64_t + operator()(const QUICStatelessResetToken &key) const + { + return static_cast(key); + } + }; + std::unordered_map _map; +}; diff --git a/iocore/net/quic/QUICRetryIntegrityTag.cc b/iocore/net/quic/QUICRetryIntegrityTag.cc new file mode 100644 index 00000000000..117dff696a4 --- /dev/null +++ b/iocore/net/quic/QUICRetryIntegrityTag.cc @@ -0,0 +1,96 @@ +/** @file + * + * A brief file description + * + * @section license License + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "QUICRetryIntegrityTag.h" + +bool +QUICRetryIntegrityTag::compute(uint8_t *out, QUICVersion version, QUICConnectionId odcid, Ptr header, + Ptr payload) +{ + EVP_CIPHER_CTX *aead_ctx; + + if (!(aead_ctx = EVP_CIPHER_CTX_new())) { + return false; + } + if (!EVP_EncryptInit_ex(aead_ctx, EVP_aes_128_gcm(), nullptr, nullptr, nullptr)) { + return false; + } + if (!EVP_CIPHER_CTX_ctrl(aead_ctx, EVP_CTRL_AEAD_SET_IVLEN, 12, nullptr)) { + return false; + } + const uint8_t *key; + const uint8_t *nonce; + switch (version) { + case 0xff00001d: // Draft-29 + key = KEY_FOR_RETRY_INTEGRITY_TAG; + nonce = NONCE_FOR_RETRY_INTEGRITY_TAG; + break; + case 0xff00001b: // Draft-27 + key = KEY_FOR_RETRY_INTEGRITY_TAG_D27; + nonce = NONCE_FOR_RETRY_INTEGRITY_TAG_D27; + break; + default: + key = KEY_FOR_RETRY_INTEGRITY_TAG; + nonce = NONCE_FOR_RETRY_INTEGRITY_TAG; + break; + } + if (!EVP_EncryptInit_ex(aead_ctx, nullptr, nullptr, key, nonce)) { + return false; + } + + // Original Destination Connection ID + size_t n; + int dummy; + uint8_t odcid_buf[1 + QUICConnectionId::MAX_LENGTH]; + // Len + odcid_buf[0] = odcid.length(); + // ID + QUICTypeUtil::write_QUICConnectionId(odcid, odcid_buf + 1, &n); + if (!EVP_EncryptUpdate(aead_ctx, nullptr, &dummy, odcid_buf, 1 + odcid.length())) { + return false; + } + + // Retry Packet + for (Ptr b = header; b; b = b->next) { + if (!EVP_EncryptUpdate(aead_ctx, nullptr, &dummy, reinterpret_cast(b->start()), b->size())) { + return false; + } + } + for (Ptr b = payload; b; b = b->next) { + if (!EVP_EncryptUpdate(aead_ctx, nullptr, &dummy, reinterpret_cast(b->start()), b->size())) { + return false; + } + } + + if (!EVP_EncryptFinal_ex(aead_ctx, nullptr, &dummy)) { + return false; + } + + if (!EVP_CIPHER_CTX_ctrl(aead_ctx, EVP_CTRL_AEAD_GET_TAG, LEN, out)) { + return false; + } + + EVP_CIPHER_CTX_free(aead_ctx); + + return true; +} diff --git a/iocore/net/quic/QUICRetryIntegrityTag.h b/iocore/net/quic/QUICRetryIntegrityTag.h new file mode 100644 index 00000000000..081bd59da77 --- /dev/null +++ b/iocore/net/quic/QUICRetryIntegrityTag.h @@ -0,0 +1,45 @@ +/** @file + * + * A brief file description + * + * @section license License + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "QUICTypes.h" + +class QUICRetryIntegrityTag +{ +public: + static constexpr int LEN = 16; + static bool compute(uint8_t *out, QUICVersion version, QUICConnectionId odcid, Ptr header, + Ptr payload); + +private: + static constexpr uint8_t KEY_FOR_RETRY_INTEGRITY_TAG[] = {0xcc, 0xce, 0x18, 0x7e, 0xd0, 0x9a, 0x09, 0xd0, + 0x57, 0x28, 0x15, 0x5a, 0x6c, 0xb9, 0x6b, 0xe1}; + static constexpr uint8_t NONCE_FOR_RETRY_INTEGRITY_TAG[] = {0xe5, 0x49, 0x30, 0xf9, 0x7f, 0x21, + 0x36, 0xf0, 0x53, 0x0a, 0x8c, 0x1c}; + // For draft 27 + static constexpr uint8_t KEY_FOR_RETRY_INTEGRITY_TAG_D27[] = {0x4d, 0x32, 0xec, 0xdb, 0x2a, 0x21, 0x33, 0xc8, + 0x41, 0xe4, 0x04, 0x3d, 0xf2, 0x7d, 0x44, 0x30}; + static constexpr uint8_t NONCE_FOR_RETRY_INTEGRITY_TAG_D27[] = {0x4d, 0x16, 0x11, 0xd0, 0x55, 0x13, + 0xa5, 0x52, 0xc5, 0x87, 0xd5, 0x75}; +}; diff --git a/iocore/net/quic/QUICStream.cc b/iocore/net/quic/QUICStream.cc index 334b9f41bca..93ac8f1f733 100644 --- a/iocore/net/quic/QUICStream.cc +++ b/iocore/net/quic/QUICStream.cc @@ -245,7 +245,7 @@ QUICStreamVConnection::_signal_read_event() } MUTEX_TRY_LOCK(lock, this->_read_vio.mutex, this_ethread()); - int event = this->_read_vio.ntodo() ? VC_EVENT_READ_READY : VC_EVENT_READ_COMPLETE; + int event = this->_read_vio.nbytes == INT64_MAX ? VC_EVENT_READ_READY : VC_EVENT_READ_COMPLETE; if (lock.is_locked()) { this->_read_vio.cont->handleEvent(event, &this->_read_vio); diff --git a/iocore/net/quic/QUICStream.h b/iocore/net/quic/QUICStream.h index bff660a0b6d..afeb03c4a23 100644 --- a/iocore/net/quic/QUICStream.h +++ b/iocore/net/quic/QUICStream.h @@ -130,5 +130,8 @@ class QUICStreamVConnection : public VConnection, public QUICStream #define QUICStreamFCDebug(fmt, ...) \ Debug("quic_flow_ctrl", "[%s] [%" PRIu64 "] [%s] " fmt, this->_connection_info->cids().data(), this->_id, \ QUICDebugNames::stream_state(this->_state.get()), ##__VA_ARGS__) +#define QUICVStreamFCDebug(fmt, ...) \ + Debug("v_quic_flow_ctrl", "[%s] [%" PRIu64 "] [%s] " fmt, this->_connection_info->cids().data(), this->_id, \ + QUICDebugNames::stream_state(this->_state.get()), ##__VA_ARGS__) extern const uint32_t MAX_STREAM_FRAME_OVERHEAD; diff --git a/iocore/net/quic/QUICStreamManager.cc b/iocore/net/quic/QUICStreamManager.cc index 91edd1b7c8b..da66d740822 100644 --- a/iocore/net/quic/QUICStreamManager.cc +++ b/iocore/net/quic/QUICStreamManager.cc @@ -29,12 +29,10 @@ static constexpr char tag[] = "quic_stream_manager"; static constexpr QUICStreamId QUIC_STREAM_TYPES = 4; -ClassAllocator quicStreamManagerAllocator("quicStreamManagerAllocator"); - -QUICStreamManager::QUICStreamManager(QUICConnectionInfoProvider *info, QUICRTTProvider *rtt_provider, QUICApplicationMap *app_map) - : _stream_factory(rtt_provider, info), _info(info), _app_map(app_map) +QUICStreamManager::QUICStreamManager(QUICContext *context, QUICApplicationMap *app_map) + : _stream_factory(context->rtt_provider(), context->connection_info()), _context(context), _app_map(app_map) { - if (this->_info->direction() == NET_VCONNECTION_OUT) { + if (this->_context->connection_info()->direction() == NET_VCONNECTION_OUT) { this->_next_stream_id_bidi = static_cast(QUICStreamType::CLIENT_BIDI); this->_next_stream_id_uni = static_cast(QUICStreamType::CLIENT_UNI); } else { @@ -92,7 +90,7 @@ QUICStreamManager::create_stream(QUICStreamId stream_id) QUICConnectionErrorUPtr error = nullptr; QUICStreamVConnection *stream_vc = this->_find_or_create_stream_vc(stream_id); if (!stream_vc) { - return std::make_unique(QUICTransErrorCode::STREAM_ID_ERROR); + return std::make_unique(QUICTransErrorCode::STREAM_LIMIT_ERROR); } QUICApplication *application = this->_app_map->get(stream_id); @@ -176,7 +174,7 @@ QUICStreamManager::_handle_frame(const QUICMaxStreamDataFrame &frame) if (stream) { return stream->recv(frame); } else { - return std::make_unique(QUICTransErrorCode::STREAM_ID_ERROR); + return std::make_unique(QUICTransErrorCode::STREAM_LIMIT_ERROR); } } @@ -187,7 +185,7 @@ QUICStreamManager::_handle_frame(const QUICStreamDataBlockedFrame &frame) if (stream) { return stream->recv(frame); } else { - return std::make_unique(QUICTransErrorCode::STREAM_ID_ERROR); + return std::make_unique(QUICTransErrorCode::STREAM_LIMIT_ERROR); } } @@ -196,7 +194,7 @@ QUICStreamManager::_handle_frame(const QUICStreamFrame &frame) { QUICStreamVConnection *stream = this->_find_or_create_stream_vc(frame.stream_id()); if (!stream) { - return std::make_unique(QUICTransErrorCode::STREAM_ID_ERROR); + return std::make_unique(QUICTransErrorCode::STREAM_LIMIT_ERROR); } QUICApplication *application = this->_app_map->get(frame.stream_id()); @@ -215,7 +213,7 @@ QUICStreamManager::_handle_frame(const QUICRstStreamFrame &frame) if (stream) { return stream->recv(frame); } else { - return std::make_unique(QUICTransErrorCode::STREAM_ID_ERROR); + return std::make_unique(QUICTransErrorCode::STREAM_LIMIT_ERROR); } } @@ -226,7 +224,7 @@ QUICStreamManager::_handle_frame(const QUICStopSendingFrame &frame) if (stream) { return stream->recv(frame); } else { - return std::make_unique(QUICTransErrorCode::STREAM_ID_ERROR); + return std::make_unique(QUICTransErrorCode::STREAM_LIMIT_ERROR); } } @@ -267,12 +265,13 @@ QUICStreamManager::_find_or_create_stream_vc(QUICStreamId stream_id) uint64_t local_max_stream_data = 0; uint64_t remote_max_stream_data = 0; + uint64_t nth_stream = this->_stream_id_to_nth_stream(stream_id); switch (QUICTypeUtil::detect_stream_type(stream_id)) { case QUICStreamType::CLIENT_BIDI: - if (this->_info->direction() == NET_VCONNECTION_OUT) { + if (this->_context->connection_info()->direction() == NET_VCONNECTION_OUT) { // client - if (this->_remote_max_streams_bidi == 0 || stream_id > this->_remote_max_streams_bidi) { + if (this->_remote_max_streams_bidi == 0 || nth_stream > this->_remote_max_streams_bidi) { return nullptr; } @@ -280,7 +279,7 @@ QUICStreamManager::_find_or_create_stream_vc(QUICStreamId stream_id) remote_max_stream_data = this->_remote_tp->getAsUInt(QUICTransportParameterId::INITIAL_MAX_STREAM_DATA_BIDI_REMOTE); } else { // server - if (this->_local_max_streams_bidi == 0 || stream_id > this->_local_max_streams_bidi) { + if (this->_local_max_streams_bidi == 0 || nth_stream > this->_local_max_streams_bidi) { return nullptr; } @@ -290,14 +289,14 @@ QUICStreamManager::_find_or_create_stream_vc(QUICStreamId stream_id) break; case QUICStreamType::CLIENT_UNI: - if (this->_info->direction() == NET_VCONNECTION_OUT) { + if (this->_context->connection_info()->direction() == NET_VCONNECTION_OUT) { // client - if (this->_remote_max_streams_uni == 0 || stream_id > this->_remote_max_streams_uni) { + if (this->_remote_max_streams_uni == 0 || nth_stream > this->_remote_max_streams_uni) { return nullptr; } } else { // server - if (this->_local_max_streams_uni == 0 || stream_id > this->_local_max_streams_uni) { + if (this->_local_max_streams_uni == 0 || nth_stream > this->_local_max_streams_uni) { return nullptr; } } @@ -307,9 +306,9 @@ QUICStreamManager::_find_or_create_stream_vc(QUICStreamId stream_id) break; case QUICStreamType::SERVER_BIDI: - if (this->_info->direction() == NET_VCONNECTION_OUT) { + if (this->_context->connection_info()->direction() == NET_VCONNECTION_OUT) { // client - if (this->_local_max_streams_bidi == 0 || stream_id > this->_local_max_streams_bidi) { + if (this->_local_max_streams_bidi == 0 || nth_stream > this->_local_max_streams_bidi) { return nullptr; } @@ -317,7 +316,7 @@ QUICStreamManager::_find_or_create_stream_vc(QUICStreamId stream_id) remote_max_stream_data = this->_remote_tp->getAsUInt(QUICTransportParameterId::INITIAL_MAX_STREAM_DATA_BIDI_LOCAL); } else { // server - if (this->_remote_max_streams_bidi == 0 || stream_id > this->_remote_max_streams_bidi) { + if (this->_remote_max_streams_bidi == 0 || nth_stream > this->_remote_max_streams_bidi) { return nullptr; } @@ -326,12 +325,12 @@ QUICStreamManager::_find_or_create_stream_vc(QUICStreamId stream_id) } break; case QUICStreamType::SERVER_UNI: - if (this->_info->direction() == NET_VCONNECTION_OUT) { - if (this->_local_max_streams_uni == 0 || stream_id > this->_local_max_streams_uni) { + if (this->_context->connection_info()->direction() == NET_VCONNECTION_OUT) { + if (this->_local_max_streams_uni == 0 || nth_stream > this->_local_max_streams_uni) { return nullptr; } } else { - if (this->_remote_max_streams_uni == 0 || stream_id > this->_remote_max_streams_uni) { + if (this->_remote_max_streams_uni == 0 || nth_stream > this->_remote_max_streams_uni) { return nullptr; } } @@ -407,7 +406,7 @@ QUICStreamManager::set_default_application(QUICApplication *app) } bool -QUICStreamManager::will_generate_frame(QUICEncryptionLevel level, ink_hrtime timestamp) +QUICStreamManager::will_generate_frame(QUICEncryptionLevel level, size_t current_packet_size, bool ack_eliciting, uint32_t seq_num) { if (!this->_is_level_matched(level)) { return false; @@ -418,8 +417,13 @@ QUICStreamManager::will_generate_frame(QUICEncryptionLevel level, ink_hrtime tim return false; } + // Don't send DATA frames if current path is not validated + if (!this->_context->path_manager()->get_verified_path().remote_ep().isValid()) { + return false; + } + for (QUICStreamVConnection *s = this->stream_list.head; s; s = s->link.next) { - if (s->will_generate_frame(level, timestamp)) { + if (s->will_generate_frame(level, current_packet_size, ack_eliciting, seq_num)) { return true; } } @@ -429,7 +433,7 @@ QUICStreamManager::will_generate_frame(QUICEncryptionLevel level, ink_hrtime tim QUICFrame * QUICStreamManager::generate_frame(uint8_t *buf, QUICEncryptionLevel level, uint64_t connection_credit, uint16_t maximum_frame_size, - ink_hrtime timestamp) + size_t current_packet_size, uint32_t seq_num) { QUICFrame *frame = nullptr; @@ -444,7 +448,7 @@ QUICStreamManager::generate_frame(uint8_t *buf, QUICEncryptionLevel level, uint6 // FIXME We should pick a stream based on priority for (QUICStreamVConnection *s = this->stream_list.head; s; s = s->link.next) { - frame = s->generate_frame(buf, level, connection_credit, maximum_frame_size, timestamp); + frame = s->generate_frame(buf, level, connection_credit, maximum_frame_size, current_packet_size, seq_num); if (frame) { break; } @@ -468,3 +472,9 @@ QUICStreamManager::_is_level_matched(QUICEncryptionLevel level) return false; } + +uint64_t +QUICStreamManager::_stream_id_to_nth_stream(QUICStreamId stream_id) +{ + return (stream_id / 4) + 1; +} diff --git a/iocore/net/quic/QUICStreamManager.h b/iocore/net/quic/QUICStreamManager.h index fbdc49d78fd..05da5fc93ab 100644 --- a/iocore/net/quic/QUICStreamManager.h +++ b/iocore/net/quic/QUICStreamManager.h @@ -31,14 +31,15 @@ #include "QUICFrame.h" #include "QUICStreamFactory.h" #include "QUICLossDetector.h" +#include "QUICPathManager.h" +#include "QUICContext.h" class QUICTransportParameters; class QUICStreamManager : public QUICFrameHandler, public QUICFrameGenerator { public: - QUICStreamManager() : _stream_factory(nullptr, nullptr){}; - QUICStreamManager(QUICConnectionInfoProvider *info, QUICRTTProvider *rtt_provider, QUICApplicationMap *app_map); + QUICStreamManager(QUICContext *context, QUICApplicationMap *app_map); void init_flow_control_params(const std::shared_ptr &local_tp, const std::shared_ptr &remote_tp); @@ -63,9 +64,9 @@ class QUICStreamManager : public QUICFrameHandler, public QUICFrameGenerator virtual QUICConnectionErrorUPtr handle_frame(QUICEncryptionLevel level, const QUICFrame &frame) override; // QUICFrameGenerator - bool will_generate_frame(QUICEncryptionLevel level, ink_hrtime timestamp) override; + bool will_generate_frame(QUICEncryptionLevel level, size_t current_packet_size, bool ack_eliciting, uint32_t timestamp) override; QUICFrame *generate_frame(uint8_t *buf, QUICEncryptionLevel level, uint64_t connection_credit, uint16_t maximum_frame_size, - ink_hrtime timestamp) override; + size_t current_packet_size, uint32_t timestamp) override; protected: virtual bool _is_level_matched(QUICEncryptionLevel level) override; @@ -83,7 +84,7 @@ class QUICStreamManager : public QUICFrameHandler, public QUICFrameGenerator QUICStreamFactory _stream_factory; - QUICConnectionInfoProvider *_info = nullptr; + QUICContext *_context = nullptr; QUICApplicationMap *_app_map = nullptr; std::shared_ptr _local_tp = nullptr; std::shared_ptr _remote_tp = nullptr; @@ -98,4 +99,6 @@ class QUICStreamManager : public QUICFrameHandler, public QUICFrameGenerator QUICEncryptionLevel::ZERO_RTT, QUICEncryptionLevel::ONE_RTT, }; + + uint64_t _stream_id_to_nth_stream(QUICStreamId stream_id); }; diff --git a/iocore/net/quic/QUICStreamState.cc b/iocore/net/quic/QUICStreamState.cc index d88348dcc17..7c5b8d1050a 100644 --- a/iocore/net/quic/QUICStreamState.cc +++ b/iocore/net/quic/QUICStreamState.cc @@ -40,13 +40,13 @@ QUICReceiveStreamStateMachine::is_allowed_to_send(QUICFrameType type) const } QUICReceiveStreamState state = this->get(); - // The receiver only sends MAX_STREAM_DATA in the “Recv” state. + // The receiver only sends MAX_STREAM_DATA in the "Recv" state. if (type == QUICFrameType::MAX_STREAM_DATA && state == QUICReceiveStreamState::Recv) { return true; } - // A receiver can send STOP_SENDING in any state where it has not received a RESET_STREAM frame; that is states other than “Reset - // Recvd” or “Reset Read”. + // A receiver can send STOP_SENDING in any state where it has not received a RESET_STREAM frame; that is states other than "Reset + // Recvd" or "Reset Read". if (type == QUICFrameType::STOP_SENDING && state != QUICReceiveStreamState::ResetRecvd && state != QUICReceiveStreamState::ResetRead) { return true; @@ -143,8 +143,8 @@ QUICReceiveStreamStateMachine::update_on_eos() void QUICReceiveStreamStateMachine::update(const QUICSendStreamState state) { - // The receiving part of a stream enters the “Recv” state when the sending part of a bidirectional stream initiated by the - // endpoint (type 0 for a client, type 1 for a server) enters the “Ready” state. + // The receiving part of a stream enters the "Recv" state when the sending part of a bidirectional stream initiated by the + // endpoint (type 0 for a client, type 1 for a server) enters the "Ready" state. switch (this->get()) { case QUICReceiveStreamState::Init: if (state == QUICSendStreamState::Ready) { @@ -187,19 +187,19 @@ QUICSendStreamStateMachine::is_allowed_to_send(QUICFrameType type) const return true; } break; - // A sender MUST NOT send any of these frames from a terminal state (“Data Recvd” or “Reset Recvd”). + // A sender MUST NOT send any of these frames from a terminal state ("Data Recvd" or "Reset Recvd"). case QUICSendStreamState::DataRecvd: case QUICSendStreamState::ResetRecvd: break; // A sender MUST NOT send STREAM or STREAM_DATA_BLOCKED after sending a RESET_STREAM; that is, in the terminal states and in the - // “Reset Sent” state. + // "Reset Sent" state. case QUICSendStreamState::ResetSent: if (type == QUICFrameType::RESET_STREAM) { return true; } break; default: - ink_assert("This shouuldn't be happen"); + ink_assert("This shouldn't be happen"); break; } @@ -221,7 +221,7 @@ QUICSendStreamStateMachine::is_allowed_to_receive(QUICFrameType type) const // A sender could receive either of these two frames(MAX_STREAM_DATA and STOP_SENDING) in any state as a result of delayed // delivery of packets. - // PS: Because we need to reply a RESET_STREAM frame. STOP_SENDING frame is accpeted in all states. But we + // PS: Because we need to reply a RESET_STREAM frame. STOP_SENDING frame is accepted in all states. But we // don't need to do anything for MAX_STREAM_DATA frame when we are in terminal state. if (type == QUICFrameType::STOP_SENDING) { return true; @@ -306,8 +306,8 @@ QUICSendStreamStateMachine::update_on_ack() void QUICSendStreamStateMachine::update(const QUICReceiveStreamState state) { - // The sending part of a bidirectional stream initiated by a peer (type 0 for a server, type 1 for a client) enters the “Ready” - // state then immediately transitions to the “Send” state if the receiving part enters the “Recv” state (Section 3.2). + // The sending part of a bidirectional stream initiated by a peer (type 0 for a server, type 1 for a client) enters the "Ready" + // state then immediately transitions to the "Send" state if the receiving part enters the "Recv" state (Section 3.2). switch (this->get()) { case QUICSendStreamState::Ready: if (state == QUICReceiveStreamState::Recv) { @@ -372,8 +372,8 @@ QUICBidirectionalStreamStateMachine::get() const void QUICBidirectionalStreamStateMachine::update_with_sending_frame(const QUICFrame &frame) { - // The receiving part of a stream enters the “Recv” state when the sending part of a bidirectional stream initiated by the - // endpoint (type 0 for a client, type 1 for a server) enters the “Ready” state. + // The receiving part of a stream enters the "Recv" state when the sending part of a bidirectional stream initiated by the + // endpoint (type 0 for a client, type 1 for a server) enters the "Ready" state. this->_send_stream_state.update_with_sending_frame(frame); // PS: It should not happen because we initialize the send side and read side together. And the SendState has the default state // "Ready". But to obey the specs, we do this as follow. @@ -386,8 +386,8 @@ QUICBidirectionalStreamStateMachine::update_with_sending_frame(const QUICFrame & void QUICBidirectionalStreamStateMachine::update_with_receiving_frame(const QUICFrame &frame) { - // The sending part of a bidirectional stream initiated by a peer (type 0 for a server, type 1 for a client) enters the “Ready” - // state then immediately transitions to the “Send” state if the receiving part enters the “Recv” state (Section 3.2). + // The sending part of a bidirectional stream initiated by a peer (type 0 for a server, type 1 for a client) enters the "Ready" + // state then immediately transitions to the "Send" state if the receiving part enters the "Recv" state (Section 3.2). this->_recv_stream_state.update_with_receiving_frame(frame); if (this->_send_stream_state.get() == QUICSendStreamState::Ready && this->_recv_stream_state.get() == QUICReceiveStreamState::Recv) { diff --git a/iocore/net/quic/QUICTLS.cc b/iocore/net/quic/QUICTLS.cc index 5736ef46191..a618e3add0a 100644 --- a/iocore/net/quic/QUICTLS.cc +++ b/iocore/net/quic/QUICTLS.cc @@ -32,6 +32,55 @@ constexpr static char tag[] = "quic_tls"; +static const char * +content_type_str(int type) +{ + switch (type) { + case SSL3_RT_CHANGE_CIPHER_SPEC: + return "CHANGE_CIPHER_SPEC"; + case SSL3_RT_ALERT: + return "ALERT"; + case SSL3_RT_HANDSHAKE: + return "HANDSHAKE"; + case SSL3_RT_APPLICATION_DATA: + return "APPLICATION_DATA"; + case SSL3_RT_HEADER: + // The buf contains the record header bytes only + return "HEADER"; + default: + return "UNKNOWN"; + } +} + +static const char * +hs_type_str(int type) +{ + switch (type) { + case SSL3_MT_CLIENT_HELLO: + return "CLIENT_HELLO"; + case SSL3_MT_SERVER_HELLO: + return "SERVER_HELLO"; + case SSL3_MT_NEWSESSION_TICKET: + return "NEWSESSION_TICKET"; + case SSL3_MT_END_OF_EARLY_DATA: + return "END_OF_EARLY_DATA"; + case SSL3_MT_ENCRYPTED_EXTENSIONS: + return "ENCRYPTED_EXTENSIONS"; + case SSL3_MT_CERTIFICATE: + return "CERTIFICATE"; + case SSL3_MT_CERTIFICATE_VERIFY: + return "CERTIFICATE_VERIFY"; + case SSL3_MT_FINISHED: + return "FINISHED"; + case SSL3_MT_KEY_UPDATE: + return "KEY_UPDATE"; + case SSL3_MT_MESSAGE_HASH: + return "MESSAGE_HASH"; + default: + return "UNKNOWN"; + } +} + SSL * QUICTLS::ssl_handle() { @@ -50,12 +99,6 @@ QUICTLS::remote_transport_parameters() return this->_remote_transport_parameters; } -void -QUICTLS::set_local_transport_parameters(std::shared_ptr tp) -{ - this->_local_transport_parameters = tp; -} - void QUICTLS::set_remote_transport_parameters(std::shared_ptr tp) { @@ -79,16 +122,30 @@ QUICTLS::~QUICTLS() SSL_free(this->_ssl); } +int +QUICTLS::handshake(QUICHandshakeMsgs **out, const QUICHandshakeMsgs *in) +{ + if (this->is_handshake_finished()) { + if (in != nullptr && in->offsets[4] != 0) { + return this->_process_post_handshake_messages(*out, in); + } + + return 1; + } + + return this->_handshake(out, in); +} + void QUICTLS::reset() { SSL_clear(this->_ssl); } -uint16_t +uint64_t QUICTLS::convert_to_quic_trans_error_code(uint8_t alert) { - return 0x100 | alert; + return static_cast(QUICTransErrorCode::CRYPTO_ERROR) + alert; } bool @@ -108,13 +165,15 @@ QUICTLS::is_ready_to_derive() const } int -QUICTLS::initialize_key_materials(QUICConnectionId cid) +QUICTLS::initialize_key_materials(QUICConnectionId cid, QUICVersion version) { this->_pp_key_info.set_cipher_initial(EVP_aes_128_gcm()); this->_pp_key_info.set_cipher_for_hp_initial(EVP_aes_128_ecb()); // Generate keys - Debug(tag, "Generating %s keys", QUICDebugNames::key_phase(QUICKeyPhase::INITIAL)); + if (is_debug_tag_set(tag)) { + Debug(tag, "Generating %s keys with cid %s", QUICDebugNames::key_phase(QUICKeyPhase::INITIAL), cid.hex().c_str()); + } uint8_t *client_key_for_hp; uint8_t *client_key; @@ -158,8 +217,8 @@ QUICTLS::initialize_key_materials(QUICConnectionId cid) server_iv_len = this->_pp_key_info.decryption_iv_len(QUICKeyPhase::INITIAL); } - this->_keygen_for_client.generate(client_key_for_hp, client_key, client_iv, client_iv_len, cid); - this->_keygen_for_server.generate(server_key_for_hp, server_key, server_iv, server_iv_len, cid); + this->_keygen_for_client.generate(version, client_key_for_hp, client_key, client_iv, client_iv_len, cid); + this->_keygen_for_server.generate(version, server_key_for_hp, server_key, server_iv, server_iv_len, cid); this->_pp_key_info.set_decryption_key_available(QUICKeyPhase::INITIAL); this->_pp_key_info.set_encryption_key_available(QUICKeyPhase::INITIAL); @@ -172,6 +231,127 @@ QUICTLS::initialize_key_materials(QUICConnectionId cid) return 1; } +void +QUICTLS::update_negotiated_cipher() +{ + this->_store_negotiated_cipher(); + this->_store_negotiated_cipher_for_hp(); +} + +void +QUICTLS::update_key_materials_for_read(QUICEncryptionLevel level, const uint8_t *secret, size_t secret_len) +{ + if (this->_state == HandshakeState::ABORTED) { + return; + } + + QUICKeyPhase phase; + const EVP_CIPHER *cipher; + + switch (level) { + case QUICEncryptionLevel::ZERO_RTT: + phase = QUICKeyPhase::ZERO_RTT; + cipher = this->_pp_key_info.get_cipher(phase); + break; + case QUICEncryptionLevel::HANDSHAKE: + this->_update_encryption_level(QUICEncryptionLevel::HANDSHAKE); + phase = QUICKeyPhase::HANDSHAKE; + break; + case QUICEncryptionLevel::ONE_RTT: + this->_update_encryption_level(QUICEncryptionLevel::ONE_RTT); + // TODO Support Key Update + phase = QUICKeyPhase::PHASE_0; + break; + default: + phase = QUICKeyPhase::INITIAL; + break; + } + + QUICHKDF hkdf(this->_get_handshake_digest()); + + uint8_t *key_for_hp; + uint8_t *key; + uint8_t *iv; + size_t key_for_hp_len; + size_t key_len; + size_t *iv_len; + + cipher = this->_pp_key_info.get_cipher(phase); + key_for_hp = this->_pp_key_info.decryption_key_for_hp(phase); + key_for_hp_len = this->_pp_key_info.decryption_key_for_hp_len(phase); + key = this->_pp_key_info.decryption_key(phase); + key_len = this->_pp_key_info.decryption_key_len(phase); + iv = this->_pp_key_info.decryption_iv(phase); + iv_len = this->_pp_key_info.decryption_iv_len(phase); + + if (this->_netvc_context == NET_VCONNECTION_IN) { + this->_keygen_for_client.regenerate(key_for_hp, key, iv, iv_len, secret, secret_len, cipher, hkdf); + this->_print_km("update - client", key_for_hp, key_for_hp_len, key, key_len, iv, *iv_len, secret, secret_len, phase); + } else { + this->_keygen_for_server.regenerate(key_for_hp, key, iv, iv_len, secret, secret_len, cipher, hkdf); + this->_print_km("update - server", key_for_hp, key_for_hp_len, key, key_len, iv, *iv_len, secret, secret_len, phase); + } + + this->_pp_key_info.set_decryption_key_available(phase); +} + +void +QUICTLS::update_key_materials_for_write(QUICEncryptionLevel level, const uint8_t *secret, size_t secret_len) +{ + if (this->_state == HandshakeState::ABORTED) { + return; + } + + QUICKeyPhase phase; + const EVP_CIPHER *cipher = nullptr; + + switch (level) { + case QUICEncryptionLevel::ZERO_RTT: + phase = QUICKeyPhase::ZERO_RTT; + cipher = this->_pp_key_info.get_cipher(phase); + break; + case QUICEncryptionLevel::HANDSHAKE: + this->_update_encryption_level(QUICEncryptionLevel::HANDSHAKE); + phase = QUICKeyPhase::HANDSHAKE; + cipher = this->_pp_key_info.get_cipher(phase); + break; + case QUICEncryptionLevel::ONE_RTT: + this->_update_encryption_level(QUICEncryptionLevel::ONE_RTT); + phase = QUICKeyPhase::PHASE_0; + cipher = this->_pp_key_info.get_cipher(phase); + break; + default: + phase = QUICKeyPhase::INITIAL; + break; + } + + QUICHKDF hkdf(this->_get_handshake_digest()); + + uint8_t *key_for_hp; + uint8_t *key; + uint8_t *iv; + size_t key_for_hp_len; + size_t key_len; + size_t *iv_len; + + key_for_hp = this->_pp_key_info.encryption_key_for_hp(phase); + key_for_hp_len = this->_pp_key_info.encryption_key_for_hp_len(phase); + key = this->_pp_key_info.encryption_key(phase); + key_len = this->_pp_key_info.encryption_key_len(phase); + iv = this->_pp_key_info.encryption_iv(phase); + iv_len = this->_pp_key_info.encryption_iv_len(phase); + + if (this->_netvc_context == NET_VCONNECTION_IN) { + this->_keygen_for_server.regenerate(key_for_hp, key, iv, iv_len, secret, secret_len, cipher, hkdf); + this->_print_km("update - server", key_for_hp, key_for_hp_len, key, key_len, iv, *iv_len, secret, secret_len, phase); + } else { + this->_keygen_for_client.regenerate(key_for_hp, key, iv, iv_len, secret, secret_len, cipher, hkdf); + this->_print_km("update - client", key_for_hp, key_for_hp_len, key, key_len, iv, *iv_len, secret, secret_len, phase); + } + + this->_pp_key_info.set_encryption_key_available(phase); +} + const char * QUICTLS::negotiated_cipher_suite() const { @@ -198,6 +378,118 @@ QUICTLS::abort_handshake() return; } +void +QUICTLS::set_ready_for_write() +{ + this->_should_flush = true; +} + +void +QUICTLS::on_handshake_data_generated(QUICEncryptionLevel level, const uint8_t *data, size_t len) +{ + int index = static_cast(level); + int next_index = index + 1; + + size_t offset = this->_out.offsets[next_index]; + size_t next_level_offset = offset + len; + + memcpy(this->_out.buf + offset, data, len); + + for (int i = next_index; i < 5; ++i) { + this->_out.offsets[i] = next_level_offset; + } +} + +void +QUICTLS::on_tls_alert(uint8_t alert) +{ + this->_has_crypto_error = true; + this->_crypto_error = QUICTLS::convert_to_quic_trans_error_code(alert); +} + +bool +QUICTLS::has_crypto_error() const +{ + return this->_has_crypto_error; +} + +uint64_t +QUICTLS::crypto_error() const +{ + return this->_crypto_error; +} + +int +QUICTLS::_handshake(QUICHandshakeMsgs **out, const QUICHandshakeMsgs *in) +{ + ink_assert(this->_ssl != nullptr); + if (this->_state == HandshakeState::ABORTED) { + return 0; + } + + int err = SSL_ERROR_NONE; + ERR_clear_error(); + int ret = 0; + + SSL_set_msg_callback(this->_ssl, QUICTLS::_msg_cb); + + this->_out.offsets[0] = 0; + this->_out.offsets[1] = 0; + this->_out.offsets[2] = 0; + this->_out.offsets[3] = 0; + this->_out.offsets[4] = 0; + + if (in) { + this->_pass_quic_data_to_ssl_impl(*in); + } + + if (this->_netvc_context == NET_VCONNECTION_IN) { + if (!this->_early_data_processed) { + if (auto ret = this->_read_early_data(); ret == 0) { + this->_early_data_processed = true; + } else if (ret < 0) { + return 0; + } else { + // Early data is not arrived yet -- can be multiple initial packets + } + } + + ret = SSL_accept(this->_ssl); + } else { + if (!this->_early_data_processed) { + if (this->_write_early_data()) { + this->_early_data_processed = true; + } + } + + ret = SSL_connect(this->_ssl); + } + + if (ret <= 0) { + err = SSL_get_error(this->_ssl, ret); + + switch (err) { + case SSL_ERROR_WANT_READ: + case SSL_ERROR_WANT_WRITE: + break; + default: + char err_buf[256] = {0}; + ERR_error_string_n(ERR_get_error(), err_buf, sizeof(err_buf)); + Debug(tag, "Handshake: %s", err_buf); + return ret; + } + } + + if (this->_should_flush) { + this->_should_flush = false; + *out = &this->_out; + } else { + *out = nullptr; + } + + return 1; +} + void QUICTLS::_update_encryption_level(QUICEncryptionLevel level) { @@ -210,10 +502,10 @@ QUICTLS::_update_encryption_level(QUICEncryptionLevel level) void QUICTLS::_print_km(const char *header, const uint8_t *key_for_hp, size_t key_for_hp_len, const uint8_t *key, size_t key_len, - const uint8_t *iv, size_t iv_len, const uint8_t *secret, size_t secret_len) + const uint8_t *iv, size_t iv_len, const uint8_t *secret, size_t secret_len, QUICKeyPhase phase) { if (is_debug_tag_set("vv_quic_crypto")) { - Debug("vv_quic_crypto", "%s", header); + Debug("vv_quic_crypto", "%s - %s", header, QUICDebugNames::key_phase(phase)); uint8_t print_buf[128]; if (secret) { QUICDebug::to_hex(print_buf, static_cast(secret), secret_len); @@ -227,3 +519,12 @@ QUICTLS::_print_km(const char *header, const uint8_t *key_for_hp, size_t key_for Debug("vv_quic_crypto", "hp=%s", print_buf); } } + +void +QUICTLS::_print_hs_message(int content_type, const void *buf, size_t len) +{ + if ((content_type == SSL3_RT_HANDSHAKE || content_type == SSL3_RT_ALERT)) { + int msg_type = reinterpret_cast(buf)[0]; + Debug(tag, "%s (%d), %s (%d) len=%zu", content_type_str(content_type), content_type, hs_type_str(msg_type), msg_type, len); + } +} diff --git a/iocore/net/quic/QUICTLS.h b/iocore/net/quic/QUICTLS.h index 7bba45aff38..938e1e2d3a0 100644 --- a/iocore/net/quic/QUICTLS.h +++ b/iocore/net/quic/QUICTLS.h @@ -36,6 +36,9 @@ #include "I_NetVConnection.h" #include "QUICHandshakeProtocol.h" +// TODO: fix size +static constexpr int MAX_HANDSHAKE_MSG_LEN = 65527; + class QUICTLS : public QUICHandshakeProtocol { public: @@ -50,7 +53,7 @@ class QUICTLS : public QUICHandshakeProtocol }; static QUICEncryptionLevel get_encryption_level(int msg_type); - static uint16_t convert_to_quic_trans_error_code(uint8_t alert); + static uint64_t convert_to_quic_trans_error_code(uint8_t alert); std::shared_ptr local_transport_parameters() override; std::shared_ptr remote_transport_parameters() override; @@ -64,16 +67,25 @@ class QUICTLS : public QUICHandshakeProtocol SSL *ssl_handle(); // QUICHandshakeProtocol - int handshake(QUICHandshakeMsgs *out, const QUICHandshakeMsgs *in) override; + int handshake(QUICHandshakeMsgs **out, const QUICHandshakeMsgs *in) override; void reset() override; bool is_handshake_finished() const override; bool is_ready_to_derive() const override; - int initialize_key_materials(QUICConnectionId cid) override; - void update_key_materials_on_key_cb(int name, const uint8_t *secret, size_t secret_len); + int initialize_key_materials(QUICConnectionId cid, QUICVersion version) override; + void update_negotiated_cipher(); + void update_key_materials_for_read(QUICEncryptionLevel level, const uint8_t *secret, size_t secret_len); + void update_key_materials_for_write(QUICEncryptionLevel level, const uint8_t *secret, size_t secret_len); const char *negotiated_cipher_suite() const override; void negotiated_application_name(const uint8_t **name, unsigned int *len) const override; QUICEncryptionLevel current_encryption_level() const override; void abort_handshake() override; + bool has_crypto_error() const override; + uint64_t crypto_error() const override; + + void set_ready_for_write(); + + void on_handshake_data_generated(QUICEncryptionLevel level, const uint8_t *data, size_t len); + void on_tls_alert(uint8_t alert); private: QUICKeyGenerator _keygen_for_client = QUICKeyGenerator(QUICKeyGenerator::Context::CLIENT); @@ -82,7 +94,8 @@ class QUICTLS : public QUICHandshakeProtocol int _read_early_data(); int _write_early_data(); - int _handshake(QUICHandshakeMsgs *out, const QUICHandshakeMsgs *in); + void _pass_quic_data_to_ssl_impl(const QUICHandshakeMsgs &in); + int _handshake(QUICHandshakeMsgs **out, const QUICHandshakeMsgs *in); int _process_post_handshake_messages(QUICHandshakeMsgs *out, const QUICHandshakeMsgs *in); void _generate_0rtt_key(); void _update_encryption_level(QUICEncryptionLevel level); @@ -91,8 +104,11 @@ class QUICTLS : public QUICHandshakeProtocol void _store_negotiated_cipher_for_hp(); void _print_km(const char *header, const uint8_t *key_for_hp, size_t key_for_hp_len, const uint8_t *key, size_t key_len, - const uint8_t *iv, size_t iv_len, const uint8_t *secret = nullptr, size_t secret_len = 0); + const uint8_t *iv, size_t iv_len, const uint8_t *secret = nullptr, size_t secret_len = 0, + QUICKeyPhase phase = QUICKeyPhase::INITIAL); + static void _print_hs_message(int content_type, const void *buf, size_t len); + static void _msg_cb(int write_p, int version, int content_type, const void *buf, size_t len, SSL *ssl, void *arg); const char *_session_file = nullptr; const char *_keylog_file = nullptr; SSL *_ssl = nullptr; @@ -105,4 +121,10 @@ class QUICTLS : public QUICHandshakeProtocol std::shared_ptr _local_transport_parameters = nullptr; std::shared_ptr _remote_transport_parameters = nullptr; + + uint8_t _out_buf[MAX_HANDSHAKE_MSG_LEN] = {0}; + QUICHandshakeMsgs _out = {_out_buf, MAX_HANDSHAKE_MSG_LEN, {0}, 0}; + bool _should_flush = false; + bool _has_crypto_error = false; + uint64_t _crypto_error = 0; }; diff --git a/iocore/net/quic/QUICTLS_boringssl.cc b/iocore/net/quic/QUICTLS_boringssl.cc index a535e79bccb..01aabfa1546 100644 --- a/iocore/net/quic/QUICTLS_boringssl.cc +++ b/iocore/net/quic/QUICTLS_boringssl.cc @@ -29,7 +29,148 @@ #include #include -// static constexpr char tag[] = "quic_tls"; +#include "QUICGlobals.h" +#include "QUICConnection.h" +#include "QUICPacketProtectionKeyInfo.h" + +static constexpr char tag[] = "quic_tls"; + +static QUICEncryptionLevel +convert_level_ats2ssl(enum ssl_encryption_level_t level) +{ + switch (level) { + case ssl_encryption_initial: + return QUICEncryptionLevel::INITIAL; + case ssl_encryption_early_data: + return QUICEncryptionLevel::ZERO_RTT; + case ssl_encryption_handshake: + return QUICEncryptionLevel::HANDSHAKE; + case ssl_encryption_application: + return QUICEncryptionLevel::ONE_RTT; + default: + return QUICEncryptionLevel::NONE; + } +} + +#if BORINGSSL_API_VERSION >= 10 +static int +set_read_secret(SSL *ssl, enum ssl_encryption_level_t level, const SSL_CIPHER *cipher, const uint8_t *secret, size_t secret_len) +{ + QUICTLS *qtls = static_cast(SSL_get_ex_data(ssl, QUIC::ssl_quic_tls_index)); + + qtls->update_negotiated_cipher(); + + QUICEncryptionLevel ats_level = convert_level_ats2ssl(level); + qtls->update_key_materials_for_read(ats_level, secret, secret_len); + + return 1; +} + +static int +set_write_secret(SSL *ssl, enum ssl_encryption_level_t level, const SSL_CIPHER *cipher, const uint8_t *secret, size_t secret_len) +{ + QUICTLS *qtls = static_cast(SSL_get_ex_data(ssl, QUIC::ssl_quic_tls_index)); + + qtls->update_negotiated_cipher(); + + QUICEncryptionLevel ats_level = convert_level_ats2ssl(level); + qtls->update_key_materials_for_write(ats_level, secret, secret_len); + + if (ats_level == QUICEncryptionLevel::ONE_RTT) { + // FIXME Where should this be placed? + const uint8_t *tp_buf; + size_t tp_buf_len; + SSL_get_peer_quic_transport_params(ssl, &tp_buf, &tp_buf_len); + const QUICConnection *qc = static_cast(SSL_get_ex_data(ssl, QUIC::ssl_quic_qc_index)); + QUICVersion version = qc->negotiated_version(); + if (SSL_is_server(ssl)) { + qtls->set_remote_transport_parameters(std::make_shared(tp_buf, tp_buf_len, version)); + } else { + qtls->set_remote_transport_parameters( + std::make_shared(tp_buf, tp_buf_len, version)); + } + } + + return 1; +} +#else +static int +set_encryption_secrets(SSL *ssl, enum ssl_encryption_level_t level, const uint8_t *read_secret, const uint8_t *write_secret, + size_t secret_len) +{ + QUICTLS *qtls = static_cast(SSL_get_ex_data(ssl, QUIC::ssl_quic_tls_index)); + + qtls->update_negotiated_cipher(); + + QUICEncryptionLevel ats_level = convert_level_ats2ssl(level); + if (read_secret) { + qtls->update_key_materials_for_read(ats_level, read_secret, secret_len); + } + if (write_secret) { + qtls->update_key_materials_for_write(ats_level, write_secret, secret_len); + } + + if (ats_level == QUICEncryptionLevel::ONE_RTT) { + // FIXME Where should this be placed? + const uint8_t *tp_buf; + size_t tp_buf_len; + SSL_get_peer_quic_transport_params(ssl, &tp_buf, &tp_buf_len); + const QUICConnection *qc = static_cast(SSL_get_ex_data(ssl, QUIC::ssl_quic_qc_index)); + QUICVersion version = qc->negotiated_version(); + if (SSL_is_server(ssl)) { + qtls->set_remote_transport_parameters(std::make_shared(tp_buf, tp_buf_len, version)); + } else { + qtls->set_remote_transport_parameters( + std::make_shared(tp_buf, tp_buf_len, version)); + } + } + + return 1; +} +#endif + +static int +add_handshake_data(SSL *ssl, enum ssl_encryption_level_t level, const uint8_t *data, size_t len) +{ + QUICEncryptionLevel ats_level = convert_level_ats2ssl(level); + + QUICTLS *qtls = static_cast(SSL_get_ex_data(ssl, QUIC::ssl_quic_tls_index)); + qtls->on_handshake_data_generated(ats_level, data, len); + + return 1; +} + +static int +flush_flight(SSL *ssl) +{ + QUICTLS *qtls = static_cast(SSL_get_ex_data(ssl, QUIC::ssl_quic_tls_index)); + qtls->set_ready_for_write(); + + return 1; +} + +static int +send_alert(SSL *ssl, enum ssl_encryption_level_t level, uint8_t alert) +{ + QUICTLS *qtls = static_cast(SSL_get_ex_data(ssl, QUIC::ssl_quic_tls_index)); + qtls->on_tls_alert(alert); + return 1; +} + +#if BORINGSSL_API_VERSION >= 10 +static const SSL_QUIC_METHOD quic_method = {set_read_secret, set_write_secret, add_handshake_data, flush_flight, send_alert}; +#else +static const SSL_QUIC_METHOD quic_method = {set_encryption_secrets, add_handshake_data, flush_flight, send_alert}; +#endif + +void +QUICTLS::_msg_cb(int write_p, int version, int content_type, const void *buf, size_t len, SSL *ssl, void *arg) +{ + // Debug for reading + if (write_p == 0) { + QUICTLS::_print_hs_message(content_type, buf, len); + } +} QUICTLS::QUICTLS(QUICPacketProtectionKeyInfo &pp_key_info, SSL_CTX *ssl_ctx, NetVConnectionContext_t nvc_ctx, const NetVCOptions &netvc_options, const char *session_file, const char *keylog_file) @@ -40,141 +181,188 @@ QUICTLS::QUICTLS(QUICPacketProtectionKeyInfo &pp_key_info, SSL_CTX *ssl_ctx, Net _netvc_context(nvc_ctx) { ink_assert(this->_netvc_context != NET_VCONNECTION_UNSET); + + if (this->_netvc_context == NET_VCONNECTION_OUT) { + SSL_set_connect_state(this->_ssl); + + SSL_set_alpn_protos(this->_ssl, reinterpret_cast(netvc_options.alpn_protos.data()), + netvc_options.alpn_protos.size()); + const ats_scoped_str &tlsext_host_name = netvc_options.sni_hostname ? netvc_options.sni_hostname : netvc_options.sni_servername; + SSL_set_tlsext_host_name(this->_ssl, tlsext_host_name.get()); + } else { + SSL_set_accept_state(this->_ssl); + } + + SSL_set_ex_data(this->_ssl, QUIC::ssl_quic_tls_index, this); + SSL_set_quic_method(this->_ssl, &quic_method); + SSL_set_early_data_enabled(this->_ssl, 1); + + if (session_file && this->_netvc_context == NET_VCONNECTION_OUT) { + auto file = BIO_new_file(session_file, "r"); + if (file == nullptr) { + Debug(tag, "Could not read tls session file %s", session_file); + return; + } + + auto session = PEM_read_bio_SSL_SESSION(file, nullptr, nullptr, nullptr); + if (session == nullptr) { + Debug(tag, "Could not read tls session file %s", session_file); + } else { + if (!SSL_set_session(this->_ssl, session)) { + Debug(tag, "Session resumption failed : %s", session_file); + } else { + Debug(tag, "Session resumption success : %s", session_file); + this->_is_session_reused = true; + } + SSL_SESSION_free(session); + } + + BIO_free(file); + } } -int -QUICTLS::handshake(QUICHandshakeMsgs *out, const QUICHandshakeMsgs *in) +void +QUICTLS::set_local_transport_parameters(std::shared_ptr tp) { - ink_assert(false); - return 0; + this->_local_transport_parameters = tp; + + uint8_t buf[UINT16_MAX]; + uint16_t len; + this->_local_transport_parameters->store(buf, &len); + SSL_set_quic_transport_params(this->_ssl, buf, len); } int QUICTLS::_process_post_handshake_messages(QUICHandshakeMsgs *out, const QUICHandshakeMsgs *in) { - ink_assert(false); - return 0; + this->_pass_quic_data_to_ssl_impl(*in); + return SSL_process_quic_post_handshake(this->_ssl); } -int -QUICTLS::_read_early_data() +void +QUICTLS::_store_negotiated_cipher() { - uint8_t early_data[8]; - size_t early_data_len = 0; - do { - ERR_clear_error(); - early_data_len = SSL_read(this->_ssl, early_data, sizeof(early_data)); - } while (SSL_in_early_data(this->_ssl)); + ink_assert(this->_ssl); - return 1; -} + const EVP_CIPHER *cipher = nullptr; + size_t tag_len = 0; + const SSL_CIPHER *ssl_cipher = SSL_get_current_cipher(this->_ssl); -/* -const EVP_AEAD * -QUICTLS::_get_evp_aead(QUICKeyPhase phase) const -{ - if (phase == QUICKeyPhase::INITIAL) { - return EVP_aead_aes_128_gcm(); - } else { - const SSL_CIPHER *cipher = SSL_get_current_cipher(this->_ssl); - if (cipher) { - switch (SSL_CIPHER_get_id(cipher)) { - case TLS1_CK_AES_128_GCM_SHA256: - return EVP_aead_aes_128_gcm(); - case TLS1_CK_AES_256_GCM_SHA384: - return EVP_aead_aes_256_gcm(); - case TLS1_CK_CHACHA20_POLY1305_SHA256: - return EVP_aead_chacha20_poly1305(); - default: - ink_assert(false); - return nullptr; - } - } else { + if (ssl_cipher) { + switch (SSL_CIPHER_get_id(ssl_cipher)) { + case TLS1_CK_AES_128_GCM_SHA256: + cipher = EVP_aes_128_gcm(); + tag_len = EVP_GCM_TLS_TAG_LEN; + break; + case TLS1_CK_AES_256_GCM_SHA384: + cipher = EVP_aes_256_gcm(); + tag_len = EVP_GCM_TLS_TAG_LEN; + break; + case TLS1_CK_CHACHA20_POLY1305_SHA256: + // cipher = EVP_chacha20_poly1305(); + cipher = nullptr; + tag_len = 16; + break; + default: ink_assert(false); - return nullptr; } + } else { + ink_assert(false); } + + this->_pp_key_info.set_cipher(cipher, tag_len); } -size_t -QUICTLS::_get_aead_tag_len(QUICKeyPhase phase) const +void +QUICTLS::_store_negotiated_cipher_for_hp() { - if (phase == QUICKeyPhase::INITIAL) { - return EVP_GCM_TLS_TAG_LEN; - } else { - const SSL_CIPHER *cipher = SSL_get_current_cipher(this->_ssl); - if (cipher) { - switch (SSL_CIPHER_get_id(cipher)) { - case TLS1_CK_AES_128_GCM_SHA256: - case TLS1_CK_AES_256_GCM_SHA384: - return EVP_GCM_TLS_TAG_LEN; - case TLS1_CK_CHACHA20_POLY1305_SHA256: - return 16; - default: - ink_assert(false); - return -1; - } - } else { + ink_assert(this->_ssl); + + const EVP_CIPHER *cipher_for_hp = nullptr; + const SSL_CIPHER *ssl_cipher = SSL_get_current_cipher(this->_ssl); + + if (ssl_cipher) { + switch (SSL_CIPHER_get_id(ssl_cipher)) { + case TLS1_CK_AES_128_GCM_SHA256: + cipher_for_hp = EVP_aes_128_ecb(); + break; + case TLS1_CK_AES_256_GCM_SHA384: + cipher_for_hp = EVP_aes_256_ecb(); + break; + case TLS1_CK_CHACHA20_POLY1305_SHA256: + // cipher_for_hp = EVP_chacha20(); + cipher_for_hp = nullptr; + break; + default: ink_assert(false); - return -1; + break; } + } else { + ink_assert(false); } + + this->_pp_key_info.set_cipher_for_hp(cipher_for_hp); } -const EVP_MD * -QUICKeyGenerator::_get_handshake_digest() +int +QUICTLS::_read_early_data() { - // TODO not implemented - return nullptr; + // This is for Hacked OpenSSL. Do nothing here. + return 1; } -bool -QUICTLS::_encrypt(uint8_t *cipher, size_t &cipher_len, size_t max_cipher_len, const uint8_t *plain, size_t plain_len, - uint64_t pkt_num, const uint8_t *ad, size_t ad_len, const KeyMaterial &km, const EVP_AEAD *aead, - size_t tag_len) const +int +QUICTLS::_write_early_data() { - uint8_t nonce[EVP_MAX_IV_LENGTH] = {0}; - size_t nonce_len = 0; - _gen_nonce(nonce, nonce_len, pkt_num, km.iv, km.iv_len); - - EVP_AEAD_CTX *aead_ctx = EVP_AEAD_CTX_new(aead, km.key, km.key_len, tag_len); - if (!aead_ctx) { - Debug(tag, "Failed to create EVP_AEAD_CTX"); - return false; - } - - if (!EVP_AEAD_CTX_seal(aead_ctx, cipher, &cipher_len, max_cipher_len, nonce, nonce_len, plain, plain_len, ad, ad_len)) { - Debug(tag, "Failed to encrypt"); - return false; - } - - EVP_AEAD_CTX_free(aead_ctx); - - return true; + // This is for Hacked OpenSSL. Do nothing here. + return 1; } -bool -QUICTLS::_decrypt(uint8_t *plain, size_t &plain_len, size_t max_plain_len, const uint8_t *cipher, size_t cipher_len, - uint64_t pkt_num, const uint8_t *ad, size_t ad_len, const KeyMaterial &km, const EVP_AEAD *aead, - size_t tag_len) const +void +QUICTLS::_pass_quic_data_to_ssl_impl(const QUICHandshakeMsgs &in) { - uint8_t nonce[EVP_MAX_IV_LENGTH] = {0}; - size_t nonce_len = 0; - _gen_nonce(nonce, nonce_len, pkt_num, km.iv, km.iv_len); - - EVP_AEAD_CTX *aead_ctx = EVP_AEAD_CTX_new(aead, km.key, km.key_len, tag_len); - if (!aead_ctx) { - Debug(tag, "Failed to create EVP_AEAD_CTX"); - return false; + for (auto level : QUIC_ENCRYPTION_LEVELS) { + int index = static_cast(level); + ssl_encryption_level_t ossl_level; + switch (level) { + case QUICEncryptionLevel::INITIAL: + ossl_level = ssl_encryption_initial; + break; + case QUICEncryptionLevel::ZERO_RTT: + ossl_level = ssl_encryption_early_data; + break; + case QUICEncryptionLevel::HANDSHAKE: + ossl_level = ssl_encryption_handshake; + break; + case QUICEncryptionLevel::ONE_RTT: + ossl_level = ssl_encryption_application; + break; + default: + // Should not be happened + ossl_level = ssl_encryption_application; + break; + } + if (in.offsets[index + 1] - in.offsets[index]) { + int start = 0; + for (int i = 0; i < index; ++i) { + start += in.offsets[index]; + } + SSL_provide_quic_data(this->_ssl, ossl_level, in.buf + start, in.offsets[index + 1] - in.offsets[index]); + } } +} - if (!EVP_AEAD_CTX_open(aead_ctx, plain, &plain_len, max_plain_len, nonce, nonce_len, cipher, cipher_len, ad, ad_len)) { - Debug(tag, "Failed to decrypt"); - return false; +const EVP_MD * +QUICTLS::_get_handshake_digest() const +{ + switch (SSL_CIPHER_get_id(SSL_get_current_cipher(this->_ssl))) { + case TLS1_CK_AES_128_GCM_SHA256: + case TLS1_CK_CHACHA20_POLY1305_SHA256: + return EVP_sha256(); + case TLS1_CK_AES_256_GCM_SHA384: + return EVP_sha384(); + default: + ink_assert(false); + return nullptr; } - - EVP_AEAD_CTX_free(aead_ctx); - - return true; } -*/ diff --git a/iocore/net/quic/QUICTLS_legacy.cc b/iocore/net/quic/QUICTLS_legacy.cc new file mode 100644 index 00000000000..d9cc410b515 --- /dev/null +++ b/iocore/net/quic/QUICTLS_legacy.cc @@ -0,0 +1,445 @@ +/** @file + * + * QUIC Crypto (TLS to Secure QUIC) using OpenSSL + * + * @section license License + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "QUICTLS.h" + +#include +#include +#include +#include +#include + +#include "QUICConfig.h" +#include "QUICGlobals.h" +#include "QUICDebugNames.h" +#include "QUICPacketProtectionKeyInfo.h" + +static constexpr char tag[] = "quic_tls"; + +using namespace std::literals; + +static constexpr std::string_view QUIC_CLIENT_EARLY_TRAFFIC_SECRET_LABEL("QUIC_CLIENT_EARLY_TRAFFIC_SECRET"sv); +static constexpr std::string_view QUIC_CLIENT_HANDSHAKE_TRAFFIC_SECRET_LABEL("QUIC_CLIENT_HANDSHAKE_TRAFFIC_SECRET"sv); +static constexpr std::string_view QUIC_SERVER_HANDSHAKE_TRAFFIC_SECRET_LABEL("QUIC_SERVER_HANDSHAKE_TRAFFIC_SECRET"sv); +// TODO: support key update +static constexpr std::string_view QUIC_CLIENT_TRAFFIC_SECRET_LABEL("QUIC_CLIENT_TRAFFIC_SECRET_0"sv); +static constexpr std::string_view QUIC_SERVER_TRAFFIC_SECRET_LABEL("QUIC_SERVER_TRAFFIC_SECRET_0"sv); + +void +QUICTLS::_msg_cb(int write_p, int version, int content_type, const void *buf, size_t len, SSL *ssl, void *arg) +{ + // Debug for reading + if (write_p == 0) { + QUICTLS::_print_hs_message(content_type, buf, len); + return; + } + + if (!write_p || (content_type != SSL3_RT_HANDSHAKE && content_type != SSL3_RT_ALERT)) { + return; + } + + QUICTLS *qtls = static_cast(SSL_get_ex_data(ssl, QUIC::ssl_quic_tls_index)); + const uint8_t *data = reinterpret_cast(buf); + if (content_type == SSL3_RT_HANDSHAKE) { + if (version != TLS1_3_VERSION) { + return; + } + + QUICEncryptionLevel level = QUICTLS::get_encryption_level(data[0]); + qtls->on_handshake_data_generated(level, data, len); + qtls->set_ready_for_write(); + } else if (content_type == SSL3_RT_ALERT && data[0] == SSL3_AL_FATAL && len == 2) { + qtls->on_tls_alert(data[1]); + } + + return; +} + +/** + This is very inspired from writing keylog format of ngtcp2's examples + https://github.com/ngtcp2/ngtcp2/blob/894ed23c970d61eede74f69d9178090af63fdf70/examples/keylog.cc + */ +static void +log_secret(SSL *ssl, int name, const unsigned char *secret, size_t secretlen) +{ + if (auto keylog_cb = SSL_CTX_get_keylog_callback(SSL_get_SSL_CTX(ssl))) { + unsigned char crandom[32]; + if (SSL_get_client_random(ssl, crandom, sizeof(crandom)) != sizeof(crandom)) { + return; + } + uint8_t line[256] = {0}; + size_t len = 0; + switch (name) { + case SSL_KEY_CLIENT_EARLY_TRAFFIC: + memcpy(line, QUIC_CLIENT_EARLY_TRAFFIC_SECRET_LABEL.data(), QUIC_CLIENT_EARLY_TRAFFIC_SECRET_LABEL.size()); + len += QUIC_CLIENT_EARLY_TRAFFIC_SECRET_LABEL.size(); + break; + case SSL_KEY_CLIENT_HANDSHAKE_TRAFFIC: + memcpy(line, QUIC_CLIENT_HANDSHAKE_TRAFFIC_SECRET_LABEL.data(), QUIC_CLIENT_HANDSHAKE_TRAFFIC_SECRET_LABEL.size()); + len += QUIC_CLIENT_HANDSHAKE_TRAFFIC_SECRET_LABEL.size(); + break; + case SSL_KEY_SERVER_HANDSHAKE_TRAFFIC: + memcpy(line, QUIC_SERVER_HANDSHAKE_TRAFFIC_SECRET_LABEL.data(), QUIC_SERVER_HANDSHAKE_TRAFFIC_SECRET_LABEL.size()); + len += QUIC_SERVER_HANDSHAKE_TRAFFIC_SECRET_LABEL.size(); + break; + case SSL_KEY_CLIENT_APPLICATION_TRAFFIC: + memcpy(line, QUIC_CLIENT_TRAFFIC_SECRET_LABEL.data(), QUIC_CLIENT_TRAFFIC_SECRET_LABEL.size()); + len += QUIC_CLIENT_TRAFFIC_SECRET_LABEL.size(); + break; + case SSL_KEY_SERVER_APPLICATION_TRAFFIC: + memcpy(line, QUIC_SERVER_TRAFFIC_SECRET_LABEL.data(), QUIC_SERVER_TRAFFIC_SECRET_LABEL.size()); + len += QUIC_SERVER_TRAFFIC_SECRET_LABEL.size(); + break; + + default: + return; + } + + line[len] = ' '; + ++len; + QUICDebug::to_hex(line + len, crandom, sizeof(crandom)); + len += sizeof(crandom) * 2; + line[len] = ' '; + ++len; + QUICDebug::to_hex(line + len, secret, secretlen); + + keylog_cb(ssl, reinterpret_cast(line)); + } +} + +static int +key_cb(SSL *ssl, int name, const unsigned char *secret, size_t secret_len, void *arg) +{ + if (arg == nullptr) { + return 0; + } + + QUICTLS *qtls = reinterpret_cast(arg); + + qtls->update_negotiated_cipher(); + + QUICEncryptionLevel level; + switch (name) { + case SSL_KEY_CLIENT_EARLY_TRAFFIC: + Debug("vv_quic_crypto", "%s", QUIC_CLIENT_EARLY_TRAFFIC_SECRET_LABEL.data()); + level = QUICEncryptionLevel::ZERO_RTT; + if (SSL_is_server(ssl)) { + qtls->update_key_materials_for_read(level, secret, secret_len); + } else { + qtls->update_key_materials_for_write(level, secret, secret_len); + } + break; + case SSL_KEY_CLIENT_HANDSHAKE_TRAFFIC: + Debug("vv_quic_crypto", "%s", QUIC_CLIENT_HANDSHAKE_TRAFFIC_SECRET_LABEL.data()); + level = QUICEncryptionLevel::HANDSHAKE; + if (SSL_is_server(ssl)) { + qtls->update_key_materials_for_read(level, secret, secret_len); + } else { + qtls->update_key_materials_for_write(level, secret, secret_len); + } + break; + case SSL_KEY_SERVER_HANDSHAKE_TRAFFIC: + Debug("vv_quic_crypto", "%s", QUIC_SERVER_HANDSHAKE_TRAFFIC_SECRET_LABEL.data()); + level = QUICEncryptionLevel::HANDSHAKE; + if (SSL_is_server(ssl)) { + qtls->update_key_materials_for_write(level, secret, secret_len); + } else { + qtls->update_key_materials_for_read(level, secret, secret_len); + } + break; + case SSL_KEY_CLIENT_APPLICATION_TRAFFIC: + Debug("vv_quic_crypto", "%s", QUIC_CLIENT_TRAFFIC_SECRET_LABEL.data()); + level = QUICEncryptionLevel::ONE_RTT; + if (SSL_is_server(ssl)) { + qtls->update_key_materials_for_read(level, secret, secret_len); + } else { + qtls->update_key_materials_for_write(level, secret, secret_len); + } + break; + case SSL_KEY_SERVER_APPLICATION_TRAFFIC: + Debug("vv_quic_crypto", "%s", QUIC_SERVER_TRAFFIC_SECRET_LABEL.data()); + level = QUICEncryptionLevel::ONE_RTT; + if (SSL_is_server(ssl)) { + qtls->update_key_materials_for_write(level, secret, secret_len); + } else { + qtls->update_key_materials_for_read(level, secret, secret_len); + } + break; + default: + level = QUICEncryptionLevel::NONE; + break; + } + + log_secret(ssl, name, secret, secret_len); + + return 1; +} + +QUICTLS::QUICTLS(QUICPacketProtectionKeyInfo &pp_key_info, SSL_CTX *ssl_ctx, NetVConnectionContext_t nvc_ctx, + const NetVCOptions &netvc_options, const char *session_file, const char *keylog_file) + : QUICHandshakeProtocol(pp_key_info), + _session_file(session_file), + _keylog_file(keylog_file), + _ssl(SSL_new(ssl_ctx)), + _netvc_context(nvc_ctx) +{ + ink_assert(this->_netvc_context != NET_VCONNECTION_UNSET); + + if (this->_netvc_context == NET_VCONNECTION_OUT) { + SSL_set_connect_state(this->_ssl); + + SSL_set_alpn_protos(this->_ssl, reinterpret_cast(netvc_options.alpn_protos.data()), + netvc_options.alpn_protos.size()); + const ats_scoped_str &tlsext_host_name = netvc_options.sni_hostname ? netvc_options.sni_hostname : netvc_options.sni_servername; + SSL_set_tlsext_host_name(this->_ssl, tlsext_host_name.get()); + } else { + SSL_set_accept_state(this->_ssl); + } + + SSL_set_ex_data(this->_ssl, QUIC::ssl_quic_tls_index, this); + SSL_set_key_callback(this->_ssl, key_cb, this); + + if (session_file && this->_netvc_context == NET_VCONNECTION_OUT) { + auto file = BIO_new_file(session_file, "r"); + if (file == nullptr) { + Debug(tag, "Could not read tls session file %s", session_file); + return; + } + + auto session = PEM_read_bio_SSL_SESSION(file, nullptr, nullptr, nullptr); + if (session == nullptr) { + Debug(tag, "Could not read tls session file %s", session_file); + } else { + if (!SSL_set_session(this->_ssl, session)) { + Debug(tag, "Session resumption failed : %s", session_file); + } else { + Debug(tag, "Session resumption success : %s", session_file); + this->_is_session_reused = true; + } + SSL_SESSION_free(session); + } + + BIO_free(file); + } +} + +QUICEncryptionLevel +QUICTLS::get_encryption_level(int msg_type) +{ + switch (msg_type) { + case SSL3_MT_CLIENT_HELLO: + case SSL3_MT_SERVER_HELLO: + return QUICEncryptionLevel::INITIAL; + case SSL3_MT_END_OF_EARLY_DATA: + return QUICEncryptionLevel::ZERO_RTT; + case SSL3_MT_ENCRYPTED_EXTENSIONS: + case SSL3_MT_CERTIFICATE_REQUEST: + case SSL3_MT_CERTIFICATE: + case SSL3_MT_CERTIFICATE_VERIFY: + case SSL3_MT_FINISHED: + return QUICEncryptionLevel::HANDSHAKE; + case SSL3_MT_KEY_UPDATE: + case SSL3_MT_NEWSESSION_TICKET: + return QUICEncryptionLevel::ONE_RTT; + default: + return QUICEncryptionLevel::NONE; + } +} + +void +QUICTLS::set_local_transport_parameters(std::shared_ptr tp) +{ + this->_local_transport_parameters = tp; +} + +int +QUICTLS::_process_post_handshake_messages(QUICHandshakeMsgs *out, const QUICHandshakeMsgs *in) +{ + ink_assert(this->_ssl != nullptr); + + int err = SSL_ERROR_NONE; + ERR_clear_error(); + int ret = 0; + + SSL_set_msg_callback(this->_ssl, QUICTLS::_msg_cb); + SSL_set_msg_callback_arg(this->_ssl, out); + + this->_pass_quic_data_to_ssl_impl(*in); + + uint8_t data[2048]; + size_t l = 0; + ret = SSL_read_ex(this->_ssl, data, 2048, &l); + + if (ret <= 0) { + err = SSL_get_error(this->_ssl, ret); + + switch (err) { + case SSL_ERROR_WANT_READ: + case SSL_ERROR_WANT_WRITE: + break; + default: + char err_buf[256] = {0}; + ERR_error_string_n(ERR_get_error(), err_buf, sizeof(err_buf)); + Debug(tag, "Handshake: %s", err_buf); + return ret; + } + } + + return 1; +} + +void +QUICTLS::_store_negotiated_cipher() +{ + ink_assert(this->_ssl); + + const QUIC_EVP_CIPHER *cipher = nullptr; + size_t tag_len = 0; + const SSL_CIPHER *ssl_cipher = SSL_get_current_cipher(this->_ssl); + + if (ssl_cipher) { + switch (SSL_CIPHER_get_id(ssl_cipher)) { + case TLS1_3_CK_AES_128_GCM_SHA256: + cipher = EVP_aes_128_gcm(); + tag_len = EVP_GCM_TLS_TAG_LEN; + break; + case TLS1_3_CK_AES_256_GCM_SHA384: + cipher = EVP_aes_256_gcm(); + tag_len = EVP_GCM_TLS_TAG_LEN; + break; + case TLS1_3_CK_CHACHA20_POLY1305_SHA256: + cipher = EVP_chacha20_poly1305(); + tag_len = EVP_CHACHAPOLY_TLS_TAG_LEN; + break; + case TLS1_3_CK_AES_128_CCM_SHA256: + cipher = EVP_aes_128_ccm(); + tag_len = EVP_GCM_TLS_TAG_LEN; + break; + case TLS1_3_CK_AES_128_CCM_8_SHA256: + cipher = EVP_aes_128_ccm(); + tag_len = EVP_CCM8_TLS_TAG_LEN; + break; + default: + ink_assert(false); + } + } else { + ink_assert(false); + } + + this->_pp_key_info.set_cipher(cipher, tag_len); +} + +void +QUICTLS::_store_negotiated_cipher_for_hp() +{ + ink_assert(this->_ssl); + + const QUIC_EVP_CIPHER *cipher_for_hp = nullptr; + const SSL_CIPHER *ssl_cipher = SSL_get_current_cipher(this->_ssl); + + if (ssl_cipher) { + switch (SSL_CIPHER_get_id(ssl_cipher)) { + case TLS1_3_CK_AES_128_GCM_SHA256: + cipher_for_hp = EVP_aes_128_ecb(); + break; + case TLS1_3_CK_AES_256_GCM_SHA384: + cipher_for_hp = EVP_aes_256_ecb(); + break; + case TLS1_3_CK_CHACHA20_POLY1305_SHA256: + cipher_for_hp = EVP_chacha20(); + break; + case TLS1_3_CK_AES_128_CCM_SHA256: + case TLS1_3_CK_AES_128_CCM_8_SHA256: + cipher_for_hp = EVP_aes_128_ecb(); + break; + default: + ink_assert(false); + break; + } + } else { + ink_assert(false); + } + + this->_pp_key_info.set_cipher_for_hp(cipher_for_hp); +} + +int +QUICTLS::_read_early_data() +{ + uint8_t early_data[8]; + size_t early_data_len = 0; + + // Early data within the TLS connection MUST NOT be used. As it is for other TLS application data, a server MUST treat receiving + // early data on the TLS connection as a connection error of type PROTOCOL_VIOLATION. + int ret = SSL_read_early_data(this->_ssl, early_data, sizeof(early_data), &early_data_len); + // error or reading empty data return 1, otherwise return 0. + if (early_data_len != 0) { + return -1; + } + if (ret == SSL_READ_EARLY_DATA_FINISH) { + return 0; + } else { + return 1; + } +} + +int +QUICTLS::_write_early_data() +{ + size_t early_data_len = 0; + + // Early data within the TLS connection MUST NOT be used. As it is for other TLS application data, a server MUST treat receiving + // early data on the TLS connection as a connection error of type PROTOCOL_VIOLATION. + SSL_write_early_data(this->_ssl, "", 0, &early_data_len); + // always return 1 + return 1; +} + +void +QUICTLS::_pass_quic_data_to_ssl_impl(const QUICHandshakeMsgs &in) +{ + // TODO: set BIO_METHOD which read from QUICHandshakeMsgs directly + BIO *rbio = BIO_new(BIO_s_mem()); + // TODO: set dummy BIO_METHOD which do nothing + BIO *wbio = BIO_new(BIO_s_mem()); + if (in.offsets[4] != 0) { + BIO_write(rbio, in.buf, in.offsets[4]); + } + SSL_set_bio(this->_ssl, rbio, wbio); +} + +const EVP_MD * +QUICTLS::_get_handshake_digest() const +{ + switch (SSL_CIPHER_get_id(SSL_get_current_cipher(this->_ssl))) { + case TLS1_3_CK_AES_128_GCM_SHA256: + case TLS1_3_CK_CHACHA20_POLY1305_SHA256: + case TLS1_3_CK_AES_128_CCM_SHA256: + case TLS1_3_CK_AES_128_CCM_8_SHA256: + return EVP_sha256(); + case TLS1_3_CK_AES_256_GCM_SHA384: + return EVP_sha384(); + default: + ink_assert(false); + return nullptr; + } +} diff --git a/iocore/net/quic/QUICTLS_openssl.cc b/iocore/net/quic/QUICTLS_openssl.cc index 0eabec13a23..c34598dd3f6 100644 --- a/iocore/net/quic/QUICTLS_openssl.cc +++ b/iocore/net/quic/QUICTLS_openssl.cc @@ -28,366 +28,100 @@ #include #include -#include "QUICConfig.h" #include "QUICGlobals.h" -#include "QUICDebugNames.h" +#include "QUICConnection.h" #include "QUICPacketProtectionKeyInfo.h" static constexpr char tag[] = "quic_tls"; -using namespace std::literals; - -static constexpr std::string_view QUIC_CLIENT_EARLY_TRAFFIC_SECRET_LABEL("QUIC_CLIENT_EARLY_TRAFFIC_SECRET"sv); -static constexpr std::string_view QUIC_CLIENT_HANDSHAKE_TRAFFIC_SECRET_LABEL("QUIC_CLIENT_HANDSHAKE_TRAFFIC_SECRET"sv); -static constexpr std::string_view QUIC_SERVER_HANDSHAKE_TRAFFIC_SECRET_LABEL("QUIC_SERVER_HANDSHAKE_TRAFFIC_SECRET"sv); -// TODO: support key update -static constexpr std::string_view QUIC_CLIENT_TRAFFIC_SECRET_LABEL("QUIC_CLIENT_TRAFFIC_SECRET_0"sv); -static constexpr std::string_view QUIC_SERVER_TRAFFIC_SECRET_LABEL("QUIC_SERVER_TRAFFIC_SECRET_0"sv); - -static const char * -content_type_str(int type) +static QUICEncryptionLevel +convert_level_ats2ssl(enum ssl_encryption_level_t level) { - switch (type) { - case SSL3_RT_CHANGE_CIPHER_SPEC: - return "CHANGE_CIPHER_SPEC"; - case SSL3_RT_ALERT: - return "ALERT"; - case SSL3_RT_HANDSHAKE: - return "HANDSHAKE"; - case SSL3_RT_APPLICATION_DATA: - return "APPLICATION_DATA"; - case SSL3_RT_HEADER: - // The buf contains the record header bytes only - return "HEADER"; - case SSL3_RT_INNER_CONTENT_TYPE: - // Used when an encrypted TLSv1.3 record is sent or received. In encrypted TLSv1.3 records the content type in the record header - // is always SSL3_RT_APPLICATION_DATA. The real content type for the record is contained in an "inner" content type. buf - // contains the encoded "inner" content type byte. - return "INNER_CONTENT_TYPE"; - default: - return "UNKNOWN"; - } -} - -static const char * -hs_type_str(int type) -{ - switch (type) { - case SSL3_MT_CLIENT_HELLO: - return "CLIENT_HELLO"; - case SSL3_MT_SERVER_HELLO: - return "SERVER_HELLO"; - case SSL3_MT_NEWSESSION_TICKET: - return "NEWSESSION_TICKET"; - case SSL3_MT_END_OF_EARLY_DATA: - return "END_OF_EARLY_DATA"; - case SSL3_MT_ENCRYPTED_EXTENSIONS: - return "ENCRYPTED_EXTENSIONS"; - case SSL3_MT_CERTIFICATE: - return "CERTIFICATE"; - case SSL3_MT_CERTIFICATE_VERIFY: - return "CERTIFICATE_VERIFY"; - case SSL3_MT_FINISHED: - return "FINISHED"; - case SSL3_MT_KEY_UPDATE: - return "KEY_UPDATE"; - case SSL3_MT_MESSAGE_HASH: - return "MESSAGE_HASH"; + switch (level) { + case ssl_encryption_initial: + return QUICEncryptionLevel::INITIAL; + case ssl_encryption_early_data: + return QUICEncryptionLevel::ZERO_RTT; + case ssl_encryption_handshake: + return QUICEncryptionLevel::HANDSHAKE; + case ssl_encryption_application: + return QUICEncryptionLevel::ONE_RTT; default: - return "UNKNOWN"; + return QUICEncryptionLevel::NONE; } } -static void -msg_cb(int write_p, int version, int content_type, const void *buf, size_t len, SSL *ssl, void *arg) +static int +set_encryption_secrets(SSL *ssl, enum ssl_encryption_level_t level, const uint8_t *read_secret, const uint8_t *write_secret, + size_t secret_len) { - // Debug for reading - if (write_p == 0 && (content_type == SSL3_RT_HANDSHAKE || content_type == SSL3_RT_ALERT)) { - const uint8_t *tmp = reinterpret_cast(buf); - int msg_type = tmp[0]; + QUICTLS *qtls = static_cast(SSL_get_ex_data(ssl, QUIC::ssl_quic_tls_index)); - Debug(tag, "%s (%d), %s (%d) len=%zu", content_type_str(content_type), content_type, hs_type_str(msg_type), msg_type, len); - return; - } + qtls->update_negotiated_cipher(); - if (!write_p || !arg || (content_type != SSL3_RT_HANDSHAKE && content_type != SSL3_RT_ALERT)) { - return; + QUICEncryptionLevel ats_level = convert_level_ats2ssl(level); + if (read_secret) { + qtls->update_key_materials_for_read(ats_level, read_secret, secret_len); } - - QUICHandshakeMsgs *msg = reinterpret_cast(arg); - if (msg == nullptr) { - return; + if (write_secret) { + qtls->update_key_materials_for_write(ats_level, write_secret, secret_len); } - const uint8_t *msg_buf = reinterpret_cast(buf); - - if (content_type == SSL3_RT_HANDSHAKE) { - if (version != TLS1_3_VERSION) { - return; - } - - int msg_type = msg_buf[0]; - - QUICEncryptionLevel level = QUICTLS::get_encryption_level(msg_type); - int index = static_cast(level); - int next_index = index + 1; - - size_t offset = msg->offsets[next_index]; - size_t next_level_offset = offset + len; - - memcpy(msg->buf + offset, buf, len); - - for (int i = next_index; i < 5; ++i) { - msg->offsets[i] = next_level_offset; + if (ats_level == QUICEncryptionLevel::ONE_RTT) { + // FIXME Where should this be placed? + const uint8_t *tp_buf; + size_t tp_buf_len; + SSL_get_peer_quic_transport_params(ssl, &tp_buf, &tp_buf_len); + const QUICConnection *qc = static_cast(SSL_get_ex_data(ssl, QUIC::ssl_quic_qc_index)); + QUICVersion version = qc->negotiated_version(); + if (SSL_is_server(ssl)) { + qtls->set_remote_transport_parameters(std::make_shared(tp_buf, tp_buf_len, version)); + } else { + qtls->set_remote_transport_parameters( + std::make_shared(tp_buf, tp_buf_len, version)); } - } else if (content_type == SSL3_RT_ALERT && msg_buf[0] == SSL3_AL_FATAL && len == 2) { - msg->error_code = QUICTLS::convert_to_quic_trans_error_code(msg_buf[1]); } - return; + return 1; } -/** - This is very inspired from writting keylog format of ngtcp2's examples - https://github.com/ngtcp2/ngtcp2/blob/894ed23c970d61eede74f69d9178090af63fdf70/examples/keylog.cc - */ -static void -log_secret(SSL *ssl, int name, const unsigned char *secret, size_t secretlen) +static int +add_handshake_data(SSL *ssl, enum ssl_encryption_level_t level, const uint8_t *data, size_t len) { - if (auto keylog_cb = SSL_CTX_get_keylog_callback(SSL_get_SSL_CTX(ssl))) { - unsigned char crandom[32]; - if (SSL_get_client_random(ssl, crandom, sizeof(crandom)) != sizeof(crandom)) { - return; - } - uint8_t line[256] = {0}; - size_t len = 0; - switch (name) { - case SSL_KEY_CLIENT_EARLY_TRAFFIC: - memcpy(line, QUIC_CLIENT_EARLY_TRAFFIC_SECRET_LABEL.data(), QUIC_CLIENT_EARLY_TRAFFIC_SECRET_LABEL.size()); - len += QUIC_CLIENT_EARLY_TRAFFIC_SECRET_LABEL.size(); - break; - case SSL_KEY_CLIENT_HANDSHAKE_TRAFFIC: - memcpy(line, QUIC_CLIENT_HANDSHAKE_TRAFFIC_SECRET_LABEL.data(), QUIC_CLIENT_HANDSHAKE_TRAFFIC_SECRET_LABEL.size()); - len += QUIC_CLIENT_HANDSHAKE_TRAFFIC_SECRET_LABEL.size(); - break; - case SSL_KEY_SERVER_HANDSHAKE_TRAFFIC: - memcpy(line, QUIC_SERVER_HANDSHAKE_TRAFFIC_SECRET_LABEL.data(), QUIC_SERVER_HANDSHAKE_TRAFFIC_SECRET_LABEL.size()); - len += QUIC_SERVER_HANDSHAKE_TRAFFIC_SECRET_LABEL.size(); - break; - case SSL_KEY_CLIENT_APPLICATION_TRAFFIC: - memcpy(line, QUIC_CLIENT_TRAFFIC_SECRET_LABEL.data(), QUIC_CLIENT_TRAFFIC_SECRET_LABEL.size()); - len += QUIC_CLIENT_TRAFFIC_SECRET_LABEL.size(); - break; - case SSL_KEY_SERVER_APPLICATION_TRAFFIC: - memcpy(line, QUIC_SERVER_TRAFFIC_SECRET_LABEL.data(), QUIC_SERVER_TRAFFIC_SECRET_LABEL.size()); - len += QUIC_SERVER_TRAFFIC_SECRET_LABEL.size(); - break; - - default: - return; - } + QUICEncryptionLevel ats_level = convert_level_ats2ssl(level); - line[len] = ' '; - ++len; - QUICDebug::to_hex(line + len, crandom, sizeof(crandom)); - len += sizeof(crandom) * 2; - line[len] = ' '; - ++len; - QUICDebug::to_hex(line + len, secret, secretlen); + QUICTLS *qtls = static_cast(SSL_get_ex_data(ssl, QUIC::ssl_quic_tls_index)); + qtls->on_handshake_data_generated(ats_level, data, len); - keylog_cb(ssl, reinterpret_cast(line)); - } + return 1; } static int -key_cb(SSL *ssl, int name, const unsigned char *secret, size_t secret_len, void *arg) +flush_flight(SSL *ssl) { - if (arg == nullptr) { - return 0; - } - - QUICTLS *qtls = reinterpret_cast(arg); - qtls->update_key_materials_on_key_cb(name, secret, secret_len); - - log_secret(ssl, name, secret, secret_len); + QUICTLS *qtls = static_cast(SSL_get_ex_data(ssl, QUIC::ssl_quic_tls_index)); + qtls->set_ready_for_write(); return 1; } -void -QUICTLS::update_key_materials_on_key_cb(int name, const uint8_t *secret, size_t secret_len) +static int +send_alert(SSL *ssl, enum ssl_encryption_level_t level, uint8_t alert) { - if (is_debug_tag_set("vv_quic_crypto")) { - switch (name) { - case SSL_KEY_CLIENT_EARLY_TRAFFIC: - Debug("vv_quic_crypto", "%s", QUIC_CLIENT_EARLY_TRAFFIC_SECRET_LABEL.data()); - break; - case SSL_KEY_CLIENT_HANDSHAKE_TRAFFIC: - Debug("vv_quic_crypto", "%s", QUIC_CLIENT_HANDSHAKE_TRAFFIC_SECRET_LABEL.data()); - break; - case SSL_KEY_SERVER_HANDSHAKE_TRAFFIC: - Debug("vv_quic_crypto", "%s", QUIC_SERVER_HANDSHAKE_TRAFFIC_SECRET_LABEL.data()); - break; - case SSL_KEY_CLIENT_APPLICATION_TRAFFIC: - Debug("vv_quic_crypto", "%s", QUIC_CLIENT_TRAFFIC_SECRET_LABEL.data()); - break; - case SSL_KEY_SERVER_APPLICATION_TRAFFIC: - Debug("vv_quic_crypto", "%s", QUIC_SERVER_TRAFFIC_SECRET_LABEL.data()); - break; - default: - break; - } - } + QUICTLS *qtls = static_cast(SSL_get_ex_data(ssl, QUIC::ssl_quic_tls_index)); + qtls->on_tls_alert(alert); + return 1; +} - if (this->_state == HandshakeState::ABORTED) { - return; - } +static const SSL_QUIC_METHOD quic_method = {set_encryption_secrets, add_handshake_data, flush_flight, send_alert}; - QUICKeyPhase phase; - const QUIC_EVP_CIPHER *cipher; - QUICHKDF hkdf(this->_get_handshake_digest()); - - this->_store_negotiated_cipher(); - this->_store_negotiated_cipher_for_hp(); - - uint8_t *key_for_hp; - uint8_t *key; - uint8_t *iv; - size_t key_for_hp_len; - size_t key_len; - size_t *iv_len; - - switch (name) { - case SSL_KEY_CLIENT_EARLY_TRAFFIC: - // this->_update_encryption_level(QUICEncryptionLevel::ZERO_RTT); - phase = QUICKeyPhase::ZERO_RTT; - cipher = this->_pp_key_info.get_cipher(phase); - if (this->_netvc_context == NET_VCONNECTION_IN) { - key_for_hp = this->_pp_key_info.decryption_key_for_hp(phase); - key_for_hp_len = this->_pp_key_info.decryption_key_for_hp_len(phase); - key = this->_pp_key_info.decryption_key(phase); - key_len = this->_pp_key_info.decryption_key_len(phase); - iv = this->_pp_key_info.decryption_iv(phase); - iv_len = this->_pp_key_info.decryption_iv_len(phase); - this->_keygen_for_client.regenerate(key_for_hp, key, iv, iv_len, secret, secret_len, cipher, hkdf); - this->_pp_key_info.set_decryption_key_available(phase); - } else { - key_for_hp = this->_pp_key_info.encryption_key_for_hp(phase); - key_for_hp_len = this->_pp_key_info.encryption_key_for_hp_len(phase); - key = this->_pp_key_info.encryption_key(phase); - key_len = this->_pp_key_info.encryption_key_len(phase); - iv = this->_pp_key_info.encryption_iv(phase); - iv_len = this->_pp_key_info.encryption_iv_len(phase); - this->_keygen_for_client.regenerate(key_for_hp, key, iv, iv_len, secret, secret_len, cipher, hkdf); - this->_pp_key_info.set_encryption_key_available(phase); - } - this->_print_km("update - client - 0rtt", key_for_hp, key_for_hp_len, key, key_len, iv, *iv_len, secret, secret_len); - break; - case SSL_KEY_CLIENT_HANDSHAKE_TRAFFIC: - this->_update_encryption_level(QUICEncryptionLevel::HANDSHAKE); - phase = QUICKeyPhase::HANDSHAKE; - cipher = this->_pp_key_info.get_cipher(phase); - if (this->_netvc_context == NET_VCONNECTION_IN) { - key_for_hp = this->_pp_key_info.decryption_key_for_hp(phase); - key_for_hp_len = this->_pp_key_info.decryption_key_for_hp_len(phase); - key = this->_pp_key_info.decryption_key(phase); - key_len = this->_pp_key_info.decryption_key_len(phase); - iv = this->_pp_key_info.decryption_iv(phase); - iv_len = this->_pp_key_info.decryption_iv_len(phase); - this->_keygen_for_client.regenerate(key_for_hp, key, iv, iv_len, secret, secret_len, cipher, hkdf); - this->_pp_key_info.set_decryption_key_available(phase); - } else { - key_for_hp = this->_pp_key_info.encryption_key_for_hp(phase); - key_for_hp_len = this->_pp_key_info.encryption_key_for_hp_len(phase); - key = this->_pp_key_info.encryption_key(phase); - key_len = this->_pp_key_info.encryption_key_len(phase); - iv = this->_pp_key_info.encryption_iv(phase); - iv_len = this->_pp_key_info.encryption_iv_len(phase); - this->_keygen_for_client.regenerate(key_for_hp, key, iv, iv_len, secret, secret_len, cipher, hkdf); - this->_pp_key_info.set_encryption_key_available(phase); - } - this->_print_km("update - client - handshake", key_for_hp, key_for_hp_len, key, key_len, iv, *iv_len, secret, secret_len); - break; - case SSL_KEY_CLIENT_APPLICATION_TRAFFIC: - this->_update_encryption_level(QUICEncryptionLevel::ONE_RTT); - phase = QUICKeyPhase::PHASE_0; - cipher = this->_pp_key_info.get_cipher(phase); - if (this->_netvc_context == NET_VCONNECTION_IN) { - key_for_hp = this->_pp_key_info.decryption_key_for_hp(phase); - key_for_hp_len = this->_pp_key_info.decryption_key_for_hp_len(phase); - key = this->_pp_key_info.decryption_key(phase); - key_len = this->_pp_key_info.decryption_key_len(phase); - iv = this->_pp_key_info.decryption_iv(phase); - iv_len = this->_pp_key_info.decryption_iv_len(phase); - this->_keygen_for_client.regenerate(key_for_hp, key, iv, iv_len, secret, secret_len, cipher, hkdf); - this->_pp_key_info.set_decryption_key_available(phase); - } else { - key_for_hp = this->_pp_key_info.encryption_key_for_hp(phase); - key_for_hp_len = this->_pp_key_info.encryption_key_for_hp_len(phase); - key = this->_pp_key_info.encryption_key(phase); - key_len = this->_pp_key_info.encryption_key_len(phase); - iv = this->_pp_key_info.encryption_iv(phase); - iv_len = this->_pp_key_info.encryption_iv_len(phase); - this->_keygen_for_client.regenerate(key_for_hp, key, iv, iv_len, secret, secret_len, cipher, hkdf); - this->_pp_key_info.set_encryption_key_available(phase); - } - this->_print_km("update - client - 1rtt", key_for_hp, key_for_hp_len, key, key_len, iv, *iv_len, secret, secret_len); - break; - case SSL_KEY_SERVER_HANDSHAKE_TRAFFIC: - this->_update_encryption_level(QUICEncryptionLevel::HANDSHAKE); - phase = QUICKeyPhase::HANDSHAKE; - cipher = this->_pp_key_info.get_cipher(phase); - if (this->_netvc_context == NET_VCONNECTION_IN) { - key_for_hp = this->_pp_key_info.encryption_key_for_hp(phase); - key_for_hp_len = this->_pp_key_info.encryption_key_for_hp_len(phase); - key = this->_pp_key_info.encryption_key(phase); - key_len = this->_pp_key_info.encryption_key_len(phase); - iv = this->_pp_key_info.encryption_iv(phase); - iv_len = this->_pp_key_info.encryption_iv_len(phase); - this->_keygen_for_server.regenerate(key_for_hp, key, iv, iv_len, secret, secret_len, cipher, hkdf); - this->_pp_key_info.set_encryption_key_available(phase); - } else { - key_for_hp = this->_pp_key_info.decryption_key_for_hp(phase); - key_for_hp_len = this->_pp_key_info.decryption_key_for_hp_len(phase); - key = this->_pp_key_info.decryption_key(phase); - key_len = this->_pp_key_info.decryption_key_len(phase); - iv = this->_pp_key_info.decryption_iv(phase); - iv_len = this->_pp_key_info.decryption_iv_len(phase); - this->_keygen_for_server.regenerate(key_for_hp, key, iv, iv_len, secret, secret_len, cipher, hkdf); - this->_pp_key_info.set_decryption_key_available(phase); - } - this->_print_km("update - server - handshake", key_for_hp, key_for_hp_len, key, key_len, iv, *iv_len, secret, secret_len); - break; - case SSL_KEY_SERVER_APPLICATION_TRAFFIC: - this->_update_encryption_level(QUICEncryptionLevel::ONE_RTT); - phase = QUICKeyPhase::PHASE_0; - cipher = this->_pp_key_info.get_cipher(phase); - if (this->_netvc_context == NET_VCONNECTION_IN) { - key_for_hp = this->_pp_key_info.encryption_key_for_hp(phase); - key_for_hp_len = this->_pp_key_info.encryption_key_for_hp_len(phase); - key = this->_pp_key_info.encryption_key(phase); - key_len = this->_pp_key_info.encryption_key_len(phase); - iv = this->_pp_key_info.encryption_iv(phase); - iv_len = this->_pp_key_info.encryption_iv_len(phase); - this->_keygen_for_server.regenerate(key_for_hp, key, iv, iv_len, secret, secret_len, cipher, hkdf); - this->_pp_key_info.set_encryption_key_available(phase); - } else { - key_for_hp = this->_pp_key_info.decryption_key_for_hp(phase); - key_for_hp_len = this->_pp_key_info.decryption_key_for_hp_len(phase); - key = this->_pp_key_info.decryption_key(phase); - key_len = this->_pp_key_info.decryption_key_len(phase); - iv = this->_pp_key_info.decryption_iv(phase); - iv_len = this->_pp_key_info.decryption_iv_len(phase); - this->_keygen_for_server.regenerate(key_for_hp, key, iv, iv_len, secret, secret_len, cipher, hkdf); - this->_pp_key_info.set_decryption_key_available(phase); - } - this->_print_km("update - server - 1rtt", key_for_hp, key_for_hp_len, key, key_len, iv, *iv_len, secret, secret_len); - break; - default: - break; +void +QUICTLS::_msg_cb(int write_p, int version, int content_type, const void *buf, size_t len, SSL *ssl, void *arg) +{ + // Debug for reading + if (write_p == 0) { + QUICTLS::_print_hs_message(content_type, buf, len); } - - return; } QUICTLS::QUICTLS(QUICPacketProtectionKeyInfo &pp_key_info, SSL_CTX *ssl_ctx, NetVConnectionContext_t nvc_ctx, @@ -405,13 +139,17 @@ QUICTLS::QUICTLS(QUICPacketProtectionKeyInfo &pp_key_info, SSL_CTX *ssl_ctx, Net SSL_set_alpn_protos(this->_ssl, reinterpret_cast(netvc_options.alpn_protos.data()), netvc_options.alpn_protos.size()); - SSL_set_tlsext_host_name(this->_ssl, netvc_options.sni_servername.get()); + const ats_scoped_str &tlsext_host_name = netvc_options.sni_hostname ? netvc_options.sni_hostname : netvc_options.sni_servername; + SSL_set_tlsext_host_name(this->_ssl, tlsext_host_name.get()); } else { SSL_set_accept_state(this->_ssl); } SSL_set_ex_data(this->_ssl, QUIC::ssl_quic_tls_index, this); - SSL_set_key_callback(this->_ssl, key_cb, this); + SSL_set_quic_method(this->_ssl, &quic_method); +#ifdef HAVE_SSL_SET_QUIC_EARLY_DATA_ENABLED + SSL_set_quic_early_data_enabled(this->_ssl, 1); +#endif if (session_file && this->_netvc_context == NET_VCONNECTION_OUT) { auto file = BIO_new_file(session_file, "r"); @@ -437,147 +175,22 @@ QUICTLS::QUICTLS(QUICPacketProtectionKeyInfo &pp_key_info, SSL_CTX *ssl_ctx, Net } } -QUICEncryptionLevel -QUICTLS::get_encryption_level(int msg_type) -{ - switch (msg_type) { - case SSL3_MT_CLIENT_HELLO: - case SSL3_MT_SERVER_HELLO: - return QUICEncryptionLevel::INITIAL; - case SSL3_MT_END_OF_EARLY_DATA: - return QUICEncryptionLevel::ZERO_RTT; - case SSL3_MT_ENCRYPTED_EXTENSIONS: - case SSL3_MT_CERTIFICATE_REQUEST: - case SSL3_MT_CERTIFICATE: - case SSL3_MT_CERTIFICATE_VERIFY: - case SSL3_MT_FINISHED: - return QUICEncryptionLevel::HANDSHAKE; - case SSL3_MT_KEY_UPDATE: - case SSL3_MT_NEWSESSION_TICKET: - return QUICEncryptionLevel::ONE_RTT; - default: - return QUICEncryptionLevel::NONE; - } -} - -int -QUICTLS::handshake(QUICHandshakeMsgs *out, const QUICHandshakeMsgs *in) -{ - if (this->is_handshake_finished()) { - if (in != nullptr && in->offsets[4] != 0) { - return this->_process_post_handshake_messages(out, in); - } - - return 0; - } - - return this->_handshake(out, in); -} - -int -QUICTLS::_handshake(QUICHandshakeMsgs *out, const QUICHandshakeMsgs *in) +void +QUICTLS::set_local_transport_parameters(std::shared_ptr tp) { - ink_assert(this->_ssl != nullptr); - if (this->_state == HandshakeState::ABORTED) { - return 0; - } - - int err = SSL_ERROR_NONE; - ERR_clear_error(); - int ret = 0; - - SSL_set_msg_callback(this->_ssl, msg_cb); - SSL_set_msg_callback_arg(this->_ssl, out); - - // TODO: set BIO_METHOD which read from QUICHandshakeMsgs directly - BIO *rbio = BIO_new(BIO_s_mem()); - // TODO: set dummy BIO_METHOD which do nothing - BIO *wbio = BIO_new(BIO_s_mem()); - if (in != nullptr && in->offsets[4] != 0) { - BIO_write(rbio, in->buf, in->offsets[4]); - } - SSL_set_bio(this->_ssl, rbio, wbio); - - if (this->_netvc_context == NET_VCONNECTION_IN) { - if (!this->_early_data_processed) { - if (this->_read_early_data() != 1) { - out->error_code = static_cast(QUICTransErrorCode::PROTOCOL_VIOLATION); - return 0; - } else { - this->_early_data_processed = true; - } - } - - ret = SSL_accept(this->_ssl); - } else { - if (!this->_early_data_processed) { - if (this->_write_early_data()) { - this->_early_data_processed = true; - } - } - - ret = SSL_connect(this->_ssl); - } + this->_local_transport_parameters = tp; - if (ret <= 0) { - err = SSL_get_error(this->_ssl, ret); - - switch (err) { - case SSL_ERROR_WANT_READ: - case SSL_ERROR_WANT_WRITE: - break; - default: - char err_buf[256] = {0}; - ERR_error_string_n(ERR_get_error(), err_buf, sizeof(err_buf)); - Debug(tag, "Handshake: %s", err_buf); - return ret; - } - } - - return 1; + uint8_t buf[UINT16_MAX]; + uint16_t len; + this->_local_transport_parameters->store(buf, &len); + SSL_set_quic_transport_params(this->_ssl, buf, len); } int QUICTLS::_process_post_handshake_messages(QUICHandshakeMsgs *out, const QUICHandshakeMsgs *in) { - ink_assert(this->_ssl != nullptr); - - int err = SSL_ERROR_NONE; - ERR_clear_error(); - int ret = 0; - - SSL_set_msg_callback(this->_ssl, msg_cb); - SSL_set_msg_callback_arg(this->_ssl, out); - - // TODO: set BIO_METHOD which read from QUICHandshakeMsgs directly - BIO *rbio = BIO_new(BIO_s_mem()); - // TODO: set dummy BIO_METHOD which do nothing - BIO *wbio = BIO_new(BIO_s_mem()); - if (in != nullptr && in->offsets[4] != 0) { - BIO_write(rbio, in->buf, in->offsets[4]); - } - SSL_set_bio(this->_ssl, rbio, wbio); - - uint8_t data[2048]; - size_t l = 0; - ret = SSL_read_ex(this->_ssl, data, 2048, &l); - - if (ret <= 0) { - err = SSL_get_error(this->_ssl, ret); - - switch (err) { - case SSL_ERROR_WANT_READ: - case SSL_ERROR_WANT_WRITE: - break; - default: - char err_buf[256] = {0}; - ERR_error_string_n(ERR_get_error(), err_buf, sizeof(err_buf)); - Debug(tag, "Handshake: %s", err_buf); - return ret; - } - } - - return 1; + this->_pass_quic_data_to_ssl_impl(*in); + return SSL_process_quic_post_handshake(this->_ssl); } void @@ -585,9 +198,9 @@ QUICTLS::_store_negotiated_cipher() { ink_assert(this->_ssl); - const QUIC_EVP_CIPHER *cipher = nullptr; - size_t tag_len = 0; - const SSL_CIPHER *ssl_cipher = SSL_get_current_cipher(this->_ssl); + const EVP_CIPHER *cipher = nullptr; + size_t tag_len = 0; + const SSL_CIPHER *ssl_cipher = SSL_get_current_cipher(this->_ssl); if (ssl_cipher) { switch (SSL_CIPHER_get_id(ssl_cipher)) { @@ -626,8 +239,8 @@ QUICTLS::_store_negotiated_cipher_for_hp() { ink_assert(this->_ssl); - const QUIC_EVP_CIPHER *cipher_for_hp = nullptr; - const SSL_CIPHER *ssl_cipher = SSL_get_current_cipher(this->_ssl); + const EVP_CIPHER *cipher_for_hp = nullptr; + const SSL_CIPHER *ssl_cipher = SSL_get_current_cipher(this->_ssl); if (ssl_cipher) { switch (SSL_CIPHER_get_id(ssl_cipher)) { @@ -658,28 +271,51 @@ QUICTLS::_store_negotiated_cipher_for_hp() int QUICTLS::_read_early_data() { - uint8_t early_data[8]; - size_t early_data_len = 0; - - // Early data within the TLS connection MUST NOT be used. As it is for other TLS application data, a server MUST treat receiving - // early data on the TLS connection as a connection error of type PROTOCOL_VIOLATION. - SSL_read_early_data(this->_ssl, early_data, sizeof(early_data), &early_data_len); - // error or reading empty data return 1, otherwise return 0. - return early_data_len != 0 ? 0 : 1; + // This is for Hacked OpenSSL. Do nothing here. + return 1; } int QUICTLS::_write_early_data() { - size_t early_data_len = 0; - - // Early data within the TLS connection MUST NOT be used. As it is for other TLS application data, a server MUST treat receiving - // early data on the TLS connection as a connection error of type PROTOCOL_VIOLATION. - SSL_write_early_data(this->_ssl, "", 0, &early_data_len); - // always return 1 + // This is for Hacked OpenSSL. Do nothing here. return 1; } +void +QUICTLS::_pass_quic_data_to_ssl_impl(const QUICHandshakeMsgs &in) +{ + for (auto level : QUIC_ENCRYPTION_LEVELS) { + int index = static_cast(level); + ssl_encryption_level_t ossl_level; + switch (level) { + case QUICEncryptionLevel::INITIAL: + ossl_level = ssl_encryption_initial; + break; + case QUICEncryptionLevel::ZERO_RTT: + ossl_level = ssl_encryption_early_data; + break; + case QUICEncryptionLevel::HANDSHAKE: + ossl_level = ssl_encryption_handshake; + break; + case QUICEncryptionLevel::ONE_RTT: + ossl_level = ssl_encryption_application; + break; + default: + // Should not be happened + ossl_level = ssl_encryption_application; + break; + } + if (in.offsets[index + 1] - in.offsets[index]) { + int start = 0; + for (int i = 0; i < index; ++i) { + start += in.offsets[index]; + } + SSL_provide_quic_data(this->_ssl, ossl_level, in.buf + start, in.offsets[index + 1] - in.offsets[index]); + } + } +} + const EVP_MD * QUICTLS::_get_handshake_digest() const { diff --git a/iocore/net/quic/QUICTokenCreator.cc b/iocore/net/quic/QUICTokenCreator.cc new file mode 100644 index 00000000000..9d1ef622b2f --- /dev/null +++ b/iocore/net/quic/QUICTokenCreator.cc @@ -0,0 +1,71 @@ +/** @file + * + * A brief file description + * + * @section license License + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "QUICTokenCreator.h" + +bool +QUICTokenCreator::will_generate_frame(QUICEncryptionLevel level, size_t current_packet_size, bool ack_eliciting, uint32_t seq_num) +{ + if (!this->_is_level_matched(level)) { + return false; + } + + return !this->_is_resumption_token_sent; +} + +QUICFrame * +QUICTokenCreator::generate_frame(uint8_t *buf, QUICEncryptionLevel level, uint64_t connection_credit, uint16_t maximum_frame_size, + size_t current_packet_size, uint32_t seq_num) +{ + QUICFrame *frame = nullptr; + + if (!this->_is_level_matched(level)) { + return frame; + } + + if (this->_is_resumption_token_sent) { + return frame; + } + + if (this->_context->connection_info()->direction() == NET_VCONNECTION_IN) { + // TODO Make expiration period configurable + QUICResumptionToken token(this->_context->connection_info()->five_tuple().source(), + this->_context->connection_info()->connection_id(), Thread::get_hrtime() + HRTIME_HOURS(24)); + frame = QUICFrameFactory::create_new_token_frame(buf, token, this->_issue_frame_id(), this); + if (frame) { + if (frame->size() < maximum_frame_size) { + this->_is_resumption_token_sent = true; + } else { + // Cancel generating frame + frame = nullptr; + } + } + } + + return frame; +} + +void +QUICTokenCreator::_on_frame_lost(QUICFrameInformationUPtr &info) +{ + this->_is_resumption_token_sent = false; +} diff --git a/iocore/net/quic/QUICTokenCreator.h b/iocore/net/quic/QUICTokenCreator.h new file mode 100644 index 00000000000..e107fb9d0f8 --- /dev/null +++ b/iocore/net/quic/QUICTokenCreator.h @@ -0,0 +1,42 @@ +/** @file + * + * A brief file description + * + * @section license License + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include "QUICFrameGenerator.h" +#include "QUICContext.h" + +class QUICTokenCreator final : public QUICFrameGenerator +{ +public: + QUICTokenCreator(QUICContext *context) : _context(context) {} + + bool will_generate_frame(QUICEncryptionLevel level, size_t current_packet_size, bool ack_eliciting, uint32_t seq_num) override; + QUICFrame *generate_frame(uint8_t *buf, QUICEncryptionLevel level, uint64_t connection_credit, uint16_t maximum_frame_size, + size_t current_packet_size, uint32_t seq_num) override; + +private: + void _on_frame_lost(QUICFrameInformationUPtr &info) override; + + bool _is_resumption_token_sent = false; + QUICContext *_context = nullptr; +}; diff --git a/iocore/net/quic/QUICTransportParameters.cc b/iocore/net/quic/QUICTransportParameters.cc index 04f0009930b..c11345818f2 100644 --- a/iocore/net/quic/QUICTransportParameters.cc +++ b/iocore/net/quic/QUICTransportParameters.cc @@ -27,16 +27,17 @@ #include "QUICIntUtil.h" #include "QUICTransportParameters.h" #include "QUICConnection.h" -#include "QUICHandshake.h" #include "QUICDebugNames.h" #include "QUICTLS.h" #include "QUICTypes.h" static constexpr char tag[] = "quic_handshake"; -static constexpr uint32_t TP_ERROR_LENGTH = 0x010000; -static constexpr uint32_t TP_ERROR_VALUE = 0x020000; -// static constexpr uint32_t TP_ERROR_MUST_EXIST = 0x030000; +static constexpr int TRANSPORT_PARAMETERS_MAXIMUM_SIZE = 65535; + +static constexpr uint32_t TP_ERROR_LENGTH = 0x010000; +static constexpr uint32_t TP_ERROR_VALUE = 0x020000; +static constexpr uint32_t TP_ERROR_MUST_EXIST = 0x030000; static constexpr uint32_t TP_ERROR_MUST_NOT_EXIST = 0x040000; QUICTransportParameters::Value::Value(const uint8_t *data, uint16_t len) : _len(len) @@ -69,6 +70,14 @@ QUICTransportParameters::Value::len() const return this->_len; } +QUICTransportParameters::QUICTransportParameters(const uint8_t *buf, size_t len, QUICVersion version) +{ + this->_load(buf, len, version); + if (is_debug_tag_set(tag)) { + this->_print(); + } +} + QUICTransportParameters::~QUICTransportParameters() { for (auto p : this->_parameters) { @@ -77,23 +86,20 @@ QUICTransportParameters::~QUICTransportParameters() } void -QUICTransportParameters::_load(const uint8_t *buf, size_t len) +QUICTransportParameters::_load(const uint8_t *buf, size_t len, QUICVersion version) { bool has_error = false; const uint8_t *p = buf; - - // Read size of parameters field - uint16_t nbytes = (p[0] << 8) + p[1]; - p += 2; + size_t l; + uint64_t param_id; + uint64_t param_len; // Read parameters - const uint8_t *end = p + nbytes; - while (p < end) { + while (len) { // Read ID - uint16_t id = 0; - if (end - p >= 2) { - id = (p[0] << 8) + p[1]; - p += 2; + if (!QUICVariableInt::decode(param_id, l, p, len)) { + len -= l; + p += l; } else { has_error = true; break; @@ -101,25 +107,25 @@ QUICTransportParameters::_load(const uint8_t *buf, size_t len) // Check duplication // An endpoint MUST treat receipt of duplicate transport parameters as a connection error of type TRANSPORT_PARAMETER_ERROR - if (this->_parameters.find(id) != this->_parameters.end()) { + if (this->_parameters.find(param_id) != this->_parameters.end()) { has_error = true; break; } // Read length of value - uint16_t len = 0; - if (end - p >= 2) { - len = (p[0] << 8) + p[1]; - p += 2; + if (!QUICVariableInt::decode(param_len, l, p, len)) { + len -= l; + p += l; } else { has_error = true; break; } // Store parameter - if (end - p >= len) { - this->_parameters.insert(std::make_pair(id, new Value(p, len))); - p += len; + if (len >= param_len) { + this->_parameters.insert(std::make_pair(param_id, new Value(p, param_len))); + len -= param_len; + p += param_len; } else { has_error = true; break; @@ -132,7 +138,7 @@ QUICTransportParameters::_load(const uint8_t *buf, size_t len) } // Validate parameters - int res = this->_validate_parameters(); + int res = this->_validate_parameters(version); if (res < 0) { Debug(tag, "Transport parameter is not valid (err=%d)", res); this->_valid = false; @@ -142,7 +148,7 @@ QUICTransportParameters::_load(const uint8_t *buf, size_t len) } int -QUICTransportParameters::_validate_parameters() const +QUICTransportParameters::_validate_parameters(QUICVersion version) const { decltype(this->_parameters)::const_iterator ite; @@ -158,12 +164,12 @@ QUICTransportParameters::_validate_parameters() const if ((ite = this->_parameters.find(QUICTransportParameterId::INITIAL_MAX_STREAMS_UNI)) != this->_parameters.end()) { } - if ((ite = this->_parameters.find(QUICTransportParameterId::IDLE_TIMEOUT)) != this->_parameters.end()) { + if ((ite = this->_parameters.find(QUICTransportParameterId::MAX_IDLE_TIMEOUT)) != this->_parameters.end()) { } - if ((ite = this->_parameters.find(QUICTransportParameterId::MAX_PACKET_SIZE)) != this->_parameters.end()) { + if ((ite = this->_parameters.find(QUICTransportParameterId::MAX_UDP_PAYLOAD_SIZE)) != this->_parameters.end()) { if (QUICIntUtil::read_nbytes_as_uint(ite->second->data(), ite->second->len()) < 1200) { - return -(TP_ERROR_VALUE | QUICTransportParameterId::MAX_PACKET_SIZE); + return -(TP_ERROR_VALUE | QUICTransportParameterId::MAX_UDP_PAYLOAD_SIZE); } } @@ -183,7 +189,7 @@ QUICTransportParameters::_validate_parameters() const if ((ite = this->_parameters.find(QUICTransportParameterId::INITIAL_MAX_STREAM_DATA_UNI)) != this->_parameters.end()) { } - if ((ite = this->_parameters.find(QUICTransportParameterId::DISABLE_MIGRATION)) != this->_parameters.end()) { + if ((ite = this->_parameters.find(QUICTransportParameterId::DISABLE_ACTIVE_MIGRATION)) != this->_parameters.end()) { } if ((ite = this->_parameters.find(QUICTransportParameterId::MAX_ACK_DELAY)) != this->_parameters.end()) { @@ -250,34 +256,24 @@ void QUICTransportParameters::store(uint8_t *buf, uint16_t *len) const { uint8_t *p = buf; + size_t l; - // Write QUIC versions - this->_store(p, len); - p += *len; - - // Write parameters - // XXX parameters_size will be written later - uint8_t *parameters_size = p; - p += sizeof(uint16_t); - + *len = 0; for (auto &it : this->_parameters) { // TODO Skip non-MUST parameters that have their default values - p[0] = (it.first & 0xff00) >> 8; - p[1] = it.first & 0xff; - p += 2; - p[0] = (it.second->len() & 0xff00) >> 8; - p[1] = it.second->len() & 0xff; - p += 2; + QUICVariableInt::encode(p, TRANSPORT_PARAMETERS_MAXIMUM_SIZE, l, it.first); + p += l; + QUICVariableInt::encode(p, TRANSPORT_PARAMETERS_MAXIMUM_SIZE, l, it.second->len()); + p += l; memcpy(p, it.second->data(), it.second->len()); p += it.second->len(); } - ptrdiff_t n = p - parameters_size - sizeof(uint16_t); - - parameters_size[0] = (n & 0xff00) >> 8; - parameters_size[1] = n & 0xff; - *len = (p - buf); + + if (is_debug_tag_set(tag)) { + this->_print(); + } } void @@ -290,25 +286,25 @@ QUICTransportParameters::_print() const uint64_t int_value; size_t int_value_len; QUICVariableInt::decode(int_value, int_value_len, p.second->data(), p.second->len()); - Debug(tag, "%s: 0x%" PRIx64 " (%" PRIu64 ")", QUICDebugNames::transport_parameter_id(p.first), int_value, int_value); + Debug(tag, "%s (%" PRIu32 "): 0x%" PRIx64 " (%" PRIu64 ")", QUICDebugNames::transport_parameter_id(p.first), + static_cast(p.first), int_value, int_value); } else if (p.second->len() <= 24) { char hex_str[65]; to_hex_str(hex_str, sizeof(hex_str), p.second->data(), p.second->len()); - Debug(tag, "%s: %s", QUICDebugNames::transport_parameter_id(p.first), hex_str); + Debug(tag, "%s (%" PRIu32 "): %s", QUICDebugNames::transport_parameter_id(p.first), static_cast(p.first), hex_str); } else if (QUICTransportParameterId::PREFERRED_ADDRESS == p.first) { QUICPreferredAddress pref_addr(p.second->data(), p.second->len()); - char cid_hex_str[QUICConnectionId::MAX_HEX_STR_LENGTH]; char token_hex_str[QUICStatelessResetToken::LEN * 2 + 1]; char ep_ipv4_hex_str[512]; char ep_ipv6_hex_str[512]; - pref_addr.cid().hex(cid_hex_str, sizeof(cid_hex_str)); to_hex_str(token_hex_str, sizeof(token_hex_str), pref_addr.token().buf(), QUICStatelessResetToken::LEN); ats_ip_nptop(pref_addr.endpoint_ipv4(), ep_ipv4_hex_str, sizeof(ep_ipv4_hex_str)); ats_ip_nptop(pref_addr.endpoint_ipv6(), ep_ipv6_hex_str, sizeof(ep_ipv6_hex_str)); Debug(tag, "%s: Endpoint(IPv4)=%s, Endpoint(IPv6)=%s, CID=%s, Token=%s", QUICDebugNames::transport_parameter_id(p.first), - ep_ipv4_hex_str, ep_ipv6_hex_str, cid_hex_str, token_hex_str); + ep_ipv4_hex_str, ep_ipv6_hex_str, pref_addr.cid().hex().c_str(), token_hex_str); } else { - Debug(tag, "%s: (long data)", QUICDebugNames::transport_parameter_id(p.first)); + Debug(tag, "%s (%" PRIu32 "): (%u byte data)", QUICDebugNames::transport_parameter_id(p.first), + static_cast(p.first), p.second->len()); } } } @@ -317,18 +313,9 @@ QUICTransportParameters::_print() const // QUICTransportParametersInClientHello // -QUICTransportParametersInClientHello::QUICTransportParametersInClientHello(const uint8_t *buf, size_t len) +QUICTransportParametersInClientHello::QUICTransportParametersInClientHello(const uint8_t *buf, size_t len, QUICVersion version) + : QUICTransportParameters(buf, len, version) { - this->_load(buf, len); - if (is_debug_tag_set(tag)) { - this->_print(); - } -} - -void -QUICTransportParametersInClientHello::_store(uint8_t *buf, uint16_t *len) const -{ - *len = 0; } std::ptrdiff_t @@ -338,9 +325,9 @@ QUICTransportParametersInClientHello::_parameters_offset(const uint8_t *) const } int -QUICTransportParametersInClientHello::_validate_parameters() const +QUICTransportParametersInClientHello::_validate_parameters(QUICVersion version) const { - int res = QUICTransportParameters::_validate_parameters(); + int res = QUICTransportParameters::_validate_parameters(version); if (res < 0) { return res; } @@ -348,14 +335,22 @@ QUICTransportParametersInClientHello::_validate_parameters() const decltype(this->_parameters)::const_iterator ite; // MUST NOTs - if ((ite = this->_parameters.find(QUICTransportParameterId::STATELESS_RESET_TOKEN)) != this->_parameters.end()) { - return -(TP_ERROR_MUST_NOT_EXIST | QUICTransportParameterId::STATELESS_RESET_TOKEN); + if ((ite = this->_parameters.find(QUICTransportParameterId::ORIGINAL_DESTINATION_CONNECTION_ID)) != this->_parameters.end()) { + return -(TP_ERROR_MUST_NOT_EXIST | QUICTransportParameterId::ORIGINAL_DESTINATION_CONNECTION_ID); } if ((ite = this->_parameters.find(QUICTransportParameterId::PREFERRED_ADDRESS)) != this->_parameters.end()) { return -(TP_ERROR_MUST_NOT_EXIST | QUICTransportParameterId::PREFERRED_ADDRESS); } + if ((ite = this->_parameters.find(QUICTransportParameterId::RETRY_SOURCE_CONNECTION_ID)) != this->_parameters.end()) { + return -(TP_ERROR_MUST_NOT_EXIST | QUICTransportParameterId::RETRY_SOURCE_CONNECTION_ID); + } + + if ((ite = this->_parameters.find(QUICTransportParameterId::STATELESS_RESET_TOKEN)) != this->_parameters.end()) { + return -(TP_ERROR_MUST_NOT_EXIST | QUICTransportParameterId::STATELESS_RESET_TOKEN); + } + return 0; } @@ -363,24 +358,10 @@ QUICTransportParametersInClientHello::_validate_parameters() const // QUICTransportParametersInEncryptedExtensions // -QUICTransportParametersInEncryptedExtensions::QUICTransportParametersInEncryptedExtensions(const uint8_t *buf, size_t len) -{ - this->_load(buf, len); - if (is_debug_tag_set(tag)) { - this->_print(); - } -} - -void -QUICTransportParametersInEncryptedExtensions::_store(uint8_t *buf, uint16_t *len) const -{ - *len = 0; -} - -void -QUICTransportParametersInEncryptedExtensions::add_version(QUICVersion version) +QUICTransportParametersInEncryptedExtensions::QUICTransportParametersInEncryptedExtensions(const uint8_t *buf, size_t len, + QUICVersion version) + : QUICTransportParameters(buf, len, version) { - this->_versions[this->_n_versions++] = version; } std::ptrdiff_t @@ -390,21 +371,36 @@ QUICTransportParametersInEncryptedExtensions::_parameters_offset(const uint8_t * } int -QUICTransportParametersInEncryptedExtensions::_validate_parameters() const +QUICTransportParametersInEncryptedExtensions::_validate_parameters(QUICVersion version) const { - int res = QUICTransportParameters::_validate_parameters(); + int res = QUICTransportParameters::_validate_parameters(version); if (res < 0) { return res; } decltype(this->_parameters)::const_iterator ite; - // MUSTs if the server sent a Retry packet - if ((ite = this->_parameters.find(QUICTransportParameterId::ORIGINAL_CONNECTION_ID)) != this->_parameters.end()) { - // We cannot check the length because it's not a fixed length. - } else { - // TODO Need a way that checks if we received a Retry from the server - // return -(TP_ERROR_MUST_EXIST | QUICTransportParameterId::ORIGINAL_CONNECTION_ID); + // MUSTs + if (version == QUIC_SUPPORTED_VERSIONS[0]) { // draft-28 + if ((ite = this->_parameters.find(QUICTransportParameterId::INITIAL_SOURCE_CONNECTION_ID)) != this->_parameters.end()) { + // We cannot check the length because it's not a fixed length. + } else { + return -(TP_ERROR_MUST_EXIST | QUICTransportParameterId::INITIAL_SOURCE_CONNECTION_ID); + } + + if ((ite = this->_parameters.find(QUICTransportParameterId::ORIGINAL_DESTINATION_CONNECTION_ID)) != this->_parameters.end()) { + // We cannot check the length because it's not a fixed length. + } else { + return -(TP_ERROR_MUST_EXIST | QUICTransportParameterId::ORIGINAL_DESTINATION_CONNECTION_ID); + } + + // MUSTs if the server sent a Retry packet, but MUST NOT if the server did not send a Retry packet + // TODO Check if the server sent Retry packet + if ((ite = this->_parameters.find(QUICTransportParameterId::RETRY_SOURCE_CONNECTION_ID)) != this->_parameters.end()) { + // return -(TP_ERROR_MUST_NOT_EXIST | QUICTransportParameterId::RETRY_SOURCE_CONNECTION_ID); + } else { + // return -(TP_ERROR_MUST_EXIST | QUICTransportParameterId::RETRY_SOURCE_CONNECTION_ID); + } } // MAYs @@ -425,8 +421,6 @@ QUICTransportParametersInEncryptedExtensions::_validate_parameters() const #ifndef OPENSSL_IS_BORINGSSL -static constexpr int TRANSPORT_PARAMETERS_MAXIMUM_SIZE = 65535; - // // QUICTransportParametersHandler // @@ -452,14 +446,15 @@ int QUICTransportParametersHandler::parse(SSL *s, unsigned int ext_type, unsigned int context, const unsigned char *in, size_t inlen, X509 *x, size_t chainidx, int *al, void *parse_arg) { - QUICTLS *qtls = static_cast(SSL_get_ex_data(s, QUIC::ssl_quic_tls_index)); - + QUICTLS *qtls = static_cast(SSL_get_ex_data(s, QUIC::ssl_quic_tls_index)); + const QUICConnection *qc = static_cast(SSL_get_ex_data(s, QUIC::ssl_quic_qc_index)); + QUICVersion version = qc->negotiated_version(); switch (context) { case SSL_EXT_CLIENT_HELLO: - qtls->set_remote_transport_parameters(std::make_shared(in, inlen)); + qtls->set_remote_transport_parameters(std::make_shared(in, inlen, version)); break; case SSL_EXT_TLS1_3_ENCRYPTED_EXTENSIONS: - qtls->set_remote_transport_parameters(std::make_shared(in, inlen)); + qtls->set_remote_transport_parameters(std::make_shared(in, inlen, version)); break; default: // Do nothing diff --git a/iocore/net/quic/QUICTransportParameters.h b/iocore/net/quic/QUICTransportParameters.h index 6886bd09af0..72533843105 100644 --- a/iocore/net/quic/QUICTransportParameters.h +++ b/iocore/net/quic/QUICTransportParameters.h @@ -34,10 +34,10 @@ class QUICTransportParameterId { public: enum { - ORIGINAL_CONNECTION_ID, - IDLE_TIMEOUT, + ORIGINAL_DESTINATION_CONNECTION_ID, + MAX_IDLE_TIMEOUT, STATELESS_RESET_TOKEN, - MAX_PACKET_SIZE, + MAX_UDP_PAYLOAD_SIZE, INITIAL_MAX_DATA, INITIAL_MAX_STREAM_DATA_BIDI_LOCAL, INITIAL_MAX_STREAM_DATA_BIDI_REMOTE, @@ -46,8 +46,11 @@ class QUICTransportParameterId INITIAL_MAX_STREAMS_UNI, ACK_DELAY_EXPONENT, MAX_ACK_DELAY, - DISABLE_MIGRATION, + DISABLE_ACTIVE_MIGRATION, PREFERRED_ADDRESS, + ACTIVE_CONNECTION_ID_LIMIT, + INITIAL_SOURCE_CONNECTION_ID, + RETRY_SOURCE_CONNECTION_ID, }; explicit operator bool() const { return true; } @@ -58,7 +61,7 @@ class QUICTransportParameterId } bool - operator==(uint16_t &x) const + operator==(const uint16_t &x) const { return this->_id == x; } @@ -74,7 +77,7 @@ class QUICTransportParameterId class QUICTransportParameters { public: - QUICTransportParameters(const uint8_t *buf, size_t len); + QUICTransportParameters(const uint8_t *buf, size_t len, QUICVersion version); virtual ~QUICTransportParameters(); bool is_valid() const; @@ -103,12 +106,11 @@ class QUICTransportParameters }; QUICTransportParameters(){}; - void _load(const uint8_t *buf, size_t len); + void _load(const uint8_t *buf, size_t len, QUICVersion version); bool _valid = false; virtual std::ptrdiff_t _parameters_offset(const uint8_t *buf) const = 0; - virtual int _validate_parameters() const; - virtual void _store(uint8_t *buf, uint16_t *len) const = 0; + virtual int _validate_parameters(QUICVersion version) const; void _print() const; std::map _parameters; @@ -118,12 +120,11 @@ class QUICTransportParametersInClientHello : public QUICTransportParameters { public: QUICTransportParametersInClientHello() : QUICTransportParameters(){}; - QUICTransportParametersInClientHello(const uint8_t *buf, size_t len); + QUICTransportParametersInClientHello(const uint8_t *buf, size_t len, QUICVersion version); protected: std::ptrdiff_t _parameters_offset(const uint8_t *buf) const override; - int _validate_parameters() const override; - void _store(uint8_t *buf, uint16_t *len) const override; + int _validate_parameters(QUICVersion version) const override; private: }; @@ -132,16 +133,11 @@ class QUICTransportParametersInEncryptedExtensions : public QUICTransportParamet { public: QUICTransportParametersInEncryptedExtensions() : QUICTransportParameters(){}; - QUICTransportParametersInEncryptedExtensions(const uint8_t *buf, size_t len); - void add_version(QUICVersion version); + QUICTransportParametersInEncryptedExtensions(const uint8_t *buf, size_t len, QUICVersion version); protected: std::ptrdiff_t _parameters_offset(const uint8_t *buf) const override; - int _validate_parameters() const override; - void _store(uint8_t *buf, uint16_t *len) const override; - - uint8_t _n_versions = 0; - QUICVersion _versions[256] = {}; + int _validate_parameters(QUICVersion version) const override; }; class QUICTransportParametersHandler diff --git a/iocore/net/quic/QUICTypes.cc b/iocore/net/quic/QUICTypes.cc index 29e09d2de00..305abc2b2b7 100644 --- a/iocore/net/quic/QUICTypes.cc +++ b/iocore/net/quic/QUICTypes.cc @@ -22,6 +22,10 @@ */ #include +#include +#include +#include + #include "QUICTypes.h" #include "QUICIntUtil.h" #include "tscore/CryptoHash.h" @@ -149,13 +153,14 @@ QUICTypeUtil::key_phase(QUICPacketType type) QUICPacketNumberSpace QUICTypeUtil::pn_space(QUICEncryptionLevel level) { + // TODO Optimize the case order switch (level) { case QUICEncryptionLevel::HANDSHAKE: - return QUICPacketNumberSpace::Handshake; + return QUICPacketNumberSpace::HANDSHAKE; case QUICEncryptionLevel::INITIAL: - return QUICPacketNumberSpace::Initial; + return QUICPacketNumberSpace::INITIAL; default: - return QUICPacketNumberSpace::ApplicationData; + return QUICPacketNumberSpace::APPLICATION_DATA; } } @@ -190,15 +195,15 @@ QUICTypeUtil::read_QUICVersion(const uint8_t *buf) } QUICStreamId -QUICTypeUtil::read_QUICStreamId(const uint8_t *buf) +QUICTypeUtil::read_QUICStreamId(const uint8_t *buf, size_t buf_len) { - return static_cast(QUICIntUtil::read_QUICVariableInt(buf)); + return static_cast(QUICIntUtil::read_QUICVariableInt(buf, buf_len)); } QUICOffset -QUICTypeUtil::read_QUICOffset(const uint8_t *buf) +QUICTypeUtil::read_QUICOffset(const uint8_t *buf, size_t buf_len) { - return static_cast(QUICIntUtil::read_QUICVariableInt(buf)); + return static_cast(QUICIntUtil::read_QUICVariableInt(buf, buf_len)); } uint16_t @@ -214,9 +219,9 @@ QUICTypeUtil::read_QUICAppErrorCode(const uint8_t *buf) } uint64_t -QUICTypeUtil::read_QUICMaxData(const uint8_t *buf) +QUICTypeUtil::read_QUICMaxData(const uint8_t *buf, size_t buf_len) { - return QUICIntUtil::read_QUICVariableInt(buf); + return QUICIntUtil::read_QUICVariableInt(buf, buf_len); } void @@ -252,15 +257,15 @@ QUICTypeUtil::write_QUICOffset(QUICOffset offset, uint8_t *buf, size_t *len) } void -QUICTypeUtil::write_QUICTransErrorCode(uint16_t error_code, uint8_t *buf, size_t *len) +QUICTypeUtil::write_QUICTransErrorCode(uint64_t error_code, uint8_t *buf, size_t *len) { - QUICIntUtil::write_uint_as_nbytes(static_cast(error_code), 2, buf, len); + QUICIntUtil::write_QUICVariableInt(static_cast(error_code), buf, len); } void QUICTypeUtil::write_QUICAppErrorCode(QUICAppErrorCode error_code, uint8_t *buf, size_t *len) { - QUICIntUtil::write_uint_as_nbytes(static_cast(error_code), 2, buf, len); + QUICIntUtil::write_QUICVariableInt(static_cast(error_code), buf, len); } void @@ -284,6 +289,27 @@ QUICStatelessResetToken::QUICStatelessResetToken(const QUICConnectionId &conn_id QUICIntUtil::write_uint_as_nbytes(_hash.u64[1], 8, _token + 8, &dummy); } +uint64_t +QUICStatelessResetToken::_hashcode() const +{ + return (static_cast(this->_token[0]) << 56) + (static_cast(this->_token[1]) << 48) + + (static_cast(this->_token[2]) << 40) + (static_cast(this->_token[3]) << 32) + + (static_cast(this->_token[4]) << 24) + (static_cast(this->_token[5]) << 16) + + (static_cast(this->_token[6]) << 8) + (static_cast(this->_token[7])); +} + +std::string +QUICStatelessResetToken::hex() const +{ + std::stringstream stream; + stream << "0x"; + for (auto i = 0; i < QUICStatelessResetToken::LEN; i++) { + stream << std::setfill('0') << std::setw(2) << std::hex; + stream << std::hex << static_cast(this->_token[i]); + } + return stream.str(); +} + QUICResumptionToken::QUICResumptionToken(const IpEndpoint &src, QUICConnectionId cid, ink_hrtime expire_time) { // TODO: read cookie secret from file like SSLTicketKeyConfig @@ -335,7 +361,7 @@ QUICResumptionToken::expire_time() const return QUICIntUtil::read_nbytes_as_uint(this->_token + (1 + 20), 4); } -QUICRetryToken::QUICRetryToken(const IpEndpoint &src, QUICConnectionId original_dcid) +QUICRetryToken::QUICRetryToken(const IpEndpoint &src, QUICConnectionId original_dcid, QUICConnectionId scid) { // TODO: read cookie secret from file like SSLTicketKeyConfig static constexpr char stateless_retry_token_secret[] = "stateless_cookie_secret"; @@ -346,8 +372,14 @@ QUICRetryToken::QUICRetryToken(const IpEndpoint &src, QUICConnectionId original_ data_len = strlen(reinterpret_cast(data)); size_t cid_len; + *(data + data_len) = original_dcid.length(); + data_len += 1; QUICTypeUtil::write_QUICConnectionId(original_dcid, data + data_len, &cid_len); data_len += cid_len; + *(data + data_len) = scid.length(); + data_len += 1; + QUICTypeUtil::write_QUICConnectionId(scid, data + data_len, &cid_len); + data_len += cid_len; this->_token[0] = static_cast(Type::RETRY); HMAC(EVP_sha1(), stateless_retry_token_secret, sizeof(stateless_retry_token_secret), data, data_len, this->_token + 1, @@ -355,21 +387,38 @@ QUICRetryToken::QUICRetryToken(const IpEndpoint &src, QUICConnectionId original_ ink_assert(this->_token_len == 20); this->_token_len += 1; + *(this->_token + this->_token_len) = original_dcid.length(); + this->_token_len += 1; QUICTypeUtil::write_QUICConnectionId(original_dcid, this->_token + this->_token_len, &cid_len); this->_token_len += cid_len; + *(this->_token + this->_token_len) = scid.length(); + this->_token_len += 1; + QUICTypeUtil::write_QUICConnectionId(scid, this->_token + this->_token_len, &cid_len); + this->_token_len += cid_len; } bool QUICRetryToken::is_valid(const IpEndpoint &src) const { - return *this == QUICRetryToken(src, this->original_dcid()); + return *this == QUICRetryToken(src, this->original_dcid(), this->scid()); } const QUICConnectionId QUICRetryToken::original_dcid() const { // Type uses 1 byte and output of EVP_sha1() should be 160 bits - return QUICTypeUtil::read_QUICConnectionId(this->_token + (1 + 20), this->_token_len - (1 + 20)); + auto len = *(this->_token + (1 + 20)); + auto start = this->_token + (1 + 20 + 1); + return QUICTypeUtil::read_QUICConnectionId(start, len); +} + +const QUICConnectionId +QUICRetryToken::scid() const +{ + auto len = *(this->_token + (1 + 20)); + auto start = this->_token + (1 + 20 + 1 + len + 1); + len = *(this->_token + (1 + 20 + 1 + len)); + return QUICTypeUtil::read_QUICConnectionId(start, len); } QUICFrameType @@ -552,6 +601,24 @@ QUICFiveTuple::protocol() const return this->_protocol; } +// +// QUICPath +// + +QUICPath::QUICPath(IpEndpoint local_ep, IpEndpoint remote_ep) : _local_ep(local_ep), _remote_ep(remote_ep) {} + +const IpEndpoint & +QUICPath::local_ep() const +{ + return this->_local_ep; +} + +const IpEndpoint & +QUICPath::remote_ep() const +{ + return this->_remote_ep; +} + // // QUICConnectionId // @@ -559,7 +626,7 @@ QUICConnectionId QUICConnectionId::ZERO() { uint8_t zero[MAX_LENGTH] = {0}; - return QUICConnectionId(zero, sizeof(zero)); + return QUICConnectionId(zero, 0); } QUICConnectionId::QUICConnectionId() @@ -569,7 +636,8 @@ QUICConnectionId::QUICConnectionId() QUICConnectionId::QUICConnectionId(const uint8_t *buf, uint8_t len) : _len(len) { - memcpy(this->_id, buf, len); + ink_assert(len <= QUICConnectionId::MAX_LENGTH); + memcpy(this->_id, buf, std::min(static_cast(len), QUICConnectionId::MAX_LENGTH)); } uint8_t @@ -618,10 +686,28 @@ QUICConnectionId::h32() const return static_cast(QUICIntUtil::read_nbytes_as_uint(this->_id, 4)); } -int -QUICConnectionId::hex(char *buf, size_t len) const +std::string +QUICConnectionId::hex() const { - return to_hex_str(buf, len, this->_id, this->_len); + std::stringstream stream; + stream << "0x"; + for (auto i = 0; i < this->_len; i++) { + stream << std::setfill('0') << std::setw(2) << std::hex; + stream << std::hex << static_cast(this->_id[i]); + } + return stream.str(); +} + +QUICFrameId +QUICSentPacketInfo::FrameInfo::id() const +{ + return this->_id; +} + +QUICFrameGenerator * +QUICSentPacketInfo::FrameInfo::generated_by() const +{ + return this->_generator; } // @@ -660,7 +746,7 @@ QUICInvariants::dcil(uint8_t &dst, const uint8_t *buf, uint64_t buf_len) return false; } - dst = buf[QUICInvariants::LH_CIL_OFFSET] >> 4; + dst = buf[QUICInvariants::LH_CIL_OFFSET]; return true; } @@ -670,11 +756,15 @@ QUICInvariants::scil(uint8_t &dst, const uint8_t *buf, uint64_t buf_len) { ink_assert(QUICInvariants::is_long_header(buf)); - if (buf_len < QUICInvariants::LH_CIL_OFFSET) { + uint8_t dcil = 0; + if (!QUICInvariants::dcil(dcil, buf, buf_len)) { return false; } - - dst = buf[QUICInvariants::LH_CIL_OFFSET] & 0x0F; + uint64_t scil_offset = QUICInvariants::LH_CIL_OFFSET + 1 + dcil; + if (scil_offset >= buf_len) { + return false; + } + dst = buf[scil_offset]; return true; } @@ -691,13 +781,12 @@ QUICInvariants::dcid(QUICConnectionId &dst, const uint8_t *buf, uint64_t buf_len return false; } - if (dcil) { - dcid_len = dcil + QUICInvariants::CIL_BASE; - } else { + if (dcil == 0) { dst = QUICConnectionId::ZERO(); return true; } + dcid_len = dcil; dcid_offset = QUICInvariants::LH_DCID_OFFSET; } else { // remote dcil is local scil @@ -705,6 +794,10 @@ QUICInvariants::dcid(QUICConnectionId &dst, const uint8_t *buf, uint64_t buf_len dcid_offset = QUICInvariants::SH_DCID_OFFSET; } + if (dcid_len > QUICConnectionId::MAX_LENGTH) { + return false; + } + if (dcid_offset + dcid_len > buf_len) { return false; } @@ -724,34 +817,45 @@ QUICInvariants::scid(QUICConnectionId &dst, const uint8_t *buf, uint64_t buf_len } uint8_t scid_offset = QUICInvariants::LH_DCID_OFFSET; - uint8_t scid_len = 0; uint8_t dcil = 0; if (!QUICInvariants::dcil(dcil, buf, buf_len)) { return false; } - if (dcil) { - scid_offset += (dcil + QUICInvariants::CIL_BASE); - } + scid_offset += dcil; uint8_t scil = 0; if (!QUICInvariants::scil(scil, buf, buf_len)) { return false; } + scid_offset += 1; - if (scil) { - scid_len = scil + QUICInvariants::CIL_BASE; - } else { + if (scil == 0) { dst = QUICConnectionId::ZERO(); return true; } - if (scid_offset + scid_len > buf_len) { + if (scid_offset + scil > buf_len) { return false; } - dst = QUICTypeUtil::read_QUICConnectionId(buf + scid_offset, scid_len); + dst = QUICTypeUtil::read_QUICConnectionId(buf + scid_offset, scil); return true; } + +namespace QUICBase +{ +std::string +to_hex(const uint8_t *buf, size_t len) +{ + std::stringstream stream; + stream << "0x"; + for (size_t i = 0; i < len; i++) { + stream << std::setfill('0') << std::setw(2) << std::hex; + stream << std::hex << static_cast(buf[i]); + } + return stream.str(); +} +} // namespace QUICBase diff --git a/iocore/net/quic/QUICTypes.h b/iocore/net/quic/QUICTypes.h index ee19ab05d90..47e830cd8d0 100644 --- a/iocore/net/quic/QUICTypes.h +++ b/iocore/net/quic/QUICTypes.h @@ -24,6 +24,7 @@ #pragma once #include +#include #include "tscore/ink_endian.h" #include "tscore/ink_hrtime.h" #include "tscore/Ptr.h" @@ -34,6 +35,7 @@ #include #include #include +#include #include "tscore/ink_memory.h" #include "tscore/ink_inet.h" #include "openssl/evp.h" @@ -42,8 +44,7 @@ using QUICPacketNumber = uint64_t; using QUICVersion = uint32_t; using QUICStreamId = uint64_t; using QUICOffset = uint64_t; - -static constexpr uint8_t kPacketNumberSpace = 3; +using QUICFrameId = uint64_t; // TODO: Update version number // Note: Prefix for drafts (0xff000000) + draft number @@ -51,9 +52,11 @@ static constexpr uint8_t kPacketNumberSpace = 3; // Note: Fix QUIC_ALPN_PROTO_LIST in QUICConfig.cc // Note: Change ExtensionType (QUICTransportParametersHandler::TRANSPORT_PARAMETER_ID) if it's changed constexpr QUICVersion QUIC_SUPPORTED_VERSIONS[] = { - 0xff000014, + 0xff00001d, + 0xff00001b, }; -constexpr QUICVersion QUIC_EXERCISE_VERSION = 0x1a2a3a4a; +constexpr QUICVersion QUIC_EXERCISE_VERSION1 = 0x1a2a3a4a; +constexpr QUICVersion QUIC_EXERCISE_VERSION2 = 0x5a6a7a8a; enum class QUICEncryptionLevel { NONE = -1, @@ -64,7 +67,7 @@ enum class QUICEncryptionLevel { }; // For range-based for loop. This starts from INITIAL to ONE_RTT. It doesn't include NONE. -// Defining begin, end, operator*, operator++ doen't work for duplicate symbol issue with libmgmt_p.a :( +// Defining begin, end, operator*, operator++ doesn't work for duplicate symbol issue with libmgmt_p.a :( // TODO: support ZERO_RTT constexpr QUICEncryptionLevel QUIC_ENCRYPTION_LEVELS[] = { QUICEncryptionLevel::INITIAL, @@ -73,14 +76,12 @@ constexpr QUICEncryptionLevel QUIC_ENCRYPTION_LEVELS[] = { QUICEncryptionLevel::ONE_RTT, }; -// introduce by draft-19 kPacketNumberSpace -enum class QUICPacketNumberSpace { - Initial, - Handshake, - ApplicationData, -}; +// kPacketNumberSpace on Recovery A.2.Constants of Interest +enum class QUICPacketNumberSpace : int { INITIAL, HANDSHAKE, APPLICATION_DATA, N_SPACES }; +// For conveniece (this removes neccesity of static_cast) +constexpr int QUIC_N_PACKET_SPACES = static_cast(QUICPacketNumberSpace::N_SPACES); -// Devide to QUICPacketType and QUICPacketLongHeaderType ? +// Divide to QUICPacketType and QUICPacketLongHeaderType ? enum class QUICPacketType : uint8_t { INITIAL = 0x00, // draft-17 version-specific type ZERO_RTT_PROTECTED = 0x01, // draft-17 version-specific type @@ -114,7 +115,8 @@ enum class QUICFrameType : uint8_t { PATH_CHALLENGE, PATH_RESPONSE, CONNECTION_CLOSE, // 0x1c - 0x1d - UNKNOWN = 0x1e, + HANDSHAKE_DONE = 0x1e, + UNKNOWN = 0x1f, }; enum class QUICVersionNegotiationStatus { @@ -147,24 +149,26 @@ enum class QUICErrorClass { APPLICATION, }; -enum class QUICTransErrorCode : uint16_t { +enum class QUICTransErrorCode : uint64_t { NO_ERROR = 0x00, INTERNAL_ERROR, - SERVER_BUSY, + CONNECTION_REFUSED, FLOW_CONTROL_ERROR, - STREAM_ID_ERROR, + STREAM_LIMIT_ERROR, STREAM_STATE_ERROR, - FINAL_OFFSET_ERROR, + FINAL_SIZE_ERROR, FRAME_ENCODING_ERROR, TRANSPORT_PARAMETER_ERROR, - VERSION_NEGOTIATION_ERROR, + CONNECTION_ID_LIMIT_ERROR, PROTOCOL_VIOLATION, - INVALID_MIGRATION = 0x0C, - CRYPTO_ERROR = 0x0100, // 0x100 - 0x1FF + INVALID_TOKEN, + APPLICATION_ERROR, + CRYPTO_BUFFER_EXCEEDED, + CRYPTO_ERROR = 0x0100, // 0x100 - 0x1FF }; // Application Protocol Error Codes defined in application -using QUICAppErrorCode = uint16_t; +using QUICAppErrorCode = uint64_t; constexpr uint16_t QUIC_APP_ERROR_CODE_STOPPING = 0; class QUICError @@ -224,9 +228,9 @@ class QUICConnectionId public: static uint8_t SCID_LEN; - static const int MIN_LENGTH_FOR_INITIAL = 8; - static const int MAX_LENGTH = 18; - static const size_t MAX_HEX_STR_LENGTH = MAX_LENGTH * 2 + 1; + static constexpr int MIN_LENGTH_FOR_INITIAL = 8; + static constexpr int MAX_LENGTH = 20; + static constexpr size_t MAX_HEX_STR_LENGTH = MAX_LENGTH * 2 + 1; static QUICConnectionId ZERO(); QUICConnectionId(); QUICConnectionId(const uint8_t *buf, uint8_t len); @@ -259,7 +263,7 @@ class QUICConnectionId * This is just for debugging. */ uint32_t h32() const; - int hex(char *buf, size_t len) const; + std::string hex() const; uint8_t length() const; bool is_zero() const; @@ -280,22 +284,36 @@ class QUICStatelessResetToken QUICStatelessResetToken(const QUICConnectionId &conn_id, uint32_t instance_id); QUICStatelessResetToken(const uint8_t *buf) { memcpy(this->_token, buf, QUICStatelessResetToken::LEN); } + /** + * Note that this returns a kind of hash code so we can use a StatelessResetToken as a key for a hashtable. + */ + operator uint64_t() const { return this->_hashcode(); } + bool operator==(const QUICStatelessResetToken &x) const { return memcmp(this->_token, x._token, QUICStatelessResetToken::LEN) == 0; } + bool + operator!=(const QUICStatelessResetToken &x) const + { + return memcmp(this->_token, x._token, QUICStatelessResetToken::LEN) != 0; + } + const uint8_t * buf() const { return _token; } + std::string hex() const; + private: uint8_t _token[LEN] = {0}; void _generate(uint64_t data); + uint64_t _hashcode() const; }; class QUICAddressValidationToken @@ -306,6 +324,8 @@ class QUICAddressValidationToken RETRY, }; + // FIXME Check token length + QUICAddressValidationToken(const uint8_t *buf, size_t len) : _token_len(len) { memcpy(this->_token, buf, len); } virtual ~QUICAddressValidationToken(){}; static Type @@ -315,15 +335,30 @@ class QUICAddressValidationToken return static_cast(buf[0]) == Type::RESUMPTION ? Type::RESUMPTION : Type::RETRY; } - virtual const uint8_t *buf() const = 0; - virtual uint8_t length() const = 0; + virtual const uint8_t * + buf() const + { + return this->_token; + } + + virtual uint8_t + length() const + { + return this->_token_len; + } + +protected: + QUICAddressValidationToken() {} + + // The size should be smaller than maximum size of Retry packet + uint8_t _token[1200] = {0}; + unsigned int _token_len; }; class QUICResumptionToken : public QUICAddressValidationToken { public: - QUICResumptionToken() {} - QUICResumptionToken(const uint8_t *buf, uint8_t len) : _token_len(len) { memcpy(this->_token, buf, len); } + QUICResumptionToken(const uint8_t *buf, uint8_t len) : QUICAddressValidationToken(buf, len) {} QUICResumptionToken(const IpEndpoint &src, QUICConnectionId cid, ink_hrtime expire_time); bool @@ -339,30 +374,13 @@ class QUICResumptionToken : public QUICAddressValidationToken const QUICConnectionId cid() const; const ink_hrtime expire_time() const; - - const uint8_t * - buf() const override - { - return this->_token; - } - - uint8_t - length() const override - { - return this->_token_len; - } - -private: - uint8_t _token[1 + EVP_MAX_MD_SIZE + QUICConnectionId::MAX_LENGTH + 4]; - unsigned int _token_len; }; class QUICRetryToken : public QUICAddressValidationToken { public: - QUICRetryToken() {} - QUICRetryToken(const uint8_t *buf, uint8_t len) : _token_len(len) { memcpy(this->_token, buf, len); } - QUICRetryToken(const IpEndpoint &src, QUICConnectionId original_dcid); + QUICRetryToken(const uint8_t *buf, size_t len) : QUICAddressValidationToken(buf, len) {} + QUICRetryToken(const IpEndpoint &src, QUICConnectionId original_dcid, QUICConnectionId scid); bool operator==(const QUICRetryToken &x) const @@ -376,30 +394,14 @@ class QUICRetryToken : public QUICAddressValidationToken bool is_valid(const IpEndpoint &src) const; const QUICConnectionId original_dcid() const; - - const uint8_t * - buf() const override - { - return this->_token; - } - - uint8_t - length() const override - { - return this->_token_len; - } - -private: - uint8_t _token[1 + EVP_MAX_MD_SIZE + QUICConnectionId::MAX_LENGTH] = {}; - unsigned int _token_len = 0; - QUICConnectionId _original_dcid; + const QUICConnectionId scid() const; }; class QUICPreferredAddress { public: - constexpr static int16_t MIN_LEN = 26; - constexpr static int16_t MAX_LEN = 295; + constexpr static int16_t MIN_LEN = 41; + constexpr static int16_t MAX_LEN = 61; QUICPreferredAddress(IpEndpoint endpoint_ipv4, IpEndpoint endpoint_ipv6, const QUICConnectionId &cid, QUICStatelessResetToken token) @@ -419,8 +421,8 @@ class QUICPreferredAddress void store(uint8_t *buf, uint16_t &len) const; private: - IpEndpoint _endpoint_ipv4; - IpEndpoint _endpoint_ipv6; + IpEndpoint _endpoint_ipv4 = {}; + IpEndpoint _endpoint_ipv6 = {}; QUICConnectionId _cid; QUICStatelessResetToken _token; bool _valid = false; @@ -457,25 +459,85 @@ class QUICFiveTuple uint64_t _hash_code = 0; }; +class QUICPath +{ +public: + QUICPath(IpEndpoint local_ep, IpEndpoint remote_ep); + const IpEndpoint &local_ep() const; + const IpEndpoint &remote_ep() const; + + inline bool + operator==(const QUICPath &x) const + { + if ((this->_local_ep.port() != 0 && x._local_ep.port() != 0) && this->_local_ep.port() != x._local_ep.port()) { + return false; + } + + if ((this->_remote_ep.port() != 0 && x._remote_ep.port() != 0) && this->_remote_ep.port() != x._remote_ep.port()) { + return false; + } + + if ((!IpAddr(this->_local_ep).isAnyAddr() && !IpAddr(x._local_ep).isAnyAddr()) && this->_local_ep != x._local_ep) { + return false; + } + + if ((!IpAddr(this->_remote_ep).isAnyAddr() || !IpAddr(x._remote_ep).isAnyAddr()) && this->_remote_ep != x._remote_ep) { + return false; + } + + return true; + } + +private: + IpEndpoint _local_ep; + IpEndpoint _remote_ep; +}; + +class QUICPathHasher +{ +public: + std::size_t + operator()(const QUICPath &k) const + { + return k.remote_ep().port(); + } +}; + +class QUICPathValidationData +{ +public: + QUICPathValidationData(const uint8_t *data) { memcpy(this->_data, data, sizeof(this->_data)); } + + inline operator const uint8_t *() const { return this->_data; } + +private: + uint8_t _data[8]; +}; + class QUICTPConfig { public: - virtual uint32_t no_activity_timeout() const = 0; - virtual const IpEndpoint *preferred_address_ipv4() const = 0; - virtual const IpEndpoint *preferred_address_ipv6() const = 0; - virtual uint32_t initial_max_data() const = 0; - virtual uint32_t initial_max_stream_data_bidi_local() const = 0; - virtual uint32_t initial_max_stream_data_bidi_remote() const = 0; - virtual uint32_t initial_max_stream_data_uni() const = 0; - virtual uint64_t initial_max_streams_bidi() const = 0; - virtual uint64_t initial_max_streams_uni() const = 0; - virtual uint8_t ack_delay_exponent() const = 0; - virtual uint8_t max_ack_delay() const = 0; + virtual ~QUICTPConfig() = default; // required + virtual uint32_t no_activity_timeout() const = 0; + virtual const IpEndpoint *preferred_address_ipv4() const = 0; + virtual const IpEndpoint *preferred_address_ipv6() const = 0; + virtual uint32_t initial_max_data() const = 0; + virtual uint32_t initial_max_stream_data_bidi_local() const = 0; + virtual uint32_t initial_max_stream_data_bidi_remote() const = 0; + virtual uint32_t initial_max_stream_data_uni() const = 0; + virtual uint64_t initial_max_streams_bidi() const = 0; + virtual uint64_t initial_max_streams_uni() const = 0; + virtual uint8_t ack_delay_exponent() const = 0; + virtual uint8_t max_ack_delay() const = 0; + virtual uint8_t active_cid_limit() const = 0; + virtual bool disable_active_migration() const = 0; + virtual std::unordered_map> additional_tp() const = 0; }; class QUICLDConfig { public: + virtual ~QUICLDConfig() {} virtual uint32_t packet_threshold() const = 0; virtual float time_threshold() const = 0; virtual ink_hrtime granularity() const = 0; @@ -485,13 +547,55 @@ class QUICLDConfig class QUICCCConfig { public: - virtual uint32_t max_datagram_size() const = 0; + virtual ~QUICCCConfig() {} virtual uint32_t initial_window() const = 0; virtual uint32_t minimum_window() const = 0; virtual float loss_reduction_factor() const = 0; virtual uint32_t persistent_congestion_threshold() const = 0; }; +class QUICFrameGenerator; + +struct QUICSentPacketInfo { + class FrameInfo + { + public: + FrameInfo(QUICFrameId id, QUICFrameGenerator *generator) : _id(id), _generator(generator) {} + + QUICFrameId id() const; + QUICFrameGenerator *generated_by() const; + + private: + QUICFrameId _id = 0; + QUICFrameGenerator *_generator; + }; + + // Recovery A.1.1. Sent Packet Fields + QUICPacketNumber packet_number; + bool ack_eliciting; + bool in_flight; + size_t sent_bytes; + ink_hrtime time_sent; + + // Additional fields + QUICPacketType type; + std::vector frames; + QUICPacketNumberSpace pn_space; + // End of additional fields +}; + +using QUICSentPacketInfoUPtr = std::unique_ptr; + +class QUICRTTProvider +{ +public: + virtual ink_hrtime smoothed_rtt() const = 0; + virtual ink_hrtime rttvar() const = 0; + virtual ink_hrtime latest_rtt() const = 0; + + virtual ink_hrtime congestion_period(uint32_t threshold) const = 0; +}; + // TODO: move version independent functions to QUICInvariants class QUICTypeUtil { @@ -508,11 +612,11 @@ class QUICTypeUtil static int read_QUICPacketNumberLen(const uint8_t *buf); static QUICPacketNumber read_QUICPacketNumber(const uint8_t *buf, int len); static QUICVersion read_QUICVersion(const uint8_t *buf); - static QUICStreamId read_QUICStreamId(const uint8_t *buf); - static QUICOffset read_QUICOffset(const uint8_t *buf); + static QUICStreamId read_QUICStreamId(const uint8_t *buf, size_t buf_len); + static QUICOffset read_QUICOffset(const uint8_t *buf, size_t buf_len); static uint16_t read_QUICTransErrorCode(const uint8_t *buf); static QUICAppErrorCode read_QUICAppErrorCode(const uint8_t *buf); - static uint64_t read_QUICMaxData(const uint8_t *buf); + static uint64_t read_QUICMaxData(const uint8_t *buf, size_t buf_len); static void write_QUICConnectionId(QUICConnectionId connection_id, uint8_t *buf, size_t *len); static void write_QUICPacketNumberLen(int len, uint8_t *buf); @@ -520,7 +624,7 @@ class QUICTypeUtil static void write_QUICVersion(QUICVersion version, uint8_t *buf, size_t *len); static void write_QUICStreamId(QUICStreamId stream_id, uint8_t *buf, size_t *len); static void write_QUICOffset(QUICOffset offset, uint8_t *buf, size_t *len); - static void write_QUICTransErrorCode(uint16_t error_code, uint8_t *buf, size_t *len); + static void write_QUICTransErrorCode(uint64_t error_code, uint8_t *buf, size_t *len); static void write_QUICAppErrorCode(QUICAppErrorCode error_code, uint8_t *buf, size_t *len); static void write_QUICMaxData(uint64_t max_data, uint8_t *buf, size_t *len); @@ -533,18 +637,11 @@ class QUICInvariants static bool is_long_header(const uint8_t *buf); static bool is_version_negotiation(QUICVersion v); static bool version(QUICVersion &dst, const uint8_t *buf, uint64_t buf_len); - /** - * This function returns the raw value. You'll need to add 3 to the returned value to get the actual connection id length. - */ static bool dcil(uint8_t &dst, const uint8_t *buf, uint64_t buf_len); - /** - * This function returns the raw value. You'll need to add 3 to the returned value to get the actual connection id length. - */ static bool scil(uint8_t &dst, const uint8_t *buf, uint64_t buf_len); static bool dcid(QUICConnectionId &dst, const uint8_t *buf, uint64_t buf_len); static bool scid(QUICConnectionId &dst, const uint8_t *buf, uint64_t buf_len); - static const size_t CIL_BASE = 3; static const size_t LH_VERSION_OFFSET = 1; static const size_t LH_CIL_OFFSET = 5; static const size_t LH_DCID_OFFSET = 6; @@ -554,3 +651,9 @@ class QUICInvariants }; int to_hex_str(char *dst, size_t dst_len, const uint8_t *src, size_t src_len); + +namespace QUICBase +{ +std::string to_hex(const uint8_t *buf, size_t len); + +} // namespace QUICBase diff --git a/iocore/net/quic/QUICUnidirectionalStream.cc b/iocore/net/quic/QUICUnidirectionalStream.cc index b6dae8f6f0a..c6a3d7a44de 100644 --- a/iocore/net/quic/QUICUnidirectionalStream.cc +++ b/iocore/net/quic/QUICUnidirectionalStream.cc @@ -125,17 +125,21 @@ QUICSendStream::state_stream_closed(int event, void *data) } bool -QUICSendStream::will_generate_frame(QUICEncryptionLevel level, ink_hrtime timestamp) +QUICSendStream::will_generate_frame(QUICEncryptionLevel level, size_t current_packet_size, bool ack_eliciting, uint32_t seq_num) { - return !this->is_retransmited_frame_queue_empty() || this->_write_vio.get_reader()->is_read_avail_more_than(0); + if (!this->is_retransmited_frame_queue_empty()) { + return true; + } + if (this->_write_vio.op != VIO::NONE && this->_write_vio.get_reader()->is_read_avail_more_than(0)) { + return true; + } + return false; } QUICFrame * QUICSendStream::generate_frame(uint8_t *buf, QUICEncryptionLevel level, uint64_t connection_credit, uint16_t maximum_frame_size, - ink_hrtime timestamp) + size_t current_packet_size, uint32_t seq_num) { - SCOPED_MUTEX_LOCK(lock, this->_write_vio.mutex, this_ethread()); - QUICFrame *frame = this->create_retransmitted_frame(buf, level, maximum_frame_size, this->_issue_frame_id(), this); if (frame != nullptr) { ink_assert(frame->type() == QUICFrameType::STREAM); @@ -146,97 +150,102 @@ QUICSendStream::generate_frame(uint8_t *buf, QUICEncryptionLevel level, uint64_t // RESET_STREAM if (this->_reset_reason && !this->_is_reset_sent) { frame = QUICFrameFactory::create_rst_stream_frame(buf, *this->_reset_reason, this->_issue_frame_id(), this); + if (frame->size() > maximum_frame_size) { + frame->~QUICFrame(); + return nullptr; + } this->_records_rst_stream_frame(level, *static_cast(frame)); this->_state.update_with_sending_frame(*frame); this->_is_reset_sent = true; return frame; } - if (!this->_state.is_allowed_to_send(QUICFrameType::STREAM)) { - return frame; - } - - uint64_t maximum_data_size = 0; - if (maximum_frame_size <= MAX_STREAM_FRAME_OVERHEAD) { - return frame; - } - maximum_data_size = maximum_frame_size - MAX_STREAM_FRAME_OVERHEAD; - - bool pure_fin = false; - bool fin = false; - if ((this->_write_vio.nbytes != 0 || this->_write_vio.nbytes != INT64_MAX) && - this->_write_vio.nbytes == static_cast(this->_send_offset)) { - // Pure FIN stream should be sent regardless status of remote flow controller, because the length is zero. - pure_fin = true; - fin = true; - } + if (this->_write_vio.op != VIO::NONE && this->_state.is_allowed_to_send(QUICFrameType::STREAM)) { + SCOPED_MUTEX_LOCK(lock, this->_write_vio.mutex, this_ethread()); - uint64_t len = 0; - IOBufferReader *reader = this->_write_vio.get_reader(); - if (!pure_fin) { - uint64_t data_len = reader->block_read_avail(); - if (data_len == 0) { + uint64_t maximum_data_size = 0; + if (maximum_frame_size <= MAX_STREAM_FRAME_OVERHEAD) { return frame; } - - // Check Connection/Stream level credit only if the generating STREAM frame is not pure fin - uint64_t stream_credit = this->_remote_flow_controller.credit(); - if (stream_credit == 0) { - // STREAM_DATA_BLOCKED - frame = this->_remote_flow_controller.generate_frame(buf, level, UINT16_MAX, maximum_frame_size, timestamp); - return frame; + maximum_data_size = maximum_frame_size - MAX_STREAM_FRAME_OVERHEAD; + + bool pure_fin = false; + bool fin = false; + if ((this->_write_vio.nbytes != 0 || this->_write_vio.nbytes != INT64_MAX) && + this->_write_vio.nbytes == static_cast(this->_send_offset)) { + // Pure FIN stream should be sent regardless status of remote flow controller, because the length is zero. + pure_fin = true; + fin = true; } - if (connection_credit == 0) { - // BLOCKED - BLOCKED frame will be sent by connection level remote flow controller - return frame; + uint64_t len = 0; + IOBufferReader *reader = this->_write_vio.get_reader(); + if (!pure_fin) { + uint64_t data_len = reader->block_read_avail(); + if (data_len == 0) { + return frame; + } + + // Check Connection/Stream level credit only if the generating STREAM frame is not pure fin + uint64_t stream_credit = this->_remote_flow_controller.credit(); + if (stream_credit == 0) { + // STREAM_DATA_BLOCKED + frame = + this->_remote_flow_controller.generate_frame(buf, level, UINT16_MAX, maximum_frame_size, current_packet_size, seq_num); + return frame; + } + + if (connection_credit == 0) { + // BLOCKED - BLOCKED frame will be sent by connection level remote flow controller + return frame; + } + + len = std::min(data_len, std::min(maximum_data_size, std::min(stream_credit, connection_credit))); + + // data_len, maximum_data_size, stream_credit and connection_credit are already checked they're larger than 0 + ink_assert(len != 0); + + if (this->_write_vio.nbytes == static_cast(this->_send_offset + len)) { + fin = true; + } } - len = std::min(data_len, std::min(maximum_data_size, std::min(stream_credit, connection_credit))); - - // data_len, maximum_data_size, stream_credit and connection_credit are already checked they're larger than 0 - ink_assert(len != 0); - - if (this->_write_vio.nbytes == static_cast(this->_send_offset + len)) { - fin = true; + Ptr block = make_ptr(reader->get_current_block()->clone()); + block->consume(reader->start_offset); + block->_end = std::min(block->start() + len, block->_buf_end); + ink_assert(static_cast(block->read_avail()) == len); + + // STREAM - Pure FIN or data length is lager than 0 + // FIXME has_length_flag and has_offset_flag should be configurable + frame = QUICFrameFactory::create_stream_frame(buf, block, this->_id, this->_send_offset, fin, true, true, + this->_issue_frame_id(), this); + if (!this->_state.is_allowed_to_send(*frame)) { + QUICStreamDebug("Canceled sending %s frame due to the stream state", QUICDebugNames::frame_type(frame->type())); + return frame; } - } - Ptr block = make_ptr(reader->get_current_block()->clone()); - block->consume(reader->start_offset); - block->_end = std::min(block->start() + len, block->_buf_end); - ink_assert(static_cast(block->read_avail()) == len); - - // STREAM - Pure FIN or data length is lager than 0 - // FIXME has_length_flag and has_offset_flag should be configurable - frame = QUICFrameFactory::create_stream_frame(buf, block, this->_id, this->_send_offset, fin, true, true, this->_issue_frame_id(), - this); - if (!this->_state.is_allowed_to_send(*frame)) { - QUICStreamDebug("Canceled sending %s frame due to the stream state", QUICDebugNames::frame_type(frame->type())); - return frame; - } + if (!pure_fin) { + int ret = this->_remote_flow_controller.update(this->_send_offset + len); + // We cannot cancel sending the frame after updating the flow controller - if (!pure_fin) { - int ret = this->_remote_flow_controller.update(this->_send_offset + len); - // We cannot cancel sending the frame after updating the flow controller + // Calling update always success, because len is always less than stream_credit + ink_assert(ret == 0); - // Calling update always success, because len is always less than stream_credit - ink_assert(ret == 0); + QUICStreamFCDebug("[REMOTE] %" PRIu64 "/%" PRIu64, this->_remote_flow_controller.current_offset(), + this->_remote_flow_controller.current_limit()); + if (this->_remote_flow_controller.current_offset() == this->_remote_flow_controller.current_limit()) { + QUICStreamDebug("Flow Controller will block sending a STREAM frame"); + } - QUICStreamFCDebug("[REMOTE] %" PRIu64 "/%" PRIu64, this->_remote_flow_controller.current_offset(), - this->_remote_flow_controller.current_limit()); - if (this->_remote_flow_controller.current_offset() == this->_remote_flow_controller.current_limit()) { - QUICStreamDebug("Flow Controller will block sending a STREAM frame"); + reader->consume(len); + this->_send_offset += len; + this->_write_vio.ndone += len; } + this->_records_stream_frame(level, *static_cast(frame)); - reader->consume(len); - this->_send_offset += len; - this->_write_vio.ndone += len; + this->_signal_write_event(); + this->_state.update_with_sending_frame(*frame); } - this->_records_stream_frame(level, *static_cast(frame)); - - this->_signal_write_event(); - this->_state.update_with_sending_frame(*frame); return frame; } @@ -526,21 +535,30 @@ QUICReceiveStream::is_cancelled() const } bool -QUICReceiveStream::will_generate_frame(QUICEncryptionLevel level, ink_hrtime timestamp) +QUICReceiveStream::will_generate_frame(QUICEncryptionLevel level, size_t current_packet_size, bool ack_eliciting, uint32_t seq_num) { - return this->_local_flow_controller.will_generate_frame(level, timestamp) || - (this->_stop_sending_reason != nullptr && this->_is_stop_sending_sent == false); + if (this->_local_flow_controller.will_generate_frame(level, current_packet_size, ack_eliciting, seq_num)) { + return true; + } + if (this->_stop_sending_reason != nullptr && this->_is_stop_sending_sent == false) { + return true; + } + return false; } QUICFrame * QUICReceiveStream::generate_frame(uint8_t *buf, QUICEncryptionLevel level, uint64_t connection_credit, uint16_t maximum_frame_size, - ink_hrtime timestamp) + size_t current_packet_size, uint32_t seq_num) { QUICFrame *frame = nullptr; // STOP_SENDING if (this->_stop_sending_reason && !this->_is_stop_sending_sent) { frame = QUICFrameFactory::create_stop_sending_frame(buf, this->id(), this->_stop_sending_reason->code, this->_issue_frame_id(), this); + if (frame->size() > maximum_frame_size) { + frame->~QUICFrame(); + return nullptr; + } this->_records_stop_sending_frame(level, *static_cast(frame)); this->_state.update_with_sending_frame(*frame); this->_is_stop_sending_sent = true; @@ -548,7 +566,8 @@ QUICReceiveStream::generate_frame(uint8_t *buf, QUICEncryptionLevel level, uint6 } // MAX_STREAM_DATA - frame = this->_local_flow_controller.generate_frame(buf, level, UINT16_MAX, maximum_frame_size, timestamp); + frame = this->_local_flow_controller.generate_frame(buf, level, UINT16_MAX, maximum_frame_size, current_packet_size, seq_num); + // maximum_frame_size should be checked in QUICFlowController return frame; } @@ -571,7 +590,7 @@ QUICReceiveStream::recv(const QUICStreamDataBlockedFrame &frame) /** * @brief Receive STREAM frame * @detail When receive STREAM frame, reorder frames and write to buffer of read_vio. - * If the reordering or writting operation is heavy, split out them to read function, + * If the reordering or writing operation is heavy, split out them to read function, * which is called by application via do_io_read() or reenable(). */ QUICConnectionErrorUPtr diff --git a/iocore/net/quic/QUICUnidirectionalStream.h b/iocore/net/quic/QUICUnidirectionalStream.h index 0c7162742a6..5ebe5527c0b 100644 --- a/iocore/net/quic/QUICUnidirectionalStream.h +++ b/iocore/net/quic/QUICUnidirectionalStream.h @@ -37,9 +37,9 @@ class QUICSendStream : public QUICStreamVConnection int state_stream_closed(int event, void *data); // QUICFrameGenerator - bool will_generate_frame(QUICEncryptionLevel level, ink_hrtime timestamp) override; + bool will_generate_frame(QUICEncryptionLevel level, size_t current_packet_size, bool ack_eliciting, uint32_t seq_num) override; QUICFrame *generate_frame(uint8_t *buf, QUICEncryptionLevel level, uint64_t connection_credit, uint16_t maximum_frame_size, - ink_hrtime timestamp) override; + size_t current_packet_size, uint32_t seq_num) override; virtual QUICConnectionErrorUPtr recv(const QUICMaxStreamDataFrame &frame) override; virtual QUICConnectionErrorUPtr recv(const QUICStopSendingFrame &frame) override; @@ -86,9 +86,9 @@ class QUICReceiveStream : public QUICStreamVConnection, public QUICTransferProgr int state_stream_closed(int event, void *data); // QUICFrameGenerator - bool will_generate_frame(QUICEncryptionLevel level, ink_hrtime timestamp) override; + bool will_generate_frame(QUICEncryptionLevel level, size_t current_packet_size, bool ack_eliciting, uint32_t seq_num) override; QUICFrame *generate_frame(uint8_t *buf, QUICEncryptionLevel level, uint64_t connection_credit, uint16_t maximum_frame_size, - ink_hrtime timestamp) override; + size_t current_packet_size, uint32_t seq_num) override; virtual QUICConnectionErrorUPtr recv(const QUICStreamFrame &frame) override; virtual QUICConnectionErrorUPtr recv(const QUICStreamDataBlockedFrame &frame) override; diff --git a/iocore/net/quic/QUICVersionNegotiator.cc b/iocore/net/quic/QUICVersionNegotiator.cc index ec1dc7938d6..43d2fbf5631 100644 --- a/iocore/net/quic/QUICVersionNegotiator.cc +++ b/iocore/net/quic/QUICVersionNegotiator.cc @@ -25,7 +25,7 @@ #include "QUICTransportParameters.h" QUICVersionNegotiationStatus -QUICVersionNegotiator::status() +QUICVersionNegotiator::status() const { return this->_status; } @@ -35,20 +35,21 @@ QUICVersionNegotiator::negotiate(const QUICPacket &packet) { switch (packet.type()) { case QUICPacketType::INITIAL: { - if (QUICTypeUtil::is_supported_version(packet.version())) { + const QUICInitialPacketR &initial_packet = static_cast(packet); + if (QUICTypeUtil::is_supported_version(initial_packet.version())) { this->_status = QUICVersionNegotiationStatus::NEGOTIATED; - this->_negotiated_version = packet.version(); + this->_negotiated_version = initial_packet.version(); } break; } case QUICPacketType::VERSION_NEGOTIATION: { - const uint8_t *supported_versions = packet.payload(); - uint16_t supported_versions_len = packet.payload_length(); - uint16_t len = 0; + const QUICVersionNegotiationPacketR &vn_packet = static_cast(packet); + uint16_t n_supported_version = vn_packet.nversions(); + uint16_t len = 0; - while (len < supported_versions_len) { - QUICVersion version = QUICTypeUtil::read_QUICVersion(supported_versions + len); + for (int i = 0; i < n_supported_version; ++i) { + QUICVersion version = vn_packet.supported_version(i); len += sizeof(QUICVersion); if (QUICTypeUtil::is_supported_version(version)) { @@ -69,13 +70,13 @@ QUICVersionNegotiator::negotiate(const QUICPacket &packet) } QUICVersionNegotiationStatus -QUICVersionNegotiator::validate() +QUICVersionNegotiator::validate() const { return this->_status; } QUICVersion -QUICVersionNegotiator::negotiated_version() +QUICVersionNegotiator::negotiated_version() const { return this->_negotiated_version; } diff --git a/iocore/net/quic/QUICVersionNegotiator.h b/iocore/net/quic/QUICVersionNegotiator.h index 6b86d7f73a5..392154e614e 100644 --- a/iocore/net/quic/QUICVersionNegotiator.h +++ b/iocore/net/quic/QUICVersionNegotiator.h @@ -28,16 +28,16 @@ #include "QUICTransportParameters.h" /** - * @brief Abstruct QUIC Application Class + * @brief Abstract QUIC Application Class * @detail Every quic application must inherits this class */ class QUICVersionNegotiator { public: - QUICVersionNegotiationStatus status(); + QUICVersionNegotiationStatus status() const; QUICVersionNegotiationStatus negotiate(const QUICPacket &initial_packet); - QUICVersionNegotiationStatus validate(); - QUICVersion negotiated_version(); + QUICVersionNegotiationStatus validate() const; + QUICVersion negotiated_version() const; private: QUICVersion _negotiated_version = 0; diff --git a/iocore/net/quic/qlog/QLog.cc b/iocore/net/quic/qlog/QLog.cc new file mode 100644 index 00000000000..71bb5fcca6e --- /dev/null +++ b/iocore/net/quic/qlog/QLog.cc @@ -0,0 +1,103 @@ +/** @file + * + * A brief file description + * + * @section license License + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +#include "QLog.h" + +namespace QLog +{ +void +Trace::encode(YAML::Node &node) +{ + node["title"] = _title; + node["description"] = _desc; + + // common fields + { + YAML::Node cf; + cf["ODCID"] = _odcid; + cf["reference_time"] = std::to_string(this->_reference_time / HRTIME_MSECOND); + node["common_fields"] = cf; + } + + { + node["event_fields"].push_back("relative_time"); + node["event_fields"].push_back("category"); + node["event_fields"].push_back("event"); + node["event_fields"].push_back("data"); + + if (_vp.name != "") { + node["vantage_point"]["name"] = _vp.name; + } + + if (vantage_point_type_name(_vp.type)) { + node["vantage_point"]["type"] = vantage_point_type_name(_vp.type); + } + + if (vantage_point_type_name(_vp.flow)) { + node["vantage_point"]["flow"] = vantage_point_type_name(_vp.flow); + } + } + + // events + for (auto &&it : _events) { + YAML::Node sub(YAML::NodeType::value::Sequence); + sub.push_back((it->get_time() - this->_reference_time) / HRTIME_MSECOND); + sub.push_back(it->category()); + sub.push_back(it->event()); + YAML::Node event; + it->encode(event); + sub.push_back(event); + node["events"].push_back(sub); + } +} + +void +QLog::dump(const std::string &dir) +{ + YAML::Node root; + root["qlog_version"] = this->_ver; + root["title"] = this->_title; + root["description"] = this->_desc; + for (auto &&it : this->_traces) { + YAML::Node node; + it->encode(node); + root["traces"].push_back(node); + } + + auto file = dir; + if (file.back() != '/') { + file += "/"; + } + file += this->last_trace().odcid() + ".qlog"; + std::ofstream ofs; + ofs.open(file, std::ofstream::in | std::ofstream::trunc); + + YAML::Emitter emitter(ofs); + emitter << YAML::DoubleQuoted << YAML::Flow << root; + ofs << "\n"; + ofs.close(); +} + +} // namespace QLog diff --git a/iocore/net/quic/qlog/QLog.h b/iocore/net/quic/qlog/QLog.h new file mode 100644 index 00000000000..c40bb472801 --- /dev/null +++ b/iocore/net/quic/qlog/QLog.h @@ -0,0 +1,152 @@ +/** @file + * + * A brief file description + * + * @section license License + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include + +#include "QLogEvent.h" + +namespace QLog +{ +class Trace +{ +public: + enum class VantagePointType : uint8_t { + client, + server, + network, + unknown, + }; + + struct VantagePoint { + std::string name; + VantagePointType type; + VantagePointType flow = VantagePointType::unknown; + }; + + Trace(const std::string &odcid, const std::string &title = "", const std::string &desc = "") + : _reference_time(Thread::get_hrtime()), _odcid(odcid) + { + } + + Trace(const VantagePoint &vp, const std::string &odcid, const std::string &title = "", const std::string &desc = "") + : Trace(odcid, title, desc) + { + set_vantage_point(vp); + } + + static const char * + vantage_point_type_name(VantagePointType ty) + { + switch (ty) { + case VantagePointType::client: + return "client"; + case VantagePointType::server: + return "server"; + case VantagePointType::network: + return "network"; + case VantagePointType::unknown: + return "unknown"; + default: + return nullptr; + } + } + + void + set_vantage_point(const VantagePoint &vp) + { + this->_vp = vp; + } + + Trace & + push_event(QLogEventUPtr e) + { + this->_events.push_back(std::move(e)); + return *this; + } + + std::string + odcid() const + { + return this->_odcid; + } + + void encode(YAML::Node &node); + +private: + int64_t _reference_time = Thread::get_hrtime(); + std::string _odcid; + std::string _title; + std::string _desc; + + VantagePoint _vp; + + std::vector _events; +}; + +class QLog +{ +public: + static constexpr char QLOG_VERSION[] = "draft-01"; + + QLog(const std::string &title = "", const std::string &desc = "", const std::string &ver = QLOG_VERSION) + : _title(title), _desc(desc), _ver(ver) + { + } + + Trace & + new_trace(Trace::VantagePoint vp, const std::string &odcid, const std::string &title = "", const std::string &desc = "") + { + this->_traces.push_back(std::make_unique(vp, odcid, title, desc)); + return *this->_traces.back().get(); + } + + Trace & + new_trace(std::string odcid, std::string title = "", std::string desc = "") + { + this->_traces.push_back(std::make_unique(odcid, title, desc)); + return *this->_traces.back().get(); + } + + Trace & + last_trace() + { + if (this->_traces.empty()) { + throw std::invalid_argument("traces is empty"); + } + + return *this->_traces.back().get(); + } + + void dump(const std::string &dir); + +private: + std::string _title; + std::string _desc; + std::string _ver; + std::vector> _traces; +}; + +} // namespace QLog diff --git a/iocore/net/quic/qlog/QLogEvent.cc b/iocore/net/quic/qlog/QLogEvent.cc new file mode 100644 index 00000000000..ce7bce35ffd --- /dev/null +++ b/iocore/net/quic/qlog/QLogEvent.cc @@ -0,0 +1,317 @@ +/** @file + * + * A brief file description + * + * @section license License + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "QLogEvent.h" + +namespace QLog +{ +void +check_and_set(YAML::Node &node, std::string key, std::string val) +{ + if (val.length() > 0) { + node[key] = val; + } +} + +void +check_and_set(YAML::Node &node, std::string key, std::vector val) +{ + if (val.size() > 0) { + node[key] = val; + } +} + +template +void +check_and_set(YAML::Node &node, std::string key, T val) +{ + if (val) { + node[key] = val; + } +} + +namespace Connectivity +{ + void + ServerListening::encode(YAML::Node &node) + { + check_and_set(node, "ip_v4", _ip_v4); + check_and_set(node, "ip_v6", _ip_v6); + check_and_set(node, "port_v4", _port_v4); + check_and_set(node, "port_v6", _port_v6); + check_and_set(node, "stateless_reset_required", _port_v6); + check_and_set(node, "quic_version", _quic_version); + check_and_set(node, "alpn_values", _alpn_values); + } + + void + ConnectionStarted::encode(YAML::Node &node) + { + check_and_set(node, "quic_version", _quic_version); + check_and_set(node, "ip_version", _ip_version); + check_and_set(node, "src_ip", _src_ip); + check_and_set(node, "dst_ip", _dst_ip); + check_and_set(node, "protocol", _protocol); + check_and_set(node, "src_port", _src_port); + check_and_set(node, "dst_port", _dst_port); + check_and_set(node, "src_cid", _src_cid); + check_and_set(node, "dst_cid", _dst_cid); + check_and_set(node, "alpn_values", _alpn_values); + } + + void + ConnectionIdUpdated::encode(YAML::Node &node) + { + check_and_set(node, "src_old", _src_old); + check_and_set(node, "src_new", _src_new); + check_and_set(node, "dst_old", _dst_old); + check_and_set(node, "dst_new", _dst_new); + } + + void + SpinBitUpdated::encode(YAML::Node &node) + { + check_and_set(node, "state", _state); + } + + void + ConnectionStateUpdated::encode(YAML::Node &node) + { + check_and_set(node, "new", static_cast(_new)); + check_and_set(node, "old", static_cast(_old)); + check_and_set(node, "trigger", trigger_name(_trigger)); + } + +} // namespace Connectivity + +namespace Security +{ + void + KeyEvent::encode(YAML::Node &node) + { + node["key_type"] = static_cast(_key_type); + node["new"] = _new; + check_and_set(node, "generation", _generation); + check_and_set(node, "old", _old); + check_and_set(node, "trigger", trigger_name(_trigger)); + } + +} // namespace Security + +namespace Transport +{ + void + ParametersSet::encode(YAML::Node &node) + { + node["owner"] = _owner ? "local" : "remote"; + check_and_set(node, "resumption_allowed", _resumption_allowed); + check_and_set(node, "early_data_enabled", _early_data_enabled); + check_and_set(node, "alpn", _alpn); + check_and_set(node, "version", _version); + check_and_set(node, "tls_cipher", _tls_cipher); + check_and_set(node, "original_connection_id", _original_connection_id); + check_and_set(node, "stateless_reset_token", _stateless_reset_token); + check_and_set(node, "disable_active_migration", _disable_active_migration); + check_and_set(node, "max_idle_timeout", _max_idle_timeout); + check_and_set(node, "max_udp_payload_size", _max_udp_payload_size); + check_and_set(node, "ack_delay_exponent", _ack_delay_exponent); + check_and_set(node, "max_ack_delay", _max_ack_delay); + check_and_set(node, "active_connection_id_limit", _active_connection_id_limit); + check_and_set(node, "initial_max_data", _initial_max_data); + check_and_set(node, "initial_max_stream_data_bidi_local", _initial_max_stream_data_bidi_local); + check_and_set(node, "initial_max_stream_data_bidi_remote", _initial_max_stream_data_bidi_remote); + check_and_set(node, "initial_max_stream_data_uni", _initial_max_stream_data_uni); + check_and_set(node, "initial_max_streams_bidi", _initial_max_streams_bidi); + check_and_set(node, "initial_max_streams_uni", _initial_max_streams_uni); + + if (_preferred_address.ip.length() > 0) { + YAML::Node sub; + check_and_set(sub, _preferred_address.ipv4 ? "ip_v4" : "ip_v6", _preferred_address.ip); + check_and_set(sub, _preferred_address.ipv4 ? "port_v4" : "port_v6", _preferred_address.port); + check_and_set(sub, "connection_id", _preferred_address.connection_id); + check_and_set(sub, "stateless_reset_token", _preferred_address.stateless_reset_token); + node["preferred_address"] = sub; + } + } + + void + PacketEvent::encode(YAML::Node &node) + { + node["packet_type"] = _packet_type; + for (auto &&it : this->_frames) { + YAML::Node sub; + it->encode(sub); + node["frames"].push_back(sub); + } + check_and_set(node, "is_coalesced", _is_coalesced); + check_and_set(node, "stateless_reset_token", _stateless_reset_token); + check_and_set(node, "supported_version", _supported_version); + check_and_set(node, "raw_encrypted", _raw_encrypted); + check_and_set(node, "raw_decrypted", _raw_decrypted); + check_and_set(node, "supported_version", _supported_version); + check_and_set(node, "supported_version", trigger_name(_trigger)); + + node["header"]["packet_number"] = _header.packet_number; + node["header"]["packet_size"] = _header.packet_size; + node["header"]["payload_length"] = _header.payload_length; + node["header"]["version"] = _header.version; + node["header"]["scil"] = _header.scil; + node["header"]["dcil"] = _header.dcil; + node["header"]["scid"] = _header.scid; + node["header"]["dcid"] = _header.dcid; + } + + void + PacketDropped::encode(YAML::Node &node) + { + node["packet_type"] = _packet_type; + check_and_set(node, "packet_size", _packet_size); + check_and_set(node, "raw", _raw); + check_and_set(node, "trigger", trigger_name(_trigger)); + } + + void + PacketBuffered::encode(YAML::Node &node) + { + node["packet_type"] = _packet_type; + check_and_set(node, "trigger", trigger_name(_trigger)); + check_and_set(node, "packet_number", trigger_name(_trigger)); + } + + void + DatagramsEvent::encode(YAML::Node &node) + { + check_and_set(node, "count", _count); + check_and_set(node, "byte_length", _byte_length); + } + + void + DatagramsDropped::encode(YAML::Node &node) + { + check_and_set(node, "byte_length", _byte_length); + } + + void + StreamStateUpdated::encode(YAML::Node &node) + { + node["new"] = static_cast(_new); + node["stream_id"] = _stream_id; + // FIXME + // node["stream_type"] = bidi ? "bidirectional" : "unidirectional"; + // node["stream_side"] = "sending"; + } + + void + FrameProcessed::encode(YAML::Node &node) + { + for (auto &&it : _frames) { + YAML::Node sub; + it->encode(sub); + node["frames"].push_back(sub); + } + } + +} // namespace Transport + +namespace Recovery +{ + void + ParametersSet::encode(YAML::Node &node) + { + check_and_set(node, "reordering_threshold", _reordering_threshold); + check_and_set(node, "time_threshold", _time_threshold); + check_and_set(node, "timer_granularity", _timer_granularity); + check_and_set(node, "initial_rtt", _initial_rtt); + check_and_set(node, "max_datagram_size", _max_datagram_size); + check_and_set(node, "initial_congestion_window", _initial_congestion_window); + check_and_set(node, "minimum_congestion_window", _minimum_congestion_window); + check_and_set(node, "loss_reduction_factor", _loss_reduction_factor); + check_and_set(node, "persistent_congestion_threshold", _persistent_congestion_threshold); + } + + void + MetricsUpdated::encode(YAML::Node &node) + { + check_and_set(node, "min_rtt", _min_rtt); + check_and_set(node, "smoothed_rtt", _smoothed_rtt); + check_and_set(node, "latest_rtt", _latest_rtt); + check_and_set(node, "rtt_variance", _rtt_variance); + check_and_set(node, "max_ack_delay", _max_ack_delay); + check_and_set(node, "pto_count", _pto_count); + check_and_set(node, "congestion_window", _congestion_window); + check_and_set(node, "bytes_in_flight", _bytes_in_flight); + check_and_set(node, "ssthresh", _ssthresh); + check_and_set(node, "packets_in_flight", _packets_in_flight); + check_and_set(node, "in_recovery", _in_recovery); + check_and_set(node, "pacing_rate", _pacing_rate); + } + + void + CongestionStateUpdated::encode(YAML::Node &node) + { + node["new"] = state_to_string(_new); + check_and_set(node, "old", state_to_string(_old)); + check_and_set(node, "old", trigger_name(_trigger)); + } + + void + LossTimerUpdated::encode(YAML::Node &node) + { + node["timer_type"] = _timer_type_ack ? "ack" : "pto"; + check_and_set(node, "event_type", event_type_name(_event_type)); + check_and_set(node, "packet_number_space", _packet_number_space); + if (_event_type == EventType::set) { + check_and_set(node, "delta", _delta); + } + } + + void + PacketLost::encode(YAML::Node &node) + { + node["packet_number"] = _packet_number; + node["packet_type"] = _packet_type; + check_and_set(node, "trigger", trigger_name(_trigger)); + YAML::Node sub; + _header.encode(sub); + node["header"] = sub; + + for (auto &&it : _frames) { + YAML::Node sub; + it->encode(sub); + node["frames"].push_back(sub); + } + } + + void + MarkedForRetransmit::encode(YAML::Node &node) + { + for (auto &&it : _frames) { + YAML::Node sub; + it->encode(sub); + node["frames"].push_back(sub); + } + } + +} // namespace Recovery + +} // namespace QLog diff --git a/iocore/net/quic/qlog/QLogEvent.h b/iocore/net/quic/qlog/QLogEvent.h new file mode 100644 index 00000000000..553a34835a4 --- /dev/null +++ b/iocore/net/quic/qlog/QLogEvent.h @@ -0,0 +1,1014 @@ +/** @file + * + * A brief file description + * + * @section license License + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include + +#include "QUICTypes.h" +#include "QLogEvent.h" +#include "QLogFrame.h" + +namespace QLog +{ +class QLogEvent +{ +public: + virtual ~QLogEvent() {} + + virtual std::string category() const = 0; + virtual std::string event() const = 0; + virtual void encode(YAML::Node &) = 0; + + virtual ink_hrtime + get_time() const + { + return this->_time; + }; + +protected: + ink_hrtime _time = Thread::get_hrtime(); +}; + +using QLogEventUPtr = std::unique_ptr; + +#define SET(field, type) \ + void set_##field(type v) { this->_node[#field] = v; } + +// enum class PacketType : uint8_t { initial, handshake, zerortt, onertt, retry, version_negotiation, unknown }; +using PacketType = std::string; + +struct PacketHeader { + std::string packet_number; + uint64_t packet_size; + uint64_t payload_length; + + // only if present in the header + // if correctly using NEW_CONNECTION_ID events, + // dcid can be skipped for 1RTT packets + std::string version; + std::string scil; + std::string dcil; + std::string scid; + std::string dcid; + + // Note: short vs long header is implicit through PacketType + void + encode(YAML::Node &node) const + { + node["packet_number"] = packet_number; + node["packet_size"] = packet_size; + node["payload_length"] = payload_length; + node["version"] = version; + node["scil"] = scil; + node["dcil"] = dcil; + node["scid"] = scid; + node["dcid"] = dcid; + } +}; + +#define SET_FUNC(cla, field, type) \ +public: \ + cla &set_##field(const type &v) \ + { \ + this->_##field = v; \ + return *this; \ + } \ + \ +private: \ + type _##field; + +#define APPEND_FUNC(cla, field, type) \ +public: \ + cla &append_##field(type v) \ + { \ + this->_##field.push_back(v); \ + return *this; \ + } \ + \ +private: \ + std::vector _##field; + +#define APPEND_FRAME_FUNC(cla) \ +public: \ + cla &append_frames(QLogFrameUPtr v) \ + { \ + this->_frames.push_back(std::move(v)); \ + return *this; \ + } \ + \ +private: \ + std::vector _frames; + +// +// connectivity +// +namespace Connectivity +{ + class ConnectivityEvent : public QLogEvent + { + public: + std::string + category() const override + { + return "connectivity"; + } + }; + + class ServerListening : public ConnectivityEvent + { + public: + ServerListening(int port, bool v6 = false) + { + if (v6) { + set_port_v6(port); + } else { + set_port_v4(port); + } + } + +#define _SET(a, b) SET_FUNC(ServerListening, a, b) +#define _APPEND(a, b) APPEND_FUNC(ServerListening, a, b) + _SET(port_v4, int) + _SET(port_v6, int) + _SET(ip_v4, std::string) + _SET(ip_v6, std::string) + _SET(stateless_reset_required, bool) + _APPEND(quic_version, std::string) + _APPEND(alpn_values, std::string) + +#undef _SET +#undef _APPEND + + void encode(YAML::Node &) override; + + std::string + event() const override + { + return "server_listening"; + } + }; + + class ConnectionStarted : public ConnectivityEvent + { + public: + ConnectionStarted(const std::string &version, const std::string &sip, const std::string &dip, int sport, int dport, + const std::string &protocol = "QUIC") + { + set_ip_version(version); + set_protocol(protocol); + set_src_ip(sip); + set_dst_ip(dip); + set_src_port(sport); + set_dst_port(dport); + } + +#define _SET(a, b) SET_FUNC(ConnectionStarted, a, b) +#define _APPEND(a, b) APPEND_FUNC(ConnectionStarted, a, b) + _SET(quic_version, std::string); + _SET(src_cid, std::string); + _SET(dst_cid, std::string); + _SET(protocol, std::string); + _SET(ip_version, std::string) + _SET(src_ip, std::string) + _SET(dst_ip, std::string) + _SET(src_port, int) + _SET(dst_port, int) + _APPEND(alpn_values, std::string) + +#undef _SET +#undef _APPEND + + void encode(YAML::Node &) override; + + std::string + event() const override + { + return "connection_started"; + } + }; + + class ConnectionIdUpdated : public ConnectivityEvent + { + public: + ConnectionIdUpdated(const std::string &old, const std::string &n, bool peer = false) + { + if (peer) { + set_dst_old(old); + set_dst_new(n); + } else { + set_src_old(old); + set_src_new(n); + } + } + +#define _SET(a, b) SET_FUNC(ConnectionIdUpdated, a, b) +#define _APPEND(a, b) APPEND_FUNC(ConnectionIdUpdated, a, b) + + _SET(src_old, std::string); + _SET(src_new, std::string); + _SET(dst_old, std::string); + _SET(dst_new, std::string); + +#undef _SET +#undef _APPEND + + void encode(YAML::Node &) override; + + std::string + event() const override + { + return "connection_id_updated"; + } + }; + + class SpinBitUpdated : public ConnectivityEvent + { + public: + SpinBitUpdated(bool state) { set_state(state); } + +#define _SET(a, b) SET_FUNC(SpinBitUpdated, a, b) + _SET(state, bool); +#undef _SET + + void encode(YAML::Node &) override; + + std::string + event() const override + { + return "spin_bit_updated"; + } + }; + + class ConnectionStateUpdated : public ConnectivityEvent + { + public: + enum class ConnectionState : uint8_t { + attempted, // client initial sent + reset, // stateless reset sent + handshake, // handshake in progress + active, // handshake successful, data exchange + keepalive, // no data for a longer period + draining, // CONNECTION_CLOSE sent + closed // connection actually fully closed, memory freed + }; + + enum class Triggered : uint8_t { + unknown, + error, // when closing because of an unexpected event + clean, // when closing normally + application // e.g., HTTP/3's GOAWAY frame + }; + + ConnectionStateUpdated(ConnectionState n, Triggered tr = Triggered::unknown) + { + set_new(n); + set_trigger(tr); + } + +#define _SET(a, b) SET_FUNC(ConnectionStateUpdated, a, b) + _SET(new, ConnectionState); + _SET(old, ConnectionState); + _SET(trigger, Triggered) + +#undef _SET + + void encode(YAML::Node &) override; + + static const char * + trigger_name(Triggered trigger) + { + switch (trigger) { + case Triggered::error: + return "error"; + case Triggered::clean: + return "clean"; + case Triggered::application: + return "application"; + default: + return nullptr; + } + } + + std::string + event() const override + { + return "connection_state_updated"; + } + }; + +} // namespace Connectivity + +namespace Security +{ + class KeyEvent : public QLogEvent + { + public: + enum class KeyType : uint8_t { + server_initial_secret, + client_initial_secret, + + server_handshake_secret, + client_handshake_secret, + + server_0rtt_secret, + client_0rtt_secret, + + server_1rtt_secret, + client_1rtt_secret + }; + + enum class Triggered : uint8_t { + unknown, + remote_update, + local_update, + tls, + }; + + KeyEvent(KeyType ty, const std::string &n, int generation, Triggered triggered = Triggered::unknown) + { + set_key_type(ty); + set_new(n); + set_generation(generation); + set_trigger(triggered); + } + +#define _SET(a, b) SET_FUNC(KeyEvent, a, b) + _SET(key_type, KeyType); + _SET(new, std::string) + _SET(old, std::string); + _SET(generation, int) + _SET(trigger, Triggered) +#undef _SET + + void encode(YAML::Node &) override; + + const char * + trigger_name(Triggered triggered) + { + switch (triggered) { + case Triggered::remote_update: + return "remote_update"; + case Triggered::local_update: + return "local_update"; + case Triggered::tls: + return "tls"; + default: + return nullptr; + } + } + + std::string + category() const override + { + return "security"; + } + }; + + class KeyUpdated : public KeyEvent + { + public: + KeyUpdated(KeyType ty, const std::string &n, int generation, Triggered triggered = KeyEvent::Triggered::unknown) + : KeyEvent(ty, n, generation, triggered) + { + } + + std::string + event() const override + { + return "key_updated"; + } + }; + + class KeyRetired : public KeyEvent + { + public: + KeyRetired(KeyType ty, const std::string &n, int generation, Triggered triggered = KeyEvent::Triggered::unknown) + : KeyEvent(ty, n, generation, triggered) + { + } + + std::string + event() const override + { + return "key_retired"; + } + }; + +} // namespace Security + +// +// transport event +// +namespace Transport +{ + class TransportEvent : public QLogEvent + { + public: + std::string + category() const override + { + return "transport"; + } + }; + + class ParametersSet : public TransportEvent + { + public: + struct PreferredAddress { + std::string ip; + int port; + std::string connection_id; + std::string stateless_reset_token; + bool ipv4 = true; + }; + + ParametersSet(bool owner) : _owner(owner) {} + + std::string + event() const override + { + return "parameters_set"; + } + +#define _SET(a, b) SET_FUNC(ParametersSet, a, b) + _SET(resumption_allowed, bool); // early data extension was enabled on the TLS layer + _SET(early_data_enabled, bool); // early data extension was enabled on the TLS layer + _SET(alpn, std::string); + _SET(version, std::string); // hex (e.g. 0x); + _SET(tls_cipher, std::string); // (e.g. AES_128_GCM_SHA256); + _SET(original_connection_id, std::string); // hex + _SET(stateless_reset_token, std::string); // hex + _SET(disable_active_migration, bool); + _SET(idle_timeout, int); + _SET(max_packet_size, int); + _SET(ack_delay_exponent, int); + _SET(max_ack_delay, int); + _SET(active_connection_id_limit, int); + _SET(initial_max_data, std::string); + _SET(initial_max_stream_data_bidi_local, std::string); + _SET(initial_max_stream_data_bidi_remote, std::string); + _SET(initial_max_stream_data_uni, std::string); + _SET(initial_max_streams_bidi, std::string); + _SET(initial_max_streams_uni, std::string); + _SET(max_idle_timeout, int64_t) + _SET(max_udp_payload_size, size_t) + _SET(preferred_address, PreferredAddress) +#undef _SET + + void encode(YAML::Node &) override; + + private: + bool _owner = false; + }; + + class PacketEvent : public TransportEvent + { + public: + enum class Triggered : uint8_t { + unknown, + keys_available, // if packet was buffered because it couldn't be decrypted before + retransmit_reordered, // draft-23 5.1.1 + retransmit_timeout, // draft-23 5.1.2 + pto_probe, // draft-23 5.3.1 + retransmit_crypto, // draft-19 6.2 + cc_bandwidth_probe, // needed for some CCs to figure out bandwidth allocations when there are no normal sends + }; + + PacketEvent(const PacketType &type, PacketHeader h, Triggered tr = Triggered::unknown) + { + set_packet_type(type).set_header(h).set_trigger(tr); + } + +#define _SET(a, b) SET_FUNC(PacketEvent, a, b) +#define _APPEND(a, b) APPEND_FUNC(PacketEvent, a, b) + _SET(packet_type, PacketType) + _SET(header, PacketHeader) + _SET(is_coalesced, bool); + _SET(raw_encrypted, std::string); + _SET(raw_decrypted, std::string); + _SET(stateless_reset_token, std::string); + _SET(trigger, Triggered); + _APPEND(supported_version, std::string); + +#undef _SET +#undef _APPEND + APPEND_FRAME_FUNC(PacketEvent) + + void encode(YAML::Node &) override; + + static const char * + trigger_name(Triggered triggered) + { + switch (triggered) { + case Triggered::retransmit_reordered: + return "retransmit_reordered"; + case Triggered::retransmit_timeout: + return "retransmit_timeout"; + case Triggered::pto_probe: + return "pto_probe"; + case Triggered::retransmit_crypto: + return "retransmit_crypto"; + case Triggered::cc_bandwidth_probe: + return "cc_bandwidth_probe"; + break; + case Triggered::keys_available: + return "keys_available"; + default: + return nullptr; + } + } + }; + + class PacketSent : public PacketEvent + { + public: + PacketSent(const PacketType &type, const PacketHeader &h, Triggered tr = Triggered::unknown) : PacketEvent(type, h, tr) {} + std::string + event() const override + { + return "packet_sent"; + } + }; + + class PacketReceived : public PacketEvent + { + public: + PacketReceived(const PacketType &type, const PacketHeader &h, Triggered tr = Triggered::unknown) : PacketEvent(type, h, tr) {} + std::string + event() const override + { + return "packet_received"; + } + }; + + class PacketDropped : public TransportEvent + { + public: + enum class Triggered : uint8_t { + unknown, + key_unavailable, + unknown_connection_id, + header_decrypt_error, + payload_decrypt_error, + protocol_violation, + dos_prevention, + unsupported_version, + unexpected_packet, + unexpected_source_connection_id, + unexpected_version, + }; + + PacketDropped(Triggered tr = Triggered::unknown) { set_trigger(tr); } + +#define _SET(a, b) SET_FUNC(PacketDropped, a, b) + _SET(packet_size, int); + _SET(raw, std::string); + _SET(trigger, Triggered); + _SET(packet_type, PacketType) +#undef _SET + + void encode(YAML::Node &) override; + + std::string + event() const override + { + return "packet_dropped"; + } + + static const char * + trigger_name(Triggered tr) + { + switch (tr) { + case Triggered::key_unavailable: + return "key_unavailable"; + case Triggered::unknown_connection_id: + return "unknown_connection_id"; + case Triggered::header_decrypt_error: + return "header_decrypt_error"; + case Triggered::payload_decrypt_error: + return "payload_decrypt_error"; + case Triggered::protocol_violation: + return "protocol_violation"; + case Triggered::dos_prevention: + return "dos_prevention"; + case Triggered::unsupported_version: + return "unsupported_version"; + case Triggered::unexpected_packet: + return "unexpected_packet"; + case Triggered::unexpected_source_connection_id: + return "unexpected_source_connection_id"; + case Triggered::unexpected_version: + return "unexpected_version"; + default: + return nullptr; + } + } + }; + + class PacketBuffered : public TransportEvent + { + public: + enum class Triggered : uint8_t { + unknown, + backpressure, + keys_unavailable, + }; + + PacketBuffered(Triggered tr = Triggered::unknown) { set_trigger(tr); } + +#define _SET(a, b) SET_FUNC(PacketBuffered, a, b) + _SET(trigger, Triggered); + _SET(packet_type, PacketType) + _SET(packet_number, std::string) +#undef _SET + + void encode(YAML::Node &) override; + + std::string + event() const override + { + return "packet_buffered"; + } + + static const char * + trigger_name(Triggered tr) + { + switch (tr) { + case Triggered::backpressure: + return "backpressure"; + case Triggered::keys_unavailable: + return "keys_unavailable"; + default: + return nullptr; + } + } + }; + + class DatagramsEvent : public TransportEvent + { + public: +#define _SET(a, b) SET_FUNC(DatagramsEvent, a, b) + _SET(count, int); + _SET(byte_length, int); +#undef _SET + void encode(YAML::Node &) override; + }; + + class DatagramsSent : public DatagramsEvent + { + public: + std::string + event() const override + { + return "datagrams_sent"; + } + }; + class DatagramReceived : public DatagramsEvent + { + public: + std::string + event() const override + { + return "datagrams_received"; + } + }; + + class DatagramsDropped : public TransportEvent + { + public: +#define _SET(a, b) SET_FUNC(DatagramsDropped, a, b) + _SET(byte_length, int); +#undef _SET + + void encode(YAML::Node &) override; + + std::string + event() const override + { + return "datagrams_dropped"; + } + }; + + class StreamStateUpdated : public TransportEvent + { + enum class StreamState { + // bidirectional stream states, draft-23 3.4. + idle, + open, + half_closed_local, + half_closed_remote, + closed, + + // sending-side stream states, draft-23 3.1. + ready, + send, + data_sent, + reset_sent, + reset_received, + + // receive-side stream states, draft-23 3.2. + receive, + size_known, + data_read, + reset_read, + + // both-side states + data_received, + + // qlog-defined + destroyed // memory actually freed + }; + + StreamStateUpdated(std::string stream_id, StreamState n) { set_new(n).set_stream_id(stream_id); } + + void encode(YAML::Node &) override; + +#define _SET(a, b) SET_FUNC(StreamStateUpdated, a, b) + _SET(new, StreamState); + _SET(old, StreamState); + _SET(stream_id, std::string); + _SET(bidi, bool); +#undef _SET + + std::string + event() const override + { + return "stream_state_updated"; + } + }; + + class FrameProcessed : public TransportEvent + { + public: + APPEND_FRAME_FUNC(FrameProcessed) + + void encode(YAML::Node &) override; + + std::string + event() const override + { + return "frame_processed"; + } + }; + +} // namespace Transport + +namespace Recovery +{ + class RecoveryEvent : public QLogEvent + { + public: + std::string + category() const override + { + return "recovery"; + } + }; + + class ParametersSet : public RecoveryEvent + { + public: +#define _SET(a, b) SET_FUNC(ParametersSet, a, b) + _SET(reordering_threshold, int); + _SET(time_threshold, int); + _SET(timer_granularity, int); + _SET(initial_rtt, int); + _SET(max_datagram_size, int); + _SET(initial_congestion_window, int); + _SET(minimum_congestion_window, int); + _SET(loss_reduction_factor, int); + _SET(persistent_congestion_threshold, int); +#undef _SET + void encode(YAML::Node &) override; + + std::string + event() const override + { + return "parameters_set"; + } + }; + + class MetricsUpdated : public RecoveryEvent + { + public: +#define _SET(a, b) SET_FUNC(MetricsUpdated, a, b) + _SET(min_rtt, int); + _SET(smoothed_rtt, int); + _SET(latest_rtt, int); + _SET(rtt_variance, int); + _SET(max_ack_delay, int); + _SET(pto_count, int); + _SET(congestion_window, int); + _SET(bytes_in_flight, int); + _SET(ssthresh, int); + _SET(packets_in_flight, int); + _SET(in_recovery, int); + _SET(pacing_rate, int); +#undef _SET + void encode(YAML::Node &) override; + + std::string + event() const override + { + return "metrics_updated"; + } + }; + + class CongestionStateUpdated : public RecoveryEvent + { + public: + enum class State : uint8_t { + slow_start, + congestion_avoidance, + application_limited, + recovery, + }; + + enum class Triggered : uint8_t { + unknown, + persistent_congestion, + ECN, + }; + + CongestionStateUpdated(State n, Triggered tr = Triggered::unknown) { set_trigger(tr).set_new(n); } + +#define _SET(a, b) SET_FUNC(CongestionStateUpdated, a, b) + _SET(trigger, Triggered) + _SET(new, State) + _SET(old, State) +#undef _SET + + void encode(YAML::Node &) override; + + std::string + event() const override + { + return "congestion_state_updated"; + } + + static const char * + trigger_name(Triggered tr) + { + switch (tr) { + case Triggered::persistent_congestion: + return "persistent_congestion"; + case Triggered::ECN: + return "ECN"; + default: + return nullptr; + } + } + + static const char * + state_to_string(State s) + { + switch (s) { + case State::slow_start: + return "slow_start"; + case State::congestion_avoidance: + return "congestion_avoidance"; + case State::application_limited: + return "application_limited"; + case State::recovery: + return "recovery"; + default: + break; + } + + return nullptr; + } + }; + + class LossTimerUpdated : public RecoveryEvent + { + public: + enum class EventType : uint8_t { + set, + expired, + cancelled, + }; + + void + set_timer_type(bool ack) + { + this->_timer_type_ack = ack; + } + + void encode(YAML::Node &) override; + +#define _SET(a, b) SET_FUNC(LossTimerUpdated, a, b) + _SET(event_type, EventType) + _SET(packet_number_space, int); + _SET(delta, int); +#undef _SET + + std::string + event() const override + { + return "loss_timer_updated"; + } + + static const char * + event_type_name(EventType et) + { + switch (et) { + case EventType::set: + return "set"; + case EventType::expired: + return "expired"; + case EventType::cancelled: + return "cancelled"; + default: + break; + } + return nullptr; + } + + private: + bool _timer_type_ack = false; + }; + + class PacketLost : public RecoveryEvent + { + public: + enum class Triggered : uint8_t { + unknown, + reordering_threshold, + time_threshold, + pto_expired, + }; + + PacketLost(PacketType pt, uint64_t pn, Triggered tr = Triggered::unknown) + { + set_trigger(tr).set_packet_type(pt).set_packet_number(pn); + } + +#define _SET(a, b) SET_FUNC(PacketLost, a, b) + _SET(header, PacketHeader) + _SET(packet_number, uint64_t); + _SET(packet_type, PacketType); + _SET(trigger, Triggered) + APPEND_FRAME_FUNC(PacketLost) +#undef _SET + + void encode(YAML::Node &) override; + + std::string + event() const override + { + return "packet_lost"; + } + + static const char * + trigger_name(Triggered tr) + { + switch (tr) { + case Triggered::pto_expired: + return "pto_expired"; + case Triggered::reordering_threshold: + return "reordering_threshold"; + case Triggered::time_threshold: + return "time_threshold"; + default: + return nullptr; + } + } + }; + + class MarkedForRetransmit : public RecoveryEvent + { + public: + APPEND_FRAME_FUNC(MarkedForRetransmit) + void encode(YAML::Node &) override; + std::string + event() const override + { + return "marked_for_retransmit"; + } + }; + +} // namespace Recovery + +} // namespace QLog diff --git a/iocore/net/quic/qlog/QLogFrame.cc b/iocore/net/quic/qlog/QLogFrame.cc new file mode 100644 index 00000000000..bcc46fcf739 --- /dev/null +++ b/iocore/net/quic/qlog/QLogFrame.cc @@ -0,0 +1,282 @@ +/** @file + * + * A brief file description + * + * @section license License + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "QLogFrame.h" + +namespace QLog +{ +template +const Real & +Convert(const QUICFrame *frame) +{ + // FIXME: dangerous + auto tmp = static_cast(frame); +#if defined(DEBUG) + auto ref = dynamic_cast(tmp); + ink_assert(ref != nullptr); + return *ref; +#endif + return *static_cast(tmp); +} + +QLogFrameUPtr +QLogFrameFactory::create(const QUICFrame *frame) +{ + switch (frame->type()) { + case QUICFrameType::ACK: + return std::make_unique(Convert(frame)); + case QUICFrameType::PADDING: + return std::make_unique(Convert(frame)); + case QUICFrameType::PING: + return std::make_unique(Convert(frame)); + case QUICFrameType::RESET_STREAM: + return std::make_unique(Convert(frame)); + case QUICFrameType::STOP_SENDING: + return std::make_unique(Convert(frame)); + case QUICFrameType::CRYPTO: + return std::make_unique(Convert(frame)); + case QUICFrameType::NEW_TOKEN: + return std::make_unique(Convert(frame)); + case QUICFrameType::STREAM: + return std::make_unique(Convert(frame)); + case QUICFrameType::MAX_DATA: + return std::make_unique(Convert(frame)); + case QUICFrameType::MAX_STREAM_DATA: + return std::make_unique(Convert(frame)); + case QUICFrameType::MAX_STREAMS: + return std::make_unique(Convert(frame)); + case QUICFrameType::DATA_BLOCKED: + return std::make_unique(Convert(frame)); + case QUICFrameType::STREAM_DATA_BLOCKED: + return std::make_unique(Convert(frame)); + case QUICFrameType::STREAMS_BLOCKED: + return std::make_unique(Convert(frame)); + case QUICFrameType::NEW_CONNECTION_ID: + return std::make_unique(Convert(frame)); + case QUICFrameType::RETIRE_CONNECTION_ID: + return std::make_unique(Convert(frame)); + case QUICFrameType::PATH_CHALLENGE: + return std::make_unique(Convert(frame)); + case QUICFrameType::PATH_RESPONSE: + return std::make_unique(Convert(frame)); + case QUICFrameType::CONNECTION_CLOSE: + return std::make_unique(Convert(frame)); + case QUICFrameType::HANDSHAKE_DONE: + return std::make_unique(Convert(frame)); + default: + ink_release_assert(0); + return nullptr; + } +} + +namespace Frame +{ + template + std::string + convert_to_string(T a) + { + return std::to_string(static_cast(a)); + } + + void + AckFrame::encode(YAML::Node &node) + { + node["frame_type"] = "ack"; + node["ack_delay"] = std::to_string(ack_delay); + for (auto &it : acked_range) { + YAML::Node sub; + sub.push_back(convert_to_string(it.first())); + sub.push_back(convert_to_string(it.last())); + node["acked_ranges"].push_back(sub); + } + + if (ect1) { + node["ect1"] = ect1; + } + + if (ect1) { + node["ect0"] = ect0; + } + + if (ce) { + node["ce"] = ce; + } + } + + void + StreamFrame::encode(YAML::Node &node) + { + node["frame_type"] = "stream"; + node["stream_id"] = stream_id; + node["offset"] = offset; + node["length"] = length; + node["fin"] = fin; + } + + void + PaddingFrame::encode(YAML::Node &node) + { + node["frame_type"] = "padding"; + } + + void + PingFrame::encode(YAML::Node &node) + { + node["frame_type"] = "ping"; + } + + void + RstStreamFrame::encode(YAML::Node &node) + { + node["frame_type"] = "reset_stream"; + node["stream_id"] = stream_id; + node["error_code"] = error_code; + node["final_size"] = final_size; + } + + void + StopSendingFrame::encode(YAML::Node &node) + { + node["frame_type"] = "stop_sending"; + node["stream_id"] = stream_id; + node["error_code"] = error_code; + } + + void + CryptoFrame::encode(YAML::Node &node) + { + node["frame_type"] = "crypto"; + node["offset"] = offset; + node["length"] = length; + } + + void + NewTokenFrame::encode(YAML::Node &node) + { + node["frame_type"] = "new_token"; + node["token"] = token; + node["length"] = length; + } + + void + MaxDataFrame::encode(YAML::Node &node) + { + node["frame_type"] = "max_data"; + node["maximum"] = maximum; + } + + void + MaxStreamDataFrame::encode(YAML::Node &node) + { + node["frame_type"] = "max_stream_data"; + node["maximum"] = maximum; + node["stream_id"] = stream_id; + } + + void + MaxStreamsFrame::encode(YAML::Node &node) + { + node["frame_type"] = "max_streams"; + node["maximum"] = maximum; + node["stream_type"] = stream_type; + } + + void + DataBlockedFrame::encode(YAML::Node &node) + { + node["frame_type"] = "data_blocked"; + node["limit"] = limit; + } + + void + StreamDataBlockedFrame::encode(YAML::Node &node) + { + node["frame_type"] = "stream_data_blocked"; + node["limit"] = limit; + node["stream_id"] = stream_id; + } + + void + StreamsBlockedFrame::encode(YAML::Node &node) + { + node["frame_type"] = "streams_blocked"; + node["stream_id"] = stream_id; + node["stream_type"] = stream_type; + } + + void + NewConnectionIDFrame::encode(YAML::Node &node) + { + node["frame_type"] = "new_connection_id"; + node["sequence_number"] = sequence_number; + node["retire_prior_to"] = retire_prior_to; + node["stateless_reset_token"] = stateless_reset_token; + node["length"] = length; + } + + void + RetireConnectionIDFrame::encode(YAML::Node &node) + { + node["frame_type"] = "retire_connection_id"; + node["sequence_number"] = sequence_number; + } + + void + PathChallengeFrame::encode(YAML::Node &node) + { + node["frame_type"] = "path_challenge"; + node["data"] = data; + } + + void + PathResponseFrame::encode(YAML::Node &node) + { + node["frame_type"] = "path_response"; + node["data"] = data; + } + + void + ConnectionCloseFrame::encode(YAML::Node &node) + { + node["frame_type"] = "connection_close"; + node["error_space"] = error_space; + node["error_code"] = error_code; + node["raw_error_code"] = raw_error_code; + node["reason"] = reason; + } + + void + HandshakeDoneFrame::encode(YAML::Node &node) + { + node["frame_type"] = "handshake_done"; + } + + void + UnknownFrame::encode(YAML::Node &node) + { + node["frame_type"] = "unknown"; + node["raw_frame_type"] = raw_frame_type; + } + +} // namespace Frame +} // namespace QLog diff --git a/iocore/net/quic/qlog/QLogFrame.h b/iocore/net/quic/qlog/QLogFrame.h new file mode 100644 index 00000000000..6dfb035b813 --- /dev/null +++ b/iocore/net/quic/qlog/QLogFrame.h @@ -0,0 +1,309 @@ +/** @file + * + * A brief file description + * + * @section license License + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include + +#include "QUICFrame.h" + +namespace QLog +{ +class QLogFrame +{ +public: + QLogFrame(QUICFrameType type) : _type(type) {} + virtual ~QLogFrame() {} + + QUICFrameType + type() const + { + return this->_type; + } + + // encode frame into YAML stype + virtual void encode(YAML::Node &node) = 0; + +protected: + QUICFrameType _type = QUICFrameType::UNKNOWN; +}; + +using QLogFrameUPtr = std::unique_ptr; + +// +// convert QUICFrame to QLogFrame +// +class QLogFrameFactory +{ +public: + // create QLogFrame + static QLogFrameUPtr create(const QUICFrame *frame); +}; + +namespace Frame +{ + struct AckFrame : public QLogFrame { + AckFrame(const QUICAckFrame &frame) : QLogFrame(frame.type()) + { + acked_range = frame.ranges(); + ack_delay = frame.ack_delay(); + if (frame.ecn_section()) { + ect0 = frame.ecn_section()->ect0_count(); + ect1 = frame.ecn_section()->ect1_count(); + ce = frame.ecn_section()->ecn_ce_count(); + } + } + + void encode(YAML::Node &) override; + + std::set acked_range; + uint64_t ect1 = 0; + uint64_t ect0 = 0; + uint64_t ce = 0; + uint64_t ack_delay = 0; + }; + + struct StreamFrame : public QLogFrame { + StreamFrame(const QUICStreamFrame &frame) : QLogFrame(frame.type()) + { + stream_id = std::to_string(static_cast(frame.stream_id())); + offset = std::to_string(static_cast(frame.offset())); + length = frame.data_length(); + fin = frame.has_fin_flag(); + } + + void encode(YAML::Node &) override; + std::string stream_id; + + // These two MUST always be set + // If not present in the Frame type, log their default values + std::string offset; + uint64_t length = 0; + + // this MAY be set any time, but MUST only be set if the value is "true" + // if absent, the value MUST be assumed to be "false" + bool fin = false; + + // FIXME raw + }; + + struct PaddingFrame : public QLogFrame { + PaddingFrame(const QUICPaddingFrame &frame) : QLogFrame(frame.type()) {} + void encode(YAML::Node &) override; + }; + + struct PingFrame : public QLogFrame { + PingFrame(const QUICPingFrame &frame) : QLogFrame(frame.type()) {} + void encode(YAML::Node &) override; + }; + + struct RstStreamFrame : public QLogFrame { + RstStreamFrame(const QUICRstStreamFrame &frame) : QLogFrame(frame.type()) + { + stream_id = std::to_string(static_cast(frame.stream_id())); + error_code = frame.error_code(); + final_size = std::to_string(frame.final_offset()); + } + + void encode(YAML::Node &) override; + std::string stream_id; + // FIXME ApplicationError + uint64_t error_code = 0; + std::string final_size; + }; + + struct StopSendingFrame : public QLogFrame { + StopSendingFrame(const QUICStopSendingFrame &frame) : QLogFrame(frame.type()) + { + stream_id = std::to_string(static_cast(frame.stream_id())); + error_code = frame.error_code(); + } + + void encode(YAML::Node &) override; + std::string stream_id; + // FIXME ApplicationError + uint64_t error_code = 0; + }; + + struct CryptoFrame : public QLogFrame { + CryptoFrame(const QUICCryptoFrame &frame) : QLogFrame(frame.type()) + { + offset = std::to_string(static_cast(frame.offset())); + length = frame.data_length(); + } + + void encode(YAML::Node &) override; + std::string offset; + uint64_t length = 0; + }; + + struct NewTokenFrame : public QLogFrame { + NewTokenFrame(const QUICNewTokenFrame &frame) : QLogFrame(frame.type()) + { + token = QUICBase::to_hex(frame.token(), frame.token_length()); + length = frame.token_length(); + } + + void encode(YAML::Node &) override; + std::string token; + uint64_t length = 0; + }; + + struct MaxDataFrame : public QLogFrame { + MaxDataFrame(const QUICMaxDataFrame &frame) : QLogFrame(frame.type()), maximum(std::to_string(frame.maximum_data())) {} + + void encode(YAML::Node &) override; + std::string maximum; + }; + + struct MaxStreamDataFrame : public QLogFrame { + MaxStreamDataFrame(const QUICMaxStreamDataFrame &frame) : QLogFrame(frame.type()) + { + stream_id = std::to_string(static_cast(frame.stream_id())); + maximum = std::to_string(frame.maximum_stream_data()); + } + + void encode(YAML::Node &) override; + std::string stream_id; + std::string maximum; + }; + + struct MaxStreamsFrame : public QLogFrame { + MaxStreamsFrame(const QUICMaxStreamsFrame &frame) : QLogFrame(frame.type()) + { + maximum = std::to_string(frame.maximum_streams()); + // FIXME + stream_type = "bidirectional"; + } + + void encode(YAML::Node &) override; + std::string stream_type; + std::string maximum; + }; + + struct DataBlockedFrame : public QLogFrame { + DataBlockedFrame(const QUICDataBlockedFrame &frame) : QLogFrame(frame.type()) + { + limit = std::to_string(static_cast(frame.offset())); + } + void encode(YAML::Node &) override; + std::string limit; + }; + + struct StreamDataBlockedFrame : public QLogFrame { + StreamDataBlockedFrame(const QUICStreamDataBlockedFrame &frame) : QLogFrame(frame.type()) + { + limit = std::to_string(static_cast(frame.offset())); + stream_id = std::to_string(static_cast(frame.stream_id())); + } + + void encode(YAML::Node &) override; + std::string stream_id, limit; + }; + + struct StreamsBlockedFrame : public QLogFrame { + StreamsBlockedFrame(const QUICStreamIdBlockedFrame &frame) : QLogFrame(frame.type()) + { + stream_type = "bidirectional"; + stream_id = std::to_string(static_cast(frame.stream_id())); + } + + void encode(YAML::Node &) override; + std::string stream_id, stream_type; + }; + + struct NewConnectionIDFrame : public QLogFrame { + NewConnectionIDFrame(const QUICNewConnectionIdFrame &frame) + : QLogFrame(frame.type()), sequence_number(std::to_string(frame.sequence())) + { + retire_prior_to = std::to_string(frame.retire_prior_to()); + connection_id = frame.connection_id().hex(); + stateless_reset_token = QUICBase::to_hex(frame.stateless_reset_token().buf(), QUICStatelessResetToken::LEN); + length = frame.connection_id().length(); + } + + void encode(YAML::Node &) override; + std::string sequence_number, retire_prior_to, connection_id, stateless_reset_token; + uint8_t length = 0; + }; + + struct RetireConnectionIDFrame : public QLogFrame { + RetireConnectionIDFrame(const QUICRetireConnectionIdFrame &frame) + : QLogFrame(frame.type()), sequence_number(std::to_string(frame.seq_num())) + { + } + void encode(YAML::Node &) override; + std::string sequence_number; + }; + + struct PathChallengeFrame : public QLogFrame { + PathChallengeFrame(const QUICPathChallengeFrame &frame) + : QLogFrame(frame.type()), data(QUICBase::to_hex(frame.data(), QUICPathChallengeFrame::DATA_LEN)) + { + } + void encode(YAML::Node &) override; + std::string data; + }; + + struct PathResponseFrame : public QLogFrame { + PathResponseFrame(const QUICPathResponseFrame &frame) + : QLogFrame(frame.type()), data(QUICBase::to_hex(frame.data(), QUICPathChallengeFrame::DATA_LEN)) + { + } + void encode(YAML::Node &) override; + std::string data; + }; + + struct ConnectionCloseFrame : public QLogFrame { + ConnectionCloseFrame(const QUICConnectionCloseFrame &frame, bool app = false) + : QLogFrame(frame.type()), error_space(app ? "application" : "transport") + { + error_code = frame.error_code(); + // FIXME + raw_error_code = error_code; + reason = frame.reason_phrase(); + } + + void encode(YAML::Node &) override; + std::string error_space, reason, trigger_frame_type; + uint64_t error_code, raw_error_code; + }; + + struct HandshakeDoneFrame : public QLogFrame { + HandshakeDoneFrame(const QUICHandshakeDoneFrame &frame) : QLogFrame(frame.type()){}; + void encode(YAML::Node &) override; + }; + + struct UnknownFrame : public QLogFrame { + UnknownFrame(const QUICUnknownFrame &frame) : QLogFrame(frame.type()) + { + // FIXME + raw_frame_type = static_cast(frame.type()); + } + + void encode(YAML::Node &) override; + uint8_t raw_frame_type = 0; + }; +} // namespace Frame +} // namespace QLog diff --git a/iocore/net/quic/qlog/QLogListener.h b/iocore/net/quic/qlog/QLogListener.h new file mode 100644 index 00000000000..9337b075f5d --- /dev/null +++ b/iocore/net/quic/qlog/QLogListener.h @@ -0,0 +1,119 @@ +/** @file + * + * A brief file description + * + * @section license License + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "QLog.h" +#include "QLogUtils.h" +#include "QUICPacket.h" +#include "QUICContext.h" + +namespace QLog +{ +class QLogListener : public QUICCallback +{ +public: + QLogListener(QUICContext &ctx, std::string odcid, std::string title = "", std::string desc = "") : _context(ctx) + { + this->_log.new_trace(odcid, title, desc); // initial new trace + } + + void + frame_recv_callback(QUICCallbackContext &, const QUICFrame &frame) override + { + this->_recv_frames.push_back(QLogFrameFactory::create(static_cast(&frame))); + } + + void + frame_packetize_callback(QUICCallbackContext &, const QUICFrame &frame) override + { + this->_send_frames.push_back(QLogFrameFactory::create(static_cast(&frame))); + } + + void + packet_send_callback(QUICCallbackContext &, const QUICPacket &packet) override + { + auto qe = std::make_unique(PacketTypeToName(packet.type()), QUICPacketToLogPacket(packet)); + for (auto &it : this->_send_frames) { + qe->append_frames(std::move(it)); + } + this->_send_frames.clear(); + this->_log.last_trace().push_event(std::move(qe)); + }; + + void + packet_recv_callback(QUICCallbackContext &, const QUICPacket &packet) override + { + auto qe = std::make_unique(PacketTypeToName(packet.type()), QUICPacketToLogPacket(packet)); + for (auto &it : this->_recv_frames) { + qe->append_frames(std::move(it)); + } + this->_recv_frames.clear(); + this->_log.last_trace().push_event(std::move(qe)); + }; + + void + packet_lost_callback(QUICCallbackContext &, const QUICSentPacketInfo &packet) override + { + auto qe = std::make_unique(PacketTypeToName(packet.type), packet.packet_number); + this->_log.last_trace().push_event(std::move(qe)); + }; + + void + cc_metrics_update_callback(QUICCallbackContext &, uint64_t congestion_window, uint64_t bytes_in_flight, uint64_t sshresh) override + { + auto qe = std::make_unique(); + qe->set_congestion_window(static_cast(congestion_window)).set_bytes_in_flight(bytes_in_flight).set_ssthresh(sshresh); + this->_log.last_trace().push_event(std::move(qe)); + } + + void + congestion_state_updated_callback(QUICCallbackContext &, QUICCongestionController::State state) override + { + if (state != this->_state) { + this->_log.last_trace().push_event(std::make_unique(CongestionStateConvert(state))); + this->_state = state; + } + } + + void + connection_close_callback(QUICCallbackContext &) override + { + this->_log.dump(this->_context.config()->qlog_dir()); + } + + Trace & + last_trace() + { + return this->_log.last_trace(); + } + +private: + QUICCongestionController::State _state = QUICCongestionController::State::SLOW_START; + std::vector _recv_frames; + std::vector _send_frames; + QLog _log; + QUICContext &_context; +}; + +} // namespace QLog diff --git a/iocore/net/quic/qlog/QLogUtils.h b/iocore/net/quic/qlog/QLogUtils.h new file mode 100644 index 00000000000..d39fce8c89f --- /dev/null +++ b/iocore/net/quic/qlog/QLogUtils.h @@ -0,0 +1,80 @@ +/** @file + * + * A brief file description + * + * @section license License + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "QLog.h" +#include "QUICPacket.h" + +namespace QLog +{ +inline static const char * +PacketTypeToName(QUICPacketType pt) +{ + switch (pt) { + case QUICPacketType::INITIAL: + return "initial"; + case QUICPacketType::HANDSHAKE: + return "handshake"; + case QUICPacketType::ZERO_RTT_PROTECTED: + return "0rtt"; + case QUICPacketType::PROTECTED: + return "1rtt"; + case QUICPacketType::RETRY: + return "retry"; + case QUICPacketType::VERSION_NEGOTIATION: + return "version_negotiation"; + case QUICPacketType::STATELESS_RESET: + return "stateless_reset"; + default: + return "unknown"; + } +} + +inline static Recovery::CongestionStateUpdated::State +CongestionStateConvert(QUICCongestionController::State state) +{ + switch (state) { + case QUICCongestionController::State::APPLICATION_LIMITED: + return Recovery::CongestionStateUpdated::State::application_limited; + case QUICCongestionController::State::SLOW_START: + return Recovery::CongestionStateUpdated::State::slow_start; + case QUICCongestionController::State::CONGESTION_AVOIDANCE: + return Recovery::CongestionStateUpdated::State::congestion_avoidance; + case QUICCongestionController::State::RECOVERY: + return Recovery::CongestionStateUpdated::State::recovery; + default: + return Recovery::CongestionStateUpdated::State::slow_start; + } +} + +inline static PacketHeader +QUICPacketToLogPacket(const QUICPacket &packet) +{ + PacketHeader ph; + ph.dcid = packet.destination_cid().hex(); + ph.packet_number = std::to_string(packet.packet_number()); + ph.packet_size = packet.size(); + ph.payload_length = packet.payload_length(); + return ph; +} + +} // namespace QLog diff --git a/iocore/net/quic/test/event_processor_main.cc b/iocore/net/quic/test/event_processor_main.cc index 4c1d2dd5b3c..8963e9f2c17 100644 --- a/iocore/net/quic/test/event_processor_main.cc +++ b/iocore/net/quic/test/event_processor_main.cc @@ -43,7 +43,7 @@ struct EventProcessorListener : Catch::TestEventListenerBase { testRunStarting(Catch::TestRunInfo const &testRunInfo) override { BaseLogFile *base_log_file = new BaseLogFile("stderr"); - diags = new Diags(testRunInfo.name.c_str(), "" /* tags */, "" /* actions */, base_log_file); + diags = new Diags(testRunInfo.name, "" /* tags */, "" /* actions */, base_log_file); diags->activate_taglist("vv_quic|quic", DiagsTagType_Debug); diags->config.enabled[DiagsTagType_Debug] = true; diags->show_location = SHOW_LOCATION_DEBUG; diff --git a/iocore/net/quic/test/main.cc b/iocore/net/quic/test/main.cc index 96767a2f97b..4e7ee5adc6f 100644 --- a/iocore/net/quic/test/main.cc +++ b/iocore/net/quic/test/main.cc @@ -30,6 +30,7 @@ #include "tscore/Diags.h" #include "RecordsConfig.h" +#include "QUICGlobals.h" #include "QUICConfig.h" struct EventProcessorListener : Catch::TestEventListenerBase { @@ -39,7 +40,7 @@ struct EventProcessorListener : Catch::TestEventListenerBase { testRunStarting(Catch::TestRunInfo const &testRunInfo) override { BaseLogFile *base_log_file = new BaseLogFile("stderr"); - diags = new Diags(testRunInfo.name.c_str(), "" /* tags */, "" /* actions */, base_log_file); + diags = new Diags(testRunInfo.name, "" /* tags */, "" /* actions */, base_log_file); diags->activate_taglist("vv_quic|quic", DiagsTagType_Debug); diags->config.enabled[DiagsTagType_Debug] = true; diags->show_location = SHOW_LOCATION_DEBUG; @@ -49,6 +50,7 @@ struct EventProcessorListener : Catch::TestEventListenerBase { LibRecordsConfigInit(); QUICConfig::startup(); + QUIC::init(); EThread *thread = new EThread(); thread->set_specific(); diff --git a/iocore/net/quic/test/test_QUICAckFrameCreator.cc b/iocore/net/quic/test/test_QUICAckFrameCreator.cc index 99b40e73555..eb96869c90f 100644 --- a/iocore/net/quic/test/test_QUICAckFrameCreator.cc +++ b/iocore/net/quic/test/test_QUICAckFrameCreator.cc @@ -32,63 +32,67 @@ TEST_CASE("QUICAckFrameManager", "[quic]") uint8_t frame_buf[QUICFrame::MAX_INSTANCE_SIZE]; // Initial state - QUICFrame *ack_frame = ack_manager.generate_frame(frame_buf, level, UINT16_MAX, UINT16_MAX, 0); + QUICFrame *ack_frame = ack_manager.generate_frame(frame_buf, level, UINT16_MAX, UINT16_MAX, 0, 0); QUICAckFrame *frame = static_cast(ack_frame); CHECK(frame == nullptr); // One packet ack_manager.update(level, 1, 1, false); - ack_frame = ack_manager.generate_frame(frame_buf, level, UINT16_MAX, UINT16_MAX, 0); + ack_frame = ack_manager.generate_frame(frame_buf, level, UINT16_MAX, UINT16_MAX, 0, 0); frame = static_cast(ack_frame); CHECK(frame != nullptr); CHECK(frame->ack_block_count() == 0); CHECK(frame->largest_acknowledged() == 1); CHECK(frame->ack_block_section()->first_ack_block() == 0); + ack_frame->~QUICFrame(); // retry - CHECK(ack_manager.will_generate_frame(level, 0) == false); + CHECK(ack_manager.will_generate_frame(level, 0, true, 0) == false); // Not sequential ack_manager.update(level, 2, 1, false); ack_manager.update(level, 5, 1, false); ack_manager.update(level, 3, 1, false); ack_manager.update(level, 4, 1, false); - ack_frame = ack_manager.generate_frame(frame_buf, level, UINT16_MAX, UINT16_MAX, 0); + ack_frame = ack_manager.generate_frame(frame_buf, level, UINT16_MAX, UINT16_MAX, 0, 0); frame = static_cast(ack_frame); CHECK(frame != nullptr); CHECK(frame->ack_block_count() == 0); CHECK(frame->largest_acknowledged() == 5); CHECK(frame->ack_block_section()->first_ack_block() == 4); + ack_frame->~QUICFrame(); // Loss ack_manager.update(level, 6, 1, false); ack_manager.update(level, 7, 1, false); ack_manager.update(level, 10, 1, false); - ack_frame = ack_manager.generate_frame(frame_buf, level, UINT16_MAX, UINT16_MAX, 0); + ack_frame = ack_manager.generate_frame(frame_buf, level, UINT16_MAX, UINT16_MAX, 0, 0); frame = static_cast(ack_frame); CHECK(frame != nullptr); CHECK(frame->ack_block_count() == 1); CHECK(frame->largest_acknowledged() == 10); CHECK(frame->ack_block_section()->first_ack_block() == 0); CHECK(frame->ack_block_section()->begin()->gap() == 1); + ack_frame->~QUICFrame(); // on frame acked ack_manager.on_frame_acked(frame->id()); - CHECK(ack_manager.will_generate_frame(level, 0) == false); - ack_frame = ack_manager.generate_frame(frame_buf, level, UINT16_MAX, UINT16_MAX, 0); + CHECK(ack_manager.will_generate_frame(level, 0, true, 0) == false); + ack_frame = ack_manager.generate_frame(frame_buf, level, UINT16_MAX, UINT16_MAX, 0, 0); CHECK(ack_frame == nullptr); ack_manager.update(level, 11, 1, false); ack_manager.update(level, 12, 1, false); ack_manager.update(level, 13, 1, false); - ack_frame = ack_manager.generate_frame(frame_buf, level, UINT16_MAX, UINT16_MAX, 0); + ack_frame = ack_manager.generate_frame(frame_buf, level, UINT16_MAX, UINT16_MAX, 0, 0); frame = static_cast(ack_frame); CHECK(frame != nullptr); CHECK(frame->ack_block_count() == 0); CHECK(frame->largest_acknowledged() == 13); CHECK(frame->ack_block_section()->first_ack_block() == 2); CHECK(frame->ack_block_section()->begin()->gap() == 0); + ack_frame->~QUICFrame(); ack_manager.on_frame_acked(frame->id()); @@ -96,17 +100,18 @@ TEST_CASE("QUICAckFrameManager", "[quic]") ack_manager.update(level, 14, 1, true); ack_manager.update(level, 15, 1, true); ack_manager.update(level, 16, 1, true); - CHECK(ack_manager.will_generate_frame(level, 0) == false); - ack_frame = ack_manager.generate_frame(frame_buf, level, UINT16_MAX, UINT16_MAX, 0); + CHECK(ack_manager.will_generate_frame(level, 0, true, 0) == false); + ack_frame = ack_manager.generate_frame(frame_buf, level, UINT16_MAX, UINT16_MAX, 0, 0); ack_manager.update(level, 17, 1, false); - ack_frame = ack_manager.generate_frame(frame_buf, level, UINT16_MAX, UINT16_MAX, 0); + ack_frame = ack_manager.generate_frame(frame_buf, level, UINT16_MAX, UINT16_MAX, 0, 0); frame = static_cast(ack_frame); CHECK(frame != nullptr); CHECK(frame->ack_block_count() == 0); CHECK(frame->largest_acknowledged() == 17); CHECK(frame->ack_block_section()->first_ack_block() == 3); CHECK(frame->ack_block_section()->begin()->gap() == 0); + ack_frame->~QUICFrame(); } TEST_CASE("QUICAckFrameManager should send", "[quic]") @@ -117,7 +122,7 @@ TEST_CASE("QUICAckFrameManager should send", "[quic]") QUICEncryptionLevel level = QUICEncryptionLevel::ONE_RTT; ack_manager.update(level, 2, 1, false); - CHECK(ack_manager.will_generate_frame(level, 0) == true); + CHECK(ack_manager.will_generate_frame(level, 0, true, 0) == true); } SECTION("QUIC delay ack and unorder packet", "[quic]") @@ -126,13 +131,36 @@ TEST_CASE("QUICAckFrameManager should send", "[quic]") QUICEncryptionLevel level = QUICEncryptionLevel::ONE_RTT; ack_manager.update(level, 0, 1, false); - CHECK(ack_manager.will_generate_frame(level, 0) == false); + CHECK(ack_manager.will_generate_frame(level, 0, true, 0) == false); + + ack_manager.update(level, 1, 1, true); + CHECK(ack_manager.will_generate_frame(level, 0, true, 0) == false); + + ack_manager.update(level, 3, 1, false); + CHECK(ack_manager.will_generate_frame(level, 0, true, 0) == true); + } + + SECTION("QUIC every two ack eliciting packet", "[quic]") + { + QUICAckFrameManager ack_manager; + uint8_t frame_buf[QUICFrame::MAX_INSTANCE_SIZE]; + + QUICEncryptionLevel level = QUICEncryptionLevel::ONE_RTT; + ack_manager.update(level, 0, 1, false); + CHECK(ack_manager.will_generate_frame(level, 0, true, 0) == false); ack_manager.update(level, 1, 1, false); - CHECK(ack_manager.will_generate_frame(level, 0) == false); + CHECK(ack_manager.will_generate_frame(level, 0, true, 0) == true); + + CHECK(ack_manager.generate_frame(frame_buf, level, UINT16_MAX, UINT16_MAX, 0, 0) != nullptr); + + ack_manager.update(level, 2, 1, false); + CHECK(ack_manager.will_generate_frame(level, 0, true, 0) == false); ack_manager.update(level, 3, 1, false); - CHECK(ack_manager.will_generate_frame(level, 0) == true); + CHECK(ack_manager.will_generate_frame(level, 0, true, 0) == true); + + CHECK(ack_manager.generate_frame(frame_buf, level, UINT16_MAX, UINT16_MAX, 0, 0) != nullptr); } SECTION("QUIC delay too much time", "[quic]") @@ -142,21 +170,21 @@ TEST_CASE("QUICAckFrameManager should send", "[quic]") QUICEncryptionLevel level = QUICEncryptionLevel::ONE_RTT; ack_manager.update(level, 0, 1, false); - CHECK(ack_manager.will_generate_frame(level, 0) == false); + CHECK(ack_manager.will_generate_frame(level, 0, true, 0) == false); sleep(1); Thread::get_hrtime_updated(); ack_manager.update(level, 1, 1, false); - CHECK(ack_manager.will_generate_frame(level, 0) == true); + CHECK(ack_manager.will_generate_frame(level, 0, true, 0) == true); } - SECTION("QUIC intial packet", "[quic]") + SECTION("QUIC initial packet", "[quic]") { QUICAckFrameManager ack_manager; QUICEncryptionLevel level = QUICEncryptionLevel::INITIAL; ack_manager.update(level, 0, 1, false); - CHECK(ack_manager.will_generate_frame(level, 0) == true); + CHECK(ack_manager.will_generate_frame(level, 0, true, 0) == true); } SECTION("QUIC handshake packet", "[quic]") @@ -165,7 +193,7 @@ TEST_CASE("QUICAckFrameManager should send", "[quic]") QUICEncryptionLevel level = QUICEncryptionLevel::HANDSHAKE; ack_manager.update(level, 0, 1, false); - CHECK(ack_manager.will_generate_frame(level, 0) == true); + CHECK(ack_manager.will_generate_frame(level, 0, true, 0) == true); } SECTION("QUIC frame fired", "[quic]") @@ -174,11 +202,11 @@ TEST_CASE("QUICAckFrameManager should send", "[quic]") QUICEncryptionLevel level = QUICEncryptionLevel::ONE_RTT; ack_manager.update(level, 0, 1, false); - CHECK(ack_manager.will_generate_frame(level, 0) == false); + CHECK(ack_manager.will_generate_frame(level, 0, true, 0) == false); sleep(1); Thread::get_hrtime_updated(); - CHECK(ack_manager.will_generate_frame(level, 0) == true); + CHECK(ack_manager.will_generate_frame(level, 0, true, 0) == true); } SECTION("QUIC refresh frame", "[quic]") @@ -187,27 +215,28 @@ TEST_CASE("QUICAckFrameManager should send", "[quic]") QUICEncryptionLevel level = QUICEncryptionLevel::ONE_RTT; uint8_t frame_buf[QUICFrame::MAX_INSTANCE_SIZE]; - QUICFrame *ack_frame = ack_manager.generate_frame(frame_buf, level, UINT16_MAX, UINT16_MAX, 0); + QUICFrame *ack_frame = ack_manager.generate_frame(frame_buf, level, UINT16_MAX, UINT16_MAX, 0, 0); QUICAckFrame *frame = static_cast(ack_frame); CHECK(frame == nullptr); // unorder frame should sent immediately ack_manager.update(level, 1, 1, false); - CHECK(ack_manager.will_generate_frame(level, 0) == true); + CHECK(ack_manager.will_generate_frame(level, 0, true, 0) == true); ack_manager.update(level, 2, 1, false); // Delay due to some reason, the frame is not valued any more, but still valued sleep(1); Thread::get_hrtime_updated(); - CHECK(ack_manager.will_generate_frame(level, 0) == true); - ack_frame = ack_manager.generate_frame(frame_buf, level, UINT16_MAX, UINT16_MAX, 0); + CHECK(ack_manager.will_generate_frame(level, 0, true, 0) == true); + ack_frame = ack_manager.generate_frame(frame_buf, level, UINT16_MAX, UINT16_MAX, 0, 0); frame = static_cast(ack_frame); CHECK(frame->ack_block_count() == 0); CHECK(frame->largest_acknowledged() == 2); CHECK(frame->ack_block_section()->first_ack_block() == 1); CHECK(frame->ack_block_section()->begin()->gap() == 0); + ack_frame->~QUICFrame(); } } @@ -218,7 +247,7 @@ TEST_CASE("QUICAckFrameManager_loss_recover", "[quic]") // Initial state uint8_t frame_buf[QUICFrame::MAX_INSTANCE_SIZE]; - QUICFrame *ack_frame = ack_manager.generate_frame(frame_buf, level, UINT16_MAX, UINT16_MAX, 0); + QUICFrame *ack_frame = ack_manager.generate_frame(frame_buf, level, UINT16_MAX, UINT16_MAX, 0, 0); QUICAckFrame *frame = static_cast(ack_frame); CHECK(frame == nullptr); @@ -228,31 +257,33 @@ TEST_CASE("QUICAckFrameManager_loss_recover", "[quic]") ack_manager.update(level, 8, 1, false); ack_manager.update(level, 9, 1, false); - ack_frame = ack_manager.generate_frame(frame_buf, level, UINT16_MAX, UINT16_MAX, 0); + ack_frame = ack_manager.generate_frame(frame_buf, level, UINT16_MAX, UINT16_MAX, 0, 0); frame = static_cast(ack_frame); CHECK(frame != nullptr); CHECK(frame->ack_block_count() == 2); CHECK(frame->largest_acknowledged() == 9); CHECK(frame->ack_block_section()->first_ack_block() == 1); CHECK(frame->ack_block_section()->begin()->gap() == 0); + ack_frame->~QUICFrame(); - CHECK(ack_manager.will_generate_frame(level, 0) == false); + CHECK(ack_manager.will_generate_frame(level, 0, true, 0) == false); ack_manager.update(level, 7, 1, false); ack_manager.update(level, 4, 1, false); - ack_frame = ack_manager.generate_frame(frame_buf, level, UINT16_MAX, UINT16_MAX, 0); + ack_frame = ack_manager.generate_frame(frame_buf, level, UINT16_MAX, UINT16_MAX, 0, 0); frame = static_cast(ack_frame); CHECK(frame != nullptr); CHECK(frame->ack_block_count() == 1); CHECK(frame->largest_acknowledged() == 9); CHECK(frame->ack_block_section()->first_ack_block() == 5); CHECK(frame->ack_block_section()->begin()->gap() == 0); + ack_frame->~QUICFrame(); } TEST_CASE("QUICAckFrameManager_QUICAckFrameCreator", "[quic]") { QUICAckFrameManager ack_manager; - QUICAckFrameManager::QUICAckFrameCreator packet_numbers(QUICPacketNumberSpace::Initial, &ack_manager); + QUICAckFrameManager::QUICAckFrameCreator packet_numbers(QUICPacketNumberSpace::INITIAL, &ack_manager); CHECK(packet_numbers.size() == 0); CHECK(packet_numbers.largest_ack_number() == 0); @@ -296,7 +327,7 @@ TEST_CASE("QUICAckFrameManager lost_frame", "[quic]") uint8_t frame_buf[QUICFrame::MAX_INSTANCE_SIZE]; // Initial state - QUICFrame *ack_frame = ack_manager.generate_frame(frame_buf, level, UINT16_MAX, UINT16_MAX, 0); + QUICFrame *ack_frame = ack_manager.generate_frame(frame_buf, level, UINT16_MAX, UINT16_MAX, 0, 0); QUICAckFrame *frame = static_cast(ack_frame); CHECK(frame == nullptr); @@ -306,39 +337,42 @@ TEST_CASE("QUICAckFrameManager lost_frame", "[quic]") ack_manager.update(level, 8, 1, false); ack_manager.update(level, 9, 1, false); - ack_frame = ack_manager.generate_frame(frame_buf, level, UINT16_MAX, UINT16_MAX, 0); + ack_frame = ack_manager.generate_frame(frame_buf, level, UINT16_MAX, UINT16_MAX, 0, 0); frame = static_cast(ack_frame); CHECK(frame != nullptr); CHECK(frame->ack_block_count() == 2); CHECK(frame->largest_acknowledged() == 9); CHECK(frame->ack_block_section()->first_ack_block() == 1); CHECK(frame->ack_block_section()->begin()->gap() == 0); + ack_frame->~QUICFrame(); ack_manager.on_frame_lost(frame->id()); - CHECK(ack_manager.will_generate_frame(level, 0) == true); - ack_frame = ack_manager.generate_frame(frame_buf, level, UINT16_MAX, UINT16_MAX, 0); + CHECK(ack_manager.will_generate_frame(level, 0, true, 0) == true); + ack_frame = ack_manager.generate_frame(frame_buf, level, UINT16_MAX, UINT16_MAX, 0, 0); frame = static_cast(ack_frame); CHECK(frame != nullptr); CHECK(frame->ack_block_count() == 2); CHECK(frame->largest_acknowledged() == 9); CHECK(frame->ack_block_section()->first_ack_block() == 1); CHECK(frame->ack_block_section()->begin()->gap() == 0); + ack_frame->~QUICFrame(); - CHECK(ack_manager.will_generate_frame(level, 0) == false); + CHECK(ack_manager.will_generate_frame(level, 0, true, 0) == false); ack_manager.on_frame_lost(frame->id()); - CHECK(ack_manager.will_generate_frame(level, 0) == true); + CHECK(ack_manager.will_generate_frame(level, 0, true, 0) == true); ack_manager.update(level, 7, 1, false); ack_manager.update(level, 4, 1, false); - ack_frame = ack_manager.generate_frame(frame_buf, level, UINT16_MAX, UINT16_MAX, 0); + ack_frame = ack_manager.generate_frame(frame_buf, level, UINT16_MAX, UINT16_MAX, 0, 0); frame = static_cast(ack_frame); CHECK(frame != nullptr); CHECK(frame->ack_block_count() == 1); CHECK(frame->largest_acknowledged() == 9); CHECK(frame->ack_block_section()->first_ack_block() == 5); CHECK(frame->ack_block_section()->begin()->gap() == 0); + ack_frame->~QUICFrame(); - CHECK(ack_manager.will_generate_frame(level, 0) == false); + CHECK(ack_manager.will_generate_frame(level, 0, true, 0) == false); } TEST_CASE("QUICAckFrameManager ack only packet", "[quic]") @@ -350,7 +384,7 @@ TEST_CASE("QUICAckFrameManager ack only packet", "[quic]") uint8_t frame_buf[QUICFrame::MAX_INSTANCE_SIZE]; // Initial state - QUICFrame *ack_frame = ack_manager.generate_frame(frame_buf, level, UINT16_MAX, UINT16_MAX, 0); + QUICFrame *ack_frame = ack_manager.generate_frame(frame_buf, level, UINT16_MAX, UINT16_MAX, 0, 0); QUICAckFrame *frame = static_cast(ack_frame); CHECK(frame == nullptr); @@ -360,19 +394,20 @@ TEST_CASE("QUICAckFrameManager ack only packet", "[quic]") ack_manager.update(level, 4, 1, false); ack_manager.update(level, 5, 1, false); - CHECK(ack_manager.will_generate_frame(level, 0) == true); + CHECK(ack_manager.will_generate_frame(level, 0, true, 0) == true); - ack_frame = ack_manager.generate_frame(frame_buf, level, UINT16_MAX, UINT16_MAX, 0); + ack_frame = ack_manager.generate_frame(frame_buf, level, UINT16_MAX, UINT16_MAX, 0, 0); frame = static_cast(ack_frame); CHECK(frame != nullptr); CHECK(frame->ack_block_count() == 0); CHECK(frame->largest_acknowledged() == 5); CHECK(frame->ack_block_section()->first_ack_block() == 4); CHECK(frame->ack_block_section()->begin()->gap() == 0); + ack_frame->~QUICFrame(); ack_manager.update(level, 6, 1, true); ack_manager.update(level, 7, 1, true); - CHECK(ack_manager.will_generate_frame(level, 0) == false); + CHECK(ack_manager.will_generate_frame(level, 0, true, 0) == false); } SECTION("ONE_RTT") @@ -382,7 +417,7 @@ TEST_CASE("QUICAckFrameManager ack only packet", "[quic]") uint8_t frame_buf[QUICFrame::MAX_INSTANCE_SIZE]; // Initial state - QUICFrame *ack_frame = ack_manager.generate_frame(frame_buf, level, UINT16_MAX, UINT16_MAX, 0); + QUICFrame *ack_frame = ack_manager.generate_frame(frame_buf, level, UINT16_MAX, UINT16_MAX, 0, 0); QUICAckFrame *frame = static_cast(ack_frame); CHECK(frame == nullptr); @@ -392,18 +427,19 @@ TEST_CASE("QUICAckFrameManager ack only packet", "[quic]") ack_manager.update(level, 4, 1, false); ack_manager.update(level, 5, 1, false); - CHECK(ack_manager.will_generate_frame(level, 0) == true); + CHECK(ack_manager.will_generate_frame(level, 0, true, 0) == true); - ack_frame = ack_manager.generate_frame(frame_buf, level, UINT16_MAX, UINT16_MAX, 0); + ack_frame = ack_manager.generate_frame(frame_buf, level, UINT16_MAX, UINT16_MAX, 0, 0); frame = static_cast(ack_frame); CHECK(frame != nullptr); CHECK(frame->ack_block_count() == 0); CHECK(frame->largest_acknowledged() == 5); CHECK(frame->ack_block_section()->first_ack_block() == 4); CHECK(frame->ack_block_section()->begin()->gap() == 0); + ack_frame->~QUICFrame(); ack_manager.update(level, 6, 1, true); ack_manager.update(level, 7, 1, true); - CHECK(ack_manager.will_generate_frame(level, 0) == false); + CHECK(ack_manager.will_generate_frame(level, 0, true, 0) == false); } } diff --git a/iocore/net/quic/test/test_QUICAltConnectionManager.cc b/iocore/net/quic/test/test_QUICAltConnectionManager.cc index d30da7b390a..298f21b775e 100644 --- a/iocore/net/quic/test/test_QUICAltConnectionManager.cc +++ b/iocore/net/quic/test/test_QUICAltConnectionManager.cc @@ -58,6 +58,7 @@ TEST_CASE("QUICPreferredAddress", "[quic]") CHECK(memcmp(pref_addr->endpoint_ipv6().sin6.sin6_addr.s6_addr, ipv6_addr.s6_addr, 16) == 0); CHECK(pref_addr->cid() == cid55); CHECK(memcmp(pref_addr->token().buf(), buf + 26, 16) == 0); + delete pref_addr; } SECTION("store") @@ -85,6 +86,7 @@ TEST_CASE("QUICPreferredAddress", "[quic]") pref_addr->store(actual, len); CHECK(sizeof(buf) == len); CHECK(memcmp(buf, actual, sizeof(buf)) == 0); + delete pref_addr; } SECTION("unavailable") @@ -93,5 +95,6 @@ TEST_CASE("QUICPreferredAddress", "[quic]") CHECK(!pref_addr->is_available()); CHECK(!pref_addr->has_ipv4()); CHECK(!pref_addr->has_ipv6()); + delete pref_addr; } } diff --git a/iocore/net/quic/test/test_QUICFlowController.cc b/iocore/net/quic/test/test_QUICFlowController.cc index d58a31172a7..598b74efe2f 100644 --- a/iocore/net/quic/test/test_QUICFlowController.cc +++ b/iocore/net/quic/test/test_QUICFlowController.cc @@ -116,7 +116,7 @@ TEST_CASE("QUICFlowController_Local_Connection", "[quic]") fc.forward_limit(2048); CHECK(fc.current_offset() == 1024); CHECK(fc.current_limit() == 2048); - QUICFrame *frame = fc.generate_frame(frame_buf, QUICEncryptionLevel::ONE_RTT, 0, 1024, 0); + QUICFrame *frame = fc.generate_frame(frame_buf, QUICEncryptionLevel::ONE_RTT, 0, 1024, 0, 0); CHECK(frame); CHECK(frame->type() == QUICFrameType::MAX_DATA); @@ -168,7 +168,7 @@ TEST_CASE("QUICFlowController_Remote_Connection", "[quic]") CHECK(fc.current_offset() == 1000); CHECK(fc.current_limit() == 1024); CHECK(ret != 0); - QUICFrame *frame = fc.generate_frame(frame_buf, QUICEncryptionLevel::ONE_RTT, 0, 1024, 0); + QUICFrame *frame = fc.generate_frame(frame_buf, QUICEncryptionLevel::ONE_RTT, 0, 1024, 0, 0); CHECK(frame); CHECK(frame->type() == QUICFrameType::DATA_BLOCKED); @@ -199,9 +199,44 @@ TEST_CASE("QUICFlowController_Remote_Connection_ZERO_Credit", "[quic]") CHECK(fc.current_limit() == 1024); CHECK(ret == 0); - CHECK(fc.will_generate_frame(QUICEncryptionLevel::ONE_RTT, 0)); + CHECK(fc.will_generate_frame(QUICEncryptionLevel::ONE_RTT, 0, true, 0)); // if there're anything to send - QUICFrame *frame = fc.generate_frame(frame_buf, QUICEncryptionLevel::ONE_RTT, 0, 1024, 0); + QUICFrame *frame = fc.generate_frame(frame_buf, QUICEncryptionLevel::ONE_RTT, 0, 1024, 0, 0); + CHECK(frame); + CHECK(frame->type() == QUICFrameType::DATA_BLOCKED); + + // MAX_STREAM_DATA + fc.forward_limit(2048); + CHECK(fc.current_offset() == 1024); + CHECK(fc.current_limit() == 2048); + + ret = fc.update(1280); + CHECK(fc.current_offset() == 1280); + CHECK(fc.current_limit() == 2048); + CHECK(ret == 0); +} + +TEST_CASE("QUICFlowController_Remote_Connection_Insufficient_maximum_frame_size", "[quic]") +{ + int ret = 0; + uint8_t frame_buf[QUICFrame::MAX_INSTANCE_SIZE]; + QUICRemoteConnectionFlowController fc(1024); + + // Check initial state + CHECK(fc.current_offset() == 0); + CHECK(fc.current_limit() == 1024); + + // Zero credit + ret = fc.update(1024); + CHECK(fc.current_offset() == 1024); + CHECK(fc.current_limit() == 1024); + CHECK(ret == 0); + + CHECK(fc.will_generate_frame(QUICEncryptionLevel::ONE_RTT, 0, true, 0)); + // if there're anything to send + QUICFrame *frame = fc.generate_frame(frame_buf, QUICEncryptionLevel::ONE_RTT, 0, 0, 0, 0); + CHECK(frame == nullptr); + frame = fc.generate_frame(frame_buf, QUICEncryptionLevel::ONE_RTT, 0, 1024, 0, 0); CHECK(frame); CHECK(frame->type() == QUICFrameType::DATA_BLOCKED); @@ -265,7 +300,7 @@ TEST_CASE("QUICFlowController_Local_Stream", "[quic]") fc.forward_limit(2048); CHECK(fc.current_offset() == 1024); CHECK(fc.current_limit() == 2048); - QUICFrame *frame = fc.generate_frame(frame_buf, QUICEncryptionLevel::ONE_RTT, 0, 1024, 0); + QUICFrame *frame = fc.generate_frame(frame_buf, QUICEncryptionLevel::ONE_RTT, 0, 1024, 0, 0); CHECK(frame); CHECK(frame->type() == QUICFrameType::MAX_STREAM_DATA); @@ -306,7 +341,7 @@ TEST_CASE("QUICFlowController_Remote_Stream", "[quic]") CHECK(ret == 0); CHECK(fc.credit() == 0); - CHECK(fc.will_generate_frame(QUICEncryptionLevel::ONE_RTT, 0)); + CHECK(fc.will_generate_frame(QUICEncryptionLevel::ONE_RTT, 0, true, 0)); // Delay ret = fc.update(512); @@ -342,23 +377,23 @@ TEST_CASE("Frame retransmission", "[quic]") QUICRemoteConnectionFlowController fc(1024); // Check initial state - auto frame = fc.generate_frame(frame_buf, level, 1024, 1024, 0); + auto frame = fc.generate_frame(frame_buf, level, 1024, 1024, 0, 0); CHECK(!frame); ret = fc.update(1024); CHECK(ret == 0); - frame = fc.generate_frame(frame_buf, level, 1024, 1024, 0); + frame = fc.generate_frame(frame_buf, level, 1024, 1024, 0, 0); REQUIRE(frame); CHECK(static_cast(frame)->offset() == 1024); QUICFrameId id = frame->id(); // Don't retransmit unless the frame is lost - frame = fc.generate_frame(frame_buf, level, 1024, 1024, 0); + frame = fc.generate_frame(frame_buf, level, 1024, 1024, 0, 0); REQUIRE(!frame); // Retransmit fc.on_frame_lost(id); - frame = fc.generate_frame(frame_buf, level, 1024, 1024, 0); + frame = fc.generate_frame(frame_buf, level, 1024, 1024, 0, 0); REQUIRE(frame); CHECK(static_cast(frame)->offset() == 1024); @@ -366,12 +401,12 @@ TEST_CASE("Frame retransmission", "[quic]") fc.on_frame_lost(frame->id()); fc.forward_limit(2048); ret = fc.update(1536); - frame = fc.generate_frame(frame_buf, level, 1024, 1024, 0); + frame = fc.generate_frame(frame_buf, level, 1024, 1024, 0, 0); CHECK(!frame); - // This should not be retransmition + // This should not be retransmission ret = fc.update(2048); - frame = fc.generate_frame(frame_buf, level, 1024, 1024, 0); + frame = fc.generate_frame(frame_buf, level, 1024, 1024, 0, 0); REQUIRE(frame); CHECK(static_cast(frame)->offset() == 2048); } @@ -383,23 +418,23 @@ TEST_CASE("Frame retransmission", "[quic]") QUICRemoteStreamFlowController fc(1024, 0); // Check initial state - auto frame = fc.generate_frame(frame_buf, level, 1024, 1024, 0); + auto frame = fc.generate_frame(frame_buf, level, 1024, 1024, 0, 0); CHECK(!frame); ret = fc.update(1024); CHECK(ret == 0); - frame = fc.generate_frame(frame_buf, level, 1024, 1024, 0); + frame = fc.generate_frame(frame_buf, level, 1024, 1024, 0, 0); REQUIRE(frame); CHECK(static_cast(frame)->offset() == 1024); QUICFrameId id = frame->id(); // Don't retransmit unless the frame is lost - frame = fc.generate_frame(frame_buf, level, 1024, 1024, 0); + frame = fc.generate_frame(frame_buf, level, 1024, 1024, 0, 0); REQUIRE(!frame); // Retransmit fc.on_frame_lost(id); - frame = fc.generate_frame(frame_buf, level, 1024, 1024, 0); + frame = fc.generate_frame(frame_buf, level, 1024, 1024, 0, 0); REQUIRE(frame); CHECK(static_cast(frame)->offset() == 1024); @@ -407,12 +442,12 @@ TEST_CASE("Frame retransmission", "[quic]") fc.on_frame_lost(frame->id()); fc.forward_limit(2048); ret = fc.update(1536); - frame = fc.generate_frame(frame_buf, level, 1024, 1024, 0); + frame = fc.generate_frame(frame_buf, level, 1024, 1024, 0, 0); CHECK(!frame); - // This should not be retransmition + // This should not be retransmission ret = fc.update(2048); - frame = fc.generate_frame(frame_buf, level, 1024, 1024, 0); + frame = fc.generate_frame(frame_buf, level, 1024, 1024, 0, 0); REQUIRE(frame); CHECK(static_cast(frame)->offset() == 2048); } @@ -424,23 +459,23 @@ TEST_CASE("Frame retransmission", "[quic]") QUICLocalConnectionFlowController fc(&rp, 1024); // Check initial state - auto frame = fc.generate_frame(frame_buf, level, 1024, 1024, 0); + auto frame = fc.generate_frame(frame_buf, level, 1024, 1024, 0, 0); CHECK(!frame); fc.update(1024); fc.forward_limit(1024); - frame = fc.generate_frame(frame_buf, level, 1024, 1024, 0); + frame = fc.generate_frame(frame_buf, level, 1024, 1024, 0, 0); REQUIRE(frame); CHECK(static_cast(frame)->maximum_data() == 1024); QUICFrameId id = frame->id(); // Don't retransmit unless the frame is lost - frame = fc.generate_frame(frame_buf, level, 1024, 1024, 0); + frame = fc.generate_frame(frame_buf, level, 1024, 1024, 0, 0); REQUIRE(!frame); // Retransmit fc.on_frame_lost(id); - frame = fc.generate_frame(frame_buf, level, 1024, 1024, 0); + frame = fc.generate_frame(frame_buf, level, 1024, 1024, 0, 0); REQUIRE(frame); CHECK(static_cast(frame)->maximum_data() == 1024); @@ -448,7 +483,7 @@ TEST_CASE("Frame retransmission", "[quic]") fc.on_frame_lost(id); fc.forward_limit(2048); fc.update(2048); - frame = fc.generate_frame(frame_buf, level, 1024, 1024, 0); + frame = fc.generate_frame(frame_buf, level, 1024, 1024, 0, 0); REQUIRE(frame); CHECK(static_cast(frame)->maximum_data() == 2048); } @@ -460,23 +495,23 @@ TEST_CASE("Frame retransmission", "[quic]") QUICLocalStreamFlowController fc(&rp, 1024, 0); // Check initial state - auto frame = fc.generate_frame(frame_buf, level, 1024, 1024, 0); + auto frame = fc.generate_frame(frame_buf, level, 1024, 1024, 0, 0); CHECK(!frame); fc.update(1024); fc.forward_limit(1024); - frame = fc.generate_frame(frame_buf, level, 1024, 1024, 0); + frame = fc.generate_frame(frame_buf, level, 1024, 1024, 0, 0); REQUIRE(frame); CHECK(static_cast(frame)->maximum_stream_data() == 1024); QUICFrameId id = frame->id(); // Don't retransmit unless the frame is lost - frame = fc.generate_frame(frame_buf, level, 1024, 1024, 0); + frame = fc.generate_frame(frame_buf, level, 1024, 1024, 0, 0); REQUIRE(!frame); // Retransmit fc.on_frame_lost(id); - frame = fc.generate_frame(frame_buf, level, 1024, 1024, 0); + frame = fc.generate_frame(frame_buf, level, 1024, 1024, 0, 0); REQUIRE(frame); CHECK(static_cast(frame)->maximum_stream_data() == 1024); @@ -484,7 +519,7 @@ TEST_CASE("Frame retransmission", "[quic]") fc.on_frame_lost(id); fc.forward_limit(2048); fc.update(2048); - frame = fc.generate_frame(frame_buf, level, 1024, 1024, 0); + frame = fc.generate_frame(frame_buf, level, 1024, 1024, 0, 0); REQUIRE(frame); CHECK(static_cast(frame)->maximum_stream_data() == 2048); } diff --git a/iocore/net/quic/test/test_QUICFrame.cc b/iocore/net/quic/test/test_QUICFrame.cc index a8f81fd5538..2eb0c5ea1d5 100644 --- a/iocore/net/quic/test/test_QUICFrame.cc +++ b/iocore/net/quic/test/test_QUICFrame.cc @@ -68,8 +68,10 @@ TEST_CASE("QUICFrame Type", "[quic]") CHECK(QUICFrame::type(reinterpret_cast("\x1c")) == QUICFrameType::CONNECTION_CLOSE); CHECK(QUICFrame::type(reinterpret_cast("\x1d")) == QUICFrameType::CONNECTION_CLOSE); - // Undefined ragne - CHECK(QUICFrame::type(reinterpret_cast("\x1e")) == QUICFrameType::UNKNOWN); + CHECK(QUICFrame::type(reinterpret_cast("\x1e")) == QUICFrameType::HANDSHAKE_DONE); + + // Undefined range + CHECK(QUICFrame::type(reinterpret_cast("\x1f")) == QUICFrameType::UNKNOWN); CHECK(QUICFrame::type(reinterpret_cast("\xff")) == QUICFrameType::UNKNOWN); } @@ -84,7 +86,7 @@ TEST_CASE("Load STREAM Frame", "[quic]") 0x01, // Stream ID 0x01, 0x02, 0x03, 0x04, // Stream Data }; - const QUICFrame *frame1 = QUICFrameFactory::create(frame_buf, buf1, sizeof(buf1)); + const QUICFrame *frame1 = QUICFrameFactory::create(frame_buf, buf1, sizeof(buf1), nullptr); CHECK(frame1->type() == QUICFrameType::STREAM); CHECK(frame1->size() == 6); const QUICStreamFrame *stream_frame = static_cast(frame1); @@ -103,7 +105,7 @@ TEST_CASE("Load STREAM Frame", "[quic]") 0x05, // Data Length 0x01, 0x02, 0x03, 0x04, 0x05, // Stream Data }; - const QUICFrame *frame1 = QUICFrameFactory::create(frame_buf, buf1, sizeof(buf1)); + const QUICFrame *frame1 = QUICFrameFactory::create(frame_buf, buf1, sizeof(buf1), nullptr); CHECK(frame1->type() == QUICFrameType::STREAM); CHECK(frame1->size() == 8); const QUICStreamFrame *stream_frame = static_cast(frame1); @@ -123,7 +125,7 @@ TEST_CASE("Load STREAM Frame", "[quic]") 0x05, // Data Length 0x01, 0x02, 0x03, 0x04, 0x05, // Stream Data }; - const QUICFrame *frame1 = QUICFrameFactory::create(frame_buf, buf1, sizeof(buf1)); + const QUICFrame *frame1 = QUICFrameFactory::create(frame_buf, buf1, sizeof(buf1), nullptr); CHECK(frame1->type() == QUICFrameType::STREAM); CHECK(frame1->size() == 9); @@ -144,7 +146,7 @@ TEST_CASE("Load STREAM Frame", "[quic]") 0x05, // Data Length 0x01, 0x02, 0x03, 0x04, 0x05, // Stream Data }; - const QUICFrame *frame1 = QUICFrameFactory::create(frame_buf, buf1, sizeof(buf1)); + const QUICFrame *frame1 = QUICFrameFactory::create(frame_buf, buf1, sizeof(buf1), nullptr); CHECK(frame1->type() == QUICFrameType::STREAM); CHECK(frame1->size() == 9); @@ -165,7 +167,7 @@ TEST_CASE("Load STREAM Frame", "[quic]") 0x05, // Data Length 0x01, 0x02, 0x03, 0x04, // BAD Stream Data }; - const QUICFrame *frame1 = QUICFrameFactory::create(frame_buf, buf1, sizeof(buf1)); + const QUICFrame *frame1 = QUICFrameFactory::create(frame_buf, buf1, sizeof(buf1), nullptr); CHECK(frame1->type() == QUICFrameType::STREAM); CHECK(frame1->valid() == false); } @@ -176,7 +178,7 @@ TEST_CASE("Load STREAM Frame", "[quic]") 0x0e, // 0b00001OLF (OLF=110) 0x01, // Stream ID }; - const QUICFrame *frame1 = QUICFrameFactory::create(frame_buf, buf1, sizeof(buf1)); + const QUICFrame *frame1 = QUICFrameFactory::create(frame_buf, buf1, sizeof(buf1), nullptr); CHECK(frame1->type() == QUICFrameType::STREAM); CHECK(frame1->valid() == false); } @@ -197,7 +199,7 @@ TEST_CASE("Store STREAM Frame", "[quic]") uint8_t raw1[] = "\x01\x02\x03\x04\x05"; Ptr block = make_ptr(new_IOBufferBlock()); - block->alloc(); + block->alloc(BUFFER_SIZE_INDEX_32K); memcpy(block->start(), raw1, 5); block->fill(5); @@ -226,7 +228,7 @@ TEST_CASE("Store STREAM Frame", "[quic]") }; uint8_t raw2[] = "\x01\x02\x03\x04\x05"; Ptr block = make_ptr(new_IOBufferBlock()); - block->alloc(); + block->alloc(BUFFER_SIZE_INDEX_32K); memcpy(block->start(), raw2, 5); block->fill(5); @@ -255,7 +257,7 @@ TEST_CASE("Store STREAM Frame", "[quic]") }; uint8_t raw3[] = "\x01\x02\x03\x04\x05"; Ptr block = make_ptr(new_IOBufferBlock()); - block->alloc(); + block->alloc(BUFFER_SIZE_INDEX_32K); memcpy(block->start(), raw3, 5); block->fill(5); @@ -284,7 +286,7 @@ TEST_CASE("Store STREAM Frame", "[quic]") }; uint8_t raw4[] = "\x01\x02\x03\x04\x05"; Ptr block = make_ptr(new_IOBufferBlock()); - block->alloc(); + block->alloc(BUFFER_SIZE_INDEX_32K); memcpy(block->start(), raw4, 5); block->fill(5); @@ -313,7 +315,7 @@ TEST_CASE("Store STREAM Frame", "[quic]") }; uint8_t raw5[] = "\x01\x02\x03\x04\x05"; Ptr block = make_ptr(new_IOBufferBlock()); - block->alloc(); + block->alloc(BUFFER_SIZE_INDEX_32K); memcpy(block->start(), raw5, 5); block->fill(5); @@ -342,7 +344,7 @@ TEST_CASE("Store STREAM Frame", "[quic]") }; uint8_t raw6[] = "\x01\x02\x03\x04\x05"; Ptr block = make_ptr(new_IOBufferBlock()); - block->alloc(); + block->alloc(BUFFER_SIZE_INDEX_32K); memcpy(block->start(), raw6, 5); block->fill(5); @@ -371,7 +373,7 @@ TEST_CASE("Store STREAM Frame", "[quic]") }; uint8_t raw7[] = "\x01\x02\x03\x04\x05"; Ptr block = make_ptr(new_IOBufferBlock()); - block->alloc(); + block->alloc(BUFFER_SIZE_INDEX_32K); memcpy(block->start(), raw7, 5); block->fill(5); @@ -400,7 +402,7 @@ TEST_CASE("Store STREAM Frame", "[quic]") }; uint8_t raw[] = "\x01\x02\x03\x04\x05"; Ptr block = make_ptr(new_IOBufferBlock()); - block->alloc(); + block->alloc(BUFFER_SIZE_INDEX_32K); memcpy(block->start(), raw, 5); block->fill(5); @@ -428,7 +430,7 @@ TEST_CASE("CRYPTO Frame", "[quic]") 0x05, // Length 0x01, 0x02, 0x03, 0x04, 0x05, // Crypto Data }; - const QUICFrame *frame = QUICFrameFactory::create(frame_buf, buf, sizeof(buf)); + const QUICFrame *frame = QUICFrameFactory::create(frame_buf, buf, sizeof(buf), nullptr); CHECK(frame->type() == QUICFrameType::CRYPTO); CHECK(frame->size() == sizeof(buf)); @@ -444,31 +446,35 @@ TEST_CASE("CRYPTO Frame", "[quic]") 0x06, // Type 0x80, 0x01, 0x00, 0x00, // Offset }; - const QUICFrame *frame = QUICFrameFactory::create(frame_buf, buf, sizeof(buf)); + const QUICFrame *frame = QUICFrameFactory::create(frame_buf, buf, sizeof(buf), nullptr); CHECK(frame->type() == QUICFrameType::CRYPTO); CHECK(frame->valid() == false); } SECTION("Storing") { - uint8_t buf[32] = {0}; - size_t len; + uint8_t buf[32] = {0}; + size_t len = 0; uint8_t expected[] = { - 0x06, // Typr + 0x06, // Type 0x80, 0x01, 0x00, 0x00, // Offset 0x05, // Length 0x01, 0x02, 0x03, 0x04, 0x05, // Crypto Data }; uint8_t raw_data[] = "\x01\x02\x03\x04\x05"; Ptr block = make_ptr(new_IOBufferBlock()); - block->alloc(); + block->alloc(BUFFER_SIZE_INDEX_32K); memcpy(block->start(), raw_data, 5); block->fill(5); QUICCryptoFrame crypto_frame(block, 0x010000); CHECK(crypto_frame.size() == sizeof(expected)); - crypto_frame.store(buf, &len, 32); + Ptr ibb = crypto_frame.to_io_buffer_block(sizeof(buf)); + for (auto b = ibb; b; b = b->next) { + memcpy(buf + len, b->start(), b->size()); + len += b->size(); + } CHECK(len == sizeof(expected)); CHECK(memcmp(buf, expected, sizeof(expected)) == 0); } @@ -486,7 +492,7 @@ TEST_CASE("Load Ack Frame 1", "[quic]") 0x00, // Ack Block Count 0x00, // Ack Block Section }; - const QUICFrame *frame1 = QUICFrameFactory::create(frame_buf, buf1, sizeof(buf1)); + const QUICFrame *frame1 = QUICFrameFactory::create(frame_buf, buf1, sizeof(buf1), nullptr); CHECK(frame1->type() == QUICFrameType::ACK); CHECK(frame1->size() == 6); const QUICAckFrame *ack_frame1 = static_cast(frame1); @@ -494,6 +500,7 @@ TEST_CASE("Load Ack Frame 1", "[quic]") CHECK(ack_frame1->ack_block_count() == 0); CHECK(ack_frame1->largest_acknowledged() == 0x12); CHECK(ack_frame1->ack_delay() == 0x3456); + frame1->~QUICFrame(); } SECTION("0 Ack Block, 8 bit packet number length, 8 bit block length") @@ -505,7 +512,7 @@ TEST_CASE("Load Ack Frame 1", "[quic]") 0x00, // Ack Block Count 0x80, 0x00, 0x00, 0x01, // Ack Block Section (First ACK Block Length) }; - const QUICFrame *frame1 = QUICFrameFactory::create(frame_buf, buf1, sizeof(buf1)); + const QUICFrame *frame1 = QUICFrameFactory::create(frame_buf, buf1, sizeof(buf1), nullptr); CHECK(frame1->type() == QUICFrameType::ACK); CHECK(frame1->size() == 12); @@ -518,6 +525,7 @@ TEST_CASE("Load Ack Frame 1", "[quic]") const QUICAckFrame::AckBlockSection *section = ack_frame1->ack_block_section(); CHECK(section->first_ack_block() == 0x01); + frame1->~QUICFrame(); } SECTION("2 Ack Block, 8 bit packet number length, 8 bit block length") @@ -534,7 +542,7 @@ TEST_CASE("Load Ack Frame 1", "[quic]") 0xc9, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, // Ack Block Section (ACK Block 2 Length) }; - const QUICFrame *frame1 = QUICFrameFactory::create(frame_buf, buf1, sizeof(buf1)); + const QUICFrame *frame1 = QUICFrameFactory::create(frame_buf, buf1, sizeof(buf1), nullptr); CHECK(frame1->type() == QUICFrameType::ACK); CHECK(frame1->size() == 21); const QUICAckFrame *ack_frame1 = static_cast(frame1); @@ -556,6 +564,7 @@ TEST_CASE("Load Ack Frame 1", "[quic]") CHECK(ite->length() == 0x090a0b0c0d0e0f10); ++ite; CHECK(ite == section->end()); + frame1->~QUICFrame(); } SECTION("load bad frame") @@ -565,9 +574,10 @@ TEST_CASE("Load Ack Frame 1", "[quic]") 0x12, // Largest Acknowledged }; - const QUICFrame *frame1 = QUICFrameFactory::create(frame_buf, buf1, sizeof(buf1)); + const QUICFrame *frame1 = QUICFrameFactory::create(frame_buf, buf1, sizeof(buf1), nullptr); CHECK(frame1->type() == QUICFrameType::ACK); CHECK(frame1->valid() == false); + frame1->~QUICFrame(); } SECTION("load bad block") @@ -583,9 +593,10 @@ TEST_CASE("Load Ack Frame 1", "[quic]") 0x85, 0x06, 0x07, 0x08, // Ack Block Section (Gap 2) }; - const QUICFrame *frame1 = QUICFrameFactory::create(frame_buf, buf1, sizeof(buf1)); + const QUICFrame *frame1 = QUICFrameFactory::create(frame_buf, buf1, sizeof(buf1), nullptr); CHECK(frame1->type() == QUICFrameType::ACK); CHECK(frame1->valid() == false); + frame1->~QUICFrame(); } SECTION("0 Ack Block, 8 bit packet number length, 8 bit block length with ECN section") @@ -601,7 +612,7 @@ TEST_CASE("Load Ack Frame 1", "[quic]") 0x02, // ECT1 0x03, // ECN-CE }; - const QUICFrame *frame1 = QUICFrameFactory::create(frame_buf, buf1, sizeof(buf1)); + const QUICFrame *frame1 = QUICFrameFactory::create(frame_buf, buf1, sizeof(buf1), nullptr); CHECK(frame1->type() == QUICFrameType::ACK); CHECK(frame1->size() == 9); const QUICAckFrame *ack_frame1 = static_cast(frame1); @@ -613,6 +624,7 @@ TEST_CASE("Load Ack Frame 1", "[quic]") CHECK(ack_frame1->ecn_section()->ect0_count() == 1); CHECK(ack_frame1->ecn_section()->ect1_count() == 2); CHECK(ack_frame1->ecn_section()->ecn_ce_count() == 3); + frame1->~QUICFrame(); } SECTION("0 Ack Block, 8 bit packet number length, 8 bit block length with ECN section") @@ -627,9 +639,10 @@ TEST_CASE("Load Ack Frame 1", "[quic]") 0x01, // ECT0 0x02, // ECT1 }; - const QUICFrame *frame1 = QUICFrameFactory::create(frame_buf, buf1, sizeof(buf1)); + const QUICFrame *frame1 = QUICFrameFactory::create(frame_buf, buf1, sizeof(buf1), nullptr); CHECK(frame1->type() == QUICFrameType::ACK); CHECK(frame1->valid() == false); + frame1->~QUICFrame(); } } @@ -638,7 +651,7 @@ TEST_CASE("Store Ack Frame", "[quic]") SECTION("0 Ack Block, 8 bit packet number length, 8 bit block length") { uint8_t buf[32] = {0}; - size_t len; + size_t len = 0; uint8_t expected[] = { 0x02, // Type @@ -651,7 +664,11 @@ TEST_CASE("Store Ack Frame", "[quic]") QUICAckFrame ack_frame(0x12, 0x3456, 0); CHECK(ack_frame.size() == 6); - ack_frame.store(buf, &len, 32); + Ptr ibb = ack_frame.to_io_buffer_block(sizeof(buf)); + for (auto b = ibb; b; b = b->next) { + memcpy(buf + len, b->start(), b->size()); + len += b->size(); + } CHECK(len == 6); CHECK(memcmp(buf, expected, len) == 0); } @@ -659,7 +676,7 @@ TEST_CASE("Store Ack Frame", "[quic]") SECTION("2 Ack Block, 8 bit packet number length, 8 bit block length") { uint8_t buf[32] = {0}; - size_t len; + size_t len = 0; uint8_t expected[] = { 0x02, // Type @@ -678,7 +695,11 @@ TEST_CASE("Store Ack Frame", "[quic]") section->add_ack_block({0x05060708, 0x090a0b0c0d0e0f10}); CHECK(ack_frame.size() == 21); - ack_frame.store(buf, &len, 32); + Ptr ibb = ack_frame.to_io_buffer_block(sizeof(buf)); + for (auto b = ibb; b; b = b->next) { + memcpy(buf + len, b->start(), b->size()); + len += b->size(); + } CHECK(len == 21); CHECK(memcmp(buf, expected, len) == 0); } @@ -692,12 +713,12 @@ TEST_CASE("Load RESET_STREAM Frame", "[quic]") uint8_t buf1[] = { 0x04, // Type 0x92, 0x34, 0x56, 0x78, // Stream ID - 0x00, 0x01, // Error Code + 0x01, // Error Code 0xd1, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88 // Final Offset }; - const QUICFrame *frame1 = QUICFrameFactory::create(frame_buf, buf1, sizeof(buf1)); + const QUICFrame *frame1 = QUICFrameFactory::create(frame_buf, buf1, sizeof(buf1), nullptr); CHECK(frame1->type() == QUICFrameType::RESET_STREAM); - CHECK(frame1->size() == 15); + CHECK(frame1->size() == 14); const QUICRstStreamFrame *rst_stream_frame1 = static_cast(frame1); CHECK(rst_stream_frame1 != nullptr); CHECK(rst_stream_frame1->error_code() == 0x0001); @@ -710,9 +731,9 @@ TEST_CASE("Load RESET_STREAM Frame", "[quic]") uint8_t buf1[] = { 0x04, // Type 0x92, 0x34, 0x56, 0x78, // Stream ID - 0x00, 0x01, // Error Code + 0x01, // Error Code }; - const QUICFrame *frame1 = QUICFrameFactory::create(frame_buf, buf1, sizeof(buf1)); + const QUICFrame *frame1 = QUICFrameFactory::create(frame_buf, buf1, sizeof(buf1), nullptr); CHECK(frame1->type() == QUICFrameType::RESET_STREAM); CHECK(frame1->valid() == false); } @@ -721,19 +742,23 @@ TEST_CASE("Load RESET_STREAM Frame", "[quic]") TEST_CASE("Store RESET_STREAM Frame", "[quic]") { uint8_t buf[65535]; - size_t len; + size_t len = 0; uint8_t expected[] = { 0x04, // Type 0x92, 0x34, 0x56, 0x78, // Stream ID - 0x00, 0x01, // Error Code + 0x01, // Error Code 0xd1, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88 // Final Offset }; QUICRstStreamFrame rst_stream_frame(0x12345678, 0x0001, 0x1122334455667788); - CHECK(rst_stream_frame.size() == 15); + CHECK(rst_stream_frame.size() == 14); - rst_stream_frame.store(buf, &len, 65535); - CHECK(len == 15); + Ptr ibb = rst_stream_frame.to_io_buffer_block(sizeof(buf)); + for (auto b = ibb; b; b = b->next) { + memcpy(buf + len, b->start(), b->size()); + len += b->size(); + } + CHECK(len == 14); CHECK(memcmp(buf, expected, len) == 0); } @@ -743,7 +768,7 @@ TEST_CASE("Load Ping Frame", "[quic]") uint8_t buf[] = { 0x01, // Type }; - const QUICFrame *frame = QUICFrameFactory::create(frame_buf, buf, sizeof(buf)); + const QUICFrame *frame = QUICFrameFactory::create(frame_buf, buf, sizeof(buf), nullptr); CHECK(frame->type() == QUICFrameType::PING); CHECK(frame->size() == 1); @@ -754,7 +779,7 @@ TEST_CASE("Load Ping Frame", "[quic]") TEST_CASE("Store Ping Frame", "[quic]") { uint8_t buf[16]; - size_t len; + size_t len = 0; uint8_t expected[] = { 0x01, // Type @@ -763,7 +788,11 @@ TEST_CASE("Store Ping Frame", "[quic]") QUICPingFrame frame; CHECK(frame.size() == 1); - frame.store(buf, &len, 16); + Ptr ibb = frame.to_io_buffer_block(sizeof(buf)); + for (auto b = ibb; b; b = b->next) { + memcpy(buf + len, b->start(), b->size()); + len += b->size(); + } CHECK(len == 1); CHECK(memcmp(buf, expected, len) == 0); } @@ -772,11 +801,11 @@ TEST_CASE("Load Padding Frame", "[quic]") { uint8_t frame_buf[QUICFrame::MAX_INSTANCE_SIZE]; uint8_t buf1[] = { - 0x00, // Type + 0x00, 0x00, 0x00 // Type }; - const QUICFrame *frame1 = QUICFrameFactory::create(frame_buf, buf1, sizeof(buf1)); + const QUICFrame *frame1 = QUICFrameFactory::create(frame_buf, buf1, sizeof(buf1), nullptr); CHECK(frame1->type() == QUICFrameType::PADDING); - CHECK(frame1->size() == 1); + CHECK(frame1->size() == 3); const QUICPaddingFrame *paddingFrame1 = static_cast(frame1); CHECK(paddingFrame1 != nullptr); } @@ -784,13 +813,52 @@ TEST_CASE("Load Padding Frame", "[quic]") TEST_CASE("Store Padding Frame", "[quic]") { uint8_t buf[65535]; - size_t len; + size_t len = 0; uint8_t expected[] = { - 0x00, // Type + 0x00, 0x00, 0x00, // Type }; - QUICPaddingFrame padding_frame; - padding_frame.store(buf, &len, 65535); + QUICPaddingFrame padding_frame(3); + Ptr ibb = padding_frame.to_io_buffer_block(sizeof(buf)); + for (auto b = ibb; b; b = b->next) { + memcpy(buf + len, b->start(), b->size()); + len += b->size(); + } + CHECK(len == 3); + CHECK(memcmp(buf, expected, len) == 0); +} + +TEST_CASE("Load HandshakeDone Frame", "[quic]") +{ + uint8_t frame_buf[QUICFrame::MAX_INSTANCE_SIZE]; + uint8_t buf[] = { + 0x1e, // Type + }; + const QUICFrame *frame = QUICFrameFactory::create(frame_buf, buf, sizeof(buf), nullptr); + CHECK(frame->type() == QUICFrameType::HANDSHAKE_DONE); + CHECK(frame->size() == 1); + + const QUICHandshakeDoneFrame *handshake_done_frame = static_cast(frame); + CHECK(handshake_done_frame != nullptr); +} + +TEST_CASE("Store HandshakeDone Frame", "[quic]") +{ + uint8_t buf[16]; + size_t len = 0; + + uint8_t expected[] = { + 0x1e, // Type + }; + + QUICHandshakeDoneFrame frame; + CHECK(frame.size() == 1); + + Ptr ibb = frame.to_io_buffer_block(sizeof(buf)); + for (auto b = ibb; b; b = b->next) { + memcpy(buf + len, b->start(), b->size()); + len += b->size(); + } CHECK(len == 1); CHECK(memcmp(buf, expected, len) == 0); } @@ -805,13 +873,13 @@ TEST_CASE("ConnectionClose Frame", "[quic]") { uint8_t buf[] = { 0x1c, // Type - 0x00, 0x0A, // Error Code + 0x0A, // Error Code 0x00, // Frame Type 0x05, // Reason Phrase Length 0x41, 0x42, 0x43, 0x44, 0x45 // Reason Phrase ("ABCDE"); }; - const QUICFrame *frame = QUICFrameFactory::create(frame_buf, buf, sizeof(buf)); + const QUICFrame *frame = QUICFrameFactory::create(frame_buf, buf, sizeof(buf), nullptr); CHECK(frame->type() == QUICFrameType::CONNECTION_CLOSE); CHECK(frame->size() == sizeof(buf)); @@ -826,13 +894,13 @@ TEST_CASE("ConnectionClose Frame", "[quic]") SECTION("Bad loading") { uint8_t buf[] = { - 0x1c, // Type - 0x00, 0x0A, // Error Code - 0x00, // Frame Type - 0x05, // Reason Phrase Length + 0x1c, // Type + 0x0A, // Error Code + 0x00, // Frame Type + 0x05, // Reason Phrase Length }; - const QUICFrame *frame = QUICFrameFactory::create(frame_buf, buf, sizeof(buf)); + const QUICFrame *frame = QUICFrameFactory::create(frame_buf, buf, sizeof(buf), nullptr); CHECK(frame->type() == QUICFrameType::CONNECTION_CLOSE); CHECK(frame->valid() == false); } @@ -840,12 +908,12 @@ TEST_CASE("ConnectionClose Frame", "[quic]") SECTION("loading w/o reason phrase") { uint8_t buf[] = { - 0x1c, // Type - 0x00, 0x0A, // Error Code - 0x04, // Frame Type - 0x00, // Reason Phrase Length + 0x1c, // Type + 0x0A, // Error Code + 0x04, // Frame Type + 0x00, // Reason Phrase Length }; - const QUICFrame *frame = QUICFrameFactory::create(frame_buf, buf, sizeof(buf)); + const QUICFrame *frame = QUICFrameFactory::create(frame_buf, buf, sizeof(buf), nullptr); CHECK(frame->type() == QUICFrameType::CONNECTION_CLOSE); CHECK(frame->size() == sizeof(buf)); @@ -859,11 +927,11 @@ TEST_CASE("ConnectionClose Frame", "[quic]") SECTION("storing w/ reason phrase") { uint8_t buf[32]; - size_t len; + size_t len = 0; uint8_t expected[] = { 0x1c, // Type - 0x00, 0x0A, // Error Code + 0x0A, // Error Code 0x08, // Frame Type 0x05, // Reason Phrase Length 0x41, 0x42, 0x43, 0x44, 0x45 // Reason Phrase ("ABCDE"); @@ -872,7 +940,11 @@ TEST_CASE("ConnectionClose Frame", "[quic]") QUICFrameType::STREAM, 5, "ABCDE"); CHECK(connection_close_frame.size() == sizeof(expected)); - connection_close_frame.store(buf, &len, 32); + Ptr ibb = connection_close_frame.to_io_buffer_block(sizeof(buf)); + for (auto b = ibb; b; b = b->next) { + memcpy(buf + len, b->start(), b->size()); + len += b->size(); + } CHECK(len == sizeof(expected)); CHECK(memcmp(buf, expected, len) == 0); } @@ -880,17 +952,21 @@ TEST_CASE("ConnectionClose Frame", "[quic]") SECTION("storing w/o reason phrase") { uint8_t buf[32]; - size_t len; + size_t len = 0; uint8_t expected[] = { - 0x1c, // Type - 0x00, 0x0A, // Error Code - 0x00, // Frame Type - 0x00, // Reason Phrase Length + 0x1c, // Type + 0x0A, // Error Code + 0x00, // Frame Type + 0x00, // Reason Phrase Length }; QUICConnectionCloseFrame connection_close_frame(static_cast(QUICTransErrorCode::PROTOCOL_VIOLATION), QUICFrameType::UNKNOWN, 0, nullptr); - connection_close_frame.store(buf, &len, 32); + Ptr ibb = connection_close_frame.to_io_buffer_block(sizeof(buf)); + for (auto b = ibb; b; b = b->next) { + memcpy(buf + len, b->start(), b->size()); + len += b->size(); + } CHECK(len == sizeof(expected)); CHECK(memcmp(buf, expected, len) == 0); } @@ -905,7 +981,7 @@ TEST_CASE("Load MaxData Frame", "[quic]") 0x10, // Type 0xd1, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88 // Maximum Data }; - const QUICFrame *frame1 = QUICFrameFactory::create(frame_buf, buf1, sizeof(buf1)); + const QUICFrame *frame1 = QUICFrameFactory::create(frame_buf, buf1, sizeof(buf1), nullptr); CHECK(frame1->type() == QUICFrameType::MAX_DATA); CHECK(frame1->size() == 9); const QUICMaxDataFrame *max_data_frame = static_cast(frame1); @@ -918,7 +994,7 @@ TEST_CASE("Load MaxData Frame", "[quic]") uint8_t buf1[] = { 0x10, // Type }; - const QUICFrame *frame1 = QUICFrameFactory::create(frame_buf, buf1, sizeof(buf1)); + const QUICFrame *frame1 = QUICFrameFactory::create(frame_buf, buf1, sizeof(buf1), nullptr); CHECK(frame1->type() == QUICFrameType::MAX_DATA); CHECK(frame1->valid() == false); } @@ -927,7 +1003,7 @@ TEST_CASE("Load MaxData Frame", "[quic]") TEST_CASE("Store MaxData Frame", "[quic]") { uint8_t buf[65535]; - size_t len; + size_t len = 0; uint8_t expected[] = { 0x10, // Type @@ -936,7 +1012,11 @@ TEST_CASE("Store MaxData Frame", "[quic]") QUICMaxDataFrame max_data_frame(0x1122334455667788, 0, nullptr); CHECK(max_data_frame.size() == 9); - max_data_frame.store(buf, &len, 65535); + Ptr ibb = max_data_frame.to_io_buffer_block(sizeof(buf)); + for (auto b = ibb; b; b = b->next) { + memcpy(buf + len, b->start(), b->size()); + len += b->size(); + } CHECK(len == 9); CHECK(memcmp(buf, expected, len) == 0); } @@ -951,7 +1031,7 @@ TEST_CASE("Load MaxStreamData Frame", "[quic]") 0x81, 0x02, 0x03, 0x04, // Stream ID 0xd1, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88 // Maximum Stream Data }; - const QUICFrame *frame1 = QUICFrameFactory::create(frame_buf, buf1, sizeof(buf1)); + const QUICFrame *frame1 = QUICFrameFactory::create(frame_buf, buf1, sizeof(buf1), nullptr); CHECK(frame1->type() == QUICFrameType::MAX_STREAM_DATA); CHECK(frame1->size() == 13); const QUICMaxStreamDataFrame *maxStreamDataFrame1 = static_cast(frame1); @@ -966,7 +1046,7 @@ TEST_CASE("Load MaxStreamData Frame", "[quic]") 0x11, // Type 0x81, 0x02, 0x03, 0x04, // Stream ID }; - const QUICFrame *frame1 = QUICFrameFactory::create(frame_buf, buf1, sizeof(buf1)); + const QUICFrame *frame1 = QUICFrameFactory::create(frame_buf, buf1, sizeof(buf1), nullptr); CHECK(frame1->type() == QUICFrameType::MAX_STREAM_DATA); CHECK(frame1->valid() == false); } @@ -975,7 +1055,7 @@ TEST_CASE("Load MaxStreamData Frame", "[quic]") TEST_CASE("Store MaxStreamData Frame", "[quic]") { uint8_t buf[65535]; - size_t len; + size_t len = 0; uint8_t expected[] = { 0x11, // Type @@ -985,7 +1065,11 @@ TEST_CASE("Store MaxStreamData Frame", "[quic]") QUICMaxStreamDataFrame max_stream_data_frame(0x01020304, 0x1122334455667788ULL); CHECK(max_stream_data_frame.size() == 13); - max_stream_data_frame.store(buf, &len, 65535); + Ptr ibb = max_stream_data_frame.to_io_buffer_block(sizeof(buf)); + for (auto b = ibb; b; b = b->next) { + memcpy(buf + len, b->start(), b->size()); + len += b->size(); + } CHECK(len == 13); CHECK(memcmp(buf, expected, len) == 0); } @@ -999,7 +1083,7 @@ TEST_CASE("Load MaxStreams Frame", "[quic]") 0x12, // Type 0x81, 0x02, 0x03, 0x04, // Stream ID }; - const QUICFrame *frame1 = QUICFrameFactory::create(frame_buf, buf1, sizeof(buf1)); + const QUICFrame *frame1 = QUICFrameFactory::create(frame_buf, buf1, sizeof(buf1), nullptr); CHECK(frame1->type() == QUICFrameType::MAX_STREAMS); CHECK(frame1->size() == 5); const QUICMaxStreamsFrame *max_streams_frame = static_cast(frame1); @@ -1011,7 +1095,7 @@ TEST_CASE("Load MaxStreams Frame", "[quic]") uint8_t buf1[] = { 0x12, // Type }; - const QUICFrame *frame1 = QUICFrameFactory::create(frame_buf, buf1, sizeof(buf1)); + const QUICFrame *frame1 = QUICFrameFactory::create(frame_buf, buf1, sizeof(buf1), nullptr); CHECK(frame1->type() == QUICFrameType::MAX_STREAMS); CHECK(frame1->valid() == false); } @@ -1020,7 +1104,7 @@ TEST_CASE("Load MaxStreams Frame", "[quic]") TEST_CASE("Store MaxStreams Frame", "[quic]") { uint8_t buf[65535]; - size_t len; + size_t len = 0; uint8_t expected[] = { 0x12, // Type @@ -1029,7 +1113,11 @@ TEST_CASE("Store MaxStreams Frame", "[quic]") QUICMaxStreamsFrame max_streams_frame(0x01020304, 0, nullptr); CHECK(max_streams_frame.size() == 5); - max_streams_frame.store(buf, &len, 65535); + Ptr ibb = max_streams_frame.to_io_buffer_block(sizeof(buf)); + for (auto b = ibb; b; b = b->next) { + memcpy(buf + len, b->start(), b->size()); + len += b->size(); + } CHECK(len == 5); CHECK(memcmp(buf, expected, len) == 0); } @@ -1043,7 +1131,7 @@ TEST_CASE("Load DataBlocked Frame", "[quic]") 0x14, // Type 0x07, // Offset }; - const QUICFrame *frame1 = QUICFrameFactory::create(frame_buf, buf1, sizeof(buf1)); + const QUICFrame *frame1 = QUICFrameFactory::create(frame_buf, buf1, sizeof(buf1), nullptr); CHECK(frame1->type() == QUICFrameType::DATA_BLOCKED); CHECK(frame1->size() == 2); const QUICDataBlockedFrame *blocked_stream_frame = static_cast(frame1); @@ -1056,7 +1144,7 @@ TEST_CASE("Load DataBlocked Frame", "[quic]") uint8_t buf1[] = { 0x14, // Type }; - const QUICFrame *frame1 = QUICFrameFactory::create(frame_buf, buf1, sizeof(buf1)); + const QUICFrame *frame1 = QUICFrameFactory::create(frame_buf, buf1, sizeof(buf1), nullptr); CHECK(frame1->type() == QUICFrameType::DATA_BLOCKED); CHECK(frame1->valid() == false); } @@ -1065,7 +1153,7 @@ TEST_CASE("Load DataBlocked Frame", "[quic]") TEST_CASE("Store DataBlocked Frame", "[quic]") { uint8_t buf[65535]; - size_t len; + size_t len = 0; uint8_t expected[] = { 0x14, // Type @@ -1074,7 +1162,11 @@ TEST_CASE("Store DataBlocked Frame", "[quic]") QUICDataBlockedFrame blocked_stream_frame(0x07, 0, nullptr); CHECK(blocked_stream_frame.size() == 2); - blocked_stream_frame.store(buf, &len, 65535); + Ptr ibb = blocked_stream_frame.to_io_buffer_block(sizeof(buf)); + for (auto b = ibb; b; b = b->next) { + memcpy(buf + len, b->start(), b->size()); + len += b->size(); + } CHECK(len == 2); CHECK(memcmp(buf, expected, len) == 0); } @@ -1089,7 +1181,7 @@ TEST_CASE("Load StreamDataBlocked Frame", "[quic]") 0x81, 0x02, 0x03, 0x04, // Stream ID 0x07, // Offset }; - const QUICFrame *frame1 = QUICFrameFactory::create(frame_buf, buf1, sizeof(buf1)); + const QUICFrame *frame1 = QUICFrameFactory::create(frame_buf, buf1, sizeof(buf1), nullptr); CHECK(frame1->type() == QUICFrameType::STREAM_DATA_BLOCKED); CHECK(frame1->size() == 6); const QUICStreamDataBlockedFrame *stream_blocked_frame = static_cast(frame1); @@ -1104,7 +1196,7 @@ TEST_CASE("Load StreamDataBlocked Frame", "[quic]") 0x15, // Type 0x81, 0x02, 0x03, 0x04, // Stream ID }; - const QUICFrame *frame1 = QUICFrameFactory::create(frame_buf, buf1, sizeof(buf1)); + const QUICFrame *frame1 = QUICFrameFactory::create(frame_buf, buf1, sizeof(buf1), nullptr); CHECK(frame1->type() == QUICFrameType::STREAM_DATA_BLOCKED); CHECK(frame1->valid() == false); } @@ -1113,7 +1205,7 @@ TEST_CASE("Load StreamDataBlocked Frame", "[quic]") TEST_CASE("Store StreamDataBlocked Frame", "[quic]") { uint8_t buf[65535]; - size_t len; + size_t len = 0; uint8_t expected[] = { 0x15, // Type @@ -1123,7 +1215,11 @@ TEST_CASE("Store StreamDataBlocked Frame", "[quic]") QUICStreamDataBlockedFrame stream_blocked_frame(0x01020304, 0x07); CHECK(stream_blocked_frame.size() == 6); - stream_blocked_frame.store(buf, &len, 65535); + Ptr ibb = stream_blocked_frame.to_io_buffer_block(sizeof(buf)); + for (auto b = ibb; b; b = b->next) { + memcpy(buf + len, b->start(), b->size()); + len += b->size(); + } CHECK(len == 6); CHECK(memcmp(buf, expected, len) == 0); } @@ -1137,7 +1233,7 @@ TEST_CASE("Load StreamsBlocked Frame", "[quic]") 0x16, // Type 0x41, 0x02, // Stream ID }; - const QUICFrame *frame1 = QUICFrameFactory::create(frame_buf, buf1, sizeof(buf1)); + const QUICFrame *frame1 = QUICFrameFactory::create(frame_buf, buf1, sizeof(buf1), nullptr); CHECK(frame1->type() == QUICFrameType::STREAMS_BLOCKED); CHECK(frame1->size() == 3); const QUICStreamIdBlockedFrame *stream_id_blocked_frame = static_cast(frame1); @@ -1150,7 +1246,7 @@ TEST_CASE("Load StreamsBlocked Frame", "[quic]") uint8_t buf1[] = { 0x16, // Type }; - const QUICFrame *frame1 = QUICFrameFactory::create(frame_buf, buf1, sizeof(buf1)); + const QUICFrame *frame1 = QUICFrameFactory::create(frame_buf, buf1, sizeof(buf1), nullptr); CHECK(frame1->type() == QUICFrameType::STREAMS_BLOCKED); CHECK(frame1->valid() == false); } @@ -1159,7 +1255,7 @@ TEST_CASE("Load StreamsBlocked Frame", "[quic]") TEST_CASE("Store StreamsBlocked Frame", "[quic]") { uint8_t buf[65535]; - size_t len; + size_t len = 0; uint8_t expected[] = { 0x16, // Type @@ -1168,7 +1264,11 @@ TEST_CASE("Store StreamsBlocked Frame", "[quic]") QUICStreamIdBlockedFrame stream_id_blocked_frame(0x0102, 0, nullptr); CHECK(stream_id_blocked_frame.size() == 3); - stream_id_blocked_frame.store(buf, &len, 65535); + Ptr ibb = stream_id_blocked_frame.to_io_buffer_block(sizeof(buf)); + for (auto b = ibb; b; b = b->next) { + memcpy(buf + len, b->start(), b->size()); + len += b->size(); + } CHECK(len == 3); CHECK(memcmp(buf, expected, len) == 0); } @@ -1176,61 +1276,74 @@ TEST_CASE("Store StreamsBlocked Frame", "[quic]") TEST_CASE("Load NewConnectionId Frame", "[quic]") { uint8_t frame_buf[QUICFrame::MAX_INSTANCE_SIZE]; + SECTION("load") { - uint8_t buf1[] = { + uint8_t buf[] = { 0x18, // Type 0x41, 0x02, // Sequence + 0x41, 0x00, // Retire Prior To 0x08, // Length 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88, // Connection ID 0x00, 0x10, 0x20, 0x30, 0x40, 0x50, 0x60, 0x70, // Stateless Reset Token 0x80, 0x90, 0xa0, 0xb0, 0xc0, 0xd0, 0xe0, 0xf0, }; - const QUICFrame *frame1 = QUICFrameFactory::create(frame_buf, buf1, sizeof(buf1)); - CHECK(frame1->type() == QUICFrameType::NEW_CONNECTION_ID); - CHECK(frame1->size() == 28); - const QUICNewConnectionIdFrame *new_con_id_frame = static_cast(frame1); + const QUICFrame *frame = QUICFrameFactory::create(frame_buf, buf, sizeof(buf), nullptr); + CHECK(frame->type() == QUICFrameType::NEW_CONNECTION_ID); + CHECK(frame->size() == sizeof(buf)); + CHECK(frame->valid() == true); + + const QUICNewConnectionIdFrame *new_con_id_frame = static_cast(frame); CHECK(new_con_id_frame != nullptr); CHECK(new_con_id_frame->sequence() == 0x0102); + CHECK(new_con_id_frame->retire_prior_to() == 0x0100); CHECK((new_con_id_frame->connection_id() == QUICConnectionId(reinterpret_cast("\x11\x22\x33\x44\x55\x66\x77\x88"), 8))); - CHECK(memcmp(new_con_id_frame->stateless_reset_token().buf(), buf1 + 12, 16) == 0); + CHECK(memcmp(new_con_id_frame->stateless_reset_token().buf(), buf + sizeof(buf) - QUICStatelessResetToken::LEN, + QUICStatelessResetToken::LEN) == 0); } SECTION("Bad Load") { - uint8_t buf1[] = { + uint8_t buf[] = { 0x18, // Type 0x41, 0x02, // Sequence + 0x41, 0x00, // Retire Prior To 0x08, // Length 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88, // Connection ID 0x00, 0x10, 0x20, 0x30, 0x40, 0x50, 0x60, 0x70, // Stateless Reset Token }; - const QUICFrame *frame1 = QUICFrameFactory::create(frame_buf, buf1, sizeof(buf1)); - CHECK(frame1->type() == QUICFrameType::NEW_CONNECTION_ID); - CHECK(frame1->valid() == false); + const QUICFrame *frame = QUICFrameFactory::create(frame_buf, buf, sizeof(buf), nullptr); + CHECK(frame->type() == QUICFrameType::NEW_CONNECTION_ID); + CHECK(frame->valid() == false); } } TEST_CASE("Store NewConnectionId Frame", "[quic]") { uint8_t buf[32]; - size_t len; + size_t len = 0; uint8_t expected[] = { 0x18, // Type - 0x41, 0x02, // Sequence + 0x41, 0x02, // Sequence Number + 0x41, 0x00, // Retire Prior To 0x08, // Length 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88, // Connection ID 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, // Stateless Reset Token 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, }; - QUICNewConnectionIdFrame new_con_id_frame(0x0102, {reinterpret_cast("\x11\x22\x33\x44\x55\x66\x77\x88"), 8}, - {expected + 12}); - CHECK(new_con_id_frame.size() == 28); - - new_con_id_frame.store(buf, &len, 32); - CHECK(len == 28); + QUICNewConnectionIdFrame new_con_id_frame(0x0102, 0x0100, + {reinterpret_cast("\x11\x22\x33\x44\x55\x66\x77\x88"), 8}, + {expected + sizeof(expected) - QUICStatelessResetToken::LEN}); + CHECK(new_con_id_frame.size() == sizeof(expected)); + + Ptr ibb = new_con_id_frame.to_io_buffer_block(sizeof(buf)); + for (auto b = ibb; b; b = b->next) { + memcpy(buf + len, b->start(), b->size()); + len += b->size(); + } + CHECK(len == sizeof(expected)); CHECK(memcmp(buf, expected, len) == 0); } @@ -1242,16 +1355,16 @@ TEST_CASE("Load STOP_SENDING Frame", "[quic]") uint8_t buf[] = { 0x05, // Type 0x92, 0x34, 0x56, 0x78, // Stream ID - 0x00, 0x01, // Error Code + 0x01, // Error Code }; - const QUICFrame *frame = QUICFrameFactory::create(frame_buf, buf, sizeof(buf)); + const QUICFrame *frame = QUICFrameFactory::create(frame_buf, buf, sizeof(buf), nullptr); CHECK(frame->type() == QUICFrameType::STOP_SENDING); - CHECK(frame->size() == 7); + CHECK(frame->size() == 6); const QUICStopSendingFrame *stop_sending_frame = static_cast(frame); CHECK(stop_sending_frame != nullptr); CHECK(stop_sending_frame->stream_id() == 0x12345678); - CHECK(stop_sending_frame->error_code() == 0x0001); + CHECK(stop_sending_frame->error_code() == 0x01); } SECTION("Bad LOAD") @@ -1260,7 +1373,7 @@ TEST_CASE("Load STOP_SENDING Frame", "[quic]") 0x05, // Type 0x92, 0x34, 0x56, 0x78, // Stream ID }; - const QUICFrame *frame = QUICFrameFactory::create(frame_buf, buf, sizeof(buf)); + const QUICFrame *frame = QUICFrameFactory::create(frame_buf, buf, sizeof(buf), nullptr); CHECK(frame->type() == QUICFrameType::STOP_SENDING); CHECK(frame->valid() == false); } @@ -1269,18 +1382,22 @@ TEST_CASE("Load STOP_SENDING Frame", "[quic]") TEST_CASE("Store STOP_SENDING Frame", "[quic]") { uint8_t buf[65535]; - size_t len; + size_t len = 0; uint8_t expected[] = { 0x05, // Type 0x92, 0x34, 0x56, 0x78, // Stream ID - 0x00, 0x01, // Error Code + 0x01, // Error Code }; QUICStopSendingFrame stop_sending_frame(0x12345678, static_cast(0x01)); - CHECK(stop_sending_frame.size() == 7); + CHECK(stop_sending_frame.size() == 6); - stop_sending_frame.store(buf, &len, 65535); - CHECK(len == 7); + Ptr ibb = stop_sending_frame.to_io_buffer_block(sizeof(buf)); + for (auto b = ibb; b; b = b->next) { + memcpy(buf + len, b->start(), b->size()); + len += b->size(); + } + CHECK(len == 6); CHECK(memcmp(buf, expected, len) == 0); } @@ -1293,13 +1410,14 @@ TEST_CASE("Load PATH_CHALLENGE Frame", "[quic]") 0x1a, // Type 0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef, // Data }; - const QUICFrame *frame = QUICFrameFactory::create(frame_buf, buf, sizeof(buf)); + const QUICFrame *frame = QUICFrameFactory::create(frame_buf, buf, sizeof(buf), nullptr); CHECK(frame->type() == QUICFrameType::PATH_CHALLENGE); CHECK(frame->size() == 9); const QUICPathChallengeFrame *path_challenge_frame = static_cast(frame); CHECK(path_challenge_frame != nullptr); CHECK(memcmp(path_challenge_frame->data(), "\x01\x23\x45\x67\x89\xab\xcd\xef", QUICPathChallengeFrame::DATA_LEN) == 0); + frame->~QUICFrame(); } SECTION("Load") @@ -1308,16 +1426,17 @@ TEST_CASE("Load PATH_CHALLENGE Frame", "[quic]") 0x1a, // Type 0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xef, // Data }; - const QUICFrame *frame = QUICFrameFactory::create(frame_buf, buf, sizeof(buf)); + const QUICFrame *frame = QUICFrameFactory::create(frame_buf, buf, sizeof(buf), nullptr); CHECK(frame->type() == QUICFrameType::PATH_CHALLENGE); CHECK(frame->valid() == false); + frame->~QUICFrame(); } } TEST_CASE("Store PATH_CHALLENGE Frame", "[quic]") { uint8_t buf[16]; - size_t len; + size_t len = 0; uint8_t expected[] = { 0x1a, // Type @@ -1332,7 +1451,11 @@ TEST_CASE("Store PATH_CHALLENGE Frame", "[quic]") QUICPathChallengeFrame frame(std::move(data)); CHECK(frame.size() == 9); - frame.store(buf, &len, 16); + Ptr ibb = frame.to_io_buffer_block(sizeof(buf)); + for (auto b = ibb; b; b = b->next) { + memcpy(buf + len, b->start(), b->size()); + len += b->size(); + } CHECK(len == 9); CHECK(memcmp(buf, expected, len) == 0); } @@ -1346,13 +1469,14 @@ TEST_CASE("Load PATH_RESPONSE Frame", "[quic]") 0x1b, // Type 0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef, // Data }; - const QUICFrame *frame = QUICFrameFactory::create(frame_buf, buf, sizeof(buf)); + const QUICFrame *frame = QUICFrameFactory::create(frame_buf, buf, sizeof(buf), nullptr); CHECK(frame->type() == QUICFrameType::PATH_RESPONSE); CHECK(frame->size() == 9); const QUICPathResponseFrame *path_response_frame = static_cast(frame); CHECK(path_response_frame != nullptr); CHECK(memcmp(path_response_frame->data(), "\x01\x23\x45\x67\x89\xab\xcd\xef", QUICPathResponseFrame::DATA_LEN) == 0); + frame->~QUICFrame(); } SECTION("Load") @@ -1361,16 +1485,17 @@ TEST_CASE("Load PATH_RESPONSE Frame", "[quic]") 0x1b, // Type 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef, // Data }; - const QUICFrame *frame = QUICFrameFactory::create(frame_buf, buf, sizeof(buf)); + const QUICFrame *frame = QUICFrameFactory::create(frame_buf, buf, sizeof(buf), nullptr); CHECK(frame->type() == QUICFrameType::PATH_RESPONSE); CHECK(frame->valid() == false); + frame->~QUICFrame(); } } TEST_CASE("Store PATH_RESPONSE Frame", "[quic]") { uint8_t buf[16]; - size_t len; + size_t len = 0; uint8_t expected[] = { 0x1b, // Type @@ -1385,7 +1510,11 @@ TEST_CASE("Store PATH_RESPONSE Frame", "[quic]") QUICPathResponseFrame frame(std::move(data)); CHECK(frame.size() == 9); - frame.store(buf, &len, 16); + Ptr ibb = frame.to_io_buffer_block(sizeof(buf)); + for (auto b = ibb; b; b = b->next) { + memcpy(buf + len, b->start(), b->size()); + len += b->size(); + } CHECK(len == 9); CHECK(memcmp(buf, expected, len) == 0); } @@ -1405,7 +1534,7 @@ TEST_CASE("NEW_TOKEN Frame", "[quic]") SECTION("load") { - const QUICFrame *frame = QUICFrameFactory::create(frame_buf, raw_new_token_frame, raw_new_token_frame_len); + const QUICFrame *frame = QUICFrameFactory::create(frame_buf, raw_new_token_frame, raw_new_token_frame_len, nullptr); CHECK(frame->type() == QUICFrameType::NEW_TOKEN); CHECK(frame->size() == raw_new_token_frame_len); @@ -1413,19 +1542,21 @@ TEST_CASE("NEW_TOKEN Frame", "[quic]") CHECK(new_token_frame != nullptr); CHECK(new_token_frame->token_length() == raw_token_len); CHECK(memcmp(new_token_frame->token(), raw_token, raw_token_len) == 0); + frame->~QUICFrame(); } SECTION("bad load") { - const QUICFrame *frame = QUICFrameFactory::create(frame_buf, raw_new_token_frame, raw_new_token_frame_len - 5); + const QUICFrame *frame = QUICFrameFactory::create(frame_buf, raw_new_token_frame, raw_new_token_frame_len - 5, nullptr); CHECK(frame->type() == QUICFrameType::NEW_TOKEN); CHECK(frame->valid() == false); + frame->~QUICFrame(); } SECTION("store") { uint8_t buf[32]; - size_t len; + size_t len = 0; ats_unique_buf token = ats_unique_malloc(raw_token_len); memcpy(token.get(), raw_token, raw_token_len); @@ -1433,7 +1564,11 @@ TEST_CASE("NEW_TOKEN Frame", "[quic]") QUICNewTokenFrame frame(std::move(token), raw_token_len); CHECK(frame.size() == raw_new_token_frame_len); - frame.store(buf, &len, 16); + Ptr ibb = frame.to_io_buffer_block(sizeof(buf)); + for (auto b = ibb; b; b = b->next) { + memcpy(buf + len, b->start(), b->size()); + len += b->size(); + } CHECK(len == raw_new_token_frame_len); CHECK(memcmp(buf, raw_new_token_frame, len) == 0); } @@ -1452,7 +1587,7 @@ TEST_CASE("RETIRE_CONNECTION_ID Frame", "[quic]") SECTION("load") { const QUICFrame *frame = - QUICFrameFactory::create(frame_buf, raw_retire_connection_id_frame, raw_retire_connection_id_frame_len); + QUICFrameFactory::create(frame_buf, raw_retire_connection_id_frame, raw_retire_connection_id_frame_len, nullptr); CHECK(frame->type() == QUICFrameType::RETIRE_CONNECTION_ID); CHECK(frame->size() == raw_retire_connection_id_frame_len); @@ -1464,7 +1599,7 @@ TEST_CASE("RETIRE_CONNECTION_ID Frame", "[quic]") SECTION("bad load") { const QUICFrame *frame = - QUICFrameFactory::create(frame_buf, raw_retire_connection_id_frame, raw_retire_connection_id_frame_len - 1); + QUICFrameFactory::create(frame_buf, raw_retire_connection_id_frame, raw_retire_connection_id_frame_len - 1, nullptr); CHECK(frame->type() == QUICFrameType::RETIRE_CONNECTION_ID); CHECK(frame->valid() == false); } @@ -1472,12 +1607,16 @@ TEST_CASE("RETIRE_CONNECTION_ID Frame", "[quic]") SECTION("store") { uint8_t buf[32]; - size_t len; + size_t len = 0; QUICRetireConnectionIdFrame frame(seq_num, 0, nullptr); CHECK(frame.size() == raw_retire_connection_id_frame_len); - frame.store(buf, &len, 16); + Ptr ibb = frame.to_io_buffer_block(sizeof(buf)); + for (auto b = ibb; b; b = b->next) { + memcpy(buf + len, b->start(), b->size()); + len += b->size(); + } CHECK(len == raw_retire_connection_id_frame_len); CHECK(memcmp(buf, raw_retire_connection_id_frame, len) == 0); } @@ -1489,7 +1628,7 @@ TEST_CASE("QUICFrameFactory Create Unknown Frame", "[quic]") uint8_t buf1[] = { 0x20, // Type }; - const QUICFrame *frame1 = QUICFrameFactory::create(frame_buf, buf1, sizeof(buf1)); + const QUICFrame *frame1 = QUICFrameFactory::create(frame_buf, buf1, sizeof(buf1), nullptr); CHECK(frame1 == nullptr); } @@ -1505,13 +1644,13 @@ TEST_CASE("QUICFrameFactory Fast Create Frame", "[quic]") 0x12, // Type 0x85, 0x06, 0x07, 0x08, // Stream Data }; - const QUICFrame &frame1 = factory.fast_create(buf1, sizeof(buf1)); + const QUICFrame &frame1 = factory.fast_create(buf1, sizeof(buf1), nullptr); CHECK(frame1.type() == QUICFrameType::MAX_STREAMS); const QUICMaxStreamsFrame &max_streams_frame1 = static_cast(frame1); CHECK(max_streams_frame1.maximum_streams() == 0x01020304); - const QUICFrame &frame2 = factory.fast_create(buf2, sizeof(buf2)); + const QUICFrame &frame2 = factory.fast_create(buf2, sizeof(buf2), nullptr); CHECK(frame2.type() == QUICFrameType::MAX_STREAMS); const QUICMaxStreamsFrame &max_streams_frame2 = static_cast(frame2); @@ -1527,7 +1666,7 @@ TEST_CASE("QUICFrameFactory Fast Create Unknown Frame", "[quic]") uint8_t buf1[] = { 0x20, // Type }; - const QUICFrame &frame1 = factory.fast_create(buf1, sizeof(buf1)); + const QUICFrame &frame1 = factory.fast_create(buf1, sizeof(buf1), nullptr); CHECK(frame1.type() == QUICFrameType::UNKNOWN); } diff --git a/iocore/net/quic/test/test_QUICFrameDispatcher.cc b/iocore/net/quic/test/test_QUICFrameDispatcher.cc index be81c2e37fe..10e2d318804 100644 --- a/iocore/net/quic/test/test_QUICFrameDispatcher.cc +++ b/iocore/net/quic/test/test_QUICFrameDispatcher.cc @@ -30,20 +30,18 @@ TEST_CASE("QUICFrameHandler", "[quic]") { Ptr block = make_ptr(new_IOBufferBlock()); - block->alloc(); + block->alloc(BUFFER_SIZE_INDEX_32K); block->fill(1); CHECK(block->read_avail() == 1); QUICStreamFrame streamFrame(block, 0x03, 0); - MockQUICLDConfig ld_config; - MockQUICCCConfig cc_config; + MockQUICContext context; + MockQUICConnection connection; - MockQUICStreamManager streamManager; + MockQUICStreamManager streamManager = {&context}; MockQUICConnectionInfoProvider info; - MockQUICCongestionController cc(&info, cc_config); - QUICRTTMeasure rtt_measure; - MockQUICLossDetector lossDetector(&info, &cc, &rtt_measure, ld_config); + MockQUICLossDetector lossDetector(context); QUICFrameDispatcher quicFrameDispatcher(&info); quicFrameDispatcher.add_handler(&connection); @@ -64,14 +62,21 @@ TEST_CASE("QUICFrameHandler", "[quic]") } bool should_send_ack; bool is_flow_controlled; - quicFrameDispatcher.receive_frames(QUICEncryptionLevel::INITIAL, buf, len, should_send_ack, is_flow_controlled, nullptr); + quicFrameDispatcher.receive_frames(context, QUICEncryptionLevel::INITIAL, buf, len, should_send_ack, is_flow_controlled, nullptr, + nullptr); CHECK(connection.getTotalFrameCount() == 0); CHECK(streamManager.getTotalFrameCount() == 1); // CONNECTION_CLOSE frame QUICConnectionCloseFrame connectionCloseFrame(0, 0, "", 0, nullptr); - connectionCloseFrame.store(buf, &len, 4096); - quicFrameDispatcher.receive_frames(QUICEncryptionLevel::INITIAL, buf, len, should_send_ack, is_flow_controlled, nullptr); + ibb = connectionCloseFrame.to_io_buffer_block(sizeof(buf)); + len = 0; + for (auto b = ibb; b; b = b->next) { + memcpy(buf + len, b->start(), b->size()); + len += b->size(); + } + quicFrameDispatcher.receive_frames(context, QUICEncryptionLevel::INITIAL, buf, len, should_send_ack, is_flow_controlled, nullptr, + nullptr); CHECK(connection.getTotalFrameCount() == 1); CHECK(streamManager.getTotalFrameCount() == 1); } diff --git a/iocore/net/quic/test/test_QUICFrameRetransmitter.cc b/iocore/net/quic/test/test_QUICFrameRetransmitter.cc index 39d50b7a18c..a72ee1a267e 100644 --- a/iocore/net/quic/test/test_QUICFrameRetransmitter.cc +++ b/iocore/net/quic/test/test_QUICFrameRetransmitter.cc @@ -28,7 +28,7 @@ constexpr static uint8_t data[] = {0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x10, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x10, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x10}; -TEST_CASE("QUICFrameRetransmitter ignore frame which can not be retranmistted", "[quic]") +TEST_CASE("QUICFrameRetransmitter ignore frame which can not be retransmitted", "[quic]") { QUICFrameRetransmitter retransmitter; QUICFrameInformationUPtr info = QUICFrameInformationUPtr(quicFrameInformationAllocator.alloc()); @@ -71,7 +71,7 @@ TEST_CASE("QUICFrameRetransmitter successfully create retransmitted frame", "[qu info->level = QUICEncryptionLevel::INITIAL; Ptr block = make_ptr(new_IOBufferBlock()); - block->alloc(); + block->alloc(BUFFER_SIZE_INDEX_32K); memcpy(block->start(), data, sizeof(data)); block->fill(sizeof(data)); @@ -96,7 +96,7 @@ TEST_CASE("QUICFrameRetransmitter successfully create stream frame", "[quic]") info->level = QUICEncryptionLevel::INITIAL; Ptr block = make_ptr(new_IOBufferBlock()); - block->alloc(); + block->alloc(BUFFER_SIZE_INDEX_32K); memcpy(block->start(), data, sizeof(data)); block->fill(sizeof(data)); @@ -120,7 +120,7 @@ TEST_CASE("QUICFrameRetransmitter successfully create stream frame", "[quic]") CHECK(memcmp(stream_frame->data()->start(), data, sizeof(data)) == 0); std::destroy_at(frame); frame = nullptr; - // Becasue the info has been released, the refcount should be 1 (var block). + // Because the info has been released, the refcount should be 1 (var block). CHECK(block->refcount() == 1); } @@ -132,7 +132,7 @@ TEST_CASE("QUICFrameRetransmitter successfully split stream frame", "[quic]") info->level = QUICEncryptionLevel::INITIAL; Ptr block = make_ptr(new_IOBufferBlock()); - block->alloc(); + block->alloc(BUFFER_SIZE_INDEX_32K); memcpy(block->start(), data, sizeof(data)); block->fill(sizeof(data)); @@ -189,7 +189,7 @@ TEST_CASE("QUICFrameRetransmitter successfully split crypto frame", "[quic]") info->level = QUICEncryptionLevel::INITIAL; Ptr block = make_ptr(new_IOBufferBlock()); - block->alloc(); + block->alloc(BUFFER_SIZE_INDEX_32K); memcpy(block->start(), data, sizeof(data)); block->fill(sizeof(data)); @@ -243,7 +243,7 @@ TEST_CASE("QUICFrameRetransmitter successfully split stream frame with fin flag" info->level = QUICEncryptionLevel::INITIAL; Ptr block = make_ptr(new_IOBufferBlock()); - block->alloc(); + block->alloc(BUFFER_SIZE_INDEX_32K); memcpy(block->start(), data, sizeof(data)); block->fill(sizeof(data)); diff --git a/iocore/net/quic/test/test_QUICHandshakeProtocol.cc b/iocore/net/quic/test/test_QUICHandshakeProtocol.cc index f90b10af3fa..e69786810d9 100644 --- a/iocore/net/quic/test/test_QUICHandshakeProtocol.cc +++ b/iocore/net/quic/test/test_QUICHandshakeProtocol.cc @@ -33,11 +33,12 @@ #include -// #include "Mock.h" #include "QUICPacketHeaderProtector.h" #include "QUICPacketPayloadProtector.h" #include "QUICPacketProtectionKeyInfo.h" #include "QUICTLS.h" +#include "QUICGlobals.h" +#include "Mock.h" // XXX For NetVCOptions::reset struct PollCont; @@ -45,9 +46,6 @@ struct PollCont; #include "P_UnixNet.h" #include "P_UnixNetVConnection.h" -// depends on size of cert -static constexpr uint32_t MAX_HANDSHAKE_MSG_LEN = 8192; - #include "./server_cert.h" static void @@ -110,54 +108,63 @@ TEST_CASE("QUICHandshakeProtocol") QUICPacketProtectionKeyInfo pp_key_info_client; QUICPacketProtectionKeyInfo pp_key_info_server; NetVCOptions netvc_options; + MockQUICConnection mock_client_connection; + MockQUICConnection mock_server_connection; QUICHandshakeProtocol *client = new QUICTLS(pp_key_info_client, client_ssl_ctx, NET_VCONNECTION_OUT, netvc_options); QUICHandshakeProtocol *server = new QUICTLS(pp_key_info_server, server_ssl_ctx, NET_VCONNECTION_IN, netvc_options); + SSL_set_ex_data(static_cast(client)->ssl_handle(), QUIC::ssl_quic_qc_index, &mock_client_connection); + SSL_set_ex_data(static_cast(server)->ssl_handle(), QUIC::ssl_quic_qc_index, &mock_server_connection); QUICPacketPayloadProtector ppp_client(pp_key_info_client); QUICPacketPayloadProtector ppp_server(pp_key_info_server); - CHECK(client->initialize_key_materials({reinterpret_cast("\x83\x94\xc8\xf0\x3e\x51\x57\x00"), 8})); - CHECK(server->initialize_key_materials({reinterpret_cast("\x83\x94\xc8\xf0\x3e\x51\x57\x00"), 8})); + auto client_tp = std::make_shared(); + auto server_tp = std::make_shared(); + client_tp->set(QUICTransportParameterId::MAX_IDLE_TIMEOUT, 10); + server_tp->set(QUICTransportParameterId::MAX_IDLE_TIMEOUT, 10); + client->set_local_transport_parameters(client_tp); + server->set_local_transport_parameters(server_tp); - // CH - QUICHandshakeMsgs msg1; - uint8_t msg1_buf[MAX_HANDSHAKE_MSG_LEN] = {0}; - msg1.buf = msg1_buf; - msg1.max_buf_len = MAX_HANDSHAKE_MSG_LEN; + CHECK(client->initialize_key_materials({reinterpret_cast("\x83\x94\xc8\xf0\x3e\x51\x57\x00"), 8}, + QUIC_SUPPORTED_VERSIONS[0])); + CHECK(server->initialize_key_materials({reinterpret_cast("\x83\x94\xc8\xf0\x3e\x51\x57\x00"), 8}, + QUIC_SUPPORTED_VERSIONS[0])); - REQUIRE(client->handshake(&msg1, nullptr) == 1); + // CH + QUICHandshakeMsgs msg0; + msg0.offsets[0] = 0; + msg0.offsets[1] = 0; + msg0.offsets[2] = 0; + msg0.offsets[3] = 0; + msg0.offsets[4] = 0; + + QUICHandshakeMsgs *msg1 = nullptr; + REQUIRE(client->handshake(&msg1, &msg0) == 1); + REQUIRE(msg1); std::cout << "### Messages from client" << std::endl; - print_hex(msg1.buf, msg1.offsets[4]); + print_hex(msg1->buf, msg1->offsets[4]); // SH, EE, CERT, CV, FIN - QUICHandshakeMsgs msg2; - uint8_t msg2_buf[MAX_HANDSHAKE_MSG_LEN] = {0}; - msg2.buf = msg2_buf; - msg2.max_buf_len = MAX_HANDSHAKE_MSG_LEN; - - REQUIRE(server->handshake(&msg2, &msg1) == 1); + QUICHandshakeMsgs *msg2 = nullptr; + REQUIRE(server->handshake(&msg2, msg1) == 1); + REQUIRE(msg2); std::cout << "### Messages from server" << std::endl; - print_hex(msg2.buf, msg2.offsets[4]); + print_hex(msg2->buf, msg2->offsets[4]); // FIN - QUICHandshakeMsgs msg3; - uint8_t msg3_buf[MAX_HANDSHAKE_MSG_LEN] = {0}; - msg3.buf = msg3_buf; - msg3.max_buf_len = MAX_HANDSHAKE_MSG_LEN; + QUICHandshakeMsgs *msg3; -#ifdef SSL_MODE_QUIC_HACK - // -- Hacks for OpenSSL with SSL_MODE_QUIC_HACK -- // SH QUICHandshakeMsgs msg2_1; uint8_t msg2_1_buf[MAX_HANDSHAKE_MSG_LEN] = {0}; msg2_1.buf = msg2_1_buf; msg2_1.max_buf_len = MAX_HANDSHAKE_MSG_LEN; - memcpy(msg2_1.buf, msg2.buf, msg2.offsets[1]); + memcpy(msg2_1.buf, msg2->buf, msg2->offsets[1]); msg2_1.offsets[0] = 0; - msg2_1.offsets[1] = msg2.offsets[1]; - msg2_1.offsets[2] = msg2.offsets[1]; - msg2_1.offsets[3] = msg2.offsets[1]; - msg2_1.offsets[4] = msg2.offsets[1]; + msg2_1.offsets[1] = msg2->offsets[1]; + msg2_1.offsets[2] = msg2->offsets[1]; + msg2_1.offsets[3] = msg2->offsets[1]; + msg2_1.offsets[4] = msg2->offsets[1]; // EE - FIN QUICHandshakeMsgs msg2_2; @@ -165,8 +172,8 @@ TEST_CASE("QUICHandshakeProtocol") msg2_2.buf = msg2_2_buf; msg2_2.max_buf_len = MAX_HANDSHAKE_MSG_LEN; - size_t len = msg2.offsets[3] - msg2.offsets[2]; - memcpy(msg2_2.buf, msg2.buf + msg2.offsets[1], len); + size_t len = msg2->offsets[3] - msg2->offsets[2]; + memcpy(msg2_2.buf, msg2->buf + msg2->offsets[1], len); msg2_2.offsets[0] = 0; msg2_2.offsets[1] = 0; msg2_2.offsets[2] = 0; @@ -175,29 +182,19 @@ TEST_CASE("QUICHandshakeProtocol") REQUIRE(client->handshake(&msg3, &msg2_1) == 1); REQUIRE(client->handshake(&msg3, &msg2_2) == 1); -#else - REQUIRE(client->handshake(&msg3, &msg2) == 1); -#endif std::cout << "### Messages from client" << std::endl; - print_hex(msg3.buf, msg3.offsets[4]); + print_hex(msg3->buf, msg3->offsets[4]); // NS - QUICHandshakeMsgs msg4; - uint8_t msg4_buf[MAX_HANDSHAKE_MSG_LEN] = {0}; - msg4.buf = msg4_buf; - msg4.max_buf_len = MAX_HANDSHAKE_MSG_LEN; - - REQUIRE(server->handshake(&msg4, &msg3) == 1); + QUICHandshakeMsgs *msg4 = nullptr; + REQUIRE(server->handshake(&msg4, msg3) == 1); + REQUIRE(msg4); std::cout << "### Messages from server" << std::endl; - print_hex(msg4.buf, msg4.offsets[4]); + print_hex(msg4->buf, msg4->offsets[4]); - QUICHandshakeMsgs msg5; - uint8_t msg5_buf[MAX_HANDSHAKE_MSG_LEN] = {0}; - msg5.buf = msg5_buf; - msg5.max_buf_len = MAX_HANDSHAKE_MSG_LEN; - REQUIRE(client->handshake(&msg5, &msg4) == 1); - std::cout << "### Messages from server" << std::endl; - print_hex(msg4.buf, msg4.offsets[4]); + QUICHandshakeMsgs *msg5 = nullptr; + REQUIRE(client->handshake(&msg5, msg4) == 1); + REQUIRE(msg5 == nullptr); // encrypt - decrypt // client (encrypt) - server (decrypt) @@ -242,74 +239,77 @@ TEST_CASE("QUICHandshakeProtocol") QUICPacketProtectionKeyInfo pp_key_info_client; QUICPacketProtectionKeyInfo pp_key_info_server; NetVCOptions netvc_options; + MockQUICConnection mock_client_connection; + MockQUICConnection mock_server_connection; QUICHandshakeProtocol *client = new QUICTLS(pp_key_info_client, client_ssl_ctx, NET_VCONNECTION_OUT, netvc_options); QUICHandshakeProtocol *server = new QUICTLS(pp_key_info_server, server_ssl_ctx, NET_VCONNECTION_IN, netvc_options); + SSL_set_ex_data(static_cast(client)->ssl_handle(), QUIC::ssl_quic_qc_index, &mock_client_connection); + SSL_set_ex_data(static_cast(server)->ssl_handle(), QUIC::ssl_quic_qc_index, &mock_server_connection); QUICPacketPayloadProtector ppp_client(pp_key_info_client); QUICPacketPayloadProtector ppp_server(pp_key_info_server); - CHECK(client->initialize_key_materials({reinterpret_cast("\x83\x94\xc8\xf0\x3e\x51\x57\x00"), 8})); - CHECK(server->initialize_key_materials({reinterpret_cast("\x83\x94\xc8\xf0\x3e\x51\x57\x00"), 8})); + auto client_tp = std::make_shared(); + auto server_tp = std::make_shared(); + client_tp->set(QUICTransportParameterId::MAX_IDLE_TIMEOUT, 10); + server_tp->set(QUICTransportParameterId::MAX_IDLE_TIMEOUT, 10); + client->set_local_transport_parameters(client_tp); + server->set_local_transport_parameters(server_tp); - // CH - QUICHandshakeMsgs msg1; - uint8_t msg1_buf[MAX_HANDSHAKE_MSG_LEN] = {0}; - msg1.buf = msg1_buf; - msg1.max_buf_len = MAX_HANDSHAKE_MSG_LEN; + CHECK(client->initialize_key_materials({reinterpret_cast("\x83\x94\xc8\xf0\x3e\x51\x57\x00"), 8}, + QUIC_SUPPORTED_VERSIONS[0])); + CHECK(server->initialize_key_materials({reinterpret_cast("\x83\x94\xc8\xf0\x3e\x51\x57\x00"), 8}, + QUIC_SUPPORTED_VERSIONS[0])); - REQUIRE(client->handshake(&msg1, nullptr) == 1); + // CH + QUICHandshakeMsgs msg0; + msg0.offsets[0] = 0; + msg0.offsets[1] = 0; + msg0.offsets[2] = 0; + msg0.offsets[3] = 0; + msg0.offsets[4] = 0; + + QUICHandshakeMsgs *msg1 = nullptr; + REQUIRE(client->handshake(&msg1, &msg0) == 1); + REQUIRE(msg1); std::cout << "### Messages from client" << std::endl; - print_hex(msg1.buf, msg1.offsets[4]); + print_hex(msg1->buf, msg1->offsets[4]); // HRR - QUICHandshakeMsgs msg2; - uint8_t msg2_buf[MAX_HANDSHAKE_MSG_LEN] = {0}; - msg2.buf = msg2_buf; - msg2.max_buf_len = MAX_HANDSHAKE_MSG_LEN; - - REQUIRE(server->handshake(&msg2, &msg1) == 1); + QUICHandshakeMsgs *msg2 = nullptr; + REQUIRE(server->handshake(&msg2, msg1) == 1); + REQUIRE(msg2); std::cout << "### Messages from server" << std::endl; - print_hex(msg2.buf, msg2.offsets[4]); + print_hex(msg2->buf, msg2->offsets[4]); // CH - QUICHandshakeMsgs msg3; - uint8_t msg3_buf[MAX_HANDSHAKE_MSG_LEN] = {0}; - msg3.buf = msg3_buf; - msg3.max_buf_len = MAX_HANDSHAKE_MSG_LEN; - - REQUIRE(client->handshake(&msg3, &msg2) == 1); + QUICHandshakeMsgs *msg3 = nullptr; + REQUIRE(client->handshake(&msg3, msg2) == 1); + REQUIRE(msg3); std::cout << "### Messages from client" << std::endl; - print_hex(msg3.buf, msg3.offsets[4]); + print_hex(msg3->buf, msg3->offsets[4]); // SH, EE, CERT, CV, FIN - QUICHandshakeMsgs msg4; - uint8_t msg4_buf[MAX_HANDSHAKE_MSG_LEN] = {0}; - msg4.buf = msg4_buf; - msg4.max_buf_len = MAX_HANDSHAKE_MSG_LEN; - - REQUIRE(server->handshake(&msg4, &msg3) == 1); + QUICHandshakeMsgs *msg4 = nullptr; + REQUIRE(server->handshake(&msg4, msg3) == 1); + REQUIRE(msg4); std::cout << "### Messages from server" << std::endl; - print_hex(msg4.buf, msg4.offsets[4]); + print_hex(msg4->buf, msg4->offsets[4]); // FIN - QUICHandshakeMsgs msg5; - uint8_t msg5_buf[MAX_HANDSHAKE_MSG_LEN] = {0}; - msg5.buf = msg5_buf; - msg5.max_buf_len = MAX_HANDSHAKE_MSG_LEN; + QUICHandshakeMsgs *msg5 = nullptr; -#ifdef SSL_MODE_QUIC_HACK - // -- Hacks for OpenSSL with SSL_MODE_QUIC_HACK -- // SH QUICHandshakeMsgs msg4_1; uint8_t msg4_1_buf[MAX_HANDSHAKE_MSG_LEN] = {0}; msg4_1.buf = msg4_1_buf; msg4_1.max_buf_len = MAX_HANDSHAKE_MSG_LEN; - memcpy(msg4_1.buf, msg4.buf, msg4.offsets[1]); + memcpy(msg4_1.buf, msg4->buf, msg4->offsets[1]); msg4_1.offsets[0] = 0; - msg4_1.offsets[1] = msg4.offsets[1]; - msg4_1.offsets[2] = msg4.offsets[1]; - msg4_1.offsets[3] = msg4.offsets[1]; - msg4_1.offsets[4] = msg4.offsets[1]; + msg4_1.offsets[1] = msg4->offsets[1]; + msg4_1.offsets[2] = msg4->offsets[1]; + msg4_1.offsets[3] = msg4->offsets[1]; + msg4_1.offsets[4] = msg4->offsets[1]; // EE - FIN QUICHandshakeMsgs msg4_2; @@ -317,8 +317,8 @@ TEST_CASE("QUICHandshakeProtocol") msg4_2.buf = msg4_2_buf; msg4_2.max_buf_len = MAX_HANDSHAKE_MSG_LEN; - size_t len = msg4.offsets[3] - msg4.offsets[2]; - memcpy(msg4_2.buf, msg4.buf + msg4.offsets[1], len); + size_t len = msg4->offsets[3] - msg4->offsets[2]; + memcpy(msg4_2.buf, msg4->buf + msg4->offsets[1], len); msg4_2.offsets[0] = 0; msg4_2.offsets[1] = 0; msg4_2.offsets[2] = 0; @@ -327,21 +327,16 @@ TEST_CASE("QUICHandshakeProtocol") REQUIRE(client->handshake(&msg5, &msg4_1) == 1); REQUIRE(client->handshake(&msg5, &msg4_2) == 1); -#else - REQUIRE(client->handshake(&msg5, &msg4) == 1); -#endif + REQUIRE(msg5); std::cout << "### Messages from client" << std::endl; - print_hex(msg5.buf, msg5.offsets[4]); + print_hex(msg5->buf, msg5->offsets[4]); // NS - QUICHandshakeMsgs msg6; - uint8_t msg6_buf[MAX_HANDSHAKE_MSG_LEN] = {0}; - msg6.buf = msg6_buf; - msg6.max_buf_len = MAX_HANDSHAKE_MSG_LEN; - - REQUIRE(server->handshake(&msg6, &msg5) == 1); + QUICHandshakeMsgs *msg6 = nullptr; + REQUIRE(server->handshake(&msg6, msg5) == 1); + REQUIRE(msg6); std::cout << "### Messages from server" << std::endl; - print_hex(msg6.buf, msg6.offsets[4]); + print_hex(msg6->buf, msg6->offsets[4]); Ptr original_ibb = make_ptr(new_IOBufferBlock()); original_ibb->set_internal(const_cast(original), sizeof(original), BUFFER_SIZE_NOT_ALLOCATED); @@ -387,7 +382,8 @@ TEST_CASE("QUICHandshakeProtocol") QUICPacketProtectionKeyInfo pp_key_info_server; NetVCOptions netvc_options; QUICHandshakeProtocol *server = new QUICTLS(pp_key_info_server, server_ssl_ctx, NET_VCONNECTION_IN, netvc_options); - CHECK(server->initialize_key_materials({reinterpret_cast("\x83\x94\xc8\xf0\x3e\x51\x57\x00"), 8})); + CHECK(server->initialize_key_materials({reinterpret_cast("\x83\x94\xc8\xf0\x3e\x51\x57\x00"), 8}, + QUIC_SUPPORTED_VERSIONS[0])); // Malformed CH (finished) uint8_t msg1_buf[] = {0x14, 0x00, 0x00, 0x30, 0x35, 0xb9, 0x82, 0x9d, 0xb9, 0x14, 0x70, 0x03, 0x60, @@ -405,13 +401,10 @@ TEST_CASE("QUICHandshakeProtocol") msg1.offsets[3] = msg1_len; msg1.offsets[4] = msg1_len; - QUICHandshakeMsgs msg2; - uint8_t msg2_buf[MAX_HANDSHAKE_MSG_LEN] = {0}; - msg2.buf = msg2_buf; - msg2.max_buf_len = MAX_HANDSHAKE_MSG_LEN; - + QUICHandshakeMsgs *msg2 = nullptr; CHECK(server->handshake(&msg2, &msg1) != 1); - CHECK(msg2.error_code == 0x10a); //< 0x100 + unexpected_message(10) + CHECK(server->has_crypto_error()); + CHECK((server->crypto_error() == 0x10a || server->crypto_error() == 0x150)); //< 0x100 + unexpected_message(10) // Teardown delete server; @@ -422,50 +415,59 @@ TEST_CASE("QUICHandshakeProtocol") QUICPacketProtectionKeyInfo pp_key_info_client; QUICPacketProtectionKeyInfo pp_key_info_server; NetVCOptions netvc_options; + MockQUICConnection mock_client_connection; + MockQUICConnection mock_server_connection; QUICHandshakeProtocol *client = new QUICTLS(pp_key_info_client, client_ssl_ctx, NET_VCONNECTION_OUT, netvc_options); QUICHandshakeProtocol *server = new QUICTLS(pp_key_info_server, server_ssl_ctx, NET_VCONNECTION_IN, netvc_options); + SSL_set_ex_data(static_cast(client)->ssl_handle(), QUIC::ssl_quic_qc_index, &mock_client_connection); + SSL_set_ex_data(static_cast(server)->ssl_handle(), QUIC::ssl_quic_qc_index, &mock_server_connection); - CHECK(client->initialize_key_materials({reinterpret_cast("\x83\x94\xc8\xf0\x3e\x51\x57\x00"), 8})); - CHECK(server->initialize_key_materials({reinterpret_cast("\x83\x94\xc8\xf0\x3e\x51\x57\x00"), 8})); + auto client_tp = std::make_shared(); + auto server_tp = std::make_shared(); + client_tp->set(QUICTransportParameterId::MAX_IDLE_TIMEOUT, 10); + server_tp->set(QUICTransportParameterId::MAX_IDLE_TIMEOUT, 10); + client->set_local_transport_parameters(client_tp); + server->set_local_transport_parameters(server_tp); + + CHECK(client->initialize_key_materials({reinterpret_cast("\x83\x94\xc8\xf0\x3e\x51\x57\x00"), 8}, + QUIC_SUPPORTED_VERSIONS[0])); + CHECK(server->initialize_key_materials({reinterpret_cast("\x83\x94\xc8\xf0\x3e\x51\x57\x00"), 8}, + QUIC_SUPPORTED_VERSIONS[0])); // # Start Handshake - // CH - QUICHandshakeMsgs msg1; - uint8_t msg1_buf[MAX_HANDSHAKE_MSG_LEN] = {0}; - msg1.buf = msg1_buf; - msg1.max_buf_len = MAX_HANDSHAKE_MSG_LEN; + QUICHandshakeMsgs msg0; + msg0.offsets[0] = 0; + msg0.offsets[1] = 0; + msg0.offsets[2] = 0; + msg0.offsets[3] = 0; + msg0.offsets[4] = 0; - REQUIRE(client->handshake(&msg1, nullptr) == 1); + // CH + QUICHandshakeMsgs *msg1 = nullptr; + REQUIRE(client->handshake(&msg1, &msg0) == 1); + REQUIRE(msg1); // SH, EE, CERT, CV, FIN - QUICHandshakeMsgs msg2; - uint8_t msg2_buf[MAX_HANDSHAKE_MSG_LEN] = {0}; - msg2.buf = msg2_buf; - msg2.max_buf_len = MAX_HANDSHAKE_MSG_LEN; - - REQUIRE(server->handshake(&msg2, &msg1) == 1); + QUICHandshakeMsgs *msg2 = nullptr; + REQUIRE(server->handshake(&msg2, msg1) == 1); + REQUIRE(msg2); // FIN - QUICHandshakeMsgs msg3; - uint8_t msg3_buf[MAX_HANDSHAKE_MSG_LEN] = {0}; - msg3.buf = msg3_buf; - msg3.max_buf_len = MAX_HANDSHAKE_MSG_LEN; + QUICHandshakeMsgs *msg3 = nullptr; -#ifdef SSL_MODE_QUIC_HACK - // -- Hacks for OpenSSL with SSL_MODE_QUIC_HACK -- // SH QUICHandshakeMsgs msg2_1; uint8_t msg2_1_buf[MAX_HANDSHAKE_MSG_LEN] = {0}; msg2_1.buf = msg2_1_buf; msg2_1.max_buf_len = MAX_HANDSHAKE_MSG_LEN; - memcpy(msg2_1.buf, msg2.buf, msg2.offsets[1]); + memcpy(msg2_1.buf, msg2->buf, msg2->offsets[1]); msg2_1.offsets[0] = 0; - msg2_1.offsets[1] = msg2.offsets[1]; - msg2_1.offsets[2] = msg2.offsets[1]; - msg2_1.offsets[3] = msg2.offsets[1]; - msg2_1.offsets[4] = msg2.offsets[1]; + msg2_1.offsets[1] = msg2->offsets[1]; + msg2_1.offsets[2] = msg2->offsets[1]; + msg2_1.offsets[3] = msg2->offsets[1]; + msg2_1.offsets[4] = msg2->offsets[1]; // EE - FIN QUICHandshakeMsgs msg2_2; @@ -473,8 +475,8 @@ TEST_CASE("QUICHandshakeProtocol") msg2_2.buf = msg2_2_buf; msg2_2.max_buf_len = MAX_HANDSHAKE_MSG_LEN; - size_t len = msg2.offsets[3] - msg2.offsets[2]; - memcpy(msg2_2.buf, msg2.buf + msg2.offsets[1], len); + size_t len = msg2->offsets[3] - msg2->offsets[2]; + memcpy(msg2_2.buf, msg2->buf + msg2->offsets[1], len); msg2_2.offsets[0] = 0; msg2_2.offsets[1] = 0; msg2_2.offsets[2] = 0; @@ -483,23 +485,15 @@ TEST_CASE("QUICHandshakeProtocol") REQUIRE(client->handshake(&msg3, &msg2_1) == 1); REQUIRE(client->handshake(&msg3, &msg2_2) == 1); -#else - REQUIRE(client->handshake(&msg3, &msg2) == 1); -#endif // NS - QUICHandshakeMsgs msg4; - uint8_t msg4_buf[MAX_HANDSHAKE_MSG_LEN] = {0}; - msg4.buf = msg4_buf; - msg4.max_buf_len = MAX_HANDSHAKE_MSG_LEN; - - REQUIRE(server->handshake(&msg4, &msg3) == 1); - - QUICHandshakeMsgs msg5; - uint8_t msg5_buf[MAX_HANDSHAKE_MSG_LEN] = {0}; - msg5.buf = msg5_buf; - msg5.max_buf_len = MAX_HANDSHAKE_MSG_LEN; - REQUIRE(client->handshake(&msg5, &msg4) == 1); + QUICHandshakeMsgs *msg4 = nullptr; + REQUIRE(server->handshake(&msg4, msg3) == 1); + REQUIRE(msg4); + + QUICHandshakeMsgs *msg5 = nullptr; + REQUIRE(client->handshake(&msg5, msg4) == 1); + REQUIRE(msg5 == nullptr); // # End Handshake diff --git a/iocore/net/quic/test/test_QUICIncomingFrameBuffer.cc b/iocore/net/quic/test/test_QUICIncomingFrameBuffer.cc index 50a3dfaf364..33dcbe72eaf 100644 --- a/iocore/net/quic/test/test_QUICIncomingFrameBuffer.cc +++ b/iocore/net/quic/test/test_QUICIncomingFrameBuffer.cc @@ -35,12 +35,12 @@ TEST_CASE("QUICIncomingStreamFrameBuffer_fin_offset", "[quic]") QUICErrorUPtr err = nullptr; Ptr block_1024 = make_ptr(new_IOBufferBlock()); - block_1024->alloc(); + block_1024->alloc(BUFFER_SIZE_INDEX_32K); block_1024->fill(1024); CHECK(block_1024->read_avail() == 1024); Ptr block_0 = make_ptr(new_IOBufferBlock()); - block_0->alloc(); + block_0->alloc(BUFFER_SIZE_INDEX_32K); CHECK(block_0->read_avail() == 0); SECTION("single frame") @@ -71,7 +71,7 @@ TEST_CASE("QUICIncomingStreamFrameBuffer_fin_offset", "[quic]") buffer.insert(new QUICStreamFrame(*stream1_frame_2_r)); err = buffer.insert(new QUICStreamFrame(*stream1_frame_3_r)); CHECK(err->cls == QUICErrorClass::TRANSPORT); - CHECK(err->code == static_cast(QUICTransErrorCode::FINAL_OFFSET_ERROR)); + CHECK(err->code == static_cast(QUICTransErrorCode::FINAL_SIZE_ERROR)); buffer.clear(); @@ -82,7 +82,7 @@ TEST_CASE("QUICIncomingStreamFrameBuffer_fin_offset", "[quic]") buffer2.insert(new QUICStreamFrame(*stream1_frame_1_r)); err = buffer2.insert(new QUICStreamFrame(*stream1_frame_2_r)); CHECK(err->cls == QUICErrorClass::TRANSPORT); - CHECK(err->code == static_cast(QUICTransErrorCode::FINAL_OFFSET_ERROR)); + CHECK(err->code == static_cast(QUICTransErrorCode::FINAL_SIZE_ERROR)); buffer2.clear(); @@ -91,7 +91,7 @@ TEST_CASE("QUICIncomingStreamFrameBuffer_fin_offset", "[quic]") buffer3.insert(new QUICStreamFrame(*stream1_frame_4_r)); err = buffer3.insert(new QUICStreamFrame(*stream1_frame_3_r)); CHECK(err->cls == QUICErrorClass::TRANSPORT); - CHECK(err->code == static_cast(QUICTransErrorCode::FINAL_OFFSET_ERROR)); + CHECK(err->code == static_cast(QUICTransErrorCode::FINAL_SIZE_ERROR)); buffer3.clear(); } @@ -127,12 +127,12 @@ TEST_CASE("QUICIncomingStreamFrameBuffer_pop", "[quic]") QUICErrorUPtr err = nullptr; Ptr block_1024 = make_ptr(new_IOBufferBlock()); - block_1024->alloc(); + block_1024->alloc(BUFFER_SIZE_INDEX_32K); block_1024->fill(1024); CHECK(block_1024->read_avail() == 1024); Ptr block_0 = make_ptr(new_IOBufferBlock()); - block_0->alloc(); + block_0->alloc(BUFFER_SIZE_INDEX_32K); CHECK(block_0->read_avail() == 0); uint8_t frame_buf0[QUICFrame::MAX_INSTANCE_SIZE]; @@ -158,14 +158,19 @@ TEST_CASE("QUICIncomingStreamFrameBuffer_pop", "[quic]") auto frame = static_cast(buffer.pop()); CHECK(frame->offset() == 0); + delete frame; frame = static_cast(buffer.pop()); CHECK(frame->offset() == 1024); + delete frame; frame = static_cast(buffer.pop()); CHECK(frame->offset() == 2048); + delete frame; frame = static_cast(buffer.pop()); CHECK(frame->offset() == 3072); + delete frame; frame = static_cast(buffer.pop()); CHECK(frame->offset() == 4096); + delete frame; CHECK(buffer.empty()); buffer.clear(); @@ -179,14 +184,19 @@ TEST_CASE("QUICIncomingStreamFrameBuffer_pop", "[quic]") frame = static_cast(buffer.pop()); CHECK(frame->offset() == 0); + delete frame; frame = static_cast(buffer.pop()); CHECK(frame->offset() == 1024); + delete frame; frame = static_cast(buffer.pop()); CHECK(frame->offset() == 2048); + delete frame; frame = static_cast(buffer.pop()); CHECK(frame->offset() == 3072); + delete frame; frame = static_cast(buffer.pop()); CHECK(frame->offset() == 4096); + delete frame; CHECK(buffer.empty()); delete stream; @@ -199,12 +209,12 @@ TEST_CASE("QUICIncomingStreamFrameBuffer_dup_frame", "[quic]") QUICErrorUPtr err = nullptr; Ptr block_1024 = make_ptr(new_IOBufferBlock()); - block_1024->alloc(); + block_1024->alloc(BUFFER_SIZE_INDEX_32K); block_1024->fill(1024); CHECK(block_1024->read_avail() == 1024); Ptr block_0 = make_ptr(new_IOBufferBlock()); - block_0->alloc(); + block_0->alloc(BUFFER_SIZE_INDEX_32K); CHECK(block_0->read_avail() == 0); uint8_t frame_buf0[QUICFrame::MAX_INSTANCE_SIZE]; @@ -228,12 +238,16 @@ TEST_CASE("QUICIncomingStreamFrameBuffer_dup_frame", "[quic]") auto frame = static_cast(buffer.pop()); CHECK(frame->offset() == 0); + delete frame; frame = static_cast(buffer.pop()); CHECK(frame->offset() == 1024); + delete frame; frame = static_cast(buffer.pop()); CHECK(frame->offset() == 2048); + delete frame; frame = static_cast(buffer.pop()); CHECK(frame == nullptr); + delete frame; CHECK(buffer.empty()); buffer.clear(); @@ -251,10 +265,13 @@ TEST_CASE("QUICIncomingStreamFrameBuffer_dup_frame", "[quic]") frame = static_cast(buffer.pop()); CHECK(frame->offset() == 0); + delete frame; frame = static_cast(buffer.pop()); CHECK(frame->offset() == 1024); + delete frame; frame = static_cast(buffer.pop()); CHECK(frame->offset() == 2048); + delete frame; frame = static_cast(buffer.pop()); CHECK(frame == nullptr); CHECK(buffer.empty()); diff --git a/iocore/net/quic/test/test_QUICInvariants.cc b/iocore/net/quic/test/test_QUICInvariants.cc index 5cdbc28b632..e57aab14311 100644 --- a/iocore/net/quic/test/test_QUICInvariants.cc +++ b/iocore/net/quic/test/test_QUICInvariants.cc @@ -37,8 +37,9 @@ TEST_CASE("Long Header - regular case", "[quic]") const uint8_t buf[] = { 0x80, // Long header, Type: NONE 0x11, 0x22, 0x33, 0x44, // Version - 0x55, // DCIL/SCIL + 0x08, // DCID Len 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, // Destination Connection ID + 0x08, // SCID Len 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, // Source Connection ID }; uint64_t buf_len = sizeof(buf); @@ -53,12 +54,12 @@ TEST_CASE("Long Header - regular case", "[quic]") CHECK(version == 0x11223344); CHECK(QUICInvariants::dcil(dcil, buf, buf_len)); - CHECK(dcil == 5); + CHECK(dcil == sizeof(raw_dcid)); CHECK(QUICInvariants::dcid(dcid, buf, buf_len)); CHECK(dcid == expected_dcid); CHECK(QUICInvariants::scil(scil, buf, buf_len)); - CHECK(scil == 5); + CHECK(scil == sizeof(raw_scid)); CHECK(QUICInvariants::scid(scid, buf, buf_len)); CHECK(scid == expected_scid); } @@ -68,7 +69,8 @@ TEST_CASE("Long Header - regular case", "[quic]") const uint8_t buf[] = { 0x80, // Long header, Type: NONE 0x11, 0x22, 0x33, 0x44, // Version - 0x05, // DCIL/SCIL + 0x00, // DCID Len + 0x08, // SCID Len 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, // Source Connection ID }; uint64_t buf_len = sizeof(buf); @@ -88,7 +90,7 @@ TEST_CASE("Long Header - regular case", "[quic]") CHECK(dcid == QUICConnectionId::ZERO()); CHECK(QUICInvariants::scil(scil, buf, buf_len)); - CHECK(scil == 5); + CHECK(scil == sizeof(raw_scid)); CHECK(QUICInvariants::scid(scid, buf, buf_len)); CHECK(scid == expected_scid); } @@ -98,8 +100,9 @@ TEST_CASE("Long Header - regular case", "[quic]") const uint8_t buf[] = { 0x80, // Long header, Type: NONE 0x11, 0x22, 0x33, 0x44, // Version - 0x50, // DCIL/SCIL + 0x08, // DCID Len 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, // Destination Connection ID + 0x00, // SCID Len }; uint64_t buf_len = sizeof(buf); @@ -113,7 +116,7 @@ TEST_CASE("Long Header - regular case", "[quic]") CHECK(version == 0x11223344); CHECK(QUICInvariants::dcil(dcil, buf, buf_len)); - CHECK(dcil == 5); + CHECK(dcil == sizeof(raw_dcid)); CHECK(QUICInvariants::dcid(dcid, buf, buf_len)); CHECK(dcid == expected_dcid); @@ -143,8 +146,9 @@ TEST_CASE("Long Header - error cases", "[quic]") const uint8_t buf[] = { 0x80, // Long header, Type: NONE 0x11, 0x22, 0x33, 0x44, // Version - 0x55, // DCIL/SCIL + 0x08, // DCID Len 0x01, 0x02, 0x03, 0x04, // Invalid Destination Connection ID + 0x00, // SCID Len }; uint64_t buf_len = sizeof(buf); @@ -164,8 +168,9 @@ TEST_CASE("Long Header - error cases", "[quic]") const uint8_t buf[] = { 0x80, // Long header, Type: NONE 0x11, 0x22, 0x33, 0x44, // Version - 0x55, // DCIL/SCIL + 0x08, // DCID Len 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, // Destination Connection ID + 0x08, // SCID Len 0x11, 0x12, 0x13, 0x14, // Invalid Source Connection ID }; uint64_t buf_len = sizeof(buf); diff --git a/iocore/net/quic/test/test_QUICKeyGenerator.cc b/iocore/net/quic/test/test_QUICKeyGenerator.cc index e24d3dbb711..cd9ff38f22e 100644 --- a/iocore/net/quic/test/test_QUICKeyGenerator.cc +++ b/iocore/net/quic/test/test_QUICKeyGenerator.cc @@ -35,8 +35,8 @@ #include "QUICKeyGenerator.h" #include "QUICPacketProtectionKeyInfo.h" -// https://github.com/quicwg/base-drafts/wiki/Test-Vector-for-the-Clear-Text-AEAD-key-derivation -TEST_CASE("draft-17 Test Vectors", "[quic]") +// https://github.com/quicwg/base-drafts/wiki/Test-Vector-for-the-Initial-AEAD-key-derivation +TEST_CASE("draft-23~27 Test Vectors", "[quic]") { SECTION("CLIENT Initial") { @@ -44,21 +44,16 @@ TEST_CASE("draft-17 Test Vectors", "[quic]") QUICConnectionId cid = {reinterpret_cast("\xc6\x54\xef\xd8\xa3\x1b\x47\x92"), 8}; - uint8_t expected_client_key[] = { - 0x86, 0xd1, 0x83, 0x04, 0x80, 0xb4, 0x0f, 0x86, 0xcf, 0x9d, 0x68, 0xdc, 0xad, 0xf3, 0x5d, 0xfe, - }; - uint8_t expected_client_iv[] = { - 0x12, 0xf3, 0x93, 0x8a, 0xca, 0x34, 0xaa, 0x02, 0x54, 0x31, 0x63, 0xd4, - }; - uint8_t expected_client_hp[] = { - 0xcd, 0x25, 0x3a, 0x36, 0xff, 0x93, 0x93, 0x7c, 0x46, 0x93, 0x84, 0xa8, 0x23, 0xaf, 0x6c, 0x56, - }; + uint8_t expected_client_key[] = {0xfc, 0x4a, 0x14, 0x7a, 0x7e, 0xe9, 0x70, 0x29, 0x1b, 0x8f, 0x1c, 0x3, 0x2d, 0x2c, 0x40, 0xf9}; + uint8_t expected_client_iv[] = {0x1e, 0x6a, 0x5d, 0xdb, 0x7c, 0x1d, 0x1a, 0xa7, 0xa0, 0xfd, 0x70, 0x5}; + uint8_t expected_client_hp[] = {0x43, 0x1d, 0x22, 0x82, 0xb4, 0x7b, 0xb9, 0x3f, 0xeb, 0xd2, 0xcf, 0x19, 0x85, 0x21, 0xe2, 0xbe}; QUICPacketProtectionKeyInfo pp_key_info; pp_key_info.set_cipher_initial(EVP_aes_128_gcm()); pp_key_info.set_cipher_for_hp_initial(EVP_aes_128_ecb()); - keygen.generate(pp_key_info.encryption_key_for_hp(QUICKeyPhase::INITIAL), pp_key_info.encryption_key(QUICKeyPhase::INITIAL), - pp_key_info.encryption_iv(QUICKeyPhase::INITIAL), pp_key_info.encryption_iv_len(QUICKeyPhase::INITIAL), cid); + keygen.generate(0xff00001b, pp_key_info.encryption_key_for_hp(QUICKeyPhase::INITIAL), + pp_key_info.encryption_key(QUICKeyPhase::INITIAL), pp_key_info.encryption_iv(QUICKeyPhase::INITIAL), + pp_key_info.encryption_iv_len(QUICKeyPhase::INITIAL), cid); CHECK(pp_key_info.encryption_key_len(QUICKeyPhase::INITIAL) == sizeof(expected_client_key)); CHECK(memcmp(pp_key_info.encryption_key(QUICKeyPhase::INITIAL), expected_client_key, sizeof(expected_client_key)) == 0); @@ -74,21 +69,17 @@ TEST_CASE("draft-17 Test Vectors", "[quic]") QUICConnectionId cid = {reinterpret_cast("\xc6\x54\xef\xd8\xa3\x1b\x47\x92"), 8}; - uint8_t expected_server_key[] = { - 0x2c, 0x78, 0x63, 0x3e, 0x20, 0x6e, 0x99, 0xad, 0x25, 0x19, 0x64, 0xf1, 0x9f, 0x6d, 0xcd, 0x6d, - }; - uint8_t expected_server_iv[] = { - 0x7b, 0x50, 0xbf, 0x36, 0x98, 0xa0, 0x6d, 0xfa, 0xbf, 0x75, 0xf2, 0x87, - }; - uint8_t expected_server_hp[] = { - 0x25, 0x79, 0xd8, 0x69, 0x6f, 0x85, 0xed, 0xa6, 0x8d, 0x35, 0x02, 0xb6, 0x55, 0x96, 0x58, 0x6b, - }; + uint8_t expected_server_key[] = {0x60, 0xc0, 0x2f, 0xa6, 0x12, 0x1e, 0xb1, 0xab, + 0xa4, 0x35, 0x1f, 0x2a, 0x63, 0xb0, 0xac, 0xf8}; + uint8_t expected_server_iv[] = {0x38, 0xd, 0xf3, 0xc0, 0xf2, 0x8d, 0x94, 0x7, 0x76, 0x5c, 0x55, 0xa1}; + uint8_t expected_server_hp[] = {0x92, 0xe8, 0x67, 0xb1, 0x20, 0xb1, 0x3f, 0x40, 0x9c, 0x1a, 0xa8, 0xef, 0x54, 0x30, 0x53, 0x51}; QUICPacketProtectionKeyInfo pp_key_info; pp_key_info.set_cipher_initial(EVP_aes_128_gcm()); pp_key_info.set_cipher_for_hp_initial(EVP_aes_128_ecb()); - keygen.generate(pp_key_info.encryption_key_for_hp(QUICKeyPhase::INITIAL), pp_key_info.encryption_key(QUICKeyPhase::INITIAL), - pp_key_info.encryption_iv(QUICKeyPhase::INITIAL), pp_key_info.encryption_iv_len(QUICKeyPhase::INITIAL), cid); + keygen.generate(0xff00001b, pp_key_info.encryption_key_for_hp(QUICKeyPhase::INITIAL), + pp_key_info.encryption_key(QUICKeyPhase::INITIAL), pp_key_info.encryption_iv(QUICKeyPhase::INITIAL), + pp_key_info.encryption_iv_len(QUICKeyPhase::INITIAL), cid); CHECK(pp_key_info.encryption_key_len(QUICKeyPhase::INITIAL) == sizeof(expected_server_key)); CHECK(memcmp(pp_key_info.encryption_key(QUICKeyPhase::INITIAL), expected_server_key, sizeof(expected_server_key)) == 0); diff --git a/iocore/net/quic/test/test_QUICLossDetector.cc b/iocore/net/quic/test/test_QUICLossDetector.cc index 00167247299..5dd827143eb 100644 --- a/iocore/net/quic/test/test_QUICLossDetector.cc +++ b/iocore/net/quic/test/test_QUICLossDetector.cc @@ -24,7 +24,11 @@ #include "catch.hpp" #include "QUICLossDetector.h" +#include "QUICPacketFactory.h" +#include "QUICAckFrameCreator.h" #include "QUICEvents.h" +#include "QUICPacketFactory.h" +#include "QUICAckFrameCreator.h" #include "Mock.h" #include "tscore/ink_hrtime.h" @@ -38,15 +42,16 @@ TEST_CASE("QUICLossDetector_Loss", "[quic]") QUICAckFrameManager afm; QUICConnectionId connection_id = {reinterpret_cast("\x01"), 1}; - MockQUICCCConfig cc_config; - MockQUICLDConfig ld_config; - MockQUICConnectionInfoProvider info; - MockQUICCongestionController cc(&info, cc_config); - QUICLossDetector detector(&info, &cc, &rtt_measure, ld_config); - ats_unique_buf payload = ats_unique_malloc(512); - size_t payload_len = 512; - QUICPacketUPtr packet = QUICPacketFactory::create_null_packet(); - QUICAckFrame *frame = nullptr; + MockQUICContext context; + QUICPinger pinger; + QUICPadder padder(NetVConnectionContext_t::NET_VCONNECTION_IN); + MockQUICCongestionController cc; + QUICLossDetector detector(context, &cc, &rtt_measure, &pinger, &padder); + Ptr payload = make_ptr(new_IOBufferBlock()); + payload->alloc(iobuffer_size_to_index(512, BUFFER_SIZE_INDEX_32K)); + size_t payload_len = 512; + QUICPacketUPtr packet = QUICPacketFactory::create_null_packet(); + QUICAckFrame *frame = nullptr; SECTION("Handshake") { @@ -54,33 +59,37 @@ TEST_CASE("QUICLossDetector_Loss", "[quic]") // Check initial state uint8_t frame_buffer[1024] = {0}; CHECK(g.lost_frame_count == 0); - QUICFrame *ping_frame = g.generate_frame(frame_buffer, QUICEncryptionLevel::HANDSHAKE, 4, UINT16_MAX, 0); + QUICFrame *ping_frame = g.generate_frame(frame_buffer, QUICEncryptionLevel::HANDSHAKE, 4, UINT16_MAX, 0, 0); uint8_t raw[4]; - size_t len; - CHECK(ping_frame->store(raw, &len, 10240) < 4); + size_t len = 0; + Ptr ibb = ping_frame->to_io_buffer_block(sizeof(raw)); + for (auto b = ibb; b; b = b->next) { + memcpy(raw + len, b->start(), b->size()); + len += b->size(); + } + CHECK(len < 4); // Send SERVER_CLEARTEXT (Handshake message) - ats_unique_buf payload = ats_unique_malloc(sizeof(raw)); - memcpy(payload.get(), raw, sizeof(raw)); + payload = make_ptr(new_IOBufferBlock()); + payload->alloc(iobuffer_size_to_index(sizeof(raw), BUFFER_SIZE_INDEX_32K)); + memcpy(payload->start(), raw, sizeof(raw)); + payload->fill(sizeof(raw)); - QUICPacketHeaderUPtr header = - QUICPacketHeader::build(QUICPacketType::HANDSHAKE, QUICKeyPhase::HANDSHAKE, - {reinterpret_cast("\xff\xdd\xbb\x99\x77\x55\x33\x11"), 8}, - {reinterpret_cast("\x11\x12\x13\x14\x15\x16\x17\x18"), 8}, 0x00000001, 0, 0x00112233, - false, std::move(payload), sizeof(raw)); - QUICPacketUPtr packet = QUICPacketUPtr(new QUICPacket(std::move(header), std::move(payload), sizeof(raw), true, false), - [](QUICPacket *p) { delete p; }); - detector.on_packet_sent(QUICPacketInfoUPtr(new QUICPacketInfo{ + QUICHandshakePacket *handshake_packet = new QUICHandshakePacket( + 0x00112233, {reinterpret_cast("\xff\xdd\xbb\x99\x77\x55\x33\x11"), 8}, + {reinterpret_cast("\x11\x12\x13\x14\x15\x16\x17\x18"), 8}, sizeof(raw), 0, true, true, false); + handshake_packet->attach_payload(payload, true); + QUICPacketUPtr packet = QUICPacketUPtr(handshake_packet, [](QUICPacket *p) { delete p; }); + detector.on_packet_sent(QUICSentPacketInfoUPtr(new QUICSentPacketInfo{ packet->packet_number(), - Thread::get_hrtime(), packet->is_ack_eliciting(), - packet->is_crypto_packet(), true, packet->size(), + Thread::get_hrtime(), packet->type(), {}, - QUICPacketNumberSpace::Handshake, + QUICPacketNumberSpace::HANDSHAKE, })); ink_hrtime_sleep(HRTIME_MSECONDS(1000)); CHECK(g.lost_frame_count >= 0); @@ -98,39 +107,69 @@ TEST_CASE("QUICLossDetector_Loss", "[quic]") SECTION("1-RTT") { // Send packet (1) to (7) - QUICPacketNumberSpace pn_space = QUICPacketNumberSpace::ApplicationData; + QUICPacketNumberSpace pn_space = QUICPacketNumberSpace::APPLICATION_DATA; QUICEncryptionLevel level = QUICEncryptionLevel::ONE_RTT; - payload = ats_unique_malloc(payload_len); - QUICPacketUPtr packet1 = pf.create_protected_packet(connection_id, detector.largest_acked_packet_number(pn_space), - std::move(payload), payload_len, true, false); + payload = make_ptr(new_IOBufferBlock()); + payload->alloc(iobuffer_size_to_index(payload_len, BUFFER_SIZE_INDEX_32K)); + payload->fill(payload_len); + uint8_t packet_1_buf[QUICPacket::MAX_INSTANCE_SIZE]; + QUICPacketUPtr packet1 = pf.create_short_header_packet( + packet_1_buf, connection_id, detector.largest_acked_packet_number(pn_space), payload, payload_len, true, false); REQUIRE(packet1 != nullptr); - payload = ats_unique_malloc(payload_len); - QUICPacketUPtr packet2 = pf.create_protected_packet(connection_id, detector.largest_acked_packet_number(pn_space), - std::move(payload), payload_len, true, false); - payload = ats_unique_malloc(payload_len); - QUICPacketUPtr packet3 = pf.create_protected_packet(connection_id, detector.largest_acked_packet_number(pn_space), - std::move(payload), payload_len, true, false); - payload = ats_unique_malloc(payload_len); - QUICPacketUPtr packet4 = pf.create_protected_packet(connection_id, detector.largest_acked_packet_number(pn_space), - std::move(payload), payload_len, true, false); - payload = ats_unique_malloc(payload_len); - QUICPacketUPtr packet5 = pf.create_protected_packet(connection_id, detector.largest_acked_packet_number(pn_space), - std::move(payload), payload_len, true, false); - payload = ats_unique_malloc(payload_len); - QUICPacketUPtr packet6 = pf.create_protected_packet(connection_id, detector.largest_acked_packet_number(pn_space), - std::move(payload), payload_len, true, false); - payload = ats_unique_malloc(payload_len); - QUICPacketUPtr packet7 = pf.create_protected_packet(connection_id, detector.largest_acked_packet_number(pn_space), - std::move(payload), payload_len, true, false); - payload = ats_unique_malloc(payload_len); - QUICPacketUPtr packet8 = pf.create_protected_packet(connection_id, detector.largest_acked_packet_number(pn_space), - std::move(payload), payload_len, true, false); - payload = ats_unique_malloc(payload_len); - QUICPacketUPtr packet9 = pf.create_protected_packet(connection_id, detector.largest_acked_packet_number(pn_space), - std::move(payload), payload_len, true, false); - payload = ats_unique_malloc(payload_len); - QUICPacketUPtr packet10 = pf.create_protected_packet(connection_id, detector.largest_acked_packet_number(pn_space), - std::move(payload), payload_len, true, false); + payload = make_ptr(new_IOBufferBlock()); + payload->alloc(iobuffer_size_to_index(payload_len, BUFFER_SIZE_INDEX_32K)); + payload->fill(payload_len); + uint8_t packet_2_buf[QUICPacket::MAX_INSTANCE_SIZE]; + QUICPacketUPtr packet2 = pf.create_short_header_packet( + packet_2_buf, connection_id, detector.largest_acked_packet_number(pn_space), payload, payload_len, true, false); + payload = make_ptr(new_IOBufferBlock()); + payload->alloc(iobuffer_size_to_index(payload_len, BUFFER_SIZE_INDEX_32K)); + payload->fill(payload_len); + uint8_t packet_3_buf[QUICPacket::MAX_INSTANCE_SIZE]; + QUICPacketUPtr packet3 = pf.create_short_header_packet( + packet_3_buf, connection_id, detector.largest_acked_packet_number(pn_space), payload, payload_len, true, false); + payload = make_ptr(new_IOBufferBlock()); + payload->alloc(iobuffer_size_to_index(payload_len, BUFFER_SIZE_INDEX_32K)); + payload->fill(payload_len); + uint8_t packet_4_buf[QUICPacket::MAX_INSTANCE_SIZE]; + QUICPacketUPtr packet4 = pf.create_short_header_packet( + packet_4_buf, connection_id, detector.largest_acked_packet_number(pn_space), payload, payload_len, true, false); + payload = make_ptr(new_IOBufferBlock()); + payload->alloc(iobuffer_size_to_index(payload_len, BUFFER_SIZE_INDEX_32K)); + payload->fill(payload_len); + uint8_t packet_5_buf[QUICPacket::MAX_INSTANCE_SIZE]; + QUICPacketUPtr packet5 = pf.create_short_header_packet( + packet_5_buf, connection_id, detector.largest_acked_packet_number(pn_space), payload, payload_len, true, false); + payload = make_ptr(new_IOBufferBlock()); + payload->alloc(iobuffer_size_to_index(payload_len, BUFFER_SIZE_INDEX_32K)); + payload->fill(payload_len); + uint8_t packet_6_buf[QUICPacket::MAX_INSTANCE_SIZE]; + QUICPacketUPtr packet6 = pf.create_short_header_packet( + packet_6_buf, connection_id, detector.largest_acked_packet_number(pn_space), payload, payload_len, true, false); + payload = make_ptr(new_IOBufferBlock()); + payload->alloc(iobuffer_size_to_index(payload_len, BUFFER_SIZE_INDEX_32K)); + payload->fill(payload_len); + uint8_t packet_7_buf[QUICPacket::MAX_INSTANCE_SIZE]; + QUICPacketUPtr packet7 = pf.create_short_header_packet( + packet_7_buf, connection_id, detector.largest_acked_packet_number(pn_space), payload, payload_len, true, false); + payload = make_ptr(new_IOBufferBlock()); + payload->alloc(iobuffer_size_to_index(payload_len, BUFFER_SIZE_INDEX_32K)); + payload->fill(payload_len); + uint8_t packet_8_buf[QUICPacket::MAX_INSTANCE_SIZE]; + QUICPacketUPtr packet8 = pf.create_short_header_packet( + packet_8_buf, connection_id, detector.largest_acked_packet_number(pn_space), payload, payload_len, true, false); + payload = make_ptr(new_IOBufferBlock()); + payload->alloc(iobuffer_size_to_index(payload_len, BUFFER_SIZE_INDEX_32K)); + payload->fill(payload_len); + uint8_t packet_9_buf[QUICPacket::MAX_INSTANCE_SIZE]; + QUICPacketUPtr packet9 = pf.create_short_header_packet( + packet_9_buf, connection_id, detector.largest_acked_packet_number(pn_space), payload, payload_len, true, false); + payload = make_ptr(new_IOBufferBlock()); + payload->alloc(iobuffer_size_to_index(payload_len, BUFFER_SIZE_INDEX_32K)); + payload->fill(payload_len); + uint8_t packet_10_buf[QUICPacket::MAX_INSTANCE_SIZE]; + QUICPacketUPtr packet10 = pf.create_short_header_packet( + packet_10_buf, connection_id, detector.largest_acked_packet_number(pn_space), payload, payload_len, true, false); QUICPacketNumber pn1 = packet1->packet_number(); QUICPacketNumber pn2 = packet2->packet_number(); @@ -143,97 +182,87 @@ TEST_CASE("QUICLossDetector_Loss", "[quic]") QUICPacketNumber pn9 = packet9->packet_number(); QUICPacketNumber pn10 = packet10->packet_number(); - QUICPacketInfoUPtr packet_info = nullptr; - detector.on_packet_sent(QUICPacketInfoUPtr(new QUICPacketInfo{packet1->packet_number(), - Thread::get_hrtime(), - packet1->is_ack_eliciting(), - packet1->is_crypto_packet(), - true, - packet1->size(), - packet1->type(), - {}, - pn_space})); - detector.on_packet_sent(QUICPacketInfoUPtr(new QUICPacketInfo{packet2->packet_number(), - Thread::get_hrtime(), - packet2->is_ack_eliciting(), - packet2->is_crypto_packet(), - true, - packet2->size(), - packet2->type(), - {}, - pn_space})); - detector.on_packet_sent(QUICPacketInfoUPtr(new QUICPacketInfo{packet3->packet_number(), - Thread::get_hrtime(), - packet3->is_ack_eliciting(), - packet3->is_crypto_packet(), - true, - packet3->size(), - packet3->type(), - {}, - pn_space})); - detector.on_packet_sent(QUICPacketInfoUPtr(new QUICPacketInfo{packet4->packet_number(), - Thread::get_hrtime(), - packet4->is_ack_eliciting(), - packet4->is_crypto_packet(), - true, - packet4->size(), - packet4->type(), - {}, - pn_space})); - detector.on_packet_sent(QUICPacketInfoUPtr(new QUICPacketInfo{packet5->packet_number(), - Thread::get_hrtime(), - packet5->is_ack_eliciting(), - packet5->is_crypto_packet(), - true, - packet5->size(), - packet5->type(), - {}, - pn_space})); - detector.on_packet_sent(QUICPacketInfoUPtr(new QUICPacketInfo{packet6->packet_number(), - Thread::get_hrtime(), - packet6->is_ack_eliciting(), - packet6->is_crypto_packet(), - true, - packet6->size(), - packet6->type(), - {}, - pn_space})); - detector.on_packet_sent(QUICPacketInfoUPtr(new QUICPacketInfo{packet7->packet_number(), - Thread::get_hrtime(), - packet6->is_ack_eliciting(), - packet7->is_crypto_packet(), - true, - packet7->size(), - packet7->type(), - {}, - pn_space})); - detector.on_packet_sent(QUICPacketInfoUPtr(new QUICPacketInfo{packet8->packet_number(), - Thread::get_hrtime(), - packet6->is_ack_eliciting(), - packet8->is_crypto_packet(), - true, - packet8->size(), - packet8->type(), - {}, - pn_space})); - detector.on_packet_sent(QUICPacketInfoUPtr(new QUICPacketInfo{packet9->packet_number(), - Thread::get_hrtime(), - packet6->is_ack_eliciting(), - packet9->is_crypto_packet(), - true, - packet9->size(), - packet9->type(), - {}, - pn_space})); - detector.on_packet_sent(QUICPacketInfoUPtr(new QUICPacketInfo{packet10->packet_number(), - Thread::get_hrtime(), - packet10->is_ack_eliciting(), - packet10->is_crypto_packet(), - true, - packet10->size(), - packet10->type(), - {}, - pn_space})); + QUICSentPacketInfoUPtr packet_info = nullptr; + detector.on_packet_sent(QUICSentPacketInfoUPtr(new QUICSentPacketInfo{packet1->packet_number(), + packet1->is_ack_eliciting(), + true, + packet1->size(), + Thread::get_hrtime(), + packet1->type(), + {}, + pn_space})); + detector.on_packet_sent(QUICSentPacketInfoUPtr(new QUICSentPacketInfo{packet2->packet_number(), + packet2->is_ack_eliciting(), + true, + packet2->size(), + Thread::get_hrtime(), + packet2->type(), + {}, + pn_space})); + detector.on_packet_sent(QUICSentPacketInfoUPtr(new QUICSentPacketInfo{packet3->packet_number(), + packet3->is_ack_eliciting(), + true, + packet3->size(), + Thread::get_hrtime(), + packet3->type(), + {}, + pn_space})); + detector.on_packet_sent(QUICSentPacketInfoUPtr(new QUICSentPacketInfo{packet4->packet_number(), + packet4->is_ack_eliciting(), + true, + packet4->size(), + Thread::get_hrtime(), + packet4->type(), + {}, + pn_space})); + detector.on_packet_sent(QUICSentPacketInfoUPtr(new QUICSentPacketInfo{packet5->packet_number(), + packet5->is_ack_eliciting(), + true, + packet5->size(), + Thread::get_hrtime(), + packet5->type(), + {}, + pn_space})); + detector.on_packet_sent(QUICSentPacketInfoUPtr(new QUICSentPacketInfo{packet6->packet_number(), + packet6->is_ack_eliciting(), + true, + packet6->size(), + Thread::get_hrtime(), + packet6->type(), + {}, + pn_space})); + detector.on_packet_sent(QUICSentPacketInfoUPtr(new QUICSentPacketInfo{packet7->packet_number(), + packet6->is_ack_eliciting(), + true, + packet7->size(), + Thread::get_hrtime(), + packet7->type(), + {}, + pn_space})); + detector.on_packet_sent(QUICSentPacketInfoUPtr(new QUICSentPacketInfo{packet8->packet_number(), + packet6->is_ack_eliciting(), + true, + packet8->size(), + Thread::get_hrtime(), + packet8->type(), + {}, + pn_space})); + detector.on_packet_sent(QUICSentPacketInfoUPtr(new QUICSentPacketInfo{packet9->packet_number(), + packet6->is_ack_eliciting(), + true, + packet9->size(), + Thread::get_hrtime(), + packet9->type(), + {}, + pn_space})); + detector.on_packet_sent(QUICSentPacketInfoUPtr(new QUICSentPacketInfo{packet10->packet_number(), + packet10->is_ack_eliciting(), + true, + packet10->size(), + Thread::get_hrtime(), + packet10->type(), + {}, + pn_space})); ink_hrtime_sleep(HRTIME_MSECONDS(2000)); // Receive an ACK for (1) (4) (5) (7) (8) (9) @@ -245,7 +274,7 @@ TEST_CASE("QUICLossDetector_Loss", "[quic]") afm.update(level, pn9, payload_len, false); afm.update(level, pn10, payload_len, false); uint8_t buf[QUICFrame::MAX_INSTANCE_SIZE]; - QUICFrame *x = afm.generate_frame(buf, level, 2048, 2048, 0); + QUICFrame *x = afm.generate_frame(buf, level, 2048, 2048, 0, 0); frame = static_cast(x); ink_hrtime_sleep(HRTIME_MSECONDS(1000)); detector.handle_frame(level, *frame); @@ -263,18 +292,19 @@ TEST_CASE("QUICLossDetector_Loss", "[quic]") CHECK(cc.lost_packets.find(pn8) == cc.lost_packets.end()); CHECK(cc.lost_packets.find(pn9) == cc.lost_packets.end()); CHECK(cc.lost_packets.find(pn9) == cc.lost_packets.end()); + x->~QUICFrame(); } } TEST_CASE("QUICLossDetector_HugeGap", "[quic]") { uint8_t frame_buf[QUICFrame::MAX_INSTANCE_SIZE]; - MockQUICConnectionInfoProvider info; - MockQUICCCConfig cc_config; - MockQUICLDConfig ld_config; - MockQUICCongestionController cc(&info, cc_config); QUICRTTMeasure rtt_measure; - QUICLossDetector detector(&info, &cc, &rtt_measure, ld_config); + MockQUICContext context; + QUICPinger pinger; + QUICPadder padder(NetVConnectionContext_t::NET_VCONNECTION_IN); + MockQUICCongestionController cc; + QUICLossDetector detector(context, &cc, &rtt_measure, &pinger, &padder); auto t1 = Thread::get_hrtime(); QUICAckFrame *ack = QUICFrameFactory::create_ack_frame(frame_buf, 100000000, 100, 10000000); @@ -282,4 +312,5 @@ TEST_CASE("QUICLossDetector_HugeGap", "[quic]") detector.handle_frame(QUICEncryptionLevel::INITIAL, *ack); auto t2 = Thread::get_hrtime(); CHECK(t2 - t1 < HRTIME_MSECONDS(100)); + ack->~QUICAckFrame(); } diff --git a/iocore/net/quic/test/test_QUICPacket.cc b/iocore/net/quic/test/test_QUICPacket.cc index f50d156af65..35ffd5ea43c 100644 --- a/iocore/net/quic/test/test_QUICPacket.cc +++ b/iocore/net/quic/test/test_QUICPacket.cc @@ -25,180 +25,154 @@ #include "quic/QUICPacket.h" -TEST_CASE("QUICPacketHeader - Long", "[quic]") +TEST_CASE("Receiving Packet", "[quic]") { - SECTION("Long Header (load) Version Negotiation Packet") + const uint8_t raw_dcid[] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, // Destination Connection ID (144) + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, // + 0x10, 0x11, // + }; + QUICConnectionId dcid(raw_dcid, sizeof(raw_dcid)); + + SECTION("Version Negotiation Packet") { - const uint8_t input[] = { + uint8_t input[] = { 0xc0, // Long header, Type: NONE 0x00, 0x00, 0x00, 0x00, // Version - 0x55, // DCIL/SCIL + 0x08, // DCID Len 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, // Destination Connection ID + 0x08, // SCID Len 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, // Source Connection ID 0x00, 0x00, 0x00, 0x08, // Supported Version 1 - 0x00, 0x00, 0x00, 0x09, // Supported Version 1 + 0x00, 0x00, 0x00, 0x09, // Supported Version 2 }; - ats_unique_buf uinput = ats_unique_malloc(sizeof(input)); - memcpy(uinput.get(), input, sizeof(input)); - - QUICPacketHeaderUPtr header = QUICPacketHeader::load({}, std::move(uinput), sizeof(input), 0); - CHECK(header->size() == 22); - CHECK(header->packet_size() == 30); - CHECK(header->type() == QUICPacketType::VERSION_NEGOTIATION); - CHECK( - (header->destination_cid() == QUICConnectionId(reinterpret_cast("\x01\x02\x03\x04\x05\x06\x07\x08"), 8))); - CHECK((header->source_cid() == QUICConnectionId(reinterpret_cast("\x11\x12\x13\x14\x15\x16\x17\x18"), 8))); - CHECK(header->has_version() == true); - CHECK(header->version() == 0x00000000); + Ptr input_ibb = make_ptr(new_IOBufferBlock()); + input_ibb->set_internal(static_cast(input), sizeof(input), BUFFER_SIZE_NOT_ALLOCATED); + + QUICVersionNegotiationPacketR packet(nullptr, {}, {}, input_ibb); + CHECK(packet.type() == QUICPacketType::VERSION_NEGOTIATION); + CHECK(packet.size() == sizeof(input)); + CHECK(packet.destination_cid() == QUICConnectionId(reinterpret_cast("\x01\x02\x03\x04\x05\x06\x07\x08"), 8)); + CHECK(packet.source_cid() == QUICConnectionId(reinterpret_cast("\x11\x12\x13\x14\x15\x16\x17\x18"), 8)); + CHECK(packet.version() == 0x00000000); } - SECTION("Long Header (load) INITIAL Packet") + SECTION("INITIAL Packet") { - const uint8_t input[] = { + uint8_t input[] = { 0xc3, // Long header, Type: INITIAL 0x11, 0x22, 0x33, 0x44, // Version - 0x55, // DCIL/SCIL + 0x08, // DCID Len 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, // Destination Connection ID + 0x08, // SCID Len 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, // Source Connection ID 0x00, // Token Length (i), Token (*) - 0x02, // Payload length + 0x06, // Length 0x01, 0x23, 0x45, 0x67, // Packet number 0xff, 0xff, // Payload (dummy) }; - ats_unique_buf uinput = ats_unique_malloc(sizeof(input)); - memcpy(uinput.get(), input, sizeof(input)); - - QUICPacketHeaderUPtr header = QUICPacketHeader::load({}, std::move(uinput), sizeof(input), 0); - CHECK(header->size() == sizeof(input) - 2); // Packet Length - Payload Length - CHECK(header->packet_size() == sizeof(input)); - CHECK(header->type() == QUICPacketType::INITIAL); - CHECK( - (header->destination_cid() == QUICConnectionId(reinterpret_cast("\x01\x02\x03\x04\x05\x06\x07\x08"), 8))); - CHECK((header->source_cid() == QUICConnectionId(reinterpret_cast("\x11\x12\x13\x14\x15\x16\x17\x18"), 8))); - CHECK(header->packet_number() == 0x01234567); - CHECK(header->has_version() == true); - CHECK(header->version() == 0x11223344); + Ptr input_ibb = make_ptr(new_IOBufferBlock()); + input_ibb->set_internal(static_cast(input), sizeof(input), BUFFER_SIZE_NOT_ALLOCATED); + + QUICInitialPacketR packet(nullptr, {}, {}, input_ibb, 0); + CHECK(packet.type() == QUICPacketType::INITIAL); + CHECK(packet.size() == sizeof(input)); // Packet Length - Payload Length + CHECK(packet.destination_cid() == QUICConnectionId(reinterpret_cast("\x01\x02\x03\x04\x05\x06\x07\x08"), 8)); + CHECK(packet.source_cid() == QUICConnectionId(reinterpret_cast("\x11\x12\x13\x14\x15\x16\x17\x18"), 8)); + CHECK(packet.packet_number() == 0x01234567); + CHECK(packet.version() == 0x11223344); } - SECTION("Long Header (load) RETRY Packet") + SECTION("RETRY Packet") { - const uint8_t input[] = { + uint8_t input[] = { 0xf5, // Long header, Type: RETRY 0x11, 0x22, 0x33, 0x44, // Version - 0x55, // DCIL/SCIL + 0x08, // DCID Len 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, // Destination Connection ID + 0x08, // SCID Len 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, // Source Connection ID - 0x08, 0x07, 0x06, 0x05, 0x04, 0x03, 0x02, 0x01, // Original Destination Connection ID - 0x00, 0x10, 0x20, 0x30, 0x40, 0x50, 0x60, 0x70, // Retry Token - 0x80, 0x90, 0xa0, 0xb0, 0xc0, 0xd0, 0xe0, 0xf0, + 0x01, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, // Retry Token + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, // + 0x10, 0x11, 0x12, 0x13, 0x14, 0xf0, 0xf1, 0xf2, // + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // Retry Integrity Tag + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + }; + Ptr input_ibb = make_ptr(new_IOBufferBlock()); + input_ibb->set_internal(input, sizeof(input), BUFFER_SIZE_NOT_ALLOCATED); + + const uint8_t retry_token[] = { + 0x01, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, + 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13, 0x14, 0xf0, 0xf1, 0xf2, }; - ats_unique_buf uinput = ats_unique_malloc(sizeof(input)); - memcpy(uinput.get(), input, sizeof(input)); - - const uint8_t retry_token[] = {0x00, 0x10, 0x20, 0x30, 0x40, 0x50, 0x60, 0x70, 0x80, 0x90, 0xa0, 0xb0, 0xc0, 0xd0, 0xe0, 0xf0}; - - QUICPacketHeaderUPtr header = QUICPacketHeader::load({}, std::move(uinput), sizeof(input), 0); - CHECK(header->size() == sizeof(input) - 16); // Packet Length - Payload Length (Retry Token) - CHECK(header->packet_size() == sizeof(input)); - CHECK(header->type() == QUICPacketType::RETRY); - CHECK( - (header->destination_cid() == QUICConnectionId(reinterpret_cast("\x01\x02\x03\x04\x05\x06\x07\x08"), 8))); - CHECK((header->source_cid() == QUICConnectionId(reinterpret_cast("\x11\x12\x13\x14\x15\x16\x17\x18"), 8))); - - QUICPacketLongHeader *retry_header = static_cast(header.get()); - CHECK((retry_header->original_dcid() == - QUICConnectionId(reinterpret_cast("\x08\x07\x06\x05\x04\x03\x02\x01"), 8))); - - CHECK(memcmp(header->payload(), retry_token, 16) == 0); - CHECK(header->has_version() == true); - CHECK(header->version() == 0x11223344); + + QUICRetryPacketR packet(nullptr, {}, {}, input_ibb); + CHECK(packet.type() == QUICPacketType::RETRY); + CHECK(packet.size() == sizeof(input)); + CHECK(packet.destination_cid() == QUICConnectionId(reinterpret_cast("\x01\x02\x03\x04\x05\x06\x07\x08"), 8)); + CHECK(packet.source_cid() == QUICConnectionId(reinterpret_cast("\x11\x12\x13\x14\x15\x16\x17\x18"), 8)); + + CHECK(memcmp(packet.token().buf(), retry_token, 24) == 0); + CHECK(packet.version() == 0x11223344); } - SECTION("Long Header (store) INITIAL Packet") + SECTION("INITIAL Packet") { - uint8_t buf[64] = {0}; - size_t len = 0; - - const uint8_t expected[] = { + uint8_t input[] = { 0xc3, // Long header, Type: INITIAL 0x11, 0x22, 0x33, 0x44, // Version - 0x55, // DCIL/SCIL + 0x08, // DCID Len 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, // Destination Connection ID + 0x08, // SCID Len 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, // Source Connection ID 0x00, // Token Length (i), Token (*) - 0x19, // Length (Not 0x09 because it will have 16 bytes of AEAD tag) + 0x06, // Length 0x01, 0x23, 0x45, 0x67, // Packet number - 0x11, 0x22, 0x33, 0x44, 0x55, // Payload (dummy) + 0xff, 0xff, // Payload (dummy) }; - ats_unique_buf payload = ats_unique_malloc(5); - memcpy(payload.get(), expected + 17, 5); - - QUICPacketHeaderUPtr header = QUICPacketHeader::build( - QUICPacketType::INITIAL, QUICKeyPhase::INITIAL, {reinterpret_cast("\x01\x02\x03\x04\x05\x06\x07\x08"), 8}, - {reinterpret_cast("\x11\x12\x13\x14\x15\x16\x17\x18"), 8}, 0x01234567, 0, 0x11223344, true, - std::move(payload), 5); - - CHECK(header->size() == sizeof(expected) - 5); - CHECK(header->packet_size() == sizeof(expected)); - CHECK(header->type() == QUICPacketType::INITIAL); - CHECK( - (header->destination_cid() == QUICConnectionId(reinterpret_cast("\x01\x02\x03\x04\x05\x06\x07\x08"), 8))); - CHECK((header->source_cid() == QUICConnectionId(reinterpret_cast("\x11\x12\x13\x14\x15\x16\x17\x18"), 8))); - CHECK(header->packet_number() == 0x01234567); - CHECK(header->has_version() == true); - CHECK(header->version() == 0x11223344); - CHECK(header->is_crypto_packet()); - - header->store(buf, &len); - CHECK(len == header->size()); - CHECK(memcmp(buf, expected, len) == 0); + Ptr input_ibb = make_ptr(new_IOBufferBlock()); + input_ibb->set_internal(input, sizeof(input), BUFFER_SIZE_NOT_ALLOCATED); + + QUICInitialPacketR packet(nullptr, {}, {}, input_ibb, 0); + + CHECK(packet.type() == QUICPacketType::INITIAL); + CHECK(packet.size() == sizeof(input)); + CHECK(packet.version() == 0x11223344); + CHECK(packet.destination_cid() == QUICConnectionId(reinterpret_cast("\x01\x02\x03\x04\x05\x06\x07\x08"), 8)); + CHECK(packet.source_cid() == QUICConnectionId(reinterpret_cast("\x11\x12\x13\x14\x15\x16\x17\x18"), 8)); + CHECK(packet.token().length() == 0); + + size_t token_length; + uint8_t token_length_field_len; + size_t token_length_field_offset; + CHECK(QUICInitialPacketR::token_length(token_length, token_length_field_len, token_length_field_offset, input, sizeof(input))); + CHECK(token_length == 0); + CHECK(token_length_field_len == 1); + CHECK(token_length_field_offset == 23); } - SECTION("Long Header (store) RETRY Packet") + SECTION("Short Header Packet") { - uint8_t buf[64] = {0}; - size_t len = 0; - - const uint8_t expected[] = { - 0xf5, // Long header, Type: RETRY - 0x11, 0x22, 0x33, 0x44, // Version - 0x55, // DCIL/SCIL - 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, // Destination Connection ID - 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, // Source Connection ID - 0x08, 0x07, 0x06, 0x05, 0x04, 0x03, 0x02, 0x01, // Original Destination Connection ID - 0x00, 0x10, 0x20, 0x30, 0x40, 0x50, 0x60, 0x70, // Retry Token - 0x80, 0x90, 0xa0, 0xb0, 0xc0, 0xd0, 0xe0, 0xf0, + uint8_t input[] = { + 0x43, // Short header with (K=0) + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, // Destination Connection ID (144) + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, // + 0x10, 0x11, // + 0x01, 0x23, 0x45, 0x67, // Packet number + 0xff, 0xff, // Payload (dummy) }; - ats_unique_buf payload = ats_unique_malloc(16); - memcpy(payload.get(), expected + 30, 16); - - QUICPacketHeaderUPtr header = - QUICPacketHeader::build(QUICPacketType::RETRY, QUICKeyPhase::INITIAL, 0x11223344, - {reinterpret_cast("\x01\x02\x03\x04\x05\x06\x07\x08"), 8}, - {reinterpret_cast("\x11\x12\x13\x14\x15\x16\x17\x18"), 8}, - {reinterpret_cast("\x08\x07\x06\x05\x04\x03\x02\x01"), 8}, std::move(payload), 16); - - CHECK(header->size() == sizeof(expected) - 16); - CHECK(header->packet_size() == sizeof(expected)); - CHECK(header->type() == QUICPacketType::RETRY); - CHECK( - (header->destination_cid() == QUICConnectionId(reinterpret_cast("\x01\x02\x03\x04\x05\x06\x07\x08"), 8))); - CHECK((header->source_cid() == QUICConnectionId(reinterpret_cast("\x11\x12\x13\x14\x15\x16\x17\x18"), 8))); - CHECK(header->has_version() == true); - CHECK(header->version() == 0x11223344); - - QUICPacketLongHeader *retry_header = static_cast(header.get()); - CHECK((retry_header->original_dcid() == - QUICConnectionId(reinterpret_cast("\x08\x07\x06\x05\x04\x03\x02\x01"), 8))); - - header->store(buf, &len); - CHECK(len == header->size()); - CHECK(memcmp(buf, expected, 22) == 0); - CHECK(memcmp(buf + 22, expected + 22, 8) == 0); + Ptr input_ibb = make_ptr(new_IOBufferBlock()); + input_ibb->set_internal(static_cast(input), sizeof(input), BUFFER_SIZE_NOT_ALLOCATED); + + QUICShortHeaderPacketR packet(nullptr, {}, {}, input_ibb, 0); + CHECK(packet.size() == 25); + CHECK(packet.key_phase() == QUICKeyPhase::PHASE_0); + CHECK(packet.destination_cid() == dcid); + CHECK(packet.packet_number() == 0x01234567); } } -TEST_CASE("QUICPacketHeader - Short", "[quic]") +TEST_CASE("Sending Packet", "[quic]") { const uint8_t raw_dcid[] = { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, // Destination Connection ID (144) @@ -207,61 +181,126 @@ TEST_CASE("QUICPacketHeader - Short", "[quic]") }; QUICConnectionId dcid(raw_dcid, sizeof(raw_dcid)); - SECTION("Short Header (load)") + SECTION("Short Header Packet (store)") { - const uint8_t input[] = { + uint8_t buf[32] = {0}; + size_t len = 0; + + const uint8_t expected[] = { 0x43, // Short header with (K=0) 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, // Destination Connection ID (144) 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, // 0x10, 0x11, // 0x01, 0x23, 0x45, 0x67, // Packet number - 0xff, 0xff, // Payload (dummy) + 0x11, 0x22, 0x33, 0x44, 0x55, // Protected Payload }; - ats_unique_buf uinput = ats_unique_malloc(sizeof(input)); - memcpy(uinput.get(), input, sizeof(input)); - - QUICPacketHeaderUPtr header = QUICPacketHeader::load({}, std::move(uinput), sizeof(input), 0); - CHECK(header->size() == 23); - CHECK(header->packet_size() == 25); - CHECK(header->key_phase() == QUICKeyPhase::PHASE_0); - CHECK(header->destination_cid() == dcid); - CHECK(header->packet_number() == 0x01234567); - CHECK(header->has_version() == false); + size_t payload_len = 5; + Ptr payload = make_ptr(new_IOBufferBlock()); + payload->alloc(iobuffer_size_to_index(5, BUFFER_SIZE_INDEX_32K)); + payload->fill(5); + memcpy(payload->start(), expected + sizeof(expected) - payload_len, payload_len); + + QUICShortHeaderPacket packet(dcid, 0x1234567, 0, QUICKeyPhase::PHASE_0, true, true); + packet.attach_payload(payload, true); + + CHECK(packet.size() - 16 == 28); + CHECK(packet.key_phase() == QUICKeyPhase::PHASE_0); + CHECK(packet.type() == QUICPacketType::PROTECTED); + CHECK(packet.destination_cid() == dcid); + CHECK(packet.packet_number() == 0x01234567); + + packet.store(buf, &len); + CHECK(len == sizeof(expected)); + CHECK(memcmp(buf, expected, sizeof(expected)) == 0); } - - SECTION("Short Header (store)") + SECTION("INITIAL Packet (store)") { - uint8_t buf[32] = {0}; + uint8_t buf[64] = {0}; size_t len = 0; const uint8_t expected[] = { - 0x43, // Short header with (K=0) - 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, // Destination Connection ID (144) - 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, // - 0x10, 0x11, // + 0xc3, // Long header, Type: INITIAL + 0x11, 0x22, 0x33, 0x44, // Version + 0x08, // DCID Len + 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, // Destination Connection ID + 0x08, // SCID Len + 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, // Source Connection ID + 0x00, // Token Length (i), Token (*) + 0x19, // Length (Not 0x09 because it will have 16 bytes of AEAD tag) 0x01, 0x23, 0x45, 0x67, // Packet number - 0x11, 0x22, 0x33, 0x44, 0x55, // Protected Payload + 0x11, 0x22, 0x33, 0x44, 0x55, // Payload (dummy) }; - size_t payload_len = 5; - size_t header_len = sizeof(expected) - 5; - - ats_unique_buf payload = ats_unique_malloc(payload_len); - memcpy(payload.get(), expected + header_len, payload_len); - - QUICPacketHeaderUPtr header = - QUICPacketHeader::build(QUICPacketType::PROTECTED, QUICKeyPhase::PHASE_0, dcid, 0x01234567, 0, std::move(payload), 32); - - CHECK(header->size() == 23); - CHECK(header->packet_size() == 0); - CHECK(header->key_phase() == QUICKeyPhase::PHASE_0); - CHECK(header->type() == QUICPacketType::PROTECTED); - CHECK(header->destination_cid() == dcid); - CHECK(header->packet_number() == 0x01234567); - CHECK(header->has_version() == false); - - header->store(buf, &len); - CHECK(len == header_len); - CHECK(memcmp(buf, expected, header_len) == 0); + Ptr payload = make_ptr(new_IOBufferBlock()); + payload->alloc(iobuffer_size_to_index(5, BUFFER_SIZE_INDEX_32K)); + payload->fill(5); + memcpy(payload->start(), expected + 17, 5); + + QUICInitialPacket packet(0x11223344, {reinterpret_cast("\x01\x02\x03\x04\x05\x06\x07\x08"), 8}, + {reinterpret_cast("\x11\x12\x13\x14\x15\x16\x17\x18"), 8}, 0, nullptr, 5, 0x01234567, + true, true, true); + packet.attach_payload(payload, true); + + CHECK(packet.size() == sizeof(expected) + 16); + CHECK(packet.type() == QUICPacketType::INITIAL); + CHECK((packet.destination_cid() == QUICConnectionId(reinterpret_cast("\x01\x02\x03\x04\x05\x06\x07\x08"), 8))); + CHECK((packet.source_cid() == QUICConnectionId(reinterpret_cast("\x11\x12\x13\x14\x15\x16\x17\x18"), 8))); + CHECK(packet.packet_number() == 0x01234567); + CHECK(packet.version() == 0x11223344); + CHECK(packet.is_crypto_packet()); + + packet.store(buf, &len); + CHECK(len == packet.size() - 16); + CHECK(memcmp(buf, expected, len - 16) == 0); + } + + SECTION("RETRY Packet (store)") + { + uint8_t buf[128] = {0}; + size_t len = 0; + + const uint8_t expected[] = { + 0xf0, // Long header, Type: RETRY + 0x11, 0x22, 0x33, 0x44, // Version + 0x08, // DCID Len + 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, // Destination Connection ID + 0x08, // SCID Len + 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, // Source Connection ID + 0x01, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, // Retry Token + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, // + 0x10, 0x11, 0x12, 0x13, 0x14, 0x08, 0x01, 0x02, // + 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x08, 0x11, // + 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, // + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // Retry Integrity Tag + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + }; + QUICRetryToken token(expected + 23, 39); + + QUICRetryPacket packet(0x11223344, {reinterpret_cast("\x01\x02\x03\x04\x05\x06\x07\x08"), 8}, + {reinterpret_cast("\x11\x12\x13\x14\x15\x16\x17\x18"), 8}, token); + CHECK(packet.size() == sizeof(expected)); + CHECK(packet.type() == QUICPacketType::RETRY); + CHECK((packet.destination_cid() == QUICConnectionId(reinterpret_cast("\x01\x02\x03\x04\x05\x06\x07\x08"), 8))); + CHECK((packet.source_cid() == QUICConnectionId(reinterpret_cast("\x11\x12\x13\x14\x15\x16\x17\x18"), 8))); + CHECK(packet.version() == 0x11223344); + + packet.store(buf, &len); + CHECK(len == packet.size()); + CHECK(memcmp(buf, expected, sizeof(expected) - 16) == 0); + } + + SECTION("VersionNegotiation Packet") + { + QUICConnectionId dummy; + QUICVersionNegotiationPacket vn1(dummy, dummy, QUIC_SUPPORTED_VERSIONS, countof(QUIC_SUPPORTED_VERSIONS), + QUIC_EXERCISE_VERSION1); + for (auto i = 0; i < vn1.nversions(); ++i) { + REQUIRE(vn1.versions()[i] != QUIC_EXERCISE_VERSION1); + } + QUICVersionNegotiationPacket vn2(dummy, dummy, QUIC_SUPPORTED_VERSIONS, countof(QUIC_SUPPORTED_VERSIONS), + QUIC_EXERCISE_VERSION2); + for (auto i = 0; i < vn2.nversions(); ++i) { + REQUIRE(vn2.versions()[i] != QUIC_EXERCISE_VERSION2); + } } } @@ -291,3 +330,379 @@ TEST_CASE("Decoding Packet Number 1", "[quic]") QUICPacket::decode_packet_number(dst, src, len, base); CHECK(dst == 0xaa8309b3); } + +TEST_CASE("read_essential_info", "[quic]") +{ + SECTION("Long header packet - INITIAL") + { + uint8_t input[] = { + 0xc3, // Long header, Type: INITIAL + 0x11, 0x22, 0x33, 0x44, // Version + 0x08, // DCID Len + 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, // Destination Connection ID + 0x08, // SCID Len + 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, // Source Connection ID + 0x00, // Token Length (i), Token (*) + 0x06, // Length + 0x01, 0x23, 0x45, 0x67, // Packet number + 0xff, 0xff, // Payload (dummy) + }; + + QUICConnectionId expected_dcid(input + 6, 8); + QUICConnectionId expected_scid(input + 15, 8); + + Ptr input_ibb = make_ptr(new_IOBufferBlock()); + input_ibb->set_internal(static_cast(input), sizeof(input), BUFFER_SIZE_NOT_ALLOCATED); + + QUICPacketType type; + QUICVersion version; + QUICConnectionId dcid; + QUICConnectionId scid; + QUICPacketNumber packet_number; + QUICKeyPhase key_phase; + bool result = QUICPacketR::read_essential_info(input_ibb, type, version, dcid, scid, packet_number, 0, key_phase); + + REQUIRE(result); + CHECK(type == QUICPacketType::INITIAL); + CHECK(version == 0x11223344); + CHECK(dcid == expected_dcid); + CHECK(scid == expected_scid); + CHECK(packet_number == 0x01234567); + } + + SECTION("Long header packet - INITIAL - 0 length CID") + { + uint8_t input[] = { + 0xc2, // Long header, Type: INITIAL + 0xff, 0x00, 0x00, 0x19, // Version + 0x08, // DCID Len + 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, // Destination Connection ID + 0x00, // SCID Len + 0x00, // Token Length (i), Token (*) + 0x42, 0x17, // Length + 0x00, 0x00, 0x00 // Packet number + }; + + Ptr input_ibb = make_ptr(new_IOBufferBlock()); + input_ibb->set_internal(static_cast(input), sizeof(input), BUFFER_SIZE_NOT_ALLOCATED); + + QUICPacketType type; + QUICVersion version; + QUICConnectionId dcid; + QUICConnectionId scid; + QUICPacketNumber packet_number; + QUICKeyPhase key_phase; + bool result = QUICPacketR::read_essential_info(input_ibb, type, version, dcid, scid, packet_number, 0, key_phase); + + REQUIRE(result); + } + + SECTION("Long header packet - RETRY") + { + uint8_t input[] = { + 0xf0, // Long header, Type: RETRY + 0x11, 0x22, 0x33, 0x44, // Version + 0x08, // DCID Len + 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, // Destination Connection ID + 0x08, // SCID Len + 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, // Source Connection ID + 0x01, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, // Retry Token + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, // + 0x10, 0x11, 0x12, 0x13, 0x14, 0xf0, 0xf1, 0xf2, // + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // Retry Integrity Tag + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + }; + + QUICConnectionId expected_dcid(input + 6, 8); + QUICConnectionId expected_scid(input + 15, 8); + + Ptr input_ibb = make_ptr(new_IOBufferBlock()); + input_ibb->set_internal(static_cast(input), sizeof(input), BUFFER_SIZE_NOT_ALLOCATED); + + QUICPacketType type; + QUICVersion version; + QUICConnectionId dcid; + QUICConnectionId scid; + QUICPacketNumber packet_number; + QUICKeyPhase key_phase; + bool result = QUICPacketR::read_essential_info(input_ibb, type, version, dcid, scid, packet_number, 0, key_phase); + + REQUIRE(result); + CHECK(type == QUICPacketType::RETRY); + CHECK(version == 0x11223344); + CHECK(dcid == expected_dcid); + CHECK(scid == expected_scid); + } + + SECTION("Long header packet - Version Negotiation") + { + uint8_t input[] = { + 0xd9, // Long header, Type: RETRY + 0x00, 0x00, 0x00, 0x00, // Version + 0x08, // DCID Len + 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, // Destination Connection ID + 0x08, // SCID Len + 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, // Source Connection ID + 0xff, 0x00, 0x00, 0x19, // Supported Version 1 + 0xa1, 0xa2, 0xa3, 0xa4, // Supported Version 2 + }; + + QUICConnectionId expected_dcid(input + 6, 8); + QUICConnectionId expected_scid(input + 15, 8); + + Ptr input_ibb = make_ptr(new_IOBufferBlock()); + input_ibb->set_internal(static_cast(input), sizeof(input), BUFFER_SIZE_NOT_ALLOCATED); + + QUICPacketType type; + QUICVersion version; + QUICConnectionId dcid; + QUICConnectionId scid; + QUICPacketNumber packet_number; + QUICKeyPhase key_phase; + bool result = QUICPacketR::read_essential_info(input_ibb, type, version, dcid, scid, packet_number, 0, key_phase); + + REQUIRE(result); + CHECK(type == QUICPacketType::VERSION_NEGOTIATION); + CHECK(version == 0x00); + CHECK(dcid == expected_dcid); + CHECK(scid == expected_scid); + } + + SECTION("Short header packet") + { + uint8_t input[] = { + 0x43, // Short header with (K=0) + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, // Destination Connection ID (144) + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, // + 0x10, 0x11, // + 0x01, 0x23, 0x45, 0x67, // Packet number + 0xff, 0xff, // Payload (dummy) + }; + + QUICConnectionId expected_dcid(input + 1, 18); + + Ptr input_ibb = make_ptr(new_IOBufferBlock()); + input_ibb->set_internal(static_cast(input), sizeof(input), BUFFER_SIZE_NOT_ALLOCATED); + QUICPacketType type; + QUICVersion version; + QUICConnectionId dcid; + QUICConnectionId scid; + QUICPacketNumber packet_number; + QUICKeyPhase key_phase; + bool result = QUICPacketR::read_essential_info(input_ibb, type, version, dcid, scid, packet_number, 0, key_phase); + + REQUIRE(result); + CHECK(type == QUICPacketType::PROTECTED); + CHECK(key_phase == QUICKeyPhase::PHASE_0); + CHECK(dcid == expected_dcid); + CHECK(packet_number == 0x01234567); + } + + SECTION("Long header packet - Malformed INITIAL 1") + { + uint8_t input[] = { + 0xc3, // Long header, Type: INITIAL + }; + + Ptr input_ibb = make_ptr(new_IOBufferBlock()); + input_ibb->set_internal(static_cast(input), sizeof(input), BUFFER_SIZE_NOT_ALLOCATED); + + QUICPacketType type; + QUICVersion version; + QUICConnectionId dcid; + QUICConnectionId scid; + QUICPacketNumber packet_number; + QUICKeyPhase key_phase; + bool result = QUICPacketR::read_essential_info(input_ibb, type, version, dcid, scid, packet_number, 0, key_phase); + + REQUIRE(!result); + } + + SECTION("Long header packet - Malformed INITIAL 2") + { + uint8_t input[] = { + 0xc3, // Long header, Type: INITIAL + 0x11, 0x22, // Version + }; + + Ptr input_ibb = make_ptr(new_IOBufferBlock()); + input_ibb->set_internal(static_cast(input), sizeof(input), BUFFER_SIZE_NOT_ALLOCATED); + + QUICPacketType type; + QUICVersion version; + QUICConnectionId dcid; + QUICConnectionId scid; + QUICPacketNumber packet_number; + QUICKeyPhase key_phase; + bool result = QUICPacketR::read_essential_info(input_ibb, type, version, dcid, scid, packet_number, 0, key_phase); + + REQUIRE(!result); + } + + SECTION("Long header packet - Malformed INITIAL 3") + { + uint8_t input[] = { + 0xc3, // Long header, Type: INITIAL + 0x11, 0x22, 0x33, 0x44, // Version + }; + + Ptr input_ibb = make_ptr(new_IOBufferBlock()); + input_ibb->set_internal(static_cast(input), sizeof(input), BUFFER_SIZE_NOT_ALLOCATED); + + QUICPacketType type; + QUICVersion version; + QUICConnectionId dcid; + QUICConnectionId scid; + QUICPacketNumber packet_number; + QUICKeyPhase key_phase; + bool result = QUICPacketR::read_essential_info(input_ibb, type, version, dcid, scid, packet_number, 0, key_phase); + + REQUIRE(!result); + } + + SECTION("Long header packet - Malformed INITIAL 4") + { + uint8_t input[] = { + 0xc3, // Long header, Type: INITIAL + 0x11, 0x22, 0x33, 0x44, // Version + 0x08, // DCID Len + }; + + Ptr input_ibb = make_ptr(new_IOBufferBlock()); + input_ibb->set_internal(static_cast(input), sizeof(input), BUFFER_SIZE_NOT_ALLOCATED); + + QUICPacketType type; + QUICVersion version; + QUICConnectionId dcid; + QUICConnectionId scid; + QUICPacketNumber packet_number; + QUICKeyPhase key_phase; + bool result = QUICPacketR::read_essential_info(input_ibb, type, version, dcid, scid, packet_number, 0, key_phase); + + REQUIRE(!result); + } + + SECTION("Long header packet - Malformed INITIAL 5") + { + uint8_t input[] = { + 0xc3, // Long header, Type: INITIAL + 0x11, 0x22, 0x33, 0x44, // Version + 0x08, // DCID Len + 0x01, 0x02, 0x03, 0x04, 0x05, // Destination Connection ID + }; + + Ptr input_ibb = make_ptr(new_IOBufferBlock()); + input_ibb->set_internal(static_cast(input), sizeof(input), BUFFER_SIZE_NOT_ALLOCATED); + + QUICPacketType type; + QUICVersion version; + QUICConnectionId dcid; + QUICConnectionId scid; + QUICPacketNumber packet_number; + QUICKeyPhase key_phase; + bool result = QUICPacketR::read_essential_info(input_ibb, type, version, dcid, scid, packet_number, 0, key_phase); + + REQUIRE(!result); + } + + SECTION("Long header packet - Malformed INITIAL 6") + { + uint8_t input[] = { + 0xc3, // Long header, Type: INITIAL + 0x11, 0x22, 0x33, 0x44, // Version + 0x08, // DCID Len + 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, // Destination Connection ID + }; + + Ptr input_ibb = make_ptr(new_IOBufferBlock()); + input_ibb->set_internal(static_cast(input), sizeof(input), BUFFER_SIZE_NOT_ALLOCATED); + + QUICPacketType type; + QUICVersion version; + QUICConnectionId dcid; + QUICConnectionId scid; + QUICPacketNumber packet_number; + QUICKeyPhase key_phase; + bool result = QUICPacketR::read_essential_info(input_ibb, type, version, dcid, scid, packet_number, 0, key_phase); + + REQUIRE(!result); + } + + SECTION("Long header packet - Malformed INITIAL 7") + { + uint8_t input[] = { + 0xc3, // Long header, Type: INITIAL + 0x11, 0x22, 0x33, 0x44, // Version + 0x08, // DCID Len + 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, // Destination Connection ID + 0x08, // SCID Len + 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, // Source Connection ID + }; + + Ptr input_ibb = make_ptr(new_IOBufferBlock()); + input_ibb->set_internal(static_cast(input), sizeof(input), BUFFER_SIZE_NOT_ALLOCATED); + + QUICPacketType type; + QUICVersion version; + QUICConnectionId dcid; + QUICConnectionId scid; + QUICPacketNumber packet_number; + QUICKeyPhase key_phase; + bool result = QUICPacketR::read_essential_info(input_ibb, type, version, dcid, scid, packet_number, 0, key_phase); + + REQUIRE(!result); + } + SECTION("Long header packet - Malformed INITIAL 8") + { + uint8_t input[] = { + 0xc3, // Long header, Type: INITIAL + 0x11, 0x22, 0x33, 0x44, // Version + 0x08, // DCID Len + 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, // Destination Connection ID + 0x08, // SCID Len + 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, // Source Connection ID + 0x80, // Token Length (i), Token (*) + }; + + Ptr input_ibb = make_ptr(new_IOBufferBlock()); + input_ibb->set_internal(static_cast(input), sizeof(input), BUFFER_SIZE_NOT_ALLOCATED); + + QUICPacketType type; + QUICVersion version; + QUICConnectionId dcid; + QUICConnectionId scid; + QUICPacketNumber packet_number; + QUICKeyPhase key_phase; + bool result = QUICPacketR::read_essential_info(input_ibb, type, version, dcid, scid, packet_number, 0, key_phase); + + REQUIRE(!result); + } + + SECTION("Long header packet - Malformed INITIAL 9") + { + uint8_t input[] = { + 0xc3, // Long header, Type: INITIAL + 0x11, 0x22, 0x33, 0x44, // Version + 0x08, // DCID Len + 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, // Destination Connection ID + 0x08, // SCID Len + 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, // Source Connection ID + 0x00, // Token Length (i), Token (*) + 0x06, // Length + 0x01, 0x23, // Packet number + }; + + Ptr input_ibb = make_ptr(new_IOBufferBlock()); + input_ibb->set_internal(static_cast(input), sizeof(input), BUFFER_SIZE_NOT_ALLOCATED); + + QUICPacketType type; + QUICVersion version; + QUICConnectionId dcid; + QUICConnectionId scid; + QUICPacketNumber packet_number; + QUICKeyPhase key_phase; + bool result = QUICPacketR::read_essential_info(input_ibb, type, version, dcid, scid, packet_number, 0, key_phase); + + REQUIRE(!result); + } +} diff --git a/iocore/net/quic/test/test_QUICPacketFactory.cc b/iocore/net/quic/test/test_QUICPacketFactory.cc index 08d94b494ca..57bf6b34885 100644 --- a/iocore/net/quic/test/test_QUICPacketFactory.cc +++ b/iocore/net/quic/test/test_QUICPacketFactory.cc @@ -24,6 +24,7 @@ #include "catch.hpp" #include "quic/QUICPacket.h" +#include "quic/QUICPacketFactory.h" #include "quic/Mock.h" TEST_CASE("QUICPacketFactory_Create_VersionNegotiationPacket", "[quic]") @@ -36,29 +37,32 @@ TEST_CASE("QUICPacketFactory_Create_VersionNegotiationPacket", "[quic]") QUICConnectionId dcid(raw_dcid, 8); QUICConnectionId scid(raw_scid, 8); - QUICPacketUPtr vn_packet = factory.create_version_negotiation_packet(scid, dcid); + QUICPacketUPtr packet = factory.create_version_negotiation_packet(scid, dcid, QUIC_EXERCISE_VERSION1); + REQUIRE(packet != nullptr); - REQUIRE(vn_packet != nullptr); - CHECK(vn_packet->type() == QUICPacketType::VERSION_NEGOTIATION); - CHECK(vn_packet->destination_cid() == scid); - CHECK(vn_packet->source_cid() == dcid); - CHECK(vn_packet->version() == 0x00); + QUICVersionNegotiationPacket &vn_packet = static_cast(*packet); + CHECK(vn_packet.type() == QUICPacketType::VERSION_NEGOTIATION); + CHECK(vn_packet.destination_cid() == scid); + CHECK(vn_packet.source_cid() == dcid); + CHECK(vn_packet.version() == 0x00); - QUICVersion supported_version = QUICTypeUtil::read_QUICVersion(vn_packet->payload()); + QUICVersion supported_version = QUICTypeUtil::read_QUICVersion(reinterpret_cast(vn_packet.payload_block()->start())); CHECK(supported_version == QUIC_SUPPORTED_VERSIONS[0]); uint8_t expected[] = { 0xa7, // Long header, Type: NONE 0x00, 0x00, 0x00, 0x00, // Version - 0x55, // DCIL/SCIL + 0x08, // DCID Len 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, // Destination Connection ID + 0x08, // SCID Len 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, // Source Connection ID - 0xff, 0x00, 0x00, 0x14, // Supported Version - 0x1a, 0x2a, 0x3a, 0x4a, // Excercise Version + 0xff, 0x00, 0x00, 0x1d, // Supported Version + 0xff, 0x00, 0x00, 0x1b, // Supported Version + 0x5a, 0x6a, 0x7a, 0x8a, // Exercise Version }; uint8_t buf[1024] = {0}; size_t buf_len; - vn_packet->store(buf, &buf_len); + vn_packet.store(buf, &buf_len); CHECK((buf[0] & 0x80) == 0x80); // Lower 7 bits of the first byte is random CHECK(memcmp(buf + 1, expected + 1, buf_len - 1) == 0); } @@ -72,17 +76,17 @@ TEST_CASE("QUICPacketFactory_Create_Retry", "[quic]") uint8_t raw[] = {0xaa, 0xbb, 0xcc, 0xdd}; QUICRetryToken token(raw, 4); - QUICPacketUPtr packet = - factory.create_retry_packet(QUICConnectionId(reinterpret_cast("\x01\x02\x03\x04"), 4), - QUICConnectionId(reinterpret_cast("\x11\x12\x13\x14"), 4), - QUICConnectionId(reinterpret_cast("\x04\x03\x02\x01"), 4), token); + QUICPacketUPtr packet = factory.create_retry_packet( + QUIC_SUPPORTED_VERSIONS[0], QUICConnectionId(reinterpret_cast("\x01\x02\x03\x04"), 4), + QUICConnectionId(reinterpret_cast("\x11\x12\x13\x14"), 4), token); REQUIRE(packet != nullptr); - CHECK(packet->type() == QUICPacketType::RETRY); - CHECK((packet->destination_cid() == QUICConnectionId(reinterpret_cast("\x01\x02\x03\x04"), 4))); - CHECK(memcmp(packet->payload(), raw, sizeof(raw)) == 0); - CHECK(packet->packet_number() == 0); - CHECK(packet->version() == QUIC_SUPPORTED_VERSIONS[0]); + + QUICRetryPacket &retry_packet = static_cast(*packet); + CHECK(retry_packet.type() == QUICPacketType::RETRY); + CHECK(retry_packet.destination_cid() == QUICConnectionId(reinterpret_cast("\x01\x02\x03\x04"), 4)); + CHECK(retry_packet.version() == QUIC_SUPPORTED_VERSIONS[0]); + CHECK(retry_packet.token() == token); } TEST_CASE("QUICPacketFactory_Create_Handshake", "[quic]") @@ -92,20 +96,24 @@ TEST_CASE("QUICPacketFactory_Create_Handshake", "[quic]") QUICPacketFactory factory(pp_key_info); factory.set_version(0x11223344); - uint8_t raw[] = {0xaa, 0xbb, 0xcc, 0xdd}; - ats_unique_buf payload = ats_unique_malloc(sizeof(raw)); - memcpy(payload.get(), raw, sizeof(raw)); + uint8_t raw[] = {0xaa, 0xbb, 0xcc, 0xdd}; + Ptr payload = make_ptr(new_IOBufferBlock()); + payload->alloc(iobuffer_size_to_index(sizeof(raw), BUFFER_SIZE_INDEX_32K)); + payload->fill(sizeof(raw)); + memcpy(payload->start(), raw, sizeof(raw)); - QUICPacketUPtr packet = - factory.create_handshake_packet(QUICConnectionId(reinterpret_cast("\x01\x02\x03\x04"), 4), - QUICConnectionId(reinterpret_cast("\x11\x12\x13\x14"), 4), 0, - std::move(payload), sizeof(raw), true, false, true); + uint8_t packet_buf[QUICPacket::MAX_INSTANCE_SIZE]; + QUICPacketUPtr packet = factory.create_handshake_packet( + packet_buf, QUICConnectionId(reinterpret_cast("\x01\x02\x03\x04"), 4), + QUICConnectionId(reinterpret_cast("\x11\x12\x13\x14"), 4), 0, payload, sizeof(raw), true, false, true); REQUIRE(packet != nullptr); - CHECK(packet->type() == QUICPacketType::HANDSHAKE); - CHECK((packet->destination_cid() == QUICConnectionId(reinterpret_cast("\x01\x02\x03\x04"), 4))); - CHECK(memcmp(packet->payload(), raw, sizeof(raw)) != 0); - CHECK(packet->packet_number() <= 0xFFFFFBFF); - CHECK(packet->version() == 0x11223344); + + QUICHandshakePacket &handshake_packet = reinterpret_cast(*packet); + CHECK(handshake_packet.type() == QUICPacketType::HANDSHAKE); + CHECK(handshake_packet.destination_cid() == QUICConnectionId(reinterpret_cast("\x01\x02\x03\x04"), 4)); + CHECK(memcmp(handshake_packet.payload_block()->start(), raw, sizeof(raw)) != 0); + CHECK(handshake_packet.packet_number() <= 0xFFFFFBFF); + CHECK(handshake_packet.version() == 0x11223344); } TEST_CASE("QUICPacketFactory_Create_StatelessResetPacket", "[quic]") @@ -114,10 +122,11 @@ TEST_CASE("QUICPacketFactory_Create_StatelessResetPacket", "[quic]") QUICPacketFactory factory(pp_key_info); QUICStatelessResetToken token({reinterpret_cast("\x30\x39"), 2}, 67890); - QUICPacketUPtr packet = - factory.create_stateless_reset_packet(QUICConnectionId(reinterpret_cast("\x01\x02\x03\x04"), 4), token); + QUICPacketUPtr packet = factory.create_stateless_reset_packet(token, 1200); REQUIRE(packet != nullptr); CHECK(packet->type() == QUICPacketType::STATELESS_RESET); - CHECK((packet->destination_cid() == QUICConnectionId(reinterpret_cast("\x01\x02\x03\x04"), 4))); + + QUICStatelessResetPacket *sr_packet = dynamic_cast(packet.get()); + CHECK(sr_packet->token() == token); } diff --git a/iocore/net/quic/test/test_QUICPacketHeaderProtector.cc b/iocore/net/quic/test/test_QUICPacketHeaderProtector.cc index dc754a4a2f6..06255956109 100644 --- a/iocore/net/quic/test/test_QUICPacketHeaderProtector.cc +++ b/iocore/net/quic/test/test_QUICPacketHeaderProtector.cc @@ -26,15 +26,14 @@ #include "QUICPacketProtectionKeyInfo.h" #include "QUICPacketHeaderProtector.h" #include "QUICTLS.h" +#include "QUICGlobals.h" +#include "Mock.h" struct PollCont; #include "P_UDPConnection.h" #include "P_UnixNet.h" #include "P_UnixNetVConnection.h" -// depends on size of cert -static constexpr uint32_t MAX_HANDSHAKE_MSG_LEN = 8192; - #include "./server_cert.h" TEST_CASE("QUICPacketHeaderProtector") @@ -70,13 +69,19 @@ TEST_CASE("QUICPacketHeaderProtector") SECTION("Long header", "[quic]") { uint8_t original[] = { - 0xC3, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, - 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, - 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, - 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, + 0xc3, // Long header, Type: INITIAL + 0x11, 0x22, 0x33, 0x44, // Version + 0x08, // DCID Len + 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, // Destination Connection ID + 0x08, // SCID Len + 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, // Source Connection ID + 0x00, // Token Length (i), Token (*) + 0x19, // Length (Not 0x09 because it will have 16 bytes of AEAD tag) + 0x01, 0x23, 0x45, 0x67, // Packet number + 0x11, 0x22, 0x33, 0x44, 0x55, // Payload (dummy) }; uint8_t tmp[64]; - memcpy(tmp, original, sizeof(tmp)); + memcpy(tmp, original, sizeof(original)); QUICPacketProtectionKeyInfo pp_key_info_client; QUICPacketProtectionKeyInfo pp_key_info_server; @@ -84,22 +89,27 @@ TEST_CASE("QUICPacketHeaderProtector") QUICHandshakeProtocol *client = new QUICTLS(pp_key_info_client, client_ssl_ctx, NET_VCONNECTION_OUT, netvc_options); QUICHandshakeProtocol *server = new QUICTLS(pp_key_info_server, server_ssl_ctx, NET_VCONNECTION_IN, netvc_options); - CHECK(client->initialize_key_materials({reinterpret_cast("\x83\x94\xc8\xf0\x3e\x51\x57\x00"), 8})); - CHECK(server->initialize_key_materials({reinterpret_cast("\x83\x94\xc8\xf0\x3e\x51\x57\x00"), 8})); + CHECK(client->initialize_key_materials({reinterpret_cast("\x83\x94\xc8\xf0\x3e\x51\x57\x00"), 8}, + QUIC_SUPPORTED_VERSIONS[0])); + CHECK(server->initialize_key_materials({reinterpret_cast("\x83\x94\xc8\xf0\x3e\x51\x57\x00"), 8}, + QUIC_SUPPORTED_VERSIONS[0])); QUICPacketHeaderProtector client_ph_protector(pp_key_info_client); QUICPacketHeaderProtector server_ph_protector(pp_key_info_server); // ## Client -> Server - client_ph_protector.protect(tmp, sizeof(tmp), 18); + REQUIRE(client_ph_protector.protect(tmp, sizeof(tmp), 18)); CHECK(memcmp(original, tmp, sizeof(original)) != 0); - server_ph_protector.unprotect(tmp, sizeof(tmp)); + REQUIRE(server_ph_protector.unprotect(tmp, sizeof(tmp))); CHECK(memcmp(original, tmp, sizeof(original)) == 0); // ## Server -> Client - server_ph_protector.protect(tmp, sizeof(tmp), 18); + REQUIRE(server_ph_protector.protect(tmp, sizeof(tmp), 18)); CHECK(memcmp(original, tmp, sizeof(original)) != 0); - client_ph_protector.unprotect(tmp, sizeof(tmp)); + REQUIRE(client_ph_protector.unprotect(tmp, sizeof(tmp))); CHECK(memcmp(original, tmp, sizeof(original)) == 0); + + delete client; + delete server; } SECTION("Short header", "[quic]") @@ -115,52 +125,61 @@ TEST_CASE("QUICPacketHeaderProtector") QUICPacketProtectionKeyInfo pp_key_info_client; QUICPacketProtectionKeyInfo pp_key_info_server; NetVCOptions netvc_options; + MockQUICConnection mock_client_connection; + MockQUICConnection mock_server_connection; QUICHandshakeProtocol *client = new QUICTLS(pp_key_info_client, client_ssl_ctx, NET_VCONNECTION_OUT, netvc_options); QUICHandshakeProtocol *server = new QUICTLS(pp_key_info_server, server_ssl_ctx, NET_VCONNECTION_IN, netvc_options); + SSL_set_ex_data(static_cast(client)->ssl_handle(), QUIC::ssl_quic_qc_index, &mock_client_connection); + SSL_set_ex_data(static_cast(server)->ssl_handle(), QUIC::ssl_quic_qc_index, &mock_server_connection); + + auto client_tp = std::make_shared(); + auto server_tp = std::make_shared(); + client_tp->set(QUICTransportParameterId::MAX_IDLE_TIMEOUT, 10); + server_tp->set(QUICTransportParameterId::MAX_IDLE_TIMEOUT, 10); + client->set_local_transport_parameters(client_tp); + server->set_local_transport_parameters(server_tp); - CHECK(client->initialize_key_materials({reinterpret_cast("\x83\x94\xc8\xf0\x3e\x51\x57\x00"), 8})); - CHECK(server->initialize_key_materials({reinterpret_cast("\x83\x94\xc8\xf0\x3e\x51\x57\x00"), 8})); + CHECK(client->initialize_key_materials({reinterpret_cast("\x83\x94\xc8\xf0\x3e\x51\x57\x00"), 8}, + QUIC_SUPPORTED_VERSIONS[0])); + CHECK(server->initialize_key_materials({reinterpret_cast("\x83\x94\xc8\xf0\x3e\x51\x57\x00"), 8}, + QUIC_SUPPORTED_VERSIONS[0])); QUICPacketHeaderProtector client_ph_protector(pp_key_info_client); QUICPacketHeaderProtector server_ph_protector(pp_key_info_server); // Handshake // CH - QUICHandshakeMsgs msg1; - uint8_t msg1_buf[MAX_HANDSHAKE_MSG_LEN] = {0}; - msg1.buf = msg1_buf; - msg1.max_buf_len = MAX_HANDSHAKE_MSG_LEN; + QUICHandshakeMsgs msg0; + msg0.offsets[0] = 0; + msg0.offsets[1] = 0; + msg0.offsets[2] = 0; + msg0.offsets[3] = 0; + msg0.offsets[4] = 0; - REQUIRE(client->handshake(&msg1, nullptr) == 1); + QUICHandshakeMsgs *msg1 = nullptr; + REQUIRE(client->handshake(&msg1, &msg0) == 1); + REQUIRE(msg1); // SH, EE, CERT, CV, FIN - QUICHandshakeMsgs msg2; - uint8_t msg2_buf[MAX_HANDSHAKE_MSG_LEN] = {0}; - msg2.buf = msg2_buf; - msg2.max_buf_len = MAX_HANDSHAKE_MSG_LEN; - - REQUIRE(server->handshake(&msg2, &msg1) == 1); + QUICHandshakeMsgs *msg2 = nullptr; + REQUIRE(server->handshake(&msg2, msg1) == 1); + REQUIRE(msg2); // FIN - QUICHandshakeMsgs msg3; - uint8_t msg3_buf[MAX_HANDSHAKE_MSG_LEN] = {0}; - msg3.buf = msg3_buf; - msg3.max_buf_len = MAX_HANDSHAKE_MSG_LEN; + QUICHandshakeMsgs *msg3 = nullptr; -#ifdef SSL_MODE_QUIC_HACK - // -- Hacks for OpenSSL with SSL_MODE_QUIC_HACK -- // SH QUICHandshakeMsgs msg2_1; uint8_t msg2_1_buf[MAX_HANDSHAKE_MSG_LEN] = {0}; msg2_1.buf = msg2_1_buf; msg2_1.max_buf_len = MAX_HANDSHAKE_MSG_LEN; - memcpy(msg2_1.buf, msg2.buf, msg2.offsets[1]); + memcpy(msg2_1.buf, msg2->buf, msg2->offsets[1]); msg2_1.offsets[0] = 0; - msg2_1.offsets[1] = msg2.offsets[1]; - msg2_1.offsets[2] = msg2.offsets[1]; - msg2_1.offsets[3] = msg2.offsets[1]; - msg2_1.offsets[4] = msg2.offsets[1]; + msg2_1.offsets[1] = msg2->offsets[1]; + msg2_1.offsets[2] = msg2->offsets[1]; + msg2_1.offsets[3] = msg2->offsets[1]; + msg2_1.offsets[4] = msg2->offsets[1]; // EE - FIN QUICHandshakeMsgs msg2_2; @@ -168,8 +187,8 @@ TEST_CASE("QUICPacketHeaderProtector") msg2_2.buf = msg2_2_buf; msg2_2.max_buf_len = MAX_HANDSHAKE_MSG_LEN; - size_t len = msg2.offsets[3] - msg2.offsets[2]; - memcpy(msg2_2.buf, msg2.buf + msg2.offsets[1], len); + size_t len = msg2->offsets[3] - msg2->offsets[2]; + memcpy(msg2_2.buf, msg2->buf + msg2->offsets[1], len); msg2_2.offsets[0] = 0; msg2_2.offsets[1] = 0; msg2_2.offsets[2] = 0; @@ -178,33 +197,36 @@ TEST_CASE("QUICPacketHeaderProtector") REQUIRE(client->handshake(&msg3, &msg2_1) == 1); REQUIRE(client->handshake(&msg3, &msg2_2) == 1); -#else - REQUIRE(client->handshake(&msg3, &msg2) == 1); -#endif + REQUIRE(msg3); // NS - QUICHandshakeMsgs msg4; - uint8_t msg4_buf[MAX_HANDSHAKE_MSG_LEN] = {0}; - msg4.buf = msg4_buf; - msg4.max_buf_len = MAX_HANDSHAKE_MSG_LEN; + QUICHandshakeMsgs *msg4 = nullptr; + REQUIRE(server->handshake(&msg4, msg3) == 1); + REQUIRE(msg4); - REQUIRE(server->handshake(&msg4, &msg3) == 1); - - QUICHandshakeMsgs msg5; - uint8_t msg5_buf[MAX_HANDSHAKE_MSG_LEN] = {0}; - msg5.buf = msg5_buf; - msg5.max_buf_len = MAX_HANDSHAKE_MSG_LEN; - REQUIRE(client->handshake(&msg5, &msg4) == 1); + QUICHandshakeMsgs *msg5 = nullptr; + REQUIRE(client->handshake(&msg5, msg4) == 1); + REQUIRE(msg5 == nullptr); // ## Client -> Server - client_ph_protector.protect(tmp, sizeof(tmp), 18); + REQUIRE(client_ph_protector.protect(tmp, sizeof(tmp), 18)); CHECK(memcmp(original, tmp, sizeof(original)) != 0); - server_ph_protector.unprotect(tmp, sizeof(tmp)); + REQUIRE(server_ph_protector.unprotect(tmp, sizeof(tmp))); CHECK(memcmp(original, tmp, sizeof(original)) == 0); // ## Server -> Client - server_ph_protector.protect(tmp, sizeof(tmp), 18); + REQUIRE(server_ph_protector.protect(tmp, sizeof(tmp), 18)); CHECK(memcmp(original, tmp, sizeof(original)) != 0); - client_ph_protector.unprotect(tmp, sizeof(tmp)); + REQUIRE(client_ph_protector.unprotect(tmp, sizeof(tmp))); CHECK(memcmp(original, tmp, sizeof(original)) == 0); + + delete client; + delete server; } + + SSL_CTX_free(client_ssl_ctx); + SSL_CTX_free(server_ssl_ctx); + BIO_free(crt_bio); + BIO_free(key_bio); + X509_free(x509); + EVP_PKEY_free(pkey); } diff --git a/iocore/net/quic/test/test_QUICPathValidator.cc b/iocore/net/quic/test/test_QUICPathValidator.cc new file mode 100644 index 00000000000..cd96ef74a2b --- /dev/null +++ b/iocore/net/quic/test/test_QUICPathValidator.cc @@ -0,0 +1,112 @@ +/** @file + * + * A brief file description + * + * @section license License + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "catch.hpp" + +#include "quic/QUICPathValidator.h" +#include "quic/Mock.h" +#include "stdio.h" +#include "stdlib.h" + +TEST_CASE("QUICPathValidator", "[quic]") +{ + MockQUICConnectionInfoProvider cinfo_provider; + QUICPathValidator pv_c(cinfo_provider, [](bool x) {}); + QUICPathValidator pv_s(cinfo_provider, [](bool x) {}); + + SECTION("interests") + { + auto interests = pv_c.interests(); + CHECK(std::find_if(interests.begin(), interests.end(), [](QUICFrameType t) { return t == QUICFrameType::PATH_CHALLENGE; }) != + interests.end()); + CHECK(std::find_if(interests.begin(), interests.end(), [](QUICFrameType t) { return t == QUICFrameType::PATH_RESPONSE; }) != + interests.end()); + CHECK(std::find_if(interests.begin(), interests.end(), [](QUICFrameType t) { + return t != QUICFrameType::PATH_CHALLENGE && t != QUICFrameType::PATH_RESPONSE; + }) == interests.end()); + } + + SECTION("basic scenario") + { + uint8_t frame_buf[1024]; + uint32_t seq_num = 1; + IpEndpoint local, remote; + ats_ip_pton("127.0.0.1:4433", &local); + ats_ip_pton("127.0.0.1:12345", &remote); + QUICPath path = {local, remote}; + + // Send a challenge + CHECK(!pv_c.is_validating(path)); + CHECK(!pv_c.is_validated(path)); + REQUIRE(!pv_c.will_generate_frame(QUICEncryptionLevel::ONE_RTT, 0, false, seq_num)); + pv_c.validate(path); + CHECK(pv_c.is_validating(path)); + CHECK(!pv_c.is_validated(path)); + REQUIRE(pv_c.will_generate_frame(QUICEncryptionLevel::ONE_RTT, 0, false, seq_num)); + auto frame = pv_c.generate_frame(frame_buf, QUICEncryptionLevel::ONE_RTT, 1024, 1024, 0, seq_num); + REQUIRE(frame); + CHECK(frame->type() == QUICFrameType::PATH_CHALLENGE); + CHECK(pv_c.is_validating(path)); + CHECK(!pv_c.is_validated(path)); + ++seq_num; + + // Receive the challenge and respond + CHECK(!pv_s.is_validating(path)); + CHECK(!pv_s.is_validated(path)); + REQUIRE(!pv_s.will_generate_frame(QUICEncryptionLevel::ONE_RTT, 0, false, seq_num)); + auto error = pv_s.handle_frame(QUICEncryptionLevel::ONE_RTT, *frame); + REQUIRE(!error); + CHECK(!pv_s.is_validating(path)); + CHECK(!pv_s.is_validated(path)); + REQUIRE(pv_s.will_generate_frame(QUICEncryptionLevel::ONE_RTT, 0, false, seq_num)); + frame->~QUICFrame(); + frame = pv_s.generate_frame(frame_buf, QUICEncryptionLevel::ONE_RTT, 1024, 1024, 0, seq_num); + REQUIRE(frame); + CHECK(frame->type() == QUICFrameType::PATH_RESPONSE); + CHECK(!pv_s.is_validating(path)); + CHECK(!pv_s.is_validated(path)); + ++seq_num; + + uint8_t buf[1024]; + size_t len = 0; + uint8_t received_frame_buf[1024]; + Ptr ibb = frame->to_io_buffer_block(sizeof(buf)); + for (auto b = ibb; b; b = b->next) { + memcpy(buf + len, b->start(), b->size()); + len += b->size(); + } + MockQUICPacketR mock_packet; + auto received_frame = QUICFrameFactory::create(received_frame_buf, buf, len, &mock_packet); + mock_packet.set_from(remote); + mock_packet.set_to(local); + + // Receive the response + error = pv_c.handle_frame(QUICEncryptionLevel::ONE_RTT, *received_frame); + REQUIRE(!error); + CHECK(!pv_c.is_validating(path)); + CHECK(pv_c.is_validated(path)); + + frame->~QUICFrame(); + received_frame->~QUICFrame(); + } +} diff --git a/iocore/net/quic/test/test_QUICPinger.cc b/iocore/net/quic/test/test_QUICPinger.cc new file mode 100644 index 00000000000..d6b58e228b6 --- /dev/null +++ b/iocore/net/quic/test/test_QUICPinger.cc @@ -0,0 +1,101 @@ +/** @file + * + * A brief file description + * + * @section license License + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "catch.hpp" + +#include "QUICPinger.h" + +static constexpr QUICEncryptionLevel level = QUICEncryptionLevel::ONE_RTT; +static uint8_t frame[1024] = {0}; + +TEST_CASE("QUICPinger", "[quic]") +{ + SECTION("request and cancel") + { + QUICPinger pinger; + pinger.request(level); + REQUIRE(pinger.count(level) == 1); + pinger.request(level); + REQUIRE(pinger.count(level) == 2); + pinger.cancel(level); + REQUIRE(pinger.count(level) == 1); + REQUIRE(pinger.generate_frame(frame, level, UINT64_MAX, UINT16_MAX, 0, 0) != nullptr); + REQUIRE(pinger.count(level) == 0); + } + + SECTION("generate PING Frame twice") + { + QUICPinger pinger; + pinger.request(level); + REQUIRE(pinger.count(level) == 1); + pinger.request(level); + REQUIRE(pinger.count(level) == 2); + REQUIRE(pinger.will_generate_frame(level, UINT64_MAX, false, 0) == true); + REQUIRE(pinger.count(level) == 2); + REQUIRE(pinger.will_generate_frame(level, UINT64_MAX, false, 0) == false); + REQUIRE(pinger.count(level) == 2); + } + + SECTION("don't generate frame when packet is ack_elicting") + { + QUICPinger pinger; + pinger.request(level); + REQUIRE(pinger.count(level) == 1); + pinger.request(level); + REQUIRE(pinger.count(level) == 2); + REQUIRE(pinger.will_generate_frame(level, UINT64_MAX, true, 0) == false); + REQUIRE(pinger.count(level) == 1); + REQUIRE(pinger.will_generate_frame(level, UINT64_MAX, true, 1) == false); + REQUIRE(pinger.count(level) == 0); + } + + SECTION("generating PING Frame for next continuous un-ack-eliciting packets") + { + QUICPinger pinger; + REQUIRE(pinger.will_generate_frame(level, UINT64_MAX, false, 0) == true); + REQUIRE(pinger.count(level) == 1); + REQUIRE(pinger.will_generate_frame(level, UINT64_MAX, true, 1) == false); + REQUIRE(pinger.count(level) == 0); + REQUIRE(pinger.will_generate_frame(level, UINT64_MAX, false, 2) == false); + REQUIRE(pinger.count(level) == 0); + REQUIRE(pinger.will_generate_frame(level, UINT64_MAX, false, 3) == true); + REQUIRE(pinger.count(level) == 1); + } + + SECTION("don't send PING Frame for empty packet") + { + QUICPinger pinger; + REQUIRE(pinger.will_generate_frame(level, 0, false, 0) == false); + REQUIRE(pinger.count(level) == 0); + REQUIRE(pinger.will_generate_frame(level, UINT64_MAX, false, 1) == true); + REQUIRE(pinger.count(level) == 1); + REQUIRE(pinger.will_generate_frame(level, UINT64_MAX, true, 2) == false); + REQUIRE(pinger.count(level) == 0); + REQUIRE(pinger.will_generate_frame(level, UINT64_MAX, false, 3) == false); + REQUIRE(pinger.count(level) == 0); + REQUIRE(pinger.will_generate_frame(level, 0, false, 4) == false); + REQUIRE(pinger.count(level) == 0); + REQUIRE(pinger.will_generate_frame(level, 1, false, 5) == true); + REQUIRE(pinger.count(level) == 1); + } +} diff --git a/iocore/net/quic/test/test_QUICStream.cc b/iocore/net/quic/test/test_QUICStream.cc index aefc4883033..828049f43b2 100644 --- a/iocore/net/quic/test/test_QUICStream.cc +++ b/iocore/net/quic/test/test_QUICStream.cc @@ -33,7 +33,7 @@ TEST_CASE("QUICBidiStream", "[quic]") uint8_t payload[] = {0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10}; uint32_t stream_id = 0x03; Ptr block = make_ptr(new_IOBufferBlock()); - block->alloc(); + block->alloc(BUFFER_SIZE_INDEX_32K); memcpy(block->start(), payload, sizeof(payload)); block->fill(sizeof(payload)); @@ -178,7 +178,7 @@ TEST_CASE("QUICBidiStream", "[quic]") stream->do_io_read(nullptr, INT64_MAX, read_buffer); Ptr block = make_ptr(new_IOBufferBlock()); - block->alloc(); + block->alloc(BUFFER_SIZE_INDEX_32K); block->fill(1024); // Start with 1024 but not 0 so received frames won't be processed @@ -229,50 +229,50 @@ TEST_CASE("QUICBidiStream", "[quic]") write_buffer->write(data, 1024); stream->handleEvent(VC_EVENT_WRITE_READY, nullptr); - CHECK(stream->will_generate_frame(level, 0) == true); - frame = stream->generate_frame(frame_buf, level, 4096, 4096, 0); + CHECK(stream->will_generate_frame(level, 0, false, 0) == true); + frame = stream->generate_frame(frame_buf, level, 4096, 4096, 0, 0); CHECK(frame->type() == QUICFrameType::STREAM); - CHECK(stream->will_generate_frame(level, 0) == false); + CHECK(stream->will_generate_frame(level, 0, false, 0) == false); write_buffer->write(data, 1024); stream->handleEvent(VC_EVENT_WRITE_READY, nullptr); - CHECK(stream->will_generate_frame(level, 0) == true); - frame = stream->generate_frame(frame_buf, level, 4096, 4096, 0); + CHECK(stream->will_generate_frame(level, 0, false, 0) == true); + frame = stream->generate_frame(frame_buf, level, 4096, 4096, 0, 0); CHECK(frame->type() == QUICFrameType::STREAM); - CHECK(stream->will_generate_frame(level, 0) == false); + CHECK(stream->will_generate_frame(level, 0, false, 0) == false); write_buffer->write(data, 1024); stream->handleEvent(VC_EVENT_WRITE_READY, nullptr); - CHECK(stream->will_generate_frame(level, 0) == true); - frame = stream->generate_frame(frame_buf, level, 4096, 4096, 0); + CHECK(stream->will_generate_frame(level, 0, false, 0) == true); + frame = stream->generate_frame(frame_buf, level, 4096, 4096, 0, 0); CHECK(frame->type() == QUICFrameType::STREAM); - CHECK(stream->will_generate_frame(level, 0) == false); + CHECK(stream->will_generate_frame(level, 0, false, 0) == false); write_buffer->write(data, 1024); stream->handleEvent(VC_EVENT_WRITE_READY, nullptr); - CHECK(stream->will_generate_frame(level, 0) == true); - frame = stream->generate_frame(frame_buf, level, 4096, 4096, 0); + CHECK(stream->will_generate_frame(level, 0, false, 0) == true); + frame = stream->generate_frame(frame_buf, level, 4096, 4096, 0, 0); CHECK(frame->type() == QUICFrameType::STREAM); - CHECK(stream->will_generate_frame(level, 0) == false); + CHECK(stream->will_generate_frame(level, 0, false, 0) == false); // This should not send a frame because of flow control write_buffer->write(data, 1024); stream->handleEvent(VC_EVENT_WRITE_READY, nullptr); - CHECK(stream->will_generate_frame(level, 0) == true); - frame = stream->generate_frame(frame_buf, level, 4096, 4096, 0); + CHECK(stream->will_generate_frame(level, 0, false, 0) == true); + frame = stream->generate_frame(frame_buf, level, 4096, 4096, 0, 0); CHECK(frame); CHECK(frame->type() == QUICFrameType::STREAM_DATA_BLOCKED); - CHECK(stream->will_generate_frame(level, 0) == true); + CHECK(stream->will_generate_frame(level, 0, false, 0) == true); // Update window stream->recv(*std::make_shared(stream_id, 5120)); // This should send a frame stream->handleEvent(VC_EVENT_WRITE_READY, nullptr); - CHECK(stream->will_generate_frame(level, 0) == true); - frame = stream->generate_frame(frame_buf, level, 4096, 4096, 0); + CHECK(stream->will_generate_frame(level, 0, false, 0) == true); + frame = stream->generate_frame(frame_buf, level, 4096, 4096, 0, 0); CHECK(frame->type() == QUICFrameType::STREAM); - CHECK(stream->will_generate_frame(level, 0) == false); + CHECK(stream->will_generate_frame(level, 0, false, 0) == false); // Update window stream->recv(*std::make_shared(stream_id, 5632)); @@ -280,24 +280,24 @@ TEST_CASE("QUICBidiStream", "[quic]") // This should send a frame write_buffer->write(data, 1024); stream->handleEvent(VC_EVENT_WRITE_READY, nullptr); - CHECK(stream->will_generate_frame(level, 0) == true); - frame = stream->generate_frame(frame_buf, level, 4096, 4096, 0); + CHECK(stream->will_generate_frame(level, 0, false, 0) == true); + frame = stream->generate_frame(frame_buf, level, 4096, 4096, 0, 0); CHECK(frame->type() == QUICFrameType::STREAM); - CHECK(stream->will_generate_frame(level, 0) == true); + CHECK(stream->will_generate_frame(level, 0, false, 0) == true); stream->handleEvent(VC_EVENT_WRITE_READY, nullptr); - CHECK(stream->will_generate_frame(level, 0) == true); - frame = stream->generate_frame(frame_buf, level, 4096, 4096, 0); + CHECK(stream->will_generate_frame(level, 0, false, 0) == true); + frame = stream->generate_frame(frame_buf, level, 4096, 4096, 0, 0); CHECK(frame->type() == QUICFrameType::STREAM_DATA_BLOCKED); // Update window stream->recv(*std::make_shared(stream_id, 6144)); stream->handleEvent(VC_EVENT_WRITE_READY, nullptr); - CHECK(stream->will_generate_frame(level, 0) == true); - frame = stream->generate_frame(frame_buf, level, 4096, 4096, 0); + CHECK(stream->will_generate_frame(level, 0, false, 0) == true); + frame = stream->generate_frame(frame_buf, level, 4096, 4096, 0, 0); CHECK(frame->type() == QUICFrameType::STREAM); - CHECK(stream->will_generate_frame(level, 0) == false); + CHECK(stream->will_generate_frame(level, 0, false, 0) == false); } /* @@ -329,13 +329,13 @@ TEST_CASE("QUICBidiStream", "[quic]") write_buffer->write(data1, sizeof(data1)); stream->handleEvent(VC_EVENT_WRITE_READY, nullptr); // Generate STREAM frame - frame = stream->generate_frame(frame_buf, level, 4096, 4096, 0); + frame = stream->generate_frame(frame_buf, level, 4096, 4096, 0, 0); frame1 = static_cast(frame); CHECK(frame->type() == QUICFrameType::STREAM); - CHECK(stream->generate_frame(frame_buf, level, 4096, 4096, 0) == nullptr); - CHECK(stream->will_generate_frame(level, 0) == false); + CHECK(stream->generate_frame(frame_buf, level, 4096, 4096, 0, 0) == nullptr); + CHECK(stream->will_generate_frame(level, 0, false, 0) == false); stream->on_frame_lost(frame->id()); - CHECK(stream->will_generate_frame(level, 0) == true); + CHECK(stream->will_generate_frame(level, 0, false, 0) == true); // Write data2 write_buffer->write(data2, sizeof(data2)); @@ -343,7 +343,7 @@ TEST_CASE("QUICBidiStream", "[quic]") // Lost the frame stream->on_frame_lost(frame->id()); // Regenerate a frame - frame = stream->generate_frame(frame_buf2, level, 4096, 4096, 0); + frame = stream->generate_frame(frame_buf2, level, 4096, 4096, 0, 0); // Lost data should be resent first frame2 = static_cast(frame); CHECK(frame->type() == QUICFrameType::STREAM); @@ -370,15 +370,15 @@ TEST_CASE("QUICBidiStream", "[quic]") QUICFrame *frame = nullptr; stream->reset(QUICStreamErrorUPtr(new QUICStreamError(stream.get(), QUIC_APP_ERROR_CODE_STOPPING))); - frame = stream->generate_frame(frame_buf, level, 4096, 4096, 0); + frame = stream->generate_frame(frame_buf, level, 4096, 4096, 0, 0); REQUIRE(frame); CHECK(frame->type() == QUICFrameType::RESET_STREAM); - // Don't send it again untill it is considers as lost - CHECK(stream->generate_frame(frame_buf, level, 4096, 4096, 0) == nullptr); + // Don't send it again until it is considers as lost + CHECK(stream->generate_frame(frame_buf, level, 4096, 4096, 0, 0) == nullptr); // Loss the frame stream->on_frame_lost(frame->id()); // After the loss the frame should be regenerated - frame = stream->generate_frame(frame_buf, level, 4096, 4096, 0); + frame = stream->generate_frame(frame_buf, level, 4096, 4096, 0, 0); REQUIRE(frame); CHECK(frame->type() == QUICFrameType::RESET_STREAM); } @@ -401,18 +401,60 @@ TEST_CASE("QUICBidiStream", "[quic]") QUICFrame *frame = nullptr; stream->stop_sending(QUICStreamErrorUPtr(new QUICStreamError(stream.get(), QUIC_APP_ERROR_CODE_STOPPING))); - frame = stream->generate_frame(frame_buf, level, 4096, 4096, 0); + frame = stream->generate_frame(frame_buf, level, 4096, 4096, 0, 0); REQUIRE(frame); CHECK(frame->type() == QUICFrameType::STOP_SENDING); - // Don't send it again untill it is considers as lost - CHECK(stream->generate_frame(frame_buf, level, 4096, 4096, 0) == nullptr); + // Don't send it again until it is considers as lost + CHECK(stream->generate_frame(frame_buf, level, 4096, 4096, 0, 0) == nullptr); // Loss the frame stream->on_frame_lost(frame->id()); // After the loss the frame should be regenerated - frame = stream->generate_frame(frame_buf, level, 4096, 4096, 0); + frame = stream->generate_frame(frame_buf, level, 4096, 4096, 0, 0); REQUIRE(frame); CHECK(frame->type() == QUICFrameType::STOP_SENDING); } + + SECTION("Insufficient max_frame_size") + { + MIOBuffer *write_buffer = new_MIOBuffer(BUFFER_SIZE_INDEX_8K); + IOBufferReader *write_buffer_reader = write_buffer->alloc_reader(); + QUICRTTMeasure rtt_provider; + MockQUICConnectionInfoProvider cinfo_provider; + QUICEncryptionLevel level = QUICEncryptionLevel::ONE_RTT; + QUICFrame *frame = nullptr; + + // STOP_SENDING + std::unique_ptr stream1( + new QUICBidirectionalStream(&rtt_provider, &cinfo_provider, stream_id, UINT64_MAX, UINT64_MAX)); + MockContinuation mock_cont1(stream1->mutex); + stream1->do_io_write(&mock_cont1, INT64_MAX, write_buffer_reader); + SCOPED_MUTEX_LOCK(lock1, stream1->mutex, this_ethread()); + stream1->stop_sending(QUICStreamErrorUPtr(new QUICStreamError(stream1.get(), QUIC_APP_ERROR_CODE_STOPPING))); + frame = stream1->generate_frame(frame_buf, level, 4096, 0, 0, 0); + CHECK(frame == nullptr); + + // RESET_STREAM + std::unique_ptr stream2( + new QUICBidirectionalStream(&rtt_provider, &cinfo_provider, stream_id, UINT64_MAX, UINT64_MAX)); + MockContinuation mock_cont2(stream2->mutex); + stream2->do_io_write(&mock_cont2, INT64_MAX, write_buffer_reader); + SCOPED_MUTEX_LOCK(lock2, stream2->mutex, this_ethread()); + stream2->reset(QUICStreamErrorUPtr(new QUICStreamError(stream2.get(), QUIC_APP_ERROR_CODE_STOPPING))); + frame = stream2->generate_frame(frame_buf, level, 4096, 0, 0, 0); + CHECK(frame == nullptr); + + // STREAM + std::unique_ptr stream3( + new QUICBidirectionalStream(&rtt_provider, &cinfo_provider, stream_id, UINT64_MAX, UINT64_MAX)); + MockContinuation mock_cont3(stream3->mutex); + stream3->do_io_write(&mock_cont3, INT64_MAX, write_buffer_reader); + SCOPED_MUTEX_LOCK(lock3, stream3->mutex, this_ethread()); + const char data[] = "this is a test data"; + write_buffer->write(data, sizeof(data)); + stream3->handleEvent(VC_EVENT_WRITE_READY, nullptr); + frame = stream3->generate_frame(frame_buf, level, 4096, 0, 0, 0); + CHECK(frame == nullptr); + } } TEST_CASE("QUIC receive only stream", "[quic]") @@ -421,7 +463,7 @@ TEST_CASE("QUIC receive only stream", "[quic]") uint8_t payload[] = {0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10}; uint32_t stream_id = 0x03; Ptr block = make_ptr(new_IOBufferBlock()); - block->alloc(); + block->alloc(BUFFER_SIZE_INDEX_32K); memcpy(block->start(), payload, sizeof(payload)); block->fill(sizeof(payload)); @@ -562,7 +604,7 @@ TEST_CASE("QUIC receive only stream", "[quic]") stream->do_io_read(nullptr, INT64_MAX, read_buffer); Ptr block = make_ptr(new_IOBufferBlock()); - block->alloc(); + block->alloc(BUFFER_SIZE_INDEX_32K); block->fill(1024); // Start with 1024 but not 0 so received frames won't be processed @@ -599,18 +641,34 @@ TEST_CASE("QUIC receive only stream", "[quic]") QUICFrame *frame = nullptr; stream->stop_sending(QUICStreamErrorUPtr(new QUICStreamError(stream.get(), QUIC_APP_ERROR_CODE_STOPPING))); - frame = stream->generate_frame(frame_buf, level, 4096, 4096, 0); + frame = stream->generate_frame(frame_buf, level, 4096, 4096, 0, 0); REQUIRE(frame); CHECK(frame->type() == QUICFrameType::STOP_SENDING); - // Don't send it again untill it is considers as lost - CHECK(stream->generate_frame(frame_buf, level, 4096, 4096, 0) == nullptr); + // Don't send it again until it is considers as lost + CHECK(stream->generate_frame(frame_buf, level, 4096, 4096, 0, 0) == nullptr); // Loss the frame stream->on_frame_lost(frame->id()); // After the loss the frame should be regenerated - frame = stream->generate_frame(frame_buf, level, 4096, 4096, 0); + frame = stream->generate_frame(frame_buf, level, 4096, 4096, 0, 0); REQUIRE(frame); CHECK(frame->type() == QUICFrameType::STOP_SENDING); } + + SECTION("Insufficient max_frame_size") + { + QUICRTTMeasure rtt_provider; + MockQUICConnectionInfoProvider cinfo_provider; + QUICEncryptionLevel level = QUICEncryptionLevel::ONE_RTT; + QUICFrame *frame = nullptr; + + // STOP_SENDING + std::unique_ptr stream1(new QUICReceiveStream(&rtt_provider, &cinfo_provider, stream_id, UINT64_MAX)); + MockContinuation mock_cont1(stream1->mutex); + SCOPED_MUTEX_LOCK(lock1, stream1->mutex, this_ethread()); + stream1->stop_sending(QUICStreamErrorUPtr(new QUICStreamError(stream1.get(), QUIC_APP_ERROR_CODE_STOPPING))); + frame = stream1->generate_frame(frame_buf, level, 4096, 0, 0, 0); + CHECK(frame == nullptr); + } } TEST_CASE("QUIC send only stream", "[quic]") @@ -619,7 +677,7 @@ TEST_CASE("QUIC send only stream", "[quic]") uint8_t payload[] = {0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10}; uint32_t stream_id = 0x03; Ptr block = make_ptr(new_IOBufferBlock()); - block->alloc(); + block->alloc(BUFFER_SIZE_INDEX_32K); memcpy(block->start(), payload, sizeof(payload)); block->fill(sizeof(payload)); @@ -672,7 +730,6 @@ TEST_CASE("QUIC send only stream", "[quic]") MIOBuffer *write_buffer = new_MIOBuffer(BUFFER_SIZE_INDEX_4K); IOBufferReader *write_buffer_reader = write_buffer->alloc_reader(); - QUICRTTMeasure rtt_provider; MockQUICConnectionInfoProvider cinfo_provider; std::unique_ptr stream(new QUICSendStream(&cinfo_provider, stream_id, 4096)); SCOPED_MUTEX_LOCK(lock, stream->mutex, this_ethread()); @@ -687,50 +744,50 @@ TEST_CASE("QUIC send only stream", "[quic]") write_buffer->write(data, 1024); stream->handleEvent(VC_EVENT_WRITE_READY, nullptr); - CHECK(stream->will_generate_frame(level, 0) == true); - frame = stream->generate_frame(frame_buf, level, 4096, 4096, 0); + CHECK(stream->will_generate_frame(level, 0, false, 0) == true); + frame = stream->generate_frame(frame_buf, level, 4096, 4096, 0, 0); CHECK(frame->type() == QUICFrameType::STREAM); - CHECK(stream->will_generate_frame(level, 0) == false); + CHECK(stream->will_generate_frame(level, 0, false, 0) == false); write_buffer->write(data, 1024); stream->handleEvent(VC_EVENT_WRITE_READY, nullptr); - CHECK(stream->will_generate_frame(level, 0) == true); - frame = stream->generate_frame(frame_buf, level, 4096, 4096, 0); + CHECK(stream->will_generate_frame(level, 0, false, 0) == true); + frame = stream->generate_frame(frame_buf, level, 4096, 4096, 0, 0); CHECK(frame->type() == QUICFrameType::STREAM); - CHECK(stream->will_generate_frame(level, 0) == false); + CHECK(stream->will_generate_frame(level, 0, false, 0) == false); write_buffer->write(data, 1024); stream->handleEvent(VC_EVENT_WRITE_READY, nullptr); - CHECK(stream->will_generate_frame(level, 0) == true); - frame = stream->generate_frame(frame_buf, level, 4096, 4096, 0); + CHECK(stream->will_generate_frame(level, 0, false, 0) == true); + frame = stream->generate_frame(frame_buf, level, 4096, 4096, 0, 0); CHECK(frame->type() == QUICFrameType::STREAM); - CHECK(stream->will_generate_frame(level, 0) == false); + CHECK(stream->will_generate_frame(level, 0, false, 0) == false); write_buffer->write(data, 1024); stream->handleEvent(VC_EVENT_WRITE_READY, nullptr); - CHECK(stream->will_generate_frame(level, 0) == true); - frame = stream->generate_frame(frame_buf, level, 4096, 4096, 0); + CHECK(stream->will_generate_frame(level, 0, false, 0) == true); + frame = stream->generate_frame(frame_buf, level, 4096, 4096, 0, 0); CHECK(frame->type() == QUICFrameType::STREAM); - CHECK(stream->will_generate_frame(level, 0) == false); + CHECK(stream->will_generate_frame(level, 0, false, 0) == false); // This should not send a frame because of flow control write_buffer->write(data, 1024); stream->handleEvent(VC_EVENT_WRITE_READY, nullptr); - CHECK(stream->will_generate_frame(level, 0) == true); - frame = stream->generate_frame(frame_buf, level, 4096, 4096, 0); + CHECK(stream->will_generate_frame(level, 0, false, 0) == true); + frame = stream->generate_frame(frame_buf, level, 4096, 4096, 0, 0); CHECK(frame); CHECK(frame->type() == QUICFrameType::STREAM_DATA_BLOCKED); - CHECK(stream->will_generate_frame(level, 0) == true); + CHECK(stream->will_generate_frame(level, 0, false, 0) == true); // Update window stream->recv(*std::make_shared(stream_id, 5120)); // This should send a frame stream->handleEvent(VC_EVENT_WRITE_READY, nullptr); - CHECK(stream->will_generate_frame(level, 0) == true); - frame = stream->generate_frame(frame_buf, level, 4096, 4096, 0); + CHECK(stream->will_generate_frame(level, 0, false, 0) == true); + frame = stream->generate_frame(frame_buf, level, 4096, 4096, 0, 0); CHECK(frame->type() == QUICFrameType::STREAM); - CHECK(stream->will_generate_frame(level, 0) == false); + CHECK(stream->will_generate_frame(level, 0, false, 0) == false); // Update window stream->recv(*std::make_shared(stream_id, 5632)); @@ -738,24 +795,24 @@ TEST_CASE("QUIC send only stream", "[quic]") // This should send a frame write_buffer->write(data, 1024); stream->handleEvent(VC_EVENT_WRITE_READY, nullptr); - CHECK(stream->will_generate_frame(level, 0) == true); - frame = stream->generate_frame(frame_buf, level, 4096, 4096, 0); + CHECK(stream->will_generate_frame(level, 0, false, 0) == true); + frame = stream->generate_frame(frame_buf, level, 4096, 4096, 0, 0); CHECK(frame->type() == QUICFrameType::STREAM); - CHECK(stream->will_generate_frame(level, 0) == true); + CHECK(stream->will_generate_frame(level, 0, false, 0) == true); stream->handleEvent(VC_EVENT_WRITE_READY, nullptr); - CHECK(stream->will_generate_frame(level, 0) == true); - frame = stream->generate_frame(frame_buf, level, 4096, 4096, 0); + CHECK(stream->will_generate_frame(level, 0, false, 0) == true); + frame = stream->generate_frame(frame_buf, level, 4096, 4096, 0, 0); CHECK(frame->type() == QUICFrameType::STREAM_DATA_BLOCKED); // Update window stream->recv(*std::make_shared(stream_id, 6144)); stream->handleEvent(VC_EVENT_WRITE_READY, nullptr); - CHECK(stream->will_generate_frame(level, 0) == true); - frame = stream->generate_frame(frame_buf, level, 4096, 4096, 0); + CHECK(stream->will_generate_frame(level, 0, false, 0) == true); + frame = stream->generate_frame(frame_buf, level, 4096, 4096, 0, 0); CHECK(frame->type() == QUICFrameType::STREAM); - CHECK(stream->will_generate_frame(level, 0) == false); + CHECK(stream->will_generate_frame(level, 0, false, 0) == false); } /* @@ -766,7 +823,6 @@ TEST_CASE("QUIC send only stream", "[quic]") MIOBuffer *write_buffer = new_MIOBuffer(BUFFER_SIZE_INDEX_8K); IOBufferReader *write_buffer_reader = write_buffer->alloc_reader(); - QUICRTTMeasure rtt_provider; MockQUICConnectionInfoProvider cinfo_provider; std::unique_ptr stream(new QUICSendStream(&cinfo_provider, stream_id, UINT64_MAX)); SCOPED_MUTEX_LOCK(lock, stream->mutex, this_ethread()); @@ -786,13 +842,13 @@ TEST_CASE("QUIC send only stream", "[quic]") write_buffer->write(data1, sizeof(data1)); stream->handleEvent(VC_EVENT_WRITE_READY, nullptr); // Generate STREAM frame - frame = stream->generate_frame(frame_buf, level, 4096, 4096, 0); + frame = stream->generate_frame(frame_buf, level, 4096, 4096, 0, 0); frame1 = static_cast(frame); CHECK(frame->type() == QUICFrameType::STREAM); - CHECK(stream->generate_frame(frame_buf, level, 4096, 4096, 0) == nullptr); - CHECK(stream->will_generate_frame(level, 0) == false); + CHECK(stream->generate_frame(frame_buf, level, 4096, 4096, 0, 0) == nullptr); + CHECK(stream->will_generate_frame(level, 0, false, 0) == false); stream->on_frame_lost(frame->id()); - CHECK(stream->will_generate_frame(level, 0) == true); + CHECK(stream->will_generate_frame(level, 0, false, 0) == true); // Write data2 write_buffer->write(data2, sizeof(data2)); @@ -800,7 +856,7 @@ TEST_CASE("QUIC send only stream", "[quic]") // Lost the frame stream->on_frame_lost(frame->id()); // Regenerate a frame - frame = stream->generate_frame(frame_buf2, level, 4096, 4096, 0); + frame = stream->generate_frame(frame_buf2, level, 4096, 4096, 0, 0); // Lost data should be resent first frame2 = static_cast(frame); CHECK(frame->type() == QUICFrameType::STREAM); @@ -814,7 +870,6 @@ TEST_CASE("QUIC send only stream", "[quic]") MIOBuffer *write_buffer = new_MIOBuffer(BUFFER_SIZE_INDEX_8K); IOBufferReader *write_buffer_reader = write_buffer->alloc_reader(); - QUICRTTMeasure rtt_provider; MockQUICConnectionInfoProvider cinfo_provider; std::unique_ptr stream(new QUICSendStream(&cinfo_provider, stream_id, UINT64_MAX)); SCOPED_MUTEX_LOCK(lock, stream->mutex, this_ethread()); @@ -826,16 +881,68 @@ TEST_CASE("QUIC send only stream", "[quic]") QUICFrame *frame = nullptr; stream->reset(QUICStreamErrorUPtr(new QUICStreamError(stream.get(), QUIC_APP_ERROR_CODE_STOPPING))); - frame = stream->generate_frame(frame_buf, level, 4096, 4096, 0); + frame = stream->generate_frame(frame_buf, level, 4096, 4096, 0, 0); REQUIRE(frame); CHECK(frame->type() == QUICFrameType::RESET_STREAM); - // Don't send it again untill it is considers as lost - CHECK(stream->generate_frame(frame_buf, level, 4096, 4096, 0) == nullptr); + // Don't send it again until it is considers as lost + CHECK(stream->generate_frame(frame_buf, level, 4096, 4096, 0, 0) == nullptr); // Loss the frame stream->on_frame_lost(frame->id()); // After the loss the frame should be regenerated - frame = stream->generate_frame(frame_buf, level, 4096, 4096, 0); + frame = stream->generate_frame(frame_buf, level, 4096, 4096, 0, 0); REQUIRE(frame); CHECK(frame->type() == QUICFrameType::RESET_STREAM); } + + SECTION("Insufficient max_frame_size") + { + MIOBuffer *write_buffer = new_MIOBuffer(BUFFER_SIZE_INDEX_8K); + IOBufferReader *write_buffer_reader = write_buffer->alloc_reader(); + MockQUICConnectionInfoProvider cinfo_provider; + QUICEncryptionLevel level = QUICEncryptionLevel::ONE_RTT; + QUICFrame *frame = nullptr; + + // RESET_STREAM + std::unique_ptr stream2(new QUICSendStream(&cinfo_provider, stream_id, UINT64_MAX)); + MockContinuation mock_cont2(stream2->mutex); + stream2->do_io_write(&mock_cont2, INT64_MAX, write_buffer_reader); + SCOPED_MUTEX_LOCK(lock2, stream2->mutex, this_ethread()); + stream2->reset(QUICStreamErrorUPtr(new QUICStreamError(stream2.get(), QUIC_APP_ERROR_CODE_STOPPING))); + frame = stream2->generate_frame(frame_buf, level, 4096, 0, 0, 0); + CHECK(frame == nullptr); + + // STREAM + std::unique_ptr stream3(new QUICSendStream(&cinfo_provider, stream_id, UINT64_MAX)); + MockContinuation mock_cont3(stream3->mutex); + stream3->do_io_write(&mock_cont3, INT64_MAX, write_buffer_reader); + SCOPED_MUTEX_LOCK(lock3, stream3->mutex, this_ethread()); + const char data[] = "this is a test data"; + write_buffer->write(data, sizeof(data)); + stream3->handleEvent(VC_EVENT_WRITE_READY, nullptr); + frame = stream3->generate_frame(frame_buf, level, 4096, 0, 0, 0); + CHECK(frame == nullptr); + } +} + +TEST_CASE("will_generate_frame", "[quic]") +{ + SECTION("Return false if a stream has not initialized for IO") + { + QUICRTTMeasure rtt_provider; + MockQUICConnectionInfoProvider cinfo_provider; + uint8_t buf[128]; + + std::unique_ptr stream_bidi( + new QUICBidirectionalStream(&rtt_provider, &cinfo_provider, 0, 1024, 1024)); + CHECK(stream_bidi->will_generate_frame(QUICEncryptionLevel::ONE_RTT, 0, false, 0) == false); + CHECK(stream_bidi->generate_frame(buf, QUICEncryptionLevel::ONE_RTT, 1024, 1024, 0, 0) == nullptr); + + std::unique_ptr stream_uni1(new QUICSendStream(&cinfo_provider, 2, 1024)); + CHECK(stream_uni1->will_generate_frame(QUICEncryptionLevel::ONE_RTT, 0, false, 0) == false); + CHECK(stream_uni1->generate_frame(buf, QUICEncryptionLevel::ONE_RTT, 1024, 1024, 0, 0) == nullptr); + + std::unique_ptr stream_uni2(new QUICReceiveStream(&rtt_provider, &cinfo_provider, 3, 1024)); + CHECK(stream_uni2->will_generate_frame(QUICEncryptionLevel::ONE_RTT, 0, false, 0) == false); + CHECK(stream_uni2->generate_frame(buf, QUICEncryptionLevel::ONE_RTT, 1024, 1024, 0, 0) == nullptr); + } } diff --git a/iocore/net/quic/test/test_QUICStreamManager.cc b/iocore/net/quic/test/test_QUICStreamManager.cc index ced351aba7e..ad9443deb54 100644 --- a/iocore/net/quic/test/test_QUICStreamManager.cc +++ b/iocore/net/quic/test/test_QUICStreamManager.cc @@ -29,6 +29,8 @@ #include "quic/QUICFrame.h" #include "quic/Mock.h" +MockQUICContext context; + TEST_CASE("QUICStreamManager_NewStream", "[quic]") { QUICEncryptionLevel level = QUICEncryptionLevel::ONE_RTT; @@ -37,32 +39,29 @@ TEST_CASE("QUICStreamManager_NewStream", "[quic]") MockQUICApplication mock_app(&connection); app_map.set_default(&mock_app); MockQUICConnectionInfoProvider cinfo_provider; - QUICRTTMeasure rtt_provider; - QUICStreamManager sm(&cinfo_provider, &rtt_provider, &app_map); + QUICStreamManager sm(&context, &app_map); uint8_t local_tp_buf[] = { - 0x00, 0x06, // size of parameters - 0x00, 0x08, // parameter id - initial_max_streams_bidi - 0x00, 0x02, // length of value - 0x40, 0x10 // value + 0x08, // parameter id - initial_max_streams_bidi + 0x02, // length of value + 0x40, 0x10 // value }; std::shared_ptr local_tp = - std::make_shared(local_tp_buf, sizeof(local_tp_buf)); + std::make_shared(local_tp_buf, sizeof(local_tp_buf), QUIC_SUPPORTED_VERSIONS[0]); uint8_t remote_tp_buf[] = { - 0x00, 0x06, // size of parameters - 0x00, 0x08, // parameter id - initial_max_streams_bidi - 0x00, 0x02, // length of value - 0x40, 0x10 // value + 0x08, // parameter id - initial_max_streams_bidi + 0x02, // length of value + 0x40, 0x10 // value }; std::shared_ptr remote_tp = - std::make_shared(remote_tp_buf, sizeof(remote_tp_buf)); + std::make_shared(remote_tp_buf, sizeof(remote_tp_buf), QUIC_SUPPORTED_VERSIONS[0]); sm.init_flow_control_params(local_tp, remote_tp); // STREAM frames create new streams Ptr block = make_ptr(new_IOBufferBlock()); - block->alloc(); + block->alloc(BUFFER_SIZE_INDEX_32K); block->fill(4); CHECK(block->read_avail() == 4); @@ -94,13 +93,6 @@ TEST_CASE("QUICStreamManager_NewStream", "[quic]") QUICFrame *stream_blocked_frame = QUICFrameFactory::create_stream_data_blocked_frame(stream_blocked_frame_buf, 0x10, 0); sm.handle_frame(level, *stream_blocked_frame); CHECK(sm.stream_count() == 5); - - // Set local maximum stream id - sm.set_max_streams_bidi(5); - uint8_t stream_blocked_frame_x_buf[QUICFrame::MAX_INSTANCE_SIZE]; - QUICFrame *stream_blocked_frame_x = QUICFrameFactory::create_stream_data_blocked_frame(stream_blocked_frame_x_buf, 0x18, 0); - sm.handle_frame(level, *stream_blocked_frame_x); - CHECK(sm.stream_count() == 5); } TEST_CASE("QUICStreamManager_first_initial_map", "[quic]") @@ -111,15 +103,14 @@ TEST_CASE("QUICStreamManager_first_initial_map", "[quic]") MockQUICApplication mock_app(&connection); app_map.set_default(&mock_app); MockQUICConnectionInfoProvider cinfo_provider; - QUICRTTMeasure rtt_provider; - QUICStreamManager sm(&cinfo_provider, &rtt_provider, &app_map); + QUICStreamManager sm(&context, &app_map); std::shared_ptr local_tp = std::make_shared(); std::shared_ptr remote_tp = std::make_shared(); sm.init_flow_control_params(local_tp, remote_tp); // STREAM frames create new streams Ptr block = make_ptr(new_IOBufferBlock()); - block->alloc(); + block->alloc(BUFFER_SIZE_INDEX_32K); block->fill(4); CHECK(block->read_avail() == 4); @@ -137,32 +128,29 @@ TEST_CASE("QUICStreamManager_total_offset_received", "[quic]") MockQUICConnection connection; MockQUICApplication mock_app(&connection); app_map.set_default(&mock_app); - QUICRTTMeasure rtt_provider; - QUICStreamManager sm(new MockQUICConnectionInfoProvider(), &rtt_provider, &app_map); + QUICStreamManager sm(&context, &app_map); uint8_t local_tp_buf[] = { - 0x00, 0x0e, // size of parameters - 0x00, 0x08, // parameter id - initial_max_streams_bidi - 0x00, 0x02, // length of value + 0x08, // parameter id - initial_max_streams_bidi + 0x02, // length of value 0x40, 0x10, // value - 0x00, 0x05, // parameter id - initial_max_stream_data_bidi_local - 0x00, 0x04, // length of value + 0x05, // parameter id - initial_max_stream_data_bidi_local + 0x04, // length of value 0xbf, 0xff, 0xff, 0xff // value }; std::shared_ptr local_tp = - std::make_shared(local_tp_buf, sizeof(local_tp_buf)); + std::make_shared(local_tp_buf, sizeof(local_tp_buf), QUIC_SUPPORTED_VERSIONS[0]); uint8_t remote_tp_buf[] = { - 0x00, 0x0e, // size of parameters - 0x00, 0x08, // parameter id - initial_max_streams_bidi - 0x00, 0x02, // length of value + 0x08, // parameter id - initial_max_streams_bidi + 0x02, // length of value 0x40, 0x10, // value - 0x00, 0x06, // parameter id - initial_max_stream_data_bidi_remote - 0x00, 0x04, // length of value + 0x06, // parameter id - initial_max_stream_data_bidi_remote + 0x04, // length of value 0xbf, 0xff, 0xff, 0xff // value }; std::shared_ptr remote_tp = - std::make_shared(remote_tp_buf, sizeof(remote_tp_buf)); + std::make_shared(remote_tp_buf, sizeof(remote_tp_buf), QUIC_SUPPORTED_VERSIONS[0]); sm.init_flow_control_params(local_tp, remote_tp); @@ -178,7 +166,7 @@ TEST_CASE("QUICStreamManager_total_offset_received", "[quic]") // total_offset should be a integer in unit of 1024 octets Ptr block = make_ptr(new_IOBufferBlock()); - block->alloc(); + block->alloc(BUFFER_SIZE_INDEX_32K); block->fill(1024); CHECK(block->read_avail() == 1024); @@ -195,38 +183,35 @@ TEST_CASE("QUICStreamManager_total_offset_sent", "[quic]") MockQUICConnection connection; MockQUICApplication mock_app(&connection); app_map.set_default(&mock_app); - QUICRTTMeasure rtt_provider; - QUICStreamManager sm(new MockQUICConnectionInfoProvider(), &rtt_provider, &app_map); + QUICStreamManager sm(&context, &app_map); uint8_t local_tp_buf[] = { - 0x00, 0x0e, // size of parameters - 0x00, 0x08, // parameter id - initial_max_streams_bidi - 0x00, 0x02, // length of value + 0x08, // parameter id - initial_max_streams_bidi + 0x02, // length of value 0x40, 0x10, // value - 0x00, 0x05, // parameter id - initial_max_stream_data_bidi_local - 0x00, 0x04, // length of value + 0x05, // parameter id - initial_max_stream_data_bidi_local + 0x04, // length of value 0xbf, 0xff, 0xff, 0xff // value }; std::shared_ptr local_tp = - std::make_shared(local_tp_buf, sizeof(local_tp_buf)); + std::make_shared(local_tp_buf, sizeof(local_tp_buf), QUIC_SUPPORTED_VERSIONS[0]); uint8_t remote_tp_buf[] = { - 0x00, 0x0e, // size of parameters - 0x00, 0x08, // parameter id - initial_max_streams_bidi - 0x00, 0x02, // length of value + 0x08, // parameter id - initial_max_streams_bidi + 0x02, // length of value 0x40, 0x10, // value - 0x00, 0x06, // parameter id - initial_max_stream_data_bidi_remote - 0x00, 0x04, // length of value + 0x06, // parameter id - initial_max_stream_data_bidi_remote + 0x04, // length of value 0xbf, 0xff, 0xff, 0xff // value }; std::shared_ptr remote_tp = - std::make_shared(remote_tp_buf, sizeof(remote_tp_buf)); + std::make_shared(remote_tp_buf, sizeof(remote_tp_buf), QUIC_SUPPORTED_VERSIONS[0]); sm.init_flow_control_params(local_tp, remote_tp); // Create a stream with STREAM_DATA_BLOCKED (== noop) Ptr block_3 = make_ptr(new_IOBufferBlock()); - block_3->alloc(); + block_3->alloc(BUFFER_SIZE_INDEX_32K); block_3->fill(3); CHECK(block_3->read_avail() == 3); @@ -240,21 +225,142 @@ TEST_CASE("QUICStreamManager_total_offset_sent", "[quic]") CHECK(sm.total_offset_sent() == 0); Ptr block_1024 = make_ptr(new_IOBufferBlock()); - block_1024->alloc(); + block_1024->alloc(BUFFER_SIZE_INDEX_32K); block_1024->fill(1024); CHECK(block_1024->read_avail() == 1024); // total_offset should be a integer in unit of octets uint8_t frame_buf[4096]; mock_app.send(reinterpret_cast(block_1024->buf()), 1024, 0); - sm.generate_frame(frame_buf, QUICEncryptionLevel::ONE_RTT, 16384, 16384, 0); + sm.generate_frame(frame_buf, QUICEncryptionLevel::ONE_RTT, 16384, 16384, 0, 0); CHECK(sm.total_offset_sent() == 1024); // total_offset should be a integer in unit of octets mock_app.send(reinterpret_cast(block_1024->buf()), 1024, 4); - sm.generate_frame(frame_buf, QUICEncryptionLevel::ONE_RTT, 16384, 16384, 0); + sm.generate_frame(frame_buf, QUICEncryptionLevel::ONE_RTT, 16384, 16384, 0, 0); CHECK(sm.total_offset_sent() == 2048); // Wait for event processing sleep(2); } + +TEST_CASE("QUICStreamManager_max_streams", "[quic]") +{ + QUICEncryptionLevel level = QUICEncryptionLevel::ONE_RTT; + QUICApplicationMap app_map; + MockQUICConnection connection; + MockQUICApplication mock_app(&connection); + app_map.set_default(&mock_app); + QUICStreamManager sm(&context, &app_map); + + uint8_t local_tp_buf[] = { + 0x08, // parameter id - initial_max_streams_bidi + 0x01, // length of value + 0x03, // value + 0x09, // parameter id - initial_max_streams_uni + 0x01, // length of value + 0x03, // value + }; + std::shared_ptr local_tp = + std::make_shared(local_tp_buf, sizeof(local_tp_buf), QUIC_SUPPORTED_VERSIONS[0]); + + uint8_t remote_tp_buf[] = { + 0x08, // parameter id - initial_max_streams_bidi + 0x01, // length of value + 0x03, // value + 0x09, // parameter id - initial_max_streams_uni + 0x01, // length of value + 0x03, // value + }; + std::shared_ptr remote_tp = + std::make_shared(remote_tp_buf, sizeof(remote_tp_buf), QUIC_SUPPORTED_VERSIONS[0]); + + sm.init_flow_control_params(local_tp, remote_tp); + + SECTION("local") + { + // RESET_STREAM frames create new streams + + // Bidirectional + uint8_t rst_stream_frame_buf[QUICFrame::MAX_INSTANCE_SIZE]; + QUICFrame *rst_stream_frame = + QUICFrameFactory::create_rst_stream_frame(rst_stream_frame_buf, 1, static_cast(0x01), 0); + sm.handle_frame(level, *rst_stream_frame); + REQUIRE(sm.stream_count() == 1); + + rst_stream_frame = QUICFrameFactory::create_rst_stream_frame(rst_stream_frame_buf, 5, static_cast(0x01), 0); + sm.handle_frame(level, *rst_stream_frame); + REQUIRE(sm.stream_count() == 2); + + rst_stream_frame = QUICFrameFactory::create_rst_stream_frame(rst_stream_frame_buf, 9, static_cast(0x01), 0); + sm.handle_frame(level, *rst_stream_frame); + REQUIRE(sm.stream_count() == 3); + + rst_stream_frame = QUICFrameFactory::create_rst_stream_frame(rst_stream_frame_buf, 13, static_cast(0x01), 0); + sm.handle_frame(level, *rst_stream_frame); + REQUIRE(sm.stream_count() == 3); + + // Unidirectional + rst_stream_frame = QUICFrameFactory::create_rst_stream_frame(rst_stream_frame_buf, 3, static_cast(0x01), 0); + sm.handle_frame(level, *rst_stream_frame); + REQUIRE(sm.stream_count() == 4); + + rst_stream_frame = QUICFrameFactory::create_rst_stream_frame(rst_stream_frame_buf, 7, static_cast(0x01), 0); + sm.handle_frame(level, *rst_stream_frame); + REQUIRE(sm.stream_count() == 5); + + rst_stream_frame = QUICFrameFactory::create_rst_stream_frame(rst_stream_frame_buf, 11, static_cast(0x01), 0); + sm.handle_frame(level, *rst_stream_frame); + REQUIRE(sm.stream_count() == 6); + + rst_stream_frame = QUICFrameFactory::create_rst_stream_frame(rst_stream_frame_buf, 15, static_cast(0x01), 0); + sm.handle_frame(level, *rst_stream_frame); + REQUIRE(sm.stream_count() == 6); + } + + SECTION("remote") + { + // Bidirection + QUICConnectionErrorUPtr error; + QUICStreamId id; + + error = sm.create_bidi_stream(id); + REQUIRE(!error); + REQUIRE(id == 0); + REQUIRE(sm.stream_count() == 1); + + error = sm.create_bidi_stream(id); + REQUIRE(!error); + REQUIRE(id == 4); + REQUIRE(sm.stream_count() == 2); + + error = sm.create_bidi_stream(id); + REQUIRE(!error); + REQUIRE(id == 8); + REQUIRE(sm.stream_count() == 3); + + error = sm.create_bidi_stream(id); + REQUIRE(error); + REQUIRE(sm.stream_count() == 3); + + // Unidirection + error = sm.create_uni_stream(id); + REQUIRE(!error); + REQUIRE(id == 2); + REQUIRE(sm.stream_count() == 4); + + error = sm.create_uni_stream(id); + REQUIRE(!error); + REQUIRE(id == 6); + REQUIRE(sm.stream_count() == 5); + + error = sm.create_uni_stream(id); + REQUIRE(!error); + REQUIRE(id == 10); + REQUIRE(sm.stream_count() == 6); + + error = sm.create_uni_stream(id); + REQUIRE(error); + REQUIRE(sm.stream_count() == 6); + } +} diff --git a/iocore/net/quic/test/test_QUICStreamState.cc b/iocore/net/quic/test/test_QUICStreamState.cc index b5538669d16..9eb11281475 100644 --- a/iocore/net/quic/test/test_QUICStreamState.cc +++ b/iocore/net/quic/test/test_QUICStreamState.cc @@ -33,7 +33,7 @@ TEST_CASE("QUICSendStreamState", "[quic]") { Ptr block_4 = make_ptr(new_IOBufferBlock()); - block_4->alloc(); + block_4->alloc(BUFFER_SIZE_INDEX_32K); block_4->fill(4); CHECK(block_4->read_avail() == 4); @@ -183,7 +183,7 @@ TEST_CASE("QUICSendStreamState", "[quic]") TEST_CASE("QUICReceiveStreamState", "[quic]") { Ptr block_4 = make_ptr(new_IOBufferBlock()); - block_4->alloc(); + block_4->alloc(BUFFER_SIZE_INDEX_32K); block_4->fill(4); CHECK(block_4->read_avail() == 4); @@ -366,7 +366,7 @@ TEST_CASE("QUICReceiveStreamState", "[quic]") TEST_CASE("QUICBidiState", "[quic]") { Ptr block_4 = make_ptr(new_IOBufferBlock()); - block_4->alloc(); + block_4->alloc(BUFFER_SIZE_INDEX_32K); block_4->fill(4); CHECK(block_4->read_avail() == 4); diff --git a/iocore/net/quic/test/test_QUICTransportParameters.cc b/iocore/net/quic/test/test_QUICTransportParameters.cc index d60e2606790..cddad460f8a 100644 --- a/iocore/net/quic/test/test_QUICTransportParameters.cc +++ b/iocore/net/quic/test/test_QUICTransportParameters.cc @@ -30,32 +30,31 @@ TEST_CASE("QUICTransportParametersInClientHello_read", "[quic]") SECTION("OK") { uint8_t buf[] = { - 0x00, 0x1c, // size of parameters - 0x00, 0x00, // parameter id - 0x00, 0x04, // length of value + 0x00, // parameter id + 0x04, // length of value 0x11, 0x22, 0x33, 0x44, // value - 0x00, 0x01, // parameter id - 0x00, 0x04, // length of value + 0x01, // parameter id + 0x04, // length of value 0x12, 0x34, 0x56, 0x78, // value - 0x00, 0x05, // parameter id - 0x00, 0x02, // length of value + 0x05, // parameter id + 0x02, // length of value 0x0a, 0x0b, // value - 0x00, 0x03, // parameter id - 0x00, 0x02, // length of value + 0x03, // parameter id + 0x02, // length of value 0x05, 0x67, // value }; - QUICTransportParametersInClientHello params_in_ch(buf, sizeof(buf)); + QUICTransportParametersInClientHello params_in_ch(buf, sizeof(buf), QUIC_SUPPORTED_VERSIONS[0]); CHECK(params_in_ch.is_valid()); uint16_t len = 0; const uint8_t *data = nullptr; - data = params_in_ch.getAsBytes(QUICTransportParameterId::ORIGINAL_CONNECTION_ID, len); + data = params_in_ch.getAsBytes(QUICTransportParameterId::ORIGINAL_DESTINATION_CONNECTION_ID, len); CHECK(len == 4); CHECK(memcmp(data, "\x11\x22\x33\x44", 4) == 0); - data = params_in_ch.getAsBytes(QUICTransportParameterId::IDLE_TIMEOUT, len); + data = params_in_ch.getAsBytes(QUICTransportParameterId::MAX_IDLE_TIMEOUT, len); CHECK(len == 4); CHECK(memcmp(data, "\x12\x34\x56\x78", 4) == 0); @@ -63,7 +62,7 @@ TEST_CASE("QUICTransportParametersInClientHello_read", "[quic]") CHECK(len == 2); CHECK(memcmp(data, "\x0a\x0b", 2) == 0); - data = params_in_ch.getAsBytes(QUICTransportParameterId::MAX_PACKET_SIZE, len); + data = params_in_ch.getAsBytes(QUICTransportParameterId::MAX_UDP_PAYLOAD_SIZE, len); CHECK(len == 2); CHECK(memcmp(data, "\x05\x67", 2) == 0); @@ -75,16 +74,15 @@ TEST_CASE("QUICTransportParametersInClientHello_read", "[quic]") SECTION("Duplicate parameters") { uint8_t buf[] = { - 0x00, 0x10, // size of parameters - 0x00, 0x00, // parameter id - 0x00, 0x04, // length of value + 0x00, // parameter id + 0x04, // length of value 0x11, 0x22, 0x33, 0x44, // value - 0x00, 0x00, // parameter id - 0x00, 0x04, // length of value + 0x00, // parameter id + 0x04, // length of value 0x12, 0x34, 0x56, 0x78, // value }; - QUICTransportParametersInClientHello params_in_ch(buf, sizeof(buf)); + QUICTransportParametersInClientHello params_in_ch(buf, sizeof(buf), QUIC_SUPPORTED_VERSIONS[0]); CHECK(!params_in_ch.is_valid()); } } @@ -95,16 +93,15 @@ TEST_CASE("QUICTransportParametersInClientHello_write", "[quic]") uint16_t len; uint8_t expected[] = { - 0x00, 0x22, // size of parameters - 0x00, 0x02, // parameter id - 0x00, 0x10, // length of value + 0x02, // parameter id + 0x10, // length of value 0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, // value 0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, // value - 0x00, 0x03, // parameter id - 0x00, 0x02, // length of value + 0x03, // parameter id + 0x02, // length of value 0x5b, 0xcd, // value - 0x00, 0x05, // parameter id - 0x00, 0x04, // length of value + 0x05, // parameter id + 0x04, // length of value 0x91, 0x22, 0x33, 0x44, // value }; @@ -114,14 +111,14 @@ TEST_CASE("QUICTransportParametersInClientHello_write", "[quic]") params_in_ch.set(QUICTransportParameterId::INITIAL_MAX_STREAM_DATA_BIDI_LOCAL, max_stream_data); uint16_t max_packet_size = 0x1bcd; - params_in_ch.set(QUICTransportParameterId::MAX_PACKET_SIZE, max_packet_size); + params_in_ch.set(QUICTransportParameterId::MAX_UDP_PAYLOAD_SIZE, max_packet_size); uint8_t stateless_reset_token[16] = {0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77}; params_in_ch.set(QUICTransportParameterId::STATELESS_RESET_TOKEN, stateless_reset_token, 16); params_in_ch.store(buf, &len); - CHECK(len == 36); + CHECK(len == 28); CHECK(memcmp(buf, expected, len) == 0); } @@ -130,22 +127,22 @@ TEST_CASE("QUICTransportParametersInEncryptedExtensions_read", "[quic]") SECTION("OK case") { uint8_t buf[] = { - 0x00, 0x2a, // size of parameters - 0x00, 0x01, // parameter id - 0x00, 0x02, // length of value - 0x51, 0x23, // value - 0x00, 0x02, // parameter id - 0x00, 0x10, // length of value - 0x00, 0x10, 0x20, 0x30, // value - 0x40, 0x50, 0x60, 0x70, 0x80, 0x90, 0xa0, 0xb0, 0xc0, 0xd0, 0xe0, 0xf0, 0x00, 0x04, // parameter id - 0x00, 0x04, // length of value - 0x92, 0x34, 0x56, 0x78, // value - 0x00, 0x06, // parameter id - 0x00, 0x04, // length of value - 0x91, 0x22, 0x33, 0x44, // value + 0x01, // parameter id + 0x02, // length of value + 0x51, 0x23, // value + 0x02, // parameter id + 0x10, // length of value + 0x00, 0x10, 0x20, 0x30, // value + 0x40, 0x50, 0x60, 0x70, 0x80, 0x90, 0xa0, 0xb0, 0xc0, 0xd0, 0xe0, 0xf0, + 0x04, // parameter id + 0x04, // length of value + 0x92, 0x34, 0x56, 0x78, // value + 0x06, // parameter id + 0x04, // length of value + 0x91, 0x22, 0x33, 0x44, // value }; - QUICTransportParametersInEncryptedExtensions params_in_ee(buf, sizeof(buf)); + QUICTransportParametersInEncryptedExtensions params_in_ee(buf, sizeof(buf), QUIC_SUPPORTED_VERSIONS[0]); CHECK(params_in_ee.is_valid()); uint16_t len = 0; @@ -159,35 +156,34 @@ TEST_CASE("QUICTransportParametersInEncryptedExtensions_read", "[quic]") CHECK(len == 4); CHECK(memcmp(data, "\x92\x34\x56\x78", 4) == 0); - data = params_in_ee.getAsBytes(QUICTransportParameterId::IDLE_TIMEOUT, len); + data = params_in_ee.getAsBytes(QUICTransportParameterId::MAX_IDLE_TIMEOUT, len); CHECK(len == 2); CHECK(memcmp(data, "\x51\x23", 2) == 0); data = params_in_ee.getAsBytes(QUICTransportParameterId::STATELESS_RESET_TOKEN, len); CHECK(len == 16); - CHECK(memcmp(data, buf + 12, 16) == 0); + CHECK(memcmp(data, buf + 6, 16) == 0); - CHECK(!params_in_ee.contains(QUICTransportParameterId::DISABLE_MIGRATION)); + CHECK(!params_in_ee.contains(QUICTransportParameterId::DISABLE_ACTIVE_MIGRATION)); } SECTION("OK case - zero length value") { uint8_t buf[] = { - 0x00, 0x1a, // size of parameters - 0x00, 0x01, // parameter id - 0x00, 0x02, // length of value + 0x01, // parameter id + 0x02, // length of value 0x51, 0x23, // value - 0x00, 0x04, // parameter id - 0x00, 0x04, // length of value + 0x04, // parameter id + 0x04, // length of value 0xa2, 0x34, 0x56, 0x78, // value - 0x00, 0x06, // parameter id - 0x00, 0x04, // length of value + 0x06, // parameter id + 0x04, // length of value 0xa1, 0x22, 0x33, 0x44, // value - 0x00, 0x0c, // parameter id - 0x00, 0x00, // length of value + 0x0c, // parameter id + 0x00, // length of value }; - QUICTransportParametersInEncryptedExtensions params_in_ee(buf, sizeof(buf)); + QUICTransportParametersInEncryptedExtensions params_in_ee(buf, sizeof(buf), QUIC_SUPPORTED_VERSIONS[0]); CHECK(params_in_ee.is_valid()); uint16_t len = 0; @@ -201,26 +197,25 @@ TEST_CASE("QUICTransportParametersInEncryptedExtensions_read", "[quic]") CHECK(len == 4); CHECK(memcmp(data, "\xa2\x34\x56\x78", 4) == 0); - data = params_in_ee.getAsBytes(QUICTransportParameterId::IDLE_TIMEOUT, len); + data = params_in_ee.getAsBytes(QUICTransportParameterId::MAX_IDLE_TIMEOUT, len); CHECK(len == 2); CHECK(memcmp(data, "\x51\x23", 2) == 0); - CHECK(params_in_ee.contains(QUICTransportParameterId::DISABLE_MIGRATION)); + CHECK(params_in_ee.contains(QUICTransportParameterId::DISABLE_ACTIVE_MIGRATION)); } SECTION("Duplicate parameters") { uint8_t buf[] = { - 0x00, 0x1e, // size of parameters - 0x00, 0x00, // parameter id - 0x00, 0x04, // length of value + 0x00, // parameter id + 0x04, // length of value 0x01, 0x02, 0x03, 0x04, // value - 0x00, 0x00, // parameter id - 0x00, 0x04, // length of value + 0x00, // parameter id + 0x04, // length of value 0x12, 0x34, 0x56, 0x78, // value }; - QUICTransportParametersInEncryptedExtensions params_in_ee(buf, sizeof(buf)); + QUICTransportParametersInEncryptedExtensions params_in_ee(buf, sizeof(buf), QUIC_SUPPORTED_VERSIONS[0]); CHECK(!params_in_ee.is_valid()); } } @@ -233,12 +228,11 @@ TEST_CASE("QUICTransportParametersEncryptedExtensions_write", "[quic]") uint16_t len; uint8_t expected[] = { - 0x00, 0x0e, // size of parameters - 0x00, 0x03, // parameter id - 0x00, 0x02, // length of value + 0x03, // parameter id + 0x02, // length of value 0x5b, 0xcd, // value - 0x00, 0x06, // parameter id - 0x00, 0x04, // length of value + 0x06, // parameter id + 0x04, // length of value 0x91, 0x22, 0x33, 0x44, // value }; @@ -248,12 +242,10 @@ TEST_CASE("QUICTransportParametersEncryptedExtensions_write", "[quic]") params_in_ee.set(QUICTransportParameterId::INITIAL_MAX_STREAM_DATA_BIDI_REMOTE, max_stream_data); uint16_t max_packet_size = 0x1bcd; - params_in_ee.set(QUICTransportParameterId::MAX_PACKET_SIZE, max_packet_size); + params_in_ee.set(QUICTransportParameterId::MAX_UDP_PAYLOAD_SIZE, max_packet_size); - params_in_ee.add_version(0x01020304); - params_in_ee.add_version(0x05060708); params_in_ee.store(buf, &len); - CHECK(len == 16); + CHECK(len == 10); CHECK(memcmp(buf, expected, len) == 0); } @@ -263,15 +255,14 @@ TEST_CASE("QUICTransportParametersEncryptedExtensions_write", "[quic]") uint16_t len; uint8_t expected[] = { - 0x00, 0x12, // size of parameters - 0x00, 0x03, // parameter id - 0x00, 0x02, // length of value + 0x03, // parameter id + 0x02, // length of value 0x5b, 0xcd, // value - 0x00, 0x06, // parameter id - 0x00, 0x04, // length of value + 0x06, // parameter id + 0x04, // length of value 0x91, 0x22, 0x33, 0x44, // value - 0x00, 0x0c, // parameter id - 0x00, 0x00, // length of value + 0x0c, // parameter id + 0x00, // length of value }; QUICTransportParametersInEncryptedExtensions params_in_ee; @@ -280,13 +271,11 @@ TEST_CASE("QUICTransportParametersEncryptedExtensions_write", "[quic]") params_in_ee.set(QUICTransportParameterId::INITIAL_MAX_STREAM_DATA_BIDI_REMOTE, max_stream_data); uint16_t max_packet_size = 0x1bcd; - params_in_ee.set(QUICTransportParameterId::MAX_PACKET_SIZE, max_packet_size); - params_in_ee.set(QUICTransportParameterId::DISABLE_MIGRATION, nullptr, 0); + params_in_ee.set(QUICTransportParameterId::MAX_UDP_PAYLOAD_SIZE, max_packet_size); + params_in_ee.set(QUICTransportParameterId::DISABLE_ACTIVE_MIGRATION, nullptr, 0); - params_in_ee.add_version(0x01020304); - params_in_ee.add_version(0x05060708); params_in_ee.store(buf, &len); - CHECK(len == 20); + CHECK(len == 12); CHECK(memcmp(buf, expected, len) == 0); } } diff --git a/iocore/net/quic/test/test_QUICType.cc b/iocore/net/quic/test/test_QUICType.cc index aff8247fb4c..b08085cf818 100644 --- a/iocore/net/quic/test/test_QUICType.cc +++ b/iocore/net/quic/test/test_QUICType.cc @@ -30,16 +30,87 @@ TEST_CASE("QUICType", "[quic]") { + SECTION("QUICPath") + { + IpEndpoint local_a, local_b, remote_a, remote_b; + QUICPath path_a = {{}, {}}, path_b = {{}, {}}; + + // The same addresses and ports -> TRUE + ats_ip_pton("192.168.0.1:4433", &local_a); + ats_ip_pton("192.168.1.1:12345", &remote_a); + ats_ip_pton("192.168.0.1:4433", &local_b); + ats_ip_pton("192.168.1.1:12345", &remote_b); + path_a = {local_a, remote_a}; + path_b = {local_b, remote_b}; + CHECK(path_a == path_b); + CHECK(path_b == path_a); + path_a = {remote_a, local_a}; + path_b = {remote_b, local_b}; + CHECK(path_a == path_b); + CHECK(path_b == path_a); + + // Different ports -> FALSE + ats_ip_pton("192.168.0.1:4433", &local_a); + ats_ip_pton("192.168.1.1:12345", &remote_a); + ats_ip_pton("192.168.0.1:4433", &local_b); + ats_ip_pton("192.168.1.1:54321", &remote_b); + path_a = {local_a, remote_a}; + path_b = {local_b, remote_b}; + CHECK(!(path_a == path_b)); + CHECK(!(path_b == path_a)); + path_a = {remote_a, local_a}; + path_b = {remote_b, local_b}; + CHECK(!(path_a == path_b)); + CHECK(!(path_b == path_a)); + + // Different addresses but the same ports -> FALSE + ats_ip_pton("192.168.0.1:4433", &local_a); + ats_ip_pton("192.168.1.1:12345", &remote_a); + ats_ip_pton("192.168.0.1:4433", &local_b); + ats_ip_pton("192.168.2.1:12345", &remote_b); + path_a = {local_a, remote_a}; + path_b = {local_b, remote_b}; + CHECK(!(path_a == path_b)); + CHECK(!(path_b == path_a)); + path_a = {remote_a, local_a}; + path_b = {remote_b, local_b}; + CHECK(!(path_a == path_b)); + CHECK(!(path_b == path_a)); + + // Server local address is any -> TRUE + ats_ip_pton("0.0.0.0:4433", &local_a); + ats_ip_pton("192.168.1.1:12345", &remote_a); + ats_ip_pton("192.168.0.1:4433", &local_b); + ats_ip_pton("192.168.1.1:12345", &remote_b); + path_a = {local_a, remote_a}; + path_b = {local_b, remote_b}; + CHECK(path_a == path_b); + CHECK(path_b == path_a); + + // Client local address and port are any -> TRUE + ats_ip_pton("0.0.0.0:0", &local_a); + ats_ip_pton("192.168.1.1:12345", &remote_a); + ats_ip_pton("192.168.0.1:4433", &local_b); + ats_ip_pton("192.168.1.1:12345", &remote_b); + path_a = {local_a, remote_a}; + path_b = {local_b, remote_b}; + CHECK(path_a == path_b); + CHECK(path_b == path_a); + } + SECTION("QUICRetryToken") { IpEndpoint ep; ats_ip4_set(&ep, 0x04030201, 0x2211); - uint8_t cid_buf[] = {0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, - 0x19, 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27}; - QUICConnectionId cid(cid_buf, sizeof(cid_buf)); + uint8_t cid1_buf[] = {0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, + 0x19, 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27}; + QUICConnectionId cid1(cid1_buf, sizeof(cid1_buf)); + uint8_t cid2_buf[] = {0xA0, 0xA1, 0x12, 0xA3, 0xA4, 0xA5, 0xA6, 0xA7, 0xA8, + 0xA9, 0xB0, 0xB1, 0xB2, 0xB3, 0xB4, 0xB5, 0xB6, 0xB7}; + QUICConnectionId cid2(cid2_buf, sizeof(cid2_buf)); - QUICRetryToken token1(ep, cid); + QUICRetryToken token1(ep, cid1, cid2); QUICRetryToken token2(token1.buf(), token1.length()); CHECK(token1.is_valid(ep)); @@ -50,6 +121,7 @@ TEST_CASE("QUICType", "[quic]") CHECK(token1.length() == token2.length()); CHECK(memcmp(token1.buf(), token2.buf(), token1.length()) == 0); CHECK(token1.original_dcid() == token2.original_dcid()); + CHECK(token1.scid() == token2.scid()); } SECTION("QUICResumptionToken") diff --git a/iocore/net/quic/test/test_QUICVersionNegotiator.cc b/iocore/net/quic/test/test_QUICVersionNegotiator.cc index be76949a89d..35397459c86 100644 --- a/iocore/net/quic/test/test_QUICVersionNegotiator.cc +++ b/iocore/net/quic/test/test_QUICVersionNegotiator.cc @@ -24,7 +24,9 @@ #include "catch.hpp" #include "quic/QUICVersionNegotiator.h" +#include "quic/QUICPacketFactory.h" #include "quic/QUICPacketProtectionKeyInfo.h" +#include "quic/QUICPacketFactory.h" #include "quic/Mock.h" TEST_CASE("QUICVersionNegotiator - Server Side", "[quic]") @@ -34,51 +36,66 @@ TEST_CASE("QUICVersionNegotiator - Server Side", "[quic]") QUICPacketFactory packet_factory(pp_key_info); QUICVersionNegotiator vn; - ats_unique_buf dummy_payload = ats_unique_malloc(128); - size_t dummy_payload_len = 128; - SECTION("Normal case") + size_t dummy_payload_len = 128; + Ptr dummy_payload = make_ptr(new_IOBufferBlock()); + dummy_payload->alloc(iobuffer_size_to_index(dummy_payload_len, BUFFER_SIZE_INDEX_32K)); + dummy_payload->fill(dummy_payload_len); + + SECTION("Match") { // Check initial state CHECK(vn.status() == QUICVersionNegotiationStatus::NOT_NEGOTIATED); // Negotiate version packet_factory.set_version(QUIC_SUPPORTED_VERSIONS[0]); + uint8_t packet_buf[QUICPacket::MAX_INSTANCE_SIZE]; QUICPacketUPtr initial_packet = - packet_factory.create_initial_packet({}, {}, 0, std::move(dummy_payload), dummy_payload_len, true, false, true); - + packet_factory.create_initial_packet(packet_buf, {}, {}, 0, dummy_payload, dummy_payload_len, true, false, true); REQUIRE(initial_packet != nullptr); - vn.negotiate(*initial_packet); + + auto blocks = initial_packet->header_block(); + blocks->next = initial_packet->payload_block(); + QUICInitialPacketR received_initial_packet(nullptr, {}, {}, blocks, 0); + vn.negotiate(received_initial_packet); CHECK(vn.status() == QUICVersionNegotiationStatus::NEGOTIATED); } - SECTION("Negotiation case") + SECTION("Unmatch") { // Check initial state CHECK(vn.status() == QUICVersionNegotiationStatus::NOT_NEGOTIATED); // Negotiate version - packet_factory.set_version(QUIC_SUPPORTED_VERSIONS[0]); + packet_factory.set_version(0xff000001); + uint8_t packet_buf[QUICPacket::MAX_INSTANCE_SIZE]; QUICPacketUPtr initial_packet = - packet_factory.create_initial_packet({}, {}, 0, std::move(dummy_payload), dummy_payload_len, true, false, true); - + packet_factory.create_initial_packet(packet_buf, {}, {}, 0, std::move(dummy_payload), dummy_payload_len, true, false, true); REQUIRE(initial_packet != nullptr); - vn.negotiate(*initial_packet); - CHECK(vn.status() == QUICVersionNegotiationStatus::NEGOTIATED); + + auto blocks = initial_packet->header_block(); + blocks->next = initial_packet->payload_block(); + QUICInitialPacketR received_initial_packet(nullptr, {}, {}, blocks, 0); + vn.negotiate(received_initial_packet); + CHECK(vn.status() == QUICVersionNegotiationStatus::NOT_NEGOTIATED); } - SECTION("Downgrade case") + SECTION("Exercise") { // Check initial state CHECK(vn.status() == QUICVersionNegotiationStatus::NOT_NEGOTIATED); // Negotiate version - packet_factory.set_version(QUIC_EXERCISE_VERSION); + packet_factory.set_version(QUIC_EXERCISE_VERSION1); + uint8_t packet_buf[QUICPacket::MAX_INSTANCE_SIZE]; QUICPacketUPtr initial_packet = - packet_factory.create_initial_packet({}, {}, 0, std::move(dummy_payload), dummy_payload_len, true, false, true); - + packet_factory.create_initial_packet(packet_buf, {}, {}, 0, std::move(dummy_payload), dummy_payload_len, true, false, true); REQUIRE(initial_packet != nullptr); - vn.negotiate(*initial_packet); + + auto blocks = initial_packet->header_block(); + blocks->next = initial_packet->payload_block(); + QUICInitialPacketR received_initial_packet(nullptr, {}, {}, blocks, 0); + vn.negotiate(received_initial_packet); CHECK(vn.status() == QUICVersionNegotiationStatus::NOT_NEGOTIATED); } } @@ -90,8 +107,10 @@ TEST_CASE("QUICVersionNegotiator - Client Side", "[quic]") QUICPacketFactory packet_factory(pp_key_info); QUICVersionNegotiator vn; - ats_unique_buf dummy_payload = ats_unique_malloc(128); - size_t dummy_payload_len = 128; + size_t dummy_payload_len = 128; + Ptr dummy_payload = make_ptr(new_IOBufferBlock()); + dummy_payload->alloc(iobuffer_size_to_index(dummy_payload_len, BUFFER_SIZE_INDEX_32K)); + dummy_payload->fill(dummy_payload_len); SECTION("Normal case") { @@ -107,18 +126,23 @@ TEST_CASE("QUICVersionNegotiator - Client Side", "[quic]") CHECK(vn.status() == QUICVersionNegotiationStatus::NOT_NEGOTIATED); // Negotiate version - packet_factory.set_version(QUIC_EXERCISE_VERSION); - QUICPacketUPtr initial_packet = - packet_factory.create_initial_packet({}, {}, 0, std::move(dummy_payload), dummy_payload_len, true, false, true); - REQUIRE(initial_packet != nullptr); + packet_factory.set_version(QUIC_EXERCISE_VERSION1); + uint8_t initial_packet_buf[QUICPacket::MAX_INSTANCE_SIZE]; + QUICPacketUPtr packet = packet_factory.create_initial_packet(initial_packet_buf, {}, {}, 0, std::move(dummy_payload), + dummy_payload_len, true, false, true); + REQUIRE(packet != nullptr); + QUICInitialPacket &initial_packet = static_cast(*packet); // Server send VN packet based on Initial packet - QUICPacketUPtr vn_packet = - packet_factory.create_version_negotiation_packet(initial_packet->source_cid(), initial_packet->destination_cid()); + QUICPacketUPtr vn_packet = packet_factory.create_version_negotiation_packet( + initial_packet.source_cid(), initial_packet.destination_cid(), QUIC_EXERCISE_VERSION1); REQUIRE(vn_packet != nullptr); + auto blocks = vn_packet->header_block(); + blocks->next = vn_packet->payload_block(); + QUICVersionNegotiationPacketR received_vn_packet(nullptr, {}, {}, blocks); // Negotiate version - vn.negotiate(*vn_packet); + vn.negotiate(received_vn_packet); CHECK(vn.status() == QUICVersionNegotiationStatus::NEGOTIATED); CHECK(vn.negotiated_version() == QUIC_SUPPORTED_VERSIONS[0]); } diff --git a/iocore/net/test_I_UDPNet.cc b/iocore/net/test_I_UDPNet.cc index 3971f995b15..9f50efe1e05 100644 --- a/iocore/net/test_I_UDPNet.cc +++ b/iocore/net/test_I_UDPNet.cc @@ -61,7 +61,7 @@ EchoServer::start() addr.sin_addr.s_addr = htonl(INADDR_LOOPBACK); addr.sin_port = 0; - udpNet.UDPBind(static_cast(this), reinterpret_cast(&addr), 1048576, 1048576); + udpNet.UDPBind(static_cast(this), reinterpret_cast(&addr), -1, 1048576, 1048576); return true; } diff --git a/iocore/utils/OneWayMultiTunnel.cc b/iocore/utils/OneWayMultiTunnel.cc index 3b8830ae3a1..2a3f8783136 100644 --- a/iocore/utils/OneWayMultiTunnel.cc +++ b/iocore/utils/OneWayMultiTunnel.cc @@ -64,9 +64,9 @@ OneWayMultiTunnel::init(VConnection *vcSource, VConnection **vcTargets, int n_vc int64_t size_index = 0; if (size_estimate) { - size_index = buffer_size_to_index(size_estimate, default_large_iobuffer_size); + size_index = buffer_size_to_index(size_estimate, BUFFER_SIZE_INDEX_32K); } else { - size_index = default_large_iobuffer_size; + size_index = BUFFER_SIZE_INDEX_32K; } tunnel_till_done = (nbytes == TUNNEL_TILL_DONE); diff --git a/iocore/utils/OneWayTunnel.cc b/iocore/utils/OneWayTunnel.cc index c2238b4c1d5..b2ac187c4a4 100644 --- a/iocore/utils/OneWayTunnel.cc +++ b/iocore/utils/OneWayTunnel.cc @@ -126,9 +126,9 @@ OneWayTunnel::init(VConnection *vcSource, VConnection *vcTarget, Continuation *a int64_t size_index = 0; if (size_estimate) { - size_index = buffer_size_to_index(size_estimate); + size_index = buffer_size_to_index(size_estimate, BUFFER_SIZE_INDEX_32K); } else { - size_index = default_large_iobuffer_size; + size_index = BUFFER_SIZE_INDEX_32K; } Debug("one_way_tunnel", "buffer size index [%" PRId64 "] [%d]", size_index, size_estimate); diff --git a/lib/Makefile.am b/lib/Makefile.am index 8b3d28e60f2..cc091b38b68 100644 --- a/lib/Makefile.am +++ b/lib/Makefile.am @@ -34,3 +34,8 @@ all-local: $(LOCAL) clean-local: $(MAKE) -C yamlcpp clean + +if EXPORT_YAML_HEADERS +install-data-local: + $(MAKE) -C yamlcpp install +endif diff --git a/lib/perl/lib/Apache/TS/AdminClient.pm b/lib/perl/lib/Apache/TS/AdminClient.pm index a8023f9f883..dbccd3eeaec 100644 --- a/lib/perl/lib/Apache/TS/AdminClient.pm +++ b/lib/perl/lib/Apache/TS/AdminClient.pm @@ -349,7 +349,6 @@ The Apache Traffic Server Administration Manual will explain what these strings proxy.config.cache.ram_cache_cutoff proxy.config.cache.ram_cache.size proxy.config.cache.select_alternate - proxy.config.cache.storage_filename proxy.config.cache.threads_per_disk proxy.config.cache.mutex_retry_delay proxy.config.cop.core_signal diff --git a/lib/records/I_RecCore.h b/lib/records/I_RecCore.h index 6a0e3b23fba..061ebb8dbb7 100644 --- a/lib/records/I_RecCore.h +++ b/lib/records/I_RecCore.h @@ -166,7 +166,7 @@ RecErrT RecLookupMatchingRecords(unsigned rec_type, const char *match, RecLookup RecErrT RecGetRecordType(const char *name, RecT *rec_type, bool lock = true); RecErrT RecGetRecordDataType(const char *name, RecDataT *data_type, bool lock = true); RecErrT RecGetRecordPersistenceType(const char *name, RecPersistT *persist_type, bool lock = true); -RecErrT RecGetRecordOrderAndId(const char *name, int *order, int *id, bool lock = true); +RecErrT RecGetRecordOrderAndId(const char *name, int *order, int *id, bool lock = true, bool check_sync_cb = false); RecErrT RecGetRecordUpdateType(const char *name, RecUpdateT *update_type, bool lock = true); RecErrT RecGetRecordCheckType(const char *name, RecCheckT *check_type, bool lock = true); RecErrT RecGetRecordCheckExpr(const char *name, char **check_expr, bool lock = true); diff --git a/lib/records/Makefile.am b/lib/records/Makefile.am index e403a81313d..8fc83fb2f95 100644 --- a/lib/records/Makefile.am +++ b/lib/records/Makefile.am @@ -44,7 +44,6 @@ librecords_COMMON = \ P_RecCore.h \ P_RecDefs.h \ P_RecFile.h \ - P_RecFile.h \ P_RecMessage.h \ P_RecUtils.h \ RecConfigParse.cc \ diff --git a/lib/records/P_RecDefs.h b/lib/records/P_RecDefs.h index 0aa4c09d62e..46e44566b3a 100644 --- a/lib/records/P_RecDefs.h +++ b/lib/records/P_RecDefs.h @@ -25,17 +25,13 @@ #include "I_RecDefs.h" -#define REC_CONFIG_FILE "records.config" -#define REC_SHADOW_EXT ".shadow" -#define REC_RAW_STATS_FILE "records.snap" - #define REC_MESSAGE_ELE_MAGIC 0xF00DF00D // We need at least this many internal record entries for our configurations and metrics. Any // additional slots in librecords will be allocated to the plugin metrics. These should be // updated if we change the internal librecords size significantly. #define REC_INTERNAL_RECORDS 1100 -#define REC_MIN_API_RECORDS 500 +#define REC_DEFAULT_API_RECORDS 1400 #define REC_CONFIG_UPDATE_INTERVAL_MS 3000 #define REC_REMOTE_SYNC_INTERVAL_MS 5000 diff --git a/lib/records/RecCore.cc b/lib/records/RecCore.cc index f8b832c4db3..fcb6474cbce 100644 --- a/lib/records/RecCore.cc +++ b/lib/records/RecCore.cc @@ -24,6 +24,7 @@ #include "tscore/ink_platform.h" #include "tscore/ink_memory.h" #include "tscore/ink_string.h" +#include "tscore/Filenames.h" #include "RecordsConfig.h" #include "P_RecFile.h" @@ -33,7 +34,7 @@ // This is needed to manage the size of the librecords record. It can't be static, because it needs to be modified // and used (read) from several binaries / modules. -int max_records_entries = REC_INTERNAL_RECORDS + REC_MIN_API_RECORDS; +int max_records_entries = REC_INTERNAL_RECORDS + REC_DEFAULT_API_RECORDS; static bool g_initialized = false; @@ -218,14 +219,10 @@ RecCoreInit(RecModeT mode_type, Diags *_diags) ink_mutex_init(&g_rec_config_lock); - g_rec_config_fpath = ats_stringdup(RecConfigReadConfigPath(nullptr, REC_CONFIG_FILE REC_SHADOW_EXT)); + g_rec_config_fpath = ats_stringdup(RecConfigReadConfigPath(nullptr, ts::filename::RECORDS)); if (RecFileExists(g_rec_config_fpath) == REC_ERR_FAIL) { - ats_free(const_cast(g_rec_config_fpath)); - g_rec_config_fpath = ats_stringdup(RecConfigReadConfigPath(nullptr, REC_CONFIG_FILE)); - if (RecFileExists(g_rec_config_fpath) == REC_ERR_FAIL) { - RecLog(DL_Warning, "Could not find '%s', system will run with defaults\n", REC_CONFIG_FILE); - file_exists = false; - } + RecLog(DL_Warning, "Could not find '%s', system will run with defaults\n", ts::filename::RECORDS); + file_exists = false; } if (file_exists) { @@ -610,7 +607,7 @@ RecGetRecordPersistenceType(const char *name, RecPersistT *persist_type, bool lo } RecErrT -RecGetRecordOrderAndId(const char *name, int *order, int *id, bool lock) +RecGetRecordOrderAndId(const char *name, int *order, int *id, bool lock, bool check_sync_cb) { RecErrT err = REC_ERR_FAIL; @@ -622,15 +619,17 @@ RecGetRecordOrderAndId(const char *name, int *order, int *id, bool lock) RecRecord *r = it->second; if (r->registered) { - rec_mutex_acquire(&(r->lock)); - if (order) { - *order = r->order; - } - if (id) { - *id = r->rsb_id; + if (!check_sync_cb || r->stat_meta.sync_cb) { + rec_mutex_acquire(&(r->lock)); + if (order) { + *order = r->order; + } + if (id) { + *id = r->rsb_id; + } + err = REC_ERR_OKAY; + rec_mutex_release(&(r->lock)); } - err = REC_ERR_OKAY; - rec_mutex_release(&(r->lock)); } } @@ -1260,7 +1259,7 @@ std::string RecConfigReadPersistentStatsPath() { std::string rundir(RecConfigReadRuntimeDir()); - return Layout::relative_to(rundir, REC_RAW_STATS_FILE); + return Layout::relative_to(rundir, ts::filename::RECORDS_STATS); } void @@ -1286,11 +1285,12 @@ RecSignalWarning(int sig, const char *fmt, ...) void RecConfigWarnIfUnregistered() { - RecDumpRecords(RECT_CONFIG, - [](RecT, void *, int registered_p, const char *name, int, RecData *) -> void { - if (!registered_p) { - Warning("Unrecognized configuration value '%s'", name); - } - }, - nullptr); + RecDumpRecords( + RECT_CONFIG, + [](RecT, void *, int registered_p, const char *name, int, RecData *) -> void { + if (!registered_p) { + Warning("Unrecognized configuration value '%s'", name); + } + }, + nullptr); } diff --git a/lib/records/RecHttp.cc b/lib/records/RecHttp.cc index d8ce6f6fd2d..4a35d29a434 100644 --- a/lib/records/RecHttp.cc +++ b/lib/records/RecHttp.cc @@ -38,39 +38,45 @@ SessionProtocolNameRegistry globalSessionProtocolNameRegistry; These are also used for NPN setup. */ -const char *const TS_ALPN_PROTOCOL_HTTP_0_9 = IP_PROTO_TAG_HTTP_0_9.data(); -const char *const TS_ALPN_PROTOCOL_HTTP_1_0 = IP_PROTO_TAG_HTTP_1_0.data(); -const char *const TS_ALPN_PROTOCOL_HTTP_1_1 = IP_PROTO_TAG_HTTP_1_1.data(); -const char *const TS_ALPN_PROTOCOL_HTTP_2_0 = IP_PROTO_TAG_HTTP_2_0.data(); -const char *const TS_ALPN_PROTOCOL_HTTP_3 = IP_PROTO_TAG_HTTP_3.data(); -const char *const TS_ALPN_PROTOCOL_HTTP_QUIC = IP_PROTO_TAG_HTTP_QUIC.data(); +const char *const TS_ALPN_PROTOCOL_HTTP_0_9 = IP_PROTO_TAG_HTTP_0_9.data(); +const char *const TS_ALPN_PROTOCOL_HTTP_1_0 = IP_PROTO_TAG_HTTP_1_0.data(); +const char *const TS_ALPN_PROTOCOL_HTTP_1_1 = IP_PROTO_TAG_HTTP_1_1.data(); +const char *const TS_ALPN_PROTOCOL_HTTP_2_0 = IP_PROTO_TAG_HTTP_2_0.data(); +const char *const TS_ALPN_PROTOCOL_HTTP_3 = IP_PROTO_TAG_HTTP_3.data(); +const char *const TS_ALPN_PROTOCOL_HTTP_QUIC = IP_PROTO_TAG_HTTP_QUIC.data(); +const char *const TS_ALPN_PROTOCOL_HTTP_3_D27 = IP_PROTO_TAG_HTTP_3_D27.data(); +const char *const TS_ALPN_PROTOCOL_HTTP_QUIC_D27 = IP_PROTO_TAG_HTTP_QUIC_D27.data(); const char *const TS_ALPN_PROTOCOL_GROUP_HTTP = "http"; const char *const TS_ALPN_PROTOCOL_GROUP_HTTP2 = "http2"; -const char *const TS_PROTO_TAG_HTTP_1_0 = TS_ALPN_PROTOCOL_HTTP_1_0; -const char *const TS_PROTO_TAG_HTTP_1_1 = TS_ALPN_PROTOCOL_HTTP_1_1; -const char *const TS_PROTO_TAG_HTTP_2_0 = TS_ALPN_PROTOCOL_HTTP_2_0; -const char *const TS_PROTO_TAG_HTTP_3 = TS_ALPN_PROTOCOL_HTTP_3; -const char *const TS_PROTO_TAG_HTTP_QUIC = TS_ALPN_PROTOCOL_HTTP_QUIC; -const char *const TS_PROTO_TAG_TLS_1_3 = IP_PROTO_TAG_TLS_1_3.data(); -const char *const TS_PROTO_TAG_TLS_1_2 = IP_PROTO_TAG_TLS_1_2.data(); -const char *const TS_PROTO_TAG_TLS_1_1 = IP_PROTO_TAG_TLS_1_1.data(); -const char *const TS_PROTO_TAG_TLS_1_0 = IP_PROTO_TAG_TLS_1_0.data(); -const char *const TS_PROTO_TAG_TCP = IP_PROTO_TAG_TCP.data(); -const char *const TS_PROTO_TAG_UDP = IP_PROTO_TAG_UDP.data(); -const char *const TS_PROTO_TAG_IPV4 = IP_PROTO_TAG_IPV4.data(); -const char *const TS_PROTO_TAG_IPV6 = IP_PROTO_TAG_IPV6.data(); +const char *const TS_PROTO_TAG_HTTP_1_0 = TS_ALPN_PROTOCOL_HTTP_1_0; +const char *const TS_PROTO_TAG_HTTP_1_1 = TS_ALPN_PROTOCOL_HTTP_1_1; +const char *const TS_PROTO_TAG_HTTP_2_0 = TS_ALPN_PROTOCOL_HTTP_2_0; +const char *const TS_PROTO_TAG_HTTP_3 = TS_ALPN_PROTOCOL_HTTP_3; +const char *const TS_PROTO_TAG_HTTP_QUIC = TS_ALPN_PROTOCOL_HTTP_QUIC; +const char *const TS_PROTO_TAG_HTTP_3_D27 = TS_ALPN_PROTOCOL_HTTP_3_D27; +const char *const TS_PROTO_TAG_HTTP_QUIC_D27 = TS_ALPN_PROTOCOL_HTTP_QUIC_D27; +const char *const TS_PROTO_TAG_TLS_1_3 = IP_PROTO_TAG_TLS_1_3.data(); +const char *const TS_PROTO_TAG_TLS_1_2 = IP_PROTO_TAG_TLS_1_2.data(); +const char *const TS_PROTO_TAG_TLS_1_1 = IP_PROTO_TAG_TLS_1_1.data(); +const char *const TS_PROTO_TAG_TLS_1_0 = IP_PROTO_TAG_TLS_1_0.data(); +const char *const TS_PROTO_TAG_TCP = IP_PROTO_TAG_TCP.data(); +const char *const TS_PROTO_TAG_UDP = IP_PROTO_TAG_UDP.data(); +const char *const TS_PROTO_TAG_IPV4 = IP_PROTO_TAG_IPV4.data(); +const char *const TS_PROTO_TAG_IPV6 = IP_PROTO_TAG_IPV6.data(); std::unordered_set TSProtoTags; // Precomputed indices for ease of use. -int TS_ALPN_PROTOCOL_INDEX_HTTP_0_9 = SessionProtocolNameRegistry::INVALID; -int TS_ALPN_PROTOCOL_INDEX_HTTP_1_0 = SessionProtocolNameRegistry::INVALID; -int TS_ALPN_PROTOCOL_INDEX_HTTP_1_1 = SessionProtocolNameRegistry::INVALID; -int TS_ALPN_PROTOCOL_INDEX_HTTP_2_0 = SessionProtocolNameRegistry::INVALID; -int TS_ALPN_PROTOCOL_INDEX_HTTP_3 = SessionProtocolNameRegistry::INVALID; -int TS_ALPN_PROTOCOL_INDEX_HTTP_QUIC = SessionProtocolNameRegistry::INVALID; +int TS_ALPN_PROTOCOL_INDEX_HTTP_0_9 = SessionProtocolNameRegistry::INVALID; +int TS_ALPN_PROTOCOL_INDEX_HTTP_1_0 = SessionProtocolNameRegistry::INVALID; +int TS_ALPN_PROTOCOL_INDEX_HTTP_1_1 = SessionProtocolNameRegistry::INVALID; +int TS_ALPN_PROTOCOL_INDEX_HTTP_2_0 = SessionProtocolNameRegistry::INVALID; +int TS_ALPN_PROTOCOL_INDEX_HTTP_3 = SessionProtocolNameRegistry::INVALID; +int TS_ALPN_PROTOCOL_INDEX_HTTP_QUIC = SessionProtocolNameRegistry::INVALID; +int TS_ALPN_PROTOCOL_INDEX_HTTP_3_D27 = SessionProtocolNameRegistry::INVALID; +int TS_ALPN_PROTOCOL_INDEX_HTTP_QUIC_D27 = SessionProtocolNameRegistry::INVALID; // Predefined protocol sets for ease of use. SessionProtocolSet HTTP_PROTOCOL_SET; @@ -395,8 +401,10 @@ HttpProxyPort::processOptions(const char *opts) af_set_p = true; } else if (0 == strcasecmp(OPT_SSL, item)) { m_type = TRANSPORT_SSL; +#if TS_USE_QUIC == 1 } else if (0 == strcasecmp(OPT_QUIC, item)) { m_type = TRANSPORT_QUIC; +#endif } else if (0 == strcasecmp(OPT_PLUGIN, item)) { m_type = TRANSPORT_PLUGIN; } else if (0 == strcasecmp(OPT_PROXY_PROTO, item)) { @@ -703,12 +711,15 @@ void ts_session_protocol_well_known_name_indices_init() { // register all the well known protocols and get the indices set. - TS_ALPN_PROTOCOL_INDEX_HTTP_0_9 = globalSessionProtocolNameRegistry.toIndexConst(std::string_view{TS_ALPN_PROTOCOL_HTTP_0_9}); - TS_ALPN_PROTOCOL_INDEX_HTTP_1_0 = globalSessionProtocolNameRegistry.toIndexConst(std::string_view{TS_ALPN_PROTOCOL_HTTP_1_0}); - TS_ALPN_PROTOCOL_INDEX_HTTP_1_1 = globalSessionProtocolNameRegistry.toIndexConst(std::string_view{TS_ALPN_PROTOCOL_HTTP_1_1}); - TS_ALPN_PROTOCOL_INDEX_HTTP_2_0 = globalSessionProtocolNameRegistry.toIndexConst(std::string_view{TS_ALPN_PROTOCOL_HTTP_2_0}); - TS_ALPN_PROTOCOL_INDEX_HTTP_3 = globalSessionProtocolNameRegistry.toIndexConst(std::string_view{TS_ALPN_PROTOCOL_HTTP_3}); - TS_ALPN_PROTOCOL_INDEX_HTTP_QUIC = globalSessionProtocolNameRegistry.toIndexConst(std::string_view{TS_ALPN_PROTOCOL_HTTP_QUIC}); + TS_ALPN_PROTOCOL_INDEX_HTTP_0_9 = globalSessionProtocolNameRegistry.toIndexConst(std::string_view{TS_ALPN_PROTOCOL_HTTP_0_9}); + TS_ALPN_PROTOCOL_INDEX_HTTP_1_0 = globalSessionProtocolNameRegistry.toIndexConst(std::string_view{TS_ALPN_PROTOCOL_HTTP_1_0}); + TS_ALPN_PROTOCOL_INDEX_HTTP_1_1 = globalSessionProtocolNameRegistry.toIndexConst(std::string_view{TS_ALPN_PROTOCOL_HTTP_1_1}); + TS_ALPN_PROTOCOL_INDEX_HTTP_2_0 = globalSessionProtocolNameRegistry.toIndexConst(std::string_view{TS_ALPN_PROTOCOL_HTTP_2_0}); + TS_ALPN_PROTOCOL_INDEX_HTTP_3 = globalSessionProtocolNameRegistry.toIndexConst(std::string_view{TS_ALPN_PROTOCOL_HTTP_3}); + TS_ALPN_PROTOCOL_INDEX_HTTP_3_D27 = globalSessionProtocolNameRegistry.toIndexConst(std::string_view{TS_ALPN_PROTOCOL_HTTP_3_D27}); + TS_ALPN_PROTOCOL_INDEX_HTTP_QUIC = globalSessionProtocolNameRegistry.toIndexConst(std::string_view{TS_ALPN_PROTOCOL_HTTP_QUIC}); + TS_ALPN_PROTOCOL_INDEX_HTTP_QUIC_D27 = + globalSessionProtocolNameRegistry.toIndexConst(std::string_view{TS_ALPN_PROTOCOL_HTTP_QUIC_D27}); // Now do the predefined protocol sets. HTTP_PROTOCOL_SET.markIn(TS_ALPN_PROTOCOL_INDEX_HTTP_0_9); @@ -722,6 +733,8 @@ ts_session_protocol_well_known_name_indices_init() DEFAULT_QUIC_SESSION_PROTOCOL_SET.markIn(TS_ALPN_PROTOCOL_INDEX_HTTP_3); DEFAULT_QUIC_SESSION_PROTOCOL_SET.markIn(TS_ALPN_PROTOCOL_INDEX_HTTP_QUIC); + DEFAULT_QUIC_SESSION_PROTOCOL_SET.markIn(TS_ALPN_PROTOCOL_INDEX_HTTP_3_D27); + DEFAULT_QUIC_SESSION_PROTOCOL_SET.markIn(TS_ALPN_PROTOCOL_INDEX_HTTP_QUIC_D27); DEFAULT_NON_TLS_SESSION_PROTOCOL_SET = HTTP_PROTOCOL_SET; @@ -730,6 +743,8 @@ ts_session_protocol_well_known_name_indices_init() TSProtoTags.insert(TS_PROTO_TAG_HTTP_2_0); TSProtoTags.insert(TS_PROTO_TAG_HTTP_3); TSProtoTags.insert(TS_PROTO_TAG_HTTP_QUIC); + TSProtoTags.insert(TS_PROTO_TAG_HTTP_3_D27); + TSProtoTags.insert(TS_PROTO_TAG_HTTP_QUIC_D27); TSProtoTags.insert(TS_PROTO_TAG_TLS_1_3); TSProtoTags.insert(TS_PROTO_TAG_TLS_1_2); TSProtoTags.insert(TS_PROTO_TAG_TLS_1_1); diff --git a/lib/records/unit_tests/unit_test_main.cc b/lib/records/unit_tests/unit_test_main.cc index bfa4a61ba27..45f9d08e1bf 100644 --- a/lib/records/unit_tests/unit_test_main.cc +++ b/lib/records/unit_tests/unit_test_main.cc @@ -27,12 +27,14 @@ #define CATCH_CONFIG_RUNNER #include "catch.hpp" -Diags *diags = new CatchDiags; extern void ts_session_protocol_well_known_name_indices_init(); int main(int argc, char *argv[]) { + // Set the global diags variable + diags = new CatchDiags; + // Global data initialization needed for the unit tests. ts_session_protocol_well_known_name_indices_init(); // Cheat for ts_host_res_global_init as there's no records.config to check for non-default. diff --git a/lib/yamlcpp/.gitignore b/lib/yamlcpp/.gitignore index 567609b1234..0e29154b528 100644 --- a/lib/yamlcpp/.gitignore +++ b/lib/yamlcpp/.gitignore @@ -1 +1,2 @@ build/ +/tags diff --git a/lib/yamlcpp/CMakeLists.txt b/lib/yamlcpp/CMakeLists.txt index d2d88102880..4732a453f10 100644 --- a/lib/yamlcpp/CMakeLists.txt +++ b/lib/yamlcpp/CMakeLists.txt @@ -1,38 +1,15 @@ ### ### CMake settings ### -## Due to Mac OSX we need to keep compatibility with CMake 2.6 # see http://www.cmake.org/Wiki/CMake_Policies -cmake_minimum_required(VERSION 2.6) -# see http://www.cmake.org/cmake/help/cmake-2-8-docs.html#policy:CMP0012 -if(POLICY CMP0012) - cmake_policy(SET CMP0012 OLD) -endif() -# see http://www.cmake.org/cmake/help/cmake-2-8-docs.html#policy:CMP0015 -if(POLICY CMP0015) - cmake_policy(SET CMP0015 OLD) -endif() -# see https://cmake.org/cmake/help/latest/policy/CMP0042.html -if(POLICY CMP0042) - # Enable MACOSX_RPATH by default. - cmake_policy(SET CMP0042 NEW) -endif() +cmake_minimum_required(VERSION 3.1) include(CheckCXXCompilerFlag) - ### ### Project settings ### -project(YAML_CPP) - -set(YAML_CPP_VERSION_MAJOR "0") -set(YAML_CPP_VERSION_MINOR "6") -set(YAML_CPP_VERSION_PATCH "2") -set(YAML_CPP_VERSION "${YAML_CPP_VERSION_MAJOR}.${YAML_CPP_VERSION_MINOR}.${YAML_CPP_VERSION_PATCH}") - -enable_testing() - +project(YAML_CPP VERSION 0.6.3) ### ### Project options @@ -41,21 +18,26 @@ enable_testing() option(YAML_CPP_BUILD_TESTS "Enable testing" ON) option(YAML_CPP_BUILD_TOOLS "Enable parse tools" ON) option(YAML_CPP_BUILD_CONTRIB "Enable contrib stuff in library" ON) +option(YAML_CPP_INSTALL "Enable generation of install target" ON) ## Build options # --> General # see http://www.cmake.org/cmake/help/cmake2.6docs.html#variable:BUILD_SHARED_LIBS # http://www.cmake.org/cmake/help/cmake2.6docs.html#command:add_library -option(BUILD_SHARED_LIBS "Build Shared Libraries" OFF) +option(YAML_BUILD_SHARED_LIBS "Build Shared Libraries" OFF) # --> Apple -option(APPLE_UNIVERSAL_BIN "Apple: Build universal binary" OFF) +if(APPLE) + option(YAML_APPLE_UNIVERSAL_BIN "Apple: Build universal binary" OFF) +endif() # --> Microsoft Visual C++ # see http://msdn.microsoft.com/en-us/library/aa278396(v=VS.60).aspx # http://msdn.microsoft.com/en-us/library/2kzt1wy3(v=VS.71).aspx -option(MSVC_SHARED_RT "MSVC: Build with shared runtime libs (/MD)" ON) -option(MSVC_STHREADED_RT "MSVC: Build with single-threaded static runtime libs (/ML until VS .NET 2003)" OFF) +if(MSVC) + option(YAML_MSVC_SHARED_RT "MSVC: Build with shared runtime libs (/MD)" ON) + option(YAML_MSVC_STHREADED_RT "MSVC: Build with single-threaded static runtime libs (/ML until VS .NET 2003)" OFF) +endif() ### ### Sources, headers, directories and libs @@ -116,9 +98,10 @@ if(VERBOSE) message(STATUS "contrib_private_headers: ${contrib_private_headers}") endif() -include_directories(${YAML_CPP_SOURCE_DIR}/src) -include_directories(${YAML_CPP_SOURCE_DIR}/include) - +if (CMAKE_VERSION VERSION_LESS 2.8.12) + include_directories(${YAML_CPP_SOURCE_DIR}/src) + include_directories(${YAML_CPP_SOURCE_DIR}/include) +endif() ### @@ -127,14 +110,14 @@ include_directories(${YAML_CPP_SOURCE_DIR}/include) set(yaml_c_flags ${CMAKE_C_FLAGS}) set(yaml_cxx_flags ${CMAKE_CXX_FLAGS}) -if(BUILD_SHARED_LIBS) +if(YAML_BUILD_SHARED_LIBS) set(LABEL_SUFFIX "shared") else() set(LABEL_SUFFIX "static") endif() if(APPLE) - if(APPLE_UNIVERSAL_BIN) + if(YAML_APPLE_UNIVERSAL_BIN) set(CMAKE_OSX_ARCHITECTURES ppc;i386) endif() endif() @@ -145,7 +128,7 @@ if(IPHONE) endif() if(WIN32) - if(BUILD_SHARED_LIBS) + if(YAML_BUILD_SHARED_LIBS) add_definitions(-D${PROJECT_NAME}_DLL) # use or build Windows DLL endif() if(CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT) @@ -155,7 +138,7 @@ endif() # GCC or Clang or Intel Compiler specialities if(CMAKE_CXX_COMPILER_ID MATCHES "GNU" OR - CMAKE_CXX_COMPILER_ID MATCHES "Clang" OR + (CMAKE_CXX_COMPILER_ID MATCHES "Clang" AND NOT "x${CMAKE_CXX_SIMULATE_ID}" STREQUAL "xMSVC") OR CMAKE_CXX_COMPILER_ID MATCHES "Intel") ### General stuff @@ -170,32 +153,27 @@ if(CMAKE_CXX_COMPILER_ID MATCHES "GNU" OR set(CMAKE_BUILD_TYPE Release) endif() # - set(CMAKE_CXX_FLAGS_RELEASE "-O2") - set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "-O2 -g") - set(CMAKE_CXX_FLAGS_DEBUG "-g") - set(CMAKE_CXX_FLAGS_MINSIZEREL "-Os") - # set(GCC_EXTRA_OPTIONS "") # - if(BUILD_SHARED_LIBS) + if(YAML_BUILD_SHARED_LIBS) set(GCC_EXTRA_OPTIONS "${GCC_EXTRA_OPTIONS} -fPIC") endif() # - set(FLAG_TESTED "-Wextra") + set(FLAG_TESTED "-Wextra -Wshadow -Weffc++ -pedantic -pedantic-errors") check_cxx_compiler_flag(${FLAG_TESTED} FLAG_WEXTRA) if(FLAG_WEXTRA) set(GCC_EXTRA_OPTIONS "${GCC_EXTRA_OPTIONS} ${FLAG_TESTED}") endif() # - set(yaml_cxx_flags "-Wall ${GCC_EXTRA_OPTIONS} -pedantic -Wno-long-long -std=c++11 ${yaml_cxx_flags}") + set(yaml_cxx_flags "-Wall ${GCC_EXTRA_OPTIONS} -pedantic -Wno-long-long ${yaml_cxx_flags}") ### Make specific if(${CMAKE_BUILD_TOOL} MATCHES make OR ${CMAKE_BUILD_TOOL} MATCHES gmake) - add_custom_target(debuggable $(MAKE) clean + add_custom_target(debuggable ${CMAKE_MAKE_PROGRAM} clean COMMAND ${CMAKE_COMMAND} -DCMAKE_BUILD_TYPE=Debug ${CMAKE_SOURCE_DIR} COMMENT "Adjusting settings for debug compilation" VERBATIM) - add_custom_target(releasable $(MAKE) clean + add_custom_target(releasable ${CMAKE_MAKE_PROGRAM} clean COMMAND ${CMAKE_COMMAND} -DCMAKE_BUILD_TYPE=Release ${CMAKE_SOURCE_DIR} COMMENT "Adjusting settings for release compilation" VERBATIM) @@ -212,8 +190,8 @@ if(MSVC) set(LIB_RT_SUFFIX "md") # CMake defaults to /MD for MSVC set(LIB_RT_OPTION "/MD") # - if(NOT MSVC_SHARED_RT) # User wants to have static runtime libraries (/MT, /ML) - if(MSVC_STHREADED_RT) # User wants to have old single-threaded static runtime libraries + if(NOT YAML_MSVC_SHARED_RT) # User wants to have static runtime libraries (/MT, /ML) + if(YAML_MSVC_STHREADED_RT) # User wants to have old single-threaded static runtime libraries set(LIB_RT_SUFFIX "ml") set(LIB_RT_OPTION "/ML") if(NOT ${MSVC_VERSION} LESS 1400) @@ -243,7 +221,7 @@ if(MSVC) set(CMAKE_STATIC_LIBRARY_PREFIX "lib") # to distinguish static libraries from DLL import libs # c) Correct suffixes for static libraries - if(NOT BUILD_SHARED_LIBS) + if(NOT YAML_BUILD_SHARED_LIBS) ### General stuff set(LIB_TARGET_SUFFIX "${LIB_SUFFIX}${LIB_RT_SUFFIX}") endif() @@ -274,7 +252,24 @@ set(_INSTALL_DESTINATIONS ### ### Library ### -add_library(yaml-cpp ${library_sources}) +if(YAML_BUILD_SHARED_LIBS) + add_library(yaml-cpp SHARED ${library_sources}) +else() + add_library(yaml-cpp STATIC ${library_sources}) +endif() + +if (NOT CMAKE_VERSION VERSION_LESS 2.8.12) + target_include_directories(yaml-cpp + PUBLIC $ + $ + PRIVATE $) +endif() + +set_target_properties(yaml-cpp PROPERTIES + CXX_STANDARD 11 + CXX_STANDARD_REQUIRED ON +) + set_target_properties(yaml-cpp PROPERTIES COMPILE_FLAGS "${yaml_c_flags} ${yaml_cxx_flags}" ) @@ -292,7 +287,7 @@ if(IPHONE) endif() if(MSVC) - if(NOT BUILD_SHARED_LIBS) + if(NOT YAML_BUILD_SHARED_LIBS) # correct library names set_target_properties(yaml-cpp PROPERTIES DEBUG_POSTFIX "${LIB_TARGET_SUFFIX}d" @@ -303,12 +298,14 @@ if(MSVC) endif() endif() -install(TARGETS yaml-cpp EXPORT yaml-cpp-targets ${_INSTALL_DESTINATIONS}) -install( - DIRECTORY ${header_directory} - DESTINATION ${INCLUDE_INSTALL_DIR} - FILES_MATCHING PATTERN "*.h" -) +if (YAML_CPP_INSTALL) + install(TARGETS yaml-cpp EXPORT yaml-cpp-targets ${_INSTALL_DESTINATIONS}) + install( + DIRECTORY ${header_directory} + DESTINATION ${INCLUDE_INSTALL_DIR} + FILES_MATCHING PATTERN "*.h" + ) +endif() export( TARGETS yaml-cpp @@ -326,6 +323,7 @@ else() set(INSTALL_CMAKE_DIR ${LIB_INSTALL_DIR}/cmake/yaml-cpp) endif() + file(RELATIVE_PATH REL_INCLUDE_DIR "${CMAKE_INSTALL_PREFIX}/${INSTALL_CMAKE_DIR}" "${CMAKE_INSTALL_PREFIX}/${INCLUDE_INSTALL_ROOT_DIR}") set(CONFIG_INCLUDE_DIRS "\${YAML_CPP_CMAKE_DIR}/${REL_INCLUDE_DIR}") configure_file(${CMAKE_CURRENT_SOURCE_DIR}/yaml-cpp-config.cmake.in @@ -334,16 +332,19 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/yaml-cpp-config.cmake.in configure_file(${CMAKE_CURRENT_SOURCE_DIR}/yaml-cpp-config-version.cmake.in "${PROJECT_BINARY_DIR}/yaml-cpp-config-version.cmake" @ONLY) -install(FILES - "${PROJECT_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/yaml-cpp-config.cmake" - "${PROJECT_BINARY_DIR}/yaml-cpp-config-version.cmake" - DESTINATION "${INSTALL_CMAKE_DIR}" COMPONENT dev) -install(EXPORT yaml-cpp-targets DESTINATION ${INSTALL_CMAKE_DIR}) +if (YAML_CPP_INSTALL) + install(FILES + "${PROJECT_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/yaml-cpp-config.cmake" + "${PROJECT_BINARY_DIR}/yaml-cpp-config-version.cmake" + DESTINATION "${INSTALL_CMAKE_DIR}" COMPONENT dev) + install(EXPORT yaml-cpp-targets DESTINATION ${INSTALL_CMAKE_DIR}) + + if(UNIX) + set(PC_FILE ${CMAKE_BINARY_DIR}/yaml-cpp.pc) + configure_file("yaml-cpp.pc.cmake" ${PC_FILE} @ONLY) + install(FILES ${PC_FILE} DESTINATION ${LIB_INSTALL_DIR}/pkgconfig) + endif() -if(UNIX) - set(PC_FILE ${CMAKE_BINARY_DIR}/yaml-cpp.pc) - configure_file("yaml-cpp.pc.cmake" ${PC_FILE} @ONLY) - install(FILES ${PC_FILE} DESTINATION ${LIB_INSTALL_DIR}/pkgconfig) endif() @@ -351,6 +352,7 @@ endif() ### Extras ### if(YAML_CPP_BUILD_TESTS) + enable_testing() add_subdirectory(test) endif() if(YAML_CPP_BUILD_TOOLS) diff --git a/lib/yamlcpp/Makefile.am b/lib/yamlcpp/Makefile.am index a451416b7c8..0bb0ab01fcb 100644 --- a/lib/yamlcpp/Makefile.am +++ b/lib/yamlcpp/Makefile.am @@ -17,8 +17,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -AM_CPPFLAGS += \ - -I$(abs_top_srcdir)/lib/yamlcpp/include +AM_CPPFLAGS += -I$(abs_top_srcdir)/lib/yamlcpp/include noinst_LTLIBRARIES = libyamlcpp.la @@ -50,3 +49,17 @@ src/simplekey.cpp \ src/singledocparser.cpp \ src/stream.cpp \ src/tag.cpp + + +yamlcpp_includedir=$(includedir)/yaml-cpp +yamlcpp_include_HEADERS = \ + $(srcdir)/include/yaml-cpp/*.h + +yamlcpp_node_includedir=$(includedir)/yaml-cpp/node +yamlcpp_node_include_HEADERS = \ + $(srcdir)/include/yaml-cpp/node/*.h + +yamlcpp_node_detail_includedir=$(includedir)/yaml-cpp/node/detail +yamlcpp_node_detail_include_HEADERS = \ + $(srcdir)/include/yaml-cpp/node/detail/*.h + diff --git a/lib/yamlcpp/README.md b/lib/yamlcpp/README.md index f33d3503a15..e793143af92 100644 --- a/lib/yamlcpp/README.md +++ b/lib/yamlcpp/README.md @@ -26,7 +26,7 @@ cd build 3. Run CMake. The basic syntax is: ``` -cmake [-G generator] [-DBUILD_SHARED_LIBS=ON|OFF] .. +cmake [-G generator] [-DYAML_BUILD_SHARED_LIBS=ON|OFF] .. ``` * The `generator` is whatever type of build system you'd like to use. To see a full list of generators on your platform, just run `cmake` (with no arguments). For example: @@ -34,7 +34,7 @@ cmake [-G generator] [-DBUILD_SHARED_LIBS=ON|OFF] .. * On OS X, you might use "Xcode" to generate an Xcode project * On a UNIX-y system, simply omit the option to generate a makefile - * yaml-cpp defaults to building a static library, but you may build a shared library by specifying `-DBUILD_SHARED_LIBS=ON`. + * yaml-cpp defaults to building a static library, but you may build a shared library by specifying `-DYAML_BUILD_SHARED_LIBS=ON`. * For more options on customizing the build, see the [CMakeLists.txt](https://github.com/jbeder/yaml-cpp/blob/master/CMakeLists.txt) file. diff --git a/lib/yamlcpp/include/yaml-cpp/binary.h b/lib/yamlcpp/include/yaml-cpp/binary.h index 29d5dbd027a..1050dae98c3 100644 --- a/lib/yamlcpp/include/yaml-cpp/binary.h +++ b/lib/yamlcpp/include/yaml-cpp/binary.h @@ -19,9 +19,13 @@ YAML_CPP_API std::vector DecodeBase64(const std::string &input); class YAML_CPP_API Binary { public: - Binary() : m_unownedData(0), m_unownedSize(0) {} Binary(const unsigned char *data_, std::size_t size_) - : m_unownedData(data_), m_unownedSize(size_) {} + : m_data{}, m_unownedData(data_), m_unownedSize(size_) {} + Binary() : Binary(nullptr, 0) {} + Binary(const Binary &) = default; + Binary(Binary &&) = default; + Binary &operator=(const Binary &) = default; + Binary &operator=(Binary &&) = default; bool owned() const { return !m_unownedData; } std::size_t size() const { return owned() ? m_data.size() : m_unownedSize; } @@ -35,7 +39,7 @@ class YAML_CPP_API Binary { rhs.clear(); rhs.resize(m_unownedSize); std::copy(m_unownedData, m_unownedData + m_unownedSize, rhs.begin()); - m_unownedData = 0; + m_unownedData = nullptr; m_unownedSize = 0; } else { m_data.swap(rhs); @@ -62,6 +66,6 @@ class YAML_CPP_API Binary { const unsigned char *m_unownedData; std::size_t m_unownedSize; }; -} +} // namespace YAML #endif // BASE64_H_62B23520_7C8E_11DE_8A39_0800200C9A66 diff --git a/lib/yamlcpp/include/yaml-cpp/contrib/anchordict.h b/lib/yamlcpp/include/yaml-cpp/contrib/anchordict.h index 78db9ec9288..1b7809b875f 100644 --- a/lib/yamlcpp/include/yaml-cpp/contrib/anchordict.h +++ b/lib/yamlcpp/include/yaml-cpp/contrib/anchordict.h @@ -22,6 +22,7 @@ namespace YAML { template class AnchorDict { public: + AnchorDict() : m_data{} {} void Register(anchor_t anchor, T value) { if (anchor > m_data.size()) { m_data.resize(anchor); @@ -34,6 +35,6 @@ class AnchorDict { private: std::vector m_data; }; -} +} // namespace YAML #endif // ANCHORDICT_H_62B23520_7C8E_11DE_8A39_0800200C9A66 diff --git a/lib/yamlcpp/include/yaml-cpp/emitter.h b/lib/yamlcpp/include/yaml-cpp/emitter.h index ef92cc4035b..6f142b01160 100644 --- a/lib/yamlcpp/include/yaml-cpp/emitter.h +++ b/lib/yamlcpp/include/yaml-cpp/emitter.h @@ -7,16 +7,18 @@ #pragma once #endif +#include #include +#include #include #include #include +#include #include "yaml-cpp/binary.h" #include "yaml-cpp/dll.h" #include "yaml-cpp/emitterdef.h" #include "yaml-cpp/emittermanip.h" -#include "yaml-cpp/noncopyable.h" #include "yaml-cpp/null.h" #include "yaml-cpp/ostream_wrapper.h" @@ -28,10 +30,12 @@ struct _Null; namespace YAML { class EmitterState; -class YAML_CPP_API Emitter : private noncopyable { +class YAML_CPP_API Emitter { public: Emitter(); explicit Emitter(std::ostream& stream); + Emitter(const Emitter&) = delete; + Emitter& operator=(const Emitter&) = delete; ~Emitter(); // output @@ -152,7 +156,28 @@ inline Emitter& Emitter::WriteStreamable(T value) { std::stringstream stream; SetStreamablePrecision(stream); - stream << value; + + bool special = false; + if (std::is_floating_point::value) { + if ((std::numeric_limits::has_quiet_NaN || + std::numeric_limits::has_signaling_NaN) && + std::isnan(value)) { + special = true; + stream << ".nan"; + } else if (std::numeric_limits::has_infinity) { + if (value == std::numeric_limits::infinity()) { + special = true; + stream << ".inf"; + } else if (value == -std::numeric_limits::infinity()) { + special = true; + stream << "-.inf"; + } + } + } + + if (!special) { + stream << value; + } m_stream << stream.str(); StartedScalar(); @@ -249,6 +274,6 @@ inline Emitter& operator<<(Emitter& emitter, _Indent indent) { inline Emitter& operator<<(Emitter& emitter, _Precision precision) { return emitter.SetLocalPrecision(precision); } -} +} // namespace YAML #endif // EMITTER_H_62B23520_7C8E_11DE_8A39_0800200C9A66 diff --git a/lib/yamlcpp/include/yaml-cpp/eventhandler.h b/lib/yamlcpp/include/yaml-cpp/eventhandler.h index efe381c6218..29718ffa1f9 100644 --- a/lib/yamlcpp/include/yaml-cpp/eventhandler.h +++ b/lib/yamlcpp/include/yaml-cpp/eventhandler.h @@ -34,7 +34,12 @@ class EventHandler { virtual void OnMapStart(const Mark& mark, const std::string& tag, anchor_t anchor, EmitterStyle::value style) = 0; virtual void OnMapEnd() = 0; + + virtual void OnAnchor(const Mark& /*mark*/, + const std::string& /*anchor_name*/) { + // empty default implementation for compatibility + } }; -} +} // namespace YAML #endif // EVENTHANDLER_H_62B23520_7C8E_11DE_8A39_0800200C9A66 diff --git a/lib/yamlcpp/include/yaml-cpp/exceptions.h b/lib/yamlcpp/include/yaml-cpp/exceptions.h index 9c96859b2c9..eef22833a81 100644 --- a/lib/yamlcpp/include/yaml-cpp/exceptions.h +++ b/lib/yamlcpp/include/yaml-cpp/exceptions.h @@ -15,7 +15,7 @@ // This is here for compatibility with older versions of Visual Studio // which don't support noexcept -#ifdef _MSC_VER +#if defined(_MSC_VER) && _MSC_VER < 1900 #define YAML_CPP_NOEXCEPT _NOEXCEPT #else #define YAML_CPP_NOEXCEPT noexcept @@ -114,6 +114,35 @@ inline const std::string KEY_NOT_FOUND_WITH_KEY( stream << KEY_NOT_FOUND << ": " << key; return stream.str(); } + +template +inline const std::string BAD_SUBSCRIPT_WITH_KEY( + const T&, typename disable_if>::type* = nullptr) { + return BAD_SUBSCRIPT; +} + +inline const std::string BAD_SUBSCRIPT_WITH_KEY(const std::string& key) { + std::stringstream stream; + stream << BAD_SUBSCRIPT << " (key: \"" << key << "\")"; + return stream.str(); +} + +template +inline const std::string BAD_SUBSCRIPT_WITH_KEY( + const T& key, typename enable_if>::type* = nullptr) { + std::stringstream stream; + stream << BAD_SUBSCRIPT << " (key: \"" << key << "\")"; + return stream.str(); +} + +inline const std::string INVALID_NODE_WITH_KEY(const std::string& key) { + std::stringstream stream; + if (key.empty()) { + return INVALID_NODE; + } + stream << "invalid node; first invalid key: \"" << key << "\""; + return stream.str(); +} } class YAML_CPP_API Exception : public std::runtime_error { @@ -131,7 +160,7 @@ class YAML_CPP_API Exception : public std::runtime_error { static const std::string build_what(const Mark& mark, const std::string& msg) { if (mark.is_null()) { - return msg.c_str(); + return msg; } std::stringstream output; @@ -194,8 +223,9 @@ inline TypedKeyNotFound MakeTypedKeyNotFound(const Mark& mark, class YAML_CPP_API InvalidNode : public RepresentationException { public: - InvalidNode() - : RepresentationException(Mark::null_mark(), ErrorMsg::INVALID_NODE) {} + InvalidNode(std::string key) + : RepresentationException(Mark::null_mark(), + ErrorMsg::INVALID_NODE_WITH_KEY(key)) {} InvalidNode(const InvalidNode&) = default; virtual ~InvalidNode() YAML_CPP_NOEXCEPT; }; @@ -224,8 +254,10 @@ class YAML_CPP_API BadDereference : public RepresentationException { class YAML_CPP_API BadSubscript : public RepresentationException { public: - BadSubscript() - : RepresentationException(Mark::null_mark(), ErrorMsg::BAD_SUBSCRIPT) {} + template + BadSubscript(const Key& key) + : RepresentationException(Mark::null_mark(), + ErrorMsg::BAD_SUBSCRIPT_WITH_KEY(key)) {} BadSubscript(const BadSubscript&) = default; virtual ~BadSubscript() YAML_CPP_NOEXCEPT; }; diff --git a/lib/yamlcpp/include/yaml-cpp/node/convert.h b/lib/yamlcpp/include/yaml-cpp/node/convert.h index 45a878ab0c0..d61b73d09db 100644 --- a/lib/yamlcpp/include/yaml-cpp/node/convert.h +++ b/lib/yamlcpp/include/yaml-cpp/node/convert.h @@ -93,7 +93,7 @@ struct convert<_Null> { struct convert { \ static Node encode(const type& rhs) { \ std::stringstream stream; \ - stream.precision(std::numeric_limits::digits10 + 1); \ + stream.precision(std::numeric_limits::max_digits10); \ stream << rhs; \ return Node(stream.str()); \ } \ @@ -116,10 +116,11 @@ struct convert<_Null> { } \ } \ \ - if (std::numeric_limits::has_quiet_NaN && \ - conversion::IsNaN(input)) { \ - rhs = std::numeric_limits::quiet_NaN(); \ - return true; \ + if (std::numeric_limits::has_quiet_NaN) { \ + if (conversion::IsNaN(input)) { \ + rhs = std::numeric_limits::quiet_NaN(); \ + return true; \ + } \ } \ \ return false; \ diff --git a/lib/yamlcpp/include/yaml-cpp/node/detail/impl.h b/lib/yamlcpp/include/yaml-cpp/node/detail/impl.h index 09e55f838c2..4123b85da54 100644 --- a/lib/yamlcpp/include/yaml-cpp/node/detail/impl.h +++ b/lib/yamlcpp/include/yaml-cpp/node/detail/impl.h @@ -17,7 +17,7 @@ template struct get_idx { static node* get(const std::vector& /* sequence */, const Key& /* key */, shared_memory_holder /* pMemory */) { - return 0; + return nullptr; } }; @@ -27,12 +27,12 @@ struct get_idx::value>::type> { static node* get(const std::vector& sequence, const Key& key, shared_memory_holder /* pMemory */) { - return key < sequence.size() ? sequence[key] : 0; + return key < sequence.size() ? sequence[key] : nullptr; } static node* get(std::vector& sequence, const Key& key, shared_memory_holder pMemory) { - if (key > sequence.size() || (key > 0 && !sequence[key-1]->is_defined())) + if (key > sequence.size() || (key > 0 && !sequence[key - 1]->is_defined())) return 0; if (key == sequence.size()) sequence.push_back(&pMemory->create_node()); @@ -46,13 +46,44 @@ struct get_idx::value>::type> { shared_memory_holder pMemory) { return key >= 0 ? get_idx::get( sequence, static_cast(key), pMemory) - : 0; + : nullptr; } static node* get(std::vector& sequence, const Key& key, shared_memory_holder pMemory) { return key >= 0 ? get_idx::get( sequence, static_cast(key), pMemory) - : 0; + : nullptr; + } +}; + +template +struct remove_idx { + static bool remove(std::vector&, const Key&) { return false; } +}; + +template +struct remove_idx< + Key, typename std::enable_if::value && + !std::is_same::value>::type> { + + static bool remove(std::vector& sequence, const Key& key) { + if (key >= sequence.size()) { + return false; + } else { + sequence.erase(sequence.begin() + key); + return true; + } + } +}; + +template +struct remove_idx::value>::type> { + + static bool remove(std::vector& sequence, const Key& key) { + return key >= 0 ? remove_idx::remove( + sequence, static_cast(key)) + : false; } }; @@ -78,13 +109,13 @@ inline node* node_data::get(const Key& key, break; case NodeType::Undefined: case NodeType::Null: - return NULL; + return nullptr; case NodeType::Sequence: if (node* pNode = get_idx::get(m_sequence, key, pMemory)) return pNode; - return NULL; + return nullptr; case NodeType::Scalar: - throw BadSubscript(); + throw BadSubscript(key); } for (node_map::const_iterator it = m_map.begin(); it != m_map.end(); ++it) { @@ -93,7 +124,7 @@ inline node* node_data::get(const Key& key, } } - return NULL; + return nullptr; } template @@ -112,7 +143,7 @@ inline node& node_data::get(const Key& key, shared_memory_holder pMemory) { convert_to_map(pMemory); break; case NodeType::Scalar: - throw BadSubscript(); + throw BadSubscript(key); } for (node_map::const_iterator it = m_map.begin(); it != m_map.end(); ++it) { @@ -129,21 +160,23 @@ inline node& node_data::get(const Key& key, shared_memory_holder pMemory) { template inline bool node_data::remove(const Key& key, shared_memory_holder pMemory) { - if (m_type != NodeType::Map) - return false; - - for (kv_pairs::iterator it = m_undefinedPairs.begin(); - it != m_undefinedPairs.end();) { - kv_pairs::iterator jt = std::next(it); - if (it->first->equals(key, pMemory)) - m_undefinedPairs.erase(it); - it = jt; - } + if (m_type == NodeType::Sequence) { + return remove_idx::remove(m_sequence, key); + } else if (m_type == NodeType::Map) { + kv_pairs::iterator it = m_undefinedPairs.begin(); + while (it != m_undefinedPairs.end()) { + kv_pairs::iterator jt = std::next(it); + if (it->first->equals(key, pMemory)) { + m_undefinedPairs.erase(it); + } + it = jt; + } - for (node_map::iterator it = m_map.begin(); it != m_map.end(); ++it) { - if (it->first->equals(key, pMemory)) { - m_map.erase(it); - return true; + for (node_map::iterator iter = m_map.begin(); iter != m_map.end(); ++iter) { + if (iter->first->equals(key, pMemory)) { + m_map.erase(iter); + return true; + } } } diff --git a/lib/yamlcpp/include/yaml-cpp/node/detail/iterator.h b/lib/yamlcpp/include/yaml-cpp/node/detail/iterator.h index deec8fb62cd..966107d959d 100644 --- a/lib/yamlcpp/include/yaml-cpp/node/detail/iterator.h +++ b/lib/yamlcpp/include/yaml-cpp/node/detail/iterator.h @@ -8,19 +8,19 @@ #endif #include "yaml-cpp/dll.h" +#include "yaml-cpp/node/detail/node_iterator.h" #include "yaml-cpp/node/node.h" #include "yaml-cpp/node/ptr.h" -#include "yaml-cpp/node/detail/node_iterator.h" #include #include + namespace YAML { namespace detail { struct iterator_value; template -class iterator_base : public std::iterator { +class iterator_base { private: template @@ -37,7 +37,11 @@ class iterator_base : public std::iterator namespace YAML { namespace detail { class node { public: - node() : m_pRef(new node_ref) {} + node() : m_pRef(new node_ref), m_dependencies{} {} node(const node&) = delete; node& operator=(const node&) = delete; @@ -163,7 +163,7 @@ class node { typedef std::set nodes; nodes m_dependencies; }; -} -} +} // namespace detail +} // namespace YAML #endif // NODE_DETAIL_NODE_H_62B23520_7C8E_11DE_8A39_0800200C9A66 diff --git a/lib/yamlcpp/include/yaml-cpp/node/detail/node_data.h b/lib/yamlcpp/include/yaml-cpp/node/detail/node_data.h index 50bcd74352d..82fb79adef9 100644 --- a/lib/yamlcpp/include/yaml-cpp/node/detail/node_data.h +++ b/lib/yamlcpp/include/yaml-cpp/node/detail/node_data.h @@ -81,7 +81,7 @@ class YAML_CPP_API node_data { shared_memory_holder pMemory); public: - static std::string empty_scalar; + static const std::string& empty_scalar(); private: void compute_seq_size() const; diff --git a/lib/yamlcpp/include/yaml-cpp/node/detail/node_iterator.h b/lib/yamlcpp/include/yaml-cpp/node/detail/node_iterator.h index 088090fe743..ab6916fea47 100644 --- a/lib/yamlcpp/include/yaml-cpp/node/detail/node_iterator.h +++ b/lib/yamlcpp/include/yaml-cpp/node/detail/node_iterator.h @@ -26,9 +26,9 @@ template struct node_iterator_value : public std::pair { typedef std::pair kv; - node_iterator_value() : kv(), pNode(0) {} + node_iterator_value() : kv(), pNode(nullptr) {} explicit node_iterator_value(V& rhs) : kv(), pNode(&rhs) {} - explicit node_iterator_value(V& key, V& value) : kv(&key, &value), pNode(0) {} + explicit node_iterator_value(V& key, V& value) : kv(&key, &value), pNode(nullptr) {} V& operator*() const { return *pNode; } V& operator->() const { return *pNode; } diff --git a/lib/yamlcpp/include/yaml-cpp/node/impl.h b/lib/yamlcpp/include/yaml-cpp/node/impl.h index 20c487a687f..7a3deacf906 100644 --- a/lib/yamlcpp/include/yaml-cpp/node/impl.h +++ b/lib/yamlcpp/include/yaml-cpp/node/impl.h @@ -7,18 +7,21 @@ #pragma once #endif -#include "yaml-cpp/node/node.h" -#include "yaml-cpp/node/iterator.h" +#include "yaml-cpp/exceptions.h" #include "yaml-cpp/node/detail/memory.h" #include "yaml-cpp/node/detail/node.h" -#include "yaml-cpp/exceptions.h" +#include "yaml-cpp/node/iterator.h" +#include "yaml-cpp/node/node.h" +#include #include namespace YAML { -inline Node::Node() : m_isValid(true), m_pNode(NULL) {} +inline Node::Node() + : m_isValid(true), m_invalidKey{}, m_pMemory(nullptr), m_pNode(nullptr) {} inline Node::Node(NodeType::value type) : m_isValid(true), + m_invalidKey{}, m_pMemory(new detail::memory_holder), m_pNode(&m_pMemory->create_node()) { m_pNode->set_type(type); @@ -27,6 +30,7 @@ inline Node::Node(NodeType::value type) template inline Node::Node(const T& rhs) : m_isValid(true), + m_invalidKey{}, m_pMemory(new detail::memory_holder), m_pNode(&m_pMemory->create_node()) { Assign(rhs); @@ -34,24 +38,30 @@ inline Node::Node(const T& rhs) inline Node::Node(const detail::iterator_value& rhs) : m_isValid(rhs.m_isValid), + m_invalidKey(rhs.m_invalidKey), m_pMemory(rhs.m_pMemory), m_pNode(rhs.m_pNode) {} inline Node::Node(const Node& rhs) : m_isValid(rhs.m_isValid), + m_invalidKey(rhs.m_invalidKey), m_pMemory(rhs.m_pMemory), m_pNode(rhs.m_pNode) {} -inline Node::Node(Zombie) : m_isValid(false), m_pNode(NULL) {} +inline Node::Node(Zombie) + : m_isValid(false), m_invalidKey{}, m_pMemory{}, m_pNode(nullptr) {} + +inline Node::Node(Zombie, const std::string& key) + : m_isValid(false), m_invalidKey(key), m_pMemory{}, m_pNode(nullptr) {} inline Node::Node(detail::node& node, detail::shared_memory_holder pMemory) - : m_isValid(true), m_pMemory(pMemory), m_pNode(&node) {} + : m_isValid(true), m_invalidKey{}, m_pMemory(pMemory), m_pNode(&node) {} inline Node::~Node() {} inline void Node::EnsureNodeExists() const { if (!m_isValid) - throw InvalidNode(); + throw InvalidNode(m_invalidKey); if (!m_pNode) { m_pMemory.reset(new detail::memory_holder); m_pNode = &m_pMemory->create_node(); @@ -68,14 +78,14 @@ inline bool Node::IsDefined() const { inline Mark Node::Mark() const { if (!m_isValid) { - throw InvalidNode(); + throw InvalidNode(m_invalidKey); } return m_pNode ? m_pNode->mark() : Mark::null_mark(); } inline NodeType::value Node::Type() const { if (!m_isValid) - throw InvalidNode(); + throw InvalidNode(m_invalidKey); return m_pNode ? m_pNode->type() : NodeType::Null; } @@ -142,7 +152,7 @@ struct as_if { template inline T Node::as() const { if (!m_isValid) - throw InvalidNode(); + throw InvalidNode(m_invalidKey); return as_if(*this)(); } @@ -155,32 +165,32 @@ inline T Node::as(const S& fallback) const { inline const std::string& Node::Scalar() const { if (!m_isValid) - throw InvalidNode(); - return m_pNode ? m_pNode->scalar() : detail::node_data::empty_scalar; + throw InvalidNode(m_invalidKey); + return m_pNode ? m_pNode->scalar() : detail::node_data::empty_scalar(); } inline const std::string& Node::Tag() const { if (!m_isValid) - throw InvalidNode(); - return m_pNode ? m_pNode->tag() : detail::node_data::empty_scalar; + throw InvalidNode(m_invalidKey); + return m_pNode ? m_pNode->tag() : detail::node_data::empty_scalar(); } inline void Node::SetTag(const std::string& tag) { if (!m_isValid) - throw InvalidNode(); + throw InvalidNode(m_invalidKey); EnsureNodeExists(); m_pNode->set_tag(tag); } inline EmitterStyle::value Node::Style() const { if (!m_isValid) - throw InvalidNode(); + throw InvalidNode(m_invalidKey); return m_pNode ? m_pNode->style() : EmitterStyle::Default; } inline void Node::SetStyle(EmitterStyle::value style) { if (!m_isValid) - throw InvalidNode(); + throw InvalidNode(m_invalidKey); EnsureNodeExists(); m_pNode->set_style(style); } @@ -188,7 +198,7 @@ inline void Node::SetStyle(EmitterStyle::value style) { // assignment inline bool Node::is(const Node& rhs) const { if (!m_isValid || !rhs.m_isValid) - throw InvalidNode(); + throw InvalidNode(m_invalidKey); if (!m_pNode || !rhs.m_pNode) return false; return m_pNode->is(*rhs.m_pNode); @@ -197,14 +207,23 @@ inline bool Node::is(const Node& rhs) const { template inline Node& Node::operator=(const T& rhs) { if (!m_isValid) - throw InvalidNode(); + throw InvalidNode(m_invalidKey); Assign(rhs); return *this; } +inline Node& Node::operator=(const Node& rhs) { + if (!m_isValid || !rhs.m_isValid) + throw InvalidNode(m_invalidKey); + if (is(rhs)) + return *this; + AssignNode(rhs); + return *this; +} + inline void Node::reset(const YAML::Node& rhs) { if (!m_isValid || !rhs.m_isValid) - throw InvalidNode(); + throw InvalidNode(m_invalidKey); m_pMemory = rhs.m_pMemory; m_pNode = rhs.m_pNode; } @@ -212,44 +231,35 @@ inline void Node::reset(const YAML::Node& rhs) { template inline void Node::Assign(const T& rhs) { if (!m_isValid) - throw InvalidNode(); + throw InvalidNode(m_invalidKey); AssignData(convert::encode(rhs)); } template <> inline void Node::Assign(const std::string& rhs) { if (!m_isValid) - throw InvalidNode(); + throw InvalidNode(m_invalidKey); EnsureNodeExists(); m_pNode->set_scalar(rhs); } inline void Node::Assign(const char* rhs) { if (!m_isValid) - throw InvalidNode(); + throw InvalidNode(m_invalidKey); EnsureNodeExists(); m_pNode->set_scalar(rhs); } inline void Node::Assign(char* rhs) { if (!m_isValid) - throw InvalidNode(); + throw InvalidNode(m_invalidKey); EnsureNodeExists(); m_pNode->set_scalar(rhs); } -inline Node& Node::operator=(const Node& rhs) { - if (!m_isValid || !rhs.m_isValid) - throw InvalidNode(); - if (is(rhs)) - return *this; - AssignNode(rhs); - return *this; -} - inline void Node::AssignData(const Node& rhs) { if (!m_isValid || !rhs.m_isValid) - throw InvalidNode(); + throw InvalidNode(m_invalidKey); EnsureNodeExists(); rhs.EnsureNodeExists(); @@ -259,7 +269,7 @@ inline void Node::AssignData(const Node& rhs) { inline void Node::AssignNode(const Node& rhs) { if (!m_isValid || !rhs.m_isValid) - throw InvalidNode(); + throw InvalidNode(m_invalidKey); rhs.EnsureNodeExists(); if (!m_pNode) { @@ -276,7 +286,7 @@ inline void Node::AssignNode(const Node& rhs) { // size/iterator inline std::size_t Node::size() const { if (!m_isValid) - throw InvalidNode(); + throw InvalidNode(m_invalidKey); return m_pNode ? m_pNode->size() : 0; } @@ -309,13 +319,13 @@ inline iterator Node::end() { template inline void Node::push_back(const T& rhs) { if (!m_isValid) - throw InvalidNode(); + throw InvalidNode(m_invalidKey); push_back(Node(rhs)); } inline void Node::push_back(const Node& rhs) { if (!m_isValid || !rhs.m_isValid) - throw InvalidNode(); + throw InvalidNode(m_invalidKey); EnsureNodeExists(); rhs.EnsureNodeExists(); @@ -366,18 +376,23 @@ template inline typename to_value_t::return_type to_value(const T& t) { return to_value_t(t)(); } +} // namespace detail + +template +std::string key_to_string(const Key& key) { + return streamable_to_string::value>().impl(key); } // indexing template inline const Node Node::operator[](const Key& key) const { if (!m_isValid) - throw InvalidNode(); + throw InvalidNode(m_invalidKey); EnsureNodeExists(); - detail::node* value = static_cast(*m_pNode) - .get(detail::to_value(key), m_pMemory); + detail::node* value = static_cast(*m_pNode).get( + detail::to_value(key), m_pMemory); if (!value) { - return Node(ZombieNode); + return Node(ZombieNode, key_to_string(key)); } return Node(*value, m_pMemory); } @@ -385,7 +400,7 @@ inline const Node Node::operator[](const Key& key) const { template inline Node Node::operator[](const Key& key) { if (!m_isValid) - throw InvalidNode(); + throw InvalidNode(m_invalidKey); EnsureNodeExists(); detail::node& value = m_pNode->get(detail::to_value(key), m_pMemory); return Node(value, m_pMemory); @@ -394,28 +409,28 @@ inline Node Node::operator[](const Key& key) { template inline bool Node::remove(const Key& key) { if (!m_isValid) - throw InvalidNode(); + throw InvalidNode(m_invalidKey); EnsureNodeExists(); return m_pNode->remove(detail::to_value(key), m_pMemory); } inline const Node Node::operator[](const Node& key) const { if (!m_isValid || !key.m_isValid) - throw InvalidNode(); + throw InvalidNode(m_invalidKey); EnsureNodeExists(); key.EnsureNodeExists(); m_pMemory->merge(*key.m_pMemory); detail::node* value = static_cast(*m_pNode).get(*key.m_pNode, m_pMemory); if (!value) { - return Node(ZombieNode); + return Node(ZombieNode, key_to_string(key)); } return Node(*value, m_pMemory); } inline Node Node::operator[](const Node& key) { if (!m_isValid || !key.m_isValid) - throw InvalidNode(); + throw InvalidNode(m_invalidKey); EnsureNodeExists(); key.EnsureNodeExists(); m_pMemory->merge(*key.m_pMemory); @@ -425,7 +440,7 @@ inline Node Node::operator[](const Node& key) { inline bool Node::remove(const Node& key) { if (!m_isValid || !key.m_isValid) - throw InvalidNode(); + throw InvalidNode(m_invalidKey); EnsureNodeExists(); key.EnsureNodeExists(); return m_pNode->remove(*key.m_pNode, m_pMemory); @@ -435,7 +450,7 @@ inline bool Node::remove(const Node& key) { template inline void Node::force_insert(const Key& key, const Value& value) { if (!m_isValid) - throw InvalidNode(); + throw InvalidNode(m_invalidKey); EnsureNodeExists(); m_pNode->force_insert(detail::to_value(key), detail::to_value(value), m_pMemory); @@ -443,6 +458,6 @@ inline void Node::force_insert(const Key& key, const Value& value) { // free functions inline bool operator==(const Node& lhs, const Node& rhs) { return lhs.is(rhs); } -} +} // namespace YAML #endif // NODE_IMPL_H_62B23520_7C8E_11DE_8A39_0800200C9A66 diff --git a/lib/yamlcpp/include/yaml-cpp/node/node.h b/lib/yamlcpp/include/yaml-cpp/node/node.h index 1ded7d27b72..49af58e5c42 100644 --- a/lib/yamlcpp/include/yaml-cpp/node/node.h +++ b/lib/yamlcpp/include/yaml-cpp/node/node.h @@ -8,6 +8,7 @@ #endif #include +#include #include "yaml-cpp/dll.h" #include "yaml-cpp/emitterstyle.h" @@ -116,6 +117,7 @@ class YAML_CPP_API Node { private: enum Zombie { ZombieNode }; explicit Node(Zombie); + explicit Node(Zombie, const std::string&); explicit Node(detail::node& node, detail::shared_memory_holder pMemory); void EnsureNodeExists() const; @@ -130,6 +132,8 @@ class YAML_CPP_API Node { private: bool m_isValid; + // String representation of invalid key, if the node is invalid. + std::string m_invalidKey; mutable detail::shared_memory_holder m_pMemory; mutable detail::node* m_pNode; }; diff --git a/lib/yamlcpp/include/yaml-cpp/noncopyable.h b/lib/yamlcpp/include/yaml-cpp/noncopyable.h deleted file mode 100644 index a261040739b..00000000000 --- a/lib/yamlcpp/include/yaml-cpp/noncopyable.h +++ /dev/null @@ -1,25 +0,0 @@ -#ifndef NONCOPYABLE_H_62B23520_7C8E_11DE_8A39_0800200C9A66 -#define NONCOPYABLE_H_62B23520_7C8E_11DE_8A39_0800200C9A66 - -#if defined(_MSC_VER) || \ - (defined(__GNUC__) && (__GNUC__ == 3 && __GNUC_MINOR__ >= 4) || \ - (__GNUC__ >= 4)) // GCC supports "pragma once" correctly since 3.4 -#pragma once -#endif - -#include "yaml-cpp/dll.h" - -namespace YAML { -// this is basically boost::noncopyable -class YAML_CPP_API noncopyable { - protected: - noncopyable() {} - ~noncopyable() {} - - private: - noncopyable(const noncopyable&); - const noncopyable& operator=(const noncopyable&); -}; -} - -#endif // NONCOPYABLE_H_62B23520_7C8E_11DE_8A39_0800200C9A66 diff --git a/lib/yamlcpp/include/yaml-cpp/ostream_wrapper.h b/lib/yamlcpp/include/yaml-cpp/ostream_wrapper.h index 09d45f39b78..cf89741d093 100644 --- a/lib/yamlcpp/include/yaml-cpp/ostream_wrapper.h +++ b/lib/yamlcpp/include/yaml-cpp/ostream_wrapper.h @@ -17,6 +17,10 @@ class YAML_CPP_API ostream_wrapper { public: ostream_wrapper(); explicit ostream_wrapper(std::ostream& stream); + ostream_wrapper(const ostream_wrapper&) = delete; + ostream_wrapper(ostream_wrapper&&) = delete; + ostream_wrapper& operator=(const ostream_wrapper&) = delete; + ostream_wrapper& operator=(ostream_wrapper&&) = delete; ~ostream_wrapper(); void write(const std::string& str); @@ -26,7 +30,7 @@ class YAML_CPP_API ostream_wrapper { const char* str() const { if (m_pStream) { - return 0; + return nullptr; } else { m_buffer[m_pos] = '\0'; return &m_buffer[0]; @@ -52,7 +56,7 @@ class YAML_CPP_API ostream_wrapper { template inline ostream_wrapper& operator<<(ostream_wrapper& stream, - const char(&str)[N]) { + const char (&str)[N]) { stream.write(str, N - 1); return stream; } @@ -67,6 +71,6 @@ inline ostream_wrapper& operator<<(ostream_wrapper& stream, char ch) { stream.write(&ch, 1); return stream; } -} +} // namespace YAML #endif // OSTREAM_WRAPPER_H_62B23520_7C8E_11DE_8A39_0800200C9A66 diff --git a/lib/yamlcpp/include/yaml-cpp/parser.h b/lib/yamlcpp/include/yaml-cpp/parser.h index ceac22d0268..2f403c35048 100644 --- a/lib/yamlcpp/include/yaml-cpp/parser.h +++ b/lib/yamlcpp/include/yaml-cpp/parser.h @@ -11,7 +11,6 @@ #include #include "yaml-cpp/dll.h" -#include "yaml-cpp/noncopyable.h" namespace YAML { class EventHandler; @@ -24,11 +23,16 @@ struct Token; * A parser turns a stream of bytes into one stream of "events" per YAML * document in the input stream. */ -class YAML_CPP_API Parser : private noncopyable { +class YAML_CPP_API Parser { public: /** Constructs an empty parser (with no input. */ Parser(); + Parser(const Parser&) = delete; + Parser(Parser&&) = delete; + Parser& operator=(const Parser&) = delete; + Parser& operator=(Parser&&) = delete; + /** * Constructs a parser from the given input stream. The input stream must * live as long as the parser. @@ -81,6 +85,6 @@ class YAML_CPP_API Parser : private noncopyable { std::unique_ptr m_pScanner; std::unique_ptr m_pDirectives; }; -} +} // namespace YAML #endif // PARSER_H_62B23520_7C8E_11DE_8A39_0800200C9A66 diff --git a/lib/yamlcpp/include/yaml-cpp/traits.h b/lib/yamlcpp/include/yaml-cpp/traits.h index f33d0e1f637..36d406ba49d 100644 --- a/lib/yamlcpp/include/yaml-cpp/traits.h +++ b/lib/yamlcpp/include/yaml-cpp/traits.h @@ -7,6 +7,11 @@ #pragma once #endif +#include +#include +#include +#include + namespace YAML { template struct is_numeric { @@ -100,4 +105,31 @@ template struct disable_if : public disable_if_c {}; } +template +struct is_streamable { + template + static auto test(int) + -> decltype(std::declval() << std::declval(), std::true_type()); + + template + static auto test(...) -> std::false_type; + + static const bool value = decltype(test(0))::value; +}; + +template +struct streamable_to_string { + static std::string impl(const Key& key) { + std::stringstream ss; + ss << key; + return ss.str(); + } +}; + +template +struct streamable_to_string { + static std::string impl(const Key&) { + return ""; + } +}; #endif // TRAITS_H_62B23520_7C8E_11DE_8A39_0800200C9A66 diff --git a/lib/yamlcpp/src/binary.cpp b/lib/yamlcpp/src/binary.cpp index a7e51301b82..4db6d0b977d 100644 --- a/lib/yamlcpp/src/binary.cpp +++ b/lib/yamlcpp/src/binary.cpp @@ -1,5 +1,7 @@ #include "yaml-cpp/binary.h" +#include + namespace YAML { static const char encoding[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; @@ -72,19 +74,24 @@ std::vector DecodeBase64(const std::string &input) { unsigned char *out = &ret[0]; unsigned value = 0; - for (std::size_t i = 0; i < input.size(); i++) { + for (std::size_t i = 0, cnt = 0; i < input.size(); i++) { + if (std::isspace(input[i])) { + // skip newlines + continue; + } unsigned char d = decoding[static_cast(input[i])]; if (d == 255) return ret_type(); value = (value << 6) | d; - if (i % 4 == 3) { + if (cnt % 4 == 3) { *out++ = value >> 16; if (i > 0 && input[i - 1] != '=') *out++ = value >> 8; if (input[i] != '=') *out++ = value; } + ++cnt; } ret.resize(out - &ret[0]); diff --git a/lib/yamlcpp/src/collectionstack.h b/lib/yamlcpp/src/collectionstack.h index 2302786e037..9feba967951 100644 --- a/lib/yamlcpp/src/collectionstack.h +++ b/lib/yamlcpp/src/collectionstack.h @@ -7,8 +7,8 @@ #pragma once #endif -#include #include +#include namespace YAML { struct CollectionType { @@ -17,6 +17,7 @@ struct CollectionType { class CollectionStack { public: + CollectionStack() : collectionStack{} {} CollectionType::value GetCurCollectionType() const { if (collectionStack.empty()) return CollectionType::NoCollection; @@ -28,12 +29,13 @@ class CollectionStack { } void PopCollectionType(CollectionType::value type) { assert(type == GetCurCollectionType()); + (void)type; collectionStack.pop(); } private: std::stack collectionStack; }; -} +} // namespace YAML #endif // COLLECTIONSTACK_H_62B23520_7C8E_11DE_8A39_0800200C9A66 diff --git a/lib/yamlcpp/src/contrib/graphbuilder.cpp b/lib/yamlcpp/src/contrib/graphbuilder.cpp index 416c1359db6..bf251627953 100644 --- a/lib/yamlcpp/src/contrib/graphbuilder.cpp +++ b/lib/yamlcpp/src/contrib/graphbuilder.cpp @@ -11,7 +11,7 @@ void* BuildGraphOfNextDocument(Parser& parser, if (parser.HandleNextDocument(eventHandler)) { return eventHandler.RootNode(); } else { - return NULL; + return nullptr; } } } diff --git a/lib/yamlcpp/src/contrib/graphbuilderadapter.cpp b/lib/yamlcpp/src/contrib/graphbuilderadapter.cpp index 02a3d972a50..b9e0b657102 100644 --- a/lib/yamlcpp/src/contrib/graphbuilderadapter.cpp +++ b/lib/yamlcpp/src/contrib/graphbuilderadapter.cpp @@ -49,7 +49,7 @@ void GraphBuilderAdapter::OnMapStart(const Mark &mark, const std::string &tag, EmitterStyle::value /* style */) { void *pNode = m_builder.NewMap(mark, tag, GetCurrentParent()); m_containers.push(ContainerFrame(pNode, m_pKeyNode)); - m_pKeyNode = NULL; + m_pKeyNode = nullptr; RegisterAnchor(anchor, pNode); } @@ -62,7 +62,7 @@ void GraphBuilderAdapter::OnMapEnd() { void *GraphBuilderAdapter::GetCurrentParent() const { if (m_containers.empty()) { - return NULL; + return nullptr; } return m_containers.top().pContainer; } @@ -83,7 +83,7 @@ void GraphBuilderAdapter::DispositionNode(void *pNode) { if (m_containers.top().isMap()) { if (m_pKeyNode) { m_builder.AssignInMap(pContainer, m_pKeyNode, pNode); - m_pKeyNode = NULL; + m_pKeyNode = nullptr; } else { m_pKeyNode = pNode; } diff --git a/lib/yamlcpp/src/contrib/graphbuilderadapter.h b/lib/yamlcpp/src/contrib/graphbuilderadapter.h index 0d1e579208c..9078cdfdacd 100644 --- a/lib/yamlcpp/src/contrib/graphbuilderadapter.h +++ b/lib/yamlcpp/src/contrib/graphbuilderadapter.h @@ -26,7 +26,15 @@ namespace YAML { class GraphBuilderAdapter : public EventHandler { public: GraphBuilderAdapter(GraphBuilderInterface& builder) - : m_builder(builder), m_pRootNode(NULL), m_pKeyNode(NULL) {} + : m_builder(builder), + m_containers{}, + m_anchors{}, + m_pRootNode(nullptr), + m_pKeyNode(nullptr) {} + GraphBuilderAdapter(const GraphBuilderAdapter&) = delete; + GraphBuilderAdapter(GraphBuilderAdapter&&) = delete; + GraphBuilderAdapter& operator=(const GraphBuilderAdapter&) = delete; + GraphBuilderAdapter& operator=(GraphBuilderAdapter&&) = delete; virtual void OnDocumentStart(const Mark& mark) { (void)mark; } virtual void OnDocumentEnd() {} @@ -50,8 +58,8 @@ class GraphBuilderAdapter : public EventHandler { struct ContainerFrame { ContainerFrame(void* pSequence) : pContainer(pSequence), pPrevKeyNode(&sequenceMarker) {} - ContainerFrame(void* pMap, void* pPrevKeyNode) - : pContainer(pMap), pPrevKeyNode(pPrevKeyNode) {} + ContainerFrame(void* pMap, void* pPreviousKeyNode) + : pContainer(pMap), pPrevKeyNode(pPreviousKeyNode) {} void* pContainer; void* pPrevKeyNode; @@ -74,6 +82,6 @@ class GraphBuilderAdapter : public EventHandler { void RegisterAnchor(anchor_t anchor, void* pNode); void DispositionNode(void* pNode); }; -} +} // namespace YAML #endif // GRAPHBUILDERADAPTER_H_62B23520_7C8E_11DE_8A39_0800200C9A66 diff --git a/lib/yamlcpp/src/contrib/yaml-cpp.natvis b/lib/yamlcpp/src/contrib/yaml-cpp.natvis new file mode 100644 index 00000000000..d5c222be52c --- /dev/null +++ b/lib/yamlcpp/src/contrib/yaml-cpp.natvis @@ -0,0 +1,32 @@ + + + + + {{invalid}} + {{pNode==nullptr}} + {{ {*m_pNode} }} + + m_pNode->m_pRef._Ptr->m_pData._Ptr->m_scalar + m_pNode->m_pRef._Ptr->m_pData._Ptr->m_sequence + m_pNode->m_pRef._Ptr->m_pData._Ptr->m_map + m_pNode->m_pRef._Ptr->m_pData._Ptr + + + + + {{node:pRef==nullptr}} + {{node:pRef->pData==nullptr}} + {{undefined}} + {{{m_pRef._Ptr->m_pData._Ptr->m_scalar}}} + {{ Map {m_pRef._Ptr->m_pData._Ptr->m_map}}} + {{ Seq {m_pRef._Ptr->m_pData._Ptr->m_sequence}}} + {{{m_pRef._Ptr->m_pData._Ptr->m_type}}} + + m_pRef._Ptr->m_pData._Ptr->m_scalar + m_pRef._Ptr->m_pData._Ptr->m_sequence + m_pRef._Ptr->m_pData._Ptr->m_map + m_pRef._Ptr->m_pData._Ptr + + + + diff --git a/lib/yamlcpp/src/contrib/yaml-cpp.natvis.md b/lib/yamlcpp/src/contrib/yaml-cpp.natvis.md new file mode 100644 index 00000000000..f1d68a86c61 --- /dev/null +++ b/lib/yamlcpp/src/contrib/yaml-cpp.natvis.md @@ -0,0 +1,9 @@ +# MSVC debugger visualizer for YAML::Node + +## How to use +Add yaml-cpp.natvis to your Visual C++ project like any other source file. It will be included in the debug information, and improve debugger display on YAML::Node and contained types. + +## Compatibility and Troubleshooting + +This has been tested for MSVC 2017. It is expected to be compatible with VS 2015 and VS 2019. If you have any problems, you can open an issue here: https://github.com/peterchen-cp/yaml-cpp-natvis + diff --git a/lib/yamlcpp/src/directives.cpp b/lib/yamlcpp/src/directives.cpp index 963bd2cd379..0c85d0ff37a 100644 --- a/lib/yamlcpp/src/directives.cpp +++ b/lib/yamlcpp/src/directives.cpp @@ -1,12 +1,7 @@ #include "directives.h" namespace YAML { -Directives::Directives() { - // version - version.isDefault = true; - version.major = 1; - version.minor = 2; -} +Directives::Directives() : version{true, 1, 2}, tags{} {} const std::string Directives::TranslateTagHandle( const std::string& handle) const { @@ -19,4 +14,4 @@ const std::string Directives::TranslateTagHandle( return it->second; } -} +} // namespace YAML diff --git a/lib/yamlcpp/src/emitfromevents.cpp b/lib/yamlcpp/src/emitfromevents.cpp index 4832649f3c7..5a51bb471e3 100644 --- a/lib/yamlcpp/src/emitfromevents.cpp +++ b/lib/yamlcpp/src/emitfromevents.cpp @@ -16,10 +16,11 @@ std::string ToString(YAML::anchor_t anchor) { stream << anchor; return stream.str(); } -} +} // namespace namespace YAML { -EmitFromEvents::EmitFromEvents(Emitter& emitter) : m_emitter(emitter) {} +EmitFromEvents::EmitFromEvents(Emitter& emitter) + : m_emitter(emitter), m_stateStack{} {} void EmitFromEvents::OnDocumentStart(const Mark&) {} @@ -116,4 +117,4 @@ void EmitFromEvents::EmitProps(const std::string& tag, anchor_t anchor) { if (anchor) m_emitter << Anchor(ToString(anchor)); } -} +} // namespace YAML diff --git a/lib/yamlcpp/src/emitter.cpp b/lib/yamlcpp/src/emitter.cpp index ebeb059554e..016beb12c0d 100644 --- a/lib/yamlcpp/src/emitter.cpp +++ b/lib/yamlcpp/src/emitter.cpp @@ -11,7 +11,7 @@ namespace YAML { class Binary; struct _Null; -Emitter::Emitter() : m_pState(new EmitterState) {} +Emitter::Emitter() : m_pState(new EmitterState), m_stream{} {} Emitter::Emitter(std::ostream& stream) : m_pState(new EmitterState), m_stream(stream) {} @@ -285,10 +285,8 @@ void Emitter::PrepareTopNode(EmitterNodeType::value child) { if (child == EmitterNodeType::NoType) return; - if (m_pState->CurGroupChildCount() > 0 && m_stream.col() > 0) { - if (child != EmitterNodeType::NoType) - EmitBeginDoc(); - } + if (m_pState->CurGroupChildCount() > 0 && m_stream.col() > 0) + EmitBeginDoc(); switch (child) { case EmitterNodeType::NoType: @@ -908,4 +906,4 @@ Emitter& Emitter::Write(const Binary& binary) { return *this; } -} +} // namespace YAML diff --git a/lib/yamlcpp/src/emitterstate.cpp b/lib/yamlcpp/src/emitterstate.cpp index 3542aaf5071..890b4f0d2e7 100644 --- a/lib/yamlcpp/src/emitterstate.cpp +++ b/lib/yamlcpp/src/emitterstate.cpp @@ -6,27 +6,31 @@ namespace YAML { EmitterState::EmitterState() : m_isGood(true), + m_lastError{}, + // default global manipulators + m_charset(EmitNonAscii), + m_strFmt(Auto), + m_boolFmt(TrueFalseBool), + m_boolLengthFmt(LongBool), + m_boolCaseFmt(LowerCase), + m_intFmt(Dec), + m_indent(2), + m_preCommentIndent(2), + m_postCommentIndent(1), + m_seqFmt(Block), + m_mapFmt(Block), + m_mapKeyFmt(Auto), + m_floatPrecision(std::numeric_limits::max_digits10), + m_doublePrecision(std::numeric_limits::max_digits10), + // + m_modifiedSettings{}, + m_globalModifiedSettings{}, + m_groups{}, m_curIndent(0), m_hasAnchor(false), m_hasTag(false), m_hasNonContent(false), - m_docCount(0) { - // set default global manipulators - m_charset.set(EmitNonAscii); - m_strFmt.set(Auto); - m_boolFmt.set(TrueFalseBool); - m_boolLengthFmt.set(LongBool); - m_boolCaseFmt.set(LowerCase); - m_intFmt.set(Dec); - m_indent.set(2); - m_preCommentIndent.set(2); - m_postCommentIndent.set(1); - m_seqFmt.set(Block); - m_mapFmt.set(Block); - m_mapKeyFmt.set(Auto); - m_floatPrecision.set(std::numeric_limits::digits10 + 1); - m_doublePrecision.set(std::numeric_limits::digits10 + 1); -} + m_docCount(0) {} EmitterState::~EmitterState() {} @@ -349,7 +353,7 @@ bool EmitterState::SetMapKeyFormat(EMITTER_MANIP value, FmtScope::value scope) { } bool EmitterState::SetFloatPrecision(std::size_t value, FmtScope::value scope) { - if (value > std::numeric_limits::digits10 + 1) + if (value > std::numeric_limits::max_digits10) return false; _Set(m_floatPrecision, value, scope); return true; @@ -357,9 +361,9 @@ bool EmitterState::SetFloatPrecision(std::size_t value, FmtScope::value scope) { bool EmitterState::SetDoublePrecision(std::size_t value, FmtScope::value scope) { - if (value > std::numeric_limits::digits10 + 1) + if (value > std::numeric_limits::max_digits10) return false; _Set(m_doublePrecision, value, scope); return true; } -} +} // namespace YAML diff --git a/lib/yamlcpp/src/emitterstate.h b/lib/yamlcpp/src/emitterstate.h index 0937f000d9f..75e216695cd 100644 --- a/lib/yamlcpp/src/emitterstate.h +++ b/lib/yamlcpp/src/emitterstate.h @@ -145,7 +145,12 @@ class EmitterState { struct Group { explicit Group(GroupType::value type_) - : type(type_), indent(0), childCount(0), longKey(false) {} + : type(type_), + flowType{}, + indent(0), + childCount(0), + longKey(false), + modifiedSettings{} {} GroupType::value type; FlowType::value flowType; @@ -198,6 +203,6 @@ void EmitterState::_Set(Setting& fmt, T value, FmtScope::value scope) { assert(false); } } -} +} // namespace YAML #endif // EMITTERSTATE_H_62B23520_7C8E_11DE_8A39_0800200C9A66 diff --git a/lib/yamlcpp/src/emitterutils.cpp b/lib/yamlcpp/src/emitterutils.cpp index 147738ad8a1..e348599d53d 100644 --- a/lib/yamlcpp/src/emitterutils.cpp +++ b/lib/yamlcpp/src/emitterutils.cpp @@ -8,8 +8,8 @@ #include "regeximpl.h" #include "stringsource.h" #include "yaml-cpp/binary.h" // IWYU pragma: keep -#include "yaml-cpp/ostream_wrapper.h" #include "yaml-cpp/null.h" +#include "yaml-cpp/ostream_wrapper.h" namespace YAML { namespace Utils { @@ -134,12 +134,12 @@ void WriteCodePoint(ostream_wrapper& out, int codePoint) { if (codePoint < 0 || codePoint > 0x10FFFF) { codePoint = REPLACEMENT_CHARACTER; } - if (codePoint < 0x7F) { + if (codePoint <= 0x7F) { out << static_cast(codePoint); - } else if (codePoint < 0x7FF) { + } else if (codePoint <= 0x7FF) { out << static_cast(0xC0 | (codePoint >> 6)) << static_cast(0x80 | (codePoint & 0x3F)); - } else if (codePoint < 0xFFFF) { + } else if (codePoint <= 0xFFFF) { out << static_cast(0xE0 | (codePoint >> 12)) << static_cast(0x80 | ((codePoint >> 6) & 0x3F)) << static_cast(0x80 | (codePoint & 0x3F)); @@ -173,12 +173,12 @@ bool IsValidPlainScalar(const std::string& str, FlowType::value flowType, // then check until something is disallowed static const RegEx& disallowed_flow = - Exp::EndScalarInFlow() || (Exp::BlankOrBreak() + Exp::Comment()) || - Exp::NotPrintable() || Exp::Utf8_ByteOrderMark() || Exp::Break() || + Exp::EndScalarInFlow() | (Exp::BlankOrBreak() + Exp::Comment()) | + Exp::NotPrintable() | Exp::Utf8_ByteOrderMark() | Exp::Break() | Exp::Tab(); static const RegEx& disallowed_block = - Exp::EndScalar() || (Exp::BlankOrBreak() + Exp::Comment()) || - Exp::NotPrintable() || Exp::Utf8_ByteOrderMark() || Exp::Break() || + Exp::EndScalar() | (Exp::BlankOrBreak() + Exp::Comment()) | + Exp::NotPrintable() | Exp::Utf8_ByteOrderMark() | Exp::Break() | Exp::Tab(); const RegEx& disallowed = flowType == FlowType::Flow ? disallowed_flow : disallowed_block; @@ -258,7 +258,7 @@ bool WriteAliasName(ostream_wrapper& out, const std::string& str) { } return true; } -} +} // namespace StringFormat::value ComputeStringFormat(const std::string& str, EMITTER_MANIP strFormat, @@ -382,7 +382,7 @@ bool WriteChar(ostream_wrapper& out, char ch) { out << "\"\\b\""; } else if (ch == '\\') { out << "\"\\\\\""; - } else if ((0x20 <= ch && ch <= 0x7e) || ch == ' ') { + } else if (0x20 <= ch && ch <= 0x7e) { out << "\"" << ch << "\""; } else { out << "\""; @@ -401,8 +401,8 @@ bool WriteComment(ostream_wrapper& out, const std::string& str, for (std::string::const_iterator i = str.begin(); GetNextCodePointAndAdvance(codePoint, i, str.end());) { if (codePoint == '\n') { - out << "\n" << IndentTo(curIndent) << "#" - << Indentation(postCommentIndent); + out << "\n" + << IndentTo(curIndent) << "#" << Indentation(postCommentIndent); out.set_comment(); } else { WriteCodePoint(out, codePoint); @@ -479,5 +479,5 @@ bool WriteBinary(ostream_wrapper& out, const Binary& binary) { false); return true; } -} -} +} // namespace Utils +} // namespace YAML diff --git a/lib/yamlcpp/src/exceptions.cpp b/lib/yamlcpp/src/exceptions.cpp index 9b6d8912c18..841549e0dfb 100644 --- a/lib/yamlcpp/src/exceptions.cpp +++ b/lib/yamlcpp/src/exceptions.cpp @@ -2,7 +2,7 @@ // This is here for compatibility with older versions of Visual Studio // which don't support noexcept -#ifdef _MSC_VER +#if defined(_MSC_VER) && _MSC_VER < 1900 #define YAML_CPP_NOEXCEPT _NOEXCEPT #else #define YAML_CPP_NOEXCEPT noexcept diff --git a/lib/yamlcpp/src/exp.h b/lib/yamlcpp/src/exp.h index 7c02cf6e451..fe36535b578 100644 --- a/lib/yamlcpp/src/exp.h +++ b/lib/yamlcpp/src/exp.h @@ -33,15 +33,15 @@ inline const RegEx& Tab() { return e; } inline const RegEx& Blank() { - static const RegEx e = Space() || Tab(); + static const RegEx e = Space() | Tab(); return e; } inline const RegEx& Break() { - static const RegEx e = RegEx('\n') || RegEx("\r\n"); + static const RegEx e = RegEx('\n') | RegEx("\r\n"); return e; } inline const RegEx& BlankOrBreak() { - static const RegEx e = Blank() || Break(); + static const RegEx e = Blank() | Break(); return e; } inline const RegEx& Digit() { @@ -49,29 +49,29 @@ inline const RegEx& Digit() { return e; } inline const RegEx& Alpha() { - static const RegEx e = RegEx('a', 'z') || RegEx('A', 'Z'); + static const RegEx e = RegEx('a', 'z') | RegEx('A', 'Z'); return e; } inline const RegEx& AlphaNumeric() { - static const RegEx e = Alpha() || Digit(); + static const RegEx e = Alpha() | Digit(); return e; } inline const RegEx& Word() { - static const RegEx e = AlphaNumeric() || RegEx('-'); + static const RegEx e = AlphaNumeric() | RegEx('-'); return e; } inline const RegEx& Hex() { - static const RegEx e = Digit() || RegEx('A', 'F') || RegEx('a', 'f'); + static const RegEx e = Digit() | RegEx('A', 'F') | RegEx('a', 'f'); return e; } // Valid Unicode code points that are not part of c-printable (YAML 1.2, sec. // 5.1) inline const RegEx& NotPrintable() { static const RegEx e = - RegEx(0) || - RegEx("\x01\x02\x03\x04\x05\x06\x07\x08\x0B\x0C\x7F", REGEX_OR) || - RegEx(0x0E, 0x1F) || - (RegEx('\xC2') + (RegEx('\x80', '\x84') || RegEx('\x86', '\x9F'))); + RegEx(0) | + RegEx("\x01\x02\x03\x04\x05\x06\x07\x08\x0B\x0C\x7F", REGEX_OR) | + RegEx(0x0E, 0x1F) | + (RegEx('\xC2') + (RegEx('\x80', '\x84') | RegEx('\x86', '\x9F'))); return e; } inline const RegEx& Utf8_ByteOrderMark() { @@ -82,19 +82,19 @@ inline const RegEx& Utf8_ByteOrderMark() { // actual tags inline const RegEx& DocStart() { - static const RegEx e = RegEx("---") + (BlankOrBreak() || RegEx()); + static const RegEx e = RegEx("---") + (BlankOrBreak() | RegEx()); return e; } inline const RegEx& DocEnd() { - static const RegEx e = RegEx("...") + (BlankOrBreak() || RegEx()); + static const RegEx e = RegEx("...") + (BlankOrBreak() | RegEx()); return e; } inline const RegEx& DocIndicator() { - static const RegEx e = DocStart() || DocEnd(); + static const RegEx e = DocStart() | DocEnd(); return e; } inline const RegEx& BlockEntry() { - static const RegEx e = RegEx('-') + (BlankOrBreak() || RegEx()); + static const RegEx e = RegEx('-') + (BlankOrBreak() | RegEx()); return e; } inline const RegEx& Key() { @@ -106,11 +106,11 @@ inline const RegEx& KeyInFlow() { return e; } inline const RegEx& Value() { - static const RegEx e = RegEx(':') + (BlankOrBreak() || RegEx()); + static const RegEx e = RegEx(':') + (BlankOrBreak() | RegEx()); return e; } inline const RegEx& ValueInFlow() { - static const RegEx e = RegEx(':') + (BlankOrBreak() || RegEx(",}", REGEX_OR)); + static const RegEx e = RegEx(':') + (BlankOrBreak() | RegEx(",}", REGEX_OR)); return e; } inline const RegEx& ValueInJSONFlow() { @@ -122,20 +122,20 @@ inline const RegEx Comment() { return e; } inline const RegEx& Anchor() { - static const RegEx e = !(RegEx("[]{},", REGEX_OR) || BlankOrBreak()); + static const RegEx e = !(RegEx("[]{},", REGEX_OR) | BlankOrBreak()); return e; } inline const RegEx& AnchorEnd() { - static const RegEx e = RegEx("?:,]}%@`", REGEX_OR) || BlankOrBreak(); + static const RegEx e = RegEx("?:,]}%@`", REGEX_OR) | BlankOrBreak(); return e; } inline const RegEx& URI() { - static const RegEx e = Word() || RegEx("#;/?:@&=+$,_.!~*'()[]", REGEX_OR) || + static const RegEx e = Word() | RegEx("#;/?:@&=+$,_.!~*'()[]", REGEX_OR) | (RegEx('%') + Hex() + Hex()); return e; } inline const RegEx& Tag() { - static const RegEx e = Word() || RegEx("#;/?:@&=+$_.~*'()", REGEX_OR) || + static const RegEx e = Word() | RegEx("#;/?:@&=+$_.~*'()", REGEX_OR) | (RegEx('%') + Hex() + Hex()); return e; } @@ -148,34 +148,34 @@ inline const RegEx& Tag() { // space. inline const RegEx& PlainScalar() { static const RegEx e = - !(BlankOrBreak() || RegEx(",[]{}#&*!|>\'\"%@`", REGEX_OR) || - (RegEx("-?:", REGEX_OR) + (BlankOrBreak() || RegEx()))); + !(BlankOrBreak() | RegEx(",[]{}#&*!|>\'\"%@`", REGEX_OR) | + (RegEx("-?:", REGEX_OR) + (BlankOrBreak() | RegEx()))); return e; } inline const RegEx& PlainScalarInFlow() { static const RegEx e = - !(BlankOrBreak() || RegEx("?,[]{}#&*!|>\'\"%@`", REGEX_OR) || + !(BlankOrBreak() | RegEx("?,[]{}#&*!|>\'\"%@`", REGEX_OR) | (RegEx("-:", REGEX_OR) + Blank())); return e; } inline const RegEx& EndScalar() { - static const RegEx e = RegEx(':') + (BlankOrBreak() || RegEx()); + static const RegEx e = RegEx(':') + (BlankOrBreak() | RegEx()); return e; } inline const RegEx& EndScalarInFlow() { static const RegEx e = - (RegEx(':') + (BlankOrBreak() || RegEx() || RegEx(",]}", REGEX_OR))) || + (RegEx(':') + (BlankOrBreak() | RegEx() | RegEx(",]}", REGEX_OR))) | RegEx(",?[]{}", REGEX_OR); return e; } inline const RegEx& ScanScalarEndInFlow() { - static const RegEx e = (EndScalarInFlow() || (BlankOrBreak() + Comment())); + static const RegEx e = (EndScalarInFlow() | (BlankOrBreak() + Comment())); return e; } inline const RegEx& ScanScalarEnd() { - static const RegEx e = EndScalar() || (BlankOrBreak() + Comment()); + static const RegEx e = EndScalar() | (BlankOrBreak() + Comment()); return e; } inline const RegEx& EscSingleQuote() { @@ -192,8 +192,8 @@ inline const RegEx& ChompIndicator() { return e; } inline const RegEx& Chomp() { - static const RegEx e = (ChompIndicator() + Digit()) || - (Digit() + ChompIndicator()) || ChompIndicator() || + static const RegEx e = (ChompIndicator() + Digit()) | + (Digit() + ChompIndicator()) | ChompIndicator() | Digit(); return e; } diff --git a/lib/yamlcpp/src/node_data.cpp b/lib/yamlcpp/src/node_data.cpp index 77cd4657806..6cfedfc3c7f 100644 --- a/lib/yamlcpp/src/node_data.cpp +++ b/lib/yamlcpp/src/node_data.cpp @@ -13,14 +13,22 @@ namespace YAML { namespace detail { -std::string node_data::empty_scalar; +const std::string& node_data::empty_scalar() { + static const std::string svalue; + return svalue; +} node_data::node_data() : m_isDefined(false), m_mark(Mark::null_mark()), m_type(NodeType::Null), + m_tag{}, m_style(EmitterStyle::Default), - m_seqSize(0) {} + m_scalar{}, + m_sequence{}, + m_seqSize(0), + m_map{}, + m_undefinedPairs{} {} void node_data::mark_defined() { if (m_type == NodeType::Undefined) @@ -188,7 +196,7 @@ void node_data::insert(node& key, node& value, shared_memory_holder pMemory) { convert_to_map(pMemory); break; case NodeType::Scalar: - throw BadSubscript(); + throw BadSubscript(key); } insert_map_pair(key, value); @@ -197,7 +205,7 @@ void node_data::insert(node& key, node& value, shared_memory_holder pMemory) { // indexing node* node_data::get(node& key, shared_memory_holder /* pMemory */) const { if (m_type != NodeType::Map) { - return NULL; + return nullptr; } for (node_map::const_iterator it = m_map.begin(); it != m_map.end(); ++it) { @@ -205,7 +213,7 @@ node* node_data::get(node& key, shared_memory_holder /* pMemory */) const { return it->second; } - return NULL; + return nullptr; } node& node_data::get(node& key, shared_memory_holder pMemory) { @@ -218,7 +226,7 @@ node& node_data::get(node& key, shared_memory_holder pMemory) { convert_to_map(pMemory); break; case NodeType::Scalar: - throw BadSubscript(); + throw BadSubscript(key); } for (node_map::const_iterator it = m_map.begin(); it != m_map.end(); ++it) { @@ -235,6 +243,14 @@ bool node_data::remove(node& key, shared_memory_holder /* pMemory */) { if (m_type != NodeType::Map) return false; + for (kv_pairs::iterator it = m_undefinedPairs.begin(); + it != m_undefinedPairs.end();) { + kv_pairs::iterator jt = std::next(it); + if (it->first->is(key)) + m_undefinedPairs.erase(it); + it = jt; + } + for (node_map::iterator it = m_map.begin(); it != m_map.end(); ++it) { if (it->first->is(key)) { m_map.erase(it); @@ -296,5 +312,5 @@ void node_data::convert_sequence_to_map(shared_memory_holder pMemory) { reset_sequence(); m_type = NodeType::Map; } -} -} +} // namespace detail +} // namespace YAML diff --git a/lib/yamlcpp/src/nodebuilder.cpp b/lib/yamlcpp/src/nodebuilder.cpp index 093d2efeb77..c90a66cb878 100644 --- a/lib/yamlcpp/src/nodebuilder.cpp +++ b/lib/yamlcpp/src/nodebuilder.cpp @@ -11,8 +11,13 @@ namespace YAML { struct Mark; NodeBuilder::NodeBuilder() - : m_pMemory(new detail::memory_holder), m_pRoot(0), m_mapDepth(0) { - m_anchors.push_back(0); // since the anchors start at 1 + : m_pMemory(new detail::memory_holder), + m_pRoot(nullptr), + m_stack{}, + m_anchors{}, + m_keys{}, + m_mapDepth(0) { + m_anchors.push_back(nullptr); // since the anchors start at 1 } NodeBuilder::~NodeBuilder() {} @@ -127,4 +132,4 @@ void NodeBuilder::RegisterAnchor(anchor_t anchor, detail::node& node) { m_anchors.push_back(&node); } } -} +} // namespace YAML diff --git a/lib/yamlcpp/src/nodebuilder.h b/lib/yamlcpp/src/nodebuilder.h index a6a47f007bb..642da36cc2a 100644 --- a/lib/yamlcpp/src/nodebuilder.h +++ b/lib/yamlcpp/src/nodebuilder.h @@ -27,6 +27,10 @@ class Node; class NodeBuilder : public EventHandler { public: NodeBuilder(); + NodeBuilder(const NodeBuilder&) = delete; + NodeBuilder(NodeBuilder&&) = delete; + NodeBuilder& operator=(const NodeBuilder&) = delete; + NodeBuilder& operator=(NodeBuilder&&) = delete; virtual ~NodeBuilder(); Node Root(); @@ -65,6 +69,6 @@ class NodeBuilder : public EventHandler { std::vector m_keys; std::size_t m_mapDepth; }; -} +} // namespace YAML #endif // NODE_NODEBUILDER_H_62B23520_7C8E_11DE_8A39_0800200C9A66 diff --git a/lib/yamlcpp/src/nodeevents.cpp b/lib/yamlcpp/src/nodeevents.cpp index 82261feb058..b7222e068ed 100644 --- a/lib/yamlcpp/src/nodeevents.cpp +++ b/lib/yamlcpp/src/nodeevents.cpp @@ -20,7 +20,7 @@ anchor_t NodeEvents::AliasManager::LookupAnchor( } NodeEvents::NodeEvents(const Node& node) - : m_pMemory(node.m_pMemory), m_root(node.m_pNode) { + : m_pMemory(node.m_pMemory), m_root(node.m_pNode), m_refCount{} { if (m_root) Setup(*m_root); } @@ -98,4 +98,4 @@ bool NodeEvents::IsAliased(const detail::node& node) const { RefCount::const_iterator it = m_refCount.find(node.ref()); return it != m_refCount.end() && it->second > 1; } -} +} // namespace YAML diff --git a/lib/yamlcpp/src/nodeevents.h b/lib/yamlcpp/src/nodeevents.h index 49c18eb854a..dbed5d2066f 100644 --- a/lib/yamlcpp/src/nodeevents.h +++ b/lib/yamlcpp/src/nodeevents.h @@ -26,13 +26,17 @@ class Node; class NodeEvents { public: explicit NodeEvents(const Node& node); + NodeEvents(const NodeEvents&) = delete; + NodeEvents(NodeEvents&&) = delete; + NodeEvents& operator=(const NodeEvents&) = delete; + NodeEvents& operator=(NodeEvents&&) = delete; void Emit(EventHandler& handler); private: class AliasManager { public: - AliasManager() : m_curAnchor(0) {} + AliasManager() : m_anchorByIdentity{}, m_curAnchor(0) {} void RegisterReference(const detail::node& node); anchor_t LookupAnchor(const detail::node& node) const; @@ -59,6 +63,6 @@ class NodeEvents { typedef std::map RefCount; RefCount m_refCount; }; -} +} // namespace YAML #endif // NODE_NODEEVENTS_H_62B23520_7C8E_11DE_8A39_0800200C9A66 diff --git a/lib/yamlcpp/src/ostream_wrapper.cpp b/lib/yamlcpp/src/ostream_wrapper.cpp index 357fc0094c4..5fa58589736 100644 --- a/lib/yamlcpp/src/ostream_wrapper.cpp +++ b/lib/yamlcpp/src/ostream_wrapper.cpp @@ -7,14 +7,19 @@ namespace YAML { ostream_wrapper::ostream_wrapper() : m_buffer(1, '\0'), - m_pStream(0), + m_pStream(nullptr), m_pos(0), m_row(0), m_col(0), m_comment(false) {} ostream_wrapper::ostream_wrapper(std::ostream& stream) - : m_pStream(&stream), m_pos(0), m_row(0), m_col(0), m_comment(false) {} + : m_buffer{}, + m_pStream(&stream), + m_pos(0), + m_row(0), + m_col(0), + m_comment(false) {} ostream_wrapper::~ostream_wrapper() {} @@ -54,4 +59,4 @@ void ostream_wrapper::update_pos(char ch) { m_comment = false; } } -} +} // namespace YAML diff --git a/lib/yamlcpp/src/parser.cpp b/lib/yamlcpp/src/parser.cpp index cd69f39fcec..7bc0c7828af 100644 --- a/lib/yamlcpp/src/parser.cpp +++ b/lib/yamlcpp/src/parser.cpp @@ -11,9 +11,9 @@ namespace YAML { class EventHandler; -Parser::Parser() {} +Parser::Parser() : m_pScanner{}, m_pDirectives{} {} -Parser::Parser(std::istream& in) { Load(in); } +Parser::Parser(std::istream& in) : Parser() { Load(in); } Parser::~Parser() {} @@ -126,4 +126,4 @@ void Parser::PrintTokens(std::ostream& out) { m_pScanner->pop(); } } -} +} // namespace YAML diff --git a/lib/yamlcpp/src/ptr_vector.h b/lib/yamlcpp/src/ptr_vector.h index 955aebd8d56..d58de04cb62 100644 --- a/lib/yamlcpp/src/ptr_vector.h +++ b/lib/yamlcpp/src/ptr_vector.h @@ -12,15 +12,17 @@ #include #include -#include "yaml-cpp/noncopyable.h" - namespace YAML { // TODO: This class is no longer needed template -class ptr_vector : private YAML::noncopyable { +class ptr_vector { public: - ptr_vector() {} + ptr_vector() : m_data{} {} + ptr_vector(const ptr_vector&) = delete; + ptr_vector(ptr_vector&&) = default; + ptr_vector& operator=(const ptr_vector&) = delete; + ptr_vector& operator=(ptr_vector&&) = default; void clear() { m_data.clear(); } @@ -38,6 +40,6 @@ class ptr_vector : private YAML::noncopyable { private: std::vector> m_data; }; -} +} // namespace YAML #endif // PTR_VECTOR_H_62B23520_7C8E_11DE_8A39_0800200C9A66 diff --git a/lib/yamlcpp/src/regex_yaml.cpp b/lib/yamlcpp/src/regex_yaml.cpp index 20b772051d2..bf1784b41d5 100644 --- a/lib/yamlcpp/src/regex_yaml.cpp +++ b/lib/yamlcpp/src/regex_yaml.cpp @@ -2,18 +2,16 @@ namespace YAML { // constructors -RegEx::RegEx() : m_op(REGEX_EMPTY) {} -RegEx::RegEx(REGEX_OP op) : m_op(op) {} +RegEx::RegEx(REGEX_OP op) : m_op(op), m_a(0), m_z(0), m_params{} {} +RegEx::RegEx() : RegEx(REGEX_EMPTY) {} -RegEx::RegEx(char ch) : m_op(REGEX_MATCH), m_a(ch) {} +RegEx::RegEx(char ch) : m_op(REGEX_MATCH), m_a(ch), m_z(0), m_params{} {} -RegEx::RegEx(char a, char z) : m_op(REGEX_RANGE), m_a(a), m_z(z) {} +RegEx::RegEx(char a, char z) : m_op(REGEX_RANGE), m_a(a), m_z(z), m_params{} {} -RegEx::RegEx(const std::string& str, REGEX_OP op) : m_op(op) { - for (std::size_t i = 0; i < str.size(); i++) - m_params.push_back(RegEx(str[i])); -} +RegEx::RegEx(const std::string& str, REGEX_OP op) + : m_op(op), m_a(0), m_z(0), m_params(str.begin(), str.end()) {} // combination constructors RegEx operator!(const RegEx& ex) { @@ -22,14 +20,14 @@ RegEx operator!(const RegEx& ex) { return ret; } -RegEx operator||(const RegEx& ex1, const RegEx& ex2) { +RegEx operator|(const RegEx& ex1, const RegEx& ex2) { RegEx ret(REGEX_OR); ret.m_params.push_back(ex1); ret.m_params.push_back(ex2); return ret; } -RegEx operator&&(const RegEx& ex1, const RegEx& ex2) { +RegEx operator&(const RegEx& ex1, const RegEx& ex2) { RegEx ret(REGEX_AND); ret.m_params.push_back(ex1); ret.m_params.push_back(ex2); @@ -42,4 +40,4 @@ RegEx operator+(const RegEx& ex1, const RegEx& ex2) { ret.m_params.push_back(ex2); return ret; } -} +} // namespace YAML diff --git a/lib/yamlcpp/src/regex_yaml.h b/lib/yamlcpp/src/regex_yaml.h index 8f28b852a29..3a347bbd082 100644 --- a/lib/yamlcpp/src/regex_yaml.h +++ b/lib/yamlcpp/src/regex_yaml.h @@ -31,14 +31,14 @@ enum REGEX_OP { class YAML_CPP_API RegEx { public: RegEx(); - RegEx(char ch); + explicit RegEx(char ch); RegEx(char a, char z); RegEx(const std::string& str, REGEX_OP op = REGEX_SEQ); ~RegEx() {} friend YAML_CPP_API RegEx operator!(const RegEx& ex); - friend YAML_CPP_API RegEx operator||(const RegEx& ex1, const RegEx& ex2); - friend YAML_CPP_API RegEx operator&&(const RegEx& ex1, const RegEx& ex2); + friend YAML_CPP_API RegEx operator|(const RegEx& ex1, const RegEx& ex2); + friend YAML_CPP_API RegEx operator&(const RegEx& ex1, const RegEx& ex2); friend YAML_CPP_API RegEx operator+(const RegEx& ex1, const RegEx& ex2); bool Matches(char ch) const; @@ -53,7 +53,7 @@ class YAML_CPP_API RegEx { int Match(const Source& source) const; private: - RegEx(REGEX_OP op); + explicit RegEx(REGEX_OP op); template bool IsValidSource(const Source& source) const; @@ -77,10 +77,11 @@ class YAML_CPP_API RegEx { private: REGEX_OP m_op; - char m_a, m_z; + char m_a{}; + char m_z{}; std::vector m_params; }; -} +} // namespace YAML #include "regeximpl.h" diff --git a/lib/yamlcpp/src/scanner.cpp b/lib/yamlcpp/src/scanner.cpp index b5cfcc12b22..ac848924fc7 100644 --- a/lib/yamlcpp/src/scanner.cpp +++ b/lib/yamlcpp/src/scanner.cpp @@ -9,10 +9,15 @@ namespace YAML { Scanner::Scanner(std::istream& in) : INPUT(in), + m_tokens{}, m_startedStream(false), m_endedStream(false), m_simpleKeyAllowed(false), - m_canBeJSONFlow(false) {} + m_canBeJSONFlow(false), + m_simpleKeys{}, + m_indents{}, + m_indentRefs{}, + m_flows{} {} Scanner::~Scanner() {} @@ -282,7 +287,7 @@ Scanner::IndentMarker* Scanner::PushIndentTo(int column, IndentMarker::INDENT_TYPE type) { // are we in flow? if (InFlowContext()) { - return 0; + return nullptr; } std::unique_ptr pIndent(new IndentMarker(column, type)); @@ -291,12 +296,12 @@ Scanner::IndentMarker* Scanner::PushIndentTo(int column, // is this actually an indentation? if (indent.column < lastIndent.column) { - return 0; + return nullptr; } if (indent.column == lastIndent.column && !(indent.type == IndentMarker::SEQ && lastIndent.type == IndentMarker::MAP)) { - return 0; + return nullptr; } // push a start token diff --git a/lib/yamlcpp/src/scanner.h b/lib/yamlcpp/src/scanner.h index 7bb2ccc71a5..c653ac69ff6 100644 --- a/lib/yamlcpp/src/scanner.h +++ b/lib/yamlcpp/src/scanner.h @@ -49,7 +49,7 @@ class Scanner { enum INDENT_TYPE { MAP, SEQ, NONE }; enum STATUS { VALID, INVALID, UNKNOWN }; IndentMarker(int column_, INDENT_TYPE type_) - : column(column_), type(type_), status(VALID), pStartToken(0) {} + : column(column_), type(type_), status(VALID), pStartToken(nullptr) {} int column; INDENT_TYPE type; diff --git a/lib/yamlcpp/src/scanscalar.cpp b/lib/yamlcpp/src/scanscalar.cpp index 10e359d4466..e5b0973ee8d 100644 --- a/lib/yamlcpp/src/scanscalar.cpp +++ b/lib/yamlcpp/src/scanscalar.cpp @@ -183,7 +183,7 @@ std::string ScanScalar(Stream& INPUT, ScanScalarParams& params) { case FOLD_FLOW: if (nextEmptyLine) { scalar += "\n"; - } else if (!emptyLine && !nextEmptyLine && !escapedNewline) { + } else if (!emptyLine && !escapedNewline) { scalar += " "; } break; diff --git a/lib/yamlcpp/src/scantoken.cpp b/lib/yamlcpp/src/scantoken.cpp index fd8758d7815..325de571502 100644 --- a/lib/yamlcpp/src/scantoken.cpp +++ b/lib/yamlcpp/src/scantoken.cpp @@ -338,7 +338,7 @@ void Scanner::ScanQuotedScalar() { // setup the scanning parameters ScanScalarParams params; - RegEx end = (single ? RegEx(quote) && !Exp::EscSingleQuote() : RegEx(quote)); + RegEx end = (single ? RegEx(quote) & !Exp::EscSingleQuote() : RegEx(quote)); params.end = &end; params.eatEnd = true; params.escape = (single ? '\'' : '\\'); @@ -434,4 +434,4 @@ void Scanner::ScanBlockScalar() { token.value = scalar; m_tokens.push(token); } -} +} // namespace YAML diff --git a/lib/yamlcpp/src/setting.h b/lib/yamlcpp/src/setting.h index b78d40e2e85..280d1e72b30 100644 --- a/lib/yamlcpp/src/setting.h +++ b/lib/yamlcpp/src/setting.h @@ -8,8 +8,8 @@ #endif #include +#include #include -#include "yaml-cpp/noncopyable.h" namespace YAML { class SettingChangeBase; @@ -18,6 +18,7 @@ template class Setting { public: Setting() : m_value() {} + Setting(const T& value) : m_value() { set(value); } const T get() const { return m_value; } std::unique_ptr set(const T& value); @@ -36,10 +37,14 @@ class SettingChangeBase { template class SettingChange : public SettingChangeBase { public: - SettingChange(Setting* pSetting) : m_pCurSetting(pSetting) { - // copy old setting to save its state - m_oldSetting = *pSetting; - } + SettingChange(Setting* pSetting) + : m_pCurSetting(pSetting), + m_oldSetting(*pSetting) // copy old setting to save its state + {} + SettingChange(const SettingChange&) = delete; + SettingChange(SettingChange&&) = delete; + SettingChange& operator=(const SettingChange&) = delete; + SettingChange& operator=(SettingChange&&) = delete; virtual void pop() { m_pCurSetting->restore(m_oldSetting); } @@ -55,9 +60,12 @@ inline std::unique_ptr Setting::set(const T& value) { return pChange; } -class SettingChanges : private noncopyable { +class SettingChanges { public: - SettingChanges() {} + SettingChanges() : m_settingChanges{} {} + SettingChanges(const SettingChanges&) = delete; + SettingChanges(SettingChanges&&) = default; + SettingChanges& operator=(const SettingChanges&) = delete; ~SettingChanges() { clear(); } void clear() { @@ -90,6 +98,6 @@ class SettingChanges : private noncopyable { typedef std::vector> setting_changes; setting_changes m_settingChanges; }; -} +} // namespace YAML #endif // SETTING_H_62B23520_7C8E_11DE_8A39_0800200C9A66 diff --git a/lib/yamlcpp/src/simplekey.cpp b/lib/yamlcpp/src/simplekey.cpp index 70f56b6ae42..c7a21355275 100644 --- a/lib/yamlcpp/src/simplekey.cpp +++ b/lib/yamlcpp/src/simplekey.cpp @@ -5,7 +5,7 @@ namespace YAML { struct Mark; Scanner::SimpleKey::SimpleKey(const Mark& mark_, std::size_t flowLevel_) - : mark(mark_), flowLevel(flowLevel_), pIndent(0), pMapStart(0), pKey(0) {} + : mark(mark_), flowLevel(flowLevel_), pIndent(nullptr), pMapStart(nullptr), pKey(nullptr) {} void Scanner::SimpleKey::Validate() { // Note: pIndent will *not* be garbage here; diff --git a/lib/yamlcpp/src/singledocparser.cpp b/lib/yamlcpp/src/singledocparser.cpp index a27c1c3b04d..be82741531a 100644 --- a/lib/yamlcpp/src/singledocparser.cpp +++ b/lib/yamlcpp/src/singledocparser.cpp @@ -18,6 +18,7 @@ SingleDocParser::SingleDocParser(Scanner& scanner, const Directives& directives) : m_scanner(scanner), m_directives(directives), m_pCollectionStack(new CollectionStack), + m_anchors{}, m_curAnchor(0) {} SingleDocParser::~SingleDocParser() {} @@ -71,8 +72,12 @@ void SingleDocParser::HandleNode(EventHandler& eventHandler) { } std::string tag; + std::string anchor_name; anchor_t anchor; - ParseProperties(tag, anchor); + ParseProperties(tag, anchor, anchor_name); + + if (!anchor_name.empty()) + eventHandler.OnAnchor(mark, anchor_name); const Token& token = m_scanner.peek(); @@ -166,10 +171,10 @@ void SingleDocParser::HandleBlockSequence(EventHandler& eventHandler) { // check for null if (!m_scanner.empty()) { - const Token& token = m_scanner.peek(); - if (token.type == Token::BLOCK_ENTRY || - token.type == Token::BLOCK_SEQ_END) { - eventHandler.OnNull(token.mark, NullAnchor); + const Token& nextToken = m_scanner.peek(); + if (nextToken.type == Token::BLOCK_ENTRY || + nextToken.type == Token::BLOCK_SEQ_END) { + eventHandler.OnNull(nextToken.mark, NullAnchor); continue; } } @@ -356,8 +361,10 @@ void SingleDocParser::HandleCompactMapWithNoKey(EventHandler& eventHandler) { // ParseProperties // . Grabs any tag or anchor tokens and deals with them. -void SingleDocParser::ParseProperties(std::string& tag, anchor_t& anchor) { +void SingleDocParser::ParseProperties(std::string& tag, anchor_t& anchor, + std::string& anchor_name) { tag.clear(); + anchor_name.clear(); anchor = NullAnchor; while (1) { @@ -369,7 +376,7 @@ void SingleDocParser::ParseProperties(std::string& tag, anchor_t& anchor) { ParseTag(tag); break; case Token::ANCHOR: - ParseAnchor(anchor); + ParseAnchor(anchor, anchor_name); break; default: return; @@ -387,11 +394,12 @@ void SingleDocParser::ParseTag(std::string& tag) { m_scanner.pop(); } -void SingleDocParser::ParseAnchor(anchor_t& anchor) { +void SingleDocParser::ParseAnchor(anchor_t& anchor, std::string& anchor_name) { Token& token = m_scanner.peek(); if (anchor) throw ParserException(token.mark, ErrorMsg::MULTIPLE_ANCHORS); + anchor_name = token.value; anchor = RegisterAnchor(token.value); m_scanner.pop(); } @@ -411,4 +419,4 @@ anchor_t SingleDocParser::LookupAnchor(const Mark& mark, return it->second; } -} +} // namespace YAML diff --git a/lib/yamlcpp/src/singledocparser.h b/lib/yamlcpp/src/singledocparser.h index 2b92067cddb..392453ec79d 100644 --- a/lib/yamlcpp/src/singledocparser.h +++ b/lib/yamlcpp/src/singledocparser.h @@ -12,7 +12,6 @@ #include #include "yaml-cpp/anchor.h" -#include "yaml-cpp/noncopyable.h" namespace YAML { class CollectionStack; @@ -23,9 +22,13 @@ struct Directives; struct Mark; struct Token; -class SingleDocParser : private noncopyable { +class SingleDocParser { public: SingleDocParser(Scanner& scanner, const Directives& directives); + SingleDocParser(const SingleDocParser&) = delete; + SingleDocParser(SingleDocParser&&) = delete; + SingleDocParser& operator=(const SingleDocParser&) = delete; + SingleDocParser& operator=(SingleDocParser&&) = delete; ~SingleDocParser(); void HandleDocument(EventHandler& eventHandler); @@ -43,9 +46,10 @@ class SingleDocParser : private noncopyable { void HandleCompactMap(EventHandler& eventHandler); void HandleCompactMapWithNoKey(EventHandler& eventHandler); - void ParseProperties(std::string& tag, anchor_t& anchor); + void ParseProperties(std::string& tag, anchor_t& anchor, + std::string& anchor_name); void ParseTag(std::string& tag); - void ParseAnchor(anchor_t& anchor); + void ParseAnchor(anchor_t& anchor, std::string& anchor_name); anchor_t RegisterAnchor(const std::string& name); anchor_t LookupAnchor(const Mark& mark, const std::string& name) const; @@ -60,6 +64,6 @@ class SingleDocParser : private noncopyable { anchor_t m_curAnchor; }; -} +} // namespace YAML #endif // SINGLEDOCPARSER_H_62B23520_7C8E_11DE_8A39_0800200C9A66 diff --git a/lib/yamlcpp/src/stream.cpp b/lib/yamlcpp/src/stream.cpp index 3b013cfa7d3..592802d1791 100644 --- a/lib/yamlcpp/src/stream.cpp +++ b/lib/yamlcpp/src/stream.cpp @@ -111,24 +111,15 @@ static UtfIntroState s_introTransitions[][uictMax] = { static char s_introUngetCount[][uictMax] = { // uict00, uictBB, uictBF, uictEF, uictFE, uictFF, uictAscii, uictOther - {0, 1, 1, 0, 0, 0, 0, 1}, - {0, 2, 2, 2, 2, 2, 2, 2}, - {3, 3, 3, 3, 0, 3, 3, 3}, - {4, 4, 4, 4, 4, 0, 4, 4}, - {1, 1, 1, 1, 1, 1, 1, 1}, - {1, 1, 1, 1, 1, 1, 1, 1}, - {2, 2, 2, 2, 2, 0, 2, 2}, - {2, 2, 2, 2, 0, 2, 2, 2}, - {0, 1, 1, 1, 1, 1, 1, 1}, - {0, 2, 2, 2, 2, 2, 2, 2}, - {1, 1, 1, 1, 1, 1, 1, 1}, - {1, 1, 1, 1, 1, 1, 1, 1}, - {0, 2, 2, 2, 2, 2, 2, 2}, - {0, 3, 3, 3, 3, 3, 3, 3}, - {4, 4, 4, 4, 4, 4, 4, 4}, - {2, 0, 2, 2, 2, 2, 2, 2}, - {3, 3, 0, 3, 3, 3, 3, 3}, - {1, 1, 1, 1, 1, 1, 1, 1}, + {0, 1, 1, 0, 0, 0, 0, 1}, {0, 2, 2, 2, 2, 2, 2, 2}, + {3, 3, 3, 3, 0, 3, 3, 3}, {4, 4, 4, 4, 4, 0, 4, 4}, + {1, 1, 1, 1, 1, 1, 1, 1}, {1, 1, 1, 1, 1, 1, 1, 1}, + {2, 2, 2, 2, 2, 0, 2, 2}, {2, 2, 2, 2, 0, 2, 2, 2}, + {0, 1, 1, 1, 1, 1, 1, 1}, {0, 2, 2, 2, 2, 2, 2, 2}, + {1, 1, 1, 1, 1, 1, 1, 1}, {1, 1, 1, 1, 1, 1, 1, 1}, + {0, 2, 2, 2, 2, 2, 2, 2}, {0, 3, 3, 3, 3, 3, 3, 3}, + {4, 4, 4, 4, 4, 4, 4, 4}, {2, 0, 2, 2, 2, 2, 2, 2}, + {3, 3, 0, 3, 3, 3, 3, 3}, {1, 1, 1, 1, 1, 1, 1, 1}, }; inline UtfIntroCharType IntroCharTypeOf(std::istream::int_type ch) { @@ -192,6 +183,9 @@ inline void QueueUnicodeCodepoint(std::deque& q, unsigned long ch) { Stream::Stream(std::istream& input) : m_input(input), + m_mark{}, + m_charSet{}, + m_readahead{}, m_pPrefetched(new unsigned char[YAML_PREFETCH_SIZE]), m_nPrefetchedAvailable(0), m_nPrefetchedUsed(0) { @@ -445,4 +439,4 @@ void Stream::StreamInUtf32() const { QueueUnicodeCodepoint(m_readahead, ch); } -} +} // namespace YAML diff --git a/lib/yamlcpp/src/stream.h b/lib/yamlcpp/src/stream.h index 42d542d5b16..ff5149e970c 100644 --- a/lib/yamlcpp/src/stream.h +++ b/lib/yamlcpp/src/stream.h @@ -7,7 +7,6 @@ #pragma once #endif -#include "yaml-cpp/noncopyable.h" #include "yaml-cpp/mark.h" #include #include @@ -17,11 +16,15 @@ #include namespace YAML { -class Stream : private noncopyable { +class Stream { public: friend class StreamCharSource; Stream(std::istream& input); + Stream(const Stream&) = delete; + Stream(Stream&&) = delete; + Stream& operator=(const Stream&) = delete; + Stream& operator=(Stream&&) = delete; ~Stream(); operator bool() const; @@ -71,6 +74,6 @@ inline bool Stream::ReadAheadTo(size_t i) const { return true; return _ReadAheadTo(i); } -} +} // namespace YAML #endif // STREAM_H_62B23520_7C8E_11DE_8A39_0800200C9A66 diff --git a/lib/yamlcpp/src/streamcharsource.h b/lib/yamlcpp/src/streamcharsource.h index 624599e65da..4b6a143ea37 100644 --- a/lib/yamlcpp/src/streamcharsource.h +++ b/lib/yamlcpp/src/streamcharsource.h @@ -7,7 +7,6 @@ #pragma once #endif -#include "yaml-cpp/noncopyable.h" #include namespace YAML { @@ -16,6 +15,9 @@ class StreamCharSource { StreamCharSource(const Stream& stream) : m_offset(0), m_stream(stream) {} StreamCharSource(const StreamCharSource& source) : m_offset(source.m_offset), m_stream(source.m_stream) {} + StreamCharSource(StreamCharSource&&) = default; + StreamCharSource& operator=(const StreamCharSource&) = delete; + StreamCharSource& operator=(StreamCharSource&&) = delete; ~StreamCharSource() {} operator bool() const; @@ -27,8 +29,6 @@ class StreamCharSource { private: std::size_t m_offset; const Stream& m_stream; - - StreamCharSource& operator=(const StreamCharSource&); // non-assignable }; inline StreamCharSource::operator bool() const { @@ -43,6 +43,6 @@ inline const StreamCharSource StreamCharSource::operator+(int i) const { source.m_offset = 0; return source; } -} +} // namespace YAML #endif // STREAMCHARSOURCE_H_62B23520_7C8E_11DE_8A39_0800200C9A66 diff --git a/lib/yamlcpp/src/tag.cpp b/lib/yamlcpp/src/tag.cpp index 51435520e46..df8a2cf4613 100644 --- a/lib/yamlcpp/src/tag.cpp +++ b/lib/yamlcpp/src/tag.cpp @@ -6,7 +6,8 @@ #include "token.h" namespace YAML { -Tag::Tag(const Token& token) : type(static_cast(token.data)) { +Tag::Tag(const Token& token) + : type(static_cast(token.data)), handle{}, value{} { switch (type) { case VERBATIM: value = token.value; @@ -46,4 +47,4 @@ const std::string Tag::Translate(const Directives& directives) { } throw std::runtime_error("yaml-cpp: internal error, bad tag type"); } -} +} // namespace YAML diff --git a/lib/yamlcpp/src/token.h b/lib/yamlcpp/src/token.h index ad0b7d0a005..9a27f3d6c12 100644 --- a/lib/yamlcpp/src/token.h +++ b/lib/yamlcpp/src/token.h @@ -14,10 +14,11 @@ namespace YAML { const std::string TokenNames[] = { - "DIRECTIVE", "DOC_START", "DOC_END", "BLOCK_SEQ_START", "BLOCK_MAP_START", - "BLOCK_SEQ_END", "BLOCK_MAP_END", "BLOCK_ENTRY", "FLOW_SEQ_START", - "FLOW_MAP_START", "FLOW_SEQ_END", "FLOW_MAP_END", "FLOW_MAP_COMPACT", - "FLOW_ENTRY", "KEY", "VALUE", "ANCHOR", "ALIAS", "TAG", "SCALAR"}; + "DIRECTIVE", "DOC_START", "DOC_END", "BLOCK_SEQ_START", + "BLOCK_MAP_START", "BLOCK_SEQ_END", "BLOCK_MAP_END", "BLOCK_ENTRY", + "FLOW_SEQ_START", "FLOW_MAP_START", "FLOW_SEQ_END", "FLOW_MAP_END", + "FLOW_MAP_COMPACT", "FLOW_ENTRY", "KEY", "VALUE", + "ANCHOR", "ALIAS", "TAG", "SCALAR"}; struct Token { // enums @@ -48,7 +49,7 @@ struct Token { // data Token(TYPE type_, const Mark& mark_) - : status(VALID), type(type_), mark(mark_), data(0) {} + : status(VALID), type(type_), mark(mark_), value{}, params{}, data(0) {} friend std::ostream& operator<<(std::ostream& out, const Token& token) { out << TokenNames[token.type] << std::string(": ") << token.value; @@ -64,6 +65,6 @@ struct Token { std::vector params; int data; }; -} +} // namespace YAML #endif // TOKEN_H_62B23520_7C8E_11DE_8A39_0800200C9A66 diff --git a/lib/yamlcpp/test/CMakeLists.txt b/lib/yamlcpp/test/CMakeLists.txt index 3633da578b3..0a669d5cc86 100644 --- a/lib/yamlcpp/test/CMakeLists.txt +++ b/lib/yamlcpp/test/CMakeLists.txt @@ -1,26 +1,39 @@ -set(gtest_force_shared_crt ${MSVC_SHARED_RT} CACHE BOOL - "Use shared (DLL) run-time lib even when Google Test built as a static lib.") -add_subdirectory(gtest-1.8.0) -include_directories(SYSTEM gtest-1.8.0/googlemock/include) -include_directories(SYSTEM gtest-1.8.0/googletest/include) - -if(WIN32 AND BUILD_SHARED_LIBS) - add_definitions("-DGTEST_LINKED_AS_SHARED_LIBRARY") +include(ExternalProject) + +if(MSVC) + # MS Visual Studio expects lib prefix on static libraries, + # but CMake compiles them without prefix + # See https://gitlab.kitware.com/cmake/cmake/issues/17338 + set(CMAKE_STATIC_LIBRARY_PREFIX "") endif() +ExternalProject_Add( + googletest_project + SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/gtest-1.8.0" + INSTALL_DIR "${CMAKE_CURRENT_BINARY_DIR}/prefix" + CMAKE_ARGS + -DCMAKE_INSTALL_PREFIX:PATH= + -DBUILD_GMOCK=ON + -Dgtest_force_shared_crt=ON +) + +add_library(gmock UNKNOWN IMPORTED) +set_target_properties(gmock PROPERTIES + IMPORTED_LOCATION + ${PROJECT_BINARY_DIR}/test/prefix/lib/${CMAKE_STATIC_LIBRARY_PREFIX}gmock${CMAKE_STATIC_LIBRARY_SUFFIX} +) + +find_package(Threads) + +include_directories(SYSTEM "${PROJECT_BINARY_DIR}/test/prefix/include") + if(CMAKE_CXX_COMPILER_ID MATCHES "GNU" OR - CMAKE_CXX_COMPILER_ID MATCHES "Clang") - set(yaml_test_flags "-Wno-variadic-macros -Wno-sign-compare") - - if(CMAKE_CXX_COMPILER_ID MATCHES "Clang") - set(yaml_test_flags "${yaml_test_flags} -Wno-c99-extensions") - endif() - - if(CMAKE_COMPILER_IS_GNUCXX) - set(yaml_test_flags "${yaml_test_flags} -std=gnu++11") - else() - set(yaml_test_flags "${yaml_test_flags} -std=c++11") - endif() + CMAKE_CXX_COMPILER_ID MATCHES "Clang") + set(yaml_test_flags "-Wno-variadic-macros -Wno-sign-compare") + + if(CMAKE_CXX_COMPILER_ID MATCHES "Clang") + set(yaml_test_flags "${yaml_test_flags} -Wno-c99-extensions") + endif() endif() file(GLOB test_headers [a-z_]*.h) @@ -30,15 +43,27 @@ file(GLOB test_new_api_sources new-api/[a-z]*.cpp) list(APPEND test_sources ${test_new_api_sources}) add_sources(${test_sources} ${test_headers}) +include_directories(${YAML_CPP_SOURCE_DIR}/src) include_directories(${YAML_CPP_SOURCE_DIR}/test) add_executable(run-tests - ${test_sources} - ${test_headers} + ${test_sources} + ${test_headers} +) + +set_target_properties(run-tests PROPERTIES + CXX_STANDARD 11 + CXX_STANDARD_REQUIRED ON ) + +add_dependencies(run-tests googletest_project) + set_target_properties(run-tests PROPERTIES - COMPILE_FLAGS "${yaml_c_flags} ${yaml_cxx_flags} ${yaml_test_flags}" + COMPILE_FLAGS "${yaml_c_flags} ${yaml_cxx_flags} ${yaml_test_flags}" ) -target_link_libraries(run-tests yaml-cpp gmock) +target_link_libraries(run-tests + yaml-cpp + gmock + ${CMAKE_THREAD_LIBS_INIT}) add_test(yaml-test ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/run-tests) diff --git a/lib/yamlcpp/test/create-emitter-tests.py b/lib/yamlcpp/test/create-emitter-tests.py index a6bf3a4dcec..e202522cee9 100644 --- a/lib/yamlcpp/test/create-emitter-tests.py +++ b/lib/yamlcpp/test/create-emitter-tests.py @@ -35,10 +35,12 @@ def doc_end(implicit=False): def scalar(value, tag='', anchor='', anchor_id=0): emit = [] + handle = [] if tag: emit += ['VerbatimTag("%s")' % encode(tag)] if anchor: emit += ['Anchor("%s")' % encode(anchor)] + handle += ['OnAnchor(_, "%s")' % encode(anchor)] if tag: out_tag = encode(tag) else: @@ -47,39 +49,46 @@ def scalar(value, tag='', anchor='', anchor_id=0): else: out_tag = '!' emit += ['"%s"' % encode(value)] - return {'emit': emit, 'handle': 'OnScalar(_, "%s", %s, "%s")' % (out_tag, anchor_id, encode(value))} + handle += ['OnScalar(_, "%s", %s, "%s")' % (out_tag, anchor_id, encode(value))] + return {'emit': emit, 'handle': handle} def comment(value): return {'emit': 'Comment("%s")' % value, 'handle': ''} def seq_start(tag='', anchor='', anchor_id=0, style='_'): emit = [] + handle = [] if tag: emit += ['VerbatimTag("%s")' % encode(tag)] if anchor: emit += ['Anchor("%s")' % encode(anchor)] + handle += ['OnAnchor(_, "%s")' % encode(anchor)] if tag: out_tag = encode(tag) else: out_tag = '?' emit += ['BeginSeq'] - return {'emit': emit, 'handle': 'OnSequenceStart(_, "%s", %s, %s)' % (out_tag, anchor_id, style)} + handle += ['OnSequenceStart(_, "%s", %s, %s)' % (out_tag, anchor_id, style)] + return {'emit': emit, 'handle': handle} def seq_end(): return {'emit': 'EndSeq', 'handle': 'OnSequenceEnd()'} def map_start(tag='', anchor='', anchor_id=0, style='_'): emit = [] + handle = [] if tag: emit += ['VerbatimTag("%s")' % encode(tag)] if anchor: emit += ['Anchor("%s")' % encode(anchor)] + handle += ['OnAnchor(_, "%s")' % encode(anchor)] if tag: out_tag = encode(tag) else: out_tag = '?' emit += ['BeginMap'] - return {'emit': emit, 'handle': 'OnMapStart(_, "%s", %s, %s)' % (out_tag, anchor_id, style)} + handle += ['OnMapStart(_, "%s", %s, %s)' % (out_tag, anchor_id, style)] + return {'emit': emit, 'handle': handle} def map_end(): return {'emit': 'EndMap', 'handle': 'OnMapEnd()'} @@ -125,7 +134,6 @@ def expand(template): for cdr in expand(template[1:]): yield car + cdr - def gen_events(): for template in gen_templates(): for events in expand(template): @@ -202,7 +210,10 @@ def create_emitter_tests(out): out.writeln('') for event in test['events']: handle = event['handle'] - if handle: + if isinstance(handle, list): + for e in handle: + out.writeln('EXPECT_CALL(handler, %s);' % e) + elif handle: out.writeln('EXPECT_CALL(handler, %s);' % handle) out.writeln('Parse(out.c_str());') out.writeln('') diff --git a/lib/yamlcpp/test/integration/emitter_test.cpp b/lib/yamlcpp/test/integration/emitter_test.cpp index 27808380d5a..3a5783bffcf 100644 --- a/lib/yamlcpp/test/integration/emitter_test.cpp +++ b/lib/yamlcpp/test/integration/emitter_test.cpp @@ -253,8 +253,9 @@ TEST_F(EmitterTest, ScalarFormat) { out << DoubleQuoted << "explicit double-quoted scalar"; out << "auto-detected\ndouble-quoted scalar"; out << "a non-\"auto-detected\" double-quoted scalar"; - out << Literal << "literal scalar\nthat may span\nmany, many\nlines " - "and have \"whatever\" crazy\tsymbols that we like"; + out << Literal + << "literal scalar\nthat may span\nmany, many\nlines " + "and have \"whatever\" crazy\tsymbols that we like"; out << EndSeq; ExpectEmit( @@ -526,9 +527,10 @@ TEST_F(EmitterTest, SimpleComment) { TEST_F(EmitterTest, MultiLineComment) { out << BeginSeq; - out << "item 1" << Comment( - "really really long\ncomment that couldn't " - "possibly\nfit on one line"); + out << "item 1" + << Comment( + "really really long\ncomment that couldn't " + "possibly\nfit on one line"); out << "item 2"; out << EndSeq; @@ -901,18 +903,18 @@ TEST_F(EmitterTest, SingleChar) { TEST_F(EmitterTest, DefaultPrecision) { out << BeginSeq; - out << 1.234f; - out << 3.14159265358979; + out << 1.3125f; + out << 1.23455810546875; out << EndSeq; - ExpectEmit("- 1.234\n- 3.14159265358979"); + ExpectEmit("- 1.3125\n- 1.23455810546875"); } TEST_F(EmitterTest, SetPrecision) { out << BeginSeq; - out << FloatPrecision(3) << 1.234f; - out << DoublePrecision(6) << 3.14159265358979; + out << FloatPrecision(3) << 1.3125f; + out << DoublePrecision(6) << 1.23455810546875; out << EndSeq; - ExpectEmit("- 1.23\n- 3.14159"); + ExpectEmit("- 1.31\n- 1.23456"); } TEST_F(EmitterTest, DashInBlockContext) { @@ -984,6 +986,45 @@ TEST_F(EmitterTest, ValueOfBackslash) { ExpectEmit("foo: \"\\\\\""); } +TEST_F(EmitterTest, Infinity) { + out << YAML::BeginMap; + out << YAML::Key << "foo" << YAML::Value + << std::numeric_limits::infinity(); + out << YAML::Key << "bar" << YAML::Value + << std::numeric_limits::infinity(); + out << YAML::EndMap; + + ExpectEmit( + "foo: .inf\n" + "bar: .inf"); +} + +TEST_F(EmitterTest, NegInfinity) { + out << YAML::BeginMap; + out << YAML::Key << "foo" << YAML::Value + << -std::numeric_limits::infinity(); + out << YAML::Key << "bar" << YAML::Value + << -std::numeric_limits::infinity(); + out << YAML::EndMap; + + ExpectEmit( + "foo: -.inf\n" + "bar: -.inf"); +} + +TEST_F(EmitterTest, NaN) { + out << YAML::BeginMap; + out << YAML::Key << "foo" << YAML::Value + << std::numeric_limits::quiet_NaN(); + out << YAML::Key << "bar" << YAML::Value + << std::numeric_limits::quiet_NaN(); + out << YAML::EndMap; + + ExpectEmit( + "foo: .nan\n" + "bar: .nan"); +} + class EmitterErrorTest : public ::testing::Test { protected: void ExpectEmitError(const std::string& expectedError) { @@ -1034,5 +1075,5 @@ TEST_F(EmitterErrorTest, InvalidAlias) { ExpectEmitError(ErrorMsg::INVALID_ALIAS); } -} -} +} // namespace +} // namespace YAML diff --git a/lib/yamlcpp/test/integration/error_messages_test.cpp b/lib/yamlcpp/test/integration/error_messages_test.cpp new file mode 100644 index 00000000000..ea921ce364e --- /dev/null +++ b/lib/yamlcpp/test/integration/error_messages_test.cpp @@ -0,0 +1,62 @@ +#include "yaml-cpp/yaml.h" // IWYU pragma: keep + +#include "gtest/gtest.h" + +#define EXPECT_THROW_EXCEPTION(exception_type, statement, message) \ + ASSERT_THROW(statement, exception_type); \ + try { \ + statement; \ + } catch (const exception_type &e) { \ + EXPECT_EQ(e.msg, message); \ + } + +namespace YAML { +namespace { + +TEST(ErrorMessageTest, BadSubscriptErrorMessage) { + const char *example_yaml = + "first:\n" + " second: 1\n" + " third: 2\n"; + + Node doc = Load(example_yaml); + + // Test that printable key is part of error message + EXPECT_THROW_EXCEPTION(YAML::BadSubscript, doc["first"]["second"]["fourth"], + "operator[] call on a scalar (key: \"fourth\")"); + + EXPECT_THROW_EXCEPTION(YAML::BadSubscript, doc["first"]["second"][37], + "operator[] call on a scalar (key: \"37\")"); + + // Non-printable key is not included in error message + EXPECT_THROW_EXCEPTION(YAML::BadSubscript, + doc["first"]["second"][std::vector()], + "operator[] call on a scalar"); + + EXPECT_THROW_EXCEPTION(YAML::BadSubscript, doc["first"]["second"][Node()], + "operator[] call on a scalar"); +} + +TEST(ErrorMessageTest, Ex9_1_InvalidNodeErrorMessage) { + const char *example_yaml = + "first:\n" + " second: 1\n" + " third: 2\n"; + + const Node doc = Load(example_yaml); + + // Test that printable key is part of error message + EXPECT_THROW_EXCEPTION(YAML::InvalidNode, doc["first"]["fourth"].as(), + "invalid node; first invalid key: \"fourth\""); + + EXPECT_THROW_EXCEPTION(YAML::InvalidNode, doc["first"][37].as(), + "invalid node; first invalid key: \"37\""); + + // Non-printable key is not included in error message + EXPECT_THROW_EXCEPTION(YAML::InvalidNode, + doc["first"][std::vector()].as(), + "invalid node; this may result from using a map " + "iterator as a sequence iterator, or vice-versa"); +} +} // namespace +} // namespace YAML diff --git a/lib/yamlcpp/test/integration/gen_emitter_test.cpp b/lib/yamlcpp/test/integration/gen_emitter_test.cpp index e44eee6da7d..35361447423 100644 --- a/lib/yamlcpp/test/integration/gen_emitter_test.cpp +++ b/lib/yamlcpp/test/integration/gen_emitter_test.cpp @@ -10,7 +10,7 @@ namespace { typedef HandlerTest GenEmitterTest; -TEST_F(GenEmitterTest, testbf4e63edf2258c91fb88) { +TEST_F(GenEmitterTest, testf2a8b6e6359fb2c30830) { Emitter out; out << Comment("comment"); out << BeginDoc; @@ -23,7 +23,7 @@ TEST_F(GenEmitterTest, testbf4e63edf2258c91fb88) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test8c2aa26989357a4c8d2d) { +TEST_F(GenEmitterTest, testa2c9c04eab06a05bf1a3) { Emitter out; out << BeginDoc; out << Comment("comment"); @@ -36,7 +36,7 @@ TEST_F(GenEmitterTest, test8c2aa26989357a4c8d2d) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, testf8818f97591e2c51179c) { +TEST_F(GenEmitterTest, testc5fae995bf84b2f62627) { Emitter out; out << BeginDoc; out << "foo"; @@ -49,7 +49,7 @@ TEST_F(GenEmitterTest, testf8818f97591e2c51179c) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test2b9d697f1ec84bdc484f) { +TEST_F(GenEmitterTest, test208b4fb7399a936fce93) { Emitter out; out << BeginDoc; out << "foo"; @@ -62,7 +62,7 @@ TEST_F(GenEmitterTest, test2b9d697f1ec84bdc484f) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test969d8cf1535db02242b4) { +TEST_F(GenEmitterTest, test402085442ada9788bc4e) { Emitter out; out << Comment("comment"); out << BeginDoc; @@ -74,7 +74,7 @@ TEST_F(GenEmitterTest, test969d8cf1535db02242b4) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test4d16d2c638f0b1131d42) { +TEST_F(GenEmitterTest, test279346c761f7d9a10aec) { Emitter out; out << BeginDoc; out << Comment("comment"); @@ -86,7 +86,7 @@ TEST_F(GenEmitterTest, test4d16d2c638f0b1131d42) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test3bdad9a4ffa67cc4201b) { +TEST_F(GenEmitterTest, test386f6766d57a48ccb316) { Emitter out; out << BeginDoc; out << "foo"; @@ -98,7 +98,7 @@ TEST_F(GenEmitterTest, test3bdad9a4ffa67cc4201b) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, testa57103d877a04b0da3c9) { +TEST_F(GenEmitterTest, test989baa41ede860374193) { Emitter out; out << BeginDoc; out << "foo"; @@ -110,7 +110,7 @@ TEST_F(GenEmitterTest, testa57103d877a04b0da3c9) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, testf838cbd6db90346652d6) { +TEST_F(GenEmitterTest, test718fa11b9bfa9dfc6632) { Emitter out; out << Comment("comment"); out << BeginDoc; @@ -123,7 +123,7 @@ TEST_F(GenEmitterTest, testf838cbd6db90346652d6) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, teste65456c6070d7ed9b292) { +TEST_F(GenEmitterTest, test7986c74c7cab2ff062e7) { Emitter out; out << BeginDoc; out << Comment("comment"); @@ -136,7 +136,7 @@ TEST_F(GenEmitterTest, teste65456c6070d7ed9b292) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test365273601c89ebaeec61) { +TEST_F(GenEmitterTest, test1a432be0760ebcf72dde) { Emitter out; out << BeginDoc; out << "foo\n"; @@ -149,7 +149,7 @@ TEST_F(GenEmitterTest, test365273601c89ebaeec61) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test92d67b382f78c6a58c2a) { +TEST_F(GenEmitterTest, test9b4714c8c6dd71f963f1) { Emitter out; out << BeginDoc; out << "foo\n"; @@ -162,7 +162,7 @@ TEST_F(GenEmitterTest, test92d67b382f78c6a58c2a) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test49e0bb235c344722e0df) { +TEST_F(GenEmitterTest, test59d039102f43b05233b2) { Emitter out; out << Comment("comment"); out << BeginDoc; @@ -174,7 +174,7 @@ TEST_F(GenEmitterTest, test49e0bb235c344722e0df) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test3010c495cd1c61d1ccf2) { +TEST_F(GenEmitterTest, test15371be5fc126b3601ee) { Emitter out; out << BeginDoc; out << Comment("comment"); @@ -186,7 +186,7 @@ TEST_F(GenEmitterTest, test3010c495cd1c61d1ccf2) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test22e48c3bc91b32853688) { +TEST_F(GenEmitterTest, test5a2a5702a41d71567a10) { Emitter out; out << BeginDoc; out << "foo\n"; @@ -198,7 +198,7 @@ TEST_F(GenEmitterTest, test22e48c3bc91b32853688) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test03e42bee2a2c6ffc1dd8) { +TEST_F(GenEmitterTest, test7975acd31f55f66c21a9) { Emitter out; out << BeginDoc; out << "foo\n"; @@ -210,7 +210,7 @@ TEST_F(GenEmitterTest, test03e42bee2a2c6ffc1dd8) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test9662984f64ea0b79b267) { +TEST_F(GenEmitterTest, test9ab358e41e3af0e1852c) { Emitter out; out << Comment("comment"); out << BeginDoc; @@ -224,7 +224,7 @@ TEST_F(GenEmitterTest, test9662984f64ea0b79b267) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, testf3867ffaec6663c515ff) { +TEST_F(GenEmitterTest, test6571b17e1089f3f34d41) { Emitter out; out << BeginDoc; out << Comment("comment"); @@ -238,7 +238,7 @@ TEST_F(GenEmitterTest, testf3867ffaec6663c515ff) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, testfd8783233e21636f7f12) { +TEST_F(GenEmitterTest, test7c8476d0a02eeab3326f) { Emitter out; out << BeginDoc; out << VerbatimTag("tag"); @@ -252,7 +252,7 @@ TEST_F(GenEmitterTest, testfd8783233e21636f7f12) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test3fc20508ecea0f4cb165) { +TEST_F(GenEmitterTest, test0883fa5d170d96324325) { Emitter out; out << BeginDoc; out << VerbatimTag("tag"); @@ -266,7 +266,7 @@ TEST_F(GenEmitterTest, test3fc20508ecea0f4cb165) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, teste120c09230c813be6c30) { +TEST_F(GenEmitterTest, test7f44d870f57878e83749) { Emitter out; out << Comment("comment"); out << BeginDoc; @@ -279,7 +279,7 @@ TEST_F(GenEmitterTest, teste120c09230c813be6c30) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test835d37d226cbacaa4b2d) { +TEST_F(GenEmitterTest, testc7b8d9af2a71da438220) { Emitter out; out << BeginDoc; out << Comment("comment"); @@ -292,7 +292,7 @@ TEST_F(GenEmitterTest, test835d37d226cbacaa4b2d) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test7a26848396e9291bf1f1) { +TEST_F(GenEmitterTest, testa27a4f0174aee7622160) { Emitter out; out << BeginDoc; out << VerbatimTag("tag"); @@ -305,7 +305,7 @@ TEST_F(GenEmitterTest, test7a26848396e9291bf1f1) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test34a821220a5e1441f553) { +TEST_F(GenEmitterTest, testf06e77dc66bc51682e57) { Emitter out; out << BeginDoc; out << VerbatimTag("tag"); @@ -318,7 +318,7 @@ TEST_F(GenEmitterTest, test34a821220a5e1441f553) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test53e5179db889a79c3ea2) { +TEST_F(GenEmitterTest, test28c636f42558c217d90b) { Emitter out; out << Comment("comment"); out << BeginDoc; @@ -327,12 +327,13 @@ TEST_F(GenEmitterTest, test53e5179db889a79c3ea2) { out << EndDoc; EXPECT_CALL(handler, OnDocumentStart(_)); + EXPECT_CALL(handler, OnAnchor(_, "anchor")); EXPECT_CALL(handler, OnScalar(_, "?", 1, "foo")); EXPECT_CALL(handler, OnDocumentEnd()); Parse(out.c_str()); } -TEST_F(GenEmitterTest, testb8450c68977e0df66c5b) { +TEST_F(GenEmitterTest, testa8e930c2f4f761519825) { Emitter out; out << BeginDoc; out << Comment("comment"); @@ -341,12 +342,13 @@ TEST_F(GenEmitterTest, testb8450c68977e0df66c5b) { out << EndDoc; EXPECT_CALL(handler, OnDocumentStart(_)); + EXPECT_CALL(handler, OnAnchor(_, "anchor")); EXPECT_CALL(handler, OnScalar(_, "?", 1, "foo")); EXPECT_CALL(handler, OnDocumentEnd()); Parse(out.c_str()); } -TEST_F(GenEmitterTest, teste0277d1ed537e53294b4) { +TEST_F(GenEmitterTest, testc73b721f492b45035034) { Emitter out; out << BeginDoc; out << Anchor("anchor"); @@ -355,12 +357,13 @@ TEST_F(GenEmitterTest, teste0277d1ed537e53294b4) { out << EndDoc; EXPECT_CALL(handler, OnDocumentStart(_)); + EXPECT_CALL(handler, OnAnchor(_, "anchor")); EXPECT_CALL(handler, OnScalar(_, "?", 1, "foo")); EXPECT_CALL(handler, OnDocumentEnd()); Parse(out.c_str()); } -TEST_F(GenEmitterTest, testd6ebe62492bf8757ddde) { +TEST_F(GenEmitterTest, testb401b54145c71ea07848) { Emitter out; out << BeginDoc; out << Anchor("anchor"); @@ -369,12 +372,13 @@ TEST_F(GenEmitterTest, testd6ebe62492bf8757ddde) { out << Comment("comment"); EXPECT_CALL(handler, OnDocumentStart(_)); + EXPECT_CALL(handler, OnAnchor(_, "anchor")); EXPECT_CALL(handler, OnScalar(_, "?", 1, "foo")); EXPECT_CALL(handler, OnDocumentEnd()); Parse(out.c_str()); } -TEST_F(GenEmitterTest, test56c67a81a5989623dad7) { +TEST_F(GenEmitterTest, test380d9af0ae2e27279526) { Emitter out; out << Comment("comment"); out << BeginDoc; @@ -382,12 +386,13 @@ TEST_F(GenEmitterTest, test56c67a81a5989623dad7) { out << "foo"; EXPECT_CALL(handler, OnDocumentStart(_)); + EXPECT_CALL(handler, OnAnchor(_, "anchor")); EXPECT_CALL(handler, OnScalar(_, "?", 1, "foo")); EXPECT_CALL(handler, OnDocumentEnd()); Parse(out.c_str()); } -TEST_F(GenEmitterTest, testea4c45819b88c22d02b6) { +TEST_F(GenEmitterTest, test6969308096c6106d1f55) { Emitter out; out << BeginDoc; out << Comment("comment"); @@ -395,12 +400,13 @@ TEST_F(GenEmitterTest, testea4c45819b88c22d02b6) { out << "foo"; EXPECT_CALL(handler, OnDocumentStart(_)); + EXPECT_CALL(handler, OnAnchor(_, "anchor")); EXPECT_CALL(handler, OnScalar(_, "?", 1, "foo")); EXPECT_CALL(handler, OnDocumentEnd()); Parse(out.c_str()); } -TEST_F(GenEmitterTest, testfa05ed7573dd54074344) { +TEST_F(GenEmitterTest, testa8afc0036fffa3b2d185) { Emitter out; out << BeginDoc; out << Anchor("anchor"); @@ -408,12 +414,13 @@ TEST_F(GenEmitterTest, testfa05ed7573dd54074344) { out << Comment("comment"); EXPECT_CALL(handler, OnDocumentStart(_)); + EXPECT_CALL(handler, OnAnchor(_, "anchor")); EXPECT_CALL(handler, OnScalar(_, "?", 1, "foo")); EXPECT_CALL(handler, OnDocumentEnd()); Parse(out.c_str()); } -TEST_F(GenEmitterTest, test52431165a20aa2a085dc) { +TEST_F(GenEmitterTest, test7b41f0a32b90bf5f138d) { Emitter out; out << BeginDoc; out << Anchor("anchor"); @@ -421,12 +428,13 @@ TEST_F(GenEmitterTest, test52431165a20aa2a085dc) { out << Comment("comment"); EXPECT_CALL(handler, OnDocumentStart(_)); + EXPECT_CALL(handler, OnAnchor(_, "anchor")); EXPECT_CALL(handler, OnScalar(_, "?", 1, "foo")); EXPECT_CALL(handler, OnDocumentEnd()); Parse(out.c_str()); } -TEST_F(GenEmitterTest, test2e1bf781941755fc5944) { +TEST_F(GenEmitterTest, test99b1e0027d74c641f4fc) { Emitter out; out << Comment("comment"); out << "foo"; @@ -438,7 +446,7 @@ TEST_F(GenEmitterTest, test2e1bf781941755fc5944) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test5405b9f863e524bb3e81) { +TEST_F(GenEmitterTest, test8e45fdb3ff3c00d56f27) { Emitter out; out << Comment("comment"); out << "foo"; @@ -450,7 +458,7 @@ TEST_F(GenEmitterTest, test5405b9f863e524bb3e81) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test0a7d85109d068170e547) { +TEST_F(GenEmitterTest, testf898ade0c92d48d498cf) { Emitter out; out << "foo"; out << Comment("comment"); @@ -462,7 +470,7 @@ TEST_F(GenEmitterTest, test0a7d85109d068170e547) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, testba8dc6889d6983fb0f05) { +TEST_F(GenEmitterTest, test3eb11fe6897b9638b90c) { Emitter out; out << "foo"; out << EndDoc; @@ -474,7 +482,7 @@ TEST_F(GenEmitterTest, testba8dc6889d6983fb0f05) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, testd8743fc1225fef185b69) { +TEST_F(GenEmitterTest, test4e7428248511a461fdae) { Emitter out; out << Comment("comment"); out << "foo"; @@ -485,7 +493,7 @@ TEST_F(GenEmitterTest, testd8743fc1225fef185b69) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, testc2f808fe5fb8b2970b89) { +TEST_F(GenEmitterTest, testb811cba8e9f57399cd40) { Emitter out; out << Comment("comment"); out << "foo"; @@ -496,7 +504,7 @@ TEST_F(GenEmitterTest, testc2f808fe5fb8b2970b89) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test984d0572a31be4451efc) { +TEST_F(GenEmitterTest, testc625669ef35d9165757f) { Emitter out; out << "foo"; out << Comment("comment"); @@ -507,7 +515,7 @@ TEST_F(GenEmitterTest, test984d0572a31be4451efc) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, testa3883cf6b7f84c32ba99) { +TEST_F(GenEmitterTest, test0bc005214f48707274f7) { Emitter out; out << "foo"; out << Comment("comment"); @@ -518,7 +526,7 @@ TEST_F(GenEmitterTest, testa3883cf6b7f84c32ba99) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test1fe1f2d242b3a00c5f83) { +TEST_F(GenEmitterTest, testdccc5288dfb2f680dd82) { Emitter out; out << Comment("comment"); out << "foo\n"; @@ -530,7 +538,7 @@ TEST_F(GenEmitterTest, test1fe1f2d242b3a00c5f83) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test80e82792ed68bb0cadbc) { +TEST_F(GenEmitterTest, test0a928620b149d5644a3b) { Emitter out; out << Comment("comment"); out << "foo\n"; @@ -542,7 +550,7 @@ TEST_F(GenEmitterTest, test80e82792ed68bb0cadbc) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test6756b87f08499449fd53) { +TEST_F(GenEmitterTest, test72bf9f8ba5207fb041c3) { Emitter out; out << "foo\n"; out << Comment("comment"); @@ -554,7 +562,7 @@ TEST_F(GenEmitterTest, test6756b87f08499449fd53) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test7d768a7e214b2e791928) { +TEST_F(GenEmitterTest, test39ba33ec287e431e70d0) { Emitter out; out << "foo\n"; out << EndDoc; @@ -566,7 +574,7 @@ TEST_F(GenEmitterTest, test7d768a7e214b2e791928) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test73470e304962e94c82ee) { +TEST_F(GenEmitterTest, testce39fe4e650942b617c6) { Emitter out; out << Comment("comment"); out << "foo\n"; @@ -577,7 +585,7 @@ TEST_F(GenEmitterTest, test73470e304962e94c82ee) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test220fcaca9f58ed63ab66) { +TEST_F(GenEmitterTest, test680e99eab986e1cdac01) { Emitter out; out << Comment("comment"); out << "foo\n"; @@ -588,7 +596,7 @@ TEST_F(GenEmitterTest, test220fcaca9f58ed63ab66) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test7e4c037d370d52aa4da4) { +TEST_F(GenEmitterTest, teste6e7f73dfac0048154af) { Emitter out; out << "foo\n"; out << Comment("comment"); @@ -599,7 +607,7 @@ TEST_F(GenEmitterTest, test7e4c037d370d52aa4da4) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test79a2ffc6c8161726f1ed) { +TEST_F(GenEmitterTest, test19e2c91493d21a389511) { Emitter out; out << "foo\n"; out << Comment("comment"); @@ -610,7 +618,7 @@ TEST_F(GenEmitterTest, test79a2ffc6c8161726f1ed) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test2a634546fd8c4b92ad18) { +TEST_F(GenEmitterTest, testc15cdbdbf9661c853def) { Emitter out; out << Comment("comment"); out << VerbatimTag("tag"); @@ -623,7 +631,7 @@ TEST_F(GenEmitterTest, test2a634546fd8c4b92ad18) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test84a311c6ca4fe200eff5) { +TEST_F(GenEmitterTest, testa349878c464f03fa6d4e) { Emitter out; out << Comment("comment"); out << VerbatimTag("tag"); @@ -636,7 +644,7 @@ TEST_F(GenEmitterTest, test84a311c6ca4fe200eff5) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, testef05b48cc1f9318b612f) { +TEST_F(GenEmitterTest, testc063a846f87b1e20e4e9) { Emitter out; out << VerbatimTag("tag"); out << "foo"; @@ -649,7 +657,7 @@ TEST_F(GenEmitterTest, testef05b48cc1f9318b612f) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, testa77250518abd6e019ab8) { +TEST_F(GenEmitterTest, testdfb3a9ec6da3b792f392) { Emitter out; out << VerbatimTag("tag"); out << "foo"; @@ -662,7 +670,7 @@ TEST_F(GenEmitterTest, testa77250518abd6e019ab8) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test3e9c6f05218917c77c62) { +TEST_F(GenEmitterTest, test6f545990782be38424bf) { Emitter out; out << Comment("comment"); out << VerbatimTag("tag"); @@ -674,7 +682,7 @@ TEST_F(GenEmitterTest, test3e9c6f05218917c77c62) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test7392dd9d6829b8569e16) { +TEST_F(GenEmitterTest, test9d7dd5e044527a4e8f31) { Emitter out; out << Comment("comment"); out << VerbatimTag("tag"); @@ -686,7 +694,7 @@ TEST_F(GenEmitterTest, test7392dd9d6829b8569e16) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test8b3e535afd61211d988f) { +TEST_F(GenEmitterTest, testba570ae83f89342779ff) { Emitter out; out << VerbatimTag("tag"); out << "foo"; @@ -698,7 +706,7 @@ TEST_F(GenEmitterTest, test8b3e535afd61211d988f) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, testa88d36caa958ac21e487) { +TEST_F(GenEmitterTest, testc6fc50c169793aa60531) { Emitter out; out << VerbatimTag("tag"); out << "foo"; @@ -710,7 +718,7 @@ TEST_F(GenEmitterTest, testa88d36caa958ac21e487) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test0afc4387fea5a0ad574d) { +TEST_F(GenEmitterTest, testd10a9c9671992acd494d) { Emitter out; out << Comment("comment"); out << Anchor("anchor"); @@ -718,12 +726,13 @@ TEST_F(GenEmitterTest, test0afc4387fea5a0ad574d) { out << EndDoc; EXPECT_CALL(handler, OnDocumentStart(_)); + EXPECT_CALL(handler, OnAnchor(_, "anchor")); EXPECT_CALL(handler, OnScalar(_, "?", 1, "foo")); EXPECT_CALL(handler, OnDocumentEnd()); Parse(out.c_str()); } -TEST_F(GenEmitterTest, test6e02b45ba1f87d0b17fa) { +TEST_F(GenEmitterTest, testd5ee8a3bdb42c8639ad4) { Emitter out; out << Comment("comment"); out << Anchor("anchor"); @@ -731,12 +740,13 @@ TEST_F(GenEmitterTest, test6e02b45ba1f87d0b17fa) { out << EndDoc; EXPECT_CALL(handler, OnDocumentStart(_)); + EXPECT_CALL(handler, OnAnchor(_, "anchor")); EXPECT_CALL(handler, OnScalar(_, "?", 1, "foo")); EXPECT_CALL(handler, OnDocumentEnd()); Parse(out.c_str()); } -TEST_F(GenEmitterTest, test1ecee6697402f1ced486) { +TEST_F(GenEmitterTest, test24914f6c2b7f7d5843c4) { Emitter out; out << Anchor("anchor"); out << "foo"; @@ -744,12 +754,13 @@ TEST_F(GenEmitterTest, test1ecee6697402f1ced486) { out << EndDoc; EXPECT_CALL(handler, OnDocumentStart(_)); + EXPECT_CALL(handler, OnAnchor(_, "anchor")); EXPECT_CALL(handler, OnScalar(_, "?", 1, "foo")); EXPECT_CALL(handler, OnDocumentEnd()); Parse(out.c_str()); } -TEST_F(GenEmitterTest, testf778d3e7e1fd4bc81ac8) { +TEST_F(GenEmitterTest, test9af19fe8c77aa18cd462) { Emitter out; out << Anchor("anchor"); out << "foo"; @@ -757,60 +768,65 @@ TEST_F(GenEmitterTest, testf778d3e7e1fd4bc81ac8) { out << Comment("comment"); EXPECT_CALL(handler, OnDocumentStart(_)); + EXPECT_CALL(handler, OnAnchor(_, "anchor")); EXPECT_CALL(handler, OnScalar(_, "?", 1, "foo")); EXPECT_CALL(handler, OnDocumentEnd()); Parse(out.c_str()); } -TEST_F(GenEmitterTest, testce2ddd97c4f7b7cad993) { +TEST_F(GenEmitterTest, testc2f9274717aaf39b0838) { Emitter out; out << Comment("comment"); out << Anchor("anchor"); out << "foo"; EXPECT_CALL(handler, OnDocumentStart(_)); + EXPECT_CALL(handler, OnAnchor(_, "anchor")); EXPECT_CALL(handler, OnScalar(_, "?", 1, "foo")); EXPECT_CALL(handler, OnDocumentEnd()); Parse(out.c_str()); } -TEST_F(GenEmitterTest, test9801aff946ce11347b78) { +TEST_F(GenEmitterTest, test75f3a7f62b5b77411653) { Emitter out; out << Comment("comment"); out << Anchor("anchor"); out << "foo"; EXPECT_CALL(handler, OnDocumentStart(_)); + EXPECT_CALL(handler, OnAnchor(_, "anchor")); EXPECT_CALL(handler, OnScalar(_, "?", 1, "foo")); EXPECT_CALL(handler, OnDocumentEnd()); Parse(out.c_str()); } -TEST_F(GenEmitterTest, test02ae081b4d9719668378) { +TEST_F(GenEmitterTest, testea51373e6b4e598b2adf) { Emitter out; out << Anchor("anchor"); out << "foo"; out << Comment("comment"); EXPECT_CALL(handler, OnDocumentStart(_)); + EXPECT_CALL(handler, OnAnchor(_, "anchor")); EXPECT_CALL(handler, OnScalar(_, "?", 1, "foo")); EXPECT_CALL(handler, OnDocumentEnd()); Parse(out.c_str()); } -TEST_F(GenEmitterTest, test1c75e643ba55491e9d58) { +TEST_F(GenEmitterTest, testa594204f0724101d4931) { Emitter out; out << Anchor("anchor"); out << "foo"; out << Comment("comment"); EXPECT_CALL(handler, OnDocumentStart(_)); + EXPECT_CALL(handler, OnAnchor(_, "anchor")); EXPECT_CALL(handler, OnScalar(_, "?", 1, "foo")); EXPECT_CALL(handler, OnDocumentEnd()); Parse(out.c_str()); } -TEST_F(GenEmitterTest, testa834d5e30e0fde106520) { +TEST_F(GenEmitterTest, test87638f2fba55c5235720) { Emitter out; out << Comment("comment"); out << BeginDoc; @@ -825,7 +841,7 @@ TEST_F(GenEmitterTest, testa834d5e30e0fde106520) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test26c5da2b48377ba6d9c3) { +TEST_F(GenEmitterTest, test786f027ec8e380bdeb45) { Emitter out; out << BeginDoc; out << Comment("comment"); @@ -840,7 +856,7 @@ TEST_F(GenEmitterTest, test26c5da2b48377ba6d9c3) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, teste5df2b4f5b7ed31f5843) { +TEST_F(GenEmitterTest, test9d9ca2fc29536ef5d392) { Emitter out; out << BeginDoc; out << BeginSeq; @@ -855,7 +871,7 @@ TEST_F(GenEmitterTest, teste5df2b4f5b7ed31f5843) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test7da7e14ccc523f8ef682) { +TEST_F(GenEmitterTest, testde9c33927d8f706e5191) { Emitter out; out << BeginDoc; out << BeginSeq; @@ -870,7 +886,7 @@ TEST_F(GenEmitterTest, test7da7e14ccc523f8ef682) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test5dc6d560b8a5defab6a6) { +TEST_F(GenEmitterTest, testa03392eb1d9af2180eb2) { Emitter out; out << BeginDoc; out << BeginSeq; @@ -885,7 +901,7 @@ TEST_F(GenEmitterTest, test5dc6d560b8a5defab6a6) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test5c2184c2ae1d09d7e486) { +TEST_F(GenEmitterTest, test8826ba8a4954e2775441) { Emitter out; out << Comment("comment"); out << BeginDoc; @@ -899,7 +915,7 @@ TEST_F(GenEmitterTest, test5c2184c2ae1d09d7e486) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, testcb7ca22e0d0c01cdfd6c) { +TEST_F(GenEmitterTest, testf116d1ab8a1646bb4295) { Emitter out; out << BeginDoc; out << Comment("comment"); @@ -913,7 +929,7 @@ TEST_F(GenEmitterTest, testcb7ca22e0d0c01cdfd6c) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test764d56ed21bef8c8ed1d) { +TEST_F(GenEmitterTest, test1d4afe394248c5d6f190) { Emitter out; out << BeginDoc; out << BeginSeq; @@ -927,7 +943,7 @@ TEST_F(GenEmitterTest, test764d56ed21bef8c8ed1d) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, testff49b69cd78b76f680f4) { +TEST_F(GenEmitterTest, test4cc7b190d6dd08368f08) { Emitter out; out << BeginDoc; out << BeginSeq; @@ -941,7 +957,7 @@ TEST_F(GenEmitterTest, testff49b69cd78b76f680f4) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, testb8bbef15bc67a7e8a4f1) { +TEST_F(GenEmitterTest, testc623063380afa67c57c4) { Emitter out; out << BeginDoc; out << BeginSeq; @@ -955,7 +971,7 @@ TEST_F(GenEmitterTest, testb8bbef15bc67a7e8a4f1) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, testfe636c029a18a80da2bc) { +TEST_F(GenEmitterTest, teste24ef3f378c4c33107b2) { Emitter out; out << Comment("comment"); out << BeginDoc; @@ -972,7 +988,7 @@ TEST_F(GenEmitterTest, testfe636c029a18a80da2bc) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, teste61e3e6d8f0e748525e6) { +TEST_F(GenEmitterTest, test2aa8f68b872fd07fde8c) { Emitter out; out << BeginDoc; out << Comment("comment"); @@ -989,7 +1005,7 @@ TEST_F(GenEmitterTest, teste61e3e6d8f0e748525e6) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test410c9f00cef2fbe84d3f) { +TEST_F(GenEmitterTest, test940bf9330572d48b476f) { Emitter out; out << BeginDoc; out << BeginSeq; @@ -1006,7 +1022,7 @@ TEST_F(GenEmitterTest, test410c9f00cef2fbe84d3f) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test2438e75da6ea8f014749) { +TEST_F(GenEmitterTest, testd710bce67052a991abfa) { Emitter out; out << BeginDoc; out << BeginSeq; @@ -1023,7 +1039,7 @@ TEST_F(GenEmitterTest, test2438e75da6ea8f014749) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test0eb1e1c40f31df48889a) { +TEST_F(GenEmitterTest, testd2ac557dae648cd1ba66) { Emitter out; out << BeginDoc; out << BeginSeq; @@ -1040,7 +1056,7 @@ TEST_F(GenEmitterTest, test0eb1e1c40f31df48889a) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test7d9b8575590f008b6dcf) { +TEST_F(GenEmitterTest, testb394f0e282404d1235d3) { Emitter out; out << BeginDoc; out << BeginSeq; @@ -1057,7 +1073,7 @@ TEST_F(GenEmitterTest, test7d9b8575590f008b6dcf) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test65e209ba68cc24d8b595) { +TEST_F(GenEmitterTest, testaf620080909b118a715d) { Emitter out; out << Comment("comment"); out << BeginDoc; @@ -1073,7 +1089,7 @@ TEST_F(GenEmitterTest, test65e209ba68cc24d8b595) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test507a8de1564951289f94) { +TEST_F(GenEmitterTest, testfc23fc6f424006e5907f) { Emitter out; out << BeginDoc; out << Comment("comment"); @@ -1089,7 +1105,7 @@ TEST_F(GenEmitterTest, test507a8de1564951289f94) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, testfad3bc4da4b3117cbaac) { +TEST_F(GenEmitterTest, testbc5517fe466dd4988ce2) { Emitter out; out << BeginDoc; out << BeginSeq; @@ -1105,7 +1121,7 @@ TEST_F(GenEmitterTest, testfad3bc4da4b3117cbaac) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test56e0e2370b233fb813d0) { +TEST_F(GenEmitterTest, testc0db52f1be33ddf93852) { Emitter out; out << BeginDoc; out << BeginSeq; @@ -1121,7 +1137,7 @@ TEST_F(GenEmitterTest, test56e0e2370b233fb813d0) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test0a4ab861864d922b0ef3) { +TEST_F(GenEmitterTest, test279a9eef5b2d2cf98ecf) { Emitter out; out << BeginDoc; out << BeginSeq; @@ -1137,7 +1153,7 @@ TEST_F(GenEmitterTest, test0a4ab861864d922b0ef3) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, testdfefc6ca543a9fdd2ab4) { +TEST_F(GenEmitterTest, test7f55b2f00c1090e43af5) { Emitter out; out << BeginDoc; out << BeginSeq; @@ -1153,7 +1169,7 @@ TEST_F(GenEmitterTest, testdfefc6ca543a9fdd2ab4) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test812eee6d9f35f45d4c74) { +TEST_F(GenEmitterTest, test1be996b4b790d9bd565d) { Emitter out; out << Comment("comment"); out << BeginDoc; @@ -1171,7 +1187,7 @@ TEST_F(GenEmitterTest, test812eee6d9f35f45d4c74) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test40cad31d965f6c4799b2) { +TEST_F(GenEmitterTest, testa5dea69e968ea27412cc) { Emitter out; out << BeginDoc; out << Comment("comment"); @@ -1189,7 +1205,7 @@ TEST_F(GenEmitterTest, test40cad31d965f6c4799b2) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, testcd959e61fe82eb217264) { +TEST_F(GenEmitterTest, test77fe0d4370db4aa8af1a) { Emitter out; out << BeginDoc; out << BeginSeq; @@ -1207,7 +1223,7 @@ TEST_F(GenEmitterTest, testcd959e61fe82eb217264) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test6d9e41274914945389ad) { +TEST_F(GenEmitterTest, test6d0319a28dd1a931f211) { Emitter out; out << BeginDoc; out << BeginSeq; @@ -1225,7 +1241,7 @@ TEST_F(GenEmitterTest, test6d9e41274914945389ad) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, testab6b07362d96686c11d0) { +TEST_F(GenEmitterTest, test0031c4cd5331366162a6) { Emitter out; out << BeginDoc; out << BeginSeq; @@ -1243,7 +1259,7 @@ TEST_F(GenEmitterTest, testab6b07362d96686c11d0) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, testd4c0512efd628009c912) { +TEST_F(GenEmitterTest, testc0c74d483811e3322ed2) { Emitter out; out << BeginDoc; out << BeginSeq; @@ -1261,7 +1277,7 @@ TEST_F(GenEmitterTest, testd4c0512efd628009c912) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test2774792a756fc53a2ca9) { +TEST_F(GenEmitterTest, test0d0938c9dca1e78401d6) { Emitter out; out << Comment("comment"); out << BeginDoc; @@ -1278,7 +1294,7 @@ TEST_F(GenEmitterTest, test2774792a756fc53a2ca9) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test9f03b22824667c3b1189) { +TEST_F(GenEmitterTest, test0c83b8f0404e62673099) { Emitter out; out << BeginDoc; out << Comment("comment"); @@ -1295,7 +1311,7 @@ TEST_F(GenEmitterTest, test9f03b22824667c3b1189) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test24dd60172cf27492a619) { +TEST_F(GenEmitterTest, test733fd2f94ae082ea6076) { Emitter out; out << BeginDoc; out << BeginSeq; @@ -1312,7 +1328,7 @@ TEST_F(GenEmitterTest, test24dd60172cf27492a619) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, testa48e3f886fe68e9b48f5) { +TEST_F(GenEmitterTest, test04b57f98a492b0f2c1ad) { Emitter out; out << BeginDoc; out << BeginSeq; @@ -1329,7 +1345,7 @@ TEST_F(GenEmitterTest, testa48e3f886fe68e9b48f5) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, testfc1258212ed3c60e6e0c) { +TEST_F(GenEmitterTest, test80d83a80f235341f1bff) { Emitter out; out << BeginDoc; out << BeginSeq; @@ -1346,7 +1362,7 @@ TEST_F(GenEmitterTest, testfc1258212ed3c60e6e0c) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test31c3570f387278586efc) { +TEST_F(GenEmitterTest, testecbe137bf7436ccd7976) { Emitter out; out << BeginDoc; out << BeginSeq; @@ -1363,7 +1379,7 @@ TEST_F(GenEmitterTest, test31c3570f387278586efc) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test0d2e56437573e1a68e06) { +TEST_F(GenEmitterTest, testf14f2e8202cacdf9252d) { Emitter out; out << Comment("comment"); out << BeginDoc; @@ -1375,13 +1391,14 @@ TEST_F(GenEmitterTest, test0d2e56437573e1a68e06) { EXPECT_CALL(handler, OnDocumentStart(_)); EXPECT_CALL(handler, OnSequenceStart(_, "?", 0, _)); + EXPECT_CALL(handler, OnAnchor(_, "anchor")); EXPECT_CALL(handler, OnScalar(_, "?", 1, "foo")); EXPECT_CALL(handler, OnSequenceEnd()); EXPECT_CALL(handler, OnDocumentEnd()); Parse(out.c_str()); } -TEST_F(GenEmitterTest, test87f2bba12c122a5ef611) { +TEST_F(GenEmitterTest, test9c029f7cf565580a56fd) { Emitter out; out << BeginDoc; out << Comment("comment"); @@ -1393,13 +1410,14 @@ TEST_F(GenEmitterTest, test87f2bba12c122a5ef611) { EXPECT_CALL(handler, OnDocumentStart(_)); EXPECT_CALL(handler, OnSequenceStart(_, "?", 0, _)); + EXPECT_CALL(handler, OnAnchor(_, "anchor")); EXPECT_CALL(handler, OnScalar(_, "?", 1, "foo")); EXPECT_CALL(handler, OnSequenceEnd()); EXPECT_CALL(handler, OnDocumentEnd()); Parse(out.c_str()); } -TEST_F(GenEmitterTest, testce2b83660fc3c47881d9) { +TEST_F(GenEmitterTest, test129cd28cda34c7b97a89) { Emitter out; out << BeginDoc; out << BeginSeq; @@ -1411,13 +1429,14 @@ TEST_F(GenEmitterTest, testce2b83660fc3c47881d9) { EXPECT_CALL(handler, OnDocumentStart(_)); EXPECT_CALL(handler, OnSequenceStart(_, "?", 0, _)); + EXPECT_CALL(handler, OnAnchor(_, "anchor")); EXPECT_CALL(handler, OnScalar(_, "?", 1, "foo")); EXPECT_CALL(handler, OnSequenceEnd()); EXPECT_CALL(handler, OnDocumentEnd()); Parse(out.c_str()); } -TEST_F(GenEmitterTest, test1192cd9aee9977c43f91) { +TEST_F(GenEmitterTest, test1c55ee081412be96e00f) { Emitter out; out << BeginDoc; out << BeginSeq; @@ -1429,13 +1448,14 @@ TEST_F(GenEmitterTest, test1192cd9aee9977c43f91) { EXPECT_CALL(handler, OnDocumentStart(_)); EXPECT_CALL(handler, OnSequenceStart(_, "?", 0, _)); + EXPECT_CALL(handler, OnAnchor(_, "anchor")); EXPECT_CALL(handler, OnScalar(_, "?", 1, "foo")); EXPECT_CALL(handler, OnSequenceEnd()); EXPECT_CALL(handler, OnDocumentEnd()); Parse(out.c_str()); } -TEST_F(GenEmitterTest, test5ec743a4a5f5d9f250e6) { +TEST_F(GenEmitterTest, testf9e9d15d9e09a8e98681) { Emitter out; out << BeginDoc; out << BeginSeq; @@ -1447,13 +1467,14 @@ TEST_F(GenEmitterTest, test5ec743a4a5f5d9f250e6) { EXPECT_CALL(handler, OnDocumentStart(_)); EXPECT_CALL(handler, OnSequenceStart(_, "?", 0, _)); + EXPECT_CALL(handler, OnAnchor(_, "anchor")); EXPECT_CALL(handler, OnScalar(_, "?", 1, "foo")); EXPECT_CALL(handler, OnSequenceEnd()); EXPECT_CALL(handler, OnDocumentEnd()); Parse(out.c_str()); } -TEST_F(GenEmitterTest, test989ce0d2a8d41f94e563) { +TEST_F(GenEmitterTest, test03dd7104722840fe7fee) { Emitter out; out << BeginDoc; out << BeginSeq; @@ -1465,13 +1486,14 @@ TEST_F(GenEmitterTest, test989ce0d2a8d41f94e563) { EXPECT_CALL(handler, OnDocumentStart(_)); EXPECT_CALL(handler, OnSequenceStart(_, "?", 0, _)); + EXPECT_CALL(handler, OnAnchor(_, "anchor")); EXPECT_CALL(handler, OnScalar(_, "?", 1, "foo")); EXPECT_CALL(handler, OnSequenceEnd()); EXPECT_CALL(handler, OnDocumentEnd()); Parse(out.c_str()); } -TEST_F(GenEmitterTest, test16b16527ce6d2047b885) { +TEST_F(GenEmitterTest, teste6c856a08270255404b6) { Emitter out; out << Comment("comment"); out << BeginDoc; @@ -1482,13 +1504,14 @@ TEST_F(GenEmitterTest, test16b16527ce6d2047b885) { EXPECT_CALL(handler, OnDocumentStart(_)); EXPECT_CALL(handler, OnSequenceStart(_, "?", 0, _)); + EXPECT_CALL(handler, OnAnchor(_, "anchor")); EXPECT_CALL(handler, OnScalar(_, "?", 1, "foo")); EXPECT_CALL(handler, OnSequenceEnd()); EXPECT_CALL(handler, OnDocumentEnd()); Parse(out.c_str()); } -TEST_F(GenEmitterTest, test190462c295b7c1b9be90) { +TEST_F(GenEmitterTest, testf285ed8797058c0e4e2f) { Emitter out; out << BeginDoc; out << Comment("comment"); @@ -1499,13 +1522,14 @@ TEST_F(GenEmitterTest, test190462c295b7c1b9be90) { EXPECT_CALL(handler, OnDocumentStart(_)); EXPECT_CALL(handler, OnSequenceStart(_, "?", 0, _)); + EXPECT_CALL(handler, OnAnchor(_, "anchor")); EXPECT_CALL(handler, OnScalar(_, "?", 1, "foo")); EXPECT_CALL(handler, OnSequenceEnd()); EXPECT_CALL(handler, OnDocumentEnd()); Parse(out.c_str()); } -TEST_F(GenEmitterTest, teste179dd43311786d15564) { +TEST_F(GenEmitterTest, test906076647b894281787e) { Emitter out; out << BeginDoc; out << BeginSeq; @@ -1516,13 +1540,14 @@ TEST_F(GenEmitterTest, teste179dd43311786d15564) { EXPECT_CALL(handler, OnDocumentStart(_)); EXPECT_CALL(handler, OnSequenceStart(_, "?", 0, _)); + EXPECT_CALL(handler, OnAnchor(_, "anchor")); EXPECT_CALL(handler, OnScalar(_, "?", 1, "foo")); EXPECT_CALL(handler, OnSequenceEnd()); EXPECT_CALL(handler, OnDocumentEnd()); Parse(out.c_str()); } -TEST_F(GenEmitterTest, test448c118909ba10e1e5c2) { +TEST_F(GenEmitterTest, test8a836336041c56130c5c) { Emitter out; out << BeginDoc; out << BeginSeq; @@ -1533,13 +1558,14 @@ TEST_F(GenEmitterTest, test448c118909ba10e1e5c2) { EXPECT_CALL(handler, OnDocumentStart(_)); EXPECT_CALL(handler, OnSequenceStart(_, "?", 0, _)); + EXPECT_CALL(handler, OnAnchor(_, "anchor")); EXPECT_CALL(handler, OnScalar(_, "?", 1, "foo")); EXPECT_CALL(handler, OnSequenceEnd()); EXPECT_CALL(handler, OnDocumentEnd()); Parse(out.c_str()); } -TEST_F(GenEmitterTest, test2aa064c8455651bbbccf) { +TEST_F(GenEmitterTest, testc7f61ada097fb34f24ce) { Emitter out; out << BeginDoc; out << BeginSeq; @@ -1550,13 +1576,14 @@ TEST_F(GenEmitterTest, test2aa064c8455651bbbccf) { EXPECT_CALL(handler, OnDocumentStart(_)); EXPECT_CALL(handler, OnSequenceStart(_, "?", 0, _)); + EXPECT_CALL(handler, OnAnchor(_, "anchor")); EXPECT_CALL(handler, OnScalar(_, "?", 1, "foo")); EXPECT_CALL(handler, OnSequenceEnd()); EXPECT_CALL(handler, OnDocumentEnd()); Parse(out.c_str()); } -TEST_F(GenEmitterTest, test68e7564605ae7013b858) { +TEST_F(GenEmitterTest, testec075d926fd1f95a1bae) { Emitter out; out << BeginDoc; out << BeginSeq; @@ -1567,13 +1594,14 @@ TEST_F(GenEmitterTest, test68e7564605ae7013b858) { EXPECT_CALL(handler, OnDocumentStart(_)); EXPECT_CALL(handler, OnSequenceStart(_, "?", 0, _)); + EXPECT_CALL(handler, OnAnchor(_, "anchor")); EXPECT_CALL(handler, OnScalar(_, "?", 1, "foo")); EXPECT_CALL(handler, OnSequenceEnd()); EXPECT_CALL(handler, OnDocumentEnd()); Parse(out.c_str()); } -TEST_F(GenEmitterTest, test11fa4cf5f845696bf8c9) { +TEST_F(GenEmitterTest, testa79ce9edc0c3593faa31) { Emitter out; out << Comment("comment"); out << BeginDoc; @@ -1586,13 +1614,14 @@ TEST_F(GenEmitterTest, test11fa4cf5f845696bf8c9) { EXPECT_CALL(handler, OnDocumentStart(_)); EXPECT_CALL(handler, OnSequenceStart(_, "?", 0, _)); + EXPECT_CALL(handler, OnAnchor(_, "anchor")); EXPECT_CALL(handler, OnScalar(_, "tag", 1, "foo")); EXPECT_CALL(handler, OnSequenceEnd()); EXPECT_CALL(handler, OnDocumentEnd()); Parse(out.c_str()); } -TEST_F(GenEmitterTest, test2b2cd9f7d03b725c77f7) { +TEST_F(GenEmitterTest, test525c133ebf8f46a1962f) { Emitter out; out << BeginDoc; out << Comment("comment"); @@ -1605,13 +1634,14 @@ TEST_F(GenEmitterTest, test2b2cd9f7d03b725c77f7) { EXPECT_CALL(handler, OnDocumentStart(_)); EXPECT_CALL(handler, OnSequenceStart(_, "?", 0, _)); + EXPECT_CALL(handler, OnAnchor(_, "anchor")); EXPECT_CALL(handler, OnScalar(_, "tag", 1, "foo")); EXPECT_CALL(handler, OnSequenceEnd()); EXPECT_CALL(handler, OnDocumentEnd()); Parse(out.c_str()); } -TEST_F(GenEmitterTest, test846d35c91609d010038a) { +TEST_F(GenEmitterTest, testb06604d03a8c9cfbe7c2) { Emitter out; out << BeginDoc; out << BeginSeq; @@ -1624,13 +1654,14 @@ TEST_F(GenEmitterTest, test846d35c91609d010038a) { EXPECT_CALL(handler, OnDocumentStart(_)); EXPECT_CALL(handler, OnSequenceStart(_, "?", 0, _)); + EXPECT_CALL(handler, OnAnchor(_, "anchor")); EXPECT_CALL(handler, OnScalar(_, "tag", 1, "foo")); EXPECT_CALL(handler, OnSequenceEnd()); EXPECT_CALL(handler, OnDocumentEnd()); Parse(out.c_str()); } -TEST_F(GenEmitterTest, test0295ace15e4d18d3efff) { +TEST_F(GenEmitterTest, teste268ba5f2d54120eb665) { Emitter out; out << BeginDoc; out << BeginSeq; @@ -1643,13 +1674,14 @@ TEST_F(GenEmitterTest, test0295ace15e4d18d3efff) { EXPECT_CALL(handler, OnDocumentStart(_)); EXPECT_CALL(handler, OnSequenceStart(_, "?", 0, _)); + EXPECT_CALL(handler, OnAnchor(_, "anchor")); EXPECT_CALL(handler, OnScalar(_, "tag", 1, "foo")); EXPECT_CALL(handler, OnSequenceEnd()); EXPECT_CALL(handler, OnDocumentEnd()); Parse(out.c_str()); } -TEST_F(GenEmitterTest, test31f01031d12ec0623b22) { +TEST_F(GenEmitterTest, test7a646350a81bba70e44a) { Emitter out; out << BeginDoc; out << BeginSeq; @@ -1662,13 +1694,14 @@ TEST_F(GenEmitterTest, test31f01031d12ec0623b22) { EXPECT_CALL(handler, OnDocumentStart(_)); EXPECT_CALL(handler, OnSequenceStart(_, "?", 0, _)); + EXPECT_CALL(handler, OnAnchor(_, "anchor")); EXPECT_CALL(handler, OnScalar(_, "tag", 1, "foo")); EXPECT_CALL(handler, OnSequenceEnd()); EXPECT_CALL(handler, OnDocumentEnd()); Parse(out.c_str()); } -TEST_F(GenEmitterTest, test9dfa64a8b15cb1e411f3) { +TEST_F(GenEmitterTest, test025df570e0d8a1f818da) { Emitter out; out << BeginDoc; out << BeginSeq; @@ -1681,13 +1714,14 @@ TEST_F(GenEmitterTest, test9dfa64a8b15cb1e411f3) { EXPECT_CALL(handler, OnDocumentStart(_)); EXPECT_CALL(handler, OnSequenceStart(_, "?", 0, _)); + EXPECT_CALL(handler, OnAnchor(_, "anchor")); EXPECT_CALL(handler, OnScalar(_, "tag", 1, "foo")); EXPECT_CALL(handler, OnSequenceEnd()); EXPECT_CALL(handler, OnDocumentEnd()); Parse(out.c_str()); } -TEST_F(GenEmitterTest, test2e707476963eb0157f56) { +TEST_F(GenEmitterTest, test897087b9aba1d5773870) { Emitter out; out << Comment("comment"); out << BeginDoc; @@ -1699,13 +1733,14 @@ TEST_F(GenEmitterTest, test2e707476963eb0157f56) { EXPECT_CALL(handler, OnDocumentStart(_)); EXPECT_CALL(handler, OnSequenceStart(_, "?", 0, _)); + EXPECT_CALL(handler, OnAnchor(_, "anchor")); EXPECT_CALL(handler, OnScalar(_, "tag", 1, "foo")); EXPECT_CALL(handler, OnSequenceEnd()); EXPECT_CALL(handler, OnDocumentEnd()); Parse(out.c_str()); } -TEST_F(GenEmitterTest, test455d60cf99b6ecf73d46) { +TEST_F(GenEmitterTest, testa45ee0501da4a4e5da54) { Emitter out; out << BeginDoc; out << Comment("comment"); @@ -1717,13 +1752,14 @@ TEST_F(GenEmitterTest, test455d60cf99b6ecf73d46) { EXPECT_CALL(handler, OnDocumentStart(_)); EXPECT_CALL(handler, OnSequenceStart(_, "?", 0, _)); + EXPECT_CALL(handler, OnAnchor(_, "anchor")); EXPECT_CALL(handler, OnScalar(_, "tag", 1, "foo")); EXPECT_CALL(handler, OnSequenceEnd()); EXPECT_CALL(handler, OnDocumentEnd()); Parse(out.c_str()); } -TEST_F(GenEmitterTest, testa370b09ea89f5df32d03) { +TEST_F(GenEmitterTest, teste751c06ea558ccca1821) { Emitter out; out << BeginDoc; out << BeginSeq; @@ -1735,13 +1771,14 @@ TEST_F(GenEmitterTest, testa370b09ea89f5df32d03) { EXPECT_CALL(handler, OnDocumentStart(_)); EXPECT_CALL(handler, OnSequenceStart(_, "?", 0, _)); + EXPECT_CALL(handler, OnAnchor(_, "anchor")); EXPECT_CALL(handler, OnScalar(_, "tag", 1, "foo")); EXPECT_CALL(handler, OnSequenceEnd()); EXPECT_CALL(handler, OnDocumentEnd()); Parse(out.c_str()); } -TEST_F(GenEmitterTest, test325554fe67b486e0ac43) { +TEST_F(GenEmitterTest, test8526d26e85cc930eecec) { Emitter out; out << BeginDoc; out << BeginSeq; @@ -1753,13 +1790,14 @@ TEST_F(GenEmitterTest, test325554fe67b486e0ac43) { EXPECT_CALL(handler, OnDocumentStart(_)); EXPECT_CALL(handler, OnSequenceStart(_, "?", 0, _)); + EXPECT_CALL(handler, OnAnchor(_, "anchor")); EXPECT_CALL(handler, OnScalar(_, "tag", 1, "foo")); EXPECT_CALL(handler, OnSequenceEnd()); EXPECT_CALL(handler, OnDocumentEnd()); Parse(out.c_str()); } -TEST_F(GenEmitterTest, test0e511740421ef2391918) { +TEST_F(GenEmitterTest, teste9a5a4a0f0e44311d01a) { Emitter out; out << BeginDoc; out << BeginSeq; @@ -1771,13 +1809,14 @@ TEST_F(GenEmitterTest, test0e511740421ef2391918) { EXPECT_CALL(handler, OnDocumentStart(_)); EXPECT_CALL(handler, OnSequenceStart(_, "?", 0, _)); + EXPECT_CALL(handler, OnAnchor(_, "anchor")); EXPECT_CALL(handler, OnScalar(_, "tag", 1, "foo")); EXPECT_CALL(handler, OnSequenceEnd()); EXPECT_CALL(handler, OnDocumentEnd()); Parse(out.c_str()); } -TEST_F(GenEmitterTest, test48e6238da58855f75d2d) { +TEST_F(GenEmitterTest, testac8a091ab93b65aee893) { Emitter out; out << BeginDoc; out << BeginSeq; @@ -1789,13 +1828,14 @@ TEST_F(GenEmitterTest, test48e6238da58855f75d2d) { EXPECT_CALL(handler, OnDocumentStart(_)); EXPECT_CALL(handler, OnSequenceStart(_, "?", 0, _)); + EXPECT_CALL(handler, OnAnchor(_, "anchor")); EXPECT_CALL(handler, OnScalar(_, "tag", 1, "foo")); EXPECT_CALL(handler, OnSequenceEnd()); EXPECT_CALL(handler, OnDocumentEnd()); Parse(out.c_str()); } -TEST_F(GenEmitterTest, test985ccda3f305aebbe7c1) { +TEST_F(GenEmitterTest, testee014788f524623b5075) { Emitter out; out << Comment("comment"); out << BeginDoc; @@ -1814,7 +1854,7 @@ TEST_F(GenEmitterTest, test985ccda3f305aebbe7c1) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test48a885b1b5192b7d6c42) { +TEST_F(GenEmitterTest, test57a067545c01c42a7b4e) { Emitter out; out << BeginDoc; out << Comment("comment"); @@ -1833,7 +1873,7 @@ TEST_F(GenEmitterTest, test48a885b1b5192b7d6c42) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test2d59de662bffd75bdd4e) { +TEST_F(GenEmitterTest, test948ac02da8825214c869) { Emitter out; out << BeginDoc; out << BeginSeq; @@ -1852,7 +1892,7 @@ TEST_F(GenEmitterTest, test2d59de662bffd75bdd4e) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test1876b50ad981242e1b5e) { +TEST_F(GenEmitterTest, testa3d6c5e8a1658c1dd726) { Emitter out; out << BeginDoc; out << BeginSeq; @@ -1871,7 +1911,7 @@ TEST_F(GenEmitterTest, test1876b50ad981242e1b5e) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test3be092fd7c3394e57a02) { +TEST_F(GenEmitterTest, test548d71006d7cafde91da) { Emitter out; out << BeginDoc; out << BeginSeq; @@ -1890,7 +1930,7 @@ TEST_F(GenEmitterTest, test3be092fd7c3394e57a02) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, testdaaf6202df0524d94ba2) { +TEST_F(GenEmitterTest, test35e08ea7459dbee9eab8) { Emitter out; out << BeginDoc; out << BeginSeq; @@ -1909,7 +1949,7 @@ TEST_F(GenEmitterTest, testdaaf6202df0524d94ba2) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test0f9c4973bc77d8baa80b) { +TEST_F(GenEmitterTest, test87e79665a4339434d781) { Emitter out; out << BeginDoc; out << BeginSeq; @@ -1928,7 +1968,7 @@ TEST_F(GenEmitterTest, test0f9c4973bc77d8baa80b) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, testfb738d9af8c3f5d0d89b) { +TEST_F(GenEmitterTest, test4928d09bc979129c05ca) { Emitter out; out << Comment("comment"); out << BeginDoc; @@ -1946,7 +1986,7 @@ TEST_F(GenEmitterTest, testfb738d9af8c3f5d0d89b) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test7ba35bde1bf4471d19fd) { +TEST_F(GenEmitterTest, test1d2f73011af6b4486504) { Emitter out; out << BeginDoc; out << Comment("comment"); @@ -1964,7 +2004,7 @@ TEST_F(GenEmitterTest, test7ba35bde1bf4471d19fd) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test2fc98e907a537e17f435) { +TEST_F(GenEmitterTest, test2460718f7277d5f42306) { Emitter out; out << BeginDoc; out << BeginSeq; @@ -1982,7 +2022,7 @@ TEST_F(GenEmitterTest, test2fc98e907a537e17f435) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test1c2cbe9d3ad5bca645f1) { +TEST_F(GenEmitterTest, test52309e87b3f0185f982b) { Emitter out; out << BeginDoc; out << BeginSeq; @@ -2000,7 +2040,7 @@ TEST_F(GenEmitterTest, test1c2cbe9d3ad5bca645f1) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test999bff6d585448f6f195) { +TEST_F(GenEmitterTest, testa51d8f1cedfead1de5ab) { Emitter out; out << BeginDoc; out << BeginSeq; @@ -2018,7 +2058,7 @@ TEST_F(GenEmitterTest, test999bff6d585448f6f195) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test8972a583224ae4553b81) { +TEST_F(GenEmitterTest, test537bf14b4d578f212f4d) { Emitter out; out << BeginDoc; out << BeginSeq; @@ -2036,7 +2076,7 @@ TEST_F(GenEmitterTest, test8972a583224ae4553b81) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test77d2c64329579c726156) { +TEST_F(GenEmitterTest, teste19e3fd4d5cd52bf6754) { Emitter out; out << BeginDoc; out << BeginSeq; @@ -2054,7 +2094,7 @@ TEST_F(GenEmitterTest, test77d2c64329579c726156) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, testf60e4d264223ca1d8ed0) { +TEST_F(GenEmitterTest, testf27e53142f2ca0e96a99) { Emitter out; out << Comment("comment"); out << BeginDoc; @@ -2070,14 +2110,16 @@ TEST_F(GenEmitterTest, testf60e4d264223ca1d8ed0) { EXPECT_CALL(handler, OnDocumentStart(_)); EXPECT_CALL(handler, OnSequenceStart(_, "?", 0, _)); + EXPECT_CALL(handler, OnAnchor(_, "anchor")); EXPECT_CALL(handler, OnScalar(_, "tag", 1, "foo")); + EXPECT_CALL(handler, OnAnchor(_, "other")); EXPECT_CALL(handler, OnScalar(_, "tag", 2, "bar")); EXPECT_CALL(handler, OnSequenceEnd()); EXPECT_CALL(handler, OnDocumentEnd()); Parse(out.c_str()); } -TEST_F(GenEmitterTest, test954d6b81ce04cb78b6a9) { +TEST_F(GenEmitterTest, test8ce13fdbb0e53e131cbe) { Emitter out; out << BeginDoc; out << Comment("comment"); @@ -2093,14 +2135,16 @@ TEST_F(GenEmitterTest, test954d6b81ce04cb78b6a9) { EXPECT_CALL(handler, OnDocumentStart(_)); EXPECT_CALL(handler, OnSequenceStart(_, "?", 0, _)); + EXPECT_CALL(handler, OnAnchor(_, "anchor")); EXPECT_CALL(handler, OnScalar(_, "tag", 1, "foo")); + EXPECT_CALL(handler, OnAnchor(_, "other")); EXPECT_CALL(handler, OnScalar(_, "tag", 2, "bar")); EXPECT_CALL(handler, OnSequenceEnd()); EXPECT_CALL(handler, OnDocumentEnd()); Parse(out.c_str()); } -TEST_F(GenEmitterTest, test24e333326d100b0fb309) { +TEST_F(GenEmitterTest, test9fa693277f014353aa34) { Emitter out; out << BeginDoc; out << BeginSeq; @@ -2116,14 +2160,16 @@ TEST_F(GenEmitterTest, test24e333326d100b0fb309) { EXPECT_CALL(handler, OnDocumentStart(_)); EXPECT_CALL(handler, OnSequenceStart(_, "?", 0, _)); + EXPECT_CALL(handler, OnAnchor(_, "anchor")); EXPECT_CALL(handler, OnScalar(_, "tag", 1, "foo")); + EXPECT_CALL(handler, OnAnchor(_, "other")); EXPECT_CALL(handler, OnScalar(_, "tag", 2, "bar")); EXPECT_CALL(handler, OnSequenceEnd()); EXPECT_CALL(handler, OnDocumentEnd()); Parse(out.c_str()); } -TEST_F(GenEmitterTest, test996f3742f7296a3dfc08) { +TEST_F(GenEmitterTest, testc3e4849fb38bc3556f45) { Emitter out; out << BeginDoc; out << BeginSeq; @@ -2139,14 +2185,16 @@ TEST_F(GenEmitterTest, test996f3742f7296a3dfc08) { EXPECT_CALL(handler, OnDocumentStart(_)); EXPECT_CALL(handler, OnSequenceStart(_, "?", 0, _)); + EXPECT_CALL(handler, OnAnchor(_, "anchor")); EXPECT_CALL(handler, OnScalar(_, "tag", 1, "foo")); + EXPECT_CALL(handler, OnAnchor(_, "other")); EXPECT_CALL(handler, OnScalar(_, "tag", 2, "bar")); EXPECT_CALL(handler, OnSequenceEnd()); EXPECT_CALL(handler, OnDocumentEnd()); Parse(out.c_str()); } -TEST_F(GenEmitterTest, test7c795dad3124d186fe05) { +TEST_F(GenEmitterTest, test34049495795f40da2d52) { Emitter out; out << BeginDoc; out << BeginSeq; @@ -2162,14 +2210,16 @@ TEST_F(GenEmitterTest, test7c795dad3124d186fe05) { EXPECT_CALL(handler, OnDocumentStart(_)); EXPECT_CALL(handler, OnSequenceStart(_, "?", 0, _)); + EXPECT_CALL(handler, OnAnchor(_, "anchor")); EXPECT_CALL(handler, OnScalar(_, "tag", 1, "foo")); + EXPECT_CALL(handler, OnAnchor(_, "other")); EXPECT_CALL(handler, OnScalar(_, "tag", 2, "bar")); EXPECT_CALL(handler, OnSequenceEnd()); EXPECT_CALL(handler, OnDocumentEnd()); Parse(out.c_str()); } -TEST_F(GenEmitterTest, testc2b03c01b73703fad10f) { +TEST_F(GenEmitterTest, test14353701fc865919ab50) { Emitter out; out << BeginDoc; out << BeginSeq; @@ -2185,14 +2235,16 @@ TEST_F(GenEmitterTest, testc2b03c01b73703fad10f) { EXPECT_CALL(handler, OnDocumentStart(_)); EXPECT_CALL(handler, OnSequenceStart(_, "?", 0, _)); + EXPECT_CALL(handler, OnAnchor(_, "anchor")); EXPECT_CALL(handler, OnScalar(_, "tag", 1, "foo")); + EXPECT_CALL(handler, OnAnchor(_, "other")); EXPECT_CALL(handler, OnScalar(_, "tag", 2, "bar")); EXPECT_CALL(handler, OnSequenceEnd()); EXPECT_CALL(handler, OnDocumentEnd()); Parse(out.c_str()); } -TEST_F(GenEmitterTest, testab877782f99ac0c76a54) { +TEST_F(GenEmitterTest, test74547fc0ba8d387c5423) { Emitter out; out << BeginDoc; out << BeginSeq; @@ -2208,14 +2260,16 @@ TEST_F(GenEmitterTest, testab877782f99ac0c76a54) { EXPECT_CALL(handler, OnDocumentStart(_)); EXPECT_CALL(handler, OnSequenceStart(_, "?", 0, _)); + EXPECT_CALL(handler, OnAnchor(_, "anchor")); EXPECT_CALL(handler, OnScalar(_, "tag", 1, "foo")); + EXPECT_CALL(handler, OnAnchor(_, "other")); EXPECT_CALL(handler, OnScalar(_, "tag", 2, "bar")); EXPECT_CALL(handler, OnSequenceEnd()); EXPECT_CALL(handler, OnDocumentEnd()); Parse(out.c_str()); } -TEST_F(GenEmitterTest, test784517e5fb6049b5eeb8) { +TEST_F(GenEmitterTest, test52d2b69b185f6ccfff4c) { Emitter out; out << Comment("comment"); out << BeginDoc; @@ -2230,14 +2284,16 @@ TEST_F(GenEmitterTest, test784517e5fb6049b5eeb8) { EXPECT_CALL(handler, OnDocumentStart(_)); EXPECT_CALL(handler, OnSequenceStart(_, "?", 0, _)); + EXPECT_CALL(handler, OnAnchor(_, "anchor")); EXPECT_CALL(handler, OnScalar(_, "tag", 1, "foo")); + EXPECT_CALL(handler, OnAnchor(_, "other")); EXPECT_CALL(handler, OnScalar(_, "tag", 2, "bar")); EXPECT_CALL(handler, OnSequenceEnd()); EXPECT_CALL(handler, OnDocumentEnd()); Parse(out.c_str()); } -TEST_F(GenEmitterTest, testbb2a1aa3c8fb0b8c763f) { +TEST_F(GenEmitterTest, test44d442585e5bc9a7644a) { Emitter out; out << BeginDoc; out << Comment("comment"); @@ -2252,14 +2308,16 @@ TEST_F(GenEmitterTest, testbb2a1aa3c8fb0b8c763f) { EXPECT_CALL(handler, OnDocumentStart(_)); EXPECT_CALL(handler, OnSequenceStart(_, "?", 0, _)); + EXPECT_CALL(handler, OnAnchor(_, "anchor")); EXPECT_CALL(handler, OnScalar(_, "tag", 1, "foo")); + EXPECT_CALL(handler, OnAnchor(_, "other")); EXPECT_CALL(handler, OnScalar(_, "tag", 2, "bar")); EXPECT_CALL(handler, OnSequenceEnd()); EXPECT_CALL(handler, OnDocumentEnd()); Parse(out.c_str()); } -TEST_F(GenEmitterTest, testb360035c1b0988456eb8) { +TEST_F(GenEmitterTest, test3dc263684801dec471c9) { Emitter out; out << BeginDoc; out << BeginSeq; @@ -2274,14 +2332,16 @@ TEST_F(GenEmitterTest, testb360035c1b0988456eb8) { EXPECT_CALL(handler, OnDocumentStart(_)); EXPECT_CALL(handler, OnSequenceStart(_, "?", 0, _)); + EXPECT_CALL(handler, OnAnchor(_, "anchor")); EXPECT_CALL(handler, OnScalar(_, "tag", 1, "foo")); + EXPECT_CALL(handler, OnAnchor(_, "other")); EXPECT_CALL(handler, OnScalar(_, "tag", 2, "bar")); EXPECT_CALL(handler, OnSequenceEnd()); EXPECT_CALL(handler, OnDocumentEnd()); Parse(out.c_str()); } -TEST_F(GenEmitterTest, test547ca5f2968b36666091) { +TEST_F(GenEmitterTest, testa04cde3245ad9b929b9a) { Emitter out; out << BeginDoc; out << BeginSeq; @@ -2296,14 +2356,16 @@ TEST_F(GenEmitterTest, test547ca5f2968b36666091) { EXPECT_CALL(handler, OnDocumentStart(_)); EXPECT_CALL(handler, OnSequenceStart(_, "?", 0, _)); + EXPECT_CALL(handler, OnAnchor(_, "anchor")); EXPECT_CALL(handler, OnScalar(_, "tag", 1, "foo")); + EXPECT_CALL(handler, OnAnchor(_, "other")); EXPECT_CALL(handler, OnScalar(_, "tag", 2, "bar")); EXPECT_CALL(handler, OnSequenceEnd()); EXPECT_CALL(handler, OnDocumentEnd()); Parse(out.c_str()); } -TEST_F(GenEmitterTest, testcfb6b5ffda618b49a876) { +TEST_F(GenEmitterTest, testd911e740ca36e0509dfa) { Emitter out; out << BeginDoc; out << BeginSeq; @@ -2318,14 +2380,16 @@ TEST_F(GenEmitterTest, testcfb6b5ffda618b49a876) { EXPECT_CALL(handler, OnDocumentStart(_)); EXPECT_CALL(handler, OnSequenceStart(_, "?", 0, _)); + EXPECT_CALL(handler, OnAnchor(_, "anchor")); EXPECT_CALL(handler, OnScalar(_, "tag", 1, "foo")); + EXPECT_CALL(handler, OnAnchor(_, "other")); EXPECT_CALL(handler, OnScalar(_, "tag", 2, "bar")); EXPECT_CALL(handler, OnSequenceEnd()); EXPECT_CALL(handler, OnDocumentEnd()); Parse(out.c_str()); } -TEST_F(GenEmitterTest, test0f42ce2ba53fb3e486f9) { +TEST_F(GenEmitterTest, testde44215fe9b2e87846ba) { Emitter out; out << BeginDoc; out << BeginSeq; @@ -2340,14 +2404,16 @@ TEST_F(GenEmitterTest, test0f42ce2ba53fb3e486f9) { EXPECT_CALL(handler, OnDocumentStart(_)); EXPECT_CALL(handler, OnSequenceStart(_, "?", 0, _)); + EXPECT_CALL(handler, OnAnchor(_, "anchor")); EXPECT_CALL(handler, OnScalar(_, "tag", 1, "foo")); + EXPECT_CALL(handler, OnAnchor(_, "other")); EXPECT_CALL(handler, OnScalar(_, "tag", 2, "bar")); EXPECT_CALL(handler, OnSequenceEnd()); EXPECT_CALL(handler, OnDocumentEnd()); Parse(out.c_str()); } -TEST_F(GenEmitterTest, test93e437d10dbd42ac83bb) { +TEST_F(GenEmitterTest, test6390021323a4889f19d2) { Emitter out; out << BeginDoc; out << BeginSeq; @@ -2362,14 +2428,16 @@ TEST_F(GenEmitterTest, test93e437d10dbd42ac83bb) { EXPECT_CALL(handler, OnDocumentStart(_)); EXPECT_CALL(handler, OnSequenceStart(_, "?", 0, _)); + EXPECT_CALL(handler, OnAnchor(_, "anchor")); EXPECT_CALL(handler, OnScalar(_, "tag", 1, "foo")); + EXPECT_CALL(handler, OnAnchor(_, "other")); EXPECT_CALL(handler, OnScalar(_, "tag", 2, "bar")); EXPECT_CALL(handler, OnSequenceEnd()); EXPECT_CALL(handler, OnDocumentEnd()); Parse(out.c_str()); } -TEST_F(GenEmitterTest, test4267b4f75e3d9582d981) { +TEST_F(GenEmitterTest, test1db2fcb7347f6cb37dd4) { Emitter out; out << Comment("comment"); out << BeginSeq; @@ -2383,7 +2451,7 @@ TEST_F(GenEmitterTest, test4267b4f75e3d9582d981) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test62c1e6598c987e3d28a2) { +TEST_F(GenEmitterTest, test06b32e9d75498ee291d2) { Emitter out; out << Comment("comment"); out << BeginSeq; @@ -2397,7 +2465,7 @@ TEST_F(GenEmitterTest, test62c1e6598c987e3d28a2) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, testbcf0ccd5a35fe73714b0) { +TEST_F(GenEmitterTest, test86654989004963952b15) { Emitter out; out << BeginSeq; out << Comment("comment"); @@ -2411,7 +2479,7 @@ TEST_F(GenEmitterTest, testbcf0ccd5a35fe73714b0) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, testa4c54e8a23771f781411) { +TEST_F(GenEmitterTest, test53d875fc5058faa44a4e) { Emitter out; out << BeginSeq; out << EndSeq; @@ -2425,7 +2493,7 @@ TEST_F(GenEmitterTest, testa4c54e8a23771f781411) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, testad0a081525db9f568450) { +TEST_F(GenEmitterTest, test3f4b49a82b6e07eb11fd) { Emitter out; out << BeginSeq; out << EndSeq; @@ -2439,7 +2507,7 @@ TEST_F(GenEmitterTest, testad0a081525db9f568450) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test1c24a291665d1cc4404c) { +TEST_F(GenEmitterTest, testae4e2fa09d6a34077b6e) { Emitter out; out << Comment("comment"); out << BeginSeq; @@ -2452,7 +2520,7 @@ TEST_F(GenEmitterTest, test1c24a291665d1cc4404c) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, testcb331bf2e56388df2f1a) { +TEST_F(GenEmitterTest, testb181b63559d96d5f848c) { Emitter out; out << Comment("comment"); out << BeginSeq; @@ -2465,7 +2533,7 @@ TEST_F(GenEmitterTest, testcb331bf2e56388df2f1a) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test122fe9269df7f8cb80f7) { +TEST_F(GenEmitterTest, test817661fec7d3730f4fa6) { Emitter out; out << BeginSeq; out << Comment("comment"); @@ -2478,7 +2546,7 @@ TEST_F(GenEmitterTest, test122fe9269df7f8cb80f7) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test6a09300420248adaddd4) { +TEST_F(GenEmitterTest, test34bb2700e9688718fa5a) { Emitter out; out << BeginSeq; out << EndSeq; @@ -2491,7 +2559,7 @@ TEST_F(GenEmitterTest, test6a09300420248adaddd4) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, testf3bd03889c1e8dceb798) { +TEST_F(GenEmitterTest, test84e3c1999b6888d2e897) { Emitter out; out << BeginSeq; out << EndSeq; @@ -2504,7 +2572,7 @@ TEST_F(GenEmitterTest, testf3bd03889c1e8dceb798) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, testce882fb9271babb66dc6) { +TEST_F(GenEmitterTest, testa9d113656780031a99f5) { Emitter out; out << Comment("comment"); out << BeginSeq; @@ -2520,7 +2588,7 @@ TEST_F(GenEmitterTest, testce882fb9271babb66dc6) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test9c237cf40d8848a18afd) { +TEST_F(GenEmitterTest, test1cd1ead50aaa7b827068) { Emitter out; out << Comment("comment"); out << BeginSeq; @@ -2536,7 +2604,7 @@ TEST_F(GenEmitterTest, test9c237cf40d8848a18afd) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, testcadbaf942d41242c21fc) { +TEST_F(GenEmitterTest, test1389f95066b07eac89ef) { Emitter out; out << BeginSeq; out << Comment("comment"); @@ -2552,7 +2620,7 @@ TEST_F(GenEmitterTest, testcadbaf942d41242c21fc) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, testfc6e5fb862b4ce920622) { +TEST_F(GenEmitterTest, test709f3a5c294f47f62c1e) { Emitter out; out << BeginSeq; out << "foo"; @@ -2568,7 +2636,7 @@ TEST_F(GenEmitterTest, testfc6e5fb862b4ce920622) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test4ecc85dc0cb57f446540) { +TEST_F(GenEmitterTest, test8a238d7fdee02a368203) { Emitter out; out << BeginSeq; out << "foo"; @@ -2584,7 +2652,7 @@ TEST_F(GenEmitterTest, test4ecc85dc0cb57f446540) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test4e844b231f7985238b21) { +TEST_F(GenEmitterTest, test0d13534e2949ea35ca96) { Emitter out; out << BeginSeq; out << "foo"; @@ -2600,7 +2668,7 @@ TEST_F(GenEmitterTest, test4e844b231f7985238b21) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test8ec201c2f040093428a7) { +TEST_F(GenEmitterTest, test10fe6827ed46e0e063a7) { Emitter out; out << Comment("comment"); out << BeginSeq; @@ -2615,7 +2683,7 @@ TEST_F(GenEmitterTest, test8ec201c2f040093428a7) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test92469ed608395adc2620) { +TEST_F(GenEmitterTest, testc7eb6d9da57005534c1c) { Emitter out; out << Comment("comment"); out << BeginSeq; @@ -2630,7 +2698,7 @@ TEST_F(GenEmitterTest, test92469ed608395adc2620) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test8a9d541f2dc9c468cbaa) { +TEST_F(GenEmitterTest, test3f424efd76e1d32727eb) { Emitter out; out << BeginSeq; out << Comment("comment"); @@ -2645,7 +2713,7 @@ TEST_F(GenEmitterTest, test8a9d541f2dc9c468cbaa) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test6a9392e1353590d282e2) { +TEST_F(GenEmitterTest, test2bdc361bc6b056f02465) { Emitter out; out << BeginSeq; out << "foo"; @@ -2660,7 +2728,7 @@ TEST_F(GenEmitterTest, test6a9392e1353590d282e2) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test87d8b037171651caa126) { +TEST_F(GenEmitterTest, test0cc1936afe5637ba1376) { Emitter out; out << BeginSeq; out << "foo"; @@ -2675,7 +2743,7 @@ TEST_F(GenEmitterTest, test87d8b037171651caa126) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test1b1bc972209abdacc529) { +TEST_F(GenEmitterTest, test7d3e2f793963d3480545) { Emitter out; out << BeginSeq; out << "foo"; @@ -2690,7 +2758,7 @@ TEST_F(GenEmitterTest, test1b1bc972209abdacc529) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test552483e6a5fe7eee1cc5) { +TEST_F(GenEmitterTest, testf3f50e76d7ef6e2b2bff) { Emitter out; out << Comment("comment"); out << BeginSeq; @@ -2707,7 +2775,7 @@ TEST_F(GenEmitterTest, test552483e6a5fe7eee1cc5) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test43d39704b3b5e188182a) { +TEST_F(GenEmitterTest, testcbf1cff67fec9148df1c) { Emitter out; out << Comment("comment"); out << BeginSeq; @@ -2724,7 +2792,7 @@ TEST_F(GenEmitterTest, test43d39704b3b5e188182a) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test7318d9b71ec1a8f05621) { +TEST_F(GenEmitterTest, test168bd4b8dc78b4d524ee) { Emitter out; out << BeginSeq; out << Comment("comment"); @@ -2741,7 +2809,7 @@ TEST_F(GenEmitterTest, test7318d9b71ec1a8f05621) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test77302e5c536183072945) { +TEST_F(GenEmitterTest, testb616ef26030304bca6ef) { Emitter out; out << BeginSeq; out << VerbatimTag("tag"); @@ -2758,7 +2826,7 @@ TEST_F(GenEmitterTest, test77302e5c536183072945) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test5b740c51d614c5cedf1c) { +TEST_F(GenEmitterTest, test9fda976f36ddb23b38ee) { Emitter out; out << BeginSeq; out << VerbatimTag("tag"); @@ -2775,7 +2843,7 @@ TEST_F(GenEmitterTest, test5b740c51d614c5cedf1c) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, testb66838e46cf76ce30c95) { +TEST_F(GenEmitterTest, test48e8c45c081edc86deb2) { Emitter out; out << BeginSeq; out << VerbatimTag("tag"); @@ -2792,7 +2860,7 @@ TEST_F(GenEmitterTest, testb66838e46cf76ce30c95) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test592cabeb320cc3d6a4a6) { +TEST_F(GenEmitterTest, test30f5136e817ddd8158de) { Emitter out; out << Comment("comment"); out << BeginSeq; @@ -2808,7 +2876,7 @@ TEST_F(GenEmitterTest, test592cabeb320cc3d6a4a6) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, testdcc6e5388f6c2058954f) { +TEST_F(GenEmitterTest, testeb51d66281f593566172) { Emitter out; out << Comment("comment"); out << BeginSeq; @@ -2824,7 +2892,7 @@ TEST_F(GenEmitterTest, testdcc6e5388f6c2058954f) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test1ebbb7521464e6cc5da7) { +TEST_F(GenEmitterTest, testef6ffa5fa4658785ef00) { Emitter out; out << BeginSeq; out << Comment("comment"); @@ -2840,7 +2908,7 @@ TEST_F(GenEmitterTest, test1ebbb7521464e6cc5da7) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test8ccc7ae170ad81b12996) { +TEST_F(GenEmitterTest, test6db34efc6b59e8a7ba18) { Emitter out; out << BeginSeq; out << VerbatimTag("tag"); @@ -2856,7 +2924,7 @@ TEST_F(GenEmitterTest, test8ccc7ae170ad81b12996) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test6421c3c3ed7a65699e0b) { +TEST_F(GenEmitterTest, test537b9ecc9d9a5b546a9c) { Emitter out; out << BeginSeq; out << VerbatimTag("tag"); @@ -2872,7 +2940,7 @@ TEST_F(GenEmitterTest, test6421c3c3ed7a65699e0b) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test096939c5c407fed2178a) { +TEST_F(GenEmitterTest, testfadd6ee259c13382f5ce) { Emitter out; out << BeginSeq; out << VerbatimTag("tag"); @@ -2888,7 +2956,7 @@ TEST_F(GenEmitterTest, test096939c5c407fed2178a) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test1e79c736bdaf36bcc331) { +TEST_F(GenEmitterTest, test974ae82483391d01787b) { Emitter out; out << Comment("comment"); out << BeginSeq; @@ -2899,13 +2967,14 @@ TEST_F(GenEmitterTest, test1e79c736bdaf36bcc331) { EXPECT_CALL(handler, OnDocumentStart(_)); EXPECT_CALL(handler, OnSequenceStart(_, "?", 0, _)); + EXPECT_CALL(handler, OnAnchor(_, "anchor")); EXPECT_CALL(handler, OnScalar(_, "?", 1, "foo")); EXPECT_CALL(handler, OnSequenceEnd()); EXPECT_CALL(handler, OnDocumentEnd()); Parse(out.c_str()); } -TEST_F(GenEmitterTest, testa932875b266f62a32a15) { +TEST_F(GenEmitterTest, test7fc68b49cfe198b30eeb) { Emitter out; out << Comment("comment"); out << BeginSeq; @@ -2916,13 +2985,14 @@ TEST_F(GenEmitterTest, testa932875b266f62a32a15) { EXPECT_CALL(handler, OnDocumentStart(_)); EXPECT_CALL(handler, OnSequenceStart(_, "?", 0, _)); + EXPECT_CALL(handler, OnAnchor(_, "anchor")); EXPECT_CALL(handler, OnScalar(_, "?", 1, "foo")); EXPECT_CALL(handler, OnSequenceEnd()); EXPECT_CALL(handler, OnDocumentEnd()); Parse(out.c_str()); } -TEST_F(GenEmitterTest, test7211a0d0a7d7b8957fbd) { +TEST_F(GenEmitterTest, test41644c59ff95f8ec5ec2) { Emitter out; out << BeginSeq; out << Comment("comment"); @@ -2933,13 +3003,14 @@ TEST_F(GenEmitterTest, test7211a0d0a7d7b8957fbd) { EXPECT_CALL(handler, OnDocumentStart(_)); EXPECT_CALL(handler, OnSequenceStart(_, "?", 0, _)); + EXPECT_CALL(handler, OnAnchor(_, "anchor")); EXPECT_CALL(handler, OnScalar(_, "?", 1, "foo")); EXPECT_CALL(handler, OnSequenceEnd()); EXPECT_CALL(handler, OnDocumentEnd()); Parse(out.c_str()); } -TEST_F(GenEmitterTest, testfe2ed84c5a19bea4324a) { +TEST_F(GenEmitterTest, testa3a24413b537aece4834) { Emitter out; out << BeginSeq; out << Anchor("anchor"); @@ -2950,13 +3021,14 @@ TEST_F(GenEmitterTest, testfe2ed84c5a19bea4324a) { EXPECT_CALL(handler, OnDocumentStart(_)); EXPECT_CALL(handler, OnSequenceStart(_, "?", 0, _)); + EXPECT_CALL(handler, OnAnchor(_, "anchor")); EXPECT_CALL(handler, OnScalar(_, "?", 1, "foo")); EXPECT_CALL(handler, OnSequenceEnd()); EXPECT_CALL(handler, OnDocumentEnd()); Parse(out.c_str()); } -TEST_F(GenEmitterTest, test05fe79e124bcadf04952) { +TEST_F(GenEmitterTest, testc4516128af938868b120) { Emitter out; out << BeginSeq; out << Anchor("anchor"); @@ -2967,13 +3039,14 @@ TEST_F(GenEmitterTest, test05fe79e124bcadf04952) { EXPECT_CALL(handler, OnDocumentStart(_)); EXPECT_CALL(handler, OnSequenceStart(_, "?", 0, _)); + EXPECT_CALL(handler, OnAnchor(_, "anchor")); EXPECT_CALL(handler, OnScalar(_, "?", 1, "foo")); EXPECT_CALL(handler, OnSequenceEnd()); EXPECT_CALL(handler, OnDocumentEnd()); Parse(out.c_str()); } -TEST_F(GenEmitterTest, test6deaaa4620537aec93bb) { +TEST_F(GenEmitterTest, testef3c20a56c8a3993cc2d) { Emitter out; out << BeginSeq; out << Anchor("anchor"); @@ -2984,13 +3057,14 @@ TEST_F(GenEmitterTest, test6deaaa4620537aec93bb) { EXPECT_CALL(handler, OnDocumentStart(_)); EXPECT_CALL(handler, OnSequenceStart(_, "?", 0, _)); + EXPECT_CALL(handler, OnAnchor(_, "anchor")); EXPECT_CALL(handler, OnScalar(_, "?", 1, "foo")); EXPECT_CALL(handler, OnSequenceEnd()); EXPECT_CALL(handler, OnDocumentEnd()); Parse(out.c_str()); } -TEST_F(GenEmitterTest, testd8f4b7fd570238af0ac6) { +TEST_F(GenEmitterTest, test83aceee2ee6446347fba) { Emitter out; out << Comment("comment"); out << BeginSeq; @@ -3000,13 +3074,14 @@ TEST_F(GenEmitterTest, testd8f4b7fd570238af0ac6) { EXPECT_CALL(handler, OnDocumentStart(_)); EXPECT_CALL(handler, OnSequenceStart(_, "?", 0, _)); + EXPECT_CALL(handler, OnAnchor(_, "anchor")); EXPECT_CALL(handler, OnScalar(_, "?", 1, "foo")); EXPECT_CALL(handler, OnSequenceEnd()); EXPECT_CALL(handler, OnDocumentEnd()); Parse(out.c_str()); } -TEST_F(GenEmitterTest, test6aa495401fa882fd6ef0) { +TEST_F(GenEmitterTest, test5a054d76c67b6de340e2) { Emitter out; out << Comment("comment"); out << BeginSeq; @@ -3016,13 +3091,14 @@ TEST_F(GenEmitterTest, test6aa495401fa882fd6ef0) { EXPECT_CALL(handler, OnDocumentStart(_)); EXPECT_CALL(handler, OnSequenceStart(_, "?", 0, _)); + EXPECT_CALL(handler, OnAnchor(_, "anchor")); EXPECT_CALL(handler, OnScalar(_, "?", 1, "foo")); EXPECT_CALL(handler, OnSequenceEnd()); EXPECT_CALL(handler, OnDocumentEnd()); Parse(out.c_str()); } -TEST_F(GenEmitterTest, test3388236bb529c50f5341) { +TEST_F(GenEmitterTest, testc6706e6b6fc94d1e4752) { Emitter out; out << BeginSeq; out << Comment("comment"); @@ -3032,13 +3108,14 @@ TEST_F(GenEmitterTest, test3388236bb529c50f5341) { EXPECT_CALL(handler, OnDocumentStart(_)); EXPECT_CALL(handler, OnSequenceStart(_, "?", 0, _)); + EXPECT_CALL(handler, OnAnchor(_, "anchor")); EXPECT_CALL(handler, OnScalar(_, "?", 1, "foo")); EXPECT_CALL(handler, OnSequenceEnd()); EXPECT_CALL(handler, OnDocumentEnd()); Parse(out.c_str()); } -TEST_F(GenEmitterTest, test79f4266262dffb3f8346) { +TEST_F(GenEmitterTest, test72f3ded341d6b5d21803) { Emitter out; out << BeginSeq; out << Anchor("anchor"); @@ -3048,13 +3125,14 @@ TEST_F(GenEmitterTest, test79f4266262dffb3f8346) { EXPECT_CALL(handler, OnDocumentStart(_)); EXPECT_CALL(handler, OnSequenceStart(_, "?", 0, _)); + EXPECT_CALL(handler, OnAnchor(_, "anchor")); EXPECT_CALL(handler, OnScalar(_, "?", 1, "foo")); EXPECT_CALL(handler, OnSequenceEnd()); EXPECT_CALL(handler, OnDocumentEnd()); Parse(out.c_str()); } -TEST_F(GenEmitterTest, teste858dd76c42bbd281706) { +TEST_F(GenEmitterTest, test7dc830828b604b5d1839) { Emitter out; out << BeginSeq; out << Anchor("anchor"); @@ -3064,13 +3142,14 @@ TEST_F(GenEmitterTest, teste858dd76c42bbd281706) { EXPECT_CALL(handler, OnDocumentStart(_)); EXPECT_CALL(handler, OnSequenceStart(_, "?", 0, _)); + EXPECT_CALL(handler, OnAnchor(_, "anchor")); EXPECT_CALL(handler, OnScalar(_, "?", 1, "foo")); EXPECT_CALL(handler, OnSequenceEnd()); EXPECT_CALL(handler, OnDocumentEnd()); Parse(out.c_str()); } -TEST_F(GenEmitterTest, test7487786b0131b1337e71) { +TEST_F(GenEmitterTest, test3a5baef0d6a62e5880ef) { Emitter out; out << BeginSeq; out << Anchor("anchor"); @@ -3080,13 +3159,14 @@ TEST_F(GenEmitterTest, test7487786b0131b1337e71) { EXPECT_CALL(handler, OnDocumentStart(_)); EXPECT_CALL(handler, OnSequenceStart(_, "?", 0, _)); + EXPECT_CALL(handler, OnAnchor(_, "anchor")); EXPECT_CALL(handler, OnScalar(_, "?", 1, "foo")); EXPECT_CALL(handler, OnSequenceEnd()); EXPECT_CALL(handler, OnDocumentEnd()); Parse(out.c_str()); } -TEST_F(GenEmitterTest, testa9b9110a8d2175a57e7c) { +TEST_F(GenEmitterTest, testfe7bf25b7a5525cab12a) { Emitter out; out << Comment("comment"); out << BeginSeq; @@ -3098,13 +3178,14 @@ TEST_F(GenEmitterTest, testa9b9110a8d2175a57e7c) { EXPECT_CALL(handler, OnDocumentStart(_)); EXPECT_CALL(handler, OnSequenceStart(_, "?", 0, _)); + EXPECT_CALL(handler, OnAnchor(_, "anchor")); EXPECT_CALL(handler, OnScalar(_, "tag", 1, "foo")); EXPECT_CALL(handler, OnSequenceEnd()); EXPECT_CALL(handler, OnDocumentEnd()); Parse(out.c_str()); } -TEST_F(GenEmitterTest, teste99a7f40fb4d939e2da7) { +TEST_F(GenEmitterTest, test817bf3d583230e503f8e) { Emitter out; out << Comment("comment"); out << BeginSeq; @@ -3116,13 +3197,14 @@ TEST_F(GenEmitterTest, teste99a7f40fb4d939e2da7) { EXPECT_CALL(handler, OnDocumentStart(_)); EXPECT_CALL(handler, OnSequenceStart(_, "?", 0, _)); + EXPECT_CALL(handler, OnAnchor(_, "anchor")); EXPECT_CALL(handler, OnScalar(_, "tag", 1, "foo")); EXPECT_CALL(handler, OnSequenceEnd()); EXPECT_CALL(handler, OnDocumentEnd()); Parse(out.c_str()); } -TEST_F(GenEmitterTest, test0e253895bd9cff694f9a) { +TEST_F(GenEmitterTest, testab122e386b3e30ea59e2) { Emitter out; out << BeginSeq; out << Comment("comment"); @@ -3134,13 +3216,14 @@ TEST_F(GenEmitterTest, test0e253895bd9cff694f9a) { EXPECT_CALL(handler, OnDocumentStart(_)); EXPECT_CALL(handler, OnSequenceStart(_, "?", 0, _)); + EXPECT_CALL(handler, OnAnchor(_, "anchor")); EXPECT_CALL(handler, OnScalar(_, "tag", 1, "foo")); EXPECT_CALL(handler, OnSequenceEnd()); EXPECT_CALL(handler, OnDocumentEnd()); Parse(out.c_str()); } -TEST_F(GenEmitterTest, testf0a8ffd3a895826b093c) { +TEST_F(GenEmitterTest, test466c3e0dbec8e9660837) { Emitter out; out << BeginSeq; out << VerbatimTag("tag"); @@ -3152,13 +3235,14 @@ TEST_F(GenEmitterTest, testf0a8ffd3a895826b093c) { EXPECT_CALL(handler, OnDocumentStart(_)); EXPECT_CALL(handler, OnSequenceStart(_, "?", 0, _)); + EXPECT_CALL(handler, OnAnchor(_, "anchor")); EXPECT_CALL(handler, OnScalar(_, "tag", 1, "foo")); EXPECT_CALL(handler, OnSequenceEnd()); EXPECT_CALL(handler, OnDocumentEnd()); Parse(out.c_str()); } -TEST_F(GenEmitterTest, testdae2c374518234f70e9f) { +TEST_F(GenEmitterTest, test9fc49f92e554cd85e349) { Emitter out; out << BeginSeq; out << VerbatimTag("tag"); @@ -3170,13 +3254,14 @@ TEST_F(GenEmitterTest, testdae2c374518234f70e9f) { EXPECT_CALL(handler, OnDocumentStart(_)); EXPECT_CALL(handler, OnSequenceStart(_, "?", 0, _)); + EXPECT_CALL(handler, OnAnchor(_, "anchor")); EXPECT_CALL(handler, OnScalar(_, "tag", 1, "foo")); EXPECT_CALL(handler, OnSequenceEnd()); EXPECT_CALL(handler, OnDocumentEnd()); Parse(out.c_str()); } -TEST_F(GenEmitterTest, testc0dae4cd70b3f409cbb4) { +TEST_F(GenEmitterTest, testf9d2f39bdbd217d70868) { Emitter out; out << BeginSeq; out << VerbatimTag("tag"); @@ -3188,13 +3273,14 @@ TEST_F(GenEmitterTest, testc0dae4cd70b3f409cbb4) { EXPECT_CALL(handler, OnDocumentStart(_)); EXPECT_CALL(handler, OnSequenceStart(_, "?", 0, _)); + EXPECT_CALL(handler, OnAnchor(_, "anchor")); EXPECT_CALL(handler, OnScalar(_, "tag", 1, "foo")); EXPECT_CALL(handler, OnSequenceEnd()); EXPECT_CALL(handler, OnDocumentEnd()); Parse(out.c_str()); } -TEST_F(GenEmitterTest, testc26f714fb6f9b1ee6cf9) { +TEST_F(GenEmitterTest, test1ce3d77707f18ec48a19) { Emitter out; out << Comment("comment"); out << BeginSeq; @@ -3205,13 +3291,14 @@ TEST_F(GenEmitterTest, testc26f714fb6f9b1ee6cf9) { EXPECT_CALL(handler, OnDocumentStart(_)); EXPECT_CALL(handler, OnSequenceStart(_, "?", 0, _)); + EXPECT_CALL(handler, OnAnchor(_, "anchor")); EXPECT_CALL(handler, OnScalar(_, "tag", 1, "foo")); EXPECT_CALL(handler, OnSequenceEnd()); EXPECT_CALL(handler, OnDocumentEnd()); Parse(out.c_str()); } -TEST_F(GenEmitterTest, test9644620dcb35431e2969) { +TEST_F(GenEmitterTest, test71df6ecc32e49ea961d4) { Emitter out; out << Comment("comment"); out << BeginSeq; @@ -3222,13 +3309,14 @@ TEST_F(GenEmitterTest, test9644620dcb35431e2969) { EXPECT_CALL(handler, OnDocumentStart(_)); EXPECT_CALL(handler, OnSequenceStart(_, "?", 0, _)); + EXPECT_CALL(handler, OnAnchor(_, "anchor")); EXPECT_CALL(handler, OnScalar(_, "tag", 1, "foo")); EXPECT_CALL(handler, OnSequenceEnd()); EXPECT_CALL(handler, OnDocumentEnd()); Parse(out.c_str()); } -TEST_F(GenEmitterTest, test1cfceae5c8e4a64a43ae) { +TEST_F(GenEmitterTest, test8f37b0a6cc287f8c922f) { Emitter out; out << BeginSeq; out << Comment("comment"); @@ -3239,13 +3327,14 @@ TEST_F(GenEmitterTest, test1cfceae5c8e4a64a43ae) { EXPECT_CALL(handler, OnDocumentStart(_)); EXPECT_CALL(handler, OnSequenceStart(_, "?", 0, _)); + EXPECT_CALL(handler, OnAnchor(_, "anchor")); EXPECT_CALL(handler, OnScalar(_, "tag", 1, "foo")); EXPECT_CALL(handler, OnSequenceEnd()); EXPECT_CALL(handler, OnDocumentEnd()); Parse(out.c_str()); } -TEST_F(GenEmitterTest, test64f296d213a7ddc7738c) { +TEST_F(GenEmitterTest, testf992e2a1f7d737647506) { Emitter out; out << BeginSeq; out << VerbatimTag("tag"); @@ -3256,13 +3345,14 @@ TEST_F(GenEmitterTest, test64f296d213a7ddc7738c) { EXPECT_CALL(handler, OnDocumentStart(_)); EXPECT_CALL(handler, OnSequenceStart(_, "?", 0, _)); + EXPECT_CALL(handler, OnAnchor(_, "anchor")); EXPECT_CALL(handler, OnScalar(_, "tag", 1, "foo")); EXPECT_CALL(handler, OnSequenceEnd()); EXPECT_CALL(handler, OnDocumentEnd()); Parse(out.c_str()); } -TEST_F(GenEmitterTest, testb645b7ae7c95adc70e6f) { +TEST_F(GenEmitterTest, testd79381f97cdd0af81ae4) { Emitter out; out << BeginSeq; out << VerbatimTag("tag"); @@ -3273,13 +3363,14 @@ TEST_F(GenEmitterTest, testb645b7ae7c95adc70e6f) { EXPECT_CALL(handler, OnDocumentStart(_)); EXPECT_CALL(handler, OnSequenceStart(_, "?", 0, _)); + EXPECT_CALL(handler, OnAnchor(_, "anchor")); EXPECT_CALL(handler, OnScalar(_, "tag", 1, "foo")); EXPECT_CALL(handler, OnSequenceEnd()); EXPECT_CALL(handler, OnDocumentEnd()); Parse(out.c_str()); } -TEST_F(GenEmitterTest, test3a6fb33f542118758a78) { +TEST_F(GenEmitterTest, test74ca1feb5f0c520a8518) { Emitter out; out << BeginSeq; out << VerbatimTag("tag"); @@ -3290,13 +3381,14 @@ TEST_F(GenEmitterTest, test3a6fb33f542118758a78) { EXPECT_CALL(handler, OnDocumentStart(_)); EXPECT_CALL(handler, OnSequenceStart(_, "?", 0, _)); + EXPECT_CALL(handler, OnAnchor(_, "anchor")); EXPECT_CALL(handler, OnScalar(_, "tag", 1, "foo")); EXPECT_CALL(handler, OnSequenceEnd()); EXPECT_CALL(handler, OnDocumentEnd()); Parse(out.c_str()); } -TEST_F(GenEmitterTest, test98b9868c139b816dcc00) { +TEST_F(GenEmitterTest, teste86e6fd56707272c091b) { Emitter out; out << Comment("comment"); out << BeginSeq; @@ -3314,7 +3406,7 @@ TEST_F(GenEmitterTest, test98b9868c139b816dcc00) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test69526d609eb86d3b7917) { +TEST_F(GenEmitterTest, test1e6f73bc378c184c786b) { Emitter out; out << Comment("comment"); out << BeginSeq; @@ -3332,7 +3424,7 @@ TEST_F(GenEmitterTest, test69526d609eb86d3b7917) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, testb569fe86774c96603a89) { +TEST_F(GenEmitterTest, test3fbac5e1aef66dc40bf7) { Emitter out; out << BeginSeq; out << Comment("comment"); @@ -3350,7 +3442,7 @@ TEST_F(GenEmitterTest, testb569fe86774c96603a89) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test03793bbef87d15c4ec74) { +TEST_F(GenEmitterTest, test558c4bf1c9c6e4e81e98) { Emitter out; out << BeginSeq; out << "foo"; @@ -3368,7 +3460,7 @@ TEST_F(GenEmitterTest, test03793bbef87d15c4ec74) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test508ca3314339f0dfb5c8) { +TEST_F(GenEmitterTest, testfa6d88b26c0072cddb26) { Emitter out; out << BeginSeq; out << "foo"; @@ -3386,7 +3478,7 @@ TEST_F(GenEmitterTest, test508ca3314339f0dfb5c8) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test7c8a731b32213075b25a) { +TEST_F(GenEmitterTest, test40a5af3360fb3d9e79f1) { Emitter out; out << BeginSeq; out << "foo"; @@ -3404,7 +3496,7 @@ TEST_F(GenEmitterTest, test7c8a731b32213075b25a) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, testb2a840af01cc1074e80f) { +TEST_F(GenEmitterTest, test451dd95b95b7e958bb03) { Emitter out; out << BeginSeq; out << "foo"; @@ -3422,7 +3514,7 @@ TEST_F(GenEmitterTest, testb2a840af01cc1074e80f) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test8e372218ea9564c579b6) { +TEST_F(GenEmitterTest, test1717ad2d772bafb9b573) { Emitter out; out << Comment("comment"); out << BeginSeq; @@ -3439,7 +3531,7 @@ TEST_F(GenEmitterTest, test8e372218ea9564c579b6) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test9135f186de91cf9e7538) { +TEST_F(GenEmitterTest, testedc6737e8b2f5b23b42e) { Emitter out; out << Comment("comment"); out << BeginSeq; @@ -3456,7 +3548,7 @@ TEST_F(GenEmitterTest, test9135f186de91cf9e7538) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test81be571777fd06470d73) { +TEST_F(GenEmitterTest, test771c7d28c0b8c184e2c7) { Emitter out; out << BeginSeq; out << Comment("comment"); @@ -3473,7 +3565,7 @@ TEST_F(GenEmitterTest, test81be571777fd06470d73) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test7614d4928827481a2d8e) { +TEST_F(GenEmitterTest, test469a446f0b22e9b6d269) { Emitter out; out << BeginSeq; out << "foo"; @@ -3490,7 +3582,7 @@ TEST_F(GenEmitterTest, test7614d4928827481a2d8e) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, teste21f9902a4a5e3c73628) { +TEST_F(GenEmitterTest, testec45b0503f312be47336) { Emitter out; out << BeginSeq; out << "foo"; @@ -3507,7 +3599,7 @@ TEST_F(GenEmitterTest, teste21f9902a4a5e3c73628) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test879d7957916fb5526b7e) { +TEST_F(GenEmitterTest, test1bfc4f39d6730acb6a12) { Emitter out; out << BeginSeq; out << "foo"; @@ -3524,7 +3616,7 @@ TEST_F(GenEmitterTest, test879d7957916fb5526b7e) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test8e4e2b3618c7fe67384b) { +TEST_F(GenEmitterTest, test9bc9a72ad06084dc8cf8) { Emitter out; out << BeginSeq; out << "foo"; @@ -3541,7 +3633,7 @@ TEST_F(GenEmitterTest, test8e4e2b3618c7fe67384b) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test5e8672f96ce5a11b846a) { +TEST_F(GenEmitterTest, test62c996cdfc1d3b77b7ec) { Emitter out; out << Comment("comment"); out << BeginSeq; @@ -3556,14 +3648,16 @@ TEST_F(GenEmitterTest, test5e8672f96ce5a11b846a) { EXPECT_CALL(handler, OnDocumentStart(_)); EXPECT_CALL(handler, OnSequenceStart(_, "?", 0, _)); + EXPECT_CALL(handler, OnAnchor(_, "anchor")); EXPECT_CALL(handler, OnScalar(_, "tag", 1, "foo")); + EXPECT_CALL(handler, OnAnchor(_, "other")); EXPECT_CALL(handler, OnScalar(_, "tag", 2, "bar")); EXPECT_CALL(handler, OnSequenceEnd()); EXPECT_CALL(handler, OnDocumentEnd()); Parse(out.c_str()); } -TEST_F(GenEmitterTest, test26c021c876f1ef91cff4) { +TEST_F(GenEmitterTest, test1d038936a340d5bef490) { Emitter out; out << Comment("comment"); out << BeginSeq; @@ -3578,14 +3672,16 @@ TEST_F(GenEmitterTest, test26c021c876f1ef91cff4) { EXPECT_CALL(handler, OnDocumentStart(_)); EXPECT_CALL(handler, OnSequenceStart(_, "?", 0, _)); + EXPECT_CALL(handler, OnAnchor(_, "anchor")); EXPECT_CALL(handler, OnScalar(_, "tag", 1, "foo")); + EXPECT_CALL(handler, OnAnchor(_, "other")); EXPECT_CALL(handler, OnScalar(_, "tag", 2, "bar")); EXPECT_CALL(handler, OnSequenceEnd()); EXPECT_CALL(handler, OnDocumentEnd()); Parse(out.c_str()); } -TEST_F(GenEmitterTest, test04da690e5c32b7bf1dc3) { +TEST_F(GenEmitterTest, test7057f64ac570dbe3c1ca) { Emitter out; out << BeginSeq; out << Comment("comment"); @@ -3600,14 +3696,16 @@ TEST_F(GenEmitterTest, test04da690e5c32b7bf1dc3) { EXPECT_CALL(handler, OnDocumentStart(_)); EXPECT_CALL(handler, OnSequenceStart(_, "?", 0, _)); + EXPECT_CALL(handler, OnAnchor(_, "anchor")); EXPECT_CALL(handler, OnScalar(_, "tag", 1, "foo")); + EXPECT_CALL(handler, OnAnchor(_, "other")); EXPECT_CALL(handler, OnScalar(_, "tag", 2, "bar")); EXPECT_CALL(handler, OnSequenceEnd()); EXPECT_CALL(handler, OnDocumentEnd()); Parse(out.c_str()); } -TEST_F(GenEmitterTest, test4bf435d6bad0ca7f8016) { +TEST_F(GenEmitterTest, testbfe0890de3ffc73f0f9d) { Emitter out; out << BeginSeq; out << VerbatimTag("tag"); @@ -3622,14 +3720,16 @@ TEST_F(GenEmitterTest, test4bf435d6bad0ca7f8016) { EXPECT_CALL(handler, OnDocumentStart(_)); EXPECT_CALL(handler, OnSequenceStart(_, "?", 0, _)); + EXPECT_CALL(handler, OnAnchor(_, "anchor")); EXPECT_CALL(handler, OnScalar(_, "tag", 1, "foo")); + EXPECT_CALL(handler, OnAnchor(_, "other")); EXPECT_CALL(handler, OnScalar(_, "tag", 2, "bar")); EXPECT_CALL(handler, OnSequenceEnd()); EXPECT_CALL(handler, OnDocumentEnd()); Parse(out.c_str()); } -TEST_F(GenEmitterTest, test38a40c04ceadf66cb77a) { +TEST_F(GenEmitterTest, test5faa7320a493247b4f8b) { Emitter out; out << BeginSeq; out << VerbatimTag("tag"); @@ -3644,14 +3744,16 @@ TEST_F(GenEmitterTest, test38a40c04ceadf66cb77a) { EXPECT_CALL(handler, OnDocumentStart(_)); EXPECT_CALL(handler, OnSequenceStart(_, "?", 0, _)); + EXPECT_CALL(handler, OnAnchor(_, "anchor")); EXPECT_CALL(handler, OnScalar(_, "tag", 1, "foo")); + EXPECT_CALL(handler, OnAnchor(_, "other")); EXPECT_CALL(handler, OnScalar(_, "tag", 2, "bar")); EXPECT_CALL(handler, OnSequenceEnd()); EXPECT_CALL(handler, OnDocumentEnd()); Parse(out.c_str()); } -TEST_F(GenEmitterTest, test73f543da75154971fa58) { +TEST_F(GenEmitterTest, test929fbc93b3d6d98b1f0a) { Emitter out; out << BeginSeq; out << VerbatimTag("tag"); @@ -3666,14 +3768,16 @@ TEST_F(GenEmitterTest, test73f543da75154971fa58) { EXPECT_CALL(handler, OnDocumentStart(_)); EXPECT_CALL(handler, OnSequenceStart(_, "?", 0, _)); + EXPECT_CALL(handler, OnAnchor(_, "anchor")); EXPECT_CALL(handler, OnScalar(_, "tag", 1, "foo")); + EXPECT_CALL(handler, OnAnchor(_, "other")); EXPECT_CALL(handler, OnScalar(_, "tag", 2, "bar")); EXPECT_CALL(handler, OnSequenceEnd()); EXPECT_CALL(handler, OnDocumentEnd()); Parse(out.c_str()); } -TEST_F(GenEmitterTest, test9c2a6b8e53a71cc271f1) { +TEST_F(GenEmitterTest, testcc7d1ad7797581b37549) { Emitter out; out << BeginSeq; out << VerbatimTag("tag"); @@ -3688,14 +3792,16 @@ TEST_F(GenEmitterTest, test9c2a6b8e53a71cc271f1) { EXPECT_CALL(handler, OnDocumentStart(_)); EXPECT_CALL(handler, OnSequenceStart(_, "?", 0, _)); + EXPECT_CALL(handler, OnAnchor(_, "anchor")); EXPECT_CALL(handler, OnScalar(_, "tag", 1, "foo")); + EXPECT_CALL(handler, OnAnchor(_, "other")); EXPECT_CALL(handler, OnScalar(_, "tag", 2, "bar")); EXPECT_CALL(handler, OnSequenceEnd()); EXPECT_CALL(handler, OnDocumentEnd()); Parse(out.c_str()); } -TEST_F(GenEmitterTest, test76e7370aa293790ca0b6) { +TEST_F(GenEmitterTest, test1115ba981ba8f739ddf2) { Emitter out; out << Comment("comment"); out << BeginSeq; @@ -3709,14 +3815,16 @@ TEST_F(GenEmitterTest, test76e7370aa293790ca0b6) { EXPECT_CALL(handler, OnDocumentStart(_)); EXPECT_CALL(handler, OnSequenceStart(_, "?", 0, _)); + EXPECT_CALL(handler, OnAnchor(_, "anchor")); EXPECT_CALL(handler, OnScalar(_, "tag", 1, "foo")); + EXPECT_CALL(handler, OnAnchor(_, "other")); EXPECT_CALL(handler, OnScalar(_, "tag", 2, "bar")); EXPECT_CALL(handler, OnSequenceEnd()); EXPECT_CALL(handler, OnDocumentEnd()); Parse(out.c_str()); } -TEST_F(GenEmitterTest, test530430c212524fd2d917) { +TEST_F(GenEmitterTest, testf7ca743a82040e1313a8) { Emitter out; out << Comment("comment"); out << BeginSeq; @@ -3730,14 +3838,16 @@ TEST_F(GenEmitterTest, test530430c212524fd2d917) { EXPECT_CALL(handler, OnDocumentStart(_)); EXPECT_CALL(handler, OnSequenceStart(_, "?", 0, _)); + EXPECT_CALL(handler, OnAnchor(_, "anchor")); EXPECT_CALL(handler, OnScalar(_, "tag", 1, "foo")); + EXPECT_CALL(handler, OnAnchor(_, "other")); EXPECT_CALL(handler, OnScalar(_, "tag", 2, "bar")); EXPECT_CALL(handler, OnSequenceEnd()); EXPECT_CALL(handler, OnDocumentEnd()); Parse(out.c_str()); } -TEST_F(GenEmitterTest, testdc001467ce9060c3e34f) { +TEST_F(GenEmitterTest, testa4e0257ad6c987178ca4) { Emitter out; out << BeginSeq; out << Comment("comment"); @@ -3751,14 +3861,16 @@ TEST_F(GenEmitterTest, testdc001467ce9060c3e34f) { EXPECT_CALL(handler, OnDocumentStart(_)); EXPECT_CALL(handler, OnSequenceStart(_, "?", 0, _)); + EXPECT_CALL(handler, OnAnchor(_, "anchor")); EXPECT_CALL(handler, OnScalar(_, "tag", 1, "foo")); + EXPECT_CALL(handler, OnAnchor(_, "other")); EXPECT_CALL(handler, OnScalar(_, "tag", 2, "bar")); EXPECT_CALL(handler, OnSequenceEnd()); EXPECT_CALL(handler, OnDocumentEnd()); Parse(out.c_str()); } -TEST_F(GenEmitterTest, teste650915c87ea88e85b7d) { +TEST_F(GenEmitterTest, testb65ceea0d4080b44180e) { Emitter out; out << BeginSeq; out << VerbatimTag("tag"); @@ -3772,14 +3884,16 @@ TEST_F(GenEmitterTest, teste650915c87ea88e85b7d) { EXPECT_CALL(handler, OnDocumentStart(_)); EXPECT_CALL(handler, OnSequenceStart(_, "?", 0, _)); + EXPECT_CALL(handler, OnAnchor(_, "anchor")); EXPECT_CALL(handler, OnScalar(_, "tag", 1, "foo")); + EXPECT_CALL(handler, OnAnchor(_, "other")); EXPECT_CALL(handler, OnScalar(_, "tag", 2, "bar")); EXPECT_CALL(handler, OnSequenceEnd()); EXPECT_CALL(handler, OnDocumentEnd()); Parse(out.c_str()); } -TEST_F(GenEmitterTest, testc288c80940cbb3539620) { +TEST_F(GenEmitterTest, test4fcd60d48dbd7b07e289) { Emitter out; out << BeginSeq; out << VerbatimTag("tag"); @@ -3793,14 +3907,16 @@ TEST_F(GenEmitterTest, testc288c80940cbb3539620) { EXPECT_CALL(handler, OnDocumentStart(_)); EXPECT_CALL(handler, OnSequenceStart(_, "?", 0, _)); + EXPECT_CALL(handler, OnAnchor(_, "anchor")); EXPECT_CALL(handler, OnScalar(_, "tag", 1, "foo")); + EXPECT_CALL(handler, OnAnchor(_, "other")); EXPECT_CALL(handler, OnScalar(_, "tag", 2, "bar")); EXPECT_CALL(handler, OnSequenceEnd()); EXPECT_CALL(handler, OnDocumentEnd()); Parse(out.c_str()); } -TEST_F(GenEmitterTest, test18256634c494bbf715ac) { +TEST_F(GenEmitterTest, test92704937d4e130b43390) { Emitter out; out << BeginSeq; out << VerbatimTag("tag"); @@ -3814,14 +3930,16 @@ TEST_F(GenEmitterTest, test18256634c494bbf715ac) { EXPECT_CALL(handler, OnDocumentStart(_)); EXPECT_CALL(handler, OnSequenceStart(_, "?", 0, _)); + EXPECT_CALL(handler, OnAnchor(_, "anchor")); EXPECT_CALL(handler, OnScalar(_, "tag", 1, "foo")); + EXPECT_CALL(handler, OnAnchor(_, "other")); EXPECT_CALL(handler, OnScalar(_, "tag", 2, "bar")); EXPECT_CALL(handler, OnSequenceEnd()); EXPECT_CALL(handler, OnDocumentEnd()); Parse(out.c_str()); } -TEST_F(GenEmitterTest, testfaecbb520871c68e74c2) { +TEST_F(GenEmitterTest, test029a31902f93dfa9ea7b) { Emitter out; out << BeginSeq; out << VerbatimTag("tag"); @@ -3835,14 +3953,16 @@ TEST_F(GenEmitterTest, testfaecbb520871c68e74c2) { EXPECT_CALL(handler, OnDocumentStart(_)); EXPECT_CALL(handler, OnSequenceStart(_, "?", 0, _)); + EXPECT_CALL(handler, OnAnchor(_, "anchor")); EXPECT_CALL(handler, OnScalar(_, "tag", 1, "foo")); + EXPECT_CALL(handler, OnAnchor(_, "other")); EXPECT_CALL(handler, OnScalar(_, "tag", 2, "bar")); EXPECT_CALL(handler, OnSequenceEnd()); EXPECT_CALL(handler, OnDocumentEnd()); Parse(out.c_str()); } -TEST_F(GenEmitterTest, testd3b1e7d0410f03b69666) { +TEST_F(GenEmitterTest, test40b4e7494e5b850d26f4) { Emitter out; out << Comment("comment"); out << BeginDoc; @@ -3857,7 +3977,7 @@ TEST_F(GenEmitterTest, testd3b1e7d0410f03b69666) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test2654cfdffd9be4c24917) { +TEST_F(GenEmitterTest, test64d2ab5993b67281212b) { Emitter out; out << BeginDoc; out << Comment("comment"); @@ -3872,7 +3992,7 @@ TEST_F(GenEmitterTest, test2654cfdffd9be4c24917) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, testad5fd5e6d524dd2c4907) { +TEST_F(GenEmitterTest, teste71b9b975d71c18a2897) { Emitter out; out << BeginDoc; out << BeginMap; @@ -3887,7 +4007,7 @@ TEST_F(GenEmitterTest, testad5fd5e6d524dd2c4907) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, testb1b59e0e48f25bf90045) { +TEST_F(GenEmitterTest, test138039761e432a5ba11e) { Emitter out; out << BeginDoc; out << BeginMap; @@ -3902,7 +4022,7 @@ TEST_F(GenEmitterTest, testb1b59e0e48f25bf90045) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test6a1cd91064a302335507) { +TEST_F(GenEmitterTest, test6684d2eacb3f094bfc84) { Emitter out; out << BeginDoc; out << BeginMap; @@ -3917,7 +4037,7 @@ TEST_F(GenEmitterTest, test6a1cd91064a302335507) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test7542cfaba20c029d74d2) { +TEST_F(GenEmitterTest, test8624a705f2167d4db358) { Emitter out; out << Comment("comment"); out << BeginDoc; @@ -3931,7 +4051,7 @@ TEST_F(GenEmitterTest, test7542cfaba20c029d74d2) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test7c64ec9fde647f95ae64) { +TEST_F(GenEmitterTest, test90877a1ec609edb69bce) { Emitter out; out << BeginDoc; out << Comment("comment"); @@ -3945,7 +4065,7 @@ TEST_F(GenEmitterTest, test7c64ec9fde647f95ae64) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, testbba41212c116596476f3) { +TEST_F(GenEmitterTest, test5f925d3c910a7e32bb99) { Emitter out; out << BeginDoc; out << BeginMap; @@ -3959,7 +4079,7 @@ TEST_F(GenEmitterTest, testbba41212c116596476f3) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test66a3e0c8ba0580adcce7) { +TEST_F(GenEmitterTest, testffeb4955bf4ee9510a88) { Emitter out; out << BeginDoc; out << BeginMap; @@ -3973,7 +4093,7 @@ TEST_F(GenEmitterTest, test66a3e0c8ba0580adcce7) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test201f78790b978c826f38) { +TEST_F(GenEmitterTest, test769ee82c3bfc52d7a85d) { Emitter out; out << BeginDoc; out << BeginMap; @@ -3987,7 +4107,7 @@ TEST_F(GenEmitterTest, test201f78790b978c826f38) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test85630834239ca1a21b5f) { +TEST_F(GenEmitterTest, testdc4e16b5a48fe16102b4) { Emitter out; out << Comment("comment"); out << BeginDoc; @@ -4006,7 +4126,7 @@ TEST_F(GenEmitterTest, test85630834239ca1a21b5f) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test6e4aaceae87fea59c777) { +TEST_F(GenEmitterTest, testd3c578e5b5a6813a73c7) { Emitter out; out << BeginDoc; out << Comment("comment"); @@ -4025,7 +4145,7 @@ TEST_F(GenEmitterTest, test6e4aaceae87fea59c777) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test430815076958c69dc3ae) { +TEST_F(GenEmitterTest, test0034b2c9905b34f7f22e) { Emitter out; out << BeginDoc; out << BeginMap; @@ -4044,7 +4164,7 @@ TEST_F(GenEmitterTest, test430815076958c69dc3ae) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test699eb267a6012eb16178) { +TEST_F(GenEmitterTest, teste911e620becf080a4d96) { Emitter out; out << BeginDoc; out << BeginMap; @@ -4063,7 +4183,7 @@ TEST_F(GenEmitterTest, test699eb267a6012eb16178) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, testf51a095fb67ab6d7a2c9) { +TEST_F(GenEmitterTest, test7f8bbf619609651a2e55) { Emitter out; out << BeginDoc; out << BeginMap; @@ -4082,7 +4202,7 @@ TEST_F(GenEmitterTest, testf51a095fb67ab6d7a2c9) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test9c926b68cb956ee2c349) { +TEST_F(GenEmitterTest, test2974bda177bed72619f4) { Emitter out; out << BeginDoc; out << BeginMap; @@ -4101,7 +4221,7 @@ TEST_F(GenEmitterTest, test9c926b68cb956ee2c349) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, testeafd3facc48783e0e3a5) { +TEST_F(GenEmitterTest, testbc7a1599883ed8c27262) { Emitter out; out << BeginDoc; out << BeginMap; @@ -4120,7 +4240,7 @@ TEST_F(GenEmitterTest, testeafd3facc48783e0e3a5) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, testd702fb9ab5607d2ac819) { +TEST_F(GenEmitterTest, test323e14a02e02b94939fb) { Emitter out; out << Comment("comment"); out << BeginDoc; @@ -4138,7 +4258,7 @@ TEST_F(GenEmitterTest, testd702fb9ab5607d2ac819) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, teste442f82a0e6799f2610c) { +TEST_F(GenEmitterTest, test705ff113324bf0b4897c) { Emitter out; out << BeginDoc; out << Comment("comment"); @@ -4156,7 +4276,7 @@ TEST_F(GenEmitterTest, teste442f82a0e6799f2610c) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test60c91197fb0bb0c6abc3) { +TEST_F(GenEmitterTest, test587f5739ba58f0e21e0e) { Emitter out; out << BeginDoc; out << BeginMap; @@ -4174,7 +4294,7 @@ TEST_F(GenEmitterTest, test60c91197fb0bb0c6abc3) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, testb51cf53880c7efae7f00) { +TEST_F(GenEmitterTest, test31a8c7da96ebe3da3f6e) { Emitter out; out << BeginDoc; out << BeginMap; @@ -4192,7 +4312,7 @@ TEST_F(GenEmitterTest, testb51cf53880c7efae7f00) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test015d48eead15f53dda15) { +TEST_F(GenEmitterTest, test0b6fe270e4cf9fc21181) { Emitter out; out << BeginDoc; out << BeginMap; @@ -4210,7 +4330,7 @@ TEST_F(GenEmitterTest, test015d48eead15f53dda15) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, testa6ef6e851b7c9bb8695a) { +TEST_F(GenEmitterTest, testaf5869c722ea0dfb3394) { Emitter out; out << BeginDoc; out << BeginMap; @@ -4228,7 +4348,7 @@ TEST_F(GenEmitterTest, testa6ef6e851b7c9bb8695a) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test77f35be9920c06becdb3) { +TEST_F(GenEmitterTest, testc348837f92793a778246) { Emitter out; out << BeginDoc; out << BeginMap; @@ -4246,7 +4366,7 @@ TEST_F(GenEmitterTest, test77f35be9920c06becdb3) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, testdabd5e4aaaf01a76ac53) { +TEST_F(GenEmitterTest, test9d26ae9ec8db76a06a6f) { Emitter out; out << Comment("comment"); out << BeginDoc; @@ -4262,14 +4382,16 @@ TEST_F(GenEmitterTest, testdabd5e4aaaf01a76ac53) { EXPECT_CALL(handler, OnDocumentStart(_)); EXPECT_CALL(handler, OnMapStart(_, "?", 0, _)); + EXPECT_CALL(handler, OnAnchor(_, "anchor")); EXPECT_CALL(handler, OnScalar(_, "tag", 1, "foo")); + EXPECT_CALL(handler, OnAnchor(_, "other")); EXPECT_CALL(handler, OnScalar(_, "tag", 2, "bar")); EXPECT_CALL(handler, OnMapEnd()); EXPECT_CALL(handler, OnDocumentEnd()); Parse(out.c_str()); } -TEST_F(GenEmitterTest, testfdfbe118771b6de1e966) { +TEST_F(GenEmitterTest, test28691969bbaa41191640) { Emitter out; out << BeginDoc; out << Comment("comment"); @@ -4285,14 +4407,16 @@ TEST_F(GenEmitterTest, testfdfbe118771b6de1e966) { EXPECT_CALL(handler, OnDocumentStart(_)); EXPECT_CALL(handler, OnMapStart(_, "?", 0, _)); + EXPECT_CALL(handler, OnAnchor(_, "anchor")); EXPECT_CALL(handler, OnScalar(_, "tag", 1, "foo")); + EXPECT_CALL(handler, OnAnchor(_, "other")); EXPECT_CALL(handler, OnScalar(_, "tag", 2, "bar")); EXPECT_CALL(handler, OnMapEnd()); EXPECT_CALL(handler, OnDocumentEnd()); Parse(out.c_str()); } -TEST_F(GenEmitterTest, test4153b34f6f6392539a57) { +TEST_F(GenEmitterTest, testb38c27cd2556a14bb479) { Emitter out; out << BeginDoc; out << BeginMap; @@ -4308,14 +4432,16 @@ TEST_F(GenEmitterTest, test4153b34f6f6392539a57) { EXPECT_CALL(handler, OnDocumentStart(_)); EXPECT_CALL(handler, OnMapStart(_, "?", 0, _)); + EXPECT_CALL(handler, OnAnchor(_, "anchor")); EXPECT_CALL(handler, OnScalar(_, "tag", 1, "foo")); + EXPECT_CALL(handler, OnAnchor(_, "other")); EXPECT_CALL(handler, OnScalar(_, "tag", 2, "bar")); EXPECT_CALL(handler, OnMapEnd()); EXPECT_CALL(handler, OnDocumentEnd()); Parse(out.c_str()); } -TEST_F(GenEmitterTest, test73cfd1e2fec4af9bcd24) { +TEST_F(GenEmitterTest, test1103d3c99e3525075da6) { Emitter out; out << BeginDoc; out << BeginMap; @@ -4331,14 +4457,16 @@ TEST_F(GenEmitterTest, test73cfd1e2fec4af9bcd24) { EXPECT_CALL(handler, OnDocumentStart(_)); EXPECT_CALL(handler, OnMapStart(_, "?", 0, _)); + EXPECT_CALL(handler, OnAnchor(_, "anchor")); EXPECT_CALL(handler, OnScalar(_, "tag", 1, "foo")); + EXPECT_CALL(handler, OnAnchor(_, "other")); EXPECT_CALL(handler, OnScalar(_, "tag", 2, "bar")); EXPECT_CALL(handler, OnMapEnd()); EXPECT_CALL(handler, OnDocumentEnd()); Parse(out.c_str()); } -TEST_F(GenEmitterTest, test9a25bda309eadfbe8323) { +TEST_F(GenEmitterTest, testeb7edb5d1dfd039c72c3) { Emitter out; out << BeginDoc; out << BeginMap; @@ -4354,14 +4482,16 @@ TEST_F(GenEmitterTest, test9a25bda309eadfbe8323) { EXPECT_CALL(handler, OnDocumentStart(_)); EXPECT_CALL(handler, OnMapStart(_, "?", 0, _)); + EXPECT_CALL(handler, OnAnchor(_, "anchor")); EXPECT_CALL(handler, OnScalar(_, "tag", 1, "foo")); + EXPECT_CALL(handler, OnAnchor(_, "other")); EXPECT_CALL(handler, OnScalar(_, "tag", 2, "bar")); EXPECT_CALL(handler, OnMapEnd()); EXPECT_CALL(handler, OnDocumentEnd()); Parse(out.c_str()); } -TEST_F(GenEmitterTest, test7600d72d8bd5c481b7d2) { +TEST_F(GenEmitterTest, testa9862d708fcb755db479) { Emitter out; out << BeginDoc; out << BeginMap; @@ -4377,14 +4507,16 @@ TEST_F(GenEmitterTest, test7600d72d8bd5c481b7d2) { EXPECT_CALL(handler, OnDocumentStart(_)); EXPECT_CALL(handler, OnMapStart(_, "?", 0, _)); + EXPECT_CALL(handler, OnAnchor(_, "anchor")); EXPECT_CALL(handler, OnScalar(_, "tag", 1, "foo")); + EXPECT_CALL(handler, OnAnchor(_, "other")); EXPECT_CALL(handler, OnScalar(_, "tag", 2, "bar")); EXPECT_CALL(handler, OnMapEnd()); EXPECT_CALL(handler, OnDocumentEnd()); Parse(out.c_str()); } -TEST_F(GenEmitterTest, teste03fdb7c4a58b419523d) { +TEST_F(GenEmitterTest, testae3e98286336f0c5d2af) { Emitter out; out << BeginDoc; out << BeginMap; @@ -4400,14 +4532,16 @@ TEST_F(GenEmitterTest, teste03fdb7c4a58b419523d) { EXPECT_CALL(handler, OnDocumentStart(_)); EXPECT_CALL(handler, OnMapStart(_, "?", 0, _)); + EXPECT_CALL(handler, OnAnchor(_, "anchor")); EXPECT_CALL(handler, OnScalar(_, "tag", 1, "foo")); + EXPECT_CALL(handler, OnAnchor(_, "other")); EXPECT_CALL(handler, OnScalar(_, "tag", 2, "bar")); EXPECT_CALL(handler, OnMapEnd()); EXPECT_CALL(handler, OnDocumentEnd()); Parse(out.c_str()); } -TEST_F(GenEmitterTest, testcfc06e7baf582f834f85) { +TEST_F(GenEmitterTest, test8bd970000ae21619e864) { Emitter out; out << Comment("comment"); out << BeginDoc; @@ -4422,14 +4556,16 @@ TEST_F(GenEmitterTest, testcfc06e7baf582f834f85) { EXPECT_CALL(handler, OnDocumentStart(_)); EXPECT_CALL(handler, OnMapStart(_, "?", 0, _)); + EXPECT_CALL(handler, OnAnchor(_, "anchor")); EXPECT_CALL(handler, OnScalar(_, "tag", 1, "foo")); + EXPECT_CALL(handler, OnAnchor(_, "other")); EXPECT_CALL(handler, OnScalar(_, "tag", 2, "bar")); EXPECT_CALL(handler, OnMapEnd()); EXPECT_CALL(handler, OnDocumentEnd()); Parse(out.c_str()); } -TEST_F(GenEmitterTest, test355c684d237adcf5d852) { +TEST_F(GenEmitterTest, test0a960fe3efeeb1b4fafe) { Emitter out; out << BeginDoc; out << Comment("comment"); @@ -4444,14 +4580,16 @@ TEST_F(GenEmitterTest, test355c684d237adcf5d852) { EXPECT_CALL(handler, OnDocumentStart(_)); EXPECT_CALL(handler, OnMapStart(_, "?", 0, _)); + EXPECT_CALL(handler, OnAnchor(_, "anchor")); EXPECT_CALL(handler, OnScalar(_, "tag", 1, "foo")); + EXPECT_CALL(handler, OnAnchor(_, "other")); EXPECT_CALL(handler, OnScalar(_, "tag", 2, "bar")); EXPECT_CALL(handler, OnMapEnd()); EXPECT_CALL(handler, OnDocumentEnd()); Parse(out.c_str()); } -TEST_F(GenEmitterTest, testbc2e12df3c174bd6b7e3) { +TEST_F(GenEmitterTest, testffec1dcba9a2622b57a3) { Emitter out; out << BeginDoc; out << BeginMap; @@ -4466,14 +4604,16 @@ TEST_F(GenEmitterTest, testbc2e12df3c174bd6b7e3) { EXPECT_CALL(handler, OnDocumentStart(_)); EXPECT_CALL(handler, OnMapStart(_, "?", 0, _)); + EXPECT_CALL(handler, OnAnchor(_, "anchor")); EXPECT_CALL(handler, OnScalar(_, "tag", 1, "foo")); + EXPECT_CALL(handler, OnAnchor(_, "other")); EXPECT_CALL(handler, OnScalar(_, "tag", 2, "bar")); EXPECT_CALL(handler, OnMapEnd()); EXPECT_CALL(handler, OnDocumentEnd()); Parse(out.c_str()); } -TEST_F(GenEmitterTest, test47220f053e2788618f28) { +TEST_F(GenEmitterTest, test9a181b6042027e7977bf) { Emitter out; out << BeginDoc; out << BeginMap; @@ -4488,14 +4628,16 @@ TEST_F(GenEmitterTest, test47220f053e2788618f28) { EXPECT_CALL(handler, OnDocumentStart(_)); EXPECT_CALL(handler, OnMapStart(_, "?", 0, _)); + EXPECT_CALL(handler, OnAnchor(_, "anchor")); EXPECT_CALL(handler, OnScalar(_, "tag", 1, "foo")); + EXPECT_CALL(handler, OnAnchor(_, "other")); EXPECT_CALL(handler, OnScalar(_, "tag", 2, "bar")); EXPECT_CALL(handler, OnMapEnd()); EXPECT_CALL(handler, OnDocumentEnd()); Parse(out.c_str()); } -TEST_F(GenEmitterTest, testa71dd9a65f4e9563d114) { +TEST_F(GenEmitterTest, test5b42728bff7e0dd63ae8) { Emitter out; out << BeginDoc; out << BeginMap; @@ -4510,14 +4652,16 @@ TEST_F(GenEmitterTest, testa71dd9a65f4e9563d114) { EXPECT_CALL(handler, OnDocumentStart(_)); EXPECT_CALL(handler, OnMapStart(_, "?", 0, _)); + EXPECT_CALL(handler, OnAnchor(_, "anchor")); EXPECT_CALL(handler, OnScalar(_, "tag", 1, "foo")); + EXPECT_CALL(handler, OnAnchor(_, "other")); EXPECT_CALL(handler, OnScalar(_, "tag", 2, "bar")); EXPECT_CALL(handler, OnMapEnd()); EXPECT_CALL(handler, OnDocumentEnd()); Parse(out.c_str()); } -TEST_F(GenEmitterTest, test188c5ac5c1d6e2174110) { +TEST_F(GenEmitterTest, testa17514c4db3a70fe5084) { Emitter out; out << BeginDoc; out << BeginMap; @@ -4532,14 +4676,16 @@ TEST_F(GenEmitterTest, test188c5ac5c1d6e2174110) { EXPECT_CALL(handler, OnDocumentStart(_)); EXPECT_CALL(handler, OnMapStart(_, "?", 0, _)); + EXPECT_CALL(handler, OnAnchor(_, "anchor")); EXPECT_CALL(handler, OnScalar(_, "tag", 1, "foo")); + EXPECT_CALL(handler, OnAnchor(_, "other")); EXPECT_CALL(handler, OnScalar(_, "tag", 2, "bar")); EXPECT_CALL(handler, OnMapEnd()); EXPECT_CALL(handler, OnDocumentEnd()); Parse(out.c_str()); } -TEST_F(GenEmitterTest, testdc89b13ed8a4694f0296) { +TEST_F(GenEmitterTest, test2ac903a52c526db4b34b) { Emitter out; out << BeginDoc; out << BeginMap; @@ -4554,14 +4700,16 @@ TEST_F(GenEmitterTest, testdc89b13ed8a4694f0296) { EXPECT_CALL(handler, OnDocumentStart(_)); EXPECT_CALL(handler, OnMapStart(_, "?", 0, _)); + EXPECT_CALL(handler, OnAnchor(_, "anchor")); EXPECT_CALL(handler, OnScalar(_, "tag", 1, "foo")); + EXPECT_CALL(handler, OnAnchor(_, "other")); EXPECT_CALL(handler, OnScalar(_, "tag", 2, "bar")); EXPECT_CALL(handler, OnMapEnd()); EXPECT_CALL(handler, OnDocumentEnd()); Parse(out.c_str()); } -TEST_F(GenEmitterTest, test91fe618a569a60fa75d0) { +TEST_F(GenEmitterTest, testbebc6bc66d04a91bfa9c) { Emitter out; out << Comment("comment"); out << BeginMap; @@ -4575,7 +4723,7 @@ TEST_F(GenEmitterTest, test91fe618a569a60fa75d0) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test1f18c55487cda597929f) { +TEST_F(GenEmitterTest, test0918e247384bfc94d831) { Emitter out; out << Comment("comment"); out << BeginMap; @@ -4589,7 +4737,7 @@ TEST_F(GenEmitterTest, test1f18c55487cda597929f) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, testa74f5aa5747c1ec09ac2) { +TEST_F(GenEmitterTest, testf8512b2ebdaad8ae4cae) { Emitter out; out << BeginMap; out << Comment("comment"); @@ -4603,7 +4751,7 @@ TEST_F(GenEmitterTest, testa74f5aa5747c1ec09ac2) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, testda71284d14e9f5902fe5) { +TEST_F(GenEmitterTest, test01a1d249079c380030ca) { Emitter out; out << BeginMap; out << EndMap; @@ -4617,7 +4765,7 @@ TEST_F(GenEmitterTest, testda71284d14e9f5902fe5) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, testb041a1dda939d84dd6ed) { +TEST_F(GenEmitterTest, testcb48737e9c352108dc56) { Emitter out; out << BeginMap; out << EndMap; @@ -4631,7 +4779,7 @@ TEST_F(GenEmitterTest, testb041a1dda939d84dd6ed) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test0f39f734136f83edaab4) { +TEST_F(GenEmitterTest, testdea8106f3dce46929197) { Emitter out; out << Comment("comment"); out << BeginMap; @@ -4644,7 +4792,7 @@ TEST_F(GenEmitterTest, test0f39f734136f83edaab4) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, teste8eed1f3ab25f5395d7b) { +TEST_F(GenEmitterTest, test2b91aa87abdaa0fc0b20) { Emitter out; out << Comment("comment"); out << BeginMap; @@ -4657,7 +4805,7 @@ TEST_F(GenEmitterTest, teste8eed1f3ab25f5395d7b) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test3c7e3bbca86317884080) { +TEST_F(GenEmitterTest, test9c8b1fe0c5bbbf6a787e) { Emitter out; out << BeginMap; out << Comment("comment"); @@ -4670,7 +4818,7 @@ TEST_F(GenEmitterTest, test3c7e3bbca86317884080) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, testdc6b30ad8f00369e0597) { +TEST_F(GenEmitterTest, test8808d4be9571f365f79a) { Emitter out; out << BeginMap; out << EndMap; @@ -4683,7 +4831,7 @@ TEST_F(GenEmitterTest, testdc6b30ad8f00369e0597) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, testa905ff4b380f9fbbb630) { +TEST_F(GenEmitterTest, teste77c95c5163513fa25c5) { Emitter out; out << BeginMap; out << EndMap; @@ -4696,7 +4844,7 @@ TEST_F(GenEmitterTest, testa905ff4b380f9fbbb630) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test7143b34a608a3c7dc4bf) { +TEST_F(GenEmitterTest, testa3ed6e26dac366240579) { Emitter out; out << Comment("comment"); out << BeginMap; @@ -4714,7 +4862,7 @@ TEST_F(GenEmitterTest, test7143b34a608a3c7dc4bf) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test3f56e373d59464d85912) { +TEST_F(GenEmitterTest, test136adbd0ad47d74cfa22) { Emitter out; out << Comment("comment"); out << BeginMap; @@ -4732,7 +4880,7 @@ TEST_F(GenEmitterTest, test3f56e373d59464d85912) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test8790906cd88c45aba794) { +TEST_F(GenEmitterTest, test77f384e8387e39b54691) { Emitter out; out << BeginMap; out << Comment("comment"); @@ -4750,7 +4898,7 @@ TEST_F(GenEmitterTest, test8790906cd88c45aba794) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test7eef04e795fe3d520c6f) { +TEST_F(GenEmitterTest, testf8f016177cf9e428fcd4) { Emitter out; out << BeginMap; out << "foo"; @@ -4768,7 +4916,7 @@ TEST_F(GenEmitterTest, test7eef04e795fe3d520c6f) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test3378c3dc3604de62c507) { +TEST_F(GenEmitterTest, test1cec69d3c95937f4137a) { Emitter out; out << BeginMap; out << "foo"; @@ -4786,7 +4934,7 @@ TEST_F(GenEmitterTest, test3378c3dc3604de62c507) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, teste9742c9cbd88de777d38) { +TEST_F(GenEmitterTest, testb1fe1a5c5c064bdfe505) { Emitter out; out << BeginMap; out << "foo"; @@ -4804,7 +4952,7 @@ TEST_F(GenEmitterTest, teste9742c9cbd88de777d38) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, testb74cac7a45f30159f880) { +TEST_F(GenEmitterTest, test14200fdf4de8797d8dfb) { Emitter out; out << BeginMap; out << "foo"; @@ -4822,7 +4970,7 @@ TEST_F(GenEmitterTest, testb74cac7a45f30159f880) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, testa4475a6e0ca5a46213db) { +TEST_F(GenEmitterTest, testde7595a96199f66d7ac0) { Emitter out; out << Comment("comment"); out << BeginMap; @@ -4839,7 +4987,7 @@ TEST_F(GenEmitterTest, testa4475a6e0ca5a46213db) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test7aa5535404499fd9e6fe) { +TEST_F(GenEmitterTest, testb1434e1f508509c0ade4) { Emitter out; out << Comment("comment"); out << BeginMap; @@ -4856,7 +5004,7 @@ TEST_F(GenEmitterTest, test7aa5535404499fd9e6fe) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test75a57ca05218b76d7aa1) { +TEST_F(GenEmitterTest, test0d3bd788298201abbe67) { Emitter out; out << BeginMap; out << Comment("comment"); @@ -4873,7 +5021,7 @@ TEST_F(GenEmitterTest, test75a57ca05218b76d7aa1) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test973f12b1e7cdd047a330) { +TEST_F(GenEmitterTest, test3c716f5c232001f04805) { Emitter out; out << BeginMap; out << "foo"; @@ -4890,7 +5038,7 @@ TEST_F(GenEmitterTest, test973f12b1e7cdd047a330) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test75086003fa734ba2e64c) { +TEST_F(GenEmitterTest, testa55ef29eecbda5bc5b69) { Emitter out; out << BeginMap; out << "foo"; @@ -4907,7 +5055,7 @@ TEST_F(GenEmitterTest, test75086003fa734ba2e64c) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, testef9ab805717a642f5d50) { +TEST_F(GenEmitterTest, test55ddd593defa5ee8da90) { Emitter out; out << BeginMap; out << "foo"; @@ -4924,7 +5072,7 @@ TEST_F(GenEmitterTest, testef9ab805717a642f5d50) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, testbbe1014c67cbcd9bff8f) { +TEST_F(GenEmitterTest, test7326f87fd5c3adff317b) { Emitter out; out << BeginMap; out << "foo"; @@ -4941,7 +5089,7 @@ TEST_F(GenEmitterTest, testbbe1014c67cbcd9bff8f) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, testa0fae19726738031a50d) { +TEST_F(GenEmitterTest, test5ebc413d376f3b965879) { Emitter out; out << Comment("comment"); out << BeginMap; @@ -4956,14 +5104,16 @@ TEST_F(GenEmitterTest, testa0fae19726738031a50d) { EXPECT_CALL(handler, OnDocumentStart(_)); EXPECT_CALL(handler, OnMapStart(_, "?", 0, _)); + EXPECT_CALL(handler, OnAnchor(_, "anchor")); EXPECT_CALL(handler, OnScalar(_, "tag", 1, "foo")); + EXPECT_CALL(handler, OnAnchor(_, "other")); EXPECT_CALL(handler, OnScalar(_, "tag", 2, "bar")); EXPECT_CALL(handler, OnMapEnd()); EXPECT_CALL(handler, OnDocumentEnd()); Parse(out.c_str()); } -TEST_F(GenEmitterTest, test8cf4c10b22758615f5dc) { +TEST_F(GenEmitterTest, test4c7159334e528e2cfff8) { Emitter out; out << Comment("comment"); out << BeginMap; @@ -4978,14 +5128,16 @@ TEST_F(GenEmitterTest, test8cf4c10b22758615f5dc) { EXPECT_CALL(handler, OnDocumentStart(_)); EXPECT_CALL(handler, OnMapStart(_, "?", 0, _)); + EXPECT_CALL(handler, OnAnchor(_, "anchor")); EXPECT_CALL(handler, OnScalar(_, "tag", 1, "foo")); + EXPECT_CALL(handler, OnAnchor(_, "other")); EXPECT_CALL(handler, OnScalar(_, "tag", 2, "bar")); EXPECT_CALL(handler, OnMapEnd()); EXPECT_CALL(handler, OnDocumentEnd()); Parse(out.c_str()); } -TEST_F(GenEmitterTest, test777130969e538d6086d6) { +TEST_F(GenEmitterTest, test6a6bc06cdfee9f58a094) { Emitter out; out << BeginMap; out << Comment("comment"); @@ -5000,14 +5152,16 @@ TEST_F(GenEmitterTest, test777130969e538d6086d6) { EXPECT_CALL(handler, OnDocumentStart(_)); EXPECT_CALL(handler, OnMapStart(_, "?", 0, _)); + EXPECT_CALL(handler, OnAnchor(_, "anchor")); EXPECT_CALL(handler, OnScalar(_, "tag", 1, "foo")); + EXPECT_CALL(handler, OnAnchor(_, "other")); EXPECT_CALL(handler, OnScalar(_, "tag", 2, "bar")); EXPECT_CALL(handler, OnMapEnd()); EXPECT_CALL(handler, OnDocumentEnd()); Parse(out.c_str()); } -TEST_F(GenEmitterTest, test5ab6ba08277cd6a6a349) { +TEST_F(GenEmitterTest, test0beedfaace1b1e71d0c6) { Emitter out; out << BeginMap; out << VerbatimTag("tag"); @@ -5022,14 +5176,16 @@ TEST_F(GenEmitterTest, test5ab6ba08277cd6a6a349) { EXPECT_CALL(handler, OnDocumentStart(_)); EXPECT_CALL(handler, OnMapStart(_, "?", 0, _)); + EXPECT_CALL(handler, OnAnchor(_, "anchor")); EXPECT_CALL(handler, OnScalar(_, "tag", 1, "foo")); + EXPECT_CALL(handler, OnAnchor(_, "other")); EXPECT_CALL(handler, OnScalar(_, "tag", 2, "bar")); EXPECT_CALL(handler, OnMapEnd()); EXPECT_CALL(handler, OnDocumentEnd()); Parse(out.c_str()); } -TEST_F(GenEmitterTest, test4e6b76c47d712e10f181) { +TEST_F(GenEmitterTest, test9f65fafc369193908b7b) { Emitter out; out << BeginMap; out << VerbatimTag("tag"); @@ -5044,14 +5200,16 @@ TEST_F(GenEmitterTest, test4e6b76c47d712e10f181) { EXPECT_CALL(handler, OnDocumentStart(_)); EXPECT_CALL(handler, OnMapStart(_, "?", 0, _)); + EXPECT_CALL(handler, OnAnchor(_, "anchor")); EXPECT_CALL(handler, OnScalar(_, "tag", 1, "foo")); + EXPECT_CALL(handler, OnAnchor(_, "other")); EXPECT_CALL(handler, OnScalar(_, "tag", 2, "bar")); EXPECT_CALL(handler, OnMapEnd()); EXPECT_CALL(handler, OnDocumentEnd()); Parse(out.c_str()); } -TEST_F(GenEmitterTest, test0ccc3ac76a8810b05ab3) { +TEST_F(GenEmitterTest, test3a5c3ac504d7a58a08ca) { Emitter out; out << BeginMap; out << VerbatimTag("tag"); @@ -5066,14 +5224,16 @@ TEST_F(GenEmitterTest, test0ccc3ac76a8810b05ab3) { EXPECT_CALL(handler, OnDocumentStart(_)); EXPECT_CALL(handler, OnMapStart(_, "?", 0, _)); + EXPECT_CALL(handler, OnAnchor(_, "anchor")); EXPECT_CALL(handler, OnScalar(_, "tag", 1, "foo")); + EXPECT_CALL(handler, OnAnchor(_, "other")); EXPECT_CALL(handler, OnScalar(_, "tag", 2, "bar")); EXPECT_CALL(handler, OnMapEnd()); EXPECT_CALL(handler, OnDocumentEnd()); Parse(out.c_str()); } -TEST_F(GenEmitterTest, test3def4b737140116db9f9) { +TEST_F(GenEmitterTest, testc007513c868038dd3a68) { Emitter out; out << BeginMap; out << VerbatimTag("tag"); @@ -5088,14 +5248,16 @@ TEST_F(GenEmitterTest, test3def4b737140116db9f9) { EXPECT_CALL(handler, OnDocumentStart(_)); EXPECT_CALL(handler, OnMapStart(_, "?", 0, _)); + EXPECT_CALL(handler, OnAnchor(_, "anchor")); EXPECT_CALL(handler, OnScalar(_, "tag", 1, "foo")); + EXPECT_CALL(handler, OnAnchor(_, "other")); EXPECT_CALL(handler, OnScalar(_, "tag", 2, "bar")); EXPECT_CALL(handler, OnMapEnd()); EXPECT_CALL(handler, OnDocumentEnd()); Parse(out.c_str()); } -TEST_F(GenEmitterTest, test9ca096634f9d9cec9b36) { +TEST_F(GenEmitterTest, test89f3ba065cbd341381ec) { Emitter out; out << Comment("comment"); out << BeginMap; @@ -5109,14 +5271,16 @@ TEST_F(GenEmitterTest, test9ca096634f9d9cec9b36) { EXPECT_CALL(handler, OnDocumentStart(_)); EXPECT_CALL(handler, OnMapStart(_, "?", 0, _)); + EXPECT_CALL(handler, OnAnchor(_, "anchor")); EXPECT_CALL(handler, OnScalar(_, "tag", 1, "foo")); + EXPECT_CALL(handler, OnAnchor(_, "other")); EXPECT_CALL(handler, OnScalar(_, "tag", 2, "bar")); EXPECT_CALL(handler, OnMapEnd()); EXPECT_CALL(handler, OnDocumentEnd()); Parse(out.c_str()); } -TEST_F(GenEmitterTest, testbf2b7d7182dae8038726) { +TEST_F(GenEmitterTest, test692c2652cee84e90c096) { Emitter out; out << Comment("comment"); out << BeginMap; @@ -5130,14 +5294,16 @@ TEST_F(GenEmitterTest, testbf2b7d7182dae8038726) { EXPECT_CALL(handler, OnDocumentStart(_)); EXPECT_CALL(handler, OnMapStart(_, "?", 0, _)); + EXPECT_CALL(handler, OnAnchor(_, "anchor")); EXPECT_CALL(handler, OnScalar(_, "tag", 1, "foo")); + EXPECT_CALL(handler, OnAnchor(_, "other")); EXPECT_CALL(handler, OnScalar(_, "tag", 2, "bar")); EXPECT_CALL(handler, OnMapEnd()); EXPECT_CALL(handler, OnDocumentEnd()); Parse(out.c_str()); } -TEST_F(GenEmitterTest, testa3528cc9ec7b90fc1d8d) { +TEST_F(GenEmitterTest, test761fba62f7617a03fbf0) { Emitter out; out << BeginMap; out << Comment("comment"); @@ -5151,14 +5317,16 @@ TEST_F(GenEmitterTest, testa3528cc9ec7b90fc1d8d) { EXPECT_CALL(handler, OnDocumentStart(_)); EXPECT_CALL(handler, OnMapStart(_, "?", 0, _)); + EXPECT_CALL(handler, OnAnchor(_, "anchor")); EXPECT_CALL(handler, OnScalar(_, "tag", 1, "foo")); + EXPECT_CALL(handler, OnAnchor(_, "other")); EXPECT_CALL(handler, OnScalar(_, "tag", 2, "bar")); EXPECT_CALL(handler, OnMapEnd()); EXPECT_CALL(handler, OnDocumentEnd()); Parse(out.c_str()); } -TEST_F(GenEmitterTest, testf5be0253b15048a37e3a) { +TEST_F(GenEmitterTest, teste960a69bc06912eb8c76) { Emitter out; out << BeginMap; out << VerbatimTag("tag"); @@ -5172,14 +5340,16 @@ TEST_F(GenEmitterTest, testf5be0253b15048a37e3a) { EXPECT_CALL(handler, OnDocumentStart(_)); EXPECT_CALL(handler, OnMapStart(_, "?", 0, _)); + EXPECT_CALL(handler, OnAnchor(_, "anchor")); EXPECT_CALL(handler, OnScalar(_, "tag", 1, "foo")); + EXPECT_CALL(handler, OnAnchor(_, "other")); EXPECT_CALL(handler, OnScalar(_, "tag", 2, "bar")); EXPECT_CALL(handler, OnMapEnd()); EXPECT_CALL(handler, OnDocumentEnd()); Parse(out.c_str()); } -TEST_F(GenEmitterTest, teste338a8f7f7aa9b63cddb) { +TEST_F(GenEmitterTest, test8f6187c6c2419dbf1770) { Emitter out; out << BeginMap; out << VerbatimTag("tag"); @@ -5193,14 +5363,16 @@ TEST_F(GenEmitterTest, teste338a8f7f7aa9b63cddb) { EXPECT_CALL(handler, OnDocumentStart(_)); EXPECT_CALL(handler, OnMapStart(_, "?", 0, _)); + EXPECT_CALL(handler, OnAnchor(_, "anchor")); EXPECT_CALL(handler, OnScalar(_, "tag", 1, "foo")); + EXPECT_CALL(handler, OnAnchor(_, "other")); EXPECT_CALL(handler, OnScalar(_, "tag", 2, "bar")); EXPECT_CALL(handler, OnMapEnd()); EXPECT_CALL(handler, OnDocumentEnd()); Parse(out.c_str()); } -TEST_F(GenEmitterTest, test3628377ede539d44f975) { +TEST_F(GenEmitterTest, testba6cb3810a074fabc55e) { Emitter out; out << BeginMap; out << VerbatimTag("tag"); @@ -5214,14 +5386,16 @@ TEST_F(GenEmitterTest, test3628377ede539d44f975) { EXPECT_CALL(handler, OnDocumentStart(_)); EXPECT_CALL(handler, OnMapStart(_, "?", 0, _)); + EXPECT_CALL(handler, OnAnchor(_, "anchor")); EXPECT_CALL(handler, OnScalar(_, "tag", 1, "foo")); + EXPECT_CALL(handler, OnAnchor(_, "other")); EXPECT_CALL(handler, OnScalar(_, "tag", 2, "bar")); EXPECT_CALL(handler, OnMapEnd()); EXPECT_CALL(handler, OnDocumentEnd()); Parse(out.c_str()); } -TEST_F(GenEmitterTest, test6eb2c2967a88d82d83d9) { +TEST_F(GenEmitterTest, testac695c3621ec3f104672) { Emitter out; out << BeginMap; out << VerbatimTag("tag"); @@ -5235,14 +5409,16 @@ TEST_F(GenEmitterTest, test6eb2c2967a88d82d83d9) { EXPECT_CALL(handler, OnDocumentStart(_)); EXPECT_CALL(handler, OnMapStart(_, "?", 0, _)); + EXPECT_CALL(handler, OnAnchor(_, "anchor")); EXPECT_CALL(handler, OnScalar(_, "tag", 1, "foo")); + EXPECT_CALL(handler, OnAnchor(_, "other")); EXPECT_CALL(handler, OnScalar(_, "tag", 2, "bar")); EXPECT_CALL(handler, OnMapEnd()); EXPECT_CALL(handler, OnDocumentEnd()); Parse(out.c_str()); } -TEST_F(GenEmitterTest, test03515aba5b291049c262) { +TEST_F(GenEmitterTest, test86494a6bcb6a65e7029e) { Emitter out; out << Comment("comment"); out << BeginMap; @@ -5259,7 +5435,7 @@ TEST_F(GenEmitterTest, test03515aba5b291049c262) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test20eea0df7f54b919cb92) { +TEST_F(GenEmitterTest, testb406fb13323c199d709c) { Emitter out; out << Comment("comment"); out << BeginMap; @@ -5276,7 +5452,7 @@ TEST_F(GenEmitterTest, test20eea0df7f54b919cb92) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test6c6ab6f66ce2828d26cd) { +TEST_F(GenEmitterTest, test4409f685a3e80b9ab415) { Emitter out; out << BeginMap; out << Comment("comment"); @@ -5293,7 +5469,7 @@ TEST_F(GenEmitterTest, test6c6ab6f66ce2828d26cd) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test7f415ecadae422f7b0f5) { +TEST_F(GenEmitterTest, testa74ace9c1f5e18cf3f2a) { Emitter out; out << BeginMap; out << "foo"; @@ -5310,7 +5486,7 @@ TEST_F(GenEmitterTest, test7f415ecadae422f7b0f5) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test0797d220a5428528ec87) { +TEST_F(GenEmitterTest, testabfc8ce2ca4c3dafa013) { Emitter out; out << BeginMap; out << "foo"; @@ -5327,7 +5503,7 @@ TEST_F(GenEmitterTest, test0797d220a5428528ec87) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test6e29f19c88864559a43e) { +TEST_F(GenEmitterTest, test07ff4bbae6104c4e30c1) { Emitter out; out << BeginMap; out << "foo"; @@ -5344,7 +5520,7 @@ TEST_F(GenEmitterTest, test6e29f19c88864559a43e) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test0fe24accf63e33a71494) { +TEST_F(GenEmitterTest, test1e3559cacab6d46c98fe) { Emitter out; out << BeginMap; out << "foo"; @@ -5361,7 +5537,7 @@ TEST_F(GenEmitterTest, test0fe24accf63e33a71494) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test6cc44b957e440b365fbc) { +TEST_F(GenEmitterTest, test795830e12e5a20213a7e) { Emitter out; out << Comment("comment"); out << BeginMap; @@ -5382,7 +5558,7 @@ TEST_F(GenEmitterTest, test6cc44b957e440b365fbc) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test5de19a26e7af5459141b) { +TEST_F(GenEmitterTest, test849f2c88c71734fcf3f3) { Emitter out; out << Comment("comment"); out << BeginMap; @@ -5403,7 +5579,7 @@ TEST_F(GenEmitterTest, test5de19a26e7af5459141b) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, testea71fb07f12c70473a81) { +TEST_F(GenEmitterTest, test7bb139ac1f14e8ae04e2) { Emitter out; out << BeginMap; out << Comment("comment"); @@ -5424,7 +5600,7 @@ TEST_F(GenEmitterTest, testea71fb07f12c70473a81) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, testa70d56c89e5219a106b6) { +TEST_F(GenEmitterTest, test58655e359c60bf73986f) { Emitter out; out << BeginMap; out << "foo"; @@ -5445,7 +5621,7 @@ TEST_F(GenEmitterTest, testa70d56c89e5219a106b6) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test9eac2d394b9528ccd982) { +TEST_F(GenEmitterTest, testde9f70648448cbd37245) { Emitter out; out << BeginMap; out << "foo"; @@ -5466,7 +5642,7 @@ TEST_F(GenEmitterTest, test9eac2d394b9528ccd982) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test25b8b26feb1a8e5e50f9) { +TEST_F(GenEmitterTest, testfff25037c90a64db8771) { Emitter out; out << BeginMap; out << "foo"; @@ -5487,7 +5663,7 @@ TEST_F(GenEmitterTest, test25b8b26feb1a8e5e50f9) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, testc9498e8d4545fb7ba842) { +TEST_F(GenEmitterTest, test94b24a286074cac9b881) { Emitter out; out << BeginMap; out << "foo"; @@ -5508,7 +5684,7 @@ TEST_F(GenEmitterTest, testc9498e8d4545fb7ba842) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, testaf3f79c8ce6b65a71427) { +TEST_F(GenEmitterTest, test0c678fe6c6dbd3ccf4eb) { Emitter out; out << BeginMap; out << "foo"; @@ -5529,7 +5705,7 @@ TEST_F(GenEmitterTest, testaf3f79c8ce6b65a71427) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test921c297e990d295cd067) { +TEST_F(GenEmitterTest, test15f6ce577f139b9f1b61) { Emitter out; out << BeginMap; out << "foo"; @@ -5550,7 +5726,7 @@ TEST_F(GenEmitterTest, test921c297e990d295cd067) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, testd61d8ef3753e51e671a8) { +TEST_F(GenEmitterTest, test5e927c8865f44b5c1abe) { Emitter out; out << Comment("comment"); out << BeginMap; @@ -5573,7 +5749,7 @@ TEST_F(GenEmitterTest, testd61d8ef3753e51e671a8) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test8fa800a208cdaa3e20d7) { +TEST_F(GenEmitterTest, test235aebc60786962899f1) { Emitter out; out << Comment("comment"); out << BeginMap; @@ -5596,7 +5772,7 @@ TEST_F(GenEmitterTest, test8fa800a208cdaa3e20d7) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, testdf78d6aa9976557adf35) { +TEST_F(GenEmitterTest, test45e109e1bc3ca312091d) { Emitter out; out << BeginMap; out << Comment("comment"); @@ -5619,7 +5795,7 @@ TEST_F(GenEmitterTest, testdf78d6aa9976557adf35) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, testf13bfc9be619d52da2b9) { +TEST_F(GenEmitterTest, test9a58086d44719b21c6b3) { Emitter out; out << BeginMap; out << "foo"; @@ -5642,7 +5818,7 @@ TEST_F(GenEmitterTest, testf13bfc9be619d52da2b9) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, testd69830b6ed705995e364) { +TEST_F(GenEmitterTest, testa3ac3fa06ae69e9f5c9d) { Emitter out; out << BeginMap; out << "foo"; @@ -5665,7 +5841,7 @@ TEST_F(GenEmitterTest, testd69830b6ed705995e364) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, teste41087bf41e163aa315d) { +TEST_F(GenEmitterTest, test9ccaf628d78cc8f27857) { Emitter out; out << BeginMap; out << "foo"; @@ -5688,7 +5864,7 @@ TEST_F(GenEmitterTest, teste41087bf41e163aa315d) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, testf46aa6778b48f4a04e21) { +TEST_F(GenEmitterTest, test6b8483101720027fd945) { Emitter out; out << BeginMap; out << "foo"; @@ -5711,7 +5887,7 @@ TEST_F(GenEmitterTest, testf46aa6778b48f4a04e21) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, teste017eaffcd10bfedf757) { +TEST_F(GenEmitterTest, test1549a08694b053f8a2cb) { Emitter out; out << BeginMap; out << "foo"; @@ -5734,7 +5910,7 @@ TEST_F(GenEmitterTest, teste017eaffcd10bfedf757) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test1cd2c3e073dac695dfdb) { +TEST_F(GenEmitterTest, test0417f66daae22676ad66) { Emitter out; out << BeginMap; out << "foo"; @@ -5757,7 +5933,7 @@ TEST_F(GenEmitterTest, test1cd2c3e073dac695dfdb) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, testd2eb393135a7a84aea37) { +TEST_F(GenEmitterTest, test6b3d4384d27d0ac90b01) { Emitter out; out << BeginMap; out << "foo"; @@ -5780,7 +5956,7 @@ TEST_F(GenEmitterTest, testd2eb393135a7a84aea37) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test79b4dcb868412729afed) { +TEST_F(GenEmitterTest, testbf166fa245c204799ea8) { Emitter out; out << Comment("comment"); out << BeginMap; @@ -5801,7 +5977,7 @@ TEST_F(GenEmitterTest, test79b4dcb868412729afed) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test379fd004b428ddfbabe2) { +TEST_F(GenEmitterTest, testdf279b50896e8f084ed3) { Emitter out; out << Comment("comment"); out << BeginMap; @@ -5822,7 +5998,7 @@ TEST_F(GenEmitterTest, test379fd004b428ddfbabe2) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test5c13b45235bb1b7e36a9) { +TEST_F(GenEmitterTest, test7d9b55fe33dfdc6cf930) { Emitter out; out << BeginMap; out << Comment("comment"); @@ -5843,7 +6019,7 @@ TEST_F(GenEmitterTest, test5c13b45235bb1b7e36a9) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test95b9c0c89b4cd2841446) { +TEST_F(GenEmitterTest, test2b0f0825ac5d9ac3baf7) { Emitter out; out << BeginMap; out << BeginSeq; @@ -5864,7 +6040,7 @@ TEST_F(GenEmitterTest, test95b9c0c89b4cd2841446) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test14810a0d0234bee070a2) { +TEST_F(GenEmitterTest, teste4865b227f48a727aafe) { Emitter out; out << BeginMap; out << BeginSeq; @@ -5885,7 +6061,7 @@ TEST_F(GenEmitterTest, test14810a0d0234bee070a2) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test8447a364c30880a42ca1) { +TEST_F(GenEmitterTest, testb02fb812ae14499cc30e) { Emitter out; out << BeginMap; out << BeginSeq; @@ -5906,7 +6082,7 @@ TEST_F(GenEmitterTest, test8447a364c30880a42ca1) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test09fbd11d82a3777ac376) { +TEST_F(GenEmitterTest, testa395515c1bce39e737b7) { Emitter out; out << BeginMap; out << BeginSeq; @@ -5927,7 +6103,7 @@ TEST_F(GenEmitterTest, test09fbd11d82a3777ac376) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, testcbb2d4bd62f1ef2f66ad) { +TEST_F(GenEmitterTest, test813be458848be1ba3bf1) { Emitter out; out << BeginMap; out << BeginSeq; @@ -5948,7 +6124,7 @@ TEST_F(GenEmitterTest, testcbb2d4bd62f1ef2f66ad) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, testa0ecef367a1f54e08d00) { +TEST_F(GenEmitterTest, test23ab6af7b3fc7957a03d) { Emitter out; out << BeginMap; out << BeginSeq; @@ -5969,7 +6145,7 @@ TEST_F(GenEmitterTest, testa0ecef367a1f54e08d00) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test89c1ba14891887445d1a) { +TEST_F(GenEmitterTest, testc5fb40e239029d9efa58) { Emitter out; out << Comment("comment"); out << BeginMap; @@ -5994,7 +6170,7 @@ TEST_F(GenEmitterTest, test89c1ba14891887445d1a) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test68436e821e03f93754e6) { +TEST_F(GenEmitterTest, test88b5c599a4f9ac52f951) { Emitter out; out << Comment("comment"); out << BeginMap; @@ -6019,7 +6195,7 @@ TEST_F(GenEmitterTest, test68436e821e03f93754e6) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test9d56b9e336a032014977) { +TEST_F(GenEmitterTest, test3995578993108fa25f88) { Emitter out; out << BeginMap; out << Comment("comment"); @@ -6044,7 +6220,7 @@ TEST_F(GenEmitterTest, test9d56b9e336a032014977) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test38701040630c71bff30d) { +TEST_F(GenEmitterTest, test61a24036be6f8cd49a28) { Emitter out; out << BeginMap; out << BeginSeq; @@ -6069,7 +6245,7 @@ TEST_F(GenEmitterTest, test38701040630c71bff30d) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, testa5ee191882708767d03a) { +TEST_F(GenEmitterTest, testc13e02ab3c1bd1db6e55) { Emitter out; out << BeginMap; out << BeginSeq; @@ -6094,7 +6270,7 @@ TEST_F(GenEmitterTest, testa5ee191882708767d03a) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test7f055e278180eb0f8c0d) { +TEST_F(GenEmitterTest, testb3ca69f6d7a888644064) { Emitter out; out << BeginMap; out << BeginSeq; @@ -6119,7 +6295,7 @@ TEST_F(GenEmitterTest, test7f055e278180eb0f8c0d) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, testb17a6726041573ad5d90) { +TEST_F(GenEmitterTest, test4e6337821c858c2f7cfa) { Emitter out; out << BeginMap; out << BeginSeq; @@ -6144,7 +6320,7 @@ TEST_F(GenEmitterTest, testb17a6726041573ad5d90) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test0fe7dcb77bf033a58a7d) { +TEST_F(GenEmitterTest, test8dc4a01d956c779dd8b0) { Emitter out; out << BeginMap; out << BeginSeq; @@ -6169,7 +6345,7 @@ TEST_F(GenEmitterTest, test0fe7dcb77bf033a58a7d) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test4df73e774cb1a7044859) { +TEST_F(GenEmitterTest, test1dae9c6350559f6b9f89) { Emitter out; out << BeginMap; out << BeginSeq; @@ -6194,7 +6370,7 @@ TEST_F(GenEmitterTest, test4df73e774cb1a7044859) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test519ee9f4db0c2cc3cb22) { +TEST_F(GenEmitterTest, test9f891b1cdde286863956) { Emitter out; out << BeginMap; out << BeginSeq; @@ -6219,7 +6395,7 @@ TEST_F(GenEmitterTest, test519ee9f4db0c2cc3cb22) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test86fdfe330b8b39c502e3) { +TEST_F(GenEmitterTest, test0704315052c50c54b17a) { Emitter out; out << BeginMap; out << BeginSeq; @@ -6244,7 +6420,7 @@ TEST_F(GenEmitterTest, test86fdfe330b8b39c502e3) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, testf1f415e2941ace087ef1) { +TEST_F(GenEmitterTest, test7f5b47cf1d2571afc033) { Emitter out; out << Comment("comment"); out << BeginMap; @@ -6271,7 +6447,7 @@ TEST_F(GenEmitterTest, testf1f415e2941ace087ef1) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test61947172d8a42f6daa33) { +TEST_F(GenEmitterTest, test5040c19e850e3046d32d) { Emitter out; out << Comment("comment"); out << BeginMap; @@ -6298,7 +6474,7 @@ TEST_F(GenEmitterTest, test61947172d8a42f6daa33) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, testad7ddc91f2603dcd8af4) { +TEST_F(GenEmitterTest, test110665e2f3409ef307ff) { Emitter out; out << BeginMap; out << Comment("comment"); @@ -6325,7 +6501,7 @@ TEST_F(GenEmitterTest, testad7ddc91f2603dcd8af4) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test3ded81db555ef0e2dd4c) { +TEST_F(GenEmitterTest, teste9e1549f96267f93651c) { Emitter out; out << BeginMap; out << BeginSeq; @@ -6352,7 +6528,7 @@ TEST_F(GenEmitterTest, test3ded81db555ef0e2dd4c) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, testb1da8147f5442f38a0af) { +TEST_F(GenEmitterTest, test6a5b6a20fdb6f7c3279e) { Emitter out; out << BeginMap; out << BeginSeq; @@ -6379,7 +6555,7 @@ TEST_F(GenEmitterTest, testb1da8147f5442f38a0af) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, testdb0f2d85399318eb7957) { +TEST_F(GenEmitterTest, test7c568c68f77e34d5c714) { Emitter out; out << BeginMap; out << BeginSeq; @@ -6406,7 +6582,7 @@ TEST_F(GenEmitterTest, testdb0f2d85399318eb7957) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test456b6f61fd4fb5b4b516) { +TEST_F(GenEmitterTest, testfd8fe783b5297c92d17f) { Emitter out; out << BeginMap; out << BeginSeq; @@ -6433,7 +6609,7 @@ TEST_F(GenEmitterTest, test456b6f61fd4fb5b4b516) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test016098295ec6d033a470) { +TEST_F(GenEmitterTest, test418b1426e630825a7c85) { Emitter out; out << BeginMap; out << BeginSeq; @@ -6460,7 +6636,7 @@ TEST_F(GenEmitterTest, test016098295ec6d033a470) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test1cbee1bf6ca60fa72aa5) { +TEST_F(GenEmitterTest, test8161c5e486d317e7864e) { Emitter out; out << BeginMap; out << BeginSeq; @@ -6487,7 +6663,7 @@ TEST_F(GenEmitterTest, test1cbee1bf6ca60fa72aa5) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, testaf9b292d92d4ec50429c) { +TEST_F(GenEmitterTest, testa48e1915ca7c919db445) { Emitter out; out << BeginMap; out << BeginSeq; @@ -6514,7 +6690,7 @@ TEST_F(GenEmitterTest, testaf9b292d92d4ec50429c) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test3aded59e694359fa222e) { +TEST_F(GenEmitterTest, test27124815aea27c053aab) { Emitter out; out << BeginMap; out << BeginSeq; @@ -6541,7 +6717,7 @@ TEST_F(GenEmitterTest, test3aded59e694359fa222e) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test6a79d05b97bf18ab27e9) { +TEST_F(GenEmitterTest, teste751377e4a74306bc555) { Emitter out; out << BeginMap; out << BeginSeq; @@ -6568,7 +6744,7 @@ TEST_F(GenEmitterTest, test6a79d05b97bf18ab27e9) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test96727e93c95d3dcb5a2b) { +TEST_F(GenEmitterTest, test36a4cc298255efdd1ef5) { Emitter out; out << Comment("comment"); out << BeginMap; @@ -6591,7 +6767,7 @@ TEST_F(GenEmitterTest, test96727e93c95d3dcb5a2b) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test19387cf6dcd0618b416f) { +TEST_F(GenEmitterTest, test43471dee97b0506909b2) { Emitter out; out << Comment("comment"); out << BeginMap; @@ -6614,7 +6790,7 @@ TEST_F(GenEmitterTest, test19387cf6dcd0618b416f) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test755733f3c3c00263610c) { +TEST_F(GenEmitterTest, test959c85b4e833f72173a6) { Emitter out; out << BeginMap; out << Comment("comment"); @@ -6637,7 +6813,7 @@ TEST_F(GenEmitterTest, test755733f3c3c00263610c) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test426a9d79d3e0421c9a57) { +TEST_F(GenEmitterTest, test3a09723dce29399bc865) { Emitter out; out << BeginMap; out << BeginMap; @@ -6660,7 +6836,7 @@ TEST_F(GenEmitterTest, test426a9d79d3e0421c9a57) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, testefad231fdbf0147b5a46) { +TEST_F(GenEmitterTest, test958e404277a1c55139af) { Emitter out; out << BeginMap; out << BeginMap; @@ -6683,7 +6859,7 @@ TEST_F(GenEmitterTest, testefad231fdbf0147b5a46) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test03772c261806afb7c9cc) { +TEST_F(GenEmitterTest, testb1823407ab0601edc9cb) { Emitter out; out << BeginMap; out << BeginMap; @@ -6706,7 +6882,7 @@ TEST_F(GenEmitterTest, test03772c261806afb7c9cc) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test65734300dcf959922c18) { +TEST_F(GenEmitterTest, test9993e9dad983b28960aa) { Emitter out; out << BeginMap; out << BeginMap; @@ -6729,7 +6905,7 @@ TEST_F(GenEmitterTest, test65734300dcf959922c18) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test815ef14ef8792dc2de1d) { +TEST_F(GenEmitterTest, test8e87f559b25a9a20e11c) { Emitter out; out << BeginMap; out << BeginMap; @@ -6752,7 +6928,7 @@ TEST_F(GenEmitterTest, test815ef14ef8792dc2de1d) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test5266cf3b9eb70be75944) { +TEST_F(GenEmitterTest, testeb431be8504451636efe) { Emitter out; out << BeginMap; out << BeginMap; @@ -6775,7 +6951,7 @@ TEST_F(GenEmitterTest, test5266cf3b9eb70be75944) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test199b993f2f9302c1c0c8) { +TEST_F(GenEmitterTest, testf876965882befc7355df) { Emitter out; out << BeginMap; out << BeginMap; @@ -6798,7 +6974,7 @@ TEST_F(GenEmitterTest, test199b993f2f9302c1c0c8) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test448f8a30d377c0c01b26) { +TEST_F(GenEmitterTest, testf7e1d47f266f0b940fed) { Emitter out; out << Comment("comment"); out << BeginMap; @@ -6825,7 +7001,7 @@ TEST_F(GenEmitterTest, test448f8a30d377c0c01b26) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, testa4a3cfe5a74aa90708a5) { +TEST_F(GenEmitterTest, teste5a4bc646f2182b78cc1) { Emitter out; out << Comment("comment"); out << BeginMap; @@ -6852,7 +7028,7 @@ TEST_F(GenEmitterTest, testa4a3cfe5a74aa90708a5) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test6de6ac6c26a8d098b529) { +TEST_F(GenEmitterTest, test6cbedca25c9a8a6bb42e) { Emitter out; out << BeginMap; out << Comment("comment"); @@ -6879,7 +7055,7 @@ TEST_F(GenEmitterTest, test6de6ac6c26a8d098b529) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, testc9e7f10371f284e94d20) { +TEST_F(GenEmitterTest, test07613cc34874a5b47577) { Emitter out; out << BeginMap; out << BeginMap; @@ -6906,7 +7082,7 @@ TEST_F(GenEmitterTest, testc9e7f10371f284e94d20) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test382360950d64ced8a0cc) { +TEST_F(GenEmitterTest, testa55b7ac19580aeb82d32) { Emitter out; out << BeginMap; out << BeginMap; @@ -6933,7 +7109,7 @@ TEST_F(GenEmitterTest, test382360950d64ced8a0cc) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test52b289c54900ca0a098e) { +TEST_F(GenEmitterTest, test458f9af92dfb9f64a488) { Emitter out; out << BeginMap; out << BeginMap; @@ -6960,7 +7136,7 @@ TEST_F(GenEmitterTest, test52b289c54900ca0a098e) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test7266716f412852506771) { +TEST_F(GenEmitterTest, testc4a9c3769e95770bb455) { Emitter out; out << BeginMap; out << BeginMap; @@ -6987,7 +7163,7 @@ TEST_F(GenEmitterTest, test7266716f412852506771) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, testd2ac7d6f40cc75a3c695) { +TEST_F(GenEmitterTest, testae56654f89fa2416f3db) { Emitter out; out << BeginMap; out << BeginMap; @@ -7014,7 +7190,7 @@ TEST_F(GenEmitterTest, testd2ac7d6f40cc75a3c695) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test368597fd8bcc0d3aaff0) { +TEST_F(GenEmitterTest, test1635f71f27e7303c730e) { Emitter out; out << BeginMap; out << BeginMap; @@ -7041,7 +7217,7 @@ TEST_F(GenEmitterTest, test368597fd8bcc0d3aaff0) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test9201309cd67ab579b883) { +TEST_F(GenEmitterTest, testcd9d835a8d8b8622c63b) { Emitter out; out << BeginMap; out << BeginMap; @@ -7068,7 +7244,7 @@ TEST_F(GenEmitterTest, test9201309cd67ab579b883) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test30e18cbdd5d04a552625) { +TEST_F(GenEmitterTest, test3c2d1f91e57040cb8fdd) { Emitter out; out << BeginMap; out << BeginMap; @@ -7095,7 +7271,7 @@ TEST_F(GenEmitterTest, test30e18cbdd5d04a552625) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, testa659d36137a8f34911a6) { +TEST_F(GenEmitterTest, test9bd5b4c002b3b637747f) { Emitter out; out << BeginMap; out << BeginMap; @@ -7122,7 +7298,7 @@ TEST_F(GenEmitterTest, testa659d36137a8f34911a6) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, teste0b8302185ee814d6552) { +TEST_F(GenEmitterTest, testf6c71199cd45b5b8e7e0) { Emitter out; out << Comment("comment"); out << BeginMap; @@ -7151,7 +7327,7 @@ TEST_F(GenEmitterTest, teste0b8302185ee814d6552) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test66793e09aa910bd924ca) { +TEST_F(GenEmitterTest, test865b02b69742e5513963) { Emitter out; out << Comment("comment"); out << BeginMap; @@ -7180,7 +7356,7 @@ TEST_F(GenEmitterTest, test66793e09aa910bd924ca) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test9beaa5ceb1fcfef2a897) { +TEST_F(GenEmitterTest, test86eb8783dc4367cda931) { Emitter out; out << BeginMap; out << Comment("comment"); @@ -7209,7 +7385,7 @@ TEST_F(GenEmitterTest, test9beaa5ceb1fcfef2a897) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test8ccc36c941ec6aa59536) { +TEST_F(GenEmitterTest, test7f6604058a9d8e2e1219) { Emitter out; out << BeginMap; out << BeginMap; @@ -7238,7 +7414,7 @@ TEST_F(GenEmitterTest, test8ccc36c941ec6aa59536) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test3c8ea05243d098395929) { +TEST_F(GenEmitterTest, test676b4abcb3cf0530e4da) { Emitter out; out << BeginMap; out << BeginMap; @@ -7267,7 +7443,7 @@ TEST_F(GenEmitterTest, test3c8ea05243d098395929) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, testd819abb35a3ff74e764a) { +TEST_F(GenEmitterTest, teste7e52980e73d442a602b) { Emitter out; out << BeginMap; out << BeginMap; @@ -7296,7 +7472,7 @@ TEST_F(GenEmitterTest, testd819abb35a3ff74e764a) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test416c49d00378b626ad21) { +TEST_F(GenEmitterTest, testc3d6f480087db4bcd3a1) { Emitter out; out << BeginMap; out << BeginMap; @@ -7325,7 +7501,7 @@ TEST_F(GenEmitterTest, test416c49d00378b626ad21) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test5cd5ecef5b2a297afe21) { +TEST_F(GenEmitterTest, test6943516ed43da6944e96) { Emitter out; out << BeginMap; out << BeginMap; @@ -7354,7 +7530,7 @@ TEST_F(GenEmitterTest, test5cd5ecef5b2a297afe21) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test9d8ea78df40a20f07e96) { +TEST_F(GenEmitterTest, testb7e880522d0778ae6e6f) { Emitter out; out << BeginMap; out << BeginMap; @@ -7383,7 +7559,7 @@ TEST_F(GenEmitterTest, test9d8ea78df40a20f07e96) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, teste7bdb2f35bec40c207ee) { +TEST_F(GenEmitterTest, testec39ee51992618c7b154) { Emitter out; out << BeginMap; out << BeginMap; @@ -7412,7 +7588,7 @@ TEST_F(GenEmitterTest, teste7bdb2f35bec40c207ee) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, teste6d6da3a612849d32b79) { +TEST_F(GenEmitterTest, test39f768713a9b3aaffe0d) { Emitter out; out << BeginMap; out << BeginMap; @@ -7441,7 +7617,7 @@ TEST_F(GenEmitterTest, teste6d6da3a612849d32b79) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test9af66f2efca487828902) { +TEST_F(GenEmitterTest, testf287a68abc5b8ff7784d) { Emitter out; out << BeginMap; out << BeginMap; @@ -7470,7 +7646,7 @@ TEST_F(GenEmitterTest, test9af66f2efca487828902) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, testb05b2fc716f368b6bdfc) { +TEST_F(GenEmitterTest, testc36154aa87842ba9699f) { Emitter out; out << BeginMap; out << BeginMap; @@ -7499,7 +7675,7 @@ TEST_F(GenEmitterTest, testb05b2fc716f368b6bdfc) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, testd0713381b33b72abdb58) { +TEST_F(GenEmitterTest, testa75da84dfc8ee5507157) { Emitter out; out << Comment("comment"); out << BeginSeq; @@ -7516,7 +7692,7 @@ TEST_F(GenEmitterTest, testd0713381b33b72abdb58) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, testc6739cb3a9de4a5c41a7) { +TEST_F(GenEmitterTest, testc54a03d1735615f7bd60) { Emitter out; out << Comment("comment"); out << BeginSeq; @@ -7533,7 +7709,7 @@ TEST_F(GenEmitterTest, testc6739cb3a9de4a5c41a7) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test2b4fa669c6b56cd6a117) { +TEST_F(GenEmitterTest, testf5d72aba828875527d6f) { Emitter out; out << BeginSeq; out << Comment("comment"); @@ -7550,7 +7726,7 @@ TEST_F(GenEmitterTest, test2b4fa669c6b56cd6a117) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test29ae2ef656132c069d22) { +TEST_F(GenEmitterTest, test9e8681a1b27a8524ec5e) { Emitter out; out << BeginSeq; out << "foo"; @@ -7567,7 +7743,7 @@ TEST_F(GenEmitterTest, test29ae2ef656132c069d22) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, testd3e2ad21ab7d0890c412) { +TEST_F(GenEmitterTest, test34678928e79e6eb160f4) { Emitter out; out << BeginSeq; out << "foo"; @@ -7584,7 +7760,7 @@ TEST_F(GenEmitterTest, testd3e2ad21ab7d0890c412) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test632b0fb6570fa3b1b682) { +TEST_F(GenEmitterTest, test07c6d6b9e133553d4532) { Emitter out; out << BeginSeq; out << "foo"; @@ -7601,7 +7777,7 @@ TEST_F(GenEmitterTest, test632b0fb6570fa3b1b682) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, testc9bcfd496ae3382d546b) { +TEST_F(GenEmitterTest, test1531642db71e1aa8dd1c) { Emitter out; out << BeginSeq; out << "foo"; @@ -7618,7 +7794,7 @@ TEST_F(GenEmitterTest, testc9bcfd496ae3382d546b) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test43fd860d560c9137add4) { +TEST_F(GenEmitterTest, test22fde02c0bd0ecb8a527) { Emitter out; out << Comment("comment"); out << BeginSeq; @@ -7639,7 +7815,7 @@ TEST_F(GenEmitterTest, test43fd860d560c9137add4) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test0e47183ab4a56c3a6d36) { +TEST_F(GenEmitterTest, test72430574ba42559cf917) { Emitter out; out << Comment("comment"); out << BeginSeq; @@ -7660,7 +7836,7 @@ TEST_F(GenEmitterTest, test0e47183ab4a56c3a6d36) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, testf165d91c1dce024c9774) { +TEST_F(GenEmitterTest, test067bce5d8aa6281a3e6f) { Emitter out; out << BeginSeq; out << Comment("comment"); @@ -7681,7 +7857,7 @@ TEST_F(GenEmitterTest, testf165d91c1dce024c9774) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test7592a1f019ddb2b9c3f4) { +TEST_F(GenEmitterTest, testb1839ee8ced911bb1ed1) { Emitter out; out << BeginSeq; out << "foo"; @@ -7702,7 +7878,7 @@ TEST_F(GenEmitterTest, test7592a1f019ddb2b9c3f4) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test758d285e89a9de8070a7) { +TEST_F(GenEmitterTest, test5899c9cd7e5a40077178) { Emitter out; out << BeginSeq; out << "foo"; @@ -7723,7 +7899,7 @@ TEST_F(GenEmitterTest, test758d285e89a9de8070a7) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test5888499c52285a1b61de) { +TEST_F(GenEmitterTest, test834f4e4a74c0ac6cd011) { Emitter out; out << BeginSeq; out << "foo"; @@ -7744,7 +7920,7 @@ TEST_F(GenEmitterTest, test5888499c52285a1b61de) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test0115edbe7dd05e84f108) { +TEST_F(GenEmitterTest, testd3aeee7524918cf227e7) { Emitter out; out << BeginSeq; out << "foo"; @@ -7765,7 +7941,7 @@ TEST_F(GenEmitterTest, test0115edbe7dd05e84f108) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, testcbbbde92f35597a9e476) { +TEST_F(GenEmitterTest, test7c587549aa0bbd6e2d53) { Emitter out; out << BeginSeq; out << "foo"; @@ -7786,7 +7962,7 @@ TEST_F(GenEmitterTest, testcbbbde92f35597a9e476) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, testa74ec451d449e2a89442) { +TEST_F(GenEmitterTest, testc684ba00d512b6009b02) { Emitter out; out << BeginSeq; out << "foo"; @@ -7807,7 +7983,7 @@ TEST_F(GenEmitterTest, testa74ec451d449e2a89442) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test60faf9f5ac584b2d5a5f) { +TEST_F(GenEmitterTest, testbe39189b638b9e0214dd) { Emitter out; out << Comment("comment"); out << BeginSeq; @@ -7830,7 +8006,7 @@ TEST_F(GenEmitterTest, test60faf9f5ac584b2d5a5f) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, testa34103141c1aec5e8fa8) { +TEST_F(GenEmitterTest, test80326240018ececfa606) { Emitter out; out << Comment("comment"); out << BeginSeq; @@ -7853,7 +8029,7 @@ TEST_F(GenEmitterTest, testa34103141c1aec5e8fa8) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, testa2a2b83cb72c22fc4ac2) { +TEST_F(GenEmitterTest, test8a74653a376d02a2b6db) { Emitter out; out << BeginSeq; out << Comment("comment"); @@ -7876,7 +8052,7 @@ TEST_F(GenEmitterTest, testa2a2b83cb72c22fc4ac2) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test1f7ddcc3e5263ec983c5) { +TEST_F(GenEmitterTest, testaa82cace20492eb66f60) { Emitter out; out << BeginSeq; out << "foo"; @@ -7899,7 +8075,7 @@ TEST_F(GenEmitterTest, test1f7ddcc3e5263ec983c5) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test638672d9ee378d2b59ec) { +TEST_F(GenEmitterTest, testcd4a1cdb4e2a24cae5c1) { Emitter out; out << BeginSeq; out << "foo"; @@ -7922,7 +8098,7 @@ TEST_F(GenEmitterTest, test638672d9ee378d2b59ec) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, testffa1014d27bcb4821993) { +TEST_F(GenEmitterTest, testa9ef5ab0eada79175f6a) { Emitter out; out << BeginSeq; out << "foo"; @@ -7945,7 +8121,7 @@ TEST_F(GenEmitterTest, testffa1014d27bcb4821993) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, tested2f908c978d8908cde2) { +TEST_F(GenEmitterTest, test4060ba4b4f9b8193dcc4) { Emitter out; out << BeginSeq; out << "foo"; @@ -7968,7 +8144,7 @@ TEST_F(GenEmitterTest, tested2f908c978d8908cde2) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test75c9a342b4ff9dcddd76) { +TEST_F(GenEmitterTest, test6cd2fc4be08857654fa0) { Emitter out; out << BeginSeq; out << "foo"; @@ -7991,7 +8167,7 @@ TEST_F(GenEmitterTest, test75c9a342b4ff9dcddd76) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, testb3cf7afef788a5168172) { +TEST_F(GenEmitterTest, testadb892f6183cde28d9cc) { Emitter out; out << BeginSeq; out << "foo"; @@ -8014,7 +8190,7 @@ TEST_F(GenEmitterTest, testb3cf7afef788a5168172) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test32c5731914c4f623f8cb) { +TEST_F(GenEmitterTest, test5e830445d6cafe856b09) { Emitter out; out << BeginSeq; out << "foo"; @@ -8037,7 +8213,7 @@ TEST_F(GenEmitterTest, test32c5731914c4f623f8cb) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, testc4e4b895f5a3615e6c92) { +TEST_F(GenEmitterTest, test16a7d875f6358bdd36ee) { Emitter out; out << Comment("comment"); out << BeginSeq; @@ -8058,7 +8234,7 @@ TEST_F(GenEmitterTest, testc4e4b895f5a3615e6c92) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test024f81ab1e1f375d06c8) { +TEST_F(GenEmitterTest, test75b4342605739c88bc3f) { Emitter out; out << Comment("comment"); out << BeginSeq; @@ -8079,7 +8255,7 @@ TEST_F(GenEmitterTest, test024f81ab1e1f375d06c8) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, testf73efa1189e8e9bab50c) { +TEST_F(GenEmitterTest, test7d42488f1a02d045d278) { Emitter out; out << BeginSeq; out << Comment("comment"); @@ -8100,7 +8276,7 @@ TEST_F(GenEmitterTest, testf73efa1189e8e9bab50c) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test074882061d59e9381988) { +TEST_F(GenEmitterTest, test9ce404764c03c6644c98) { Emitter out; out << BeginSeq; out << BeginSeq; @@ -8121,7 +8297,7 @@ TEST_F(GenEmitterTest, test074882061d59e9381988) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, testb81b19396e93a0e29157) { +TEST_F(GenEmitterTest, testf9e413cd5405efcbd432) { Emitter out; out << BeginSeq; out << BeginSeq; @@ -8142,7 +8318,7 @@ TEST_F(GenEmitterTest, testb81b19396e93a0e29157) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, testd4776155c00bb794aec9) { +TEST_F(GenEmitterTest, test757c82faa95e2e507ee9) { Emitter out; out << BeginSeq; out << BeginSeq; @@ -8163,7 +8339,7 @@ TEST_F(GenEmitterTest, testd4776155c00bb794aec9) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test0d3813f2674c45d54be7) { +TEST_F(GenEmitterTest, testb73d72bf5de11964969f) { Emitter out; out << BeginSeq; out << BeginSeq; @@ -8184,7 +8360,7 @@ TEST_F(GenEmitterTest, test0d3813f2674c45d54be7) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, testaee95b4f9b1161f6fd14) { +TEST_F(GenEmitterTest, test3926f169b9dce6db913f) { Emitter out; out << BeginSeq; out << BeginSeq; @@ -8205,7 +8381,7 @@ TEST_F(GenEmitterTest, testaee95b4f9b1161f6fd14) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test3f7b4338eae92e7a575f) { +TEST_F(GenEmitterTest, test77b0cc7e618a09e0556d) { Emitter out; out << BeginSeq; out << BeginSeq; @@ -8226,7 +8402,7 @@ TEST_F(GenEmitterTest, test3f7b4338eae92e7a575f) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test13a3de779d0756db2635) { +TEST_F(GenEmitterTest, test6817f3e5da2ad8823025) { Emitter out; out << Comment("comment"); out << BeginSeq; @@ -8251,7 +8427,7 @@ TEST_F(GenEmitterTest, test13a3de779d0756db2635) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test6cc39fc73a32cb816c15) { +TEST_F(GenEmitterTest, test817ae48b78359d60888b) { Emitter out; out << Comment("comment"); out << BeginSeq; @@ -8276,7 +8452,7 @@ TEST_F(GenEmitterTest, test6cc39fc73a32cb816c15) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test33c4f4a8854d92180427) { +TEST_F(GenEmitterTest, test9db1bf6278dd7de937e6) { Emitter out; out << BeginSeq; out << Comment("comment"); @@ -8301,7 +8477,7 @@ TEST_F(GenEmitterTest, test33c4f4a8854d92180427) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test1e5612a63642939843fc) { +TEST_F(GenEmitterTest, test4d5ca5c891442ddf7e84) { Emitter out; out << BeginSeq; out << BeginSeq; @@ -8326,7 +8502,7 @@ TEST_F(GenEmitterTest, test1e5612a63642939843fc) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test774fccdfa9e9ea982df2) { +TEST_F(GenEmitterTest, testfb6eb22f4bf080b9ac8b) { Emitter out; out << BeginSeq; out << BeginSeq; @@ -8351,7 +8527,7 @@ TEST_F(GenEmitterTest, test774fccdfa9e9ea982df2) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, testecdd8159ca0150ac9084) { +TEST_F(GenEmitterTest, test3ce4b4ec89282d701502) { Emitter out; out << BeginSeq; out << BeginSeq; @@ -8376,7 +8552,7 @@ TEST_F(GenEmitterTest, testecdd8159ca0150ac9084) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test8081bb348b980caf237e) { +TEST_F(GenEmitterTest, testaf53ae415739a8812200) { Emitter out; out << BeginSeq; out << BeginSeq; @@ -8401,7 +8577,7 @@ TEST_F(GenEmitterTest, test8081bb348b980caf237e) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test5dd9e7d23a6868f8da14) { +TEST_F(GenEmitterTest, test72d3de78c6508500cb00) { Emitter out; out << BeginSeq; out << BeginSeq; @@ -8426,7 +8602,7 @@ TEST_F(GenEmitterTest, test5dd9e7d23a6868f8da14) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test5cfe1848cf1771ee5a62) { +TEST_F(GenEmitterTest, test6dd3d3703718b37fa2a4) { Emitter out; out << BeginSeq; out << BeginSeq; @@ -8451,7 +8627,7 @@ TEST_F(GenEmitterTest, test5cfe1848cf1771ee5a62) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test1a78787fd377fecaa98a) { +TEST_F(GenEmitterTest, testc0beca9064d8081d45c1) { Emitter out; out << BeginSeq; out << BeginSeq; @@ -8476,7 +8652,7 @@ TEST_F(GenEmitterTest, test1a78787fd377fecaa98a) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test3659c5e006def41d781a) { +TEST_F(GenEmitterTest, test14c55f7cd295d89763ca) { Emitter out; out << BeginSeq; out << BeginSeq; @@ -8501,7 +8677,7 @@ TEST_F(GenEmitterTest, test3659c5e006def41d781a) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, testcffd2b80f823bc7b88bc) { +TEST_F(GenEmitterTest, test72a93f054d9296607dff) { Emitter out; out << Comment("comment"); out << BeginSeq; @@ -8528,7 +8704,7 @@ TEST_F(GenEmitterTest, testcffd2b80f823bc7b88bc) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test3646d0fd40876cdc7d90) { +TEST_F(GenEmitterTest, testf0ac164fe5c38cc36922) { Emitter out; out << Comment("comment"); out << BeginSeq; @@ -8555,7 +8731,7 @@ TEST_F(GenEmitterTest, test3646d0fd40876cdc7d90) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, testce3c9ef9e0c642d65146) { +TEST_F(GenEmitterTest, test74efb7c560fd057d25ba) { Emitter out; out << BeginSeq; out << Comment("comment"); @@ -8582,7 +8758,7 @@ TEST_F(GenEmitterTest, testce3c9ef9e0c642d65146) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test666f93032b0297bd6f81) { +TEST_F(GenEmitterTest, test43adceaba606a7f5013f) { Emitter out; out << BeginSeq; out << BeginSeq; @@ -8609,7 +8785,7 @@ TEST_F(GenEmitterTest, test666f93032b0297bd6f81) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test7939e2697c6f4d8b91e9) { +TEST_F(GenEmitterTest, test94160894abf5f0650ec9) { Emitter out; out << BeginSeq; out << BeginSeq; @@ -8636,7 +8812,7 @@ TEST_F(GenEmitterTest, test7939e2697c6f4d8b91e9) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test3c153927514f7330e37d) { +TEST_F(GenEmitterTest, testb77f1131af63dae91031) { Emitter out; out << BeginSeq; out << BeginSeq; @@ -8663,7 +8839,7 @@ TEST_F(GenEmitterTest, test3c153927514f7330e37d) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test59b085e35e59562026da) { +TEST_F(GenEmitterTest, test296aa575c385013e91f0) { Emitter out; out << BeginSeq; out << BeginSeq; @@ -8690,7 +8866,7 @@ TEST_F(GenEmitterTest, test59b085e35e59562026da) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test367f38cb15951f58f294) { +TEST_F(GenEmitterTest, test339bddce4b70064141c4) { Emitter out; out << BeginSeq; out << BeginSeq; @@ -8717,7 +8893,7 @@ TEST_F(GenEmitterTest, test367f38cb15951f58f294) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, testbb3feccd2d65d126c9e4) { +TEST_F(GenEmitterTest, test991a70285cf143adb7fe) { Emitter out; out << BeginSeq; out << BeginSeq; @@ -8744,7 +8920,7 @@ TEST_F(GenEmitterTest, testbb3feccd2d65d126c9e4) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test4c5079959ae8b908aee3) { +TEST_F(GenEmitterTest, test1b1ae70c1b5e7a1a2502) { Emitter out; out << BeginSeq; out << BeginSeq; @@ -8771,7 +8947,7 @@ TEST_F(GenEmitterTest, test4c5079959ae8b908aee3) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, testfa82743c73819854df98) { +TEST_F(GenEmitterTest, test02e58fb30f5a5b3616ec) { Emitter out; out << BeginSeq; out << BeginSeq; @@ -8798,7 +8974,7 @@ TEST_F(GenEmitterTest, testfa82743c73819854df98) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, teste354627da20a33a0ec48) { +TEST_F(GenEmitterTest, testbdc3952445cad78094e2) { Emitter out; out << BeginSeq; out << BeginSeq; @@ -8825,7 +9001,7 @@ TEST_F(GenEmitterTest, teste354627da20a33a0ec48) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test202a2c088ee50d08f182) { +TEST_F(GenEmitterTest, test5d24f2ab8e24cb71d6c9) { Emitter out; out << Comment("comment"); out << BeginSeq; @@ -8848,7 +9024,7 @@ TEST_F(GenEmitterTest, test202a2c088ee50d08f182) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, testc7277e5979e7eaf4d016) { +TEST_F(GenEmitterTest, test1ca2c58583cb7dd8a765) { Emitter out; out << Comment("comment"); out << BeginSeq; @@ -8871,7 +9047,7 @@ TEST_F(GenEmitterTest, testc7277e5979e7eaf4d016) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, teste2b7015fc02e6a5cb416) { +TEST_F(GenEmitterTest, test6086aee45faab48750ad) { Emitter out; out << BeginSeq; out << Comment("comment"); @@ -8894,7 +9070,7 @@ TEST_F(GenEmitterTest, teste2b7015fc02e6a5cb416) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test30e30b89df20ef693d7b) { +TEST_F(GenEmitterTest, testdac42de03b96b1207ec4) { Emitter out; out << BeginSeq; out << BeginMap; @@ -8917,7 +9093,7 @@ TEST_F(GenEmitterTest, test30e30b89df20ef693d7b) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test5cab866f8818d65f5cfe) { +TEST_F(GenEmitterTest, test10d18ea5e198359e218b) { Emitter out; out << BeginSeq; out << BeginMap; @@ -8940,7 +9116,7 @@ TEST_F(GenEmitterTest, test5cab866f8818d65f5cfe) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test6c2133597d99c14720d0) { +TEST_F(GenEmitterTest, test56218e461d6be3a18500) { Emitter out; out << BeginSeq; out << BeginMap; @@ -8963,7 +9139,7 @@ TEST_F(GenEmitterTest, test6c2133597d99c14720d0) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test243b3bb00a830d4d7c5c) { +TEST_F(GenEmitterTest, test9acfd124b72471e34bbd) { Emitter out; out << BeginSeq; out << BeginMap; @@ -8986,7 +9162,7 @@ TEST_F(GenEmitterTest, test243b3bb00a830d4d7c5c) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test7fcdb9d71a9717069b9f) { +TEST_F(GenEmitterTest, test2a1c3780a4dfaa43646e) { Emitter out; out << BeginSeq; out << BeginMap; @@ -9009,7 +9185,7 @@ TEST_F(GenEmitterTest, test7fcdb9d71a9717069b9f) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test0ce9909c470ae5b1d3ae) { +TEST_F(GenEmitterTest, test91e4c547fdab9e8b1c67) { Emitter out; out << BeginSeq; out << BeginMap; @@ -9032,7 +9208,7 @@ TEST_F(GenEmitterTest, test0ce9909c470ae5b1d3ae) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, testc077fd929177c75e3072) { +TEST_F(GenEmitterTest, test3d7e8318208742fe4358) { Emitter out; out << BeginSeq; out << BeginMap; @@ -9055,7 +9231,7 @@ TEST_F(GenEmitterTest, testc077fd929177c75e3072) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test3b500af23a2b660e0330) { +TEST_F(GenEmitterTest, test2e4a92f93d5f9d8c5fed) { Emitter out; out << Comment("comment"); out << BeginSeq; @@ -9082,7 +9258,7 @@ TEST_F(GenEmitterTest, test3b500af23a2b660e0330) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test6d037fb77412a7b06f24) { +TEST_F(GenEmitterTest, test9abf5d48ef7c6f2ed8a0) { Emitter out; out << Comment("comment"); out << BeginSeq; @@ -9109,7 +9285,7 @@ TEST_F(GenEmitterTest, test6d037fb77412a7b06f24) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, testa153abd07d892e352889) { +TEST_F(GenEmitterTest, testc3428819fe7cfe88cf10) { Emitter out; out << BeginSeq; out << Comment("comment"); @@ -9136,7 +9312,7 @@ TEST_F(GenEmitterTest, testa153abd07d892e352889) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, testf328e682c5d872fbb7fa) { +TEST_F(GenEmitterTest, test8007ba3728b0fdbb0cb8) { Emitter out; out << BeginSeq; out << BeginMap; @@ -9163,7 +9339,7 @@ TEST_F(GenEmitterTest, testf328e682c5d872fbb7fa) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, testb59b29e1f0612f8333bf) { +TEST_F(GenEmitterTest, test6eedc1e3db4ceee9caf6) { Emitter out; out << BeginSeq; out << BeginMap; @@ -9190,7 +9366,7 @@ TEST_F(GenEmitterTest, testb59b29e1f0612f8333bf) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test0d2de653c36fff7e875f) { +TEST_F(GenEmitterTest, testd892f2048c7066c74b7e) { Emitter out; out << BeginSeq; out << BeginMap; @@ -9217,7 +9393,7 @@ TEST_F(GenEmitterTest, test0d2de653c36fff7e875f) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test18454002edb2d8ce5dff) { +TEST_F(GenEmitterTest, test736430339c2a221b6d89) { Emitter out; out << BeginSeq; out << BeginMap; @@ -9244,7 +9420,7 @@ TEST_F(GenEmitterTest, test18454002edb2d8ce5dff) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test3869b95fb927574c1cfa) { +TEST_F(GenEmitterTest, test6d51f33adb9324f438d1) { Emitter out; out << BeginSeq; out << BeginMap; @@ -9271,7 +9447,7 @@ TEST_F(GenEmitterTest, test3869b95fb927574c1cfa) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test5f9d2836779171944bd5) { +TEST_F(GenEmitterTest, test00d50067643ed73a3f7f) { Emitter out; out << BeginSeq; out << BeginMap; @@ -9298,7 +9474,7 @@ TEST_F(GenEmitterTest, test5f9d2836779171944bd5) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test00ca741ed5db71e7197c) { +TEST_F(GenEmitterTest, test5fc029e9d46151d31a80) { Emitter out; out << BeginSeq; out << BeginMap; @@ -9325,7 +9501,7 @@ TEST_F(GenEmitterTest, test00ca741ed5db71e7197c) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test63b2dee62c4656039c0d) { +TEST_F(GenEmitterTest, test328542a5a4b65371d2c6) { Emitter out; out << BeginSeq; out << BeginMap; @@ -9352,7 +9528,7 @@ TEST_F(GenEmitterTest, test63b2dee62c4656039c0d) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test6fca12b5bf897f3b933f) { +TEST_F(GenEmitterTest, testf791a97db1c96e9f16c7) { Emitter out; out << BeginSeq; out << BeginMap; @@ -9379,7 +9555,7 @@ TEST_F(GenEmitterTest, test6fca12b5bf897f3b933f) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, testf047a3a7cd147dc69553) { +TEST_F(GenEmitterTest, test44ac18a00d604391a169) { Emitter out; out << Comment("comment"); out << BeginSeq; @@ -9408,7 +9584,7 @@ TEST_F(GenEmitterTest, testf047a3a7cd147dc69553) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, testf191f92a0c15e5624272) { +TEST_F(GenEmitterTest, teste7ff269f6d95faa06abe) { Emitter out; out << Comment("comment"); out << BeginSeq; @@ -9437,7 +9613,7 @@ TEST_F(GenEmitterTest, testf191f92a0c15e5624272) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test874f18e2970f54d92f34) { +TEST_F(GenEmitterTest, test8d056d159803415c2c85) { Emitter out; out << BeginSeq; out << Comment("comment"); @@ -9466,7 +9642,7 @@ TEST_F(GenEmitterTest, test874f18e2970f54d92f34) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test7e94b62bc08fb63b7062) { +TEST_F(GenEmitterTest, test47ef0ff3da945fda8680) { Emitter out; out << BeginSeq; out << BeginMap; @@ -9495,7 +9671,7 @@ TEST_F(GenEmitterTest, test7e94b62bc08fb63b7062) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test50d40871122b935a2937) { +TEST_F(GenEmitterTest, test1f981851e5a72a91614b) { Emitter out; out << BeginSeq; out << BeginMap; @@ -9524,7 +9700,7 @@ TEST_F(GenEmitterTest, test50d40871122b935a2937) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, testb6a8d9eb931455caa603) { +TEST_F(GenEmitterTest, test783be6c196784ca7ff30) { Emitter out; out << BeginSeq; out << BeginMap; @@ -9553,7 +9729,7 @@ TEST_F(GenEmitterTest, testb6a8d9eb931455caa603) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test2f23bfff4a1d201b1f37) { +TEST_F(GenEmitterTest, test217dcab50ef45ac6d344) { Emitter out; out << BeginSeq; out << BeginMap; @@ -9582,7 +9758,7 @@ TEST_F(GenEmitterTest, test2f23bfff4a1d201b1f37) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test41ef23e040f0db2c06c2) { +TEST_F(GenEmitterTest, testec71f98bc646a34c9327) { Emitter out; out << BeginSeq; out << BeginMap; @@ -9611,7 +9787,7 @@ TEST_F(GenEmitterTest, test41ef23e040f0db2c06c2) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, testb8c7b55b5607df008fbe) { +TEST_F(GenEmitterTest, test04595ac13c58c0740048) { Emitter out; out << BeginSeq; out << BeginMap; @@ -9640,7 +9816,7 @@ TEST_F(GenEmitterTest, testb8c7b55b5607df008fbe) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test21daf188179c0c0c7bef) { +TEST_F(GenEmitterTest, test18d724e2ff0f869e9947) { Emitter out; out << BeginSeq; out << BeginMap; @@ -9669,7 +9845,7 @@ TEST_F(GenEmitterTest, test21daf188179c0c0c7bef) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test8ca560e6b97325079e8f) { +TEST_F(GenEmitterTest, test9f80798acafd4cfec0aa) { Emitter out; out << BeginSeq; out << BeginMap; @@ -9698,7 +9874,7 @@ TEST_F(GenEmitterTest, test8ca560e6b97325079e8f) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test7fe0ce86b72c7c1ed7de) { +TEST_F(GenEmitterTest, testeabc051f6366e0275232) { Emitter out; out << BeginSeq; out << BeginMap; @@ -9727,7 +9903,7 @@ TEST_F(GenEmitterTest, test7fe0ce86b72c7c1ed7de) { Parse(out.c_str()); } -TEST_F(GenEmitterTest, test8cc25a6c1aea65ad7de1) { +TEST_F(GenEmitterTest, teste33a98fb01ea45cc91dc) { Emitter out; out << BeginSeq; out << BeginMap; @@ -9755,5 +9931,6 @@ TEST_F(GenEmitterTest, test8cc25a6c1aea65ad7de1) { EXPECT_CALL(handler, OnDocumentEnd()); Parse(out.c_str()); } -} -} + +} // namespace +} // namespace YAML diff --git a/lib/yamlcpp/test/integration/handler_spec_test.cpp b/lib/yamlcpp/test/integration/handler_spec_test.cpp index d142a0d302e..fcb76d583cc 100644 --- a/lib/yamlcpp/test/integration/handler_spec_test.cpp +++ b/lib/yamlcpp/test/integration/handler_spec_test.cpp @@ -199,6 +199,7 @@ TEST_F(HandlerSpecTest, Ex2_10_SimpleAnchor) { EXPECT_CALL(handler, OnScalar(_, "?", 0, "hr")); EXPECT_CALL(handler, OnSequenceStart(_, "?", 0, EmitterStyle::Block)); EXPECT_CALL(handler, OnScalar(_, "?", 0, "Mark McGwire")); + EXPECT_CALL(handler, OnAnchor(_, "SS")); EXPECT_CALL(handler, OnScalar(_, "?", 1, "Sammy Sosa")); EXPECT_CALL(handler, OnSequenceEnd()); EXPECT_CALL(handler, OnScalar(_, "?", 0, "rbi")); @@ -376,6 +377,7 @@ TEST_F(HandlerSpecTest, Ex2_24_GlobalTags) { EXPECT_CALL(handler, OnMapStart(_, "tag:clarkevans.com,2002:circle", 0, EmitterStyle::Block)); EXPECT_CALL(handler, OnScalar(_, "?", 0, "center")); + EXPECT_CALL(handler, OnAnchor(_, "ORIGIN")); EXPECT_CALL(handler, OnMapStart(_, "?", 1, EmitterStyle::Flow)); EXPECT_CALL(handler, OnScalar(_, "?", 0, "x")); EXPECT_CALL(handler, OnScalar(_, "?", 0, "73")); @@ -456,6 +458,7 @@ TEST_F(HandlerSpecTest, Ex2_27_Invoice) { EXPECT_CALL(handler, OnScalar(_, "?", 0, "date")); EXPECT_CALL(handler, OnScalar(_, "?", 0, "2001-01-23")); EXPECT_CALL(handler, OnScalar(_, "?", 0, "bill-to")); + EXPECT_CALL(handler, OnAnchor(_, "id001")); EXPECT_CALL(handler, OnMapStart(_, "?", 1, EmitterStyle::Block)); EXPECT_CALL(handler, OnScalar(_, "?", 0, "given")); EXPECT_CALL(handler, OnScalar(_, "?", 0, "Chris")); @@ -616,6 +619,7 @@ TEST_F(HandlerSpecTest, Ex5_6_NodePropertyIndicators) { EXPECT_CALL(handler, OnDocumentStart(_)); EXPECT_CALL(handler, OnMapStart(_, "?", 0, EmitterStyle::Block)); EXPECT_CALL(handler, OnScalar(_, "?", 0, "anchored")); + EXPECT_CALL(handler, OnAnchor(_, "anchor")); EXPECT_CALL(handler, OnScalar(_, "!local", 1, "value")); EXPECT_CALL(handler, OnScalar(_, "?", 0, "alias")); EXPECT_CALL(handler, OnAlias(_, 1)); @@ -909,8 +913,10 @@ TEST_F(HandlerSpecTest, Ex6_22_GlobalTagPrefix) { TEST_F(HandlerSpecTest, Ex6_23_NodeProperties) { EXPECT_CALL(handler, OnDocumentStart(_)); EXPECT_CALL(handler, OnMapStart(_, "?", 0, EmitterStyle::Block)); + EXPECT_CALL(handler, OnAnchor(_, "a1")); EXPECT_CALL(handler, OnScalar(_, "tag:yaml.org,2002:str", 1, "foo")); EXPECT_CALL(handler, OnScalar(_, "tag:yaml.org,2002:str", 0, "bar")); + EXPECT_CALL(handler, OnAnchor(_, "a2")); EXPECT_CALL(handler, OnScalar(_, "?", 2, "baz")); EXPECT_CALL(handler, OnAlias(_, 1)); EXPECT_CALL(handler, OnMapEnd()); @@ -972,6 +978,7 @@ TEST_F(HandlerSpecTest, Ex6_29_NodeAnchors) { EXPECT_CALL(handler, OnDocumentStart(_)); EXPECT_CALL(handler, OnMapStart(_, "?", 0, EmitterStyle::Block)); EXPECT_CALL(handler, OnScalar(_, "?", 0, "First occurrence")); + EXPECT_CALL(handler, OnAnchor(_, "anchor")); EXPECT_CALL(handler, OnScalar(_, "?", 1, "Value")); EXPECT_CALL(handler, OnScalar(_, "?", 0, "Second occurrence")); EXPECT_CALL(handler, OnAlias(_, 1)); @@ -984,10 +991,12 @@ TEST_F(HandlerSpecTest, Ex7_1_AliasNodes) { EXPECT_CALL(handler, OnDocumentStart(_)); EXPECT_CALL(handler, OnMapStart(_, "?", 0, EmitterStyle::Block)); EXPECT_CALL(handler, OnScalar(_, "?", 0, "First occurrence")); + EXPECT_CALL(handler, OnAnchor(_, "anchor")); EXPECT_CALL(handler, OnScalar(_, "?", 1, "Foo")); EXPECT_CALL(handler, OnScalar(_, "?", 0, "Second occurrence")); EXPECT_CALL(handler, OnAlias(_, 1)); EXPECT_CALL(handler, OnScalar(_, "?", 0, "Override anchor")); + EXPECT_CALL(handler, OnAnchor(_, "anchor")); EXPECT_CALL(handler, OnScalar(_, "?", 2, "Bar")); EXPECT_CALL(handler, OnScalar(_, "?", 0, "Reuse anchor")); EXPECT_CALL(handler, OnAlias(_, 2)); @@ -1307,6 +1316,7 @@ TEST_F(HandlerSpecTest, Ex7_24_FlowNodes) { EXPECT_CALL(handler, OnSequenceStart(_, "?", 0, EmitterStyle::Block)); EXPECT_CALL(handler, OnScalar(_, "tag:yaml.org,2002:str", 0, "a")); EXPECT_CALL(handler, OnScalar(_, "!", 0, "b")); + EXPECT_CALL(handler, OnAnchor(_, "anchor")); EXPECT_CALL(handler, OnScalar(_, "!", 1, "c")); EXPECT_CALL(handler, OnAlias(_, 1)); EXPECT_CALL(handler, OnScalar(_, "tag:yaml.org,2002:str", 0, "")); @@ -1607,5 +1617,5 @@ TEST_F(HandlerSpecTest, Ex8_22_BlockCollectionNodes) { EXPECT_CALL(handler, OnDocumentEnd()); Parse(ex8_22); } -} -} +} // namespace +} // namespace YAML diff --git a/lib/yamlcpp/test/integration/load_node_test.cpp b/lib/yamlcpp/test/integration/load_node_test.cpp index 28850847604..8ed3b0c39d5 100644 --- a/lib/yamlcpp/test/integration/load_node_test.cpp +++ b/lib/yamlcpp/test/integration/load_node_test.cpp @@ -55,6 +55,35 @@ TEST(LoadNodeTest, Binary) { node[1].as()); } +TEST(LoadNodeTest, BinaryWithWhitespaces) { + Node node = Load( + "binaryText: !binary |-\n" + " " + "TWFuIGlzIGRpc3Rpbmd1aXNoZWQsIG5vdCBvbmx5IGJ5IGhpcyByZWFzb24sIGJ1dCBieS\n" + " " + "B0aGlzIHNpbmd1bGFyIHBhc3Npb24gZnJvbSBvdGhlciBhbmltYWxzLCB3aGljaCBpcyBhIG" + "\n" + " " + "x1c3Qgb2YgdGhlIG1pbmQsIHRoYXQgYnkgYSBwZXJzZXZlcmFuY2Ugb2YgZGVsaWdodCBpbi" + "\n" + " " + "B0aGUgY29udGludWVkIGFuZCBpbmRlZmF0aWdhYmxlIGdlbmVyYXRpb24gb2Yga25vd2xlZG" + "\n" + " " + "dlLCBleGNlZWRzIHRoZSBzaG9ydCB2ZWhlbWVuY2Ugb2YgYW55IGNhcm5hbCBwbGVhc3VyZS" + "\n" + " 4K"); + EXPECT_EQ(Binary(reinterpret_cast( + "Man is distinguished, not only by his reason, " + "but by this singular passion from other " + "animals, which is a lust of the mind, that by " + "a perseverance of delight in the continued and " + "indefatigable generation of knowledge, exceeds " + "the short vehemence of any carnal pleasure.\n"), + 270), + node["binaryText"].as()); +} + TEST(LoadNodeTest, IterateSequence) { Node node = Load("[1, 3, 5, 7]"); int seq[] = {1, 3, 5, 7}; @@ -208,8 +237,8 @@ struct ParserExceptionTestCase { TEST(NodeTest, IncompleteJson) { std::vector tests = { {"JSON map without value", "{\"access\"", ErrorMsg::END_OF_MAP_FLOW}, - {"JSON map with colon but no value", "{\"access\":", - ErrorMsg::END_OF_MAP_FLOW}, + {"JSON map with colon but no value", + "{\"access\":", ErrorMsg::END_OF_MAP_FLOW}, {"JSON map with unclosed value quote", "{\"access\":\"", ErrorMsg::END_OF_MAP_FLOW}, {"JSON map without end brace", "{\"access\":\"abc\"", @@ -232,9 +261,9 @@ TEST(NodeTest, LoadTildeAsNull) { } TEST(NodeTest, LoadTagWithParenthesis) { - Node node = Load("!Complex(Tag) foo"); - EXPECT_EQ(node.Tag(), "!Complex(Tag)"); - EXPECT_EQ(node.as(), "foo"); + Node node = Load("!Complex(Tag) foo"); + EXPECT_EQ(node.Tag(), "!Complex(Tag)"); + EXPECT_EQ(node.as(), "foo"); } } // namespace diff --git a/lib/yamlcpp/test/mock_event_handler.h b/lib/yamlcpp/test/mock_event_handler.h index 49d1f0c3342..2c1d15a4b80 100644 --- a/lib/yamlcpp/test/mock_event_handler.h +++ b/lib/yamlcpp/test/mock_event_handler.h @@ -22,5 +22,6 @@ class MockEventHandler : public EventHandler { MOCK_METHOD4(OnMapStart, void(const Mark&, const std::string&, anchor_t, EmitterStyle::value)); MOCK_METHOD0(OnMapEnd, void()); + MOCK_METHOD2(OnAnchor, void(const Mark&, const std::string&)); }; -} +} // namespace YAML diff --git a/lib/yamlcpp/test/node/node_test.cpp b/lib/yamlcpp/test/node/node_test.cpp index 485ad09e1ae..18234dbde01 100644 --- a/lib/yamlcpp/test/node/node_test.cpp +++ b/lib/yamlcpp/test/node/node_test.cpp @@ -1,10 +1,10 @@ +#include "yaml-cpp/node/node.h" #include "yaml-cpp/emitter.h" +#include "yaml-cpp/node/convert.h" +#include "yaml-cpp/node/detail/impl.h" #include "yaml-cpp/node/emit.h" -#include "yaml-cpp/node/node.h" #include "yaml-cpp/node/impl.h" -#include "yaml-cpp/node/convert.h" #include "yaml-cpp/node/iterator.h" -#include "yaml-cpp/node/detail/impl.h" #include "gmock/gmock.h" #include "gtest/gtest.h" @@ -47,6 +47,30 @@ TEST(NodeTest, SimpleAppendSequence) { EXPECT_TRUE(node.IsSequence()); } +TEST(NodeTest, SequenceElementRemoval) { + Node node; + node[0] = "a"; + node[1] = "b"; + node[2] = "c"; + node.remove(1); + EXPECT_TRUE(node.IsSequence()); + EXPECT_EQ(2, node.size()); + EXPECT_EQ("a", node[0].as()); + EXPECT_EQ("c", node[1].as()); +} + +TEST(NodeTest, SequenceLastElementRemoval) { + Node node; + node[0] = "a"; + node[1] = "b"; + node[2] = "c"; + node.remove(2); + EXPECT_TRUE(node.IsSequence()); + EXPECT_EQ(2, node.size()); + EXPECT_EQ("a", node[0].as()); + EXPECT_EQ("b", node[1].as()); +} + TEST(NodeTest, MapElementRemoval) { Node node; node["foo"] = "bar"; @@ -54,6 +78,16 @@ TEST(NodeTest, MapElementRemoval) { EXPECT_TRUE(!node["foo"]); } +TEST(NodeTest, MapIntegerElementRemoval) { + Node node; + node[1] = "hello"; + node[2] = 'c'; + node["foo"] = "bar"; + EXPECT_TRUE(node.IsMap()); + node.remove(1); + EXPECT_TRUE(node.IsMap()); +} + TEST(NodeTest, SimpleAssignSequence) { Node node; node[0] = 10; @@ -106,6 +140,14 @@ TEST(NodeTest, RemoveUnassignedNode) { EXPECT_EQ(0, node.size()); } +TEST(NodeTest, RemoveUnassignedNodeFromMap) { + Node node(NodeType::Map); + Node n; + node[n]; + node.remove(n); + EXPECT_EQ(0, node.size()); +} + TEST(NodeTest, MapForceInsert) { Node node; Node k1("k1"); @@ -349,7 +391,13 @@ TEST(NodeTest, AutoBoolConversion) { EXPECT_TRUE(!!node["foo"]); } -TEST(NodeTest, FloatingPrecision) { +TEST(NodeTest, FloatingPrecisionFloat) { + const float x = 0.123456789; + Node node = Node(x); + EXPECT_EQ(x, node.as()); +} + +TEST(NodeTest, FloatingPrecisionDouble) { const double x = 0.123456789; Node node = Node(x); EXPECT_EQ(x, node.as()); @@ -410,108 +458,108 @@ class NodeEmitterTest : public ::testing::Test { TEST_F(NodeEmitterTest, SimpleFlowSeqNode) { Node node; node.SetStyle(EmitterStyle::Flow); - node.push_back(1.01); - node.push_back(2.01); - node.push_back(3.01); + node.push_back(1.5); + node.push_back(2.25); + node.push_back(3.125); - ExpectOutput("[1.01, 2.01, 3.01]", node); + ExpectOutput("[1.5, 2.25, 3.125]", node); } TEST_F(NodeEmitterTest, NestFlowSeqNode) { Node node, cell0, cell1; - cell0.push_back(1.01); - cell0.push_back(2.01); - cell0.push_back(3.01); + cell0.push_back(1.5); + cell0.push_back(2.25); + cell0.push_back(3.125); - cell1.push_back(4.01); - cell1.push_back(5.01); - cell1.push_back(6.01); + cell1.push_back(4.5); + cell1.push_back(5.25); + cell1.push_back(6.125); node.SetStyle(EmitterStyle::Flow); node.push_back(cell0); node.push_back(cell1); - ExpectOutput("[[1.01, 2.01, 3.01], [4.01, 5.01, 6.01]]", node); + ExpectOutput("[[1.5, 2.25, 3.125], [4.5, 5.25, 6.125]]", node); } TEST_F(NodeEmitterTest, MixBlockFlowSeqNode) { Node node, cell0, cell1; cell0.SetStyle(EmitterStyle::Flow); - cell0.push_back(1.01); - cell0.push_back(2.01); - cell0.push_back(3.01); + cell0.push_back(1.5); + cell0.push_back(2.25); + cell0.push_back(3.125); - cell1.push_back(4.01); - cell1.push_back(5.01); - cell1.push_back(6.01); + cell1.push_back(4.5); + cell1.push_back(5.25); + cell1.push_back(6.125); node.SetStyle(EmitterStyle::Block); node.push_back(cell0); node.push_back(cell1); - ExpectOutput("- [1.01, 2.01, 3.01]\n-\n - 4.01\n - 5.01\n - 6.01", node); + ExpectOutput("- [1.5, 2.25, 3.125]\n-\n - 4.5\n - 5.25\n - 6.125", node); } TEST_F(NodeEmitterTest, NestBlockFlowMapListNode) { Node node, mapNode, blockNode; - node.push_back(1.01); - node.push_back(2.01); - node.push_back(3.01); + node.push_back(1.5); + node.push_back(2.25); + node.push_back(3.125); mapNode.SetStyle(EmitterStyle::Flow); mapNode["position"] = node; - blockNode.push_back(1.01); + blockNode.push_back(1.0625); blockNode.push_back(mapNode); - ExpectOutput("- 1.01\n- {position: [1.01, 2.01, 3.01]}", blockNode); + ExpectOutput("- 1.0625\n- {position: [1.5, 2.25, 3.125]}", blockNode); } TEST_F(NodeEmitterTest, NestBlockMixMapListNode) { Node node, mapNode, blockNode; - node.push_back(1.01); - node.push_back(2.01); - node.push_back(3.01); + node.push_back(1.5); + node.push_back(2.25); + node.push_back(3.125); mapNode.SetStyle(EmitterStyle::Flow); mapNode["position"] = node; - blockNode["scalar"] = 1.01; + blockNode["scalar"] = 1.0625; blockNode["object"] = mapNode; ExpectAnyOutput(blockNode, - "scalar: 1.01\nobject: {position: [1.01, 2.01, 3.01]}", - "object: {position: [1.01, 2.01, 3.01]}\nscalar: 1.01"); + "scalar: 1.0625\nobject: {position: [1.5, 2.25, 3.125]}", + "object: {position: [1.5, 2.25, 3.125]}\nscalar: 1.5"); } TEST_F(NodeEmitterTest, NestBlockMapListNode) { Node node, mapNode; - node.push_back(1.01); - node.push_back(2.01); - node.push_back(3.01); + node.push_back(1.5); + node.push_back(2.25); + node.push_back(3.125); mapNode.SetStyle(EmitterStyle::Block); mapNode["position"] = node; - ExpectOutput("position:\n - 1.01\n - 2.01\n - 3.01", mapNode); + ExpectOutput("position:\n - 1.5\n - 2.25\n - 3.125", mapNode); } TEST_F(NodeEmitterTest, NestFlowMapListNode) { Node node, mapNode; - node.push_back(1.01); - node.push_back(2.01); - node.push_back(3.01); + node.push_back(1.5); + node.push_back(2.25); + node.push_back(3.125); mapNode.SetStyle(EmitterStyle::Flow); mapNode["position"] = node; - ExpectOutput("{position: [1.01, 2.01, 3.01]}", mapNode); + ExpectOutput("{position: [1.5, 2.25, 3.125]}", mapNode); } } } diff --git a/lib/yamlcpp/test/regex_test.cpp b/lib/yamlcpp/test/regex_test.cpp index 7589d2e4bf8..658db9e64dc 100644 --- a/lib/yamlcpp/test/regex_test.cpp +++ b/lib/yamlcpp/test/regex_test.cpp @@ -1,6 +1,6 @@ -#include "gtest/gtest.h" #include "regex_yaml.h" #include "stream.h" +#include "gtest/gtest.h" using YAML::RegEx; using YAML::Stream; @@ -106,8 +106,8 @@ TEST(RegExTest, OperatorOr) { for (int j = i + 1; j < 128; ++j) { auto iStr = std::string(1, char(i)); auto jStr = std::string(1, char(j)); - RegEx ex1 = RegEx(iStr) || RegEx(jStr); - RegEx ex2 = RegEx(jStr) || RegEx(iStr); + RegEx ex1 = RegEx(iStr) | RegEx(jStr); + RegEx ex2 = RegEx(jStr) | RegEx(iStr); for (int k = MIN_CHAR; k < 128; ++k) { auto str = std::string(1, char(k)); @@ -128,8 +128,8 @@ TEST(RegExTest, OperatorOr) { } TEST(RegExTest, OperatorOrShortCircuits) { - RegEx ex1 = RegEx(std::string("aaaa")) || RegEx(std::string("aa")); - RegEx ex2 = RegEx(std::string("aa")) || RegEx(std::string("aaaa")); + RegEx ex1 = RegEx(std::string("aaaa")) | RegEx(std::string("aa")); + RegEx ex2 = RegEx(std::string("aa")) | RegEx(std::string("aaaa")); EXPECT_TRUE(ex1.Matches(std::string("aaaaa"))); EXPECT_EQ(4, ex1.Match(std::string("aaaaa"))); @@ -139,13 +139,13 @@ TEST(RegExTest, OperatorOrShortCircuits) { } TEST(RegExTest, OperatorAnd) { - RegEx emptySet = RegEx('a') && RegEx(); + RegEx emptySet = RegEx('a') & RegEx(); EXPECT_FALSE(emptySet.Matches(std::string("a"))); } TEST(RegExTest, OperatorAndShortCircuits) { - RegEx ex1 = RegEx(std::string("aaaa")) && RegEx(std::string("aa")); - RegEx ex2 = RegEx(std::string("aa")) && RegEx(std::string("aaaa")); + RegEx ex1 = RegEx(std::string("aaaa")) & RegEx(std::string("aa")); + RegEx ex2 = RegEx(std::string("aa")) & RegEx(std::string("aaaa")); EXPECT_TRUE(ex1.Matches(std::string("aaaaa"))); EXPECT_EQ(4, ex1.Match(std::string("aaaaa"))); @@ -174,4 +174,4 @@ TEST(RegExTest, StringOr) { EXPECT_EQ(1, ex.Match(str)); } -} +} // namespace diff --git a/lib/yamlcpp/util/CMakeLists.txt b/lib/yamlcpp/util/CMakeLists.txt index 22866273c74..09dafa24c90 100644 --- a/lib/yamlcpp/util/CMakeLists.txt +++ b/lib/yamlcpp/util/CMakeLists.txt @@ -1,14 +1,26 @@ +cmake_minimum_required(VERSION 3.5) + add_sources(parse.cpp) add_executable(parse parse.cpp) +set_target_properties(parse PROPERTIES + CXX_STANDARD 11 + CXX_STANDARD_REQUIRED ON +) target_link_libraries(parse yaml-cpp) -set_target_properties(parse PROPERTIES COMPILE_FLAGS "-std=c++11") add_sources(sandbox.cpp) add_executable(sandbox sandbox.cpp) +set_target_properties(sandbox PROPERTIES + CXX_STANDARD 11 + CXX_STANDARD_REQUIRED ON +) target_link_libraries(sandbox yaml-cpp) -set_target_properties(sandbox PROPERTIES COMPILE_FLAGS "-std=c++11") add_sources(read.cpp) add_executable(read read.cpp) +set_target_properties(read PROPERTIES + CXX_STANDARD 11 + CXX_STANDARD_REQUIRED ON +) target_link_libraries(read yaml-cpp) -set_target_properties(read PROPERTIES COMPILE_FLAGS "-std=c++11") + diff --git a/mgmt/ConfigManager.cc b/mgmt/ConfigManager.cc index e5ffa732349..19304e02d19 100644 --- a/mgmt/ConfigManager.cc +++ b/mgmt/ConfigManager.cc @@ -42,8 +42,9 @@ #define TS_ARCHIVE_STAT_MTIME(t) ((t).st_mtime * 1000000000) #endif -ConfigManager::ConfigManager(const char *fileName_, const char *configName_, bool root_access_needed_, ConfigManager *parentConfig_) - : root_access_needed(root_access_needed_), parentConfig(parentConfig_) +ConfigManager::ConfigManager(const char *fileName_, const char *configName_, bool root_access_needed_, bool isRequired_, + ConfigManager *parentConfig_) + : root_access_needed(root_access_needed_), isRequired(isRequired_), parentConfig(parentConfig_) { ExpandingArray existVer(25, true); // Existing versions struct stat fileInfo; @@ -59,14 +60,15 @@ ConfigManager::ConfigManager(const char *fileName_, const char *configName_, boo configName = ats_strdup(configName_); ink_mutex_init(&fileAccessLock); - // Check to make sure that our configuration file exists // if (statFile(&fileInfo) < 0) { - // If we can't find an active version because there is none we have a hard failure. - mgmt_fatal(0, "[ConfigManager::ConfigManager] Unable to find configuration file %s.\n\tStat failed : %s\n", fileName, - strerror(errno)); + mgmt_log("[ConfigManager::ConfigManager] %s Unable to load: %s", fileName, strerror(errno)); + if (isRequired) { + mgmt_fatal(0, "[ConfigManager::ConfigManager] Unable to open required configuration file %s.\n\tStat failed : %s\n", fileName, + strerror(errno)); + } } else { fileLastModified = TS_ARCHIVE_STAT_MTIME(fileInfo); } @@ -95,12 +97,12 @@ ConfigManager::statFile(struct stat *buf) return statResult; } -// bool ConfigManager::checkForUserUpdate() +// bool ConfigManager::checkForUserUpdate(RollBackCheckType how) // // Called to check if the file has been changed by the user. // Timestamps are compared to see if a change occurred bool -ConfigManager::checkForUserUpdate() +ConfigManager::checkForUserUpdate(RollBackCheckType how) { struct stat fileInfo; bool result; @@ -113,10 +115,11 @@ ConfigManager::checkForUserUpdate() } if (fileLastModified < TS_ARCHIVE_STAT_MTIME(fileInfo)) { - fileLastModified = TS_ARCHIVE_STAT_MTIME(fileInfo); - configFiles->fileChanged(fileName, configName); - mgmt_log("User has changed config file %s\n", fileName); - + if (how == ROLLBACK_CHECK_AND_UPDATE) { + fileLastModified = TS_ARCHIVE_STAT_MTIME(fileInfo); + configFiles->fileChanged(fileName, configName); + mgmt_log("User has changed config file %s\n", fileName); + } result = true; } else { result = false; diff --git a/mgmt/ConfigManager.h b/mgmt/ConfigManager.h index d27aed10a5a..b10a9ae3cfe 100644 --- a/mgmt/ConfigManager.h +++ b/mgmt/ConfigManager.h @@ -31,6 +31,11 @@ class TextBuffer; class ExpandingArray; +enum RollBackCheckType { + ROLLBACK_CHECK_AND_UPDATE, + ROLLBACK_CHECK_ONLY, +}; + // // class ConfigManager // @@ -49,7 +54,8 @@ class ConfigManager { public: // fileName_ should be rooted or a base file name. - ConfigManager(const char *fileName_, const char *configName_, bool root_access_needed, ConfigManager *parentConfig_); + ConfigManager(const char *fileName_, const char *configName_, bool root_access_needed, bool isRequired_, + ConfigManager *parentConfig_); ~ConfigManager(); // Manual take out of lock required @@ -66,7 +72,7 @@ class ConfigManager }; // Check if a file has changed, automatically holds the lock. Used by FileManager. - bool checkForUserUpdate(); + bool checkForUserUpdate(RollBackCheckType); // These are getters, for FileManager to get info about a particular configuration. const char * @@ -99,6 +105,12 @@ class ConfigManager return root_access_needed; } + bool + getIsRequired() const + { + return isRequired; + } + FileManager *configFiles = nullptr; // Manager to notify on an update. // noncopyable @@ -112,6 +124,7 @@ class ConfigManager char *fileName; char *configName; bool root_access_needed; + bool isRequired; ConfigManager *parentConfig; time_t fileLastModified = 0; }; diff --git a/mgmt/FileManager.cc b/mgmt/FileManager.cc index e5737e9458d..fce09a0dcf2 100644 --- a/mgmt/FileManager.cc +++ b/mgmt/FileManager.cc @@ -92,20 +92,22 @@ FileManager::registerCallback(FileCallbackFunc func) // Pointers to the new objects are stored in the bindings hashtable // void -FileManager::addFile(const char *fileName, const char *configName, bool root_access_needed, ConfigManager *parentConfig) +FileManager::addFile(const char *fileName, const char *configName, bool root_access_needed, bool isRequired, + ConfigManager *parentConfig) { ink_mutex_acquire(&accessLock); - addFileHelper(fileName, configName, root_access_needed, parentConfig); + addFileHelper(fileName, configName, root_access_needed, isRequired, parentConfig); ink_mutex_release(&accessLock); } // caller must hold the lock void -FileManager::addFileHelper(const char *fileName, const char *configName, bool root_access_needed, ConfigManager *parentConfig) +FileManager::addFileHelper(const char *fileName, const char *configName, bool root_access_needed, bool isRequired, + ConfigManager *parentConfig) { ink_assert(fileName != nullptr); - ConfigManager *rb = new ConfigManager(fileName, configName, root_access_needed, parentConfig); + ConfigManager *rb = new ConfigManager(fileName, configName, root_access_needed, isRequired, parentConfig); rb->configFiles = this; bindings.emplace(rb->getFileName(), rb); @@ -177,7 +179,7 @@ FileManager::rereadConfig() rb = it.second; // ToDo: rb->isVersions() was always true before, because numberBackups was always >= 1. So ROLLBACK_CHECK_ONLY could not // happen at all... - if (rb->checkForUserUpdate()) { + if (rb->checkForUserUpdate(ROLLBACK_CHECK_AND_UPDATE)) { changedFiles.push_back(rb); if (rb->isChildManaged()) { if (std::find(parentFileNeedChange.begin(), parentFileNeedChange.end(), rb->getParentConfig()) == @@ -237,7 +239,7 @@ FileManager::isConfigStale() ink_mutex_acquire(&accessLock); for (auto &&it : bindings) { rb = it.second; - if (rb->checkForUserUpdate()) { + if (rb->checkForUserUpdate(ROLLBACK_CHECK_ONLY)) { stale = true; break; } @@ -257,7 +259,7 @@ FileManager::configFileChild(const char *parent, const char *child) ink_mutex_acquire(&accessLock); if (auto it = bindings.find(parent); it != bindings.end()) { parentConfig = it->second; - addFileHelper(child, "", parentConfig->rootAccessNeeded(), parentConfig); + addFileHelper(child, "", parentConfig->rootAccessNeeded(), parentConfig->getIsRequired(), parentConfig); } ink_mutex_release(&accessLock); } diff --git a/mgmt/FileManager.h b/mgmt/FileManager.h index dfc59cf3fed..64725b620a7 100644 --- a/mgmt/FileManager.h +++ b/mgmt/FileManager.h @@ -48,7 +48,7 @@ enum lockAction_t { // // public functions: // -// addFile(char*, char *, configFileInfo*) - adds a new config file to be +// addFile(char*, char *, bool, configFileInfo*) - adds a new config file to be // managed. A ConfigManager object is created for the file. // if the file_info ptr is not NULL, a WebFileEdit object // is also created @@ -80,7 +80,8 @@ class FileManager public: FileManager(); ~FileManager(); - void addFile(const char *fileName, const char *configName, bool root_access_needed, ConfigManager *parentConfig = nullptr); + void addFile(const char *fileName, const char *configName, bool root_access_needed, bool isRequired, + ConfigManager *parentConfig = nullptr); bool getConfigObj(const char *fileName, ConfigManager **rbPtr); void registerCallback(FileCallbackFunc func); void fileChanged(const char *fileName, const char *configName); @@ -93,7 +94,8 @@ class FileManager ink_mutex cbListLock; // Protects the CallBack List DLL cblist; std::unordered_map bindings; - void addFileHelper(const char *fileName, const char *configName, bool root_access_needed, ConfigManager *parentConfig); + void addFileHelper(const char *fileName, const char *configName, bool root_access_needed, bool isRequired, + ConfigManager *parentConfig); }; void initializeRegistry(); // implemented in AddConfigFilesHere.cc diff --git a/mgmt/LocalManager.cc b/mgmt/LocalManager.cc index 946cf772969..5f55df39027 100644 --- a/mgmt/LocalManager.cc +++ b/mgmt/LocalManager.cc @@ -38,6 +38,7 @@ #include "tscpp/util/TextView.h" #include "tscore/BufferWriter.h" #include "tscore/bwf_std_format.h" +#include "tscore/Filenames.h" #if TS_USE_POSIX_CAP #include @@ -405,7 +406,7 @@ LocalManager::pollMgmtProcessServer() // read the message if ((res = mgmt_read_pipe(watched_process_fd, reinterpret_cast(&mh_hdr), sizeof(MgmtMessageHdr))) > 0) { - MgmtMessageHdr *mh_full = static_cast(alloca(sizeof(MgmtMessageHdr) + mh_hdr.data_len)); + MgmtMessageHdr *mh_full = static_cast(malloc(sizeof(MgmtMessageHdr) + mh_hdr.data_len)); memcpy(mh_full, &mh_hdr, sizeof(MgmtMessageHdr)); char *data_raw = reinterpret_cast(mh_full) + sizeof(MgmtMessageHdr); if ((res = mgmt_read_pipe(watched_process_fd, data_raw, mh_hdr.data_len)) > 0) { @@ -413,6 +414,7 @@ LocalManager::pollMgmtProcessServer() } else if (res < 0) { mgmt_fatal(0, "[LocalManager::pollMgmtProcessServer] Error in read (errno: %d)\n", -res); } + free(mh_full); } else if (res < 0) { mgmt_fatal(0, "[LocalManager::pollMgmtProcessServer] Error in read (errno: %d)\n", -res); } @@ -749,9 +751,9 @@ LocalManager::processEventQueue() // check if we have a local file update if (mh->msg_id == MGMT_EVENT_CONFIG_FILE_UPDATE) { // records.config - if (!(strcmp(payload.begin(), REC_CONFIG_FILE))) { + if (!(strcmp(payload.begin(), ts::filename::RECORDS))) { if (RecReadConfigFile() != REC_ERR_OKAY) { - mgmt_elog(errno, "[fileUpdated] Config update failed for records.config\n"); + mgmt_elog(errno, "[fileUpdated] Config update failed for %s\n", ts::filename::RECORDS); } else { RecConfigWarnIfUnregistered(); } @@ -939,29 +941,91 @@ LocalManager::listenForProxy() // We are not already bound, bind the port for (auto &p : lmgmt->m_proxy_ports) { if (ts::NO_FD == p.m_fd) { - this->bindProxyPort(p); + // Check the protocol (TCP or UDP) and create an appropriate socket + if (p.isQUIC()) { + this->bindUdpProxyPort(p); + } else { + this->bindTcpProxyPort(p); + } } - // read backlog configuration value and overwrite the default value if found - bool found; std::string_view fam{ats_ip_family_name(p.m_family)}; - RecInt backlog = REC_readInteger("proxy.config.net.listen_backlog", &found); - backlog = (found && backlog >= 0) ? backlog : ats_tcp_somaxconn(); + if (p.isQUIC()) { + // Can we do something like listen backlog for QUIC(UDP) ?? + // Do nothing for now + } else { + // read backlog configuration value and overwrite the default value if found + bool found; + RecInt backlog = REC_readInteger("proxy.config.net.listen_backlog", &found); + backlog = (found && backlog >= 0) ? backlog : ats_tcp_somaxconn(); - if ((listen(p.m_fd, backlog)) < 0) { - mgmt_fatal(errno, "[LocalManager::listenForProxy] Unable to listen on port: %d (%.*s)\n", p.m_port, fam.size(), fam.data()); + if ((listen(p.m_fd, backlog)) < 0) { + mgmt_fatal(errno, "[LocalManager::listenForProxy] Unable to listen on port: %d (%.*s)\n", p.m_port, fam.size(), fam.data()); + } } + mgmt_log("[LocalManager::listenForProxy] Listening on port: %d (%.*s)\n", p.m_port, fam.size(), fam.data()); } return; } /* - * bindProxyPort() + * bindUdpProxyPort() + * Function binds the accept port of the proxy + */ +void +LocalManager::bindUdpProxyPort(HttpProxyPort &port) +{ + int one = 1; + int priv = (port.m_port < 1024 && 0 != geteuid()) ? ElevateAccess::LOW_PORT_PRIVILEGE : 0; + + ElevateAccess access(priv); + + if ((port.m_fd = socket(port.m_family, SOCK_DGRAM, 0)) < 0) { + mgmt_fatal(0, "[bindProxyPort] Unable to create socket : %s\n", strerror(errno)); + } + + if (port.m_family == AF_INET6) { + if (setsockopt(port.m_fd, IPPROTO_IPV6, IPV6_V6ONLY, SOCKOPT_ON, sizeof(int)) < 0) { + mgmt_log("[bindProxyPort] Unable to set socket options: %d : %s\n", port.m_port, strerror(errno)); + } + } + if (setsockopt(port.m_fd, SOL_SOCKET, SO_REUSEADDR, reinterpret_cast(&one), sizeof(int)) < 0) { + mgmt_fatal(0, "[bindProxyPort] Unable to set socket options: %d : %s\n", port.m_port, strerror(errno)); + } + + IpEndpoint ip; + if (port.m_inbound_ip.isValid()) { + ip.assign(port.m_inbound_ip); + } else if (AF_INET6 == port.m_family) { + if (m_inbound_ip6.isValid()) { + ip.assign(m_inbound_ip6); + } else { + ip.setToAnyAddr(AF_INET6); + } + } else if (AF_INET == port.m_family) { + if (m_inbound_ip4.isValid()) { + ip.assign(m_inbound_ip4); + } else { + ip.setToAnyAddr(AF_INET); + } + } else { + mgmt_fatal(0, "[bindProxyPort] Proxy port with invalid address type %d\n", port.m_family); + } + ip.port() = htons(port.m_port); + if (bind(port.m_fd, &ip.sa, ats_ip_size(&ip)) < 0) { + mgmt_fatal(0, "[bindProxyPort] Unable to bind socket: %d : %s\n", port.m_port, strerror(errno)); + } + + Debug("lm", "[bindProxyPort] Successfully bound proxy port %d", port.m_port); +} + +/* + * bindTcpProxyPort() * Function binds the accept port of the proxy */ void -LocalManager::bindProxyPort(HttpProxyPort &port) +LocalManager::bindTcpProxyPort(HttpProxyPort &port) { int one = 1; int priv = (port.m_port < 1024 && 0 != geteuid()) ? ElevateAccess::LOW_PORT_PRIVILEGE : 0; diff --git a/mgmt/LocalManager.h b/mgmt/LocalManager.h index 4c9cb37fa17..f5121a41e63 100644 --- a/mgmt/LocalManager.h +++ b/mgmt/LocalManager.h @@ -76,7 +76,8 @@ class LocalManager : public BaseManager void processEventQueue(); bool startProxy(const char *onetime_options); void listenForProxy(); - void bindProxyPort(HttpProxyPort &); + void bindUdpProxyPort(HttpProxyPort &); + void bindTcpProxyPort(HttpProxyPort &); void closeProxyPorts(); void mgmtCleanup(); diff --git a/mgmt/Makefile.am b/mgmt/Makefile.am index a52678d743b..b5ce714dae9 100644 --- a/mgmt/Makefile.am +++ b/mgmt/Makefile.am @@ -65,8 +65,6 @@ libmgmt_lm_la_SOURCES = \ LocalManager.h \ ConfigManager.cc \ ConfigManager.h \ - DerivativeMetrics.cc \ - DerivativeMetrics.h \ WebMgmtUtils.cc \ WebMgmtUtils.h diff --git a/mgmt/MgmtDefs.h b/mgmt/MgmtDefs.h index 6d1aad9a473..2245ee5b9db 100644 --- a/mgmt/MgmtDefs.h +++ b/mgmt/MgmtDefs.h @@ -72,7 +72,7 @@ struct MgmtConverter { * This is passed a @c void* which is a pointer to the member in the configuration instance. * This function must return a @c MgmtInt converted from that value. */ - MgmtInt (*load_int)(void *) = nullptr; + MgmtInt (*load_int)(const void *) = nullptr; /** Store a @c MgmtInt into a native type. * @@ -86,7 +86,7 @@ struct MgmtConverter { * This is passed a @c void* which is a pointer to the member in the configuration instance. * This function must return a @c MgmtFloat converted from that value. */ - MgmtFloat (*load_float)(void *) = nullptr; + MgmtFloat (*load_float)(const void *) = nullptr; /** Store a @c MgmtFloat into a native type. * @@ -100,7 +100,7 @@ struct MgmtConverter { * This is passed a @c void* which is a pointer to the member in the configuration instance. * This function must return a @c string_view which contains the text for the member. */ - std::string_view (*load_string)(void *) = nullptr; + std::string_view (*load_string)(const void *) = nullptr; /** Store a view in a native type. * @@ -110,30 +110,33 @@ struct MgmtConverter { void (*store_string)(void *, std::string_view) = nullptr; // Convenience constructors because generally only one pair is valid. - MgmtConverter(MgmtInt (*load)(void *), void (*store)(void *, MgmtInt)); - MgmtConverter(MgmtFloat (*load)(void *), void (*store)(void *, MgmtFloat)); - MgmtConverter(std::string_view (*load)(void *), void (*store)(void *, std::string_view)); + MgmtConverter(MgmtInt (*load)(const void *), void (*store)(void *, MgmtInt)); + MgmtConverter(MgmtFloat (*load)(const void *), void (*store)(void *, MgmtFloat)); + MgmtConverter(std::string_view (*load)(const void *), void (*store)(void *, std::string_view)); - MgmtConverter(MgmtInt (*_load_int)(void *), void (*_store_int)(void *, MgmtInt), MgmtFloat (*_load_float)(void *), - void (*_store_float)(void *, MgmtFloat), std::string_view (*_load_string)(void *), + MgmtConverter(MgmtInt (*_load_int)(const void *), void (*_store_int)(void *, MgmtInt), MgmtFloat (*_load_float)(const void *), + void (*_store_float)(void *, MgmtFloat), std::string_view (*_load_string)(const void *), void (*_store_string)(void *, std::string_view)); }; -inline MgmtConverter::MgmtConverter(MgmtInt (*load)(void *), void (*store)(void *, MgmtInt)) : load_int(load), store_int(store) {} +inline MgmtConverter::MgmtConverter(MgmtInt (*load)(const void *), void (*store)(void *, MgmtInt)) + : load_int(load), store_int(store) +{ +} -inline MgmtConverter::MgmtConverter(MgmtFloat (*load)(void *), void (*store)(void *, MgmtFloat)) +inline MgmtConverter::MgmtConverter(MgmtFloat (*load)(const void *), void (*store)(void *, MgmtFloat)) : load_float(load), store_float(store) { } -inline MgmtConverter::MgmtConverter(std::string_view (*load)(void *), void (*store)(void *, std::string_view)) +inline MgmtConverter::MgmtConverter(std::string_view (*load)(const void *), void (*store)(void *, std::string_view)) : load_string(load), store_string(store) { } -inline MgmtConverter::MgmtConverter(MgmtInt (*_load_int)(void *), void (*_store_int)(void *, MgmtInt), - MgmtFloat (*_load_float)(void *), void (*_store_float)(void *, MgmtFloat), - std::string_view (*_load_string)(void *), void (*_store_string)(void *, std::string_view)) +inline MgmtConverter::MgmtConverter(MgmtInt (*_load_int)(const void *), void (*_store_int)(void *, MgmtInt), + MgmtFloat (*_load_float)(const void *), void (*_store_float)(void *, MgmtFloat), + std::string_view (*_load_string)(const void *), void (*_store_string)(void *, std::string_view)) : load_int(_load_int), store_int(_store_int), load_float(_load_float), diff --git a/mgmt/ProcessManager.cc b/mgmt/ProcessManager.cc index ef18e237c6a..e3c9baced24 100644 --- a/mgmt/ProcessManager.cc +++ b/mgmt/ProcessManager.cc @@ -82,11 +82,12 @@ read_management_message(int sockfd, MgmtMessageHdr **msg) } void -ProcessManager::start(std::function const &cb) +ProcessManager::start(std::function const &cb_init, std::function const &cb_destroy) { Debug("pmgmt", "starting process manager"); - init = cb; + init = cb_init; + destroy = cb_destroy; ink_release_assert(running == 0); ink_atomic_increment(&running, 1); @@ -153,7 +154,7 @@ ProcessManager::processManagerThread(void *arg) } if (pmgmt->init) { - pmgmt->init(); + pmgmt->managerThread = pmgmt->init(); } // Start pumping messages between the local process and the process @@ -178,6 +179,11 @@ ProcessManager::processManagerThread(void *arg) } } + if (pmgmt->destroy && pmgmt->managerThread != nullptr) { + pmgmt->destroy(pmgmt->managerThread); + pmgmt->managerThread = nullptr; + } + return ret; } @@ -266,6 +272,10 @@ ProcessManager::signalManager(int msg_id, std::string_view text) void ProcessManager::signalManager(MgmtMessageHdr *mh) { + if (!this->running) { + Warning("MgmtMessageHdr is ignored. Because ProcessManager is not running"); + return; + } ink_release_assert(::enqueue(mgmt_signal_queue, mh)); #if HAVE_EVENTFD diff --git a/mgmt/ProcessManager.h b/mgmt/ProcessManager.h index fdf555da45f..c19b14cd9b1 100644 --- a/mgmt/ProcessManager.h +++ b/mgmt/ProcessManager.h @@ -28,6 +28,8 @@ #include #include +#include + #include "MgmtUtils.h" #include "BaseManager.h" #include "tscore/ink_sock.h" @@ -49,7 +51,8 @@ class ProcessManager : public BaseManager // Start a thread for the process manager. If @a cb is set then it // is called after the thread is started and before any messages are // processed. - void start(std::function const &cb = std::function()); + void start(std::function const &cb_init = std::function(), + std::function const &cb_destroy = std::function()); // Stop the process manager, dropping any unprocessed messages. void stop(); @@ -94,7 +97,9 @@ class ProcessManager : public BaseManager /// Thread initialization callback. /// This allows @c traffic_server and @c traffic_manager to perform different initialization in the thread. - std::function init; + std::function init; + std::function destroy; + TSThread managerThread = nullptr; int local_manager_sockfd; #if HAVE_EVENTFD diff --git a/mgmt/ProxyConfig.cc b/mgmt/ProxyConfig.cc index e18fc64429e..d30d06544b5 100644 --- a/mgmt/ProxyConfig.cc +++ b/mgmt/ProxyConfig.cc @@ -29,67 +29,6 @@ ConfigProcessor configProcessor; -void * -config_int_cb(void *data, void *value) -{ - *static_cast(data) = *static_cast(value); - return nullptr; -} - -void * -config_float_cb(void *data, void *value) -{ - *static_cast(data) = *static_cast(value); - return nullptr; -} - -void * -config_long_long_cb(void *data, void *value) -{ - *static_cast(data) = *static_cast(value); - return nullptr; -} - -///////////////////////////////////////////////////////////// -// -// config_string_alloc_cb() -// -// configuration callback function. The function is called -// by the manager when a string configuration variable -// changed. It allocates new memory for the new data. -// the old variable is scheduled to be freed using -// ConfigFreerContinuation which will free the memory -// used for this variable after long time, assuming that -// during all this time all the users of this memory will -// disappear. -///////////////////////////////////////////////////////////// -void * -config_string_alloc_cb(void *data, void *value) -{ - char *_ss = static_cast(value); - char *_new_value = nullptr; - -#if defined(DEBUG_CONFIG_STRING_UPDATE) - printf("config callback [new, old] = [%s : %s]\n", (_ss) ? (_ss) : (""), (*(char **)data) ? (*(char **)data) : ("")); -#endif - - if (_ss) { - int len = strlen(_ss); - _new_value = static_cast(ats_malloc(len + 1)); - memcpy(_new_value, _ss, len + 1); - } - - char *_temp2 = *static_cast(data); - *static_cast(data) = _new_value; - - // free old data - if (_temp2 != nullptr) { - new_Freer(_temp2, HRTIME_DAY); - } - - return nullptr; -} - class ConfigInfoReleaser : public Continuation { public: diff --git a/mgmt/ProxyConfig.h b/mgmt/ProxyConfig.h index abd7c780cdc..493dec230e2 100644 --- a/mgmt/ProxyConfig.h +++ b/mgmt/ProxyConfig.h @@ -30,12 +30,6 @@ class ProxyMutex; -void *config_int_cb(void *data, void *value); -void *config_long_long_cb(void *data, void *value); -void *config_float_cb(void *data, void *value); -void *config_string511_cb(void *data, void *value); -void *config_string_alloc_cb(void *data, void *value); - // // Macros that spin waiting for the data to be bound // @@ -68,7 +62,11 @@ class ConfigProcessor ~scoped_config() { ClassType::release(ptr); } operator bool() const { return ptr != nullptr; } operator const ConfigType *() const { return ptr; } - const ConfigType *operator->() const { return ptr; } + const ConfigType * + operator->() const + { + return ptr; + } private: ConfigType *ptr; diff --git a/mgmt/RecordsConfig.cc b/mgmt/RecordsConfig.cc index 71214fb5b5a..3ac18d2928c 100644 --- a/mgmt/RecordsConfig.cc +++ b/mgmt/RecordsConfig.cc @@ -22,6 +22,7 @@ */ #include "tscore/ink_config.h" +#include "tscore/Filenames.h" #include "RecordsConfig.h" #if TS_USE_REMOTE_UNWINDING @@ -110,12 +111,14 @@ static const RecordElement RecordsConfig[] = // By default Traffic Server set number of execution threads equal to total CPUs {RECT_CONFIG, "proxy.config.exec_thread.autoconfig", RECD_INT, "1", RECU_RESTART_TS, RR_NULL, RECC_INT, "[0-1]", RECA_READ_ONLY} , - {RECT_CONFIG, "proxy.config.exec_thread.autoconfig.scale", RECD_FLOAT, "1.5", RECU_RESTART_TS, RR_NULL, RECC_NULL, nullptr, RECA_READ_ONLY} + {RECT_CONFIG, "proxy.config.exec_thread.autoconfig.scale", RECD_FLOAT, "1.0", RECU_RESTART_TS, RR_NULL, RECC_NULL, nullptr, RECA_READ_ONLY} , {RECT_CONFIG, "proxy.config.exec_thread.limit", RECD_INT, "2", RECU_RESTART_TS, RR_NULL, RECC_INT, "[1-" TS_STR(TS_MAX_NUMBER_EVENT_THREADS) "]", RECA_READ_ONLY} , {RECT_CONFIG, "proxy.config.exec_thread.affinity", RECD_INT, "1", RECU_RESTART_TS, RR_NULL, RECC_INT, "[0-4]", RECA_READ_ONLY} , + {RECT_CONFIG, "proxy.config.exec_thread.listen", RECD_INT, "0", RECU_RESTART_TS, RR_NULL, RECC_INT, "[0-1]", RECA_READ_ONLY} + , {RECT_CONFIG, "proxy.config.accept_threads", RECD_INT, "1", RECU_RESTART_TS, RR_NULL, RECC_INT, "[0-" TS_STR(TS_MAX_NUMBER_EVENT_THREADS) "]", RECA_READ_ONLY} , {RECT_CONFIG, "proxy.config.task_threads", RECD_INT, "2", RECU_RESTART_TS, RR_NULL, RECC_INT, "[1-" TS_STR(TS_MAX_NUMBER_EVENT_THREADS) "]", RECA_READ_ONLY} @@ -339,6 +342,8 @@ static const RecordElement RecordsConfig[] = , {RECT_CONFIG, "proxy.config.http.keep_alive_post_out", RECD_INT, "1", RECU_DYNAMIC, RR_NULL, RECC_NULL, nullptr, RECA_NULL} , + {RECT_CONFIG, "proxy.config.http.request_buffer_enabled", RECD_INT, "0", RECU_DYNAMIC, RR_NULL, RECC_NULL, nullptr, RECA_NULL} + , {RECT_CONFIG, "proxy.config.http.chunking_enabled", RECD_INT, "1", RECU_DYNAMIC, RR_NULL, RECC_NULL, nullptr, RECA_NULL} , {RECT_CONFIG, "proxy.config.http.chunking.size", RECD_INT, "4096", RECU_DYNAMIC, RR_NULL, RECC_NULL, nullptr, RECA_NULL} @@ -381,7 +386,7 @@ static const RecordElement RecordsConfig[] = , {RECT_CONFIG, "proxy.config.http.per_server.connection.max", RECD_INT, "0", RECU_DYNAMIC, RR_NULL, RECC_STR, "^[0-9]+$", RECA_NULL} , - {RECT_CONFIG, "proxy.config.http.per_server.connection.match", RECD_STRING, "ip", RECU_DYNAMIC, RR_NULL, RECC_STR, "^(?:ip|host|both|none)$", RECA_NULL} + {RECT_CONFIG, "proxy.config.http.per_server.connection.match", RECD_STRING, "both", RECU_DYNAMIC, RR_NULL, RECC_STR, "^(?:ip|host|both|none)$", RECA_NULL} , {RECT_CONFIG, "proxy.config.http.per_server.connection.alert_delay", RECD_INT, "60", RECU_DYNAMIC, RR_NULL, RECC_STR, "^[0-9]+$", RECA_NULL} , @@ -395,7 +400,7 @@ static const RecordElement RecordsConfig[] = , {RECT_CONFIG, "proxy.config.net.max_connections_in", RECD_INT, "30000", RECU_DYNAMIC, RR_NULL, RECC_STR, "^[0-9]+$", RECA_NULL} , - {RECT_CONFIG, "proxy.config.net.max_connections_active_in", RECD_INT, "10000", RECU_DYNAMIC, RR_NULL, RECC_STR, "^[0-9]+$", RECA_NULL} + {RECT_CONFIG, "proxy.config.net.max_requests_in", RECD_INT, "0", RECU_DYNAMIC, RR_NULL, RECC_STR, "^[0-9]+$", RECA_NULL} , // ########################### @@ -416,7 +421,7 @@ static const RecordElement RecordsConfig[] = // ############################## {RECT_CONFIG, "proxy.config.http.parent_proxies", RECD_STRING, nullptr, RECU_DYNAMIC, RR_NULL, RECC_STR, ".*", RECA_NULL} , - {RECT_CONFIG, "proxy.config.http.parent_proxy.file", RECD_STRING, "parent.config", RECU_DYNAMIC, RR_NULL, RECC_NULL, nullptr, RECA_NULL} + {RECT_CONFIG, "proxy.config.http.parent_proxy.file", RECD_STRING, ts::filename::PARENT, RECU_DYNAMIC, RR_NULL, RECC_NULL, nullptr, RECA_NULL} , {RECT_CONFIG, "proxy.config.http.parent_proxy.retry_time", RECD_INT, "300", RECU_DYNAMIC, RR_NULL, RECC_NULL, nullptr, RECA_NULL} , @@ -522,7 +527,7 @@ static const RecordElement RecordsConfig[] = , {RECT_CONFIG, "proxy.config.http.insert_forwarded", RECD_STRING, "none", RECU_DYNAMIC, RR_NULL, RECC_NULL, nullptr, RECA_NULL} , - {RECT_CONFIG, "proxy.config.http.proxy_protocol_whitelist", RECD_STRING, "none", RECU_NULL, RR_NULL, RECC_NULL, nullptr, RECA_NULL} + {RECT_CONFIG, "proxy.config.http.proxy_protocol_allowlist", RECD_STRING, "none", RECU_NULL, RR_NULL, RECC_NULL, nullptr, RECA_NULL} , {RECT_CONFIG, "proxy.config.http.insert_age_in_response", RECD_INT, "1", RECU_DYNAMIC, RR_NULL, RECC_INT, "[0-1]", RECA_NULL} , @@ -663,13 +668,13 @@ static const RecordElement RecordsConfig[] = , {RECT_CONFIG, "proxy.config.body_factory.enable_logging", RECD_INT, "0", RECU_RESTART_TS, RR_NULL, RECC_INT, "[0-1]", RECA_NULL} , - {RECT_CONFIG, "proxy.config.body_factory.template_sets_dir", RECD_STRING, "body_factory", RECU_DYNAMIC, RR_NULL, RECC_STR, "^[^[:space:]]+$", RECA_NULL} + {RECT_CONFIG, "proxy.config.body_factory.template_sets_dir", RECD_STRING, TS_BUILD_SYSCONFDIR "/body_factory", RECU_DYNAMIC, RR_NULL, RECC_STR, "^[^[:space:]]+$", RECA_NULL} , {RECT_CONFIG, "proxy.config.body_factory.response_max_size", RECD_INT, "8192", RECU_DYNAMIC, RR_NULL, RECC_NULL, nullptr, RECA_NULL} , //# 0 - never suppress generated responses //# 1 - always suppress generated responses - //# 2 - suppress responses for intercepted traffic + //# 2 - suppress responses for internal traffic {RECT_CONFIG, "proxy.config.body_factory.response_suppression_mode", RECD_INT, "0", RECU_RESTART_TS, RR_NULL, RECC_INT, "[0-2]", RECA_NULL} , {RECT_CONFIG, "proxy.config.body_factory.template_base", RECD_STRING, "NONE", RECU_DYNAMIC, RR_NULL, RECC_STR, ".*", RECA_NULL} @@ -683,7 +688,7 @@ static const RecordElement RecordsConfig[] = , {RECT_CONFIG, "proxy.config.socks.socks_version", RECD_INT, "4", RECU_RESTART_TS, RR_NULL, RECC_INT, "[4-5]", RECA_NULL} , - {RECT_CONFIG, "proxy.config.socks.socks_config_file", RECD_STRING, "socks.config", RECU_RESTART_TS, RR_NULL, RECC_NULL, nullptr, RECA_NULL} + {RECT_CONFIG, "proxy.config.socks.socks_config_file", RECD_STRING, ts::filename::SOCKS, RECU_RESTART_TS, RR_NULL, RECC_NULL, nullptr, RECA_NULL} , {RECT_CONFIG, "proxy.config.socks.socks_timeout", RECD_INT, "100", RECU_RESTART_TS, RR_NULL, RECC_STR, "^[0-9]+$", RECA_NULL} , @@ -713,7 +718,15 @@ static const RecordElement RecordsConfig[] = //# I/O Subsystem //# //############################################################################## - {RECT_CONFIG, "proxy.config.io.max_buffer_size", RECD_INT, "32768", RECU_NULL, RR_NULL, RECC_NULL, nullptr, RECA_NULL} + {RECT_CONFIG, "proxy.config.ssl.misc.io.max_buffer_index", RECD_INT, "8", RECU_NULL, RR_NULL, RECC_NULL, nullptr, RECA_NULL} + , + {RECT_CONFIG, "proxy.config.hostdb.io.max_buffer_index", RECD_INT, "8", RECU_NULL, RR_NULL, RECC_NULL, nullptr, RECA_NULL} + , + {RECT_CONFIG, "proxy.config.payload.io.max_buffer_index", RECD_INT, "8", RECU_NULL, RR_NULL, RECC_NULL, nullptr, RECA_NULL} + , + {RECT_CONFIG, "proxy.config.msg.io.max_buffer_index", RECD_INT, "8", RECU_NULL, RR_NULL, RECC_NULL, nullptr, RECA_NULL} + , + {RECT_CONFIG, "proxy.config.log.io.max_buffer_index", RECD_INT, "8", RECU_NULL, RR_NULL, RECC_NULL, nullptr, RECA_NULL} , //############################################################################## @@ -792,15 +805,13 @@ static const RecordElement RecordsConfig[] = //# Cache //# //############################################################################## - {RECT_CONFIG, "proxy.config.cache.storage_filename", RECD_STRING, "storage.config", RECU_RESTART_TS, RR_NULL, RECC_NULL, nullptr, RECA_NULL} - , - {RECT_CONFIG, "proxy.config.cache.control.filename", RECD_STRING, "cache.config", RECU_DYNAMIC, RR_NULL, RECC_NULL, nullptr, RECA_NULL} + {RECT_CONFIG, "proxy.config.cache.control.filename", RECD_STRING, ts::filename::CACHE, RECU_DYNAMIC, RR_NULL, RECC_NULL, nullptr, RECA_NULL} , - {RECT_CONFIG, "proxy.config.cache.ip_allow.filename", RECD_STRING, "ip_allow.yaml", RECU_DYNAMIC, RR_NULL, RECC_NULL, nullptr, RECA_NULL} + {RECT_CONFIG, "proxy.config.cache.ip_allow.filename", RECD_STRING, ts::filename::IP_ALLOW, RECU_DYNAMIC, RR_NULL, RECC_NULL, nullptr, RECA_NULL} , - {RECT_CONFIG, "proxy.config.cache.hosting_filename", RECD_STRING, "hosting.config", RECU_DYNAMIC, RR_NULL, RECC_NULL, nullptr, RECA_NULL} + {RECT_CONFIG, "proxy.config.cache.hosting_filename", RECD_STRING, ts::filename::HOSTING, RECU_DYNAMIC, RR_NULL, RECC_NULL, nullptr, RECA_NULL} , - {RECT_CONFIG, "proxy.config.cache.volume_filename", RECD_STRING, "volume.config", RECU_RESTART_TS, RR_NULL, RECC_NULL, nullptr, RECA_NULL} + {RECT_CONFIG, "proxy.config.cache.volume_filename", RECD_STRING, ts::filename::VOLUME, RECU_RESTART_TS, RR_NULL, RECC_NULL, nullptr, RECA_NULL} , {RECT_CONFIG, "proxy.config.cache.permit.pinning", RECD_INT, "0", RECU_DYNAMIC, RR_NULL, RECC_INT, "[0-1]", RECA_NULL} , @@ -877,7 +888,7 @@ static const RecordElement RecordsConfig[] = , {RECT_CONFIG, "proxy.config.dns.splitDNS.enabled", RECD_INT, "0", RECU_DYNAMIC, RR_NULL, RECC_INT, "[0-1]", RECA_NULL} , - {RECT_CONFIG, "proxy.config.dns.splitdns.filename", RECD_STRING, "splitdns.config", RECU_NULL, RR_NULL, RECC_NULL, nullptr, RECA_NULL} + {RECT_CONFIG, "proxy.config.dns.splitdns.filename", RECD_STRING, ts::filename::SPLITDNS, RECU_NULL, RR_NULL, RECC_NULL, nullptr, RECA_NULL} , {RECT_CONFIG, "proxy.config.dns.nameservers", RECD_STRING, nullptr, RECU_DYNAMIC, RR_NULL, RECC_NULL, nullptr, RECA_NULL} , @@ -998,7 +1009,7 @@ static const RecordElement RecordsConfig[] = , {RECT_CONFIG, "proxy.config.log.logfile_perm", RECD_STRING, "rw-r--r--", RECU_DYNAMIC, RR_NULL, RECC_NULL, nullptr, RECA_NULL} , - {RECT_CONFIG, "proxy.config.log.config.filename", RECD_STRING, "logging.yaml", RECU_DYNAMIC, RR_NULL, RECC_NULL, nullptr, RECA_NULL} + {RECT_CONFIG, "proxy.config.log.config.filename", RECD_STRING, ts::filename::LOGGING, RECU_DYNAMIC, RR_NULL, RECC_NULL, nullptr, RECA_NULL} , {RECT_CONFIG, "proxy.config.log.preproc_threads", RECD_INT, "1", RECU_DYNAMIC, RR_REQUIRED, RECC_INT, "[1-128]", RECA_NULL} , @@ -1039,12 +1050,16 @@ static const RecordElement RecordsConfig[] = //############################################################################## {RECT_CONFIG, "proxy.config.reverse_proxy.enabled", RECD_INT, "1", RECU_DYNAMIC, RR_REQUIRED, RECC_INT, "[0-1]", RECA_NULL} , - {RECT_CONFIG, "proxy.config.url_remap.filename", RECD_STRING, "remap.config", RECU_DYNAMIC, RR_NULL, RECC_NULL, nullptr, RECA_NULL} + {RECT_CONFIG, "proxy.config.url_remap.filename", RECD_STRING, ts::filename::REMAP, RECU_DYNAMIC, RR_NULL, RECC_NULL, nullptr, RECA_NULL} + , + {RECT_CONFIG, "proxy.config.url_remap.strategies.filename", RECD_STRING, "strategies.yaml", RECU_DYNAMIC, RR_NULL, RECC_NULL, nullptr, RECA_NULL} , {RECT_CONFIG, "proxy.config.url_remap.remap_required", RECD_INT, "1", RECU_DYNAMIC, RR_NULL, RECC_INT, "[0-1]", RECA_NULL} , {RECT_CONFIG, "proxy.config.url_remap.pristine_host_hdr", RECD_INT, "0", RECU_DYNAMIC, RR_NULL, RECC_INT, "[0-1]", RECA_NULL} , + {RECT_CONFIG, "proxy.config.plugin.dynamic_reload_mode", RECD_INT, "1", RECU_RESTART_TS, RR_NULL, RECC_INT, "[0-1]", RECA_NULL} + , //############################################################################## //# @@ -1053,6 +1068,8 @@ static const RecordElement RecordsConfig[] = //############################################################################## {RECT_CONFIG, "proxy.config.ssl.server.session_ticket.enable", RECD_INT, "1", RECU_DYNAMIC, RR_NULL, RECC_INT, "[0-1]", RECA_NULL} , + {RECT_CONFIG, "proxy.config.ssl.server.session_ticket.number", RECD_INT, "2", RECU_DYNAMIC, RR_NULL, RECC_NULL, nullptr, RECA_NULL}, + {RECT_CONFIG, "proxy.config.ssl.TLSv1", RECD_INT, "0", RECU_RESTART_TS, RR_NULL, RECC_INT, "[0-1]", RECA_NULL} , {RECT_CONFIG, "proxy.config.ssl.TLSv1_1", RECD_INT, "0", RECU_RESTART_TS, RR_NULL, RECC_INT, "[0-1]", RECA_NULL} @@ -1070,23 +1087,26 @@ static const RecordElement RecordsConfig[] = , {RECT_CONFIG, "proxy.config.ssl.client.TLSv1_3", RECD_INT, "1", RECU_RESTART_TS, RR_NULL, RECC_INT, "[0-1]", RECA_NULL} , - {RECT_CONFIG, "proxy.config.ssl.server.cipher_suite", RECD_STRING, "ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:DHE-DSS-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-DSS-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES128-SHA:DHE-RSA-AES256-SHA256:DHE-DSS-AES256-SHA256:DHE-RSA-AES128-SHA256:DHE-DSS-AES128-SHA256:DHE-RSA-AES256-SHA:DHE-DSS-AES256-SHA:DHE-RSA-AES128-SHA:DHE-DSS-AES128-SHA:!aNULL:!eNULL:!EXPORT:!DES:!RC4:!MD5:!PSK:!aECDH:!EDH-DSS-DES-CBC3-SHA:!EDH-RSA-DES-CBC3-SHA:!KRB5-DES-CBC3-SHA", RECU_RESTART_TS, RR_NULL, RECC_NULL, nullptr, RECA_NULL} + {RECT_CONFIG, "proxy.config.ssl.server.cipher_suite", RECD_STRING, "ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-CCM:ECDHE-ECDSA-AES128-CCM:ECDHE-ECDSA-AES256-CCM8:ECDHE-ECDSA-AES128-CCM8:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES256-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-RSA-AES256-SHA384:ECDHE-RSA-AES128-SHA256:DHE-RSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-CCM8:DHE-RSA-AES128-CCM8:DHE-RSA-AES256-CCM:DHE-RSA-AES128-CCM:DHE-RSA-AES256-SHA256:DHE-RSA-AES128-SHA256:AES256-GCM-SHA384:AES128-GCM-SHA256:AES256-CCM8:AES128-CCM8:AES256-CCM:AES128-CCM:AES256-SHA256:AES128-SHA2", RECU_RESTART_TS, RR_NULL, RECC_NULL, nullptr, RECA_NULL} , - {RECT_CONFIG, "proxy.config.ssl.client.cipher_suite", RECD_STRING, nullptr, RECU_RESTART_TS, RR_NULL, RECC_NULL, nullptr, RECA_NULL} + // Computed from openssl ciphers 'ALL:!aNULL:!aDH:!aECDH:!aPSK:!aSRP:!eNULL:!kSRP:!SSLv2:!SSLv3:!TLSv1:!TLSv1.1:@STRENGTH' + {RECT_CONFIG, "proxy.config.ssl.client.cipher_suite", RECD_STRING, "ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:DHE-DSS-AES256-GCM-SHA384:DHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-CCM8:ECDHE-ECDSA-AES256-CCM:DHE-RSA-AES256-CCM8:DHE-RSA-AES256-CCM:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-ARIA256-GCM-SHA384:ECDHE-ARIA256-GCM-SHA384:DHE-DSS-ARIA256-GCM-SHA384:DHE-RSA-ARIA256-GCM-SHA384:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:DHE-RSA-AES256-SHA256:DHE-DSS-AES256-SHA256:ECDHE-ECDSA-CAMELLIA256-SHA384:ECDHE-RSA-CAMELLIA256-SHA384:DHE-RSA-CAMELLIA256-SHA256:DHE-DSS-CAMELLIA256-SHA256:RSA-PSK-AES256-GCM-SHA384:RSA-PSK-CHACHA20-POLY1305:RSA-PSK-ARIA256-GCM-SHA384:AES256-GCM-SHA384:AES256-CCM8:AES256-CCM:ARIA256-GCM-SHA384:AES256-SHA256:CAMELLIA256-SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:DHE-DSS-AES128-GCM-SHA256:DHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-CCM8:ECDHE-ECDSA-AES128-CCM:DHE-RSA-AES128-CCM8:DHE-RSA-AES128-CCM:ECDHE-ECDSA-ARIA128-GCM-SHA256:ECDHE-ARIA128-GCM-SHA256:DHE-DSS-ARIA128-GCM-SHA256:DHE-RSA-ARIA128-GCM-SHA256:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA256:DHE-DSS-AES128-SHA256:ECDHE-ECDSA-CAMELLIA128-SHA256:ECDHE-RSA-CAMELLIA128-SHA256:DHE-RSA-CAMELLIA128-SHA256:DHE-DSS-CAMELLIA128-SHA256:RSA-PSK-AES128-GCM-SHA256:RSA-PSK-ARIA128-GCM-SHA256:AES128-GCM-SHA256:AES128-CCM8:AES128-CCM:ARIA128-GCM-SHA256:AES128-SHA256:CAMELLIA128-SHA256", RECU_RESTART_TS, RR_NULL, RECC_NULL, nullptr, RECA_NULL} , {RECT_CONFIG, "proxy.config.ssl.server.honor_cipher_order", RECD_INT, "1", RECU_RESTART_TS, RR_NULL, RECC_INT, "[0-1]", RECA_NULL} , + {RECT_CONFIG, "proxy.config.ssl.server.prioritize_chacha", RECD_INT, "0", RECU_RESTART_TS, RR_NULL, RECC_INT, "[0-1]", RECA_NULL} + , {RECT_CONFIG, "proxy.config.ssl.client.certification_level", RECD_INT, "0", RECU_RESTART_TS, RR_NULL, RECC_INT, "[0-2]", RECA_NULL} , {RECT_CONFIG, "proxy.config.ssl.server.cert.path", RECD_STRING, TS_BUILD_SYSCONFDIR, RECU_RESTART_TS, RR_NULL, RECC_NULL, nullptr, RECA_NULL} , {RECT_CONFIG, "proxy.config.ssl.server.cert_chain.filename", RECD_STRING, nullptr, RECU_RESTART_TS, RR_NULL, RECC_NULL, nullptr, RECA_NULL} , - {RECT_CONFIG, "proxy.config.ssl.server.multicert.filename", RECD_STRING, "ssl_multicert.config", RECU_RESTART_TS, RR_NULL, RECC_NULL, nullptr, RECA_NULL} + {RECT_CONFIG, "proxy.config.ssl.server.multicert.filename", RECD_STRING, ts::filename::SSL_MULTICERT, RECU_RESTART_TS, RR_NULL, RECC_NULL, nullptr, RECA_NULL} , {RECT_CONFIG, "proxy.config.ssl.server.multicert.exit_on_load_fail", RECD_INT, "1", RECU_RESTART_TS, RR_NULL, RECC_NULL, "[0-1]", RECA_NULL} , - {RECT_CONFIG, "proxy.config.ssl.servername.filename", RECD_STRING, "sni.yaml", RECU_RESTART_TS, RR_NULL, RECC_NULL, nullptr, RECA_NULL} + {RECT_CONFIG, "proxy.config.ssl.servername.filename", RECD_STRING, ts::filename::SNI, RECU_RESTART_TS, RR_NULL, RECC_NULL, nullptr, RECA_NULL} , {RECT_CONFIG, "proxy.config.ssl.server.ticket_key.filename", RECD_STRING, nullptr, RECU_DYNAMIC, RR_NULL, RECC_NULL, nullptr, RECA_NULL} , @@ -1096,9 +1116,7 @@ static const RecordElement RecordsConfig[] = , {RECT_CONFIG, "proxy.config.ssl.CA.cert.path", RECD_STRING, TS_BUILD_SYSCONFDIR, RECU_RESTART_TS, RR_NULL, RECC_NULL, nullptr, RECA_NULL} , - {RECT_CONFIG, "proxy.config.ssl.client.verify.server", RECD_INT, "0", RECU_DYNAMIC, RR_NULL, RECC_INT, "[0-2]", RECA_NULL} - , - {RECT_CONFIG, "proxy.config.ssl.client.verify.server.policy", RECD_STRING, "DISABLED", RECU_DYNAMIC, RR_NULL, RECC_NULL, nullptr, RECA_NULL} + {RECT_CONFIG, "proxy.config.ssl.client.verify.server.policy", RECD_STRING, "PERMISSIVE", RECU_DYNAMIC, RR_NULL, RECC_NULL, nullptr, RECA_NULL} , {RECT_CONFIG, "proxy.config.ssl.client.verify.server.properties", RECD_STRING, "ALL", RECU_DYNAMIC, RR_NULL, RECC_NULL, nullptr, RECA_NULL} , @@ -1138,7 +1156,7 @@ static const RecordElement RecordsConfig[] = , {RECT_CONFIG, "proxy.config.ssl.server.dhparams_file", RECD_STRING, nullptr, RECU_RESTART_TS, RR_NULL, RECC_NULL, nullptr, RECA_NULL} , - {RECT_CONFIG, "proxy.config.ssl.handshake_timeout_in", RECD_INT, "0", RECU_RESTART_TS, RR_NULL, RECC_INT, "[0-65535]", RECA_NULL} + {RECT_CONFIG, "proxy.config.ssl.handshake_timeout_in", RECD_INT, "30", RECU_RESTART_TS, RR_NULL, RECC_INT, "[0-65535]", RECA_NULL} , {RECT_CONFIG, "proxy.config.ssl.cert.load_elevated", RECD_INT, "0", RECU_RESTART_TS, RR_NULL, RECC_INT, "[0-1]", RECA_READ_ONLY} , @@ -1146,7 +1164,10 @@ static const RecordElement RecordsConfig[] = , {RECT_CONFIG, "proxy.config.ssl.client.groups_list", RECD_STRING, nullptr, RECU_RESTART_TS, RR_NULL, RECC_NULL, nullptr, RECA_NULL} , - + {RECT_CONFIG, "proxy.config.ssl.server.max_early_data", RECD_INT, "0", RECU_RESTART_TS, RR_NULL, RECC_NULL, nullptr, RECA_NULL} + , + {RECT_CONFIG, "proxy.config.ssl.server.allow_early_data_params", RECD_INT, "0", RECU_RESTART_TS, RR_NULL, RECC_NULL, nullptr, RECA_NULL} + , //############################################################################## //# //# OCSP (Online Certificate Status Protocol) Stapling Configuration @@ -1172,11 +1193,9 @@ static const RecordElement RecordsConfig[] = //# Configuration for TLSv1.3 and above //# //############################################################################## - // The default value (nullptr) means the default value of TLS stack will be used. - // - e.g. OpenSSL : "TLS_AES_256_GCM_SHA384:TLS_CHACHA20_POLY1305_SHA256:TLS_AES_128_GCM_SHA256" - {RECT_CONFIG, "proxy.config.ssl.server.TLSv1_3.cipher_suites", RECD_STRING, nullptr, RECU_RESTART_TS, RR_NULL, RECC_NULL, nullptr, RECA_NULL} + {RECT_CONFIG, "proxy.config.ssl.server.TLSv1_3.cipher_suites", RECD_STRING, "TLS_AES_256_GCM_SHA384:TLS_AES_128_GCM_SHA256:TLS_CHACHA20_POLY1305_SHA256", RECU_RESTART_TS, RR_NULL, RECC_NULL, nullptr, RECA_NULL} , - {RECT_CONFIG, "proxy.config.ssl.client.TLSv1_3.cipher_suites", RECD_STRING, nullptr, RECU_RESTART_TS, RR_NULL, RECC_NULL, nullptr, RECA_NULL} + {RECT_CONFIG, "proxy.config.ssl.client.TLSv1_3.cipher_suites", RECD_STRING, "TLS_AES_256_GCM_SHA384:TLS_AES_128_GCM_SHA256:TLS_CHACHA20_POLY1305_SHA256", RECU_RESTART_TS, RR_NULL, RECC_NULL, nullptr, RECA_NULL} , //############################################################################ @@ -1275,7 +1294,7 @@ static const RecordElement RecordsConfig[] = , {RECT_CONFIG, "proxy.config.http2.max_active_streams_in", RECD_INT, "0", RECU_DYNAMIC, RR_NULL, RECC_STR, "^[0-9]+$", RECA_NULL} , - {RECT_CONFIG, "proxy.config.http2.initial_window_size_in", RECD_INT, "1048576", RECU_DYNAMIC, RR_NULL, RECC_STR, "^[0-9]+$", RECA_NULL} + {RECT_CONFIG, "proxy.config.http2.initial_window_size_in", RECD_INT, "65535", RECU_DYNAMIC, RR_NULL, RECC_STR, "^[0-9]+$", RECA_NULL} , {RECT_CONFIG, "proxy.config.http2.max_frame_size", RECD_INT, "16384", RECU_DYNAMIC, RR_NULL, RECC_STR, "^[0-9]+$", RECA_NULL} , @@ -1307,6 +1326,8 @@ static const RecordElement RecordsConfig[] = , {RECT_CONFIG, "proxy.config.http2.min_avg_window_update", RECD_FLOAT, "2560.0", RECU_DYNAMIC, RR_NULL, RECC_NULL, nullptr, RECA_NULL} , + {RECT_CONFIG, "proxy.config.http2.header_table_size_limit", RECD_INT, "65536", RECU_DYNAMIC, RR_NULL, RECC_STR, "^[0-9]+$", RECA_NULL} + , //############ //# @@ -1342,6 +1363,10 @@ static const RecordElement RecordsConfig[] = , {RECT_CONFIG, "proxy.config.quic.client.cm_exercise_enabled", RECD_INT, "0", RECU_DYNAMIC, RR_NULL, RECC_INT, "[0-1]", RECA_NULL} , + {RECT_CONFIG, "proxy.config.quic.client.quantum_readiness_test_enabled", RECD_INT, "0", RECU_DYNAMIC, RR_NULL, RECC_INT, "[0-1]", RECA_NULL} + , + {RECT_CONFIG, "proxy.config.quic.server.quantum_readiness_test_enabled", RECD_INT, "0", RECU_DYNAMIC, RR_NULL, RECC_INT, "[0-1]", RECA_NULL} + , {RECT_CONFIG, "proxy.config.quic.server.supported_groups", RECD_STRING, "P-256:X25519:P-384:P-521" , RECU_RESTART_TS, RR_NULL, RECC_NULL, nullptr, RECA_NULL} , {RECT_CONFIG, "proxy.config.quic.client.supported_groups", RECD_STRING, "P-256:X25519:P-384:P-521" , RECU_RESTART_TS, RR_NULL, RECC_NULL, nullptr, RECA_NULL} @@ -1350,6 +1375,8 @@ static const RecordElement RecordsConfig[] = , {RECT_CONFIG, "proxy.config.quic.client.keylog_file", RECD_STRING, nullptr , RECU_RESTART_TS, RR_NULL, RECC_NULL, nullptr, RECA_NULL} , + {RECT_CONFIG, "proxy.config.quic.qlog_dir", RECD_STRING, nullptr , RECU_RESTART_TS, RR_NULL, RECC_NULL, nullptr, RECA_NULL} + , // Transport Parameters {RECT_CONFIG, "proxy.config.quic.no_activity_timeout_in", RECD_INT, "30000", RECU_DYNAMIC, RR_NULL, RECC_STR, "^-?[0-9]+$", RECA_NULL} , @@ -1373,7 +1400,7 @@ static const RecordElement RecordsConfig[] = , {RECT_CONFIG, "proxy.config.quic.initial_max_stream_data_uni_in", RECD_INT, "4096", RECU_DYNAMIC, RR_NULL, RECC_STR, "^-?[0-9]+$", RECA_NULL} , - {RECT_CONFIG, "proxy.config.quic.initial_max_stream_data_uni_out", RECD_INT, "0", RECU_DYNAMIC, RR_NULL, RECC_STR, "^-?[0-9]+$", RECA_NULL} + {RECT_CONFIG, "proxy.config.quic.initial_max_stream_data_uni_out", RECD_INT, "4096", RECU_DYNAMIC, RR_NULL, RECC_STR, "^-?[0-9]+$", RECA_NULL} , {RECT_CONFIG, "proxy.config.quic.initial_max_streams_bidi_in", RECD_INT, "100", RECU_DYNAMIC, RR_NULL, RECC_STR, "^-?[0-9]+$", RECA_NULL} , @@ -1391,6 +1418,12 @@ static const RecordElement RecordsConfig[] = , {RECT_CONFIG, "proxy.config.quic.max_ack_delay_out", RECD_INT, "25", RECU_DYNAMIC, RR_NULL, RECC_STR, "^-?[0-9]+$", RECA_NULL} , + {RECT_CONFIG, "proxy.config.quic.active_cid_limit_in", RECD_INT, "4", RECU_DYNAMIC, RR_NULL, RECC_STR, "^-?[0-9]+$", RECA_NULL} + , + {RECT_CONFIG, "proxy.config.quic.active_cid_limit_out", RECD_INT, "8", RECU_DYNAMIC, RR_NULL, RECC_STR, "^-?[0-9]+$", RECA_NULL} + , + {RECT_CONFIG, "proxy.config.quic.disable_active_migration", RECD_INT, "0", RECU_DYNAMIC, RR_NULL, RECC_STR, "[0-1]", RECA_NULL} + , // Constants of Loss Detection {RECT_CONFIG, "proxy.config.quic.loss_detection.packet_threshold", RECD_INT, "3", RECU_DYNAMIC, RR_NULL, RECC_STR, "^-?[0-9]+$", RECA_NULL} , @@ -1398,19 +1431,19 @@ static const RecordElement RecordsConfig[] = , {RECT_CONFIG, "proxy.config.quic.loss_detection.granularity", RECD_INT, "1", RECU_DYNAMIC, RR_NULL, RECC_STR, "[0-1]", RECA_NULL} , - {RECT_CONFIG, "proxy.config.quic.loss_detection.initial_rtt", RECD_INT, "100", RECU_DYNAMIC, RR_NULL, RECC_STR, "^-?[0-9]+$", RECA_NULL} + {RECT_CONFIG, "proxy.config.quic.loss_detection.initial_rtt", RECD_INT, "500", RECU_DYNAMIC, RR_NULL, RECC_STR, "^-?[0-9]+$", RECA_NULL} , // Constatns of Congestion Control {RECT_CONFIG, "proxy.config.quic.congestion_control.max_datagram_size", RECD_INT, "1200", RECU_DYNAMIC, RR_NULL, RECC_STR, "^-?[0-9]+$", RECA_NULL} , - {RECT_CONFIG, "proxy.config.quic.congestion_control.initial_window_scale", RECD_INT, "10", RECU_DYNAMIC, RR_NULL, RECC_STR, "^-?[0-9]+$", RECA_NULL} + {RECT_CONFIG, "proxy.config.quic.congestion_control.initial_window", RECD_INT, "12000", RECU_DYNAMIC, RR_NULL, RECC_STR, "^-?[0-9]+$", RECA_NULL} , - {RECT_CONFIG, "proxy.config.quic.congestion_control.minimum_window_scale", RECD_INT, "2", RECU_DYNAMIC, RR_NULL, RECC_STR, "^-?[0-9]+$", RECA_NULL} + {RECT_CONFIG, "proxy.config.quic.congestion_control.minimum_window", RECD_INT, "2400", RECU_DYNAMIC, RR_NULL, RECC_STR, "^-?[0-9]+$", RECA_NULL} , {RECT_CONFIG, "proxy.config.quic.congestion_control.loss_reduction_factor", RECD_FLOAT, "0.5", RECU_DYNAMIC, RR_NULL, RECC_STR, "^-?[\\.0-9]+$", RECA_NULL} , - {RECT_CONFIG, "proxy.config.quic.congestion_control.persistent_congestion_threshold", RECD_INT, "2", RECU_DYNAMIC, RR_NULL, RECC_STR, "^-?[\\.0-9]+$", RECA_NULL} + {RECT_CONFIG, "proxy.config.quic.congestion_control.persistent_congestion_threshold", RECD_INT, "3", RECU_DYNAMIC, RR_NULL, RECC_STR, "^-?[\\.0-9]+$", RECA_NULL} , //# Add LOCAL Records Here @@ -1438,24 +1471,17 @@ static const RecordElement RecordsConfig[] = {RECT_CONFIG, "proxy.config.allocator.dontdump_iobuffers", RECD_INT, "1", RECU_RESTART_TS, RR_NULL, RECC_NULL, "[0-1]", RECA_NULL} , - //############ - //# - //# Eric's super cool remap processor - //# - //############ - {RECT_CONFIG, "proxy.config.remap.num_remap_threads", RECD_INT, "0", RECU_NULL, RR_NULL, RECC_NULL, nullptr, RECA_NULL} - , + // Controls for TLS ASYN_JOBS and engine loading + {RECT_CONFIG, "proxy.config.ssl.async.handshake.enabled", RECD_INT, "0", RECU_RESTART_TS, RR_NULL, RECC_NULL, "[0-1]", RECA_NULL}, + {RECT_CONFIG, "proxy.config.ssl.engine.conf_file", RECD_STRING, nullptr, RECU_NULL, RR_NULL, RECC_NULL, nullptr, RECA_NULL}, //########### //# - //# Temporary and esoteric values. + //# Control how the host/sni name mismatches are handled + //# 0 - no checking. 1 - log in mismatch. 2 - enforcing //# //########### - {RECT_CONFIG, "proxy.config.cache.http.compatibility.4-2-0-fixup", RECD_INT, "1", RECU_DYNAMIC, RR_NULL, RECC_NULL, nullptr, RECA_NULL}, - - // Controls for TLS ASYN_JOBS and engine loading - {RECT_CONFIG, "proxy.config.ssl.async.handshake.enabled", RECD_INT, "0", RECU_RESTART_TS, RR_NULL, RECC_NULL, "[0-1]", RECA_NULL}, - {RECT_CONFIG, "proxy.config.ssl.engine.conf_file", RECD_STRING, nullptr, RECU_NULL, RR_NULL, RECC_NULL, nullptr, RECA_NULL}, + {RECT_CONFIG, "proxy.config.http.host_sni_policy", RECD_INT, "2", RECU_NULL, RR_NULL, RECC_NULL, "[0-2]", RECA_NULL}, }; // clang-format on diff --git a/mgmt/WebMgmtUtils.cc b/mgmt/WebMgmtUtils.cc index 383d7df7967..eb004e28c34 100644 --- a/mgmt/WebMgmtUtils.cc +++ b/mgmt/WebMgmtUtils.cc @@ -1065,9 +1065,8 @@ recordIPCheck(const char *pattern, const char *value) // regex_t regex; // int result; bool check; - const char *range_pattern = - R"(\[[0-9]+\-[0-9]+\]\\\.\[[0-9]+\-[0-9]+\]\\\.\[[0-9]+\-[0-9]+\]\\\.\[[0-9]+\-[0-9]+\])"; - const char *ip_pattern = "[0-9]*[0-9]*[0-9].[0-9]*[0-9]*[0-9].[0-9]*[0-9]*[0-9].[0-9]*[0-9]*[0-9]"; + const char *range_pattern = R"(\[[0-9]+\-[0-9]+\]\\\.\[[0-9]+\-[0-9]+\]\\\.\[[0-9]+\-[0-9]+\]\\\.\[[0-9]+\-[0-9]+\])"; + const char *ip_pattern = "[0-9]*[0-9]*[0-9].[0-9]*[0-9]*[0-9].[0-9]*[0-9]*[0-9].[0-9]*[0-9]*[0-9]"; Tokenizer dotTok1("."); Tokenizer dotTok2("."); diff --git a/mgmt/api/CoreAPI.cc b/mgmt/api/CoreAPI.cc index be5bce4c13f..c9b19e0d797 100644 --- a/mgmt/api/CoreAPI.cc +++ b/mgmt/api/CoreAPI.cc @@ -170,7 +170,8 @@ ProxyStateSet(TSProxyStateT state, TSCacheClearT clear) // Start with the default options from records.config. if (RecGetRecordString_Xmalloc("proxy.config.proxy_binary_opts", &proxy_options) == REC_ERR_OKAY) { - if (max_records_entries == (REC_INTERNAL_RECORDS + REC_MIN_API_RECORDS)) { // Default size, don't need to pass down to _server + if (max_records_entries == + (REC_INTERNAL_RECORDS + REC_DEFAULT_API_RECORDS)) { // Default size, don't need to pass down to _server snprintf(tsArgs, sizeof(tsArgs), "%s", proxy_options); } else { snprintf(tsArgs, sizeof(tsArgs), "%s --maxRecords %d", proxy_options, max_records_entries); diff --git a/mgmt/api/EventControlMain.cc b/mgmt/api/EventControlMain.cc index 94e6ea07fea..ff154cb687c 100644 --- a/mgmt/api/EventControlMain.cc +++ b/mgmt/api/EventControlMain.cc @@ -243,7 +243,7 @@ event_callback_main(void *arg) Debug("event", "[event_callback_main] listen on socket = %d", con_socket_fd); // initialize queue for holding mgmt events - if ((ret = init_mgmt_events()) != TS_ERR_OKAY) { + if (init_mgmt_events() != TS_ERR_OKAY) { return nullptr; } // register callback with alarms processor diff --git a/mgmt/api/INKMgmtAPI.cc b/mgmt/api/INKMgmtAPI.cc index 245c198280b..0c4c376f53b 100644 --- a/mgmt/api/INKMgmtAPI.cc +++ b/mgmt/api/INKMgmtAPI.cc @@ -688,7 +688,7 @@ TSTerminate() /*--- plugin initialization -----------------------------------------------*/ inkexp extern void -TSPluginInit(int /* argc ATS_UNUSED */, const char * /* argv ATS_UNUSED */ []) +TSPluginInit(int /* argc ATS_UNUSED */, const char * /* argv ATS_UNUSED */[]) { } diff --git a/mgmt/utils/ExpandingArray.cc b/mgmt/utils/ExpandingArray.cc index 85217356e64..0ec893f35d8 100644 --- a/mgmt/utils/ExpandingArray.cc +++ b/mgmt/utils/ExpandingArray.cc @@ -48,7 +48,8 @@ ExpandingArray::~ExpandingArray() ats_free(internalArray); } -void *ExpandingArray::operator[](int index) +void * +ExpandingArray::operator[](int index) { if (index < numValidValues) { return internalArray[index]; diff --git a/plugins/Makefile.am b/plugins/Makefile.am index 9818f9c00b3..4bf76783524 100644 --- a/plugins/Makefile.am +++ b/plugins/Makefile.am @@ -25,7 +25,7 @@ pkglib_LTLIBRARIES = SUBDIRS = -AM_LDFLAGS = $(TS_PLUGIN_LD_FLAGS) +AM_LDFLAGS += $(TS_PLUGIN_LD_FLAGS) include authproxy/Makefile.inc include background_fetch/Makefile.inc @@ -50,6 +50,7 @@ include regex_remap/Makefile.inc include regex_revalidate/Makefile.inc include remap_purge/Makefile.inc include s3_auth/Makefile.inc +include server_push_preload/Makefile.inc include stats_over_http/Makefile.inc include tcpinfo/Makefile.inc include xdebug/Makefile.inc @@ -67,13 +68,20 @@ include experimental/geoip_acl/Makefile.inc include experimental/header_freq/Makefile.inc include experimental/hook-trace/Makefile.inc include experimental/inliner/Makefile.inc + +if BUILD_MAXMIND_ACL_PLUGIN +include experimental/maxmind_acl/Makefile.inc +endif + include experimental/memcache/Makefile.inc +include experimental/memory_profile/Makefile.inc include experimental/metalink/Makefile.inc include experimental/money_trace/Makefile.inc include experimental/mp4/Makefile.inc -include experimental/server_push_preload/Makefile.inc +include experimental/remap_stats/Makefile.inc include experimental/slice/Makefile.inc include experimental/sslheaders/Makefile.inc +include experimental/statichit/Makefile.inc include experimental/stream_editor/Makefile.inc include experimental/system_stats/Makefile.inc include experimental/traffic_dump/Makefile.inc @@ -92,10 +100,6 @@ if BUILD_SSL_SESSION_REUSE_PLUGIN include experimental/ssl_session_reuse/Makefile.inc endif -if BUILD_REMAP_STATS_PLUGIN -include experimental/remap_stats/Makefile.inc -endif - if HAS_KYOTOCABINET include experimental/cache_key_genid/Makefile.inc endif diff --git a/plugins/authproxy/authproxy.cc b/plugins/authproxy/authproxy.cc index d8ad2cd888d..6e4f5388844 100644 --- a/plugins/authproxy/authproxy.cc +++ b/plugins/authproxy/authproxy.cc @@ -170,7 +170,7 @@ struct AuthRequestContext { { AuthOptions *opt; - opt = static_cast(TSHttpTxnArgGet(this->txn, AuthTaggedRequestArg)); + opt = static_cast(TSUserArgGet(this->txn, AuthTaggedRequestArg)); return opt ? opt : AuthGlobalOptions; } @@ -621,7 +621,7 @@ StateAuthorized(AuthRequestContext *auth, void *) static bool AuthRequestIsTagged(TSHttpTxn txn) { - return AuthTaggedRequestArg != -1 && TSHttpTxnArgGet(txn, AuthTaggedRequestArg) != nullptr; + return AuthTaggedRequestArg != -1 && TSUserArgGet(txn, AuthTaggedRequestArg) != nullptr; } static int @@ -736,7 +736,8 @@ TSPluginInit(int argc, const char *argv[]) AuthLogError("plugin registration failed"); } - TSReleaseAssert(TSHttpTxnArgIndexReserve("AuthProxy", "AuthProxy authorization tag", &AuthTaggedRequestArg) == TS_SUCCESS); + TSReleaseAssert(TSUserArgIndexReserve(TS_USER_ARGS_TXN, "AuthProxy", "AuthProxy authorization tag", &AuthTaggedRequestArg) == + TS_SUCCESS); AuthOsDnsContinuation = TSContCreate(AuthProxyGlobalHook, nullptr); AuthGlobalOptions = AuthParseOptions(argc, argv); @@ -749,7 +750,8 @@ TSPluginInit(int argc, const char *argv[]) TSReturnCode TSRemapInit(TSRemapInterface * /* api ATS_UNUSED */, char * /* err ATS_UNUSED */, int /* errsz ATS_UNUSED */) { - TSReleaseAssert(TSHttpTxnArgIndexReserve("AuthProxy", "AuthProxy authorization tag", &AuthTaggedRequestArg) == TS_SUCCESS); + TSReleaseAssert(TSUserArgIndexReserve(TS_USER_ARGS_TXN, "AuthProxy", "AuthProxy authorization tag", &AuthTaggedRequestArg) == + TS_SUCCESS); AuthOsDnsContinuation = TSContCreate(AuthProxyGlobalHook, nullptr); return TS_SUCCESS; @@ -785,7 +787,7 @@ TSRemapDoRemap(void *instance, TSHttpTxn txn, TSRemapRequestInfo * /* rri ATS_UN { AuthOptions *options = static_cast(instance); - TSHttpTxnArgSet(txn, AuthTaggedRequestArg, options); + TSUserArgSet(txn, AuthTaggedRequestArg, options); TSHttpTxnHookAdd(txn, TS_HTTP_POST_REMAP_HOOK, AuthOsDnsContinuation); return TSREMAP_NO_REMAP; diff --git a/plugins/authproxy/utils.h b/plugins/authproxy/utils.h index fae4b90acee..51cf01c387f 100644 --- a/plugins/authproxy/utils.h +++ b/plugins/authproxy/utils.h @@ -16,6 +16,8 @@ * limitations under the License. */ +#pragma once + #include #include #include diff --git a/plugins/background_fetch/background_fetch.cc b/plugins/background_fetch/background_fetch.cc index fbeba7dd318..13aaff4a15e 100644 --- a/plugins/background_fetch/background_fetch.cc +++ b/plugins/background_fetch/background_fetch.cc @@ -531,9 +531,9 @@ cont_handle_response(TSCont contp, TSEvent event, void *edata) TSDebug(PLUGIN_NAME, "Testing: response status code: %d?", status); if (TS_HTTP_STATUS_PARTIAL_CONTENT == status || (config->allow304() && TS_HTTP_STATUS_NOT_MODIFIED == status)) { // Everything looks good so far, add a TXN hook for SEND_RESPONSE_HDR - TSCont contp = TSContCreate(cont_check_cacheable, nullptr); + TSCont localcontp = TSContCreate(cont_check_cacheable, nullptr); - TSHttpTxnHookAdd(txnp, TS_HTTP_SEND_RESPONSE_HDR_HOOK, contp); + TSHttpTxnHookAdd(txnp, TS_HTTP_SEND_RESPONSE_HDR_HOOK, localcontp); } // Release the response MLoc TSHandleMLocRelease(response, TS_NULL_MLOC, resp_hdr); diff --git a/plugins/background_fetch/configs.h b/plugins/background_fetch/configs.h index ba6dd35fb5c..0517ffad314 100644 --- a/plugins/background_fetch/configs.h +++ b/plugins/background_fetch/configs.h @@ -34,7 +34,7 @@ const char PLUGIN_NAME[] = "background_fetch"; /////////////////////////////////////////////////////////////////////////// -// This holds one complete background fetch rule, which is also ref-counted. +// This holds one complete background fetch rule // class BgFetchConfig { diff --git a/plugins/cache_promote/Makefile.inc b/plugins/cache_promote/Makefile.inc index a0029d37e50..7c330c361e6 100644 --- a/plugins/cache_promote/Makefile.inc +++ b/plugins/cache_promote/Makefile.inc @@ -17,4 +17,8 @@ pkglib_LTLIBRARIES += cache_promote/cache_promote.la cache_promote_cache_promote_la_SOURCES = \ - cache_promote/cache_promote.cc + cache_promote/cache_promote.cc \ + cache_promote/configs.cc \ + cache_promote/policy.cc \ + cache_promote/lru_policy.cc \ + cache_promote/policy_manager.cc diff --git a/plugins/cache_promote/cache_promote.cc b/plugins/cache_promote/cache_promote.cc index 2924b375783..79372917da3 100644 --- a/plugins/cache_promote/cache_promote.cc +++ b/plugins/cache_promote/cache_promote.cc @@ -15,396 +15,22 @@ See the License for the specific language governing permissions and limitations under the License. */ - #include #include -#include -#include -#include -#include -#include - -#include -#include -#include #include "ts/ts.h" #include "ts/remap.h" -#include "tscore/ink_config.h" - -#define MINIMUM_BUCKET_SIZE 10 - -static const char *PLUGIN_NAME = "cache_promote"; - -////////////////////////////////////////////////////////////////////////////////////////////// -// Note that all options for all policies has to go here. Not particularly pretty... -// -static const struct option longopt[] = { - {const_cast("policy"), required_argument, nullptr, 'p'}, - // This is for both Chance and LRU (optional) policy - {const_cast("sample"), required_argument, nullptr, 's'}, - // For the LRU policy - {const_cast("buckets"), required_argument, nullptr, 'b'}, - {const_cast("hits"), required_argument, nullptr, 'h'}, - // EOF - {nullptr, no_argument, nullptr, '\0'}, -}; - -////////////////////////////////////////////////////////////////////////////////////////////// -// Abstract base class for all policies. -// -class PromotionPolicy -{ -public: - PromotionPolicy() - { - // This doesn't have to be perfect, since this is just chance sampling. - // coverity[dont_call] - srand48(static_cast(time(nullptr))); - } - - void - setSample(char *s) - { - _sample = strtof(s, nullptr) / 100.0; - } - - float - getSample() const - { - return _sample; - } - - bool - doSample() const - { - if (_sample > 0) { - // coverity[dont_call] - double r = drand48(); - - if (_sample > r) { - TSDebug(PLUGIN_NAME, "checking sampling, is %f > %f? Yes!", _sample, r); - } else { - TSDebug(PLUGIN_NAME, "checking sampling, is %f > %f? No!", _sample, r); - return false; - } - } - return true; - } - - virtual ~PromotionPolicy() = default; - ; - - virtual bool - parseOption(int opt, char *optarg) - { - return false; - } - - // These are pure virtual - virtual bool doPromote(TSHttpTxn txnp) = 0; - virtual const char *policyName() const = 0; - virtual void usage() const = 0; - -private: - float _sample = 0.0; -}; - -////////////////////////////////////////////////////////////////////////////////////////////// -// This is the simplest of all policies, just give each request a (small) -// percentage chance to be promoted to cache. -// -class ChancePolicy : public PromotionPolicy -{ -public: - bool doPromote(TSHttpTxn /* txnp ATS_UNUSED */) override - { - TSDebug(PLUGIN_NAME, "ChancePolicy::doPromote(%f)", getSample()); - return true; - } - - void - usage() const override - { - TSError("[%s] Usage: @plugin=%s.so @pparam=--policy=chance @pparam=--sample=%%", PLUGIN_NAME, PLUGIN_NAME); - } - - const char * - policyName() const override - { - return "chance"; - } -}; - -////////////////////////////////////////////////////////////////////////////////////////////// -// The LRU based policy keeps track of number of URLs, with a counter for each slot. -// Objects are not promoted unless the counter reaches before it gets evicted. An -// optional parameter can be used to sample hits, this can reduce contention and -// churning in the LRU as well. -// -class LRUHash -{ - friend struct LRUHashHasher; - -public: - LRUHash() { TSDebug(PLUGIN_NAME, "In LRUHash()"); } - ~LRUHash() { TSDebug(PLUGIN_NAME, "In ~LRUHash()"); } - LRUHash & - operator=(const LRUHash &h) - { - TSDebug(PLUGIN_NAME, "copying an LRUHash object"); - if (this != &h) { - memcpy(_hash, h._hash, sizeof(_hash)); - } - return *this; - } - - void - init(char *data, int len) - { - SHA_CTX sha; - - SHA1_Init(&sha); - SHA1_Update(&sha, data, len); - SHA1_Final(_hash, &sha); - } - -private: - u_char _hash[SHA_DIGEST_LENGTH]; -}; - -struct LRUHashHasher { - bool - operator()(const LRUHash *s1, const LRUHash *s2) const - { - return 0 == memcmp(s1->_hash, s2->_hash, sizeof(s2->_hash)); - } - - size_t - operator()(const LRUHash *s) const - { - return *(reinterpret_cast(s->_hash)) ^ *(reinterpret_cast(s->_hash + 9)); - } -}; - -typedef std::pair LRUEntry; -using LRUList = std::list; -typedef std::unordered_map LRUMap; - -static LRUEntry NULL_LRU_ENTRY; // Used to create an "empty" new LRUEntry - -class LRUPolicy : public PromotionPolicy -{ -public: - LRUPolicy() : PromotionPolicy(), _lock(TSMutexCreate()) {} - ~LRUPolicy() override - { - TSDebug(PLUGIN_NAME, "deleting LRUPolicy object"); - TSMutexLock(_lock); - _map.clear(); - _list.clear(); - _list_size = 0; - _freelist.clear(); - _freelist_size = 0; +#include "policy_manager.h" +#include "configs.h" - TSMutexUnlock(_lock); - TSMutexDestroy(_lock); - } - - bool - parseOption(int opt, char *optarg) override - { - switch (opt) { - case 'b': - _buckets = static_cast(strtol(optarg, nullptr, 10)); - if (_buckets < MINIMUM_BUCKET_SIZE) { - TSError("%s: Enforcing minimum LRU bucket size of %d", PLUGIN_NAME, MINIMUM_BUCKET_SIZE); - TSDebug(PLUGIN_NAME, "Enforcing minimum bucket size of %d", MINIMUM_BUCKET_SIZE); - _buckets = MINIMUM_BUCKET_SIZE; - } - break; - case 'h': - _hits = static_cast(strtol(optarg, nullptr, 10)); - break; - default: - // All other options are unsupported for this policy - return false; - } - - // This doesn't have to be perfect, since this is just chance sampling. - // coverity[dont_call] - srand48(static_cast(time(nullptr)) ^ static_cast(getpid()) ^ static_cast(getppid())); - - return true; - } - - bool - doPromote(TSHttpTxn txnp) override - { - LRUHash hash; - LRUMap::iterator map_it; - char *url = nullptr; - int url_len = 0; - bool ret = false; - TSMBuffer request; - TSMLoc req_hdr; - - if (TS_SUCCESS == TSHttpTxnClientReqGet(txnp, &request, &req_hdr)) { - TSMLoc c_url = TS_NULL_MLOC; - - // Get the cache key URL (for now), since this has better lookup behavior when using - // e.g. the cachekey plugin. - if (TS_SUCCESS == TSUrlCreate(request, &c_url)) { - if (TS_SUCCESS == TSHttpTxnCacheLookupUrlGet(txnp, request, c_url)) { - url = TSUrlStringGet(request, c_url, &url_len); - TSHandleMLocRelease(request, TS_NULL_MLOC, c_url); - } - } - TSHandleMLocRelease(request, TS_NULL_MLOC, req_hdr); - } - - // Generally shouldn't happen ... - if (!url) { - return false; - } - - TSDebug(PLUGIN_NAME, "LRUPolicy::doPromote(%.*s%s)", url_len > 100 ? 100 : url_len, url, url_len > 100 ? "..." : ""); - hash.init(url, url_len); - TSfree(url); - - // We have to hold the lock across all list and hash access / updates - TSMutexLock(_lock); - - map_it = _map.find(&hash); - if (_map.end() != map_it) { - // We have an entry in the LRU - TSAssert(_list_size > 0); // mismatch in the LRUs hash and list, shouldn't happen - if (++(map_it->second->second) >= _hits) { - // Promoted! Cleanup the LRU, and signal success. Save the promoted entry on the freelist. - TSDebug(PLUGIN_NAME, "saving the LRUEntry to the freelist"); - _freelist.splice(_freelist.begin(), _list, map_it->second); - ++_freelist_size; - --_list_size; - _map.erase(map_it->first); - ret = true; - } else { - // It's still not promoted, make sure it's moved to the front of the list - TSDebug(PLUGIN_NAME, "still not promoted, got %d hits so far", map_it->second->second); - _list.splice(_list.begin(), _list, map_it->second); - } - } else { - // New LRU entry for the URL, try to repurpose the list entry as much as possible - if (_list_size >= _buckets) { - TSDebug(PLUGIN_NAME, "repurposing last LRUHash entry"); - _list.splice(_list.begin(), _list, --_list.end()); - _map.erase(&(_list.begin()->first)); - } else if (_freelist_size > 0) { - TSDebug(PLUGIN_NAME, "reusing LRUEntry from freelist"); - _list.splice(_list.begin(), _freelist, _freelist.begin()); - --_freelist_size; - ++_list_size; - } else { - TSDebug(PLUGIN_NAME, "creating new LRUEntry"); - _list.push_front(NULL_LRU_ENTRY); - ++_list_size; - } - // Update the "new" LRUEntry and add it to the hash - _list.begin()->first = hash; - _list.begin()->second = 1; - _map[&(_list.begin()->first)] = _list.begin(); - } - - TSMutexUnlock(_lock); - - return ret; - } - - void - usage() const override - { - TSError("[%s] Usage: @plugin=%s.so @pparam=--policy=lru @pparam=--buckets= --hits= --sample=", PLUGIN_NAME, - PLUGIN_NAME); - } +const char *PLUGIN_NAME = "cache_promote"; - const char * - policyName() const override - { - return "LRU"; - } - -private: - unsigned _buckets = 1000; - unsigned _hits = 10; - // For the LRU. Note that we keep track of the List sizes, because some versions fo STL have broken - // implementations of size(), making them obsessively slow on calling ::size(). - TSMutex _lock; - LRUMap _map; - LRUList _list, _freelist; - size_t _list_size = 0, _freelist_size = 0; -}; - -////////////////////////////////////////////////////////////////////////////////////////////// -// This holds the configuration for a remap rule, as well as parses the configurations. -// -class PromotionConfig -{ -public: - PromotionConfig() = default; - ~PromotionConfig() { delete _policy; } - PromotionPolicy * - getPolicy() const - { - return _policy; - } - - // Parse the command line arguments to the plugin, and instantiate the appropriate policy - bool - factory(int argc, char *argv[]) - { - while (true) { - int opt = getopt_long(argc, (char *const *)argv, "psbh", longopt, nullptr); - - if (opt == -1) { - break; - } else if (opt == 'p') { - if (0 == strncasecmp(optarg, "chance", 6)) { - _policy = new ChancePolicy(); - } else if (0 == strncasecmp(optarg, "lru", 3)) { - _policy = new LRUPolicy(); - } else { - TSError("[%s] Unknown policy --policy=%s", PLUGIN_NAME, optarg); - return false; - } - if (_policy) { - TSDebug(PLUGIN_NAME, "created remap with cache promotion policy = %s", _policy->policyName()); - } - } else { - if (_policy) { - // The --sample (-s) option is allowed for all configs, but only after --policy is specified. - if (opt == 's') { - _policy->setSample(optarg); - } else { - if (!_policy->parseOption(opt, optarg)) { - TSError("[%s] The specified policy (%s) does not support the -%c option", PLUGIN_NAME, _policy->policyName(), opt); - delete _policy; - _policy = nullptr; - return false; - } - } - } else { - TSError("[%s] The --policy= parameter must come first on the remap configuration", PLUGIN_NAME); - return false; - } - } - } - - return true; - } - -private: - PromotionPolicy *_policy = nullptr; -}; +// This has to be a global here. I tried doing a classic singleton (with a getInstance()) in the PolicyManager, +// but then reloading the DSO does not work. What happens is that the old singleton is stil there, even though +// the rest of the plugin is reloaded. Very scary, and not what we need / want; if the plugin reloads, the +// PolicyManager has to reload (and start fresh) as well. +static PolicyManager gManager; ////////////////////////////////////////////////////////////////////////////////////////////// // Main "plugin", a TXN hook in the TS_HTTP_READ_CACHE_HDR_HOOK. Unless the policy allows @@ -440,17 +66,25 @@ cont_handle_policy(TSCont contp, TSEvent event, void *edata) default: // Do nothing, just let it handle the lookup. TSDebug(PLUGIN_NAME, "cache-status is %d (hit), nothing to do", obj_status); + + if (config->getPolicy()->stats_enabled) { + TSStatIntIncrement(config->getPolicy()->cache_hits_id, 1); + } break; } } + + if (config->getPolicy()->stats_enabled) { + TSStatIntIncrement(config->getPolicy()->total_requests_id, 1); + } } else { - TSDebug(PLUGIN_NAME, "Request is an internal (plugin) request, implicitly promoted"); + TSDebug(PLUGIN_NAME, "request is an internal (plugin) request, implicitly promoted"); } break; // Should not happen default: - TSDebug(PLUGIN_NAME, "Unhandled event %d", static_cast(event)); + TSDebug(PLUGIN_NAME, "unhandled event %d", static_cast(event)); break; } @@ -480,10 +114,17 @@ TSRemapInit(TSRemapInterface *api_info, char *errbuf, int errbuf_size) return TS_SUCCESS; /* success */ } +void +TSRemapDone() +{ + TSDebug(PLUGIN_NAME, "called TSRemapDone()"); + gManager.clear(); +} + TSReturnCode TSRemapNewInstance(int argc, char *argv[], void **ih, char * /* errbuf */, int /* errbuf_size */) { - PromotionConfig *config = new PromotionConfig; + PromotionConfig *config = new PromotionConfig(&gManager); --argc; ++argv; @@ -506,7 +147,7 @@ TSRemapDeleteInstance(void *ih) TSCont contp = static_cast(ih); PromotionConfig *config = static_cast(TSContDataGet(contp)); - delete config; + delete config; // This will return the PromotionPolicy to the PromotionManager as well TSContDestroy(contp); } @@ -517,7 +158,7 @@ TSRemapStatus TSRemapDoRemap(void *ih, TSHttpTxn rh, TSRemapRequestInfo * /* ATS_UNUSED rri */) { if (nullptr == ih) { - TSDebug(PLUGIN_NAME, "No promotion rules configured, this is probably a plugin bug"); + TSDebug(PLUGIN_NAME, "no promotion rules configured, this is probably a plugin bug"); } else { TSCont contp = static_cast(ih); diff --git a/plugins/cache_promote/chance_policy.h b/plugins/cache_promote/chance_policy.h new file mode 100644 index 00000000000..649c5e7a238 --- /dev/null +++ b/plugins/cache_promote/chance_policy.h @@ -0,0 +1,75 @@ +/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ +#pragma once + +#include "policy.h" + +////////////////////////////////////////////////////////////////////////////////////////////// +// This is the simplest of all policies, just give each request a (small) +// percentage chance to be promoted to cache. +// +class ChancePolicy : public PromotionPolicy +{ +public: + bool doPromote(TSHttpTxn /* txnp ATS_UNUSED */) override + { + TSDebug(PLUGIN_NAME, "ChancePolicy::doPromote(%f)", getSample()); + incrementStat(promoted_id, 1); + + return true; + } + + void + usage() const override + { + TSError("[%s] Usage: @plugin=%s.so @pparam=--policy=chance @pparam=--sample=%%", PLUGIN_NAME, PLUGIN_NAME); + } + + const char * + policyName() const override + { + return "chance"; + } + + bool + stats_add(const char *remap_id) override + { + std::string_view remap_identifier = remap_id; + const std::tuple stats[] = { + {"cache_hits", &cache_hits_id}, + {"promoted", &promoted_id}, + {"total_requests", &total_requests_id}, + }; + + if (nullptr == remap_id) { + TSError("[%s] no remap identifier specified for for stats, no stats will be used", PLUGIN_NAME); + return false; + } + + for (int ii = 0; ii < 3; ii++) { + std::string_view name = std::get<0>(stats[ii]); + int *id = std::get<1>(stats[ii]); + + if ((*(id) = create_stat(name, remap_identifier)) == TS_ERROR) { + return false; + } + } + + return true; + } +}; diff --git a/plugins/cache_promote/configs.cc b/plugins/cache_promote/configs.cc new file mode 100644 index 00000000000..82e13029bba --- /dev/null +++ b/plugins/cache_promote/configs.cc @@ -0,0 +1,104 @@ +/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ +#include + +#include "configs.h" +#include "lru_policy.h" +#include "chance_policy.h" + +////////////////////////////////////////////////////////////////////////////////////////////// +// ToDo: It's ugly that this is a "global" options list, clearly each policy should be able +// to add to this list, making them more modular. +static const struct option longopt[] = { + {const_cast("policy"), required_argument, nullptr, 'p'}, + // This is for both Chance and LRU (optional) policy + {const_cast("sample"), required_argument, nullptr, 's'}, + // For the LRU policy + {const_cast("buckets"), required_argument, nullptr, 'b'}, + {const_cast("hits"), required_argument, nullptr, 'h'}, + {const_cast("stats-enable-with-id"), required_argument, nullptr, 'e'}, + {const_cast("label"), required_argument, nullptr, 'l'}, + // EOF + {nullptr, no_argument, nullptr, '\0'}, +}; + +// The destructor is responsible for returning the policy to the PolicyManager. +PromotionConfig::~PromotionConfig() +{ + _manager->releasePolicy(_policy); +} + +// Parse the command line arguments to the plugin, and instantiate the appropriate policy +bool +PromotionConfig::factory(int argc, char *argv[]) +{ + while (true) { + int opt = getopt_long(argc, (char *const *)argv, "", longopt, nullptr); + + if (opt == -1) { + break; + } else if (opt == 'p') { + if (0 == strncasecmp(optarg, "chance", 6)) { + _policy = new ChancePolicy(); + } else if (0 == strncasecmp(optarg, "lru", 3)) { + _policy = new LRUPolicy(); + } else { + TSError("[%s] Unknown policy --policy=%s", PLUGIN_NAME, optarg); + return false; + } + if (_policy) { + TSDebug(PLUGIN_NAME, "created remap with cache promotion policy = %s", _policy->policyName()); + } + } else if (opt == 'e') { + if (optarg == nullptr) { + TSError("[%s] the -%c option requires an argument, the remap identifier.", PLUGIN_NAME, opt); + return false; + } else { + if (_policy && _policy->stats_add(optarg)) { + _policy->stats_enabled = true; + TSDebug(PLUGIN_NAME, "stats collection is enabled"); + } + } + } else { + if (_policy) { + // The --sample (-s) option is allowed for all configs, but only after --policy is specified. + if (opt == 's') { + _policy->setSample(optarg); + } else { + if (!_policy->parseOption(opt, optarg)) { + TSError("[%s] The specified policy (%s) does not support the -%c option", PLUGIN_NAME, _policy->policyName(), opt); + delete _policy; + _policy = nullptr; + return false; + } + } + } else { + TSError("[%s] The --policy= parameter must come first on the remap configuration", PLUGIN_NAME); + return false; + } + } + } + + // Coalesce any LRU policies via the LRU manager. This is a little ugly, but it makes configuration + // easier, and order of options doesn't matter. + + // This can return the same policy, or an existing one, in which case, this one is deleted by the Manager + _policy = _manager->coalescePolicy(_policy); + + return true; +} diff --git a/plugins/cache_promote/configs.h b/plugins/cache_promote/configs.h new file mode 100644 index 00000000000..ce4e973f8e3 --- /dev/null +++ b/plugins/cache_promote/configs.h @@ -0,0 +1,44 @@ +/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ +#pragma once + +#include "policy.h" +#include "policy_manager.h" + +////////////////////////////////////////////////////////////////////////////////////////////// +// This holds the configuration for a remap rule, as well as parses the configurations. +// +class PromotionConfig +{ +public: + PromotionConfig(PolicyManager *mgr) : _manager(mgr){}; + + virtual ~PromotionConfig(); + + PromotionPolicy * + getPolicy() const + { + return _policy; + } + + bool factory(int argc, char *argv[]); + +private: + PromotionPolicy *_policy = nullptr; + PolicyManager *_manager; +}; diff --git a/plugins/cache_promote/lru_policy.cc b/plugins/cache_promote/lru_policy.cc new file mode 100644 index 00000000000..07eeef8b1f0 --- /dev/null +++ b/plugins/cache_promote/lru_policy.cc @@ -0,0 +1,186 @@ +/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ +#include + +#include "lru_policy.h" + +#define MINIMUM_BUCKET_SIZE 10 +static LRUEntry NULL_LRU_ENTRY; // Used to create an "empty" new LRUEntry + +LRUPolicy::~LRUPolicy() +{ + TSDebug(PLUGIN_NAME, "LRUPolicy DTOR"); + TSMutexLock(_lock); + + _map.clear(); + _list.clear(); + _list_size = 0; + _freelist.clear(); + _freelist_size = 0; + + TSMutexUnlock(_lock); + TSMutexDestroy(_lock); +} + +bool +LRUPolicy::parseOption(int opt, char *optarg) +{ + switch (opt) { + case 'b': + _buckets = static_cast(strtol(optarg, nullptr, 10)); + if (_buckets < MINIMUM_BUCKET_SIZE) { + TSError("%s: Enforcing minimum LRU bucket size of %d", PLUGIN_NAME, MINIMUM_BUCKET_SIZE); + TSDebug(PLUGIN_NAME, "enforcing minimum bucket size of %d", MINIMUM_BUCKET_SIZE); + _buckets = MINIMUM_BUCKET_SIZE; + } + break; + case 'h': + _hits = static_cast(strtol(optarg, nullptr, 10)); + break; + case 'l': + _label = optarg; + break; + default: + // All other options are unsupported for this policy + return false; + } + + // This doesn't have to be perfect, since this is just chance sampling. + // coverity[dont_call] + srand48(static_cast(time(nullptr)) ^ static_cast(getpid()) ^ static_cast(getppid())); + + return true; +} + +bool +LRUPolicy::doPromote(TSHttpTxn txnp) +{ + LRUHash hash; + LRUMap::iterator map_it; + char *url = nullptr; + int url_len = 0; + bool ret = false; + TSMBuffer request; + TSMLoc req_hdr; + + if (TS_SUCCESS == TSHttpTxnClientReqGet(txnp, &request, &req_hdr)) { + TSMLoc c_url = TS_NULL_MLOC; + + // Get the cache key URL (for now), since this has better lookup behavior when using + // e.g. the cachekey plugin. + if (TS_SUCCESS == TSUrlCreate(request, &c_url)) { + if (TS_SUCCESS == TSHttpTxnCacheLookupUrlGet(txnp, request, c_url)) { + url = TSUrlStringGet(request, c_url, &url_len); + TSHandleMLocRelease(request, TS_NULL_MLOC, c_url); + } + } + TSHandleMLocRelease(request, TS_NULL_MLOC, req_hdr); + } + + // Generally shouldn't happen ... + if (!url) { + return false; + } + + TSDebug(PLUGIN_NAME, "LRUPolicy::doPromote(%.*s%s)", url_len > 100 ? 100 : url_len, url, url_len > 100 ? "..." : ""); + hash.init(url, url_len); + TSfree(url); + + // We have to hold the lock across all list and hash access / updates + TSMutexLock(_lock); + + map_it = _map.find(&hash); + if (_map.end() != map_it) { + // We have an entry in the LRU + TSAssert(_list_size > 0); // mismatch in the LRUs hash and list, shouldn't happen + incrementStat(lru_hit_id, 1); + if (++(map_it->second->second) >= _hits) { + // Promoted! Cleanup the LRU, and signal success. Save the promoted entry on the freelist. + TSDebug(PLUGIN_NAME, "saving the LRUEntry to the freelist"); + _freelist.splice(_freelist.begin(), _list, map_it->second); + ++_freelist_size; + --_list_size; + _map.erase(map_it->first); + incrementStat(promoted_id, 1); + incrementStat(freelist_size_id, 1); + decrementStat(lru_size_id, 1); + ret = true; + } else { + // It's still not promoted, make sure it's moved to the front of the list + TSDebug(PLUGIN_NAME, "still not promoted, got %d hits so far", map_it->second->second); + _list.splice(_list.begin(), _list, map_it->second); + } + } else { + // New LRU entry for the URL, try to repurpose the list entry as much as possible + incrementStat(lru_miss_id, 1); + if (_list_size >= _buckets) { + TSDebug(PLUGIN_NAME, "repurposing last LRUHash entry"); + _list.splice(_list.begin(), _list, --_list.end()); + _map.erase(&(_list.begin()->first)); + incrementStat(lru_vacated_id, 1); + } else if (_freelist_size > 0) { + TSDebug(PLUGIN_NAME, "reusing LRUEntry from freelist"); + _list.splice(_list.begin(), _freelist, _freelist.begin()); + --_freelist_size; + ++_list_size; + incrementStat(lru_size_id, 1); + decrementStat(freelist_size_id, 1); + } else { + TSDebug(PLUGIN_NAME, "creating new LRUEntry"); + _list.push_front(NULL_LRU_ENTRY); + ++_list_size; + incrementStat(lru_size_id, 1); + } + // Update the "new" LRUEntry and add it to the hash + _list.begin()->first = hash; + _list.begin()->second = 1; + _map[&(_list.begin()->first)] = _list.begin(); + } + + TSMutexUnlock(_lock); + + return ret; +} + +bool +LRUPolicy::stats_add(const char *remap_id) + +{ + std::string_view remap_identifier = remap_id; + const std::tuple stats[] = { + {"cache_hits", &cache_hits_id}, {"freelist_size", &freelist_size_id}, + {"lru_size", &lru_size_id}, {"lru_hit", &lru_hit_id}, + {"lru_miss", &lru_miss_id}, {"lru_vacated", &lru_vacated_id}, + {"promoted", &promoted_id}, {"total_requests", &total_requests_id}, + }; + + if (nullptr == remap_id) { + TSError("[%s] no remap identifier specified for for stats, no stats will be used", PLUGIN_NAME); + return false; + } + + for (const auto &stat : stats) { + std::string_view name = std::get<0>(stat); + int *id = std::get<1>(stat); + if ((*(id) = create_stat(name, remap_identifier)) == TS_ERROR) { + return false; + } + } + + return true; +} diff --git a/plugins/cache_promote/lru_policy.h b/plugins/cache_promote/lru_policy.h new file mode 100644 index 00000000000..390a9497b93 --- /dev/null +++ b/plugins/cache_promote/lru_policy.h @@ -0,0 +1,132 @@ +/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ +#pragma once + +#include +#include +#include +#include + +#include "policy.h" + +#define MINIMUM_BUCKET_SIZE 10 + +////////////////////////////////////////////////////////////////////////////////////////////// +// The LRU based policy keeps track of number of URLs, with a counter for each slot. +// Objects are not promoted unless the counter reaches before it gets evicted. An +// optional parameter can be used to sample hits, this can reduce contention and +// churning in the LRU as well. +// +class LRUHash +{ + friend struct LRUHashHasher; + +public: + LRUHash() { TSDebug(PLUGIN_NAME, "LRUHash() CTOR"); } + ~LRUHash() { TSDebug(PLUGIN_NAME, "~LRUHash() DTOR"); } + + LRUHash & + operator=(const LRUHash &h) + { + TSDebug(PLUGIN_NAME, "copying an LRUHash object"); + if (this != &h) { + memcpy(_hash, h._hash, sizeof(_hash)); + } + return *this; + } + + void + init(char *data, int len) + { + SHA_CTX sha; + + SHA1_Init(&sha); + SHA1_Update(&sha, data, len); + SHA1_Final(_hash, &sha); + } + +private: + u_char _hash[SHA_DIGEST_LENGTH]; +}; + +struct LRUHashHasher { + bool + operator()(const LRUHash *s1, const LRUHash *s2) const + { + return 0 == memcmp(s1->_hash, s2->_hash, sizeof(s2->_hash)); + } + + size_t + operator()(const LRUHash *s) const + { + return *(reinterpret_cast(s->_hash)) ^ *(reinterpret_cast(s->_hash + 9)); + } +}; + +typedef std::pair LRUEntry; +using LRUList = std::list; +typedef std::unordered_map LRUMap; + +class LRUPolicy : public PromotionPolicy +{ +public: + LRUPolicy() : PromotionPolicy(), _lock(TSMutexCreate()) {} + ~LRUPolicy() override; + + bool parseOption(int opt, char *optarg) override; + bool doPromote(TSHttpTxn txnp) override; + bool stats_add(const char *remap_id) override; + + void + usage() const override + { + TSError("[%s] Usage: @plugin=%s.so @pparam=--policy=lru @pparam=--buckets= --hits= --sample=", PLUGIN_NAME, + PLUGIN_NAME); + } + + const char * + policyName() const override + { + return "LRU"; + } + + const std::string + id() const override + { + return _label + ";LRU=b:" + std::to_string(_buckets) + ",h:" + std::to_string(_hits); + } + +private: + unsigned _buckets = 1000; + unsigned _hits = 10; + + // For the LRU. Note that we keep track of the List sizes, because some versions fo STL have broken + // implementations of size(), making them obsessively slow on calling ::size(). + TSMutex _lock; + LRUMap _map; + LRUList _list, _freelist; + size_t _list_size = 0, _freelist_size = 0; + + // internal stats ids + int freelist_size_id = -1; + int lru_size_id = -1; + int lru_hit_id = -1; + int lru_miss_id = -1; + int lru_vacated_id = -1; + int promoted_id = -1; +}; diff --git a/plugins/cache_promote/policy.cc b/plugins/cache_promote/policy.cc new file mode 100644 index 00000000000..8bfaf4fba42 --- /dev/null +++ b/plugins/cache_promote/policy.cc @@ -0,0 +1,58 @@ +/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ +#include "tscore/BufferWriter.h" +#include "policy.h" + +bool +PromotionPolicy::doSample() const +{ + if (_sample > 0) { + // coverity[dont_call] + double r = drand48(); + + if (_sample > r) { + TSDebug(PLUGIN_NAME, "checking sampling, is %f > %f? Yes!", _sample, r); + } else { + TSDebug(PLUGIN_NAME, "checking sampling, is %f > %f? No!", _sample, r); + return false; + } + } + return true; +} + +int +PromotionPolicy::create_stat(std::string_view name, std::string_view remap_identifier) +{ + int stat_id = -1; + ts::LocalBufferWriter stat_name; + + stat_name.reset().clip(1); + stat_name.print("plugin.{}.{}.{}", PLUGIN_NAME, remap_identifier, name); + stat_name.extend(1).write('\0'); + + if (TS_ERROR == TSStatFindName(stat_name.data(), &stat_id)) { + stat_id = TSStatCreate(stat_name.data(), TS_RECORDDATATYPE_INT, TS_STAT_NON_PERSISTENT, TS_STAT_SYNC_SUM); + if (TS_ERROR == stat_id) { + TSDebug(PLUGIN_NAME, "error creating stat_name: %s", stat_name.data()); + } else { + TSDebug(PLUGIN_NAME, "created stat_name: %s, stat_id: %d", stat_name.data(), stat_id); + } + } + + return stat_id; +} diff --git a/plugins/cache_promote/policy.h b/plugins/cache_promote/policy.h new file mode 100644 index 00000000000..ef06f1bfea0 --- /dev/null +++ b/plugins/cache_promote/policy.h @@ -0,0 +1,105 @@ +/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ +#pragma once + +#include +#include + +#include "ts/ts.h" +#include "ts/remap.h" + +#define MAX_STAT_LENGTH (1 << 8) +extern const char *PLUGIN_NAME; + +////////////////////////////////////////////////////////////////////////////////////////////// +// Abstract base class for all policies. +// +class PromotionPolicy +{ +public: + PromotionPolicy() + { + // This doesn't have to be perfect, since this is just chance sampling. + // coverity[dont_call] + TSDebug(PLUGIN_NAME, "PromotionPolicy() CTOR"); + srand48(static_cast(time(nullptr))); + } + + virtual ~PromotionPolicy() = default; + + void + setSample(char *s) + { + _sample = strtof(s, nullptr) / 100.0; + } + + float + getSample() const + { + return _sample; + } + + void + decrementStat(const int stat, const int amount) + { + if (stats_enabled) { + TSStatIntDecrement(stat, amount); + } + } + + void + incrementStat(const int stat, const int amount) + { + if (stats_enabled) { + TSStatIntIncrement(stat, amount); + } + } + + virtual bool + parseOption(int opt, char *optarg) + { + return false; + } + + virtual const std::string + id() const + { + return ""; + } + + bool doSample() const; + int create_stat(std::string_view name, std::string_view remap_identifier); + + // These are pure virtual + virtual bool doPromote(TSHttpTxn txnp) = 0; + virtual const char *policyName() const = 0; + virtual void usage() const = 0; + virtual bool stats_add(const char *remap_id) = 0; + + // when true stats are incremented. + bool stats_enabled = false; + int cache_hits_id = -1; + int promoted_id = -1; + int total_requests_id = -1; + +private: + float _sample = 0.0; + +protected: + std::string _label = ""; +}; diff --git a/plugins/cache_promote/policy_manager.cc b/plugins/cache_promote/policy_manager.cc new file mode 100644 index 00000000000..de3ecb0f82b --- /dev/null +++ b/plugins/cache_promote/policy_manager.cc @@ -0,0 +1,67 @@ +/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ +#include "policy_manager.h" + +PromotionPolicy * +PolicyManager::coalescePolicy(PromotionPolicy *policy) +{ + std::string tag = policy->id(); + + if (tag.size() != 0) { + auto res = _policies.find(tag); + + TSDebug(PLUGIN_NAME, "looking up policy by tag: %s", tag.c_str()); + + if (res != _policies.end()) { + TSDebug(PLUGIN_NAME, "repurposing policy for tag: %s", tag.c_str()); + + ++res->second.second; + // Repurpose the existing policy, nuking this placeholder. + delete policy; + return res->second.first; + } else { + TSDebug(PLUGIN_NAME, "inserting policy for tag: %s", tag.c_str()); + _policies[tag] = std::make_pair(policy, 1); + } + } + + return policy; +} + +void +PolicyManager::releasePolicy(PromotionPolicy *policy) +{ + std::string tag = policy->id(); + + if (tag.size() != 0) { + auto res = _policies.find(tag); + + if (res != _policies.end()) { + if (0 == --res->second.second) { + TSDebug(PLUGIN_NAME, "releasing unused PromotionPolicy"); + delete res->second.first; + _policies.erase(res); + } + } else { + TSAssert(!"Trying to release a policy which was not acquired via PolicyManager"); + } + } else { + // Not managed by the policy manager, so just nuke it. + delete policy; + } +} diff --git a/plugins/cache_promote/policy_manager.h b/plugins/cache_promote/policy_manager.h new file mode 100644 index 00000000000..bdf7bfba1f1 --- /dev/null +++ b/plugins/cache_promote/policy_manager.h @@ -0,0 +1,49 @@ +/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ +#pragma once + +#include +#include + +#include "policy.h" + +class PolicyManager +{ +public: + PolicyManager() { TSDebug(PLUGIN_NAME, "PolicyManager() CTOR"); } + virtual ~PolicyManager() { TSDebug(PLUGIN_NAME, "~PolicyManger() DTOR"); } + + // This is sort of a no-op right now, but it should be called regardless. + void + clear() + { + // This should always be zero here, otherwise, we have not released policies properly. + TSReleaseAssert(_policies.size() == 0); + } + + // This is the main entry point. + PromotionPolicy *coalescePolicy(PromotionPolicy *policy); + void releasePolicy(PromotionPolicy *policy); + + // Don't allow copy-constructors. + PolicyManager(PolicyManager const &) = delete; + void operator=(PolicyManager const &) = delete; + +private: + std::unordered_map>> _policies; +}; diff --git a/plugins/cache_range_requests/README b/plugins/cache_range_requests/README deleted file mode 100644 index 5495f687d0f..00000000000 --- a/plugins/cache_range_requests/README +++ /dev/null @@ -1,45 +0,0 @@ - -Thousands of range requests for a very large object in the traffic server cache -are likely to increase system load averages due to I/O wait as objects are stored -on a single stripe or disk drive. - -This plugin allows you to remap individual range requests so that they are stored -as individual objects in the ATS cache when subsequent range requests are likely -to use the same range. This spreads range requests over multiple stripes thereby -reducing I/O wait and system load averages. - -This plugin reads the range request header byte range value and then creates a -new cache key url using the original request url with the range value appended -to it. The range header is removed where appropriate from the requests and the -origin server response code is changed from a 206 to a 200 to insure that the -object is written to cache using the new cache key url. The response code sent -to the client will be changed back to a 206 and all requests to the origin server -will contain the range header so that the correct response is received. - -Configuration: - - Add @plugin=cache_range_requests.so to your remap.config rules. - - Or for a global plugin where all range requests are processed, - Add cache_range_requests.so to the plugin.config - -Parent Selection Mode (consistent-hash only): - - default: Parent selection is based solely on the hash of a URL Path - In this mode, all partial content of a URL is requested from the same - upstream parent cache listed in parent.config - - cache_key_url: Parent selection is based on the full cache_key_url which - includes information about the partial content range. - In this mode, all requests (include partial content) will use - consistent hashing method for parent selection. - - To enable cache_key_url parent select mode, the following param must be set: - - Global Plugin (plugin.config): - - cache_range_requests.so ps_mode:cache_key_url - - Remap Plugin (remap.config): - - @plugin=cache_range_requests.so @pparam=ps_mode:cache_key_url diff --git a/plugins/cache_range_requests/README.md b/plugins/cache_range_requests/README.md new file mode 100644 index 00000000000..ef567a70944 --- /dev/null +++ b/plugins/cache_range_requests/README.md @@ -0,0 +1,80 @@ + +Thousands of range requests for a very large object in the traffic server +cache are likely to increase system load averages due to I/O wait as +objects are stored on a single stripe or disk drive. + +This plugin allows you to remap individual range requests so that they +are stored as individual objects in the ATS cache when subsequent range +requests are likely to use the same range. This spreads range requests +over multiple stripes thereby reducing I/O wait and system load averages. + +This plugin reads the range request header byte range value and then +creates a new cache key url using the original request url with the range +value appended to it. The range header is removed where appropriate +from the requests and the origin server response code is changed from +a 206 to a 200 to insure that the object is written to cache using the +new cache key url. The response code sent to the client will be changed +back to a 206 and all requests to the origin server will contain the +range header so that the correct response is received. + +Configuration: + + Add @plugin=cache_range_requests.so to your remap.config rules. + + Or for a global plugin where all range requests are processed, + Add cache_range_requests.so to the plugin.config + +Parent Selection Mode (consistent-hash only): + + default: Parent selection is based solely on the hash of a URL Path + In this mode, all partial content of a URL is requested + from the same upstream parent cache listed in parent.config + + Cache_key_url: Parent selection is based on the full cache_key_url + which includes information about the partial content + range. In this mode, all requests (include partial + content) will use consistent hashing method for + parent selection. + + To enable cache_key_url parent select mode, the following param must be set: + + Global Plugin (plugin.config): + + cache_range_requests.so -p + cache_range_requests.so --ps-cachekey + + Remap Plugin (remap.config): + + @plugin=cache_range_requests.so @pparam=--ps-cachekey + @plugin=cache_range_requests.so @pparam=-p + +X-CRR-IMS header support + + To support slice plugin self healing an option to force + revalidation after cache lookup complete was added. This option + is triggered by a special header: + + This optional header looks like: + + X-CRR-IMS: Tue, 19 Nov 2019 13:26:45 GMT + + If the cache lookup was a cache hit and the cache header date + is *less* than this header value then the cache state is switched + from FRESH to STALE which results in If-Modified-Since or + If-Match request being passed to the parent. + + In order for this option to be enabled the the following parameter + must be set: + + Global Plugin (plugin.config): + + cache_range_requests.so --consider-ims + cache_range_requests.so -c + + Remap Plugin (remap.config): + + @plugin=cache_range_requests.so @pparam=--consider-ims + @plugin=cache_range_requests.so @pparam=-c + + Consider using the header_rewrite plugin to protect the parent + from using this option as an attack vector against an origin. diff --git a/plugins/cache_range_requests/cache_range_requests.cc b/plugins/cache_range_requests/cache_range_requests.cc index 60f512caf7c..41a770221ba 100644 --- a/plugins/cache_range_requests/cache_range_requests.cc +++ b/plugins/cache_range_requests/cache_range_requests.cc @@ -26,68 +26,113 @@ * requests. */ -#include -#include #include "ts/ts.h" #include "ts/remap.h" +#include +#include +#include +#include +#include + #define PLUGIN_NAME "cache_range_requests" #define DEBUG_LOG(fmt, ...) TSDebug(PLUGIN_NAME, "[%s:%d] %s(): " fmt, __FILE__, __LINE__, __func__, ##__VA_ARGS__) #define ERROR_LOG(fmt, ...) TSError("[%s:%d] %s(): " fmt, __FILE__, __LINE__, __func__, ##__VA_ARGS__) +namespace +{ typedef enum parent_select_mode { PS_DEFAULT, // Default ATS parent selection mode PS_CACHEKEY_URL, // Set parent selection url to cache_key url } parent_select_mode_t; struct pluginconfig { - parent_select_mode_t ps_mode; + parent_select_mode_t ps_mode{PS_DEFAULT}; + bool consider_ims_header{false}; + bool modify_cache_key{true}; }; struct txndata { - char *range_value; + std::string range_value; + time_t ims_time{0}; }; -static int handle_read_request_header(TSCont, TSEvent, void *); -static void range_header_check(TSHttpTxn txnp, struct pluginconfig *pc); -static void handle_send_origin_request(TSCont, TSHttpTxn, struct txndata *); -static void handle_client_send_response(TSHttpTxn, struct txndata *); -static void handle_server_read_response(TSHttpTxn, struct txndata *); -static int remove_header(TSMBuffer, TSMLoc, const char *, int); -static bool set_header(TSMBuffer, TSMLoc, const char *, int, const char *, int); -static int transaction_handler(TSCont, TSEvent, void *); -static struct pluginconfig *create_pluginconfig(int argc, const char *argv[]); -static void delete_pluginconfig(struct pluginconfig *); +// Header for optional revalidation +constexpr std::string_view X_IMS_HEADER = {"X-CRR-IMS"}; // pluginconfig struct (global plugin only) -static struct pluginconfig *gPluginConfig = nullptr; +pluginconfig *gPluginConfig = {nullptr}; + +int handle_read_request_header(TSCont, TSEvent, void *); +void range_header_check(TSHttpTxn, pluginconfig *const); +void handle_send_origin_request(TSCont, TSHttpTxn, txndata *const); +void handle_client_send_response(TSHttpTxn, txndata *const); +void handle_server_read_response(TSHttpTxn, txndata *const); +int remove_header(TSMBuffer, TSMLoc, const char *, int); +bool set_header(TSMBuffer, TSMLoc, const char *, int, const char *, int); +int transaction_handler(TSCont, TSEvent, void *); +struct pluginconfig *create_pluginconfig(int argc, char *const argv[]); +void delete_pluginconfig(pluginconfig *const); /** * Creates pluginconfig data structure * Sets default parent url selection mode * Walk plugin argument list and updates config */ -static struct pluginconfig * -create_pluginconfig(int argc, const char *argv[]) +pluginconfig * +create_pluginconfig(int argc, char *const argv[]) { - struct pluginconfig *pc = nullptr; + DEBUG_LOG("Number of arguments: %d", argc); + for (int index = 0; index < argc; ++index) { + DEBUG_LOG("args[%d] = %s", index, argv[index]); + } - pc = static_cast(TSmalloc(sizeof(struct pluginconfig))); + pluginconfig *const pc = new pluginconfig; if (nullptr == pc) { ERROR_LOG("Can't allocate pluginconfig"); return nullptr; } - // Plugin uses default ATS selection (hash of URL path) - pc->ps_mode = PS_DEFAULT; + static const struct option longopts[] = { + {const_cast("ps-cachekey"), no_argument, nullptr, 'p'}, + {const_cast("consider-ims"), no_argument, nullptr, 'c'}, + {const_cast("no-modify-cachekey"), no_argument, nullptr, 'n'}, + {nullptr, 0, nullptr, 0}, + }; - // Walk through param list. - for (int c = 0; c < argc; c++) { - if (strcmp("ps_mode:cache_key_url", argv[c]) == 0) { - pc->ps_mode = PS_CACHEKEY_URL; + // getopt assumes args start at '1' + ++argc; + --argv; + + for (;;) { + int const opt = getopt_long(argc, argv, "", longopts, nullptr); + if (-1 == opt) { break; } + + switch (opt) { + case 'p': { + DEBUG_LOG("Plugin modifies parent selection key"); + pc->ps_mode = PS_CACHEKEY_URL; + } break; + case 'c': { + DEBUG_LOG("Plugin considers the '%.*s' header", (int)X_IMS_HEADER.size(), X_IMS_HEADER.data()); + pc->consider_ims_header = true; + } break; + case 'n': { + DEBUG_LOG("Plugin doesn't modify cache key"); + pc->modify_cache_key = false; + } break; + default: { + } break; + } + } + + // Backwards compatibility + if (optind < argc && 0 == strcmp("ps_mode:cache_key_url", argv[optind])) { + DEBUG_LOG("Plugin modifies parent selection key (deprecated)"); + pc->ps_mode = PS_CACHEKEY_URL; } return pc; @@ -96,21 +141,19 @@ create_pluginconfig(int argc, const char *argv[]) /** * Destroy pluginconfig data structure. */ -static void -delete_pluginconfig(struct pluginconfig *pc) +void +delete_pluginconfig(pluginconfig *const pc) { if (nullptr != pc) { DEBUG_LOG("Delete struct pluginconfig"); - TSfree(pc); - pc = nullptr; + delete pc; } } /** * Entry point when used as a global plugin. - * */ -static int +int handle_read_request_header(TSCont txn_contp, TSEvent event, void *edata) { TSHttpTxn txnp = static_cast(edata); @@ -131,63 +174,82 @@ handle_read_request_header(TSCont txn_contp, TSEvent event, void *edata) * 3. Schedules TS_HTTP_SEND_REQUEST_HDR_HOOK, TS_HTTP_SEND_RESPONSE_HDR_HOOK, * and TS_HTTP_TXN_CLOSE_HOOK for further processing. */ -static void -range_header_check(TSHttpTxn txnp, struct pluginconfig *pc) +void +range_header_check(TSHttpTxn txnp, pluginconfig *const pc) { char cache_key_url[8192] = {0}; char *req_url; int length, url_length, cache_key_url_length; - struct txndata *txn_state; - TSMBuffer hdr_bufp; - TSMLoc req_hdrs = nullptr; - TSMLoc loc = nullptr; + txndata *txn_state; + TSMBuffer hdr_buf; + TSMLoc hdr_loc = nullptr; + TSMLoc loc = nullptr; TSCont txn_contp; - if (TS_SUCCESS == TSHttpTxnClientReqGet(txnp, &hdr_bufp, &req_hdrs)) { - loc = TSMimeHdrFieldFind(hdr_bufp, req_hdrs, TS_MIME_FIELD_RANGE, TS_MIME_LEN_RANGE); + if (TS_SUCCESS == TSHttpTxnClientReqGet(txnp, &hdr_buf, &hdr_loc)) { + loc = TSMimeHdrFieldFind(hdr_buf, hdr_loc, TS_MIME_FIELD_RANGE, TS_MIME_LEN_RANGE); if (TS_NULL_MLOC != loc) { - const char *hdr_value = TSMimeHdrFieldValueStringGet(hdr_bufp, req_hdrs, loc, 0, &length); + const char *hdr_value = TSMimeHdrFieldValueStringGet(hdr_buf, hdr_loc, loc, 0, &length); + TSHandleMLocRelease(hdr_buf, hdr_loc, loc); + if (!hdr_value || length <= 0) { DEBUG_LOG("Not a range request."); } else { if (nullptr == (txn_contp = TSContCreate(static_cast(transaction_handler), nullptr))) { ERROR_LOG("failed to create the transaction handler continuation."); } else { - txn_state = static_cast(TSmalloc(sizeof(struct txndata))); - txn_state->range_value = TSstrndup(hdr_value, length); - DEBUG_LOG("length: %d, txn_state->range_value: %s", length, txn_state->range_value); - txn_state->range_value[length] = '\0'; // workaround for bug in core - + txn_state = new txndata; + std::string &rv = txn_state->range_value; + rv.assign(hdr_value, length); + DEBUG_LOG("length: %d, txn_state->range_value: %s", length, rv.c_str()); req_url = TSHttpTxnEffectiveUrlStringGet(txnp, &url_length); - cache_key_url_length = snprintf(cache_key_url, 8192, "%s-%s", req_url, txn_state->range_value); + cache_key_url_length = snprintf(cache_key_url, 8192, "%s-%s", req_url, rv.c_str()); DEBUG_LOG("Rewriting cache URL for %s to %s", req_url, cache_key_url); if (req_url != nullptr) { TSfree(req_url); } - // set the cache key. - if (TS_SUCCESS != TSCacheUrlSet(txnp, cache_key_url, cache_key_url_length)) { - DEBUG_LOG("failed to change the cache url to %s.", cache_key_url); - } + if (nullptr != pc) { + // set the cache key if configured to. + if (pc->modify_cache_key && TS_SUCCESS != TSCacheUrlSet(txnp, cache_key_url, cache_key_url_length)) { + ERROR_LOG("failed to change the cache url to %s.", cache_key_url); + ERROR_LOG("Disabling cache for this transaction to avoid cache poisoning."); + TSHttpTxnServerRespNoStoreSet(txnp, 1); + TSHttpTxnRespCacheableSet(txnp, 0); + TSHttpTxnReqCacheableSet(txnp, 0); + } - // Optionally set the parent_selection_url to the cache_key url or path - if (nullptr != pc && PS_DEFAULT != pc->ps_mode) { - TSMLoc ps_loc = nullptr; - - if (PS_CACHEKEY_URL == pc->ps_mode) { - const char *start = cache_key_url; - const char *end = cache_key_url + cache_key_url_length; - if (TS_SUCCESS == TSUrlCreate(hdr_bufp, &ps_loc) && - TS_PARSE_DONE == TSUrlParse(hdr_bufp, ps_loc, &start, end) && // This should always succeed. - TS_SUCCESS == TSHttpTxnParentSelectionUrlSet(txnp, hdr_bufp, ps_loc)) { - DEBUG_LOG("Set Parent Selection URL to cache_key_url: %s", cache_key_url); - TSHandleMLocRelease(hdr_bufp, TS_NULL_MLOC, ps_loc); + // Optionally set the parent_selection_url to the cache_key url or path + if (PS_DEFAULT != pc->ps_mode) { + TSMLoc ps_loc = nullptr; + + if (PS_CACHEKEY_URL == pc->ps_mode) { + const char *start = cache_key_url; + const char *end = cache_key_url + cache_key_url_length; + if (TS_SUCCESS == TSUrlCreate(hdr_buf, &ps_loc) && + TS_PARSE_DONE == TSUrlParse(hdr_buf, ps_loc, &start, end) && // This should always succeed. + TS_SUCCESS == TSHttpTxnParentSelectionUrlSet(txnp, hdr_buf, ps_loc)) { + DEBUG_LOG("Set Parent Selection URL to cache_key_url: %s", cache_key_url); + TSHandleMLocRelease(hdr_buf, TS_NULL_MLOC, ps_loc); + } + } + } + + // optionally consider an X-CRR-IMS header + if (pc->consider_ims_header) { + TSMLoc const imsloc = TSMimeHdrFieldFind(hdr_buf, hdr_loc, X_IMS_HEADER.data(), X_IMS_HEADER.size()); + if (TS_NULL_MLOC != imsloc) { + time_t const itime = TSMimeHdrFieldValueDateGet(hdr_buf, hdr_loc, imsloc); + TSHandleMLocRelease(hdr_buf, hdr_loc, imsloc); + if (0 < itime) { + txn_state->ims_time = itime; + } } } } // remove the range request header. - if (remove_header(hdr_bufp, req_hdrs, TS_MIME_FIELD_RANGE, TS_MIME_LEN_RANGE) > 0) { + if (remove_header(hdr_buf, hdr_loc, TS_MIME_FIELD_RANGE, TS_MIME_LEN_RANGE) > 0) { DEBUG_LOG("Removed the Range: header from the request."); } @@ -196,13 +258,18 @@ range_header_check(TSHttpTxn txnp, struct pluginconfig *pc) TSHttpTxnHookAdd(txnp, TS_HTTP_SEND_RESPONSE_HDR_HOOK, txn_contp); TSHttpTxnHookAdd(txnp, TS_HTTP_TXN_CLOSE_HOOK, txn_contp); DEBUG_LOG("Added TS_HTTP_SEND_REQUEST_HDR_HOOK, TS_HTTP_SEND_RESPONSE_HDR_HOOK, and TS_HTTP_TXN_CLOSE_HOOK"); + + if (0 < txn_state->ims_time) { + TSHttpTxnHookAdd(txnp, TS_HTTP_CACHE_LOOKUP_COMPLETE_HOOK, txn_contp); + DEBUG_LOG("Also Added TS_HTTP_CACHE_LOOKUP_COMPLETE_HOOK"); + } } } - TSHandleMLocRelease(hdr_bufp, req_hdrs, loc); + // TSHandleMLocRelease(hdr_buf, hdr_loc, loc); } else { DEBUG_LOG("no range request header."); } - TSHandleMLocRelease(hdr_bufp, TS_NULL_MLOC, req_hdrs); + TSHandleMLocRelease(hdr_buf, TS_NULL_MLOC, hdr_loc); } else { DEBUG_LOG("failed to retrieve the server request"); } @@ -212,41 +279,44 @@ range_header_check(TSHttpTxn txnp, struct pluginconfig *pc) * Restores the range request header if the request must be * satisfied from the origin and schedules the TS_READ_RESPONSE_HDR_HOOK. */ -static void -handle_send_origin_request(TSCont contp, TSHttpTxn txnp, struct txndata *txn_state) +void +handle_send_origin_request(TSCont contp, TSHttpTxn txnp, txndata *const txn_state) { - TSMBuffer hdr_bufp; - TSMLoc req_hdrs = nullptr; + TSMBuffer hdr_buf; + TSMLoc hdr_loc = nullptr; + + std::string const &rv = txn_state->range_value; - if (TS_SUCCESS == TSHttpTxnServerReqGet(txnp, &hdr_bufp, &req_hdrs) && txn_state->range_value != nullptr) { - if (set_header(hdr_bufp, req_hdrs, TS_MIME_FIELD_RANGE, TS_MIME_LEN_RANGE, txn_state->range_value, - strlen(txn_state->range_value))) { - DEBUG_LOG("Added range header: %s", txn_state->range_value); + if (TS_SUCCESS == TSHttpTxnServerReqGet(txnp, &hdr_buf, &hdr_loc) && !rv.empty()) { + if (set_header(hdr_buf, hdr_loc, TS_MIME_FIELD_RANGE, TS_MIME_LEN_RANGE, rv.data(), rv.length())) { + DEBUG_LOG("Added range header: %s", rv.c_str()); TSHttpTxnHookAdd(txnp, TS_HTTP_READ_RESPONSE_HDR_HOOK, contp); } } - TSHandleMLocRelease(hdr_bufp, TS_NULL_MLOC, req_hdrs); + TSHandleMLocRelease(hdr_buf, TS_NULL_MLOC, hdr_loc); } /** * Changes the response code back to a 206 Partial content before * replying to the client that requested a range. */ -static void -handle_client_send_response(TSHttpTxn txnp, struct txndata *txn_state) +void +handle_client_send_response(TSHttpTxn txnp, txndata *const txn_state) { bool partial_content_reason = false; char *p; int length; - TSMBuffer response, hdr_bufp; - TSMLoc resp_hdr, req_hdrs = nullptr; + TSMBuffer resp_buf = nullptr; + TSMBuffer req_buf = nullptr; + TSMLoc resp_loc = nullptr; + TSMLoc req_loc = nullptr; - TSReturnCode result = TSHttpTxnClientRespGet(txnp, &response, &resp_hdr); + TSReturnCode result = TSHttpTxnClientRespGet(txnp, &resp_buf, &resp_loc); DEBUG_LOG("result: %d", result); if (TS_SUCCESS == result) { - TSHttpStatus status = TSHttpHdrStatusGet(response, resp_hdr); + TSHttpStatus status = TSHttpHdrStatusGet(resp_buf, resp_loc); // a cached result will have a TS_HTTP_OK with a 'Partial Content' reason - if ((p = const_cast(TSHttpHdrReasonGet(response, resp_hdr, &length))) != nullptr) { + if ((p = const_cast(TSHttpHdrReasonGet(resp_buf, resp_loc, &length))) != nullptr) { if ((length == 15) && (0 == strncasecmp(p, "Partial Content", length))) { partial_content_reason = true; } @@ -254,23 +324,23 @@ handle_client_send_response(TSHttpTxn txnp, struct txndata *txn_state) DEBUG_LOG("%d %.*s", status, length, p); if (TS_HTTP_STATUS_OK == status && partial_content_reason) { DEBUG_LOG("Got TS_HTTP_STATUS_OK."); - TSHttpHdrStatusSet(response, resp_hdr, TS_HTTP_STATUS_PARTIAL_CONTENT); + TSHttpHdrStatusSet(resp_buf, resp_loc, TS_HTTP_STATUS_PARTIAL_CONTENT); DEBUG_LOG("Set response header to TS_HTTP_STATUS_PARTIAL_CONTENT."); } } + std::string const &rv = txn_state->range_value; // add the range request header back in so that range requests may be logged. - if (TS_SUCCESS == TSHttpTxnClientReqGet(txnp, &hdr_bufp, &req_hdrs) && txn_state->range_value != nullptr) { - if (set_header(hdr_bufp, req_hdrs, TS_MIME_FIELD_RANGE, TS_MIME_LEN_RANGE, txn_state->range_value, - strlen(txn_state->range_value))) { - DEBUG_LOG("added range header: %s", txn_state->range_value); + if (TS_SUCCESS == TSHttpTxnClientReqGet(txnp, &req_buf, &req_loc) && !rv.empty()) { + if (set_header(req_buf, req_loc, TS_MIME_FIELD_RANGE, TS_MIME_LEN_RANGE, rv.data(), rv.length())) { + DEBUG_LOG("added range header: %s", rv.c_str()); } else { DEBUG_LOG("set_header() failed."); } } else { DEBUG_LOG("failed to get Request Headers"); } - TSHandleMLocRelease(response, TS_NULL_MLOC, resp_hdr); - TSHandleMLocRelease(hdr_bufp, TS_NULL_MLOC, req_hdrs); + TSHandleMLocRelease(resp_buf, TS_NULL_MLOC, resp_loc); + TSHandleMLocRelease(req_buf, TS_NULL_MLOC, req_loc); } /** @@ -278,20 +348,20 @@ handle_client_send_response(TSHttpTxn txnp, struct txndata *txn_state) * the response code from a 206 Partial content to a 200 OK so that * the response will be written to cache. */ -static void -handle_server_read_response(TSHttpTxn txnp, struct txndata *txn_state) +void +handle_server_read_response(TSHttpTxn txnp, txndata *const txn_state) { - TSMBuffer response; - TSMLoc resp_hdr; + TSMBuffer resp_buf = nullptr; + TSMLoc resp_loc = nullptr; TSHttpStatus status; - if (TS_SUCCESS == TSHttpTxnServerRespGet(txnp, &response, &resp_hdr)) { - status = TSHttpHdrStatusGet(response, resp_hdr); + if (TS_SUCCESS == TSHttpTxnServerRespGet(txnp, &resp_buf, &resp_loc)) { + status = TSHttpHdrStatusGet(resp_buf, resp_loc); if (TS_HTTP_STATUS_PARTIAL_CONTENT == status) { DEBUG_LOG("Got TS_HTTP_STATUS_PARTIAL_CONTENT."); - TSHttpHdrStatusSet(response, resp_hdr, TS_HTTP_STATUS_OK); + TSHttpHdrStatusSet(resp_buf, resp_loc, TS_HTTP_STATUS_OK); DEBUG_LOG("Set response header to TS_HTTP_STATUS_OK."); - bool cacheable = TSHttpTxnIsCacheable(txnp, nullptr, response); + bool cacheable = TSHttpTxnIsCacheable(txnp, nullptr, resp_buf); DEBUG_LOG("range is cacheable: %d", cacheable); } else if (TS_HTTP_STATUS_OK == status) { DEBUG_LOG("The origin does not support range requests, attempting to disable cache write."); @@ -302,7 +372,7 @@ handle_server_read_response(TSHttpTxn txnp, struct txndata *txn_state) } } } - TSHandleMLocRelease(response, TS_NULL_MLOC, resp_hdr); + TSHandleMLocRelease(resp_buf, TS_NULL_MLOC, resp_loc); } /** @@ -311,18 +381,18 @@ handle_server_read_response(TSHttpTxn txnp, struct txndata *txn_state) * * From background_fetch.cc */ -static int -remove_header(TSMBuffer bufp, TSMLoc hdr_loc, const char *header, int len) +int +remove_header(TSMBuffer buf, TSMLoc hdr_loc, const char *header, int len) { - TSMLoc field = TSMimeHdrFieldFind(bufp, hdr_loc, header, len); + TSMLoc field = TSMimeHdrFieldFind(buf, hdr_loc, header, len); int cnt = 0; while (field) { - TSMLoc tmp = TSMimeHdrFieldNextDup(bufp, hdr_loc, field); + TSMLoc tmp = TSMimeHdrFieldNextDup(buf, hdr_loc, field); ++cnt; - TSMimeHdrFieldDestroy(bufp, hdr_loc, field); - TSHandleMLocRelease(bufp, hdr_loc, field); + TSMimeHdrFieldDestroy(buf, hdr_loc, field); + TSHandleMLocRelease(buf, hdr_loc, field); field = tmp; } @@ -336,25 +406,25 @@ remove_header(TSMBuffer bufp, TSMLoc hdr_loc, const char *header, int len) * * From background_fetch.cc */ -static bool -set_header(TSMBuffer bufp, TSMLoc hdr_loc, const char *header, int len, const char *val, int val_len) +bool +set_header(TSMBuffer buf, TSMLoc hdr_loc, const char *header, int len, const char *val, int val_len) { - if (!bufp || !hdr_loc || !header || len <= 0 || !val || val_len <= 0) { + if (!buf || !hdr_loc || !header || len <= 0 || !val || val_len <= 0) { return false; } DEBUG_LOG("header: %s, len: %d, val: %s, val_len: %d", header, len, val, val_len); bool ret = false; - TSMLoc field_loc = TSMimeHdrFieldFind(bufp, hdr_loc, header, len); + TSMLoc field_loc = TSMimeHdrFieldFind(buf, hdr_loc, header, len); if (!field_loc) { // No existing header, so create one - if (TS_SUCCESS == TSMimeHdrFieldCreateNamed(bufp, hdr_loc, header, len, &field_loc)) { - if (TS_SUCCESS == TSMimeHdrFieldValueStringSet(bufp, hdr_loc, field_loc, -1, val, val_len)) { - TSMimeHdrFieldAppend(bufp, hdr_loc, field_loc); + if (TS_SUCCESS == TSMimeHdrFieldCreateNamed(buf, hdr_loc, header, len, &field_loc)) { + if (TS_SUCCESS == TSMimeHdrFieldValueStringSet(buf, hdr_loc, field_loc, -1, val, val_len)) { + TSMimeHdrFieldAppend(buf, hdr_loc, field_loc); ret = true; } - TSHandleMLocRelease(bufp, hdr_loc, field_loc); + TSHandleMLocRelease(buf, hdr_loc, field_loc); } } else { TSMLoc tmp = nullptr; @@ -363,14 +433,14 @@ set_header(TSMBuffer bufp, TSMLoc hdr_loc, const char *header, int len, const ch while (field_loc) { if (first) { first = false; - if (TS_SUCCESS == TSMimeHdrFieldValueStringSet(bufp, hdr_loc, field_loc, -1, val, val_len)) { + if (TS_SUCCESS == TSMimeHdrFieldValueStringSet(buf, hdr_loc, field_loc, -1, val, val_len)) { ret = true; } } else { - TSMimeHdrFieldDestroy(bufp, hdr_loc, field_loc); + TSMimeHdrFieldDestroy(buf, hdr_loc, field_loc); } - tmp = TSMimeHdrFieldNextDup(bufp, hdr_loc, field_loc); - TSHandleMLocRelease(bufp, hdr_loc, field_loc); + tmp = TSMimeHdrFieldNextDup(buf, hdr_loc, field_loc); + TSHandleMLocRelease(buf, hdr_loc, field_loc); field_loc = tmp; } } @@ -378,6 +448,91 @@ set_header(TSMBuffer bufp, TSMLoc hdr_loc, const char *header, int len, const ch return ret; } +time_t +get_date_from_cached_hdr(TSHttpTxn txn) +{ + TSMBuffer buf; + TSMLoc hdr_loc, date_loc; + time_t date = 0; + + if (TSHttpTxnCachedRespGet(txn, &buf, &hdr_loc) == TS_SUCCESS) { + date_loc = TSMimeHdrFieldFind(buf, hdr_loc, TS_MIME_FIELD_DATE, TS_MIME_LEN_DATE); + if (date_loc != TS_NULL_MLOC) { + date = TSMimeHdrFieldValueDateGet(buf, hdr_loc, date_loc); + TSHandleMLocRelease(buf, hdr_loc, date_loc); + } + TSHandleMLocRelease(buf, TS_NULL_MLOC, hdr_loc); + } + + return date; +} + +/** + * Handle a special IMS request. + */ +void +handle_cache_lookup_complete(TSHttpTxn txnp, txndata *const txn_state) +{ + int cachestat; + if (TS_SUCCESS == TSHttpTxnCacheLookupStatusGet(txnp, &cachestat)) { + if (TS_CACHE_LOOKUP_HIT_FRESH == cachestat) { + time_t const ch_time = get_date_from_cached_hdr(txnp); + if (ch_time < txn_state->ims_time) { + TSHttpTxnCacheLookupStatusSet(txnp, TS_CACHE_LOOKUP_HIT_STALE); + if (TSIsDebugTagSet(PLUGIN_NAME)) { + int url_len = 0; + char *const req_url = TSHttpTxnEffectiveUrlStringGet(txnp, &url_len); + if (nullptr != req_url) { + std::string const &rv = txn_state->range_value; + DEBUG_LOG("Forced revalidate %.*s-%s", url_len, req_url, rv.c_str()); + + TSfree(req_url); + } + } + } + } + } +} + +/** + * Transaction event handler. + */ +int +transaction_handler(TSCont contp, TSEvent event, void *edata) +{ + TSHttpTxn txnp = static_cast(edata); + txndata *const txn_state = static_cast(TSContDataGet(contp)); + + switch (event) { + case TS_EVENT_HTTP_READ_RESPONSE_HDR: + handle_server_read_response(txnp, txn_state); + break; + case TS_EVENT_HTTP_SEND_REQUEST_HDR: + handle_send_origin_request(contp, txnp, txn_state); + break; + case TS_EVENT_HTTP_SEND_RESPONSE_HDR: + handle_client_send_response(txnp, txn_state); + break; + case TS_EVENT_HTTP_CACHE_LOOKUP_COMPLETE: + handle_cache_lookup_complete(txnp, txn_state); + break; + case TS_EVENT_HTTP_TXN_CLOSE: + if (txn_state != nullptr) { + TSContDataSet(contp, nullptr); + delete txn_state; + } + TSContDestroy(contp); + break; + default: + TSAssert(!"Unexpected event"); + break; + } + TSHttpTxnReenable(txnp, TS_EVENT_HTTP_CONTINUE); + return 0; +} + +} // namespace + /** * Remap initialization. */ @@ -411,11 +566,10 @@ TSRemapNewInstance(int argc, char *argv[], void **ih, char * /*errbuf */, int /* } // Skip over the Remap params - const char **plugin_argv = const_cast(argv + 2); - argc -= 2; + char *const *plugin_argv = const_cast(argv); // Parse the argument list. - *ih = create_pluginconfig(argc, plugin_argv); + *ih = static_cast(create_pluginconfig(argc - 2, plugin_argv + 2)); if (*ih == nullptr) { ERROR_LOG("Can't create pluginconfig"); @@ -430,7 +584,7 @@ TSRemapNewInstance(int argc, char *argv[], void **ih, char * /*errbuf */, int /* void TSRemapDeleteInstance(void *ih) { - struct pluginconfig *pc = static_cast(ih); + pluginconfig *const pc = static_cast(ih); if (nullptr != pc) { delete_pluginconfig(pc); @@ -443,7 +597,7 @@ TSRemapDeleteInstance(void *ih) TSRemapStatus TSRemapDoRemap(void *ih, TSHttpTxn txnp, TSRemapRequestInfo * /* rri */) { - struct pluginconfig *pc = static_cast(ih); + pluginconfig *const pc = static_cast(ih); range_header_check(txnp, pc); @@ -470,11 +624,9 @@ TSPluginInit(int argc, const char *argv[]) } if (nullptr == gPluginConfig) { - if (argc > 1) { - // Skip ahead of first param (name of traffic server plugin shared object) - const char **plugin_argv = const_cast(argv + 1); - argc -= 1; - gPluginConfig = create_pluginconfig(argc, plugin_argv); + if (1 < argc) { + char *const *plugin_argv = const_cast(argv); + gPluginConfig = create_pluginconfig(argc - 1, plugin_argv + 1); } } @@ -485,39 +637,3 @@ TSPluginInit(int argc, const char *argv[]) TSHttpHookAdd(TS_HTTP_READ_REQUEST_HDR_HOOK, txnp_cont); } } - -/** - * Transaction event handler. - */ -static int -transaction_handler(TSCont contp, TSEvent event, void *edata) -{ - TSHttpTxn txnp = static_cast(edata); - struct txndata *txn_state = static_cast(TSContDataGet(contp)); - - switch (event) { - case TS_EVENT_HTTP_READ_RESPONSE_HDR: - handle_server_read_response(txnp, txn_state); - break; - case TS_EVENT_HTTP_SEND_REQUEST_HDR: - handle_send_origin_request(contp, txnp, txn_state); - break; - case TS_EVENT_HTTP_SEND_RESPONSE_HDR: - handle_client_send_response(txnp, txn_state); - break; - case TS_EVENT_HTTP_TXN_CLOSE: - if (txn_state != nullptr && txn_state->range_value != nullptr) { - TSfree(txn_state->range_value); - } - if (txn_state != nullptr) { - TSfree(txn_state); - } - TSContDestroy(contp); - break; - default: - TSAssert(!"Unexpected event"); - break; - } - TSHttpTxnReenable(txnp, TS_EVENT_HTTP_CONTINUE); - return 0; -} diff --git a/plugins/cachekey/cachekey.cc b/plugins/cachekey/cachekey.cc index 9cf4454210f..5f128894bfa 100644 --- a/plugins/cachekey/cachekey.cc +++ b/plugins/cachekey/cachekey.cc @@ -188,6 +188,51 @@ getUri(TSMBuffer buf, TSMLoc url) return uri; } +static String +getCanonicalUrl(TSMBuffer buf, TSMLoc url, bool canonicalPrefix, bool provideDefaultKey) +{ + String canonicalUrl; + + String scheme; + int schemeLen; + const char *schemePtr = TSUrlSchemeGet(buf, url, &schemeLen); + if (nullptr != schemePtr && 0 != schemeLen) { + scheme.assign(schemePtr, schemeLen); + } else { + CacheKeyError("failed to get scheme"); + return canonicalUrl; + } + + String host; + int hostLen; + const char *hostPtr = TSUrlHostGet(buf, url, &hostLen); + if (nullptr != hostPtr && 0 != hostLen) { + host.assign(hostPtr, hostLen); + } else { + CacheKeyError("failed to get host"); + return canonicalUrl; + } + + String port; + int portInt = TSUrlPortGet(buf, url); + ::append(port, portInt); + + if (canonicalPrefix) { + /* return the same for both regex input or default key, results in 'scheme://host:port' */ + canonicalUrl.assign(scheme).append("://").append(host).append(":").append(port); + } else { + if (provideDefaultKey) { + /* return the key default - results in '/host/port' */ + canonicalUrl.assign("/").append(host).append("/").append(port); + } else { + /* return regex input string - results in 'host:port' (use-case kept for compatibility reasons) */ + canonicalUrl.assign(host).append(":").append(port); + } + } + + return canonicalUrl; +} + /** * @brief Constructor setting up the cache key prefix, initializing request info. * @param txn transaction handle. @@ -195,8 +240,8 @@ getUri(TSMBuffer buf, TSMLoc url) * @param uriType type of the URI used to create the cachekey ("remap" or "pristine") * @param rri remap request info */ -CacheKey::CacheKey(TSHttpTxn txn, String separator, CacheKeyUriType uriType, TSRemapRequestInfo *rri) - : _txn(txn), _separator(std::move(separator)), _uriType(uriType) +CacheKey::CacheKey(TSHttpTxn txn, String separator, CacheKeyUriType uriType, CacheKeyKeyType keyType, TSRemapRequestInfo *rri) + : _txn(txn), _separator(std::move(separator)), _uriType(uriType), _keyType(keyType) { _key.reserve(512); @@ -205,8 +250,9 @@ CacheKey::CacheKey(TSHttpTxn txn, String separator, CacheKeyUriType uriType, TSR /* Get the URI and header to base the cachekey on. * @TODO it might make sense to add more supported URI types */ + CacheKeyDebug("setting %s from a %s plugin", getCacheKeyKeyTypeName(_keyType), _remap ? "remap" : "global"); + if (_remap) { - CacheKeyDebug("setting cache key from a remap plugin"); if (PRISTINE == _uriType) { if (TS_SUCCESS != TSHttpTxnPristineUrlGet(_txn, &_buf, &_url)) { /* Failing here is unlikely. No action seems the only reasonable thing to do from within this plug-in */ @@ -221,7 +267,6 @@ CacheKey::CacheKey(TSHttpTxn txn, String separator, CacheKeyUriType uriType, TSR } _hdrs = rri->requestHdrp; } else { - CacheKeyDebug("setting cache key from a global plugin"); if (TS_SUCCESS != TSHttpTxnClientReqGet(_txn, &_buf, &_hdrs)) { /* Failing here is unlikely. No action seems the only reasonable thing to do from within this plug-in */ CacheKeyError("failed to get client request handle"); @@ -289,6 +334,16 @@ CacheKey::append(const String &s) ::appendEncoded(_key, s.data(), s.size()); } +void +CacheKey::append(const String &s, bool useSeparator) +{ + if (useSeparator) { + append(s); + } else { + _key.append(s); + } +} + /** * @brief Append null-terminated C-style string to the key. * @param s null-terminated C-style string. @@ -319,42 +374,31 @@ CacheKey::append(const char *s, unsigned n) * @param prefix if not empty string will append the static prefix to the cache key. * @param prefixCapture if not empty will append regex capture/replacement from the host:port. * @param prefixCaptureUri if not empty will append regex capture/replacement from the whole URI. + * @param canonicalPrefix false - use 'host:port' as starting point of all transformations, true - use 'scheme://host:port' * @note if both prefix and pattern are not empty prefix will be added first, followed by the results from pattern. */ void -CacheKey::appendPrefix(const String &prefix, Pattern &prefixCapture, Pattern &prefixCaptureUri) +CacheKey::appendPrefix(const String &prefix, Pattern &prefixCapture, Pattern &prefixCaptureUri, bool canonicalPrefix) { - // "true" would mean that the plugin config meant to override the default prefix (host:port). + // "true" would mean that the plugin config meant to override the default prefix, "false" means use default. bool customPrefix = false; - String host; - int port = 0; + + /* For all the following operations if a canonical prefix is required then appned to the key with no separator + * to leave the door open for potential valid host name formed in the final resulting cache key. */ if (!prefix.empty()) { customPrefix = true; - append(prefix); + append(prefix, /* useSeparator */ !canonicalPrefix); CacheKeyDebug("added static prefix, key: '%s'", _key.c_str()); } - int hostLen; - const char *hostPtr = TSUrlHostGet(_buf, _url, &hostLen); - if (nullptr != hostPtr && 0 != hostLen) { - host.assign(hostPtr, hostLen); - } else { - CacheKeyError("failed to get host"); - } - port = TSUrlPortGet(_buf, _url); - if (!prefixCapture.empty()) { customPrefix = true; - String hostAndPort; - hostAndPort.append(host).append(":"); - ::append(hostAndPort, port); - StringVector captures; - if (prefixCapture.process(hostAndPort, captures)) { + if (prefixCapture.process(getCanonicalUrl(_buf, _url, canonicalPrefix, /* provideDefaultKey */ false), captures)) { for (auto &capture : captures) { - append(capture); + append(capture, /* useSeparator */ !canonicalPrefix); } CacheKeyDebug("added host:port capture prefix, key: '%s'", _key.c_str()); } @@ -368,7 +412,7 @@ CacheKey::appendPrefix(const String &prefix, Pattern &prefixCapture, Pattern &pr StringVector captures; if (prefixCaptureUri.process(uri, captures)) { for (auto &capture : captures) { - append(capture); + append(capture, /* useSeparator */ !canonicalPrefix); } CacheKeyDebug("added URI capture prefix, key: '%s'", _key.c_str()); } @@ -376,8 +420,8 @@ CacheKey::appendPrefix(const String &prefix, Pattern &prefixCapture, Pattern &pr } if (!customPrefix) { - append(host); - append(port); + /* nothing was customized => default prefix */ + append(getCanonicalUrl(_buf, _url, canonicalPrefix, /* provideDefaultKey */ true), /* useSeparator */ false); CacheKeyDebug("added default prefix, key: '%s'", _key.c_str()); } } @@ -701,24 +745,67 @@ CacheKey::appendUaClass(Classifier &classifier) bool CacheKey::finalize() const { - bool res = true; - CacheKeyDebug("finalizing cache key '%s' from a %s plugin", _key.c_str(), (_remap ? "remap" : "global")); - if (TS_SUCCESS != TSCacheUrlSet(_txn, &(_key[0]), _key.size())) { - int len; - char *url = TSHttpTxnEffectiveUrlStringGet(_txn, &len); - if (nullptr != url) { + bool res = false; + String msg; + + CacheKeyDebug("finalizing %s '%s' from a %s plugin", getCacheKeyKeyTypeName(_keyType), _key.c_str(), + (_remap ? "remap" : "global")); + switch (_keyType) { + case CACHE_KEY: { + if (TS_SUCCESS == TSCacheUrlSet(_txn, &(_key[0]), _key.size())) { + /* Set cache key succesfully */ + msg.assign("set cache key to ").append(_key); + res = true; + } else { if (_remap) { /* Remap instance. Always runs first by design (before TS_HTTP_POST_REMAP_HOOK) */ - CacheKeyError("failed to set cache key for url %.*s", len, url); + msg.assign("failed to set cache key"); } else { /* Global instance. We would fail and get here if a per-remap instance has already set the cache key * (currently TSCacheUrlSet() can be called only once successfully). Don't error, just debug. * @todo avoid the consecutive attempts and error only on unexpected failures. */ - CacheKeyDebug("failed to set cache key for url %.*s", len, url); + msg.assign("failed to set cache key"); } + } + } break; + case PARENT_SELECTION_URL: { + /* parent selection */ + const char *start = _key.c_str(); + const char *end = _key.c_str() + _key.length(); + TSMLoc new_url_loc; + if (TS_SUCCESS == TSUrlCreate(_buf, &new_url_loc)) { + if (TS_PARSE_DONE == TSUrlParse(_buf, new_url_loc, &start, end)) { + if (TS_SUCCESS == TSHttpTxnParentSelectionUrlSet(_txn, _buf, new_url_loc)) { + msg.assign("set parent selection URL to ").append(_key); + res = true; + } else { + msg.assign("failed to set parent selection URL"); + } + } else { + msg.assign("failed to parse parent selection URL"); + } + TSHandleMLocRelease(_buf, TS_NULL_MLOC, new_url_loc); + } else { + msg.assign("failed to create parent selection URL"); + } + } break; + default: { + msg.assign("unknown target URI type"); + } break; + } + + /* Report status - debug level in case of success, error in case of failure. + * Since getting effective URI is expensive add it only in case of failure */ + if (res) { + CacheKeyDebug("%.*s", static_cast(msg.length()), msg.c_str()); + } else { + int len; + char *url = TSHttpTxnEffectiveUrlStringGet(_txn, &len); + if (nullptr != url) { + msg.append(" for url ").append(url, len); TSfree(url); } - res = false; + CacheKeyError("%.*s", static_cast(msg.length()), msg.c_str()); } return res; } diff --git a/plugins/cachekey/cachekey.h b/plugins/cachekey/cachekey.h index 7ea058cb6cf..0b47e85984d 100644 --- a/plugins/cachekey/cachekey.h +++ b/plugins/cachekey/cachekey.h @@ -50,14 +50,16 @@ class CacheKey { public: - CacheKey(TSHttpTxn txn, String separator, CacheKeyUriType urlType, TSRemapRequestInfo *rri = nullptr); + CacheKey(TSHttpTxn txn, String separator, CacheKeyUriType urlType, CacheKeyKeyType targetUrlType, + TSRemapRequestInfo *rri = nullptr); ~CacheKey(); void append(unsigned number); void append(const String &); + void append(const String &s, bool useSeparator); void append(const char *s); void append(const char *n, unsigned s); - void appendPrefix(const String &prefix, Pattern &prefixCapture, Pattern &prefixCaptureUri); + void appendPrefix(const String &prefix, Pattern &prefixCapture, Pattern &prefixCaptureUri, bool canonicalPrefix); void appendPath(Pattern &pathCapture, Pattern &pathCaptureUri); void appendHeaders(const ConfigHeaders &config); void appendQuery(const ConfigQuery &config); @@ -85,7 +87,8 @@ class CacheKey bool _valid = false; /**< @brief shows if the constructor discovered the input correctly */ bool _remap = false; /**< @brief shows if the input URI was from remap info */ - String _key; /**< @brief cache key */ - String _separator; /**< @brief a separator used to separate the cache key elements extracted from the URI */ - CacheKeyUriType _uriType; /**< @brief the URI type used as a cachekey base: pristine, remap, etc. */ + String _key; /**< @brief cache key */ + String _separator; /**< @brief a separator used to separate the cache key elements extracted from the URI */ + CacheKeyUriType _uriType = REMAP; /**< @brief the URI type used as a cachekey base: pristine, remap, etc. */ + CacheKeyKeyType _keyType = CACHE_KEY; /**< @brief the target URI type: cache key, parent selection, etc. */ }; diff --git a/plugins/cachekey/configs.cc b/plugins/cachekey/configs.cc index 2320db14336..b2bc42d5e70 100644 --- a/plugins/cachekey/configs.cc +++ b/plugins/cachekey/configs.cc @@ -274,11 +274,11 @@ makeConfigPath(const String &path) /** * @brief a helper function which loads the classifier from files. * @param args classname + filename in ':' format. - * @param blacklist true - load as a blacklist classifier, false - whitelist. + * @param denylist true - load as a denylist classifier, false - allowlist. * @return true if successful, false otherwise. */ bool -Configs::loadClassifiers(const String &args, bool blacklist) +Configs::loadClassifiers(const String &args, bool denylist) { static const char *EXPECTED_FORMAT = ":"; @@ -310,7 +310,7 @@ Configs::loadClassifiers(const String &args, bool blacklist) } MultiPattern *multiPattern; - if (blacklist) { + if (denylist) { multiPattern = new NonMatchingMultiPattern(classname); } else { multiPattern = new MultiPattern(classname); @@ -341,11 +341,11 @@ Configs::loadClassifiers(const String &args, bool blacklist) p = new Pattern(); if (nullptr != p && p->init(regex)) { - if (blacklist) { - CacheKeyDebug("Added pattern '%s' to black list '%s'", regex.c_str(), classname.c_str()); + if (denylist) { + CacheKeyDebug("Added pattern '%s' to deny list '%s'", regex.c_str(), classname.c_str()); multiPattern->add(p); } else { - CacheKeyDebug("Added pattern '%s' to white list '%s'", regex.c_str(), classname.c_str()); + CacheKeyDebug("Added pattern '%s' to allow list '%s'", regex.c_str(), classname.c_str()); multiPattern->add(p); } } else { @@ -385,8 +385,8 @@ Configs::init(int argc, const char *argv[], bool perRemapConfig) {const_cast("include-headers"), optional_argument, nullptr, 'g'}, {const_cast("include-cookies"), optional_argument, nullptr, 'h'}, {const_cast("ua-capture"), optional_argument, nullptr, 'i'}, - {const_cast("ua-whitelist"), optional_argument, nullptr, 'j'}, - {const_cast("ua-blacklist"), optional_argument, nullptr, 'k'}, + {const_cast("ua-allowlist"), optional_argument, nullptr, 'j'}, + {const_cast("ua-denylist"), optional_argument, nullptr, 'k'}, {const_cast("static-prefix"), optional_argument, nullptr, 'l'}, {const_cast("capture-prefix"), optional_argument, nullptr, 'm'}, {const_cast("capture-prefix-uri"), optional_argument, nullptr, 'n'}, @@ -396,7 +396,10 @@ Configs::init(int argc, const char *argv[], bool perRemapConfig) {const_cast("remove-path"), optional_argument, nullptr, 'r'}, {const_cast("separator"), optional_argument, nullptr, 's'}, {const_cast("uri-type"), optional_argument, nullptr, 't'}, - {const_cast("capture-header"), optional_argument, nullptr, 'u'}, + {const_cast("key-type"), optional_argument, nullptr, 'u'}, + {const_cast("capture-header"), optional_argument, nullptr, 'v'}, + {const_cast("canonical-prefix"), optional_argument, nullptr, 'w'}, + /* reserve 'z' for 'config' files */ {nullptr, 0, nullptr, 0}, }; @@ -449,15 +452,15 @@ Configs::init(int argc, const char *argv[], bool perRemapConfig) status = false; } break; - case 'j': /* ua-whitelist */ - if (!loadClassifiers(optarg, /* blacklist = */ false)) { - CacheKeyError("failed to load User-Agent pattern white-list '%s'", optarg); + case 'j': /* ua-allowlist */ + if (!loadClassifiers(optarg, /* denylist = */ false)) { + CacheKeyError("failed to load User-Agent pattern allow-list '%s'", optarg); status = false; } break; - case 'k': /* ua-blacklist */ - if (!loadClassifiers(optarg, /* blacklist = */ true)) { - CacheKeyError("failed to load User-Agent pattern black-list '%s'", optarg); + case 'k': /* ua-denylist */ + if (!loadClassifiers(optarg, /* denylist = */ true)) { + CacheKeyError("failed to load User-Agent pattern deny-list '%s'", optarg); status = false; } break; @@ -501,9 +504,15 @@ Configs::init(int argc, const char *argv[], bool perRemapConfig) case 't': /* uri-type */ setUriType(optarg); break; - case 'u': /* capture-header */ + case 'u': /* key-type */ + setKeyType(optarg); + break; + case 'v': /* capture-header */ _headers.addCapture(optarg); break; + case 'w': /* canonical-prefix */ + _canonicalPrefix = isTrue(optarg); + break; } } @@ -520,6 +529,10 @@ Configs::init(int argc, const char *argv[], bool perRemapConfig) bool Configs::finalize() { + if (_keyTypes.empty()) { + CacheKeyDebug("setting cache key"); + _keyTypes = {CACHE_KEY}; + } return _query.finalize() && _headers.finalize() && _cookies.finalize(); } @@ -535,6 +548,12 @@ Configs::pathToBeRemoved() return _pathToBeRemoved; } +bool +Configs::canonicalPrefix() +{ + return _canonicalPrefix; +} + void Configs::setSeparator(const char *arg) { @@ -567,8 +586,63 @@ Configs::setUriType(const char *arg) } } +void +Configs::setKeyType(const char *arg) +{ + if (nullptr != arg) { + StringVector types; + ::commaSeparateString(types, arg); + + for (auto type : types) { + if (9 == type.length() && 0 == strncasecmp(type.c_str(), "cache_key", 9)) { + _keyTypes.insert(CacheKeyKeyType::CACHE_KEY); + CacheKeyDebug("setting cache key"); + } else if (20 == type.length() && 0 == strncasecmp(type.c_str(), "parent_selection_url", 20)) { + _keyTypes.insert(CacheKeyKeyType::PARENT_SELECTION_URL); + CacheKeyDebug("setting parent selection URL"); + } else { + CacheKeyError("unrecognized key type '%s', using default 'cache_key'", arg); + } + } + } else { + CacheKeyError("found an empty key type, using default 'cache_key'"); + } +} + CacheKeyUriType Configs::getUriType() { return _uriType; } + +CacheKeyKeyTypeSet & +Configs::getKeyType() +{ + return _keyTypes; +} + +const char * +getCacheKeyUriTypeName(CacheKeyUriType type) +{ + switch (type) { + case REMAP: + return "remap"; + case PRISTINE: + return "pristine"; + default: + return "unknown"; + } +} + +const char * +getCacheKeyKeyTypeName(CacheKeyKeyType type) +{ + switch (type) { + case CACHE_KEY: + return "cache key"; + case PARENT_SELECTION_URL: + return "parent selection url"; + default: + return "unknown"; + } +} diff --git a/plugins/cachekey/configs.h b/plugins/cachekey/configs.h index 947b21931ea..e98b69afd48 100644 --- a/plugins/cachekey/configs.h +++ b/plugins/cachekey/configs.h @@ -33,6 +33,16 @@ enum CacheKeyUriType { PRISTINE, }; +enum CacheKeyKeyType { + CACHE_KEY, + PARENT_SELECTION_URL, +}; + +const char *getCacheKeyUriTypeName(CacheKeyUriType type); +const char *getCacheKeyKeyTypeName(CacheKeyKeyType type); + +typedef std::set CacheKeyKeyTypeSet; + /** * @brief Plug-in configuration elements (query / headers / cookies). * @@ -162,6 +172,11 @@ class Configs */ bool pathToBeRemoved(); + /** + * @brief keep URI scheme and authority elements. + */ + bool canonicalPrefix(); + /** * @brief set the cache key elements separator string. */ @@ -177,11 +192,21 @@ class Configs */ void setUriType(const char *arg); + /** + * @brief sets the target URI Type. + */ + void setKeyType(const char *arg); + /** * @brief get URI type. */ CacheKeyUriType getUriType(); + /** + * @brief get target URI type. + */ + CacheKeyKeyTypeSet &getKeyType(); + /* Make the following members public to avoid unnecessary accessors */ ConfigQuery _query; /**< @brief query parameter related configuration */ ConfigHeaders _headers; /**< @brief headers related configuration */ @@ -192,19 +217,21 @@ class Configs Pattern _prefixCaptureUri; /**< @brief cache key prefix captured from the URI as a whole */ Pattern _pathCapture; /**< @brief cache key element captured from the URI path */ Pattern _pathCaptureUri; /**< @brief cache key element captured from the URI as a whole */ - Classifier _classifier; /**< @brief blacklist and white-list classifier used to classify User-Agent header */ + Classifier _classifier; /**< @brief denylist and allow-list classifier used to classify User-Agent header */ private: /** * @brief a helper function which loads the classifier from files. * @param args classname + filename in ':' format. - * @param blacklist true - load as a blacklist classifier, false - white-list. + * @param denylist true - load as a denylist classifier, false - allow-list. * @return true if successful, false otherwise. */ - bool loadClassifiers(const String &args, bool blacklist = true); + bool loadClassifiers(const String &args, bool denylist = true); bool _prefixToBeRemoved = false; /**< @brief instructs the prefix (i.e. host:port) not to added to the cache key */ bool _pathToBeRemoved = false; /**< @brief instructs the path not to added to the cache key */ + bool _canonicalPrefix = false; /**< @brief keep the URI scheme and authority element used as input to transforming into key */ String _separator = "/"; /**< @brief a separator used to separate the cache key elements extracted from the URI */ CacheKeyUriType _uriType = REMAP; /**< @brief shows which URI the cache key will be based on */ + CacheKeyKeyTypeSet _keyTypes; /**< @brief target URI to be modified, cache key or paren selection */ }; diff --git a/plugins/cachekey/plugin.cc b/plugins/cachekey/plugin.cc index 43bad79e3c0..d92c079271a 100644 --- a/plugins/cachekey/plugin.cc +++ b/plugins/cachekey/plugin.cc @@ -38,34 +38,38 @@ Configs *globalConfig = nullptr; static void setCacheKey(TSHttpTxn txn, Configs *config, TSRemapRequestInfo *rri = nullptr) { - /* Initial cache key facility from the requested URL. */ - CacheKey cachekey(txn, config->getSeparator(), config->getUriType(), rri); + const CacheKeyKeyTypeSet &keyTypes = config->getKeyType(); - /* Append custom prefix or the host:port */ - if (!config->prefixToBeRemoved()) { - cachekey.appendPrefix(config->_prefix, config->_prefixCapture, config->_prefixCaptureUri); - } - /* Classify User-Agent and append the class name to the cache key if matched. */ - cachekey.appendUaClass(config->_classifier); + for (auto type : keyTypes) { + /* Initial cache key facility from the requested URL. */ + CacheKey cachekey(txn, config->getSeparator(), config->getUriType(), type, rri); - /* Capture from User-Agent header. */ - cachekey.appendUaCaptures(config->_uaCapture); + /* Append custom prefix or the host:port */ + if (!config->prefixToBeRemoved()) { + cachekey.appendPrefix(config->_prefix, config->_prefixCapture, config->_prefixCaptureUri, config->canonicalPrefix()); + } + /* Classify User-Agent and append the class name to the cache key if matched. */ + cachekey.appendUaClass(config->_classifier); - /* Append headers to the cache key. */ - cachekey.appendHeaders(config->_headers); + /* Capture from User-Agent header. */ + cachekey.appendUaCaptures(config->_uaCapture); - /* Append cookies to the cache key. */ - cachekey.appendCookies(config->_cookies); + /* Append headers to the cache key. */ + cachekey.appendHeaders(config->_headers); - /* Append the path to the cache key. */ - if (!config->pathToBeRemoved()) { - cachekey.appendPath(config->_pathCapture, config->_pathCaptureUri); - } - /* Append query parameters to the cache key. */ - cachekey.appendQuery(config->_query); + /* Append cookies to the cache key. */ + cachekey.appendCookies(config->_cookies); - /* Set the cache key */ - cachekey.finalize(); + /* Append the path to the cache key. */ + if (!config->pathToBeRemoved()) { + cachekey.appendPath(config->_pathCapture, config->_pathCaptureUri); + } + /* Append query parameters to the cache key. */ + cachekey.appendQuery(config->_query); + + /* Set the cache key */ + cachekey.finalize(); + } } static int diff --git a/plugins/certifier/certifier.cc b/plugins/certifier/certifier.cc index de1469947a8..452b694f3dc 100644 --- a/plugins/certifier/certifier.cc +++ b/plugins/certifier/certifier.cc @@ -534,7 +534,7 @@ shadow_cert_generator(TSCont contp, TSEvent event, void *edata) TSDebug(PLUGIN_NAME, "\tClearing the queue size %lu", localQ.size()); TSVConn ssl_vc = reinterpret_cast(localQ.front()); localQ.pop(); - TSSslConnection sslobj = TSVConnSSLConnectionGet(ssl_vc); + TSSslConnection sslobj = TSVConnSslConnectionGet(ssl_vc); SSL *ssl = reinterpret_cast(sslobj); SSL_set_SSL_CTX(ssl, ref_ctx); TSVConnReenable(ssl_vc); @@ -549,7 +549,7 @@ static int cert_retriever(TSCont contp, TSEvent event, void *edata) { TSVConn ssl_vc = reinterpret_cast(edata); - TSSslConnection sslobj = TSVConnSSLConnectionGet(ssl_vc); + TSSslConnection sslobj = TSVConnSslConnectionGet(ssl_vc); SSL *ssl = reinterpret_cast(sslobj); const char *servername = SSL_get_servername(ssl, TLSEXT_NAMETYPE_host_name); SSL_CTX *ref_ctx = nullptr; diff --git a/plugins/compress/compress.cc b/plugins/compress/compress.cc index c6c560b082b..b7efe6dba3d 100644 --- a/plugins/compress/compress.cc +++ b/plugins/compress/compress.cc @@ -24,6 +24,8 @@ #include #include +#include "ink_autoconf.h" + #if HAVE_BROTLI_ENCODE_H #include #endif @@ -873,7 +875,6 @@ transform_plugin(TSCont contp, TSEvent event, void *edata) case TS_EVENT_HTTP_TXN_CLOSE: // Release the ocnif lease, and destroy this continuation - hc->release(); TSContDestroy(contp); break; @@ -904,9 +905,9 @@ handle_request(TSHttpTxn txnp, Configuration *config) if (TSHttpTxnClientReqGet(txnp, &req_buf, &req_loc) == TS_SUCCESS) { if (config == nullptr) { - hc = find_host_configuration(txnp, req_buf, req_loc, nullptr); // Get a lease on the global config + hc = find_host_configuration(txnp, req_buf, req_loc, nullptr); } else { - hc = find_host_configuration(txnp, req_buf, req_loc, config); // Get a lease on the local config passed through doRemap + hc = find_host_configuration(txnp, req_buf, req_loc, config); } bool allowed = false; @@ -929,8 +930,6 @@ handle_request(TSHttpTxn txnp, Configuration *config) normalize_accept_encoding(txnp, req_buf, req_loc); TSHttpTxnHookAdd(txnp, TS_HTTP_CACHE_LOOKUP_COMPLETE_HOOK, transform_contp); TSHttpTxnHookAdd(txnp, TS_HTTP_TXN_CLOSE_HOOK, transform_contp); // To release the config - } else { - hc->release(); // No longer need this configuration, release it. } TSHandleMLocRelease(req_buf, TS_NULL_MLOC, req_loc); } @@ -965,11 +964,7 @@ load_global_configuration(TSCont contp) debug("config swapped, old config %p", oldconfig); - // First, if there was a previous configuration, clean that one out. This avoids the - // small race condition tht exist between doing a find() and calling hold() on a - // HostConfig object. if (prev_config) { - prev_config->release_all(); debug("deleting previous configuration container, %p", prev_config); delete prev_config; } @@ -1072,7 +1067,6 @@ TSRemapDeleteInstance(void *instance) { debug("Cleanup configs read from remap"); auto c = static_cast(instance); - c->release_all(); delete c; } diff --git a/plugins/compress/configuration.cc b/plugins/compress/configuration.cc index a8afd3f9b0b..b6211a4c7b6 100644 --- a/plugins/compress/configuration.cc +++ b/plugins/compress/configuration.cc @@ -21,12 +21,15 @@ limitations under the License. */ +#include "ink_autoconf.h" #include "configuration.h" #include #include #include #include +#include "debug_macros.h" + namespace Gzip { using namespace std; @@ -110,7 +113,6 @@ enum ParserState { void Configuration::add_host_configuration(HostConfiguration *hc) { - hc->hold(); // We hold a lease on the HostConfig while it's in this container host_configurations_.push_back(hc); } @@ -152,18 +154,9 @@ Configuration::find(const char *host, int host_length) } } - host_configuration->hold(); // Hold a lease return host_configuration; } -void -Configuration::release_all() -{ - for (auto &host_configuration : host_configurations_) { - host_configuration->release(); - } -} - bool HostConfiguration::is_url_allowed(const char *url, int url_len) { @@ -204,7 +197,10 @@ HostConfiguration::is_content_type_compressible(const char *content_type, int co for (StringContainer::iterator it = compressible_content_types_.begin(); it != compressible_content_types_.end(); ++it) { const char *match_string = it->c_str(); - bool exclude = match_string[0] == '!'; + if (match_string == nullptr) { + continue; + } + bool exclude = match_string[0] == '!'; if (exclude) { ++match_string; // skip '!' diff --git a/plugins/compress/configuration.h b/plugins/compress/configuration.h index df5f74de0ad..67e81ff945c 100644 --- a/plugins/compress/configuration.h +++ b/plugins/compress/configuration.h @@ -26,8 +26,8 @@ #include #include #include -#include "debug_macros.h" -#include "tscore/ink_atomic.h" + +#include "ts/ts.h" #include "tscpp/api/noncopyable.h" namespace Gzip @@ -51,8 +51,7 @@ class HostConfiguration : private atscppapi::noncopyable remove_accept_encoding_(false), flush_(false), compression_algorithms_(ALGORITHM_GZIP), - minimum_content_length_(1024), - ref_count_(0) + minimum_content_length_(1024) { } @@ -128,21 +127,6 @@ class HostConfiguration : private atscppapi::noncopyable void add_compression_algorithms(std::string &algorithms); int compression_algorithms(); - // Ref-counting these host configuration objects - void - hold() - { - ink_atomic_increment(&ref_count_, 1); - } - void - release() - { - if (1 >= ink_atomic_decrement(&ref_count_, 1)) { - debug("released and deleting HostConfiguration for %s settings", host_.size() > 0 ? host_.c_str() : "global"); - delete this; - } - } - private: std::string host_; bool enabled_; @@ -151,7 +135,6 @@ class HostConfiguration : private atscppapi::noncopyable bool flush_; int compression_algorithms_; unsigned int minimum_content_length_; - int ref_count_; StringContainer compressible_content_types_; StringContainer allows_; @@ -169,7 +152,6 @@ class Configuration : private atscppapi::noncopyable public: static Configuration *Parse(const char *path); HostConfiguration *find(const char *host, int host_length); - void release_all(); private: explicit Configuration() {} diff --git a/plugins/compress/misc.cc b/plugins/compress/misc.cc index 57708443ff5..56c3fa9a24c 100644 --- a/plugins/compress/misc.cc +++ b/plugins/compress/misc.cc @@ -23,6 +23,7 @@ #include "ts/ts.h" #include "tscore/ink_defs.h" +#include "tscpp/util/TextView.h" #include "misc.h" #include @@ -41,40 +42,69 @@ gzip_free(voidpf /* opaque ATS_UNUSED */, voidpf address) TSfree(address); } +namespace +{ +// Strips parameters from value. Returns cleared TextView if a q=f parameter present, where f is less than or equal to +// zero. +// +void +strip_ae_value(ts::TextView &value) +{ + ts::TextView compression{value.take_prefix_at(';')}; + compression.trim(" \t"); + while (value) { + ts::TextView param{value.take_prefix_at(';')}; + ts::TextView name{param.take_prefix_at('=')}; + name.trim(" \t"); + if (strcasecmp("q", name) == 0) { + // If q value is valid and is zero, suppress compression types. + param.trim(" \t"); + if (param) { + ts::TextView whole{param.take_prefix_at('.')}; + whole.ltrim(" \t"); + if ("0" == whole) { + param.trim('0'); + if (!param) { + // Suppress compression type. + compression.clear(); + break; + } + } + } + } + } + value = compression; +} +} // end anonymous namespace + void normalize_accept_encoding(TSHttpTxn /* txnp ATS_UNUSED */, TSMBuffer reqp, TSMLoc hdr_loc) { TSMLoc field = TSMimeHdrFieldFind(reqp, hdr_loc, TS_MIME_FIELD_ACCEPT_ENCODING, TS_MIME_LEN_ACCEPT_ENCODING); - int deflate = 0; - int gzip = 0; - int br = 0; + bool deflate = false; + bool gzip = false; + bool br = false; // remove the accept encoding field(s), // while finding out if gzip or deflate is supported. while (field) { - TSMLoc tmp; - - if (!deflate && !gzip) { - int value_count = TSMimeHdrFieldValuesCount(reqp, hdr_loc, field); - - while (value_count > 0) { - int val_len = 0; - const char *val; - - --value_count; - val = TSMimeHdrFieldValueStringGet(reqp, hdr_loc, field, value_count, &val_len); - - if (val_len == static_cast(strlen("br"))) { - br = !strncmp(val, "br", val_len); - } - if (val_len == static_cast(strlen("gzip"))) { - gzip = !strncmp(val, "gzip", val_len); - } else if (val_len == static_cast(strlen("deflate"))) { - deflate = !strncmp(val, "deflate", val_len); + int val_len; + const char *values_ = TSMimeHdrFieldValueStringGet(reqp, hdr_loc, field, -1, &val_len); + if (values_ && val_len) { + ts::TextView values(values_, val_len); + while (values) { + ts::TextView next{values.take_prefix_at(',')}; + strip_ae_value(next); + if (strcasecmp("gzip", next) == 0) { + gzip = true; + } else if (strcasecmp("br", next) == 0) { + br = true; + } else if (strcasecmp("deflate", next) == 0) { + deflate = true; } } } - tmp = TSMimeHdrFieldNextDup(reqp, hdr_loc, field); + TSMLoc tmp = TSMimeHdrFieldNextDup(reqp, hdr_loc, field); TSMimeHdrFieldDestroy(reqp, hdr_loc, field); // catch retval? TSHandleMLocRelease(reqp, hdr_loc, field); field = tmp; @@ -142,6 +172,7 @@ init_hidden_header_name() hidden_header_name = static_cast(TSmalloc(hidden_header_name_len + 1)); hidden_header_name[hidden_header_name_len] = 0; sprintf(hidden_header_name, "x-accept-encoding-%s", result); + TSfree(result); } return hidden_header_name; } diff --git a/plugins/esi/README.combo b/plugins/esi/README.combo index e2042e1427a..211e8e0dc89 100644 --- a/plugins/esi/README.combo +++ b/plugins/esi/README.combo @@ -1,6 +1,8 @@ Combohandler -------------------- +NOTE: THIS FILE IS OBSOLETE, SEE THE TRAFFICSERVER ADMIN GUIDE. + This plugin provides that functionality (and more) with the same interface but with these differences in configuration: diff --git a/plugins/esi/combo_handler.cc b/plugins/esi/combo_handler.cc index 72d27fafba7..baacf797fad 100644 --- a/plugins/esi/combo_handler.cc +++ b/plugins/esi/combo_handler.cc @@ -23,10 +23,12 @@ #include #include +#include #include #include #include #include +#include #include #include #include @@ -36,6 +38,7 @@ #include "ts/experimental.h" #include "ts/remap.h" #include "tscore/ink_defs.h" +#include "tscpp/util/TextView.h" #include "HttpDataFetcherImpl.h" #include "gzip.h" @@ -63,7 +66,7 @@ unsigned MaxFileCount = DEFAULT_MAX_FILE_COUNT; int arg_idx; static string SIG_KEY_NAME; -static vector HEADER_WHITELIST; +static vector HEADER_ALLOWLIST; #define DEFAULT_COMBO_HANDLER_PATH "admin/v1/combo" static string COMBO_HANDLER_PATH{DEFAULT_COMBO_HANDLER_PATH}; @@ -168,6 +171,32 @@ struct CacheControlHeader { bool _immutable = true; }; +class ContentTypeHandler +{ +public: + ContentTypeHandler(std::string &resp_header_fields) : _resp_header_fields(resp_header_fields) {} + + // Returns false if _content_type_allowlist is not empty, and content-type field is either not present or not in the + // allowlist. Adds first Content-type field it encounters in the headers passed to this function. + // + bool nextObjectHeader(TSMBuffer bufp, TSMLoc hdr_loc); + + // Load allowlist from config file. + // + static void loadAllowList(std::string const &file_spec); + +private: + // Add Content-Type field to these. + // + std::string &_resp_header_fields; + + bool _added_content_type{false}; + + static vector _content_type_allowlist; +}; + +vector ContentTypeHandler::_content_type_allowlist; + bool InterceptData::init(TSVConn vconn) { @@ -309,7 +338,6 @@ static bool writeResponse(InterceptData &int_data); static bool writeErrorResponse(InterceptData &int_data, int &n_bytes_written); static bool writeStandardHeaderFields(InterceptData &int_data, int &n_bytes_written); static void prepareResponse(InterceptData &int_data, ByteBlockList &body_blocks, string &resp_header_fields); -static bool getContentType(TSMBuffer bufp, TSMLoc hdr_loc, string &resp_header_fields); static bool getDefaultBucket(TSHttpTxn txnp, TSMBuffer bufp, TSMLoc hdr_obj, ClientRequest &creq); // libesi TLS key. @@ -383,15 +411,29 @@ TSPluginInit(int argc, const char *argv[]) stringstream strstream(argv[optind++]); string header; while (getline(strstream, header, ':')) { - HEADER_WHITELIST.push_back(header); + HEADER_ALLOWLIST.push_back(header); } } ++optind; - for (unsigned int i = 0; i < HEADER_WHITELIST.size(); i++) { - LOG_DEBUG("WhiteList: %s", HEADER_WHITELIST[i].c_str()); + for (unsigned int i = 0; i < HEADER_ALLOWLIST.size(); i++) { + LOG_DEBUG("AllowList: %s", HEADER_ALLOWLIST[i].c_str()); } + std::string content_type_allowlist_filespec = (argc > optind && (argv[optind][0] != '-' || argv[optind][1])) ? argv[optind] : ""; + if (content_type_allowlist_filespec.empty()) { + LOG_DEBUG("No Content-Type allowlist file specified (all content types allowed)"); + } else { + // If we have a path and it's not an absolute path, make it relative to the + // configuration directory. + if (content_type_allowlist_filespec[0] != '/') { + content_type_allowlist_filespec = std::string(TSConfigDirGet()) + '/' + content_type_allowlist_filespec; + } + LOG_DEBUG("Content-Type allowlist file: %s", content_type_allowlist_filespec.c_str()); + ContentTypeHandler::loadAllowList(content_type_allowlist_filespec); + } + ++optind; + TSReleaseAssert(pthread_key_create(&threadKey, nullptr) == 0); TSCont rrh_contp = TSContCreate(handleReadRequestHeader, nullptr); @@ -402,7 +444,7 @@ TSPluginInit(int argc, const char *argv[]) TSHttpHookAdd(TS_HTTP_OS_DNS_HOOK, rrh_contp); - if (TSHttpTxnArgIndexReserve(DEBUG_TAG, "will save plugin-enable flag here", &arg_idx) != TS_SUCCESS) { + if (TSUserArgIndexReserve(TS_USER_ARGS_TXN, DEBUG_TAG, "will save plugin-enable flag here", &arg_idx) != TS_SUCCESS) { LOG_ERROR("failed to reserve private data slot"); return; } else { @@ -433,7 +475,7 @@ handleReadRequestHeader(TSCont /* contp ATS_UNUSED */, TSEvent event, void *edat return 0; } - if (1 != reinterpret_cast(TSHttpTxnArgGet(txnp, arg_idx))) { + if (1 != reinterpret_cast(TSUserArgGet(txnp, arg_idx))) { LOG_DEBUG("combo is disabled for this channel"); TSHttpTxnReenable(txnp, TS_EVENT_HTTP_CONTINUE); return 0; @@ -926,14 +968,12 @@ writeResponse(InterceptData &int_data) static void prepareResponse(InterceptData &int_data, ByteBlockList &body_blocks, string &resp_header_fields) { - bool got_content_type = false; - if (int_data.creq.status == TS_HTTP_STATUS_OK) { HttpDataFetcherImpl::ResponseData resp_data; TSMLoc field_loc; time_t expires_time; bool got_expires_time = false; - int num_headers = HEADER_WHITELIST.size(); + int num_headers = HEADER_ALLOWLIST.size(); int flags_list[num_headers]; CacheControlHeader cch; @@ -941,12 +981,16 @@ prepareResponse(InterceptData &int_data, ByteBlockList &body_blocks, string &res flags_list[i] = 0; } + ContentTypeHandler cth(resp_header_fields); + for (StringList::iterator iter = int_data.creq.file_urls.begin(); iter != int_data.creq.file_urls.end(); ++iter) { if (int_data.fetcher->getData(*iter, resp_data) && resp_data.status == TS_HTTP_STATUS_OK) { body_blocks.push_back(ByteBlock(resp_data.content, resp_data.content_len)); - if (find(HEADER_WHITELIST.begin(), HEADER_WHITELIST.end(), TS_MIME_FIELD_CONTENT_TYPE) == HEADER_WHITELIST.end()) { - if (!got_content_type) { - got_content_type = getContentType(resp_data.bufp, resp_data.hdr_loc, resp_header_fields); + if (find(HEADER_ALLOWLIST.begin(), HEADER_ALLOWLIST.end(), TS_MIME_FIELD_CONTENT_TYPE) == HEADER_ALLOWLIST.end()) { + if (!cth.nextObjectHeader(resp_data.bufp, resp_data.hdr_loc)) { + LOG_ERROR("Content type missing or forbidden for requested URL [%s]", iter->c_str()); + int_data.creq.status = TS_HTTP_STATUS_FORBIDDEN; + break; } } @@ -974,7 +1018,7 @@ prepareResponse(InterceptData &int_data, ByteBlockList &body_blocks, string &res continue; } - const string &header = HEADER_WHITELIST[i]; + const string &header = HEADER_ALLOWLIST[i]; field_loc = TSMimeHdrFieldFind(resp_data.bufp, resp_data.hdr_loc, header.c_str(), header.size()); if (field_loc != TS_NULL_MLOC) { @@ -1010,16 +1054,17 @@ prepareResponse(InterceptData &int_data, ByteBlockList &body_blocks, string &res } if (int_data.creq.status == TS_HTTP_STATUS_OK) { // Add in Cache-Control header - if (find(HEADER_WHITELIST.begin(), HEADER_WHITELIST.end(), TS_MIME_FIELD_CACHE_CONTROL) == HEADER_WHITELIST.end()) { + if (find(HEADER_ALLOWLIST.begin(), HEADER_ALLOWLIST.end(), TS_MIME_FIELD_CACHE_CONTROL) == HEADER_ALLOWLIST.end()) { resp_header_fields.append(cch.generate()); } - if (find(HEADER_WHITELIST.begin(), HEADER_WHITELIST.end(), TS_MIME_FIELD_EXPIRES) == HEADER_WHITELIST.end()) { + if (find(HEADER_ALLOWLIST.begin(), HEADER_ALLOWLIST.end(), TS_MIME_FIELD_EXPIRES) == HEADER_ALLOWLIST.end()) { if (got_expires_time) { if (expires_time <= 0) { resp_header_fields.append("Expires: 0\r\n"); } else { char line_buf[128]; - int line_size = strftime(line_buf, 128, "Expires: %a, %d %b %Y %T GMT\r\n", gmtime(&expires_time)); + struct tm gm_expires_time; + int line_size = strftime(line_buf, 128, "Expires: %a, %d %b %Y %T GMT\r\n", gmtime_r(&expires_time, &gm_expires_time)); resp_header_fields.append(line_buf, line_size); } } @@ -1040,10 +1085,9 @@ prepareResponse(InterceptData &int_data, ByteBlockList &body_blocks, string &res } } -static bool -getContentType(TSMBuffer bufp, TSMLoc hdr_loc, string &resp_header_fields) +bool +ContentTypeHandler::nextObjectHeader(TSMBuffer bufp, TSMLoc hdr_loc) { - bool retval = false; TSMLoc field_loc = TSMimeHdrFieldFind(bufp, hdr_loc, TS_MIME_FIELD_CONTENT_TYPE, TS_MIME_LEN_CONTENT_TYPE); if (field_loc != TS_NULL_MLOC) { bool values_added = false; @@ -1052,21 +1096,86 @@ getContentType(TSMBuffer bufp, TSMLoc hdr_loc, string &resp_header_fields) int n_values = TSMimeHdrFieldValuesCount(bufp, hdr_loc, field_loc); for (int i = 0; i < n_values; ++i) { value = TSMimeHdrFieldValueStringGet(bufp, hdr_loc, field_loc, i, &value_len); - if (!values_added) { - resp_header_fields.append("Content-Type: "); - values_added = true; - } else { - resp_header_fields.append(", "); + ts::TextView tv{value, value_len}; + tv = tv.prefix(';').rtrim(std::string_view(" \t")); + if (_content_type_allowlist.empty()) { + ; + } else if (std::find_if(_content_type_allowlist.begin(), _content_type_allowlist.end(), [tv](ts::TextView tv2) -> bool { + return strcasecmp(tv, tv2) == 0; + }) == _content_type_allowlist.end()) { + return false; + } else if (tv.empty()) { + // allowlist is bad, contains an empty string. + return false; + } + if (!_added_content_type) { + if (!values_added) { + _resp_header_fields.append("Content-Type: "); + values_added = true; + } else { + _resp_header_fields.append(", "); + } + _resp_header_fields.append(value, value_len); } - resp_header_fields.append(value, value_len); } TSHandleMLocRelease(bufp, hdr_loc, field_loc); if (values_added) { - resp_header_fields.append("\r\n"); - retval = true; + _resp_header_fields.append("\r\n"); + + // Assume that the Content-type field from the first header covers all the responses being combined. + _added_content_type = true; } + return true; } - return retval; + // No content type header field so doesn't pass allowlist if there is one. + return _content_type_allowlist.empty(); +} + +void +ContentTypeHandler::loadAllowList(std::string const &file_spec) +{ + std::fstream fs; + char line_buffer[256]; + bool extra_junk_on_line{false}; + int line_num = 0; + + fs.open(file_spec); + if (fs.good()) { + for (;;) { + ++line_num; + fs.getline(line_buffer, sizeof(line_buffer)); + if (!fs.good()) { + break; + } + constexpr std::string_view bs{" \t"sv}; + ts::TextView line{line_buffer, std::size_t(fs.gcount() - 1)}; + line.ltrim(bs); + if (line.empty() || line[0] == '#') { + // Empty/comment line. + continue; + } + ts::TextView content_type{line.take_prefix_at(bs)}; + line.trim(bs); + if (line.size() && (line[0] != '#')) { + extra_junk_on_line = true; + break; + } + _content_type_allowlist.emplace_back(content_type); + } + } + if (fs.fail() && !(fs.eof() && (fs.gcount() == 0))) { + LOG_ERROR("Error reading Content-Type allowlist config file %s, line %d", file_spec.c_str(), line_num); + } else if (extra_junk_on_line) { + LOG_ERROR("More than one type on line %d in Content-Type allowlist config file %s", line_num, file_spec.c_str()); + } else if (_content_type_allowlist.empty()) { + LOG_ERROR("Content-type allowlist config file %s must have at least one entry", file_spec.c_str()); + } else { + // End of file. + return; + } + _content_type_allowlist.clear(); + // An empty string marks object as bad. + _content_type_allowlist.emplace_back(""); } static const char INVARIANT_FIELD_LINES[] = {"Vary: Accept-Encoding\r\n"}; @@ -1075,7 +1184,7 @@ static const char INVARIANT_FIELD_LINES_SIZE = sizeof(INVARIANT_FIELD_LINES) - 1 static bool writeStandardHeaderFields(InterceptData &int_data, int &n_bytes_written) { - if (find(HEADER_WHITELIST.begin(), HEADER_WHITELIST.end(), TS_MIME_FIELD_VARY) == HEADER_WHITELIST.end()) { + if (find(HEADER_ALLOWLIST.begin(), HEADER_ALLOWLIST.end(), TS_MIME_FIELD_VARY) == HEADER_ALLOWLIST.end()) { if (TSIOBufferWrite(int_data.output.buffer, INVARIANT_FIELD_LINES, INVARIANT_FIELD_LINES_SIZE) == TS_ERROR) { LOG_ERROR("Error while writing invariant fields"); return false; @@ -1083,10 +1192,12 @@ writeStandardHeaderFields(InterceptData &int_data, int &n_bytes_written) n_bytes_written += INVARIANT_FIELD_LINES_SIZE; } - if (find(HEADER_WHITELIST.begin(), HEADER_WHITELIST.end(), TS_MIME_FIELD_LAST_MODIFIED) == HEADER_WHITELIST.end()) { + if (find(HEADER_ALLOWLIST.begin(), HEADER_ALLOWLIST.end(), TS_MIME_FIELD_LAST_MODIFIED) == HEADER_ALLOWLIST.end()) { time_t time_now = static_cast(TShrtime() / 1000000000); // it returns nanoseconds! char last_modified_line[128]; - int last_modified_line_size = strftime(last_modified_line, 128, "Last-Modified: %a, %d %b %Y %T GMT\r\n", gmtime(&time_now)); + struct tm gmnow; + int last_modified_line_size = + strftime(last_modified_line, 128, "Last-Modified: %a, %d %b %Y %T GMT\r\n", gmtime_r(&time_now, &gmnow)); if (TSIOBufferWrite(int_data.output.buffer, last_modified_line, last_modified_line_size) == TS_ERROR) { LOG_ERROR("Error while writing last-modified fields"); return false; @@ -1123,8 +1234,8 @@ writeErrorResponse(InterceptData &int_data, int &n_bytes_written) TSRemapStatus TSRemapDoRemap(void *ih, TSHttpTxn rh, TSRemapRequestInfo *rri) { - TSHttpTxnArgSet(rh, arg_idx, (void *)1); /* Save for later hooks */ - return TSREMAP_NO_REMAP; /* Continue with next remap plugin in chain */ + TSUserArgSet(rh, arg_idx, (void *)1); /* Save for later hooks */ + return TSREMAP_NO_REMAP; /* Continue with next remap plugin in chain */ } /* diff --git a/plugins/esi/esi.cc b/plugins/esi/esi.cc index cf02b3b4640..0a01c9b091e 100644 --- a/plugins/esi/esi.cc +++ b/plugins/esi/esi.cc @@ -61,7 +61,7 @@ struct OptionInfo { }; static HandlerManager *gHandlerManager = nullptr; -static Utils::HeaderValueList gWhitelistCookies; +static Utils::HeaderValueList gAllowlistCookies; #define DEBUG_TAG "plugin_esi" #define PROCESSOR_DEBUG_TAG "plugin_esi_processor" @@ -256,7 +256,7 @@ ContData::init() data_fetcher = new HttpDataFetcherImpl(contp, client_addr, createDebugTag(FETCHER_DEBUG_TAG, contp, fetcher_tag)); } if (!esi_vars) { - esi_vars = new Variables(createDebugTag(VARS_DEBUG_TAG, contp, vars_tag), &TSDebug, &TSError, gWhitelistCookies); + esi_vars = new Variables(createDebugTag(VARS_DEBUG_TAG, contp, vars_tag), &TSDebug, &TSError, gAllowlistCookies); } esi_proc = new EsiProcessor( @@ -290,7 +290,7 @@ ContData::getClientState() if (!esi_vars) { string vars_tag; - esi_vars = new Variables(createDebugTag(VARS_DEBUG_TAG, contp, vars_tag), &TSDebug, &TSError, gWhitelistCookies); + esi_vars = new Variables(createDebugTag(VARS_DEBUG_TAG, contp, vars_tag), &TSDebug, &TSError, gAllowlistCookies); } if (!data_fetcher) { string fetcher_tag; @@ -1212,7 +1212,7 @@ maskOsCacheHeaders(TSHttpTxn txnp) TSMLoc field_loc; const char *name, *value; int name_len, value_len, n_field_values; - bool os_response_cacheable, is_cache_header, mask_header; + bool os_response_cacheable, mask_header; string masked_name; os_response_cacheable = true; for (int i = 0; i < n_mime_headers; ++i) { @@ -1223,14 +1223,14 @@ maskOsCacheHeaders(TSHttpTxn txnp) } name = TSMimeHdrFieldNameGet(bufp, hdr_loc, field_loc, &name_len); if (name) { - mask_header = is_cache_header = false; - n_field_values = TSMimeHdrFieldValuesCount(bufp, hdr_loc, field_loc); + mask_header = false; + n_field_values = TSMimeHdrFieldValuesCount(bufp, hdr_loc, field_loc); for (int j = 0; j < n_field_values; ++j) { value = TSMimeHdrFieldValueStringGet(bufp, hdr_loc, field_loc, j, &value_len); if (nullptr == value || !value_len) { TSDebug(DEBUG_TAG, "[%s] Error while getting value #%d of header [%.*s]", __FUNCTION__, j, name_len, name); } else { - is_cache_header = checkForCacheHeader(name, name_len, value, value_len, os_response_cacheable); + bool is_cache_header = checkForCacheHeader(name, name_len, value, value_len, os_response_cacheable); if (!os_response_cacheable) { break; } @@ -1567,7 +1567,7 @@ loadHandlerConf(const char *file_name, Utils::KeyValueMap &handler_conf) conf_lines.push_back(string(buf)); } TSfclose(conf_file); - Utils::parseKeyValueConfig(conf_lines, handler_conf, gWhitelistCookies); + Utils::parseKeyValueConfig(conf_lines, handler_conf, gAllowlistCookies); TSDebug(DEBUG_TAG, "[%s] Loaded handler conf file [%s]", __FUNCTION__, file_name); } else { TSError("[esi][%s] Failed to open handler config file [%s]", __FUNCTION__, file_name); diff --git a/plugins/esi/lib/Utils.cc b/plugins/esi/lib/Utils.cc index 9af3c64267d..cfd0f846408 100644 --- a/plugins/esi/lib/Utils.cc +++ b/plugins/esi/lib/Utils.cc @@ -111,7 +111,7 @@ Utils::getAttribute(const string &data, const string &attr, size_t curr_pos, siz } void -Utils::parseKeyValueConfig(const std::list &lines, KeyValueMap &kvMap, HeaderValueList &whitelistCookies) +Utils::parseKeyValueConfig(const std::list &lines, KeyValueMap &kvMap, HeaderValueList &allowlistCookies) { string key, value; std::istringstream iss; @@ -125,8 +125,8 @@ Utils::parseKeyValueConfig(const std::list &lines, KeyValueMap &kvMap, H if (iss.good()) { iss >> key; iss >> value; - if (key == "whitelistCookie") { - whitelistCookies.push_back(value); + if (key == "allowlistCookie") { + allowlistCookies.push_back(value); continue; } if (key.size() && value.size()) { diff --git a/plugins/esi/lib/Utils.h b/plugins/esi/lib/Utils.h index c7c412534ee..c093441f4e6 100644 --- a/plugins/esi/lib/Utils.h +++ b/plugins/esi/lib/Utils.h @@ -106,8 +106,8 @@ namespace Utils // parses given lines (assumes format) and // stores them in supplied map; Lines beginning with '#' are ignored - // also if line starts with "whitelistCookie", we store next token in a list - void parseKeyValueConfig(const std::list &lines, KeyValueMap &kvMap, HeaderValueList &whitelistCookies); + // also if line starts with "allowlistCookie", we store next token in a list + void parseKeyValueConfig(const std::list &lines, KeyValueMap &kvMap, HeaderValueList &allowlistCookies); inline std::string unescape(const char *str, int len = -1) diff --git a/plugins/esi/lib/Variables.cc b/plugins/esi/lib/Variables.cc index adec87e72e1..8fe12f41bd3 100644 --- a/plugins/esi/lib/Variables.cc +++ b/plugins/esi/lib/Variables.cc @@ -371,8 +371,8 @@ Variables::_parseCookieString(const char *str, int str_len) } bool found = false; - for (auto &_whitelistCookie : _whitelistCookies) { - if ((_whitelistCookie == "*") || (_whitelistCookie == cookie)) { + for (auto &_allowlistCookie : _allowlistCookies) { + if ((_allowlistCookie == "*") || (_allowlistCookie == cookie)) { found = true; } } diff --git a/plugins/esi/lib/Variables.h b/plugins/esi/lib/Variables.h index 18fb8457065..c747ba9d216 100644 --- a/plugins/esi/lib/Variables.h +++ b/plugins/esi/lib/Variables.h @@ -37,14 +37,14 @@ class Variables : private ComponentBase { public: Variables(const char *debug_tag, ComponentBase::Debug debug_func, ComponentBase::Error error_func, - Utils::HeaderValueList whitelistCookies) + Utils::HeaderValueList allowlistCookies) : ComponentBase(debug_tag, debug_func, error_func), _headers_parsed(false), _query_string(""), _query_string_parsed(false), _cookie_jar_created(false) { - _whitelistCookies.insert(_whitelistCookies.end(), whitelistCookies.begin(), whitelistCookies.end()); + _allowlistCookies.insert(_allowlistCookies.end(), allowlistCookies.begin(), allowlistCookies.end()); }; /** currently 'host', 'referer', 'accept-language', 'cookie' and 'user-agent' headers are parsed */ @@ -150,7 +150,7 @@ class Variables : private ComponentBase Utils::HeaderValueList _cached_simple_headers[N_SIMPLE_HEADERS]; Utils::HeaderValueList _cached_special_headers[N_SPECIAL_HEADERS]; - Utils::HeaderValueList _whitelistCookies; + Utils::HeaderValueList _allowlistCookies; std::string _cookie_str; bool _headers_parsed; std::string _query_string; diff --git a/plugins/esi/test/processor_test.cc b/plugins/esi/test/processor_test.cc index 850ce5bae9c..f08abfaece5 100644 --- a/plugins/esi/test/processor_test.cc +++ b/plugins/esi/test/processor_test.cc @@ -44,8 +44,8 @@ static const int FETCHER_STATIC_DATA_SIZE = 30; int main() { - Utils::HeaderValueList whitelistCookies; - Variables esi_vars("vars", &Debug, &Error, whitelistCookies); + Utils::HeaderValueList allowlistCookies; + Variables esi_vars("vars", &Debug, &Error, allowlistCookies); HandlerManager handler_mgr("handler_mgr", &Debug, &Error); pthread_key_create(&threadKey, nullptr); diff --git a/plugins/esi/test/utils_test.cc b/plugins/esi/test/utils_test.cc index 481bfbc9a87..288a59aa847 100644 --- a/plugins/esi/test/utils_test.cc +++ b/plugins/esi/test/utils_test.cc @@ -121,8 +121,8 @@ main() cout << "Test 11 " << endl; std::list lines; - lines.push_back("whitelistCookie AGE"); - lines.push_back("whitelistCookie GRADE"); + lines.push_back("allowlistCookie AGE"); + lines.push_back("allowlistCookie GRADE"); lines.push_back("a b"); Utils::KeyValueMap kv; Utils::HeaderValueList list; diff --git a/plugins/esi/test/vars_test.cc b/plugins/esi/test/vars_test.cc index 0bd11b78521..4a313bc5cb2 100644 --- a/plugins/esi/test/vars_test.cc +++ b/plugins/esi/test/vars_test.cc @@ -75,13 +75,13 @@ main() { cout << endl << "===================== Test 1" << endl; - Utils::HeaderValueList whitelistCookies; - whitelistCookies.push_back("c1"); - whitelistCookies.push_back("c2"); - whitelistCookies.push_back("c3"); - whitelistCookies.push_back("c4"); - whitelistCookies.push_back("c5"); - Variables esi_vars("vars_test", &Debug, &Error, whitelistCookies); + Utils::HeaderValueList allowlistCookies; + allowlistCookies.push_back("c1"); + allowlistCookies.push_back("c2"); + allowlistCookies.push_back("c3"); + allowlistCookies.push_back("c4"); + allowlistCookies.push_back("c5"); + Variables esi_vars("vars_test", &Debug, &Error, allowlistCookies); const char *strings[] = {"Cookie", "; c1=v1; c2=v2; ; c3; c4=; c5=v5 ", "Host", @@ -314,8 +314,8 @@ main() { cout << endl << "===================== Test 2" << endl; gFakeDebugLog.assign(""); - Utils::HeaderValueList whitelistCookies; - Variables esi_vars("vars_test", &fakeDebug, &Error, whitelistCookies); + Utils::HeaderValueList allowlistCookies; + Variables esi_vars("vars_test", &fakeDebug, &Error, allowlistCookies); esi_vars.populate(HttpHeader("Host", -1, "example.com", -1)); esi_vars.populate(HttpHeader("Referer", -1, "google.com", -1)); @@ -344,17 +344,17 @@ main() { cout << endl << "===================== Test 3" << endl; - Utils::HeaderValueList whitelistCookies; - whitelistCookies.push_back("age"); - whitelistCookies.push_back("grade"); - whitelistCookies.push_back("avg"); - whitelistCookies.push_back("t1"); - whitelistCookies.push_back("t2"); - whitelistCookies.push_back("t3"); - whitelistCookies.push_back("t4"); - whitelistCookies.push_back("t5"); - whitelistCookies.push_back("c1"); - Variables esi_vars("vars_test", &Debug, &Error, whitelistCookies); + Utils::HeaderValueList allowlistCookies; + allowlistCookies.push_back("age"); + allowlistCookies.push_back("grade"); + allowlistCookies.push_back("avg"); + allowlistCookies.push_back("t1"); + allowlistCookies.push_back("t2"); + allowlistCookies.push_back("t3"); + allowlistCookies.push_back("t4"); + allowlistCookies.push_back("t5"); + allowlistCookies.push_back("c1"); + Variables esi_vars("vars_test", &Debug, &Error, allowlistCookies); esi_vars.populate(HttpHeader("Host", -1, "example.com", -1)); esi_vars.populate(HttpHeader("Referer", -1, "google.com", -1)); @@ -390,15 +390,15 @@ main() { cout << endl << "===================== Test 4" << endl; - Utils::HeaderValueList whitelistCookies; - whitelistCookies.push_back("FPS"); - whitelistCookies.push_back("mb"); - whitelistCookies.push_back("Y"); - whitelistCookies.push_back("C"); - whitelistCookies.push_back("F"); - whitelistCookies.push_back("a"); - whitelistCookies.push_back("c"); - Variables esi_vars("vars_test", &Debug, &Error, whitelistCookies); + Utils::HeaderValueList allowlistCookies; + allowlistCookies.push_back("FPS"); + allowlistCookies.push_back("mb"); + allowlistCookies.push_back("Y"); + allowlistCookies.push_back("C"); + allowlistCookies.push_back("F"); + allowlistCookies.push_back("a"); + allowlistCookies.push_back("c"); + Variables esi_vars("vars_test", &Debug, &Error, allowlistCookies); string cookie_str("FPS=dl; mb=d=OPsv7rvU4FFaAOoIRi75BBuqdMdbMLFuDwQmk6nKrCgno7L4xuN44zm7QBQJRmQSh8ken6GSVk8-&v=1; C=mg=1; " "Y=v=1&n=fmaptagvuff50&l=fc0d94i7/o&p=m2f0000313000400&r=8j&lg=en-US&intl=us; " "F=a=4KvLV9IMvTJnIAqCk25y9Use6hnPALtUf3n78PihlcIqvmzoW.Ax8UyW8_oxtgFNrrdmooqZmPa7WsX4gE." @@ -440,8 +440,8 @@ main() { cout << endl << "===================== Test 5" << endl; - Utils::HeaderValueList whitelistCookies; - Variables esi_vars("vars_test", &Debug, &Error, whitelistCookies); + Utils::HeaderValueList allowlistCookies; + Variables esi_vars("vars_test", &Debug, &Error, allowlistCookies); esi_vars.populate(HttpHeader("hdr1", -1, "hval1", -1)); esi_vars.populate(HttpHeader("Hdr2", -1, "hval2", -1)); esi_vars.populate(HttpHeader("@Intenal-hdr1", -1, "internal-hval1", -1)); @@ -457,9 +457,9 @@ main() { cout << endl << "===================== Test 6" << endl; - Utils::HeaderValueList whitelistCookies; - whitelistCookies.push_back("*"); - Variables esi_vars("vars_test", &Debug, &Error, whitelistCookies); + Utils::HeaderValueList allowlistCookies; + allowlistCookies.push_back("*"); + Variables esi_vars("vars_test", &Debug, &Error, allowlistCookies); esi_vars.populate(HttpHeader("Host", -1, "example.com", -1)); esi_vars.populate(HttpHeader("Cookie", -1, "age=21; grade=-5; avg=4.3; t1=\" \"; t2=0.0", -1)); diff --git a/plugins/experimental/access_control/access_control.cc b/plugins/experimental/access_control/access_control.cc index 7401c5fc8b1..3f71c0f0b97 100644 --- a/plugins/experimental/access_control/access_control.cc +++ b/plugins/experimental/access_control/access_control.cc @@ -164,7 +164,7 @@ AccessToken::validateTiming(time_t time) } /* "issued at" time-stamp is currently only for info, so check if the time-stamp is valid only */ - if (!_issuedAt.empty() && 0 == (t = string2int(_issuedAt))) { + if (!_issuedAt.empty() && 0 == string2int(_issuedAt)) { return _state = INVALID_FIELD_VALUE; } diff --git a/plugins/experimental/access_control/config.cc b/plugins/experimental/access_control/config.cc index 4e08ffbc5d5..b401e6fc838 100644 --- a/plugins/experimental/access_control/config.cc +++ b/plugins/experimental/access_control/config.cc @@ -261,14 +261,14 @@ AccessControlConfig::init(int argc, char *argv[]) _useRedirects = ::isTrue(optarg); } break; case 'o': /* include-uri-paths-file */ - if (!loadMultiPatternsFromFile(optarg, /* blacklist = */ false)) { - AccessControlError("failed to load uri-path multi-pattern white-list '%s'", optarg); + if (!loadMultiPatternsFromFile(optarg, /* denylist = */ false)) { + AccessControlError("failed to load uri-path multi-pattern allow-list '%s'", optarg); status = false; } break; case 'p': /* exclude-uri-paths-file */ - if (!loadMultiPatternsFromFile(optarg, /* blacklist = */ true)) { - AccessControlError("failed to load uri-path multi-pattern black-list '%s'", optarg); + if (!loadMultiPatternsFromFile(optarg, /* denylist = */ true)) { + AccessControlError("failed to load uri-path multi-pattern deny-list '%s'", optarg); status = false; } break; @@ -297,11 +297,11 @@ AccessControlConfig::init(int argc, char *argv[]) /** * @brief a helper function which loads the classifier from files. * @param filename file name - * @param blacklist true - load as a blacklist of patterns, false - white-list of patterns + * @param denylist true - load as a denylist of patterns, false - allow-list of patterns * @return true if successful, false otherwise. */ bool -AccessControlConfig::loadMultiPatternsFromFile(const String &filename, bool blacklist) +AccessControlConfig::loadMultiPatternsFromFile(const String &filename, bool denylist) { if (filename.empty()) { AccessControlError("filename cannot be empty"); @@ -322,7 +322,7 @@ AccessControlConfig::loadMultiPatternsFromFile(const String &filename, bool blac /* Have the multiplattern be named as same as the filename, would be used only for debugging. */ MultiPattern *multiPattern; - if (blacklist) { + if (denylist) { multiPattern = new NonMatchingMultiPattern(filename); AccessControlDebug("NonMatchingMultiPattern('%s')", filename.c_str()); } else { @@ -355,11 +355,11 @@ AccessControlConfig::loadMultiPatternsFromFile(const String &filename, bool blac p = new Pattern(); if (nullptr != p && p->init(regex)) { - if (blacklist) { - AccessControlDebug("Added pattern '%s' to black list uri-path multi-pattern '%s'", regex.c_str(), filename.c_str()); + if (denylist) { + AccessControlDebug("Added pattern '%s' to deny list uri-path multi-pattern '%s'", regex.c_str(), filename.c_str()); multiPattern->add(p); } else { - AccessControlDebug("Added pattern '%s' to white list uri-path multi-pattern '%s'", regex.c_str(), filename.c_str()); + AccessControlDebug("Added pattern '%s' to allow list uri-path multi-pattern '%s'", regex.c_str(), filename.c_str()); multiPattern->add(p); } } else { diff --git a/plugins/experimental/access_control/config.h b/plugins/experimental/access_control/config.h index 4d1ab9e0a52..1391a4a9d02 100644 --- a/plugins/experimental/access_control/config.h +++ b/plugins/experimental/access_control/config.h @@ -38,7 +38,7 @@ class AccessControlConfig virtual ~AccessControlConfig() { delete _tokenFactory; } bool init(int argc, char *argv[]); - bool loadMultiPatternsFromFile(const String &filename, bool blacklist = true); + bool loadMultiPatternsFromFile(const String &filename, bool denylist = true); StringMap _symmetricKeysMap; /** @brief a map secrets accessible by key string (KID) */ @@ -66,5 +66,5 @@ class AccessControlConfig String _extrTokenIdHdrName; /** @brief header name to extract the token id, if empty => no extraction */ String _extrValidationHdrName; /** @brief header name to extract the token validation status, if empty => no extraction */ bool _useRedirects = false; /** @brief true - use redirect to set the access token cookie, @todo not used yet */ - Classifier _uriPathScope; /**< @brief blacklist (exclude) and white-list (include) which path should have the access control */ + Classifier _uriPathScope; /**< @brief denylist (exclude) and allow-list (include) which path should have the access control */ }; diff --git a/plugins/experimental/access_control/plugin.cc b/plugins/experimental/access_control/plugin.cc index aab28d36168..38d062102b0 100644 --- a/plugins/experimental/access_control/plugin.cc +++ b/plugins/experimental/access_control/plugin.cc @@ -385,7 +385,7 @@ contHandleAccessControl(const TSCont contp, TSEvent event, void *edata) /* Secure - instructs the UA to include the cookie in an HTTP request only if the request is transmitted over * a secure channel, typically HTTP over Transport Layer Security (TLS) - * HttpOnly - instructs the UA to omit the cookie when providing access to cookies via “non-HTTP” APIs such as a web + * HttpOnly - instructs the UA to omit the cookie when providing access to cookies via "non-HTTP" APIs such as a web * browser API that exposes cookies to scripts */ cookieValue.append("path=/; Secure; HttpOnly"); diff --git a/plugins/experimental/cert_reporting_tool/cert_reporting_tool.cc b/plugins/experimental/cert_reporting_tool/cert_reporting_tool.cc index 0103418b2a7..fd6ab40b5d9 100644 --- a/plugins/experimental/cert_reporting_tool/cert_reporting_tool.cc +++ b/plugins/experimental/cert_reporting_tool/cert_reporting_tool.cc @@ -115,7 +115,7 @@ dump_context(const char *ca_path, const char *ck_path) // Serial number int64_t sn = 0; -#if OPENSSL_VERSION_NUMBER >= 0x010100000 +#if !defined(OPENSSL_IS_BORINGSSL) && (OPENSSL_VERSION_NUMBER >= 0x010100000) ASN1_INTEGER_get_int64(&sn, serial); #else sn = ASN1_INTEGER_get(serial); diff --git a/plugins/experimental/cookie_remap/cookie_remap.cc b/plugins/experimental/cookie_remap/cookie_remap.cc index b5eff3e06be..421c52d6c04 100644 --- a/plugins/experimental/cookie_remap/cookie_remap.cc +++ b/plugins/experimental/cookie_remap/cookie_remap.cc @@ -178,7 +178,6 @@ void urlencode(std::string &str) { size_t pos = 0; - std::string replacement; for (; pos < str.length(); pos++) { if (!isalnum(str[pos])) { char dec[2]; @@ -765,8 +764,6 @@ using OpMap = std::vector; static bool build_op(op &o, OpMap const &q) { - StringPair m; - subop *sub = new subop(); // loop through the array of key->value pairs @@ -873,10 +870,10 @@ TSRemapNewInstance(int argc, char *argv[], void **ih, char *errbuf, int errbuf_s } for (YAML::const_iterator it2 = it->second.begin(); it2 != it->second.end(); ++it2) { - const YAML::Node &first = it2->first; - const YAML::Node &second = it2->second; + const YAML::Node first = it2->first; + const YAML::Node second = it2->second; - if (second.Type() != YAML::NodeType::Scalar) { + if (second.IsScalar() == false) { const string reason = "All op nodes must be of type scalar"; TSError("Invalid YAML Configuration format for cookie_remap: %s, reason: %s", filename.c_str(), reason.c_str()); return TS_ERROR; diff --git a/plugins/experimental/cookie_remap/cookiejar.cc b/plugins/experimental/cookie_remap/cookiejar.cc index 0b3a05a41dc..1f7d5d33dbe 100644 --- a/plugins/experimental/cookie_remap/cookiejar.cc +++ b/plugins/experimental/cookie_remap/cookiejar.cc @@ -82,14 +82,13 @@ CookieJar::parse(const string &arg, const char *sepstr, bool val_check, bool mai cp = arg_copy; char empty[] = ""; for (key = strsep(&cp, sepstr); key != nullptr; key = strsep(&cp, sepstr)) { - int val_len; char *val = strchr(key, '='); char *addme = nullptr; if (val) { /* split key and value */ - *val++ = '\0'; - val_len = strlen(val); + *val++ = '\0'; + int val_len = strlen(val); if (val_len > 0) { /* if we have DQUOTES around our value then drop them */ diff --git a/plugins/experimental/cookie_remap/cookiejar.h b/plugins/experimental/cookie_remap/cookiejar.h index 34e9907317b..25475817f2e 100644 --- a/plugins/experimental/cookie_remap/cookiejar.h +++ b/plugins/experimental/cookie_remap/cookiejar.h @@ -16,8 +16,7 @@ limitations under the License. */ -#ifndef CKREMAP_COOKIEJAR_H_ -#define CKREMAP_COOKIEJAR_H_ +#pragma once #include #include @@ -58,5 +57,3 @@ class CookieJar unordered_map m_jar; }; - -#endif // CKREMAP_COOKIEJAR_H_ diff --git a/plugins/experimental/cookie_remap/hash.h b/plugins/experimental/cookie_remap/hash.h index 6d6c94131bb..18375b3ecb9 100644 --- a/plugins/experimental/cookie_remap/hash.h +++ b/plugins/experimental/cookie_remap/hash.h @@ -16,8 +16,7 @@ limitations under the License. */ -#ifndef _CKREMAP_HASH_H_ -#define _CKREMAP_HASH_H_ +#pragma once #include // NOLINT(modernize-deprecated-headers) #include @@ -50,5 +49,3 @@ uint32_t hash_fnv32_buckets(const char *buf, size_t len, uint32_t num_buckets); #ifdef __cplusplus } #endif - -#endif /* _CKREMAP_HASH_H_ */ diff --git a/plugins/experimental/cookie_remap/strip.h b/plugins/experimental/cookie_remap/strip.h index 5a6e97c3551..67af1d08f13 100644 --- a/plugins/experimental/cookie_remap/strip.h +++ b/plugins/experimental/cookie_remap/strip.h @@ -16,8 +16,7 @@ limitations under the License. */ -#ifndef CKREMAP_IV_H -#define CKREMAP_IV_H +#pragma once #include @@ -142,5 +141,3 @@ int get_stripped(const char *in, ssize_t in_len, char *out, int *out_len, unsign #ifdef __cplusplus } #endif - -#endif /* CKREMAP_IV_H */ diff --git a/plugins/experimental/fastcgi/src/Readme b/plugins/experimental/fastcgi/src/Readme index f9771460749..24cb8c6085e 100644 --- a/plugins/experimental/fastcgi/src/Readme +++ b/plugins/experimental/fastcgi/src/Readme @@ -14,11 +14,11 @@ Extract the file and set the flag "C_Cpp.clang_format_path" to file path. 2. Adding pre-commit hook : Copy the file path/to/trafficserver/tools/git/pre-commit under .git/hook/ directory. This will does the formatting every time does a commit. - + ----------------------------------------------------------------------------- ----------------------------------------------------------------------------- -Build and Install cmd = +Build and Install cmd = #building CPP ats_fastcgi.so @@ -40,7 +40,7 @@ tsxs -o ats_fastcgi.so \ # (for the case where the sudo context doesn't know about tsxs) mtsxs=$(which tsxs) -sudo $mtsxs -o ats_fastcgi.so -i +sudo $mtsxs -o ats_fastcgi.so -i ``` OR Build and install using Makefile utility: @@ -50,7 +50,7 @@ OR Build and install using Makefile utility: Afterwords command, $> make && sudo -E make install For example: - make ATS_SRC=/home/oschaaf/code/apache/trafficserver ATS_EXEC=/usr/local/ CPPFLAGS=-std=c++11 + make ATS_SRC=/home/oschaaf/code/apache/trafficserver ATS_EXEC=/usr/local/ CPPFLAGS=-std=c++11 ------------------------------------------------------------------------------ @@ -60,8 +60,8 @@ ATS (Apache Traffic Server) FastCGI Plugin This plugin collapses connections with identical CacheUrl/EffectiveUrl. If an entry was created for a given CacheUrl/EffectiveUrl in our global hashTable, -successive GET requests with identical CacheUrl/EffectiveUrl will be blocked in -POST_REMAP hook until the hash entry was removed. (POST_REMAP hook is the last +successive GET requests with identical CacheUrl/EffectiveUrl will be blocked in +POST_REMAP hook until the hash entry was removed. (POST_REMAP hook is the last hook before 'cache lookup') For requests going into 'cache lookup' stage, @@ -75,7 +75,7 @@ For requests going into 'read server response header' stage, if proxy.config.cache.enable_read_while_writer is enabled, hash entry will be removed at earliest possible time. else - hash entry will be removed in TXN_CLOSE hook. + hash entry will be removed in TXN_CLOSE hook. if response is not public cacheable, we will update hash entry with a special value to let successive requests pass collapsed check, this special hash entry will be removed in TXN_CLOSE hook if value of keep_pass_record_time is 0, @@ -122,7 +122,7 @@ Meaning of these configurable options: fcgi server port number in string format e.g 60000 root_directory root directory path from where fcgi server will server resources/web_contents - min_connections + min_connections min number of connections plugin will open to the php server max_connections max number of connections plugin will make to the php server @@ -145,7 +145,7 @@ Meaning of these configurable options: and require the C++11 standard. a. download libfcgi-dev package :i.e libfcgi-dev_2.4.0-8.4+b1_arm64.deb from https://packages.debian.org/sid/arm64/libfcgi-dev/download - + b. download php7.0-cli php7.0-fpm with : sudo apt-get install php7.0-cli php7.0-fpm @@ -166,4 +166,3 @@ Using Profiler: #if ATS_FCGI_PROFILER ats_plugin::ProfileTaker profile_taker(&profiler, "functinName", (std::size_t)&plugin, "B"); #endif - \ No newline at end of file diff --git a/plugins/experimental/geoip_acl/acl.h b/plugins/experimental/geoip_acl/acl.h index b8e6b8cf10d..7fa9387157e 100644 --- a/plugins/experimental/geoip_acl/acl.h +++ b/plugins/experimental/geoip_acl/acl.h @@ -15,6 +15,9 @@ See the License for the specific language governing permissions and limitations under the License. */ + +#pragma once + #include #include #include diff --git a/plugins/experimental/geoip_acl/lulu.h b/plugins/experimental/geoip_acl/lulu.h index c4c9cd99b5e..b76ce1ace2f 100644 --- a/plugins/experimental/geoip_acl/lulu.h +++ b/plugins/experimental/geoip_acl/lulu.h @@ -25,7 +25,6 @@ #include #include "tscore/ink_defs.h" -#include "tscore/ink_atomic.h" // Used for Debug etc. static const char *PLUGIN_NAME = "geoip_acl"; diff --git a/plugins/experimental/ja3_fingerprint/ja3_fingerprint.cc b/plugins/experimental/ja3_fingerprint/ja3_fingerprint.cc index 6b4ecfb0472..fe3a33e8705 100644 --- a/plugins/experimental/ja3_fingerprint/ja3_fingerprint.cc +++ b/plugins/experimental/ja3_fingerprint/ja3_fingerprint.cc @@ -302,7 +302,7 @@ client_hello_ja3_handler(TSCont contp, TSEvent event, void *edata) #error OpenSSL cannot be 1.1.0 #endif - TSSslConnection sslobj = TSVConnSSLConnectionGet(ssl_vc); + TSSslConnection sslobj = TSVConnSslConnectionGet(ssl_vc); // OpenSSL handle SSL *ssl = reinterpret_cast(sslobj); @@ -311,7 +311,7 @@ client_hello_ja3_handler(TSCont contp, TSEvent event, void *edata) data->ja3_string.append(custom_get_ja3(ssl)); getIP(TSNetVConnRemoteAddrGet(ssl_vc), data->ip_addr); - TSVConnArgSet(ssl_vc, ja3_idx, static_cast(data)); + TSUserArgSet(ssl_vc, ja3_idx, static_cast(data)); TSDebug(PLUGIN_NAME, "client_hello_ja3_handler(): JA3: %s", data->ja3_string.c_str()); // MD5 hash @@ -326,14 +326,14 @@ client_hello_ja3_handler(TSCont contp, TSEvent event, void *edata) } case TS_EVENT_VCONN_CLOSE: { // Clean up - ja3_data *data = static_cast(TSVConnArgGet(ssl_vc, ja3_idx)); + ja3_data *data = static_cast(TSUserArgGet(ssl_vc, ja3_idx)); if (data == nullptr) { TSDebug(PLUGIN_NAME, "client_hello_ja3_handler(): Failed to retrieve ja3 data at VCONN_CLOSE."); return TS_ERROR; } - TSVConnArgSet(ssl_vc, ja3_idx, nullptr); + TSUserArgSet(ssl_vc, ja3_idx, nullptr); delete data; break; @@ -361,7 +361,7 @@ req_hdr_ja3_handler(TSCont contp, TSEvent event, void *edata) } // Retrieve ja3_data from vconn args - ja3_data *data = static_cast(TSVConnArgGet(vconn, ja3_idx)); + ja3_data *data = static_cast(TSUserArgGet(vconn, ja3_idx)); if (data) { // Decide global or remap ja3_remap_info *info = static_cast(TSContDataGet(contp)); @@ -444,7 +444,7 @@ TSPluginInit(int argc, const char *argv[]) } // SNI handler TSCont ja3_cont = TSContCreate(client_hello_ja3_handler, nullptr); - TSVConnArgIndexReserve(PLUGIN_NAME, "used to pass ja3", &ja3_idx); + TSUserArgIndexReserve(TS_USER_ARGS_VCONN, PLUGIN_NAME, "used to pass ja3", &ja3_idx); #if OPENSSL_VERSION_NUMBER < 0x10100000L TSHttpHookAdd(TS_SSL_SERVERNAME_HOOK, ja3_cont); #elif OPENSSL_VERSION_NUMBER >= 0x10101000L @@ -473,7 +473,7 @@ TSRemapInit(TSRemapInterface *api_info, char *errbuf, int errbuf_size) // Set up SNI handler for all TLS connections TSCont ja3_cont = TSContCreate(client_hello_ja3_handler, nullptr); - TSVConnArgIndexReserve(PLUGIN_NAME, "Used to pass ja3", &ja3_idx); + TSUserArgIndexReserve(TS_USER_ARGS_VCONN, PLUGIN_NAME, "Used to pass ja3", &ja3_idx); #if OPENSSL_VERSION_NUMBER < 0x10100000L TSHttpHookAdd(TS_SSL_SERVERNAME_HOOK, ja3_cont); #elif OPENSSL_VERSION_NUMBER >= 0x10101000L diff --git a/plugins/experimental/magick/magick.cc b/plugins/experimental/magick/magick.cc index 00f4755a3de..d320d017045 100644 --- a/plugins/experimental/magick/magick.cc +++ b/plugins/experimental/magick/magick.cc @@ -184,7 +184,7 @@ struct EVPKey { EVPKey() : key(EVP_PKEY_new()) { assert(nullptr != key); } bool - assign(const char *const k) const + assign(char *k) const { assert(nullptr != k); const int rc = EVP_PKEY_assign_RSA(key, k); @@ -196,7 +196,7 @@ struct EVPKey { bool assign(T &t) { - return assign(reinterpret_cast(t)); + return assign(reinterpret_cast(t)); } }; @@ -352,7 +352,9 @@ struct QueryMap { QueryMap(std::string &&s) : content_(s) { parse(); } - template const Vector &operator[](T &&k) const + template + const Vector & + operator[](T &&k) const { const auto iterator = map_.find(k); if (iterator != map_.end()) { diff --git a/plugins/experimental/maxmind_acl/Makefile.inc b/plugins/experimental/maxmind_acl/Makefile.inc new file mode 100644 index 00000000000..5a613b5f77f --- /dev/null +++ b/plugins/experimental/maxmind_acl/Makefile.inc @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +pkglib_LTLIBRARIES += experimental/maxmind_acl/maxmind_acl.la + +experimental_maxmind_acl_maxmind_acl_la_SOURCES = \ + experimental/maxmind_acl/maxmind_acl.cc \ + experimental/maxmind_acl/mmdb.cc + +experimental_maxmind_acl_maxmind_acl_la_LIBADD = $(MAXMINDDB_LIBS) + +experimental_maxmind_acl_maxmind_acl_la_LDFLAGS = \ + $(AM_LDFLAGS) + +AM_CPPFLAGS += @YAMLCPP_INCLUDES@ diff --git a/plugins/experimental/maxmind_acl/maxmind_acl.cc b/plugins/experimental/maxmind_acl/maxmind_acl.cc new file mode 100644 index 00000000000..b3e4a355e09 --- /dev/null +++ b/plugins/experimental/maxmind_acl/maxmind_acl.cc @@ -0,0 +1,87 @@ +/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#include "mmdb.h" + +/////////////////////////////////////////////////////////////////////////////// +// Initialize the plugin as a remap plugin. +// +TSReturnCode +TSRemapInit(TSRemapInterface *api_info, char *errbuf, int errbuf_size) +{ + if (api_info->size < sizeof(TSRemapInterface)) { + strncpy(errbuf, "[tsremap_init] - Incorrect size of TSRemapInterface structure", errbuf_size - 1); + return TS_ERROR; + } + + if (api_info->tsremap_version < TSREMAP_VERSION) { + snprintf(errbuf, errbuf_size, "[tsremap_init] - Incorrect API version %ld.%ld", api_info->tsremap_version >> 16, + (api_info->tsremap_version & 0xffff)); + return TS_ERROR; + } + + TSDebug(PLUGIN_NAME, "remap plugin is successfully initialized"); + return TS_SUCCESS; +} + +TSReturnCode +TSRemapNewInstance(int argc, char *argv[], void **ih, char * /* errbuf */, int /* errbuf_size */) +{ + if (argc < 3) { + TSError("[%s] Unable to create remap instance, missing configuration file", PLUGIN_NAME); + return TS_ERROR; + } + + Acl *a = new Acl(); + *ih = static_cast(a); + if (!a->init(argv[2])) { + TSError("[%s] Failed to initialize maxmind with %s", PLUGIN_NAME, argv[2]); + return TS_ERROR; + } + + TSDebug(PLUGIN_NAME, "created remap instance with configuration %s", argv[2]); + return TS_SUCCESS; +} + +void +TSRemapDeleteInstance(void *ih) +{ + if (nullptr != ih) { + Acl *const a = static_cast(ih); + delete a; + } +} + +/////////////////////////////////////////////////////////////////////////////// +// Main entry point when used as a remap plugin. +// +TSRemapStatus +TSRemapDoRemap(void *ih, TSHttpTxn rh, TSRemapRequestInfo *rri) +{ + if (nullptr == ih) { + TSDebug(PLUGIN_NAME, "No ACLs configured"); + } else { + Acl *a = static_cast(ih); + if (!a->eval(rri, rh)) { + TSDebug(PLUGIN_NAME, "denying request"); + TSHttpTxnStatusSet(rh, TS_HTTP_STATUS_FORBIDDEN); + a->send_html(rh); + } + } + return TSREMAP_NO_REMAP; +} diff --git a/plugins/experimental/maxmind_acl/mmdb.cc b/plugins/experimental/maxmind_acl/mmdb.cc new file mode 100644 index 00000000000..1634fe8412a --- /dev/null +++ b/plugins/experimental/maxmind_acl/mmdb.cc @@ -0,0 +1,563 @@ +/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#include "mmdb.h" + +/////////////////////////////////////////////////////////////////////////////// +// Load the config file from param +// check for basics +// Clear out any existing data since this may be a reload +bool +Acl::init(char const *filename) +{ + std::string configloc; + struct stat s; + bool status = false; + + YAML::Node maxmind; + + if (filename[0] != '/') { + // relative file + configloc = TSConfigDirGet(); + configloc += "/"; + configloc.append(filename); + } else { + configloc.assign(filename); + } + + if (stat(configloc.c_str(), &s) < 0) { + TSDebug(PLUGIN_NAME, "Could not stat %s", configloc.c_str()); + return status; + } + + try { + _config = YAML::LoadFile(configloc.c_str()); + + if (_config.IsNull()) { + TSDebug(PLUGIN_NAME, "Config file not found or unreadable"); + return status; + } + if (!_config["maxmind"]) { + TSDebug(PLUGIN_NAME, "Config file not in maxmind namespace"); + return status; + } + + // Get our root maxmind node + maxmind = _config["maxmind"]; +#if 0 + // Test junk + for (YAML::const_iterator it = maxmind.begin(); it != maxmind.end(); ++it) { + const std::string &name = it->first.as(); + YAML::NodeType::value type = it->second.Type(); + TSDebug(PLUGIN_NAME, "name: %s, value: %d", name.c_str(), type); + } +#endif + } catch (const YAML::Exception &e) { + TSError("[%s] YAML::Exception %s when parsing YAML config file %s for maxmind", PLUGIN_NAME, e.what(), configloc.c_str()); + return status; + } + + // Find our database name and convert to full path as needed + status = loaddb(maxmind["database"]); + + if (!status) { + TSDebug(PLUGIN_NAME, "Failed to load MaxMind Database"); + return status; + } + + // Clear out existing data, these may no longer exist in a new config and so we + // dont want old ones left behind + allow_country.clear(); + allow_ip_map.clear(); + deny_ip_map.clear(); + allow_regex.clear(); + deny_regex.clear(); + _html.clear(); + default_allow = false; + + if (loadallow(maxmind["allow"])) { + TSDebug(PLUGIN_NAME, "Loaded Allow ruleset"); + status = true; + } else { + // We have no proper allow ruleset + // setting to allow by default to only apply deny rules + default_allow = true; + } + + if (loaddeny(maxmind["deny"])) { + TSDebug(PLUGIN_NAME, "Loaded Deny ruleset"); + status = true; + } + + loadhtml(maxmind["html"]); + + if (!status) { + TSDebug(PLUGIN_NAME, "Failed to load any rulesets, none specified"); + status = false; + } + + return status; +} + +/////////////////////////////////////////////////////////////////////////////// +// Parse the deny list country codes and IPs +bool +Acl::loaddeny(YAML::Node denyNode) +{ + if (!denyNode) { + TSDebug(PLUGIN_NAME, "No Deny rules set"); + return false; + } + if (denyNode.IsNull()) { + TSDebug(PLUGIN_NAME, "Deny rules are NULL"); + return false; + } + +#if 0 + // Test junk + for (YAML::const_iterator it = denyNode.begin(); it != denyNode.end(); ++it) { + const std::string &name = it->first.as(); + YAML::NodeType::value type = it->second.Type(); + TSDebug(PLUGIN_NAME, "name: %s, value: %d", name.c_str(), type); + } +#endif + + // Load Allowable Country codes + try { + if (denyNode["country"]) { + YAML::Node country = denyNode["country"]; + if (!country.IsNull()) { + if (country.IsSequence()) { + for (std::size_t i = 0; i < country.size(); i++) { + allow_country.insert_or_assign(country[i].as(), false); + } + } else { + TSDebug(PLUGIN_NAME, "Invalid country code allow list yaml"); + } + } + } + } catch (const YAML::Exception &e) { + TSDebug(PLUGIN_NAME, "YAML::Exception %s when parsing YAML config file country code deny list for maxmind", e.what()); + return false; + } + + // Load Denyable IPs + try { + if (denyNode["ip"]) { + YAML::Node ip = denyNode["ip"]; + if (!ip.IsNull()) { + if (ip.IsSequence()) { + // Do IP Deny processing + for (std::size_t i = 0; i < ip.size(); i++) { + IpAddr min, max; + ats_ip_range_parse(std::string_view{ip[i].as()}, min, max); + deny_ip_map.fill(min, max, nullptr); + TSDebug(PLUGIN_NAME, "loading ip: valid: %d, fam %d ", min.isValid(), min.family()); + } + } else { + TSDebug(PLUGIN_NAME, "Invalid IP deny list yaml"); + } + } + } + } catch (const YAML::Exception &e) { + TSDebug(PLUGIN_NAME, "YAML::Exception %s when parsing YAML config file ip deny list for maxmind", e.what()); + return false; + } + + if (denyNode["regex"]) { + YAML::Node regex = denyNode["regex"]; + parseregex(regex, false); + } + +#if 0 + std::unordered_map::iterator cursor; + TSDebug(PLUGIN_NAME, "Deny Country List:"); + for (cursor = allow_country.begin(); cursor != allow_country.end(); cursor++) { + TSDebug(PLUGIN_NAME, "%s:%d", cursor->first.c_str(), cursor->second); + } +#endif + + return true; +} + +// Parse the allow list country codes and IPs +bool +Acl::loadallow(YAML::Node allowNode) +{ + if (!allowNode) { + TSDebug(PLUGIN_NAME, "No Allow rules set"); + return false; + } + if (allowNode.IsNull()) { + TSDebug(PLUGIN_NAME, "Allow rules are NULL"); + return false; + } + +#if 0 + // Test junk + for (YAML::const_iterator it = allowNode.begin(); it != allowNode.end(); ++it) { + const std::string &name = it->first.as(); + YAML::NodeType::value type = it->second.Type(); + TSDebug(PLUGIN_NAME, "name: %s, value: %d", name.c_str(), type); + } +#endif + + // Load Allowable Country codes + try { + if (allowNode["country"]) { + YAML::Node country = allowNode["country"]; + if (!country.IsNull()) { + if (country.IsSequence()) { + for (std::size_t i = 0; i < country.size(); i++) { + allow_country.insert_or_assign(country[i].as(), true); + } + + } else { + TSDebug(PLUGIN_NAME, "Invalid country code allow list yaml"); + } + } + } + } catch (const YAML::Exception &e) { + TSDebug(PLUGIN_NAME, "YAML::Exception %s when parsing YAML config file country code allow list for maxmind", e.what()); + return false; + } + + // Load Allowable IPs + try { + if (allowNode["ip"]) { + YAML::Node ip = allowNode["ip"]; + if (!ip.IsNull()) { + if (ip.IsSequence()) { + // Do IP Allow processing + for (std::size_t i = 0; i < ip.size(); i++) { + IpAddr min, max; + ats_ip_range_parse(std::string_view{ip[i].as()}, min, max); + allow_ip_map.fill(min, max, nullptr); + TSDebug(PLUGIN_NAME, "loading ip: valid: %d, fam %d ", min.isValid(), min.family()); + } + } else { + TSDebug(PLUGIN_NAME, "Invalid IP allow list yaml"); + } + } + } + } catch (const YAML::Exception &e) { + TSDebug(PLUGIN_NAME, "YAML::Exception %s when parsing YAML config file ip allow list for maxmind", e.what()); + return false; + } + + if (allowNode["regex"]) { + YAML::Node regex = allowNode["regex"]; + parseregex(regex, true); + } + +#if 0 + std::unordered_map::iterator cursor; + TSDebug(PLUGIN_NAME, "Allow Country List:"); + for (cursor = allow_country.begin(); cursor != allow_country.end(); cursor++) { + TSDebug(PLUGIN_NAME, "%s:%d", cursor->first.c_str(), cursor->second); + } +#endif + + return true; +} + +void +Acl::parseregex(YAML::Node regex, bool allow) +{ + try { + if (!regex.IsNull()) { + if (regex.IsSequence()) { + // Parse each country-regex pair + for (std::size_t i = 0; i < regex.size(); i++) { + plugin_regex temp; + auto temprule = regex[i].as>(); + temp._regex_s = temprule.back(); + const char *error; + int erroffset; + temp._rex = pcre_compile(temp._regex_s.c_str(), 0, &error, &erroffset, nullptr); + + // Compile the regex for this set of countries + if (nullptr != temp._rex) { + temp._extra = pcre_study(temp._rex, 0, &error); + if ((nullptr == temp._extra) && error && (*error != 0)) { + TSError("[%s] Failed to study regular expression in %s:%s", PLUGIN_NAME, temp._regex_s.c_str(), error); + return; + } + } else { + TSError("[%s] Failed to compile regular expression in %s: %s", PLUGIN_NAME, temp._regex_s.c_str(), error); + return; + } + + for (std::size_t y = 0; y < temprule.size() - 1; y++) { + TSDebug(PLUGIN_NAME, "Adding regex: %s, for country: %s", temp._regex_s.c_str(), regex[i][y].as().c_str()); + if (allow) { + allow_regex[regex[i][y].as()].push_back(temp); + } else { + deny_regex[regex[i][y].as()].push_back(temp); + } + } + } + } + } + } catch (const YAML::Exception &e) { + TSDebug(PLUGIN_NAME, "YAML::Exception %s when parsing YAML config file regex allow list for maxmind", e.what()); + return; + } +} + +void +Acl::loadhtml(YAML::Node htmlNode) +{ + std::string htmlname, htmlloc; + std::ifstream f; + + if (!htmlNode) { + TSDebug(PLUGIN_NAME, "No html field set"); + return; + } + + if (htmlNode.IsNull()) { + TSDebug(PLUGIN_NAME, "Html field not set"); + return; + } + + htmlname = htmlNode.as(); + if (htmlname[0] != '/') { + htmlloc = TSConfigDirGet(); + htmlloc += "/"; + htmlloc.append(htmlname); + } else { + htmlloc.assign(htmlname); + } + + f.open(htmlloc, std::ios::in); + if (f.is_open()) { + _html.append(std::istreambuf_iterator(f), std::istreambuf_iterator()); + f.close(); + TSDebug(PLUGIN_NAME, "Loaded HTML from %s", htmlloc.c_str()); + } else { + TSError("[%s] Unable to open HTML file %s", PLUGIN_NAME, htmlloc.c_str()); + } +} +/////////////////////////////////////////////////////////////////////////////// +// Load the maxmind database from the config parameter +bool +Acl::loaddb(YAML::Node dbNode) +{ + std::string dbloc, dbname; + + if (!dbNode) { + TSDebug(PLUGIN_NAME, "No Database field set"); + return false; + } + if (dbNode.IsNull()) { + TSDebug(PLUGIN_NAME, "Database file not set"); + return false; + } + dbname = dbNode.as(); + if (dbname[0] != '/') { + dbloc = TSConfigDirGet(); + dbloc += "/"; + dbloc.append(dbname); + } else { + dbloc.assign(dbname); + } + + // Make sure we close any previously opened DBs in case this is a reload + if (db_loaded) { + MMDB_close(&_mmdb); + } + + int status = MMDB_open(dbloc.c_str(), MMDB_MODE_MMAP, &_mmdb); + if (MMDB_SUCCESS != status) { + TSDebug(PLUGIN_NAME, "Cant open DB %s - %s", dbloc.c_str(), MMDB_strerror(status)); + return false; + } + + db_loaded = true; + TSDebug(PLUGIN_NAME, "Initialized MMDB with %s", dbloc.c_str()); + return true; +} + +bool +Acl::eval(TSRemapRequestInfo *rri, TSHttpTxn txnp) +{ + bool ret = default_allow; + int mmdb_error; + MMDB_lookup_result_s result = MMDB_lookup_sockaddr(&_mmdb, TSHttpTxnClientAddrGet(txnp), &mmdb_error); + + if (MMDB_SUCCESS != mmdb_error) { + TSDebug(PLUGIN_NAME, "Error during sockaddr lookup: %s", MMDB_strerror(mmdb_error)); + ret = false; + return ret; + } + + MMDB_entry_data_list_s *entry_data_list = nullptr; + if (result.found_entry) { + int status = MMDB_get_entry_data_list(&result.entry, &entry_data_list); + if (MMDB_SUCCESS != status) { + TSDebug(PLUGIN_NAME, "Error looking up entry data: %s", MMDB_strerror(status)); + ret = false; + return ret; + } + + if (NULL != entry_data_list) { + // This is useful to be able to dump out a full record of a + // mmdb entry for debug. Enabling can help if you want to figure + // out how to add new fields +#if 0 + // Block of test stuff to dump output, remove later + char buffer[4096]; + FILE *temp = fmemopen(&buffer[0], 4096, "wb+"); + int status = MMDB_dump_entry_data_list(temp, entry_data_list, 0); + fflush(temp); + TSDebug(PLUGIN_NAME, "Entry: %s, status: %s, type: %d", buffer, MMDB_strerror(status), entry_data_list->entry_data.type); +#endif + + MMDB_entry_data_s entry_data; + int path_len = 0; + const char *path = nullptr; + if (!allow_regex.empty() || !deny_regex.empty()) { + path = TSUrlPathGet(rri->requestBufp, rri->requestUrl, &path_len); + } + // Test for country code + if (!allow_country.empty() || !allow_regex.empty() || !deny_regex.empty()) { + status = MMDB_get_value(&result.entry, &entry_data, "country", "iso_code", NULL); + if (MMDB_SUCCESS != status) { + TSDebug(PLUGIN_NAME, "err on get country code value: %s", MMDB_strerror(status)); + return false; + } + if (entry_data.has_data) { + ret = eval_country(&entry_data, path, path_len); + } + } else { + // Country map is empty as well as regexes, use our default rejection + ret = default_allow; + } + } + } else { + TSDebug(PLUGIN_NAME, "No Country Code entry for this IP was found"); + ret = false; + } + + // Test for allowable IPs based on our lists + switch (eval_ip(TSHttpTxnClientAddrGet(txnp))) { + case ALLOW_IP: + TSDebug(PLUGIN_NAME, "Saw explicit allow of this IP"); + ret = true; + break; + case DENY_IP: + TSDebug(PLUGIN_NAME, "Saw explicit deny of this IP"); + ret = false; + break; + case UNKNOWN_IP: + TSDebug(PLUGIN_NAME, "Unknown IP, following default from ruleset: %d", ret); + break; + default: + TSDebug(PLUGIN_NAME, "Unknown client addr ip state, should not get here"); + ret = false; + break; + } + + if (NULL != entry_data_list) { + MMDB_free_entry_data_list(entry_data_list); + } + + return ret; +} + +/////////////////////////////////////////////////////////////////////////////// +// Returns true if entry data contains an +// allowable country code from our map. +// False otherwise +bool +Acl::eval_country(MMDB_entry_data_s *entry_data, const char *path, int path_len) +{ + bool ret = false; + bool allow = default_allow; + char *output = NULL; + output = (char *)malloc((sizeof(char) * entry_data->data_size)); + strncpy(output, entry_data->utf8_string, entry_data->data_size); + TSDebug(PLUGIN_NAME, "This IP Country Code: %s", output); + auto exists = allow_country.count(output); + + // If the country exists in our map then set its allow value here + // Otherwise we will use our default value + if (exists) { + allow = allow_country[output]; + } + + if (allow) { + TSDebug(PLUGIN_NAME, "Found country code of IP in allow list or allow by default"); + ret = true; + } + + if (nullptr != path && 0 != path_len) { + if (!allow_regex[output].empty()) { + for (auto &i : allow_regex[output]) { + if (PCRE_ERROR_NOMATCH != pcre_exec(i._rex, i._extra, path, path_len, 0, PCRE_NOTEMPTY, nullptr, 0)) { + TSDebug(PLUGIN_NAME, "Got a regex allow hit on regex: %s, country: %s", i._regex_s.c_str(), output); + ret = true; + } + } + } + if (!deny_regex[output].empty()) { + for (auto &i : deny_regex[output]) { + if (PCRE_ERROR_NOMATCH != pcre_exec(i._rex, i._extra, path, path_len, 0, PCRE_NOTEMPTY, nullptr, 0)) { + TSDebug(PLUGIN_NAME, "Got a regex deny hit on regex: %s, country: %s", i._regex_s.c_str(), output); + ret = false; + } + } + } + } + + free(output); + return ret; +} + +/////////////////////////////////////////////////////////////////////////////// +// Returns enum based on current client: +// ALLOW_IP if IP is in the allow list +// DENY_IP if IP is in the deny list +// UNKNOWN_IP if it does not exist in either, this is then used to determine +// action based on the default allow action +ipstate +Acl::eval_ip(const sockaddr *sock) const +{ +#if 0 + for (auto &spot : allow_ip_map) { + char text[INET6_ADDRSTRLEN]; + TSDebug(PLUGIN_NAME, "IP: %s", ats_ip_ntop(spot.min(), text, sizeof text)); + if (0 != ats_ip_addr_cmp(spot.min(), spot.max())) { + TSDebug(PLUGIN_NAME, "stuff: %s", ats_ip_ntop(spot.max(), text, sizeof text)); + } + } +#endif + + if (allow_ip_map.contains(sock, nullptr)) { + // Allow map has this ip, we know we want to allow it + return ALLOW_IP; + } + + if (deny_ip_map.contains(sock, nullptr)) { + // Deny map has this ip, explicitly deny + return DENY_IP; + } + + return UNKNOWN_IP; +} diff --git a/plugins/experimental/maxmind_acl/mmdb.h b/plugins/experimental/maxmind_acl/mmdb.h new file mode 100644 index 00000000000..aa73c3bb244 --- /dev/null +++ b/plugins/experimental/maxmind_acl/mmdb.h @@ -0,0 +1,108 @@ +/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "tscore/IpMap.h" + +#ifdef HAVE_PCRE_PCRE_H +#include +#else +#include +#endif + +#define PLUGIN_NAME "maxmind_acl" +#define CONFIG_TMOUT 60000 + +typedef struct { + std::string _regex_s; + pcre *_rex; + pcre_extra *_extra; +} plugin_regex; + +typedef enum { ALLOW_IP, DENY_IP, UNKNOWN_IP } ipstate; + +// Base class for all ACLs +class Acl +{ +public: + Acl() {} + ~Acl() + { + if (db_loaded) { + MMDB_close(&_mmdb); + } + } + + bool eval(TSRemapRequestInfo *rri, TSHttpTxn txnp); + bool init(char const *filename); + + void + send_html(TSHttpTxn txnp) const + { + if (_html.size() > 0) { + char *msg = TSstrdup(_html.c_str()); + + TSHttpTxnErrorBodySet(txnp, msg, _html.size(), nullptr); // Defaults to text/html + } + } + +protected: + // Class members + YAML::Node _config; + MMDB_s _mmdb; + std::string _html; + std::unordered_map allow_country; + + std::unordered_map> allow_regex; + std::unordered_map> deny_regex; + + IpMap allow_ip_map; + IpMap deny_ip_map; + + // Do we want to allow by default or not? Useful + // for deny only rules + bool default_allow = false; + bool db_loaded = false; + + bool loaddb(YAML::Node dbNode); + bool loadallow(YAML::Node allowNode); + bool loaddeny(YAML::Node denyNode); + void loadhtml(YAML::Node htmlNode); + bool eval_country(MMDB_entry_data_s *entry_data, const char *path, int path_len); + void parseregex(YAML::Node regex, bool allow); + ipstate eval_ip(const sockaddr *sock) const; +}; diff --git a/plugins/experimental/memcache/tsmemcache.cc b/plugins/experimental/memcache/tsmemcache.cc index f8e5789227a..532c2bb4b31 100644 --- a/plugins/experimental/memcache/tsmemcache.cc +++ b/plugins/experimental/memcache/tsmemcache.cc @@ -145,7 +145,7 @@ MC::new_connection(NetVConnection *netvc, EThread *thread) rbuf = new_MIOBuffer(MAX_IOBUFFER_SIZE); rbuf->water_mark = TSMEMCACHE_TMP_CMD_BUFFER_SIZE; reader = rbuf->alloc_reader(); - wbuf = new_empty_MIOBuffer(); + wbuf = new_empty_MIOBuffer(BUFFER_SIZE_INDEX_32K); cbuf = 0; writer = wbuf->alloc_reader(); SCOPED_MUTEX_LOCK(lock, mutex, thread); @@ -375,8 +375,9 @@ MC::write_binary_response(const void *d, int hlen, int keylen, int dlen) if (dlen) { MCDebug("tsmemcache", "response dlen %d\n", dlen); wbuf->write(d, dlen); - } else + } else { MCDebug("tsmemcache", "no response\n"); + } } return writer->read_avail(); } @@ -826,7 +827,7 @@ MC::ascii_set_event(int event, void *data) if (f.set_append) { TS_PUSH_HANDLER(&MC::tunnel_event); if (!cbuf) { - cbuf = new_empty_MIOBuffer(); + cbuf = new_empty_MIOBuffer(BUFFER_SIZE_INDEX_32K); } creader = cbuf->alloc_reader(); crvio = crvc->do_io_read(this, rcache_header->nbytes, cbuf); @@ -838,7 +839,7 @@ MC::ascii_set_event(int event, void *data) a = static_cast(nbytes); } if (!cbuf) { - cbuf = new_empty_MIOBuffer(); + cbuf = new_empty_MIOBuffer(BUFFER_SIZE_INDEX_32K); } creader = cbuf->alloc_reader(); if (a) { @@ -962,13 +963,13 @@ MC::ascii_incr_decr_event(int event, void *data) } header.cas = ink_atomic_increment(&next_cas, 1); { - char *data = nullptr; - int len = 0; + char *localdata = nullptr; + int len = 0; // must be huge, why convert to a counter ?? - if (cwvc->get_single_data((void **)&data, &len) < 0) { + if (cwvc->get_single_data((void **)&localdata, &len) < 0) { goto Lfail; } - uint64_t new_value = xatoull(data, data + len); + uint64_t new_value = xatoull(localdata, localdata + len); if (f.set_incr) { new_value += delta; } else { @@ -1628,7 +1629,7 @@ TSPluginInit(int argc, const char *argv[]) if (argc < 2) { TSError("[tsmemcache] Usage: tsmemcache.so [accept_port]\n"); goto error; - } else if (argc > 1) { + } else { int port = atoi(argv[1]); if (!port) { TSError("[tsmemcache] bad accept_port '%s'\n", argv[1]); diff --git a/plugins/experimental/memcache/tsmemcache.h b/plugins/experimental/memcache/tsmemcache.h index fc6c81188a5..5c591ca1ff3 100644 --- a/plugins/experimental/memcache/tsmemcache.h +++ b/plugins/experimental/memcache/tsmemcache.h @@ -21,9 +21,6 @@ limitations under the License. */ -#ifndef tsmemcache_h -#define tsmemcache_h - #include "I_EventSystem.h" #include "I_Net.h" #include "I_Cache.h" @@ -241,5 +238,3 @@ xatoull(char *s, char *e) } return n; } - -#endif diff --git a/plugins/experimental/memory_profile/Makefile.inc b/plugins/experimental/memory_profile/Makefile.inc new file mode 100644 index 00000000000..1c080f5239a --- /dev/null +++ b/plugins/experimental/memory_profile/Makefile.inc @@ -0,0 +1,23 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +experimental_memory_profile_memory_profile_la_CPPFLAGS = \ + $(AM_CPPFLAGS) + +pkglib_LTLIBRARIES += experimental/memory_profile/memory_profile.la + +experimental_memory_profile_memory_profile_la_SOURCES = \ + experimental/memory_profile/memory_profile.cc diff --git a/plugins/experimental/memory_profile/memory_profile.cc b/plugins/experimental/memory_profile/memory_profile.cc new file mode 100644 index 00000000000..85a4c707f6c --- /dev/null +++ b/plugins/experimental/memory_profile/memory_profile.cc @@ -0,0 +1,109 @@ +/** @file + + A brief file description + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +/* memory_profile.cc + * Responds to plugin messages to dump and activate memory profiling + * System must be built with jemalloc to be useful + */ + +#include +#include +#include +#include +#include +#include +#include +#if TS_HAS_JEMALLOC +#include +#endif + +#define PLUGIN_NAME "memory_profile" + +int +CallbackHandler(TSCont cont, TSEvent id, void *data) +{ + (void)cont; // make compiler shut up about unused variable. + + if (id == TS_EVENT_LIFECYCLE_MSG) { + TSPluginMsg *msg = (TSPluginMsg *)data; + TSDebug(PLUGIN_NAME, "Message to '%s' - %zu bytes of data", msg->tag, msg->data_size); + if (strcmp(PLUGIN_NAME, msg->tag) == 0) { // Message is for us +#if TS_HAS_JEMALLOC + int retval = 0; + if (strncmp((char *)msg->data, "dump", msg->data_size) == 0) { + if ((retval = mallctl("prof.dump", nullptr, nullptr, nullptr, 0)) != 0) { + TSError("mallct(prof.dump) failed retval=%d errno=%d", retval, errno); + } + } else if (strncmp((char *)msg->data, "activate", msg->data_size) == 0) { + bool active = true; + + if ((retval = mallctl("prof.active", nullptr, nullptr, &active, sizeof(active))) != 0) { + TSError("mallct(prof.activate) on failed retval=%d errno=%d", retval, errno); + } + } else if (strncmp((char *)msg->data, "deactivate", msg->data_size) == 0) { + bool active = false; + if ((retval = mallctl("prof.active", nullptr, nullptr, &active, sizeof(active))) != 0) { + TSError("mallct(prof.activate) off failed retval=%d errno=%d", retval, errno); + } + } else if (strncmp((char *)msg->data, "stats", msg->data_size) == 0) { + malloc_stats_print(nullptr, nullptr, nullptr); + } else { + TSError("Unexpected msg %*.s", (int)msg->data_size, (char *)msg->data); + } +#else + TSError("Not built with jemalloc"); +#endif + } + } else { + TSError("Unexpected event %d", id); + } + return TS_EVENT_NONE; +} + +void +TSPluginInit(int argc, const char *argv[]) +{ + TSPluginRegistrationInfo info; + TSCont cb; + + info.plugin_name = PLUGIN_NAME; + info.vendor_name = "Apache Software Foundation"; + info.support_email = "dev@trafficserver.apache.org"; + + if (TSPluginRegister(&info) != TS_SUCCESS) { + TSError("[%s] Plugin registration failed", PLUGIN_NAME); + + goto Lerror; + } + + cb = TSContCreate(CallbackHandler, NULL); + + TSLifecycleHookAdd(TS_LIFECYCLE_MSG_HOOK, cb); + + TSDebug(PLUGIN_NAME, "online"); + + return; + +Lerror: + TSError("[%s] Unable to initialize plugin (disabled)", PLUGIN_NAME); +} diff --git a/plugins/experimental/metalink/metalink.cc b/plugins/experimental/metalink/metalink.cc index e1819fc4845..9f7d201a410 100644 --- a/plugins/experimental/metalink/metalink.cc +++ b/plugins/experimental/metalink/metalink.cc @@ -44,19 +44,18 @@ /* TSCacheWrite() and TSVConnWrite() data: Write the digest to the * cache and store the request URL at that key */ -typedef struct { +struct WriteData { TSHttpTxn txnp; TSCacheKey key; TSVConn connp; TSIOBuffer cache_bufp; - -} WriteData; +}; /* TSTransformCreate() data: Compute the SHA-256 digest of the content */ -typedef struct { +struct TransformData { TSHttpTxn txnp; /* Null transformation */ @@ -65,13 +64,12 @@ typedef struct { /* Message digest handle */ SHA256_CTX c; - -} TransformData; +}; /* TSCacheRead() and TSVConnRead() data: Check the Location and Digest * headers */ -typedef struct { +struct SendData { TSHttpTxn txnp; TSMBuffer resp_bufp; @@ -95,8 +93,7 @@ typedef struct { const char *value; int64_t length; - -} SendData; +}; /* Implement TS_HTTP_READ_RESPONSE_HDR_HOOK to implement a null * transformation */ @@ -888,7 +885,7 @@ handler(TSCont contp, TSEvent event, void *edata) } void -TSPluginInit(int /* argc ATS_UNUSED */, const char * /* argv ATS_UNUSED */ []) +TSPluginInit(int /* argc ATS_UNUSED */, const char * /* argv ATS_UNUSED */[]) { TSPluginRegistrationInfo info; diff --git a/plugins/experimental/metalink/test/chunkedEncoding b/plugins/experimental/metalink/test/chunkedEncoding index 000019be12b..e2c26139a51 100755 --- a/plugins/experimental/metalink/test/chunkedEncoding +++ b/plugins/experimental/metalink/test/chunkedEncoding @@ -16,12 +16,12 @@ # implied. See the License for the specific language governing # permissions and limitations under the License. +from twisted.web import http +from twisted.internet import error, protocol, reactor, tcp + print '''1..1 chunkedEncoding # The proxy forwards the final chunk at the end of a chunked response''' -from twisted.internet import error, protocol, reactor, tcp -from twisted.web import http - def callback(): print 'not ok 1 - No final chunk yet' diff --git a/plugins/experimental/metalink/test/chunkedEncodingDisconnect b/plugins/experimental/metalink/test/chunkedEncodingDisconnect index 8e5cd458b9e..0fa3bc193bf 100755 --- a/plugins/experimental/metalink/test/chunkedEncodingDisconnect +++ b/plugins/experimental/metalink/test/chunkedEncodingDisconnect @@ -16,13 +16,13 @@ # implied. See the License for the specific language governing # permissions and limitations under the License. +from twisted.internet import error, protocol, reactor, tcp +from twisted.web import http + print '''1..1 chunkedEncodingDisconnect # The proxy closes the client connection and doesn't send a final # chunk if the origin disconnects without sending one''' -from twisted.internet import error, protocol, reactor, tcp -from twisted.web import http - def callback(): print 'not ok 1 - The client was left hanging' diff --git a/plugins/experimental/metalink/test/clientDisconnect b/plugins/experimental/metalink/test/clientDisconnect index 54398d2ecdd..af2258a5735 100755 --- a/plugins/experimental/metalink/test/clientDisconnect +++ b/plugins/experimental/metalink/test/clientDisconnect @@ -16,12 +16,12 @@ # implied. See the License for the specific language governing # permissions and limitations under the License. -print '''1..1 clientDissconnect -# The proxy doesn't crash if the client disconnects prematurely''' - from twisted.internet import error, protocol, reactor, tcp from twisted.web import http +print '''1..1 clientDissconnect +# The proxy doesn't crash if the client disconnects prematurely''' + def callback(): print 'not ok 1 - Why didn\'t the test finish yet?' diff --git a/plugins/experimental/metalink/test/contentLength b/plugins/experimental/metalink/test/contentLength index d9635b2cb17..f3bab947f80 100755 --- a/plugins/experimental/metalink/test/contentLength +++ b/plugins/experimental/metalink/test/contentLength @@ -16,12 +16,12 @@ # implied. See the License for the specific language governing # permissions and limitations under the License. -print '''1..1 contentLength -# The proxy forwards the Content-Length header to the client''' - from twisted.internet import error, protocol, reactor, tcp from twisted.web import http +print '''1..1 contentLength +# The proxy forwards the Content-Length header to the client''' + def callback(): print 'not ok 1 - Why didn\'t the test finish yet?' diff --git a/plugins/experimental/metalink/test/contentLengthDisconnect b/plugins/experimental/metalink/test/contentLengthDisconnect index 17c371a7f49..cb39cd36997 100755 --- a/plugins/experimental/metalink/test/contentLengthDisconnect +++ b/plugins/experimental/metalink/test/contentLengthDisconnect @@ -16,13 +16,13 @@ # implied. See the License for the specific language governing # permissions and limitations under the License. +from twisted.internet import error, protocol, reactor, tcp +from twisted.web import http + print '''1..2 contentLengthDisconnect # The proxy closes the client connection if the origin disconnects # prematurely''' -from twisted.internet import error, protocol, reactor, tcp -from twisted.web import http - def callback(): print 'not ok 2 - The client was left hanging' diff --git a/plugins/experimental/metalink/test/finalChunkedEncodingDisconnect b/plugins/experimental/metalink/test/finalChunkedEncodingDisconnect index 41887a92a33..8db9e4e0712 100755 --- a/plugins/experimental/metalink/test/finalChunkedEncodingDisconnect +++ b/plugins/experimental/metalink/test/finalChunkedEncodingDisconnect @@ -16,13 +16,13 @@ # implied. See the License for the specific language governing # permissions and limitations under the License. +from twisted.internet import error, protocol, reactor, tcp +from twisted.web import http + print '''1..1 finalChunkEncodingDisconnect # The proxy forwards the final chunk even if the origin disconnects # immediately afterward''' -from twisted.internet import error, protocol, reactor, tcp -from twisted.web import http - def callback(): print 'not ok 1 - No final chunk yet' diff --git a/plugins/experimental/metalink/test/headers b/plugins/experimental/metalink/test/headers index adc071a12c5..b6451e46a9b 100755 --- a/plugins/experimental/metalink/test/headers +++ b/plugins/experimental/metalink/test/headers @@ -16,12 +16,12 @@ # implied. See the License for the specific language governing # permissions and limitations under the License. -print '''1..1 headers -# The response isn't held up until the origin starts sending content''' - from twisted.internet import error, protocol, reactor, tcp from twisted.web import http +print '''1..1 headers +# The response isn't held up until the origin starts sending content''' + def callback(): print 'not ok 1 - No response yet' diff --git a/plugins/experimental/metalink/test/http09 b/plugins/experimental/metalink/test/http09 index 675459a9ef4..0e7e0ef42e3 100755 --- a/plugins/experimental/metalink/test/http09 +++ b/plugins/experimental/metalink/test/http09 @@ -16,9 +16,6 @@ # implied. See the License for the specific language governing # permissions and limitations under the License. -print '''1..1 http09 -# The proxy doesn't crash on an HTTP/0.9 response''' - # http://www.w3.org/Protocols/HTTP/AsImplemented # # The proxy crashes only after the response is complete. It closes @@ -32,6 +29,9 @@ print '''1..1 http09 from twisted.internet import error, protocol, reactor, tcp from twisted.web import http +print '''1..1 http09 +# The proxy doesn't crash on an HTTP/0.9 response''' + def callback(): print 'not ok 1 - Why didn\'t the test finish yet?' diff --git a/plugins/experimental/metalink/test/location b/plugins/experimental/metalink/test/location index 30ed48846d0..4a0b7704e1f 100755 --- a/plugins/experimental/metalink/test/location +++ b/plugins/experimental/metalink/test/location @@ -16,12 +16,12 @@ # implied. See the License for the specific language governing # permissions and limitations under the License. -print '''1..2 location -# The proxy rewrites the Location header if the file is already cached''' - from twisted.internet import error, protocol, reactor, tcp from twisted.web import http +print '''1..2 location +# The proxy rewrites the Location header if the file is already cached''' + def callback(): print 'not ok 1 - Why didn\'t the test finish yet?' diff --git a/plugins/experimental/metalink/test/longer b/plugins/experimental/metalink/test/longer index d56d18c9eba..d7e48f36f93 100755 --- a/plugins/experimental/metalink/test/longer +++ b/plugins/experimental/metalink/test/longer @@ -16,16 +16,16 @@ # implied. See the License for the specific language governing # permissions and limitations under the License. -print '''1..1 longer -# The proxy doesn't choke if the origin sends more content than it -# advertised''' - # Unlike the contentLength test, don't stop the reactor at the end of # the headers. Give the proxy time to choke. from twisted.internet import error, protocol, reactor, tcp from twisted.web import http +print '''1..1 longer +# The proxy doesn't choke if the origin sends more content than it +# advertised''' + def callback(): print 'not ok 1 - No Content-Length header' diff --git a/plugins/experimental/metalink/test/notCacheable b/plugins/experimental/metalink/test/notCacheable index 8d6c529d3f2..217d44dab65 100755 --- a/plugins/experimental/metalink/test/notCacheable +++ b/plugins/experimental/metalink/test/notCacheable @@ -16,12 +16,12 @@ # implied. See the License for the specific language governing # permissions and limitations under the License. -print '''1..1 notCacheable -# The digest of a file that wasn't cacheable doesn't crash the proxy''' - from twisted.internet import error, protocol, reactor, tcp from twisted.web import http +print '''1..1 notCacheable +# The digest of a file that wasn't cacheable doesn't crash the proxy''' + def callback(): print 'not ok 1 - Why didn\'t the test finish yet?' diff --git a/plugins/experimental/metalink/test/notModified b/plugins/experimental/metalink/test/notModified index 4e664aeb184..2d78433dfd9 100755 --- a/plugins/experimental/metalink/test/notModified +++ b/plugins/experimental/metalink/test/notModified @@ -16,12 +16,12 @@ # implied. See the License for the specific language governing # permissions and limitations under the License. -print '''1..2 notModified -# The proxy doesn't crash on a 304 Not Modified response''' - from twisted.internet import error, protocol, reactor, tcp from twisted.web import http +print '''1..2 notModified +# The proxy doesn't crash on a 304 Not Modified response''' + def callback(): print 'ok 2 - The proxy didn\'t crash (the client connection didn\'t close yet)' diff --git a/plugins/experimental/metalink/test/pipeliningDisconnect b/plugins/experimental/metalink/test/pipeliningDisconnect index 501576fa74a..4943d197879 100755 --- a/plugins/experimental/metalink/test/pipeliningDisconnect +++ b/plugins/experimental/metalink/test/pipeliningDisconnect @@ -16,13 +16,13 @@ # implied. See the License for the specific language governing # permissions and limitations under the License. +from twisted.internet import error, protocol, reactor, tcp +from twisted.web import http + print '''1..1 pipeliningDisconnect # The proxy doesn't crash if INKVConnInternal::do_io_close() gets # called after a message is already complete''' -from twisted.internet import error, protocol, reactor, tcp -from twisted.web import http - def callback(): print 'ok 1 - Did the connection close before the proxy made the second request?' diff --git a/plugins/experimental/metalink/test/shortChunkedEncodingDisconnect b/plugins/experimental/metalink/test/shortChunkedEncodingDisconnect index f4d7e74ec7b..f75aeaf7b39 100755 --- a/plugins/experimental/metalink/test/shortChunkedEncodingDisconnect +++ b/plugins/experimental/metalink/test/shortChunkedEncodingDisconnect @@ -16,14 +16,14 @@ # implied. See the License for the specific language governing # permissions and limitations under the License. +from twisted.internet import error, protocol, reactor, tcp +from twisted.web import http + print '''1..1 shortChunkedEncodingDisconnect # The proxy closes the client connection and doesn't send a final # chunk if the origin disconnects without sending one, before the # proxy sends the response headers''' -from twisted.internet import error, protocol, reactor, tcp -from twisted.web import http - def callback(): print 'not ok 1 - The client was left hanging' diff --git a/plugins/experimental/metalink/test/shortClientDisconnect b/plugins/experimental/metalink/test/shortClientDisconnect index 03ec5a52645..d78e411eeb7 100755 --- a/plugins/experimental/metalink/test/shortClientDisconnect +++ b/plugins/experimental/metalink/test/shortClientDisconnect @@ -16,13 +16,13 @@ # implied. See the License for the specific language governing # permissions and limitations under the License. +from twisted.internet import error, protocol, reactor, tcp +from twisted.web import http + print '''1..1 shortClientDisconnect # The proxy doesn't crash if the client disconnects before the proxy # sends the response headers''' -from twisted.internet import error, protocol, reactor, tcp -from twisted.web import http - def callback(): print 'not ok 1 - Why didn\'t the test finish yet?' diff --git a/plugins/experimental/metalink/test/shortContentLengthDisconnect b/plugins/experimental/metalink/test/shortContentLengthDisconnect index 9c40f398469..b508d805f19 100755 --- a/plugins/experimental/metalink/test/shortContentLengthDisconnect +++ b/plugins/experimental/metalink/test/shortContentLengthDisconnect @@ -16,14 +16,14 @@ # implied. See the License for the specific language governing # permissions and limitations under the License. +from twisted.internet import error, protocol, reactor, tcp +from twisted.web import http + print '''1..2 shortContentLengthDisconnect # The proxy sends the right Content-Length header and closes the # client connection if the origin disconnects before the proxy sends # the response headers''' -from twisted.internet import error, protocol, reactor, tcp -from twisted.web import http - def callback(): print 'not ok 2 - The client was left hanging' diff --git a/plugins/experimental/metalink/test/zero b/plugins/experimental/metalink/test/zero index 71fe60d8961..18a98c807a4 100755 --- a/plugins/experimental/metalink/test/zero +++ b/plugins/experimental/metalink/test/zero @@ -16,12 +16,12 @@ # implied. See the License for the specific language governing # permissions and limitations under the License. -print '''1..1 zero -# The proxy doesn't crash if the Content-Length is zero''' - from twisted.internet import error, protocol, reactor, tcp from twisted.web import http +print '''1..1 zero +# The proxy doesn't crash if the Content-Length is zero''' + def callback(): print 'not ok 1 - Why didn\'t the test finish yet?' diff --git a/plugins/experimental/mp4/mp4_meta.cc b/plugins/experimental/mp4/mp4_meta.cc index 87ae60efaa4..873f5a0c5d5 100644 --- a/plugins/experimental/mp4/mp4_meta.cc +++ b/plugins/experimental/mp4/mp4_meta.cc @@ -1012,7 +1012,7 @@ int Mp4Meta::mp4_update_stts_atom(Mp4Trak *trak) { uint32_t i, entries, count, duration, pass; - uint32_t start_sample, left, start_count; + uint32_t start_sample, left; uint32_t key_sample, old_sample; uint64_t start_time, sum; int64_t atom_size; @@ -1022,7 +1022,7 @@ Mp4Meta::mp4_update_stts_atom(Mp4Trak *trak) return -1; } - sum = start_count = 0; + sum = 0; entries = trak->time_to_sample_entries; start_time = this->start * trak->timescale / 1000; diff --git a/plugins/experimental/remap_stats/Makefile.inc b/plugins/experimental/remap_stats/Makefile.inc index 9cf318ea159..ce922580071 100644 --- a/plugins/experimental/remap_stats/Makefile.inc +++ b/plugins/experimental/remap_stats/Makefile.inc @@ -17,4 +17,4 @@ pkglib_LTLIBRARIES += experimental/remap_stats/remap_stats.la experimental_remap_stats_remap_stats_la_SOURCES = \ - experimental/remap_stats/remap_stats.c + experimental/remap_stats/remap_stats.cc diff --git a/plugins/experimental/remap_stats/remap_stats.c b/plugins/experimental/remap_stats/remap_stats.cc similarity index 55% rename from plugins/experimental/remap_stats/remap_stats.c rename to plugins/experimental/remap_stats/remap_stats.cc index 79c931598b8..727c752a42b 100644 --- a/plugins/experimental/remap_stats/remap_stats.c +++ b/plugins/experimental/remap_stats/remap_stats.cc @@ -21,57 +21,44 @@ #include "tscore/ink_config.h" #include "tscore/ink_defs.h" +#include "tscore/BufferWriter.h" #include "ts/ts.h" -#include -#include -#include -#include -#include -#include + +#include +#include +#include #define PLUGIN_NAME "remap_stats" #define DEBUG_TAG PLUGIN_NAME #define MAX_STAT_LENGTH (1 << 8) -typedef struct { +struct config_t { bool post_remap_host; int txn_slot; TSStatPersistence persist_type; TSMutex stat_creation_mutex; -} config_t; +}; // From "core".... sigh, but we need it for now at least. extern int max_records_entries; -static void -stat_add(char *name, TSMgmtInt amount, TSStatPersistence persist_type, TSMutex create_mutex) +namespace +{ +void +stat_add(const char *name, TSMgmtInt amount, TSStatPersistence persist_type, TSMutex create_mutex) { + static thread_local std::unordered_map hash; int stat_id = -1; - ENTRY search, *result = NULL; - static __thread struct hsearch_data stat_cache; - static __thread bool hash_init = false; - - if (unlikely(!hash_init)) { - // NOLINTNEXTLINE - hcreate_r(max_records_entries << 1, &stat_cache); // This is weird, but oh well. - hash_init = true; - TSDebug(DEBUG_TAG, "stat cache hash init"); - } - search.key = name; - search.data = 0; - // NOLINTNEXTLINE - hsearch_r(search, FIND, &result, &stat_cache); - - if (unlikely(result == NULL)) { + if (unlikely(hash.find(name) == hash.cend())) { // This is an unlikely path because we most likely have the stat cached // so this mutex won't be much overhead and it fixes a race condition // in the RecCore. Hopefully this can be removed in the future. TSMutexLock(create_mutex); - if (TS_ERROR == TSStatFindName((const char *)name, &stat_id)) { - stat_id = TSStatCreate((const char *)name, TS_RECORDDATATYPE_INT, persist_type, TS_STAT_SYNC_SUM); + if (TS_ERROR == TSStatFindName(name, &stat_id)) { + stat_id = TSStatCreate(name, TS_RECORDDATATYPE_INT, persist_type, TS_STAT_SYNC_SUM); if (stat_id == TS_ERROR) { TSDebug(DEBUG_TAG, "Error creating stat_name: %s", name); } else { @@ -81,14 +68,11 @@ stat_add(char *name, TSMgmtInt amount, TSStatPersistence persist_type, TSMutex c TSMutexUnlock(create_mutex); if (stat_id >= 0) { - search.key = TSstrdup(name); - search.data = (void *)((intptr_t)stat_id); - // NOLINTNEXTLINE - hsearch_r(search, ENTER, &result, &stat_cache); + hash.emplace(name, stat_id); TSDebug(DEBUG_TAG, "Cached stat_name: %s stat_id: %d", name, stat_id); } } else { - stat_id = (int)((intptr_t)result->data); + stat_id = hash.at(name); } if (likely(stat_id >= 0)) { @@ -98,7 +82,7 @@ stat_add(char *name, TSMgmtInt amount, TSStatPersistence persist_type, TSMutex c } } -static char * +char * get_effective_host(TSHttpTxn txn) { char *effective_url, *tmp; @@ -111,7 +95,7 @@ get_effective_host(TSHttpTxn txn) if (TS_SUCCESS != TSUrlCreate(buf, &url_loc)) { TSDebug(DEBUG_TAG, "unable to create url"); TSMBufferDestroy(buf); - return NULL; + return nullptr; } tmp = effective_url = TSHttpTxnEffectiveUrlStringGet(txn, &len); TSUrlParse(buf, url_loc, (const char **)(&tmp), (const char *)(effective_url + len)); @@ -123,36 +107,36 @@ get_effective_host(TSHttpTxn txn) return tmp; } -static int +int handle_read_req_hdr(TSCont cont, TSEvent event ATS_UNUSED, void *edata) { - TSHttpTxn txn = (TSHttpTxn)edata; + TSHttpTxn txn = static_cast(edata); config_t *config; void *txnd; - config = (config_t *)TSContDataGet(cont); + config = static_cast(TSContDataGet(cont)); txnd = (void *)get_effective_host(txn); // low bit left 0 because we do not know that remap succeeded yet - TSHttpTxnArgSet(txn, config->txn_slot, txnd); + TSUserArgSet(txn, config->txn_slot, txnd); TSHttpTxnReenable(txn, TS_EVENT_HTTP_CONTINUE); TSDebug(DEBUG_TAG, "Read Req Handler Finished"); return 0; } -static int +int handle_post_remap(TSCont cont, TSEvent event ATS_UNUSED, void *edata) { - TSHttpTxn txn = (TSHttpTxn)edata; + TSHttpTxn txn = static_cast(edata); config_t *config; void *txnd = (void *)0x01; // low bit 1 because we are post remap and thus success - config = (config_t *)TSContDataGet(cont); + config = static_cast(TSContDataGet(cont)); if (config->post_remap_host) { - TSHttpTxnArgSet(txn, config->txn_slot, txnd); + TSUserArgSet(txn, config->txn_slot, txnd); } else { - txnd = (void *)((uintptr_t)txnd | (uintptr_t)TSHttpTxnArgGet(txn, config->txn_slot)); // We need the hostname pre-remap - TSHttpTxnArgSet(txn, config->txn_slot, txnd); + txnd = (void *)((uintptr_t)txnd | (uintptr_t)TSUserArgGet(txn, config->txn_slot)); // We need the hostname pre-remap + TSUserArgSet(txn, config->txn_slot, txnd); } TSHttpTxnReenable(txn, TS_EVENT_HTTP_CONTINUE); @@ -160,80 +144,87 @@ handle_post_remap(TSCont cont, TSEvent event ATS_UNUSED, void *edata) return 0; } -#define CREATE_STAT_NAME(s, h, b) snprintf(s, MAX_STAT_LENGTH, "plugin.%s.%s.%s", PLUGIN_NAME, h, b) +void +create_stat_name(ts::FixedBufferWriter &stat_name, std::string_view h, std::string_view b) +{ + stat_name.reset().clip(1); + stat_name.print("plugin.{}.{}.{}", PLUGIN_NAME, h, b); + stat_name.extend(1).write('\0'); +} -static int +int handle_txn_close(TSCont cont, TSEvent event ATS_UNUSED, void *edata) { - TSHttpTxn txn = (TSHttpTxn)edata; - config_t *config; - void *txnd; - int status_code = 0; - TSMBuffer buf; - TSMLoc hdr_loc; - uint64_t out_bytes, in_bytes; - char *remap, *hostname; - char *unknown = "unknown"; - char stat_name[MAX_STAT_LENGTH]; + TSHttpTxn const txn = static_cast(edata); + char const *remap = nullptr; + char *hostname = nullptr; + char *effective_hostname = nullptr; + + static char const *const unknown = "unknown"; - config = (config_t *)TSContDataGet(cont); - txnd = TSHttpTxnArgGet(txn, config->txn_slot); + config_t const *const config = static_cast(TSContDataGet(cont)); + void const *const txnd = TSUserArgGet(txn, config->txn_slot); - hostname = (char *)((uintptr_t)txnd & (~((uintptr_t)0x01))); // Get hostname + hostname = reinterpret_cast(reinterpret_cast(txnd) & ~0x01); // Get hostname - if (txnd) { - if ((uintptr_t)txnd & 0x01) // remap succeeded? + if (nullptr != txnd) { + if (reinterpret_cast(txnd) & 0x01) // remap succeeded? { if (!config->post_remap_host) { remap = hostname; } else { - remap = get_effective_host(txn); + effective_hostname = get_effective_host(txn); + remap = effective_hostname; } - if (!remap) { + if (nullptr == remap) { remap = unknown; } - in_bytes = TSHttpTxnClientReqHdrBytesGet(txn); + uint64_t in_bytes = TSHttpTxnClientReqHdrBytesGet(txn); in_bytes += TSHttpTxnClientReqBodyBytesGet(txn); - CREATE_STAT_NAME(stat_name, remap, "in_bytes"); - stat_add(stat_name, (TSMgmtInt)in_bytes, config->persist_type, config->stat_creation_mutex); + ts::LocalBufferWriter stat_name; - out_bytes = TSHttpTxnClientRespHdrBytesGet(txn); + create_stat_name(stat_name, remap, "in_bytes"); + stat_add(stat_name.data(), static_cast(in_bytes), config->persist_type, config->stat_creation_mutex); + + uint64_t out_bytes = TSHttpTxnClientRespHdrBytesGet(txn); out_bytes += TSHttpTxnClientRespBodyBytesGet(txn); - CREATE_STAT_NAME(stat_name, remap, "out_bytes"); - stat_add(stat_name, (TSMgmtInt)out_bytes, config->persist_type, config->stat_creation_mutex); + create_stat_name(stat_name, remap, "out_bytes"); + stat_add(stat_name.data(), static_cast(out_bytes), config->persist_type, config->stat_creation_mutex); + TSMBuffer buf = nullptr; + TSMLoc hdr_loc = nullptr; if (TSHttpTxnClientRespGet(txn, &buf, &hdr_loc) == TS_SUCCESS) { - status_code = (int)TSHttpHdrStatusGet(buf, hdr_loc); + int const status_code = static_cast(TSHttpHdrStatusGet(buf, hdr_loc)); TSHandleMLocRelease(buf, TS_NULL_MLOC, hdr_loc); if (status_code < 200) { - CREATE_STAT_NAME(stat_name, remap, "status_other"); + create_stat_name(stat_name, remap, "status_other"); } else if (status_code <= 299) { - CREATE_STAT_NAME(stat_name, remap, "status_2xx"); + create_stat_name(stat_name, remap, "status_2xx"); } else if (status_code <= 399) { - CREATE_STAT_NAME(stat_name, remap, "status_3xx"); + create_stat_name(stat_name, remap, "status_3xx"); } else if (status_code <= 499) { - CREATE_STAT_NAME(stat_name, remap, "status_4xx"); + create_stat_name(stat_name, remap, "status_4xx"); } else if (status_code <= 599) { - CREATE_STAT_NAME(stat_name, remap, "status_5xx"); + create_stat_name(stat_name, remap, "status_5xx"); } else { - CREATE_STAT_NAME(stat_name, remap, "status_other"); + create_stat_name(stat_name, remap, "status_other"); } - stat_add(stat_name, 1, config->persist_type, config->stat_creation_mutex); + stat_add(stat_name.data(), 1, config->persist_type, config->stat_creation_mutex); } else { - CREATE_STAT_NAME(stat_name, remap, "status_unknown"); - stat_add(stat_name, 1, config->persist_type, config->stat_creation_mutex); + create_stat_name(stat_name, remap, "status_unknown"); + stat_add(stat_name.data(), 1, config->persist_type, config->stat_creation_mutex); } - if (remap != unknown) { - TSfree(remap); + if (nullptr != effective_hostname) { + TSfree(effective_hostname); } - } else if (hostname) { + } else if (nullptr != hostname) { TSfree(hostname); } } @@ -243,12 +234,13 @@ handle_txn_close(TSCont cont, TSEvent event ATS_UNUSED, void *edata) return 0; } +} // namespace + void TSPluginInit(int argc, const char *argv[]) { TSPluginRegistrationInfo info; TSCont pre_remap_cont, post_remap_cont, global_cont; - config_t *config; info.plugin_name = PLUGIN_NAME; info.vendor_name = "Apache Software Foundation"; @@ -262,46 +254,39 @@ TSPluginInit(int argc, const char *argv[]) TSDebug(DEBUG_TAG, "Plugin registration succeeded"); } - config = TSmalloc(sizeof(config_t)); + auto config = new config_t; config->post_remap_host = false; config->persist_type = TS_STAT_NON_PERSISTENT; config->stat_creation_mutex = TSMutexCreate(); if (argc > 1) { - int c; - static const struct option longopts[] = { - {"post-remap-host", no_argument, NULL, 'P'}, {"persistent", no_argument, NULL, 'p'}, {NULL, 0, NULL, 0}}; - - while ((c = getopt_long(argc, (char *const *)argv, "Pp", longopts, NULL)) != -1) { - switch (c) { - case 'P': + // Argument parser + for (int ii = 0; ii < argc; ++ii) { + std::string_view const arg(argv[ii]); + if (arg == "-P" || arg == "--post-remap-host") { config->post_remap_host = true; TSDebug(DEBUG_TAG, "Using post remap hostname"); - break; - case 'p': + } else if (arg == "-p" || arg == "--persistent") { config->persist_type = TS_STAT_PERSISTENT; TSDebug(DEBUG_TAG, "Using persistent stats"); - break; - default: - break; } } } - TSHttpTxnArgIndexReserve(PLUGIN_NAME, "txn data", &(config->txn_slot)); + TSUserArgIndexReserve(TS_USER_ARGS_TXN, PLUGIN_NAME, "txn data", &(config->txn_slot)); if (!config->post_remap_host) { - pre_remap_cont = TSContCreate(handle_read_req_hdr, NULL); - TSContDataSet(pre_remap_cont, (void *)config); + pre_remap_cont = TSContCreate(handle_read_req_hdr, nullptr); + TSContDataSet(pre_remap_cont, static_cast(config)); TSHttpHookAdd(TS_HTTP_READ_REQUEST_HDR_HOOK, pre_remap_cont); } - post_remap_cont = TSContCreate(handle_post_remap, NULL); - TSContDataSet(post_remap_cont, (void *)config); + post_remap_cont = TSContCreate(handle_post_remap, nullptr); + TSContDataSet(post_remap_cont, static_cast(config)); TSHttpHookAdd(TS_HTTP_POST_REMAP_HOOK, post_remap_cont); - global_cont = TSContCreate(handle_txn_close, NULL); - TSContDataSet(global_cont, (void *)config); + global_cont = TSContCreate(handle_txn_close, nullptr); + TSContDataSet(global_cont, static_cast(config)); TSHttpHookAdd(TS_HTTP_TXN_CLOSE_HOOK, global_cont); TSDebug(DEBUG_TAG, "Init complete"); diff --git a/plugins/experimental/server_push_preload/README.md b/plugins/experimental/server_push_preload/README.md deleted file mode 100644 index 296794c8e15..00000000000 --- a/plugins/experimental/server_push_preload/README.md +++ /dev/null @@ -1,3 +0,0 @@ -Parse origin response Link headers and use H2 Server Push to initiate push requests of assets that have the preload keyword. - -https://www.w3.org/TR/preload/ diff --git a/plugins/experimental/slice/Config.cc b/plugins/experimental/slice/Config.cc index 2198011d0e4..176ba9480c6 100644 --- a/plugins/experimental/slice/Config.cc +++ b/plugins/experimental/slice/Config.cc @@ -26,24 +26,39 @@ #include "ts/experimental.h" +Config::~Config() +{ + if (nullptr != m_regex_extra) { +#ifndef PCRE_STUDY_JIT_COMPILE + pcre_free(m_regex_extra); +#else + pcre_free_study(m_regex_extra); +#endif + } + if (nullptr != m_regex) { + pcre_free(m_regex); + } +} + int64_t Config::bytesFrom(char const *const valstr) { - char *endptr = nullptr; - int64_t blockbytes = strtoll(valstr, &endptr, 10); + char *endptr = nullptr; + int64_t blockbytes = strtoll(valstr, &endptr, 10); + constexpr int64_t kib = 1024; if (nullptr != endptr && valstr < endptr) { size_t const dist = endptr - valstr; if (dist < strlen(valstr) && 0 <= blockbytes) { switch (tolower(*endptr)) { case 'g': - blockbytes *= (static_cast(1024) * static_cast(1024) * static_cast(1024)); + blockbytes *= (kib * kib * kib); break; case 'm': - blockbytes *= (static_cast(1024) * static_cast(1024)); + blockbytes *= (kib * kib); break; case 'k': - blockbytes *= static_cast(1024); + blockbytes *= kib; break; default: break; @@ -66,7 +81,7 @@ Config::fromArgs(int const argc, char const *const argv[]) DEBUG_LOG("args[%d] = %s", index, argv[index]); } - // current "best" blockbytes from configuration + // look for lowest priority deprecated blockbytes int64_t blockbytes = 0; // backwards compat: look for blockbytes @@ -91,19 +106,22 @@ Config::fromArgs(int const argc, char const *const argv[]) } // standard parsing - constexpr const struct option longopts[] = { + constexpr struct option longopts[] = { {const_cast("blockbytes"), required_argument, nullptr, 'b'}, - {const_cast("test-blockbytes"), required_argument, nullptr, 't'}, - {const_cast("pace-errorlog"), required_argument, nullptr, 'p'}, {const_cast("disable-errorlog"), no_argument, nullptr, 'd'}, + {const_cast("exclude-regex"), required_argument, nullptr, 'e'}, + {const_cast("include-regex"), required_argument, nullptr, 'i'}, + {const_cast("throttle"), no_argument, nullptr, 'o'}, + {const_cast("pace-errorlog"), required_argument, nullptr, 'p'}, + {const_cast("remap-host"), required_argument, nullptr, 'r'}, + {const_cast("blockbytes-test"), required_argument, nullptr, 't'}, {nullptr, 0, nullptr, 0}, }; // getopt assumes args start at '1' so this hack is needed char *const *argvp = (const_cast(argv) - 1); - for (;;) { - int const opt = getopt_long(argc + 1, argvp, "b:t:p:d", longopts, nullptr); + int const opt = getopt_long(argc + 1, argvp, "b:de:i:op:r:t:", longopts, nullptr); if (-1 == opt) { break; } @@ -120,30 +138,74 @@ Config::fromArgs(int const argc, char const *const argv[]) ERROR_LOG("Invalid blockbytes: %s", optarg); } } break; - case 't': - if (0 == blockbytes) { - int64_t const bytesread = bytesFrom(optarg); - if (0 < bytesread) { - DEBUG_LOG("Using blockbytestest %" PRId64, bytesread); - blockbytes = bytesread; - } else { - ERROR_LOG("Invalid blockbytestest: %s", optarg); - } + case 'd': { + m_paceerrsecs = -1; + } break; + case 'e': { + if (None != m_regex_type) { + ERROR_LOG("Regex already specified!"); + break; + } + + const char *errptr; + int erroffset; + m_regexstr = optarg; + m_regex = pcre_compile(m_regexstr.c_str(), 0, &errptr, &erroffset, nullptr); + if (nullptr == m_regex) { + ERROR_LOG("Invalid regex: '%s'", m_regexstr.c_str()); } else { - DEBUG_LOG("Skipping blockbytestest in favor of blockbytes"); + m_regex_type = Exclude; + m_regex_extra = pcre_study(m_regex, 0, &errptr); + DEBUG_LOG("Using regex for url exclude: '%s'", m_regexstr.c_str()); } - break; + } break; + case 'i': { + if (None != m_regex_type) { + ERROR_LOG("Regex already specified!"); + break; + } + + const char *errptr; + int erroffset; + m_regexstr = optarg; + m_regex = pcre_compile(m_regexstr.c_str(), 0, &errptr, &erroffset, nullptr); + if (nullptr == m_regex) { + ERROR_LOG("Invalid regex: '%s'", m_regexstr.c_str()); + } else { + m_regex_type = Include; + m_regex_extra = pcre_study(m_regex, 0, &errptr); + DEBUG_LOG("Using regex for url include: '%s'", m_regexstr.c_str()); + } + } break; + case 'o': { + m_throttle = true; + DEBUG_LOG("Enabling internal block throttling"); + } break; case 'p': { int const secsread = atoi(optarg); if (0 < secsread) { m_paceerrsecs = std::min(secsread, 60); } else { - DEBUG_LOG("Ignoring pace-errlog argument"); + ERROR_LOG("Ignoring pace-errlog argument"); + } + } break; + case 'r': { + m_remaphost = optarg; + DEBUG_LOG("Using loopback remap host override: %s", m_remaphost.c_str()); + } break; + case 't': { + if (0 == blockbytes) { + int64_t const bytesread = bytesFrom(optarg); + if (0 < bytesread) { + DEBUG_LOG("Using blockbytes-test %" PRId64, bytesread); + blockbytes = bytesread; + } else { + ERROR_LOG("Invalid blockbytes-test: %s", optarg); + } + } else { + DEBUG_LOG("Skipping blockbytes-test in favor of blockbytes"); } } break; - case 'd': - m_paceerrsecs = -1; - break; default: break; } @@ -170,8 +232,6 @@ Config::fromArgs(int const argc, char const *const argv[]) bool Config::canLogError() { - std::lock_guard const guard(m_mutex); - if (m_paceerrsecs < 0) { return false; } else if (0 == m_paceerrsecs) { @@ -180,14 +240,42 @@ Config::canLogError() #if !defined(UNITTEST) TSHRTime const timenow = TShrtime(); +#endif + + std::lock_guard const guard(m_mutex); + +#if !defined(UNITTEST) if (timenow < m_nextlogtime) { return false; } m_nextlogtime = timenow + TS_HRTIME_SECONDS(m_paceerrsecs); #else - m_nextlogtime = 0; // thanks clang + m_nextlogtime = 0; // needed by clang #endif return true; } + +bool +Config::matchesRegex(char const *const url, int const urllen) const +{ + bool matches = true; + + switch (m_regex_type) { + case Exclude: { + if (0 <= pcre_exec(m_regex, m_regex_extra, url, urllen, 0, 0, nullptr, 0)) { + matches = false; + } + } break; + case Include: { + if (pcre_exec(m_regex, m_regex_extra, url, urllen, 0, 0, nullptr, 0) < 0) { + matches = false; + } + } break; + default: + break; + } + + return matches; +} diff --git a/plugins/experimental/slice/Config.h b/plugins/experimental/slice/Config.h index 8c4ab2498fa..5edf4638f07 100644 --- a/plugins/experimental/slice/Config.h +++ b/plugins/experimental/slice/Config.h @@ -20,6 +20,12 @@ #include "slice.h" +#ifdef HAVE_PCRE_PCRE_H +#include +#else +#include +#endif + #include // Data Structures and Classes @@ -29,17 +35,37 @@ struct Config { static constexpr int64_t const blockbytesdefault = 1024 * 1024; // 1MB int64_t m_blockbytes{blockbytesdefault}; - int m_paceerrsecs{0}; // -1 disable logging, 0 no pacing, max 60s + std::string m_remaphost; // remap host to use for loopback slice GET + std::string m_regexstr; // regex string for things to slice (default all) + enum RegexType { None, Include, Exclude }; + RegexType m_regex_type{None}; + pcre *m_regex{nullptr}; + pcre_extra *m_regex_extra{nullptr}; + bool m_throttle{false}; // internal block throttling + int m_paceerrsecs{0}; // -1 disable logging, 0 no pacing, max 60s // Convert optarg to bytes static int64_t bytesFrom(char const *const valstr); + // clean up pcre if applicable + ~Config(); + // Parse from args, ast one wins bool fromArgs(int const argc, char const *const argv[]); // Check if the error should can be logged, if sucessful may update m_nexttime bool canLogError(); + // Check if regex supplied + bool + hasRegex() const + { + return None != m_regex_type; + } + + // If no null reg, true, otherwise check against regex + bool matchesRegex(char const *const url, int const urllen) const; + private: TSHRTime m_nextlogtime{0}; // next time to log in ns std::mutex m_mutex; diff --git a/plugins/experimental/slice/Data.h b/plugins/experimental/slice/Data.h index 45383522103..3150bd299b8 100644 --- a/plugins/experimental/slice/Data.h +++ b/plugins/experimental/slice/Data.h @@ -20,16 +20,13 @@ #include "ts/ts.h" -#include "Config.h" #include "HttpHeader.h" #include "Range.h" #include "Stage.h" #include -void incrData(); - -void decrData(); +struct Config; struct Data { Data(Data const &) = delete; @@ -39,8 +36,8 @@ struct Data { sockaddr_storage m_client_ip; - // for pristine url coming in - TSMBuffer m_urlbuffer{nullptr}; + // for pristine/effective url coming in + TSMBuffer m_urlbuf{nullptr}; TSMLoc m_urlloc{nullptr}; char m_hostname[8192]; @@ -61,7 +58,9 @@ struct Data { int64_t m_blockexpected; // body bytes expected int64_t m_blockskip; // number of bytes to skip in this block int64_t m_blockconsumed; // body bytes consumed - bool m_iseos; // server in EOS state + + enum BlockState { Pending, Active, Done, Fail }; + BlockState m_blockstate; // is there an active slice block int64_t m_bytestosend; // header + content bytes to send int64_t m_bytessent; // number of bytes written to the client @@ -80,44 +79,43 @@ struct Data { explicit Data(Config *const config) : m_config(config), m_client_ip(), - m_urlbuffer(nullptr), + m_urlbuf(nullptr), m_urlloc(nullptr), m_hostlen(0), m_etaglen(0), m_lastmodifiedlen(0), m_statustype(TS_HTTP_STATUS_NONE), - m_bail(false), m_req_range(-1, -1), - m_contentlen(-1) - - , + m_contentlen(-1), m_blocknum(-1), m_blockexpected(0), m_blockskip(0), m_blockconsumed(0), - m_iseos(false) - - , + m_blockstate(Pending), m_bytestosend(0), m_bytessent(0), m_server_block_header_parsed(false), m_server_first_header_parsed(false), m_http_parser(nullptr) { - // incrData(); m_hostname[0] = '\0'; m_lastmodified[0] = '\0'; m_etag[0] = '\0'; +#if defined(COLLECT_STATS) + TSStatIntIncrement(stats::DataCreate, 1); +#endif } ~Data() { - // decrData(); - if (nullptr != m_urlbuffer) { +#if defined(COLLECT_STATS) + TSStatIntIncrement(stats::DataDestroy, 1); +#endif + if (nullptr != m_urlbuf) { if (nullptr != m_urlloc) { - TSHandleMLocRelease(m_urlbuffer, TS_NULL_MLOC, m_urlloc); + TSHandleMLocRelease(m_urlbuf, TS_NULL_MLOC, m_urlloc); } - TSMBufferDestroy(m_urlbuffer); + TSMBufferDestroy(m_urlbuf); } if (nullptr != m_http_parser) { TSHttpParserDestroy(m_http_parser); diff --git a/plugins/experimental/slice/HttpHeader.cc b/plugins/experimental/slice/HttpHeader.cc index 332370576c1..f70c42e264c 100644 --- a/plugins/experimental/slice/HttpHeader.cc +++ b/plugins/experimental/slice/HttpHeader.cc @@ -54,18 +54,20 @@ HttpHeader::setStatus(TSHttpStatus const newstatus) } char * -HttpHeader ::urlString(int *const urllen) const +HttpHeader::urlString(int *const urllen) const { char *urlstr = nullptr; TSAssert(nullptr != urllen); TSMLoc locurl = nullptr; TSReturnCode const rcode = TSHttpHdrUrlGet(m_buffer, m_lochdr, &locurl); - if (TS_SUCCESS == rcode && nullptr != locurl) { - urlstr = TSUrlStringGet(m_buffer, locurl, urllen); + if (nullptr != locurl) { + if (TS_SUCCESS == rcode) { + urlstr = TSUrlStringGet(m_buffer, locurl, urllen); + } else { + *urllen = 0; + } TSHandleMLocRelease(m_buffer, m_lochdr, locurl); - } else { - *urllen = 0; } return urlstr; @@ -311,7 +313,8 @@ HttpHeader::toString() const /////// HdrMgr TSParseResult -HdrMgr::populateFrom(TSHttpParser const http_parser, TSIOBufferReader const reader, HeaderParseFunc const parsefunc) +HdrMgr::populateFrom(TSHttpParser const http_parser, TSIOBufferReader const reader, HeaderParseFunc const parsefunc, + int64_t *const bytes) { TSParseResult parse_res = TS_PARSE_CONT; @@ -322,14 +325,14 @@ HdrMgr::populateFrom(TSHttpParser const http_parser, TSIOBufferReader const read m_lochdr = TSHttpHdrCreate(m_buffer); } - int64_t read_avail = TSIOBufferReaderAvail(reader); - if (0 < read_avail) { + int64_t avail = TSIOBufferReaderAvail(reader); + if (0 < avail) { TSIOBufferBlock block = TSIOBufferReaderStart(reader); int64_t consumed = 0; parse_res = TS_PARSE_CONT; - while (nullptr != block && 0 < read_avail) { + while (nullptr != block && 0 < avail) { int64_t blockbytes = 0; char const *const bstart = TSIOBufferBlockReadStart(block, reader, &blockbytes); @@ -341,7 +344,7 @@ HdrMgr::populateFrom(TSHttpParser const http_parser, TSIOBufferReader const read int64_t const bytes_parsed(ptr - bstart); consumed += bytes_parsed; - read_avail -= bytes_parsed; + avail -= bytes_parsed; if (TS_PARSE_CONT == parse_res) { block = TSIOBufferBlockNext(block); @@ -350,6 +353,12 @@ HdrMgr::populateFrom(TSHttpParser const http_parser, TSIOBufferReader const read } } TSIOBufferReaderConsume(reader, consumed); + + if (nullptr != bytes) { + *bytes = consumed; + } + } else if (nullptr != bytes) { + *bytes = 0; } return parse_res; diff --git a/plugins/experimental/slice/HttpHeader.h b/plugins/experimental/slice/HttpHeader.h index bfd9c4d6f4e..001e5f6480a 100644 --- a/plugins/experimental/slice/HttpHeader.h +++ b/plugins/experimental/slice/HttpHeader.h @@ -53,6 +53,16 @@ struct HttpHeader { return nullptr != m_buffer && nullptr != m_lochdr; } + int + byteSize() const + { + if (isValid()) { + return TSHttpHdrLengthGet(m_buffer, m_lochdr); + } else { + return 0; + } + } + // TS_HTTP_TYPE_UNKNOWN, TS_HTTP_TYPE_REQUEST, TS_HTTP_TYPE_RESPONSE TSHttpType type() const; @@ -106,7 +116,7 @@ struct HttpHeader { char *const valstr, // <-- return string value int *const vallen, // <-- pass in capacity, returns len of string int const index = -1 // retrieves all values - ) const; + ) const; /** Sets or adds a key/value @@ -205,7 +215,8 @@ struct HdrMgr { TSHttpHdrParseResp Call this multiple times if necessary. */ - TSParseResult populateFrom(TSHttpParser const http_parser, TSIOBufferReader const reader, HeaderParseFunc const parsefunc); + TSParseResult populateFrom(TSHttpParser const http_parser, TSIOBufferReader const reader, HeaderParseFunc const parsefunc, + int64_t *const consumed); bool isValid() const diff --git a/plugins/experimental/slice/Makefile.inc b/plugins/experimental/slice/Makefile.inc index dbac02b4940..be376ca1c64 100644 --- a/plugins/experimental/slice/Makefile.inc +++ b/plugins/experimental/slice/Makefile.inc @@ -23,7 +23,6 @@ experimental_slice_slice_la_SOURCES = \ experimental/slice/Config.h \ experimental/slice/ContentRange.cc \ experimental/slice/ContentRange.h \ - experimental/slice/Data.cc \ experimental/slice/Data.h \ experimental/slice/HttpHeader.cc \ experimental/slice/HttpHeader.h \ @@ -39,7 +38,9 @@ experimental_slice_slice_la_SOURCES = \ experimental/slice/slice.h \ experimental/slice/Stage.h \ experimental/slice/transfer.cc \ - experimental/slice/transfer.h + experimental/slice/transfer.h \ + experimental/slice/util.cc \ + experimental/slice/util.h check_PROGRAMS += experimental/slice/test_content_range @@ -48,6 +49,8 @@ experimental_slice_test_content_range_SOURCES = \ experimental/slice/unit-tests/test_content_range.cc \ experimental/slice/ContentRange.cc +experimental_slice_test_content_range_LDADD = @LIBPCRE@ + check_PROGRAMS += experimental/slice/test_range experimental_slice_test_range_CPPFLAGS = $(AM_CPPFLAGS) -I$(abs_top_srcdir)/tests/include -DUNITTEST @@ -55,9 +58,13 @@ experimental_slice_test_range_SOURCES = \ experimental/slice/unit-tests/test_range.cc \ experimental/slice/Range.cc +experimental_slice_test_range_LDADD = @LIBPCRE@ + check_PROGRAMS += experimental/slice/test_config experimental_slice_test_config_CPPFLAGS = $(AM_CPPFLAGS) -I$(abs_top_srcdir)/tests/include -DUNITTEST experimental_slice_test_config_SOURCES = \ experimental/slice/unit-tests/test_config.cc \ experimental/slice/Config.cc + +experimental_slice_test_config_LDADD = @LIBPCRE@ diff --git a/plugins/experimental/slice/Makefile.tsxs b/plugins/experimental/slice/Makefile.tsxs index 0992d8cb27e..f8022a1714e 100644 --- a/plugins/experimental/slice/Makefile.tsxs +++ b/plugins/experimental/slice/Makefile.tsxs @@ -22,7 +22,6 @@ all: $(PLUGIN).so SOURCES = \ Config.cc \ ContentRange.cc \ - Data.cc \ HttpHeader.cc \ Range.cc \ client.cc \ @@ -62,5 +61,5 @@ TSCXXFLAGS = $(shell tsxs -q CXXFLAGS) slice_test: slice_test.cc ContentRange.cc Range.cc $(TSCXX) -o $@ $^ $(TSCXXFLAGS) -I$(TSINCLUDE) -DUNITTEST -clean: +clean: rm -fv *.lo *.so diff --git a/plugins/experimental/slice/README.md b/plugins/experimental/slice/README.md index ec3d1bc420f..6ae1399037a 100644 --- a/plugins/experimental/slice/README.md +++ b/plugins/experimental/slice/README.md @@ -15,66 +15,86 @@ The plugin uses TSHttpConnect to delegate each block request to cache_range_requests.so which handles all cache and parent interaction. To enable the plugin, specify the plugin library via @plugin at the end -of a remap line as follows (2MB slice in this example): +of a remap line as follows (default 1MB slice in this example): ``` -map http://ats-cache/ http://parent/ @plugin=slice.so @pparam=--blockbytes=2M @plugin=cache_range_requests.so +map http://ats-cache/ http://parent/ @plugin=slice.so @plugin=cache_range_requests.so +map https://ats-cache/ http://parent/ @plugin=slice.so @plugin=cache_range_requests.so ``` -alternatively +alternatively (2MB slice block) ``` map http://ats-cache/ http://parent/ @plugin=slice.so @pparam=-b @pparam=2M @plugin=cache_range_requests.so -``` - -for global plugins. - -``` -slice.so --blockbytes=2097152 -cache_range_requests.so -``` - -alternatively: - -``` -slice.so -b 2M -cache_range_requests.so +map https://ats-cache/ http://parent/ @plugin=slice.so @pparam=--blockbytes=2M @plugin=cache_range_requests.so ``` Options for the slice plugin (typically last one wins): ``` --blockbytes= (optional) Slice block size. - Default is 1m or 1048576 bytes. + Default is 1m or 1048576 bytes. also -b - Suffix k,m,g supported. - Limited to 32k and 32m inclusive. - For backwards compatibility blockbytes: is also supported. + Suffix k,m,g supported. + Limited to 32k and 32m inclusive. + For backwards compatibility blockbytes: is also supported. ---test-blockbytes= (optional) +--blockbytes-test= (optional) Slice block size for testing. also -t - Suffix k,m,g supported. - Limited to any positive number. - Ignored if --blockbytes is provided. + Suffix k,m,g supported. + Limited to any positive number. + Ignored if --blockbytes is provided. + +--remap-host= (optional) + Uses effective url with given host and port 0 for remapping. + Requires setting up an intermediate loopback remap rule. + -r for short --pace-errorlog= (optional) Limit stitching error logs to every 'n' second(s) Default is to log all errors (no pacing). - also -p + also -e --disable-errorlog (optional) Disable writing stitching errors to the error log. also -d ``` -**Note**: cache_range_requests **MUST** follow slice.so Put these plugins -at the end of the plugin list +By default the plugin uses the pristine url to loopback call back +into the same rule as each range slice is issued. The effective url +with loopback remap host may be used by adding the '-r ' or +'--remap-host=' plugin option. + +Using the `--remap-host` option splits the plugin chain into 2 remap rules. +One remap rule for all the incoming requests and the other for just the block +range requests. This allows for easier trouble shooting via logs and +also allows for more effecient plugin rules. The default pristine method +runs the remap plugins twice, one for the incoming request and one for +eace slice. Splitting the rules allows for plugins like URI signing to +be done on the client request only. + +NOTE: Requests NOT handled by the slice plugin (ie: HEAD requests) are +handled as with a typical remap rule. GET requests intercepted by the +slice plugin are virtually reissued into ATS and are forward proxied +through the cache_range_requests plugin. + +``` +map http://ats/ http://parent/ @plugin=slice.so @pparam=--blockbytes=512k @pparam=--remap-host=loopback +map https://ats/ https://parent/ @plugin=slice.so @pparam=--blockbytes=512k @pparam=--remap-host=loopback + +# Virtual forward proxy for slice range blocks +map http://loopback/ http://parent/ @plugin=cache_range_requests.so +map https://loopback/ http://parent/ @plugin=cache_range_requests.so +``` + +**Note**: For default pristine behavior cache_range_requests **MUST** +follow slice.so Put these plugins at the end of the plugin list **Note**: blockbytes is defined in bytes. Postfix for 'K', 'M' and 'G' may be used. 1048576 (1MB) is the default. -For testing purposes an unchecked value of "bytesover" is also available. +For testing purposes an unchecked value of "blockbytes-test" is also available. Debug output can be enable by setting the debug tag: **slice**. If debug is enabled all block stitch errors will log to diags.log @@ -90,16 +110,16 @@ provided to help with debugging. Below is a sample error log entry:: Current error types logged: ``` Mismatch block Etag - Mismatch block Last-Modified - Non 206 internal block response - Mismatch/Bad block Content-Range + Mismatch block Last-Modified + Non 206 internal block response + Mismatch/Bad block Content-Range ``` With slice error logs disabled these type errors can typically be detected by observing crc=ERR_READ_ERROR and pscl=0 in normal logs. -At the current time only single range requests or the first part of a +At the current time only single range requests or the first part of a multi part range request of the forms: ``` Range: bytes=- diff --git a/plugins/experimental/slice/Range.cc b/plugins/experimental/slice/Range.cc index bb06099722b..c8c4fa3036b 100644 --- a/plugins/experimental/slice/Range.cc +++ b/plugins/experimental/slice/Range.cc @@ -121,7 +121,7 @@ Range::fromStringClosed(char const *const rangestr) bool Range::toStringClosed(char *const bufstr, int *const buflen // returns actual bytes used - ) const +) const { if (!isValid()) { if (0 < *buflen) { @@ -152,6 +152,16 @@ Range::firstBlockFor(int64_t const blocksize) const } } +int64_t +Range::lastBlockFor(int64_t const blocksize) const +{ + if (0 < blocksize && isValid()) { + return std::max(static_cast(0), (m_end - 1) / blocksize); + } else { + return -1; + } +} + Range Range::intersectedWith(Range const &other) const { @@ -162,7 +172,6 @@ bool Range::blockIsInside(int64_t const blocksize, int64_t const blocknum) const { Range const blockrange(blocksize * blocknum, blocksize * (blocknum + 1)); - Range const isec(blockrange.intersectedWith(*this)); return isec.isValid(); diff --git a/plugins/experimental/slice/Range.h b/plugins/experimental/slice/Range.h index 0fb3145ddae..8c8826d2091 100644 --- a/plugins/experimental/slice/Range.h +++ b/plugins/experimental/slice/Range.h @@ -56,6 +56,10 @@ struct Range { */ int64_t firstBlockFor(int64_t const blockbytes) const; + /** block number of last (inclusive) range block + */ + int64_t lastBlockFor(int64_t const blockbytes) const; + /** block intersection */ Range intersectedWith(Range const &other) const; diff --git a/plugins/experimental/slice/Stage.h b/plugins/experimental/slice/Stage.h index 4166071e5eb..a2d7c4fdc6f 100644 --- a/plugins/experimental/slice/Stage.h +++ b/plugins/experimental/slice/Stage.h @@ -20,6 +20,11 @@ #include "ts/ts.h" +#include "slice.h" +#include "util.h" + +#include + struct Channel { TSVIO m_vio{nullptr}; TSIOBuffer m_iobuf{nullptr}; @@ -29,45 +34,67 @@ struct Channel { { if (nullptr != m_reader) { TSIOBufferReaderFree(m_reader); +#if defined(COLLECT_STATS) + TSStatIntDecrement(stats::Reader, 1); +#endif } if (nullptr != m_iobuf) { TSIOBufferDestroy(m_iobuf); } } - void + int64_t drainReader() { - TSAssert(nullptr != m_reader); - int64_t const bytes_avail = TSIOBufferReaderAvail(m_reader); - TSIOBufferReaderConsume(m_reader, bytes_avail); + int64_t consumed = 0; + + if (nullptr != m_reader && reader_avail_more_than(m_reader, 0)) { + int64_t const avail = TSIOBufferReaderAvail(m_reader); + TSIOBufferReaderConsume(m_reader, avail); + consumed = avail; + if (nullptr != m_vio) { + TSVIONDoneSet(m_vio, TSVIONDoneGet(m_vio) + consumed); + } + } + + return consumed; } bool - setForRead(TSVConn vc, TSCont contp, int64_t const bytesin //=INT64_MAX - ) + setForRead(TSVConn vc, TSCont contp, int64_t const bytesin) { TSAssert(nullptr != vc); if (nullptr == m_iobuf) { m_iobuf = TSIOBufferCreate(); m_reader = TSIOBufferReaderAlloc(m_iobuf); +#if defined(COLLECT_STATS) + TSStatIntIncrement(stats::Reader, 1); +#endif } else { - drainReader(); + int64_t const drained = drainReader(); + if (0 < drained) { + DEBUG_LOG("Drained from reader: %" PRId64, drained); + } } m_vio = TSVConnRead(vc, contp, m_iobuf, bytesin); return nullptr != m_vio; } bool - setForWrite(TSVConn vc, TSCont contp, int64_t const bytesout //=INT64_MAX - ) + setForWrite(TSVConn vc, TSCont contp, int64_t const bytesout) { TSAssert(nullptr != vc); if (nullptr == m_iobuf) { m_iobuf = TSIOBufferCreate(); m_reader = TSIOBufferReaderAlloc(m_iobuf); +#if defined(COLLECT_STATS) + TSStatIntIncrement(stats::Reader, 1); +#endif } else { - drainReader(); + int64_t const drained = drainReader(); + if (0 < drained) { + DEBUG_LOG("Drained from reader: %" PRId64, drained); + } } m_vio = TSVConnWrite(vc, contp, m_reader, bytesout); return nullptr != m_vio; @@ -85,7 +112,13 @@ struct Channel { bool isOpen() const { - return nullptr != m_iobuf && nullptr != m_reader && nullptr != m_vio; + return nullptr != m_vio; + } + + bool + isDrained() const + { + return nullptr == m_reader || !reader_avail_more_than(m_reader, 0); } }; @@ -112,38 +145,48 @@ struct Stage // upstream or downstream (server or client) if (nullptr != m_vc) { TSVConnClose(m_vc); } - m_vc = vc; - m_read.m_vio = nullptr; - m_write.m_vio = nullptr; + m_read.close(); + m_write.close(); + m_vc = vc; } void - setupVioRead(TSCont contp, int64_t const bytesin = INT64_MAX) + setupVioRead(TSCont contp, int64_t const bytesin) { m_read.setForRead(m_vc, contp, bytesin); } void - setupVioWrite(TSCont contp, int64_t const bytesout = INT64_MAX) + setupVioWrite(TSCont contp, int64_t const bytesout) { m_write.setForWrite(m_vc, contp, bytesout); } void - close() + abort() { + if (nullptr != m_vc) { + TSVConnAbort(m_vc, TS_VC_CLOSE_ABORT); + m_vc = nullptr; + } m_read.close(); m_write.close(); + } + void + close() + { if (nullptr != m_vc) { TSVConnClose(m_vc); m_vc = nullptr; } + m_read.close(); + m_write.close(); } bool isOpen() const { - return nullptr != m_vc && m_read.isOpen() && m_write.isOpen(); + return nullptr != m_vc && (m_read.isOpen() || m_write.isOpen()); } }; diff --git a/plugins/experimental/slice/client.cc b/plugins/experimental/slice/client.cc index 47ac4e3383c..6d8d44c92ed 100644 --- a/plugins/experimental/slice/client.cc +++ b/plugins/experimental/slice/client.cc @@ -18,95 +18,38 @@ #include "client.h" -#include "transfer.h" +#include "Config.h" +#include "util.h" -namespace -{ -void -shutdown(TSCont const contp, Data *const data) -{ - DEBUG_LOG("shutting down transaction"); - delete data; - TSContDestroy(contp); -} - -// create and issue a block request -bool -requestBlock(TSCont contp, Data *const data) -{ - int64_t const blockbeg = (data->m_config->m_blockbytes * data->m_blocknum); - Range blockbe(blockbeg, blockbeg + data->m_config->m_blockbytes); - - char rangestr[1024]; - int rangelen = sizeof(rangestr); - bool const rpstat = blockbe.toStringClosed(rangestr, &rangelen); - TSAssert(rpstat); - - DEBUG_LOG("requestBlock: %s", rangestr); - - // reuse the incoming client header, just change the range - HttpHeader header(data->m_req_hdrmgr.m_buffer, data->m_req_hdrmgr.m_lochdr); - - // add/set sub range key and add slicer tag - bool const rangestat = header.setKeyVal(TS_MIME_FIELD_RANGE, TS_MIME_LEN_RANGE, rangestr, rangelen); - - if (!rangestat) { - ERROR_LOG("Error trying to set range request header %s", rangestr); - return false; - } - - // create virtual connection back into ATS - TSVConn const upvc = TSHttpConnect(reinterpret_cast(&data->m_client_ip)); - - // set up connection with the HttpConnect server, maybe clear old one - data->m_upstream.setupConnection(upvc); - data->m_upstream.setupVioWrite(contp); - - TSHttpHdrPrint(header.m_buffer, header.m_lochdr, data->m_upstream.m_write.m_iobuf); - TSVIOReenable(data->m_upstream.m_write.m_vio); - - /* - std::string const headerstr(header.toString()); - DEBUG_LOG("Headers\n%s", headerstr.c_str()); - */ - - // get ready for data back from the server - data->m_upstream.setupVioRead(contp); - - // anticipate the next server response header - TSHttpParserClear(data->m_http_parser); - data->m_resp_hdrmgr.resetHeader(); - - data->m_blockexpected = 0; - data->m_blockconsumed = 0; - data->m_iseos = false; - data->m_server_block_header_parsed = false; - - return true; -} - -} // namespace +#include // this is called once per transaction when the client sends a req header bool handle_client_req(TSCont contp, TSEvent event, Data *const data) { - if (TS_EVENT_VCONN_READ_READY == event || TS_EVENT_VCONN_READ_COMPLETE == event) { + switch (event) { + case TS_EVENT_VCONN_READ_READY: + case TS_EVENT_VCONN_READ_COMPLETE: { if (nullptr == data->m_http_parser) { data->m_http_parser = TSHttpParserCreate(); } - // the client request header didn't fit into the input buffer: + // Read the header from the buffer + int64_t consumed = 0; if (TS_PARSE_DONE != - data->m_req_hdrmgr.populateFrom(data->m_http_parser, data->m_dnstream.m_read.m_reader, TSHttpHdrParseReq)) { + data->m_req_hdrmgr.populateFrom(data->m_http_parser, data->m_dnstream.m_read.m_reader, TSHttpHdrParseReq, &consumed)) { return false; } + // update the VIO + TSVIO const input_vio = data->m_dnstream.m_read.m_vio; + TSVIONDoneSet(input_vio, TSVIONDoneGet(input_vio) + consumed); + // make the header manipulator HttpHeader header(data->m_req_hdrmgr.m_buffer, data->m_req_hdrmgr.m_lochdr); // set the request url back to pristine in case of plugin stacking - header.setUrl(data->m_urlbuffer, data->m_urlloc); + header.setUrl(data->m_urlbuf, data->m_urlloc); header.setKeyVal(TS_MIME_FIELD_HOST, TS_MIME_LEN_HOST, data->m_hostname, data->m_hostlen); @@ -123,18 +66,18 @@ handle_client_req(TSCont contp, TSEvent event, Data *const data) bool const isRangeGood = rangebe.fromStringClosed(rangestr); if (isRangeGood) { - DEBUG_LOG("Partial content request"); + DEBUG_LOG("%p Partial content request", data); data->m_statustype = TS_HTTP_STATUS_PARTIAL_CONTENT; } else // signal a 416 needs to be formed and sent { - DEBUG_LOG("Ill formed/unhandled range: %s", rangestr); + DEBUG_LOG("%p Ill formed/unhandled range: %s", data, rangestr); data->m_statustype = TS_HTTP_STATUS_REQUESTED_RANGE_NOT_SATISFIABLE; // First block will give Content-Length rangebe = Range(0, data->m_config->m_blockbytes); } } else { - DEBUG_LOG("Full content request"); + DEBUG_LOG("%p Full content request", data); static char const *const valstr = "-"; static size_t const vallen = strlen(valstr); header.setKeyVal(SLICER_MIME_FIELD_INFO, strlen(SLICER_MIME_FIELD_INFO), valstr, vallen); @@ -151,8 +94,8 @@ handle_client_req(TSCont contp, TSEvent event, Data *const data) header.removeKey(TS_MIME_FIELD_X_FORWARDED_FOR, TS_MIME_LEN_X_FORWARDED_FOR); // send the first block request to server - if (!requestBlock(contp, data)) { - shutdown(contp, data); + if (!request_block(contp, data)) { + abort(contp, data); return false; } @@ -163,6 +106,10 @@ handle_client_req(TSCont contp, TSEvent event, Data *const data) header.removeKey(TS_MIME_FIELD_IF_NONE_MATCH, TS_MIME_LEN_IF_NONE_MATCH); header.removeKey(TS_MIME_FIELD_IF_RANGE, TS_MIME_LEN_IF_RANGE); header.removeKey(TS_MIME_FIELD_IF_UNMODIFIED_SINCE, TS_MIME_LEN_IF_UNMODIFIED_SINCE); + } break; + default: { + DEBUG_LOG("%p handle_client_req unhandled event %d %s", data, event, TSHttpEventNameLookup(event)); + } break; } return true; @@ -172,61 +119,47 @@ handle_client_req(TSCont contp, TSEvent event, Data *const data) void handle_client_resp(TSCont contp, TSEvent event, Data *const data) { - if (TS_EVENT_VCONN_WRITE_READY == event || TS_EVENT_VCONN_WRITE_COMPLETE == event) { - transfer_content_bytes(data); - - // done transferring from server to client buffer? - if (data->m_bytestosend <= data->m_bytessent) { - // real amount transferred to client - int64_t const bytessent(TSVIONDoneGet(data->m_dnstream.m_write.m_vio)); - - // is the output buffer drained? - if (data->m_bytestosend <= bytessent) { - data->m_dnstream.close(); - if (!data->m_upstream.m_read.isOpen()) { - shutdown(contp, data); - return; + DEBUG_LOG("%p handle_client_resp %s", data, TSHttpEventNameLookup(event)); + +#if defined(COLLECT_STATS) + TSStatIntIncrement(stats::Client, 1); +#endif + + switch (event) { + case TS_EVENT_VCONN_WRITE_READY: { + if (Data::BlockState::Pending == data->m_blockstate) { + bool start_next_block = true; + + if (data->m_config->m_throttle) { + TSVIO const output_vio = data->m_dnstream.m_write.m_vio; + int64_t const output_done = TSVIONDoneGet(output_vio); + int64_t const output_sent = data->m_bytessent; + int64_t const threshout = data->m_config->m_blockbytes; + + if (threshout < (output_sent - output_done)) { + start_next_block = false; + DEBUG_LOG("%p handle_client_resp: throttling %" PRId64, data, (output_sent - output_done)); } } - // continue allowing the downstream to drain - return; - } - - // error condition from the server side - if (data->m_bail) { - shutdown(contp, data); - return; - } - - // check for upstream eos, maybe request next block - if (data->m_iseos) { - // still need to drain the server side - if (0 < TSIOBufferReaderAvail(data->m_upstream.m_read.m_reader)) { - TSVIOReenable(data->m_dnstream.m_write.m_vio); - return; - } - - // if done or partial block - if (data->m_blocknum < 0 || data->m_blockconsumed < data->m_blockexpected) { - shutdown(contp, data); - return; + if (start_next_block) { + request_block(contp, data); } - - // ready for next block - requestBlock(contp, data); } - } - // client closed connection - else if (TS_EVENT_ERROR == event) { - DEBUG_LOG("got a TS_EVENT_ERROR from the client"); + } break; + case TS_EVENT_VCONN_WRITE_COMPLETE: { + if (TSIsDebugTagSet(PLUGIN_NAME) && reader_avail_more_than(data->m_upstream.m_read.m_reader, 0)) { + int64_t const left = TSIOBufferReaderAvail(data->m_upstream.m_read.m_reader); + DEBUG_LOG("%p WRITE_COMPLETE called with %" PRId64 " bytes left", data, left); + } - // allow the upstream server to drain data->m_dnstream.close(); if (!data->m_upstream.m_read.isOpen()) { shutdown(contp, data); } - } else { - DEBUG_LOG("Unhandled event: %d", event); + } break; + default: { + DEBUG_LOG("%p handle_client_resp unhandled event %d %s", data, event, TSHttpEventNameLookup(event)); + } break; } } diff --git a/plugins/experimental/slice/client.h b/plugins/experimental/slice/client.h index cc27bee3b1d..b91516165d8 100644 --- a/plugins/experimental/slice/client.h +++ b/plugins/experimental/slice/client.h @@ -27,6 +27,8 @@ * New block requests are also initiated by the client. */ +bool requestBlock(TSCont contp, Data *const data); + /** returns true if the incoming vio can be turned off */ bool handle_client_req(TSCont contp, TSEvent event, Data *const data); diff --git a/plugins/experimental/slice/intercept.cc b/plugins/experimental/slice/intercept.cc index e3a325b3d4b..2b6ef3bdfcd 100644 --- a/plugins/experimental/slice/intercept.cc +++ b/plugins/experimental/slice/intercept.cc @@ -26,11 +26,10 @@ int intercept_hook(TSCont contp, TSEvent event, void *edata) { - // DEBUG_LOG("intercept_hook: %d", event); - Data *const data = static_cast(TSContDataGet(contp)); + if (nullptr == data) { - DEBUG_LOG("Events handled after data already torn down"); + ERROR_LOG("intercept_hook called without data"); TSContDestroy(contp); return TS_EVENT_ERROR; } @@ -42,15 +41,15 @@ intercept_hook(TSCont contp, TSEvent event, void *edata) // set up reader from client TSVConn const downvc = static_cast(edata); data->m_dnstream.setupConnection(downvc); - data->m_dnstream.setupVioRead(contp); + data->m_dnstream.setupVioRead(contp, INT64_MAX); } break; + case TS_EVENT_NET_ACCEPT_FAILED: case TS_EVENT_VCONN_INACTIVITY_TIMEOUT: case TS_EVENT_VCONN_ACTIVE_TIMEOUT: - case TS_EVENT_HTTP_TXN_CLOSE: - delete data; - TSContDestroy(contp); - break; + case TS_EVENT_ERROR: { + abort(contp, data); + } break; default: { // data from client -- only the initial header @@ -66,7 +65,7 @@ intercept_hook(TSCont contp, TSEvent event, void *edata) // DEBUG_LOG("shutting down send to server pipe"); TSVConnShutdown(data->m_upstream.m_vc, 0, 1); } - // server has data for us, typically handle just the header + // server has data for us else if (data->m_upstream.m_read.isOpen() && edata == data->m_upstream.m_read.m_vio) { handle_server_resp(contp, event, data); } @@ -75,13 +74,8 @@ intercept_hook(TSCont contp, TSEvent event, void *edata) handle_client_resp(contp, event, data); } else { ERROR_LOG("Unhandled event: %d", event); - /* - std::cerr << __func__ - << ": events received after intercept state torn down" - << std::endl; - */ } - } + } break; } return TS_EVENT_CONTINUE; diff --git a/plugins/experimental/slice/server.cc b/plugins/experimental/slice/server.cc index 8c9c7c6fa28..25d21472109 100644 --- a/plugins/experimental/slice/server.cc +++ b/plugins/experimental/slice/server.cc @@ -18,9 +18,11 @@ #include "server.h" +#include "Config.h" #include "ContentRange.h" #include "response.h" #include "transfer.h" +#include "util.h" #include "ts/experimental.h" @@ -28,14 +30,6 @@ namespace { -void -shutdown(TSCont const contp, Data *const data) -{ - DEBUG_LOG("shutting down transaction"); - delete data; - TSContDestroy(contp); -} - ContentRange contentRangeFrom(HttpHeader const &header) { @@ -66,15 +60,20 @@ handleFirstServerHeader(Data *const data, TSCont const contp) // DEBUG_LOG("First header\n%s", header.toString().c_str()); - data->m_dnstream.setupVioWrite(contp); + data->m_dnstream.setupVioWrite(contp, INT64_MAX); + + TSVIO const output_vio = data->m_dnstream.m_write.m_vio; + TSIOBuffer const output_buf = data->m_dnstream.m_write.m_iobuf; - // only process a 206, everything else gets a pass through + // only process a 206, everything else gets a (possibly incomplete) + // pass through if (TS_HTTP_STATUS_PARTIAL_CONTENT != header.status()) { DEBUG_LOG("Initial response other than 206: %d", header.status()); - data->m_bail = true; - - TSHttpHdrPrint(header.m_buffer, header.m_lochdr, data->m_dnstream.m_write.m_iobuf); + // Should run TSVIONSetBytes(output_io, hlen + bodybytes); + // int const hlen = TSHttpHdrLengthGet(header.m_buffer, header.m_lochdr); + // TSVIONBytesSet(output_vio, hlen); + TSHttpHdrPrint(header.m_buffer, header.m_lochdr, output_buf); transfer_all_bytes(data); return false; @@ -83,13 +82,10 @@ handleFirstServerHeader(Data *const data, TSCont const contp) ContentRange const blockcr = contentRangeFrom(header); // 206 with bad content range? if (!blockcr.isValid()) { - data->m_bail = true; - static std::string const &msg502 = string502(); - - TSIOBufferWrite(data->m_dnstream.m_write.m_iobuf, msg502.data(), msg502.size()); - TSVIOReenable(data->m_dnstream.m_write.m_vio); - + TSVIONBytesSet(output_vio, msg502.size()); + TSIOBufferWrite(output_buf, msg502.data(), msg502.size()); + TSVIOReenable(output_vio); return false; } @@ -108,18 +104,21 @@ handleFirstServerHeader(Data *const data, TSCont const contp) int64_t const bodybytes = data->m_req_range.size(); - // range past end of data, assume 416 needs to be sent + // range begins past end of data but inside last block, send 416 bool const send416 = (bodybytes <= 0 || TS_HTTP_STATUS_REQUESTED_RANGE_NOT_SATISFIABLE == data->m_statustype); if (send416) { - data->m_bail = true; std::string const &bodystr = bodyString416(); form416HeaderAndBody(header, data->m_contentlen, bodystr); - TSHttpHdrPrint(header.m_buffer, header.m_lochdr, data->m_dnstream.m_write.m_iobuf); + int const hlen = TSHttpHdrLengthGet(header.m_buffer, header.m_lochdr); + int64_t const blen = bodystr.size(); - TSIOBufferWrite(data->m_dnstream.m_write.m_iobuf, bodystr.data(), bodystr.size()); + TSVIONBytesSet(output_vio, int64_t(hlen) + blen); + TSHttpHdrPrint(header.m_buffer, header.m_lochdr, output_buf); + TSIOBufferWrite(output_buf, bodystr.data(), bodystr.size()); + TSVIOReenable(output_vio); - TSVIOReenable(data->m_dnstream.m_write.m_vio); + data->m_upstream.m_read.close(); return false; } @@ -146,8 +145,6 @@ handleFirstServerHeader(Data *const data, TSCont const contp) // corner case, return 500 ?? if (!crstat) { - data->m_bail = true; - data->m_upstream.close(); data->m_dnstream.close(); @@ -156,9 +153,7 @@ handleFirstServerHeader(Data *const data, TSCont const contp) } header.setKeyVal(TS_MIME_FIELD_CONTENT_RANGE, TS_MIME_LEN_CONTENT_RANGE, rangestr, rangelen); - } - // fix up for 200 response - else if (TS_HTTP_STATUS_OK == data->m_statustype) { + } else if (TS_HTTP_STATUS_OK == data->m_statustype) { header.setStatus(TS_HTTP_STATUS_OK); static char const *const reason = TSHttpHdrReasonLookup(TS_HTTP_STATUS_OK); header.setReason(reason, strlen(reason)); @@ -170,15 +165,13 @@ handleFirstServerHeader(Data *const data, TSCont const contp) header.setKeyVal(TS_MIME_FIELD_CONTENT_LENGTH, TS_MIME_LEN_CONTENT_LENGTH, bufstr, buflen); // add the response header length to the total bytes to send - int64_t const headerbytes = TSHttpHdrLengthGet(header.m_buffer, header.m_lochdr); + int const hbytes = TSHttpHdrLengthGet(header.m_buffer, header.m_lochdr); - data->m_bytestosend = headerbytes + bodybytes; - - TSHttpHdrPrint(header.m_buffer, header.m_lochdr, data->m_dnstream.m_write.m_iobuf); - - data->m_bytessent = headerbytes; - - TSVIOReenable(data->m_dnstream.m_write.m_vio); + TSVIONBytesSet(output_vio, hbytes + bodybytes); + data->m_bytestosend = hbytes + bodybytes; + TSHttpHdrPrint(header.m_buffer, header.m_lochdr, output_buf); + data->m_bytessent = hbytes; + TSVIOReenable(output_vio); return true; } @@ -304,7 +297,6 @@ handleNextServerHeader(Data *const data, TSCont const contp) // only process a 206, everything else just aborts if (TS_HTTP_STATUS_PARTIAL_CONTENT != header.status()) { logSliceError("Non 206 internal block response", data, header); - data->m_bail = true; return false; } @@ -312,7 +304,6 @@ handleNextServerHeader(Data *const data, TSCont const contp) ContentRange const blockcr = contentRangeFrom(header); if (!blockcr.isValid() || blockcr.m_length != data->m_contentlen) { logSliceError("Mismatch/Bad block Content-Range", data, header); - data->m_bail = true; return false; } @@ -341,7 +332,7 @@ handleNextServerHeader(Data *const data, TSCont const contp) } if (!same) { - data->m_bail = true; + data->m_upstream.close(); return false; } @@ -356,30 +347,46 @@ handleNextServerHeader(Data *const data, TSCont const contp) void handle_server_resp(TSCont contp, TSEvent event, Data *const data) { - if (TS_EVENT_VCONN_READ_READY == event || TS_EVENT_VCONN_READ_COMPLETE == event) { + DEBUG_LOG("%p handle_server_resp: %s", data, TSHttpEventNameLookup(event)); + +#if defined(COLLECT_STATS) + TSStatIntIncrement(stats::Server, 1); +#endif + + switch (event) { + case TS_EVENT_VCONN_READ_READY: { // has block response header been parsed?? if (!data->m_server_block_header_parsed) { - // the server response header didn't fit into the input buffer?? - if (TS_PARSE_DONE != - data->m_resp_hdrmgr.populateFrom(data->m_http_parser, data->m_upstream.m_read.m_reader, TSHttpHdrParseResp)) { + int64_t consumed = 0; + TSIOBufferReader const reader = data->m_upstream.m_read.m_reader; + TSVIO const input_vio = data->m_upstream.m_read.m_vio; + TSParseResult const res = data->m_resp_hdrmgr.populateFrom(data->m_http_parser, reader, TSHttpHdrParseResp, &consumed); + + TSVIONDoneSet(input_vio, TSVIONDoneGet(input_vio) + consumed); + + // the server response header didn't fit into the input buffer. + if (TS_PARSE_CONT == res) { return; } // very first server response header bool headerStat = false; - if (!data->m_server_first_header_parsed) { - headerStat = handleFirstServerHeader(data, contp); - data->m_server_first_header_parsed = true; - } else { - headerStat = handleNextServerHeader(data, contp); - } - data->m_server_block_header_parsed = true; + if (TS_PARSE_DONE == res) { + if (!data->m_server_first_header_parsed) { + headerStat = handleFirstServerHeader(data, contp); + data->m_server_first_header_parsed = true; + } else { + headerStat = handleNextServerHeader(data, contp); + } + + data->m_server_block_header_parsed = true; + } // kill the upstream and allow dnstream to clean up if (!headerStat) { - data->m_upstream.close(); - data->m_bail = true; + data->m_upstream.abort(); + data->m_blockstate = Data::BlockState::Fail; if (data->m_dnstream.m_write.isOpen()) { TSVIOReenable(data->m_dnstream.m_write.m_vio); } else { @@ -393,41 +400,22 @@ handle_server_resp(TSCont contp, TSEvent event, Data *const data) } transfer_content_bytes(data); - } else if (TS_EVENT_VCONN_EOS == event) { - // from testing as far as I can tell, if the sub transaction returns - // a valid header TS_EVENT_VCONN_READ_READY event is always called first. - // this event being called means the input stream is null. - // An upstream transaction that aborts immediately (or a few bytes) - // after it sends a header may end up here with nothing in the upstream - // buffer. - - // this is called when the upstream connection is done. - // make sure to drain all the bytes out before - // issuing the next block request - data->m_iseos = true; - - // corner condition, good source header + 0 length aborted content - // results in no header being read, just an EOS. - // trying to delete the upstream will crash ATS (??) - if (0 == data->m_blockexpected) { - shutdown(contp, data); // this will crash if first block + } break; + case TS_EVENT_VCONN_READ_COMPLETE: { + // fprintf(stderr, "%p: TS_EVENT_VCONN_READ_COMPLETE\n", data); + } break; + case TS_EVENT_VCONN_EOS: { + data->m_blockstate = Data::BlockState::Pending; + data->m_upstream.close(); + + // check for block truncation + if (data->m_blockconsumed < data->m_blockexpected) { + DEBUG_LOG("%p handle_server_resp truncation: %" PRId64 "\n", data, data->m_blockexpected - data->m_blockconsumed); + data->m_blockstate = Data::BlockState::Fail; + // shutdown(contp, data); return; } - transfer_content_bytes(data); - - if (!data->m_dnstream.m_write.isOpen()) // server drain condition - { - shutdown(contp, data); - return; - } - - // all bytes left transferred to client buffer - if (0 == TSIOBufferReaderAvail(data->m_upstream.m_read.m_reader)) { - data->m_upstream.close(); - TSVIOReenable(data->m_dnstream.m_write.m_vio); - } - // prepare for the next request block ++data->m_blocknum; @@ -435,16 +423,45 @@ handle_server_resp(TSCont contp, TSEvent event, Data *const data) // issues a speculative request for the first block // in that case fast forward to the real first in range block // Btw this isn't implemented yet, to be handled - int64_t const firstblock(data->m_req_range.firstBlockFor(data->m_config->m_blockbytes)); + int64_t const firstblock = data->m_req_range.firstBlockFor(data->m_config->m_blockbytes); if (data->m_blocknum < firstblock) { data->m_blocknum = firstblock; } - // done processing blocks? - if (!data->m_req_range.blockIsInside(data->m_config->m_blockbytes, data->m_blocknum)) { - data->m_blocknum = -1; // signal value no more blocks + // continue processing blocks? + if (data->m_req_range.blockIsInside(data->m_config->m_blockbytes, data->m_blocknum)) { + // Don't immediately request the next slice if the client + // isn't keeping up + + bool start_next_block = true; + + // throttle condition + if (data->m_config->m_throttle && data->m_dnstream.m_read.isOpen()) { + TSVIO const output_vio = data->m_dnstream.m_write.m_vio; + int64_t const output_done = TSVIONDoneGet(output_vio); + int64_t const output_sent = data->m_bytessent; + int64_t const threshout = data->m_config->m_blockbytes; + + if (threshout < (output_sent - output_done)) { + start_next_block = false; + DEBUG_LOG("%p handle_server_resp: throttling %" PRId64, data, (output_sent - output_done)); + } + } + + if (start_next_block) { + request_block(contp, data); + } + + } else { + data->m_upstream.close(); + data->m_blockstate = Data::BlockState::Done; + if (!data->m_dnstream.m_read.isOpen()) { + shutdown(contp, data); + } } - } else { - DEBUG_LOG("Unhandled event: %d", event); + } break; + default: { + DEBUG_LOG("%p handle_server_resp uhandled event: %s", data, TSHttpEventNameLookup(event)); + } break; } } diff --git a/plugins/experimental/slice/slice.cc b/plugins/experimental/slice/slice.cc index 5b5c90b48d9..a091f5c7ac9 100644 --- a/plugins/experimental/slice/slice.cc +++ b/plugins/experimental/slice/slice.cc @@ -27,6 +27,19 @@ #include "ts/ts.h" #include +#include +#include + +#if defined(COLLECT_STATS) +namespace stats +{ +int DataCreate = -1; +int DataDestroy = -1; +int Reader = -1; +int Server = -1; +int Client = -1; +} // namespace stats +#endif // COLLECT_STATS namespace { @@ -43,6 +56,29 @@ read_request(TSHttpTxn txnp, Config *const config) if (TS_HTTP_METHOD_GET == header.method()) { static int const SLICER_MIME_LEN_INFO = strlen(SLICER_MIME_FIELD_INFO); if (!header.hasKey(SLICER_MIME_FIELD_INFO, SLICER_MIME_LEN_INFO)) { + // check if any previous plugin has monkeyed with the transaction status + TSHttpStatus const txnstat = TSHttpTxnStatusGet(txnp); + if (0 != static_cast(txnstat)) { + DEBUG_LOG("txn status change detected (%d), skipping plugin\n", (int)txnstat); + return false; + } + + if (config->hasRegex()) { + int urllen = 0; + char *const urlstr = TSHttpTxnEffectiveUrlStringGet(txnp, &urllen); + if (nullptr != urlstr) { + bool const shouldslice = config->matchesRegex(urlstr, urllen); + if (!shouldslice) { + DEBUG_LOG("request failed regex, not slicing: '%.*s'", urllen, urlstr); + TSfree(urlstr); + return false; + } + + DEBUG_LOG("request passed regex, slicing: '%.*s'", urllen, urlstr); + TSfree(urlstr); + } + } + // turn off any and all transaction caching (shouldn't matter) TSHttpTxnServerRespNoStoreSet(txnp, 1); TSHttpTxnRespCacheableSet(txnp, 0); @@ -76,32 +112,77 @@ read_request(TSHttpTxn txnp, Config *const config) return false; } - // need the pristine url, especially for global plugins - TSMBuffer urlbuf; - TSMLoc urlloc; - TSReturnCode rcode = TSHttpTxnPristineUrlGet(txnp, &urlbuf, &urlloc); - - if (TS_SUCCESS == rcode) { - TSMBuffer const newbuf = TSMBufferCreate(); - TSMLoc newloc = nullptr; - rcode = TSUrlClone(newbuf, urlbuf, urlloc, &newloc); - TSHandleMLocRelease(urlbuf, TS_NULL_MLOC, urlloc); - - if (TS_SUCCESS != rcode) { - ERROR_LOG("Error cloning pristine url"); - delete data; - TSMBufferDestroy(newbuf); - return false; + // is the plugin configured to use a remap host? + std::string const &newhost = config->m_remaphost; + if (newhost.empty()) { + TSMBuffer urlbuf = nullptr; + TSMLoc urlloc = nullptr; + TSReturnCode rcode = TSHttpTxnPristineUrlGet(txnp, &urlbuf, &urlloc); + + if (TS_SUCCESS == rcode) { + TSMBuffer const newbuf = TSMBufferCreate(); + TSMLoc newloc = nullptr; + rcode = TSUrlClone(newbuf, urlbuf, urlloc, &newloc); + TSHandleMLocRelease(urlbuf, TS_NULL_MLOC, urlloc); + + if (TS_SUCCESS != rcode) { + ERROR_LOG("Error cloning pristine url"); + TSMBufferDestroy(newbuf); + delete data; + return false; + } + + data->m_urlbuf = newbuf; + data->m_urlloc = newloc; + } + } else { // grab the effective url, swap out the host and zero the port + int len = 0; + char *const effstr = TSHttpTxnEffectiveUrlStringGet(txnp, &len); + + if (nullptr != effstr) { + TSMBuffer const newbuf = TSMBufferCreate(); + TSMLoc newloc = nullptr; + bool okay = false; + + if (TS_SUCCESS == TSUrlCreate(newbuf, &newloc)) { + char const *start = effstr; + if (TS_PARSE_DONE == TSUrlParse(newbuf, newloc, &start, start + len)) { + if (TS_SUCCESS == TSUrlHostSet(newbuf, newloc, newhost.c_str(), newhost.size()) && + TS_SUCCESS == TSUrlPortSet(newbuf, newloc, 0)) { + okay = true; + } + } + } + + TSfree(effstr); + + if (!okay) { + ERROR_LOG("Error cloning effective url"); + if (nullptr != newloc) { + TSHandleMLocRelease(newbuf, nullptr, newloc); + } + TSMBufferDestroy(newbuf); + delete data; + return false; + } + + data->m_urlbuf = newbuf; + data->m_urlloc = newloc; } + } - data->m_urlbuffer = newbuf; - data->m_urlloc = newloc; + if (TSIsDebugTagSet(PLUGIN_NAME)) { + int len = 0; + char *const urlstr = TSUrlStringGet(data->m_urlbuf, data->m_urlloc, &len); + DEBUG_LOG("slice url: %.*s", len, urlstr); + TSfree(urlstr); } // we'll intercept this GET and do it ourselves - TSCont const icontp(TSContCreate(intercept_hook, TSMutexCreate())); + TSMutex const mutex = TSContMutexGet(reinterpret_cast(txnp)); + // TSMutex const mutex = TSMutexCreate(); + TSCont const icontp(TSContCreate(intercept_hook, mutex)); TSContDataSet(icontp, (void *)data); - // TSHttpTxnHookAdd(txnp, TS_HTTP_TXN_CLOSE_HOOK, icontp); TSHttpTxnIntercept(icontp, txnp); return true; } else { @@ -175,7 +256,44 @@ SLICE_EXPORT TSReturnCode TSRemapInit(TSRemapInterface *api_info, char *errbug, int errbuf_size) { - DEBUG_LOG("slice remap is successfully initialized."); + DEBUG_LOG("slice remap initializing."); + +#if defined(COLLECT_STATS) + static bool init = false; + static std::mutex mutex; + + std::lock_guard lock(mutex); + + if (!init) { + init = true; + + std::string const namedatacreate = std::string(PLUGIN_NAME) + ".DataCreate"; + stats::DataCreate = TSStatCreate(namedatacreate.c_str(), TS_RECORDDATATYPE_INT, TS_STAT_NON_PERSISTENT, TS_STAT_SYNC_SUM); + + assert(0 <= stats::DataCreate); + + std::string const namedatadestroy = std::string(PLUGIN_NAME) + ".DataDestroy"; + stats::DataDestroy = TSStatCreate(namedatadestroy.c_str(), TS_RECORDDATATYPE_INT, TS_STAT_NON_PERSISTENT, TS_STAT_SYNC_SUM); + + assert(0 <= stats::DataDestroy); + + std::string const namereader = std::string(PLUGIN_NAME) + ".Reader"; + stats::Reader = TSStatCreate(namereader.c_str(), TS_RECORDDATATYPE_INT, TS_STAT_NON_PERSISTENT, TS_STAT_SYNC_SUM); + + assert(0 <= stats::Reader); + + std::string const nameserver = std::string(PLUGIN_NAME) + ".Server"; + stats::Server = TSStatCreate(nameserver.c_str(), TS_RECORDDATATYPE_INT, TS_STAT_NON_PERSISTENT, TS_STAT_SYNC_SUM); + + assert(0 <= stats::Server); + + std::string const nameclient = std::string(PLUGIN_NAME) + ".Client"; + stats::Client = TSStatCreate(nameclient.c_str(), TS_RECORDDATATYPE_INT, TS_STAT_NON_PERSISTENT, TS_STAT_SYNC_SUM); + + assert(0 <= stats::Client); + } +#endif // COLLECT_STATS + return TS_SUCCESS; } diff --git a/plugins/experimental/slice/slice.h b/plugins/experimental/slice/slice.h index 682c0168c5a..d4e040b32f2 100644 --- a/plugins/experimental/slice/slice.h +++ b/plugins/experimental/slice/slice.h @@ -52,3 +52,14 @@ #define ERROR_LOG(fmt, ...) #endif + +#if defined(COLLECT_STATS) +namespace stats +{ +extern int DataCreate; +extern int DataDestroy; +extern int Reader; +extern int Server; +extern int Client; +} // namespace stats +#endif // COLLECT_STATS diff --git a/plugins/experimental/slice/transfer.cc b/plugins/experimental/slice/transfer.cc index 4ab7a8bb7c1..f83b0882c0b 100644 --- a/plugins/experimental/slice/transfer.cc +++ b/plugins/experimental/slice/transfer.cc @@ -18,64 +18,81 @@ #include "transfer.h" -int64_t transfer_content_bytes(Data *const data) // , char const * const fstr) +int64_t +transfer_content_bytes(Data *const data) { - int64_t consumed(0); - - // is the downstream is fulfilled or closed - if (!data->m_dnstream.m_write.isOpen()) { - // drain the upstream - if (data->m_upstream.m_read.isOpen()) { - int64_t const avail = TSIOBufferReaderAvail(data->m_upstream.m_read.m_reader); - TSIOBufferReaderConsume(data->m_upstream.m_read.m_reader, avail); - consumed += avail; - } - } else // if (data->m_dnstream.m_write.isOpen()) - { - if (data->m_upstream.m_read.isOpen()) { - int64_t avail = TSIOBufferReaderAvail(data->m_upstream.m_read.m_reader); - if (0 < avail) { - int64_t const toskip = std::min(data->m_blockskip, avail); - - // consume any up front (first block) padding - if (0 < toskip) { - TSIOBufferReaderConsume(data->m_upstream.m_read.m_reader, toskip); - data->m_blockskip -= toskip; - avail -= toskip; - consumed += toskip; - } + // nothing to transfer if there's no source. + if (nullptr == data->m_upstream.m_read.m_reader) { + return 0; + } - if (0 < avail) { - int64_t const bytesleft = (data->m_bytestosend - data->m_bytessent); - int64_t const tocopy = std::min(avail, bytesleft); + TSIOBufferReader const reader = data->m_upstream.m_read.m_reader; + TSIOBuffer const output_buf = data->m_dnstream.m_write.m_iobuf; + TSVIO const output_vio = data->m_dnstream.m_write.m_vio; - if (0 < tocopy) { - int64_t const copied(TSIOBufferCopy(data->m_dnstream.m_write.m_iobuf, data->m_upstream.m_read.m_reader, tocopy, 0)); + int64_t consumed = 0; // input vio bytes visited + int64_t copied = 0; // output bytes transferred - data->m_bytessent += copied; + bool const canWrite = data->m_dnstream.m_write.isOpen(); + bool done = false; - TSIOBufferReaderConsume(data->m_upstream.m_read.m_reader, copied); + TSIOBufferBlock block = TSIOBufferReaderStart(reader); - avail -= copied; - consumed += copied; - } - } + while (!done && nullptr != block) { + int64_t bavail = TSIOBufferBlockReadAvail(block, reader); + + if (0 == bavail) { + block = TSIOBufferBlockNext(block); + } else { + int64_t toconsume = 0; - // if hit fulfillment start bulk consuming - if (0 < avail && data->m_bytestosend <= data->m_bytessent) { - TSIOBufferReaderConsume(data->m_upstream.m_read.m_reader, avail); - consumed += avail; + if (canWrite) { + int64_t const toskip = std::min(data->m_blockskip, bavail); + if (0 < toskip) { // before bytes + toconsume = toskip; + data->m_blockskip -= toskip; + } else { + int64_t const bytesleft = data->m_bytestosend - data->m_bytessent; + if (0 < bytesleft) { // transfer bytes + int64_t const tocopy = std::min(bavail, bytesleft); + int64_t const nbytes = TSIOBufferCopy(output_buf, reader, tocopy, 0); + + done = (nbytes < tocopy); // output buffer stuffed + + copied += nbytes; + data->m_bytessent += nbytes; + + toconsume = nbytes; + } else { // after bytes + toconsume = bavail; + } } + } else { // drain + toconsume = bavail; } - if (0 < consumed) { - TSVIOReenable(data->m_dnstream.m_write.m_vio); + if (0 < toconsume) { + if (bavail == toconsume) { + block = TSIOBufferBlockNext(block); + } + TSIOBufferReaderConsume(reader, toconsume); + consumed += toconsume; } } } + // tell output more data is available + if (0 < copied) { + TSVIOReenable(output_vio); + } + if (0 < consumed) { data->m_blockconsumed += consumed; + + TSVIO const input_vio = data->m_upstream.m_read.m_vio; + if (nullptr != input_vio) { + TSVIONDoneSet(input_vio, TSVIONDoneGet(input_vio) + consumed); + } } return consumed; @@ -85,21 +102,50 @@ int64_t transfer_content_bytes(Data *const data) // , char const * const fstr) int64_t transfer_all_bytes(Data *const data) { - DEBUG_LOG("transfer_all_bytes"); - int64_t consumed = 0; + // nothing to transfer if there's no source. + if (nullptr == data->m_upstream.m_read.m_reader || !data->m_dnstream.m_write.isOpen()) { + return 0; + } + + int64_t consumed = 0; // input vio bytes visited + + TSIOBufferReader const reader = data->m_upstream.m_read.m_reader; + TSIOBuffer const output_buf = data->m_dnstream.m_write.m_iobuf; + + bool done = false; + + TSIOBufferBlock block = TSIOBufferReaderStart(reader); - if (data->m_dnstream.m_write.isOpen()) { - int64_t const read_avail = TSIOBufferReaderAvail(data->m_upstream.m_read.m_reader); + while (!done && nullptr != block) { + int64_t bavail = TSIOBufferBlockReadAvail(block, reader); - if (0 < read_avail) { - int64_t const copied(TSIOBufferCopy(data->m_dnstream.m_write.m_iobuf, data->m_upstream.m_read.m_reader, read_avail, 0)); + if (0 == bavail) { + block = TSIOBufferBlockNext(block); + } else { + int64_t const nbytes = TSIOBufferCopy(output_buf, reader, bavail, 0); + done = nbytes < bavail; // output buffer is full - if (0 < copied) { - TSIOBufferReaderConsume(data->m_upstream.m_read.m_reader, copied); - consumed = copied; + if (0 < nbytes) { + if (bavail == nbytes) { + block = TSIOBufferBlockNext(block); + } + TSIOBufferReaderConsume(reader, nbytes); + consumed += nbytes; } } } + if (0 < consumed) { + TSVIO const output_vio = data->m_dnstream.m_write.m_vio; + if (nullptr != output_vio) { + TSVIOReenable(output_vio); + } + + TSVIO const input_vio = data->m_upstream.m_read.m_vio; + if (nullptr != input_vio) { + TSVIONDoneSet(input_vio, TSVIONDoneGet(input_vio) + consumed); + } + } + return consumed; } diff --git a/plugins/experimental/slice/transfer.h b/plugins/experimental/slice/transfer.h index de3e1766d04..a63bb243a2d 100644 --- a/plugins/experimental/slice/transfer.h +++ b/plugins/experimental/slice/transfer.h @@ -32,3 +32,6 @@ int64_t transfer_content_bytes(Data *const data); // , char const * const fstr); // transfer all bytes from the server (error condition) int64_t transfer_all_bytes(Data *const data); + +// Signal the input about write state +void signal_input(TSVIO const input_vio, int64_t const consumed); diff --git a/plugins/experimental/slice/util.cc b/plugins/experimental/slice/util.cc new file mode 100644 index 00000000000..6f33559289f --- /dev/null +++ b/plugins/experimental/slice/util.cc @@ -0,0 +1,134 @@ +/** @file + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#include "util.h" + +#include "Config.h" +#include "Data.h" + +void +shutdown(TSCont const contp, Data *const data) +{ + DEBUG_LOG("shutting down transaction"); + TSContDataSet(contp, nullptr); + delete data; + TSContDestroy(contp); +} + +void +abort(TSCont const contp, Data *const data) +{ + DEBUG_LOG("aborting transaction"); + TSContDataSet(contp, nullptr); + data->m_upstream.abort(); + data->m_dnstream.abort(); + delete data; + TSContDestroy(contp); +} + +// create and issue a block request +bool +request_block(TSCont contp, Data *const data) +{ + // ensure no upstream connection + if (data->m_upstream.m_read.isOpen()) { + ERROR_LOG("Block request already in flight!"); + return false; + } + + if (Data::BlockState::Pending != data->m_blockstate) { + ERROR_LOG("request_block called with non Pending state!"); + return false; + } + + int64_t const blockbeg = (data->m_config->m_blockbytes * data->m_blocknum); + Range blockbe(blockbeg, blockbeg + data->m_config->m_blockbytes); + + char rangestr[1024]; + int rangelen = sizeof(rangestr); + bool const rpstat = blockbe.toStringClosed(rangestr, &rangelen); + TSAssert(rpstat); + + DEBUG_LOG("requestBlock: %s", rangestr); + + // reuse the incoming client header, just change the range + HttpHeader header(data->m_req_hdrmgr.m_buffer, data->m_req_hdrmgr.m_lochdr); + + // add/set sub range key and add slicer tag + bool const rangestat = header.setKeyVal(TS_MIME_FIELD_RANGE, TS_MIME_LEN_RANGE, rangestr, rangelen); + + if (!rangestat) { + ERROR_LOG("Error trying to set range request header %s", rangestr); + return false; + } + + // create virtual connection back into ATS + TSVConn const upvc = TSHttpConnectWithPluginId(reinterpret_cast(&data->m_client_ip), PLUGIN_NAME, 0); + + int const hlen = TSHttpHdrLengthGet(header.m_buffer, header.m_lochdr); + + // set up connection with the HttpConnect server + data->m_upstream.setupConnection(upvc); + data->m_upstream.setupVioWrite(contp, hlen); + + // Send full request + TSHttpHdrPrint(header.m_buffer, header.m_lochdr, data->m_upstream.m_write.m_iobuf); + TSVIOReenable(data->m_upstream.m_write.m_vio); + + /* + std::string const headerstr(header.toString()); + DEBUG_LOG("Headers\n%s", headerstr.c_str()); + */ + + // get ready for data back from the server + data->m_upstream.setupVioRead(contp, INT64_MAX); + + // anticipate the next server response header + TSHttpParserClear(data->m_http_parser); + data->m_resp_hdrmgr.resetHeader(); + + data->m_blockexpected = 0; + data->m_blockconsumed = 0; + data->m_blockstate = Data::BlockState::Active; + data->m_server_block_header_parsed = false; + + return true; +} + +bool +reader_avail_more_than(TSIOBufferReader const reader, int64_t bytes) +{ + TSIOBufferBlock block = TSIOBufferReaderStart(reader); + + if (nullptr == block) { + return false; + } + + while (nullptr != block) { + int64_t const blockbytes = TSIOBufferBlockReadAvail(block, reader); + if (bytes < blockbytes) { + return true; + } else { + bytes -= blockbytes; + } + + block = TSIOBufferBlockNext(block); + } + + return false; +} diff --git a/plugins/experimental/slice/util.h b/plugins/experimental/slice/util.h new file mode 100644 index 00000000000..9da6f368a3b --- /dev/null +++ b/plugins/experimental/slice/util.h @@ -0,0 +1,36 @@ +/** @file + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#pragma once + +#include "ts/ts.h" + +struct Data; + +/** Functions to deal with the connection to the client. + * Body content transfers are handled by the client. + * New block requests are also initiated by the client. + */ + +void shutdown(TSCont const contp, Data *const data); + +void abort(TSCont const contp, Data *const data); + +bool request_block(TSCont contp, Data *const data); + +bool reader_avail_more_than(TSIOBufferReader const reader, int64_t bytes); diff --git a/plugins/experimental/ssl_session_reuse/Makefile.inc b/plugins/experimental/ssl_session_reuse/Makefile.inc index 5e5a95607c2..8984463e7b5 100644 --- a/plugins/experimental/ssl_session_reuse/Makefile.inc +++ b/plugins/experimental/ssl_session_reuse/Makefile.inc @@ -18,17 +18,17 @@ pkglib_LTLIBRARIES += experimental/ssl_session_reuse/ssl_session_reuse.la experimental_ssl_session_reuse_ssl_session_reuse_la_SOURCES = \ - experimental/ssl_session_reuse/src/openssl_utils.cc \ - experimental/ssl_session_reuse/src/session_process.cc \ - experimental/ssl_session_reuse/src/ssl_init.cc \ - experimental/ssl_session_reuse/src/config.cc \ - experimental/ssl_session_reuse/src/redis_endpoint.cc \ - experimental/ssl_session_reuse/src/simple_pool.cc \ - experimental/ssl_session_reuse/src/ssl_key_utils.cc \ - experimental/ssl_session_reuse/src/connection.cc \ - experimental/ssl_session_reuse/src/publish.cc \ - experimental/ssl_session_reuse/src/subscriber.cc \ - experimental/ssl_session_reuse/src/ats_ssl_plugin.cc - -experimental_ssl_session_reuse_ssl_session_reuse_la_LIBADD = @LIB_HIREDIS@ + experimental/ssl_session_reuse/src/ats_ssl_plugin.cc \ + experimental/ssl_session_reuse/src/common.cc \ + experimental/ssl_session_reuse/src/config.cc \ + experimental/ssl_session_reuse/src/connection.cc \ + experimental/ssl_session_reuse/src/openssl_utils.cc \ + experimental/ssl_session_reuse/src/publish.cc \ + experimental/ssl_session_reuse/src/redis_endpoint.cc \ + experimental/ssl_session_reuse/src/session_process.cc \ + experimental/ssl_session_reuse/src/simple_pool.cc \ + experimental/ssl_session_reuse/src/ssl_init.cc \ + experimental/ssl_session_reuse/src/ssl_key_utils.cc \ + experimental/ssl_session_reuse/src/subscriber.cc +experimental_ssl_session_reuse_ssl_session_reuse_la_LIBADD = @LIB_HIREDIS@ diff --git a/plugins/experimental/ssl_session_reuse/example_config.config b/plugins/experimental/ssl_session_reuse/example_config.config index e74ab0292ea..55d96a50716 100644 --- a/plugins/experimental/ssl_session_reuse/example_config.config +++ b/plugins/experimental/ssl_session_reuse/example_config.config @@ -21,7 +21,7 @@ redis.RedisEndpoints=host1.com:6379,host2.com:6379 # in milliseconds redis.RedisConnectTimeout=20000 # in milliseconds -resis.RedisRetryDelay=5000000 +redis.RedisRetryDelay=5000000 ## end generic redis config parameters ## start pub config settings diff --git a/plugins/experimental/ssl_session_reuse/src/Config.h b/plugins/experimental/ssl_session_reuse/src/Config.h index 652a9f7dddf..e4ac1ffd454 100644 --- a/plugins/experimental/ssl_session_reuse/src/Config.h +++ b/plugins/experimental/ssl_session_reuse/src/Config.h @@ -31,7 +31,6 @@ struct fromstring { fromstring(const std::string &string) : _string(string) {} - template operator _T() const { _T t; diff --git a/plugins/experimental/ssl_session_reuse/src/ats_ssl_plugin.cc b/plugins/experimental/ssl_session_reuse/src/ats_ssl_plugin.cc index 81be94ae71b..64b03dfbdf3 100644 --- a/plugins/experimental/ssl_session_reuse/src/ats_ssl_plugin.cc +++ b/plugins/experimental/ssl_session_reuse/src/ats_ssl_plugin.cc @@ -23,16 +23,15 @@ */ #include +#include #include #include -#include +#include "common.h" #include "ssl_utils.h" PluginThreads plugin_threads; -int SSL_session_callback(TSCont contp, TSEvent event, void *edata); - static int shutdown_handler(TSCont contp, TSEvent event, void *edata) { @@ -49,7 +48,7 @@ TSPluginInit(int argc, const char *argv[]) info.plugin_name = (char *)("ats_session_reuse"); info.vendor_name = (char *)("ats"); - info.support_email = (char *)("ats-devel@oath.com"); + info.support_email = (char *)("ats-devel@verizonmedia.com"); TSLifecycleHookAdd(TS_LIFECYCLE_SHUTDOWN_HOOK, TSContCreate(shutdown_handler, nullptr)); @@ -63,10 +62,11 @@ TSPluginInit(int argc, const char *argv[]) } #endif if (argc < 2) { - TSError("Must specify config file"); + TSError("Must specify config file."); } else if (!init_ssl_params(argv[1])) { init_subscriber(); TSCont cont = TSContCreate(SSL_session_callback, nullptr); + TSDebug(PLUGIN, "TSPluginInit adding TS_SSL_SESSION_HOOK."); TSHttpHookAdd(TS_SSL_SESSION_HOOK, cont); } else { TSError("init_ssl_params failed."); diff --git a/plugins/experimental/ssl_session_reuse/src/common.cc b/plugins/experimental/ssl_session_reuse/src/common.cc new file mode 100644 index 00000000000..33b16dd0db3 --- /dev/null +++ b/plugins/experimental/ssl_session_reuse/src/common.cc @@ -0,0 +1,197 @@ +/** @file + + common.cc - Some common functions everyone needs + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + */ + +#include +#include +#include +#include +#include + +#include "common.h" + +const unsigned char salt[] = {115, 97, 108, 117, 0, 85, 137, 229}; +const unsigned char hex_chars[] = {'0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'A', 'B', 'C', 'D', 'E', 'F'}; + +std::string +hex_str(std::string const &str) +{ + size_t len = str.size() * 2 + 1; + char hex_str[len]; + for (unsigned long int i = 0; i < str.size(); ++i) { + unsigned char c = str.at(i); + hex_str[i * 2] = hex_chars[(c & 0xF0) >> 4]; + hex_str[i * 2 + 1] = hex_chars[(c & 0x0F)]; + } + hex_str[len - 1] = '\0'; + return std::string(hex_str, len); +} + +int +encrypt_encode64(const unsigned char *key, int key_length, const unsigned char *in_data, int in_data_len, char *out_data, + size_t out_data_size, size_t *out_data_len) +{ + if (!key || !in_data || !out_data || !out_data_len) { + return -1; + } + + int cipher_block_size = 0; + unsigned char *encrypted = nullptr; + int encrypted_len = 0; + int encrypted_len_extra = 0; + int ret = -1; + + // Initialize context + EVP_CIPHER_CTX *context = EVP_CIPHER_CTX_new(); + unsigned char iv[EVP_MAX_IV_LENGTH]; + unsigned char gen_key[EVP_MAX_KEY_LENGTH]; + + // generate key and iv + if (EVP_BytesToKey(EVP_aes_256_cbc(), EVP_md5(), salt, key, key_length, 1, gen_key, iv) <= 0) { + TSDebug(PLUGIN, "Error generating key."); + ret = -2; + goto Cleanup; + } + + // Set context AES128 with the generated key and iv + if (1 != EVP_EncryptInit_ex(context, EVP_aes_256_cbc(), nullptr, gen_key, iv)) { + TSDebug(PLUGIN, "EVP_EncryptInit_ex failed."); + ret = -3; + goto Cleanup; + } + + // https://www.openssl.org/docs/manmaster/man3/EVP_EncryptUpdate.html + // EVP_EncryptUpdate() needs (inl + cipher_block_size - 1) bytes. + // EVP_EncryptFinal_ex needs (inl + cipher_block_size) bytes. + cipher_block_size = EVP_CIPHER_CTX_block_size(context); + encrypted = new unsigned char[in_data_len + cipher_block_size * 2]; + if (1 != EVP_EncryptUpdate(context, encrypted, &encrypted_len, in_data, in_data_len)) { + TSDebug(PLUGIN, "EVP_EncryptUpdate failed."); + ret = -4; + goto Cleanup; + } + + if (1 != EVP_EncryptFinal_ex(context, encrypted + encrypted_len, &encrypted_len_extra)) { + TSDebug(PLUGIN, "EVP_EncryptFinal_ex failed."); + ret = -5; + goto Cleanup; + } + + // We must encode it to base64 here, since the encryption doesn't guarantee that there are no + // null bytes in the output. Which will cause a problem when sending it through redis since + // the redis command needs to be formatted to a C string. + if (TSBase64Encode(reinterpret_cast(encrypted), encrypted_len + encrypted_len_extra, out_data, out_data_size, + out_data_len) != 0) { + TSDebug(PLUGIN, "Base 64 encoding failed."); + ret = -6; + goto Cleanup; + } + + TSDebug(PLUGIN, "Encrypted buffer of size %d to buffer of size %lu.", in_data_len, *out_data_len); + ret = 0; + +Cleanup: + + if (encrypted) { + delete[] encrypted; + } + + if (context) { + EVP_CIPHER_CTX_free(context); + } + + return ret; +} + +int +decrypt_decode64(const unsigned char *key, int key_length, const char *in_data, int in_data_len, unsigned char *out_data, + size_t out_data_size, size_t *out_data_len) +{ + if (!key || !in_data || !out_data || !out_data_len) { + return -1; + } + + size_t decoded_size = DECODED_LEN(in_data_len); + size_t decoded_len = 0; + unsigned char *decoded = new unsigned char[decoded_size]; + int decrypted_len = 0; + int decrypted_len_extra = 0; + int ret = -1; + + // Initialize context + EVP_CIPHER_CTX *context = EVP_CIPHER_CTX_new(); + unsigned char iv[EVP_MAX_IV_LENGTH]; + unsigned char gen_key[EVP_MAX_KEY_LENGTH]; + + // Decode base64 + std::memset(decoded, 0, decoded_size); + if (TSBase64Decode(in_data, in_data_len, decoded, decoded_size, &decoded_len) != 0) { + TSDebug(PLUGIN, "Base 64 decoding failed."); + ret = -2; + goto Cleanup; + } + + // generate key and iv + if (EVP_BytesToKey(EVP_aes_256_cbc(), EVP_md5(), salt, key, key_length, 1, gen_key, iv) <= 0) { + TSDebug(PLUGIN, "Error generating key."); + ret = -3; + goto Cleanup; + } + // set context with the generated key and iv + if (1 != EVP_DecryptInit_ex(context, EVP_aes_256_cbc(), nullptr, gen_key, iv)) { + TSDebug(PLUGIN, "EVP_DecryptInit_ex failed."); + ret = -4; + goto Cleanup; + } + + // https://www.openssl.org/docs/manmaster/man3/EVP_DecryptUpdate.html + // EVP_DecryptUpdate() and EVP_DecryptFinal_ex() have the exact same requirements as their encrypt counterparts. + if (1 != EVP_DecryptUpdate(context, out_data, &decrypted_len, decoded, decoded_len)) { + TSDebug(PLUGIN, "EVP_DecryptUpdate failed."); + ret = -5; + goto Cleanup; + } + + if (1 != EVP_DecryptFinal_ex(context, out_data + decrypted_len, &decrypted_len_extra)) { + TSDebug(PLUGIN, "EVP_DecryptFinal_ex failed."); + ret = -6; + goto Cleanup; + } + + *out_data_len = decrypted_len + decrypted_len_extra; + + TSDebug(PLUGIN, "Decrypted buffer of size %d to buffer of size %lu.", in_data_len, *out_data_len); + ret = 0; + +Cleanup: + + if (decoded) { + delete[] decoded; + } + + if (context) { + EVP_CIPHER_CTX_free(context); + } + + return ret; +} diff --git a/plugins/experimental/ssl_session_reuse/src/common.h b/plugins/experimental/ssl_session_reuse/src/common.h index 86b158e8027..17ed4fafea7 100644 --- a/plugins/experimental/ssl_session_reuse/src/common.h +++ b/plugins/experimental/ssl_session_reuse/src/common.h @@ -23,6 +23,61 @@ */ #pragma once +#include +#include +#include +#include +#include +#include +#include + #define PLUGIN "ssl_session_reuse" +// Base 64 encoding takes 4*(ceil(n/3)) bytes +#define ENCODED_LEN(len) (((int)ceil(1.34 * (len) + 5)) + 1) +#define DECODED_LEN(len) (((int)ceil(0.75 * (len))) + 1) +// 3DES encryption will take at most 8 extra bytes. Plus we base 64 encode the result +#define ENCRYPT_LEN(len) ((int)ceil(1.34 * ((len) + 8) + 5) + 1) +#define DECRYPT_LEN(len) ((int)ceil(1.34 * ((len) + 8) + 5) + 1) + +class PluginThreads +{ +public: + bool shutdown = false; + + void + store(const pthread_t &th) + { + std::lock_guard lock(threads_mutex); + threads_queue.push_back(th); + } + + void + terminate() + { + shutdown = true; + + std::lock_guard lock(threads_mutex); + while (!threads_queue.empty()) { + pthread_t th = threads_queue.front(); + ::pthread_join(th, nullptr); + threads_queue.pop_front(); + } + } + +private: + std::deque threads_queue; + std::mutex threads_mutex; +}; + +std::string hex_str(std::string const &str); + +int encrypt_encode64(const unsigned char *key, int key_length, const unsigned char *in_data, int in_data_len, char *out_data, + size_t out_data_size, size_t *out_data_len); + +int decrypt_decode64(const unsigned char *key, int key_length, const char *in_data, int in_data_len, unsigned char *out_data, + size_t out_data_size, size_t *out_data_len); + extern const unsigned char salt[]; + +extern PluginThreads plugin_threads; diff --git a/plugins/experimental/ssl_session_reuse/src/config.cc b/plugins/experimental/ssl_session_reuse/src/config.cc index 7f3927150d5..6501048b2c3 100644 --- a/plugins/experimental/ssl_session_reuse/src/config.cc +++ b/plugins/experimental/ssl_session_reuse/src/config.cc @@ -22,17 +22,17 @@ */ -#include -#include #include #include #include #include #include -#include "tscpp/util/TextView.h" +#include +#include #include "Config.h" #include "common.h" +#include "tscpp/util/TextView.h" Config::Config() { @@ -101,9 +101,9 @@ Config::setLastConfigChange() memset(&s, 0, sizeof(s)); stat(m_filename.c_str(), &s); - m_lastmtime = s.st_mtim.tv_sec; + m_lastmtime = s.st_mtime; - if (s.st_mtim.tv_sec > oldLastmtime) { + if (s.st_mtime > oldLastmtime) { return true; } return false; diff --git a/plugins/experimental/ssl_session_reuse/src/message.h b/plugins/experimental/ssl_session_reuse/src/message.h index 41e94c052eb..1de48d1a1cb 100644 --- a/plugins/experimental/ssl_session_reuse/src/message.h +++ b/plugins/experimental/ssl_session_reuse/src/message.h @@ -25,6 +25,7 @@ #include #include + #include "redis_endpoint.h" typedef struct message { @@ -34,7 +35,7 @@ typedef struct message { std::set hosts_tried; message() {} + message(const struct message &m) : channel(m.channel), data(m.data), cleanup(m.cleanup), hosts_tried(m.hosts_tried) {} message(const std::string &c, const std::string &d, bool quit = false) : channel(c), data(d), cleanup(quit) {} virtual ~message() {} - } Message; diff --git a/plugins/experimental/ssl_session_reuse/src/openssl_utils.cc b/plugins/experimental/ssl_session_reuse/src/openssl_utils.cc index 2909e728591..919bb103f70 100644 --- a/plugins/experimental/ssl_session_reuse/src/openssl_utils.cc +++ b/plugins/experimental/ssl_session_reuse/src/openssl_utils.cc @@ -22,16 +22,15 @@ */ -#include -#include -#include -#include #include -#include #include #include -#include +#include +#include #include +#include +#include +#include #include #include #include @@ -48,28 +47,32 @@ ssl_new_session(TSSslSessionID &sid) std::string encoded_id; int ret = encode_id(sid.bytes, sid.len, encoded_id); if (ret < 0) { - TSError("encoded id failed."); + TSError("Encoded id failed."); return 0; } + std::string redis_channel = ssl_param.cluster_name + "." + encoded_id; int session_ret_len = SSL_SESSION_MAX_DER; char session_data[SSL_SESSION_MAX_DER]; - if (!TSSslSessionGetBuffer(&sid, session_data, &session_ret_len)) { - TSError("session data is too large. %d", session_ret_len); + const auto buffer_length = TSSslSessionGetBuffer(&sid, session_data, &session_ret_len); + if (buffer_length == 0) { + TSError("Failed to find a session buffer."); + return 0; + } else if (buffer_length > session_ret_len) { + TSError("Session data is too large. Its size is: %d but our max buffer size is: %d.", buffer_length, SSL_SESSION_MAX_DER); return 0; } std::string encrypted_data; ret = encrypt_session(session_data, session_ret_len, (unsigned char *)get_key_ptr(), get_key_length(), encrypted_data); if (ret < 0) { - TSError("encrypt_session failed."); + TSError("Encrypt_session failed."); return 0; } - std::string redis_channel = ssl_param.cluster_name + "." + encoded_id; ssl_param.pub->publish(redis_channel, encrypted_data); - TSDebug(PLUGIN, "create new session id: %s encoded: \"%s\" channel: %s", encoded_id.c_str(), encrypted_data.c_str(), + TSDebug(PLUGIN, "Create new session id: %s encoded: %s channel: %s", encoded_id.c_str(), encrypted_data.c_str(), redis_channel.c_str()); return 0; @@ -88,7 +91,7 @@ ssl_del_session(TSSslSessionID &sid) int ret = encode_id(sid.bytes, sid.len, encoded_id); if (!ret) { - TSDebug(PLUGIN, "session is deleted. id: \"%s\"", encoded_id.c_str()); + TSDebug(PLUGIN, "Session is deleted. id: %s", encoded_id.c_str()); } return 0; @@ -97,6 +100,7 @@ ssl_del_session(TSSslSessionID &sid) int SSL_session_callback(TSCont contp, TSEvent event, void *edata) { + TSDebug(PLUGIN, "SSL_session_callback event: %d", event); TSSslSessionID *sessionid = reinterpret_cast(edata); switch (event) { diff --git a/plugins/experimental/ssl_session_reuse/src/publish.cc b/plugins/experimental/ssl_session_reuse/src/publish.cc index 794260611e3..7c07a0ab192 100644 --- a/plugins/experimental/ssl_session_reuse/src/publish.cc +++ b/plugins/experimental/ssl_session_reuse/src/publish.cc @@ -22,23 +22,30 @@ */ -#include -#include #include #include #include #include +#include +#include #include + #include "common.h" #include "publisher.h" #include "Config.h" #include "redis_auth.h" #include "ssl_utils.h" +#include +#include + +std::mutex q_mutex; +std::condition_variable q_checker; +bool q_ready = false; void * RedisPublisher::start_worker_thread(void *arg) { - plugin_threads.store(::pthread_self()); + plugin_threads.store(pthread_self()); ::pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, nullptr); ::pthread_setcanceltype(PTHREAD_CANCEL_ASYNCHRONOUS, nullptr); @@ -49,14 +56,12 @@ RedisPublisher::start_worker_thread(void *arg) RedisPublisher::RedisPublisher(const std::string &conf) : m_redisEndpointsStr(cDefaultRedisEndpoint), - m_numWorkers(cPubNumWorkerThreads), m_redisConnectTimeout(cDefaultRedisConnectTimeout), m_redisConnectTries(cDefaultRedisConnectTries), m_redisPublishTries(cDefaultRedisPublishTries), m_redisRetryDelay(cDefaultRedisRetryDelay), m_maxQueuedMessages(cDefaultMaxQueuedMessages) - { if (Config::getSingleton().loadConfig(conf)) { Config::getSingleton().getValue("pubconfig", "PubNumWorkers", m_numWorkers); @@ -73,7 +78,7 @@ RedisPublisher::RedisPublisher(const std::string &conf) char redis_auth_key[MAX_REDIS_KEYSIZE]; if (!(get_redis_auth_key(redis_auth_key, MAX_REDIS_KEYSIZE))) { err = true; - TSError("RedisPublisher::RedisPublisher.Cannot get redis AUTH password."); + TSError("RedisPublisher::RedisPublisher: Cannot get redis AUTH password."); redis_passwd.clear(); } else { redis_passwd = redis_auth_key; @@ -82,29 +87,28 @@ RedisPublisher::RedisPublisher(const std::string &conf) addto_endpoint_vector(m_redisEndpoints, m_redisEndpointsStr); - TSDebug(PLUGIN, "RedisPublisher::RedisPublisher.NumWorkers= %d RedisConnectTimeout=%d", m_numWorkers, m_redisConnectTimeout); + TSDebug(PLUGIN, "RedisPublisher::RedisPublisher: NumWorkers: %d RedisConnectTimeout: %d", m_numWorkers, m_redisConnectTimeout); TSDebug(PLUGIN, - "RedisPublisher::RedisPublisher.RedisPublishTries= %d RedisConnectTries=%d RedisRetryDelay=%d MaxQueuedMessages=%d", + "RedisPublisher::RedisPublisher: RedisPublishTries: %d RedisConnectTries: %d RedisRetryDelay: %d MaxQueuedMessages: %d", m_redisPublishTries, m_redisConnectTries, m_redisRetryDelay, m_maxQueuedMessages); - TSDebug(PLUGIN, "RedisPublisher::RedisPublisher.Redis Publish endpoints are as follows:"); - for (std::vector::iterator it = m_redisEndpoints.begin(); it != m_redisEndpoints.end(); ++it) { - // LOGDEBUG(PUB, *it); - simple_pool *pool = simple_pool::create(it->m_hostname, it->m_port, m_poolRedisConnectTimeout); + TSDebug(PLUGIN, "RedisPublisher::RedisPublisher: Redis Publish endpoints are as follows:"); + for (auto &m_redisEndpoint : m_redisEndpoints) { + simple_pool *pool = simple_pool::create(m_redisEndpoint.m_hostname, m_redisEndpoint.m_port, m_poolRedisConnectTimeout); pools.push_back(pool); } - TSDebug(PLUGIN, "RedisPublisher::RedisPublisher.PoolRedisConnectTimeout= %d", m_poolRedisConnectTimeout); + TSDebug(PLUGIN, "RedisPublisher::RedisPublisher: PoolRedisConnectTimeout: %d", m_poolRedisConnectTimeout); ::sem_init(&m_workerSem, 0, 0); if (m_redisEndpoints.size() > m_numWorkers) { err = true; - TSError("RedisPublisher::RedisPublisher.Number of threads in the thread pool less than the number of redis endpoints"); + TSError("RedisPublisher::RedisPublisher: Number of threads in the thread pool less than the number of redis endpoints."); } if (!err) { - for (int i = 0; i < static_cast(m_numWorkers); i++) { + for (unsigned int i = 0; i < m_redisEndpoints.size(); i++) { TSThreadCreate(RedisPublisher::start_worker_thread, static_cast(this)); } } @@ -119,8 +123,11 @@ RedisPublisher::is_good() ::redisContext * RedisPublisher::setup_connection(const RedisEndpoint &re) { - ::pthread_t my_id = ::pthread_self(); - TSDebug(PLUGIN, "RedisPublisher::setup_connection.called by threadId: %d", static_cast(my_id)); + uint64_t my_id = 0; + if (TSIsDebugTagSet(PLUGIN)) { + my_id = static_cast(pthread_self()); + TSDebug(PLUGIN, "RedisPublisher::setup_connection: Called by threadId: %" PRIx64, my_id); + } RedisContextPtr my_context; struct ::timeval timeout; @@ -130,33 +137,33 @@ RedisPublisher::setup_connection(const RedisEndpoint &re) for (int i = 0; i < static_cast(m_redisConnectTries); ++i) { my_context.reset(::redisConnectWithTimeout(re.m_hostname.c_str(), re.m_port, timeout)); if (!my_context) { - TSError("RedisPublisher::setup_connection.connect to host: %s and port: %d fail count: %d threadId: %d", - re.m_hostname.c_str(), re.m_port, i + 1, static_cast(my_id)); + TSError("RedisPublisher::setup_connection: Connect to host: %s port: %d fail count: %d threadId: %" PRIx64, + re.m_hostname.c_str(), re.m_port, i + 1, my_id); } else if (my_context->err) { - TSError("RedisPublisher::setup_connection.connect to host: %s port: %d fail count: %d and threadId: %d", - re.m_hostname.c_str(), re.m_port, i + 1, static_cast(my_id)); + TSError("RedisPublisher::setup_connection: Connect to host: %s port: %d fail count: %d threadId: %" PRIx64, + re.m_hostname.c_str(), re.m_port, i + 1, my_id); my_context.reset(nullptr); } else { - TSDebug(PLUGIN, "RedisPublisher::setup_connection.threadId: %d successfully connected to the redis instance", - static_cast(my_id)); + TSDebug(PLUGIN, "RedisPublisher::setup_connection: threadId: %" PRIx64 " Successfully connected to the redis instance.", + my_id); redisReply *reply = static_cast(redisCommand(my_context.get(), "AUTH %s", redis_passwd.c_str())); if (reply == nullptr) { - TSError("RedisPublisher::setup_connection. Cannot AUTH redis server, no reply."); + TSError("RedisPublisher::setup_connection: Cannot AUTH redis server, no reply."); my_context.reset(nullptr); } else if (reply->type == REDIS_REPLY_ERROR) { - TSError("RedisPublisher::setup_connection. Cannot AUTH redis server, error reply."); + TSError("RedisPublisher::setup_connection: Cannot AUTH redis server, error reply."); freeReplyObject(reply); my_context.reset(nullptr); } else { - TSDebug(PLUGIN, "RedisPublisher::setup_connection. Successfully AUTH redis server."); + TSDebug(PLUGIN, "RedisPublisher::setup_connection: Successfully AUTH redis server."); freeReplyObject(reply); } break; } - TSError("RedisPublisher::setup_connection.connect failed.will wait for: %d microseconds and try again.", m_redisRetryDelay); + TSError("RedisPublisher::setup_connection: Connect failed, will wait for: %d microseconds and try again.", m_redisRetryDelay); ::usleep(m_redisRetryDelay); } @@ -166,8 +173,11 @@ RedisPublisher::setup_connection(const RedisEndpoint &re) ::redisReply * RedisPublisher::send_publish(RedisContextPtr &ctx, const RedisEndpoint &re, const Message &msg) { - ::pthread_t my_id = ::pthread_self(); - TSDebug(PLUGIN, "RedisPublisher::send_publish.called by threadId: %d", static_cast(my_id)); + uint64_t my_id = 0; + if (TSIsDebugTagSet(PLUGIN)) { + my_id = static_cast(pthread_self()); + TSDebug(PLUGIN, "RedisPublisher::send_publish: Called by threadId: %" PRIx64, my_id); + } ::redisReply *current_reply(nullptr); @@ -176,21 +186,21 @@ RedisPublisher::send_publish(RedisContextPtr &ctx, const RedisEndpoint &re, cons ctx.reset(setup_connection(re)); if (!ctx) { - TSError("RedisPublisher::send_publish.Unable to setup a connection to the redis server: %s:%d .ThreadId: %d Try: %d", - re.m_hostname.c_str(), re.m_port, static_cast(my_id), (i + 1)); + TSError("RedisPublisher::send_publish: Unable to setup a connection to the redis server: %s:%d threadId: %" PRIx64 + " try: %d", + re.m_hostname.c_str(), re.m_port, my_id, (i + 1)); continue; } } current_reply = static_cast(::redisCommand(ctx.get(), "PUBLISH %s %s", msg.channel.c_str(), msg.data.c_str())); if (!current_reply) { - TSError("RedisPublisher::send_publish.Unable to get a reply from the server for publish.ThreadId: %d Try: %d", - static_cast(my_id), (i + 1)); + TSError("RedisPublisher::send_publish: Unable to get a reply from the server for publish. threadId: %" PRIx64 " try: %d", + my_id, (i + 1)); ctx.reset(nullptr); // Clean up previous attempt } else if (REDIS_REPLY_ERROR == current_reply->type) { - TSError("RedisPublisher::send_publish.Server responded with error for publish.ThreadId: %d Try: %d", static_cast(my_id), - i + 1); + TSError("RedisPublisher::send_publish: Server responded with error for publish. threadId: %" PRIx64 " try: %d", my_id, i + 1); clear_reply(current_reply); current_reply = nullptr; ctx.reset(nullptr); // Clean up previous attempt @@ -223,57 +233,71 @@ RedisPublisher::runWorker() } m_endpointIndexMutex.unlock(); - ::pthread_t my_id = ::pthread_self(); RedisContextPtr my_context; ::redisReply *current_reply(nullptr); - while (true) { - ::sem_post(&m_workerSem); - int readyWorkers = 0; - ::sem_getvalue(&m_workerSem, &readyWorkers); - - // LOGDEBUG(PUB, "RedisPublisher::runWorker.Ready worker count: " << readyWorkers); + while (!plugin_threads.shutdown) { + try { + ::sem_post(&m_workerSem); + int readyWorkers = 0; + ::sem_getvalue(&m_workerSem, &readyWorkers); - m_messageQueueMutex.lock(); + // LOGDEBUG(PUB, "RedisPublisher::runWorker.Ready worker count: " << readyWorkers); - if (m_messageQueue.empty()) { - ::sem_wait(&m_workerSem); - m_messageQueueMutex.unlock(); - usleep(1000); - continue; - } + m_messageQueueMutex.lock(); - Message ¤t_message(m_messageQueue.front()); - if (!current_message.cleanup) { - m_messageQueue.pop_front(); - } + if (m_messageQueue.empty()) { + ::sem_wait(&m_workerSem); + m_messageQueueMutex.unlock(); + std::unique_lock lock(q_mutex); + q_checker.wait(lock, [] { return q_ready; }); + q_ready = false; + continue; + } - m_messageQueueMutex.unlock(); - ::sem_wait(&m_workerSem); + // Can't do reference here, since we pop it off the queue, the reference will be invalid. + Message current_message(m_messageQueue.front()); + if (!current_message.cleanup) { + m_messageQueue.pop_front(); + } - if (current_message.cleanup) { - TSDebug(PLUGIN, "RedisPublisher::runWorker.Thread id: %d received the cleanup message. exiting!", static_cast(my_id)); - break; - } - current_reply = send_publish(my_context, my_endpoint, current_message); + m_messageQueueMutex.unlock(); + ::sem_wait(&m_workerSem); - if (!current_reply) { - current_message.hosts_tried.insert(my_endpoint); - if (current_message.hosts_tried.size() < m_redisEndpoints.size()) { - // all endpoints are not tried - // someone else might be able to transmit - m_messageQueueMutex.lock(); - - if (!m_messageQueue.front().cleanup) { - m_messageQueue.push_front(current_message); + if (current_message.cleanup) { + if (TSIsDebugTagSet(PLUGIN)) { + auto my_id = static_cast(pthread_self()); + TSDebug(PLUGIN, "RedisPublisher::runWorker: threadId: %" PRIx64 " received the cleanup message. Exiting!", my_id); + } + break; + } + current_reply = send_publish(my_context, my_endpoint, current_message); + + if (!current_reply) { + current_message.hosts_tried.insert(my_endpoint); + if (current_message.hosts_tried.size() < m_redisEndpoints.size()) { + // all endpoints are not tried + // someone else might be able to transmit + m_messageQueueMutex.lock(); + if (!m_messageQueue.front().cleanup) { + m_messageQueue.push_front(current_message); + } + m_messageQueueMutex.unlock(); + + { + std::lock_guard lock(q_mutex); + q_ready = true; + } + q_checker.notify_one(); } - - m_messageQueueMutex.unlock(); } - } - clear_reply(current_reply); - current_reply = nullptr; + clear_reply(current_reply); + current_reply = nullptr; + } catch (...) { + TSDebug(PLUGIN, "RedisPublisher::runWorker exception"); + break; + } } my_context.reset(nullptr); clear_reply(current_reply); @@ -284,39 +308,49 @@ RedisPublisher::runWorker() int RedisPublisher::publish(const std::string &channel, const std::string &data) { - TSDebug(PLUGIN, "RedisPublisher::publish.Publish request for channel: %s and message: \"%s\" received", channel.c_str(), - data.c_str()); + TSDebug(PLUGIN, "RedisPublisher::publish: Publish request for channel: %s and message: \"%s\" received.", channel.c_str(), + hex_str(data).c_str()); m_messageQueueMutex.lock(); m_messageQueue.emplace_back(channel, data); - if (m_maxQueuedMessages < m_messageQueue.size()) { + if (m_messageQueue.size() > m_maxQueuedMessages) { m_messageQueue.pop_front(); } m_messageQueueMutex.unlock(); + { + std::lock_guard lock(q_mutex); + q_ready = true; + } + q_checker.notify_one(); + return SUCCESS; } int RedisPublisher::signal_cleanup() { - TSDebug(PLUGIN, "RedisPublisher::signal_cleanup.called"); + TSDebug(PLUGIN, "RedisPublisher::signal_cleanup: Called."); Message cleanup_message("", "", true); m_messageQueueMutex.lock(); - m_messageQueue.push_front(cleanup_message); // highest priority - m_messageQueueMutex.unlock(); + { + std::lock_guard lock(q_mutex); + q_ready = true; + } + q_checker.notify_one(); + return SUCCESS; } RedisPublisher::~RedisPublisher() { - TSDebug(PLUGIN, "RedisPublisher::~RedisPublisher.called"); + TSDebug(PLUGIN, "RedisPublisher::~RedisPublisher: Called."); RedisPublisher::signal_cleanup(); ::sem_destroy(&m_workerSem); @@ -325,18 +359,22 @@ RedisPublisher::~RedisPublisher() std::string RedisPublisher::get_session(const std::string &channel) { - TSDebug(PLUGIN, "RedisPublisher::get_session. Called by threadId: %d", static_cast(pthread_self())); + if (TSIsDebugTagSet(PLUGIN)) { + auto my_id = static_cast(pthread_self()); + TSDebug(PLUGIN, "RedisPublisher::get_session: Called by threadId: %" PRIx64, my_id); + } + std::string ret; uint32_t index = get_hash_index(channel); redisReply *reply = nullptr; - TSDebug(PLUGIN, "RedisPublisher::get_session. Start to try to get session."); + TSDebug(PLUGIN, "RedisPublisher::get_session: Start to try to get session."); for (uint32_t i = 0; i < m_redisEndpoints.size(); i++) { connection *conn = pools[index]->get(); if (conn) { reply = static_cast(redisCommand(conn->c_ptr(), "GET %s", channel.c_str())); if (reply && reply->type == REDIS_REPLY_STRING) { - TSDebug(PLUGIN, "RedisPublisher::get_session. Success to GET a value from redis server index: %d", index); + TSDebug(PLUGIN, "RedisPublisher::get_session: Success to GET a value from redis server index: %d", index); pools[index]->put(conn); ret = reply->str; clear_reply(reply); @@ -345,19 +383,23 @@ RedisPublisher::get_session(const std::string &channel) pools[index]->put(conn); clear_reply(reply); } - TSError("RedisPublisher::get_session. Fail to GET a value from this redis server index: %d", index); + TSError("RedisPublisher::get_session: Fail to GET a value from this redis server index: %d", index); index = get_next_index(index); - TSDebug(PLUGIN, "RedisPublisher::get_session. Will try the next redis server: %d", index); + TSDebug(PLUGIN, "RedisPublisher::get_session: Will try the next redis server: %d", index); } - TSError("RedisPublisher::get_session. Fail to GET a value from all redis servers!"); + TSError("RedisPublisher::get_session: Fail to GET a value from all redis servers!"); return ret; } redisReply * RedisPublisher::set_session(const Message &msg) { - TSDebug(PLUGIN, "RedisPublisher::set_session. Called by threadId: %d", static_cast(pthread_self())); + if (TSIsDebugTagSet(PLUGIN)) { + auto my_id = static_cast(pthread_self()); + TSDebug(PLUGIN, "RedisPublisher::set_session: Called by threadId: %" PRIx64, my_id); + } + uint32_t index = get_hash_index(msg.channel); redisReply *reply = nullptr; for (uint32_t i = 0; i < m_redisEndpoints.size(); i++) { @@ -366,7 +408,7 @@ RedisPublisher::set_session(const Message &msg) if (conn) { reply = static_cast(redisCommand(conn->c_ptr(), "SET %s %s", msg.channel.c_str(), msg.data.c_str())); if (reply && reply->type == REDIS_REPLY_STATUS && strcasecmp(reply->str, "OK") == 0) { - TSDebug(PLUGIN, "RedisPublisher::set_session. Success to SET a value to redis server: %s:%d", + TSDebug(PLUGIN, "RedisPublisher::set_session: Success to SET a value to redis server: %s:%d", m_redisEndpoints[index].m_hostname.c_str(), m_redisEndpoints[index].m_port); pools[index]->put(conn); return reply; @@ -374,14 +416,14 @@ RedisPublisher::set_session(const Message &msg) pools[index]->put(conn); clear_reply(reply); } - TSError("RedisPublisher::set_session. Fail to SET a value to this redis server %s:%d", + TSError("RedisPublisher::set_session: Fail to SET a value to this redis server %s:%d", m_redisEndpoints[index].m_hostname.c_str(), m_redisEndpoints[index].m_port); index = get_next_index(index); - TSDebug(PLUGIN, "RedisPublisher::set_session. Will try the next redis server: %s:%d", + TSDebug(PLUGIN, "RedisPublisher::set_session: Will try the next redis server: %s:%d", m_redisEndpoints[index].m_hostname.c_str(), m_redisEndpoints[index].m_port); } - TSError("RedisPublisher::set_session. Fail to SET a value to all redis servers!"); + TSError("RedisPublisher::set_session: Fail to SET a value to all redis servers!"); return nullptr; } diff --git a/plugins/experimental/ssl_session_reuse/src/publisher.h b/plugins/experimental/ssl_session_reuse/src/publisher.h index 82fec012efd..a76880c9cbb 100644 --- a/plugins/experimental/ssl_session_reuse/src/publisher.h +++ b/plugins/experimental/ssl_session_reuse/src/publisher.h @@ -30,6 +30,7 @@ #include #include #include + #include "message.h" #include "globals.h" #include "redis_endpoint.h" diff --git a/plugins/experimental/ssl_session_reuse/src/redis_endpoint.h b/plugins/experimental/ssl_session_reuse/src/redis_endpoint.h index 2762d9b80fc..55ff737745a 100644 --- a/plugins/experimental/ssl_session_reuse/src/redis_endpoint.h +++ b/plugins/experimental/ssl_session_reuse/src/redis_endpoint.h @@ -25,6 +25,7 @@ #include #include + #include "globals.h" typedef struct redis_endpoint { diff --git a/plugins/experimental/ssl_session_reuse/src/session_process.cc b/plugins/experimental/ssl_session_reuse/src/session_process.cc index 3f69e18f3ea..a40b33c1ee9 100644 --- a/plugins/experimental/ssl_session_reuse/src/session_process.cc +++ b/plugins/experimental/ssl_session_reuse/src/session_process.cc @@ -32,88 +32,45 @@ #include "session_process.h" #include "ssl_utils.h" - #include "common.h" -const unsigned char salt[] = {115, 97, 108, 117, 0, 85, 137, 229}; +const uint64_t protocol_version = 2; int encrypt_session(const char *session_data, int32_t session_data_len, const unsigned char *key, int key_length, std::string &encrypted_data) { - size_t len_all = 0; - size_t offset = 0; - char *pBuf = nullptr; - - int encrypted_buffer_size = 0; - int encrypted_msg_len = 0; - unsigned char *encrypted_msg = nullptr; - int ret = 0; - - offset = 0; - len_all = sizeof(int64_t) + sizeof(int32_t) + session_data_len; + if (!key || !session_data) { + return -1; + } - pBuf = new char[len_all]; + int data_len = sizeof(int64_t) + sizeof(int32_t) + session_data_len; + unsigned char *data = new unsigned char[data_len]; + int offset = 0; + size_t encrypted_size = ENCODED_LEN(data_len + EVP_MAX_BLOCK_LENGTH * 2); + size_t encrypted_len = 0; + char *encrypted = new char[encrypted_size]; + int ret = 0; - // Put in a fixed experation time of 7 hours, just to have communication consistency with the original - // protocol - int64_t expire_time = time(nullptr) + 2 * 3600; - memcpy(pBuf + offset, &expire_time, sizeof(expire_time)); + // Transition the expiration time into a protocol version field. + // Keeping it unnecessarily long at 64 bits to have consistency with previous version + // Version 1, had a fixed expiration time of 2*3600 seconds + std::memcpy(data + offset, &protocol_version, sizeof(int64_t)); offset += sizeof(int64_t); - - memcpy(pBuf + offset, &session_data_len, sizeof(int32_t)); + std::memcpy(data + offset, &session_data_len, sizeof(int32_t)); offset += sizeof(session_data_len); - memcpy(pBuf + offset, session_data, session_data_len); - - // Initialize context - EVP_CIPHER_CTX *context = EVP_CIPHER_CTX_new(); - unsigned char iv[EVP_MAX_IV_LENGTH]; - unsigned char gen_key[EVP_MAX_KEY_LENGTH]; - - // generate key and iv - int generated_key_len = EVP_BytesToKey(EVP_aes_256_cbc(), EVP_md5(), (unsigned char *)salt, key, key_length, 1, gen_key, iv); - if (generated_key_len <= 0) { - TSDebug(PLUGIN, "Error generating key"); - } - - // Set context AES128 with the generated key and iv - if (1 != EVP_EncryptInit_ex(context, EVP_aes_256_cbc(), nullptr, gen_key, iv)) { - TSDebug(PLUGIN, "Encryption of session data failed"); - ret = -1; - goto Cleanup; - } + std::memcpy(data + offset, session_data, session_data_len); - int elen; - encrypted_buffer_size = ENCRYPT_LEN(len_all); - encrypted_msg = new unsigned char[encrypted_buffer_size]; - if (1 != EVP_EncryptUpdate(context, encrypted_msg, &elen, reinterpret_cast(pBuf), len_all)) { - TSDebug(PLUGIN, "Encryption of session data failed"); - ret = -1; - goto Cleanup; - } - encrypted_msg_len = elen; - if (1 != EVP_EncryptFinal_ex(context, encrypted_msg + elen, &elen)) { - TSDebug(PLUGIN, "Encryption of session data failed"); - ret = -1; - goto Cleanup; + std::memset(encrypted, 0, encrypted_size); + ret = encrypt_encode64(key, key_length, data, data_len, encrypted, encrypted_size, &encrypted_len); + if (ret == 0) { + encrypted_data.assign(encrypted, encrypted_len); + } else { + TSDebug(PLUGIN, "encrypt_session calling encrypt_encode64 failed, error: %d", ret); } - encrypted_msg_len += elen; - - TSDebug(PLUGIN, "Encrypted buffer of size %d to buffer of size %d\n", session_data_len, encrypted_msg_len); - encrypted_data.assign(reinterpret_cast(encrypted_msg), encrypted_msg_len); - -Cleanup: - - if (pBuf) { - delete[] pBuf; - } - if (encrypted_msg) { - delete[] encrypted_msg; - } - if (context) { - EVP_CIPHER_CTX_free(context); - } + delete[] data; + delete[] encrypted; return ret; } @@ -128,85 +85,57 @@ int decrypt_session(const std::string &encrypted_data, const unsigned char *key, int key_length, char *session_data, int32_t &session_data_len) { - unsigned char *ssl_sess_ptr = nullptr; - int decrypted_buffer_size = 0; - int decrypted_msg_len = 0; - unsigned char *decrypted_msg = nullptr; - int ret = 0; - - // Initialize context - // Initialize context - EVP_CIPHER_CTX *context = EVP_CIPHER_CTX_new(); - unsigned char iv[EVP_MAX_IV_LENGTH]; - unsigned char gen_key[EVP_MAX_KEY_LENGTH]; - - // generate key and iv - int generated_key_len = EVP_BytesToKey(EVP_aes_256_cbc(), EVP_md5(), (unsigned char *)salt, key, key_length, 1, gen_key, iv); - - if (generated_key_len <= 0) { - TSDebug(PLUGIN, "Error generating key"); - } - // set context with the generated key and iv - if (1 != EVP_DecryptInit_ex(context, EVP_aes_256_cbc(), nullptr, gen_key, iv)) { - TSDebug(PLUGIN, "Decryption of encrypted session data failed"); - goto Cleanup; + if (!key || !session_data) { + return -1; } - decrypted_buffer_size = DECRYPT_LEN(encrypted_data.length()); - decrypted_msg = reinterpret_cast(new char[decrypted_buffer_size + 1]); - decrypted_msg_len = decrypted_buffer_size + 1; - if (decrypted_msg == nullptr) { - TSError("decrypted_msg allocate failure"); - } - if (1 != EVP_DecryptUpdate(context, decrypted_msg, &decrypted_msg_len, (unsigned char *)encrypted_data.c_str(), - encrypted_data.length())) { - TSDebug(PLUGIN, "Decryption of encrypted session data failed"); + unsigned char *ssl_sess_ptr = nullptr; + size_t decrypted_size = DECODED_LEN(encrypted_data.length()) + EVP_MAX_BLOCK_LENGTH * 2; + size_t decrypted_len = 0; + unsigned char *decrypted = new unsigned char[decrypted_size]; + int ret = -1; + size_t len_all = 0; + + std::memset(decrypted, 0, decrypted_size); + if ((ret = decrypt_decode64(key, key_length, encrypted_data.c_str(), encrypted_data.length(), decrypted, decrypted_size, + &decrypted_len)) != 0) { + TSDebug(PLUGIN, "decrypt_session calling decrypt_decode64 failed, error: %d", ret); goto Cleanup; } // Retrieve ssl_session - ssl_sess_ptr = decrypted_msg; - - // Skip the expiration time. Just a place holder to interact with the old version - ssl_sess_ptr += sizeof(int64_t); - - // Length - ret = *reinterpret_cast(ssl_sess_ptr); - ssl_sess_ptr += sizeof(int32_t); - TSDebug(PLUGIN, "Decrypted buffer of size %d from buffer of size %d\n", ret, session_data_len); - // If there is less data than the maxiumum buffer size, reduce accordingly - if (ret < session_data_len) { - session_data_len = ret; + ssl_sess_ptr = decrypted; + + // The first 64 bits are now the protocol version. Make sure it matches what we expect + if (protocol_version == *(reinterpret_cast(ssl_sess_ptr))) { + // Move beyond the protocol version + ssl_sess_ptr += sizeof(int64_t); + + // Length + ret = *reinterpret_cast(ssl_sess_ptr); + ssl_sess_ptr += sizeof(int32_t); + + len_all = ret + sizeof(int64_t) + sizeof(int32_t); + if (decrypted_len < len_all) { + TSDebug(PLUGIN, "Session data length mismatch, got %lu, should be %lu.", decrypted_len, len_all); + ret = -1; + goto Cleanup; + } + + // If there is less data than the maxiumum buffer size, reduce accordingly + if (ret < session_data_len) { + session_data_len = ret; + } + std::memcpy(session_data, ssl_sess_ptr, session_data_len); } - memcpy(session_data, ssl_sess_ptr, session_data_len); Cleanup: - if (decrypted_msg) { - delete[] decrypted_msg; - } - - if (context) { - EVP_CIPHER_CTX_free(context); - } + delete[] decrypted; return ret; } -int -decode_id(const std::string &encoded_id, char *decoded_data, int &decoded_data_len) -{ - size_t decode_len = 0; - memset(decoded_data, 0, decoded_data_len); - if (TSBase64Decode(static_cast(encoded_id.c_str()), encoded_id.length(), - reinterpret_cast(decoded_data), decoded_data_len, &decode_len) != 0) { - TSError("Base 64 decoding failed."); - return -1; - } - decoded_data_len = decode_len; - return 0; -} - int encode_id(const char *id, int idlen, std::string &encoded_data) { @@ -214,14 +143,15 @@ encode_id(const char *id, int idlen, std::string &encoded_data) memset(encoded, 0, ENCODED_LEN(idlen)); size_t encoded_len = 0; if (TSBase64Encode(id, idlen, encoded, ENCODED_LEN(idlen), &encoded_len) != 0) { - TSError("Base 64 encoding failed."); + TSError("ID base 64 encoding failed."); if (encoded) { delete[] encoded; } return -1; } - encoded_data.assign(encoded); + encoded_data.assign(encoded, encoded_len); + if (encoded) { delete[] encoded; } @@ -229,19 +159,37 @@ encode_id(const char *id, int idlen, std::string &encoded_data) return 0; } +int +decode_id(const std::string &encoded_id, char *decoded_data, int &decoded_data_len) +{ + size_t decode_len = 0; + memset(decoded_data, 0, decoded_data_len); + if (TSBase64Decode(static_cast(encoded_id.c_str()), encoded_id.length(), + reinterpret_cast(decoded_data), decoded_data_len, &decode_len) != 0) { + TSError("ID base 64 decoding failed."); + return -1; + } + decoded_data_len = decode_len; + return 0; +} + int add_session(char *session_id, int session_id_len, const std::string &encrypted_session) { + std::string session(session_id, session_id_len); + TSDebug(PLUGIN, "add_session session_id: %s", hex_str(session).c_str()); char session_data[SSL_SESSION_MAX_DER]; int32_t session_data_len = SSL_SESSION_MAX_DER; - if (decrypt_session(encrypted_session, (unsigned char *)get_key_ptr(), get_key_length(), session_data, session_data_len) < 0) { - TSError("Failed to decrypt session %.*s", session_id_len, session_id); - return -1; + int ret = decrypt_session(encrypted_session, (unsigned char *)get_key_ptr(), get_key_length(), session_data, session_data_len); + if (ret < 0) { + TSError("Failed to decrypt session %.*s, error: %d", session_id_len, hex_str(session).c_str(), ret); + return ret; } const unsigned char *loc = reinterpret_cast(session_data); SSL_SESSION *sess = d2i_SSL_SESSION(nullptr, &loc, session_data_len); if (nullptr == sess) { - TSError("Failed to transform session buffer %.*s", session_id_len, session_id); + TSError("Failed to transform session buffer %.*s", session_id_len, hex_str(session).c_str()); + return -1; } TSSslSessionID sid; memcpy(reinterpret_cast(sid.bytes), session_id, session_id_len); @@ -249,7 +197,7 @@ add_session(char *session_id, int session_id_len, const std::string &encrypted_s if (sid.len > sizeof(sid.bytes)) { sid.len = sizeof(sid.bytes); } - TSSslSessionInsert(&sid, reinterpret_cast(sess)); + TSSslSessionInsert(&sid, reinterpret_cast(sess), nullptr); // Free the sesison object created by d2i_SSL_SESSION // We should make an API that just takes the ASN buffer SSL_SESSION_free(sess); diff --git a/plugins/experimental/ssl_session_reuse/src/session_process.h b/plugins/experimental/ssl_session_reuse/src/session_process.h index a676db2b3ab..f987787f94d 100644 --- a/plugins/experimental/ssl_session_reuse/src/session_process.h +++ b/plugins/experimental/ssl_session_reuse/src/session_process.h @@ -26,14 +26,7 @@ #include #include -#define SSL_SESSION_MAX_DER 1024 * 10 - -// Base 64 encoding takes 4*(floor(n/3)) bytes -#define ENCODED_LEN(len) ((int)ceil(1.34 * len + 5)) + 1 -#define DECODED_LEN(len) ((int)ceil(0.75 * len)) + 1 -// 3DES encryption will take at most 8 extra bytes. Plus we base 64 encode the result -#define ENCRYPT_LEN(len) (int)ceil(1.34 * (len + 8) + 5) + 1 -#define DECRYPT_LEN(len) (int)ceil(1.34 * (len + 8) + 5) + 1 +#define SSL_SESSION_MAX_DER (1024 * 10) int encrypt_session(const char *session_data, int32_t session_data_len, const unsigned char *key, int key_length, std::string &encrypted_data); diff --git a/plugins/experimental/ssl_session_reuse/src/simple_pool.h b/plugins/experimental/ssl_session_reuse/src/simple_pool.h index 1931e8ac9c1..2977e356a38 100644 --- a/plugins/experimental/ssl_session_reuse/src/simple_pool.h +++ b/plugins/experimental/ssl_session_reuse/src/simple_pool.h @@ -23,10 +23,11 @@ */ #pragma once -#include "connection.h" #include #include +#include "connection.h" + /** * @brief Manages a pool of connections to a single Redis server */ diff --git a/plugins/experimental/ssl_session_reuse/src/ssl_init.cc b/plugins/experimental/ssl_session_reuse/src/ssl_init.cc index fd0c456a966..afc88fffeb0 100644 --- a/plugins/experimental/ssl_session_reuse/src/ssl_init.cc +++ b/plugins/experimental/ssl_session_reuse/src/ssl_init.cc @@ -24,13 +24,14 @@ #include #include -#include "ssl_utils.h" -#include "Config.h" -#include "common.h" #include #include #include +#include "ssl_utils.h" +#include "Config.h" +#include "common.h" + ssl_session_param ssl_param; // <- global containing all operational info std::string conf_file; @@ -60,26 +61,27 @@ init_ssl_params(const std::string &conf) if (ssl_param.key_update_interval > STEK_MAX_LIFETIME) { ssl_param.key_update_interval = STEK_MAX_LIFETIME; - TSDebug(PLUGIN, "KeyUpdateInterval too high, resetting session ticket key rotation to %d seconds", + TSDebug(PLUGIN, "KeyUpdateInterval too high, resetting session ticket key rotation to %d seconds.", ssl_param.key_update_interval); } - TSDebug(PLUGIN, "init_ssl_params: I %s been configured to initially be stek_master", + TSDebug(PLUGIN, "init_ssl_params: I %s been configured to initially be stek_master.", ((ssl_param.stek_master) ? "HAVE" : "HAVE NOT")); - TSDebug(PLUGIN, "init_ssl_params: Rotation interval (ssl_param.key_update_interval)set to %d\n", ssl_param.key_update_interval); + TSDebug(PLUGIN, "init_ssl_params: Rotation interval (ssl_param.key_update_interval) set to %d", ssl_param.key_update_interval); TSDebug(PLUGIN, "init_ssl_params: cluster_name set to %s", ssl_param.cluster_name.c_str()); - int ret = STEK_init_keys(); - if (ret < 0) { - TSError("init keys failure. %s", conf.c_str()); - return -1; - } - ssl_param.pub = new RedisPublisher(conf); if ((!ssl_param.pub) || (!ssl_param.pub->is_good())) { TSError("Construct RedisPublisher error."); return -1; } + + int ret = STEK_init_keys(); + if (ret < 0) { + TSError("STEK_init_keys failure: %s", conf.c_str()); + return -1; + } + return 0; } @@ -93,12 +95,13 @@ ssl_session_param::~ssl_session_param() } /* - Read the redis auth key from file ssl_param.redis_auth_key_file in retKeyBuff - + Read the redis auth key from file ssl_param.redis_auth_key_file in retKeyBuff + Return length of key read. */ int get_redis_auth_key(char *retKeyBuff, int buffSize) { + int retval = 0; // Get the Key if (ssl_param.redis_auth_key_file.length()) { int fd = open(ssl_param.redis_auth_key_file.c_str(), O_RDONLY); @@ -112,12 +115,13 @@ get_redis_auth_key(char *retKeyBuff, int buffSize) while (read_len > 1 && key_data[read_len - 1] == '\n') { --read_len; } + memset(retKeyBuff, 0, buffSize); strncpy(retKeyBuff, key_data.c_str(), read_len); + retval = key_data.length(); } } else { - TSError("can not get redis auth key"); - return 0; /* error */ + TSError("Can not get redis auth key."); } - return 1; /* ok */ + return retval; } diff --git a/plugins/experimental/ssl_session_reuse/src/ssl_key_utils.cc b/plugins/experimental/ssl_session_reuse/src/ssl_key_utils.cc index 7c53020450a..86359ca2fdd 100644 --- a/plugins/experimental/ssl_session_reuse/src/ssl_key_utils.cc +++ b/plugins/experimental/ssl_session_reuse/src/ssl_key_utils.cc @@ -22,15 +22,15 @@ */ -#include #include #include +#include +#include #include -#include #include +#include #include "ssl_utils.h" -#include #include "redis_auth.h" #include "stek.h" #include "common.h" @@ -48,14 +48,14 @@ static char channel_key[MAX_REDIS_KEYSIZE] = { }; static int channel_key_length = 0; -static int stek_master_setter_running = 0; /* stek master setter thread running */ +static std::atomic stek_master_setter_running(false); /* stek master setter thread running */ -static bool stek_initialized = false; +static std::atomic stek_initialized(false); bool isSTEKMaster() { - return stek_master_setter_running != 0; + return stek_master_setter_running; } /***************************************************************************** @@ -157,103 +157,63 @@ STEK_CreateNew(struct ssl_ticket_key_t *returnSTEK, int globalkey, int entropyEn } static int -STEK_encrypt(struct ssl_ticket_key_t *stek, const char *key, int key_length, char *retEncrypted, int *retLength) +STEK_encrypt(struct ssl_ticket_key_t *stek, const char *key, int key_length, char *ret_encrypted, int *ret_len) { - EVP_CIPHER_CTX *context = EVP_CIPHER_CTX_new(); - unsigned char iv[EVP_MAX_IV_LENGTH]; - unsigned char gen_key[EVP_MAX_KEY_LENGTH]; - int retval = 0; - - /* Encrypted stek will be placed in caller allocated retEncrypted buffer */ - /* NOTE: retLength must initially contain the size of the retEncrypted buffer */ + /* Encrypted stek will be placed in caller allocated ret_encrypted buffer */ + /* NOTE: ret_len must initially contain the size of the ret_encrypted buffer */ /* return 1 on success, 0 on failure */ - - int stek_len = sizeof(struct ssl_ticket_key_t); - int stek_enc_len = 0; - - // generate key and iv - int generated_key_len = EVP_BytesToKey(EVP_aes_256_cbc(), EVP_md5(), (const unsigned char *)salt, - reinterpret_cast(key), key_length, 1, gen_key, iv); - if (generated_key_len <= 0) { - TSDebug(PLUGIN, "Key setup for encryption of session ticket failed"); - goto cleanup; - } - - if (1 != EVP_EncryptInit_ex(context, EVP_aes_256_cbc(), nullptr, (const unsigned char *)gen_key, iv)) { - TSDebug(PLUGIN, "Encryption init of session ticket failed"); - goto cleanup; - } - - if (1 != EVP_EncryptUpdate(context, reinterpret_cast(retEncrypted), &stek_enc_len, - reinterpret_cast(stek), stek_len)) { - TSDebug(PLUGIN, "Encryption of session ticket failed"); - goto cleanup; - } - *retLength = stek_enc_len; - if (1 != EVP_EncryptFinal_ex(context, reinterpret_cast(retEncrypted) + stek_enc_len, &stek_enc_len)) { - TSDebug(PLUGIN, "Final encryption of session ticket failed"); - goto cleanup; + int stek_len = sizeof(struct ssl_ticket_key_t); + size_t encrypted_size = *ret_len; + size_t encrypted_len = 0; + int ret = -1; + + if ((ret = encrypt_encode64(reinterpret_cast(key), key_length, reinterpret_cast(stek), + stek_len, ret_encrypted, encrypted_size, &encrypted_len)) == 0) { + *ret_len = encrypted_len; + } else { + TSDebug(PLUGIN, "STEK_encrypt calling encrypt_encode64 failed, error: %d", ret); } - *retLength += stek_enc_len; - - retval = 1; -cleanup: - EVP_CIPHER_CTX_free(context); - return retval; // success + return ret; } static int -STEK_decrypt(const std::string &encrypted_data, const char *key, int key_length, struct ssl_ticket_key_t *retSTEK) +STEK_decrypt(const std::string &encrypted_data, const char *key, int key_length, struct ssl_ticket_key_t *ret_STEK) { - if (!retSTEK) { - return 0; + if (!ret_STEK) { + return -1; } - EVP_CIPHER_CTX *context = EVP_CIPHER_CTX_new(); - unsigned char iv[EVP_MAX_IV_LENGTH]; - unsigned char gen_key[EVP_MAX_KEY_LENGTH]; - unsigned char decryptBuff[4 * sizeof(struct ssl_ticket_key_t)] = { - 0, - }; - int decryptLength = sizeof(decryptBuff); - int retval = 0; + TSDebug(PLUGIN, "STEK_decrypt: requested to decrypt %lu bytes", encrypted_data.length()); - TSDebug(PLUGIN, "STEK_decrypt(): requested to decrypt %d bytes", static_cast(encrypted_data.length())); + int ret = -1; + size_t decrypted_size = DECODED_LEN(encrypted_data.length()) + EVP_MAX_BLOCK_LENGTH * 2; + size_t decrypted_len = 0; + unsigned char *decrypted = new unsigned char[decrypted_size]; - // generate key and iv - int generated_key_len = EVP_BytesToKey(EVP_aes_256_cbc(), EVP_md5(), (const unsigned char *)salt, - reinterpret_cast(key), key_length, 1, gen_key, iv); - if (generated_key_len <= 0) { - TSDebug(PLUGIN, "Key setup for decryption of session ticket failed"); - goto cleanup; + std::memset(decrypted, 0, decrypted_size); + if ((ret = decrypt_decode64(reinterpret_cast(key), key_length, encrypted_data.c_str(), + encrypted_data.length(), decrypted, decrypted_size, &decrypted_len)) != 0) { + TSDebug(PLUGIN, "STEK_decrypt calling decrypt_decode64 failed, error: %d", ret); + goto Cleanup; } - if (1 != EVP_DecryptInit_ex(context, EVP_aes_256_cbc(), nullptr, (const unsigned char *)gen_key, iv)) { - TSDebug(PLUGIN, "Encryption of session data failed"); - goto cleanup; + if (sizeof(struct ssl_ticket_key_t) != decrypted_len) { + TSError("STEK data length mismatch, got %lu, should be %lu", decrypted_len, sizeof(struct ssl_ticket_key_t)); + ret = -1; + goto Cleanup; } - if (1 != - EVP_DecryptUpdate(context, decryptBuff, &decryptLength, (unsigned char *)encrypted_data.c_str(), encrypted_data.length())) { - TSDebug(PLUGIN, "Decryption of encrypted ticket key failed"); - goto cleanup; - } + memcpy(ret_STEK, decrypted, sizeof(struct ssl_ticket_key_t)); + memset(decrypted, 0, decrypted_size); // warm fuzzies + ret = 0; - if (sizeof(struct ssl_ticket_key_t) != decryptLength) { - TSError("STEK received is unexpected size length=%d not %d", decryptLength, static_cast(sizeof(struct ssl_ticket_key_t))); - goto cleanup; - } +Cleanup: - memcpy(retSTEK, decryptBuff, sizeof(struct ssl_ticket_key_t)); - memset(decryptBuff, 0, sizeof(decryptBuff)); // warm fuzzies - retval = 1; - -cleanup: - if (context) { - EVP_CIPHER_CTX_free(context); + if (decrypted) { + delete[] decrypted; } - return retval; /* ok, length of data in retSTEK will be sizeof(struct ssl_ticket_key_t) */ + return ret; /* ok, length of data in ret_STEK will be sizeof(struct ssl_ticket_key_t) */ } int @@ -271,8 +231,8 @@ STEK_Send_To_Network(struct ssl_ticket_key_t *stekToSend) // encrypt the STEK before sending encryptedDataLength = sizeof(encryptedData); - if (!STEK_encrypt(stekToSend, get_key_ptr(), get_key_length(), encryptedData, &encryptedDataLength)) { - TSError("Can't encrypt STEK. Not sending"); + if (STEK_encrypt(stekToSend, get_key_ptr(), get_key_length(), encryptedData, &encryptedDataLength) != 0) { + TSError("STEK_encrypt failed, not sending."); return 0; // failure } @@ -306,46 +266,50 @@ STEK_Update_Setter_Thread(void *arg) return nullptr; } - stek_master_setter_running = 1; - TSDebug(PLUGIN, "Will now act as the STEK rotator for pod"); - - while (true) { - // Create new STEK, set it for me, and then send it to the POD. - if ((!STEK_CreateNew(&newKey, 0, 1 /* entropy ensured */)) || (!STEK_Send_To_Network(&newKey))) { - // Error occurred. We will retry after a short interval. - // perhaps publishig isn't up yet. - startProblem++; // count how many times we have this problem. - sleepInterval = 60; // short sleep for error retry - TSError("Could not create/send new STEK for key rotation...try again in %d seconds", sleepInterval); - } else { - // Everything good. will sleep for normal rotation time period and then repeat - startProblem = 0; - TSDebug(PLUGIN, "New POD STEK created and sent to network."); - - sleepInterval = ssl_param.key_update_interval; - } + stek_master_setter_running = true; + TSDebug(PLUGIN, "Will now act as the STEK rotator for POD."); + + while (!plugin_threads.shutdown) { + try { + // Create new STEK, set it for me, and then send it to the POD. + if ((!STEK_CreateNew(&newKey, 0, 1 /* entropy ensured */)) || (!STEK_Send_To_Network(&newKey))) { + // Error occurred. We will retry after a short interval. + // perhaps publishig isn't up yet. + startProblem++; // count how many times we have this problem. + sleepInterval = 60; // short sleep for error retry + TSError("Could not create/send new STEK for key rotation... Try again in %d seconds.", sleepInterval); + } else { + // Everything good. will sleep for normal rotation time period and then repeat + startProblem = 0; + TSDebug(PLUGIN, "New POD STEK created and sent to network."); + + sleepInterval = ssl_param.key_update_interval; + } - ::sleep(sleepInterval); + ::sleep(sleepInterval); - if ((!startProblem) && (memcmp(&newKey, &ssl_param.ticket_keys[0], sizeof(struct ssl_ticket_key_t)))) { - /* I am not using the key I set before sleeping. This means node (and POD) has - * sync'd onto a more recent master, which we will now yield to by exiting - * out of this thread. */ - goto done_master_setter; - } + if ((!startProblem) && (memcmp(&newKey, &ssl_param.ticket_keys[0], sizeof(struct ssl_ticket_key_t)))) { + /* I am not using the key I set before sleeping. This means node (and POD) has + * sync'd onto a more recent master, which we will now yield to by exiting + * out of this thread. */ + goto done_master_setter; + } - if (startProblem > 60) { - /* We've been trying every minute for more than an hour. Time to give up and move on..*/ - /* Another node in pod will notice and pick up responsibility, else we'll try again later */ + if (startProblem > 60) { + /* We've been trying every minute for more than an hour. Time to give up and move on..*/ + /* Another node in pod will notice and pick up responsibility, else we'll try again later */ + goto done_master_setter; + } + } catch (...) { + TSDebug(PLUGIN, "STEK_Update_Setter_Thread exception"); goto done_master_setter; } - } // while(forever) done_master_setter: - TSDebug(PLUGIN, "Yielding STEK-Master rotation responsibility to another node in POD"); + TSDebug(PLUGIN, "Yielding STEK-Master rotation responsibility to another node in POD."); memset(&newKey, 0, sizeof(struct ssl_ticket_key_t)); - stek_master_setter_running = 0; + stek_master_setter_running = false; return nullptr; } @@ -355,8 +319,8 @@ void STEK_update(const std::string &encrypted_stek) { struct ssl_ticket_key_t newSTEK; - if (STEK_decrypt(encrypted_stek, get_key_ptr(), get_key_length(), &newSTEK)) { - if (memcmp(&newSTEK, &ssl_param.ticket_keys[0], sizeof(struct ssl_ticket_key_t))) { + if (STEK_decrypt(encrypted_stek, get_key_ptr(), get_key_length(), &newSTEK) == 0) { + if (memcmp(&newSTEK, &ssl_param.ticket_keys[0], sizeof(struct ssl_ticket_key_t)) != 0) { /* ... and it's a new one, so we will now set and use it. */ ssl_key_lock.lock(); memcpy(&ssl_param.ticket_keys[1], &ssl_param.ticket_keys[0], sizeof(struct ssl_ticket_key_t)); @@ -387,53 +351,68 @@ STEK_Update_Checker_Thread(void *arg) * that something is up with our STEK master, and nominate a new STEK master. */ - TSDebug(PLUGIN, "Starting Update Checker Thread"); + TSDebug(PLUGIN, "Starting STEK_Update_Checker_Thread."); lastChangeTime = lastWarningTime = time(¤tTime); // init to current to supress a startup warning. + int check_count = 0; // Keep track of how many times we've checked whether we got a new STEK. + + while (!plugin_threads.shutdown) { + try { + if (!stek_initialized && ssl_param.pub) { + // Launch a request for the master to resend you the ticket key + std::string redis_channel = ssl_param.cluster_name + "." + STEK_ID_RESEND; + ssl_param.pub->publish(redis_channel, ""); // send it + TSDebug(PLUGIN, "Request for ticket."); + } + time(¤tTime); + time_t sleepUntil; + if (stek_initialized) { + // Sleep until we are overdue for a key update + sleepUntil = 2 * STEK_MAX_LIFETIME - (currentTime - lastChangeTime); + stek_initialized = false; + check_count = 0; + } else { + // Wait for a while in hopes that the server gets back to us + sleepUntil = 30; + ++check_count; + } + ::sleep(sleepUntil); - while (true) { - if (!stek_initialized && ssl_param.pub) { - // Launch a request for the master to resend you the ticket key - std::string redis_channel = ssl_param.cluster_name + "." + STEK_ID_RESEND; - ssl_param.pub->publish(redis_channel, ""); // send it - TSDebug(PLUGIN, "Request for ticket"); - } - time(¤tTime); - time_t sleepUntil; - if (stek_initialized) { - // Sleep until we are overdue for a key update - sleepUntil = 2 * STEK_MAX_LIFETIME - (currentTime - lastChangeTime); - } else { - // Wait for a while in hopes that the server gets back to us - sleepUntil = 30; - } - ::sleep(sleepUntil); - - /* We track last time STEK changed. If we haven't gotten a new STEK in twice the max, - * then we figure something is wrong with the POD STEK master and nominate a new master. - * STEK master may have been misconfigured, disconnected, died or who-knows. - * ...no problem we will will recover POD STEK rotation now */ - - time(¤tTime); - if ((currentTime - lastChangeTime) > (2 * STEK_MAX_LIFETIME)) { - // Yes we were way past due for a new STEK, and haven't received it. - if ((currentTime - lastWarningTime) > STEK_NOT_CHANGED_WARNING_INTERVAL) { - // Yes it's time to put another warning in log file. - TSError("Session Ticket Encryption Key not syncd in past %d hours.", - static_cast(((currentTime - lastChangeTime) / 3600))); - - lastWarningTime = currentTime; + if (check_count == 0) { + continue; } - /* Time to nominate a new stek master for pod key rotation... */ - if (!stek_master_setter_running) { - TSDebug(PLUGIN, "Will nominate a new STEK-master thread now for pod key rotation"); - TSThreadCreate(STEK_Update_Setter_Thread, nullptr); + /* We track last time STEK changed. If we haven't gotten a new STEK in twice the max, + * then we figure something is wrong with the POD STEK master and nominate a new master. + * STEK master may have been misconfigured, disconnected, died or who-knows. + * If we're have been checking in the past five minutes and still haven't got a new + * STEK, we believe that the master has died, so "now, I am the master". + * ...no problem we will recover POD STEK rotation now */ + + time(¤tTime); + if ((currentTime - lastChangeTime) > (2 * STEK_MAX_LIFETIME) || check_count > 10) { + // Yes we were way past due for a new STEK, and haven't received it. + if ((currentTime - lastWarningTime) > STEK_NOT_CHANGED_WARNING_INTERVAL) { + // Yes it's time to put another warning in log file. + TSError("Session Ticket Encryption Key not syncd in past %d hours.", + static_cast(((currentTime - lastChangeTime) / 3600))); + + lastWarningTime = currentTime; + } + + /* Time to nominate a new stek master for pod key rotation... */ + if (!stek_master_setter_running) { + TSDebug(PLUGIN, "Will nominate a new STEK-master thread now for pod key rotation."); + TSThreadCreate(STEK_Update_Setter_Thread, nullptr); + } } + } catch (...) { + TSDebug(PLUGIN, "STEK_Update_Checker_Thread exception"); + break; } - } // while(forever) + return nullptr; } // STEK_Update_Checker_Thread() int @@ -441,15 +420,16 @@ STEK_init_keys() { ssl_ticket_key_t initKey; - if (!get_redis_auth_key(channel_key, MAX_REDIS_KEYSIZE)) { - TSError("STEK_init_keys: could not get redis authentication key"); + channel_key_length = get_redis_auth_key(channel_key, MAX_REDIS_KEYSIZE); + if (channel_key_length <= 0) { + TSError("STEK_init_keys: Could not get redis authentication key."); return -1; } // Initialize starter Session Ticket Encryption Key // Will sync up with master later if (!STEK_CreateNew(&initKey, 0, 0 /*fast start*/)) { - TSError("Cant init STEK"); + TSError("Cant init STEK."); return -1; } memcpy(&ssl_param.ticket_keys[0], &initKey, sizeof(struct ssl_ticket_key_t)); diff --git a/plugins/experimental/ssl_session_reuse/src/ssl_utils.h b/plugins/experimental/ssl_session_reuse/src/ssl_utils.h index 262728cb758..adfc39c80d7 100644 --- a/plugins/experimental/ssl_session_reuse/src/ssl_utils.h +++ b/plugins/experimental/ssl_session_reuse/src/ssl_utils.h @@ -23,15 +23,15 @@ */ #pragma once -#include #include +#include +#include +#include +#include #include -#include -#include #include "publisher.h" #include "subscriber.h" - #include "stek.h" struct ssl_session_param { @@ -47,35 +47,6 @@ struct ssl_session_param { ~ssl_session_param(); }; -class PluginThreads -{ -public: - void - store(const pthread_t &th) - { - std::lock_guard lock(threads_mutex); - threads_queue.push_back(th); - } - - void - terminate() - { - std::lock_guard lock(threads_mutex); - for (pthread_t th : threads_queue) { - ::pthread_cancel(th); - } - while (!threads_queue.empty()) { - pthread_t th = threads_queue.front(); - ::pthread_join(th, nullptr); - threads_queue.pop_front(); - } - } - -private: - std::deque threads_queue; - std::mutex threads_mutex; -}; - int STEK_init_keys(); const char *get_key_ptr(); @@ -97,5 +68,3 @@ int init_subscriber(); int SSL_session_callback(TSCont contp, TSEvent event, void *edata); extern ssl_session_param ssl_param; // almost everything one needs is stored in here - -extern PluginThreads plugin_threads; diff --git a/plugins/experimental/ssl_session_reuse/src/stek.h b/plugins/experimental/ssl_session_reuse/src/stek.h index 76492cefc2f..f1f9725bb1b 100644 --- a/plugins/experimental/ssl_session_reuse/src/stek.h +++ b/plugins/experimental/ssl_session_reuse/src/stek.h @@ -25,8 +25,8 @@ /* STEK - Session Ticket Encryption Key stuff */ -#define STEK_ID_NAME "stek" // ACTUALLY it is redis channel minus cluster_name prefix, aka mdbm keyname -#define STEK_ID_RESEND "resendstek" +#define STEK_ID_NAME "@stek@" // ACTUALLY it is redis channel minus cluster_name prefix, aka mdbm keyname +#define STEK_ID_RESEND "@resendstek@" #define STEK_MAX_LIFETIME 86400 // 24 hours max - should rotate STEK #define STEK_NOT_CHANGED_WARNING_INTERVAL (2 * STEK_MAX_LIFETIME) // warn on non-stek rotate every X secs. diff --git a/plugins/experimental/ssl_session_reuse/src/subscriber.cc b/plugins/experimental/ssl_session_reuse/src/subscriber.cc index 7dfc5560350..c21cf0fa85d 100644 --- a/plugins/experimental/ssl_session_reuse/src/subscriber.cc +++ b/plugins/experimental/ssl_session_reuse/src/subscriber.cc @@ -22,13 +22,13 @@ */ +#include #include -#include #include #include -#include -#include "common.h" +#include +#include "common.h" #include "subscriber.h" #include "Config.h" #include "redis_auth.h" @@ -67,7 +67,7 @@ RedisSubscriber::RedisSubscriber(const std::string &conf) char redis_auth_key[MAX_REDIS_KEYSIZE]; if (!(get_redis_auth_key(redis_auth_key, MAX_REDIS_KEYSIZE))) { err = true; - TSError("RedisPublisher::RedisPublisher.Cannot get redis AUTH password."); + TSError("RedisPublisher::RedisPublisher: Cannot get redis AUTH password."); redis_passwd.clear(); } else { redis_passwd = redis_auth_key; @@ -76,7 +76,7 @@ RedisSubscriber::RedisSubscriber(const std::string &conf) m_channel_prefix = m_channel.substr(0, m_channel.find('*')); - TSDebug(PLUGIN, "RedisSubscriber::RedisSubscriber.SubscriberChannel= %s SubscriberChannelPrefix= %s", m_channel.c_str(), + TSDebug(PLUGIN, "RedisSubscriber::RedisSubscriber: SubscriberChannel: %s SubscriberChannelPrefix: %s", m_channel.c_str(), m_channel_prefix.c_str()); addto_endpoint_vector(m_redisEndpoints, redisEndpointsStr); @@ -96,14 +96,13 @@ RedisSubscriber::is_good() int RedisSubscriber::get_endpoint_index() { - int retval = m_redisEndpointsIndex++; - return retval; + return m_redisEndpointsIndex++; } ::redisContext * RedisSubscriber::setup_connection(int index) { - TSDebug(PLUGIN, "RedisSubscriber::setup_connection.called for host= %s and port= %d", m_redisEndpoints[index].m_hostname.c_str(), + TSDebug(PLUGIN, "RedisSubscriber::setup_connection: Called for host: %s port: %d", m_redisEndpoints[index].m_hostname.c_str(), m_redisEndpoints[index].m_port); ::redisContext *my_context(nullptr); @@ -115,31 +114,31 @@ RedisSubscriber::setup_connection(int index) my_context = ::redisConnectWithTimeout(m_redisEndpoints[index].m_hostname.c_str(), m_redisEndpoints[index].m_port, timeout_connect); if (!my_context) { - TSError("RedisSubscriber::setup_connection.connect to host: %s and port: %d failed", - m_redisEndpoints[index].m_hostname.c_str(), m_redisEndpoints[index].m_port); + TSError("RedisSubscriber::setup_connection: Connect to host: %s port: %d failed.", m_redisEndpoints[index].m_hostname.c_str(), + m_redisEndpoints[index].m_port); } else if (my_context->err) { - TSError("RedisSubscriber::setup_connection.connect to host: %s and port: %d failed.", - m_redisEndpoints[index].m_hostname.c_str(), m_redisEndpoints[index].m_port); + TSError("RedisSubscriber::setup_connection: Connect to host: %s port: %d failed.", m_redisEndpoints[index].m_hostname.c_str(), + m_redisEndpoints[index].m_port); } else { - TSDebug(PLUGIN, "RedisSubscriber::setup_connection: successfully connected to the redis host: %s on port: %d", + TSDebug(PLUGIN, "RedisSubscriber::setup_connection: Successfully connected to the redis host: %s port: %d", m_redisEndpoints[index].m_hostname.c_str(), m_redisEndpoints[index].m_port); redisReply *reply = static_cast(redisCommand(my_context, "AUTH %s", redis_passwd.c_str())); if (reply == nullptr) { - TSError("RedisSubscriber::setup_connection Cannot AUTH redis server, no reply."); + TSError("RedisSubscriber::setup_connection: Cannot AUTH redis server, no reply."); } else if (reply->type == REDIS_REPLY_ERROR) { - TSError("RedisSubscriber::setup_connection Cannot AUTH redis server, error reply."); + TSError("RedisSubscriber::setup_connection: Cannot AUTH redis server, error reply."); freeReplyObject(reply); } else { - TSDebug(PLUGIN, "RedisSubscriber::setup_connection. Successfully AUTH redis server."); + TSDebug(PLUGIN, "RedisSubscriber::setup_connection: Successfully AUTH redis server."); freeReplyObject(reply); } break; } - TSError("RedisSubscriber::setup_connection.will wait for: %d microseconds and try again.", m_redisRetryDelay); + TSError("RedisSubscriber::setup_connection: Will wait for: %d microseconds and try again.", m_redisRetryDelay); ::usleep(m_redisRetryDelay); } @@ -150,71 +149,77 @@ RedisSubscriber::setup_connection(int index) void RedisSubscriber::run() { - TSDebug(PLUGIN, "RedisSubscriber::runMaster.called"); + TSDebug(PLUGIN, "RedisSubscriber::run: Called."); int my_endpoint_index = get_endpoint_index(); ::redisContext *my_context(setup_connection(my_endpoint_index)); ::redisReply *current_reply(nullptr); - while (true) { - while ((!my_context) || (my_context->err)) { - ::usleep(m_redisRetryDelay); - my_context = setup_connection(my_endpoint_index); - } + while (!plugin_threads.shutdown) { + try { + while ((!my_context) || (my_context->err)) { + ::usleep(m_redisRetryDelay); + my_context = setup_connection(my_endpoint_index); + } - TSDebug(PLUGIN, "RedisSubscriber::runMaster.Issuing command: \"PSUBSCRIBE %s\"", m_channel.c_str()); - current_reply = static_cast(::redisCommand(my_context, "PSUBSCRIBE %s", m_channel.c_str())); + TSDebug(PLUGIN, "RedisSubscriber::run: Issuing command: PSUBSCRIBE %s", m_channel.c_str()); + current_reply = static_cast(::redisCommand(my_context, "PSUBSCRIBE %s", m_channel.c_str())); - if (!current_reply || (REDIS_REPLY_ERROR == current_reply->type)) { - TSError("RedisSubscriber::runMaster.subscribe to redis server on channel: %s failed.", m_channel.c_str()); - ::usleep(1000 * 1000); - continue; - } else { - TSDebug(PLUGIN, "RedisSubscriber::runMaster.Successfully subscribed to channel: \"%s\"", m_channel.c_str()); - TSDebug(PLUGIN, "RedisSubscriber::runMaster.Waiting for messages to appear on the channel!"); - ::freeReplyObject(current_reply); - } - - // Blocking read - while (REDIS_OK == ::redisGetReply(my_context, reinterpret_cast(¤t_reply))) { - // Process Message - std::string channel(current_reply->element[2]->str, current_reply->element[2]->len); - std::string data(current_reply->element[3]->str, current_reply->element[3]->len); - TSDebug(PLUGIN, "Redis request channel=%s message=%s", channel.c_str(), data.c_str()); - - std::string key = ""; - // Strip the channel name to get the key - if (channel.compare(0, m_channel_prefix.length(), m_channel_prefix) == 0) { - key = channel.substr(m_channel_prefix.length()); + if (!current_reply || (REDIS_REPLY_ERROR == current_reply->type)) { + TSError("RedisSubscriber::run: Subscribe to redis server on channel: %s failed.", m_channel.c_str()); + ::usleep(1000 * 1000); + continue; + } else { + TSDebug(PLUGIN, "RedisSubscriber::run: Successfully subscribed to channel: %s", m_channel.c_str()); + TSDebug(PLUGIN, "RedisSubscriber::run: Waiting for messages to appear on the channel!"); + ::freeReplyObject(current_reply); } - // If this is new keys, go do the ticket key updates - if (strncmp(key.c_str(), STEK_ID_NAME, strlen(STEK_ID_NAME)) == 0) { - STEK_update(data); - // Requesting last ticket to be resent - } else if (strncmp(key.c_str(), STEK_ID_RESEND, strlen(STEK_ID_RESEND)) == 0) { - if (isSTEKMaster()) { - TSDebug(PLUGIN, "RedisSubscriber Resend ticket"); - STEK_Send_To_Network(ssl_param.ticket_keys); + // Blocking read + while (!plugin_threads.shutdown && REDIS_OK == ::redisGetReply(my_context, reinterpret_cast(¤t_reply))) { + // Process Message + std::string channel(current_reply->element[2]->str, current_reply->element[2]->len); + std::string data(current_reply->element[3]->str, current_reply->element[3]->len); + TSDebug(PLUGIN, "RedisSubscriber::run: Redis request channel: %s message: %s", channel.c_str(), hex_str(data).c_str()); + + std::string key = ""; + // Strip the channel name to get the key + if (channel.compare(0, m_channel_prefix.length(), m_channel_prefix) == 0) { + key = channel.substr(m_channel_prefix.length()); } - } else { // Otherwise this is a new session. Let ATS core know about it - char session_id[SSL_MAX_SSL_SESSION_ID_LENGTH * 2]; - int session_id_len = sizeof(session_id); - if (decode_id(key, session_id, session_id_len) == 0) { // Decrypt the data - TSDebug(PLUGIN, "Add session encoded_id=%s decoded_id=%.*s %d", key.c_str(), session_id_len, session_id, session_id_len); - add_session(session_id, session_id_len, data); - } else { - TSDebug(PLUGIN, "failed to decode key=%s", key.c_str()); + + // If this is new keys, go do the ticket key updates + if (strncmp(key.c_str(), STEK_ID_NAME, strlen(STEK_ID_NAME)) == 0) { + STEK_update(data); + // Requesting last ticket to be resent + } else if (strncmp(key.c_str(), STEK_ID_RESEND, strlen(STEK_ID_RESEND)) == 0) { + if (isSTEKMaster()) { + TSDebug(PLUGIN, "RedisSubscriber::run: Resend ticket."); + STEK_Send_To_Network(ssl_param.ticket_keys); + } + } else { // Otherwise this is a new session. Let ATS core know about it + char session_id[SSL_MAX_SSL_SESSION_ID_LENGTH * 2]; + int session_id_len = sizeof(session_id); + if (decode_id(key, session_id, session_id_len) == 0) { // Decrypt the data + TSDebug(PLUGIN, "RedisSubscriber::run: Add session encoded_id: %s decoded_id: %.*s %d", key.c_str(), session_id_len, + hex_str(std::string(session_id, session_id_len)).c_str(), session_id_len); + add_session(session_id, session_id_len, data); + } else { + TSDebug(PLUGIN, "RedisSubscriber::run: Failed to decode key: %s", key.c_str()); + } } - } - ::freeReplyObject(current_reply); + ::freeReplyObject(current_reply); - TSDebug(PLUGIN, "RedisSubscriber::runMaster.Got message: %s on channel: %s", data.c_str(), channel.c_str()); + TSDebug(PLUGIN, "RedisSubscriber::run: Got message: %s channel: %s", hex_str(data).c_str(), channel.c_str()); + } + } catch (...) { + TSDebug(PLUGIN, "RedisSubscriber::run exception"); + break; } } } RedisSubscriber::~RedisSubscriber() { - TSDebug(PLUGIN, "RedisSubscriber::~RedisSubscriber.called for endpoint"); + TSDebug(PLUGIN, "RedisSubscriber::~RedisSubscriber: Called for endpoint."); } diff --git a/plugins/experimental/ssl_session_reuse/src/subscriber.h b/plugins/experimental/ssl_session_reuse/src/subscriber.h index 19c18127b6c..4c5bb2e6db6 100644 --- a/plugins/experimental/ssl_session_reuse/src/subscriber.h +++ b/plugins/experimental/ssl_session_reuse/src/subscriber.h @@ -25,6 +25,8 @@ #include #include +#include + #include "message.h" #include "globals.h" #include "redis_endpoint.h" @@ -35,7 +37,7 @@ class RedisSubscriber std::string redis_passwd; std::vector m_redisEndpoints; - int m_redisEndpointsIndex; + std::atomic m_redisEndpointsIndex; std::string m_channel; std::string m_channel_prefix; diff --git a/plugins/experimental/ssl_session_reuse/tests/plug-load.test.py b/plugins/experimental/ssl_session_reuse/tests/plug-load.test.py index cd3801e6f51..10a6468f711 100644 --- a/plugins/experimental/ssl_session_reuse/tests/plug-load.test.py +++ b/plugins/experimental/ssl_session_reuse/tests/plug-load.test.py @@ -20,45 +20,45 @@ ''' pluginName = 'ats_ssl_plugin' -path=os.path.abspath(".") -configFile='./packages/rhel.6.5.package/conf/trafficserver/ats_ssl_session_reuse.xml' +path = os.path.abspath(".") +configFile = './packages/rhel.6.5.package/conf/trafficserver/ats_ssl_session_reuse.xml' -Test.ContinueOnFail=True +Test.ContinueOnFail = True # Define default ATS -ts=Test.MakeATSProcess("ts") -server=Test.MakeOriginServer("server") +ts = Test.MakeATSProcess("ts") +server = Test.MakeOriginServer("server") Test.testName = "" -request_header={"headers": "GET / HTTP/1.1\r\nHost: www.example.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} -#expected response from the origin server -response_header={"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +request_header = {"headers": "GET / HTTP/1.1\r\nHost: www.example.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +# expected response from the origin server +response_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": ""} -#add response to the server dictionary +# add response to the server dictionary server.addResponse("sessionfile.log", request_header, response_header) ts.Disk.records_config.update({ - 'proxy.config.diags.debug.enabled': 1, - 'proxy.config.diags.debug.tags': '{0}'.format(pluginName), - }) + 'proxy.config.diags.debug.enabled': 1, + 'proxy.config.diags.debug.tags': '{0}'.format(pluginName), +}) ts.Disk.plugin_config.AddLine( - '# {1}/{0}.so {2}'.format(pluginName,path,configFile) + '# {1}/{0}.so {2}'.format(pluginName, path, configFile) ) ts.Disk.remap_config.AddLine( 'map http://www.example.com http://127.0.0.1:{0}'.format(server.Variables.Port) ) -goldFile = os.path.join(Test.RunDirectory,"{0}.gold".format(pluginName)) -with open(goldFile,'w+') as jf: - jf.write("``loading plugin ``{0}.so``".format(pluginName)) +goldFile = os.path.join(Test.RunDirectory, "{0}.gold".format(pluginName)) +with open(goldFile, 'w+') as jf: + jf.write("``loading plugin ``{0}.so``".format(pluginName)) # call localhost straight -tr=Test.AddTestRun() -tr.Processes.Default.Command='curl --proxy 127.0.0.1:{0} "http://www.example.com" --verbose'.format(ts.Variables.port) -tr.Processes.Default.ReturnCode=0 +tr = Test.AddTestRun() +tr.Processes.Default.Command = 'curl --proxy 127.0.0.1:{0} "http://www.example.com" --verbose'.format(ts.Variables.port) +tr.Processes.Default.ReturnCode = 0 # time delay as proxy.config.http.wait_for_cache could be broken -tr.Processes.Default.StartBefore(server,ready=When.PortOpen(server.Variables.Port)) +tr.Processes.Default.StartBefore(server, ready=When.PortOpen(server.Variables.Port)) tr.Processes.Default.StartBefore(Test.Processes.ts) -tr.StillRunningAfter=server +tr.StillRunningAfter = server -#ts.Streams.All=goldFile -#ts.Disk.diags_log.Content+=goldFile +# ts.Streams.All=goldFile +# ts.Disk.diags_log.Content+=goldFile diff --git a/plugins/experimental/sslheaders/sslheaders.cc b/plugins/experimental/sslheaders/sslheaders.cc index 0c4dd1fbcfd..92b42992340 100644 --- a/plugins/experimental/sslheaders/sslheaders.cc +++ b/plugins/experimental/sslheaders/sslheaders.cc @@ -36,7 +36,7 @@ SslHdrExpandRequestHook(TSCont cont, TSEvent event, void *edata) txn = static_cast(edata); hdr = static_cast(TSContDataGet(cont)); TSVConn vconn = TSHttpSsnClientVConnGet(TSHttpTxnSsnGet(txn)); - TSSslConnection ssl = TSVConnSSLConnectionGet(vconn); + TSSslConnection ssl = TSVConnSslConnectionGet(vconn); switch (event) { case TS_EVENT_HTTP_READ_REQUEST_HDR: diff --git a/plugins/experimental/sslheaders/sslheaders.h b/plugins/experimental/sslheaders/sslheaders.h index 571ea573b17..c3134aae9cf 100644 --- a/plugins/experimental/sslheaders/sslheaders.h +++ b/plugins/experimental/sslheaders/sslheaders.h @@ -16,6 +16,8 @@ * limitations under the License. */ +#pragma once + #include #include #include diff --git a/plugins/experimental/statichit/Makefile.inc b/plugins/experimental/statichit/Makefile.inc new file mode 100644 index 00000000000..b45e5e0d3b8 --- /dev/null +++ b/plugins/experimental/statichit/Makefile.inc @@ -0,0 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +pkglib_LTLIBRARIES += experimental/statichit/statichit.la + +experimental_statichit_statichit_la_SOURCES = \ + experimental/statichit/statichit.cc diff --git a/plugins/experimental/statichit/README.md b/plugins/experimental/statichit/README.md new file mode 100644 index 00000000000..cb7001d525b --- /dev/null +++ b/plugins/experimental/statichit/README.md @@ -0,0 +1,38 @@ +This is a simple plugin to serve static content from a local filesystem. It shares some of the same functionality as the `healthchecks` plugin, but can be used in the remap context(thereby making it reloadable). It does not use fsnotify for watching the source files. + +If the file specified by the `--file-path` parameter is not found, the plugin will return the status code specified +by the `--failure-code` parameter, defaulting to a 404. If the file is found, it will return the contents of the file +setting the `Content-Type` header to the value specified on the `--mime-type` parameter. + +Metrics +-------- + +The plugin publishes the following metrics: + +statichit.response_bytes: + The total number of bytes emitted +statichit.response_count: + The number of HTTP responses generated by the plugin + +Examples: +----------- + +remap.config: +``` +map /internal/healthcheck \ + http://127.0.0.1 \ + @plugin=statichit.so @pparam=--file-path=/var/run/trafficserver/healthcheck.txt \ + @pparam=--mime-type=text/plain \ + @pparam=--success-code=200 \ + @pparam=--failure-code=403 \ + @pparam=--max-age=0 + +map http://content.example.com/content.txt \ + http://127.0.0.1 \ + @plugin=statichit.so @pparam=--file-path=/opt/ats/etc/trafficserver/static/content_source.txt \ + @pparam=--failure-code=404 \ + @pparam=--max-age=604800 + +``` + +NOTE: The remap origin is never contacted because this plugin intercepts the request and acts as the origin server. For that reason, the origin specification must be syntactically valid and resolvable in DNS. diff --git a/plugins/experimental/statichit/statichit.cc b/plugins/experimental/statichit/statichit.cc new file mode 100644 index 00000000000..15203508faa --- /dev/null +++ b/plugins/experimental/statichit/statichit.cc @@ -0,0 +1,637 @@ +/** @file + + Static Hit Content Serving + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#include +#include + +#include +#include +#include +#include + +#include +#include + +#include +#include + +#include +#include + +#include "ts/ts.h" +#include "ts/remap.h" + +constexpr char PLUGIN[] = "statichit"; + +#define VDEBUG(fmt, ...) TSDebug(PLUGIN, fmt, ##__VA_ARGS__) + +#if DEBUG +#define VERROR(fmt, ...) TSDebug(PLUGIN, fmt, ##__VA_ARGS__) +#else +#define VERROR(fmt, ...) TSError("[%s] %s: " fmt, PLUGIN, __FUNCTION__, ##__VA_ARGS__) +#endif + +#define VIODEBUG(vio, fmt, ...) \ + VDEBUG("vio=%p vio.cont=%p, vio.cont.data=%p, vio.vc=%p " fmt, (vio), TSVIOContGet(vio), TSContDataGet(TSVIOContGet(vio)), \ + TSVIOVConnGet(vio), ##__VA_ARGS__) + +static TSCont TxnHook; + +static int StatCountBytes = -1; +static int StatCountResponses = -1; + +static int StaticHitInterceptHook(TSCont contp, TSEvent event, void *edata); +static int StaticHitTxnHook(TSCont contp, TSEvent event, void *edata); + +struct StaticHitConfig { + explicit StaticHitConfig(const std::string &filePath, const std::string &mimeType) : filePath(filePath), mimeType(mimeType) {} + + ~StaticHitConfig() {} + + std::string filePath; + std::string mimeType; + + int successCode = 200; + int failureCode = 404; + int maxAge = 0; +}; + +struct StaticHitRequest; + +union argument_type { + void *ptr; + intptr_t ecode; + TSVConn vc; + TSVIO vio; + TSHttpTxn txn; + StaticHitRequest *trq; + + argument_type(void *_p) : ptr(_p) {} +}; + +// This structure represents the state of a streaming I/O request. It +// is directional (ie. either a read or a write). We need two of these +// for each TSVConn; one to push data into the TSVConn and one to pull +// data out. +struct IOChannel { + TSVIO vio = nullptr; + TSIOBuffer iobuf; + TSIOBufferReader reader; + + IOChannel() : iobuf(TSIOBufferSizedCreate(TS_IOBUFFER_SIZE_INDEX_32K)), reader(TSIOBufferReaderAlloc(iobuf)) {} + ~IOChannel() + { + if (this->reader) { + TSIOBufferReaderFree(this->reader); + } + + if (this->iobuf) { + TSIOBufferDestroy(this->iobuf); + } + } + + void + read(TSVConn vc, TSCont contp) + { + this->vio = TSVConnRead(vc, contp, this->iobuf, INT64_MAX); + } + + void + write(TSVConn vc, TSCont contp) + { + this->vio = TSVConnWrite(vc, contp, this->reader, INT64_MAX); + } +}; + +struct StaticHitHttpHeader { + TSMBuffer buffer; + TSMLoc header; + TSHttpParser parser; + + StaticHitHttpHeader() + { + this->buffer = TSMBufferCreate(); + this->header = TSHttpHdrCreate(this->buffer); + this->parser = TSHttpParserCreate(); + } + + ~StaticHitHttpHeader() + { + if (this->parser) { + TSHttpParserDestroy(this->parser); + } + + TSHttpHdrDestroy(this->buffer, this->header); + TSHandleMLocRelease(this->buffer, TS_NULL_MLOC, this->header); + TSMBufferDestroy(this->buffer); + } +}; + +struct StaticHitRequest { + StaticHitRequest() {} + + off_t nbytes = 0; // Number of bytes to generate. + unsigned maxAge = 0; // Max age for cache responses. + unsigned statusCode = 200; + IOChannel readio; + IOChannel writeio; + StaticHitHttpHeader rqheader; + + std::string body; + std::string mimeType; + + static StaticHitRequest * + createStaticHitRequest(StaticHitConfig *tc) + { + StaticHitRequest *shr = new StaticHitRequest; + std::ifstream ifstr; + + ifstr.open(tc->filePath); + if (!ifstr) { + shr->statusCode = tc->failureCode; + return shr; + } + + std::stringstream sstr; + sstr << ifstr.rdbuf(); + shr->body = sstr.str(); + shr->nbytes = shr->body.size(); + shr->mimeType = tc->mimeType; + shr->statusCode = tc->successCode; + shr->maxAge = tc->maxAge; + + return shr; + } + + ~StaticHitRequest() = default; +}; + +// Destroy a StaticHitRequest, including the per-txn continuation. +static void +StaticHitRequestDestroy(StaticHitRequest *trq, TSVIO vio, TSCont contp) +{ + if (vio) { + TSVConnClose(TSVIOVConnGet(vio)); + } + + TSContDestroy(contp); + delete trq; +} + +// NOTE: This will always append a new "field_name: value" +static void +HeaderFieldDateSet(const StaticHitHttpHeader &http, const char *field_name, int64_t field_len, time_t value) +{ + TSMLoc field; + + TSMimeHdrFieldCreateNamed(http.buffer, http.header, field_name, field_len, &field); + TSMimeHdrFieldValueDateSet(http.buffer, http.header, field, value); + TSMimeHdrFieldAppend(http.buffer, http.header, field); + TSHandleMLocRelease(http.buffer, http.header, field); +} + +// NOTE: This will always append a new "field_name: value" +static void +HeaderFieldIntSet(const StaticHitHttpHeader &http, const char *field_name, int64_t field_len, int64_t value) +{ + TSMLoc field; + + TSMimeHdrFieldCreateNamed(http.buffer, http.header, field_name, field_len, &field); + TSMimeHdrFieldValueInt64Set(http.buffer, http.header, field, -1, value); + TSMimeHdrFieldAppend(http.buffer, http.header, field); + TSHandleMLocRelease(http.buffer, http.header, field); +} + +// NOTE: This will always append a new "field_name: value" +static void +HeaderFieldStringSet(const StaticHitHttpHeader &http, const char *field_name, int64_t field_len, const char *value) +{ + TSMLoc field; + + TSMimeHdrFieldCreateNamed(http.buffer, http.header, field_name, field_len, &field); + TSMimeHdrFieldValueStringSet(http.buffer, http.header, field, -1, value, -1); + TSMimeHdrFieldAppend(http.buffer, http.header, field); + TSHandleMLocRelease(http.buffer, http.header, field); +} + +static void +WriteResponseHeader(StaticHitRequest *trq, TSCont contp, TSHttpStatus status) +{ + StaticHitHttpHeader response; + + VDEBUG("writing response header"); + + TSReleaseAssert(TSHttpHdrTypeSet(response.buffer, response.header, TS_HTTP_TYPE_RESPONSE) == TS_SUCCESS); + TSReleaseAssert(TSHttpHdrVersionSet(response.buffer, response.header, TS_HTTP_VERSION(1, 1)) == TS_SUCCESS); + TSReleaseAssert(TSHttpHdrStatusSet(response.buffer, response.header, status) == TS_SUCCESS); + + TSHttpHdrReasonSet(response.buffer, response.header, TSHttpHdrReasonLookup(status), -1); + + if (status == TS_HTTP_STATUS_OK) { + // Set the Content-Length header. + HeaderFieldIntSet(response, TS_MIME_FIELD_CONTENT_LENGTH, TS_MIME_LEN_CONTENT_LENGTH, trq->nbytes); + + // Set the Cache-Control header. + if (trq->maxAge > 0) { + char buf[64]; + + snprintf(buf, sizeof(buf), "max-age=%u", trq->maxAge); + HeaderFieldStringSet(response, TS_MIME_FIELD_CACHE_CONTROL, TS_MIME_LEN_CACHE_CONTROL, buf); + HeaderFieldDateSet(response, TS_MIME_FIELD_LAST_MODIFIED, TS_MIME_LEN_LAST_MODIFIED, time(nullptr)); + + } else { + HeaderFieldStringSet(response, TS_MIME_FIELD_CACHE_CONTROL, TS_MIME_LEN_CACHE_CONTROL, "no-cache"); + } + + HeaderFieldStringSet(response, TS_MIME_FIELD_CONTENT_TYPE, TS_MIME_LEN_CONTENT_TYPE, trq->mimeType.c_str()); + } + + // Write the header to the IO buffer. Set the VIO bytes so that we can get a WRITE_COMPLETE + // event when this is done. + int hdrlen = TSHttpHdrLengthGet(response.buffer, response.header); + + TSHttpHdrPrint(response.buffer, response.header, trq->writeio.iobuf); + TSVIONBytesSet(trq->writeio.vio, hdrlen); + TSVIOReenable(trq->writeio.vio); + + TSStatIntIncrement(StatCountBytes, hdrlen); +} + +static bool +StaticHitParseRequest(StaticHitRequest *trq) +{ + const char *path; + int pathsz; + + // Make sure this is a GET request + path = TSHttpHdrMethodGet(trq->rqheader.buffer, trq->rqheader.header, &pathsz); + if (path != TS_HTTP_METHOD_GET) { + VDEBUG("%.*s method is not supported", pathsz, path); + return false; + } + + return true; +} + +// Handle events from TSHttpTxnServerIntercept. The intercept +// starts with TS_EVENT_NET_ACCEPT, and then continues with +// TSVConn events. +static int +StaticHitInterceptHook(TSCont contp, TSEvent event, void *edata) +{ + VDEBUG("StaticHitInterceptHook: %p ", edata); + + argument_type arg(edata); + + VDEBUG("contp=%p, event=%s (%d), edata=%p", contp, TSHttpEventNameLookup(event), event, arg.ptr); + + switch (event) { + case TS_EVENT_NET_ACCEPT: { + // TS_EVENT_NET_ACCEPT will be delivered when the server intercept + // is set up by the core. We just need to allocate a statichit + // request state and start reading the VC. + StaticHitRequest *trq = static_cast(TSContDataGet(contp)); + + TSStatIntIncrement(StatCountResponses, 1); + VDEBUG("allocated server intercept statichit trq=%p", trq); + + // This continuation was allocated in StaticHitTxnHook. Reset the + // data to keep track of this generator request. + TSContDataSet(contp, trq); + + // Start reading the request from the server intercept VC. + trq->readio.read(arg.vc, contp); + VIODEBUG(trq->readio.vio, "started reading statichit request"); + + return TS_EVENT_NONE; + } + + case TS_EVENT_NET_ACCEPT_FAILED: { + // TS_EVENT_NET_ACCEPT_FAILED will be delivered if the + // transaction is cancelled before we start tunnelling + // through the server intercept. One way that this can happen + // is if the intercept is attached early, and then we serve + // the document out of cache. + + // There's nothing to do here except nuke the continuation + // that was allocated in StaticHitTxnHook(). + + StaticHitRequest *trq = static_cast(TSContDataGet(contp)); + delete trq; + + TSContDestroy(contp); + return TS_EVENT_NONE; + } + + case TS_EVENT_VCONN_READ_READY: { + argument_type cdata = TSContDataGet(contp); + StaticHitHttpHeader &rqheader = cdata.trq->rqheader; + + VDEBUG("reading vio=%p vc=%p, trq=%p", arg.vio, TSVIOVConnGet(arg.vio), cdata.trq); + + TSIOBufferBlock blk; + ssize_t consumed = 0; + TSParseResult result = TS_PARSE_CONT; + + for (blk = TSIOBufferReaderStart(cdata.trq->readio.reader); blk; blk = TSIOBufferBlockNext(blk)) { + const char *ptr; + const char *end; + int64_t nbytes; + TSHttpStatus status = static_cast(cdata.trq->statusCode); + + ptr = TSIOBufferBlockReadStart(blk, cdata.trq->readio.reader, &nbytes); + if (ptr == nullptr || nbytes == 0) { + continue; + } + + end = ptr + nbytes; + result = TSHttpHdrParseReq(rqheader.parser, rqheader.buffer, rqheader.header, &ptr, end); + switch (result) { + case TS_PARSE_ERROR: + // If we got a bad request, just shut it down. + VDEBUG("bad request on trq=%p, sending an error", cdata.trq); + StaticHitRequestDestroy(cdata.trq, arg.vio, contp); + return TS_EVENT_ERROR; + + case TS_PARSE_DONE: + // Check the response. + VDEBUG("parsed request on trq=%p, sending a response", cdata.trq); + if (!StaticHitParseRequest(cdata.trq)) { + status = TS_HTTP_STATUS_METHOD_NOT_ALLOWED; + } + + // Start the vconn write. + cdata.trq->writeio.write(TSVIOVConnGet(arg.vio), contp); + TSVIONBytesSet(cdata.trq->writeio.vio, 0); + + WriteResponseHeader(cdata.trq, contp, status); + return TS_EVENT_NONE; + + case TS_PARSE_CONT: + // We consumed the buffer we got minus the remainder. + consumed += (nbytes - std::distance(ptr, end)); + } + } + + TSReleaseAssert(result == TS_PARSE_CONT); + + // Reenable the read VIO to get more events. + TSVIOReenable(arg.vio); + return TS_EVENT_NONE; + } + + case TS_EVENT_VCONN_WRITE_READY: { + argument_type cdata = TSContDataGet(contp); + + if (cdata.trq->nbytes) { + int64_t nbytes = cdata.trq->nbytes; + + VIODEBUG(arg.vio, "writing %" PRId64 " bytes for trq=%p", nbytes, cdata.trq); + nbytes = TSIOBufferWrite(cdata.trq->writeio.iobuf, cdata.trq->body.c_str(), nbytes); + + cdata.trq->nbytes -= nbytes; + TSStatIntIncrement(StatCountBytes, nbytes); + + // Update the number of bytes to write. + TSVIONBytesSet(arg.vio, TSVIONBytesGet(arg.vio) + nbytes); + TSVIOReenable(arg.vio); + } + + return TS_EVENT_NONE; + } + + case TS_EVENT_ERROR: + case TS_EVENT_VCONN_EOS: { + argument_type cdata = TSContDataGet(contp); + + VIODEBUG(arg.vio, "received EOS or ERROR for trq=%p", cdata.trq); + StaticHitRequestDestroy(cdata.trq, arg.vio, contp); + return event == TS_EVENT_ERROR ? TS_EVENT_ERROR : TS_EVENT_NONE; + } + + case TS_EVENT_VCONN_READ_COMPLETE: + // We read data forever, so we should never get a READ_COMPLETE. + VIODEBUG(arg.vio, "unexpected TS_EVENT_VCONN_READ_COMPLETE"); + return TS_EVENT_NONE; + + case TS_EVENT_VCONN_WRITE_COMPLETE: { + argument_type cdata = TSContDataGet(contp); + + // If we still have bytes to write, kick off a new write operation, otherwise + // we are done and we can shut down the VC. + if (cdata.trq->nbytes) { + cdata.trq->writeio.write(TSVIOVConnGet(arg.vio), contp); + TSVIONBytesSet(cdata.trq->writeio.vio, cdata.trq->nbytes); + } else { + VIODEBUG(arg.vio, "TS_EVENT_VCONN_WRITE_COMPLETE %" PRId64 " todo", TSVIONTodoGet(arg.vio)); + StaticHitRequestDestroy(cdata.trq, arg.vio, contp); + } + + return TS_EVENT_NONE; + } + + case TS_EVENT_TIMEOUT: { + return TS_EVENT_NONE; + } + + case TS_EVENT_VCONN_INACTIVITY_TIMEOUT: + VERROR("unexpected event %s (%d) edata=%p", TSHttpEventNameLookup(event), event, arg.ptr); + return TS_EVENT_ERROR; + + default: + VERROR("unexpected event %s (%d) edata=%p", TSHttpEventNameLookup(event), event, arg.ptr); + return TS_EVENT_ERROR; + } +} + +static void +StaticHitSetupIntercept(StaticHitConfig *cfg, TSHttpTxn txn) +{ + StaticHitRequest *req = StaticHitRequest::createStaticHitRequest(cfg); + if (req == nullptr) { + VERROR("could not create request for %s", cfg->filePath.c_str()); + return; + } + + TSCont cnt = TSContCreate(StaticHitInterceptHook, TSMutexCreate()); + TSContDataSet(cnt, req); + + TSHttpTxnServerIntercept(cnt, txn); + + return; +} + +// Handle events that occur on the TSHttpTxn. +static int +StaticHitTxnHook(TSCont contp, TSEvent event, void *edata) +{ + argument_type arg(edata); + + VDEBUG("contp=%p, event=%s (%d), edata=%p", contp, TSHttpEventNameLookup(event), event, edata); + + switch (event) { + case TS_EVENT_HTTP_CACHE_LOOKUP_COMPLETE: { + int method_length, status; + TSMBuffer bufp; + TSMLoc hdr_loc; + const char *method; + + TSReleaseAssert(TSHttpTxnCacheLookupStatusGet(arg.txn, &status) == TS_SUCCESS); + + if (TSHttpTxnClientReqGet(arg.txn, &bufp, &hdr_loc) != TS_SUCCESS) { + VERROR("Couldn't retrieve client request header"); + goto done; + } + + method = TSHttpHdrMethodGet(bufp, hdr_loc, &method_length); + if (NULL == method) { + VERROR("Couldn't retrieve client request method"); + goto done; + } + + if (status != TS_CACHE_LOOKUP_HIT_FRESH || method != TS_HTTP_METHOD_GET) { + StaticHitSetupIntercept(static_cast(TSContDataGet(contp)), arg.txn); + } + + break; + } + + default: + VERROR("unexpected event %s (%d)", TSHttpEventNameLookup(event), event); + break; + } + +done: + TSHttpTxnReenable(arg.txn, TS_EVENT_HTTP_CONTINUE); + return TS_EVENT_NONE; +} + +TSReturnCode +TSRemapInit(TSRemapInterface * /* api_info */, char * /* errbuf */, int /* errbuf_size */) +{ + TxnHook = TSContCreate(StaticHitTxnHook, nullptr); + + if (TSStatFindName("statichit.response_bytes", &StatCountBytes) == TS_ERROR) { + StatCountBytes = TSStatCreate("statichit.response_bytes", TS_RECORDDATATYPE_COUNTER, TS_STAT_NON_PERSISTENT, TS_STAT_SYNC_SUM); + } + + if (TSStatFindName("statichit.response_count", &StatCountResponses) == TS_ERROR) { + StatCountResponses = + TSStatCreate("statichit.response_count", TS_RECORDDATATYPE_COUNTER, TS_STAT_NON_PERSISTENT, TS_STAT_SYNC_COUNT); + } + return TS_SUCCESS; +} + +TSRemapStatus +TSRemapDoRemap(void *ih, TSHttpTxn rh, TSRemapRequestInfo *rri) +{ + StaticHitConfig *cfg = static_cast(ih); + + if (!cfg) { + VERROR("No remap context available, check code / config"); + TSHttpTxnStatusSet(rh, TS_HTTP_STATUS_INTERNAL_SERVER_ERROR); + return TSREMAP_NO_REMAP; + } + + if (!cfg->maxAge) { + TSHttpTxnConfigIntSet(rh, TS_CONFIG_HTTP_CACHE_HTTP, 0); + StaticHitSetupIntercept(static_cast(ih), rh); + } else { + TSHttpTxnHookAdd(rh, TS_HTTP_CACHE_LOOKUP_COMPLETE_HOOK, TxnHook); + } + TSContDataSet(TxnHook, ih); + + return TSREMAP_NO_REMAP; // This plugin never rewrites anything. +} + +TSReturnCode +TSRemapNewInstance(int argc, char *argv[], void **ih, char * /* errbuf ATS_UNUSED */, int /* errbuf_size ATS_UNUSED */) +{ + static const struct option longopt[] = { + {"file-path", required_argument, NULL, 'f'}, {"mime-type", required_argument, NULL, 'm'}, + {"max-age", required_argument, NULL, 'a'}, {"failure-code", required_argument, NULL, 'c'}, + {"success-code", required_argument, NULL, 's'}, {NULL, no_argument, NULL, '\0'}}; + + std::string filePath; + std::string mimeType = "text/plain"; + int maxAge = 0, failureCode = 0, successCode = 0; + + // argv contains the "to" and "from" URLs. Skip the first so that the + // second one poses as the program name. + --argc; + ++argv; + optind = 0; + + while (true) { + int opt = getopt_long(argc, (char *const *)argv, "f:m:a:c:s:", longopt, NULL); + + switch (opt) { + case 'f': { + filePath = std::string(optarg); + } break; + case 'm': { + mimeType = std::string(optarg); + } break; + case 'a': { + maxAge = atoi(optarg); + } break; + case 'c': { + failureCode = atoi(optarg); + } break; + case 's': { + successCode = atoi(optarg); + } break; + } + + if (opt == -1) { + break; + } + } + + if (filePath.size() == 0) { + printf("Need to specify --file-path\n"); + return TS_ERROR; + } + + StaticHitConfig *tc = new StaticHitConfig(filePath, mimeType); + if (maxAge > 0) { + tc->maxAge = maxAge; + } + if (failureCode > 0) { + tc->failureCode = failureCode; + } + if (successCode > 0) { + tc->successCode = successCode; + } + + *ih = static_cast(tc); + + return TS_SUCCESS; +} + +void +TSRemapDeleteInstance(void *ih) +{ + StaticHitConfig *tc = static_cast(ih); + delete tc; +} diff --git a/plugins/experimental/stream_editor/stream_editor.cc b/plugins/experimental/stream_editor/stream_editor.cc index 9be3e232365..839be9e9b7c 100644 --- a/plugins/experimental/stream_editor/stream_editor.cc +++ b/plugins/experimental/stream_editor/stream_editor.cc @@ -98,6 +98,7 @@ #include #include #include +#include #include "ts/ts.h" struct edit_t; @@ -616,8 +617,8 @@ process_block(contdata_t *contdata, TSIOBufferReader reader) editset_t edits; - for (rule_p r = contdata->rules.begin(); r != contdata->rules.end(); ++r) { - r->apply(buf, buflen, edits); + for (const auto &rule : contdata->rules) { + rule.apply(buf, buflen, edits); } for (edit_p p = edits.begin(); p != edits.end(); ++p) { @@ -766,13 +767,13 @@ streamedit_setup(TSCont contp, TSEvent event, void *edata) assert((event == TS_EVENT_HTTP_READ_RESPONSE_HDR) || (event == TS_EVENT_HTTP_READ_REQUEST_HDR)); /* make a new list comprising those rules that are in scope */ - for (rule_p r = rules_in->begin(); r != rules_in->end(); ++r) { - if (r->in_scope(txn)) { + for (const auto &r : *rules_in) { + if (r.in_scope(txn)) { if (contdata == nullptr) { contdata = new contdata_t(); } - contdata->rules.push_back(*r); - contdata->set_cont_size(r->cont_size()); + contdata->rules.push_back(r); + contdata->set_cont_size(r.cont_size()); } } diff --git a/plugins/experimental/system_stats/system_stats.c b/plugins/experimental/system_stats/system_stats.c index d263eacfbd8..c4f2588535a 100644 --- a/plugins/experimental/system_stats/system_stats.c +++ b/plugins/experimental/system_stats/system_stats.c @@ -49,7 +49,7 @@ // Load Average Strings #define LOAD_AVG_ONE_MIN "plugin." PLUGIN_NAME ".loadavg.one" #define LOAD_AVG_FIVE_MIN "plugin." PLUGIN_NAME ".loadavg.five" -#define LOAD_AVG_TEN_MIN "plugin." PLUGIN_NAME ".loadavg.ten" +#define LOAD_AVG_FIFTEEN_MIN "plugin." PLUGIN_NAME ".loadavg.fifteen" // Process Strings #define CURRENT_PROCESSES "plugin." PLUGIN_NAME ".current_processes" @@ -206,7 +206,7 @@ getStats(TSMutex stat_creation_mutex) statSet(LOAD_AVG_ONE_MIN, info.loads[0], stat_creation_mutex); statSet(LOAD_AVG_FIVE_MIN, info.loads[1], stat_creation_mutex); - statSet(LOAD_AVG_TEN_MIN, info.loads[2], stat_creation_mutex); + statSet(LOAD_AVG_FIFTEEN_MIN, info.loads[2], stat_creation_mutex); statSet(CURRENT_PROCESSES, info.procs, stat_creation_mutex); #endif // #ifdef HAVE_SYS_SYSINFO_H netStatsInfo(stat_creation_mutex); diff --git a/plugins/experimental/traffic_dump/Makefile.inc b/plugins/experimental/traffic_dump/Makefile.inc index ac875961f02..411a9f78198 100644 --- a/plugins/experimental/traffic_dump/Makefile.inc +++ b/plugins/experimental/traffic_dump/Makefile.inc @@ -16,4 +16,30 @@ pkglib_LTLIBRARIES += experimental/traffic_dump/traffic_dump.la -experimental_traffic_dump_traffic_dump_la_SOURCES = experimental/traffic_dump/traffic_dump.cc +experimental_traffic_dump_traffic_dump_la_SOURCES = \ + experimental/traffic_dump/global_variables.h \ + experimental/traffic_dump/json_utils.cc \ + experimental/traffic_dump/json_utils.h \ + experimental/traffic_dump/sensitive_fields.h \ + experimental/traffic_dump/session_data.cc \ + experimental/traffic_dump/session_data.h \ + experimental/traffic_dump/traffic_dump.cc \ + experimental/traffic_dump/transaction_data.cc \ + experimental/traffic_dump/transaction_data.h + +check_PROGRAMS += \ + experimental/traffic_dump/test_traffic_dump + +experimental_traffic_dump_test_traffic_dump_CPPFLAGS = \ + $(AM_CPPFLAGS) \ + -I$(abs_top_srcdir)/plugins/experimental/traffic_dump \ + -I$(abs_top_srcdir)/tests/include + +experimental_traffic_dump_test_traffic_dump_SOURCES = \ + experimental/traffic_dump/unit_tests/unit_test_main.cc \ + experimental/traffic_dump/unit_tests/test_json_utils.cc \ + experimental/traffic_dump/unit_tests/test_sensitive_fields.cc \ + experimental/traffic_dump/json_utils.cc \ + experimental/traffic_dump/sensitive_fields.h + +# vim: ft=make ts=8 sw=8 et: diff --git a/plugins/experimental/traffic_dump/global_variables.h b/plugins/experimental/traffic_dump/global_variables.h new file mode 100644 index 00000000000..18a9555926f --- /dev/null +++ b/plugins/experimental/traffic_dump/global_variables.h @@ -0,0 +1,25 @@ +/** @sensitive_fields.h + The set of fields considered user-sensitive. + @section license License + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#pragma once + +namespace traffic_dump +{ +constexpr char debug_tag[] = "traffic_dump"; + +} // namespace traffic_dump diff --git a/plugins/experimental/traffic_dump/json_utils.cc b/plugins/experimental/traffic_dump/json_utils.cc new file mode 100644 index 00000000000..eb2ff37796b --- /dev/null +++ b/plugins/experimental/traffic_dump/json_utils.cc @@ -0,0 +1,178 @@ +/** @json_utils.cc + Implementation of JSON formatting functions. + @section license License + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#include +#include + +#include "json_utils.h" + +namespace +{ +/** Write the content of buf to jsonfile between prevIdx (inclusive) and idx (not + * inclusive). + * + * This function is used to help deal with escaped characters with a low number + * of write calls. This is meant to be used like so: the caller inspects every + * character in a buffer, doing one of two things with each character: + * + * - The character does not need to be escaped. In this case, no call to + * write_buffered_content is made and the idx is advanced and prevIdx is not + * advanced. Eventually this character will be written once the contiguous set + * of non-escaped characters is collected. + * + * - The character needs to be escaped. In this case, it will call this + * function. Anything between prevIdx and up to (but not including) idx is + * written. The caller then writes the escaped sequence for the character + * pointed to by idx ("\\t" instead of '\t', for instance). Then prevIdx is set + * past the escaped character. + * + * Finally, once all characters are inspected, a final write_buffered_content + * call is made with idx set to one past the last character in buf. This + * results in all characters between prevIdx and the last character in buf + * (including that character) to be written. + * + * @param[in] buf The buffer containing characters that need to be written to jsonfile. + * + * @param[in,out] prevIdx The pointer to the beginning of the set of characters + * in buf that have not been written yet. This is always updated before + * function exit to one past idx. + * + * @param[in] idx The current character in buf being inspected. + * + * @param[out] jsonfile The stream to conditionally write buffer content to. + */ +inline void +write_buffered_context(char const *buf, int64_t &prevIdx, int64_t idx, std::ostream &jsonfile) +{ + if (prevIdx < idx) { + jsonfile.write(buf + prevIdx, idx - prevIdx); + } + prevIdx = idx + 1; +} + +int +esc_json_out(const char *buf, int64_t len, std::ostream &jsonfile) +{ + if (buf == nullptr) { + return 0; + } + int64_t idx = 0, prevIdx = 0; + // For an explanation of the algorithm here, see the doxygen comment for + // write_buffered_content. + for (idx = 0; idx < len; idx++) { + char c = *(buf + idx); + switch (c) { + case '"': + case '\\': { + write_buffered_context(buf, prevIdx, idx, jsonfile); + jsonfile << "\\" << c; + break; + } + case '\b': { + write_buffered_context(buf, prevIdx, idx, jsonfile); + jsonfile << "\\b"; + break; + } + case '\f': { + write_buffered_context(buf, prevIdx, idx, jsonfile); + jsonfile << "\\f"; + break; + } + case '\n': { + write_buffered_context(buf, prevIdx, idx, jsonfile); + jsonfile << "\\n"; + break; + } + case '\r': { + write_buffered_context(buf, prevIdx, idx, jsonfile); + jsonfile << "\\r"; + break; + } + case '\t': { + write_buffered_context(buf, prevIdx, idx, jsonfile); + jsonfile << "\\t"; + break; + } + default: { + if ('\x00' <= c && c <= '\x1f') { + write_buffered_context(buf, prevIdx, idx, jsonfile); + jsonfile << "\\u" << std::hex << std::setw(4) << std::setfill('0') << static_cast(c); + } + break; + // else: The character does not need to be escaped. Do not call + // write_buffered_content so nothing is written and prevIdx remains + // pointing to the first character that needs to be written on the next + // call to write_buffered_content. + } + } + } + // Finally, call write_buffered_content to write out any data that has not + // been written yet. + write_buffered_context(buf, prevIdx, idx, jsonfile); + + return len; +} +/** Escape characters in a string as needed and return the resultant escaped string. + * + * @param[in] s The characters that need to be escaped. + */ +std::string +escape_json(std::string_view s) +{ + std::ostringstream o; + esc_json_out(s.data(), s.length(), o); + return o.str(); +} + +/** An escape_json overload for a char buffer. + * + * @param[in] buf The char buffer pointer with characters that need to be escaped. + * + * @param[in] size The size of the buf char array. + */ +std::string +escape_json(char const *buf, int64_t size) +{ + std::ostringstream o; + esc_json_out(buf, size, o); + return o.str(); +} + +} // anonymous namespace + +namespace traffic_dump +{ +std::string +json_entry(std::string_view name, std::string_view value) +{ + return "\"" + escape_json(name) + "\":\"" + escape_json(value) + "\""; +} + +std::string +json_entry(std::string_view name, char const *value, int64_t size) +{ + return "\"" + escape_json(name) + "\":\"" + escape_json(value, size) + "\""; +} + +std::string +json_entry_array(std::string_view name, std::string_view value) +{ + return "[\"" + escape_json(name) + "\",\"" + escape_json(value) + "\"]"; +} + +} // namespace traffic_dump diff --git a/plugins/experimental/traffic_dump/json_utils.h b/plugins/experimental/traffic_dump/json_utils.h new file mode 100644 index 00000000000..7d26012e96c --- /dev/null +++ b/plugins/experimental/traffic_dump/json_utils.h @@ -0,0 +1,58 @@ +/** @json_utils.h + JSON formatting functions. + @section license License + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#pragma once + +#include +#include + +namespace traffic_dump +{ +/** Create the name and value as an escaped JSON map entry. + * + * @param[in] name The key name for the map entry. + * + * @param[in] value The value to write. + * + * @return The JSON map string. + */ +std::string json_entry(std::string_view name, std::string_view value); + +/** Create the name and value as an escaped JSON map entry. + * + * @param[in] name The key name for the map entry. + * + * @param[in] value The buffer for the value to write. + * + * @param[in] size The size of the value buffer. + * + * @return The JSON map string. + */ +std::string json_entry(std::string_view name, char const *value, int64_t size); + +/** Create the name and value as an escaped JSON array entry. + * + * @param[in] name The key name for the map entry. + * + * @param[in] value The value to write for the JSON map entry. + * + * @return The JSON array string. + */ +std::string json_entry_array(std::string_view name, std::string_view value); + +} // namespace traffic_dump diff --git a/plugins/experimental/traffic_dump/post_process.py b/plugins/experimental/traffic_dump/post_process.py new file mode 100755 index 00000000000..0472cbc0735 --- /dev/null +++ b/plugins/experimental/traffic_dump/post_process.py @@ -0,0 +1,426 @@ +#!/usr/bin/env python3 +''' +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +''' + +from collections import defaultdict +from copy import deepcopy +from queue import Queue +from threading import Thread +import argparse +import json +import logging +import os +import sys + +description = ''' +Post process replay files produced by traffic_dump and clean it of any +incomplete transactions or sessions which got partially written because Traffic +Server was interrupted mid-connection. This also merges sessions to the same +client and, by default, formats the output files with human readable spacing. +''' + + +# Base replay file template with basic elements +TEMPLATE = json.loads('{"meta": {"version":"1.0"},"sessions":[]}') + + +class PostProcessError(Exception): + ''' Base class for post processing errors. + ''' + + def __init__(self, message=None): + self.message = message + + def __str__(self, *args): + if self.message: + return self.message + else: + return 'PostProcessError raised' + + +class VerifyError(PostProcessError): + ''' Base class for node node verification errors. + ''' + pass + + +class VerifyRequestError(VerifyError): + ''' There was a problem verifying a request node. + ''' + pass + + +class VerifyResponseError(VerifyError): + ''' There was a problem verifying a response node. + ''' + pass + + +class VerifySessionError(VerifyError): + ''' There was a problem verifying a session node. + ''' + pass + + +def verify_request(request): + """ Function to verify request with method, url, and headers + Args: + request (json object) + + Raises: + VerifyRequestError if there is a problem with the request. + """ + if not request: + raise VerifyRequestError('No request found.') + if "method" not in request or not request["method"]: + raise VerifyRequestError("Request did not have a method.") + if "url" not in request or not request["url"]: + raise VerifyRequestError("Request did not have a url.") + if "headers" not in request or not request["headers"]: + raise VerifyRequestError("Request did not have headers.") + + +def verify_response(response): + """ Function to verify response with status + Args: + response (json object) + + Raises: + VerifyResponseError if there is a problem with the response. + """ + if not response: + raise VerifyResponseError("No response found.") + if "status" not in response or not response["status"]: + raise VerifyResponseError("Response did not have a status.") + + +def verify_transaction(transaction, fabricate_proxy_requests=False): + """ Function to verify that a transaction looks complete. + + Args: + transaction (json object) + fabricate_proxy_requests (bool) Whether the post-processor should + fabricate proxy requests if they don't exist because the proxy served + the response locally. + + Raises: + VerifySessionError if there is no transaction. + VerifyRequestError if there is a problem with a request. + VerifyResponseError if there is a problem with a response. + """ + if not transaction: + raise VerifySessionError('No transaction found in the session.') + + if "client-request" not in transaction: + raise VerifyRequestError('client-request not found in transaction') + else: + verify_request(transaction["client-request"]) + + if "proxy-request" not in transaction and fabricate_proxy_requests: + if "proxy-response" not in transaction: + raise VerifyRequestError('proxy-response not found in transaction with a client-request') + transaction["proxy-request"] = transaction["client-request"] + if "server-response" not in transaction: + transaction["server-response"] = transaction["proxy-response"] + + # proxy-response nodes can be empty. + if "proxy-response" not in transaction: + raise VerifyResponseError('proxy-response not found in transaction') + + if "proxy-request" in transaction or "server-response" in transaction: + # proxy-request nodes can be empty, so no need to verify_response. + if "proxy-request" not in transaction: + raise VerifyRequestError('proxy-request not found in transaction') + + if "server-response" not in transaction: + raise VerifyResponseError('server-response not found in transaction') + else: + verify_response(transaction["server-response"]) + + +def verify_session(session, fabricate_proxy_requests=False): + """ Function to verify that a session looks complete. + + A valid session contains a valid list of transactions. + + Args: + transaction (json object) + fabricate_proxy_requests (bool) Whether the post-processor should + fabricate proxy requests if they don't exist because the proxy served + the response locally. + + Raises: + VerifyError if there is a problem with the session. + """ + if not session: + raise VerifySessionError('Session not found.') + if "transactions" not in session or not session["transactions"]: + raise VerifySessionError('No transactions found in session.') + for transaction in session["transactions"]: + verify_transaction(transaction, fabricate_proxy_requests) + + +def write_sessions(sessions, filename, indent): + """ Write the JSON sessions to the given filename. + + Args: + sessions The parsed JSON sessions to dump into filename. + filename (string) The path to the file to write the parsed JSON file to. + indent (int) The number of spaces per line to write to the file. A + value of None causes the whole JSON file to be written as a single line. + """ + new_json = deepcopy(TEMPLATE) + new_json["sessions"] = deepcopy(sessions) + with open(filename, "w") as f: + json.dump(new_json, f, ensure_ascii=False, indent=indent) + logging.debug("{} has {} sessions".format(filename, len(sessions))) + + +class ParseJSONError(PostProcessError): + ''' There was an error opening or parsing the replay file. + ''' + pass + + +def parse_json(replay_file): + """ Open and parse the replay_file. + + Args: + replay_file (string) The file with JSON content to parse. + + Return: + The json package parsed JSON file or None if there was a problem + parsing the file. + """ + try: + fd = open(replay_file, 'r') + except Exception as e: + logging.exception("Failed to open %s.", replay_file) + raise ParseJSONError(e) + + try: + parsed_json = json.load(fd) + except Exception as e: + message = e.msg.split(':')[0] + logging.error("Failed to load %s as a JSON object: %s", replay_file, e) + raise ParseJSONError(message) + + return parsed_json + + +def readAndCombine(replay_dir, num_sessions_per_file, indent, fabricate_proxy_requests, out_dir): + """ Read raw dump files, filter out incomplete sessions, and merge + them into output files. + + Args: + replay_dir (string) Full path to dumps + num_sessions_per_file (int) number of sessions in each output file + indent (int) The number of spaces per line in the output replay files. + fabricate_proxy_requests (bool) Whether the post-processor should + fabricate proxy requests if they don't exist because the proxy served + the response locally. + out_dir (string) Output directory for post-processed json files. + """ + session_count = 0 + batch_count = 0 + transaction_count = 0 + error_count = defaultdict(int) + + base_name = os.path.basename(replay_dir) + + sessions = [] + for f in os.listdir(replay_dir): + replay_file = os.path.join(replay_dir, f) + if not os.path.isfile(replay_file): + continue + + try: + parsed_json = parse_json(replay_file) + except ParseJSONError as e: + error_count[e.message] += 1 + continue + + for session in parsed_json["sessions"]: + try: + verify_session(session, fabricate_proxy_requests) + except VerifyError as e: + connection_time = session['connection-time'] + if not connection_time: + connection_time = session['start-time'] + if connection_time: + logging.debug("Omitting session in %s with connection-time: %d: %s", + replay_file, session['connection-time'], e) + else: + logging.debug("Omitting a session in %s, could not find a connection time: %s", + replay_file, e) + continue + sessions.append(session) + session_count += 1 + transaction_count += len(session["transactions"]) + if len(sessions) >= num_sessions_per_file: + write_sessions(sessions, "{}/{}_{}.json".format(out_dir, base_name, batch_count), indent) + sessions = [] + batch_count += 1 + if sessions: + write_sessions(sessions, "{}/{}_{}.json".format(out_dir, base_name, batch_count), indent) + + return session_count, transaction_count, error_count + + +def post_process(in_dir, subdir_q, out_dir, num_sessions_per_file, single_line, fabricate_proxy_requests, cnt_q): + """ Function used to set up individual threads. + + Each thread loops over the subdir_q, pulls a directory from there, and + process the replay files in that directory. The threads finish when the + subdir queue is empty, meaning each subdir has been processed. + + Args: + in_dir (string) Path to parent of the subdirectories in subdir_q. + subdir_q (Queue) Queue of subdir to read from. + out_dir (string) The directory into which the post processed replay files + are placed. + num_sessions_per_file (int) traffic_dump will emit a separate file per + session. This mechanism merges sessions within a single subdir. + num_sessions_per_file is the limit to the number of sessions merged + into a single replay file. + single_line (bool) Whether to emit replay files as a single line. If + false, the file is spaced out in a human readable fashion. + fabricate_proxy_requests (bool) Whether the post-processor should + fabricate proxy requests if they don't exist because the proxy served + the response locally. + cnt_q (Queue) Session, transaction, error count queue populated by each + thread. + """ + while not subdir_q.empty(): + subdir = subdir_q.get() + subdir_path = os.path.join(in_dir, subdir) + indent = 2 + if single_line: + indent = None + cnt = readAndCombine(subdir_path, num_sessions_per_file, indent, fabricate_proxy_requests, out_dir) + cnt_q.put(cnt) + + +def configure_logging(use_debug=False): + ''' Configure the logging mechanism. + + Args: + use_debug (bool) Whether to configure debug-level logging. + ''' + log_format = '%(levelname)s: %(message)s' + if use_debug: + logging.basicConfig(format=log_format, level=logging.DEBUG) + else: + logging.basicConfig(format=log_format, level=logging.INFO) + + +def parse_args(): + ''' Parse the command line arguments. + ''' + parser = argparse.ArgumentParser(description=description) + + parser.add_argument("in_dir", type=str, + help='''The input directory of traffic_dump replay + files. The expectation is that this will contain + sub-directories that themselves contain replay files. + This is written to accommodate the directory populated + by traffic_dump via the --logdir option.''') + parser.add_argument("out_dir", type=str, + help="The output directory of post processed replay files.") + parser.add_argument("-n", "--num_sessions", type=int, default=10, + help='''The maximum number of sessions merged into + single replay output files. The default is 10.''') + parser.add_argument("--no-human-readable", action="store_true", + help='''By default, post processor will generate replay + files that are spaced out in a human readable format. + This turns off that behavior and leaves the files as + single-line entries.''') + parser.add_argument("--no-fabricate-proxy-requests", action="store_true", + help='''By default, post processor will fabricate proxy + requests and server responses for transactions served + out of the proxy. Presumably in replay conditions, + these fabricated requests and responses will not hurt + anything because the Proxy Verifier server will not + notice if the proxy replies locally in replay + conditions. However, if it doesn't reply locally, then + the server will not know how to reply to these + requests. Using this option turns off this fabrication + behavior.''') + parser.add_argument("-j", "--num_threads", type=int, default=32, + help='''The maximum number of threads to use.''') + parser.add_argument("-d", "--debug", action="store_true", + help="Enable debug level logging.") + return parser.parse_args() + + +def main(): + args = parse_args() + configure_logging(use_debug=args.debug) + logging.debug("Original options: %s", " ".join(sys.argv)) + + if not os.path.exists(args.out_dir): + os.mkdir(args.out_dir) + + # generate thread arguments + subdir_q = Queue() + cnt_q = Queue() + for subdir in os.listdir(args.in_dir): + if os.path.isdir(os.path.join(args.in_dir, subdir)): + subdir_q.put(subdir) + + threads = [] + nthreads = min(max(subdir_q.qsize(), 1), args.num_threads) + + # Start up the threads. + for _ in range(nthreads): + t = Thread(target=post_process, + args=(args.in_dir, subdir_q, args.out_dir, + args.num_sessions, args.no_human_readable, + not args.no_fabricate_proxy_requests, cnt_q)) + t.start() + threads.append(t) + + # Wait for them to finish. + for t in threads: + t.join() + + # Retrieve the counts + session_count = 0 + transaction_count = 0 + errors = defaultdict(int) + for count_tuple in list(cnt_q.queue): + session_count += count_tuple[0] + transaction_count += count_tuple[1] + for e in count_tuple[2]: + errors[e] += count_tuple[2][e] + summary = "Total {} sessions and {} transactions.".format(session_count, transaction_count) + logging.info(summary) + if errors: + logging.info("Total errors:") + for e in errors: + logging.info("{}: {}".format(e, errors[e])) + else: + logging.info("Total errors: 0") + + with open("{}/summary.txt".format(args.out_dir), "w", encoding="ascii") as f: + f.write("{}\n".format(summary)) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/plugins/experimental/traffic_dump/sensitive_fields.h b/plugins/experimental/traffic_dump/sensitive_fields.h new file mode 100644 index 00000000000..8e0db2d0002 --- /dev/null +++ b/plugins/experimental/traffic_dump/sensitive_fields.h @@ -0,0 +1,54 @@ +/** @sensitive_fields.h + Define the type used to store user-sensitive HTTP fields (such as cookies). + @section license License + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#pragma once + +#include +#include +#include +#include +#include + +namespace traffic_dump +{ +// A case-insensitive comparitor used for comparing HTTP field names. +struct InsensitiveCompare { + bool + operator()(std::string_view a, std::string_view b) const + { + return strcasecmp(a.data(), b.data()) == 0; + } +}; + +// A case-insensitive hash functor for HTTP field names. +struct StringHashByLower { +public: + size_t + operator()(std::string_view str) const + { + std::string lower; + std::transform(str.begin(), str.end(), lower.begin(), [](unsigned char c) -> unsigned char { return std::tolower(c); }); + return std::hash()(lower); + } +}; + +/** The type used to store the set of user-sensitive HTTP fields, such as + * "Cookie" and "Set-Cookie". */ +using sensitive_fields_t = std::unordered_set; + +} // namespace traffic_dump diff --git a/plugins/experimental/traffic_dump/session_data.cc b/plugins/experimental/traffic_dump/session_data.cc new file mode 100644 index 00000000000..4c91f20c766 --- /dev/null +++ b/plugins/experimental/traffic_dump/session_data.cc @@ -0,0 +1,476 @@ +/** @session_handler.h + Traffic Dump session handling implementation + @section license License + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "session_data.h" +#include "global_variables.h" +#include "transaction_data.h" + +namespace +{ +/** The final string used to close a JSON session. */ +char const constexpr *const json_closing = "]}]}"; +} // namespace + +namespace traffic_dump +{ +// Static member initialization. +int SessionData::session_arg_index = -1; +std::atomic SessionData::sample_pool_size = default_sample_pool_size; +std::atomic SessionData::max_disk_usage = default_max_disk_usage; +std::atomic SessionData::disk_usage = 0; +ts::file::path SessionData::log_directory{default_log_directory}; +uint64_t SessionData::session_counter = 0; +std::string SessionData::sni_filter; + +int +SessionData::get_session_arg_index() +{ + return session_arg_index; +} + +void +SessionData::set_sample_pool_size(int64_t new_sample_size) +{ + sample_pool_size = new_sample_size; +} + +void +SessionData::reset_disk_usage() +{ + disk_usage = 0; +} + +void +SessionData::set_max_disk_usage(int64_t new_max_disk_usage) +{ + max_disk_usage = new_max_disk_usage; +} + +bool +SessionData::init(std::string_view log_directory, int64_t max_disk_usage, int64_t sample_size) +{ + SessionData::log_directory = log_directory; + SessionData::max_disk_usage = max_disk_usage; + SessionData::sample_pool_size = sample_size; + + if (TS_SUCCESS != TSUserArgIndexReserve(TS_USER_ARGS_SSN, debug_tag, "Track log related data", &session_arg_index)) { + TSError("[%s] Unable to initialize plugin (disabled). Failed to reserve ssn arg.", traffic_dump::debug_tag); + return false; + } + + TSCont ssncont = TSContCreate(global_session_handler, nullptr); + TSHttpHookAdd(TS_HTTP_SSN_START_HOOK, ssncont); + TSHttpHookAdd(TS_HTTP_SSN_CLOSE_HOOK, ssncont); + + TSDebug(debug_tag, "Initialized with log directory: %s", SessionData::log_directory.c_str()); + TSDebug(debug_tag, "Initialized with sample pool size %" PRId64 " bytes and disk limit %" PRId64 " bytes", sample_size, + max_disk_usage); + return true; +} + +bool +SessionData::init(std::string_view log_directory, int64_t max_disk_usage, int64_t sample_size, std::string_view sni_filter) +{ + if (!init(log_directory, max_disk_usage, sample_size)) { + return false; + } + SessionData::sni_filter = sni_filter; + TSDebug(debug_tag, "Filtering to only dump connections with SNI: %s", SessionData::sni_filter.c_str()); + return true; +} + +/** Create a TLS characteristics node. + * + * This function encapsulates the logic common between the client-side and + * server-side logic for populating the TLS node. + * + * @param[in] ssnp The pointer for this session. + * + * @return The node describing the TLS properties of this session. + */ +std::string +get_tls_description_helper(TSVConn ssn_vc) +{ + TSSslConnection ssl_conn = TSVConnSslConnectionGet(ssn_vc); + SSL *ssl_obj = reinterpret_cast(ssl_conn); + if (ssl_obj == nullptr) { + return ""; + } + std::ostringstream tls_description; + tls_description << R"("tls":{)"; + char const *sni_ptr = SSL_get_servername(ssl_obj, TLSEXT_NAMETYPE_host_name); + if (sni_ptr != nullptr) { + std::string_view sni{sni_ptr}; + if (!sni.empty()) { + tls_description << R"("sni":")" << sni << R"(",)"; + } + } + tls_description << R"("verify_mode":")" << std::to_string(SSL_get_verify_mode(ssl_obj)) << R"(")"; + tls_description << "}"; + return tls_description.str(); +} + +/** Create a server-side TLS characteristics node. + * + * @param[in] ssnp The pointer for this session. + * + * @return The node describing the TLS properties of this session. + */ +std::string +get_server_tls_description(TSHttpSsn ssnp) +{ + TSVConn ssn_vc = TSHttpSsnServerVConnGet(ssnp); + return get_tls_description_helper(ssn_vc); +} + +/** Create a client-side TLS characteristics node. + * + * @param[in] ssnp The pointer for this session. + * + * @return The node describing the TLS properties of this session. + */ +std::string +get_client_tls_description(TSHttpSsn ssnp) +{ + TSVConn ssn_vc = TSHttpSsnClientVConnGet(ssnp); + return get_tls_description_helper(ssn_vc); +} + +/// A named boolean for callers who pass the is_client parameter. +constexpr bool IS_CLIENT = true; + +/** Create the nodes that describe the session's sub-HTTP protocols. + * + * This function encapsulates the logic common between the client-side and + * server-side logic for describing the session's characteristics. + * + * This will create the string representing the "protocol" and "tls" nodes. The + * "tls" node will only be present if the connection is over SSL/TLS. + * + * @param[in] ssnp The pointer for this session. + * + * @return The description of the protocol stack and certain TLS attributes. + */ +std::string +get_protocol_description_helper(TSHttpSsn ssnp, bool is_client) +{ + std::ostringstream protocol_description; + protocol_description << R"("protocol":[)"; + + char const *protocol[10]; + int count = -1; + if (is_client) { + TSAssert(TS_SUCCESS == TSHttpSsnClientProtocolStackGet(ssnp, 10, protocol, &count)); + } else { + // See the TODO below in the commented out defintion of get_server_protocol_description. + // TSAssert(TS_SUCCESS == TSHttpSsnServerProtocolStackGet(ssnp, 10, protocol, &count)); + } + for (int i = 0; i < count; i++) { + if (i > 0) { + protocol_description << ","; + } + protocol_description << '"' << std::string(protocol[i]) << '"'; + } + + protocol_description << "]"; + std::string tls_description; + if (is_client) { + tls_description = get_client_tls_description(ssnp); + } else { + tls_description = get_server_tls_description(ssnp); + } + if (!tls_description.empty()) { + protocol_description << "," << tls_description; + } + return protocol_description.str(); +} + +#if 0 +// See the TODO above the get_server_protocol_description declaration. +// +// It will be important to add this eventually, but +// TSHttpSsnServerProtocolStackGet is not defined yet. Once it (or some other +// mechanism for getting the server side stack) is implemented, we will call +// this as a part of writing the server-response node. +std::string +SessionData::get_server_protocol_description(TSHttpSsn ssnp) +{ + return get_protocol_description_helper(ssnp, !IS_CLIENT); +} +#endif + +std::string +SessionData::get_client_protocol_description(TSHttpSsn ssnp) +{ + return get_protocol_description_helper(ssnp, IS_CLIENT); +} + +SessionData::SessionData() +{ + aio_cont = TSContCreate(session_aio_handler, TSMutexCreate()); + txn_cont = TSContCreate(TransactionData::global_transaction_handler, nullptr); +} + +SessionData::~SessionData() +{ + if (aio_cont) { + TSContDestroy(aio_cont); + } + if (txn_cont) { + TSContDestroy(txn_cont); + } +} + +/* + * Note this assumes that the caller holds the disk_io_mutex lock. This is a + * private member function. The two publically accessible functions hold the + * lock before calling this. + */ +int +SessionData::write_to_disk_no_lock(std::string_view content) +{ + char *pBuf = nullptr; + // Allocate a buffer for aio writing + if ((pBuf = static_cast(TSmalloc(sizeof(char) * content.size())))) { + memcpy(pBuf, content.data(), content.size()); + if (TS_SUCCESS == TSAIOWrite(log_fd, write_offset, pBuf, content.size(), aio_cont)) { + // Update offset within file and aio events count + write_offset += content.size(); + aio_count += 1; + + return TS_SUCCESS; + } + TSfree(pBuf); + } + return TS_ERROR; +} + +int +SessionData::write_to_disk(std::string_view content) +{ + const std::lock_guard _(disk_io_mutex); + const int result = write_to_disk_no_lock(content); + return result; +} + +int +SessionData::write_transaction_to_disk(std::string_view content) +{ + const std::lock_guard _(disk_io_mutex); + + int result = TS_SUCCESS; + if (has_written_first_transaction) { + // Prepend a comma. + std::string with_comma; + with_comma.reserve(content.size() + 1); + with_comma.insert(0, ","); + with_comma.insert(1, content); + result = write_to_disk_no_lock(with_comma); + } else { + result = write_to_disk_no_lock(content); + has_written_first_transaction = true; + } + + return result; +} + +int +SessionData::session_aio_handler(TSCont contp, TSEvent event, void *edata) +{ + switch (event) { + case TS_EVENT_AIO_DONE: { + TSAIOCallback cb = static_cast(edata); + SessionData *ssnData = static_cast(TSContDataGet(contp)); + if (!ssnData) { + TSDebug(debug_tag, "session_aio_handler(): No valid ssnData. Abort."); + return TS_ERROR; + } + char *buf = TSAIOBufGet(cb); + const std::lock_guard _(ssnData->disk_io_mutex); + + // Free the allocated buffer and update aio_count + if (buf) { + TSfree(buf); + if (--ssnData->aio_count == 0 && ssnData->ssn_closed) { + // check for ssn close, if closed, do clean up + TSContDataSet(contp, nullptr); + close(ssnData->log_fd); + std::error_code ec; + ts::file::file_status st = ts::file::status(ssnData->log_name, ec); + if (!ec) { + disk_usage += ts::file::file_size(st); + TSDebug(debug_tag, "Finish a session with log file of %" PRIuMAX "bytes", ts::file::file_size(st)); + } + delete ssnData; + return TS_SUCCESS; + } + } + return TS_SUCCESS; + } + default: + TSDebug(debug_tag, "session_aio_handler(): unhandled events %d", event); + return TS_ERROR; + } + return TS_SUCCESS; +} + +int +SessionData::global_session_handler(TSCont contp, TSEvent event, void *edata) +{ + TSHttpSsn ssnp = static_cast(edata); + + switch (event) { + case TS_EVENT_HTTP_SSN_START: { + // Grab session id for logging against a global value rather than the local + // session_counter. + int64_t id = TSHttpSsnIdGet(ssnp); + + // If the user has asked for SNI filtering, filter on that first because + // any sampling will apply just to that subset of connections that match + // that SNI. + if (!sni_filter.empty()) { + TSVConn ssn_vc = TSHttpSsnClientVConnGet(ssnp); + TSSslConnection ssl_conn = TSVConnSslConnectionGet(ssn_vc); + SSL *ssl_obj = reinterpret_cast(ssl_conn); + if (ssl_obj == nullptr) { + TSDebug(debug_tag, "global_session_handler(): Ignore non-HTTPS session %" PRId64 "...", id); + break; + } + char const *sni_ptr = SSL_get_servername(ssl_obj, TLSEXT_NAMETYPE_host_name); + if (sni_ptr == nullptr) { + TSDebug(debug_tag, "global_session_handler(): Ignore HTTPS session with non-existent SNI."); + break; + } else { + const std::string_view sni{sni_ptr}; + if (sni != sni_filter) { + TSDebug(debug_tag, "global_session_handler(): Ignore HTTPS session with non-filtered SNI: %s", sni_ptr); + break; + } + } + } + const auto this_session_count = session_counter++; + if (this_session_count % sample_pool_size != 0) { + TSDebug(debug_tag, "global_session_handler(): Ignore session %" PRId64 "...", id); + break; + } else if (disk_usage >= max_disk_usage) { + TSDebug(debug_tag, "global_session_handler(): Ignore session %" PRId64 "due to disk usage %" PRId64 "bytes", id, + disk_usage.load()); + break; + } + // Beginning of a new session + /// Get epoch time + auto start = std::chrono::duration_cast(std::chrono::system_clock::now().time_since_epoch()); + + // Create new per session data + SessionData *ssnData = new SessionData; + TSUserArgSet(ssnp, session_arg_index, ssnData); + + TSContDataSet(ssnData->aio_cont, ssnData); + + // "protocol":(string),"tls":(string) + // The "tls" node will only be present if the session is over SSL/TLS. + std::string protocol_description = get_client_protocol_description(ssnp); + + std::string beginning = R"({"meta":{"version":"1.0"},"sessions":[{)" + protocol_description + R"(,"connection-time":)" + + std::to_string(start.count()) + R"(,"transactions":[)"; + + // Use the session count's hex string as the filename. + std::stringstream stream; + stream << std::setw(16) << std::setfill('0') << std::hex << this_session_count; + std::string session_hex_name = stream.str(); + + // Use client ip as sub directory name + char client_str[INET6_ADDRSTRLEN]; + sockaddr const *client_ip = TSHttpSsnClientAddrGet(ssnp); + if (AF_INET == client_ip->sa_family) { + inet_ntop(AF_INET, &(reinterpret_cast(client_ip)->sin_addr), client_str, INET_ADDRSTRLEN); + } else if (AF_INET6 == client_ip->sa_family) { + inet_ntop(AF_INET6, &(reinterpret_cast(client_ip)->sin6_addr), client_str, INET6_ADDRSTRLEN); + } else { + TSDebug(debug_tag, "global_session_handler(): Unknown address family."); + snprintf(client_str, INET6_ADDRSTRLEN, "unknown"); + } + + // Initialize AIO file + const std::lock_guard _(ssnData->disk_io_mutex); + if (ssnData->log_fd < 0) { + ts::file::path log_p = log_directory / ts::file::path(std::string(client_str, 3)); + ts::file::path log_f = log_p / ts::file::path(session_hex_name); + + // Create subdir if not existing + std::error_code ec; + ts::file::status(log_p, ec); + if (ec && mkdir(log_p.c_str(), 0755) == -1) { + TSDebug(debug_tag, "global_session_handler(): Failed to create dir %s", log_p.c_str()); + TSError("[%s] Failed to create dir %s", debug_tag, log_p.c_str()); + } + + // Try to open log files for AIO + ssnData->log_fd = open(log_f.c_str(), O_RDWR | O_CREAT, S_IRWXU); + if (ssnData->log_fd < 0) { + TSDebug(debug_tag, "global_session_handler(): Failed to open log files %s. Abort.", log_f.c_str()); + TSHttpSsnReenable(ssnp, TS_EVENT_HTTP_CONTINUE); + return TS_EVENT_HTTP_CONTINUE; + } + ssnData->log_name = log_f; + // Write log file beginning to disk + ssnData->write_to_disk(beginning); + } + + TSHttpSsnHookAdd(ssnp, TS_HTTP_TXN_START_HOOK, ssnData->txn_cont); + TSHttpSsnHookAdd(ssnp, TS_HTTP_TXN_CLOSE_HOOK, ssnData->txn_cont); + break; + } + case TS_EVENT_HTTP_SSN_CLOSE: { + // Write session and close the log file. + int64_t id = TSHttpSsnIdGet(ssnp); + TSDebug(debug_tag, "global_session_handler(): Closing session %" PRId64 "...", id); + // Retrieve SessionData + SessionData *ssnData = static_cast(TSUserArgGet(ssnp, session_arg_index)); + // If no valid ssnData, continue transaction as if nothing happened + if (!ssnData) { + TSDebug(debug_tag, "global_session_handler(): [TS_EVENT_HTTP_SSN_CLOSE] No ssnData found. Abort."); + TSHttpSsnReenable(ssnp, TS_EVENT_HTTP_CONTINUE); + return TS_SUCCESS; + } + ssnData->write_to_disk(json_closing); + { + const std::lock_guard _(ssnData->disk_io_mutex); + ssnData->ssn_closed = true; + } + + break; + } + default: + break; + } + TSHttpSsnReenable(ssnp, TS_EVENT_HTTP_CONTINUE); + return TS_SUCCESS; +} + +} // namespace traffic_dump diff --git a/plugins/experimental/traffic_dump/session_data.h b/plugins/experimental/traffic_dump/session_data.h new file mode 100644 index 00000000000..7551501e933 --- /dev/null +++ b/plugins/experimental/traffic_dump/session_data.h @@ -0,0 +1,186 @@ +/** @session_handler.h + Traffic Dump session handling encapsulation. + @section license License + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#pragma once + +#include +#include +#include +#include + +#include "ts/ts.h" +#include "tscore/ts_file.h" + +namespace traffic_dump +{ +/** The information associated with an individual session. + * + * This class is responsible for containing the members associated with a + * particular session and defines the session handler callback. + */ +class SessionData +{ +public: + /// By default, Traffic Dump logs will go into a directory called "dump". + static char const constexpr *const default_log_directory = "dump"; + /// By default, 1 out of 1000 sessions will be dumped. + static constexpr int64_t default_sample_pool_size = 1000; + /// By default, logging will stop after 10 MB have been dumped. + static constexpr int64_t default_max_disk_usage = 10 * 1000 * 1000; + +private: + // + // Instance Variables + // + + /// Log file descriptor for this session's dump file. + int log_fd = -1; + /// The count of the currently outstanding AIO operations. + int aio_count = 0; + /// The offset of the last point written to so for in the dump file for this + /// session. + int64_t write_offset = 0; + /// Whether this session has been closed. + bool ssn_closed = false; + /// The filename for this session's dump file. + ts::file::path log_name; + /// Whether the first transaction in this session has been written. + bool has_written_first_transaction = false; + + TSCont aio_cont = nullptr; /// AIO continuation callback + TSCont txn_cont = nullptr; /// Transaction continuation callback + + // The following has to be recursive because the stack does not unwind + // between event invocations. + std::recursive_mutex disk_io_mutex; /// The mutex for guarding IO calls. + + // + // Static Variables + // + + // The index to be used for the TS API for storing this SessionData on a + // per-session basis. + static int session_arg_index; + + /// The rate at which to dump sessions. Every one out of sample_pool_size will + /// be dumped. + static std::atomic sample_pool_size; + /// The maximum space logs should take up before stopping the dumping of new + /// sessions. + static std::atomic max_disk_usage; + /// The amount of bytes currently written to dump files. + static std::atomic disk_usage; + + /// The directory into which to put the dump files. + static ts::file::path log_directory; + + /// Only sessions with this SNI will be dumped (if set). + static std::string sni_filter; + + /// The running counter of all sessions dumped by traffic_dump. + static uint64_t session_counter; + +public: + SessionData(); + ~SessionData(); + + /** The getter for the session_arg_index value. */ + static int get_session_arg_index(); + + /** Initialize the cross-session values of managing sessions. + * + * @return True if initialization is successful, false otherwise. + */ + static bool init(std::string_view log_directory, int64_t max_disk_usage, int64_t sample_size); + static bool init(std::string_view log_directory, int64_t max_disk_usage, int64_t sample_size, std::string_view sni_filter); + + /** Set the sample_pool_size to a new value. + * + * @param[in] new_sample_size The new value to set for sample_pool_size. + */ + static void set_sample_pool_size(int64_t new_sample_size); + + /** Reset the disk usage counter to 0. */ + static void reset_disk_usage(); + + /** Set the max_disk_usage to a new value. + * + * @param[in] new_max_disk_usage The new value to set for max_disk_usage. + */ + static void set_max_disk_usage(int64_t new_max_disk_usage); + +#if 0 + // TODO: This will eventually be used by TransactionData to dump + // the server protocol description in the "server-response" node, + // but the TS API does not yet support this. + + /** Get the JSON string that describes the server session stack. + * + * @param[in] ssnp The reference to the server session. + * + * @return A JSON description of the server protocol stack. + */ + static std::string get_server_protocol_description(TSHttpSsn ssnp); +#endif + + /** Write the string to the session's dump file. + * + * @param[in] content The content to write to the file. + * + * @return TS_SUCCESS if the write is successfully scheduled with the AIO + * system, TS_ERROR otherwise. + */ + int write_to_disk(std::string_view content); + + /** Write the transaction to the session's dump file. + * + * @param[in] content The transaction content to write to the file. + * + * @return TS_SUCCESS if the write is successfully scheduled with the AIO + * system, TS_ERROR otherwise. + */ + int write_transaction_to_disk(std::string_view content); + +private: + /** Write the string to the session's dump file. + * + * This assumes that the caller acquired the required AIO lock. + * + * @param[in] content The content to write to the file. + * + * @return TS_SUCCESS if the write is successfully scheduled with the AIO + * system, TS_ERROR otherwise. + */ + int write_to_disk_no_lock(std::string_view content); + + /** Get the JSON string that describes the client session stack. + * + * @param[in] ssnp The reference to the client session. + * + * @return A JSON description of the client protocol stack. + */ + static std::string get_client_protocol_description(TSHttpSsn ssnp); + + /** The handler callback for when async IO is done. Used for cleanup. */ + static int session_aio_handler(TSCont contp, TSEvent event, void *edata); + + /** The handler callback for session events. */ + static int global_session_handler(TSCont contp, TSEvent event, void *edata); +}; + +} // namespace traffic_dump diff --git a/plugins/experimental/traffic_dump/traffic_dump.cc b/plugins/experimental/traffic_dump/traffic_dump.cc index dc73e16cf02..271b3b8c58f 100644 --- a/plugins/experimental/traffic_dump/traffic_dump.cc +++ b/plugins/experimental/traffic_dump/traffic_dump.cc @@ -18,554 +18,108 @@ limitations under the License. */ -#include - -#include -#include -#include #include -#include - -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include - -#include "tscore/ts_file.h" -#include "ts/ts.h" - -namespace -{ -const char *PLUGIN_NAME = "traffic_dump"; -const std::string closing = "]}]}"; - -ts::file::path log_path{"dump"}; // default log directory -int s_arg_idx = 0; // Session Arg Index to pass on session data -std::atomic sample_pool_size(1000); // Sampling ratio -std::atomic max_disk_usage(10000000); //< Max disk space for logs (approximate) -std::atomic disk_usage(0); //< Actual disk usage -// handler declaration -int session_aio_handler(TSCont contp, TSEvent event, void *edata); -int session_txn_handler(TSCont contp, TSEvent event, void *edata); - -/// Custom structure for per session data -struct SsnData { - int log_fd = -1; //< Log file descriptor - int aio_count = 0; //< Active AIO counts - int64_t write_offset = 0; //< AIO write offset - bool first = true; //< First Transaction - bool ssn_closed = false; //< Session closed flag - ts::file::path log_name; //< Log file path - - TSCont aio_cont = nullptr; //< AIO callback - TSCont txn_cont = nullptr; //< Transaction callback - TSMutex disk_io_mutex = nullptr; //< AIO mutex - - SsnData() - { - disk_io_mutex = TSMutexCreate(); - aio_cont = TSContCreate(session_aio_handler, TSMutexCreate()); - txn_cont = TSContCreate(session_txn_handler, nullptr); - } - - ~SsnData() - { - if (disk_io_mutex) { - TSMutexDestroy(disk_io_mutex); - } - if (aio_cont) { - TSContDestroy(aio_cont); - } - if (txn_cont) { - TSContDestroy(txn_cont); - } - } - - /// write_to_disk(): Takes a string object and writes to file via AIO - int - write_to_disk(const std::string &body) - { - TSMutexLock(disk_io_mutex); - char *pBuf = nullptr; - // Allocate a buffer for aio writing - if ((pBuf = static_cast(TSmalloc(sizeof(char) * body.size())))) { - memcpy(pBuf, body.c_str(), body.size()); - if (TS_SUCCESS == TSAIOWrite(log_fd, write_offset, pBuf, body.size(), aio_cont)) { - // Update offset within file and aio events count - write_offset += body.size(); - aio_count += 1; - - TSMutexUnlock(disk_io_mutex); - return TS_SUCCESS; - } - TSfree(pBuf); - } - TSMutexUnlock(disk_io_mutex); - return TS_ERROR; - } -}; - -/// Local helper functions about json formatting -/// min_write(): Inline function for repeating code -inline void -min_write(const char *buf, int64_t &prevIdx, int64_t &idx, std::ostream &jsonfile) -{ - if (prevIdx < idx) { - jsonfile.write(buf + prevIdx, idx - prevIdx); - } - prevIdx = idx + 1; -} - -/// esc_json_out(): Escape characters in a buffer and output to ofstream object -/// in a way to minimize ofstream operations -int -esc_json_out(const char *buf, int64_t len, std::ostream &jsonfile) -{ - if (buf == nullptr) { - return 0; - } - int64_t idx, prevIdx = 0; - for (idx = 0; idx < len; idx++) { - char c = *(buf + idx); - switch (c) { - case '"': - case '\\': { - min_write(buf, prevIdx, idx, jsonfile); - jsonfile << "\\" << c; - break; - } - case '\b': { - min_write(buf, prevIdx, idx, jsonfile); - jsonfile << "\\b"; - break; - } - case '\f': { - min_write(buf, prevIdx, idx, jsonfile); - jsonfile << "\\f"; - break; - } - case '\n': { - min_write(buf, prevIdx, idx, jsonfile); - jsonfile << "\\n"; - break; - } - case '\r': { - min_write(buf, prevIdx, idx, jsonfile); - jsonfile << "\\r"; - break; - } - case '\t': { - min_write(buf, prevIdx, idx, jsonfile); - jsonfile << "\\t"; - break; - } - default: { - if ('\x00' <= c && c <= '\x1f') { - min_write(buf, prevIdx, idx, jsonfile); - jsonfile << "\\u" << std::hex << std::setw(4) << std::setfill('0') << static_cast(c); - } - break; - } - } - } - min_write(buf, prevIdx, idx, jsonfile); - - return len; -} - -/// escape_json(): escape chars in a string and returns json string -std::string -escape_json(std::string const &s) -{ - std::ostringstream o; - esc_json_out(s.c_str(), s.length(), o); - return o.str(); -} -std::string -escape_json(const char *buf, int64_t size) -{ - std::ostringstream o; - esc_json_out(buf, size, o); - return o.str(); -} - -inline std::string -json_entry(std::string const &name, const char *buf, int64_t size) -{ - return "\"" + escape_json(name) + "\":\"" + escape_json(buf, size) + "\""; -} - -/// json_entry_array(): Formats to array-style entry i.e. ["field","value"] -inline std::string -json_entry_array(const char *name, int name_len, const char *value, int value_len) -{ - return "[\"" + escape_json(name, name_len) + "\", \"" + escape_json(value, value_len) + "\"]"; -} - -/// Helper functions to collect txn information from TSMBuffer -std::string -collect_headers(TSMBuffer &buffer, TSMLoc &hdr_loc, int64_t body_bytes) -{ - std::string result = "{"; - int len = 0; - const char *cp = nullptr; - TSMLoc url_loc = nullptr; - - // Log scheme+method or status+reason based on header type - if (TSHttpHdrTypeGet(buffer, hdr_loc) == TS_HTTP_TYPE_REQUEST) { - // 1. "version" - int version = TSHttpHdrVersionGet(buffer, hdr_loc); - result += R"("version":")" + std::to_string(TS_HTTP_MAJOR(version)) + "." + std::to_string(TS_HTTP_MINOR(version)) + '"'; - - // 2. "scheme": - TSAssert(TS_SUCCESS == TSHttpHdrUrlGet(buffer, hdr_loc, &url_loc)); - cp = TSUrlSchemeGet(buffer, url_loc, &len); - TSDebug(PLUGIN_NAME, "collect_headers(): found scheme %d ", len); - result += "," + json_entry("scheme", cp, len); - - // 3. "method":(string) - cp = TSHttpHdrMethodGet(buffer, hdr_loc, &len); - result += "," + json_entry("method", cp, len); - - // 4. "url" - cp = TSUrlStringGet(buffer, url_loc, &len); - TSDebug(PLUGIN_NAME, "collect_headers(): found url %.*s", len, cp); - result += "," + json_entry("url", cp, len); - TSHandleMLocRelease(buffer, hdr_loc, url_loc); - } else { - // 1. "status":(string) - result += R"("status":)" + std::to_string(TSHttpHdrStatusGet(buffer, hdr_loc)); - // 2. "reason":(string) - cp = TSHttpHdrReasonGet(buffer, hdr_loc, &len); - result += "," + json_entry("reason", cp, len); - // 3. "encoding" - } - - // "content" - // "encoding" - // "size" - result += R"(,"content":{"encoding":"plain","size":)" + std::to_string(body_bytes) + '}'; - - // "headers": [[name(string), value(string)]] - result += R"(,"headers":{"encoding":"esc_json", "fields": [)"; - TSMLoc field_loc = TSMimeHdrFieldGet(buffer, hdr_loc, 0); - while (field_loc) { - TSMLoc next_field_loc; - const char *name; - const char *value; - int name_len, value_len; - // Append to "fields" list if valid value exists - if ((name = TSMimeHdrFieldNameGet(buffer, hdr_loc, field_loc, &name_len)) && name_len) { - value = TSMimeHdrFieldValueStringGet(buffer, hdr_loc, field_loc, -1, &value_len); - result += json_entry_array(name, name_len, value, value_len); - } - next_field_loc = TSMimeHdrFieldNext(buffer, hdr_loc, field_loc); - TSHandleMLocRelease(buffer, hdr_loc, field_loc); - if ((field_loc = next_field_loc) != nullptr) { - result += ","; - } - } +#include "global_variables.h" +#include "session_data.h" +#include "transaction_data.h" - return result + "]}}"; -} - -// Per session AIO handler: update AIO counts and clean up -int -session_aio_handler(TSCont contp, TSEvent event, void *edata) +namespace traffic_dump { - switch (event) { - case TS_EVENT_AIO_DONE: { - TSAIOCallback cb = static_cast(edata); - SsnData *ssnData = static_cast(TSContDataGet(contp)); - if (!ssnData) { - TSDebug(PLUGIN_NAME, "session_aio_handler(): No valid ssnData. Abort."); - return TS_ERROR; - } - char *buf = TSAIOBufGet(cb); - TSMutexLock(ssnData->disk_io_mutex); - - // Free the allocated buffer and update aio_count - if (buf) { - TSfree(buf); - if (--ssnData->aio_count == 0 && ssnData->ssn_closed) { - // check for ssn close, if closed, do clean up - TSContDataSet(contp, nullptr); - close(ssnData->log_fd); - TSMutexUnlock(ssnData->disk_io_mutex); - std::error_code ec; - ts::file::file_status st = ts::file::status(ssnData->log_name, ec); - if (!ec) { - disk_usage += ts::file::file_size(st); - TSDebug(PLUGIN_NAME, "Finish a session with log file of %" PRIuMAX "bytes", ts::file::file_size(st)); - } - delete ssnData; - return TS_SUCCESS; - } - } - TSMutexUnlock(ssnData->disk_io_mutex); - return TS_SUCCESS; - } - default: - TSDebug(PLUGIN_NAME, "session_aio_handler(): unhandled events %d", event); - return TS_ERROR; - } - return TS_SUCCESS; -} - -// Transaction handler: writes headers to the log file using AIO -int -session_txn_handler(TSCont contp, TSEvent event, void *edata) -{ - TSHttpTxn txnp = static_cast(edata); - - // Retrieve SsnData - TSHttpSsn ssnp = TSHttpTxnSsnGet(txnp); - SsnData *ssnData = static_cast(TSHttpSsnArgGet(ssnp, s_arg_idx)); - - // If no valid ssnData, continue transaction as if nothing happened - if (!ssnData) { - TSDebug(PLUGIN_NAME, "session_txn_handler(): No ssnData found. Abort."); - TSHttpTxnReenable(txnp, TS_EVENT_HTTP_CONTINUE); - return TS_SUCCESS; - } - - switch (event) { - case TS_EVENT_HTTP_TXN_CLOSE: { - // Get UUID - char uuid[TS_CRUUID_STRING_LEN + 1]; - TSAssert(TS_SUCCESS == TSClientRequestUuidGet(txnp, uuid)); - - // Generate per transaction json records - std::string txn_info; - if (!ssnData->first) { - txn_info += ","; - } - ssnData->first = false; - - // "uuid":(string) - txn_info += "{" + json_entry("uuid", uuid, strlen(uuid)); - - // "connect-time":(number) - TSHRTime start_time; - TSHttpTxnMilestoneGet(txnp, TS_MILESTONE_UA_BEGIN, &start_time); - txn_info += ",\"start-time\":" + std::to_string(start_time); - - // client/proxy-request/response headers - TSMBuffer buffer; - TSMLoc hdr_loc; - if (TS_SUCCESS == TSHttpTxnClientReqGet(txnp, &buffer, &hdr_loc)) { - TSDebug(PLUGIN_NAME, "Found client request"); - txn_info += R"(,"client-request":)" + collect_headers(buffer, hdr_loc, TSHttpTxnClientReqBodyBytesGet(txnp)); - TSHandleMLocRelease(buffer, TS_NULL_MLOC, hdr_loc); - buffer = nullptr; - } - if (TS_SUCCESS == TSHttpTxnServerReqGet(txnp, &buffer, &hdr_loc)) { - TSDebug(PLUGIN_NAME, "Found proxy request"); - txn_info += R"(,"proxy-request":)" + collect_headers(buffer, hdr_loc, TSHttpTxnServerReqBodyBytesGet(txnp)); - TSHandleMLocRelease(buffer, TS_NULL_MLOC, hdr_loc); - buffer = nullptr; - } - if (TS_SUCCESS == TSHttpTxnServerRespGet(txnp, &buffer, &hdr_loc)) { - TSDebug(PLUGIN_NAME, "Found server response"); - txn_info += R"(,"server-response":)" + collect_headers(buffer, hdr_loc, TSHttpTxnServerRespBodyBytesGet(txnp)); - TSHandleMLocRelease(buffer, TS_NULL_MLOC, hdr_loc); - buffer = nullptr; - } - if (TS_SUCCESS == TSHttpTxnClientRespGet(txnp, &buffer, &hdr_loc)) { - TSDebug(PLUGIN_NAME, "Found proxy response"); - txn_info += R"(,"proxy-response":)" + collect_headers(buffer, hdr_loc, TSHttpTxnClientRespBodyBytesGet(txnp)); - TSHandleMLocRelease(buffer, TS_NULL_MLOC, hdr_loc); - buffer = nullptr; - } - - txn_info += "}"; - ssnData->write_to_disk(txn_info); - break; - } - default: - TSDebug(PLUGIN_NAME, "session_txn_handler(): Unhandled events %d", event); - TSHttpTxnReenable(txnp, TS_EVENT_HTTP_ERROR); - return TS_ERROR; - } - - TSHttpTxnReenable(txnp, TS_EVENT_HTTP_CONTINUE); - return TS_SUCCESS; -} - -// Session handler for global hooks; Assign per-session data structure and log files +/// Handle LIFECYCLE_MSG from traffic_ctl. static int -global_ssn_handler(TSCont contp, TSEvent event, void *edata) +global_message_handler(TSCont contp, TSEvent event, void *edata) { - TSHttpSsn ssnp = static_cast(edata); - switch (event) { - // Also handles LIFECYCLE_MSG from traffic_ctl case TS_EVENT_LIFECYCLE_MSG: { TSPluginMsg *msg = static_cast(edata); - // String view of plugin message prefix static constexpr std::string_view PLUGIN_PREFIX("traffic_dump."_sv); std::string_view tag(msg->tag, strlen(msg->tag)); - if (tag.substr(0, PLUGIN_PREFIX.size()) == PLUGIN_PREFIX) { tag.remove_prefix(PLUGIN_PREFIX.size()); if (tag == "sample") { - sample_pool_size = static_cast(strtol(static_cast(msg->data), nullptr, 0)); - TSDebug(PLUGIN_NAME, "TS_EVENT_LIFECYCLE_MSG: Received Msg to change sample size to %" PRId64 "bytes", - sample_pool_size.load()); + const auto new_sample_size = static_cast(strtol(static_cast(msg->data), nullptr, 0)); + TSDebug(debug_tag, "TS_EVENT_LIFECYCLE_MSG: Received Msg to change sample size to %" PRId64 "bytes", new_sample_size); + SessionData::set_sample_pool_size(new_sample_size); } else if (tag == "reset") { - disk_usage = 0; - TSDebug(PLUGIN_NAME, "TS_EVENT_LIFECYCLE_MSG: Received Msg to reset disk usage counter"); + TSDebug(debug_tag, "TS_EVENT_LIFECYCLE_MSG: Received Msg to reset disk usage counter"); + SessionData::reset_disk_usage(); } else if (tag == "limit") { - max_disk_usage = static_cast(strtol(static_cast(msg->data), nullptr, 0)); - TSDebug(PLUGIN_NAME, "TS_EVENT_LIFECYCLE_MSG: Received Msg to change max disk usage to %" PRId64 "bytes", - max_disk_usage.load()); + const auto new_max_disk_usage = static_cast(strtol(static_cast(msg->data), nullptr, 0)); + TSDebug(debug_tag, "TS_EVENT_LIFECYCLE_MSG: Received Msg to change max disk usage to %" PRId64 "bytes", new_max_disk_usage); + SessionData::set_max_disk_usage(new_max_disk_usage); } } return TS_SUCCESS; } - case TS_EVENT_HTTP_SSN_START: { - // Grab session id to do sampling - int64_t id = TSHttpSsnIdGet(ssnp); - if (id % sample_pool_size != 0) { - TSDebug(PLUGIN_NAME, "global_ssn_handler(): Ignore session %" PRId64 "...", id); - break; - } else if (disk_usage >= max_disk_usage) { - TSDebug(PLUGIN_NAME, "global_ssn_handler(): Ignore session %" PRId64 "due to disk usage %" PRId64 "bytes", id, - disk_usage.load()); - break; - } - // Beginning of a new session - /// Get epoch time - auto start = std::chrono::duration_cast(std::chrono::system_clock::now().time_since_epoch()); - - // Create new per session data - SsnData *ssnData = new SsnData; - TSHttpSsnArgSet(ssnp, s_arg_idx, ssnData); - - TSContDataSet(ssnData->aio_cont, ssnData); - - // 1. "protocol":(string) - const char *protocol[10]; - int count = 0; - TSAssert(TS_SUCCESS == TSHttpSsnClientProtocolStackGet(ssnp, 10, protocol, &count)); - std::string result; - for (int i = 0; i < count; i++) { - if (i > 0) { - result += ","; - } - result += '"' + std::string(protocol[i]) + '"'; - } - - std::string beginning = R"({"meta":{"version":"1.0"},"sessions":[{"protocol":[)" + result + "]" + R"(,"connection-time":)" + - std::to_string(start.count()) + R"(,"transactions":[)"; - - // Grab session id and use its hex string as fname - std::stringstream stream; - stream << std::setw(16) << std::setfill('0') << std::hex << id; - std::string session_id = stream.str(); - - // Use client ip as sub directory name - char client_str[INET6_ADDRSTRLEN]; - const sockaddr *client_ip = TSHttpSsnClientAddrGet(ssnp); - if (AF_INET == client_ip->sa_family) { - inet_ntop(AF_INET, &(reinterpret_cast(client_ip)->sin_addr), client_str, INET_ADDRSTRLEN); - } else if (AF_INET6 == client_ip->sa_family) { - inet_ntop(AF_INET6, &(reinterpret_cast(client_ip)->sin6_addr), client_str, INET6_ADDRSTRLEN); - } else { - TSDebug(PLUGIN_NAME, "global_ssn_handler(): Unknown address family."); - snprintf(client_str, INET6_ADDRSTRLEN, "unknown"); - } - - // Initialize AIO file - TSMutexLock(ssnData->disk_io_mutex); - if (ssnData->log_fd < 0) { - ts::file::path log_p = log_path / ts::file::path(std::string(client_str, 3)); - ts::file::path log_f = log_p / ts::file::path(session_id); - - // Create subdir if not existing - std::error_code ec; - ts::file::status(log_p, ec); - if (ec && mkdir(log_p.c_str(), 0755) == -1) { - TSDebug(PLUGIN_NAME, "global_ssn_handler(): Failed to create dir %s", log_p.c_str()); - TSError("[%s] Failed to create dir %s", PLUGIN_NAME, log_p.c_str()); - } - - // Try to open log files for AIO - ssnData->log_fd = open(log_f.c_str(), O_RDWR | O_CREAT, S_IRWXU); - if (ssnData->log_fd < 0) { - TSMutexUnlock(ssnData->disk_io_mutex); - TSDebug(PLUGIN_NAME, "global_ssn_handler(): Failed to open log files %s. Abort.", log_f.c_str()); - TSHttpSsnReenable(ssnp, TS_EVENT_HTTP_CONTINUE); - return TS_EVENT_HTTP_CONTINUE; - } - ssnData->log_name = log_f; - // Write log file beginning to disk - ssnData->write_to_disk(beginning); - } - TSMutexUnlock(ssnData->disk_io_mutex); - - TSHttpSsnHookAdd(ssnp, TS_HTTP_TXN_CLOSE_HOOK, ssnData->txn_cont); - break; - } - case TS_EVENT_HTTP_SSN_CLOSE: { - // Write session and log file closing - int64_t id = TSHttpSsnIdGet(ssnp); - TSDebug(PLUGIN_NAME, "global_ssn_handler(): Closing session %" PRId64 "...", id); - // Retrieve SsnData - SsnData *ssnData = static_cast(TSHttpSsnArgGet(ssnp, s_arg_idx)); - // If no valid ssnData, continue transaction as if nothing happened - if (!ssnData) { - TSDebug(PLUGIN_NAME, "global_ssn_handler(): [TS_EVENT_HTTP_SSN_CLOSE] No ssnData found. Abort."); - TSHttpSsnReenable(ssnp, TS_EVENT_HTTP_CONTINUE); - return TS_SUCCESS; - } - ssnData->write_to_disk(closing); - TSMutexLock(ssnData->disk_io_mutex); - ssnData->ssn_closed = true; - TSMutexUnlock(ssnData->disk_io_mutex); - - break; - } default: - break; + TSDebug(debug_tag, "session_aio_handler(): unhandled events %d", event); + return TS_ERROR; } - TSHttpSsnReenable(ssnp, TS_EVENT_HTTP_CONTINUE); - return TS_SUCCESS; } -} // End of anonymous namespace +} // namespace traffic_dump void -TSPluginInit(int argc, const char *argv[]) +TSPluginInit(int argc, char const *argv[]) { - TSDebug(PLUGIN_NAME, "initializing plugin"); + TSDebug(traffic_dump::debug_tag, "initializing plugin"); TSPluginRegistrationInfo info; info.plugin_name = "traffic_dump"; info.vendor_name = "Apache Software Foundation"; info.support_email = "dev@trafficserver.apache.org"; + if (TS_SUCCESS != TSPluginRegister(&info)) { + TSError("[%s] Unable to initialize plugin (disabled). Failed to register plugin.", traffic_dump::debug_tag); + return; + } + + bool sensitive_fields_were_specified = false; + traffic_dump::sensitive_fields_t user_specified_fields; + ts::file::path log_dir{traffic_dump::SessionData::default_log_directory}; + int64_t sample_pool_size = traffic_dump::SessionData::default_sample_pool_size; + int64_t max_disk_usage = traffic_dump::SessionData::default_max_disk_usage; + std::string sni_filter; + /// Commandline options - static const struct option longopts[] = {{"logdir", required_argument, nullptr, 'l'}, - {"sample", required_argument, nullptr, 's'}, - {"limit", required_argument, nullptr, 'm'}, - {nullptr, no_argument, nullptr, 0}}; - int opt = 0; + static const struct option longopts[] = { + {"logdir", required_argument, nullptr, 'l'}, {"sample", required_argument, nullptr, 's'}, + {"limit", required_argument, nullptr, 'm'}, {"sensitive-fields", required_argument, nullptr, 'f'}, + {"sni-filter", required_argument, nullptr, 'n'}, {nullptr, no_argument, nullptr, 0}}; + int opt = 0; while (opt >= 0) { opt = getopt_long(argc, const_cast(argv), "l:", longopts, nullptr); switch (opt) { + case 'f': { + // --sensitive-fields takes a comma-separated list of HTTP fields that + // are sensitive. The field values for these fields will be replaced + // with generic traffic_dump generated data. + // + // If this option is not used, then the default values in + // default_sensitive_fields is used. If this option is used, then it + // replaced the default sensitive fields with the user-supplied list of + // sensitive fields. + sensitive_fields_were_specified = true; + ts::TextView input_filter_fields{std::string_view{optarg}}; + ts::TextView filter_field; + while (!(filter_field = input_filter_fields.take_prefix_at(',')).empty()) { + filter_field.trim_if(&isspace); + if (filter_field.empty()) { + continue; + } + user_specified_fields.emplace(filter_field); + } + break; + } + case 'n': { + // --sni-filter is used to filter sessions based upon an SNI. + sni_filter = std::string(optarg); + break; + } case 'l': { - log_path = ts::file::path{optarg}; + log_dir = ts::file::path{optarg}; break; } case 's': { @@ -580,31 +134,41 @@ TSPluginInit(int argc, const char *argv[]) break; default: - TSDebug(PLUGIN_NAME, "Unexpected options."); - TSError("[%s] Unexpected options error.", PLUGIN_NAME); + TSDebug(traffic_dump::debug_tag, "Unexpected options."); + TSError("[%s] Unexpected options error.", traffic_dump::debug_tag); return; } } - - // Make absolute path if not - if (!log_path.is_absolute()) { - log_path = ts::file::path(TSInstallDirGet()) / log_path; + if (!log_dir.is_absolute()) { + log_dir = ts::file::path(TSInstallDirGet()) / log_dir; + } + if (sni_filter.empty()) { + if (!traffic_dump::SessionData::init(log_dir.view(), max_disk_usage, sample_pool_size)) { + TSError("[%s] Failed to initialize session state.", traffic_dump::debug_tag); + return; + } + } else { + if (!traffic_dump::SessionData::init(log_dir.view(), max_disk_usage, sample_pool_size, sni_filter)) { + TSError("[%s] Failed to initialize session state with an SNI filter.", traffic_dump::debug_tag); + return; + } } - TSDebug(PLUGIN_NAME, "Initialized with log directory: %s", log_path.c_str()); - if (TS_SUCCESS != TSPluginRegister(&info)) { - TSError("[%s] Unable to initialize plugin (disabled). Failed to register plugin.", PLUGIN_NAME); - } else if (TS_SUCCESS != TSHttpSsnArgIndexReserve(PLUGIN_NAME, "Track log related data", &s_arg_idx)) { - TSError("[%s] Unable to initialize plugin (disabled). Failed to reserve ssn arg.", PLUGIN_NAME); + if (sensitive_fields_were_specified) { + if (!traffic_dump::TransactionData::init(std::move(user_specified_fields))) { + TSError("[%s] Failed to initialize transaction state with user-specified fields.", traffic_dump::debug_tag); + return; + } } else { - /// Add global hooks - TSCont ssncont = TSContCreate(global_ssn_handler, nullptr); - TSHttpHookAdd(TS_HTTP_SSN_START_HOOK, ssncont); - TSHttpHookAdd(TS_HTTP_SSN_CLOSE_HOOK, ssncont); - TSLifecycleHookAdd(TS_LIFECYCLE_MSG_HOOK, ssncont); - TSDebug(PLUGIN_NAME, "Initialized with sample pool size %" PRId64 " bytes and disk limit %" PRId64 "bytes", - sample_pool_size.load(), max_disk_usage.load()); + // The user did not provide their own list of sensitive fields. Use the + // default. + if (!traffic_dump::TransactionData::init()) { + TSError("[%s] Failed to initialize transaction state.", traffic_dump::debug_tag); + return; + } } + TSCont message_continuation = TSContCreate(traffic_dump::global_message_handler, nullptr); + TSLifecycleHookAdd(TS_LIFECYCLE_MSG_HOOK, message_continuation); return; } diff --git a/plugins/experimental/traffic_dump/transaction_data.cc b/plugins/experimental/traffic_dump/transaction_data.cc new file mode 100644 index 00000000000..ec16fffee3e --- /dev/null +++ b/plugins/experimental/traffic_dump/transaction_data.cc @@ -0,0 +1,359 @@ +/** @txn_data.cc + Implementation of Traffic Dump transaction data. + @section license License + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#include "transaction_data.h" +#include "global_variables.h" +#include "json_utils.h" +#include "session_data.h" + +namespace traffic_dump +{ +int TransactionData::transaction_arg_index = 0; +sensitive_fields_t TransactionData::sensitive_fields; + +std::string defaut_sensitive_field_value; + +/// Fields considered sensitive because they may contain user-private +/// information. These fields are replaced with auto-generated generic content by +/// default. To override this behavior, the user should specify their own fields +/// they consider sensitive with --sensitive-fields. +/// +/// While these are specified with case, they are matched case-insensitively. +sensitive_fields_t default_sensitive_fields = { + "Set-Cookie", + "Cookie", +}; + +void +TransactionData::initialize_default_sensitive_field() +{ + // 128 KB is the maximum size supported for all headers, so this size should + // be plenty large for our needs. + constexpr size_t default_field_size = 128 * 1024; + defaut_sensitive_field_value.resize(default_field_size); + + char *field_buffer = defaut_sensitive_field_value.data(); + for (auto i = 0u; i < default_field_size; i += 8) { + sprintf(field_buffer, "%07x ", i / 8); + field_buffer += 8; + } +} + +/// The set of fields, default and user-specified, that are sensitive and whose +/// values will be replaced with auto-generated generic content. +sensitive_fields_t sensitive_fields; + +std::string +TransactionData::get_sensitive_field_description() +{ + std::string sensitive_fields_string; + bool is_first = true; + for (const auto &field : sensitive_fields) { + if (!is_first) { + sensitive_fields_string += ", "; + } + is_first = false; + sensitive_fields_string += field; + } + return sensitive_fields_string; +} + +bool +TransactionData::init(sensitive_fields_t &&new_fields) +{ + sensitive_fields = std::move(new_fields); + return init_helper(); +} + +bool +TransactionData::init() +{ + sensitive_fields = default_sensitive_fields; + return init_helper(); +} + +std::string_view +TransactionData::replace_sensitive_fields(std::string_view name, std::string_view original_value) +{ + auto search = sensitive_fields.find(std::string(name)); + if (search == sensitive_fields.end()) { + return original_value; + } + auto new_value_size = original_value.size(); + if (original_value.size() > defaut_sensitive_field_value.size()) { + new_value_size = defaut_sensitive_field_value.size(); + TSError("[%s] Encountered a sensitive field value larger than our default " + "field size. Default size: %zu, incoming field size: %zu", + debug_tag, defaut_sensitive_field_value.size(), original_value.size()); + } + return std::string_view{defaut_sensitive_field_value.data(), new_value_size}; +} + +std::string +TransactionData::write_content_node(int64_t num_body_bytes) +{ + return std::string(R"(,"content":{"encoding":"plain","size":)" + std::to_string(num_body_bytes) + '}'); +} + +std::string +TransactionData::write_message_node_no_content(TSMBuffer &buffer, TSMLoc &hdr_loc) +{ + std::string result = "{"; + int len = 0; + char const *cp = nullptr; + TSMLoc url_loc = nullptr; + + // 1. "version" + // Note that we print this for both requests and responses, so the first + // element in each has to start with a comma. + int version = TSHttpHdrVersionGet(buffer, hdr_loc); + result += R"("version":")" + std::to_string(TS_HTTP_MAJOR(version)) + "." + std::to_string(TS_HTTP_MINOR(version)) + '"'; + + // Log scheme+method+request-target or status+reason based on header type + if (TSHttpHdrTypeGet(buffer, hdr_loc) == TS_HTTP_TYPE_REQUEST) { + TSAssert(TS_SUCCESS == TSHttpHdrUrlGet(buffer, hdr_loc, &url_loc)); + // 2. "scheme": + cp = TSUrlSchemeGet(buffer, url_loc, &len); + TSDebug(debug_tag, "write_message_node(): found scheme %.*s ", len, cp); + result += "," + traffic_dump::json_entry("scheme", cp, len); + + // 3. "method":(string) + cp = TSHttpHdrMethodGet(buffer, hdr_loc, &len); + TSDebug(debug_tag, "write_message_node(): found method %.*s ", len, cp); + result += "," + traffic_dump::json_entry("method", cp, len); + + // 4. "url" + cp = TSUrlHostGet(buffer, url_loc, &len); + std::string_view host{cp, static_cast(len)}; + + char *url = TSUrlStringGet(buffer, url_loc, &len); + std::string_view url_string{url, static_cast(len)}; + + if (host.empty()) { + // TSUrlStringGet will add the scheme to the URL, even if the request + // target doesn't contain it. However, we cannot just always remove the + // scheme because the original request target may include it. We assume + // here that a URL with a scheme but not a host is artificial and thus + // we remove it. + url_string = remove_scheme_prefix(url_string); + } + + TSDebug(debug_tag, "write_message_node(): found host target %.*s", static_cast(url_string.size()), url_string.data()); + result += "," + traffic_dump::json_entry("url", url_string); + TSfree(url); + TSHandleMLocRelease(buffer, hdr_loc, url_loc); + } else { + // 2. "status":(string) + result += R"(,"status":)" + std::to_string(TSHttpHdrStatusGet(buffer, hdr_loc)); + // 3. "reason":(string) + cp = TSHttpHdrReasonGet(buffer, hdr_loc, &len); + result += "," + traffic_dump::json_entry("reason", cp, len); + } + + // "headers": [[name(string), value(string)]] + result += R"(,"headers":{"encoding":"esc_json", "fields": [)"; + TSMLoc field_loc = TSMimeHdrFieldGet(buffer, hdr_loc, 0); + while (field_loc) { + TSMLoc next_field_loc; + char const *name = nullptr; + char const *value = nullptr; + int name_len = 0, value_len = 0; + // Append to "fields" list if valid value exists + if ((name = TSMimeHdrFieldNameGet(buffer, hdr_loc, field_loc, &name_len)) && name_len) { + std::string_view name_view{name, static_cast(name_len)}; + value = TSMimeHdrFieldValueStringGet(buffer, hdr_loc, field_loc, -1, &value_len); + std::string_view value_view{value, static_cast(value_len)}; + std::string_view new_value = replace_sensitive_fields(name_view, value_view); + result += traffic_dump::json_entry_array(name_view, new_value); + } + + next_field_loc = TSMimeHdrFieldNext(buffer, hdr_loc, field_loc); + TSHandleMLocRelease(buffer, hdr_loc, field_loc); + if ((field_loc = next_field_loc) != nullptr) { + result += ","; + } + } + return result += "]}"; +} + +std::string +TransactionData::write_message_node(TSMBuffer &buffer, TSMLoc &hdr_loc, int64_t num_body_bytes) +{ + std::string result = write_message_node_no_content(buffer, hdr_loc); + result += write_content_node(num_body_bytes); + return result + "}"; +} + +std::string_view +TransactionData::remove_scheme_prefix(std::string_view url) +{ + const auto scheme_separator = url.find("://"); + if (scheme_separator == std::string::npos) { + return url; + } + url.remove_prefix(scheme_separator + 3); + return url; +} + +bool +TransactionData::init_helper() +{ + initialize_default_sensitive_field(); + const std::string sensitive_fields_string = get_sensitive_field_description(); + TSDebug(debug_tag, "Sensitive fields for which generic values will be dumped: %s", sensitive_fields_string.c_str()); + + if (TS_SUCCESS != + TSUserArgIndexReserve(TS_USER_ARGS_TXN, traffic_dump::debug_tag, "Track transaction related data", &transaction_arg_index)) { + TSError("[%s] Unable to initialize plugin (disabled). Failed to reserve transaction arg.", traffic_dump::debug_tag); + return false; + } + + // Register the collecting of client-request headers at the global level so + // we can process requests before other plugins. (Global hooks are + // processed before session and transaction ones.) + TSCont txn_cont = TSContCreate(global_transaction_handler, nullptr); + TSHttpHookAdd(TS_HTTP_READ_REQUEST_HDR_HOOK, txn_cont); + return true; +} + +// Transaction handler: writes headers to the log file using AIO +int +TransactionData::global_transaction_handler(TSCont contp, TSEvent event, void *edata) +{ + TSHttpTxn txnp = static_cast(edata); + + // Retrieve SessionData + TSHttpSsn ssnp = TSHttpTxnSsnGet(txnp); + SessionData *ssnData = static_cast(TSUserArgGet(ssnp, SessionData::get_session_arg_index())); + + // If no valid ssnData, continue transaction as if nothing happened. This transaction + // must be filtered out by our filter criteria. + if (!ssnData) { + TSDebug(debug_tag, "session_txn_handler(): No ssnData found. Abort."); + TSHttpTxnReenable(txnp, TS_EVENT_HTTP_CONTINUE); + return TS_SUCCESS; + } + + switch (event) { + case TS_EVENT_HTTP_TXN_START: { + // We will piece together JSON content accumulated across several hooks of + // the transaction. The catch is that hooks across transactions in a + // session may fire interleaved in HTTP/2. Thus, in order to get + // non-garbled JSON content, we accumulate the data for an entire + // transaction and write that atomically once the transaction is completed. + TransactionData *txnData = new TransactionData; + TSUserArgSet(txnp, transaction_arg_index, txnData); + // Get UUID + char uuid[TS_CRUUID_STRING_LEN + 1]; + TSAssert(TS_SUCCESS == TSClientRequestUuidGet(txnp, uuid)); + std::string_view uuid_view{uuid, strnlen(uuid, TS_CRUUID_STRING_LEN)}; + + // Generate per transaction json records + txnData->txn_json += "{"; + // "connection-time":(number) + TSHRTime start_time; + TSHttpTxnMilestoneGet(txnp, TS_MILESTONE_UA_BEGIN, &start_time); + txnData->txn_json += "\"connection-time\":" + std::to_string(start_time); + + // "uuid":(string) + // The uuid is a header field for each message in the transaction. Use the + // "all" node to apply to each message. + std::string_view name = "uuid"; + txnData->txn_json += R"(,"all":{"headers":{"fields":[)" + json_entry_array(name, uuid_view); + txnData->txn_json += "]}}"; + break; + } + + case TS_EVENT_HTTP_READ_REQUEST_HDR: { + TransactionData *txnData = static_cast(TSUserArgGet(txnp, transaction_arg_index)); + if (!txnData) { + TSError("[%s] No transaction data found for the header hook we registered for.", traffic_dump::debug_tag); + break; + } + // This hook is registered globally, not at TS_EVENT_HTTP_SSN_START in + // global_session_handler(). As such, this handler will be called with every + // transaction. However, we know that we are dumping this transaction + // because there is a ssnData associated with it. + + // We must grab the client request information before remap happens because + // the remap process modifies the request buffer. + TSMBuffer buffer; + TSMLoc hdr_loc; + if (TS_SUCCESS == TSHttpTxnClientReqGet(txnp, &buffer, &hdr_loc)) { + TSDebug(debug_tag, "Found client request"); + // We don't have an accurate view of the body size until TXN_CLOSE so we hold + // off on writing the content:size node until then. + txnData->txn_json += R"(,"client-request":)" + txnData->write_message_node_no_content(buffer, hdr_loc); + TSHandleMLocRelease(buffer, TS_NULL_MLOC, hdr_loc); + buffer = nullptr; + } + break; + } + + case TS_EVENT_HTTP_TXN_CLOSE: { + TransactionData *txnData = static_cast(TSUserArgGet(txnp, SessionData::get_session_arg_index())); + if (!txnData) { + TSError("[%s] No transaction data found for the close hook we registered for.", traffic_dump::debug_tag); + break; + } + // proxy-request/response headers + TSMBuffer buffer; + TSMLoc hdr_loc; + if (TS_SUCCESS == TSHttpTxnClientReqGet(txnp, &buffer, &hdr_loc)) { + txnData->txn_json += txnData->write_content_node(TSHttpTxnClientReqBodyBytesGet(txnp)) + "}"; + TSHandleMLocRelease(buffer, TS_NULL_MLOC, hdr_loc); + buffer = nullptr; + } + if (TS_SUCCESS == TSHttpTxnServerReqGet(txnp, &buffer, &hdr_loc)) { + TSDebug(debug_tag, "Found proxy request"); + txnData->txn_json += + R"(,"proxy-request":)" + txnData->write_message_node(buffer, hdr_loc, TSHttpTxnServerReqBodyBytesGet(txnp)); + TSHandleMLocRelease(buffer, TS_NULL_MLOC, hdr_loc); + buffer = nullptr; + } + if (TS_SUCCESS == TSHttpTxnServerRespGet(txnp, &buffer, &hdr_loc)) { + TSDebug(debug_tag, "Found server response"); + txnData->txn_json += + R"(,"server-response":)" + txnData->write_message_node(buffer, hdr_loc, TSHttpTxnServerRespBodyBytesGet(txnp)); + TSHandleMLocRelease(buffer, TS_NULL_MLOC, hdr_loc); + buffer = nullptr; + } + if (TS_SUCCESS == TSHttpTxnClientRespGet(txnp, &buffer, &hdr_loc)) { + TSDebug(debug_tag, "Found proxy response"); + txnData->txn_json += + R"(,"proxy-response":)" + txnData->write_message_node(buffer, hdr_loc, TSHttpTxnClientRespBodyBytesGet(txnp)); + TSHandleMLocRelease(buffer, TS_NULL_MLOC, hdr_loc); + buffer = nullptr; + } + + txnData->txn_json += "}"; + ssnData->write_transaction_to_disk(txnData->txn_json); + delete txnData; + break; + } + default: + TSDebug(debug_tag, "session_txn_handler(): Unhandled events %d", event); + TSHttpTxnReenable(txnp, TS_EVENT_HTTP_ERROR); + return TS_ERROR; + } + + TSHttpTxnReenable(txnp, TS_EVENT_HTTP_CONTINUE); + return TS_SUCCESS; +} + +} // namespace traffic_dump diff --git a/plugins/experimental/traffic_dump/transaction_data.h b/plugins/experimental/traffic_dump/transaction_data.h new file mode 100644 index 00000000000..c5b5139119a --- /dev/null +++ b/plugins/experimental/traffic_dump/transaction_data.h @@ -0,0 +1,110 @@ +/** @txn_data.h + Traffic Dump data specific to transactions. + @section license License + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#pragma once + +#include + +#include "ts/ts.h" + +#include "sensitive_fields.h" + +namespace traffic_dump +{ +/** The information associated with an single transaction. + * + * This class is responsible for containing the members associated with a + * particular transaction and defines the transaction handler callback. + */ +class TransactionData +{ +private: + /** The string for the JSON content of this transaction. */ + std::string txn_json; + + // The index to be used for the TS API for storing this TransactionData on a + // per-transaction basis. + static int transaction_arg_index; + + /// The set of fields, default and user-specified, that are sensitive and + /// whose values will be replaced with auto-generated generic content. + static sensitive_fields_t sensitive_fields; + +public: + /** Initialize TransactionData, using the provided sensitive fields. + * + * @return True if initialization is successful, false otherwise. + */ + static bool init(sensitive_fields_t &&sensitive_fields); + + /** Initialize TransactionData, using default sensitive fields. + * @return True if initialization is successful, false otherwise. + */ + static bool init(); + + /// Read the txn information from TSMBuffer and write the header information. + /// This function does not write the content node. + std::string write_message_node_no_content(TSMBuffer &buffer, TSMLoc &hdr_loc); + + /// Read the txn information from TSMBuffer and write the header information including + /// the content node describing the body characteristics. + std::string write_message_node(TSMBuffer &buffer, TSMLoc &hdr_loc, int64_t num_body_bytes); + + /// The handler callback for transaction events. + static int global_transaction_handler(TSCont contp, TSEvent event, void *edata); + +private: + /** Common logic for the init overloads. */ + static bool init_helper(); + + /** Initialize the generic sensitive field to be dumped. This is used instead + * of the sensitive field values seen on the wire. + */ + static void initialize_default_sensitive_field(); + + /** Return a separated string representing the HTTP fields considered sensitive. + * + * @return A comma-separated string representing the sensitive HTTP fields. + */ + static std::string get_sensitive_field_description(); + + /** Inspect the field to see whether it is sensitive and return a generic value + * of equal size to the original if it is. + * + * @param[in] name The field name to inspect. + * @param[in] original_value The field value to inspect. + * + * @return The value traffic_dump should dump for the given field. + */ + std::string_view replace_sensitive_fields(std::string_view name, std::string_view original_value); + + /// Write the content JSON node for an HTTP message. + // + /// "content" + /// "encoding" + /// "size" + std::string write_content_node(int64_t num_body_bytes); + + /** Remove the scheme prefix from the url. + * + * @return The view without the scheme prefix. + */ + std::string_view remove_scheme_prefix(std::string_view url); +}; + +} // namespace traffic_dump diff --git a/plugins/experimental/traffic_dump/unit_tests/test_json_utils.cc b/plugins/experimental/traffic_dump/unit_tests/test_json_utils.cc new file mode 100644 index 00000000000..4c27a91562e --- /dev/null +++ b/plugins/experimental/traffic_dump/unit_tests/test_json_utils.cc @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "json_utils.h" + +#include +#include + +using namespace traffic_dump; + +TEST_CASE("JsonUtilsMap", "[json_entry]") +{ + SECTION("Test the string_view value overload") + { + CHECK(std::string(R"("name":"value")") == json_entry("name", "value")); + CHECK(std::string(R"("":"value")") == json_entry("", "value")); + CHECK(std::string(R"("name":"")") == json_entry("name", "")); + } + + SECTION("Test the char buffer value overload") + { + CHECK(std::string(R"("name":"value")") == json_entry("name", "value", strlen("value"))); + CHECK(std::string(R"("name":"val")") == json_entry("name", "value", 3)); + CHECK(std::string(R"("":"value")") == json_entry("", "value", strlen("value"))); + CHECK(std::string(R"("name":"")") == json_entry("name", "", 0)); + } + + SECTION("Test that escaped characters are encoded as expected") + { + // Note that the raw strings on the left, i.e., R"(...)", leaves "\b" as + // two characters, not a single escaped one. The escaped characters on + // the right, by contrast, such as '\b', is a single escaped character. + CHECK(std::string(R"("name":"val\bue")") == json_entry("name", "val\bue")); + CHECK(std::string(R"("name":"\\value")") == json_entry("name", "\\value")); + CHECK(std::string(R"("name":"value\f")") == json_entry("name", "value\f")); + CHECK(std::string(R"("na\rme":"\tva\nlue\f")") == json_entry("na\rme", "\tva\nlue\f")); + CHECK(std::string(R"("\r":"\t\n\f")") == json_entry("\r", "\t\n\f")); + } +} + +TEST_CASE("JsonUtilsArray", "[json_entry_array]") +{ + CHECK(std::string(R"(["name","value"])") == json_entry_array("name", "value")); +} diff --git a/plugins/experimental/traffic_dump/unit_tests/test_sensitive_fields.cc b/plugins/experimental/traffic_dump/unit_tests/test_sensitive_fields.cc new file mode 100644 index 00000000000..19845c99e82 --- /dev/null +++ b/plugins/experimental/traffic_dump/unit_tests/test_sensitive_fields.cc @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "sensitive_fields.h" + +#include +#include + +using namespace traffic_dump; + +TEST_CASE("SensitiveFields", "[sensitive_fields_t]") +{ + sensitive_fields_t test_fields = {"one", "tWO", "THREE"}; + CHECK(test_fields.count("one") == 1); + CHECK(test_fields.count("oNe") == 1); + + CHECK(test_fields.count("tWO") == 1); + CHECK(test_fields.count("two") == 1); + + CHECK(test_fields.count("THREE") == 1); + CHECK(test_fields.count("three") == 1); +} diff --git a/plugins/experimental/traffic_dump/unit_tests/unit_test_main.cc b/plugins/experimental/traffic_dump/unit_tests/unit_test_main.cc new file mode 100644 index 00000000000..6aed3a63097 --- /dev/null +++ b/plugins/experimental/traffic_dump/unit_tests/unit_test_main.cc @@ -0,0 +1,25 @@ +/** @file + + This file used for catch based tests. It is the main() stub. + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#define CATCH_CONFIG_MAIN +#include "catch.hpp" diff --git a/plugins/experimental/uri_signing/Makefile.inc b/plugins/experimental/uri_signing/Makefile.inc index 864d2b48b38..7632c492467 100644 --- a/plugins/experimental/uri_signing/Makefile.inc +++ b/plugins/experimental/uri_signing/Makefile.inc @@ -30,7 +30,7 @@ experimental_uri_signing_uri_signing_la_LIBADD = @LIBJANSSON@ @LIBCJOSE@ @LIBPCR check_PROGRAMS += experimental/uri_signing/test_uri_signing -experimental_uri_signing_test_uri_signing_CPPFLAGS = $(AM_CPPFLAGS) -I$(abs_top_srcdir)/tests/include -DURI_SIGNING_UNIT_TEST +experimental_uri_signing_test_uri_signing_CPPFLAGS = $(AM_CPPFLAGS) -I$(abs_top_srcdir)/tests/include -DURI_SIGNING_UNIT_TEST experimental_uri_signing_test_uri_signing_LDADD = @LIBJANSSON@ @LIBCJOSE@ @LIBPCRE@ -lm -lcrypto experimental_uri_signing_test_uri_signing_SOURCES = \ experimental/uri_signing/unit_tests/uri_signing_test.cc \ diff --git a/plugins/experimental/uri_signing/README.md b/plugins/experimental/uri_signing/README.md index 398b9869958..02d7c20f4a2 100644 --- a/plugins/experimental/uri_signing/README.md +++ b/plugins/experimental/uri_signing/README.md @@ -79,13 +79,13 @@ a path. ### More Configuration Options **Strip Token** -When the strip_token parameter is set to true, the plugin removes the -token from both the url that is sent upstream to the origin and the url that +When the strip_token parameter is set to true, the plugin removes the +token from both the url that is sent upstream to the origin and the url that is used as the cache key. The strip_token parameter defaults to false and should be set by only one issuer. **ID** The id field takes a string indicating the identification of the entity processing the request. -This is used in aud claim checks to ensure that the receiver is the intended audience of a +This is used in aud claim checks to ensure that the receiver is the intended audience of a tokenized request. The id parameter can only be set by one issuer. Example: @@ -140,7 +140,7 @@ These claims are not supported. If they are present, the token will not validate - `cdnicrit` - `cdniip` -In addition, the `cdniuc` container of `hash` is +In addition, the `cdniuc` container of `hash` is **not supported**. ### Token Renewal diff --git a/plugins/experimental/uri_signing/common.h b/plugins/experimental/uri_signing/common.h index 467d0cee257..9a51bb61b97 100644 --- a/plugins/experimental/uri_signing/common.h +++ b/plugins/experimental/uri_signing/common.h @@ -33,7 +33,10 @@ void PrintToStdErr(const char *fmt, ...); #else #include "ts/ts.h" -#define PluginDebug(...) TSDebug("uri_signing", PLUGIN_NAME " " __VA_ARGS__) -#define PluginError(...) PluginDebug(__VA_ARGS__), TSError(PLUGIN_NAME " " __VA_ARGS__) +#define __FILENAME__ (strrchr(__FILE__, '/') ? strrchr(__FILE__, '/') + 1 : __FILE__) +#define PluginDebug(fmt, ...) TSDebug(PLUGIN_NAME, "[%s:% 4d] %s(): " fmt, __FILENAME__, __LINE__, __func__, ##__VA_ARGS__); +#define PluginError(fmt, ...) \ + PluginDebug(fmt, ##__VA_ARGS__); \ + TSError("[%s:% 4d] %s(): " fmt, __FILENAME__, __LINE__, __func__, ##__VA_ARGS__); #endif diff --git a/plugins/experimental/uri_signing/config.c b/plugins/experimental/uri_signing/config.c index 815f12c5efa..8727e9f3c98 100644 --- a/plugins/experimental/uri_signing/config.c +++ b/plugins/experimental/uri_signing/config.c @@ -289,7 +289,6 @@ read_config(const char *path) PluginDebug("Found Id in the config: %s", cfg->id); } } - json_decref(id_json); json_t *strip_json = json_object_get(jwks, "strip_token"); if (strip_json) { diff --git a/plugins/experimental/uri_signing/config.h b/plugins/experimental/uri_signing/config.h index 87688374c85..16bb6c9fb24 100644 --- a/plugins/experimental/uri_signing/config.h +++ b/plugins/experimental/uri_signing/config.h @@ -16,6 +16,8 @@ * limitations under the License. */ +#pragma once + #include #include diff --git a/plugins/experimental/uri_signing/cookie.h b/plugins/experimental/uri_signing/cookie.h index a5d0f20b4db..1944df9b4cc 100644 --- a/plugins/experimental/uri_signing/cookie.h +++ b/plugins/experimental/uri_signing/cookie.h @@ -16,5 +16,7 @@ * limitations under the License. */ +#pragma once + #include const char *get_cookie_value(const char **cookie, size_t *cookie_ct, const char *key, size_t *ct); diff --git a/plugins/experimental/uri_signing/jwt.h b/plugins/experimental/uri_signing/jwt.h index 95efbbc1da0..1e4d58fefda 100644 --- a/plugins/experimental/uri_signing/jwt.h +++ b/plugins/experimental/uri_signing/jwt.h @@ -16,6 +16,8 @@ * limitations under the License. */ +#pragma once + #include #include diff --git a/plugins/experimental/uri_signing/normalize.c b/plugins/experimental/uri_signing/normalize.c index e51411190a0..cd475868e2e 100644 --- a/plugins/experimental/uri_signing/normalize.c +++ b/plugins/experimental/uri_signing/normalize.c @@ -218,7 +218,7 @@ normalize_uri(const char *uri, int uri_ct, char *normal_uri, int normal_ct) const char *uri_end = uri + uri_ct; const char *buff_end = normal_uri + normal_ct; - if (normal_uri && normal_ct < uri_ct + 1) { + if ((normal_uri == NULL) || (normal_uri && normal_ct < uri_ct + 1)) { PluginDebug("Buffer to Normalize URI not large enough."); return -1; } diff --git a/plugins/experimental/uri_signing/normalize.h b/plugins/experimental/uri_signing/normalize.h index a84c9979718..ced84bf635c 100644 --- a/plugins/experimental/uri_signing/normalize.h +++ b/plugins/experimental/uri_signing/normalize.h @@ -16,5 +16,7 @@ * limitations under the License. */ +#pragma once + int normalize_uri(const char *uri, int uri_ct, char *uri_normal, int buffer_size); int remove_dot_segments(const char *path, int path_ct, char *ret_buffer, int buff_ct); diff --git a/plugins/experimental/uri_signing/parse.c b/plugins/experimental/uri_signing/parse.c index 3ca10b21d23..f577e740a7e 100644 --- a/plugins/experimental/uri_signing/parse.c +++ b/plugins/experimental/uri_signing/parse.c @@ -31,8 +31,8 @@ cjose_jws_t * get_jws_from_uri(const char *uri, size_t uri_ct, const char *paramName, char *strip_uri, size_t buff_ct, size_t *strip_ct) { /* Reserved characters as defined by the URI Generic Syntax RFC: https://tools.ietf.org/html/rfc3986#section-2.2 */ - static const char *reserved_string = ":/?#[]@!$&\'()*+,;="; - static const char *sub_delim_string = "!$&\'()*+,;="; + static char const *const reserved_string = ":/?#[]@!$&\'()*+,;="; + static char const *const sub_delim_string = "!$&\'()*+,;="; /* If param name ends in reserved character this will be treated as the termination symbol when parsing for package. Default is * '='. */ diff --git a/plugins/experimental/uri_signing/parse.h b/plugins/experimental/uri_signing/parse.h index d05bfd59adb..98a35abcc28 100644 --- a/plugins/experimental/uri_signing/parse.h +++ b/plugins/experimental/uri_signing/parse.h @@ -16,9 +16,13 @@ * limitations under the License. */ +#pragma once + #include struct _cjose_jws_int; + +/* For now strip_ct returns size of string *including* the null terminator */ struct _cjose_jws_int *get_jws_from_uri(const char *uri, size_t uri_ct, const char *paramName, char *strip_uri, size_t buff_ct, size_t *strip_ct); struct _cjose_jws_int *get_jws_from_cookie(const char **cookie, size_t *cookie_ct, const char *paramName); diff --git a/plugins/experimental/uri_signing/python_signer/README.md b/plugins/experimental/uri_signing/python_signer/README.md new file mode 100644 index 00000000000..daa41bb8f2c --- /dev/null +++ b/plugins/experimental/uri_signing/python_signer/README.md @@ -0,0 +1,36 @@ +Python URI Signer +================== + +Given a configuration file and a URI, this python script will generate a signed URI according to the URI signing protocol outlined [here](https://tools.ietf.org/html/draft-ietf-cdni-uri-signing-16): + +The script takes a config file and a uri as command line arguments. It picks one of the keys located in the json file at random +and embeds a valid JWT as a query string parameter into the uri and prints this new signed URI to standard out. + +** Disclaimer ** +Please note that this script is provided as a very simple example of how to implement a signer should not be considered production ready. + +Requirements +------ + +[python-jose](https://pypi.org/project/python-jose/) library must be installed (pip install python-jose). + +Config +------ + +The config file should be a JSON object that contains the following: + + - `iss`: A string representing the issuer of the token + - `token_lifetime`: The lifetime of the token in seconds. Expiry of the token is calculated as now + token_lifetime + - `aud`: A string representing the intended audience of the token. + - `cdnistt`: Boolean value which if set to true uses cookie signed token transport, allowing the validator of the token to + to issue subsequent tokens via set cookie headers. + - `cdniets`: Must be set if using cdnistt. Provides means of setting Expiry Times when generating subsequent tokens. It denotes + the number of seconds to be added to the time at which the JWT is verified that gives the value of the Expiry Time claim of the + next signed JWT. + - `keys`: A list of json objects, each one representing a key. Each key should have the following attributes: + - `alg`: The Cryptographic algorithm to be used with the key. + - `kid`: The key identifier + - `kty`: The key type + - `k`: The key itself + +example_config.json can be used as a template for the configuration file. diff --git a/plugins/experimental/uri_signing/python_signer/example_config.json b/plugins/experimental/uri_signing/python_signer/example_config.json new file mode 100644 index 00000000000..4039796ebad --- /dev/null +++ b/plugins/experimental/uri_signing/python_signer/example_config.json @@ -0,0 +1,33 @@ +{ + "iss": "Example Issuer", + "token_lifetime": 90, + "aud": "Caching Software", + "cdnistt": true, + "cdniets": 30, + "keys": [ + { + "alg": "HS256", + "kid": 0, + "kty": "oct", + "k": "SECRET1" + }, + { + "alg": "HS256", + "kid": 1, + "kty": "oct", + "k": "SECRET2" + }, + { + "alg": "HS256", + "kid": 2, + "kty": "oct", + "k": "SECRET3" + }, + { + "alg": "HS256", + "kid": 3, + "kty": "oct", + "k": "SECRET4" + } + ] +} diff --git a/plugins/experimental/uri_signing/python_signer/uri_signer.py b/plugins/experimental/uri_signing/python_signer/uri_signer.py new file mode 100755 index 00000000000..549080a4d9a --- /dev/null +++ b/plugins/experimental/uri_signing/python_signer/uri_signer.py @@ -0,0 +1,133 @@ +#!/usr/bin/env python3 + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import argparse +import random +import time +import os + +# https://github.com/mpdavis/python-jose +from jose import jwk, jwt + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('-c', '--config', + help="Configuration File", + required=True) + parser.add_argument('-u', '--uri', + help="URI to sign", + required=True) + + # helpers + parser.add_argument('--key_index', type=int, nargs=1) + parser.add_argument('--token_lifetime', type=int, nargs=1) + + # override arguments -- claims + parser.add_argument('--aud', nargs=1) + parser.add_argument('--cdniets', type=int, nargs=1) + parser.add_argument('--cdnistd', type=int, nargs=1) + parser.add_argument('--cdnistt', type=int, nargs=1) + parser.add_argument('--exp', type=int, nargs=1) + parser.add_argument('--iss', nargs=1) + + # override arguments -- key + parser.add_argument('--alg', nargs=1) + parser.add_argument('--k', nargs=1) + parser.add_argument('--kid', nargs=1) + parser.add_argument('--kty', nargs=1) + + args = parser.parse_args() + + with open(args.config, 'r') as f: + config = json.load(f) + + keys = config["keys"] + + # Select a key, either explicitly or randomly + key_index = 0 + if args.key_index: + key_index = args.key_index[0] + print("args key_index " + str(key_index)) + else: + key_index = random.randint(0, len(keys) - 1) + print("randomizing key index") + + print("Using key_index " + str(key_index)) + + print("Using Key: " + str(keys[key_index]["kid"]) + " to sign URI.") + key = keys[key_index] + + # Build Out claimset + claimset = {} + if "iss" in config.keys(): + claimset["iss"] = config["iss"] + + if "token_lifetime" in config.keys(): + claimset["exp"] = int(time.time()) + config["token_lifetime"] + else: + claimset["exp"] = int(time.time()) + 30 + + if "aud" in config.keys(): + claimset["aud"] = config["aud"] + + if "cdnistt" in config.keys(): + if config["cdnistt"]: + claimset["cdnistt"] = 1 + if "cdniets" in config.keys(): + claimset["cdniets"] = config["cdniets"] + else: + claimset["cdniets"] = 30 + + # process override args - simple + if args.iss: + claimset["iss"] = args.iss[0] + if args.exp: + claimset["exp"] = args.exp[0] + if args.aud: + claimset["aud"] = args.aud[0] + + # process override args - complex + if args.cdnistt: + claimset["cdnistt"] = args.cdnistt[0] + + if "cdnistt" in config.keys(): + if args.cdniets: + claimset["cdniets"] = arg.cdniets[0] + + # specific key overrides + if args.alg: + key["alg"] = args.alg[0] + if args.kid: + key["kid"] = args.kid[0] + if args.kty: + key["kty"] = args.kty[0] + if args.k: + key["k"] = args.k[0] + + print(claimset) + print(key) + + Token = jwt.encode(claimset, key, algorithm=key["alg"]) + + print("Signed URL: " + args.uri + "?URISigningPackage=" + Token) + + +if __name__ == "__main__": + main() diff --git a/plugins/experimental/uri_signing/timing.h b/plugins/experimental/uri_signing/timing.h index 9511bcdd5b3..20768a58bf9 100644 --- a/plugins/experimental/uri_signing/timing.h +++ b/plugins/experimental/uri_signing/timing.h @@ -16,6 +16,8 @@ * limitations under the License. */ +#pragma once + #include #include diff --git a/plugins/experimental/uri_signing/uri_signing.c b/plugins/experimental/uri_signing/uri_signing.c index f85a87b5ed3..fe6793bee84 100644 --- a/plugins/experimental/uri_signing/uri_signing.c +++ b/plugins/experimental/uri_signing/uri_signing.c @@ -62,10 +62,23 @@ TSRemapNewInstance(int argc, char *argv[], void **ih, char *errbuf, int errbuf_s TSDebug(PLUGIN_NAME, "Initializing remap function of %s -> %s with config from %s", argv[0], argv[1], argv[2]); - const char *install_dir = TSInstallDirGet(); - size_t config_file_ct = snprintf(NULL, 0, "%s/%s/%s", install_dir, "etc/trafficserver", argv[2]); - char *config_file = malloc(config_file_ct + 1); - (void)snprintf(config_file, config_file_ct + 1, "%s/%s/%s", install_dir, "etc/trafficserver", argv[2]); + char const *const fname = argv[2]; + + if (0 == strlen(fname)) { + snprintf(errbuf, errbuf_size, "[TSRemapNewKeyInstance] - Invalid config file name for %s -> %s", argv[0], argv[1]); + return TS_ERROR; + } + + char *config_file = NULL; + if ('/' == fname[0]) { + config_file = strdup(fname); + } else { + char const *const config_dir = TSConfigDirGet(); + size_t const config_file_ct = snprintf(NULL, 0, "%s/%s", config_dir, fname); + config_file = malloc(config_file_ct + 1); + (void)snprintf(config_file, config_file_ct + 1, "%s/%s", config_dir, fname); + } + TSDebug(PLUGIN_NAME, "config file name: %s", config_file); struct config *cfg = read_config(config_file); if (!cfg) { @@ -159,8 +172,9 @@ TSRemapDoRemap(void *ih, TSHttpTxn txnp, TSRemapRequestInfo *rri) const char *url = NULL; char *strip_uri = NULL; TSRemapStatus status = TSREMAP_NO_REMAP; + bool checked_auth = false; - const char *package = "URISigningPackage"; + static char const *const package = "URISigningPackage"; TSMBuffer mbuf; TSMLoc ul; @@ -174,9 +188,7 @@ TSRemapDoRemap(void *ih, TSHttpTxn txnp, TSRemapRequestInfo *rri) TSHandleMLocRelease(mbuf, TS_NULL_MLOC, ul); PluginDebug("Processing request for %.*s.", url_ct, url); - if (cpi < max_cpi) { - checkpoints[cpi++] = mark_timer(&t); - } + checkpoints[cpi++] = mark_timer(&t); int strip_size = url_ct + 1; strip_uri = (char *)TSmalloc(strip_size); @@ -185,9 +197,8 @@ TSRemapDoRemap(void *ih, TSHttpTxn txnp, TSRemapRequestInfo *rri) size_t strip_ct; cjose_jws_t *jws = get_jws_from_uri(url, url_ct, package, strip_uri, strip_size, &strip_ct); - if (cpi < max_cpi) { - checkpoints[cpi++] = mark_timer(&t); - } + checkpoints[cpi++] = mark_timer(&t); + int checked_cookies = 0; if (!jws) { check_cookies: @@ -207,7 +218,11 @@ TSRemapDoRemap(void *ih, TSHttpTxn txnp, TSRemapRequestInfo *rri) field = TSMimeHdrFieldFind(buffer, hdr, "Cookie", 6); if (field == TS_NULL_MLOC) { TSHandleMLocRelease(buffer, TS_NULL_MLOC, hdr); - goto fail; + if (!checked_auth) { + goto check_auth; + } else { + goto fail; + } } const char *client_cookie; @@ -218,7 +233,11 @@ TSRemapDoRemap(void *ih, TSHttpTxn txnp, TSRemapRequestInfo *rri) TSHandleMLocRelease(buffer, TS_NULL_MLOC, hdr); if (!client_cookie || !client_cookie_ct) { - goto fail; + if (!checked_auth) { + goto check_auth; + } else { + goto fail; + } } size_t client_cookie_sz_ct = client_cookie_ct; check_more_cookies: @@ -246,11 +265,15 @@ TSRemapDoRemap(void *ih, TSHttpTxn txnp, TSRemapRequestInfo *rri) cjose_jws_t *map_jws = get_jws_from_uri(map_url, map_url_ct, package, map_strip_uri, map_strip_size, &map_strip_ct); cjose_jws_release(map_jws); - char *strip_uri_start = &map_strip_uri[0]; - char *strip_uri_end = &map_strip_uri[map_strip_ct]; - PluginDebug("Stripping token from upstream url to: %s", strip_uri_start); + char const *strip_uri_start = map_strip_uri; + + /* map_strip_uri is null terminated */ + size_t const mlen = strlen(strip_uri_start); + char const *strip_uri_end = strip_uri_start + mlen; - TSParseResult parse_rc = TSUrlParse(rri->requestBufp, rri->requestUrl, (const char **)&strip_uri_start, strip_uri_end); + PluginDebug("Stripping token from upstream url to: %.*s", (int)mlen, strip_uri_start); + + TSParseResult parse_rc = TSUrlParse(rri->requestBufp, rri->requestUrl, &strip_uri_start, strip_uri_end); if (map_url != NULL) { TSfree(map_url); } @@ -266,8 +289,10 @@ TSRemapDoRemap(void *ih, TSHttpTxn txnp, TSRemapRequestInfo *rri) } } } +check_auth: /* Check auth_dir and pass through if configured */ if (uri_matches_auth_directive((struct config *)ih, url, url_ct)) { + PluginDebug("Auth directive matched for %.*s", url_ct, url); if (url != NULL) { TSfree((void *)url); } @@ -276,6 +301,8 @@ TSRemapDoRemap(void *ih, TSHttpTxn txnp, TSRemapRequestInfo *rri) } return TSREMAP_NO_REMAP; } + checked_auth = true; + if (!jws) { goto fail; } diff --git a/plugins/experimental/url_sig/url_sig.c b/plugins/experimental/url_sig/url_sig.c index 6782869c416..249a011366c 100644 --- a/plugins/experimental/url_sig/url_sig.c +++ b/plugins/experimental/url_sig/url_sig.c @@ -165,11 +165,11 @@ TSRemapNewInstance(int argc, char *argv[], void **ih, char *errbuf, int errbuf_s return TS_ERROR; } if (strncmp(line, "key", 3) == 0) { - if (strncmp((line + 3), "0", 1) == 0) { + if (strncmp(line + 3, "0", 1) == 0) { keynum = 0; } else { TSDebug(PLUGIN_NAME, ">>> %s <<<", line + 3); - keynum = atoi((line + 3)); + keynum = atoi(line + 3); if (keynum == 0) { keynum = -1; // Not a Number } @@ -349,14 +349,18 @@ fixedBufferWrite(char **dest_end, int *dest_len, const char *src, int src_len) } static char * -urlParse(char *url, char *anchor, char *new_path_seg, int new_path_seg_len, char *signed_seg, unsigned int signed_seg_len) +urlParse(char const *const url_in, char *anchor, char *new_path_seg, int new_path_seg_len, char *signed_seg, + unsigned int signed_seg_len) { char *segment[MAX_SEGMENTS]; + char url[8192] = {'\0'}; unsigned char decoded_string[2048] = {'\0'}; char new_url[8192]; /* new_url is not null_terminated */ char *p = NULL, *sig_anchor = NULL, *saveptr = NULL; int i = 0, numtoks = 0, decoded_len = 0, sig_anchor_seg = 0; + strncat(url, url_in, sizeof(url) - strlen(url) - 1); + char *new_url_end = new_url; int new_url_len_left = sizeof(new_url); @@ -373,7 +377,7 @@ urlParse(char *url, char *anchor, char *new_path_seg, int new_path_seg_len, char TSError("insufficient space to copy schema into new_path_seg buffer."); return NULL; } - TSDebug(PLUGIN_NAME, "%s:%d - new_url: %*s\n", __FILE__, __LINE__, (int)(new_url_end - new_url), new_url); + TSDebug(PLUGIN_NAME, "%s:%d - new_url: %.*s\n", __FILE__, __LINE__, (int)(new_url_end - new_url), new_url); // parse the url. if ((p = strtok_r(skip, "/", &saveptr)) != NULL) { @@ -582,19 +586,25 @@ TSRemapDoRemap(void *ih, TSHttpTxn txnp, TSRemapRequestInfo *rri) // check for path params. if (query == NULL || strstr(query, "E=") == NULL) { char *const parsed = urlParse(url, cfg->sig_anchor, new_path, 8192, path_params, 8192); - if (NULL == parsed) { - err_log(url, "Has no signing query string or signing path parameters."); + if (parsed == NULL) { + err_log(url, "Unable to parse/decode new url path parameters"); goto deny; } - url = parsed; has_path_params = true; - query = strstr(url, ";"); + query = strstr(parsed, ";"); if (query == NULL) { err_log(url, "Has no signing query string or signing path parameters."); + TSfree(parsed); goto deny; } + + if (url != current_url) { + TSfree(url); + } + + url = parsed; } /* first, parse the query string */ diff --git a/plugins/generator/generator.cc b/plugins/generator/generator.cc index 551c6ec5a6f..751a2531393 100644 --- a/plugins/generator/generator.cc +++ b/plugins/generator/generator.cc @@ -80,7 +80,7 @@ static uint8_t GeneratorData[32 * 1024]; static int StatCountBytes = -1; static int StatCountResponses = -1; -static int GeneratorInterceptionHook(TSCont contp, TSEvent event, void *edata); +static int GeneratorInterceptHook(TSCont contp, TSEvent event, void *edata); static int GeneratorTxnHook(TSCont contp, TSEvent event, void *edata); struct GeneratorRequest; @@ -406,7 +406,7 @@ GeneratorParseRequest(GeneratorRequest *grq) // starts with TS_EVENT_NET_ACCEPT, and then continues with // TSVConn events. static int -GeneratorInterceptionHook(TSCont contp, TSEvent event, void *edata) +GeneratorInterceptHook(TSCont contp, TSEvent event, void *edata) { argument_type arg(edata); @@ -594,6 +594,21 @@ GeneratorInterceptionHook(TSCont contp, TSEvent event, void *edata) } } +// Little helper function, to turn off the cache on requests which aren't cacheable to begin with. +// This helps performance, a lot. +static void +CheckCacheable(TSHttpTxn txnp, TSMLoc url, TSMBuffer bufp) +{ + int pathsz = 0; + const char *path = TSUrlPathGet(bufp, url, &pathsz); + + if (path && (pathsz >= 8) && (0 == memcmp(path, "nocache/", 8))) { + // It's not cacheable, so, turn off the cache. This avoids major serialization and performance issues. + VDEBUG("turning off the cache, uncacehable"); + TSHttpTxnConfigIntSet(txnp, TS_CONFIG_HTTP_CACHE_HTTP, 0); + } +} + // Handle events that occur on the TSHttpTxn. static int GeneratorTxnHook(TSCont contp, TSEvent event, void *edata) @@ -603,6 +618,19 @@ GeneratorTxnHook(TSCont contp, TSEvent event, void *edata) VDEBUG("contp=%p, event=%s (%d), edata=%p", contp, TSHttpEventNameLookup(event), event, edata); switch (event) { + case TS_EVENT_HTTP_READ_REQUEST_HDR: { + TSMBuffer recp; + TSMLoc url_loc; + TSMLoc hdr_loc; + + TSReleaseAssert(TSHttpTxnClientReqGet(arg.txn, &recp, &hdr_loc) == TS_SUCCESS); + TSReleaseAssert(TSHttpHdrUrlGet(recp, hdr_loc, &url_loc) == TS_SUCCESS); + CheckCacheable(arg.txn, url_loc, recp); + TSHandleMLocRelease(recp, hdr_loc, url_loc); + TSHandleMLocRelease(recp, TS_NULL_MLOC, hdr_loc); + break; + } + case TS_EVENT_HTTP_CACHE_LOOKUP_COMPLETE: { int status; @@ -610,9 +638,8 @@ GeneratorTxnHook(TSCont contp, TSEvent event, void *edata) if (status != TS_CACHE_LOOKUP_HIT_FRESH) { // This transaction is going to be a cache miss, so intercept it. VDEBUG("intercepting origin server request for txn=%p", arg.txn); - TSHttpTxnServerIntercept(TSContCreate(GeneratorInterceptionHook, TSMutexCreate()), arg.txn); + TSHttpTxnServerIntercept(TSContCreate(GeneratorInterceptHook, TSMutexCreate()), arg.txn); } - break; } @@ -642,7 +669,7 @@ GeneratorInitialize() } void -TSPluginInit(int /* argc */, const char * /* argv */ []) +TSPluginInit(int /* argc */, const char * /* argv */[]) { TSPluginRegistrationInfo info; @@ -656,6 +683,10 @@ TSPluginInit(int /* argc */, const char * /* argv */ []) GeneratorInitialize(); + // We want to check early on if the request is cacheable or not, and if it's not cacheable, + // we benefit signifciantly from turning off the cache completely. + TSHttpHookAdd(TS_HTTP_READ_REQUEST_HDR_HOOK, TxnHook); + // Wait until after the cache lookup to decide whether to // intercept a request. For cache hits we will never intercept. TSHttpHookAdd(TS_HTTP_CACHE_LOOKUP_COMPLETE_HOOK, TxnHook); @@ -669,15 +700,16 @@ TSRemapInit(TSRemapInterface * /* api_info */, char * /* errbuf */, int /* errbu } TSRemapStatus -TSRemapDoRemap(void * /* ih */, TSHttpTxn txn, TSRemapRequestInfo * /* rri ATS_UNUSED */) +TSRemapDoRemap(void * /* ih */, TSHttpTxn txn, TSRemapRequestInfo *rri) { + // Check if we should turn off the cache before doing anything else ... + CheckCacheable(txn, rri->requestUrl, rri->requestBufp); TSHttpTxnHookAdd(txn, TS_HTTP_CACHE_LOOKUP_COMPLETE_HOOK, TxnHook); return TSREMAP_NO_REMAP; // This plugin never rewrites anything. } TSReturnCode -TSRemapNewInstance(int /* argc */, char * /* argv */ [], void **ih, char * /* errbuf ATS_UNUSED */, - int /* errbuf_size ATS_UNUSED */) +TSRemapNewInstance(int /* argc */, char * /* argv */[], void **ih, char * /* errbuf ATS_UNUSED */, int /* errbuf_size ATS_UNUSED */) { *ih = nullptr; return TS_SUCCESS; diff --git a/plugins/header_rewrite/conditions.cc b/plugins/header_rewrite/conditions.cc index 380d270c80e..b9df7b89e28 100644 --- a/plugins/header_rewrite/conditions.cc +++ b/plugins/header_rewrite/conditions.cc @@ -274,11 +274,17 @@ ConditionUrl::append_value(std::string &s, const Resources &res) TSMLoc url = nullptr; TSMBuffer bufp = nullptr; - if (res._rri != nullptr) { + if (_type == CLIENT) { + // CLIENT always uses the pristine URL + TSDebug(PLUGIN_NAME, " Using the pristine url"); + if (TSHttpTxnPristineUrlGet(res.txnp, &bufp, &url) != TS_SUCCESS) { + TSError("[header_rewrite] Error getting the pristine URL"); + return; + } + } else if (res._rri != nullptr) { // called at the remap hook bufp = res._rri->requestBufp; - if (_type == URL || _type == CLIENT) { - // res._rri->requestBufp and res.client_bufp are the same if it is at the remap hook + if (_type == URL) { TSDebug(PLUGIN_NAME, " Using the request url"); url = res._rri->requestUrl; } else if (_type == FROM) { @@ -292,21 +298,17 @@ ConditionUrl::append_value(std::string &s, const Resources &res) return; } } else { - TSMLoc hdr_loc = nullptr; - if (_type == CLIENT) { - bufp = res.client_bufp; - hdr_loc = res.client_hdr_loc; - } else if (_type == URL) { - bufp = res.bufp; - hdr_loc = res.hdr_loc; + if (_type == URL) { + bufp = res.bufp; + TSMLoc hdr_loc = res.hdr_loc; + if (TSHttpHdrUrlGet(bufp, hdr_loc, &url) != TS_SUCCESS) { + TSError("[header_rewrite] Error getting the URL"); + return; + } } else { TSError("[header_rewrite] Rule not supported at this hook"); return; } - if (TSHttpHdrUrlGet(bufp, hdr_loc, &url) != TS_SUCCESS) { - TSError("[header_rewrite] Error getting the URL"); - return; - } } int i; @@ -344,12 +346,15 @@ ConditionUrl::append_value(std::string &s, const Resources &res) TSDebug(PLUGIN_NAME, " Scheme to match is: %.*s", i, q_str); break; case URL_QUAL_URL: - case URL_QUAL_NONE: - q_str = TSUrlStringGet(bufp, url, &i); - s.append(q_str, i); - TSDebug(PLUGIN_NAME, " URL to match is: %.*s", i, q_str); + case URL_QUAL_NONE: { + // TSUrlStringGet returns an allocated char * we must free + char *non_const_q_str = TSUrlStringGet(bufp, url, &i); + s.append(non_const_q_str, i); + TSDebug(PLUGIN_NAME, " URL to match is: %.*s", i, non_const_q_str); + TSfree(non_const_q_str); break; } + } } bool @@ -1113,17 +1118,17 @@ ConditionCidr::append_value(std::string &s, const Resources &res) if (addr) { switch (addr->sa_family) { case AF_INET: { - char res[INET_ADDRSTRLEN]; + char resource[INET_ADDRSTRLEN]; struct in_addr ipv4 = reinterpret_cast(addr)->sin_addr; ipv4.s_addr &= _v4_mask.s_addr; - inet_ntop(AF_INET, &ipv4, res, INET_ADDRSTRLEN); - if (res[0]) { - s += res; + inet_ntop(AF_INET, &ipv4, resource, INET_ADDRSTRLEN); + if (resource[0]) { + s += resource; } } break; case AF_INET6: { - char res[INET6_ADDRSTRLEN]; + char resource[INET6_ADDRSTRLEN]; struct in6_addr ipv6 = reinterpret_cast(addr)->sin6_addr; if (_v6_zero_bytes > 0) { @@ -1132,9 +1137,9 @@ ConditionCidr::append_value(std::string &s, const Resources &res) if (_v6_mask != 0xff) { ipv6.s6_addr[16 - _v6_zero_bytes] &= _v6_mask; } - inet_ntop(AF_INET6, &ipv6, res, INET6_ADDRSTRLEN); - if (res[0]) { - s += res; + inet_ntop(AF_INET6, &ipv6, resource, INET6_ADDRSTRLEN); + if (resource[0]) { + s += resource; } } break; } diff --git a/plugins/header_rewrite/header_rewrite.cc b/plugins/header_rewrite/header_rewrite.cc index 09825449579..a0e6707f2fe 100644 --- a/plugins/header_rewrite/header_rewrite.cc +++ b/plugins/header_rewrite/header_rewrite.cc @@ -22,8 +22,6 @@ #include "ts/ts.h" #include "ts/remap.h" -#include "tscore/ink_atomic.h" - #include "parser.h" #include "ruleset.h" #include "resources.h" @@ -236,7 +234,7 @@ RulesConfig::parse_config(const std::string &fname, TSHttpHookID default_hook) } } } catch (std::runtime_error &e) { - TSError("[%s] header_rewrite configuration exception: %s", PLUGIN_NAME, e.what()); + TSError("[%s] header_rewrite configuration exception: %s in file: %s", PLUGIN_NAME, e.what(), fname.c_str()); delete rule; return false; } diff --git a/plugins/header_rewrite/header_rewrite_test.cc b/plugins/header_rewrite/header_rewrite_test.cc index cd032f2eba4..704c68d0a7c 100644 --- a/plugins/header_rewrite/header_rewrite_test.cc +++ b/plugins/header_rewrite/header_rewrite_test.cc @@ -69,10 +69,10 @@ class ParserTest : public Parser bool res; }; -class SimpleTokenizerTest : public SimpleTokenizer +class SimpleTokenizerTest : public HRWSimpleTokenizer { public: - SimpleTokenizerTest(const std::string &line) : SimpleTokenizer(line), res(true) + SimpleTokenizerTest(const std::string &line) : HRWSimpleTokenizer(line), res(true) { std::cout << "Finished tokenizer test: " << line << std::endl; } diff --git a/plugins/header_rewrite/operators.cc b/plugins/header_rewrite/operators.cc index 4c58f2fc764..f0cd8b4b159 100644 --- a/plugins/header_rewrite/operators.cc +++ b/plugins/header_rewrite/operators.cc @@ -406,7 +406,9 @@ OperatorSetRedirect::exec(const Resources &res) const const char *end = value.size() + start; if (remap) { // Set new location. - TSUrlParse(bufp, url_loc, &start, end); + if (TS_PARSE_ERROR == TSUrlParse(bufp, url_loc, &start, end)) { + TSDebug(PLUGIN_NAME, "Could not set Location field value to: %s", value.c_str()); + } // Set the new status. TSHttpTxnStatusSet(res.txnp, static_cast(_status.get_int_value())); const_cast(res).changed_url = true; diff --git a/plugins/header_rewrite/parser.cc b/plugins/header_rewrite/parser.cc index 7e5e40e6fd1..cb6e2c5cf3c 100644 --- a/plugins/header_rewrite/parser.cc +++ b/plugins/header_rewrite/parser.cc @@ -248,7 +248,7 @@ Parser::cond_is_hook(TSHttpHookID &hook) const return false; } -SimpleTokenizer::SimpleTokenizer(const std::string &original_line) +HRWSimpleTokenizer::HRWSimpleTokenizer(const std::string &original_line) { std::string line = original_line; ParserState state = PARSER_DEFAULT; diff --git a/plugins/header_rewrite/parser.h b/plugins/header_rewrite/parser.h index 691a817b72b..4b3b468c037 100644 --- a/plugins/header_rewrite/parser.h +++ b/plugins/header_rewrite/parser.h @@ -99,14 +99,14 @@ class Parser std::vector _tokens; }; -class SimpleTokenizer +class HRWSimpleTokenizer { public: - explicit SimpleTokenizer(const std::string &line); + explicit HRWSimpleTokenizer(const std::string &line); // noncopyable - SimpleTokenizer(const SimpleTokenizer &) = delete; - void operator=(const SimpleTokenizer &) = delete; + HRWSimpleTokenizer(const HRWSimpleTokenizer &) = delete; + void operator=(const HRWSimpleTokenizer &) = delete; const std::vector & get_tokens() const diff --git a/plugins/header_rewrite/value.cc b/plugins/header_rewrite/value.cc index fc4dcc0e09b..e6c8ddc0dea 100644 --- a/plugins/header_rewrite/value.cc +++ b/plugins/header_rewrite/value.cc @@ -44,7 +44,7 @@ Value::set_value(const std::string &val) _value = val; if (_value.find("%{") != std::string::npos) { - SimpleTokenizer tokenizer(_value); + HRWSimpleTokenizer tokenizer(_value); auto tokens = tokenizer.get_tokens(); for (auto token : tokens) { diff --git a/plugins/lua/ts_lua.c b/plugins/lua/ts_lua.c index 5aef7b551ef..1451c560ec5 100644 --- a/plugins/lua/ts_lua.c +++ b/plugins/lua/ts_lua.c @@ -20,41 +20,309 @@ #include #include #include +#include #include "ts_lua_util.h" #define TS_LUA_MAX_STATE_COUNT 256 +#define TS_LUA_STATS_TIMEOUT 5000 // 5s -- convert to configurable +#define TS_LUA_STATS_BUFFER_SIZE 10 // stats buffer + +#define TS_LUA_IND_STATE 0 +#define TS_LUA_IND_GC_BYTES 1 +#define TS_LUA_IND_THREADS 2 +#define TS_LUA_IND_SIZE 3 + static uint64_t ts_lua_http_next_id = 0; static uint64_t ts_lua_g_http_next_id = 0; -static ts_lua_main_ctx *ts_lua_main_ctx_array; -static ts_lua_main_ctx *ts_lua_g_main_ctx_array; +static ts_lua_main_ctx *ts_lua_main_ctx_array = NULL; +static ts_lua_main_ctx *ts_lua_g_main_ctx_array = NULL; + +// records.config entry injected by plugin +static char const *const ts_lua_mgmt_state_str = "proxy.config.plugin.lua.max_states"; +static char const *const ts_lua_mgmt_state_regex = "^[1-9][0-9]*$"; + +// this is set the first time global configuration is probed. +static int ts_lua_max_state_count = 0; + +// lifecycle message tag +static char const *const print_tag = "stats_print"; +static char const *const reset_tag = "stats_reset"; + +// stat record strings +static char const *const ts_lua_stat_strs[] = { + "plugin.lua.remap.states", + "plugin.lua.remap.gc_bytes", + "plugin.lua.remap.threads", + NULL, +}; +static char const *const ts_lua_g_stat_strs[] = { + "plugin.lua.global.states", + "plugin.lua.global.gc_bytes", + "plugin.lua.global.threads", + NULL, +}; + +typedef struct { + ts_lua_main_ctx *main_ctx_array; + + int gc_kb; // last collected gc in kb + int threads; // last collected number active threads + + int stat_inds[TS_LUA_IND_SIZE]; // stats indices + +} ts_lua_plugin_stats; + +ts_lua_plugin_stats * +create_plugin_stats(ts_lua_main_ctx *const main_ctx_array, char const *const *stat_strs) +{ + ts_lua_plugin_stats *const stats = TSmalloc(sizeof(ts_lua_plugin_stats)); + memset(stats, 0, sizeof(ts_lua_plugin_stats)); + + stats->main_ctx_array = main_ctx_array; + + // sample buffers + stats->gc_kb = 0; + stats->threads = 0; + + int const max_state_count = ts_lua_max_state_count; + + for (int ind = 0; ind < TS_LUA_IND_SIZE; ++ind) { + stats->stat_inds[ind] = TSStatCreate(stat_strs[ind], TS_RECORDDATATYPE_INT, TS_STAT_NON_PERSISTENT, TS_STAT_SYNC_SUM); + } + + // initialize the number of states stat + int const sid = stats->stat_inds[TS_LUA_IND_STATE]; + if (TS_ERROR != sid) { + TSStatIntSet(sid, max_state_count); + } + + return stats; +} + +ts_lua_main_ctx * +create_lua_vms() +{ + ts_lua_main_ctx *ctx_array = NULL; + + // Inject the setting into records.config + static bool ts_mgt_int_inserted = false; + if (!ts_mgt_int_inserted) { + if (TS_SUCCESS == TSMgmtIntCreate(TS_RECORDTYPE_CONFIG, ts_lua_mgmt_state_str, TS_LUA_MAX_STATE_COUNT, + TS_RECORDUPDATE_RESTART_TS, TS_RECORDCHECK_INT, ts_lua_mgmt_state_regex, + TS_RECORDACCESS_READ_ONLY)) { + TSDebug(TS_LUA_DEBUG_TAG, "[%s] registered config string %s: with default [%d]", __FUNCTION__, ts_lua_mgmt_state_str, + TS_LUA_MAX_STATE_COUNT); + } else { + TSError("[%s][%s] failed to register %s", TS_LUA_DEBUG_TAG, __FUNCTION__, ts_lua_mgmt_state_str); + } + ts_mgt_int_inserted = true; + } + + if (0 == ts_lua_max_state_count) { + TSMgmtInt mgmt_state = 0; + + if (TS_SUCCESS != TSMgmtIntGet(ts_lua_mgmt_state_str, &mgmt_state)) { + TSDebug(TS_LUA_DEBUG_TAG, "[%s] setting max state to default: %d", __FUNCTION__, TS_LUA_MAX_STATE_COUNT); + ts_lua_max_state_count = TS_LUA_MAX_STATE_COUNT; + } else { + ts_lua_max_state_count = (int)mgmt_state; + TSDebug(TS_LUA_DEBUG_TAG, "[%s] found %s: [%d]", __FUNCTION__, ts_lua_mgmt_state_str, ts_lua_max_state_count); + } + + if (ts_lua_max_state_count < 1) { + TSError("[ts_lua][%s] invalid %s: %d", __FUNCTION__, ts_lua_mgmt_state_str, ts_lua_max_state_count); + ts_lua_max_state_count = 0; + return NULL; + } + } + + ctx_array = TSmalloc(sizeof(ts_lua_main_ctx) * ts_lua_max_state_count); + memset(ctx_array, 0, sizeof(ts_lua_main_ctx) * ts_lua_max_state_count); + + int const ret = ts_lua_create_vm(ctx_array, ts_lua_max_state_count); + + if (ret) { + ts_lua_destroy_vm(ctx_array, ts_lua_max_state_count); + TSfree(ctx_array); + ctx_array = NULL; + return NULL; + } + + // Initalize the GC numbers, no need to lock here + for (int index = 0; index < ts_lua_max_state_count; ++index) { + ts_lua_main_ctx *const main_ctx = (ctx_array + index); + lua_State *const lstate = main_ctx->lua; + ts_lua_ctx_stats *const stats = main_ctx->stats; + + stats->gc_kb = stats->gc_kb_max = lua_getgccount(lstate); + } + + return ctx_array; +} + +// dump exhaustive per state summary stats +static void +collectStats(ts_lua_plugin_stats *const plugin_stats) +{ + TSMgmtInt gc_kb_total = 0; + TSMgmtInt threads_total = 0; + + ts_lua_main_ctx *const main_ctx_array = plugin_stats->main_ctx_array; + + // aggregate stats on the states + for (int index = 0; index < ts_lua_max_state_count; ++index) { + ts_lua_main_ctx *const main_ctx = (main_ctx_array + index); + if (NULL != main_ctx) { + ts_lua_ctx_stats *const stats = main_ctx->stats; + + TSMutexLock(stats->mutexp); + gc_kb_total += (TSMgmtInt)stats->gc_kb; + threads_total += (TSMgmtInt)stats->threads; + TSMutexUnlock(stats->mutexp); + } + } + + // set the stats sample slot + plugin_stats->gc_kb = gc_kb_total; + plugin_stats->threads = threads_total; +} + +static void +publishStats(ts_lua_plugin_stats *const plugin_stats) +{ + TSMgmtInt const gc_bytes = plugin_stats->gc_kb * 1024; + TSStatIntSet(plugin_stats->stat_inds[TS_LUA_IND_GC_BYTES], gc_bytes); + TSStatIntSet(plugin_stats->stat_inds[TS_LUA_IND_THREADS], plugin_stats->threads); +} + +// dump exhaustive per state summary stats +static int +statsHandler(TSCont contp, TSEvent event, void *edata) +{ + ts_lua_plugin_stats *const plugin_stats = (ts_lua_plugin_stats *)TSContDataGet(contp); + + collectStats(plugin_stats); + publishStats(plugin_stats); + + TSContScheduleOnPool(contp, TS_LUA_STATS_TIMEOUT, TS_THREAD_POOL_TASK); + + return TS_EVENT_NONE; +} + +static void +get_time_now_str(char *const buf, size_t const buflen) +{ + TSHRTime const timenowusec = TShrtime(); + int64_t const timemsec = (int64_t)(timenowusec / 1000000); + time_t const timesec = (time_t)(timemsec / 1000); + int const ms = (int)(timemsec % 1000); + + struct tm tm; + gmtime_r(×ec, &tm); + size_t const dtlen = strftime(buf, buflen, "%b %e %H:%M:%S", &tm); + + // tack on the ms + snprintf(buf + dtlen, buflen - dtlen, ".%03d", ms); +} + +// dump exhaustive per state summary stats +static int +lifecycleHandler(TSCont contp, TSEvent event, void *edata) +{ + // ensure the message is for ts_lua + TSPluginMsg *const msgp = (TSPluginMsg *)edata; + if (0 != strncasecmp(msgp->tag, TS_LUA_DEBUG_TAG, strlen(msgp->tag))) { + return TS_EVENT_NONE; + } + + ts_lua_main_ctx *const main_ctx_array = (ts_lua_main_ctx *)TSContDataGet(contp); + + static char const *const remapstr = "remap"; + static char const *const globalstr = "global"; + + char const *labelstr = NULL; + + if (main_ctx_array == ts_lua_main_ctx_array) { + labelstr = remapstr; + } else { + labelstr = globalstr; + } + + char timebuf[128]; + get_time_now_str(timebuf, 128); + + char const *const msgstr = (char *)msgp->data; + enum State { Print, Reset } state; + state = Print; + size_t const reset_tag_len = strlen(reset_tag); + + if (reset_tag_len <= msgp->data_size && 0 == strncasecmp(reset_tag, msgstr, reset_tag_len)) { + TSDebug(TS_LUA_DEBUG_TAG, "[%s] LIFECYCLE_MSG: %s", __FUNCTION__, reset_tag); + state = Reset; + fprintf(stderr, "[%s] %s (%s) resetting per state gc_kb_max and threads_max\n", timebuf, TS_LUA_DEBUG_TAG, labelstr); + } else { + TSDebug(TS_LUA_DEBUG_TAG, "[%s] LIFECYCLE_MSG: %s", __FUNCTION__, print_tag); + } + + for (int index = 0; index < ts_lua_max_state_count; ++index) { + ts_lua_main_ctx *const main_ctx = (main_ctx_array + index); + if (NULL != main_ctx) { + ts_lua_ctx_stats *const stats = main_ctx->stats; + if (NULL != main_ctx) { + TSMutexLock(stats->mutexp); + + switch (state) { + case Reset: + stats->threads_max = stats->threads; + stats->gc_kb_max = stats->gc_kb; + break; + + case Print: + default: + fprintf(stderr, "[%s] %s (%s) id: %3d gc_kb: %6d gc_kb_max: %6d threads: %4d threads_max: %4d\n", timebuf, + TS_LUA_DEBUG_TAG, labelstr, index, stats->gc_kb, stats->gc_kb_max, stats->threads, stats->threads_max); + break; + } + + TSMutexUnlock(stats->mutexp); + } + } + } + + return TS_EVENT_NONE; +} TSReturnCode TSRemapInit(TSRemapInterface *api_info, char *errbuf, int errbuf_size) { - int ret; - if (!api_info || api_info->size < sizeof(TSRemapInterface)) { strncpy(errbuf, "[TSRemapInit] - Incorrect size of TSRemapInterface structure", errbuf_size - 1); errbuf[errbuf_size - 1] = '\0'; return TS_ERROR; } - if (ts_lua_main_ctx_array != NULL) { - return TS_SUCCESS; - } - - ts_lua_main_ctx_array = TSmalloc(sizeof(ts_lua_main_ctx) * TS_LUA_MAX_STATE_COUNT); - memset(ts_lua_main_ctx_array, 0, sizeof(ts_lua_main_ctx) * TS_LUA_MAX_STATE_COUNT); + if (NULL == ts_lua_main_ctx_array) { + ts_lua_main_ctx_array = create_lua_vms(); + if (NULL != ts_lua_main_ctx_array) { + TSCont const lcontp = TSContCreate(lifecycleHandler, TSMutexCreate()); + TSContDataSet(lcontp, ts_lua_main_ctx_array); + TSLifecycleHookAdd(TS_LIFECYCLE_MSG_HOOK, lcontp); - ret = ts_lua_create_vm(ts_lua_main_ctx_array, TS_LUA_MAX_STATE_COUNT); + ts_lua_plugin_stats *const plugin_stats = create_plugin_stats(ts_lua_main_ctx_array, ts_lua_stat_strs); - if (ret) { - ts_lua_destroy_vm(ts_lua_main_ctx_array, TS_LUA_MAX_STATE_COUNT); - TSfree(ts_lua_main_ctx_array); - return TS_ERROR; + // start the stats management + if (NULL != plugin_stats) { + TSDebug(TS_LUA_DEBUG_TAG, "Starting up stats management continuation"); + TSCont const scontp = TSContCreate(statsHandler, TSMutexCreate()); + TSContDataSet(scontp, plugin_stats); + TSContScheduleOnPool(scontp, TS_LUA_STATS_TIMEOUT, TS_THREAD_POOL_TASK); + } + } else { + return TS_ERROR; + } } return TS_SUCCESS; @@ -67,7 +335,7 @@ TSRemapNewInstance(int argc, char *argv[], void **ih, char *errbuf, int errbuf_s char script[TS_LUA_MAX_SCRIPT_FNAME_LENGTH]; char *inline_script = ""; int fn = 0; - int states = TS_LUA_MAX_STATE_COUNT; + int states = ts_lua_max_state_count; static const struct option longopt[] = { {"states", required_argument, 0, 's'}, {"inline", required_argument, 0, 'i'}, @@ -84,7 +352,7 @@ TSRemapNewInstance(int argc, char *argv[], void **ih, char *errbuf, int errbuf_s switch (opt) { case 's': states = atoi(optarg); - TSDebug(TS_LUA_DEBUG_TAG, "[%s] setting number of lua VM [%d]", __FUNCTION__, states); + TSDebug(TS_LUA_DEBUG_TAG, "[%s] setting number of lua VMs [%d]", __FUNCTION__, states); // set state break; case 'i': @@ -96,9 +364,9 @@ TSRemapNewInstance(int argc, char *argv[], void **ih, char *errbuf, int errbuf_s } } - if (states > TS_LUA_MAX_STATE_COUNT || states < 1) { + if (states < 1 || ts_lua_max_state_count < states) { snprintf(errbuf, errbuf_size, "[TSRemapNewInstance] - invalid state in option input. Must be between 1 and %d", - TS_LUA_MAX_STATE_COUNT); + ts_lua_max_state_count); return TS_ERROR; } @@ -130,7 +398,9 @@ TSRemapNewInstance(int argc, char *argv[], void **ih, char *errbuf, int errbuf_s TSDebug(TS_LUA_DEBUG_TAG, "[%s] checking if script has been registered", __FUNCTION__); // we only need to check the first lua VM for script registration + TSMutexLock(ts_lua_main_ctx_array[0].mutexp); conf = ts_lua_script_registered(ts_lua_main_ctx_array[0].lua, script); + TSMutexUnlock(ts_lua_main_ctx_array[0].mutexp); } if (!conf) { @@ -165,7 +435,9 @@ TSRemapNewInstance(int argc, char *argv[], void **ih, char *errbuf, int errbuf_s // register the script only if it is from a file and has no __init__ function if (fn && !conf->init_func) { // we only need to register the script for the first lua VM + TSMutexLock(ts_lua_main_ctx_array[0].mutexp); ts_lua_script_register(ts_lua_main_ctx_array[0].lua, conf->script, conf); + TSMutexUnlock(ts_lua_main_ctx_array[0].mutexp); } } @@ -455,19 +727,27 @@ TSPluginInit(int argc, const char *argv[]) TSError("[ts_lua] Plugin registration failed"); } - int ret = 0; - ts_lua_g_main_ctx_array = TSmalloc(sizeof(ts_lua_main_ctx) * TS_LUA_MAX_STATE_COUNT); - memset(ts_lua_g_main_ctx_array, 0, sizeof(ts_lua_main_ctx) * TS_LUA_MAX_STATE_COUNT); + if (NULL == ts_lua_g_main_ctx_array) { + ts_lua_g_main_ctx_array = create_lua_vms(); + if (NULL != ts_lua_g_main_ctx_array) { + TSCont const contp = TSContCreate(lifecycleHandler, TSMutexCreate()); + TSContDataSet(contp, ts_lua_g_main_ctx_array); + TSLifecycleHookAdd(TS_LIFECYCLE_MSG_HOOK, contp); - ret = ts_lua_create_vm(ts_lua_g_main_ctx_array, TS_LUA_MAX_STATE_COUNT); + ts_lua_plugin_stats *const plugin_stats = create_plugin_stats(ts_lua_g_main_ctx_array, ts_lua_g_stat_strs); - if (ret) { - ts_lua_destroy_vm(ts_lua_g_main_ctx_array, TS_LUA_MAX_STATE_COUNT); - TSfree(ts_lua_g_main_ctx_array); - return; + if (NULL != plugin_stats) { + TSCont const scontp = TSContCreate(statsHandler, TSMutexCreate()); + TSContDataSet(scontp, plugin_stats); + TSContScheduleOnPool(scontp, TS_LUA_STATS_TIMEOUT, TS_THREAD_POOL_TASK); + } + } else { + return; + } } - int states = TS_LUA_MAX_STATE_COUNT; + int states = ts_lua_max_state_count; + int reload = 0; static const struct option longopt[] = { {"states", required_argument, 0, 's'}, @@ -495,8 +775,8 @@ TSPluginInit(int argc, const char *argv[]) } } - if (states > TS_LUA_MAX_STATE_COUNT || states < 1) { - TSError("[ts_lua][%s] invalid # of states from option input. Must be between 1 and %d", __FUNCTION__, TS_LUA_MAX_STATE_COUNT); + if (states < 1 || ts_lua_max_state_count < states) { + TSError("[ts_lua][%s] invalid # of states from option input. Must be between 1 and %d", __FUNCTION__, ts_lua_max_state_count); return; } @@ -528,8 +808,9 @@ TSPluginInit(int argc, const char *argv[]) ts_lua_init_instance(conf); char errbuf[TS_LUA_MAX_STR_LENGTH]; - int errbuf_len = sizeof(errbuf); - ret = ts_lua_add_module(conf, ts_lua_g_main_ctx_array, conf->states, argc - optind, (char **)&argv[optind], errbuf, errbuf_len); + int const errbuf_len = sizeof(errbuf); + int const ret = + ts_lua_add_module(conf, ts_lua_g_main_ctx_array, conf->states, argc - optind, (char **)&argv[optind], errbuf, errbuf_len); if (ret != 0) { TSError(errbuf, NULL); diff --git a/plugins/lua/ts_lua_client_request.c b/plugins/lua/ts_lua_client_request.c index b851eb19d7e..a78b5b9f203 100644 --- a/plugins/lua/ts_lua_client_request.c +++ b/plugins/lua/ts_lua_client_request.c @@ -16,8 +16,8 @@ limitations under the License. */ -#include "tscore/ink_platform.h" #include +#include #include "ts_lua_util.h" static void ts_lua_inject_client_request_client_addr_api(lua_State *L); @@ -66,6 +66,15 @@ static int ts_lua_client_request_client_addr_get_port(lua_State *L); static int ts_lua_client_request_client_addr_get_addr(lua_State *L); static int ts_lua_client_request_client_addr_get_incoming_port(lua_State *L); +static void ts_lua_inject_client_request_ssl_reused_api(lua_State *L); +static int ts_lua_client_request_get_ssl_reused(lua_State *L); +static void ts_lua_inject_client_request_ssl_cipher_api(lua_State *L); +static int ts_lua_client_request_get_ssl_cipher(lua_State *L); +static void ts_lua_inject_client_request_ssl_protocol_api(lua_State *L); +static int ts_lua_client_request_get_ssl_protocol(lua_State *L); +static void ts_lua_inject_client_request_ssl_curve_api(lua_State *L); +static int ts_lua_client_request_get_ssl_curve(lua_State *L); + void ts_lua_inject_client_request_api(lua_State *L) { @@ -82,6 +91,10 @@ ts_lua_inject_client_request_api(lua_State *L) ts_lua_inject_client_request_version_api(L); ts_lua_inject_client_request_body_size_api(L); ts_lua_inject_client_request_header_size_api(L); + ts_lua_inject_client_request_ssl_reused_api(L); + ts_lua_inject_client_request_ssl_cipher_api(L); + ts_lua_inject_client_request_ssl_protocol_api(L); + ts_lua_inject_client_request_ssl_curve_api(L); lua_setfield(L, -2, "client_request"); } @@ -924,3 +937,118 @@ ts_lua_client_request_get_header_size(lua_State *L) return 1; } + +static void +ts_lua_inject_client_request_ssl_reused_api(lua_State *L) +{ + lua_pushcfunction(L, ts_lua_client_request_get_ssl_reused); + lua_setfield(L, -2, "get_ssl_reused"); +} + +static int +ts_lua_client_request_get_ssl_reused(lua_State *L) +{ + int ssl_reused = 0; + ts_lua_http_ctx *http_ctx; + TSHttpSsn ssnp; + TSVConn client_conn; + + GET_HTTP_CONTEXT(http_ctx, L); + ssnp = TSHttpTxnSsnGet(http_ctx->txnp); + client_conn = TSHttpSsnClientVConnGet(ssnp); + + if (TSVConnIsSsl(client_conn)) { + ssl_reused = TSVConnIsSslReused(client_conn); + } + + lua_pushnumber(L, ssl_reused); + + return 1; +} + +static void +ts_lua_inject_client_request_ssl_cipher_api(lua_State *L) +{ + lua_pushcfunction(L, ts_lua_client_request_get_ssl_cipher); + lua_setfield(L, -2, "get_ssl_cipher"); +} + +static int +ts_lua_client_request_get_ssl_cipher(lua_State *L) +{ + const char *ssl_cipher = "-"; + ts_lua_http_ctx *http_ctx; + TSHttpSsn ssnp; + TSVConn client_conn; + + GET_HTTP_CONTEXT(http_ctx, L); + + ssnp = TSHttpTxnSsnGet(http_ctx->txnp); + client_conn = TSHttpSsnClientVConnGet(ssnp); + + if (TSVConnIsSsl(client_conn)) { + ssl_cipher = TSVConnSslCipherGet(client_conn); + } + + lua_pushstring(L, ssl_cipher); + + return 1; +} + +static void +ts_lua_inject_client_request_ssl_protocol_api(lua_State *L) +{ + lua_pushcfunction(L, ts_lua_client_request_get_ssl_protocol); + lua_setfield(L, -2, "get_ssl_protocol"); +} + +static int +ts_lua_client_request_get_ssl_protocol(lua_State *L) +{ + const char *ssl_protocol = "-"; + ts_lua_http_ctx *http_ctx; + TSHttpSsn ssnp; + TSVConn client_conn; + + GET_HTTP_CONTEXT(http_ctx, L); + + ssnp = TSHttpTxnSsnGet(http_ctx->txnp); + client_conn = TSHttpSsnClientVConnGet(ssnp); + + if (TSVConnIsSsl(client_conn)) { + ssl_protocol = TSVConnSslProtocolGet(client_conn); + } + + lua_pushstring(L, ssl_protocol); + + return 1; +} + +static void +ts_lua_inject_client_request_ssl_curve_api(lua_State *L) +{ + lua_pushcfunction(L, ts_lua_client_request_get_ssl_curve); + lua_setfield(L, -2, "get_ssl_curve"); +} + +static int +ts_lua_client_request_get_ssl_curve(lua_State *L) +{ + const char *ssl_curve = "-"; + ts_lua_http_ctx *http_ctx; + TSHttpSsn ssnp; + TSVConn client_conn; + + GET_HTTP_CONTEXT(http_ctx, L); + + ssnp = TSHttpTxnSsnGet(http_ctx->txnp); + client_conn = TSHttpSsnClientVConnGet(ssnp); + + if (TSVConnIsSsl(client_conn)) { + ssl_curve = TSVConnSslCurveGet(client_conn); + } + + lua_pushstring(L, ssl_curve); + + return 1; +} diff --git a/plugins/lua/ts_lua_common.h b/plugins/lua/ts_lua_common.h index dba8b7d77e0..d076ae13db5 100644 --- a/plugins/lua/ts_lua_common.h +++ b/plugins/lua/ts_lua_common.h @@ -21,6 +21,7 @@ #include #include #include +#include #include "lua.h" #include "lualib.h" @@ -29,7 +30,6 @@ #include #include #include -#include "tscore/ink_defs.h" #include "ts_lua_coroutine.h" #define TS_LUA_FUNCTION_REMAP "do_remap" @@ -91,8 +91,8 @@ typedef struct { char script[TS_LUA_MAX_SCRIPT_FNAME_LENGTH]; void *conf_vars[TS_LUA_MAX_CONFIG_VARS_COUNT]; - int _first : 1; // create current instance for 1st ts_lua_main_ctx - int _last : 1; // create current instance for the last ts_lua_main_ctx + unsigned int _first : 1; // create current instance for 1st ts_lua_main_ctx + unsigned int _last : 1; // create current instance for the last ts_lua_main_ctx int remap; int states; @@ -160,10 +160,10 @@ typedef struct { ts_lua_http_ctx *hctx; int64_t to_flush; - int reuse : 1; - int recv_complete : 1; - int send_complete : 1; - int all_ready : 1; + unsigned int reuse : 1; + unsigned int recv_complete : 1; + unsigned int send_complete : 1; + unsigned int all_ready : 1; } ts_lua_http_intercept_ctx; #define TS_LUA_RELEASE_IO_HANDLE(ih) \ @@ -177,3 +177,7 @@ typedef struct { ih->buffer = NULL; \ } \ } while (0) + +#ifndef ATS_UNUSED +#define ATS_UNUSED __attribute__((unused)) +#endif diff --git a/plugins/lua/ts_lua_coroutine.h b/plugins/lua/ts_lua_coroutine.h index 831066dc24b..7b86b34a38b 100644 --- a/plugins/lua/ts_lua_coroutine.h +++ b/plugins/lua/ts_lua_coroutine.h @@ -28,11 +28,21 @@ struct async_item; typedef int (*async_clean)(struct async_item *item); +/* context stats */ +typedef struct { + TSMutex mutexp; // mutex for the following stats + int gc_kb; // last recorded gc kbytes + int gc_kb_max; // maximum recorded gc kbytes + int threads; // associated coroutines + int threads_max; // max coroutines +} ts_lua_ctx_stats; + /* main context*/ typedef struct { - lua_State *lua; // basic lua vm, injected - TSMutex mutexp; // mutex for lua vm - int gref; // reference for lua vm self, in reg table + lua_State *lua; // basic lua vm, injected + TSMutex mutexp; // mutex for lua vm + int gref; // reference for lua vm self, in reg table + ts_lua_ctx_stats *stats; // per vm stats } ts_lua_main_ctx; /* coroutine */ @@ -59,7 +69,7 @@ typedef struct async_item { void *data; // private data async_clean cleanup; // cleanup function - int deleted : 1; + unsigned int deleted : 1; } ts_lua_async_item; ts_lua_async_item *ts_lua_async_create_item(TSCont cont, async_clean func, void *d, ts_lua_cont_info *ci); diff --git a/plugins/lua/ts_lua_crypto.c b/plugins/lua/ts_lua_crypto.c index 73a79a965ab..b1e02d7ac3f 100644 --- a/plugins/lua/ts_lua_crypto.c +++ b/plugins/lua/ts_lua_crypto.c @@ -18,11 +18,13 @@ #include #include +#include #include "ts_lua_string.h" #include "ts_lua_util.h" #define TS_LUA_MD5_DIGEST_LENGTH 16 -#define TS_LUA_SHA_DIGEST_LENGTH 20 +#define TS_LUA_SHA1_DIGEST_LENGTH 20 +#define TS_LUA_SHA256_DIGEST_LENGTH 32 static int ts_lua_md5(lua_State *L); static int ts_lua_md5_bin(lua_State *L); @@ -30,6 +32,13 @@ static int ts_lua_md5_bin(lua_State *L); static int ts_lua_sha1(lua_State *L); static int ts_lua_sha1_bin(lua_State *L); +static int ts_lua_sha256(lua_State *L); +static int ts_lua_sha256_bin(lua_State *L); + +static int ts_lua_hmac_md5(lua_State *L); +static int ts_lua_hmac_sha1(lua_State *L); +static int ts_lua_hmac_sha256(lua_State *L); + static int ts_lua_base64_encode(lua_State *L); static int ts_lua_base64_decode(lua_State *L); @@ -47,7 +56,7 @@ ts_lua_inject_crypto_api(lua_State *L) lua_pushcfunction(L, ts_lua_md5_bin); lua_setfield(L, -2, "md5_bin"); - /* ts.sha1_bin(...) */ + /* ts.sha1(...) */ lua_pushcfunction(L, ts_lua_sha1); lua_setfield(L, -2, "sha1"); @@ -55,6 +64,26 @@ ts_lua_inject_crypto_api(lua_State *L) lua_pushcfunction(L, ts_lua_sha1_bin); lua_setfield(L, -2, "sha1_bin"); + /* ts.sha256(...) */ + lua_pushcfunction(L, ts_lua_sha256); + lua_setfield(L, -2, "sha256"); + + /* ts.sha256_bin(...) */ + lua_pushcfunction(L, ts_lua_sha256_bin); + lua_setfield(L, -2, "sha256_bin"); + + /* ts.hmac_md5(...) */ + lua_pushcfunction(L, ts_lua_hmac_md5); + lua_setfield(L, -2, "hmac_md5"); + + /* ts.hmac_sha1(...) */ + lua_pushcfunction(L, ts_lua_hmac_sha1); + lua_setfield(L, -2, "hmac_sha1"); + + /* ts.hmac_sha256(...) */ + lua_pushcfunction(L, ts_lua_hmac_sha256); + lua_setfield(L, -2, "hmac_sha256"); + /* ts.base64_encode(...) */ lua_pushcfunction(L, ts_lua_base64_encode); lua_setfield(L, -2, "base64_encode"); @@ -142,7 +171,7 @@ ts_lua_sha1(lua_State *L) size_t slen; SHA_CTX sha; - u_char sha_buf[TS_LUA_SHA_DIGEST_LENGTH]; + u_char sha_buf[TS_LUA_SHA1_DIGEST_LENGTH]; u_char hex_buf[2 * sizeof(sha_buf)]; if (lua_gettop(L) != 1) { @@ -174,7 +203,7 @@ ts_lua_sha1_bin(lua_State *L) size_t slen; SHA_CTX sha; - u_char sha_buf[TS_LUA_SHA_DIGEST_LENGTH]; + u_char sha_buf[TS_LUA_SHA1_DIGEST_LENGTH]; if (lua_gettop(L) != 1) { return luaL_error(L, "expecting one argument"); @@ -197,6 +226,233 @@ ts_lua_sha1_bin(lua_State *L) return 1; } +static int +ts_lua_sha256(lua_State *L) +{ + u_char *src; + size_t slen; + + SHA256_CTX sha; + u_char sha_buf[TS_LUA_SHA256_DIGEST_LENGTH]; + u_char hex_buf[2 * sizeof(sha_buf)]; + + if (lua_gettop(L) != 1) { + return luaL_error(L, "expecting one argument"); + } + + if (lua_isnil(L, 1)) { + src = (u_char *)""; + slen = 0; + + } else { + src = (u_char *)luaL_checklstring(L, 1, &slen); + } + + SHA256_Init(&sha); + SHA256_Update(&sha, src, slen); + SHA256_Final(sha_buf, &sha); + + ts_lua_hex_dump(hex_buf, sha_buf, sizeof(sha_buf)); + lua_pushlstring(L, (char *)hex_buf, sizeof(hex_buf)); + + return 1; +} + +static int +ts_lua_sha256_bin(lua_State *L) +{ + u_char *src; + size_t slen; + + SHA256_CTX sha; + u_char sha_buf[TS_LUA_SHA256_DIGEST_LENGTH]; + + if (lua_gettop(L) != 1) { + return luaL_error(L, "expecting one argument"); + } + + if (lua_isnil(L, 1)) { + src = (u_char *)""; + slen = 0; + + } else { + src = (u_char *)luaL_checklstring(L, 1, &slen); + } + + SHA256_Init(&sha); + SHA256_Update(&sha, src, slen); + SHA256_Final(sha_buf, &sha); + + lua_pushlstring(L, (char *)sha_buf, sizeof(sha_buf)); + + return 1; +} + +static int +ts_lua_hmac_md5(lua_State *L) +{ + u_char *key; + u_char *src; + size_t klen; + size_t slen; + + unsigned char *key_bin; + unsigned int key_bin_len; + + u_char sha_buf[TS_LUA_MD5_DIGEST_LENGTH]; + u_char hex_buf[2 * sizeof(sha_buf)]; + unsigned int output_length; + + if (lua_gettop(L) != 2) { + return luaL_error(L, "expecting two arguments"); + } + + if (lua_isnil(L, 1)) { + key = (u_char *)""; + klen = 0; + + } else { + key = (u_char *)luaL_checklstring(L, 1, &klen); + } + + if (lua_isnil(L, 2)) { + src = (u_char *)""; + slen = 0; + + } else { + src = (u_char *)luaL_checklstring(L, 2, &slen); + } + + key_bin = TSmalloc((int)((klen / 2) + 1)); + if (key_bin == NULL) { + TSDebug(TS_LUA_DEBUG_TAG, "unable to allocate buffer for hex to binary conversion"); + return luaL_error(L, "unable to allocate buffer for hex to binary conversion"); + } + if (ts_lua_hex_to_bin(key_bin, key, klen) == NULL) { + TSfree(key_bin); + return luaL_error(L, "hex to binary conversion failed"); + } + key_bin_len = klen / 2; + + HMAC(EVP_md5(), key_bin, key_bin_len, src, slen, sha_buf, &output_length); + + ts_lua_hex_dump(hex_buf, sha_buf, sizeof(sha_buf)); + lua_pushlstring(L, (char *)hex_buf, sizeof(hex_buf)); + + TSfree(key_bin); + return 1; +} + +static int +ts_lua_hmac_sha1(lua_State *L) +{ + u_char *key; + u_char *src; + size_t klen; + size_t slen; + + unsigned char *key_bin; + unsigned int key_bin_len; + + u_char sha_buf[TS_LUA_SHA1_DIGEST_LENGTH]; + u_char hex_buf[2 * sizeof(sha_buf)]; + unsigned int output_length; + + if (lua_gettop(L) != 2) { + return luaL_error(L, "expecting two arguments"); + } + + if (lua_isnil(L, 1)) { + key = (u_char *)""; + klen = 0; + + } else { + key = (u_char *)luaL_checklstring(L, 1, &klen); + } + + if (lua_isnil(L, 2)) { + src = (u_char *)""; + slen = 0; + + } else { + src = (u_char *)luaL_checklstring(L, 2, &slen); + } + + key_bin = TSmalloc((int)((klen / 2) + 1)); + if (key_bin == NULL) { + TSDebug(TS_LUA_DEBUG_TAG, "unable to allocate buffer for hex to binary conversion"); + return luaL_error(L, "unable to allocate buffer for hex to binary conversion"); + } + if (ts_lua_hex_to_bin(key_bin, key, klen) == NULL) { + TSfree(key_bin); + return luaL_error(L, "hex to binary conversion failed"); + } + key_bin_len = klen / 2; + + HMAC(EVP_sha1(), key_bin, key_bin_len, src, slen, sha_buf, &output_length); + + ts_lua_hex_dump(hex_buf, sha_buf, sizeof(sha_buf)); + lua_pushlstring(L, (char *)hex_buf, sizeof(hex_buf)); + + TSfree(key_bin); + return 1; +} + +static int +ts_lua_hmac_sha256(lua_State *L) +{ + u_char *key; + u_char *src; + size_t klen; + size_t slen; + + unsigned char *key_bin; + unsigned int key_bin_len; + + u_char sha_buf[TS_LUA_SHA256_DIGEST_LENGTH]; + u_char hex_buf[2 * sizeof(sha_buf)]; + unsigned int output_length; + + if (lua_gettop(L) != 2) { + return luaL_error(L, "expecting two arguments"); + } + + if (lua_isnil(L, 1)) { + key = (u_char *)""; + klen = 0; + + } else { + key = (u_char *)luaL_checklstring(L, 1, &klen); + } + + if (lua_isnil(L, 2)) { + src = (u_char *)""; + slen = 0; + + } else { + src = (u_char *)luaL_checklstring(L, 2, &slen); + } + + key_bin = TSmalloc((int)((klen / 2) + 1)); + if (key_bin == NULL) { + TSDebug(TS_LUA_DEBUG_TAG, "unable to allocate buffer for hex to binary conversion"); + return luaL_error(L, "unable to allocate buffer for hex to binary conversion"); + } + if (ts_lua_hex_to_bin(key_bin, key, klen) == NULL) { + TSfree(key_bin); + return luaL_error(L, "hex to binary conversion failed"); + } + key_bin_len = klen / 2; + + HMAC(EVP_sha256(), key_bin, key_bin_len, src, slen, sha_buf, &output_length); + + ts_lua_hex_dump(hex_buf, sha_buf, sizeof(sha_buf)); + lua_pushlstring(L, (char *)hex_buf, sizeof(hex_buf)); + + TSfree(key_bin); + return 1; +} + static int ts_lua_base64_encode(lua_State *L) { @@ -316,8 +572,8 @@ ts_lua_unescape_uri(lua_State *L) return 1; } - /* the unescaped string can only be smaller */ - dlen = len; + /* the unescaped string can not be larger, but need to account for terminating null. */ + dlen = len + 1; dst = lua_newuserdata(L, dlen); if (TS_SUCCESS == TSStringPercentDecode((const char *)src, len, (char *)dst, dlen, &length)) { diff --git a/plugins/lua/ts_lua_fetch.h b/plugins/lua/ts_lua_fetch.h index 04c0d47a56c..c4b600cc4d8 100644 --- a/plugins/lua/ts_lua_fetch.h +++ b/plugins/lua/ts_lua_fetch.h @@ -30,8 +30,8 @@ typedef struct { TSIOBufferReader reader; TSFetchSM fch; - int over : 1; - int failed : 1; + unsigned int over : 1; + unsigned int failed : 1; } ts_lua_fetch_info; typedef struct fetch_multi_info { diff --git a/plugins/lua/ts_lua_http_config.c b/plugins/lua/ts_lua_http_config.c index dd72d147415..d9738be80d6 100644 --- a/plugins/lua/ts_lua_http_config.c +++ b/plugins/lua/ts_lua_http_config.c @@ -136,6 +136,7 @@ typedef enum { TS_LUA_CONFIG_SSL_CLIENT_SNI_POLICY = TS_CONFIG_SSL_CLIENT_SNI_POLICY, TS_LUA_CONFIG_SSL_CLIENT_PRIVATE_KEY_FILENAME = TS_CONFIG_SSL_CLIENT_PRIVATE_KEY_FILENAME, TS_LUA_CONFIG_SSL_CLIENT_CA_CERT_FILENAME = TS_CONFIG_SSL_CLIENT_CA_CERT_FILENAME, + TS_LUA_CONFIG_HTTP_HOST_RESOLUTION_PREFERENCE = TS_CONFIG_HTTP_HOST_RESOLUTION_PREFERENCE, TS_LUA_CONFIG_LAST_ENTRY = TS_CONFIG_LAST_ENTRY, } TSLuaOverridableConfigKey; @@ -261,6 +262,7 @@ ts_lua_var_item ts_lua_http_config_vars[] = { TS_LUA_MAKE_VAR_ITEM(TS_CONFIG_SSL_CLIENT_SNI_POLICY), TS_LUA_MAKE_VAR_ITEM(TS_CONFIG_SSL_CLIENT_PRIVATE_KEY_FILENAME), TS_LUA_MAKE_VAR_ITEM(TS_CONFIG_SSL_CLIENT_CA_CERT_FILENAME), + TS_LUA_MAKE_VAR_ITEM(TS_CONFIG_HTTP_HOST_RESOLUTION_PREFERENCE), TS_LUA_MAKE_VAR_ITEM(TS_CONFIG_HTTP_SERVER_MIN_KEEP_ALIVE_CONNS), TS_LUA_MAKE_VAR_ITEM(TS_LUA_CONFIG_HTTP_PER_SERVER_CONNECTION_MAX), TS_LUA_MAKE_VAR_ITEM(TS_LUA_CONFIG_HTTP_PER_SERVER_CONNECTION_MATCH), diff --git a/plugins/lua/ts_lua_misc.c b/plugins/lua/ts_lua_misc.c index e12e5a1fb1b..30e362a3f26 100644 --- a/plugins/lua/ts_lua_misc.c +++ b/plugins/lua/ts_lua_misc.c @@ -16,8 +16,8 @@ limitations under the License. */ -#include "tscore/ink_platform.h" #include +#include #include "ts_lua_util.h" static int ts_lua_get_process_id(lua_State *L); diff --git a/plugins/lua/ts_lua_server_request.c b/plugins/lua/ts_lua_server_request.c index c9272176efc..6b5cf85db38 100644 --- a/plugins/lua/ts_lua_server_request.c +++ b/plugins/lua/ts_lua_server_request.c @@ -16,8 +16,8 @@ limitations under the License. */ -#include "tscore/ink_platform.h" #include +#include #include "ts_lua_util.h" #define TS_LUA_CHECK_SERVER_REQUEST_HDR(http_ctx) \ @@ -51,6 +51,7 @@ static void ts_lua_inject_server_request_uri_api(lua_State *L); static void ts_lua_inject_server_request_uri_args_api(lua_State *L); static void ts_lua_inject_server_request_uri_params_api(lua_State *L); static void ts_lua_inject_server_request_url_api(lua_State *L); +static void ts_lua_inject_server_request_method_api(lua_State *L); static int ts_lua_server_request_header_get(lua_State *L); static int ts_lua_server_request_header_set(lua_State *L); @@ -63,10 +64,14 @@ static int ts_lua_server_request_set_uri_args(lua_State *L); static int ts_lua_server_request_get_uri_args(lua_State *L); static int ts_lua_server_request_set_uri_params(lua_State *L); static int ts_lua_server_request_get_uri_params(lua_State *L); +static int ts_lua_server_request_get_method(lua_State *L); +static int ts_lua_server_request_set_method(lua_State *L); static int ts_lua_server_request_get_url_host(lua_State *L); static int ts_lua_server_request_set_url_host(lua_State *L); static int ts_lua_server_request_get_url_scheme(lua_State *L); static int ts_lua_server_request_set_url_scheme(lua_State *L); +static int ts_lua_server_request_get_version(lua_State *L); +static int ts_lua_server_request_set_version(lua_State *L); static int ts_lua_server_request_server_addr_get_ip(lua_State *L); static int ts_lua_server_request_server_addr_get_port(lua_State *L); @@ -87,7 +92,7 @@ ts_lua_inject_server_request_api(lua_State *L) ts_lua_inject_server_request_headers_api(L); ts_lua_inject_server_request_get_header_size_api(L); ts_lua_inject_server_request_get_body_size_api(L); - + ts_lua_inject_server_request_method_api(L); ts_lua_inject_server_request_uri_api(L); ts_lua_inject_server_request_uri_args_api(L); ts_lua_inject_server_request_uri_params_api(L); @@ -563,6 +568,11 @@ ts_lua_inject_server_request_url_api(lua_State *L) lua_setfield(L, -2, "get_url_scheme"); lua_pushcfunction(L, ts_lua_server_request_set_url_scheme); lua_setfield(L, -2, "set_url_scheme"); + + lua_pushcfunction(L, ts_lua_server_request_get_version); + lua_setfield(L, -2, "get_version"); + lua_pushcfunction(L, ts_lua_server_request_set_version); + lua_setfield(L, -2, "set_version"); } static int @@ -659,6 +669,55 @@ ts_lua_server_request_set_url_scheme(lua_State *L) return 0; } +static int +ts_lua_server_request_get_version(lua_State *L) +{ + int version; + char buf[32]; + int n; + + ts_lua_http_ctx *http_ctx; + + GET_HTTP_CONTEXT(http_ctx, L); + TS_LUA_CHECK_SERVER_REQUEST_HDR(http_ctx); + + version = TSHttpHdrVersionGet(http_ctx->server_request_bufp, http_ctx->server_request_hdrp); + + n = snprintf(buf, sizeof(buf), "%d.%d", TS_HTTP_MAJOR(version), TS_HTTP_MINOR(version)); + + if (n >= (int)sizeof(buf)) { + lua_pushlstring(L, buf, sizeof(buf) - 1); + } else if (n > 0) { + lua_pushlstring(L, buf, n); + } + + return 1; +} + +static int +ts_lua_server_request_set_version(lua_State *L) +{ + const char *version; + size_t len; + int major, minor; + + ts_lua_http_ctx *http_ctx; + + GET_HTTP_CONTEXT(http_ctx, L); + + TS_LUA_CHECK_SERVER_REQUEST_HDR(http_ctx); + + version = luaL_checklstring(L, 1, &len); + + if (sscanf(version, "%2u.%2u", &major, &minor) != 2) { + return luaL_error(L, "failed to set version. Format must be X.Y"); + } + + TSHttpHdrVersionSet(http_ctx->server_request_bufp, http_ctx->server_request_hdrp, TS_HTTP_VERSION(major, minor)); + + return 0; +} + static int ts_lua_server_request_server_addr_get_ip(lua_State *L) { @@ -924,3 +983,53 @@ ts_lua_server_request_server_addr_set_outgoing_addr(lua_State *L) return 0; } + +static void +ts_lua_inject_server_request_method_api(lua_State *L) +{ + lua_pushcfunction(L, ts_lua_server_request_get_method); + lua_setfield(L, -2, "get_method"); + + lua_pushcfunction(L, ts_lua_server_request_set_method); + lua_setfield(L, -2, "set_method"); +} + +static int +ts_lua_server_request_get_method(lua_State *L) +{ + const char *method; + int method_len; + + ts_lua_http_ctx *http_ctx; + + GET_HTTP_CONTEXT(http_ctx, L); + + method = TSHttpHdrMethodGet(http_ctx->server_request_bufp, http_ctx->server_request_hdrp, &method_len); + + if (method && method_len) { + lua_pushlstring(L, method, method_len); + } else { + lua_pushnil(L); + } + + return 1; +} + +static int +ts_lua_server_request_set_method(lua_State *L) +{ + const char *method; + size_t method_len; + + ts_lua_http_ctx *http_ctx; + + GET_HTTP_CONTEXT(http_ctx, L); + + method = luaL_checklstring(L, 1, &method_len); + + if (method) { + TSHttpHdrMethodSet(http_ctx->server_request_bufp, http_ctx->server_request_hdrp, method, method_len); + } + + return 0; +} diff --git a/plugins/lua/ts_lua_string.c b/plugins/lua/ts_lua_string.c index 76747a65dc1..727b5a30b47 100644 --- a/plugins/lua/ts_lua_string.c +++ b/plugins/lua/ts_lua_string.c @@ -17,6 +17,7 @@ */ #include "ts_lua_string.h" +#include "ts_lua_util.h" u_char * ts_lua_hex_dump(u_char *dst, u_char *src, size_t len) @@ -30,3 +31,40 @@ ts_lua_hex_dump(u_char *dst, u_char *src, size_t len) return dst; } + +unsigned char +hex_to_int(unsigned char c) +{ + if (c >= '0' && c <= '9') { + return (c - '0'); + } + if (c >= 'A' && c <= 'F') { + return (c - 'A' + 10); + } + if (c >= 'a' && c <= 'f') { + return (c - 'a' + 10); + } + return 255; +} + +u_char * +ts_lua_hex_to_bin(u_char *dst, u_char *src, size_t len) +{ + if (len % 2 != 0) { + TSDebug(TS_LUA_DEBUG_TAG, "ts_lua_hex_to_bin(): not an even number of hex digits"); + return NULL; + } + + for (unsigned int x = 0; x < len; x += 2) { + unsigned char a = hex_to_int(src[x]); + unsigned char b = hex_to_int(src[x + 1]); + if (a == 255 || b == 255) { + TSDebug(TS_LUA_DEBUG_TAG, "ts_lua_hex_to_bin(): failure in hex to binary conversion"); + return NULL; + } + unsigned char result = (a << 4) + b; + dst[x / 2] = result; + } + + return dst; +} diff --git a/plugins/lua/ts_lua_string.h b/plugins/lua/ts_lua_string.h index 35c5e652805..bc36a6c4f30 100644 --- a/plugins/lua/ts_lua_string.h +++ b/plugins/lua/ts_lua_string.h @@ -24,3 +24,4 @@ #include u_char *ts_lua_hex_dump(u_char *dst, u_char *src, size_t len); +u_char *ts_lua_hex_to_bin(u_char *dst, u_char *src, size_t len); diff --git a/plugins/lua/ts_lua_util.c b/plugins/lua/ts_lua_util.c index 34d2b1ea54e..76d86fd88c6 100644 --- a/plugins/lua/ts_lua_util.c +++ b/plugins/lua/ts_lua_util.c @@ -40,6 +40,8 @@ static lua_State *ts_lua_new_state(); static void ts_lua_init_registry(lua_State *L); static void ts_lua_init_globals(lua_State *L); static void ts_lua_inject_ts_api(lua_State *L); +static ts_lua_ctx_stats *ts_lua_create_ctx_stats(); +static void ts_lua_destroy_ctx_stats(ts_lua_ctx_stats *stats); int ts_lua_create_vm(ts_lua_main_ctx *arr, int n) @@ -58,6 +60,7 @@ ts_lua_create_vm(ts_lua_main_ctx *arr, int n) arr[i].gref = luaL_ref(L, LUA_REGISTRYINDEX); /* L[REG][gref] = L[GLOBAL] */ arr[i].lua = L; arr[i].mutexp = TSMutexCreate(); + arr[i].stats = ts_lua_create_ctx_stats(); } return 0; @@ -68,11 +71,26 @@ ts_lua_destroy_vm(ts_lua_main_ctx *arr, int n) { int i; lua_State *L; + TSMutex mutexp; + ts_lua_ctx_stats *stats; for (i = 0; i < n; i++) { L = arr[i].lua; - if (L) + if (L) { lua_close(L); + arr[i].lua = NULL; + } + mutexp = arr[i].mutexp; + if (mutexp) { + TSMutexDestroy(mutexp); + arr[i].mutexp = NULL; + } + + stats = arr[i].stats; + if (stats) { + ts_lua_destroy_ctx_stats(stats); + arr[i].stats = NULL; + } } return; @@ -98,6 +116,29 @@ ts_lua_new_state() return L; } +ts_lua_ctx_stats * +ts_lua_create_ctx_stats() +{ + ts_lua_ctx_stats *stats = NULL; + + stats = TSmalloc(sizeof(ts_lua_ctx_stats)); + memset(stats, 0, sizeof(ts_lua_ctx_stats)); + + stats->mutexp = TSMutexCreate(); + + return stats; +} + +void +ts_lua_destroy_ctx_stats(ts_lua_ctx_stats *stats) +{ + if (stats) { + TSMutexDestroy(stats->mutexp); + stats->mutexp = NULL; + TSfree(stats); + } +} + ts_lua_instance_conf * ts_lua_script_registered(lua_State *L, char *script) { @@ -264,8 +305,6 @@ ts_lua_add_module(ts_lua_instance_conf *conf, ts_lua_main_ctx *arr, int n, int a lua_newtable(L); lua_replace(L, LUA_GLOBALSINDEX); /* L[GLOBAL] = EMPTY */ - lua_gc(L, LUA_GCCOLLECT, 0); - TSMutexUnlock(arr[i].mutexp); } @@ -306,8 +345,6 @@ ts_lua_del_module(ts_lua_instance_conf *conf, ts_lua_main_ctx *arr, int n) lua_newtable(L); lua_replace(L, LUA_GLOBALSINDEX); /* L[GLOBAL] = EMPTY */ - lua_gc(L, LUA_GCCOLLECT, 0); - TSMutexUnlock(arr[i].mutexp); } @@ -369,8 +406,6 @@ ts_lua_reload_module(ts_lua_instance_conf *conf, ts_lua_main_ctx *arr, int n) lua_newtable(L); lua_replace(L, LUA_GLOBALSINDEX); /* L[GLOBAL] = EMPTY */ - lua_gc(L, LUA_GCCOLLECT, 0); - TSMutexUnlock(arr[i].mutexp); } @@ -501,6 +536,17 @@ ts_lua_create_async_ctx(lua_State *L, ts_lua_cont_info *hci, int n) crt->lua = l; crt->ref = luaL_ref(L, LUA_REGISTRYINDEX); + // update thread stats + ts_lua_main_ctx *const main_ctx = crt->mctx; + ts_lua_ctx_stats *const stats = main_ctx->stats; + + TSMutexLock(stats->mutexp); + ++stats->threads; + if (stats->threads_max < stats->threads) { + stats->threads_max = stats->threads; + } + TSMutexUnlock(stats->mutexp); + // replace the param; start with 2 because first two params are not needed for (i = 2; i < n; i++) { lua_pushvalue(L, i + 1); @@ -518,6 +564,14 @@ ts_lua_destroy_async_ctx(ts_lua_http_ctx *http_ctx) ci = &http_ctx->cinfo; + // update thread stats + ts_lua_main_ctx *const main_ctx = ci->routine.mctx; + ts_lua_ctx_stats *const stats = main_ctx->stats; + + TSMutexLock(stats->mutexp); + --stats->threads; + TSMutexUnlock(stats->mutexp); + ts_lua_release_cont_info(ci); TSfree(http_ctx); } @@ -580,6 +634,16 @@ ts_lua_create_http_ctx(ts_lua_main_ctx *main_ctx, ts_lua_instance_conf *conf) crt->lua = l; crt->mctx = main_ctx; + // update thread stats + ts_lua_ctx_stats *const stats = main_ctx->stats; + + TSMutexLock(stats->mutexp); + ++stats->threads; + if (stats->threads_max < stats->threads) { + stats->threads_max = stats->threads; + } + TSMutexUnlock(stats->mutexp); + http_ctx->instance_conf = conf; ts_lua_set_http_ctx(l, http_ctx); @@ -623,6 +687,14 @@ ts_lua_destroy_http_ctx(ts_lua_http_ctx *http_ctx) TSMBufferDestroy(http_ctx->cached_response_bufp); } + // update thread stats + ts_lua_main_ctx *const main_ctx = ci->routine.mctx; + ts_lua_ctx_stats *const stats = main_ctx->stats; + + TSMutexLock(stats->mutexp); + --stats->threads; + TSMutexUnlock(stats->mutexp); + ts_lua_release_cont_info(ci); TSfree(http_ctx); } @@ -806,6 +878,7 @@ ts_lua_http_cont_handler(TSCont contp, TSEvent ev, void *edata) rc = ret = 0; TSMutexLock(main_ctx->mutexp); + ts_lua_set_cont_info(L, ci); switch (event) { @@ -951,8 +1024,23 @@ ts_lua_http_cont_handler(TSCont contp, TSEvent ev, void *edata) break; } + // current memory in use by this state + int const gc_kb = lua_getgccount(L); + TSMutexUnlock(main_ctx->mutexp); + // collect state memory stats + ts_lua_ctx_stats *const stats = main_ctx->stats; + + TSMutexLock(stats->mutexp); + if (gc_kb != stats->gc_kb) { + stats->gc_kb = gc_kb; + if (stats->gc_kb_max < stats->gc_kb) { + stats->gc_kb_max = stats->gc_kb; + } + } + TSMutexUnlock(stats->mutexp); + if (rc == 0) { TSHttpTxnReenable(txnp, TS_EVENT_HTTP_CONTINUE); diff --git a/plugins/multiplexer/ats-multiplexer.cc b/plugins/multiplexer/ats-multiplexer.cc index e89f23dfe10..d539b50784d 100644 --- a/plugins/multiplexer/ats-multiplexer.cc +++ b/plugins/multiplexer/ats-multiplexer.cc @@ -118,6 +118,8 @@ DoRemap(const Instance &i, TSHttpTxn t) CHECK(TSMimeHdrFieldValueStringSet(buffer, location, field, -1, "original", 8)); CHECK(TSMimeHdrFieldAppend(buffer, location, field)); + + CHECK(TSHandleMLocRelease(buffer, location, field)); } Requests requests; diff --git a/plugins/multiplexer/dispatch.cc b/plugins/multiplexer/dispatch.cc index 675b23a349f..d5f4c3a75fb 100644 --- a/plugins/multiplexer/dispatch.cc +++ b/plugins/multiplexer/dispatch.cc @@ -168,9 +168,9 @@ class Handler if (TSIsDebugTagSet(PLUGIN_TAG) > 0) { const TSIOBuffer buffer = TSIOBufferCreate(); TSHttpHdrPrint(b, l, buffer); - std::string b; - read(buffer, b); - TSDebug(PLUGIN_TAG, "Response header for \"%s\" was:\n%s", url.c_str(), b.c_str()); + std::string buf; + read(buffer, buf); + TSDebug(PLUGIN_TAG, "Response header for \"%s\" was:\n%s", url.c_str(), buf.c_str()); TSIOBufferDestroy(buffer); } } diff --git a/plugins/regex_remap/regex_remap.cc b/plugins/regex_remap/regex_remap.cc index 0426a6cc0fb..27201407ca7 100644 --- a/plugins/regex_remap/regex_remap.cc +++ b/plugins/regex_remap/regex_remap.cc @@ -394,7 +394,8 @@ RemapRegex::compile(const char *&error, int &erroffset) return -1; } - _extra->match_limit_recursion = 2047; + // POOMA - also dependent on actual stack size. Crashes with previous value of 2047, + _extra->match_limit_recursion = 1750; _extra->flags |= PCRE_EXTRA_MATCH_LIMIT_RECURSION; if (pcre_fullinfo(_rex, _extra, PCRE_INFO_CAPTURECOUNT, &ccount) != 0) { diff --git a/plugins/s3_auth/aws_auth_v4.cc b/plugins/s3_auth/aws_auth_v4.cc index 072a900065f..3f9aea07777 100644 --- a/plugins/s3_auth/aws_auth_v4.cc +++ b/plugins/s3_auth/aws_auth_v4.cc @@ -385,7 +385,11 @@ getCanonicalRequestSha256Hash(TsInterface &api, bool signPayload, const StringSe const char *trimValue = trimWhiteSpaces(value, valueLen, trimValueLen); signedHeadersSet.insert(lowercaseName); - headersMap[lowercaseName] = String(trimValue, trimValueLen); + if (headersMap.find(lowercaseName) == headersMap.end()) { + headersMap[lowercaseName] = String(trimValue, trimValueLen); + } else { + headersMap[lowercaseName].append(",").append(String(trimValue, trimValueLen)); + } } for (const auto &it : signedHeadersSet) { diff --git a/plugins/s3_auth/aws_auth_v4.h b/plugins/s3_auth/aws_auth_v4.h index aa929de57be..865a199385f 100644 --- a/plugins/s3_auth/aws_auth_v4.h +++ b/plugins/s3_auth/aws_auth_v4.h @@ -36,6 +36,7 @@ typedef std::string String; typedef std::set StringSet; typedef std::map StringMap; +typedef std::multimap HeaderMultiMap; class HeaderIterator; diff --git a/plugins/s3_auth/s3_auth.cc b/plugins/s3_auth/s3_auth.cc index ce760201902..426140650ac 100644 --- a/plugins/s3_auth/s3_auth.cc +++ b/plugins/s3_auth/s3_auth.cc @@ -41,8 +41,6 @@ #include #include "tscore/ink_config.h" -// Special snowflake here, only available when building inside the ATS source tree. -#include "tscore/ink_atomic.h" #include "aws_auth_v4.h" /////////////////////////////////////////////////////////////////////////////// @@ -203,29 +201,11 @@ class S3Config } } else { /* 4 == _version */ - if (_virt_host_modified) { - TSError("[%s] virtual host not used with AWS auth v4, parameter ignored", PLUGIN_NAME); - } + // NOTE: virtual host not used with AWS auth v4, parameter ignored } return true; } - void - acquire() - { - ink_atomic_increment(&_ref_count, 1); - } - - void - release() - { - TSDebug(PLUGIN_NAME, "ref_count is %d", _ref_count); - if (1 >= ink_atomic_decrement(&_ref_count, 1)) { - TSDebug(PLUGIN_NAME, "configuration deleted, due to ref-counting"); - delete this; - } - } - // Used to copy relevant configurations that can be configured in a config file. Note: we intentionally // don't override/use the assignment operator, since we only copy things IF they have been modified. void @@ -409,7 +389,6 @@ class S3Config schedule(TSHttpTxn txnp) const { TSHttpTxnHookAdd(txnp, TS_HTTP_SEND_REQUEST_HDR_HOOK, _cont); - TSHttpTxnHookAdd(txnp, TS_HTTP_TXN_CLOSE_HOOK, _cont); // To release the config lease } private: @@ -424,7 +403,6 @@ class S3Config bool _version_modified = false; bool _virt_host_modified = false; TSCont _cont = nullptr; - int _ref_count = 1; StringSet _v4includeHeaders; bool _v4includeHeaders_modified = false; StringSet _v4excludeHeaders; @@ -524,15 +502,11 @@ ConfigCache::get(const char *fname) TSDebug(PLUGIN_NAME, "Configuration from %s is stale, reloading", config_fname.c_str()); it->second.second = tv.tv_sec; - if (nullptr != it->second.first) { - // The previous config update / reload attempt did not fail, safe to call release. - it->second.first->release(); - } if (s3->parse_config(config_fname)) { it->second.first = s3; } else { // Failed the configuration parse... Set the cache response to nullptr - s3->release(); + delete s3; it->second.first = nullptr; } } else { @@ -547,7 +521,7 @@ ConfigCache::get(const char *fname) _cache[config_fname] = std::make_pair(s3, tv.tv_sec); TSDebug(PLUGIN_NAME, "Parsing and caching configuration from %s, version:%d", config_fname.c_str(), s3->version()); } else { - s3->release(); + delete s3; return nullptr; } @@ -913,9 +887,6 @@ event_handler(TSCont cont, TSEvent event, void *edata) enable_event = TS_EVENT_HTTP_ERROR; } break; - case TS_EVENT_HTTP_TXN_CLOSE: - s3->release(); // Release the configuration lease when txn closes - break; default: TSError("[%s] Unknown event for this plugin", PLUGIN_NAME); TSDebug(PLUGIN_NAME, "unknown event for this plugin"); @@ -983,7 +954,6 @@ TSRemapNewInstance(int argc, char *argv[], void **ih, char * /* errbuf ATS_UNUSE if (!file_config) { TSError("[%s] invalid configuration file, %s", PLUGIN_NAME, optarg); *ih = nullptr; - s3->release(); return TS_ERROR; } break; @@ -1026,12 +996,10 @@ TSRemapNewInstance(int argc, char *argv[], void **ih, char * /* errbuf ATS_UNUSE // Make sure we got both the shared secret and the AWS secret if (!s3->valid()) { TSError("[%s] requires both shared and AWS secret configuration", PLUGIN_NAME); - s3->release(); *ih = nullptr; return TS_ERROR; } - // Note that we don't acquire() the s3 config, it's implicit that we hold at least one ref *ih = static_cast(s3); TSDebug(PLUGIN_NAME, "New rule: access_key=%s, virtual_host=%s, version=%d", s3->keyid(), s3->virt_host() ? "yes" : "no", s3->version()); @@ -1043,8 +1011,7 @@ void TSRemapDeleteInstance(void *ih) { S3Config *s3 = static_cast(ih); - - s3->release(); + delete s3; } /////////////////////////////////////////////////////////////////////////////// @@ -1057,7 +1024,6 @@ TSRemapDoRemap(void *ih, TSHttpTxn txnp, TSRemapRequestInfo * /* rri */) if (s3) { TSAssert(s3->valid()); - s3->acquire(); // Increase ref-count // Now schedule the continuation to update the URL when going to origin. // Note that in most cases, this is a No-Op, assuming you have reasonable // cache hit ratio. However, the scheduling is next to free (very cheap). diff --git a/plugins/s3_auth/unit_tests/test_aws_auth_v4.cc b/plugins/s3_auth/unit_tests/test_aws_auth_v4.cc index 4bf58af5da4..595fe003e26 100644 --- a/plugins/s3_auth/unit_tests/test_aws_auth_v4.cc +++ b/plugins/s3_auth/unit_tests/test_aws_auth_v4.cc @@ -405,10 +405,10 @@ TEST_CASE("AWSAuthSpecByExample: GET Object", "[AWS][auth][SpecByExample]") api._host.assign("examplebucket.s3.amazonaws.com"); api._path.assign("test.txt"); api._query.assign(""); - api._headers["Host"] = "examplebucket.s3.amazonaws.com"; - api._headers["Range"] = "bytes=0-9"; - api._headers["x-amz-content-sha256"] = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"; - api._headers["x-amz-date"] = "20130524T000000Z"; + api._headers.insert(std::make_pair("Host", "examplebucket.s3.amazonaws.com")); + api._headers.insert(std::make_pair("Range", "bytes=0-9")); + api._headers.insert(std::make_pair("x-amz-content-sha256", "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855")); + api._headers.insert(std::make_pair("x-amz-date", "20130524T000000Z")); const char *bench[] = { /* Authorization Header */ @@ -450,9 +450,9 @@ TEST_CASE("AWSAuthSpecByExample: GET Bucket Lifecycle", "[AWS][auth][SpecByExamp api._host.assign("examplebucket.s3.amazonaws.com"); api._path.assign(""); api._query.assign("lifecycle"); - api._headers["Host"] = "examplebucket.s3.amazonaws.com"; - api._headers["x-amz-content-sha256"] = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"; - api._headers["x-amz-date"] = "20130524T000000Z"; + api._headers.insert(std::make_pair("Host", "examplebucket.s3.amazonaws.com")); + api._headers.insert(std::make_pair("x-amz-content-sha256", "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855")); + api._headers.insert(std::make_pair("x-amz-date", "20130524T000000Z")); const char *bench[] = { /* Authorization Header */ @@ -494,9 +494,9 @@ TEST_CASE("AWSAuthSpecByExample: Get Bucket List Objects", "[AWS][auth][SpecByEx api._host.assign("examplebucket.s3.amazonaws.com"); api._path.assign(""); api._query.assign("max-keys=2&prefix=J"); - api._headers["Host"] = "examplebucket.s3.amazonaws.com"; - api._headers["x-amz-content-sha256"] = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"; - api._headers["x-amz-date"] = "20130524T000000Z"; + api._headers.insert(std::make_pair("Host", "examplebucket.s3.amazonaws.com")); + api._headers.insert(std::make_pair("x-amz-content-sha256", "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855")); + api._headers.insert(std::make_pair("x-amz-date", "20130524T000000Z")); const char *bench[] = { /* Authorization Header */ @@ -539,9 +539,9 @@ TEST_CASE("AWSAuthSpecByExample: GET Bucket List Objects, unsigned pay-load", "[ api._host.assign("examplebucket.s3.amazonaws.com"); api._path.assign(""); api._query.assign("max-keys=2&prefix=J"); - api._headers["Host"] = "examplebucket.s3.amazonaws.com"; - api._headers["x-amz-content-sha256"] = "UNSIGNED-PAYLOAD"; - api._headers["x-amz-date"] = "20130524T000000Z"; + api._headers.insert(std::make_pair("Host", "examplebucket.s3.amazonaws.com")); + api._headers.insert(std::make_pair("x-amz-content-sha256", "UNSIGNED-PAYLOAD")); + api._headers.insert(std::make_pair("x-amz-date", "20130524T000000Z")); const char *bench[] = { /* Authorization Header */ @@ -585,12 +585,13 @@ TEST_CASE("AWSAuthSpecByExample: GET Bucket List Objects, unsigned pay-load, exc api._host.assign("examplebucket.s3.amazonaws.com"); api._path.assign(""); api._query.assign("max-keys=2&prefix=J"); - api._headers["Host"] = "examplebucket.s3.amazonaws.com"; - api._headers["x-amz-content-sha256"] = "UNSIGNED-PAYLOAD"; - api._headers["x-amz-date"] = "20130524T000000Z"; - api._headers["@internal"] = "internal value"; - api._headers["x-forwarded-for"] = "192.168.7.1"; - api._headers["via"] = "http/1.1 tcp ipv4 ats_dev[7e67ac60-c204-450d-90be-a426dd3b569f] (ApacheTrafficServer/7.2.0)"; + api._headers.insert(std::make_pair("Host", "examplebucket.s3.amazonaws.com")); + api._headers.insert(std::make_pair("x-amz-content-sha256", "UNSIGNED-PAYLOAD")); + api._headers.insert(std::make_pair("x-amz-date", "20130524T000000Z")); + api._headers.insert(std::make_pair("@internal", "internal value")); + api._headers.insert(std::make_pair("x-forwarded-for", "192.168.7.1")); + api._headers.insert( + std::make_pair("via", "http/1.1 tcp ipv4 ats_dev[7e67ac60-c204-450d-90be-a426dd3b569f] (ApacheTrafficServer/7.2.0)")); const char *bench[] = { /* Authorization Header */ @@ -633,9 +634,9 @@ TEST_CASE("AWSAuthSpecByExample: GET Bucket List Objects, query param value alre api._host.assign("examplebucket.s3.amazonaws.com"); api._path.assign("PATH=="); api._query.assign("key=TEST=="); - api._headers["Host"] = "examplebucket.s3.amazonaws.com"; - api._headers["x-amz-content-sha256"] = "UNSIGNED-PAYLOAD"; - api._headers["x-amz-date"] = "20130524T000000Z"; + api._headers.insert(std::make_pair("Host", "examplebucket.s3.amazonaws.com")); + api._headers.insert(std::make_pair("x-amz-content-sha256", "UNSIGNED-PAYLOAD")); + api._headers.insert(std::make_pair("x-amz-date", "20130524T000000Z")); const char *bench[] = { /* Authorization Header */ @@ -669,7 +670,54 @@ TEST_CASE("AWSAuthSpecByExample: GET Bucket List Objects, query param value alre ValidateBench(api, /*signePayload */ false, &now, bench, defaultIncludeHeaders, defaultExcludeHeaders); } -/* Utility parameters related tests ******************************************************************************** */ +TEST_CASE("S3AuthV4UtilParams: signing multiple same name fields", "[AWS][auth][utility]") +{ + time_t now = 1369353600; /* 5/24/2013 00:00:00 GMT */ + + /* Define the HTTP request elements */ + MockTsInterface api; + api._method.assign("GET"); + api._host.assign("examplebucket.s3.amazonaws.com"); + api._path.assign(""); + api._query.assign("max-keys=2&prefix=J"); + api._headers.insert(std::make_pair("Host", "examplebucket.s3.amazonaws.com")); + api._headers.insert(std::make_pair("Content-Type", "gzip")); + api._headers.insert(std::make_pair("x-amz-content-sha256", "UNSIGNED-PAYLOAD")); + api._headers.insert(std::make_pair("x-amz-date", "20130524T000000Z")); + api._headers.insert(std::make_pair("HeaderA", "HeaderAValue")); + api._headers.insert(std::make_pair("HeaderB", "HeaderBValue1")); + api._headers.insert(std::make_pair("HeaderB", "HeaderBValue2")); + api._headers.insert(std::make_pair("HeaderC", " HeaderCValue1, HeaderCValue2 ")); // LWS between values + api._headers.insert(std::make_pair("HeaderC", "HeaderCValue3,HeaderCValue4")); // No LWS between values + + const char *bench[] = { + /* Authorization Header */ + "AWS4-HMAC-SHA256 " + "Credential=AKIAIOSFODNN7EXAMPLE/20130524/us-east-1/s3/aws4_request," + "SignedHeaders=content-type;headera;headerb;headerc;host;x-amz-content-sha256;x-amz-date," + "Signature=c9f57637044ddb0633430a8631f946a35d0ffff0cf7647aeaa2d0e985a69e674", + + /* Canonical Request sha256 */ + "e3429bb6a99fea0162c703ba6aaf771e16d2bb8f637c9300cd468fcbe1b66a0e", + /* Date and time*/ + "20130524T000000Z", + /* String to sign */ + "AWS4-HMAC-SHA256\n" + "20130524T000000Z\n" + "20130524/us-east-1/s3/aws4_request\n" + "e3429bb6a99fea0162c703ba6aaf771e16d2bb8f637c9300cd468fcbe1b66a0e", + /* Signature */ + "c9f57637044ddb0633430a8631f946a35d0ffff0cf7647aeaa2d0e985a69e674", + /* Payload hash */ + "UNSIGNED-PAYLOAD", + /* Signed Headers */ + "content-type;headera;headerb;headerc;host;x-amz-content-sha256;x-amz-date", + }; + + ValidateBench(api, /*signePayload */ false, &now, bench, defaultIncludeHeaders, defaultExcludeHeaders); +} + +///* Utility parameters related tests ******************************************************************************** */ void ValidateBenchCanonicalRequest(TsInterface &api, bool signPayload, time_t *now, const char *bench[], @@ -696,16 +744,16 @@ TEST_CASE("S3AuthV4UtilParams: include all headers by default", "[AWS][auth][uti api._host.assign("examplebucket.s3.amazonaws.com"); api._path.assign(""); api._query.assign("max-keys=2&prefix=J"); - api._headers["Host"] = "examplebucket.s3.amazonaws.com"; - api._headers["Content-Type"] = "gzip"; - api._headers["x-amz-content-sha256"] = "UNSIGNED-PAYLOAD"; - api._headers["x-amz-date"] = "20130524T000000Z"; - api._headers["HeaderA"] = "HeaderAValue"; - api._headers["HeaderB"] = "HeaderBValue"; - api._headers["HeaderC"] = "HeaderCValue"; - api._headers["HeaderD"] = "HeaderDValue"; - api._headers["HeaderE"] = "HeaderEValue"; - api._headers["HeaderF"] = "HeaderFValue"; + api._headers.insert(std::make_pair("Host", "examplebucket.s3.amazonaws.com")); + api._headers.insert(std::make_pair("Content-Type", "gzip")); + api._headers.insert(std::make_pair("x-amz-content-sha256", "UNSIGNED-PAYLOAD")); + api._headers.insert(std::make_pair("x-amz-date", "20130524T000000Z")); + api._headers.insert(std::make_pair("HeaderA", "HeaderAValue")); + api._headers.insert(std::make_pair("HeaderB", "HeaderBValue")); + api._headers.insert(std::make_pair("HeaderC", "HeaderCValue")); + api._headers.insert(std::make_pair("HeaderD", "HeaderDValue")); + api._headers.insert(std::make_pair("HeaderE", "HeaderEValue")); + api._headers.insert(std::make_pair("HeaderF", "HeaderFValue")); StringSet include = defaultIncludeHeaders; StringSet exclude = defaultExcludeHeaders; @@ -730,16 +778,16 @@ TEST_CASE("S3AuthV4UtilParams: include all headers explicit", "[AWS][auth][SpecB api._host.assign("examplebucket.s3.amazonaws.com"); api._path.assign(""); api._query.assign("max-keys=2&prefix=J"); - api._headers["Host"] = "examplebucket.s3.amazonaws.com"; - api._headers["Content-Type"] = "gzip"; - api._headers["x-amz-content-sha256"] = "UNSIGNED-PAYLOAD"; - api._headers["x-amz-date"] = "20130524T000000Z"; - api._headers["HeaderA"] = "HeaderAValue"; - api._headers["HeaderB"] = "HeaderBValue"; - api._headers["HeaderC"] = "HeaderCValue"; - api._headers["HeaderD"] = "HeaderDValue"; - api._headers["HeaderE"] = "HeaderEValue"; - api._headers["HeaderF"] = "HeaderFValue"; + api._headers.insert(std::make_pair("Host", "examplebucket.s3.amazonaws.com")); + api._headers.insert(std::make_pair("Content-Type", "gzip")); + api._headers.insert(std::make_pair("x-amz-content-sha256", "UNSIGNED-PAYLOAD")); + api._headers.insert(std::make_pair("x-amz-date", "20130524T000000Z")); + api._headers.insert(std::make_pair("HeaderA", "HeaderAValue")); + api._headers.insert(std::make_pair("HeaderB", "HeaderBValue")); + api._headers.insert(std::make_pair("HeaderC", "HeaderCValue")); + api._headers.insert(std::make_pair("HeaderD", "HeaderDValue")); + api._headers.insert(std::make_pair("HeaderE", "HeaderEValue")); + api._headers.insert(std::make_pair("HeaderF", "HeaderFValue")); StringSet include; commaSeparateString(include, "HeaderA,HeaderB,HeaderC,HeaderD,HeaderE,HeaderF"); @@ -765,16 +813,16 @@ TEST_CASE("S3AuthV4UtilParams: exclude all headers explicit", "[AWS][auth][utili api._host.assign("examplebucket.s3.amazonaws.com"); api._path.assign(""); api._query.assign("max-keys=2&prefix=J"); - api._headers["Host"] = "examplebucket.s3.amazonaws.com"; - api._headers["Content-Type"] = "gzip"; - api._headers["x-amz-content-sha256"] = "UNSIGNED-PAYLOAD"; - api._headers["x-amz-date"] = "20130524T000000Z"; - api._headers["HeaderA"] = "HeaderAValue"; - api._headers["HeaderB"] = "HeaderBValue"; - api._headers["HeaderC"] = "HeaderCValue"; - api._headers["HeaderD"] = "HeaderDValue"; - api._headers["HeaderE"] = "HeaderEValue"; - api._headers["HeaderF"] = "HeaderFValue"; + api._headers.insert(std::make_pair("Host", "examplebucket.s3.amazonaws.com")); + api._headers.insert(std::make_pair("Content-Type", "gzip")); + api._headers.insert(std::make_pair("x-amz-content-sha256", "UNSIGNED-PAYLOAD")); + api._headers.insert(std::make_pair("x-amz-date", "20130524T000000Z")); + api._headers.insert(std::make_pair("HeaderA", "HeaderAValue")); + api._headers.insert(std::make_pair("HeaderB", "HeaderBValue")); + api._headers.insert(std::make_pair("HeaderC", "HeaderCValue")); + api._headers.insert(std::make_pair("HeaderD", "HeaderDValue")); + api._headers.insert(std::make_pair("HeaderE", "HeaderEValue")); + api._headers.insert(std::make_pair("HeaderF", "HeaderFValue")); StringSet include = defaultIncludeHeaders; StringSet exclude; @@ -800,16 +848,15 @@ TEST_CASE("S3AuthV4UtilParams: include/exclude non overlapping headers", "[AWS][ api._host.assign("examplebucket.s3.amazonaws.com"); api._path.assign(""); api._query.assign("max-keys=2&prefix=J"); - api._headers["Host"] = "examplebucket.s3.amazonaws.com"; - api._headers["Content-Type"] = "gzip"; - api._headers["x-amz-content-sha256"] = "UNSIGNED-PAYLOAD"; - api._headers["x-amz-date"] = "20130524T000000Z"; - api._headers["HeaderA"] = "HeaderAValue"; - api._headers["HeaderB"] = "HeaderBValue"; - api._headers["HeaderC"] = "HeaderCValue"; - api._headers["HeaderD"] = "HeaderDValue"; - api._headers["HeaderE"] = "HeaderEValue"; - api._headers["HeaderF"] = "HeaderFValue"; + api._headers.insert(std::make_pair("Host", "examplebucket.s3.amazonaws.com")); + api._headers.insert(std::make_pair("Content-Type", "gzip")); + api._headers.insert(std::make_pair("x-amz-content-sha256", "UNSIGNED-PAYLOAD")); + api._headers.insert(std::make_pair("x-amz-date", "20130524T000000Z")); + api._headers.insert(std::make_pair("HeaderA", "HeaderAValue")); + api._headers.insert(std::make_pair("HeaderB", "HeaderBValue")); + api._headers.insert(std::make_pair("HeaderC", "HeaderCValue")); + api._headers.insert(std::make_pair("HeaderD", "HeaderEValue")); + api._headers.insert(std::make_pair("HeaderF", "HeaderFValue")); StringSet include, exclude; commaSeparateString(include, "HeaderA,HeaderB,HeaderC"); @@ -835,16 +882,16 @@ TEST_CASE("S3AuthV4UtilParams: include/exclude overlapping headers", "[AWS][auth api._host.assign("examplebucket.s3.amazonaws.com"); api._path.assign(""); api._query.assign("max-keys=2&prefix=J"); - api._headers["Host"] = "examplebucket.s3.amazonaws.com"; - api._headers["Content-Type"] = "gzip"; - api._headers["x-amz-content-sha256"] = "UNSIGNED-PAYLOAD"; - api._headers["x-amz-date"] = "20130524T000000Z"; - api._headers["HeaderA"] = "HeaderAValue"; - api._headers["HeaderB"] = "HeaderBValue"; - api._headers["HeaderC"] = "HeaderCValue"; - api._headers["HeaderD"] = "HeaderDValue"; - api._headers["HeaderE"] = "HeaderEValue"; - api._headers["HeaderF"] = "HeaderFValue"; + api._headers.insert(std::make_pair("Host", "examplebucket.s3.amazonaws.com")); + api._headers.insert(std::make_pair("Content-Type", "gzip")); + api._headers.insert(std::make_pair("x-amz-content-sha256", "UNSIGNED-PAYLOAD")); + api._headers.insert(std::make_pair("x-amz-date", "20130524T000000Z")); + api._headers.insert(std::make_pair("HeaderA", "HeaderAValue")); + api._headers.insert(std::make_pair("HeaderB", "HeaderBValue")); + api._headers.insert(std::make_pair("HeaderC", "HeaderCValue")); + api._headers.insert(std::make_pair("HeaderD", "HeaderDValue")); + api._headers.insert(std::make_pair("HeaderE", "HeaderEValue")); + api._headers.insert(std::make_pair("HeaderF", "HeaderFValue")); StringSet include, exclude; commaSeparateString(include, "HeaderA,HeaderB,HeaderC"); @@ -870,16 +917,16 @@ TEST_CASE("S3AuthV4UtilParams: include/exclude overlapping headers missing inclu api._host.assign("examplebucket.s3.amazonaws.com"); api._path.assign(""); api._query.assign("max-keys=2&prefix=J"); - api._headers["Host"] = "examplebucket.s3.amazonaws.com"; - api._headers["Content-Type"] = "gzip"; - api._headers["x-amz-content-sha256"] = "UNSIGNED-PAYLOAD"; - api._headers["x-amz-date"] = "20130524T000000Z"; - api._headers["HeaderA"] = "HeaderAValue"; - api._headers["HeaderB"] = "HeaderBValue"; - api._headers["HeaderC"] = "HeaderCValue"; - api._headers["HeaderD"] = "HeaderDValue"; - api._headers["HeaderE"] = "HeaderEValue"; - api._headers["HeaderF"] = "HeaderFValue"; + api._headers.insert(std::make_pair("Host", "examplebucket.s3.amazonaws.com")); + api._headers.insert(std::make_pair("Content-Type", "gzip")); + api._headers.insert(std::make_pair("x-amz-content-sha256", "UNSIGNED-PAYLOAD")); + api._headers.insert(std::make_pair("x-amz-date", "20130524T000000Z")); + api._headers.insert(std::make_pair("HeaderA", "HeaderAValue")); + api._headers.insert(std::make_pair("HeaderB", "HeaderBValue")); + api._headers.insert(std::make_pair("HeaderC", "HeaderCValue")); + api._headers.insert(std::make_pair("HeaderD", "HeaderDValue")); + api._headers.insert(std::make_pair("HeaderE", "HeaderEValue")); + api._headers.insert(std::make_pair("HeaderF", "HeaderFValue")); StringSet include, exclude; commaSeparateString(include, "HeaderA,HeaderC"); @@ -905,16 +952,16 @@ TEST_CASE("S3AuthV4UtilParams: include/exclude overlapping headers missing exclu api._host.assign("examplebucket.s3.amazonaws.com"); api._path.assign(""); api._query.assign("max-keys=2&prefix=J"); - api._headers["Host"] = "examplebucket.s3.amazonaws.com"; - api._headers["Content-Type"] = "gzip"; - api._headers["x-amz-content-sha256"] = "UNSIGNED-PAYLOAD"; - api._headers["x-amz-date"] = "20130524T000000Z"; - api._headers["HeaderA"] = "HeaderAValue"; - api._headers["HeaderB"] = "HeaderBValue"; - api._headers["HeaderC"] = "HeaderCValue"; - api._headers["HeaderD"] = "HeaderDValue"; - api._headers["HeaderE"] = "HeaderEValue"; - api._headers["HeaderF"] = "HeaderFValue"; + api._headers.insert(std::make_pair("Host", "examplebucket.s3.amazonaws.com")); + api._headers.insert(std::make_pair("Content-Type", "gzip")); + api._headers.insert(std::make_pair("x-amz-content-sha256", "UNSIGNED-PAYLOAD")); + api._headers.insert(std::make_pair("x-amz-date", "20130524T000000Z")); + api._headers.insert(std::make_pair("HeaderA", "HeaderAValue")); + api._headers.insert(std::make_pair("HeaderB", "HeaderBValue")); + api._headers.insert(std::make_pair("HeaderC", "HeaderCValue")); + api._headers.insert(std::make_pair("HeaderD", "HeaderDValue")); + api._headers.insert(std::make_pair("HeaderE", "HeaderEValue")); + api._headers.insert(std::make_pair("HeaderF", "HeaderFValue")); StringSet include, exclude; commaSeparateString(include, "HeaderA,HeaderB,HeaderC"); @@ -943,10 +990,10 @@ TEST_CASE("S3AuthV4UtilParams: include content type", "[AWS][auth][utility]") api._host.assign("examplebucket.s3.amazonaws.com"); api._path.assign(""); api._query.assign("max-keys=2&prefix=J"); - api._headers["Host"] = "examplebucket.s3.amazonaws.com"; - api._headers["Content-Type"] = "gzip"; - api._headers["x-amz-content-sha256"] = "UNSIGNED-PAYLOAD"; - api._headers["x-amz-date"] = "20130524T000000Z"; + api._headers.insert(std::make_pair("Host", "examplebucket.s3.amazonaws.com")); + api._headers.insert(std::make_pair("Content-Type", "gzip")); + api._headers.insert(std::make_pair("x-amz-content-sha256", "UNSIGNED-PAYLOAD")); + api._headers.insert(std::make_pair("x-amz-date", "20130524T000000Z")); StringSet include = defaultIncludeHeaders; StringSet exclude; @@ -976,9 +1023,9 @@ TEST_CASE("S3AuthV4UtilParams: include missing content type", "[AWS][auth][utili api._host.assign("examplebucket.s3.amazonaws.com"); api._path.assign(""); api._query.assign("max-keys=2&prefix=J"); - api._headers["Host"] = "examplebucket.s3.amazonaws.com"; - api._headers["x-amz-content-sha256"] = "UNSIGNED-PAYLOAD"; - api._headers["x-amz-date"] = "20130524T000000Z"; + api._headers.insert(std::make_pair("Host", "examplebucket.s3.amazonaws.com")); + api._headers.insert(std::make_pair("x-amz-content-sha256", "UNSIGNED-PAYLOAD")); + api._headers.insert(std::make_pair("x-amz-date", "20130524T000000Z")); StringSet include = defaultIncludeHeaders; StringSet exclude; diff --git a/plugins/s3_auth/unit_tests/test_aws_auth_v4.h b/plugins/s3_auth/unit_tests/test_aws_auth_v4.h index 134151797b4..e295d750f08 100644 --- a/plugins/s3_auth/unit_tests/test_aws_auth_v4.h +++ b/plugins/s3_auth/unit_tests/test_aws_auth_v4.h @@ -30,7 +30,7 @@ class HeaderIterator { public: - HeaderIterator(const StringMap::iterator &it) { _it = it; } + HeaderIterator(const HeaderMultiMap::iterator &it) { _it = it; } HeaderIterator(const HeaderIterator &i) { _it = i._it; } ~HeaderIterator() {} HeaderIterator & @@ -69,7 +69,7 @@ class HeaderIterator *len = _it->second.length(); return _it->second.c_str(); } - StringMap::iterator _it; + HeaderMultiMap::iterator _it; }; /* Define a mock API to be used in unit-tests */ @@ -115,7 +115,7 @@ class MockTsInterface : public TsInterface String _host; String _path; String _query; - StringMap _headers; + HeaderMultiMap _headers; }; /* Expose the following methods only to the unit tests */ diff --git a/plugins/experimental/server_push_preload/Makefile.inc b/plugins/server_push_preload/Makefile.inc similarity index 71% rename from plugins/experimental/server_push_preload/Makefile.inc rename to plugins/server_push_preload/Makefile.inc index 5a04108ac2e..518d71c7fb3 100644 --- a/plugins/experimental/server_push_preload/Makefile.inc +++ b/plugins/server_push_preload/Makefile.inc @@ -14,14 +14,14 @@ # See the License for the specific language governing permissions and # limitations under the License. -pkglib_LTLIBRARIES += experimental/server_push_preload/server_push_preload.la +pkglib_LTLIBRARIES += server_push_preload/server_push_preload.la -experimental_server_push_preload_server_push_preload_la_SOURCES = \ - experimental/server_push_preload/server_push_preload.cc +server_push_preload_server_push_preload_la_SOURCES = \ + server_push_preload/server_push_preload.cc -experimental_server_push_preload_server_push_preload_la_LDFLAGS = \ +server_push_preload_server_push_preload_la_LDFLAGS = \ $(AM_LDFLAGS) -experimental_server_push_preload_server_push_preload_la_LIBADD = \ +server_push_preload_server_push_preload_la_LIBADD = \ $(top_builddir)/src/tscpp/api/libtscppapi.la diff --git a/plugins/server_push_preload/README.md b/plugins/server_push_preload/README.md new file mode 100644 index 00000000000..4079c895ff4 --- /dev/null +++ b/plugins/server_push_preload/README.md @@ -0,0 +1,17 @@ +Parse origin response Link headers and use H2 Server Push to initiate push requests of assets that have the preload keyword. + +https://www.w3.org/TR/preload/ + +This plugin can be used as a global plugin or a remap plugin. + +To use it as a global plugin for all your remaps, add this line to your `plugins.config` file: + +``` +server_push_preload.so +``` + +To use it as a remap plugin add it to one of your remaps in the `remap.config` file: + +``` +map https://foo.cow.com/ https://bar.cow.com @plugin=server_push_preload.so +``` \ No newline at end of file diff --git a/plugins/experimental/server_push_preload/server_push_preload.cc b/plugins/server_push_preload/server_push_preload.cc similarity index 74% rename from plugins/experimental/server_push_preload/server_push_preload.cc rename to plugins/server_push_preload/server_push_preload.cc index def21cca672..68c44101200 100644 --- a/plugins/experimental/server_push_preload/server_push_preload.cc +++ b/plugins/server_push_preload/server_push_preload.cc @@ -26,9 +26,9 @@ #include #include #include -#include #include "tscpp/api/GlobalPlugin.h" -#include "tscpp/api/utils.h" +#include "tscpp/api/RemapPlugin.h" +#include "tscpp/api/TransactionPlugin.h" #define PLUGIN_NAME "server_push_preload" #define PRELOAD_PARAM "rel=preload" @@ -41,16 +41,16 @@ static regex linkRegexp("<([^>]+)>;(.+)"); namespace { -GlobalPlugin *plugin; -} +GlobalPlugin *globalPlugin; +RemapPlugin *remapPlugin; +} // namespace -class LinkServerPushPlugin : public GlobalPlugin +class ServerPushTransaction : public TransactionPlugin { public: - LinkServerPushPlugin() + explicit ServerPushTransaction(Transaction &transaction) : TransactionPlugin(transaction) { - TSDebug(PLUGIN_NAME, "registering transaction hooks"); - LinkServerPushPlugin::registerHook(HOOK_SEND_RESPONSE_HEADERS); + TransactionPlugin::registerHook(HOOK_SEND_RESPONSE_HEADERS); } void @@ -134,6 +134,32 @@ class LinkServerPushPlugin : public GlobalPlugin } }; +class ServerPushRemap : public RemapPlugin +{ +public: + explicit ServerPushRemap(void **instance_handle) : RemapPlugin(instance_handle) {} + + Result + doRemap(const Url &map_from_url, const Url &map_to_url, Transaction &transaction, bool &redirect) override + { + transaction.addPlugin(new ServerPushTransaction(transaction)); + return RESULT_DID_REMAP; + } +}; + +class ServerPushGlobal : public GlobalPlugin +{ +public: + ServerPushGlobal() { GlobalPlugin::registerHook(HOOK_READ_REQUEST_HEADERS_PRE_REMAP); } + + void + handleReadRequestHeadersPreRemap(Transaction &transaction) override + { + transaction.addPlugin(new ServerPushTransaction(transaction)); + transaction.resume(); + } +}; + void TSPluginInit(int argc ATSCPPAPI_UNUSED, const char *argv[] ATSCPPAPI_UNUSED) { @@ -141,5 +167,14 @@ TSPluginInit(int argc ATSCPPAPI_UNUSED, const char *argv[] ATSCPPAPI_UNUSED) if (!RegisterGlobalPlugin("ServerPushPreloadPlugin", PLUGIN_NAME, "dev@trafficserver.apache.org")) { return; } - plugin = new LinkServerPushPlugin(); + globalPlugin = new ServerPushGlobal(); } + +TSReturnCode +TSRemapNewInstance(int argc ATSCPPAPI_UNUSED, char *argv[] ATSCPPAPI_UNUSED, void **instance_handle, char *errbuf ATSCPPAPI_UNUSED, + int errbuf_size ATSCPPAPI_UNUSED) +{ + TSDebug(PLUGIN_NAME, "New Instance"); + remapPlugin = new ServerPushRemap(instance_handle); + return TS_SUCCESS; +} \ No newline at end of file diff --git a/plugins/stats_over_http/stats_over_http.c b/plugins/stats_over_http/stats_over_http.c index 3f44b688678..9783e627725 100644 --- a/plugins/stats_over_http/stats_over_http.c +++ b/plugins/stats_over_http/stats_over_http.c @@ -33,18 +33,59 @@ #include #include #include +#include +#include +#include +#include +#include #include "tscore/ink_defs.h" #define PLUGIN_NAME "stats_over_http" +#define FREE_TMOUT 300000 +#define STR_BUFFER_SIZE 1024 + +#define SYSTEM_RECORD_TYPE (0x100) +#define DEFAULT_RECORD_TYPES (SYSTEM_RECORD_TYPE | TS_RECORDTYPE_PROCESS | TS_RECORDTYPE_PLUGIN) + +#define DEFAULT_IP "0.0.0.0" +#define DEFAULT_IP6 "::" /* global holding the path used for access to this JSON data */ -static const char *url_path = "_stats"; -static int url_path_len; +#define DEFAULT_URL_PATH "_stats" static bool integer_counters = false; static bool wrap_counters = false; +typedef struct { + unsigned int recordTypes; + char *stats_path; + int stats_path_len; + char *allowIps; + int ipCount; + char *allowIps6; + int ip6Count; +} config_t; +typedef struct { + char *config_path; + volatile time_t last_load; + config_t *config; +} config_holder_t; + +typedef enum { JSON_OUTPUT, CSV_OUTPUT } output_format; + +int configReloadRequests = 0; +int configReloads = 0; +time_t lastReloadRequest = 0; +time_t lastReload = 0; +time_t astatsLoad = 0; + +static int free_handler(TSCont cont, TSEvent event, void *edata); +static int config_handler(TSCont cont, TSEvent event, void *edata); +static config_t *get_config(TSCont cont); +static config_holder_t *new_config_holder(const char *path); +static bool is_ip_allowed(const config_t *config, const struct sockaddr *addr); + typedef struct stats_state_t { TSVConn net_vc; TSVIO read_vio; @@ -56,8 +97,17 @@ typedef struct stats_state_t { int output_bytes; int body_written; + output_format output; } stats_state; +static char * +nstr(const char *s) +{ + char *mys = (char *)TSmalloc(strlen(s) + 1); + strcpy(mys, s); + return mys; +} + static void stats_cleanup(TSCont contp, stats_state *my_state) { @@ -94,12 +144,24 @@ stats_add_data_to_resp_buffer(const char *s, stats_state *my_state) return s_len; } -static const char RESP_HEADER[] = "HTTP/1.0 200 Ok\r\nContent-Type: text/javascript\r\nCache-Control: no-cache\r\n\r\n"; +static const char RESP_HEADER_JSON[] = "HTTP/1.0 200 Ok\r\nContent-Type: text/json\r\nCache-Control: no-cache\r\n\r\n"; +static const char RESP_HEADER_CSV[] = "HTTP/1.0 200 Ok\r\nContent-Type: text/csv\r\nCache-Control: no-cache\r\n\r\n"; static int stats_add_resp_header(stats_state *my_state) { - return stats_add_data_to_resp_buffer(RESP_HEADER, my_state); + switch (my_state->output) { + case JSON_OUTPUT: + return stats_add_data_to_resp_buffer(RESP_HEADER_JSON, my_state); + break; + case CSV_OUTPUT: + return stats_add_data_to_resp_buffer(RESP_HEADER_CSV, my_state); + break; + default: + TSError("stats_add_resp_header: Unknown output format"); + break; + } + return stats_add_data_to_resp_buffer(RESP_HEADER_JSON, my_state); } static void @@ -124,13 +186,13 @@ stats_process_read(TSCont contp, TSEvent event, stats_state *my_state) } #define APPEND(a) my_state->output_bytes += stats_add_data_to_resp_buffer(a, my_state) -#define APPEND_STAT(a, fmt, v) \ +#define APPEND_STAT_JSON(a, fmt, v) \ do { \ char b[256]; \ if (snprintf(b, sizeof(b), "\"%s\": \"" fmt "\",\n", a, v) < (int)sizeof(b)) \ APPEND(b); \ } while (0) -#define APPEND_STAT_NUMERIC(a, fmt, v) \ +#define APPEND_STAT_JSON_NUMERIC(a, fmt, v) \ do { \ char b[256]; \ if (integer_counters) { \ @@ -144,6 +206,20 @@ stats_process_read(TSCont contp, TSEvent event, stats_state *my_state) } \ } while (0) +#define APPEND_STAT_CSV(a, fmt, v) \ + do { \ + char b[256]; \ + if (snprintf(b, sizeof(b), "%s," fmt "\n", a, v) < (int)sizeof(b)) \ + APPEND(b); \ + } while (0) +#define APPEND_STAT_CSV_NUMERIC(a, fmt, v) \ + do { \ + char b[256]; \ + if (snprintf(b, sizeof(b), "%s," fmt "\n", a, v) < (int)sizeof(b)) { \ + APPEND(b); \ + } \ + } while (0) + // This wraps uint64_t values to the int64_t range to fit into a Java long. Java 8 has an unsigned long which // can interoperate with a full uint64_t, but it's unlikely that much of the ecosystem supports that yet. static uint64_t @@ -164,22 +240,47 @@ json_out_stat(TSRecordType rec_type ATS_UNUSED, void *edata, int registered ATS_ switch (data_type) { case TS_RECORDDATATYPE_COUNTER: - APPEND_STAT_NUMERIC(name, "%" PRIu64, wrap_unsigned_counter(datum->rec_counter)); + APPEND_STAT_JSON_NUMERIC(name, "%" PRIu64, wrap_unsigned_counter(datum->rec_counter)); + break; + case TS_RECORDDATATYPE_INT: + APPEND_STAT_JSON_NUMERIC(name, "%" PRIu64, wrap_unsigned_counter(datum->rec_int)); + break; + case TS_RECORDDATATYPE_FLOAT: + APPEND_STAT_JSON_NUMERIC(name, "%f", datum->rec_float); + break; + case TS_RECORDDATATYPE_STRING: + APPEND_STAT_JSON(name, "%s", datum->rec_string); + break; + default: + TSDebug(PLUGIN_NAME, "unknown type for %s: %d", name, data_type); + break; + } +} + +static void +csv_out_stat(TSRecordType rec_type ATS_UNUSED, void *edata, int registered ATS_UNUSED, const char *name, TSRecordDataType data_type, + TSRecordData *datum) +{ + stats_state *my_state = edata; + switch (data_type) { + case TS_RECORDDATATYPE_COUNTER: + APPEND_STAT_CSV_NUMERIC(name, "%" PRIu64, wrap_unsigned_counter(datum->rec_counter)); break; case TS_RECORDDATATYPE_INT: - APPEND_STAT_NUMERIC(name, "%" PRIu64, wrap_unsigned_counter(datum->rec_int)); + APPEND_STAT_CSV_NUMERIC(name, "%" PRIu64, wrap_unsigned_counter(datum->rec_int)); break; case TS_RECORDDATATYPE_FLOAT: - APPEND_STAT_NUMERIC(name, "%f", datum->rec_float); + APPEND_STAT_CSV_NUMERIC(name, "%f", datum->rec_float); break; case TS_RECORDDATATYPE_STRING: - APPEND_STAT(name, "%s", datum->rec_string); + APPEND_STAT_CSV(name, "%s", datum->rec_string); break; default: TSDebug(PLUGIN_NAME, "unknown type for %s: %d", name, data_type); break; } } + static void json_out_stats(stats_state *my_state) { @@ -194,6 +295,15 @@ json_out_stats(stats_state *my_state) APPEND(" }\n}\n"); } +static void +csv_out_stats(stats_state *my_state) +{ + const char *version; + TSRecordDump((TSRecordType)(TS_RECORDTYPE_PLUGIN | TS_RECORDTYPE_NODE | TS_RECORDTYPE_PROCESS), csv_out_stat, my_state); + version = TSTrafficServerVersionGet(); + APPEND_STAT_CSV("version", "%s", version); +} + static void stats_process_write(TSCont contp, TSEvent event, stats_state *my_state) { @@ -201,7 +311,17 @@ stats_process_write(TSCont contp, TSEvent event, stats_state *my_state) if (my_state->body_written == 0) { TSDebug(PLUGIN_NAME, "plugin adding response body"); my_state->body_written = 1; - json_out_stats(my_state); + switch (my_state->output) { + case JSON_OUTPUT: + json_out_stats(my_state); + break; + case CSV_OUTPUT: + csv_out_stats(my_state); + break; + default: + TSError("stats_process_write: Unknown output type\n"); + break; + } TSVIONBytesSet(my_state->write_vio, my_state->output_bytes); } TSVIOReenable(my_state->write_vio); @@ -236,12 +356,14 @@ stats_origin(TSCont contp ATS_UNUSED, TSEvent event ATS_UNUSED, void *edata) { TSCont icontp; stats_state *my_state; + config_t *config; TSHttpTxn txnp = (TSHttpTxn)edata; TSMBuffer reqp; - TSMLoc hdr_loc = NULL, url_loc = NULL; + TSMLoc hdr_loc = NULL, url_loc = NULL, accept_field = NULL; TSEvent reenable = TS_EVENT_HTTP_CONTINUE; TSDebug(PLUGIN_NAME, "in the read stuff"); + config = get_config(contp); if (TSHttpTxnClientReqGet(txnp, &reqp, &hdr_loc) != TS_SUCCESS) { goto cleanup; @@ -255,7 +377,13 @@ stats_origin(TSCont contp ATS_UNUSED, TSEvent event ATS_UNUSED, void *edata) const char *path = TSUrlPathGet(reqp, url_loc, &path_len); TSDebug(PLUGIN_NAME, "Path: %.*s", path_len, path); - if (!(path_len != 0 && path_len == url_path_len && !memcmp(path, url_path, url_path_len))) { + if (!(path_len != 0 && path_len == config->stats_path_len && !memcmp(path, config->stats_path, config->stats_path_len))) { + goto notforme; + } + + const struct sockaddr *addr = TSHttpTxnClientAddrGet(txnp); + if (!is_ip_allowed(config, addr)) { + TSDebug(PLUGIN_NAME, "not right ip"); goto notforme; } @@ -267,6 +395,22 @@ stats_origin(TSCont contp ATS_UNUSED, TSEvent event ATS_UNUSED, void *edata) icontp = TSContCreate(stats_dostuff, TSMutexCreate()); my_state = (stats_state *)TSmalloc(sizeof(*my_state)); memset(my_state, 0, sizeof(*my_state)); + + accept_field = TSMimeHdrFieldFind(reqp, hdr_loc, TS_MIME_FIELD_ACCEPT, TS_MIME_LEN_ACCEPT); + my_state->output = JSON_OUTPUT; // default to json output + // accept header exists, use it to determine response type + if (accept_field != TS_NULL_MLOC) { + int len = -1; + const char *str = TSMimeHdrFieldValueStringGet(reqp, hdr_loc, accept_field, -1, &len); + + // Parse the Accept header, default to JSON output unless its another supported format + if (!strncasecmp(str, "text/csv", len)) { + my_state->output = CSV_OUTPUT; + } else { + my_state->output = JSON_OUTPUT; + } + } + TSContDataSet(icontp, my_state); TSHttpTxnIntercept(icontp, txnp); goto cleanup; @@ -280,7 +424,9 @@ stats_origin(TSCont contp ATS_UNUSED, TSEvent event ATS_UNUSED, void *edata) if (hdr_loc) { TSHandleMLocRelease(reqp, TS_NULL_MLOC, hdr_loc); } - + if (accept_field) { + TSHandleMLocRelease(reqp, TS_NULL_MLOC, accept_field); + } TSHttpTxnReenable(txnp, reenable); return 0; } @@ -294,6 +440,8 @@ TSPluginInit(int argc, const char *argv[]) static const struct option longopts[] = {{(char *)("integer-counters"), no_argument, NULL, 'i'}, {(char *)("wrap-counters"), no_argument, NULL, 'w'}, {NULL, 0, NULL, 0}}; + TSCont main_cont, config_cont; + config_holder_t *config_holder; info.plugin_name = PLUGIN_NAME; info.vendor_name = "Apache Software Foundation"; @@ -301,6 +449,7 @@ TSPluginInit(int argc, const char *argv[]) if (TSPluginRegister(&info) != TS_SUCCESS) { TSError("[%s] registration failed", PLUGIN_NAME); + goto done; } for (;;) { @@ -322,13 +471,379 @@ TSPluginInit(int argc, const char *argv[]) argc -= optind; argv += optind; - if (argc > 0) { - url_path = TSstrdup(argv[0] + ('/' == argv[0][0] ? 1 : 0)); /* Skip leading / */ + config_holder = new_config_holder(argc > 0 ? argv[0] : NULL); + + /* Path was not set during load, so the param was not a config file, we also + have an argument so it must be the path, set it here. Otherwise if no argument + then use the default _stats path */ + if ((config_holder->config != NULL) && (config_holder->config->stats_path == 0) && (argc > 0) && + (config_holder->config_path == NULL)) { + config_holder->config->stats_path = TSstrdup(argv[0] + ('/' == argv[0][0] ? 1 : 0)); + config_holder->config->stats_path_len = strlen(config_holder->config->stats_path); + } else if ((config_holder->config != NULL) && (config_holder->config->stats_path == 0)) { + config_holder->config->stats_path = nstr(DEFAULT_URL_PATH); + config_holder->config->stats_path_len = strlen(config_holder->config->stats_path); } - url_path_len = strlen(url_path); /* Create a continuation with a mutex as there is a shared global structure containing the headers to add */ - TSHttpHookAdd(TS_HTTP_READ_REQUEST_HDR_HOOK, TSContCreate(stats_origin, TSMutexCreate())); - TSDebug(PLUGIN_NAME, "stats module registered"); + main_cont = TSContCreate(stats_origin, NULL); + TSContDataSet(main_cont, (void *)config_holder); + TSHttpHookAdd(TS_HTTP_READ_REQUEST_HDR_HOOK, main_cont); + + /* Create continuation for management updates to re-read config file */ + config_cont = TSContCreate(config_handler, TSMutexCreate()); + TSContDataSet(config_cont, (void *)config_holder); + TSMgmtUpdateRegister(config_cont, PLUGIN_NAME); + TSDebug(PLUGIN_NAME, "stats module registered with path %s", config_holder->config->stats_path); + +done: + return; +} + +static bool +is_ip_match(const char *ip, char *ipmask, char mask) +{ + unsigned int j, i, k; + char cm; + // to be able to set mask to 128 + unsigned int umask = 0xff & mask; + + for (j = 0, i = 0; ((i + 1) * 8) <= umask; i++) { + if (ip[i] != ipmask[i]) { + return false; + } + j += 8; + } + cm = 0; + for (k = 0; j < umask; j++, k++) { + cm |= 1 << (7 - k); + } + + if ((ip[i] & cm) != (ipmask[i] & cm)) { + return false; + } + return true; +} + +static bool +is_ip_allowed(const config_t *config, const struct sockaddr *addr) +{ + char ip_port_text_buffer[INET6_ADDRSTRLEN]; + int i; + char *ipmask; + if (!addr) { + return true; + } + + if (addr->sa_family == AF_INET && config->allowIps) { + const struct sockaddr_in *addr_in = (struct sockaddr_in *)addr; + const char *ip = (char *)&addr_in->sin_addr; + + for (i = 0; i < config->ipCount; i++) { + ipmask = config->allowIps + (i * (sizeof(struct in_addr) + 1)); + if (is_ip_match(ip, ipmask, ipmask[4])) { + TSDebug(PLUGIN_NAME, "clientip is %s--> ALLOW", inet_ntop(AF_INET, ip, ip_port_text_buffer, INET6_ADDRSTRLEN)); + return true; + } + } + TSDebug(PLUGIN_NAME, "clientip is %s--> DENY", inet_ntop(AF_INET, ip, ip_port_text_buffer, INET6_ADDRSTRLEN)); + return false; + + } else if (addr->sa_family == AF_INET6 && config->allowIps6) { + const struct sockaddr_in6 *addr_in6 = (struct sockaddr_in6 *)addr; + const char *ip = (char *)&addr_in6->sin6_addr; + + for (i = 0; i < config->ip6Count; i++) { + ipmask = config->allowIps6 + (i * (sizeof(struct in6_addr) + 1)); + if (is_ip_match(ip, ipmask, ipmask[sizeof(struct in6_addr)])) { + TSDebug(PLUGIN_NAME, "clientip6 is %s--> ALLOW", inet_ntop(AF_INET6, ip, ip_port_text_buffer, INET6_ADDRSTRLEN)); + return true; + } + } + TSDebug(PLUGIN_NAME, "clientip6 is %s--> DENY", inet_ntop(AF_INET6, ip, ip_port_text_buffer, INET6_ADDRSTRLEN)); + return false; + } + return true; +} + +static void +parseIps(config_t *config, char *ipStr) +{ + char buffer[STR_BUFFER_SIZE]; + char *p, *tok1, *tok2, *ip; + int i, mask; + char ip_port_text_buffer[INET_ADDRSTRLEN]; + + if (!ipStr) { + config->ipCount = 1; + ip = config->allowIps = TSmalloc(sizeof(struct in_addr) + 1); + inet_pton(AF_INET, DEFAULT_IP, ip); + ip[4] = 0; + return; + } + + strcpy(buffer, ipStr); + p = buffer; + while (strtok_r(p, ", \n", &p)) { + config->ipCount++; + } + if (!config->ipCount) { + return; + } + config->allowIps = TSmalloc(5 * config->ipCount); // 4 bytes for ip + 1 for bit mask + strcpy(buffer, ipStr); + p = buffer; + i = 0; + while ((tok1 = strtok_r(p, ", \n", &p))) { + TSDebug(PLUGIN_NAME, "%d) parsing: %s", i + 1, tok1); + tok2 = strtok_r(tok1, "/", &tok1); + ip = config->allowIps + ((sizeof(struct in_addr) + 1) * i); + if (!inet_pton(AF_INET, tok2, ip)) { + TSDebug(PLUGIN_NAME, "%d) skipping: %s", i + 1, tok1); + continue; + } + + if (tok1 != NULL) { + tok2 = strtok_r(tok1, "/", &tok1); + } + if (!tok2) { + mask = 32; + } else { + mask = atoi(tok2); + } + ip[4] = mask; + TSDebug(PLUGIN_NAME, "%d) adding netmask: %s/%d", i + 1, inet_ntop(AF_INET, ip, ip_port_text_buffer, INET_ADDRSTRLEN), ip[4]); + i++; + } +} +static void +parseIps6(config_t *config, char *ipStr) +{ + char buffer[STR_BUFFER_SIZE]; + char *p, *tok1, *tok2, *ip; + int i, mask; + char ip_port_text_buffer[INET6_ADDRSTRLEN]; + + if (!ipStr) { + config->ip6Count = 1; + ip = config->allowIps6 = TSmalloc(sizeof(struct in6_addr) + 1); + inet_pton(AF_INET6, DEFAULT_IP6, ip); + ip[sizeof(struct in6_addr)] = 0; + return; + } + + strcpy(buffer, ipStr); + p = buffer; + while (strtok_r(p, ", \n", &p)) { + config->ip6Count++; + } + if (!config->ip6Count) { + return; + } + + config->allowIps6 = TSmalloc((sizeof(struct in6_addr) + 1) * config->ip6Count); // 16 bytes for ip + 1 for bit mask + strcpy(buffer, ipStr); + p = buffer; + i = 0; + while ((tok1 = strtok_r(p, ", \n", &p))) { + TSDebug(PLUGIN_NAME, "%d) parsing: %s", i + 1, tok1); + tok2 = strtok_r(tok1, "/", &tok1); + ip = config->allowIps6 + ((sizeof(struct in6_addr) + 1) * i); + if (!inet_pton(AF_INET6, tok2, ip)) { + TSDebug(PLUGIN_NAME, "%d) skipping: %s", i + 1, tok1); + continue; + } + + if (tok1 != NULL) { + tok2 = strtok_r(tok1, "/", &tok1); + } + + if (!tok2) { + mask = 128; + } else { + mask = atoi(tok2); + } + ip[sizeof(struct in6_addr)] = mask; + TSDebug(PLUGIN_NAME, "%d) adding netmask: %s/%d", i + 1, inet_ntop(AF_INET6, ip, ip_port_text_buffer, INET6_ADDRSTRLEN), + ip[sizeof(struct in6_addr)]); + i++; + } +} + +static config_t * +new_config(TSFile fh) +{ + char buffer[STR_BUFFER_SIZE]; + config_t *config = NULL; + config = (config_t *)TSmalloc(sizeof(config_t)); + config->stats_path = 0; + config->stats_path_len = 0; + config->allowIps = 0; + config->ipCount = 0; + config->allowIps6 = 0; + config->ip6Count = 0; + config->recordTypes = DEFAULT_RECORD_TYPES; + + if (!fh) { + TSDebug(PLUGIN_NAME, "No config file, using defaults"); + return config; + } + + while (TSfgets(fh, buffer, STR_BUFFER_SIZE - 1)) { + if (*buffer == '#') { + continue; /* # Comments, only at line beginning */ + } + char *p = 0; + if ((p = strstr(buffer, "path="))) { + p += strlen("path="); + if (p[0] == '/') { + p++; + } + config->stats_path = nstr(strtok_r(p, " \n", &p)); + config->stats_path_len = strlen(config->stats_path); + } else if ((p = strstr(buffer, "record_types="))) { + p += strlen("record_types="); + config->recordTypes = strtol(strtok_r(p, " \n", &p), NULL, 16); + } else if ((p = strstr(buffer, "allow_ip="))) { + p += strlen("allow_ip="); + parseIps(config, p); + } else if ((p = strstr(buffer, "allow_ip6="))) { + p += strlen("allow_ip6="); + parseIps6(config, p); + } + } + if (!config->ipCount) { + parseIps(config, NULL); + } + if (!config->ip6Count) { + parseIps6(config, NULL); + } + TSDebug(PLUGIN_NAME, "config path=%s", config->stats_path); + + return config; +} + +static void +delete_config(config_t *config) +{ + TSDebug(PLUGIN_NAME, "Freeing config"); + TSfree(config->allowIps); + TSfree(config->allowIps6); + TSfree(config->stats_path); + TSfree(config); +} + +// standard api below... +static config_t * +get_config(TSCont cont) +{ + config_holder_t *configh = (config_holder_t *)TSContDataGet(cont); + if (!configh) { + return 0; + } + return configh->config; +} + +static void +load_config_file(config_holder_t *config_holder) +{ + TSFile fh = NULL; + struct stat s; + + config_t *newconfig, *oldconfig; + TSCont free_cont; + + configReloadRequests++; + lastReloadRequest = time(NULL); + + // check date + if ((config_holder->config_path == NULL) || (stat(config_holder->config_path, &s) < 0)) { + TSDebug(PLUGIN_NAME, "Could not stat %s", config_holder->config_path); + config_holder->config_path = NULL; + if (config_holder->config) { + return; + } + } else { + TSDebug(PLUGIN_NAME, "s.st_mtime=%lu, last_load=%lu", s.st_mtime, config_holder->last_load); + if (s.st_mtime < config_holder->last_load) { + return; + } + } + + if (config_holder->config_path != NULL) { + TSDebug(PLUGIN_NAME, "Opening config file: %s", config_holder->config_path); + fh = TSfopen(config_holder->config_path, "r"); + } + + if (!fh) { + TSError("[%s] Unable to open config: %s. Will use the param as the path, or %s if null\n", PLUGIN_NAME, + config_holder->config_path, DEFAULT_URL_PATH); + if (config_holder->config) { + return; + } + } + + newconfig = 0; + newconfig = new_config(fh); + if (newconfig) { + configReloads++; + lastReload = lastReloadRequest; + config_holder->last_load = lastReloadRequest; + config_t **confp = &(config_holder->config); + oldconfig = __sync_lock_test_and_set(confp, newconfig); + if (oldconfig) { + TSDebug(PLUGIN_NAME, "scheduling free: %p (%p)", oldconfig, newconfig); + free_cont = TSContCreate(free_handler, TSMutexCreate()); + TSContDataSet(free_cont, (void *)oldconfig); + TSContScheduleOnPool(free_cont, FREE_TMOUT, TS_THREAD_POOL_TASK); + } + } + if (fh) { + TSfclose(fh); + } + return; +} + +static config_holder_t * +new_config_holder(const char *path) +{ + config_holder_t *config_holder = TSmalloc(sizeof(config_holder_t)); + config_holder->config_path = 0; + config_holder->config = 0; + config_holder->last_load = 0; + + if (path) { + config_holder->config_path = nstr(path); + } else { + config_holder->config_path = NULL; + } + load_config_file(config_holder); + return config_holder; +} + +static int +free_handler(TSCont cont, TSEvent event, void *edata) +{ + config_t *config; + config = (config_t *)TSContDataGet(cont); + delete_config(config); + TSContDestroy(cont); + return 0; +} + +static int +config_handler(TSCont cont, TSEvent event, void *edata) +{ + config_holder_t *config_holder; + config_holder = (config_holder_t *)TSContDataGet(cont); + load_config_file(config_holder); + + /* We received a reload, check if the path value was removed since it was not set after load. + If unset, then we'll use the default */ + if (config_holder->config->stats_path == 0) { + config_holder->config->stats_path = nstr(DEFAULT_URL_PATH); + config_holder->config->stats_path_len = strlen(config_holder->config->stats_path); + } + return 0; } diff --git a/plugins/tcpinfo/tcpinfo.cc b/plugins/tcpinfo/tcpinfo.cc index cb8a82043ba..0c9d07fd250 100644 --- a/plugins/tcpinfo/tcpinfo.cc +++ b/plugins/tcpinfo/tcpinfo.cc @@ -54,7 +54,6 @@ #define TCPI_HOOK_SSN_START 0x01u #define TCPI_HOOK_TXN_START 0x02u #define TCPI_HOOK_SEND_RESPONSE 0x04u -#define TCPI_HOOK_SSN_CLOSE 0x08u #define TCPI_HOOK_TXN_CLOSE 0x10u // Log format headers. These are emitted once at the start of a log file. Note that we @@ -215,10 +214,6 @@ tcp_info_hook(TSCont contp, TSEvent event, void *edata) ssnp = TSHttpTxnSsnGet(txnp); event_name = "send_resp_hdr"; break; - case TS_EVENT_HTTP_SSN_CLOSE: - ssnp = static_cast(edata); - event_name = "ssn_close"; - break; default: return 0; } @@ -291,8 +286,11 @@ parse_hook_list(const char *hook_list) const struct hookmask { const char *name; unsigned mask; - } hooks[] = {{"ssn_start", TCPI_HOOK_SSN_START}, {"txn_start", TCPI_HOOK_TXN_START}, {"send_resp_hdr", TCPI_HOOK_SEND_RESPONSE}, - {"ssn_close", TCPI_HOOK_SSN_CLOSE}, {"txn_close", TCPI_HOOK_TXN_CLOSE}, {nullptr, 0u}}; + } hooks[] = {{"ssn_start", TCPI_HOOK_SSN_START}, + {"txn_start", TCPI_HOOK_TXN_START}, + {"send_resp_hdr", TCPI_HOOK_SEND_RESPONSE}, + {"txn_close", TCPI_HOOK_TXN_CLOSE}, + {nullptr, 0u}}; str = TSstrdup(hook_list); @@ -461,11 +459,6 @@ TSPluginInit(int argc, const char *argv[]) TSDebug("tcpinfo", "added hook to the sending of the headers"); } - if (hooks & TCPI_HOOK_SSN_CLOSE) { - TSHttpHookAdd(TS_HTTP_SSN_CLOSE_HOOK, cont); - TSDebug("tcpinfo", "added hook to the close of the TCP connection"); - } - if (hooks & TCPI_HOOK_TXN_CLOSE) { TSHttpHookAdd(TS_HTTP_TXN_CLOSE_HOOK, cont); TSDebug("tcpinfo", "added hook to the close of the transaction"); diff --git a/plugins/xdebug/Cleanup.h b/plugins/xdebug/Cleanup.h new file mode 100644 index 00000000000..7a1c5b3351f --- /dev/null +++ b/plugins/xdebug/Cleanup.h @@ -0,0 +1,186 @@ +/** + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +/** + * @file Cleanup.h + * @brief Easy-to-use utilities to avoid resource leaks or double-releases of resources. Independent of the rest + * of the CPPAPI. + */ + +#pragma once + +#include +#include + +#include + +namespace atscppapi +{ +// For TS API types TSXxx with a TSXxxDestroy function, define standard deleter TSXxxDeleter, and use it to +// define TSXxxUniqPtr (specialization of std::unique_ptr). X() is used when the destroy function returns void, +// Y() is used when the destroy function returns TSReturnCode. + +#if defined(X) +#error "X defined as preprocessor symbol" +#endif + +#define X(NAME_SEGMENT) \ + struct TS##NAME_SEGMENT##Deleter { \ + void \ + operator()(TS##NAME_SEGMENT ptr) \ + { \ + TS##NAME_SEGMENT##Destroy(ptr); \ + } \ + }; \ + using TS##NAME_SEGMENT##UniqPtr = std::unique_ptr, TS##NAME_SEGMENT##Deleter>; + +#if defined(Y) +#error "Y defined as preprocessor symbol" +#endif + +#define Y(NAME_SEGMENT) \ + struct TS##NAME_SEGMENT##Deleter { \ + void \ + operator()(TS##NAME_SEGMENT ptr) \ + { \ + TSAssert(TS##NAME_SEGMENT##Destroy(ptr) == TS_SUCCESS); \ + } \ + }; \ + using TS##NAME_SEGMENT##UniqPtr = std::unique_ptr, TS##NAME_SEGMENT##Deleter>; + +Y(MBuffer) // Defines TSMBufferDeleter and TSMBufferUniqPtr. +X(MimeParser) // Defines TSMimeParserDeleter and TSMimeParserUniqPtr. +X(Thread) // Defines TSThreadDeleter and TSThreadUniqPtr. +X(Mutex) // Defines TSMutexDeleter and TSMutexUniqPtr. +Y(CacheKey) // Defines TSCacheKeyDeleter and TSCacheKeyUniqPtr. +X(Cont) // Defines TSContDeleter and TSContUniqPtr. +X(SslContext) // Defines TSSslContextDeleter and TSSslContextUniqPtr. +X(IOBuffer) // Defines TSIOBufferDeleter and TSIOBufferUniqPtr. +Y(TextLogObject) // Defines TSTextLogObjectDeleter and TSTextLogObjectUniqPtr. +X(Uuid) // Defines TSUuidDeleter and TSUuidUniqPtr. + +#undef X +#undef Y + +// Deleter and unique pointer for memory buffer returned by TSalloc(), TSrealloc(), Tstrdup(), TSsrtndup(). +// +struct TSMemDeleter { + void + operator()(void *ptr) + { + TSfree(ptr); + } +}; +using TSMemUniqPtr = std::unique_ptr; + +// Deleter and unique pointer for TSIOBufferReader. Care must be taken that the reader is deleted before the +// TSIOBuffer to which it refers is deleted. +// +struct TSIOBufferReaderDeleter { + void + operator()(TSIOBufferReader ptr) + { + TSIOBufferReaderFree(ptr); + } +}; +using TSIOBufferReaderUniqPtr = std::unique_ptr, TSIOBufferReaderDeleter>; + +class TxnAuxDataMgrBase +{ +protected: + struct MgrData_ { + TSCont txnCloseContp = nullptr; + int txnArgIndex = -1; + }; + +public: + class MgrData : private MgrData_ + { + friend class TxnAuxDataMgrBase; + }; + +protected: + static MgrData_ & + access(MgrData &md) + { + return md; + } +}; + +using TxnAuxMgrData = TxnAuxDataMgrBase::MgrData; + +// Class to manage auxilliary data for a transaction. If an instance is created for the transaction, the instance +// will be deleted on the TXN_CLOSE transaction hook (which is always triggered for all transactions). +// The TxnAuxData class must have a public default constructor. +// +template class TxnAuxDataMgr : private TxnAuxDataMgrBase +{ +public: + using Data = TxnAuxData; + + // This must be called from the plugin init function. arg_name is the name for the transaction argument used + // to store the pointer to the auxiliary data class instance. Repeated calls are ignored. + // + static void + init(char const *arg_name, char const *arg_desc = "per-transaction auxiliary data") + { + MgrData_ &md = access(MDRef); + + if (md.txnArgIndex >= 0) { + return; + } + + TSReleaseAssert(TSUserArgIndexReserve(TS_USER_ARGS_TXN, arg_name, arg_desc, &md.txnArgIndex) == TS_SUCCESS); + TSReleaseAssert(md.txnCloseContp = TSContCreate(_deleteAuxData, nullptr)); + } + + // Get a reference to the auxiliary data for a transaction. + // + static TxnAuxData & + data(TSHttpTxn txn) + { + MgrData_ &md = access(MDRef); + + TSAssert(md.txnArgIndex >= 0); + + auto d = static_cast(TSUserArgGet(txn, md.txnArgIndex)); + if (!d) { + d = new TxnAuxData; + + TSUserArgSet(txn, md.txnArgIndex, d); + + TSHttpTxnHookAdd(txn, TS_HTTP_TXN_CLOSE_HOOK, md.txnCloseContp); + } + return *d; + } + +private: + static int + _deleteAuxData(TSCont, TSEvent, void *edata) + { + MgrData_ &md = access(MDRef); + + auto txn = static_cast(edata); + auto data = static_cast(TSUserArgGet(txn, md.txnArgIndex)); + delete data; + TSHttpTxnReenable(txn, TS_EVENT_HTTP_CONTINUE); + return 0; + }; +}; + +} // end namespace atscppapi diff --git a/plugins/xdebug/Makefile.inc b/plugins/xdebug/Makefile.inc index 1e41324b72c..a69e223ffc8 100644 --- a/plugins/xdebug/Makefile.inc +++ b/plugins/xdebug/Makefile.inc @@ -15,4 +15,6 @@ # limitations under the License. pkglib_LTLIBRARIES += xdebug/xdebug.la -xdebug_xdebug_la_SOURCES = xdebug/xdebug.cc +xdebug_xdebug_la_SOURCES = \ +xdebug/Cleanup.h \ +xdebug/xdebug.cc diff --git a/plugins/xdebug/xdebug.cc b/plugins/xdebug/xdebug.cc index 1b6d877f58b..30a1d108c58 100644 --- a/plugins/xdebug/xdebug.cc +++ b/plugins/xdebug/xdebug.cc @@ -22,6 +22,8 @@ #include #include #include +#include +#include #include #include #include @@ -32,6 +34,34 @@ #include "tscore/ink_defs.h" #include "tscpp/util/PostScript.h" #include "tscpp/util/TextView.h" +#include "Cleanup.h" + +namespace +{ +struct BodyBuilder { + atscppapi::TSContUniqPtr transform_connp; + atscppapi::TSIOBufferUniqPtr output_buffer; + // It's important that output_reader comes after output_buffer so it will be deleted first. + atscppapi::TSIOBufferReaderUniqPtr output_reader; + TSVIO output_vio = nullptr; + bool wrote_prebody = false; + bool wrote_body = false; + bool hdr_ready = false; + std::atomic_flag wrote_postbody; + + int64_t nbytes = 0; +}; + +struct XDebugTxnAuxData { + std::unique_ptr body_builder; + unsigned xheaders = 0; +}; + +atscppapi::TxnAuxMgrData mgrData; + +using AuxDataMgr = atscppapi::TxnAuxDataMgr; + +} // end anonymous namespace #include "xdebug_headers.cc" #include "xdebug_transforms.cc" @@ -50,10 +80,9 @@ enum { XHEADER_X_DUMP_HEADERS = 1u << 7, XHEADER_X_REMAP = 1u << 8, XHEADER_X_PROBE_HEADERS = 1u << 9, + XHEADER_X_PSELECT_KEY = 1u << 10, }; -static int XArgIndex = 0; -static int BodyBuilderArgIndex = 0; static TSCont XInjectHeadersCont = nullptr; static TSCont XDeleteDebugHdrCont = nullptr; @@ -131,7 +160,7 @@ InjectCacheKeyHeader(TSHttpTxn txn, TSMBuffer buffer, TSMLoc hdr) } // Now copy the cache lookup URL into the response header. - TSReleaseAssert(TSMimeHdrFieldValueStringInsert(buffer, hdr, dst, 0 /* idx */, strval.ptr, strval.len) == TS_SUCCESS); + TSReleaseAssert(TSMimeHdrFieldValueStringInsert(buffer, hdr, dst, -1 /* idx */, strval.ptr, strval.len) == TS_SUCCESS); done: if (dst != TS_NULL_MLOC) { @@ -168,11 +197,11 @@ InjectCacheHeader(TSHttpTxn txn, TSMBuffer buffer, TSMLoc hdr) if (TSHttpTxnCacheLookupStatusGet(txn, &status) == TS_ERROR) { // If the cache lookup hasn't happened yes, TSHttpTxnCacheLookupStatusGet will fail. - TSReleaseAssert(TSMimeHdrFieldValueStringInsert(buffer, hdr, dst, 0 /* idx */, "none", 4) == TS_SUCCESS); + TSReleaseAssert(TSMimeHdrFieldValueStringInsert(buffer, hdr, dst, -1 /* idx */, "none", 4) == TS_SUCCESS); } else { const char *msg = (status < 0 || status >= static_cast(countof(names))) ? "unknown" : names[status]; - TSReleaseAssert(TSMimeHdrFieldValueStringInsert(buffer, hdr, dst, 0 /* idx */, msg, -1) == TS_SUCCESS); + TSReleaseAssert(TSMimeHdrFieldValueStringInsert(buffer, hdr, dst, -1 /* idx */, msg, -1) == TS_SUCCESS); } done: @@ -243,7 +272,7 @@ InjectMilestonesHeader(TSHttpTxn txn, TSMBuffer buffer, TSMLoc hdr) double elapsed = static_cast(time - epoch) / 1000000000.0; int len = snprintf(hdrval, sizeof(hdrval), "%s=%1.9lf", milestones[i].msname, elapsed); - TSReleaseAssert(TSMimeHdrFieldValueStringInsert(buffer, hdr, dst, 0 /* idx */, hdrval, len) == TS_SUCCESS); + TSReleaseAssert(TSMimeHdrFieldValueStringInsert(buffer, hdr, dst, -1 /* idx */, hdrval, len) == TS_SUCCESS); } } @@ -305,7 +334,7 @@ InjectRemapHeader(TSHttpTxn txn, TSMBuffer buffer, TSMLoc hdr) TSfree(const_cast(toUrlStr)); } - TSReleaseAssert(TSMimeHdrFieldValueStringInsert(buffer, hdr, dst, 0 /* idx */, buf, len) == TS_SUCCESS); + TSReleaseAssert(TSMimeHdrFieldValueStringInsert(buffer, hdr, dst, -1 /* idx */, buf, len) == TS_SUCCESS); TSHandleMLocRelease(buffer, hdr, dst); } } @@ -320,9 +349,56 @@ InjectTxnUuidHeader(TSHttpTxn txn, TSMBuffer buffer, TSMLoc hdr) TSUuid uuid = TSProcessUuidGet(); int len = snprintf(buf, sizeof(buf), "%s-%" PRIu64 "", TSUuidStringGet(uuid), TSHttpTxnIdGet(txn)); - TSReleaseAssert(TSMimeHdrFieldValueStringInsert(buffer, hdr, dst, 0 /* idx */, buf, len) == TS_SUCCESS); + TSReleaseAssert(TSMimeHdrFieldValueStringInsert(buffer, hdr, dst, -1 /* idx */, buf, len) == TS_SUCCESS); + TSHandleMLocRelease(buffer, hdr, dst); + } +} + +static void +InjectParentSelectionKeyHeader(TSHttpTxn txn, TSMBuffer buffer, TSMLoc hdr) +{ + TSMLoc url = TS_NULL_MLOC; + TSMLoc dst = TS_NULL_MLOC; + + struct { + char *ptr; + int len; + } strval = {nullptr, 0}; + + TSDebug("xdebug", "attempting to inject X-ParentSelection-Key header"); + + if (TSUrlCreate(buffer, &url) != TS_SUCCESS) { + goto done; + } + + if (TSHttpTxnParentSelectionUrlGet(txn, buffer, url) != TS_SUCCESS) { + goto done; + } + + strval.ptr = TSUrlStringGet(buffer, url, &strval.len); + if (strval.ptr == nullptr || strval.len == 0) { + goto done; + } + + // Create a new response header field. + dst = FindOrMakeHdrField(buffer, hdr, "X-ParentSelection-Key", lengthof("X-ParentSelection-Key")); + if (dst == TS_NULL_MLOC) { + goto done; + } + + // Now copy the parent selection lookup URL into the response header. + TSReleaseAssert(TSMimeHdrFieldValueStringInsert(buffer, hdr, dst, -1 /* idx */, strval.ptr, strval.len) == TS_SUCCESS); + +done: + if (dst != TS_NULL_MLOC) { TSHandleMLocRelease(buffer, hdr, dst); } + + if (url != TS_NULL_MLOC) { + TSHandleMLocRelease(buffer, TS_NULL_MLOC, url); + } + + TSfree(strval.ptr); } static int @@ -334,7 +410,7 @@ XInjectResponseHeaders(TSCont /* contp */, TSEvent event, void *edata) TSReleaseAssert(event == TS_EVENT_HTTP_SEND_RESPONSE_HDR); - uintptr_t xheaders = reinterpret_cast(TSHttpTxnArgGet(txn, XArgIndex)); + unsigned xheaders = AuxDataMgr::data(txn).xheaders; if (xheaders == 0) { goto done; } @@ -374,14 +450,18 @@ XInjectResponseHeaders(TSCont /* contp */, TSEvent event, void *edata) } if (xheaders & XHEADER_X_PROBE_HEADERS) { - BodyBuilder *data = static_cast(TSHttpTxnArgGet(txn, BodyBuilderArgIndex)); + BodyBuilder *data = AuxDataMgr::data(txn).body_builder.get(); TSDebug("xdebug_transform", "XInjectResponseHeaders(): client resp header ready"); if (data == nullptr) { TSHttpTxnReenable(txn, TS_EVENT_HTTP_ERROR); return TS_ERROR; } data->hdr_ready = true; - writePostBody(data); + writePostBody(txn, data); + } + + if (xheaders & XHEADER_X_PSELECT_KEY) { + InjectParentSelectionKeyHeader(txn, buffer, hdr); } done: @@ -441,9 +521,9 @@ isFwdFieldValue(std::string_view value, intmax_t &fwdCnt) static int XScanRequestHeaders(TSCont /* contp */, TSEvent event, void *edata) { - TSHttpTxn txn = static_cast(edata); - uintptr_t xheaders = 0; - intmax_t fwdCnt = 0; + TSHttpTxn txn = static_cast(edata); + unsigned xheaders = 0; + intmax_t fwdCnt = 0; TSMLoc field, next; TSMBuffer buffer; TSMLoc hdr; @@ -498,26 +578,17 @@ XScanRequestHeaders(TSCont /* contp */, TSEvent event, void *edata) } else if (header_field_eq("probe", value, vsize)) { xheaders |= XHEADER_X_PROBE_HEADERS; + auto &auxData = AuxDataMgr::data(txn); + // prefix request headers and postfix response headers BodyBuilder *data = new BodyBuilder(); - data->txn = txn; + auxData.body_builder.reset(data); TSVConn connp = TSTransformCreate(body_transform, txn); - TSContDataSet(connp, data); + data->transform_connp.reset(connp); + TSContDataSet(connp, txn); TSHttpTxnHookAdd(txn, TS_HTTP_RESPONSE_TRANSFORM_HOOK, connp); - // store data pointer in txnarg to use in global cont XInjectResponseHeaders - TSHttpTxnArgSet(txn, BodyBuilderArgIndex, data); - - // create a self-cleanup on close - auto cleanupBodyBuilder = [](TSCont /* contp */, TSEvent event, void *edata) -> int { - TSHttpTxn txn = static_cast(edata); - BodyBuilder *data = static_cast(TSHttpTxnArgGet(txn, BodyBuilderArgIndex)); - delete data; - return TS_EVENT_NONE; - }; - TSHttpTxnHookAdd(txn, TS_HTTP_TXN_CLOSE_HOOK, TSContCreate(cleanupBodyBuilder, nullptr)); - // disable writing to cache because we are injecting data into the body. TSHttpTxnReqCacheableSet(txn, 0); TSHttpTxnRespCacheableSet(txn, 0); @@ -525,6 +596,9 @@ XScanRequestHeaders(TSCont /* contp */, TSEvent event, void *edata) TSHttpTxnTransformedRespCache(txn, 0); TSHttpTxnUntransformedRespCache(txn, 0); + } else if (header_field_eq("x-parentselection-key", value, vsize)) { + xheaders |= XHEADER_X_PSELECT_KEY; + } else if (isFwdFieldValue(std::string_view(value, vsize), fwdCnt)) { if (fwdCnt > 0) { // Decrement forward count in X-Debug header. @@ -553,7 +627,7 @@ XScanRequestHeaders(TSCont /* contp */, TSEvent event, void *edata) TSDebug("xdebug", "adding response hook for header mask %p and forward count %" PRIiMAX, reinterpret_cast(xheaders), fwdCnt); TSHttpTxnHookAdd(txn, TS_HTTP_SEND_RESPONSE_HDR_HOOK, XInjectHeadersCont); - TSHttpTxnArgSet(txn, XArgIndex, reinterpret_cast(xheaders)); + AuxDataMgr::data(txn).xheaders = xheaders; if (fwdCnt == 0) { // X-Debug header has to be deleted, but not too soon for other plugins to see it. @@ -585,7 +659,6 @@ XDeleteDebugHdr(TSCont /* contp */, TSEvent event, void *edata) field = TSMimeHdrFieldFind(buffer, hdr, xDebugHeader.str, xDebugHeader.len); if (field == TS_NULL_MLOC) { - TSError("Missing %s header", xDebugHeader.str); return TS_EVENT_NONE; } @@ -633,9 +706,16 @@ TSPluginInit(int argc, const char *argv[]) } xDebugHeader.len = strlen(xDebugHeader.str); + // Make xDebugHeader available to other plugins, as a C-style string. + // + int idx = -1; + TSReleaseAssert(TSUserArgIndexReserve(TS_USER_ARGS_GLB, "XDebugHeader", "XDebug header name", &idx) == TS_SUCCESS); + TSReleaseAssert(idx >= 0); + TSUserArgSet(nullptr, idx, const_cast(xDebugHeader.str)); + + AuxDataMgr::init("xdebug"); + // Setup the global hook - TSReleaseAssert(TSHttpTxnArgIndexReserve("xdebug", "xdebug header requests", &XArgIndex) == TS_SUCCESS); - TSReleaseAssert(TSHttpTxnArgIndexReserve("bodyTransform", "BodyBuilder*", &XArgIndex) == TS_SUCCESS); TSReleaseAssert(XInjectHeadersCont = TSContCreate(XInjectResponseHeaders, nullptr)); TSReleaseAssert(XDeleteDebugHdrCont = TSContCreate(XDeleteDebugHdr, nullptr)); TSHttpHookAdd(TS_HTTP_READ_REQUEST_HDR_HOOK, TSContCreate(XScanRequestHeaders, nullptr)); diff --git a/plugins/xdebug/xdebug_headers.cc b/plugins/xdebug/xdebug_headers.cc index aef6871146a..003658f76a5 100644 --- a/plugins/xdebug/xdebug_headers.cc +++ b/plugins/xdebug/xdebug_headers.cc @@ -28,44 +28,88 @@ #define DEBUG_TAG_LOG_HEADERS "xdebug.headers" -std::string_view -escape_char_for_json(char const &c, bool &parsing_key) +class EscapeCharForJson { - switch (c) { - case '\'': - return {"\\\'"}; - case '"': - return {"\\\""}; - case '\\': - return {"\\\\"}; - case '\b': - return {"\\b"}; - case '\f': - return {"\\f"}; - case '\t': - return {"\\t"}; - - // Special header reformatting - case '\r': - return {""}; - case '\n': - parsing_key = true; - return {"',\r\n\t'"}; // replace new line with pair delimiter - case ':': - if (parsing_key) { - return {"' : "}; // replace colon after key with quote + colon +public: + std::string_view + operator()(char const &c) + { + if ((_state != IN_VALUE) && ((' ' == c) || ('\t' == c))) { + return {""}; } - return {":"}; - case ' ': - if (parsing_key) { - parsing_key = false; - return {"'"}; // replace first space after the key to be a quote + if ((IN_NAME == _state) && (':' == c)) { + _state = BEFORE_VALUE; + return {"' : '"}; + } + if ('\r' == c) { + return {""}; + } + if ('\n' == c) { + std::string_view result{_after_value()}; + + if (BEFORE_NAME == _state) { + return {""}; + } else if (BEFORE_VALUE == _state) { + // Failsafe -- missing value -- this should never happen. + result = _missing_value(); + } + _state = BEFORE_NAME; + return result; + } + if (BEFORE_NAME == _state) { + _state = IN_NAME; + } else if (BEFORE_VALUE == _state) { + _state = IN_VALUE; + } + switch (c) { + case '\'': + return {"\\\'"}; + case '"': + return {"\\\""}; + case '\\': + return {"\\\\"}; + case '\b': + return {"\\b"}; + case '\f': + return {"\\f"}; + case '\t': + return {"\\t"}; + default: + return {&c, 1}; } - return {" "}; - default: - return {&c, 1}; } -} + + // After last header line, back up and throw away everything but the closing quote. + // + static std::size_t + backup() + { + return _after_value().size() - 1; + } + +private: + static std::string_view + _missing_value() + { + return {"' : '',\n\t'"}; + } + + static std::string_view + _after_name() + { + return {_missing_value().data(), 5}; + } + + static std::string_view + _after_value() + { + return {_missing_value().data() + 5, 5}; + } + + enum _State { BEFORE_NAME, IN_NAME, BEFORE_VALUE, IN_VALUE }; + + _State _state{BEFORE_VALUE}; +}; /////////////////////////////////////////////////////////////////////////// // Dump a header on stderr, useful together with TSDebug(). @@ -77,37 +121,33 @@ print_headers(TSHttpTxn txn, TSMBuffer bufp, TSMLoc hdr_loc, std::stringstream & TSIOBufferBlock block; const char *block_start; int64_t block_avail; - bool parsing_key = true; - size_t print_rewind = ss.str().length(); - output_buffer = TSIOBufferCreate(); - reader = TSIOBufferReaderAlloc(output_buffer); + EscapeCharForJson escape_char_for_json; + output_buffer = TSIOBufferCreate(); + reader = TSIOBufferReaderAlloc(output_buffer); - ss << "\t'"; - /* This will print just MIMEFields and not the http request line */ - TSMimeHdrPrint(bufp, hdr_loc, output_buffer); + ss << "\t'Start-Line' : '"; + + // Print all message header lines. + TSHttpHdrPrint(bufp, hdr_loc, output_buffer); /* We need to loop over all the buffer blocks, there can be more than 1 */ block = TSIOBufferReaderStart(reader); do { block_start = TSIOBufferBlockReadStart(block, reader, &block_avail); for (const char *c = block_start; c < block_start + block_avail; ++c) { - bool was_parsing_key = parsing_key; - ss << escape_char_for_json(*c, parsing_key); - if (parsing_key && !was_parsing_key) { - print_rewind = ss.str().length() - 1; - } + ss << escape_char_for_json(*c); } TSIOBufferReaderConsume(reader, block_avail); block = TSIOBufferReaderStart(reader); } while (block && block_avail != 0); - ss.seekp(print_rewind); + ss.seekp(-escape_char_for_json.backup(), std::ios_base::end); /* Free up the TSIOBuffer that we used to print out the header */ TSIOBufferReaderFree(reader); TSIOBufferDestroy(output_buffer); - TSDebug(DEBUG_TAG_LOG_HEADERS, "%s", ss.str().c_str()); + TSDebug(DEBUG_TAG_LOG_HEADERS, "%.*s", static_cast(ss.tellp()), ss.str().data()); } void @@ -128,13 +168,13 @@ print_request_headers(TSHttpTxn txn, std::stringstream &output) if (TSHttpTxnClientReqGet(txn, &buf_c, &hdr_loc) == TS_SUCCESS) { output << "{'type':'request', 'side':'client', 'headers': {\n"; print_headers(txn, buf_c, hdr_loc, output); - output << "}}"; + output << "\n\t}}"; TSHandleMLocRelease(buf_c, TS_NULL_MLOC, hdr_loc); } if (TSHttpTxnServerReqGet(txn, &buf_s, &hdr_loc) == TS_SUCCESS) { output << ",{'type':'request', 'side':'server', 'headers': {\n"; print_headers(txn, buf_s, hdr_loc, output); - output << "}}"; + output << "\n\t}}"; TSHandleMLocRelease(buf_s, TS_NULL_MLOC, hdr_loc); } } @@ -147,13 +187,13 @@ print_response_headers(TSHttpTxn txn, std::stringstream &output) if (TSHttpTxnServerRespGet(txn, &buf_s, &hdr_loc) == TS_SUCCESS) { output << "{'type':'response', 'side':'server', 'headers': {\n"; print_headers(txn, buf_s, hdr_loc, output); - output << "}},"; + output << "\n\t}},"; TSHandleMLocRelease(buf_s, TS_NULL_MLOC, hdr_loc); } if (TSHttpTxnClientRespGet(txn, &buf_c, &hdr_loc) == TS_SUCCESS) { output << "{'type':'response', 'side':'client', 'headers': {\n"; print_headers(txn, buf_c, hdr_loc, output); - output << "}}"; + output << "\n\t}}"; TSHandleMLocRelease(buf_c, TS_NULL_MLOC, hdr_loc); } } diff --git a/plugins/xdebug/xdebug_transforms.cc b/plugins/xdebug/xdebug_transforms.cc index a5bc75a0a73..e1019e726ec 100644 --- a/plugins/xdebug/xdebug_transforms.cc +++ b/plugins/xdebug/xdebug_transforms.cc @@ -21,25 +21,11 @@ #include #include #include -#include #include "ts/ts.h" static const std::string_view MultipartBoundary{"\r\n--- ATS xDebug Probe Injection Boundary ---\r\n\r\n"}; -struct BodyBuilder { - TSVIO output_vio = nullptr; - TSIOBuffer output_buffer = nullptr; - TSIOBufferReader output_reader = nullptr; - bool wrote_prebody = false; - bool wrote_body = false; - bool hdr_ready = false; - std::atomic_flag wrote_postbody; - - int64_t nbytes = 0; - TSHttpTxn txn = nullptr; -}; - static char Hostname[1024]; static std::string @@ -65,12 +51,12 @@ getPostBody(TSHttpTxn txn) } static void -writePostBody(BodyBuilder *data) +writePostBody(TSHttpTxn txn, BodyBuilder *data) { if (data->wrote_body && data->hdr_ready && !data->wrote_postbody.test_and_set()) { TSDebug("xdebug_transform", "body_transform(): Writing postbody headers..."); - std::string postbody = getPostBody(data->txn); - TSIOBufferWrite(data->output_buffer, postbody.data(), postbody.length()); + std::string postbody = getPostBody(txn); + TSIOBufferWrite(data->output_buffer.get(), postbody.data(), postbody.length()); data->nbytes += postbody.length(); TSVIONBytesSet(data->output_vio, data->nbytes); TSVIOReenable(data->output_vio); @@ -80,15 +66,13 @@ writePostBody(BodyBuilder *data) static int body_transform(TSCont contp, TSEvent event, void *edata) { - BodyBuilder *data = static_cast(TSContDataGet(contp)); + TSHttpTxn txn = static_cast(TSContDataGet(contp)); + BodyBuilder *data = AuxDataMgr::data(txn).body_builder.get(); if (!data) { - TSContDestroy(contp); return TS_ERROR; } if (TSVConnClosedGet(contp)) { - // write connection destroyed. cleanup. - delete data; - TSContDestroy(contp); + // write connection destroyed. return 0; } @@ -108,16 +92,16 @@ body_transform(TSCont contp, TSEvent event, void *edata) TSDebug("xdebug_transform", "body_transform(): Event is TS_EVENT_VCONN_WRITE_READY"); // fall through default: - if (!data->output_buffer) { - data->output_buffer = TSIOBufferCreate(); - data->output_reader = TSIOBufferReaderAlloc(data->output_buffer); - data->output_vio = TSVConnWrite(TSTransformOutputVConnGet(contp), contp, data->output_reader, INT64_MAX); + if (!data->output_buffer.get()) { + data->output_buffer.reset(TSIOBufferCreate()); + data->output_reader.reset(TSIOBufferReaderAlloc(data->output_buffer.get())); + data->output_vio = TSVConnWrite(TSTransformOutputVConnGet(contp), contp, data->output_reader.get(), INT64_MAX); } if (data->wrote_prebody == false) { TSDebug("xdebug_transform", "body_transform(): Writing prebody headers..."); - std::string prebody = getPreBody(data->txn); - TSIOBufferWrite(data->output_buffer, prebody.data(), prebody.length()); // write prebody + std::string prebody = getPreBody(txn); + TSIOBufferWrite(data->output_buffer.get(), prebody.data(), prebody.length()); // write prebody data->wrote_prebody = true; data->nbytes += prebody.length(); } @@ -127,7 +111,7 @@ body_transform(TSCont contp, TSEvent event, void *edata) if (!src_buf) { // upstream continuation shuts down write operation. data->wrote_body = true; - writePostBody(data); + writePostBody(txn, data); return 0; } @@ -150,7 +134,7 @@ body_transform(TSCont contp, TSEvent event, void *edata) // Write post body content and update output VIO data->wrote_body = true; data->nbytes += TSVIONDoneGet(src_vio); - writePostBody(data); + writePostBody(txn, data); TSContCall(TSVIOContGet(src_vio), TS_EVENT_VCONN_WRITE_COMPLETE, src_vio); } } diff --git a/proxy/CacheControl.cc b/proxy/CacheControl.cc index 072c4709e99..5a0a229bc7c 100644 --- a/proxy/CacheControl.cc +++ b/proxy/CacheControl.cc @@ -31,6 +31,7 @@ #include #include "tscore/ink_config.h" +#include "tscore/Filenames.h" #include "CacheControl.h" #include "ControlMatcher.h" #include "Main.h" @@ -143,20 +144,20 @@ initCacheControl() void reloadCacheControl() { - Note("cache.config loading ..."); + Note("%s loading ...", ts::filename::CACHE); CC_table *newTable; - Debug("cache_control", "cache.config updated, reloading"); + Debug("cache_control", "%s updated, reloading", ts::filename::CACHE); eventProcessor.schedule_in(new CC_FreerContinuation(CacheControlTable), CACHE_CONTROL_TIMEOUT, ET_CACHE); newTable = new CC_table("proxy.config.cache.control.filename", modulePrefix, &http_dest_tags); ink_atomic_swap(&CacheControlTable, newTable); - Note("cache.config finished loading"); + Note("%s finished loading", ts::filename::CACHE); } void -getCacheControl(CacheControlResult *result, HttpRequestData *rdata, OverridableHttpConfigParams *h_txn_conf, char *tag) +getCacheControl(CacheControlResult *result, HttpRequestData *rdata, const OverridableHttpConfigParams *h_txn_conf, char *tag) { rdata->tag = tag; CacheControlTable->Match(rdata, result); @@ -297,7 +298,7 @@ CacheControlRecord::Init(matcher_line *line_info) directive = CC_IGNORE_SERVER_NO_CACHE; d_found = true; } else { - return Result::failure("%s Invalid action at line %d in cache.config", modulePrefix, line_num); + return Result::failure("%s Invalid action at line %d in %s", modulePrefix, line_num, ts::filename::CACHE); } } else { if (strcasecmp(label, "revalidate") == 0) { @@ -317,7 +318,7 @@ CacheControlRecord::Init(matcher_line *line_info) this->time_arg = time_in; } else { - return Result::failure("%s %s at line %d in cache.config", modulePrefix, tmp, line_num); + return Result::failure("%s %s at line %d in %s", modulePrefix, tmp, line_num, ts::filename::CACHE); } } } @@ -331,14 +332,14 @@ CacheControlRecord::Init(matcher_line *line_info) } if (d_found == false) { - return Result::failure("%s No directive in cache.config at line %d", modulePrefix, line_num); + return Result::failure("%s No directive in %s at line %d", modulePrefix, ts::filename::CACHE, line_num); } // Process any modifiers to the directive, if they exist if (line_info->num_el > 0) { tmp = ProcessModifiers(line_info); if (tmp != nullptr) { - return Result::failure("%s %s at line %d in cache.config", modulePrefix, tmp, line_num); + return Result::failure("%s %s at line %d in %s", modulePrefix, tmp, line_num, ts::filename::CACHE); } } diff --git a/proxy/CacheControl.h b/proxy/CacheControl.h index 83ebc46a9e0..e6730c6990a 100644 --- a/proxy/CacheControl.h +++ b/proxy/CacheControl.h @@ -121,7 +121,7 @@ class URL; struct HttpConfigParams; struct OverridableHttpConfigParams; -inkcoreapi void getCacheControl(CacheControlResult *result, HttpRequestData *rdata, OverridableHttpConfigParams *h_txn_conf, +inkcoreapi void getCacheControl(CacheControlResult *result, HttpRequestData *rdata, const OverridableHttpConfigParams *h_txn_conf, char *tag = nullptr); inkcoreapi bool host_rule_in_CacheControlTable(); inkcoreapi bool ip_rule_in_CacheControlTable(); diff --git a/proxy/ClassH.txt b/proxy/ClassH.txt deleted file mode 100644 index abe6701d8f4..00000000000 --- a/proxy/ClassH.txt +++ /dev/null @@ -1,97 +0,0 @@ - - - Thread { thread_key, processor } - - | - | - - EventThread { event_mutex, - DLL timeout_events, - DLL free_events - } - - | - | - - IOThread { DLL io_events, - DLL free_connections; - } - - / \ - / \ - DNSThread AcceptThread - - - - - - - VConnection - - | - | - - EventContinuation - - / \ - / \ - - IOVConnection DNSEntry - - / \ \ - / \ \ - \ \ -DiskIOVConnection \ \ - NetIOVConnection \ - DNSIOVConnection - - - - - - Processor - - | - | - EventProcessor - - | - | - - IOProcessor - - / \ - / \ - \ - DiskProcessor - NetProcessor - - | - | - - DNSProcessor - - - - - -class VConnection: -================== -vio_queue : queue of io operations. -read_vio : current read vio. -wrire_vio : current write vio. - - -Thread: -======= -Processor: a reference to the processor that started the thread. - -EventThread: -============ -timeout_events: a list (DLL) of EventContinuation -free_events: - -IOThread: -========= -io_events: list of IOVConnection - diff --git a/proxy/ControlBase.h b/proxy/ControlBase.h index 76ffb25861d..4c79d544865 100644 --- a/proxy/ControlBase.h +++ b/proxy/ControlBase.h @@ -65,10 +65,10 @@ class ControlBase @return @c true if the request is matched, @c false if not. */ virtual bool check(HttpRequestData *req ///< Request to check. - ) const = 0; + ) const = 0; /// Print the mod information. virtual void print(FILE *f ///< Output stream. - ) const = 0; + ) const = 0; }; ControlBase(); diff --git a/proxy/ControlMatcher.cc b/proxy/ControlMatcher.cc index 7dff4745fb5..0a85f73a620 100644 --- a/proxy/ControlMatcher.cc +++ b/proxy/ControlMatcher.cc @@ -33,6 +33,7 @@ #include "tscore/ink_config.h" #include "tscore/MatcherUtils.h" #include "tscore/Tokenizer.h" +#include "tscore/ts_file.h" #include "ProxyConfig.h" #include "ControlMatcher.h" #include "CacheControl.h" @@ -499,10 +500,10 @@ RegexMatcher::Match(RequestData *rdata, MatchResult *result) if (url_str == nullptr) { url_str = ats_strdup(""); } + // INKqa12980 // The function unescapifyStr() is already called in // HttpRequestData::get_string(); therefore, no need to call again here. - // unescapifyStr(url_str); for (int i = 0; i < num_el; i++) { r = pcre_exec(re_array[i], nullptr, url_str, strlen(url_str), 0, 0, nullptr, 0); @@ -663,10 +664,10 @@ void IpMatcher::Print() { printf("\tIp Matcher with %d elements, %zu ranges.\n", num_el, ip_map.count()); - for (IpMap::iterator spot(ip_map.begin()), limit(ip_map.end()); spot != limit; ++spot) { + for (auto &spot : ip_map) { char b1[INET6_ADDRSTRLEN], b2[INET6_ADDRSTRLEN]; - printf("\tRange %s - %s ", ats_ip_ntop(spot->min(), b1, sizeof b1), ats_ip_ntop(spot->max(), b2, sizeof b2)); - static_cast(spot->data())->Print(); + printf("\tRange %s - %s ", ats_ip_ntop(spot.min(), b1, sizeof b1), ats_ip_ntop(spot.max(), b2, sizeof b2)); + static_cast(spot.data())->Print(); } } @@ -795,6 +796,7 @@ ControlMatcher::BuildTableFromString(char *file_buf) // We have an empty file return 0; } + // First get the number of entries tmp = bufTok.iterFirst(&i_state); while (tmp != nullptr) { @@ -911,8 +913,7 @@ ControlMatcher::BuildTableFromString(char *file_buf) Result::failure("%s discarding %s entry with unknown type at line %d", matcher_name, config_file_path, current->line_num); } - // Check to see if there was an error in creating - // the NewEntry + // Check to see if there was an error in creating the NewEntry if (error.failed()) { SignalError(error.message(), alarmAlready); } @@ -935,48 +936,25 @@ template int ControlMatcher::BuildTable() { - // File I/O Locals - char *file_buf; - int ret; - - file_buf = readIntoBuffer(config_file_path, matcher_name, nullptr); - - if (file_buf == nullptr) { - return 1; + std::error_code ec; + std::string content{ts::file::load(ts::file::path{config_file_path}, ec)}; + if (ec) { + switch (ec.value()) { + case ENOENT: + Warning("ControlMatcher - Cannot open config file: %s - %s", config_file_path, strerror(ec.value())); + break; + default: + Error("ControlMatcher - %s failed to load: %s", config_file_path, strerror(ec.value())); + return 1; + } } - ret = BuildTableFromString(file_buf); - ats_free(file_buf); - return ret; + return BuildTableFromString(content.data()); } /**************************************************************** * TEMPLATE INSTANTIATIONS GO HERE * - * We have to explicitly instantiate the templates so that - * everything works on with dec ccx, sun CC, and g++ - * - * Details on the different comipilers: - * - * dec ccx: Does not seem to instantiate anything automatically - * so it needs all templates manually instantiated - * - * sun CC: Automatic instantiation works but since we make - * use of the templates in other files, instantiation - * only occurs when those files are compiled, breaking - * the dependency system. Explict instantiation - * in this file causes the templates to be reinstantiated - * when this file changes. - * - * Also, does not give error messages about template - * compilation problems. Requires the -verbose=template - * flage to error messages - * - * g++: Requires instantiation to occur in the same file as the - * the implementation. Instantiating ControlMatcher - * automatically instatiatiates the other templates since - * ControlMatcher makes use of them - * ****************************************************************/ template class ControlMatcher; diff --git a/proxy/ControlMatcher.h b/proxy/ControlMatcher.h index 50cb2916e6e..7b48ab6c6f3 100644 --- a/proxy/ControlMatcher.h +++ b/proxy/ControlMatcher.h @@ -327,25 +327,13 @@ template class ControlMatcher return reMatch; } - UrlMatcher * - getUrlMatcher() - { - return urlMatch; - } - IpMatcher * getIPMatcher() { return ipMatch; } - HostRegexMatcher * - getHrMatcher() - { - return hrMatch; - } - - // private: + // private RegexMatcher *reMatch; UrlMatcher *urlMatch; HostMatcher *hostMatch; diff --git a/proxy/IPAllow.cc b/proxy/IPAllow.cc index 1761a0c7e54..776c8b5fd34 100644 --- a/proxy/IPAllow.cc +++ b/proxy/IPAllow.cc @@ -29,6 +29,7 @@ #include "tscore/BufferWriter.h" #include "tscore/ts_file.h" #include "tscore/ink_memory.h" +#include "tscore/Filenames.h" #include "yaml-cpp/yaml.h" @@ -136,14 +137,14 @@ IpAllow::reconfigure() { self_type *new_table; - Note("ip_allow.yaml loading ..."); + Note("%s loading ...", ts::filename::IP_ALLOW); new_table = new self_type("proxy.config.cache.ip_allow.filename"); new_table->BuildTable(); configid = configProcessor.set(configid, new_table); - Note("ip_allow.yaml finished loading"); + Note("%s finished loading", ts::filename::IP_ALLOW); } IpAllow * diff --git a/proxy/InkAPIInternal.h b/proxy/InkAPIInternal.h index 25fab38b991..d140996afa6 100644 --- a/proxy/InkAPIInternal.h +++ b/proxy/InkAPIInternal.h @@ -40,8 +40,6 @@ /* Some defines that might be candidates for configurable settings later. */ -#define TS_HTTP_MAX_USER_ARG 16 /* max number of user arguments for Transactions and Sessions */ - typedef int8_t TSMgmtByte; // Not for external use /* ****** Cache Structure ********* */ @@ -226,7 +224,9 @@ FeatureAPIHooks::get(ID id) const return likely(is_valid(id)) ? m_hooks[id].head() : nullptr; } -template APIHooks const *FeatureAPIHooks::operator[](ID id) const +template +APIHooks const * +FeatureAPIHooks::operator[](ID id) const { return likely(is_valid(id)) ? &(m_hooks[id]) : nullptr; } @@ -357,8 +357,9 @@ class HttpHookState protected: /// Track the state of one scope of hooks. struct Scope { - APIHook const *_c; ///< Current hook (candidate for invocation). - APIHook const *_p; ///< Previous hook (already invoked). + APIHook const *_c; ///< Current hook (candidate for invocation). + APIHook const *_p; ///< Previous hook (already invoked). + APIHooks const *_hooks; ///< Reference to the real hook list /// Initialize the scope. void init(HttpAPIHooks const *scope, TSHttpHookID id); diff --git a/proxy/Makefile.am b/proxy/Makefile.am index bac2ea445ff..33a2ad1dc22 100644 --- a/proxy/Makefile.am +++ b/proxy/Makefile.am @@ -18,6 +18,8 @@ include $(top_srcdir)/build/tidy.mk +include private/Makefile.inc + SUBDIRS = hdrs shared http http2 logging if ENABLE_QUIC SUBDIRS += http3 @@ -44,6 +46,7 @@ noinst_HEADERS = \ Show.h libproxy_a_SOURCES = \ + $(PRIVATE_SOURCES_) \ CacheControl.cc \ CacheControl.h \ ControlBase.cc \ diff --git a/proxy/Milestones.h b/proxy/Milestones.h index 744a326be79..737fd0523c8 100644 --- a/proxy/Milestones.h +++ b/proxy/Milestones.h @@ -33,8 +33,16 @@ template class Milestones { public: - ink_hrtime &operator[](T ms) { return this->_milestones[static_cast(ms)]; } - ink_hrtime operator[](T ms) const { return this->_milestones[static_cast(ms)]; } + ink_hrtime & + operator[](T ms) + { + return this->_milestones[static_cast(ms)]; + } + ink_hrtime + operator[](T ms) const + { + return this->_milestones[static_cast(ms)]; + } /** * Mark given milestone with timestamp if it's not marked yet diff --git a/proxy/ParentConsistentHash.cc b/proxy/ParentConsistentHash.cc index 1d374e5574c..568fba97d92 100644 --- a/proxy/ParentConsistentHash.cc +++ b/proxy/ParentConsistentHash.cc @@ -108,7 +108,7 @@ ParentConsistentHash::getPathHash(HttpRequestData *hrdata, ATSHash64 *h) // Helper function to abstract calling ATSConsistentHash lookup_by_hashval() vs lookup(). static pRecord * chash_lookup(ATSConsistentHash *fhash, uint64_t path_hash, ATSConsistentHashIter *chashIter, bool *wrap_around, - ATSHash64Sip24 *hash, bool *chash_init) + ATSHash64Sip24 *hash, bool *chash_init, bool *mapWrapped) { pRecord *prtmp; @@ -118,6 +118,12 @@ chash_lookup(ATSConsistentHash *fhash, uint64_t path_hash, ATSConsistentHashIter } else { prtmp = (pRecord *)fhash->lookup(nullptr, chashIter, wrap_around, hash); } + // Do not set wrap_around to true until we try all the parents atleast once. + bool wrapped = *wrap_around; + *wrap_around = (*mapWrapped && *wrap_around) ? true : false; + if (!*mapWrapped && wrapped) { + *mapWrapped = true; + } return prtmp; } @@ -188,7 +194,7 @@ ParentConsistentHash::selectParent(bool first_call, ParentResult *result, Reques fhash = chash[last_lookup]; do { // search until we've selected a different parent if !firstCall prtmp = chash_lookup(fhash, path_hash, &result->chashIter[last_lookup], &wrap_around[last_lookup], &hash, - &result->chash_init[last_lookup]); + &result->chash_init[last_lookup], &result->mapWrapped[last_lookup]); lookups++; if (prtmp) { pRec = (parents[last_lookup] + prtmp->idx); @@ -219,10 +225,6 @@ ParentConsistentHash::selectParent(bool first_call, ParentResult *result, Reques } } if (!pRec || (pRec && !pRec->available) || host_stat == HOST_STATUS_DOWN) { - if (firstCall) { - result->chash_init[PRIMARY] = false; - result->chash_init[SECONDARY] = false; - } do { // check if the host is retryable. It's retryable if the retry window has elapsed // and the global host status is HOST_STATUS_UP @@ -276,7 +278,7 @@ ParentConsistentHash::selectParent(bool first_call, ParentResult *result, Reques } fhash = chash[last_lookup]; prtmp = chash_lookup(fhash, path_hash, &result->chashIter[last_lookup], &wrap_around[last_lookup], &hash, - &result->chash_init[last_lookup]); + &result->chash_init[last_lookup], &result->mapWrapped[last_lookup]); lookups++; if (prtmp) { pRec = (parents[last_lookup] + prtmp->idx); diff --git a/proxy/ParentRoundRobin.cc b/proxy/ParentRoundRobin.cc index edd089cbfb4..5c955bb2415 100644 --- a/proxy/ParentRoundRobin.cc +++ b/proxy/ParentRoundRobin.cc @@ -99,8 +99,8 @@ ParentRoundRobin::selectParent(bool first_call, ParentResult *result, RequestDat } break; case P_STRICT_ROUND_ROBIN: - cur_index = ink_atomic_increment(reinterpret_cast(&result->rec->rr_next), 1); - cur_index = result->start_parent = cur_index % num_parents; + cur_index = result->start_parent = + ink_atomic_increment(reinterpret_cast(&result->rec->rr_next), 1) % num_parents; break; case P_NO_ROUND_ROBIN: cur_index = result->start_parent = 0; diff --git a/proxy/ParentSelection.cc b/proxy/ParentSelection.cc index e8a787798d0..1b6a861e3f6 100644 --- a/proxy/ParentSelection.cc +++ b/proxy/ParentSelection.cc @@ -30,6 +30,7 @@ #include "HTTP.h" #include "HttpTransact.h" #include "I_Machine.h" +#include "tscore/Filenames.h" #define MAX_SIMPLE_RETRIES 5 #define MAX_UNAVAILABLE_SERVER_RETRIES 5 @@ -47,8 +48,6 @@ static const char *default_var = "proxy.config.http.parent_proxies"; static const char *retry_var = "proxy.config.http.parent_proxy.retry_time"; static const char *threshold_var = "proxy.config.http.parent_proxy.fail_threshold"; -static const char *ParentResultStr[] = {"PARENT_UNDEFINED", "PARENT_DIRECT", "PARENT_SPECIFIED", "PARENT_AGENT", "PARENT_FAIL"}; - // // Config Callback Prototypes // @@ -107,12 +106,6 @@ ParentConfigParams::findParent(HttpRequestData *rdata, ParentResult *result, uns ParentRecord *defaultPtr = DefaultParent; ParentRecord *rec; - Debug("parent_select", "In ParentConfigParams::findParent(): parent_table: %p.", parent_table); - ink_assert(result->result == PARENT_UNDEFINED); - - // Initialize the result structure - result->reset(); - // Check to see if the parent was set through the // api if (apiParentExists(rdata)) { @@ -127,6 +120,9 @@ ParentConfigParams::findParent(HttpRequestData *rdata, ParentResult *result, uns return; } + // Initialize the result structure + result->reset(); + tablePtr->Match(rdata, result); rec = result->rec; @@ -227,17 +223,38 @@ ParentConfigParams::nextParent(HttpRequestData *rdata, ParentResult *result, uns bool ParentConfigParams::parentExists(HttpRequestData *rdata) { - unsigned int fail_threshold = policy.FailThreshold; - unsigned int retry_time = policy.ParentRetryTime; + P_table *tablePtr = parent_table; + ParentRecord *rec = nullptr; ParentResult result; - findParent(rdata, &result, fail_threshold, retry_time); + // Initialize the result structure; + result.reset(); - if (result.result == PARENT_SPECIFIED) { - return true; - } else { + tablePtr->Match(rdata, &result); + rec = result.rec; + + if (rec == nullptr) { + Debug("parent_select", "No matching parent record was found for the request."); return false; } + + if (rec->num_parents > 0) { + for (int ii = 0; ii < rec->num_parents; ii++) { + if (rec->parents[ii].available) { + Debug("parent_select", "found available parent: %s", rec->parents[ii].hostname); + return true; + } + } + } + if (rec->secondary_parents && rec->num_secondary_parents > 0) { + for (int ii = 0; ii < rec->num_secondary_parents; ii++) { + if (rec->secondary_parents[ii].available) { + Debug("parent_select", "found available parent: %s", rec->secondary_parents[ii].hostname); + return true; + } + } + } + return false; } int ParentConfig::m_id = 0; @@ -264,7 +281,7 @@ ParentConfig::startup() void ParentConfig::reconfigure() { - Note("parent.config loading ..."); + Note("%s loading ...", ts::filename::PARENT); ParentConfigParams *params = nullptr; @@ -280,7 +297,7 @@ ParentConfig::reconfigure() ParentConfig::print(); } - Note("parent.config finished loading"); + Note("%s finished loading", ts::filename::PARENT); } // void ParentConfig::print @@ -333,6 +350,33 @@ UnavailableServerResponseCodes::UnavailableServerResponseCodes(char *val) std::sort(codes.begin(), codes.end()); } +SimpleRetryResponseCodes::SimpleRetryResponseCodes(char *val) +{ + Tokenizer pTok(", \t\r"); + int numTok = 0, c; + + if (val == nullptr) { + Warning("SimpleRetryResponseCodes - simple_server_retry_responses is null loading default 404 code."); + codes.push_back(HTTP_STATUS_NOT_FOUND); + return; + } + numTok = pTok.Initialize(val, SHARE_TOKS); + if (numTok == 0) { + c = atoi(val); + if (c > 399 && c < 500) { + codes.push_back(HTTP_STATUS_NOT_FOUND); + } + } + for (int i = 0; i < numTok; i++) { + c = atoi(pTok[i]); + if (c > 399 && c < 500) { + Debug("parent_select", "loading simple response code: %d", c); + codes.push_back(c); + } + } + std::sort(codes.begin(), codes.end()); +} + void ParentRecord::PreProcessParents(const char *val, const int line_num, char *buf, size_t len) { @@ -416,7 +460,6 @@ ParentRecord::ProcessParents(char *val, bool isPrimary) if (numTok == 0) { return "No parents specified"; } - HostStatus &hs = HostStatus::instance(); // Allocate the parents array if (isPrimary) { this->parents = static_cast(ats_malloc(sizeof(pRecord) * numTok)); @@ -500,10 +543,6 @@ ParentRecord::ProcessParents(char *val, bool isPrimary) memcpy(this->parents[i].hash_string, tmp3 + 1, strlen(tmp3)); this->parents[i].name = this->parents[i].hash_string; } - HostStatRec *hst = hs.getHostStatus(this->parents[i].hostname); - if (hst == nullptr) { - hs.setHostStatus(this->parents[i].hostname, HOST_STATUS_UP, 0, Reason::MANUAL); - } } else { memcpy(this->secondary_parents[i].hostname, current, tmp - current); this->secondary_parents[i].hostname[tmp - current] = '\0'; @@ -519,10 +558,6 @@ ParentRecord::ProcessParents(char *val, bool isPrimary) memcpy(this->secondary_parents[i].hash_string, tmp3 + 1, strlen(tmp3)); this->secondary_parents[i].name = this->secondary_parents[i].hash_string; } - HostStatRec *hst = hs.getHostStatus(this->secondary_parents[i].hostname); - if (hst == nullptr) { - hs.setHostStatus(this->secondary_parents[i].hostname, HOST_STATUS_UP, 0, Reason::MANUAL); - } } tmp3 = nullptr; } @@ -686,6 +721,9 @@ ParentRecord::Init(matcher_line *line_info) } else if (strcasecmp(label, "unavailable_server_retry_responses") == 0 && unavailable_server_retry_responses == nullptr) { unavailable_server_retry_responses = new UnavailableServerResponseCodes(val); used = true; + } else if (strcasecmp(label, "simple_server_retry_responses") == 0 && simple_server_retry_responses == nullptr) { + simple_server_retry_responses = new SimpleRetryResponseCodes(val); + used = true; } else if (strcasecmp(label, "max_simple_retries") == 0) { int v = atoi(val); if (v >= 1 && v < MAX_SIMPLE_RETRIES) { @@ -742,15 +780,27 @@ ParentRecord::Init(matcher_line *line_info) unavailable_server_retry_responses = new UnavailableServerResponseCodes(nullptr); } + // delete simple_server_retry_responses if simple_retry is not enabled. + if (simple_server_retry_responses != nullptr && !(parent_retry & PARENT_RETRY_SIMPLE)) { + Warning("%s ignore simple_server_Retry_responses directive on line %d, as simple_server_retry is not enabled.", modulePrefix, + line_num); + delete simple_server_retry_responses; + simple_server_retry_responses = nullptr; + } else if (simple_server_retry_responses == nullptr && (parent_retry & PARENT_RETRY_SIMPLE)) { + // initialize simple server respones codes to the default value if simple_retry is enabled. + Warning("%s initializing SimpleRetryResponseCodes on line %d to 404 default.", modulePrefix, line_num); + simple_server_retry_responses = new SimpleRetryResponseCodes(nullptr); + } + if (this->parents == nullptr && go_direct == false) { - return Result::failure("%s No parent specified in parent.config at line %d", modulePrefix, line_num); + return Result::failure("%s No parent specified in %s at line %d", modulePrefix, ts::filename::PARENT, line_num); } // Process any modifiers to the directive, if they exist if (line_info->num_el > 0) { tmp = ProcessModifiers(line_info); if (tmp != nullptr) { - return Result::failure("%s %s at line %d in parent.config", modulePrefix, tmp, line_num); + return Result::failure("%s %s at line %d in %s", modulePrefix, tmp, line_num, ts::filename::PARENT); } // record SCHEME modifier if present. // NULL if not present @@ -895,7 +945,7 @@ setup_socks_servers(ParentRecord *rec_arr, int len) void SocksServerConfig::reconfigure() { - Note("socks.config loading ..."); + Note("%s loading ...", ts::filename::SOCKS); char *default_val = nullptr; int retry_time = 30; @@ -935,7 +985,7 @@ SocksServerConfig::reconfigure() SocksServerConfig::print(); } - Note("socks.config finished loading"); + Note("%s finished loading", ts::filename::SOCKS); } void diff --git a/proxy/ParentSelection.h b/proxy/ParentSelection.h index 759a23eaf4a..f3353d058d7 100644 --- a/proxy/ParentSelection.h +++ b/proxy/ParentSelection.h @@ -59,12 +59,15 @@ enum ParentResultType { PARENT_FAIL, }; +static const char *ParentResultStr[] = {"PARENT_UNDEFINED", "PARENT_DIRECT", "PARENT_SPECIFIED", "PARENT_AGENT", "PARENT_FAIL"}; + enum ParentRR_t { P_NO_ROUND_ROBIN = 0, P_STRICT_ROUND_ROBIN, P_HASH_ROUND_ROBIN, P_CONSISTENT_HASH, P_LATCHED_ROUND_ROBIN, + P_UNDEFINED }; enum ParentRetry_t { @@ -89,6 +92,19 @@ struct UnavailableServerResponseCodes { std::vector codes; }; +struct SimpleRetryResponseCodes { + SimpleRetryResponseCodes(char *val); + ~SimpleRetryResponseCodes(){}; + + bool + contains(int code) + { + return binary_search(codes.begin(), codes.end(), code); + } + +private: + std::vector codes; +}; // struct pRecord // // A record for an individual parent @@ -142,6 +158,7 @@ class ParentRecord : public ControlBase bool parent_is_proxy = true; ParentSelectionStrategy *selection_strategy = nullptr; UnavailableServerResponseCodes *unavailable_server_retry_responses = nullptr; + SimpleRetryResponseCodes *simple_server_retry_responses = nullptr; ParentRetry_t parent_retry = PARENT_RETRY_NONE; int max_simple_retries = 1; int max_unavailable_server_retries = 1; @@ -155,6 +172,11 @@ class ParentRecord : public ControlBase // between HttpTransact & the parent selection code. The following ParentRecord *const extApiRecord = (ParentRecord *)0xeeeeffff; +// used here to to set the number of ATSConsistentHashIter's +// used in NextHopSelectionStrategy to limit the host group +// size as well, group size is one to one with the number of rings +constexpr const uint32_t MAX_GROUP_RINGS = 5; + struct ParentResult { ParentResult() { reset(); } // For outside consumption @@ -162,15 +184,17 @@ struct ParentResult { const char *hostname; int port; bool retry; - bool chash_init[2] = {false, false}; + bool chash_init[MAX_GROUP_RINGS] = {false}; HostStatus_t first_choice_status = HostStatus_t::HOST_STATUS_INIT; void reset() { ink_zero(*this); - line_number = -1; - result = PARENT_UNDEFINED; + line_number = -1; + result = PARENT_UNDEFINED; + mapWrapped[0] = false; + mapWrapped[1] = false; } bool @@ -232,7 +256,20 @@ struct ParentResult { bool response_is_retryable(HTTPStatus response_code) const { - return (retry_type() & PARENT_RETRY_UNAVAILABLE_SERVER) && rec->unavailable_server_retry_responses->contains(response_code); + Debug("parent_select", "In response_is_retryable, code: %d", response_code); + if (retry_type() == PARENT_RETRY_BOTH) { + Debug("parent_select", "Saw retry both"); + return (rec->unavailable_server_retry_responses->contains(response_code) || + rec->simple_server_retry_responses->contains(response_code)); + } else if (retry_type() == PARENT_RETRY_UNAVAILABLE_SERVER) { + Debug("parent_select", "Saw retry unavailable server"); + return rec->unavailable_server_retry_responses->contains(response_code); + } else if (retry_type() == PARENT_RETRY_SIMPLE) { + Debug("parent_select", "Saw retry simple retry"); + return rec->simple_server_retry_responses->contains(response_code); + } else { + return false; + } } bool @@ -248,6 +285,15 @@ struct ParentResult { } } + void + print() + { + printf("ParentResult - hostname: %s, port: %d, retry: %s, line_number: %d, last_parent: %d, start_parent: %d, wrap_around: %s, " + "last_lookup: %d, result: %s\n", + hostname, port, (retry) ? "true" : "false", line_number, last_parent, start_parent, (wrap_around) ? "true" : "false", + last_lookup, ParentResultStr[result]); + } + private: // Internal use only // Not to be modified by HTTP @@ -255,11 +301,16 @@ struct ParentResult { ParentRecord *rec; uint32_t last_parent; uint32_t start_parent; + uint32_t last_group; bool wrap_around; + bool mapWrapped[2]; // state for consistent hash. int last_lookup; - ATSConsistentHashIter chashIter[2]; + ATSConsistentHashIter chashIter[MAX_GROUP_RINGS]; + friend class NextHopSelectionStrategy; + friend class NextHopRoundRobin; + friend class NextHopConsistentHash; friend class ParentConsistentHash; friend class ParentRoundRobin; friend class ParentConfigParams; @@ -383,9 +434,6 @@ struct ParentConfig { // Helper Functions ParentRecord *createDefaultParent(char *val); -void reloadDefaultParent(char *val); -void reloadParentFile(); -int parentSelection_CB(const char *name, RecDataT data_type, RecData data, void *cookie); // Unit Test Functions void show_result(ParentResult *aParentResult); diff --git a/proxy/Plugin.cc b/proxy/Plugin.cc index 4687ddb5528..2a604d8bb7a 100644 --- a/proxy/Plugin.cc +++ b/proxy/Plugin.cc @@ -30,9 +30,39 @@ #include "InkAPIInternal.h" #include "Plugin.h" #include "tscore/ink_cap.h" +#include "tscore/Filenames.h" #define MAX_PLUGIN_ARGS 64 +static PluginDynamicReloadMode plugin_dynamic_reload_mode = PluginDynamicReloadMode::RELOAD_ON; + +bool +isPluginDynamicReloadEnabled() +{ + return PluginDynamicReloadMode::RELOAD_ON == plugin_dynamic_reload_mode; +} + +void +parsePluginDynamicReloadConfig() +{ + int int_plugin_dynamic_reload_mode; + + REC_ReadConfigInteger(int_plugin_dynamic_reload_mode, "proxy.config.plugin.dynamic_reload_mode"); + plugin_dynamic_reload_mode = static_cast(int_plugin_dynamic_reload_mode); + + if (plugin_dynamic_reload_mode < 0 || plugin_dynamic_reload_mode >= PluginDynamicReloadMode::RELOAD_COUNT) { + Warning("proxy.config.plugin.dynamic_reload_mode out of range. using default value."); + plugin_dynamic_reload_mode = PluginDynamicReloadMode::RELOAD_ON; + } + Note("Initialized plugin_dynamic_reload_mode: %d", plugin_dynamic_reload_mode); +} + +void +parsePluginConfig() +{ + parsePluginDynamicReloadConfig(); +} + static const char *plugin_dir = "."; using init_func_t = void (*)(int, char **); @@ -69,8 +99,29 @@ PluginRegInfo::~PluginRegInfo() } } +bool +plugin_dso_load(const char *path, void *&handle, void *&init, std::string &error) +{ + handle = dlopen(path, RTLD_NOW); + init = nullptr; + if (!handle) { + error.assign("unable to load '").append(path).append("': ").append(dlerror()); + Error("%s", error.c_str()); + return false; + } + + init = dlsym(handle, "TSPluginInit"); + if (!init) { + error.assign("unable to find TSPluginInit function in '").append(path).append("': ").append(dlerror()); + Error("%s", error.c_str()); + return false; + } + + return true; +} + static bool -plugin_load(int argc, char *argv[], bool validateOnly) +single_plugin_init(int argc, char *argv[], bool validateOnly) { char path[PATH_NAME_MAX]; init_func_t init; @@ -97,12 +148,17 @@ plugin_load(int argc, char *argv[], bool validateOnly) REC_ReadConfigInteger(elevate_access, "proxy.config.plugin.load_elevated"); ElevateAccess access(elevate_access ? ElevateAccess::FILE_PRIVILEGE : 0); - void *handle = dlopen(path, RTLD_NOW); - if (!handle) { + void *handle, *initptr = nullptr; + std::string error; + bool loaded = plugin_dso_load(path, handle, initptr, error); + init = reinterpret_cast(initptr); + + if (!loaded) { if (validateOnly) { return false; } - Fatal("unable to load '%s': %s", path, dlerror()); + Fatal("%s", error.c_str()); + return false; // this line won't get called since Fatal brings down ATS } // Allocate a new registration structure for the @@ -112,16 +168,6 @@ plugin_load(int argc, char *argv[], bool validateOnly) plugin_reg_current->plugin_path = ats_strdup(path); plugin_reg_current->dlh = handle; - init = reinterpret_cast(dlsym(plugin_reg_current->dlh, "TSPluginInit")); - if (!init) { - delete plugin_reg_current; - if (validateOnly) { - return false; - } - Fatal("unable to find TSPluginInit function in '%s': %s", path, dlerror()); - return false; // this line won't get called since Fatal brings down ATS - } - #if (!defined(kfreebsd) && defined(freebsd)) || defined(darwin) optreset = 1; #endif @@ -208,7 +254,7 @@ plugin_expand(char *arg) } not_found: - Warning("plugin.config: unable to find parameter %s", arg); + Warning("%s: unable to find parameter %s", ts::filename::PLUGIN, arg); return nullptr; } @@ -231,11 +277,11 @@ plugin_init(bool validateOnly) INIT_ONCE = false; } - Note("plugin.config loading ..."); - path = RecConfigReadConfigPath(nullptr, "plugin.config"); + Note("%s loading ...", ts::filename::PLUGIN); + path = RecConfigReadConfigPath(nullptr, ts::filename::PLUGIN); fd = open(path, O_RDONLY); if (fd < 0) { - Warning("plugin.config failed to load: %d, %s", errno, strerror(errno)); + Warning("%s failed to load: %d, %s", ts::filename::PLUGIN, errno, strerror(errno)); return false; } @@ -302,7 +348,7 @@ plugin_init(bool validateOnly) } else { argv[MAX_PLUGIN_ARGS - 1] = nullptr; } - retVal = plugin_load(argc, argv, validateOnly); + retVal = single_plugin_init(argc, argv, validateOnly); for (i = 0; i < argc; i++) { ats_free(vars[i]); @@ -311,9 +357,9 @@ plugin_init(bool validateOnly) close(fd); if (retVal) { - Note("plugin.config finished loading"); + Note("%s finished loading", ts::filename::PLUGIN); } else { - Error("plugin.config failed to load"); + Error("%s failed to load", ts::filename::PLUGIN); } return retVal; } diff --git a/proxy/Plugin.h b/proxy/Plugin.h index 09c3de69fca..86691a535f2 100644 --- a/proxy/Plugin.h +++ b/proxy/Plugin.h @@ -23,8 +23,16 @@ #pragma once +#include #include "tscore/List.h" +typedef enum { RELOAD_OFF, RELOAD_ON, RELOAD_COUNT } PluginDynamicReloadMode; + +// read records.config to parse plugin related configs +void parsePluginConfig(); + +bool isPluginDynamicReloadEnabled(); + struct PluginRegInfo { PluginRegInfo(); ~PluginRegInfo(); @@ -46,6 +54,7 @@ extern DLL plugin_reg_list; extern PluginRegInfo *plugin_reg_current; bool plugin_init(bool validateOnly = false); +bool plugin_dso_load(const char *path, void *&handle, void *&init, std::string &error); /** Abstract interface class for plugin based continuations. diff --git a/proxy/PluginVC.cc b/proxy/PluginVC.cc index f6cc4270592..fae0e96af18 100644 --- a/proxy/PluginVC.cc +++ b/proxy/PluginVC.cc @@ -886,6 +886,18 @@ PluginVC::set_inactivity_timeout(ink_hrtime timeout_in) } } +void +PluginVC::set_default_inactivity_timeout(ink_hrtime timeout_in) +{ + set_inactivity_timeout(timeout_in); +} + +bool +PluginVC::is_default_inactivity_timeout() +{ + return false; +} + void PluginVC::cancel_active_timeout() { @@ -926,7 +938,7 @@ bool PluginVC::add_to_active_queue() { // do nothing - return false; + return true; } SOCKET diff --git a/proxy/PluginVC.h b/proxy/PluginVC.h index ce420fd5861..cb0460de0b2 100644 --- a/proxy/PluginVC.h +++ b/proxy/PluginVC.h @@ -89,6 +89,8 @@ class PluginVC : public NetVConnection, public PluginIdentity // Timeouts void set_active_timeout(ink_hrtime timeout_in) override; void set_inactivity_timeout(ink_hrtime timeout_in) override; + void set_default_inactivity_timeout(ink_hrtime timeout_in) override; + bool is_default_inactivity_timeout() override; void cancel_active_timeout() override; void cancel_inactivity_timeout() override; void add_to_keep_alive_queue() override; diff --git a/proxy/ProtocolProbeSessionAccept.cc b/proxy/ProtocolProbeSessionAccept.cc index ed23291fe07..31c7f25fff8 100644 --- a/proxy/ProtocolProbeSessionAccept.cc +++ b/proxy/ProtocolProbeSessionAccept.cc @@ -93,7 +93,7 @@ struct ProtocolProbeTrampoline : public Continuation, public ProtocolProbeSessio } // if proxy_protocol is enabled via port descriptor AND the src IP is in - // the trusted whitelist for proxy protocol, then check to see if it is + // the trusted allowlist for proxy protocol, then check to see if it is // present IpMap *pp_ipmap; @@ -102,20 +102,20 @@ struct ProtocolProbeTrampoline : public Continuation, public ProtocolProbeSessio if (netvc->get_is_proxy_protocol()) { Debug("proxyprotocol", "ioCompletionEvent: proxy protocol is enabled on this port"); if (pp_ipmap->count() > 0) { - Debug("proxyprotocol", "ioCompletionEvent: proxy protocol has a configured whitelist of trusted IPs - checking"); + Debug("proxyprotocol", "ioCompletionEvent: proxy protocol has a configured allowlist of trusted IPs - checking"); void *payload = nullptr; if (!pp_ipmap->contains(netvc->get_remote_addr(), &payload)) { Debug("proxyprotocol", - "ioCompletionEvent: proxy protocol src IP is NOT in the configured whitelist of trusted IPs - closing connection"); + "ioCompletionEvent: proxy protocol src IP is NOT in the configured allowlist of trusted IPs - closing connection"); goto done; } else { char new_host[INET6_ADDRSTRLEN]; - Debug("proxyprotocol", "ioCompletionEvent: Source IP [%s] is trusted in the whitelist for proxy protocol", + Debug("proxyprotocol", "ioCompletionEvent: Source IP [%s] is trusted in the allowlist for proxy protocol", ats_ip_ntop(netvc->get_remote_addr(), new_host, sizeof(new_host))); } } else { Debug("proxyprotocol", - "ioCompletionEvent: proxy protocol DOES NOT have a configured whitelist of trusted IPs but proxy protocol is " + "ioCompletionEvent: proxy protocol DOES NOT have a configured allowlist of trusted IPs but proxy protocol is " "ernabled on this port - processing all connections"); } diff --git a/proxy/ProxySession.cc b/proxy/ProxySession.cc index 7063b9f3bea..3d9c2039fc4 100644 --- a/proxy/ProxySession.cc +++ b/proxy/ProxySession.cc @@ -24,13 +24,11 @@ #include "HttpConfig.h" #include "HttpDebugNames.h" #include "ProxySession.h" +#include "P_SSLNetVConnection.h" -static int64_t next_cs_id = 0; +ProxySession::ProxySession() : VConnection(nullptr) {} -ProxySession::ProxySession() : VConnection(nullptr) -{ - ink_zero(this->user_args); -} +ProxySession::ProxySession(NetVConnection *vc) : VConnection(nullptr), _vc(vc) {} void ProxySession::set_session_active() @@ -50,12 +48,6 @@ ProxySession::clear_session_active() } } -int64_t -ProxySession::next_connection_id() -{ - return ink_atomic_increment(&next_cs_id, 1); -} - static const TSEvent eventmap[TS_HTTP_LAST_HOOK + 1] = { TS_EVENT_HTTP_READ_REQUEST_HDR, // TS_HTTP_READ_REQUEST_HDR_HOOK TS_EVENT_HTTP_OS_DNS, // TS_HTTP_OS_DNS_HOOK @@ -87,6 +79,7 @@ ProxySession::free() this->api_hooks.clear(); this->mutex.clear(); this->acl.clear(); + this->_ssl.reset(); } int @@ -108,7 +101,7 @@ ProxySession::state_api_callout(int event, void *data) if (nullptr != cur_hook) { APIHook const *hook = cur_hook; - MUTEX_TRY_LOCK(lock, hook->m_cont->mutex, mutex->thread_holding); + WEAK_MUTEX_TRY_LOCK(lock, hook->m_cont->mutex, mutex->thread_holding); // Have a mutex but didn't get the lock, reschedule if (!lock.is_locked()) { @@ -116,7 +109,7 @@ ProxySession::state_api_callout(int event, void *data) if (!schedule_event) { // Don't bother if there is already one schedule_event = mutex->thread_holding->schedule_in(this, HRTIME_MSECONDS(10)); } - return 0; + return -1; } cur_hook = nullptr; // mark current callback at dispatched. @@ -140,7 +133,7 @@ ProxySession::state_api_callout(int event, void *data) return 0; } -void +int ProxySession::do_api_callout(TSHttpHookID id) { ink_assert(id == TS_HTTP_SSN_START_HOOK || id == TS_HTTP_SSN_CLOSE_HOOK); @@ -149,10 +142,11 @@ ProxySession::do_api_callout(TSHttpHookID id) cur_hook = hook_state.getNext(); if (nullptr != cur_hook) { SET_HANDLER(&ProxySession::state_api_callout); - this->state_api_callout(EVENT_NONE, nullptr); + return this->state_api_callout(EVENT_NONE, nullptr); } else { this->handle_api_return(TS_EVENT_HTTP_CONTINUE); } + return 0; } void @@ -182,53 +176,6 @@ ProxySession::handle_api_return(int event) } } -void * - -ProxySession::get_user_arg(unsigned ix) const -{ - ink_assert(ix < countof(user_args)); - return this->user_args[ix]; -} - -void -ProxySession::set_user_arg(unsigned ix, void *arg) -{ - ink_assert(ix < countof(user_args)); - user_args[ix] = arg; -} - -void -ProxySession::set_debug(bool flag) -{ - debug_on = flag; -} - -// Return whether debugging is enabled for this session. -bool -ProxySession::debug() const -{ - return this->debug_on; -} - -bool -ProxySession::is_active() const -{ - return m_active; -} - -bool -ProxySession::is_draining() const -{ - return TSSystemState::is_draining(); -} - -// Override if your session protocol allows this. -bool -ProxySession::is_transparent_passthrough_allowed() const -{ - return false; -} - bool ProxySession::is_chunked_encoding_supported() const { @@ -247,24 +194,6 @@ ProxySession::get_half_close_flag() const return false; } -in_port_t -ProxySession::get_outbound_port() const -{ - return outbound_port; -} - -IpAddr -ProxySession::get_outbound_ip4() const -{ - return outbound_ip4; -} - -IpAddr -ProxySession::get_outbound_ip6() const -{ - return outbound_ip6; -} - int64_t ProxySession::connection_id() const { @@ -282,80 +211,90 @@ ProxySession::get_server_session() const return nullptr; } -TSHttpHookID -ProxySession::get_hookid() const -{ - return hook_state.id(); -} - void ProxySession::set_active_timeout(ink_hrtime timeout_in) { + if (_vc) { + _vc->set_active_timeout(timeout_in); + } } void ProxySession::set_inactivity_timeout(ink_hrtime timeout_in) { + if (_vc) { + _vc->set_inactivity_timeout(timeout_in); + } } void ProxySession::cancel_inactivity_timeout() { -} - -bool -ProxySession::is_client_closed() const -{ - return get_netvc() == nullptr; + if (_vc) { + _vc->cancel_inactivity_timeout(); + } } int ProxySession::populate_protocol(std::string_view *result, int size) const { - auto vc = this->get_netvc(); - return vc ? vc->populate_protocol(result, size) : 0; + return _vc ? _vc->populate_protocol(result, size) : 0; } const char * ProxySession::protocol_contains(std::string_view tag_prefix) const { - auto vc = this->get_netvc(); - return vc ? vc->protocol_contains(tag_prefix) : nullptr; + return _vc ? _vc->protocol_contains(tag_prefix) : nullptr; } sockaddr const * ProxySession::get_client_addr() { - NetVConnection *netvc = get_netvc(); - return netvc ? netvc->get_remote_addr() : nullptr; + return _vc ? _vc->get_remote_addr() : nullptr; } + sockaddr const * ProxySession::get_local_addr() { - NetVConnection *netvc = get_netvc(); - return netvc ? netvc->get_local_addr() : nullptr; + return _vc ? _vc->get_local_addr() : nullptr; } void -ProxySession::hook_add(TSHttpHookID id, INKContInternal *cont) +ProxySession::_handle_if_ssl(NetVConnection *new_vc) +{ + auto ssl_vc = dynamic_cast(new_vc); + if (ssl_vc) { + _ssl = std::make_unique(); + _ssl.get()->init(*ssl_vc); + } +} + +VIO * +ProxySession::do_io_read(Continuation *c, int64_t nbytes, MIOBuffer *buf) +{ + return _vc ? this->_vc->do_io_read(c, nbytes, buf) : nullptr; +} + +VIO * +ProxySession::do_io_write(Continuation *c, int64_t nbytes, IOBufferReader *buf, bool owner) { - this->api_hooks.append(id, cont); + return _vc ? this->_vc->do_io_write(c, nbytes, buf, owner) : nullptr; } -APIHook * -ProxySession::hook_get(TSHttpHookID id) const +void +ProxySession::do_io_shutdown(ShutdownHowTo_t howto) { - return this->api_hooks.get(id); + this->_vc->do_io_shutdown(howto); } -HttpAPIHooks const * -ProxySession::feature_hooks() const +void +ProxySession::reenable(VIO *vio) { - return &api_hooks; + this->_vc->reenable(vio); } bool -ProxySession::has_hooks() const +ProxySession::support_sni() const { - return this->api_hooks.has_hooks() || http_global_hooks->has_hooks(); + return _vc ? _vc->support_sni() : false; } diff --git a/proxy/ProxySession.h b/proxy/ProxySession.h index 5f40af71829..ca24bc42c4c 100644 --- a/proxy/ProxySession.h +++ b/proxy/ProxySession.h @@ -27,10 +27,13 @@ #include "tscore/ink_resolver.h" #include "tscore/TSSystemState.h" #include +#include #include "P_Net.h" #include "InkAPIInternal.h" #include "http/Http1ServerSession.h" +#include "http/HttpSessionAccept.h" #include "IPAllow.h" +#include "private/SSLProxySession.h" // Emit a debug message conditional on whether this particular client session // has debugging enabled. This should only be called from within a client session @@ -71,10 +74,11 @@ struct ProxyError { }; /// Abstract class for HttpSM to interface with any session -class ProxySession : public VConnection +class ProxySession : public VConnection, public PluginUserArgs { public: ProxySession(); + ProxySession(NetVConnection *vc); // noncopyable ProxySession(ProxySession &) = delete; @@ -96,10 +100,9 @@ class ProxySession : public VConnection virtual void decrement_current_active_client_connections_stat() = 0; // Virtual Accessors - virtual NetVConnection *get_netvc() const = 0; + NetVConnection *get_netvc() const; virtual int get_transact_count() const = 0; virtual const char *get_protocol_string() const = 0; - virtual bool is_transparent_passthrough_allowed() const; virtual void hook_add(TSHttpHookID id, INKContInternal *cont); @@ -108,9 +111,6 @@ class ProxySession : public VConnection virtual void set_half_close_flag(bool flag); virtual bool get_half_close_flag() const; - virtual in_port_t get_outbound_port() const; - virtual IpAddr get_outbound_ip4() const; - virtual IpAddr get_outbound_ip6() const; virtual Http1ServerSession *get_server_session() const; // Replicate NetVConnection API @@ -125,11 +125,7 @@ class ProxySession : public VConnection virtual const char *protocol_contains(std::string_view tag_prefix) const; // Non-Virtual Methods - void do_api_callout(TSHttpHookID id); - - // Non-Virtual Accessors - void *get_user_arg(unsigned ix) const; - void set_user_arg(unsigned ix, void *arg); + int do_api_callout(TSHttpHookID id); void set_debug(bool flag); bool debug() const; @@ -144,18 +140,26 @@ class ProxySession : public VConnection TSHttpHookID get_hookid() const; bool has_hooks() const; + virtual bool support_sni() const; + APIHook *hook_get(TSHttpHookID id) const; HttpAPIHooks const *feature_hooks() const; + + // Returns null pointer if session does not use a TLS connection. + SSLProxySession const *ssl() const; + + // Implement VConnection interface + VIO *do_io_read(Continuation *c, int64_t nbytes = INT64_MAX, MIOBuffer *buf = nullptr) override; + VIO *do_io_write(Continuation *c = nullptr, int64_t nbytes = INT64_MAX, IOBufferReader *buf = 0, bool owner = false) override; + void do_io_shutdown(ShutdownHowTo_t howto) override; + void reenable(VIO *vio) override; + //////////////////// // Members IpAllow::ACL acl; ///< IpAllow based method ACL. - IpAddr outbound_ip4; ///< Local address for outbound connection. - IpAddr outbound_ip6; ///< Local address for outbound connection. - in_port_t outbound_port{0}; ///< Local port for outbound connection. - - HostResStyle host_res_style = HOST_RES_NONE; ///< DNS resolution preferences. + HttpSessionAccept::Options const *accept_options; ///< connection info // L7R TODO: set in constructor ink_hrtime ssn_start_time = 0; ink_hrtime ssn_last_txn_time = 0; @@ -174,17 +178,108 @@ class ProxySession : public VConnection int64_t con_id = 0; Event *schedule_event = nullptr; + // This function should be called in all overrides of new_connection() where + // the new_vc may be an SSLNetVConnection object. + void _handle_if_ssl(NetVConnection *new_vc); + + NetVConnection *_vc = nullptr; // The netvc associated with the concrete session class + private: void handle_api_return(int event); int state_api_callout(int event, void *edata); APIHook const *cur_hook = nullptr; HttpAPIHooks api_hooks; - void *user_args[TS_HTTP_MAX_USER_ARG]; // for DI. An active connection is one that a request has // been successfully parsed (PARSE_DONE) and it remains to // be active until the transaction goes through or the client // aborts. bool m_active = false; + + std::unique_ptr _ssl; }; + +/////////////////// +// INLINE + +inline int64_t next_cs_id = 0; + +inline int64_t +ProxySession::next_connection_id() +{ + return ink_atomic_increment(&next_cs_id, 1); +} + +inline void +ProxySession::set_debug(bool flag) +{ + debug_on = flag; +} + +// Return whether debugging is enabled for this session. +inline bool +ProxySession::debug() const +{ + return this->debug_on; +} + +inline bool +ProxySession::is_active() const +{ + return m_active; +} + +inline bool +ProxySession::is_draining() const +{ + return TSSystemState::is_draining(); +} + +inline bool +ProxySession::is_client_closed() const +{ + return get_netvc() == nullptr; +} + +inline TSHttpHookID +ProxySession::get_hookid() const +{ + return hook_state.id(); +} + +inline void +ProxySession::hook_add(TSHttpHookID id, INKContInternal *cont) +{ + this->api_hooks.append(id, cont); +} + +inline APIHook * +ProxySession::hook_get(TSHttpHookID id) const +{ + return this->api_hooks.get(id); +} + +inline HttpAPIHooks const * +ProxySession::feature_hooks() const +{ + return &api_hooks; +} + +inline bool +ProxySession::has_hooks() const +{ + return this->api_hooks.has_hooks() || http_global_hooks->has_hooks(); +} + +inline SSLProxySession const * +ProxySession::ssl() const +{ + return _ssl.get(); +} + +inline NetVConnection * +ProxySession::get_netvc() const +{ + return _vc; +} diff --git a/proxy/ProxyTransaction.cc b/proxy/ProxyTransaction.cc index 67b9c842e15..9be892ff999 100644 --- a/proxy/ProxyTransaction.cc +++ b/proxy/ProxyTransaction.cc @@ -30,241 +30,155 @@ ProxyTransaction::ProxyTransaction() : VConnection(nullptr) {} void -ProxyTransaction::new_transaction() +ProxyTransaction::new_transaction(bool from_early_data) { - ink_assert(current_reader == nullptr); + ink_assert(_sm == nullptr); // Defensive programming, make sure nothing persists across // connection re-use - ink_release_assert(proxy_ssn != nullptr); - current_reader = HttpSM::allocate(); - current_reader->init(); - HttpTxnDebug("[%" PRId64 "] Starting transaction %d using sm [%" PRId64 "]", proxy_ssn->connection_id(), - proxy_ssn->get_transact_count(), current_reader->sm_id); - - PluginIdentity *pi = dynamic_cast(this->get_netvc()); - if (pi) { - current_reader->plugin_tag = pi->getPluginTag(); - current_reader->plugin_id = pi->getPluginId(); + ink_release_assert(_proxy_ssn != nullptr); + _sm = HttpSM::allocate(); + _sm->init(from_early_data); + HttpTxnDebug("[%" PRId64 "] Starting transaction %d using sm [%" PRId64 "]", _proxy_ssn->connection_id(), + _proxy_ssn->get_transact_count(), _sm->sm_id); + + // PI tag valid only for internal requests + if (this->get_netvc()->get_is_internal_request()) { + PluginIdentity *pi = dynamic_cast(this->get_netvc()); + if (pi) { + _sm->plugin_tag = pi->getPluginTag(); + _sm->plugin_id = pi->getPluginId(); + } } this->increment_client_transactions_stat(); - current_reader->attach_client_session(this, sm_reader); + _sm->attach_client_session(this, _reader); } void ProxyTransaction::release(IOBufferReader *r) { - HttpTxnDebug("[%" PRId64 "] session released by sm [%" PRId64 "]", proxy_ssn ? proxy_ssn->connection_id() : 0, - current_reader ? current_reader->sm_id : 0); + HttpTxnDebug("[%" PRId64 "] session released by sm [%" PRId64 "]", _proxy_ssn ? _proxy_ssn->connection_id() : 0, + _sm ? _sm->sm_id : 0); this->decrement_client_transactions_stat(); // Pass along the release to the session - if (proxy_ssn) { - proxy_ssn->release(this); + if (_proxy_ssn) { + _proxy_ssn->release(this); } } void ProxyTransaction::attach_server_session(Http1ServerSession *ssession, bool transaction_done) { - proxy_ssn->attach_server_session(ssession, transaction_done); + _proxy_ssn->attach_server_session(ssession, transaction_done); } void ProxyTransaction::destroy() { - current_reader = nullptr; + _sm = nullptr; this->mutex.clear(); } -// See if we need to schedule on the primary thread for the transaction or change the thread that is associated with the VC. -// If we reschedule, the scheduled action is returned. Otherwise, NULL is returned -Action * -ProxyTransaction::adjust_thread(Continuation *cont, int event, void *data) -{ - NetVConnection *vc = this->get_netvc(); - EThread *this_thread = this_ethread(); - if (vc && vc->thread != this_thread) { - if (vc->thread->is_event_type(ET_NET)) { - return vc->thread->schedule_imm(cont, event, data); - } else { // Not a net thread, take over this thread - vc->thread = this_thread; - } - } - return nullptr; -} - void ProxyTransaction::set_rx_error_code(ProxyError e) { - if (this->current_reader) { - this->current_reader->t_state.client_info.rx_error_code = e; + if (this->_sm) { + this->_sm->t_state.client_info.rx_error_code = e; } } void ProxyTransaction::set_tx_error_code(ProxyError e) { - if (this->current_reader) { - this->current_reader->t_state.client_info.tx_error_code = e; + if (this->_sm) { + this->_sm->t_state.client_info.tx_error_code = e; } } NetVConnection * ProxyTransaction::get_netvc() const { - return (proxy_ssn) ? proxy_ssn->get_netvc() : nullptr; + return (_proxy_ssn) ? _proxy_ssn->get_netvc() : nullptr; } bool ProxyTransaction::is_first_transaction() const { - return proxy_ssn->get_transact_count() == 1; -} -// Ask your session if this is allowed -bool -ProxyTransaction::is_transparent_passthrough_allowed() -{ - return proxy_ssn ? proxy_ssn->is_transparent_passthrough_allowed() : false; -} - -bool -ProxyTransaction::is_chunked_encoding_supported() const -{ - return proxy_ssn ? proxy_ssn->is_chunked_encoding_supported() : false; -} - -void -ProxyTransaction::set_half_close_flag(bool flag) -{ - if (proxy_ssn) { - proxy_ssn->set_half_close_flag(flag); - } -} - -bool -ProxyTransaction::get_half_close_flag() const -{ - return proxy_ssn ? proxy_ssn->get_half_close_flag() : false; -} - -// What are the debug and hooks_enabled used for? How are they set? -// Just calling through to proxy session for now -bool -ProxyTransaction::debug() const -{ - return proxy_ssn ? proxy_ssn->debug() : false; -} - -APIHook * -ProxyTransaction::hook_get(TSHttpHookID id) const -{ - return proxy_ssn ? proxy_ssn->hook_get(id) : nullptr; -} - -HttpAPIHooks const * -ProxyTransaction::feature_hooks() const -{ - return proxy_ssn ? proxy_ssn->feature_hooks() : nullptr; -} - -bool -ProxyTransaction::has_hooks() const -{ - return proxy_ssn->has_hooks(); + return _proxy_ssn->get_transact_count() == 1; } void ProxyTransaction::set_session_active() { - if (proxy_ssn) { - proxy_ssn->set_session_active(); + if (_proxy_ssn) { + _proxy_ssn->set_session_active(); } } void ProxyTransaction::clear_session_active() { - if (proxy_ssn) { - proxy_ssn->clear_session_active(); + if (_proxy_ssn) { + _proxy_ssn->clear_session_active(); } } -/// DNS resolution preferences. -HostResStyle -ProxyTransaction::get_host_res_style() const -{ - return host_res_style; -} -void -ProxyTransaction::set_host_res_style(HostResStyle style) -{ - host_res_style = style; -} - const IpAllow::ACL & ProxyTransaction::get_acl() const { - return proxy_ssn ? proxy_ssn->acl : IpAllow::DENY_ALL_ACL; + return _proxy_ssn ? _proxy_ssn->acl : IpAllow::DENY_ALL_ACL; } // outbound values Set via the server port definition. Really only used for Http1 at the moment in_port_t ProxyTransaction::get_outbound_port() const { - return outbound_port; + return upstream_outbound_options.outbound_port; +} +void +ProxyTransaction::set_outbound_port(in_port_t port) +{ + upstream_outbound_options.outbound_port = port; } + IpAddr ProxyTransaction::get_outbound_ip4() const { - return outbound_ip4; + return upstream_outbound_options.outbound_ip4; } + IpAddr ProxyTransaction::get_outbound_ip6() const { - return outbound_ip6; -} -void -ProxyTransaction::set_outbound_port(in_port_t port) -{ - outbound_port = port; + return upstream_outbound_options.outbound_ip6; } + void ProxyTransaction::set_outbound_ip(const IpAddr &new_addr) { if (new_addr.isIp4()) { - outbound_ip4 = new_addr; + upstream_outbound_options.outbound_ip4 = new_addr; } else if (new_addr.isIp6()) { - outbound_ip6 = new_addr; + upstream_outbound_options.outbound_ip6 = new_addr; } else { - outbound_ip4.invalidate(); - outbound_ip6.invalidate(); + upstream_outbound_options.outbound_ip4.invalidate(); + upstream_outbound_options.outbound_ip6.invalidate(); } } bool ProxyTransaction::is_outbound_transparent() const { - return false; -} -void -ProxyTransaction::set_outbound_transparent(bool flag) -{ -} - -ProxySession * -ProxyTransaction::get_proxy_ssn() -{ - return proxy_ssn; + return upstream_outbound_options.f_outbound_transparent; } void -ProxyTransaction::set_proxy_ssn(ProxySession *new_proxy_ssn) +ProxyTransaction::set_outbound_transparent(bool flag) { - proxy_ssn = new_proxy_ssn; - host_res_style = proxy_ssn->host_res_style; + upstream_outbound_options.f_outbound_transparent = flag; } void @@ -272,43 +186,14 @@ ProxyTransaction::set_h2c_upgrade_flag() { } -Http1ServerSession * -ProxyTransaction::get_server_session() const -{ - return proxy_ssn ? proxy_ssn->get_server_session() : nullptr; -} - -HttpSM * -ProxyTransaction::get_sm() const -{ - return current_reader; -} - -const char * -ProxyTransaction::get_protocol_string() -{ - return proxy_ssn ? proxy_ssn->get_protocol_string() : nullptr; -} - -void -ProxyTransaction::set_restart_immediate(bool val) -{ - restart_immediate = true; -} -bool -ProxyTransaction::get_restart_immediate() const -{ - return restart_immediate; -} - int -ProxyTransaction::populate_protocol(std::string_view *result, int size) const +ProxyTransaction::get_transaction_priority_weight() const { - return proxy_ssn ? proxy_ssn->populate_protocol(result, size) : 0; + return 0; } -const char * -ProxyTransaction::protocol_contains(std::string_view tag_prefix) const +int +ProxyTransaction::get_transaction_priority_dependence() const { - return proxy_ssn ? proxy_ssn->protocol_contains(tag_prefix) : nullptr; + return 0; } diff --git a/proxy/ProxyTransaction.h b/proxy/ProxyTransaction.h index 7b16646f472..83a2111610c 100644 --- a/proxy/ProxyTransaction.h +++ b/proxy/ProxyTransaction.h @@ -37,7 +37,7 @@ class ProxyTransaction : public VConnection /// Virtual Methods // - virtual void new_transaction(); + virtual void new_transaction(bool from_early_data = false); virtual void attach_server_session(Http1ServerSession *ssession, bool transaction_done = true); Action *adjust_thread(Continuation *cont, int event, void *data); virtual void release(IOBufferReader *r); @@ -50,9 +50,11 @@ class ProxyTransaction : public VConnection virtual void set_inactivity_timeout(ink_hrtime timeout_in) = 0; virtual void cancel_inactivity_timeout() = 0; virtual int get_transaction_id() const = 0; - virtual bool allow_half_open() const = 0; - virtual void increment_client_transactions_stat() = 0; - virtual void decrement_client_transactions_stat() = 0; + virtual int get_transaction_priority_weight() const; + virtual int get_transaction_priority_dependence() const; + virtual bool allow_half_open() const = 0; + virtual void increment_client_transactions_stat() = 0; + virtual void decrement_client_transactions_stat() = 0; virtual NetVConnection *get_netvc() const; virtual bool is_first_transaction() const; @@ -73,14 +75,11 @@ class ProxyTransaction : public VConnection virtual void set_proxy_ssn(ProxySession *set_proxy_ssn); virtual void set_h2c_upgrade_flag(); - virtual const char *get_protocol_string(); - - virtual int populate_protocol(std::string_view *result, int size) const; - - virtual const char *protocol_contains(std::string_view tag_prefix) const; - /// Non-Virtual Methods // + const char *get_protocol_string(); + int populate_protocol(std::string_view *result, int size) const; + const char *protocol_contains(std::string_view tag_prefix) const; /// Non-Virtual Accessors // @@ -102,28 +101,140 @@ class ProxyTransaction : public VConnection Http1ServerSession *get_server_session() const; HttpSM *get_sm() const; - void set_restart_immediate(bool val); - bool get_restart_immediate() const; - // This function must return a non-negative number that is different for two in-progress transactions with the same proxy_ssn // session. // void set_rx_error_code(ProxyError e); void set_tx_error_code(ProxyError e); -protected: - ProxySession *proxy_ssn = nullptr; - HttpSM *current_reader = nullptr; - IOBufferReader *sm_reader = nullptr; + bool support_sni() const; - /// DNS resolution preferences. - HostResStyle host_res_style = HOST_RES_NONE; - /// Local outbound address control. - in_port_t outbound_port{0}; - IpAddr outbound_ip4; - IpAddr outbound_ip6; + /// Variables + // + HttpSessionAccept::Options upstream_outbound_options; // overwritable copy of options - bool restart_immediate = false; +protected: + ProxySession *_proxy_ssn = nullptr; + HttpSM *_sm = nullptr; + IOBufferReader *_reader = nullptr; private: }; + +//////////////////////////////////////////////////////////// +// INLINE + +inline bool +ProxyTransaction::is_transparent_passthrough_allowed() +{ + return upstream_outbound_options.f_transparent_passthrough; +} +inline bool +ProxyTransaction::is_chunked_encoding_supported() const +{ + return _proxy_ssn ? _proxy_ssn->is_chunked_encoding_supported() : false; +} +inline void +ProxyTransaction::set_half_close_flag(bool flag) +{ + if (_proxy_ssn) { + _proxy_ssn->set_half_close_flag(flag); + } +} + +inline bool +ProxyTransaction::get_half_close_flag() const +{ + return _proxy_ssn ? _proxy_ssn->get_half_close_flag() : false; +} + +// What are the debug and hooks_enabled used for? How are they set? +// Just calling through to proxy session for now +inline bool +ProxyTransaction::debug() const +{ + return _proxy_ssn ? _proxy_ssn->debug() : false; +} + +inline APIHook * +ProxyTransaction::hook_get(TSHttpHookID id) const +{ + return _proxy_ssn ? _proxy_ssn->hook_get(id) : nullptr; +} + +inline HttpAPIHooks const * +ProxyTransaction::feature_hooks() const +{ + return _proxy_ssn ? _proxy_ssn->feature_hooks() : nullptr; +} + +inline bool +ProxyTransaction::has_hooks() const +{ + return _proxy_ssn->has_hooks(); +} + +inline ProxySession * +ProxyTransaction::get_proxy_ssn() +{ + return _proxy_ssn; +} + +inline void +ProxyTransaction::set_proxy_ssn(ProxySession *new_proxy_ssn) +{ + _proxy_ssn = new_proxy_ssn; +} + +inline Http1ServerSession * +ProxyTransaction::get_server_session() const +{ + return _proxy_ssn ? _proxy_ssn->get_server_session() : nullptr; +} + +inline HttpSM * +ProxyTransaction::get_sm() const +{ + return _sm; +} + +inline const char * +ProxyTransaction::get_protocol_string() +{ + return _proxy_ssn ? _proxy_ssn->get_protocol_string() : nullptr; +} + +inline int +ProxyTransaction::populate_protocol(std::string_view *result, int size) const +{ + return _proxy_ssn ? _proxy_ssn->populate_protocol(result, size) : 0; +} + +inline const char * +ProxyTransaction::protocol_contains(std::string_view tag_prefix) const +{ + return _proxy_ssn ? _proxy_ssn->protocol_contains(tag_prefix) : nullptr; +} + +inline bool +ProxyTransaction::support_sni() const +{ + return _proxy_ssn ? _proxy_ssn->support_sni() : false; +} + +// See if we need to schedule on the primary thread for the transaction or change the thread that is associated with the VC. +// If we reschedule, the scheduled action is returned. Otherwise, NULL is returned +inline Action * +ProxyTransaction::adjust_thread(Continuation *cont, int event, void *data) +{ + NetVConnection *vc = this->get_netvc(); + EThread *this_thread = this_ethread(); + if (vc && vc->thread != this_thread) { + if (vc->thread->is_event_type(ET_NET)) { + return vc->thread->schedule_imm(cont, event, data); + } else { // Not a net thread, take over this thread + vc->thread = this_thread; + } + } + return nullptr; +} diff --git a/proxy/ReverseProxy.cc b/proxy/ReverseProxy.cc index d7d3ae09e45..22b44af4ff8 100644 --- a/proxy/ReverseProxy.cc +++ b/proxy/ReverseProxy.cc @@ -28,6 +28,7 @@ */ #include "tscore/ink_platform.h" +#include "tscore/Filenames.h" #include #include "P_EventSystem.h" #include "P_Cache.h" @@ -64,11 +65,11 @@ init_reverse_proxy() reconfig_mutex = new_ProxyMutex(); rewrite_table = new UrlRewrite(); - Note("remap.config loading ..."); + Note("%s loading ...", ts::filename::REMAP); if (!rewrite_table->load()) { - Fatal("remap.config failed to load"); + Fatal("%s failed to load", ts::filename::REMAP); } - Note("remap.config finished loading"); + Note("%s finished loading", ts::filename::REMAP); REC_RegisterConfigUpdateFunc("proxy.config.url_remap.filename", url_rewrite_CB, (void *)FILE_CHANGED); REC_RegisterConfigUpdateFunc("proxy.config.proxy_name", url_rewrite_CB, (void *)TSNAME_CHANGED); @@ -136,11 +137,11 @@ reloadUrlRewrite() { UrlRewrite *newTable, *oldTable; - Note("remap.config loading ..."); - Debug("url_rewrite", "remap.config updated, reloading..."); + Note("%s loading ...", ts::filename::REMAP); + Debug("url_rewrite", "%s updated, reloading...", ts::filename::REMAP); newTable = new UrlRewrite(); if (newTable->load()) { - static const char *msg = "remap.config finished loading"; + static const char *msg_format = "%s finished loading"; // Hold at least one lease, until we reload the configuration newTable->acquire(); @@ -151,18 +152,17 @@ reloadUrlRewrite() ink_assert(oldTable != nullptr); // Release the old one - oldTable->pluginFactory.indicateReload(); oldTable->release(); - Debug("url_rewrite", "%s", msg); - Note("%s", msg); + Debug("url_rewrite", msg_format, ts::filename::REMAP); + Note(msg_format, ts::filename::REMAP); return true; } else { - static const char *msg = "remap.config failed to load"; + static const char *msg_format = "%s failed to load"; delete newTable; - Debug("url_rewrite", "%s", msg); - Error("%s", msg); + Debug("url_rewrite", msg_format, ts::filename::REMAP); + Error(msg_format, ts::filename::REMAP); return false; } } diff --git a/proxy/Transform.cc b/proxy/Transform.cc index e51b26c543b..dd8e4766e16 100644 --- a/proxy/Transform.cc +++ b/proxy/Transform.cc @@ -125,11 +125,11 @@ TransformTerminus::TransformTerminus(TransformVConnection *tvc) SET_HANDLER(&TransformTerminus::handle_event); } -#define RETRY() \ - if (ink_atomic_increment((int *)&m_event_count, 1) < 0) { \ - ink_assert(!"not reached"); \ - } \ - eventProcessor.schedule_in(this, HRTIME_MSECONDS(10), ET_NET); \ +#define RETRY() \ + if (ink_atomic_increment((int *)&m_event_count, 1) < 0) { \ + ink_assert(!"not reached"); \ + } \ + this_ethread()->schedule_in(this, HRTIME_MSECONDS(10)); \ return 0; int @@ -280,7 +280,7 @@ TransformTerminus::do_io_read(Continuation *c, int64_t nbytes, MIOBuffer *buf) } Debug("transform", "[TransformTerminus::do_io_read] event_count %d", m_event_count); - eventProcessor.schedule_imm(this, ET_NET); + this_ethread()->schedule_imm_local(this); return &m_read_vio; } @@ -305,7 +305,7 @@ TransformTerminus::do_io_write(Continuation *c, int64_t nbytes, IOBufferReader * } Debug("transform", "[TransformTerminus::do_io_write] event_count %d", m_event_count); - eventProcessor.schedule_imm(this, ET_NET); + this_ethread()->schedule_imm_local(this); return &m_write_vio; } @@ -335,7 +335,7 @@ TransformTerminus::do_io_close(int error) m_write_vio.op = VIO::NONE; m_write_vio.buffer.clear(); - eventProcessor.schedule_imm(this, ET_NET); + this_ethread()->schedule_imm_local(this); } /*------------------------------------------------------------------------- @@ -368,7 +368,7 @@ TransformTerminus::reenable(VIO *vio) ink_assert(!"not reached"); } Debug("transform", "[TransformTerminus::reenable] event_count %d", m_event_count); - eventProcessor.schedule_imm(this, ET_NET); + this_ethread()->schedule_imm_local(this); } else { Debug("transform", "[TransformTerminus::reenable] skipping due to " "pending events"); @@ -549,7 +549,7 @@ TransformControl::handle_event(int event, void * /* edata ATS_UNUSED */) } ink_assert(m_tvc != nullptr); - m_write_buf = new_MIOBuffer(); + m_write_buf = new_MIOBuffer(BUFFER_SIZE_INDEX_32K); s = m_write_buf->end(); e = m_write_buf->buf_end(); @@ -561,7 +561,7 @@ TransformControl::handle_event(int event, void * /* edata ATS_UNUSED */) } case TRANSFORM_READ_READY: { - MIOBuffer *buf = new_empty_MIOBuffer(); + MIOBuffer *buf = new_empty_MIOBuffer(BUFFER_SIZE_INDEX_32K); m_read_buf = buf->alloc_reader(); m_tvc->do_io_read(this, INT64_MAX, buf); @@ -652,7 +652,7 @@ NullTransform::handle_event(int event, void *edata) ink_assert(m_output_vc != nullptr); if (!m_output_vio) { - m_output_buf = new_empty_MIOBuffer(); + m_output_buf = new_empty_MIOBuffer(BUFFER_SIZE_INDEX_32K); m_output_reader = m_output_buf->alloc_reader(); m_output_vio = m_output_vc->do_io_write(this, m_write_vio.nbytes, m_output_reader); } @@ -731,7 +731,7 @@ void TransformTest::run() { if (is_action_tag_set("transform_test")) { - eventProcessor.schedule_imm(new TransformControl(), ET_NET); + this_ethread()->schedule_imm_local(new TransformControl()); } } #endif @@ -802,7 +802,7 @@ RangeTransform::handle_event(int event, void *edata) ink_assert(m_output_vc != nullptr); if (!m_output_vio) { - m_output_buf = new_empty_MIOBuffer(); + m_output_buf = new_empty_MIOBuffer(BUFFER_SIZE_INDEX_32K); m_output_reader = m_output_buf->alloc_reader(); m_output_vio = m_output_vc->do_io_write(this, m_output_cl, m_output_reader); diff --git a/proxy/hdrs/HTTP.cc b/proxy/hdrs/HTTP.cc index 7c18f9162c1..c557172858e 100644 --- a/proxy/hdrs/HTTP.cc +++ b/proxy/hdrs/HTTP.cc @@ -959,8 +959,8 @@ http_parser_parse_req(HTTPParser *parser, HdrHeap *heap, HTTPHdrImpl *hh, const return PARSE_RESULT_ERROR; } - ParseResult ret = - mime_parser_parse(&parser->m_mime_parser, heap, hh->m_fields_impl, start, end, must_copy_strings, eof, max_hdr_field_size); + ParseResult ret = mime_parser_parse(&parser->m_mime_parser, heap, hh->m_fields_impl, start, end, must_copy_strings, eof, + false, max_hdr_field_size); // If we're done with the main parse do some validation if (ret == PARSE_RESULT_DONE) { ret = validate_hdr_host(hh); // check HOST header @@ -1114,20 +1114,18 @@ http_parser_parse_req(HTTPParser *parser, HdrHeap *heap, HTTPHdrImpl *hh, const end = real_end; parser->m_parsing_http = false; - - ParseResult ret = - mime_parser_parse(&parser->m_mime_parser, heap, hh->m_fields_impl, start, end, must_copy_strings, eof, max_hdr_field_size); - // If we're done with the main parse do some validation - if (ret == PARSE_RESULT_DONE) { - ret = validate_hdr_host(hh); // check HOST header - } - if (ret == PARSE_RESULT_DONE) { - ret = validate_hdr_content_length(heap, hh); - } - return ret; } - return mime_parser_parse(&parser->m_mime_parser, heap, hh->m_fields_impl, start, end, must_copy_strings, eof, max_hdr_field_size); + ParseResult ret = mime_parser_parse(&parser->m_mime_parser, heap, hh->m_fields_impl, start, end, must_copy_strings, eof, false, + max_hdr_field_size); + // If we're done with the main parse do some validation + if (ret == PARSE_RESULT_DONE) { + ret = validate_hdr_host(hh); // check HOST header + } + if (ret == PARSE_RESULT_DONE) { + ret = validate_hdr_content_length(heap, hh); + } + return ret; } ParseResult @@ -1179,7 +1177,7 @@ validate_hdr_content_length(HdrHeap *heap, HTTPHdrImpl *hh) if (mime_hdr_field_find(hh->m_fields_impl, MIME_FIELD_TRANSFER_ENCODING, MIME_LEN_TRANSFER_ENCODING) != nullptr) { // Delete all Content-Length headers Debug("http", "Transfer-Encoding header and Content-Length headers the request, removing all Content-Length headers"); - mime_hdr_field_delete(heap, hh->m_fields_impl, content_length_field); + mime_hdr_field_delete(heap, hh->m_fields_impl, content_length_field, true); return PARSE_RESULT_DONE; } @@ -1289,7 +1287,7 @@ http_parser_parse_resp(HTTPParser *parser, HdrHeap *heap, HTTPHdrImpl *hh, const end = real_end; parser->m_parsing_http = false; - return mime_parser_parse(&parser->m_mime_parser, heap, hh->m_fields_impl, start, end, must_copy_strings, eof); + return mime_parser_parse(&parser->m_mime_parser, heap, hh->m_fields_impl, start, end, must_copy_strings, eof, true); } #endif @@ -1405,7 +1403,7 @@ http_parser_parse_resp(HTTPParser *parser, HdrHeap *heap, HTTPHdrImpl *hh, const parser->m_parsing_http = false; } - return mime_parser_parse(&parser->m_mime_parser, heap, hh->m_fields_impl, start, end, must_copy_strings, eof); + return mime_parser_parse(&parser->m_mime_parser, heap, hh->m_fields_impl, start, end, must_copy_strings, eof, true); } /*------------------------------------------------------------------------- diff --git a/proxy/hdrs/HTTP.h b/proxy/hdrs/HTTP.h index 30fcf6c9ae5..0ae1077b585 100644 --- a/proxy/hdrs/HTTP.h +++ b/proxy/hdrs/HTTP.h @@ -78,6 +78,7 @@ enum HTTPStatus { HTTP_STATUS_REQUEST_URI_TOO_LONG = 414, HTTP_STATUS_UNSUPPORTED_MEDIA_TYPE = 415, HTTP_STATUS_RANGE_NOT_SATISFIABLE = 416, + HTTP_STATUS_TOO_EARLY = 425, HTTP_STATUS_INTERNAL_SERVER_ERROR = 500, HTTP_STATUS_NOT_IMPLEMENTED = 501, @@ -506,6 +507,8 @@ class HTTPHdr : public MIMEHdr /// also had a port, @c false otherwise. mutable bool m_port_in_header = false; + mutable bool early_data = false; + HTTPHdr() = default; // Force the creation of the default constructor int valid() const; @@ -583,7 +586,7 @@ class HTTPHdr : public MIMEHdr @note The results are cached so this is fast after the first call. @return A pointer to the host name. */ - const char *host_get(int *length = nullptr); + const char *host_get(int *length = nullptr) const; /** Get the target port. If the target port is not found then it is adjusted to the @@ -623,12 +626,15 @@ class HTTPHdr : public MIMEHdr /// header internals, they must be able to do this. void mark_target_dirty() const; - HTTPStatus status_get(); + HTTPStatus status_get() const; void status_set(HTTPStatus status); const char *reason_get(int *length); void reason_set(const char *value, int length); + void mark_early_data(bool flag = true) const; + bool is_early_data() const; + ParseResult parse_req(HTTPParser *parser, const char **start, const char *end, bool eof, bool strict_uri_parsing = false, size_t max_request_line_size = UINT16_MAX, size_t max_hdr_field_size = 131070); ParseResult parse_resp(HTTPParser *parser, const char **start, const char *end, bool eof); @@ -642,6 +648,7 @@ class HTTPHdr : public MIMEHdr bool is_cache_control_set(const char *cc_directive_wks); bool is_pragma_no_cache_set(); bool is_keep_alive_set() const; + bool expect_final_response() const; HTTPKeepAlive keep_alive_get() const; protected: @@ -850,7 +857,7 @@ HTTPHdr::_test_and_fill_target_cache() const -------------------------------------------------------------------------*/ inline const char * -HTTPHdr::host_get(int *length) +HTTPHdr::host_get(int *length) const { this->_test_and_fill_target_cache(); if (m_target_in_url) { @@ -1001,6 +1008,24 @@ HTTPHdr::is_keep_alive_set() const return this->keep_alive_get() == HTTP_KEEPALIVE; } +/** + Check the status code is informational and expecting final response + - e.g. "100 Continue", "103 Early Hints" + + Please note that "101 Switching Protocol" is not included. + */ +inline bool +HTTPHdr::expect_final_response() const +{ + switch (this->status_get()) { + case HTTP_STATUS_CONTINUE: + case HTTP_STATUS_EARLY_HINTS: + return true; + default: + return false; + } +} + /*------------------------------------------------------------------------- -------------------------------------------------------------------------*/ @@ -1151,7 +1176,7 @@ http_hdr_status_get(HTTPHdrImpl *hh) -------------------------------------------------------------------------*/ inline HTTPStatus -HTTPHdr::status_get() +HTTPHdr::status_get() const { ink_assert(valid()); @@ -1199,6 +1224,26 @@ HTTPHdr::reason_set(const char *value, int length) http_hdr_reason_set(m_heap, m_http, value, length, true); } +/*------------------------------------------------------------------------- + -------------------------------------------------------------------------*/ + +inline void +HTTPHdr::mark_early_data(bool flag) const +{ + ink_assert(valid()); + early_data = flag; +} + +/*------------------------------------------------------------------------- + -------------------------------------------------------------------------*/ + +inline bool +HTTPHdr::is_early_data() const +{ + ink_assert(valid()); + return early_data; +} + /*------------------------------------------------------------------------- -------------------------------------------------------------------------*/ diff --git a/proxy/hdrs/HdrHeap.cc b/proxy/hdrs/HdrHeap.cc index c10d6e56c92..dfa387abddc 100644 --- a/proxy/hdrs/HdrHeap.cc +++ b/proxy/hdrs/HdrHeap.cc @@ -400,6 +400,9 @@ HdrHeap::evacuate_from_str_heaps(HdrStrHeap *new_heap) while (data < h->m_free_start) { HdrHeapObjImpl *obj = reinterpret_cast(data); + // Object length cannot be 0 by design, otherwise something is wrong + infinite loop here! + ink_release_assert(0 != obj->m_length); + switch (obj->m_type) { case HDR_HEAP_OBJ_URL: ((URLImpl *)obj)->move_strings(new_heap); @@ -440,6 +443,9 @@ HdrHeap::required_space_for_evacuation() while (data < h->m_free_start) { HdrHeapObjImpl *obj = reinterpret_cast(data); + // Object length cannot be 0 by design, otherwise something is wrong + infinite loop here! + ink_release_assert(0 != obj->m_length); + switch (obj->m_type) { case HDR_HEAP_OBJ_URL: ret += ((URLImpl *)obj)->strings_length(); @@ -514,6 +520,9 @@ HdrHeap::sanity_check_strs() while (data < h->m_free_start) { HdrHeapObjImpl *obj = reinterpret_cast(data); + // Object length cannot be 0 by design, otherwise something is wrong + infinite loop here! + ink_release_assert(0 != obj->m_length); + switch (obj->m_type) { case HDR_HEAP_OBJ_URL: ((URLImpl *)obj)->check_strings(heaps, num_heaps); @@ -937,6 +946,9 @@ HdrHeap::unmarshal(int buf_length, int obj_type, HdrHeapObjImpl **found_obj, Ref HdrHeapObjImpl *obj = reinterpret_cast(obj_data); ink_assert(obj_is_aligned(obj)); + // Object length cannot be 0 by design, otherwise something is wrong + infinite loop here! + ink_release_assert(0 != obj->m_length); + if (obj->m_type == static_cast(obj_type) && *found_obj == nullptr) { *found_obj = obj; } @@ -1124,6 +1136,24 @@ HdrHeap::dump_heap(int len) fprintf(stderr, "\n-------------- End header heap dump -----------\n"); } +uint64_t +HdrHeap::total_used_size() const +{ + uint64_t size = 0; + const HdrHeap *h = this; + + while (h) { + size += (h->m_free_start - h->m_data_start); + h = h->m_next; + } + + return size; +} + +// +// HdrStrHeap +// + void HdrStrHeap::free() { diff --git a/proxy/hdrs/HdrHeap.h b/proxy/hdrs/HdrHeap.h index e7c075c3c4c..1d83c85cae1 100644 --- a/proxy/hdrs/HdrHeap.h +++ b/proxy/hdrs/HdrHeap.h @@ -287,6 +287,8 @@ class HdrHeap size_t required_space_for_evacuation(); bool attach_str_heap(char const *h_start, int h_len, RefCountObj *h_ref_obj, int *index); + uint64_t total_used_size() const; + /** Struct to prevent garbage collection on heaps. This bumps the reference count to the heap containing the pointer while the instance of this class exists. When it goes out of scope diff --git a/proxy/hdrs/HdrTest.cc b/proxy/hdrs/HdrTest.cc deleted file mode 100644 index 9d8e30fe7dc..00000000000 --- a/proxy/hdrs/HdrTest.cc +++ /dev/null @@ -1,2109 +0,0 @@ -/** @file - - A brief file description - - @section license License - - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - */ - -/**************************************************************************** - - HdrTest.cc - - Description: - Unit test code for sanity checking the header system is functioning - properly - - - ****************************************************************************/ - -#include "tscore/ink_platform.h" -#include "tscore/ink_memory.h" -#include "tscore/ink_time.h" - -#include "tscore/Arena.h" -#include "HTTP.h" -#include "MIME.h" -#include "tscore/Regex.h" -#include "URL.h" -#include "HttpCompat.h" - -#include "HdrTest.h" - -////////////////////// -// Main Test Driver // -////////////////////// - -int -HdrTest::go(RegressionTest *t, int /* atype ATS_UNUSED */) -{ - HdrTest::rtest = t; - int status = 1; - - hdrtoken_init(); - url_init(); - mime_init(); - http_init(); - - status = status & test_http_hdr_print_and_copy(); - status = status & test_comma_vals(); - status = status & test_parse_comma_list(); - status = status & test_set_comma_vals(); - status = status & test_delete_comma_vals(); - status = status & test_extend_comma_vals(); - status = status & test_insert_comma_vals(); - status = status & test_accept_language_match(); - status = status & test_accept_charset_match(); - status = status & test_parse_date(); - status = status & test_format_date(); - status = status & test_url(); - status = status & test_arena(); - status = status & test_regex(); - status = status & test_http_mutation(); - status = status & test_mime(); - status = status & test_http(); - - return (status ? REGRESSION_TEST_PASSED : REGRESSION_TEST_FAILED); -} - -//////////////////////////////////////////////////////////// -// Individual Tests --- return 1 on success, 0 on failure // -//////////////////////////////////////////////////////////// - -int -HdrTest::test_parse_date() -{ - static struct { - const char *fast; - const char *slow; - } dates[] = { - {"Sun, 06 Nov 1994 08:49:37 GMT", "Sunday, 06-Nov-1994 08:49:37 GMT"}, - {"Mon, 07 Nov 1994 08:49:37 GMT", "Monday, 07-Nov-1994 08:49:37 GMT"}, - {"Tue, 08 Nov 1994 08:49:37 GMT", "Tuesday, 08-Nov-1994 08:49:37 GMT"}, - {"Wed, 09 Nov 1994 08:49:37 GMT", "Wednesday, 09-Nov-1994 08:49:37 GMT"}, - {"Thu, 10 Nov 1994 08:49:37 GMT", "Thursday, 10-Nov-1994 08:49:37 GMT"}, - {"Fri, 11 Nov 1994 08:49:37 GMT", "Friday, 11-Nov-1994 08:49:37 GMT"}, - {"Sat, 11 Nov 1994 08:49:37 GMT", "Saturday, 11-Nov-1994 08:49:37 GMT"}, - {"Sun, 03 Jan 1999 08:49:37 GMT", "Sunday, 03-Jan-1999 08:49:37 GMT"}, - {"Sun, 07 Feb 1999 08:49:37 GMT", "Sunday, 07-Feb-1999 08:49:37 GMT"}, - {"Sun, 07 Mar 1999 08:49:37 GMT", "Sunday, 07-Mar-1999 08:49:37 GMT"}, - {"Sun, 04 Apr 1999 08:49:37 GMT", "Sunday, 04-Apr-1999 08:49:37 GMT"}, - {"Sun, 02 May 1999 08:49:37 GMT", "Sunday, 02-May-1999 08:49:37 GMT"}, - {"Sun, 06 Jun 1999 08:49:37 GMT", "Sunday, 06-Jun-1999 08:49:37 GMT"}, - {"Sun, 04 Jul 1999 08:49:37 GMT", "Sunday, 04-Jul-1999 08:49:37 GMT"}, - {"Sun, 01 Aug 1999 08:49:37 GMT", "Sunday, 01-Aug-1999 08:49:37 GMT"}, - {"Sun, 05 Sep 1999 08:49:37 GMT", "Sunday, 05-Sep-1999 08:49:37 GMT"}, - {"Sun, 03 Oct 1999 08:49:37 GMT", "Sunday, 03-Oct-1999 08:49:37 GMT"}, - {"Sun, 07 Nov 1999 08:49:37 GMT", "Sunday, 07-Nov-1999 08:49:37 GMT"}, - {"Sun, 05 Dec 1999 08:49:37 GMT", "Sunday, 05-Dec-1999 08:49:37 GMT"}, - {nullptr, nullptr}, - }; - - int i; - int failures = 0; - time_t fast_t, slow_t; - - bri_box("test_parse_date"); - - for (i = 0; dates[i].fast; i++) { - fast_t = mime_parse_date(dates[i].fast, dates[i].fast + static_cast(strlen(dates[i].fast))); - slow_t = mime_parse_date(dates[i].slow, dates[i].slow + static_cast(strlen(dates[i].slow))); - // compare with strptime here! - if (fast_t != slow_t) { - printf("FAILED: date %lu (%s) != %lu (%s)\n", static_cast(fast_t), dates[i].fast, - static_cast(slow_t), dates[i].slow); - ++failures; - } - } - - return (failures_to_status("test_parse_date", failures)); -} - -/*------------------------------------------------------------------------- - -------------------------------------------------------------------------*/ - -int -HdrTest::test_format_date() -{ - static const char *dates[] = { - "Sun, 06 Nov 1994 08:49:37 GMT", - "Sun, 03 Jan 1999 08:49:37 GMT", - "Sun, 05 Dec 1999 08:49:37 GMT", - "Tue, 25 Apr 2000 20:29:53 GMT", - nullptr, - }; - - bri_box("test_format_date"); - - // (1) Test a few hand-created dates - - int i; - time_t t, t2, t3; - char buffer[128], buffer2[128]; - static const char *envstr = "TZ=GMT0"; - int failures = 0; - - // shift into GMT timezone for cftime conversions - putenv(const_cast(envstr)); - tzset(); - - for (i = 0; dates[i]; i++) { - t = mime_parse_date(dates[i], dates[i] + static_cast(strlen(dates[i]))); - - cftime_replacement(buffer, sizeof(buffer), "%a, %d %b %Y %T %Z", &t); - if (memcmp(dates[i], buffer, 29) != 0) { - printf("FAILED: original date doesn't match cftime date\n"); - printf(" input date: %s\n", dates[i]); - printf(" cftime date: %s\n", buffer); - ++failures; - } - - mime_format_date(buffer, t); - if (memcmp(dates[i], buffer, 29) != 0) { - printf("FAILED: original date doesn't match mime_format_date date\n"); - printf(" input date: %s\n", dates[i]); - printf(" cftime date: %s\n", buffer); - ++failures; - } - } - - // (2) test a few times per day from 1/1/1970 to past 2010 - - // coverity[dont_call] - for (t = 0; t < 40 * 366 * (24 * 60 * 60); t += static_cast(drand48() * (24 * 60 * 60))) { - cftime_replacement(buffer, sizeof(buffer), "%a, %d %b %Y %T %Z", &t); - t2 = mime_parse_date(buffer, buffer + static_cast(strlen(buffer))); - if (t2 != t) { - printf("FAILED: parsed time_t doesn't match original time_t\n"); - printf(" input time_t: %d (%s)\n", static_cast(t), buffer); - printf(" parsed time_t: %d\n", static_cast(t2)); - ++failures; - } - mime_format_date(buffer2, t); - if (memcmp(buffer, buffer2, 29) != 0) { - printf("FAILED: formatted date doesn't match original date\n"); - printf(" original date: %s\n", buffer); - printf(" formatted date: %s\n", buffer2); - ++failures; - } - t3 = mime_parse_date(buffer2, buffer2 + static_cast(strlen(buffer2))); - if (t != t3) { - printf("FAILED: parsed time_t doesn't match original time_t\n"); - printf(" input time_t: %d (%s)\n", static_cast(t), buffer2); - printf(" parsed time_t: %d\n", static_cast(t3)); - ++failures; - } - - if (failures > 20) { - // Already too many failures, don't need to continue - break; - } - } - - return (failures_to_status("test_format_date", failures)); -} - -/*------------------------------------------------------------------------- - -------------------------------------------------------------------------*/ - -int -HdrTest::test_url() -{ - static const char *strs[] = { - "http://some.place/path;params?query#fragment", - - // Start with an easy one... - "http://trafficserver.apache.org/index.html", - - // "cheese://bogosity", This fails, but it's not clear it should work... - - "some.place", "some.place/", "http://some.place", "http://some.place/", "http://some.place/path", - "http://some.place/path;params", "http://some.place/path;params?query", "http://some.place/path;params?query#fragment", - "http://some.place/path?query#fragment", "http://some.place/path#fragment", - - "some.place:80", "some.place:80/", "http://some.place:80", "http://some.place:80/", - - "foo@some.place:80", "foo@some.place:80/", "http://foo@some.place:80", "http://foo@some.place:80/", - - "foo:bar@some.place:80", "foo:bar@some.place:80/", "http://foo:bar@some.place:80", "http://foo:bar@some.place:80/", - - // Some address stuff - "http://172.16.28.101", "http://172.16.28.101:8080", "http://[::]", "http://[::1]", "http://[fc01:172:16:28::101]", - "http://[fc01:172:16:28::101]:80", "http://[fc01:172:16:28:BAAD:BEEF:DEAD:101]", - "http://[fc01:172:16:28:BAAD:BEEF:DEAD:101]:8080", "http://172.16.28.101/some/path", "http://172.16.28.101:8080/some/path", - "http://[::1]/some/path", "http://[fc01:172:16:28::101]/some/path", "http://[fc01:172:16:28::101]:80/some/path", - "http://[fc01:172:16:28:BAAD:BEEF:DEAD:101]/some/path", "http://[fc01:172:16:28:BAAD:BEEF:DEAD:101]:8080/some/path", - "http://172.16.28.101/", "http://[fc01:172:16:28:BAAD:BEEF:DEAD:101]:8080/", - - "foo:bar@some.place", "foo:bar@some.place/", "http://foo:bar@some.place", "http://foo:bar@some.place/", - "http://foo:bar@[::1]:8080/", "http://foo@[::1]", - - "mms://sm02.tsqa.example.com/0102rally.asf", "pnm://foo:bar@some.place:80/path;params?query#fragment", - "rtsp://foo:bar@some.place:80/path;params?query#fragment", "rtspu://foo:bar@some.place:80/path;params?query#fragment", - "/finance/external/cbsm/*http://cbs.marketwatch.com/archive/19990713/news/current/net.htx?source=blq/yhoo&dist=yhoo", - "http://a.b.com/xx.jpg?newpath=http://bob.dave.com"}; - - static const char *bad[] = { - "http://[1:2:3:4:5:6:7:8:9]", - "http://1:2:3:4:5:6:7:8:A:B", - "http://bob.com[::1]", - "http://[::1].com", - "http://foo:bar:baz@bob.com/", - "http://foo:bar:baz@[::1]:8080/", - "http://]", - "http://:", - "http:/", - }; - - int err, failed; - URL url; - const char *start; - const char *end; - int old_length, new_length; - - bri_box("test_url"); - - failed = 0; - for (unsigned i = 0; i < countof(strs); i++) { - old_length = static_cast(strlen(strs[i])); - start = strs[i]; - end = start + old_length; - - url.create(nullptr); - err = url.parse(&start, end); - if (err < 0) { - failed = 1; - break; - } - - char print_buf[1024]; - new_length = 0; - int offset = 0; - url.print(print_buf, 1024, &new_length, &offset); - print_buf[new_length] = '\0'; - - const char *fail_text = nullptr; - - if (old_length == new_length) { - if (memcmp(print_buf, strs[i], new_length) != 0) { - fail_text = "URLS DIFFER"; - } - } else if (old_length == new_length - 1) { - // Check to see if the difference is the trailing - // slash we add - if (memcmp(print_buf, strs[i], old_length) != 0 || print_buf[new_length - 1] != '/' || (strs[i])[old_length - 1] == '/') { - fail_text = "TRAILING SLASH"; - } - } else { - fail_text = "LENGTHS DIFFER"; - } - - if (fail_text) { - failed = 1; - printf("%16s: OLD: (%4d) %s\n", fail_text, old_length, strs[i]); - printf("%16s: NEW: (%4d) %s\n", "", new_length, print_buf); - obj_describe(url.m_url_impl, true); - } else { - printf("%16s: '%s'\n", "PARSE SUCCESS", strs[i]); - } - - url.destroy(); - } - - for (unsigned i = 0; i < countof(bad); ++i) { - const char *x = bad[i]; - - url.create(nullptr); - err = url.parse(x, strlen(x)); - url.destroy(); - if (err == PARSE_RESULT_DONE) { - failed = 1; - printf("Successfully parsed invalid url '%s'", x); - break; - } - } - - return (failures_to_status("test_url", failed)); -} - -/*------------------------------------------------------------------------- - -------------------------------------------------------------------------*/ - -int -HdrTest::test_mime() -{ - // This can not be a static string (any more) since we unfold the headers - // in place. - char mime[] = { - // "Date: Tuesday, 08-Dec-98 20:32:17 GMT\r\n" - "Date: 6 Nov 1994 08:49:37 GMT\r\n" - "Max-Forwards: 65535\r\n" - "Cache-Control: private\r\n" - "accept: foo\r\n" - "accept: bar\n" - ": (null) field name\r\n" - "aCCept: \n" - "ACCEPT\r\n" - "foo: bar\r\n" - "foo: argh\r\n" - "foo: three, four\r\n" - "word word: word \r\n" - "accept: \"fazzle, dazzle\"\r\n" - "accept: 1, 2, 3, 4, 5, 6, 7, 8\r\n" - "continuation: part1\r\n" - " part2\r\n" - "scooby: doo\r\n" - " scooby: doo\r\n" - "bar: foo\r\n" - "\r\n", - }; - - int err; - MIMEHdr hdr; - MIMEParser parser; - const char *start; - const char *end; - - bri_box("test_mime"); - - printf(" <<< MUST BE HAND-VERIFIED FOR FULL-BENEFIT>>>\n\n"); - - start = mime; - end = start + strlen(start); - - mime_parser_init(&parser); - - bool must_copy_strs = false; - - hdr.create(nullptr); - err = hdr.parse(&parser, &start, end, must_copy_strs, false); - - if (err < 0) { - return (failures_to_status("test_mime", 1)); - } - - // Test the (new) continuation line folding to be correct. This should replace the - // \r\n with two spaces (so a total of three between "part1" and "part2"). - int length = 0; - const char *continuation = hdr.value_get("continuation", 12, &length); - - if ((13 != length)) { - printf("FAILED: continue header folded line was too short\n"); - return (failures_to_status("test_mime", 1)); - } - - if (strncmp(continuation + 5, " ", 3)) { - printf("FAILED: continue header unfolding did not produce correct WS's\n"); - return (failures_to_status("test_mime", 1)); - } - - if (strncmp(continuation, "part1 part2", 13)) { - printf("FAILED: continue header unfolding was not correct\n"); - return (failures_to_status("test_mime", 1)); - } - - hdr.field_delete("not_there", 9); - hdr.field_delete("accept", 6); - hdr.field_delete("scooby", 6); - hdr.field_delete("scooby", 6); - hdr.field_delete("bar", 3); - hdr.field_delete("continuation", 12); - - int count = hdr.fields_count(); - printf("hdr.fields_count() = %d\n", count); - - int i_max_forwards = hdr.value_get_int("Max-Forwards", 12); - int u_max_forwards = hdr.value_get_uint("Max-Forwards", 12); - printf("i_max_forwards = %d u_max_forwards = %d\n", i_max_forwards, u_max_forwards); - - hdr.set_age(9999); - - length = hdr.length_get(); - printf("hdr.length_get() = %d\n", length); - - time_t t0, t1, t2; - - t0 = hdr.get_date(); - if (t0 == 0) { - printf("FAILED: Initial date is zero but shouldn't be\n"); - return (failures_to_status("test_mime", 1)); - } - - t1 = time(nullptr); - hdr.set_date(t1); - t2 = hdr.get_date(); - if (t1 != t2) { - printf("FAILED: set_date(%" PRId64 ") ... get_date = %" PRId64 "\n\n", static_cast(t1), static_cast(t2)); - return (failures_to_status("test_mime", 1)); - } - - hdr.value_append("Cache-Control", 13, "no-cache", 8, true); - - MIMEField *cc_field; - StrList slist; - - cc_field = hdr.field_find("Cache-Control", 13); - - if (cc_field == nullptr) { - printf("FAILED: missing Cache-Control header\n\n"); - return (failures_to_status("test_mime", 1)); - } - - // TODO: Do we need to check the "count" returned? - cc_field->value_get_comma_list(&slist); // FIX: correct usage? - - if (cc_field->value_get_index("Private", 7) < 0) { - printf("Failed: value_get_index of Cache-Control did not find private"); - return (failures_to_status("test_mime", 1)); - } - if (cc_field->value_get_index("Bogus", 5) >= 0) { - printf("Failed: value_get_index of Cache-Control incorrectly found bogus"); - return (failures_to_status("test_mime", 1)); - } - if (hdr.value_get_index("foo", 3, "three", 5) < 0) { - printf("Failed: value_get_index of foo did not find three"); - return (failures_to_status("test_mime", 1)); - } - if (hdr.value_get_index("foo", 3, "bar", 3) < 0) { - printf("Failed: value_get_index of foo did not find bar"); - return (failures_to_status("test_mime", 1)); - } - if (hdr.value_get_index("foo", 3, "Bogus", 5) >= 0) { - printf("Failed: value_get_index of foo incorrectly found bogus"); - return (failures_to_status("test_mime", 1)); - } - - mime_parser_clear(&parser); - - hdr.print(nullptr, 0, nullptr, nullptr); - printf("\n"); - - obj_describe((HdrHeapObjImpl *)(hdr.m_mime), true); - - hdr.fields_clear(); - - hdr.destroy(); - - return (failures_to_status("test_mime", 0)); -} - -/*------------------------------------------------------------------------- - -------------------------------------------------------------------------*/ - -int -HdrTest::test_http_aux(const char *request, const char *response) -{ - int err; - HTTPHdr req_hdr, rsp_hdr; - HTTPParser parser; - const char *start; - const char *end; - - int status = 1; - - printf(" <<< MUST BE HAND-VERIFIED FOR FULL BENEFIT >>>\n\n"); - - /*** (1) parse the request string into req_hdr ***/ - - start = request; - end = start + strlen(start); // 1 character past end of string - - http_parser_init(&parser); - - req_hdr.create(HTTP_TYPE_REQUEST); - rsp_hdr.create(HTTP_TYPE_RESPONSE); - - printf("======== parsing\n\n"); - while (true) { - err = req_hdr.parse_req(&parser, &start, end, true); - if (err != PARSE_RESULT_CONT) { - break; - } - } - if (err == PARSE_RESULT_ERROR) { - req_hdr.destroy(); - rsp_hdr.destroy(); - return (failures_to_status("test_http_aux", (status == 0))); - } - - /*** useless copy to exercise copy function ***/ - - HTTPHdr new_hdr; - new_hdr.create(HTTP_TYPE_REQUEST); - new_hdr.copy(&req_hdr); - new_hdr.destroy(); - - /*** (2) print out the request ***/ - - printf("======== real request (length=%d)\n\n", static_cast(strlen(request))); - printf("%s\n", request); - - printf("\n["); - req_hdr.print(nullptr, 0, nullptr, nullptr); - printf("]\n\n"); - - obj_describe(req_hdr.m_http, true); - - // req_hdr.destroy (); - // ink_release_assert(!"req_hdr.destroy() not defined"); - - /*** (3) parse the response string into rsp_hdr ***/ - - start = response; - end = start + strlen(start); - - http_parser_clear(&parser); - http_parser_init(&parser); - - while (true) { - err = rsp_hdr.parse_resp(&parser, &start, end, true); - if (err != PARSE_RESULT_CONT) { - break; - } - } - if (err == PARSE_RESULT_ERROR) { - req_hdr.destroy(); - rsp_hdr.destroy(); - return (failures_to_status("test_http_aux", (status == 0))); - } - - http_parser_clear(&parser); - - /*** (4) print out the response ***/ - - printf("\n======== real response (length=%d)\n\n", static_cast(strlen(response))); - printf("%s\n", response); - - printf("\n["); - rsp_hdr.print(nullptr, 0, nullptr, nullptr); - printf("]\n\n"); - - obj_describe(rsp_hdr.m_http, true); - -#define NNN 1000 - { - char buf[NNN]; - int bufindex, last_bufindex; - int tmp; - int i; - - bufindex = 0; - - do { - last_bufindex = bufindex; - tmp = bufindex; - buf[0] = '#'; // make it obvious if hdr.print doesn't print anything - err = rsp_hdr.print(buf, NNN, &bufindex, &tmp); - - // printf("test_header: tmp = %d err = %d bufindex = %d\n", tmp, err, bufindex); - putchar('{'); - for (i = 0; i < bufindex - last_bufindex; i++) { - if (!iscntrl(buf[i])) { - putchar(buf[i]); - } else { - printf("\\%o", buf[i]); - } - } - putchar('}'); - } while (!err); - } - - // rsp_hdr.print (NULL, 0, NULL, NULL); - - req_hdr.destroy(); - rsp_hdr.destroy(); - - return (failures_to_status("test_http_aux", (status == 0))); -} - -int -HdrTest::test_http_req_parse_error(const char *request, const char *response) -{ - int err; - HTTPHdr req_hdr, rsp_hdr; - HTTPParser parser; - const char *start; - const char *end; - - int status = 1; - - /*** (1) parse the request string into req_hdr ***/ - - start = request; - end = start + strlen(start); // 1 character past end of string - - http_parser_init(&parser); - - req_hdr.create(HTTP_TYPE_REQUEST); - rsp_hdr.create(HTTP_TYPE_RESPONSE); - - printf("======== test_http_req_parse_error parsing\n\n"); - err = req_hdr.parse_req(&parser, &start, end, true, true, 1); - if (err != PARSE_RESULT_ERROR) { - status = 0; - } - - http_parser_clear(&parser); - - /*** (4) print out the response ***/ - - printf("\n======== real response (length=%d)\n\n", static_cast(strlen(response))); - printf("%s\n", response); - - obj_describe(rsp_hdr.m_http, true); - - req_hdr.destroy(); - rsp_hdr.destroy(); - - return (failures_to_status("test_http_req_parse_error", (status == 0))); -} - -/*------------------------------------------------------------------------- - -------------------------------------------------------------------------*/ - -int -HdrTest::test_http_hdr_print_and_copy() -{ - static struct { - const char *req; - const char *req_tgt; - const char *rsp; - const char *rsp_tgt; - } tests[] = { - {"GET http://foo.com/bar.txt HTTP/1.0\r\n" - "Accept-Language: fjdfjdslkf dsjkfdj flkdsfjlk sjfdlk ajfdlksa\r\n" - "\r\n", - "GET http://foo.com/bar.txt HTTP/1.0\r\n" - "Accept-Language: fjdfjdslkf dsjkfdj flkdsfjlk sjfdlk ajfdlksa\r\n" - "\r\n", - "HTTP/1.0 200 OK\r\n" - "\r\n", - "HTTP/1.0 200 OK\r\n" - "\r\n"}, - {"GET http://foo.com/bar.txt HTTP/1.0\r\n" - "Accept-Language: fjdfjdslkf dsjkfdj flkdsfjlk sjfdlk ajfdlksa fjfj dslkfjdslk fjsdafkl dsajfkldsa jfkldsafj " - "klsafjs lkafjdsalk fsdjakfl sdjaflkdsaj flksdjflsd ;ffd salfdjs lf;sdaf ;dsaf jdsal;fdjsaflkjsda \r\n" - "\r\n", - "GET http://foo.com/bar.txt HTTP/1.0\r\n" - "Accept-Language: fjdfjdslkf dsjkfdj flkdsfjlk sjfdlk ajfdlksa fjfj dslkfjdslk fjsdafkl dsajfkldsa jfkldsafj " - "klsafjs lkafjdsalk fsdjakfl sdjaflkdsaj flksdjflsd ;ffd salfdjs lf;sdaf ;dsaf jdsal;fdjsaflkjsda \r\n" - "\r\n", - "HTTP/1.0 200 OK\r\n" - "\r\n", - "HTTP/1.0 200 OK\r\n" - "\r\n"}, - {"GET http://foo.com/bar.txt HTTP/1.0\r\n" - "Accept-Language: fjdfjdslkf dsjkfdj flkdsfjlk sjfdlk ajfdlksa fjfj dslkfjdslk fjsdafkl dsajfkldsa jfkldsafj " - "klsafjs lkafjdsalk fsdjakfl sdjaflkdsaj flksdjflsd ;ffd salfdjs lf;sdaf ;dsaf jdsal;fdjsaflkjsda kfl; fsdajfl; " - "sdjafl;dsajlsjfl;sdafjsdal;fjds al;fdjslaf ;slajdk;f\r\n" - "\r\n", - "GET http://foo.com/bar.txt HTTP/1.0\r\n" - "Accept-Language: fjdfjdslkf dsjkfdj flkdsfjlk sjfdlk ajfdlksa fjfj dslkfjdslk fjsdafkl dsajfkldsa jfkldsafj " - "klsafjs lkafjdsalk fsdjakfl sdjaflkdsaj flksdjflsd ;ffd salfdjs lf;sdaf ;dsaf jdsal;fdjsaflkjsda kfl; fsdajfl; " - "sdjafl;dsajlsjfl;sdafjsdal;fjds al;fdjslaf ;slajdk;f\r\n" - "\r\n", - "HTTP/1.0 200 OK\r\n" - "\r\n", - "HTTP/1.0 200 OK\r\n" - "\r\n"}, - {"GET http://people.netscape.com/jwz/hacks-1.gif HTTP/1.0\r\n" - "If-Modified-Since: Wednesday, 26-Feb-97 06:58:17 GMT; length=842\r\n" - "Referer: chocolate fribble\r\n", // missing final CRLF - "GET http://people.netscape.com/jwz/hacks-1.gif HTTP/1.0\r\n" - "If-Modified-Since: Wednesday, 26-Feb-97 06:58:17 GMT; length=842\r\n" - "Referer: chocolate fribble\r\n" - "\r\n", - "HTTP/1.0 200 OK\r\n" - "MIME-Version: 1.0\r\n" - "Server: WebSTAR/2.1 ID/30013\r\n" - "Content-Type: text/html\r\n" - "Content-Length: 939\r\n" - "Last-Modified: Thursday, 01-Jan-04 05:00:00 GMT\r\n", // missing final CRLF - "HTTP/1.0 200 OK\r\n" - "MIME-Version: 1.0\r\n" - "Server: WebSTAR/2.1 ID/30013\r\n" - "Content-Type: text/html\r\n" - "Content-Length: 939\r\n" - "Last-Modified: Thursday, 01-Jan-04 05:00:00 GMT\r\n" - "\r\n"}, - {"GET http://people.netscape.com/jwz/hacks-1.gif HTTP/1.0\r\n" - "If-Modified-Since: Wednesday, 26-Feb-97 06:58:17 GMT; length=842\r\n" - "Referer: \r\n", // missing final CRLF - "GET http://people.netscape.com/jwz/hacks-1.gif HTTP/1.0\r\n" - "If-Modified-Since: Wednesday, 26-Feb-97 06:58:17 GMT; length=842\r\n" - "Referer: \r\n" - "\r\n", - "HTTP/1.0 200 OK\r\n" - "MIME-Version: 1.0\r\n" - "Server: WebSTAR/2.1 ID/30013\r\n" - "Content-Type: text/html\r\n" - "Content-Length: 939\r\n" - "Last-Modified: Thursday, 01-Jan-04 05:00:00 GMT\r\n" - "\r\n", - "HTTP/1.0 200 OK\r\n" - "MIME-Version: 1.0\r\n" - "Server: WebSTAR/2.1 ID/30013\r\n" - "Content-Type: text/html\r\n" - "Content-Length: 939\r\n" - "Last-Modified: Thursday, 01-Jan-04 05:00:00 GMT\r\n" - "\r\n"}, - {"GET http://www.news.com:80/ HTTP/1.0\r\n" - "Proxy-Connection: Keep-Alive\r\n" - "User-Agent: Mozilla/4.04 [en] (X11; I; Linux 2.0.33 i586)\r\n" - "Pragma: no-cache\r\n" - "Host: www.news.com\r\n" - "Accept: image/gif, image/x-xbitmap, image/jpeg, image/pjpeg, image/png, */*\r\n" - "Accept-Language: en\r\n" - "Accept-Charset: iso-8859-1, *, utf-8\r\n" - "Client-ip: D1012148\r\n" - "Foo: abcdefghijklmnopqrtu\r\n" - "\r\n", - "GET http://www.news.com:80/ HTTP/1.0\r\n" - "Proxy-Connection: Keep-Alive\r\n" - "User-Agent: Mozilla/4.04 [en] (X11; I; Linux 2.0.33 i586)\r\n" - "Pragma: no-cache\r\n" - "Host: www.news.com\r\n" - "Accept: image/gif, image/x-xbitmap, image/jpeg, image/pjpeg, image/png, */*\r\n" - "Accept-Language: en\r\n" - "Accept-Charset: iso-8859-1, *, utf-8\r\n" - "Client-ip: D1012148\r\n" - "Foo: abcdefghijklmnopqrtu\r\n" - "\r\n", - "HTTP/1.0 200 OK\r\n" - "Content-Length: 16428\r\n" - "Content-Type: text/html\r\n" - "\r\n", - "HTTP/1.0 200 OK\r\n" - "Content-Length: 16428\r\n" - "Content-Type: text/html\r\n" - "\r\n"}, - {"GET http://people.netscape.com/jwz/hacks-1.gif HTTP/1.0\r\n" - "If-Modified-Since: Wednesday, 26-Feb-97 06:58:17 GMT; length=842\r\n" - "Referer: http://people.netscape.com/jwz/index.html\r\n" - "Proxy-Connection: Keep-Alive\r\n" - "User-Agent: Mozilla/3.01 (X11; I; Linux 2.0.28 i586)\r\n" - "Pragma: no-cache\r\n" - "Host: people.netscape.com\r\n" - "Accept: image/gif, image/x-xbitmap, image/jpeg, image/pjpeg, */*\r\n" - "\r\n", - "GET http://people.netscape.com/jwz/hacks-1.gif HTTP/1.0\r\n" - "If-Modified-Since: Wednesday, 26-Feb-97 06:58:17 GMT; length=842\r\n" - "Referer: http://people.netscape.com/jwz/index.html\r\n" - "Proxy-Connection: Keep-Alive\r\n" - "User-Agent: Mozilla/3.01 (X11; I; Linux 2.0.28 i586)\r\n" - "Pragma: no-cache\r\n" - "Host: people.netscape.com\r\n" - "Accept: image/gif, image/x-xbitmap, image/jpeg, image/pjpeg, */*\r\n" - "\r\n", - "HTTP/1.0 200 OK\r\n" - "Content-Length: 16428\r\n" - "Content-Type: text/html\r\n" - "\r\n", - "HTTP/1.0 200 OK\r\n" - "Content-Length: 16428\r\n" - "Content-Type: text/html\r\n" - "\r\n"}, - }; - - int ntests = sizeof(tests) / sizeof(tests[0]); - int i, failures; - - failures = 0; - - bri_box("test_http_hdr_print_and_copy"); - - for (i = 0; i < ntests; i++) { - int status = test_http_hdr_print_and_copy_aux(i + 1, tests[i].req, tests[i].req_tgt, tests[i].rsp, tests[i].rsp_tgt); - if (status == 0) { - ++failures; - } - - // Test for expected failures - // parse with a '\0' in the header. Should fail - status = test_http_hdr_null_char(i + 1, tests[i].req, tests[i].req_tgt); - if (status == 0) { - ++failures; - } - - // Parse with a CTL character in the method name. Should fail - status = test_http_hdr_ctl_char(i + 1, tests[i].req, tests[i].req_tgt); - if (status == 0) { - ++failures; - } - } - - return (failures_to_status("test_http_hdr_print_and_copy", failures)); -} - -/*------------------------------------------------------------------------- - -------------------------------------------------------------------------*/ -static const char * -comp_http_hdr(HTTPHdr *h1, HTTPHdr *h2) -{ - int h1_len, h2_len; - int p_index, p_dumpoffset, rval; - char *h1_pbuf, *h2_pbuf; - - h1_len = h1->length_get(); - h2_len = h2->length_get(); - - if (h1_len != h2_len) { - return "length mismatch"; - } - - h1_pbuf = static_cast(ats_malloc(h1_len + 1)); - h2_pbuf = static_cast(ats_malloc(h2_len + 1)); - - p_index = p_dumpoffset = 0; - rval = h1->print(h1_pbuf, h1_len, &p_index, &p_dumpoffset); - if (rval != 1) { - ats_free(h1_pbuf); - ats_free(h2_pbuf); - return "hdr print failed"; - } - - p_index = p_dumpoffset = 0; - rval = h2->print(h2_pbuf, h2_len, &p_index, &p_dumpoffset); - if (rval != 1) { - ats_free(h1_pbuf); - ats_free(h2_pbuf); - return "hdr print failed"; - } - - rval = memcmp(h1_pbuf, h2_pbuf, h1_len); - ats_free(h1_pbuf); - ats_free(h2_pbuf); - - if (rval != 0) { - return "compare failed"; - } else { - return nullptr; - } -} - -/*------------------------------------------------------------------------- - -------------------------------------------------------------------------*/ - -int -HdrTest::test_http_hdr_copy_over_aux(int testnum, const char *request, const char *response) -{ - int err; - HTTPHdr req_hdr; - HTTPHdr resp_hdr; - HTTPHdr copy1; - HTTPHdr copy2; - - HTTPParser parser; - const char *start; - const char *end; - const char *comp_str = nullptr; - - /*** (1) parse the request string into hdr ***/ - - req_hdr.create(HTTP_TYPE_REQUEST); - - start = request; - end = start + strlen(start); // 1 character past end of string - - http_parser_init(&parser); - - while (true) { - err = req_hdr.parse_req(&parser, &start, end, true); - if (err != PARSE_RESULT_CONT) { - break; - } - } - - if (err == PARSE_RESULT_ERROR) { - printf("FAILED: (test #%d) parse error parsing request hdr\n", testnum); - return (0); - } - http_parser_clear(&parser); - - /*** (2) parse the response string into hdr ***/ - - resp_hdr.create(HTTP_TYPE_RESPONSE); - - start = response; - end = start + strlen(start); // 1 character past end of string - - http_parser_init(&parser); - - while (true) { - err = resp_hdr.parse_resp(&parser, &start, end, true); - if (err != PARSE_RESULT_CONT) { - break; - } - } - - if (err == PARSE_RESULT_ERROR) { - printf("FAILED: (test #%d) parse error parsing response hdr\n", testnum); - return (0); - } - - /*** (3) Basic copy testing ***/ - copy1.create(HTTP_TYPE_REQUEST); - copy1.copy(&req_hdr); - comp_str = comp_http_hdr(&req_hdr, ©1); - if (comp_str) { - goto done; - } - - copy2.create(HTTP_TYPE_RESPONSE); - copy2.copy(&resp_hdr); - comp_str = comp_http_hdr(&resp_hdr, ©2); - if (comp_str) { - goto done; - } - - // The APIs for copying headers uses memcpy() which can be unsafe for - // overlapping memory areas. It's unclear to me why these tests were - // created in the first place honestly, since nothing else does this. - - /*** (4) Gender bending copying ***/ - copy1.copy(&resp_hdr); - comp_str = comp_http_hdr(&resp_hdr, ©1); - if (comp_str) { - goto done; - } - - copy2.copy(&req_hdr); - comp_str = comp_http_hdr(&req_hdr, ©2); - if (comp_str) { - goto done; - } - -done: - req_hdr.destroy(); - resp_hdr.destroy(); - copy1.destroy(); - copy2.destroy(); - - if (comp_str) { - printf("FAILED: (test #%d) copy & compare: %s\n", testnum, comp_str); - printf("REQ:\n[%.*s]\n", static_cast(strlen(request)), request); - printf("RESP :\n[%.*s]\n", static_cast(strlen(response)), response); - return (0); - } else { - return (1); - } -} - -/*------------------------------------------------------------------------- - -------------------------------------------------------------------------*/ -int -HdrTest::test_http_hdr_null_char(int testnum, const char *request, const char * /*request_tgt*/) -{ - int err; - HTTPHdr hdr; - HTTPParser parser; - const char *start; - char cpy_buf[2048]; - const char *cpy_buf_ptr = cpy_buf; - - /*** (1) parse the request string into hdr ***/ - - hdr.create(HTTP_TYPE_REQUEST); - - start = request; - - if (strlen(start) > sizeof(cpy_buf)) { - printf("FAILED: (test #%d) Internal buffer too small for null char test\n", testnum); - return (0); - } - strcpy(cpy_buf, start); - - // Put a null character somewhere in the header - int length = strlen(start); - cpy_buf[length / 2] = '\0'; - http_parser_init(&parser); - - while (true) { - err = hdr.parse_req(&parser, &cpy_buf_ptr, cpy_buf_ptr + length, true); - if (err != PARSE_RESULT_CONT) { - break; - } - } - if (err != PARSE_RESULT_ERROR) { - printf("FAILED: (test #%d) no parse error parsing request with null char\n", testnum); - return (0); - } - return 1; -} - -/*------------------------------------------------------------------------- - -------------------------------------------------------------------------*/ -int -HdrTest::test_http_hdr_ctl_char(int testnum, const char *request, const char * /*request_tgt */) -{ - int err; - HTTPHdr hdr; - HTTPParser parser; - const char *start; - char cpy_buf[2048]; - const char *cpy_buf_ptr = cpy_buf; - - /*** (1) parse the request string into hdr ***/ - - hdr.create(HTTP_TYPE_REQUEST); - - start = request; - - if (strlen(start) > sizeof(cpy_buf)) { - printf("FAILED: (test #%d) Internal buffer too small for ctl char test\n", testnum); - return (0); - } - strcpy(cpy_buf, start); - - // Replace a character in the method - cpy_buf[1] = 16; - - http_parser_init(&parser); - - while (true) { - err = hdr.parse_req(&parser, &cpy_buf_ptr, cpy_buf_ptr + strlen(start), true); - if (err != PARSE_RESULT_CONT) { - break; - } - } - - if (err != PARSE_RESULT_ERROR) { - printf("FAILED: (test #%d) no parse error parsing method with ctl char\n", testnum); - return (0); - } - return 1; -} - -/*------------------------------------------------------------------------- - -------------------------------------------------------------------------*/ - -int -HdrTest::test_http_hdr_print_and_copy_aux(int testnum, const char *request, const char *request_tgt, const char *response, - const char *response_tgt) -{ - int err; - HTTPHdr hdr; - HTTPParser parser; - const char *start; - const char *end; - - char prt_buf[2048]; - int prt_bufsize = sizeof(prt_buf); - int prt_bufindex, prt_dumpoffset, prt_ret; - - char cpy_buf[2048]; - int cpy_bufsize = sizeof(cpy_buf); - int cpy_bufindex, cpy_dumpoffset, cpy_ret; - - char *marshal_buf = static_cast(ats_malloc(2048)); - int marshal_bufsize = sizeof(cpy_buf); - - /*** (1) parse the request string into hdr ***/ - - hdr.create(HTTP_TYPE_REQUEST); - - start = request; - end = start + strlen(start); // 1 character past end of string - - http_parser_init(&parser); - - while (true) { - err = hdr.parse_req(&parser, &start, end, true); - if (err != PARSE_RESULT_CONT) { - break; - } - } - - if (err == PARSE_RESULT_ERROR) { - printf("FAILED: (test #%d) parse error parsing request hdr\n", testnum); - ats_free(marshal_buf); - return (0); - } - - /*** (2) copy the request header ***/ - HTTPHdr new_hdr, marshal_hdr; - RefCountObj ref; - - // Pretend to pin this object with a refcount. - ref.refcount_inc(); - - int marshal_len = hdr.m_heap->marshal(marshal_buf, marshal_bufsize); - marshal_hdr.create(HTTP_TYPE_REQUEST); - marshal_hdr.unmarshal(marshal_buf, marshal_len, &ref); - new_hdr.create(HTTP_TYPE_REQUEST); - new_hdr.copy(&marshal_hdr); - - /*** (3) print the request header and copy to buffers ***/ - - prt_bufindex = prt_dumpoffset = 0; - prt_ret = hdr.print(prt_buf, prt_bufsize, &prt_bufindex, &prt_dumpoffset); - - cpy_bufindex = cpy_dumpoffset = 0; - cpy_ret = new_hdr.print(cpy_buf, cpy_bufsize, &cpy_bufindex, &cpy_dumpoffset); - - ats_free(marshal_buf); - - if ((prt_ret != 1) || (cpy_ret != 1)) { - printf("FAILED: (test #%d) couldn't print req hdr or copy --- prt_ret=%d, cpy_ret=%d\n", testnum, prt_ret, cpy_ret); - return (0); - } - - if ((static_cast(prt_bufindex) != strlen(request_tgt)) || (static_cast(cpy_bufindex) != strlen(request_tgt))) { - printf("FAILED: (test #%d) print req output size mismatch --- tgt=%d, prt_bufsize=%d, cpy_bufsize=%d\n", testnum, - static_cast(strlen(request_tgt)), prt_bufindex, cpy_bufindex); - - printf("ORIGINAL:\n[%.*s]\n", static_cast(strlen(request)), request); - printf("TARGET :\n[%.*s]\n", static_cast(strlen(request_tgt)), request_tgt); - printf("PRT_BUFF:\n[%.*s]\n", prt_bufindex, prt_buf); - printf("CPY_BUFF:\n[%.*s]\n", cpy_bufindex, cpy_buf); - return (0); - } - - if ((strncasecmp(request_tgt, prt_buf, strlen(request_tgt)) != 0) || - (strncasecmp(request_tgt, cpy_buf, strlen(request_tgt)) != 0)) { - printf("FAILED: (test #%d) print req output mismatch\n", testnum); - printf("ORIGINAL:\n[%.*s]\n", static_cast(strlen(request)), request); - printf("TARGET :\n[%.*s]\n", static_cast(strlen(request_tgt)), request_tgt); - printf("PRT_BUFF:\n[%.*s]\n", prt_bufindex, prt_buf); - printf("CPY_BUFF:\n[%.*s]\n", cpy_bufindex, cpy_buf); - return (0); - } - - hdr.destroy(); - new_hdr.destroy(); - - /*** (4) parse the response string into hdr ***/ - - hdr.create(HTTP_TYPE_RESPONSE); - - start = response; - end = start + strlen(start); // 1 character past end of string - - http_parser_init(&parser); - - while (true) { - err = hdr.parse_resp(&parser, &start, end, true); - if (err != PARSE_RESULT_CONT) { - break; - } - } - - if (err == PARSE_RESULT_ERROR) { - printf("FAILED: (test #%d) parse error parsing response hdr\n", testnum); - return (0); - } - - /*** (2) copy the response header ***/ - - new_hdr.create(HTTP_TYPE_RESPONSE); - new_hdr.copy(&hdr); - - /*** (3) print the response header and copy to buffers ***/ - - prt_bufindex = prt_dumpoffset = 0; - prt_ret = hdr.print(prt_buf, prt_bufsize, &prt_bufindex, &prt_dumpoffset); - - cpy_bufindex = cpy_dumpoffset = 0; - cpy_ret = new_hdr.print(cpy_buf, cpy_bufsize, &cpy_bufindex, &cpy_dumpoffset); - - if ((prt_ret != 1) || (cpy_ret != 1)) { - printf("FAILED: (test #%d) couldn't print rsp hdr or copy --- prt_ret=%d, cpy_ret=%d\n", testnum, prt_ret, cpy_ret); - return (0); - } - - if ((static_cast(prt_bufindex) != strlen(response_tgt)) || (static_cast(cpy_bufindex) != strlen(response_tgt))) { - printf("FAILED: (test #%d) print rsp output size mismatch --- tgt=%d, prt_bufsize=%d, cpy_bufsize=%d\n", testnum, - static_cast(strlen(response_tgt)), prt_bufindex, cpy_bufindex); - printf("ORIGINAL:\n[%.*s]\n", static_cast(strlen(response)), response); - printf("TARGET :\n[%.*s]\n", static_cast(strlen(response_tgt)), response_tgt); - printf("PRT_BUFF:\n[%.*s]\n", prt_bufindex, prt_buf); - printf("CPY_BUFF:\n[%.*s]\n", cpy_bufindex, cpy_buf); - return (0); - } - - if ((strncasecmp(response_tgt, prt_buf, strlen(response_tgt)) != 0) || - (strncasecmp(response_tgt, cpy_buf, strlen(response_tgt)) != 0)) { - printf("FAILED: (test #%d) print rsp output mismatch\n", testnum); - printf("ORIGINAL:\n[%.*s]\n", static_cast(strlen(response)), response); - printf("TARGET :\n[%.*s]\n", static_cast(strlen(response_tgt)), response_tgt); - printf("PRT_BUFF:\n[%.*s]\n", prt_bufindex, prt_buf); - printf("CPY_BUFF:\n[%.*s]\n", cpy_bufindex, cpy_buf); - return (0); - } - - hdr.destroy(); - new_hdr.destroy(); - - if (test_http_hdr_copy_over_aux(testnum, request, response) == 0) { - return 0; - } - - return (1); -} - -/*------------------------------------------------------------------------- - -------------------------------------------------------------------------*/ - -int -HdrTest::test_http() -{ - int status = 1; - - static const char request0[] = { - "GET http://www.news.com:80/ HTTP/1.0\r\n" - "Proxy-Connection: Keep-Alive\r\n" - "User-Agent: Mozilla/4.04 [en] (X11; I; Linux 2.0.33 i586)\r\n" - "Pragma: no-cache\r\n" - "Host: www.news.com\r\n" - "Accept: image/gif, image/x-xbitmap, image/jpeg, image/pjpeg, image/png, */*\r\n" - "Accept-Language: en\r\n" - "Accept-Charset: iso-8859-1, *, utf-8\r\n" - "Cookie: u_vid_0_0=00031ba3; " - "s_cur_0_0=0101sisi091314775496e7d3Jx4+POyJakrMybmNOsq6XOn5bVn5Z6a4Ln5crU5M7Rxq2lm5aWpqupo20=; " - "SC_Cnet001=Sampled; SC_Cnet002=Sampled\r\n" - "Client-ip: D1012148\r\n" - "Foo: abcdefghijklmnopqrtu\r\n" - "\r\n", - }; - - static const char request09[] = { - "GET /index.html\r\n" - "\r\n", - }; - - static const char request1[] = { - "GET http://people.netscape.com/jwz/hacks-1.gif HTTP/1.0\r\n" - "If-Modified-Since: Wednesday, 26-Feb-97 06:58:17 GMT; length=842\r\n" - "Referer: http://people.netscape.com/jwz/index.html\r\n" - "Proxy-Connection: Keep-Alive\r\n" - "User-Agent: Mozilla/3.01 (X11; I; Linux 2.0.28 i586)\r\n" - "Pragma: no-cache\r\n" - "Host: people.netscape.com\r\n" - "Accept: image/gif, image/x-xbitmap, image/jpeg, image/pjpeg, */*\r\n" - "\r\n", - }; - - static const char request_no_colon[] = { - "GET http://people.netscape.com/jwz/hacks-1.gif HTTP/1.0\r\n" - "If-Modified-Since Wednesday, 26-Feb-97 06:58:17 GMT; length=842\r\n" - "Referer http://people.netscape.com/jwz/index.html\r\n" - "Proxy-Connection Keep-Alive\r\n" - "User-Agent Mozilla/3.01 (X11; I; Linux 2.0.28 i586)\r\n" - "Pragma no-cache\r\n" - "Host people.netscape.com\r\n" - "Accept image/gif, image/x-xbitmap, image/jpeg, image/pjpeg, */*\r\n" - "\r\n", - }; - - static const char request_no_val[] = { - "GET http://people.netscape.com/jwz/hacks-1.gif HTTP/1.0\r\n" - "If-Modified-Since:\r\n" - "Referer: " - "Proxy-Connection:\r\n" - "User-Agent: \r\n" - "Host:::\r\n" - "\r\n", - }; - - static const char request_multi_fblock[] = { - "GET http://people.netscape.com/jwz/hacks-1.gif HTTP/1.0\r\n" - "If-Modified-Since: Wednesday, 26-Feb-97 06:58:17 GMT; length=842\r\n" - "Referer: http://people.netscape.com/jwz/index.html\r\n" - "Proxy-Connection: Keep-Alive\r\n" - "User-Agent: Mozilla/3.01 (X11; I; Linux 2.0.28 i586)\r\n" - "Pragma: no-cache\r\n" - "Host: people.netscape.com\r\n" - "Accept: image/gif, image/x-xbitmap, image/jpeg, image/pjpeg, */*\r\n" - "X-1: blah\r\n" - "X-2: blah\r\n" - "X-3: blah\r\n" - "X-4: blah\r\n" - "X-5: blah\r\n" - "X-6: blah\r\n" - "X-7: blah\r\n" - "X-8: blah\r\n" - "X-9: blah\r\n" - "Pragma: no-cache\r\n" - "X-X-1: blah\r\n" - "X-X-2: blah\r\n" - "X-X-3: blah\r\n" - "X-X-4: blah\r\n" - "X-X-5: blah\r\n" - "X-X-6: blah\r\n" - "X-X-7: blah\r\n" - "X-X-8: blah\r\n" - "X-X-9: blah\r\n" - "\r\n", - }; - - static const char request_leading_space[] = { - " GET http://www.news.com:80/ HTTP/1.0\r\n" - "Proxy-Connection: Keep-Alive\r\n" - "User-Agent: Mozilla/4.04 [en] (X11; I; Linux 2.0.33 i586)\r\n" - "\r\n", - }; - - static const char request_padding[] = { - "GET http://www.padding.com:80/ HTTP/1.0\r\n" - "X-1: blah1\r\n" - // "X-2: blah2\r\n" - "X-3: blah3\r\n" - // "X-4: blah4\r\n" - "X-5: blah5\r\n" - // "X-6: blah6\r\n" - "X-7: blah7\r\n" - // "X-8: blah8\r\n" - "X-9: blah9\r\n" - "\r\n", - }; - - static const char request_09p[] = { - "GET http://www.news09.com/\r\n" - "\r\n", - }; - - static const char request_09ht[] = { - "GET http://www.news09.com/ HT\r\n" - "\r\n", - }; - - static const char request_11[] = { - "GET http://www.news.com/ HTTP/1.1\r\n" - "Connection: close\r\n" - "\r\n", - }; - - static const char request_too_long[] = { - "GET http://www.news.com/i/am/too/long HTTP/1.1\r\n" - "Connection: close\r\n" - "\r\n", - }; - - static const char request_unterminated[] = { - "GET http://www.unterminated.com/ HTTP/1.1", - }; - - static const char request_blank[] = { - "\r\n", - }; - - static const char request_blank2[] = { - "\r\n" - "\r\n", - }; - - static const char request_blank3[] = { - " " - "\r\n", - }; - - //////////////////////////////////////////////////// - - static const char response0[] = { - "HTTP/1.0 200 OK\r\n" - "MIME-Version: 1.0\r\n" - "Server: WebSTAR/2.1 ID/30013\r\n" - "Content-Type: text/html\r\n" - "Content-Length: 939\r\n" - "Last-Modified: Thursday, 01-Jan-04 05:00:00 GMT\r\n" - "\r\n", - }; - - static const char response1[] = { - "HTTP/1.0 200 OK\r\n" - "Server: Netscape-Communications/1.12\r\n" - "Date: Tuesday, 08-Dec-98 20:32:17 GMT\r\n" - "Content-Type: text/html\r\n" - "\r\n", - }; - - static const char response_no_colon[] = { - "HTTP/1.0 200 OK\r\n" - "Server Netscape-Communications/1.12\r\n" - "Date: Tuesday, 08-Dec-98 20:32:17 GMT\r\n" - "Content-Type: text/html\r\n" - "\r\n", - }; - - static const char response_unterminated[] = { - "HTTP/1.0 200 OK", - }; - - static const char response09[] = { - "", - }; - - static const char response_blank[] = { - "\r\n", - }; - - static const char response_blank2[] = { - "\r\n" - "\r\n", - }; - - static const char response_blank3[] = { - " " - "\r\n", - }; - - static const char response_too_long_req[] = { - "HTTP/1.0 414 URI Too Long\r\n" - "\r\n", - }; - - status = status & test_http_aux(request0, response0); - status = status & test_http_aux(request09, response09); - status = status & test_http_aux(request1, response1); - status = status & test_http_aux(request_no_colon, response_no_colon); - status = status & test_http_aux(request_no_val, response_no_colon); - status = status & test_http_aux(request_leading_space, response0); - status = status & test_http_aux(request_multi_fblock, response0); - status = status & test_http_aux(request_padding, response0); - status = status & test_http_aux(request_09p, response0); - status = status & test_http_aux(request_09ht, response0); - status = status & test_http_aux(request_11, response0); - status = status & test_http_aux(request_unterminated, response_unterminated); - status = status & test_http_aux(request_blank, response_blank); - status = status & test_http_aux(request_blank2, response_blank2); - status = status & test_http_aux(request_blank3, response_blank3); - status = status & test_http_req_parse_error(request_too_long, response_too_long_req); - - return (failures_to_status("test_http", (status == 0))); -} - -/*------------------------------------------------------------------------- - -------------------------------------------------------------------------*/ - -int -HdrTest::test_http_mutation() -{ - int status = 1; - - bri_box("test_http_mutation"); - - printf(" <<< MUST BE HAND-VERIFIED FOR FULL BENEFIT>>>\n\n"); - - HTTPHdr resp_hdr; - int err, i; - HTTPParser parser; - const char base_resp[] = "HTTP/1.0 200 OK\r\n\r\n"; - const char *start, *end; - - /*** (1) parse the response string into req_hdr ***/ - - start = base_resp; - end = start + strlen(start); - - http_parser_init(&parser); - - resp_hdr.create(HTTP_TYPE_RESPONSE); - - while (true) { - err = resp_hdr.parse_resp(&parser, &start, end, true); - if (err != PARSE_RESULT_CONT) { - break; - } - } - - printf("\n======== before mutation ==========\n\n"); - printf("\n["); - resp_hdr.print(nullptr, 0, nullptr, nullptr); - printf("]\n\n"); - - /*** (2) add in a bunch of header fields ****/ - char field_name[1024]; - char field_value[1024]; - for (i = 1; i <= 100; i++) { - snprintf(field_name, sizeof(field_name), "Test%d", i); - snprintf(field_value, sizeof(field_value), "%d %d %d %d %d", i, i, i, i, i); - resp_hdr.value_set(field_name, static_cast(strlen(field_name)), field_value, static_cast(strlen(field_value))); - } - - /**** (3) delete all the even numbered fields *****/ - for (i = 2; i <= 100; i += 2) { - snprintf(field_name, sizeof(field_name), "Test%d", i); - resp_hdr.field_delete(field_name, static_cast(strlen(field_name))); - } - - /***** (4) add in secondary fields for all multiples of 3 ***/ - for (i = 3; i <= 100; i += 3) { - snprintf(field_name, sizeof(field_name), "Test%d", i); - MIMEField *f = resp_hdr.field_create(field_name, static_cast(strlen(field_name))); - resp_hdr.field_attach(f); - snprintf(field_value, sizeof(field_value), "d %d %d %d %d %d", i, i, i, i, i); - f->value_set(resp_hdr.m_heap, resp_hdr.m_mime, field_value, static_cast(strlen(field_value))); - } - - /***** (5) append all fields with multiples of 5 ***/ - for (i = 5; i <= 100; i += 5) { - snprintf(field_name, sizeof(field_name), "Test%d", i); - snprintf(field_value, sizeof(field_value), "a %d", i); - - resp_hdr.value_append(field_name, static_cast(strlen(field_name)), field_value, static_cast(strlen(field_value)), - true); - } - - /**** (6) delete all multiples of nine *****/ - for (i = 9; i <= 100; i += 9) { - snprintf(field_name, sizeof(field_name), "Test%d", i); - resp_hdr.field_delete(field_name, static_cast(strlen(field_name))); - } - - printf("\n======== mutated response ==========\n\n"); - printf("\n["); - resp_hdr.print(nullptr, 0, nullptr, nullptr); - printf("]\n\n"); - - resp_hdr.destroy(); - - return (failures_to_status("test_http_mutation", (status == 0))); -} - -/*------------------------------------------------------------------------- - -------------------------------------------------------------------------*/ - -int -HdrTest::test_arena_aux(Arena *arena, int len) -{ - char *str = arena->str_alloc(len); - int verify_len = static_cast(arena->str_length(str)); - - if (len != verify_len) { - printf("FAILED: requested %d, got %d bytes\n", len, verify_len); - return (1); // 1 error (different return convention) - } else { - return (0); // no errors (different return convention) - } -} - -/*------------------------------------------------------------------------- - -------------------------------------------------------------------------*/ - -int -HdrTest::test_arena() -{ - bri_box("test_arena"); - - Arena *arena; - int failures = 0; - - arena = new Arena; - - failures += test_arena_aux(arena, 1); - failures += test_arena_aux(arena, 127); - failures += test_arena_aux(arena, 128); - failures += test_arena_aux(arena, 129); - failures += test_arena_aux(arena, 255); - failures += test_arena_aux(arena, 256); - failures += test_arena_aux(arena, 16384); - failures += test_arena_aux(arena, 16385); - failures += test_arena_aux(arena, 16511); - failures += test_arena_aux(arena, 16512); - failures += test_arena_aux(arena, 2097152); - failures += test_arena_aux(arena, 2097153); - failures += test_arena_aux(arena, 2097279); - failures += test_arena_aux(arena, 2097280); - - delete arena; - - return (failures_to_status("test_arena", failures)); -} - -/*------------------------------------------------------------------------- - -------------------------------------------------------------------------*/ - -int -HdrTest::test_regex() -{ - DFA dfa; - int status = 1; - - const char *test_harness[] = {"foo", "(.*\\.apache\\.org)", "(.*\\.example\\.com)"}; - - bri_box("test_regex"); - - dfa.compile(test_harness, SIZEOF(test_harness)); - status = status & (dfa.match("trafficserver.apache.org") == 1); - status = status & (dfa.match("www.example.com") == 2); - status = status & (dfa.match("aaaaaafooooooooinktomi....com.org") == -1); - status = status & (dfa.match("foo") == 0); - - return (failures_to_status("test_regex", (status != 1))); -} - -/*------------------------------------------------------------------------- - -------------------------------------------------------------------------*/ - -int -HdrTest::test_accept_language_match() -{ - bri_box("test_accept_language_match"); - - struct { - const char *content_language; - const char *accept_language; - float Q; - int L; - int I; - } test_cases[] = { - {"en", "*", 1.0, 1, 1}, - {"en", "fr", 0.0, 0, 0}, - {"en", "de, fr, en;q=0.7", 0.7, 2, 3}, - {"en-cockney", "de, fr, en;q=0.7", 0.7, 2, 3}, - {"en-cockney", "de, fr, en-foobar;q=0.8, en;q=0.7", 0.7, 2, 4}, - {"en-cockney", "de, fr, en-cockney;q=0.8, en;q=0.7", 0.8, 10, 3}, - {"en-cockney", "de, fr, en;q=0.8, en;q=0.7", 0.8, 2, 3}, - {"en-cockney", "de, fr, en;q=0.7, en;q=0.8", 0.8, 2, 4}, - {"en-cockney", "de, fr, en;q=0.8, en;q=0.8", 0.8, 2, 3}, - {"en-cockney", "de, fr, en-cockney;q=0.7, en;q=0.8", 0.7, 10, 3}, - {"en-cockney", "de, fr, en;q=0.8, en-cockney;q=0.7", 0.7, 10, 4}, - {"en-cockney", "de, fr, en-cockney;q=0.8, en;q=0.8", 0.8, 10, 3}, - {"en-cockney", "de, fr, en-cockney;q=0.8, en;q=0.7", 0.8, 10, 3}, - {"en-cockney", "de, fr, en-american", 0.0, 0, 0}, - {"en-cockney", "de, fr, en;q=0.8, en;q=0.8, *", 0.8, 2, 3}, - {"en-cockney", "de, fr, en;q=0.8, en;q=0.8, *;q=0.9", 0.8, 2, 3}, - {"en-foobar", "de, fr, en;q=0.8, en;q=0.8, *;q=0.9", 0.8, 2, 3}, - {"oo-foobar", "de, fr, en;q=0.8, en;q=0.8, *;q=0.9", 0.9, 1, 5}, - {"oo-foobar", "de, fr, en;q=0.8, en;q=0.8, *;q=0.9, *", 1.0, 1, 6}, - {"oo-foobar", "de, fr, en;q=0.8, en;q=0.8, *, *;q=0.9", 1.0, 1, 5}, - {"fr-belgian", "de, fr;hi-there;q=0.9, fr;q=0.8, en", 0.9, 2, 2}, - {"fr-belgian", "de, fr;q=0.8, fr;hi-there;q=0.9, en", 0.9, 2, 3}, - {nullptr, nullptr, 0.0, 0, 0}, - }; - - int i, I, L; - float Q; - int failures = 0; - - for (i = 0; test_cases[i].accept_language; i++) { - StrList acpt_lang_list(false); - HttpCompat::parse_comma_list(&acpt_lang_list, test_cases[i].accept_language, - static_cast(strlen(test_cases[i].accept_language))); - - Q = HttpCompat::match_accept_language(test_cases[i].content_language, static_cast(strlen(test_cases[i].content_language)), - &acpt_lang_list, &L, &I); - - if ((Q != test_cases[i].Q) || (L != test_cases[i].L) || (I != test_cases[i].I)) { - printf("FAILED: (#%d) got { Q = %.3f; L = %d; I = %d; }, expected { Q = %.3f; L = %d; I = %d; }, from matching\n '%s' " - "against '%s'\n", - i, Q, L, I, test_cases[i].Q, test_cases[i].L, test_cases[i].I, test_cases[i].content_language, - test_cases[i].accept_language); - ++failures; - } - } - - return (failures_to_status("test_accept_language_match", failures)); -} - -/*------------------------------------------------------------------------- - -------------------------------------------------------------------------*/ - -int -HdrTest::test_accept_charset_match() -{ - bri_box("test_accept_charset_match"); - - struct { - const char *content_charset; - const char *accept_charset; - float Q; - int I; - } test_cases[] = { - {"iso-8859-1", "*", 1.0, 1}, - {"iso-8859-1", "iso-8859-2", 0.0, 0}, - {"iso-8859-1", "iso-8859", 0.0, 0}, - {"iso-8859-1", "iso-8859-12", 0.0, 0}, - {"iso-8859-1", "koi-8-r", 0.0, 0}, - {"euc-jp", "shift_jis, iso-2022-jp, euc-jp;q=0.7", 0.7, 3}, - {"euc-jp", "shift_jis, iso-2022-jp, euc-jp;q=0.7", 0.7, 3}, - {"euc-jp", "shift_jis, iso-2022-jp, euc-jp;q=0.8, euc-jp;q=0.7", 0.8, 3}, - {"euc-jp", "shift_jis, iso-2022-jp, euc-jp;q=0.7, euc-jp;q=0.8", 0.8, 4}, - {"euc-jp", "euc-jp;q=0.9, shift_jis, iso-2022-jp, euc-jp;q=0.7, euc-jp;q=0.8", 0.9, 1}, - {"EUC-JP", "euc-jp;q=0.9, shift_jis, iso-2022-jp, euc-jp, euc-jp;q=0.8", 1.0, 4}, - {"euc-jp", "euc-jp;q=0.9, shift_jis, iso-2022-jp, EUC-JP, euc-jp;q=0.8", 1.0, 4}, - {"euc-jp", "shift_jis, iso-2022-jp, euc-jp-foobar", 0.0, 0}, - {"euc-jp", "shift_jis, iso-2022-jp, euc-jp-foobar, *", 1.0, 4}, - {"euc-jp", "shift_jis, iso-2022-jp, euc-jp-foobar, *;q=0.543", 0.543, 4}, - {"euc-jp", "shift_jis, iso-2022-jp, euc-jp-foobar, *;q=0.0", 0.0, 4}, - {"euc-jp", "shift_jis, iso-2022-jp, *;q=0.0, euc-jp-foobar, *;q=0.0", 0.0, 3}, - {"euc-jp", "shift_jis, iso-2022-jp, *;q=0.0, euc-jp-foobar, *;q=0.5", 0.5, 5}, - {"euc-jp", "shift_jis, iso-2022-jp, *;q=0.5, euc-jp-foobar, *;q=0.0", 0.5, 3}, - {"euc-jp", "shift_jis, iso-2022-jp, *;q=0.5, euc-jp-foobar, *, *;q=0.0", 1.0, 5}, - {"euc-jp", "shift_jis, euc-jp;hi-there;q=0.5, iso-2022-jp", 0.5, 2}, - {"euc-jp", "shift_jis, euc-jp;hi-there;q= 0.5, iso-2022-jp", 0.5, 2}, - {"euc-jp", "shift_jis, euc-jp;hi-there;q = 0.5, iso-2022-jp", 0.5, 2}, - {"euc-jp", "shift_jis, euc-jp;hi-there ; q = 0.5, iso-2022-jp", 0.5, 2}, - {"euc-jp", "shift_jis, euc-jp;hi-there ;; q = 0.5, iso-2022-jp", 0.5, 2}, - {"euc-jp", "shift_jis, euc-jp;hi-there ;; Q = 0.5, iso-2022-jp", 0.5, 2}, - {nullptr, nullptr, 0.0, 0}, - }; - - int i, I; - float Q; - int failures = 0; - - for (i = 0; test_cases[i].accept_charset; i++) { - StrList acpt_lang_list(false); - HttpCompat::parse_comma_list(&acpt_lang_list, test_cases[i].accept_charset, - static_cast(strlen(test_cases[i].accept_charset))); - - Q = HttpCompat::match_accept_charset(test_cases[i].content_charset, static_cast(strlen(test_cases[i].content_charset)), - &acpt_lang_list, &I); - - if ((Q != test_cases[i].Q) || (I != test_cases[i].I)) { - printf("FAILED: (#%d) got { Q = %.3f; I = %d; }, expected { Q = %.3f; I = %d; }, from matching\n '%s' against '%s'\n", i, Q, - I, test_cases[i].Q, test_cases[i].I, test_cases[i].content_charset, test_cases[i].accept_charset); - ++failures; - } - } - - return (failures_to_status("test_accept_charset_match", failures)); -} - -/*------------------------------------------------------------------------- - -------------------------------------------------------------------------*/ - -int -HdrTest::test_comma_vals() -{ - static struct { - const char *value; - int value_count; - struct { - int offset; - int len; - } pieces[4]; - } tests[] = { - {",", 2, {{0, 0}, {1, 0}, {-1, 0}, {-1, 0}}}, - {"", 1, {{0, 0}, {-1, 0}, {-1, 0}, {-1, 0}}}, - {" ", 1, {{0, 0}, {-1, 0}, {-1, 0}, {-1, 0}}}, - {", ", 2, {{0, 0}, {1, 0}, {-1, 0}, {-1, 0}}}, - {",,", 3, {{0, 0}, {1, 0}, {2, 0}, {-1, 0}}}, - {" ,", 2, {{0, 0}, {2, 0}, {-1, 0}, {-1, 0}}}, - {" , ", 2, {{0, 0}, {2, 0}, {-1, 0}, {-1, 0}}}, - {"a, ", 2, {{0, 1}, {2, 0}, {-1, 0}, {-1, 0}}}, - {" a, ", 2, {{1, 1}, {3, 0}, {-1, 0}, {-1, 0}}}, - {" ,a", 2, {{0, 0}, {2, 1}, {-1, 0}, {-1, 0}}}, - {" , a", 2, {{0, 0}, {3, 1}, {-1, 0}, {-1, 0}}}, - {"a,a", 2, {{0, 1}, {2, 1}, {-1, 0}, {-1, 0}}}, - {"foo", 1, {{0, 3}, {-1, 0}, {-1, 0}, {-1, 0}}}, - {"foo,", 2, {{0, 3}, {4, 0}, {-1, 0}, {-1, 0}}}, - {"foo, ", 2, {{0, 3}, {4, 0}, {-1, 0}, {-1, 0}}}, - {"foo, bar", 2, {{0, 3}, {5, 3}, {-1, 0}, {-1, 0}}}, - {"foo, bar,", 3, {{0, 3}, {5, 3}, {9, 0}, {-1, 0}}}, - {"foo, bar, ", 3, {{0, 3}, {5, 3}, {9, 0}, {-1, 0}}}, - { - ",foo,bar,", - 4, - {{0, 0}, {1, 3}, {5, 3}, {9, 0}}, - }, - }; - - bri_box("test_comma_vals"); - - HTTPHdr hdr; - char field_name[32]; - int i, j, len, failures, ntests, ncommavals; - - failures = 0; - ntests = sizeof(tests) / sizeof(tests[0]); - - hdr.create(HTTP_TYPE_REQUEST); - - for (i = 0; i < ntests; i++) { - snprintf(field_name, sizeof(field_name), "Test%d", i); - - MIMEField *f = hdr.field_create(field_name, static_cast(strlen(field_name))); - ink_release_assert(f->m_ptr_value == nullptr); - - hdr.field_attach(f); - ink_release_assert(f->m_ptr_value == nullptr); - - hdr.field_value_set(f, tests[i].value, strlen(tests[i].value)); - ink_release_assert(f->m_ptr_value != tests[i].value); // should be copied - ink_release_assert(f->m_len_value == strlen(tests[i].value)); - ink_release_assert(memcmp(f->m_ptr_value, tests[i].value, f->m_len_value) == 0); - - ncommavals = mime_field_value_get_comma_val_count(f); - if (ncommavals != tests[i].value_count) { - ++failures; - printf("FAILED: test #%d (field value '%s') expected val count %d, got %d\n", i + 1, tests[i].value, tests[i].value_count, - ncommavals); - } - - for (j = 0; j < tests[i].value_count; j++) { - const char *val = mime_field_value_get_comma_val(f, &len, j); - int offset = ((val == nullptr) ? -1 : (val - f->m_ptr_value)); - - if ((offset != tests[i].pieces[j].offset) || (len != tests[i].pieces[j].len)) { - ++failures; - printf("FAILED: test #%d (field value '%s', commaval idx %d) expected [offset %d, len %d], got [offset %d, len %d]\n", - i + 1, tests[i].value, j, tests[i].pieces[j].offset, tests[i].pieces[j].len, offset, len); - } - } - } - - hdr.destroy(); - return (failures_to_status("test_comma_vals", failures)); -} - -/*------------------------------------------------------------------------- - -------------------------------------------------------------------------*/ - -int -HdrTest::test_set_comma_vals() -{ - static struct { - const char *old_raw; - int idx; - const char *slice; - const char *new_raw; - } tests[] = { - {"a,b,c", 0, "fred", "fred, b, c"}, - {"a,b,c", 1, "fred", "a, fred, c"}, - {"a,b,c", 2, "fred", "a, b, fred"}, - {"a,b,c", 3, "fred", "a,b,c"}, - {"", 0, "", ""}, - {"", 0, "foo", "foo"}, - {"", 1, "foo", ""}, - {" ", 0, "", ""}, - {" ", 0, "foo", "foo"}, - {" ", 1, "foo", " "}, - {",", 0, "foo", "foo, "}, - {",", 1, "foo", ", foo"}, - {",,", 0, "foo", "foo, , "}, - {",,", 1, "foo", ", foo, "}, - {",,", 2, "foo", ", , foo"}, - {"foo", 0, "abc", "abc"}, - {"foo", 1, "abc", "foo"}, - {"foo", 0, "abc,", "abc,"}, - {"foo", 0, ",abc", ",abc"}, - {",,", 1, ",,,", ", ,,,, "}, - {" a , b , c", 0, "fred", "fred, b, c"}, - {" a , b , c", 1, "fred", "a, fred, c"}, - {" a , b , c", 2, "fred", "a, b, fred"}, - {" a , b , c", 3, "fred", " a , b , c"}, - {" a , b ", 0, "fred", "fred, b"}, - {" a , b ", 1, "fred", "a, fred"}, - {" a , b ", 1, "fred", "a, fred"}, - {" a ,b ", 1, "fred", "a, fred"}, - {"a, , , , e, , g,", 0, "fred", "fred, , , , e, , g, "}, - {"a, , , , e, , g,", 1, "fred", "a, fred, , , e, , g, "}, - {"a, , , , e, , g,", 2, "fred", "a, , fred, , e, , g, "}, - {"a, , , , e, , g,", 5, "fred", "a, , , , e, fred, g, "}, - {"a, , , , e, , g,", 7, "fred", "a, , , , e, , g, fred"}, - {"a, , , , e, , g,", 8, "fred", "a, , , , e, , g,"}, - {"a, \"boo,foo\", c", 0, "wawa", "wawa, \"boo,foo\", c"}, - {"a, \"boo,foo\", c", 1, "wawa", "a, wawa, c"}, - {"a, \"boo,foo\", c", 2, "wawa", "a, \"boo,foo\", wawa"}, - }; - - bri_box("test_set_comma_vals"); - - HTTPHdr hdr; - char field_name[32]; - int i, failures, ntests; - - failures = 0; - ntests = sizeof(tests) / sizeof(tests[0]); - - hdr.create(HTTP_TYPE_REQUEST); - - for (i = 0; i < ntests; i++) { - snprintf(field_name, sizeof(field_name), "Test%d", i); - - MIMEField *f = hdr.field_create(field_name, static_cast(strlen(field_name))); - hdr.field_value_set(f, tests[i].old_raw, strlen(tests[i].old_raw)); - mime_field_value_set_comma_val(hdr.m_heap, hdr.m_mime, f, tests[i].idx, tests[i].slice, strlen(tests[i].slice)); - ink_release_assert(f->m_ptr_value != nullptr); - - if ((f->m_len_value != strlen(tests[i].new_raw)) || (memcmp(f->m_ptr_value, tests[i].new_raw, f->m_len_value) != 0)) { - ++failures; - printf("FAILED: test #%d (setting idx %d of '%s' to '%s') expected '%s' len %d, got '%.*s' len %d\n", i + 1, tests[i].idx, - tests[i].old_raw, tests[i].slice, tests[i].new_raw, static_cast(strlen(tests[i].new_raw)), f->m_len_value, - f->m_ptr_value, f->m_len_value); - } - } - - hdr.destroy(); - return (failures_to_status("test_set_comma_vals", failures)); -} - -/*------------------------------------------------------------------------- - -------------------------------------------------------------------------*/ - -int -HdrTest::test_delete_comma_vals() -{ - bri_box("test_delete_comma_vals"); - rprintf(rtest, " HdrTest test_delete_comma_vals: TEST NOT IMPLEMENTED\n"); - return (1); -} - -/*------------------------------------------------------------------------- - -------------------------------------------------------------------------*/ - -int -HdrTest::test_extend_comma_vals() -{ - bri_box("test_extend_comma_vals"); - rprintf(rtest, " HdrTest test_extend_comma_vals: TEST NOT IMPLEMENTED\n"); - return (1); -} - -/*------------------------------------------------------------------------- - -------------------------------------------------------------------------*/ - -int -HdrTest::test_insert_comma_vals() -{ - bri_box("test_insert_comma_vals"); - rprintf(rtest, " HdrTest test_insert_comma_vals: TEST NOT IMPLEMENTED\n"); - return (1); -} - -/*------------------------------------------------------------------------- - -------------------------------------------------------------------------*/ - -int -HdrTest::test_parse_comma_list() -{ - static struct { - const char *value; - int count; - struct { - int offset; - int len; - } pieces[3]; - } tests[] = { - {"", 1, {{0, 0}, {-1, 0}, {-1, 0}}}, - {",", 2, {{0, 0}, {1, 0}, {-1, 0}}}, - {" ,", 2, {{0, 0}, {2, 0}, {-1, 0}}}, - {", ", 2, {{0, 0}, {1, 0}, {-1, 0}}}, - {" , ", 2, {{0, 0}, {2, 0}, {-1, 0}}}, - {"abc,", 2, {{0, 3}, {4, 0}, {-1, 0}}}, - {"abc, ", 2, {{0, 3}, {4, 0}, {-1, 0}}}, - {"", 1, {{0, 0}, {-1, 0}, {-1, 0}}}, - {" ", 1, {{0, 0}, {-1, 0}, {-1, 0}}}, - {" ", 1, {{0, 0}, {-1, 0}, {-1, 0}}}, - {"a", 1, {{0, 1}, {-1, 0}, {-1, 0}}}, - {" a", 1, {{1, 1}, {-1, 0}, {-1, 0}}}, - {" a ", 1, {{2, 1}, {-1, 0}, {-1, 0}}}, - {"abc,defg", 2, {{0, 3}, {4, 4}, {-1, 0}}}, - {" abc,defg", 2, {{1, 3}, {5, 4}, {-1, 0}}}, - {" abc, defg", 2, {{1, 3}, {6, 4}, {-1, 0}}}, - {" abc , defg", 2, {{1, 3}, {7, 4}, {-1, 0}}}, - {" abc , defg ", 2, {{1, 3}, {7, 4}, {-1, 0}}}, - {" abc , defg, ", 3, {{1, 3}, {7, 4}, {12, 0}}}, - {" abc , defg ,", 3, {{1, 3}, {7, 4}, {13, 0}}}, - {", abc , defg ", 3, {{0, 0}, {2, 3}, {8, 4}}}, - {" ,abc , defg ", 3, {{0, 0}, {2, 3}, {8, 4}}}, - {"a,b", 2, {{0, 1}, {2, 1}, {-1, 0}}}, - {"a,,b", 3, {{0, 1}, {2, 0}, {3, 1}}}, - {"a, ,b", 3, {{0, 1}, {2, 0}, {4, 1}}}, - {"a ,,b", 3, {{0, 1}, {3, 0}, {4, 1}}}, - {",", 2, {{0, 0}, {1, 0}, {-1, 0}}}, - {" ,", 2, {{0, 0}, {2, 0}, {-1, 0}}}, - {", ", 2, {{0, 0}, {1, 0}, {-1, 0}}}, - {" , ", 2, {{0, 0}, {2, 0}, {-1, 0}}}, - {"a,b,", 3, {{0, 1}, {2, 1}, {4, 0}}}, - {"a,b, ", 3, {{0, 1}, {2, 1}, {4, 0}}}, - {"a,b, ", 3, {{0, 1}, {2, 1}, {4, 0}}}, - {"a,b, c", 3, {{0, 1}, {2, 1}, {6, 1}}}, - {"a,b, c ", 3, {{0, 1}, {2, 1}, {6, 1}}}, - {"a,\"b,c\",d", 3, {{0, 1}, {3, 3}, {8, 1}}}, - }; - - bri_box("test_parse_comma_list"); - - int i, j, failures, ntests, offset; - - failures = (offset = 0); - ntests = sizeof(tests) / sizeof(tests[0]); - - for (i = 0; i < ntests; i++) { - StrList list(false); - HttpCompat::parse_comma_list(&list, tests[i].value); - if (list.count != tests[i].count) { - ++failures; - printf("FAILED: test #%d (string '%s') expected list count %d, got %d\n", i + 1, tests[i].value, tests[i].count, list.count); - } - - for (j = 0; j < tests[i].count; j++) { - Str *cell = list.get_idx(j); - if (cell != nullptr) { - offset = cell->str - tests[i].value; - } - - if (tests[i].pieces[j].offset == -1) // should not have a piece - { - if (cell != nullptr) { - ++failures; - printf("FAILED: test #%d (string '%s', idx %d) expected NULL piece, got [offset %d len %d]\n", i + 1, tests[i].value, j, - offset, static_cast(cell->len)); - } - } else // should have a piece - { - if (cell == nullptr) { - ++failures; - printf("FAILED: test #%d (string '%s', idx %d) expected [offset %d len %d], got NULL piece\n", i + 1, tests[i].value, j, - tests[i].pieces[j].offset, tests[i].pieces[j].len); - } else if ((offset != tests[i].pieces[j].offset) || (cell->len != static_cast(tests[i].pieces[j].len))) { - ++failures; - printf("FAILED: test #%d (string '%s', idx %d) expected [offset %d len %d], got [offset %d len %d]\n", i + 1, - tests[i].value, j, tests[i].pieces[j].offset, tests[i].pieces[j].len, offset, static_cast(cell->len)); - } - } - } - } - - return (failures_to_status("test_parse_comma_list", failures)); -} - -/*------------------------------------------------------------------------- - -------------------------------------------------------------------------*/ - -void -HdrTest::bri_box(const char *s) -{ - int i, len; - - len = static_cast(strlen(s)); - printf("\n+-"); - for (i = 0; i < len; i++) { - putchar('-'); - } - printf("-+\n"); - printf("| %s |\n", s); - printf("+-"); - for (i = 0; i < len; i++) { - putchar('-'); - } - printf("-+\n\n"); -} - -/*------------------------------------------------------------------------- - -------------------------------------------------------------------------*/ - -int -HdrTest::failures_to_status(const char *testname, int nfail) -{ - rprintf(rtest, " HdrTest %s: %s\n", testname, ((nfail > 0) ? "FAILED" : "PASSED")); - return ((nfail > 0) ? 0 : 1); -} diff --git a/proxy/hdrs/HdrTest.h b/proxy/hdrs/HdrTest.h deleted file mode 100644 index 76128fac841..00000000000 --- a/proxy/hdrs/HdrTest.h +++ /dev/null @@ -1,77 +0,0 @@ -/** @file - - A brief file description - - @section license License - - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - */ - -/**************************************************************************** - - HdrTest.cc - - Description: - Unit test code for sanity checking the header system is functioning - properly - - - ****************************************************************************/ - -#pragma once - -#include "tscore/Regression.h" - -class HdrTest -{ -public: - RegressionTest *rtest = nullptr; - - HdrTest(){}; - ~HdrTest(){}; - - int go(RegressionTest *t, int atype); - -private: - int test_http_hdr_print_and_copy(); - int test_parse_date(); - int test_format_date(); - int test_url(); - int test_arena(); - int test_regex(); - int test_accept_language_match(); - int test_accept_charset_match(); - int test_comma_vals(); - int test_set_comma_vals(); - int test_delete_comma_vals(); - int test_extend_comma_vals(); - int test_insert_comma_vals(); - int test_parse_comma_list(); - int test_mime(); - int test_http(); - int test_http_mutation(); - - int test_http_hdr_print_and_copy_aux(int testnum, const char *req, const char *req_tgt, const char *rsp, const char *rsp_tgt); - int test_http_hdr_null_char(int testnum, const char *req, const char *req_tgt); - int test_http_hdr_ctl_char(int testnum, const char *req, const char *req_tgt); - int test_http_hdr_copy_over_aux(int testnum, const char *request, const char *response); - int test_http_aux(const char *request, const char *response); - int test_http_req_parse_error(const char *request, const char *response); - int test_arena_aux(Arena *arena, int len); - void bri_box(const char *s); - int failures_to_status(const char *testname, int nfail); -}; diff --git a/proxy/hdrs/HdrToken.cc b/proxy/hdrs/HdrToken.cc index a1ddec4349c..fcfc5c656b6 100644 --- a/proxy/hdrs/HdrToken.cc +++ b/proxy/hdrs/HdrToken.cc @@ -110,7 +110,10 @@ static const char *_hdrtoken_strs[] = { "X-ID", "X-Forwarded-For", "TE", "Strict-Transport-Security", "100-continue", // RFC-2739 - "Forwarded"}; + "Forwarded", + + // RFC-8470 + "Early-Data"}; static HdrTokenTypeBinding _hdrtoken_strs_type_initializers[] = { {"file", HDRTOKEN_TYPE_SCHEME}, @@ -359,7 +362,10 @@ static const char *_hdrtoken_commonly_tokenized_strs[] = { "X-ID", "X-Forwarded-For", "TE", "Strict-Transport-Security", "100-continue", // RFC-2739 - "Forwarded"}; + "Forwarded", + + // RFC-8470 + "Early-Data"}; /*------------------------------------------------------------------------- -------------------------------------------------------------------------*/ diff --git a/proxy/hdrs/MIME.cc b/proxy/hdrs/MIME.cc index f37b95df346..fb7bd33a799 100644 --- a/proxy/hdrs/MIME.cc +++ b/proxy/hdrs/MIME.cc @@ -156,6 +156,7 @@ const char *MIME_FIELD_FORWARDED; const char *MIME_FIELD_SEC_WEBSOCKET_KEY; const char *MIME_FIELD_SEC_WEBSOCKET_VERSION; const char *MIME_FIELD_HTTP2_SETTINGS; +const char *MIME_FIELD_EARLY_DATA; const char *MIME_VALUE_BYTES; const char *MIME_VALUE_CHUNKED; @@ -272,6 +273,7 @@ int MIME_LEN_FORWARDED; int MIME_LEN_SEC_WEBSOCKET_KEY; int MIME_LEN_SEC_WEBSOCKET_VERSION; int MIME_LEN_HTTP2_SETTINGS; +int MIME_LEN_EARLY_DATA; int MIME_WKSIDX_ACCEPT; int MIME_WKSIDX_ACCEPT_CHARSET; @@ -351,6 +353,7 @@ int MIME_WKSIDX_FORWARDED; int MIME_WKSIDX_SEC_WEBSOCKET_KEY; int MIME_WKSIDX_SEC_WEBSOCKET_VERSION; int MIME_WKSIDX_HTTP2_SETTINGS; +int MIME_WKSIDX_EARLY_DATA; /*********************************************************************** * * @@ -485,6 +488,8 @@ mime_hdr_set_accelerators_and_presence_bits(MIMEHdrImpl *mh, MIMEField *field) return; } + ink_assert(mh); + mime_hdr_presence_set(mh, field->m_wks_idx); slot_id = hdrtoken_index_to_slotid(field->m_wks_idx); @@ -625,8 +630,9 @@ mime_hdr_sanity_check(MIMEHdrImpl *mh) if (field->m_next_dup) { bool found = false; for (blk = &(mh->m_first_fblock); blk != nullptr; blk = blk->m_next) { - const char *addr = (const char *)(field->m_next_dup); - if ((addr >= (const char *)(blk)) && (addr < (const char *)(blk) + sizeof(MIMEFieldBlockImpl))) { + const char *addr = reinterpret_cast(field->m_next_dup); + if ((addr >= reinterpret_cast(blk)) && + (addr < reinterpret_cast(blk) + sizeof(MIMEFieldBlockImpl))) { found = true; break; } @@ -743,11 +749,10 @@ mime_init() MIME_FIELD_X_ID = hdrtoken_string_to_wks("X-ID"); MIME_FIELD_X_FORWARDED_FOR = hdrtoken_string_to_wks("X-Forwarded-For"); MIME_FIELD_FORWARDED = hdrtoken_string_to_wks("Forwarded"); - - MIME_FIELD_SEC_WEBSOCKET_KEY = hdrtoken_string_to_wks("Sec-WebSocket-Key"); - MIME_FIELD_SEC_WEBSOCKET_VERSION = hdrtoken_string_to_wks("Sec-WebSocket-Version"); - - MIME_FIELD_HTTP2_SETTINGS = hdrtoken_string_to_wks("HTTP2-Settings"); + MIME_FIELD_SEC_WEBSOCKET_KEY = hdrtoken_string_to_wks("Sec-WebSocket-Key"); + MIME_FIELD_SEC_WEBSOCKET_VERSION = hdrtoken_string_to_wks("Sec-WebSocket-Version"); + MIME_FIELD_HTTP2_SETTINGS = hdrtoken_string_to_wks("HTTP2-Settings"); + MIME_FIELD_EARLY_DATA = hdrtoken_string_to_wks("Early-Data"); MIME_LEN_ACCEPT = hdrtoken_wks_to_length(MIME_FIELD_ACCEPT); MIME_LEN_ACCEPT_CHARSET = hdrtoken_wks_to_length(MIME_FIELD_ACCEPT_CHARSET); @@ -824,11 +829,10 @@ mime_init() MIME_LEN_X_ID = hdrtoken_wks_to_length(MIME_FIELD_X_ID); MIME_LEN_X_FORWARDED_FOR = hdrtoken_wks_to_length(MIME_FIELD_X_FORWARDED_FOR); MIME_LEN_FORWARDED = hdrtoken_wks_to_length(MIME_FIELD_FORWARDED); - - MIME_LEN_SEC_WEBSOCKET_KEY = hdrtoken_wks_to_length(MIME_FIELD_SEC_WEBSOCKET_KEY); - MIME_LEN_SEC_WEBSOCKET_VERSION = hdrtoken_wks_to_length(MIME_FIELD_SEC_WEBSOCKET_VERSION); - - MIME_LEN_HTTP2_SETTINGS = hdrtoken_wks_to_length(MIME_FIELD_HTTP2_SETTINGS); + MIME_LEN_SEC_WEBSOCKET_KEY = hdrtoken_wks_to_length(MIME_FIELD_SEC_WEBSOCKET_KEY); + MIME_LEN_SEC_WEBSOCKET_VERSION = hdrtoken_wks_to_length(MIME_FIELD_SEC_WEBSOCKET_VERSION); + MIME_LEN_HTTP2_SETTINGS = hdrtoken_wks_to_length(MIME_FIELD_HTTP2_SETTINGS); + MIME_LEN_EARLY_DATA = hdrtoken_wks_to_length(MIME_FIELD_EARLY_DATA); MIME_WKSIDX_ACCEPT = hdrtoken_wks_to_index(MIME_FIELD_ACCEPT); MIME_WKSIDX_ACCEPT_CHARSET = hdrtoken_wks_to_index(MIME_FIELD_ACCEPT_CHARSET); @@ -907,6 +911,7 @@ mime_init() MIME_WKSIDX_SEC_WEBSOCKET_KEY = hdrtoken_wks_to_index(MIME_FIELD_SEC_WEBSOCKET_KEY); MIME_WKSIDX_SEC_WEBSOCKET_VERSION = hdrtoken_wks_to_index(MIME_FIELD_SEC_WEBSOCKET_VERSION); MIME_WKSIDX_HTTP2_SETTINGS = hdrtoken_wks_to_index(MIME_FIELD_HTTP2_SETTINGS); + MIME_WKSIDX_EARLY_DATA = hdrtoken_wks_to_index(MIME_FIELD_EARLY_DATA); MIME_VALUE_BYTES = hdrtoken_string_to_wks("bytes"); MIME_VALUE_CHUNKED = hdrtoken_string_to_wks("chunked"); @@ -1570,6 +1575,7 @@ mime_hdr_field_attach(MIMEHdrImpl *mh, MIMEField *field, int check_for_dups, MIM void mime_hdr_field_detach(MIMEHdrImpl *mh, MIMEField *field, bool detach_all_dups) { + ink_assert(mh); MIMEField *next_dup = field->m_next_dup; // If this field is already detached, there's nothing to do. There must @@ -1639,20 +1645,8 @@ mime_hdr_field_delete(HdrHeap *heap, MIMEHdrImpl *mh, MIMEField *field, bool del { if (delete_all_dups) { while (field) { - // NOTE: we pass zero to field_detach for detach_all_dups - // since this loop will already detach each dup MIMEField *next = field->m_next_dup; - - heap->free_string(field->m_ptr_name, field->m_len_name); - heap->free_string(field->m_ptr_value, field->m_len_value); - - MIME_HDR_SANITY_CHECK(mh); - mime_hdr_field_detach(mh, field, false); - - MIME_HDR_SANITY_CHECK(mh); - mime_field_destroy(mh, field); - - MIME_HDR_SANITY_CHECK(mh); + mime_hdr_field_delete(heap, mh, field, false); field = next; } } else { @@ -1664,6 +1658,32 @@ mime_hdr_field_delete(HdrHeap *heap, MIMEHdrImpl *mh, MIMEField *field, bool del MIME_HDR_SANITY_CHECK(mh); mime_field_destroy(mh, field); + + MIMEFieldBlockImpl *prev_block = nullptr; + bool can_destroy_block = true; + for (auto fblock = &(mh->m_first_fblock); fblock != nullptr; fblock = fblock->m_next) { + if (prev_block != nullptr) { + if (fblock->m_freetop == MIME_FIELD_BLOCK_SLOTS && fblock->contains(field)) { + // Check if fields in all slots are deleted + for (auto &m_field_slot : fblock->m_field_slots) { + if (m_field_slot.m_readiness != MIME_FIELD_SLOT_READINESS_DELETED) { + can_destroy_block = false; + break; + } + } + // Destroy a block and maintain the chain + if (can_destroy_block) { + prev_block->m_next = fblock->m_next; + _mime_field_block_destroy(heap, fblock); + if (prev_block->m_next == nullptr) { + mh->m_fblock_list_tail = prev_block; + } + } + break; + } + } + prev_block = fblock; + } } MIME_HDR_SANITY_CHECK(mh); @@ -2364,7 +2384,7 @@ MIMEScanner::get(TextView &input, TextView &output, bool &output_shares_input, b } else { // This really should be an error (spec doesn't permit lone CR) but the regression tests // require it. - this->append({&RAW_CR, 1}); + this->append(TextView(&RAW_CR, 1)); // This is to fix a core dump of the icc 19.1 compiler when {&RAW_CR, 1} is used m_state = MIME_PARSE_INSIDE; } break; @@ -2493,7 +2513,7 @@ mime_parser_clear(MIMEParser *parser) ParseResult mime_parser_parse(MIMEParser *parser, HdrHeap *heap, MIMEHdrImpl *mh, const char **real_s, const char *real_e, - bool must_copy_strings, bool eof, size_t max_hdr_field_size) + bool must_copy_strings, bool eof, bool remove_ws_from_field_name, size_t max_hdr_field_size) { ParseResult err; bool line_is_real; @@ -2553,8 +2573,15 @@ mime_parser_parse(MIMEParser *parser, HdrHeap *heap, MIMEHdrImpl *mh, const char // server MUST reject any received request message that contains // whitespace between a header field-name and colon with a response code // of 400 (Bad Request). + // A proxy MUST remove any such whitespace from a response message before + // forwarding the message downstream. + bool raw_print_field = true; if (is_ws(field_name.back())) { - return PARSE_RESULT_ERROR; + if (!remove_ws_from_field_name) { + return PARSE_RESULT_ERROR; + } + field_name.rtrim_if(&ParseRules::is_ws); + raw_print_field = false; } // find value first @@ -2590,7 +2617,7 @@ mime_parser_parse(MIMEParser *parser, HdrHeap *heap, MIMEHdrImpl *mh, const char MIMEField *field = mime_field_create(heap, mh); mime_field_name_value_set(heap, mh, field, field_name_wks_idx, field_name.data(), field_name.size(), field_value.data(), - field_value.size(), true, parsed.size(), false); + field_value.size(), raw_print_field, parsed.size(), false); mime_hdr_field_attach(mh, field, 1, nullptr); } } diff --git a/proxy/hdrs/MIME.h b/proxy/hdrs/MIME.h index ebf1677bc90..b15d665092d 100644 --- a/proxy/hdrs/MIME.h +++ b/proxy/hdrs/MIME.h @@ -458,6 +458,7 @@ extern const char *MIME_FIELD_FORWARDED; extern const char *MIME_FIELD_SEC_WEBSOCKET_KEY; extern const char *MIME_FIELD_SEC_WEBSOCKET_VERSION; extern const char *MIME_FIELD_HTTP2_SETTINGS; +extern const char *MIME_FIELD_EARLY_DATA; extern const char *MIME_VALUE_BYTES; extern const char *MIME_VALUE_CHUNKED; @@ -559,7 +560,6 @@ extern int MIME_LEN_ATS_INTERNAL; extern int MIME_LEN_X_ID; extern int MIME_LEN_X_FORWARDED_FOR; extern int MIME_LEN_FORWARDED; - extern int MIME_LEN_BYTES; extern int MIME_LEN_CHUNKED; extern int MIME_LEN_CLOSE; @@ -582,11 +582,10 @@ extern int MIME_LEN_PROXY_REVALIDATE; extern int MIME_LEN_PUBLIC; extern int MIME_LEN_S_MAXAGE; extern int MIME_LEN_NEED_REVALIDATE_ONCE; - extern int MIME_LEN_SEC_WEBSOCKET_KEY; extern int MIME_LEN_SEC_WEBSOCKET_VERSION; - extern int MIME_LEN_HTTP2_SETTINGS; +extern int MIME_LEN_EARLY_DATA; extern int MIME_WKSIDX_ACCEPT; extern int MIME_WKSIDX_ACCEPT_CHARSET; @@ -664,6 +663,7 @@ extern int MIME_WKSIDX_X_ID; extern int MIME_WKSIDX_SEC_WEBSOCKET_KEY; extern int MIME_WKSIDX_SEC_WEBSOCKET_VERSION; extern int MIME_WKSIDX_HTTP2_SETTINGS; +extern int MIME_WKSIDX_EARLY_DATA; /*********************************************************************** * * @@ -718,6 +718,9 @@ void mime_hdr_field_attach(MIMEHdrImpl *mh, MIMEField *field, int check_for_dups void mime_hdr_field_detach(MIMEHdrImpl *mh, MIMEField *field, bool detach_all_dups = false); void mime_hdr_field_delete(HdrHeap *heap, MIMEHdrImpl *mh, MIMEField *field, bool delete_all_dups = false); +/** + * Returned slotnum is not a persistent value. A slotnum may refer a different field after making changes to a mime header. + */ int mime_hdr_field_slotnum(MIMEHdrImpl *mh, MIMEField *field); inkcoreapi MIMEField *mime_hdr_prepare_for_value_set(HdrHeap *heap, MIMEHdrImpl *mh, const char *name, int name_length); @@ -758,7 +761,7 @@ void mime_field_value_append(HdrHeap *heap, MIMEHdrImpl *mh, MIMEField *field, c void mime_parser_init(MIMEParser *parser); void mime_parser_clear(MIMEParser *parser); ParseResult mime_parser_parse(MIMEParser *parser, HdrHeap *heap, MIMEHdrImpl *mh, const char **real_s, const char *real_e, - bool must_copy_strings, bool eof, size_t max_hdr_field_size = 131070); + bool must_copy_strings, bool eof, bool remove_ws_from_field_name, size_t max_hdr_field_size = 131070); void mime_hdr_describe(HdrHeapObjImpl *raw, bool recurse); void mime_field_block_describe(HdrHeapObjImpl *raw, bool recurse); @@ -1014,7 +1017,7 @@ class MIMEHdr : public HdrHeapSDKHandle int print(char *buf, int bufsize, int *bufindex, int *chars_to_skip); - int parse(MIMEParser *parser, const char **start, const char *end, bool must_copy_strs, bool eof, + int parse(MIMEParser *parser, const char **start, const char *end, bool must_copy_strs, bool eof, bool remove_ws_from_field_name, size_t max_hdr_field_size = UINT16_MAX); int value_get_index(const char *name, int name_length, const char *value, int value_length) const; @@ -1036,7 +1039,7 @@ class MIMEHdr : public HdrHeapSDKHandle void value_append(const char *name, int name_length, const char *value, int value_length, bool prepend_comma = false, const char separator = ','); - void field_value_set(MIMEField *field, const char *value, int value_length); + void field_value_set(MIMEField *field, const char *value, int value_length, bool reuse_heaps = false); void field_value_set_int(MIMEField *field, int32_t value); void field_value_set_uint(MIMEField *field, uint32_t value); void field_value_set_int64(MIMEField *field, int64_t value); @@ -1092,6 +1095,12 @@ class MIMEHdr : public HdrHeapSDKHandle // No gratuitous copies & refcounts! MIMEHdr(const MIMEHdr &m) = delete; MIMEHdr &operator=(const MIMEHdr &m) = delete; + +private: + // Interface to replace (overwrite) field value without + // changing the heap as long as the new value is not longer + // than the current value + bool field_value_replace(MIMEField *field, const char *value, int value_length); }; /*------------------------------------------------------------------------- @@ -1290,7 +1299,8 @@ MIMEHdr::print(char *buf, int bufsize, int *bufindex, int *chars_to_skip) -------------------------------------------------------------------------*/ inline int -MIMEHdr::parse(MIMEParser *parser, const char **start, const char *end, bool must_copy_strs, bool eof, size_t max_hdr_field_size) +MIMEHdr::parse(MIMEParser *parser, const char **start, const char *end, bool must_copy_strs, bool eof, + bool remove_ws_from_field_name, size_t max_hdr_field_size) { if (!m_heap) m_heap = new_HdrHeap(); @@ -1298,7 +1308,7 @@ MIMEHdr::parse(MIMEParser *parser, const char **start, const char *end, bool mus if (!m_mime) m_mime = mime_hdr_create(m_heap); - return mime_parser_parse(parser, m_heap, m_mime, start, end, must_copy_strs, eof, max_hdr_field_size); + return mime_parser_parse(parser, m_heap, m_mime, start, end, must_copy_strs, eof, remove_ws_from_field_name, max_hdr_field_size); } /*------------------------------------------------------------------------- @@ -1397,10 +1407,23 @@ MIMEHdr::value_get_comma_list(const char *name, int name_length, StrList *list) /*------------------------------------------------------------------------- -------------------------------------------------------------------------*/ +inline bool +MIMEHdr::field_value_replace(MIMEField *field, const char *value, int value_length) +{ + if (field->m_len_value >= value_length) { + memcpy((char *)field->m_ptr_value, value, value_length); + field->m_len_value = value_length; + return true; + } + return false; +} + inline void -MIMEHdr::field_value_set(MIMEField *field, const char *value, int value_length) +MIMEHdr::field_value_set(MIMEField *field, const char *value, int value_length, bool reuse_heaps) { - field->value_set(m_heap, m_mime, value, value_length); + if (!reuse_heaps || !field_value_replace(field, value, value_length)) { + field->value_set(m_heap, m_mime, value, value_length); + } } inline void diff --git a/proxy/hdrs/Makefile.am b/proxy/hdrs/Makefile.am index 7067f5514a4..7fcd2484a7f 100644 --- a/proxy/hdrs/Makefile.am +++ b/proxy/hdrs/Makefile.am @@ -49,12 +49,6 @@ libhdrs_a_SOURCES = \ XPACK.cc \ XPACK.h -if BUILD_TESTS -libhdrs_a_SOURCES += \ - HdrTest.cc \ - HdrTest.h -endif - load_http_hdr_SOURCES = \ HTTP.h \ HdrHeap.h \ @@ -66,7 +60,6 @@ load_http_hdr_LDADD = -L. -lhdrs \ $(top_builddir)/src/tscpp/util/libtscpputil.la check_PROGRAMS = \ - test_mime \ test_proxy_hdrs \ test_hdr_heap \ test_Huffmancode \ @@ -74,25 +67,15 @@ check_PROGRAMS = \ TESTS = $(check_PROGRAMS) -test_mime_LDADD = -L. -lhdrs \ - $(top_builddir)/src/tscore/libtscore.la \ - $(top_builddir)/src/tscpp/util/libtscpputil.la \ - $(top_builddir)/iocore/eventsystem/libinkevent.a \ - $(top_builddir)/lib/records/librecords_p.a \ - $(top_builddir)/mgmt/libmgmt_p.la \ - $(top_builddir)/proxy/shared/libUglyLogStubs.a \ - @HWLOC_LIBS@ \ - @LIBCAP@ - -test_mime_SOURCES = test_mime.cc - test_proxy_hdrs_CPPFLAGS = $(AM_CPPFLAGS) \ -I$(abs_top_srcdir)/tests/include test_proxy_hdrs_SOURCES = \ unit_tests/unit_test_main.cc \ unit_tests/test_Hdrs.cc \ - unit_tests/test_HdrUtils.cc + unit_tests/test_HdrUtils.cc \ + unit_tests/test_URL.cc \ + unit_tests/test_mime.cc test_proxy_hdrs_LDADD = \ $(top_builddir)/src/tscore/libtscore.la \ diff --git a/proxy/hdrs/URL.cc b/proxy/hdrs/URL.cc index 8c94eb5f27c..c2a903b41de 100644 --- a/proxy/hdrs/URL.cc +++ b/proxy/hdrs/URL.cc @@ -290,10 +290,11 @@ url_copy_onto_as_server_url(URLImpl *s_url, HdrHeap *s_heap, URLImpl *d_url, Hdr { url_nuke_proxy_stuff(d_url); - d_url->m_ptr_path = s_url->m_ptr_path; - d_url->m_ptr_params = s_url->m_ptr_params; - d_url->m_ptr_query = s_url->m_ptr_query; - d_url->m_ptr_fragment = s_url->m_ptr_fragment; + d_url->m_ptr_path = s_url->m_ptr_path; + d_url->m_path_is_empty = s_url->m_path_is_empty; + d_url->m_ptr_params = s_url->m_ptr_params; + d_url->m_ptr_query = s_url->m_ptr_query; + d_url->m_ptr_fragment = s_url->m_ptr_fragment; url_clear_string_ref(d_url); d_url->m_len_path = s_url->m_len_path; @@ -827,9 +828,13 @@ url_length_get(URLImpl *url) } if (url->m_ptr_path) { - length += url->m_len_path + 1; // +1 for / - } else { - length += 1; // +1 for / + length += url->m_len_path; + } + + if (!url->m_path_is_empty) { + // m_ptr_path does not contain the initial "/" and thus m_len_path does not + // count it. We account for it here. + length += 1; // +1 for "/" } if (url->m_ptr_params && url->m_len_params > 0) { @@ -1117,34 +1122,41 @@ url_parse_scheme(HdrHeap *heap, URLImpl *url, const char **start, const char *en const char *scheme_end = nullptr; int scheme_wks_idx; + // Skip over spaces while (' ' == *cur && ++cur < end) { - ; } + if (cur < end) { scheme_start = scheme_end = cur; - // special case 'http:' for performance - if ((end - cur >= 5) && (((cur[0] ^ 'h') | (cur[1] ^ 't') | (cur[2] ^ 't') | (cur[3] ^ 'p') | (cur[4] ^ ':')) == 0)) { - scheme_end = cur + 4; // point to colon - url_scheme_set(heap, url, scheme_start, URL_WKSIDX_HTTP, 4, copy_strings_p); - } else if ('/' != *cur) { - // For forward transparent mode, the URL for the method can just be a path, - // so don't scan that for a scheme, as we could find a false positive if there - // is a URL in the parameters (which is legal). + + // If the URL is more complex then a path, parse to see if there is a scheme + if ('/' != *cur) { + // Search for a : it could be part of a scheme or a username:password while (':' != *cur && ++cur < end) { - ; } - if (cur < end) { // found a colon - scheme_wks_idx = hdrtoken_tokenize(scheme_start, cur - scheme_start, &scheme_wks); - - /* Distinguish between a scheme only and a username by looking past the colon. If it is missing - or it's a slash, presume scheme. Otherwise it's a username with a password. - */ - if ((scheme_wks_idx > 0 && hdrtoken_wks_to_token_type(scheme_wks) == HDRTOKEN_TYPE_SCHEME) || // known scheme - (cur >= end - 1 || cur[1] == '/')) // no more data or slash past colon - { - scheme_end = cur; - url_scheme_set(heap, url, scheme_start, scheme_wks_idx, scheme_end - scheme_start, copy_strings_p); + + // If there is a :// then there is a scheme + if (cur + 2 < end && cur[1] == '/' && cur[2] == '/') { // found "://" + scheme_end = cur; + scheme_wks_idx = hdrtoken_tokenize(scheme_start, scheme_end - scheme_start, &scheme_wks); + + if (!(scheme_wks_idx > 0 && hdrtoken_wks_to_token_type(scheme_wks) == HDRTOKEN_TYPE_SCHEME)) { + // Unknown scheme, validate the scheme + + // RFC 3986 Section 3.1 + // These are the valid characters in a scheme: + // scheme = ALPHA *( ALPHA / DIGIT / "+" / "-" / "." ) + // return an error if there is another character in the scheme + if (!ParseRules::is_alpha(*scheme_start)) { + return PARSE_RESULT_ERROR; + } + for (cur = scheme_start + 1; cur < scheme_end; ++cur) { + if (!(ParseRules::is_alnum(*cur) != 0 || *cur == '+' || *cur == '-' || *cur == '.')) { + return PARSE_RESULT_ERROR; + } + } } + url_scheme_set(heap, url, scheme_start, scheme_wks_idx, scheme_end - scheme_start, copy_strings_p); } } *start = scheme_end; @@ -1153,11 +1165,16 @@ url_parse_scheme(HdrHeap *heap, URLImpl *url, const char **start, const char *en return PARSE_RESULT_ERROR; // no non-whitespace found } +// This implementation namespace is necessary because this function is tested by a Catch unit test +// in another source file. +// +namespace UrlImpl +{ /** * This method will return TRUE if the uri is strictly compliant with * RFC 3986 and it will return FALSE if not. */ -static bool +bool url_is_strictly_compliant(const char *start, const char *end) { for (const char *i = start; i < end; ++i) { @@ -1169,27 +1186,34 @@ url_is_strictly_compliant(const char *start, const char *end) return true; } +} // namespace UrlImpl +using namespace UrlImpl; + ParseResult -url_parse(HdrHeap *heap, URLImpl *url, const char **start, const char *end, bool copy_strings_p, bool strict_uri_parsing) +url_parse(HdrHeap *heap, URLImpl *url, const char **start, const char *end, bool copy_strings_p, bool strict_uri_parsing, + bool verify_host_characters) { if (strict_uri_parsing && !url_is_strictly_compliant(*start, end)) { return PARSE_RESULT_ERROR; } ParseResult zret = url_parse_scheme(heap, url, start, end, copy_strings_p); - return PARSE_RESULT_CONT == zret ? url_parse_http(heap, url, start, end, copy_strings_p) : zret; + return PARSE_RESULT_CONT == zret ? url_parse_http(heap, url, start, end, copy_strings_p, verify_host_characters) : zret; } ParseResult -url_parse_no_path_component_breakdown(HdrHeap *heap, URLImpl *url, const char **start, const char *end, bool copy_strings_p) +url_parse_regex(HdrHeap *heap, URLImpl *url, const char **start, const char *end, bool copy_strings_p) { ParseResult zret = url_parse_scheme(heap, url, start, end, copy_strings_p); - return PARSE_RESULT_CONT == zret ? url_parse_http_no_path_component_breakdown(heap, url, start, end, copy_strings_p) : zret; + return PARSE_RESULT_CONT == zret ? url_parse_http_regex(heap, url, start, end, copy_strings_p) : zret; } /** Parse internet URL. + After this function completes, start will point to the first character after the + host or @a end if there are not characters after it. + @verbatim [://][user[:password]@]host[:port] @@ -1204,7 +1228,8 @@ url_parse_no_path_component_breakdown(HdrHeap *heap, URLImpl *url, const char ** */ ParseResult -url_parse_internet(HdrHeap *heap, URLImpl *url, const char **start, char const *end, bool copy_strings_p) +url_parse_internet(HdrHeap *heap, URLImpl *url, const char **start, char const *end, bool copy_strings_p, + bool verify_host_characters) { const char *cur = *start; const char *base; // Base for host/port field. @@ -1281,8 +1306,14 @@ url_parse_internet(HdrHeap *heap, URLImpl *url, const char **start, char const * bracket = cur; // location and flag. ++cur; break; - case '/': // we're done with this phase. - end = cur; // cause loop exit + // RFC 3986, section 3.2: + // The authority component is ... terminated by the next slash ("/"), + // question mark ("?"), or number sign ("#") character, or by the end of + // the URI. + case '/': + case '?': + case '#': + end = cur; // We're done parsing authority, cause loop exit. break; default: ++cur; @@ -1309,7 +1340,7 @@ url_parse_internet(HdrHeap *heap, URLImpl *url, const char **start, char const * } } if (host._size) { - if (validate_host_name(std::string_view(host._ptr, host._size))) { + if (!verify_host_characters || validate_host_name(std::string_view(host._ptr, host._size))) { url_host_set(heap, url, host._ptr, host._size, copy_strings_p); } else { return PARSE_RESULT_ERROR; @@ -1324,9 +1355,6 @@ url_parse_internet(HdrHeap *heap, URLImpl *url, const char **start, char const * } url_port_set(heap, url, port._ptr, port._size, copy_strings_p); } - if ('/' == *cur) { - ++cur; // must do this after filling in host/port. - } *start = cur; return PARSE_RESULT_DONE; } @@ -1337,7 +1365,7 @@ url_parse_internet(HdrHeap *heap, URLImpl *url, const char **start, char const * // empties params/query/fragment component ParseResult -url_parse_http(HdrHeap *heap, URLImpl *url, const char **start, const char *end, bool copy_strings) +url_parse_http(HdrHeap *heap, URLImpl *url, const char **start, const char *end, bool copy_strings, bool verify_host_characters) { ParseResult err; const char *cur; @@ -1351,18 +1379,28 @@ url_parse_http(HdrHeap *heap, URLImpl *url, const char **start, const char *end, const char *fragment_end = nullptr; char mask; - err = url_parse_internet(heap, url, start, end, copy_strings); + err = url_parse_internet(heap, url, start, end, copy_strings, verify_host_characters); if (err < 0) { return err; } cur = *start; + if (url->m_ptr_host == nullptr && ((end - cur) >= 2) && '/' == *cur && '/' == *(cur + 1)) { + // RFC 3986 section-3.3: + // If a URI does not contain an authority component, then the path cannot + // begin with two slash characters ("//"). + return PARSE_RESULT_ERROR; + } + bool nothing_after_host = false; if (*start == end) { + nothing_after_host = true; goto done; } - path_start = cur; - mask = ';' & '?' & '#'; + if (*cur == '/') { + path_start = cur; + } + mask = ';' & '?' & '#'; parse_path2: if ((*cur & mask) == mask) { if (*cur == ';') { @@ -1416,10 +1454,42 @@ url_parse_http(HdrHeap *heap, URLImpl *url, const char **start, const char *end, done: if (path_start) { + // There was an explicit path set with '/'. if (!path_end) { path_end = cur; } + if (path_start == path_end) { + url->m_path_is_empty = true; + } else { + url->m_path_is_empty = false; + // Per RFC 3986 section 3, the query string does not contain the initial + // '?' nor does the fragment contain the initial '#'. The path however + // does contain the initial '/' and a path can be empty, containing no + // characters at all, not even the initial '/'. Our path_get interface, + // however, has long not behaved accordingly, returning only the + // characters after the first '/'. This does not allow users to tell + // whether the path was absolutely empty. Further, callers have to + // account for the missing first '/' character themselves, either in URL + // length calculations or when piecing together their own URL. There are + // various examples of this in core and in the plugins shipped with Traffic + // Server. + // + // Correcting this behavior by having path_get return the entire path, + // (inclusive of any first '/') and an empty string if there were no + // characters specified in the path would break existing functionality, + // including various plugins that expect this behavior. Rather than + // correcting this behavior, therefore, we maintain the current + // functionality but add state to determine whether the path was + // absolutely empty so we can reconstruct such URLs. + ++path_start; + } url_path_set(heap, url, path_start, path_end - path_start, copy_strings); + } else if (!nothing_after_host) { + // There was no path set via '/': it is absolutely empty. However, if there + // is no path, query, or fragment after the host, we by convention add a + // slash after the authority. Users of URL expect this behavior. Thus the + // nothing_after_host check. + url->m_path_is_empty = true; } if (params_start) { if (!params_end) { @@ -1428,12 +1498,14 @@ url_parse_http(HdrHeap *heap, URLImpl *url, const char **start, const char *end, url_params_set(heap, url, params_start, params_end - params_start, copy_strings); } if (query_start) { + // There was a query string marked by '?'. if (!query_end) { query_end = cur; } url_query_set(heap, url, query_start, query_end - query_start, copy_strings); } if (fragment_start) { + // There was a fragment string marked by '#'. if (!fragment_end) { fragment_end = cur; } @@ -1445,7 +1517,7 @@ url_parse_http(HdrHeap *heap, URLImpl *url, const char **start, const char *end, } ParseResult -url_parse_http_no_path_component_breakdown(HdrHeap *heap, URLImpl *url, const char **start, const char *end, bool copy_strings) +url_parse_http_regex(HdrHeap *heap, URLImpl *url, const char **start, const char *end, bool copy_strings) { const char *cur = *start; const char *host_end; @@ -1557,8 +1629,9 @@ url_print(URLImpl *url, char *buf_start, int buf_length, int *buf_index_inout, i } } - TRY(mime_mem_print("/", 1, buf_start, buf_length, buf_index_inout, buf_chars_to_skip_inout)); - + if (!url->m_path_is_empty) { + TRY(mime_mem_print("/", 1, buf_start, buf_length, buf_index_inout, buf_chars_to_skip_inout)); + } if (url->m_ptr_path) { TRY(mime_mem_print(url->m_ptr_path, url->m_len_path, buf_start, buf_length, buf_index_inout, buf_chars_to_skip_inout)); } @@ -1794,78 +1867,3 @@ url_host_CryptoHash_get(URLImpl *url, CryptoHash *hash) ctx.update(&port, sizeof(port)); ctx.finalize(*hash); } - -/*------------------------------------------------------------------------- - * Regression tests - -------------------------------------------------------------------------*/ -#if TS_HAS_TESTS -#include "tscore/TestBox.h" - -const static struct { - const char *const text; - bool valid; -} http_validate_hdr_field_test_case[] = {{"yahoo", true}, - {"yahoo.com", true}, - {"yahoo.wow.com", true}, - {"yahoo.wow.much.amaze.com", true}, - {"209.131.52.50", true}, - {"192.168.0.1", true}, - {"localhost", true}, - {"3ffe:1900:4545:3:200:f8ff:fe21:67cf", true}, - {"fe80:0:0:0:200:f8ff:fe21:67cf", true}, - {"fe80::200:f8ff:fe21:67cf", true}, - {"", false}, // Sample host header XSS attack - {"jlads;f8-9349*(D&F*D(234jD*(FSD*(VKLJ#(*$@()#$)))))", false}, - {"\"\t\n", false}, - {"!@#$%^ &*(*&^%$#@#$%^&*(*&^%$#))", false}, - {":):(:O!!!!!!", false}}; - -REGRESSION_TEST(VALIDATE_HDR_FIELD)(RegressionTest *t, int /* level ATS_UNUSED */, int *pstatus) -{ - TestBox box(t, pstatus); - box = REGRESSION_TEST_PASSED; - - for (auto i : http_validate_hdr_field_test_case) { - const char *const txt = i.text; - box.check(validate_host_name({txt}) == i.valid, "Validation of FQDN (host) header: \"%s\", expected %s, but not", txt, - (i.valid ? "true" : "false")); - } -} - -REGRESSION_TEST(ParseRules_strict_URI)(RegressionTest *t, int /* level ATS_UNUSED */, int *pstatus) -{ - const struct { - const char *const uri; - bool valid; - } http_strict_uri_parsing_test_case[] = {{"/home", true}, - {"/path/data?key=value#id", true}, - {"/ABCDEFGHIJKLMNOPQRSTUVWXYZ", true}, - {"/abcdefghijklmnopqrstuvwxyz", true}, - {"/0123456789", true}, - {":/?#[]@", true}, - {"!$&'()*+,;=", true}, - {"-._~", true}, - {"%", true}, - {"\n", false}, - {"\"", false}, - {"<", false}, - {">", false}, - {"\\", false}, - {"^", false}, - {"`", false}, - {"{", false}, - {"|", false}, - {"}", false}, - {"é", false}}; - - TestBox box(t, pstatus); - box = REGRESSION_TEST_PASSED; - - for (auto i : http_strict_uri_parsing_test_case) { - const char *const uri = i.uri; - box.check(url_is_strictly_compliant(uri, uri + strlen(uri)) == i.valid, "Strictly parse URI: \"%s\", expected %s, but not", uri, - (i.valid ? "true" : "false")); - } -} - -#endif // TS_HAS_TESTS diff --git a/proxy/hdrs/URL.h b/proxy/hdrs/URL.h index 025d2dbabb8..87c9207de46 100644 --- a/proxy/hdrs/URL.h +++ b/proxy/hdrs/URL.h @@ -74,7 +74,9 @@ struct URLImpl : public HdrHeapObjImpl { // 6 bytes uint32_t m_clean : 1; - // 8 bytes + 1 bit, will result in padding + /// Whether the URI had an absolutely empty path, not even an initial '/'. + uint32_t m_path_is_empty : 1; + // 8 bytes + 2 bits, will result in padding // Marshaling Functions int marshal(MarshalXlate *str_xlate, int num_xlate); @@ -194,14 +196,19 @@ void url_params_set(HdrHeap *heap, URLImpl *url, const char *value, int length, void url_query_set(HdrHeap *heap, URLImpl *url, const char *value, int length, bool copy_string); void url_fragment_set(HdrHeap *heap, URLImpl *url, const char *value, int length, bool copy_string); +constexpr bool USE_STRICT_URI_PARSING = true; + ParseResult url_parse(HdrHeap *heap, URLImpl *url, const char **start, const char *end, bool copy_strings, - bool strict_uri_parsing = false); -ParseResult url_parse_no_path_component_breakdown(HdrHeap *heap, URLImpl *url, const char **start, const char *end, - bool copy_strings); -ParseResult url_parse_internet(HdrHeap *heap, URLImpl *url, const char **start, const char *end, bool copy_strings); -ParseResult url_parse_http(HdrHeap *heap, URLImpl *url, const char **start, const char *end, bool copy_strings); -ParseResult url_parse_http_no_path_component_breakdown(HdrHeap *heap, URLImpl *url, const char **start, const char *end, - bool copy_strings); + bool strict_uri_parsing = false, bool verify_host_characters = true); + +constexpr bool COPY_STRINGS = true; + +ParseResult url_parse_regex(HdrHeap *heap, URLImpl *url, const char **start, const char *end, bool copy_strings); +ParseResult url_parse_internet(HdrHeap *heap, URLImpl *url, const char **start, const char *end, bool copy_strings, + bool verify_host_characters); +ParseResult url_parse_http(HdrHeap *heap, URLImpl *url, const char **start, const char *end, bool copy_strings, + bool verify_host_characters); +ParseResult url_parse_http_regex(HdrHeap *heap, URLImpl *url, const char **start, const char *end, bool copy_strings); char *url_unescapify(Arena *arena, const char *str, int length); @@ -276,15 +283,48 @@ class URL : public HdrHeapSDKHandle const char *fragment_get(int *length); void fragment_set(const char *value, int length); + /** + * Parse the given URL string and populate URL state with the parts. + * + * @param[in] url The URL to parse. + * + * @return PARSE_RESULT_DONE if parsing was successful, PARSE_RESULT_ERROR + * otherwise. + */ + ParseResult parse(std::string_view url); + + /** Same as parse() but do not verify that the host has proper FQDN + * characters. + * + * This is useful for RemapConfig To targets which have "$[0-9]" references + * in their host names which will later be substituted for other text. + */ + ParseResult parse_no_host_check(std::string_view url); + ParseResult parse(const char **start, const char *end); ParseResult parse(const char *str, int length); - ParseResult parse_no_path_component_breakdown(const char *str, int length); + + /** Perform more simplified parsing that is resilient to receiving regular + * expressions. + * + * This simply looks for the first '/' in a URL and considers that the end of + * the authority and the beginning of the rest of the URL. This allows for + * the '?' character in an authority as a part of a regex without it being + * considered a query parameter and, thus, avoids confusing the parser. + * + * This is only used in RemapConfig and may have no other uses. + */ + ParseResult parse_regex(std::string_view url); + ParseResult parse_regex(const char *str, int length); public: static char *unescapify(Arena *arena, const char *str, int length); // No gratuitous copies! URL(const URL &u) = delete; URL &operator=(const URL &u) = delete; + +private: + static constexpr bool VERIFY_HOST_CHARACTERS = true; }; /*------------------------------------------------------------------------- @@ -681,6 +721,31 @@ URL::fragment_set(const char *value, int length) url_fragment_set(m_heap, m_url_impl, value, length, true); } +/** + Parser doesn't clear URL first, so if you parse over a non-clear URL, + the resulting URL may contain some of the previous data. + + */ +inline ParseResult +URL::parse(std::string_view url) +{ + return this->parse(url.data(), static_cast(url.size())); +} + +/** + Parser doesn't clear URL first, so if you parse over a non-clear URL, + the resulting URL may contain some of the previous data. + + */ +inline ParseResult +URL::parse_no_host_check(std::string_view url) +{ + ink_assert(valid()); + const char *start = url.data(); + const char *end = url.data() + url.length(); + return url_parse(m_heap, m_url_impl, &start, end, COPY_STRINGS, !USE_STRICT_URI_PARSING, !VERIFY_HOST_CHARACTERS); +} + /** Parser doesn't clear URL first, so if you parse over a non-clear URL, the resulting URL may contain some of the previous data. @@ -690,7 +755,7 @@ inline ParseResult URL::parse(const char **start, const char *end) { ink_assert(valid()); - return url_parse(m_heap, m_url_impl, start, end, true); + return url_parse(m_heap, m_url_impl, start, end, COPY_STRINGS); } /** @@ -713,13 +778,26 @@ URL::parse(const char *str, int length) */ inline ParseResult -URL::parse_no_path_component_breakdown(const char *str, int length) +URL::parse_regex(std::string_view url) +{ + ink_assert(valid()); + const char *str = url.data(); + return url_parse_regex(m_heap, m_url_impl, &str, str + url.length(), COPY_STRINGS); +} + +/** + Parser doesn't clear URL first, so if you parse over a non-clear URL, + the resulting URL may contain some of the previous data. + + */ +inline ParseResult +URL::parse_regex(const char *str, int length) { ink_assert(valid()); if (length < 0) length = (int)strlen(str); ink_assert(valid()); - return url_parse_no_path_component_breakdown(m_heap, m_url_impl, &str, str + length, true); + return url_parse_regex(m_heap, m_url_impl, &str, str + length, COPY_STRINGS); } /*------------------------------------------------------------------------- diff --git a/proxy/hdrs/XPACK.cc b/proxy/hdrs/XPACK.cc index e987479b933..009d5bd93d4 100644 --- a/proxy/hdrs/XPACK.cc +++ b/proxy/hdrs/XPACK.cc @@ -82,7 +82,7 @@ xpack_decode_string(Arena &arena, char **str, uint64_t &str_length, const uint8_ } p += len; - if ((p + encoded_string_len) > buf_end) { + if (buf_end < p || static_cast(buf_end - p) < encoded_string_len) { return XPACK_ERROR_COMPRESSION_ERROR; } diff --git a/proxy/hdrs/test_mime.cc b/proxy/hdrs/test_mime.cc deleted file mode 100644 index eba287d1b19..00000000000 --- a/proxy/hdrs/test_mime.cc +++ /dev/null @@ -1,104 +0,0 @@ -/** @file - - A brief file description - - @section license License - - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - */ - -#include "tscore/TestBox.h" -#include "I_EventSystem.h" -#include "MIME.h" - -REGRESSION_TEST(MIME)(RegressionTest *t, int /* atype ATS_UNUSED */, int *pstatus) -{ - TestBox box(t, pstatus); - box = REGRESSION_TEST_PASSED; - - MIMEField *field; - MIMEHdr hdr; - hdr.create(NULL); - - hdr.field_create("Test1", 5); - hdr.field_create("Test2", 5); - hdr.field_create("Test3", 5); - hdr.field_create("Test4", 5); - field = hdr.field_create("Test5", 5); - - box.check(hdr.m_mime->m_first_fblock.contains(field), "The field block doesn't contain the field but it should"); - box.check(!hdr.m_mime->m_first_fblock.contains(field + (1L << 32)), "The field block contains the field but it shouldn't"); - - int slot_num = mime_hdr_field_slotnum(hdr.m_mime, field); - box.check(slot_num == 4, "Slot number is %d but should be 4", slot_num); - - slot_num = mime_hdr_field_slotnum(hdr.m_mime, field + (1L << 32)); - box.check(slot_num == -1, "Slot number is %d but should be -1", slot_num); - - hdr.destroy(); -} - -REGRESSION_TEST(MIME_PARSERS)(RegressionTest *t, int /* atype ATS_UNUSED */, int *pstatus) -{ - const char *end; - int value; - TestBox box(t, pstatus); - box = REGRESSION_TEST_PASSED; - - std::vector> tests = {{"0", 0}, - {"1234", 1234}, - {"-1234", -1234}, - {"2147483647", 2147483647}, - {"-2147483648", 2147483648}, - {"2147483648", INT_MAX}, - {"-2147483649", INT_MIN}, - {"2147483647", INT_MAX}, - {"-2147483648", INT_MIN}, - {"999999999999", INT_MAX}, - {"-999999999999", INT_MIN}}; - - for (const auto &it : tests) { - auto [buf, val] = it; - - end = buf + strlen(buf); - box.check(mime_parse_int(buf, end) == val, "Failed mime_parse_int"); - box.check(mime_parse_integer(buf, end, &value), "Failed mime_parse_integer call"); - box.check(value == val, "Failed mime_parse_integer value"); - } - - // Also check the date parser, which relies heavily on the mime_parse_integer() function - const char *date1 = "Sun, 05 Dec 1999 08:49:37 GMT"; - const char *date2 = "Sunday, 05-Dec-1999 08:49:37 GMT"; - - int d1 = mime_parse_date(date1, date1 + strlen(date1)); - int d2 = mime_parse_date(date2, date2 + strlen(date2)); - - box.check(d1 == d2, "Failed mime_parse_date"); - - printf("Date1: %d\n", d1); - printf("Date2: %d\n", d2); -} - -int -main(int argc, const char **argv) -{ - Thread *main_thread = new EThread(); - main_thread->set_specific(); - mime_init(); - - return RegressionTest::main(argc, argv, REGRESSION_TEST_QUICK); -} diff --git a/proxy/hdrs/unit_tests/test_HdrUtils.cc b/proxy/hdrs/unit_tests/test_HdrUtils.cc index 50cca3a926c..b86b34963ab 100644 --- a/proxy/hdrs/unit_tests/test_HdrUtils.cc +++ b/proxy/hdrs/unit_tests/test_HdrUtils.cc @@ -56,7 +56,7 @@ TEST_CASE("HdrUtils", "[proxy][hdrutils]") mime.create(heap); mime_parser_init(&parser); - auto result = mime_parser_parse(&parser, heap, mime.m_mime, &real_s, real_e, false, true); + auto result = mime_parser_parse(&parser, heap, mime.m_mime, &real_s, real_e, false, true, false); REQUIRE(PARSE_RESULT_DONE == result); HdrCsvIter iter; @@ -143,7 +143,7 @@ TEST_CASE("HdrUtils 2", "[proxy][hdrutils]") mime.create(heap); mime_parser_init(&parser); - auto result = mime_parser_parse(&parser, heap, mime.m_mime, &real_s, real_e, false, true); + auto result = mime_parser_parse(&parser, heap, mime.m_mime, &real_s, real_e, false, true, false); REQUIRE(PARSE_RESULT_DONE == result); MIMEField *field{mime.field_find(connection_tag.data(), int(connection_tag.size()))}; @@ -185,7 +185,7 @@ TEST_CASE("HdrUtils 3", "[proxy][hdrutils]") mime.create(heap); mime_parser_init(&parser); - auto result = mime_parser_parse(&parser, heap, mime.m_mime, &real_s, real_e, false, true); + auto result = mime_parser_parse(&parser, heap, mime.m_mime, &real_s, real_e, false, true, false); REQUIRE(PARSE_RESULT_DONE == result); MIMEField *field{mime.field_find(connection_tag.data(), int(connection_tag.size()))}; diff --git a/proxy/hdrs/unit_tests/test_Hdrs.cc b/proxy/hdrs/unit_tests/test_Hdrs.cc index a471339f8ad..20ecfa829d3 100644 --- a/proxy/hdrs/unit_tests/test_Hdrs.cc +++ b/proxy/hdrs/unit_tests/test_Hdrs.cc @@ -26,13 +26,19 @@ #include #include #include +#include +#include + +#include "tscore/Regex.h" +#include "tscore/ink_time.h" #include "catch.hpp" #include "HTTP.h" +#include "HttpCompat.h" // replaces test_http_parser_eos_boundary_cases -TEST_CASE("HdrTest", "[proxy][hdrtest]") +TEST_CASE("HdrTestHttpParse", "[proxy][hdrtest]") { struct Test { ts::TextView msg; @@ -115,3 +121,1915 @@ TEST_CASE("MIMEScanner_fragments", "[proxy][mimescanner_fragments]") REQUIRE(message == output); } + +namespace +{ +static const char * +comp_http_hdr(HTTPHdr *h1, HTTPHdr *h2) +{ + int h1_len = h1->length_get(); + int h2_len = h2->length_get(); + + if (h1_len != h2_len) { + return "length mismatch"; + } + + std::unique_ptr h1_pbuf(new char[h1_len + 1]); + std::unique_ptr h2_pbuf(new char[h2_len + 1]); + + int p_index = 0, p_dumpoffset = 0; + + int rval = h1->print(h1_pbuf.get(), h1_len, &p_index, &p_dumpoffset); + if (rval != 1) { + return "hdr print failed"; + } + + p_index = p_dumpoffset = 0; + rval = h2->print(h2_pbuf.get(), h2_len, &p_index, &p_dumpoffset); + if (rval != 1) { + return "hdr print failed"; + } + + rval = memcmp(h1_pbuf.get(), h2_pbuf.get(), h1_len); + + if (rval != 0) { + return "compare failed"; + } else { + return nullptr; + } +} + +int +test_http_hdr_copy_over_aux(int testnum, const char *request, const char *response) +{ + int err; + HTTPHdr req_hdr; + HTTPHdr resp_hdr; + HTTPHdr copy1; + HTTPHdr copy2; + + HTTPParser parser; + const char *start; + const char *end; + const char *comp_str = nullptr; + + /*** (1) parse the request string into hdr ***/ + + req_hdr.create(HTTP_TYPE_REQUEST); + + start = request; + end = start + strlen(start); // 1 character past end of string + + http_parser_init(&parser); + + while (true) { + err = req_hdr.parse_req(&parser, &start, end, true); + if (err != PARSE_RESULT_CONT) { + break; + } + } + + if (err == PARSE_RESULT_ERROR) { + std::printf("FAILED: (test #%d) parse error parsing request hdr\n", testnum); + return (0); + } + http_parser_clear(&parser); + + /*** (2) parse the response string into hdr ***/ + + resp_hdr.create(HTTP_TYPE_RESPONSE); + + start = response; + end = start + strlen(start); // 1 character past end of string + + http_parser_init(&parser); + + while (true) { + err = resp_hdr.parse_resp(&parser, &start, end, true); + if (err != PARSE_RESULT_CONT) { + break; + } + } + + if (err == PARSE_RESULT_ERROR) { + printf("FAILED: (test #%d) parse error parsing response hdr\n", testnum); + return (0); + } + + /*** (3) Basic copy testing ***/ + copy1.create(HTTP_TYPE_REQUEST); + copy1.copy(&req_hdr); + comp_str = comp_http_hdr(&req_hdr, ©1); + if (comp_str) { + goto done; + } + + copy2.create(HTTP_TYPE_RESPONSE); + copy2.copy(&resp_hdr); + comp_str = comp_http_hdr(&resp_hdr, ©2); + if (comp_str) { + goto done; + } + + // The APIs for copying headers uses memcpy() which can be unsafe for + // overlapping memory areas. It's unclear to me why these tests were + // created in the first place honestly, since nothing else does this. + + /*** (4) Gender bending copying ***/ + copy1.copy(&resp_hdr); + comp_str = comp_http_hdr(&resp_hdr, ©1); + if (comp_str) { + goto done; + } + + copy2.copy(&req_hdr); + comp_str = comp_http_hdr(&req_hdr, ©2); + if (comp_str) { + goto done; + } + +done: + req_hdr.destroy(); + resp_hdr.destroy(); + copy1.destroy(); + copy2.destroy(); + + if (comp_str) { + printf("FAILED: (test #%d) copy & compare: %s\n", testnum, comp_str); + printf("REQ:\n[%.*s]\n", static_cast(strlen(request)), request); + printf("RESP :\n[%.*s]\n", static_cast(strlen(response)), response); + return (0); + } else { + return (1); + } +} + +int +test_http_hdr_null_char(int testnum, const char *request, const char * /*request_tgt*/) +{ + int err; + HTTPHdr hdr; + HTTPParser parser; + const char *start; + char cpy_buf[2048]; + const char *cpy_buf_ptr = cpy_buf; + + /*** (1) parse the request string into hdr ***/ + + hdr.create(HTTP_TYPE_REQUEST); + + start = request; + + if (strlen(start) > sizeof(cpy_buf)) { + std::printf("FAILED: (test #%d) Internal buffer too small for null char test\n", testnum); + return (0); + } + strcpy(cpy_buf, start); + + // Put a null character somewhere in the header + int length = strlen(start); + cpy_buf[length / 2] = '\0'; + http_parser_init(&parser); + + while (true) { + err = hdr.parse_req(&parser, &cpy_buf_ptr, cpy_buf_ptr + length, true); + if (err != PARSE_RESULT_CONT) { + break; + } + } + if (err != PARSE_RESULT_ERROR) { + std::printf("FAILED: (test #%d) no parse error parsing request with null char\n", testnum); + return (0); + } + return 1; +} + +int +test_http_hdr_ctl_char(int testnum, const char *request, const char * /*request_tgt */) +{ + int err; + HTTPHdr hdr; + HTTPParser parser; + const char *start; + char cpy_buf[2048]; + const char *cpy_buf_ptr = cpy_buf; + + /*** (1) parse the request string into hdr ***/ + + hdr.create(HTTP_TYPE_REQUEST); + + start = request; + + if (strlen(start) > sizeof(cpy_buf)) { + std::printf("FAILED: (test #%d) Internal buffer too small for ctl char test\n", testnum); + return (0); + } + strcpy(cpy_buf, start); + + // Replace a character in the method + cpy_buf[1] = 16; + + http_parser_init(&parser); + + while (true) { + err = hdr.parse_req(&parser, &cpy_buf_ptr, cpy_buf_ptr + strlen(start), true); + if (err != PARSE_RESULT_CONT) { + break; + } + } + + if (err != PARSE_RESULT_ERROR) { + std::printf("FAILED: (test #%d) no parse error parsing method with ctl char\n", testnum); + return (0); + } + return 1; +} + +int +test_http_hdr_print_and_copy_aux(int testnum, const char *request, const char *request_tgt, const char *response, + const char *response_tgt) +{ + int err; + HTTPHdr hdr; + HTTPParser parser; + const char *start; + const char *end; + + char prt_buf[2048]; + int prt_bufsize = sizeof(prt_buf); + int prt_bufindex, prt_dumpoffset, prt_ret; + + char cpy_buf[2048]; + int cpy_bufsize = sizeof(cpy_buf); + int cpy_bufindex, cpy_dumpoffset, cpy_ret; + + std::unique_ptr marshal_buf(new char[2048]); + int marshal_bufsize = sizeof(cpy_buf); + + /*** (1) parse the request string into hdr ***/ + + hdr.create(HTTP_TYPE_REQUEST); + + start = request; + end = start + strlen(start); // 1 character past end of string + + http_parser_init(&parser); + + while (true) { + err = hdr.parse_req(&parser, &start, end, true); + if (err != PARSE_RESULT_CONT) { + break; + } + } + + if (err == PARSE_RESULT_ERROR) { + std::printf("FAILED: (test #%d) parse error parsing request hdr\n", testnum); + return (0); + } + + /*** (2) copy the request header ***/ + HTTPHdr new_hdr, marshal_hdr; + RefCountObj ref; + + // Pretend to pin this object with a refcount. + ref.refcount_inc(); + + int marshal_len = hdr.m_heap->marshal(marshal_buf.get(), marshal_bufsize); + marshal_hdr.create(HTTP_TYPE_REQUEST); + marshal_hdr.unmarshal(marshal_buf.get(), marshal_len, &ref); + new_hdr.create(HTTP_TYPE_REQUEST); + new_hdr.copy(&marshal_hdr); + + /*** (3) print the request header and copy to buffers ***/ + + prt_bufindex = prt_dumpoffset = 0; + prt_ret = hdr.print(prt_buf, prt_bufsize, &prt_bufindex, &prt_dumpoffset); + + cpy_bufindex = cpy_dumpoffset = 0; + cpy_ret = new_hdr.print(cpy_buf, cpy_bufsize, &cpy_bufindex, &cpy_dumpoffset); + + if ((prt_ret != 1) || (cpy_ret != 1)) { + std::printf("FAILED: (test #%d) couldn't print req hdr or copy --- prt_ret=%d, cpy_ret=%d\n", testnum, prt_ret, cpy_ret); + return (0); + } + + if ((static_cast(prt_bufindex) != strlen(request_tgt)) || (static_cast(cpy_bufindex) != strlen(request_tgt))) { + std::printf("FAILED: (test #%d) print req output size mismatch --- tgt=%d, prt_bufsize=%d, cpy_bufsize=%d\n", testnum, + static_cast(strlen(request_tgt)), prt_bufindex, cpy_bufindex); + + std::printf("ORIGINAL:\n[%.*s]\n", static_cast(strlen(request)), request); + std::printf("TARGET :\n[%.*s]\n", static_cast(strlen(request_tgt)), request_tgt); + std::printf("PRT_BUFF:\n[%.*s]\n", prt_bufindex, prt_buf); + std::printf("CPY_BUFF:\n[%.*s]\n", cpy_bufindex, cpy_buf); + return (0); + } + + if ((strncasecmp(request_tgt, prt_buf, strlen(request_tgt)) != 0) || + (strncasecmp(request_tgt, cpy_buf, strlen(request_tgt)) != 0)) { + std::printf("FAILED: (test #%d) print req output mismatch\n", testnum); + std::printf("ORIGINAL:\n[%.*s]\n", static_cast(strlen(request)), request); + std::printf("TARGET :\n[%.*s]\n", static_cast(strlen(request_tgt)), request_tgt); + std::printf("PRT_BUFF:\n[%.*s]\n", prt_bufindex, prt_buf); + std::printf("CPY_BUFF:\n[%.*s]\n", cpy_bufindex, cpy_buf); + return (0); + } + + hdr.destroy(); + new_hdr.destroy(); + + /*** (4) parse the response string into hdr ***/ + + hdr.create(HTTP_TYPE_RESPONSE); + + start = response; + end = start + strlen(start); // 1 character past end of string + + http_parser_init(&parser); + + while (true) { + err = hdr.parse_resp(&parser, &start, end, true); + if (err != PARSE_RESULT_CONT) { + break; + } + } + + if (err == PARSE_RESULT_ERROR) { + std::printf("FAILED: (test #%d) parse error parsing response hdr\n", testnum); + return (0); + } + + /*** (2) copy the response header ***/ + + new_hdr.create(HTTP_TYPE_RESPONSE); + new_hdr.copy(&hdr); + + /*** (3) print the response header and copy to buffers ***/ + + prt_bufindex = prt_dumpoffset = 0; + prt_ret = hdr.print(prt_buf, prt_bufsize, &prt_bufindex, &prt_dumpoffset); + + cpy_bufindex = cpy_dumpoffset = 0; + cpy_ret = new_hdr.print(cpy_buf, cpy_bufsize, &cpy_bufindex, &cpy_dumpoffset); + + if ((prt_ret != 1) || (cpy_ret != 1)) { + std::printf("FAILED: (test #%d) couldn't print rsp hdr or copy --- prt_ret=%d, cpy_ret=%d\n", testnum, prt_ret, cpy_ret); + return (0); + } + + if ((static_cast(prt_bufindex) != strlen(response_tgt)) || (static_cast(cpy_bufindex) != strlen(response_tgt))) { + std::printf("FAILED: (test #%d) print rsp output size mismatch --- tgt=%d, prt_bufsize=%d, cpy_bufsize=%d\n", testnum, + static_cast(strlen(response_tgt)), prt_bufindex, cpy_bufindex); + std::printf("ORIGINAL:\n[%.*s]\n", static_cast(strlen(response)), response); + std::printf("TARGET :\n[%.*s]\n", static_cast(strlen(response_tgt)), response_tgt); + std::printf("PRT_BUFF:\n[%.*s]\n", prt_bufindex, prt_buf); + std::printf("CPY_BUFF:\n[%.*s]\n", cpy_bufindex, cpy_buf); + return (0); + } + + if ((strncasecmp(response_tgt, prt_buf, strlen(response_tgt)) != 0) || + (strncasecmp(response_tgt, cpy_buf, strlen(response_tgt)) != 0)) { + std::printf("FAILED: (test #%d) print rsp output mismatch\n", testnum); + std::printf("ORIGINAL:\n[%.*s]\n", static_cast(strlen(response)), response); + std::printf("TARGET :\n[%.*s]\n", static_cast(strlen(response_tgt)), response_tgt); + std::printf("PRT_BUFF:\n[%.*s]\n", prt_bufindex, prt_buf); + std::printf("CPY_BUFF:\n[%.*s]\n", cpy_bufindex, cpy_buf); + return (0); + } + + hdr.destroy(); + new_hdr.destroy(); + + if (test_http_hdr_copy_over_aux(testnum, request, response) == 0) { + return 0; + } + + return (1); +} + +int +test_arena_aux(Arena *arena, int len) +{ + char *str = arena->str_alloc(len); + int verify_len = static_cast(arena->str_length(str)); + + if (len != verify_len) { + std::printf("FAILED: requested %d, got %d bytes\n", len, verify_len); + return (1); // 1 error (different return convention) + } else { + return (0); // no errors (different return convention) + } +} + +} // end anonymous namespace + +TEST_CASE("HdrTest", "[proxy][hdrtest]") +{ + hdrtoken_init(); + url_init(); + mime_init(); + http_init(); + + SECTION("Test parse date") + { + static struct { + const char *fast; + const char *slow; + } dates[] = { + {"Sun, 06 Nov 1994 08:49:37 GMT", "Sunday, 06-Nov-1994 08:49:37 GMT"}, + {"Mon, 07 Nov 1994 08:49:37 GMT", "Monday, 07-Nov-1994 08:49:37 GMT"}, + {"Tue, 08 Nov 1994 08:49:37 GMT", "Tuesday, 08-Nov-1994 08:49:37 GMT"}, + {"Wed, 09 Nov 1994 08:49:37 GMT", "Wednesday, 09-Nov-1994 08:49:37 GMT"}, + {"Thu, 10 Nov 1994 08:49:37 GMT", "Thursday, 10-Nov-1994 08:49:37 GMT"}, + {"Fri, 11 Nov 1994 08:49:37 GMT", "Friday, 11-Nov-1994 08:49:37 GMT"}, + {"Sat, 11 Nov 1994 08:49:37 GMT", "Saturday, 11-Nov-1994 08:49:37 GMT"}, + {"Sun, 03 Jan 1999 08:49:37 GMT", "Sunday, 03-Jan-1999 08:49:37 GMT"}, + {"Sun, 07 Feb 1999 08:49:37 GMT", "Sunday, 07-Feb-1999 08:49:37 GMT"}, + {"Sun, 07 Mar 1999 08:49:37 GMT", "Sunday, 07-Mar-1999 08:49:37 GMT"}, + {"Sun, 04 Apr 1999 08:49:37 GMT", "Sunday, 04-Apr-1999 08:49:37 GMT"}, + {"Sun, 02 May 1999 08:49:37 GMT", "Sunday, 02-May-1999 08:49:37 GMT"}, + {"Sun, 06 Jun 1999 08:49:37 GMT", "Sunday, 06-Jun-1999 08:49:37 GMT"}, + {"Sun, 04 Jul 1999 08:49:37 GMT", "Sunday, 04-Jul-1999 08:49:37 GMT"}, + {"Sun, 01 Aug 1999 08:49:37 GMT", "Sunday, 01-Aug-1999 08:49:37 GMT"}, + {"Sun, 05 Sep 1999 08:49:37 GMT", "Sunday, 05-Sep-1999 08:49:37 GMT"}, + {"Sun, 03 Oct 1999 08:49:37 GMT", "Sunday, 03-Oct-1999 08:49:37 GMT"}, + {"Sun, 07 Nov 1999 08:49:37 GMT", "Sunday, 07-Nov-1999 08:49:37 GMT"}, + {"Sun, 05 Dec 1999 08:49:37 GMT", "Sunday, 05-Dec-1999 08:49:37 GMT"}, + {nullptr, nullptr}, + }; + + int i; + time_t fast_t, slow_t; + + for (i = 0; dates[i].fast; i++) { + fast_t = mime_parse_date(dates[i].fast, dates[i].fast + static_cast(strlen(dates[i].fast))); + slow_t = mime_parse_date(dates[i].slow, dates[i].slow + static_cast(strlen(dates[i].slow))); + // compare with strptime here! + if (fast_t != slow_t) { + std::printf("FAILED: date %lu (%s) != %lu (%s)\n", static_cast(fast_t), dates[i].fast, + static_cast(slow_t), dates[i].slow); + CHECK(false); + } + } + } + + SECTION("Test format date") + { + static const char *dates[] = { + "Sun, 06 Nov 1994 08:49:37 GMT", + "Sun, 03 Jan 1999 08:49:37 GMT", + "Sun, 05 Dec 1999 08:49:37 GMT", + "Tue, 25 Apr 2000 20:29:53 GMT", + nullptr, + }; + + // (1) Test a few hand-created dates + + int i; + time_t t, t2, t3; + char buffer[128], buffer2[128]; + static const char *envstr = "TZ=GMT0"; + + // shift into GMT timezone for cftime conversions + putenv(const_cast(envstr)); + tzset(); + + for (i = 0; dates[i]; i++) { + t = mime_parse_date(dates[i], dates[i] + static_cast(strlen(dates[i]))); + + cftime_replacement(buffer, sizeof(buffer), "%a, %d %b %Y %T %Z", &t); + if (memcmp(dates[i], buffer, 29) != 0) { + std::printf("FAILED: original date doesn't match cftime date\n"); + std::printf(" input date: %s\n", dates[i]); + std::printf(" cftime date: %s\n", buffer); + CHECK(false); + } + + mime_format_date(buffer, t); + if (memcmp(dates[i], buffer, 29) != 0) { + std::printf("FAILED: original date doesn't match mime_format_date date\n"); + std::printf(" input date: %s\n", dates[i]); + std::printf(" cftime date: %s\n", buffer); + CHECK(false); + } + } + + // (2) test a few times per day from 1/1/1970 to past 2010 + + // coverity[dont_call] + for (t = 0; t < 40 * 366 * (24 * 60 * 60); t += static_cast(drand48() * (24 * 60 * 60))) { + cftime_replacement(buffer, sizeof(buffer), "%a, %d %b %Y %T %Z", &t); + t2 = mime_parse_date(buffer, buffer + static_cast(strlen(buffer))); + if (t2 != t) { + std::printf("FAILED: parsed time_t doesn't match original time_t\n"); + std::printf(" input time_t: %d (%s)\n", static_cast(t), buffer); + std::printf(" parsed time_t: %d\n", static_cast(t2)); + CHECK(false); + } + mime_format_date(buffer2, t); + if (memcmp(buffer, buffer2, 29) != 0) { + std::printf("FAILED: formatted date doesn't match original date\n"); + std::printf(" original date: %s\n", buffer); + std::printf(" formatted date: %s\n", buffer2); + CHECK(false); + } + t3 = mime_parse_date(buffer2, buffer2 + static_cast(strlen(buffer2))); + if (t != t3) { + std::printf("FAILED: parsed time_t doesn't match original time_t\n"); + std::printf(" input time_t: %d (%s)\n", static_cast(t), buffer2); + std::printf(" parsed time_t: %d\n", static_cast(t3)); + CHECK(false); + } + } + } + + SECTION("Test url") + { + static const char *strs[] = { + "http://some.place/path;params?query#fragment", + + // Start with an easy one... + "http://trafficserver.apache.org/index.html", + + "cheese://bogosity", + + "some.place", + "some.place/", + "http://some.place", + "http://some.place/", + "http://some.place/path", + "http://some.place/path;params", + "http://some.place/path;params?query", + "http://some.place/path;params?query#fragment", + "http://some.place/path?query#fragment", + "http://some.place/path#fragment", + + "some.place:80", + "some.place:80/", + "http://some.place:80", + "http://some.place:80/", + + "foo@some.place:80", + "foo@some.place:80/", + "http://foo@some.place:80", + "http://foo@some.place:80/", + + "foo:bar@some.place:80", + "foo:bar@some.place:80/", + "http://foo:bar@some.place:80", + "http://foo:bar@some.place:80/", + + // Some address stuff + "http://172.16.28.101", + "http://172.16.28.101:8080", + "http://[::]", + "http://[::1]", + "http://[fc01:172:16:28::101]", + "http://[fc01:172:16:28::101]:80", + "http://[fc01:172:16:28:BAAD:BEEF:DEAD:101]", + "http://[fc01:172:16:28:BAAD:BEEF:DEAD:101]:8080", + "http://172.16.28.101/some/path", + "http://172.16.28.101:8080/some/path", + "http://[::1]/some/path", + "http://[fc01:172:16:28::101]/some/path", + "http://[fc01:172:16:28::101]:80/some/path", + "http://[fc01:172:16:28:BAAD:BEEF:DEAD:101]/some/path", + "http://[fc01:172:16:28:BAAD:BEEF:DEAD:101]:8080/some/path", + "http://172.16.28.101/", + "http://[fc01:172:16:28:BAAD:BEEF:DEAD:101]:8080/", + + // "foo:@some.place", TODO - foo:@some.place is change to foo@some.place in the test + "foo:bar@some.place", + "foo:bar@some.place/", + "http://foo:bar@some.place", + "http://foo:bar@some.place/", + "http://foo:bar@[::1]:8080/", + "http://foo@[::1]", + + "mms://sm02.tsqa.example.com/0102rally.asf", + "pnm://foo:bar@some.place:80/path;params?query#fragment", + "rtsp://foo:bar@some.place:80/path;params?query#fragment", + "rtspu://foo:bar@some.place:80/path;params?query#fragment", + "/finance/external/cbsm/*http://cbs.marketwatch.com/archive/19990713/news/current/net.htx?source=blq/yhoo&dist=yhoo", + "http://a.b.com/xx.jpg?newpath=http://bob.dave.com", + + "ht-tp://a.b.com", + "ht+tp://a.b.com", + "ht.tp://a.b.com", + + "h1ttp://a.b.com", + "http1://a.b.com", + }; + + static const char *bad[] = { + "http://[1:2:3:4:5:6:7:8:9]", + "http://1:2:3:4:5:6:7:8:A:B", + "http://bob.com[::1]", + "http://[::1].com", + + "http://foo:bar:baz@bob.com/", + "http://foo:bar:baz@[::1]:8080/", + + "http://]", + "http://:", + + "http:/", + "http:/foo.bar.com/", + "~http://invalid.char.in.scheme/foo", + "http~://invalid.char.in.scheme/foo", + "ht~tp://invalid.char.in.scheme/foo", + "1http://first.char.not.alpha", + "some.domain.com/http://invalid.domain/foo", + ":", + "://", + + // maybe this should be a valid URL + "a.b.com/xx.jpg?newpath=http://bob.dave.com", + }; + + int err, failed = 0; + URL url; + const char *start; + const char *end; + int old_length, new_length; + + for (unsigned i = 0; i < countof(strs); i++) { + old_length = static_cast(strlen(strs[i])); + start = strs[i]; + end = start + old_length; + + url.create(nullptr); + err = url.parse(&start, end); + if (err < 0) { + std::printf("Failed to parse url '%s'\n", start); + failed = 1; + break; + } + + char print_buf[1024]; + new_length = 0; + int offset = 0; + url.print(print_buf, 1024, &new_length, &offset); + print_buf[new_length] = '\0'; + + const char *fail_text = nullptr; + + if (old_length == new_length) { + if (memcmp(print_buf, strs[i], new_length) != 0) { + fail_text = "URLS DIFFER"; + } + } else if (old_length == new_length - 1) { + // Check to see if the difference is the trailing + // slash we add + if (memcmp(print_buf, strs[i], old_length) != 0 || print_buf[new_length - 1] != '/' || (strs[i])[old_length - 1] == '/') { + fail_text = "TRAILING SLASH"; + } + } else { + fail_text = "LENGTHS DIFFER"; + } + + if (fail_text) { + failed = 1; + std::printf("%16s: OLD: (%4d) %s\n", fail_text, old_length, strs[i]); + std::printf("%16s: NEW: (%4d) %s\n", "", new_length, print_buf); + obj_describe(url.m_url_impl, true); + } else { + std::printf("%16s: '%s'\n", "PARSE SUCCESS", strs[i]); + } + + url.destroy(); + } + + for (unsigned i = 0; i < countof(bad); ++i) { + const char *x = bad[i]; + + url.create(nullptr); + err = url.parse(x, strlen(x)); + url.destroy(); + if (err == PARSE_RESULT_DONE) { + failed = 1; + std::printf("Successfully parsed invalid url '%s'", x); + break; + } else { + std::printf(" bad URL - PARSE FAILED: '%s'\n", bad[i]); + } + } + + CHECK(failed == 0); + } + + SECTION("Test mime") + { + // This can not be a static string (any more) since we unfold the headers + // in place. + char mime[] = { + // "Date: Tuesday, 08-Dec-98 20:32:17 GMT\r\n" + "Date: 6 Nov 1994 08:49:37 GMT\r\n" + "Max-Forwards: 65535\r\n" + "Cache-Control: private\r\n" + "accept: foo\r\n" + "accept: bar\n" + ": (null) field name\r\n" + "aCCept: \n" + "ACCEPT\r\n" + "foo: bar\r\n" + "foo: argh\r\n" + "foo: three, four\r\n" + "word word: word \r\n" + "accept: \"fazzle, dazzle\"\r\n" + "accept: 1, 2, 3, 4, 5, 6, 7, 8\r\n" + "continuation: part1\r\n" + " part2\r\n" + "scooby: doo\r\n" + " scooby: doo\r\n" + "bar: foo\r\n" + "\r\n", + }; + + int err; + MIMEHdr hdr; + MIMEParser parser; + const char *start; + const char *end; + + std::printf(" <<< MUST BE HAND-VERIFIED FOR FULL-BENEFIT>>>\n\n"); + + start = mime; + end = start + strlen(start); + + mime_parser_init(&parser); + + bool must_copy_strs = false; + + hdr.create(nullptr); + err = hdr.parse(&parser, &start, end, must_copy_strs, false, false); + + REQUIRE(err >= 0); + + // Test the (new) continuation line folding to be correct. This should replace the + // \r\n with two spaces (so a total of three between "part1" and "part2"). + int length = 0; + const char *continuation = hdr.value_get("continuation", 12, &length); + + if ((13 != length)) { + std::printf("FAILED: continue header folded line was too short\n"); + REQUIRE(false); + } + + if (strncmp(continuation + 5, " ", 3)) { + std::printf("FAILED: continue header unfolding did not produce correct WS's\n"); + REQUIRE(false); + } + + if (strncmp(continuation, "part1 part2", 13)) { + std::printf("FAILED: continue header unfolding was not correct\n"); + REQUIRE(false); + } + + hdr.field_delete("not_there", 9); + hdr.field_delete("accept", 6); + hdr.field_delete("scooby", 6); + hdr.field_delete("scooby", 6); + hdr.field_delete("bar", 3); + hdr.field_delete("continuation", 12); + + int count = hdr.fields_count(); + std::printf("hdr.fields_count() = %d\n", count); + + int i_max_forwards = hdr.value_get_int("Max-Forwards", 12); + int u_max_forwards = hdr.value_get_uint("Max-Forwards", 12); + std::printf("i_max_forwards = %d u_max_forwards = %d\n", i_max_forwards, u_max_forwards); + + hdr.set_age(9999); + + length = hdr.length_get(); + std::printf("hdr.length_get() = %d\n", length); + + time_t t0, t1, t2; + + t0 = hdr.get_date(); + if (t0 == 0) { + std::printf("FAILED: Initial date is zero but shouldn't be\n"); + REQUIRE(false); + } + + t1 = time(nullptr); + hdr.set_date(t1); + t2 = hdr.get_date(); + if (t1 != t2) { + std::printf("FAILED: set_date(%" PRId64 ") ... get_date = %" PRId64 "\n\n", static_cast(t1), + static_cast(t2)); + REQUIRE(false); + } + + hdr.value_append("Cache-Control", 13, "no-cache", 8, true); + + MIMEField *cc_field; + StrList slist; + + cc_field = hdr.field_find("Cache-Control", 13); + + if (cc_field == nullptr) { + std::printf("FAILED: missing Cache-Control header\n\n"); + REQUIRE(false); + } + + // TODO: Do we need to check the "count" returned? + cc_field->value_get_comma_list(&slist); // FIX: correct usage? + + if (cc_field->value_get_index("Private", 7) < 0) { + std::printf("Failed: value_get_index of Cache-Control did not find private"); + REQUIRE(false); + } + if (cc_field->value_get_index("Bogus", 5) >= 0) { + std::printf("Failed: value_get_index of Cache-Control incorrectly found bogus"); + REQUIRE(false); + } + if (hdr.value_get_index("foo", 3, "three", 5) < 0) { + std::printf("Failed: value_get_index of foo did not find three"); + REQUIRE(false); + } + if (hdr.value_get_index("foo", 3, "bar", 3) < 0) { + std::printf("Failed: value_get_index of foo did not find bar"); + REQUIRE(false); + } + if (hdr.value_get_index("foo", 3, "Bogus", 5) >= 0) { + std::printf("Failed: value_get_index of foo incorrectly found bogus"); + REQUIRE(false); + } + + mime_parser_clear(&parser); + + hdr.print(nullptr, 0, nullptr, nullptr); + std::printf("\n"); + + obj_describe((HdrHeapObjImpl *)(hdr.m_mime), true); + + const char *field_name = "Test_heap_reuse"; + + MIMEField *f = hdr.field_create(field_name, static_cast(strlen(field_name))); + REQUIRE(f->m_ptr_value == nullptr); + + hdr.field_attach(f); + REQUIRE(f->m_ptr_value == nullptr); + + const char *test_value = "mytest"; + + std::printf("Testing Heap Reuse..\n"); + hdr.field_value_set(f, "orig_value", strlen("orig_value")); + const char *m_ptr_value_orig = f->m_ptr_value; + hdr.field_value_set(f, test_value, strlen(test_value), true); + REQUIRE(f->m_ptr_value != test_value); // should be copied + REQUIRE(f->m_ptr_value == m_ptr_value_orig); // heap doesn't change + REQUIRE(f->m_len_value == strlen(test_value)); + REQUIRE(memcmp(f->m_ptr_value, test_value, f->m_len_value) == 0); + + m_ptr_value_orig = f->m_ptr_value; + const char *new_test_value = "myTest"; + hdr.field_value_set(f, new_test_value, strlen(new_test_value), false); + REQUIRE(f->m_ptr_value != new_test_value); // should be copied + REQUIRE(f->m_ptr_value != m_ptr_value_orig); // new heap + REQUIRE(f->m_len_value == strlen(new_test_value)); + REQUIRE(memcmp(f->m_ptr_value, new_test_value, f->m_len_value) == 0); + + hdr.fields_clear(); + + hdr.destroy(); + } + + SECTION("Test http hdr print and copy") + { + static struct { + const char *req; + const char *req_tgt; + const char *rsp; + const char *rsp_tgt; + } tests[] = { + {"GET http://foo.com/bar.txt HTTP/1.0\r\n" + "Accept-Language: fjdfjdslkf dsjkfdj flkdsfjlk sjfdlk ajfdlksa\r\n" + "\r\n", + "GET http://foo.com/bar.txt HTTP/1.0\r\n" + "Accept-Language: fjdfjdslkf dsjkfdj flkdsfjlk sjfdlk ajfdlksa\r\n" + "\r\n", + "HTTP/1.0 200 OK\r\n" + "\r\n", + "HTTP/1.0 200 OK\r\n" + "\r\n"}, + {"GET http://foo.com/bar.txt HTTP/1.0\r\n" + "Accept-Language: fjdfjdslkf dsjkfdj flkdsfjlk sjfdlk ajfdlksa fjfj dslkfjdslk fjsdafkl dsajfkldsa jfkldsafj " + "klsafjs lkafjdsalk fsdjakfl sdjaflkdsaj flksdjflsd ;ffd salfdjs lf;sdaf ;dsaf jdsal;fdjsaflkjsda \r\n" + "\r\n", + "GET http://foo.com/bar.txt HTTP/1.0\r\n" + "Accept-Language: fjdfjdslkf dsjkfdj flkdsfjlk sjfdlk ajfdlksa fjfj dslkfjdslk fjsdafkl dsajfkldsa jfkldsafj " + "klsafjs lkafjdsalk fsdjakfl sdjaflkdsaj flksdjflsd ;ffd salfdjs lf;sdaf ;dsaf jdsal;fdjsaflkjsda \r\n" + "\r\n", + "HTTP/1.0 200 OK\r\n" + "\r\n", + "HTTP/1.0 200 OK\r\n" + "\r\n"}, + {"GET http://foo.com/bar.txt HTTP/1.0\r\n" + "Accept-Language: fjdfjdslkf dsjkfdj flkdsfjlk sjfdlk ajfdlksa fjfj dslkfjdslk fjsdafkl dsajfkldsa jfkldsafj " + "klsafjs lkafjdsalk fsdjakfl sdjaflkdsaj flksdjflsd ;ffd salfdjs lf;sdaf ;dsaf jdsal;fdjsaflkjsda kfl; fsdajfl; " + "sdjafl;dsajlsjfl;sdafjsdal;fjds al;fdjslaf ;slajdk;f\r\n" + "\r\n", + "GET http://foo.com/bar.txt HTTP/1.0\r\n" + "Accept-Language: fjdfjdslkf dsjkfdj flkdsfjlk sjfdlk ajfdlksa fjfj dslkfjdslk fjsdafkl dsajfkldsa jfkldsafj " + "klsafjs lkafjdsalk fsdjakfl sdjaflkdsaj flksdjflsd ;ffd salfdjs lf;sdaf ;dsaf jdsal;fdjsaflkjsda kfl; fsdajfl; " + "sdjafl;dsajlsjfl;sdafjsdal;fjds al;fdjslaf ;slajdk;f\r\n" + "\r\n", + "HTTP/1.0 200 OK\r\n" + "\r\n", + "HTTP/1.0 200 OK\r\n" + "\r\n"}, + {"GET http://people.netscape.com/jwz/hacks-1.gif HTTP/1.0\r\n" + "If-Modified-Since: Wednesday, 26-Feb-97 06:58:17 GMT; length=842\r\n" + "Referer: chocolate fribble\r\n", // missing final CRLF + "GET http://people.netscape.com/jwz/hacks-1.gif HTTP/1.0\r\n" + "If-Modified-Since: Wednesday, 26-Feb-97 06:58:17 GMT; length=842\r\n" + "Referer: chocolate fribble\r\n" + "\r\n", + "HTTP/1.0 200 OK\r\n" + "MIME-Version: 1.0\r\n" + "Server: WebSTAR/2.1 ID/30013\r\n" + "Content-Type: text/html\r\n" + "Content-Length: 939\r\n" + "Last-Modified: Thursday, 01-Jan-04 05:00:00 GMT\r\n", // missing final CRLF + "HTTP/1.0 200 OK\r\n" + "MIME-Version: 1.0\r\n" + "Server: WebSTAR/2.1 ID/30013\r\n" + "Content-Type: text/html\r\n" + "Content-Length: 939\r\n" + "Last-Modified: Thursday, 01-Jan-04 05:00:00 GMT\r\n" + "\r\n"}, + {"GET http://people.netscape.com/jwz/hacks-1.gif HTTP/1.0\r\n" + "If-Modified-Since: Wednesday, 26-Feb-97 06:58:17 GMT; length=842\r\n" + "Referer: \r\n", // missing final CRLF + "GET http://people.netscape.com/jwz/hacks-1.gif HTTP/1.0\r\n" + "If-Modified-Since: Wednesday, 26-Feb-97 06:58:17 GMT; length=842\r\n" + "Referer: \r\n" + "\r\n", + "HTTP/1.0 200 OK\r\n" + "MIME-Version: 1.0\r\n" + "Server: WebSTAR/2.1 ID/30013\r\n" + "Content-Type: text/html\r\n" + "Content-Length: 939\r\n" + "Last-Modified: Thursday, 01-Jan-04 05:00:00 GMT\r\n" + "\r\n", + "HTTP/1.0 200 OK\r\n" + "MIME-Version: 1.0\r\n" + "Server: WebSTAR/2.1 ID/30013\r\n" + "Content-Type: text/html\r\n" + "Content-Length: 939\r\n" + "Last-Modified: Thursday, 01-Jan-04 05:00:00 GMT\r\n" + "\r\n"}, + {"GET http://www.news.com:80/ HTTP/1.0\r\n" + "Proxy-Connection: Keep-Alive\r\n" + "User-Agent: Mozilla/4.04 [en] (X11; I; Linux 2.0.33 i586)\r\n" + "Pragma: no-cache\r\n" + "Host: www.news.com\r\n" + "Accept: image/gif, image/x-xbitmap, image/jpeg, image/pjpeg, image/png, */*\r\n" + "Accept-Language: en\r\n" + "Accept-Charset: iso-8859-1, *, utf-8\r\n" + "Client-ip: D1012148\r\n" + "Foo: abcdefghijklmnopqrtu\r\n" + "\r\n", + "GET http://www.news.com:80/ HTTP/1.0\r\n" + "Proxy-Connection: Keep-Alive\r\n" + "User-Agent: Mozilla/4.04 [en] (X11; I; Linux 2.0.33 i586)\r\n" + "Pragma: no-cache\r\n" + "Host: www.news.com\r\n" + "Accept: image/gif, image/x-xbitmap, image/jpeg, image/pjpeg, image/png, */*\r\n" + "Accept-Language: en\r\n" + "Accept-Charset: iso-8859-1, *, utf-8\r\n" + "Client-ip: D1012148\r\n" + "Foo: abcdefghijklmnopqrtu\r\n" + "\r\n", + "HTTP/1.0 200 OK\r\n" + "Content-Length: 16428\r\n" + "Content-Type: text/html\r\n" + "\r\n", + "HTTP/1.0 200 OK\r\n" + "Content-Length: 16428\r\n" + "Content-Type: text/html\r\n" + "\r\n"}, + {"GET http://people.netscape.com/jwz/hacks-1.gif HTTP/1.0\r\n" + "If-Modified-Since: Wednesday, 26-Feb-97 06:58:17 GMT; length=842\r\n" + "Referer: http://people.netscape.com/jwz/index.html\r\n" + "Proxy-Connection: Keep-Alive\r\n" + "User-Agent: Mozilla/3.01 (X11; I; Linux 2.0.28 i586)\r\n" + "Pragma: no-cache\r\n" + "Host: people.netscape.com\r\n" + "Accept: image/gif, image/x-xbitmap, image/jpeg, image/pjpeg, */*\r\n" + "\r\n", + "GET http://people.netscape.com/jwz/hacks-1.gif HTTP/1.0\r\n" + "If-Modified-Since: Wednesday, 26-Feb-97 06:58:17 GMT; length=842\r\n" + "Referer: http://people.netscape.com/jwz/index.html\r\n" + "Proxy-Connection: Keep-Alive\r\n" + "User-Agent: Mozilla/3.01 (X11; I; Linux 2.0.28 i586)\r\n" + "Pragma: no-cache\r\n" + "Host: people.netscape.com\r\n" + "Accept: image/gif, image/x-xbitmap, image/jpeg, image/pjpeg, */*\r\n" + "\r\n", + "HTTP/1.0 200 OK\r\n" + "Content-Length: 16428\r\n" + "Content-Type: text/html\r\n" + "\r\n", + "HTTP/1.0 200 OK\r\n" + "Content-Length: 16428\r\n" + "Content-Type: text/html\r\n" + "\r\n"}, + }; + + int ntests = sizeof(tests) / sizeof(tests[0]); + int i; + + for (i = 0; i < ntests; i++) { + int status = test_http_hdr_print_and_copy_aux(i + 1, tests[i].req, tests[i].req_tgt, tests[i].rsp, tests[i].rsp_tgt); + CHECK(status != 0); + + // Test for expected failures + // parse with a '\0' in the header. Should fail + status = test_http_hdr_null_char(i + 1, tests[i].req, tests[i].req_tgt); + CHECK(status != 0); + + // Parse with a CTL character in the method name. Should fail + status = test_http_hdr_ctl_char(i + 1, tests[i].req, tests[i].req_tgt); + CHECK(status != 0); + } + } + + SECTION("Test http") + { + static const char request0[] = { + "GET http://www.news.com:80/ HTTP/1.0\r\n" + "Proxy-Connection: Keep-Alive\r\n" + "User-Agent: Mozilla/4.04 [en] (X11; I; Linux 2.0.33 i586)\r\n" + "Pragma: no-cache\r\n" + "Host: www.news.com\r\n" + "Accept: image/gif, image/x-xbitmap, image/jpeg, image/pjpeg, image/png, */*\r\n" + "Accept-Language: en\r\n" + "Accept-Charset: iso-8859-1, *, utf-8\r\n" + "Cookie: u_vid_0_0=00031ba3; " + "s_cur_0_0=0101sisi091314775496e7d3Jx4+POyJakrMybmNOsq6XOn5bVn5Z6a4Ln5crU5M7Rxq2lm5aWpqupo20=; " + "SC_Cnet001=Sampled; SC_Cnet002=Sampled\r\n" + "Client-ip: D1012148\r\n" + "Foo: abcdefghijklmnopqrtu\r\n" + "\r\n", + }; + + static const char request09[] = { + "GET /index.html\r\n" + "\r\n", + }; + + static const char request1[] = { + "GET http://people.netscape.com/jwz/hacks-1.gif HTTP/1.0\r\n" + "If-Modified-Since: Wednesday, 26-Feb-97 06:58:17 GMT; length=842\r\n" + "Referer: http://people.netscape.com/jwz/index.html\r\n" + "Proxy-Connection: Keep-Alive\r\n" + "User-Agent: Mozilla/3.01 (X11; I; Linux 2.0.28 i586)\r\n" + "Pragma: no-cache\r\n" + "Host: people.netscape.com\r\n" + "Accept: image/gif, image/x-xbitmap, image/jpeg, image/pjpeg, */*\r\n" + "\r\n", + }; + + static const char request_no_colon[] = { + "GET http://people.netscape.com/jwz/hacks-1.gif HTTP/1.0\r\n" + "If-Modified-Since Wednesday, 26-Feb-97 06:58:17 GMT; length=842\r\n" + "Referer http://people.netscape.com/jwz/index.html\r\n" + "Proxy-Connection Keep-Alive\r\n" + "User-Agent Mozilla/3.01 (X11; I; Linux 2.0.28 i586)\r\n" + "Pragma no-cache\r\n" + "Host people.netscape.com\r\n" + "Accept image/gif, image/x-xbitmap, image/jpeg, image/pjpeg, */*\r\n" + "\r\n", + }; + + static const char request_no_val[] = { + "GET http://people.netscape.com/jwz/hacks-1.gif HTTP/1.0\r\n" + "If-Modified-Since:\r\n" + "Referer: " + "Proxy-Connection:\r\n" + "User-Agent: \r\n" + "Host:::\r\n" + "\r\n", + }; + + static const char request_multi_fblock[] = { + "GET http://people.netscape.com/jwz/hacks-1.gif HTTP/1.0\r\n" + "If-Modified-Since: Wednesday, 26-Feb-97 06:58:17 GMT; length=842\r\n" + "Referer: http://people.netscape.com/jwz/index.html\r\n" + "Proxy-Connection: Keep-Alive\r\n" + "User-Agent: Mozilla/3.01 (X11; I; Linux 2.0.28 i586)\r\n" + "Pragma: no-cache\r\n" + "Host: people.netscape.com\r\n" + "Accept: image/gif, image/x-xbitmap, image/jpeg, image/pjpeg, */*\r\n" + "X-1: blah\r\n" + "X-2: blah\r\n" + "X-3: blah\r\n" + "X-4: blah\r\n" + "X-5: blah\r\n" + "X-6: blah\r\n" + "X-7: blah\r\n" + "X-8: blah\r\n" + "X-9: blah\r\n" + "Pragma: no-cache\r\n" + "X-X-1: blah\r\n" + "X-X-2: blah\r\n" + "X-X-3: blah\r\n" + "X-X-4: blah\r\n" + "X-X-5: blah\r\n" + "X-X-6: blah\r\n" + "X-X-7: blah\r\n" + "X-X-8: blah\r\n" + "X-X-9: blah\r\n" + "\r\n", + }; + + static const char request_leading_space[] = { + " GET http://www.news.com:80/ HTTP/1.0\r\n" + "Proxy-Connection: Keep-Alive\r\n" + "User-Agent: Mozilla/4.04 [en] (X11; I; Linux 2.0.33 i586)\r\n" + "\r\n", + }; + + static const char request_padding[] = { + "GET http://www.padding.com:80/ HTTP/1.0\r\n" + "X-1: blah1\r\n" + // "X-2: blah2\r\n" + "X-3: blah3\r\n" + // "X-4: blah4\r\n" + "X-5: blah5\r\n" + // "X-6: blah6\r\n" + "X-7: blah7\r\n" + // "X-8: blah8\r\n" + "X-9: blah9\r\n" + "\r\n", + }; + + static const char request_09p[] = { + "GET http://www.news09.com/\r\n" + "\r\n", + }; + + static const char request_09ht[] = { + "GET http://www.news09.com/ HT\r\n" + "\r\n", + }; + + static const char request_11[] = { + "GET http://www.news.com/ HTTP/1.1\r\n" + "Connection: close\r\n" + "\r\n", + }; + + static const char request_too_long[] = { + "GET http://www.news.com/i/am/too/long HTTP/1.1\r\n" + "Connection: close\r\n" + "\r\n", + }; + + static const char request_unterminated[] = { + "GET http://www.unterminated.com/ HTTP/1.1", + }; + + static const char request_blank[] = { + "\r\n", + }; + + static const char request_blank2[] = { + "\r\n" + "\r\n", + }; + + static const char request_blank3[] = { + " " + "\r\n", + }; + + //////////////////////////////////////////////////// + + static const char response0[] = { + "HTTP/1.0 200 OK\r\n" + "MIME-Version: 1.0\r\n" + "Server: WebSTAR/2.1 ID/30013\r\n" + "Content-Type: text/html\r\n" + "Content-Length: 939\r\n" + "Last-Modified: Thursday, 01-Jan-04 05:00:00 GMT\r\n" + "\r\n", + }; + + static const char response1[] = { + "HTTP/1.0 200 OK\r\n" + "Server: Netscape-Communications/1.12\r\n" + "Date: Tuesday, 08-Dec-98 20:32:17 GMT\r\n" + "Content-Type: text/html\r\n" + "\r\n", + }; + + static const char response_no_colon[] = { + "HTTP/1.0 200 OK\r\n" + "Server Netscape-Communications/1.12\r\n" + "Date: Tuesday, 08-Dec-98 20:32:17 GMT\r\n" + "Content-Type: text/html\r\n" + "\r\n", + }; + + static const char response_unterminated[] = { + "HTTP/1.0 200 OK", + }; + + static const char response09[] = { + "", + }; + + static const char response_blank[] = { + "\r\n", + }; + + static const char response_blank2[] = { + "\r\n" + "\r\n", + }; + + static const char response_blank3[] = { + " " + "\r\n", + }; + + static const char response_too_long_req[] = { + "HTTP/1.0 414 URI Too Long\r\n" + "\r\n", + }; + + struct RequestResponse { + char const *request; + char const *response; + }; + + RequestResponse rr[] = {{request0, response0}, + {request09, response09}, + {request1, response1}, + {request_no_colon, response_no_colon}, + {request_no_val, response_no_colon}, + {request_leading_space, response0}, + {request_multi_fblock, response0}, + {request_padding, response0}, + {request_09p, response0}, + {request_09ht, response0}, + {request_11, response0}, + {request_unterminated, response_unterminated}, + {request_blank, response_blank}, + {request_blank2, response_blank2}, + {request_blank3, response_blank3}}; + + int err; + HTTPHdr req_hdr, rsp_hdr; + HTTPParser parser; + const char *start; + const char *end; + const char *request; + const char *response; + + for (unsigned idx = 0; idx < (sizeof(rr) / sizeof(*rr)); ++idx) { + request = rr[idx].request; + response = rr[idx].response; + + std::printf(" <<< MUST BE HAND-VERIFIED FOR FULL BENEFIT >>>\n\n"); + + /*** (1) parse the request string into req_hdr ***/ + + start = request; + end = start + strlen(start); // 1 character past end of string + + http_parser_init(&parser); + + req_hdr.create(HTTP_TYPE_REQUEST); + rsp_hdr.create(HTTP_TYPE_RESPONSE); + + std::printf("======== parsing\n\n"); + while (true) { + err = req_hdr.parse_req(&parser, &start, end, true); + if (err != PARSE_RESULT_CONT) { + break; + } + } + if (err == PARSE_RESULT_ERROR) { + req_hdr.destroy(); + rsp_hdr.destroy(); + break; + } + + /*** useless copy to exercise copy function ***/ + + HTTPHdr new_hdr; + new_hdr.create(HTTP_TYPE_REQUEST); + new_hdr.copy(&req_hdr); + new_hdr.destroy(); + + /*** (2) print out the request ***/ + + std::printf("======== real request (length=%d)\n\n", static_cast(strlen(request))); + std::printf("%s\n", request); + + std::printf("\n["); + req_hdr.print(nullptr, 0, nullptr, nullptr); + std::printf("]\n\n"); + + obj_describe(req_hdr.m_http, true); + + // req_hdr.destroy (); + // REQUIRE(!"req_hdr.destroy() not defined"); + + /*** (3) parse the response string into rsp_hdr ***/ + + start = response; + end = start + strlen(start); + + http_parser_clear(&parser); + http_parser_init(&parser); + + while (true) { + err = rsp_hdr.parse_resp(&parser, &start, end, true); + if (err != PARSE_RESULT_CONT) { + break; + } + } + if (err == PARSE_RESULT_ERROR) { + req_hdr.destroy(); + rsp_hdr.destroy(); + break; + } + + http_parser_clear(&parser); + + /*** (4) print out the response ***/ + + std::printf("\n======== real response (length=%d)\n\n", static_cast(strlen(response))); + std::printf("%s\n", response); + + std::printf("\n["); + rsp_hdr.print(nullptr, 0, nullptr, nullptr); + std::printf("]\n\n"); + + obj_describe(rsp_hdr.m_http, true); + + const int NNN = 1000; + { + char buf[NNN]; + int bufindex, last_bufindex; + int tmp; + int i; + + bufindex = 0; + + do { + last_bufindex = bufindex; + tmp = bufindex; + buf[0] = '#'; // make it obvious if hdr.print doesn't print anything + err = rsp_hdr.print(buf, NNN, &bufindex, &tmp); + + // std::printf("test_header: tmp = %d err = %d bufindex = %d\n", tmp, err, bufindex); + putchar('{'); + for (i = 0; i < bufindex - last_bufindex; i++) { + if (!iscntrl(buf[i])) { + putchar(buf[i]); + } else { + std::printf("\\%o", buf[i]); + } + } + putchar('}'); + } while (!err); + } + + // rsp_hdr.print (NULL, 0, NULL, NULL); + + req_hdr.destroy(); + rsp_hdr.destroy(); + } + + { + request = request_too_long; + response = response_too_long_req; + + int status = 1; + + /*** (1) parse the request string into req_hdr ***/ + + start = request; + end = start + strlen(start); // 1 character past end of string + + http_parser_init(&parser); + + req_hdr.create(HTTP_TYPE_REQUEST); + rsp_hdr.create(HTTP_TYPE_RESPONSE); + + std::printf("======== test_http_req_parse_error parsing\n\n"); + err = req_hdr.parse_req(&parser, &start, end, true, true, 1); + if (err != PARSE_RESULT_ERROR) { + status = 0; + } + + http_parser_clear(&parser); + + /*** (4) print out the response ***/ + + std::printf("\n======== real response (length=%d)\n\n", static_cast(strlen(response))); + std::printf("%s\n", response); + + obj_describe(rsp_hdr.m_http, true); + + req_hdr.destroy(); + rsp_hdr.destroy(); + + CHECK(status != 0); + } + } + + SECTION("Test http mutation") + { + std::printf(" <<< MUST BE HAND-VERIFIED FOR FULL BENEFIT>>>\n\n"); + + HTTPHdr resp_hdr; + int err, i; + HTTPParser parser; + const char base_resp[] = "HTTP/1.0 200 OK\r\n\r\n"; + const char *start, *end; + + /*** (1) parse the response string into req_hdr ***/ + + start = base_resp; + end = start + strlen(start); + + http_parser_init(&parser); + + resp_hdr.create(HTTP_TYPE_RESPONSE); + + while (true) { + err = resp_hdr.parse_resp(&parser, &start, end, true); + if (err != PARSE_RESULT_CONT) { + break; + } + } + + std::printf("\n======== before mutation ==========\n\n"); + std::printf("\n["); + resp_hdr.print(nullptr, 0, nullptr, nullptr); + std::printf("]\n\n"); + + /*** (2) add in a bunch of header fields ****/ + char field_name[1024]; + char field_value[1024]; + for (i = 1; i <= 100; i++) { + snprintf(field_name, sizeof(field_name), "Test%d", i); + snprintf(field_value, sizeof(field_value), "%d %d %d %d %d", i, i, i, i, i); + resp_hdr.value_set(field_name, static_cast(strlen(field_name)), field_value, static_cast(strlen(field_value))); + } + + /**** (3) delete all the even numbered fields *****/ + for (i = 2; i <= 100; i += 2) { + snprintf(field_name, sizeof(field_name), "Test%d", i); + resp_hdr.field_delete(field_name, static_cast(strlen(field_name))); + } + + /***** (4) add in secondary fields for all multiples of 3 ***/ + for (i = 3; i <= 100; i += 3) { + snprintf(field_name, sizeof(field_name), "Test%d", i); + MIMEField *f = resp_hdr.field_create(field_name, static_cast(strlen(field_name))); + resp_hdr.field_attach(f); + snprintf(field_value, sizeof(field_value), "d %d %d %d %d %d", i, i, i, i, i); + f->value_set(resp_hdr.m_heap, resp_hdr.m_mime, field_value, static_cast(strlen(field_value))); + } + + /***** (5) append all fields with multiples of 5 ***/ + for (i = 5; i <= 100; i += 5) { + snprintf(field_name, sizeof(field_name), "Test%d", i); + snprintf(field_value, sizeof(field_value), "a %d", i); + + resp_hdr.value_append(field_name, static_cast(strlen(field_name)), field_value, static_cast(strlen(field_value)), + true); + } + + /**** (6) delete all multiples of nine *****/ + for (i = 9; i <= 100; i += 9) { + snprintf(field_name, sizeof(field_name), "Test%d", i); + resp_hdr.field_delete(field_name, static_cast(strlen(field_name))); + } + + std::printf("\n======== mutated response ==========\n\n"); + std::printf("\n["); + resp_hdr.print(nullptr, 0, nullptr, nullptr); + std::printf("]\n\n"); + + resp_hdr.destroy(); + } + + SECTION("Test arena") + { + Arena *arena; + + arena = new Arena; + + CHECK(test_arena_aux(arena, 1) != 1); + CHECK(test_arena_aux(arena, 127) != 1); + CHECK(test_arena_aux(arena, 128) != 1); + CHECK(test_arena_aux(arena, 129) != 1); + CHECK(test_arena_aux(arena, 255) != 1); + CHECK(test_arena_aux(arena, 256) != 1); + CHECK(test_arena_aux(arena, 16384) != 1); + CHECK(test_arena_aux(arena, 16385) != 1); + CHECK(test_arena_aux(arena, 16511) != 1); + CHECK(test_arena_aux(arena, 16512) != 1); + CHECK(test_arena_aux(arena, 2097152) != 1); + CHECK(test_arena_aux(arena, 2097153) != 1); + CHECK(test_arena_aux(arena, 2097279) != 1); + CHECK(test_arena_aux(arena, 2097280) != 1); + + delete arena; + } + + SECTION("Test regex") + { + DFA dfa; + + const char *test_harness[] = {"foo", "(.*\\.apache\\.org)", "(.*\\.example\\.com)"}; + + dfa.compile(test_harness, SIZEOF(test_harness)); + CHECK(dfa.match("trafficserver.apache.org") == 1); + CHECK(dfa.match("www.example.com") == 2); + CHECK(dfa.match("aaaaaafooooooooinktomi....com.org") == -1); + CHECK(dfa.match("foo") == 0); + } + + SECTION("Test accept language match") + { + struct { + const char *content_language; + const char *accept_language; + float Q; + int L; + int I; + } test_cases[] = { + {"en", "*", 1.0, 1, 1}, + {"en", "fr", 0.0, 0, 0}, + {"en", "de, fr, en;q=0.7", 0.7, 2, 3}, + {"en-cockney", "de, fr, en;q=0.7", 0.7, 2, 3}, + {"en-cockney", "de, fr, en-foobar;q=0.8, en;q=0.7", 0.7, 2, 4}, + {"en-cockney", "de, fr, en-cockney;q=0.8, en;q=0.7", 0.8, 10, 3}, + {"en-cockney", "de, fr, en;q=0.8, en;q=0.7", 0.8, 2, 3}, + {"en-cockney", "de, fr, en;q=0.7, en;q=0.8", 0.8, 2, 4}, + {"en-cockney", "de, fr, en;q=0.8, en;q=0.8", 0.8, 2, 3}, + {"en-cockney", "de, fr, en-cockney;q=0.7, en;q=0.8", 0.7, 10, 3}, + {"en-cockney", "de, fr, en;q=0.8, en-cockney;q=0.7", 0.7, 10, 4}, + {"en-cockney", "de, fr, en-cockney;q=0.8, en;q=0.8", 0.8, 10, 3}, + {"en-cockney", "de, fr, en-cockney;q=0.8, en;q=0.7", 0.8, 10, 3}, + {"en-cockney", "de, fr, en-american", 0.0, 0, 0}, + {"en-cockney", "de, fr, en;q=0.8, en;q=0.8, *", 0.8, 2, 3}, + {"en-cockney", "de, fr, en;q=0.8, en;q=0.8, *;q=0.9", 0.8, 2, 3}, + {"en-foobar", "de, fr, en;q=0.8, en;q=0.8, *;q=0.9", 0.8, 2, 3}, + {"oo-foobar", "de, fr, en;q=0.8, en;q=0.8, *;q=0.9", 0.9, 1, 5}, + {"oo-foobar", "de, fr, en;q=0.8, en;q=0.8, *;q=0.9, *", 1.0, 1, 6}, + {"oo-foobar", "de, fr, en;q=0.8, en;q=0.8, *, *;q=0.9", 1.0, 1, 5}, + {"fr-belgian", "de, fr;hi-there;q=0.9, fr;q=0.8, en", 0.9, 2, 2}, + {"fr-belgian", "de, fr;q=0.8, fr;hi-there;q=0.9, en", 0.9, 2, 3}, + {nullptr, nullptr, 0.0, 0, 0}, + }; + + int i, I, L; + float Q; + + for (i = 0; test_cases[i].accept_language; i++) { + StrList acpt_lang_list(false); + HttpCompat::parse_comma_list(&acpt_lang_list, test_cases[i].accept_language, + static_cast(strlen(test_cases[i].accept_language))); + + Q = HttpCompat::match_accept_language(test_cases[i].content_language, + static_cast(strlen(test_cases[i].content_language)), &acpt_lang_list, &L, &I); + + if ((Q != test_cases[i].Q) || (L != test_cases[i].L) || (I != test_cases[i].I)) { + std::printf( + "FAILED: (#%d) got { Q = %.3f; L = %d; I = %d; }, expected { Q = %.3f; L = %d; I = %d; }, from matching\n '%s' " + "against '%s'\n", + i, Q, L, I, test_cases[i].Q, test_cases[i].L, test_cases[i].I, test_cases[i].content_language, + test_cases[i].accept_language); + CHECK(false); + } + } + } + + SECTION("Test accept charset match") + { + struct { + const char *content_charset; + const char *accept_charset; + float Q; + int I; + } test_cases[] = { + {"iso-8859-1", "*", 1.0, 1}, + {"iso-8859-1", "iso-8859-2", 0.0, 0}, + {"iso-8859-1", "iso-8859", 0.0, 0}, + {"iso-8859-1", "iso-8859-12", 0.0, 0}, + {"iso-8859-1", "koi-8-r", 0.0, 0}, + {"euc-jp", "shift_jis, iso-2022-jp, euc-jp;q=0.7", 0.7, 3}, + {"euc-jp", "shift_jis, iso-2022-jp, euc-jp;q=0.7", 0.7, 3}, + {"euc-jp", "shift_jis, iso-2022-jp, euc-jp;q=0.8, euc-jp;q=0.7", 0.8, 3}, + {"euc-jp", "shift_jis, iso-2022-jp, euc-jp;q=0.7, euc-jp;q=0.8", 0.8, 4}, + {"euc-jp", "euc-jp;q=0.9, shift_jis, iso-2022-jp, euc-jp;q=0.7, euc-jp;q=0.8", 0.9, 1}, + {"EUC-JP", "euc-jp;q=0.9, shift_jis, iso-2022-jp, euc-jp, euc-jp;q=0.8", 1.0, 4}, + {"euc-jp", "euc-jp;q=0.9, shift_jis, iso-2022-jp, EUC-JP, euc-jp;q=0.8", 1.0, 4}, + {"euc-jp", "shift_jis, iso-2022-jp, euc-jp-foobar", 0.0, 0}, + {"euc-jp", "shift_jis, iso-2022-jp, euc-jp-foobar, *", 1.0, 4}, + {"euc-jp", "shift_jis, iso-2022-jp, euc-jp-foobar, *;q=0.543", 0.543, 4}, + {"euc-jp", "shift_jis, iso-2022-jp, euc-jp-foobar, *;q=0.0", 0.0, 4}, + {"euc-jp", "shift_jis, iso-2022-jp, *;q=0.0, euc-jp-foobar, *;q=0.0", 0.0, 3}, + {"euc-jp", "shift_jis, iso-2022-jp, *;q=0.0, euc-jp-foobar, *;q=0.5", 0.5, 5}, + {"euc-jp", "shift_jis, iso-2022-jp, *;q=0.5, euc-jp-foobar, *;q=0.0", 0.5, 3}, + {"euc-jp", "shift_jis, iso-2022-jp, *;q=0.5, euc-jp-foobar, *, *;q=0.0", 1.0, 5}, + {"euc-jp", "shift_jis, euc-jp;hi-there;q=0.5, iso-2022-jp", 0.5, 2}, + {"euc-jp", "shift_jis, euc-jp;hi-there;q= 0.5, iso-2022-jp", 0.5, 2}, + {"euc-jp", "shift_jis, euc-jp;hi-there;q = 0.5, iso-2022-jp", 0.5, 2}, + {"euc-jp", "shift_jis, euc-jp;hi-there ; q = 0.5, iso-2022-jp", 0.5, 2}, + {"euc-jp", "shift_jis, euc-jp;hi-there ;; q = 0.5, iso-2022-jp", 0.5, 2}, + {"euc-jp", "shift_jis, euc-jp;hi-there ;; Q = 0.5, iso-2022-jp", 0.5, 2}, + {nullptr, nullptr, 0.0, 0}, + }; + + int i, I; + float Q; + + for (i = 0; test_cases[i].accept_charset; i++) { + StrList acpt_lang_list(false); + HttpCompat::parse_comma_list(&acpt_lang_list, test_cases[i].accept_charset, + static_cast(strlen(test_cases[i].accept_charset))); + + Q = HttpCompat::match_accept_charset(test_cases[i].content_charset, static_cast(strlen(test_cases[i].content_charset)), + &acpt_lang_list, &I); + + if ((Q != test_cases[i].Q) || (I != test_cases[i].I)) { + std::printf("FAILED: (#%d) got { Q = %.3f; I = %d; }, expected { Q = %.3f; I = %d; }, from matching\n '%s' against '%s'\n", + i, Q, I, test_cases[i].Q, test_cases[i].I, test_cases[i].content_charset, test_cases[i].accept_charset); + CHECK(false); + } + } + } + + SECTION("Test comma vals") + { + static struct { + const char *value; + int value_count; + struct { + int offset; + int len; + } pieces[4]; + } tests[] = { + {",", 2, {{0, 0}, {1, 0}, {-1, 0}, {-1, 0}}}, + {"", 1, {{0, 0}, {-1, 0}, {-1, 0}, {-1, 0}}}, + {" ", 1, {{0, 0}, {-1, 0}, {-1, 0}, {-1, 0}}}, + {", ", 2, {{0, 0}, {1, 0}, {-1, 0}, {-1, 0}}}, + {",,", 3, {{0, 0}, {1, 0}, {2, 0}, {-1, 0}}}, + {" ,", 2, {{0, 0}, {2, 0}, {-1, 0}, {-1, 0}}}, + {" , ", 2, {{0, 0}, {2, 0}, {-1, 0}, {-1, 0}}}, + {"a, ", 2, {{0, 1}, {2, 0}, {-1, 0}, {-1, 0}}}, + {" a, ", 2, {{1, 1}, {3, 0}, {-1, 0}, {-1, 0}}}, + {" ,a", 2, {{0, 0}, {2, 1}, {-1, 0}, {-1, 0}}}, + {" , a", 2, {{0, 0}, {3, 1}, {-1, 0}, {-1, 0}}}, + {"a,a", 2, {{0, 1}, {2, 1}, {-1, 0}, {-1, 0}}}, + {"foo", 1, {{0, 3}, {-1, 0}, {-1, 0}, {-1, 0}}}, + {"foo,", 2, {{0, 3}, {4, 0}, {-1, 0}, {-1, 0}}}, + {"foo, ", 2, {{0, 3}, {4, 0}, {-1, 0}, {-1, 0}}}, + {"foo, bar", 2, {{0, 3}, {5, 3}, {-1, 0}, {-1, 0}}}, + {"foo, bar,", 3, {{0, 3}, {5, 3}, {9, 0}, {-1, 0}}}, + {"foo, bar, ", 3, {{0, 3}, {5, 3}, {9, 0}, {-1, 0}}}, + { + ",foo,bar,", + 4, + {{0, 0}, {1, 3}, {5, 3}, {9, 0}}, + }, + }; + + HTTPHdr hdr; + char field_name[32]; + int i, j, len, ntests, ncommavals; + + ntests = sizeof(tests) / sizeof(tests[0]); + + hdr.create(HTTP_TYPE_REQUEST); + + for (i = 0; i < ntests; i++) { + snprintf(field_name, sizeof(field_name), "Test%d", i); + + MIMEField *f = hdr.field_create(field_name, static_cast(strlen(field_name))); + REQUIRE(f->m_ptr_value == nullptr); + + hdr.field_attach(f); + REQUIRE(f->m_ptr_value == nullptr); + + hdr.field_value_set(f, tests[i].value, strlen(tests[i].value)); + REQUIRE(f->m_ptr_value != tests[i].value); // should be copied + REQUIRE(f->m_len_value == strlen(tests[i].value)); + REQUIRE(memcmp(f->m_ptr_value, tests[i].value, f->m_len_value) == 0); + + ncommavals = mime_field_value_get_comma_val_count(f); + if (ncommavals != tests[i].value_count) { + std::printf("FAILED: test #%d (field value '%s') expected val count %d, got %d\n", i + 1, tests[i].value, + tests[i].value_count, ncommavals); + CHECK(false); + } + + for (j = 0; j < tests[i].value_count; j++) { + const char *val = mime_field_value_get_comma_val(f, &len, j); + int offset = ((val == nullptr) ? -1 : (val - f->m_ptr_value)); + + if ((offset != tests[i].pieces[j].offset) || (len != tests[i].pieces[j].len)) { + std::printf( + "FAILED: test #%d (field value '%s', commaval idx %d) expected [offset %d, len %d], got [offset %d, len %d]\n", i + 1, + tests[i].value, j, tests[i].pieces[j].offset, tests[i].pieces[j].len, offset, len); + CHECK(false); + } + } + } + + hdr.destroy(); + } + + SECTION("Test set comma vals") + { + static struct { + const char *old_raw; + int idx; + const char *slice; + const char *new_raw; + } tests[] = { + {"a,b,c", 0, "fred", "fred, b, c"}, + {"a,b,c", 1, "fred", "a, fred, c"}, + {"a,b,c", 2, "fred", "a, b, fred"}, + {"a,b,c", 3, "fred", "a,b,c"}, + {"", 0, "", ""}, + {"", 0, "foo", "foo"}, + {"", 1, "foo", ""}, + {" ", 0, "", ""}, + {" ", 0, "foo", "foo"}, + {" ", 1, "foo", " "}, + {",", 0, "foo", "foo, "}, + {",", 1, "foo", ", foo"}, + {",,", 0, "foo", "foo, , "}, + {",,", 1, "foo", ", foo, "}, + {",,", 2, "foo", ", , foo"}, + {"foo", 0, "abc", "abc"}, + {"foo", 1, "abc", "foo"}, + {"foo", 0, "abc,", "abc,"}, + {"foo", 0, ",abc", ",abc"}, + {",,", 1, ",,,", ", ,,,, "}, + {" a , b , c", 0, "fred", "fred, b, c"}, + {" a , b , c", 1, "fred", "a, fred, c"}, + {" a , b , c", 2, "fred", "a, b, fred"}, + {" a , b , c", 3, "fred", " a , b , c"}, + {" a , b ", 0, "fred", "fred, b"}, + {" a , b ", 1, "fred", "a, fred"}, + {" a , b ", 1, "fred", "a, fred"}, + {" a ,b ", 1, "fred", "a, fred"}, + {"a, , , , e, , g,", 0, "fred", "fred, , , , e, , g, "}, + {"a, , , , e, , g,", 1, "fred", "a, fred, , , e, , g, "}, + {"a, , , , e, , g,", 2, "fred", "a, , fred, , e, , g, "}, + {"a, , , , e, , g,", 5, "fred", "a, , , , e, fred, g, "}, + {"a, , , , e, , g,", 7, "fred", "a, , , , e, , g, fred"}, + {"a, , , , e, , g,", 8, "fred", "a, , , , e, , g,"}, + {"a, \"boo,foo\", c", 0, "wawa", "wawa, \"boo,foo\", c"}, + {"a, \"boo,foo\", c", 1, "wawa", "a, wawa, c"}, + {"a, \"boo,foo\", c", 2, "wawa", "a, \"boo,foo\", wawa"}, + }; + + HTTPHdr hdr; + char field_name[32]; + int i, ntests; + + ntests = sizeof(tests) / sizeof(tests[0]); + + hdr.create(HTTP_TYPE_REQUEST); + + for (i = 0; i < ntests; i++) { + snprintf(field_name, sizeof(field_name), "Test%d", i); + + MIMEField *f = hdr.field_create(field_name, static_cast(strlen(field_name))); + hdr.field_value_set(f, tests[i].old_raw, strlen(tests[i].old_raw)); + mime_field_value_set_comma_val(hdr.m_heap, hdr.m_mime, f, tests[i].idx, tests[i].slice, strlen(tests[i].slice)); + REQUIRE(f->m_ptr_value != nullptr); + + if ((f->m_len_value != strlen(tests[i].new_raw)) || (memcmp(f->m_ptr_value, tests[i].new_raw, f->m_len_value) != 0)) { + std::printf("FAILED: test #%d (setting idx %d of '%s' to '%s') expected '%s' len %d, got '%.*s' len %d\n", i + 1, + tests[i].idx, tests[i].old_raw, tests[i].slice, tests[i].new_raw, static_cast(strlen(tests[i].new_raw)), + f->m_len_value, f->m_ptr_value, f->m_len_value); + CHECK(false); + } + } + + hdr.destroy(); + } + + SECTION("Test delete comma vals") + { + // TEST NOT IMPLEMENTED + } + + SECTION("Test extend comma vals") + { + // TEST NOT IMPLEMENTED + } + + SECTION("Test insert comma vals") + { + // TEST NOT IMPLEMENTED + } + + SECTION("Test parse comma list") + { + static struct { + const char *value; + int count; + struct { + int offset; + int len; + } pieces[3]; + } tests[] = { + {"", 1, {{0, 0}, {-1, 0}, {-1, 0}}}, + {",", 2, {{0, 0}, {1, 0}, {-1, 0}}}, + {" ,", 2, {{0, 0}, {2, 0}, {-1, 0}}}, + {", ", 2, {{0, 0}, {1, 0}, {-1, 0}}}, + {" , ", 2, {{0, 0}, {2, 0}, {-1, 0}}}, + {"abc,", 2, {{0, 3}, {4, 0}, {-1, 0}}}, + {"abc, ", 2, {{0, 3}, {4, 0}, {-1, 0}}}, + {"", 1, {{0, 0}, {-1, 0}, {-1, 0}}}, + {" ", 1, {{0, 0}, {-1, 0}, {-1, 0}}}, + {" ", 1, {{0, 0}, {-1, 0}, {-1, 0}}}, + {"a", 1, {{0, 1}, {-1, 0}, {-1, 0}}}, + {" a", 1, {{1, 1}, {-1, 0}, {-1, 0}}}, + {" a ", 1, {{2, 1}, {-1, 0}, {-1, 0}}}, + {"abc,defg", 2, {{0, 3}, {4, 4}, {-1, 0}}}, + {" abc,defg", 2, {{1, 3}, {5, 4}, {-1, 0}}}, + {" abc, defg", 2, {{1, 3}, {6, 4}, {-1, 0}}}, + {" abc , defg", 2, {{1, 3}, {7, 4}, {-1, 0}}}, + {" abc , defg ", 2, {{1, 3}, {7, 4}, {-1, 0}}}, + {" abc , defg, ", 3, {{1, 3}, {7, 4}, {12, 0}}}, + {" abc , defg ,", 3, {{1, 3}, {7, 4}, {13, 0}}}, + {", abc , defg ", 3, {{0, 0}, {2, 3}, {8, 4}}}, + {" ,abc , defg ", 3, {{0, 0}, {2, 3}, {8, 4}}}, + {"a,b", 2, {{0, 1}, {2, 1}, {-1, 0}}}, + {"a,,b", 3, {{0, 1}, {2, 0}, {3, 1}}}, + {"a, ,b", 3, {{0, 1}, {2, 0}, {4, 1}}}, + {"a ,,b", 3, {{0, 1}, {3, 0}, {4, 1}}}, + {",", 2, {{0, 0}, {1, 0}, {-1, 0}}}, + {" ,", 2, {{0, 0}, {2, 0}, {-1, 0}}}, + {", ", 2, {{0, 0}, {1, 0}, {-1, 0}}}, + {" , ", 2, {{0, 0}, {2, 0}, {-1, 0}}}, + {"a,b,", 3, {{0, 1}, {2, 1}, {4, 0}}}, + {"a,b, ", 3, {{0, 1}, {2, 1}, {4, 0}}}, + {"a,b, ", 3, {{0, 1}, {2, 1}, {4, 0}}}, + {"a,b, c", 3, {{0, 1}, {2, 1}, {6, 1}}}, + {"a,b, c ", 3, {{0, 1}, {2, 1}, {6, 1}}}, + {"a,\"b,c\",d", 3, {{0, 1}, {3, 3}, {8, 1}}}, + }; + + int i, j, ntests, offset; + + offset = 0; + ntests = sizeof(tests) / sizeof(tests[0]); + + for (i = 0; i < ntests; i++) { + StrList list(false); + HttpCompat::parse_comma_list(&list, tests[i].value); + if (list.count != tests[i].count) { + std::printf("FAILED: test #%d (string '%s') expected list count %d, got %d\n", i + 1, tests[i].value, tests[i].count, + list.count); + CHECK(false); + } + + for (j = 0; j < tests[i].count; j++) { + Str *cell = list.get_idx(j); + if (cell != nullptr) { + offset = cell->str - tests[i].value; + } + + if (tests[i].pieces[j].offset == -1) // should not have a piece + { + if (cell != nullptr) { + std::printf("FAILED: test #%d (string '%s', idx %d) expected NULL piece, got [offset %d len %d]\n", i + 1, + tests[i].value, j, offset, static_cast(cell->len)); + CHECK(false); + } + } else // should have a piece + { + if (cell == nullptr) { + std::printf("FAILED: test #%d (string '%s', idx %d) expected [offset %d len %d], got NULL piece\n", i + 1, + tests[i].value, j, tests[i].pieces[j].offset, tests[i].pieces[j].len); + CHECK(false); + } else if ((offset != tests[i].pieces[j].offset) || (cell->len != static_cast(tests[i].pieces[j].len))) { + std::printf("FAILED: test #%d (string '%s', idx %d) expected [offset %d len %d], got [offset %d len %d]\n", i + 1, + tests[i].value, j, tests[i].pieces[j].offset, tests[i].pieces[j].len, offset, static_cast(cell->len)); + CHECK(false); + } + } + } + } + } +} diff --git a/proxy/hdrs/unit_tests/test_URL.cc b/proxy/hdrs/unit_tests/test_URL.cc new file mode 100644 index 00000000000..975bd5ea094 --- /dev/null +++ b/proxy/hdrs/unit_tests/test_URL.cc @@ -0,0 +1,468 @@ +/** @file + + Catch-based unit tests for URL + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. + See the NOTICE file distributed with this work for additional information regarding copyright + ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance with the License. You may obtain a + copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software distributed under the License + is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + or implied. See the License for the specific language governing permissions and limitations under + the License. + */ + +#include + +#include "catch.hpp" + +#include "URL.h" + +TEST_CASE("ValidateURL", "[proxy][validurl]") +{ + static const struct { + const char *const text; + bool valid; + } http_validate_hdr_field_test_case[] = {{"yahoo", true}, + {"yahoo.com", true}, + {"yahoo.wow.com", true}, + {"yahoo.wow.much.amaze.com", true}, + {"209.131.52.50", true}, + {"192.168.0.1", true}, + {"localhost", true}, + {"3ffe:1900:4545:3:200:f8ff:fe21:67cf", true}, + {"fe80:0:0:0:200:f8ff:fe21:67cf", true}, + {"fe80::200:f8ff:fe21:67cf", true}, + {"", false}, // Sample host header XSS attack + {"jlads;f8-9349*(D&F*D(234jD*(FSD*(VKLJ#(*$@()#$)))))", false}, + {"\"\t\n", false}, + {"!@#$%^ &*(*&^%$#@#$%^&*(*&^%$#))", false}, + {":):(:O!!!!!!", false}}; + for (auto i : http_validate_hdr_field_test_case) { + const char *const txt = i.text; + if (validate_host_name({txt}) != i.valid) { + std::printf("Validation of FQDN (host) header: \"%s\", expected %s, but not\n", txt, (i.valid ? "true" : "false")); + CHECK(false); + } + } +} + +namespace UrlImpl +{ +bool url_is_strictly_compliant(const char *start, const char *end); +} +using namespace UrlImpl; + +TEST_CASE("ParseRulesStrictURI", "[proxy][parseuri]") +{ + const struct { + const char *const uri; + bool valid; + } http_strict_uri_parsing_test_case[] = {{"/home", true}, + {"/path/data?key=value#id", true}, + {"/ABCDEFGHIJKLMNOPQRSTUVWXYZ", true}, + {"/abcdefghijklmnopqrstuvwxyz", true}, + {"/0123456789", true}, + {":/?#[]@", true}, + {"!$&'()*+,;=", true}, + {"-._~", true}, + {"%", true}, + {"\n", false}, + {"\"", false}, + {"<", false}, + {">", false}, + {"\\", false}, + {"^", false}, + {"`", false}, + {"{", false}, + {"|", false}, + {"}", false}, + {"é", false}}; + + for (auto i : http_strict_uri_parsing_test_case) { + const char *const uri = i.uri; + if (url_is_strictly_compliant(uri, uri + strlen(uri)) != i.valid) { + std::printf("Strictly parse URI: \"%s\", expected %s, but not\n", uri, (i.valid ? "true" : "false")); + CHECK(false); + } + } +} + +struct url_parse_test_case { + const std::string input_uri; + const std::string expected_printed_url; + const bool verify_host_characters; + const std::string expected_printed_url_regex; + const bool is_valid; + const bool is_valid_regex; +}; + +constexpr bool IS_VALID = true; +constexpr bool VERIFY_HOST_CHARACTERS = true; + +// clang-format off +std::vector url_parse_test_cases = { + { + // The following scheme-only URI is technically valid per the spec, but we + // have historically returned this as invalid and I'm not comfortable + // changing it in case something depends upon this behavior. Besides, a + // scheme-only URI is probably not helpful to us nor something likely + // Traffic Server will see. + "http://", + "", + VERIFY_HOST_CHARACTERS, + "", + !IS_VALID, + !IS_VALID + }, + { + "https:///", + "https:///", + VERIFY_HOST_CHARACTERS, + "https:///", + IS_VALID, + IS_VALID + }, + { + // RFC 3986 section-3: When authority is not present, the path cannot begin + // with two slash characters ("//"). The parse_regex, though, is more + // forgiving. + "https:////", + "", + VERIFY_HOST_CHARACTERS, + "https:////", + !IS_VALID, + IS_VALID + }, + { + // By convention, our url_print() function adds a path of '/' at the end of + // URLs that have no path, query, or fragment after the authority. + "mailto:Test.User@example.com", + "mailto:Test.User@example.com/", + VERIFY_HOST_CHARACTERS, + "mailto:Test.User@example.com/", + IS_VALID, + IS_VALID + }, + { + "mailto:Test.User@example.com:25", + "mailto:Test.User@example.com:25/", + VERIFY_HOST_CHARACTERS, + "mailto:Test.User@example.com:25/", + IS_VALID, + IS_VALID + }, + { + "https://www.example.com", + "https://www.example.com/", + VERIFY_HOST_CHARACTERS, + "https://www.example.com/", + IS_VALID, + IS_VALID + }, + { + "https://www.example.com/", + "https://www.example.com/", + VERIFY_HOST_CHARACTERS, + "https://www.example.com/", + IS_VALID, + IS_VALID + }, + { + "https://www.example.com//", + "https://www.example.com//", + VERIFY_HOST_CHARACTERS, + "https://www.example.com//", + IS_VALID, + IS_VALID + }, + { + "https://127.0.0.1", + "https://127.0.0.1/", + VERIFY_HOST_CHARACTERS, + "https://127.0.0.1/", + IS_VALID, + IS_VALID + }, + { + "https://[::1]", + "https://[::1]/", + VERIFY_HOST_CHARACTERS, + "https://[::1]/", + IS_VALID, + IS_VALID + }, + { + "https://127.0.0.1/", + "https://127.0.0.1/", + VERIFY_HOST_CHARACTERS, + "https://127.0.0.1/", + IS_VALID, + IS_VALID + }, + { + "https://www.example.com:8888", + "https://www.example.com:8888/", + VERIFY_HOST_CHARACTERS, + "https://www.example.com:8888/", + IS_VALID, + IS_VALID + }, + { + "https://www.example.com:8888/", + "https://www.example.com:8888/", + VERIFY_HOST_CHARACTERS, + "https://www.example.com:8888/", + IS_VALID, + IS_VALID + }, + { + "https://www.example.com/a/path", + "https://www.example.com/a/path", + VERIFY_HOST_CHARACTERS, + "https://www.example.com/a/path", + IS_VALID, + IS_VALID + }, + { + "https://www.example.com//a/path", + "https://www.example.com//a/path", + VERIFY_HOST_CHARACTERS, + "https://www.example.com//a/path", + IS_VALID, + IS_VALID + }, + + // Technically a trailing '?' with an empty query string is valid, but we + // drop the '?'. The parse_regex, however, makes no distinction between + // query, fragment, and path components so it does not cut it out. + { + "https://www.example.com/a/path?", + "https://www.example.com/a/path", + VERIFY_HOST_CHARACTERS, + "https://www.example.com/a/path?", + IS_VALID, + IS_VALID}, + { + "https://www.example.com/a/path?name=value", + "https://www.example.com/a/path?name=value", + VERIFY_HOST_CHARACTERS, + "https://www.example.com/a/path?name=value", + IS_VALID, + IS_VALID + }, + { + "https://www.example.com/a/path?name=/a/path/value", + "https://www.example.com/a/path?name=/a/path/value", + VERIFY_HOST_CHARACTERS, + "https://www.example.com/a/path?name=/a/path/value", + IS_VALID, + IS_VALID + }, + { + "https://www.example.com/a/path?name=/a/path/value;some=other_value", + "https://www.example.com/a/path?name=/a/path/value;some=other_value", + VERIFY_HOST_CHARACTERS, + "https://www.example.com/a/path?name=/a/path/value;some=other_value", + IS_VALID, + IS_VALID + }, + { + "https://www.example.com/a/path?name=/a/path/value;some=other_value/", + "https://www.example.com/a/path?name=/a/path/value;some=other_value/", + VERIFY_HOST_CHARACTERS, + "https://www.example.com/a/path?name=/a/path/value;some=other_value/", + IS_VALID, + IS_VALID + }, + + // Again, URL::parse drops a final '?'. + { + "https://www.example.com?", + "https://www.example.com", + VERIFY_HOST_CHARACTERS, + "https://www.example.com?/", + IS_VALID, + IS_VALID + }, + { + "https://www.example.com?name=value", + "https://www.example.com?name=value", + VERIFY_HOST_CHARACTERS, + "https://www.example.com?name=value/", + IS_VALID, + IS_VALID + }, + { + "https://www.example.com?name=value/", + "https://www.example.com?name=value/", + VERIFY_HOST_CHARACTERS, + "https://www.example.com?name=value/", + IS_VALID, + IS_VALID + }, + + // URL::parse also drops the final '#'. + { + "https://www.example.com#", + "https://www.example.com", + VERIFY_HOST_CHARACTERS, + "https://www.example.com#/", + IS_VALID, + IS_VALID + }, + { + "https://www.example.com#some=value", + "https://www.example.com#some=value", + VERIFY_HOST_CHARACTERS, + "https://www.example.com#some=value/", + IS_VALID, + IS_VALID}, + { + "https://www.example.com/a/path#", + "https://www.example.com/a/path", + VERIFY_HOST_CHARACTERS, + "https://www.example.com/a/path#", + IS_VALID, + IS_VALID + }, + { + "https://www.example.com/a/path#some=value", + "https://www.example.com/a/path#some=value", + VERIFY_HOST_CHARACTERS, + "https://www.example.com/a/path#some=value", + IS_VALID, + IS_VALID + }, + { + // Note that this final '?' is not for a query parameter but is a part of + // the fragment. + "https://www.example.com/a/path#some=value?", + "https://www.example.com/a/path#some=value?", + VERIFY_HOST_CHARACTERS, + "https://www.example.com/a/path#some=value?", + IS_VALID, + IS_VALID + }, + { + "https://www.example.com/a/path#some=value?with_question", + "https://www.example.com/a/path#some=value?with_question", + VERIFY_HOST_CHARACTERS, + "https://www.example.com/a/path#some=value?with_question", + IS_VALID, + IS_VALID + }, + { + "https://www.example.com/a/path?name=value?_with_question#some=value?with_question/", + "https://www.example.com/a/path?name=value?_with_question#some=value?with_question/", + VERIFY_HOST_CHARACTERS, + "https://www.example.com/a/path?name=value?_with_question#some=value?with_question/", + IS_VALID, + IS_VALID + }, + + // The following are some examples of strings we expect from regex_map in + // remap.config. The "From" portion, which are regular expressions, are + // often not parsible by URL::parse but are by URL::parse_regex, which is the + // purpose of its existence. + { + R"(http://(.*)?reactivate\.mail\.yahoo\.com/)", + "", + VERIFY_HOST_CHARACTERS, + R"(http://(.*)?reactivate\.mail\.yahoo\.com/)", + !IS_VALID, + IS_VALID + }, + { + // The following is an example of a "To" URL in a regex_map line. We'll + // first verify that the '$' is flagged as invalid for a host in this case. + "http://$1reactivate.real.mail.yahoo.com/", + "http://$1reactivate.real.mail.yahoo.com/", + VERIFY_HOST_CHARACTERS, + "http://$1reactivate.real.mail.yahoo.com/", + !IS_VALID, + IS_VALID + }, + { + // Same as above, but this time we pass in !VERIFY_HOST_CHARACTERS. This is + // how RemapConfig will call this parse() function. + "http://$1reactivate.real.mail.yahoo.com/", + "http://$1reactivate.real.mail.yahoo.com/", + !VERIFY_HOST_CHARACTERS, + "http://$1reactivate.real.mail.yahoo.com/", + IS_VALID, + IS_VALID + } +}; +// clang-format on + +constexpr bool URL_PARSE = true; +constexpr bool URL_PARSE_REGEX = false; + +/** Test the specified url.parse function. + * + * URL::parse and URL::parse_regex should behave the same. This function + * performs the same behavior for each. + * + * @param[in] test_case The test case specification to run. + * + * @param[in] parse_function Whether to run parse() or + * parse_regex(). + */ +void +test_parse(url_parse_test_case const &test_case, bool parse_function) +{ + URL url; + HdrHeap *heap = new_HdrHeap(); + url.create(heap); + ParseResult result = PARSE_RESULT_OK; + if (parse_function == URL_PARSE) { + if (test_case.verify_host_characters) { + result = url.parse(test_case.input_uri); + } else { + result = url.parse_no_host_check(test_case.input_uri); + } + } else if (parse_function == URL_PARSE_REGEX) { + result = url.parse_regex(test_case.input_uri); + } + bool expected_is_valid = test_case.is_valid; + if (parse_function == URL_PARSE_REGEX) { + expected_is_valid = test_case.is_valid_regex; + } + if (expected_is_valid && result != PARSE_RESULT_DONE) { + std::printf("Parse URI: \"%s\", expected it to be valid but it was parsed invalid (%d)\n", test_case.input_uri.c_str(), result); + CHECK(false); + } else if (!expected_is_valid && result != PARSE_RESULT_ERROR) { + std::printf("Parse URI: \"%s\", expected it to be invalid but it was parsed valid (%d)\n", test_case.input_uri.c_str(), result); + CHECK(false); + } + if (result == PARSE_RESULT_DONE) { + char buf[1024]; + int index = 0; + int offset = 0; + url.print(buf, sizeof(buf), &index, &offset); + std::string printed_url{buf, static_cast(index)}; + if (parse_function == URL_PARSE) { + CHECK(test_case.expected_printed_url == printed_url); + CHECK(test_case.expected_printed_url.size() == printed_url.size()); + } else if (parse_function == URL_PARSE_REGEX) { + CHECK(test_case.expected_printed_url_regex == printed_url); + CHECK(test_case.expected_printed_url_regex.size() == printed_url.size()); + } + } + heap->destroy(); +} + +TEST_CASE("UrlParse", "[proxy][parseurl]") +{ + for (auto const &test_case : url_parse_test_cases) { + test_parse(test_case, URL_PARSE); + test_parse(test_case, URL_PARSE_REGEX); + } +} diff --git a/proxy/hdrs/unit_tests/test_mime.cc b/proxy/hdrs/unit_tests/test_mime.cc new file mode 100644 index 00000000000..0ad70384ac9 --- /dev/null +++ b/proxy/hdrs/unit_tests/test_mime.cc @@ -0,0 +1,249 @@ +/** @file + + A brief file description + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#include + +#include "catch.hpp" + +#include "I_EventSystem.h" +#include "MIME.h" + +TEST_CASE("Mime", "[proxy][mime]") +{ + MIMEField *field; + MIMEHdr hdr; + hdr.create(NULL); + + hdr.field_create("Test1", 5); + hdr.field_create("Test2", 5); + hdr.field_create("Test3", 5); + hdr.field_create("Test4", 5); + field = hdr.field_create("Test5", 5); + + if (!hdr.m_mime->m_first_fblock.contains(field)) { + std::printf("The field block doesn't contain the field but it should\n"); + CHECK(false); + } + if (hdr.m_mime->m_first_fblock.contains(field + (1L << 32))) { + std::printf("The field block contains the field but it shouldn't\n"); + CHECK(false); + } + + int slot_num = mime_hdr_field_slotnum(hdr.m_mime, field); + if (slot_num != 4) { + std::printf("Slot number is %d but should be 4\n", slot_num); + CHECK(false); + } + + slot_num = mime_hdr_field_slotnum(hdr.m_mime, field + (1L << 32)); + if (slot_num != -1) { + std::printf("Slot number is %d but should be -1\n", slot_num); + CHECK(false); + } + + hdr.destroy(); +} + +TEST_CASE("MimeGetHostPortValues", "[proxy][mimeport]") +{ + MIMEHdr hdr; + hdr.create(NULL); + + const char *header_value; + const char *host; + int host_len; + const char *port; + int port_len; + + header_value = "host"; + hdr.value_set("Host", 4, header_value, strlen(header_value)); + hdr.get_host_port_values(&host, &host_len, &port, &port_len); + if (host_len != 4) { + std::printf("host length doesn't match\n"); + CHECK(false); + } + if (strncmp(host, "host", host_len) != 0) { + std::printf("host string doesn't match\n"); + CHECK(false); + } + if (port_len != 0) { + std::printf("port length doesn't match\n"); + CHECK(false); + } + if (port != nullptr) { + std::printf("port string doesn't match\n"); + CHECK(false); + } + + header_value = "host:"; + hdr.value_set("Host", 4, header_value, strlen(header_value)); + hdr.get_host_port_values(&host, &host_len, &port, &port_len); + if (host_len != 4) { + std::printf("host length doesn't match\n"); + CHECK(false); + } + if (strncmp(host, "host", host_len) != 0) { + std::printf("host string doesn't match\n"); + CHECK(false); + } + if (port_len != 0) { + std::printf("port length doesn't match\n"); + CHECK(false); + } + if (port != nullptr) { + std::printf("port string doesn't match\n"); + CHECK(false); + } + + header_value = "[host]"; + hdr.value_set("Host", 4, header_value, strlen(header_value)); + hdr.get_host_port_values(&host, &host_len, &port, &port_len); + if (host_len != 6) { + std::printf("host length doesn't match\n"); + CHECK(false); + } + if (strncmp(host, "[host]", host_len) != 0) { + std::printf("host string doesn't match\n"); + CHECK(false); + } + if (port_len != 0) { + std::printf("port length doesn't match\n"); + CHECK(false); + } + if (port != nullptr) { + std::printf("port string doesn't match\n"); + CHECK(false); + } + + header_value = "host:port"; + hdr.value_set("Host", 4, header_value, strlen(header_value)); + hdr.get_host_port_values(&host, &host_len, &port, &port_len); + if (host_len != 4) { + std::printf("host length doesn't match\n"); + CHECK(false); + } + if (strncmp(host, "host", host_len) != 0) { + std::printf("host string doesn't match\n"); + CHECK(false); + } + if (port_len != 4) { + std::printf("port length doesn't match\n"); + CHECK(false); + } + if (strncmp(port, "port", port_len) != 0) { + std::printf("port string doesn't match\n"); + CHECK(false); + } + + header_value = "[host]:port"; + hdr.value_set("Host", 4, header_value, strlen(header_value)); + hdr.get_host_port_values(&host, &host_len, &port, &port_len); + if (host_len != 6) { + std::printf("host length doesn't match\n"); + CHECK(false); + } + if (strncmp(host, "[host]", host_len) != 0) { + std::printf("host string doesn't match\n"); + CHECK(false); + } + if (port_len != 4) { + std::printf("port length doesn't match\n"); + CHECK(false); + } + if (strncmp(port, "port", port_len) != 0) { + std::printf("port string doesn't match\n"); + CHECK(false); + } + + header_value = "[host]:"; + hdr.value_set("Host", 4, header_value, strlen(header_value)); + hdr.get_host_port_values(&host, &host_len, &port, &port_len); + if (host_len != 6) { + std::printf("host length doesn't match\n"); + CHECK(false); + } + if (strncmp(host, "[host]", host_len) != 0) { + std::printf("host string doesn't match\n"); + CHECK(false); + } + if (port_len != 0) { + std::printf("port length doesn't match\n"); + CHECK(false); + } + if (port != nullptr) { + std::printf("port string doesn't match\n"); + CHECK(false); + } + + hdr.destroy(); +} + +TEST_CASE("MimeParsers", "[proxy][mimeparsers]") +{ + const char *end; + int value; + + static const std::vector> tests = {{"0", 0}, + {"1234", 1234}, + {"-1234", -1234}, + {"2147483647", 2147483647}, + {"-2147483648", 2147483648}, + {"2147483648", INT_MAX}, + {"-2147483649", INT_MIN}, + {"2147483647", INT_MAX}, + {"-2147483648", INT_MIN}, + {"999999999999", INT_MAX}, + {"-999999999999", INT_MIN}}; + + for (const auto &it : tests) { + auto [buf, val] = it; + + end = buf + strlen(buf); + if (mime_parse_int(buf, end) != val) { + std::printf("Failed mime_parse_int\n"); + CHECK(false); + } + if (!mime_parse_integer(buf, end, &value)) { + std::printf("Failed mime_parse_integer call\n"); + CHECK(false); + } else if (value != val) { + std::printf("Failed mime_parse_integer value\n"); + CHECK(false); + } + } + + // Also check the date parser, which relies heavily on the mime_parse_integer() function + const char *date1 = "Sun, 05 Dec 1999 08:49:37 GMT"; + const char *date2 = "Sunday, 05-Dec-1999 08:49:37 GMT"; + + int d1 = mime_parse_date(date1, date1 + strlen(date1)); + int d2 = mime_parse_date(date2, date2 + strlen(date2)); + + if (d1 != d2) { + std::printf("Failed mime_parse_date\n"); + CHECK(false); + } + + std::printf("Date1: %d\n", d1); + std::printf("Date2: %d\n", d2); +} diff --git a/proxy/http/Http1ClientSession.cc b/proxy/http/Http1ClientSession.cc index de6c81e9044..9177c0e4da3 100644 --- a/proxy/http/Http1ClientSession.cc +++ b/proxy/http/Http1ClientSession.cc @@ -74,7 +74,6 @@ Http1ClientSession::destroy() in_destroy = true; HttpSsnDebug("[%" PRId64 "] session destroy", con_id); - ink_release_assert(!client_vc); ink_assert(read_buffer); ink_release_assert(transact_count == released_transactions); do_api_callout(TS_HTTP_SSN_CLOSE_HOOK); @@ -88,7 +87,14 @@ Http1ClientSession::release_transaction() { released_transactions++; if (transact_count == released_transactions) { - destroy(); + // Make sure we previously called release() or do_io_close() on the session + ink_release_assert(read_state != HCS_INIT); + if (read_state == HCS_ACTIVE_READER) { + // (in)active timeout + do_io_close(HTTP_ERRNO); + } else { + destroy(); + } } } @@ -112,15 +118,12 @@ Http1ClientSession::free() conn_decrease = false; } - // Clean up the write VIO in case of inactivity timeout - this->do_io_write(nullptr, 0, nullptr); - // Free the transaction resources this->trans.super_type::destroy(); - if (client_vc) { - client_vc->do_io_close(); - client_vc = nullptr; + if (_vc) { + _vc->do_io_close(); + _vc = nullptr; } super::free(); @@ -131,14 +134,20 @@ void Http1ClientSession::new_connection(NetVConnection *new_vc, MIOBuffer *iobuf, IOBufferReader *reader) { ink_assert(new_vc != nullptr); - ink_assert(client_vc == nullptr); - client_vc = new_vc; + ink_assert(_vc == nullptr); + _vc = new_vc; magic = HTTP_CS_MAGIC_ALIVE; mutex = new_vc->mutex; trans.mutex = mutex; // Share this mutex with the transaction ssn_start_time = Thread::get_hrtime(); in_destroy = false; + SSLNetVConnection *ssl_vc = dynamic_cast(new_vc); + if (ssl_vc != nullptr) { + read_from_early_data = ssl_vc->read_from_early_data; + Debug("ssl_early_data", "read_from_early_data = %" PRId64, read_from_early_data); + } + MUTEX_TRY_LOCK(lock, mutex, this_ethread()); ink_assert(lock.is_locked()); @@ -183,11 +192,13 @@ Http1ClientSession::new_connection(NetVConnection *new_vc, MIOBuffer *iobuf, IOB HttpSsnDebug("[%" PRId64 "] session born, netvc %p", con_id, new_vc); - client_vc->set_tcp_congestion_control(CLIENT_SIDE); + _vc->set_tcp_congestion_control(CLIENT_SIDE); read_buffer = iobuf ? iobuf : new_MIOBuffer(HTTP_HEADER_BUFFER_SIZE_INDEX); - sm_reader = reader ? reader : read_buffer->alloc_reader(); - trans.set_reader(sm_reader); + _reader = reader ? reader : read_buffer->alloc_reader(); + trans.set_reader(_reader); + + _handle_if_ssl(new_vc); // INKqa11186: Use a local pointer to the mutex as // when we return from do_api_callout, the ClientSession may @@ -200,28 +211,6 @@ Http1ClientSession::new_connection(NetVConnection *new_vc, MIOBuffer *iobuf, IOB lmutex.clear(); } -VIO * -Http1ClientSession::do_io_read(Continuation *c, int64_t nbytes, MIOBuffer *buf) -{ - return (client_vc) ? client_vc->do_io_read(c, nbytes, buf) : nullptr; -} - -VIO * -Http1ClientSession::do_io_write(Continuation *c, int64_t nbytes, IOBufferReader *buf, bool owner) -{ - if (client_vc) { - return client_vc->do_io_write(c, nbytes, buf, owner); - } else { - return nullptr; - } -} - -void -Http1ClientSession::do_io_shutdown(ShutdownHowTo_t howto) -{ - client_vc->do_io_shutdown(howto); -} - void Http1ClientSession::do_io_close(int alerrno) { @@ -243,45 +232,42 @@ Http1ClientSession::do_io_close(int alerrno) slave_ka_vio = nullptr; } // Completed the last transaction. Just shutdown already - if (transact_count == released_transactions) { + // Or the do_io_close is due to a network error + if (transact_count == released_transactions || alerrno == HTTP_ERRNO) { half_close = false; } + if (half_close && this->trans.get_sm()) { read_state = HCS_HALF_CLOSED; SET_HANDLER(&Http1ClientSession::state_wait_for_close); HttpSsnDebug("[%" PRId64 "] session half close", con_id); - if (client_vc) { - // We want the client to know that that we're finished - // writing. The write shutdown accomplishes this. Unfortunately, - // the IO Core semantics don't stop us from getting events - // on the write side of the connection like timeouts so we - // need to zero out the write of the continuation with - // the do_io_write() call (INKqa05309) - client_vc->do_io_shutdown(IO_SHUTDOWN_WRITE); + if (_vc) { + _vc->do_io_shutdown(IO_SHUTDOWN_WRITE); - ka_vio = client_vc->do_io_read(this, INT64_MAX, read_buffer); + ka_vio = _vc->do_io_read(this, INT64_MAX, read_buffer); ink_assert(slave_ka_vio != ka_vio); // Set the active timeout to the same as the inactive time so // that this connection does not hang around forever if // the ua hasn't closed - client_vc->set_active_timeout(HRTIME_SECONDS(trans.get_sm()->t_state.txn_conf->keep_alive_no_activity_timeout_in)); + _vc->set_active_timeout(HRTIME_SECONDS(trans.get_sm()->t_state.txn_conf->keep_alive_no_activity_timeout_in)); } // [bug 2610799] Drain any data read. // If the buffer is full and the client writes again, we will not receive a // READ_READY event. - sm_reader->consume(sm_reader->read_avail()); + _reader->consume(_reader->read_avail()); } else { - read_state = HCS_CLOSED; HttpSsnDebug("[%" PRId64 "] session closed", con_id); HTTP_SUM_DYN_STAT(http_transactions_per_client_con, transact_count); - HTTP_DECREMENT_DYN_STAT(http_current_client_connections_stat); - conn_decrease = false; - if (client_vc) { - client_vc->do_io_close(); - client_vc = nullptr; + read_state = HCS_CLOSED; + + // Can go ahead and close the netvc now, but keeping around the session object + // until all the transactions are closed + if (_vc) { + _vc->do_io_close(); + _vc = nullptr; } } if (transact_count == released_transactions) { @@ -308,11 +294,11 @@ Http1ClientSession::state_wait_for_close(int event, void *data) case VC_EVENT_ACTIVE_TIMEOUT: case VC_EVENT_INACTIVITY_TIMEOUT: half_close = false; - this->do_io_close(); + this->do_io_close(EHTTP_ERROR); break; case VC_EVENT_READ_READY: // Drain any data read - sm_reader->consume(sm_reader->read_avail()); + _reader->consume(_reader->read_avail()); break; default: @@ -365,7 +351,7 @@ Http1ClientSession::state_slave_keep_alive(int event, void *data) int Http1ClientSession::state_keep_alive(int event, void *data) { - // Route the event. It is either for client vc or + // Route the event. It is either for vc or // the origin server slave vc if (data && data == slave_ka_vio) { return state_slave_keep_alive(event, data); @@ -384,7 +370,7 @@ Http1ClientSession::state_keep_alive(int event, void *data) break; case VC_EVENT_EOS: - this->do_io_close(); + this->do_io_close(EHTTP_ERROR); break; case VC_EVENT_READ_COMPLETE: @@ -396,17 +382,12 @@ Http1ClientSession::state_keep_alive(int event, void *data) case VC_EVENT_ACTIVE_TIMEOUT: case VC_EVENT_INACTIVITY_TIMEOUT: // Keep-alive timed out - this->do_io_close(); + this->do_io_close(EHTTP_ERROR); break; } return 0; } -void -Http1ClientSession::reenable(VIO *vio) -{ - client_vc->reenable(vio); -} // Called from the Http1Transaction::release void @@ -414,30 +395,28 @@ Http1ClientSession::release(ProxyTransaction *trans) { ink_assert(read_state == HCS_ACTIVE_READER || read_state == HCS_INIT); - // Clean up the write VIO in case of inactivity timeout - this->do_io_write(nullptr, 0, nullptr); + // Timeout events should be delivered to the session + this->do_io_write(this, 0, nullptr); // Check to see there is remaining data in the // buffer. If there is, spin up a new state // machine to process it. Otherwise, issue an // IO to wait for new data - bool more_to_read = this->sm_reader->is_read_avail_more_than(0); + bool more_to_read = this->_reader->is_read_avail_more_than(0); if (more_to_read) { trans->destroy(); - trans->set_restart_immediate(true); HttpSsnDebug("[%" PRId64 "] data already in buffer, starting new transaction", con_id); new_transaction(); } else { HttpSsnDebug("[%" PRId64 "] initiating io for next header", con_id); - trans->set_restart_immediate(false); read_state = HCS_KEEP_ALIVE; SET_HANDLER(&Http1ClientSession::state_keep_alive); ka_vio = this->do_io_read(this, INT64_MAX, read_buffer); ink_assert(slave_ka_vio != ka_vio); - if (client_vc) { - client_vc->cancel_active_timeout(); - client_vc->add_to_keep_alive_queue(); + if (_vc) { + _vc->cancel_active_timeout(); + _vc->add_to_keep_alive_queue(); } trans->destroy(); } @@ -447,11 +426,17 @@ void Http1ClientSession::new_transaction() { // If the client connection terminated during API callouts we're done. - if (nullptr == client_vc) { + if (nullptr == _vc) { this->do_io_close(); // calls the SSN_CLOSE hooks to match the SSN_START hooks. return; } + if (!_vc->add_to_active_queue()) { + // no room in the active queue close the connection + this->do_io_close(); + return; + } + // Defensive programming, make sure nothing persists across // connection re-use half_close = false; @@ -461,8 +446,7 @@ Http1ClientSession::new_transaction() trans.set_proxy_ssn(this); transact_count++; - client_vc->add_to_active_queue(); - trans.new_transaction(); + trans.new_transaction(read_from_early_data > 0 ? true : false); } void @@ -484,6 +468,7 @@ Http1ClientSession::attach_server_session(Http1ServerSession *ssession, bool tra // the server net conneciton from calling back a dead sm SET_HANDLER(&Http1ClientSession::state_keep_alive); slave_ka_vio = ssession->do_io_read(this, INT64_MAX, ssession->read_buffer); + this->do_io_write(this, 0, nullptr); // Capture the inactivity timeouts ink_assert(slave_ka_vio != ka_vio); // Transfer control of the write side as well @@ -516,3 +501,59 @@ Http1ClientSession::decrement_current_active_client_connections_stat() { HTTP_DECREMENT_DYN_STAT(http_current_active_client_connections_stat); } + +void +Http1ClientSession::start() +{ + // Troll for data to get a new transaction + this->release(&trans); +} + +bool +Http1ClientSession::allow_half_open() const +{ + // Only allow half open connections if the not over TLS + return (_vc && dynamic_cast(_vc) == nullptr); +} + +void +Http1ClientSession::set_half_close_flag(bool flag) +{ + half_close = flag; +} + +bool +Http1ClientSession::get_half_close_flag() const +{ + return half_close; +} + +bool +Http1ClientSession::is_chunked_encoding_supported() const +{ + return true; +} + +int +Http1ClientSession::get_transact_count() const +{ + return transact_count; +} + +bool +Http1ClientSession::is_outbound_transparent() const +{ + return f_outbound_transparent; +} + +Http1ServerSession * +Http1ClientSession::get_server_session() const +{ + return bound_ss; +} + +const char * +Http1ClientSession::get_protocol_string() const +{ + return "http"; +} diff --git a/proxy/http/Http1ClientSession.h b/proxy/http/Http1ClientSession.h index f6a1de0445a..aef989de0ca 100644 --- a/proxy/http/Http1ClientSession.h +++ b/proxy/http/Http1ClientSession.h @@ -53,114 +53,28 @@ class Http1ClientSession : public ProxySession Http1ClientSession(); // Implement ProxySession interface. + void new_connection(NetVConnection *new_vc, MIOBuffer *iobuf, IOBufferReader *reader) override; + void start() override; + void release(ProxyTransaction *trans) override; // Indicate we are done with a transaction + void release_transaction(); void destroy() override; void free() override; - void release_transaction(); - void - start() override - { - // Troll for data to get a new transaction - this->release(&trans); - } - - void new_connection(NetVConnection *new_vc, MIOBuffer *iobuf, IOBufferReader *reader) override; + void attach_server_session(Http1ServerSession *ssession, bool transaction_done = true) override; // Implement VConnection interface. - VIO *do_io_read(Continuation *c, int64_t nbytes = INT64_MAX, MIOBuffer *buf = nullptr) override; - VIO *do_io_write(Continuation *c = nullptr, int64_t nbytes = INT64_MAX, IOBufferReader *buf = nullptr, - bool owner = false) override; - void do_io_close(int lerrno = -1) override; - void do_io_shutdown(ShutdownHowTo_t howto) override; - void reenable(VIO *vio) override; - - bool - allow_half_open() - { - // Only allow half open connections if the not over TLS - return (client_vc && dynamic_cast(client_vc) == nullptr); - } - - void - set_half_close_flag(bool flag) override - { - half_close = flag; - } - - bool - get_half_close_flag() const override - { - return half_close; - } - - bool - is_chunked_encoding_supported() const override - { - return true; - } - - NetVConnection * - get_netvc() const override - { - return client_vc; - } - - int - get_transact_count() const override - { - return transact_count; - } - - virtual bool - is_outbound_transparent() const - { - return f_outbound_transparent; - } - - // Indicate we are done with a transaction - void release(ProxyTransaction *trans) override; - void attach_server_session(Http1ServerSession *ssession, bool transaction_done = true) override; + // Accessor Methods + bool allow_half_open() const; + void set_half_close_flag(bool flag) override; + bool get_half_close_flag() const override; + bool is_chunked_encoding_supported() const override; + int get_transact_count() const override; + virtual bool is_outbound_transparent() const; - Http1ServerSession * - get_server_session() const override - { - return bound_ss; - } - - void - set_active_timeout(ink_hrtime timeout_in) override - { - if (client_vc) - client_vc->set_active_timeout(timeout_in); - } - - void - set_inactivity_timeout(ink_hrtime timeout_in) override - { - if (client_vc) - client_vc->set_inactivity_timeout(timeout_in); - } - - void - cancel_inactivity_timeout() override - { - if (client_vc) - client_vc->cancel_inactivity_timeout(); - } - - const char * - get_protocol_string() const override - { - return "http"; - } - - bool - is_transparent_passthrough_allowed() const override - { - return f_transparent_passthrough; - } + Http1ServerSession *get_server_session() const override; + const char *get_protocol_string() const override; void increment_current_active_client_connections_stat() override; void decrement_current_active_client_connections_stat() override; @@ -182,14 +96,13 @@ class Http1ClientSession : public ProxySession HCS_CLOSED, }; - NetVConnection *client_vc = nullptr; - int magic = HTTP_SS_MAGIC_DEAD; - int transact_count = 0; - bool half_close = false; - bool conn_decrease = false; + int magic = HTTP_SS_MAGIC_DEAD; + int transact_count = 0; + bool half_close = false; + bool conn_decrease = false; - MIOBuffer *read_buffer = nullptr; - IOBufferReader *sm_reader = nullptr; + MIOBuffer *read_buffer = nullptr; + IOBufferReader *_reader = nullptr; C_Read_State read_state = HCS_INIT; @@ -200,14 +113,14 @@ class Http1ClientSession : public ProxySession int released_transactions = 0; + int64_t read_from_early_data = 0; + public: // Link debug_link; LINK(Http1ClientSession, debug_link); /// Set outbound connection to transparent. bool f_outbound_transparent = false; - /// Transparently pass-through non-HTTP traffic. - bool f_transparent_passthrough = false; Http1Transaction trans; }; diff --git a/proxy/http/Http1ServerSession.cc b/proxy/http/Http1ServerSession.cc index 23d7b53c228..2adaf28ff41 100644 --- a/proxy/http/Http1ServerSession.cc +++ b/proxy/http/Http1ServerSession.cc @@ -180,7 +180,7 @@ Http1ServerSession::release() server_vc->control_flags.set_flags(0); // Private sessions are never released back to the shared pool - if (private_session || TS_SERVER_SESSION_SHARING_MATCH_NONE == sharing_match) { + if (private_session || sharing_match == 0) { this->do_io_close(); return; } @@ -196,6 +196,7 @@ Http1ServerSession::release() // due to lock contention // FIX: should retry instead of closing this->do_io_close(); + HTTP_INCREMENT_DYN_STAT(http_origin_shutdown_pool_lock_contention); } else { // The session was successfully put into the session // manager and it will manage it @@ -203,3 +204,37 @@ Http1ServerSession::release() ink_assert(r == HSM_DONE); } } + +NetVConnection * +Http1ServerSession::get_netvc() const +{ + return server_vc; +}; + +void +Http1ServerSession::set_netvc(NetVConnection *new_vc) +{ + server_vc = new_vc; +} + +// Keys for matching hostnames +IpEndpoint const & +Http1ServerSession::get_server_ip() const +{ + ink_release_assert(server_vc != nullptr); + return server_vc->get_remote_endpoint(); +} + +int +Http1ServerSession::populate_protocol(std::string_view *result, int size) const +{ + auto vc = this->get_netvc(); + return vc ? vc->populate_protocol(result, size) : 0; +} + +const char * +Http1ServerSession::protocol_contains(std::string_view tag_prefix) const +{ + auto vc = this->get_netvc(); + return vc ? vc->protocol_contains(tag_prefix) : nullptr; +} diff --git a/proxy/http/Http1ServerSession.h b/proxy/http/Http1ServerSession.h index e1851ef6f73..a46c6dd6d08 100644 --- a/proxy/http/Http1ServerSession.h +++ b/proxy/http/Http1ServerSession.h @@ -63,55 +63,32 @@ class Http1ServerSession : public VConnection Http1ServerSession(self_type const &) = delete; self_type &operator=(self_type const &) = delete; - void destroy(); + //////////////////// + // Methods void new_connection(NetVConnection *new_vc); + void release(); + void destroy(); - /** Enable tracking the number of outbound session. - * - * @param group The connection tracking group. - * - * The @a group must have already incremented the connection count. It will be cleaned up when the - * session terminates. - */ - void enable_outbound_connection_tracking(OutboundConnTrack::Group *group); - - IOBufferReader * - get_reader() - { - return buf_reader; - }; - + // VConnection Methods VIO *do_io_read(Continuation *c, int64_t nbytes = INT64_MAX, MIOBuffer *buf = nullptr) override; - VIO *do_io_write(Continuation *c = nullptr, int64_t nbytes = INT64_MAX, IOBufferReader *buf = nullptr, bool owner = false) override; - void do_io_close(int lerrno = -1) override; void do_io_shutdown(ShutdownHowTo_t howto) override; void reenable(VIO *vio) override; - void release(); + void enable_outbound_connection_tracking(OutboundConnTrack::Group *group); + IOBufferReader *get_reader(); void attach_hostname(const char *hostname); - NetVConnection * - get_netvc() const - { - return server_vc; - }; - void - set_netvc(NetVConnection *new_vc) - { - server_vc = new_vc; - } - - // Keys for matching hostnames - IpEndpoint const & - get_server_ip() const - { - ink_release_assert(server_vc != nullptr); - return server_vc->get_remote_endpoint(); - } - + NetVConnection *get_netvc() const; + void set_netvc(NetVConnection *new_vc); + IpEndpoint const &get_server_ip() const; + int populate_protocol(std::string_view *result, int size) const; + const char *protocol_contains(std::string_view tag_prefix) const; + + //////////////////// + // Variables CryptoHash hostname_hash; int64_t con_id = 0; @@ -134,9 +111,8 @@ class Http1ServerSession : public VConnection bool private_session = false; // Copy of the owning SM's server session sharing settings - TSServerSessionSharingMatchType sharing_match = TS_SERVER_SESSION_SHARING_MATCH_BOTH; + TSServerSessionSharingMatchMask sharing_match = TS_SERVER_SESSION_SHARING_MATCH_MASK_NONE; TSServerSessionSharingPoolType sharing_pool = TS_SERVER_SESSION_SHARING_POOL_GLOBAL; - // int share_session; /// Hash map descriptor class for IP map. struct IPLinkage { @@ -178,20 +154,6 @@ class Http1ServerSession : public VConnection // an asynchronous cancel on NT MIOBuffer *read_buffer = nullptr; - virtual int - populate_protocol(std::string_view *result, int size) const - { - auto vc = this->get_netvc(); - return vc ? vc->populate_protocol(result, size) : 0; - } - - virtual const char * - protocol_contains(std::string_view tag_prefix) const - { - auto vc = this->get_netvc(); - return vc ? vc->protocol_contains(tag_prefix) : nullptr; - } - private: NetVConnection *server_vc = nullptr; int magic = HTTP_SS_MAGIC_DEAD; @@ -201,7 +163,8 @@ class Http1ServerSession : public VConnection extern ClassAllocator httpServerSessionAllocator; -// --- Implementation --- +//////////////////////////////////////////// +// INLINE inline void Http1ServerSession::attach_hostname(const char *hostname) @@ -211,6 +174,15 @@ Http1ServerSession::attach_hostname(const char *hostname) } } +inline IOBufferReader * +Http1ServerSession::get_reader() +{ + return buf_reader; +}; + +// +// LINKAGE + inline Http1ServerSession *& Http1ServerSession::IPLinkage::next_ptr(self_type *ssn) { diff --git a/proxy/http/Http1Transaction.cc b/proxy/http/Http1Transaction.cc index a3d811cddf8..d2f8b2c3ad0 100644 --- a/proxy/http/Http1Transaction.cc +++ b/proxy/http/Http1Transaction.cc @@ -30,16 +30,16 @@ Http1Transaction::release(IOBufferReader *r) { // Must set this inactivity count here rather than in the session because the state machine // is not available then - MgmtInt ka_in = current_reader->t_state.txn_conf->keep_alive_no_activity_timeout_in; + MgmtInt ka_in = _sm->t_state.txn_conf->keep_alive_no_activity_timeout_in; set_inactivity_timeout(HRTIME_SECONDS(ka_in)); - proxy_ssn->clear_session_active(); - proxy_ssn->ssn_last_txn_time = Thread::get_hrtime(); + _proxy_ssn->clear_session_active(); + _proxy_ssn->ssn_last_txn_time = Thread::get_hrtime(); // Make sure that the state machine is returning // correct buffer reader - ink_assert(r == sm_reader); - if (r != sm_reader) { + ink_assert(r == _reader); + if (r != _reader) { this->do_io_close(); } else { super_type::release(r); @@ -47,36 +47,32 @@ Http1Transaction::release(IOBufferReader *r) } void -Http1Transaction::set_proxy_ssn(ProxySession *new_proxy_ssn) +Http1Transaction::destroy() // todo make ~Http1Transaction() { - Http1ClientSession *http1_proxy_ssn = dynamic_cast(new_proxy_ssn); - - if (http1_proxy_ssn) { - outbound_port = http1_proxy_ssn->outbound_port; - outbound_ip4 = http1_proxy_ssn->outbound_ip4; - outbound_ip6 = http1_proxy_ssn->outbound_ip6; - outbound_transparent = http1_proxy_ssn->f_outbound_transparent; - super_type::set_proxy_ssn(new_proxy_ssn); - } else { - proxy_ssn = nullptr; - } + _sm = nullptr; } void Http1Transaction::transaction_done() { - if (proxy_ssn) { - static_cast(proxy_ssn)->release_transaction(); + if (_proxy_ssn) { + static_cast(_proxy_ssn)->release_transaction(); } } +void +Http1Transaction::reenable(VIO *vio) +{ + _proxy_ssn->reenable(vio); +} + bool Http1Transaction::allow_half_open() const { - bool config_allows_it = (current_reader) ? current_reader->t_state.txn_conf->allow_half_open > 0 : true; + bool config_allows_it = (_sm) ? _sm->t_state.txn_conf->allow_half_open > 0 : true; if (config_allows_it) { // Check with the session to make sure the underlying transport allows the half open scenario - return static_cast(proxy_ssn)->allow_half_open(); + return static_cast(_proxy_ssn)->allow_half_open(); } return false; } @@ -92,3 +88,60 @@ Http1Transaction::decrement_client_transactions_stat() { HTTP_DECREMENT_DYN_STAT(http_current_client_transactions_stat); } + +// Implement VConnection interface. +VIO * +Http1Transaction::do_io_read(Continuation *c, int64_t nbytes, MIOBuffer *buf) +{ + return _proxy_ssn->do_io_read(c, nbytes, buf); +} +VIO * +Http1Transaction::do_io_write(Continuation *c, int64_t nbytes, IOBufferReader *buf, bool owner) +{ + return _proxy_ssn->do_io_write(c, nbytes, buf, owner); +} + +void +Http1Transaction::do_io_close(int lerrno) +{ + _proxy_ssn->do_io_close(lerrno); + // this->destroy(); Parent owns this data structure. No need for separate destroy. +} + +void +Http1Transaction::do_io_shutdown(ShutdownHowTo_t howto) +{ + _proxy_ssn->do_io_shutdown(howto); +} + +void +Http1Transaction::set_active_timeout(ink_hrtime timeout_in) +{ + if (_proxy_ssn) { + _proxy_ssn->set_active_timeout(timeout_in); + } +} +void +Http1Transaction::set_inactivity_timeout(ink_hrtime timeout_in) +{ + if (_proxy_ssn) { + _proxy_ssn->set_inactivity_timeout(timeout_in); + } +} +void +Http1Transaction::cancel_inactivity_timeout() +{ + if (_proxy_ssn) { + _proxy_ssn->cancel_inactivity_timeout(); + } +} +// +int +Http1Transaction::get_transaction_id() const +{ + // For HTTP/1 there is only one on-going transaction at a time per session/connection. Therefore, the transaction count can be + // presumed not to increase during the lifetime of a transaction, thus this function will return a consistent unique transaction + // identifier. + // + return _proxy_ssn->get_transact_count(); +} diff --git a/proxy/http/Http1Transaction.h b/proxy/http/Http1Transaction.h index dfc9e3ab522..407d73ec030 100644 --- a/proxy/http/Http1Transaction.h +++ b/proxy/http/Http1Transaction.h @@ -33,103 +33,43 @@ class Http1Transaction : public ProxyTransaction using super_type = ProxyTransaction; Http1Transaction() {} - // Implement VConnection interface. - VIO * - do_io_read(Continuation *c, int64_t nbytes = INT64_MAX, MIOBuffer *buf = nullptr) override - { - return proxy_ssn->do_io_read(c, nbytes, buf); - } - VIO * - do_io_write(Continuation *c = nullptr, int64_t nbytes = INT64_MAX, IOBufferReader *buf = nullptr, bool owner = false) override - { - return proxy_ssn->do_io_write(c, nbytes, buf, owner); - } - - void - do_io_close(int lerrno = -1) override - { - proxy_ssn->do_io_close(lerrno); - // this->destroy(); Parent owns this data structure. No need for separate destroy. - } - - // Don't destroy your elements. Rely on the Http1ClientSession to clean up the - // Http1Transaction class as necessary. The super::destroy() clears the - // mutex, which Http1ClientSession owns. - void - destroy() override - { - current_reader = nullptr; - } - - void - do_io_shutdown(ShutdownHowTo_t howto) override - { - proxy_ssn->do_io_shutdown(howto); - } - - void - reenable(VIO *vio) override - { - proxy_ssn->reenable(vio); - } - - void - set_reader(IOBufferReader *reader) - { - sm_reader = reader; - } + //////////////////// + // Methods void release(IOBufferReader *r) override; + void destroy() override; // todo make ~Http1Transaction() - bool allow_half_open() const override; + // Implement VConnection interface. + VIO *do_io_read(Continuation *c, int64_t nbytes = INT64_MAX, MIOBuffer *buf = nullptr) override; + VIO *do_io_write(Continuation *c = nullptr, int64_t nbytes = INT64_MAX, IOBufferReader *buf = nullptr, + bool owner = false) override; + void do_io_close(int lerrno = -1) override; + void do_io_shutdown(ShutdownHowTo_t howto) override; + void reenable(VIO *vio) override; - void set_proxy_ssn(ProxySession *new_proxy_ssn) override; - - bool - is_outbound_transparent() const override - { - return outbound_transparent; - } - void - set_outbound_transparent(bool flag) override - { - outbound_transparent = flag; - } - - // Pass on the timeouts to the netvc - void - set_active_timeout(ink_hrtime timeout_in) override - { - if (proxy_ssn) - proxy_ssn->set_active_timeout(timeout_in); - } - void - set_inactivity_timeout(ink_hrtime timeout_in) override - { - if (proxy_ssn) - proxy_ssn->set_inactivity_timeout(timeout_in); - } - void - cancel_inactivity_timeout() override - { - if (proxy_ssn) - proxy_ssn->cancel_inactivity_timeout(); - } + bool allow_half_open() const override; + void set_active_timeout(ink_hrtime timeout_in) override; + void set_inactivity_timeout(ink_hrtime timeout_in) override; + void cancel_inactivity_timeout() override; void transaction_done() override; - - int - get_transaction_id() const override - { - // For HTTP/1 there is only one on-going transaction at a time per session/connection. Therefore, the transaction count can be - // presumed not to increase during the lifetime of a transaction, thus this function will return a consistent unique transaction - // identifier. - // - return proxy_ssn->get_transact_count(); - } - + int get_transaction_id() const override; void increment_client_transactions_stat() override; void decrement_client_transactions_stat() override; + void set_reader(IOBufferReader *reader); + + //////////////////// + // Variables + protected: bool outbound_transparent{false}; }; + +////////////////////////////////// +// INLINE + +inline void +Http1Transaction::set_reader(IOBufferReader *reader) +{ + _reader = reader; +} diff --git a/proxy/http/HttpBodyFactory.cc b/proxy/http/HttpBodyFactory.cc index 65b44c2d287..6d2150d7ba0 100644 --- a/proxy/http/HttpBodyFactory.cc +++ b/proxy/http/HttpBodyFactory.cc @@ -31,6 +31,7 @@ #include "tscore/ink_platform.h" #include "tscore/ink_sprintf.h" #include "tscore/ink_file.h" +#include "tscore/Filenames.h" #include "HttpBodyFactory.h" #include #include @@ -72,6 +73,16 @@ HttpBodyFactory::fabricate_with_old_api(const char *type, HttpTransact::State *c bool found_requested_template = false; bool plain_flag = false; + /////////////////////////////////////////////// + // if suppressing this response, return NULL // + /////////////////////////////////////////////// + if (is_response_suppressed(context)) { + if (enable_logging) { + Log::error("BODY_FACTORY: suppressing '%s' response for url '%s'", type, url); + } + return nullptr; + } + lock(); *resulting_buffer_length = 0; @@ -98,16 +109,6 @@ HttpBodyFactory::fabricate_with_old_api(const char *type, HttpTransact::State *c } } } - /////////////////////////////////////////////// - // if suppressing this response, return NULL // - /////////////////////////////////////////////// - if (is_response_suppressed(context)) { - if (enable_logging) { - Log::error("BODY_FACTORY: suppressing '%s' response for url '%s'", type, url); - } - unlock(); - return nullptr; - } ////////////////////////////////////////////////////////////////////////////////// // if language-targeting activated, get client Accept-Language & Accept-Charset // ////////////////////////////////////////////////////////////////////////////////// @@ -240,6 +241,7 @@ void HttpBodyFactory::reconfigure() { RecInt e; + RecString s = nullptr; bool all_found; int rec_err; @@ -275,15 +277,22 @@ HttpBodyFactory::reconfigure() all_found = all_found && (rec_err == REC_ERR_OKAY); Debug("body_factory", "response_suppression_mode = %d (found = %" PRId64 ")", response_suppression_mode, e); - ats_scoped_str directory_of_template_sets(RecConfigReadConfigPath("proxy.config.body_factory.template_sets_dir", "body_factory")); + ats_scoped_str directory_of_template_sets; - if (access(directory_of_template_sets, R_OK) < 0) { - Warning("Unable to access() directory '%s': %d, %s", (const char *)directory_of_template_sets, errno, strerror(errno)); - Warning(" Please set 'proxy.config.body_factory.template_sets_dir' "); + rec_err = RecGetRecordString_Xmalloc("proxy.config.body_factory.template_sets_dir", &s); + all_found = all_found && (rec_err == REC_ERR_OKAY); + if (rec_err == REC_ERR_OKAY) { + directory_of_template_sets = Layout::get()->relative(s); + if (access(directory_of_template_sets, R_OK) < 0) { + Warning("Unable to access() directory '%s': %d, %s", (const char *)directory_of_template_sets, errno, strerror(errno)); + Warning(" Please set 'proxy.config.body_factory.template_sets_dir' "); + } } Debug("body_factory", "directory_of_template_sets = '%s' ", (const char *)directory_of_template_sets); + ats_free(s); + if (!all_found) { Warning("config changed, but can't fetch all proxy.config.body_factory values"); } @@ -334,7 +343,7 @@ HttpBodyFactory::HttpBodyFactory() for (i = 0; config_record_names[i] != nullptr; i++) { status = REC_RegisterConfigUpdateFunc(config_record_names[i], config_callback, (void *)this); if (status != REC_ERR_OKAY) { - Warning("couldn't register variable '%s', is records.config up to date?", config_record_names[i]); + Warning("couldn't register variable '%s', is %s up to date?", config_record_names[i], ts::filename::RECORDS); } no_registrations_failed = no_registrations_failed && (status == REC_ERR_OKAY); } @@ -672,11 +681,7 @@ HttpBodyFactory::is_response_suppressed(HttpTransact::State *context) } else if (response_suppression_mode == 1) { return true; } else if (response_suppression_mode == 2) { - if (context->req_flavor == HttpTransact::REQ_FLAVOR_INTERCEPTED) { - return true; - } else { - return false; - } + return context->request_data.internal_txn; } else { return false; } diff --git a/proxy/http/HttpCacheSM.cc b/proxy/http/HttpCacheSM.cc index 731ebf2ee76..2bab3be01f4 100644 --- a/proxy/http/HttpCacheSM.cc +++ b/proxy/http/HttpCacheSM.cc @@ -159,7 +159,8 @@ HttpCacheSM::state_cache_open_write(int event, void *data) { STATE_ENTER(&HttpCacheSM::state_cache_open_write, event); ink_assert(captive_action.cancelled == 0); - pending_action = nullptr; + pending_action = nullptr; + bool read_retry_on_write_fail = false; switch (event) { case CACHE_EVENT_OPEN_WRITE: @@ -171,9 +172,32 @@ HttpCacheSM::state_cache_open_write(int event, void *data) break; case CACHE_EVENT_OPEN_WRITE_FAILED: - if (open_write_tries <= master_sm->t_state.txn_conf->max_cache_open_write_retries) { + if (master_sm->t_state.txn_conf->cache_open_write_fail_action == CACHE_WL_FAIL_ACTION_READ_RETRY) { + // fall back to open_read_tries + // Note that when CACHE_WL_FAIL_ACTION_READ_RETRY is configured, max_cache_open_write_retries + // is automatically ignored. Make sure to not disable max_cache_open_read_retries + // with CACHE_WL_FAIL_ACTION_READ_RETRY as this results in proxy'ing to origin + // without write retries in both a cache miss or a cache refresh scenario. + if (open_write_tries <= master_sm->t_state.txn_conf->max_cache_open_write_retries) { + Debug("http_cache", "[%" PRId64 "] [state_cache_open_write] cache open write failure %d. read retry triggered", + master_sm->sm_id, open_write_tries); + if (master_sm->t_state.txn_conf->max_cache_open_read_retries <= 0) { + Debug("http_cache", + "[%" PRId64 "] [state_cache_open_write] invalid config, cache write fail set to" + " read retry, but, max_cache_open_read_retries is not enabled", + master_sm->sm_id); + } + open_read_tries = 0; + read_retry_on_write_fail = true; + // make sure it doesn't loop indefinitely + open_write_tries = master_sm->t_state.txn_conf->max_cache_open_write_retries + 1; + } + } + if (read_retry_on_write_fail || open_write_tries <= master_sm->t_state.txn_conf->max_cache_open_write_retries) { // Retry open write; open_write_cb = false; + // reset captive_action since HttpSM cancelled it + captive_action.cancelled = 0; do_schedule_in(); } else { // The cache is hosed or full or something. @@ -188,18 +212,27 @@ HttpCacheSM::state_cache_open_write(int event, void *data) break; case EVENT_INTERVAL: - // Retry the cache open write if the number retries is less - // than or equal to the max number of open write retries - ink_assert(open_write_tries <= master_sm->t_state.txn_conf->max_cache_open_write_retries); - Debug("http_cache", - "[%" PRId64 "] [state_cache_open_write] cache open write failure %d. " - "retrying cache open write...", - master_sm->sm_id, open_write_tries); - - open_write(&cache_key, lookup_url, read_request_hdr, master_sm->t_state.cache_info.object_read, - static_cast( - (master_sm->t_state.cache_control.pin_in_cache_for < 0) ? 0 : master_sm->t_state.cache_control.pin_in_cache_for), - retry_write, false); + if (master_sm->t_state.txn_conf->cache_open_write_fail_action == CACHE_WL_FAIL_ACTION_READ_RETRY) { + Debug("http_cache", + "[%" PRId64 "] [state_cache_open_write] cache open write failure %d. " + "falling back to read retry...", + master_sm->sm_id, open_write_tries); + open_read_cb = false; + master_sm->handleEvent(CACHE_EVENT_OPEN_READ, data); + } else { + Debug("http_cache", + "[%" PRId64 "] [state_cache_open_write] cache open write failure %d. " + "retrying cache open write...", + master_sm->sm_id, open_write_tries); + + // Retry the cache open write if the number retries is less + // than or equal to the max number of open write retries + ink_assert(open_write_tries <= master_sm->t_state.txn_conf->max_cache_open_write_retries); + open_write(&cache_key, lookup_url, read_request_hdr, master_sm->t_state.cache_info.object_read, + static_cast( + (master_sm->t_state.cache_control.pin_in_cache_for < 0) ? 0 : master_sm->t_state.cache_control.pin_in_cache_for), + retry_write, false); + } break; default: @@ -233,6 +266,8 @@ HttpCacheSM::do_cache_open_read(const HttpCacheKey &key) } else { ink_assert(open_read_cb == false); } + // reset captive_action since HttpSM cancelled it during open read retry + captive_action.cancelled = 0; // Initialising read-while-write-inprogress flag this->readwhilewrite_inprogress = false; Action *action_handle = cacheProcessor.open_read(this, &key, this->read_request_hdr, this->http_params, this->read_pin_in_cache); @@ -253,7 +288,8 @@ HttpCacheSM::do_cache_open_read(const HttpCacheKey &key) } Action * -HttpCacheSM::open_read(const HttpCacheKey *key, URL *url, HTTPHdr *hdr, OverridableHttpConfigParams *params, time_t pin_in_cache) +HttpCacheSM::open_read(const HttpCacheKey *key, URL *url, HTTPHdr *hdr, const OverridableHttpConfigParams *params, + time_t pin_in_cache) { Action *act_return; diff --git a/proxy/http/HttpCacheSM.h b/proxy/http/HttpCacheSM.h index 89b8542ec58..2fc0c0ec2aa 100644 --- a/proxy/http/HttpCacheSM.h +++ b/proxy/http/HttpCacheSM.h @@ -65,7 +65,8 @@ class HttpCacheSM : public Continuation captive_action.init(this); } - Action *open_read(const HttpCacheKey *key, URL *url, HTTPHdr *hdr, OverridableHttpConfigParams *params, time_t pin_in_cache); + Action *open_read(const HttpCacheKey *key, URL *url, HTTPHdr *hdr, const OverridableHttpConfigParams *params, + time_t pin_in_cache); Action *open_write(const HttpCacheKey *key, URL *url, HTTPHdr *request, CacheHTTPInfo *old_info, time_t pin_in_cache, bool retry, bool allow_multiple); @@ -194,10 +195,10 @@ class HttpCacheSM : public Continuation bool open_write_cb = false; // Open read parameters - int open_read_tries = 0; - HTTPHdr *read_request_hdr = nullptr; - OverridableHttpConfigParams *http_params = nullptr; - time_t read_pin_in_cache = 0; + int open_read_tries = 0; + HTTPHdr *read_request_hdr = nullptr; + const OverridableHttpConfigParams *http_params = nullptr; + time_t read_pin_in_cache = 0; // Open write parameters bool retry_write = true; diff --git a/proxy/http/HttpConfig.cc b/proxy/http/HttpConfig.cc index 46f0f0979be..b32e39c9831 100644 --- a/proxy/http/HttpConfig.cc +++ b/proxy/http/HttpConfig.cc @@ -1,4 +1,5 @@ /** @file + * A brief file description @@ -22,6 +23,7 @@ */ #include "tscore/ink_config.h" +#include "tscore/Filenames.h" #include #include #include "HttpConfig.h" @@ -74,11 +76,12 @@ template struct ConfigEnumPair { /// If found @a value is set to the corresponding value in @a list. template static bool -http_config_enum_search(const char *key, const ConfigEnumPair (&list)[N], MgmtByte &value) +http_config_enum_search(std::string_view key, const ConfigEnumPair (&list)[N], MgmtByte &value) { + Debug("http_config", "enum element %.*s", static_cast(key.size()), key.data()); // We don't expect any of these lists to be more than 10 long, so a linear search is the best choice. for (unsigned i = 0; i < N; ++i) { - if (0 == strcasecmp(list[i]._key, key)) { + if (key.compare(list[i]._key) == 0) { value = list[i]._value; return true; } @@ -108,10 +111,56 @@ http_config_enum_read(const char *name, const ConfigEnumPair (&list)[N], Mgmt //////////////////////////////////////////////////////////////// /// Session sharing match types. static const ConfigEnumPair SessionSharingMatchStrings[] = { - {TS_SERVER_SESSION_SHARING_MATCH_NONE, "none"}, - {TS_SERVER_SESSION_SHARING_MATCH_IP, "ip"}, - {TS_SERVER_SESSION_SHARING_MATCH_HOST, "host"}, - {TS_SERVER_SESSION_SHARING_MATCH_BOTH, "both"}}; + {TS_SERVER_SESSION_SHARING_MATCH_NONE, "none"}, {TS_SERVER_SESSION_SHARING_MATCH_IP, "ip"}, + {TS_SERVER_SESSION_SHARING_MATCH_HOST, "host"}, {TS_SERVER_SESSION_SHARING_MATCH_HOST, "hostsni"}, + {TS_SERVER_SESSION_SHARING_MATCH_BOTH, "both"}, {TS_SERVER_SESSION_SHARING_MATCH_HOSTONLY, "hostonly"}, + {TS_SERVER_SESSION_SHARING_MATCH_SNI, "sni"}, {TS_SERVER_SESSION_SHARING_MATCH_CERT, "cert"}}; + +bool +HttpConfig::load_server_session_sharing_match(const char *key, MgmtByte &mask) +{ + MgmtByte value; + mask = 0; + // Parse through and build up mask + std::string_view key_list(key); + size_t start = 0; + size_t offset = 0; + Debug("http_config", "enum mask value %s", key); + do { + offset = key_list.find(',', start); + if (offset == std::string_view::npos) { + std::string_view one_key = key_list.substr(start); + if (!http_config_enum_search(one_key, SessionSharingMatchStrings, value)) { + return false; + } + } else { + std::string_view one_key = key_list.substr(start, offset - start); + if (!http_config_enum_search(one_key, SessionSharingMatchStrings, value)) { + return false; + } + start = offset + 1; + } + if (value < TS_SERVER_SESSION_SHARING_MATCH_NONE) { + mask |= (1 << value); + } else if (value == TS_SERVER_SESSION_SHARING_MATCH_BOTH) { + mask |= TS_SERVER_SESSION_SHARING_MATCH_MASK_IP | TS_SERVER_SESSION_SHARING_MATCH_MASK_HOSTONLY | + TS_SERVER_SESSION_SHARING_MATCH_MASK_HOSTSNISYNC; + } else if (value == TS_SERVER_SESSION_SHARING_MATCH_HOST) { + mask |= TS_SERVER_SESSION_SHARING_MATCH_MASK_HOSTONLY | TS_SERVER_SESSION_SHARING_MATCH_MASK_HOSTSNISYNC; + } + } while (offset != std::string_view::npos); + return true; +} + +static bool +http_config_enum_mask_read(const char *name, MgmtByte &value) +{ + char key[512]; // it's just one key - painful UI if keys are longer than this + if (REC_ERR_OKAY == RecGetRecordString(name, key, sizeof(key))) { + return HttpConfig::load_server_session_sharing_match(key, value); + } + return false; +} static const ConfigEnumPair SessionSharingPoolStrings[] = { {TS_SERVER_SESSION_SHARING_POOL_GLOBAL, "global"}, @@ -137,7 +186,7 @@ HttpConfigCont::handle_event(int /* event ATS_UNUSED */, void * /* edata ATS_UNU return 0; } -static int +int http_config_cb(const char * /* name ATS_UNUSED */, RecDataT /* data_type ATS_UNUSED */, RecData /* data ATS_UNUSED */, void * /* cookie ATS_UNUSED */) { @@ -149,6 +198,42 @@ http_config_cb(const char * /* name ATS_UNUSED */, RecDataT /* data_type ATS_UNU return 0; } +void +Enable_Config_Var(std::string_view const &name, bool (*cb)(const char *, RecDataT, RecData, void *), void *cookie) +{ + // Must use this indirection because the API requires a pure function, therefore no values can + // be bound in the lambda. Instead this is needed to pass in the data for both the lambda and + // the actual callback. + using Context = std::tuple; + + // To deal with process termination cleanup, store the context instances in a deque where + // tail insertion doesn't invalidate pointers. These persist until process shutdown. + static std::deque storage; + + Context &ctx = storage.emplace_back(cb, cookie); + // Register the call back - this handles external updates. + RecRegisterConfigUpdateCb( + name.data(), + [](const char *name, RecDataT dtype, RecData data, void *ctx) -> int { + auto &&[cb, cookie] = *static_cast(ctx); + if ((*cb)(name, dtype, data, cookie)) { + http_config_cb(name, dtype, data, cookie); // signal runtime config update. + } + return REC_ERR_OKAY; + }, + &ctx); + + // Use the record to do the initial data load. + // Look it up and call the updater @a cb on that data. + RecLookupRecord( + name.data(), + [](RecRecord const *r, void *ctx) -> void { + auto &&[cb, cookie] = *static_cast(ctx); + (*cb)(r->name, r->data_type, r->data, cookie); + }, + &ctx); +} + // [amc] Not sure which is uglier, this switch or having a micro-function for each var. // Oh, how I long for when we can use C++eleventy lambdas without compiler problems! // I think for 5.0 when the BC stuff is yanked, we should probably revert this to independent callbacks. @@ -162,7 +247,7 @@ http_server_session_sharing_cb(const char *name, RecDataT dtype, RecData data, v MgmtByte &match = c->oride.server_session_sharing_match; if (RECD_INT == dtype) { match = static_cast(data.rec_int); - } else if (RECD_STRING == dtype && http_config_enum_search(data.rec_string, SessionSharingMatchStrings, match)) { + } else if (RECD_STRING == dtype && HttpConfig::load_server_session_sharing_match(data.rec_string, match)) { // empty } else { valid_p = false; @@ -265,6 +350,44 @@ register_stat_callbacks() RecRegisterRawStat(http_rsb, RECT_PROCESS, "proxy.process.http.total_parent_marked_down_count", RECD_COUNTER, RECP_PERSISTENT, (int)http_total_parent_marked_down_count, RecRawStatSyncCount); + // Stats to track causes of ATS initiated origin shutdowns + RecRegisterRawStat(http_rsb, RECT_PROCESS, "proxy.process.http.origin_shutdown.pool_lock_contention", RECD_INT, + RECP_NON_PERSISTENT, (int)http_origin_shutdown_pool_lock_contention, RecRawStatSyncCount); + RecRegisterRawStat(http_rsb, RECT_PROCESS, "proxy.process.http.origin_shutdown.migration_failure", RECD_INT, RECP_NON_PERSISTENT, + (int)http_origin_shutdown_migration_failure, RecRawStatSyncCount); + RecRegisterRawStat(http_rsb, RECT_PROCESS, "proxy.process.http.origin_shutdown.tunnel_server", RECD_INT, RECP_NON_PERSISTENT, + (int)http_origin_shutdown_tunnel_server, RecRawStatSyncCount); + RecRegisterRawStat(http_rsb, RECT_PROCESS, "proxy.process.http.origin_shutdown.tunnel_server_no_keep_alive", RECD_INT, + RECP_NON_PERSISTENT, (int)http_origin_shutdown_tunnel_server_no_keep_alive, RecRawStatSyncCount); + RecRegisterRawStat(http_rsb, RECT_PROCESS, "proxy.process.http.origin_shutdown.tunnel_server_eos", RECD_INT, RECP_NON_PERSISTENT, + (int)http_origin_shutdown_tunnel_server_eos, RecRawStatSyncCount); + RecRegisterRawStat(http_rsb, RECT_PROCESS, "proxy.process.http.origin_shutdown.tunnel_server_plugin_tunnel", RECD_INT, + RECP_NON_PERSISTENT, (int)http_origin_shutdown_tunnel_server_plugin_tunnel, RecRawStatSyncCount); + RecRegisterRawStat(http_rsb, RECT_PROCESS, "proxy.process.http.origin_shutdown.tunnel_server_detach", RECD_INT, + RECP_NON_PERSISTENT, (int)http_origin_shutdown_tunnel_server_detach, RecRawStatSyncCount); + RecRegisterRawStat(http_rsb, RECT_PROCESS, "proxy.process.http.origin_shutdown.tunnel_client", RECD_INT, RECP_NON_PERSISTENT, + (int)http_origin_shutdown_tunnel_client, RecRawStatSyncCount); + RecRegisterRawStat(http_rsb, RECT_PROCESS, "proxy.process.http.origin_shutdown.tunnel_transform_read", RECD_INT, + RECP_NON_PERSISTENT, (int)http_origin_shutdown_tunnel_transform_read, RecRawStatSyncCount); + RecRegisterRawStat(http_rsb, RECT_PROCESS, "proxy.process.http.origin_shutdown.release_no_sharing", RECD_INT, RECP_NON_PERSISTENT, + (int)http_origin_shutdown_release_no_sharing, RecRawStatSyncCount); + RecRegisterRawStat(http_rsb, RECT_PROCESS, "proxy.process.http.origin_shutdown.release_no_server", RECD_INT, RECP_NON_PERSISTENT, + (int)http_origin_shutdown_release_no_server, RecRawStatSyncCount); + RecRegisterRawStat(http_rsb, RECT_PROCESS, "proxy.process.http.origin_shutdown.release_no_keep_alive", RECD_INT, + RECP_NON_PERSISTENT, (int)http_origin_shutdown_release_no_keep_alive, RecRawStatSyncCount); + RecRegisterRawStat(http_rsb, RECT_PROCESS, "proxy.process.http.origin_shutdown.release_invalid_response", RECD_INT, + RECP_NON_PERSISTENT, (int)http_origin_shutdown_release_invalid_response, RecRawStatSyncCount); + RecRegisterRawStat(http_rsb, RECT_PROCESS, "proxy.process.http.origin_shutdown.release_invalid_request", RECD_INT, + RECP_NON_PERSISTENT, (int)http_origin_shutdown_release_invalid_request, RecRawStatSyncCount); + RecRegisterRawStat(http_rsb, RECT_PROCESS, "proxy.process.http.origin_shutdown.release_modified", RECD_INT, RECP_NON_PERSISTENT, + (int)http_origin_shutdown_release_modified, RecRawStatSyncCount); + RecRegisterRawStat(http_rsb, RECT_PROCESS, "proxy.process.http.origin_shutdown.release_misc", RECD_INT, RECP_NON_PERSISTENT, + (int)http_origin_shutdown_release_misc, RecRawStatSyncCount); + RecRegisterRawStat(http_rsb, RECT_PROCESS, "proxy.process.http.origin_shutdown.cleanup_entry", RECD_INT, RECP_NON_PERSISTENT, + (int)http_origin_shutdown_cleanup_entry, RecRawStatSyncCount); + RecRegisterRawStat(http_rsb, RECT_PROCESS, "proxy.process.http.origin_shutdown.tunnel_abort", RECD_INT, RECP_NON_PERSISTENT, + (int)http_origin_shutdown_tunnel_abort, RecRawStatSyncCount); + // Upstream current connections stats RecRegisterRawStat(http_rsb, RECT_PROCESS, "proxy.process.http.current_parent_proxy_connections", RECD_INT, RECP_NON_PERSISTENT, (int)http_current_parent_proxy_connections_stat, RecRawStatSyncSum); @@ -356,9 +479,6 @@ register_stat_callbacks() RecRegisterRawStat(http_rsb, RECT_PROCESS, "proxy.process.http.tunnels", RECD_COUNTER, RECP_PERSISTENT, (int)http_tunnels_stat, RecRawStatSyncCount); - RecRegisterRawStat(http_rsb, RECT_PROCESS, "proxy.process.http.throttled_proxy_only", RECD_COUNTER, RECP_PERSISTENT, - (int)http_throttled_proxy_only_stat, RecRawStatSyncCount); - RecRegisterRawStat(http_rsb, RECT_PROCESS, "proxy.process.http.parent_proxy_transaction_time", RECD_INT, RECP_PERSISTENT, (int)http_parent_proxy_transaction_time_stat, RecRawStatSyncSum); @@ -860,6 +980,12 @@ register_stat_callbacks() (int)http_origin_connections_throttled_stat, RecRawStatSyncCount); RecRegisterRawStat(http_rsb, RECT_PROCESS, "proxy.process.http.post_body_too_large", RECD_COUNTER, RECP_PERSISTENT, (int)http_post_body_too_large, RecRawStatSyncCount); + RecRegisterRawStat(http_rsb, RECT_PROCESS, "proxy.process.http.origin.connect.adjust_thread", RECD_COUNTER, RECP_NON_PERSISTENT, + (int)http_origin_connect_adjust_thread_stat, RecRawStatSyncCount); + HTTP_CLEAR_DYN_STAT(http_origin_connect_adjust_thread_stat); + RecRegisterRawStat(http_rsb, RECT_PROCESS, "proxy.process.http.cache.open_write.adjust_thread", RECD_COUNTER, RECP_NON_PERSISTENT, + (int)http_cache_open_write_adjust_thread_stat, RecRawStatSyncCount); + HTTP_CLEAR_DYN_STAT(http_cache_open_write_adjust_thread_stat); // milestones RecRegisterRawStat(http_rsb, RECT_PROCESS, "proxy.process.http.milestone.ua_begin", RECD_COUNTER, RECP_PERSISTENT, (int)http_ua_begin_time_stat, RecRawStatSyncSum); @@ -979,7 +1105,7 @@ HttpConfig::startup() RecHttpLoadIp("proxy.local.incoming_ip_to_bind", c.inbound_ip4, c.inbound_ip6); RecHttpLoadIp("proxy.local.outgoing_ip_to_bind", c.outbound_ip4, c.outbound_ip6); - RecHttpLoadIpMap("proxy.config.http.proxy_protocol_whitelist", c.config_proxy_protocol_ipmap); + RecHttpLoadIpMap("proxy.config.http.proxy_protocol_allowlist", c.config_proxy_protocol_ipmap); SSLConfigInit(&c.config_proxy_protocol_ipmap); HttpEstablishStaticConfigLongLong(c.server_max_connections, "proxy.config.http.server_max_connections"); @@ -1024,8 +1150,8 @@ HttpConfig::startup() // [amc] This is a bit of a mess, need to figure out to make this cleaner. RecRegisterConfigUpdateCb("proxy.config.http.server_session_sharing.match", &http_server_session_sharing_cb, &c); - http_config_enum_read("proxy.config.http.server_session_sharing.match", SessionSharingMatchStrings, - c.oride.server_session_sharing_match); + http_config_enum_mask_read("proxy.config.http.server_session_sharing.match", c.oride.server_session_sharing_match); + HttpEstablishStaticConfigStringAlloc(c.oride.server_session_sharing_match_str, "proxy.config.http.server_session_sharing.match"); http_config_enum_read("proxy.config.http.server_session_sharing.pool", SessionSharingPoolStrings, c.server_session_sharing_pool); RecRegisterConfigUpdateCb("proxy.config.http.insert_forwarded", &http_insert_forwarded_cb, &c); @@ -1203,6 +1329,8 @@ HttpConfig::startup() HttpEstablishStaticConfigByte(c.enable_http_info, "proxy.config.http.enable_http_info"); HttpEstablishStaticConfigLongLong(c.max_post_size, "proxy.config.http.max_post_size"); + HttpEstablishStaticConfigLongLong(c.max_payload_iobuf_index, "proxy.config.payload.io.max_buffer_index"); + HttpEstablishStaticConfigLongLong(c.max_msg_iobuf_index, "proxy.config.msg.io.max_buffer_index"); //############################################################################## //# @@ -1216,6 +1344,7 @@ HttpConfig::startup() HttpEstablishStaticConfigLongLong(c.oride.number_of_redirections, "proxy.config.http.number_of_redirections"); HttpEstablishStaticConfigLongLong(c.post_copy_size, "proxy.config.http.post_copy_size"); HttpEstablishStaticConfigStringAlloc(c.redirect_actions_string, "proxy.config.http.redirect.actions"); + HttpEstablishStaticConfigByte(c.http_host_sni_policy, "proxy.config.http.host_sni_policy"); HttpEstablishStaticConfigStringAlloc(c.oride.ssl_client_sni_policy, "proxy.config.ssl.client.sni_policy"); @@ -1269,9 +1398,9 @@ HttpConfig::reconfigure() if (params->outbound_conntrack.queue_size > 0 && !(params->oride.outbound_conntrack.max > 0 || params->oride.outbound_conntrack.min > 0)) { Warning("'%s' is set, but neither '%s' nor '%s' are " - "set, please correct your records.config", + "set, please correct your %s", OutboundConnTrack::CONFIG_VAR_QUEUE_SIZE.data(), OutboundConnTrack::CONFIG_VAR_MAX.data(), - OutboundConnTrack::CONFIG_VAR_MIN.data()); + OutboundConnTrack::CONFIG_VAR_MIN.data(), ts::filename::RECORDS); } params->oride.attach_server_session_to_client = m_master.oride.attach_server_session_to_client; @@ -1279,8 +1408,8 @@ HttpConfig::reconfigure() params->http_hdr_field_max_size = m_master.http_hdr_field_max_size; if (params->oride.outbound_conntrack.max > 0 && params->oride.outbound_conntrack.max < params->oride.outbound_conntrack.min) { - Warning("'%s' < per_server.min_keep_alive_connections, setting min=max , please correct your records.config", - OutboundConnTrack::CONFIG_VAR_MAX.data()); + Warning("'%s' < per_server.min_keep_alive_connections, setting min=max , please correct your %s", + OutboundConnTrack::CONFIG_VAR_MAX.data(), ts::filename::RECORDS); params->oride.outbound_conntrack.min = params->oride.outbound_conntrack.max; } @@ -1321,10 +1450,11 @@ HttpConfig::reconfigure() params->oride.flow_high_water_mark = params->oride.flow_low_water_mark = 0; } - params->oride.server_session_sharing_match = m_master.oride.server_session_sharing_match; - params->oride.server_min_keep_alive_conns = m_master.oride.server_min_keep_alive_conns; - params->server_session_sharing_pool = m_master.server_session_sharing_pool; - params->oride.keep_alive_post_out = m_master.oride.keep_alive_post_out; + params->oride.server_session_sharing_match = m_master.oride.server_session_sharing_match; + params->oride.server_session_sharing_match_str = ats_strdup(m_master.oride.server_session_sharing_match_str); + params->oride.server_min_keep_alive_conns = m_master.oride.server_min_keep_alive_conns; + params->server_session_sharing_pool = m_master.server_session_sharing_pool; + params->oride.keep_alive_post_out = m_master.oride.keep_alive_post_out; params->oride.keep_alive_no_activity_timeout_in = m_master.oride.keep_alive_no_activity_timeout_in; params->oride.keep_alive_no_activity_timeout_out = m_master.oride.keep_alive_no_activity_timeout_out; @@ -1434,9 +1564,19 @@ HttpConfig::reconfigure() params->keepalive_internal_vc = INT_TO_BOOL(m_master.keepalive_internal_vc); params->oride.cache_open_write_fail_action = m_master.oride.cache_open_write_fail_action; + if (params->oride.cache_open_write_fail_action == CACHE_WL_FAIL_ACTION_READ_RETRY) { + if (params->oride.max_cache_open_read_retries <= 0 || params->oride.max_cache_open_write_retries <= 0) { + Warning("Invalid config, cache_open_write_fail_action (%d), max_cache_open_read_retries (%" PRIu64 "), " + "max_cache_open_write_retries (%" PRIu64 ")", + params->oride.cache_open_write_fail_action, params->oride.max_cache_open_read_retries, + params->oride.max_cache_open_write_retries); + } + } params->oride.cache_when_to_revalidate = m_master.oride.cache_when_to_revalidate; params->max_post_size = m_master.max_post_size; + params->max_payload_iobuf_index = m_master.max_payload_iobuf_index; + params->max_msg_iobuf_index = m_master.max_msg_iobuf_index; params->oride.cache_required_headers = m_master.oride.cache_required_headers; params->oride.cache_range_lookup = INT_TO_BOOL(m_master.oride.cache_range_lookup); @@ -1487,11 +1627,15 @@ HttpConfig::reconfigure() params->post_copy_size = m_master.post_copy_size; params->redirect_actions_string = ats_strdup(m_master.redirect_actions_string); params->redirect_actions_map = parse_redirect_actions(params->redirect_actions_string, params->redirect_actions_self_action); + params->http_host_sni_policy = m_master.http_host_sni_policy; params->oride.ssl_client_sni_policy = ats_strdup(m_master.oride.ssl_client_sni_policy); params->negative_caching_list = m_master.negative_caching_list; + params->oride.host_res_data = m_master.oride.host_res_data; + params->oride.host_res_data.conf_value = ats_strdup(m_master.oride.host_res_data.conf_value); + m_id = configProcessor.set(m_id, params); } diff --git a/proxy/http/HttpConfig.h b/proxy/http/HttpConfig.h index 45f922d0ea8..f4f146b6e15 100644 --- a/proxy/http/HttpConfig.h +++ b/proxy/http/HttpConfig.h @@ -44,6 +44,7 @@ #include "tscore/ink_platform.h" #include "tscore/ink_inet.h" +#include "tscore/ink_resolver.h" #include "tscore/IpMap.h" #include "tscore/Regex.h" #include "string_view" @@ -121,7 +122,6 @@ enum { http_cache_deletes_stat, http_tunnels_stat, - http_throttled_proxy_only_stat, // document size stats http_user_agent_request_header_total_size_stat, @@ -328,9 +328,41 @@ enum { http_origin_connections_throttled_stat, + http_origin_connect_adjust_thread_stat, + http_cache_open_write_adjust_thread_stat, + + http_origin_shutdown_pool_lock_contention, + http_origin_shutdown_migration_failure, + http_origin_shutdown_tunnel_server, + http_origin_shutdown_tunnel_server_no_keep_alive, + http_origin_shutdown_tunnel_server_eos, + http_origin_shutdown_tunnel_server_plugin_tunnel, + http_origin_shutdown_tunnel_server_detach, + http_origin_shutdown_tunnel_client, + http_origin_shutdown_tunnel_transform_read, + http_origin_shutdown_release_no_sharing, + http_origin_shutdown_release_no_server, + http_origin_shutdown_release_no_keep_alive, + http_origin_shutdown_release_invalid_response, + http_origin_shutdown_release_invalid_request, + http_origin_shutdown_release_modified, + http_origin_shutdown_release_misc, + http_origin_shutdown_cleanup_entry, + http_origin_shutdown_tunnel_abort, + http_stat_count }; +enum CacheOpenWriteFailAction_t { + CACHE_WL_FAIL_ACTION_DEFAULT = 0x00, + CACHE_WL_FAIL_ACTION_ERROR_ON_MISS = 0x01, + CACHE_WL_FAIL_ACTION_STALE_ON_REVALIDATE = 0x02, + CACHE_WL_FAIL_ACTION_ERROR_ON_MISS_STALE_ON_REVALIDATE = 0x03, + CACHE_WL_FAIL_ACTION_ERROR_ON_MISS_OR_REVALIDATE = 0x04, + CACHE_WL_FAIL_ACTION_READ_RETRY = 0x05, + TOTAL_CACHE_WL_FAIL_ACTION_TYPES +}; + extern RecRawStatBlock *http_rsb; /* Stats should only be accessed using these macros */ @@ -455,9 +487,9 @@ struct OverridableHttpConfigParams { MgmtByte keep_alive_enabled_out = 1; MgmtByte keep_alive_post_out = 1; // share server sessions for post - MgmtInt server_min_keep_alive_conns = 0; - MgmtByte server_session_sharing_match = TS_SERVER_SESSION_SHARING_MATCH_BOTH; - // MgmtByte share_server_sessions; + MgmtInt server_min_keep_alive_conns = 0; + MgmtByte server_session_sharing_match = 0; + char *server_session_sharing_match_str = nullptr; MgmtByte auth_server_session_private = 1; MgmtByte fwd_proxy_auth_to_parent = 0; MgmtByte uncacheable_requests_bypass_parent = 1; @@ -553,9 +585,9 @@ struct OverridableHttpConfigParams { ///////////////////////////////////////////////// MgmtByte allow_half_open = 1; - ///////////////////////////// - // server verification mode// - ///////////////////////////// + ////////////////////////////// + // server verification mode // + ////////////////////////////// MgmtByte ssl_client_verify_server = 0; char *ssl_client_verify_server_policy = nullptr; char *ssl_client_verify_server_properties = nullptr; @@ -661,7 +693,7 @@ struct OverridableHttpConfigParams { size_t proxy_response_server_string_len = 0; // Updated when server_string is set. /////////////////////////////////////////////////////////////////// - // Global User Agent header // + // Global User Agent header // /////////////////////////////////////////////////////////////////// char *global_user_agent_header = nullptr; // This does not get free'd by us! size_t global_user_agent_header_size = 0; // Updated when user_agent is set. @@ -673,6 +705,9 @@ struct OverridableHttpConfigParams { char *ssl_client_cert_filename = nullptr; char *ssl_client_private_key_filename = nullptr; char *ssl_client_ca_cert_filename = nullptr; + + // Host Resolution order + HostResData host_res_data; }; ///////////////////////////////////////////////////////////// @@ -735,6 +770,9 @@ struct HttpConfigParams : public ConfigInfo { MgmtInt post_copy_size = 2048; MgmtInt max_post_size = 0; + MgmtInt max_payload_iobuf_index = BUFFER_SIZE_INDEX_32K; + MgmtInt max_msg_iobuf_index = BUFFER_SIZE_INDEX_32K; + char *redirect_actions_string = nullptr; IpMap *redirect_actions_map = nullptr; RedirectEnabled::Action redirect_actions_self_action = RedirectEnabled::Action::INVALID; @@ -788,6 +826,8 @@ struct HttpConfigParams : public ConfigInfo { MgmtInt http_request_line_max_size = 65535; MgmtInt http_hdr_field_max_size = 131070; + MgmtByte http_host_sni_policy = 0; + // noncopyable ///////////////////////////////////// // operator = and copy constructor // @@ -811,6 +851,8 @@ class HttpConfig inkcoreapi static HttpConfigParams *acquire(); inkcoreapi static void release(HttpConfigParams *params); + static bool load_server_session_sharing_match(const char *key, MgmtByte &mask); + // parse ssl ports configuration string static HttpConfigPortRange *parse_ports_list(char *ports_str); @@ -838,6 +880,7 @@ inline HttpConfigParams::~HttpConfigParams() ats_free(proxy_response_via_string); ats_free(anonymize_other_header_list); ats_free(oride.body_factory_template_base); + ats_free(oride.server_session_sharing_match_str); ats_free(oride.proxy_response_server_string); ats_free(oride.global_user_agent_header); ats_free(oride.ssl_client_cert_filename); @@ -847,7 +890,33 @@ inline HttpConfigParams::~HttpConfigParams() ats_free(reverse_proxy_no_host_redirect); ats_free(redirect_actions_string); ats_free(oride.ssl_client_sni_policy); + ats_free(oride.host_res_data.conf_value); delete connect_ports; delete redirect_actions_map; } + +/** Enable a dynamic configuration variable. + * + * @param name Configuration var name. + * @param cb Callback to do the actual update of the master record. + * @param cookie Extra data for @a cb + * + * The purpose of this is to unite the different ways and times a configuration variable needs + * to be loaded. These are + * - Process start. + * - Dynamic update. + * - Plugin API update. + * + * @a cb is expected to perform the update. It must return a @c bool which is + * - @c true if the value was changed. + * - @c false if the value was not changed. + * + * Based on that, a run time configuration update is triggered or not. + * + * In addition, this invokes @a cb and passes it the information in the configuration variable + * global table in order to perform the initial loading of the value. No update is triggered for + * that call as it is not needed. + * + */ +extern void Enable_Config_Var(std::string_view const &name, bool (*cb)(const char *, RecDataT, RecData, void *), void *cookie); diff --git a/proxy/http/HttpConnectionCount.cc b/proxy/http/HttpConnectionCount.cc index fde59d0adc4..cd64e8d38d5 100644 --- a/proxy/http/HttpConnectionCount.cc +++ b/proxy/http/HttpConnectionCount.cc @@ -22,28 +22,32 @@ */ #include +#include #include +#include #include "HttpConnectionCount.h" #include "tscore/bwf_std_format.h" #include "tscore/BufferWriter.h" using namespace std::literals; +extern int http_config_cb(const char *, RecDataT, RecData, void *); + OutboundConnTrack::Imp OutboundConnTrack::_imp; OutboundConnTrack::GlobalConfig *OutboundConnTrack::_global_config{nullptr}; const MgmtConverter OutboundConnTrack::MAX_CONV( - [](void *data) -> MgmtInt { return static_cast(*static_cast(data)); }, + [](const void *data) -> MgmtInt { return static_cast(*static_cast(data)); }, [](void *data, MgmtInt i) -> void { *static_cast(data) = static_cast(i); }); const MgmtConverter OutboundConnTrack::MIN_CONV( - [](void *data) -> MgmtInt { return static_cast(*static_cast(data)); }, + [](const void *data) -> MgmtInt { return static_cast(*static_cast(data)); }, [](void *data, MgmtInt i) -> void { *static_cast(data) = static_cast(i); }); // Do integer and string conversions. const MgmtConverter OutboundConnTrack::MATCH_CONV{ - [](void *data) -> MgmtInt { return static_cast(*static_cast(data)); }, + [](const void *data) -> MgmtInt { return static_cast(*static_cast(data)); }, [](void *data, MgmtInt i) -> void { // Problem - the InkAPITest requires being able to set an arbitrary value, so this can either // correctly clamp or pass the regression tests. Currently it passes the tests. @@ -53,8 +57,8 @@ const MgmtConverter OutboundConnTrack::MATCH_CONV{ }, nullptr, nullptr, - [](void *data) -> std::string_view { - auto t = *static_cast(data); + [](const void *data) -> std::string_view { + auto t = *static_cast(data); return t < 0 || t > OutboundConnTrack::MATCH_BOTH ? "Invalid"sv : OutboundConnTrack::MATCH_TYPE_NAME[t]; }, [](void *data, std::string_view src) -> void { @@ -76,51 +80,55 @@ static_assert(OutboundConnTrack::Group::Clock::period::den >= 1000); // Configuration callback functions. namespace { -int +bool Config_Update_Conntrack_Min(const char *name, RecDataT dtype, RecData data, void *cookie) { auto config = static_cast(cookie); if (RECD_INT == dtype) { config->min = data.rec_int; + return true; } - return REC_ERR_OKAY; + return false; } -int +bool Config_Update_Conntrack_Max(const char *name, RecDataT dtype, RecData data, void *cookie) { auto config = static_cast(cookie); if (RECD_INT == dtype) { config->max = data.rec_int; + return true; } - return REC_ERR_OKAY; + return false; } -int +bool Config_Update_Conntrack_Queue_Size(const char *name, RecDataT dtype, RecData data, void *cookie) { auto config = static_cast(cookie); if (RECD_INT == dtype) { config->queue_size = data.rec_int; + return true; } - return REC_ERR_OKAY; + return false; } -int +bool Config_Update_Conntrack_Queue_Delay(const char *name, RecDataT dtype, RecData data, void *cookie) { auto config = static_cast(cookie); if (RECD_INT == dtype && data.rec_int > 0) { config->queue_delay = std::chrono::milliseconds(data.rec_int); + return true; } - return REC_ERR_OKAY; + return false; } -int +bool Config_Update_Conntrack_Match(const char *name, RecDataT dtype, RecData data, void *cookie) { auto config = static_cast(cookie); @@ -130,35 +138,26 @@ Config_Update_Conntrack_Match(const char *name, RecDataT dtype, RecData data, vo std::string_view tag{data.rec_string}; if (OutboundConnTrack::lookup_match_type(tag, match_type)) { config->match = match_type; + return true; } else { OutboundConnTrack::Warning_Bad_Match_Type(tag); } } else { Warning("Invalid type for '%s' - must be 'INT'", OutboundConnTrack::CONFIG_VAR_MATCH.data()); } - return REC_ERR_OKAY; + return false; } -int +bool Config_Update_Conntrack_Alert_Delay(const char *name, RecDataT dtype, RecData data, void *cookie) { auto config = static_cast(cookie); if (RECD_INT == dtype && data.rec_int >= 0) { config->alert_delay = std::chrono::seconds(data.rec_int); + return true; } - return REC_ERR_OKAY; -} - -// Do the initial load of a configuration var by grabbing the raw value from the records data -// and calling the update callback. This must be a function because that's how the records -// interface works. Everything needed is already in the record @a r. -void -Load_Config_Var(RecRecord const *r, void *) -{ - for (auto cb = r->config_meta.update_cb_list; nullptr != cb; cb = cb->next) { - cb->update_cb(r->name, r->data_type, r->data, cb->update_cookie); - } + return false; } } // namespace @@ -169,20 +168,12 @@ OutboundConnTrack::config_init(GlobalConfig *global, TxnConfig *txn) _global_config = global; // remember this for later retrieval. // Per transaction lookup must be done at call time because it changes. - RecRegisterConfigUpdateCb(CONFIG_VAR_MIN.data(), &Config_Update_Conntrack_Min, txn); - RecRegisterConfigUpdateCb(CONFIG_VAR_MAX.data(), &Config_Update_Conntrack_Max, txn); - RecRegisterConfigUpdateCb(CONFIG_VAR_MATCH.data(), &Config_Update_Conntrack_Match, txn); - RecRegisterConfigUpdateCb(CONFIG_VAR_QUEUE_SIZE.data(), &Config_Update_Conntrack_Queue_Size, global); - RecRegisterConfigUpdateCb(CONFIG_VAR_QUEUE_DELAY.data(), &Config_Update_Conntrack_Queue_Delay, global); - RecRegisterConfigUpdateCb(CONFIG_VAR_ALERT_DELAY.data(), &Config_Update_Conntrack_Alert_Delay, global); - - // Load 'em up by firing off the config update callback. - RecLookupRecord(CONFIG_VAR_MIN.data(), &Load_Config_Var, nullptr, true); - RecLookupRecord(CONFIG_VAR_MAX.data(), &Load_Config_Var, nullptr, true); - RecLookupRecord(CONFIG_VAR_MATCH.data(), &Load_Config_Var, nullptr, true); - RecLookupRecord(CONFIG_VAR_QUEUE_SIZE.data(), &Load_Config_Var, nullptr, true); - RecLookupRecord(CONFIG_VAR_QUEUE_DELAY.data(), &Load_Config_Var, nullptr, true); - RecLookupRecord(CONFIG_VAR_ALERT_DELAY.data(), &Load_Config_Var, nullptr, true); + Enable_Config_Var(CONFIG_VAR_MIN, &Config_Update_Conntrack_Min, txn); + Enable_Config_Var(CONFIG_VAR_MAX, &Config_Update_Conntrack_Max, txn); + Enable_Config_Var(CONFIG_VAR_MATCH, &Config_Update_Conntrack_Match, txn); + Enable_Config_Var(CONFIG_VAR_QUEUE_SIZE, &Config_Update_Conntrack_Queue_Size, global); + Enable_Config_Var(CONFIG_VAR_QUEUE_DELAY, &Config_Update_Conntrack_Queue_Delay, global); + Enable_Config_Var(CONFIG_VAR_ALERT_DELAY, &Config_Update_Conntrack_Alert_Delay, global); } OutboundConnTrack::TxnState @@ -381,7 +372,7 @@ OutboundConnTrack::Warning_Bad_Match_Type(std::string_view tag) } void -OutboundConnTrack::TxnState::Note_Unblocked(TxnConfig *config, int count, sockaddr const *addr) +OutboundConnTrack::TxnState::Note_Unblocked(const TxnConfig *config, int count, sockaddr const *addr) { time_t lat; // last alert time (epoch seconds) @@ -397,7 +388,8 @@ OutboundConnTrack::TxnState::Note_Unblocked(TxnConfig *config, int count, sockad } void -OutboundConnTrack::TxnState::Warn_Blocked(TxnConfig *config, int64_t sm_id, int count, sockaddr const *addr, char const *debug_tag) +OutboundConnTrack::TxnState::Warn_Blocked(const TxnConfig *config, int64_t sm_id, int count, sockaddr const *addr, + char const *debug_tag) { bool alert_p = _g->should_alert(); auto blocked = alert_p ? _g->_blocked.exchange(0) : _g->_blocked.load(); diff --git a/proxy/http/HttpConnectionCount.h b/proxy/http/HttpConnectionCount.h index 01ad368d65b..494d2ca3be7 100644 --- a/proxy/http/HttpConnectionCount.h +++ b/proxy/http/HttpConnectionCount.h @@ -184,7 +184,7 @@ class OutboundConnTrack * @param count Current connection count for display in message. * @param addr IP address of the upstream. */ - void Note_Unblocked(TxnConfig *config, int count, const sockaddr *addr); + void Note_Unblocked(const TxnConfig *config, int count, const sockaddr *addr); /** Generate a Warning that a connection was blocked. * @@ -194,7 +194,7 @@ class OutboundConnTrack * @param addr IP address of the upstream. * @param debug_tag Tag to use for the debug message. If no debug message should be generated set this to @c nullptr. */ - void Warn_Blocked(TxnConfig *config, int64_t sm_id, int count, const sockaddr *addr, const char *debug_tag = nullptr); + void Warn_Blocked(const TxnConfig *config, int64_t sm_id, int count, const sockaddr *addr, const char *debug_tag = nullptr); }; /** Get or create the @c Group for the specified session properties. diff --git a/proxy/http/HttpPages.cc b/proxy/http/HttpPages.cc index 44dc0d1e151..37061dad546 100644 --- a/proxy/http/HttpPages.cc +++ b/proxy/http/HttpPages.cc @@ -134,16 +134,16 @@ HttpPagesHandler::dump_tunnel_info(HttpSM *sm) // Col 3 - ndone resp_begin_column(); if (producer.alive && producer.read_vio) { - resp_add("%d", producer.read_vio->ndone); + resp_add("%" PRId64, producer.read_vio->ndone); } else { - resp_add("%d", producer.bytes_read); + resp_add("%" PRId64, producer.bytes_read); } resp_end_column(); // Col 4 - nbytes resp_begin_column(); if (producer.alive && producer.read_vio) { - resp_add("%d", producer.read_vio->nbytes); + resp_add("%" PRId64, producer.read_vio->nbytes); } else { resp_add("-"); } @@ -173,16 +173,16 @@ HttpPagesHandler::dump_tunnel_info(HttpSM *sm) // Col 3 - ndone resp_begin_column(); if (consumer.alive && consumer.write_vio) { - resp_add("%d", consumer.write_vio->ndone); + resp_add("%" PRId64, consumer.write_vio->ndone); } else { - resp_add("%d", consumer.bytes_written); + resp_add("%" PRId64, consumer.bytes_written); } resp_end_column(); // Col 4 - nbytes resp_begin_column(); if (consumer.alive && consumer.write_vio) { - resp_add("%d", consumer.write_vio->nbytes); + resp_add("%" PRId64, consumer.write_vio->nbytes); } else { resp_add("-"); } @@ -191,7 +191,7 @@ HttpPagesHandler::dump_tunnel_info(HttpSM *sm) // Col 5 - read avail resp_begin_column(); if (consumer.alive && consumer.buffer_reader) { - resp_add("%d", consumer.buffer_reader->read_avail()); + resp_add("%" PRId64, consumer.buffer_reader->read_avail()); } else { resp_add("-"); } diff --git a/proxy/http/HttpProxyAPIEnums.h b/proxy/http/HttpProxyAPIEnums.h index 5a94a10bd9f..f0d84557210 100644 --- a/proxy/http/HttpProxyAPIEnums.h +++ b/proxy/http/HttpProxyAPIEnums.h @@ -29,24 +29,37 @@ #pragma once -// This is use to signal apidefs.h to not define these again. -#ifndef _HTTP_PROXY_API_ENUMS_H_ -#define _HTTP_PROXY_API_ENUMS_H_ - /// Server session sharing values - match typedef enum { + TS_SERVER_SESSION_SHARING_MATCH_IP, + TS_SERVER_SESSION_SHARING_MATCH_HOSTONLY, + TS_SERVER_SESSION_SHARING_MATCH_HOSTSNISYNC, + TS_SERVER_SESSION_SHARING_MATCH_SNI, + TS_SERVER_SESSION_SHARING_MATCH_CERT, TS_SERVER_SESSION_SHARING_MATCH_NONE, TS_SERVER_SESSION_SHARING_MATCH_BOTH, - TS_SERVER_SESSION_SHARING_MATCH_IP, - TS_SERVER_SESSION_SHARING_MATCH_HOST + TS_SERVER_SESSION_SHARING_MATCH_HOST, } TSServerSessionSharingMatchType; +typedef enum { + TS_SERVER_SESSION_SHARING_MATCH_MASK_NONE = 0, + TS_SERVER_SESSION_SHARING_MATCH_MASK_IP = 0x1, + TS_SERVER_SESSION_SHARING_MATCH_MASK_HOSTONLY = 0x2, + TS_SERVER_SESSION_SHARING_MATCH_MASK_HOSTSNISYNC = 0x4, + TS_SERVER_SESSION_SHARING_MATCH_MASK_SNI = 0x8, + TS_SERVER_SESSION_SHARING_MATCH_MASK_CERT = 0x10 +} TSServerSessionSharingMatchMask; + /// Server session sharing values - pool typedef enum { TS_SERVER_SESSION_SHARING_POOL_GLOBAL, TS_SERVER_SESSION_SHARING_POOL_THREAD, } TSServerSessionSharingPoolType; +// This is use to signal apidefs.h to not define these again. +#ifndef _HTTP_PROXY_API_ENUMS_H_ +#define _HTTP_PROXY_API_ENUMS_H_ + /// Values for per server outbound connection tracking group definition. /// See proxy.config.http.per_server.match typedef enum { @@ -55,4 +68,5 @@ typedef enum { TS_SERVER_OUTBOUND_MATCH_HOST, TS_SERVER_OUTBOUND_MATCH_BOTH } TSOutboundConnectionMatchType; + #endif diff --git a/proxy/http/HttpProxyServerMain.cc b/proxy/http/HttpProxyServerMain.cc index 48ea161c874..2dbac63a9ac 100644 --- a/proxy/http/HttpProxyServerMain.cc +++ b/proxy/http/HttpProxyServerMain.cc @@ -40,6 +40,7 @@ #include "HttpConnectionCount.h" #include "HttpProxyServerMain.h" #if TS_USE_QUIC == 1 +#include "P_QUICNetProcessor.h" #include "P_QUICNextProtocolAccept.h" #include "http3/Http3SessionAccept.h" #endif @@ -237,12 +238,18 @@ MakeHttpProxyAcceptor(HttpProxyAcceptor &acceptor, HttpProxyPort &port, unsigned quic->enableProtocols(port.m_session_protocol_preference); - // HTTP/3 - quic->registerEndpoint(TS_ALPN_PROTOCOL_HTTP_3, new Http3SessionAccept(accept_opt)); + // HTTP/0.9 over QUIC draft-27 (for interop only, will be removed) + quic->registerEndpoint(TS_ALPN_PROTOCOL_HTTP_QUIC_D27, new Http3SessionAccept(accept_opt)); + + // HTTP/3 draft-27 + quic->registerEndpoint(TS_ALPN_PROTOCOL_HTTP_3_D27, new Http3SessionAccept(accept_opt)); // HTTP/0.9 over QUIC (for interop only, will be removed) quic->registerEndpoint(TS_ALPN_PROTOCOL_HTTP_QUIC, new Http3SessionAccept(accept_opt)); + // HTTP/3 + quic->registerEndpoint(TS_ALPN_PROTOCOL_HTTP_3, new Http3SessionAccept(accept_opt)); + quic->proxyPort = &port; acceptor._accept = quic; #endif @@ -308,9 +315,9 @@ init_accept_HttpProxyServer(int n_accept_threads) * start_HttpProxyServer(). */ void -init_HttpProxyServer(EThread *) +init_HttpProxyServer() { - if (eventProcessor.thread_group[ET_NET]._started == num_of_net_threads) { + if (eventProcessor.has_tg_started(ET_NET)) { std::unique_lock lock(proxyServerMutex); et_net_threads_ready = true; lock.unlock(); diff --git a/proxy/http/HttpProxyServerMain.h b/proxy/http/HttpProxyServerMain.h index 96c461abecc..1de9ced2e98 100644 --- a/proxy/http/HttpProxyServerMain.h +++ b/proxy/http/HttpProxyServerMain.h @@ -38,7 +38,7 @@ void init_accept_HttpProxyServer(int n_accept_threads = 0); /** Checkes whether we can call start_HttpProxyServer(). */ -void init_HttpProxyServer(EThread *); +void init_HttpProxyServer(); /** Start the proxy server. The port data should have been created by @c prep_HttpProxyServer(). diff --git a/proxy/http/HttpSM.cc b/proxy/http/HttpSM.cc index 523feec90cf..9e853ea8b19 100644 --- a/proxy/http/HttpSM.cc +++ b/proxy/http/HttpSM.cc @@ -40,10 +40,12 @@ #include "RemapProcessor.h" #include "Transform.h" #include "P_SSLConfig.h" +#include "P_SSLSNI.h" #include "HttpPages.h" #include "IPAllow.h" #include "tscore/I_Layout.h" #include "tscore/bwf_std_format.h" +#include "ts/sdt.h" #include #include @@ -194,26 +196,10 @@ HttpVCTable::remove_entry(HttpVCTableEntry *e) free_MIOBuffer(e->write_buffer); e->write_buffer = nullptr; } - if (e->read_vio != nullptr && e->read_vio->cont == sm) { - // Cleanup dangling i/o - if (e == sm->get_ua_entry() && sm->get_ua_txn() != nullptr) { - e->read_vio = sm->get_ua_txn()->do_io_read(nullptr, 0, nullptr); - } else if (e == sm->get_server_entry() && sm->get_server_session()) { - e->read_vio = sm->get_server_session()->do_io_read(nullptr, 0, nullptr); - } else { - ink_release_assert(false); - } - } - if (e->write_vio != nullptr && e->write_vio->cont == sm) { - // Cleanup dangling i/o - if (e == sm->get_ua_entry() && sm->get_ua_txn()) { - e->write_vio = sm->get_ua_txn()->do_io_write(nullptr, 0, nullptr); - } else if (e == sm->get_server_entry() && sm->get_server_session()) { - e->write_vio = sm->get_server_session()->do_io_write(nullptr, 0, nullptr); - } else { - ink_release_assert(false); - } - } + // Cannot reach in to checkout the netvc + // for remaining I/O operations because the netvc + // may have been deleted at this point and the pointer + // could be stale. e->read_vio = nullptr; e->write_vio = nullptr; e->vc_handler = nullptr; @@ -243,6 +229,9 @@ HttpVCTable::cleanup_entry(HttpVCTableEntry *e) break; } + if (e->vc_type == HTTP_SERVER_VC) { + HTTP_INCREMENT_DYN_STAT(http_origin_shutdown_cleanup_entry); + } e->vc->do_io_close(); e->vc = nullptr; } @@ -290,8 +279,6 @@ HttpSM::cleanup() api_hooks.clear(); http_parser_clear(&http_parser); - // t_state.content_control.cleanup(); - HttpConfig::release(t_state.http_config_param); m_remap->release(); @@ -311,10 +298,12 @@ HttpSM::destroy() } void -HttpSM::init() +HttpSM::init(bool from_early_data) { milestones[TS_MILESTONE_SM_START] = Thread::get_hrtime(); + _from_early_data = from_early_data; + magic = HTTP_SM_MAGIC_ALIVE; // Unique state machine identifier @@ -341,6 +330,9 @@ HttpSM::init() SET_HANDLER(&HttpSM::main_handler); + // Remember where this SM is running so it gets returned correctly + this->setThreadAffinity(this_ethread()); + #ifdef USE_HTTP_DEBUG_LISTS ink_mutex_acquire(&debug_sm_list_mutex); debug_sm_list.push(this); @@ -354,13 +346,14 @@ HttpSM::set_ua_half_close_flag() ua_txn->set_half_close_flag(true); } -inline void +inline int HttpSM::do_api_callout() { if (hooks_set) { - do_api_callout_internal(); + return do_api_callout_internal(); } else { handle_api_return(); + return 0; } } @@ -386,7 +379,16 @@ HttpSM::state_add_to_list(int event, void * /* data ATS_UNUSED */) } t_state.api_next_action = HttpTransact::SM_ACTION_API_SM_START; - do_api_callout(); + if (do_api_callout() < 0) { + // Didn't get the hook continuation lock. Clear the read and wait for next event + if (ua_entry->read_vio) { + // Seems like ua_entry->read_vio->disable(); should work, but that was + // not sufficient to stop the state machine from processing IO events until the + // TXN_START hooks had completed + ua_entry->read_vio = ua_entry->vc->do_io_read(nullptr, 0, nullptr); + } + return EVENT_CONT; + } return EVENT_DONE; } @@ -453,7 +455,9 @@ HttpSM::attach_client_session(ProxyTransaction *client_vc, IOBufferReader *buffe // It seems to be possible that the ua_txn pointer will go stale before log entries for this HTTP transaction are // generated. Therefore, collect information that may be needed for logging from the ua_txn object at this point. // - _client_transaction_id = ua_txn->get_transaction_id(); + _client_transaction_id = ua_txn->get_transaction_id(); + _client_transaction_priority_weight = ua_txn->get_transaction_priority_weight(); + _client_transaction_priority_dependence = ua_txn->get_transaction_priority_dependence(); { auto p = ua_txn->get_proxy_ssn(); @@ -485,6 +489,7 @@ HttpSM::attach_client_session(ProxyTransaction *client_vc, IOBufferReader *buffe milestones[TS_MILESTONE_TLS_HANDSHAKE_END] = ssl_vc->sslHandshakeEndTime; } } + const char *protocol_str = client_vc->get_protocol_string(); client_protocol = protocol_str ? protocol_str : "-"; @@ -494,6 +499,16 @@ HttpSM::attach_client_session(ProxyTransaction *client_vc, IOBufferReader *buffe debug_on = true; } + t_state.setup_per_txn_configs(); + + ink_assert(ua_txn->get_proxy_ssn()); + ink_assert(ua_txn->get_proxy_ssn()->accept_options); + + // default the upstream IP style host resolution order from inbound + std::copy(std::begin(ua_txn->get_proxy_ssn()->accept_options->host_res_preference), + std::end(ua_txn->get_proxy_ssn()->accept_options->host_res_preference), + std::begin(t_state.my_txn_conf().host_res_data.order)); + start_sub_sm(); // Allocate a user agent entry in the state machine's @@ -525,7 +540,8 @@ HttpSM::attach_client_session(ProxyTransaction *client_vc, IOBufferReader *buffe // this hook maybe asynchronous, we need to disable IO on // client but set the continuation to be the state machine // so if we get an timeout events the sm handles them - ua_entry->read_vio = client_vc->do_io_read(this, 0, buffer_reader->mbuf); + ua_entry->read_vio = client_vc->do_io_read(this, 0, buffer_reader->mbuf); + ua_entry->write_vio = client_vc->do_io_write(this, 0, nullptr); ///////////////////////// // set up timeouts // @@ -536,6 +552,7 @@ HttpSM::attach_client_session(ProxyTransaction *client_vc, IOBufferReader *buffe ++reentrancy_count; // Add our state sm to the sm list state_add_to_list(EVENT_NONE, nullptr); + // This is another external entry point and it is possible for the state machine to get terminated // while down the call chain from @c state_add_to_list. So we need to use the reentrancy_count to // prevent cleanup there and do it here as we return to the external caller. @@ -574,6 +591,7 @@ HttpSM::setup_blind_tunnel_port() t_state.hdr_info.client_request.url_create(&u); u.scheme_set(URL_SCHEME_TUNNEL, URL_LEN_TUNNEL); t_state.hdr_info.client_request.url_set(&u); + if (ssl_vc->has_tunnel_destination()) { const char *tunnel_host = ssl_vc->get_tunnel_host(); t_state.hdr_info.client_request.url_get()->host_set(tunnel_host, strlen(tunnel_host)); @@ -583,7 +601,7 @@ HttpSM::setup_blind_tunnel_port() t_state.hdr_info.client_request.url_get()->port_set(t_state.state_machine->ua_txn->get_netvc()->get_local_port()); } } else { - t_state.hdr_info.client_request.url_get()->host_set(ssl_vc->serverName, strlen(ssl_vc->serverName)); + t_state.hdr_info.client_request.url_get()->host_set(ssl_vc->get_server_name(), strlen(ssl_vc->get_server_name())); t_state.hdr_info.client_request.url_get()->port_set(t_state.state_machine->ua_txn->get_netvc()->get_local_port()); } } @@ -663,7 +681,7 @@ HttpSM::state_read_client_request_header(int event, void *data) } // We need to handle EOS as well as READ_READY because the client - // may have sent all of the data already followed by a fIN and that + // may have sent all of the data already followed by a FIN and that // should be OK. if (ua_raw_buffer_reader != nullptr) { bool do_blind_tunnel = false; @@ -754,6 +772,24 @@ HttpSM::state_read_client_request_header(int event, void *data) case PARSE_RESULT_DONE: SMDebug("http", "[%" PRId64 "] done parsing client request header", sm_id); + if (_from_early_data) { + // Only allow early data for safe methods defined in RFC7231 Section 4.2.1. + // https://tools.ietf.org/html/rfc7231#section-4.2.1 + SMDebug("ssl_early_data", "%d", t_state.hdr_info.client_request.method_get_wksidx()); + if (!HttpTransactHeaders::is_method_safe(t_state.hdr_info.client_request.method_get_wksidx())) { + SMDebug("http", "client request was from early data but is NOT safe"); + call_transact_and_set_next_state(HttpTransact::TooEarly); + return 0; + } else if (!SSLConfigParams::server_allow_early_data_params && + (t_state.hdr_info.client_request.m_http->u.req.m_url_impl->m_len_params > 0 || + t_state.hdr_info.client_request.m_http->u.req.m_url_impl->m_len_query > 0)) { + SMDebug("http", "client request was from early data but HAS parameters"); + call_transact_and_set_next_state(HttpTransact::TooEarly); + return 0; + } + t_state.hdr_info.client_request.mark_early_data(); + } + ua_txn->set_session_active(); if (t_state.hdr_info.client_request.version_get() == HTTPVersion(1, 1) && @@ -765,7 +801,7 @@ HttpSM::state_read_client_request_header(int event, void *data) // When receive an "Expect: 100-continue" request from client, ATS sends a "100 Continue" response to client // immediately, before receive the real response from original server. if ((len == HTTP_LEN_100_CONTINUE) && (strncasecmp(expect, HTTP_VALUE_100_CONTINUE, HTTP_LEN_100_CONTINUE) == 0)) { - int64_t alloc_index = buffer_size_to_index(len_100_continue_response); + int64_t alloc_index = buffer_size_to_index(len_100_continue_response, t_state.http_config_param->max_payload_iobuf_index); if (ua_entry->write_buffer) { free_MIOBuffer(ua_entry->write_buffer); ua_entry->write_buffer = nullptr; @@ -820,7 +856,7 @@ HttpSM::wait_for_full_body() alloc_index = DEFAULT_REQUEST_BUFFER_SIZE_INDEX; } } else { - alloc_index = buffer_size_to_index(t_state.hdr_info.request_content_length); + alloc_index = buffer_size_to_index(t_state.hdr_info.request_content_length, t_state.http_config_param->max_payload_iobuf_index); } MIOBuffer *post_buffer = new_MIOBuffer(alloc_index); IOBufferReader *buf_start = post_buffer->alloc_reader(); @@ -1395,7 +1431,7 @@ plugins required to work with sni_routing. t_state.hdr_info.client_request.url_get()->port_set(t_state.state_machine->ua_txn->get_netvc()->get_local_port()); } } else if (ssl_vc) { - t_state.hdr_info.client_request.url_get()->host_set(ssl_vc->serverName, strlen(ssl_vc->serverName)); + t_state.hdr_info.client_request.url_get()->host_set(ssl_vc->get_server_name(), strlen(ssl_vc->get_server_name())); t_state.hdr_info.client_request.url_get()->port_set(t_state.state_machine->ua_txn->get_netvc()->get_local_port()); } } @@ -1409,7 +1445,7 @@ plugins required to work with sni_routing. callout_state = HTTP_API_IN_CALLOUT; } - MUTEX_TRY_LOCK(lock, cur_hook->m_cont->mutex, mutex->thread_holding); + WEAK_MUTEX_TRY_LOCK(lock, cur_hook->m_cont->mutex, mutex->thread_holding); // Have a mutex but didn't get the lock, reschedule if (!lock.is_locked()) { @@ -1417,7 +1453,7 @@ plugins required to work with sni_routing. HTTP_SM_SET_DEFAULT_HANDLER(&HttpSM::state_api_callout); ink_assert(pending_action == nullptr); pending_action = mutex->thread_holding->schedule_in(this, HRTIME_MSECONDS(10)); - return 0; + return -1; } SMDebug("http", "[%" PRId64 "] calling plugin on hook %s at hook %p", sm_id, HttpDebugNames::get_api_hook_name(cur_hook_id), @@ -1704,7 +1740,7 @@ HttpSM::state_http_server_open(int event, void *data) THREAD_ALLOC_INIT(httpServerSessionAllocator, mutex->thread_holding) : httpServerSessionAllocator.alloc(); session->sharing_pool = static_cast(t_state.http_config_param->server_session_sharing_pool); - session->sharing_match = static_cast(t_state.txn_conf->server_session_sharing_match); + session->sharing_match = static_cast(t_state.txn_conf->server_session_sharing_match); netvc = static_cast(data); session->attach_hostname(t_state.current.server->name); @@ -1717,6 +1753,8 @@ HttpSM::state_http_server_open(int event, void *data) session->new_connection(vc); + ATS_PROBE1(new_origin_server_connection, t_state.current.server->name); + session->state = HSS_ACTIVE; ats_ip_copy(&t_state.server_info.src_addr, netvc->get_local_addr()); @@ -1771,7 +1809,14 @@ HttpSM::state_http_server_open(int event, void *data) case VC_EVENT_INACTIVITY_TIMEOUT: case VC_EVENT_ACTIVE_TIMEOUT: case VC_EVENT_ERROR: - case NET_EVENT_OPEN_FAILED: + case NET_EVENT_OPEN_FAILED: { + if (server_session) { + NetVConnection *vc = server_session->get_netvc(); + if (vc) { + server_connection_provided_cert = vc->provided_cert(); + } + } + t_state.current.state = HttpTransact::CONNECTION_ERROR; // save the errno from the connect fail for future use (passed as negative value, flip back) t_state.current.server->set_connect_fail(event == NET_EVENT_OPEN_FAILED ? -reinterpret_cast(data) : ECONNABORTED); @@ -1803,7 +1848,7 @@ HttpSM::state_http_server_open(int event, void *data) call_transact_and_set_next_state(HttpTransact::HandleResponse); } return 0; - + } default: Error("[HttpSM::state_http_server_open] Unknown event: %d", event); ink_release_assert(0); @@ -2078,7 +2123,7 @@ HttpSM::process_srv_info(HostDBInfo *r) if (!r || !r->is_srv || !r->round_robin) { t_state.dns_info.srv_hostname[0] = '\0'; t_state.dns_info.srv_lookup_success = false; - t_state.txn_conf->srv_enabled = false; + t_state.my_txn_conf().srv_enabled = false; SMDebug("dns_srv", "No SRV records were available, continuing to lookup %s", t_state.dns_info.lookup_name); } else { HostDBRoundRobin *rr = r->rr(); @@ -2090,7 +2135,7 @@ HttpSM::process_srv_info(HostDBInfo *r) if (!srv) { t_state.dns_info.srv_lookup_success = false; t_state.dns_info.srv_hostname[0] = '\0'; - t_state.txn_conf->srv_enabled = false; + t_state.my_txn_conf().srv_enabled = false; SMDebug("dns_srv", "SRV records empty for %s", t_state.dns_info.lookup_name); } else { t_state.dns_info.srv_lookup_success = true; @@ -2207,7 +2252,6 @@ int HttpSM::state_hostdb_lookup(int event, void *data) { STATE_ENTER(&HttpSM::state_hostdb_lookup, event); - // ink_assert (m_origin_server_vc == 0); // REQ_FLAVOR_SCHEDULED_UPDATE can be transformed into // REQ_FLAVOR_REVPROXY @@ -2230,7 +2274,7 @@ HttpSM::state_hostdb_lookup(int event, void *data) opt.flags = (t_state.cache_info.directives.does_client_permit_dns_storing) ? HostDBProcessor::HOSTDB_DO_NOT_FORCE_DNS : HostDBProcessor::HOSTDB_FORCE_DNS_RELOAD; opt.timeout = (t_state.api_txn_dns_timeout_value != -1) ? t_state.api_txn_dns_timeout_value : 0; - opt.host_res_style = ua_txn->get_host_res_style(); + opt.host_res_style = ats_host_res_from(ua_txn->get_netvc()->get_local_addr()->sa_family, t_state.txn_conf->host_res_data.order); Action *dns_lookup_action_handle = hostDBProcessor.getbyname_imm(this, (cb_process_result_pfn)&HttpSM::process_hostdb_info, host_name, 0, opt); @@ -2375,6 +2419,7 @@ HttpSM::state_cache_open_write(int event, void *data) pending_action->cancel(); } if ((pending_action = ua_txn->adjust_thread(this, event, data))) { + HTTP_INCREMENT_DYN_STAT(http_cache_open_write_adjust_thread_stat); return 0; // Go away if we reschedule } } @@ -2395,17 +2440,17 @@ HttpSM::state_cache_open_write(int event, void *data) // for reading if (t_state.redirect_info.redirect_in_process) { SMDebug("http_redirect", "[%" PRId64 "] CACHE_EVENT_OPEN_WRITE_FAILED during redirect follow", sm_id); - t_state.cache_open_write_fail_action = HttpTransact::CACHE_WL_FAIL_ACTION_DEFAULT; + t_state.cache_open_write_fail_action = CACHE_WL_FAIL_ACTION_DEFAULT; t_state.cache_info.write_lock_state = HttpTransact::CACHE_WL_FAIL; break; } - if (t_state.txn_conf->cache_open_write_fail_action == HttpTransact::CACHE_WL_FAIL_ACTION_DEFAULT) { + if (t_state.txn_conf->cache_open_write_fail_action == CACHE_WL_FAIL_ACTION_DEFAULT) { t_state.cache_info.write_lock_state = HttpTransact::CACHE_WL_FAIL; break; } else { t_state.cache_open_write_fail_action = t_state.txn_conf->cache_open_write_fail_action; if (!t_state.cache_info.object_read || - (t_state.cache_open_write_fail_action == HttpTransact::CACHE_WL_FAIL_ACTION_ERROR_ON_MISS_OR_REVALIDATE)) { + (t_state.cache_open_write_fail_action == CACHE_WL_FAIL_ACTION_ERROR_ON_MISS_OR_REVALIDATE)) { // cache miss, set wl_state to fail SMDebug("http", "[%" PRId64 "] cache object read %p, cache_wl_fail_action %d", sm_id, t_state.cache_info.object_read, t_state.cache_open_write_fail_action); @@ -2416,6 +2461,15 @@ HttpSM::state_cache_open_write(int event, void *data) // INTENTIONAL FALL THROUGH // Allow for stale object to be served case CACHE_EVENT_OPEN_READ: + if (!t_state.cache_info.object_read) { + t_state.cache_open_write_fail_action = t_state.txn_conf->cache_open_write_fail_action; + // Note that CACHE_LOOKUP_COMPLETE may be invoked more than once + // if CACHE_WL_FAIL_ACTION_READ_RETRY is configured + ink_assert(t_state.cache_open_write_fail_action == CACHE_WL_FAIL_ACTION_READ_RETRY); + t_state.cache_lookup_result = HttpTransact::CACHE_LOOKUP_NONE; + t_state.cache_info.write_lock_state = HttpTransact::CACHE_WL_READ_RETRY; + break; + } // The write vector was locked and the cache_sm retried // and got the read vector again. cache_sm.cache_read_vc->get_http_info(&t_state.cache_info.object_read); @@ -2679,7 +2733,6 @@ HttpSM::tunnel_handler_post(int event, void *data) if (ua_entry->write_buffer) { free_MIOBuffer(ua_entry->write_buffer); ua_entry->write_buffer = nullptr; - ua_entry->vc->do_io_write(nullptr, 0, nullptr); } if (!p->handler_state) { p->handler_state = HTTP_SM_POST_UA_FAIL; @@ -2709,6 +2762,8 @@ HttpSM::tunnel_handler_post(int event, void *data) handle_post_failure(); break; case HTTP_SM_POST_UA_FAIL: + // Client side failed. Shutdown and go home. No need to communicate back to UA + terminate_sm = true; break; case HTTP_SM_POST_SUCCESS: // It's time to start reading the response @@ -2895,6 +2950,13 @@ HttpSM::tunnel_handler_server(int event, HttpTunnelProducer *p) plugin_tunnel_type == HTTP_NO_PLUGIN_TUNNEL && t_state.txn_conf->keep_alive_enabled_out == 1) { close_connection = false; } else { + if (t_state.current.server->keep_alive != HTTP_KEEPALIVE) { + HTTP_INCREMENT_DYN_STAT(http_origin_shutdown_tunnel_server_no_keep_alive); + } else if (server_entry->eos == true) { + HTTP_INCREMENT_DYN_STAT(http_origin_shutdown_tunnel_server_eos); + } else { + HTTP_INCREMENT_DYN_STAT(http_origin_shutdown_tunnel_server_plugin_tunnel); + } close_connection = true; } @@ -2923,6 +2985,7 @@ HttpSM::tunnel_handler_server(int event, HttpTunnelProducer *p) break; } + HTTP_INCREMENT_DYN_STAT(http_origin_shutdown_tunnel_server); close_connection = true; ink_assert(p->vc_type == HT_HTTP_SERVER); @@ -2991,7 +3054,8 @@ HttpSM::tunnel_handler_server(int event, HttpTunnelProducer *p) p->read_success = true; t_state.current.server->state = HttpTransact::TRANSACTION_COMPLETE; t_state.current.server->abort = HttpTransact::DIDNOT_ABORT; - close_connection = true; + HTTP_INCREMENT_DYN_STAT(http_origin_shutdown_tunnel_server_detach); + close_connection = true; break; case VC_EVENT_READ_READY: @@ -3183,6 +3247,11 @@ HttpSM::tunnel_handler_ua(int event, HttpTunnelConsumer *c) server_session->get_netvc()->set_active_timeout(HRTIME_SECONDS(t_state.txn_conf->background_fill_active_timeout)); } + // Even with the background fill, the client side should go down + c->write_vio = nullptr; + c->vc->do_io_close(EHTTP_ERROR); + c->alive = false; + } else { // No background fill p = c->producer; @@ -3425,9 +3494,6 @@ HttpSM::tunnel_handler_post_ua(int event, HttpTunnelProducer *p) // if it is active timeout case, we need to give another chance to send 408 response; ua_txn->set_active_timeout(HRTIME_SECONDS(t_state.txn_conf->transaction_active_timeout_in)); - p->vc->do_io_write(nullptr, 0, nullptr); - p->vc->do_io_shutdown(IO_SHUTDOWN_READ); - return 0; } // fall through @@ -3439,12 +3505,8 @@ HttpSM::tunnel_handler_post_ua(int event, HttpTunnelProducer *p) // server and close the ua p->handler_state = HTTP_SM_POST_UA_FAIL; set_ua_abort(HttpTransact::ABORTED, event); - tunnel.chain_abort_all(p); server_session = nullptr; - p->read_vio = nullptr; - p->vc->do_io_close(EHTTP_ERROR); - // the in_tunnel status on both the ua & and // it's consumer must already be set to true. Previously // we were setting it again to true but incorrectly in @@ -3452,9 +3514,12 @@ HttpSM::tunnel_handler_post_ua(int event, HttpTunnelProducer *p) hsm_release_assert(ua_entry->in_tunnel == true); if (p->consumer_list.head && p->consumer_list.head->vc_type == HT_TRANSFORM) { hsm_release_assert(post_transform_info.entry->in_tunnel == true); - } else if (server_entry != nullptr) { - hsm_release_assert(server_entry->in_tunnel == true); - } + } // server side may have completed before the user agent side, so it may no longer be in tunnel + + // In the error case, start to take down the client session. There should + // be no reuse here + vc_table.remove_entry(this->ua_entry); + ua_txn->do_io_close(); break; case VC_EVENT_READ_COMPLETE: @@ -3471,8 +3536,7 @@ HttpSM::tunnel_handler_post_ua(int event, HttpTunnelProducer *p) } } - // Initiate another read to watch catch aborts and - // timeouts + // Initiate another read to catch aborts and timeouts. ua_entry->vc_handler = &HttpSM::state_watch_for_client_abort; ua_entry->read_vio = p->vc->do_io_read(this, INT64_MAX, ua_buffer_reader->mbuf); break; @@ -3591,7 +3655,8 @@ HttpSM::tunnel_handler_post_server(int event, HttpTunnelConsumer *c) case VC_EVENT_WRITE_COMPLETE: // Completed successfully - c->write_success = true; + c->write_success = true; + server_entry->in_tunnel = false; break; default: ink_release_assert(0); @@ -3852,6 +3917,7 @@ HttpSM::tunnel_handler_transform_read(int event, HttpTunnelProducer *p) // transform hasn't detached yet. If it is still alive, // don't close the transform vc if (p->self_consumer->alive == false) { + HTTP_INCREMENT_DYN_STAT(http_origin_shutdown_tunnel_transform_read); p->vc->do_io_close(); } p->handler_state = HTTP_SM_TRANSFORM_CLOSED; @@ -3918,6 +3984,49 @@ HttpSM::state_remap_request(int event, void * /* data ATS_UNUSED */) return 0; } +// This check must be called before remap. Otherwise, the client_request host +// name may be changed. +void +HttpSM::check_sni_host() +{ + // Check that the SNI and host name fields match, if it matters + // Issue warning or mark the transaction to be terminated as necessary + int host_len; + const char *host_name = t_state.hdr_info.client_request.host_get(&host_len); + if (host_name && host_len) { + if (ua_txn->support_sni()) { + int host_sni_policy = t_state.http_config_param->http_host_sni_policy; + NetVConnection *netvc = ua_txn->get_netvc(); + if (netvc) { + IpEndpoint ip = netvc->get_remote_endpoint(); + if (SNIConfig::TestClientAction(std::string{host_name, static_cast(host_len)}.c_str(), ip, host_sni_policy) && + host_sni_policy > 0) { + // In a SNI/Host mismatch where the Host would have triggered SNI policy, mark the transaction + // to be considered for rejection after the remap phase passes. Gives the opportunity to conf_remap + // override the policy.:w + // + // + // to be rejected + // in the end_remap logic + const char *sni_value = netvc->get_server_name(); + const char *action_value = host_sni_policy == 2 ? "terminate" : "continue"; + if (!sni_value || sni_value[0] == '\0') { // No SNI + Warning("No SNI for TLS request with hostname %.*s action=%s", host_len, host_name, action_value); + if (host_sni_policy == 2) { + this->t_state.client_connection_enabled = false; + } + } else if (strncmp(host_name, sni_value, host_len) != 0) { // Name mismatch + Warning("SNI/hostname mismatch sni=%s host=%.*s action=%s", sni_value, host_len, host_name, action_value); + if (host_sni_policy == 2) { + this->t_state.client_connection_enabled = false; + } + } + } + } + } + } +} + void HttpSM::do_remap_request(bool run_inline) { @@ -3925,6 +4034,8 @@ HttpSM::do_remap_request(bool run_inline) SMDebug("url_rewrite", "Starting a possible remapping for request [%" PRId64 "]", sm_id); bool ret = remapProcessor.setup_for_remap(&t_state, m_remap); + check_sni_host(); + // Preserve effective url before remap t_state.unmapped_url.create(t_state.hdr_info.client_request.url_get()->m_heap); t_state.unmapped_url.copy(t_state.hdr_info.client_request.url_get()); @@ -3966,18 +4077,6 @@ HttpSM::do_remap_request(bool run_inline) void HttpSM::do_hostdb_lookup() { - /* - ////////////////////////////////////////// - // if a connection to the origin server // - // is currently opened --- close it. // - ////////////////////////////////////////// - if (m_origin_server_vc != 0) { - origin_server_close(CLOSE_CONNECTION); - if (m_response_body_tunnel_buffer_.buf() != 0) - m_response_body_tunnel_buffer_.reset(); - } - */ - ink_assert(t_state.dns_info.lookup_name != nullptr); ink_assert(pending_action == nullptr); @@ -4010,8 +4109,9 @@ HttpSM::do_hostdb_lookup() t_state.hdr_info.client_request.port_get(); opt.flags = (t_state.cache_info.directives.does_client_permit_dns_storing) ? HostDBProcessor::HOSTDB_DO_NOT_FORCE_DNS : HostDBProcessor::HOSTDB_FORCE_DNS_RELOAD; - opt.timeout = (t_state.api_txn_dns_timeout_value != -1) ? t_state.api_txn_dns_timeout_value : 0; - opt.host_res_style = ua_txn->get_host_res_style(); + opt.timeout = (t_state.api_txn_dns_timeout_value != -1) ? t_state.api_txn_dns_timeout_value : 0; + opt.host_res_style = + ats_host_res_from(ua_txn->get_netvc()->get_local_addr()->sa_family, t_state.txn_conf->host_res_data.order); Action *dns_lookup_action_handle = hostDBProcessor.getbyname_imm(this, (cb_process_result_pfn)&HttpSM::process_hostdb_info, host_name, 0, opt); @@ -4045,8 +4145,9 @@ HttpSM::do_hostdb_lookup() opt.port = server_port; opt.flags = (t_state.cache_info.directives.does_client_permit_dns_storing) ? HostDBProcessor::HOSTDB_DO_NOT_FORCE_DNS : HostDBProcessor::HOSTDB_FORCE_DNS_RELOAD; - opt.timeout = (t_state.api_txn_dns_timeout_value != -1) ? t_state.api_txn_dns_timeout_value : 0; - opt.host_res_style = ua_txn->get_host_res_style(); + opt.timeout = (t_state.api_txn_dns_timeout_value != -1) ? t_state.api_txn_dns_timeout_value : 0; + + opt.host_res_style = ats_host_res_from(ua_txn->get_netvc()->get_local_addr()->sa_family, t_state.txn_conf->host_res_data.order); Action *dns_lookup_action_handle = hostDBProcessor.getbyname_imm(this, (cb_process_result_pfn)&HttpSM::process_hostdb_info, t_state.dns_info.lookup_name, 0, opt); @@ -4238,7 +4339,7 @@ HttpSM::parse_range_and_compare(MIMEField *field, int64_t content_length) ; } - if (s < e || start < 0) { + if (s < e) { t_state.range_setup = HttpTransact::RANGE_NONE; goto Lfaild; } @@ -4278,7 +4379,7 @@ HttpSM::parse_range_and_compare(MIMEField *field, int64_t content_length) ; } - if (s < e || end < 0) { + if (s < e) { t_state.range_setup = HttpTransact::RANGE_NONE; goto Lfaild; } @@ -4627,7 +4728,7 @@ HttpSM::send_origin_throttled_response() } static void -set_tls_options(NetVCOptions &opt, OverridableHttpConfigParams *txn_conf) +set_tls_options(NetVCOptions &opt, const OverridableHttpConfigParams *txn_conf) { char *verify_server = nullptr; if (txn_conf->ssl_client_verify_server_policy == nullptr) { @@ -4666,6 +4767,32 @@ set_tls_options(NetVCOptions &opt, OverridableHttpConfigParams *txn_conf) } } +std::string_view +HttpSM::get_outbound_cert() const +{ + const char *cert_name = t_state.txn_conf->ssl_client_cert_filename; + if (cert_name == nullptr) { + cert_name = ""; + } + return std::string_view(cert_name); +} + +std::string_view +HttpSM::get_outbound_sni() const +{ + const char *sni_name = nullptr; + size_t len = 0; + if (t_state.txn_conf->ssl_client_sni_policy == nullptr || !strcmp(t_state.txn_conf->ssl_client_sni_policy, "host")) { + // By default the host header field value is used for the SNI. + sni_name = t_state.hdr_info.server_request.host_get(reinterpret_cast(&len)); + } else { + // If other is specified, like "remap" and "verify_with_name_source", the remapped origin name is used for the SNI value + len = strlen(t_state.server_info.name); + sni_name = t_state.server_info.name; + } + return std::string_view(sni_name, len); +} + ////////////////////////////////////////////////////////////////////////// // // HttpSM::do_http_server_open() @@ -4681,6 +4808,7 @@ HttpSM::do_http_server_open(bool raw) // Make sure we are on the "right" thread if (ua_txn) { if ((pending_action = ua_txn->adjust_thread(this, EVENT_INTERVAL, nullptr))) { + HTTP_INCREMENT_DYN_STAT(http_origin_connect_adjust_thread_stat); return; // Go away if we reschedule } } @@ -4770,6 +4898,12 @@ HttpSM::do_http_server_open(bool raw) } } + // Check for self loop. + if (HttpTransact::will_this_request_self_loop(&t_state)) { + call_transact_and_set_next_state(HttpTransact::SelfLoop); + return; + } + // If this is not a raw connection, we try to get a session from the // shared session pool. Raw connections are for SSLs tunnel and // require a new connection @@ -4807,7 +4941,7 @@ HttpSM::do_http_server_open(bool raw) set_server_session_private(true); } - if (raw == false && TS_SERVER_SESSION_SHARING_MATCH_NONE != t_state.txn_conf->server_session_sharing_match && + if ((raw == false) && TS_SERVER_SESSION_SHARING_MATCH_NONE != t_state.txn_conf->server_session_sharing_match && (t_state.txn_conf->keep_alive_post_out == 1 || t_state.hdr_info.request_content_length == 0) && !is_private() && ua_txn != nullptr) { HSMresult_t shared_result; @@ -4958,7 +5092,13 @@ HttpSM::do_http_server_open(bool raw) opt.ip_family = ip_family; + int scheme_to_use = t_state.scheme; // get initial scheme + bool tls_upstream = scheme_to_use == URL_WKSIDX_HTTPS; if (ua_txn) { + SSLNetVConnection *ssl_vc = dynamic_cast(ua_txn->get_netvc()); + if (ssl_vc && raw) { + tls_upstream = ssl_vc->upstream_tls(); + } opt.local_port = ua_txn->get_outbound_port(); const IpAddr &outbound_ip = AF_INET6 == opt.ip_family ? ua_txn->get_outbound_ip6() : ua_txn->get_outbound_ip4(); @@ -4981,8 +5121,6 @@ HttpSM::do_http_server_open(bool raw) } } - int scheme_to_use = t_state.scheme; // get initial scheme - if (!t_state.is_websocket) { // if not websocket, then get scheme from server request int new_scheme_to_use = t_state.hdr_info.server_request.url_get()->scheme_get_wksidx(); // if the server_request url scheme was never set, try the client_request @@ -4992,29 +5130,36 @@ HttpSM::do_http_server_open(bool raw) if (new_scheme_to_use >= 0) { // found a new scheme, use it scheme_to_use = new_scheme_to_use; } + if (!raw || !tls_upstream) { + tls_upstream = scheme_to_use == URL_WKSIDX_HTTPS; + } } // draft-stenberg-httpbis-tcp recommends only enabling TFO on indempotent methods or // those with intervening protocol layers (eg. TLS). - if (scheme_to_use == URL_WKSIDX_HTTPS || HttpTransactHeaders::is_method_idempotent(t_state.method)) { + if (tls_upstream || HttpTransactHeaders::is_method_idempotent(t_state.method)) { opt.f_tcp_fastopen = (t_state.txn_conf->sock_option_flag_out & NetVCOptions::SOCK_OPT_TCP_FAST_OPEN); } - opt.ssl_client_cert_name = t_state.txn_conf->ssl_client_cert_filename; + + opt.set_ssl_client_cert_name(t_state.txn_conf->ssl_client_cert_filename); opt.ssl_client_private_key_name = t_state.txn_conf->ssl_client_private_key_filename; opt.ssl_client_ca_cert_name = t_state.txn_conf->ssl_client_ca_cert_filename; - if (scheme_to_use == URL_WKSIDX_HTTPS) { + if (tls_upstream) { SMDebug("http", "calling sslNetProcessor.connect_re"); + std::string_view sni_name = this->get_outbound_sni(); + if (sni_name.length() > 0) { + opt.set_sni_servername(sni_name.data(), sni_name.length()); + } int len = 0; - if (t_state.txn_conf->ssl_client_sni_policy != nullptr && !strcmp(t_state.txn_conf->ssl_client_sni_policy, "remap")) { - len = strlen(t_state.server_info.name); - opt.set_sni_servername(t_state.server_info.name, len); - } else { // Do the default of host header for SNI + if (t_state.txn_conf->ssl_client_sni_policy != nullptr && + !strcmp(t_state.txn_conf->ssl_client_sni_policy, "verify_with_name_source")) { + // also set sni_hostname with host header from server request in this policy const char *host = t_state.hdr_info.server_request.host_get(&len); if (host && len > 0) { - opt.set_sni_servername(host, len); + opt.set_sni_hostname(host, len); } } if (t_state.server_info.name) { @@ -5039,7 +5184,7 @@ HttpSM::do_http_server_open(bool raw) return; } -void +int HttpSM::do_api_callout_internal() { switch (t_state.api_next_action) { @@ -5080,7 +5225,7 @@ HttpSM::do_api_callout_internal() case HttpTransact::SM_ACTION_API_SM_SHUTDOWN: if (callout_state == HTTP_API_IN_CALLOUT || callout_state == HTTP_API_DEFERED_SERVER_ERROR) { callout_state = HTTP_API_DEFERED_CLOSE; - return; + return 0; } else { cur_hook_id = TS_HTTP_TXN_CLOSE_HOOK; } @@ -5093,7 +5238,7 @@ HttpSM::do_api_callout_internal() hook_state.init(cur_hook_id, http_global_hooks, ua_txn ? ua_txn->feature_hooks() : nullptr, &api_hooks); cur_hook = nullptr; cur_hooks = 0; - state_api_callout(0, nullptr); + return state_api_callout(0, nullptr); } VConnection * @@ -5281,6 +5426,23 @@ HttpSM::release_server_session(bool serve_from_cache) ua_txn->attach_server_session(server_session, false); } } else { + if (TS_SERVER_SESSION_SHARING_MATCH_NONE == t_state.txn_conf->server_session_sharing_match) { + HTTP_INCREMENT_DYN_STAT(http_origin_shutdown_release_no_sharing); + } else if (t_state.current.server == nullptr) { + HTTP_INCREMENT_DYN_STAT(http_origin_shutdown_release_no_server); + } else if (t_state.current.server->keep_alive != HTTP_KEEPALIVE) { + HTTP_INCREMENT_DYN_STAT(http_origin_shutdown_release_no_keep_alive); + } else if (!t_state.hdr_info.server_response.valid()) { + HTTP_INCREMENT_DYN_STAT(http_origin_shutdown_release_invalid_response); + } else if (!t_state.hdr_info.server_request.valid()) { + HTTP_INCREMENT_DYN_STAT(http_origin_shutdown_release_invalid_request); + } else if (t_state.hdr_info.server_response.status_get() != HTTP_STATUS_NOT_MODIFIED && + (t_state.hdr_info.server_request.method_get_wksidx() != HTTP_WKSIDX_HEAD || + t_state.www_auth_content == HttpTransact::CACHE_AUTH_NONE)) { + HTTP_INCREMENT_DYN_STAT(http_origin_shutdown_release_modified); + } else { + HTTP_INCREMENT_DYN_STAT(http_origin_shutdown_release_misc); + } server_session->do_io_close(); } @@ -5363,13 +5525,16 @@ HttpSM::handle_http_server_open() // server session's first transaction. if (nullptr != server_session) { NetVConnection *vc = server_session->get_netvc(); - if (vc != nullptr && (vc->options.sockopt_flags != t_state.txn_conf->sock_option_flag_out || - vc->options.packet_mark != t_state.txn_conf->sock_packet_mark_out || - vc->options.packet_tos != t_state.txn_conf->sock_packet_tos_out)) { - vc->options.sockopt_flags = t_state.txn_conf->sock_option_flag_out; - vc->options.packet_mark = t_state.txn_conf->sock_packet_mark_out; - vc->options.packet_tos = t_state.txn_conf->sock_packet_tos_out; - vc->apply_options(); + if (vc) { + server_connection_provided_cert = vc->provided_cert(); + if (vc->options.sockopt_flags != t_state.txn_conf->sock_option_flag_out || + vc->options.packet_mark != t_state.txn_conf->sock_packet_mark_out || + vc->options.packet_tos != t_state.txn_conf->sock_packet_tos_out) { + vc->options.sockopt_flags = t_state.txn_conf->sock_option_flag_out; + vc->options.packet_mark = t_state.txn_conf->sock_packet_mark_out; + vc->options.packet_tos = t_state.txn_conf->sock_packet_tos_out; + vc->apply_options(); + } } } @@ -5528,7 +5693,7 @@ HttpSM::setup_transform_to_server_transfer() ink_assert(post_transform_info.entry->vc == post_transform_info.vc); int64_t nbytes = t_state.hdr_info.transform_request_cl; - int64_t alloc_index = buffer_size_to_index(nbytes); + int64_t alloc_index = buffer_size_to_index(nbytes, t_state.http_config_param->max_payload_iobuf_index); MIOBuffer *post_buffer = new_MIOBuffer(alloc_index); IOBufferReader *buf_start = post_buffer->alloc_reader(); @@ -5591,7 +5756,7 @@ HttpSM::do_setup_post_tunnel(HttpVC_t to_vc_type) (t_state.redirect_info.redirect_in_process && enable_redirection && this->_postbuf.postdata_copy_buffer_start != nullptr)) { post_redirect = true; // copy the post data into a new producer buffer for static producer - MIOBuffer *postdata_producer_buffer = new_empty_MIOBuffer(); + MIOBuffer *postdata_producer_buffer = new_empty_MIOBuffer(t_state.http_config_param->max_payload_iobuf_index); IOBufferReader *postdata_producer_reader = postdata_producer_buffer->alloc_reader(); postdata_producer_buffer->write(this->_postbuf.postdata_copy_buffer_start); @@ -5608,7 +5773,8 @@ HttpSM::do_setup_post_tunnel(HttpVC_t to_vc_type) alloc_index = DEFAULT_REQUEST_BUFFER_SIZE_INDEX; } } else { - alloc_index = buffer_size_to_index(t_state.hdr_info.request_content_length); + alloc_index = + buffer_size_to_index(t_state.hdr_info.request_content_length, t_state.http_config_param->max_payload_iobuf_index); } MIOBuffer *post_buffer = new_MIOBuffer(alloc_index); IOBufferReader *buf_start = post_buffer->alloc_reader(); @@ -5842,6 +6008,17 @@ HttpSM::attach_server_session(Http1ServerSession *s) hsm_release_assert(s->state == HSS_ACTIVE); server_session = s; server_transact_count = server_session->transact_count++; + + // update the dst_addr when using an existing session + // for e.g using Host based session pools may ignore the DNS IP + if (!ats_ip_addr_eq(&t_state.current.server->dst_addr, &server_session->get_server_ip())) { + ip_port_text_buffer ipb1, ipb2; + Debug("http_ss", "updating ip when attaching server session from %s to %s", + ats_ip_ntop(&t_state.current.server->dst_addr.sa, ipb1, sizeof(ipb1)), + ats_ip_ntop(&server_session->get_server_ip(), ipb2, sizeof(ipb2))); + ats_ip_copy(&t_state.current.server->dst_addr, &server_session->get_server_ip()); + } + // Propagate the per client IP debugging if (ua_txn) { s->get_netvc()->control_flags.set_flags(get_cont_flags().get_flags()); @@ -5975,6 +6152,13 @@ HttpSM::setup_server_send_request() milestones[TS_MILESTONE_SERVER_BEGIN_WRITE] = Thread::get_hrtime(); server_entry->write_vio = server_entry->vc->do_io_write(this, hdr_length, buf_start); + + // Make sure the VC is using correct timeouts. We may be reusing a previously used server session + if (t_state.api_txn_no_activity_timeout_value != -1) { + server_session->get_netvc()->set_inactivity_timeout(HRTIME_MSECONDS(t_state.api_txn_no_activity_timeout_value)); + } else { + server_session->get_netvc()->set_inactivity_timeout(HRTIME_SECONDS(t_state.txn_conf->transaction_no_activity_timeout_out)); + } } void @@ -6030,7 +6214,8 @@ HttpSM::setup_cache_read_transfer() ink_assert(cache_sm.cache_read_vc != nullptr); doc_size = t_state.cache_info.object_read->object_size_get(); - alloc_index = buffer_size_to_index(doc_size + index_to_buffer_size(HTTP_HEADER_BUFFER_SIZE_INDEX)); + alloc_index = buffer_size_to_index(doc_size + index_to_buffer_size(HTTP_HEADER_BUFFER_SIZE_INDEX), + t_state.http_config_param->max_payload_iobuf_index); #ifndef USE_NEW_EMPTY_MIOBUFFER MIOBuffer *buf = new_MIOBuffer(alloc_index); @@ -6083,7 +6268,7 @@ HttpSM::setup_cache_transfer_to_transform() cache_response_hdr_bytes = t_state.hdr_info.cache_response.length_get(); doc_size = t_state.cache_info.object_read->object_size_get(); - alloc_index = buffer_size_to_index(doc_size); + alloc_index = buffer_size_to_index(doc_size, t_state.http_config_param->max_payload_iobuf_index); MIOBuffer *buf = new_MIOBuffer(alloc_index); IOBufferReader *buf_start = buf->alloc_reader(); @@ -6228,7 +6413,7 @@ HttpSM::setup_internal_transfer(HttpSMHandler handler_arg) int64_t buf_size = index_to_buffer_size(HTTP_HEADER_BUFFER_SIZE_INDEX) + (is_msg_buf_present ? t_state.internal_msg_buffer_size : 0); - MIOBuffer *buf = new_MIOBuffer(buffer_size_to_index(buf_size)); + MIOBuffer *buf = new_MIOBuffer(buffer_size_to_index(buf_size, t_state.http_config_param->max_payload_iobuf_index)); IOBufferReader *buf_start = buf->alloc_reader(); // First write the client response header into the buffer @@ -6296,7 +6481,7 @@ HttpSM::find_http_resp_buffer_size(int64_t content_length) } } else { int64_t buf_size = index_to_buffer_size(HTTP_HEADER_BUFFER_SIZE_INDEX) + content_length; - alloc_index = buffer_size_to_index(buf_size); + alloc_index = buffer_size_to_index(buf_size, t_state.http_config_param->max_payload_iobuf_index); } return alloc_index; @@ -6779,7 +6964,12 @@ HttpSM::kill_this() // the terminate_flag terminate_sm = false; t_state.api_next_action = HttpTransact::SM_ACTION_API_SM_SHUTDOWN; - do_api_callout(); + if (do_api_callout() < 0) { // Failed to get a continuation lock + // Need to hang out until we can complete the TXN_CLOSE hook + terminate_sm = false; + reentrancy_count--; + return; + } } // The reentrancy_count is still valid up to this point since // the api shutdown hook is asynchronous and double frees can @@ -6808,6 +6998,23 @@ HttpSM::kill_this() update_stats(); } + ////////////// + // Log Data // + ////////////// + SMDebug("http_seq", "[HttpSM::update_stats] Logging transaction"); + if (Log::transaction_logging_enabled() && t_state.api_info.logging_enabled) { + LogAccess accessor(this); + + int ret = Log::access(&accessor); + + if (ret & Log::FULL) { + SMDebug("http", "[update_stats] Logging system indicates FULL."); + } + if (ret & Log::FAIL) { + Log::error("failed to log transaction for at least one log object"); + } + } + if (ua_txn) { ua_txn->transaction_done(); } @@ -6826,23 +7033,6 @@ HttpSM::kill_this() HTTP_SM_SET_DEFAULT_HANDLER(nullptr); - ////////////// - // Log Data // - ////////////// - SMDebug("http_seq", "[HttpSM::update_stats] Logging transaction"); - if (Log::transaction_logging_enabled() && t_state.api_info.logging_enabled) { - LogAccess accessor(this); - - int ret = Log::access(&accessor); - - if (ret & Log::FULL) { - SMDebug("http", "[update_stats] Logging system indicates FULL."); - } - if (ret & Log::FAIL) { - Log::error("failed to log transaction for at least one log object"); - } - } - if (redirect_url != nullptr) { ats_free((void *)redirect_url); redirect_url = nullptr; @@ -7150,21 +7340,16 @@ HttpSM::set_next_state() } case HttpTransact::SM_ACTION_REMAP_REQUEST: { - if (!remapProcessor.using_separate_thread()) { - do_remap_request(true); /* run inline */ - SMDebug("url_rewrite", "completed inline remapping request for [%" PRId64 "]", sm_id); - t_state.url_remap_success = remapProcessor.finish_remap(&t_state, m_remap); - if (t_state.next_action == HttpTransact::SM_ACTION_SEND_ERROR_CACHE_NOOP && t_state.transact_return_point == nullptr) { - // It appears that we can now set the next_action to error and transact_return_point to nullptr when - // going through do_remap_request presumably due to a plugin setting an error. In that case, it seems - // that the error message has already been setup, so we can just return and avoid the further - // call_transact_and_set_next_state - } else { - call_transact_and_set_next_state(nullptr); - } + do_remap_request(true); /* run inline */ + SMDebug("url_rewrite", "completed inline remapping request for [%" PRId64 "]", sm_id); + t_state.url_remap_success = remapProcessor.finish_remap(&t_state, m_remap); + if (t_state.next_action == HttpTransact::SM_ACTION_SEND_ERROR_CACHE_NOOP && t_state.transact_return_point == nullptr) { + // It appears that we can now set the next_action to error and transact_return_point to nullptr when + // going through do_remap_request presumably due to a plugin setting an error. In that case, it seems + // that the error message has already been setup, so we can just return and avoid the further + // call_transact_and_set_next_state } else { - HTTP_SM_SET_DEFAULT_HANDLER(&HttpSM::state_remap_request); - do_remap_request(false); /* dont run inline (iow on another thread) */ + call_transact_and_set_next_state(nullptr); } break; } @@ -7722,16 +7907,18 @@ HttpSM::redirect_request(const char *arg_redirect_url, const int arg_redirect_le // will do that in handle_api_return under the // HttpTransact::SM_ACTION_REDIRECT_READ state t_state.parent_result.reset(); - t_state.request_sent_time = 0; - t_state.response_received_time = 0; - t_state.cache_info.write_lock_state = HttpTransact::CACHE_WL_INIT; - t_state.next_action = HttpTransact::SM_ACTION_REDIRECT_READ; + t_state.request_sent_time = 0; + t_state.response_received_time = 0; + t_state.next_action = HttpTransact::SM_ACTION_REDIRECT_READ; // we have a new OS and need to have DNS lookup the new OS t_state.dns_info.lookup_success = false; t_state.force_dns = false; t_state.server_info.clear(); t_state.parent_info.clear(); + // Must reset whether the InkAPI has set the destination address + t_state.api_server_addr_set = false; + if (t_state.txn_conf->cache_http) { t_state.cache_info.object_read = nullptr; } @@ -7905,7 +8092,7 @@ inline bool HttpSM::is_redirect_required() { bool redirect_required = (enable_redirection && (redirection_tries <= t_state.txn_conf->number_of_redirections) && - !HttpTransact::is_cache_hit(t_state.cache_lookup_result)); + !HttpTransact::is_fresh_cache_hit(t_state.cache_lookup_result)); SMDebug("http_redirect", "is_redirect_required %u", redirect_required); diff --git a/proxy/http/HttpSM.h b/proxy/http/HttpSM.h index ab26c66eab7..e3725524457 100644 --- a/proxy/http/HttpSM.h +++ b/proxy/http/HttpSM.h @@ -198,7 +198,7 @@ class PostDataBuffers ~PostDataBuffers(); }; -class HttpSM : public Continuation +class HttpSM : public Continuation, public PluginUserArgs { friend class HttpPagesHandler; friend class CoreUtils; @@ -212,8 +212,10 @@ class HttpSM : public Continuation HttpCacheSM &get_cache_sm(); // Added to get the object of CacheSM YTS Team, yamsat HttpVCTableEntry *get_ua_entry(); // Added to get the ua_entry pointer - YTS-TEAM HttpVCTableEntry *get_server_entry(); // Added to get the server_entry pointer + std::string_view get_outbound_sni() const; + std::string_view get_outbound_cert() const; - void init(); + void init(bool from_early_data = false); void attach_client_session(ProxyTransaction *client_vc_arg, IOBufferReader *buffer_reader); @@ -338,6 +340,10 @@ class HttpSM : public Continuation bool get_postbuf_done(); bool is_postbuf_valid(); + // See if we should allow the transaction + // based on sni and host name header values + void check_sni_host(); + protected: int reentrancy_count = 0; @@ -459,8 +465,8 @@ class HttpSM : public Continuation void do_cache_prepare_action(HttpCacheSM *c_sm, CacheHTTPInfo *object_read_info, bool retry, bool allow_multiple = false); void do_cache_delete_all_alts(Continuation *cont); void do_auth_callout(); - void do_api_callout(); - void do_api_callout_internal(); + int do_api_callout(); + int do_api_callout_internal(); void do_redirect(); void redirect_request(const char *redirect_url, const int redirect_len); void do_drain_request_body(HTTPHdr &response); @@ -520,25 +526,26 @@ class HttpSM : public Continuation public: // TODO: Now that bodies can be empty, should the body counters be set to -1 ? TS-2213 // Stats & Logging Info - int client_request_hdr_bytes = 0; - int64_t client_request_body_bytes = 0; - int server_request_hdr_bytes = 0; - int64_t server_request_body_bytes = 0; - int server_response_hdr_bytes = 0; - int64_t server_response_body_bytes = 0; - int client_response_hdr_bytes = 0; - int64_t client_response_body_bytes = 0; - int cache_response_hdr_bytes = 0; - int64_t cache_response_body_bytes = 0; - int pushed_response_hdr_bytes = 0; - int64_t pushed_response_body_bytes = 0; - bool client_tcp_reused = false; - bool client_ssl_reused = false; - bool client_connection_is_ssl = false; - bool is_internal = false; - bool server_connection_is_ssl = false; - bool is_waiting_for_full_body = false; - bool is_using_post_buffer = false; + int client_request_hdr_bytes = 0; + int64_t client_request_body_bytes = 0; + int server_request_hdr_bytes = 0; + int64_t server_request_body_bytes = 0; + int server_response_hdr_bytes = 0; + int64_t server_response_body_bytes = 0; + int client_response_hdr_bytes = 0; + int64_t client_response_body_bytes = 0; + int cache_response_hdr_bytes = 0; + int64_t cache_response_body_bytes = 0; + int pushed_response_hdr_bytes = 0; + int64_t pushed_response_body_bytes = 0; + int server_connection_provided_cert = 0; + bool client_tcp_reused = false; + bool client_ssl_reused = false; + bool client_connection_is_ssl = false; + bool is_internal = false; + bool server_connection_is_ssl = false; + bool is_waiting_for_full_body = false; + bool is_using_post_buffer = false; std::optional mptcp_state; // Don't initialize, that marks it as "not defined". const char *client_protocol = "-"; const char *client_sec_protocol = "-"; @@ -612,9 +619,28 @@ class HttpSM : public Continuation return _client_transaction_id; } + int + client_transaction_priority_weight() const + { + return _client_transaction_priority_weight; + } + + int + client_transaction_priority_dependence() const + { + return _client_transaction_priority_dependence; + } + + void set_server_netvc_inactivity_timeout(NetVConnection *netvc); + void set_server_netvc_active_timeout(NetVConnection *netvc); + void set_server_netvc_connect_timeout(NetVConnection *netvc); + void rewind_state_machine(); + private: PostDataBuffers _postbuf; int _client_connection_id = -1, _client_transaction_id = -1; + int _client_transaction_priority_weight = -1, _client_transaction_priority_dependence = -1; + bool _from_early_data = false; }; // Function to get the cache_sm object - YTS Team, yamsat diff --git a/proxy/http/HttpSessionAccept.cc b/proxy/http/HttpSessionAccept.cc index ecad5c5a719..e7fd20fb4d5 100644 --- a/proxy/http/HttpSessionAccept.cc +++ b/proxy/http/HttpSessionAccept.cc @@ -51,17 +51,13 @@ HttpSessionAccept::accept(NetVConnection *netvc, MIOBuffer *iobuf, IOBufferReade Http1ClientSession *new_session = THREAD_ALLOC_INIT(http1ClientSessionAllocator, this_ethread()); - // copy over session related data. - new_session->f_outbound_transparent = f_outbound_transparent; - new_session->f_transparent_passthrough = f_transparent_passthrough; - new_session->outbound_ip4 = outbound_ip4; - new_session->outbound_ip6 = outbound_ip6; - new_session->outbound_port = outbound_port; - new_session->host_res_style = ats_host_res_from(client_ip->sa_family, host_res_preference); - new_session->acl = std::move(acl); + new_session->accept_options = static_cast(this); + new_session->acl = std::move(acl); new_session->new_connection(netvc, iobuf, reader); + new_session->trans.upstream_outbound_options = *new_session->accept_options; + return true; } diff --git a/proxy/http/HttpSessionManager.cc b/proxy/http/HttpSessionManager.cc index 4215ca8a9c4..8952a617a09 100644 --- a/proxy/http/HttpSessionManager.cc +++ b/proxy/http/HttpSessionManager.cc @@ -64,79 +64,130 @@ ServerSessionPool::purge() bool ServerSessionPool::match(Http1ServerSession *ss, sockaddr const *addr, CryptoHash const &hostname_hash, - TSServerSessionSharingMatchType match_style) + TSServerSessionSharingMatchMask match_style) { - return TS_SERVER_SESSION_SHARING_MATCH_NONE != - match_style && // if no matching allowed, fail immediately. - // The hostname matches if we're not checking it or it (and the port!) is a match. - (TS_SERVER_SESSION_SHARING_MATCH_IP == match_style || - (ats_ip_port_cast(addr) == ats_ip_port_cast(ss->get_server_ip()) && ss->hostname_hash == hostname_hash)) && - // The IP address matches if we're not checking it or it is a match. - (TS_SERVER_SESSION_SHARING_MATCH_HOST == match_style || ats_ip_addr_port_eq(ss->get_server_ip(), addr)); + bool retval = match_style != 0; + if (retval && (TS_SERVER_SESSION_SHARING_MATCH_MASK_IP & match_style)) { + retval = ats_ip_addr_port_eq(ss->get_server_ip(), addr); + } + if (retval && (TS_SERVER_SESSION_SHARING_MATCH_MASK_HOSTONLY & match_style)) { + retval = (ats_ip_port_cast(addr) == ats_ip_port_cast(ss->get_server_ip()) && ss->hostname_hash == hostname_hash); + } + return retval; +} + +bool +ServerSessionPool::validate_host_sni(HttpSM *sm, NetVConnection *netvc) +{ + bool retval = true; + if (sm->t_state.scheme == URL_WKSIDX_HTTPS) { + // The sni_servername of the connection was set on HttpSM::do_http_server_open + // by fetching the hostname from the server request. So the connection should only + // be reused if the hostname in the new request is the same as the host name in the + // original request + const char *session_sni = netvc->get_sni_servername(); + if (session_sni) { + // TS-4468: If the connection matches, make sure the SNI server + // name (if present) matches the request hostname + int len = 0; + const char *req_host = sm->t_state.hdr_info.server_request.host_get(&len); + retval = strncasecmp(session_sni, req_host, len) == 0; + Debug("http_ss", "validate_host_sni host=%*.s, sni=%s", len, req_host, session_sni); + } + } + return retval; } bool ServerSessionPool::validate_sni(HttpSM *sm, NetVConnection *netvc) { - // TS-4468: If the connection matches, make sure the SNI server - // name (if present) matches the request hostname - int len = 0; - const char *req_host = sm->t_state.hdr_info.server_request.host_get(&len); - // The sni_servername of the connection was set on HttpSM::do_http_server_open - // by fetching the hostname from the server request. So the connection should only - // be reused if the hostname in the new request is the same as the host name in the - // original request - const char *session_sni = netvc->options.sni_servername; - - return ((sm->t_state.scheme != URL_WKSIDX_HTTPS) || !session_sni || strncasecmp(session_sni, req_host, len) == 0); + bool retval = true; + // Verify that the sni name on this connection would match the sni we would have use to create + // a new connection. + // + if (sm->t_state.scheme == URL_WKSIDX_HTTPS) { + const char *session_sni = netvc->get_sni_servername(); + std::string_view proposed_sni = sm->get_outbound_sni(); + Debug("http_ss", "validate_sni proposed_sni=%s, sni=%s", proposed_sni.data(), session_sni); + if (!session_sni || proposed_sni.length() == 0) { + retval = session_sni == nullptr && proposed_sni.length() == 0; + } else { + retval = proposed_sni.compare(session_sni) == 0; + } + } + return retval; +} + +bool +ServerSessionPool::validate_cert(HttpSM *sm, NetVConnection *netvc) +{ + bool retval = true; + // Verify that the cert file associated this connection would match the cert file we would have use to create + // a new connection. + // + if (sm->t_state.scheme == URL_WKSIDX_HTTPS) { + const char *session_cert = netvc->options.ssl_client_cert_name.get(); + std::string_view proposed_cert = sm->get_outbound_cert(); + Debug("http_ss", "validate_cert proposed_cert=%.*s, cert=%s", static_cast(proposed_cert.size()), proposed_cert.data(), + session_cert); + if (!session_cert || proposed_cert.length() == 0) { + retval = session_cert == nullptr && proposed_cert.length() == 0; + } else { + retval = proposed_cert.compare(session_cert) == 0; + } + } + return retval; } HSMresult_t ServerSessionPool::acquireSession(sockaddr const *addr, CryptoHash const &hostname_hash, - TSServerSessionSharingMatchType match_style, HttpSM *sm, Http1ServerSession *&to_return) + TSServerSessionSharingMatchMask match_style, HttpSM *sm, Http1ServerSession *&to_return) { HSMresult_t zret = HSM_NOT_FOUND; to_return = nullptr; - if (TS_SERVER_SESSION_SHARING_MATCH_HOST == match_style) { + if ((TS_SERVER_SESSION_SHARING_MATCH_MASK_HOSTONLY & match_style) && !(TS_SERVER_SESSION_SHARING_MATCH_MASK_IP & match_style)) { // This is broken out because only in this case do we check the host hash first. The range must be checked // to verify an upstream that matches port and SNI name is selected. Walk backwards to select oldest. in_port_t port = ats_ip_port_cast(addr); - FQDNTable::iterator first, last; - // FreeBSD/clang++ bug workaround: explicit cast to super type to make overload work. Not needed on Fedora27 nor gcc. - // Not fixed on FreeBSD as of llvm 6.0.1. - std::tie(first, last) = static_cast(m_fqdn_pool.equal_range(hostname_hash)); - while (last != first) { - --last; - if (port == ats_ip_port_cast(last->get_server_ip()) && validate_sni(sm, last->get_netvc())) { + auto first = m_fqdn_pool.find(hostname_hash); + while (first != m_fqdn_pool.end() && first->hostname_hash == hostname_hash) { + if (port == ats_ip_port_cast(first->get_server_ip()) && + (!(match_style & TS_SERVER_SESSION_SHARING_MATCH_MASK_SNI) || validate_sni(sm, first->get_netvc())) && + (!(match_style & TS_SERVER_SESSION_SHARING_MATCH_MASK_HOSTSNISYNC) || validate_host_sni(sm, first->get_netvc())) && + (!(match_style & TS_SERVER_SESSION_SHARING_MATCH_MASK_CERT) || validate_cert(sm, first->get_netvc()))) { zret = HSM_DONE; break; } + ++first; } if (zret == HSM_DONE) { - to_return = last; - m_fqdn_pool.erase(last); + to_return = first; + m_fqdn_pool.erase(first); m_ip_pool.erase(to_return); } - } else if (TS_SERVER_SESSION_SHARING_MATCH_NONE != match_style) { // matching is not disabled. - IPTable::iterator first, last; - // FreeBSD/clang++ bug workaround: explicit cast to super type to make overload work. Not needed on Fedora27 nor gcc. - // Not fixed on FreeBSD as of llvm 6.0.1. - std::tie(first, last) = static_cast(m_ip_pool.equal_range(addr)); - // The range is all that is needed in the match IP case, otherwise need to scan for matching fqdn. + } else if (TS_SERVER_SESSION_SHARING_MATCH_MASK_IP & match_style) { // matching is not disabled. + auto first = m_ip_pool.find(addr); + // The range is all that is needed in the match IP case, otherwise need to scan for matching fqdn + // And matches the other constraints as well // Note the port is matched as part of the address key so it doesn't need to be checked again. - if (TS_SERVER_SESSION_SHARING_MATCH_IP != match_style) { - while (last != first) { - --last; - if (last->hostname_hash == hostname_hash && validate_sni(sm, last->get_netvc())) { + if (match_style & (~TS_SERVER_SESSION_SHARING_MATCH_MASK_IP)) { + while (first != m_ip_pool.end() && ats_ip_addr_port_eq(first->get_server_ip(), addr)) { + if ((!(match_style & TS_SERVER_SESSION_SHARING_MATCH_MASK_HOSTONLY) || first->hostname_hash == hostname_hash) && + (!(match_style & TS_SERVER_SESSION_SHARING_MATCH_MASK_SNI) || validate_sni(sm, first->get_netvc())) && + (!(match_style & TS_SERVER_SESSION_SHARING_MATCH_MASK_HOSTSNISYNC) || validate_host_sni(sm, first->get_netvc())) && + (!(match_style & TS_SERVER_SESSION_SHARING_MATCH_MASK_CERT) || validate_cert(sm, first->get_netvc()))) { zret = HSM_DONE; break; } + ++first; } + } else if (first != m_ip_pool.end()) { + zret = HSM_DONE; } if (zret == HSM_DONE) { - to_return = last; - m_ip_pool.erase(last); + to_return = first; + m_ip_pool.erase(first); m_fqdn_pool.erase(to_return); } } @@ -279,8 +330,8 @@ HttpSessionManager::acquire_session(Continuation * /* cont ATS_UNUSED */, sockad ProxyTransaction *ua_txn, HttpSM *sm) { Http1ServerSession *to_return = nullptr; - TSServerSessionSharingMatchType match_style = - static_cast(sm->t_state.txn_conf->server_session_sharing_match); + TSServerSessionSharingMatchMask match_style = + static_cast(sm->t_state.txn_conf->server_session_sharing_match); CryptoHash hostname_hash; HSMresult_t retval = HSM_NOT_FOUND; @@ -297,7 +348,12 @@ HttpSessionManager::acquire_session(Continuation * /* cont ATS_UNUSED */, sockad // the IP/hostname here seems a bit redundant too // if (ServerSessionPool::match(to_return, ip, hostname_hash, match_style) && - ServerSessionPool::validate_sni(sm, to_return->get_netvc())) { + (!(match_style & TS_SERVER_SESSION_SHARING_MATCH_MASK_SNI) || + ServerSessionPool::validate_sni(sm, to_return->get_netvc())) && + (!(match_style & TS_SERVER_SESSION_SHARING_MATCH_MASK_HOSTSNISYNC) || + ServerSessionPool::validate_host_sni(sm, to_return->get_netvc())) && + (!(match_style & TS_SERVER_SESSION_SHARING_MATCH_MASK_CERT) || + ServerSessionPool::validate_cert(sm, to_return->get_netvc()))) { Debug("http_ss", "[%" PRId64 "] [acquire session] returning attached session ", to_return->con_id); to_return->state = HSS_ACTIVE; sm->attach_server_session(to_return); @@ -340,17 +396,22 @@ HttpSessionManager::acquire_session(Continuation * /* cont ATS_UNUSED */, sockad UnixNetVConnection *server_vc = dynamic_cast(to_return->get_netvc()); if (server_vc) { UnixNetVConnection *new_vc = server_vc->migrateToCurrentThread(sm, ethread); - if (new_vc->thread != ethread) { - // Failed to migrate, put it back to global session pool - m_g_pool->releaseSession(to_return); - to_return = nullptr; - retval = HSM_NOT_FOUND; - } else if (new_vc != server_vc) { - // The VC migrated, keep things from timing out on us - new_vc->set_inactivity_timeout(new_vc->get_inactivity_timeout()); - to_return->set_netvc(new_vc); + // The VC moved, free up the original one + if (new_vc != server_vc) { + ink_assert(new_vc == nullptr || new_vc->nh != nullptr); + if (!new_vc) { + // Close out to_return, we were't able to get a connection + HTTP_INCREMENT_DYN_STAT(http_origin_shutdown_migration_failure); + to_return->do_io_close(); + to_return = nullptr; + retval = HSM_NOT_FOUND; + } else { + // Keep things from timing out on us + new_vc->set_inactivity_timeout(new_vc->get_inactivity_timeout()); + to_return->set_netvc(new_vc); + } } else { - // The VC moved, keep things from timing out on us + // Keep things from timing out on us server_vc->set_inactivity_timeout(server_vc->get_inactivity_timeout()); } } diff --git a/proxy/http/HttpSessionManager.h b/proxy/http/HttpSessionManager.h index 3a6ff020722..dcb80c9bac4 100644 --- a/proxy/http/HttpSessionManager.h +++ b/proxy/http/HttpSessionManager.h @@ -64,7 +64,9 @@ class ServerSessionPool : public Continuation ServerSessionPool(); /// Handle events from server sessions. int eventHandler(int event, void *data); + static bool validate_host_sni(HttpSM *sm, NetVConnection *netvc); static bool validate_sni(HttpSM *sm, NetVConnection *netvc); + static bool validate_cert(HttpSM *sm, NetVConnection *netvc); protected: using IPTable = IntrusiveHashMap; @@ -74,7 +76,7 @@ class ServerSessionPool : public Continuation /** Check if a session matches address and host name. */ static bool match(Http1ServerSession *ss, sockaddr const *addr, CryptoHash const &host_hash, - TSServerSessionSharingMatchType match_style); + TSServerSessionSharingMatchMask match_style); /** Get a session from the pool. @@ -83,7 +85,7 @@ class ServerSessionPool : public Continuation @return A pointer to the session or @c NULL if not matching session was found. */ - HSMresult_t acquireSession(sockaddr const *addr, CryptoHash const &host_hash, TSServerSessionSharingMatchType match_style, + HSMresult_t acquireSession(sockaddr const *addr, CryptoHash const &host_hash, TSServerSessionSharingMatchMask match_style, HttpSM *sm, Http1ServerSession *&server_session); /** Release a session to to pool. */ diff --git a/proxy/http/HttpTransact.cc b/proxy/http/HttpTransact.cc index 3dbc1760779..7134cd1235a 100644 --- a/proxy/http/HttpTransact.cc +++ b/proxy/http/HttpTransact.cc @@ -33,6 +33,7 @@ #include "HttpDebugNames.h" #include #include "tscore/ParseRules.h" +#include "tscore/Filenames.h" #include "HTTP.h" #include "HdrUtils.h" #include "logging/Log.h" @@ -45,6 +46,16 @@ #include "../IPAllow.h" #include "I_Machine.h" +// Support ip_resolve override. +const MgmtConverter HttpTransact::HOST_RES_CONV{[](const void *data) -> std::string_view { + const HostResData *host_res_data = static_cast(data); + return host_res_data->conf_value; + }, + [](void *data, std::string_view src) -> void { + HostResData *res_data = static_cast(data); + parse_host_res_preference(src.data(), res_data->order); + }}; + static char range_type[] = "multipart/byteranges; boundary=RANGE_SEPARATOR"; #define RANGE_NUMBERS_LENGTH 60 @@ -77,6 +88,201 @@ static char range_type[] = "multipart/byteranges; boundary=RANGE_SEPARATOR"; extern HttpBodyFactory *body_factory; +// wrapper to choose between a remap next hop strategy or use parent.config +// remap next hop strategy is preferred +inline static bool +bypass_ok(HttpTransact::State *s) +{ + bool r = false; + url_mapping *mp = s->url_map.getMapping(); + + if (mp && mp->strategy) { + // remap strategies do not support the TSHttpTxnParentProxySet API. + r = mp->strategy->go_direct; + } else if (s->parent_params) { + r = s->parent_result.bypass_ok(); + } + return r; +} + +// wrapper to choose between a remap next hop strategy or use parent.config +// remap next hop strategy is preferred +inline static bool +is_api_result(HttpTransact::State *s) +{ + bool r = false; + url_mapping *mp = s->url_map.getMapping(); + + if (mp && mp->strategy) { + // remap strategies do not support the TSHttpTxnParentProxySet API. + r = false; + } else if (s->parent_params) { + r = s->parent_result.is_api_result(); + } + return r; +} + +// wrapper to choose between a remap next hop strategy or use parent.config +// remap next hop strategy is preferred +inline static unsigned +max_retries(HttpTransact::State *s, ParentRetry_t method) +{ + unsigned int r = 0; + url_mapping *mp = s->url_map.getMapping(); + + if (mp && mp->strategy) { + // remap strategies does not support unavailable_server_responses + if (method == PARENT_RETRY_SIMPLE) { + r = mp->strategy->max_simple_retries; + } + } else if (s->parent_params) { + r = s->parent_result.max_retries(method); + } + return r; +} + +// wrapper to choose between a remap next hop strategy or use parent.config +// remap next hop strategy is preferred +inline static uint32_t +numParents(HttpTransact::State *s) +{ + uint32_t r = 0; + url_mapping *mp = s->url_map.getMapping(); + + if (mp && mp->strategy) { + r = mp->strategy->num_parents; + } else if (s->parent_params) { + r = s->parent_params->numParents(&s->parent_result); + } + return r; +} + +// wrapper to choose between a remap next hop strategy or use parent.config +// remap next hop strategy is preferred +inline static bool +parent_is_proxy(HttpTransact::State *s) +{ + bool r = false; + url_mapping *mp = s->url_map.getMapping(); + + if (mp && mp->strategy) { + r = mp->strategy->parent_is_proxy; + } else if (s->parent_params) { + r = s->parent_result.parent_is_proxy(); + } + return r; +} + +// wrapper to choose between a remap next hop strategy or use parent.config +// remap next hop strategy is preferred +inline static bool +response_is_retryable(HttpTransact::State *s, HTTPStatus response_code) +{ + bool r = false; + url_mapping *mp = s->url_map.getMapping(); + + if (mp && mp->strategy) { + if (mp->strategy->resp_codes.codes.size() > 0) { + r = mp->strategy->resp_codes.contains(response_code); + } + } else if (s->parent_params) { + r = s->parent_result.response_is_retryable(response_code); + } + return r; +} + +// wrapper to choose between a remap next hop strategy or use parent.config +// remap next hop strategy is preferred +inline static unsigned +retry_type(HttpTransact::State *s) +{ + unsigned r = PARENT_RETRY_NONE; + url_mapping *mp = s->url_map.getMapping(); + + if (mp && mp->strategy) { + if (mp->strategy->resp_codes.codes.size() > 0) { + r = PARENT_RETRY_SIMPLE; + } + } else if (s->parent_params) { + r = s->parent_result.retry_type(); + } + return r; +} + +// wrapper to choose between a remap next hop strategy or use parent.config +// remap next hop strategy is preferred +inline static void +findParent(HttpTransact::State *s) +{ + url_mapping *mp = s->url_map.getMapping(); + + if (mp && mp->strategy) { + return mp->strategy->findNextHop(s->state_machine->sm_id, s->parent_result, s->request_data, s->txn_conf->parent_fail_threshold, + s->txn_conf->parent_retry_time); + } else if (s->parent_params) { + return s->parent_params->findParent(&s->request_data, &s->parent_result, s->txn_conf->parent_fail_threshold, + s->txn_conf->parent_retry_time); + } +} + +// wrapper to choose between a remap next hop strategy or use parent.config +// remap next hop strategy is preferred +inline static void +markParentDown(HttpTransact::State *s) +{ + url_mapping *mp = s->url_map.getMapping(); + + if (mp && mp->strategy) { + return mp->strategy->markNextHopDown(s->state_machine->sm_id, s->parent_result, s->txn_conf->parent_fail_threshold, + s->txn_conf->parent_retry_time); + } else if (s->parent_params) { + return s->parent_params->markParentDown(&s->parent_result, s->txn_conf->parent_fail_threshold, s->txn_conf->parent_retry_time); + } +} + +// wrapper to choose between a remap next hop strategy or use parent.config +// remap next hop strategy is preferred +inline static void +markParentUp(HttpTransact::State *s) +{ + url_mapping *mp = s->url_map.getMapping(); + if (mp && mp->strategy) { + return mp->strategy->markNextHopUp(s->state_machine->sm_id, s->parent_result); + } else if (s->parent_params) { + return s->parent_params->markParentUp(&s->parent_result); + } +} + +// wrapper to choose between a remap next hop strategy or use parent.config +// remap next hop strategy is preferred +inline static bool +parentExists(HttpTransact::State *s) +{ + url_mapping *mp = s->url_map.getMapping(); + if (mp && mp->strategy) { + return mp->strategy->nextHopExists(s->state_machine->sm_id); + } else if (s->parent_params) { + return s->parent_params->parentExists(&s->request_data); + } + return false; +} + +// wrapper to choose between a remap next hop strategy or use parent.config +// remap next hop strategy is preferred +inline static void +nextParent(HttpTransact::State *s) +{ + url_mapping *mp = s->url_map.getMapping(); + if (mp && mp->strategy) { + // NextHop only has a findNextHop() function. + return mp->strategy->findNextHop(s->state_machine->sm_id, s->parent_result, s->request_data, s->txn_conf->parent_fail_threshold, + s->txn_conf->parent_retry_time); + } else if (s->parent_params) { + return s->parent_params->nextParent(&s->request_data, &s->parent_result, s->txn_conf->parent_fail_threshold, + s->txn_conf->parent_retry_time); + } +} + inline static bool is_localhost(const char *name, int len) { @@ -84,6 +290,26 @@ is_localhost(const char *name, int len) return (len == (sizeof(local) - 1)) && (memcmp(name, local, len) == 0); } +inline static bool +is_response_simple_code(HTTPStatus response_code) +{ + if (static_cast(response_code) < 400 || static_cast(response_code) > 499) { + return false; + } + + return true; +} + +inline static bool +is_response_unavailable_code(HTTPStatus response_code) +{ + if (static_cast(response_code) < 500 || static_cast(response_code) > 599) { + return false; + } + + return true; +} + inline static void simple_or_unavailable_server_retry(HttpTransact::State *s) { @@ -91,14 +317,12 @@ simple_or_unavailable_server_retry(HttpTransact::State *s) HTTPStatus server_response = http_hdr_status_get(s->hdr_info.server_response.m_http); TxnDebug("http_trans", "[simple_or_unavailabe_server_retry] server_response = %d, simple_retry_attempts: %d, numParents:%d ", - server_response, s->current.simple_retry_attempts, s->parent_params->numParents(&s->parent_result)); - + server_response, s->current.simple_retry_attempts, numParents(s)); // simple retry is enabled, 0x1 - if ((s->parent_result.retry_type() & PARENT_RETRY_SIMPLE) && - s->current.simple_retry_attempts < s->parent_result.max_retries(PARENT_RETRY_SIMPLE) && - server_response == HTTP_STATUS_NOT_FOUND) { - TxnDebug("http_trans", "RECEIVED A SIMPLE RETRY RESPONSE"); - if (s->current.simple_retry_attempts < s->parent_params->numParents(&s->parent_result)) { + if ((retry_type(s) & PARENT_RETRY_SIMPLE) && is_response_simple_code(server_response) && + s->current.simple_retry_attempts < max_retries(s, PARENT_RETRY_SIMPLE) && response_is_retryable(s, server_response)) { + TxnDebug("parent_select", "RECEIVED A SIMPLE RETRY RESPONSE"); + if (s->current.simple_retry_attempts < numParents(s)) { s->current.state = HttpTransact::PARENT_RETRY; s->current.retry_type = PARENT_RETRY_SIMPLE; return; @@ -108,11 +332,11 @@ simple_or_unavailable_server_retry(HttpTransact::State *s) } } // unavailable server retry is enabled 0x2 - else if ((s->parent_result.retry_type() & PARENT_RETRY_UNAVAILABLE_SERVER) && - s->current.unavailable_server_retry_attempts < s->parent_result.max_retries(PARENT_RETRY_UNAVAILABLE_SERVER) && - s->parent_result.response_is_retryable(server_response)) { + else if ((retry_type(s) & PARENT_RETRY_UNAVAILABLE_SERVER) && is_response_unavailable_code(server_response) && + s->current.unavailable_server_retry_attempts < max_retries(s, PARENT_RETRY_UNAVAILABLE_SERVER) && + response_is_retryable(s, server_response)) { TxnDebug("parent_select", "RECEIVED A PARENT_RETRY_UNAVAILABLE_SERVER RESPONSE"); - if (s->current.unavailable_server_retry_attempts < s->parent_params->numParents(&s->parent_result)) { + if (s->current.unavailable_server_retry_attempts < numParents(s)) { s->current.state = HttpTransact::PARENT_RETRY; s->current.retry_type = PARENT_RETRY_UNAVAILABLE_SERVER; return; @@ -172,7 +396,7 @@ update_cache_control_information_from_config(HttpTransact::State *s) // Less than 0 means it wasn't overridden, so leave it alone. if (s->cache_control.cache_responses_to_cookies >= 0) { - s->txn_conf->cache_responses_to_cookies = s->cache_control.cache_responses_to_cookies; + s->my_txn_conf().cache_responses_to_cookies = s->cache_control.cache_responses_to_cookies; } } @@ -264,13 +488,11 @@ find_server_and_update_current_info(HttpTransact::State *s) s->parent_result.result = PARENT_DIRECT; } else if (s->method == HTTP_WKSIDX_CONNECT && s->http_config_param->disable_ssl_parenting) { if (s->parent_result.result == PARENT_SPECIFIED) { - s->parent_params->nextParent(&s->request_data, &s->parent_result, s->txn_conf->parent_fail_threshold, - s->txn_conf->parent_retry_time); + nextParent(s); } else { - s->parent_params->findParent(&s->request_data, &s->parent_result, s->txn_conf->parent_fail_threshold, - s->txn_conf->parent_retry_time); + findParent(s); } - if (!s->parent_result.is_some() || s->parent_result.is_api_result() || s->parent_result.parent_is_proxy()) { + if (!s->parent_result.is_some() || is_api_result(s) || parent_is_proxy(s)) { TxnDebug("http_trans", "request not cacheable, so bypass parent"); s->parent_result.result = PARENT_DIRECT; } @@ -283,25 +505,21 @@ find_server_and_update_current_info(HttpTransact::State *s) // with respect to whether a request is cacheable or not. // For example, the cache_urls_that_look_dynamic variable. if (s->parent_result.result == PARENT_SPECIFIED) { - s->parent_params->nextParent(&s->request_data, &s->parent_result, s->txn_conf->parent_fail_threshold, - s->txn_conf->parent_retry_time); + nextParent(s); } else { - s->parent_params->findParent(&s->request_data, &s->parent_result, s->txn_conf->parent_fail_threshold, - s->txn_conf->parent_retry_time); + findParent(s); } - if (!s->parent_result.is_some() || s->parent_result.is_api_result() || s->parent_result.parent_is_proxy()) { + if (!s->parent_result.is_some() || is_api_result(s) || parent_is_proxy(s)) { TxnDebug("http_trans", "request not cacheable, so bypass parent"); s->parent_result.result = PARENT_DIRECT; } } else { switch (s->parent_result.result) { case PARENT_UNDEFINED: - s->parent_params->findParent(&s->request_data, &s->parent_result, s->txn_conf->parent_fail_threshold, - s->txn_conf->parent_retry_time); + findParent(s); break; case PARENT_SPECIFIED: - s->parent_params->nextParent(&s->request_data, &s->parent_result, s->txn_conf->parent_fail_threshold, - s->txn_conf->parent_retry_time); + nextParent(s); // Hack! // We already have a parent that failed, if we are now told @@ -318,8 +536,8 @@ find_server_and_update_current_info(HttpTransact::State *s) // 1) the config permitted us to dns the origin server // 2) the config permits us // 3) the parent was not set from API - if (s->http_config_param->no_dns_forward_to_parent == 0 && s->parent_result.bypass_ok() && - s->parent_result.parent_is_proxy() && !s->parent_params->apiParentExists(&s->request_data)) { + if (s->http_config_param->no_dns_forward_to_parent == 0 && bypass_ok(s) && parent_is_proxy(s) && + !s->parent_params->apiParentExists(&s->request_data)) { s->parent_result.result = PARENT_DIRECT; } break; @@ -349,6 +567,13 @@ find_server_and_update_current_info(HttpTransact::State *s) return HttpTransact::HOST_NONE; case PARENT_DIRECT: + // if the configuration does not allow the origin to be dns'd + // we're unable to go direct to the origin. + if (s->http_config_param->no_dns_forward_to_parent) { + Warning("no available parents and the config proxy.config.http.no_dns_just_forward_to_parent, prevents origin lookups."); + s->parent_result.result = PARENT_FAIL; + return HttpTransact::HOST_NONE; + } /* fall through */ default: update_current_info(&s->current, &s->server_info, HttpTransact::ORIGIN_SERVER, (s->current.attempts)++); @@ -569,6 +794,26 @@ HttpTransact::Forbidden(State *s) TRANSACT_RETURN(SM_ACTION_SEND_ERROR_CACHE_NOOP, nullptr); } +void +HttpTransact::SelfLoop(State *s) +{ + TxnDebug("http_trans", "[Loop]" + "Request will selfloop."); + bootstrap_state_variables_from_request(s, &s->hdr_info.client_request); + build_error_response(s, HTTP_STATUS_BAD_REQUEST, "Direct self loop detected", "request#cycle_detected"); + TRANSACT_RETURN(SM_ACTION_SEND_ERROR_CACHE_NOOP, nullptr); +} + +void +HttpTransact::TooEarly(State *s) +{ + TxnDebug("http_trans", "[TooEarly]" + "Early Data method is not safe"); + bootstrap_state_variables_from_request(s, &s->hdr_info.client_request); + build_error_response(s, HTTP_STATUS_TOO_EARLY, "Too Early", "too#early"); + TRANSACT_RETURN(SM_ACTION_SEND_ERROR_CACHE_NOOP, nullptr); +} + void HttpTransact::HandleBlindTunnel(State *s) { @@ -645,6 +890,7 @@ HttpTransact::StartRemapRequest(State *s) TxnDebug("http_trans", "Before Remapping:"); obj_describe(s->hdr_info.client_request.m_http, true); } + DUMP_HEADER("http_hdrs", &s->hdr_info.client_request, s->state_machine_id, "Incoming Request"); if (s->http_config_param->referer_filter_enabled) { s->filter_mask = URL_REMAP_FILTER_REFERER; @@ -654,6 +900,14 @@ HttpTransact::StartRemapRequest(State *s) } TxnDebug("http_trans", "END HttpTransact::StartRemapRequest"); + + TxnDebug("http_trans", "Checking if transaction wants to upgrade"); + if (handle_upgrade_request(s)) { + // everything should be handled by the upgrade handler. + TxnDebug("http_trans", "Transaction will be upgraded by the appropriate upgrade handler."); + return; + } + TRANSACT_RETURN(SM_ACTION_API_PRE_REMAP, HttpTransact::PerformRemap); } @@ -845,6 +1099,9 @@ HttpTransact::EndRemapRequest(State *s) bool HttpTransact::handle_upgrade_request(State *s) { + HTTPHdr &request = s->hdr_info.client_request; + s->method = request.method_get_wksidx(); + // Quickest way to determine that this is defintely not an upgrade. /* RFC 6455 The method of the request MUST be GET, and the HTTP version MUST be at least 1.1. */ @@ -957,7 +1214,7 @@ HttpTransact::handle_websocket_upgrade_pre_remap(State *s) TRANSACT_RETURN(SM_ACTION_SEND_ERROR_CACHE_NOOP, nullptr); } - TRANSACT_RETURN(SM_ACTION_API_READ_REQUEST_HDR, HttpTransact::StartRemapRequest); + TRANSACT_RETURN(SM_ACTION_API_PRE_REMAP, HttpTransact::PerformRemap); } void @@ -1002,7 +1259,7 @@ HttpTransact::ModifyRequest(State *s) bootstrap_state_variables_from_request(s, &request); //////////////////////////////////////////////// - // If there is no scheme default to http // + // If there is no scheme, default to http // //////////////////////////////////////////////// URL *url = request.url_get(); @@ -1071,13 +1328,6 @@ HttpTransact::ModifyRequest(State *s) } TxnDebug("http_trans", "END HttpTransact::ModifyRequest"); - TxnDebug("http_trans", "Checking if transaction wants to upgrade"); - - if (handle_upgrade_request(s)) { - // everything should be handled by the upgrade handler. - TxnDebug("http_trans", "Transaction will be upgraded by the appropriate upgrade handler."); - return; - } TRANSACT_RETURN(SM_ACTION_API_READ_REQUEST_HDR, HttpTransact::StartRemapRequest); } @@ -1118,7 +1368,7 @@ HttpTransact::HandleRequest(State *s) { TxnDebug("http_trans", "START HttpTransact::HandleRequest"); - if (!s->state_machine->is_waiting_for_full_body) { + if (!s->state_machine->is_waiting_for_full_body && !s->state_machine->is_using_post_buffer) { ink_assert(!s->hdr_info.server_request.valid()); HTTP_INCREMENT_DYN_STAT(http_incoming_requests_stat); @@ -1216,11 +1466,6 @@ HttpTransact::HandleRequest(State *s) TRANSACT_RETURN(SM_ACTION_INTERNAL_REQUEST, nullptr); } - // this needs to be called after initializing state variables from request - // it adds the client-ip to the incoming client request. - - DUMP_HEADER("http_hdrs", &s->hdr_info.client_request, s->state_machine_id, "Incoming Request"); - if (s->state_machine->plugin_tunnel_type == HTTP_PLUGIN_AS_INTERCEPT) { setup_plugin_request_intercept(s); return; @@ -1230,7 +1475,7 @@ HttpTransact::HandleRequest(State *s) if (s->txn_conf->srv_enabled) { IpEndpoint addr; ats_ip_pton(s->server_info.name, &addr); - s->txn_conf->srv_enabled = !ats_is_ip(&addr); + s->my_txn_conf().srv_enabled = !ats_is_ip(&addr); } // if the request is a trace or options request, decrement the @@ -1255,7 +1500,7 @@ HttpTransact::HandleRequest(State *s) ats_ip_copy(&s->request_data.dest_ip, &addr); } - if (s->parent_params->parentExists(&s->request_data)) { + if (parentExists(s)) { // If the proxy is behind and firewall and there is no // DNS service available, we just want to forward the request // the parent proxy. In this case, we never find out the @@ -1423,7 +1668,7 @@ HttpTransact::PPDNSLookup(State *s) if (!s->dns_info.lookup_success) { // Mark parent as down due to resolving failure HTTP_INCREMENT_DYN_STAT(http_total_parent_marked_down_count); - s->parent_params->markParentDown(&s->parent_result, s->txn_conf->parent_fail_threshold, s->txn_conf->parent_retry_time); + markParentDown(s); // DNS lookup of parent failed, find next parent or o.s. if (find_server_and_update_current_info(s) == HttpTransact::HOST_NONE) { ink_assert(s->current.request_to == HOST_NONE); @@ -1572,17 +1817,6 @@ HttpTransact::OSDNSLookup(State *s) TRANSACT_RETURN(SM_ACTION_SEND_ERROR_CACHE_NOOP, nullptr); } - // detect whether we are about to self loop. the client may have - // specified the proxy as the origin server (badness). - // Check if this procedure is already done - YTS Team, yamsat - if (!s->request_will_not_selfloop) { - if (will_this_request_self_loop(s)) { - TxnDebug("http_trans", "[OSDNSLookup] request will selfloop - bailing out"); - SET_VIA_STRING(VIA_DETAIL_TUNNEL, VIA_DETAIL_TUNNEL_NO_FORWARD); - TRANSACT_RETURN(SM_ACTION_SEND_ERROR_CACHE_NOOP, nullptr); - } - } - if (!s->dns_info.lookup_success) { // maybe the name can be expanded (e.g cnn -> www.cnn.com) HostNameExpansionError_t host_name_expansion = try_to_expand_host_name(s); @@ -1685,22 +1919,27 @@ HttpTransact::OSDNSLookup(State *s) ink_release_assert(s->http_config_param->redirect_actions_map->contains(s->host_db_info.ip(), reinterpret_cast(&x))); action = static_cast(x); } - switch (action) { - case RedirectEnabled::Action::FOLLOW: + + if (action == RedirectEnabled::Action::FOLLOW) { TxnDebug("http_trans", "[OSDNSLookup] Invalid redirect address. Following"); - break; - case RedirectEnabled::Action::REJECT: - TxnDebug("http_trans", "[OSDNSLookup] Invalid redirect address. Rejecting."); + } else if (action == RedirectEnabled::Action::REJECT || s->hdr_info.server_response.valid() == false) { + if (action == RedirectEnabled::Action::REJECT) { + TxnDebug("http_trans", "[OSDNSLookup] Invalid redirect address. Rejecting."); + } else { + // Invalid server response, since we can't copy it we are going to reject + TxnDebug("http_trans", "[OSDNSLookup] Invalid server response. Rejecting."); + Error("Invalid server response. Rejecting."); + } build_error_response(s, HTTP_STATUS_FORBIDDEN, nullptr, "request#syntax_error"); SET_VIA_STRING(VIA_DETAIL_TUNNEL, VIA_DETAIL_TUNNEL_NO_FORWARD); TRANSACT_RETURN(SM_ACTION_SEND_ERROR_CACHE_NOOP, nullptr); - break; - case RedirectEnabled::Action::RETURN: - TxnDebug("http_trans", "[OSDNSLookup] Configured to return on invalid redirect address."); - // fall-through - default: - // Return this 3xx to the client as-is. - TxnDebug("http_trans", "[OSDNSLookup] Invalid redirect address. Returning."); + } else { + // Return this 3xx to the client as-is + if (action == RedirectEnabled::Action::RETURN) { + TxnDebug("http_trans", "[OSDNSLookup] Configured to return on invalid redirect address."); + } else { + TxnDebug("http_trans", "[OSDNSLookup] Invalid redirect address. Returning."); + } build_response_copy(s, &s->hdr_info.server_response, &s->hdr_info.client_response, s->client_info.http_version); TRANSACT_RETURN(SM_ACTION_INTERNAL_CACHE_NOOP, nullptr); } @@ -1817,11 +2056,6 @@ HttpTransact::DecideCacheLookup(State *s) } } - if (service_transaction_in_proxy_only_mode(s)) { - s->cache_info.action = CACHE_DO_NO_ACTION; - s->current.mode = TUNNELLING_PROXY; - HTTP_INCREMENT_DYN_STAT(http_throttled_proxy_only_stat); - } // at this point the request is ready to continue down the // traffic server path. @@ -1887,7 +2121,12 @@ HttpTransact::DecideCacheLookup(State *s) if (s->redirect_info.redirect_in_process) { // without calling out the CACHE_LOOKUP_COMPLETE_HOOK if (s->txn_conf->cache_http) { - s->cache_info.action = CACHE_DO_WRITE; + if (s->cache_info.write_lock_state == CACHE_WL_FAIL) { + s->cache_info.action = CACHE_PREPARE_TO_WRITE; + s->cache_info.write_lock_state = HttpTransact::CACHE_WL_INIT; + } else if (s->cache_info.write_lock_state == CACHE_WL_SUCCESS) { + s->cache_info.action = CACHE_DO_WRITE; + } } LookupSkipOpenServer(s); } else { @@ -2896,7 +3135,6 @@ HttpTransact::handle_cache_write_lock(State *s) } TRANSACT_RETURN(SM_ACTION_SEND_ERROR_CACHE_NOOP, nullptr); - return; default: s->cache_info.write_status = CACHE_WRITE_LOCK_MISS; remove_ims = true; @@ -2904,14 +3142,25 @@ HttpTransact::handle_cache_write_lock(State *s) } break; case CACHE_WL_READ_RETRY: - // Write failed but retried and got a vector to read - // We need to clean up our state so that transact does - // not assert later on. Then handle the open read hit - // s->request_sent_time = UNDEFINED_TIME; s->response_received_time = UNDEFINED_TIME; s->cache_info.action = CACHE_DO_LOOKUP; - remove_ims = true; + if (!s->cache_info.object_read) { + // Write failed and read retry triggered + // Clean up server_request and re-initiate + // Cache Lookup + ink_assert(s->cache_open_write_fail_action == CACHE_WL_FAIL_ACTION_READ_RETRY); + s->cache_info.write_status = CACHE_WRITE_LOCK_MISS; + StateMachineAction_t next; + next = SM_ACTION_CACHE_LOOKUP; + s->next_action = next; + s->hdr_info.server_request.destroy(); + TRANSACT_RETURN(next, nullptr); + } + // Write failed but retried and got a vector to read + // We need to clean up our state so that transact does + // not assert later on. Then handle the open read hit + remove_ims = true; SET_VIA_STRING(VIA_DETAIL_CACHE_TYPE, VIA_DETAIL_CACHE); break; case CACHE_WL_INIT: @@ -3302,7 +3551,7 @@ HttpTransact::handle_response_from_parent(State *s) // response is from a parent origin server. if (is_response_valid(s, &s->hdr_info.server_response) && s->current.request_to == HttpTransact::PARENT_PROXY) { // check for a retryable response if simple or unavailable server retry are enabled. - if (s->parent_result.retry_type() & (PARENT_RETRY_SIMPLE | PARENT_RETRY_UNAVAILABLE_SERVER)) { + if (retry_type(s) & (PARENT_RETRY_SIMPLE | PARENT_RETRY_UNAVAILABLE_SERVER)) { simple_or_unavailable_server_retry(s); } } @@ -3314,13 +3563,13 @@ HttpTransact::handle_response_from_parent(State *s) s->current.server->connect_result = 0; SET_VIA_STRING(VIA_DETAIL_PP_CONNECT, VIA_DETAIL_PP_SUCCESS); if (s->parent_result.retry) { - s->parent_params->markParentUp(&s->parent_result); + markParentUp(s); } handle_forward_server_connection_open(s); break; case PARENT_RETRY: if (s->current.retry_type == PARENT_RETRY_SIMPLE) { - if (s->current.simple_retry_attempts >= s->parent_result.max_retries(PARENT_RETRY_SIMPLE)) { + if (s->current.simple_retry_attempts >= max_retries(s, PARENT_RETRY_SIMPLE)) { TxnDebug("http_trans", "PARENT_RETRY_SIMPLE: retried all parents, send error to client."); s->current.retry_type = PARENT_RETRY_NONE; } else { @@ -3330,7 +3579,7 @@ HttpTransact::handle_response_from_parent(State *s) next_lookup = find_server_and_update_current_info(s); } } else if (s->current.retry_type == PARENT_RETRY_UNAVAILABLE_SERVER) { - if (s->current.unavailable_server_retry_attempts >= s->parent_result.max_retries(PARENT_RETRY_UNAVAILABLE_SERVER)) { + if (s->current.unavailable_server_retry_attempts >= max_retries(s, PARENT_RETRY_UNAVAILABLE_SERVER)) { TxnDebug("http_trans", "PARENT_RETRY_UNAVAILABLE_SERVER: retried all parents, send error to client."); s->current.retry_type = PARENT_RETRY_NONE; } else { @@ -3338,7 +3587,7 @@ HttpTransact::handle_response_from_parent(State *s) TxnDebug("http_trans", "PARENT_RETRY_UNAVAILABLE_SERVER: marking parent down and trying another."); s->current.retry_type = PARENT_RETRY_NONE; HTTP_INCREMENT_DYN_STAT(http_total_parent_marked_down_count); - s->parent_params->markParentDown(&s->parent_result, s->txn_conf->parent_fail_threshold, s->txn_conf->parent_retry_time); + markParentDown(s); next_lookup = find_server_and_update_current_info(s); } } @@ -3364,7 +3613,7 @@ HttpTransact::handle_response_from_parent(State *s) if (!is_request_retryable(s)) { if (s->current.state != OUTBOUND_CONGESTION) { HTTP_INCREMENT_DYN_STAT(http_total_parent_marked_down_count); - s->parent_params->markParentDown(&s->parent_result, s->txn_conf->parent_fail_threshold, s->txn_conf->parent_retry_time); + markParentDown(s); } s->parent_result.result = PARENT_FAIL; handle_parent_died(s); @@ -3392,7 +3641,7 @@ HttpTransact::handle_response_from_parent(State *s) // us to mark the parent down if (s->current.state == CONNECTION_ERROR) { HTTP_INCREMENT_DYN_STAT(http_total_parent_marked_down_count); - s->parent_params->markParentDown(&s->parent_result, s->txn_conf->parent_fail_threshold, s->txn_conf->parent_retry_time); + markParentDown(s); } // We are done so look for another parent if any next_lookup = find_server_and_update_current_info(s); @@ -3404,7 +3653,7 @@ HttpTransact::handle_response_from_parent(State *s) TxnDebug("http_trans", "[handle_response_from_parent] Error. No more retries."); if (s->current.state == CONNECTION_ERROR) { HTTP_INCREMENT_DYN_STAT(http_total_parent_marked_down_count); - s->parent_params->markParentDown(&s->parent_result, s->txn_conf->parent_fail_threshold, s->txn_conf->parent_retry_time); + markParentDown(s); } s->parent_result.result = PARENT_FAIL; next_lookup = HOST_NONE; @@ -3515,7 +3764,7 @@ HttpTransact::handle_response_from_server(State *s) // Force host resolution to have the same family as the client. // Because this is a transparent connection, we can't switch address // families - that is locked in by the client source address. - s->state_machine->ua_txn->set_host_res_style(ats_host_res_match(&s->current.server->dst_addr.sa)); + ats_force_order_by_family(&s->current.server->dst_addr.sa, s->my_txn_conf().host_res_data.order); return CallOSDNSLookup(s); } else if ((s->dns_info.srv_lookup_success || s->host_db_info.is_rr_elt()) && (s->txn_conf->connect_attempts_rr_retries > 0) && @@ -3704,7 +3953,7 @@ HttpTransact::handle_server_connection_not_open(State *s) ink_assert(s->cache_info.object_read != nullptr); ink_assert(s->cache_info.action == CACHE_DO_UPDATE || s->cache_info.action == CACHE_DO_SERVE); ink_assert(s->internal_msg_buffer == nullptr); - + s->source = SOURCE_CACHE; TxnDebug("http_trans", "[hscno] serving stale doc to client"); build_response_from_cache(s, HTTP_WARNING_CODE_REVALIDATION_FAILED); } else { @@ -5205,7 +5454,8 @@ HttpTransact::check_request_validity(State *s, HTTPHdr *incoming_hdr) HttpTransact::ResponseError_t HttpTransact::check_response_validity(State *s, HTTPHdr *incoming_hdr) { - ink_assert(s->next_hop_scheme == URL_WKSIDX_HTTP || s->next_hop_scheme == URL_WKSIDX_HTTPS); + ink_assert(s->next_hop_scheme == URL_WKSIDX_HTTP || s->next_hop_scheme == URL_WKSIDX_HTTPS || + s->next_hop_scheme == URL_WKSIDX_TUNNEL); if (incoming_hdr == nullptr) { return NON_EXISTANT_RESPONSE_HEADER; @@ -5326,8 +5576,9 @@ HttpTransact::handle_trace_and_options_requests(State *s, HTTPHdr *incoming_hdr) s->free_internal_msg_buffer(); s->internal_msg_buffer_size = req_length * 2; - if (s->internal_msg_buffer_size <= max_iobuffer_size) { - s->internal_msg_buffer_fast_allocator_size = buffer_size_to_index(s->internal_msg_buffer_size); + if (s->internal_msg_buffer_size <= BUFFER_SIZE_FOR_INDEX(s->http_config_param->max_msg_iobuf_index)) { + s->internal_msg_buffer_fast_allocator_size = + buffer_size_to_index(s->internal_msg_buffer_size, s->http_config_param->max_msg_iobuf_index); s->internal_msg_buffer = static_cast(ioBufAllocator[s->internal_msg_buffer_fast_allocator_size].alloc_void()); } else { s->internal_msg_buffer_fast_allocator_size = -1; @@ -6061,8 +6312,8 @@ HttpTransact::is_response_cacheable(State *s, HTTPHdr *request, HTTPHdr *respons // If a ttl is set, allow caching even if response contains // Cache-Control headers to prevent caching if (s->cache_control.ttl_in_cache > 0) { - TxnDebug("http_trans", - "[is_response_cacheable] Cache-control header directives in response overridden by ttl in cache.config"); + TxnDebug("http_trans", "[is_response_cacheable] Cache-control header directives in response overridden by ttl in %s", + ts::filename::CACHE); } else { TxnDebug("http_trans", "[is_response_cacheable] NO by response cache control"); return false; @@ -6325,28 +6576,6 @@ HttpTransact::is_response_valid(State *s, HTTPHdr *incoming_response) } } -/////////////////////////////////////////////////////////////////////////////// -// Name : service_transaction_in_proxy_only_mode -// Description: uses some metric to force this transaction to be proxy-only -// -// Details : -// -// Some metric may be employed to force the traffic server to enter -// a proxy-only mode temporarily. This function is called to determine -// if the current transaction should be proxy-only. The function is -// called from initialize_state_variables_from_request and is used to -// set s->current.mode to TUNNELLING_PROXY and just for safety to set -// s->cache_info.action to CACHE_DO_NO_ACTION. -// -// Currently the function is just a placeholder and always returns false. -// -/////////////////////////////////////////////////////////////////////////////// -bool -HttpTransact::service_transaction_in_proxy_only_mode(State * /* s ATS_UNUSED */) -{ - return false; -} - void HttpTransact::process_quick_http_filter(State *s, int method) { @@ -6463,7 +6692,6 @@ HttpTransact::will_this_request_self_loop(State *s) via_field = via_field->m_next_dup; } } - s->request_will_not_selfloop = true; return false; } @@ -6757,11 +6985,11 @@ HttpTransact::handle_response_keep_alive_headers(State *s, HTTPVersion ver, HTTP // to the client to keep the connection alive. // Insert a Transfer-Encoding header in the response if necessary. - // check that the client is HTTP 1.1 and the conf allows chunking or the client - // protocol unchunks before returning to the user agent (i.e. is http/2) - if (s->client_info.http_version == HTTPVersion(1, 1) && - (s->txn_conf->chunking_enabled == 1 || - (s->state_machine->plugin_tag && (!strncmp(s->state_machine->plugin_tag, "http/2", 6)))) && + // check that the client protocol is HTTP/1.1 and the conf allows chunking or + // the client protocol doesn't support chunked transfer coding (i.e. HTTP/1.0, HTTP/2, and HTTP/3) + if (s->state_machine->ua_txn && s->state_machine->ua_txn->is_chunked_encoding_supported() && + s->client_info.http_version == HTTPVersion(1, 1) && s->txn_conf->chunking_enabled == 1 && + s->state_machine->ua_txn->is_chunked_encoding_supported() && // if we're not sending a body, don't set a chunked header regardless of server response !is_response_body_precluded(s->hdr_info.client_response.status_get(), s->method) && // we do not need chunked encoding for internal error messages @@ -6794,11 +7022,11 @@ HttpTransact::handle_response_keep_alive_headers(State *s, HTTPVersion ver, HTTP // Close the connection if client_info is not keep-alive. // Otherwise, if we cannot trust the content length, we will close the connection - // unless we are going to use chunked encoding or the client issued - // a PUSH request + // unless we are going to use chunked encoding on HTTP/1.1 or the client issued a PUSH request if (s->client_info.keep_alive != HTTP_KEEPALIVE) { ka_action = KA_DISABLED; - } else if (s->hdr_info.trust_response_cl == false && + } else if (s->hdr_info.trust_response_cl == false && s->state_machine->ua_txn && + s->state_machine->ua_txn->is_chunked_encoding_supported() && !(s->client_info.receive_chunked_response == true || (s->method == HTTP_WKSIDX_PUSH && s->client_info.keep_alive == HTTP_KEEPALIVE))) { ka_action = KA_CLOSE; @@ -6942,6 +7170,23 @@ HttpTransact::does_client_request_permit_storing(CacheControlResult *c, HTTPHdr return (true); } +int +HttpTransact::get_max_age(HTTPHdr *response) +{ + int max_age = -1; + uint32_t cc_mask = response->get_cooked_cc_mask(); + + if (cc_mask & MIME_COOKED_MASK_CC_S_MAXAGE) { + // Precedence to s-maxage + max_age = static_cast(response->get_cooked_cc_s_maxage()); + } else if (cc_mask & MIME_COOKED_MASK_CC_MAX_AGE) { + // If s-maxage isn't set, try max-age + max_age = static_cast(response->get_cooked_cc_max_age()); + } + + return max_age; +} + int HttpTransact::calculate_document_freshness_limit(State *s, HTTPHdr *response, time_t response_date, bool *heuristic) { @@ -6949,19 +7194,13 @@ HttpTransact::calculate_document_freshness_limit(State *s, HTTPHdr *response, ti time_t date_value, expires_value, last_modified_value; MgmtInt min_freshness_bounds, max_freshness_bounds; int freshness_limit = 0; - uint32_t cc_mask = response->get_cooked_cc_mask(); + int max_age = get_max_age(response); *heuristic = false; - if (cc_mask & (MIME_COOKED_MASK_CC_S_MAXAGE | MIME_COOKED_MASK_CC_MAX_AGE)) { - if (cc_mask & MIME_COOKED_MASK_CC_S_MAXAGE) { - freshness_limit = static_cast(response->get_cooked_cc_s_maxage()); - TxnDebug("http_match", "calculate_document_freshness_limit --- s_max_age set, freshness_limit = %d", freshness_limit); - } else if (cc_mask & MIME_COOKED_MASK_CC_MAX_AGE) { - freshness_limit = static_cast(response->get_cooked_cc_max_age()); - TxnDebug("http_match", "calculate_document_freshness_limit --- max_age set, freshness_limit = %d", freshness_limit); - } - freshness_limit = std::min(std::max(0, freshness_limit), static_cast(s->txn_conf->cache_guaranteed_max_lifetime)); + if (max_age >= 0) { + freshness_limit = std::min(std::max(0, max_age), static_cast(s->txn_conf->cache_guaranteed_max_lifetime)); + TxnDebug("http_match", "calculate_document_freshness_limit --- freshness_limit = %d", freshness_limit); } else { date_set = last_modified_set = false; @@ -7459,10 +7698,16 @@ HttpTransact::is_request_likely_cacheable(State *s, HTTPHdr *request) return false; } +bool +HttpTransact::is_fresh_cache_hit(CacheLookupResult_t r) +{ + return (r == CACHE_LOOKUP_HIT_FRESH || r == CACHE_LOOKUP_HIT_WARNING); +} + bool HttpTransact::is_cache_hit(CacheLookupResult_t r) { - return (r == CACHE_LOOKUP_HIT_FRESH || r == CACHE_LOOKUP_HIT_WARNING || r == CACHE_LOOKUP_HIT_STALE); + return (is_fresh_cache_hit(r) || r == CACHE_LOOKUP_HIT_STALE); } void @@ -7495,12 +7740,6 @@ HttpTransact::build_request(State *s, HTTPHdr *base_request, HTTPHdr *outgoing_r HttpTransactHeaders::add_global_user_agent_header_to_request(s->txn_conf, outgoing_request); handle_request_keep_alive_headers(s, outgoing_version, outgoing_request); - // handle_conditional_headers appears to be obsolete. Nothing happens - // unless s->cache_info.action == HttpTransact::CACHE_DO_UPDATE. In that - // case an assert will go off. The functionality of this method - // (e.g., setting the if-modfied-since header occurs in issue_revalidate - // HttpTransactHeaders::handle_conditional_headers(&s->cache_info, outgoing_request); - if (s->next_hop_scheme < 0) { s->next_hop_scheme = URL_WKSIDX_HTTP; } @@ -7544,7 +7783,7 @@ HttpTransact::build_request(State *s, HTTPHdr *base_request, HTTPHdr *outgoing_r if (outgoing_request->method_get_wksidx() == HTTP_WKSIDX_CONNECT) { // CONNECT method requires a target in the URL, so always force it from the Host header. outgoing_request->set_url_target_from_host_field(); - } else if (s->current.request_to == PARENT_PROXY && s->parent_result.parent_is_proxy()) { + } else if (s->current.request_to == PARENT_PROXY && parent_is_proxy(s)) { // If we have a parent proxy set the URL target field. if (!outgoing_request->is_target_in_url()) { TxnDebug("http_trans", "[build_request] adding target to URL for parent proxy"); @@ -7586,6 +7825,10 @@ HttpTransact::build_request(State *s, HTTPHdr *base_request, HTTPHdr *outgoing_r TxnDebug("http_trans", "[build_request] request expect 100-continue headers removed"); } + if (base_request->is_early_data()) { + outgoing_request->value_set_int(MIME_FIELD_EARLY_DATA, MIME_LEN_EARLY_DATA, 1); + } + s->request_sent_time = ink_local_time(); s->current.now = s->request_sent_time; @@ -7987,24 +8230,6 @@ HttpTransact::build_redirect_response(State *s) s->arena.str_free(to_free); } -void -HttpTransact::build_upgrade_response(State *s) -{ - TxnDebug("http_upgrade", "[HttpTransact::build_upgrade_response]"); - - // 101 Switching Protocols - HTTPStatus status_code = HTTP_STATUS_SWITCHING_PROTOCOL; - const char *reason_phrase = http_hdr_reason_lookup(status_code); - build_response(s, &s->hdr_info.client_response, s->client_info.http_version, status_code, reason_phrase); - - ////////////////////////// - // set upgrade headers // - ////////////////////////// - HTTPHdr *h = &s->hdr_info.client_response; - h->value_set(MIME_FIELD_CONNECTION, MIME_LEN_CONNECTION, "Upgrade", strlen("Upgrade")); - h->value_set(MIME_FIELD_UPGRADE, MIME_LEN_UPGRADE, MIME_UPGRADE_H2C_TOKEN, strlen(MIME_UPGRADE_H2C_TOKEN)); -} - const char * HttpTransact::get_error_string(int erno) { diff --git a/proxy/http/HttpTransact.h b/proxy/http/HttpTransact.h index 6c713d3e046..966bb7fc842 100644 --- a/proxy/http/HttpTransact.h +++ b/proxy/http/HttpTransact.h @@ -23,6 +23,7 @@ #pragma once +#include "tscore/ink_assert.h" #include "tscore/ink_platform.h" #include "P_HostDB.h" #include "P_Net.h" @@ -40,6 +41,7 @@ #include "UrlMapping.h" #include "records/I_RecHttp.h" #include "ProxySession.h" +#include "MgmtDefs.h" #define HTTP_RELEASE_ASSERT(X) ink_release_assert(X) @@ -245,15 +247,6 @@ class HttpTransact TOTAL_CACHE_ACTION_TYPES }; - enum CacheOpenWriteFailAction_t { - CACHE_WL_FAIL_ACTION_DEFAULT = 0x00, - CACHE_WL_FAIL_ACTION_ERROR_ON_MISS = 0x01, - CACHE_WL_FAIL_ACTION_STALE_ON_REVALIDATE = 0x02, - CACHE_WL_FAIL_ACTION_ERROR_ON_MISS_STALE_ON_REVALIDATE = 0x03, - CACHE_WL_FAIL_ACTION_ERROR_ON_MISS_OR_REVALIDATE = 0x04, - TOTAL_CACHE_WL_FAIL_ACTION_TYPES - }; - enum CacheWriteLock_t { CACHE_WL_INIT, CACHE_WL_SUCCESS, @@ -638,6 +631,7 @@ class HttpTransact bool srv_lookup_success = false; short srv_port = 0; HostDBApplicationInfo srv_app; + /*** Set to true by default. If use_client_target_address is set * to 1, this value will be set to false if the client address is * not in the DNS pool */ @@ -646,6 +640,9 @@ class HttpTransact _DNSLookupInfo() {} } DNSLookupInfo; + // Conversion handling for DNS host resolution type. + static const MgmtConverter HOST_RES_CONV; + typedef struct _HeaderInfo { HTTPHdr client_request; HTTPHdr client_response; @@ -690,7 +687,6 @@ class HttpTransact bool force_dns = false; MgmtByte cache_open_write_fail_action = 0; bool is_revalidation_necessary = false; // Added to check if revalidation is necessary - YTS Team, yamsat - bool request_will_not_selfloop = false; // To determine if process done - YTS Team, yamsat ConnectionAttributes client_info; ConnectionAttributes parent_info; ConnectionAttributes server_info; @@ -714,7 +710,8 @@ class HttpTransact bool first_dns_lookup = true; HttpRequestData request_data; - ParentConfigParams *parent_params = nullptr; + ParentConfigParams *parent_params = nullptr; + std::shared_ptr next_hop_strategy = nullptr; ParentResult parent_result; CacheControlResult cache_control; CacheLookupResult_t cache_lookup_result = CACHE_LOOKUP_NONE; @@ -765,8 +762,6 @@ class HttpTransact // for authenticated content caching CacheAuth_t www_auth_content = CACHE_AUTH_NONE; - // INK API/Remap API plugin interface - void *user_args[TS_HTTP_MAX_USER_ARG]; RemapPluginInst *os_response_plugin_inst = nullptr; HTTPStatus http_return_code = HTTP_STATUS_NONE; @@ -816,8 +811,16 @@ class HttpTransact int64_t range_output_cl = 0; RangeRecord *ranges = nullptr; - OverridableHttpConfigParams *txn_conf = nullptr; - OverridableHttpConfigParams my_txn_conf; // Storage for plugins, to avoid malloc + OverridableHttpConfigParams const *txn_conf = nullptr; + OverridableHttpConfigParams & + my_txn_conf() // Storage for plugins, to avoid malloc + { + auto p = reinterpret_cast(_my_txn_conf); + + ink_assert(p == txn_conf); + + return *p; + } bool transparent_passthrough = false; bool range_in_cache = false; @@ -853,7 +856,6 @@ class HttpTransact via_string[VIA_DETAIL_SERVER_DESCRIPTOR] = VIA_DETAIL_SERVER_DESCRIPTOR_STRING; via_string[MAX_VIA_INDICES] = '\0'; - memset(user_args, 0, sizeof(user_args)); memset((void *)&host_db_info, 0, sizeof(host_db_info)); } @@ -897,10 +899,9 @@ class HttpTransact void setup_per_txn_configs() { - if (txn_conf != &my_txn_conf) { - // Make sure we copy it first. - memcpy(&my_txn_conf, &http_config_param->oride, sizeof(my_txn_conf)); - txn_conf = &my_txn_conf; + if (txn_conf != reinterpret_cast(_my_txn_conf)) { + txn_conf = reinterpret_cast(_my_txn_conf); + memcpy(_my_txn_conf, &http_config_param->oride, sizeof(_my_txn_conf)); } } @@ -920,6 +921,10 @@ class HttpTransact NetVConnection::ProxyProtocol pp_info; + private: + // Make this a raw byte array, so it will be accessed through the my_txn_conf() member function. + alignas(OverridableHttpConfigParams) char _my_txn_conf[sizeof(OverridableHttpConfigParams)]; + }; // End of State struct. static void HandleBlindTunnel(State *s); @@ -937,6 +942,8 @@ class HttpTransact static void HandleRequestAuthorized(State *s); static void BadRequest(State *s); static void Forbidden(State *s); + static void TooEarly(State *s); + static void SelfLoop(State *s); static void PostActiveTimeoutResponse(State *s); static void PostInactiveTimeoutResponse(State *s); static void DecideCacheLookup(State *s); @@ -995,7 +1002,6 @@ class HttpTransact static bool get_ka_info_from_config(State *s, ConnectionAttributes *server_info); static void get_ka_info_from_host_db(State *s, ConnectionAttributes *server_info, ConnectionAttributes *client_info, HostDBInfo *host_db_info); - static bool service_transaction_in_proxy_only_mode(State *s); static void setup_plugin_request_intercept(State *s); static void add_client_ip_to_outgoing_request(State *s, HTTPHdr *request); static RequestError_t check_request_validity(State *s, HTTPHdr *incoming_hdr); @@ -1032,6 +1038,7 @@ class HttpTransact static bool will_this_request_self_loop(State *s); static bool is_request_likely_cacheable(State *s, HTTPHdr *request); static bool is_cache_hit(CacheLookupResult_t r); + static bool is_fresh_cache_hit(CacheLookupResult_t r); static void build_request(State *s, HTTPHdr *base_request, HTTPHdr *outgoing_request, HTTPVersion outgoing_version); static void build_response(State *s, HTTPHdr *base_response, HTTPHdr *outgoing_response, HTTPVersion outgoing_version, @@ -1046,6 +1053,7 @@ class HttpTransact static void handle_request_keep_alive_headers(State *s, HTTPVersion ver, HTTPHdr *heads); static void handle_response_keep_alive_headers(State *s, HTTPVersion ver, HTTPHdr *heads); + static int get_max_age(HTTPHdr *response); static int calculate_document_freshness_limit(State *s, HTTPHdr *response, time_t response_date, bool *heuristic); static int calculate_freshness_fuzz(State *s, int fresh_limit); static Freshness_t what_is_document_freshness(State *s, HTTPHdr *client_request, HTTPHdr *cached_obj_response); @@ -1056,7 +1064,6 @@ class HttpTransact static void build_error_response(State *s, HTTPStatus status_code, const char *reason_phrase_or_null, const char *error_body_type); static void build_redirect_response(State *s); - static void build_upgrade_response(State *s); static const char *get_error_string(int erno); // the stat functions diff --git a/proxy/http/HttpTransactCache.cc b/proxy/http/HttpTransactCache.cc index 8ea46c37345..0c5ec7fc9b5 100644 --- a/proxy/http/HttpTransactCache.cc +++ b/proxy/http/HttpTransactCache.cc @@ -164,7 +164,7 @@ is_empty(char *s) */ int HttpTransactCache::SelectFromAlternates(CacheHTTPInfoVector *cache_vector, HTTPHdr *client_request, - OverridableHttpConfigParams *http_config_params) + const OverridableHttpConfigParams *http_config_params) { time_t current_age, best_age = CacheHighAgeWatermark; time_t t_now = 0; @@ -284,7 +284,7 @@ HttpTransactCache::SelectFromAlternates(CacheHTTPInfoVector *cache_vector, HTTPH */ float -HttpTransactCache::calculate_quality_of_match(OverridableHttpConfigParams *http_config_param, HTTPHdr *client_request, +HttpTransactCache::calculate_quality_of_match(const OverridableHttpConfigParams *http_config_param, HTTPHdr *client_request, HTTPHdr *obj_client_request, HTTPHdr *obj_origin_server_response) { // For PURGE requests, any alternate is good really. @@ -486,13 +486,6 @@ HttpTransactCache::calculate_quality_of_match(OverridableHttpConfigParams *http_ @return quality (-1: no match, 0..1: poor..good). */ -static inline bool -do_content_types_match(char *type1, char *subtype1, char *type2, char *subtype2) -{ - return ((is_asterisk(type1) || is_empty(type1) || (strcasecmp(type1, type2) == 0)) && - (is_asterisk(subtype1) || is_empty(subtype1) || (strcasecmp(subtype1, subtype2) == 0))); -} - float HttpTransactCache::calculate_quality_of_accept_match(MIMEField *accept_field, MIMEField *content_field) { @@ -526,6 +519,9 @@ HttpTransactCache::calculate_quality_of_accept_match(MIMEField *accept_field, MI // Parse the type and subtype of the Content-Type field. HttpCompat::parse_mime_type(c_param->str, c_type, c_subtype, sizeof(c_type), sizeof(c_subtype)); + // Special case for webp because Safari is has Accept: */*, but doesn't support webp + bool content_type_webp = ((strcasecmp("webp", c_subtype) == 0) && (strcasecmp("image", c_type) == 0)); + // Now loop over Accept field values. // TODO: Should we check the return value (count) from this? accept_field->value_get_comma_list(&a_values_list); @@ -549,19 +545,25 @@ HttpTransactCache::calculate_quality_of_accept_match(MIMEField *accept_field, MI char a_type[32], a_subtype[32]; HttpCompat::parse_mime_type(a_param->str, a_type, a_subtype, sizeof(a_type), sizeof(a_subtype)); - // printf("matching Content-type; '%s/%s' with Accept value '%s/%s'\n", - // c_type,c_subtype,a_type,a_subtype); - - // Is there a wildcard in the type or subtype? - if (is_asterisk(a_type)) { - wildcard_type_present = true; - wildcard_type_q = HttpCompat::find_Q_param_in_strlist(&a_param_list); - } else if (is_asterisk(a_subtype) && (strcasecmp(a_type, c_type) == 0)) { - wildcard_subtype_present = true; - wildcard_subtype_q = HttpCompat::find_Q_param_in_strlist(&a_param_list); - } else { - // No wildcard. Do explicit matching of accept and content values. - if (do_content_types_match(a_type, a_subtype, c_type, c_subtype)) { + Debug("http_match", "matching Content-type; '%s/%s' with Accept value '%s/%s'\n", c_type, c_subtype, a_type, a_subtype); + + bool wildcard_found = true; + // Only do wildcard checks if the content type is not image/webp + if (content_type_webp == false) { + // Is there a wildcard in the type or subtype? + if (is_asterisk(a_type)) { + wildcard_type_present = true; + wildcard_type_q = HttpCompat::find_Q_param_in_strlist(&a_param_list); + } else if (is_asterisk(a_subtype) && (strcasecmp(a_type, c_type) == 0)) { + wildcard_subtype_present = true; + wildcard_subtype_q = HttpCompat::find_Q_param_in_strlist(&a_param_list); + } else { + wildcard_found = false; + } + } + if (content_type_webp == true || wildcard_found == false) { + // No wildcard or the content type is image/webp. Do explicit matching of accept and content values. + if ((strcasecmp(a_type, c_type) == 0) && (strcasecmp(a_subtype, c_subtype) == 0)) { float tq; tq = HttpCompat::find_Q_param_in_strlist(&a_param_list); q = (tq > q ? tq : q); @@ -582,6 +584,7 @@ HttpTransactCache::calculate_quality_of_accept_match(MIMEField *accept_field, MI if ((q == -1.0) && (wildcard_type_present == true)) { q = wildcard_type_q; } + return (q); } @@ -1124,7 +1127,7 @@ HttpTransactCache::calculate_quality_of_accept_language_match(MIMEField *accept_ */ Variability_t -HttpTransactCache::CalcVariability(OverridableHttpConfigParams *http_config_params, HTTPHdr *client_request, +HttpTransactCache::CalcVariability(const OverridableHttpConfigParams *http_config_params, HTTPHdr *client_request, HTTPHdr *obj_client_request, HTTPHdr *obj_origin_server_response) { ink_assert(http_config_params != nullptr); @@ -1381,13 +1384,13 @@ HttpTransactCache::match_response_to_request_conditionals(HTTPHdr *request, HTTP return HTTP_STATUS_RANGE_NOT_SATISFIABLE; } } - // this a Date, similar to If-Unmodified-Since + // this a Date, similar to If-Unmodified-Since but must be an exact match else { // lm_value is zero if Last-modified not exists ink_time_t lm_value = response->get_last_modified(); // condition fails if Last-modified not exists - if ((request->get_if_range_date() < lm_value) || (lm_value == 0)) { + if ((request->get_if_range_date() != lm_value) || (lm_value == 0)) { return HTTP_STATUS_RANGE_NOT_SATISFIABLE; } else { return response->status_get(); diff --git a/proxy/http/HttpTransactCache.h b/proxy/http/HttpTransactCache.h index 4f7e142b894..eba10037a85 100644 --- a/proxy/http/HttpTransactCache.h +++ b/proxy/http/HttpTransactCache.h @@ -52,9 +52,9 @@ class HttpTransactCache ///////////////////////////////// static int SelectFromAlternates(CacheHTTPInfoVector *cache_vector_data, HTTPHdr *client_request, - OverridableHttpConfigParams *cache_lookup_http_config_params); + const OverridableHttpConfigParams *cache_lookup_http_config_params); - static float calculate_quality_of_match(OverridableHttpConfigParams *http_config_params, HTTPHdr *client_request, + static float calculate_quality_of_match(const OverridableHttpConfigParams *http_config_params, HTTPHdr *client_request, HTTPHdr *obj_client_request, HTTPHdr *obj_origin_server_response); static float calculate_quality_of_accept_match(MIMEField *accept_field, MIMEField *content_field); @@ -75,7 +75,7 @@ class HttpTransactCache // variability & server negotiation routines // /////////////////////////////////////////////// - static Variability_t CalcVariability(OverridableHttpConfigParams *http_config_params, HTTPHdr *client_request, + static Variability_t CalcVariability(const OverridableHttpConfigParams *http_config_params, HTTPHdr *client_request, HTTPHdr *obj_client_request, HTTPHdr *obj_origin_server_response); static HTTPStatus match_response_to_request_conditionals(HTTPHdr *ua_request, HTTPHdr *c_response, diff --git a/proxy/http/HttpTransactHeaders.cc b/proxy/http/HttpTransactHeaders.cc index f3c9b5b90fe..07dcf7437d8 100644 --- a/proxy/http/HttpTransactHeaders.cc +++ b/proxy/http/HttpTransactHeaders.cc @@ -961,12 +961,13 @@ HttpTransactHeaders::remove_host_name_from_url(HTTPHdr *outgoing_request) } void -HttpTransactHeaders::add_global_user_agent_header_to_request(OverridableHttpConfigParams *http_txn_conf, HTTPHdr *header) +HttpTransactHeaders::add_global_user_agent_header_to_request(const OverridableHttpConfigParams *http_txn_conf, HTTPHdr *header) { if (http_txn_conf->global_user_agent_header) { MIMEField *ua_field; - Debug("http_trans", "Adding User-Agent: %s", http_txn_conf->global_user_agent_header); + Debug("http_trans", "Adding User-Agent: %.*s", static_cast(http_txn_conf->global_user_agent_header_size), + http_txn_conf->global_user_agent_header); if ((ua_field = header->field_find(MIME_FIELD_USER_AGENT, MIME_LEN_USER_AGENT)) == nullptr) { if (likely((ua_field = header->field_create(MIME_FIELD_USER_AGENT, MIME_LEN_USER_AGENT)) != nullptr)) { header->field_attach(ua_field); @@ -1159,7 +1160,7 @@ HttpTransactHeaders::add_forwarded_field_to_request(HttpTransact::State *s, HTTP } // end HttpTransact::add_forwarded_field_to_outgoing_request() void -HttpTransactHeaders::add_server_header_to_response(OverridableHttpConfigParams *http_txn_conf, HTTPHdr *header) +HttpTransactHeaders::add_server_header_to_response(const OverridableHttpConfigParams *http_txn_conf, HTTPHdr *header) { if (http_txn_conf->proxy_response_server_enabled && http_txn_conf->proxy_response_server_string) { MIMEField *ua_field; @@ -1185,7 +1186,7 @@ HttpTransactHeaders::add_server_header_to_response(OverridableHttpConfigParams * void HttpTransactHeaders::remove_privacy_headers_from_request(HttpConfigParams *http_config_param, - OverridableHttpConfigParams *http_txn_conf, HTTPHdr *header) + const OverridableHttpConfigParams *http_txn_conf, HTTPHdr *header) { if (!header) { return; diff --git a/proxy/http/HttpTransactHeaders.h b/proxy/http/HttpTransactHeaders.h index 96b236ab49e..cb921870b0a 100644 --- a/proxy/http/HttpTransactHeaders.h +++ b/proxy/http/HttpTransactHeaders.h @@ -66,10 +66,6 @@ class HttpTransactHeaders static int write_hdr_protocol_stack(char *hdr_string, size_t len, ProtocolStackDetail pSDetail, std::string_view *proto_buf, int n_proto, char separator = ' '); - // Removing handle_conditional_headers. Functionality appears to be elsewhere (issue_revalidate) - // and the only condition when it does anything causes an assert to go - // off - // static void handle_conditional_headers(HttpTransact::CacheLookupInfo * cache_info, HTTPHdr * header); static void insert_warning_header(HttpConfigParams *http_config_param, HTTPHdr *header, HTTPWarningCode code, const char *warn_text = nullptr, int warn_text_len = 0); static void insert_time_and_age_headers_in_response(ink_time_t request_sent_time, ink_time_t response_received_time, @@ -87,10 +83,10 @@ class HttpTransactHeaders static void remove_conditional_headers(HTTPHdr *outgoing); static void remove_100_continue_headers(HttpTransact::State *s, HTTPHdr *outgoing); static void remove_host_name_from_url(HTTPHdr *outgoing_request); - static void add_global_user_agent_header_to_request(OverridableHttpConfigParams *http_txn_conf, HTTPHdr *header); - static void add_server_header_to_response(OverridableHttpConfigParams *http_txn_conf, HTTPHdr *header); - static void remove_privacy_headers_from_request(HttpConfigParams *http_config_param, OverridableHttpConfigParams *http_txn_conf, - HTTPHdr *header); + static void add_global_user_agent_header_to_request(const OverridableHttpConfigParams *http_txn_conf, HTTPHdr *header); + static void add_server_header_to_response(const OverridableHttpConfigParams *http_txn_conf, HTTPHdr *header); + static void remove_privacy_headers_from_request(HttpConfigParams *http_config_param, + const OverridableHttpConfigParams *http_txn_conf, HTTPHdr *header); static void add_connection_close(HTTPHdr *header); static int nstrcpy(char *d, const char *as); diff --git a/proxy/http/HttpTunnel.cc b/proxy/http/HttpTunnel.cc index 882e1e5ca1c..0588acd311f 100644 --- a/proxy/http/HttpTunnel.cc +++ b/proxy/http/HttpTunnel.cc @@ -479,8 +479,8 @@ HttpTunnel::reset() num_producers = 0; num_consumers = 0; - memset(consumers, 0, sizeof(consumers)); - memset(producers, 0, sizeof(producers)); + ink_zero(consumers); + ink_zero(producers); } void @@ -811,10 +811,9 @@ HttpTunnel::producer_run(HttpTunnelProducer *p) consumer_n = (producer_n = INT64_MAX); } - // Do the IO on the consumers first so - // data doesn't disappear out from - // under the tunnel - for (c = p->consumer_list.head; c;) { + // At least set up the consumer readers first so the data + // doesn't disappear out from under the tunnel + for (c = p->consumer_list.head; c; c = c->link.next) { // Create a reader for each consumer. The reader allows // us to implement skip bytes if (c->vc_type == HT_CACHE_WRITE) { @@ -845,45 +844,6 @@ HttpTunnel::producer_run(HttpTunnelProducer *p) ink_assert(c->skip_bytes <= c->buffer_reader->read_avail()); c->buffer_reader->consume(c->skip_bytes); } - int64_t c_write = consumer_n; - - // INKqa05109 - if we don't know the length leave it at - // INT64_MAX or else the cache may bounce the write - // because it thinks the document is too big. INT64_MAX - // is a special case for the max document size code - // in the cache - if (c_write != INT64_MAX) { - c_write -= c->skip_bytes; - } - // Fix for problems with not chunked content being chunked and - // not sending the entire data. The content length grows when - // it is being chunked. - if (p->do_chunking == true) { - c_write = INT64_MAX; - } - - if (c_write == 0) { - // Nothing to do, call back the cleanup handlers - c->write_vio = nullptr; - consumer_handler(VC_EVENT_WRITE_COMPLETE, c); - } else { - // In the client half close case, all the data that will be sent - // from the client is already in the buffer. Go ahead and set - // the amount to read since we know it. We will forward the FIN - // to the server on VC_EVENT_WRITE_COMPLETE. - if (p->vc_type == HT_HTTP_CLIENT) { - ProxyTransaction *ua_vc = static_cast(p->vc); - if (ua_vc->get_half_close_flag()) { - c_write = c->buffer_reader->read_avail(); - p->alive = false; - p->handler_state = HTTP_SM_POST_SUCCESS; - } - } - c->write_vio = c->vc->do_io_write(this, c_write, c->buffer_reader); - ink_assert(c_write > 0); - } - - c = c->link.next; } // YTS Team, yamsat Plugin @@ -943,7 +903,64 @@ HttpTunnel::producer_run(HttpTunnelProducer *p) producer_n = 0; } } + for (c = p->consumer_list.head; c; c = c->link.next) { + int64_t c_write = consumer_n; + // Don't bother to set up the consumer if it is dead + if (!c->alive) { + continue; + } + + if (!p->alive) { + // Adjust the amount of chunked data to write if the only data was in the initial read + // The amount to write in some cases is dependent on the type of the consumer, so this + // value must be computed for each consumer + c_write = final_consumer_bytes_to_write(p, c); + } else { + // INKqa05109 - if we don't know the length leave it at + // INT64_MAX or else the cache may bounce the write + // because it thinks the document is too big. INT64_MAX + // is a special case for the max document size code + // in the cache + if (c_write != INT64_MAX) { + c_write -= c->skip_bytes; + } + // Fix for problems with not chunked content being chunked and + // not sending the entire data. The content length grows when + // it is being chunked. + if (p->do_chunking == true) { + c_write = INT64_MAX; + } + } + + if (c_write == 0) { + // Nothing to do, call back the cleanup handlers + c->write_vio = nullptr; + consumer_handler(VC_EVENT_WRITE_COMPLETE, c); + } else { + // In the client half close case, all the data that will be sent + // from the client is already in the buffer. Go ahead and set + // the amount to read since we know it. We will forward the FIN + // to the server on VC_EVENT_WRITE_COMPLETE. + if (p->vc_type == HT_HTTP_CLIENT) { + ProxyTransaction *ua_vc = static_cast(p->vc); + if (ua_vc->get_half_close_flag()) { + int tmp = c->buffer_reader->read_avail(); + if (tmp < c_write) { + c_write = tmp; + } + p->alive = false; + p->handler_state = HTTP_SM_POST_SUCCESS; + } + } + // Start the writes now that we know we will consume all the initial data + c->write_vio = c->vc->do_io_write(this, c_write, c->buffer_reader); + ink_assert(c_write > 0); + if (c->write_vio == nullptr) { + consumer_handler(VC_EVENT_ERROR, c); + } + } + } if (p->alive) { ink_assert(producer_n >= 0); @@ -1124,8 +1141,8 @@ HttpTunnel::producer_handler(int event, HttpTunnelProducer *p) } } // end of added logic for partial copy of POST - Debug("http_redirect", "[HttpTunnel::producer_handler] enable_redirection: [%d %d %d] event: %d", p->alive == true, - sm->enable_redirection, (p->self_consumer && p->self_consumer->alive == true), event); + Debug("http_redirect", "[HttpTunnel::producer_handler] enable_redirection: [%d %d %d] event: %d, state: %d", p->alive == true, + sm->enable_redirection, (p->self_consumer && p->self_consumer->alive == true), event, p->chunked_handler.state); switch (event) { case VC_EVENT_READ_READY: @@ -1182,8 +1199,12 @@ HttpTunnel::producer_handler(int event, HttpTunnelProducer *p) case VC_EVENT_INACTIVITY_TIMEOUT: case HTTP_TUNNEL_EVENT_CONSUMER_DETACH: if (p->alive) { - p->alive = false; - p->bytes_read = p->read_vio->ndone; + p->alive = false; + if (p->read_vio) { + p->bytes_read = p->read_vio->ndone; + } else { + p->bytes_read = 0; + } // Clear any outstanding reads so they don't // collide with future tunnel IO's p->vc->do_io_read(nullptr, 0, nullptr); @@ -1398,25 +1419,23 @@ HttpTunnel::chain_abort_all(HttpTunnelProducer *p) } p->read_vio = nullptr; p->vc->do_io_close(EHTTP_ERROR); + HTTP_INCREMENT_DYN_STAT(http_origin_shutdown_tunnel_abort); update_stats_after_abort(p->vc_type); } } -// void HttpTunnel::chain_finish_internal(HttpTunnelProducer* p) // -// Internal function for finishing all consumers. Takes -// chain argument about where to finish just immediate -// consumer or all those downstream +// Determine the number of bytes a consumer should read from a producer // -void -HttpTunnel::finish_all_internal(HttpTunnelProducer *p, bool chain) +int64_t +HttpTunnel::final_consumer_bytes_to_write(HttpTunnelProducer *p, HttpTunnelConsumer *c) { - ink_assert(p->alive == false); - HttpTunnelConsumer *c = p->consumer_list.head; - int64_t total_bytes = 0; - TunnelChunkingAction_t action = p->chunking_action; - - while (c) { + int64_t total_bytes = 0; + int64_t consumer_n = 0; + if (p->alive) { + consumer_n = INT64_MAX; + } else { + TunnelChunkingAction_t action = p->chunking_action; if (c->alive) { if (c->vc_type == HT_CACHE_WRITE) { switch (action) { @@ -1435,12 +1454,48 @@ HttpTunnel::finish_all_internal(HttpTunnelProducer *p, bool chain) total_bytes = p->chunked_handler.skip_bytes + p->chunked_handler.chunked_size; } else if (action == TCA_DECHUNK_CONTENT) { total_bytes = p->chunked_handler.skip_bytes + p->chunked_handler.dechunked_size; + } else if (action == TCA_PASSTHRU_CHUNKED_CONTENT) { + total_bytes = p->bytes_read + p->init_bytes_done; } else { total_bytes = p->bytes_read + p->init_bytes_done; } + consumer_n = total_bytes - c->skip_bytes; + } + } + return consumer_n; +} +// +// void HttpTunnel::finish_all_internal(HttpTunnelProducer* p) +// +// Internal function for finishing all consumers. Takes +// chain argument about where to finish just immediate +// consumer or all those downstream +// +void +HttpTunnel::finish_all_internal(HttpTunnelProducer *p, bool chain) +{ + ink_assert(p->alive == false); + HttpTunnelConsumer *c = p->consumer_list.head; + int64_t total_bytes = 0; + TunnelChunkingAction_t action = p->chunking_action; + + if (action == TCA_PASSTHRU_CHUNKED_CONTENT) { + // if the only chunked data was in the initial read, make sure we don't consume too much + if (p->bytes_read == 0 && p->buffer_start != nullptr) { + int num_read = p->buffer_start->read_avail() - p->chunked_handler.chunked_reader->read_avail(); + if (num_read < p->init_bytes_done) { + p->init_bytes_done = num_read; + } + } + } + + while (c) { + if (c->alive) { if (c->write_vio) { - c->write_vio->nbytes = total_bytes - c->skip_bytes; + // Adjust the number of bytes to write in the case of + // a completed unlimited producer + c->write_vio->nbytes = final_consumer_bytes_to_write(p, c); ink_assert(c->write_vio->nbytes >= 0); if (c->write_vio->nbytes < 0) { @@ -1457,7 +1512,7 @@ HttpTunnel::finish_all_internal(HttpTunnelProducer *p, bool chain) // is nothing to do. Check to see if there is // nothing to do and take the appripriate // action - if (c->write_vio && c->write_vio->nbytes == c->write_vio->ndone) { + if (c->write_vio && c->alive && c->write_vio->nbytes == c->write_vio->ndone) { consumer_handler(VC_EVENT_WRITE_COMPLETE, c); } } diff --git a/proxy/http/HttpTunnel.h b/proxy/http/HttpTunnel.h index 272b33e674f..9ba0f976d68 100644 --- a/proxy/http/HttpTunnel.h +++ b/proxy/http/HttpTunnel.h @@ -312,6 +312,7 @@ class HttpTunnel : public Continuation void chain_abort_all(HttpTunnelProducer *p); void abort_cache_write_finish_others(HttpTunnelProducer *p); void append_message_to_producer_buffer(HttpTunnelProducer *p, const char *msg, int64_t msg_len); + int64_t final_consumer_bytes_to_write(HttpTunnelProducer *p, HttpTunnelConsumer *c); /** Mark a producer and consumer as the same underlying object. diff --git a/proxy/http/HttpUpdateTester.cc b/proxy/http/HttpUpdateTester.cc index 581eafbe150..3841aa2b0f6 100644 --- a/proxy/http/HttpUpdateTester.cc +++ b/proxy/http/HttpUpdateTester.cc @@ -87,9 +87,9 @@ UpTest::make_requests() test_req.parse_req(&http_parser, &req, req + strlen(req), false); http_parser_clear(&http_parser); - HttpUpdateSM *current_reader = HttpUpdateSM::allocate(); - current_reader->init(); - Action *a = current_reader->start_scheduled_update(this, test_req); + HttpUpdateSM *_sm = HttpUpdateSM::allocate(); + _sm->init(); + Action *a = _sm->start_scheduled_update(this, test_req); (void)a; active_req++; diff --git a/proxy/http/Makefile.am b/proxy/http/Makefile.am index 31a06c1b086..edc10ff5a9b 100644 --- a/proxy/http/Makefile.am +++ b/proxy/http/Makefile.am @@ -33,7 +33,8 @@ AM_CPPFLAGS += \ -I$(abs_top_srcdir)/proxy/logging \ -I$(abs_top_srcdir)/proxy/http2 \ -I$(abs_top_srcdir)/proxy/http3 \ - $(TS_INCLUDES) + $(TS_INCLUDES) \ + @YAMLCPP_INCLUDES@ noinst_HEADERS = HttpProxyServerMain.h noinst_LIBRARIES = libhttp.a diff --git a/proxy/http/remap/Makefile.am b/proxy/http/remap/Makefile.am index ad705d3d26c..af15a57556a 100644 --- a/proxy/http/remap/Makefile.am +++ b/proxy/http/remap/Makefile.am @@ -29,13 +29,22 @@ AM_CPPFLAGS += \ -I$(abs_top_srcdir)/proxy/hdrs \ -I$(abs_top_srcdir)/proxy/shared \ -I$(abs_top_srcdir)/proxy/http \ - $(TS_INCLUDES) + $(TS_INCLUDES) \ + @YAMLCPP_INCLUDES@ noinst_LIBRARIES = libhttp_remap.a libhttp_remap_a_SOURCES = \ AclFiltering.cc \ AclFiltering.h \ + NextHopSelectionStrategy.h \ + NextHopSelectionStrategy.cc \ + NextHopConsistentHash.h \ + NextHopConsistentHash.cc \ + NextHopRoundRobin.h \ + NextHopRoundRobin.cc \ + NextHopStrategyFactory.h \ + NextHopStrategyFactory.cc \ RemapConfig.cc \ RemapConfig.h \ RemapPluginInfo.cc \ @@ -54,25 +63,40 @@ libhttp_remap_a_SOURCES = \ UrlRewrite.cc \ UrlRewrite.h +COMMON_PLUGINDSO_LDADDS = \ + $(OPENSSL_LIBS) \ + $(top_builddir)/iocore/eventsystem/libinkevent.a \ + $(top_builddir)/lib/records/librecords_p.a \ + $(top_builddir)/iocore/eventsystem/libinkevent.a \ + $(top_builddir)/src/tscore/libtscore.la \ + $(top_builddir)/mgmt/libmgmt_p.la \ + $(top_builddir)/proxy/shared/libUglyLogStubs.a \ + @HWLOC_LIBS@ + clang-tidy-local: $(libhttp_remap_a_SOURCES) $(CXX_Clang_Tidy) TESTS = $(check_PROGRAMS) -check_PROGRAMS = test_PluginDso test_PluginFactory test_RemapPluginInfo +check_PROGRAMS = test_PluginDso test_PluginFactory test_RemapPluginInfo test_NextHopStrategyFactory test_NextHopRoundRobin test_NextHopConsistentHash test_PluginDso_CPPFLAGS = $(AM_CPPFLAGS) -I$(abs_top_srcdir)/tests/include -DPLUGIN_DSO_TESTS +test_PluginDso_LIBTOOLFLAGS = --preserve-dup-deps EXTRA_test_PluginDso_DEPENDENCIES = unit-tests/plugin_v1.la -test_PluginDso_LDADD = $(OPENSSL_LIBS) -test_PluginDso_LDFLAGS = $(AM_LDFLAGS) -L$(top_builddir)/src/tscore/.libs -ltscore +test_PluginDso_LDADD = $(COMMON_PLUGINDSO_LDADDS) +test_PluginDso_LDFLAGS = $(AM_LDFLAGS) test_PluginDso_SOURCES = \ unit-tests/test_PluginDso.cc \ unit-tests/plugin_testing_common.cc \ PluginDso.cc test_PluginFactory_CPPFLAGS = $(AM_CPPFLAGS) -I$(abs_top_srcdir)/tests/include -DPLUGIN_DSO_TESTS -EXTRA_test_PluginFactory_DEPENDENCIES = unit-tests/plugin_v1.la -test_PluginFactory_LDADD = $(OPENSSL_LIBS) -test_PluginFactory_LDFLAGS = $(AM_LDFLAGS) -L$(top_builddir)/src/tscore/.libs -ltscore +test_PluginFactory_LIBTOOLFLAGS = --preserve-dup-deps +EXTRA_test_PluginFactory_DEPENDENCIES = \ + unit-tests/plugin_v1.la \ + unit-tests/plugin_init_fail.la \ + unit-tests/plugin_instinit_fail.la +test_PluginFactory_LDADD = $(COMMON_PLUGINDSO_LDADDS) +test_PluginFactory_LDFLAGS = $(AM_LDFLAGS) test_PluginFactory_SOURCES = \ unit-tests/test_PluginFactory.cc \ unit-tests/plugin_testing_common.cc \ @@ -81,6 +105,7 @@ test_PluginFactory_SOURCES = \ RemapPluginInfo.cc test_RemapPluginInfo_CPPFLAGS = $(AM_CPPFLAGS) -I$(abs_top_srcdir)/tests/include -DPLUGIN_DSO_TESTS +test_RemapPluginInfo_LIBTOOLFLAGS = --preserve-dup-deps EXTRA_test_RemapPluginInfo_DEPENDENCIES = \ unit-tests/plugin_missing_init.la \ unit-tests/plugin_missing_doremap.la \ @@ -88,27 +113,116 @@ EXTRA_test_RemapPluginInfo_DEPENDENCIES = \ unit-tests/plugin_required_cb.la \ unit-tests/plugin_missing_newinstance.la \ unit-tests/plugin_testing_calls.la -test_RemapPluginInfo_LDADD = \ - $(OPENSSL_LIBS) -test_RemapPluginInfo_LDFLAGS = $(AM_LDFLAGS) -L$(top_builddir)/src/tscore/.libs -ltscore +test_RemapPluginInfo_LDADD = $(COMMON_PLUGINDSO_LDADDS) +test_RemapPluginInfo_LDFLAGS = $(AM_LDFLAGS) test_RemapPluginInfo_SOURCES = \ unit-tests/plugin_testing_common.cc \ - unit-tests/plugin_testing_calls.cc \ unit-tests/test_RemapPlugin.cc \ PluginDso.cc \ RemapPluginInfo.cc +test_NextHopStrategyFactory_CPPFLAGS = \ + $(AM_CPPFLAGS) \ + -D_NH_UNIT_TESTS_ \ + -DTS_SRC_DIR=\"$(abs_top_srcdir)/proxy/http/remap/\" \ + -I$(abs_top_srcdir)/tests/include \ + $(TS_INCLUDES) \ + @YAMLCPP_INCLUDES@ + +test_NextHopStrategyFactory_LDADD = \ + $(top_builddir)/src/tscpp/util/libtscpputil.la \ + $(top_builddir)/src/tscore/libtscore.la \ + $(top_builddir)/proxy/hdrs/libhdrs.a \ + $(top_builddir)/iocore/eventsystem/libinkevent.a \ + $(top_builddir)/lib/records/librecords_p.a \ + $(top_builddir)/proxy/logging/liblogging.a \ + $(top_builddir)/mgmt/libmgmt_p.la \ + $(top_builddir)/iocore/utils/libinkutils.a \ + @YAMLCPP_LIBS@ \ + @HWLOC_LIBS@ + +test_NextHopStrategyFactory_LDFLAGS = $(AM_LDFLAGS) -L$(top_builddir)/src/tscore/.libs -ltscore + +test_NextHopStrategyFactory_SOURCES = \ + NextHopSelectionStrategy.cc \ + NextHopStrategyFactory.cc \ + NextHopRoundRobin.cc \ + NextHopConsistentHash.cc \ + unit-tests/test_NextHopStrategyFactory.cc \ + unit-tests/nexthop_test_stubs.cc + +test_NextHopRoundRobin_CPPFLAGS = \ + $(AM_CPPFLAGS) \ + -D_NH_UNIT_TESTS_ \ + -DTS_SRC_DIR=\"$(abs_top_srcdir)/proxy/http/remap/\" \ + -I$(abs_top_srcdir)/tests/include \ + $(TS_INCLUDES) \ + @YAMLCPP_INCLUDES@ + +test_NextHopRoundRobin_LDADD = \ + $(top_builddir)/src/tscpp/util/libtscpputil.la \ + $(top_builddir)/src/tscore/libtscore.la \ + $(top_builddir)/proxy/hdrs/libhdrs.a \ + $(top_builddir)/iocore/eventsystem/libinkevent.a \ + $(top_builddir)/lib/records/librecords_p.a \ + $(top_builddir)/proxy/logging/liblogging.a \ + $(top_builddir)/mgmt/libmgmt_p.la \ + $(top_builddir)/iocore/utils/libinkutils.a \ + @YAMLCPP_LIBS@ \ + @HWLOC_LIBS@ + +test_NextHopRoundRobin_LDFLAGS = $(AM_LDFLAGS) -L$(top_builddir)/src/tscore/.libs -ltscore + +test_NextHopRoundRobin_SOURCES = \ + NextHopSelectionStrategy.cc \ + NextHopStrategyFactory.cc \ + NextHopRoundRobin.cc \ + NextHopConsistentHash.cc \ + unit-tests/test_NextHopRoundRobin.cc \ + unit-tests/nexthop_test_stubs.cc + +test_NextHopConsistentHash_CPPFLAGS = \ + $(AM_CPPFLAGS) \ + -D_NH_UNIT_TESTS_ \ + -DTS_SRC_DIR=\"$(abs_top_srcdir)/proxy/http/remap/\" \ + -I$(abs_top_srcdir)/tests/include \ + $(TS_INCLUDES) \ + @YAMLCPP_INCLUDES@ + +test_NextHopConsistentHash_LDADD = \ + $(top_builddir)/src/tscpp/util/libtscpputil.la \ + $(top_builddir)/src/tscore/libtscore.la \ + $(top_builddir)/proxy/hdrs/libhdrs.a \ + $(top_builddir)/iocore/eventsystem/libinkevent.a \ + $(top_builddir)/lib/records/librecords_p.a \ + $(top_builddir)/proxy/logging/liblogging.a \ + $(top_builddir)/mgmt/libmgmt_p.la \ + $(top_builddir)/iocore/utils/libinkutils.a \ + @YAMLCPP_LIBS@ \ + @HWLOC_LIBS@ + +test_NextHopConsistentHash_LDFLAGS = $(AM_LDFLAGS) -L$(top_builddir)/src/tscore/.libs -ltscore + +test_NextHopConsistentHash_SOURCES = \ + NextHopSelectionStrategy.cc \ + NextHopStrategyFactory.cc \ + NextHopConsistentHash.cc \ + NextHopRoundRobin.cc \ + unit-tests/test_NextHopConsistentHash.cc \ + unit-tests/nexthop_test_stubs.cc DSO_LDFLAGS = \ -module \ -shared \ -avoid-version \ - -export-symbols-regex '^(TSRemapInit|TSRemapDone|TSRemapDoRemap|TSRemapNewInstance|TSRemapDeleteInstance|TSRemapOSResponse|TSRemapConfigReload|TSPluginInit|pluginDsoVersionTest|getPluginDebugObjectTest)$$' + -export-symbols-regex '^(TSRemapInit|TSRemapDone|TSRemapDoRemap|TSRemapNewInstance|TSRemapDeleteInstance|TSRemapOSResponse|TSRemapPreConfigReload|TSRemapPostConfigReload|TSPluginInit|pluginDsoVersionTest|getPluginDebugObjectTest)$$' # Build plugins for unit testing the plugin (re)load. pkglib_LTLIBRARIES = \ unit-tests/plugin_v1.la \ unit-tests/plugin_v2.la \ + unit-tests/plugin_init_fail.la \ + unit-tests/plugin_instinit_fail.la \ unit-tests/plugin_required_cb.la \ unit-tests/plugin_missing_deleteinstance.la \ unit-tests/plugin_missing_doremap.la \ @@ -121,6 +235,12 @@ unit_tests_plugin_v1_la_CPPFLAGS = -DPLUGINDSOVER=1 $(AM_CPPFLAGS) unit_tests_plugin_v2_la_SOURCES = unit-tests/plugin_misc_cb.cc unit_tests_plugin_v2_la_LDFLAGS = $(DSO_LDFLAGS) unit_tests_plugin_v2_la_CPPFLAGS = -DPLUGINDSOVER=2 $(AM_CPPFLAGS) +unit_tests_plugin_init_fail_la_SOURCES = unit-tests/plugin_init_fail.cc +unit_tests_plugin_init_fail_la_LDFLAGS = $(DSO_LDFLAGS) +unit_tests_plugin_init_fail_la_CPPFLAGS = $(AM_CPPFLAGS) +unit_tests_plugin_instinit_fail_la_SOURCES = unit-tests/plugin_instinit_fail.cc +unit_tests_plugin_instinit_fail_la_LDFLAGS = $(DSO_LDFLAGS) +unit_tests_plugin_instinit_fail_la_CPPFLAGS = $(AM_CPPFLAGS) unit_tests_plugin_required_cb_la_SOURCES = unit-tests/plugin_required_cb.cc unit_tests_plugin_required_cb_la_LDFLAGS = $(DSO_LDFLAGS) unit_tests_plugin_required_cb_la_CPPFLAGS = -DPLUGINDSOVER=1 $(AM_CPPFLAGS) diff --git a/proxy/http/remap/NextHopConsistentHash.cc b/proxy/http/remap/NextHopConsistentHash.cc new file mode 100644 index 00000000000..b28a6c10b77 --- /dev/null +++ b/proxy/http/remap/NextHopConsistentHash.cc @@ -0,0 +1,409 @@ +/** @file + + Implementation of nexthop consistent hash selections strategies. + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#include + +#include "tscore/HashSip.h" +#include "NextHopConsistentHash.h" + +// hash_key strings. +constexpr std::string_view hash_key_url = "url"; +constexpr std::string_view hash_key_hostname = "hostname"; +constexpr std::string_view hash_key_path = "path"; +constexpr std::string_view hash_key_path_query = "path+query"; +constexpr std::string_view hash_key_path_fragment = "path+fragment"; +constexpr std::string_view hash_key_cache = "cache_key"; + +static HostRecord * +chash_lookup(const std::shared_ptr &ring, uint64_t hash_key, ATSConsistentHashIter *iter, bool *wrapped, + ATSHash64Sip24 *hash, bool *hash_init, bool *mapWrapped, uint64_t sm_id) +{ + HostRecord *host_rec = nullptr; + + if (*hash_init == false) { + host_rec = static_cast(ring->lookup_by_hashval(hash_key, iter, wrapped)); + *hash_init = true; + } else { + host_rec = static_cast(ring->lookup(nullptr, iter, wrapped, hash)); + } + bool wrap_around = *wrapped; + *wrapped = (*mapWrapped && *wrapped) ? true : false; + if (!*mapWrapped && wrap_around) { + *mapWrapped = true; + } + + return host_rec; +} + +NextHopConsistentHash::~NextHopConsistentHash() +{ + NH_Debug(NH_DEBUG_TAG, "destructor called for strategy named: %s", strategy_name.c_str()); +} + +bool +NextHopConsistentHash::Init(const YAML::Node &n) +{ + ATSHash64Sip24 hash; + + try { + if (n["hash_key"]) { + auto hash_key_val = n["hash_key"].Scalar(); + if (hash_key_val == hash_key_url) { + hash_key = NH_URL_HASH_KEY; + } else if (hash_key_val == hash_key_hostname) { + hash_key = NH_HOSTNAME_HASH_KEY; + } else if (hash_key_val == hash_key_path) { + hash_key = NH_PATH_HASH_KEY; + } else if (hash_key_val == hash_key_path_query) { + hash_key = NH_PATH_QUERY_HASH_KEY; + } else if (hash_key_val == hash_key_path_fragment) { + hash_key = NH_PATH_FRAGMENT_HASH_KEY; + } else if (hash_key_val == hash_key_cache) { + hash_key = NH_CACHE_HASH_KEY; + } else { + hash_key = NH_PATH_HASH_KEY; + NH_Note("Invalid 'hash_key' value, '%s', for the strategy named '%s', using default '%s'.", hash_key_val.c_str(), + strategy_name.c_str(), hash_key_path.data()); + } + } + } catch (std::exception &ex) { + NH_Note("Error parsing the strategy named '%s' due to '%s', this strategy will be ignored.", strategy_name.c_str(), ex.what()); + return false; + } + + bool result = NextHopSelectionStrategy::Init(n); + if (!result) { + return false; + } + + // load up the hash rings. + for (uint32_t i = 0; i < groups; i++) { + std::shared_ptr hash_ring = std::make_shared(); + for (uint32_t j = 0; j < host_groups[i].size(); j++) { + // ATSConsistentHash needs the raw pointer. + HostRecord *p = host_groups[i][j].get(); + // need to copy the 'hash_string' or 'hostname' cstring to 'name' for insertion into ATSConsistentHash. + if (!p->hash_string.empty()) { + p->name = const_cast(p->hash_string.c_str()); + } else { + p->name = const_cast(p->hostname.c_str()); + } + p->group_index = host_groups[i][j]->group_index; + p->host_index = host_groups[i][j]->host_index; + hash_ring->insert(p, p->weight, &hash); + NH_Debug(NH_DEBUG_TAG, "Loading hash rings - ring: %d, host record: %d, name: %s, hostname: %s, stategy: %s", i, j, p->name, + p->hostname.c_str(), strategy_name.c_str()); + } + hash.clear(); + rings.push_back(std::move(hash_ring)); + } + return true; +} + +// returns a hash key calculated from the request and 'hash_key' configuration +// parameter. +uint64_t +NextHopConsistentHash::getHashKey(uint64_t sm_id, HttpRequestData *hrdata, ATSHash64 *h) +{ + URL *url = hrdata->hdr->url_get(); + URL *ps_url = nullptr; + int len = 0; + const char *url_string_ref = nullptr; + + // calculate a hash using the selected config. + switch (hash_key) { + case NH_URL_HASH_KEY: + url_string_ref = url->string_get_ref(&len, true); + if (url_string_ref && len > 0) { + h->update(url_string_ref, len); + NH_Debug(NH_DEBUG_TAG, "[%" PRIu64 "] url hash string: %s", sm_id, url_string_ref); + } + break; + // hostname hash + case NH_HOSTNAME_HASH_KEY: + url_string_ref = url->host_get(&len); + if (url_string_ref && len > 0) { + h->update(url_string_ref, len); + } + break; + // path + query string + case NH_PATH_QUERY_HASH_KEY: + url_string_ref = url->path_get(&len); + h->update("/", 1); + if (url_string_ref && len > 0) { + h->update(url_string_ref, len); + } + url_string_ref = url->query_get(&len); + if (url_string_ref && len > 0) { + h->update("?", 1); + h->update(url_string_ref, len); + } + break; + // path + fragment hash + case NH_PATH_FRAGMENT_HASH_KEY: + url_string_ref = url->path_get(&len); + h->update("/", 1); + if (url_string_ref && len > 0) { + h->update(url_string_ref, len); + } + url_string_ref = url->fragment_get(&len); + if (url_string_ref && len > 0) { + h->update("?", 1); + h->update(url_string_ref, len); + } + break; + // use the cache key created by the cache-key plugin. + case NH_CACHE_HASH_KEY: + ps_url = *(hrdata->cache_info_parent_selection_url); + if (ps_url) { + url_string_ref = ps_url->string_get_ref(&len); + if (url_string_ref && len > 0) { + NH_Debug(NH_DEBUG_TAG, "[%" PRIu64 "] using parent selection over-ride string:'%.*s'.", sm_id, len, url_string_ref); + h->update(url_string_ref, len); + } + } else { + url_string_ref = url->path_get(&len); + h->update("/", 1); + if (url_string_ref && len > 0) { + NH_Debug(NH_DEBUG_TAG, "[%" PRIu64 "] the parent selection over-ride url is not set, using default path: %s.", sm_id, + url_string_ref); + h->update(url_string_ref, len); + } + } + break; + // use the path as the hash, default. + case NH_PATH_HASH_KEY: + default: + url_string_ref = url->path_get(&len); + h->update("/", 1); + if (url_string_ref && len > 0) { + h->update(url_string_ref, len); + } + break; + } + + h->final(); + return h->get(); +} + +void +NextHopConsistentHash::findNextHop(const uint64_t sm_id, ParentResult &result, RequestData &rdata, const uint64_t fail_threshold, + const uint64_t retry_time, time_t now) +{ + time_t _now = now; + bool firstcall = false; + bool nextHopRetry = false; + bool wrapped = false; + std::vector wrap_around(groups, false); + uint32_t cur_ring = 0; // there is a hash ring for each host group + uint64_t hash_key = 0; + uint32_t lookups = 0; + ATSHash64Sip24 hash; + HttpRequestData *request_info = static_cast(&rdata); + HostRecord *hostRec = nullptr; + std::shared_ptr pRec = nullptr; + HostStatus &pStatus = HostStatus::instance(); + HostStatus_t host_stat = HostStatus_t::HOST_STATUS_INIT; + HostStatRec *hst = nullptr; + + if (result.line_number == -1 && result.result == PARENT_UNDEFINED) { + firstcall = true; + } + + if (firstcall) { + NH_Debug(NH_DEBUG_TAG, "[%" PRIu64 "] firstcall, line_number: %d, result: %s", sm_id, result.line_number, + ParentResultStr[result.result]); + result.line_number = distance; + cur_ring = 0; + for (uint32_t i = 0; i < groups; i++) { + result.chash_init[i] = false; + wrap_around[i] = false; + } + } else { + NH_Debug(NH_DEBUG_TAG, "[%" PRIu64 "] not firstcall, line_number: %d, result: %s", sm_id, result.line_number, + ParentResultStr[result.result]); + switch (ring_mode) { + case NH_ALTERNATE_RING: + if (groups > 1) { + cur_ring = (result.last_group + 1) % groups; + } else { + cur_ring = result.last_group; + } + break; + case NH_EXHAUST_RING: + default: + if (!wrapped) { + cur_ring = result.last_group; + } else if (groups > 1) { + cur_ring = (result.last_group + 1) % groups; + } + break; + } + } + + // Do the initial parent look-up. + hash_key = getHashKey(sm_id, request_info, &hash); + + do { // search until we've selected a different parent if !firstcall + std::shared_ptr r = rings[cur_ring]; + hostRec = chash_lookup(r, hash_key, &result.chashIter[cur_ring], &wrapped, &hash, &result.chash_init[cur_ring], + &result.mapWrapped[cur_ring], sm_id); + wrap_around[cur_ring] = wrapped; + lookups++; + // the 'available' flag is maintained in 'host_groups' and not the hash ring. + if (hostRec) { + pRec = host_groups[hostRec->group_index][hostRec->host_index]; + if (firstcall) { + hst = (pRec) ? pStatus.getHostStatus(pRec->hostname.c_str()) : nullptr; + result.first_choice_status = (hst) ? hst->status : HostStatus_t::HOST_STATUS_UP; + break; + } + } else { + pRec = nullptr; + } + } while (pRec && result.hostname && strcmp(pRec->hostname.c_str(), result.hostname) == 0); + + NH_Debug(NH_DEBUG_TAG, "[%" PRIu64 "] Initial parent lookups: %d", sm_id, lookups); + + // ---------------------------------------------------------------------------------------------------- + // Validate initial parent look-up and perform additional look-ups if required. + // ---------------------------------------------------------------------------------------------------- + + hst = (pRec) ? pStatus.getHostStatus(pRec->hostname.c_str()) : nullptr; + host_stat = (hst) ? hst->status : HostStatus_t::HOST_STATUS_UP; + // if the config ignore_self_detect is set to true and the host is down due to SELF_DETECT reason + // ignore the down status and mark it as avaialble + if ((pRec && ignore_self_detect) && (hst && hst->status == HOST_STATUS_DOWN)) { + if (hst->reasons == Reason::SELF_DETECT) { + host_stat = HOST_STATUS_UP; + } + } + if (!pRec || (pRec && !pRec->available) || host_stat == HOST_STATUS_DOWN) { + do { + // check if an unavailable server is now retryable, use it if it is. + if (pRec && !pRec->available && host_stat == HOST_STATUS_UP) { + _now == 0 ? _now = time(nullptr) : _now = now; + // check if the host is retryable. It's retryable if the retry window has elapsed + if ((pRec->failedAt + retry_time) < static_cast(_now)) { + nextHopRetry = true; + result.last_parent = pRec->host_index; + result.last_lookup = pRec->group_index; + result.retry = nextHopRetry; + result.result = PARENT_SPECIFIED; + NH_Debug(NH_DEBUG_TAG, "[%" PRIu64 "] next hop %s is now retryable, marked it available.", sm_id, pRec->hostname.c_str()); + break; + } + } + switch (ring_mode) { + case NH_ALTERNATE_RING: + if (groups > 0) { + cur_ring = (pRec->group_index + 1) % groups; + } + break; + case NH_EXHAUST_RING: + default: + if (wrap_around[cur_ring] && groups > 1) { + cur_ring = (cur_ring + 1) % groups; + } + break; + } + std::shared_ptr r = rings[cur_ring]; + hostRec = chash_lookup(r, hash_key, &result.chashIter[cur_ring], &wrapped, &hash, &result.chash_init[cur_ring], + &result.mapWrapped[cur_ring], sm_id); + wrap_around[cur_ring] = wrapped; + lookups++; + if (hostRec) { + pRec = host_groups[hostRec->group_index][hostRec->host_index]; + hst = (pRec) ? pStatus.getHostStatus(pRec->hostname.c_str()) : nullptr; + host_stat = (hst) ? hst->status : HostStatus_t::HOST_STATUS_UP; + // if the config ignore_self_detect is set to true and the host is down due to SELF_DETECT reason + // ignore the down status and mark it as avaialble + if ((pRec && ignore_self_detect) && (hst && hst->status == HOST_STATUS_DOWN)) { + if (hst->reasons == Reason::SELF_DETECT) { + host_stat = HOST_STATUS_UP; + } + } + NH_Debug(NH_DEBUG_TAG, "[%" PRIu64 "] Selected a new parent: %s, available: %s, wrapped: %s, lookups: %d.", sm_id, + pRec->hostname.c_str(), (pRec->available) ? "true" : "false", (wrapped) ? "true" : "false", lookups); + // use available host. + if (pRec->available && host_stat == HOST_STATUS_UP) { + break; + } + } else { + pRec = nullptr; + } + bool all_wrapped = true; + for (uint32_t c = 0; c < groups; c++) { + if (wrap_around[c] == false) { + all_wrapped = false; + } + } + if (all_wrapped) { + NH_Debug(NH_DEBUG_TAG, "[%" PRIu64 "] No available parents.", sm_id); + if (pRec) { + pRec = nullptr; + } + break; + } + } while (!pRec || (pRec && !pRec->available) || host_stat == HOST_STATUS_DOWN); + } + + // ---------------------------------------------------------------------------------------------------- + // Validate and return the final result. + // ---------------------------------------------------------------------------------------------------- + + // use the available or marked for retry parent. + hst = (pRec) ? pStatus.getHostStatus(pRec->hostname.c_str()) : nullptr; + host_stat = (hst) ? hst->status : HostStatus_t::HOST_STATUS_UP; + + if (pRec && host_stat == HOST_STATUS_UP && (pRec->available || result.retry)) { + result.result = PARENT_SPECIFIED; + result.hostname = pRec->hostname.c_str(); + result.last_parent = pRec->host_index; + result.last_lookup = result.last_group = cur_ring; + switch (scheme) { + case NH_SCHEME_NONE: + case NH_SCHEME_HTTP: + result.port = pRec->getPort(scheme); + break; + case NH_SCHEME_HTTPS: + result.port = pRec->getPort(scheme); + break; + } + result.retry = nextHopRetry; + ink_assert(result.hostname != nullptr); + ink_assert(result.port != 0); + NH_Debug(NH_DEBUG_TAG, "[%" PRIu64 "] Chosen parent: %s.%d", sm_id, result.hostname, result.port); + } else { + if (go_direct == true) { + result.result = PARENT_DIRECT; + } else { + result.result = PARENT_FAIL; + } + result.hostname = nullptr; + result.port = 0; + result.retry = false; + } + + return; +} diff --git a/proxy/http/remap/NextHopConsistentHash.h b/proxy/http/remap/NextHopConsistentHash.h new file mode 100644 index 00000000000..47fcf9e1aae --- /dev/null +++ b/proxy/http/remap/NextHopConsistentHash.h @@ -0,0 +1,54 @@ +/** @file + + Implementation of nexthop consistent hash selections strategies. + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#pragma once + +#include +#include +#include "NextHopSelectionStrategy.h" + +enum NHHashKeyType { + NH_URL_HASH_KEY = 0, + NH_HOSTNAME_HASH_KEY, + NH_PATH_HASH_KEY, // default, consistent hash uses the request url path + NH_PATH_QUERY_HASH_KEY, + NH_PATH_FRAGMENT_HASH_KEY, + NH_CACHE_HASH_KEY +}; + +class NextHopConsistentHash : public NextHopSelectionStrategy +{ + std::vector> rings; + + uint64_t getHashKey(uint64_t sm_id, HttpRequestData *hrdata, ATSHash64 *h); + +public: + NHHashKeyType hash_key = NH_PATH_HASH_KEY; + + NextHopConsistentHash() = delete; + NextHopConsistentHash(const std::string_view name, const NHPolicyType &policy) : NextHopSelectionStrategy(name, policy) {} + ~NextHopConsistentHash(); + bool Init(const YAML::Node &n); + void findNextHop(const uint64_t sm_id, ParentResult &result, RequestData &rdata, const uint64_t fail_threshold, + const uint64_t retry_time, time_t now = 0) override; +}; diff --git a/proxy/http/remap/NextHopRoundRobin.cc b/proxy/http/remap/NextHopRoundRobin.cc new file mode 100644 index 00000000000..8bdcbc59205 --- /dev/null +++ b/proxy/http/remap/NextHopRoundRobin.cc @@ -0,0 +1,219 @@ +/** @file + + Implementation of various round robin nexthop selections strategies. + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#include +#include + +#include "NextHopRoundRobin.h" + +NextHopRoundRobin::~NextHopRoundRobin() +{ + NH_Debug(NH_DEBUG_TAG, "destructor called for strategy named: %s", strategy_name.c_str()); +} + +void +NextHopRoundRobin::findNextHop(const uint64_t sm_id, ParentResult &result, RequestData &rdata, const uint64_t fail_threshold, + const uint64_t retry_time, time_t now) +{ + time_t _now = now; + bool firstcall = true; + bool parentUp = false; + bool parentRetry = false; + bool wrapped = result.wrap_around; + std::vector wrap_around(groups, false); + uint32_t cur_hst_index = 0; + uint32_t cur_grp_index = 0; + uint32_t hst_size = host_groups[cur_grp_index].size(); + uint32_t start_group = 0; + uint32_t start_host = 0; + std::shared_ptr cur_host; + HostStatus &pStatus = HostStatus::instance(); + HttpRequestData *request_info = static_cast(&rdata); + HostStatus_t host_stat = HostStatus_t::HOST_STATUS_UP; + + if (result.line_number != -1 && result.result != PARENT_UNDEFINED) { + firstcall = false; + } + + if (firstcall) { + // distance is the index into the strategies map, this is the equivalent to the old line_number in parent.config. + result.line_number = distance; + NH_Debug(NH_DEBUG_TAG, "[%" PRIu64 "] first call , cur_grp_index: %d, cur_hst_index: %d, distance: %d", sm_id, cur_grp_index, + cur_hst_index, distance); + switch (policy_type) { + case NH_FIRST_LIVE: + result.start_parent = cur_hst_index = 0; + cur_grp_index = 0; + break; + case NH_RR_STRICT: { + std::lock_guard lock(_mutex); + cur_hst_index = result.start_parent = this->hst_index; + cur_grp_index = 0; + this->hst_index = (this->hst_index + 1) % hst_size; + } break; + case NH_RR_IP: + cur_grp_index = 0; + if (rdata.get_client_ip() != nullptr) { + cur_hst_index = result.start_parent = ntohl(ats_ip_hash(rdata.get_client_ip())) % hst_size; + } else { + cur_hst_index = this->hst_index; + } + break; + case NH_RR_LATCHED: + cur_grp_index = 0; + cur_hst_index = result.start_parent = latched_index; + break; + default: + ink_assert(0); + break; + } + cur_host = host_groups[cur_grp_index][cur_hst_index]; + NH_Debug(NH_DEBUG_TAG, "[%" PRIu64 "] first call, cur_grp_index: %d, cur_hst_index: %d", sm_id, cur_grp_index, cur_hst_index); + } else { + NH_Debug(NH_DEBUG_TAG, "[%" PRIu64 "] next call, cur_grp_index: %d, cur_hst_index: %d, distance: %d", sm_id, cur_grp_index, + cur_hst_index, distance); + // Move to next parent due to failure + latched_index = cur_hst_index = (result.last_parent + 1) % hst_size; + cur_host = host_groups[cur_grp_index][cur_hst_index]; + + // Check to see if we have wrapped around + if (static_cast(cur_hst_index) == result.start_parent) { + // We've wrapped around so bypass if we can + if (go_direct == true) { + result.result = PARENT_DIRECT; + } else { + result.result = PARENT_FAIL; + } + result.hostname = nullptr; + result.port = 0; + result.wrap_around = true; + return; + } + } + start_group = cur_grp_index; + start_host = cur_hst_index; + + // Verify that the 'cur_hst' is available or retryable, if not loop through the array of parents seeing if any are up or + // should be retried + do { + HostStatRec *hst = pStatus.getHostStatus(cur_host->hostname.c_str()); + host_stat = (hst) ? hst->status : HostStatus_t::HOST_STATUS_UP; + // if the config ignore_self_detect is set to true and the host is down due to SELF_DETECT reason + // ignore the down status and mark it as avaialble + if (ignore_self_detect && (hst && hst->status == HOST_STATUS_DOWN)) { + if (hst->reasons == Reason::SELF_DETECT) { + host_stat = HOST_STATUS_UP; + } + } + + NH_Debug(NH_DEBUG_TAG, + "[%" PRIu64 "] Selected a parent, %s, failCount (faileAt: %d failCount: %d), FailThreshold: %" PRIu64 + ", request_info->xact_start: %ld", + sm_id, cur_host->hostname.c_str(), (unsigned)cur_host->failedAt, cur_host->failCount, fail_threshold, + request_info->xact_start); + // check if 'cur_host' is available, mark it up if it is. + if ((cur_host->failedAt == 0) || (cur_host->failCount < fail_threshold)) { + if (host_stat == HOST_STATUS_UP) { + NH_Debug(NH_DEBUG_TAG, + "[%" PRIu64 + "] Selecting a parent, %s, due to little failCount (faileAt: %d failCount: %d), FailThreshold: %" PRIu64, + sm_id, cur_host->hostname.c_str(), (unsigned)cur_host->failedAt, cur_host->failCount, fail_threshold); + parentUp = true; + } + } else { // if not available, check to see if it can be retried. If so, set the retry flag and temporairly mark it as + // available. + _now == 0 ? _now = time(nullptr) : _now = now; + if (((result.wrap_around) || (cur_host->failedAt + retry_time) < static_cast(_now)) && + host_stat == HOST_STATUS_UP) { + // Reuse the parent + parentUp = true; + parentRetry = true; + NH_Debug(NH_DEBUG_TAG, "[%" PRIu64 "] NextHop marked for retry %s:%d", sm_id, cur_host->hostname.c_str(), + host_groups[cur_grp_index][cur_hst_index]->getPort(scheme)); + } else { // not retryable or available. + parentUp = false; + } + } + NH_Debug(NH_DEBUG_TAG, "[%" PRIu64 "] parentUp: %s, hostname: %s, host status: %s", sm_id, parentUp ? "true" : "false", + cur_host->hostname.c_str(), HostStatusNames[host_stat]); + + // The selected host is available or retryable, return the search result. + if (parentUp == true && host_stat != HOST_STATUS_DOWN) { + NH_Debug(NH_DEBUG_TAG, "[%" PRIu64 "] status for %s: %s", sm_id, cur_host->hostname.c_str(), HostStatusNames[host_stat]); + result.result = PARENT_SPECIFIED; + result.hostname = cur_host->hostname.c_str(); + result.port = cur_host->getPort(scheme); + result.last_parent = cur_hst_index; + result.last_group = cur_grp_index; + result.retry = parentRetry; + ink_assert(result.hostname != nullptr); + ink_assert(result.port != 0); + NH_Debug(NH_DEBUG_TAG, "[%" PRIu64 "] Chosen parent = %s.%d", sm_id, result.hostname, result.port); + return; + } + + // only one host group is available, find another host if we have not wrapped. + if (groups == 1) { + latched_index = cur_hst_index = (cur_hst_index + 1) % hst_size; + if (start_host == cur_hst_index) { + wrap_around[cur_grp_index] = wrapped = result.wrap_around = true; + } + } else { // search the fail over groups. + if (ring_mode == NH_ALTERNATE_RING) { // use alternating ring mode. + cur_grp_index = (cur_grp_index + 1) % groups; + hst_size = host_groups[cur_grp_index].size(); + if (cur_grp_index == start_group) { + latched_index = cur_hst_index = (cur_hst_index + 1) % hst_size; + if (cur_hst_index == start_host) { + wrapped = wrap_around[cur_grp_index] = result.wrap_around = true; + } + } + } else { // use the exhaust ring mode. + latched_index = cur_hst_index = (cur_hst_index + 1) % hst_size; + if (cur_hst_index == start_host) { + wrap_around[cur_grp_index] = true; + cur_grp_index = (cur_grp_index + 1) % groups; + if (cur_grp_index == start_group) { + wrapped = wrap_around[cur_grp_index] = result.wrap_around = true; + } else { + start_host = cur_hst_index = 0; + } + } + } + } + cur_host = host_groups[cur_grp_index][cur_hst_index]; + NH_Debug( + NH_DEBUG_TAG, + "[%" PRIu64 "] host: %s, groups: %d, cur_grp_index: %d, cur_hst_index: %d, wrapped: %s, start_group: %d, start_host: %d", + sm_id, cur_host->hostname.c_str(), groups, cur_grp_index, cur_hst_index, wrapped ? "true" : "false", start_group, start_host); + } while (!wrapped); + + if (go_direct == true) { + result.result = PARENT_DIRECT; + } else { + result.result = PARENT_FAIL; + } + + result.hostname = nullptr; + result.port = 0; +} diff --git a/proxy/http/remap/NextHopRoundRobin.h b/proxy/http/remap/NextHopRoundRobin.h new file mode 100644 index 00000000000..80b05e01af5 --- /dev/null +++ b/proxy/http/remap/NextHopRoundRobin.h @@ -0,0 +1,45 @@ +/** @file + + Implementation of various round robin nexthop selections strategies. + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#pragma once + +#include +#include "NextHopSelectionStrategy.h" + +class NextHopRoundRobin : public NextHopSelectionStrategy +{ + std::mutex _mutex; + uint32_t latched_index = 0; + +public: + NextHopRoundRobin() = delete; + NextHopRoundRobin(const std::string_view &name, const NHPolicyType &policy) : NextHopSelectionStrategy(name, policy) {} + ~NextHopRoundRobin(); + bool + Init(const YAML::Node &n) + { + return NextHopSelectionStrategy::Init(n); + } + void findNextHop(const uint64_t sm_id, ParentResult &result, RequestData &rdata, const uint64_t fail_threshold, + const uint64_t retry_time, time_t now = 0) override; +}; diff --git a/proxy/http/remap/NextHopSelectionStrategy.cc b/proxy/http/remap/NextHopSelectionStrategy.cc new file mode 100644 index 00000000000..20d6a3915ec --- /dev/null +++ b/proxy/http/remap/NextHopSelectionStrategy.cc @@ -0,0 +1,389 @@ +/** @file + + A brief file description + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#include +#include "I_Machine.h" +#include "NextHopSelectionStrategy.h" + +// ring mode strings +constexpr std::string_view alternate_rings = "alternate_ring"; +constexpr std::string_view exhaust_rings = "exhaust_ring"; + +// health check strings +constexpr std::string_view active_health_check = "active"; +constexpr std::string_view passive_health_check = "passive"; + +constexpr const char *policy_strings[] = {"NH_UNDEFINED", "NH_FIRST_LIVE", "NH_RR_STRICT", + "NH_RR_IP", "NH_RR_LATCHED", "NH_CONSISTENT_HASH"}; + +NextHopSelectionStrategy::NextHopSelectionStrategy(const std::string_view &name, const NHPolicyType &policy) +{ + strategy_name = name; + policy_type = policy; + NH_Debug(NH_DEBUG_TAG, "Using a selection strategy of type %s", policy_strings[policy]); +} + +// +// parse out the data for this strategy. +// +bool +NextHopSelectionStrategy::Init(const YAML::Node &n) +{ + NH_Debug(NH_DEBUG_TAG, "calling Init()"); + + try { + if (n["scheme"]) { + auto scheme_val = n["scheme"].Scalar(); + if (scheme_val == "http") { + scheme = NH_SCHEME_HTTP; + } else if (scheme_val == "https") { + scheme = NH_SCHEME_HTTPS; + } else { + scheme = NH_SCHEME_NONE; + NH_Note("Invalid 'scheme' value, '%s', for the strategy named '%s', setting to NH_SCHEME_NONE", scheme_val.c_str(), + strategy_name.c_str()); + } + } + + // go_direct config. + if (n["go_direct"]) { + go_direct = n["go_direct"].as(); + } + + // parent_is_proxy config. + if (n["parent_is_proxy"]) { + parent_is_proxy = n["parent_is_proxy"].as(); + } + + // ignore_self_detect + if (n["ignore_self_detect"]) { + ignore_self_detect = n["ignore_self_detect"].as(); + } + + // failover node. + YAML::Node failover_node; + if (n["failover"]) { + failover_node = n["failover"]; + if (failover_node["ring_mode"]) { + auto ring_mode_val = failover_node["ring_mode"].Scalar(); + if (ring_mode_val == alternate_rings) { + ring_mode = NH_ALTERNATE_RING; + } else if (ring_mode_val == exhaust_rings) { + ring_mode = NH_EXHAUST_RING; + } else { + ring_mode = NH_ALTERNATE_RING; + NH_Note("Invalid 'ring_mode' value, '%s', for the strategy named '%s', using default '%s'.", ring_mode_val.c_str(), + strategy_name.c_str(), alternate_rings.data()); + } + } + if (failover_node["max_simple_retries"]) { + max_simple_retries = failover_node["max_simple_retries"].as(); + } + + YAML::Node resp_codes_node; + if (failover_node["response_codes"]) { + resp_codes_node = failover_node["response_codes"]; + if (resp_codes_node.Type() != YAML::NodeType::Sequence) { + NH_Error("Error in the response_codes definition for the strategy named '%s', skipping response_codes.", + strategy_name.c_str()); + } else { + for (auto &&k : resp_codes_node) { + auto code = k.as(); + if (code > 300 && code < 599) { + resp_codes.add(code); + } else { + NH_Note("Skipping invalid response code '%d' for the strategy named '%s'.", code, strategy_name.c_str()); + } + } + resp_codes.sort(); + } + } + YAML::Node health_check_node; + if (failover_node["health_check"]) { + health_check_node = failover_node["health_check"]; + if (health_check_node.Type() != YAML::NodeType::Sequence) { + NH_Error("Error in the health_check definition for the strategy named '%s', skipping health_checks.", + strategy_name.c_str()); + } else { + for (auto it = health_check_node.begin(); it != health_check_node.end(); ++it) { + auto health_check = it->as(); + if (health_check.compare(active_health_check) == 0) { + health_checks.active = true; + } + if (health_check.compare(passive_health_check) == 0) { + health_checks.passive = true; + } + } + } + } + } + + // parse and load the host data + YAML::Node groups_node; + if (n["groups"]) { + groups_node = n["groups"]; + // a groups list is required. + if (groups_node.Type() != YAML::NodeType::Sequence) { + throw std::invalid_argument("Invalid groups definition, expected a sequence, '" + strategy_name + "' cannot be loaded."); + } else { + Machine *mach = Machine::instance(); + HostStatus &h_stat = HostStatus::instance(); + uint32_t grp_size = groups_node.size(); + if (grp_size > MAX_GROUP_RINGS) { + NH_Note("the groups list exceeds the maximum of %d for the strategy '%s'. Only the first %d groups will be configured.", + MAX_GROUP_RINGS, strategy_name.c_str(), MAX_GROUP_RINGS); + groups = MAX_GROUP_RINGS; + } else { + groups = groups_node.size(); + } + // resize the hosts vector. + host_groups.reserve(groups); + // loop through the groups + for (unsigned int grp = 0; grp < groups; ++grp) { + YAML::Node hosts_list = groups_node[grp]; + + // a list of hosts is required. + if (hosts_list.Type() != YAML::NodeType::Sequence) { + throw std::invalid_argument("Invalid hosts definition, expected a sequence, '" + strategy_name + "' cannot be loaded."); + } else { + // loop through the hosts list. + std::vector> hosts_inner; + + for (unsigned int hst = 0; hst < hosts_list.size(); ++hst) { + std::shared_ptr host_rec = std::make_shared(hosts_list[hst].as()); + host_rec->group_index = grp; + host_rec->host_index = hst; + if (mach->is_self(host_rec->hostname.c_str())) { + h_stat.setHostStatus(host_rec->hostname.c_str(), HostStatus_t::HOST_STATUS_DOWN, 0, Reason::SELF_DETECT); + } + hosts_inner.push_back(std::move(host_rec)); + num_parents++; + } + host_groups.push_back(std::move(hosts_inner)); + } + } + } + } + } catch (std::exception &ex) { + NH_Note("Error parsing the strategy named '%s' due to '%s', this strategy will be ignored.", strategy_name.c_str(), ex.what()); + return false; + } + + return true; +} + +void +NextHopSelectionStrategy::markNextHopDown(const uint64_t sm_id, ParentResult &result, const uint64_t fail_threshold, + const uint64_t retry_time, time_t now) +{ + time_t _now; + now == 0 ? _now = time(nullptr) : _now = now; + uint32_t new_fail_count = 0; + + // Make sure that we are being called back with with a + // result structure with a selected parent. + if (result.result != PARENT_SPECIFIED) { + return; + } + // If we were set through the API we currently have not failover + // so just return fail + if (result.is_api_result()) { + ink_assert(0); + return; + } + uint32_t hst_size = host_groups[result.last_group].size(); + ink_assert(result.last_parent < hst_size); + std::shared_ptr h = host_groups[result.last_group][result.last_parent]; + + // If the parent has already been marked down, just increment + // the failure count. If this is the first mark down on a + // parent we need to both set the failure time and set + // count to one. If this was the result of a retry, we + // must update move the failedAt timestamp to now so that we + // continue negative cache the parent + if (h->failedAt == 0 || result.retry == true) { + { // start of lock_guard scope. + std::lock_guard lock(h->_mutex); + if (h->failedAt == 0) { + // Mark the parent failure time. + h->failedAt = _now; + if (result.retry == false) { + new_fail_count = h->failCount = 1; + } + } else if (result.retry == true) { + h->failedAt = _now; + } + } // end of lock_guard scope + NH_Note("[%" PRIu64 "] NextHop %s marked as down %s:%d", sm_id, (result.retry) ? "retry" : "initially", h->hostname.c_str(), + h->getPort(scheme)); + + } else { + int old_count = 0; + + // if the last failure was outside the retry window, set the failcount to 1 and failedAt to now. + { // start of lock_guard_scope + std::lock_guard lock(h->_mutex); + if ((h->failedAt + retry_time) < static_cast(_now)) { + h->failCount = 1; + h->failedAt = _now; + } else { + old_count = h->failCount = 1; + } + new_fail_count = old_count + 1; + } // end of lock_guard + NH_Debug(NH_DEBUG_TAG, "[%" PRIu64 "] Parent fail count increased to %d for %s:%d", sm_id, new_fail_count, h->hostname.c_str(), + h->getPort(scheme)); + } + + if (new_fail_count >= fail_threshold) { + h->set_unavailable(); + NH_Note("[%" PRIu64 "] Failure threshold met failcount:%d >= threshold:%" PRIu64 ", http parent proxy %s:%d marked down", sm_id, + new_fail_count, fail_threshold, h->hostname.c_str(), h->getPort(scheme)); + NH_Debug(NH_DEBUG_TAG, "[%" PRIu64 "] NextHop %s:%d marked unavailable, h->available=%s", sm_id, h->hostname.c_str(), + h->getPort(scheme), (h->available) ? "true" : "false"); + } +} + +void +NextHopSelectionStrategy::markNextHopUp(const uint64_t sm_id, ParentResult &result) +{ + // Make sure that we are being called back with with a + // result structure with a parent that is being retried + ink_assert(result.retry == true); + if (result.result != PARENT_SPECIFIED) { + return; + } + // If we were set through the API we currently have not failover + // so just return fail + if (result.is_api_result()) { + ink_assert(0); + return; + } + uint32_t hst_size = host_groups[result.last_group].size(); + ink_assert(result.last_parent < hst_size); + std::shared_ptr h = host_groups[result.last_group][result.last_parent]; + + if (!h->available) { + h->set_available(); + NH_Note("[%" PRIu64 "] http parent proxy %s:%d restored", sm_id, h->hostname.c_str(), h->getPort(scheme)); + } +} + +bool +NextHopSelectionStrategy::nextHopExists(const uint64_t sm_id) +{ + for (uint32_t gg = 0; gg < groups; gg++) { + for (auto &hh : host_groups[gg]) { + HostRecord *p = hh.get(); + if (p->available) { + NH_Debug(NH_DEBUG_TAG, "[%" PRIu64 "] found available next hop %s", sm_id, p->hostname.c_str()); + return true; + } + } + } + return false; +} + +namespace YAML +{ +template <> struct convert { + static bool + decode(const Node &node, HostRecord &nh) + { + YAML::Node nd; + bool merge_tag_used = false; + + // check for YAML merge tag. + if (node["<<"]) { + nd = node["<<"]; + merge_tag_used = true; + } else { + nd = node; + } + + // lookup the hostname + if (nd["host"]) { + nh.hostname = nd["host"].Scalar(); + } else { + throw std::invalid_argument("Invalid host definition, missing host name."); + } + + // lookup the port numbers supported by this host. + YAML::Node proto = nd["protocol"]; + + if (proto.Type() != YAML::NodeType::Sequence) { + throw std::invalid_argument("Invalid host protocol definition, expected a sequence."); + } else { + for (auto &&ii : proto) { + const YAML::Node &protocol_node = ii; + std::shared_ptr pr = std::make_shared(protocol_node.as()); + nh.protocols.push_back(std::move(pr)); + } + } + + // get the host's weight + YAML::Node weight; + if (merge_tag_used) { + weight = node["weight"]; + nh.weight = weight.as(); + } else if ((weight = nd["weight"])) { + nh.weight = weight.as(); + } else { + NH_Note("No weight is defined for the host '%s', using default 1.0", nh.hostname.data()); + nh.weight = 1.0; + } + + // get the host's optional hash_string + YAML::Node hash; + if ((hash = nd["hash_string"])) { + nh.hash_string = hash.Scalar(); + } + + return true; + } +}; + +template <> struct convert { + static bool + decode(const Node &node, NHProtocol &nh) + { + if (node["scheme"]) { + if (node["scheme"].Scalar() == "http") { + nh.scheme = NH_SCHEME_HTTP; + } else if (node["scheme"].Scalar() == "https") { + nh.scheme = NH_SCHEME_HTTPS; + } else { + nh.scheme = NH_SCHEME_NONE; + } + } + if (node["port"]) { + nh.port = node["port"].as(); + } + if (node["health_check_url"]) { + nh.health_check_url = node["health_check_url"].Scalar(); + } + return true; + } +}; +}; // namespace YAML +// namespace YAML diff --git a/proxy/http/remap/NextHopSelectionStrategy.h b/proxy/http/remap/NextHopSelectionStrategy.h new file mode 100644 index 00000000000..77a8b11cea6 --- /dev/null +++ b/proxy/http/remap/NextHopSelectionStrategy.h @@ -0,0 +1,216 @@ +/** @file + + A brief file description + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#pragma once + +#include "ParentSelection.h" + +#ifndef _NH_UNIT_TESTS_ +#define NH_Debug(tag, ...) Debug(tag, __VA_ARGS__) +#define NH_Error(...) DiagsError(DL_Error, __VA_ARGS__) +#define NH_Note(...) DiagsError(DL_Note, __VA_ARGS__) +#define NH_Warn(...) DiagsError(DL_Warning, __VA_ARGS__) +#else +#include "unit-tests/nexthop_test_stubs.h" +#endif /* _NH_UNIT_TESTS_ */ + +constexpr const char *NH_DEBUG_TAG = "next_hop"; + +namespace YAML +{ +class Node; +} + +enum NHPolicyType { + NH_UNDEFINED = 0, + NH_FIRST_LIVE, // first available nexthop + NH_RR_STRICT, // strict round robin + NH_RR_IP, // round robin by client ip. + NH_RR_LATCHED, // latched to available next hop. + NH_CONSISTENT_HASH // consistent hashing strategy. +}; + +enum NHSchemeType { NH_SCHEME_NONE = 0, NH_SCHEME_HTTP, NH_SCHEME_HTTPS }; + +enum NHRingMode { NH_ALTERNATE_RING = 0, NH_EXHAUST_RING }; + +enum NH_HHealthCheck { NH_ACTIVE, NH_PASSIVE }; + +// response codes container +struct ResponseCodes { + ResponseCodes(){}; + std::vector codes; + void + add(short code) + { + codes.push_back(code); + } + bool + contains(short code) + { + return std::binary_search(codes.begin(), codes.end(), code); + } + void + sort() + { + std::sort(codes.begin(), codes.end()); + } +}; + +struct HealthChecks { + bool active = false; + bool passive = false; +}; + +struct NHProtocol { + NHSchemeType scheme; + uint32_t port; + std::string health_check_url; +}; + +struct HostRecord : ATSConsistentHashNode { + std::mutex _mutex; + std::string hostname; + time_t failedAt; + uint32_t failCount; + time_t upAt; + float weight; + std::string hash_string; + int host_index; + int group_index; + std::vector> protocols; + + // construct without locking the _mutex. + HostRecord() + { + hostname = ""; + failedAt = 0; + failCount = 0; + upAt = 0; + weight = 0; + hash_string = ""; + host_index = -1; + group_index = -1; + available = true; + } + + // copy constructor to avoid copying the _mutex. + HostRecord(const HostRecord &o) + { + hostname = o.hostname; + failedAt = o.failedAt; + failCount = o.failCount; + upAt = o.upAt; + weight = o.weight; + hash_string = ""; + host_index = -1; + group_index = -1; + available = true; + protocols = o.protocols; + } + + // assign without copying the _mutex. + HostRecord & + operator=(const HostRecord &o) + { + hostname = o.hostname; + failedAt = o.failedAt; + upAt = o.upAt; + weight = o.weight; + hash_string = o.hash_string; + host_index = o.host_index; + group_index = o.group_index; + available = o.available; + protocols = o.protocols; + return *this; + } + + // locks the record when marking this host down. + void + set_unavailable() + { + if (available) { + std::lock_guard lock(_mutex); + failedAt = time(nullptr); + available = false; + } + } + + // locks the record when marking this host up. + void + set_available() + { + if (!available) { + std::lock_guard lock(_mutex); + failedAt = 0; + failCount = 0; + upAt = time(nullptr); + available = true; + } + } + + int + getPort(NHSchemeType scheme) + { + int port = 0; + for (uint32_t i = 0; i < protocols.size(); i++) { + if (protocols[i]->scheme == scheme) { + port = protocols[i]->port; + break; + } + } + return port; + } +}; + +class NextHopSelectionStrategy +{ +public: + NextHopSelectionStrategy(); + NextHopSelectionStrategy(const std::string_view &name, const NHPolicyType &type); + virtual ~NextHopSelectionStrategy(){}; + bool Init(const YAML::Node &n); + virtual void findNextHop(const uint64_t sm_id, ParentResult &result, RequestData &rdata, const uint64_t fail_threshold, + const uint64_t retry_time, time_t now = 0) = 0; + void markNextHopDown(const uint64_t sm_id, ParentResult &result, const uint64_t fail_threshold, const uint64_t retry_time, + time_t now = 0); + void markNextHopUp(const uint64_t sm_id, ParentResult &result); + bool nextHopExists(const uint64_t sm_id); + + std::string strategy_name; + bool go_direct = true; + bool parent_is_proxy = true; + bool ignore_self_detect = false; + NHPolicyType policy_type = NH_UNDEFINED; + NHSchemeType scheme = NH_SCHEME_NONE; + NHRingMode ring_mode = NH_ALTERNATE_RING; + ResponseCodes resp_codes; + HealthChecks health_checks; + std::vector>> host_groups; + uint32_t max_simple_retries = 1; + uint32_t groups = 0; + uint32_t grp_index = 0; + uint32_t hst_index = 0; + uint32_t num_parents = 0; + uint32_t distance = 0; // index into the strategies list. +}; diff --git a/proxy/http/remap/NextHopStrategyFactory.cc b/proxy/http/remap/NextHopStrategyFactory.cc new file mode 100644 index 00000000000..4de0b8aa521 --- /dev/null +++ b/proxy/http/remap/NextHopStrategyFactory.cc @@ -0,0 +1,260 @@ +/** @file + + A brief file description + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#include + +#include +#include + +#include "NextHopStrategyFactory.h" +#include "NextHopConsistentHash.h" +#include "NextHopRoundRobin.h" + +NextHopStrategyFactory::NextHopStrategyFactory(const char *file) +{ + YAML::Node config; + YAML::Node strategies; + std::stringstream doc; + std::unordered_set include_once; + std::string_view fn = file; + + // strategy policies. + constexpr std::string_view consistent_hash = "consistent_hash"; + constexpr std::string_view first_live = "first_live"; + constexpr std::string_view rr_strict = "rr_strict"; + constexpr std::string_view rr_ip = "rr_ip"; + constexpr std::string_view latched = "latched"; + + strategies_loaded = true; + const char *basename = fn.substr(fn.find_last_of('/') + 1).data(); + + // load the strategies yaml config file. + try { + NH_Note("%s loading ...", basename); + loadConfigFile(fn.data(), doc, include_once); + + config = YAML::Load(doc); + if (config.IsNull()) { + NH_Note("No NextHop strategy configs were loaded."); + strategies_loaded = false; + } else { + strategies = config["strategies"]; + if (strategies.Type() != YAML::NodeType::Sequence) { + NH_Error("malformed %s file, expected a 'strategies' sequence", basename); + strategies_loaded = false; + } + } + // loop through the strategies document. + for (auto &&strategie : strategies) { + YAML::Node strategy = strategie; + auto name = strategy["strategy"].as(); + auto policy = strategy["policy"]; + if (!policy) { + NH_Error("No policy is defined for the strategy named '%s', this strategy will be ignored.", name.c_str()); + continue; + } + const auto &policy_value = policy.Scalar(); + NHPolicyType policy_type = NH_UNDEFINED; + + if (policy_value == consistent_hash) { + policy_type = NH_CONSISTENT_HASH; + } else if (policy_value == first_live) { + policy_type = NH_FIRST_LIVE; + } else if (policy_value == rr_strict) { + policy_type = NH_RR_STRICT; + } else if (policy_value == rr_ip) { + policy_type = NH_RR_IP; + } else if (policy_value == latched) { + policy_type = NH_RR_LATCHED; + } + if (policy_type == NH_UNDEFINED) { + NH_Error("Invalid policy '%s' for the strategy named '%s', this strategy will be ignored.", policy_value.c_str(), + name.c_str()); + } else { + createStrategy(name, policy_type, strategy); + } + } + } catch (std::exception &ex) { + NH_Note("%s", ex.what()); + strategies_loaded = false; + } + if (strategies_loaded) { + NH_Note("%s finished loading", basename); + } +} + +NextHopStrategyFactory::~NextHopStrategyFactory() +{ + NH_Debug(NH_DEBUG_TAG, "destroying NextHopStrategyFactory"); +} + +void +NextHopStrategyFactory::createStrategy(const std::string &name, const NHPolicyType policy_type, const YAML::Node &node) +{ + std::shared_ptr strat; + std::shared_ptr strat_rr; + std::shared_ptr strat_chash; + + strat = strategyInstance(name.c_str()); + if (strat != nullptr) { + NH_Note("A strategy named '%s' has already been loaded and another will not be created.", name.data()); + return; + } + + switch (policy_type) { + case NH_FIRST_LIVE: + case NH_RR_STRICT: + case NH_RR_IP: + case NH_RR_LATCHED: + strat_rr = std::make_shared(name, policy_type); + if (strat_rr->Init(node)) { + _strategies.emplace(std::make_pair(std::string(name), strat_rr)); + } else { + strat.reset(); + } + break; + case NH_CONSISTENT_HASH: + strat_chash = std::make_shared(name, policy_type); + if (strat_chash->Init(node)) { + _strategies.emplace(std::make_pair(std::string(name), strat_chash)); + } else { + strat_chash.reset(); + } + break; + default: // handles P_UNDEFINED, no strategy is added + break; + }; +} + +std::shared_ptr +NextHopStrategyFactory::strategyInstance(const char *name) +{ + std::shared_ptr ps_strategy; + + if (!strategies_loaded) { + NH_Error("no strategy configurations were defined, see defintions in '%s' file", fn.c_str()); + return nullptr; + } else { + auto it = _strategies.find(name); + if (it == _strategies.end()) { + // NH_Error("no strategy found for name: %s", name); + return nullptr; + } else { + ps_strategy = it->second; + ps_strategy->distance = std::distance(_strategies.begin(), it); + } + } + + return ps_strategy; +} + +/* + * loads the contents of a file into a std::stringstream document. If the file has a '#include file' + * directive, that 'file' is read into the document beginning at the the point where the + * '#include' was found. This allows the 'strategy' and 'hosts' yaml files to be separate. The + * 'strategy' yaml file would then normally have the '#include hosts.yml' in it's begining. + */ +void +NextHopStrategyFactory::loadConfigFile(const std::string &fileName, std::stringstream &doc, + std::unordered_set &include_once) +{ + const char *sep = " \t"; + char *tok, *last; + struct stat buf; + std::string line; + + if (stat(fileName.c_str(), &buf) == -1) { + std::string err_msg = strerror(errno); + throw std::invalid_argument("Unable to stat '" + fileName + "': " + err_msg); + } + + // if fileName is a directory, concatenate all '.yaml' files alphanumerically + // into a single document stream. No #include is supported. + if (S_ISDIR(buf.st_mode)) { + DIR *dir = nullptr; + struct dirent *dir_ent = nullptr; + std::vector files; + + NH_Note("loading strategy YAML files from the directory %s", fileName.c_str()); + if ((dir = opendir(fileName.c_str())) == nullptr) { + std::string err_msg = strerror(errno); + throw std::invalid_argument("Unable to open the directory '" + fileName + "': " + err_msg); + } else { + while ((dir_ent = readdir(dir)) != nullptr) { + // filename should be greater that 6 characters to have a '.yaml' suffix. + if (strlen(dir_ent->d_name) < 6) { + continue; + } + std::string_view sv = dir_ent->d_name; + if (sv.find(".yaml", sv.size() - 5) == sv.size() - 5) { + files.push_back(sv); + } + } + // sort the files alphanumerically + std::sort(files.begin(), files.end(), + [](const std::string_view lhs, const std::string_view rhs) { return lhs.compare(rhs) < 0; }); + + for (auto &i : files) { + std::ifstream file(fileName + "/" + i.data()); + if (file.is_open()) { + while (std::getline(file, line)) { + if (line[0] == '#') { + // continue; + } + doc << line << "\n"; + } + file.close(); + } else { + throw std::invalid_argument("Unable to open and read '" + fileName + "/" + i.data() + "'"); + } + } + } + closedir(dir); + } else { + std::ifstream file(fileName); + if (file.is_open()) { + while (std::getline(file, line)) { + if (line[0] == '#') { + tok = strtok_r(const_cast(line.c_str()), sep, &last); + if (tok != nullptr && strcmp(tok, "#include") == 0) { + std::string f = strtok_r(nullptr, sep, &last); + if (include_once.find(f) == include_once.end()) { + include_once.insert(f); + // try to load included file. + try { + loadConfigFile(f, doc, include_once); + } catch (std::exception &ex) { + throw std::invalid_argument("Unable to open included file '" + f + "' from '" + fileName + "'"); + } + } + } + } else { + doc << line << "\n"; + } + } + file.close(); + } else { + throw std::invalid_argument("Unable to open and read '" + fileName + "'"); + } + } +} diff --git a/proxy/http/remap/NextHopStrategyFactory.h b/proxy/http/remap/NextHopStrategyFactory.h new file mode 100644 index 00000000000..ecb25a8943c --- /dev/null +++ b/proxy/http/remap/NextHopStrategyFactory.h @@ -0,0 +1,54 @@ +/** @file + + A brief file description + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#pragma once + +#include +#include +#include +#include + +#include "tscore/Diags.h" +#include "NextHopSelectionStrategy.h" + +namespace YAML +{ +class Node; +}; + +class NextHopStrategyFactory +{ +public: + NextHopStrategyFactory() = delete; + NextHopStrategyFactory(const char *file); + ~NextHopStrategyFactory(); + std::shared_ptr strategyInstance(const char *name); + + bool strategies_loaded; + +private: + std::string fn; + void loadConfigFile(const std::string &file, std::stringstream &doc, std::unordered_set &include_once); + void createStrategy(const std::string &name, const NHPolicyType policy_type, const YAML::Node &node); + std::unordered_map> _strategies; +}; diff --git a/proxy/http/remap/PluginDso.cc b/proxy/http/remap/PluginDso.cc index 24490fc044d..34280062aeb 100644 --- a/proxy/http/remap/PluginDso.cc +++ b/proxy/http/remap/PluginDso.cc @@ -28,15 +28,21 @@ */ #include "PluginDso.h" +#include "P_Freer.h" +#include "P_EventSystem.h" #ifdef PLUGIN_DSO_TESTS #include "unit-tests/plugin_testing_common.h" #else #include "tscore/Diags.h" +#define PluginDebug Debug +#define PluginError Error #endif PluginDso::PluginDso(const fs::path &configPath, const fs::path &effectivePath, const fs::path &runtimePath) : _configPath(configPath), _effectivePath(effectivePath), _runtimePath(runtimePath) { + PluginDebug(_tag, "PluginDso (%p) created _configPath: [%s] _effectivePath: [%s] _runtimePath: [%s]", this, _configPath.c_str(), + _effectivePath.c_str(), _runtimePath.c_str()); } PluginDso::~PluginDso() @@ -58,30 +64,30 @@ PluginDso::load(std::string &error) return false; } - Debug(_tag, "plugin '%s' started loading DSO", _configPath.c_str()); + PluginDebug(_tag, "plugin '%s' started loading DSO", _configPath.c_str()); /* Find plugin DSO looking through the search dirs */ if (_effectivePath.empty()) { error.append("empty effective path"); result = false; } else { - Debug(_tag, "plugin '%s' effective path: %s", _configPath.c_str(), _effectivePath.c_str()); + PluginDebug(_tag, "plugin '%s' effective path: %s", _configPath.c_str(), _effectivePath.c_str()); - /* Copy the installed plugin DSO to a runtime directory */ + /* Copy the installed plugin DSO to a runtime directory if dynamic reload enabled */ std::error_code ec; - if (!copy(_effectivePath, _runtimePath, ec)) { + if (isDynamicReloadEnabled() && !copy(_effectivePath, _runtimePath, ec)) { std::string temp_error; temp_error.append("failed to create a copy: ").append(strerror(ec.value())); error.assign(temp_error); result = false; } else { - Debug(_tag, "plugin '%s' runtime path: %s", _configPath.c_str(), _runtimePath.c_str()); + PluginDebug(_tag, "plugin '%s' runtime path: %s", _configPath.c_str(), _runtimePath.c_str()); /* Save the time for later checking if DSO got modified in consecutive DSO reloads */ std::error_code ec; fs::file_status fs = fs::status(_effectivePath, ec); _mtime = fs::modification_time(fs); - Debug(_tag, "plugin '%s' mоdification time %ld", _configPath.c_str(), _mtime); + PluginDebug(_tag, "plugin '%s' modification time %ld", _configPath.c_str(), _mtime); /* Now attemt to load the plugin DSO */ if ((_dlh = dlopen(_runtimePath.c_str(), RTLD_NOW)) == nullptr) { @@ -96,7 +102,7 @@ PluginDso::load(std::string &error) clean(error); result = false; - Error("plugin '%s' failed to load: %s", _configPath.c_str(), error.c_str()); + PluginError("plugin '%s' failed to load: %s", _configPath.c_str(), error.c_str()); } } @@ -105,7 +111,7 @@ PluginDso::load(std::string &error) clean(error); } } - Debug(_tag, "plugin '%s' finished loading DSO", _configPath.c_str()); + PluginDebug(_tag, "plugin '%s' finished loading DSO", _configPath.c_str()); return result; } @@ -211,6 +217,18 @@ PluginDso::modTime() const return _mtime; } +/** + * @brief file handle returned by dlopen syscall + * + * @return dlopen filehandle + */ + +void * +PluginDso::dlOpenHandle() const +{ + return _dlh; +} + /** * @brief clean files created by the plugin instance and handle errors * @@ -220,6 +238,10 @@ PluginDso::modTime() const void PluginDso::clean(std::string &error) { + if (!isDynamicReloadEnabled()) { + return; + } + if (false == remove(_runtimePath, _errorCode)) { error.append("failed to remove runtime copy: ").append(_errorCode.message()); } @@ -229,17 +251,16 @@ void PluginDso::acquire() { this->refcount_inc(); - Debug(_tag, "plugin DSO acquire (ref-count:%d, dso-addr:%p)", this->refcount(), this); + PluginDebug(_tag, "plugin DSO acquire (ref-count:%d, dso-addr:%p)", this->refcount(), this); } void PluginDso::release() { - Debug(_tag, "plugin DSO release (ref-count:%d, dso-addr:%p)", this->refcount() - 1, this); + PluginDebug(_tag, "plugin DSO release (ref-count:%d, dso-addr:%p)", this->refcount() - 1, this); if (0 == this->refcount_dec()) { - Debug(_tag, "unloading plugin DSO '%s' (dso-addr:%p)", _configPath.c_str(), this); - _list.erase(this); - delete this; + PluginDebug(_tag, "unloading plugin DSO '%s' (dso-addr:%p)", _configPath.c_str(), this); + _plugins->remove(this); } } @@ -247,14 +268,14 @@ void PluginDso::incInstanceCount() { _instanceCount.refcount_inc(); - Debug(_tag, "instance count (inst-count:%d, dso-addr:%p)", _instanceCount.refcount(), this); + PluginDebug(_tag, "instance count (inst-count:%d, dso-addr:%p)", _instanceCount.refcount(), this); } void PluginDso::decInstanceCount() { _instanceCount.refcount_dec(); - Debug(_tag, "instance count (inst-count:%d, dso-addr:%p)", _instanceCount.refcount(), this); + PluginDebug(_tag, "instance count (inst-count:%d, dso-addr:%p)", _instanceCount.refcount(), this); } int @@ -263,4 +284,128 @@ PluginDso::instanceCount() return _instanceCount.refcount(); } -PluginDso::PluginList PluginDso::_list; +bool +PluginDso::isDynamicReloadEnabled() const +{ + return (_runtimePath != _effectivePath); +} + +void +PluginDso::LoadedPlugins::add(PluginDso *plugin) +{ + SCOPED_MUTEX_LOCK(lock, _mutex, this_ethread()); + + _list.append(plugin); +} + +void +PluginDso::LoadedPlugins::remove(PluginDso *plugin) +{ + SCOPED_MUTEX_LOCK(lock, _mutex, this_ethread()); + + _list.erase(plugin); + this_ethread()->schedule_imm(new DeleterContinuation(plugin)); +} + +/* check if need to reload the plugin DSO + * if dynamic reload not enabled: check if plugin Dso with same effective path already loaded + * if dynamic reload enabled: check if plugin Dso with same effective path and same time stamp already loaded + * return pointer to already loaded plugin if found, else return null + */ +PluginDso * +PluginDso::LoadedPlugins::findByEffectivePath(const fs::path &path, bool dynamicReloadEnabled) +{ + struct stat sb; + time_t mtime = 0; + if (0 == stat(path.c_str(), &sb)) { + mtime = sb.st_mtime; + } + + SCOPED_MUTEX_LOCK(lock, _mutex, this_ethread()); + + auto spot = std::find_if(_list.begin(), _list.end(), [&](PluginDso const &plugin) -> bool { + return ((!dynamicReloadEnabled || (mtime == plugin.modTime())) && + (0 == path.string().compare(plugin.effectivePath().string()))); + }); + return spot == _list.end() ? nullptr : static_cast(spot); +} + +void +PluginDso::LoadedPlugins::indicatePreReload(const char *factoryId) +{ + SCOPED_MUTEX_LOCK(lock, _mutex, this_ethread()); + + PluginDebug(_tag, "indicated config is going to be reloaded by factory '%s' to %zu plugin%s", factoryId, _list.count(), + _list.count() != 1 ? "s" : ""); + + _list.apply([](PluginDso &plugin) -> void { plugin.indicatePreReload(); }); +} + +void +PluginDso::LoadedPlugins::indicatePostReload(bool reloadSuccessful, const std::unordered_map &pluginUsed, + const char *factoryId) +{ + SCOPED_MUTEX_LOCK(lock, _mutex, this_ethread()); + + PluginDebug(_tag, "indicated config is done reloading by factory '%s' to %zu plugin%s", factoryId, _list.count(), + _list.count() != 1 ? "s" : ""); + + for (auto &plugin : _list) { + TSRemapReloadStatus status = TSREMAP_CONFIG_RELOAD_FAILURE; + if (reloadSuccessful) { + /* reload succeeded but was the plugin instantiated by this factory? */ + status = (pluginUsed.end() == pluginUsed.find(&plugin) ? TSREMAP_CONFIG_RELOAD_SUCCESS_PLUGIN_UNUSED : + TSREMAP_CONFIG_RELOAD_SUCCESS_PLUGIN_USED); + } + plugin.indicatePostReload(status); + } +} + +bool +PluginDso::LoadedPlugins::addPluginPathToDsoOptOutTable(std::string_view pluginPath) +{ + std::error_code ec; + auto effectivePath = fs::canonical(fs::path{pluginPath}, ec); + + if (ec) { + PluginError("Error getting the canonical path: %s", ec.message().c_str()); + return false; + } + + { + SCOPED_MUTEX_LOCK(lock, _mutex, this_ethread()); + _optoutDsoReloadPlugins.push_front(DisableDSOReloadPluginInfo{effectivePath}); + } + + return true; +} + +void +PluginDso::LoadedPlugins::removePluginPathFromDsoOptOutTable(std::string_view pluginPath) +{ + std::error_code ec; + auto effectivePath = fs::canonical(fs::path{pluginPath}, ec); + + if (ec) { + PluginError("Error getting the canonical path: %s", ec.message().c_str()); + return; + } + + { + SCOPED_MUTEX_LOCK(lock, _mutex, this_ethread()); + _optoutDsoReloadPlugins.remove_if([&effectivePath](auto const &info) { return info.dsoEffectivePath == effectivePath; }); + } +} + +bool +PluginDso::LoadedPlugins::isPluginInDsoOptOutTable(const fs::path &effectivePath) +{ + SCOPED_MUTEX_LOCK(lock, _mutex, this_ethread()); + const auto found = std::find_if(std::begin(this->_optoutDsoReloadPlugins), std::end(this->_optoutDsoReloadPlugins), + [&effectivePath](const auto &info) { return info.dsoEffectivePath == effectivePath; }); + + // if is found, then the plugin opt out. + return (found != std::end(this->_optoutDsoReloadPlugins)); +} + +Ptr PluginDso::_plugins; diff --git a/proxy/http/remap/PluginDso.h b/proxy/http/remap/PluginDso.h index 4554c6b8756..9e6df2d1a74 100644 --- a/proxy/http/remap/PluginDso.h +++ b/proxy/http/remap/PluginDso.h @@ -29,16 +29,23 @@ #pragma once +#include #include #include +#include #include +#include "ts/apidefs.h" +#include "ts/remap.h" #include "tscore/ts_file.h" namespace fs = ts::file; #include "tscore/Ptr.h" +#include "I_EventSystem.h" #include "tscpp/util/IntrusiveDList.h" +#include "Plugin.h" + class PluginThreadContext : public RefCountObj { public: @@ -65,6 +72,7 @@ class PluginDso : public PluginThreadContext const fs::path &effectivePath() const; const fs::path &runtimePath() const; time_t modTime() const; + void *dlOpenHandle() const; /* List used by the plugin factory */ using self_type = PluginDso; ///< Self reference type. @@ -73,10 +81,11 @@ class PluginDso : public PluginThreadContext using Linkage = ts::IntrusiveLinkage; using PluginList = ts::IntrusiveDList; - /* Methods to be called when processing a list of plugins, to overloaded by the remap or the global plugins correspondingly */ - virtual void indicateReload() = 0; - virtual bool init(std::string &error) = 0; - virtual void done() = 0; + /* Methods to be called when processing a list of plugins, to be overloaded by the remap or the global plugins correspondingly */ + virtual void indicatePreReload() = 0; + virtual void indicatePostReload(TSRemapReloadStatus reloadStatus) = 0; + virtual bool init(std::string &error) = 0; + virtual void done() = 0; void acquire(); void release(); @@ -84,6 +93,61 @@ class PluginDso : public PluginThreadContext void incInstanceCount(); void decInstanceCount(); int instanceCount(); + bool isDynamicReloadEnabled() const; + + class LoadedPlugins : public RefCountObj + { + public: + LoadedPlugins() : _mutex(new_ProxyMutex()) {} + void add(PluginDso *plugin); + void remove(PluginDso *plugin); + PluginDso *findByEffectivePath(const fs::path &path, bool dynamicReloadEnabled); + void indicatePreReload(const char *factoryId); + void indicatePostReload(bool reloadSuccessful, const std::unordered_map &pluginUsed, const char *factoryId); + + /** + * @brief Check if the opt out table contains the passed plugin's effective path. + * @return true If the plugin was marked to not participate in the dynamic reload by + * TSPluginDSOReloadEnable() API. + * @param effectivePath canonical value of the plugin's path. + */ + bool isPluginInDsoOptOutTable(const fs::path &effectivePath); + /** + * @brief Add the plugin's path to the opt out table in order to let the Plugin Factory + * that this plugin is not interested in taking part of the dynamic reloading. + * This function will store the plugin's canonical path. + * @param effectivePath Plugin's path. + * @return false if any errors converting the plugin's path to a canonical path, true otherwise. + */ + bool addPluginPathToDsoOptOutTable(std::string_view pluginPath); + /** + * @brief Removes the passed plugin's effective path from the opt out list. + * @note This is mostly used by unit test than needs to remove the plugin's effectivePath + * from the out out list. + * @param pluginPath plugin's path. + * @note This function is mainly for unit tests purposes. + */ + void removePluginPathFromDsoOptOutTable(std::string_view pluginPath); + + private: + PluginList _list; /** @brief plugin list */ + Ptr _mutex; /** @brief mutex used when updating the plugin list from multiple threads */ + + struct DisableDSOReloadPluginInfo { + fs::path dsoEffectivePath; + }; + + std::forward_list _optoutDsoReloadPlugins; + }; + + static const Ptr & + loadedPlugins() + { + if (!_plugins) { + _plugins = new LoadedPlugins(); + } + return _plugins; + } protected: void clean(std::string &error); @@ -99,6 +163,7 @@ class PluginDso : public PluginThreadContext time_t _mtime = 0; /* @brief modification time of the DSO's file, used for checking */ bool _preventiveCleaning = true; - static PluginList _list; /** @brief a global list of plugins, usually maintained by a plugin factory or plugin instance itself */ + static Ptr + _plugins; /** @brief a global list of plugins, usually maintained by a plugin factory or plugin instance itself */ RefCountObj _instanceCount; /** @brief used for properly calling "done" and "indicate config reload" methods by the factory */ }; diff --git a/proxy/http/remap/PluginFactory.cc b/proxy/http/remap/PluginFactory.cc index c6b1c8a7f31..1837ddd7411 100644 --- a/proxy/http/remap/PluginFactory.cc +++ b/proxy/http/remap/PluginFactory.cc @@ -22,41 +22,52 @@ */ +#include + #include "RemapPluginInfo.h" #include "PluginFactory.h" #ifdef PLUGIN_DSO_TESTS #include "unit-tests/plugin_testing_common.h" #else #include "tscore/Diags.h" +#define PluginDebug Debug +#define PluginError Error #endif +#include "P_EventSystem.h" #include /* std::swap */ RemapPluginInst::RemapPluginInst(RemapPluginInfo &plugin) : _plugin(plugin) { _plugin.acquire(); - _plugin.incInstanceCount(); } RemapPluginInst::~RemapPluginInst() { - _plugin.decInstanceCount(); _plugin.release(); } -bool -RemapPluginInst::init(int argc, char **argv, std::string &error) +RemapPluginInst * +RemapPluginInst::init(RemapPluginInfo *plugin, int argc, char **argv, std::string &error) { - bool result = false; - result = _plugin.initInstance(argc, argv, &_instance, error); - - return result; + RemapPluginInst *inst = new RemapPluginInst(*plugin); + if (plugin->initInstance(argc, argv, &(inst->_instance), error)) { + plugin->incInstanceCount(); + return inst; + } + delete inst; + return nullptr; } void RemapPluginInst::done() { + _plugin.decInstanceCount(); _plugin.doneInstance(_instance); + + if (0 == _plugin.instanceCount()) { + _plugin.done(); + } } TSRemapStatus @@ -83,7 +94,7 @@ PluginFactory::PluginFactory() } } - Debug(_tag, "created plugin factory %s", getUuid()); + PluginDebug(_tag, "created plugin factory %s", getUuid()); } PluginFactory::~PluginFactory() @@ -93,7 +104,7 @@ PluginFactory::~PluginFactory() fs::remove(_runtimeDir, _ec); - Debug(_tag, "destroyed plugin factory %s", getUuid()); + PluginDebug(_tag, "destroyed plugin factory %s", getUuid()); delete _uuid; } @@ -101,7 +112,7 @@ PluginFactory & PluginFactory::addSearchDir(const fs::path &searchDir) { _searchDirs.push_back(searchDir); - Debug(_tag, "added plugin search dir %s", searchDir.c_str()); + PluginDebug(_tag, "added plugin search dir %s", searchDir.c_str()); return *this; } @@ -109,7 +120,7 @@ PluginFactory & PluginFactory::setRuntimeDir(const fs::path &runtimeDir) { _runtimeDir = runtimeDir / fs::path(getUuid()); - Debug(_tag, "set plugin runtime dir %s", runtimeDir.c_str()); + PluginDebug(_tag, "set plugin runtime dir %s", runtimeDir.c_str()); return *this; } @@ -130,56 +141,84 @@ PluginFactory::getUuid() * @return pointer to a plugin instance, nullptr if failure */ RemapPluginInst * -PluginFactory::getRemapPlugin(const fs::path &configPath, int argc, char **argv, std::string &error) +PluginFactory::getRemapPlugin(const fs::path &configPath, int argc, char **argv, std::string &error, bool dynamicReloadEnabled) { /* Discover the effective path by looking into the search dirs */ fs::path effectivePath = getEffectivePath(configPath); if (effectivePath.empty()) { error.assign("failed to find plugin '").append(configPath.string()).append("'"); + // The error will be reported by the caller but add debug log entry with this tag for convenience. + PluginDebug(_tag, "%s", error.c_str()); return nullptr; } + // The plugin may have opt out by `TSPluginDSOReloadEnable`, let's check and overwrite + if (dynamicReloadEnabled && PluginDso::loadedPlugins()->isPluginInDsoOptOutTable(effectivePath)) { + // plugin not interested to be reload. + PluginDebug(_tag, "Plugin %s not interested in taking part of the reload.", effectivePath.c_str()); + dynamicReloadEnabled = false; + } + /* Only one plugin with this effective path can be loaded by a plugin factory */ - RemapPluginInfo *plugin = dynamic_cast(findByEffectivePath(effectivePath)); + RemapPluginInfo *plugin = dynamic_cast(findByEffectivePath(effectivePath, dynamicReloadEnabled)); RemapPluginInst *inst = nullptr; if (nullptr == plugin) { /* The plugin requested have not been loaded yet. */ - Debug(_tag, "plugin '%s' has not been loaded yet, loading as remap plugin", configPath.c_str()); + PluginDebug(_tag, "plugin '%s' has not been loaded yet, loading as remap plugin", configPath.c_str()); fs::path runtimePath; - runtimePath /= _runtimeDir; - runtimePath /= effectivePath.relative_path(); - fs::path parent = runtimePath.parent_path(); - if (!fs::create_directories(parent, _ec)) { - error.assign("failed to create plugin runtime dir"); - return nullptr; + // if dynamic reload enabled then create a temporary location to copy .so and load from there + // else load from original location + if (dynamicReloadEnabled) { + runtimePath /= _runtimeDir; + runtimePath /= effectivePath.relative_path(); + + fs::path parent = runtimePath.parent_path(); + PluginDebug(_tag, "Using effectivePath: [%s] runtimePath: [%s] parent: [%s]", effectivePath.c_str(), runtimePath.c_str(), + parent.c_str()); + if (!fs::create_directories(parent, _ec)) { + error.assign("failed to create plugin runtime dir"); + return nullptr; + } + } else { + runtimePath = effectivePath; + PluginDebug(_tag, "Using effectivePath: [%s] runtimePath: [%s]", effectivePath.c_str(), runtimePath.c_str()); } plugin = new RemapPluginInfo(configPath, effectivePath, runtimePath); if (nullptr != plugin) { if (plugin->load(error)) { - _list.append(plugin); - if (plugin->init(error)) { - inst = new RemapPluginInst(*plugin); - inst->init(argc, argv, error); - _instList.append(inst); + PluginDso::loadedPlugins()->add(plugin); + inst = RemapPluginInst::init(plugin, argc, argv, error); + if (nullptr != inst) { + /* Plugin loading and instance init went fine. */ + _instList.append(inst); + } + } else { + /* Plugin DSO load succeeded but instance init failed. */ + PluginDebug(_tag, "plugin '%s' instance init failed", configPath.c_str()); + plugin->unload(error); + delete plugin; } - if (_preventiveCleaning) { + if (dynamicReloadEnabled && _preventiveCleaning) { clean(error); } } else { - return nullptr; + /* Plugin DSO load failed. */ + PluginDebug(_tag, "plugin '%s' DSO load failed", configPath.c_str()); + delete plugin; } } } else { - Debug(_tag, "plugin '%s' has already been loaded", configPath.c_str()); - inst = new RemapPluginInst(*plugin); - inst->init(argc, argv, error); - _instList.append(inst); + PluginDebug(_tag, "plugin '%s' has already been loaded", configPath.c_str()); + inst = RemapPluginInst::init(plugin, argc, argv, error); + if (nullptr != inst) { + _instList.append(inst); + } } return inst; @@ -222,39 +261,48 @@ PluginFactory::getEffectivePath(const fs::path &configPath) * @return plugin found or nullptr if not found */ PluginDso * -PluginFactory::findByEffectivePath(const fs::path &path) +PluginFactory::findByEffectivePath(const fs::path &path, bool dynamicReloadEnabled) { - struct stat sb; - time_t mtime = 0; - if (0 == stat(path.c_str(), &sb)) { - mtime = sb.st_mtime; - } - auto spot = std::find_if(_list.begin(), _list.end(), [&](PluginDso const &plugin) -> bool { - return (0 == path.string().compare(plugin.effectivePath().string()) && (mtime == plugin.modTime())); - }); - return spot == _list.end() ? nullptr : static_cast(spot); + return PluginDso::loadedPlugins()->findByEffectivePath(path, dynamicReloadEnabled); } /** - * @brief Tell all plugins (that so wish) that remap.config is being reloaded + * @brief Tell all plugins instantiated by this factory that the configuration + * they are using is no longer the active one. * * This method would be useful only in case configs are reloaded independently from * factory/plugins instantiation and initialization. */ void -PluginFactory::indicateReload() +PluginFactory::deactivate() { - Debug(_tag, "indicated config reload to factory '%s'", getUuid()); + PluginDebug(_tag, "deactivate configuration used by factory '%s'", getUuid()); _instList.apply([](RemapPluginInst &pluginInst) -> void { pluginInst.done(); }); +} - _list.apply([](PluginDso &plugin) -> void { - if (1 == plugin.instanceCount()) { - plugin.done(); - } else { - plugin.indicateReload(); - } - }); +/** + * @brief Tell all plugins (that so wish) that remap.config is going to be reloaded + */ +void +PluginFactory::indicatePreReload() +{ + PluginDso::loadedPlugins()->indicatePreReload(getUuid()); +} + +/** + * @brief Tell all plugins (that so wish) that remap.config is done reloading + */ +void +PluginFactory::indicatePostReload(bool reloadSuccessful) +{ + /* Find out which plugins (DSO) are actually instantiated by this factory */ + std::unordered_map pluginUsed; + for (auto &inst : _instList) { + pluginUsed[&(inst._plugin)]++; + } + + PluginDso::loadedPlugins()->indicatePostReload(reloadSuccessful, pluginUsed, getUuid()); } void diff --git a/proxy/http/remap/PluginFactory.h b/proxy/http/remap/PluginFactory.h index 1cb0661903e..eb2ca5c1366 100644 --- a/proxy/http/remap/PluginFactory.h +++ b/proxy/http/remap/PluginFactory.h @@ -47,7 +47,7 @@ class RemapPluginInst ~RemapPluginInst(); /* Used by the PluginFactory */ - bool init(int argc, char **argv, std::string &error); + static RemapPluginInst *init(RemapPluginInfo *plugin, int argc, char **argv, std::string &error); void done(); /* Used by the traffic server core while processing requests */ @@ -82,11 +82,12 @@ class RemapPluginInst * filesystem links and different dl library implementations. * * @note This is meant to unify the way global and remap plugins are (re)loaded (global plugin support is not implemented yet). + * @note In the case of a mixed plugin, getRemapPlugin/dynamicReloadEnabled can be internally overwritten if the plugin opt out to + * take part in the dynamic reload process. */ class PluginFactory { - using PluginInstList = ts::IntrusiveDList; - PluginDso::PluginList &_list = PluginDso::_list; + using PluginInstList = ts::IntrusiveDList; public: PluginFactory(); @@ -95,15 +96,17 @@ class PluginFactory PluginFactory &setRuntimeDir(const fs::path &runtimeDir); PluginFactory &addSearchDir(const fs::path &searchDir); - RemapPluginInst *getRemapPlugin(const fs::path &configPath, int argc, char **argv, std::string &error); + RemapPluginInst *getRemapPlugin(const fs::path &configPath, int argc, char **argv, std::string &error, bool dynamicReloadEnabled); virtual const char *getUuid(); void clean(std::string &error); - void indicateReload(); + void deactivate(); + void indicatePreReload(); + void indicatePostReload(bool reloadSuccessful); protected: - PluginDso *findByEffectivePath(const fs::path &path); + PluginDso *findByEffectivePath(const fs::path &path, bool dynamicReloadEnabled); fs::path getEffectivePath(const fs::path &configPath); std::vector _searchDirs; /** @brief ordered list of search paths where we look for plugins */ diff --git a/proxy/http/remap/RemapConfig.cc b/proxy/http/remap/RemapConfig.cc index e231c098b40..d8a2ce58bbf 100644 --- a/proxy/http/remap/RemapConfig.cc +++ b/proxy/http/remap/RemapConfig.cc @@ -31,6 +31,7 @@ #include "tscore/ink_cap.h" #include "tscore/ink_file.h" #include "tscore/Tokenizer.h" +#include "tscore/ts_file.h" #include "IPAllow.h" #include "PluginFactory.h" @@ -691,6 +692,14 @@ remap_check_option(const char **argv, int argc, unsigned long findmode, int *_re idx = i; } ret_flags |= REMAP_OPTFLG_INTERNAL; + } else if (!strncasecmp(argv[i], "strategy=", 9)) { + if ((findmode & REMAP_OPTFLG_STRATEGY) != 0) { + idx = i; + } + if (argptr) { + *argptr = &argv[i][9]; + } + ret_flags |= REMAP_OPTFLG_STRATEGY; } else { Warning("ignoring invalid remap option '%s'", argv[i]); } @@ -809,12 +818,14 @@ remap_load_plugin(const char **argv, int argc, url_mapping *mp, char *errbuf, in REC_ReadConfigInteger(elevate_access, "proxy.config.plugin.load_elevated"); ElevateAccess access(elevate_access ? ElevateAccess::FILE_PRIVILEGE : 0); - pi = rewrite->pluginFactory.getRemapPlugin(ts::file::path(const_cast(c)), parc, pargv, error); + pi = rewrite->pluginFactory.getRemapPlugin(ts::file::path(const_cast(c)), parc, pargv, error, + isPluginDynamicReloadEnabled()); } // done elevating access bool result = true; if (nullptr == pi) { snprintf(errbuf, errbufsize, "%s", error.c_str()); + result = false; } else { mp->add_plugin_instance(pi); } @@ -836,7 +847,6 @@ process_regex_mapping_config(const char *from_host_lower, url_mapping *new_mappi const char *to_host; int to_host_len; int substitution_id; - int substitution_count = 0; int captures; reg_map->to_url_host_template = nullptr; @@ -866,10 +876,6 @@ process_regex_mapping_config(const char *from_host_lower, url_mapping *new_mappi to_host = new_mapping->toURL.host_get(&to_host_len); for (int i = 0; i < (to_host_len - 1); ++i) { if (to_host[i] == '$') { - if (substitution_count > UrlRewrite::MAX_REGEX_SUBS) { - Warning("Cannot have more than %d substitutions in mapping with host [%s]", UrlRewrite::MAX_REGEX_SUBS, from_host_lower); - goto lFail; - } substitution_id = to_host[i + 1] - '0'; if ((substitution_id < 0) || (substitution_id > captures)) { Warning("Substitution id [%c] has no corresponding capture pattern in regex [%s]", to_host[i + 1], from_host_lower); @@ -936,15 +942,22 @@ remap_parse_config_bti(const char *path, BUILD_TABLE_INFO *bti) bool is_cur_mapping_regex; const char *type_id_str; - ats_scoped_str file_buf(readIntoBuffer(path, modulePrefix, nullptr)); - if (!file_buf) { - Warning("can't load remapping configuration file %s", path); - return false; + std::error_code ec; + std::string content{ts::file::load(ts::file::path{path}, ec)}; + if (ec) { + switch (ec.value()) { + case ENOENT: + Warning("Can't open remapping configuration file %s - %s", path, strerror(ec.value())); + break; + default: + Error("Failed load remapping configuration file %s - %s", path, strerror(ec.value())); + return false; + } } Debug("url_rewrite", "[BuildTable] UrlRewrite::BuildTable()"); - for (cur_line = tokLine(file_buf, &tok_state, '\\'); cur_line != nullptr;) { + for (cur_line = tokLine(content.data(), &tok_state, '\\'); cur_line != nullptr;) { reg_map = nullptr; new_mapping = nullptr; errStrBuf[0] = 0; @@ -968,7 +981,7 @@ remap_parse_config_bti(const char *path, BUILD_TABLE_INFO *bti) --cur_line_tmp; } - if ((cur_line_size = strlen(cur_line)) <= 0 || *cur_line == '#' || *cur_line == '\0') { + if (strlen(cur_line) <= 0 || *cur_line == '#' || *cur_line == '\0') { cur_line = tokLine(nullptr, &tok_state, '\\'); ++cln; continue; @@ -1042,7 +1055,7 @@ remap_parse_config_bti(const char *path, BUILD_TABLE_INFO *bti) new_mapping = new url_mapping(); // apply filter rules if we have to - if ((errStr = process_filter_opt(new_mapping, bti, errStrBuf, sizeof(errStrBuf))) != nullptr) { + if (process_filter_opt(new_mapping, bti, errStrBuf, sizeof(errStrBuf)) != nullptr) { errStr = errStrBuf; goto MAP_ERROR; } @@ -1073,12 +1086,13 @@ remap_parse_config_bti(const char *path, BUILD_TABLE_INFO *bti) } new_mapping->fromURL.create(nullptr); - rparse = new_mapping->fromURL.parse_no_path_component_breakdown(tmp, length); + rparse = new_mapping->fromURL.parse_regex(tmp, length); map_from_start[origLength] = '\0'; // Unwhack if (rparse != PARSE_RESULT_DONE) { - errStr = "malformed From URL"; + snprintf(errStrBuf, sizeof(errStrBuf), "malformed From URL: %.*s", length, tmp); + errStr = errStrBuf; goto MAP_ERROR; } @@ -1088,11 +1102,12 @@ remap_parse_config_bti(const char *path, BUILD_TABLE_INFO *bti) tmp = map_to; new_mapping->toURL.create(nullptr); - rparse = new_mapping->toURL.parse_no_path_component_breakdown(tmp, length); + rparse = new_mapping->toURL.parse_no_host_check(std::string_view(tmp, length)); map_to_start[origLength] = '\0'; // Unwhack if (rparse != PARSE_RESULT_DONE) { - errStr = "malformed To URL"; + snprintf(errStrBuf, sizeof(errStrBuf), "malformed To URL: %.*s", length, tmp); + errStr = errStrBuf; goto MAP_ERROR; } @@ -1261,6 +1276,25 @@ remap_parse_config_bti(const char *path, BUILD_TABLE_INFO *bti) } } + // check for a 'strategy' and if wire it up if one exists. + if ((bti->remap_optflg & REMAP_OPTFLG_STRATEGY) != 0 && + (maptype == FORWARD_MAP || maptype == FORWARD_MAP_REFERER || maptype == FORWARD_MAP_WITH_RECV_PORT)) { + const char *strategy = strchr(bti->argv[0], static_cast('=')); + if (strategy == nullptr) { + errStr = "missing 'strategy' name argument, unable to add mapping rule"; + goto MAP_ERROR; + } else { + strategy++; + new_mapping->strategy = bti->rewrite->strategyFactory->strategyInstance(strategy); + if (!new_mapping->strategy) { + snprintf(errStrBuf, sizeof(errStrBuf), "no strategy named '%s' is defined in the config", strategy); + errStr = errStrBuf; + goto MAP_ERROR; + } + Debug("url_rewrite_regex", "mapped the 'strategy' named %s", strategy); + } + } + // Check "remap" plugin options and load .so object if ((bti->remap_optflg & REMAP_OPTFLG_PLUGIN) != 0 && (maptype == FORWARD_MAP || maptype == FORWARD_MAP_REFERER || maptype == FORWARD_MAP_WITH_RECV_PORT)) { @@ -1320,9 +1354,16 @@ remap_parse_config(const char *path, UrlRewrite *rewrite) { BUILD_TABLE_INFO bti; - // If this happens to be a config reload, the list of loaded remap plugins is non-empty, and we - // can signal all these plugins that a reload has begun. - rewrite->pluginFactory.indicateReload(); + /* If this happens to be a config reload, the list of loaded remap plugins is non-empty, and we + * can signal all these plugins that a reload has begun. */ + rewrite->pluginFactory.indicatePreReload(); + bti.rewrite = rewrite; - return remap_parse_config_bti(path, &bti); + bool status = remap_parse_config_bti(path, &bti); + + /* Now after we parsed the configuration and (re)loaded plugins and plugin instances + * accordingly notify all plugins that we are done */ + rewrite->pluginFactory.indicatePostReload(status); + + return status; } diff --git a/proxy/http/remap/RemapConfig.h b/proxy/http/remap/RemapConfig.h index 8cac3d821b3..276b1218e47 100644 --- a/proxy/http/remap/RemapConfig.h +++ b/proxy/http/remap/RemapConfig.h @@ -38,6 +38,7 @@ class UrlRewrite; #define REMAP_OPTFLG_ACTION 0x0020u /* "action=" option (used for ACL filtering) */ #define REMAP_OPTFLG_INTERNAL 0x0040u /* only allow internal requests to hit this remap */ #define REMAP_OPTFLG_IN_IP 0x0080u /* "in_ip=" option (used for ACL filtering)*/ +#define REMAP_OPTFLG_STRATEGY 0x0100u /* "strategy=" the name of the nexthop selection strategy */ #define REMAP_OPTFLG_MAP_ID 0x0800u /* associate a map ID with this rule */ #define REMAP_OPTFLG_INVERT 0x80000000u /* "invert" the rule (for src_ip at least) */ #define REMAP_OPTFLG_ALL_FILTERS (REMAP_OPTFLG_METHOD | REMAP_OPTFLG_SRC_IP | REMAP_OPTFLG_ACTION | REMAP_OPTFLG_INTERNAL) diff --git a/proxy/http/remap/RemapPluginInfo.cc b/proxy/http/remap/RemapPluginInfo.cc index 8a1c00ed134..736fd4d6727 100644 --- a/proxy/http/remap/RemapPluginInfo.cc +++ b/proxy/http/remap/RemapPluginInfo.cc @@ -34,6 +34,8 @@ #include "unit-tests/plugin_testing_common.h" #else #include "tscore/Diags.h" +#define PluginDebug Debug +#define PluginError Error #endif /** @@ -52,7 +54,9 @@ RemapPluginInfo::getFunctionSymbol(const char *symbol) { std::string error; /* ignore the error, return nullptr if symbol not defined */ void *address = nullptr; - getSymbol(symbol, address, error); + if (getSymbol(symbol, address, error)) { + PluginDebug(_tag, "plugin '%s' found symbol '%s'", _configPath.c_str(), symbol); + } return reinterpret_cast(address); } @@ -81,13 +85,14 @@ RemapPluginInfo::load(std::string &error) return false; } - init_cb = getFunctionSymbol(TSREMAP_FUNCNAME_INIT); - config_reload_cb = getFunctionSymbol(TSREMAP_FUNCNAME_CONFIG_RELOAD); - done_cb = getFunctionSymbol(TSREMAP_FUNCNAME_DONE); - new_instance_cb = getFunctionSymbol(TSREMAP_FUNCNAME_NEW_INSTANCE); - delete_instance_cb = getFunctionSymbol(TSREMAP_FUNCNAME_DELETE_INSTANCE); - do_remap_cb = getFunctionSymbol(TSREMAP_FUNCNAME_DO_REMAP); - os_response_cb = getFunctionSymbol(TSREMAP_FUNCNAME_OS_RESPONSE); + init_cb = getFunctionSymbol(TSREMAP_FUNCNAME_INIT); + pre_config_reload_cb = getFunctionSymbol(TSREMAP_FUNCNAME_PRE_CONFIG_RELOAD); + post_config_reload_cb = getFunctionSymbol(TSREMAP_FUNCNAME_POST_CONFIG_RELOAD); + done_cb = getFunctionSymbol(TSREMAP_FUNCNAME_DONE); + new_instance_cb = getFunctionSymbol(TSREMAP_FUNCNAME_NEW_INSTANCE); + delete_instance_cb = getFunctionSymbol(TSREMAP_FUNCNAME_DELETE_INSTANCE); + do_remap_cb = getFunctionSymbol(TSREMAP_FUNCNAME_DO_REMAP); + os_response_cb = getFunctionSymbol(TSREMAP_FUNCNAME_OS_RESPONSE); /* Validate if the callback TSREMAP functions are specified correctly in the plugin. */ bool valid = true; @@ -106,9 +111,9 @@ RemapPluginInfo::load(std::string &error) } if (valid) { - Debug(_tag, "plugin '%s' callbacks validated", _configPath.c_str()); + PluginDebug(_tag, "plugin '%s' callbacks validated", _configPath.c_str()); } else { - Error("plugin '%s' callbacks validation failed: %s", _configPath.c_str(), error.c_str()); + PluginError("plugin '%s' callbacks validation failed: %s", _configPath.c_str(), error.c_str()); } return valid; } @@ -120,7 +125,7 @@ RemapPluginInfo::init(std::string &error) TSRemapInterface ri; bool result = true; - Debug(_tag, "started initializing plugin '%s'", _configPath.c_str()); + PluginDebug(_tag, "started initializing plugin '%s'", _configPath.c_str()); /* A buffer to get the error from the plugin instance init function, be defensive here. */ char tmpbuf[2048]; @@ -142,7 +147,7 @@ RemapPluginInfo::init(std::string &error) resetPluginContext(); - Debug(_tag, "finished initializing plugin '%s'", _configPath.c_str()); + PluginDebug(_tag, "finished initializing plugin '%s'", _configPath.c_str()); return result; } @@ -162,7 +167,7 @@ RemapPluginInfo::initInstance(int argc, char **argv, void **ih, std::string &err TSReturnCode res = TS_SUCCESS; bool result = true; - Debug(_tag, "started initializing instance of plugin '%s'", _configPath.c_str()); + PluginDebug(_tag, "started initializing instance of plugin '%s'", _configPath.c_str()); /* A buffer to get the error from the plugin instance init function, be defensive here. */ char tmpbuf[2048]; @@ -195,7 +200,7 @@ RemapPluginInfo::initInstance(int argc, char **argv, void **ih, std::string &err } } - Debug(_tag, "finished initializing instance of plugin '%s'", _configPath.c_str()); + PluginDebug(_tag, "finished initializing instance of plugin '%s'", _configPath.c_str()); return result; } @@ -243,12 +248,24 @@ RemapPluginInfo::osResponse(void *ih, TSHttpTxn rh, int os_response_type) RemapPluginInfo::~RemapPluginInfo() {} void -RemapPluginInfo::indicateReload() +RemapPluginInfo::indicatePreReload() +{ + setPluginContext(); + + if (pre_config_reload_cb) { + pre_config_reload_cb(); + } + + resetPluginContext(); +} + +void +RemapPluginInfo::indicatePostReload(TSRemapReloadStatus reloadStatus) { setPluginContext(); - if (config_reload_cb) { - config_reload_cb(); + if (post_config_reload_cb) { + post_config_reload_cb(reloadStatus); } resetPluginContext(); @@ -259,12 +276,12 @@ RemapPluginInfo::setPluginContext() { _tempContext = pluginThreadContext; pluginThreadContext = this; - Debug(_tag, "change plugin context from dso-addr:%p to dso-addr:%p", pluginThreadContext, _tempContext); + PluginDebug(_tag, "change plugin context from dso-addr:%p to dso-addr:%p", pluginThreadContext, _tempContext); } inline void RemapPluginInfo::resetPluginContext() { - Debug(_tag, "change plugin context from dso-addr:%p to dso-addr:%p (restore)", this, pluginThreadContext); + PluginDebug(_tag, "change plugin context from dso-addr:%p to dso-addr:%p (restore)", this, pluginThreadContext); pluginThreadContext = _tempContext; } diff --git a/proxy/http/remap/RemapPluginInfo.h b/proxy/http/remap/RemapPluginInfo.h index cc5941db64a..16156dc749c 100644 --- a/proxy/http/remap/RemapPluginInfo.h +++ b/proxy/http/remap/RemapPluginInfo.h @@ -36,13 +36,14 @@ class url_mapping; extern thread_local PluginThreadContext *pluginThreadContext; -static constexpr const char *const TSREMAP_FUNCNAME_INIT = "TSRemapInit"; -static constexpr const char *const TSREMAP_FUNCNAME_CONFIG_RELOAD = "TSRemapConfigReload"; -static constexpr const char *const TSREMAP_FUNCNAME_DONE = "TSRemapDone"; -static constexpr const char *const TSREMAP_FUNCNAME_NEW_INSTANCE = "TSRemapNewInstance"; -static constexpr const char *const TSREMAP_FUNCNAME_DELETE_INSTANCE = "TSRemapDeleteInstance"; -static constexpr const char *const TSREMAP_FUNCNAME_DO_REMAP = "TSRemapDoRemap"; -static constexpr const char *const TSREMAP_FUNCNAME_OS_RESPONSE = "TSRemapOSResponse"; +static constexpr const char *const TSREMAP_FUNCNAME_INIT = "TSRemapInit"; +static constexpr const char *const TSREMAP_FUNCNAME_PRE_CONFIG_RELOAD = "TSRemapPreConfigReload"; +static constexpr const char *const TSREMAP_FUNCNAME_POST_CONFIG_RELOAD = "TSRemapPostConfigReload"; +static constexpr const char *const TSREMAP_FUNCNAME_DONE = "TSRemapDone"; +static constexpr const char *const TSREMAP_FUNCNAME_NEW_INSTANCE = "TSRemapNewInstance"; +static constexpr const char *const TSREMAP_FUNCNAME_DELETE_INSTANCE = "TSRemapDeleteInstance"; +static constexpr const char *const TSREMAP_FUNCNAME_DO_REMAP = "TSRemapDoRemap"; +static constexpr const char *const TSREMAP_FUNCNAME_OS_RESPONSE = "TSRemapOSResponse"; /** * Holds information for a remap plugin, remap specific callback entry points for plugin init/done and instance init/done, do_remap, @@ -53,8 +54,10 @@ class RemapPluginInfo : public PluginDso public: /// Initialization function, called on library load. using Init_F = TSReturnCode(TSRemapInterface *api_info, char *errbuf, int errbuf_size); - /// Reload function, called to inform the plugin of a configuration reload. - using Reload_F = void(); + /// Reload function, called to inform the plugin that configuration is going to be reloaded. + using PreReload_F = void(); + /// Reload function, called to inform the plugin that configuration is done reloading. + using PostReload_F = void(TSRemapReloadStatus); /// Called when remapping for a transaction has finished. using Done_F = void(); /// Create an rule instance. @@ -68,7 +71,8 @@ class RemapPluginInfo : public PluginDso void *dl_handle = nullptr; /* "handle" for the dynamic library */ Init_F *init_cb = nullptr; - Reload_F *config_reload_cb = nullptr; + PreReload_F *pre_config_reload_cb = nullptr; + PostReload_F *post_config_reload_cb = nullptr; Done_F *done_cb = nullptr; New_Instance_F *new_instance_cb = nullptr; Delete_Instance_F *delete_instance_cb = nullptr; @@ -79,11 +83,11 @@ class RemapPluginInfo : public PluginDso ~RemapPluginInfo(); /* Overload to add / execute remap plugin specific tasks during the plugin loading */ - virtual bool load(std::string &error); + bool load(std::string &error) override; /* Used by the factory to invoke callbacks during plugin load, init and unload */ - virtual bool init(std::string &error); - virtual void done(void); + bool init(std::string &error) override; + void done(void) override; /* Used by the facility that handles remap plugin instances to invoke callbacks per plugin instance */ bool initInstance(int argc, char **argv, void **ih, std::string &error); @@ -94,7 +98,8 @@ class RemapPluginInfo : public PluginDso void osResponse(void *ih, TSHttpTxn rh, int os_response_type); /* Used by traffic server core to indicate configuration reload */ - virtual void indicateReload(); + void indicatePreReload() override; + void indicatePostReload(TSRemapReloadStatus reloadStatus) override; protected: /* Utility to be used only with unit testing */ diff --git a/proxy/http/remap/RemapProcessor.cc b/proxy/http/remap/RemapProcessor.cc index 50928862cb5..3081ae7b712 100644 --- a/proxy/http/remap/RemapProcessor.cc +++ b/proxy/http/remap/RemapProcessor.cc @@ -26,16 +26,6 @@ RemapProcessor remapProcessor; extern ClassAllocator pluginAllocator; -int -RemapProcessor::start(int num_threads, size_t stacksize) -{ - if (_use_separate_remap_thread) { - ET_REMAP = eventProcessor.spawn_event_threads("ET_REMAP", num_threads, stacksize); // ET_REMAP is a class member - } - - return 0; -} - /** Most of this comes from UrlRewrite::Remap(). Generally, all this does is set "map" to the appropriate entry from the HttpSM's leased m_remap @@ -154,6 +144,11 @@ RemapProcessor::finish_remap(HttpTransact::State *s, UrlRewrite *table) if (!map) { return false; } + + // if there is a configured next hop strategy, make it available in the state. + if (map->strategy) { + s->next_hop_strategy = map->strategy; + } // Do fast ACL filtering (it is safe to check map here) table->PerformACLFiltering(s, map); @@ -300,29 +295,11 @@ RemapProcessor::perform_remap(Continuation *cont, HttpTransact::State *s) return ACTION_RESULT_DONE; } - if (_use_separate_remap_thread) { - RemapPlugins *plugins = pluginAllocator.alloc(); - - plugins->setState(s); - plugins->setRequestUrl(request_url); - plugins->setRequestHeader(request_header); - plugins->setHostHeaderInfo(hh_info); + RemapPlugins plugins(s, request_url, request_header, hh_info); - // Execute "inline" if not using separate remap threads. - ink_assert(cont->mutex->thread_holding == this_ethread()); - plugins->mutex = cont->mutex; - plugins->action = cont; - SET_CONTINUATION_HANDLER(plugins, &RemapPlugins::run_remap); - eventProcessor.schedule_imm(plugins, ET_REMAP); - - return &plugins->action; - } else { - RemapPlugins plugins(s, request_url, request_header, hh_info); - - while (!plugins.run_single_remap()) { - ; // EMPTY - } - - return ACTION_RESULT_DONE; + while (!plugins.run_single_remap()) { + ; // EMPTY } + + return ACTION_RESULT_DONE; } diff --git a/proxy/http/remap/RemapProcessor.h b/proxy/http/remap/RemapProcessor.h index d5e5aced5b2..e21aee5f697 100644 --- a/proxy/http/remap/RemapProcessor.h +++ b/proxy/http/remap/RemapProcessor.h @@ -35,31 +35,16 @@ #define EVENT_REMAP_ERROR (REMAP_EVENT_EVENTS_START + 1) #define EVENT_REMAP_COMPLETE (REMAP_EVENT_EVENTS_START + 2) -class RemapProcessor : public Processor +class RemapProcessor { public: RemapProcessor() {} - ~RemapProcessor() override {} + ~RemapProcessor() {} bool setup_for_remap(HttpTransact::State *s, UrlRewrite *table); bool finish_remap(HttpTransact::State *s, UrlRewrite *table); Action *perform_remap(Continuation *cont, HttpTransact::State *s); - int start(int num_threads, size_t stacksize) override; bool LessThan(HttpTransact::State *, HttpTransact::State *); - void - setUseSeparateThread() - { - _use_separate_remap_thread = true; - } - bool - using_separate_thread() - { - return _use_separate_remap_thread == true; - } - -private: - EventType ET_REMAP = 0; - bool _use_separate_remap_thread = false; }; /** diff --git a/proxy/http/remap/UrlMapping.h b/proxy/http/remap/UrlMapping.h index c6de7683bde..2fa52bed765 100644 --- a/proxy/http/remap/UrlMapping.h +++ b/proxy/http/remap/UrlMapping.h @@ -34,6 +34,8 @@ #include "tscore/Regex.h" #include "tscore/List.h" +class NextHopSelectionStrategy; + /** * Used to store http referer strings (and/or regexp) **/ @@ -109,6 +111,7 @@ class url_mapping bool ip_allow_check_enabled_p = false; acl_filter_rule *filter = nullptr; // acl filtering (list of rules) LINK(url_mapping, link); // For use with the main Queue linked list holding all the mapping + std::shared_ptr strategy = nullptr; int getRank() const diff --git a/proxy/http/remap/UrlRewrite.cc b/proxy/http/remap/UrlRewrite.cc index 39052138174..aab1b215398 100644 --- a/proxy/http/remap/UrlRewrite.cc +++ b/proxy/http/remap/UrlRewrite.cc @@ -27,6 +27,7 @@ #include "ReverseProxy.h" #include "RemapConfig.h" #include "tscore/I_Layout.h" +#include "tscore/Filenames.h" #include "HttpSM.h" #define modulePrefix "[ReverseProxy]" @@ -55,10 +56,10 @@ UrlRewrite::load() { ats_scoped_str config_file_path; - config_file_path = RecConfigReadConfigPath("proxy.config.url_remap.filename", "remap.config"); + config_file_path = RecConfigReadConfigPath("proxy.config.url_remap.filename", ts::filename::REMAP); if (!config_file_path) { pmgmt->signalManager(MGMT_SIGNAL_CONFIG_ERROR, "Unable to find proxy.config.url_remap.filename"); - Warning("%s Unable to locate remap.config. No remappings in effect", modulePrefix); + Warning("%s Unable to locate %s. No remappings in effect", modulePrefix, ts::filename::REMAP); return false; } @@ -83,6 +84,11 @@ UrlRewrite::load() /* Initialize the plugin factory */ pluginFactory.setRuntimeDir(RecConfigReadRuntimeDir()).addSearchDir(RecConfigReadPluginDir()); + /* Initialize the next hop strategy factory */ + std::string sf = RecConfigReadConfigPath("proxy.config.url_remap.strategies.filename", "strategies.yaml"); + Debug("url_rewrite_regex", "strategyFactory file: %s", sf.c_str()); + strategyFactory = new NextHopStrategyFactory(sf.c_str()); + if (0 == this->BuildTable(config_file_path)) { _valid = true; if (is_debug_tag_set("url_rewrite")) { @@ -105,6 +111,10 @@ UrlRewrite::~UrlRewrite() DestroyStore(temporary_redirects); DestroyStore(forward_mappings_with_recv_port); _valid = false; + + /* Deactivate the factory when all SM are gone for sure. */ + pluginFactory.deactivate(); + delete strategyFactory; } /** Sets the reverse proxy flag. */ diff --git a/proxy/http/remap/UrlRewrite.h b/proxy/http/remap/UrlRewrite.h index 054e631ce1c..7419edd9a09 100644 --- a/proxy/http/remap/UrlRewrite.h +++ b/proxy/http/remap/UrlRewrite.h @@ -30,6 +30,7 @@ #include "HttpTransact.h" #include "tscore/Regex.h" #include "PluginFactory.h" +#include "NextHopStrategyFactory.h" #include @@ -210,6 +211,7 @@ class UrlRewrite : public RefCountObj int num_rules_forward_with_recv_port = 0; PluginFactory pluginFactory; + NextHopStrategyFactory *strategyFactory = nullptr; private: bool _valid = false; diff --git a/proxy/http/remap/unit-tests/combined.yaml b/proxy/http/remap/unit-tests/combined.yaml new file mode 100644 index 00000000000..de4d347a3c8 --- /dev/null +++ b/proxy/http/remap/unit-tests/combined.yaml @@ -0,0 +1,170 @@ +# @file +# +# Unit test data strategy.yaml file for testing the NextHopStrategyFactory +# +# @section license License +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# @section details Details +# +# +# unit test combined hosts and strategies, combined.yaml example +# +hosts: + - &p1 # shorthand name of host object, with an "anchor name" + host: p1.foo.com # name or IP of host + hash_string: slsklslsk # optional hash string that replaces the hostname in consistent hashing. + protocol: + - scheme: http + port: 80 + health_check_url: http://192.168.1.1:80 + - scheme: https + port: 443 + health_check_url: https://192.168.1.1:443 + - &p2 + host: p2.foo.com + protocol: + - scheme: http + port: 80 + health_check_url: http://192.168.1.2:80 + - scheme: https + port: 443 + health_check_url: https://192.168.1.2:443 + - &s1 + host: s1.bar.com + protocol: + - scheme: http + port: 8080 + health_check_url: http://192.168.2.1:8080 + - scheme: https + port: 8443 + health_check_url: https://192.168.2.1:8443 + - &s2 + host: s2.bar.com + protocol: + - scheme: http + port: 8080 + health_check_url: http://192.168.2.2:8080 + - scheme: https + port: 8443 + health_check_url: https://192.168.2.2:8443 +groups: + - &g1 + - <<: *p1 + weight: 0.5 + - <<: *p2 + weight: 0.5 + - &g2 + - <<: *s1 + weight: 2.0 + - <<: *s2 + weight: 1.0 +strategies: + - strategy: "mid-tier-north" + policy: rr_ip # Selection strategy policy: Enum of 'consistent_hash' or 'first_live' or 'rr_strict' or 'rr_ip' or 'latched' + go_direct: true # transactions may routed directly to the origin true/false default is true. + parent_is_proxy: false # next hop hosts are origin servers when set to 'false', defaults to true and indicates next hop hosts are ats cache's. + groups: # groups of hosts, these groups are used as rings in consistent hash and arrays of host groups for round_robin. + - *g1 + - *g2 + scheme: http # enumerated, 'http' or 'https'. by default uses the remapped scheme + failover: + max_simple_retries: 2 # default is 1, indicates the maximum number of simple retries for the listed response codes. + ring_mode: + exhaust_ring # enumerated as exhaust_ring or alternate_ring + #1) in 'exhaust_ring' mode all the servers in a ring are exhausted before failing over to secondary ring + #2) in 'alternate_ring' mode causes the failover to another server in secondary ring. + response_codes: # defines the responses codes for failover in exhaust_ring mode + - 404 + - 502 + - 503 + health_check: # specifies the list of healthchecks that should be considered for failover. A list of enums: 'passive' or 'active' + - passive + - active + - strategy: "mid-tier-south" + policy: latched + go_direct: false + parent_is_proxy: false # next hop hosts are origin servers + ignore_self_detect: false + groups: + - *g1 + - *g2 + scheme: http + failover: + max_simple_retries: 2 + ring_mode: alternate_ring + response_codes: + - 404 + - 502 + - 503 + health_check: + - passive + - active + - strategy: "mid-tier-east" + policy: first_live + go_direct: false + parent_is_proxy: false # next hop hosts are origin servers + ignore_self_detect: true + groups: + - *g1 + - *g2 + scheme: https + failover: + max_simple_retries: 2 + ring_mode: alternate_ring + response_codes: + - 404 + - 502 + - 503 + health_check: + - passive + - strategy: "mid-tier-west" + policy: rr_strict + go_direct: true + parent_is_proxy: false # next hop hosts are origin servers + groups: + - *g1 + - *g2 + scheme: https + failover: + max_simple_retries: 2 + ring_mode: exhaust_ring + response_codes: + - 404 + - 502 + - 503 + health_check: + - active + - strategy: "mid-tier-midwest" + policy: consistent_hash + hash_key: cache_key + go_direct: true + parent_is_proxy: false # next hop hosts are origin servers + groups: + - *g1 + - *g2 + scheme: https + failover: + max_simple_retries: 2 + ring_mode: exhaust_ring + response_codes: + - 404 + - 502 + - 503 + health_check: + - active diff --git a/proxy/http/remap/unit-tests/consistent-hash-tests.yaml b/proxy/http/remap/unit-tests/consistent-hash-tests.yaml new file mode 100644 index 00000000000..5bb1d9bfeea --- /dev/null +++ b/proxy/http/remap/unit-tests/consistent-hash-tests.yaml @@ -0,0 +1,171 @@ +# @file +# +# Unit test data consistent-hash-tests.yaml file for testing the NextHopStrategyFactory +# +# @section license License +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# @section details Details +# +# +# unit testing strategies for NextHopConsistentHash. +# +strategies: + - strategy: "consistent-hash-1" + policy: consistent_hash + hash_key: path + groups: + - &g1 + - host: p1.foo.com + protocol: + - scheme: http + port: 80 + health_check_url: http://192.168.1.1:80 + - scheme: https + port: 443 + health_check_url: https://192.168.1.1:443 + weight: 1.0 + - host: p2.foo.com + protocol: + - scheme: http + port: 80 + health_check_url: http://192.168.1.2:80 + - scheme: https + port: 443 + health_check_url: https://192.168.1.2:443 + weight: 1.0 + - &g2 + - host: s1.bar.com + protocol: + - scheme: http + port: 80 + health_check_url: http://192.168.2.1:80 + - scheme: https + port: 443 + health_check_url: https://192.168.2.1:443 + weight: 1.0 + - host: s2.bar.com + protocol: + - scheme: http + port: 80 + health_check_url: http://192.168.2.2:80 + - scheme: https + port: 443 + health_check_url: https://192.168.2.2:443 + weight: 1.0 + - &g3 + - host: q1.bar.com + protocol: + - scheme: http + port: 80 + health_check_url: http://192.168.3.1:80 + - scheme: https + port: 443 + health_check_url: https://192.168.3.1:443 + weight: 1.0 + - host: q2.bar.com + protocol: + - scheme: http + port: 80 + health_check_url: http://192.168.3.2:80 + - scheme: https + port: 443 + health_check_url: https://192.168.3.2:443 + weight: 1.0 + scheme: http + failover: + ring_mode: exhaust_ring + response_codes: + - 404 + - 502 + - 503 + health_check: + - passive + - active + - strategy: "consistent-hash-2" + policy: consistent_hash + hash_key: path + go_direct: false + groups: + - &g1 + - host: c1.foo.com + protocol: + - scheme: http + port: 80 + health_check_url: http://192.168.1.1:80 + - scheme: https + port: 443 + health_check_url: https://192.168.1.1:443 + weight: 1.0 + - host: c2.foo.com + protocol: + - scheme: http + port: 80 + health_check_url: http://192.168.1.2:80 + - scheme: https + port: 443 + health_check_url: https://192.168.1.2:443 + weight: 1.0 + - &g2 + - host: c3.bar.com + protocol: + - scheme: http + port: 80 + health_check_url: http://192.168.2.1:80 + - scheme: https + port: 443 + health_check_url: https://192.168.2.1:443 + weight: 1.0 + - host: c4.bar.com + protocol: + - scheme: http + port: 80 + health_check_url: http://192.168.2.2:80 + - scheme: https + port: 443 + health_check_url: https://192.168.2.2:443 + weight: 1.0 + - &g3 + - host: c5.bar.com + protocol: + - scheme: http + port: 80 + health_check_url: http://192.168.3.1:80 + - scheme: https + port: 443 + health_check_url: https://192.168.3.1:443 + weight: 1.0 + - host: c6.bar.com + protocol: + - scheme: http + port: 80 + health_check_url: http://192.168.3.2:80 + - scheme: https + port: 443 + health_check_url: https://192.168.3.2:443 + weight: 1.0 + scheme: http + failover: + ring_mode: alternate_ring + response_codes: + - 404 + - 502 + - 503 + health_check: + - passive + - active diff --git a/proxy/http/remap/unit-tests/hosts.yaml b/proxy/http/remap/unit-tests/hosts.yaml new file mode 100644 index 00000000000..f8ab3294072 --- /dev/null +++ b/proxy/http/remap/unit-tests/hosts.yaml @@ -0,0 +1,71 @@ +# @file +# +# Unit test data hosts.yaml file for testing the NextHopStrategyFactory +# +# @section license License +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# @section details Details +# +# For unit testing hosts.yaml +# +hosts: + - &p1 + host: "p1.foo.com" + protocol: + - scheme: http + port: 80 + health_check_url: "http://192.168.1.1:80" + - scheme: https + port: 443 + health_check_url: "https://192.168.1.1:443" + - &p2 + host: "p2.foo.com" + protocol: + - scheme: http + port: 80 + health_check_url: "http://192.168.1.2:80" + - &p3 + host: "p3.foo.com" + protocol: + - scheme: http + port: 8080 + health_check_url: "http://192.168.1.3:8080" + - scheme: https + port: 8443 + health_check_url: "https://192.168.1.3:8443" + - &p4 + host: "p4.foo.com" + protocol: + - scheme: http + port: 8080 + health_check_url: "http://192.168.1.4:8080" + - scheme: https + port: 8443 + health_check_url: "https://192.168.1.4:8443" +groups: + - &g1 + - <<: *p1 + weight: 1.5 + - <<: *p2 + weight: 1.5 + - &g2 + - <<: *p3 + weight: 0.5 + - <<: *p4 + weight: 1.5 diff --git a/proxy/http/remap/unit-tests/nexthop_test_stubs.cc b/proxy/http/remap/unit-tests/nexthop_test_stubs.cc new file mode 100644 index 00000000000..91a168568f2 --- /dev/null +++ b/proxy/http/remap/unit-tests/nexthop_test_stubs.cc @@ -0,0 +1,152 @@ +/** @file + + nexthop unit test stubs for unit testing nexthop strategies. + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + @section details Details + + Implements code necessary for Reverse Proxy which mostly consists of + general purpose hostname substitution in URLs. + + */ + +#include "nexthop_test_stubs.h" + +#include "HttpTransact.h" + +void +br_destroy(HttpRequestData &h) +{ + delete h.hdr; + delete h.api_info; + ats_free(h.hostname_str); +} + +void +build_request(HttpRequestData &h, const char *os_hostname) +{ + HdrHeap *heap = nullptr; + + if (h.hdr == nullptr) { + h.hdr = new HTTPHdr(); + h.hdr->create(HTTP_TYPE_REQUEST, heap); + h.xact_start = time(nullptr); + h.incoming_port = 80; + ink_zero(h.src_ip); + ink_zero(h.dest_ip); + } + if (h.hostname_str != nullptr) { + ats_free(h.hostname_str); + h.hostname_str = ats_strdup(os_hostname); + } + if (h.api_info == nullptr) { + h.api_info = new HttpApiInfo(); + } +} + +void +PrintToStdErr(const char *fmt, ...) +{ + va_list args; + va_start(args, fmt); + vfprintf(stderr, fmt, args); + va_end(args); +} + +char * +HttpRequestData::get_string() +{ + return nullptr; +} +const char * +HttpRequestData::get_host() +{ + return nullptr; +} +sockaddr const * +HttpRequestData::get_ip() +{ + return nullptr; +} + +sockaddr const * +HttpRequestData::get_client_ip() +{ + return nullptr; +} + +#include "InkAPIInternal.h" +void +ConfigUpdateCbTable::invoke(char const *p) +{ +} + +#include "I_Machine.h" + +static Machine *my_machine = nullptr; + +Machine::Machine(char const *hostname, sockaddr const *addr) {} +Machine::~Machine() +{ + delete my_machine; +} +Machine * +Machine::instance() +{ + if (my_machine == nullptr) { + my_machine = new Machine(nullptr, nullptr); + } + return my_machine; +} +bool +Machine::is_self(const char *name) +{ + return false; +} + +#include "HostStatus.h" + +HostStatRec::HostStatRec(){}; +HostStatus::HostStatus() {} +HostStatus::~HostStatus(){}; +HostStatRec * +HostStatus::getHostStatus(const char *name) +{ + // for unit tests only, always return a record with HOST_STATUS_UP + static HostStatRec rec; + rec.status = HostStatus_t::HOST_STATUS_UP; + return &rec; +} +void +HostStatus::setHostStatus(char const *host, HostStatus_t status, unsigned int, unsigned int) +{ + NH_Debug("next_hop", "setting host status for '%s' to %s", host, HostStatusNames[status]); +} + +#include "I_UDPConnection.h" + +void +UDPConnection::Release() +{ +} + +#include "P_UDPPacket.h" +inkcoreapi ClassAllocator udpPacketAllocator("udpPacketAllocator"); +// for UDPPacketInternal::free() diff --git a/proxy/http/remap/unit-tests/nexthop_test_stubs.h b/proxy/http/remap/unit-tests/nexthop_test_stubs.h new file mode 100644 index 00000000000..bc05c5075b7 --- /dev/null +++ b/proxy/http/remap/unit-tests/nexthop_test_stubs.h @@ -0,0 +1,89 @@ +/** @file + + unit test stubs header for linking nexthop unit tests. + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + @section details Details + + Implements code necessary for Reverse Proxy which mostly consists of + general purpose hostname substitution in URLs. + + */ + +#pragma once + +#include +#include +#include + +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif /* __cplusplus */ + +#define NH_Debug(tag, fmt, ...) PrintToStdErr("%s %s:%d:%s() " fmt "\n", tag, __FILE__, __LINE__, __func__, ##__VA_ARGS__) +#define NH_Error(fmt, ...) PrintToStdErr("%s:%d:%s() " fmt "\n", __FILE__, __LINE__, __func__, ##__VA_ARGS__) +#define NH_Note(fmt, ...) PrintToStdErr("%s:%d:%s() " fmt "\n", __FILE__, __LINE__, __func__, ##__VA_ARGS__) +#define NH_Warn(fmt, ...) PrintToStdErr("%s:%d:%s() " fmt "\n", __FILE__, __LINE__, __func__, ##__VA_ARGS__) + +class HttpRequestData; + +void PrintToStdErr(const char *fmt, ...); +void br_destroy(HttpRequestData &h); +void build_request(HttpRequestData &h, const char *os_hostname); + +#ifdef __cplusplus +} +#endif /* __cplusplus */ + +#include "ControlMatcher.h" +struct TestData : public HttpRequestData { + std::string hostname; + sockaddr client_ip; + sockaddr server_ip; + + TestData() + { + client_ip.sa_family = AF_INET; + memset(client_ip.sa_data, 0, sizeof(client_ip.sa_data)); + } + const char * + get_host() + { + return hostname.c_str(); + } + sockaddr const * + get_ip() + { + return &server_ip; + } + sockaddr const * + get_client_ip() + { + return &client_ip; + } + char * + get_string() + { + return nullptr; + } +}; diff --git a/proxy/http/remap/unit-tests/plugin_init_fail.cc b/proxy/http/remap/unit-tests/plugin_init_fail.cc new file mode 100644 index 00000000000..151382f7009 --- /dev/null +++ b/proxy/http/remap/unit-tests/plugin_init_fail.cc @@ -0,0 +1,58 @@ +/** @file + + A test plugin for testing Plugin's Dynamic Shared Objects (DSO) + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + @section details Details + + Implements code necessary for Reverse Proxy which mostly consists of + general purpose hostname substitution in URLs. + + */ + +#include "plugin_testing_common.h" + +#ifdef __cplusplus +extern "C" { +#endif /* __cplusplus */ + +#include "ts/ts.h" +#include "ts/remap.h" + +TSReturnCode +TSRemapInit(TSRemapInterface *api_info, char *errbuf, int errbuf_size) +{ + return TS_ERROR; +} + +void +TSRemapDone(void) +{ +} + +TSRemapStatus +TSRemapDoRemap(void *ih, TSHttpTxn rh, TSRemapRequestInfo *rri) +{ + return TSREMAP_NO_REMAP; +} + +#ifdef __cplusplus +} +#endif /* __cplusplus */ diff --git a/proxy/http/remap/unit-tests/plugin_instinit_fail.cc b/proxy/http/remap/unit-tests/plugin_instinit_fail.cc new file mode 100644 index 00000000000..41bf6b1bbc9 --- /dev/null +++ b/proxy/http/remap/unit-tests/plugin_instinit_fail.cc @@ -0,0 +1,69 @@ +/** @file + + A test plugin for testing Plugin's Dynamic Shared Objects (DSO) + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + @section details Details + + Implements code necessary for Reverse Proxy which mostly consists of + general purpose hostname substitution in URLs. + + */ + +#include "plugin_testing_common.h" + +#ifdef __cplusplus +extern "C" { +#endif /* __cplusplus */ + +#include "ts/ts.h" +#include "ts/remap.h" + +TSReturnCode +TSRemapInit(TSRemapInterface *api_info, char *errbuf, int errbuf_size) +{ + return TS_SUCCESS; +} + +void +TSRemapDone(void) +{ +} + +TSReturnCode +TSRemapNewInstance(int argc, char *argv[], void **ih, char *errbuf, int errbuf_size) +{ + return TS_ERROR; +} + +void +TSRemapDeleteInstance(void *) +{ +} + +TSRemapStatus +TSRemapDoRemap(void *ih, TSHttpTxn rh, TSRemapRequestInfo *rri) +{ + return TSREMAP_NO_REMAP; +} + +#ifdef __cplusplus +} +#endif /* __cplusplus */ diff --git a/proxy/http/remap/unit-tests/plugin_testing_calls.cc b/proxy/http/remap/unit-tests/plugin_testing_calls.cc index 89c8df2c749..883b971cdc6 100644 --- a/proxy/http/remap/unit-tests/plugin_testing_calls.cc +++ b/proxy/http/remap/unit-tests/plugin_testing_calls.cc @@ -103,9 +103,16 @@ TSRemapOSResponse(void *ih, TSHttpTxn rh, int os_response_type) } void -TSRemapConfigReload(void) +TSRemapPreConfigReload(void) { - debugObject.reloadConfigCalled++; + debugObject.preReloadConfigCalled++; +} + +void +TSRemapPostConfigReload(TSRemapReloadStatus reloadStatus) +{ + debugObject.postReloadConfigCalled++; + debugObject.postReloadConfigStatus = reloadStatus; } /* The folowing functions are meant for unit testing */ diff --git a/proxy/http/remap/unit-tests/plugin_testing_common.cc b/proxy/http/remap/unit-tests/plugin_testing_common.cc index a54422c184a..07a44e15f29 100644 --- a/proxy/http/remap/unit-tests/plugin_testing_common.cc +++ b/proxy/http/remap/unit-tests/plugin_testing_common.cc @@ -50,3 +50,24 @@ getTemporaryDir() return fs::path(mkdtemp(dirNameTemplate)); } + +// implement functions to support unit-testing of option to enable/disable dynamic reload of plugins +static PluginDynamicReloadMode plugin_dynamic_reload_mode = PluginDynamicReloadMode::RELOAD_ON; + +bool +isPluginDynamicReloadEnabled() +{ + return PluginDynamicReloadMode::RELOAD_ON == plugin_dynamic_reload_mode; +} + +void +enablePluginDynamicReload() +{ + plugin_dynamic_reload_mode = PluginDynamicReloadMode::RELOAD_ON; +} + +void +disablePluginDynamicReload() +{ + plugin_dynamic_reload_mode = PluginDynamicReloadMode::RELOAD_OFF; +} diff --git a/proxy/http/remap/unit-tests/plugin_testing_common.h b/proxy/http/remap/unit-tests/plugin_testing_common.h index 66a8dd686ee..1669a303f02 100644 --- a/proxy/http/remap/unit-tests/plugin_testing_common.h +++ b/proxy/http/remap/unit-tests/plugin_testing_common.h @@ -51,17 +51,19 @@ class PluginDebugObject void clear() { - contextInit = nullptr; - contextInitInstance = nullptr; - doRemapCalled = 0; - initCalled = 0; - doneCalled = 0; - initInstanceCalled = 0; - deleteInstanceCalled = 0; - reloadConfigCalled = 0; - ih = nullptr; - argc = 0; - argv = nullptr; + contextInit = nullptr; + contextInitInstance = nullptr; + doRemapCalled = 0; + initCalled = 0; + doneCalled = 0; + initInstanceCalled = 0; + deleteInstanceCalled = 0; + preReloadConfigCalled = 0; + postReloadConfigCalled = 0; + postReloadConfigStatus = TSREMAP_CONFIG_RELOAD_FAILURE; + ih = nullptr; + argc = 0; + argv = nullptr; } /* Input fields used to set the test behavior of the plugin call-backs */ @@ -69,17 +71,19 @@ class PluginDebugObject void *input_ih; /* the value to be returned by the plugin instance init function */ /* Output fields showing what happend during the test */ - const PluginThreadContext *contextInit = nullptr; /* plugin initialization context */ - const PluginThreadContext *contextInitInstance = nullptr; /* plugin instance initialization context */ - int doRemapCalled = 0; /* mark if remap was called */ - int initCalled = 0; /* mark if plugin init was called */ - int doneCalled = 0; /* mark if done was called */ - int initInstanceCalled = 0; /* mark if instance init was called */ - int deleteInstanceCalled = 0; /* mark if delete instance was called */ - int reloadConfigCalled = 0; /* mark if reload config was called */ - void *ih = nullptr; /* instance handler */ - int argc = 0; /* number of plugin instance parameters received by the plugin */ - char **argv = nullptr; /* plugin instance parameters received by the plugin */ + const PluginThreadContext *contextInit = nullptr; /* plugin initialization context */ + const PluginThreadContext *contextInitInstance = nullptr; /* plugin instance initialization context */ + int doRemapCalled = 0; /* mark if remap was called */ + int initCalled = 0; /* mark if plugin init was called */ + int doneCalled = 0; /* mark if done was called */ + int initInstanceCalled = 0; /* mark if instance init was called */ + int deleteInstanceCalled = 0; /* mark if delete instance was called */ + int preReloadConfigCalled = 0; /* mark if pre-reload config was called */ + int postReloadConfigCalled = 0; /* mark if post-reload config was called */ + TSRemapReloadStatus postReloadConfigStatus = TSREMAP_CONFIG_RELOAD_FAILURE; /* mark if plugin reload status is passed correctly */ + void *ih = nullptr; /* instance handler */ + int argc = 0; /* number of plugin instance parameters received by the plugin */ + char **argv = nullptr; /* plugin instance parameters received by the plugin */ }; #ifdef __cplusplus @@ -89,10 +93,15 @@ extern "C" { typedef void *GetPluginDebugObjectFunction(void); GetPluginDebugObjectFunction getPluginDebugObjectTest; -#define Debug(category, fmt, ...) PrintToStdErr("(%s) %s:%d:%s() " fmt "\n", category, __FILE__, __LINE__, __func__, ##__VA_ARGS__) -#define Error(fmt, ...) PrintToStdErr("%s:%d:%s() " fmt "\n", __FILE__, __LINE__, __func__, ##__VA_ARGS__) +#define PluginDebug(category, fmt, ...) \ + PrintToStdErr("(%s) %s:%d:%s() " fmt "\n", category, __FILE__, __LINE__, __func__, ##__VA_ARGS__) +#define PluginError(fmt, ...) PrintToStdErr("%s:%d:%s() " fmt "\n", __FILE__, __LINE__, __func__, ##__VA_ARGS__) void PrintToStdErr(const char *fmt, ...); #ifdef __cplusplus } #endif /* __cplusplus */ + +// functions to support unit-testing of option to enable/disable dynamic reload of plugins +void enablePluginDynamicReload(); +void disablePluginDynamicReload(); \ No newline at end of file diff --git a/proxy/http/remap/unit-tests/round-robin-tests.yaml b/proxy/http/remap/unit-tests/round-robin-tests.yaml new file mode 100644 index 00000000000..0ff9267d032 --- /dev/null +++ b/proxy/http/remap/unit-tests/round-robin-tests.yaml @@ -0,0 +1,206 @@ +# @file +# +# Unit test data round-robin-tests.yaml file for testing the NextHopStrategyFactory +# +# @section license License +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# @section details Details +# +# +# unit testing strategies for NextHopRoundRobin. +# +strategies: + - strategy: "first-live" + policy: first_live + groups: + - &g1 + - host: p1.foo.com + hash_string: slsklslsk + protocol: + - scheme: http + port: 80 + health_check_url: http://192.168.1.1:80 + - scheme: https + port: 443 + health_check_url: https://192.168.1.1:443 + weight: 1.0 + - host: p2.foo.com + hash_string: srskrsrsk + protocol: + - scheme: http + port: 80 + health_check_url: http://192.168.1.2:80 + - scheme: https + port: 443 + health_check_url: https://192.168.1.2:443 + weight: 1.0 + - &g2 + - host: s1.bar.com + hash_string: lslalalal + protocol: + - scheme: http + port: 80 + health_check_url: http://192.168.2.1:80 + - scheme: https + port: 443 + health_check_url: https://192.168.2.1:443 + weight: 1.0 + - host: s2.bar.com + hash_string: alalalalal + protocol: + - scheme: http + port: 80 + health_check_url: http://192.168.2.2:80 + - scheme: https + port: 443 + health_check_url: https://192.168.2.2:443 + weight: 1.0 + scheme: http + failover: + ring_mode: exhaust_ring + response_codes: + - 404 + - 502 + - 503 + health_check: + - passive + - active + - strategy: "rr-strict-exhaust-ring" + policy: rr_strict + groups: + - &g1 + - host: p1.foo.com + hash_string: slsklslsk + protocol: + - scheme: http + port: 80 + health_check_url: http://192.168.1.1:80 + - scheme: https + port: 443 + health_check_url: https://192.168.1.1:443 + weight: 1.0 + - host: p2.foo.com + hash_string: srskrsrsk + protocol: + - scheme: http + port: 80 + health_check_url: http://192.168.1.2:80 + - scheme: https + port: 443 + health_check_url: https://192.168.1.2:443 + weight: 1.0 + - &g2 + - host: s1.bar.com + hash_string: lslalalal + protocol: + - scheme: http + port: 80 + health_check_url: http://192.168.2.1:80 + - scheme: https + port: 443 + health_check_url: https://192.168.2.1:443 + weight: 1.0 + - host: s2.bar.com + hash_string: alalalalal + protocol: + - scheme: http + port: 80 + health_check_url: http://192.168.2.2:80 + - scheme: https + port: 443 + health_check_url: https://192.168.2.2:443 + weight: 1.0 + scheme: http + failover: + ring_mode: exhaust_ring + response_codes: + - 404 + - 502 + - 503 + health_check: + - passive + - active + - strategy: "rr-ip" + policy: rr_ip + groups: + - &g1 + - host: p3.foo.com + hash_string: slsklslsk + protocol: + - scheme: http + port: 80 + health_check_url: http://192.168.1.3:80 + - scheme: https + port: 443 + health_check_url: https://192.168.1.3:443 + weight: 1.0 + - host: p4.foo.com + hash_string: srskrsrsk + protocol: + - scheme: http + port: 80 + health_check_url: http://192.168.1.4:80 + - scheme: https + port: 443 + health_check_url: https://192.168.1.4:443 + weight: 1.0 + scheme: http + failover: + ring_mode: alternate_ring + response_codes: + - 404 + - 502 + - 503 + health_check: + - passive + - active + - strategy: "latched" + policy: latched + groups: + - &g1 + - host: p3.foo.com + hash_string: slsklslsk + protocol: + - scheme: http + port: 80 + health_check_url: http://192.168.1.3:80 + - scheme: https + port: 443 + health_check_url: https://192.168.1.3:443 + weight: 1.0 + - host: p4.foo.com + hash_string: srskrsrsk + protocol: + - scheme: http + port: 80 + health_check_url: http://192.168.1.4:80 + - scheme: https + port: 443 + health_check_url: https://192.168.1.4:443 + weight: 1.0 + scheme: http + failover: + ring_mode: alternate_ring + response_codes: + - 404 + - 502 + - 503 + health_check: + - passive + - active diff --git a/proxy/http/remap/unit-tests/simple-strategy.yaml b/proxy/http/remap/unit-tests/simple-strategy.yaml new file mode 100644 index 00000000000..23ef1d694a7 --- /dev/null +++ b/proxy/http/remap/unit-tests/simple-strategy.yaml @@ -0,0 +1,117 @@ +# @file +# +# Unit test data strategy.yaml file for testing the NextHopStrategyFactory +# +# @section license License +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# @section details Details +# +# +# unit test simple-strategy.yaml example +# +strategies: + - strategy: "strategy-3" + policy: rr_ip + groups: + - &g1 + - host: p1.foo.com + hash_string: slsklslsk + protocol: + - scheme: http + port: 80 + health_check_url: http://192.168.1.1:80 + - scheme: https + port: 443 + health_check_url: https://192.168.1.1:443 + weight: 1.0 + - host: p2.foo.com + hash_string: srskrsrsk + protocol: + - scheme: http + port: 80 + health_check_url: http://192.168.1.2:80 + - scheme: https + port: 443 + health_check_url: https://192.168.1.2:443 + weight: 1.0 + - &g2 + - host: s1.bar.com + hash_string: lslalalal + protocol: + - scheme: http + port: 80 + health_check_url: http://192.168.2.1:80 + - scheme: https + port: 443 + health_check_url: https://192.168.2.1:443 + weight: 1.0 + - host: s2.bar.com + hash_string: alalalalal + protocol: + - scheme: http + port: 80 + health_check_url: http://192.168.2.2:80 + - scheme: https + port: 443 + health_check_url: https://192.168.2.2:443 + weight: 1.0 + scheme: https + failover: + ring_mode: exhaust_ring + response_codes: + - 404 + - 502 + - 503 + health_check: + - passive + - active + - strategy: "strategy-4" + policy: latched + groups: + - &g1 + - host: p3.foo.com + hash_string: slsklslsk + protocol: + - scheme: http + port: 80 + health_check_url: http://192.168.1.3:80 + - scheme: https + port: 443 + health_check_url: https://192.168.1.3:443 + weight: 1.0 + - host: p4.foo.com + hash_string: srskrsrsk + protocol: + - scheme: http + port: 80 + health_check_url: http://192.168.1.4:80 + - scheme: https + port: 443 + health_check_url: https://192.168.1.4:443 + weight: 1.0 + scheme: http + failover: + ring_mode: alternate_ring + response_codes: + - 404 + - 502 + - 503 + health_check: + - passive + - active diff --git a/proxy/http/remap/unit-tests/strategies-dir/01-hosts.yaml b/proxy/http/remap/unit-tests/strategies-dir/01-hosts.yaml new file mode 100644 index 00000000000..99af17c3952 --- /dev/null +++ b/proxy/http/remap/unit-tests/strategies-dir/01-hosts.yaml @@ -0,0 +1,60 @@ +# @file +# +# Unit test data hosts.yaml file for testing the NextHopStrategyFactory +# +# @section license License +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# @section details Details +# +# For unit testing hosts.yaml +# +hosts: + - &p1 + host: "p1.foo.com" + protocol: + - scheme: http + port: 80 + health_check_url: "http://192.168.1.1:80" + - scheme: https + port: 443 + health_check_url: "https://192.168.1.1:443" + - &p2 + host: "p2.foo.com" + protocol: + - scheme: http + port: 80 + health_check_url: "http://192.168.1.2:80" + - &p3 + host: "p3.foo.com" + protocol: + - scheme: http + port: 8080 + health_check_url: "http://192.168.1.3:8080" + - scheme: https + port: 8443 + health_check_url: "https://192.168.1.3:8443" + - &p4 + host: "p4.foo.com" + protocol: + - scheme: http + port: 8080 + health_check_url: "http://192.168.1.4:8080" + - scheme: https + port: 8443 + health_check_url: "https://192.168.1.4:8443" diff --git a/proxy/http/remap/unit-tests/strategies-dir/02-groups.yaml b/proxy/http/remap/unit-tests/strategies-dir/02-groups.yaml new file mode 100644 index 00000000000..6d66e98ecea --- /dev/null +++ b/proxy/http/remap/unit-tests/strategies-dir/02-groups.yaml @@ -0,0 +1,37 @@ +# @file +# +# Unit test data hosts.yaml file for testing the NextHopStrategyFactory +# +# @section license License +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# @section details Details +# +# For unit testing hosts.yaml +# +groups: + - &g1 + - <<: *p1 + weight: 0.5 + - <<: *p2 + weight: 0.5 + - &g2 + - <<: *p3 + weight: 0.5 + - <<: *p4 + weight: 0.5 diff --git a/proxy/http/remap/unit-tests/strategies-dir/03-strategies.yaml b/proxy/http/remap/unit-tests/strategies-dir/03-strategies.yaml new file mode 100644 index 00000000000..b60101d1602 --- /dev/null +++ b/proxy/http/remap/unit-tests/strategies-dir/03-strategies.yaml @@ -0,0 +1,65 @@ +# @file +# +# Unit test data strategy.yaml file for testing the NextHopStrategyFactory +# +# @section license License +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# @section details Details +# +# +# unit test data. +# +strategies: + - strategy: 'mid-tier-north' + policy: rr_ip + go_direct: true + parent_is_proxy: false + groups: + - *g1 + - *g2 + scheme: http + failover: + max_simple_retries: 2 + ring_mode: exhaust_ring + response_codes: + - 404 + - 502 + - 503 + health_check: + - passive + - active + - strategy: 'mid-tier-south' + policy: latched + go_direct: false + parent_is_proxy: false + ignore_self_detect: false + groups: + - *g1 + - *g2 + scheme: http + failover: + max_simple_retries: 2 + ring_mode: alternate_ring + response_codes: + - 404 + - 502 + - 503 + health_check: + - passive + - active diff --git a/proxy/http/remap/unit-tests/strategy.yaml b/proxy/http/remap/unit-tests/strategy.yaml new file mode 100644 index 00000000000..b3f438a1bb2 --- /dev/null +++ b/proxy/http/remap/unit-tests/strategy.yaml @@ -0,0 +1,59 @@ +# @file +# +# Unit test data strategy.yaml file for testing the NextHopStrategyFactory +# +# @section license License +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# @section details Details +# +# +# unit test strategy.yaml example +# +#include unit-tests/hosts.yaml +# +strategies: + - strategy: 'strategy-1' + policy: consistent_hash + hash_key: cache_key + go_direct: false + groups: + - *g1 + - *g2 + scheme: http + failover: + ring_mode: exhaust_ring + response_codes: + - 404 + - 503 + health_check: + - passive + - strategy: 'strategy-2' + policy: rr_strict + go_direct: true + groups: + - *g1 + - *g2 + scheme: http + failover: + ring_mode: exhaust_ring + response_codes: + - 404 + - 503 + health_check: + - passive diff --git a/proxy/http/remap/unit-tests/test_NextHopConsistentHash.cc b/proxy/http/remap/unit-tests/test_NextHopConsistentHash.cc new file mode 100644 index 00000000000..f831253064e --- /dev/null +++ b/proxy/http/remap/unit-tests/test_NextHopConsistentHash.cc @@ -0,0 +1,400 @@ +/** @file + + Unit tests for the NextHopConsistentHash. + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + @section details Details + + Unit testing the NextHopConsistentHash class. + + */ + +#define CATCH_CONFIG_MAIN /* include main function */ + +#include /* catch unit-test framework */ +#include + +#include "nexthop_test_stubs.h" +#include "NextHopSelectionStrategy.h" +#include "NextHopStrategyFactory.h" +#include "NextHopConsistentHash.h" + +#include "HTTP.h" +extern int cmd_disable_pfreelist; + +SCENARIO("Testing NextHopConsistentHash class, using policy 'consistent_hash'", "[NextHopConsistentHash]") +{ + // We need this to build a HdrHeap object in build_request(); + // No thread setup, forbid use of thread local allocators. + cmd_disable_pfreelist = true; + // Get all of the HTTP WKS items populated. + http_init(); + + GIVEN("Loading the consistent-hash-tests.yaml config for 'consistent_hash' tests.") + { + // load the configuration strtegies. + std::shared_ptr strategy; + NextHopStrategyFactory nhf(TS_SRC_DIR "unit-tests/consistent-hash-tests.yaml"); + strategy = nhf.strategyInstance("consistent-hash-1"); + + WHEN("the config is loaded.") + { + THEN("then testing consistent hash.") + { + REQUIRE(nhf.strategies_loaded == true); + REQUIRE(strategy != nullptr); + REQUIRE(strategy->groups == 3); + } + } + + WHEN("requests are received.") + { + HttpRequestData request; + ParentResult result; + TestData rdata; + rdata.xact_start = time(nullptr); + uint64_t fail_threshold = 1; + uint64_t retry_time = 1; + + // need to run these checks in succession so there + // are no host status state changes. + // + // These tests simulate failed requests using a selected host. + // markNextHopDown() is called by the state machine when + // there is a request failure due to a connection error or + // timeout. the 'result' struct has the information on the + // host used in the failed request and when called, marks the + // host indicated in the 'request' struct as unavailable. + // + // Here we walk through making requests then marking the selected + // host down until all are down and the origin is finally chosen. + // + THEN("when making requests and taking nodes down.") + { + REQUIRE(nhf.strategies_loaded == true); + REQUIRE(strategy != nullptr); + + // first request. + build_request(request, "rabbit.net"); + result.reset(); + strategy->findNextHop(10001, result, request, fail_threshold, retry_time); + + CHECK(result.result == ParentResultType::PARENT_SPECIFIED); + CHECK(strcmp(result.hostname, "p1.foo.com") == 0); + + // mark down p1.foo.com. markNextHopDown looks at the 'result' + // and uses the host index there mark down the host selected + // from a + strategy->markNextHopDown(10001, result, 1, fail_threshold); + + // second request - reusing the ParentResult from the last request + // simulating a failure triggers a search for another parent, not firstcall. + build_request(request, "rabbit.net"); + strategy->findNextHop(10002, result, request, fail_threshold, retry_time); + + CHECK(result.result == ParentResultType::PARENT_SPECIFIED); + CHECK(strcmp(result.hostname, "p2.foo.com") == 0); + + // mark down p2.foo.com + strategy->markNextHopDown(10002, result, 1, fail_threshold); + + // third request - reusing the ParentResult from the last request + // simulating a failure triggers a search for another parent, not firstcall. + build_request(request, "rabbit.net"); + strategy->findNextHop(10003, result, request, fail_threshold, retry_time); + + CHECK(result.result == ParentResultType::PARENT_SPECIFIED); + CHECK(strcmp(result.hostname, "s2.bar.com") == 0); + + // mark down s2.bar.com + strategy->markNextHopDown(10003, result, 1, fail_threshold); + + // fourth request - reusing the ParentResult from the last request + // simulating a failure triggers a search for another parent, not firstcall. + build_request(request, "rabbit.net"); + strategy->findNextHop(10004, result, request, fail_threshold, retry_time); + + CHECK(result.result == ParentResultType::PARENT_SPECIFIED); + CHECK(strcmp(result.hostname, "s1.bar.com") == 0); + + // mark down s1.bar.com. + strategy->markNextHopDown(10004, result, 1, fail_threshold); + + // fifth request - reusing the ParentResult from the last request + // simulating a failure triggers a search for another parent, not firstcall. + build_request(request, "rabbit.net"); + strategy->findNextHop(10005, result, request, fail_threshold, retry_time); + + CHECK(result.result == ParentResultType::PARENT_SPECIFIED); + CHECK(strcmp(result.hostname, "q1.bar.com") == 0); + + // mark down q1.bar.com + strategy->markNextHopDown(10005, result, 1, fail_threshold); + // sixth request - reusing the ParentResult from the last request + // simulating a failure triggers a search for another parent, not firstcall. + build_request(request, "rabbit.net"); + strategy->findNextHop(10006, result, request, fail_threshold, retry_time); + + CHECK(result.result == ParentResultType::PARENT_SPECIFIED); + CHECK(strcmp(result.hostname, "q2.bar.com") == 0); + + // mark down q2.bar.com + strategy->markNextHopDown(10006, result, 1, fail_threshold); + // seventh request - reusing the ParentResult from the last request + // simulating a failure triggers a search for another parent, not firstcall. + build_request(request, "rabbit.net"); + strategy->findNextHop(10007, result, request, fail_threshold, retry_time); + + CHECK(result.result == ParentResultType::PARENT_DIRECT); + CHECK(result.hostname == nullptr); + + // sleep and test that q2 is becomes retryable; + time_t now = time(nullptr) + 5; + + // eighth request - reusing the ParentResult from the last request + // simulating a failure triggers a search for another parent, not firstcall. + build_request(request, "rabbit.net"); + strategy->findNextHop(10008, result, request, fail_threshold, retry_time, now); + CHECK(result.result == ParentResultType::PARENT_SPECIFIED); + CHECK(strcmp(result.hostname, "q2.bar.com") == 0); + } + // free up request resources. + br_destroy(request); + } + } +} + +SCENARIO("Testing NextHopConsistentHash class (all firstcalls), using policy 'consistent_hash'", "[NextHopConsistentHash]") +{ + // We need this to build a HdrHeap object in build_request(); + // No thread setup, forbid use of thread local allocators. + cmd_disable_pfreelist = true; + // Get all of the HTTP WKS items populated. + http_init(); + + GIVEN("Loading the consistent-hash-tests.yaml config for 'consistent_hash' tests.") + { + std::shared_ptr strategy; + NextHopStrategyFactory nhf(TS_SRC_DIR "unit-tests/consistent-hash-tests.yaml"); + strategy = nhf.strategyInstance("consistent-hash-1"); + + WHEN("the config is loaded.") + { + THEN("then testing consistent hash.") + { + REQUIRE(nhf.strategies_loaded == true); + REQUIRE(strategy != nullptr); + REQUIRE(strategy->groups == 3); + } + } + + // Same test procedure as the first scenario but we clear the 'result' struct + // so that we are making initial requests and simulating that hosts were + // removed by different transactions. + // + // these checks need to be run in sequence so that there are no host status + // state changes induced by using multiple WHEN() and THEN() + WHEN("initial requests are made and hosts are unavailable .") + { + uint64_t fail_threshold = 1; + uint64_t retry_time = 1; + TestData rdata; + rdata.xact_start = time(nullptr); + HttpRequestData request; + ParentResult result; + + THEN("when making requests and taking nodes down.") + { + REQUIRE(nhf.strategies_loaded == true); + REQUIRE(strategy != nullptr); + + // first request. + build_request(request, "rabbit.net"); + result.reset(); + strategy->findNextHop(20001, result, request, fail_threshold, retry_time); + CHECK(result.result == ParentResultType::PARENT_SPECIFIED); + CHECK(strcmp(result.hostname, "p1.foo.com") == 0); + + // mark down p1.foo.com + strategy->markNextHopDown(20001, result, 1, fail_threshold); + // second request + build_request(request, "rabbit.net"); + result.reset(); + strategy->findNextHop(20002, result, request, fail_threshold, retry_time); + CHECK(result.result == ParentResultType::PARENT_SPECIFIED); + CHECK(strcmp(result.hostname, "p2.foo.com") == 0); + + // mark down p2.foo.com + strategy->markNextHopDown(20002, result, 1, fail_threshold); + + // third request + build_request(request, "rabbit.net"); + result.reset(); + strategy->findNextHop(20003, result, request, fail_threshold, retry_time); + CHECK(result.result == ParentResultType::PARENT_SPECIFIED); + CHECK(strcmp(result.hostname, "s2.bar.com") == 0); + + // mark down s2.bar.com + strategy->markNextHopDown(20003, result, 1, fail_threshold); + + // fourth request + build_request(request, "rabbit.net"); + result.reset(); + strategy->findNextHop(20004, result, request, fail_threshold, retry_time); + CHECK(result.result == ParentResultType::PARENT_SPECIFIED); + CHECK(strcmp(result.hostname, "s1.bar.com") == 0); + + // mark down s1.bar.com + strategy->markNextHopDown(20004, result, 1, fail_threshold); + + // fifth request + build_request(request, "rabbit.net/asset1"); + result.reset(); + strategy->findNextHop(20005, result, request, fail_threshold, retry_time); + CHECK(result.result == ParentResultType::PARENT_SPECIFIED); + CHECK(strcmp(result.hostname, "q1.bar.com") == 0); + + // sixth request - wait and p1 should now become available + time_t now = time(nullptr) + 5; + build_request(request, "rabbit.net"); + result.reset(); + strategy->findNextHop(20006, result, request, fail_threshold, retry_time, now); + CHECK(result.result == ParentResultType::PARENT_SPECIFIED); + CHECK(strcmp(result.hostname, "p1.foo.com") == 0); + } + // free up request resources. + br_destroy(request); + } + } +} + +SCENARIO("Testing NextHopConsistentHash class (alternating rings), using policy 'consistent_hash'", "[NextHopConsistentHash]") +{ + // We need this to build a HdrHeap object in build_request(); + // No thread setup, forbid use of thread local allocators. + cmd_disable_pfreelist = true; + // Get all of the HTTP WKS items populated. + http_init(); + + GIVEN("Loading the consistent-hash-tests.yaml config for 'consistent_hash' tests.") + { + std::shared_ptr strategy; + NextHopStrategyFactory nhf(TS_SRC_DIR "unit-tests/consistent-hash-tests.yaml"); + strategy = nhf.strategyInstance("consistent-hash-2"); + + WHEN("the config is loaded.") + { + THEN("then testing consistent hash.") + { + REQUIRE(nhf.strategies_loaded == true); + REQUIRE(strategy != nullptr); + REQUIRE(strategy->groups == 3); + } + } + + // makeing requests and marking down hosts with a config set for alternating ring mode. + WHEN("requests are made in a config set for alternating rings and hosts are marked down.") + { + uint64_t fail_threshold = 1; + uint64_t retry_time = 1; + TestData rdata; + rdata.xact_start = time(nullptr); + HttpRequestData request; + ParentResult result; + + THEN("expect the following results when making requests and marking hosts down.") + { + REQUIRE(nhf.strategies_loaded == true); + REQUIRE(strategy != nullptr); + + // first request. + build_request(request, "bunny.net/asset1"); + result.reset(); + strategy->findNextHop(30001, result, request, fail_threshold, retry_time); + CHECK(result.result == ParentResultType::PARENT_SPECIFIED); + CHECK(strcmp(result.hostname, "c2.foo.com") == 0); + + // simulated failure, mark c2 down and retry request + strategy->markNextHopDown(30001, result, 1, fail_threshold); + + // second request + build_request(request, "bunny.net.net/asset1"); + strategy->findNextHop(30002, result, request, fail_threshold, retry_time); + CHECK(result.result == ParentResultType::PARENT_SPECIFIED); + CHECK(strcmp(result.hostname, "c3.bar.com") == 0); + + // mark down c3.bar.com + strategy->markNextHopDown(30002, result, 1, fail_threshold); + + // third request + build_request(request, "bunny.net/asset2"); + result.reset(); + strategy->findNextHop(30003, result, request, fail_threshold, retry_time); + CHECK(result.result == ParentResultType::PARENT_SPECIFIED); + CHECK(strcmp(result.hostname, "c6.bar.com") == 0); + + // just mark it down and retry request + strategy->markNextHopDown(30003, result, 1, fail_threshold); + // fourth request + build_request(request, "bunny.net/asset2"); + strategy->findNextHop(30004, result, request, fail_threshold, retry_time); + CHECK(result.result == ParentResultType::PARENT_SPECIFIED); + CHECK(strcmp(result.hostname, "c1.foo.com") == 0); + + // mark it down + strategy->markNextHopDown(30004, result, 1, fail_threshold); + // fifth request - new request + build_request(request, "bunny.net/asset3"); + result.reset(); + strategy->findNextHop(30005, result, request, fail_threshold, retry_time); + CHECK(result.result == ParentResultType::PARENT_SPECIFIED); + CHECK(strcmp(result.hostname, "c4.bar.com") == 0); + + // mark it down and retry + strategy->markNextHopDown(30005, result, 1, fail_threshold); + // sixth request + build_request(request, "bunny.net/asset3"); + result.reset(); + strategy->findNextHop(30006, result, request, fail_threshold, retry_time); + CHECK(result.result == ParentResultType::PARENT_SPECIFIED); + CHECK(strcmp(result.hostname, "c5.bar.com") == 0); + + // mark it down + strategy->markNextHopDown(30006, result, 1, fail_threshold); + // seventh request - new request with all hosts down and go_direct is false. + build_request(request, "bunny.net/asset4"); + result.reset(); + strategy->findNextHop(30007, result, request, fail_threshold, retry_time); + CHECK(result.result == ParentResultType::PARENT_FAIL); + CHECK(result.hostname == nullptr); + + // eighth request - retry after waiting for the retry window to expire. + time_t now = time(nullptr) + 5; + build_request(request, "bunny.net/asset4"); + result.reset(); + strategy->findNextHop(30008, result, request, fail_threshold, retry_time, now); + CHECK(result.result == ParentResultType::PARENT_SPECIFIED); + CHECK(strcmp(result.hostname, "c2.foo.com") == 0); + } + // free up request resources. + br_destroy(request); + } + } +} diff --git a/proxy/http/remap/unit-tests/test_NextHopRoundRobin.cc b/proxy/http/remap/unit-tests/test_NextHopRoundRobin.cc new file mode 100644 index 00000000000..05c260cc5de --- /dev/null +++ b/proxy/http/remap/unit-tests/test_NextHopRoundRobin.cc @@ -0,0 +1,337 @@ +/** @file + + Unit tests for the NextHopRoundRobin. + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + @section details Details + + Unit testing the NextHopRoundRobin class. + + */ + +#define CATCH_CONFIG_MAIN /* include main function */ + +#include /* catch unit-test framework */ +#include + +#include "nexthop_test_stubs.h" +#include "NextHopSelectionStrategy.h" +#include "NextHopStrategyFactory.h" +#include "NextHopRoundRobin.h" + +SCENARIO("Testing NextHopRoundRobin class, using policy 'rr-strict'", "[NextHopRoundRobin]") +{ + GIVEN("Loading the round-robin-tests.yaml config for round robin 'rr-strict' tests.") + { + std::shared_ptr strategy; + NextHopStrategyFactory nhf(TS_SRC_DIR "unit-tests/round-robin-tests.yaml"); + strategy = nhf.strategyInstance("rr-strict-exhaust-ring"); + + WHEN("the config is loaded.") + { + THEN("the rr-strict strategy is ready for use.") + { + REQUIRE(nhf.strategies_loaded == true); + REQUIRE(strategy != nullptr); + REQUIRE(strategy->policy_type == NH_RR_STRICT); + } + } + + WHEN("making requests using a 'rr-strict' policy.") + { + uint64_t fail_threshold = 1; + uint64_t retry_time = 1; + TestData rdata; + + THEN("then testing rr-strict.") + { + REQUIRE(nhf.strategies_loaded == true); + REQUIRE(strategy != nullptr); + + // first request. + ParentResult result; + strategy->findNextHop(10000, result, rdata, fail_threshold, retry_time); + CHECK(strcmp(result.hostname, "p1.foo.com") == 0); + + // second request. + result.reset(); + strategy->findNextHop(10001, result, rdata, fail_threshold, retry_time); + CHECK(strcmp(result.hostname, "p2.foo.com") == 0); + + // third request. + result.reset(); + strategy->findNextHop(10002, result, rdata, fail_threshold, retry_time); + CHECK(strcmp(result.hostname, "p1.foo.com") == 0); + + // did not reset result, kept it as last parent selected was p1.fo.com, mark it down and we should only select p2.foo.com + strategy->markNextHopDown(10003, result, 1, fail_threshold); + + // fourth request, p1 is down should select p2. + result.reset(); + strategy->findNextHop(10004, result, rdata, fail_threshold, retry_time); + CHECK(strcmp(result.hostname, "p2.foo.com") == 0); + + // fifth request, p1 is down should still select p2. + result.reset(); + strategy->findNextHop(10005, result, rdata, fail_threshold, retry_time); + CHECK(strcmp(result.hostname, "p2.foo.com") == 0); + + // mark down p2. + strategy->markNextHopDown(10006, result, 1, fail_threshold); + + // fifth request, p1 and p2 are both down, should get s1.bar.com from failover ring. + result.reset(); + strategy->findNextHop(10007, result, rdata, fail_threshold, retry_time); + CHECK(strcmp(result.hostname, "s1.bar.com") == 0); + + // sixth request, p1 and p2 are still down, should get s1.bar.com from failover ring. + result.reset(); + strategy->findNextHop(10008, result, rdata, fail_threshold, retry_time); + CHECK(strcmp(result.hostname, "s1.bar.com") == 0); + + // mark down s1. + strategy->markNextHopDown(10009, result, 1, fail_threshold); + + // seventh request, p1, p2, s1 are down, should get s2.bar.com from failover ring. + result.reset(); + strategy->findNextHop(10010, result, rdata, fail_threshold, retry_time); + CHECK(strcmp(result.hostname, "s2.bar.com") == 0); + + // mark down s2. + strategy->markNextHopDown(10011, result, 1, fail_threshold); + + // eighth request, p1, p2, s1, s2 are down, should get PARENT_DIRECT as go_direct is true + result.reset(); + strategy->findNextHop(10012, result, rdata, fail_threshold, retry_time); + CHECK(result.result == ParentResultType::PARENT_DIRECT); + + // check that nextHopExists() returns false when all parents are down. + CHECK(strategy->nextHopExists(10012) == false); + + // change the request time to trigger a retry. + time_t now = (time(nullptr) + 5); + + // ninth request, p1 and p2 are still down, should get p2.foo.com as it will be retried + result.reset(); + strategy->findNextHop(10013, result, rdata, fail_threshold, retry_time, now); + REQUIRE(result.result == ParentResultType::PARENT_SPECIFIED); + CHECK(strcmp(result.hostname, "p2.foo.com") == 0); + + // tenth request, p1 should now be retried. + result.reset(); + strategy->findNextHop(10014, result, rdata, fail_threshold, retry_time, now); + REQUIRE(result.result == ParentResultType::PARENT_SPECIFIED); + CHECK(strcmp(result.hostname, "p1.foo.com") == 0); + } + } + } +} + +SCENARIO("Testing NextHopRoundRobin class, using policy 'first-live'", "[NextHopRoundRobin]") +{ + GIVEN("Loading the round-robin-tests.yaml config for round robin 'first-live' tests.") + { + std::shared_ptr strategy; + NextHopStrategyFactory nhf(TS_SRC_DIR "unit-tests/round-robin-tests.yaml"); + strategy = nhf.strategyInstance("first-live"); + + WHEN("the config is loaded.") + { + THEN("the 'first-live' strategy is available.") + { + REQUIRE(nhf.strategies_loaded == true); + REQUIRE(strategy != nullptr); + REQUIRE(strategy->policy_type == NH_FIRST_LIVE); + } + } + + WHEN("when using a strategy with a 'first-live' policy.") + { + uint64_t fail_threshold = 1; + uint64_t retry_time = 1; + TestData rdata; + + THEN("when making requests and marking down hosts.") + { + REQUIRE(nhf.strategies_loaded == true); + REQUIRE(strategy != nullptr); + + // first request. + ParentResult result; + strategy->findNextHop(20000, result, rdata, fail_threshold, retry_time); + CHECK(strcmp(result.hostname, "p1.foo.com") == 0); + + // second request. + result.reset(); + strategy->findNextHop(20001, result, rdata, fail_threshold, retry_time); + CHECK(strcmp(result.hostname, "p1.foo.com") == 0); + + // mark down p1. + strategy->markNextHopDown(20002, result, 1, fail_threshold); + + // third request. + result.reset(); + strategy->findNextHop(20003, result, rdata, fail_threshold, retry_time); + CHECK(strcmp(result.hostname, "p2.foo.com") == 0); + + // change the request time to trigger a retry. + time_t now = (time(nullptr) + 5); + + // fourth request, p1 should be marked for retry + result.reset(); + strategy->findNextHop(20004, result, rdata, fail_threshold, retry_time, now); + CHECK(strcmp(result.hostname, "p1.foo.com") == 0); + } + } + } +} + +SCENARIO("Testing NextHopRoundRobin class, using policy 'rr-ip'", "[NextHopRoundRobin]") +{ + GIVEN("Loading the round-robin-tests.yaml config for round robin 'rr-ip' tests.") + { + std::shared_ptr strategy; + NextHopStrategyFactory nhf(TS_SRC_DIR "unit-tests/round-robin-tests.yaml"); + strategy = nhf.strategyInstance("rr-ip"); + sockaddr_in sa1, sa2; + sa1.sin_port = 10000; + sa1.sin_family = AF_INET; + inet_pton(AF_INET, "192.168.1.1", &(sa1.sin_addr)); + sa2.sin_port = 10001; + sa2.sin_family = AF_INET; + inet_pton(AF_INET, "192.168.1.2", &(sa2.sin_addr)); + WHEN("the config is loaded.") + { + THEN("then the 'rr-strict' strategy is ready.") + { + REQUIRE(nhf.strategies_loaded == true); + REQUIRE(strategy != nullptr); + REQUIRE(strategy->policy_type == NH_RR_IP); + } + } + + WHEN("using the 'rr-strict' strategy.") + { + uint64_t fail_threshold = 1; + uint64_t retry_time = 1; + TestData rdata; + + THEN("when making requests and marking down hosts.") + { + REQUIRE(nhf.strategies_loaded == true); + REQUIRE(strategy != nullptr); + + // call and test parentExists(), this call should not affect + // findNextHop() round robin strict results + CHECK(strategy->nextHopExists(29000) == true); + + // first request. + memcpy(&rdata.client_ip, &sa1, sizeof(sa1)); + ParentResult result; + strategy->findNextHop(30000, result, rdata, fail_threshold, retry_time); + CHECK(strcmp(result.hostname, "p4.foo.com") == 0); + + // call and test parentExists(), this call should not affect + // findNextHop round robin strict results. + CHECK(strategy->nextHopExists(29000) == true); + + // second request. + memcpy(&rdata.client_ip, &sa2, sizeof(sa2)); + result.reset(); + strategy->findNextHop(30001, result, rdata, fail_threshold, retry_time); + CHECK(strcmp(result.hostname, "p3.foo.com") == 0); + + // call and test parentExists(), this call should not affect + // findNextHop() round robin strict results + CHECK(strategy->nextHopExists(29000) == true); + + // third request with same client ip, result should still be p3 + result.reset(); + strategy->findNextHop(30002, result, rdata, fail_threshold, retry_time); + CHECK(strcmp(result.hostname, "p3.foo.com") == 0); + + // call and test parentExists(), this call should not affect + // findNextHop() round robin strict results. + CHECK(strategy->nextHopExists(29000) == true); + + // fourth request with same client ip and same result indicating a failure should result in p4 + // being selected. + strategy->findNextHop(30003, result, rdata, fail_threshold, retry_time); + CHECK(strcmp(result.hostname, "p4.foo.com") == 0); + } + } + } +} + +SCENARIO("Testing NextHopRoundRobin class, using policy 'latched'", "[NextHopRoundRobin]") +{ + GIVEN("Loading the round-robin-tests.yaml config for round robin 'latched' tests.") + { + std::shared_ptr strategy; + NextHopStrategyFactory nhf(TS_SRC_DIR "unit-tests/round-robin-tests.yaml"); + strategy = nhf.strategyInstance("latched"); + + WHEN("the config is loaded.") + { + THEN("then the 'latched' strategy is available.") + { + REQUIRE(nhf.strategies_loaded == true); + REQUIRE(strategy != nullptr); + REQUIRE(strategy->policy_type == NH_RR_LATCHED); + } + } + + WHEN("using a strategy having a 'latched' policy.") + { + uint64_t fail_threshold = 1; + uint64_t retry_time = 1; + TestData rdata; + + THEN("when making requests and marking down hosts.") + { + REQUIRE(nhf.strategies_loaded == true); + REQUIRE(strategy != nullptr); + + // first request should select p3 + ParentResult result; + strategy->findNextHop(40000, result, rdata, fail_threshold, retry_time); + CHECK(strcmp(result.hostname, "p3.foo.com") == 0); + + // second request should select p3 + result.reset(); + strategy->findNextHop(40001, result, rdata, fail_threshold, retry_time); + CHECK(strcmp(result.hostname, "p3.foo.com") == 0); + + // third request, use previous result to simulate a failure, we should now select p4. + strategy->findNextHop(40002, result, rdata, fail_threshold, retry_time); + CHECK(strcmp(result.hostname, "p4.foo.com") == 0); + + // fourth request we should be latched on p4 + result.reset(); + strategy->findNextHop(40003, result, rdata, fail_threshold, retry_time); + CHECK(strcmp(result.hostname, "p4.foo.com") == 0); + + // fifth request, use previous result to simulate a failure, we should now select p3. + strategy->findNextHop(40004, result, rdata, fail_threshold, retry_time); + CHECK(strcmp(result.hostname, "p3.foo.com") == 0); + } + } + } +} diff --git a/proxy/http/remap/unit-tests/test_NextHopStrategyFactory.cc b/proxy/http/remap/unit-tests/test_NextHopStrategyFactory.cc new file mode 100644 index 00000000000..de9a71f8aa8 --- /dev/null +++ b/proxy/http/remap/unit-tests/test_NextHopStrategyFactory.cc @@ -0,0 +1,956 @@ +/** @file + + Unit tests for the NextHopStrategyFactory. + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + @section details Details + + Unit testing the NextHopStrategy factory. + + */ + +#define CATCH_CONFIG_MAIN /* include main function */ +#include /* catch unit-test framework */ +#include /* ofstream */ +#include +#include +#include + +#include "nexthop_test_stubs.h" +#include "NextHopSelectionStrategy.h" +#include "NextHopStrategyFactory.h" +#include "NextHopConsistentHash.h" +#include "NextHopRoundRobin.h" + +SCENARIO("factory tests loading yaml configs", "[loadConfig]") +{ + GIVEN("Loading the strategy.yaml with included 'hosts.yaml'.") + { +#ifdef TS_SRC_DIR + REQUIRE(chdir(TS_SRC_DIR) == 0); +#endif + NextHopStrategyFactory nhf(TS_SRC_DIR "unit-tests/strategy.yaml"); + + WHEN("the two files are loaded.") + { + THEN("there are two strategies defined in the config, 'strategy-1' and 'strategy-2'") + { + REQUIRE(nhf.strategies_loaded == true); + REQUIRE(nhf.strategyInstance("strategy-1") != nullptr); + REQUIRE(nhf.strategyInstance("strategy-2") != nullptr); + REQUIRE(nhf.strategyInstance("notthere") == nullptr); + } + } + + WHEN("'strategy-1' details are checked.") + { + THEN("Expect that these results for 'strategy-1'") + { + std::shared_ptr strategy = nhf.strategyInstance("strategy-1"); + REQUIRE(strategy != nullptr); + CHECK(strategy->parent_is_proxy == true); + CHECK(strategy->max_simple_retries == 1); + CHECK(strategy->policy_type == NH_CONSISTENT_HASH); + + // down cast here using the stored pointer so that I can verify the hash_key was set + // properly. + NextHopConsistentHash *ptr = static_cast(strategy.get()); + REQUIRE(ptr != nullptr); + CHECK(ptr->hash_key == NH_CACHE_HASH_KEY); + + CHECK(strategy->go_direct == false); + CHECK(strategy->scheme == NH_SCHEME_HTTP); + CHECK(strategy->ring_mode == NH_EXHAUST_RING); + CHECK(strategy->groups == 2); + std::shared_ptr h = strategy->host_groups[0][0]; + CHECK(h != nullptr); + for (unsigned int i = 0; i < strategy->groups; i++) { + CHECK(strategy->host_groups[i].size() == 2); + for (unsigned int j = 0; j < strategy->host_groups[i].size(); j++) { + h = strategy->host_groups[i][j]; + switch (i) { + case 0: + switch (j) { + case 0: + CHECK(h->hostname == "p1.foo.com"); + CHECK(h->protocols[0]->scheme == NH_SCHEME_HTTP); + CHECK(h->protocols[0]->port == 80); + CHECK(h->protocols[0]->health_check_url == "http://192.168.1.1:80"); + CHECK(h->protocols[1]->scheme == NH_SCHEME_HTTPS); + CHECK(h->protocols[1]->port == 443); + CHECK(h->protocols[1]->health_check_url == "https://192.168.1.1:443"); + CHECK(h->weight == 1.5); + CHECK(h->available == true); + break; + case 1: + CHECK(h->hostname == "p2.foo.com"); + CHECK(h->protocols[0]->scheme == NH_SCHEME_HTTP); + CHECK(h->protocols[0]->port == 80); + CHECK(h->protocols[0]->health_check_url == "http://192.168.1.2:80"); + CHECK(h->weight == 1.5); + CHECK(h->available == true); + break; + } + break; + case 1: + switch (j) { + case 0: + CHECK(h->hostname == "p3.foo.com"); + CHECK(h->protocols[0]->scheme == NH_SCHEME_HTTP); + CHECK(h->protocols[0]->port == 8080); + CHECK(h->protocols[0]->health_check_url == "http://192.168.1.3:8080"); + CHECK(h->protocols[1]->scheme == NH_SCHEME_HTTPS); + CHECK(h->protocols[1]->port == 8443); + CHECK(h->protocols[1]->health_check_url == "https://192.168.1.3:8443"); + CHECK(h->weight == 0.5); + CHECK(h->available == true); + break; + case 1: + CHECK(h->hostname == "p4.foo.com"); + CHECK(h->protocols[0]->scheme == NH_SCHEME_HTTP); + CHECK(h->protocols[0]->port == 8080); + CHECK(h->protocols[0]->health_check_url == "http://192.168.1.4:8080"); + CHECK(h->protocols[1]->scheme == NH_SCHEME_HTTPS); + CHECK(h->protocols[1]->port == 8443); + CHECK(h->protocols[1]->health_check_url == "https://192.168.1.4:8443"); + CHECK(h->weight == 1.5); + CHECK(h->available == true); + break; + } + break; + } + } + } + CHECK(strategy->resp_codes.contains(404)); + CHECK(strategy->resp_codes.contains(503)); + CHECK(!strategy->resp_codes.contains(604)); + } + } + + WHEN("'strategy-2' details are checked.") + { + THEN("Expect that these results for 'strategy-2'") + { + std::shared_ptr strategy = nhf.strategyInstance("strategy-2"); + REQUIRE(strategy != nullptr); + CHECK(strategy->policy_type == NH_RR_STRICT); + CHECK(strategy->go_direct == true); + CHECK(strategy->scheme == NH_SCHEME_HTTP); + CHECK(strategy->ring_mode == NH_EXHAUST_RING); + CHECK(strategy->groups == 2); + std::shared_ptr h = strategy->host_groups[0][0]; + CHECK(h != nullptr); + for (unsigned int i = 0; i < strategy->groups; i++) { + CHECK(strategy->host_groups[i].size() == 2); + for (unsigned int j = 0; j < strategy->host_groups[i].size(); j++) { + h = strategy->host_groups[i][j]; + switch (i) { + case 0: + switch (j) { + case 0: + CHECK(h->hostname == "p1.foo.com"); + CHECK(h->protocols[0]->scheme == NH_SCHEME_HTTP); + CHECK(h->protocols[0]->port == 80); + CHECK(h->protocols[0]->health_check_url == "http://192.168.1.1:80"); + CHECK(h->protocols[1]->scheme == NH_SCHEME_HTTPS); + CHECK(h->protocols[1]->port == 443); + CHECK(h->protocols[1]->health_check_url == "https://192.168.1.1:443"); + CHECK(h->weight == 1.5); + break; + case 1: + CHECK(h->hostname == "p2.foo.com"); + CHECK(h->protocols[0]->scheme == NH_SCHEME_HTTP); + CHECK(h->protocols[0]->port == 80); + CHECK(h->protocols[0]->health_check_url == "http://192.168.1.2:80"); + CHECK(h->weight == 1.5); + break; + } + break; + case 1: + switch (j) { + case 0: + CHECK(h->hostname == "p3.foo.com"); + CHECK(h->protocols[0]->scheme == NH_SCHEME_HTTP); + CHECK(h->protocols[0]->port == 8080); + CHECK(h->protocols[0]->health_check_url == "http://192.168.1.3:8080"); + CHECK(h->protocols[1]->scheme == NH_SCHEME_HTTPS); + CHECK(h->protocols[1]->port == 8443); + CHECK(h->protocols[1]->health_check_url == "https://192.168.1.3:8443"); + CHECK(h->weight == 0.5); + break; + case 1: + CHECK(h->hostname == "p4.foo.com"); + CHECK(h->protocols[0]->scheme == NH_SCHEME_HTTP); + CHECK(h->protocols[0]->port == 8080); + CHECK(h->protocols[0]->health_check_url == "http://192.168.1.4:8080"); + CHECK(h->protocols[1]->scheme == NH_SCHEME_HTTPS); + CHECK(h->protocols[1]->port == 8443); + CHECK(h->protocols[1]->health_check_url == "https://192.168.1.4:8443"); + CHECK(h->weight == 1.5); + break; + } + break; + } + } + } + CHECK(strategy->resp_codes.contains(404)); + CHECK(strategy->resp_codes.contains(503)); + CHECK(!strategy->resp_codes.contains(604)); + } + } + } + + GIVEN("loading a yaml config, simple-strategy.yaml ") + { + NextHopStrategyFactory nhf(TS_SRC_DIR "unit-tests/simple-strategy.yaml"); + + WHEN("loading the single file") + { + THEN("loading the simple-strategy.yaml") + { + REQUIRE(nhf.strategies_loaded == true); + REQUIRE(nhf.strategyInstance("strategy-3") != nullptr); + REQUIRE(nhf.strategyInstance("strategy-4") != nullptr); + } + } + + WHEN("'strategy-3' details are checked.") + { + THEN("Expect that these results for 'strategy-3'") + { + std::shared_ptr strategy = nhf.strategyInstance("strategy-3"); + REQUIRE(strategy != nullptr); + CHECK(strategy->policy_type == NH_RR_IP); + CHECK(strategy->go_direct == true); + CHECK(strategy->scheme == NH_SCHEME_HTTPS); + CHECK(strategy->ring_mode == NH_EXHAUST_RING); + CHECK(strategy->groups == 2); + std::shared_ptr h = strategy->host_groups[0][0]; + CHECK(h != nullptr); + for (unsigned int i = 0; i < strategy->groups; i++) { + CHECK(strategy->host_groups[i].size() == 2); + for (unsigned int j = 0; j < strategy->host_groups[i].size(); j++) { + h = strategy->host_groups[i][j]; + switch (i) { + case 0: + switch (j) { + case 0: + CHECK(h->hostname == "p1.foo.com"); + CHECK(h->protocols[0]->scheme == NH_SCHEME_HTTP); + CHECK(h->protocols[0]->port == 80); + CHECK(h->protocols[0]->health_check_url == "http://192.168.1.1:80"); + CHECK(h->protocols[1]->scheme == NH_SCHEME_HTTPS); + CHECK(h->protocols[1]->port == 443); + CHECK(h->protocols[1]->health_check_url == "https://192.168.1.1:443"); + CHECK(h->weight == 1.0); + break; + case 1: + CHECK(h->hostname == "p2.foo.com"); + CHECK(h->protocols[0]->scheme == NH_SCHEME_HTTP); + CHECK(h->protocols[0]->port == 80); + CHECK(h->protocols[0]->health_check_url == "http://192.168.1.2:80"); + CHECK(h->protocols[1]->scheme == NH_SCHEME_HTTPS); + CHECK(h->protocols[1]->port == 443); + CHECK(h->protocols[1]->health_check_url == "https://192.168.1.2:443"); + CHECK(h->weight == 1.0); + break; + } + break; + case 1: + switch (j) { + case 0: + CHECK(h->hostname == "s1.bar.com"); + CHECK(h->protocols[0]->scheme == NH_SCHEME_HTTP); + CHECK(h->protocols[0]->port == 80); + CHECK(h->protocols[0]->health_check_url == "http://192.168.2.1:80"); + CHECK(h->protocols[1]->scheme == NH_SCHEME_HTTPS); + CHECK(h->protocols[1]->port == 443); + CHECK(h->protocols[1]->health_check_url == "https://192.168.2.1:443"); + CHECK(h->weight == 1.0); + break; + case 1: + CHECK(h->hostname == "s2.bar.com"); + CHECK(h->protocols[0]->scheme == NH_SCHEME_HTTP); + CHECK(h->protocols[0]->port == 80); + CHECK(h->protocols[0]->health_check_url == "http://192.168.2.2:80"); + CHECK(h->protocols[1]->scheme == NH_SCHEME_HTTPS); + CHECK(h->protocols[1]->port == 443); + CHECK(h->protocols[1]->health_check_url == "https://192.168.2.2:443"); + CHECK(h->weight == 1.0); + break; + } + break; + } + } + } + CHECK(strategy->resp_codes.contains(404)); + CHECK(strategy->resp_codes.contains(503)); + CHECK(!strategy->resp_codes.contains(604)); + } + } + + WHEN("'strategy-4' details are checked.") + { + THEN("Expect that these results for 'strategy-4'") + { + std::shared_ptr strategy = nhf.strategyInstance("strategy-4"); + REQUIRE(strategy != nullptr); + CHECK(strategy->policy_type == NH_RR_LATCHED); + CHECK(strategy->go_direct == true); + CHECK(strategy->scheme == NH_SCHEME_HTTP); + CHECK(strategy->ring_mode == NH_ALTERNATE_RING); + CHECK(strategy->groups == 1); + std::shared_ptr h = strategy->host_groups[0][0]; + CHECK(h != nullptr); + for (unsigned int i = 0; i < strategy->groups; i++) { + CHECK(strategy->host_groups[i].size() == 2); + for (unsigned int j = 0; j < strategy->host_groups[i].size(); j++) { + h = strategy->host_groups[i][j]; + switch (i) { + case 0: + switch (j) { + case 0: + CHECK(h->hostname == "p3.foo.com"); + CHECK(h->protocols[0]->scheme == NH_SCHEME_HTTP); + CHECK(h->protocols[0]->port == 80); + CHECK(h->protocols[0]->health_check_url == "http://192.168.1.3:80"); + CHECK(h->protocols[1]->scheme == NH_SCHEME_HTTPS); + CHECK(h->protocols[1]->port == 443); + CHECK(h->protocols[1]->health_check_url == "https://192.168.1.3:443"); + CHECK(h->weight == 1.0); + break; + case 1: + CHECK(h->hostname == "p4.foo.com"); + CHECK(h->protocols[0]->scheme == NH_SCHEME_HTTP); + CHECK(h->protocols[0]->port == 80); + CHECK(h->protocols[0]->health_check_url == "http://192.168.1.4:80"); + CHECK(h->protocols[1]->scheme == NH_SCHEME_HTTPS); + CHECK(h->protocols[1]->port == 443); + CHECK(h->protocols[1]->health_check_url == "https://192.168.1.4:443"); + CHECK(h->weight == 1.0); + break; + } + break; + } + } + } + CHECK(strategy->resp_codes.contains(404)); + CHECK(strategy->resp_codes.contains(503)); + CHECK(!strategy->resp_codes.contains(604)); + } + } + } + + GIVEN("loading a yaml config combining hosts and strategies into one file, combined.yaml") + { + NextHopStrategyFactory nhf(TS_SRC_DIR "unit-tests/combined.yaml"); + + WHEN("loading the single file") + { + THEN("loading the single file when there is only one strategy, 'mid-tier-east'") + { + REQUIRE(nhf.strategies_loaded == true); + REQUIRE(nhf.strategyInstance("mid-tier-east") != nullptr); + REQUIRE(nhf.strategyInstance("notthere") == nullptr); + } + } + + WHEN("the strategy 'mid-tier-north' details are checked.") + { + THEN("expect the following details.") + { + std::shared_ptr strategy = nhf.strategyInstance("mid-tier-north"); + REQUIRE(strategy != nullptr); + CHECK(strategy->parent_is_proxy == false); + CHECK(strategy->max_simple_retries == 2); + CHECK(strategy->policy_type == NH_RR_IP); + CHECK(strategy->go_direct == true); + CHECK(strategy->scheme == NH_SCHEME_HTTP); + CHECK(strategy->ring_mode == NH_EXHAUST_RING); + CHECK(strategy->groups == 2); + CHECK(strategy->resp_codes.contains(404)); + CHECK(strategy->resp_codes.contains(502)); + CHECK(!strategy->resp_codes.contains(604)); + CHECK(strategy->health_checks.active == true); + CHECK(strategy->health_checks.passive == true); + std::shared_ptr h = strategy->host_groups[0][0]; + CHECK(h != nullptr); + for (unsigned int i = 0; i < strategy->groups; i++) { + CHECK(strategy->host_groups[i].size() == 2); + for (unsigned int j = 0; j < strategy->host_groups[i].size(); j++) { + h = strategy->host_groups[i][j]; + switch (i) { + case 0: + switch (j) { + case 0: + CHECK(h->hostname == "p1.foo.com"); + CHECK(h->protocols[0]->scheme == NH_SCHEME_HTTP); + CHECK(h->protocols[0]->port == 80); + CHECK(h->protocols[0]->health_check_url == "http://192.168.1.1:80"); + CHECK(h->protocols[1]->scheme == NH_SCHEME_HTTPS); + CHECK(h->protocols[1]->port == 443); + CHECK(h->protocols[1]->health_check_url == "https://192.168.1.1:443"); + CHECK(h->weight == 0.5); + break; + case 1: + CHECK(h->hostname == "p2.foo.com"); + CHECK(h->protocols[0]->scheme == NH_SCHEME_HTTP); + CHECK(h->protocols[0]->port == 80); + CHECK(h->protocols[0]->health_check_url == "http://192.168.1.2:80"); + CHECK(h->weight == 0.5); + break; + } + break; + case 1: + switch (j) { + case 0: + CHECK(h->hostname == "s1.bar.com"); + CHECK(h->protocols[0]->scheme == NH_SCHEME_HTTP); + CHECK(h->protocols[0]->port == 8080); + CHECK(h->protocols[0]->health_check_url == "http://192.168.2.1:8080"); + CHECK(h->protocols[1]->scheme == NH_SCHEME_HTTPS); + CHECK(h->protocols[1]->port == 8443); + CHECK(h->protocols[1]->health_check_url == "https://192.168.2.1:8443"); + CHECK(h->weight == 2.0); + break; + case 1: + CHECK(h->hostname == "s2.bar.com"); + CHECK(h->protocols[0]->scheme == NH_SCHEME_HTTP); + CHECK(h->protocols[0]->port == 8080); + CHECK(h->protocols[0]->health_check_url == "http://192.168.2.2:8080"); + CHECK(h->protocols[1]->scheme == NH_SCHEME_HTTPS); + CHECK(h->protocols[1]->port == 8443); + CHECK(h->protocols[1]->health_check_url == "https://192.168.2.2:8443"); + CHECK(h->weight == 1.0); + break; + } + break; + } + } + } + CHECK(strategy->resp_codes.contains(404)); + CHECK(strategy->resp_codes.contains(503)); + CHECK(!strategy->resp_codes.contains(604)); + } + } + + WHEN("the strategy 'mid-tier-south' details are checked.") + { + THEN("expect the following results.") + { + std::shared_ptr strategy = nhf.strategyInstance("mid-tier-south"); + REQUIRE(strategy != nullptr); + CHECK(strategy->policy_type == NH_RR_LATCHED); + CHECK(strategy->parent_is_proxy == false); + CHECK(strategy->ignore_self_detect == false); + CHECK(strategy->max_simple_retries == 2); + CHECK(strategy->go_direct == false); + CHECK(strategy->scheme == NH_SCHEME_HTTP); + CHECK(strategy->ring_mode == NH_ALTERNATE_RING); + CHECK(strategy->groups == 2); + CHECK(strategy->resp_codes.contains(404)); + CHECK(strategy->resp_codes.contains(502)); + CHECK(!strategy->resp_codes.contains(604)); + CHECK(strategy->health_checks.active == true); + CHECK(strategy->health_checks.passive == true); + std::shared_ptr h = strategy->host_groups[0][0]; + CHECK(h != nullptr); + for (unsigned int i = 0; i < strategy->groups; i++) { + CHECK(strategy->host_groups[i].size() == 2); + for (unsigned int j = 0; j < strategy->host_groups[i].size(); j++) { + h = strategy->host_groups[i][j]; + switch (i) { + case 0: + switch (j) { + case 0: + CHECK(h->hostname == "p1.foo.com"); + CHECK(h->protocols[0]->scheme == NH_SCHEME_HTTP); + CHECK(h->protocols[0]->port == 80); + CHECK(h->protocols[0]->health_check_url == "http://192.168.1.1:80"); + CHECK(h->protocols[1]->scheme == NH_SCHEME_HTTPS); + CHECK(h->protocols[1]->port == 443); + CHECK(h->protocols[1]->health_check_url == "https://192.168.1.1:443"); + CHECK(h->weight == 0.5); + break; + case 1: + CHECK(h->hostname == "p2.foo.com"); + CHECK(h->protocols[0]->scheme == NH_SCHEME_HTTP); + CHECK(h->protocols[0]->port == 80); + CHECK(h->protocols[0]->health_check_url == "http://192.168.1.2:80"); + CHECK(h->weight == 0.5); + break; + } + break; + case 1: + switch (j) { + case 0: + CHECK(h->hostname == "s1.bar.com"); + CHECK(h->protocols[0]->scheme == NH_SCHEME_HTTP); + CHECK(h->protocols[0]->port == 8080); + CHECK(h->protocols[0]->health_check_url == "http://192.168.2.1:8080"); + CHECK(h->protocols[1]->scheme == NH_SCHEME_HTTPS); + CHECK(h->protocols[1]->port == 8443); + CHECK(h->protocols[1]->health_check_url == "https://192.168.2.1:8443"); + CHECK(h->weight == 2.0); + break; + case 1: + CHECK(h->hostname == "s2.bar.com"); + CHECK(h->protocols[0]->scheme == NH_SCHEME_HTTP); + CHECK(h->protocols[0]->port == 8080); + CHECK(h->protocols[0]->health_check_url == "http://192.168.2.2:8080"); + CHECK(h->protocols[1]->scheme == NH_SCHEME_HTTPS); + CHECK(h->protocols[1]->port == 8443); + CHECK(h->protocols[1]->health_check_url == "https://192.168.2.2:8443"); + CHECK(h->weight == 1.0); + break; + } + break; + } + } + } + CHECK(strategy->resp_codes.contains(404)); + CHECK(strategy->resp_codes.contains(503)); + CHECK(!strategy->resp_codes.contains(604)); + } + } + + WHEN("the strategy 'mid-tier-east' details are checked.") + { + THEN("expect the following results.") + { + std::shared_ptr strategy = nhf.strategyInstance("mid-tier-east"); + REQUIRE(strategy != nullptr); + CHECK(strategy->policy_type == NH_FIRST_LIVE); + CHECK(strategy->parent_is_proxy == false); + CHECK(strategy->ignore_self_detect == true); + CHECK(strategy->max_simple_retries == 2); + CHECK(strategy->go_direct == false); + CHECK(strategy->scheme == NH_SCHEME_HTTPS); + CHECK(strategy->ring_mode == NH_ALTERNATE_RING); + CHECK(strategy->groups == 2); + CHECK(strategy->resp_codes.contains(404)); + CHECK(strategy->resp_codes.contains(502)); + CHECK(!strategy->resp_codes.contains(604)); + CHECK(strategy->health_checks.active == false); + CHECK(strategy->health_checks.passive == true); + std::shared_ptr h = strategy->host_groups[0][0]; + CHECK(h != nullptr); + for (unsigned int i = 0; i < strategy->groups; i++) { + CHECK(strategy->host_groups[i].size() == 2); + for (unsigned int j = 0; j < strategy->host_groups[i].size(); j++) { + h = strategy->host_groups[i][j]; + switch (i) { + case 0: + switch (j) { + case 0: + CHECK(h->hostname == "p1.foo.com"); + CHECK(h->protocols[0]->scheme == NH_SCHEME_HTTP); + CHECK(h->protocols[0]->port == 80); + CHECK(h->protocols[0]->health_check_url == "http://192.168.1.1:80"); + CHECK(h->protocols[1]->scheme == NH_SCHEME_HTTPS); + CHECK(h->protocols[1]->port == 443); + CHECK(h->protocols[1]->health_check_url == "https://192.168.1.1:443"); + CHECK(h->weight == 0.5); + break; + case 1: + CHECK(h->hostname == "p2.foo.com"); + CHECK(h->protocols[0]->scheme == NH_SCHEME_HTTP); + CHECK(h->protocols[0]->port == 80); + CHECK(h->protocols[0]->health_check_url == "http://192.168.1.2:80"); + CHECK(h->weight == 0.5); + break; + } + break; + case 1: + switch (j) { + case 0: + CHECK(h->hostname == "s1.bar.com"); + CHECK(h->protocols[0]->scheme == NH_SCHEME_HTTP); + CHECK(h->protocols[0]->port == 8080); + CHECK(h->protocols[0]->health_check_url == "http://192.168.2.1:8080"); + CHECK(h->protocols[1]->scheme == NH_SCHEME_HTTPS); + CHECK(h->protocols[1]->port == 8443); + CHECK(h->protocols[1]->health_check_url == "https://192.168.2.1:8443"); + CHECK(h->weight == 2.0); + break; + case 1: + CHECK(h->hostname == "s2.bar.com"); + CHECK(h->protocols[0]->scheme == NH_SCHEME_HTTP); + CHECK(h->protocols[0]->port == 8080); + CHECK(h->protocols[0]->health_check_url == "http://192.168.2.2:8080"); + CHECK(h->protocols[1]->scheme == NH_SCHEME_HTTPS); + CHECK(h->protocols[1]->port == 8443); + CHECK(h->protocols[1]->health_check_url == "https://192.168.2.2:8443"); + CHECK(h->weight == 1.0); + break; + } + break; + } + } + } + CHECK(strategy->resp_codes.contains(404)); + CHECK(strategy->resp_codes.contains(503)); + CHECK(!strategy->resp_codes.contains(604)); + } + } + + WHEN("the strategy 'mid-tier-west' details are checked.") + { + THEN("expect the following results.") + { + std::shared_ptr strategy = nhf.strategyInstance("mid-tier-west"); + REQUIRE(strategy != nullptr); + CHECK(strategy->policy_type == NH_RR_STRICT); + CHECK(strategy->go_direct == true); + CHECK(strategy->scheme == NH_SCHEME_HTTPS); + CHECK(strategy->parent_is_proxy == false); + CHECK(strategy->max_simple_retries == 2); + CHECK(strategy->ring_mode == NH_EXHAUST_RING); + CHECK(strategy->groups == 2); + CHECK(strategy->resp_codes.contains(404)); + CHECK(strategy->resp_codes.contains(502)); + CHECK(!strategy->resp_codes.contains(604)); + CHECK(strategy->health_checks.active == true); + CHECK(strategy->health_checks.passive == false); + std::shared_ptr h = strategy->host_groups[0][0]; + CHECK(h != nullptr); + for (unsigned int i = 0; i < strategy->groups; i++) { + CHECK(strategy->host_groups[i].size() == 2); + for (unsigned int j = 0; j < strategy->host_groups[i].size(); j++) { + h = strategy->host_groups[i][j]; + switch (i) { + case 0: + switch (j) { + case 0: + CHECK(h->hostname == "p1.foo.com"); + CHECK(h->protocols[0]->scheme == NH_SCHEME_HTTP); + CHECK(h->protocols[0]->port == 80); + CHECK(h->protocols[0]->health_check_url == "http://192.168.1.1:80"); + CHECK(h->protocols[1]->scheme == NH_SCHEME_HTTPS); + CHECK(h->protocols[1]->port == 443); + CHECK(h->protocols[1]->health_check_url == "https://192.168.1.1:443"); + CHECK(h->weight == 0.5); + break; + case 1: + CHECK(h->hostname == "p2.foo.com"); + CHECK(h->protocols[0]->scheme == NH_SCHEME_HTTP); + CHECK(h->protocols[0]->port == 80); + CHECK(h->protocols[0]->health_check_url == "http://192.168.1.2:80"); + CHECK(h->weight == 0.5); + break; + } + break; + case 1: + switch (j) { + case 0: + CHECK(h->hostname == "s1.bar.com"); + CHECK(h->protocols[0]->scheme == NH_SCHEME_HTTP); + CHECK(h->protocols[0]->port == 8080); + CHECK(h->protocols[0]->health_check_url == "http://192.168.2.1:8080"); + CHECK(h->protocols[1]->scheme == NH_SCHEME_HTTPS); + CHECK(h->protocols[1]->port == 8443); + CHECK(h->protocols[1]->health_check_url == "https://192.168.2.1:8443"); + CHECK(h->weight == 2.0); + break; + case 1: + CHECK(h->hostname == "s2.bar.com"); + CHECK(h->protocols[0]->scheme == NH_SCHEME_HTTP); + CHECK(h->protocols[0]->port == 8080); + CHECK(h->protocols[0]->health_check_url == "http://192.168.2.2:8080"); + CHECK(h->protocols[1]->scheme == NH_SCHEME_HTTPS); + CHECK(h->protocols[1]->port == 8443); + CHECK(h->protocols[1]->health_check_url == "https://192.168.2.2:8443"); + CHECK(h->weight == 1.0); + break; + } + break; + } + } + } + CHECK(strategy->resp_codes.contains(404)); + CHECK(strategy->resp_codes.contains(503)); + CHECK(!strategy->resp_codes.contains(604)); + } + } + + WHEN("the strategy 'mid-tier-midwest' details are checked.") + { + THEN("expect the following results.") + { + std::shared_ptr strategy = nhf.strategyInstance("mid-tier-midwest"); + REQUIRE(strategy != nullptr); + CHECK(strategy->policy_type == NH_CONSISTENT_HASH); + CHECK(strategy->parent_is_proxy == false); + CHECK(strategy->max_simple_retries == 2); + + // I need to down cast here using the stored pointer so that I can verify that + // the hash_key was set properly. + NextHopConsistentHash *ptr = static_cast(strategy.get()); + REQUIRE(ptr != nullptr); + CHECK(ptr->hash_key == NH_CACHE_HASH_KEY); + + CHECK(strategy->go_direct == true); + CHECK(strategy->scheme == NH_SCHEME_HTTPS); + CHECK(strategy->ring_mode == NH_EXHAUST_RING); + CHECK(strategy->groups == 2); + CHECK(strategy->resp_codes.contains(404)); + CHECK(strategy->resp_codes.contains(502)); + CHECK(!strategy->resp_codes.contains(604)); + CHECK(strategy->health_checks.active == true); + CHECK(strategy->health_checks.passive == false); + std::shared_ptr h = strategy->host_groups[0][0]; + CHECK(h != nullptr); + for (unsigned int i = 0; i < strategy->groups; i++) { + CHECK(strategy->host_groups[i].size() == 2); + for (unsigned int j = 0; j < strategy->host_groups[i].size(); j++) { + h = strategy->host_groups[i][j]; + switch (i) { + case 0: + switch (j) { + case 0: + CHECK(h->hostname == "p1.foo.com"); + CHECK(h->protocols[0]->scheme == NH_SCHEME_HTTP); + CHECK(h->protocols[0]->port == 80); + CHECK(h->protocols[0]->health_check_url == "http://192.168.1.1:80"); + CHECK(h->protocols[1]->scheme == NH_SCHEME_HTTPS); + CHECK(h->protocols[1]->port == 443); + CHECK(h->protocols[1]->health_check_url == "https://192.168.1.1:443"); + CHECK(h->weight == 0.5); + break; + case 1: + CHECK(h->hostname == "p2.foo.com"); + CHECK(h->protocols[0]->scheme == NH_SCHEME_HTTP); + CHECK(h->protocols[0]->port == 80); + CHECK(h->protocols[0]->health_check_url == "http://192.168.1.2:80"); + CHECK(h->weight == 0.5); + break; + } + break; + case 1: + switch (j) { + case 0: + CHECK(h->hostname == "s1.bar.com"); + CHECK(h->protocols[0]->scheme == NH_SCHEME_HTTP); + CHECK(h->protocols[0]->port == 8080); + CHECK(h->protocols[0]->health_check_url == "http://192.168.2.1:8080"); + CHECK(h->protocols[1]->scheme == NH_SCHEME_HTTPS); + CHECK(h->protocols[1]->port == 8443); + CHECK(h->protocols[1]->health_check_url == "https://192.168.2.1:8443"); + CHECK(h->weight == 2.0); + break; + case 1: + CHECK(h->hostname == "s2.bar.com"); + CHECK(h->protocols[0]->scheme == NH_SCHEME_HTTP); + CHECK(h->protocols[0]->port == 8080); + CHECK(h->protocols[0]->health_check_url == "http://192.168.2.2:8080"); + CHECK(h->protocols[1]->scheme == NH_SCHEME_HTTPS); + CHECK(h->protocols[1]->port == 8443); + CHECK(h->protocols[1]->health_check_url == "https://192.168.2.2:8443"); + CHECK(h->weight == 1.0); + break; + } + break; + } + } + } + CHECK(strategy->resp_codes.contains(404)); + CHECK(strategy->resp_codes.contains(503)); + CHECK(!strategy->resp_codes.contains(604)); + } + } + } +} + +SCENARIO("factory tests loading yaml configs from a directory", "[loadConfig]") +{ + GIVEN("Loading the strategies using a directory of 'yaml' files") + { + NextHopStrategyFactory nhf(TS_SRC_DIR "unit-tests/strategies-dir"); + + WHEN("the two files are loaded.") + { + THEN("there are two strategies defined in the config, 'strategy-1' and 'strategy-2'") + { + REQUIRE(nhf.strategies_loaded == true); + REQUIRE(nhf.strategyInstance("mid-tier-north") != nullptr); + REQUIRE(nhf.strategyInstance("mid-tier-south") != nullptr); + } + } + + WHEN("the strategy 'mid-tier-north' details are checked.") + { + THEN("expect the following results.") + { + std::shared_ptr strategy = nhf.strategyInstance("mid-tier-north"); + REQUIRE(strategy != nullptr); + CHECK(strategy->parent_is_proxy == false); + CHECK(strategy->max_simple_retries == 2); + CHECK(strategy->policy_type == NH_RR_IP); + CHECK(strategy->go_direct == true); + CHECK(strategy->scheme == NH_SCHEME_HTTP); + CHECK(strategy->ring_mode == NH_EXHAUST_RING); + CHECK(strategy->groups == 2); + CHECK(strategy->resp_codes.contains(404)); + CHECK(strategy->resp_codes.contains(502)); + CHECK(!strategy->resp_codes.contains(604)); + CHECK(strategy->health_checks.active == true); + CHECK(strategy->health_checks.passive == true); + std::shared_ptr h = strategy->host_groups[0][0]; + CHECK(h != nullptr); + for (unsigned int i = 0; i < strategy->groups; i++) { + CHECK(strategy->host_groups[i].size() == 2); + for (unsigned int j = 0; j < strategy->host_groups[i].size(); j++) { + h = strategy->host_groups[i][j]; + switch (i) { + case 0: + switch (j) { + case 0: + CHECK(h->hostname == "p1.foo.com"); + CHECK(h->protocols[0]->scheme == NH_SCHEME_HTTP); + CHECK(h->protocols[0]->port == 80); + CHECK(h->protocols[0]->health_check_url == "http://192.168.1.1:80"); + CHECK(h->protocols[1]->scheme == NH_SCHEME_HTTPS); + CHECK(h->protocols[1]->port == 443); + CHECK(h->protocols[1]->health_check_url == "https://192.168.1.1:443"); + CHECK(h->weight == 0.5); + break; + case 1: + CHECK(h->hostname == "p2.foo.com"); + CHECK(h->protocols[0]->scheme == NH_SCHEME_HTTP); + CHECK(h->protocols[0]->port == 80); + CHECK(h->protocols[0]->health_check_url == "http://192.168.1.2:80"); + CHECK(h->weight == 0.5); + break; + } + break; + case 1: + switch (j) { + case 0: + CHECK(h->hostname == "p3.foo.com"); + CHECK(h->protocols[0]->scheme == NH_SCHEME_HTTP); + CHECK(h->protocols[0]->port == 8080); + CHECK(h->protocols[0]->health_check_url == "http://192.168.1.3:8080"); + CHECK(h->protocols[1]->scheme == NH_SCHEME_HTTPS); + CHECK(h->protocols[1]->port == 8443); + CHECK(h->protocols[1]->health_check_url == "https://192.168.1.3:8443"); + CHECK(h->weight == 0.5); + break; + case 1: + CHECK(h->hostname == "p4.foo.com"); + CHECK(h->protocols[0]->scheme == NH_SCHEME_HTTP); + CHECK(h->protocols[0]->port == 8080); + CHECK(h->protocols[0]->health_check_url == "http://192.168.1.4:8080"); + CHECK(h->protocols[1]->scheme == NH_SCHEME_HTTPS); + CHECK(h->protocols[1]->port == 8443); + CHECK(h->protocols[1]->health_check_url == "https://192.168.1.4:8443"); + CHECK(h->weight == 0.5); + break; + } + break; + } + } + } + CHECK(strategy->resp_codes.contains(404)); + CHECK(strategy->resp_codes.contains(503)); + CHECK(!strategy->resp_codes.contains(604)); + } + } + + WHEN("the strategy 'mid-tier-south' details are checked.") + { + THEN("expect the following results.") + { + std::shared_ptr strategy = nhf.strategyInstance("mid-tier-south"); + REQUIRE(strategy != nullptr); + CHECK(strategy->policy_type == NH_RR_LATCHED); + CHECK(strategy->parent_is_proxy == false); + CHECK(strategy->ignore_self_detect == false); + CHECK(strategy->max_simple_retries == 2); + CHECK(strategy->go_direct == false); + CHECK(strategy->scheme == NH_SCHEME_HTTP); + CHECK(strategy->ring_mode == NH_ALTERNATE_RING); + CHECK(strategy->groups == 2); + CHECK(strategy->resp_codes.contains(404)); + CHECK(strategy->resp_codes.contains(502)); + CHECK(!strategy->resp_codes.contains(604)); + CHECK(strategy->health_checks.active == true); + CHECK(strategy->health_checks.passive == true); + std::shared_ptr h = strategy->host_groups[0][0]; + CHECK(h != nullptr); + for (unsigned int i = 0; i < strategy->groups; i++) { + CHECK(strategy->host_groups[i].size() == 2); + for (unsigned int j = 0; j < strategy->host_groups[i].size(); j++) { + h = strategy->host_groups[i][j]; + switch (i) { + case 0: + switch (j) { + case 0: + CHECK(h->hostname == "p1.foo.com"); + CHECK(h->protocols[0]->scheme == NH_SCHEME_HTTP); + CHECK(h->protocols[0]->port == 80); + CHECK(h->protocols[0]->health_check_url == "http://192.168.1.1:80"); + CHECK(h->protocols[1]->scheme == NH_SCHEME_HTTPS); + CHECK(h->protocols[1]->port == 443); + CHECK(h->protocols[1]->health_check_url == "https://192.168.1.1:443"); + CHECK(h->weight == 0.5); + break; + case 1: + CHECK(h->hostname == "p2.foo.com"); + CHECK(h->protocols[0]->scheme == NH_SCHEME_HTTP); + CHECK(h->protocols[0]->port == 80); + CHECK(h->protocols[0]->health_check_url == "http://192.168.1.2:80"); + CHECK(h->weight == 0.5); + break; + } + break; + case 1: + switch (j) { + case 0: + CHECK(h->hostname == "p3.foo.com"); + CHECK(h->protocols[0]->scheme == NH_SCHEME_HTTP); + CHECK(h->protocols[0]->port == 8080); + CHECK(h->protocols[0]->health_check_url == "http://192.168.1.3:8080"); + CHECK(h->protocols[1]->scheme == NH_SCHEME_HTTPS); + CHECK(h->protocols[1]->port == 8443); + CHECK(h->protocols[1]->health_check_url == "https://192.168.1.3:8443"); + CHECK(h->weight == 0.5); + break; + case 1: + CHECK(h->hostname == "p4.foo.com"); + CHECK(h->protocols[0]->scheme == NH_SCHEME_HTTP); + CHECK(h->protocols[0]->port == 8080); + CHECK(h->protocols[0]->health_check_url == "http://192.168.1.4:8080"); + CHECK(h->protocols[1]->scheme == NH_SCHEME_HTTPS); + CHECK(h->protocols[1]->port == 8443); + CHECK(h->protocols[1]->health_check_url == "https://192.168.1.4:8443"); + CHECK(h->weight == 0.5); + break; + } + break; + } + } + } + CHECK(strategy->resp_codes.contains(404)); + CHECK(strategy->resp_codes.contains(503)); + CHECK(!strategy->resp_codes.contains(604)); + } + } + } +} diff --git a/proxy/http/remap/unit-tests/test_PluginDso.cc b/proxy/http/remap/unit-tests/test_PluginDso.cc index 092261b76fd..2db5d1c43fe 100644 --- a/proxy/http/remap/unit-tests/test_PluginDso.cc +++ b/proxy/http/remap/unit-tests/test_PluginDso.cc @@ -69,7 +69,11 @@ class PluginDsoUnitTest : public PluginDso } virtual void - indicateReload() + indicatePreReload() + { + } + virtual void + indicatePostReload(TSRemapReloadStatus reloadStatus) { } virtual bool diff --git a/proxy/http/remap/unit-tests/test_PluginFactory.cc b/proxy/http/remap/unit-tests/test_PluginFactory.cc index 9c2d81942f2..7f0c8fcee2a 100644 --- a/proxy/http/remap/unit-tests/test_PluginFactory.cc +++ b/proxy/http/remap/unit-tests/test_PluginFactory.cc @@ -35,6 +35,30 @@ #include "plugin_testing_common.h" #include "../PluginFactory.h" #include "../PluginDso.h" +#include "I_EventSystem.h" +#include "tscore/I_Layout.h" +#include "diags.i" + +#define TEST_THREADS 2 +struct EventProcessorListener : Catch::TestEventListenerBase { + using TestEventListenerBase::TestEventListenerBase; + + void + testRunStarting(Catch::TestRunInfo const &testRunInfo) override + { + Layout::create(); + init_diags("", nullptr); + RecProcessInit(RECM_STAND_ALONE); + + ink_event_system_init(EVENT_SYSTEM_MODULE_PUBLIC_VERSION); + eventProcessor.start(TEST_THREADS, 1048576); + + EThread *main_thread = new EThread; + main_thread->set_specific(); + } +}; + +CATCH_REGISTER_LISTENER(EventProcessorListener); thread_local PluginThreadContext *pluginThreadContext; @@ -90,6 +114,78 @@ clean() fs::remove(sandboxDir, ec); } +static int +getPluginVersion(const PluginDso &plugin) +{ + std::string error; + void *s = nullptr; + CHECK(plugin.getSymbol("pluginDsoVersionTest", s, error)); + int (*version)() = reinterpret_cast(s); + return version ? version() : -1; +} + +// this is a simple class to simulate loading of Plugin Dso as done during loading of global plugins +class GlobalPluginInfo +{ +public: + GlobalPluginInfo() : _dlh(nullptr){}; + ~GlobalPluginInfo(){}; + + bool + loadDso(const fs::path &configPath) + { + CHECK(fs::exists(configPath)); + + void *handle = dlopen(configPath.c_str(), RTLD_NOW); + if (!handle) { + return false; + } + + _dlh = handle; + return true; + } + + bool + getSymbol(const char *symbol, void *&address, std::string &error) const + { + /* Clear the errors */ + dlerror(); + error.clear(); + + address = dlsym(_dlh, symbol); + char *err = dlerror(); + + if (nullptr == address && nullptr != err) { + /* symbol really cannot be found */ + error.assign(err); + return false; + } + + return true; + } + + void * + dlOpenHandle() + { + return _dlh; + } + + int + getPluginVersion() + { + std::string error; + void *s = nullptr; + CHECK(getSymbol("pluginDsoVersionTest", s, error)); + auto version = reinterpret_cast(s); + return version ? version() : -1; + } + +private: + void *_dlh; +}; + +// copies the .so file (pluginBuildPath) in the target directory (effectivePath) +// as the desired .so filename (configPath) with the desired timestamp (mtime) static void setupConfigPathTest(const fs::path &configPath, const fs::path &pluginBuildPath, const fs::path &uuid, fs::path &effectivePath, fs::path &runtimePath, time_t mtime = 0, bool append = false) @@ -100,7 +196,11 @@ setupConfigPathTest(const fs::path &configPath, const fs::path &pluginBuildPath, } effectivePath = configPath.is_absolute() ? configPath : searchDir / configPath; - runtimePath = runtimeRootDir / uuid / effectivePath.relative_path(); + if (isPluginDynamicReloadEnabled()) { + runtimePath = runtimeRootDir / uuid / effectivePath.relative_path(); + } else { + runtimePath = effectivePath; + } /* Create the directory structure and install plugins */ fs::create_directories(effectivePath.parent_path(), ec); @@ -151,6 +251,7 @@ SCENARIO("loading plugins", "[plugin][core]") fs::path effectivePath; fs::path runtimePath; std::string error; + enablePluginDynamicReload(); GIVEN("an existing plugin") { @@ -164,9 +265,13 @@ SCENARIO("loading plugins", "[plugin][core]") setupConfigPathTest(configPath, buildPath, tempComponent, effectivePath, runtimePath); PluginFactoryUnitTest *factory = getFactory(tempComponent); - RemapPluginInst *plugin = factory->getRemapPlugin(configPath, 0, nullptr, error); + RemapPluginInst *plugin = factory->getRemapPlugin(configPath, 0, nullptr, error, isPluginDynamicReloadEnabled()); - THEN("expect it to successfully load") { validateSuccessfulConfigPathTest(plugin, error, effectivePath, runtimePath); } + THEN("expect it to successfully load") + { + validateSuccessfulConfigPathTest(plugin, error, effectivePath, runtimePath); + CHECK(nullptr != PluginDso::loadedPlugins()->findByEffectivePath(effectivePath, isPluginDynamicReloadEnabled())); + } teardownConfigPathTest(factory); } @@ -178,25 +283,92 @@ SCENARIO("loading plugins", "[plugin][core]") setupConfigPathTest(configPath, buildPath, tempComponent, effectivePath, runtimePath); PluginFactoryUnitTest *factory = getFactory(tempComponent); - RemapPluginInst *plugin = factory->getRemapPlugin(configPath, 0, nullptr, error); + RemapPluginInst *plugin = factory->getRemapPlugin(configPath, 0, nullptr, error, isPluginDynamicReloadEnabled()); + + THEN("expect it to successfully load") + { + validateSuccessfulConfigPathTest(plugin, error, effectivePath, runtimePath); + CHECK(nullptr != PluginDso::loadedPlugins()->findByEffectivePath(effectivePath, isPluginDynamicReloadEnabled())); + } + + teardownConfigPathTest(factory); + } + + WHEN("config is using plugin absolute path - dynamic reload is ENABLED") + { + fs::path configPath = searchDir / "subdir" / pluginName; + CHECK(configPath.is_absolute()); /* make sure this is absolute path - this is what we are testing */ + setupConfigPathTest(configPath, buildPath, tempComponent, effectivePath, runtimePath); + PluginFactoryUnitTest *factory = getFactory(tempComponent); + RemapPluginInst *plugin = factory->getRemapPlugin(configPath, 0, nullptr, error, isPluginDynamicReloadEnabled()); + + THEN("expect it to successfully load") + { + validateSuccessfulConfigPathTest(plugin, error, effectivePath, runtimePath); + CHECK(nullptr != PluginDso::loadedPlugins()->findByEffectivePath(effectivePath, isPluginDynamicReloadEnabled())); + + // check Dso at effective path still exists while copy at runtime path doesn't + CHECK(fs::exists(plugin->_plugin.effectivePath())); + CHECK(!fs::exists(plugin->_plugin.runtimePath())); + } + + teardownConfigPathTest(factory); + } + + WHEN("config is using plugin absolute path - dynamic reload is DISABLED") + { + disablePluginDynamicReload(); + fs::path configPath = searchDir / "subdir" / pluginName; + CHECK(configPath.is_absolute()); /* make sure this is absolute path - this is what we are testing */ + setupConfigPathTest(configPath, buildPath, tempComponent, effectivePath, runtimePath); + PluginFactoryUnitTest *factory = getFactory(tempComponent); + RemapPluginInst *plugin = factory->getRemapPlugin(configPath, 0, nullptr, error, isPluginDynamicReloadEnabled()); - THEN("expect it to successfully load") { validateSuccessfulConfigPathTest(plugin, error, effectivePath, runtimePath); } + THEN("expect it to successfully load") + { + validateSuccessfulConfigPathTest(plugin, error, effectivePath, runtimePath); + CHECK(nullptr != PluginDso::loadedPlugins()->findByEffectivePath(effectivePath, isPluginDynamicReloadEnabled())); + + // check Dso still exists + CHECK(plugin->_plugin.effectivePath() == plugin->_plugin.runtimePath()); + CHECK(fs::exists(plugin->_plugin.effectivePath())); + } teardownConfigPathTest(factory); + enablePluginDynamicReload(); } - WHEN("config is using plugin absolute path") + WHEN("config is using plugin absolute path - dynamic reload is ENABLED but overwritten by optout") { + enablePluginDynamicReload(); fs::path configPath = searchDir / "subdir" / pluginName; CHECK(configPath.is_absolute()); /* make sure this is absolute path - this is what we are testing */ + // We have to force the path to be the effective == runtime + disablePluginDynamicReload(); setupConfigPathTest(configPath, buildPath, tempComponent, effectivePath, runtimePath); + enablePluginDynamicReload(); + PluginFactoryUnitTest *factory = getFactory(tempComponent); - RemapPluginInst *plugin = factory->getRemapPlugin(configPath, 0, nullptr, error); + // make the factory take this plugin as it opt out. + PluginDso::loadedPlugins()->addPluginPathToDsoOptOutTable(effectivePath.string()); + + RemapPluginInst *plugin = factory->getRemapPlugin(configPath, 0, nullptr, error, isPluginDynamicReloadEnabled()); - THEN("expect it to successfully load") { validateSuccessfulConfigPathTest(plugin, error, effectivePath, runtimePath); } + THEN("expect it to successfully load") + { + validateSuccessfulConfigPathTest(plugin, error, effectivePath, runtimePath); + static const bool disableDynamicReloadByOptOut{false}; + CHECK(nullptr != PluginDso::loadedPlugins()->findByEffectivePath(effectivePath, disableDynamicReloadByOptOut)); + // check Dso still exists + CHECK(plugin->_plugin.effectivePath() == plugin->_plugin.runtimePath()); + CHECK(fs::exists(plugin->_plugin.effectivePath())); + } + // we need to remove this from the list so next tests can work as expected. + PluginDso::loadedPlugins()->removePluginPathFromDsoOptOutTable(effectivePath.string()); teardownConfigPathTest(factory); + enablePluginDynamicReload(); } WHEN("config using nonexisting relative plugin file name") @@ -209,7 +381,7 @@ SCENARIO("loading plugins", "[plugin][core]") setupConfigPathTest(relativeExistingPath, buildPath, tempComponent, effectivePath, runtimePath); PluginFactoryUnitTest *factory = getFactory(tempComponent); - RemapPluginInst *plugin = factory->getRemapPlugin(relativeNonexistingPath, 0, nullptr, error); + RemapPluginInst *plugin = factory->getRemapPlugin(relativeNonexistingPath, 0, nullptr, error, isPluginDynamicReloadEnabled()); THEN("expect it to fail with appropriate error message") { @@ -231,7 +403,7 @@ SCENARIO("loading plugins", "[plugin][core]") setupConfigPathTest(relativeExistingPath, buildPath, tempComponent, effectivePath, runtimePath); PluginFactoryUnitTest *factory = getFactory(tempComponent); - RemapPluginInst *plugin = factory->getRemapPlugin(absoluteNonexistingPath, 0, nullptr, error); + RemapPluginInst *plugin = factory->getRemapPlugin(absoluteNonexistingPath, 0, nullptr, error, isPluginDynamicReloadEnabled()); THEN("expect it to fail with appropriate error message") { @@ -243,12 +415,47 @@ SCENARIO("loading plugins", "[plugin][core]") teardownConfigPathTest(factory); } + + WHEN("plugin initialization fails") + { + fs::path configPath = fs::path("plugin_init_fail.so"); + fs::path buildPath = pluginBuildDir / configPath; + setupConfigPathTest(configPath, buildPath, tempComponent, effectivePath, runtimePath); + PluginFactoryUnitTest *factory = getFactory(tempComponent); + RemapPluginInst *plugin = factory->getRemapPlugin(configPath, 0, nullptr, error, isPluginDynamicReloadEnabled()); + + THEN("expect it to unload the plugin dso") + { + CHECK(nullptr == plugin); + CHECK(nullptr == PluginDso::loadedPlugins()->findByEffectivePath(effectivePath, isPluginDynamicReloadEnabled())); + } + + teardownConfigPathTest(factory); + } + + WHEN("instance initialization fails") + { + fs::path configPath = fs::path("plugin_instinit_fail.so"); + fs::path buildPath = pluginBuildDir / configPath; + setupConfigPathTest(configPath, buildPath, tempComponent, effectivePath, runtimePath); + PluginFactoryUnitTest *factory = getFactory(tempComponent); + RemapPluginInst *plugin = factory->getRemapPlugin(configPath, 0, nullptr, error, isPluginDynamicReloadEnabled()); + + THEN("expect it to unload the plugin dso") + { + CHECK(nullptr == plugin); + CHECK(nullptr == PluginDso::loadedPlugins()->findByEffectivePath(effectivePath, isPluginDynamicReloadEnabled())); + } + + teardownConfigPathTest(factory); + } } } SCENARIO("multiple search dirs + multiple or no plugins installed", "[plugin][core]") { REQUIRE_FALSE(sandboxDir.empty()); + enablePluginDynamicReload(); GIVEN("multiple search dirs specified for the plugin search") { @@ -298,7 +505,7 @@ SCENARIO("multiple search dirs + multiple or no plugins installed", "[plugin][co CHECK(fs::exists(abEffectivePath)); /* Now use an absolute path containing the unregistered search directory */ - RemapPluginInst *pluginInst = factory.getRemapPlugin(abEffectivePath, 0, nullptr, error); + RemapPluginInst *pluginInst = factory.getRemapPlugin(abEffectivePath, 0, nullptr, error, isPluginDynamicReloadEnabled()); THEN("Expect it to successfully load") { @@ -312,7 +519,7 @@ SCENARIO("multiple search dirs + multiple or no plugins installed", "[plugin][co WHEN("a valid plugin is found in the first search path") { - RemapPluginInst *pluginInst = factory.getRemapPlugin(configPath, 0, nullptr, error); + RemapPluginInst *pluginInst = factory.getRemapPlugin(configPath, 0, nullptr, error, isPluginDynamicReloadEnabled()); THEN("Expect it to successfully load the one found in the first search dir and copy it in the runtime dir") { @@ -327,7 +534,7 @@ SCENARIO("multiple search dirs + multiple or no plugins installed", "[plugin][co WHEN("the first search dir is missing the plugin but the second search has it") { CHECK(fs::remove(effectivePath1, ec)); - RemapPluginInst *pluginInst = factory.getRemapPlugin(configPath, 0, nullptr, error); + RemapPluginInst *pluginInst = factory.getRemapPlugin(configPath, 0, nullptr, error, isPluginDynamicReloadEnabled()); THEN("Expect it to successfully load the one found in the second search dir") { @@ -343,7 +550,7 @@ SCENARIO("multiple search dirs + multiple or no plugins installed", "[plugin][co { CHECK(fs::remove(effectivePath1, ec)); CHECK(fs::remove(effectivePath2, ec)); - RemapPluginInst *pluginInst = factory.getRemapPlugin(configPath, 0, nullptr, error); + RemapPluginInst *pluginInst = factory.getRemapPlugin(configPath, 0, nullptr, error, isPluginDynamicReloadEnabled()); THEN("Expect it to successfully load the one found in the third search dir") { @@ -363,7 +570,7 @@ SCENARIO("multiple search dirs + multiple or no plugins installed", "[plugin][co THEN("expect the plugin load to fail.") { - RemapPluginInst *pluginInst = factory.getRemapPlugin(configPath, 0, nullptr, error); + RemapPluginInst *pluginInst = factory.getRemapPlugin(configPath, 0, nullptr, error, isPluginDynamicReloadEnabled()); CHECK(nullptr == pluginInst); CHECK(std::string("failed to find plugin '").append(configPath.string()).append("'") == error); CHECK_FALSE(fs::exists(runtimePath1)); @@ -375,30 +582,124 @@ SCENARIO("multiple search dirs + multiple or no plugins installed", "[plugin][co } } -static int -getPluginVersion(const PluginDso &plugin) +void +checkTwoLoadedVersionsDifferent(const RemapPluginInst *plugin_v1, const RemapPluginInst *plugin_v2) +{ + void *tsRemapInitSym_v1_t2 = nullptr; /* callback address from DSO v1 at moment t2 */ + void *tsRemapInitSym_v2_t2 = nullptr; /* callback address from DSO v2 at moment t2 */ + std::string error; + + /* Make sure we ended up with different DSO objects and runtime paths are different - new plugin was indeed loaded */ + CHECK(&(plugin_v1->_plugin) != &(plugin_v2->_plugin)); + CHECK(plugin_v1->_plugin.runtimePath() != plugin_v2->_plugin.runtimePath()); + CHECK(plugin_v1->_plugin.dlOpenHandle() != plugin_v2->_plugin.dlOpenHandle()); + + /* Make sure what we installed and loaded first was v1 and after the plugin reload we run v2 */ + CHECK(1 == getPluginVersion(plugin_v1->_plugin)); + CHECK(2 == getPluginVersion(plugin_v2->_plugin)); + + /* Make sure the symbols we get from the 2 loaded plugins don't yield the same callback function pointer */ + plugin_v1->_plugin.getSymbol("TSRemapInit", tsRemapInitSym_v1_t2, error); + plugin_v2->_plugin.getSymbol("TSRemapInit", tsRemapInitSym_v2_t2, error); + CHECK(nullptr != tsRemapInitSym_v1_t2); + CHECK(nullptr != tsRemapInitSym_v2_t2); + CHECK(tsRemapInitSym_v1_t2 != tsRemapInitSym_v2_t2); + + // 2 versions can be different only when dynamic reload enabled + CHECK(isPluginDynamicReloadEnabled()); + + // check Dso at effective path still exists while the copy at runtime path doesn't + CHECK(plugin_v1->_plugin.effectivePath() != plugin_v1->_plugin.runtimePath()); + CHECK(fs::exists(plugin_v1->_plugin.effectivePath())); + CHECK(!fs::exists(plugin_v1->_plugin.runtimePath())); + CHECK(plugin_v2->_plugin.effectivePath() != plugin_v2->_plugin.runtimePath()); + CHECK(fs::exists(plugin_v2->_plugin.effectivePath())); + CHECK(!fs::exists(plugin_v2->_plugin.runtimePath())); +} + +void +checkTwoLoadedVersionsSame(RemapPluginInst *plugin_v1, RemapPluginInst *plugin_v2, bool pluginOptOut = false) { + void *tsRemapInitSym_v1_t2 = nullptr; /* callback address from DSO v1 at moment t2 */ + void *tsRemapInitSym_v2_t2 = nullptr; /* callback address from DSO v2 at moment t2 */ std::string error; - void *s = nullptr; - CHECK(plugin.getSymbol("pluginDsoVersionTest", s, error)); - int (*version)() = reinterpret_cast(s); - return version ? version() : -1; + + /* Make sure we ended up with the same DSO object and runtime paths should be same - no new plugin was loaded */ + CHECK(&(plugin_v1->_plugin) == &(plugin_v2->_plugin)); + CHECK(plugin_v1->_plugin.runtimePath() == plugin_v2->_plugin.runtimePath()); + CHECK(plugin_v1->_plugin.dlOpenHandle() == plugin_v2->_plugin.dlOpenHandle()); + + /* Make sure v2 DSO was NOT really loaded - both instances should return same v1 version */ + CHECK(1 == getPluginVersion(plugin_v1->_plugin)); + CHECK(1 == getPluginVersion(plugin_v2->_plugin)); + + /* Make sure the symbols we get from the 2 loaded plugins yield the same callback function pointer */ + plugin_v1->_plugin.getSymbol("TSRemapInit", tsRemapInitSym_v1_t2, error); + plugin_v2->_plugin.getSymbol("TSRemapInit", tsRemapInitSym_v2_t2, error); + CHECK(nullptr != tsRemapInitSym_v1_t2); + CHECK(nullptr != tsRemapInitSym_v2_t2); + CHECK(tsRemapInitSym_v1_t2 == tsRemapInitSym_v2_t2); + + // 2 versions can be same even when dynamic reload is enabled + // 2 versions must be same when dynamic reload is disabled or the plugin opt out + // check presence/absence of Dso files + if (pluginOptOut || !isPluginDynamicReloadEnabled()) { + CHECK(plugin_v1->_plugin.effectivePath() == plugin_v1->_plugin.runtimePath()); + CHECK(fs::exists(plugin_v1->_plugin.effectivePath())); + } else { + CHECK(plugin_v1->_plugin.effectivePath() != plugin_v1->_plugin.runtimePath()); + CHECK(fs::exists(plugin_v1->_plugin.effectivePath())); + CHECK(!fs::exists(plugin_v1->_plugin.runtimePath())); + } +} + +std::tuple +testSetupLoadPlugin(const fs::path &configName, const fs::path &buildPath, const fs::path &uuid, const time_t mtime, + fs::path &effectivePath, fs::path &runtimePath) +{ + std::string error; + setupConfigPathTest(configName, buildPath, uuid, effectivePath, runtimePath, mtime); + auto factory = getFactory(uuid); + auto pluginInst = factory->getRemapPlugin(configName, 0, nullptr, error, isPluginDynamicReloadEnabled()); + + return {pluginInst, factory}; +} + +std::tuple +testSetupLoadPluginWithOptOut(const fs::path &configName, const fs::path &buildPath, const fs::path &uuid, const time_t mtime, + fs::path &effectivePath, fs::path &runtimePath) +{ + std::string error; + + // force paths to be as dynamic disabled which will be the case for a plugin that opt out. + disablePluginDynamicReload(); + setupConfigPathTest(configName, buildPath, uuid, effectivePath, runtimePath, mtime); + enablePluginDynamicReload(); + + auto factory = getFactory(uuid); + + // Set factory's opt out plugin information. + PluginDso::loadedPlugins()->addPluginPathToDsoOptOutTable(effectivePath.string()); + auto pluginInst = factory->getRemapPlugin(configName, 0, nullptr, error, isPluginDynamicReloadEnabled()); + // make sure we remove it so there is no evidence of this in the Plugin's list. + PluginDso::loadedPlugins()->removePluginPathFromDsoOptOutTable(effectivePath.string()); + return {pluginInst, factory}; } SCENARIO("loading multiple version of the same plugin at the same time", "[plugin][core]") { REQUIRE_FALSE(sandboxDir.empty()); + enablePluginDynamicReload(); - static fs::path uuid_t1 = fs::path("c71e2bab-90dc-4770-9535-c9304c3de381"); /* UUID at moment t1 */ - static fs::path uuid_t2 = fs::path("c71e2bab-90dc-4770-9535-e7304c3ee732"); /* UUID at moment t2 */ + static fs::path uuid_t1 = fs::path("c71e2bab-90dc-4770-9535-c9304c3de381"); /* UUID at moment t1 */ + static fs::path uuid_t2 = fs::path("c71e2bab-90dc-4770-9535-e7304c3ee732"); /* UUID at moment t2 */ + void *tsRemapInitSym_v1_t1 = nullptr; /* callback address from DSO v1 at moment t1 */ + void *tsRemapInitSym_v1_t2 = nullptr; /* callback address from DSO v1 at moment t2 */ - fs::path effectivePath_v1; /* expected effective path for DSO v1 */ - fs::path effectivePath_v2; /* expected effective path for DSO v2 */ - fs::path runtimePath_v1; /* expected runtime path for DSO v1 */ - fs::path runtimePath_v2; /* expected runtime path for DSO v2 */ - void *tsRemapInitSym_v1_t1 = nullptr; /* callback address from DSO v1 at moment t1 */ - void *tsRemapInitSym_v1_t2 = nullptr; /* callback address from DSO v1 at moment t2 */ - void *tsRemapInitSym_v2_t2 = nullptr; /* callback address from DSO v2 at moment t2 */ + fs::path effectivePath_v1; /* expected effective path for DSO v1 */ + fs::path effectivePath_v2; /* expected effective path for DSO v2 */ + fs::path runtimePath_v1; /* expected runtime path for DSO v1 */ + fs::path runtimePath_v2; /* expected runtime path for DSO v2 */ std::string error; std::string error1; @@ -408,26 +709,26 @@ SCENARIO("loading multiple version of the same plugin at the same time", "[plugi fs::path buildPath_v1 = pluginBuildDir / fs::path("plugin_v1.so"); /* DSO v1 */ fs::path buildPath_v2 = pluginBuildDir / fs::path("plugin_v2.so"); /* DSO v1 */ - GIVEN("two different versions v1 and v2 of same plugin") + GIVEN("two different versions v1 and v2 of same plugin with different time stamps - dynamic plugin reload ENABLED") { WHEN("(1) loading v1, (2) overwriting with v2 and then (3) reloading by using the same plugin name, " "(*) v1 and v2 DSOs modification time are different (changed)") { + enablePluginDynamicReload(); + /* Simulate installing plugin plugin_v1.so (ver 1) as plugin.so and loading it at some point of time t1 */ - setupConfigPathTest(configName, buildPath_v1, uuid_t1, effectivePath_v1, runtimePath_v1, 1556825556); - PluginFactoryUnitTest *factory1 = getFactory(uuid_t1); - RemapPluginInst *plugin_v1 = factory1->getRemapPlugin(configName, 0, nullptr, error1); + auto [plugin_v1, factory1] = + testSetupLoadPlugin(configName, buildPath_v1, uuid_t1, 1556825556, effectivePath_v1, runtimePath_v1); plugin_v1->_plugin.getSymbol("TSRemapInit", tsRemapInitSym_v1_t1, error); /* Simulate installing plugin plugin_v2.so (v1) as plugin.so and loading it at some point of time t2 */ /* Note that during the installation plugin_v2.so (v2) is "barberically" overriding the existing plugin.so which was v1 */ - setupConfigPathTest(configName, buildPath_v2, uuid_t2, effectivePath_v2, runtimePath_v2, 1556825557); - PluginFactoryUnitTest *factory2 = getFactory(uuid_t2); - RemapPluginInst *plugin_v2 = factory2->getRemapPlugin(configName, 0, nullptr, error2); + auto [plugin_v2, factory2] = + testSetupLoadPlugin(configName, buildPath_v2, uuid_t2, 1556825557, effectivePath_v2, runtimePath_v2); + plugin_v1->_plugin.getSymbol("TSRemapInit", tsRemapInitSym_v1_t2, error); /* Make sure plugin.so was overriden */ CHECK(effectivePath_v1 == effectivePath_v2); - /* Although effective path is the same runtime paths should be different */ CHECK(runtimePath_v1 != runtimePath_v2); @@ -437,16 +738,7 @@ SCENARIO("loading multiple version of the same plugin at the same time", "[plugi validateSuccessfulConfigPathTest(plugin_v1, error1, effectivePath_v1, runtimePath_v1); validateSuccessfulConfigPathTest(plugin_v2, error2, effectivePath_v2, runtimePath_v2); - /* Make sure what we installed and loaded first was v1 and after the plugin reload we run v2 */ - CHECK(1 == getPluginVersion(plugin_v1->_plugin)); - CHECK(2 == getPluginVersion(plugin_v2->_plugin)); - - /* Make sure the symbols we get from the 2 loaded plugins don't yield the same callback function pointer */ - plugin_v1->_plugin.getSymbol("TSRemapInit", tsRemapInitSym_v1_t2, error); - plugin_v2->_plugin.getSymbol("TSRemapInit", tsRemapInitSym_v2_t2, error); - CHECK(nullptr != tsRemapInitSym_v1_t2); - CHECK(nullptr != tsRemapInitSym_v2_t2); - CHECK(tsRemapInitSym_v1_t2 != tsRemapInitSym_v2_t2); + checkTwoLoadedVersionsDifferent(plugin_v1, plugin_v2); /* Make sure v1 callback functions addresses did not change for v1 after v2 was loaded */ CHECK(tsRemapInitSym_v1_t1 == tsRemapInitSym_v1_t2); @@ -457,23 +749,25 @@ SCENARIO("loading multiple version of the same plugin at the same time", "[plugi } } - GIVEN("two different versions v1 and v2 of same plugin") + GIVEN("two different versions v1 and v2 of same plugin with same time stamps - dynamic plugin reload ENABLED") { WHEN("(1) loading v1, (2) overwriting with v2 and then (3) reloading by using the same plugin name, " "(*) v1 and v2 DSOs modification time are same (did NOT change)") { + enablePluginDynamicReload(); + /* Simulate installing plugin plugin_v1.so (ver 1) as plugin.so and loading it at some point of time t1 */ - setupConfigPathTest(configName, buildPath_v1, uuid_t1, effectivePath_v1, runtimePath_v1, 1556825556); - PluginFactoryUnitTest *factory1 = getFactory(uuid_t1); - RemapPluginInst *plugin_v1 = factory1->getRemapPlugin(configName, 0, nullptr, error1); + auto [plugin_v1, factory1] = + testSetupLoadPlugin(configName, buildPath_v1, uuid_t1, 1556825556, effectivePath_v1, runtimePath_v1); + plugin_v1->_plugin.getSymbol("TSRemapInit", tsRemapInitSym_v1_t1, error); /* Simulate installing plugin plugin_v2.so (v1) as plugin.so and loading it at some point of time t2 */ /* Note that during the installation plugin_v2.so (v2) is "barberically" overriding the existing plugin.so which was v1, since the modification time is exactly the same the new v2 plugin would not be loaded and we should get the same PluginDso address and same effective and runtime paths */ - setupConfigPathTest(configName, buildPath_v2, uuid_t2, effectivePath_v2, runtimePath_v2, 1556825556); - PluginFactoryUnitTest *factory2 = getFactory(uuid_t2); - RemapPluginInst *plugin_v2 = factory2->getRemapPlugin(configName, 0, nullptr, error2); + auto [plugin_v2, factory2] = + testSetupLoadPlugin(configName, buildPath_v2, uuid_t2, 1556825556, effectivePath_v2, runtimePath_v2); + plugin_v1->_plugin.getSymbol("TSRemapInit", tsRemapInitSym_v1_t2, error); /* Make sure plugin.so was overriden */ CHECK(effectivePath_v1 == effectivePath_v2); @@ -484,24 +778,142 @@ SCENARIO("loading multiple version of the same plugin at the same time", "[plugi validateSuccessfulConfigPathTest(plugin_v1, error1, effectivePath_v1, runtimePath_v1); validateSuccessfulConfigPathTest(plugin_v2, error2, effectivePath_v2, runtimePath_v1); - /* Make sure we ended up with the same DSO object and runtime paths should be same - no new plugin was loaded */ - CHECK(&(plugin_v1->_plugin) == &(plugin_v2->_plugin)); - CHECK(plugin_v1->_plugin.runtimePath() == plugin_v2->_plugin.runtimePath()); + checkTwoLoadedVersionsSame(plugin_v1, plugin_v2); - /* Make sure v2 DSO was NOT loaded both instances should return same v1 version */ - CHECK(1 == getPluginVersion(plugin_v1->_plugin)); - CHECK(1 == getPluginVersion(plugin_v2->_plugin)); + /* Make sure v1 callback functions addresses did not change for v1 after v2 was loaded */ + CHECK(tsRemapInitSym_v1_t1 == tsRemapInitSym_v1_t2); + } - /* Make sure the symbols we get from the 2 loaded plugins yield the same callback function pointer */ - plugin_v1->_plugin.getSymbol("TSRemapInit", tsRemapInitSym_v1_t2, error); - plugin_v2->_plugin.getSymbol("TSRemapInit", tsRemapInitSym_v2_t2, error); - CHECK(nullptr != tsRemapInitSym_v1_t2); - CHECK(nullptr != tsRemapInitSym_v2_t2); - CHECK(tsRemapInitSym_v1_t2 == tsRemapInitSym_v2_t2); + teardownConfigPathTest(factory1); + teardownConfigPathTest(factory2); + } + } + + GIVEN("two different versions v1 and v2 of same plugin with different time stamps - dynamic plugin reload DISABLED") + { + WHEN("(1) loading v1, (2) overwriting with v2 and then (3) reloading by using the same plugin name, " + "(*) v1 and v2 DSOs modification time are different (changed)") + { + disablePluginDynamicReload(); + + /* Simulate installing plugin plugin_v1.so (ver 1) as plugin.so and loading it at some point of time t1 */ + auto [plugin_v1, factory1] = + testSetupLoadPlugin(configName, buildPath_v1, uuid_t1, 1556825556, effectivePath_v1, runtimePath_v1); + plugin_v1->_plugin.getSymbol("TSRemapInit", tsRemapInitSym_v1_t1, error); + + /* Simulate installing plugin plugin_v2.so (v1) as plugin.so and loading it at some point of time t2 */ + /* Note that during the installation plugin_v2.so (v2) is "barberically" overriding the existing plugin.so which was v1 */ + auto [plugin_v2, factory2] = + testSetupLoadPlugin(configName, buildPath_v2, uuid_t2, 1556825557, effectivePath_v2, runtimePath_v2); + plugin_v1->_plugin.getSymbol("TSRemapInit", tsRemapInitSym_v1_t2, error); + + /* Make sure plugin.so was overriden */ + CHECK(effectivePath_v1 == effectivePath_v2); + /* since dynamic reload is disabled, runtimepaths should be same */ + CHECK(runtimePath_v1 == runtimePath_v2); + + THEN("expect v1 plugin to remain loaded since dynamic reload is disabled, even though the timestamp has changed") + { + /* Both getRemapPlugin() calls should succeed but only v1 plugin DSO should be used */ + validateSuccessfulConfigPathTest(plugin_v1, error1, effectivePath_v1, runtimePath_v1); + validateSuccessfulConfigPathTest(plugin_v2, error2, effectivePath_v2, runtimePath_v1); + + checkTwoLoadedVersionsSame(plugin_v1, plugin_v2); + + /* Make sure v1 callback functions addresses did not change for v1 after v2 was loaded */ + CHECK(tsRemapInitSym_v1_t1 == tsRemapInitSym_v1_t2); + } + + teardownConfigPathTest(factory1); + teardownConfigPathTest(factory2); + enablePluginDynamicReload(); + } + } + + GIVEN("two different versions v1 and v2 of same plugin with different time stamps - dynamic plugin reload ENABLED") + { + WHEN("(1) loading v1, (2) overwriting with v2 and then (3) reloading by using the same plugin name, " + "(*) v1 and v2 DSOs modification time are different (changed)") + { + enablePluginDynamicReload(); + + /* Simulate installing plugin plugin_v1.so (ver 1) as plugin.so and loading it at some point of time t1 */ + auto [plugin_v1, factory1] = + testSetupLoadPluginWithOptOut(configName, buildPath_v1, uuid_t1, 1556825556, effectivePath_v1, runtimePath_v1); + + plugin_v1->_plugin.getSymbol("TSRemapInit", tsRemapInitSym_v1_t1, error); + + /* Simulate installing plugin plugin_v2.so (v1) as plugin.so and loading it at some point of time t2 */ + /* Note that during the installation plugin_v2.so (v2) is "barberically" overriding the existing plugin.so which was v1 */ + auto [plugin_v2, factory2] = + testSetupLoadPluginWithOptOut(configName, buildPath_v2, uuid_t2, 1556825557, effectivePath_v2, runtimePath_v2); + plugin_v1->_plugin.getSymbol("TSRemapInit", tsRemapInitSym_v1_t2, error); + + /* Make sure plugin.so was overriden */ + CHECK(effectivePath_v1 == effectivePath_v2); + + /* since dynamic reload is disabled, runtimepaths should be same */ + CHECK(runtimePath_v1 == runtimePath_v2); + + THEN("expect v1 plugin to remain loaded since dynamic reload is disabled, even though the timestamp has changed") + { + /* Both getRemapPlugin() calls should succeed but only v1 plugin DSO should be used */ + validateSuccessfulConfigPathTest(plugin_v1, error1, effectivePath_v1, runtimePath_v1); + validateSuccessfulConfigPathTest(plugin_v2, error2, effectivePath_v2, runtimePath_v1); + + const bool pluginOptOut{true}; + checkTwoLoadedVersionsSame(plugin_v1, plugin_v2, pluginOptOut); + + /* Make sure v1 callback functions addresses did not change for v1 after v2 was loaded */ + CHECK(tsRemapInitSym_v1_t1 == tsRemapInitSym_v1_t2); + } + + teardownConfigPathTest(factory1); + teardownConfigPathTest(factory2); + enablePluginDynamicReload(); + } + } + + GIVEN("two different versions v1 and v2 of same plugin with same time stamp - dynamic plugin reload DISABLED") + { + WHEN("(1) loading v1, (2) overwriting with v2 and then (3) reloading by using the same plugin name, " + "(*) v1 and v2 DSOs modification time are same (did NOT change)") + { + disablePluginDynamicReload(); + + /* Simulate installing plugin plugin_v1.so (ver 1) as plugin.so and loading it at some point of time t1 */ + auto [plugin_v1, factory1] = + testSetupLoadPlugin(configName, buildPath_v1, uuid_t1, 1556825556, effectivePath_v1, runtimePath_v1); + plugin_v1->_plugin.getSymbol("TSRemapInit", tsRemapInitSym_v1_t1, error); + + /* Simulate installing plugin plugin_v2.so (v1) as plugin.so and loading it at some point of time t2 */ + /* Note that during the installation plugin_v2.so (v2) is "barberically" overriding the existing plugin.so + which was v1, since dynamic reload is disabled the new v2 plugin would not be loaded and + we should get the same PluginDso address and same effective and runtime paths */ + auto [plugin_v2, factory2] = + testSetupLoadPlugin(configName, buildPath_v2, uuid_t2, 1556825556, effectivePath_v2, runtimePath_v2); + plugin_v1->_plugin.getSymbol("TSRemapInit", tsRemapInitSym_v1_t2, error); + + /* Make sure plugin.so was overriden */ + CHECK(effectivePath_v1 == effectivePath_v2); + /* since dynamic reload is disabled, runtimepaths should be same */ + CHECK(runtimePath_v1 == runtimePath_v2); + + THEN("expect v1 plugin to be loaded since dynamic reload is disabled") + { + /* Both getRemapPlugin() calls should succeed but only v1 plugin DSO should be used */ + validateSuccessfulConfigPathTest(plugin_v1, error1, effectivePath_v1, runtimePath_v1); + validateSuccessfulConfigPathTest(plugin_v2, error2, effectivePath_v2, runtimePath_v1); + + checkTwoLoadedVersionsSame(plugin_v1, plugin_v2); + + /* Make sure v1 callback functions addresses did not change for v1 after v2 was loaded */ + CHECK(tsRemapInitSym_v1_t1 == tsRemapInitSym_v1_t2); } teardownConfigPathTest(factory1); teardownConfigPathTest(factory2); + enablePluginDynamicReload(); } } @@ -515,9 +927,8 @@ SCENARIO("loading multiple version of the same plugin at the same time", "[plugi PluginFactoryUnitTest *factory1 = getFactory(uuid_t1); /* Now provision and load a plugin using a second factory */ - setupConfigPathTest(configName, buildPath_v2, uuid_t2, effectivePath_v2, runtimePath_v2, 1556825556); - PluginFactoryUnitTest *factory2 = getFactory(uuid_t2); - RemapPluginInst *plugin_v2 = factory2->getRemapPlugin(configName, 0, nullptr, error2); + auto [plugin_v2, factory2] = + testSetupLoadPlugin(configName, buildPath_v2, uuid_t2, 1556825556, effectivePath_v2, runtimePath_v2); THEN("the plugin from the second factory to work") { @@ -533,9 +944,241 @@ SCENARIO("loading multiple version of the same plugin at the same time", "[plugi } } +SCENARIO("loading multiple version of the same plugin in mixed mode - global as well as remap plugin", "[plugin][core]") +{ + REQUIRE_FALSE(sandboxDir.empty()); + enablePluginDynamicReload(); + + static fs::path uuid_t1 = fs::path("c71e2bab-90dc-4770-9535-c9304c3de381"); /* UUID at moment t1 */ + static fs::path uuid_t2 = fs::path("c71e2bab-90dc-4770-9535-e7304c3ee732"); /* UUID at moment t2 */ + + fs::path effectivePath_v1; /* expected effective path for DSO v1 */ + fs::path effectivePath_v2; /* expected effective path for DSO v2 */ + fs::path runtimePath_v1; /* expected runtime path for DSO v1 */ + fs::path runtimePath_v2; /* expected runtime path for DSO v2 */ + void *tsRemapInitSym_v1_t1 = nullptr; /* callback address from DSO v1 at moment t1 */ + void *tsRemapInitSym_v1_t2 = nullptr; /* callback address from DSO v1 at moment t2 */ + void *tsRemapInitSym_v2_t2 = nullptr; /* callback address from DSO v2 at moment t2 */ + + std::string error; + std::string error1; + std::string error2; + + fs::path configName = fs::path("plugin.so"); /* use same config name for all following tests */ + fs::path buildPath_v1 = pluginBuildDir / fs::path("plugin_v1.so"); /* DSO v1 */ + fs::path buildPath_v2 = pluginBuildDir / fs::path("plugin_v2.so"); /* DSO v1 */ + + GIVEN("two different versions v1 and v2 of same plugin in mixed mode - dynamic plugin reload DISABLED") + { + WHEN("(1) loading v1, (2) overwriting with v2 and then (3) reloading by using the same plugin name, " + "(*) v1 and v2 DSOs modification time are different (changed)") + { + disablePluginDynamicReload(); + /* Simulate installing plugin plugin_v1.so (ver 1) as plugin.so and loading it at some point of time t1 */ + setupConfigPathTest(configName, buildPath_v1, uuid_t1, effectivePath_v1, runtimePath_v1, 1556825556); + GlobalPluginInfo global_plugin_v1; + CHECK(global_plugin_v1.loadDso(effectivePath_v1) == true); + global_plugin_v1.getSymbol("TSRemapInit", tsRemapInitSym_v1_t1, error); + + /* Simulate installing plugin plugin_v2.so (v1) as plugin.so and loading it at some point of time t2 */ + /* Note that during the installation plugin_v2.so (v2) is "barberically" overriding the existing plugin.so which was v1 */ + setupConfigPathTest(configName, buildPath_v2, uuid_t2, effectivePath_v2, runtimePath_v2, 1556825557); + PluginFactoryUnitTest *factory2 = getFactory(uuid_t2); + RemapPluginInst *plugin_v2 = factory2->getRemapPlugin(configName, 0, nullptr, error2, isPluginDynamicReloadEnabled()); + + /* Make sure plugin.so was overriden */ + CHECK(effectivePath_v1 == effectivePath_v2); + + /* since dynamic reload is disabled, runtimepaths should be same */ + CHECK(runtimePath_v1 == runtimePath_v2); + + THEN("expect v1 plugin to remain loaded since dynamic reload is disabled, even though the timestamp has changed") + { + validateSuccessfulConfigPathTest(plugin_v2, error2, effectivePath_v2, runtimePath_v2); + + /* Make sure we ended up with the same DSO object and runtime paths should be same - no new plugin was loaded */ + CHECK(global_plugin_v1.dlOpenHandle() == plugin_v2->_plugin.dlOpenHandle()); + + /* Make sure v2 DSO was NOT really loaded - both instances should return same v1 version */ + CHECK(1 == global_plugin_v1.getPluginVersion()); + CHECK(1 == getPluginVersion(plugin_v2->_plugin)); + + /* Make sure the symbols we get from the 2 loaded plugins yield the same callback function pointer */ + global_plugin_v1.getSymbol("TSRemapInit", tsRemapInitSym_v1_t2, error); + plugin_v2->_plugin.getSymbol("TSRemapInit", tsRemapInitSym_v2_t2, error); + CHECK(nullptr != tsRemapInitSym_v1_t2); + CHECK(nullptr != tsRemapInitSym_v2_t2); + CHECK(tsRemapInitSym_v1_t2 == tsRemapInitSym_v2_t2); + + /* check that the symbol we got originally is still valid */ + CHECK(tsRemapInitSym_v1_t1 == tsRemapInitSym_v1_t2); + } + + teardownConfigPathTest(factory2); + enablePluginDynamicReload(); + } + } + + GIVEN("two different versions v1 and v2 of same plugin in mixed mode - dynamic plugin reload ENABLED but overwritten by opt out") + { + WHEN("(1) loading v1, (2) overwriting with v2 and then (3) reloading by using the same plugin name, " + "(*) v1 and v2 DSOs modification time are different (changed)") + { + // just to get the proper path's + disablePluginDynamicReload(); + /* Simulate installing plugin plugin_v1.so (ver 1) as plugin.so and loading it at some point of time t1 */ + setupConfigPathTest(configName, buildPath_v1, uuid_t1, effectivePath_v1, runtimePath_v1, 1556825556); + + GlobalPluginInfo global_plugin_v1; + CHECK(global_plugin_v1.loadDso(effectivePath_v1) == true); + global_plugin_v1.getSymbol("TSRemapInit", tsRemapInitSym_v1_t1, error); + + /* Simulate installing plugin plugin_v2.so (v1) as plugin.so and loading it at some point of time t2 */ + /* Note that during the installation plugin_v2.so (v2) is "barberically" overriding the existing plugin.so which was v1 */ + setupConfigPathTest(configName, buildPath_v2, uuid_t2, effectivePath_v2, runtimePath_v2, 1556825557); + enablePluginDynamicReload(); + PluginFactoryUnitTest *factory = getFactory(uuid_t2); + + // Set factory's opt out plugin information. + PluginDso::loadedPlugins()->addPluginPathToDsoOptOutTable(effectivePath_v1.string()); + + RemapPluginInst *plugin_v2 = factory->getRemapPlugin(configName, 0, nullptr, error2, isPluginDynamicReloadEnabled()); + + /* Make sure plugin.so was overriden */ + CHECK(effectivePath_v1 == effectivePath_v2); + + /* since dynamic reload is disabled, runtimepaths should be same */ + CHECK(runtimePath_v1 == runtimePath_v2); + + THEN("expect v1 plugin to remain loaded since dynamic reload is disabled, even though the timestamp has changed") + { + validateSuccessfulConfigPathTest(plugin_v2, error2, effectivePath_v2, runtimePath_v2); + + /* Make sure we ended up with the same DSO object and runtime paths should be same - no new plugin was loaded */ + CHECK(global_plugin_v1.dlOpenHandle() == plugin_v2->_plugin.dlOpenHandle()); + + /* Make sure v2 DSO was NOT really loaded - both instances should return same v1 version */ + CHECK(1 == global_plugin_v1.getPluginVersion()); + CHECK(1 == getPluginVersion(plugin_v2->_plugin)); + + /* Make sure the symbols we get from the 2 loaded plugins yield the same callback function pointer */ + global_plugin_v1.getSymbol("TSRemapInit", tsRemapInitSym_v1_t2, error); + plugin_v2->_plugin.getSymbol("TSRemapInit", tsRemapInitSym_v2_t2, error); + CHECK(nullptr != tsRemapInitSym_v1_t2); + CHECK(nullptr != tsRemapInitSym_v2_t2); + CHECK(tsRemapInitSym_v1_t2 == tsRemapInitSym_v2_t2); + + /* check that the symbol we got originally is still valid */ + CHECK(tsRemapInitSym_v1_t1 == tsRemapInitSym_v1_t2); + } + PluginDso::loadedPlugins()->removePluginPathFromDsoOptOutTable(effectivePath_v1.string()); + teardownConfigPathTest(factory); + enablePluginDynamicReload(); + } + } + + GIVEN("two different versions v1 and v2 of same plugin in mixed mode - dynamic plugin reload DISABLED") + { + WHEN("(1) loading v1, (2) overwriting with v2 and then (3) reloading by using the same plugin name, " + "(*) v1 and v2 DSOs modification times are unchanged") + { + disablePluginDynamicReload(); + /* Simulate installing plugin plugin_v1.so (ver 1) as plugin.so and loading it at some point of time t1 */ + setupConfigPathTest(configName, buildPath_v1, uuid_t1, effectivePath_v1, runtimePath_v1, 1556825556); + GlobalPluginInfo global_plugin_v1; + CHECK(global_plugin_v1.loadDso(effectivePath_v1) == true); + global_plugin_v1.getSymbol("TSRemapInit", tsRemapInitSym_v1_t1, error); + + /* Simulate installing plugin plugin_v2.so (v1) as plugin.so and loading it at some point of time t2 */ + /* Note that during the installation plugin_v2.so (v2) is "barberically" overriding the existing plugin.so which was v1 */ + setupConfigPathTest(configName, buildPath_v2, uuid_t2, effectivePath_v2, runtimePath_v2, 1556825556); + PluginFactoryUnitTest *factory2 = getFactory(uuid_t2); + RemapPluginInst *plugin_v2 = factory2->getRemapPlugin(configName, 0, nullptr, error2, isPluginDynamicReloadEnabled()); + + /* Make sure plugin.so was overriden */ + CHECK(effectivePath_v1 == effectivePath_v2); + + /* since dynamic reload is disabled, runtimepaths should be same */ + CHECK(runtimePath_v1 == runtimePath_v2); + + THEN("expect v1 plugin to remain loaded since dynamic reload is disabled, even though the timestamp has changed") + { + validateSuccessfulConfigPathTest(plugin_v2, error2, effectivePath_v2, runtimePath_v2); + + /* Make sure we ended up with the same DSO object and runtime paths should be same - no new plugin was loaded */ + CHECK(global_plugin_v1.dlOpenHandle() == plugin_v2->_plugin.dlOpenHandle()); + + /* Make sure v2 DSO was NOT really loaded - both instances should return same v1 version */ + CHECK(1 == global_plugin_v1.getPluginVersion()); + CHECK(1 == getPluginVersion(plugin_v2->_plugin)); + + /* Make sure the symbols we get from the 2 loaded plugins yield the same callback function pointer */ + global_plugin_v1.getSymbol("TSRemapInit", tsRemapInitSym_v1_t2, error); + plugin_v2->_plugin.getSymbol("TSRemapInit", tsRemapInitSym_v2_t2, error); + CHECK(nullptr != tsRemapInitSym_v1_t2); + CHECK(nullptr != tsRemapInitSym_v2_t2); + CHECK(tsRemapInitSym_v1_t2 == tsRemapInitSym_v2_t2); + + /* check that the symbol we got originally is still valid */ + CHECK(tsRemapInitSym_v1_t1 == tsRemapInitSym_v1_t2); + } + + teardownConfigPathTest(factory2); + enablePluginDynamicReload(); + } + } + + GIVEN("two different versions v1 and v2 of same plugin in mixed mode - dynamic plugin reload ENABLED (negative test)") + { + WHEN("(1) loading v1, (2) overwriting with v2 and then (3) reloading by using the same plugin name, " + "(*) v1 and v2 DSOs modification time are different (changed)") + { + enablePluginDynamicReload(); + /* Simulate installing plugin plugin_v1.so (ver 1) as plugin.so and loading it at some point of time t1 */ + setupConfigPathTest(configName, buildPath_v1, uuid_t1, effectivePath_v1, runtimePath_v1, 1556825556); + GlobalPluginInfo global_plugin_v1; + CHECK(global_plugin_v1.loadDso(effectivePath_v1) == true); + global_plugin_v1.getSymbol("TSRemapInit", tsRemapInitSym_v1_t1, error); + + /* Simulate installing plugin plugin_v2.so (v1) as plugin.so and loading it at some point of time t2 */ + /* Note that during the installation plugin_v2.so (v2) is "barberically" overriding the existing plugin.so which was v1 */ + auto [plugin_v2, factory2] = + testSetupLoadPlugin(configName, buildPath_v2, uuid_t2, 1556825557, effectivePath_v2, runtimePath_v2); + + CHECK(effectivePath_v1 == effectivePath_v2); + /* since dynamic reload is enabled runtime paths would be different */ + CHECK(runtimePath_v1 != runtimePath_v2); + + THEN("expect both to be successfully loaded and used simultaneously") + { + validateSuccessfulConfigPathTest(plugin_v2, error2, effectivePath_v2, runtimePath_v2); + + /* Make sure what we installed and loaded first was v1 and after the plugin reload we run v2 */ + CHECK(1 == global_plugin_v1.getPluginVersion()); + CHECK(2 == getPluginVersion(plugin_v2->_plugin)); + CHECK(global_plugin_v1.dlOpenHandle() != plugin_v2->_plugin.dlOpenHandle()); + + /* Make sure the symbols we get from the 2 loaded plugins don't yield the same callback function pointer */ + global_plugin_v1.getSymbol("TSRemapInit", tsRemapInitSym_v1_t2, error); + plugin_v2->_plugin.getSymbol("TSRemapInit", tsRemapInitSym_v2_t2, error); + CHECK(nullptr != tsRemapInitSym_v1_t2); + CHECK(nullptr != tsRemapInitSym_v2_t2); + CHECK(tsRemapInitSym_v1_t2 != tsRemapInitSym_v2_t2); + + /* Make sure v1 callback functions addresses did not change for v1 after v2 was loaded */ + CHECK(tsRemapInitSym_v1_t1 == tsRemapInitSym_v1_t2); + } + + teardownConfigPathTest(factory2); + enablePluginDynamicReload(); + } + } +} + SCENARIO("notifying plugins of config reload", "[plugin][core]") { REQUIRE_FALSE(sandboxDir.empty()); + enablePluginDynamicReload(); /* use 2 copies of the same plugin to test */ fs::path configName1 = fs::path("plugin_testing_calls_1.so"); @@ -554,29 +1197,55 @@ SCENARIO("notifying plugins of config reload", "[plugin][core]") GIVEN("simple configuration with 1 plugin and 1 factory") { - WHEN("indicating config reload") + WHEN("(1) signal pre-new-config-load, (2) signal post-new-config-load, (3) old-config deactivate") { - /* Simulate configuration without plugins - an unused factory */ + /* Simulate configuration with 1 factory and 1 plugin */ setupConfigPathTest(configName1, buildPath, uuid_t1, effectivePath1, runtimePath1, 1556825556); PluginFactoryUnitTest *factory1 = getFactory(uuid_t1); - RemapPluginInst *plugin1 = factory1->getRemapPlugin(configName1, 0, nullptr, error); + RemapPluginInst *plugin1 = factory1->getRemapPlugin(configName1, 0, nullptr, error, isPluginDynamicReloadEnabled()); /* check if loaded successfully */ validateSuccessfulConfigPathTest(plugin1, error, effectivePath1, runtimePath1); /* Prapare the debug object */ PluginDebugObject *debugObject = getDebugObject(plugin1->_plugin); - debugObject->clear(); - THEN("expect 'done' methods to be called for plugin and the instance but not the 'reload config' methods") + THEN("expect pre-, post- and done/delete_instance to be called accordingly ") { - /* Simulate reloading the config */ - factory1->indicateReload(); - - /* was "done" method called? */ - CHECK(1 == debugObject->doneCalled); + /* Signal before loading the config */ + debugObject->clear(); + + factory1->indicatePreReload(); + CHECK(0 == debugObject->deleteInstanceCalled); + CHECK(0 == debugObject->doneCalled); + CHECK(1 == debugObject->preReloadConfigCalled); + + /* ... parse the new remap config ... */ + + /* Assume (re)load was done OK and test */ + debugObject->clear(); + factory1->indicatePostReload(/* reload succeeded */ true); + CHECK(0 == debugObject->deleteInstanceCalled); + CHECK(0 == debugObject->doneCalled); + CHECK(1 == debugObject->postReloadConfigCalled); + CHECK(TSREMAP_CONFIG_RELOAD_SUCCESS_PLUGIN_USED == debugObject->postReloadConfigStatus); + + /* Assume (re)load failed and test */ + debugObject->clear(); + factory1->indicatePostReload(/* reload succeeded */ false); + CHECK(0 == debugObject->deleteInstanceCalled); + CHECK(0 == debugObject->doneCalled); + CHECK(1 == debugObject->postReloadConfigCalled); + CHECK(TSREMAP_CONFIG_RELOAD_FAILURE == debugObject->postReloadConfigStatus); + + /* ... swap the new and the old config ... */ + + /* Signal de-activation of the old config */ + debugObject->clear(); + factory1->deactivate(); CHECK(1 == debugObject->deleteInstanceCalled); - CHECK(0 == debugObject->reloadConfigCalled); + CHECK(1 == debugObject->doneCalled); + CHECK(0 == debugObject->preReloadConfigCalled); } teardownConfigPathTest(factory1); @@ -585,14 +1254,14 @@ SCENARIO("notifying plugins of config reload", "[plugin][core]") GIVEN("configuration with 2 plugins loaded by 1 factory") { - WHEN("indicating config reload") + WHEN("(1) signal pre-new-config-load, (2) signal post-new-config-load, (3) old-config deactivate") { - /* Simulate configuration without plugins - an unused factory */ + /* Simulate configuration with 1 factories and 2 plugin */ setupConfigPathTest(configName1, buildPath, uuid_t1, effectivePath1, runtimePath1, 1556825556); setupConfigPathTest(configName2, buildPath, uuid_t1, effectivePath2, runtimePath2, 1556825556, /* append */ true); PluginFactoryUnitTest *factory1 = getFactory(uuid_t1); - RemapPluginInst *plugin1 = factory1->getRemapPlugin(configName1, 0, nullptr, error); - RemapPluginInst *plugin2 = factory1->getRemapPlugin(configName2, 0, nullptr, error); + RemapPluginInst *plugin1 = factory1->getRemapPlugin(configName1, 0, nullptr, error, isPluginDynamicReloadEnabled()); + RemapPluginInst *plugin2 = factory1->getRemapPlugin(configName2, 0, nullptr, error, isPluginDynamicReloadEnabled()); /* check if loaded successfully */ validateSuccessfulConfigPathTest(plugin1, error, effectivePath1, runtimePath1); @@ -601,21 +1270,60 @@ SCENARIO("notifying plugins of config reload", "[plugin][core]") /* Prapare the debug objects */ PluginDebugObject *debugObject1 = getDebugObject(plugin1->_plugin); PluginDebugObject *debugObject2 = getDebugObject(plugin2->_plugin); - debugObject1->clear(); - debugObject2->clear(); - THEN("expect 'done' methods to be called but not the 'reload config' methods") + THEN("expect pre-, post- and done/delete_instance to be called accordingly") { - /* Simulate reloading the config */ - factory1->indicateReload(); + /* Signal before loading the config */ + debugObject1->clear(); + debugObject2->clear(); + factory1->indicatePreReload(); + CHECK(0 == debugObject1->deleteInstanceCalled); + CHECK(0 == debugObject1->doneCalled); + CHECK(1 == debugObject1->preReloadConfigCalled); + CHECK(0 == debugObject2->doneCalled); + CHECK(0 == debugObject2->deleteInstanceCalled); + CHECK(1 == debugObject2->preReloadConfigCalled); - /* Was "done" method called? */ - CHECK(1 == debugObject1->doneCalled); + /* ... parse the new remap config ... */ + + /* Assume (re)load was done OK */ + debugObject1->clear(); + debugObject2->clear(); + factory1->indicatePostReload(/* reload succeeded */ true); + CHECK(0 == debugObject1->deleteInstanceCalled); + CHECK(0 == debugObject1->doneCalled); + CHECK(1 == debugObject1->postReloadConfigCalled); + CHECK(TSREMAP_CONFIG_RELOAD_SUCCESS_PLUGIN_USED == debugObject1->postReloadConfigStatus); + CHECK(0 == debugObject2->deleteInstanceCalled); + CHECK(0 == debugObject2->doneCalled); + CHECK(1 == debugObject2->postReloadConfigCalled); + CHECK(TSREMAP_CONFIG_RELOAD_SUCCESS_PLUGIN_USED == debugObject2->postReloadConfigStatus); + + /* Assume (re)load failed */ + debugObject1->clear(); + debugObject2->clear(); + factory1->indicatePostReload(/* reload succeeded */ false); + CHECK(0 == debugObject1->deleteInstanceCalled); + CHECK(0 == debugObject1->doneCalled); + CHECK(1 == debugObject1->postReloadConfigCalled); + CHECK(TSREMAP_CONFIG_RELOAD_FAILURE == debugObject1->postReloadConfigStatus); + CHECK(0 == debugObject2->deleteInstanceCalled); + CHECK(0 == debugObject2->doneCalled); + CHECK(1 == debugObject2->postReloadConfigCalled); + CHECK(TSREMAP_CONFIG_RELOAD_FAILURE == debugObject2->postReloadConfigStatus); + + /* ... swap the new and the old config ... */ + + /* Signal de-activation of the old config */ + debugObject1->clear(); + debugObject2->clear(); + factory1->deactivate(); CHECK(1 == debugObject1->deleteInstanceCalled); - CHECK(0 == debugObject1->reloadConfigCalled); - CHECK(1 == debugObject2->doneCalled); + CHECK(1 == debugObject1->doneCalled); + CHECK(0 == debugObject1->preReloadConfigCalled); CHECK(1 == debugObject2->deleteInstanceCalled); - CHECK(0 == debugObject2->reloadConfigCalled); + CHECK(1 == debugObject2->doneCalled); + CHECK(0 == debugObject2->preReloadConfigCalled); } teardownConfigPathTest(factory1); @@ -624,36 +1332,95 @@ SCENARIO("notifying plugins of config reload", "[plugin][core]") GIVEN("configuration with 1 plugin loaded by 2 separate factories") { - WHEN("indicating config reload") + WHEN("indicating de-activation of the factories") { - /* Simulate configuration without plugins - an unused factory */ + /* Simulate configuration with 2 factories and 1 plugin */ setupConfigPathTest(configName1, buildPath, uuid_t1, effectivePath1, runtimePath1, 1556825556); PluginFactoryUnitTest *factory1 = getFactory(uuid_t1); PluginFactoryUnitTest *factory2 = getFactory(uuid_t2); - RemapPluginInst *plugin1 = factory1->getRemapPlugin(configName1, 0, nullptr, error); - RemapPluginInst *plugin2 = factory2->getRemapPlugin(configName1, 0, nullptr, error); + RemapPluginInst *plugin1 = factory1->getRemapPlugin(configName1, 0, nullptr, error, isPluginDynamicReloadEnabled()); + RemapPluginInst *plugin2 = factory2->getRemapPlugin(configName1, 0, nullptr, error, isPluginDynamicReloadEnabled()); - /* Prapare the debug objects */ + /* Prepare the debug objects */ PluginDebugObject *debugObject1 = getDebugObject(plugin1->_plugin); PluginDebugObject *debugObject2 = getDebugObject(plugin2->_plugin); - THEN("expect instance 'done' to be always called, but plugin 'done' called only after destroying one factory") + THEN("expect instance 'done' to be always called, but plugin 'done' called only after destroying both factories") { debugObject2->clear(); - factory2->indicateReload(); + factory2->deactivate(); CHECK(0 == debugObject2->doneCalled); CHECK(1 == debugObject2->deleteInstanceCalled); - CHECK(1 == debugObject2->reloadConfigCalled); + CHECK(0 == debugObject2->preReloadConfigCalled); delete factory2; debugObject1->clear(); - factory1->indicateReload(); + factory1->deactivate(); CHECK(1 == debugObject1->doneCalled); CHECK(1 == debugObject1->deleteInstanceCalled); - CHECK(0 == debugObject1->reloadConfigCalled); + CHECK(0 == debugObject1->preReloadConfigCalled); + + delete factory1; + } + + clean(); + } + } + + GIVEN("configuration with 2 plugin loaded by 2 factories") + { + WHEN("2 plugins are loaded by the 1st factory and only one of them loaded by the 2nd factory") + { + /* Simulate configuration with 2 factories and 2 different plugins */ + setupConfigPathTest(configName1, buildPath, uuid_t1, effectivePath1, runtimePath1, 1556825556, /* append */ false); + setupConfigPathTest(configName2, buildPath, uuid_t1, effectivePath2, runtimePath2, 1556825556, /* append */ true); + PluginFactoryUnitTest *factory1 = getFactory(uuid_t1); + PluginFactoryUnitTest *factory2 = getFactory(uuid_t2); + + /* 2 plugins loaded by the 1st factory */ + RemapPluginInst *pluginInst1 = factory1->getRemapPlugin(configName1, 0, nullptr, error, isPluginDynamicReloadEnabled()); + RemapPluginInst *pluginInst2 = factory1->getRemapPlugin(configName2, 0, nullptr, error, isPluginDynamicReloadEnabled()); + + /* only 1 plugin loaded by the 2st factory */ + RemapPluginInst *pluginInst3 = factory2->getRemapPlugin(configName1, 0, nullptr, error, isPluginDynamicReloadEnabled()); + + /* pluginInst1 and pluginInst3 should be using the same plugin DSO named configName1 + * pluginInst2 should be using plugin DSO named configName 2*/ + CHECK_FALSE(nullptr == pluginInst1); + CHECK_FALSE(nullptr == pluginInst2); + CHECK_FALSE(nullptr == pluginInst3); + CHECK(&pluginInst1->_plugin == &pluginInst3->_plugin); + + /* Get test objects for the 2 plugins used by the 3 instances from the 2 factories */ + PluginDebugObject *debugObject1 = getDebugObject(pluginInst1->_plugin); + PluginDebugObject *debugObject2 = getDebugObject(pluginInst2->_plugin); + + THEN( + "expect the plugin that is not loaded by the second factory to receive 'plugin unused' notification from the 2nd factory") + { + /* Factory 1: reload was OK and both plugins were part of the configuration that used/instantiated that factory */ + debugObject1->clear(); + debugObject2->clear(); + + factory1->indicatePostReload(/* reload succeeded */ true); + CHECK(1 == debugObject1->postReloadConfigCalled); + CHECK(TSREMAP_CONFIG_RELOAD_SUCCESS_PLUGIN_USED == debugObject1->postReloadConfigStatus); + CHECK(1 == debugObject2->postReloadConfigCalled); + CHECK(TSREMAP_CONFIG_RELOAD_SUCCESS_PLUGIN_USED == debugObject2->postReloadConfigStatus); + + /* Factory 2: (re)load was OK and only 1 plugin was part of the configuration that used/instantiated that factory */ + debugObject1->clear(); + debugObject2->clear(); + + factory2->indicatePostReload(/* reload succeeded */ true); + CHECK(1 == debugObject1->postReloadConfigCalled); + CHECK(TSREMAP_CONFIG_RELOAD_SUCCESS_PLUGIN_USED == debugObject1->postReloadConfigStatus); + CHECK(1 == debugObject2->postReloadConfigCalled); + CHECK(TSREMAP_CONFIG_RELOAD_SUCCESS_PLUGIN_UNUSED == debugObject2->postReloadConfigStatus); delete factory1; + delete factory2; } clean(); diff --git a/proxy/http/remap/unit-tests/test_RemapPlugin.cc b/proxy/http/remap/unit-tests/test_RemapPlugin.cc index 1b385cc51da..0184ab214b2 100644 --- a/proxy/http/remap/unit-tests/test_RemapPlugin.cc +++ b/proxy/http/remap/unit-tests/test_RemapPlugin.cc @@ -422,13 +422,51 @@ SCENARIO("config reload", "[plugin][core]") bool result = loadPlugin(plugin, error, debugObject); CHECK(true == result); - WHEN("'config reload' is called") + WHEN("'config reload' failed") { debugObject->clear(); - plugin->indicateReload(); + plugin->indicatePreReload(); + plugin->indicatePostReload(TSREMAP_CONFIG_RELOAD_FAILURE); - THEN("expect it to run") { CHECK(1 == debugObject->reloadConfigCalled); } + THEN("expect it to run") + { + CHECK(1 == debugObject->preReloadConfigCalled); + CHECK(1 == debugObject->postReloadConfigCalled); + CHECK(TSREMAP_CONFIG_RELOAD_FAILURE == debugObject->postReloadConfigStatus); + } + cleanupSandBox(plugin); + } + + WHEN("'config reload' is successful and the plugin is part of the new configuration") + { + debugObject->clear(); + + plugin->indicatePreReload(); + plugin->indicatePostReload(TSREMAP_CONFIG_RELOAD_SUCCESS_PLUGIN_USED); + + THEN("expect it to run") + { + CHECK(1 == debugObject->preReloadConfigCalled); + CHECK(1 == debugObject->postReloadConfigCalled); + CHECK(TSREMAP_CONFIG_RELOAD_SUCCESS_PLUGIN_USED == debugObject->postReloadConfigStatus); + } + cleanupSandBox(plugin); + } + + WHEN("'config reload' is successful and the plugin is part of the new configuration") + { + debugObject->clear(); + + plugin->indicatePreReload(); + plugin->indicatePostReload(TSREMAP_CONFIG_RELOAD_SUCCESS_PLUGIN_UNUSED); + + THEN("expect it to run") + { + CHECK(1 == debugObject->preReloadConfigCalled); + CHECK(1 == debugObject->postReloadConfigCalled); + CHECK(TSREMAP_CONFIG_RELOAD_SUCCESS_PLUGIN_UNUSED == debugObject->postReloadConfigStatus); + } cleanupSandBox(plugin); } } diff --git a/proxy/http/unit_tests/unit_test_main.cc b/proxy/http/unit_tests/unit_test_main.cc index 6aed3a63097..6217bdafd22 100644 --- a/proxy/http/unit_tests/unit_test_main.cc +++ b/proxy/http/unit_tests/unit_test_main.cc @@ -22,4 +22,9 @@ */ #define CATCH_CONFIG_MAIN + #include "catch.hpp" + +#include "tscore/I_Version.h" + +AppVersionInfo appVersionInfo; diff --git a/proxy/http2/HPACK.cc b/proxy/http2/HPACK.cc index 7fa62ccb219..2680600223b 100644 --- a/proxy/http2/HPACK.cc +++ b/proxy/http2/HPACK.cc @@ -166,6 +166,21 @@ static const StaticTable STATIC_TABLE[] = {{"", ""}, {"via", ""}, {"www-authenticate", ""}}; +/** + Threshold for total HdrHeap size which used by HPAK Dynamic Table. + The HdrHeap is filled by MIMEHdrImpl and MIMEFieldBlockImpl like below. + This threshold allow to allocate 3 HdrHeap at maximum. + + +------------------+-----------------------------+ + HdrHeap 1 (2048): | MIMEHdrImpl(592) | MIMEFieldBlockImpl(528) x 2 | + +------------------+-----------------------------+--...--+ + HdrHeap 2 (4096): | MIMEFieldBlockImpl(528) x 7 | + +------------------------------------------------+--...--+--...--+ + HdrHeap 3 (8192): | MIMEFieldBlockImpl(528) x 15 | + +------------------------------------------------+--...--+--...--+ +*/ +static constexpr uint32_t HPACK_HDR_HEAP_THRESHOLD = sizeof(MIMEHdrImpl) + sizeof(MIMEFieldBlockImpl) * (2 + 7 + 15); + /****************** * Local functions ******************/ @@ -312,16 +327,34 @@ HpackIndexingTable::size() const return _dynamic_table->size(); } -bool +void HpackIndexingTable::update_maximum_size(uint32_t new_size) { - return _dynamic_table->update_maximum_size(new_size); + _dynamic_table->update_maximum_size(new_size); +} + +// +// HpackDynamicTable +// +HpackDynamicTable::~HpackDynamicTable() +{ + this->_headers.clear(); + + this->_mhdr->fields_clear(); + this->_mhdr->destroy(); + delete this->_mhdr; + + if (this->_mhdr_old != nullptr) { + this->_mhdr_old->fields_clear(); + this->_mhdr_old->destroy(); + delete this->_mhdr_old; + } } const MIMEField * HpackDynamicTable::get_header_field(uint32_t index) const { - return this->_headers.at(this->_headers.size() - index - 1); + return this->_headers.at(index); } void @@ -339,6 +372,14 @@ HpackDynamicTable::add_header_field(const MIMEField *field) // table causes the table to be emptied of all existing entries. this->_headers.clear(); this->_mhdr->fields_clear(); + + if (this->_mhdr_old) { + this->_mhdr_old->fields_clear(); + this->_mhdr_old->destroy(); + delete this->_mhdr_old; + this->_mhdr_old = nullptr; + } + this->_current_size = 0; } else { this->_current_size += header_size; @@ -347,8 +388,7 @@ HpackDynamicTable::add_header_field(const MIMEField *field) MIMEField *new_field = this->_mhdr->field_create(name, name_len); new_field->value_set(this->_mhdr->m_heap, this->_mhdr->m_mime, value, value_len); this->_mhdr->field_attach(new_field); - // XXX Because entire Vec instance is copied, Its too expensive! - this->_headers.push_back(new_field); + this->_headers.push_front(new_field); } } @@ -371,49 +411,70 @@ HpackDynamicTable::size() const // are evicted from the end of the header table until the size of the // header table is less than or equal to the maximum size. // -bool +void HpackDynamicTable::update_maximum_size(uint32_t new_size) { this->_maximum_size = new_size; - return this->_evict_overflowed_entries(); + this->_evict_overflowed_entries(); } uint32_t HpackDynamicTable::length() const { - return _headers.size(); + return this->_headers.size(); } -bool +void HpackDynamicTable::_evict_overflowed_entries() { if (this->_current_size <= this->_maximum_size) { // Do nothing - return true; + return; } - size_t count = 0; - for (auto &h : this->_headers) { + for (auto h = this->_headers.rbegin(); h != this->_headers.rend(); ++h) { int name_len, value_len; - h->name_get(&name_len); - h->value_get(&value_len); + (*h)->name_get(&name_len); + (*h)->value_get(&value_len); this->_current_size -= ADDITIONAL_OCTETS + name_len + value_len; - this->_mhdr->field_delete(h, false); - ++count; + + if (this->_mhdr_old && this->_mhdr_old->fields_count() != 0) { + this->_mhdr_old->field_delete(*h, false); + } else { + this->_mhdr->field_delete(*h, false); + } + + this->_headers.pop_back(); if (this->_current_size <= this->_maximum_size) { break; } } - this->_headers.erase(this->_headers.begin(), this->_headers.begin() + count); + this->_mime_hdr_gc(); +} - if (this->_headers.size() == 0) { - return false; +/** + When HdrHeap size of current MIMEHdr exceeds the threshold, allocate new MIMEHdr and HdrHeap. + The old MIMEHdr and HdrHeap will be freed, when all MIMEFiled are deleted by HPACK Entry Eviction. + */ +void +HpackDynamicTable::_mime_hdr_gc() +{ + if (this->_mhdr_old == nullptr) { + if (this->_mhdr->m_heap->total_used_size() >= HPACK_HDR_HEAP_THRESHOLD) { + this->_mhdr_old = this->_mhdr; + this->_mhdr = new MIMEHdr(); + this->_mhdr->create(); + } + } else { + if (this->_mhdr_old->fields_count() == 0) { + this->_mhdr_old->destroy(); + delete this->_mhdr_old; + this->_mhdr_old = nullptr; + } } - - return true; } int64_t @@ -638,7 +699,9 @@ decode_literal_header_field(MIMEFieldWrapper &header, const uint8_t *buf_start, // Decode header field name if (index) { - indexing_table.get_header_field(index, header); + if (indexing_table.get_header_field(index, header) == HPACK_ERROR_COMPRESSION_ERROR) { + return HPACK_ERROR_COMPRESSION_ERROR; + } } else { char *name_str = nullptr; uint64_t name_str_len = 0; @@ -719,9 +782,7 @@ update_dynamic_table_size(const uint8_t *buf_start, const uint8_t *buf_end, Hpac return HPACK_ERROR_COMPRESSION_ERROR; } - if (indexing_table.update_maximum_size(size) == false) { - return HPACK_ERROR_COMPRESSION_ERROR; - } + indexing_table.update_maximum_size(size); return len; } @@ -786,7 +847,11 @@ hpack_decode_header_block(HpackIndexingTable &indexing_table, HTTPHdr *hdr, cons field->name_get(&name_len); field->value_get(&value_len); - total_header_size += name_len + value_len; + + // [RFC 7540] 6.5.2. SETTINGS_MAX_HEADER_LIST_SIZE: + // The value is based on the uncompressed size of header fields, including the length of the name and value in octets plus an + // overhead of 32 octets for each header field. + total_header_size += name_len + value_len + ADDITIONAL_OCTETS; if (total_header_size > max_header_size) { return HPACK_ERROR_SIZE_EXCEEDED_ERROR; diff --git a/proxy/http2/HPACK.h b/proxy/http2/HPACK.h index 16c8e015010..70f95eddf92 100644 --- a/proxy/http2/HPACK.h +++ b/proxy/http2/HPACK.h @@ -28,7 +28,7 @@ #include "HTTP.h" #include "../hdrs/XPACK.h" -#include +#include // It means that any header field can be compressed/decompressed by ATS const static int HPACK_ERROR_COMPRESSION_ERROR = -1; @@ -112,13 +112,7 @@ class HpackDynamicTable _mhdr->create(); } - ~HpackDynamicTable() - { - _headers.clear(); - _mhdr->fields_clear(); - _mhdr->destroy(); - delete _mhdr; - } + ~HpackDynamicTable(); // noncopyable HpackDynamicTable(HpackDynamicTable &) = delete; @@ -129,18 +123,20 @@ class HpackDynamicTable uint32_t maximum_size() const; uint32_t size() const; - bool update_maximum_size(uint32_t new_size); + void update_maximum_size(uint32_t new_size); uint32_t length() const; private: - bool _evict_overflowed_entries(); + void _evict_overflowed_entries(); + void _mime_hdr_gc(); - uint32_t _current_size; - uint32_t _maximum_size; + uint32_t _current_size = 0; + uint32_t _maximum_size = 0; - MIMEHdr *_mhdr; - std::vector _headers; + MIMEHdr *_mhdr = nullptr; + MIMEHdr *_mhdr_old = nullptr; + std::deque _headers; }; // [RFC 7541] 2.3. Indexing Table @@ -161,7 +157,7 @@ class HpackIndexingTable void add_header_field(const MIMEField *field); uint32_t maximum_size() const; uint32_t size() const; - bool update_maximum_size(uint32_t new_size); + void update_maximum_size(uint32_t new_size); private: HpackDynamicTable *_dynamic_table; diff --git a/proxy/http2/HTTP2.cc b/proxy/http2/HTTP2.cc index 81f741d6459..bb4b4df8f79 100644 --- a/proxy/http2/HTTP2.cc +++ b/proxy/http2/HTTP2.cc @@ -42,8 +42,18 @@ const unsigned HTTP2_LEN_AUTHORITY = countof(":authority") - 1; const unsigned HTTP2_LEN_PATH = countof(":path") - 1; const unsigned HTTP2_LEN_STATUS = countof(":status") - 1; -static size_t HTTP2_LEN_STATUS_VALUE_STR = 3; -static const int HTTP2_MAX_TABLE_SIZE_LIMIT = 64 * 1024; +static size_t HTTP2_LEN_STATUS_VALUE_STR = 3; +static const uint32_t HTTP2_MAX_TABLE_SIZE_LIMIT = 64 * 1024; + +namespace +{ +struct Http2HeaderName { + const char *name = nullptr; + int name_len = 0; +}; + +Http2HeaderName http2_connection_specific_headers[5] = {}; +} // namespace // Statistics RecRawStatBlock *http2_rsb; @@ -62,6 +72,15 @@ static const char *const HTTP2_STAT_SESSION_DIE_INACTIVE_NAME = "pro static const char *const HTTP2_STAT_SESSION_DIE_EOS_NAME = "proxy.process.http2.session_die_eos"; static const char *const HTTP2_STAT_SESSION_DIE_ERROR_NAME = "proxy.process.http2.session_die_error"; static const char *const HTTP2_STAT_SESSION_DIE_HIGH_ERROR_RATE_NAME = "proxy.process.http2.session_die_high_error_rate"; +static const char *const HTTP2_STAT_MAX_SETTINGS_PER_FRAME_EXCEEDED_NAME = "proxy.process.http2.max_settings_per_frame_exceeded"; +static const char *const HTTP2_STAT_MAX_SETTINGS_PER_MINUTE_EXCEEDED_NAME = "proxy.process.http2.max_settings_per_minute_exceeded"; +static const char *const HTTP2_STAT_MAX_SETTINGS_FRAMES_PER_MINUTE_EXCEEDED_NAME = + "proxy.process.http2.max_settings_frames_per_minute_exceeded"; +static const char *const HTTP2_STAT_MAX_PING_FRAMES_PER_MINUTE_EXCEEDED_NAME = + "proxy.process.http2.max_ping_frames_per_minute_exceeded"; +static const char *const HTTP2_STAT_MAX_PRIORITY_FRAMES_PER_MINUTE_EXCEEDED_NAME = + "proxy.process.http2.max_priority_frames_per_minute_exceeded"; +static const char *const HTTP2_STAT_INSUFFICIENT_AVG_WINDOW_UPDATE_NAME = "proxy.process.http2.insufficient_avg_window_update"; union byte_pointer { byte_pointer(void *p) : ptr(p) {} @@ -234,24 +253,6 @@ http2_write_frame_header(const Http2FrameHeader &hdr, IOVec iov) return true; } -bool -http2_write_data(const uint8_t *src, size_t length, const IOVec &iov) -{ - byte_pointer ptr(iov.iov_base); - write_and_advance(ptr, src, length); - - return true; -} - -bool -http2_write_headers(const uint8_t *src, size_t length, const IOVec &iov) -{ - byte_pointer ptr(iov.iov_base); - write_and_advance(ptr, src, length); - - return true; -} - bool http2_write_rst_stream(uint32_t error_code, IOVec iov) { @@ -510,99 +511,160 @@ http2_convert_header_from_2_to_1_1(HTTPHdr *headers) return PARSE_RESULT_DONE; } +/** + Initialize HTTPHdr for HTTP/2 + + Reserve HTTP/2 Pseudo-Header Fields in front of HTTPHdr. Value of these header fields will be set by + `http2_convert_header_from_1_1_to_2()`. When a HTTPHdr for HTTP/2 headers is created, this should be called immediately. + Because all pseudo-header fields MUST appear in the header block before regular header fields. + */ void -http2_generate_h2_header_from_1_1(HTTPHdr *headers, HTTPHdr *h2_headers) +http2_init_pseudo_headers(HTTPHdr &hdr) { - h2_headers->create(http_hdr_type_get(headers->m_http)); + switch (http_hdr_type_get(hdr.m_http)) { + case HTTP_TYPE_REQUEST: { + MIMEField *method = hdr.field_create(HTTP2_VALUE_METHOD, HTTP2_LEN_METHOD); + hdr.field_attach(method); + + MIMEField *scheme = hdr.field_create(HTTP2_VALUE_SCHEME, HTTP2_LEN_SCHEME); + hdr.field_attach(scheme); - if (http_hdr_type_get(headers->m_http) == HTTP_TYPE_RESPONSE) { - // Add ':status' header field - char status_str[HTTP2_LEN_STATUS_VALUE_STR + 1]; - snprintf(status_str, sizeof(status_str), "%d", headers->status_get()); - MIMEField *status_field = h2_headers->field_create(HTTP2_VALUE_STATUS, HTTP2_LEN_STATUS); - status_field->value_set(h2_headers->m_heap, h2_headers->m_mime, status_str, HTTP2_LEN_STATUS_VALUE_STR); - h2_headers->field_attach(status_field); + MIMEField *authority = hdr.field_create(HTTP2_VALUE_AUTHORITY, HTTP2_LEN_AUTHORITY); + hdr.field_attach(authority); - } else if (http_hdr_type_get(headers->m_http) == HTTP_TYPE_REQUEST) { - MIMEField *field; - const char *value; - int value_len; + MIMEField *path = hdr.field_create(HTTP2_VALUE_PATH, HTTP2_LEN_PATH); + hdr.field_attach(path); + + break; + } + case HTTP_TYPE_RESPONSE: { + MIMEField *status = hdr.field_create(HTTP2_VALUE_STATUS, HTTP2_LEN_STATUS); + hdr.field_attach(status); + + break; + } + default: + ink_abort("HTTP_TYPE_UNKNOWN"); + } +} + +/** + Convert HTTP/1.1 HTTPHdr to HTTP/2 + + Assuming HTTP/2 Pseudo-Header Fields are reserved by `http2_init_pseudo_headers()`. + */ +ParseResult +http2_convert_header_from_1_1_to_2(HTTPHdr *headers) +{ + switch (http_hdr_type_get(headers->m_http)) { + case HTTP_TYPE_REQUEST: { + // :method + if (MIMEField *field = headers->field_find(HTTP2_VALUE_METHOD, HTTP2_LEN_METHOD); field != nullptr) { + int value_len; + const char *value = headers->method_get(&value_len); + + field->value_set(headers->m_heap, headers->m_mime, value, value_len); + } else { + ink_abort("initialize HTTP/2 pseudo-headers"); + return PARSE_RESULT_ERROR; + } + + // :scheme + if (MIMEField *field = headers->field_find(HTTP2_VALUE_SCHEME, HTTP2_LEN_SCHEME); field != nullptr) { + int value_len; + const char *value = headers->scheme_get(&value_len); + + if (value != nullptr) { + field->value_set(headers->m_heap, headers->m_mime, value, value_len); + } else { + field->value_set(headers->m_heap, headers->m_mime, URL_SCHEME_HTTPS, URL_LEN_HTTPS); + } + } else { + ink_abort("initialize HTTP/2 pseudo-headers"); + return PARSE_RESULT_ERROR; + } + + // :authority + if (MIMEField *field = headers->field_find(HTTP2_VALUE_AUTHORITY, HTTP2_LEN_AUTHORITY); field != nullptr) { + int value_len; + const char *value = headers->host_get(&value_len); + + if (headers->is_port_in_header()) { + int port = headers->port_get(); + char *host_and_port = static_cast(ats_malloc(value_len + 8)); + value_len = snprintf(host_and_port, value_len + 8, "%.*s:%d", value_len, value, port); + + field->value_set(headers->m_heap, headers->m_mime, host_and_port, value_len); + ats_free(host_and_port); + } else { + field->value_set(headers->m_heap, headers->m_mime, value, value_len); + } + } else { + ink_abort("initialize HTTP/2 pseudo-headers"); + return PARSE_RESULT_ERROR; + } + + // :path + if (MIMEField *field = headers->field_find(HTTP2_VALUE_PATH, HTTP2_LEN_PATH); field != nullptr) { + int value_len; + const char *value = headers->path_get(&value_len); + char *path = static_cast(ats_malloc(value_len + 1)); + path[0] = '/'; + memcpy(path + 1, value, value_len); + + field->value_set(headers->m_heap, headers->m_mime, path, value_len + 1); + ats_free(path); + } else { + ink_abort("initialize HTTP/2 pseudo-headers"); + return PARSE_RESULT_ERROR; + } - // Add ':authority' header field // TODO: remove host/Host header - // [RFC 7540] 8.1.2.3. Clients that generate HTTP/2 requests directly SHOULD use the ":authority" pseudo-header field instead of - // the Host header field. - field = h2_headers->field_create(HTTP2_VALUE_AUTHORITY, HTTP2_LEN_AUTHORITY); - value = headers->host_get(&value_len); - if (headers->is_port_in_header()) { - int port = headers->port_get(); - char *host_and_port = static_cast(ats_malloc(value_len + 8)); - value_len = snprintf(host_and_port, value_len + 8, "%.*s:%d", value_len, value, port); - field->value_set(h2_headers->m_heap, h2_headers->m_mime, host_and_port, value_len); - ats_free(host_and_port); + // [RFC 7540] 8.1.2.3. Clients that generate HTTP/2 requests directly SHOULD use the ":authority" pseudo-header field instead + // of the Host header field. + + break; + } + case HTTP_TYPE_RESPONSE: { + // :status + if (MIMEField *field = headers->field_find(HTTP2_VALUE_STATUS, HTTP2_LEN_STATUS); field != nullptr) { + // ink_small_itoa() requires 5+ buffer length + char status_str[HTTP2_LEN_STATUS_VALUE_STR + 3]; + mime_format_int(status_str, headers->status_get(), sizeof(status_str)); + + field->value_set(headers->m_heap, headers->m_mime, status_str, HTTP2_LEN_STATUS_VALUE_STR); } else { - field->value_set(h2_headers->m_heap, h2_headers->m_mime, value, value_len); + ink_abort("initialize HTTP/2 pseudo-headers"); + return PARSE_RESULT_ERROR; } - h2_headers->field_attach(field); - - // Add ':method' header field - field = h2_headers->field_create(HTTP2_VALUE_METHOD, HTTP2_LEN_METHOD); - value = headers->method_get(&value_len); - field->value_set(h2_headers->m_heap, h2_headers->m_mime, value, value_len); - h2_headers->field_attach(field); - - // Add ':path' header field - field = h2_headers->field_create(HTTP2_VALUE_PATH, HTTP2_LEN_PATH); - value = headers->path_get(&value_len); - char *path = static_cast(ats_malloc(value_len + 1)); - path[0] = '/'; - memcpy(path + 1, value, value_len); - field->value_set(h2_headers->m_heap, h2_headers->m_mime, path, value_len + 1); - ats_free(path); - h2_headers->field_attach(field); - - // Add ':scheme' header field - field = h2_headers->field_create(HTTP2_VALUE_SCHEME, HTTP2_LEN_SCHEME); - value = headers->scheme_get(&value_len); - field->value_set(h2_headers->m_heap, h2_headers->m_mime, value, value_len); - h2_headers->field_attach(field); - } - - // Copy headers + break; + } + default: + ink_abort("HTTP_TYPE_UNKNOWN"); + } + // Intermediaries SHOULD remove connection-specific header fields. - MIMEFieldIter field_iter; - for (MIMEField *field = headers->iter_get_first(&field_iter); field != nullptr; field = headers->iter_get_next(&field_iter)) { - const char *name; - int name_len; - const char *value; - int value_len; - name = field->name_get(&name_len); - if ((name_len == MIME_LEN_CONNECTION && strncasecmp(name, MIME_FIELD_CONNECTION, name_len) == 0) || - (name_len == MIME_LEN_KEEP_ALIVE && strncasecmp(name, MIME_FIELD_KEEP_ALIVE, name_len) == 0) || - (name_len == MIME_LEN_PROXY_CONNECTION && strncasecmp(name, MIME_FIELD_PROXY_CONNECTION, name_len) == 0) || - (name_len == MIME_LEN_TRANSFER_ENCODING && strncasecmp(name, MIME_FIELD_TRANSFER_ENCODING, name_len) == 0) || - (name_len == MIME_LEN_UPGRADE && strncasecmp(name, MIME_FIELD_UPGRADE, name_len) == 0)) { - continue; + for (auto &h : http2_connection_specific_headers) { + if (MIMEField *field = headers->field_find(h.name, h.name_len); field != nullptr) { + headers->field_delete(field); } - MIMEField *newfield; - name = field->name_get(&name_len); - newfield = h2_headers->field_create(name, name_len); - value = field->value_get(&value_len); - newfield->value_set(h2_headers->m_heap, h2_headers->m_mime, value, value_len); - h2_headers->field_attach(newfield); } + + return PARSE_RESULT_DONE; } Http2ErrorCode http2_encode_header_blocks(HTTPHdr *in, uint8_t *out, uint32_t out_len, uint32_t *len_written, HpackHandle &handle, int32_t maximum_table_size) { - // Limit the maximum table size to 64kB, which is the size advertised by major clients - maximum_table_size = std::min(maximum_table_size, HTTP2_MAX_TABLE_SIZE_LIMIT); + // Limit the maximum table size to the configured value or 64kB at maximum, which is the size advertised by major clients + maximum_table_size = + std::min(maximum_table_size, static_cast(std::min(Http2::header_table_size_limit, HTTP2_MAX_TABLE_SIZE_LIMIT))); // Set maximum table size only if it is different from current maximum size if (maximum_table_size == hpack_get_maximum_table_size(handle)) { maximum_table_size = -1; } + // TODO: It would be better to split Cookie header value int64_t result = hpack_encode_header_block(handle, out, out_len, in, maximum_table_size); if (result < 0) { @@ -726,7 +788,7 @@ uint32_t Http2::min_concurrent_streams_in = 10; uint32_t Http2::max_active_streams_in = 0; bool Http2::throttling = false; uint32_t Http2::stream_priority_enabled = 0; -uint32_t Http2::initial_window_size = 1048576; +uint32_t Http2::initial_window_size = 65535; uint32_t Http2::max_frame_size = 16384; uint32_t Http2::header_table_size = 4096; uint32_t Http2::max_header_list_size = 4294967295; @@ -744,6 +806,7 @@ uint32_t Http2::max_priority_frames_per_minute = 120; float Http2::min_avg_window_update = 2560.0; uint32_t Http2::con_slow_log_threshold = 0; uint32_t Http2::stream_slow_log_threshold = 0; +uint32_t Http2::header_table_size_limit = 65536; void Http2::init() @@ -770,6 +833,7 @@ Http2::init() REC_EstablishStaticConfigFloat(min_avg_window_update, "proxy.config.http2.min_avg_window_update"); REC_EstablishStaticConfigInt32U(con_slow_log_threshold, "proxy.config.http2.connection.slow.log.threshold"); REC_EstablishStaticConfigInt32U(stream_slow_log_threshold, "proxy.config.http2.stream.slow.log.threshold"); + REC_EstablishStaticConfigInt32U(header_table_size_limit, "proxy.config.http2.header_table_size_limit"); // If any settings is broken, ATS should not start ink_release_assert(http2_settings_parameter_is_valid({HTTP2_SETTINGS_MAX_CONCURRENT_STREAMS, max_concurrent_streams_in})); @@ -820,6 +884,39 @@ Http2::init() static_cast(HTTP2_STAT_SESSION_DIE_ERROR), RecRawStatSyncSum); RecRegisterRawStat(http2_rsb, RECT_PROCESS, HTTP2_STAT_SESSION_DIE_HIGH_ERROR_RATE_NAME, RECD_INT, RECP_PERSISTENT, static_cast(HTTP2_STAT_SESSION_DIE_HIGH_ERROR_RATE), RecRawStatSyncSum); + RecRegisterRawStat(http2_rsb, RECT_PROCESS, HTTP2_STAT_MAX_SETTINGS_PER_FRAME_EXCEEDED_NAME, RECD_INT, RECP_PERSISTENT, + static_cast(HTTP2_STAT_MAX_SETTINGS_PER_FRAME_EXCEEDED), RecRawStatSyncSum); + RecRegisterRawStat(http2_rsb, RECT_PROCESS, HTTP2_STAT_MAX_SETTINGS_PER_MINUTE_EXCEEDED_NAME, RECD_INT, RECP_PERSISTENT, + static_cast(HTTP2_STAT_MAX_SETTINGS_PER_MINUTE_EXCEEDED), RecRawStatSyncSum); + RecRegisterRawStat(http2_rsb, RECT_PROCESS, HTTP2_STAT_MAX_SETTINGS_FRAMES_PER_MINUTE_EXCEEDED_NAME, RECD_INT, RECP_PERSISTENT, + static_cast(HTTP2_STAT_MAX_SETTINGS_FRAMES_PER_MINUTE_EXCEEDED), RecRawStatSyncSum); + RecRegisterRawStat(http2_rsb, RECT_PROCESS, HTTP2_STAT_MAX_PING_FRAMES_PER_MINUTE_EXCEEDED_NAME, RECD_INT, RECP_PERSISTENT, + static_cast(HTTP2_STAT_MAX_PING_FRAMES_PER_MINUTE_EXCEEDED), RecRawStatSyncSum); + RecRegisterRawStat(http2_rsb, RECT_PROCESS, HTTP2_STAT_MAX_PRIORITY_FRAMES_PER_MINUTE_EXCEEDED_NAME, RECD_INT, RECP_PERSISTENT, + static_cast(HTTP2_STAT_MAX_PRIORITY_FRAMES_PER_MINUTE_EXCEEDED), RecRawStatSyncSum); + RecRegisterRawStat(http2_rsb, RECT_PROCESS, HTTP2_STAT_INSUFFICIENT_AVG_WINDOW_UPDATE_NAME, RECD_INT, RECP_PERSISTENT, + static_cast(HTTP2_STAT_INSUFFICIENT_AVG_WINDOW_UPDATE), RecRawStatSyncSum); + + http2_init(); +} + +/** + mime_init() needs to be called + */ +void +http2_init() +{ + ink_assert(MIME_FIELD_CONNECTION != nullptr); + ink_assert(MIME_FIELD_KEEP_ALIVE != nullptr); + ink_assert(MIME_FIELD_PROXY_CONNECTION != nullptr); + ink_assert(MIME_FIELD_TRANSFER_ENCODING != nullptr); + ink_assert(MIME_FIELD_UPGRADE != nullptr); + + http2_connection_specific_headers[0] = {MIME_FIELD_CONNECTION, MIME_LEN_CONNECTION}; + http2_connection_specific_headers[1] = {MIME_FIELD_KEEP_ALIVE, MIME_LEN_KEEP_ALIVE}; + http2_connection_specific_headers[2] = {MIME_FIELD_PROXY_CONNECTION, MIME_LEN_PROXY_CONNECTION}; + http2_connection_specific_headers[3] = {MIME_FIELD_TRANSFER_ENCODING, MIME_LEN_TRANSFER_ENCODING}; + http2_connection_specific_headers[4] = {MIME_FIELD_UPGRADE, MIME_LEN_UPGRADE}; } #if TS_HAS_TESTS diff --git a/proxy/http2/HTTP2.h b/proxy/http2/HTTP2.h index 85e2669bf0e..966164508a5 100644 --- a/proxy/http2/HTTP2.h +++ b/proxy/http2/HTTP2.h @@ -33,6 +33,9 @@ class HTTPHdr; typedef unsigned Http2StreamId; +constexpr Http2StreamId HTTP2_CONNECTION_CONTROL_STRTEAM = 0; +constexpr uint8_t HTTP2_FRAME_NO_FLAG = 0; + // [RFC 7540] 6.9.2. Initial Flow Control Window Size // the flow control window can be come negative so we need to track it with a signed type. typedef int32_t Http2WindowSize; @@ -40,6 +43,18 @@ typedef int32_t Http2WindowSize; extern const char *const HTTP2_CONNECTION_PREFACE; const size_t HTTP2_CONNECTION_PREFACE_LEN = 24; +extern const char *HTTP2_VALUE_SCHEME; +extern const char *HTTP2_VALUE_METHOD; +extern const char *HTTP2_VALUE_AUTHORITY; +extern const char *HTTP2_VALUE_PATH; +extern const char *HTTP2_VALUE_STATUS; + +extern const unsigned HTTP2_LEN_SCHEME; +extern const unsigned HTTP2_LEN_METHOD; +extern const unsigned HTTP2_LEN_AUTHORITY; +extern const unsigned HTTP2_LEN_PATH; +extern const unsigned HTTP2_LEN_STATUS; + const size_t HTTP2_FRAME_HEADER_LEN = 9; const size_t HTTP2_DATA_PADLEN_LEN = 1; const size_t HTTP2_HEADERS_PADLEN_LEN = 1; @@ -84,6 +99,12 @@ enum { HTTP2_STAT_SESSION_DIE_EOS, HTTP2_STAT_SESSION_DIE_ERROR, HTTP2_STAT_SESSION_DIE_HIGH_ERROR_RATE, + HTTP2_STAT_MAX_SETTINGS_PER_FRAME_EXCEEDED, + HTTP2_STAT_MAX_SETTINGS_PER_MINUTE_EXCEEDED, + HTTP2_STAT_MAX_SETTINGS_FRAMES_PER_MINUTE_EXCEEDED, + HTTP2_STAT_MAX_PING_FRAMES_PER_MINUTE_EXCEEDED, + HTTP2_STAT_MAX_PRIORITY_FRAMES_PER_MINUTE_EXCEEDED, + HTTP2_STAT_INSUFFICIENT_AVG_WINDOW_UPDATE, HTTP2_N_STATS // Terminal counter, NOT A STAT INDEX. }; @@ -293,7 +314,6 @@ struct Http2RstStream { // [RFC 7540] 6.6 PUSH_PROMISE Format struct Http2PushPromise { - Http2PushPromise() {} uint8_t pad_length = 0; Http2StreamId promised_streamid = 0; }; @@ -314,10 +334,6 @@ bool http2_parse_frame_header(IOVec, Http2FrameHeader &); bool http2_write_frame_header(const Http2FrameHeader &, IOVec); -bool http2_write_data(const uint8_t *, size_t, const IOVec &); - -bool http2_write_headers(const uint8_t *, size_t, const IOVec &); - bool http2_write_rst_stream(uint32_t, IOVec); bool http2_write_settings(const Http2SettingsParameter &, const IOVec &); @@ -351,7 +367,9 @@ Http2ErrorCode http2_decode_header_blocks(HTTPHdr *, const uint8_t *, const uint Http2ErrorCode http2_encode_header_blocks(HTTPHdr *, uint8_t *, uint32_t, uint32_t *, HpackHandle &, int32_t); ParseResult http2_convert_header_from_2_to_1_1(HTTPHdr *); -void http2_generate_h2_header_from_1_1(HTTPHdr *headers, HTTPHdr *h2_headers); +ParseResult http2_convert_header_from_1_1_to_2(HTTPHdr *); +void http2_init_pseudo_headers(HTTPHdr &); +void http2_init(); // Not sure where else to put this, but figure this is as good of a start as // anything else. @@ -384,6 +402,7 @@ class Http2 static float min_avg_window_update; static uint32_t con_slow_log_threshold; static uint32_t stream_slow_log_threshold; + static uint32_t header_table_size_limit; static void init(); }; diff --git a/proxy/http2/Http2ClientSession.cc b/proxy/http2/Http2ClientSession.cc index a4658d5b319..fb2650ce757 100644 --- a/proxy/http2/Http2ClientSession.cc +++ b/proxy/http2/Http2ClientSession.cc @@ -87,9 +87,9 @@ Http2ClientSession::free() this->_reenable_event = nullptr; } - if (client_vc) { - client_vc->do_io_close(); - client_vc = nullptr; + if (_vc) { + _vc->do_io_close(); + _vc = nullptr; } // Make sure the we are at the bottom of the stack @@ -103,6 +103,9 @@ Http2ClientSession::free() REMEMBER(NO_EVENT, this->recursion) Http2SsnDebug("session free"); + // Don't free active ProxySession + ink_release_assert(is_active() == false); + this->_milestones.mark(Http2SsnMilestone::CLOSE); ink_hrtime total_time = this->_milestones.elapsed(Http2SsnMilestone::OPEN, Http2SsnMilestone::CLOSE); @@ -151,8 +154,9 @@ Http2ClientSession::free() } } - ink_release_assert(this->client_vc == nullptr); + ink_release_assert(this->_vc == nullptr); + delete _h2_pushed_urls; this->connection_state.destroy(); super::free(); @@ -176,7 +180,7 @@ Http2ClientSession::start() this->connection_state.init(); send_connection_event(&this->connection_state, HTTP2_SESSION_EVENT_INIT, this); - if (this->sm_reader->is_read_avail_more_than(0)) { + if (this->_reader->is_read_avail_more_than(0)) { this->handleEvent(VC_EVENT_READ_READY, read_vio); } } @@ -190,26 +194,35 @@ Http2ClientSession::new_connection(NetVConnection *new_vc, MIOBuffer *iobuf, IOB this->_milestones.mark(Http2SsnMilestone::OPEN); // Unique client session identifier. - this->con_id = ProxySession::next_connection_id(); - this->client_vc = new_vc; - client_vc->set_inactivity_timeout(HRTIME_SECONDS(Http2::accept_no_activity_timeout)); + this->con_id = ProxySession::next_connection_id(); + this->_vc = new_vc; + _vc->set_inactivity_timeout(HRTIME_SECONDS(Http2::accept_no_activity_timeout)); this->schedule_event = nullptr; this->mutex = new_vc->mutex; this->in_destroy = false; this->connection_state.mutex = this->mutex; - Http2SsnDebug("session born, netvc %p", this->client_vc); + SSLNetVConnection *ssl_vc = dynamic_cast(new_vc); + if (ssl_vc != nullptr) { + this->read_from_early_data = ssl_vc->read_from_early_data; + Debug("ssl_early_data", "read_from_early_data = %" PRId64, this->read_from_early_data); + } + + Http2SsnDebug("session born, netvc %p", this->_vc); - this->client_vc->set_tcp_congestion_control(CLIENT_SIDE); + this->_vc->set_tcp_congestion_control(CLIENT_SIDE); this->read_buffer = iobuf ? iobuf : new_MIOBuffer(HTTP2_HEADER_BUFFER_SIZE_INDEX); this->read_buffer->water_mark = connection_state.server_settings.get(HTTP2_SETTINGS_MAX_FRAME_SIZE); - this->sm_reader = reader ? reader : this->read_buffer->alloc_reader(); + this->_reader = reader ? reader : this->read_buffer->alloc_reader(); - this->write_buffer = new_MIOBuffer(HTTP2_HEADER_BUFFER_SIZE_INDEX); + // Set write buffer size to max size of TLS record (16KB) + this->write_buffer = new_MIOBuffer(BUFFER_SIZE_INDEX_16K); this->sm_writer = this->write_buffer->alloc_reader(); + this->_handle_if_ssl(new_vc); + do_api_callout(TS_HTTP_SSN_START_HOOK); } @@ -252,32 +265,6 @@ Http2ClientSession::set_upgrade_context(HTTPHdr *h) upgrade_context.req_header->field_delete(MIME_FIELD_HTTP2_SETTINGS, MIME_LEN_HTTP2_SETTINGS); } -VIO * -Http2ClientSession::do_io_read(Continuation *c, int64_t nbytes, MIOBuffer *buf) -{ - if (client_vc) { - return this->client_vc->do_io_read(c, nbytes, buf); - } else { - return nullptr; - } -} - -VIO * -Http2ClientSession::do_io_write(Continuation *c, int64_t nbytes, IOBufferReader *buf, bool owner) -{ - if (client_vc) { - return this->client_vc->do_io_write(c, nbytes, buf, owner); - } else { - return nullptr; - } -} - -void -Http2ClientSession::do_io_shutdown(ShutdownHowTo_t howto) -{ - this->client_vc->do_io_shutdown(howto); -} - // XXX Currently, we don't have a half-closed state, but we will need to // implement that. After we send a GOAWAY, there // are scenarios where we would like to complete the outstanding streams. @@ -291,28 +278,15 @@ Http2ClientSession::do_io_close(int alerrno) ink_assert(this->mutex->thread_holding == this_ethread()); send_connection_event(&this->connection_state, HTTP2_SESSION_EVENT_FINI, this); - // Don't send the SSN_CLOSE_HOOK until we got rid of all the streams - // And handled all the TXN_CLOSE_HOOK's - if (client_vc) { - // Copy aside the client address before releasing the vc - cached_client_addr.assign(client_vc->get_remote_addr()); - cached_local_addr.assign(client_vc->get_local_addr()); - client_vc->do_io_close(); - client_vc = nullptr; - } - { SCOPED_MUTEX_LOCK(lock, this->connection_state.mutex, this_ethread()); - this->connection_state.release_stream(nullptr); + this->connection_state.release_stream(); } this->clear_session_active(); -} -void -Http2ClientSession::reenable(VIO *vio) -{ - this->client_vc->reenable(vio); + // Clean up the write VIO in case of inactivity timeout + this->do_io_write(this, 0, nullptr); } void @@ -324,6 +298,19 @@ Http2ClientSession::set_half_close_local_flag(bool flag) half_close_local = flag; } +int64_t +Http2ClientSession::xmit(const Http2TxFrame &frame) +{ + int64_t len = frame.write_to(this->write_buffer); + + if (len > 0) { + total_write_len += len; + write_reenable(); + } + + return len; +} + int Http2ClientSession::main_event_handler(int event, void *edata) { @@ -348,16 +335,6 @@ Http2ClientSession::main_event_handler(int event, void *edata) break; } - case HTTP2_SESSION_EVENT_XMIT: { - Http2Frame *frame = static_cast(edata); - total_write_len += frame->size(); - write_vio->nbytes = total_write_len; - frame->xmit(this->write_buffer); - write_reenable(); - retval = 0; - break; - } - case HTTP2_SESSION_EVENT_REENABLE: // VIO will be reenableed in this handler retval = (this->*session_handler)(VC_EVENT_READ_READY, static_cast(e->cookie)); @@ -369,20 +346,20 @@ Http2ClientSession::main_event_handler(int event, void *edata) case VC_EVENT_INACTIVITY_TIMEOUT: case VC_EVENT_ERROR: case VC_EVENT_EOS: + Http2SsnDebug("Closing event %d", event); this->set_dying_event(event); this->do_io_close(); retval = 0; break; case VC_EVENT_WRITE_READY: - retval = 0; - break; - case VC_EVENT_WRITE_COMPLETE: - // Seems as this is being closed already + this->connection_state.restart_streams(); + retval = 0; break; + case HTTP2_SESSION_EVENT_XMIT: default: Http2SsnDebug("unexpected event=%d edata=%p", event, edata); ink_release_assert(0); @@ -402,9 +379,9 @@ Http2ClientSession::main_event_handler(int event, void *edata) Http2::stream_error_rate_threshold) { // For a case many stream errors happened ip_port_text_buffer ipb; const char *client_ip = ats_ip_ntop(get_client_addr(), ipb, sizeof(ipb)); - Error("HTTP/2 session error client_ip=%s session_id=%" PRId64 - " closing a connection, because its stream error rate (%f) exceeded the threshold (%f)", - client_ip, connection_id(), this->connection_state.get_stream_error_rate(), Http2::stream_error_rate_threshold); + Warning("HTTP/2 session error client_ip=%s session_id=%" PRId64 + " closing a connection, because its stream error rate (%f) exceeded the threshold (%f)", + client_ip, connection_id(), this->connection_state.get_stream_error_rate(), Http2::stream_error_rate_threshold); Http2SsnDebug("Preparing for graceful shutdown because of a high stream error rate"); cause_of_death = Http2SessionCod::HIGH_ERROR_RATE; this->connection_state.set_shutdown_state(HTTP2_SHUTDOWN_NOT_INITIATED, Http2ErrorCode::HTTP2_ERROR_ENHANCE_YOUR_CALM); @@ -430,11 +407,11 @@ Http2ClientSession::state_read_connection_preface(int event, void *edata) STATE_ENTER(&Http2ClientSession::state_read_connection_preface, event); ink_assert(event == VC_EVENT_READ_COMPLETE || event == VC_EVENT_READ_READY); - if (this->sm_reader->read_avail() >= static_cast(HTTP2_CONNECTION_PREFACE_LEN)) { + if (this->_reader->read_avail() >= static_cast(HTTP2_CONNECTION_PREFACE_LEN)) { char buf[HTTP2_CONNECTION_PREFACE_LEN]; unsigned nbytes; - nbytes = copy_from_buffer_reader(buf, this->sm_reader, sizeof(buf)); + nbytes = copy_from_buffer_reader(buf, this->_reader, sizeof(buf)); ink_release_assert(nbytes == HTTP2_CONNECTION_PREFACE_LEN); if (memcmp(HTTP2_CONNECTION_PREFACE, buf, nbytes) != 0) { @@ -443,17 +420,22 @@ Http2ClientSession::state_read_connection_preface(int event, void *edata) return 0; } + // Check whether data is read from early data + if (this->read_from_early_data > 0) { + this->read_from_early_data -= this->read_from_early_data > nbytes ? nbytes : this->read_from_early_data; + } + Http2SsnDebug("received connection preface"); - this->sm_reader->consume(nbytes); + this->_reader->consume(nbytes); HTTP2_SET_SESSION_HANDLER(&Http2ClientSession::state_start_frame_read); - client_vc->set_inactivity_timeout(HRTIME_SECONDS(Http2::no_activity_timeout_in)); - client_vc->set_active_timeout(HRTIME_SECONDS(Http2::active_timeout_in)); + _vc->set_inactivity_timeout(HRTIME_SECONDS(Http2::no_activity_timeout_in)); + _vc->set_active_timeout(HRTIME_SECONDS(Http2::active_timeout_in)); // XXX start the write VIO ... // If we have unconsumed data, start tranferring frames now. - if (this->sm_reader->is_read_avail_more_than(0)) { + if (this->_reader->is_read_avail_more_than(0)) { return this->handleEvent(VC_EVENT_READ_READY, vio); } } @@ -481,24 +463,31 @@ int Http2ClientSession::do_start_frame_read(Http2ErrorCode &ret_error) { ret_error = Http2ErrorCode::HTTP2_ERROR_NO_ERROR; - ink_release_assert(this->sm_reader->read_avail() >= (int64_t)HTTP2_FRAME_HEADER_LEN); + ink_release_assert(this->_reader->read_avail() >= (int64_t)HTTP2_FRAME_HEADER_LEN); uint8_t buf[HTTP2_FRAME_HEADER_LEN]; unsigned nbytes; Http2SsnDebug("receiving frame header"); - nbytes = copy_from_buffer_reader(buf, this->sm_reader, sizeof(buf)); + nbytes = copy_from_buffer_reader(buf, this->_reader, sizeof(buf)); + this->cur_frame_from_early_data = false; if (!http2_parse_frame_header(make_iovec(buf), this->current_hdr)) { Http2SsnDebug("frame header parse failure"); this->do_io_close(); return -1; } + // Check whether data is read from early data + if (this->read_from_early_data > 0) { + this->read_from_early_data -= this->read_from_early_data > nbytes ? nbytes : this->read_from_early_data; + this->cur_frame_from_early_data = true; + } + Http2SsnDebug("frame header length=%u, type=%u, flags=0x%x, streamid=%u", (unsigned)this->current_hdr.length, (unsigned)this->current_hdr.type, (unsigned)this->current_hdr.flags, this->current_hdr.streamid); - this->sm_reader->consume(nbytes); + this->_reader->consume(nbytes); if (!http2_frame_header_is_valid(this->current_hdr, this->connection_state.server_settings.get(HTTP2_SETTINGS_MAX_FRAME_SIZE))) { ret_error = Http2ErrorCode::HTTP2_ERROR_PROTOCOL_ERROR; @@ -528,7 +517,7 @@ Http2ClientSession::state_complete_frame_read(int event, void *edata) VIO *vio = static_cast(edata); STATE_ENTER(&Http2ClientSession::state_complete_frame_read, event); ink_assert(event == VC_EVENT_READ_COMPLETE || event == VC_EVENT_READ_READY); - if (this->sm_reader->read_avail() < this->current_hdr.length) { + if (this->_reader->read_avail() < this->current_hdr.length) { if (this->_should_do_something_else()) { if (this->_reenable_event == nullptr) { vio->disable(); @@ -541,7 +530,7 @@ Http2ClientSession::state_complete_frame_read(int event, void *edata) } return 0; } - Http2SsnDebug("completed frame read, %" PRId64 " bytes available", this->sm_reader->read_avail()); + Http2SsnDebug("completed frame read, %" PRId64 " bytes available", this->_reader->read_avail()); return state_process_frame_read(event, vio, true); } @@ -550,11 +539,16 @@ int Http2ClientSession::do_complete_frame_read() { // XXX parse the frame and handle it ... - ink_release_assert(this->sm_reader->read_avail() >= this->current_hdr.length); + ink_release_assert(this->_reader->read_avail() >= this->current_hdr.length); - Http2Frame frame(this->current_hdr, this->sm_reader); + Http2Frame frame(this->current_hdr, this->_reader, this->cur_frame_from_early_data); send_connection_event(&this->connection_state, HTTP2_SESSION_EVENT_RECV, &frame); - this->sm_reader->consume(this->current_hdr.length); + // Check whether data is read from early data + if (this->read_from_early_data > 0) { + this->read_from_early_data -= + this->read_from_early_data > this->current_hdr.length ? this->current_hdr.length : this->read_from_early_data; + } + this->_reader->consume(this->current_hdr.length); ++(this->_n_frame_read); // Set the event handler if there is no more data to process a new frame @@ -570,7 +564,7 @@ Http2ClientSession::state_process_frame_read(int event, VIO *vio, bool inside_fr do_complete_frame_read(); } - while (this->sm_reader->read_avail() >= static_cast(HTTP2_FRAME_HEADER_LEN)) { + while (this->_reader->read_avail() >= static_cast(HTTP2_FRAME_HEADER_LEN)) { // Cancel reading if there was an error or connection is closed if (connection_state.tx_error_code.code != static_cast(Http2ErrorCode::HTTP2_ERROR_NO_ERROR) || connection_state.is_state_closed()) { @@ -582,9 +576,9 @@ Http2ClientSession::state_process_frame_read(int event, VIO *vio, bool inside_fr if (this->connection_state.get_stream_error_rate() > std::min(1.0, Http2::stream_error_rate_threshold * 2.0)) { ip_port_text_buffer ipb; const char *client_ip = ats_ip_ntop(get_client_addr(), ipb, sizeof(ipb)); - Error("HTTP/2 session error client_ip=%s session_id=%" PRId64 - " closing a connection, because its stream error rate (%f) is too high", - client_ip, connection_id(), this->connection_state.get_stream_error_rate()); + Warning("HTTP/2 session error client_ip=%s session_id=%" PRId64 + " closing a connection, because its stream error rate (%f) exceeded the threshold (%f)", + client_ip, connection_id(), this->connection_state.get_stream_error_rate(), Http2::stream_error_rate_threshold); err = Http2ErrorCode::HTTP2_ERROR_ENHANCE_YOUR_CALM; } @@ -596,14 +590,13 @@ Http2ClientSession::state_process_frame_read(int event, VIO *vio, bool inside_fr if (!this->connection_state.is_state_closed()) { this->connection_state.send_goaway_frame(this->connection_state.get_latest_stream_id_in(), err); this->set_half_close_local_flag(true); - this->do_io_close(); } } return 0; } // If there is no more data to finish the frame, set up the event handler and reenable - if (this->sm_reader->read_avail() < this->current_hdr.length) { + if (this->_reader->read_avail() < this->current_hdr.length) { HTTP2_SET_SESSION_HANDLER(&Http2ClientSession::state_complete_frame_read); break; } @@ -649,3 +642,84 @@ Http2ClientSession::_should_do_something_else() // Do something else every 128 incoming frames return (this->_n_frame_read & 0x7F) == 0; } + +sockaddr const * +Http2ClientSession::get_client_addr() +{ + return _vc ? _vc->get_remote_addr() : &cached_client_addr.sa; +} + +sockaddr const * +Http2ClientSession::get_local_addr() +{ + return _vc ? _vc->get_local_addr() : &cached_local_addr.sa; +} + +int64_t +Http2ClientSession::write_avail() +{ + return this->write_buffer->write_avail(); +} + +void +Http2ClientSession::write_reenable() +{ + write_vio->reenable(); +} + +int +Http2ClientSession::get_transact_count() const +{ + return connection_state.get_stream_requests(); +} + +void +Http2ClientSession::release(ProxyTransaction *trans) +{ +} + +const char * +Http2ClientSession::get_protocol_string() const +{ + return "http/2"; +} + +int +Http2ClientSession::populate_protocol(std::string_view *result, int size) const +{ + int retval = 0; + if (size > retval) { + result[retval++] = IP_PROTO_TAG_HTTP_2_0; + if (size > retval) { + retval += super::populate_protocol(result + retval, size - retval); + } + } + return retval; +} + +const char * +Http2ClientSession::protocol_contains(std::string_view prefix) const +{ + const char *retval = nullptr; + + if (prefix.size() <= IP_PROTO_TAG_HTTP_2_0.size() && strncmp(IP_PROTO_TAG_HTTP_2_0.data(), prefix.data(), prefix.size()) == 0) { + retval = IP_PROTO_TAG_HTTP_2_0.data(); + } else { + retval = super::protocol_contains(prefix); + } + return retval; +} + +void +Http2ClientSession::add_url_to_pushed_table(const char *url, int url_len) +{ + // Delay std::unordered_set allocation until when it used + if (_h2_pushed_urls == nullptr) { + this->_h2_pushed_urls = new std::unordered_set(); + this->_h2_pushed_urls->reserve(Http2::push_diary_size); + } + + if (_h2_pushed_urls->size() < Http2::push_diary_size) { + _h2_pushed_urls->emplace(url); + } +} diff --git a/proxy/http2/Http2ClientSession.h b/proxy/http2/Http2ClientSession.h index 433be890b73..0f1f64bb1cc 100644 --- a/proxy/http2/Http2ClientSession.h +++ b/proxy/http2/Http2ClientSession.h @@ -27,6 +27,7 @@ #include "Plugin.h" #include "ProxySession.h" #include "Http2ConnectionState.h" +#include "Http2Frame.h" #include #include "tscore/ink_inet.h" #include "tscore/History.h" @@ -77,239 +78,67 @@ struct Http2UpgradeContext { Http2ConnectionSettings client_settings; }; -class Http2Frame -{ -public: - Http2Frame(const Http2FrameHeader &h, IOBufferReader *r) : hdr(h), ioreader(r) {} - Http2Frame(Http2FrameType type, Http2StreamId streamid, uint8_t flags) : hdr({0, (uint8_t)type, flags, streamid}) {} - - IOBufferReader * - reader() const - { - return ioreader; - } - - const Http2FrameHeader & - header() const - { - return this->hdr; - } - - // Allocate an IOBufferBlock for payload of this frame. - void - alloc(int index) - { - this->ioblock = new_IOBufferBlock(); - this->ioblock->alloc(index); - } - - // Return the writeable buffer space for frame payload - IOVec - write() - { - return make_iovec(this->ioblock->end(), this->ioblock->write_avail()); - } - - // Once the frame has been serialized, update the payload length of frame header. - void - finalize(size_t nbytes) - { - if (this->ioblock) { - ink_assert((int64_t)nbytes <= this->ioblock->write_avail()); - this->ioblock->fill(nbytes); - - this->hdr.length = this->ioblock->size(); - } - } - - void - xmit(MIOBuffer *iobuffer) - { - // Write frame header - uint8_t buf[HTTP2_FRAME_HEADER_LEN]; - http2_write_frame_header(hdr, make_iovec(buf)); - iobuffer->write(buf, sizeof(buf)); - - // Write frame payload - // It could be empty (e.g. SETTINGS frame with ACK flag) - if (ioblock && ioblock->read_avail() > 0) { - iobuffer->append_block(this->ioblock.get()); - } - } - - int64_t - size() - { - if (ioblock) { - return HTTP2_FRAME_HEADER_LEN + ioblock->size(); - } else { - return HTTP2_FRAME_HEADER_LEN; - } - } - - // noncopyable - Http2Frame(Http2Frame &) = delete; - Http2Frame &operator=(const Http2Frame &) = delete; - -private: - Http2FrameHeader hdr; // frame header - Ptr ioblock; // frame payload - IOBufferReader *ioreader = nullptr; -}; - class Http2ClientSession : public ProxySession { public: - typedef ProxySession super; ///< Parent type. - typedef int (Http2ClientSession::*SessionHandler)(int, void *); + using super = ProxySession; ///< Parent type. + using SessionHandler = int (Http2ClientSession::*)(int, void *); Http2ClientSession(); - // Implement ProxySession interface. - void start() override; - void destroy() override; - void free() override; - void new_connection(NetVConnection *new_vc, MIOBuffer *iobuf, IOBufferReader *reader) override; + ///////////////////// + // Methods - bool - ready_to_free() const - { - return kill_me; - } - - // Implement VConnection interface. - VIO *do_io_read(Continuation *c, int64_t nbytes = INT64_MAX, MIOBuffer *buf = nullptr) override; - VIO *do_io_write(Continuation *c = nullptr, int64_t nbytes = INT64_MAX, IOBufferReader *buf = nullptr, - bool owner = false) override; + // Implement VConnection interface void do_io_close(int lerrno = -1) override; - void do_io_shutdown(ShutdownHowTo_t howto) override; - void reenable(VIO *vio) override; - - NetVConnection * - get_netvc() const override - { - return client_vc; - } - - sockaddr const * - get_client_addr() override - { - return client_vc ? client_vc->get_remote_addr() : &cached_client_addr.sa; - } - sockaddr const * - get_local_addr() override - { - return client_vc ? client_vc->get_local_addr() : &cached_local_addr.sa; - } - - void - write_reenable() - { - write_vio->reenable(); - } - - void set_upgrade_context(HTTPHdr *h); - - const Http2UpgradeContext & - get_upgrade_context() const - { - return upgrade_context; - } - - int - get_transact_count() const override - { - return connection_state.get_stream_requests(); - } - - void - release(ProxyTransaction *trans) override - { - } - - Http2ConnectionState connection_state; - void - set_dying_event(int event) - { - dying_event = event; - } - - int - get_dying_event() const - { - return dying_event; - } - - bool - is_recursing() const - { - return recursion > 0; - } - - const char * - get_protocol_string() const override - { - return "http/2"; - } - - int - populate_protocol(std::string_view *result, int size) const override - { - int retval = 0; - if (size > retval) { - result[retval++] = IP_PROTO_TAG_HTTP_2_0; - if (size > retval) { - retval += super::populate_protocol(result + retval, size - retval); - } - } - return retval; - } - - const char * - protocol_contains(std::string_view prefix) const override - { - const char *retval = nullptr; - - if (prefix.size() <= IP_PROTO_TAG_HTTP_2_0.size() && strncmp(IP_PROTO_TAG_HTTP_2_0.data(), prefix.data(), prefix.size()) == 0) { - retval = IP_PROTO_TAG_HTTP_2_0.data(); - } else { - retval = super::protocol_contains(prefix); - } - return retval; - } + // Implement ProxySession interface + void new_connection(NetVConnection *new_vc, MIOBuffer *iobuf, IOBufferReader *reader) override; + void start() override; + void destroy() override; + void release(ProxyTransaction *trans) override; + void free() override; + // more methods + void write_reenable(); + int64_t xmit(const Http2TxFrame &frame); + + //////////////////// + // Accessors + sockaddr const *get_client_addr() override; + sockaddr const *get_local_addr() override; + int get_transact_count() const override; + const char *get_protocol_string() const override; + int populate_protocol(std::string_view *result, int size) const override; + const char *protocol_contains(std::string_view prefix) const override; void increment_current_active_client_connections_stat() override; void decrement_current_active_client_connections_stat() override; + void set_upgrade_context(HTTPHdr *h); + const Http2UpgradeContext &get_upgrade_context() const; + void set_dying_event(int event); + int get_dying_event() const; + bool ready_to_free() const; + bool is_recursing() const; void set_half_close_local_flag(bool flag); - bool - get_half_close_local_flag() const - { - return half_close_local; - } - - bool - is_url_pushed(const char *url, int url_len) - { - return h2_pushed_urls.find(url) != h2_pushed_urls.end(); - } - - void - add_url_to_pushed_table(const char *url, int url_len) - { - if (h2_pushed_urls.size() < Http2::push_diary_size) { - h2_pushed_urls.emplace(url); - } - } + bool get_half_close_local_flag() const; + bool is_url_pushed(const char *url, int url_len); + void add_url_to_pushed_table(const char *url, int url_len); + int64_t write_buffer_size(); // Record history from Http2ConnectionState void remember(const SourceLocation &location, int event, int reentrant = NO_REENTRANT); + int64_t write_avail(); + // noncopyable Http2ClientSession(Http2ClientSession &) = delete; Http2ClientSession &operator=(const Http2ClientSession &) = delete; + /////////////////// + // Variables + Http2ConnectionState connection_state; + private: int main_event_handler(int, void *); @@ -327,9 +156,8 @@ class Http2ClientSession : public ProxySession int64_t total_write_len = 0; SessionHandler session_handler = nullptr; - NetVConnection *client_vc = nullptr; MIOBuffer *read_buffer = nullptr; - IOBufferReader *sm_reader = nullptr; + IOBufferReader *_reader = nullptr; MIOBuffer *write_buffer = nullptr; IOBufferReader *sm_writer = nullptr; Http2FrameHeader current_hdr = {0, 0, 0, 0}; @@ -350,10 +178,68 @@ class Http2ClientSession : public ProxySession bool half_close_local = false; int recursion = 0; - std::unordered_set h2_pushed_urls; + std::unordered_set *_h2_pushed_urls = nullptr; Event *_reenable_event = nullptr; int _n_frame_read = 0; + + int64_t read_from_early_data = 0; + bool cur_frame_from_early_data = false; }; extern ClassAllocator http2ClientSessionAllocator; + +/////////////////////////////////////////////// +// INLINE + +inline const Http2UpgradeContext & +Http2ClientSession::get_upgrade_context() const +{ + return upgrade_context; +} + +inline bool +Http2ClientSession::ready_to_free() const +{ + return kill_me; +} + +inline void +Http2ClientSession::set_dying_event(int event) +{ + dying_event = event; +} + +inline int +Http2ClientSession::get_dying_event() const +{ + return dying_event; +} + +inline bool +Http2ClientSession::is_recursing() const +{ + return recursion > 0; +} + +inline bool +Http2ClientSession::get_half_close_local_flag() const +{ + return half_close_local; +} + +inline bool +Http2ClientSession::is_url_pushed(const char *url, int url_len) +{ + if (_h2_pushed_urls == nullptr) { + return false; + } + + return _h2_pushed_urls->find(url) != _h2_pushed_urls->end(); +} + +inline int64_t +Http2ClientSession::write_buffer_size() +{ + return write_buffer->max_read_avail(); +} diff --git a/proxy/http2/Http2ConnectionState.cc b/proxy/http2/Http2ConnectionState.cc index d27f3818585..23beaffb123 100644 --- a/proxy/http2/Http2ConnectionState.cc +++ b/proxy/http2/Http2ConnectionState.cc @@ -25,8 +25,12 @@ #include "Http2ConnectionState.h" #include "Http2ClientSession.h" #include "Http2Stream.h" +#include "Http2Frame.h" #include "Http2DebugNames.h" #include "HttpDebugNames.h" + +#include "tscpp/util/PostScript.h" + #include #include @@ -49,12 +53,12 @@ static const int buffer_size_index[HTTP2_FRAME_TYPE_MAX] = { BUFFER_SIZE_INDEX_16K, // HTTP2_FRAME_TYPE_DATA BUFFER_SIZE_INDEX_16K, // HTTP2_FRAME_TYPE_HEADERS -1, // HTTP2_FRAME_TYPE_PRIORITY - BUFFER_SIZE_INDEX_128, // HTTP2_FRAME_TYPE_RST_STREAM - BUFFER_SIZE_INDEX_128, // HTTP2_FRAME_TYPE_SETTINGS + -1, // HTTP2_FRAME_TYPE_RST_STREAM + -1, // HTTP2_FRAME_TYPE_SETTINGS BUFFER_SIZE_INDEX_16K, // HTTP2_FRAME_TYPE_PUSH_PROMISE - BUFFER_SIZE_INDEX_128, // HTTP2_FRAME_TYPE_PING - BUFFER_SIZE_INDEX_128, // HTTP2_FRAME_TYPE_GOAWAY - BUFFER_SIZE_INDEX_128, // HTTP2_FRAME_TYPE_WINDOW_UPDATE + -1, // HTTP2_FRAME_TYPE_PING + -1, // HTTP2_FRAME_TYPE_GOAWAY + -1, // HTTP2_FRAME_TYPE_WINDOW_UPDATE BUFFER_SIZE_INDEX_16K, // HTTP2_FRAME_TYPE_CONTINUATION }; @@ -138,11 +142,17 @@ rcv_data_frame(Http2ConnectionState &cstate, const Http2Frame &frame) return Http2Error(Http2ErrorClass::HTTP2_ERROR_CLASS_CONNECTION, Http2ErrorCode::HTTP2_ERROR_PROTOCOL_ERROR, "recv data bad payload length"); } - } - // If Data length is 0, do nothing. - if (payload_length == 0) { - return Http2Error(Http2ErrorClass::HTTP2_ERROR_CLASS_NONE); + // Pure END_STREAM + if (payload_length == 0) { + stream->signal_read_event(VC_EVENT_READ_COMPLETE); + return Http2Error(Http2ErrorClass::HTTP2_ERROR_CLASS_NONE); + } + } else { + // If payload length is 0 without END_STREAM flag, do nothing + if (payload_length == 0) { + return Http2Error(Http2ErrorClass::HTTP2_ERROR_CLASS_NONE); + } } // Check whether Window Size is acceptable @@ -159,7 +169,18 @@ rcv_data_frame(Http2ConnectionState &cstate, const Http2Frame &frame) cstate.decrement_server_rwnd(payload_length); stream->decrement_server_rwnd(payload_length); + if (is_debug_tag_set("http2_con")) { + uint32_t rwnd = cstate.server_settings.get(HTTP2_SETTINGS_INITIAL_WINDOW_SIZE); + Http2StreamDebug(cstate.ua_session, id, "Received DATA frame: rwnd con=%zd/%" PRId32 " stream=%zd/%" PRId32, + cstate.server_rwnd(), rwnd, stream->server_rwnd(), rwnd); + } + const uint32_t unpadded_length = payload_length - pad_length; + MIOBuffer *writer = stream->read_vio_writer(); + if (writer == nullptr) { + return Http2Error(Http2ErrorClass::HTTP2_ERROR_CLASS_STREAM, Http2ErrorCode::HTTP2_ERROR_INTERNAL_ERROR); + } + // If we call write() multiple times, we must keep the same reader, so we can // update its offset via consume. Otherwise, we will read the same data on the // second time through @@ -168,31 +189,26 @@ rcv_data_frame(Http2ConnectionState &cstate, const Http2Frame &frame) if (frame.header().flags & HTTP2_FLAGS_DATA_PADDED) { myreader->consume(HTTP2_DATA_PADLEN_LEN); } - while (nbytes < payload_length - pad_length) { + + if (nbytes < unpadded_length) { size_t read_len = BUFFER_SIZE_FOR_INDEX(buffer_size_index[HTTP2_FRAME_TYPE_DATA]); if (nbytes + read_len > unpadded_length) { read_len -= nbytes + read_len - unpadded_length; } - nbytes += stream->request_buffer.write(myreader, read_len); - myreader->consume(nbytes); - // If there is an outstanding read, update the buffer - stream->update_read_request(INT64_MAX, true); + unsigned int num_written = writer->write(myreader, read_len); + if (num_written != read_len) { + myreader->writer()->dealloc_reader(myreader); + return Http2Error(Http2ErrorClass::HTTP2_ERROR_CLASS_STREAM, Http2ErrorCode::HTTP2_ERROR_INTERNAL_ERROR); + } + myreader->consume(num_written); } myreader->writer()->dealloc_reader(myreader); - uint32_t initial_rwnd = cstate.server_settings.get(HTTP2_SETTINGS_INITIAL_WINDOW_SIZE); - uint32_t min_rwnd = std::min(initial_rwnd, cstate.server_settings.get(HTTP2_SETTINGS_MAX_FRAME_SIZE)); - // Connection level WINDOW UPDATE - if (cstate.server_rwnd() <= min_rwnd) { - Http2WindowSize diff_size = initial_rwnd - cstate.server_rwnd(); - cstate.increment_server_rwnd(diff_size); - cstate.send_window_update_frame(0, diff_size); - } - // Stream level WINDOW UPDATE - if (stream->server_rwnd() <= min_rwnd) { - Http2WindowSize diff_size = initial_rwnd - stream->server_rwnd(); - stream->increment_server_rwnd(diff_size); - cstate.send_window_update_frame(stream->get_id(), diff_size); + if (frame.header().flags & HTTP2_FLAGS_DATA_END_STREAM) { + // TODO: set total written size to read_vio.nbytes + stream->signal_read_event(VC_EVENT_READ_COMPLETE); + } else { + stream->signal_read_event(VC_EVENT_READ_READY); } return Http2Error(Http2ErrorClass::HTTP2_ERROR_CLASS_NONE); @@ -225,9 +241,12 @@ rcv_headers_frame(Http2ConnectionState &cstate, const Http2Frame &frame) if (cstate.is_valid_streamid(stream_id)) { stream = cstate.find_stream(stream_id); - if (stream == nullptr || !stream->has_trailing_header()) { + if (stream == nullptr) { return Http2Error(Http2ErrorClass::HTTP2_ERROR_CLASS_CONNECTION, Http2ErrorCode::HTTP2_ERROR_STREAM_CLOSED, "recv headers cannot find existing stream_id"); + } else if (!stream->has_trailing_header()) { + return Http2Error(Http2ErrorClass::HTTP2_ERROR_CLASS_CONNECTION, Http2ErrorCode::HTTP2_ERROR_PROTOCOL_ERROR, + "recv headers cannot find existing stream_id"); } } else { // Create new stream @@ -244,13 +263,6 @@ rcv_headers_frame(Http2ConnectionState &cstate, const Http2Frame &frame) return Http2Error(Http2ErrorClass::HTTP2_ERROR_CLASS_NONE); } - // keep track of how many bytes we get in the frame - stream->request_header_length += payload_length; - if (stream->request_header_length > Http2::max_header_list_size) { - return Http2Error(Http2ErrorClass::HTTP2_ERROR_CLASS_STREAM, Http2ErrorCode::HTTP2_ERROR_PROTOCOL_ERROR, - "recv headers payload for headers greater than header length"); - } - Http2HeadersParameter params; uint32_t header_block_fragment_offset = 0; uint32_t header_block_fragment_length = payload_length; @@ -269,7 +281,8 @@ rcv_headers_frame(Http2ConnectionState &cstate, const Http2Frame &frame) "recv headers failed to parse"); } - if (params.pad_length > payload_length) { + // Payload length can't be smaller than the pad length + if ((params.pad_length + HTTP2_HEADERS_PADLEN_LEN) > header_block_fragment_length) { return Http2Error(Http2ErrorClass::HTTP2_ERROR_CLASS_CONNECTION, Http2ErrorCode::HTTP2_ERROR_PROTOCOL_ERROR, "recv headers pad > payload length"); } @@ -285,7 +298,7 @@ rcv_headers_frame(Http2ConnectionState &cstate, const Http2Frame &frame) frame.reader()->memcpy(buf, HTTP2_PRIORITY_LEN, header_block_fragment_offset); if (!http2_parse_priority_parameter(make_iovec(buf, HTTP2_PRIORITY_LEN), params.priority)) { return Http2Error(Http2ErrorClass::HTTP2_ERROR_CLASS_CONNECTION, Http2ErrorCode::HTTP2_ERROR_PROTOCOL_ERROR, - "recv headers prioirity parameters failed parse"); + "recv headers priority parameters failed parse"); } // Protocol error if the stream depends on itself if (stream_id == params.priority.stream_dependency) { @@ -293,6 +306,12 @@ rcv_headers_frame(Http2ConnectionState &cstate, const Http2Frame &frame) "recv headers self dependency"); } + // Payload length can't be smaller than the priority length + if (HTTP2_PRIORITY_LEN > header_block_fragment_length) { + return Http2Error(Http2ErrorClass::HTTP2_ERROR_CLASS_CONNECTION, Http2ErrorCode::HTTP2_ERROR_PROTOCOL_ERROR, + "recv priority length > payload length"); + } + header_block_fragment_offset += HTTP2_PRIORITY_LEN; header_block_fragment_length -= HTTP2_PRIORITY_LEN; } @@ -312,11 +331,19 @@ rcv_headers_frame(Http2ConnectionState &cstate, const Http2Frame &frame) } } + stream->header_blocks_length = header_block_fragment_length; + + // ATS advertises SETTINGS_MAX_HEADER_LIST_SIZE as a limit of total header blocks length. (Details in [RFC 7560] 10.5.1.) + // Make it double to relax the limit in cases of 1) HPACK is used naively, or 2) Huffman Encoding generates large header blocks. + // The total "decoded" header length is strictly checked by hpack_decode_header_block(). + if (stream->header_blocks_length > std::max(Http2::max_header_list_size, Http2::max_header_list_size * 2)) { + return Http2Error(Http2ErrorClass::HTTP2_ERROR_CLASS_CONNECTION, Http2ErrorCode::HTTP2_ERROR_ENHANCE_YOUR_CALM, + "header blocks too large"); + } + stream->header_blocks = static_cast(ats_malloc(header_block_fragment_length)); frame.reader()->memcpy(stream->header_blocks, header_block_fragment_length, header_block_fragment_offset); - stream->header_blocks_length = header_block_fragment_length; - if (frame.header().flags & HTTP2_FLAGS_HEADERS_END_HEADERS) { // NOTE: If there are END_HEADERS flag, decode stored Header Blocks. if (!stream->change_state(HTTP2_FRAME_TYPE_HEADERS, frame.header().flags) && stream->has_trailing_header() == false) { @@ -358,9 +385,12 @@ rcv_headers_frame(Http2ConnectionState &cstate, const Http2Frame &frame) if (!empty_request) { SCOPED_MUTEX_LOCK(stream_lock, stream->mutex, this_ethread()); stream->mark_milestone(Http2StreamMilestone::START_TXN); - stream->new_transaction(); + stream->new_transaction(frame.is_from_early_data()); // Send request header to SM stream->send_request(cstate); + } else { + // Signal VC_EVENT_READ_COMPLETE becasue received trailing header fields with END_STREAM flag + stream->signal_read_event(VC_EVENT_READ_COMPLETE); } } else { // NOTE: Expect CONTINUATION Frame. Do NOT change state of stream or decode @@ -417,10 +447,16 @@ rcv_priority_frame(Http2ConnectionState &cstate, const Http2Frame &frame) "PRIORITY frame depends on itself"); } + if (!Http2::stream_priority_enabled) { + return Http2Error(Http2ErrorClass::HTTP2_ERROR_CLASS_NONE); + } + // Update PRIORITY frame count per minute cstate.increment_received_priority_frame_count(); // Close this conection if its priority frame count received exceeds a limit - if (cstate.get_received_priority_frame_count() > Http2::max_priority_frames_per_minute) { + if (Http2::max_priority_frames_per_minute != 0 && + cstate.get_received_priority_frame_count() > Http2::max_priority_frames_per_minute) { + HTTP2_INCREMENT_THREAD_DYN_STAT(HTTP2_STAT_MAX_PRIORITY_FRAMES_PER_MINUTE_EXCEEDED, this_ethread()); Http2StreamDebug(cstate.ua_session, stream_id, "Observed too frequent priority changes: %u priority changes within a last minute", cstate.get_received_priority_frame_count()); @@ -428,10 +464,6 @@ rcv_priority_frame(Http2ConnectionState &cstate, const Http2Frame &frame) "recv priority too frequent priority changes"); } - if (!Http2::stream_priority_enabled) { - return Http2Error(Http2ErrorClass::HTTP2_ERROR_CLASS_NONE); - } - Http2StreamDebug(cstate.ua_session, stream_id, "PRIORITY - dep: %d, weight: %d, excl: %d, tree size: %d", priority.stream_dependency, priority.weight, priority.exclusive_flag, cstate.dependency_tree->size()); @@ -513,7 +545,7 @@ rcv_rst_stream_frame(Http2ConnectionState &cstate, const Http2Frame &frame) Http2StreamDebug(cstate.ua_session, stream_id, "RST_STREAM: Error Code: %u", rst_stream.error_code); stream->set_rx_error_code({ProxyErrorClass::TXN, static_cast(rst_stream.error_code)}); - cstate.delete_stream(stream); + stream->initiating_close(); } return Http2Error(Http2ErrorClass::HTTP2_ERROR_CLASS_NONE); @@ -537,6 +569,7 @@ rcv_settings_frame(Http2ConnectionState &cstate, const Http2Frame &frame) cstate.increment_received_settings_frame_count(); // Close this conection if its SETTINGS frame count exceeds a limit if (cstate.get_received_settings_frame_count() > Http2::max_settings_frames_per_minute) { + HTTP2_INCREMENT_THREAD_DYN_STAT(HTTP2_STAT_MAX_SETTINGS_FRAMES_PER_MINUTE_EXCEEDED, this_ethread()); Http2StreamDebug(cstate.ua_session, stream_id, "Observed too frequent SETTINGS frames: %u frames within a last minute", cstate.get_received_settings_frame_count()); return Http2Error(Http2ErrorClass::HTTP2_ERROR_CLASS_CONNECTION, Http2ErrorCode::HTTP2_ERROR_ENHANCE_YOUR_CALM, @@ -575,6 +608,7 @@ rcv_settings_frame(Http2ConnectionState &cstate, const Http2Frame &frame) uint32_t n_settings = 0; while (nbytes < frame.header().length) { if (n_settings >= Http2::max_settings_per_frame) { + HTTP2_INCREMENT_THREAD_DYN_STAT(HTTP2_STAT_MAX_SETTINGS_PER_FRAME_EXCEEDED, this_ethread()); Http2StreamDebug(cstate.ua_session, stream_id, "Observed too many settings in a frame"); return Http2Error(Http2ErrorClass::HTTP2_ERROR_CLASS_CONNECTION, Http2ErrorCode::HTTP2_ERROR_ENHANCE_YOUR_CALM, "recv settings too many settings in a frame"); @@ -615,6 +649,7 @@ rcv_settings_frame(Http2ConnectionState &cstate, const Http2Frame &frame) cstate.increment_received_settings_count(n_settings); // Close this conection if its settings count received exceeds a limit if (cstate.get_received_settings_count() > Http2::max_settings_per_minute) { + HTTP2_INCREMENT_THREAD_DYN_STAT(HTTP2_STAT_MAX_SETTINGS_PER_MINUTE_EXCEEDED, this_ethread()); Http2StreamDebug(cstate.ua_session, stream_id, "Observed too frequent setting changes: %u settings within a last minute", cstate.get_received_settings_count()); return Http2Error(Http2ErrorClass::HTTP2_ERROR_CLASS_CONNECTION, Http2ErrorCode::HTTP2_ERROR_ENHANCE_YOUR_CALM, @@ -623,8 +658,8 @@ rcv_settings_frame(Http2ConnectionState &cstate, const Http2Frame &frame) // [RFC 7540] 6.5. Once all values have been applied, the recipient MUST // immediately emit a SETTINGS frame with the ACK flag set. - Http2Frame ackFrame(HTTP2_FRAME_TYPE_SETTINGS, 0, HTTP2_FLAGS_SETTINGS_ACK); - cstate.ua_session->handleEvent(HTTP2_SESSION_EVENT_XMIT, &ackFrame); + Http2SettingsFrame ack_frame(0, HTTP2_FLAGS_SETTINGS_ACK); + cstate.ua_session->xmit(ack_frame); return Http2Error(Http2ErrorClass::HTTP2_ERROR_CLASS_NONE); } @@ -668,6 +703,7 @@ rcv_ping_frame(Http2ConnectionState &cstate, const Http2Frame &frame) cstate.increment_received_ping_frame_count(); // Close this conection if its ping count received exceeds a limit if (cstate.get_received_ping_frame_count() > Http2::max_ping_frames_per_minute) { + HTTP2_INCREMENT_THREAD_DYN_STAT(HTTP2_STAT_MAX_PING_FRAMES_PER_MINUTE_EXCEEDED, this_ethread()); Http2StreamDebug(cstate.ua_session, stream_id, "Observed too frequent PING frames: %u PING frames within a last minute", cstate.get_received_ping_frame_count()); return Http2Error(Http2ErrorClass::HTTP2_ERROR_CLASS_CONNECTION, Http2ErrorCode::HTTP2_ERROR_ENHANCE_YOUR_CALM, @@ -865,21 +901,20 @@ rcv_continuation_frame(Http2ConnectionState &cstate, const Http2Frame &frame) } } - // keep track of how many bytes we get in the frame - stream->request_header_length += payload_length; - if (stream->request_header_length > Http2::max_header_list_size) { - return Http2Error(Http2ErrorClass::HTTP2_ERROR_CLASS_CONNECTION, Http2ErrorCode::HTTP2_ERROR_PROTOCOL_ERROR, - "continuation payload for headers exceeded"); - } - uint32_t header_blocks_offset = stream->header_blocks_length; stream->header_blocks_length += payload_length; - if (stream->header_blocks_length > 0) { - stream->header_blocks = static_cast(ats_realloc(stream->header_blocks, stream->header_blocks_length)); - frame.reader()->memcpy(stream->header_blocks + header_blocks_offset, payload_length); + // ATS advertises SETTINGS_MAX_HEADER_LIST_SIZE as a limit of total header blocks length. (Details in [RFC 7560] 10.5.1.) + // Make it double to relax the limit in cases of 1) HPACK is used naively, or 2) Huffman Encoding generates large header blocks. + // The total "decoded" header length is strictly checked by hpack_decode_header_block(). + if (stream->header_blocks_length > std::max(Http2::max_header_list_size, Http2::max_header_list_size * 2)) { + return Http2Error(Http2ErrorClass::HTTP2_ERROR_CLASS_CONNECTION, Http2ErrorCode::HTTP2_ERROR_ENHANCE_YOUR_CALM, + "header blocks too large"); } + stream->header_blocks = static_cast(ats_realloc(stream->header_blocks, stream->header_blocks_length)); + frame.reader()->memcpy(stream->header_blocks + header_blocks_offset, payload_length); + if (frame.header().flags & HTTP2_FLAGS_HEADERS_END_HEADERS) { // NOTE: If there are END_HEADERS flag, decode stored Header Blocks. cstate.clear_continued_stream_id(); @@ -908,7 +943,9 @@ rcv_continuation_frame(Http2ConnectionState &cstate, const Http2Frame &frame) // Set up the State Machine SCOPED_MUTEX_LOCK(stream_lock, stream->mutex, this_ethread()); stream->mark_milestone(Http2StreamMilestone::START_TXN); - stream->new_transaction(); + // This should be fine, need to verify whether we need to replace this with the + // "from_early_data" flag from the associated HEADERS frame. + stream->new_transaction(frame.is_from_early_data()); // Send request header to SM stream->send_request(cstate); } else { @@ -978,8 +1015,8 @@ Http2ConnectionState::main_event_handler(int event, void *edata) ink_assert(this->fini_received == false); this->fini_received = true; cleanup_streams(); + release_stream(); SET_HANDLER(&Http2ConnectionState::state_closed); - this->release_stream(nullptr); } break; case HTTP2_SESSION_EVENT_XMIT: { @@ -1003,6 +1040,25 @@ Http2ConnectionState::main_event_handler(int event, void *edata) break; } + // We need to be careful here, certain frame types are not safe over 0-rtt, tentative for now. + // DATA: NO + // HEADERS: YES (safe http methods only, can only be checked after parsing the payload). + // PRIORITY: YES + // RST_STREAM: NO + // SETTINGS: YES + // PUSH_PROMISE: NO + // PING: YES + // GOAWAY: NO + // WINDOW_UPDATE: YES + // CONTINUATION: YES (safe http methods only, same as HEADERS frame). + if (frame->is_from_early_data() && + (frame->header().type == HTTP2_FRAME_TYPE_DATA || frame->header().type == HTTP2_FRAME_TYPE_RST_STREAM || + frame->header().type == HTTP2_FRAME_TYPE_PUSH_PROMISE || frame->header().type == HTTP2_FRAME_TYPE_GOAWAY)) { + Http2StreamDebug(ua_session, stream_id, "Discard a frame which is received from early data and has type=%x", + frame->header().type); + break; + } + if (frame_handlers[frame->header().type]) { error = frame_handlers[frame->header().type](*this, *frame); } else { @@ -1107,6 +1163,13 @@ Http2ConnectionState::state_closed(int event, void *edata) Http2Stream * Http2ConnectionState::create_stream(Http2StreamId new_id, Http2Error &error) { + // first check if we've hit the active connection limit + if (!ua_session->get_netvc()->add_to_active_queue()) { + error = Http2Error(Http2ErrorClass::HTTP2_ERROR_CLASS_CONNECTION, Http2ErrorCode::HTTP2_ERROR_NO_ERROR, + "refused to create new stream, maxed out active connections"); + return nullptr; + } + // In half_close state, TS doesn't create new stream. Because GOAWAY frame is sent to client if (ua_session->get_half_close_local_flag()) { error = Http2Error(Http2ErrorClass::HTTP2_ERROR_CLASS_STREAM, Http2ErrorCode::HTTP2_ERROR_REFUSED_STREAM, @@ -1180,7 +1243,6 @@ Http2ConnectionState::create_stream(Http2StreamId new_id, Http2Error &error) new_stream->mutex = new_ProxyMutex(); new_stream->is_first_transaction_flag = get_stream_requests() == 0; increment_stream_requests(); - ua_session->get_netvc()->add_to_active_queue(); return new_stream; } @@ -1236,6 +1298,35 @@ Http2ConnectionState::restart_streams() } } +void +Http2ConnectionState::restart_receiving(Http2Stream *stream) +{ + uint32_t initial_rwnd = this->server_settings.get(HTTP2_SETTINGS_INITIAL_WINDOW_SIZE); + uint32_t min_rwnd = std::min(initial_rwnd, this->server_settings.get(HTTP2_SETTINGS_MAX_FRAME_SIZE)); + + // Connection level WINDOW UPDATE + if (this->server_rwnd() < min_rwnd) { + Http2WindowSize diff_size = initial_rwnd - this->server_rwnd(); + this->increment_server_rwnd(diff_size); + this->send_window_update_frame(0, diff_size); + } + + // Stream level WINDOW UPDATE + if (stream == nullptr || stream->server_rwnd() >= min_rwnd) { + return; + } + + // If read_vio is buffering data, do not fully update window + int64_t data_size = stream->read_vio_read_avail(); + if (data_size >= initial_rwnd) { + return; + } + + Http2WindowSize diff_size = initial_rwnd - std::max(static_cast(stream->server_rwnd()), data_size); + stream->increment_server_rwnd(diff_size); + this->send_window_update_frame(stream->get_id(), diff_size); +} + void Http2ConnectionState::cleanup_streams() { @@ -1248,13 +1339,11 @@ Http2ConnectionState::cleanup_streams() if (this->tx_error_code.cls != ProxyErrorClass::NONE) { s->set_tx_error_code(this->tx_error_code); } - this->delete_stream(s); + s->initiating_close(); ink_assert(s != next); s = next; } - ink_assert(stream_list.empty()); - if (!is_state_closed()) { SCOPED_MUTEX_LOCK(lock, this->ua_session->mutex, this_ethread()); @@ -1317,22 +1406,18 @@ Http2ConnectionState::delete_stream(Http2Stream *stream) } void -Http2ConnectionState::release_stream(Http2Stream *stream) +Http2ConnectionState::release_stream() { REMEMBER(NO_EVENT, this->recursion) - if (stream) { - // Decrement total_client_streams_count here, because it's a counter include streams in the process of shutting down. - // Other counters (client_streams_in_count/client_streams_out_count) are already decremented in delete_stream(). - --total_client_streams_count; - } - SCOPED_MUTEX_LOCK(lock, this->mutex, this_ethread()); if (this->ua_session) { ink_assert(this->mutex == ua_session->mutex); if (total_client_streams_count == 0) { if (fini_received) { + ua_session->clear_session_active(); + // We were shutting down, go ahead and terminate the session // this is a member of Http2ConnectionState and will be freed // when ua_session is destroyed @@ -1342,6 +1427,7 @@ Http2ConnectionState::release_stream(Http2Stream *stream) // or we can use a local variable to do it. // ua_session = nullptr; } else if (shutdown_state == HTTP2_SHUTDOWN_IN_PROGRESS && fini_event == nullptr) { + ua_session->clear_session_active(); fini_event = this_ethread()->schedule_imm_local((Continuation *)this, HTTP2_SESSION_EVENT_FINI); } else if (ua_session->is_active()) { // If the number of clients is 0, HTTP2_SESSION_EVENT_FINI is not received or sent, and ua_session is active, @@ -1410,7 +1496,7 @@ Http2ConnectionState::send_data_frames_depends_on_priority() switch (result) { case Http2SendDataFrameResult::NO_ERROR: { // No response body to send - if (len == 0 && !stream->is_body_done()) { + if (len == 0 && !stream->is_write_vio_done()) { dependency_tree->deactivate(node, len); } else { dependency_tree->update(node, len); @@ -1422,7 +1508,7 @@ Http2ConnectionState::send_data_frames_depends_on_priority() } case Http2SendDataFrameResult::DONE: { dependency_tree->deactivate(node, len); - delete_stream(stream); + stream->initiating_close(); break; } default: @@ -1443,27 +1529,34 @@ Http2ConnectionState::send_a_data_frame(Http2Stream *stream, size_t &payload_len const size_t write_available_size = std::min(buf_len, static_cast(window_size)); payload_length = 0; - uint8_t flags = 0x00; - uint8_t payload_buffer[buf_len]; - IOBufferReader *current_reader = stream->response_get_data_reader(); + uint8_t flags = 0x00; + IOBufferReader *resp_reader = stream->response_get_data_reader(); SCOPED_MUTEX_LOCK(stream_lock, stream->mutex, this_ethread()); - if (!current_reader) { + if (!resp_reader) { Http2StreamDebug(this->ua_session, stream->get_id(), "couldn't get data reader"); return Http2SendDataFrameResult::ERROR; } + if (this->ua_session->write_avail() == 0) { + Http2StreamDebug(this->ua_session, stream->get_id(), "Not write avail"); + return Http2SendDataFrameResult::NOT_WRITE_AVAIL; + } + // Select appropriate payload length - if (current_reader->is_read_avail_more_than(0)) { + if (resp_reader->is_read_avail_more_than(0)) { // We only need to check for window size when there is a payload if (window_size <= 0) { Http2StreamDebug(this->ua_session, stream->get_id(), "No window"); return Http2SendDataFrameResult::NO_WINDOW; } - // Copy into the payload buffer. Seems like we should be able to skip this copy step - payload_length = write_available_size; - payload_length = current_reader->read(payload_buffer, static_cast(write_available_size)); + + if (resp_reader->is_read_avail_more_than(write_available_size)) { + payload_length = write_available_size; + } else { + payload_length = resp_reader->read_avail(); + } } else { payload_length = 0; } @@ -1471,12 +1564,12 @@ Http2ConnectionState::send_a_data_frame(Http2Stream *stream, size_t &payload_len // Are we at the end? // If we return here, we never send the END_STREAM in the case of a early terminating OS. // OK if there is no body yet. Otherwise continue on to send a DATA frame and delete the stream - if (!stream->is_body_done() && payload_length == 0) { + if (!stream->is_write_vio_done() && payload_length == 0) { Http2StreamDebug(this->ua_session, stream->get_id(), "No payload"); return Http2SendDataFrameResult::NO_PAYLOAD; } - if (stream->is_body_done() && !current_reader->is_read_avail_more_than(0)) { + if (stream->is_write_vio_done() && !resp_reader->is_read_avail_more_than(0)) { flags |= HTTP2_FLAGS_DATA_END_STREAM; } @@ -1488,22 +1581,16 @@ Http2ConnectionState::send_a_data_frame(Http2Stream *stream, size_t &payload_len Http2StreamDebug(ua_session, stream->get_id(), "Send a DATA frame - client window con: %5zd stream: %5zd payload: %5zd", _client_rwnd, stream->client_rwnd(), payload_length); - Http2Frame data(HTTP2_FRAME_TYPE_DATA, stream->get_id(), flags); - data.alloc(buffer_size_index[HTTP2_FRAME_TYPE_DATA]); - http2_write_data(payload_buffer, payload_length, data.write()); - data.finalize(payload_length); + Http2DataFrame data(stream->get_id(), flags, resp_reader, payload_length); + this->ua_session->xmit(data); stream->update_sent_count(payload_length); - // xmit event - SCOPED_MUTEX_LOCK(lock, this->ua_session->mutex, this_ethread()); - this->ua_session->handleEvent(HTTP2_SESSION_EVENT_XMIT, &data); - if (flags & HTTP2_FLAGS_DATA_END_STREAM) { - Http2StreamDebug(ua_session, stream->get_id(), "End of DATA frame"); + Http2StreamDebug(ua_session, stream->get_id(), "END_STREAM"); stream->send_end_stream = true; // Setting to the same state shouldn't be erroneous - stream->change_state(data.header().type, data.header().flags); + stream->change_state(HTTP2_FRAME_TYPE_DATA, flags); return Http2SendDataFrameResult::DONE; } @@ -1519,7 +1606,7 @@ Http2ConnectionState::send_data_frames(Http2Stream *stream) if (stream->get_state() == Http2StreamState::HTTP2_STREAM_STATE_HALF_CLOSED_LOCAL || stream->get_state() == Http2StreamState::HTTP2_STREAM_STATE_CLOSED) { Http2StreamDebug(this->ua_session, stream->get_id(), "Shutdown half closed local stream"); - this->delete_stream(stream); + stream->initiating_close(); return; } @@ -1534,7 +1621,7 @@ Http2ConnectionState::send_data_frames(Http2Stream *stream) // RST_STREAM and WINDOW_UPDATE. // See 'closed' state written at [RFC 7540] 5.1. Http2StreamDebug(this->ua_session, stream->get_id(), "Shutdown stream"); - this->delete_stream(stream); + stream->initiating_close(); } } @@ -1548,28 +1635,23 @@ Http2ConnectionState::send_headers_frame(Http2Stream *stream) uint32_t buf_len = 0; uint32_t header_blocks_size = 0; int payload_length = 0; - uint64_t sent = 0; uint8_t flags = 0x00; - HTTPHdr *resp_header = &stream->response_header; - Http2StreamDebug(ua_session, stream->get_id(), "Send HEADERS frame"); - HTTPHdr h2_hdr; - http2_generate_h2_header_from_1_1(resp_header, &h2_hdr); + HTTPHdr *resp_hdr = &stream->response_header; + http2_convert_header_from_1_1_to_2(resp_hdr); - buf_len = resp_header->length_get() * 2; // Make it double just in case + buf_len = resp_hdr->length_get() * 2; // Make it double just in case buf = static_cast(ats_malloc(buf_len)); if (buf == nullptr) { - h2_hdr.destroy(); return; } stream->mark_milestone(Http2StreamMilestone::START_ENCODE_HEADERS); - Http2ErrorCode result = http2_encode_header_blocks(&h2_hdr, buf, buf_len, &header_blocks_size, *(this->remote_hpack_handle), + Http2ErrorCode result = http2_encode_header_blocks(resp_hdr, buf, buf_len, &header_blocks_size, *(this->remote_hpack_handle), client_settings.get(HTTP2_SETTINGS_HEADER_TABLE_SIZE)); if (result != Http2ErrorCode::HTTP2_ERROR_NO_ERROR) { - h2_hdr.destroy(); ats_free(buf); return; } @@ -1578,7 +1660,9 @@ Http2ConnectionState::send_headers_frame(Http2Stream *stream) if (header_blocks_size <= static_cast(BUFFER_SIZE_FOR_INDEX(buffer_size_index[HTTP2_FRAME_TYPE_HEADERS]))) { payload_length = header_blocks_size; flags |= HTTP2_FLAGS_HEADERS_END_HEADERS; - if (h2_hdr.presence(MIME_PRESENCE_CONTENT_LENGTH) && h2_hdr.get_content_length() == 0) { + if ((resp_hdr->presence(MIME_PRESENCE_CONTENT_LENGTH) && resp_hdr->get_content_length() == 0) || + (!resp_hdr->expect_final_response() && stream->is_write_vio_done())) { + Http2StreamDebug(ua_session, stream->get_id(), "END_STREAM"); flags |= HTTP2_FLAGS_HEADERS_END_STREAM; stream->send_end_stream = true; } @@ -1586,10 +1670,6 @@ Http2ConnectionState::send_headers_frame(Http2Stream *stream) } else { payload_length = BUFFER_SIZE_FOR_INDEX(buffer_size_index[HTTP2_FRAME_TYPE_HEADERS]); } - Http2Frame headers(HTTP2_FRAME_TYPE_HEADERS, stream->get_id(), flags); - headers.alloc(buffer_size_index[HTTP2_FRAME_TYPE_HEADERS]); - http2_write_headers(buf, payload_length, headers.write()); - headers.finalize(payload_length); // Change stream state if (!stream->change_state(HTTP2_FRAME_TYPE_HEADERS, flags)) { @@ -1599,15 +1679,13 @@ Http2ConnectionState::send_headers_frame(Http2Stream *stream) fini_event = this_ethread()->schedule_imm_local((Continuation *)this, HTTP2_SESSION_EVENT_FINI); } - h2_hdr.destroy(); ats_free(buf); return; } - // xmit event - SCOPED_MUTEX_LOCK(lock, this->ua_session->mutex, this_ethread()); - this->ua_session->handleEvent(HTTP2_SESSION_EVENT_XMIT, &headers); - sent += payload_length; + Http2HeadersFrame headers(stream->get_id(), flags, buf, payload_length); + this->ua_session->xmit(headers); + uint64_t sent = payload_length; // Send CONTINUATION frames flags = 0; @@ -1618,71 +1696,62 @@ Http2ConnectionState::send_headers_frame(Http2Stream *stream) if (sent + payload_length == header_blocks_size) { flags |= HTTP2_FLAGS_CONTINUATION_END_HEADERS; } - Http2Frame continuation_frame(HTTP2_FRAME_TYPE_CONTINUATION, stream->get_id(), flags); - continuation_frame.alloc(buffer_size_index[HTTP2_FRAME_TYPE_CONTINUATION]); - http2_write_headers(buf + sent, payload_length, continuation_frame.write()); - continuation_frame.finalize(payload_length); - stream->change_state(continuation_frame.header().type, continuation_frame.header().flags); - // xmit event - SCOPED_MUTEX_LOCK(lock, this->ua_session->mutex, this_ethread()); - this->ua_session->handleEvent(HTTP2_SESSION_EVENT_XMIT, &continuation_frame); + stream->change_state(HTTP2_FRAME_TYPE_CONTINUATION, flags); + + Http2ContinuationFrame continuation_frame(stream->get_id(), flags, buf + sent, payload_length); + this->ua_session->xmit(continuation_frame); sent += payload_length; } - h2_hdr.destroy(); ats_free(buf); } -void +bool Http2ConnectionState::send_push_promise_frame(Http2Stream *stream, URL &url, const MIMEField *accept_encoding) { - HTTPHdr h1_hdr, h2_hdr; uint8_t *buf = nullptr; uint32_t buf_len = 0; uint32_t header_blocks_size = 0; int payload_length = 0; - uint64_t sent = 0; uint8_t flags = 0x00; if (client_settings.get(HTTP2_SETTINGS_ENABLE_PUSH) == 0) { - return; + return false; } Http2StreamDebug(ua_session, stream->get_id(), "Send PUSH_PROMISE frame"); - h1_hdr.create(HTTP_TYPE_REQUEST); - h1_hdr.url_set(&url); - h1_hdr.method_set("GET", 3); + HTTPHdr hdr; + ts::PostScript hdr_defer([&]() -> void { hdr.destroy(); }); + hdr.create(HTTP_TYPE_REQUEST); + http2_init_pseudo_headers(hdr); + hdr.url_set(&url); + hdr.method_set(HTTP_METHOD_GET, HTTP_LEN_GET); + if (accept_encoding != nullptr) { - MIMEField *f; - const char *name; int name_len; - const char *value; - int value_len; + const char *name = accept_encoding->name_get(&name_len); + MIMEField *f = hdr.field_create(name, name_len); - name = accept_encoding->name_get(&name_len); - f = h1_hdr.field_create(name, name_len); - value = accept_encoding->value_get(&value_len); - f->value_set(h1_hdr.m_heap, h1_hdr.m_mime, value, value_len); + int value_len; + const char *value = accept_encoding->value_get(&value_len); + f->value_set(hdr.m_heap, hdr.m_mime, value, value_len); - h1_hdr.field_attach(f); + hdr.field_attach(f); } - http2_generate_h2_header_from_1_1(&h1_hdr, &h2_hdr); + http2_convert_header_from_1_1_to_2(&hdr); - buf_len = h1_hdr.length_get() * 2; // Make it double just in case - h1_hdr.destroy(); - buf = static_cast(ats_malloc(buf_len)); + buf_len = hdr.length_get() * 2; // Make it double just in case + buf = static_cast(ats_malloc(buf_len)); if (buf == nullptr) { - h2_hdr.destroy(); - return; + return false; } - Http2ErrorCode result = http2_encode_header_blocks(&h2_hdr, buf, buf_len, &header_blocks_size, *(this->remote_hpack_handle), + Http2ErrorCode result = http2_encode_header_blocks(&hdr, buf, buf_len, &header_blocks_size, *(this->remote_hpack_handle), client_settings.get(HTTP2_SETTINGS_HEADER_TABLE_SIZE)); if (result != Http2ErrorCode::HTTP2_ERROR_NO_ERROR) { - h2_hdr.destroy(); ats_free(buf); - return; + return false; } // Send a PUSH_PROMISE frame @@ -1695,16 +1764,13 @@ Http2ConnectionState::send_push_promise_frame(Http2Stream *stream, URL &url, con payload_length = BUFFER_SIZE_FOR_INDEX(buffer_size_index[HTTP2_FRAME_TYPE_PUSH_PROMISE]) - sizeof(push_promise.promised_streamid); } - Http2Frame push_promise_frame(HTTP2_FRAME_TYPE_PUSH_PROMISE, stream->get_id(), flags); - push_promise_frame.alloc(buffer_size_index[HTTP2_FRAME_TYPE_PUSH_PROMISE]); + Http2StreamId id = this->get_latest_stream_id_out() + 2; push_promise.promised_streamid = id; - http2_write_push_promise(push_promise, buf, payload_length, push_promise_frame.write()); - push_promise_frame.finalize(sizeof(push_promise.promised_streamid) + payload_length); - // xmit event - SCOPED_MUTEX_LOCK(lock, this->ua_session->mutex, this_ethread()); - this->ua_session->handleEvent(HTTP2_SESSION_EVENT_XMIT, &push_promise_frame); - sent += payload_length; + + Http2PushPromiseFrame push_promise_frame(stream->get_id(), flags, push_promise, buf, payload_length); + this->ua_session->xmit(push_promise_frame); + uint64_t sent = payload_length; // Send CONTINUATION frames flags = 0; @@ -1715,13 +1781,9 @@ Http2ConnectionState::send_push_promise_frame(Http2Stream *stream, URL &url, con if (sent + payload_length == header_blocks_size) { flags |= HTTP2_FLAGS_CONTINUATION_END_HEADERS; } - Http2Frame continuation_frame(HTTP2_FRAME_TYPE_CONTINUATION, stream->get_id(), flags); - continuation_frame.alloc(buffer_size_index[HTTP2_FRAME_TYPE_CONTINUATION]); - http2_write_headers(buf + sent, payload_length, continuation_frame.write()); - continuation_frame.finalize(payload_length); - // xmit event - SCOPED_MUTEX_LOCK(lock, this->ua_session->mutex, this_ethread()); - this->ua_session->handleEvent(HTTP2_SESSION_EVENT_XMIT, &continuation_frame); + + Http2ContinuationFrame continuation(stream->get_id(), flags, buf + sent, payload_length); + this->ua_session->xmit(continuation); sent += payload_length; } ats_free(buf); @@ -1729,8 +1791,7 @@ Http2ConnectionState::send_push_promise_frame(Http2Stream *stream, URL &url, con Http2Error error(Http2ErrorClass::HTTP2_ERROR_CLASS_NONE); stream = this->create_stream(id, error); if (!stream) { - h2_hdr.destroy(); - return; + return false; } SCOPED_MUTEX_LOCK(stream_lock, stream->mutex, this_ethread()); @@ -1748,11 +1809,11 @@ Http2ConnectionState::send_push_promise_frame(Http2Stream *stream, URL &url, con } } stream->change_state(HTTP2_FRAME_TYPE_PUSH_PROMISE, HTTP2_FLAGS_PUSH_PROMISE_END_HEADERS); - stream->set_request_headers(h2_hdr); + stream->set_request_headers(hdr); stream->new_transaction(); stream->send_request(*this); - h2_hdr.destroy(); + return true; } void @@ -1765,12 +1826,6 @@ Http2ConnectionState::send_rst_stream_frame(Http2StreamId id, Http2ErrorCode ec) ++stream_error_count; } - Http2Frame rst_stream(HTTP2_FRAME_TYPE_RST_STREAM, id, 0); - - rst_stream.alloc(buffer_size_index[HTTP2_FRAME_TYPE_RST_STREAM]); - http2_write_rst_stream(static_cast(ec), rst_stream.write()); - rst_stream.finalize(HTTP2_RST_STREAM_LEN); - // change state to closed Http2Stream *stream = find_stream(id); if (stream != nullptr) { @@ -1786,9 +1841,8 @@ Http2ConnectionState::send_rst_stream_frame(Http2StreamId id, Http2ErrorCode ec) } } - // xmit event - SCOPED_MUTEX_LOCK(lock, this->ua_session->mutex, this_ethread()); - this->ua_session->handleEvent(HTTP2_SESSION_EVENT_XMIT, &rst_stream); + Http2RstStreamFrame rst_stream(id, static_cast(ec)); + this->ua_session->xmit(rst_stream); } void @@ -1798,11 +1852,8 @@ Http2ConnectionState::send_settings_frame(const Http2ConnectionSettings &new_set Http2StreamDebug(ua_session, stream_id, "Send SETTINGS frame"); - Http2Frame settings(HTTP2_FRAME_TYPE_SETTINGS, stream_id, 0); - settings.alloc(buffer_size_index[HTTP2_FRAME_TYPE_SETTINGS]); - - IOVec iov = settings.write(); - uint32_t settings_length = 0; + Http2SettingsParameter params[HTTP2_SETTINGS_MAX]; + size_t params_size = 0; for (int i = HTTP2_SETTINGS_HEADER_TABLE_SIZE; i < HTTP2_SETTINGS_MAX; ++i) { Http2SettingsIdentifier id = static_cast(i); @@ -1810,32 +1861,17 @@ Http2ConnectionState::send_settings_frame(const Http2ConnectionSettings &new_set // Send only difference if (settings_value != server_settings.get(id)) { - const Http2SettingsParameter param = {static_cast(id), settings_value}; + Http2StreamDebug(ua_session, stream_id, " %s : %u", Http2DebugNames::get_settings_param_name(id), settings_value); - // Write settings to send buffer - if (!http2_write_settings(param, iov)) { - this->send_goaway_frame(this->latest_streamid_in, Http2ErrorCode::HTTP2_ERROR_INTERNAL_ERROR); - this->ua_session->set_half_close_local_flag(true); - if (fini_event == nullptr) { - fini_event = this_ethread()->schedule_imm_local((Continuation *)this, HTTP2_SESSION_EVENT_FINI); - } - - return; - } - iov.iov_base = reinterpret_cast(iov.iov_base) + HTTP2_SETTINGS_PARAMETER_LEN; - iov.iov_len -= HTTP2_SETTINGS_PARAMETER_LEN; - settings_length += HTTP2_SETTINGS_PARAMETER_LEN; + params[params_size++] = {static_cast(id), settings_value}; // Update current settings server_settings.set(id, new_settings.get(id)); - - Http2StreamDebug(ua_session, stream_id, " %s : %u", Http2DebugNames::get_settings_param_name(param.id), param.value); } } - settings.finalize(settings_length); - SCOPED_MUTEX_LOCK(lock, this->ua_session->mutex, this_ethread()); - this->ua_session->handleEvent(HTTP2_SESSION_EVENT_XMIT, &settings); + Http2SettingsFrame settings(stream_id, HTTP2_FRAME_NO_FLAG, params, params_size); + this->ua_session->xmit(settings); } void @@ -1843,15 +1879,8 @@ Http2ConnectionState::send_ping_frame(Http2StreamId id, uint8_t flag, const uint { Http2StreamDebug(ua_session, id, "Send PING frame"); - Http2Frame ping(HTTP2_FRAME_TYPE_PING, id, flag); - - ping.alloc(buffer_size_index[HTTP2_FRAME_TYPE_PING]); - http2_write_ping(opaque_data, ping.write()); - ping.finalize(HTTP2_PING_LEN); - - // xmit event - SCOPED_MUTEX_LOCK(lock, this->ua_session->mutex, this_ethread()); - this->ua_session->handleEvent(HTTP2_SESSION_EVENT_XMIT, &ping); + Http2PingFrame ping(id, flag, opaque_data); + this->ua_session->xmit(ping); } // As for gracefull shutdown, TS should process outstanding stream as long as possible. @@ -1867,37 +1896,24 @@ Http2ConnectionState::send_goaway_frame(Http2StreamId id, Http2ErrorCode ec) HTTP2_INCREMENT_THREAD_DYN_STAT(HTTP2_STAT_CONNECTION_ERRORS_COUNT, this_ethread()); } - Http2Frame frame(HTTP2_FRAME_TYPE_GOAWAY, 0, 0); - Http2Goaway goaway; + this->tx_error_code = {ProxyErrorClass::SSN, static_cast(ec)}; + Http2Goaway goaway; goaway.last_streamid = id; goaway.error_code = ec; - frame.alloc(buffer_size_index[HTTP2_FRAME_TYPE_GOAWAY]); - http2_write_goaway(goaway, frame.write()); - frame.finalize(HTTP2_GOAWAY_LEN); - - this->tx_error_code = {ProxyErrorClass::SSN, static_cast(ec)}; - - // xmit event - SCOPED_MUTEX_LOCK(lock, this->ua_session->mutex, this_ethread()); - this->ua_session->handleEvent(HTTP2_SESSION_EVENT_XMIT, &frame); + Http2GoawayFrame frame(goaway); + this->ua_session->xmit(frame); } void Http2ConnectionState::send_window_update_frame(Http2StreamId id, uint32_t size) { - Http2StreamDebug(ua_session, id, "Send WINDOW_UPDATE frame"); + Http2StreamDebug(ua_session, id, "Send WINDOW_UPDATE frame: size=%" PRIu32, size); // Create WINDOW_UPDATE frame - Http2Frame window_update(HTTP2_FRAME_TYPE_WINDOW_UPDATE, id, 0x0); - window_update.alloc(buffer_size_index[HTTP2_FRAME_TYPE_WINDOW_UPDATE]); - http2_write_window_update(static_cast(size), window_update.write()); - window_update.finalize(sizeof(uint32_t)); - - // xmit event - SCOPED_MUTEX_LOCK(lock, this->ua_session->mutex, this_ethread()); - this->ua_session->handleEvent(HTTP2_SESSION_EVENT_XMIT, &window_update); + Http2WindowUpdateFrame window_update(id, size); + this->ua_session->xmit(window_update); } void @@ -1998,6 +2014,7 @@ Http2ConnectionState::increment_client_rwnd(size_t amount) double sum = std::accumulate(this->_recent_rwnd_increment.begin(), this->_recent_rwnd_increment.end(), 0.0); double avg = sum / this->_recent_rwnd_increment.size(); if (avg < Http2::min_avg_window_update) { + HTTP2_INCREMENT_THREAD_DYN_STAT(HTTP2_STAT_INSUFFICIENT_AVG_WINDOW_UPDATE, this_ethread()); return Http2ErrorCode::HTTP2_ERROR_ENHANCE_YOUR_CALM; } return Http2ErrorCode::HTTP2_ERROR_NO_ERROR; diff --git a/proxy/http2/Http2ConnectionState.h b/proxy/http2/Http2ConnectionState.h index 01c80cfa21c..a31cf17e924 100644 --- a/proxy/http2/Http2ConnectionState.h +++ b/proxy/http2/Http2ConnectionState.h @@ -36,6 +36,7 @@ enum class Http2SendDataFrameResult { NO_ERROR = 0, NO_WINDOW, NO_PAYLOAD, + NOT_WRITE_AVAIL, ERROR, DONE, }; @@ -130,6 +131,8 @@ class Http2ConnectionState : public Continuation void init() { + this->_server_rwnd = Http2::initial_window_size; + local_hpack_handle = new HpackHandle(HTTP2_HEADER_TABLE_SIZE); remote_hpack_handle = new HpackHandle(HTTP2_HEADER_TABLE_SIZE); if (Http2::stream_priority_enabled) { @@ -140,8 +143,14 @@ class Http2ConnectionState : public Continuation void destroy() { + if (in_destroy) { + schedule_zombie_event(); + return; + } + in_destroy = true; if (shutdown_cont_event) { shutdown_cont_event->cancel(); + shutdown_cont_event = nullptr; } cleanup_streams(); @@ -172,9 +181,9 @@ class Http2ConnectionState : public Continuation Http2Stream *find_stream(Http2StreamId id) const; void restart_streams(); bool delete_stream(Http2Stream *stream); - void release_stream(Http2Stream *stream); + void release_stream(); void cleanup_streams(); - + void restart_receiving(Http2Stream *stream); void update_initial_rwnd(Http2WindowSize new_size); Http2StreamId @@ -224,11 +233,18 @@ class Http2ConnectionState : public Continuation return client_streams_in_count; } + void + decrement_stream_count() + { + --total_client_streams_count; + } + double get_stream_error_rate() const { int total = get_stream_requests(); - if (total > 0) { + + if (total >= (1 / Http2::stream_error_rate_threshold)) { return (double)stream_error_count / (double)total; } else { return 0; @@ -247,7 +263,7 @@ class Http2ConnectionState : public Continuation void send_data_frames(Http2Stream *stream); Http2SendDataFrameResult send_a_data_frame(Http2Stream *stream, size_t &payload_length); void send_headers_frame(Http2Stream *stream); - void send_push_promise_frame(Http2Stream *stream, URL &url, const MIMEField *accept_encoding); + bool send_push_promise_frame(Http2Stream *stream, URL &url, const MIMEField *accept_encoding); void send_rst_stream_frame(Http2StreamId id, Http2ErrorCode ec); void send_settings_frame(const Http2ConnectionSettings &new_settings); void send_ping_frame(Http2StreamId id, uint8_t flag, const uint8_t *opaque_data); @@ -354,7 +370,7 @@ class Http2ConnectionState : public Continuation // Connection level window size ssize_t _client_rwnd = HTTP2_INITIAL_WINDOW_SIZE; - ssize_t _server_rwnd = Http2::initial_window_size; + ssize_t _server_rwnd = 0; std::vector _recent_rwnd_increment = {SIZE_MAX, SIZE_MAX, SIZE_MAX, SIZE_MAX, SIZE_MAX}; int _recent_rwnd_increment_index = 0; @@ -374,6 +390,7 @@ class Http2ConnectionState : public Continuation Http2StreamId continued_stream_id = 0; bool _scheduled = false; bool fini_received = false; + bool in_destroy = false; int recursion = 0; Http2ShutdownState shutdown_state = HTTP2_SHUTDOWN_NONE; Http2ErrorCode shutdown_reason = Http2ErrorCode::HTTP2_ERROR_MAX; diff --git a/proxy/http2/Http2Frame.cc b/proxy/http2/Http2Frame.cc new file mode 100644 index 00000000000..a731f09e2a8 --- /dev/null +++ b/proxy/http2/Http2Frame.cc @@ -0,0 +1,253 @@ +/** @file + + Http2Frame + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#include "Http2Frame.h" + +// +// Http2Frame +// +IOBufferReader * +Http2Frame::reader() const +{ + return this->_ioreader; +} + +const Http2FrameHeader & +Http2Frame::header() const +{ + return this->_hdr; +} + +bool +Http2Frame::is_from_early_data() const +{ + return this->_from_early_data; +} + +// +// DATA Frame +// +int64_t +Http2DataFrame::write_to(MIOBuffer *iobuffer) const +{ + // Write frame header + uint8_t buf[HTTP2_FRAME_HEADER_LEN]; + http2_write_frame_header(this->_hdr, make_iovec(buf)); + int64_t len = iobuffer->write(buf, sizeof(buf)); + + // Write frame payload + if (this->_reader && this->_payload_len > 0) { + int64_t written = 0; + // Fill current IOBufferBlock as much as possible to reduce SSL_write() calls + while (written < this->_payload_len) { + int64_t read_len = std::min(this->_payload_len - written, this->_reader->block_read_avail()); + written += iobuffer->write(this->_reader->start(), read_len); + this->_reader->consume(read_len); + } + len += written; + } + + return len; +} + +// +// HEADERS Frame +// +int64_t +Http2HeadersFrame::write_to(MIOBuffer *iobuffer) const +{ + // Validation + if (this->_hdr_block_len > Http2::max_frame_size) { + return -1; + } + + // Write frame header + uint8_t buf[HTTP2_FRAME_HEADER_LEN]; + http2_write_frame_header(this->_hdr, make_iovec(buf)); + int64_t len = iobuffer->write(buf, sizeof(buf)); + + // Write frame payload + if (this->_hdr_block && this->_hdr_block_len > 0) { + len += iobuffer->write(this->_hdr_block, this->_hdr_block_len); + } + + return len; +} + +// +// PRIORITY Frame +// +int64_t +Http2PriorityFrame::write_to(MIOBuffer *iobuffer) const +{ + ink_abort("not supported yet"); + + return 0; +} + +// +// RST_STREM Frame +// +int64_t +Http2RstStreamFrame::write_to(MIOBuffer *iobuffer) const +{ + // Write frame header + uint8_t buf[HTTP2_FRAME_HEADER_LEN]; + http2_write_frame_header(this->_hdr, make_iovec(buf)); + int64_t len = iobuffer->write(buf, sizeof(buf)); + + // Write frame payload + uint8_t payload[HTTP2_RST_STREAM_LEN]; + http2_write_rst_stream(this->_error_code, make_iovec(payload)); + len += iobuffer->write(payload, sizeof(payload)); + + return len; +} + +// +// SETTINGS Frame +// +int64_t +Http2SettingsFrame::write_to(MIOBuffer *iobuffer) const +{ + // Write frame header + uint8_t buf[HTTP2_FRAME_HEADER_LEN]; + http2_write_frame_header(this->_hdr, make_iovec(buf)); + int64_t len = iobuffer->write(buf, sizeof(buf)); + + // Write frame payload + for (uint32_t i = 0; i < this->_psize; ++i) { + Http2SettingsParameter *p = this->_params + i; + + uint8_t p_buf[HTTP2_SETTINGS_PARAMETER_LEN]; + http2_write_settings(*p, make_iovec(p_buf)); + len += iobuffer->write(p_buf, sizeof(p_buf)); + } + + return len; +} + +// +// PUSH_PROMISE Frame +// +int64_t +Http2PushPromiseFrame::write_to(MIOBuffer *iobuffer) const +{ + // Validation + if (this->_hdr_block_len > Http2::max_frame_size) { + return -1; + } + + // Write frame header + uint8_t buf[HTTP2_FRAME_HEADER_LEN]; + http2_write_frame_header(this->_hdr, make_iovec(buf)); + int64_t len = iobuffer->write(buf, sizeof(buf)); + + // Write frame payload + uint8_t p_buf[HTTP2_MAX_FRAME_SIZE]; + http2_write_push_promise(this->_params, this->_hdr_block, this->_hdr_block_len, make_iovec(p_buf)); + len += iobuffer->write(p_buf, sizeof(Http2StreamId) + this->_hdr_block_len); + + return len; +} + +// +// PING Frame +// +int64_t +Http2PingFrame::write_to(MIOBuffer *iobuffer) const +{ + // Write frame header + uint8_t buf[HTTP2_FRAME_HEADER_LEN]; + http2_write_frame_header(this->_hdr, make_iovec(buf)); + int64_t len = iobuffer->write(buf, sizeof(buf)); + + // Write frame payload + uint8_t payload[HTTP2_PING_LEN] = {0}; + http2_write_ping(this->_opaque_data, make_iovec(payload)); + len += iobuffer->write(payload, sizeof(payload)); + + return len; +} + +// +// GOAWAY Frame +// +int64_t +Http2GoawayFrame::write_to(MIOBuffer *iobuffer) const +{ + // Write frame header + uint8_t buf[HTTP2_FRAME_HEADER_LEN]; + http2_write_frame_header(this->_hdr, make_iovec(buf)); + int64_t len = iobuffer->write(buf, sizeof(buf)); + + // Write frame payload + uint8_t payload[HTTP2_GOAWAY_LEN]; + http2_write_goaway(this->_params, make_iovec(payload)); + len += iobuffer->write(payload, sizeof(payload)); + + return len; +} + +// +// WINDOW_UPDATE Frame +// +int64_t +Http2WindowUpdateFrame::write_to(MIOBuffer *iobuffer) const +{ + // Write frame header + uint8_t buf[HTTP2_FRAME_HEADER_LEN]; + http2_write_frame_header(this->_hdr, make_iovec(buf)); + int64_t len = iobuffer->write(buf, sizeof(buf)); + + // Write frame payload + uint8_t payload[HTTP2_WINDOW_UPDATE_LEN]; + http2_write_window_update(this->_window, make_iovec(payload)); + len += iobuffer->write(payload, sizeof(payload)); + + return len; +} + +// +// CONTINUATION Frame +// +int64_t +Http2ContinuationFrame::write_to(MIOBuffer *iobuffer) const +{ + // Validation + if (this->_hdr_block_len > Http2::max_frame_size) { + return -1; + } + + // Write frame header + uint8_t buf[HTTP2_FRAME_HEADER_LEN]; + http2_write_frame_header(this->_hdr, make_iovec(buf)); + int64_t len = iobuffer->write(buf, sizeof(buf)); + + // Write frame payload + if (this->_hdr_block && this->_hdr_block_len > 0) { + len += iobuffer->write(this->_hdr_block, this->_hdr_block_len); + } + + return len; +} diff --git a/proxy/http2/Http2Frame.h b/proxy/http2/Http2Frame.h new file mode 100644 index 00000000000..b0bcd35d022 --- /dev/null +++ b/proxy/http2/Http2Frame.h @@ -0,0 +1,256 @@ +/** @file + + Http2Frame + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#pragma once + +#include "P_Net.h" + +#include "HTTP2.h" + +/** + Incoming HTTP/2 Frame + */ +class Http2Frame +{ +public: + Http2Frame(const Http2FrameHeader &h, IOBufferReader *r, bool e = false) : _hdr(h), _ioreader(r), _from_early_data(e) {} + + // Accessor + IOBufferReader *reader() const; + const Http2FrameHeader &header() const; + bool is_from_early_data() const; + +private: + Http2FrameHeader _hdr; + IOBufferReader *_ioreader = nullptr; + bool _from_early_data = false; +}; + +/** + Outgoing HTTP/2 Frame + */ +class Http2TxFrame +{ +public: + Http2TxFrame(const Http2FrameHeader &h) : _hdr(h) {} + virtual ~Http2TxFrame() {} + + // Don't allocate on heap + void *operator new(std::size_t) = delete; + void *operator new[](std::size_t) = delete; + + virtual int64_t write_to(MIOBuffer *iobuffer) const = 0; + +protected: + Http2FrameHeader _hdr; +}; + +/** + DATA Frame + */ +class Http2DataFrame : public Http2TxFrame +{ +public: + Http2DataFrame(Http2StreamId stream_id, uint8_t flags, IOBufferReader *r, uint32_t l) + : Http2TxFrame({l, HTTP2_FRAME_TYPE_DATA, flags, stream_id}), _reader(r), _payload_len(l) + { + } + + int64_t write_to(MIOBuffer *iobuffer) const override; + +private: + IOBufferReader *_reader = nullptr; + uint32_t _payload_len = 0; +}; + +/** + HEADERS Frame + + TODO: support priority info & padding using Http2HeadersParameter + */ +class Http2HeadersFrame : public Http2TxFrame +{ +public: + Http2HeadersFrame(Http2StreamId stream_id, uint8_t flags, uint8_t *h, uint32_t l) + : Http2TxFrame({l, HTTP2_FRAME_TYPE_HEADERS, flags, stream_id}), _hdr_block(h), _hdr_block_len(l) + { + } + + int64_t write_to(MIOBuffer *iobuffer) const override; + +private: + uint8_t *_hdr_block = nullptr; + uint32_t _hdr_block_len = 0; +}; + +/** + PRIORITY Frame + + TODO: implement xmit function + */ +class Http2PriorityFrame : public Http2TxFrame +{ +public: + Http2PriorityFrame(Http2StreamId stream_id, uint8_t flags, Http2Priority p) + : Http2TxFrame({HTTP2_PRIORITY_LEN, HTTP2_FRAME_TYPE_PRIORITY, flags, stream_id}), _params(p) + { + } + + int64_t write_to(MIOBuffer *iobuffer) const override; + +private: + Http2Priority _params; +}; + +/** + RST_STREAM Frame + */ +class Http2RstStreamFrame : public Http2TxFrame +{ +public: + Http2RstStreamFrame(Http2StreamId stream_id, uint32_t e) + : Http2TxFrame({HTTP2_RST_STREAM_LEN, HTTP2_FRAME_TYPE_RST_STREAM, HTTP2_FRAME_NO_FLAG, stream_id}), _error_code(e) + { + } + + int64_t write_to(MIOBuffer *iobuffer) const override; + +private: + uint32_t _error_code; +}; + +/** + SETTINGS Frame + */ +class Http2SettingsFrame : public Http2TxFrame +{ +public: + Http2SettingsFrame(Http2StreamId stream_id, uint8_t flags) : Http2TxFrame({0, HTTP2_FRAME_TYPE_SETTINGS, flags, stream_id}) {} + Http2SettingsFrame(Http2StreamId stream_id, uint8_t flags, Http2SettingsParameter *p, uint32_t s) + : Http2TxFrame({static_cast(HTTP2_SETTINGS_PARAMETER_LEN) * s, HTTP2_FRAME_TYPE_SETTINGS, flags, stream_id}), + _params(p), + _psize(s) + { + } + + int64_t write_to(MIOBuffer *iobuffer) const override; + +private: + Http2SettingsParameter *_params = nullptr; + uint32_t _psize = 0; +}; + +/** + PUSH_PROMISE Frame + + TODO: support padding + */ +class Http2PushPromiseFrame : public Http2TxFrame +{ +public: + Http2PushPromiseFrame(Http2StreamId stream_id, uint8_t flags, Http2PushPromise p, uint8_t *h, uint32_t l) + : Http2TxFrame({l + static_cast(sizeof(Http2StreamId)), HTTP2_FRAME_TYPE_PUSH_PROMISE, flags, stream_id}), + _params(p), + _hdr_block(h), + _hdr_block_len(l) + { + } + + int64_t write_to(MIOBuffer *iobuffer) const override; + +private: + Http2PushPromise _params; + uint8_t *_hdr_block = nullptr; + uint32_t _hdr_block_len = 0; +}; + +/** + PING Frame + */ +class Http2PingFrame : public Http2TxFrame +{ +public: + Http2PingFrame(Http2StreamId stream_id, uint8_t flags, const uint8_t *data) + : Http2TxFrame({HTTP2_PING_LEN, HTTP2_FRAME_TYPE_PING, flags, stream_id}), _opaque_data(data) + { + } + + int64_t write_to(MIOBuffer *iobuffer) const override; + +private: + const uint8_t *_opaque_data; +}; + +/** + GOAWAY Frame + + TODO: support Additional Debug Data + */ +class Http2GoawayFrame : public Http2TxFrame +{ +public: + Http2GoawayFrame(Http2Goaway p) + : Http2TxFrame({HTTP2_GOAWAY_LEN, HTTP2_FRAME_TYPE_GOAWAY, HTTP2_FRAME_NO_FLAG, HTTP2_CONNECTION_CONTROL_STRTEAM}), _params(p) + { + } + + int64_t write_to(MIOBuffer *iobuffer) const override; + +private: + Http2Goaway _params; +}; + +/** + WINDOW_UPDATE Frame + */ +class Http2WindowUpdateFrame : public Http2TxFrame +{ +public: + Http2WindowUpdateFrame(Http2StreamId stream_id, uint32_t w) + : Http2TxFrame({HTTP2_WINDOW_UPDATE_LEN, HTTP2_FRAME_TYPE_WINDOW_UPDATE, HTTP2_FRAME_NO_FLAG, stream_id}), _window(w) + { + } + + int64_t write_to(MIOBuffer *iobuffer) const override; + +private: + uint32_t _window = 0; +}; + +/** + CONTINUATION Frame + */ +class Http2ContinuationFrame : public Http2TxFrame +{ +public: + Http2ContinuationFrame(Http2StreamId stream_id, uint8_t flags, uint8_t *h, uint32_t l) + : Http2TxFrame({l, HTTP2_FRAME_TYPE_CONTINUATION, flags, stream_id}), _hdr_block(h), _hdr_block_len(l) + { + } + + int64_t write_to(MIOBuffer *iobuffer) const override; + +private: + uint8_t *_hdr_block = nullptr; + uint32_t _hdr_block_len = 0; +}; diff --git a/proxy/http2/Http2FrequencyCounter.cc b/proxy/http2/Http2FrequencyCounter.cc index dfe08b98507..9fab3e6acc6 100644 --- a/proxy/http2/Http2FrequencyCounter.cc +++ b/proxy/http2/Http2FrequencyCounter.cc @@ -26,7 +26,7 @@ void Http2FrequencyCounter::increment(uint16_t amount) { - ink_hrtime hrtime_sec = ink_hrtime_to_sec(Thread::get_hrtime()); + ink_hrtime hrtime_sec = this->_get_hrtime(); uint8_t counter_index = ((hrtime_sec % 60) >= 30); uint8_t last_index = ((this->_last_update % 60) >= 30); @@ -53,3 +53,9 @@ Http2FrequencyCounter::get_count() { return this->_count[0] + this->_count[1]; } + +ink_hrtime +Http2FrequencyCounter::_get_hrtime() +{ + return ink_hrtime_to_sec(Thread::get_hrtime()); +} diff --git a/proxy/http2/Http2FrequencyCounter.h b/proxy/http2/Http2FrequencyCounter.h index 9d2ed73b884..bcd3dbba561 100644 --- a/proxy/http2/Http2FrequencyCounter.h +++ b/proxy/http2/Http2FrequencyCounter.h @@ -31,8 +31,12 @@ class Http2FrequencyCounter public: void increment(uint16_t amount = 1); uint32_t get_count(); + virtual ~Http2FrequencyCounter() {} protected: uint16_t _count[2] = {0}; ink_hrtime _last_update = 0; + +private: + virtual ink_hrtime _get_hrtime(); }; diff --git a/proxy/http2/Http2SessionAccept.cc b/proxy/http2/Http2SessionAccept.cc index 291a4fadcf8..7f68e64fd85 100644 --- a/proxy/http2/Http2SessionAccept.cc +++ b/proxy/http2/Http2SessionAccept.cc @@ -55,10 +55,7 @@ Http2SessionAccept::accept(NetVConnection *netvc, MIOBuffer *iobuf, IOBufferRead Http2ClientSession *new_session = THREAD_ALLOC_INIT(http2ClientSessionAllocator, this_ethread()); new_session->acl = std::move(session_acl); - new_session->host_res_style = ats_host_res_from(client_ip->sa_family, options.host_res_preference); - new_session->outbound_ip4 = options.outbound_ip4; - new_session->outbound_ip6 = options.outbound_ip6; - new_session->outbound_port = options.outbound_port; + new_session->accept_options = &options; new_session->new_connection(netvc, iobuf, reader); return true; diff --git a/proxy/http2/Http2Stream.cc b/proxy/http2/Http2Stream.cc index da681d3d3ae..eef44600e84 100644 --- a/proxy/http2/Http2Stream.cc +++ b/proxy/http2/Http2Stream.cc @@ -34,10 +34,35 @@ } #define Http2StreamDebug(fmt, ...) \ - SsnDebug(proxy_ssn, "http2_stream", "[%" PRId64 "] [%u] " fmt, proxy_ssn->connection_id(), this->get_id(), ##__VA_ARGS__); + SsnDebug(_proxy_ssn, "http2_stream", "[%" PRId64 "] [%u] " fmt, _proxy_ssn->connection_id(), this->get_id(), ##__VA_ARGS__); ClassAllocator http2StreamAllocator("http2StreamAllocator"); +Http2Stream::Http2Stream(Http2StreamId sid, ssize_t initial_rwnd) : _id(sid), _client_rwnd(initial_rwnd) +{ + SET_HANDLER(&Http2Stream::main_event_handler); +} + +void +Http2Stream::init(Http2StreamId sid, ssize_t initial_rwnd) +{ + this->mark_milestone(Http2StreamMilestone::OPEN); + + this->_id = sid; + this->_thread = this_ethread(); + this->_client_rwnd = initial_rwnd; + this->_server_rwnd = Http2::initial_window_size; + + this->_reader = this->_request_buffer.alloc_reader(); + + _req_header.create(HTTP_TYPE_REQUEST); + response_header.create(HTTP_TYPE_RESPONSE); + // TODO: init _req_header instead of response_header if this Http2Stream is outgoing + http2_init_pseudo_headers(response_header); + + http_parser_init(&http_parser); +} + int Http2Stream::main_event_handler(int event, void *edata) { @@ -52,7 +77,15 @@ Http2Stream::main_event_handler(int event, void *edata) Event *e = static_cast(edata); reentrancy_count++; - if (e == cross_thread_event) { + if (e == _read_vio_event) { + _read_vio_event = nullptr; + this->signal_read_event(e->callback_event); + return 0; + } else if (e == _write_vio_event) { + _write_vio_event = nullptr; + this->signal_write_event(e->callback_event); + return 0; + } else if (e == cross_thread_event) { cross_thread_event = nullptr; } else if (e == active_event) { event = VC_EVENT_ACTIVE_TIMEOUT; @@ -73,33 +106,18 @@ Http2Stream::main_event_handler(int event, void *edata) switch (event) { case VC_EVENT_ACTIVE_TIMEOUT: case VC_EVENT_INACTIVITY_TIMEOUT: - if (current_reader && read_vio.ntodo() > 0) { - MUTEX_TRY_LOCK(lock, read_vio.mutex, this_ethread()); - if (lock.is_locked()) { - read_vio.cont->handleEvent(event, &read_vio); - } else { - this_ethread()->schedule_imm(read_vio.cont, event, &read_vio); - } - } else if (current_reader && write_vio.ntodo() > 0) { - MUTEX_TRY_LOCK(lock, write_vio.mutex, this_ethread()); - if (lock.is_locked()) { - write_vio.cont->handleEvent(event, &write_vio); - } else { - this_ethread()->schedule_imm(write_vio.cont, event, &write_vio); - } + if (_sm && read_vio.ntodo() > 0) { + this->signal_read_event(event); + } else if (_sm && write_vio.ntodo() > 0) { + this->signal_write_event(event); } break; case VC_EVENT_WRITE_READY: case VC_EVENT_WRITE_COMPLETE: inactive_timeout_at = Thread::get_hrtime() + inactive_timeout; if (e->cookie == &write_vio) { - if (write_vio.mutex && write_vio.cont && this->current_reader) { - MUTEX_TRY_LOCK(lock, write_vio.mutex, this_ethread()); - if (lock.is_locked()) { - write_vio.cont->handleEvent(event, &write_vio); - } else { - this_ethread()->schedule_imm(write_vio.cont, event, &write_vio); - } + if (write_vio.mutex && write_vio.cont && this->_sm) { + this->signal_write_event(event); } } else { update_write_request(write_vio.get_reader(), INT64_MAX, true); @@ -109,13 +127,8 @@ Http2Stream::main_event_handler(int event, void *edata) case VC_EVENT_READ_READY: inactive_timeout_at = Thread::get_hrtime() + inactive_timeout; if (e->cookie == &read_vio) { - if (read_vio.mutex && read_vio.cont && this->current_reader) { - MUTEX_TRY_LOCK(lock, read_vio.mutex, this_ethread()); - if (lock.is_locked()) { - read_vio.cont->handleEvent(event, &read_vio); - } else { - this_ethread()->schedule_imm(read_vio.cont, event, &read_vio); - } + if (read_vio.mutex && read_vio.cont && this->_sm) { + signal_read_event(event); } } else { this->update_read_request(INT64_MAX, true); @@ -148,8 +161,8 @@ Http2Stream::decode_header_blocks(HpackHandle &hpack_handle, uint32_t maximum_ta void Http2Stream::send_request(Http2ConnectionState &cstate) { - ink_release_assert(this->current_reader != nullptr); - this->_http_sm_id = this->current_reader->sm_id; + ink_release_assert(this->_sm != nullptr); + this->_http_sm_id = this->_sm->sm_id; // Convert header to HTTP/1.1 format http2_convert_header_from_2_to_1_1(&_req_header); @@ -162,21 +175,30 @@ Http2Stream::send_request(Http2ConnectionState &cstate) do { bufindex = 0; tmp = dumpoffset; - IOBufferBlock *block = request_buffer.get_current_block(); + IOBufferBlock *block = this->_request_buffer.get_current_block(); if (!block) { - request_buffer.add_block(); - block = request_buffer.get_current_block(); + this->_request_buffer.add_block(); + block = this->_request_buffer.get_current_block(); } done = _req_header.print(block->start(), block->write_avail(), &bufindex, &tmp); dumpoffset += bufindex; - request_buffer.fill(bufindex); + this->_request_buffer.fill(bufindex); if (!done) { - request_buffer.add_block(); + this->_request_buffer.add_block(); } } while (!done); - // Is there a read_vio request waiting? - this->update_read_request(INT64_MAX, true); + if (bufindex == 0) { + // No data to signal read event + return; + } + + if (this->recv_end_stream) { + this->read_vio.nbytes = bufindex; + this->signal_read_event(VC_EVENT_READ_COMPLETE); + } else { + this->signal_read_event(VC_EVENT_READ_READY); + } } bool @@ -297,9 +319,7 @@ Http2Stream::do_io_read(Continuation *c, int64_t nbytes, MIOBuffer *buf) read_vio.vc_server = this; read_vio.op = VIO::READ; - // Is there already data in the request_buffer? If so, copy it over and then - // schedule a READ_READY or READ_COMPLETE event after we return. - update_read_request(nbytes, false, true); + // TODO: re-enable read_vio return &read_vio; } @@ -320,8 +340,12 @@ Http2Stream::do_io_write(Continuation *c, int64_t nbytes, IOBufferReader *abuffe write_vio.op = VIO::WRITE; response_reader = abuffer; - update_write_request(abuffer, nbytes, false); - + if (c != nullptr && nbytes > 0 && this->is_client_state_writeable()) { + update_write_request(abuffer, nbytes, false); + } else if (!this->is_client_state_writeable()) { + // Cannot start a write on a closed stream + return nullptr; + } return &write_vio; } @@ -341,10 +365,12 @@ Http2Stream::do_io_close(int /* flags */) // by the time this is called from transaction_done. closed = true; - if (proxy_ssn && this->is_client_state_writeable()) { + if (_proxy_ssn && this->is_client_state_writeable()) { // Make sure any trailing end of stream frames are sent // Wee will be removed at send_data_frames or closing connection phase - static_cast(proxy_ssn)->connection_state.send_data_frames(this); + Http2ClientSession *h2_proxy_ssn = static_cast(this->_proxy_ssn); + SCOPED_MUTEX_LOCK(lock, h2_proxy_ssn->connection_state.mutex, this_ethread()); + h2_proxy_ssn->connection_state.send_data_frames(this); } clear_timers(); @@ -367,10 +393,10 @@ Http2Stream::transaction_done() } if (!closed) { - do_io_close(); // Make sure we've been closed. If we didn't close the proxy_ssn session better still be open + do_io_close(); // Make sure we've been closed. If we didn't close the _proxy_ssn session better still be open } - ink_release_assert(closed || !static_cast(proxy_ssn)->connection_state.is_state_closed()); - current_reader = nullptr; + ink_release_assert(closed || !static_cast(_proxy_ssn)->connection_state.is_state_closed()); + _sm = nullptr; if (closed) { // Safe to initiate SSN_CLOSE if this is the last stream @@ -387,9 +413,8 @@ Http2Stream::terminate_if_possible() if (terminate_stream && reentrancy_count == 0) { REMEMBER(NO_EVENT, this->reentrancy_count); - Http2ClientSession *h2_parent = static_cast(proxy_ssn); - SCOPED_MUTEX_LOCK(lock, h2_parent->connection_state.mutex, this_ethread()); - h2_parent->connection_state.delete_stream(this); + Http2ClientSession *h2_proxy_ssn = static_cast(this->_proxy_ssn); + SCOPED_MUTEX_LOCK(lock, h2_proxy_ssn->connection_state.mutex, this_ethread()); destroy(); } } @@ -409,21 +434,20 @@ Http2Stream::initiating_close() _state = Http2StreamState::HTTP2_STREAM_STATE_CLOSED; // leaving the reference to the SM, so we can detach from the SM when we actually destroy - // current_reader = NULL; + // _sm = NULL; // Leaving reference to client session as well, so we can signal once the // TXN_CLOSE has been sent - // proxy_ssn = NULL; + // _proxy_ssn = NULL; clear_timers(); clear_io_events(); // This should result in do_io_close or release being called. That will schedule the final // kill yourself signal - // Send the SM the EOS signal if there are no active VIO's to signal // We are sending signals rather than calling the handlers directly to avoid the case where // the HttpTunnel handler causes the HttpSM to be deleted on the stack. bool sent_write_complete = false; - if (current_reader) { + if (_sm) { // Push out any last IO events if (write_vio.cont) { SCOPED_MUTEX_LOCK(lock, write_vio.mutex, this_ethread()); @@ -439,16 +463,13 @@ Http2Stream::initiating_close() } } // Send EOS to let SM know that we aren't sticking around - if (current_reader && read_vio.cont) { + if (_sm && read_vio.cont) { // Only bother with the EOS if we haven't sent the write complete if (!sent_write_complete) { SCOPED_MUTEX_LOCK(lock, read_vio.mutex, this_ethread()); Http2StreamDebug("send EOS to read cont"); read_event = send_tracked_event(read_event, VC_EVENT_EOS, &read_vio); } - } else if (current_reader) { - SCOPED_MUTEX_LOCK(lock, current_reader->mutex, this_ethread()); - current_reader->handleEvent(VC_EVENT_ERROR); } else if (!sent_write_complete) { // Transaction is already gone or not started. Kill yourself do_io_close(); @@ -479,7 +500,7 @@ Http2Stream::send_tracked_event(Event *event, int send_event, VIO *vio) void Http2Stream::update_read_request(int64_t read_len, bool call_update, bool check_eos) { - if (closed || proxy_ssn == nullptr || current_reader == nullptr || read_vio.mutex == nullptr) { + if (closed || _proxy_ssn == nullptr || _sm == nullptr || read_vio.mutex == nullptr) { return; } @@ -490,44 +511,26 @@ Http2Stream::update_read_request(int64_t read_len, bool call_update, bool check_ ink_release_assert(this->_thread == this_ethread()); SCOPED_MUTEX_LOCK(lock, read_vio.mutex, this_ethread()); - if (read_vio.nbytes > 0 && read_vio.ndone <= read_vio.nbytes) { - // If this vio has a different buffer, we must copy - ink_release_assert(this_ethread() == this->_thread); - if (read_vio.buffer.writer() != (&request_buffer)) { - int64_t num_to_read = read_vio.nbytes - read_vio.ndone; - if (num_to_read > read_len) { - num_to_read = read_len; - } - if (num_to_read > 0) { - int bytes_added = read_vio.buffer.writer()->write(request_reader, num_to_read); - if (bytes_added > 0 || (check_eos && recv_end_stream)) { - request_reader->consume(bytes_added); - read_vio.ndone += bytes_added; - int send_event = (read_vio.nbytes == read_vio.ndone || recv_end_stream) ? VC_EVENT_READ_COMPLETE : VC_EVENT_READ_READY; - if (call_update) { // Safe to call vio handler directly - inactive_timeout_at = Thread::get_hrtime() + inactive_timeout; - if (read_vio.cont && this->current_reader) { - read_vio.cont->handleEvent(send_event, &read_vio); - } - } else { // Called from do_io_read. Still setting things up. Send event to handle this after the dust settles - read_event = send_tracked_event(read_event, send_event, &read_vio); - } - } - } - } else { - // Try to be smart and only signal if there was additional data - int send_event = (read_vio.nbytes == read_vio.ndone) ? VC_EVENT_READ_COMPLETE : VC_EVENT_READ_READY; - if (request_reader->read_avail() > 0 || send_event == VC_EVENT_READ_COMPLETE) { - if (call_update) { // Safe to call vio handler directly - inactive_timeout_at = Thread::get_hrtime() + inactive_timeout; - if (read_vio.cont && this->current_reader) { - read_vio.cont->handleEvent(send_event, &read_vio); - } - } else { // Called from do_io_read. Still setting things up. Send event - // to handle this after the dust settles - read_event = send_tracked_event(read_event, send_event, &read_vio); - } + if (read_vio.nbytes == 0) { + return; + } + + // Try to be smart and only signal if there was additional data + int send_event = VC_EVENT_READ_READY; + if (read_vio.ntodo() == 0 || (this->recv_end_stream && this->read_vio.nbytes != INT64_MAX)) { + send_event = VC_EVENT_READ_COMPLETE; + } + + int64_t read_avail = this->read_vio.buffer.writer()->max_read_avail(); + if (read_avail > 0 || send_event == VC_EVENT_READ_COMPLETE) { + if (call_update) { // Safe to call vio handler directly + inactive_timeout_at = Thread::get_hrtime() + inactive_timeout; + if (read_vio.cont && this->_sm) { + read_vio.cont->handleEvent(send_event, &read_vio); } + } else { // Called from do_io_read. Still setting things up. Send event + // to handle this after the dust settles + read_event = send_tracked_event(read_event, send_event, &read_vio); } } } @@ -535,14 +538,27 @@ Http2Stream::update_read_request(int64_t read_len, bool call_update, bool check_ void Http2Stream::restart_sending() { + if (!this->response_header_done) { + return; + } + + IOBufferReader *reader = this->response_get_data_reader(); + if (reader && !reader->is_read_avail_more_than(0)) { + return; + } + + if (this->write_vio.mutex && this->write_vio.ntodo() == 0) { + return; + } + this->send_response_body(true); } void Http2Stream::update_write_request(IOBufferReader *buf_reader, int64_t write_len, bool call_update) { - if (!this->is_client_state_writeable() || closed || proxy_ssn == nullptr || write_vio.mutex == nullptr || - (buf_reader == nullptr && write_len == 0)) { + if (!this->is_client_state_writeable() || closed || _proxy_ssn == nullptr || write_vio.mutex == nullptr || + (buf_reader == nullptr && write_len == 0) || this->response_reader == nullptr) { return; } @@ -552,26 +568,11 @@ Http2Stream::update_write_request(IOBufferReader *buf_reader, int64_t write_len, } ink_release_assert(this->_thread == this_ethread()); - Http2ClientSession *proxy_ssn = static_cast(this->get_proxy_ssn()); + Http2ClientSession *h2_proxy_ssn = static_cast(this->_proxy_ssn); SCOPED_MUTEX_LOCK(lock, write_vio.mutex, this_ethread()); - // if response is chunked, limit the dechunked_buffer size. - bool is_done = false; - if (this->chunked) { - if (chunked_handler.dechunked_buffer && chunked_handler.dechunked_buffer->max_read_avail() > HTTP2_MAX_BUFFER_USAGE) { - if (buffer_full_write_event == nullptr) { - buffer_full_write_event = _thread->schedule_imm(this, VC_EVENT_WRITE_READY); - } - } else { - this->response_process_data(is_done); - } - } - - if (this->response_get_data_reader() == nullptr) { - return; - } - int64_t bytes_avail = this->response_get_data_reader()->read_avail(); + int64_t bytes_avail = this->response_reader->read_avail(); if (write_vio.nbytes > 0 && write_vio.ntodo() > 0) { int64_t num_to_write = write_vio.ntodo(); if (num_to_write > write_len) { @@ -586,7 +587,7 @@ Http2Stream::update_write_request(IOBufferReader *buf_reader, int64_t write_len, ", reader.read_avail=%" PRId64, write_vio.nbytes, write_vio.ndone, write_vio.get_writer()->write_avail(), bytes_avail); - if (bytes_avail <= 0 && !is_done) { + if (bytes_avail <= 0) { return; } @@ -608,30 +609,32 @@ Http2Stream::update_write_request(IOBufferReader *buf_reader, int64_t write_len, int len; const char *value = field->value_get(&len); if (memcmp(HTTP_VALUE_CLOSE, value, HTTP_LEN_CLOSE) == 0) { - SCOPED_MUTEX_LOCK(lock, proxy_ssn->connection_state.mutex, this_ethread()); - if (proxy_ssn->connection_state.get_shutdown_state() == HTTP2_SHUTDOWN_NONE) { - proxy_ssn->connection_state.set_shutdown_state(HTTP2_SHUTDOWN_NOT_INITIATED, Http2ErrorCode::HTTP2_ERROR_NO_ERROR); + SCOPED_MUTEX_LOCK(lock, h2_proxy_ssn->connection_state.mutex, this_ethread()); + if (h2_proxy_ssn->connection_state.get_shutdown_state() == HTTP2_SHUTDOWN_NONE) { + h2_proxy_ssn->connection_state.set_shutdown_state(HTTP2_SHUTDOWN_NOT_INITIATED, Http2ErrorCode::HTTP2_ERROR_NO_ERROR); } } } { - SCOPED_MUTEX_LOCK(lock, proxy_ssn->connection_state.mutex, this_ethread()); + SCOPED_MUTEX_LOCK(lock, h2_proxy_ssn->connection_state.mutex, this_ethread()); // Send the response header back - proxy_ssn->connection_state.send_headers_frame(this); + h2_proxy_ssn->connection_state.send_headers_frame(this); } - // See if the response is chunked. Set up the dechunking logic if it is - // Make sure to check if the chunk is complete and signal appropriately - this->response_initialize_data_handling(is_done); + // Roll back states of response header to read final response + if (this->response_header.expect_final_response()) { + this->response_header_done = false; + response_header.destroy(); + response_header.create(HTTP_TYPE_RESPONSE); + http2_init_pseudo_headers(response_header); + http_parser_clear(&http_parser); + http_parser_init(&http_parser); + } - // If there is additional data, send it along in a data frame. Or if this was header only - // make sure to send the end of stream - is_done |= (write_vio.ntodo() + this->response_header.length_get()) == bytes_avail; - if (this->response_is_data_available() || is_done) { - if (is_done) { - this->mark_body_done(); - } + this->signal_write_event(call_update); + + if (this->response_reader->is_read_avail_more_than(0)) { this->_milestones.mark(Http2StreamMilestone::START_TX_DATA_FRAMES); this->send_response_body(call_update); } @@ -644,9 +647,6 @@ Http2Stream::update_write_request(IOBufferReader *buf_reader, int64_t write_len, break; } } else { - if (write_vio.ntodo() == bytes_avail || is_done) { - this->mark_body_done(); - } this->_milestones.mark(Http2StreamMilestone::START_TX_DATA_FRAMES); this->send_response_body(call_update); } @@ -654,6 +654,46 @@ Http2Stream::update_write_request(IOBufferReader *buf_reader, int64_t write_len, return; } +void +Http2Stream::signal_read_event(int event) +{ + if (this->read_vio.cont == nullptr || this->read_vio.cont->mutex == nullptr || this->read_vio.op == VIO::NONE) { + return; + } + + MUTEX_TRY_LOCK(lock, read_vio.cont->mutex, this_ethread()); + if (lock.is_locked()) { + inactive_timeout_at = Thread::get_hrtime() + inactive_timeout; + this->read_vio.cont->handleEvent(event, &this->read_vio); + } else { + if (this->_read_vio_event) { + this->_read_vio_event->cancel(); + } + this->_read_vio_event = this_ethread()->schedule_in(this, retry_delay, event, &read_vio); + } +} + +void +Http2Stream::signal_write_event(int event) +{ + // Don't signal a write event if in fact nothing was written + if (this->write_vio.cont == nullptr || this->write_vio.cont->mutex == nullptr || this->write_vio.op == VIO::NONE || + this->write_vio.nbytes == 0) { + return; + } + + MUTEX_TRY_LOCK(lock, write_vio.cont->mutex, this_ethread()); + if (lock.is_locked()) { + inactive_timeout_at = Thread::get_hrtime() + inactive_timeout; + this->write_vio.cont->handleEvent(event, &this->write_vio); + } else { + if (this->_write_vio_event) { + this->_write_vio_event->cancel(); + } + this->_write_vio_event = this_ethread()->schedule_in(this, retry_delay, event, &write_vio); + } +} + void Http2Stream::signal_write_event(bool call_update) { @@ -669,7 +709,7 @@ Http2Stream::signal_write_event(bool call_update) if (call_update) { // Coming from reenable. Safe to call the handler directly - if (write_vio.cont && this->current_reader) { + if (write_vio.cont && this->_sm) { write_vio.cont->handleEvent(send_event, &write_vio); } } else { @@ -678,28 +718,28 @@ Http2Stream::signal_write_event(bool call_update) } } -void +bool Http2Stream::push_promise(URL &url, const MIMEField *accept_encoding) { - Http2ClientSession *proxy_ssn = static_cast(this->get_proxy_ssn()); - SCOPED_MUTEX_LOCK(lock, proxy_ssn->connection_state.mutex, this_ethread()); - proxy_ssn->connection_state.send_push_promise_frame(this, url, accept_encoding); + Http2ClientSession *h2_proxy_ssn = static_cast(this->_proxy_ssn); + SCOPED_MUTEX_LOCK(lock, h2_proxy_ssn->connection_state.mutex, this_ethread()); + return h2_proxy_ssn->connection_state.send_push_promise_frame(this, url, accept_encoding); } void Http2Stream::send_response_body(bool call_update) { - Http2ClientSession *proxy_ssn = static_cast(this->get_proxy_ssn()); - inactive_timeout_at = Thread::get_hrtime() + inactive_timeout; + Http2ClientSession *h2_proxy_ssn = static_cast(this->_proxy_ssn); + inactive_timeout_at = Thread::get_hrtime() + inactive_timeout; if (Http2::stream_priority_enabled) { - SCOPED_MUTEX_LOCK(lock, proxy_ssn->connection_state.mutex, this_ethread()); - proxy_ssn->connection_state.schedule_stream(this); + SCOPED_MUTEX_LOCK(lock, h2_proxy_ssn->connection_state.mutex, this_ethread()); + h2_proxy_ssn->connection_state.schedule_stream(this); // signal_write_event() will be called from `Http2ConnectionState::send_data_frames_depends_on_priority()` // when write_vio is consumed } else { - SCOPED_MUTEX_LOCK(lock, proxy_ssn->connection_state.mutex, this_ethread()); - proxy_ssn->connection_state.send_data_frames(this); + SCOPED_MUTEX_LOCK(lock, h2_proxy_ssn->connection_state.mutex, this_ethread()); + h2_proxy_ssn->connection_state.send_data_frames(this); this->signal_write_event(call_update); // XXX The call to signal_write_event can destroy/free the Http2Stream. // Don't modify the Http2Stream after calling this method. @@ -709,11 +749,17 @@ Http2Stream::send_response_body(bool call_update) void Http2Stream::reenable(VIO *vio) { - if (this->proxy_ssn) { + if (this->_proxy_ssn) { if (vio->op == VIO::WRITE) { SCOPED_MUTEX_LOCK(lock, this->mutex, this_ethread()); update_write_request(vio->get_reader(), INT64_MAX, true); } else if (vio->op == VIO::READ) { + Http2ClientSession *h2_proxy_ssn = static_cast(this->_proxy_ssn); + { + SCOPED_MUTEX_LOCK(ssn_lock, h2_proxy_ssn->connection_state.mutex, this_ethread()); + h2_proxy_ssn->connection_state.restart_receiving(this); + } + SCOPED_MUTEX_LOCK(lock, this->mutex, this_ethread()); update_read_request(INT64_MAX, true); } @@ -733,17 +779,21 @@ Http2Stream::destroy() uint64_t cid = 0; // Safe to initiate SSN_CLOSE if this is the last stream - if (proxy_ssn) { - Http2ClientSession *h2_proxy_ssn = static_cast(proxy_ssn); + if (_proxy_ssn) { + cid = _proxy_ssn->connection_id(); + + Http2ClientSession *h2_proxy_ssn = static_cast(_proxy_ssn); SCOPED_MUTEX_LOCK(lock, h2_proxy_ssn->connection_state.mutex, this_ethread()); // Make sure the stream is removed from the stream list and priority tree // In many cases, this has been called earlier, so this call is a no-op h2_proxy_ssn->connection_state.delete_stream(this); + h2_proxy_ssn->connection_state.decrement_stream_count(); + // Update session's stream counts, so it accurately goes into keep-alive state - h2_proxy_ssn->connection_state.release_stream(this); + h2_proxy_ssn->connection_state.release_stream(); - cid = proxy_ssn->connection_id(); + // Do not access `_proxy_ssn` in below. It might be freed by `release_stream`. } // Clean up the write VIO in case of inactivity timeout @@ -778,7 +828,7 @@ Http2Stream::destroy() response_header.destroy(); // Drop references to all buffer data - request_buffer.clear(); + this->_request_buffer.clear(); // Free the mutexes in the VIO read_vio.mutex.clear(); @@ -787,7 +837,6 @@ Http2Stream::destroy() if (header_blocks) { ats_free(header_blocks); } - chunked_handler.clear(); clear_timers(); clear_io_events(); http_parser_clear(&http_parser); @@ -796,53 +845,10 @@ Http2Stream::destroy() THREAD_FREE(this, http2StreamAllocator, this_ethread()); } -void -Http2Stream::response_initialize_data_handling(bool &is_done) -{ - is_done = false; - int chunked_index = response_header.value_get_index(TS_MIME_FIELD_TRANSFER_ENCODING, TS_MIME_LEN_TRANSFER_ENCODING, - TS_HTTP_VALUE_CHUNKED, TS_HTTP_LEN_CHUNKED); - // -1 means this value was not found for this field - if (chunked_index >= 0) { - Http2StreamDebug("Response is chunked"); - chunked = true; - this->chunked_handler.init_by_action(this->response_reader, ChunkedHandler::ACTION_DECHUNK); - this->chunked_handler.state = ChunkedHandler::CHUNK_READ_SIZE; - this->chunked_handler.dechunked_reader = this->chunked_handler.dechunked_buffer->alloc_reader(); - this->response_reader->dealloc(); - this->response_reader = nullptr; - // Get things going if there is already data waiting - if (this->chunked_handler.chunked_reader->is_read_avail_more_than(0)) { - response_process_data(is_done); - } - } -} - -void -Http2Stream::response_process_data(bool &done) -{ - done = false; - if (chunked) { - do { - if (chunked_handler.state == ChunkedHandler::CHUNK_FLOW_CONTROL) { - chunked_handler.state = ChunkedHandler::CHUNK_READ_SIZE_START; - } - done = this->chunked_handler.process_chunked_content(); - } while (chunked_handler.state == ChunkedHandler::CHUNK_FLOW_CONTROL); - } -} - -bool -Http2Stream::response_is_data_available() const -{ - IOBufferReader *reader = this->response_get_data_reader(); - return reader ? reader->is_read_avail_more_than(0) : false; -} - IOBufferReader * Http2Stream::response_get_data_reader() const { - return (chunked) ? chunked_handler.dechunked_reader : response_reader; + return this->response_reader; } void @@ -905,24 +911,34 @@ Http2Stream::clear_io_events() { if (read_event) { read_event->cancel(); + read_event = nullptr; } - read_event = nullptr; + if (write_event) { write_event->cancel(); + write_event = nullptr; } - write_event = nullptr; if (buffer_full_write_event) { buffer_full_write_event->cancel(); buffer_full_write_event = nullptr; } + + if (this->_read_vio_event) { + this->_read_vio_event->cancel(); + this->_read_vio_event = nullptr; + } + + if (this->_write_vio_event) { + this->_write_vio_event->cancel(); + this->_write_vio_event = nullptr; + } } void Http2Stream::release(IOBufferReader *r) { super::release(r); - current_reader = nullptr; // State machine is on its own way down. this->do_io_close(); } @@ -996,12 +1012,6 @@ Http2Stream::decrement_server_rwnd(size_t amount) } } -void -Http2Stream::mark_milestone(Http2StreamMilestone type) -{ - this->_milestones.mark(type); -} - bool Http2Stream::_switch_thread_if_not_on_right_thread(int event, void *edata) { @@ -1015,3 +1025,30 @@ Http2Stream::_switch_thread_if_not_on_right_thread(int event, void *edata) } return true; } + +int +Http2Stream::get_transaction_priority_weight() const +{ + return priority_node ? priority_node->weight : 0; +} + +int +Http2Stream::get_transaction_priority_dependence() const +{ + if (!priority_node) { + return -1; + } else { + return priority_node->parent ? priority_node->parent->id : 0; + } +} + +int64_t +Http2Stream::read_vio_read_avail() +{ + MIOBuffer *writer = this->read_vio.get_writer(); + if (writer) { + return writer->max_read_avail(); + } + + return 0; +} diff --git a/proxy/http2/Http2Stream.h b/proxy/http2/Http2Stream.h index a005a624658..e923a5d1b6e 100644 --- a/proxy/http2/Http2Stream.h +++ b/proxy/http2/Http2Stream.h @@ -24,9 +24,8 @@ #pragma once #include "HTTP2.h" -#include "../ProxyTransaction.h" +#include "ProxyTransaction.h" #include "Http2DebugNames.h" -#include "../http/HttpTunnel.h" // To get ChunkedHandler #include "Http2DependencyTree.h" #include "tscore/History.h" #include "Milestones.h" @@ -50,124 +49,38 @@ enum class Http2StreamMilestone { class Http2Stream : public ProxyTransaction { public: - typedef ProxyTransaction super; ///< Parent type. - Http2Stream(Http2StreamId sid = 0, ssize_t initial_rwnd = Http2::initial_window_size) : _id(sid), _client_rwnd(initial_rwnd) - { - SET_HANDLER(&Http2Stream::main_event_handler); - } - - void - init(Http2StreamId sid, ssize_t initial_rwnd) - { - this->mark_milestone(Http2StreamMilestone::OPEN); - - this->_id = sid; - this->_thread = this_ethread(); - this->_client_rwnd = initial_rwnd; - - sm_reader = request_reader = request_buffer.alloc_reader(); - // FIXME: Are you sure? every "stream" needs request_header? - _req_header.create(HTTP_TYPE_REQUEST); - response_header.create(HTTP_TYPE_RESPONSE); - http_parser_init(&http_parser); - } + const int retry_delay = HRTIME_MSECONDS(10); + using super = ProxyTransaction; ///< Parent type. - int main_event_handler(int event, void *edata); - - void destroy() override; + Http2Stream(Http2StreamId sid = 0, ssize_t initial_rwnd = Http2::initial_window_size); - bool - is_body_done() const - { - return body_done; - } - - void - mark_body_done() - { - body_done = true; - if (response_is_chunked()) { - ink_assert(chunked_handler.state == ChunkedHandler::CHUNK_READ_DONE || - chunked_handler.state == ChunkedHandler::CHUNK_READ_ERROR); - this->write_vio.nbytes = response_header.length_get() + chunked_handler.dechunked_size; - } - } - - void - update_sent_count(unsigned num_bytes) - { - bytes_sent += num_bytes; - this->write_vio.ndone += num_bytes; - } - - Http2StreamId - get_id() const - { - return _id; - } - - int - get_transaction_id() const override - { - return _id; - } - - Http2StreamState - get_state() const - { - return _state; - } + void init(Http2StreamId sid, ssize_t initial_rwnd); - bool change_state(uint8_t type, uint8_t flags); + int main_event_handler(int event, void *edata); - void - update_initial_rwnd(Http2WindowSize new_size) - { - this->_client_rwnd = new_size; - } - - bool - has_trailing_header() const - { - return trailing_header; - } - - void - set_request_headers(HTTPHdr &h2_headers) - { - _req_header.copy(&h2_headers); - } - - // Check entire DATA payload length if content-length: header is exist - void - increment_data_length(uint64_t length) - { - data_length += length; - } - - bool - payload_length_is_valid() const - { - uint32_t content_length = _req_header.get_content_length(); - return content_length == 0 || content_length == data_length; - } + void destroy() override; + void release(IOBufferReader *r) override; + void reenable(VIO *vio) override; + void transaction_done() override; - Http2ErrorCode decode_header_blocks(HpackHandle &hpack_handle, uint32_t maximum_table_size); - void send_request(Http2ConnectionState &cstate); + void do_io_shutdown(ShutdownHowTo_t) override {} VIO *do_io_read(Continuation *c, int64_t nbytes, MIOBuffer *buf) override; VIO *do_io_write(Continuation *c, int64_t nbytes, IOBufferReader *abuffer, bool owner = false) override; void do_io_close(int lerrno = -1) override; + + Http2ErrorCode decode_header_blocks(HpackHandle &hpack_handle, uint32_t maximum_table_size); + void send_request(Http2ConnectionState &cstate); void initiating_close(); void terminate_if_possible(); - void do_io_shutdown(ShutdownHowTo_t) override {} void update_read_request(int64_t read_len, bool send_update, bool check_eos = false); void update_write_request(IOBufferReader *buf_reader, int64_t write_len, bool send_update); + + void signal_read_event(int event); + void signal_write_event(int event); void signal_write_event(bool call_update); - void reenable(VIO *vio) override; - void transaction_done() override; void restart_sending(); - void push_promise(URL &url, const MIMEField *accept_encoding); + bool push_promise(URL &url, const MIMEField *accept_encoding); // Stream level window size ssize_t client_rwnd() const; @@ -177,11 +90,49 @@ class Http2Stream : public ProxyTransaction Http2ErrorCode increment_server_rwnd(size_t amount); Http2ErrorCode decrement_server_rwnd(size_t amount); + ///////////////// + // Accessors + void set_active_timeout(ink_hrtime timeout_in) override; + void set_inactivity_timeout(ink_hrtime timeout_in) override; + void cancel_inactivity_timeout() override; + + bool allow_half_open() const override; + bool is_first_transaction() const override; + void increment_client_transactions_stat() override; + void decrement_client_transactions_stat() override; + int get_transaction_id() const override; + int get_transaction_priority_weight() const override; + int get_transaction_priority_dependence() const override; + + void clear_inactive_timer(); + void clear_active_timer(); + void clear_timers(); + void clear_io_events(); + + bool is_client_state_writeable() const; + bool is_closed() const; + IOBufferReader *response_get_data_reader() const; + + void mark_milestone(Http2StreamMilestone type); + + void increment_data_length(uint64_t length); + bool payload_length_is_valid() const; + bool is_write_vio_done() const; + void update_sent_count(unsigned num_bytes); + Http2StreamId get_id() const; + Http2StreamState get_state() const; + bool change_state(uint8_t type, uint8_t flags); + void update_initial_rwnd(Http2WindowSize new_size); + bool has_trailing_header() const; + void set_request_headers(HTTPHdr &h2_headers); + MIOBuffer *read_vio_writer() const; + int64_t read_vio_read_avail(); + + ////////////////// + // Variables uint8_t *header_blocks = nullptr; - uint32_t header_blocks_length = 0; // total length of header blocks (not include - // Padding or other fields) - uint32_t request_header_length = 0; // total length of payload (include Padding - // and other fields) + uint32_t header_blocks_length = 0; // total length of header blocks (not include Padding or other fields) + bool recv_end_stream = false; bool send_end_stream = false; @@ -192,60 +143,9 @@ class Http2Stream : public ProxyTransaction HTTPHdr response_header; IOBufferReader *response_reader = nullptr; - IOBufferReader *request_reader = nullptr; - MIOBuffer request_buffer = CLIENT_CONNECTION_FIRST_READ_BUFFER_SIZE_INDEX; Http2DependencyTree::Node *priority_node = nullptr; - IOBufferReader *response_get_data_reader() const; - bool - response_is_chunked() const - { - return chunked; - } - - void release(IOBufferReader *r) override; - - bool - allow_half_open() const override - { - return false; - } - - void set_active_timeout(ink_hrtime timeout_in) override; - void set_inactivity_timeout(ink_hrtime timeout_in) override; - void cancel_inactivity_timeout() override; - void clear_inactive_timer(); - void clear_active_timer(); - void clear_timers(); - void clear_io_events(); - bool - is_client_state_writeable() const - { - return _state == Http2StreamState::HTTP2_STREAM_STATE_OPEN || - _state == Http2StreamState::HTTP2_STREAM_STATE_HALF_CLOSED_REMOTE || - _state == Http2StreamState::HTTP2_STREAM_STATE_RESERVED_LOCAL; - } - - bool - is_closed() const - { - return closed; - } - - bool - is_first_transaction() const override - { - return is_first_transaction_flag; - } - - void increment_client_transactions_stat() override; - void decrement_client_transactions_stat() override; - - void mark_milestone(Http2StreamMilestone type); - private: - void response_initialize_data_handling(bool &is_done); - void response_process_data(bool &is_done); bool response_is_data_available() const; Event *send_tracked_event(Event *event, int send_event, VIO *vio); void send_response_body(bool call_update); @@ -264,6 +164,8 @@ class Http2Stream : public ProxyTransaction int64_t _http_sm_id = -1; HTTPHdr _req_header; + MIOBuffer _request_buffer = CLIENT_CONNECTION_FIRST_READ_BUFFER_SIZE_INDEX; + int64_t read_vio_nbytes; VIO read_vio; VIO write_vio; @@ -271,8 +173,6 @@ class Http2Stream : public ProxyTransaction Milestones(Http2StreamMilestone::LAST_ENTRY)> _milestones; bool trailing_header = false; - bool body_done = false; - bool chunked = false; // A brief discussion of similar flags and state variables: _state, closed, terminate_stream // @@ -300,13 +200,12 @@ class Http2Stream : public ProxyTransaction uint64_t data_length = 0; uint64_t bytes_sent = 0; - ssize_t _client_rwnd; - ssize_t _server_rwnd = Http2::initial_window_size; + ssize_t _client_rwnd = 0; + ssize_t _server_rwnd = 0; std::vector _recent_rwnd_increment = {SIZE_MAX, SIZE_MAX, SIZE_MAX, SIZE_MAX, SIZE_MAX}; int _recent_rwnd_increment_index = 0; - ChunkedHandler chunked_handler; Event *cross_thread_event = nullptr; Event *buffer_full_write_event = nullptr; @@ -318,8 +217,113 @@ class Http2Stream : public ProxyTransaction ink_hrtime inactive_timeout_at = 0; Event *inactive_event = nullptr; - Event *read_event = nullptr; - Event *write_event = nullptr; + Event *read_event = nullptr; + Event *write_event = nullptr; + Event *_read_vio_event = nullptr; + Event *_write_vio_event = nullptr; }; extern ClassAllocator http2StreamAllocator; + +//////////////////////////////////////////////////// +// INLINE + +inline void +Http2Stream::mark_milestone(Http2StreamMilestone type) +{ + this->_milestones.mark(type); +} + +inline bool +Http2Stream::is_write_vio_done() const +{ + return this->write_vio.ntodo() == 0; +} + +inline void +Http2Stream::update_sent_count(unsigned num_bytes) +{ + bytes_sent += num_bytes; + this->write_vio.ndone += num_bytes; +} + +inline Http2StreamId +Http2Stream::get_id() const +{ + return _id; +} + +inline int +Http2Stream::get_transaction_id() const +{ + return _id; +} + +inline Http2StreamState +Http2Stream::get_state() const +{ + return _state; +} + +inline void +Http2Stream::update_initial_rwnd(Http2WindowSize new_size) +{ + this->_client_rwnd = new_size; +} + +inline bool +Http2Stream::has_trailing_header() const +{ + return trailing_header; +} + +inline void +Http2Stream::set_request_headers(HTTPHdr &h2_headers) +{ + _req_header.copy(&h2_headers); +} + +// Check entire DATA payload length if content-length: header is exist +inline void +Http2Stream::increment_data_length(uint64_t length) +{ + data_length += length; +} + +inline bool +Http2Stream::payload_length_is_valid() const +{ + uint32_t content_length = _req_header.get_content_length(); + return content_length == 0 || content_length == data_length; +} + +inline bool +Http2Stream::allow_half_open() const +{ + return false; +} + +inline bool +Http2Stream::is_client_state_writeable() const +{ + return _state == Http2StreamState::HTTP2_STREAM_STATE_OPEN || _state == Http2StreamState::HTTP2_STREAM_STATE_HALF_CLOSED_REMOTE || + _state == Http2StreamState::HTTP2_STREAM_STATE_RESERVED_LOCAL; +} + +inline bool +Http2Stream::is_closed() const +{ + return closed; +} + +inline bool +Http2Stream::is_first_transaction() const +{ + return is_first_transaction_flag; +} + +inline MIOBuffer * +Http2Stream::read_vio_writer() const +{ + return this->read_vio.get_writer(); +} diff --git a/proxy/http2/Makefile.am b/proxy/http2/Makefile.am index 3b33d361227..100909bb8a6 100644 --- a/proxy/http2/Makefile.am +++ b/proxy/http2/Makefile.am @@ -38,6 +38,8 @@ libhttp2_a_SOURCES = \ HPACK.h \ HTTP2.cc \ HTTP2.h \ + Http2Frame.cc \ + Http2Frame.h \ Http2ClientSession.cc \ Http2ClientSession.h \ Http2ConnectionState.cc \ @@ -58,14 +60,33 @@ libhttp2_a_SOURCES += \ endif check_PROGRAMS = \ + test_libhttp2 \ test_Http2DependencyTree \ test_Http2FrequencyCounter \ test_HPACK -TESTS = \ - test_Http2DependencyTree \ - test_Http2FrequencyCounter \ - test_HPACK +TESTS = $(check_PROGRAMS) + +# The order of libinkevent.a and libhdrs.a is sensitive for LLD on debug build. +# Be careful if you change the order. Details in GitHub #6666 +test_libhttp2_LDADD = \ + libhttp2.a \ + $(top_builddir)/iocore/eventsystem/libinkevent.a \ + $(top_builddir)/proxy/hdrs/libhdrs.a \ + $(top_builddir)/src/tscore/libtscore.la \ + $(top_builddir)/src/tscpp/util/libtscpputil.la \ + $(top_builddir)/lib/records/librecords_p.a \ + $(top_builddir)/mgmt/libmgmt_p.la \ + $(top_builddir)/proxy/shared/libUglyLogStubs.a \ + @HWLOC_LIBS@ + +test_libhttp2_CPPFLAGS = $(AM_CPPFLAGS)\ + -I$(abs_top_srcdir)/tests/include + +test_libhttp2_SOURCES = \ + unit_tests/test_HTTP2.cc \ + unit_tests/test_Http2Frame.cc \ + unit_tests/main.cc test_Http2DependencyTree_LDADD = \ $(top_builddir)/src/tscore/libtscore.la \ diff --git a/proxy/http2/unit_tests/main.cc b/proxy/http2/unit_tests/main.cc new file mode 100644 index 00000000000..aa19e099b95 --- /dev/null +++ b/proxy/http2/unit_tests/main.cc @@ -0,0 +1,55 @@ +/** @file + + The main file for test_libhttp2 + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#define CATCH_CONFIG_MAIN +#include "catch.hpp" + +#include "tscore/I_Layout.h" + +#include "I_EventSystem.h" +#include "RecordsConfig.h" + +#include "diags.i" + +#define TEST_THREADS 1 + +struct EventProcessorListener : Catch::TestEventListenerBase { + using TestEventListenerBase::TestEventListenerBase; + + void + testRunStarting(Catch::TestRunInfo const &testRunInfo) override + { + Layout::create(); + init_diags("", nullptr); + RecProcessInit(RECM_STAND_ALONE); + LibRecordsConfigInit(); + + ink_event_system_init(EVENT_SYSTEM_MODULE_PUBLIC_VERSION); + eventProcessor.start(TEST_THREADS); + + EThread *main_thread = new EThread; + main_thread->set_specific(); + } +}; + +CATCH_REGISTER_LISTENER(EventProcessorListener); diff --git a/proxy/http2/unit_tests/test_HTTP2.cc b/proxy/http2/unit_tests/test_HTTP2.cc new file mode 100644 index 00000000000..21a772ae9f7 --- /dev/null +++ b/proxy/http2/unit_tests/test_HTTP2.cc @@ -0,0 +1,169 @@ +/** @file + + Unit tests for HTTP2 + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#include "catch.hpp" + +#include "HTTP2.h" + +#include "tscpp/util/PostScript.h" + +TEST_CASE("Convert HTTPHdr", "[HTTP2]") +{ + url_init(); + mime_init(); + http_init(); + http2_init(); + + HTTPParser parser; + ts::PostScript parser_defer([&]() -> void { http_parser_clear(&parser); }); + http_parser_init(&parser); + + SECTION("request") + { + const char request[] = "GET /index.html HTTP/1.1\r\n" + "Host: trafficserver.apache.org\r\n" + "User-Agent: foobar\r\n" + "\r\n"; + + HTTPHdr hdr_1; + ts::PostScript hdr_1_defer([&]() -> void { hdr_1.destroy(); }); + hdr_1.create(HTTP_TYPE_REQUEST); + http2_init_pseudo_headers(hdr_1); + + // parse + const char *start = request; + const char *end = request + sizeof(request) - 1; + hdr_1.parse_req(&parser, &start, end, true); + + // convert to HTTP/2 + http2_convert_header_from_1_1_to_2(&hdr_1); + + // check pseudo headers + // :method + { + MIMEField *f = hdr_1.field_find(HTTP2_VALUE_METHOD, HTTP2_LEN_METHOD); + REQUIRE(f != nullptr); + std::string_view v = f->value_get(); + CHECK(v == "GET"); + } + + // :scheme + { + MIMEField *f = hdr_1.field_find(HTTP2_VALUE_SCHEME, HTTP2_LEN_SCHEME); + REQUIRE(f != nullptr); + std::string_view v = f->value_get(); + CHECK(v == "https"); + } + + // :authority + { + MIMEField *f = hdr_1.field_find(HTTP2_VALUE_AUTHORITY, HTTP2_LEN_AUTHORITY); + REQUIRE(f != nullptr); + std::string_view v = f->value_get(); + CHECK(v == "trafficserver.apache.org"); + } + + // :path + { + MIMEField *f = hdr_1.field_find(HTTP2_VALUE_PATH, HTTP2_LEN_PATH); + REQUIRE(f != nullptr); + std::string_view v = f->value_get(); + CHECK(v == "/index.html"); + } + + // convert to HTTP/1.1 + HTTPHdr hdr_2; + ts::PostScript hdr_2_defer([&]() -> void { hdr_2.destroy(); }); + hdr_2.create(HTTP_TYPE_REQUEST); + hdr_2.copy(&hdr_1); + + http2_convert_header_from_2_to_1_1(&hdr_2); + + // dump + char buf[128] = {0}; + int bufindex = 0; + int dumpoffset = 0; + + hdr_2.print(buf, sizeof(buf), &bufindex, &dumpoffset); + + // check + CHECK_THAT(buf, Catch::StartsWith("GET https://trafficserver.apache.org/index.html HTTP/1.1\r\n" + "Host: trafficserver.apache.org\r\n" + "User-Agent: foobar\r\n" + "\r\n")); + } + + SECTION("response") + { + const char response[] = "HTTP/1.1 200 OK\r\n" + "Connection: close\r\n" + "\r\n"; + + HTTPHdr hdr_1; + ts::PostScript hdr_1_defer([&]() -> void { hdr_1.destroy(); }); + hdr_1.create(HTTP_TYPE_RESPONSE); + http2_init_pseudo_headers(hdr_1); + + // parse + const char *start = response; + const char *end = response + sizeof(response) - 1; + hdr_1.parse_resp(&parser, &start, end, true); + + // convert to HTTP/2 + http2_convert_header_from_1_1_to_2(&hdr_1); + + // check pseudo headers + // :status + { + MIMEField *f = hdr_1.field_find(HTTP2_VALUE_STATUS, HTTP2_LEN_STATUS); + REQUIRE(f != nullptr); + std::string_view v = f->value_get(); + CHECK(v == "200"); + } + + // no connection header + { + MIMEField *f = hdr_1.field_find(MIME_FIELD_CONNECTION, MIME_LEN_CONNECTION); + CHECK(f == nullptr); + } + + // convert to HTTP/1.1 + HTTPHdr hdr_2; + ts::PostScript hdr_2_defer([&]() -> void { hdr_2.destroy(); }); + hdr_2.create(HTTP_TYPE_REQUEST); + hdr_2.copy(&hdr_1); + + http2_convert_header_from_2_to_1_1(&hdr_2); + + // dump + char buf[128] = {0}; + int bufindex = 0; + int dumpoffset = 0; + + hdr_2.print(buf, sizeof(buf), &bufindex, &dumpoffset); + + // check + REQUIRE(bufindex > 0); + CHECK_THAT(buf, Catch::StartsWith("HTTP/1.1 200 OK\r\n\r\n")); + } +} diff --git a/proxy/http2/unit_tests/test_Http2Frame.cc b/proxy/http2/unit_tests/test_Http2Frame.cc new file mode 100644 index 00000000000..8bd05c1543a --- /dev/null +++ b/proxy/http2/unit_tests/test_Http2Frame.cc @@ -0,0 +1,64 @@ +/** @file + + Unit tests for Http2Frame + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#include "catch.hpp" + +#include "Http2Frame.h" + +TEST_CASE("Http2Frame", "[http2][Http2Frame]") +{ + MIOBuffer *miob = new_MIOBuffer(BUFFER_SIZE_INDEX_32K); + IOBufferReader *miob_r = miob->alloc_reader(); + + SECTION("PUSH_PROMISE") + { + Http2StreamId id = 1; + uint8_t flags = HTTP2_FLAGS_PUSH_PROMISE_END_HEADERS; + Http2PushPromise pp{0, 2}; + uint8_t hdr_block[] = {0xbe, 0xef, 0xbe, 0xef, 0xbe, 0xef, 0xbe, 0xef, 0xbe, 0xef}; + uint8_t hdr_block_len = sizeof(hdr_block); + + Http2PushPromiseFrame frame(id, flags, pp, hdr_block, hdr_block_len); + uint64_t written = frame.write_to(miob); + + CHECK(written == HTTP2_FRAME_HEADER_LEN + sizeof(Http2StreamId) + hdr_block_len); + CHECK(written == miob_r->read_avail()); + + uint8_t buf[32] = {0}; + uint64_t read = miob_r->read(buf, written); + CHECK(read == written); + + uint8_t expected[] = { + 0x00, 0x00, 0x0e, ///< Length + 0x05, ///< Type + 0x04, ///< Flags + 0x00, 0x00, 0x00, 0x01, ///< Stream Identifier (31) + 0x00, 0x00, 0x00, 0x02, ///< Promised Stream ID + 0xbe, 0xef, 0xbe, 0xef, 0xbe, 0xef, 0xbe, 0xef, 0xbe, 0xef ///< Header Block Fragment + }; + + CHECK(memcmp(buf, expected, written) == 0); + } + + free_MIOBuffer(miob); +} diff --git a/proxy/http2/unit_tests/test_Http2FrequencyCounter.cc b/proxy/http2/unit_tests/test_Http2FrequencyCounter.cc index a1c0487420d..b69d58f1ef9 100644 --- a/proxy/http2/unit_tests/test_Http2FrequencyCounter.cc +++ b/proxy/http2/unit_tests/test_Http2FrequencyCounter.cc @@ -29,12 +29,22 @@ class TestHttp2FrequencyCounter : public Http2FrequencyCounter { public: void - set_internal_state(ink_hrtime last_update_sec, uint16_t count_0, uint16_t count_1) + set_internal_state(ink_hrtime now, ink_hrtime last_update_sec, uint16_t count_0, uint16_t count_1) { + this->_now = now; this->_last_update = last_update_sec; this->_count[0] = count_0; this->_count[1] = count_1; } + +private: + ink_hrtime + _get_hrtime() override + { + return this->_now; + } + + ink_hrtime _now = 0; }; TEST_CASE("Http2FrequencyCounter_basic", "[http2][Http2FrequencyCounter]") @@ -49,47 +59,45 @@ TEST_CASE("Http2FrequencyCounter_basic", "[http2][Http2FrequencyCounter]") counter.increment(2); REQUIRE(counter.get_count() == 3); - counter.set_internal_state(ink_hrtime_to_sec(Thread::get_hrtime()) - 10, 1, 2); + ink_hrtime now = ink_hrtime_to_sec(Thread::get_hrtime_updated()); + counter.set_internal_state(now, now - 10, 1, 2); REQUIRE(counter.get_count() == 3); } SECTION("Update at 0") { ink_hrtime now = ink_hrtime_to_sec(Thread::get_hrtime_updated()); - while (now % 60 != 0) { - sleep(1); - now = ink_hrtime_to_sec(Thread::get_hrtime_updated()); - } + now -= now % 60; - counter.set_internal_state(now - 5, 1, 2); + counter.set_internal_state(now, now - 5, 1, 2); counter.increment(); CHECK(counter.get_count() == 3); - counter.set_internal_state(now - 10, 1, 2); + counter.set_internal_state(now, now - 10, 1, 2); counter.increment(); CHECK(counter.get_count() == 3); - counter.set_internal_state(now - 20, 1, 2); + counter.set_internal_state(now, now - 20, 1, 2); counter.increment(); CHECK(counter.get_count() == 3); - counter.set_internal_state(now - 30, 1, 2); + counter.set_internal_state(now, now - 30, 1, 2); counter.increment(); CHECK(counter.get_count() == 3); - counter.set_internal_state(now - 40, 1, 2); + counter.set_internal_state(now, now - 40, 1, 2); counter.increment(); CHECK(counter.get_count() == 1); - counter.set_internal_state(now - 50, 1, 2); + counter.set_internal_state(now, now - 50, 1, 2); counter.increment(); CHECK(counter.get_count() == 1); - counter.set_internal_state(now - 60, 1, 2); + counter.set_internal_state(now, now - 60, 1, 2); counter.increment(); CHECK(counter.get_count() == 1); - counter.set_internal_state(now - 70, 1, 2); + counter.set_internal_state(now, now - 70, 1, 2); counter.increment(); CHECK(counter.get_count() == 1); } @@ -97,40 +105,38 @@ TEST_CASE("Http2FrequencyCounter_basic", "[http2][Http2FrequencyCounter]") SECTION("Update at 10") { ink_hrtime now = ink_hrtime_to_sec(Thread::get_hrtime_updated()); - while (now % 60 != 10) { - sleep(1); - now = ink_hrtime_to_sec(Thread::get_hrtime_updated()); - } + now -= now % 60; + now += 10; - counter.set_internal_state(now - 5, 1, 2); + counter.set_internal_state(now, now - 5, 1, 2); counter.increment(); CHECK(counter.get_count() == 4); - counter.set_internal_state(now - 10, 1, 2); + counter.set_internal_state(now, now - 10, 1, 2); counter.increment(); CHECK(counter.get_count() == 4); - counter.set_internal_state(now - 20, 1, 2); + counter.set_internal_state(now, now - 20, 1, 2); counter.increment(); CHECK(counter.get_count() == 3); - counter.set_internal_state(now - 30, 1, 2); + counter.set_internal_state(now, now - 30, 1, 2); counter.increment(); CHECK(counter.get_count() == 3); - counter.set_internal_state(now - 40, 1, 2); + counter.set_internal_state(now, now - 40, 1, 2); counter.increment(); CHECK(counter.get_count() == 3); - counter.set_internal_state(now - 50, 1, 2); + counter.set_internal_state(now, now - 50, 1, 2); counter.increment(); CHECK(counter.get_count() == 1); - counter.set_internal_state(now - 60, 1, 2); + counter.set_internal_state(now, now - 60, 1, 2); counter.increment(); CHECK(counter.get_count() == 1); - counter.set_internal_state(now - 70, 1, 2); + counter.set_internal_state(now, now - 70, 1, 2); counter.increment(); CHECK(counter.get_count() == 1); } @@ -138,40 +144,38 @@ TEST_CASE("Http2FrequencyCounter_basic", "[http2][Http2FrequencyCounter]") SECTION("Update at 30") { ink_hrtime now = ink_hrtime_to_sec(Thread::get_hrtime_updated()); - while (now % 60 != 30) { - sleep(1); - now = ink_hrtime_to_sec(Thread::get_hrtime_updated()); - } + now -= now % 60; + now += 30; - counter.set_internal_state(now - 5, 1, 2); + counter.set_internal_state(now, now - 5, 1, 2); counter.increment(); CHECK(counter.get_count() == 2); - counter.set_internal_state(now - 10, 1, 2); + counter.set_internal_state(now, now - 10, 1, 2); counter.increment(); CHECK(counter.get_count() == 2); - counter.set_internal_state(now - 20, 1, 2); + counter.set_internal_state(now, now - 20, 1, 2); counter.increment(); CHECK(counter.get_count() == 2); - counter.set_internal_state(now - 30, 1, 2); + counter.set_internal_state(now, now - 30, 1, 2); counter.increment(); CHECK(counter.get_count() == 2); - counter.set_internal_state(now - 40, 1, 2); + counter.set_internal_state(now, now - 40, 1, 2); counter.increment(); CHECK(counter.get_count() == 1); - counter.set_internal_state(now - 50, 1, 2); + counter.set_internal_state(now, now - 50, 1, 2); counter.increment(); CHECK(counter.get_count() == 1); - counter.set_internal_state(now - 60, 1, 2); + counter.set_internal_state(now, now - 60, 1, 2); counter.increment(); CHECK(counter.get_count() == 1); - counter.set_internal_state(now - 70, 1, 2); + counter.set_internal_state(now, now - 70, 1, 2); counter.increment(); CHECK(counter.get_count() == 1); } @@ -179,40 +183,38 @@ TEST_CASE("Http2FrequencyCounter_basic", "[http2][Http2FrequencyCounter]") SECTION("Update at 40") { ink_hrtime now = ink_hrtime_to_sec(Thread::get_hrtime_updated()); - while (now % 60 != 40) { - sleep(1); - now = ink_hrtime_to_sec(Thread::get_hrtime_updated()); - } + now -= now % 60; + now += 40; - counter.set_internal_state(now - 5, 1, 2); + counter.set_internal_state(now, now - 5, 1, 2); counter.increment(); CHECK(counter.get_count() == 4); - counter.set_internal_state(now - 10, 1, 2); + counter.set_internal_state(now, now - 10, 1, 2); counter.increment(); CHECK(counter.get_count() == 4); - counter.set_internal_state(now - 20, 1, 2); + counter.set_internal_state(now, now - 20, 1, 2); counter.increment(); CHECK(counter.get_count() == 2); - counter.set_internal_state(now - 30, 1, 2); + counter.set_internal_state(now, now - 30, 1, 2); counter.increment(); CHECK(counter.get_count() == 2); - counter.set_internal_state(now - 40, 1, 2); + counter.set_internal_state(now, now - 40, 1, 2); counter.increment(); CHECK(counter.get_count() == 2); - counter.set_internal_state(now - 50, 1, 2); + counter.set_internal_state(now, now - 50, 1, 2); counter.increment(); CHECK(counter.get_count() == 1); - counter.set_internal_state(now - 60, 1, 2); + counter.set_internal_state(now, now - 60, 1, 2); counter.increment(); CHECK(counter.get_count() == 1); - counter.set_internal_state(now - 70, 1, 2); + counter.set_internal_state(now, now - 70, 1, 2); counter.increment(); CHECK(counter.get_count() == 1); } diff --git a/proxy/http3/Http09App.cc b/proxy/http3/Http09App.cc index 0b317ff0816..cbde81222ab 100644 --- a/proxy/http3/Http09App.cc +++ b/proxy/http3/Http09App.cc @@ -27,6 +27,7 @@ #include "P_Net.h" #include "P_VConnection.h" +#include "P_QUICNetVConnection.h" #include "QUICDebugNames.h" #include "Http3Session.h" @@ -38,14 +39,9 @@ static constexpr char debug_tag_v[] = "v_quic_simple_app"; Http09App::Http09App(QUICNetVConnection *client_vc, IpAllow::ACL &&session_acl, const HttpSessionAccept::Options &options) : QUICApplication(client_vc) { - this->_ssn = new Http09Session(client_vc); - this->_ssn->acl = std::move(session_acl); - // TODO: avoid const cast - this->_ssn->host_res_style = - ats_host_res_from(client_vc->get_remote_addr()->sa_family, const_cast(options.host_res_preference)); - this->_ssn->outbound_ip4 = options.outbound_ip4; - this->_ssn->outbound_ip6 = options.outbound_ip6; - this->_ssn->outbound_port = options.outbound_port; + this->_ssn = new Http09Session(client_vc); + this->_ssn->acl = std::move(session_acl); + this->_ssn->accept_options = &options; this->_ssn->new_connection(client_vc, nullptr, nullptr); this->_qc->stream_manager()->set_default_application(this); diff --git a/proxy/http3/Http3App.cc b/proxy/http3/Http3App.cc index 946a73d7a59..f24cc7442fc 100644 --- a/proxy/http3/Http3App.cc +++ b/proxy/http3/Http3App.cc @@ -27,6 +27,7 @@ #include "P_Net.h" #include "P_VConnection.h" +#include "P_QUICNetVConnection.h" #include "Http3.h" #include "Http3Config.h" @@ -40,15 +41,9 @@ static constexpr char debug_tag_v[] = "v_http3"; Http3App::Http3App(QUICNetVConnection *client_vc, IpAllow::ACL &&session_acl, const HttpSessionAccept::Options &options) : QUICApplication(client_vc) { - this->_ssn = new Http3Session(client_vc); - this->_ssn->acl = std::move(session_acl); - // TODO: avoid const cast - this->_ssn->host_res_style = - ats_host_res_from(client_vc->get_remote_addr()->sa_family, const_cast(options.host_res_preference)); - this->_ssn->outbound_ip4 = options.outbound_ip4; - this->_ssn->outbound_ip6 = options.outbound_ip6; - this->_ssn->outbound_port = options.outbound_port; - + this->_ssn = new Http3Session(client_vc); + this->_ssn->acl = std::move(session_acl); + this->_ssn->accept_options = &options; this->_ssn->new_connection(client_vc, nullptr, nullptr); this->_qc->stream_manager()->set_default_application(this); @@ -200,7 +195,7 @@ Http3App::_handle_uni_stream_on_read_ready(int /* event */, QUICStreamIO *stream case Http3StreamType::QPACK_DECODER: { this->_set_qpack_stream(type, stream_io); } - case Http3StreamType::UNKOWN: + case Http3StreamType::UNKNOWN: default: // TODO: just ignore or trigger QUIC STOP_SENDING frame with HTTP_UNKNOWN_STREAM_TYPE break; @@ -246,7 +241,7 @@ Http3App::_handle_uni_stream_on_write_ready(int /* event */, QUICStreamIO *strea case Http3StreamType::QPACK_DECODER: { this->_set_qpack_stream(it->second, stream_io); } - case Http3StreamType::UNKOWN: + case Http3StreamType::UNKNOWN: case Http3StreamType::PUSH: default: break; diff --git a/proxy/http3/Http3App.h b/proxy/http3/Http3App.h index fd1893d9722..a48432a7dfe 100644 --- a/proxy/http3/Http3App.h +++ b/proxy/http3/Http3App.h @@ -51,7 +51,7 @@ class Http3App : public QUICApplication virtual void start(); virtual int main_event_handler(int event, Event *data); - // TODO: Return StreamIO. It looks bother that coller have to look up StreamIO by stream id. + // TODO: Return StreamIO. It looks bother that caller have to look up StreamIO by stream id. // Why not create_bidi_stream ? QUICConnectionErrorUPtr create_uni_stream(QUICStreamId &new_stream_id, Http3StreamType type); @@ -106,6 +106,6 @@ class Http3SettingsFramer : public Http3FrameGenerator private: NetVConnectionContext_t _context; - bool _is_done = false; ///< Becarefull when set FIN flag on CONTROL stream. Maybe never? + bool _is_done = false; ///< Be careful when setting FIN flag on CONTROL stream. Maybe never? bool _is_sent = false; ///< Send SETTINGS frame only once }; diff --git a/proxy/http3/Http3Frame.cc b/proxy/http3/Http3Frame.cc index b9102dc3afd..db8c98b1adf 100644 --- a/proxy/http3/Http3Frame.cc +++ b/proxy/http3/Http3Frame.cc @@ -241,11 +241,11 @@ Http3SettingsFrame::Http3SettingsFrame(const uint8_t *buf, size_t buf_len, uint3 } size_t id_len = QUICVariableInt::size(buf + len); - uint16_t id = QUICIntUtil::read_QUICVariableInt(buf + len); + uint16_t id = QUICIntUtil::read_QUICVariableInt(buf + len, buf_len - len); len += id_len; size_t value_len = QUICVariableInt::size(buf + len); - uint64_t value = QUICIntUtil::read_QUICVariableInt(buf + len); + uint64_t value = QUICIntUtil::read_QUICVariableInt(buf + len, buf_len - len); len += value_len; // Ignore any SETTINGS identifier it does not understand. diff --git a/proxy/http3/Http3HeaderFramer.cc b/proxy/http3/Http3HeaderFramer.cc index 0931acc36eb..b69f298c092 100644 --- a/proxy/http3/Http3HeaderFramer.cc +++ b/proxy/http3/Http3HeaderFramer.cc @@ -71,9 +71,9 @@ Http3HeaderFramer::is_done() const } void -Http3HeaderFramer::_convert_header_from_1_1_to_3(HTTPHdr *h3_hdrs, HTTPHdr *h1_hdrs) +Http3HeaderFramer::_convert_header_from_1_1_to_3(HTTPHdr *hdrs) { - http2_generate_h2_header_from_1_1(h1_hdrs, h3_hdrs); + http2_convert_header_from_1_1_to_2(hdrs); } void @@ -85,28 +85,29 @@ Http3HeaderFramer::_generate_header_block() if (this->_transaction->direction() == NET_VCONNECTION_OUT) { this->_header.create(HTTP_TYPE_REQUEST); + http2_init_pseudo_headers(this->_header); parse_result = this->_header.parse_req(&this->_http_parser, this->_source_vio->get_reader(), &bytes_used, false); } else { this->_header.create(HTTP_TYPE_RESPONSE); + http2_init_pseudo_headers(this->_header); parse_result = this->_header.parse_resp(&this->_http_parser, this->_source_vio->get_reader(), &bytes_used, false); } - this->_source_vio->ndone += this->_header.length_get(); + this->_source_vio->ndone += bytes_used; switch (parse_result) { case PARSE_RESULT_DONE: { - HTTPHdr h3_hdr; - this->_convert_header_from_1_1_to_3(&h3_hdr, &this->_header); + this->_convert_header_from_1_1_to_3(&this->_header); - this->_header_block = new_MIOBuffer(); + this->_header_block = new_MIOBuffer(BUFFER_SIZE_INDEX_32K); this->_header_block_reader = this->_header_block->alloc_reader(); - this->_qpack->encode(this->_stream_id, h3_hdr, this->_header_block, this->_header_block_len); + this->_qpack->encode(this->_stream_id, this->_header, this->_header_block, this->_header_block_len); break; } case PARSE_RESULT_CONT: break; default: - Debug("http3_trans", "Ignore ivalid headers"); + Debug("http3_trans", "Ignore invalid headers"); break; } } diff --git a/proxy/http3/Http3HeaderFramer.h b/proxy/http3/Http3HeaderFramer.h index 55ce58627a7..2e70338586e 100644 --- a/proxy/http3/Http3HeaderFramer.h +++ b/proxy/http3/Http3HeaderFramer.h @@ -55,6 +55,6 @@ class Http3HeaderFramer : public Http3FrameGenerator HTTPParser _http_parser; HTTPHdr _header; - void _convert_header_from_1_1_to_3(HTTPHdr *h3_hdrs, HTTPHdr *h1_hdrs); + void _convert_header_from_1_1_to_3(HTTPHdr *hdrs); void _generate_header_block(); }; diff --git a/proxy/http3/Http3HeaderVIOAdaptor.h b/proxy/http3/Http3HeaderVIOAdaptor.h index 1df2b71a75f..f81456b02d1 100644 --- a/proxy/http3/Http3HeaderVIOAdaptor.h +++ b/proxy/http3/Http3HeaderVIOAdaptor.h @@ -27,7 +27,7 @@ #include "Http3FrameHandler.h" -// TODO: rename, this is not VIOAdoptor anymore +// TODO: rename, this is not VIOAdaptor anymore class Http3HeaderVIOAdaptor : public Http3FrameHandler { public: diff --git a/proxy/http3/Http3Session.cc b/proxy/http3/Http3Session.cc index e552f953f9d..aaea1dfd9d3 100644 --- a/proxy/http3/Http3Session.cc +++ b/proxy/http3/Http3Session.cc @@ -22,13 +22,21 @@ */ #include "Http3Session.h" +#include "P_QUICNetVConnection.h" #include "Http3.h" // // HQSession // -HQSession ::~HQSession() +HQSession::HQSession(NetVConnection *vc) : ProxySession(vc) +{ + auto app_name = static_cast(vc)->negotiated_application_name(); + memcpy(this->_protocol_string, app_name.data(), std::min(app_name.length(), sizeof(this->_protocol_string))); + this->_protocol_string[app_name.length()] = '\0'; +} + +HQSession::~HQSession() { for (HQTransaction *t = this->_transaction_list.head; t; t = static_cast(t->link.next)) { delete t; @@ -43,6 +51,25 @@ HQSession::add_transaction(HQTransaction *trans) return; } +const char * +HQSession::get_protocol_string() const +{ + return this->_protocol_string; +} + +int +HQSession::populate_protocol(std::string_view *result, int size) const +{ + int retval = 0; + if (size > retval) { + result[retval++] = this->get_protocol_string(); + if (size > retval) { + retval += super::populate_protocol(result + retval, size - retval); + } + } + return retval; +} + HQTransaction * HQSession::get_transaction(QUICStreamId id) { @@ -118,12 +145,6 @@ HQSession::release(ProxyTransaction *trans) return; } -NetVConnection * -HQSession::get_netvc() const -{ - return this->_client_vc; -} - int HQSession::get_transact_count() const { @@ -143,30 +164,11 @@ Http3Session::Http3Session(NetVConnection *vc) : HQSession(vc) Http3Session::~Http3Session() { - this->_client_vc = nullptr; + this->_vc = nullptr; delete this->_local_qpack; delete this->_remote_qpack; } -const char * -Http3Session::get_protocol_string() const -{ - return IP_PROTO_TAG_HTTP_3.data(); -} - -int -Http3Session::populate_protocol(std::string_view *result, int size) const -{ - int retval = 0; - if (size > retval) { - result[retval++] = IP_PROTO_TAG_HTTP_3; - if (size > retval) { - retval += super::populate_protocol(result + retval, size - retval); - } - } - return retval; -} - void Http3Session::increment_current_active_client_connections_stat() { @@ -196,26 +198,7 @@ Http3Session::remote_qpack() // Http09Session::~Http09Session() { - this->_client_vc = nullptr; -} - -const char * -Http09Session::get_protocol_string() const -{ - return IP_PROTO_TAG_HTTP_QUIC.data(); -} - -int -Http09Session::populate_protocol(std::string_view *result, int size) const -{ - int retval = 0; - if (size > retval) { - result[retval++] = IP_PROTO_TAG_HTTP_QUIC; - if (size > retval) { - retval += super::populate_protocol(result + retval, size - retval); - } - } - return retval; + this->_vc = nullptr; } void diff --git a/proxy/http3/Http3Session.h b/proxy/http3/Http3Session.h index 49fb40c185f..a6aec410d42 100644 --- a/proxy/http3/Http3Session.h +++ b/proxy/http3/Http3Session.h @@ -32,7 +32,7 @@ class HQSession : public ProxySession public: using super = ProxySession; ///< Parent type - HQSession(NetVConnection *vc) : _client_vc(vc){}; + HQSession(NetVConnection *vc); virtual ~HQSession(); // Implement VConnection interface @@ -43,23 +43,23 @@ class HQSession : public ProxySession void reenable(VIO *vio) override; // Implement ProxySession interface + const char *get_protocol_string() const override; + int populate_protocol(std::string_view *result, int size) const override; void new_connection(NetVConnection *new_vc, MIOBuffer *iobuf, IOBufferReader *reader) override; void start() override; void destroy() override; void release(ProxyTransaction *trans) override; - NetVConnection *get_netvc() const override; int get_transact_count() const override; // HQSession void add_transaction(HQTransaction *); HQTransaction *get_transaction(QUICStreamId); -protected: - NetVConnection *_client_vc = nullptr; - private: // this should be unordered map? Queue _transaction_list; + + char _protocol_string[16]; }; class Http3Session : public HQSession @@ -71,8 +71,6 @@ class Http3Session : public HQSession ~Http3Session(); // ProxySession interface - const char *get_protocol_string() const override; - int populate_protocol(std::string_view *result, int size) const override; void increment_current_active_client_connections_stat() override; void decrement_current_active_client_connections_stat() override; @@ -96,8 +94,6 @@ class Http09Session : public HQSession ~Http09Session(); // ProxySession interface - const char *get_protocol_string() const override; - int populate_protocol(std::string_view *result, int size) const override; void increment_current_active_client_connections_stat() override; void decrement_current_active_client_connections_stat() override; diff --git a/proxy/http3/Http3SessionAccept.cc b/proxy/http3/Http3SessionAccept.cc index 9763f95c067..93dfbd88bb4 100644 --- a/proxy/http3/Http3SessionAccept.cc +++ b/proxy/http3/Http3SessionAccept.cc @@ -22,6 +22,7 @@ */ #include "Http3SessionAccept.h" +#include "P_QUICNetVConnection.h" #include "P_Net.h" #include "I_Machine.h" @@ -61,16 +62,16 @@ Http3SessionAccept::accept(NetVConnection *netvc, MIOBuffer *iobuf, IOBufferRead std::string_view alpn = qvc->negotiated_application_name(); - if (alpn.empty() || IP_PROTO_TAG_HTTP_QUIC.compare(alpn) == 0) { + if (alpn.empty() || IP_PROTO_TAG_HTTP_QUIC.compare(alpn) == 0 || IP_PROTO_TAG_HTTP_QUIC_D27.compare(alpn) == 0) { if (alpn.empty()) { Debug("http3", "[%s] start HTTP/0.9 app (ALPN=null)", qvc->cids().data()); } else { - Debug("http3", "[%s] start HTTP/0.9 app (ALPN=%s)", qvc->cids().data(), IP_PROTO_TAG_HTTP_QUIC.data()); + Debug("http3", "[%s] start HTTP/0.9 app (ALPN=%.*s)", qvc->cids().data(), static_cast(alpn.length()), alpn.data()); } new Http09App(qvc, std::move(session_acl), this->options); - } else if (IP_PROTO_TAG_HTTP_3.compare(alpn) == 0) { - Debug("http3", "[%s] start HTTP/3 app (ALPN=%s)", qvc->cids().data(), IP_PROTO_TAG_HTTP_3.data()); + } else if (IP_PROTO_TAG_HTTP_3.compare(alpn) == 0 || IP_PROTO_TAG_HTTP_3_D27.compare(alpn) == 0) { + Debug("http3", "[%s] start HTTP/3 app (ALPN=%.*s)", qvc->cids().data(), static_cast(alpn.length()), alpn.data()); Http3App *app = new Http3App(qvc, std::move(session_acl), this->options); app->start(); diff --git a/proxy/http3/Http3SessionAccept.h b/proxy/http3/Http3SessionAccept.h index 8bb388e3daf..5c872261a37 100644 --- a/proxy/http3/Http3SessionAccept.h +++ b/proxy/http3/Http3SessionAccept.h @@ -21,8 +21,7 @@ limitations under the License. */ -#ifndef __HTTP_QUIC_SESSION_ACCEPT_H__ -#define __HTTP_QUIC_SESSION_ACCEPT_H__ +#pragma once #include "tscore/ink_platform.h" #include "I_Net.h" @@ -53,5 +52,3 @@ class Http3SessionAccept : public SessionAccept HttpSessionAccept::Options options; }; - -#endif // __HTTP_QUIC_SESSION_ACCEPT_H__ diff --git a/proxy/http3/Http3StreamDataVIOAdaptor.cc b/proxy/http3/Http3StreamDataVIOAdaptor.cc index e5b6875c209..15712e1c5c8 100644 --- a/proxy/http3/Http3StreamDataVIOAdaptor.cc +++ b/proxy/http3/Http3StreamDataVIOAdaptor.cc @@ -42,6 +42,13 @@ Http3StreamDataVIOAdaptor::handle_frame(std::shared_ptr frame) MIOBuffer *writer = this->_sink_vio->get_writer(); writer->write(dframe->payload(), dframe->payload_length()); + this->_total_data_length += dframe->payload_length(); return Http3ErrorUPtr(new Http3NoError()); } + +void +Http3StreamDataVIOAdaptor::finalize() +{ + this->_sink_vio->nbytes = this->_total_data_length; +} diff --git a/proxy/http3/Http3StreamDataVIOAdaptor.h b/proxy/http3/Http3StreamDataVIOAdaptor.h index f62fd319c64..19ac384320a 100644 --- a/proxy/http3/Http3StreamDataVIOAdaptor.h +++ b/proxy/http3/Http3StreamDataVIOAdaptor.h @@ -36,6 +36,10 @@ class Http3StreamDataVIOAdaptor : public Http3FrameHandler std::vector interests() override; Http3ErrorUPtr handle_frame(std::shared_ptr frame) override; + // Http3StreamDataVIOAdaptor + void finalize(); + private: - VIO *_sink_vio = nullptr; + VIO *_sink_vio = nullptr; + int64_t _total_data_length = 0; }; diff --git a/proxy/http3/Http3Transaction.cc b/proxy/http3/Http3Transaction.cc index b72f33a9b02..6ffb895e11b 100644 --- a/proxy/http3/Http3Transaction.cc +++ b/proxy/http3/Http3Transaction.cc @@ -22,6 +22,7 @@ */ #include "Http3Transaction.h" +#include "P_QUICNetVConnection.h" #include "QUICDebugNames.h" @@ -33,14 +34,14 @@ #include "HttpSM.h" #include "HTTP2.h" -#define Http3TransDebug(fmt, ...) \ - Debug("http3_trans", "[%s] [%" PRIx32 "] " fmt, \ - static_cast(reinterpret_cast(this->proxy_ssn->get_netvc()))->cids().data(), \ +#define Http3TransDebug(fmt, ...) \ + Debug("http3_trans", "[%s] [%" PRIx32 "] " fmt, \ + static_cast(reinterpret_cast(this->_proxy_ssn->get_netvc()))->cids().data(), \ this->get_transaction_id(), ##__VA_ARGS__) -#define Http3TransVDebug(fmt, ...) \ - Debug("v_http3_trans", "[%s] [%" PRIx32 "] " fmt, \ - static_cast(reinterpret_cast(this->proxy_ssn->get_netvc()))->cids().data(), \ +#define Http3TransVDebug(fmt, ...) \ + Debug("v_http3_trans", "[%s] [%" PRIx32 "] " fmt, \ + static_cast(reinterpret_cast(this->_proxy_ssn->get_netvc()))->cids().data(), \ this->get_transaction_id(), ##__VA_ARGS__) // static void @@ -63,7 +64,7 @@ HQTransaction::HQTransaction(HQSession *session, QUICStreamIO *stream_io) : supe this->set_proxy_ssn(session); - this->sm_reader = this->_read_vio_buf.alloc_reader(); + this->_reader = this->_read_vio_buf.alloc_reader(); HTTPType http_type = HTTP_TYPE_UNKNOWN; if (this->direction() == NET_VCONNECTION_OUT) { @@ -83,24 +84,24 @@ HQTransaction::~HQTransaction() void HQTransaction::set_active_timeout(ink_hrtime timeout_in) { - if (this->proxy_ssn) { - this->proxy_ssn->set_active_timeout(timeout_in); + if (this->_proxy_ssn) { + this->_proxy_ssn->set_active_timeout(timeout_in); } } void HQTransaction::set_inactivity_timeout(ink_hrtime timeout_in) { - if (this->proxy_ssn) { - this->proxy_ssn->set_inactivity_timeout(timeout_in); + if (this->_proxy_ssn) { + this->_proxy_ssn->set_inactivity_timeout(timeout_in); } } void HQTransaction::cancel_inactivity_timeout() { - if (this->proxy_ssn) { - this->proxy_ssn->cancel_inactivity_timeout(); + if (this->_proxy_ssn) { + this->_proxy_ssn->cancel_inactivity_timeout(); } } @@ -109,7 +110,7 @@ HQTransaction::release(IOBufferReader *r) { super::release(r); this->do_io_close(); - this->current_reader = nullptr; + this->_sm = nullptr; } bool @@ -156,8 +157,11 @@ HQTransaction::do_io_write(Continuation *c, int64_t nbytes, IOBufferReader *buf, this->_write_vio.vc_server = this; this->_write_vio.op = VIO::WRITE; - this->_process_write_vio(); - this->_send_tracked_event(this->_write_event, VC_EVENT_WRITE_READY, &this->_write_vio); + if (c != nullptr && nbytes > 0) { + // TODO Return nullptr if the stream is not on writable state + this->_process_write_vio(); + this->_send_tracked_event(this->_write_event, VC_EVENT_WRITE_READY, &this->_write_vio); + } return &this->_write_vio; } @@ -185,7 +189,7 @@ HQTransaction::do_io_close(int lerrno) this->_write_vio.op = VIO::NONE; this->_write_vio.cont = nullptr; - this->proxy_ssn->do_io_close(lerrno); + this->_proxy_ssn->do_io_close(lerrno); } void @@ -217,7 +221,7 @@ HQTransaction::reenable(VIO *vio) void HQTransaction::destroy() { - current_reader = nullptr; + _sm = nullptr; } void @@ -248,7 +252,7 @@ HQTransaction::decrement_client_transactions_stat() NetVConnectionContext_t HQTransaction::direction() const { - return this->proxy_ssn->get_netvc()->get_context(); + return this->_proxy_ssn->get_netvc()->get_context(); } /** @@ -280,7 +284,7 @@ HQTransaction::_signal_read_event() if (this->_read_vio.cont == nullptr || this->_read_vio.op == VIO::NONE) { return; } - int event = this->_read_vio.ntodo() ? VC_EVENT_READ_READY : VC_EVENT_READ_COMPLETE; + int event = this->_read_vio.nbytes == INT64_MAX ? VC_EVENT_READ_READY : VC_EVENT_READ_COMPLETE; MUTEX_TRY_LOCK(lock, this->_read_vio.mutex, this_ethread()); if (lock.is_locked()) { @@ -318,7 +322,7 @@ HQTransaction::_signal_write_event() // Http3Transaction::Http3Transaction(Http3Session *session, QUICStreamIO *stream_io) : super(session, stream_io) { - static_cast(this->proxy_ssn)->add_transaction(static_cast(this)); + static_cast(this->_proxy_ssn)->add_transaction(static_cast(this)); this->_header_framer = new Http3HeaderFramer(this, &this->_write_vio, session->local_qpack(), stream_io->stream_id()); this->_data_framer = new Http3DataFramer(this, &this->_write_vio); @@ -364,29 +368,36 @@ Http3Transaction::state_stream_open(int event, void *edata) switch (event) { case VC_EVENT_READ_READY: - case VC_EVENT_READ_COMPLETE: { Http3TransVDebug("%s (%d)", get_vc_event_name(event), event); - int64_t len = this->_process_read_vio(); // if no progress, don't need to signal - if (len > 0) { + if (this->_process_read_vio() > 0) { this->_signal_read_event(); } this->_stream_io->read_reenable(); - break; - } + case VC_EVENT_READ_COMPLETE: + Http3TransVDebug("%s (%d)", get_vc_event_name(event), event); + this->_process_read_vio(); + this->_data_handler->finalize(); + // always signal regardless of progress + this->_signal_read_event(); + this->_stream_io->read_reenable(); + break; case VC_EVENT_WRITE_READY: - case VC_EVENT_WRITE_COMPLETE: { Http3TransVDebug("%s (%d)", get_vc_event_name(event), event); - int64_t len = this->_process_write_vio(); // if no progress, don't need to signal - if (len > 0) { + if (this->_process_write_vio() > 0) { this->_signal_write_event(); } this->_stream_io->write_reenable(); - break; - } + case VC_EVENT_WRITE_COMPLETE: + Http3TransVDebug("%s (%d)", get_vc_event_name(event), event); + this->_process_write_vio(); + // always signal regardless of progress + this->_signal_write_event(); + this->_stream_io->write_reenable(); + break; case VC_EVENT_EOS: case VC_EVENT_ERROR: case VC_EVENT_INACTIVITY_TIMEOUT: @@ -597,7 +608,7 @@ Http3Transaction::_on_qpack_decode_complete() // Http09Transaction::Http09Transaction(Http09Session *session, QUICStreamIO *stream_io) : super(session, stream_io) { - static_cast(this->proxy_ssn)->add_transaction(static_cast(this)); + static_cast(this->_proxy_ssn)->add_transaction(static_cast(this)); SET_HANDLER(&Http09Transaction::state_stream_open); } diff --git a/proxy/http3/Http3Transaction.h b/proxy/http3/Http3Transaction.h index 1bed5b058fd..5eebfbad310 100644 --- a/proxy/http3/Http3Transaction.h +++ b/proxy/http3/Http3Transaction.h @@ -34,6 +34,7 @@ class Http09Session; class Http3Session; class Http3HeaderFramer; class Http3DataFramer; +class Http3StreamDataVIOAdaptor; class HQTransaction : public ProxyTransaction { @@ -115,10 +116,10 @@ class Http3Transaction : public HQTransaction // These are for HTTP/3 Http3FrameDispatcher _frame_dispatcher; Http3FrameCollector _frame_collector; - Http3FrameGenerator *_header_framer = nullptr; - Http3FrameGenerator *_data_framer = nullptr; - Http3FrameHandler *_header_handler = nullptr; - Http3FrameHandler *_data_handler = nullptr; + Http3FrameGenerator *_header_framer = nullptr; + Http3FrameGenerator *_data_framer = nullptr; + Http3FrameHandler *_header_handler = nullptr; + Http3StreamDataVIOAdaptor *_data_handler = nullptr; }; /** diff --git a/proxy/http3/Http3Types.cc b/proxy/http3/Http3Types.cc index 77eb2158fb8..d8c1e4ab0a7 100644 --- a/proxy/http3/Http3Types.cc +++ b/proxy/http3/Http3Types.cc @@ -36,6 +36,6 @@ Http3Stream::type(const uint8_t *buf) case static_cast(Http3StreamType::QPACK_DECODER): return Http3StreamType::QPACK_DECODER; default: - return Http3StreamType::UNKOWN; + return Http3StreamType::UNKNOWN; } } diff --git a/proxy/http3/Http3Types.h b/proxy/http3/Http3Types.h index c6bba88483f..f2ac5e8f7dd 100644 --- a/proxy/http3/Http3Types.h +++ b/proxy/http3/Http3Types.h @@ -33,7 +33,7 @@ enum class Http3StreamType : uint8_t { QPACK_ENCODER = 0x02, ///< QPACK : encoder -> decoder QPACK_DECODER = 0x03, ///< QPACK : decoder -> encoder RESERVED = 0x21, - UNKOWN = 0xFF, + UNKNOWN = 0xFF, }; enum class Http3SettingsId : uint64_t { diff --git a/proxy/http3/Makefile.am b/proxy/http3/Makefile.am index f5d4de5906f..979f3a656e9 100644 --- a/proxy/http3/Makefile.am +++ b/proxy/http3/Makefile.am @@ -19,6 +19,7 @@ AM_CPPFLAGS += \ $(iocore_include_dirs) \ + -I$(abs_top_srcdir)/include \ -I$(abs_top_srcdir)/proxy/api/ts \ -I$(abs_top_srcdir)/lib \ -I$(abs_top_srcdir)/lib/records \ @@ -30,7 +31,8 @@ AM_CPPFLAGS += \ -I$(abs_top_srcdir)/proxy/hdrs \ -I$(abs_top_srcdir)/proxy/shared \ -I$(abs_top_srcdir)/proxy/http/remap \ - $(TS_INCLUDES) + $(TS_INCLUDES) \ + @YAMLCPP_INCLUDES@ noinst_LIBRARIES = libhttp3.a diff --git a/proxy/http3/QPACK.cc b/proxy/http3/QPACK.cc index c331dce81fe..21702148176 100644 --- a/proxy/http3/QPACK.cc +++ b/proxy/http3/QPACK.cc @@ -1106,7 +1106,7 @@ QPACK::_on_encoder_stream_read_ready(QUICStreamIO &stream_io) } QPACKDebug("Received Dynamic Table Size Update: max_size=%d", max_size); this->_dynamic_table.update_size(max_size); - } else { // Duplicatex + } else { // Duplicates uint16_t index; if (this->_read_duplicate(stream_io, index) < 0) { this->_abort_decode(); @@ -1266,7 +1266,7 @@ QPACK::DynamicTable::lookup(const char *name, int name_len, const char *value, i return {candidate_index, match_type}; } - // TODO Use a tree for better perfomance + // TODO Use a tree for better performance for (; i <= end; i = (i + 1) % this->_max_entries) { if (name_len != 0 && this->_entries[i].name_len == name_len) { this->_storage->read(this->_entries[i].offset, &tmp_name, this->_entries[i].name_len, &tmp_value, @@ -1397,7 +1397,7 @@ QPACK::DynamicTable::unref_entry(uint16_t index) } uint16_t -QPACK::DynamicTable::largest_index() +QPACK::DynamicTable::largest_index() const { return this->_entries_inserted; } @@ -1531,7 +1531,7 @@ QPACK::_write_dynamic_table_size_update(uint16_t max_size) } int -QPACK::_write_table_state_syncrhonize(uint16_t insert_count) +QPACK::_write_table_state_synchronize(uint16_t insert_count) { IOBufferBlock *instruction = new_IOBufferBlock(); instruction->alloc(TS_IOBUFFER_SIZE_INDEX_128); @@ -1802,7 +1802,8 @@ QPACK::DynamicTableStorage::~DynamicTableStorage() } void -QPACK::DynamicTableStorage::read(uint16_t offset, const char **name, uint16_t name_len, const char **value, uint16_t value_len) +QPACK::DynamicTableStorage::read(uint16_t offset, const char **name, uint16_t name_len, const char **value, + uint16_t value_len) const { *name = reinterpret_cast(this->_data + offset); *value = reinterpret_cast(this->_data + offset + name_len); diff --git a/proxy/http3/QPACK.h b/proxy/http3/QPACK.h index 85bd7106701..e81b4ab0673 100644 --- a/proxy/http3/QPACK.h +++ b/proxy/http3/QPACK.h @@ -23,11 +23,17 @@ #pragma once +#include + #include "I_EventSystem.h" #include "I_Event.h" +#include "I_IOBuffer.h" +#include "tscore/Arena.h" #include "tscpp/util/IntrusiveDList.h" #include "MIME.h" +#include "HTTP.h" #include "QUICApplication.h" +#include "QUICConnection.h" class HTTPHdr; @@ -106,7 +112,7 @@ class QPACK : public QUICApplication public: DynamicTableStorage(uint16_t size); ~DynamicTableStorage(); - void read(uint16_t offset, const char **name, uint16_t name_len, const char **value, uint16_t value_len); + void read(uint16_t offset, const char **name, uint16_t name_len, const char **value, uint16_t value_len) const; uint16_t write(const char *name, uint16_t name_len, const char *value, uint16_t value_len); void erase(uint16_t name_len, uint16_t value_len); @@ -133,7 +139,7 @@ class QPACK : public QUICApplication void update_size(uint16_t max_size); void ref_entry(uint16_t index); void unref_entry(uint16_t index); - uint16_t largest_index(); + uint16_t largest_index() const; private: uint16_t _available = 0; @@ -151,19 +157,19 @@ class QPACK : public QUICApplication { public: DecodeRequest(uint16_t largest_reference, EThread *thread, Continuation *continuation, uint64_t stream_id, - const uint8_t *header_blcok, size_t header_block_len, HTTPHdr &hdr) + const uint8_t *header_block, size_t header_block_len, HTTPHdr &hdr) : _largest_reference(largest_reference), _thread(thread), _continuation(continuation), _stream_id(stream_id), - _header_block(header_blcok), + _header_block(header_block), _header_block_len(header_block_len), _hdr(hdr) { } uint16_t - largest_reference() + largest_reference() const { return this->_largest_reference; } @@ -181,19 +187,19 @@ class QPACK : public QUICApplication } uint64_t - stream_id() + stream_id() const { return this->_stream_id; } const uint8_t * - header_block() + header_block() const { return this->_header_block; } size_t - header_block_len() + header_block_len() const { return this->_header_block_len; } @@ -275,7 +281,7 @@ class QPACK : public QUICApplication int _read_table_state_synchronize(QUICStreamIO &stream_io, uint16_t &insert_count); int _read_header_acknowledgement(QUICStreamIO &stream_io, uint64_t &stream_id); int _read_stream_cancellation(QUICStreamIO &stream_io, uint64_t &stream_id); - int _write_table_state_syncrhonize(uint16_t insert_count); + int _write_table_state_synchronize(uint16_t insert_count); int _write_header_acknowledgement(uint64_t stream_id); int _write_stream_cancellation(uint64_t stream_id); diff --git a/proxy/http3/test/main.cc b/proxy/http3/test/main.cc index 002e9d6dd8b..247e118ea92 100644 --- a/proxy/http3/test/main.cc +++ b/proxy/http3/test/main.cc @@ -39,7 +39,7 @@ struct EventProcessorListener : Catch::TestEventListenerBase { testRunStarting(Catch::TestRunInfo const &testRunInfo) override { BaseLogFile *base_log_file = new BaseLogFile("stderr"); - diags = new Diags(testRunInfo.name.c_str(), "" /* tags */, "" /* actions */, base_log_file); + diags = new Diags(testRunInfo.name, "" /* tags */, "" /* actions */, base_log_file); diags->activate_taglist("vv_quic|quic", DiagsTagType_Debug); diags->config.enabled[DiagsTagType_Debug] = true; diags->show_location = SHOW_LOCATION_DEBUG; diff --git a/proxy/http3/test/main_qpack.cc b/proxy/http3/test/main_qpack.cc index d0626372708..9bac453d037 100644 --- a/proxy/http3/test/main_qpack.cc +++ b/proxy/http3/test/main_qpack.cc @@ -56,7 +56,7 @@ struct EventProcessorListener : Catch::TestEventListenerBase { testRunStarting(Catch::TestRunInfo const &testRunInfo) override { BaseLogFile *base_log_file = new BaseLogFile("stderr"); - diags = new Diags(testRunInfo.name.c_str(), "" /* tags */, "" /* actions */, base_log_file); + diags = new Diags(testRunInfo.name, "" /* tags */, "" /* actions */, base_log_file); diags->activate_taglist("qpack", DiagsTagType_Debug); diags->config.enabled[DiagsTagType_Debug] = true; diags->show_location = SHOW_LOCATION_DEBUG; diff --git a/proxy/http3/test/test_Http3Frame.cc b/proxy/http3/test/test_Http3Frame.cc index a7f90d8938d..9aa4efd4c9d 100644 --- a/proxy/http3/test/test_Http3Frame.cc +++ b/proxy/http3/test/test_Http3Frame.cc @@ -29,7 +29,7 @@ TEST_CASE("Http3Frame Type", "[http3]") { CHECK(Http3Frame::type(reinterpret_cast("\x00\x00"), 2) == Http3FrameType::DATA); - // Undefined ragne + // Undefined range CHECK(Http3Frame::type(reinterpret_cast("\x0f\x00"), 2) == Http3FrameType::UNKNOWN); CHECK(Http3Frame::type(reinterpret_cast("\xff\xff\xff\xff\xff\xff\xff\x00"), 9) == Http3FrameType::UNKNOWN); } diff --git a/proxy/http3/test/test_QPACK.cc b/proxy/http3/test/test_QPACK.cc index f76626d457c..bae1eb008c5 100644 --- a/proxy/http3/test/test_QPACK.cc +++ b/proxy/http3/test/test_QPACK.cc @@ -41,7 +41,7 @@ extern char appname[256]; extern char pattern[256]; constexpr int ACK_MODE_IMMEDIATE = 1; -constexpr int ACK_MODE_NONE = 0; +// constexpr int ACK_MODE_NONE = 0; constexpr int MAX_SEQUENCE = 1024; @@ -158,7 +158,7 @@ output_encoder_stream_data(FILE *fd, TestQUICStream *stream) uint64_t stream_id = 0; fwrite(reinterpret_cast(&stream_id), 8, 1, fd); - // Skip 32 bits for Legnth + // Skip 32 bits for Length fseek(fd, 4, SEEK_CUR); // Write QPACKData @@ -169,7 +169,7 @@ output_encoder_stream_data(FILE *fd, TestQUICStream *stream) total += nread; } - // Back to the posistion for Length + // Back to the position for Length fseek(fd, -(total + 4), SEEK_CUR); // Write Length @@ -189,7 +189,7 @@ output_encoded_data(FILE *fd, uint64_t stream_id, IOBufferReader *header_block_r stream_id = htobe64(stream_id); fwrite(reinterpret_cast(&stream_id), 8, 1, fd); - // Skip 32 bits for Legnth + // Skip 32 bits for Length fseek(fd, 4, SEEK_CUR); // Write QPACKData @@ -200,7 +200,7 @@ output_encoded_data(FILE *fd, uint64_t stream_id, IOBufferReader *header_block_r total += nread; } - // Back to the posistion for Length + // Back to the position for Length fseek(fd, -(total + 4), SEEK_CUR); // Write Length @@ -301,7 +301,7 @@ test_encode(const char *qif_file, const char *out_file, int dts, int mbs, int am qpack->set_stream(decoder_stream); uint64_t stream_id = 1; - MIOBuffer *header_block = new_MIOBuffer(); + MIOBuffer *header_block = new_MIOBuffer(BUFFER_SIZE_INDEX_32K); uint64_t header_block_len = 0; IOBufferReader *header_block_reader = header_block->alloc_reader(); for (int i = 0; i < n_requests; ++i) { diff --git a/proxy/issues.txt b/proxy/issues.txt deleted file mode 100644 index 76a81b41ab9..00000000000 --- a/proxy/issues.txt +++ /dev/null @@ -1,252 +0,0 @@ -****************************************************************************** -* * -* Open Issues, Queustions, and Comments about the I/O Core * -* * -****************************************************************************** - - -+--------------+ -| proxy/List.h | -+--------------+ - -DLL::in((C *c,Link &l) - - what is the meaning of in() ? - - is c a member of this list ? - - is c a member of some list ? - -+-----------------------+ -| proxy/NetProcessor.cc | -+-----------------------+ - -server open - - The user interface for accepting on a port with the server() - call is a bit weird. We currently get notified that a new - connection occurred as a side-effect of the vc clone() call. - That's really a hack. - - Can we pass in a continuation for notification, list we do with - all these other calls? - -void detach_read(); -void detach_write(); - Hide VConnection::detach_read(VIO * vio = NULL) and - VConnection::detach_write(VIO * vio = NULL) - -+---------------------+ -| proxy/VConnection.h | -+---------------------+ - -old code - - How many of the methods in VConnection.h are really supported - anymore? Can we delete the vestiges of old designs so we don't - confuse people? - -inline void VConnection::clear_vio_queue() - - When blowing away unexecuted VIOs, if there is a passing - continuation, we call it with a return_state of -1. What - does -1 signify, and what other values are legal? Can we be - sure the user won't use -1? - - We need to document these APIs. - -Mutex * mutex - - What is the meaning of mutex field in class VConnection. When is it - used ? It looks that IOVConnection constructor creates it, but never - uses it. It also looks that locking is done through the mutex passed in - VIO. Should we remove the mutex field from class VConnection ? - -activate_io() - - Is it part of the public interface ? - - -+---------------+ -| proxy/Event.h | -+---------------+ - -void EventContinuation::disable() - - We disable timeout events by setting their timeout to a large - value. We disable quiescence events by patching them out of the - queue. Wouldn't it be better to have disabling work the same way - for both? - - For example, could disabling timeout events just move the events - from the active queue to the disabled queue? - - Finally, the disabled timeout value is set to 200 years past some - unknown time in the past. Do we have any guarantees about that - unknown base reference point? I'd hate the base point to be set to - 200 years in the past at some new OS revision, and our timeouts - start acting screwy. Should we change the ink_hrtime routines to - explicitly subtract off some startup base time, such as the time - when the system starts up? This way ink_hrtime would represent - nanoseconds since proxy startup. - -timeout_absolute - - What is this field? The time of the final timeout, after which no - timeouts should occur? - -+----------------+ -| proxy/Event.cc | -+----------------+ - -types of timeouts - - I need to precisely understand the different types of timeouts, - such as absolute, periodic, immediate. Can I timeout be both - absolute & periodic? The intended semantics aren't immediately - obvious from the code alone. - -void EventThread::execute() - - What is the purpose of the execute_hook_main() and execute_hook_init_select() - virtual hooks? - - We have code to check for reference counts going to zero in the - EventThread main loop. Is it possible to have a global memory - manager do this checking, or have the decrementer do this? Are - we doing this to make the locking easier? It seems minorly - annoying for every Thread to have to worry about its own - garbage collection. - -void EventContinuation::check_timeout() - - What does the (timeout_at >= EVENT_DISABLE_TIMEOUT) do? It - presumably handles disabled timeouts, but I'm not sure how. - - There appears to be an abstraction violation here, where - IOVConnection and EventContinuation functionality is mixed. We - are casting the EventContinuation into an IOVConnection, and running - IOVConnection methods on the continuation. There appear to be two - bits: read_event and write_event that enable this. - - If EventContinuation and IOVConnection are really supposed to be - separate, then we should probably fix up this routine. - - Also, periodic events have their timeout_at value incremented by the - timeout period, independent of the current wallclock time. It seems - possible for small timeout periods, and some processing delay, that - the timeout timestamps could end up shifting permanently behind the - real wallclock time. In other words, the scheduler can't keep up - with the timing requests. Is this something we should check for? - -void EventContinuation::destroy_event() - - The semantics of destruction appear to be undocumented. What happens? - What should the user expect? What is the free list used for? - -void EventThread::execute_hook_init() - - Is execute_hook_init() only called at thread startup time? It appears - that execute_hook_init() is called once each time the execute() main loop - is called --- is the main loop only invoked once? - - If this is only called once, why do we have to sort the timeout - events? It seems like no events should be set up yet, right? - - I'm confused about the control flow to this routine. - -static void sort_timeout_events(EventThread * thr) - - We patch out the list from the EventThread's timeout queue and - explicitly set head to NULL. We should have a method for doing - this, so we can change the structure of the DLL. - -EventContinuation::add_timeout() - - After scanning for the right timeout slot, why do we need this line? - - cur = prev ? prev->timeout.next : thr()->timeout_events.head; - -quiescence - - Is quiescence implemented at all? What is it intended to do? Do - we care about it? Should we keep the support? - -void EventThread::execute_hook_scan_events(int n) - - What is a "zombie" event? - -void EventContinuation::reset_periodic_timeout(ink_hrtime t) - - Why do we remove the timeout twice? - -+-------------+ -| proxy/IO.cc | -+-------------+ - -IO semantics - - What is the basic model implemented by IO processors? Is it the - "open vconnection and enqueue operations on the vconnection" model? - IO.cc doesn't seem to implement open(), but it does implement - read/write an real connections, so I'm confused. - - Are the read_event and write_event flags still used, or are they - superceded by active_read_vio and active_write_vio? - -void IOProcessor::accept_read - - What is accept_read? Why is the accept stuff in IO? - -void IOThread::drain_queues() - - Why two input queues? What purpose does each serve? - -void spawn_io_event(IOThread * thr, IOVConnection * e) - - How can destroyed events end up here? - - The relationship of IOVConnections and vio_queues still - isn't quite clear to me. - - What does activate_io() do? - - Why the close case, what do the read_vio and write_vio flags - signify, and how do they relate to the vio queue? - -void IOThread::execute_hook_scan_events(int n) - - Should accept handling really be handled here? - - There are a lot of jumps to Lrestart, running the entire scan - from the beginning. Will this ever infinite loop? Could we - do it more efficiently & cleanly? - - This comment is in the code: - - ??? shouldn't we update the timeout queue first ??? - - I don't understand how flush() works. - -+---------------------------+ -| proxy/DiskProcessor.h | -+---------------------------+ - -new do_io() interface - - Are open_fd(), close(), read(), write(), fseek() and lstat() part of - the interface ? Sould they be removed from the DiskIOVConnection - interface ? - -void * buf; - What is the meaning of this field ? Where is it used ? - Is it part of the public interface ? - - -+-----------------------------+ -| include/tscore/ink_hrtime.h | -+-----------------------------+ - -conversion routines - - I added a bunch of hrtime-->units and units-->hrtime converters - that we should use. - diff --git a/proxy/logging/Log.cc b/proxy/logging/Log.cc index 6ceeffa1cc9..fe4afa38d7e 100644 --- a/proxy/logging/Log.cc +++ b/proxy/logging/Log.cc @@ -462,6 +462,21 @@ Log::init_fields() global_field_list.add(field, false); field_symbol_hash.emplace("cluc", field); + field = new LogField("client_sni_server_name", "cssn", LogField::STRING, &LogAccess::marshal_client_sni_server_name, + reinterpret_cast(&LogAccess::unmarshal_str)); + global_field_list.add(field, false); + field_symbol_hash.emplace("cssn", field); + + field = new LogField("client_ssl_cert_provided", "cscert", LogField::STRING, &LogAccess::marshal_client_provided_cert, + reinterpret_cast(&LogAccess::unmarshal_int_to_str)); + global_field_list.add(field, false); + field_symbol_hash.emplace("cscert", field); + + field = new LogField("proxy_ssl_cert_provided", "pscert", LogField::STRING, &LogAccess::marshal_proxy_provided_cert, + reinterpret_cast(&LogAccess::unmarshal_int_to_str)); + global_field_list.add(field, false); + field_symbol_hash.emplace("pscert", field); + field = new LogField("process_uuid", "puuid", LogField::STRING, &LogAccess::marshal_process_uuid, reinterpret_cast(&LogAccess::unmarshal_str)); global_field_list.add(field, false); @@ -887,6 +902,51 @@ Log::init_fields() global_field_list.add(field, false); field_symbol_hash.emplace("ctid", field); + field = new LogField("cache_read_retry_attempts", "crra", LogField::dINT, &LogAccess::marshal_cache_read_retries, + &LogAccess::unmarshal_int_to_str); + global_field_list.add(field, false); + field_symbol_hash.emplace("crra", field); + + field = new LogField("cache_write_retry_attempts", "cwra", LogField::dINT, &LogAccess::marshal_cache_write_retries, + &LogAccess::unmarshal_int_to_str); + global_field_list.add(field, false); + field_symbol_hash.emplace("cwra", field); + + field = new LogField("cache_collapsed_connection_success", "cccs", LogField::dINT, + &LogAccess::marshal_cache_collapsed_connection_success, &LogAccess::unmarshal_int_to_str); + global_field_list.add(field, false); + field_symbol_hash.emplace("cccs", field); + + field = new LogField("client_transaction_priority_weight", "ctpw", LogField::sINT, + &LogAccess::marshal_client_http_transaction_priority_weight, &LogAccess::unmarshal_int_to_str); + global_field_list.add(field, false); + field_symbol_hash.emplace("ctpw", field); + + field = new LogField("client_transaction_priority_dependence", "ctpd", LogField::sINT, + &LogAccess::marshal_client_http_transaction_priority_dependence, &LogAccess::unmarshal_int_to_str); + global_field_list.add(field, false); + field_symbol_hash.emplace("ctpd", field); + + field = new LogField("proxy_protocol_version", "ppv", LogField::dINT, &LogAccess::marshal_proxy_protocol_version, + reinterpret_cast(&LogAccess::unmarshal_str)); + global_field_list.add(field, false); + field_symbol_hash.emplace("ppv", field); + + field = new LogField("proxy_protocol_src_ip", "pps", LogField::IP, &LogAccess::marshal_proxy_protocol_src_ip, + &LogAccess::unmarshal_ip_to_str); + global_field_list.add(field, false); + field_symbol_hash.emplace("ppsip", field); + + field = new LogField("proxy_protocol_dst_ip", "ppd", LogField::IP, &LogAccess::marshal_proxy_protocol_dst_ip, + &LogAccess::unmarshal_ip_to_str); + global_field_list.add(field, false); + field_symbol_hash.emplace("ppdip", field); + + field = new LogField("version_build_number", "vbn", LogField::STRING, &LogAccess::marshal_version_build_number, + reinterpret_cast(&LogAccess::unmarshal_str)); + global_field_list.add(field, false); + field_symbol_hash.emplace("vbn", field); + init_status |= FIELDS_INITIALIZED; } @@ -1005,10 +1065,10 @@ Log::init_when_enabled() // setup global scrap object // global_scrap_format = MakeTextLogFormat(); - global_scrap_object = - new LogObject(global_scrap_format, Log::config->logfile_dir, "scrapfile.log", LOG_FILE_BINARY, nullptr, - Log::config->rolling_enabled, Log::config->preproc_threads, Log::config->rolling_interval_sec, - Log::config->rolling_offset_hr, Log::config->rolling_size_mb); + global_scrap_object = new LogObject( + global_scrap_format, Log::config->logfile_dir, "scrapfile.log", LOG_FILE_BINARY, nullptr, Log::config->rolling_enabled, + Log::config->preproc_threads, Log::config->rolling_interval_sec, Log::config->rolling_offset_hr, Log::config->rolling_size_mb, + /* auto create */ false, Log::config->rolling_max_count, Log::config->rolling_min_count); // create the flush thread create_threads(); diff --git a/proxy/logging/LogAccess.cc b/proxy/logging/LogAccess.cc index e1c631b916f..08957c379e7 100644 --- a/proxy/logging/LogAccess.cc +++ b/proxy/logging/LogAccess.cc @@ -30,6 +30,8 @@ #include "LogFormat.h" #include "LogBuffer.h" +extern AppVersionInfo appVersionInfo; + char INVALID_STR[] = "!INVALID_STR!"; #define HIDDEN_CONTENT_TYPE "@Content-Type" @@ -352,7 +354,7 @@ LogAccess::marshal_str(char *dest, const char *source, int padded_len) // bytes to avoid UMR errors when the buffer is written. // size_t real_len = (::strlen(source) + 1); - while ((int)real_len < padded_len) { + while (static_cast(real_len) < padded_len) { dest[real_len] = '$'; real_len++; } @@ -1142,7 +1144,7 @@ void LogAccess::set_client_req_url(char *buf, int len) { if (buf) { - m_client_req_url_len = len; + m_client_req_url_len = std::min(len, m_client_req_url_len); ink_strlcpy(m_client_req_url_str, buf, m_client_req_url_len + 1); } } @@ -1151,7 +1153,7 @@ void LogAccess::set_client_req_url_canon(char *buf, int len) { if (buf) { - m_client_req_url_canon_len = len; + m_client_req_url_canon_len = std::min(len, m_client_req_url_canon_len); ink_strlcpy(m_client_req_url_canon_str, buf, m_client_req_url_canon_len + 1); } } @@ -1160,7 +1162,7 @@ void LogAccess::set_client_req_unmapped_url_canon(char *buf, int len) { if (buf && m_client_req_unmapped_url_canon_str) { - m_client_req_unmapped_url_canon_len = len; + m_client_req_unmapped_url_canon_len = std::min(len, m_client_req_unmapped_url_canon_len); ink_strlcpy(m_client_req_unmapped_url_canon_str, buf, m_client_req_unmapped_url_canon_len + 1); } } @@ -1169,7 +1171,7 @@ void LogAccess::set_client_req_unmapped_url_path(char *buf, int len) { if (buf && m_client_req_unmapped_url_path_str) { - m_client_req_unmapped_url_path_len = len; + m_client_req_unmapped_url_path_len = std::min(len, m_client_req_unmapped_url_path_len); ink_strlcpy(m_client_req_unmapped_url_path_str, buf, m_client_req_unmapped_url_path_len + 1); } } @@ -1178,7 +1180,7 @@ void LogAccess::set_client_req_unmapped_url_host(char *buf, int len) { if (buf && m_client_req_unmapped_url_host_str) { - m_client_req_unmapped_url_host_len = len; + m_client_req_unmapped_url_host_len = std::min(len, m_client_req_unmapped_url_host_len); ink_strlcpy(m_client_req_unmapped_url_host_str, buf, m_client_req_unmapped_url_host_len + 1); } } @@ -1188,7 +1190,7 @@ LogAccess::set_client_req_url_path(char *buf, int len) { //?? use m_client_req_unmapped_url_path_str for now..may need to enhance later.. if (buf && m_client_req_unmapped_url_path_str) { - m_client_req_url_path_len = len; + m_client_req_url_path_len = std::min(len, m_client_req_url_path_len); ink_strlcpy(m_client_req_unmapped_url_path_str, buf, m_client_req_url_path_len + 1); } } @@ -1266,7 +1268,145 @@ LogAccess::marshal_cache_lookup_url_canon(char *buf) /*------------------------------------------------------------------------- -------------------------------------------------------------------------*/ +int +LogAccess::marshal_client_sni_server_name(char *buf) +{ + // NOTE: For this string_view, data() must always be nul-terminated, but the nul character must not be included in + // the length. + // + std::string_view server_name = ""; + + if (m_http_sm) { + auto txn = m_http_sm->get_ua_txn(); + if (txn) { + auto ssn = txn->get_proxy_ssn(); + if (ssn) { + auto ssl = ssn->ssl(); + if (ssl) { + auto server_name_str = ssl->client_sni_server_name(); + if (server_name_str) { + server_name = server_name_str; + } + } + } + } + } + int len = round_strlen(server_name.length() + 1); + if (buf) { + marshal_str(buf, server_name.data(), len); + } + return len; +} + +/*------------------------------------------------------------------------- + -------------------------------------------------------------------------*/ +int +LogAccess::marshal_client_provided_cert(char *buf) +{ + int provided_cert = 0; + if (m_http_sm) { + auto txn = m_http_sm->get_ua_txn(); + if (txn) { + auto ssn = txn->get_proxy_ssn(); + if (ssn) { + auto ssl = ssn->ssl(); + if (ssl) { + provided_cert = ssl->client_provided_certificate(); + } + } + } + } + if (buf) { + marshal_int(buf, provided_cert); + } + return INK_MIN_ALIGN; +} + +/*------------------------------------------------------------------------- + -------------------------------------------------------------------------*/ +int +LogAccess::marshal_proxy_provided_cert(char *buf) +{ + int provided_cert = 0; + if (m_http_sm) { + provided_cert = m_http_sm->server_connection_provided_cert; + } + if (buf) { + marshal_int(buf, provided_cert); + } + return INK_MIN_ALIGN; +} +/*------------------------------------------------------------------------- + -------------------------------------------------------------------------*/ + +int +LogAccess::marshal_version_build_number(char *buf) +{ + int len = LogAccess::strlen(appVersionInfo.BldNumStr); + if (buf) { + marshal_str(buf, appVersionInfo.BldNumStr, len); + } + return len; +} + +/*------------------------------------------------------------------------- + -------------------------------------------------------------------------*/ + +int +LogAccess::marshal_proxy_protocol_version(char *buf) +{ + const char *version_str = nullptr; + int len = INK_MIN_ALIGN; + + if (m_http_sm) { + NetVConnection::ProxyProtocolVersion ver = m_http_sm->t_state.pp_info.proxy_protocol_version; + switch (ver) { + case NetVConnection::ProxyProtocolVersion::V1: + version_str = "V1"; + break; + case NetVConnection::ProxyProtocolVersion::V2: + version_str = "V2"; + break; + case NetVConnection::ProxyProtocolVersion::UNDEFINED: + default: + version_str = "-"; + break; + } + len = LogAccess::strlen(version_str); + } + + if (buf) { + marshal_str(buf, version_str, len); + } + return len; +} + +/*------------------------------------------------------------------------- + -------------------------------------------------------------------------*/ +int +LogAccess::marshal_proxy_protocol_src_ip(char *buf) +{ + sockaddr const *ip = nullptr; + if (m_http_sm && m_http_sm->t_state.pp_info.proxy_protocol_version != NetVConnection::ProxyProtocolVersion::UNDEFINED) { + ip = &m_http_sm->t_state.pp_info.src_addr.sa; + } + return marshal_ip(buf, ip); +} + +/*------------------------------------------------------------------------- + -------------------------------------------------------------------------*/ +int +LogAccess::marshal_proxy_protocol_dst_ip(char *buf) +{ + sockaddr const *ip = nullptr; + if (m_http_sm && m_http_sm->t_state.pp_info.proxy_protocol_version != NetVConnection::ProxyProtocolVersion::UNDEFINED) { + ip = &m_http_sm->t_state.pp_info.dst_addr.sa; + } + return marshal_ip(buf, ip); +} +/*------------------------------------------------------------------------- + -------------------------------------------------------------------------*/ int LogAccess::marshal_client_host_port(char *buf) { @@ -2552,6 +2692,93 @@ LogAccess::marshal_client_http_transaction_id(char *buf) return INK_MIN_ALIGN; } +/*------------------------------------------------------------------------- + -------------------------------------------------------------------------*/ + +int +LogAccess::marshal_client_http_transaction_priority_weight(char *buf) +{ + if (buf) { + int64_t id = 0; + if (m_http_sm) { + id = m_http_sm->client_transaction_priority_weight(); + } + marshal_int(buf, id); + } + return INK_MIN_ALIGN; +} + +/*------------------------------------------------------------------------- + -------------------------------------------------------------------------*/ + +int +LogAccess::marshal_client_http_transaction_priority_dependence(char *buf) +{ + if (buf) { + int64_t id = 0; + if (m_http_sm) { + id = m_http_sm->client_transaction_priority_dependence(); + } + marshal_int(buf, id); + } + return INK_MIN_ALIGN; +} + +/*------------------------------------------------------------------------- + -------------------------------------------------------------------------*/ + +int +LogAccess::marshal_cache_read_retries(char *buf) +{ + if (buf) { + int64_t id = 0; + if (m_http_sm) { + id = m_http_sm->get_cache_sm().get_open_read_tries(); + } + marshal_int(buf, id); + } + return INK_MIN_ALIGN; +} + +/*------------------------------------------------------------------------- + -------------------------------------------------------------------------*/ + +int +LogAccess::marshal_cache_write_retries(char *buf) +{ + if (buf) { + int64_t id = 0; + if (m_http_sm) { + id = m_http_sm->get_cache_sm().get_open_write_tries(); + } + marshal_int(buf, id); + } + return INK_MIN_ALIGN; +} + +int +LogAccess::marshal_cache_collapsed_connection_success(char *buf) +{ + if (buf) { + int64_t id = 0; // default - no collapse attempt + if (m_http_sm) { + SquidLogCode code = m_http_sm->t_state.squid_codes.log_code; + + // We increment open_write_tries beyond the max when we want to jump back to the read state for collapsing + if ((m_http_sm->get_cache_sm().get_open_write_tries() > (m_http_sm->t_state.txn_conf->max_cache_open_write_retries)) && + ((code == SQUID_LOG_TCP_HIT) || (code == SQUID_LOG_TCP_MEM_HIT) || (code == SQUID_LOG_TCP_DISK_HIT))) { + // Attempted collapsed connection and got a hit, success + id = 1; + } else if (m_http_sm->get_cache_sm().get_open_write_tries() > (m_http_sm->t_state.txn_conf->max_cache_open_write_retries)) { + // Attempted collapsed connection with no hit, failure, we can also get +2 retries in a failure state + id = -1; + } + } + + marshal_int(buf, id); + } + return INK_MIN_ALIGN; +} /*------------------------------------------------------------------------- -------------------------------------------------------------------------*/ @@ -2841,7 +3068,10 @@ LogAccess::set_http_header_field(LogField::Container container, char *field, cha // Loop over dups, update each of them // while (fld) { - header->value_set((const char *)field, static_cast(::strlen(field)), buf, len); + // make sure to reuse header heaps as otherwise + // coalesce logic in header heap may free up + // memory pointed to by cquuc or other log fields + header->field_value_set(fld, buf, len, true); fld = fld->m_next_dup; } } diff --git a/proxy/logging/LogAccess.h b/proxy/logging/LogAccess.h index 1a8a2ef24f7..9b14644f3ec 100644 --- a/proxy/logging/LogAccess.h +++ b/proxy/logging/LogAccess.h @@ -239,15 +239,27 @@ class LogAccess // other fields // - inkcoreapi int marshal_transfer_time_ms(char *); // INT - inkcoreapi int marshal_transfer_time_s(char *); // INT - inkcoreapi int marshal_file_size(char *); // INT - inkcoreapi int marshal_plugin_identity_id(char *); // INT - inkcoreapi int marshal_plugin_identity_tag(char *); // STR - inkcoreapi int marshal_process_uuid(char *); // STR - inkcoreapi int marshal_client_http_connection_id(char *); // INT - inkcoreapi int marshal_client_http_transaction_id(char *); // INT - inkcoreapi int marshal_cache_lookup_url_canon(char *); // STR + inkcoreapi int marshal_transfer_time_ms(char *); // INT + inkcoreapi int marshal_transfer_time_s(char *); // INT + inkcoreapi int marshal_file_size(char *); // INT + inkcoreapi int marshal_plugin_identity_id(char *); // INT + inkcoreapi int marshal_plugin_identity_tag(char *); // STR + inkcoreapi int marshal_process_uuid(char *); // STR + inkcoreapi int marshal_client_http_connection_id(char *); // INT + inkcoreapi int marshal_client_http_transaction_id(char *); // INT + inkcoreapi int marshal_client_http_transaction_priority_weight(char *); // INT + inkcoreapi int marshal_client_http_transaction_priority_dependence(char *); // INT + inkcoreapi int marshal_cache_lookup_url_canon(char *); // STR + inkcoreapi int marshal_client_sni_server_name(char *); // STR + inkcoreapi int marshal_client_provided_cert(char *); // INT + inkcoreapi int marshal_proxy_provided_cert(char *); // INT + inkcoreapi int marshal_version_build_number(char *); // STR + inkcoreapi int marshal_cache_read_retries(char *); // INT + inkcoreapi int marshal_cache_write_retries(char *); // INT + inkcoreapi int marshal_cache_collapsed_connection_success(char *); // INT + inkcoreapi int marshal_proxy_protocol_version(char *); // STR + inkcoreapi int marshal_proxy_protocol_src_ip(char *); // STR + inkcoreapi int marshal_proxy_protocol_dst_ip(char *); // STR // named fields from within a http header // diff --git a/proxy/logging/LogBuffer.cc b/proxy/logging/LogBuffer.cc index 90472fcb513..5dc19f40533 100644 --- a/proxy/logging/LogBuffer.cc +++ b/proxy/logging/LogBuffer.cc @@ -111,8 +111,8 @@ LogBuffer::LogBuffer(LogObject *owner, size_t size, size_t buf_align, size_t wri // int64_t alloc_size = size + buf_align; - if (alloc_size <= max_iobuffer_size) { - m_buffer_fast_allocator_size = buffer_size_to_index(alloc_size); + if (alloc_size <= BUFFER_SIZE_FOR_INDEX(Log::config->logbuffer_max_iobuf_index)) { + m_buffer_fast_allocator_size = buffer_size_to_index(alloc_size, Log::config->logbuffer_max_iobuf_index); m_unaligned_buffer = static_cast(ioBufAllocator[m_buffer_fast_allocator_size].alloc_void()); } else { m_buffer_fast_allocator_size = -1; diff --git a/proxy/logging/LogBuffer.h b/proxy/logging/LogBuffer.h index 1c64a18c4f6..581d99cd1a9 100644 --- a/proxy/logging/LogBuffer.h +++ b/proxy/logging/LogBuffer.h @@ -133,7 +133,8 @@ class LogBuffer LogBuffer(LogObject *owner, LogBufferHeader *header); ~LogBuffer(); - char &operator[](int idx) + char & + operator[](int idx) { ink_assert(idx >= 0); ink_assert((size_t)idx < m_size); diff --git a/proxy/logging/LogConfig.cc b/proxy/logging/LogConfig.cc index 5160554204b..d35eed14201 100644 --- a/proxy/logging/LogConfig.cc +++ b/proxy/logging/LogConfig.cc @@ -34,6 +34,7 @@ #include "tscore/ink_file.h" #include "tscore/List.h" +#include "tscore/Filenames.h" #include "Log.h" #include "LogField.h" @@ -61,6 +62,7 @@ #define PARTITION_HEADROOM_MB 10 #define DIAGS_LOG_FILENAME "diags.log" +#define MANAGER_LOG_FILENAME "manager.log" void LogConfig::setup_default_values() @@ -94,17 +96,30 @@ LogConfig::setup_default_values() file_stat_frequency = 16; space_used_frequency = 900; - ascii_buffer_size = 4 * 9216; - max_line_size = 9216; // size of pipe buffer for SunOS 5.6 + ascii_buffer_size = 4 * 9216; + max_line_size = 9216; // size of pipe buffer for SunOS 5.6 + logbuffer_max_iobuf_index = BUFFER_SIZE_INDEX_32K; } -void -LogConfig::reconfigure_mgmt_variables(ts::MemSpan) +void LogConfig::reconfigure_mgmt_variables(ts::MemSpan) { Note("received log reconfiguration event, rolling now"); Log::config->roll_log_files_now = true; } +void +LogConfig::register_rolled_log_auto_delete(std::string_view logname, int rolling_min_count) +{ + if (!auto_delete_rolled_files) { + // Nothing to do if auto-deletion is not configured. + return; + } + + Debug("logspace", "Registering rotated log deletion for %s with min roll count %d", std::string(logname).c_str(), + rolling_min_count); + rolledLogDeleter.register_log_type_for_deletion(logname, rolling_min_count); +} + void LogConfig::read_configuration_variables() { @@ -131,6 +146,11 @@ LogConfig::read_configuration_variables() max_space_mb_headroom = val; } + val = static_cast(REC_ConfigReadInteger("proxy.config.log.io.max_buffer_index")); + if (val > 0) { + logbuffer_max_iobuf_index = val; + } + ptr = REC_ConfigReadString("proxy.config.log.logfile_perm"); int logfile_perm_parsed = ink_fileperm_parse(ptr); if (logfile_perm_parsed != -1) { @@ -184,22 +204,24 @@ LogConfig::read_configuration_variables() // Read in min_count control values for auto deletion if (auto_delete_rolled_files) { + // The majority of register_rolled_log_auto_delete() updates come in + // through LogObject. However, not all ATS logs are managed by LogObject. + // The following register these other core logs for log rotation deletion. + // For diagnostic logs val = static_cast(REC_ConfigReadInteger("proxy.config.diags.logfile.rolling_min_count")); - val = ((val == 0) ? INT_MAX : val); - deleting_info.insert(new LogDeletingInfo(DIAGS_LOG_FILENAME, val)); + register_rolled_log_auto_delete(DIAGS_LOG_FILENAME, val); + register_rolled_log_auto_delete(MANAGER_LOG_FILENAME, val); // For traffic.out - ats_scoped_str name(REC_ConfigReadString("proxy.config.output.logfile")); - val = static_cast(REC_ConfigReadInteger("proxy.config.output.logfile.rolling_min_count")); - val = ((val == 0) ? INT_MAX : val); - if (name) { - deleting_info.insert(new LogDeletingInfo(name.get(), val)); - } else { - deleting_info.insert(new LogDeletingInfo("traffic.out", val)); - } + char *configured_name(REC_ConfigReadString("proxy.config.output.logfile")); + const char *traffic_logname = configured_name ? configured_name : "traffic.out"; + val = static_cast(REC_ConfigReadInteger("proxy.config.output.logfile.rolling_min_count")); + register_rolled_log_auto_delete(traffic_logname, val); rolling_max_count = static_cast(REC_ConfigReadInteger("proxy.config.log.rolling_max_count")); + + ats_free(configured_name); } // PERFORMANCE val = static_cast(REC_ConfigReadInteger("proxy.config.log.sampling_frequency")); @@ -285,7 +307,8 @@ LogConfig::init(LogConfig *prev_config) Debug("log", "creating predefined error log object"); errlog = new LogObject(fmt.get(), logfile_dir, "error.log", LOG_FILE_ASCII, nullptr, rolling_enabled, preproc_threads, - rolling_interval_sec, rolling_offset_hr, rolling_size_mb); + rolling_interval_sec, rolling_offset_hr, rolling_size_mb, /* auto_created */ false, rolling_max_count, + rolling_min_count); log_object_manager.manage_object(errlog); errlog->set_fmt_timestamps(); @@ -342,6 +365,7 @@ LogConfig::display(FILE *fd) fprintf(fd, " sampling_frequency = %d\n", sampling_frequency); fprintf(fd, " file_stat_frequency = %d\n", file_stat_frequency); fprintf(fd, " space_used_frequency = %d\n", space_used_frequency); + fprintf(fd, " logbuffer_max_iobuf_index = %d\n", logbuffer_max_iobuf_index); fprintf(fd, "\n"); fprintf(fd, "************ Log Objects (%u objects) ************\n", log_object_manager.get_num_objects()); @@ -417,6 +441,7 @@ LogConfig::register_config_callbacks() "proxy.config.log.rolling_offset_hr", "proxy.config.log.rolling_size_mb", "proxy.config.log.auto_delete_rolled_files", "proxy.config.log.rolling_max_count", "proxy.config.log.rolling_allow_empty", "proxy.config.log.config.filename", "proxy.config.log.sampling_frequency", "proxy.config.log.file_stat_frequency", "proxy.config.log.space_used_frequency", + "proxy.config.log.io.max_buffer_index", }; for (unsigned i = 0; i < countof(names); ++i) { @@ -561,7 +586,6 @@ LogConfig::update_space_used() return; } - int candidate_count; int64_t total_space_used, partition_space_left; char path[MAXPATHLEN]; int sret; @@ -603,7 +627,6 @@ LogConfig::update_space_used() } total_space_used = 0LL; - candidate_count = 0; while ((entry = readdir(ld))) { snprintf(path, MAXPATHLEN, "%s/%s", logfile_dir, entry->d_name); @@ -613,21 +636,7 @@ LogConfig::update_space_used() total_space_used += static_cast(sbuf.st_size); if (auto_delete_rolled_files && LogFile::rolled_logfile(entry->d_name)) { - // - // then check if the candidate belongs to any given log type - // - ts::TextView type_name(entry->d_name, strlen(entry->d_name)); - auto suffix = type_name; - type_name.remove_suffix(suffix.remove_prefix(suffix.find('.') + 1).remove_prefix(suffix.find('.')).size()); - auto iter = deleting_info.find(type_name); - if (iter == deleting_info.end()) { - // We won't delete the log if its name doesn't match any give type. - break; - } - - auto &candidates = iter->candidates; - candidates.push_back(LogDeleteCandidate(path, static_cast(sbuf.st_size), sbuf.st_mtime)); - candidate_count++; + rolledLogDeleter.consider_for_candidacy(path, sbuf.st_size, sbuf.st_mtime); } } } @@ -670,62 +679,50 @@ LogConfig::update_space_used() int64_t max_space = static_cast(get_max_space_mb()) * LOG_MEGABYTE; int64_t headroom = static_cast(max_space_mb_headroom) * LOG_MEGABYTE; - if (candidate_count > 0 && !space_to_write(headroom)) { + if (!space_to_write(headroom)) { Debug("logspace", "headroom reached, trying to clear space ..."); - Debug("logspace", "sorting %d delete candidates ...", candidate_count); - - deleting_info.apply([](LogDeletingInfo &info) { - std::sort(info.candidates.begin(), info.candidates.end(), - [](LogDeleteCandidate const &a, LogDeleteCandidate const &b) { return a.mtime > b.mtime; }); - }); + if (!rolledLogDeleter.has_candidates()) { + Note("Cannot clear space because there are no recognized Traffic Server rolled logs for auto deletion."); + } else { + Debug("logspace", "Considering %zu delete candidates ...", rolledLogDeleter.get_candidate_count()); + } - while (candidate_count > 0) { + while (rolledLogDeleter.has_candidates()) { if (space_to_write(headroom + log_buffer_size)) { Debug("logspace", "low water mark reached; stop deleting"); break; } - // Select the group with biggest ratio - auto target = - std::max_element(deleting_info.begin(), deleting_info.end(), [](LogDeletingInfo const &A, LogDeletingInfo const &B) { - return static_cast(A.candidates.size()) / A.min_count < static_cast(B.candidates.size()) / B.min_count; - }); - - auto &candidates = target->candidates; - + auto victim = rolledLogDeleter.take_next_candidate_to_delete(); // Check if any candidate exists - if (candidates.empty()) { + if (!victim) { // This shouldn't be triggered unless min_count are configured wrong or extra non-log files occupy the directory - Debug("logspace", "No more victims for log type %s. Check your rolling_min_count settings and logging directory.", - target->name.c_str()); + Debug("logspace", "No more victims. Check your rolling_min_count settings and logging directory."); } else { - auto &victim = candidates.back(); - Debug("logspace", "auto-deleting %s", victim.name.c_str()); + Debug("logspace", "auto-deleting %s", victim->rolled_log_path.c_str()); - if (unlink(victim.name.c_str()) < 0) { - Note("Traffic Server was Unable to auto-delete rolled " + if (unlink(victim->rolled_log_path.c_str()) < 0) { + Note("Traffic Server was unable to auto-delete rolled " "logfile %s: %s.", - victim.name.c_str(), strerror(errno)); + victim->rolled_log_path.c_str(), strerror(errno)); } else { Debug("logspace", "The rolled logfile, %s, was auto-deleted; " "%" PRId64 " bytes were reclaimed.", - victim.name.c_str(), victim.size); + victim->rolled_log_path.c_str(), victim->size); // Update after successful unlink; - m_space_used -= victim.size; - m_partition_space_left += victim.size; + m_space_used -= victim->size; + m_partition_space_left += victim->size; } - // Update total candidates and remove victim - --candidate_count; - candidates.pop_back(); } } } - // - // Clean up the candidate array - // - deleting_info.apply([](LogDeletingInfo &info) { info.clear(); }); + + // The set of files in the logs dir may change between iterations to check + // for logs to delete. To deal with this, we simply clear our internal + // candidates metadata and regenerate it on each iteration. + rolledLogDeleter.clear_candidates(); // // Now that we've updated the m_space_used value, see if we need to @@ -802,21 +799,21 @@ LogConfig::update_space_used() bool LogConfig::evaluate_config() { - ats_scoped_str path(RecConfigReadConfigPath("proxy.config.log.config.filename", "logging.yaml")); + ats_scoped_str path(RecConfigReadConfigPath("proxy.config.log.config.filename", ts::filename::LOGGING)); struct stat sbuf; if (stat(path.get(), &sbuf) == -1 && errno == ENOENT) { Warning("logging configuration '%s' doesn't exist", path.get()); return false; } - Note("logging.yaml loading ..."); + Note("%s loading ...", path.get()); YamlLogConfig y(this); bool zret = y.parse(path.get()); if (zret) { - Note("logging.yaml finished loading"); + Note("%s finished loading", path.get()); } else { - Note("logging.yaml failed to load"); + Note("%s failed to load", path.get()); } return zret; diff --git a/proxy/logging/LogConfig.h b/proxy/logging/LogConfig.h index 6d69cdf1499..858d64e8cd9 100644 --- a/proxy/logging/LogConfig.h +++ b/proxy/logging/LogConfig.h @@ -26,11 +26,11 @@ #include #include -#include "tscore/IntrusiveHashMap.h" #include "tscore/ink_platform.h" #include "records/P_RecProcess.h" #include "ProxyConfig.h" #include "LogObject.h" +#include "RolledLogDeleter.h" #include "tscpp/util/MemSpan.h" /* Instead of enumerating the stats in DynamicStats.h, each module needs @@ -78,73 +78,6 @@ extern RecRawStatBlock *log_rsb; struct dirent; -/*------------------------------------------------------------------------- - LogDeleteCandidate, LogDeletingInfo&Descriptor - -------------------------------------------------------------------------*/ - -struct LogDeleteCandidate { - std::string name; - int64_t size; - time_t mtime; - - LogDeleteCandidate(char *p_name, int64_t st_size, time_t st_time) : name(p_name), size(st_size), mtime(st_time) {} -}; - -struct LogDeletingInfo { - std::string name; - int min_count; - std::vector candidates; - - LogDeletingInfo *_next{nullptr}; - LogDeletingInfo *_prev{nullptr}; - - LogDeletingInfo(const char *type, int limit) : name(type), min_count(limit) {} - LogDeletingInfo(std::string_view type, int limit) : name(type), min_count(limit) {} - - void - clear() - { - candidates.clear(); - } -}; - -struct LogDeletingInfoDescriptor { - using key_type = std::string_view; - using value_type = LogDeletingInfo; - - static key_type - key_of(value_type *value) - { - return value->name; - } - - static bool - equal(key_type const &lhs, key_type const &rhs) - { - return lhs == rhs; - } - - static value_type *& - next_ptr(value_type *value) - { - return value->_next; - } - - static value_type *& - prev_ptr(value_type *value) - { - return value->_prev; - } - - static constexpr std::hash hasher{}; - - static auto - hash_of(key_type s) -> decltype(hasher(s)) - { - return hasher(s); - } -}; - /*------------------------------------------------------------------------- this object keeps the state of the logging configuraion variables. upon construction, the log configuration file is read and the logging @@ -224,6 +157,18 @@ class LogConfig : public ConfigInfo return log_object_manager.has_api_objects(); } + /** Register rolled logs of logname for auto-deletion when there are space + * constraints. + * + * @param[in] logname The name of the unrolled log to register, such as + * "diags.log". + * + * @param[in] rolling_min_count The minimum amount of rolled logs of logname + * to try to keep around. A value of 0 expresses a desire to keep all rolled + * files, if possible. + */ + void register_rolled_log_auto_delete(std::string_view logname, int rolling_min_count); + public: bool initialized = false; bool reconfiguration_needed = false; @@ -254,14 +199,13 @@ class LogConfig : public ConfigInfo bool rolling_allow_empty; bool auto_delete_rolled_files; - IntrusiveHashMap deleting_info; - int sampling_frequency; int file_stat_frequency; int space_used_frequency; int ascii_buffer_size; int max_line_size; + int logbuffer_max_iobuf_index; char *hostname; char *logfile_dir; @@ -278,6 +222,8 @@ class LogConfig : public ConfigInfo bool m_partition_low = false; bool m_log_directory_inaccessible = false; + RolledLogDeleter rolledLogDeleter; + // noncopyable // -- member functions not allowed -- LogConfig(const LogConfig &) = delete; diff --git a/proxy/logging/LogFile.cc b/proxy/logging/LogFile.cc index c0928beb6b7..89bdc808d59 100644 --- a/proxy/logging/LogFile.cc +++ b/proxy/logging/LogFile.cc @@ -63,12 +63,13 @@ -------------------------------------------------------------------------*/ LogFile::LogFile(const char *name, const char *header, LogFileFormat format, uint64_t signature, size_t ascii_buffer_size, - size_t max_line_size) + size_t max_line_size, int pipe_buffer_size) : m_file_format(format), m_name(ats_strdup(name)), m_header(ats_strdup(header)), m_signature(signature), - m_max_line_size(max_line_size) + m_max_line_size(max_line_size), + m_pipe_buffer_size(pipe_buffer_size) { if (m_file_format != LOG_FILE_PIPE) { m_log = new BaseLogFile(name, m_signature); @@ -97,6 +98,7 @@ LogFile::LogFile(const LogFile ©) m_signature(copy.m_signature), m_ascii_buffer_size(copy.m_ascii_buffer_size), m_max_line_size(copy.m_max_line_size), + m_pipe_buffer_size(copy.m_pipe_buffer_size), m_fd(copy.m_fd) { ink_release_assert(m_ascii_buffer_size >= m_max_line_size); @@ -116,6 +118,12 @@ LogFile::LogFile(const LogFile ©) LogFile::~LogFile() { Debug("log-file", "entering LogFile destructor, this=%p", this); + + // close_file() checks whether a file is open before attempting to close, so + // this is safe to call even if a file had not been opened. Further, calling + // close_file() here ensures that we do not leak file descriptors. + close_file(); + delete m_log; ats_free(m_header); ats_free(m_name); @@ -185,6 +193,30 @@ LogFile::open_file() Debug("log-file", "no readers for pipe %s", m_name); return LOG_FILE_NO_PIPE_READERS; } + +#ifdef F_GETPIPE_SZ + // adjust pipe size if necessary + if (m_pipe_buffer_size) { + long pipe_size = static_cast(fcntl(m_fd, F_GETPIPE_SZ)); + if (pipe_size == -1) { + Error("Get pipe size failed for pipe %s: %s", m_name, strerror(errno)); + } else { + Debug("log-file", "Previous buffer size for pipe %s: %ld", m_name, pipe_size); + } + + int ret = fcntl(m_fd, F_SETPIPE_SZ, m_pipe_buffer_size); + if (ret == -1) { + Error("Set pipe size failed for pipe %s to size %d: %s", m_name, m_pipe_buffer_size, strerror(errno)); + } + + pipe_size = static_cast(fcntl(m_fd, F_GETPIPE_SZ)); + if (pipe_size == -1) { + Error("Get pipe size after setting it failed for pipe %s: %s", m_name, strerror(errno)); + } else { + Debug("log-file", "New buffer size for pipe %s: %ld", m_name, pipe_size); + } + } +#endif // F_GETPIPE_SZ } else { if (m_log) { int status = m_log->open_file(Log::config->logfile_perm); @@ -226,18 +258,24 @@ LogFile::close_file() { if (is_open()) { if (m_file_format == LOG_FILE_PIPE) { - ::close(m_fd); - Debug("log-file", "LogFile %s (fd=%d) is closed", m_name, m_fd); + if (::close(m_fd)) { + Error("Error closing LogFile %s: %s.", m_name, strerror(errno)); + } else { + Debug("log-file", "LogFile %s (fd=%d) is closed", m_name, m_fd); + RecIncrRawStat(log_rsb, this_thread()->mutex->thread_holding, log_stat_log_files_open_stat, -1); + } m_fd = -1; } else if (m_log) { - m_log->close_file(); - Debug("log-file", "LogFile %s is closed", m_log->get_name()); + if (m_log->close_file()) { + Error("Error closing LogFile %s: %s.", m_log->get_name(), strerror(errno)); + } else { + Debug("log-file", "LogFile %s is closed", m_log->get_name()); + RecIncrRawStat(log_rsb, this_thread()->mutex->thread_holding, log_stat_log_files_open_stat, -1); + } } else { Warning("LogFile %s is open but was not closed", m_name); } } - - RecIncrRawStat(log_rsb, this_thread()->mutex->thread_holding, log_stat_log_files_open_stat, -1); } struct RolledFile { @@ -337,7 +375,9 @@ LogFile::roll(long interval_start, long interval_end, bool reopen_after_rolling) // Since these two methods of using BaseLogFile are not compatible, we perform the logging log file specific // close file operation here within the containing LogFile object. if (m_log->roll(interval_start, interval_end)) { - m_log->close_file(); + if (m_log->close_file()) { + Error("Error closing LogFile %s: %s.", m_log->get_name(), strerror(errno)); + } if (reopen_after_rolling) { /* If we re-open now log file will be created even if there is nothing being logged */ diff --git a/proxy/logging/LogFile.h b/proxy/logging/LogFile.h index 4f2fd65853d..1a9a7f1ee8c 100644 --- a/proxy/logging/LogFile.h +++ b/proxy/logging/LogFile.h @@ -43,7 +43,7 @@ class LogFile : public LogBufferSink, public RefCountObj { public: LogFile(const char *name, const char *header, LogFileFormat format, uint64_t signature, size_t ascii_buffer_size = 4 * 9216, - size_t max_line_size = 9216); + size_t max_line_size = 9216, int pipe_buffer_size = 0); LogFile(const LogFile &); ~LogFile() override; @@ -120,6 +120,7 @@ class LogFile : public LogBufferSink, public RefCountObj uint64_t m_signature; // signature of log object stored size_t m_ascii_buffer_size; // size of ascii buffer size_t m_max_line_size; // size of longest log line (record) + int m_pipe_buffer_size; // this is the size of the pipe buffer set by fcntl int m_fd; // this could back m_log or a pipe, depending on the situation public: diff --git a/proxy/logging/LogFilter.h b/proxy/logging/LogFilter.h index 1cb55fead46..083dc08e6f0 100644 --- a/proxy/logging/LogFilter.h +++ b/proxy/logging/LogFilter.h @@ -382,6 +382,72 @@ LogFilterString::_checkCondition(OperatorFunction f, const char *field_value, si return retVal; } +/*--------------------------------------------------------------------------- + * find pattern from the query param + * 1) if pattern is not in the query param, return nullptr + * 2) if got the pattern in one param's value, search again until it's in one param name, or nullptr if can't find it from param +name +---------------------------------------------------------------------------*/ +static const char * +findPatternFromParamName(const char *lookup_query_param, const char *pattern) +{ + const char *pattern_in_query_param = strstr(lookup_query_param, pattern); + while (pattern_in_query_param) { + // wipe pattern in param name, need to search again if find pattern in param value + const char *param_value_str = strchr(pattern_in_query_param, '='); + if (!param_value_str) { + // no "=" after pattern_in_query_param, means pattern_in_query_param is not in the param name, and no more param after it + pattern_in_query_param = nullptr; + break; + } + const char *param_name_str = strchr(pattern_in_query_param, '&'); + if (param_name_str && param_value_str > param_name_str) { + //"=" is after "&" followd by pattern_in_query_param, means pattern_in_query_param is not in the param name + pattern_in_query_param = strstr(param_name_str, pattern); + continue; + } + // ensure pattern_in_query_param is in the param name now + break; + } + return pattern_in_query_param; +} + +/*--------------------------------------------------------------------------- + * replace param value whose name contains pattern with same count 'X' of original value str length +---------------------------------------------------------------------------*/ +static void +updatePatternForFieldValue(char **field, const char *pattern_str, int field_pos, char *buf_dest) +{ + int buf_dest_len = strlen(buf_dest); + char buf_dest_to_field[buf_dest_len + 1]; + char *temp_text = buf_dest_to_field; + memcpy(temp_text, buf_dest, (pattern_str - buf_dest)); + temp_text += (pattern_str - buf_dest); + const char *value_str = strchr(pattern_str, '='); + if (value_str) { + value_str++; + memcpy(temp_text, pattern_str, (value_str - pattern_str)); + temp_text += (value_str - pattern_str); + const char *next_param_str = strchr(value_str, '&'); + if (next_param_str) { + for (int i = 0; i < (next_param_str - value_str); i++) { + temp_text[i] = 'X'; + } + temp_text += (next_param_str - value_str); + memcpy(temp_text, next_param_str, ((buf_dest + buf_dest_len) - next_param_str)); + } else { + for (int i = 0; i < ((buf_dest + buf_dest_len) - value_str); i++) { + temp_text[i] = 'X'; + } + } + } else { + return; + } + + buf_dest_to_field[buf_dest_len] = '\0'; + strcpy(*field, buf_dest_to_field); +} + /*--------------------------------------------------------------------------- wipeField : Given a dest buffer, wipe the first occurrence of the value of the field in the buffer. @@ -394,44 +460,25 @@ wipeField(char **field, char *pattern, const char *uppercase_field) const char *lookup_dest = uppercase_field ? uppercase_field : *field; if (buf_dest) { - char *query_param = strstr(buf_dest, "?"); - const char *lookup_query_param = strstr(lookup_dest, "?"); + char *query_param = strchr(buf_dest, '?'); + const char *lookup_query_param = strchr(lookup_dest, '?'); if (!query_param || !lookup_query_param) { return; } - const char *p1 = strstr(lookup_query_param, pattern); - int field_pos = p1 - lookup_query_param; - p1 = query_param + field_pos; - - if (p1) { - char tmp_text[strlen(buf_dest) + 10]; - char *temp_text = tmp_text; - memcpy(temp_text, buf_dest, (p1 - buf_dest)); - temp_text += (p1 - buf_dest); - const char *p2 = strstr(p1, "="); - if (p2) { - p2++; - memcpy(temp_text, p1, (p2 - p1)); - temp_text += (p2 - p1); - const char *p3 = strstr(p2, "&"); - if (p3) { - for (int i = 0; i < (p3 - p2); i++) { - temp_text[i] = 'X'; - } - temp_text += (p3 - p2); - memcpy(temp_text, p3, ((buf_dest + strlen(buf_dest)) - p3)); - } else { - for (int i = 0; i < ((buf_dest + strlen(buf_dest)) - p2); i++) { - temp_text[i] = 'X'; - } - } + const char *pattern_in_param_name = findPatternFromParamName(lookup_query_param, pattern); + while (pattern_in_param_name) { + int field_pos = pattern_in_param_name - lookup_query_param; + pattern_in_param_name = query_param + field_pos; + updatePatternForFieldValue(field, pattern_in_param_name, field_pos, buf_dest); + + // search new param again + const char *new_param = strchr(lookup_query_param + field_pos, '&'); + if (new_param && (new_param + 1)) { + pattern_in_param_name = findPatternFromParamName(new_param + 1, pattern); } else { - return; + break; } - - tmp_text[strlen(buf_dest)] = '\0'; - strcpy(*field, tmp_text); } } } diff --git a/proxy/logging/LogObject.cc b/proxy/logging/LogObject.cc index 8032dbdd296..86d2a454599 100644 --- a/proxy/logging/LogObject.cc +++ b/proxy/logging/LogObject.cc @@ -90,7 +90,8 @@ LogBufferManager::preproc_buffers(LogBufferSink *sink) LogObject::LogObject(const LogFormat *format, const char *log_dir, const char *basename, LogFileFormat file_format, const char *header, Log::RollingEnabledValues rolling_enabled, int flush_threads, int rolling_interval_sec, - int rolling_offset_hr, int rolling_size_mb, bool auto_created, int max_rolled, bool reopen_after_rolling) + int rolling_offset_hr, int rolling_size_mb, bool auto_created, int rolling_max_count, int rolling_min_count, + bool reopen_after_rolling, int pipe_buffer_size) : m_alt_filename(nullptr), m_flags(0), m_signature(0), @@ -99,9 +100,11 @@ LogObject::LogObject(const LogFormat *format, const char *log_dir, const char *b m_rolling_offset_hr(rolling_offset_hr), m_rolling_size_mb(rolling_size_mb), m_last_roll_time(0), - m_max_rolled(max_rolled), + m_max_rolled(rolling_max_count), + m_min_rolled(rolling_min_count), m_reopen_after_rolling(reopen_after_rolling), - m_buffer_manager_idx(0) + m_buffer_manager_idx(0), + m_pipe_buffer_size(pipe_buffer_size) { ink_release_assert(format); m_format = new LogFormat(*format); @@ -118,7 +121,8 @@ LogObject::LogObject(const LogFormat *format, const char *log_dir, const char *b // compute_signature is a static function m_signature = compute_signature(m_format, m_basename, m_flags); - m_logFile = new LogFile(m_filename, header, file_format, m_signature, Log::config->ascii_buffer_size, Log::config->max_line_size); + m_logFile = new LogFile(m_filename, header, file_format, m_signature, Log::config->ascii_buffer_size, Log::config->max_line_size, + m_pipe_buffer_size); if (m_reopen_after_rolling) { m_logFile->open_file(); @@ -147,8 +151,10 @@ LogObject::LogObject(LogObject &rhs) m_rolling_size_mb(rhs.m_rolling_size_mb), m_last_roll_time(rhs.m_last_roll_time), m_max_rolled(rhs.m_max_rolled), + m_min_rolled(rhs.m_min_rolled), m_reopen_after_rolling(rhs.m_reopen_after_rolling), - m_buffer_manager_idx(rhs.m_buffer_manager_idx) + m_buffer_manager_idx(rhs.m_buffer_manager_idx), + m_pipe_buffer_size(rhs.m_pipe_buffer_size) { m_format = new LogFormat(*(rhs.m_format)); m_buffer_manager = new LogBufferManager[m_flush_threads]; @@ -683,7 +689,7 @@ LogObject::_setup_rolling(Log::RollingEnabledValues rolling_enabled, int rolling m_rolling_size_mb = rolling_size_mb; } } - + Log::config->register_rolled_log_auto_delete(m_basename, m_min_rolled); m_rolling_enabled = rolling_enabled; } } @@ -785,9 +791,11 @@ const LogFormat *TextLogObject::textfmt = MakeTextLogFormat(); TextLogObject::TextLogObject(const char *name, const char *log_dir, bool timestamps, const char *header, Log::RollingEnabledValues rolling_enabled, int flush_threads, int rolling_interval_sec, - int rolling_offset_hr, int rolling_size_mb, int max_rolled, bool reopen_after_rolling) + int rolling_offset_hr, int rolling_size_mb, int rolling_max_count, int rolling_min_count, + bool reopen_after_rolling) : LogObject(TextLogObject::textfmt, log_dir, name, LOG_FILE_ASCII, header, rolling_enabled, flush_threads, rolling_interval_sec, - rolling_offset_hr, rolling_size_mb, max_rolled, reopen_after_rolling) + rolling_offset_hr, rolling_size_mb, /* auto_created */ false, rolling_max_count, rolling_min_count, + reopen_after_rolling) { if (timestamps) { this->set_fmt_timestamps(); @@ -957,24 +965,20 @@ LogObjectManager::_solve_filename_conflicts(LogObject *log_object, int maxConfli LogUtils::manager_alarm(LogUtils::LOG_ALARM_ERROR, msg, filename); retVal = CANNOT_SOLVE_FILENAME_CONFLICTS; } else { - // either the meta file could not be read, or the new object's - // signature and the metafile signature do not match ==> - // roll old filename so the new object can use the filename + // Either the meta file could not be read, or the new object's + // signature and the metafile signature do not match. + // Roll the old filename so the new object can use the filename // it requested (previously we used to rename the NEW file - // but now we roll the OLD file), or if the log object writes - // to a pipe, just remove the file if it was open as a pipe + // but now we roll the OLD file). However, if the log object writes to + // a pipe don't roll because rolling is not applicable to pipes. bool roll_file = true; if (log_object->writes_to_pipe()) { - // determine if existing file is a pipe, and remove it if - // that is the case so the right metadata for the new pipe - // is created later - // + // Verify whether the existing file is a pipe. If it is, + // disable the roll_file flag so we don't attempt rolling. struct stat s; if (stat(filename, &s) < 0) { - // an error happened while trying to get file info - // const char *msg = "Cannot stat log file %s: %s"; char *se = strerror(errno); @@ -984,7 +988,6 @@ LogObjectManager::_solve_filename_conflicts(LogObject *log_object, int maxConfli roll_file = false; } else { if (S_ISFIFO(s.st_mode)) { - unlink(filename); roll_file = false; } } diff --git a/proxy/logging/LogObject.h b/proxy/logging/LogObject.h index fcf604662f5..87f5c8061aa 100644 --- a/proxy/logging/LogObject.h +++ b/proxy/logging/LogObject.h @@ -95,7 +95,8 @@ class LogObject : public RefCountObj LogObject(const LogFormat *format, const char *log_dir, const char *basename, LogFileFormat file_format, const char *header, Log::RollingEnabledValues rolling_enabled, int flush_threads, int rolling_interval_sec = 0, int rolling_offset_hr = 0, - int rolling_size_mb = 0, bool auto_created = false, int rolling_max_count = 0, bool reopen_after_rolling = false); + int rolling_size_mb = 0, bool auto_created = false, int rolling_max_count = 0, int rolling_min_count = 0, + bool reopen_after_rolling = false, int pipe_buffer_size = 0); LogObject(LogObject &); ~LogObject() override; @@ -279,12 +280,15 @@ class LogObject : public RefCountObj int m_rolling_size_mb; // size at which the log file rolls long m_last_roll_time; // the last time this object rolled its files int m_max_rolled; // maximum number of rolled logs to be kept, 0 no limit + int m_min_rolled; // minimum number of rolled logs to be kept, 0 no limit bool m_reopen_after_rolling; // reopen log file after rolling (normally it is just renamed and closed) head_p m_log_buffer; // current work buffer unsigned m_buffer_manager_idx; LogBufferManager *m_buffer_manager; + int m_pipe_buffer_size; + void generate_filenames(const char *log_dir, const char *basename, LogFileFormat file_format); void _setup_rolling(Log::RollingEnabledValues rolling_enabled, int rolling_interval_sec, int rolling_offset_hr, int rolling_size_mb); @@ -310,7 +314,8 @@ class TextLogObject : public LogObject public: inkcoreapi TextLogObject(const char *name, const char *log_dir, bool timestamps, const char *header, Log::RollingEnabledValues rolling_enabled, int flush_threads, int rolling_interval_sec, - int rolling_offset_hr, int rolling_size_mb, int max_rolled, bool reopen_after_rolling); + int rolling_offset_hr, int rolling_size_mb, int rolling_max_count, int rolling_min_count, + bool reopen_after_rolling); inkcoreapi int write(const char *format, ...) TS_PRINTFLIKE(2, 3); inkcoreapi int va_write(const char *format, va_list ap); @@ -411,7 +416,7 @@ LogObject::operator==(LogObject &old) strcmp(m_logFile->get_name(), old.m_logFile->get_name()) == 0 && (m_filter_list == old.m_filter_list) && (m_rolling_interval_sec == old.m_rolling_interval_sec && m_rolling_offset_hr == old.m_rolling_offset_hr && m_rolling_size_mb == old.m_rolling_size_mb && m_reopen_after_rolling == old.m_reopen_after_rolling && - m_max_rolled == old.m_max_rolled)); + m_max_rolled == old.m_max_rolled && m_min_rolled == old.m_min_rolled)); } inline off_t diff --git a/proxy/logging/LogUtils.cc b/proxy/logging/LogUtils.cc index 95e3852bc12..4ec68e9eb97 100644 --- a/proxy/logging/LogUtils.cc +++ b/proxy/logging/LogUtils.cc @@ -478,6 +478,34 @@ LogUtils::seconds_to_next_roll(time_t time_now, int rolling_offset, int rolling_ return ((tr >= sidl ? (tr - sidl) % rolling_interval : (86400 - (sidl - tr)) % rolling_interval)); } +ts::TextView +LogUtils::get_unrolled_filename(ts::TextView rolled_filename) +{ + auto unrolled_name = rolled_filename; + + // A rolled log will look something like: + // squid.log_some.hostname.com.20191029.18h15m02s-20191029.18h30m02s.old + auto suffix = rolled_filename; + + suffix.remove_prefix_at('.'); + // Using the above squid.log example, suffix now looks like: + // log_some.hostname.com.20191029.18h15m02s-20191029.18h30m02s.old + + // Some suffixes do not have the hostname. Rolled diags.log files will look + // something like this, for example: + // diags.log.20191114.21h43m16s-20191114.21h43m17s.old + // + // For these, the second delimeter will be a period. For this reason, we also + // split_prefix_at with a period as well. + if (suffix.split_prefix_at('_') || suffix.split_prefix_at('.')) { + // ' + 1' to remove the '_' or second '.': + return unrolled_name.remove_suffix(suffix.size() + 1); + } + // If there isn't a '.' or an '_' after the first '.', then this + // doesn't look like a rolled file. + return rolled_filename; +} + // Checks if the file pointed to by full_filename either is a regular // file or a pipe and has write permission, or, if the file does not // exist, if the path prefix of full_filename names a directory that diff --git a/proxy/logging/LogUtils.h b/proxy/logging/LogUtils.h index 2860db1b63f..ade74181593 100644 --- a/proxy/logging/LogUtils.h +++ b/proxy/logging/LogUtils.h @@ -26,6 +26,7 @@ #include "tscore/ink_platform.h" #include "tscore/Arena.h" +#include class MIMEHdr; @@ -62,6 +63,22 @@ int seconds_to_next_roll(time_t time_now, int rolling_offset, int rolling_interv int file_is_writeable(const char *full_filename, off_t *size_bytes = nullptr, bool *has_size_limit = nullptr, uint64_t *current_size_limit_bytes = nullptr); +/** Given a rolled file, determine the unrolled filename. + * + * For example, given this: + * diags.log.20191114.21h43m16s-20191114.21h43m17s.old + * + * Return this: + * diags.log + * + * @param[in] rolled_filename The rolled filename from which to derive the + * unrolled filename. + * + * @return The unrolled filename if it looked like a rolled log file or the + * input filename if it didn't. + */ +ts::TextView get_unrolled_filename(ts::TextView rolled_filename); + // Marshals header tags and values together, with a single terminating nul character. Returns buffer space required. 'buf' points // to where to put the marshaled data. If 'buf' is null, no data is marshaled, but the function returns the amount of space that // would have been used. diff --git a/proxy/logging/Makefile.am b/proxy/logging/Makefile.am index abf8e866cfc..58b6b556d9a 100644 --- a/proxy/logging/Makefile.am +++ b/proxy/logging/Makefile.am @@ -61,19 +61,24 @@ liblogging_a_SOURCES = \ LogObject.h \ LogUtils.cc \ LogUtils.h \ + RolledLogDeleter.cc \ + RolledLogDeleter.h \ YamlLogConfig.cc \ YamlLogConfigDecoders.cc \ YamlLogConfig.h check_PROGRAMS = \ test_LogUtils \ - test_LogUtils2 + test_LogUtils2 \ + test_RolledLogDeleter TESTS = \ test_LogUtils \ - test_LogUtils2 + test_LogUtils2 \ + test_RolledLogDeleter -test_LogUtils_CPPFLAGS = $(AM_CPPFLAGS)\ +test_LogUtils_CPPFLAGS = \ + $(AM_CPPFLAGS) \ -DTEST_LOG_UTILS test_LogUtils_SOURCES = \ @@ -84,7 +89,8 @@ test_LogUtils_LDADD = \ $(top_builddir)/src/tscpp/util/libtscpputil.la \ $(top_builddir)/iocore/eventsystem/libinkevent.a -test_LogUtils2_CPPFLAGS = $(AM_CPPFLAGS)\ +test_LogUtils2_CPPFLAGS = \ + $(AM_CPPFLAGS) \ -DTEST_LOG_UTILS \ -I$(abs_top_srcdir)/tests/include @@ -98,5 +104,20 @@ test_LogUtils2_LDADD = \ $(top_builddir)/src/tscpp/util/libtscpputil.la \ $(top_builddir)/iocore/eventsystem/libinkevent.a +test_RolledLogDeleter_CPPFLAGS = \ + $(AM_CPPFLAGS) \ + -DTEST_LOG_UTILS \ + -I$(abs_top_srcdir)/tests/include + +test_RolledLogDeleter_SOURCES = \ + RolledLogDeleter.cc \ + LogUtils.cc \ + unit-tests/test_RolledLogDeleter.cc + +test_RolledLogDeleter_LDADD = \ + $(top_builddir)/src/tscore/libtscore.la \ + $(top_builddir)/src/tscpp/util/libtscpputil.la \ + $(top_builddir)/iocore/eventsystem/libinkevent.a + clang-tidy-local: $(liblogging_a_SOURCES) $(EXTRA_DIST) $(CXX_Clang_Tidy) diff --git a/proxy/logging/RolledLogDeleter.cc b/proxy/logging/RolledLogDeleter.cc new file mode 100644 index 00000000000..2d5173888cc --- /dev/null +++ b/proxy/logging/RolledLogDeleter.cc @@ -0,0 +1,135 @@ +/** @file + + This file implements the rolled log deletion. + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#include +#include +#include +#include + +#include "RolledLogDeleter.h" +#include "LogUtils.h" +#include "tscore/ts_file.h" +#include "tscpp/util/TextView.h" + +namespace fs = ts::file; + +LogDeletingInfo::LogDeletingInfo(const char *_logname, int _min_count) + : logname(_logname), + /** + * A min_count of zero indicates a request to try to keep all rotated logs + * around. By setting min_count to INT_MAX in these cases, we make the rolled + * log deletion priority small. + * + * @note This cannot have a zero value because it is used as the denominator + * in a division operation when calculating the log deletion preference. + */ + min_count((_min_count > 0) ? _min_count : INT_MAX) +{ +} + +LogDeletingInfo::LogDeletingInfo(std::string_view _logname, int _min_count) + : logname(_logname), + /** + * A min_count of zero indicates a request to try to keep all rotated logs + * around. By setting min_count to INT_MAX in these cases, we make the rolled + * log deletion priority small. + * + * @note This cannot have a zero value because it is used as the denominator + * in a division operation when calculating the log deletion preference. + */ + min_count((_min_count > 0) ? _min_count : INT_MAX) +{ +} + +void +RolledLogDeleter::register_log_type_for_deletion(std::string_view log_type, int rolling_min_count) +{ + auto deletingInfo = std::make_unique(log_type, rolling_min_count); + auto *deletingInfoPtr = deletingInfo.get(); + + deletingInfoList.push_back(std::move(deletingInfo)); + deleting_info.insert(deletingInfoPtr); +} + +bool +RolledLogDeleter::consider_for_candidacy(std::string_view log_path, int64_t file_size, time_t modification_time) +{ + const fs::path rolled_log_file = fs::filename(log_path); + auto iter = deleting_info.find(LogUtils::get_unrolled_filename(rolled_log_file.view())); + if (iter == deleting_info.end()) { + return false; + } + auto &candidates = iter->candidates; + candidates.push_back(std::make_unique(log_path, file_size, modification_time)); + ++num_candidates; + + std::sort( + candidates.begin(), candidates.end(), + [](std::unique_ptr const &a, std::unique_ptr const &b) { return a->mtime > b->mtime; }); + + return true; +} + +std::unique_ptr +RolledLogDeleter::take_next_candidate_to_delete() +{ + if (!has_candidates()) { + return nullptr; + } + // Select the highest priority type (diags.log, traffic.out, etc.) from which + // to select a candidate. + auto target_type = + std::max_element(deleting_info.begin(), deleting_info.end(), [](LogDeletingInfo const &A, LogDeletingInfo const &B) { + return static_cast(A.candidates.size()) / A.min_count < static_cast(B.candidates.size()) / B.min_count; + }); + + auto &candidates = target_type->candidates; + if (candidates.empty()) { + return nullptr; + } + + // Return the highest priority candidate among the candidates of that type. + auto victim = std::move(candidates.back()); + candidates.pop_back(); + --num_candidates; + return victim; +} + +bool +RolledLogDeleter::has_candidates() const +{ + return get_candidate_count() != 0; +} + +size_t +RolledLogDeleter::get_candidate_count() const +{ + return num_candidates; +} + +void +RolledLogDeleter::clear_candidates() +{ + deleting_info.apply([](LogDeletingInfo &info) { info.clear(); }); + num_candidates = 0; +} diff --git a/proxy/logging/RolledLogDeleter.h b/proxy/logging/RolledLogDeleter.h new file mode 100644 index 00000000000..f03785f6e44 --- /dev/null +++ b/proxy/logging/RolledLogDeleter.h @@ -0,0 +1,214 @@ +/** @file + + This contains the rotated log deletion mechanism. + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +#include "tscore/IntrusiveHashMap.h" + +/*------------------------------------------------------------------------- + LogDeleteCandidate, LogDeletingInfo&Descriptor + -------------------------------------------------------------------------*/ + +struct LogDeleteCandidate { + /** The filename for this rolled log deletion candidate. + * + * For example: /var/log/my_log.log_a_host_name.20191122.20h18m35s-20191122.20h18m51s.old + */ + std::string rolled_log_path; + int64_t size; + time_t mtime; + + LogDeleteCandidate(std::string_view p_name, int64_t st_size, time_t st_time) + : rolled_log_path(p_name), size(st_size), mtime(st_time) + { + } +}; + +/** Configure rolled log deletion for a set of logs. + * + * This contains the configuration and set of log deletion candidates for a set + * of log files associated with logname. There will be an instance of this for + * diags.log and its associated rolled log files, one for traffic.out, etc. + */ +struct LogDeletingInfo { + /** The unrolled log name (such as "diags.log"). */ + const std::string logname; + + /** The minimum number of rolled log files to try to keep around. + * + * @note This is guaranteed to be a positive (non-zero) value. + */ + int min_count; + + std::vector> candidates; + + LogDeletingInfo *_next{nullptr}; + LogDeletingInfo *_prev{nullptr}; + + /** + * @param[in] logname The unrolled log name. + * + * @param[in] min_count The minimum number of rolled files to try to keep + * around when deleting rolled logs. A zero indicates a desire to keep all + * rolled logs around. + * + * @note The min_count is used as a part of a calculation to determine which + * set of deletion candidates should be used for selecting a rolled log file + * to delete. If space is particularly constrained, even LogDeletingInfo + * instances with a min_count of 0 may be selected for deletion. + */ + LogDeletingInfo(const char *logname, int min_count); + LogDeletingInfo(std::string_view logname, int min_count); + + void + clear() + { + candidates.clear(); + } +}; + +struct LogDeletingInfoDescriptor { + using key_type = std::string_view; + using value_type = LogDeletingInfo; + + static key_type + key_of(value_type *value) + { + return value->logname; + } + + static bool + equal(key_type const &lhs, key_type const &rhs) + { + return lhs == rhs; + } + + static value_type *& + next_ptr(value_type *value) + { + return value->_next; + } + + static value_type *& + prev_ptr(value_type *value) + { + return value->_prev; + } + + static constexpr std::hash hasher{}; + + static auto + hash_of(key_type s) -> decltype(hasher(s)) + { + return hasher(s); + } +}; + +/** + * RolledLogDeleter is responsible for keeping track of rolled log candidates + * and presenting them for deletion in a prioritized order based on size and + * last modified time stamp. + * + * Terminology: + * + * log type: An unrolled log name that represents a category of rolled log + * files that are candidates for deletion. This may be something like + * diags.log, traffic.out, etc. + * + * candidate: A rolled log file which is a candidate for deletion at some + * point. This may be something like: + * squid.log_some.hostname.com.20191125.19h00m04s-20191125.19h15m04s.old. + */ +class RolledLogDeleter +{ +public: + /** Register a new log type for candidates for log deletion. + * + * @param[in] log_type The unrolled name for a set of rolled log files to + * consider for deletion. This may be something like diags.log, for example. + * + * @param[in] rolling_min_count The minimum number of rolled log files to + * keep around. + */ + void register_log_type_for_deletion(std::string_view log_type, int rolling_min_count); + + /** Evaluate a rolled log file to see whether it is a candidate for deletion. + * + * If the rolled log file is a valid candidate, it will be stored and considered + * for deletion upon later calls to deleteALogFile. + * + * @param[in] log_path The rolled log file path. + * + * @param[in] file_size The size of the rolled log file. + * + * @param[in] modification_time The time the rolled log file was last modified. + * candidate for deletion. + * + * @return True if the rolled log file is a deletion candidate, false otherwise. + */ + bool consider_for_candidacy(std::string_view log_path, int64_t file_size, time_t modification_time); + + /** Retrieve the next rolled log file to delete. + * + * This removes the returned rolled file from the candidates list. + * + * @return The next rolled log candidate to delete or nullptr if there is no + * such candidate. + */ + std::unique_ptr take_next_candidate_to_delete(); + + /** Whether there are any candidates for possible deletion. + * + * @return True if there are candidates for deletion, false otherwise. + */ + bool has_candidates() const; + + /** Retrieve the number of rolled log deletion candidates. + * + * @return The number of rolled logs that are candidates for deletion. + */ + size_t get_candidate_count() const; + + /** Clear the internal candidates array. + */ + void clear_candidates(); + +private: + /** The owning references to the set of LogDeletingInfo added to the below + * hash map. */ + std::list> deletingInfoList; + + /** The set of candidates for deletion keyed by log_type. */ + IntrusiveHashMap deleting_info; + + /** The number of tracked candidates. */ + size_t num_candidates = 0; +}; diff --git a/proxy/logging/YamlLogConfig.cc b/proxy/logging/YamlLogConfig.cc index 398bfcbbc80..5780aa6b030 100644 --- a/proxy/logging/YamlLogConfig.cc +++ b/proxy/logging/YamlLogConfig.cc @@ -54,14 +54,14 @@ YamlLogConfig::loadLogConfig(const char *cfgFilename) } if (!config.IsMap()) { - Error("malformed logging.yaml file; expected a map"); + Error("malformed %s file; expected a map", cfgFilename); return false; } if (config["logging"]) { config = config["logging"]; } else { - Error("malformed logging.yaml file; expected a toplevel 'logging' node"); + Error("malformed %s file; expected a toplevel 'logging' node", cfgFilename); return false; } @@ -109,9 +109,19 @@ TsEnumDescriptor ROLLING_MODE_TEXT = {{{"none", 0}, {"time", 1}, {"size", 2}, {" TsEnumDescriptor ROLLING_MODE_LUA = { {{"log.roll.none", 0}, {"log.roll.time", 1}, {"log.roll.size", 2}, {"log.roll.both", 3}, {"log.roll.any", 4}}}; -std::set valid_log_object_keys = { - "filename", "format", "mode", "header", "rolling_enabled", "rolling_interval_sec", - "rolling_offset_hr", "rolling_size_mb", "filters", "min_count", "rolling_max_count", "rolling_allow_empty"}; +std::set valid_log_object_keys = {"filename", + "format", + "mode", + "header", + "rolling_enabled", + "rolling_interval_sec", + "rolling_offset_hr", + "rolling_size_mb", + "filters", + "rolling_min_count", + "rolling_max_count", + "rolling_allow_empty", + "pipe_buffer_size"}; LogObject * YamlLogConfig::decodeLogObject(const YAML::Node &node) @@ -119,7 +129,7 @@ YamlLogConfig::decodeLogObject(const YAML::Node &node) for (auto const &item : node) { if (std::none_of(valid_log_object_keys.begin(), valid_log_object_keys.end(), [&item](const std::string &s) { return s == item.first.as(); })) { - throw YAML::ParserException(item.Mark(), "log: unsupported key '" + item.first.as() + "'"); + throw YAML::ParserException(item.first.Mark(), "log: unsupported key '" + item.first.as() + "'"); } } @@ -157,7 +167,7 @@ YamlLogConfig::decodeLogObject(const YAML::Node &node) int obj_rolling_interval_sec = cfg->rolling_interval_sec; int obj_rolling_offset_hr = cfg->rolling_offset_hr; int obj_rolling_size_mb = cfg->rolling_size_mb; - int obj_min_count = cfg->rolling_min_count; + int obj_rolling_min_count = cfg->rolling_min_count; int obj_rolling_max_count = cfg->rolling_max_count; int obj_rolling_allow_empty = cfg->rolling_allow_empty; @@ -183,8 +193,8 @@ YamlLogConfig::decodeLogObject(const YAML::Node &node) if (node["rolling_size_mb"]) { obj_rolling_size_mb = node["rolling_size_mb"].as(); } - if (node["min_count"]) { - obj_min_count = node["min_count"].as(); + if (node["rolling_min_count"]) { + obj_rolling_min_count = node["rolling_min_count"].as(); } if (node["rolling_max_count"]) { obj_rolling_max_count = node["rolling_max_count"].as(); @@ -196,11 +206,21 @@ YamlLogConfig::decodeLogObject(const YAML::Node &node) Warning("Invalid log rolling value '%d' in log object", obj_rolling_enabled); } - auto logObject = - new LogObject(fmt, Log::config->logfile_dir, filename.c_str(), file_type, header.c_str(), - static_cast(obj_rolling_enabled), Log::config->preproc_threads, - obj_rolling_interval_sec, obj_rolling_offset_hr, obj_rolling_size_mb, /* auto_created */ false, - /* rolling_max_count */ obj_rolling_max_count, /* reopen_after_rolling */ obj_rolling_allow_empty > 0); + // get buffer for pipe + int pipe_buffer_size = 0; + if (node["pipe_buffer_size"]) { + if (file_type != LOG_FILE_PIPE) { + Warning("Pipe buffer size field should only be set for log.pipe object."); + } else { + pipe_buffer_size = node["pipe_buffer_size"].as(); + } + } + + auto logObject = new LogObject(fmt, Log::config->logfile_dir, filename.c_str(), file_type, header.c_str(), + static_cast(obj_rolling_enabled), Log::config->preproc_threads, + obj_rolling_interval_sec, obj_rolling_offset_hr, obj_rolling_size_mb, /* auto_created */ false, + /* rolling_max_count */ obj_rolling_max_count, /* rolling_min_count */ obj_rolling_min_count, + /* reopen_after_rolling */ obj_rolling_allow_empty > 0, pipe_buffer_size); // Generate LogDeletingInfo entry for later use std::string ext; @@ -217,7 +237,6 @@ YamlLogConfig::decodeLogObject(const YAML::Node &node) default: break; } - cfg->deleting_info.insert(new LogDeletingInfo(filename + ext, ((obj_min_count == 0) ? INT_MAX : obj_min_count))); // filters auto filters = node["filters"]; diff --git a/proxy/logging/YamlLogConfigDecoders.cc b/proxy/logging/YamlLogConfigDecoders.cc index 57264458614..fca2817f578 100644 --- a/proxy/logging/YamlLogConfigDecoders.cc +++ b/proxy/logging/YamlLogConfigDecoders.cc @@ -36,10 +36,10 @@ namespace YAML bool convert>::decode(const Node &node, std::unique_ptr &logFormat) { - for (auto &&item : node) { + for (const auto &item : node) { if (std::none_of(valid_log_format_keys.begin(), valid_log_format_keys.end(), [&item](const std::string &s) { return s == item.first.as(); })) { - throw YAML::ParserException(node.Mark(), "format: unsupported key '" + item.first.as() + "'"); + throw YAML::ParserException(item.first.Mark(), "format: unsupported key '" + item.first.as() + "'"); } } diff --git a/proxy/logging/unit-tests/test_LogUtils.h b/proxy/logging/unit-tests/test_LogUtils.h index 3c9ed225932..c5b4da9cbeb 100644 --- a/proxy/logging/unit-tests/test_LogUtils.h +++ b/proxy/logging/unit-tests/test_LogUtils.h @@ -23,6 +23,8 @@ #pragma once +#include + struct MIMEField { const char *tag, *value; diff --git a/proxy/logging/unit-tests/test_LogUtils2.cc b/proxy/logging/unit-tests/test_LogUtils2.cc index 31376b3f4d0..d50a89bfc6b 100644 --- a/proxy/logging/unit-tests/test_LogUtils2.cc +++ b/proxy/logging/unit-tests/test_LogUtils2.cc @@ -130,3 +130,26 @@ void RecSignalManager(int, char const *, std::size_t) { } + +TEST_CASE("get_unrolled_filename parses possible log files as expected", "[get_unrolled_filename]") +{ + // Rolled log inputs. + constexpr ts::TextView with_underscore = "squid.log_some.hostname.com.20191029.18h15m02s-20191029.18h30m02s.old"; + REQUIRE(get_unrolled_filename(with_underscore) == "squid.log"); + + constexpr ts::TextView without_underscore = "diags.log.20191114.21h43m16s-20191114.21h43m17s.old"; + REQUIRE(get_unrolled_filename(without_underscore) == "diags.log"); + + constexpr ts::TextView dot_file = ".log.20191114.21h43m16s-20191114.21h43m17s.old"; + // Maybe strange, but why not? + REQUIRE(get_unrolled_filename(dot_file) == ".log"); + + // Non-rolled log inputs. + REQUIRE(get_unrolled_filename("") == ""); + + constexpr ts::TextView not_a_log = "logging.yaml"; + REQUIRE(get_unrolled_filename(not_a_log) == not_a_log); + + constexpr ts::TextView no_dot = "logging_yaml"; + REQUIRE(get_unrolled_filename(no_dot) == no_dot); +} diff --git a/proxy/logging/unit-tests/test_RolledLogDeleter.cc b/proxy/logging/unit-tests/test_RolledLogDeleter.cc new file mode 100644 index 00000000000..884030d0266 --- /dev/null +++ b/proxy/logging/unit-tests/test_RolledLogDeleter.cc @@ -0,0 +1,319 @@ +/** @file + + Catch-based tests for RolledLogDeleter. + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#include + +#include "tscore/ts_file.h" + +#define CATCH_CONFIG_MAIN +#include "catch.hpp" + +namespace fs = ts::file; + +const fs::path log_dir("/home/y/logs/trafficserver"); + +void +verify_there_are_no_candidates(RolledLogDeleter &deleter) +{ + CHECK_FALSE(deleter.has_candidates()); + CHECK(deleter.get_candidate_count() == 0); +} + +void +verify_rolled_log_behavior(RolledLogDeleter &deleter, fs::path rolled_log1, fs::path rolled_log2, fs::path rolled_log3) +{ + SECTION("Verify we can add a single rolled files") + { + constexpr int64_t file_size = 100; + constexpr time_t last_modified = 30; + + REQUIRE(deleter.consider_for_candidacy(rolled_log1.string(), file_size, last_modified)); + + CHECK(deleter.has_candidates()); + CHECK(deleter.get_candidate_count() == 1); + + const auto next_candidate = deleter.take_next_candidate_to_delete(); + CHECK(next_candidate->rolled_log_path == rolled_log1.string()); + + // Everything has been taken. + verify_there_are_no_candidates(deleter); + } + + SECTION("Verify we can add two rolled log files") + { + constexpr int64_t file_size = 100; + constexpr time_t oldest_timestamp = 30; + constexpr time_t youngest_timestamp = 60; + + // Intentionally insert them out of order (that is the first one to delete + // is the second added). + REQUIRE(deleter.consider_for_candidacy(rolled_log2.string(), file_size, youngest_timestamp)); + REQUIRE(deleter.consider_for_candidacy(rolled_log1.string(), file_size, oldest_timestamp)); + + CHECK(deleter.has_candidates()); + CHECK(deleter.get_candidate_count() == 2); + + // The first candidate should be the oldest modified one. + auto next_candidate = deleter.take_next_candidate_to_delete(); + CHECK(next_candidate->rolled_log_path == rolled_log1.string()); + + CHECK(deleter.has_candidates()); + CHECK(deleter.get_candidate_count() == 1); + + // The second candidate should be the remaining one. + next_candidate = deleter.take_next_candidate_to_delete(); + CHECK(next_candidate->rolled_log_path == rolled_log2.string()); + + // Everything has been taken. + verify_there_are_no_candidates(deleter); + } + + SECTION("Verify we can add three rolled log files") + { + constexpr int64_t file_size = 100; + + constexpr time_t oldest_timestamp = 30; + constexpr time_t youngest_timestamp = 60; + constexpr time_t middle_timestamp = 45; + + // Intentionally insert them out of order. + REQUIRE(deleter.consider_for_candidacy(rolled_log2.string(), file_size, youngest_timestamp)); + REQUIRE(deleter.consider_for_candidacy(rolled_log1.string(), file_size, oldest_timestamp)); + REQUIRE(deleter.consider_for_candidacy(rolled_log3.string(), file_size, middle_timestamp)); + + CHECK(deleter.has_candidates()); + CHECK(deleter.get_candidate_count() == 3); + + // The first candidate should be the oldest modified one. + auto next_candidate = deleter.take_next_candidate_to_delete(); + CHECK(next_candidate->rolled_log_path == rolled_log1.string()); + + CHECK(deleter.has_candidates()); + CHECK(deleter.get_candidate_count() == 2); + + // The second candidate should be the second oldest. + next_candidate = deleter.take_next_candidate_to_delete(); + CHECK(next_candidate->rolled_log_path == rolled_log3.string()); + + CHECK(deleter.has_candidates()); + CHECK(deleter.get_candidate_count() == 1); + + // The third candidate should be the remaining one. + next_candidate = deleter.take_next_candidate_to_delete(); + CHECK(next_candidate->rolled_log_path == rolled_log2.string()); + + // Everything has been taken. + verify_there_are_no_candidates(deleter); + } +} + +TEST_CASE("Rotated diags logs can be added and removed", "[RolledLogDeleter]") +{ + RolledLogDeleter deleter; + constexpr auto min_count = 0; + deleter.register_log_type_for_deletion("diags.log", min_count); + + const fs::path rolled_log1 = log_dir / "diags.log.20191117.16h43m15s-20191118.16h43m15s.old"; + const fs::path rolled_log2 = log_dir / "diags.log.20191118.16h43m15s-20191122.04h07m09s.old"; + const fs::path rolled_log3 = log_dir / "diags.log.20191122.04h07m09s-20191124.00h12m47s.old"; + + verify_there_are_no_candidates(deleter); + verify_rolled_log_behavior(deleter, rolled_log1, rolled_log2, rolled_log3); +} + +TEST_CASE("Rotated squid logs can be added and removed", "[RolledLogDeleter]") +{ + RolledLogDeleter deleter; + constexpr auto min_count = 0; + deleter.register_log_type_for_deletion("squid.log", min_count); + const fs::path rolled_log1 = log_dir / "squid.log_some.hostname.com.20191125.19h00m04s-20191125.19h15m04s.old"; + const fs::path rolled_log2 = log_dir / "squid.log_some.hostname.com.20191125.19h15m04s-20191125.19h30m04s.old"; + const fs::path rolled_log3 = log_dir / "squid.log_some.hostname.com.20191125.19h30m04s-20191125.19h45m04s.old"; + + verify_there_are_no_candidates(deleter); + verify_rolled_log_behavior(deleter, rolled_log1, rolled_log2, rolled_log3); +} + +TEST_CASE("clear removes all candidates", "[RolledLogDeleter]") +{ + RolledLogDeleter deleter; + constexpr auto min_count = 0; + deleter.register_log_type_for_deletion("squid.log", min_count); + deleter.register_log_type_for_deletion("diags.log", min_count); + + constexpr auto size = 10; + constexpr time_t time_stamp = 20; + + // Add some candidates. + REQUIRE(deleter.consider_for_candidacy("squid.log_arbitrary-text-1", size, time_stamp)); + REQUIRE(deleter.consider_for_candidacy("squid.log_arbitrary-text-2", size, time_stamp)); + REQUIRE(deleter.consider_for_candidacy("squid.log_arbitrary-text-3", size, time_stamp)); + + REQUIRE(deleter.consider_for_candidacy("diags.log.arbitrary-text-1", size, time_stamp)); + REQUIRE(deleter.consider_for_candidacy("diags.log.arbitrary-text-2", size, time_stamp)); + REQUIRE(deleter.consider_for_candidacy("diags.log.arbitrary-text-3", size, time_stamp)); + + REQUIRE(deleter.has_candidates()); + REQUIRE(deleter.get_candidate_count() == 6); + + deleter.clear_candidates(); + verify_there_are_no_candidates(deleter); +} + +TEST_CASE("verify priority enforcement", "[RolledLogDeleter]") +{ + RolledLogDeleter deleter; + + constexpr auto low_min_count = 1; + constexpr auto medium_min_count = 3; + constexpr auto highest_min_count = 0; + + constexpr int64_t a_size = 10; + constexpr time_t a_time = 30; + + deleter.register_log_type_for_deletion("squid.log", low_min_count); + deleter.register_log_type_for_deletion("traffic.out", medium_min_count); + deleter.register_log_type_for_deletion("diags.log", highest_min_count); + + /* The previous tests verify selection within a log_type which is done based + * upon last modified time stamp. These tests focus on selection of + * candidates across log types, which is based upon number of candidates and + * the desired min_count. */ + SECTION("Verify selection of a candidate when there is only one.") + { + const fs::path rolled_squid = log_dir / "squid.log_some.hostname.com.20191125.19h00m04s-20191125.19h15m04s.old"; + REQUIRE(deleter.consider_for_candidacy(rolled_squid.view(), a_size, a_time)); + const auto next_candidate = deleter.take_next_candidate_to_delete(); + CHECK(next_candidate->rolled_log_path == rolled_squid.string()); + verify_there_are_no_candidates(deleter); + } + + SECTION("Verify selection of candidates across three types.") + { + const fs::path rolled_squid = log_dir / "squid.log_some.hostname.com.20191125.19h00m04s-20191125.19h15m04s.old"; + const fs::path rolled_traffic = log_dir / "traffic.out.20191118.16h43m11s-20191122.01h30m30s.old"; + const fs::path rolled_diags = log_dir / "diags.log.20191117.16h43m15s-20191118.16h43m15s.old"; + + REQUIRE(deleter.consider_for_candidacy(rolled_squid.view(), a_size, a_time)); + REQUIRE(deleter.consider_for_candidacy(rolled_traffic.view(), a_size, a_time)); + REQUIRE(deleter.consider_for_candidacy(rolled_diags.view(), a_size, a_time)); + + // Since the time stamps of both are the same, selection should be made + // based upon min_count. + auto next_candidate = deleter.take_next_candidate_to_delete(); + CHECK(next_candidate->rolled_log_path == rolled_squid.string()); + + next_candidate = deleter.take_next_candidate_to_delete(); + CHECK(next_candidate->rolled_log_path == rolled_traffic.string()); + + next_candidate = deleter.take_next_candidate_to_delete(); + CHECK(next_candidate->rolled_log_path == rolled_diags.string()); + + verify_there_are_no_candidates(deleter); + } + + SECTION("Verify that number of candidates is taken into account.") + { + const fs::path rolled_squid = log_dir / "squid.log_some.hostname.com.20191125.19h00m04s-20191125.19h15m04s.old"; + const fs::path rolled_traffic1 = log_dir / "traffic.out.20191117.16h43m15s-20191118.16h43m15s.old"; + const fs::path rolled_traffic2 = log_dir / "traffic.out.20191118.16h43m15s-20191122.04h07m09s.old"; + const fs::path rolled_traffic3 = log_dir / "traffic.out.20191122.04h07m09s-20191124.00h12m47s.old"; + const fs::path rolled_traffic4 = log_dir / "traffic.out.20191124.00h12m44s-20191125.00h12m44s.old"; + + constexpr time_t old = 60; + constexpr time_t older = 30; + constexpr time_t oldest = 10; + constexpr time_t oldestest = 5; + + REQUIRE(deleter.consider_for_candidacy(rolled_squid.view(), a_size, a_time)); + REQUIRE(deleter.consider_for_candidacy(rolled_traffic1.view(), a_size, old)); + REQUIRE(deleter.consider_for_candidacy(rolled_traffic2.view(), a_size, older)); + REQUIRE(deleter.consider_for_candidacy(rolled_traffic3.view(), a_size, oldest)); + REQUIRE(deleter.consider_for_candidacy(rolled_traffic4.view(), a_size, oldestest)); + + // The user has requested a higher number of traffic.out files, but since + // there are so many of them, the oldest of them should be selected next. + auto next_candidate = deleter.take_next_candidate_to_delete(); + CHECK(next_candidate->rolled_log_path == rolled_traffic4.string()); + + // Next, squid.log should be chosen. + next_candidate = deleter.take_next_candidate_to_delete(); + CHECK(next_candidate->rolled_log_path == rolled_squid.string()); + + // Now, there's only traffic.out files. + next_candidate = deleter.take_next_candidate_to_delete(); + CHECK(next_candidate->rolled_log_path == rolled_traffic3.string()); + next_candidate = deleter.take_next_candidate_to_delete(); + CHECK(next_candidate->rolled_log_path == rolled_traffic2.string()); + next_candidate = deleter.take_next_candidate_to_delete(); + CHECK(next_candidate->rolled_log_path == rolled_traffic1.string()); + + verify_there_are_no_candidates(deleter); + } + + SECTION("A mincount of 0 should shield from deletion as much as possible") + { + const fs::path rolled_traffic = log_dir / "traffic.out.20191117.16h43m15s-20191118.16h43m15s.old"; + const fs::path rolled_diags1 = log_dir / "diags.log.20191117.16h43m15s-20191118.16h43m15s.old"; + const fs::path rolled_diags2 = log_dir / "diags.log.20191118.16h43m15s-20191122.04h07m09s.old"; + const fs::path rolled_diags3 = log_dir / "diags.log.20191122.04h07m09s-20191124.00h12m47s.old"; + const fs::path rolled_diags4 = log_dir / "diags.log.20191124.00h12m44s-20191125.00h12m44s.old"; + + constexpr time_t old = 60; + constexpr time_t older = 30; + constexpr time_t oldest = 10; + constexpr time_t oldestest = 5; + + REQUIRE(deleter.consider_for_candidacy(rolled_traffic.view(), a_size, a_time)); + REQUIRE(deleter.consider_for_candidacy(rolled_diags1.view(), a_size, old)); + REQUIRE(deleter.consider_for_candidacy(rolled_diags2.view(), a_size, older)); + REQUIRE(deleter.consider_for_candidacy(rolled_diags3.view(), a_size, oldest)); + REQUIRE(deleter.consider_for_candidacy(rolled_diags4.view(), a_size, oldestest)); + + // Even with so many diags.log files, the traffic.out one should be + // selected first because the min_count of diags.log is 0. + auto next_candidate = deleter.take_next_candidate_to_delete(); + CHECK(next_candidate->rolled_log_path == rolled_traffic.string()); + + // Now there's only diags.log files. + next_candidate = deleter.take_next_candidate_to_delete(); + CHECK(next_candidate->rolled_log_path == rolled_diags4.string()); + next_candidate = deleter.take_next_candidate_to_delete(); + CHECK(next_candidate->rolled_log_path == rolled_diags3.string()); + next_candidate = deleter.take_next_candidate_to_delete(); + CHECK(next_candidate->rolled_log_path == rolled_diags2.string()); + next_candidate = deleter.take_next_candidate_to_delete(); + CHECK(next_candidate->rolled_log_path == rolled_diags1.string()); + + verify_there_are_no_candidates(deleter); + } +} + +// +// Stub +// +void +RecSignalManager(int, const char *, unsigned long) +{ + ink_release_assert(false); +} diff --git a/proxy/private/Makefile.inc b/proxy/private/Makefile.inc new file mode 100644 index 00000000000..1a4a127c56a --- /dev/null +++ b/proxy/private/Makefile.inc @@ -0,0 +1,22 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Header files in this directory should only be included by files in the +# parent directory. + +PRIVATE_SOURCES_ = \ + private/SSLProxySession.cc \ + private/SSLProxySession.h diff --git a/proxy/private/SSLProxySession.cc b/proxy/private/SSLProxySession.cc new file mode 100644 index 00000000000..e2b10857b1d --- /dev/null +++ b/proxy/private/SSLProxySession.cc @@ -0,0 +1,40 @@ +/** @file + + Implementation file for SSLProxySession class. + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#include + +#include "SSLProxySession.h" +#include "P_Net.h" + +void +SSLProxySession::init(SSLNetVConnection const &new_vc) +{ + char const *name = new_vc.get_server_name(); + int length = std::strlen(name) + 1; + if (length > 1) { + char *n = new char[length]; + std::memcpy(n, name, length); + _client_sni_server_name.reset(n); + } + _client_provided_cert = new_vc.peer_provided_cert(); +} diff --git a/proxy/private/SSLProxySession.h b/proxy/private/SSLProxySession.h new file mode 100644 index 00000000000..8974b6c8e3d --- /dev/null +++ b/proxy/private/SSLProxySession.h @@ -0,0 +1,53 @@ +/** @file + + Header file for SSLProxySession class. + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#pragma once + +#include +#include + +class SSLNetVConnection; + +class SSLProxySession +{ +public: + // Returns null pointer if no SNI server name, otherwise pointer to null-terminated string. + // + char const * + client_sni_server_name() const + { + return _client_sni_server_name.get(); + } + + bool + client_provided_certificate() const + { + return _client_provided_cert; + } + + void init(SSLNetVConnection const &new_vc); + +private: + std::unique_ptr _client_sni_server_name; + bool _client_provided_cert = false; +}; diff --git a/proxy/shared/DiagsConfig.cc b/proxy/shared/DiagsConfig.cc index b1ce1fe33cb..9c5c8be3a97 100644 --- a/proxy/shared/DiagsConfig.cc +++ b/proxy/shared/DiagsConfig.cc @@ -25,6 +25,7 @@ #include "tscore/ink_memory.h" #include "tscore/ink_file.h" #include "tscore/I_Layout.h" +#include "tscore/Filenames.h" #include "DiagsConfig.h" #include "records/P_RecCore.h" @@ -244,7 +245,8 @@ DiagsConfig::config_diags_norecords() #endif } -DiagsConfig::DiagsConfig(const char *prefix_string, const char *filename, const char *tags, const char *actions, bool use_records) +DiagsConfig::DiagsConfig(std::string_view prefix_string, const char *filename, const char *tags, const char *actions, + bool use_records) : callbacks_established(false), diags_log(nullptr), diags(nullptr) { char diags_logpath[PATH_NAME_MAX]; @@ -337,7 +339,7 @@ DiagsConfig::register_diags_callbacks() for (i = 0; config_record_names[i] != nullptr; i++) { status = (REC_RegisterConfigUpdateFunc(config_record_names[i], diags_config_callback, o) == REC_ERR_OKAY); if (!status) { - Warning("couldn't register variable '%s', is records.config up to date?", config_record_names[i]); + Warning("couldn't register variable '%s', is %s up to date?", config_record_names[i], ts::filename::RECORDS); } total_status = total_status && status; } diff --git a/proxy/shared/DiagsConfig.h b/proxy/shared/DiagsConfig.h index 76877c97e53..bccc071fc52 100644 --- a/proxy/shared/DiagsConfig.h +++ b/proxy/shared/DiagsConfig.h @@ -32,7 +32,7 @@ struct DiagsConfig { void parse_output_string(char *s, DiagsModeOutput *o); void register_diags_callbacks(); - DiagsConfig(const char *prefix_string, const char *filename, const char *tags, const char *actions, bool use_records = true); + DiagsConfig(std::string_view prefix_string, const char *filename, const char *tags, const char *actions, bool use_records = true); ~DiagsConfig(); private: diff --git a/rc/trafficserver.conf.in b/rc/trafficserver.conf.in index 487e73fba55..6d2aef2ab49 100644 --- a/rc/trafficserver.conf.in +++ b/rc/trafficserver.conf.in @@ -31,8 +31,7 @@ stop on runlevel [06] respawn pre-start script - if [ ! -d @exp_runtimedir@ ] - then + if [ ! -d @exp_runtimedir@ ]; then mkdir -p @exp_runtimedir@ chown @pkgsysuser@:@pkgsysgroup@ @exp_runtimedir@ fi diff --git a/rc/trafficserver.in b/rc/trafficserver.in index 63539e85c75..b0512ada4ee 100644 --- a/rc/trafficserver.in +++ b/rc/trafficserver.in @@ -177,8 +177,7 @@ test -f /lib/lsb/init-functions && . /lib/lsb/init-functions # with native OS rc.subr(8) features. test -f /etc/rc.subr && . /etc/rc.subr -if [ ! -d $TS_BASE@exp_runtimedir@ ] -then +if [ ! -d $TS_BASE@exp_runtimedir@ ]; then mkdir -p $TS_BASE@exp_runtimedir@ chown @pkgsysuser@:@pkgsysgroup@ $TS_BASE@exp_runtimedir@ fi @@ -201,8 +200,7 @@ forkdaemon() while (( $i < $PIDFILE_CHECK_RETRIES )) do # check for regular file and size greater than 0 - if [[ -f $TM_PIDFILE ]] && [[ -s $TM_PIDFILE ]] - then + if [[ -f $TM_PIDFILE ]] && [[ -s $TM_PIDFILE ]]; then success return 0 fi @@ -331,7 +329,7 @@ case "$1" in echo "Starting ${TS_PACKAGE_NAME}" name="$TM_NAME" command="/usr/sbin/daemon" - command_args="$TM_DAEMON $TM_DAEMON_ARGS" + command_args="-o $STDOUTLOG $TM_DAEMON $TM_DAEMON_ARGS" pidfile="$TM_PIDFILE" run_rc_command "$1" else diff --git a/rc/trafficserver.service.in b/rc/trafficserver.service.in index 4a993df5185..7aab8488dbd 100644 --- a/rc/trafficserver.service.in +++ b/rc/trafficserver.service.in @@ -27,8 +27,8 @@ Restart=on-failure RestartSec=5s LimitNOFILE=1000000 ExecStopPost=/bin/sh -c ' \ - export TM_PIDFILE=$(@exp_bindir@/traffic_layout} 2>/dev/null | grep RUNTIMEDIR | cut -d: -f2)/manager.lock ; \ - /bin/rm -f $TM_PIDFILE ; \ + export TM_PIDFILE=$(@exp_bindir@/traffic_layout 2>/dev/null | grep RUNTIMEDIR | cut -d: -f2)/manager.lock ; \ + /bin/rm $TM_PIDFILE ; \ if [[ $? -ne 0 ]]; then echo "ERROR: Unable to delete PID"; exit 1; fi' TimeoutStopSec=5s ExecReload=@exp_bindir@/traffic_ctl config reload diff --git a/src/shared/overridable_txn_vars.cc b/src/shared/overridable_txn_vars.cc new file mode 100644 index 00000000000..3eb2a8dd976 --- /dev/null +++ b/src/shared/overridable_txn_vars.cc @@ -0,0 +1,163 @@ +/** @file + + Map of transaction overridable configuration variables and names. + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#include "shared/overridable_txn_vars.h" + +const std::unordered_map> + ts::Overridable_Txn_Vars( + {{"proxy.config.srv_enabled", {TS_CONFIG_SRV_ENABLED, TS_RECORDDATATYPE_INT}}, + {"proxy.config.http.cache.http", {TS_CONFIG_HTTP_CACHE_HTTP, TS_RECORDDATATYPE_INT}}, + {"proxy.config.ssl.hsts_max_age", {TS_CONFIG_SSL_HSTS_MAX_AGE, TS_RECORDDATATYPE_INT}}, + {"proxy.config.http.normalize_ae", {TS_CONFIG_HTTP_NORMALIZE_AE, TS_RECORDDATATYPE_INT}}, + {"proxy.config.http.chunking.size", {TS_CONFIG_HTTP_CHUNKING_SIZE, TS_RECORDDATATYPE_INT}}, + {"proxy.config.ssl.client.cert.path", {TS_CONFIG_SSL_CERT_FILEPATH, TS_RECORDDATATYPE_STRING}}, + {"proxy.config.http.allow_half_open", {TS_CONFIG_HTTP_ALLOW_HALF_OPEN, TS_RECORDDATATYPE_INT}}, + {"proxy.config.http.chunking_enabled", {TS_CONFIG_HTTP_CHUNKING_ENABLED, TS_RECORDDATATYPE_INT}}, + {"proxy.config.http.cache.generation", {TS_CONFIG_HTTP_CACHE_GENERATION, TS_RECORDDATATYPE_INT}}, + {"proxy.config.http.insert_client_ip", {TS_CONFIG_HTTP_ANONYMIZE_INSERT_CLIENT_IP, TS_RECORDDATATYPE_INT}}, + {"proxy.config.http.insert_forwarded", {TS_CONFIG_HTTP_INSERT_FORWARDED, TS_RECORDDATATYPE_STRING}}, + {"proxy.config.http.cache.range.write", {TS_CONFIG_HTTP_CACHE_RANGE_WRITE, TS_RECORDDATATYPE_INT}}, + {"proxy.config.http.allow_multi_range", {TS_CONFIG_HTTP_ALLOW_MULTI_RANGE, TS_RECORDDATATYPE_INT}}, + {"proxy.config.http.cache.range.lookup", {TS_CONFIG_HTTP_CACHE_RANGE_LOOKUP, TS_RECORDDATATYPE_INT}}, + {"proxy.config.net.sock_packet_tos_out", {TS_CONFIG_NET_SOCK_PACKET_TOS_OUT, TS_RECORDDATATYPE_INT}}, + {"proxy.config.http.slow.log.threshold", {TS_CONFIG_HTTP_SLOW_LOG_THRESHOLD, TS_RECORDDATATYPE_INT}}, + {"proxy.config.http.cache.max_stale_age", {TS_CONFIG_HTTP_CACHE_MAX_STALE_AGE, TS_RECORDDATATYPE_INT}}, + {"proxy.config.http.default_buffer_size", {TS_CONFIG_HTTP_DEFAULT_BUFFER_SIZE, TS_RECORDDATATYPE_INT}}, + {"proxy.config.http.response_server_str", {TS_CONFIG_HTTP_RESPONSE_SERVER_STR, TS_RECORDDATATYPE_STRING}}, + {"proxy.config.http.keep_alive_post_out", {TS_CONFIG_HTTP_KEEP_ALIVE_POST_OUT, TS_RECORDDATATYPE_INT}}, + {"proxy.config.net.sock_option_flag_out", {TS_CONFIG_NET_SOCK_OPTION_FLAG_OUT, TS_RECORDDATATYPE_INT}}, + {"proxy.config.net.sock_packet_mark_out", {TS_CONFIG_NET_SOCK_PACKET_MARK_OUT, TS_RECORDDATATYPE_INT}}, + {"proxy.config.websocket.active_timeout", {TS_CONFIG_WEBSOCKET_ACTIVE_TIMEOUT, TS_RECORDDATATYPE_INT}}, + {"proxy.config.http.flow_control.enabled", {TS_CONFIG_HTTP_FLOW_CONTROL_ENABLED, TS_RECORDDATATYPE_INT}}, + {"proxy.config.http.send_http11_requests", {TS_CONFIG_HTTP_SEND_HTTP11_REQUESTS, TS_RECORDDATATYPE_INT}}, + {"proxy.config.body_factory.template_base", {TS_CONFIG_BODY_FACTORY_TEMPLATE_BASE, TS_RECORDDATATYPE_STRING}}, + {"proxy.config.http.anonymize_remove_from", {TS_CONFIG_HTTP_ANONYMIZE_REMOVE_FROM, TS_RECORDDATATYPE_INT}}, + {"proxy.config.http.keep_alive_enabled_in", {TS_CONFIG_HTTP_KEEP_ALIVE_ENABLED_IN, TS_RECORDDATATYPE_INT}}, + {"proxy.config.http.doc_in_cache_skip_dns", {TS_CONFIG_HTTP_DOC_IN_CACHE_SKIP_DNS, TS_RECORDDATATYPE_INT}}, + {"proxy.config.http.forward_connect_method", {TS_CONFIG_HTTP_FORWARD_CONNECT_METHOD, TS_RECORDDATATYPE_INT}}, + {"proxy.config.http.request_buffer_enabled", {TS_CONFIG_HTTP_REQUEST_BUFFER_ENABLED, TS_RECORDDATATYPE_INT}}, + {"proxy.config.http.down_server.cache_time", {TS_CONFIG_HTTP_DOWN_SERVER_CACHE_TIME, TS_RECORDDATATYPE_INT}}, + {"proxy.config.http.insert_age_in_response", {TS_CONFIG_HTTP_INSERT_AGE_IN_RESPONSE, TS_RECORDDATATYPE_INT}}, + {"proxy.config.url_remap.pristine_host_hdr", {TS_CONFIG_URL_REMAP_PRISTINE_HOST_HDR, TS_RECORDDATATYPE_INT}}, + {"proxy.config.http.insert_request_via_str", {TS_CONFIG_HTTP_INSERT_REQUEST_VIA_STR, TS_RECORDDATATYPE_INT}}, + {"proxy.config.http.flow_control.low_water", {TS_CONFIG_HTTP_FLOW_CONTROL_LOW_WATER_MARK, TS_RECORDDATATYPE_INT}}, + {"proxy.config.http.cache.required_headers", {TS_CONFIG_HTTP_CACHE_REQUIRED_HEADERS, TS_RECORDDATATYPE_INT}}, + {"proxy.config.ssl.hsts_include_subdomains", {TS_CONFIG_SSL_HSTS_INCLUDE_SUBDOMAINS, TS_RECORDDATATYPE_INT}}, + {"proxy.config.http.number_of_redirections", {TS_CONFIG_HTTP_NUMBER_OF_REDIRECTIONS, TS_RECORDDATATYPE_INT}}, + {"proxy.config.http.keep_alive_enabled_out", {TS_CONFIG_HTTP_KEEP_ALIVE_ENABLED_OUT, TS_RECORDDATATYPE_INT}}, + {"proxy.config.http.response_server_enabled", {TS_CONFIG_HTTP_RESPONSE_SERVER_ENABLED, TS_RECORDDATATYPE_INT}}, + {"proxy.config.http.anonymize_remove_cookie", {TS_CONFIG_HTTP_ANONYMIZE_REMOVE_COOKIE, TS_RECORDDATATYPE_INT}}, + {"proxy.config.http.request_header_max_size", {TS_CONFIG_HTTP_REQUEST_HEADER_MAX_SIZE, TS_RECORDDATATYPE_INT}}, + {"proxy.config.http.parent_proxy.retry_time", {TS_CONFIG_HTTP_PARENT_PROXY_RETRY_TIME, TS_RECORDDATATYPE_INT}}, + {"proxy.config.http.insert_response_via_str", {TS_CONFIG_HTTP_INSERT_RESPONSE_VIA_STR, TS_RECORDDATATYPE_INT}}, + {"proxy.config.http.flow_control.high_water", {TS_CONFIG_HTTP_FLOW_CONTROL_HIGH_WATER_MARK, TS_RECORDDATATYPE_INT}}, + {"proxy.config.http.negative_caching_enabled", {TS_CONFIG_HTTP_NEGATIVE_CACHING_ENABLED, TS_RECORDDATATYPE_INT}}, + {"proxy.config.http.cache.when_to_revalidate", {TS_CONFIG_HTTP_CACHE_WHEN_TO_REVALIDATE, TS_RECORDDATATYPE_INT}}, + {"proxy.config.http.response_header_max_size", {TS_CONFIG_HTTP_RESPONSE_HEADER_MAX_SIZE, TS_RECORDDATATYPE_INT}}, + {"proxy.config.http.anonymize_remove_referer", {TS_CONFIG_HTTP_ANONYMIZE_REMOVE_REFERER, TS_RECORDDATATYPE_INT}}, + {"proxy.config.http.global_user_agent_header", {TS_CONFIG_HTTP_GLOBAL_USER_AGENT_HEADER, TS_RECORDDATATYPE_STRING}}, + {"proxy.config.net.sock_recv_buffer_size_out", {TS_CONFIG_NET_SOCK_RECV_BUFFER_SIZE_OUT, TS_RECORDDATATYPE_INT}}, + {"proxy.config.net.sock_send_buffer_size_out", {TS_CONFIG_NET_SOCK_SEND_BUFFER_SIZE_OUT, TS_RECORDDATATYPE_INT}}, + {"proxy.config.http.connect_attempts_timeout", {TS_CONFIG_HTTP_CONNECT_ATTEMPTS_TIMEOUT, TS_RECORDDATATYPE_INT}}, + {"proxy.config.websocket.no_activity_timeout", {TS_CONFIG_WEBSOCKET_NO_ACTIVITY_TIMEOUT, TS_RECORDDATATYPE_INT}}, + {"proxy.config.http.negative_caching_lifetime", {TS_CONFIG_HTTP_NEGATIVE_CACHING_LIFETIME, TS_RECORDDATATYPE_INT}}, + {"proxy.config.http.default_buffer_water_mark", {TS_CONFIG_HTTP_DEFAULT_BUFFER_WATER_MARK, TS_RECORDDATATYPE_INT}}, + {"proxy.config.http.cache.heuristic_lm_factor", {TS_CONFIG_HTTP_CACHE_HEURISTIC_LM_FACTOR, TS_RECORDDATATYPE_FLOAT}}, + {OutboundConnTrack::CONFIG_VAR_MAX, {TS_CONFIG_HTTP_PER_SERVER_CONNECTION_MAX, TS_RECORDDATATYPE_INT}}, + {OutboundConnTrack::CONFIG_VAR_MIN, {TS_CONFIG_HTTP_SERVER_MIN_KEEP_ALIVE_CONNS, TS_RECORDDATATYPE_INT}}, + {"proxy.config.http.anonymize_remove_client_ip", {TS_CONFIG_HTTP_ANONYMIZE_REMOVE_CLIENT_IP, TS_RECORDDATATYPE_INT}}, + {"proxy.config.http.cache.open_read_retry_time", {TS_CONFIG_HTTP_CACHE_OPEN_READ_RETRY_TIME, TS_RECORDDATATYPE_INT}}, + {"proxy.config.http.down_server.abort_threshold", {TS_CONFIG_HTTP_DOWN_SERVER_ABORT_THRESHOLD, TS_RECORDDATATYPE_INT}}, + {OutboundConnTrack::CONFIG_VAR_MATCH, {TS_CONFIG_HTTP_PER_SERVER_CONNECTION_MATCH, TS_RECORDDATATYPE_INT}}, + {"proxy.config.http.parent_proxy.fail_threshold", {TS_CONFIG_HTTP_PARENT_PROXY_FAIL_THRESHOLD, TS_RECORDDATATYPE_INT}}, + {"proxy.config.http.cache.ignore_authentication", {TS_CONFIG_HTTP_CACHE_IGNORE_AUTHENTICATION, TS_RECORDDATATYPE_INT}}, + {"proxy.config.http.anonymize_remove_user_agent", {TS_CONFIG_HTTP_ANONYMIZE_REMOVE_USER_AGENT, TS_RECORDDATATYPE_INT}}, + {"proxy.config.http.connect_attempts_rr_retries", {TS_CONFIG_HTTP_CONNECT_ATTEMPTS_RR_RETRIES, TS_RECORDDATATYPE_INT}}, + {"proxy.config.http.cache.max_open_read_retries", {TS_CONFIG_HTTP_CACHE_MAX_OPEN_READ_RETRIES, TS_RECORDDATATYPE_INT}}, + {"proxy.config.http.auth_server_session_private", {TS_CONFIG_HTTP_AUTH_SERVER_SESSION_PRIVATE, TS_RECORDDATATYPE_INT}}, + {"proxy.config.http.redirect_use_orig_cache_key", {TS_CONFIG_HTTP_REDIRECT_USE_ORIG_CACHE_KEY, TS_RECORDDATATYPE_INT}}, + {"proxy.config.http.cache.ignore_client_no_cache", {TS_CONFIG_HTTP_CACHE_IGNORE_CLIENT_NO_CACHE, TS_RECORDDATATYPE_INT}}, + {"proxy.config.http.cache.ims_on_client_no_cache", {TS_CONFIG_HTTP_CACHE_IMS_ON_CLIENT_NO_CACHE, TS_RECORDDATATYPE_INT}}, + {"proxy.config.http.cache.ignore_server_no_cache", {TS_CONFIG_HTTP_CACHE_IGNORE_SERVER_NO_CACHE, TS_RECORDDATATYPE_INT}}, + {"proxy.config.http.cache.heuristic_min_lifetime", {TS_CONFIG_HTTP_CACHE_HEURISTIC_MIN_LIFETIME, TS_RECORDDATATYPE_INT}}, + {"proxy.config.http.cache.heuristic_max_lifetime", {TS_CONFIG_HTTP_CACHE_HEURISTIC_MAX_LIFETIME, TS_RECORDDATATYPE_INT}}, + {"proxy.config.http.server_session_sharing.match", {TS_CONFIG_HTTP_SERVER_SESSION_SHARING_MATCH, TS_RECORDDATATYPE_STRING}}, + {"proxy.config.http.cache.ignore_accept_mismatch", {TS_CONFIG_HTTP_CACHE_IGNORE_ACCEPT_MISMATCH, TS_RECORDDATATYPE_INT}}, + {"proxy.config.http.cache.open_write_fail_action", {TS_CONFIG_HTTP_CACHE_OPEN_WRITE_FAIL_ACTION, TS_RECORDDATATYPE_INT}}, + {"proxy.config.http.insert_squid_x_forwarded_for", {TS_CONFIG_HTTP_INSERT_SQUID_X_FORWARDED_FOR, TS_RECORDDATATYPE_INT}}, + {"proxy.config.http.connect_attempts_max_retries", {TS_CONFIG_HTTP_CONNECT_ATTEMPTS_MAX_RETRIES, TS_RECORDDATATYPE_INT}}, + {"proxy.config.http.cache.max_open_write_retries", {TS_CONFIG_HTTP_CACHE_MAX_OPEN_WRITE_RETRIES, TS_RECORDDATATYPE_INT}}, + {"proxy.config.http.forward.proxy_auth_to_parent", {TS_CONFIG_HTTP_FORWARD_PROXY_AUTH_TO_PARENT, TS_RECORDDATATYPE_INT}}, + {"proxy.config.http.parent_proxy.mark_down_hostdb", {TS_CONFIG_PARENT_FAILURES_UPDATE_HOSTDB, TS_RECORDDATATYPE_INT}}, + {"proxy.config.http.negative_revalidating_enabled", {TS_CONFIG_HTTP_NEGATIVE_REVALIDATING_ENABLED, TS_RECORDDATATYPE_INT}}, + {"proxy.config.http.cache.guaranteed_min_lifetime", {TS_CONFIG_HTTP_CACHE_GUARANTEED_MIN_LIFETIME, TS_RECORDDATATYPE_INT}}, + {"proxy.config.http.cache.guaranteed_max_lifetime", {TS_CONFIG_HTTP_CACHE_GUARANTEED_MAX_LIFETIME, TS_RECORDDATATYPE_INT}}, + {"proxy.config.http.transaction_active_timeout_in", {TS_CONFIG_HTTP_TRANSACTION_ACTIVE_TIMEOUT_IN, TS_RECORDDATATYPE_INT}}, + {"proxy.config.http.post_connect_attempts_timeout", {TS_CONFIG_HTTP_POST_CONNECT_ATTEMPTS_TIMEOUT, TS_RECORDDATATYPE_INT}}, + {"proxy.config.http.cache.ignore_client_cc_max_age", {TS_CONFIG_HTTP_CACHE_IGNORE_CLIENT_CC_MAX_AGE, TS_RECORDDATATYPE_INT}}, + {"proxy.config.http.negative_revalidating_lifetime", {TS_CONFIG_HTTP_NEGATIVE_REVALIDATING_LIFETIME, TS_RECORDDATATYPE_INT}}, + {"proxy.config.http.transaction_active_timeout_out", {TS_CONFIG_HTTP_TRANSACTION_ACTIVE_TIMEOUT_OUT, TS_RECORDDATATYPE_INT}}, + {"proxy.config.http.background_fill_active_timeout", {TS_CONFIG_HTTP_BACKGROUND_FILL_ACTIVE_TIMEOUT, TS_RECORDDATATYPE_INT}}, + {"proxy.config.http.attach_server_session_to_client", {TS_CONFIG_HTTP_ATTACH_SERVER_SESSION_TO_CLIENT, TS_RECORDDATATYPE_INT}}, + {"proxy.config.http.cache.cache_responses_to_cookies", + {TS_CONFIG_HTTP_CACHE_CACHE_RESPONSES_TO_COOKIES, TS_RECORDDATATYPE_INT}}, + {"proxy.config.http.keep_alive_no_activity_timeout_in", + {TS_CONFIG_HTTP_KEEP_ALIVE_NO_ACTIVITY_TIMEOUT_IN, TS_RECORDDATATYPE_INT}}, + {"proxy.config.http.post.check.content_length.enabled", + {TS_CONFIG_HTTP_POST_CHECK_CONTENT_LENGTH_ENABLED, TS_RECORDDATATYPE_INT}}, + {"proxy.config.http.cache.cache_urls_that_look_dynamic", + {TS_CONFIG_HTTP_CACHE_CACHE_URLS_THAT_LOOK_DYNAMIC, TS_RECORDDATATYPE_INT}}, + {"proxy.config.http.transaction_no_activity_timeout_in", + {TS_CONFIG_HTTP_TRANSACTION_NO_ACTIVITY_TIMEOUT_IN, TS_RECORDDATATYPE_INT}}, + {"proxy.config.http.keep_alive_no_activity_timeout_out", + {TS_CONFIG_HTTP_KEEP_ALIVE_NO_ACTIVITY_TIMEOUT_OUT, TS_RECORDDATATYPE_INT}}, + {"proxy.config.http.uncacheable_requests_bypass_parent", + {TS_CONFIG_HTTP_UNCACHEABLE_REQUESTS_BYPASS_PARENT, TS_RECORDDATATYPE_INT}}, + {"proxy.config.http.transaction_no_activity_timeout_out", + {TS_CONFIG_HTTP_TRANSACTION_NO_ACTIVITY_TIMEOUT_OUT, TS_RECORDDATATYPE_INT}}, + {"proxy.config.http.background_fill_completed_threshold", + {TS_CONFIG_HTTP_BACKGROUND_FILL_COMPLETED_THRESHOLD, TS_RECORDDATATYPE_FLOAT}}, + {"proxy.config.http.parent_proxy.total_connect_attempts", + {TS_CONFIG_HTTP_PARENT_PROXY_TOTAL_CONNECT_ATTEMPTS, TS_RECORDDATATYPE_INT}}, + {"proxy.config.http.cache.ignore_accept_charset_mismatch", + {TS_CONFIG_HTTP_CACHE_IGNORE_ACCEPT_CHARSET_MISMATCH, TS_RECORDDATATYPE_INT}}, + {"proxy.config.http.cache.ignore_accept_language_mismatch", + {TS_CONFIG_HTTP_CACHE_IGNORE_ACCEPT_LANGUAGE_MISMATCH, TS_RECORDDATATYPE_INT}}, + {"proxy.config.http.cache.ignore_accept_encoding_mismatch", + {TS_CONFIG_HTTP_CACHE_IGNORE_ACCEPT_ENCODING_MISMATCH, TS_RECORDDATATYPE_INT}}, + {"proxy.config.http.parent_proxy.connect_attempts_timeout", + {TS_CONFIG_HTTP_PARENT_CONNECT_ATTEMPT_TIMEOUT, TS_RECORDDATATYPE_INT}}, + {"proxy.config.http.connect_attempts_max_retries_dead_server", + {TS_CONFIG_HTTP_CONNECT_ATTEMPTS_MAX_RETRIES_DEAD_SERVER, TS_RECORDDATATYPE_INT}}, + {"proxy.config.http.parent_proxy.per_parent_connect_attempts", + {TS_CONFIG_HTTP_PER_PARENT_CONNECT_ATTEMPTS, TS_RECORDDATATYPE_INT}}, + {"proxy.config.ssl.client.verify.server", {TS_CONFIG_SSL_CLIENT_VERIFY_SERVER, TS_RECORDDATATYPE_INT}}, + {"proxy.config.ssl.client.verify.server.policy", {TS_CONFIG_SSL_CLIENT_VERIFY_SERVER_POLICY, TS_RECORDDATATYPE_STRING}}, + {"proxy.config.ssl.client.verify.server.properties", + {TS_CONFIG_SSL_CLIENT_VERIFY_SERVER_PROPERTIES, TS_RECORDDATATYPE_STRING}}, + {"proxy.config.ssl.client.sni_policy", {TS_CONFIG_SSL_CLIENT_SNI_POLICY, TS_RECORDDATATYPE_STRING}}, + {"proxy.config.ssl.client.cert.filename", {TS_CONFIG_SSL_CLIENT_CERT_FILENAME, TS_RECORDDATATYPE_STRING}}, + {"proxy.config.ssl.client.cert.path", {TS_CONFIG_SSL_CERT_FILEPATH, TS_RECORDDATATYPE_STRING}}, + {"proxy.config.ssl.client.private_key.filename", {TS_CONFIG_SSL_CLIENT_PRIVATE_KEY_FILENAME, TS_RECORDDATATYPE_STRING}}, + {"proxy.config.ssl.client.CA.cert.filename", {TS_CONFIG_SSL_CLIENT_CA_CERT_FILENAME, TS_RECORDDATATYPE_STRING}}, + {"proxy.config.hostdb.ip_resolve", {TS_CONFIG_HTTP_HOST_RESOLUTION_PREFERENCE, TS_RECORDDATATYPE_STRING}}}); diff --git a/src/traffic_cache_tool/CacheTool.cc b/src/traffic_cache_tool/CacheTool.cc index c5b56d7a3d5..85ceced2aeb 100644 --- a/src/traffic_cache_tool/CacheTool.cc +++ b/src/traffic_cache_tool/CacheTool.cc @@ -255,6 +255,8 @@ class VolumeAllocator : _config(config), _size(size), _deficit(deficit), _shares(shares) { } + V(const V &v) = default; + V & operator=(V const &that) { @@ -520,8 +522,8 @@ Cache::loadSpanConfig(ts::file::path const &path) if (line.empty() || '#' == *line) { continue; } - ts::TextView path = line.take_prefix_if(&isspace); - if (path) { + ts::TextView localpath = line.take_prefix_if(&isspace); + if (localpath) { // After this the line is [size] [id=string] [volume=#] while (line) { ts::TextView value(line.take_prefix_if(&isspace)); @@ -539,7 +541,7 @@ Cache::loadSpanConfig(ts::file::path const &path) } } } - zret = this->loadSpan(ts::file::path(path)); + zret = this->loadSpan(ts::file::path(localpath)); } } } else { @@ -552,7 +554,7 @@ Errata Cache::loadURLs(ts::file::path const &path) { static const ts::TextView TAG_VOL("url"); - ts::URLparser parser; + ts::URLparser loadURLparser; Errata zret; std::error_code ec; @@ -568,7 +570,7 @@ Cache::loadURLs(ts::file::path const &path) std::string url; url.assign(blob.data(), blob.size()); int port_ptr = -1, port_len = -1; - int port = parser.getPort(url, port_ptr, port_len); + int port = loadURLparser.getPort(url, port_ptr, port_len); if (port_ptr >= 0 && port_len > 0) { url.erase(port_ptr, port_len + 1); // get rid of :PORT } diff --git a/src/traffic_ctl/Makefile.inc b/src/traffic_ctl/Makefile.inc index 446e56539f9..3808c19f0c7 100644 --- a/src/traffic_ctl/Makefile.inc +++ b/src/traffic_ctl/Makefile.inc @@ -23,6 +23,8 @@ traffic_ctl_traffic_ctl_CPPFLAGS = \ $(AM_CPPFLAGS) \ $(iocore_include_dirs) \ -I$(abs_top_srcdir)/include \ + -I$(abs_top_srcdir)/proxy/http \ + -I$(abs_top_srcdir)/proxy/hdrs \ -I$(abs_top_srcdir)/lib \ -I$(abs_top_srcdir)/mgmt \ -I$(abs_top_srcdir)/mgmt/api/include \ @@ -37,6 +39,7 @@ traffic_ctl_traffic_ctl_SOURCES = \ traffic_ctl/server.cc \ traffic_ctl/storage.cc \ traffic_ctl/host.cc \ + shared/overridable_txn_vars.cc \ traffic_ctl/traffic_ctl.cc traffic_ctl_traffic_ctl_LDADD = \ diff --git a/src/traffic_ctl/config.cc b/src/traffic_ctl/config.cc index 61af1998bc5..d60cb3a07d0 100644 --- a/src/traffic_ctl/config.cc +++ b/src/traffic_ctl/config.cc @@ -25,6 +25,11 @@ #include #include "records/I_RecDefs.h" #include "records/P_RecUtils.h" +#include "ts/apidefs.h" +#include "HTTP.h" +#include "HttpConnectionCount.h" +#include "shared/overridable_txn_vars.h" +#include struct RecordDescriptionPolicy { using entry_type = TSConfigRecordDescription *; @@ -168,6 +173,30 @@ rec_labelof(int rec_class) } } +static const char * +rec_datatypeof(TSRecordDataType dt) +{ + switch (dt) { + case TS_RECORDDATATYPE_INT: + return "int"; + case TS_RECORDDATATYPE_NULL: + return "null"; + case TS_RECORDDATATYPE_FLOAT: + return "float"; + case TS_RECORDDATATYPE_STRING: + return "string"; + case TS_RECORDDATATYPE_COUNTER: + return "counter"; + case TS_RECORDDATATYPE_STAT_CONST: + return "constant stat"; + case TS_RECORDDATATYPE_STAT_FX: + return "stat fx"; + case TS_RECORDDATATYPE_MAX: + return "*"; + } + return "?"; +} + static std::string timestr(time_t tm) { @@ -223,24 +252,31 @@ CtrlEngine::config_describe() return; } - std::cout << "Name: " << desc.rec_name << std::endl; - std::cout << "Current Value: " << CtrlMgmtRecordValue(desc.rec_type, desc.rec_value).c_str() << std::endl; - std::cout << "Default Value: " << CtrlMgmtRecordValue(desc.rec_type, desc.rec_default).c_str() << std::endl; - std::cout << "Record Type: " << rec_classof(desc.rec_class) << std::endl; - std::cout << "Data Type: " << rec_typeof(desc.rec_type) << std::endl; - std::cout << "Access Control: " << rec_accessof(desc.rec_access) << std::endl; - std::cout << "Update Type: " << rec_updateof(desc.rec_updatetype) << std::endl; - std::cout << "Update Status: " << desc.rec_update << std::endl; - std::cout << "Source: " << rec_sourceof(desc.rec_source) << std::endl; + auto ov_iter = ts::Overridable_Txn_Vars.find(it); + bool overridable_p = (ov_iter != ts::Overridable_Txn_Vars.end()); + + std::string text; + std::cout << ts::bwprint(text, "{:16s}: {}\n", "Name", desc.rec_name); + std::cout << ts::bwprint(text, "{:16s}: {}\n", "Current Value ", CtrlMgmtRecordValue(desc.rec_type, desc.rec_value).c_str()); + std::cout << ts::bwprint(text, "{:16s}: {}\n", "Default Value ", CtrlMgmtRecordValue(desc.rec_type, desc.rec_default).c_str()); + std::cout << ts::bwprint(text, "{:16s}: {}\n", "Record Type ", rec_classof(desc.rec_class)); + std::cout << ts::bwprint(text, "{:16s}: {}\n", "Data Type ", rec_typeof(desc.rec_type)); + std::cout << ts::bwprint(text, "{:16s}: {}\n", "Access Control ", rec_accessof(desc.rec_access)); + std::cout << ts::bwprint(text, "{:16s}: {}\n", "Update Type ", rec_updateof(desc.rec_updatetype)); + std::cout << ts::bwprint(text, "{:16s}: {}\n", "Update Status ", desc.rec_update); + std::cout << ts::bwprint(text, "{:16s}: {}\n", "Source ", rec_sourceof(desc.rec_source)); + std::cout << ts::bwprint(text, "{:16s}: {} {}\n", "Overridable", overridable_p ? "yes" : "no", + overridable_p ? rec_datatypeof(std::get<1>(ov_iter->second)) : ""); if (strlen(desc.rec_checkexpr)) { - std::cout << "Syntax Check: " << rec_checkof(desc.rec_checktype) << desc.rec_checkexpr << std::endl; + std::cout << ts::bwprint(text, "{:16s}: {}\n", "Syntax Check ", rec_checkof(desc.rec_checktype), desc.rec_checkexpr); } else { - std::cout << "Syntax Check: " << rec_checkof(desc.rec_checktype) << std::endl; + std::cout << ts::bwprint(text, "{:16s}: {}\n", "Syntax Check ", rec_checkof(desc.rec_checktype)); } - std::cout << "Version: " << desc.rec_version << std::endl; - std::cout << "Order: " << desc.rec_order << std::endl; - std::cout << "Raw Stat Block: " << desc.rec_rsb << std::endl; + + std::cout << ts::bwprint(text, "{:16s}: {}\n", "Version ", desc.rec_version); + std::cout << ts::bwprint(text, "{:16s}: {}\n", "Order ", desc.rec_order); + std::cout << ts::bwprint(text, "{:16s}: {}\n", "Raw Stat Block ", desc.rec_rsb); TSConfigRecordDescriptionFree(&desc); } diff --git a/src/traffic_ctl/traffic_ctl.cc b/src/traffic_ctl/traffic_ctl.cc index f8908ab049b..33b363e5c6a 100644 --- a/src/traffic_ctl/traffic_ctl.cc +++ b/src/traffic_ctl/traffic_ctl.cc @@ -275,6 +275,9 @@ main(int argc, const char **argv) argparser_runroot_handler(engine.arguments.get("--run-root").value(), argv[0]); Layout::create(); + + // This is a little bit of a hack, for now it'll suffice. + max_records_entries = 262144; RecProcessInit(RECM_STAND_ALONE, diags); LibRecordsConfigInit(); diff --git a/src/traffic_layout/info.cc b/src/traffic_layout/info.cc index 6b521eaf94d..05495c59598 100644 --- a/src/traffic_layout/info.cc +++ b/src/traffic_layout/info.cc @@ -21,8 +21,10 @@ limitations under the License. */ +#include #include #include "tscore/I_Layout.h" +#include "tscore/Filenames.h" #include "tscore/BufferWriter.h" #include "records/I_RecProcess.h" #include "RecordsConfig.h" @@ -40,8 +42,7 @@ #include #endif -// Produce output about compile time features, useful for checking how things were built, as well -// as for our TSQA test harness. +// Produce output about compile time features, useful for checking how things were built static void print_feature(std::string_view name, int value, bool json, bool last = false) { @@ -88,6 +89,11 @@ produce_features(bool json) #else print_feature("TS_HAS_BROTLI", 0, json); #endif +#ifdef F_GETPIPE_SZ + print_feature("TS_HAS_PIPE_BUFFER_SIZE_CONFIG", 1, json); +#else + print_feature("TS_HAS_PIPE_BUFFER_SIZE_CONFIG", 0, json); +#endif /* F_GETPIPE_SZ */ print_feature("TS_HAS_JEMALLOC", TS_HAS_JEMALLOC, json); print_feature("TS_HAS_TCMALLOC", TS_HAS_TCMALLOC, json); print_feature("TS_HAS_IN6_IS_ADDR_UNSPECIFIED", TS_HAS_IN6_IS_ADDR_UNSPECIFIED, json); @@ -105,6 +111,7 @@ produce_features(bool json) print_feature("TS_USE_HWLOC", TS_USE_HWLOC, json); print_feature("TS_USE_SET_RBIO", TS_USE_SET_RBIO, json); print_feature("TS_USE_TLS13", TS_USE_TLS13, json); + print_feature("TS_USE_QUIC", TS_USE_QUIC, json); print_feature("TS_USE_LINUX_NATIVE_AIO", TS_USE_LINUX_NATIVE_AIO, json); print_feature("TS_HAS_SO_PEERCRED", TS_HAS_SO_PEERCRED, json); print_feature("TS_USE_REMOTE_UNWINDING", TS_USE_REMOTE_UNWINDING, json); @@ -154,14 +161,14 @@ produce_layout(bool json) print_var("PLUGINDIR", RecConfigReadPluginDir(), json); print_var("INCLUDEDIR", Layout::get()->includedir, json); - print_var("records.config", RecConfigReadConfigPath(nullptr, REC_CONFIG_FILE), json); - print_var("remap.config", RecConfigReadConfigPath("proxy.config.url_remap.filename"), json); - print_var("plugin.config", RecConfigReadConfigPath(nullptr, "plugin.config"), json); - print_var("ssl_multicert.config", RecConfigReadConfigPath("proxy.config.ssl.server.multicert.filename"), json); - print_var("storage.config", RecConfigReadConfigPath("proxy.config.cache.storage_filename"), json); - print_var("hosting.config", RecConfigReadConfigPath("proxy.config.cache.hosting_filename"), json); - print_var("volume.config", RecConfigReadConfigPath("proxy.config.cache.volume_filename"), json); - print_var("ip_allow.yaml", RecConfigReadConfigPath("proxy.config.cache.ip_allow.filename"), json, true); + print_var(ts::filename::RECORDS, RecConfigReadConfigPath(nullptr, ts::filename::RECORDS), json); + print_var(ts::filename::REMAP, RecConfigReadConfigPath("proxy.config.url_remap.filename"), json); + print_var(ts::filename::PLUGIN, RecConfigReadConfigPath(nullptr, ts::filename::PLUGIN), json); + print_var(ts::filename::SSL_MULTICERT, RecConfigReadConfigPath("proxy.config.ssl.server.multicert.filename"), json); + print_var(ts::filename::STORAGE, RecConfigReadConfigPath(nullptr, ts::filename::STORAGE), json); + print_var(ts::filename::HOSTING, RecConfigReadConfigPath("proxy.config.cache.hosting_filename"), json); + print_var(ts::filename::VOLUME, RecConfigReadConfigPath("proxy.config.cache.volume_filename"), json); + print_var(ts::filename::IP_ALLOW, RecConfigReadConfigPath("proxy.config.cache.ip_allow.filename"), json, true); if (json) { printf("}\n"); } diff --git a/src/traffic_layout/info.h b/src/traffic_layout/info.h index fc3831c0454..9717d14dcbf 100644 --- a/src/traffic_layout/info.h +++ b/src/traffic_layout/info.h @@ -21,6 +21,8 @@ limitations under the License. */ +#pragma once + // the original traffic_layout void produce_features(bool json); diff --git a/src/traffic_logcat/logcat.cc b/src/traffic_logcat/logcat.cc index debbaf0a433..5ac69aa0563 100644 --- a/src/traffic_logcat/logcat.cc +++ b/src/traffic_logcat/logcat.cc @@ -208,14 +208,14 @@ process_file(int in_fd, int out_fd) } static int -open_output_file(char *output_file) +open_output_file(char *output_file_p) { int file_desc = 0; if (!overwrite_existing_file) { - if (access(output_file, F_OK)) { + if (access(output_file_p, F_OK)) { if (errno != ENOENT) { - fprintf(stderr, "Error accessing output file %s: ", output_file); + fprintf(stderr, "Error accessing output file %s: ", output_file_p); perror(nullptr); file_desc = -1; } @@ -223,16 +223,16 @@ open_output_file(char *output_file) fprintf(stderr, "Error, output file %s already exists.\n" "Select a different filename or use the -w flag\n", - output_file); + output_file_p); file_desc = -1; } } if (file_desc == 0) { - file_desc = open(output_file, O_WRONLY | O_TRUNC | O_CREAT, 0640); + file_desc = open(output_file_p, O_WRONLY | O_TRUNC | O_CREAT, 0640); if (file_desc < 0) { - fprintf(stderr, "Error while opening output file %s: ", output_file); + fprintf(stderr, "Error while opening output file %s: ", output_file_p); perror(nullptr); } } diff --git a/src/traffic_logstats/logstats.cc b/src/traffic_logstats/logstats.cc index 53be58e4b5a..c581e1f223a 100644 --- a/src/traffic_logstats/logstats.cc +++ b/src/traffic_logstats/logstats.cc @@ -336,10 +336,10 @@ struct hash_fnv32 { } }; -using LruStack = std::list; -typedef std::unordered_map OriginStorage; -typedef std::unordered_set OriginSet; -typedef std::unordered_map LruHash; +using LruStack = std::list; +using OriginStorage = std::unordered_map; +using OriginSet = std::unordered_set; +using LruHash = std::unordered_map; // Resize a hash-based container. template @@ -1998,7 +1998,7 @@ format_line(const char *desc, const StatsCounter &stat, const StatsCounter &tota } // Little "helpers" for the vector we use to sort the Origins. -typedef pair OriginPair; +using OriginPair = pair; inline bool operator<(const OriginPair &a, const OriginPair &b) { diff --git a/src/traffic_manager/AddConfigFilesHere.cc b/src/traffic_manager/AddConfigFilesHere.cc index 5f894ccadb2..3fdf572075c 100644 --- a/src/traffic_manager/AddConfigFilesHere.cc +++ b/src/traffic_manager/AddConfigFilesHere.cc @@ -22,12 +22,15 @@ */ #include "tscore/ink_platform.h" +#include "tscore/Filenames.h" #include "MgmtUtils.h" #include "tscore/Diags.h" #include "FileManager.h" extern FileManager *configFiles; +static constexpr bool REQUIRED{true}; +static constexpr bool NOT_REQUIRED{false}; /**************************************************************************** * * AddConfigFilesHere.cc - Structs for config files and @@ -42,14 +45,14 @@ testcall(char *foo, char * /*configName */) } void -registerFile(const char *configName, const char *defaultName) +registerFile(const char *configName, const char *defaultName, bool isRequired) { bool found = false; const char *fname = REC_readString(configName, &found); if (!found) { fname = defaultName; } - configFiles->addFile(fname, configName, false); + configFiles->addFile(fname, configName, false, isRequired); } // @@ -72,20 +75,20 @@ initializeRegistry() ink_assert(!"Configuration Object Registry Initialized More than Once"); } - registerFile("proxy.config.log.config.filename", "logging.yaml"); - registerFile("", "storage.config"); - registerFile("proxy.config.socks.socks_config_file", "socks.config"); - registerFile("records.config", "records.config"); - registerFile("proxy.config.cache.control.filename", "cache.config"); - registerFile("proxy.config.cache.ip_allow.filename", "ip_allow.yaml"); - registerFile("proxy.config.http.parent_proxy.file", "parent.config"); - registerFile("proxy.config.url_remap.filename", "remap.config"); - registerFile("", "volume.config"); - registerFile("proxy.config.cache.hosting_filename", "hosting.config"); - registerFile("", "plugin.config"); - registerFile("proxy.config.dns.splitdns.filename", "splitdns.config"); - registerFile("proxy.config.ssl.server.multicert.filename", "ssl_multicert.config"); - registerFile("proxy.config.ssl.servername.filename", "sni.config"); + registerFile("proxy.config.log.config.filename", ts::filename::LOGGING, NOT_REQUIRED); + registerFile("", ts::filename::STORAGE, REQUIRED); + registerFile("proxy.config.socks.socks_config_file", ts::filename::SOCKS, NOT_REQUIRED); + registerFile(ts::filename::RECORDS, ts::filename::RECORDS, NOT_REQUIRED); + registerFile("proxy.config.cache.control.filename", ts::filename::CACHE, NOT_REQUIRED); + registerFile("proxy.config.cache.ip_allow.filename", ts::filename::IP_ALLOW, NOT_REQUIRED); + registerFile("proxy.config.http.parent_proxy.file", ts::filename::PARENT, NOT_REQUIRED); + registerFile("proxy.config.url_remap.filename", ts::filename::REMAP, NOT_REQUIRED); + registerFile("", ts::filename::VOLUME, NOT_REQUIRED); + registerFile("proxy.config.cache.hosting_filename", ts::filename::HOSTING, NOT_REQUIRED); + registerFile("", ts::filename::PLUGIN, NOT_REQUIRED); + registerFile("proxy.config.dns.splitdns.filename", ts::filename::SPLITDNS, NOT_REQUIRED); + registerFile("proxy.config.ssl.server.multicert.filename", ts::filename::SSL_MULTICERT, NOT_REQUIRED); + registerFile("proxy.config.ssl.servername.filename", ts::filename::SNI, NOT_REQUIRED); configFiles->registerCallback(testcall); } diff --git a/src/traffic_manager/traffic_manager.cc b/src/traffic_manager/traffic_manager.cc index e6f7192bb46..e31eb78aca9 100644 --- a/src/traffic_manager/traffic_manager.cc +++ b/src/traffic_manager/traffic_manager.cc @@ -28,6 +28,7 @@ #include "tscore/ink_args.h" #include "tscore/ink_syslog.h" #include "tscore/runroot.h" +#include "tscore/Filenames.h" #include "WebMgmtUtils.h" #include "MgmtUtils.h" @@ -90,7 +91,7 @@ static char bind_stderr[512] = ""; static const char *mgmt_path = nullptr; // By default, set the current directory as base -static const char *recs_conf = "records.config"; +static const char *recs_conf = ts::filename::RECORDS; static int fds_limit; @@ -195,7 +196,7 @@ waited_enough() return false; } - return (lmgmt->mgmt_shutdown_triggered_at + timeout >= time(nullptr)); + return (timeout ? (lmgmt->mgmt_shutdown_triggered_at + timeout <= time(nullptr)) : false); } static void @@ -557,6 +558,11 @@ main(int argc, const char **argv) } RecGetRecordInt("proxy.config.net.connections_throttle", &fds_throttle); + RecInt listen_per_thread = 0; + RecGetRecordInt("proxy.config.exec_thread.listen", &listen_per_thread); + if (listen_per_thread > 0) { // Turn off listening. Traffic server is going to listen on all the threads. + listen_off = true; + } set_process_limits(fds_throttle); // as root diff --git a/src/traffic_quic/Makefile.inc b/src/traffic_quic/Makefile.inc index 963576f5c2d..c00faf01e8a 100644 --- a/src/traffic_quic/Makefile.inc +++ b/src/traffic_quic/Makefile.inc @@ -20,6 +20,7 @@ bin_PROGRAMS += traffic_quic/traffic_quic traffic_quic_traffic_quic_CPPFLAGS = \ $(AM_CPPFLAGS) \ $(iocore_include_dirs) \ + -I$(abs_top_srcdir)/include \ -I$(abs_top_srcdir)/lib \ -I$(abs_top_srcdir)/lib/records \ -I$(abs_top_srcdir)/mgmt \ @@ -32,7 +33,7 @@ traffic_quic_traffic_quic_CPPFLAGS = \ -I$(abs_top_srcdir)/proxy/logging \ -I$(abs_top_srcdir)/proxy/shared \ $(TS_INCLUDES) \ - @OPENSSL_INCLUDES@ + @OPENSSL_INCLUDES@ @YAMLCPP_INCLUDES@ traffic_quic_traffic_quic_LDFLAGS = \ $(AM_LDFLAGS) \ diff --git a/src/traffic_quic/quic_client.cc b/src/traffic_quic/quic_client.cc index 007c20ae3e8..04fb44d654c 100644 --- a/src/traffic_quic/quic_client.cc +++ b/src/traffic_quic/quic_client.cc @@ -28,13 +28,14 @@ #include #include "Http3Transaction.h" +#include "P_QUICNetVConnection.h" // OpenSSL protocol-lists format (vector of 8-bit length-prefixed, byte strings) // https://www.openssl.org/docs/manmaster/man3/SSL_CTX_set_alpn_protos.html // Should be integrate with IP_PROTO_TAG_HTTP_QUIC in ts/ink_inet.h ? using namespace std::literals; -static constexpr std::string_view HQ_ALPN_PROTO_LIST("\5hq-20"sv); -static constexpr std::string_view H3_ALPN_PROTO_LIST("\5h3-20"sv); +static constexpr std::string_view HQ_ALPN_PROTO_LIST("\5hq-29\5hq-27"sv); +static constexpr std::string_view H3_ALPN_PROTO_LIST("\5h3-29\5h3-27"sv); QUICClient::QUICClient(const QUICClientConfig *config) : Continuation(new_ProxyMutex()), _config(config) { @@ -80,7 +81,11 @@ QUICClient::start(int, void *) opt.socket_recv_bufsize = 1048576; opt.socket_send_bufsize = 1048576; opt.alpn_protos = alpn_protos; - opt.set_sni_servername(this->_config->addr, strnlen(this->_config->addr, 1023)); + if (strlen(this->_config->server_name) == 0) { + opt.set_sni_servername(this->_config->addr, strnlen(this->_config->addr, 1023)); + } else { + opt.set_sni_servername(this->_config->server_name, strnlen(this->_config->server_name, 1023)); + } SCOPED_MUTEX_LOCK(lock, this->mutex, this_ethread()); @@ -224,7 +229,8 @@ Http09ClientApp::main_event_handler(int event, Event *data) if (stream_io->is_read_done() && this->_config->close) { // Connection Close Exercise - this->_qc->close(QUICConnectionErrorUPtr(new QUICConnectionError(QUICTransErrorCode::NO_ERROR, "Close Exercise"))); + this->_qc->close_quic_connection( + QUICConnectionErrorUPtr(new QUICConnectionError(QUICTransErrorCode::NO_ERROR, "Close Exercise"))); } break; @@ -268,11 +274,20 @@ Http3ClientApp::~Http3ClientApp() void Http3ClientApp::start() { - this->_req_buf = new_MIOBuffer(); - this->_resp_buf = new_MIOBuffer(); + this->_req_buf = new_MIOBuffer(BUFFER_SIZE_INDEX_32K); + this->_resp_buf = new_MIOBuffer(BUFFER_SIZE_INDEX_32K); IOBufferReader *resp_buf_reader = _resp_buf->alloc_reader(); - this->_resp_handler = new RespHandler(this->_config, resp_buf_reader); + this->_resp_handler = new RespHandler(this->_config, resp_buf_reader, [&](void) { + if (this->_config->close) { + // Connection Close Exercise + this->_qc->close_quic_connection( + QUICConnectionErrorUPtr(new QUICConnectionError(QUICTransErrorCode::NO_ERROR, "Close Exercise"))); + } else if (this->_config->reset) { + // Stateless Reset Exercise + this->_qc->reset_quic_connection(); + } + }); super::start(); this->_do_http_request(); @@ -309,7 +324,13 @@ Http3ClientApp::_do_http_request() format = "GET https://%s/%s HTTP/1.1\r\n\r\n"; } - int request_len = snprintf(request, sizeof(request), format.c_str(), this->_config->addr, this->_config->path); + const char *authority; + if (strlen(this->_config->server_name) == 0) { + authority = this->_config->addr; + } else { + authority = this->_config->server_name; + } + int request_len = snprintf(request, sizeof(request), format.c_str(), authority, this->_config->path); Http09ClientAppDebug("\n%s", request); @@ -322,8 +343,8 @@ Http3ClientApp::_do_http_request() // // Response Handler // -RespHandler::RespHandler(const QUICClientConfig *config, IOBufferReader *reader) - : Continuation(new_ProxyMutex()), _config(config), _reader(reader) +RespHandler::RespHandler(const QUICClientConfig *config, IOBufferReader *reader, std::function on_complete) + : Continuation(new_ProxyMutex()), _config(config), _reader(reader), _on_complete(on_complete) { if (this->_config->output[0] != 0x0) { this->_filename = this->_config->output; @@ -372,6 +393,10 @@ RespHandler::main_event_handler(int event, Event *data) std::cout.rdbuf(default_stream); } + if (event == VC_EVENT_READ_COMPLETE) { + this->_on_complete(); + } + break; } case VC_EVENT_WRITE_READY: diff --git a/src/traffic_quic/quic_client.h b/src/traffic_quic/quic_client.h index 9af9f0d3593..fc12dbc436c 100644 --- a/src/traffic_quic/quic_client.h +++ b/src/traffic_quic/quic_client.h @@ -38,8 +38,10 @@ struct QUICClientConfig { char output[1024] = {0}; char port[16] = "4433"; char path[1018] = "/"; + char server_name[128] = ""; char debug_tags[1024] = "quic|vv_quic_crypto|http3|qpack"; int close = false; + int reset = false; int http0_9 = true; int http3 = false; }; @@ -47,7 +49,7 @@ struct QUICClientConfig { class RespHandler : public Continuation { public: - RespHandler(const QUICClientConfig *config, IOBufferReader *reader); + RespHandler(const QUICClientConfig *config, IOBufferReader *reader, std::function on_complete); int main_event_handler(int event, Event *data); void set_read_vio(VIO *vio); @@ -56,6 +58,7 @@ class RespHandler : public Continuation const char *_filename = nullptr; IOBufferReader *_reader = nullptr; VIO *_read_vio = nullptr; + std::function _on_complete; }; class QUICClient : public Continuation diff --git a/src/traffic_quic/traffic_quic.cc b/src/traffic_quic/traffic_quic.cc index 1c290eb6934..829704e477b 100644 --- a/src/traffic_quic/traffic_quic.cc +++ b/src/traffic_quic/traffic_quic.cc @@ -61,8 +61,10 @@ main(int argc, const char **argv) {"output", 'o', "Write to FILE instead of stdout", "S1023", config.output, nullptr, nullptr}, {"port", 'p', "Port", "S15", config.port, nullptr, nullptr}, {"path", 'P', "Path", "S1017", config.path, nullptr, nullptr}, + {"server", 's', "Server name", "S127", config.server_name, nullptr, nullptr}, {"debug", 'T', "Vertical-bar-separated Debug Tags", "S1023", config.debug_tags, nullptr, nullptr}, - {"close", 'c', "Enable connection close excercise", "F", &config.close, nullptr, nullptr}, + {"close", 'c', "Enable connection close exercise", "F", &config.close, nullptr, nullptr}, + {"reset", 'r', "Enable stateless reset exercise", "F", &config.reset, nullptr, nullptr}, {"http0_9", '-', "Enable HTTP/0.9", "T", &config.http0_9, nullptr, nullptr}, {"http3", '-', "Enable HTTP/3", "F", &config.http3, nullptr, nullptr}, @@ -312,7 +314,7 @@ HttpSM::attach_client_session(ProxyTransaction *, IOBufferReader *) } void -HttpSM::init() +HttpSM::init(bool from_early_data) { ink_abort("do not call stub"); } diff --git a/src/traffic_server/CoreUtils.cc b/src/traffic_server/CoreUtils.cc deleted file mode 100644 index ef3d9be6bb5..00000000000 --- a/src/traffic_server/CoreUtils.cc +++ /dev/null @@ -1,911 +0,0 @@ -/** @file - - A brief file description - - @section license License - - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - */ - -/**************************************************************************** - - CoreUtils.cc - - Description: Automated processing of core files on Linux - ****************************************************************************/ - -/* - Stack Unwinding procedure on ix86 architecture on Linux : - Get the first frame pointer in $ebp. - The value stored in $ebp is the address of prev frame pointer. - Keep on unwinding till it is Ox0. - $ebp+4 in each frame represents $eip.(PC) -*/ - -/* - * Accessing arguments on 386 : - * ---------------------------- - * We need to start from $ebp+4 and then keep on reading args - * till we reach the base pointer for prev. frame - * - * - * (high memory) - * + + - * | Callers Stack Frame | - * +---------------------+ - * | function call | - * | arguments | - * +---------------------+ - * | Return Address + - * +-------------------- + - * | Old base pointer + Base pointer BP - * +-------------------- + - * | | - * | | - * | | Local (automatic) variables - * | | - * | | - * | | - * | | - * | | - * | | - * +---------------------+ Stack pointer SP - * | free stack | (low memory, top of the stack) - * | begins here | - * + + - * - * - * +-----------------+ +-----------------+ - * FP -> | previous FP --------> | previous FP ------>... - * | | | | - * | return address | | return address | - * +-----------------+ +-----------------+ - */ - -/* 32-bit arguments are pushed down stack in reverse syntactic order (hence accessed/popped in the right order), above the 32-bit - * near return address. %ebp, %esi, %edi, %ebx are callee-saved, other registers are caller-saved; %eax is to hold the result, or - * %edx:%eax for 64-bit results */ - -/* has -fomit-frame-pointer has any repercussions?? - We assume that all the code is generated with frame pointers set. */ - -/* modify the "note" in process_core */ -/* Document properly */ - -#include "tscore/ink_config.h" - -#if defined(linux) -#include "CoreUtils.h" - -#define __p_type p_type // ugly hack? - see resolv.h -#define D(x) x /* for debugging */ -intptr_t f1, f2; -int framepointer = 0; -int program_counter = 0; -#endif // linux check - -#if defined(darwin) || defined(freebsd) || defined(solaris) || defined(openbsd) // FIXME: solaris x86 -// TODO: Cleanup multiple includes -#include -#include -#include -#include "tscore/ink_platform.h" -#include "CoreUtils.h" -#endif /* darwin || freebsd || solaris */ - -#include "EventName.h" -#include "http/HttpSM.h" - -#include -#include - -bool inTable; -FILE *fp; -memTable default_memTable = {0, 0, 0}; -std::vector arrayMem(0, default_memTable); - -HTTPHdrImpl *global_http; -HttpSM *last_seen_http_sm = nullptr; - -char ethread_ptr_str[256] = ""; -char netvc_ptr_str[256] = ""; - -HdrHeap *swizzle_heap; -char *ptr_data; - -// returns the index of the vaddr or the index after where it should be -intptr_t -CoreUtils::find_vaddr(intptr_t vaddr, intptr_t upper, intptr_t lower) -{ - intptr_t index = static_cast(floor(static_cast((upper + lower) / 2))); - - // match in table, returns index to be inserted into - if (arrayMem[index].vaddr == vaddr) { - inTable = true; - return index + 1; - // no match - } else if (upper == lower) { - inTable = false; - return upper; - // no match - } else if (index == lower) { - inTable = false; - if ((index == 0) && (arrayMem[index].vaddr > vaddr)) { - return 0; - } else { - return index + 1; - } - } else { - if (arrayMem[index].vaddr > vaddr) { - return find_vaddr(vaddr, index, lower); - } else { - return find_vaddr(vaddr, upper, index); - } - } - assert(0); - return -1; -} - -// inserts virtual address struct into the list -void -CoreUtils::insert_table(intptr_t vaddr1, intptr_t offset1, intptr_t fsize1) -{ - if (arrayMem.empty()) { - arrayMem.push_back({vaddr1, offset1, fsize1}); - } else { - unsigned index = find_vaddr(vaddr1, arrayMem.size(), 0); - arrayMem.insert(arrayMem.begin() + index, {vaddr1, offset1, fsize1}); - } -} - -// returns -1 on failure otherwise fills the buffer and -// returns the number of bytes read -intptr_t -CoreUtils::read_from_core(intptr_t vaddr, intptr_t bytes, char *buf) -{ - intptr_t index = find_vaddr(vaddr, arrayMem.size(), 0); - intptr_t vadd = arrayMem[index - 1].vaddr; - intptr_t offset = arrayMem[index - 1].offset; - intptr_t size = arrayMem[index - 1].fsize; - intptr_t offset2 = std::abs(vaddr - vadd); - - if (bytes > (size - offset2)) { - return -1; - } else { - if (fseek(fp, offset2 + offset, SEEK_SET) != -1) { - char *frameoff; - if ((frameoff = (char *)ats_malloc(sizeof(char) * bytes))) { - if (fread(frameoff, bytes, 1, fp) == 1) { - memcpy(buf, frameoff, bytes); - /*for(int j =0; j < bytes; j++) { - *buf++ = getc(fp); - } - buf -= bytes;*/ - ats_free(frameoff); - return bytes; - } - ats_free(frameoff); - } - } else { - return -1; - } - } - - return -1; -} - -/* Linux Specific functions */ - -#if defined(linux) -// copies stack info for the thread's base frame to the given -// core_stack_state pointer -void -CoreUtils::get_base_frame(intptr_t framep, core_stack_state *coress) -{ - // finds vaddress less than framep - intptr_t index = find_vaddr(framep, arrayMem.size(), 0); - intptr_t vadd = arrayMem[index - 1].vaddr; - intptr_t off = arrayMem[index - 1].offset; - intptr_t off2 = std::abs(vadd - framep); - intptr_t size = arrayMem[index - 1].fsize; - intptr_t i = 0; - - memset(coress, 0, sizeof(*coress)); - D(printf("stkbase=%p\n", (void *)(vadd + size))); - // seek to the framep offset - if (fseek(fp, off + off2, SEEK_SET) != -1) { - void **frameoff; - if ((frameoff = (void **)ats_malloc(sizeof(long)))) { - if (fread(frameoff, 4, 1, fp) == 1) { - coress->framep = (intptr_t)*frameoff; - if (fread(frameoff, 4, 1, fp) == 1) { - coress->pc = (intptr_t)*frameoff; - } - // read register arguments - for (i = 0; i < NO_OF_ARGS; i++) { - if (fread(frameoff, 4, 1, fp) == 1) { - coress->arg[i] = (intptr_t)*frameoff; - } - } - } - ats_free(frameoff); - } - } else { - printf("Failed to seek to top of the stack\n"); - } - // coress->stkbase = vadd+size; -} - -// returns 0 if current frame is already at the top of the stack -// or returns 1 and moves up the stack once -int -CoreUtils::get_next_frame(core_stack_state *coress) -{ - intptr_t i = 0; - intptr_t framep = coress->framep; - - intptr_t index = find_vaddr(framep, arrayMem.size(), 0); - - // finds vaddress less than framep - intptr_t vadd = arrayMem[index - 1].vaddr; - intptr_t off = arrayMem[index - 1].offset; - intptr_t off2 = std::abs(vadd - framep); - - // seek to the framep offset - if (fseek(fp, off + off2, SEEK_SET) != -1) { - void **frameoff; - if ((frameoff = (void **)ats_malloc(sizeof(long)))) { - if (fread(frameoff, 4, 1, fp) == 1) { - coress->framep = (intptr_t)*frameoff; - if (*frameoff == nullptr) { - ats_free(frameoff); - return 0; - } - if (fread(frameoff, 4, 1, fp) == 1) { - coress->pc = (intptr_t)*frameoff; - } - for (i = 0; i < NO_OF_ARGS; i++) { - if (fread(frameoff, 4, 1, fp) == 1) { - coress->arg[i] = (intptr_t)*frameoff; - } - } - } - ats_free(frameoff); - } - return 1; - } - - return 0; -} - -// prints the http header -void -CoreUtils::find_stuff(StuffTest_f f) -{ - intptr_t framep = framepointer; - intptr_t pc = program_counter; - core_stack_state coress; - intptr_t i; - void *test_val; - int framecount = 0; - - // Unwinding the stack - D(printf("\nStack Trace:\n")); - D(printf("stack frame#%d framep=%p pc=%p\n", framecount, (void *)framep, (void *)pc)); - framecount++; - get_base_frame(framep, &coress); - f2 = framep; - do { - f1 = f2; - f2 = coress.framep; - D(printf("stack frame#%d framep=%p pc=%p f1-f2=%p coress=%p %p %p %p %p\n", framecount, (void *)coress.framep, - (void *)coress.pc, (void *)(f2 - f1), (void *)coress.arg[0], (void *)coress.arg[1], (void *)coress.arg[2], - (void *)coress.arg[3], (void *)coress.arg[4])); - - for (i = 0; i < NO_OF_ARGS; i++) { - test_val = (void *)coress.arg[i]; - f(test_val); - } - framecount++; - } while (get_next_frame(&coress) != 0); -} -#endif // linux check - -// test whether a given register is an HttpSM -// if it is, call process_HttpSM on it -void -CoreUtils::test_HdrHeap(void *arg) -{ - HdrHeap *hheap_test = static_cast(arg); - uint32_t *magic_ptr = &(hheap_test->m_magic); - uint32_t magic = 0; - - if (read_from_core((intptr_t)magic_ptr, sizeof(uint32_t), reinterpret_cast(&magic)) != 0) { - if (magic == HDR_BUF_MAGIC_ALIVE || magic == HDR_BUF_MAGIC_DEAD || magic == HDR_BUF_MAGIC_CORRUPT || - magic == HDR_BUF_MAGIC_MARSHALED) { - printf("Found Hdr Heap @ 0x%p\n", arg); - } - } -} - -// test whether a given register is an HttpSM -// if it is, call process_HttpSM on it -// -// This code generates errors from Clang, on hsm_test not being initialized -// properly. Currently this is not used, so ifdef'ing out to suppress. -#ifndef __clang_analyzer__ -void -CoreUtils::test_HttpSM_from_tunnel(void *arg) -{ - char *tmp = (char *)arg; - intptr_t offset = (intptr_t) & (((HttpTunnel *)nullptr)->sm); - HttpSM **hsm_ptr = (HttpSM **)(tmp + offset); - HttpSM *hsm_test = nullptr; - - if (read_from_core((intptr_t)hsm_ptr, sizeof(HttpSM *), (char *)&hsm_test) == 0) { - return; - } - - unsigned int *magic_ptr = &(hsm_test->magic); - unsigned int magic = 0; - - if (read_from_core((intptr_t)magic_ptr, sizeof(int), (char *)&magic) != 0) { - if (magic == HTTP_SM_MAGIC_ALIVE || magic == HTTP_SM_MAGIC_DEAD) { - process_HttpSM(hsm_test); - } - } -} -#endif - -// test whether a given register is an HttpSM -// if it is, call process_HttpSM on it -void -CoreUtils::test_HttpSM(void *arg) -{ - HttpSM *hsm_test = static_cast(arg); - unsigned int *magic_ptr = &(hsm_test->magic); - unsigned int magic = 0; - - if (read_from_core((intptr_t)magic_ptr, sizeof(int), reinterpret_cast(&magic)) != 0) { - if (magic == HTTP_SM_MAGIC_ALIVE || magic == HTTP_SM_MAGIC_DEAD) { - printf("test_HttpSM:******MATCH*****\n"); - process_HttpSM(hsm_test); - } - } -} - -void -CoreUtils::process_HttpSM(HttpSM *core_ptr) -{ - // extracting the HttpSM from the core file - if (last_seen_http_sm != core_ptr) { - HttpSM *http_sm = (HttpSM *)ats_malloc(sizeof(HttpSM)); - - if (read_from_core((intptr_t)core_ptr, sizeof(HttpSM), (char *)http_sm) < 0) { - printf("ERROR: Failed to read httpSM @ 0x%p from core\n", core_ptr); - ats_free(http_sm); - return; - } - - if (http_sm->magic == HTTP_SM_MAGIC_ALIVE) { - last_seen_http_sm = core_ptr; - - if (is_debug_tag_set("magic")) { -#if defined(linux) - printf("\n*****match-ALIVE*****\n"); -#endif - } - printf("---- Found HttpSM --- id %" PRId64 " ------ @ 0x%p -----\n\n", http_sm->sm_id, http_sm); - - print_http_hdr(&http_sm->t_state.hdr_info.client_request, "Client Request"); - print_http_hdr(&http_sm->t_state.hdr_info.server_request, "Server Request"); - print_http_hdr(&http_sm->t_state.hdr_info.server_response, "Server Response"); - print_http_hdr(&http_sm->t_state.hdr_info.client_response, "Client Response"); - - dump_history(http_sm); - - printf("------------------------------------------------\n\n\n"); - } else if (http_sm->magic == HTTP_SM_MAGIC_DEAD) { - if (is_debug_tag_set("magic")) { -#if defined(linux) - printf("\n*****match-DEAD*****\n"); -#endif - } - } - ats_free(http_sm); - } else { - printf("process_HttpSM : last_seen_http_sm == core_ptr\n"); - } -} - -void -CoreUtils::print_http_hdr(HTTPHdr *h, const char *name) -{ - HTTPHdr new_handle; - - if (h->m_heap && h->m_http) { - int r = load_http_hdr(h, &new_handle); - - if (r > 0 && new_handle.m_http) { - printf("----------- %s ------------\n", name); - new_handle.m_mime = new_handle.m_http->m_fields_impl; - new_handle.print(nullptr, 0, nullptr, nullptr); - printf("-----------------------------\n\n"); - } - } -} - -int -CoreUtils::load_http_hdr(HTTPHdr *core_hdr, HTTPHdr *live_hdr) -{ - char buf[sizeof(char) * sizeof(HdrHeap)]; - // Load HdrHeap chain - HTTPHdr *http_hdr = core_hdr; - HdrHeap *heap = (HdrHeap *)core_hdr->m_heap; - HdrHeap *heap_ptr = (HdrHeap *)http_hdr->m_heap; - intptr_t ptr_heaps = 0; - intptr_t ptr_heap_size = 0; - intptr_t str_size = 0; - intptr_t str_heaps = 0; - std::vector ptr_xlation(2); - intptr_t i; - intptr_t copy_size; - - // extracting the header heap from the core file - do { - if (read_from_core((intptr_t)heap, sizeof(HdrHeap), buf) == -1) { - printf("Cannot read from core\n"); - ::exit(0); - } - heap = (HdrHeap *)buf; - copy_size = (int)(heap->m_free_start - heap->m_data_start); - ptr_heap_size += copy_size; - heap = heap->m_next; - } while (heap && ((intptr_t)heap != 0x1)); - - swizzle_heap = (HdrHeap *)ats_malloc(sizeof(HdrHeap)); - live_hdr->m_heap = swizzle_heap; - ptr_data = (char *)ats_malloc(sizeof(char) * ptr_heap_size); - // heap = (HdrHeap*)http_hdr->m_heap; - - // Build Hdr Heap Translation Table - do { - if (read_from_core((intptr_t)heap_ptr, sizeof(HdrHeap), buf) == -1) { - printf("Cannot read from core\n"); - ::exit(0); - } - heap_ptr = (HdrHeap *)buf; - copy_size = (int)(heap_ptr->m_free_start - heap_ptr->m_data_start); - - if (read_from_core((intptr_t)heap_ptr->m_data_start, copy_size, ptr_data) == -1) { - printf("Cannot read from core\n"); - ::exit(0); - } - // Expand ptr xlation table if necessary - if (static_cast(ptr_heaps) >= ptr_xlation.size()) { - ptr_xlation.resize(ptr_heaps + 1); - } - - char *data, *free, *off; - data = heap_ptr->m_data_start; - free = heap_ptr->m_free_start; - off = (char *)(heap_ptr->m_data_start - ptr_data); - - ptr_xlation[ptr_heaps].start = data; - ptr_xlation[ptr_heaps].end = free; - ptr_xlation[ptr_heaps].offset = off; - ptr_data += copy_size; - ptr_heaps++; - heap_ptr = heap_ptr->m_next; - } while (heap_ptr && ((intptr_t)heap_ptr != 0x1)); - - heap = (HdrHeap *)http_hdr->m_heap; - if (read_from_core((intptr_t)heap, sizeof(HdrHeap), buf) == -1) { - printf("Cannot read from core\n"); - ::exit(0); - } - heap = (HdrHeap *)buf; - // filling in the live_hdr - swizzle_heap->m_free_start = nullptr; - swizzle_heap->m_data_start = (char *)ptr_data - ptr_heap_size; // offset - swizzle_heap->m_magic = HDR_BUF_MAGIC_ALIVE; - swizzle_heap->m_writeable = false; - swizzle_heap->m_size = ptr_heap_size; - swizzle_heap->m_next = nullptr; - swizzle_heap->m_free_size = 0; - swizzle_heap->m_read_write_heap.m_ptr = nullptr; - - // We'have one read-only string heap after marshalling - swizzle_heap->m_ronly_heap[0].m_heap_start = (char *)(intptr_t)swizzle_heap->m_size; // offset - swizzle_heap->m_ronly_heap[0].m_ref_count_ptr.m_ptr = nullptr; - - for (unsigned i = 1; i < HDR_BUF_RONLY_HEAPS; ++i) { - swizzle_heap->m_ronly_heap[i].m_heap_start = nullptr; - } - - // Next order of business is to copy over string heaps - // As we are copying over the string heaps, build - // translation table for string marshaling in the heap - // objects - MarshalXlate str_xlation[HDR_BUF_RONLY_HEAPS + 1]; - - // Local String Heaps, building translation table - if (heap->m_read_write_heap) { - HdrStrHeap *hdr = (HdrStrHeap *)heap->m_read_write_heap.m_ptr; - char *copy_start = ((char *)heap->m_read_write_heap.m_ptr) + sizeof(HdrStrHeap); - char *str_hdr = (char *)ats_malloc(sizeof(char) * sizeof(HdrStrHeap)); - if (read_from_core((intptr_t)hdr, sizeof(HdrStrHeap), str_hdr) == -1) { - printf("Cannot read from core\n"); - ::exit(0); - } - - char *free_start = (char *)(((HdrStrHeap *)str_hdr)->m_free_start); - int nto_copy = std::abs(copy_start - free_start); - ats_free(str_hdr); - char rw_heap[sizeof(char) * nto_copy]; - if (read_from_core((intptr_t)copy_start, nto_copy, rw_heap) == -1) { - printf("Cannot read from core\n"); - ::exit(0); - } - // FIX ME - possible offset overflow issues? - str_xlation[str_heaps].start = copy_start; - str_xlation[str_heaps].end = copy_start + nto_copy; - str_xlation[str_heaps].offset = (char *)(copy_start - rw_heap); - - str_size += nto_copy; - str_heaps++; - } - - for (i = 0; i < HDR_BUF_RONLY_HEAPS; i++) { - if (heap->m_ronly_heap[i].m_heap_start != nullptr) { - char ro_heap[sizeof(char) * heap->m_ronly_heap[i].m_heap_len]; - if (read_from_core((intptr_t)heap->m_ronly_heap[i].m_heap_start, heap->m_ronly_heap[i].m_heap_len, ro_heap) == -1) { - printf("Cannot read from core\n"); - ::exit(0); - } - // Add translation table entry for string heaps - str_xlation[str_heaps].start = heap->m_ronly_heap[i].m_heap_start; - str_xlation[str_heaps].end = heap->m_ronly_heap[i].m_heap_start + heap->m_ronly_heap[i].m_heap_len; - str_xlation[str_heaps].offset = (char *)(heap->m_ronly_heap[i].m_heap_start - ro_heap); - - ink_assert(str_xlation[str_heaps].start <= str_xlation[str_heaps].end); - - str_heaps++; - str_size += heap->m_ronly_heap[i].m_heap_len; - } - } - - // Patch the str heap len - swizzle_heap->m_ronly_heap[0].m_heap_len = str_size; - - char *obj_data = swizzle_heap->m_data_start; - char *mheap_end = swizzle_heap->m_data_start + swizzle_heap->m_size; - - while (obj_data < mheap_end) { - HdrHeapObjImpl *obj = (HdrHeapObjImpl *)obj_data; - ink_assert(obj_is_aligned(obj)); - - switch (obj->m_type) { - case HDR_HEAP_OBJ_URL: - if (((URLImpl *)obj)->marshal(str_xlation, str_heaps) < 0) { - goto Failed; - } - break; - case HDR_HEAP_OBJ_HTTP_HEADER: - if (((HTTPHdrImpl *)obj)->marshal(&ptr_xlation[0], ptr_heaps, str_xlation, str_heaps) < 0) { - goto Failed; - } - live_hdr->m_http = (HTTPHdrImpl *)obj; - break; - case HDR_HEAP_OBJ_FIELD_BLOCK: - if (((MIMEFieldBlockImpl *)obj)->marshal(&ptr_xlation[0], ptr_heaps, str_xlation, str_heaps) < 0) { - goto Failed; - } - break; - case HDR_HEAP_OBJ_MIME_HEADER: - if (((MIMEHdrImpl *)obj)->marshal(&ptr_xlation[0], ptr_heaps, str_xlation, str_heaps)) { - goto Failed; - } - break; - case HDR_HEAP_OBJ_EMPTY: - break; - case HDR_HEAP_OBJ_RAW: - // Check to make sure we aren't stuck - // in an infinite loop - if (obj->m_length <= 0) { - ink_assert(0); - goto Failed; - } - // Nothing to do - break; - default: - ink_release_assert(0); - } - obj_data = obj_data + obj->m_length; - } - - // Add up the total bytes used - return HdrHeapMarshalBlocks{ts::round_up(ptr_heap_size + str_size + HDR_HEAP_HDR_SIZE)}; - -Failed: - swizzle_heap->m_magic = HDR_BUF_MAGIC_CORRUPT; - return -1; -} - -void -CoreUtils::dump_history(HttpSM *hsm) -{ - printf("-------- Begin History -------------\n"); - - // Loop through the history and dump it - for (unsigned int i = 0; i < hsm->history.size(); i++) { - char loc[256]; - int r = (int)hsm->history[i].reentrancy; - int e = (int)hsm->history[i].event; - char *fileline = load_string(hsm->history[i].location.str(loc, sizeof(loc))); - - fileline = (fileline != nullptr) ? fileline : ats_strdup("UNKNOWN"); - - printf("%d %d %s", e, r, fileline); - - char buffer[32]; - const char *msg = event_int_to_string(e, sizeof(buffer), buffer); - printf(" event string: \"%s\"\n", msg); - - ats_free(fileline); - } - - printf("-------- End History -----------\n\n"); -} - -void -CoreUtils::process_EThread(EThread *eth_test) -{ - char *buf = (char *)ats_malloc(sizeof(char) * sizeof(EThread)); - - if (read_from_core((intptr_t)eth_test, sizeof(EThread), buf) != -1) { - EThread *loaded_eth = reinterpret_cast(buf); - - printf("----------- EThread @ 0x%p ----------\n", eth_test); -#if !defined(kfreebsd) && (defined(freebsd) || defined(darwin) || defined(openbsd)) - printf(" thread_id: %p\n", loaded_eth->tid); -#else - printf(" thread_id: %i\n", (int)loaded_eth->tid); -#endif - // printf(" NetHandler: 0x%x\n\n", (int) loaded_eth->netHandler); - } - - ats_free(buf); -} - -void -CoreUtils::print_netstate(NetState *n) -{ - printf(" enabled: %d\n", n->enabled); - printf(" op: %d cont: 0x%p\n", n->vio.op, n->vio.cont); - printf(" nbytes: %d done: %d\n", (int)n->vio.nbytes, (int)n->vio.ndone); - printf(" vc_server: 0x%p mutex: 0x%p\n\n", n->vio.vc_server, n->vio.mutex.m_ptr); -} - -void -CoreUtils::process_NetVC(UnixNetVConnection *nvc_test) -{ - char *buf = (char *)ats_malloc(sizeof(char) * sizeof(UnixNetVConnection)); - - if (read_from_core((intptr_t)nvc_test, sizeof(UnixNetVConnection), buf) != -1) { - UnixNetVConnection *loaded_nvc = reinterpret_cast(buf); - char addrbuf[INET6_ADDRSTRLEN]; - - printf("----------- UnixNetVConnection @ 0x%p ----------\n", nvc_test); - printf(" ip: %s port: %d\n", ats_ip_ntop(loaded_nvc->get_remote_addr(), addrbuf, sizeof(addrbuf)), - ats_ip_port_host_order(loaded_nvc->get_remote_addr())); - printf(" closed: %d\n\n", loaded_nvc->closed); - printf(" read state: \n"); - print_netstate(&loaded_nvc->read); - printf(" write state: \n"); - print_netstate(&loaded_nvc->write); - } - - ats_free(buf); -} - -char * -CoreUtils::load_string(const char *addr) -{ - char buf[2048]; - int index = 0; - - if (addr == nullptr) { - return ats_strdup("NONE"); - } - - while (index < 2048) { - if (read_from_core((intptr_t)(addr + index), 1, buf + index) < 0) { - return nullptr; - } - - if (buf[index] == '\0') { - return ats_strdup(buf); - } - index++; - } - - return nullptr; -} - -// parses core file -#if defined(linux) -void -process_core(char *fname) -{ - Elf32_Ehdr ehdr; - Elf32_Phdr phdr; - int phoff, phnum, phentsize; - int framep = 0, pc = 0; - - /* Open the input file */ - if (!(fp = fopen(fname, "r"))) { - printf("cannot open file\n"); - ::exit(1); - } - - /* Obtain the .shstrtab data buffer */ - if (fread(&ehdr, sizeof ehdr, 1, fp) != 1) { - printf("Unable to read ehdr\n"); - ::exit(1); - } - // program header offset - phoff = ehdr.e_phoff; - // number of program headers - phnum = ehdr.e_phnum; - // size of each program header - phentsize = ehdr.e_phentsize; - - for (int i = 0; i < phnum; i++) { - if (fseek(fp, phoff + i * phentsize, SEEK_SET) == -1) { - fprintf(stderr, "Unable to seek to Phdr %d\n", i); - ::exit(1); - } - - if (fread(&phdr, sizeof phdr, 1, fp) != 1) { - fprintf(stderr, "Unable to read Phdr %d\n", i); - ::exit(1); - } - int poffset, psize; - int pvaddr; - /* This member gives the virtual address at which the first byte of the - segment resides in memory. */ - pvaddr = phdr.p_vaddr; - /* This member gives the offset from the beginning of the file at which - the first byte of the segment resides. */ - poffset = phdr.p_offset; - /* This member gives the number of bytes in the file image of the - segment; it may be zero. */ - psize = phdr.p_filesz; - - if (pvaddr != 0) { - CoreUtils::insert_table(pvaddr, poffset, psize); - } - - if (is_debug_tag_set("phdr")) { - printf("\n******* PHDR %d *******\n", i); - printf("p_type = %u ", phdr.p_type); - printf("p_offset = %u ", phdr.p_offset); - printf("p_vaddr = %#x ", pvaddr); - - printf("p_paddr = %#x\n", phdr.p_paddr); - printf("p_filesz = %u ", phdr.p_filesz); - printf("p_memsz = %u ", phdr.p_memsz); - printf("p_flags = %u ", phdr.p_flags); - printf("p_align = %u\n", phdr.p_align); - } - - if (phdr.p_type == PT_NOTE) { - printf("NOTE\n"); - if (fseek(fp, phdr.p_offset, SEEK_SET) != -1) { - Elf32_Nhdr *nhdr, *thdr; - if ((nhdr = (Elf32_Nhdr *)ats_malloc(sizeof(Elf32_Nhdr) * phdr.p_filesz))) { - if (fread(nhdr, phdr.p_filesz, 1, fp) == 1) { - int size = phdr.p_filesz; - int sum = 0; - thdr = nhdr; - while (size) { - int len; - - len = sizeof *thdr + ((thdr->n_namesz + 3) & ~3) + ((thdr->n_descsz + 3) & ~3); - // making sure the offset is byte aligned - char *offset = reinterpret_cast(thdr + 1) + ((thdr->n_namesz + 3) & ~3); - - if (len < 0 || len > size) { - ::exit(1); - } - printf("size=%d, len=%d\n", size, len); - - prstatus_t pstat; - prstatus_t *ps; - prpsinfo_t infostat, *ist; - elf_gregset_t rinfo; - unsigned int j; - - switch (thdr->n_type) { - case NT_PRSTATUS: - ps = reinterpret_cast(offset); - memcpy(&pstat, ps, sizeof(prstatus_t)); - printf("\n*** printing registers****\n"); - for (j = 0; j < ELF_NGREG; j++) { - rinfo[j] = pstat.pr_reg[j]; - printf("%#x ", static_cast(rinfo[j])); - } - printf("\n"); - - printf("\n**** NT_PRSTATUS ****\n"); - - printf("Process id = %d\n", pstat.pr_pid); - printf("Parent Process id = %d\n", pstat.pr_ppid); - - printf("Signal that caused this core dump is signal = %d\n", pstat.pr_cursig); - - printf("stack pointer = %#x\n", static_cast(pstat.pr_reg[SP_REGNUM])); // UESP - framep = pstat.pr_reg[FP_REGNUM]; - pc = pstat.pr_reg[PC_REGNUM]; - printf("frame pointer = %#x\n", static_cast(pstat.pr_reg[FP_REGNUM])); // EBP - printf("program counter if no save = %#x\n", static_cast(pstat.pr_reg[PC_REGNUM])); - break; - - case NT_PRPSINFO: - ist = reinterpret_cast(offset); - memcpy(&infostat, ist, sizeof(prpsinfo_t)); - - if (is_debug_tag_set("note")) { - printf("\n**** NT_PRPSINFO of active process****\n"); - printf("process state = %c\n", infostat.pr_state); - printf("Name of the executable = %s\n", infostat.pr_fname); - printf("Arg List = %s\n", infostat.pr_psargs); - - printf("process id = %d\n", infostat.pr_pid); - } - break; - } - thdr = reinterpret_cast(reinterpret_cast(thdr) + len); - sum += len; - size -= len; - } - } - ats_free(nhdr); - } - } - } - } - framepointer = framep; - program_counter = pc; - - // Write your actual tests here - CoreUtils::find_stuff(&CoreUtils::test_HdrHeap); - CoreUtils::find_stuff(&CoreUtils::test_HttpSM); - - fclose(fp); -} -#endif - -#if !defined(linux) -void -process_core(char *fname) -{ - // do not make it fatal!!!! - Warning("Only supported on Sparc Solaris and Linux"); -} -#endif diff --git a/src/traffic_server/CoreUtils.h b/src/traffic_server/CoreUtils.h deleted file mode 100644 index b3d2a77135b..00000000000 --- a/src/traffic_server/CoreUtils.h +++ /dev/null @@ -1,198 +0,0 @@ -/** @file - - A brief file description - - @section license License - - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - */ - -/**************************************************************************** - - CoreUtils.h - - Description: Automated processing of core files on Sparc & Linux - ****************************************************************************/ - -#pragma once - -#if defined(linux) -#include -#include -#include -#include -#include -#include -#include - -#define SP_REGNUM 15 /* Contains address of top of stack USP */ -#define PC_REGNUM 12 /* Contains program counter EIP */ -#define FP_REGNUM 5 /* Virtual frame pointer EBP */ -#define NO_OF_ARGS \ - 10 /* The argument depth up to which we would be looking into \ - the stack */ - -// contains local and in registers, frame pointer, and stack base -struct core_stack_state { - intptr_t framep; // int stkbase; - intptr_t pc; - intptr_t arg[NO_OF_ARGS]; -}; -#endif // linux check - -#if defined(darwin) || defined(freebsd) || defined(solaris) || defined(openbsd) // FIXME: solaris x86 -#include -#include -#include -#include -#include - -#define NO_OF_ARGS \ - 10 /* The argument depth up to which we would be looking into \ - the stack */ - -// contains local and in registers, frame pointer, and stack base -struct core_stack_state { - intptr_t framep; // int stkbase; - intptr_t pc; - intptr_t arg[NO_OF_ARGS]; -}; -#endif /* darwin || freebsd || solaris */ - -// to be sorted by virtual address -struct memTable { - intptr_t vaddr; - intptr_t offset; - intptr_t fsize; -}; - -typedef void (*StuffTest_f)(void *); - -class HttpSM; -class HTTPHdr; -class HdrHeap; - -class EThread; -class UnixNetVConnection; -struct NetState; - -class CoreUtils -{ -public: - /********************************************************************** - * purpose: finds the index of the virtual address or finds what the - * index should be if the virtual address is not there - * inputs: int vaddr, int upper, int lower - * outputs: returns the index of the vaddr or the index after where it - * should be - * notes: upper and lower stands for the lowest and highest indices - * the function should search between - **********************************************************************/ - static intptr_t find_vaddr(intptr_t vaddr, intptr_t upper, intptr_t lower); - - /********************************************************************** - * purpose: function used to build the virtual address table - * inputs: int vaddr1, int offset1, int fsize1 - * outputs: none - **********************************************************************/ - static void insert_table(intptr_t vaddr1, intptr_t offset1, intptr_t fsize1); - - /********************************************************************** - * purpose: fills the buffer with num of given bytes from the - * beginning of the memory section - * inputs: int vaddr, int bytes, char* buf - * outputs: returns -1 on read error or num of bytes read - **********************************************************************/ - static intptr_t read_from_core(intptr_t vaddr, intptr_t bytes, char *buf); - -/********************************************************************** - * purpose: returns the base core_stack_state for the given thread id - * inputs: int threadId, core_stack_state* coress - * outputs: returns the base core_stack_state for the given thread id - **********************************************************************/ -#if defined(linux) - static void get_base_frame(intptr_t framep, core_stack_state *coress); -#endif - - /********************************************************************** - * purpose: returns the core_stack_state of the next frame up - * inputs: core_stack_state* coress - * outputs: returns 0 if current frame is already at the top of the stack - * or returns 1 and moves up the stack once - **********************************************************************/ - static int get_next_frame(core_stack_state *coress); - - /********************************************************************** - * purpose: loop ups over local & in registers on the stack and calls - * f on all of them - * inputs: none - * outputs: none - **********************************************************************/ - static void find_stuff(StuffTest_f f); - - /********************************************************************** - * purpose: tests whether a given register is an HttpSM - * inputs: reg_type t, int i, core_stack_state coress - * outputs: none - **********************************************************************/ - static void test_HttpSM(void *); - static void test_HttpSM_from_tunnel(void *); - - /********************************************************************** - * purpose: prints out info about a give HttpSM - * inputs: HttpSM* core_ptr (ptr to http_sm in core) - * outputs: none - **********************************************************************/ - static void process_HttpSM(HttpSM *core_ptr); - static void process_EThread(EThread *eth_test); - static void process_NetVC(UnixNetVConnection *eth_test); - - /********************************************************************** - * purpose: dumps the given state machine's history - * inputs: HttpSM* hsm - * outputs: none - **********************************************************************/ - static void dump_history(HttpSM *hsm); - - /********************************************************************** - * purpose: fills in the given HTTPHdr * live_hdr with live information - * take from core_hdr the core file - * inputs: HTTPHdr* core_hdr, HTTPHdr* live_hdr - * outputs: -1 on failure or total bytes in the header heaps - **********************************************************************/ - static int load_http_hdr(HTTPHdr *core_hdr, HTTPHdr *live_hdr); - - /********************************************************************** - * purpose: loads the http hdr from handle h in the core file - * inputs: HTTPHdr* h, char* name - * outputs: none - **********************************************************************/ - static void print_http_hdr(HTTPHdr *h, const char *name); - static void print_netstate(NetState *n); - - /********************************************************************** - * purpose: loads a null terminated string from the core file - * inputs: core file string address - * outputs: char* pointing to live string. call deallocs via ats_free() - ***********************************************************************/ - static char *load_string(const char *addr); - - static void test_HdrHeap(void *arg); -}; - -// parses the core file -void process_core(char *fname); diff --git a/src/traffic_server/FetchSM.cc b/src/traffic_server/FetchSM.cc index 9a01e1800fe..aa735978475 100644 --- a/src/traffic_server/FetchSM.cc +++ b/src/traffic_server/FetchSM.cc @@ -359,6 +359,7 @@ void FetchSM::get_info_from_buffer(IOBufferReader *reader) { char *buf, *info; + IOBufferBlock *blk; int64_t read_avail, read_done; if (!reader) { @@ -366,6 +367,10 @@ FetchSM::get_info_from_buffer(IOBufferReader *reader) return; } + /* Read the data out of the reader */ + if (reader->block != NULL) + reader->skip_empty_blocks(); + read_avail = reader->read_avail(); Debug(DEBUG_TAG, "[%s] total avail %" PRId64, __FUNCTION__, read_avail); if (!read_avail) { @@ -376,21 +381,44 @@ FetchSM::get_info_from_buffer(IOBufferReader *reader) info = (char *)ats_malloc(sizeof(char) * (read_avail + 1)); client_response = info; - // To maintain backwards compatibility we don't allow chunking when it's not streaming. - if (!(fetch_flags & TS_FETCH_FLAGS_STREAM) || !check_chunked()) { + blk = reader->block.get(); + + // This is the equivalent of TSIOBufferBlockReadStart() + buf = blk->start() + reader->start_offset; + read_done = blk->read_avail() - reader->start_offset; + + if (header_done == 0 && read_done > 0) { + int bytes_used = 0; + header_done = true; + if (client_response_hdr.parse_resp(&http_parser, reader, &bytes_used, 0) == PARSE_RESULT_DONE) { + if ((bytes_used > 0) && (bytes_used <= read_avail)) { + memcpy(info, buf, bytes_used); + info += bytes_used; + client_bytes += bytes_used; + } + } else { + Error("Failed to parse headers in FetchSM buffer"); + } + // adjust the read_avail + read_avail -= bytes_used; + } + + // Send the body without dechunk when neither streaming nor dechunk flag is set + // Or when the body is not chunked + if (!((fetch_flags & TS_FETCH_FLAGS_STREAM) || (fetch_flags & TS_FETCH_FLAGS_DECHUNK)) || !check_chunked()) { /* Read the data out of the reader */ while (read_avail > 0) { if (reader->block) { reader->skip_empty_blocks(); } - IOBufferBlock *blk = reader->block.get(); + blk = reader->block.get(); // This is the equivalent of TSIOBufferBlockReadStart() buf = blk->start() + reader->start_offset; read_done = blk->read_avail() - reader->start_offset; - if (read_done > 0) { + if ((read_done > 0) && ((read_done <= read_avail))) { memcpy(info, buf, read_done); reader->consume(read_done); read_avail -= read_done; @@ -425,7 +453,7 @@ FetchSM::get_info_from_buffer(IOBufferReader *reader) buf = blk->start() + reader->start_offset; read_done = blk->read_avail() - reader->start_offset; - if (read_done > 0) { + if ((read_done > 0) && (read_done <= read_avail)) { memcpy(info, buf, read_done); reader->consume(read_done); read_avail -= read_done; diff --git a/src/traffic_server/FetchSM.h b/src/traffic_server/FetchSM.h index 0e57e4ce8b4..984887ddc8e 100644 --- a/src/traffic_server/FetchSM.h +++ b/src/traffic_server/FetchSM.h @@ -61,7 +61,6 @@ class FetchSM : public Continuation callback_events = events; callback_options = options; _addr.assign(addr); - fetch_flags = TS_FETCH_FLAGS_DECHUNK; writeRequest(headers, length); mutex = new_ProxyMutex(); @@ -76,6 +75,12 @@ class FetchSM : public Continuation resp_buffer->water_mark = INT64_MAX; } + void + set_fetch_flags(int flags) + { + fetch_flags = flags; + } + int fetch_handler(int event, void *data); void process_fetch_read(int event); void process_fetch_write(int event); diff --git a/src/traffic_server/HostStatus.cc b/src/traffic_server/HostStatus.cc index 9c23103c180..ea4372209a5 100644 --- a/src/traffic_server/HostStatus.cc +++ b/src/traffic_server/HostStatus.cc @@ -405,10 +405,6 @@ HostStatus::getHostStatus(const char *name) } _status->reasons = reasons; } - // didn't find this host in host status db, create the record - if (!lookup) { - createHostStat(name); - } return _status; } diff --git a/src/traffic_server/InkAPI.cc b/src/traffic_server/InkAPI.cc index 7fa63d003c1..fb17310bb34 100644 --- a/src/traffic_server/InkAPI.cc +++ b/src/traffic_server/InkAPI.cc @@ -30,6 +30,7 @@ #include "tscore/ink_platform.h" #include "tscore/ink_base64.h" +#include "tscore/PluginUserArgs.h" #include "tscore/I_Layout.h" #include "tscore/I_Version.h" @@ -72,6 +73,7 @@ #include "records/I_RecCore.h" #include "I_Machine.h" #include "HttpProxyServerMain.h" +#include "shared/overridable_txn_vars.h" #include "ts/ts.h" @@ -102,22 +104,15 @@ static RecRawStatBlock *api_rsb; /** Reservation for a user arg. */ struct UserArg { - /// Types of user args. - enum Type { - TXN, ///< Transaction based. - SSN, ///< Session based - VCONN, ///< VConnection based - COUNT ///< Fake enum, # of valid entries. - }; - + TSUserArgType type; std::string name; ///< Name of reserving plugin. std::string description; ///< Description of use for this arg. }; -/// Table of reservations, indexed by type and then index. -UserArg UserArgTable[UserArg::Type::COUNT][TS_HTTP_MAX_USER_ARG]; -/// Table of next reserved index. -std::atomic UserArgIdx[UserArg::Type::COUNT]; +// Managing the user args tables, and the global storage (which is assumed to be the biggest, by far). +UserArg UserArgTable[TS_USER_ARGS_COUNT][MAX_USER_ARGS[TS_USER_ARGS_GLB]]; +static PluginUserArgs global_user_args; +std::atomic UserArgIdx[TS_USER_ARGS_COUNT]; // Table of next reserved index. /* URL schemes */ tsapi const char *TS_URL_SCHEME_FILE; @@ -1330,7 +1325,7 @@ APIHook::invoke(int event, void *edata) const ink_assert(!"not reached"); } } - MUTEX_TRY_LOCK(lock, m_cont->mutex, this_ethread()); + WEAK_MUTEX_TRY_LOCK(lock, m_cont->mutex, this_ethread()); if (!lock.is_locked()) { // If we cannot get the lock, the caller needs to restructure to handle rescheduling ink_release_assert(0); @@ -1426,16 +1421,21 @@ HttpHookState::is_enabled() void HttpHookState::Scope::init(HttpAPIHooks const *feature_hooks, TSHttpHookID id) { - APIHooks const *hooks = (*feature_hooks)[id]; + _hooks = (*feature_hooks)[id]; _p = nullptr; - _c = hooks->head(); + _c = _hooks->head(); } APIHook const * HttpHookState::Scope::candidate() { /// Simply returns _c hook for now. Later will do priority checking here + + // Check to see if a hook has been added since this was initialized empty + if (nullptr == _c && nullptr == _p && _hooks != nullptr) { + _c = _hooks->head(); + } return _c; } @@ -1449,6 +1449,7 @@ HttpHookState::Scope::operator++() void HttpHookState::Scope::clear() { + _hooks = nullptr; _p = _c = nullptr; } @@ -1965,6 +1966,23 @@ TSPluginRegister(const TSPluginRegistrationInfo *plugin_info) return TS_SUCCESS; } +TSReturnCode +TSPluginDSOReloadEnable(int enabled) +{ + TSReturnCode ret = TS_SUCCESS; + if (!plugin_reg_current) { + return TS_ERROR; + } + + if (!enabled) { + if (!PluginDso::loadedPlugins()->addPluginPathToDsoOptOutTable(plugin_reg_current->plugin_path)) { + ret = TS_ERROR; + } + } + + return ret; +} + //////////////////////////////////////////////////////////////////// // // API file management @@ -2738,7 +2756,8 @@ TSMimeHdrParse(TSMimeParser parser, TSMBuffer bufp, TSMLoc obj, const char **sta MIMEHdrImpl *mh = _hdr_mloc_to_mime_hdr_impl(obj); - return (TSParseResult)mime_parser_parse((MIMEParser *)parser, ((HdrHeapSDKHandle *)bufp)->m_heap, mh, start, end, false, false); + return (TSParseResult)mime_parser_parse((MIMEParser *)parser, ((HdrHeapSDKHandle *)bufp)->m_heap, mh, start, end, false, false, + false); } int @@ -4516,6 +4535,9 @@ TSContSchedule(TSCont contp, TSHRTime timeout) { sdk_assert(sdk_sanity_check_iocore_structure(contp) == TS_SUCCESS); + /* ensure we are on a EThread */ + sdk_assert(sdk_sanity_check_null_ptr((void *)this_ethread()) == TS_SUCCESS); + FORCE_PLUGIN_SCOPED_MUTEX(contp); INKContInternal *i = reinterpret_cast(contp); @@ -4526,7 +4548,8 @@ TSContSchedule(TSCont contp, TSHRTime timeout) EThread *eth = i->getThreadAffinity(); if (eth == nullptr) { - return nullptr; + eth = this_ethread(); + i->setThreadAffinity(eth); } TSAction action; @@ -4546,6 +4569,9 @@ TSContScheduleOnPool(TSCont contp, TSHRTime timeout, TSThreadPool tp) { sdk_assert(sdk_sanity_check_iocore_structure(contp) == TS_SUCCESS); + /* ensure we are on a EThread */ + sdk_assert(sdk_sanity_check_null_ptr((void *)this_ethread()) == TS_SUCCESS); + FORCE_PLUGIN_SCOPED_MUTEX(contp); INKContInternal *i = reinterpret_cast(contp); @@ -4563,15 +4589,9 @@ TSContScheduleOnPool(TSCont contp, TSHRTime timeout, TSThreadPool tp) case TS_THREAD_POOL_TASK: etype = ET_TASK; break; - case TS_THREAD_POOL_SSL: - etype = ET_TASK; // Should be ET_SSL - break; case TS_THREAD_POOL_DNS: etype = ET_DNS; break; - case TS_THREAD_POOL_REMAP: - etype = ET_TASK; // Should be ET_REMAP - break; case TS_THREAD_POOL_UDP: etype = ET_UDP; break; @@ -4629,6 +4649,9 @@ TSContScheduleEvery(TSCont contp, TSHRTime every /* millisecs */) { sdk_assert(sdk_sanity_check_iocore_structure(contp) == TS_SUCCESS); + /* ensure we are on a EThread */ + sdk_assert(sdk_sanity_check_null_ptr((void *)this_ethread()) == TS_SUCCESS); + FORCE_PLUGIN_SCOPED_MUTEX(contp); INKContInternal *i = reinterpret_cast(contp); @@ -4639,7 +4662,8 @@ TSContScheduleEvery(TSCont contp, TSHRTime every /* millisecs */) EThread *eth = i->getThreadAffinity(); if (eth == nullptr) { - return nullptr; + eth = this_ethread(); + i->setThreadAffinity(eth); } TSAction action = reinterpret_cast(eth->schedule_every(i, HRTIME_MSECONDS(every))); @@ -4654,6 +4678,9 @@ TSContScheduleEveryOnPool(TSCont contp, TSHRTime every, TSThreadPool tp) { sdk_assert(sdk_sanity_check_iocore_structure(contp) == TS_SUCCESS); + /* ensure we are on a EThread */ + sdk_assert(sdk_sanity_check_null_ptr((void *)this_ethread()) == TS_SUCCESS); + FORCE_PLUGIN_SCOPED_MUTEX(contp); INKContInternal *i = reinterpret_cast(contp); @@ -4785,7 +4812,7 @@ int TSContCall(TSCont contp, TSEvent event, void *edata) { Continuation *c = (Continuation *)contp; - MUTEX_TRY_LOCK(lock, c->mutex, this_ethread()); + WEAK_MUTEX_TRY_LOCK(lock, c->mutex, this_ethread()); if (!lock.is_locked()) { // If we cannot get the lock, the caller needs to restructure to handle rescheduling ink_release_assert(0); @@ -5725,7 +5752,7 @@ TSHttpTxnOutgoingAddrSet(TSHttpTxn txnp, const struct sockaddr *addr) sdk_assert(sdk_sanity_check_txn(txnp) == TS_SUCCESS); HttpSM *sm = (HttpSM *)txnp; - sm->ua_txn->set_outbound_port(ats_ip_port_host_order(addr)); + sm->ua_txn->upstream_outbound_options.outbound_port = ats_ip_port_host_order(addr); sm->ua_txn->set_outbound_ip(IpAddr(addr)); return TS_SUCCESS; } @@ -6087,45 +6114,46 @@ TSHttpTxnReenable(TSHttpTxn txnp, TSEvent event) // created using the ATS EThread API, eth will be NULL, and the // continuation needs to be called back on a REGULAR thread. // - // If this function is being executed on a thread created by the API - // which is DEDICATED, the continuation needs to be called back on a - // REGULAR thread. - if (eth == nullptr || eth->tt != REGULAR || !eth->is_event_type(ET_NET)) { - eventProcessor.schedule_imm(new TSHttpSMCallback(sm, event), ET_NET); - } else { + // If we are not coming from the thread associated with the state machine, + // reschedule. Also reschedule if we cannot get the state machine lock. + if (eth != nullptr && sm->getThreadAffinity() == eth) { MUTEX_TRY_LOCK(trylock, sm->mutex, eth); - if (!trylock.is_locked()) { - eventProcessor.schedule_imm(new TSHttpSMCallback(sm, event), ET_NET); - } else { + if (trylock.is_locked()) { ink_assert(eth->is_event_type(ET_NET)); sm->state_api_callback((int)event, nullptr); + return; } } + // Couldn't call the handler directly, schedule to the original SM thread + TSHttpSMCallback *cb = new TSHttpSMCallback(sm, event); + cb->setThreadAffinity(sm->getThreadAffinity()); + eventProcessor.schedule_imm(cb, ET_NET); } -TSReturnCode TSHttpArgIndexNameLookup(UserArg::Type type, const char *name, int *arg_idx, const char **description); +TSReturnCode TSUserArgIndexNameLookup(TSUserArgType type, const char *name, int *arg_idx, const char **description); TSReturnCode -TSHttpArgIndexReserve(UserArg::Type type, const char *name, const char *description, int *ptr_idx) +TSUserArgIndexReserve(TSUserArgType type, const char *name, const char *description, int *ptr_idx) { sdk_assert(sdk_sanity_check_null_ptr(ptr_idx) == TS_SUCCESS); sdk_assert(sdk_sanity_check_null_ptr(name) == TS_SUCCESS); - sdk_assert(0 <= type && type < UserArg::Type::COUNT); + sdk_assert(0 <= type && type < TS_USER_ARGS_COUNT); int idx; /* Since this function is meant to be called during plugin initialization we could end up "leaking" indices during plugins reload. - * Make sure we allocate 1 index per name, also current TSHttpArgIndexNameLookup() implementation assumes 1-1 relationship as + * Make sure we allocate 1 index per name, also current TSUserArgIndexNameLookup() implementation assumes 1-1 relationship as * well. */ const char *desc; - if (TS_SUCCESS == TSHttpArgIndexNameLookup(type, name, &idx, &desc)) { + + if (TS_SUCCESS == TSUserArgIndexNameLookup(type, name, &idx, &desc)) { // Found existing index. *ptr_idx = idx; return TS_SUCCESS; } idx = UserArgIdx[type]++; - int limit = (type == UserArg::Type::VCONN) ? TS_VCONN_MAX_USER_ARG : TS_HTTP_MAX_USER_ARG; + int limit = MAX_USER_ARGS[type]; if (idx < limit) { UserArg &arg(UserArgTable[type][idx]); @@ -6141,9 +6169,9 @@ TSHttpArgIndexReserve(UserArg::Type type, const char *name, const char *descript } TSReturnCode -TSHttpArgIndexLookup(UserArg::Type type, int idx, const char **name, const char **description) +TSUserArgIndexLookup(TSUserArgType type, int idx, const char **name, const char **description) { - sdk_assert(0 <= type && type < UserArg::Type::COUNT); + sdk_assert(0 <= type && type < TS_USER_ARGS_COUNT); if (sdk_sanity_check_null_ptr(name) == TS_SUCCESS) { if (idx < UserArgIdx[type]) { UserArg &arg(UserArgTable[type][idx]); @@ -6159,10 +6187,10 @@ TSHttpArgIndexLookup(UserArg::Type type, int idx, const char **name, const char // Not particularly efficient, but good enough for now. TSReturnCode -TSHttpArgIndexNameLookup(UserArg::Type type, const char *name, int *arg_idx, const char **description) +TSUserArgIndexNameLookup(TSUserArgType type, const char *name, int *arg_idx, const char **description) { sdk_assert(sdk_sanity_check_null_ptr(arg_idx) == TS_SUCCESS); - sdk_assert(0 <= type && type < UserArg::Type::COUNT); + sdk_assert(0 <= type && type < TS_USER_ARGS_COUNT); std::string_view n{name}; @@ -6178,86 +6206,114 @@ TSHttpArgIndexNameLookup(UserArg::Type type, const char *name, int *arg_idx, con return TS_ERROR; } +// ------------- +void +TSUserArgSet(void *data, int arg_idx, void *arg) +{ + if (nullptr != data) { + PluginUserArgsMixin *user_args = dynamic_cast(static_cast(data)); + sdk_assert(user_args); + + user_args->set_user_arg(arg_idx, arg); + } else { + global_user_args.set_user_arg(arg_idx, arg); + } +} + +void * +TSUserArgGet(void *data, int arg_idx) +{ + if (nullptr != data) { + PluginUserArgsMixin *user_args = dynamic_cast(static_cast(data)); + sdk_assert(user_args); + + return user_args->get_user_arg(arg_idx); + } else { + return global_user_args.get_user_arg(arg_idx); + } +} + // ------------- TSReturnCode TSHttpTxnArgIndexReserve(const char *name, const char *description, int *arg_idx) { - return TSHttpArgIndexReserve(UserArg::TXN, name, description, arg_idx); + return TSUserArgIndexReserve(TS_USER_ARGS_TXN, name, description, arg_idx); } TSReturnCode TSHttpTxnArgIndexLookup(int arg_idx, const char **name, const char **description) { - return TSHttpArgIndexLookup(UserArg::TXN, arg_idx, name, description); + return TSUserArgIndexLookup(TS_USER_ARGS_TXN, arg_idx, name, description); } TSReturnCode TSHttpTxnArgIndexNameLookup(const char *name, int *arg_idx, const char **description) { - return TSHttpArgIndexNameLookup(UserArg::TXN, name, arg_idx, description); + return TSUserArgIndexNameLookup(TS_USER_ARGS_TXN, name, arg_idx, description); } TSReturnCode TSHttpSsnArgIndexReserve(const char *name, const char *description, int *arg_idx) { - return TSHttpArgIndexReserve(UserArg::SSN, name, description, arg_idx); + return TSUserArgIndexReserve(TS_USER_ARGS_SSN, name, description, arg_idx); } TSReturnCode TSHttpSsnArgIndexLookup(int arg_idx, const char **name, const char **description) { - return TSHttpArgIndexLookup(UserArg::SSN, arg_idx, name, description); + return TSUserArgIndexLookup(TS_USER_ARGS_SSN, arg_idx, name, description); } TSReturnCode TSHttpSsnArgIndexNameLookup(const char *name, int *arg_idx, const char **description) { - return TSHttpArgIndexNameLookup(UserArg::SSN, name, arg_idx, description); + return TSUserArgIndexNameLookup(TS_USER_ARGS_SSN, name, arg_idx, description); } TSReturnCode TSVConnArgIndexReserve(const char *name, const char *description, int *arg_idx) { - return TSHttpArgIndexReserve(UserArg::VCONN, name, description, arg_idx); + return TSUserArgIndexReserve(TS_USER_ARGS_VCONN, name, description, arg_idx); } TSReturnCode TSVConnArgIndexLookup(int arg_idx, const char **name, const char **description) { - return TSHttpArgIndexLookup(UserArg::VCONN, arg_idx, name, description); + return TSUserArgIndexLookup(TS_USER_ARGS_VCONN, arg_idx, name, description); } TSReturnCode TSVConnArgIndexNameLookup(const char *name, int *arg_idx, const char **description) { - return TSHttpArgIndexNameLookup(UserArg::VCONN, name, arg_idx, description); + return TSUserArgIndexNameLookup(TS_USER_ARGS_VCONN, name, arg_idx, description); } void TSHttpTxnArgSet(TSHttpTxn txnp, int arg_idx, void *arg) { sdk_assert(sdk_sanity_check_txn(txnp) == TS_SUCCESS); - sdk_assert(arg_idx >= 0 && arg_idx < TS_HTTP_MAX_USER_ARG); + sdk_assert(arg_idx >= 0 && static_cast(arg_idx) < MAX_USER_ARGS[TS_USER_ARGS_TXN]); - HttpSM *sm = reinterpret_cast(txnp); - sm->t_state.user_args[arg_idx] = arg; + HttpSM *sm = reinterpret_cast(txnp); + + sm->set_user_arg(arg_idx, arg); } void * TSHttpTxnArgGet(TSHttpTxn txnp, int arg_idx) { sdk_assert(sdk_sanity_check_txn(txnp) == TS_SUCCESS); - sdk_assert(arg_idx >= 0 && arg_idx < TS_HTTP_MAX_USER_ARG); + sdk_assert(arg_idx >= 0 && static_cast(arg_idx) < MAX_USER_ARGS[TS_USER_ARGS_TXN]); HttpSM *sm = reinterpret_cast(txnp); - return sm->t_state.user_args[arg_idx]; + return sm->get_user_arg(arg_idx); } void TSHttpSsnArgSet(TSHttpSsn ssnp, int arg_idx, void *arg) { sdk_assert(sdk_sanity_check_http_ssn(ssnp) == TS_SUCCESS); - sdk_assert(arg_idx >= 0 && arg_idx < TS_HTTP_MAX_USER_ARG); + sdk_assert(arg_idx >= 0 && static_cast(arg_idx) < MAX_USER_ARGS[TS_USER_ARGS_SSN]); ProxySession *cs = reinterpret_cast(ssnp); @@ -6268,7 +6324,7 @@ void * TSHttpSsnArgGet(TSHttpSsn ssnp, int arg_idx) { sdk_assert(sdk_sanity_check_http_ssn(ssnp) == TS_SUCCESS); - sdk_assert(arg_idx >= 0 && arg_idx < TS_HTTP_MAX_USER_ARG); + sdk_assert(arg_idx >= 0 && static_cast(arg_idx) < MAX_USER_ARGS[TS_USER_ARGS_SSN]); ProxySession *cs = reinterpret_cast(ssnp); return cs->get_user_arg(arg_idx); @@ -6278,19 +6334,22 @@ void TSVConnArgSet(TSVConn connp, int arg_idx, void *arg) { sdk_assert(sdk_sanity_check_iocore_structure(connp) == TS_SUCCESS); - sdk_assert(arg_idx >= 0 && arg_idx < TS_VCONN_MAX_USER_ARG); - AnnotatedVConnection *avc = reinterpret_cast(connp); - avc->set_user_arg(arg_idx, arg); + sdk_assert(arg_idx >= 0 && static_cast(arg_idx) < MAX_USER_ARGS[TS_USER_ARGS_VCONN]); + PluginUserArgsMixin *user_args = dynamic_cast(reinterpret_cast(connp)); + sdk_assert(user_args); + + user_args->set_user_arg(arg_idx, arg); } void * TSVConnArgGet(TSVConn connp, int arg_idx) { sdk_assert(sdk_sanity_check_iocore_structure(connp) == TS_SUCCESS); - sdk_assert(arg_idx >= 0 && arg_idx < TS_VCONN_MAX_USER_ARG); + sdk_assert(arg_idx >= 0 && static_cast(arg_idx) < MAX_USER_ARGS[TS_USER_ARGS_VCONN]); + PluginUserArgsMixin *user_args = dynamic_cast(reinterpret_cast(connp)); + sdk_assert(user_args); - AnnotatedVConnection *avc = reinterpret_cast(connp); - return avc->get_user_arg(arg_idx); + return user_args->get_user_arg(arg_idx); } void @@ -6487,6 +6546,42 @@ TSHttpTxnClientRespBodyBytesGet(TSHttpTxn txnp) return sm->client_response_body_bytes; } +int +TSVConnIsSslReused(TSVConn sslp) +{ + NetVConnection *vc = reinterpret_cast(sslp); + SSLNetVConnection *ssl_vc = dynamic_cast(vc); + + return ssl_vc ? ssl_vc->getSSLSessionCacheHit() : 0; +} + +const char * +TSVConnSslCipherGet(TSVConn sslp) +{ + NetVConnection *vc = reinterpret_cast(sslp); + SSLNetVConnection *ssl_vc = dynamic_cast(vc); + + return ssl_vc ? ssl_vc->getSSLCipherSuite() : nullptr; +} + +const char * +TSVConnSslProtocolGet(TSVConn sslp) +{ + NetVConnection *vc = reinterpret_cast(sslp); + SSLNetVConnection *ssl_vc = dynamic_cast(vc); + + return ssl_vc ? ssl_vc->getSSLProtocol() : nullptr; +} + +const char * +TSVConnSslCurveGet(TSVConn sslp) +{ + NetVConnection *vc = reinterpret_cast(sslp); + SSLNetVConnection *ssl_vc = dynamic_cast(vc); + + return ssl_vc ? ssl_vc->getSSLCurve() : nullptr; +} + int TSHttpTxnPushedRespHdrBytesGet(TSHttpTxn txnp) { @@ -6741,19 +6836,28 @@ TSHttpConnectTransparent(sockaddr const *client_addr, sockaddr const *server_add void TSActionCancel(TSAction actionp) { - Action *a; + Action *thisaction; INKContInternal *i; + // Nothing to cancel + if (actionp == nullptr) { + return; + } + /* This is a hack. Should be handled in ink_types */ if ((uintptr_t)actionp & 0x1) { - a = (Action *)((uintptr_t)actionp - 1); - i = (INKContInternal *)a->continuation; - i->handle_event_count(EVENT_IMMEDIATE); + thisaction = (Action *)((uintptr_t)actionp - 1); + if (thisaction) { + i = (INKContInternal *)thisaction->continuation; + i->handle_event_count(EVENT_IMMEDIATE); + } else { // The action pointer for an INKContInternal was effectively null, just go away + return; + } } else { - a = (Action *)actionp; + thisaction = (Action *)actionp; } - a->cancel(); + thisaction->cancel(); } // Currently no error handling necessary, actionp can be anything. @@ -7374,7 +7478,7 @@ TSStatFindName(const char *name, int *idp) sdk_assert(sdk_sanity_check_null_ptr((void *)name) == TS_SUCCESS); - if (RecGetRecordOrderAndId(name, nullptr, &id) != REC_ERR_OKAY) { + if (RecGetRecordOrderAndId(name, nullptr, &id, true, true) != REC_ERR_OKAY) { return TS_ERROR; } @@ -7411,7 +7515,7 @@ TSIsDebugTagSet(const char *t) void TSDebugSpecific(int debug_flag, const char *tag, const char *format_str, ...) { - if (is_debug_tag_set(tag) || (debug_flag && diags->on())) { + if ((debug_flag && diags->on()) || is_debug_tag_set(tag)) { va_list ap; va_start(ap, format_str); @@ -7450,7 +7554,7 @@ TSTextLogObjectCreate(const char *filename, int mode, TSTextLogObject *new_objec TextLogObject *tlog = new TextLogObject( filename, Log::config->logfile_dir, (bool)mode & TS_LOG_MODE_ADD_TIMESTAMP, nullptr, Log::config->rolling_enabled, Log::config->preproc_threads, Log::config->rolling_interval_sec, Log::config->rolling_offset_hr, Log::config->rolling_size_mb, - Log::config->rolling_max_count, Log::config->rolling_allow_empty); + Log::config->rolling_max_count, Log::config->rolling_min_count, Log::config->rolling_allow_empty); if (tlog == nullptr) { *new_object = nullptr; return TS_ERROR; @@ -7747,34 +7851,6 @@ TSCacheHttpInfoSizeSet(TSCacheHttpInfo infop, int64_t size) info->object_size_set(size); } -// This API tells the core to follow normal (301/302) redirects using the -// standard Location: URL. This does not need to be called if you set an -// explicit URL using TSHttpTxnRedirectUrlSet(). -TSReturnCode -TSHttpTxnFollowRedirect(TSHttpTxn txnp, int on) -{ - sdk_assert(sdk_sanity_check_txn(txnp) == TS_SUCCESS); - - HttpSM *sm = (HttpSM *)txnp; - - // This is necessary since we might not have setup these overridable configurations - sm->t_state.setup_per_txn_configs(); - - if (on) { - sm->redirection_tries = 0; - sm->enable_redirection = true; - // Make sure we allow for at least one redirection. - if (sm->t_state.txn_conf->number_of_redirections <= 0) { - sm->t_state.txn_conf->number_of_redirections = 1; - } - } else { - sm->enable_redirection = false; - sm->t_state.txn_conf->number_of_redirections = 0; - } - - return TS_SUCCESS; -} - // this function should be called at TS_EVENT_HTTP_READ_RESPONSE_HDR void TSHttpTxnRedirectUrlSet(TSHttpTxn txnp, const char *url, const int url_len) @@ -7798,7 +7874,7 @@ TSHttpTxnRedirectUrlSet(TSHttpTxn txnp, const char *url, const int url_len) // Make sure we allow for at least one redirection. if (sm->t_state.txn_conf->number_of_redirections <= 0) { sm->t_state.setup_per_txn_configs(); - sm->t_state.txn_conf->number_of_redirections = 1; + sm->t_state.my_txn_conf().number_of_redirections = 1; } } @@ -7867,7 +7943,7 @@ TSFetchPages(TSFetchUrlParams_t *params) } } -void +TSFetchSM TSFetchUrl(const char *headers, int request_len, sockaddr const *ip, TSCont contp, TSFetchWakeUpOptions callback_options, TSFetchEvent events) { @@ -7879,6 +7955,15 @@ TSFetchUrl(const char *headers, int request_len, sockaddr const *ip, TSCont cont fetch_sm->init((Continuation *)contp, callback_options, events, headers, request_len, ip); fetch_sm->httpConnect(); + + return reinterpret_cast(fetch_sm); +} + +void +TSFetchFlagSet(TSFetchSM fetch_sm, int flags) +{ + sdk_assert(sdk_sanity_check_fetch_sm(fetch_sm) == TS_SUCCESS); + (reinterpret_cast(fetch_sm))->set_fetch_flags(flags); } TSFetchSM @@ -7990,7 +8075,7 @@ TSHttpTxnIsInternal(TSHttpTxn txnp) return TSHttpSsnIsInternal(TSHttpTxnSsnGet(txnp)); } -void +TSReturnCode TSHttpTxnServerPush(TSHttpTxn txnp, const char *url, int url_len) { sdk_assert(sdk_sanity_check_txn(txnp) == TS_SUCCESS); @@ -7999,26 +8084,37 @@ TSHttpTxnServerPush(TSHttpTxn txnp, const char *url, int url_len) url_obj.create(nullptr); if (url_obj.parse(url, url_len) == PARSE_RESULT_ERROR) { url_obj.destroy(); - return; + return TS_ERROR; } HttpSM *sm = reinterpret_cast(txnp); Http2Stream *stream = dynamic_cast(sm->ua_txn); - if (stream) { - Http2ClientSession *ua_session = static_cast(stream->get_proxy_ssn()); - SCOPED_MUTEX_LOCK(lock, ua_session->mutex, this_ethread()); - if (!ua_session->connection_state.is_state_closed() && !ua_session->is_url_pushed(url, url_len)) { - HTTPHdr *hptr = &(sm->t_state.hdr_info.client_request); - TSMLoc obj = reinterpret_cast(hptr->m_http); - - MIMEHdrImpl *mh = _hdr_mloc_to_mime_hdr_impl(obj); - MIMEField *f = mime_hdr_field_find(mh, MIME_FIELD_ACCEPT_ENCODING, MIME_LEN_ACCEPT_ENCODING); - stream->push_promise(url_obj, f); - - ua_session->add_url_to_pushed_table(url, url_len); - } + if (stream == nullptr) { + url_obj.destroy(); + return TS_ERROR; + } + + Http2ClientSession *ua_session = static_cast(stream->get_proxy_ssn()); + SCOPED_MUTEX_LOCK(lock, ua_session->mutex, this_ethread()); + if (ua_session->connection_state.is_state_closed() || ua_session->is_url_pushed(url, url_len)) { + url_obj.destroy(); + return TS_ERROR; } + + HTTPHdr *hptr = &(sm->t_state.hdr_info.client_request); + TSMLoc obj = reinterpret_cast(hptr->m_http); + + MIMEHdrImpl *mh = _hdr_mloc_to_mime_hdr_impl(obj); + MIMEField *f = mime_hdr_field_find(mh, MIME_FIELD_ACCEPT_ENCODING, MIME_LEN_ACCEPT_ENCODING); + if (!stream->push_promise(url_obj, f)) { + url_obj.destroy(); + return TS_ERROR; + } + + ua_session->add_url_to_pushed_table(url, url_len); + url_obj.destroy(); + return TS_SUCCESS; } TSReturnCode @@ -8148,7 +8244,7 @@ _memberp_to_generic(T *ptr, MgmtConverter const *&conv) inline void * _memberp_to_generic(MgmtInt *ptr, MgmtConverter const *&conv) { - static const MgmtConverter converter([](void *data) -> MgmtInt { return *static_cast(data); }, + static const MgmtConverter converter([](const void *data) -> MgmtInt { return *static_cast(data); }, [](void *data, MgmtInt i) -> void { *static_cast(data) = i; }); conv = &converter; @@ -8159,7 +8255,7 @@ _memberp_to_generic(MgmtInt *ptr, MgmtConverter const *&conv) inline void * _memberp_to_generic(MgmtByte *ptr, MgmtConverter const *&conv) { - static const MgmtConverter converter{[](void *data) -> MgmtInt { return *static_cast(data); }, + static const MgmtConverter converter{[](const void *data) -> MgmtInt { return *static_cast(data); }, [](void *data, MgmtInt i) -> void { *static_cast(data) = i; }}; conv = &converter; @@ -8170,7 +8266,7 @@ _memberp_to_generic(MgmtByte *ptr, MgmtConverter const *&conv) inline void * _memberp_to_generic(MgmtFloat *ptr, MgmtConverter const *&conv) { - static const MgmtConverter converter{[](void *data) -> MgmtFloat { return *static_cast(data); }, + static const MgmtConverter converter{[](const void *data) -> MgmtFloat { return *static_cast(data); }, [](void *data, MgmtFloat f) -> void { *static_cast(data) = f; }}; conv = &converter; @@ -8183,8 +8279,9 @@ template inline auto _memberp_to_generic(MgmtFloat *ptr, MgmtConverter const *&conv) -> typename std::enable_if::value, void *>::type { - static const MgmtConverter converter{[](void *data) -> MgmtInt { return static_cast(*static_cast(data)); }, - [](void *data, MgmtInt i) -> void { *static_cast(data) = static_cast(i); }}; + static const MgmtConverter converter{ + [](const void *data) -> MgmtInt { return static_cast(*static_cast(data)); }, + [](void *data, MgmtInt i) -> void { *static_cast(data) = static_cast(i); }}; conv = &converter; return ptr; @@ -8540,6 +8637,10 @@ _conf_to_memberp(TSOverridableConfigKey conf, OverridableHttpConfigParams *overr ret = &overridableHttpConfig->outbound_conntrack.match; conv = &OutboundConnTrack::MATCH_CONV; break; + case TS_CONFIG_HTTP_HOST_RESOLUTION_PREFERENCE: + ret = &overridableHttpConfig->host_res_data; + conv = &HttpTransact::HOST_RES_CONV; + break; // This helps avoiding compiler warnings, yet detect unhandled enum members. case TS_CONFIG_NULL: case TS_CONFIG_LAST_ENTRY: @@ -8549,6 +8650,13 @@ _conf_to_memberp(TSOverridableConfigKey conf, OverridableHttpConfigParams *overr return ret; } +// 2nd little helper function to find the struct member for getting. +static const void * +_conf_to_memberp(TSOverridableConfigKey conf, const OverridableHttpConfigParams *overridableHttpConfig, MgmtConverter const *&conv) +{ + return _conf_to_memberp(conf, const_cast(overridableHttpConfig), conv); +} + /* APIs to manipulate the overridable configuration options. */ TSReturnCode @@ -8561,7 +8669,7 @@ TSHttpTxnConfigIntSet(TSHttpTxn txnp, TSOverridableConfigKey conf, TSMgmtInt val s->t_state.setup_per_txn_configs(); - void *dest = _conf_to_memberp(conf, s->t_state.txn_conf, conv); + void *dest = _conf_to_memberp(conf, &(s->t_state.my_txn_conf()), conv); if (!dest || !conv->store_int) { return TS_ERROR; @@ -8580,7 +8688,7 @@ TSHttpTxnConfigIntGet(TSHttpTxn txnp, TSOverridableConfigKey conf, TSMgmtInt *va HttpSM *s = reinterpret_cast(txnp); MgmtConverter const *conv; - void *src = _conf_to_memberp(conf, s->t_state.txn_conf, conv); + const void *src = _conf_to_memberp(conf, s->t_state.txn_conf, conv); if (!src || !conv->load_int) { return TS_ERROR; @@ -8601,7 +8709,7 @@ TSHttpTxnConfigFloatSet(TSHttpTxn txnp, TSOverridableConfigKey conf, TSMgmtFloat s->t_state.setup_per_txn_configs(); - void *dest = _conf_to_memberp(conf, s->t_state.txn_conf, conv); + void *dest = _conf_to_memberp(conf, &(s->t_state.my_txn_conf()), conv); if (!dest || !conv->store_float) { return TS_ERROR; @@ -8619,7 +8727,7 @@ TSHttpTxnConfigFloatGet(TSHttpTxn txnp, TSOverridableConfigKey conf, TSMgmtFloat sdk_assert(sdk_sanity_check_null_ptr(static_cast(value)) == TS_SUCCESS); MgmtConverter const *conv; - void *src = _conf_to_memberp(conf, reinterpret_cast(txnp)->t_state.txn_conf, conv); + const void *src = _conf_to_memberp(conf, reinterpret_cast(txnp)->t_state.txn_conf, conv); if (!src || !conv->load_float) { return TS_ERROR; @@ -8645,29 +8753,29 @@ TSHttpTxnConfigStringSet(TSHttpTxn txnp, TSOverridableConfigKey conf, const char switch (conf) { case TS_CONFIG_HTTP_RESPONSE_SERVER_STR: if (value && length > 0) { - s->t_state.txn_conf->proxy_response_server_string = const_cast(value); // The "core" likes non-const char* - s->t_state.txn_conf->proxy_response_server_string_len = length; + s->t_state.my_txn_conf().proxy_response_server_string = const_cast(value); // The "core" likes non-const char* + s->t_state.my_txn_conf().proxy_response_server_string_len = length; } else { - s->t_state.txn_conf->proxy_response_server_string = nullptr; - s->t_state.txn_conf->proxy_response_server_string_len = 0; + s->t_state.my_txn_conf().proxy_response_server_string = nullptr; + s->t_state.my_txn_conf().proxy_response_server_string_len = 0; } break; case TS_CONFIG_HTTP_GLOBAL_USER_AGENT_HEADER: if (value && length > 0) { - s->t_state.txn_conf->global_user_agent_header = const_cast(value); // The "core" likes non-const char* - s->t_state.txn_conf->global_user_agent_header_size = length; + s->t_state.my_txn_conf().global_user_agent_header = const_cast(value); // The "core" likes non-const char* + s->t_state.my_txn_conf().global_user_agent_header_size = length; } else { - s->t_state.txn_conf->global_user_agent_header = nullptr; - s->t_state.txn_conf->global_user_agent_header_size = 0; + s->t_state.my_txn_conf().global_user_agent_header = nullptr; + s->t_state.my_txn_conf().global_user_agent_header_size = 0; } break; case TS_CONFIG_BODY_FACTORY_TEMPLATE_BASE: if (value && length > 0) { - s->t_state.txn_conf->body_factory_template_base = const_cast(value); - s->t_state.txn_conf->body_factory_template_base_len = length; + s->t_state.my_txn_conf().body_factory_template_base = const_cast(value); + s->t_state.my_txn_conf().body_factory_template_base_len = length; } else { - s->t_state.txn_conf->body_factory_template_base = nullptr; - s->t_state.txn_conf->body_factory_template_base_len = 0; + s->t_state.my_txn_conf().body_factory_template_base = nullptr; + s->t_state.my_txn_conf().body_factory_template_base_len = 0; } break; case TS_CONFIG_HTTP_INSERT_FORWARDED: @@ -8675,48 +8783,59 @@ TSHttpTxnConfigStringSet(TSHttpTxn txnp, TSOverridableConfigKey conf, const char ts::LocalBufferWriter<1024> error; HttpForwarded::OptionBitSet bs = HttpForwarded::optStrToBitset(std::string_view(value, length), error); if (!error.size()) { - s->t_state.txn_conf->insert_forwarded = bs; + s->t_state.my_txn_conf().insert_forwarded = bs; } else { Error("HTTP %.*s", static_cast(error.size()), error.data()); } } break; + case TS_CONFIG_HTTP_SERVER_SESSION_SHARING_MATCH: + if (value && length > 0) { + HttpConfig::load_server_session_sharing_match(value, s->t_state.my_txn_conf().server_session_sharing_match); + s->t_state.my_txn_conf().server_session_sharing_match_str = const_cast(value); + } + break; case TS_CONFIG_SSL_CLIENT_VERIFY_SERVER_POLICY: if (value && length > 0) { - s->t_state.txn_conf->ssl_client_verify_server_policy = const_cast(value); + s->t_state.my_txn_conf().ssl_client_verify_server_policy = const_cast(value); } break; case TS_CONFIG_SSL_CLIENT_VERIFY_SERVER_PROPERTIES: if (value && length > 0) { - s->t_state.txn_conf->ssl_client_verify_server_properties = const_cast(value); + s->t_state.my_txn_conf().ssl_client_verify_server_properties = const_cast(value); } break; case TS_CONFIG_SSL_CLIENT_SNI_POLICY: if (value && length > 0) { - s->t_state.txn_conf->ssl_client_sni_policy = const_cast(value); + s->t_state.my_txn_conf().ssl_client_sni_policy = const_cast(value); } break; case TS_CONFIG_SSL_CLIENT_CERT_FILENAME: if (value && length > 0) { - s->t_state.txn_conf->ssl_client_cert_filename = const_cast(value); + s->t_state.my_txn_conf().ssl_client_cert_filename = const_cast(value); } break; case TS_CONFIG_SSL_CLIENT_PRIVATE_KEY_FILENAME: if (value && length > 0) { - s->t_state.txn_conf->ssl_client_private_key_filename = const_cast(value); + s->t_state.my_txn_conf().ssl_client_private_key_filename = const_cast(value); } break; case TS_CONFIG_SSL_CLIENT_CA_CERT_FILENAME: if (value && length > 0) { - s->t_state.txn_conf->ssl_client_ca_cert_filename = const_cast(value); + s->t_state.my_txn_conf().ssl_client_ca_cert_filename = const_cast(value); } break; case TS_CONFIG_SSL_CERT_FILEPATH: /* noop */ break; + case TS_CONFIG_HTTP_HOST_RESOLUTION_PREFERENCE: + if (value && length > 0) { + s->t_state.my_txn_conf().host_res_data.conf_value = const_cast(value); + } + [[fallthrough]]; default: { MgmtConverter const *conv; - void *dest = _conf_to_memberp(conf, s->t_state.txn_conf, conv); + void *dest = _conf_to_memberp(conf, &(s->t_state.my_txn_conf()), conv); if (dest != nullptr && conv != nullptr && conv->store_string) { conv->store_string(dest, std::string_view(value, length)); } else { @@ -8751,9 +8870,13 @@ TSHttpTxnConfigStringGet(TSHttpTxn txnp, TSOverridableConfigKey conf, const char *value = sm->t_state.txn_conf->body_factory_template_base; *length = sm->t_state.txn_conf->body_factory_template_base_len; break; + case TS_CONFIG_HTTP_SERVER_SESSION_SHARING_MATCH: + *value = sm->t_state.txn_conf->server_session_sharing_match_str; + *length = *value ? strlen(*value) : 0; + break; default: { MgmtConverter const *conv; - void *src = _conf_to_memberp(conf, sm->t_state.txn_conf, conv); + const void *src = _conf_to_memberp(conf, sm->t_state.txn_conf, conv); if (src != nullptr && conv != nullptr && conv->load_string) { auto sv = conv->load_string(src); *value = sv.data(); @@ -8768,142 +8891,6 @@ TSHttpTxnConfigStringGet(TSHttpTxn txnp, TSOverridableConfigKey conf, const char return TS_SUCCESS; } -// This is a map of all overridable configurations, with the data type -static const std::unordered_map> Overridable_Map( - {{"proxy.config.srv_enabled", {TS_CONFIG_SRV_ENABLED, TS_RECORDDATATYPE_INT}}, - {"proxy.config.http.cache.http", {TS_CONFIG_HTTP_CACHE_HTTP, TS_RECORDDATATYPE_INT}}, - {"proxy.config.ssl.hsts_max_age", {TS_CONFIG_SSL_HSTS_MAX_AGE, TS_RECORDDATATYPE_INT}}, - {"proxy.config.http.normalize_ae", {TS_CONFIG_HTTP_NORMALIZE_AE, TS_RECORDDATATYPE_INT}}, - {"proxy.config.http.chunking.size", {TS_CONFIG_HTTP_CHUNKING_SIZE, TS_RECORDDATATYPE_INT}}, - {"proxy.config.ssl.client.cert.path", {TS_CONFIG_SSL_CERT_FILEPATH, TS_RECORDDATATYPE_STRING}}, - {"proxy.config.http.allow_half_open", {TS_CONFIG_HTTP_ALLOW_HALF_OPEN, TS_RECORDDATATYPE_INT}}, - {"proxy.config.http.chunking_enabled", {TS_CONFIG_HTTP_CHUNKING_ENABLED, TS_RECORDDATATYPE_INT}}, - {"proxy.config.http.cache.generation", {TS_CONFIG_HTTP_CACHE_GENERATION, TS_RECORDDATATYPE_INT}}, - {"proxy.config.http.insert_client_ip", {TS_CONFIG_HTTP_ANONYMIZE_INSERT_CLIENT_IP, TS_RECORDDATATYPE_INT}}, - {"proxy.config.http.insert_forwarded", {TS_CONFIG_HTTP_INSERT_FORWARDED, TS_RECORDDATATYPE_STRING}}, - {"proxy.config.http.cache.range.write", {TS_CONFIG_HTTP_CACHE_RANGE_WRITE, TS_RECORDDATATYPE_INT}}, - {"proxy.config.http.allow_multi_range", {TS_CONFIG_HTTP_ALLOW_MULTI_RANGE, TS_RECORDDATATYPE_INT}}, - {"proxy.config.http.cache.range.lookup", {TS_CONFIG_HTTP_CACHE_RANGE_LOOKUP, TS_RECORDDATATYPE_INT}}, - {"proxy.config.net.sock_packet_tos_out", {TS_CONFIG_NET_SOCK_PACKET_TOS_OUT, TS_RECORDDATATYPE_INT}}, - {"proxy.config.http.slow.log.threshold", {TS_CONFIG_HTTP_SLOW_LOG_THRESHOLD, TS_RECORDDATATYPE_INT}}, - {"proxy.config.http.cache.max_stale_age", {TS_CONFIG_HTTP_CACHE_MAX_STALE_AGE, TS_RECORDDATATYPE_INT}}, - {"proxy.config.http.default_buffer_size", {TS_CONFIG_HTTP_DEFAULT_BUFFER_SIZE, TS_RECORDDATATYPE_INT}}, - {"proxy.config.http.response_server_str", {TS_CONFIG_HTTP_RESPONSE_SERVER_STR, TS_RECORDDATATYPE_STRING}}, - {"proxy.config.http.keep_alive_post_out", {TS_CONFIG_HTTP_KEEP_ALIVE_POST_OUT, TS_RECORDDATATYPE_INT}}, - {"proxy.config.net.sock_option_flag_out", {TS_CONFIG_NET_SOCK_OPTION_FLAG_OUT, TS_RECORDDATATYPE_INT}}, - {"proxy.config.net.sock_packet_mark_out", {TS_CONFIG_NET_SOCK_PACKET_MARK_OUT, TS_RECORDDATATYPE_INT}}, - {"proxy.config.websocket.active_timeout", {TS_CONFIG_WEBSOCKET_ACTIVE_TIMEOUT, TS_RECORDDATATYPE_INT}}, - {"proxy.config.http.flow_control.enabled", {TS_CONFIG_HTTP_FLOW_CONTROL_ENABLED, TS_RECORDDATATYPE_INT}}, - {"proxy.config.http.send_http11_requests", {TS_CONFIG_HTTP_SEND_HTTP11_REQUESTS, TS_RECORDDATATYPE_INT}}, - {"proxy.config.body_factory.template_base", {TS_CONFIG_BODY_FACTORY_TEMPLATE_BASE, TS_RECORDDATATYPE_STRING}}, - {"proxy.config.http.anonymize_remove_from", {TS_CONFIG_HTTP_ANONYMIZE_REMOVE_FROM, TS_RECORDDATATYPE_INT}}, - {"proxy.config.http.keep_alive_enabled_in", {TS_CONFIG_HTTP_KEEP_ALIVE_ENABLED_IN, TS_RECORDDATATYPE_INT}}, - {"proxy.config.http.doc_in_cache_skip_dns", {TS_CONFIG_HTTP_DOC_IN_CACHE_SKIP_DNS, TS_RECORDDATATYPE_INT}}, - {"proxy.config.http.forward_connect_method", {TS_CONFIG_HTTP_FORWARD_CONNECT_METHOD, TS_RECORDDATATYPE_INT}}, - {"proxy.config.http.request_buffer_enabled", {TS_CONFIG_HTTP_REQUEST_BUFFER_ENABLED, TS_RECORDDATATYPE_INT}}, - {"proxy.config.http.down_server.cache_time", {TS_CONFIG_HTTP_DOWN_SERVER_CACHE_TIME, TS_RECORDDATATYPE_INT}}, - {"proxy.config.http.insert_age_in_response", {TS_CONFIG_HTTP_INSERT_AGE_IN_RESPONSE, TS_RECORDDATATYPE_INT}}, - {"proxy.config.url_remap.pristine_host_hdr", {TS_CONFIG_URL_REMAP_PRISTINE_HOST_HDR, TS_RECORDDATATYPE_INT}}, - {"proxy.config.http.insert_request_via_str", {TS_CONFIG_HTTP_INSERT_REQUEST_VIA_STR, TS_RECORDDATATYPE_INT}}, - {"proxy.config.http.flow_control.low_water", {TS_CONFIG_HTTP_FLOW_CONTROL_LOW_WATER_MARK, TS_RECORDDATATYPE_INT}}, - {"proxy.config.http.cache.required_headers", {TS_CONFIG_HTTP_CACHE_REQUIRED_HEADERS, TS_RECORDDATATYPE_INT}}, - {"proxy.config.ssl.hsts_include_subdomains", {TS_CONFIG_SSL_HSTS_INCLUDE_SUBDOMAINS, TS_RECORDDATATYPE_INT}}, - {"proxy.config.http.number_of_redirections", {TS_CONFIG_HTTP_NUMBER_OF_REDIRECTIONS, TS_RECORDDATATYPE_INT}}, - {"proxy.config.http.keep_alive_enabled_out", {TS_CONFIG_HTTP_KEEP_ALIVE_ENABLED_OUT, TS_RECORDDATATYPE_INT}}, - {"proxy.config.http.response_server_enabled", {TS_CONFIG_HTTP_RESPONSE_SERVER_ENABLED, TS_RECORDDATATYPE_INT}}, - {"proxy.config.http.anonymize_remove_cookie", {TS_CONFIG_HTTP_ANONYMIZE_REMOVE_COOKIE, TS_RECORDDATATYPE_INT}}, - {"proxy.config.http.request_header_max_size", {TS_CONFIG_HTTP_REQUEST_HEADER_MAX_SIZE, TS_RECORDDATATYPE_INT}}, - {"proxy.config.http.parent_proxy.retry_time", {TS_CONFIG_HTTP_PARENT_PROXY_RETRY_TIME, TS_RECORDDATATYPE_INT}}, - {"proxy.config.http.insert_response_via_str", {TS_CONFIG_HTTP_INSERT_RESPONSE_VIA_STR, TS_RECORDDATATYPE_INT}}, - {"proxy.config.http.flow_control.high_water", {TS_CONFIG_HTTP_FLOW_CONTROL_HIGH_WATER_MARK, TS_RECORDDATATYPE_INT}}, - {"proxy.config.http.negative_caching_enabled", {TS_CONFIG_HTTP_NEGATIVE_CACHING_ENABLED, TS_RECORDDATATYPE_INT}}, - {"proxy.config.http.cache.when_to_revalidate", {TS_CONFIG_HTTP_CACHE_WHEN_TO_REVALIDATE, TS_RECORDDATATYPE_INT}}, - {"proxy.config.http.response_header_max_size", {TS_CONFIG_HTTP_RESPONSE_HEADER_MAX_SIZE, TS_RECORDDATATYPE_INT}}, - {"proxy.config.http.anonymize_remove_referer", {TS_CONFIG_HTTP_ANONYMIZE_REMOVE_REFERER, TS_RECORDDATATYPE_INT}}, - {"proxy.config.http.global_user_agent_header", {TS_CONFIG_HTTP_GLOBAL_USER_AGENT_HEADER, TS_RECORDDATATYPE_STRING}}, - {"proxy.config.net.sock_recv_buffer_size_out", {TS_CONFIG_NET_SOCK_RECV_BUFFER_SIZE_OUT, TS_RECORDDATATYPE_INT}}, - {"proxy.config.net.sock_send_buffer_size_out", {TS_CONFIG_NET_SOCK_SEND_BUFFER_SIZE_OUT, TS_RECORDDATATYPE_INT}}, - {"proxy.config.http.connect_attempts_timeout", {TS_CONFIG_HTTP_CONNECT_ATTEMPTS_TIMEOUT, TS_RECORDDATATYPE_INT}}, - {"proxy.config.websocket.no_activity_timeout", {TS_CONFIG_WEBSOCKET_NO_ACTIVITY_TIMEOUT, TS_RECORDDATATYPE_INT}}, - {"proxy.config.http.negative_caching_lifetime", {TS_CONFIG_HTTP_NEGATIVE_CACHING_LIFETIME, TS_RECORDDATATYPE_INT}}, - {"proxy.config.http.default_buffer_water_mark", {TS_CONFIG_HTTP_DEFAULT_BUFFER_WATER_MARK, TS_RECORDDATATYPE_INT}}, - {"proxy.config.http.cache.heuristic_lm_factor", {TS_CONFIG_HTTP_CACHE_HEURISTIC_LM_FACTOR, TS_RECORDDATATYPE_FLOAT}}, - {OutboundConnTrack::CONFIG_VAR_MAX, {TS_CONFIG_HTTP_PER_SERVER_CONNECTION_MAX, TS_RECORDDATATYPE_INT}}, - {OutboundConnTrack::CONFIG_VAR_MIN, {TS_CONFIG_HTTP_SERVER_MIN_KEEP_ALIVE_CONNS, TS_RECORDDATATYPE_INT}}, - {"proxy.config.http.anonymize_remove_client_ip", {TS_CONFIG_HTTP_ANONYMIZE_REMOVE_CLIENT_IP, TS_RECORDDATATYPE_INT}}, - {"proxy.config.http.cache.open_read_retry_time", {TS_CONFIG_HTTP_CACHE_OPEN_READ_RETRY_TIME, TS_RECORDDATATYPE_INT}}, - {"proxy.config.http.down_server.abort_threshold", {TS_CONFIG_HTTP_DOWN_SERVER_ABORT_THRESHOLD, TS_RECORDDATATYPE_INT}}, - {OutboundConnTrack::CONFIG_VAR_MATCH, {TS_CONFIG_HTTP_PER_SERVER_CONNECTION_MATCH, TS_RECORDDATATYPE_INT}}, - {"proxy.config.http.parent_proxy.fail_threshold", {TS_CONFIG_HTTP_PARENT_PROXY_FAIL_THRESHOLD, TS_RECORDDATATYPE_INT}}, - {"proxy.config.http.cache.ignore_authentication", {TS_CONFIG_HTTP_CACHE_IGNORE_AUTHENTICATION, TS_RECORDDATATYPE_INT}}, - {"proxy.config.http.anonymize_remove_user_agent", {TS_CONFIG_HTTP_ANONYMIZE_REMOVE_USER_AGENT, TS_RECORDDATATYPE_INT}}, - {"proxy.config.http.connect_attempts_rr_retries", {TS_CONFIG_HTTP_CONNECT_ATTEMPTS_RR_RETRIES, TS_RECORDDATATYPE_INT}}, - {"proxy.config.http.cache.max_open_read_retries", {TS_CONFIG_HTTP_CACHE_MAX_OPEN_READ_RETRIES, TS_RECORDDATATYPE_INT}}, - {"proxy.config.http.auth_server_session_private", {TS_CONFIG_HTTP_AUTH_SERVER_SESSION_PRIVATE, TS_RECORDDATATYPE_INT}}, - {"proxy.config.http.redirect_use_orig_cache_key", {TS_CONFIG_HTTP_REDIRECT_USE_ORIG_CACHE_KEY, TS_RECORDDATATYPE_INT}}, - {"proxy.config.http.cache.ignore_client_no_cache", {TS_CONFIG_HTTP_CACHE_IGNORE_CLIENT_NO_CACHE, TS_RECORDDATATYPE_INT}}, - {"proxy.config.http.cache.ims_on_client_no_cache", {TS_CONFIG_HTTP_CACHE_IMS_ON_CLIENT_NO_CACHE, TS_RECORDDATATYPE_INT}}, - {"proxy.config.http.cache.ignore_server_no_cache", {TS_CONFIG_HTTP_CACHE_IGNORE_SERVER_NO_CACHE, TS_RECORDDATATYPE_INT}}, - {"proxy.config.http.cache.heuristic_min_lifetime", {TS_CONFIG_HTTP_CACHE_HEURISTIC_MIN_LIFETIME, TS_RECORDDATATYPE_INT}}, - {"proxy.config.http.cache.heuristic_max_lifetime", {TS_CONFIG_HTTP_CACHE_HEURISTIC_MAX_LIFETIME, TS_RECORDDATATYPE_INT}}, - {"proxy.config.http.server_session_sharing.match", {TS_CONFIG_HTTP_SERVER_SESSION_SHARING_MATCH, TS_RECORDDATATYPE_INT}}, - {"proxy.config.http.cache.ignore_accept_mismatch", {TS_CONFIG_HTTP_CACHE_IGNORE_ACCEPT_MISMATCH, TS_RECORDDATATYPE_INT}}, - {"proxy.config.http.cache.open_write_fail_action", {TS_CONFIG_HTTP_CACHE_OPEN_WRITE_FAIL_ACTION, TS_RECORDDATATYPE_INT}}, - {"proxy.config.http.insert_squid_x_forwarded_for", {TS_CONFIG_HTTP_INSERT_SQUID_X_FORWARDED_FOR, TS_RECORDDATATYPE_INT}}, - {"proxy.config.http.connect_attempts_max_retries", {TS_CONFIG_HTTP_CONNECT_ATTEMPTS_MAX_RETRIES, TS_RECORDDATATYPE_INT}}, - {"proxy.config.http.cache.max_open_write_retries", {TS_CONFIG_HTTP_CACHE_MAX_OPEN_WRITE_RETRIES, TS_RECORDDATATYPE_INT}}, - {"proxy.config.http.forward.proxy_auth_to_parent", {TS_CONFIG_HTTP_FORWARD_PROXY_AUTH_TO_PARENT, TS_RECORDDATATYPE_INT}}, - {"proxy.config.http.parent_proxy.mark_down_hostdb", {TS_CONFIG_PARENT_FAILURES_UPDATE_HOSTDB, TS_RECORDDATATYPE_INT}}, - {"proxy.config.http.negative_revalidating_enabled", {TS_CONFIG_HTTP_NEGATIVE_REVALIDATING_ENABLED, TS_RECORDDATATYPE_INT}}, - {"proxy.config.http.cache.guaranteed_min_lifetime", {TS_CONFIG_HTTP_CACHE_GUARANTEED_MIN_LIFETIME, TS_RECORDDATATYPE_INT}}, - {"proxy.config.http.cache.guaranteed_max_lifetime", {TS_CONFIG_HTTP_CACHE_GUARANTEED_MAX_LIFETIME, TS_RECORDDATATYPE_INT}}, - {"proxy.config.http.transaction_active_timeout_in", {TS_CONFIG_HTTP_TRANSACTION_ACTIVE_TIMEOUT_IN, TS_RECORDDATATYPE_INT}}, - {"proxy.config.http.post_connect_attempts_timeout", {TS_CONFIG_HTTP_POST_CONNECT_ATTEMPTS_TIMEOUT, TS_RECORDDATATYPE_INT}}, - {"proxy.config.http.cache.ignore_client_cc_max_age", {TS_CONFIG_HTTP_CACHE_IGNORE_CLIENT_CC_MAX_AGE, TS_RECORDDATATYPE_INT}}, - {"proxy.config.http.negative_revalidating_lifetime", {TS_CONFIG_HTTP_NEGATIVE_REVALIDATING_LIFETIME, TS_RECORDDATATYPE_INT}}, - {"proxy.config.http.transaction_active_timeout_out", {TS_CONFIG_HTTP_TRANSACTION_ACTIVE_TIMEOUT_OUT, TS_RECORDDATATYPE_INT}}, - {"proxy.config.http.background_fill_active_timeout", {TS_CONFIG_HTTP_BACKGROUND_FILL_ACTIVE_TIMEOUT, TS_RECORDDATATYPE_INT}}, - {"proxy.config.http.attach_server_session_to_client", {TS_CONFIG_HTTP_ATTACH_SERVER_SESSION_TO_CLIENT, TS_RECORDDATATYPE_INT}}, - {"proxy.config.http.cache.cache_responses_to_cookies", {TS_CONFIG_HTTP_CACHE_CACHE_RESPONSES_TO_COOKIES, TS_RECORDDATATYPE_INT}}, - {"proxy.config.http.keep_alive_no_activity_timeout_in", - {TS_CONFIG_HTTP_KEEP_ALIVE_NO_ACTIVITY_TIMEOUT_IN, TS_RECORDDATATYPE_INT}}, - {"proxy.config.http.post.check.content_length.enabled", - {TS_CONFIG_HTTP_POST_CHECK_CONTENT_LENGTH_ENABLED, TS_RECORDDATATYPE_INT}}, - {"proxy.config.http.cache.cache_urls_that_look_dynamic", - {TS_CONFIG_HTTP_CACHE_CACHE_URLS_THAT_LOOK_DYNAMIC, TS_RECORDDATATYPE_INT}}, - {"proxy.config.http.transaction_no_activity_timeout_in", - {TS_CONFIG_HTTP_TRANSACTION_NO_ACTIVITY_TIMEOUT_IN, TS_RECORDDATATYPE_INT}}, - {"proxy.config.http.keep_alive_no_activity_timeout_out", - {TS_CONFIG_HTTP_KEEP_ALIVE_NO_ACTIVITY_TIMEOUT_OUT, TS_RECORDDATATYPE_INT}}, - {"proxy.config.http.uncacheable_requests_bypass_parent", - {TS_CONFIG_HTTP_UNCACHEABLE_REQUESTS_BYPASS_PARENT, TS_RECORDDATATYPE_INT}}, - {"proxy.config.http.transaction_no_activity_timeout_out", - {TS_CONFIG_HTTP_TRANSACTION_NO_ACTIVITY_TIMEOUT_OUT, TS_RECORDDATATYPE_INT}}, - {"proxy.config.http.background_fill_completed_threshold", - {TS_CONFIG_HTTP_BACKGROUND_FILL_COMPLETED_THRESHOLD, TS_RECORDDATATYPE_FLOAT}}, - {"proxy.config.http.parent_proxy.total_connect_attempts", - {TS_CONFIG_HTTP_PARENT_PROXY_TOTAL_CONNECT_ATTEMPTS, TS_RECORDDATATYPE_INT}}, - {"proxy.config.http.cache.ignore_accept_charset_mismatch", - {TS_CONFIG_HTTP_CACHE_IGNORE_ACCEPT_CHARSET_MISMATCH, TS_RECORDDATATYPE_INT}}, - {"proxy.config.http.cache.ignore_accept_language_mismatch", - {TS_CONFIG_HTTP_CACHE_IGNORE_ACCEPT_LANGUAGE_MISMATCH, TS_RECORDDATATYPE_INT}}, - {"proxy.config.http.cache.ignore_accept_encoding_mismatch", - {TS_CONFIG_HTTP_CACHE_IGNORE_ACCEPT_ENCODING_MISMATCH, TS_RECORDDATATYPE_INT}}, - {"proxy.config.http.parent_proxy.connect_attempts_timeout", - {TS_CONFIG_HTTP_PARENT_CONNECT_ATTEMPT_TIMEOUT, TS_RECORDDATATYPE_INT}}, - {"proxy.config.http.connect_attempts_max_retries_dead_server", - {TS_CONFIG_HTTP_CONNECT_ATTEMPTS_MAX_RETRIES_DEAD_SERVER, TS_RECORDDATATYPE_INT}}, - {"proxy.config.http.parent_proxy.per_parent_connect_attempts", - {TS_CONFIG_HTTP_PER_PARENT_CONNECT_ATTEMPTS, TS_RECORDDATATYPE_INT}}, - {"proxy.config.ssl.client.verify.server", {TS_CONFIG_SSL_CLIENT_VERIFY_SERVER, TS_RECORDDATATYPE_INT}}, - {"proxy.config.ssl.client.verify.server.policy", {TS_CONFIG_SSL_CLIENT_VERIFY_SERVER_POLICY, TS_RECORDDATATYPE_STRING}}, - {"proxy.config.ssl.client.verify.server.properties", {TS_CONFIG_SSL_CLIENT_VERIFY_SERVER_PROPERTIES, TS_RECORDDATATYPE_STRING}}, - {"proxy.config.ssl.client.sni_policy", {TS_CONFIG_SSL_CLIENT_SNI_POLICY, TS_RECORDDATATYPE_STRING}}, - {"proxy.config.ssl.client.cert.filename", {TS_CONFIG_SSL_CLIENT_CERT_FILENAME, TS_RECORDDATATYPE_STRING}}, - {"proxy.config.ssl.client.cert.path", {TS_CONFIG_SSL_CERT_FILEPATH, TS_RECORDDATATYPE_STRING}}, - {"proxy.config.ssl.client.private_key.filename", {TS_CONFIG_SSL_CLIENT_PRIVATE_KEY_FILENAME, TS_RECORDDATATYPE_STRING}}, - {"proxy.config.ssl.client.CA.cert.filename", {TS_CONFIG_SSL_CLIENT_CA_CERT_FILENAME, TS_RECORDDATATYPE_STRING}}}); - TSReturnCode TSHttpTxnConfigFind(const char *name, int length, TSOverridableConfigKey *conf, TSRecordDataType *type) { @@ -8911,11 +8898,8 @@ TSHttpTxnConfigFind(const char *name, int length, TSOverridableConfigKey *conf, sdk_assert(sdk_sanity_check_null_ptr(conf) == TS_SUCCESS); std::string_view name_sv(name, length < 0 ? strlen(name) : length); - auto config = Overridable_Map.find(name_sv); - - if (config != Overridable_Map.end()) { + if (auto config = ts::Overridable_Txn_Vars.find(name_sv); config != ts::Overridable_Txn_Vars.end()) { std::tie(*conf, *type) = config->second; - return TS_SUCCESS; } @@ -9070,6 +9054,30 @@ TSHttpTxnIsCacheable(TSHttpTxn txnp, TSMBuffer request, TSMBuffer response) return (req->valid() && resp->valid() && HttpTransact::is_response_cacheable(&(sm->t_state), req, resp)) ? 1 : 0; } +int +TSHttpTxnGetMaxAge(TSHttpTxn txnp, TSMBuffer response) +{ + sdk_assert(sdk_sanity_check_txn(txnp) == TS_SUCCESS); + HttpSM *sm = (HttpSM *)txnp; + HTTPHdr *resp; + + if (response) { + // Make sure the response we got as a parameter is valid + sdk_assert(sdk_sanity_check_mbuffer(response) == TS_SUCCESS); + resp = reinterpret_cast(response); + } else { + // Use the transactions origin response if the user passed NULL + resp = &(sm->t_state.hdr_info.server_response); + } + + if (!resp || !resp->valid()) { + return -1; + } + + // We have a valid response, return max_age + return HttpTransact::get_max_age(resp); +} + // Lookup various debug names for common HTTP types. const char * TSHttpServerStateNameLookup(TSServerState state) @@ -9126,7 +9134,7 @@ TSVConnTunnel(TSVConn sslp) } TSSslConnection -TSVConnSSLConnectionGet(TSVConn sslp) +TSVConnSslConnectionGet(TSVConn sslp) { TSSslConnection ssl = nullptr; NetVConnection *vc = reinterpret_cast(sslp); @@ -9515,7 +9523,7 @@ TSSslSessionGet(const TSSslSessionID *session_id) { SSL_SESSION *session = nullptr; if (session_id && session_cache) { - session_cache->getSession(reinterpret_cast(*session_id), &session); + session_cache->getSession(reinterpret_cast(*session_id), &session, nullptr); } return reinterpret_cast(session); } @@ -9532,7 +9540,7 @@ TSSslSessionGetBuffer(const TSSslSessionID *session_id, char *buffer, int *len_p } TSReturnCode -TSSslSessionInsert(const TSSslSessionID *session_id, TSSslSession add_session) +TSSslSessionInsert(const TSSslSessionID *session_id, TSSslSession add_session, TSSslConnection ssl_conn) { // Don't insert if there is no session id or the cache is not yet set up if (session_id && session_cache) { @@ -9543,7 +9551,8 @@ TSSslSessionInsert(const TSSslSessionID *session_id, TSSslSession add_session) Debug("ssl.session_cache.insert", "TSSslSessionInsert: Inserting session '%s' ", buf); } SSL_SESSION *session = reinterpret_cast(add_session); - session_cache->insertSession(reinterpret_cast(*session_id), session); + SSL *ssl = reinterpret_cast(ssl_conn); + session_cache->insertSession(reinterpret_cast(*session_id), session, ssl); // insertSession returns void, assume all went well return TS_SUCCESS; } else { diff --git a/src/traffic_server/InkAPITest.cc b/src/traffic_server/InkAPITest.cc index d6713a8180b..2b137453b92 100644 --- a/src/traffic_server/InkAPITest.cc +++ b/src/traffic_server/InkAPITest.cc @@ -40,6 +40,7 @@ #include "tscore/ink_sprintf.h" #include "tscore/ink_file.h" #include "tscore/Regression.h" +#include "tscore/Filenames.h" #include "ts/ts.h" #include "ts/experimental.h" #include "records/I_RecCore.h" @@ -89,7 +90,7 @@ using TxnHandler = int (*)(TSCont, TSEvent, void *); /* Server transaction structure */ -typedef struct { +struct ServerTxn { TSVConn vconn; TSVIO read_vio; @@ -105,24 +106,24 @@ typedef struct { TxnHandler current_handler; unsigned int magic; -} ServerTxn; +}; /* Server structure */ -typedef struct { +struct SocketServer { int accept_port; TSAction accept_action; TSCont accept_cont; unsigned int magic; -} SocketServer; +}; -typedef enum { +enum RequestStatus { REQUEST_SUCCESS, REQUEST_INPROGRESS, REQUEST_FAILURE, -} RequestStatus; +}; /* Client structure */ -typedef struct { +struct ClientTxn { TSVConn vconn; TSVIO read_vio; @@ -147,7 +148,7 @@ typedef struct { TxnHandler current_handler; unsigned int magic; -} ClientTxn; +}; ////////////////////////////////////////////////////////////////////////////// // DECLARATIONS @@ -1596,7 +1597,7 @@ int *SDK_Cache_pstatus; static char content[OBJECT_SIZE]; static int read_counter = 0; -typedef struct { +struct CacheVConnStruct { TSIOBuffer bufp; TSIOBuffer out_bufp; TSIOBufferReader readerp; @@ -1608,7 +1609,7 @@ typedef struct { TSVIO write_vio; TSCacheKey key; -} CacheVConnStruct; +}; int cache_handler(TSCont contp, TSEvent event, void *data) @@ -1995,7 +1996,6 @@ REGRESSION_TEST(SDK_API_TSCache)(RegressionTest *test, int /* atype ATS_UNUSED * // TSfread // TSfwrite ////////////////////////////////////////////// -#define PFX "plugin.config" // Note that for each test, if it fails, we set the error status and return. REGRESSION_TEST(SDK_API_TSfopen)(RegressionTest *test, int /* atype ATS_UNUSED */, int *pstatus) @@ -2013,8 +2013,7 @@ REGRESSION_TEST(SDK_API_TSfopen)(RegressionTest *test, int /* atype ATS_UNUSED * struct stat stat_buffer_pre, stat_buffer_post, stat_buffer_input; char *ret_val; int read = 0, wrote = 0; - int64_t read_amount = 0; - char INPUT_TEXT_FILE[] = "plugin.config"; + int64_t read_amount = 0; char input_file_full_path[BUFSIZ]; // Set full path to file at run time. @@ -2027,7 +2026,7 @@ REGRESSION_TEST(SDK_API_TSfopen)(RegressionTest *test, int /* atype ATS_UNUSED * return; } // Add "etc/trafficserver" to point to config directory - ink_filepath_make(input_file_full_path, sizeof(input_file_full_path), TSConfigDirGet(), INPUT_TEXT_FILE); + ink_filepath_make(input_file_full_path, sizeof(input_file_full_path), TSConfigDirGet(), ts::filename::PLUGIN); // open existing file for reading if (!(source_read_file = TSfopen(input_file_full_path, "r"))) { @@ -2041,7 +2040,7 @@ REGRESSION_TEST(SDK_API_TSfopen)(RegressionTest *test, int /* atype ATS_UNUSED * } // Create unique tmp _file_name_, do not use any TS file_name - snprintf(write_file_name, PATH_NAME_MAX, "/tmp/%sXXXXXX", PFX); + snprintf(write_file_name, PATH_NAME_MAX, "/tmp/%sXXXXXX", ts::filename::PLUGIN); int write_file_fd; // this file will be reopened below if ((write_file_fd = mkstemp(write_file_name)) <= 0) { SDK_RPRINT(test, "mkstemp", "std func", TC_FAIL, "can't create file for writing"); @@ -2528,10 +2527,10 @@ static RegressionTest *SDK_ContData_test; static int *SDK_ContData_pstatus; // this is specific for this test -typedef struct { +struct MyData { int data1; int data2; -} MyData; +}; int cont_data_handler(TSCont contp, TSEvent /* event ATS_UNUSED */, void * /* edata ATS_UNUSED */) @@ -2947,7 +2946,7 @@ REGRESSION_TEST(SDK_API_TSIOBufferBlockReadAvail)(RegressionTest *test, int /* a SDK_RPRINT(test, "TSIOBufferBlockWriteStart", "TestCase1", TC_FAIL, "failed"); } - if ((TSIOBufferBlockReadAvail(blockp, readerp) + TSIOBufferBlockWriteAvail(blockp)) == 4096) { + if ((TSIOBufferBlockReadAvail(blockp, readerp) + TSIOBufferBlockWriteAvail(blockp)) == 32768) { SDK_RPRINT(test, "TSIOBufferBlockReadAvail", "TestCase1", TC_PASS, "ok"); SDK_RPRINT(test, "TSIOBufferBlockWriteAvail", "TestCase1", TC_PASS, "ok"); test_passed_2 = true; @@ -3039,7 +3038,7 @@ REGRESSION_TEST(SDK_API_TSContSchedule)(RegressionTest *test, int /* atype ATS_U #define HTTP_HOOK_TEST_REQUEST_ID 1 -typedef struct { +struct SocketTest { RegressionTest *regtest; int *pstatus; SocketServer *os; @@ -3059,7 +3058,7 @@ typedef struct { bool test_client_protocol_stack_contains; unsigned int magic; -} SocketTest; +}; // This func is called by us from mytest_handler to test TSHttpTxnClientIPGet static int @@ -6293,13 +6292,13 @@ REGRESSION_TEST(SDK_API_TSUrlParse)(RegressionTest *test, int /* atype ATS_UNUSE ////////////////////////////////////////////// #define LOG_TEST_PATTERN "SDK team rocks" -typedef struct { +struct LogTestData { RegressionTest *test; int *pstatus; char *fullpath_logname; unsigned long magic; TSTextLogObject log; -} LogTestData; +}; static int log_test_handler(TSCont contp, TSEvent event, void * /* edata ATS_UNUSED */) @@ -6528,19 +6527,19 @@ REGRESSION_TEST(SDK_API_TSMgmtGet)(RegressionTest *test, int /* atype ATS_UNUSED } \ } -typedef enum { +enum ORIG_TSParseResult { ORIG_TS_PARSE_ERROR = -1, ORIG_TS_PARSE_DONE = 0, ORIG_TS_PARSE_CONT = 1, -} ORIG_TSParseResult; +}; -typedef enum { +enum ORIG_TSHttpType { ORIG_TS_HTTP_TYPE_UNKNOWN, ORIG_TS_HTTP_TYPE_REQUEST, ORIG_TS_HTTP_TYPE_RESPONSE, -} ORIG_TSHttpType; +}; -typedef enum { +enum ORIG_TSHttpStatus { ORIG_TS_HTTP_STATUS_NONE = 0, ORIG_TS_HTTP_STATUS_CONTINUE = 100, @@ -6585,9 +6584,9 @@ typedef enum { ORIG_TS_HTTP_STATUS_SERVICE_UNAVAILABLE = 503, ORIG_TS_HTTP_STATUS_GATEWAY_TIMEOUT = 504, ORIG_TS_HTTP_STATUS_HTTPVER_NOT_SUPPORTED = 505 -} ORIG_TSHttpStatus; +}; -typedef enum { +enum ORIG_TSHttpHookID { ORIG_TS_HTTP_READ_REQUEST_HDR_HOOK, ORIG_TS_HTTP_OS_DNS_HOOK, ORIG_TS_HTTP_SEND_REQUEST_HDR_HOOK, @@ -6619,9 +6618,9 @@ typedef enum { ORIG_TS_SSL_LAST_HOOK = ORIG_TS_VCONN_OUTBOUND_CLOSE_HOOK, ORIG_TS_HTTP_REQUEST_BUFFER_READ_COMPLETE_HOOK, ORIG_TS_HTTP_LAST_HOOK -} ORIG_TSHttpHookID; +}; -typedef enum { +enum ORIG_TSEvent { ORIG_TS_EVENT_NONE = 0, ORIG_TS_EVENT_IMMEDIATE = 1, ORIG_TS_EVENT_TIMEOUT = 2, @@ -6672,44 +6671,44 @@ typedef enum { ORIG_TS_EVENT_HTTP_CACHE_LOOKUP_COMPLETE = 60015, ORIG_TS_EVENT_MGMT_UPDATE = 60300 -} ORIG_TSEvent; +}; -typedef enum { +enum ORIG_TSCacheLookupResult { ORIG_TS_CACHE_LOOKUP_MISS, ORIG_TS_CACHE_LOOKUP_HIT_STALE, ORIG_TS_CACHE_LOOKUP_HIT_FRESH, -} ORIG_TSCacheLookupResult; +}; -typedef enum { +enum ORIG_TSCacheDataType { ORIG_TS_CACHE_DATA_TYPE_NONE, ORIG_TS_CACHE_DATA_TYPE_HTTP, ORIG_TS_CACHE_DATA_TYPE_OTHER, -} ORIG_TSCacheDataType; +}; -typedef enum { +enum ORIG_TSCacheError { ORIG_TS_CACHE_ERROR_NO_DOC = -20400, ORIG_TS_CACHE_ERROR_DOC_BUSY = -20401, ORIG_TS_CACHE_ERROR_NOT_READY = -20407 -} ORIG_TSCacheError; +}; -typedef enum { +enum ORIG_TSCacheScanResult { ORIG_TS_CACHE_SCAN_RESULT_DONE = 0, ORIG_TS_CACHE_SCAN_RESULT_CONTINUE = 1, ORIG_TS_CACHE_SCAN_RESULT_DELETE = 10, ORIG_TS_CACHE_SCAN_RESULT_DELETE_ALL_ALTERNATES, ORIG_TS_CACHE_SCAN_RESULT_UPDATE, ORIG_TS_CACHE_SCAN_RESULT_RETRY -} ORIG_TSCacheScanResult; +}; -typedef enum { +enum ORIG_TSVConnCloseFlags { ORIG_TS_VC_CLOSE_ABORT = -1, ORIG_TS_VC_CLOSE_NORMAL = 1, -} ORIG_TSVConnCloseFlags; +}; -typedef enum { +enum ORIG_TSReturnCode { ORIG_TS_ERROR = -1, ORIG_TS_SUCCESS = 0, -} ORIG_TSReturnCode; +}; REGRESSION_TEST(SDK_API_TSConstant)(RegressionTest *test, int /* atype ATS_UNUSED */, int *pstatus) { @@ -6869,7 +6868,7 @@ REGRESSION_TEST(SDK_API_TSConstant)(RegressionTest *test, int /* atype ATS_UNUSE // TSHttpTxnParentProxySet ////////////////////////////////////////////// -typedef struct { +struct ContData { RegressionTest *test; int *pstatus; SocketServer *os; @@ -6882,7 +6881,7 @@ typedef struct { int test_passed_txn_error_body_set; bool test_passed_Parent_Proxy; int magic; -} ContData; +}; static int checkHttpTxnParentProxy(ContData *data, TSHttpTxn txnp) @@ -7348,7 +7347,7 @@ EXCLUSIVE_REGRESSION_TEST(SDK_API_HttpParentProxySet_Success)(RegressionTest *te // TSHttpTxnCacheLookupStatusGet ///////////////////////////////////////////////////// -typedef struct { +struct CacheTestData { RegressionTest *test; int *pstatus; SocketServer *os; @@ -7360,7 +7359,7 @@ typedef struct { bool test_passed_txn_cache_lookup_status; bool first_time; int magic; -} CacheTestData; +}; static int cache_hook_handler(TSCont contp, TSEvent event, void *edata) @@ -7567,7 +7566,7 @@ EXCLUSIVE_REGRESSION_TEST(SDK_API_HttpTxnCache)(RegressionTest *test, int /* aty /** Append Transform Data Structure Ends **/ -typedef struct { +struct TransformTestData { RegressionTest *test; int *pstatus; SocketServer *os; @@ -7583,7 +7582,7 @@ typedef struct { bool test_passed_transform_create; int req_no; uint32_t magic; -} TransformTestData; +}; /** Append Transform Data Structure **/ struct AppendTransformTestData { @@ -8102,7 +8101,7 @@ EXCLUSIVE_REGRESSION_TEST(SDK_API_HttpTxnTransform)(RegressionTest *test, int /* // TSHttpTxnCachedRespGet ////////////////////////////////////////////// -typedef struct { +struct AltInfoTestData { RegressionTest *test; int *pstatus; SocketServer *os; @@ -8119,7 +8118,7 @@ typedef struct { bool run_at_least_once; bool first_time; int magic; -} AltInfoTestData; +}; static int altinfo_hook_handler(TSCont contp, TSEvent event, void *edata) @@ -8337,7 +8336,7 @@ EXCLUSIVE_REGRESSION_TEST(SDK_API_HttpAltInfo)(RegressionTest *test, int /* atyp #define TEST_CASE_CONNECT_ID1 9 // TSHttpTxnIntercept #define TEST_CASE_CONNECT_ID2 10 // TSHttpTxnServerIntercept -typedef struct { +struct ConnectTestData { RegressionTest *test; int *pstatus; int test_case; @@ -8346,7 +8345,7 @@ typedef struct { ClientTxn *browser; char *request; unsigned long magic; -} ConnectTestData; +}; static int cont_test_handler(TSCont contp, TSEvent event, void *edata) @@ -8661,7 +8660,8 @@ std::array SDK_Overridable_Configs = { "proxy.config.ssl.client.verify.server.properties", "proxy.config.ssl.client.sni_policy", "proxy.config.ssl.client.private_key.filename", - "proxy.config.ssl.client.CA.cert.filename"}}; + "proxy.config.ssl.client.CA.cert.filename", + "proxy.config.hostdb.ip_resolve"}}; REGRESSION_TEST(SDK_API_OVERRIDABLE_CONFIGS)(RegressionTest *test, int /* atype ATS_UNUSED */, int *pstatus) { diff --git a/src/traffic_server/InkIOCoreAPI.cc b/src/traffic_server/InkIOCoreAPI.cc index 25618ef05e0..82adea6fa33 100644 --- a/src/traffic_server/InkIOCoreAPI.cc +++ b/src/traffic_server/InkIOCoreAPI.cc @@ -235,14 +235,17 @@ TSEventThreadSelf(void) //////////////////////////////////////////////////////////////////// // -// Mutexes +// Mutexes: For TSMutexCreate and TSMutexDestroy, the refcount of the +// ProxyMutex object is not incremented or decremented. If the resulting +// ProxyMutex is passed to a INKContInternal, it's mutex smart pointer +// will take ownership of the ProxyMutex and delete it when the last +// reference is removed. TSMutexDestroy should not be called in that case. // //////////////////////////////////////////////////////////////////// TSMutex TSMutexCreate() { ProxyMutex *mutexp = new_ProxyMutex(); - mutexp->refcount_inc(); // TODO: Remove this when allocations can never fail. sdk_assert(sdk_sanity_check_mutex((TSMutex)mutexp) == TS_SUCCESS); @@ -255,9 +258,9 @@ TSMutexDestroy(TSMutex m) { sdk_assert(sdk_sanity_check_mutex(m) == TS_SUCCESS); ProxyMutex *mutexp = reinterpret_cast(m); - // Decrement the refcount added in TSMutexCreate. Delete if this - // was the last ref count - if (mutexp && mutexp->refcount_dec() == 0) { + + if (mutexp) { + ink_release_assert(mutexp->refcount() == 0); mutexp->free(); } } @@ -296,7 +299,7 @@ void TSMutexLock(TSMutex mutexp) { sdk_assert(sdk_sanity_check_mutex(mutexp) == TS_SUCCESS); - Ptr proxy_mutex(reinterpret_cast(mutexp)); + ProxyMutex *proxy_mutex = reinterpret_cast(mutexp); MUTEX_TAKE_LOCK(proxy_mutex, this_ethread()); } @@ -304,7 +307,7 @@ TSReturnCode TSMutexLockTry(TSMutex mutexp) { sdk_assert(sdk_sanity_check_mutex(mutexp) == TS_SUCCESS); - Ptr proxy_mutex(reinterpret_cast(mutexp)); + ProxyMutex *proxy_mutex = reinterpret_cast(mutexp); return (MUTEX_TAKE_TRY_LOCK(proxy_mutex, this_ethread()) ? TS_SUCCESS : TS_ERROR); } @@ -312,7 +315,7 @@ void TSMutexUnlock(TSMutex mutexp) { sdk_assert(sdk_sanity_check_mutex(mutexp) == TS_SUCCESS); - Ptr proxy_mutex(reinterpret_cast(mutexp)); + ProxyMutex *proxy_mutex(reinterpret_cast(mutexp)); MUTEX_UNTAKE_LOCK(proxy_mutex, this_ethread()); } @@ -440,7 +443,7 @@ INKUDPBind(TSCont contp, unsigned int ip, int port) ats_ip4_set(&addr, ip, htons(port)); return reinterpret_cast( - udpNet.UDPBind((Continuation *)contp, ats_ip_sa_cast(&addr), INK_ETHERNET_MTU_SIZE, INK_ETHERNET_MTU_SIZE)); + udpNet.UDPBind((Continuation *)contp, ats_ip_sa_cast(&addr), -1, INK_ETHERNET_MTU_SIZE, INK_ETHERNET_MTU_SIZE)); } TSAction @@ -564,7 +567,7 @@ INKUDPPacketGet(INKUDPacketQueue queuep) TSIOBuffer TSIOBufferCreate() { - MIOBuffer *b = new_empty_MIOBuffer(); + MIOBuffer *b = new_empty_MIOBuffer(BUFFER_SIZE_INDEX_32K); // TODO: Should remove this when memory allocations can't fail. sdk_assert(sdk_sanity_check_iocore_structure(b) == TS_SUCCESS); diff --git a/src/traffic_server/Makefile.inc b/src/traffic_server/Makefile.inc index 41d3839759c..d189bd727d2 100644 --- a/src/traffic_server/Makefile.inc +++ b/src/traffic_server/Makefile.inc @@ -42,8 +42,6 @@ traffic_server_traffic_server_LDFLAGS = \ @YAMLCPP_LDFLAGS@ traffic_server_traffic_server_SOURCES = \ - traffic_server/CoreUtils.cc \ - traffic_server/CoreUtils.h \ traffic_server/Crash.cc \ traffic_server/EventName.cc \ traffic_server/EventName.h \ @@ -53,6 +51,7 @@ traffic_server_traffic_server_SOURCES = \ traffic_server/InkAPI.cc \ traffic_server/InkIOCoreAPI.cc \ traffic_server/SocksProxy.cc \ + shared/overridable_txn_vars.cc \ traffic_server/traffic_server.cc if BUILD_TESTS diff --git a/src/traffic_server/SocksProxy.cc b/src/traffic_server/SocksProxy.cc index 06e88f812e5..c1f44c25559 100644 --- a/src/traffic_server/SocksProxy.cc +++ b/src/traffic_server/SocksProxy.cc @@ -144,7 +144,7 @@ void SocksProxy::init(NetVConnection *netVC) { mutex = new_ProxyMutex(); - buf = new_MIOBuffer(); + buf = new_MIOBuffer(BUFFER_SIZE_INDEX_32K); reader = buf->alloc_reader(); SCOPED_MUTEX_LOCK(lock, mutex, this_ethread()); diff --git a/src/traffic_server/traffic_server.cc b/src/traffic_server/traffic_server.cc index 591ce0b0c49..15fe8ff7b50 100644 --- a/src/traffic_server/traffic_server.cc +++ b/src/traffic_server/traffic_server.cc @@ -38,11 +38,15 @@ #include "tscore/ink_syslog.h" #include "tscore/hugepages.h" #include "tscore/runroot.h" +#include "tscore/Filenames.h" +#include "tscore/ts_file.h" #include "ts/ts.h" // This is sadly needed because of us using TSThreadInit() for some reason. #include #include +#include +#include #if !defined(linux) #include @@ -62,6 +66,7 @@ extern "C" int plock(int); #include "tscore/signals.h" #include "P_EventSystem.h" #include "P_Net.h" +#include "P_QUICNetProcessor.h" #include "P_UDPNet.h" #include "P_DNS.h" #include "P_SplitDNS.h" @@ -88,8 +93,8 @@ extern "C" int plock(int); #include "HuffmanCodec.h" #include "Plugin.h" #include "DiagsConfig.h" -#include "CoreUtils.h" #include "RemapConfig.h" +#include "RemapPluginInfo.h" #include "RemapProcessor.h" #include "I_Tasks.h" #include "InkAPIInternal.h" @@ -138,7 +143,6 @@ static int num_task_threads = 0; static char *http_accept_port_descriptor; int http_accept_file_descriptor = NO_FD; -static char core_file[255] = ""; static bool enable_core_file_p = false; // Enable core file dump? int command_flag = DEFAULT_COMMAND_FLAG; int command_index = -1; @@ -204,17 +208,13 @@ static ArgumentDescription argument_descriptions[] = { {"remote_management", 'M', "Remote Management", "T", &remote_management_flag, "PROXY_REMOTE_MANAGEMENT", nullptr}, {"command", 'C', "Maintenance Command to Execute\n" - " Commands: list, check, clear, clear_cache, clear_hostdb, verify_config, help", + " Commands: list, check, clear, clear_cache, clear_hostdb, verify_config, verify_global_plugin, verify_remap_plugin, help", "S511", &command_string, "PROXY_COMMAND_STRING", nullptr}, {"conf_dir", 'D', "config dir to verify", "S511", &conf_dir, "PROXY_CONFIG_CONFIG_DIR", nullptr}, {"clear_hostdb", 'k', "Clear HostDB on Startup", "F", &auto_clear_hostdb_flag, "PROXY_CLEAR_HOSTDB", nullptr}, {"clear_cache", 'K', "Clear Cache on Startup", "F", &cacheProcessor.auto_clear_flag, "PROXY_CLEAR_CACHE", nullptr}, {"bind_stdout", '-', "Regular file to bind stdout to", "S512", &bind_stdout, "PROXY_BIND_STDOUT", nullptr}, {"bind_stderr", '-', "Regular file to bind stderr to", "S512", &bind_stderr, "PROXY_BIND_STDERR", nullptr}, -#if defined(linux) - {"read_core", 'c', "Read Core file", "S255", &core_file, nullptr, nullptr}, -#endif - {"accept_mss", '-', "MSS for client connections", "I", &accept_mss, nullptr, nullptr}, {"poll_timeout", 't', "poll timeout in milliseconds", "I", &poll_timeout, nullptr, nullptr}, HELP_ARGUMENT_DESCRIPTION(), @@ -230,7 +230,7 @@ struct AutoStopCont : public Continuation { APIHook *hook = lifecycle_hooks->get(TS_LIFECYCLE_SHUTDOWN_HOOK); while (hook) { - SCOPED_MUTEX_LOCK(lock, hook->m_cont->mutex, this_ethread()); + WEAK_SCOPED_MUTEX_LOCK(lock, hook->m_cont->mutex, this_ethread()); hook->invoke(TS_EVENT_LIFECYCLE_SHUTDOWN, nullptr); hook = hook->next(); } @@ -625,8 +625,8 @@ initialize_process_manager() pmgmt = new ProcessManager(remote_management_flag); // Lifecycle callbacks can potentially be invoked from this thread, so force thread initialization - // to make the TS API work. Use a lambda to avoid dealing with compiler dependent casting issues. - pmgmt->start([]() -> void { TSThreadInit(); }); + // to make the TS API work. + pmgmt->start(TSThreadInit, TSThreadDestroy); RecProcessInitMessage(remote_management_flag ? RECM_CLIENT : RECM_STAND_ALONE); pmgmt->reconfigure(); @@ -678,17 +678,27 @@ cmd_list(char * /* cmd ATS_UNUSED */) } } +/** Parse the given string and skip the first word. + * + * Words are assumed to be separated by spaces or tabs. + * + * @param[in] cmd The string whose first word will be skipped. + * + * @return The pointer in the string cmd to the second word in the string, or + * nullptr if there is no second word. + */ static char * -skip(char *cmd, int null_ok = 0) +skip(char *cmd) { + // Skip initial white space. cmd += strspn(cmd, " \t"); + // Point to the beginning of the next white space. cmd = strpbrk(cmd, " \t"); if (!cmd) { - if (!null_ok) { - printf("Error: argument missing\n"); - } return cmd; } + // Skip the second white space so that cmd now points to the beginning of the + // second word. cmd += strspn(cmd, " \t"); return cmd; } @@ -857,23 +867,23 @@ cmd_verify(char * /* cmd ATS_UNUSED */) if (!urlRewriteVerify()) { exitStatus |= (1 << 0); - fprintf(stderr, "ERROR: Failed to load remap.config, exitStatus %d\n\n", exitStatus); + fprintf(stderr, "ERROR: Failed to load %s, exitStatus %d\n\n", ts::filename::REMAP, exitStatus); } else { - fprintf(stderr, "INFO: Successfully loaded remap.config\n\n"); + fprintf(stderr, "INFO: Successfully loaded %s\n\n", ts::filename::REMAP); } if (RecReadConfigFile() != REC_ERR_OKAY) { exitStatus |= (1 << 1); - fprintf(stderr, "ERROR: Failed to load records.config, exitStatus %d\n\n", exitStatus); + fprintf(stderr, "ERROR: Failed to load %s, exitStatus %d\n\n", ts::filename::RECORDS, exitStatus); } else { - fprintf(stderr, "INFO: Successfully loaded records.config\n\n"); + fprintf(stderr, "INFO: Successfully loaded %s\n\n", ts::filename::RECORDS); } if (!plugin_init(true)) { exitStatus |= (1 << 2); - fprintf(stderr, "ERROR: Failed to load plugin.config, exitStatus %d\n\n", exitStatus); + fprintf(stderr, "ERROR: Failed to load %s, exitStatus %d\n\n", ts::filename::PLUGIN, exitStatus); } else { - fprintf(stderr, "INFO: Successfully loaded plugin.config\n\n"); + fprintf(stderr, "INFO: Successfully loaded %s\n\n", ts::filename::PLUGIN); } SSLInitializeLibrary(); @@ -900,6 +910,115 @@ cmd_verify(char * /* cmd ATS_UNUSED */) return 0; } +enum class plugin_type_t { + GLOBAL, + REMAP, +}; + +/** Attempt to load a plugin shared object file. + * + * @param[in] plugin_type The type of plugin for which to create a PluginInfo. + * @param[in] plugin_path The path to the plugin's shared object file. + * @param[out] error Some description of why the plugin failed to load if + * loading it fails. + * + * @return True if the plugin loaded successfully, false otherwise. + */ +static bool +load_plugin(plugin_type_t plugin_type, const fs::path &plugin_path, std::string &error) +{ + switch (plugin_type) { + case plugin_type_t::GLOBAL: { + void *handle, *initptr; + return plugin_dso_load(plugin_path.c_str(), handle, initptr, error); + } + case plugin_type_t::REMAP: { + auto temporary_directory = fs::temp_directory_path(); + temporary_directory /= fs::path(std::string("verify_plugin_") + std::to_string(getpid())); + std::error_code ec; + if (!fs::create_directories(temporary_directory, ec)) { + std::ostringstream error_os; + error_os << "Could not create temporary directory " << temporary_directory.string() << ": " << ec.message(); + error = error_os.str(); + return false; + } + const auto runtime_path = temporary_directory / ts::file::filename(plugin_path); + const fs::path unused_config; + auto plugin_info = std::make_unique(unused_config, plugin_path, runtime_path); + bool loaded = plugin_info->load(error); + if (!fs::remove(temporary_directory, ec)) { + fprintf(stderr, "ERROR: could not remove temporary directory '%s': %s\n", temporary_directory.c_str(), ec.message().c_str()); + } + return loaded; + } + } + // Unreached. + return false; +} + +/** A helper for the verify plugin command functions. + * + * @param[in] args The arguments passed to the -C command option. This includes + * verify_global_plugin. + * + * @param[in] symbols The expected symbols to verify exist in the plugin file. + * + * @return a CMD status code. See the CMD_ defines above in this file. + */ +static int +verify_plugin_helper(char *args, plugin_type_t plugin_type) +{ + const auto *plugin_filename = skip(args); + if (!plugin_filename) { + fprintf(stderr, "ERROR: verifying a plugin requires a plugin SO file path argument\n"); + return CMD_FAILED; + } + + fs::path plugin_path(plugin_filename); + fprintf(stderr, "NOTE: verifying plugin '%s'...\n", plugin_filename); + + if (!fs::exists(plugin_path)) { + fprintf(stderr, "ERROR: verifying plugin '%s' Fail: No such file or directory\n", plugin_filename); + return CMD_FAILED; + } + + auto ret = CMD_OK; + std::string error; + if (load_plugin(plugin_type, plugin_path, error)) { + fprintf(stderr, "NOTE: verifying plugin '%s' Success\n", plugin_filename); + } else { + fprintf(stderr, "ERROR: verifying plugin '%s' Fail: %s\n", plugin_filename, error.c_str()); + ret = CMD_FAILED; + } + return ret; +} + +/** Verify whether a given SO file looks like a valid global plugin. + * + * @param[in] args The arguments passed to the -C command option. This includes + * verify_global_plugin. + * + * @return a CMD status code. See the CMD_ defines above in this file. + */ +static int +cmd_verify_global_plugin(char *args) +{ + return verify_plugin_helper(args, plugin_type_t::GLOBAL); +} + +/** Verify whether a given SO file looks like a valid remap plugin. + * + * @param[in] args The arguments passed to the -C command option. This includes + * verify_global_plugin. + * + * @return a CMD status code. See the CMD_ defines above in this file. + */ +static int +cmd_verify_remap_plugin(char *args) +{ + return verify_plugin_helper(args, plugin_type_t::REMAP); +} + static int cmd_help(char *cmd); static const struct CMD { @@ -960,6 +1079,22 @@ static const struct CMD { "\n" "Load the config and verify traffic_server comes up correctly. \n", cmd_verify, true}, + {"verify_global_plugin", "Verify a global plugin's shared object file", + "VERIFY_GLOBAL_PLUGIN\n" + "\n" + "FORMAT: verify_global_plugin [global_plugin_so_file]\n" + "\n" + "Load a global plugin's shared object file and verify it meets\n" + "minimal plugin API requirements. \n", + cmd_verify_global_plugin, false}, + {"verify_remap_plugin", "Verify a remap plugin's shared object file", + "VERIFY_REMAP_PLUGIN\n" + "\n" + "FORMAT: verify_remap_plugin [remap_plugin_so_file]\n" + "\n" + "Load a remap plugin's shared object file and verify it meets\n" + "minimal plugin API requirements. \n", + cmd_verify_remap_plugin, false}, {"help", "Obtain a short description of a command (e.g. 'help clear')", "HELP\n" "\n" @@ -992,16 +1127,24 @@ find_cmd_index(const char *p) return -1; } +/** Print the maintenance command help output. + */ +static void +print_cmd_help() +{ + for (unsigned i = 0; i < countof(commands); i++) { + printf("%25s %s\n", commands[i].n, commands[i].d); + } +} + static int cmd_help(char *cmd) { (void)cmd; printf("HELP\n\n"); - cmd = skip(cmd, true); + cmd = skip(cmd); if (!cmd) { - for (unsigned i = 0; i < countof(commands); i++) { - printf("%15s %s\n", commands[i].n, commands[i].d); - } + print_cmd_help(); } else { int i; if ((i = find_cmd_index(cmd)) < 0) { @@ -1044,6 +1187,10 @@ cmd_mode() return commands[command_index].f(command_string); } else if (*command_string) { Warning("unrecognized command: '%s'", command_string); + printf("\n"); + printf("WARNING: Unrecognized command: '%s'\n", command_string); + printf("\n"); + print_cmd_help(); return CMD_FAILED; // in error } else { printf("\n"); @@ -1304,7 +1451,7 @@ syslog_log_configure() ats_free(facility_str); if (facility < 0) { - syslog(LOG_WARNING, "Bad syslog facility in records.config. Keeping syslog at LOG_DAEMON"); + syslog(LOG_WARNING, "Bad syslog facility in %s. Keeping syslog at LOG_DAEMON", ts::filename::RECORDS); } else { Debug("server", "Setting syslog facility to %d", facility); closelog(); @@ -1474,7 +1621,8 @@ change_uid_gid(const char *user) "\tand then rebuild the server.\n" "\tIt is strongly suggested that you instead modify the\n" "\tproxy.config.admin.user_id directive in your\n" - "\trecords.config file to list a non-root user.\n"); + "\t%s file to list a non-root user.\n", + ts::filename::RECORDS); } #endif } @@ -1488,27 +1636,27 @@ change_uid_gid(const char *user) * This must work without the ability to elevate privilege if the files are accessible without. */ void -bind_outputs(const char *bind_stdout, const char *bind_stderr) +bind_outputs(const char *bind_stdout_p, const char *bind_stderr_p) { int log_fd; unsigned int flags = O_WRONLY | O_APPEND | O_CREAT | O_SYNC; - if (*bind_stdout != 0) { - Debug("log", "binding stdout to %s", bind_stdout); - log_fd = elevating_open(bind_stdout, flags, 0644); + if (*bind_stdout_p != 0) { + Debug("log", "binding stdout to %s", bind_stdout_p); + log_fd = elevating_open(bind_stdout_p, flags, 0644); if (log_fd < 0) { - fprintf(stdout, "[Warning]: TS unable to open log file \"%s\" [%d '%s']\n", bind_stdout, errno, strerror(errno)); + fprintf(stdout, "[Warning]: TS unable to open log file \"%s\" [%d '%s']\n", bind_stdout_p, errno, strerror(errno)); } else { Debug("log", "duping stdout"); dup2(log_fd, STDOUT_FILENO); close(log_fd); } } - if (*bind_stderr != 0) { - Debug("log", "binding stderr to %s", bind_stderr); - log_fd = elevating_open(bind_stderr, O_WRONLY | O_APPEND | O_CREAT | O_SYNC, 0644); + if (*bind_stderr_p != 0) { + Debug("log", "binding stderr to %s", bind_stderr_p); + log_fd = elevating_open(bind_stderr_p, O_WRONLY | O_APPEND | O_CREAT | O_SYNC, 0644); if (log_fd < 0) { - fprintf(stdout, "[Warning]: TS unable to open log file \"%s\" [%d '%s']\n", bind_stderr, errno, strerror(errno)); + fprintf(stdout, "[Warning]: TS unable to open log file \"%s\" [%d '%s']\n", bind_stderr_p, errno, strerror(errno)); } else { Debug("log", "duping stderr"); dup2(log_fd, STDERR_FILENO); @@ -1707,12 +1855,6 @@ main(int /* argc ATS_UNUSED */, const char **argv) } #endif - // Check for core file - if (core_file[0] != '\0') { - process_core(core_file); - ::exit(0); - } - // setup callback for tracking remap included files load_remap_file_cb = load_remap_file_callback; @@ -1813,10 +1955,10 @@ main(int /* argc ATS_UNUSED */, const char **argv) quic_NetProcessor.init(); #endif - // If num_accept_threads == 0, let the ET_NET threads to set the condition variable, + // If num_accept_threads == 0, let the ET_NET threads set the condition variable, // Else we set it here so when checking the condition variable later it returns immediately. - if (num_accept_threads == 0) { - eventProcessor.schedule_spawn(&init_HttpProxyServer, ET_NET); + if (num_accept_threads == 0 || command_flag) { + eventProcessor.thread_group[ET_NET]._afterStartCallback = init_HttpProxyServer; } else { std::unique_lock lock(proxyServerMutex); et_net_threads_ready = true; @@ -1828,17 +1970,6 @@ main(int /* argc ATS_UNUSED */, const char **argv) // This means any spawn scheduling must be done before this point. eventProcessor.start(num_of_net_threads, stacksize); - int num_remap_threads = 0; - REC_ReadConfigInteger(num_remap_threads, "proxy.config.remap.num_remap_threads"); - if (num_remap_threads < 1) { - num_remap_threads = 0; - } - - if (num_remap_threads > 0) { - Note("using the new remap processor system with %d threads", num_remap_threads); - remapProcessor.setUseSeparateThread(); - } - eventProcessor.schedule_every(new SignalContinuation, HRTIME_MSECOND * 500, ET_CALL); eventProcessor.schedule_every(new DiagsLogContinuation, HRTIME_SECOND, ET_TASK); eventProcessor.schedule_every(new MemoryLimit, HRTIME_SECOND * 10, ET_TASK); @@ -1858,6 +1989,12 @@ main(int /* argc ATS_UNUSED */, const char **argv) int cmd_ret = cmd_mode(); if (cmd_ret != CMD_IN_PROGRESS) { + // Check the condition variable. + { + std::unique_lock lock(proxyServerMutex); + proxyServerCheck.wait(lock, [] { return et_net_threads_ready; }); + } + if (cmd_ret >= 0) { ::exit(0); // everything is OK } else { @@ -1865,7 +2002,6 @@ main(int /* argc ATS_UNUSED */, const char **argv) } } } else { - remapProcessor.start(num_remap_threads, stacksize); RecProcessStart(); initCacheControl(); IpAllow::startup(); @@ -1895,6 +2031,8 @@ main(int /* argc ATS_UNUSED */, const char **argv) // initialize logging (after event and net processor) Log::init(remote_management_flag ? 0 : Log::NO_REMOTE_MANAGEMENT); + (void)parsePluginConfig(); + // Init plugins as soon as logging is ready. (void)plugin_init(); // plugin.config @@ -2014,23 +2152,7 @@ main(int /* argc ATS_UNUSED */, const char **argv) delete main_thread; } -#if TS_HAS_TESTS -////////////////////////////// -// Unit Regression Test Hook // -////////////////////////////// - -#include "HdrTest.h" - -REGRESSION_TEST(Hdrs)(RegressionTest *t, int atype, int *pstatus) -{ - HdrTest ht; - *pstatus = ht.go(t, atype); - return; -} -#endif - -static void -mgmt_restart_shutdown_callback(ts::MemSpan) +static void mgmt_restart_shutdown_callback(ts::MemSpan) { sync_cache_dir_on_shutdown(); } @@ -2099,13 +2221,13 @@ init_ssl_ctx_callback(void *ctx, bool server) static void load_ssl_file_callback(const char *ssl_file) { - pmgmt->signalConfigFileChild("ssl_multicert.config", ssl_file); + pmgmt->signalConfigFileChild(ts::filename::SSL_MULTICERT, ssl_file); } static void load_remap_file_callback(const char *remap_file) { - pmgmt->signalConfigFileChild("remap.config", remap_file); + pmgmt->signalConfigFileChild(ts::filename::REMAP, remap_file); } static void @@ -2113,7 +2235,7 @@ task_threads_started_callback() { APIHook *hook = lifecycle_hooks->get(TS_LIFECYCLE_TASK_THREADS_READY_HOOK); while (hook) { - SCOPED_MUTEX_LOCK(lock, hook->m_cont->mutex, this_ethread()); + WEAK_SCOPED_MUTEX_LOCK(lock, hook->m_cont->mutex, this_ethread()); hook->invoke(TS_EVENT_LIFECYCLE_TASK_THREADS_READY, nullptr); hook = hook->next(); } diff --git a/src/traffic_top/stats.h b/src/traffic_top/stats.h index 1789fd0ed02..7b508491ff5 100644 --- a/src/traffic_top/stats.h +++ b/src/traffic_top/stats.h @@ -32,8 +32,6 @@ #include #include "mgmtapi.h" -using namespace std; - struct LookupItem { LookupItem(const char *s, const char *n, const int t) : pretty(s), name(n), numerator(""), denominator(""), type(t) {} LookupItem(const char *s, const char *n, const char *d, const int t) : pretty(s), name(n), numerator(n), denominator(d), type(t) @@ -49,7 +47,7 @@ extern size_t write_data(void *ptr, size_t size, size_t nmemb, void *stream); #if HAS_CURL extern char curl_error[CURL_ERROR_SIZE]; #endif -extern string response; +extern std::string response; namespace constant { @@ -62,6 +60,9 @@ const char end[] = "\",\n"; //---------------------------------------------------------------------------- class Stats { + using string = std::string; + template using map = std::map; + public: Stats(const string &url) : _url(url) { @@ -130,14 +131,14 @@ class Stats make_pair("client_curr_conn_h1", LookupItem("Curr Conn HTTP/1.x", "proxy.process.http.current_client_connections", 1))); lookup_table.insert( make_pair("client_curr_conn_h2", LookupItem("Curr Conn HTTP/2", "proxy.process.http2.current_client_connections", 1))); - lookup_table.insert(make_pair("client_curr_conn", LookupItem("Curr Conn", "client_curr_conn_h1", "client_curr_conn_h2", 6))); + lookup_table.insert(make_pair("client_curr_conn", LookupItem("Curr Conn", "client_curr_conn_h1", "client_curr_conn_h2", 9))); // current_active_client_connections lookup_table.insert(make_pair("client_actv_conn_h1", LookupItem("Active Con HTTP/1.x", "proxy.process.http.current_active_client_connections", 1))); lookup_table.insert(make_pair("client_actv_conn_h2", LookupItem("Active Con HTTP/2", "proxy.process.http2.current_active_client_connections", 1))); - lookup_table.insert(make_pair("client_actv_conn", LookupItem("Active Con", "client_actv_conn_h1", "client_actv_conn_h2", 6))); + lookup_table.insert(make_pair("client_actv_conn", LookupItem("Active Con", "client_actv_conn_h1", "client_actv_conn_h2", 9))); lookup_table.insert(make_pair("server_req", LookupItem("Requests", "proxy.process.http.outgoing_requests", 2))); lookup_table.insert(make_pair("server_conn", LookupItem("New Conn", "proxy.process.http.total_server_connections", 2))); @@ -395,6 +396,9 @@ class Stats void getStat(const string &key, double &value, string &prettyName, int &type, int overrideType = 0) { + // set default value + value = 0; + map::const_iterator lookup_it = lookup_table.find(key); assert(lookup_it != lookup_table.end()); const LookupItem &item = lookup_it->second; @@ -432,14 +436,22 @@ class Stats value *= 100; } } else if (type == 6 || type == 7) { - double numerator; - double denominator; - getStat(item.numerator, numerator, 2); - getStat(item.denominator, denominator, 2); - value = numerator + denominator; + // add rate + double first; + double second; + getStat(item.numerator, first, 2); + getStat(item.denominator, second, 2); + value = first + second; if (type == 7) { value *= 8; } + } else if (type == 9) { + // add + double first; + double second; + getStat(item.numerator, first); + getStat(item.denominator, second); + value = first + second; } if (type == 8) { @@ -524,6 +536,12 @@ class Stats } private: + std::pair + make_pair(std::string s, LookupItem i) + { + return std::make_pair(s, i); + } + map *_stats; map *_old_stats; map lookup_table; diff --git a/src/traffic_top/traffic_top.cc b/src/traffic_top/traffic_top.cc index 539e9853c7b..d33d682e1c0 100644 --- a/src/traffic_top/traffic_top.cc +++ b/src/traffic_top/traffic_top.cc @@ -228,10 +228,11 @@ help(const string &host, const string &version) while (true) { clear(); - time_t now = time(nullptr); - struct tm *nowtm = localtime(&now); + time_t now = time(nullptr); + struct tm nowtm; char timeBuf[32]; - strftime(timeBuf, sizeof(timeBuf), "%H:%M:%S", nowtm); + localtime_r(&now, &nowtm); + strftime(timeBuf, sizeof(timeBuf), "%H:%M:%S", &nowtm); // clear(); attron(A_BOLD); @@ -400,7 +401,7 @@ main(int argc, const char **argv) version.setup(PACKAGE_NAME, "traffic_top", PACKAGE_VERSION, __DATE__, __TIME__, BUILD_MACHINE, BUILD_PERSON, ""); const ArgumentDescription argument_descriptions[] = { - {"sleep", 's', "Enable debugging output", "I", &sleep_time, nullptr, nullptr}, + {"sleep", 's', "Sets the delay between updates (in seconds)", "I", &sleep_time, nullptr, nullptr}, HELP_ARGUMENT_DESCRIPTION(), VERSION_ARGUMENT_DESCRIPTION(), RUNROOT_ARGUMENT_DESCRIPTION(), @@ -467,10 +468,11 @@ main(int argc, const char **argv) attron(A_BOLD); string version; - time_t now = time(nullptr); - struct tm *nowtm = localtime(&now); + time_t now = time(nullptr); + struct tm nowtm; char timeBuf[32]; - strftime(timeBuf, sizeof(timeBuf), "%H:%M:%S", nowtm); + localtime_r(&now, &nowtm); + strftime(timeBuf, sizeof(timeBuf), "%H:%M:%S", &nowtm); stats.getStat("version", version); mvprintw(23, 0, "%-20.20s %30s (q)uit (h)elp (%c)bsolute ", host.c_str(), page_alt.c_str(), absolute ? 'A' : 'a'); diff --git a/src/traffic_via/traffic_via.cc b/src/traffic_via/traffic_via.cc index 197032bc8e3..65fb617e465 100644 --- a/src/traffic_via/traffic_via.cc +++ b/src/traffic_via/traffic_via.cc @@ -24,11 +24,11 @@ #include "tscore/ink_platform.h" #include "tscore/ink_args.h" #include "tscore/I_Version.h" -#include "tscore/Tokenizer.h" -#include "tscore/TextBuffer.h" #include "mgmtapi.h" #include #include +#include +#include #include "tscore/Regex.h" /// XXX Use DFA or Regex wrappers? @@ -43,11 +43,11 @@ static AppVersionInfo appVersionInfo; struct VIA { - VIA(const char *t) : title(t), next(nullptr) { memset(viaData, 0, sizeof(viaData)); } + VIA(const char *t) : title(t) {} ~VIA() { delete next; } const char *title; - const char *viaData[128]; - VIA *next; + const char *viaData[128] = {}; // zero initialize + VIA *next = nullptr; }; // Function to get via header table for every field/category in the via header @@ -186,7 +186,7 @@ standardViaLookup(char flag) // Function to print via header static void -printViaHeader(const char *header) +printViaHeader(std::string_view header) { VIA *viaTable = nullptr; VIA *viaEntry = nullptr; @@ -195,22 +195,22 @@ printViaHeader(const char *header) printf("Via Header Details:\n"); // Loop through input via header flags - for (const char *c = header; *c; ++c) { - if ((*c == ':') || (*c == ';')) { + for (char c : header) { + if ((c == ':') || (c == ';')) { isDetail = true; continue; } - if (islower(*c)) { + if (islower(c)) { // Get the via header table delete viaTable; - viaEntry = viaTable = isDetail ? detailViaLookup(*c) : standardViaLookup(*c); + viaEntry = viaTable = isDetail ? detailViaLookup(c) : standardViaLookup(c); } else { // This is a one of the sequence of (uppercase) VIA codes. if (viaEntry) { + unsigned char idx = c; printf("%-55s:", viaEntry->title); - printf("%s\n", viaEntry->viaData[static_cast(*c)] ? viaEntry->viaData[static_cast(*c)] : - "Invalid sequence"); + printf("%s\n", viaEntry->viaData[idx] ? viaEntry->viaData[idx] : "Invalid sequence"); viaEntry = viaEntry->next; } } @@ -220,32 +220,29 @@ printViaHeader(const char *header) // Check validity of via header and then decode it static TSMgmtError -decodeViaHeader(const char *str) +decodeViaHeader(std::string_view text) { - size_t viaHdrLength = strlen(str); - char tmp[viaHdrLength + 2]; - char *Via = tmp; - - memcpy(Via, str, viaHdrLength); - Via[viaHdrLength] = '\0'; // null terminate - // Via header inside square brackets - if (Via[0] == '[' && Via[viaHdrLength - 1] == ']') { - viaHdrLength = viaHdrLength - 2; - Via++; - Via[viaHdrLength] = '\0'; // null terminate the string after trimming + if (!text.empty() && text.front() == '[' && text.back() == ']') { + text.remove_prefix(1); + text.remove_suffix(1); + } + if (text.empty()) { + return TS_ERR_FAIL; } - printf("Via header is [%s], Length is %zu\n", Via, viaHdrLength); + printf("Via header is [%.*s], Length is %zu\n", int(text.size()), text.data(), text.size()); - if (viaHdrLength == 5) { - Via = strcat(Via, " "); // Add one space character before decoding via header - ++viaHdrLength; + char extender[6]; + if (text.size() == 5) { // add a trailing space in this case. + memcpy(extender, text.data(), text.size()); + extender[5] = ' '; + text = std::string_view{extender, 6}; } - if (viaHdrLength == 22 || viaHdrLength == 6) { + if (text.size() == 22 || text.size() == 6) { // Decode via header - printViaHeader(Via); + printViaHeader(text); return TS_ERR_OKAY; } // Invalid header size, come out. @@ -269,8 +266,7 @@ filterViaHeader() int i; const char *viaPattern = R"(\[([ucsfpe]+[^\]]+)\])"; // Regex to match via header with in [] which can start with character class ucsfpe - char *viaHeaderString; - char viaHeader[1024]; + std::string line; // Compile PCRE via header pattern compiledReg = pcre_compile(viaPattern, 0, &err, &errOffset, nullptr); @@ -281,15 +277,9 @@ filterViaHeader() } // Read all lines from stdin - while (fgets(viaHeader, sizeof(viaHeader), stdin)) { - // Trim new line character and null terminate it - char *newLinePtr = strchr(viaHeader, '\n'); - if (newLinePtr) { - *newLinePtr = '\0'; - } + while (std::getline(std::cin, line)) { // Match for via header pattern - pcreExecCode = pcre_exec(compiledReg, extraReg, viaHeader, static_cast(sizeof(viaHeader)), 0, 0, subStringVector, - SUBSTRING_VECTOR_COUNT); + pcreExecCode = pcre_exec(compiledReg, extraReg, line.data(), line.size(), 0, 0, subStringVector, SUBSTRING_VECTOR_COUNT); // Match failed, don't worry. Continue to next line. if (pcreExecCode < 0) { @@ -304,14 +294,9 @@ filterViaHeader() // Loop based on number of matches found for (i = 1; i < pcreExecCode; i++) { - // Point to beginning of matched substring - char *subStringBegin = viaHeader + subStringVector[2 * i]; - // Get length of matched substring - int subStringLen = subStringVector[2 * i + 1] - subStringVector[2 * i]; - viaHeaderString = subStringBegin; - sprintf(viaHeaderString, "%.*s", subStringLen, subStringBegin); + std::string_view match{line.data() + subStringVector[2 * i], size_t(subStringVector[2 * i + 1] - subStringVector[2 * i])}; // Decode matched substring - decodeViaHeader(viaHeaderString); + decodeViaHeader(match); } } return TS_ERR_OKAY; @@ -338,7 +323,7 @@ main(int /* argc ATS_UNUSED */, const char **argv) // Filter arguments provided from stdin status = filterViaHeader(); } else { - status = decodeViaHeader(file_arguments[i]); + status = decodeViaHeader(std::string_view{file_arguments[i], strlen(file_arguments[i])}); } if (status != TS_ERR_OKAY) { diff --git a/src/tscore/Arena.cc b/src/tscore/Arena.cc index 81af84cff65..e26a20a6a01 100644 --- a/src/tscore/Arena.cc +++ b/src/tscore/Arena.cc @@ -48,8 +48,8 @@ blk_alloc(int size) } blk->next = nullptr; - blk->m_heap_end = &blk->data[size]; - blk->m_water_level = &blk->data[0]; + blk->m_heap_end = blk->data + size; + blk->m_water_level = blk->data; return blk; } diff --git a/src/tscore/BaseLogFile.cc b/src/tscore/BaseLogFile.cc index 1dac88fef52..280ad17068b 100644 --- a/src/tscore/BaseLogFile.cc +++ b/src/tscore/BaseLogFile.cc @@ -339,18 +339,32 @@ BaseLogFile::open_file(int perm) return LOG_FILE_NO_ERROR; } -/* - * Closes actual log file, not metainfo +/** + * @brief Close the managed log file. + * + * @note This closes the actual log file, not its metainfo. + * + * @return The result of calling fclose on the file descriptor or 0 if the file + * was not open. If the result is non-zero, fclose failed and errno is set + * appropriately. */ -void +int BaseLogFile::close_file() { + int ret = 0; if (is_open()) { - fclose(m_fp); log_log_trace("BaseLogFile %s is closed\n", m_name.get()); + + // Both log_log_trace and fclose may set errno. Thus, keep fclose after + // log_log_trace so that by the time this function completes if errno was + // set by fclose it will remain upon function return. + ret = fclose(m_fp); m_fp = nullptr; m_is_init = false; + delete m_meta_info; + m_meta_info = nullptr; } + return ret; } /* diff --git a/src/tscore/BufferWriterFormat.cc b/src/tscore/BufferWriterFormat.cc index 0060a2c673b..b1070227974 100644 --- a/src/tscore/BufferWriterFormat.cc +++ b/src/tscore/BufferWriterFormat.cc @@ -467,17 +467,18 @@ namespace bw_fmt } w.write(digits); } else { // use generic Write_Aligned - Write_Aligned(w, - [&]() { - if (prefix1) { - w.write(prefix1); - if (prefix2) { - w.write(prefix2); - } - } - w.write(digits); - }, - spec._align, width, spec._fill, neg); + Write_Aligned( + w, + [&]() { + if (prefix1) { + w.write(prefix1); + if (prefix2) { + w.write(prefix2); + } + } + w.write(digits); + }, + spec._align, width, spec._fill, neg); } return w; } @@ -574,13 +575,14 @@ namespace bw_fmt std::string_view whole_digits{whole + sizeof(whole) - l, l}; std::string_view frac_digits{fraction + sizeof(fraction) - r, r}; - Write_Aligned(w, - [&]() { - w.write(whole_digits); - w.write(dec); - w.write(frac_digits); - }, - spec._align, width, spec._fill, neg); + Write_Aligned( + w, + [&]() { + w.write(whole_digits); + w.write(dec); + w.write(frac_digits); + }, + spec._align, width, spec._fill, neg); return w; } @@ -611,7 +613,8 @@ bwformat(BufferWriter &w, BWFSpec const &spec, std::string_view sv) return bwformat(w, spec, bwf::detail::MemDump(sv.data(), sv.size())); } else { width -= sv.size(); - bw_fmt::Write_Aligned(w, [&w, &sv]() { w.write(sv); }, spec._align, width, spec._fill, 0); + bw_fmt::Write_Aligned( + w, [&w, &sv]() { w.write(sv); }, spec._align, width, spec._fill, 0); } return w; } @@ -963,8 +966,8 @@ bwformat(BufferWriter &w, BWFSpec const &spec, bwf::detail::MemDump const &hex) w.write(fmt_type); width -= 2; } - bw_fmt::Write_Aligned(w, [&w, &hex, digits]() { bw_fmt::Format_As_Hex(w, hex._view, digits); }, spec._align, width, spec._fill, - 0); + bw_fmt::Write_Aligned( + w, [&w, &hex, digits]() { bw_fmt::Format_As_Hex(w, hex._view, digits); }, spec._align, width, spec._fill, 0); return w; } diff --git a/src/tscore/Diags.cc b/src/tscore/Diags.cc index 4463d17de0f..bc143f06da8 100644 --- a/src/tscore/Diags.cc +++ b/src/tscore/Diags.cc @@ -88,15 +88,17 @@ location(const SourceLocation *loc, DiagsShowLocation show, DiagsLevel level) // ////////////////////////////////////////////////////////////////////////////// -Diags::Diags(const char *prefix_string, const char *bdt, const char *bat, BaseLogFile *_diags_log, int dl_perm, int ol_perm) +Diags::Diags(std::string_view prefix_string, const char *bdt, const char *bat, BaseLogFile *_diags_log, int dl_perm, int ol_perm) : diags_log(nullptr), stdout_log(nullptr), stderr_log(nullptr), magic(DIAGS_MAGIC), show_location(SHOW_LOCATION_NONE), base_debug_tags(nullptr), - base_action_tags(nullptr) + base_action_tags(nullptr), + prefix_str(prefix_string) { + ink_release_assert(!prefix_str.empty()); int i; cleanup_func = nullptr; @@ -116,11 +118,8 @@ Diags::Diags(const char *prefix_string, const char *bdt, const char *bat, BaseLo config.enabled[DiagsTagType_Debug] = (base_debug_tags != nullptr); config.enabled[DiagsTagType_Action] = (base_action_tags != nullptr); diags_on_for_plugins = config.enabled[DiagsTagType_Debug]; - prefix_str = prefix_string; // The caller must always provide a non-empty prefix. - ink_release_assert(prefix_str); - ink_release_assert(*prefix_str); for (i = 0; i < DiagsLevel_Count; i++) { config.outputs[i].to_stdout = false; @@ -461,15 +460,12 @@ Diags::dump(FILE *fp) const void Diags::error_va(DiagsLevel level, const SourceLocation *loc, const char *format_string, va_list ap) const { - va_list ap2; - - if (DiagsLevel_IsTerminal(level)) { - va_copy(ap2, ap); - } - print_va(nullptr, level, loc, format_string, ap); if (DiagsLevel_IsTerminal(level)) { + va_list ap2; + + va_copy(ap2, ap); if (cleanup_func) { cleanup_func(); } @@ -480,9 +476,8 @@ Diags::error_va(DiagsLevel level, const SourceLocation *loc, const char *format_ } else { ink_fatal_va(format_string, ap2); } + va_end(ap2); } - - va_end(ap2); } /* diff --git a/src/tscore/EventNotify.cc b/src/tscore/EventNotify.cc index a18decc3021..fb88fefcf12 100644 --- a/src/tscore/EventNotify.cc +++ b/src/tscore/EventNotify.cc @@ -66,7 +66,7 @@ EventNotify::signal() #ifdef HAVE_EVENTFD uint64_t value = 1; // - // If the addition would cause the counter’s value of eventfd + // If the addition would cause the counter's value of eventfd // to exceed the maximum, write() will fail with the errno EAGAIN, // which is acceptable as the receiver will be notified eventually. // @@ -104,7 +104,8 @@ EventNotify::wait() #endif } -int EventNotify::timedwait(int timeout) // milliseconds +int +EventNotify::timedwait(int timeout) // milliseconds { #ifdef HAVE_EVENTFD ssize_t nr, nr_fd = 0; diff --git a/src/tscore/Extendible.cc b/src/tscore/Extendible.cc index 2ca6237223b..7d39f8d968b 100644 --- a/src/tscore/Extendible.cc +++ b/src/tscore/Extendible.cc @@ -44,7 +44,7 @@ namespace details void Schema::updateMemOffsets() { - ink_release_assert(instance_count == 0); + ink_release_assert(cnt_constructed == cnt_destructed); uint32_t acc_offset = 0; alloc_align = 1; @@ -86,7 +86,7 @@ namespace details bool Schema::reset() { - if (instance_count > 0) { + if (cnt_constructed > cnt_destructed) { // free instances before calling this so we don't leak memory return false; } @@ -99,7 +99,9 @@ namespace details Schema::callConstructor(uintptr_t ext_loc) { ink_assert(ext_loc); - ++instance_count; // don't allow schema modification + ++cnt_fld_constructed; // don't allow schema modification + ink_assert(cnt_fld_constructed <= cnt_constructed); + // init all extendible memory to 0, incase constructors don't memset(reinterpret_cast(ext_loc), 0, alloc_size); @@ -119,7 +121,6 @@ namespace details elm.second.destructor(FieldPtr(ext_loc + elm.second.field_offset)); } } - --instance_count; } size_t @@ -132,7 +133,7 @@ namespace details bool Schema::no_instances() const { - return instance_count == 0; + return cnt_constructed == cnt_destructed; } } // namespace details diff --git a/src/tscore/HostLookup.cc b/src/tscore/HostLookup.cc index 72759d988cc..b70a45a3511 100644 --- a/src/tscore/HostLookup.cc +++ b/src/tscore/HostLookup.cc @@ -211,7 +211,7 @@ struct CharIndexBlock { // ----------- ------------ // 0 | | | | | | // . | | | | | | -// CharIndexBlock . | | | | | | +// CharIndexBlock . | | | | | | // ---------- . | | | | | | // 0 | | | . | | | |-->23| ptr| 0 | (ptr is to the // . | | | |-------->25| 0 | -----| | | | hostBranch for @@ -277,8 +277,8 @@ CharIndex::~CharIndex() { // clean up the illegal key table. if (illegalKey) { - for (auto spot = illegalKey->begin(), limit = illegalKey->end(); spot != limit; delete &*(spot++)) { - ; // empty + for (auto &item : *illegalKey) { + delete item.second; } } } @@ -301,7 +301,7 @@ CharIndex::Insert(string_view match_data, HostBranch *toInsert) illegalKey.reset(new Table); } toInsert->key = match_data; - illegalKey->emplace(match_data, toInsert); + illegalKey->emplace(toInsert->key, toInsert); } else { while (true) { index = asciiToTable[static_cast(match_data.front())]; @@ -388,13 +388,15 @@ CharIndex::end() -> iterator return {}; } -auto CharIndex::iterator::operator-> () -> value_type * +auto +CharIndex::iterator::operator->() -> value_type * { ink_assert(state.block != nullptr); // clang! return state.block->array[state.index].branch; } -auto CharIndex::iterator::operator*() -> value_type & +auto +CharIndex::iterator::operator*() -> value_type & { ink_assert(state.block != nullptr); // clang! return *(state.block->array[state.index].branch); @@ -428,9 +430,14 @@ CharIndex::iterator::advance() -> self_type & break; } else if (state.block->array[state.index].block != nullptr) { // There is a lower level block to iterate over, store our current state and descend - q[cur_level++] = state; - state.block = state.block->array[state.index].block.get(); - state.index = 0; + if (static_cast(q.size()) <= cur_level) { + q.push_back(state); + } else { + q[cur_level] = state; + } + cur_level++; + state.block = state.block->array[state.index].block.get(); + state.index = 0; } else { ++state.index; } @@ -556,8 +563,9 @@ HostBranch::~HostBranch() break; case HOST_HASH: { HostTable *ht = next_level._table; - for (auto spot = ht->begin(), limit = ht->end(); spot != limit; delete &*(spot++)) { - } // empty + for (auto &item : *ht) { + delete item.second; + } delete ht; } break; case HOST_INDEX: { @@ -675,10 +683,10 @@ HostLookup::InsertBranch(HostBranch *insert_in, string_view level_data) ink_release_assert(0); break; case HostBranch::HOST_HASH: - insert_in->next_level._table->emplace(level_data, new_branch); + insert_in->next_level._table->emplace(new_branch->key, new_branch); break; case HostBranch::HOST_INDEX: - insert_in->next_level._index->Insert(level_data, new_branch); + insert_in->next_level._index->Insert(new_branch->key, new_branch); break; case HostBranch::HOST_ARRAY: { auto array = insert_in->next_level._array; @@ -686,9 +694,9 @@ HostLookup::InsertBranch(HostBranch *insert_in, string_view level_data) // The array is out of space, time to move to a hash table auto ha = insert_in->next_level._array; auto ht = new HostTable; - ht->emplace(level_data, new_branch); + ht->emplace(new_branch->key, new_branch); for (auto &item : *array) { - ht->emplace(item.match_data, item.branch); + ht->emplace(item.branch->key, item.branch); } // Ring out the old, ring in the new delete ha; @@ -785,15 +793,15 @@ HostLookup::TableInsert(string_view match_data, int index, bool domain_record) // leaf node to make sure we have a match if (domain_record == false) { if (match.empty()) { - leaf_array[index].type = HostLeaf::HOST_PARTIAL; - } else { leaf_array[index].type = HostLeaf::HOST_COMPLETE; + } else { + leaf_array[index].type = HostLeaf::HOST_PARTIAL; } } else { if (match.empty()) { - leaf_array[index].type = HostLeaf::DOMAIN_PARTIAL; - } else { leaf_array[index].type = HostLeaf::DOMAIN_COMPLETE; + } else { + leaf_array[index].type = HostLeaf::DOMAIN_PARTIAL; } } diff --git a/src/tscore/IpMap.cc b/src/tscore/IpMap.cc index c24445fc832..481ea0307f7 100644 --- a/src/tscore/IpMap.cc +++ b/src/tscore/IpMap.cc @@ -180,7 +180,7 @@ namespace detail */ bool contains(ArgType target, ///< Search target value. void **ptr = nullptr ///< Client data return. - ) const; + ) const; /** Remove all addresses in the map. @@ -339,22 +339,22 @@ namespace detail N *n = this->lowerBound(rmin); N *x = nullptr; // New node (if any). // Need copies because we will modify these. - Metric min = N::deref(rmin); - Metric max = N::deref(rmax); + Metric localmin = N::deref(rmin); + Metric localmax = N::deref(rmax); // Handle cases involving a node of interest to the left of the // range. if (n) { - if (n->_min < min) { - Metric min_1 = min; + if (n->_min < localmin) { + Metric min_1 = localmin; N::dec(min_1); // dec is OK because min isn't zero. if (n->_max < min_1) { // no overlap or adj. n = next(n); - } else if (n->_max >= max) { // incoming range is covered, just discard. + } else if (n->_max >= localmax) { // incoming range is covered, just discard. return *this; } else if (n->_data != payload) { // different payload, clip range on left. - min = n->_max; - N::inc(min); + localmin = n->_max; + N::inc(localmin); n = next(n); } else { // skew overlap with same payload, use node and continue. x = n; @@ -371,7 +371,7 @@ namespace detail // Careful here -- because max_plus1 might wrap we need to use it only if we can be certain it // didn't. This is done by ordering the range tests so that when max_plus1 is used when we know // there exists a larger value than max. - Metric max_plus1 = max; + Metric max_plus1 = localmax; N::inc(max_plus1); /* Notes: @@ -381,7 +381,7 @@ namespace detail while (n) { if (n->_data == payload) { if (x) { - if (n->_max <= max) { // next range is covered, so we can remove and continue. + if (n->_max <= localmax) { // next range is covered, so we can remove and continue. #if defined(__clang_analyzer__) x->_next = n->_next; // done in @c remove, but CA doesn't realize that. // It's insufficient to assert(x->_next != n) after the remove. @@ -395,52 +395,52 @@ namespace detail return *this; } else { // have the space to finish off the range. - x->setMax(max); + x->setMax(localmax); return *this; } - } else { // not carrying a span. - if (n->_max <= max) { // next range is covered - use it. + } else { // not carrying a span. + if (n->_max <= localmax) { // next range is covered - use it. x = n; - x->setMin(min); + x->setMin(localmin); n = next(n); } else if (n->_min <= max_plus1) { - n->setMin(min); + n->setMin(localmin); return *this; } else { // no overlap, space to complete range. - this->insert_before(n, new N(min, max, payload)); + this->insert_before(n, new N(localmin, localmax, payload)); return *this; } } } else { // different payload if (x) { - if (max < n->_min) { // range ends before n starts, done. - x->setMax(max); + if (localmax < n->_min) { // range ends before n starts, done. + x->setMax(localmax); return *this; - } else if (max <= n->_max) { // range ends before n, done. + } else if (localmax <= n->_max) { // range ends before n, done. x->setMaxMinusOne(n->_min); return *this; } else { // n is contained in range, skip over it. x->setMaxMinusOne(n->_min); - x = nullptr; - min = n->_max; - N::inc(min); // OK because n->_max maximal => next is null. + x = nullptr; + localmin = n->_max; + N::inc(localmin); // OK because n->_max maximal => next is null. n = next(n); } - } else { // no carry node. - if (max < n->_min) { // entirely before next span. - this->insert_before(n, new N(min, max, payload)); + } else { // no carry node. + if (localmax < n->_min) { // entirely before next span. + this->insert_before(n, new N(localmin, localmax, payload)); return *this; } else { - if (min < n->_min) { // leading section, need node. - N *y = new N(min, n->_min, payload); + if (localmin < n->_min) { // leading section, need node. + N *y = new N(localmin, n->_min, payload); y->decrementMax(); this->insert_before(n, y); } - if (max <= n->_max) { // nothing past node + if (localmax <= n->_max) { // nothing past node return *this; } - min = n->_max; - N::inc(min); + localmin = n->_max; + N::inc(localmin); n = next(n); } } @@ -448,9 +448,9 @@ namespace detail } // Invariant: min is larger than any existing range maximum. if (x) { - x->setMax(max); + x->setMax(localmax); } else { - this->append(new N(min, max, payload)); + this->append(new N(localmin, localmax, payload)); } return *this; } diff --git a/src/tscore/Makefile.am b/src/tscore/Makefile.am index 755df4ad664..c342179c890 100644 --- a/src/tscore/Makefile.am +++ b/src/tscore/Makefile.am @@ -19,7 +19,11 @@ include $(top_srcdir)/build/tidy.mk noinst_PROGRAMS = mkdfa CompileParseRules -check_PROGRAMS = test_atomic test_freelist test_geometry test_X509HostnameValidator test_tscore +check_PROGRAMS = test_geometry test_X509HostnameValidator test_tscore + +if EXPENSIVE_TESTS +check_PROGRAMS += test_atomic test_freelist +endif TESTS_ENVIRONMENT = LSAN_OPTIONS=suppressions=$(abs_top_srcdir)/ci/asan_leak_suppression/unit_tests.txt diff --git a/src/tscore/RbTree.cc b/src/tscore/RbTree.cc index 156899725a4..23bb41741de 100644 --- a/src/tscore/RbTree.cc +++ b/src/tscore/RbTree.cc @@ -301,7 +301,7 @@ namespace detail parent = n->_parent; d = NONE; // Cancel any leaf node logic } else { - if (wfc->_color == BLACK) { + if (wfc == BLACK) { w->getChild(near)->_color = BLACK; w->_color = RED; w->rotate(far); diff --git a/src/tscore/Regex.cc b/src/tscore/Regex.cc index 33487d18349..9d93c8d5247 100644 --- a/src/tscore/Regex.cc +++ b/src/tscore/Regex.cc @@ -28,7 +28,7 @@ #include "tscore/ink_memory.h" #include "tscore/Regex.h" -#ifdef PCRE_CONFIG_JIT +#if defined(PCRE_CONFIG_JIT) && !defined(darwin) // issue with macOS Catalina and pcre 8.43 struct RegexThreadKey { RegexThreadKey() { ink_thread_key_create(&this->key, reinterpret_cast(&pcre_jit_stack_free)); } ink_thread_key key; @@ -82,13 +82,13 @@ Regex::compile(const char *pattern, const unsigned flags) return false; } -#ifdef PCRE_CONFIG_JIT +#if defined(PCRE_CONFIG_JIT) && !defined(darwin) // issue with macOS Catalina and pcre 8.43 study_opts |= PCRE_STUDY_JIT_COMPILE; #endif regex_extra = pcre_study(regex, study_opts, &error); -#ifdef PCRE_CONFIG_JIT +#if defined(PCRE_CONFIG_JIT) && !defined(darwin) // issue with macOS Catalina and pcre 8.43 if (regex_extra) { pcre_assign_jit_stack(regex_extra, &get_jit_stack, nullptr); } @@ -127,7 +127,7 @@ Regex::exec(std::string_view const &str, int *ovector, int ovecsize) Regex::~Regex() { if (regex_extra) { -#ifdef PCRE_CONFIG_JIT +#if defined(PCRE_CONFIG_JIT) && !defined(darwin) // issue with macOS Catalina and pcre 8.43 pcre_free_study(regex_extra); #else pcre_free(regex_extra); diff --git a/src/tscore/Tokenizer.cc b/src/tscore/Tokenizer.cc index 068b4e0a697..f40973f3b20 100644 --- a/src/tscore/Tokenizer.cc +++ b/src/tscore/Tokenizer.cc @@ -275,7 +275,8 @@ Tokenizer::addToken(char *startAddr, int length) } } -const char *Tokenizer::operator[](unsigned index) const +const char * +Tokenizer::operator[](unsigned index) const { const tok_node *cur_node = &start_node; unsigned cur_start = 0; diff --git a/src/tscore/ink_hrtime.cc b/src/tscore/ink_hrtime.cc index ad47f335f63..9bffa1a53ce 100644 --- a/src/tscore/ink_hrtime.cc +++ b/src/tscore/ink_hrtime.cc @@ -35,8 +35,10 @@ #if defined(freebsd) #include #include +#ifdef HAVE_SYS_SYSCTL_H #include #endif +#endif #include #include @@ -47,18 +49,19 @@ int64_to_str(char *buf, unsigned int buf_size, int64_t val, unsigned int *total_ char local_buf[local_buf_size]; bool using_local_buffer = false; bool negative = false; - char *out_buf; + char *out_buf = buf; + char *working_buf; if (buf_size < 22) { // int64_t may not fit in provided buffer, use the local one - out_buf = &local_buf[local_buf_size - 1]; + working_buf = &local_buf[local_buf_size - 1]; using_local_buffer = true; } else { - out_buf = &buf[buf_size - 1]; + working_buf = &buf[buf_size - 1]; } unsigned int num_chars = 1; // includes eos - *out_buf-- = 0; + *working_buf-- = 0; if (val < 0) { val = -val; @@ -66,11 +69,11 @@ int64_to_str(char *buf, unsigned int buf_size, int64_t val, unsigned int *total_ } if (val < 10) { - *out_buf-- = '0' + static_cast(val); + *working_buf-- = '0' + static_cast(val); ++num_chars; } else { do { - *out_buf-- = static_cast(val % 10) + '0'; + *working_buf-- = static_cast(val % 10) + '0'; val /= 10; ++num_chars; } while (val); @@ -81,10 +84,10 @@ int64_to_str(char *buf, unsigned int buf_size, int64_t val, unsigned int *total_ if (req_width) { // add minus sign if padding character is not 0 if (negative && pad_char != '0') { - *out_buf = '-'; + *working_buf = '-'; ++num_chars; } else { - out_buf++; + working_buf++; } if (req_width > buf_size) { req_width = buf_size; @@ -94,19 +97,19 @@ int64_to_str(char *buf, unsigned int buf_size, int64_t val, unsigned int *total_ num_padding = req_width - num_chars; switch (num_padding) { case 3: - *--out_buf = pad_char; + *--working_buf = pad_char; // fallthrough case 2: - *--out_buf = pad_char; + *--working_buf = pad_char; // fallthrough case 1: - *--out_buf = pad_char; + *--working_buf = pad_char; break; default: - for (unsigned int i = 0; i < num_padding; ++i, *--out_buf = pad_char) { + for (unsigned int i = 0; i < num_padding; ++i, *--working_buf = pad_char) { ; } } @@ -115,23 +118,23 @@ int64_to_str(char *buf, unsigned int buf_size, int64_t val, unsigned int *total_ // add minus sign if padding character is 0 if (negative && pad_char == '0') { if (num_padding) { - *out_buf = '-'; // overwrite padding + *working_buf = '-'; // overwrite padding } else { - *--out_buf = '-'; + *--working_buf = '-'; ++num_chars; } } } else if (negative) { - *out_buf = '-'; + *working_buf = '-'; ++num_chars; } else { - out_buf++; + working_buf++; } if (using_local_buffer) { if (num_chars <= buf_size) { - memcpy(buf, out_buf, num_chars); - out_buf = buf; + memcpy(buf, working_buf, num_chars); + // out_buf is already pointing to buf } else { // data does not fit return nullptr out_buf = nullptr; @@ -141,6 +144,7 @@ int64_to_str(char *buf, unsigned int buf_size, int64_t val, unsigned int *total_ if (total_chars) { *total_chars = num_chars; } + return out_buf; } diff --git a/src/tscore/ink_inet.cc b/src/tscore/ink_inet.cc index 648385070c9..6f781cc85c0 100644 --- a/src/tscore/ink_inet.cc +++ b/src/tscore/ink_inet.cc @@ -49,9 +49,11 @@ const std::string_view IP_PROTO_TAG_TLS_1_3("tls/1.3"sv); const std::string_view IP_PROTO_TAG_HTTP_0_9("http/0.9"sv); const std::string_view IP_PROTO_TAG_HTTP_1_0("http/1.0"sv); const std::string_view IP_PROTO_TAG_HTTP_1_1("http/1.1"sv); -const std::string_view IP_PROTO_TAG_HTTP_2_0("h2"sv); // HTTP/2 over TLS -const std::string_view IP_PROTO_TAG_HTTP_QUIC("hq-20"sv); // HTTP/0.9 over QUIC -const std::string_view IP_PROTO_TAG_HTTP_3("h3-20"sv); // HTTP/3 over QUIC +const std::string_view IP_PROTO_TAG_HTTP_2_0("h2"sv); // HTTP/2 over TLS +const std::string_view IP_PROTO_TAG_HTTP_QUIC("hq-29"sv); // HTTP/0.9 over QUIC +const std::string_view IP_PROTO_TAG_HTTP_3("h3-29"sv); // HTTP/3 over QUIC +const std::string_view IP_PROTO_TAG_HTTP_QUIC_D27("hq-27"sv); // HTTP/0.9 over QUIC (draft-27) +const std::string_view IP_PROTO_TAG_HTTP_3_D27("h3-27"sv); // HTTP/3 over QUIC (draft-27) const std::string_view UNIX_PROTO_TAG{"unix"sv}; @@ -64,7 +66,7 @@ ink_inet_addr(const char *s) uint32_t base = 10; if (nullptr == s) { - return htonl((uint32_t)-1); + return htonl(static_cast(-1)); } while (n < 4) { @@ -103,7 +105,7 @@ ink_inet_addr(const char *s) } if (*pc && !ParseRules::is_wslfcr(*pc)) { - return htonl((uint32_t)-1); + return htonl(static_cast(-1)); } switch (n) { @@ -111,21 +113,21 @@ ink_inet_addr(const char *s) return htonl(u[0]); case 2: if (u[0] > 0xff || u[1] > 0xffffff) { - return htonl((uint32_t)-1); + return htonl(static_cast(-1)); } return htonl((u[0] << 24) | u[1]); case 3: if (u[0] > 0xff || u[1] > 0xff || u[2] > 0xffff) { - return htonl((uint32_t)-1); + return htonl(static_cast(-1)); } return htonl((u[0] << 24) | (u[1] << 16) | u[2]); case 4: if (u[0] > 0xff || u[1] > 0xff || u[2] > 0xff || u[3] > 0xff) { - return htonl((uint32_t)-1); + return htonl(static_cast(-1)); } return htonl((u[0] << 24) | (u[1] << 16) | (u[2] << 8) | u[3]); } - return htonl((uint32_t)-1); + return htonl(static_cast(-1)); } const char * diff --git a/src/tscore/ink_queue.cc b/src/tscore/ink_queue.cc index 22f3b32382e..f4a2841c06f 100644 --- a/src/tscore/ink_queue.cc +++ b/src/tscore/ink_queue.cc @@ -227,9 +227,10 @@ freelist_new(InkFreeList *f) for (i = 0; i < f->chunk_size; i++) { char *a = (static_cast(FREELIST_POINTER(item))) + i * f->type_size; #ifdef DEADBEEF - const char str[4] = {(char)0xde, (char)0xad, (char)0xbe, (char)0xef}; - for (int j = 0; j < (int)f->type_size; j++) + const char str[4] = {static_cast(0xde), static_cast(0xad), static_cast(0xbe), static_cast(0xef)}; + for (int j = 0; j < static_cast(f->type_size); j++) { a[j] = str[j % 4]; + } #endif freelist_free(f, a); } @@ -240,12 +241,15 @@ freelist_new(InkFreeList *f) #ifdef SANITY if (result) { - if (FREELIST_POINTER(item) == TO_PTR(FREELIST_POINTER(next))) + if (FREELIST_POINTER(item) == TO_PTR(FREELIST_POINTER(next))) { ink_abort("ink_freelist_new: loop detected"); - if (((uintptr_t)(TO_PTR(FREELIST_POINTER(next)))) & 3) + } + if (((uintptr_t)(TO_PTR(FREELIST_POINTER(next)))) & 3) { ink_abort("ink_freelist_new: bad list"); - if (TO_PTR(FREELIST_POINTER(next))) - fake_global_for_ink_queue = *(int *)TO_PTR(FREELIST_POINTER(next)); + } + if (TO_PTR(FREELIST_POINTER(next))) { + fake_global_for_ink_queue = *static_cast(TO_PTR(FREELIST_POINTER(next))); + } } #endif /* SANITY */ } @@ -291,23 +295,27 @@ freelist_free(InkFreeList *f, void *item) #ifdef DEADBEEF { - static const char str[4] = {(char)0xde, (char)0xad, (char)0xbe, (char)0xef}; + static const char str[4] = {static_cast(0xde), static_cast(0xad), static_cast(0xbe), static_cast(0xef)}; // set the entire item to DEADBEEF - for (int j = 0; j < (int)f->type_size; j++) - ((char *)item)[j] = str[j % 4]; + for (int j = 0; j < static_cast(f->type_size); j++) { + (static_cast(item))[j] = str[j % 4]; + } } #endif /* DEADBEEF */ while (!result) { INK_QUEUE_LD(h, f->head); #ifdef SANITY - if (TO_PTR(FREELIST_POINTER(h)) == item) + if (TO_PTR(FREELIST_POINTER(h)) == item) { ink_abort("ink_freelist_free: trying to free item twice"); - if (((uintptr_t)(TO_PTR(FREELIST_POINTER(h)))) & 3) + } + if (((uintptr_t)(TO_PTR(FREELIST_POINTER(h)))) & 3) { ink_abort("ink_freelist_free: bad list"); - if (TO_PTR(FREELIST_POINTER(h))) - fake_global_for_ink_queue = *(int *)TO_PTR(FREELIST_POINTER(h)); + } + if (TO_PTR(FREELIST_POINTER(h))) { + fake_global_for_ink_queue = *static_cast(TO_PTR(FREELIST_POINTER(h))); + } #endif /* SANITY */ *adr_of_next = FREELIST_POINTER(h); SET_FREELIST_POINTER_VERSION(item_pair, FROM_PTR(item), FREELIST_VERSION(h)); @@ -347,13 +355,14 @@ freelist_bulkfree(InkFreeList *f, void *head, void *tail, size_t num_item) #ifdef DEADBEEF { - static const char str[4] = {(char)0xde, (char)0xad, (char)0xbe, (char)0xef}; + static const char str[4] = {static_cast(0xde), static_cast(0xad), static_cast(0xbe), static_cast(0xef)}; // set the entire item to DEADBEEF; void *temp = head; for (size_t i = 0; i < num_item; i++) { - for (int j = sizeof(void *); j < (int)f->type_size; j++) - ((char *)temp)[j] = str[j % 4]; + for (int j = sizeof(void *); j < static_cast(f->type_size); j++) { + (static_cast(temp))[j] = str[j % 4]; + } *ADDRESS_OF_NEXT(temp, 0) = FROM_PTR(*ADDRESS_OF_NEXT(temp, 0)); temp = TO_PTR(*ADDRESS_OF_NEXT(temp, 0)); } @@ -363,12 +372,15 @@ freelist_bulkfree(InkFreeList *f, void *head, void *tail, size_t num_item) while (!result) { INK_QUEUE_LD(h, f->head); #ifdef SANITY - if (TO_PTR(FREELIST_POINTER(h)) == head) + if (TO_PTR(FREELIST_POINTER(h)) == head) { ink_abort("ink_freelist_free: trying to free item twice"); - if (((uintptr_t)(TO_PTR(FREELIST_POINTER(h)))) & 3) + } + if (((uintptr_t)(TO_PTR(FREELIST_POINTER(h)))) & 3) { ink_abort("ink_freelist_free: bad list"); - if (TO_PTR(FREELIST_POINTER(h))) - fake_global_for_ink_queue = *(int *)TO_PTR(FREELIST_POINTER(h)); + } + if (TO_PTR(FREELIST_POINTER(h))) { + fake_global_for_ink_queue = *static_cast(TO_PTR(FREELIST_POINTER(h))); + } #endif /* SANITY */ *adr_of_next = FREELIST_POINTER(h); SET_FREELIST_POINTER_VERSION(item_pair, FROM_PTR(head), FREELIST_VERSION(h)); diff --git a/src/tscore/ink_res_init.cc b/src/tscore/ink_res_init.cc index be4dccce24d..fd293c7798c 100644 --- a/src/tscore/ink_res_init.cc +++ b/src/tscore/ink_res_init.cc @@ -168,8 +168,9 @@ ink_res_setoptions(ink_res_state statp, const char *options, const char *source int i; #ifdef DEBUG - if (statp->options & INK_RES_DEBUG) + if (statp->options & INK_RES_DEBUG) { printf(";; res_setoptions(\"%s\", \"%s\")...\n", options, source); + } #endif while (*cp) { /* skip leading and inner runs of spaces */ @@ -185,8 +186,9 @@ ink_res_setoptions(ink_res_state statp, const char *options, const char *source statp->ndots = INK_RES_MAXNDOTS; } #ifdef DEBUG - if (statp->options & INK_RES_DEBUG) + if (statp->options & INK_RES_DEBUG) { printf(";;\tndots=%d\n", statp->ndots); + } #endif } else if (!strncmp(cp, "timeout:", sizeof("timeout:") - 1)) { i = atoi(cp + sizeof("timeout:") - 1); @@ -196,8 +198,9 @@ ink_res_setoptions(ink_res_state statp, const char *options, const char *source statp->retrans = INK_RES_MAXRETRANS; } #ifdef DEBUG - if (statp->options & INK_RES_DEBUG) + if (statp->options & INK_RES_DEBUG) { printf(";;\ttimeout=%d\n", statp->retrans); + } #endif #ifdef SOLARIS2 } else if (!strncmp(cp, "retrans:", sizeof("retrans:") - 1)) { @@ -223,8 +226,9 @@ ink_res_setoptions(ink_res_state statp, const char *options, const char *source statp->retry = INK_RES_MAXRETRY; } #ifdef DEBUG - if (statp->options & INK_RES_DEBUG) + if (statp->options & INK_RES_DEBUG) { printf(";;\tattempts=%d\n", statp->retry); + } #endif } else if (!strncmp(cp, "debug", sizeof("debug") - 1)) { #ifdef DEBUG @@ -568,8 +572,9 @@ ink_res_init(ink_res_state statp, ///< State object to update. #ifdef DEBUG if (statp->options & INK_RES_DEBUG) { printf(";; res_init()... default dnsrch list:\n"); - for (pp = statp->dnsrch; *pp; pp++) + for (pp = statp->dnsrch; *pp; pp++) { printf(";;\t%s\n", *pp); + } printf(";;\t..END..\n"); } #endif diff --git a/src/tscore/ink_res_mkquery.cc b/src/tscore/ink_res_mkquery.cc index 4615816832d..eaffd041e8a 100644 --- a/src/tscore/ink_res_mkquery.cc +++ b/src/tscore/ink_res_mkquery.cc @@ -88,14 +88,15 @@ * Form all types of queries. * Returns the size of the result or -1. */ -int ink_res_mkquery(ink_res_state statp, int op, /*!< opcode of query */ - const char *dname, /*!< domain name */ - int _class, int type, /*!< _class and type of query */ - const u_char *data, /*!< resource record data */ - int datalen, /*!< length of data */ - const u_char * /* newrr_in ATS_UNUSED */, /*!< new rr for modify or append */ - u_char *buf, /*!< buffer to put query */ - int buflen) /*!< size of buffer */ +int +ink_res_mkquery(ink_res_state statp, int op, /*!< opcode of query */ + const char *dname, /*!< domain name */ + int _class, int type, /*!< _class and type of query */ + const u_char *data, /*!< resource record data */ + int datalen, /*!< length of data */ + const u_char * /* newrr_in ATS_UNUSED */, /*!< new rr for modify or append */ + u_char *buf, /*!< buffer to put query */ + int buflen) /*!< size of buffer */ { HEADER *hp; u_char *cp, *ep; @@ -510,13 +511,12 @@ ns_name_ntop(const u_char *src, char *dst, size_t dstsiz) } HostResStyle -ats_host_res_from(int family, HostResPreferenceOrder order) +ats_host_res_from(int family, HostResPreferenceOrder const &order) { bool v4 = false, v6 = false; HostResPreference client = AF_INET6 == family ? HOST_RES_PREFER_IPV6 : HOST_RES_PREFER_IPV4; - for (int i = 0; i < N_HOST_RES_PREFERENCE_ORDER; ++i) { - HostResPreference p = order[i]; + for (auto p : order) { if (HOST_RES_PREFER_CLIENT == p) { p = client; // CLIENT -> actual value } @@ -544,14 +544,16 @@ ats_host_res_from(int family, HostResPreferenceOrder order) return HOST_RES_NONE; } -HostResStyle -ats_host_res_match(sockaddr const *addr) +void +ats_force_order_by_family(sockaddr const *addr, HostResPreferenceOrder order) { - HostResStyle zret = HOST_RES_NONE; + int pos{0}; if (ats_is_ip6(addr)) { - zret = HOST_RES_IPV6_ONLY; + order[pos++] = HOST_RES_PREFER_IPV6; } else if (ats_is_ip4(addr)) { - zret = HOST_RES_IPV4_ONLY; + order[pos++] = HOST_RES_PREFER_IPV4; + } + for (; pos < N_HOST_RES_PREFERENCE_ORDER; pos++) { + order[pos] = HOST_RES_PREFER_NONE; } - return zret; } diff --git a/src/tscore/ink_rwlock.cc b/src/tscore/ink_rwlock.cc index 1d44876f12b..19f4c8eea59 100644 --- a/src/tscore/ink_rwlock.cc +++ b/src/tscore/ink_rwlock.cc @@ -29,126 +29,23 @@ // // Note: This should be called only once. //------------------------------------------------------------------------- -int +void ink_rwlock_init(ink_rwlock *rw) { - ink_mutex_init(&rw->rw_mutex); - - ink_cond_init(&rw->rw_condreaders); - ink_cond_init(&rw->rw_condwriters); - rw->rw_nwaitreaders = 0; - rw->rw_nwaitwriters = 0; - // coverity[missing_lock] - rw->rw_refcount = 0; - rw->rw_magic = RW_MAGIC; - - return 0; + int error = pthread_rwlock_init(rw, nullptr); + if (unlikely(error != 0)) { + ink_abort("pthread_rwlock_init(%p) failed: %s (%d)", rw, strerror(error), error); + } } //------------------------------------------------------------------------- // ink_rwlock_destroy //------------------------------------------------------------------------- - -int +void ink_rwlock_destroy(ink_rwlock *rw) { - if (rw->rw_magic != RW_MAGIC) { - return EINVAL; - } - if (rw->rw_refcount != 0 || rw->rw_nwaitreaders != 0 || rw->rw_nwaitwriters != 0) { - return EBUSY; + int error = pthread_rwlock_destroy(rw); + if (unlikely(error != 0)) { + ink_abort("pthread_rwlock_destroy(%p) failed: %s (%d)", rw, strerror(error), error); } - - ink_mutex_destroy(&rw->rw_mutex); - ink_cond_destroy(&rw->rw_condreaders); - ink_cond_destroy(&rw->rw_condwriters); - rw->rw_magic = 0; - - return 0; -} - -//------------------------------------------------------------------------- -// ink_rwlock_rdlock -//------------------------------------------------------------------------- - -int -ink_rwlock_rdlock(ink_rwlock *rw) -{ - if (rw->rw_magic != RW_MAGIC) { - return EINVAL; - } - - ink_mutex_acquire(&rw->rw_mutex); - - /* give preference to waiting writers */ - while (rw->rw_refcount < 0 || rw->rw_nwaitwriters > 0) { - rw->rw_nwaitreaders++; - ink_cond_wait(&rw->rw_condreaders, &rw->rw_mutex); - rw->rw_nwaitreaders--; - } - rw->rw_refcount++; /* another reader has a read lock */ - - ink_mutex_release(&rw->rw_mutex); - - return 0; -} - -//------------------------------------------------------------------------- -// ink_rwlock_wrlock -//------------------------------------------------------------------------- - -int -ink_rwlock_wrlock(ink_rwlock *rw) -{ - if (rw->rw_magic != RW_MAGIC) { - return EINVAL; - } - - ink_mutex_acquire(&rw->rw_mutex); - - while (rw->rw_refcount != 0) { - rw->rw_nwaitwriters++; - ink_cond_wait(&rw->rw_condwriters, &rw->rw_mutex); - rw->rw_nwaitwriters--; - } - rw->rw_refcount = -1; - - ink_mutex_release(&rw->rw_mutex); - - return 0; -} - -//------------------------------------------------------------------------- -// ink_rwlock_unlock -//------------------------------------------------------------------------- - -int -ink_rwlock_unlock(ink_rwlock *rw) -{ - if (rw->rw_magic != RW_MAGIC) { - return EINVAL; - } - - ink_mutex_acquire(&rw->rw_mutex); - - if (rw->rw_refcount > 0) { - rw->rw_refcount--; /* releasing a reader */ - } else if (rw->rw_refcount == -1) { - rw->rw_refcount = 0; /* releasing a reader */ - } else { - ink_abort("invalid refcount %d on ink_rwlock %p", rw->rw_refcount, rw); - } - - /* give preference to waiting writers over waiting readers */ - if (rw->rw_nwaitwriters > 0) { - if (rw->rw_refcount == 0) { - ink_cond_signal(&rw->rw_condwriters); - } - } else if (rw->rw_nwaitreaders > 0) { - ink_cond_broadcast(&rw->rw_condreaders); - } - - ink_mutex_release(&rw->rw_mutex); - - return 0; } diff --git a/src/tscore/test_atomic.cc b/src/tscore/test_atomic.cc index f498645a620..bb2ff0f1be7 100644 --- a/src/tscore/test_atomic.cc +++ b/src/tscore/test_atomic.cc @@ -139,7 +139,7 @@ cycle_data(void *d) #endif // LONG_ATOMICLIST_TEST int -main(int /* argc ATS_UNUSED */, const char * /* argv ATS_UNUSED */ []) +main(int /* argc ATS_UNUSED */, const char * /* argv ATS_UNUSED */[]) { #ifndef LONG_ATOMICLIST_TEST int32_t m = 1, n = 100; diff --git a/src/tscore/test_freelist.cc b/src/tscore/test_freelist.cc index 8c5776babf9..5106f5122a7 100644 --- a/src/tscore/test_freelist.cc +++ b/src/tscore/test_freelist.cc @@ -66,7 +66,7 @@ test(void *d) } int -main(int /* argc ATS_UNUSED */, char * /*argv ATS_UNUSED */ []) +main(int /* argc ATS_UNUSED */, char * /*argv ATS_UNUSED */[]) { int i; diff --git a/src/tscore/ts_file.cc b/src/tscore/ts_file.cc index c990adfa258..f77faaefe58 100644 --- a/src/tscore/ts_file.cc +++ b/src/tscore/ts_file.cc @@ -100,6 +100,13 @@ namespace file return path(); } + path + filename(const path &p) + { + const size_t last_slash_idx = p.string().find_last_of(p.preferred_separator); + return p.string().substr(last_slash_idx + 1); + } + bool exists(const path &p) { @@ -170,9 +177,8 @@ namespace file path final_to; file_status s = status(to, err); if (!(err && ENOENT == err.value()) && is_dir(s)) { - const size_t last_slash_idx = from.string().find_last_of(from.preferred_separator); - std::string filename = from.string().substr(last_slash_idx + 1); - final_to = to / filename; + const auto file = filename(from); + final_to = to / file; } else { final_to = to; } diff --git a/src/tscore/unit_tests/test_BufferWriterFormat.cc b/src/tscore/unit_tests/test_BufferWriterFormat.cc index a3ef870720d..90f95d899ea 100644 --- a/src/tscore/unit_tests/test_BufferWriterFormat.cc +++ b/src/tscore/unit_tests/test_BufferWriterFormat.cc @@ -29,7 +29,10 @@ #include "tscore/BufferWriter.h" #include "tscore/bwf_std_format.h" #include "tscpp/util/MemSpan.h" +#include "tscore/ink_config.h" +#if TS_ENABLE_FIPS == 0 #include "tscore/INK_MD5.h" +#endif #include "tscore/CryptoHash.h" using namespace std::literals; @@ -252,6 +255,7 @@ TEST_CASE("BWFormat numerics", "[bwprint][bwformat]") bw20.print("012345|{:^10s}|6789abc", true); REQUIRE(bw20.view() == "012345| true |67"); +#if TS_ENABLE_FIPS == 0 INK_MD5 md5; bw.reduce(0); bw.print("{}", md5); @@ -260,6 +264,7 @@ TEST_CASE("BWFormat numerics", "[bwprint][bwformat]") bw.reduce(0); bw.print("{}", md5); REQUIRE(bw.view() == "e99a18c428cb38d5f260853678922e03"); +#endif bw.reset().print("Char '{}'", 'a'); REQUIRE(bw.view() == "Char 'a'"); @@ -594,12 +599,14 @@ TEST_CASE("bwstring std formats", "[libts][bwprint]") w.reset().print("{} bytes {} digits {}", sizeof(double), std::numeric_limits::digits10, ts::bwf::Hex_Dump(2.718281828)); REQUIRE(w.view() == "8 bytes 15 digits 9b91048b0abf0540"); +#if TS_ENABLE_FIPS == 0 INK_MD5 md5; w.reset().print("{}", ts::bwf::Hex_Dump(md5)); REQUIRE(w.view() == "00000000000000000000000000000000"); CryptoContext().hash_immediate(md5, s2.data(), s2.size()); w.reset().print("{}", ts::bwf::Hex_Dump(md5)); REQUIRE(w.view() == "f240ccd7a95c7ec66d6c111e2925b23e"); +#endif } // Normally there's no point in running the performance tests, but it's worth keeping the code diff --git a/src/tscore/unit_tests/test_Extendible.cc b/src/tscore/unit_tests/test_Extendible.cc index a6b0210555a..6f3d306b08d 100644 --- a/src/tscore/unit_tests/test_Extendible.cc +++ b/src/tscore/unit_tests/test_Extendible.cc @@ -65,6 +65,8 @@ TEST_CASE("AtomicBit Atomic test") // Extendible Inheritance Tests struct A : public Extendible { + using self_type = A; + DEF_EXT_NEW_DEL(self_type); uint16_t a = {1}; }; @@ -74,18 +76,32 @@ class B : public A { public: using super_type = A; - uint16_t b = {2}; + using self_type = B; + DEF_EXT_NEW_DEL(self_type); + uint16_t b = {2}; }; class C : public B, public Extendible { public: using super_type = B; - uint16_t c = {3}; + using self_type = C; + DEF_EXT_NEW_DEL(self_type); + uint16_t c = {3}; // operator[] - template decltype(auto) operator[](F field) const { return ext::get(*this, field); } - template decltype(auto) operator[](F field) { return ext::set(*this, field); } + template + decltype(auto) + operator[](F field) const + { + return ext::get(*this, field); + } + template + decltype(auto) + operator[](F field) + { + return ext::set(*this, field); + } }; ext::FieldId> ext_c_1; @@ -95,6 +111,46 @@ memDelta(void *p, void *q) { return uintptr_t(q) - uintptr_t(p); } +A *a_ptr = nullptr; +TEST_CASE("Create A", "") +{ + ext::details::areFieldsFinalized() = true; + a_ptr = ext::create(); + CHECK(Extendible::schema.no_instances() == false); +} +TEST_CASE("Delete A", "") +{ + delete a_ptr; + CHECK(Extendible::schema.no_instances()); +} +TEST_CASE("Create B", "") +{ + a_ptr = ext::create(); + CHECK(Extendible::schema.no_instances() == false); +} +TEST_CASE("Delete B", "") +{ + delete a_ptr; + CHECK(Extendible::schema.no_instances()); +} +TEST_CASE("Create C", "") +{ + a_ptr = ext::create(); + CHECK(Extendible::schema.no_instances() == false); + CHECK(Extendible::schema.no_instances() == false); +} +TEST_CASE("Delete C", "") +{ + delete static_cast(a_ptr); + CHECK(Extendible::schema.no_instances()); + CHECK(Extendible::schema.no_instances()); + CHECK(Extendible::schema.cnt_constructed == 3); + CHECK(Extendible::schema.cnt_fld_constructed == 3); + CHECK(Extendible::schema.cnt_destructed == 3); + CHECK(Extendible::schema.cnt_constructed == 1); + CHECK(Extendible::schema.cnt_fld_constructed == 1); + CHECK(Extendible::schema.cnt_destructed == 1); +} TEST_CASE("Extendible Memory Allocations", "") { ext::details::areFieldsFinalized() = false; @@ -107,7 +163,7 @@ TEST_CASE("Extendible Memory Allocations", "") CHECK(ext::sizeOf() == w * 4); CHECK(ext::sizeOf() == w * 7); - C &x = *(ext::alloc()); + C &x = *(ext::create()); // 0 1 2 3 4 5 6 //[ EA*, a, b,EC*, c, EA, EC] // @@ -136,7 +192,7 @@ TEST_CASE("Extendible Memory Allocations", "") TEST_CASE("Extendible Pointer Math", "") { - C &x = *(ext::alloc()); + C &x = *(ext::create()); CHECK(x.a == 1); CHECK(x.b == 2); @@ -169,11 +225,23 @@ TEST_CASE("Extendible Pointer Math", "") // Extendible is abstract and must be derived in a CRTP struct Derived : Extendible { + using self_type = Derived; + DEF_EXT_NEW_DEL(self_type); string m_str; // operator[] for shorthand - template decltype(auto) operator[](F field) const { return ext::get(*this, field); } - template decltype(auto) operator[](F field) { return ext::set(*this, field); } + template + decltype(auto) + operator[](F field) const + { + return ext::get(*this, field); + } + template + decltype(auto) + operator[](F field) + { + return ext::set(*this, field); + } static const string testFormat() @@ -193,7 +261,7 @@ struct Derived : Extendible { void * DerivedExtalloc() { - return ext::alloc(); + return ext::create(); } void DerivedExtFree(void *ptr) @@ -265,7 +333,7 @@ TEST_CASE("Extendible", "") // I don't use SECTIONS because this modifies static variables many times, is not thread safe. INFO("Extendible()") { - ptr = ext::alloc(); + ptr = ext::create(); REQUIRE(ptr != nullptr); } @@ -277,7 +345,7 @@ TEST_CASE("Extendible", "") INFO("Schema Reset") { - ptr = ext::alloc(); + ptr = ext::create(); REQUIRE(Derived::schema.no_instances() == false); REQUIRE(Derived::schema.reset() == false); delete ptr; @@ -289,7 +357,7 @@ TEST_CASE("Extendible", "") INFO("shared_ptr") { - shared_ptr sptr(ext::alloc()); + shared_ptr sptr(ext::create()); REQUIRE(Derived::schema.no_instances() == false); REQUIRE(sptr); } @@ -306,7 +374,7 @@ TEST_CASE("Extendible", "") INFO("Extendible delete ptr"); { for (int i = 0; i < 10; i++) { - ptr = ext::alloc(); + ptr = ext::create(); REQUIRE(ptr != nullptr); INFO(__LINE__); REQUIRE(Derived::schema.no_instances() == false); @@ -319,7 +387,7 @@ TEST_CASE("Extendible", "") INFO("test bit field"); { - shared_ptr sptr{ext::alloc()}; + shared_ptr sptr{ext::create()}; Derived &ref = *sptr; CHECK(ext::viewFormat(ref) == Derived::testFormat()); @@ -356,7 +424,7 @@ TEST_CASE("Extendible", "") CHECK(ext::sizeOf() == expected_size); ext::details::areFieldsFinalized() = true; - shared_ptr sptr(ext::alloc()); + shared_ptr sptr(ext::create()); Derived &ref = *sptr; CHECK(ext::viewFormat(ref) == Derived::testFormat()); using Catch::Matchers::Contains; @@ -385,7 +453,7 @@ TEST_CASE("Extendible", "") size_t expected_size = sizeof(Derived) + 1 + sizeof(std::atomic_int) * 2; CHECK(ext::sizeOf() == expected_size); - shared_ptr sptr(ext::alloc()); + shared_ptr sptr(ext::create()); Derived &ref = *sptr; CHECK(ext::get(ref, int_a) == 0); CHECK(ext::get(ref, int_b) == 0); diff --git a/src/tscore/unit_tests/test_IntrusiveHashMap.cc b/src/tscore/unit_tests/test_IntrusiveHashMap.cc index f8f5b882e23..e182dd64f61 100644 --- a/src/tscore/unit_tests/test_IntrusiveHashMap.cc +++ b/src/tscore/unit_tests/test_IntrusiveHashMap.cc @@ -123,7 +123,7 @@ TEST_CASE("IntrusiveHashMap", "[libts][IntrusiveHashMap]") auto r = map.equal_range("dup"sv); REQUIRE(r.first != r.second); REQUIRE(r.first->_payload == "dup"sv); - REQUIRE(r.first->_n == 79); + REQUIRE(r.first->_n == 81); Map::iterator idx; @@ -138,13 +138,13 @@ TEST_CASE("IntrusiveHashMap", "[libts][IntrusiveHashMap]") REQUIRE(r.first != r.second); idx = r.first; REQUIRE(idx->_payload == "dup"sv); - REQUIRE(idx->_n == 79); + REQUIRE(idx->_n == 81); REQUIRE((++idx)->_payload == "dup"sv); REQUIRE(idx->_n != r.first->_n); - REQUIRE(idx->_n == 80); + REQUIRE(idx->_n == 79); REQUIRE((++idx)->_payload == "dup"sv); REQUIRE(idx->_n != r.first->_n); - REQUIRE(idx->_n == 81); + REQUIRE(idx->_n == 80); REQUIRE(++idx == map.end()); // Verify only the "dup" items are left. for (auto &&elt : map) { @@ -200,7 +200,7 @@ TEST_CASE("IntrusiveHashMapManyStrings", "[IntrusiveHashMap]") } for (int idx = 23; idx < N; idx += 23) { auto spot = ihm.find(strings[idx]); - if (spot == ihm.end() || spot->_n != idx || ++spot == ihm.end() || spot->_n != 2000 + idx) { + if (spot == ihm.end() || spot->_n != 2000 + idx || ++spot == ihm.end() || spot->_n != idx) { miss_p = true; } } @@ -213,7 +213,7 @@ TEST_CASE("IntrusiveHashMapManyStrings", "[IntrusiveHashMap]") } for (int idx = 31; idx < N; idx += 31) { auto spot = ihm.find(strings[idx]); - if (spot == ihm.end() || spot->_n != idx || ++spot == ihm.end() || (idx != (23 * 31) && spot->_n != 3000 + idx) || + if (spot == ihm.end() || spot->_n != 3000 + idx || ++spot == ihm.end() || (idx != (23 * 31) && spot->_n != idx) || (idx == (23 * 31) && spot->_n != 2000 + idx)) { miss_p = true; } diff --git a/src/tscore/unit_tests/test_Regex.cc b/src/tscore/unit_tests/test_Regex.cc index b0ee29b9067..e457345fd85 100644 --- a/src/tscore/unit_tests/test_Regex.cc +++ b/src/tscore/unit_tests/test_Regex.cc @@ -28,15 +28,15 @@ #include "tscore/Regex.h" #include "catch.hpp" -typedef struct { +struct subject_match_t { std::string_view subject; bool match; -} subject_match_t; +}; -typedef struct { +struct test_t { std::string_view regex; std::array tests; -} test_t; +}; std::array test_data{{{{"^foo"}, {{{{"foo"}, true}, {{"bar"}, false}, {{"foobar"}, true}, {{"foobarbaz"}, true}}}}, {{"foo$"}, {{{{"foo"}, true}, {{"bar"}, false}, {{"foobar"}, false}, {{"foobarbaz"}, false}}}}}}; diff --git a/src/tscore/unit_tests/test_layout.cc b/src/tscore/unit_tests/test_layout.cc index e478f137c3d..736393bd847 100644 --- a/src/tscore/unit_tests/test_layout.cc +++ b/src/tscore/unit_tests/test_layout.cc @@ -23,6 +23,7 @@ #include "tscore/I_Layout.h" #include "tscore/ink_platform.h" +#include "tscore/Filenames.h" std::string append_slash(const char *path) @@ -79,8 +80,8 @@ TEST_CASE("relative to test", "[relative_to]") // relative to (4 parameters) char config_file[PATH_NAME_MAX]; - Layout::relative_to(config_file, sizeof(config_file), Layout::get()->sysconfdir, "records.config"); - std::string a = Layout::relative_to(Layout::get()->sysconfdir, "records.config"); + Layout::relative_to(config_file, sizeof(config_file), Layout::get()->sysconfdir, ts::filename::RECORDS); + std::string a = Layout::relative_to(Layout::get()->sysconfdir, ts::filename::RECORDS); std::string b = config_file; REQUIRE(a == b); } diff --git a/src/tscore/unit_tests/test_ts_file.cc b/src/tscore/unit_tests/test_ts_file.cc index 603ded33653..b8f788f8416 100644 --- a/src/tscore/unit_tests/test_ts_file.cc +++ b/src/tscore/unit_tests/test_ts_file.cc @@ -222,6 +222,22 @@ TEST_CASE("ts_file::path::canonical", "[libts][fs_file]") CHECK_FALSE(ts::file::exists(testdir1)); } +TEST_CASE("ts_file::path::filename", "[libts][fs_file]") +{ + CHECK(ts::file::filename(path("/foo/bar.txt")) == path("bar.txt")); + CHECK(ts::file::filename(path("/foo/.bar")) == path(".bar")); + CHECK(ts::file::filename(path("/foo/bar")) == path("bar")); + CHECK(ts::file::filename(path("/foo/bar/")) == path("")); + CHECK(ts::file::filename(path("/foo/.")) == path(".")); + CHECK(ts::file::filename(path("/foo/..")) == path("..")); + CHECK(ts::file::filename(path("/foo/../bar")) == path("bar")); + CHECK(ts::file::filename(path("/foo/../bar/")) == path("")); + CHECK(ts::file::filename(path(".")) == path(".")); + CHECK(ts::file::filename(path("..")) == path("..")); + CHECK(ts::file::filename(path("/")) == path("")); + CHECK(ts::file::filename(path("//host")) == path("host")); +} + TEST_CASE("ts_file::path::copy", "[libts][fs_file]") { std::error_code ec; @@ -259,4 +275,4 @@ TEST_CASE("ts_file::path::copy", "[libts][fs_file]") // Cleanup CHECK(ts::file::remove(testdir1, ec)); CHECK_FALSE(ts::file::exists(testdir1)); -} \ No newline at end of file +} diff --git a/src/tscpp/api/GlobalPlugin.cc b/src/tscpp/api/GlobalPlugin.cc index b1be2302a1f..8e5f05c2e7e 100644 --- a/src/tscpp/api/GlobalPlugin.cc +++ b/src/tscpp/api/GlobalPlugin.cc @@ -87,6 +87,7 @@ GlobalPlugin::~GlobalPlugin() void GlobalPlugin::registerHook(Plugin::HookType hook_type) { + assert(hook_type != Plugin::HOOK_TXN_CLOSE); TSHttpHookID hook_id = utils::internal::convertInternalHookToTsHook(hook_type); TSHttpHookAdd(hook_id, state_->cont_); LOG_DEBUG("Registered global plugin %p for hook %s", this, HOOK_TYPE_STRINGS[hook_type].c_str()); diff --git a/src/tscpp/api/Headers.cc b/src/tscpp/api/Headers.cc index 4a04a59c3d4..6b19ffdc171 100644 --- a/src/tscpp/api/Headers.cc +++ b/src/tscpp/api/Headers.cc @@ -133,7 +133,8 @@ header_field_value_iterator::~header_field_value_iterator() delete state_; } -std::string header_field_value_iterator::operator*() +std::string +header_field_value_iterator::operator*() { if (state_->index_ >= 0) { int length = 0; @@ -363,7 +364,8 @@ HeaderField::operator=(const char *field_value) return append(field_value); } -std::string HeaderField::operator[](const int index) +std::string +HeaderField::operator[](const int index) { return *header_field_value_iterator(iter_.state_->mloc_container_->hdr_buf_, iter_.state_->mloc_container_->hdr_loc_, iter_.state_->mloc_container_->field_loc_, index); @@ -464,7 +466,8 @@ header_field_iterator::operator!=(const header_field_iterator &rhs) const return !operator==(rhs); } -HeaderField header_field_iterator::operator*() +HeaderField +header_field_iterator::operator*() { return HeaderField(*this); } @@ -670,7 +673,8 @@ Headers::set(const std::string &key, const std::string &value) return append(key, value); } -HeaderField Headers::operator[](const std::string &key) +HeaderField +Headers::operator[](const std::string &key) { // In STL fashion if the key doesn't exist it will be added with an empty value. header_field_iterator it = find(key); diff --git a/src/tscpp/api/InterceptPlugin.cc b/src/tscpp/api/InterceptPlugin.cc index abd05731092..2eb76175635 100644 --- a/src/tscpp/api/InterceptPlugin.cc +++ b/src/tscpp/api/InterceptPlugin.cc @@ -190,7 +190,7 @@ InterceptPlugin::getSslConnection() return nullptr; } - return TSVConnSSLConnectionGet(state_->net_vc_); + return TSVConnSslConnectionGet(state_->net_vc_); } bool diff --git a/src/tscpp/api/RemapPlugin.cc b/src/tscpp/api/RemapPlugin.cc index 7be742ca01e..2e9bb1e63f7 100644 --- a/src/tscpp/api/RemapPlugin.cc +++ b/src/tscpp/api/RemapPlugin.cc @@ -30,12 +30,9 @@ using namespace atscppapi; TSRemapStatus TSRemapDoRemap(void *ih, TSHttpTxn rh, TSRemapRequestInfo *rri) { - RemapPlugin *remap_plugin = static_cast(ih); - Url map_from_url(rri->requestBufp, rri->mapFromUrl), map_to_url(rri->requestBufp, rri->mapToUrl); + RemapPlugin *remap_plugin = static_cast(ih); Transaction &transaction = utils::internal::getTransaction(rh); - bool redirect = false; - RemapPlugin::Result result = remap_plugin->doRemap(map_from_url, map_to_url, transaction, redirect); - rri->redirect = redirect ? 1 : 0; + RemapPlugin::Result result = remap_plugin->remapTransaction(transaction, rri); switch (result) { case RemapPlugin::RESULT_ERROR: return TSREMAP_ERROR; diff --git a/src/tscpp/api/TransactionPlugin.cc b/src/tscpp/api/TransactionPlugin.cc index 5aba50b1fc7..3e1ba75655d 100644 --- a/src/tscpp/api/TransactionPlugin.cc +++ b/src/tscpp/api/TransactionPlugin.cc @@ -71,6 +71,12 @@ TransactionPlugin::getMutex() return state_->mutex_; } +std::shared_ptr +TransactionPlugin::getMutex(TSHttpTxn txnp) +{ + return state_->ats_txn_handle_ == txnp ? state_->mutex_ : nullptr; +} + bool TransactionPlugin::isWebsocket() const { diff --git a/src/tscpp/api/Url.cc b/src/tscpp/api/Url.cc index a8f7144176e..c8e043081a6 100644 --- a/src/tscpp/api/Url.cc +++ b/src/tscpp/api/Url.cc @@ -63,11 +63,6 @@ bool inline Url::isInitialized() const return state_->hdr_buf_ && state_->url_loc_; } -void -Url::reset() -{ -} - std::string Url::getUrlString() const { diff --git a/src/tscpp/api/utils_internal.cc b/src/tscpp/api/utils_internal.cc index 923c48d961a..a4da599a0a4 100644 --- a/src/tscpp/api/utils_internal.cc +++ b/src/tscpp/api/utils_internal.cc @@ -49,6 +49,29 @@ resetTransactionHandles(Transaction &transaction, TSEvent event) return; } +void +cleanupTransaction(Transaction &transaction, TSHttpTxn ats_txn_handle) +{ + delete &transaction; + // reset the txn arg to prevent use-after-free + TSUserArgSet(ats_txn_handle, TRANSACTION_STORAGE_INDEX, nullptr); +} + +void +cleanupTransactionPlugin(Plugin *plugin, TSHttpTxn ats_txn_handle) +{ + TransactionPlugin *transaction_plugin = static_cast(plugin); + std::shared_ptr trans_mutex = utils::internal::getTransactionPluginMutex(*transaction_plugin, ats_txn_handle); + if (trans_mutex == nullptr) { + LOG_ERROR("TransactionPlugin use-after-free! plugin %p, txn %p", plugin, ats_txn_handle); + return; + } + LOG_DEBUG("Locking TransactionPlugin mutex to delete transaction plugin at %p", transaction_plugin); + trans_mutex->lock(); + delete transaction_plugin; + trans_mutex->unlock(); +} + int handleTransactionEvents(TSCont cont, TSEvent event, void *edata) { @@ -61,7 +84,6 @@ handleTransactionEvents(TSCont cont, TSEvent event, void *edata) utils::internal::setTransactionEvent(transaction, event); switch (event) { case TS_EVENT_HTTP_POST_REMAP: - transaction.getClientRequest().getUrl().reset(); // This is here to force a refresh of the cached client request url TSMBuffer hdr_buf; TSMLoc hdr_loc; @@ -78,14 +100,9 @@ handleTransactionEvents(TSCont cont, TSEvent event, void *edata) resetTransactionHandles(transaction, event); const std::list &plugins = utils::internal::getTransactionPlugins(transaction); for (auto plugin : plugins) { - std::shared_ptr trans_mutex = utils::internal::getTransactionPluginMutex(*plugin); - LOG_DEBUG("Locking TransactionPlugin mutex to delete transaction plugin at %p", plugin); - trans_mutex->lock(); - LOG_DEBUG("Locked Mutex...Deleting transaction plugin at %p", plugin); - delete plugin; - trans_mutex->unlock(); + cleanupTransactionPlugin(plugin, ats_txn_handle); } - delete &transaction; + cleanupTransaction(transaction, ats_txn_handle); } break; default: assert(false); /* we should never get here */ @@ -99,7 +116,7 @@ void setupTransactionManagement() { // Reserve a transaction slot - TSAssert(TS_SUCCESS == TSHttpTxnArgIndexReserve("atscppapi", "ATS CPP API", &TRANSACTION_STORAGE_INDEX)); + TSAssert(TS_SUCCESS == TSUserArgIndexReserve(TS_USER_ARGS_TXN, "atscppapi", "ATS CPP API", &TRANSACTION_STORAGE_INDEX)); // We must always have a cleanup handler available TSMutex mutex = nullptr; TSCont cont = TSContCreate(handleTransactionEvents, mutex); @@ -142,6 +159,15 @@ void inline invokePluginForEvent(Plugin *plugin, TSHttpTxn ats_txn_handle, TSEve case TS_EVENT_HTTP_CACHE_LOOKUP_COMPLETE: plugin->handleReadCacheLookupComplete(transaction); break; + case TS_EVENT_HTTP_TXN_CLOSE: + if (plugin) { + plugin->handleTxnClose(transaction); + cleanupTransactionPlugin(plugin, ats_txn_handle); + } else { + LOG_ERROR("stray event TS_EVENT_HTTP_TXN_CLOSE, no transaction plugin to handle it!"); + } + cleanupTransaction(transaction, ats_txn_handle); + break; default: assert(false); /* we should never get here */ break; @@ -153,19 +179,19 @@ void inline invokePluginForEvent(Plugin *plugin, TSHttpTxn ats_txn_handle, TSEve Transaction & utils::internal::getTransaction(TSHttpTxn ats_txn_handle) { - Transaction *transaction = static_cast(TSHttpTxnArgGet(ats_txn_handle, TRANSACTION_STORAGE_INDEX)); + Transaction *transaction = static_cast(TSUserArgGet(ats_txn_handle, TRANSACTION_STORAGE_INDEX)); if (!transaction) { transaction = new Transaction(static_cast(ats_txn_handle)); LOG_DEBUG("Created new transaction object at %p for ats pointer %p", transaction, ats_txn_handle); - TSHttpTxnArgSet(ats_txn_handle, TRANSACTION_STORAGE_INDEX, transaction); + TSUserArgSet(ats_txn_handle, TRANSACTION_STORAGE_INDEX, transaction); } return *transaction; } std::shared_ptr -utils::internal::getTransactionPluginMutex(TransactionPlugin &transaction_plugin) +utils::internal::getTransactionPluginMutex(TransactionPlugin &transaction_plugin, TSHttpTxn txnp) { - return transaction_plugin.getMutex(); + return transaction_plugin.getMutex(txnp); } TSHttpHookID @@ -192,6 +218,8 @@ utils::internal::convertInternalHookToTsHook(Plugin::HookType hooktype) return TS_HTTP_CACHE_LOOKUP_COMPLETE_HOOK; case Plugin::HOOK_SELECT_ALT: return TS_HTTP_SELECT_ALT_HOOK; + case Plugin::HOOK_TXN_CLOSE: + return TS_HTTP_TXN_CLOSE_HOOK; default: assert(false); // shouldn't happen, let's catch it early break; diff --git a/src/tscpp/api/utils_internal.h b/src/tscpp/api/utils_internal.h index eacd1e4e9ed..a66f0a6f3ec 100644 --- a/src/tscpp/api/utils_internal.h +++ b/src/tscpp/api/utils_internal.h @@ -55,7 +55,7 @@ namespace utils static HttpVersion getHttpVersion(TSMBuffer hdr_buf, TSMLoc hdr_loc); static void initTransactionManagement(); static std::string consumeFromTSIOBufferReader(TSIOBufferReader); - static std::shared_ptr getTransactionPluginMutex(TransactionPlugin &); + static std::shared_ptr getTransactionPluginMutex(TransactionPlugin &, TSHttpTxn); static Transaction &getTransaction(TSHttpTxn); static AsyncHttpFetchState * diff --git a/src/tscpp/util/unit_tests/test_TextView.cc b/src/tscpp/util/unit_tests/test_TextView.cc index d9ce540b1c8..869c278b0b6 100644 --- a/src/tscpp/util/unit_tests/test_TextView.cc +++ b/src/tscpp/util/unit_tests/test_TextView.cc @@ -36,10 +36,8 @@ TEST_CASE("TextView Constructor", "[libts][TextView]") { static std::string base = "Evil Dave Rulez!"; TextView tv(base); - TextView a{"Evil Dave Rulez"}; TextView b{base.data(), base.size()}; TextView c{std::string_view(base)}; - constexpr TextView d{"Grigor!"sv}; } TEST_CASE("TextView Operations", "[libts][TextView]") @@ -111,6 +109,8 @@ TEST_CASE("TextView Affixes", "[libts][TextView]") left = tv3; right = left.split_suffix_at(";:,"); TextView pre{tv3}, post{pre.split_suffix_at(7)}; + + REQUIRE(post.size() == 7); REQUIRE(right.size() == 7); REQUIRE(left.size() == 7); REQUIRE(left == "abcdefg"); diff --git a/src/wccp/WccpLocal.h b/src/wccp/WccpLocal.h index 773ce911cc9..7e173ab7c6d 100644 --- a/src/wccp/WccpLocal.h +++ b/src/wccp/WccpLocal.h @@ -296,7 +296,7 @@ class RouterAssignListElt ); /// Access const element. RouterAssignElt const &elt(int idx ///< Index of target element. - ) const; + ) const; /// Get the number of elements. uint32_t getCount() const; //@} @@ -537,7 +537,7 @@ class HashAssignElt uint32_t getCount() const; /// Get a cache address. uint32_t getAddr(int idx ///< Index of target address. - ) const; + ) const; /// Set a cache address. self &setAddr(int idx, ///< Index of target address. uint32_t addr ///< Address value to set. @@ -547,7 +547,7 @@ class HashAssignElt ); /// Access a const bucket. Bucket const &operator[](size_t idx ///< Bucket index (0..N_BUCKETS-1) - ) const; + ) const; //@} /** Do a round robin assignment. @@ -1046,7 +1046,7 @@ class SecurityComp : public CompWithHeader ); bool validate(MsgBuffer const &msg ///< Message data. - ) const; + ) const; protected: /// Local to this message shared key / password. @@ -1105,7 +1105,7 @@ class ServiceComp : public CompWithHeader /// Get a port value. uint16_t getPort(int idx ///< Index of target port. - ) const; + ) const; /// Set a port value. self &setPort(int idx, ///< Index of port. uint16_t port ///< Value for port. @@ -1195,7 +1195,7 @@ class RouterIdComp : public CompWithHeader uint32_t getFromCount() const; /// Get received from address. uint32_t getFromAddr(int idx ///< Index of address. - ) const; + ) const; /// Set received from address. self &setFromAddr(int idx, ///< Index of address. uint32_t addr ///< Address value. @@ -1335,14 +1335,14 @@ class RouterViewComp : public CompWithHeader ); /// Access cache element. CacheIdBox const &cacheId(int idx ///< Index of target element. - ) const; + ) const; /// Get router count field. /// @note No @c setf method because this cannot be changed independently. /// @see fill uint32_t getRouterCount() const; /// Get router address. uint32_t getRouterAddr(int idx ///< Index of router. - ) const; + ) const; /// Set router address. self &setRouterAddr(int idx, ///< Index of router. uint32_t addr ///< Address value. @@ -1418,7 +1418,7 @@ class CacheViewComp : public CompWithHeader uint32_t getCacheCount() const; /// Get a cache address. uint32_t getCacheAddr(int idx ///< Index of target address. - ) const; + ) const; /// Set a cache address. self &setCacheAddr(int idx, ///< Index of target address. uint32_t addr ///< Address value to set. @@ -1499,7 +1499,7 @@ class AssignInfoComp : public CompWithHeader uint32_t getCacheCount() const; /// Get a cache address. uint32_t getCacheAddr(int idx ///< Index of target address. - ) const; + ) const; /// Set a cache address. self &setCacheAddr(int idx, ///< Index of target address. uint32_t addr ///< Address value to set. @@ -1509,7 +1509,7 @@ class AssignInfoComp : public CompWithHeader ); /// Access a bucket. Bucket const &bucket(int idx ///< Index of target bucket. - ) const; + ) const; //@} /// Fill out the component from an @c Assignment. @@ -1560,7 +1560,7 @@ class CapComp : public CompWithHeader CapabilityElt &elt(int idx ///< Index of target element. ); CapabilityElt const &elt(int idx ///< Index of target element. - ) const; + ) const; /// Get the element count. /// @note No corresponding @c setf_ because that cannot be changed once set. /// @see fill @@ -2268,7 +2268,7 @@ class Impl : public ts::IntrusivePtrCounter /// @return Security option to use during message fill. SecurityOption setSecurity(BaseMsg &msg, ///< Message. GroupData const &group ///< Group data used to control fill. - ) const; + ) const; /// Validate a security component. bool validateSecurity(BaseMsg &msg, ///< Message data (including security component). GroupData const &group ///< Group data for message. @@ -2301,11 +2301,11 @@ namespace detail /// Time until next packet. /// @return Seconds until a packet should be sent. time_t waitTime(time_t now ///< Current time. - ) const; + ) const; /// Time till next ping. /// @return Seconds until a HERE_I_AM should be sent. time_t pingTime(time_t now ///< Current time. - ) const; + ) const; uint32_t m_addr; ///< Router identifying IP address. uint32_t m_generation; ///< Router's view change number. @@ -2406,7 +2406,7 @@ namespace detail /// Time until next event. /// @return The number of seconds until the next event of interest. time_t waitTime(time_t now ///< Current time. - ) const; + ) const; /** Cull routers. Routers that have not replied recently are moved from the @@ -2503,7 +2503,7 @@ class CacheImpl : public Impl */ virtual ts::Errata checkRouterAssignment(GroupData const &group, ///< Group with assignment. RouterViewComp const &comp ///< Assignment reported by router. - ) const; + ) const; protected: /// Generate contents in HERE_I_AM @a msg for seed router. @@ -3369,11 +3369,13 @@ HashAssignElt::getBucketBase() // coverity[ptr_arith] return reinterpret_cast((&m_count + 1 + this->getCount())); } -inline HashAssignElt::Bucket &HashAssignElt::operator[](size_t idx) +inline HashAssignElt::Bucket & +HashAssignElt::operator[](size_t idx) { return this->getBucketBase()[idx]; } -inline HashAssignElt::Bucket const &HashAssignElt::operator[](size_t idx) const +inline HashAssignElt::Bucket const & +HashAssignElt::operator[](size_t idx) const { return (*(const_cast(this)))[idx]; } @@ -3386,7 +3388,8 @@ MaskAssignElt::getCount() const { return ntohl(m_count); } -inline MaskValueSetElt *MaskAssignElt::appender::operator->() +inline MaskValueSetElt * +MaskAssignElt::appender::operator->() { return m_set; } @@ -3486,7 +3489,7 @@ AssignInfoComp::bucket(int idx) const } inline RouterViewComp::RouterViewComp() : m_cache_count(0) { - memset(m_cache_ids, 0, sizeof(m_cache_ids)); + ink_zero(m_cache_ids); } inline CapComp::CapComp() {} diff --git a/tests/.gitignore b/tests/.gitignore index 4cbfa0ff584..bafde3c70e2 100644 --- a/tests/.gitignore +++ b/tests/.gitignore @@ -1 +1,2 @@ _sandbox/ +done diff --git a/tests/Makefile.am b/tests/Makefile.am new file mode 100644 index 00000000000..6b0687bb91e --- /dev/null +++ b/tests/Makefile.am @@ -0,0 +1,41 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +include $(top_srcdir)/build/plugins.mk +include $(top_srcdir)/build/tidy.mk + +noinst_LTLIBRARIES = +noinst_PROGRAMS = + +SUBDIRS = + +AM_LDFLAGS += $(TS_PLUGIN_LD_FLAGS) + +# Automake is pretty draconian about not creating shared object (.so) files for +# non-installed files. However we do not want to install our test plugins so +# we prefix them with noinst_. The following -rpath argument coerces the +# generation of so objects for these test files. +AM_LDFLAGS += -rpath $(abs_builddir) + +include gold_tests/chunked_encoding/Makefile.inc +include gold_tests/tls/Makefile.inc +include tools/plugins/Makefile.inc + +TESTS = $(check_PROGRAMS) + +clang-tidy-local: $(DIST_SOURCES) + $(CXX_Clang_Tidy) + $(CC_Clang_Tidy) diff --git a/tests/Pipfile b/tests/Pipfile index 83578aca495..6cb10137ee4 100644 --- a/tests/Pipfile +++ b/tests/Pipfile @@ -20,9 +20,11 @@ url = "https://pypi.org/simple" verify_ssl = true [dev-packages] +autopep8 = "*" +pyflakes = "*" [packages] -autest = "==1.7.3" +autest = "==1.8.1" traffic-replay = "*" # this should install TRLib, MicroServer, MicroDNS, Traffic-Replay hyper = "*" dnslib = "*" @@ -30,6 +32,9 @@ dnslib = "*" requests = "*" gunicorn = "*" httpbin = "*" +microserver = ">=1.0.4" +jsonschema = "*" +python-jose = "*" [requires] python_version = "3" diff --git a/tests/README.md b/tests/README.md index db84c568e21..b0edbaec985 100644 --- a/tests/README.md +++ b/tests/README.md @@ -6,7 +6,7 @@ This directory contains different tests for Apache Trafficserver. It is recommen ## Layout The current layout is: -**gold_tests/** - contains all the TSQA v4 based tests that run on the Reusable Gold Testing System (AuTest) +**gold_tests/** - contains all the tests that run on the Reusable Gold Testing System (AuTest) **tools/** - contains programs used to help with testing. @@ -23,16 +23,16 @@ This file is a simple wrapper that will call the Reusable Gold Testing System (A This script will check for the necessary packages needed to create a pipenv that can run Autest. If any package is missing, the script will alert the user. If all packages are available, it install a virtual environment using the provided Pipfile. ### Pipfile -This file is used to setup a virtual environment using pipenv. It contains information including the packages needed for Autest. +This file is used to setup a virtual environment using pipenv. It contains information including the packages needed for Autest. A set of commands for pipenv: - * **pipenv install**: create virtual environment from the Pipfile. + * **pipenv install**: create virtual environment from the Pipfile. ( If you're going to add tests, add `-d` option to install dev packages ) * **pipenv shell**: launch a shell with the environment running(type "exit" to leave the shell). * **pipenv run cmd**: run command in the virtual environment without entering a shell, where cmd is the shell command to run. * **pipenv --rm**: remove the environment. # Basic setup -AuTest can be run using the script file autest.sh listed above. Run the file from the tests/ directory followed by --ats-bin and the bin name. (ie ~/ats/bin) This will run the wrapper for the tests. +AuTest can be run using the script file autest.sh listed above. Run the file from the tests/ directory followed by --ats-bin and the bin name. (ie ~/ats/bin) This will run the wrapper for the tests. To run autest manually, the recommended way is to follow these steps: 1. **pipenv install**: create the virtual environment(only needed once). @@ -44,7 +44,7 @@ To run autest manually, the recommended way is to follow these steps: AuTest and the relevant tools can be install manually instead of using the wrapper script. By doing this, it is often easier to debug issues with the testing system, or the tests. There are two ways this can be done. 1. Run the bootstrap script then source the path with a "source ./env-test/bin/activate" command. At this point autest command should run without the wrapper script -2. Make sure you install python 3.5 or better on your system. From there install these python packages ( ie pip install ): +2. Make sure you install python 3.6 or better on your system. From there install these python packages ( ie pip install ): - hyper - git+https://bitbucket.org/autestsuite/reusable-gold-testing-system.git - [traffic-replay](https://bitbucket.org/autestsuite/trafficreplay/src/master/) (This will automatically install [MicroDNS](https://bitbucket.org/autestsuite/microdns/src/master/), [MicroServer](https://bitbucket.org/autestsuite/microserver/src/master/), [TrafficReplayLibrary](https://bitbucket.org/autestsuite/trafficreplaylibrary/src/master/), and dnslib as part of the dependencies.) @@ -53,7 +53,7 @@ AuTest and the relevant tools can be install manually instead of using the wrapp When writing for the AuTest system please refer to the current [Online Documentation](https://autestsuite.bitbucket.io/) for general use of the system. To use CurlHeader tester for testing output of curl, please refer to [CurlHeader README](gold_tests/autest-site/readme.md) ## Documentation of AuTest extension for ATS. -Autest allows for the creation of extensions to help specialize and simplify test writing for a given application domain. Minus API addition the extension code will check that python 3.5 or better is used. There is also a new command line argumented added specifically for Trafficserver: +Autest allows for the creation of extensions to help specialize and simplify test writing for a given application domain. Minus API addition the extension code will check that python 3.6 or better is used. There is also a new command line argumented added specifically for Trafficserver: --ats-bin < path to bin directory > @@ -65,7 +65,7 @@ This command line argument will point to your build of ATS you want to test. At * select_ports - have the testing system automatically select a nonSSL port to use for this instance of ATS * enable_tls - have the testing system also auto-select a SSL port to use (NOTE: This does not set up certs and other TLS-related configs.) -This function will define a sandbox for an instance of trafficserver to run under. The function will return a AuTest process object that will have a number of files and variables define for making it easier to define a test. If both *select_ports* and *enable_tls* are toggled to **False**, then the test writer will be responsible for setting up the ports and the ready condition for an instance of ATS. +This function will define a sandbox for an instance of trafficserver to run under. The function will return a AuTest process object that will have a number of files and variables define for making it easier to define a test. If both *select_ports* and *enable_tls* are toggled to **False**, then the test writer will be responsible for setting up the ports and the ready condition for an instance of ATS. #### Environment The environment of the process will have a number of added environment variables to control trafficserver running the in the sandbox location correctly. This can be used to easily setup other commands that should run under same environment. @@ -101,13 +101,13 @@ A number of file objects are defined to help with adding values to a given confi ##### config files * records.config * cache.config - * congestion.config * hosting.config - * ip_allow.config + * ip_allow.yaml * logging.yaml * parent.config * plugin.config * remap.config + * sni.yaml * socks.config * splitdns.config * ssl_multicert.config @@ -311,6 +311,7 @@ ts.Disk.remap_config.AddLine( * TS_HAS_IP_TOS * TS_USE_HWLOC * TS_USE_SET_RBIO + * TS_USE_QUIC * TS_USE_LINUX_NATIVE_AIO * TS_HAS_SO_PEERCRED * TS_USE_REMOTE_UNWINDING diff --git a/tests/gold_tests/autest-site/build.test.ext b/tests/gold_tests/autest-site/build.test.ext deleted file mode 100644 index ef8aa422e66..00000000000 --- a/tests/gold_tests/autest-site/build.test.ext +++ /dev/null @@ -1,53 +0,0 @@ -''' -Build random code for running as part of a test -''' -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import re -import autest.common.is_a as is_a - - -def Build(Test, target, sources, CPPFLAGS='', LDFLAGS='', LIBS='', CC=None): - if is_a.OrderedSequence(sources): - sources = " ".join(sources) - tr = Test.AddTestRun("Build", "Build test files: {0}".format(sources)) - vars = Test.ComposeVariables() - if CC is None: - cc = vars.CXX - else: - cc = CC - - tr.Processes.Default.Command = '{cc} -o {target} {cppflags} {sources} {ldflags} {libs}'.format( - cppflags="{0} {1}".format(vars.CPPFLAGS, CPPFLAGS), - ldflags="{0} {1}".format(vars.LDFLAGS, LDFLAGS), - libs="{0} {1}".format(vars.LIBS, LIBS), - target=target, - sources=sources, - cc=cc - ) - tr.Processes.Default.ForceUseShell = True - tr.ReturnCode = 0 - tr.Streams.All = Testers.ExcludesExpression( - r'(\A|\s)error?\s?(([?!: ])|(\.\s))\D', - "Build should not contain errors", - reflags=re.IGNORECASE - ) - - return tr - - -ExtendTest(Build, name="Build") diff --git a/tests/gold_tests/autest-site/cli_tools.test.ext b/tests/gold_tests/autest-site/cli_tools.test.ext index d82a30b395a..b253f03b955 100644 --- a/tests/gold_tests/autest-site/cli_tools.test.ext +++ b/tests/gold_tests/autest-site/cli_tools.test.ext @@ -31,19 +31,19 @@ Tools to help with TestRun commands # # Note that by default, the Default Process is created in this function. -def spawn_commands(self, cmdstr, count, retcode = 0, use_default=True): - ret=[] +def spawn_commands(self, cmdstr, count, retcode=0, use_default=True): + ret = [] if use_default: count = int(count) - 1 - for cnt in range(0,count): + for cnt in range(0, count): ret.append( self.Processes.Process( name="cmdline-{num}".format(num=cnt), - cmdstr = cmdstr, - returncode = retcode - ) + cmdstr=cmdstr, + returncode=retcode ) + ) if use_default: self.Processes.Default.Command = cmdstr self.Processes.Default.ReturnCode = retcode @@ -52,4 +52,5 @@ def spawn_commands(self, cmdstr, count, retcode = 0, use_default=True): ) return ret + ExtendTestRun(spawn_commands, name="SpawnCommands") diff --git a/tests/gold_tests/autest-site/conditions.test.ext b/tests/gold_tests/autest-site/conditions.test.ext index a8a91c6c8dd..c9d3a0d3221 100644 --- a/tests/gold_tests/autest-site/conditions.test.ext +++ b/tests/gold_tests/autest-site/conditions.test.ext @@ -20,9 +20,10 @@ import subprocess import json import re + def HasOpenSSLVersion(self, version): output = subprocess.check_output( - os.path.join(self.Variables.BINDIR, "traffic_layout") + " info --versions --json", shell = True + os.path.join(self.Variables.BINDIR, "traffic_layout") + " info --versions --json", shell=True ) json_data = output.decode('utf-8') openssl_str = json.loads(json_data)['openssl_str'] @@ -34,8 +35,10 @@ def HasOpenSSLVersion(self, version): "openssl library version is " + exe_ver + ", must be at least " + version ) + def HasCurlVersion(self, version): - return self.EnsureVersion(["curl","--version"],min_version=version) + return self.EnsureVersion(["curl", "--version"], min_version=version) + def HasCurlFeature(self, feature): @@ -58,6 +61,8 @@ def HasCurlFeature(self, feature): default, "Curl needs to support feature: {feature}".format(feature=feature) ) + + def HasCurlOption(self, option): def default(output): tag = option.lower() @@ -75,20 +80,23 @@ def HasCurlOption(self, option): "Curl needs to support option: {option}".format(option=option) ) + def HasATSFeature(self, feature): val = self.Variables.get(feature, None) return self.Condition( - lambda: val == True, + lambda: val, "ATS feature not enabled: {feature}".format(feature=feature) ) -#test if a plugin exists in the libexec folder +# test if a plugin exists in the libexec folder + + def PluginExists(self, pluginname): path = os.path.join(self.Variables.PLUGINDIR, pluginname) - return self.Condition(lambda: os.path.isfile(path) == True, path + " not found." ) + return self.Condition(lambda: os.path.isfile(path), path + " not found.") ExtendCondition(HasOpenSSLVersion) @@ -97,4 +105,3 @@ ExtendCondition(HasCurlVersion) ExtendCondition(HasCurlFeature) ExtendCondition(HasCurlOption) ExtendCondition(PluginExists) - diff --git a/tests/gold_tests/autest-site/curl_header.test.ext b/tests/gold_tests/autest-site/curl_header.test.ext index b592b45be96..3ce674ece11 100644 --- a/tests/gold_tests/autest-site/curl_header.test.ext +++ b/tests/gold_tests/autest-site/curl_header.test.ext @@ -22,6 +22,7 @@ import hosts.output as host import re + class CurlHeader(Tester): def __init__(self, value, @@ -38,13 +39,13 @@ class CurlHeader(Tester): stack=host.getCurrentStack(1) ) - ops = ("equal","equal_re") + ops = ("equal", "equal_re") gold_dict = value headers = tuple(gold_dict.keys()) if description is None: - description = 'Checking that all {} headers exist and have matching values: {}'.format(len(headers), \ - ', '.join(headers) if len(headers) <= 10 else ', '.join(headers[0:10]) + ', etc.') + description = 'Checking that all {} headers exist and have matching values: {}'.format( + len(headers), ', '.join(headers) if len(headers) <= 10 else ', '.join(headers[0:10]) + ', etc.') # sanity check for input dictionary format for header, target in gold_dict.items(): @@ -54,34 +55,30 @@ class CurlHeader(Tester): stack=self._stack ) - if target == None or isinstance(target, str): + if target is None or isinstance(target, str): continue elif isinstance(target, dict): for op, pos_val in target.items(): if op not in ops: host.WriteError( - 'CurlHeader Input: Unsupported operation \'{}\' for value at header \'{}\'. The available operations are: {}.'.format(op, header, ', '.join(ops)), - stack=self._stack - ) - elif pos_val == None or isinstance(pos_val, str): + 'CurlHeader Input: Unsupported operation \'{}\' for value at header \'{}\'. The available operations are: {}.'.format( + op, header, ', '.join(ops)), stack=self._stack) + elif pos_val is None or isinstance(pos_val, str): continue elif isinstance(pos_val, list): for str_ in pos_val: - if not isinstance(str_, str) and str_ != None: + if not isinstance(str_, str) and str_ is not None: host.WriteError( - 'CurlHeader Input: Value {} has unsupported type \'{}\' for header \'{}\'. Need to provide a string or None.'.format(str_, str_.__class__.__name__, header), - stack=self._stack - ) + 'CurlHeader Input: Value {} has unsupported type \'{}\' for header \'{}\'. Need to provide a string or None.'.format( + str_, str_.__class__.__name__, header), stack=self._stack) else: host.WriteError( - 'CurlHeader Input: Value {} has unsupported type \'{}\' for header \'{}\'. Need to provide a string, a list or None for possible curl values.'.format(pos_val, pos_val.__class__.__name__, header), - stack=self._stack - ) + 'CurlHeader Input: Value {} has unsupported type \'{}\' for header \'{}\'. Need to provide a string, a list or None for possible curl values.'.format( + pos_val, pos_val.__class__.__name__, header), stack=self._stack) else: host.WriteError( - 'CurlHeader Input: Value {} has unsupported type \'{}\' for header \'{}\'. Need to provide either a string, a dictionary or None.'.format(target, target.__class__.__name__, header), - stack=self._stack - ) + 'CurlHeader Input: Value {} has unsupported type \'{}\' for header \'{}\'. Need to provide either a string, a dictionary or None.'.format( + target, target.__class__.__name__, header), stack=self._stack) super(CurlHeader, self).__init__( value=value, @@ -120,16 +117,16 @@ class CurlHeader(Tester): val_dict[vals[0].lower()] = ': '.join(vals[1:]) # generate a gold dictionary with lowercase header - gold_dict = {k.lower() : v for k, v in gold_dict.items()} + gold_dict = {k.lower(): v for k, v in gold_dict.items()} p_flag = 1 reason = '' for header in gold_dict.keys(): v = val_dict.get(header) - if v != None: + if v is not None: res = self.match(gold_dict, header, v) - if res == None: + if res is None: continue else: reason += 'In field: {} \n'.format(header) + res @@ -140,7 +137,6 @@ class CurlHeader(Tester): p_flag = 0 reason += '\n--------------------------------------------------------------------------------\n' - if p_flag == 1: self.Result = tester.ResultType.Passed self.Reason = "Curl headers and values match" @@ -166,7 +162,7 @@ class CurlHeader(Tester): def match(self, gold_dictionary, header, val_value): target = gold_dictionary[header] - if target == None: + if target is None: return None elif isinstance(target, str): # if given a string, check for an exact match @@ -176,21 +172,21 @@ class CurlHeader(Tester): elif isinstance(target, dict): # if given a dict, check for valid operations indicated by the keys for op, pos_val in target.items(): - if pos_val == None: + if pos_val is None: return None elif isinstance(pos_val, list): # print('Need to provide a list of possible values.') # continue if op == 'equal': for str_ in pos_val: - if val_value == str_ or str_ == None: + if val_value == str_ or str_ is None: return None # return 'No matching strings in the list.' elif op == 'equal_re': for regex in pos_val: - if regex == None: + if regex is None: return None - elif re.fullmatch(regex, val_value) != None: + elif re.fullmatch(regex, val_value) is not None: return None elif isinstance(pos_val, str): @@ -200,12 +196,12 @@ class CurlHeader(Tester): # else 'Not an exact match.\nExpected : {}\nActual : {}'.format(pos_val, val_value) elif op == 'equal_re': - if re.fullmatch(pos_val, val_value) != None: + if re.fullmatch(pos_val, val_value) is not None: return None ret = '' - ops = {'equal' : 'Any of the following strings: ', - 'equal_re' : 'Any of the following regular expression: '} + ops = {'equal': 'Any of the following strings: ', + 'equal_re': 'Any of the following regular expression: '} for op, pos_val in target.items(): ret += ' {}: \'{}\'\n'.format(ops[op], '\', \''.join(pos_val)) if isinstance(pos_val, list) \ @@ -213,4 +209,5 @@ class CurlHeader(Tester): return 'Value \'{}\' matches none of: \n{}'.format(val_value, ret.strip('\n')) + AddTester(CurlHeader) diff --git a/tests/gold_tests/autest-site/init.cli.ext b/tests/gold_tests/autest-site/init.cli.ext index 6d7e89b13e2..a8d0641ee46 100644 --- a/tests/gold_tests/autest-site/init.cli.ext +++ b/tests/gold_tests/autest-site/init.cli.ext @@ -16,17 +16,20 @@ # See the License for the specific language governing permissions and # limitations under the License. -import os - import sys -if sys.version_info < (3, 5, 0): + +if sys.version_info < (3, 6, 0): host.WriteError( - "You need python 3.5 or later to run these tests\n", show_stack=False) + "You need python 3.6 or later to run these tests\n", show_stack=False) -autest_version ="1.7.2" +autest_version = "1.8.1" if AuTestVersion() < autest_version: host.WriteError( - "Tests need AuTest version {ver} or better\n Please update AuTest:\n pip install --upgrade autest\n".format(ver=autest_version), show_stack=False) + "Tests need AuTest version {needed_version} or better, found version {found_version}\n" + "Please update AuTest:\n pip install --upgrade autest\n".format( + needed_version=autest_version, + found_version=AuTestVersion()), + show_stack=False) Settings.path_argument(["--ats-bin"], diff --git a/tests/gold_tests/autest-site/ip.test.ext b/tests/gold_tests/autest-site/ip.test.ext new file mode 100755 index 00000000000..74ea68fc311 --- /dev/null +++ b/tests/gold_tests/autest-site/ip.test.ext @@ -0,0 +1,40 @@ +''' +Extend Autest with IP-related utilities. +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from ports import get_port + +# this forms is for the global process define + +# New member function for Test object, that reserves TCP port(s) from the list of available TCP ports, for +# use in the test. Each argument is a string, containing the name of a variable to add to Test.Variables . +# For each argument, a port will be reserved, and its number will be assigned to the new variable for the +# argument. +# + + +def get_tcp_port(obj, *newVariables): + for v in newVariables: + if not isinstance(v, str): + raise TypeError("all function arguments must be strings") + get_port(obj, v) + + +#AddTestEntityMember(get_tcp_port, name="GetTcpPort") +ExtendTest(get_tcp_port, name="GetTcpPort") diff --git a/tests/gold_tests/autest-site/microDNS.test.ext b/tests/gold_tests/autest-site/microDNS.test.ext index fdeffe00088..f33b095f10d 100644 --- a/tests/gold_tests/autest-site/microDNS.test.ext +++ b/tests/gold_tests/autest-site/microDNS.test.ext @@ -18,14 +18,12 @@ import json import os -import sys import trlib.ipconstants as IPConstants from ports import get_port -# AddRecord registers a list of ip address against hostname - +# AddRecord registers a list of ip address against hostname def AddRecord(hostname, list_ip_addr): record = dict() @@ -84,6 +82,9 @@ def MakeDNServer(obj, name, filename="dns_file.json", port=False, ip='INADDR_LOO jsondata = {'mappings': []} if default: + if isinstance(default, str): + # MicroDNS expects 'otherwise' to be a sequence. + default = [default] jsondata['otherwise'] = default with open(filepath, 'w') as f: diff --git a/tests/gold_tests/autest-site/microserver.test.ext b/tests/gold_tests/autest-site/microserver.test.ext index bef027f09c7..5be699b9883 100644 --- a/tests/gold_tests/autest-site/microserver.test.ext +++ b/tests/gold_tests/autest-site/microserver.test.ext @@ -19,10 +19,9 @@ import json import socket import ssl -import time -import sys from autest.api import AddWhenFunction +import hosts.output as host from ports import get_port import trlib.ipconstants as IPConstants @@ -57,6 +56,7 @@ def getHeaderFieldVal(request_header, field): val = field_v[0].strip() return val + # addResponse adds customized response with respect to request_header. request_header and response_header are both dictionaries def addResponse(self, filename, request_header, response_header): client_request = Request.fromRequestLine(request_header["headers"], request_header["body"], None if "options" not in request_header else request_header["options"]) @@ -69,9 +69,8 @@ def addResponse(self, filename, request_header, response_header): addTransactionToSession(txn, absFilepath) return -# adds transaction in json format to the specified file - +# adds transaction in json format to the specified file def addTransactionToSession(txn, JFile): jsondata = None if not os.path.exists(os.path.dirname(JFile)): @@ -82,7 +81,7 @@ def addTransactionToSession(txn, JFile): # hard coding only 1 session per file # since for the purpose of testing, we don't need multiple sessions in a file - if jsondata == None: + if jsondata is None: jsondata = {} jsondata["sessions"] = [] @@ -96,6 +95,7 @@ def addTransactionToSession(txn, JFile): with open(JFile, 'w+') as jf: jf.write(json.dumps(jsondata)) + def addSessionFromFiles(self, session_dir): self.Setup.Copy(session_dir, self.Variables.DataDir) @@ -126,6 +126,16 @@ def uServerUpAndRunning(serverHost, port, isSsl, isIPv6, request, clientcert='', try: sock.connect((serverHost, port)) except ConnectionRefusedError: + host.WriteDebug( + ['uServerUpAndRunning', 'when'], + "Connection refused: {0}:{1}".format( + serverHost, port)) + return False + except ssl.SSLError: + host.WriteDebug( + ['uServerUpAndRunning', 'when'], + "SSL connection error: {0}:{1}".format( + serverHost, port)) return False sock.sendall(request.encode()) @@ -155,7 +165,19 @@ def uServerUpAndRunning(serverHost, port, isSsl, isIPv6, request, clientcert='', AddWhenFunction(uServerUpAndRunning) -def MakeOriginServer(obj, name, port=None, s_port=None, ip='INADDR_LOOPBACK', delay=None, ssl=False, lookup_key=DEFAULT_LOOKUP_KEY, clientcert='', clientkey='', both=False, options={}): +def MakeOriginServer( + obj, + name, + port=None, + s_port=None, + ip='INADDR_LOOPBACK', + delay=None, + ssl=False, + lookup_key=DEFAULT_LOOKUP_KEY, + clientcert='', + clientkey='', + both=False, + options={}): data_dir = os.path.join(obj.RunDirectory, name) p = obj.Processes.Process(name) @@ -177,8 +199,8 @@ def MakeOriginServer(obj, name, port=None, s_port=None, ip='INADDR_LOOPBACK', de command += " --cert {0}".format(cert) command += " --s_port {0}".format(s_port) - # this might break if user specifies both both and ssl - if not ssl: # in both or HTTP only mode + # this might break if user specifies both both and ssl + if not ssl: # in both or HTTP only mode if not port: port = get_port(p, "Port") @@ -219,7 +241,14 @@ def MakeOriginServer(obj, name, port=None, s_port=None, ip='INADDR_LOOPBACK', de "options": {"skipHooks": None} }) - p.Ready = When.uServerUpAndRunning(ipaddr, s_port if ssl else port, ssl, IPConstants.isIPv6(ip), healthcheck_request["headers"], clientcert=clientcert, clientkey=clientkey) + p.Ready = When.uServerUpAndRunning( + ipaddr, + s_port if ssl else port, + ssl, + IPConstants.isIPv6(ip), + healthcheck_request["headers"], + clientcert=clientcert, + clientkey=clientkey) p.ReturnCode = Any(None, 0) return p diff --git a/tests/gold_tests/autest-site/min_cfg/cache.config b/tests/gold_tests/autest-site/min_cfg/cache.config deleted file mode 100644 index 6ac656b1b66..00000000000 --- a/tests/gold_tests/autest-site/min_cfg/cache.config +++ /dev/null @@ -1 +0,0 @@ -# this files just needs to exist \ No newline at end of file diff --git a/tests/gold_tests/autest-site/min_cfg/hosting.config b/tests/gold_tests/autest-site/min_cfg/hosting.config deleted file mode 100644 index 6ac656b1b66..00000000000 --- a/tests/gold_tests/autest-site/min_cfg/hosting.config +++ /dev/null @@ -1 +0,0 @@ -# this files just needs to exist \ No newline at end of file diff --git a/tests/gold_tests/autest-site/min_cfg/logging.yaml b/tests/gold_tests/autest-site/min_cfg/logging.yaml deleted file mode 100644 index 6ac656b1b66..00000000000 --- a/tests/gold_tests/autest-site/min_cfg/logging.yaml +++ /dev/null @@ -1 +0,0 @@ -# this files just needs to exist \ No newline at end of file diff --git a/tests/gold_tests/autest-site/min_cfg/parent.config b/tests/gold_tests/autest-site/min_cfg/parent.config deleted file mode 100644 index 6ac656b1b66..00000000000 --- a/tests/gold_tests/autest-site/min_cfg/parent.config +++ /dev/null @@ -1 +0,0 @@ -# this files just needs to exist \ No newline at end of file diff --git a/tests/gold_tests/autest-site/min_cfg/plugin.config b/tests/gold_tests/autest-site/min_cfg/plugin.config deleted file mode 100644 index 6ac656b1b66..00000000000 --- a/tests/gold_tests/autest-site/min_cfg/plugin.config +++ /dev/null @@ -1 +0,0 @@ -# this files just needs to exist \ No newline at end of file diff --git a/tests/gold_tests/autest-site/min_cfg/readme.txt b/tests/gold_tests/autest-site/min_cfg/readme.txt index 8060e963e16..369a4dbf2af 100644 --- a/tests/gold_tests/autest-site/min_cfg/readme.txt +++ b/tests/gold_tests/autest-site/min_cfg/readme.txt @@ -1,3 +1,4 @@ -Contains the miniumn set of config file and setting to allow trafficserver to start in a usable way. +Contains the minimum set of config file and setting to allow trafficserver to start in a usable way. The goal is to remove the need for any of these files to exist. -Each of these files should provide an understanding of what we need to fix in the code +ip_allow.yaml is needed to allow tests the minimum access to traffic_server. +storage.config needs to exist for now. diff --git a/tests/gold_tests/autest-site/min_cfg/records.config b/tests/gold_tests/autest-site/min_cfg/records.config deleted file mode 100644 index 8d97ae7bd01..00000000000 --- a/tests/gold_tests/autest-site/min_cfg/records.config +++ /dev/null @@ -1 +0,0 @@ -# some stuff diff --git a/tests/gold_tests/autest-site/min_cfg/remap.config b/tests/gold_tests/autest-site/min_cfg/remap.config deleted file mode 100644 index 6ac656b1b66..00000000000 --- a/tests/gold_tests/autest-site/min_cfg/remap.config +++ /dev/null @@ -1 +0,0 @@ -# this files just needs to exist \ No newline at end of file diff --git a/tests/gold_tests/autest-site/min_cfg/sni.yaml b/tests/gold_tests/autest-site/min_cfg/sni.yaml deleted file mode 100644 index 6ac656b1b66..00000000000 --- a/tests/gold_tests/autest-site/min_cfg/sni.yaml +++ /dev/null @@ -1 +0,0 @@ -# this files just needs to exist \ No newline at end of file diff --git a/tests/gold_tests/autest-site/min_cfg/socks.config b/tests/gold_tests/autest-site/min_cfg/socks.config deleted file mode 100644 index 6ac656b1b66..00000000000 --- a/tests/gold_tests/autest-site/min_cfg/socks.config +++ /dev/null @@ -1 +0,0 @@ -# this files just needs to exist \ No newline at end of file diff --git a/tests/gold_tests/autest-site/min_cfg/splitdns.config b/tests/gold_tests/autest-site/min_cfg/splitdns.config deleted file mode 100644 index 6ac656b1b66..00000000000 --- a/tests/gold_tests/autest-site/min_cfg/splitdns.config +++ /dev/null @@ -1 +0,0 @@ -# this files just needs to exist \ No newline at end of file diff --git a/tests/gold_tests/autest-site/min_cfg/ssl_multicert.config b/tests/gold_tests/autest-site/min_cfg/ssl_multicert.config deleted file mode 100644 index 6ac656b1b66..00000000000 --- a/tests/gold_tests/autest-site/min_cfg/ssl_multicert.config +++ /dev/null @@ -1 +0,0 @@ -# this files just needs to exist \ No newline at end of file diff --git a/tests/gold_tests/autest-site/min_cfg/volume.config b/tests/gold_tests/autest-site/min_cfg/volume.config deleted file mode 100644 index 6ac656b1b66..00000000000 --- a/tests/gold_tests/autest-site/min_cfg/volume.config +++ /dev/null @@ -1 +0,0 @@ -# this files just needs to exist \ No newline at end of file diff --git a/tests/gold_tests/autest-site/ports.py b/tests/gold_tests/autest-site/ports.py index 7932be82e4f..db6a66f104b 100644 --- a/tests/gold_tests/autest-site/ports.py +++ b/tests/gold_tests/autest-site/ports.py @@ -31,8 +31,25 @@ g_ports = None # ports we can use +class PortQueueSelectionError(Exception): + """ + An exception for when there are problems selecting a port from the port + queue. + """ + pass + + def PortOpen(port, address=None): + """ + Detect whether the port is open, that is a socket is currently using that port. + + Open ports are currently in use by an open socket and are therefore not available + for a server to listen on them. + Returns: + True if there is a connection currently listening on the port, False if + there is no server listening on the port currently. + """ ret = False if address is None: address = "localhost" @@ -40,25 +57,84 @@ def PortOpen(port, address=None): address = (address, port) try: + # Try to connect on that port. If we can connect on it, then someone is + # listening on that port and therefore the port is open. s = socket.create_connection(address, timeout=.5) s.close() ret = True + host.WriteDebug( + 'PortOpen', + "Connection to port {} succeeded, the port is open, " + "and a future connection cannot use it".format(port)) except socket.error: s = None - ret = False + host.WriteDebug( + 'PortOpen', + "socket error for port {0}, port is closed, " + "and therefore a future connection can use it".format(port)) except socket.timeout: s = None + host.WriteDebug( + 'PortOpen', + "Timeout error for port {0}, port is closed, " + "and therefore a future connection can use it".format(port)) return ret -def setup_port_queue(amount=1000): +def _get_available_port(queue): + """ + Get the next available port from the port queue and return it. + + Since there can be a delay between when the queue is populated and when the + port is requested, this checks the next port in the queue to see whether it + is still not open. If it isn't, it is dropped from the queue and the next + one is inspected. This process continues until either an available port is + found, or, if the queue is exhausted, PortQueueSelectionError is raised. + + Returns: + An available (i.e., non-open) port. + + Throws: + PortQueueSelectionError if the port queue is exhausted. + """ + + if queue.qsize() == 0: + host.WriteWarning("Port queue is empty.") + raise PortQueueSelectionError( + "Could not get a valid port because the queue is empty") + + port = queue.get() + while PortOpen(port): + host.WriteDebug( + '_get_available_port' + "Port was open but now is used: {}".format(port)) + if queue.qsize() == 0: + host.WriteWarning("Port queue is empty.") + raise PortQueueSelectionError( + "Could not get a valid port because the queue is empty") + port = queue.get() + return port + + +def _setup_port_queue(amount=1000): + """ + Build up the set of ports that the OS in theory will not use. + """ global g_ports if g_ports is None: - g_ports = Queue.LifoQueue() + host.WriteDebug( + '_setup_port_queue', + "Populating the port queue.") + g_ports = Queue.Queue() else: + # The queue has already been populated. + host.WriteDebug( + '_setup_port_queue', + "Queue was previously populated. Queue size: {}".format(g_ports.qsize())) return try: + # Use sysctl to find the range of ports that the OS publishes it uses. # some docker setups don't have sbin setup correctly new_env = os.environ.copy() new_env['PATH'] = "/sbin:/usr/sbin:" + new_env['PATH'] @@ -78,7 +154,7 @@ def setup_port_queue(amount=1000): ).decode().split("=")[1].split() dmin = int(dmin) dmax = int(dmax) - except: + except Exception: host.WriteWarning("Unable to call sysctrl!\n Tests may fail because of bad port selection!") return @@ -86,43 +162,89 @@ def setup_port_queue(amount=1000): rmax = 65536 - dmax if rmax > amount: - # fill in ports + # Fill in ports, starting above the upper OS-usable port range. port = dmax + 1 while port < 65536 and g_ports.qsize() < amount: - # if port good: if not PortOpen(port): + host.WriteDebug( + '_setup_port_queue', + "Adding a possible port to connect to: {0}".format(port)) g_ports.put(port) + else: + host.WriteDebug( + '_setup_port_queue', + "Rejecting a possible port to connect to: {0}".format(port)) port += 1 if rmin > amount and g_ports.qsize() < amount: port = 2001 + # Fill in more ports, starting at 2001, well above well known ports, + # and going up until the minimum port range used by the OS. while port < dmin and g_ports.qsize() < amount: - # if port good: if not PortOpen(port): + host.WriteDebug( + '_setup_port_queue', + "Adding a possible port to connect to: {0}".format(port)) g_ports.put(port) + else: + host.WriteDebug( + '_setup_port_queue', + "Rejecting a possible port to connect to: {0}".format(port)) port += 1 -def get_port(obj, name): - ''' - Get a port and set it to a variable on the object +def _get_port_by_bind(): + """ + Create a socket, bind with REUSEADDR, and return the bound port. - ''' + Because SO_REUSEADDR is applied as an option to the socket, the future + server should be able to use this port. This method is considered + sub-optimal in comparison to the OS-unused port range method used above + because another process might bind to this port in the meantime. - setup_port_queue() - if g_ports.qsize(): - # get port - port = g_ports.get() - # assign to variable - obj.Variables[name] = port - # setup clean up step to recycle the port - obj.Setup.Lambda(func_cleanup=lambda: g_ports.put( - port), description="recycling port") - return port - - # use old code + Returns: + A port value that can be used for a connection. + """ sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) sock.bind(('', 0)) # bind to all interfaces on an ephemeral port port = sock.getsockname()[1] + return port + + +def get_port(obj, name): + ''' + Get a port and set it to the specified variable on the object. + + Args: + obj: The object upon which to set the port variable. + name: The name of the variable that receives the port value. + + Returns: + The port value. + ''' + _setup_port_queue() + port = 0 + if g_ports.qsize() > 0: + try: + port = _get_available_port(g_ports) + host.WriteVerbose( + "get_port", + "Using port from port queue: {}".format(port)) + # setup clean up step to recycle the port + obj.Setup.Lambda(func_cleanup=lambda: g_ports.put( + port), description="recycling port") + except PortQueueSelectionError: + port = _get_port_by_bind() + host.WriteVerbose( + "get_port", + "Queue was drained. Using port from a bound socket: {}".format(port)) + else: + # Since the queue could not be populated, use a port via bind. + port = _get_port_by_bind() + host.WriteVerbose( + "get_port", + "Queue is empty. Using port from a bound socket: {}".format(port)) + + # Assign to the named variable. obj.Variables[name] = port return port diff --git a/tests/gold_tests/autest-site/readme.md b/tests/gold_tests/autest-site/readme.md index 8ab499d1a15..72c89c7b419 100644 --- a/tests/gold_tests/autest-site/readme.md +++ b/tests/gold_tests/autest-site/readme.md @@ -14,18 +14,18 @@ To use the CurlHeader tester when writing tests, which is not one of the default Examples: * To check for header 'X-Cache' that have value 'miss' and header 'cache-control' that have value starting with 'max', provide the following dictionary to tester: - - { - 'X-Cache' : 'miss', + + { + 'X-Cache' : 'miss', 'cache-control' : {'equal_re' : 'max.*'} } * To check for header 'Age' that can have any value and header 'etag' that have value either matching 'myetag' or end with 'p', provide the following dictionary to tester: - + { 'Age' : None, 'etag' : { 'equal' : 'myetag', 'equal_re' : '.*p' } - } \ No newline at end of file + } diff --git a/tests/gold_tests/autest-site/setup.cli.ext b/tests/gold_tests/autest-site/setup.cli.ext index d0f1e6412de..c28c5d4b0cc 100644 --- a/tests/gold_tests/autest-site/setup.cli.ext +++ b/tests/gold_tests/autest-site/setup.cli.ext @@ -36,7 +36,7 @@ if ENV['ATS_BIN'] is not None: hint = '' if os.path.isfile(os.path.join(ENV['ATS_BIN'], 'bin', 'traffic_layout')): hint = "\nDid you mean '--ats-bin {}'?".\ - format(os.path.join(ENV['ATS_BIN'],'bin')) + format(os.path.join(ENV['ATS_BIN'], 'bin')) host.WriteError("traffic_layout is not found. Aborting tests - Bad build or install.{}".format(hint), show_stack=False) try: out = subprocess.check_output([traffic_layout, "--json"]) @@ -80,6 +80,7 @@ if ENV['ATS_BIN'] is not None: Variables.update(out) Variables.AtsExampleDir = os.path.join(AutestSitePath, '../../../example') Variables.AtsTestToolsDir = os.path.join(AutestSitePath, '../../tools') +Variables.AtsTestPluginsDir = os.path.join(AutestSitePath, '../../tools/plugins/.libs') # modify delay times as we always have to kill Trafficserver # no need to wait diff --git a/tests/gold_tests/autest-site/traffic_replay.test.ext b/tests/gold_tests/autest-site/traffic_replay.test.ext index 340fae93d48..c4b2e6910b7 100644 --- a/tests/gold_tests/autest-site/traffic_replay.test.ext +++ b/tests/gold_tests/autest-site/traffic_replay.test.ext @@ -16,10 +16,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -# default 'mixed' for connection type since it doesn't hurt +# default 'mixed' for connection type since it doesn't hurt + + def Replay(obj, name, replay_dir, key=None, cert=None, conn_type='mixed', options={}): # ATS setup - one line because we leave records and remap config to user - ts = obj.MakeATSProcess("ts", select_ports=False) # select ports can be disabled once we add ssl port selection in extension + ts = obj.MakeATSProcess("ts", select_ports=False) # select ports can be disabled once we add ssl port selection in extension # TEMP ts.Variables.ssl_port = 4443 @@ -61,8 +63,9 @@ def Replay(obj, name, replay_dir, key=None, cert=None, conn_type='mixed', option if not cert: cert = os.path.join(obj.Variables["AtsTestToolsDir"], "microserver", "ssl", "server.crt") - - command = 'traffic-replay --log_dir {0} --type {1} --verify --host {2} --port {3} --s_port {4} '.format(data_dir, conn_type, hostIP, ts.Variables.port, ts.Variables.ssl_port) + + command = 'traffic-replay --log_dir {0} --type {1} --verify --host {2} --port {3} --s_port {4} '.format( + data_dir, conn_type, hostIP, ts.Variables.port, ts.Variables.ssl_port) if key: command += "-k {0} ".format(key) @@ -88,4 +91,5 @@ def Replay(obj, name, replay_dir, key=None, cert=None, conn_type='mixed', option # return all the stuff in case user wants to do extra optimization return (ts, server, dns, tr) + AddTestRunSet(Replay) diff --git a/tests/gold_tests/autest-site/trafficserver.test.ext b/tests/gold_tests/autest-site/trafficserver.test.ext index 0ca41eb12a6..423af0f1cf1 100755 --- a/tests/gold_tests/autest-site/trafficserver.test.ext +++ b/tests/gold_tests/autest-site/trafficserver.test.ext @@ -18,9 +18,7 @@ from __future__ import print_function import os -import socket from ports import get_port -from autest.api import AddWhenFunction def make_id(s): @@ -263,6 +261,7 @@ def MakeATSProcess(obj, name, command='traffic_server', select_ports=True, enabl # set up default ports # get some ports TODO make it so we can hold on to the socket if select_ports: + # some system have a bug in which ipv4 and ipv6 share port space # Make two different ports to avoid this get_port(p, "port") @@ -270,12 +269,13 @@ def MakeATSProcess(obj, name, command='traffic_server', select_ports=True, enabl if enable_tls: get_port(p, "ssl_port") + get_port(p, "ssl_portv6") # p.Ready = When.PortOpen(p.Variables.port) else: p.Variables.port = 8080 p.Variables.portv6 = 8080 p.Variables.ssl_port = 4443 - + get_port(p, "manager_port") get_port(p, "admin_port") @@ -289,7 +289,7 @@ def MakeATSProcess(obj, name, command='traffic_server', select_ports=True, enabl port_str = "{port} {v6_port}:ipv6 ".format(port=p.Variables.port, v6_port=p.Variables.portv6) if enable_tls: - port_str += "{ssl_port}:ssl".format(ssl_port=p.Variables.ssl_port) + port_str += "{ssl_port}:ssl {ssl_portv6}:ssl:ipv6".format(ssl_port=p.Variables.ssl_port,ssl_portv6=p.Variables.ssl_portv6) p.Env['PROXY_CONFIG_HTTP_SERVER_PORTS'] = port_str @@ -304,7 +304,6 @@ def MakeATSProcess(obj, name, command='traffic_server', select_ports=True, enabl else: p.ReturnCode = 0 - return p diff --git a/tests/gold_tests/autest-site/trafficserver_plugins.test.ext b/tests/gold_tests/autest-site/trafficserver_plugins.test.ext index e6e3087af7a..4e234bc489c 100644 --- a/tests/gold_tests/autest-site/trafficserver_plugins.test.ext +++ b/tests/gold_tests/autest-site/trafficserver_plugins.test.ext @@ -19,47 +19,101 @@ Builds, installs, and enables an ATS plugin in the sandbox environment import os -def prepare_plugin(self, path, tsproc, plugin_args = "", extra_build_args=''): - """Builds, installs, and enables an ATS plugin in the sandbox environment +import hosts.output as host - The source file at the given path is copied to the sandbox directory of the - given traffic server process and compiled into a binary with the file - extensioned replaced with '.so'. An entry for this plugin is added to - the 'plugin.config' file.""" - # Copy the source to the sandbox directory. +def prepare_plugin_helper(so_name, tsproc, plugin_args="", copy_plugin=True): + """ + Installs and enables an ATS plugin in the sandbox environment. + + Args: + so_name (str): The path or filename of a built .so file. + + tsproc (Process): The Traffic Server process whose plugin.config should + be configured to use the provided plugin. + + plugin_args (str): The arguments to provide the plugin in the + plugin.config. + + copy_plugin (bool): Whether to copy the plugin to the sandbox's plugin + directory. + """ + + filename, extension = os.path.splitext(so_name) + if extension != ".so": + raise ValueError('so_name argument must have a ".so" extension. ' + 'Received: {}'.format(so_name)) + plugin_dir = tsproc.Env['PROXY_CONFIG_PLUGIN_PLUGIN_DIR'] - tsproc.Setup.Copy(path, plugin_dir) - - tsxs = os.path.join(self.Variables.BINDIR,'tsxs') - # get the top level object ( ie Test) to add a condition - # need to change this API in AuTest to be a better name as it now has value - # to be called by user API - dragon512 - self._RootRunable.SkipUnless( - Condition.HasProgram(tsxs, "tsxs needs be installed with trafficserver package for this test to run") - ) - - link_gxx="export CC=c++ &&" - # Compile the plugin. - in_basename = os.path.basename(path) - if in_basename.endswith(".c"): - link_gxx = '' - in_path = os.path.join(plugin_dir, in_basename) - out_basename = os.path.splitext(in_basename)[0] + '.so' - out_path = os.path.join(plugin_dir, out_basename) - tsproc.Setup.RunCommand( - "{pre_args} {tsxs} {args} -c {0} -o {1}".format( - in_path, - out_path, - tsxs=tsxs, - args=" -L {0} {1}".format(self.Variables.LIBDIR, extra_build_args), - pre_args=link_gxx) - ) + if copy_plugin: + host.WriteVerbose( + "prepare_plugin", + "Copying down {} into {}.".format(so_name, plugin_dir)) + tsproc.Setup.Copy(so_name, plugin_dir) + else: + host.WriteVerbose( + "prepare_plugin", + "Skipping copying {} into {} due to configuration.".format( + so_name, plugin_dir)) # Add an entry to plugin.config. - tsproc.Disk.plugin_config.AddLine("{0} {1}".format(out_basename,plugin_args)) + basename = os.path.basename(so_name) + config_line = "{0} {1}".format(basename, plugin_args) + host.WriteVerbose( + "prepare_plugin", + 'Adding line to plugin.config: "{}"'.format(config_line)) + tsproc.Disk.plugin_config.AddLine(config_line) + + +def prepare_test_plugin(self, so_path, tsproc, plugin_args=""): + """ + Installs and enables an ATS plugin in the sandbox environment. + + Args: + so_path (str): The path to a built .so file. + + tsproc (Process): The Traffic Server process whose plugin.config should + be configured to use the provided plugin. + + plugin_args (str): The arguments to provide the plugin in the + plugin.config. + """ + if not os.path.exists(so_path): + raise ValueError( + 'PrepareTestPlugin: file does not exist: "{}"'.format(so_path)) + + prepare_plugin_helper(so_path, tsproc, plugin_args, copy_plugin=True) + + +def prepare_installed_plugin(self, so_name, tsproc, plugin_args=""): + """ + Configures an already-installed ATS plugin in the sandbox environment. + + Args: + so_name (str): The name of a plugin to configure. + + tsproc (Process): The Traffic Server process whose plugin.config should + be configured to use the provided plugin. + + plugin_args (str): The arguments to provide the plugin in the + plugin.config. + """ + if os.path.dirname(so_name): + raise ValueError( + 'PrepareInstalledPlugin expects a filename not a path: ' + '"{}"'.format(so_name)) + prepare_plugin_helper(so_name, tsproc, plugin_args, copy_plugin=False) + -# remove this later -ExtendTest(prepare_plugin, name="prepare_plugin") +""" +PrepareTestPlugin should be used for the test-specific plugins that need to +be copied down into the sandbox directory. +""" +ExtendTest(prepare_test_plugin, name="PrepareTestPlugin") -ExtendTest(prepare_plugin, name="PreparePlugin") +""" +PrepareInstalledPlugin should be used for the plugins installed via Automake +make install. They are already sym linked into the test directory via the +MakeATSProcess extension. +""" +ExtendTest(prepare_installed_plugin, name="PrepareInstalledPlugin") diff --git a/tests/gold_tests/autest-site/when.test.ext b/tests/gold_tests/autest-site/when.test.ext new file mode 100644 index 00000000000..3f0a27600ca --- /dev/null +++ b/tests/gold_tests/autest-site/when.test.ext @@ -0,0 +1,85 @@ +''' +When extensions. +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from autest.api import AddWhenFunction +import hosts.output as host +import os +import re + + +def FileContains(haystack, needle, desired_count=1): + """ + Return whether the file haystack contains the string needle. + + Args: + haystack (str): The path to the file to be inspected. + needle (str): The content to look for in haystack. This can be a + regular expression which will be used in Python's re.search + function. + desired_count (int): How many times the caller desires to see needle in + haystack before considering the When condition fulfilled. + + Returns: + True if the haystack exists as a file and contains needle, False + otherwise. + """ + + if desired_count < 0: + raise ValueError("Cannot pass a negative desired_count value to FileContains.") + if desired_count == 0: + raise ValueError("Cannot pass a desired_count of 0 to FileContains.") + + if not os.path.exists(haystack): + host.WriteDebug( + ['FileContains', 'when'], + "Testing for file content '{0}' in file '{1}': file does not exist".format( + needle, haystack)) + return False + + needle_regex = re.compile(needle) + with open(haystack) as f: + needle_seen_count = 0 + line_count = 0 + for line in f: + line_count += 1 + if needle_regex.search(line): + host.WriteDebug( + ['FileContains', 'when'], + "Found '{0}' in file '{1}' in line: '{2}', line number: {3}".format( + needle, haystack, line.rstrip(), line_count)) + needle_seen_count += 1 + + if needle_seen_count >= desired_count: + host.WriteDebug( + ['FileContains', 'when'], + "Testing for file content '{0}' in file '{1}', " + "successfully found it the desired {2} times".format( + needle, haystack, needle_seen_count)) + return True + + host.WriteDebug( + ['FileContains', 'when'], + "Testing for file content '{0}' in file '{1}', only seen {2} " + "out of the desired {3} times".format( + needle, haystack, needle_seen_count, desired_count)) + + return False + + +AddWhenFunction(FileContains) diff --git a/tests/gold_tests/basic/config.test.py b/tests/gold_tests/basic/config.test.py index 4045b86ce1d..bfe048b1854 100644 --- a/tests/gold_tests/basic/config.test.py +++ b/tests/gold_tests/basic/config.test.py @@ -27,4 +27,3 @@ t.Command = "curl 127.0.0.1:{port}".format(port=ts.Variables.port) t.ReturnCode = 0 t.StillRunningAfter = ts - diff --git a/tests/gold_tests/basic/copy_config.test.py b/tests/gold_tests/basic/copy_config.test.py index 8cfea1bbd7d..612f6531440 100644 --- a/tests/gold_tests/basic/copy_config.test.py +++ b/tests/gold_tests/basic/copy_config.test.py @@ -43,4 +43,4 @@ t.Command = "curl 127.0.0.1:{port}".format(port=ts2.Variables.port) t.ReturnCode = 0 t.StillRunningAfter = ts1 -t.StillRunningAfter += ts2 \ No newline at end of file +t.StillRunningAfter += ts2 diff --git a/tests/gold_tests/basic/deny0.test.py b/tests/gold_tests/basic/deny0.test.py index 8b451632d26..cf808c4ac53 100644 --- a/tests/gold_tests/basic/deny0.test.py +++ b/tests/gold_tests/basic/deny0.test.py @@ -18,14 +18,13 @@ # limitations under the License. import os -import socket Test.Summary = ''' Test that Trafficserver rejects requests for host 0 ''' Test.ContinueOnFail = True -HOST1='redirect.test' +HOST1 = 'redirect.test' redirect_serv = Test.MakeOriginServer("redirect_serv", ip='0.0.0.0') @@ -34,17 +33,11 @@ ts = Test.MakeATSProcess("ts") ts.Disk.records_config.update({ - 'proxy.config.diags.debug.enabled': 1 - ,'proxy.config.diags.debug.tags': 'http|dns|redirect' - ,'proxy.config.http.redirection_enabled': 1 - ,'proxy.config.http.number_of_redirections': 1 - ,'proxy.config.http.cache.http': 0 - ,'proxy.config.dns.nameservers': '127.0.0.1:{0}'.format(dns.Variables.Port) - ,'proxy.config.dns.resolv_conf': 'NULL' - ,'proxy.config.url_remap.remap_required': 0 # need this so the domain gets a chance to be evaluated through DNS + # need this so the domain gets a chance to be evaluated through DNS + 'proxy.config.diags.debug.enabled': 1, 'proxy.config.diags.debug.tags': 'http|dns|redirect', 'proxy.config.http.redirection_enabled': 1, 'proxy.config.http.number_of_redirections': 1, 'proxy.config.http.cache.http': 0, 'proxy.config.dns.nameservers': '127.0.0.1:{0}'.format(dns.Variables.Port), 'proxy.config.dns.resolv_conf': 'NULL', 'proxy.config.url_remap.remap_required': 0 }) -Test.Setup.Copy(os.path.join(Test.Variables.AtsTestToolsDir,'tcp_client.py')) +Test.Setup.Copy(os.path.join(Test.Variables.AtsTestToolsDir, 'tcp_client.py')) data_dirname = 'generated_test_data' data_path = os.path.join(Test.TestDirectory, data_dirname) @@ -54,30 +47,33 @@ f.write('HTTP/1.1 400 Bad Destination Address\r\n') isFirstTest = True + + def buildMetaTest(testName, requestString): - tr = Test.AddTestRun(testName) - global isFirstTest - if isFirstTest: - isFirstTest = False - tr.Processes.Default.StartBefore(ts, ready=When.PortOpen(ts.Variables.port)) - tr.Processes.Default.StartBefore(redirect_serv, ready=When.PortOpen(redirect_serv.Variables.Port)) - tr.Processes.Default.StartBefore(dns) - with open(os.path.join(data_path, tr.Name), 'w') as f: - f.write(requestString) - tr.Processes.Default.Command = "python tcp_client.py 127.0.0.1 {0} {1} | head -1".format(ts.Variables.port, os.path.join(data_dirname, tr.Name)) - tr.ReturnCode = 0 - tr.Processes.Default.Streams.stdout = gold_filepath - tr.StillRunningAfter = ts - tr.StillRunningAfter = redirect_serv - tr.StillRunningAfter = dns + tr = Test.AddTestRun(testName) + global isFirstTest + if isFirstTest: + isFirstTest = False + tr.Processes.Default.StartBefore(ts, ready=When.PortOpen(ts.Variables.port)) + tr.Processes.Default.StartBefore(redirect_serv, ready=When.PortOpen(redirect_serv.Variables.Port)) + tr.Processes.Default.StartBefore(dns) + with open(os.path.join(data_path, tr.Name), 'w') as f: + f.write(requestString) + tr.Processes.Default.Command = "python3 tcp_client.py 127.0.0.1 {0} {1} | head -1".format( + ts.Variables.port, os.path.join(data_dirname, tr.Name)) + tr.ReturnCode = 0 + tr.Processes.Default.Streams.stdout = gold_filepath + tr.StillRunningAfter = ts + tr.StillRunningAfter = redirect_serv + tr.StillRunningAfter = dns buildMetaTest('RejectInterfaceAnyIpv4', - 'GET / HTTP/1.1\r\nHost: 0:{port}\r\nConnection: close\r\n\r\n'.format(port=ts.Variables.port)) + 'GET / HTTP/1.1\r\nHost: 0:{port}\r\nConnection: close\r\n\r\n'.format(port=ts.Variables.port)) buildMetaTest('RejectInterfaceAnyIpv6', - 'GET / HTTP/1.1\r\nHost: [::]:{port}\r\nConnection: close\r\n\r\n'.format(port=ts.Variables.portv6)) + 'GET / HTTP/1.1\r\nHost: [::]:{port}\r\nConnection: close\r\n\r\n'.format(port=ts.Variables.portv6)) # Sets up redirect to IPv4 ANY address @@ -87,7 +83,7 @@ def buildMetaTest(testName, requestString): redirect_serv.addResponse("sessionfile.log", redirect_request_header, redirect_response_header) buildMetaTest('RejectRedirectToInterfaceAnyIpv4', - 'GET /redirect-0 HTTP/1.1\r\nHost: {host}:{port}\r\n\r\n'.format(host=HOST1, port=redirect_serv.Variables.Port)) + 'GET /redirect-0 HTTP/1.1\r\nHost: {host}:{port}\r\n\r\n'.format(host=HOST1, port=redirect_serv.Variables.Port)) # Sets up redirect to IPv6 ANY address @@ -97,7 +93,7 @@ def buildMetaTest(testName, requestString): redirect_serv.addResponse("sessionfile.log", redirect_request_header, redirect_response_header) buildMetaTest('RejectRedirectToInterfaceAnyIpv6', - 'GET /redirect-0v6 HTTP/1.1\r\nHost: {host}:{port}\r\n\r\n'.format(host=HOST1, port=redirect_serv.Variables.Port)) + 'GET /redirect-0v6 HTTP/1.1\r\nHost: {host}:{port}\r\n\r\n'.format(host=HOST1, port=redirect_serv.Variables.Port)) Test.Setup.Copy(data_path) diff --git a/tests/gold_tests/body_factory/data/www.customplugin204.test_get.txt b/tests/gold_tests/body_factory/data/www.customplugin204.test_get.txt index be603f95f17..4cf018aea16 100644 --- a/tests/gold_tests/body_factory/data/www.customplugin204.test_get.txt +++ b/tests/gold_tests/body_factory/data/www.customplugin204.test_get.txt @@ -1,2 +1,2 @@ -GET HTTP://www.customplugin204.test/ HTTP/1.1 - +GET HTTP://www.customplugin204.test/ HTTP/1.1 + diff --git a/tests/gold_tests/body_factory/data/www.customtemplate204.test_get.txt b/tests/gold_tests/body_factory/data/www.customtemplate204.test_get.txt index 395d7985f69..f16fbc74628 100644 --- a/tests/gold_tests/body_factory/data/www.customtemplate204.test_get.txt +++ b/tests/gold_tests/body_factory/data/www.customtemplate204.test_get.txt @@ -1,2 +1,2 @@ -GET HTTP://www.customtemplate204.test/ HTTP/1.1 - +GET HTTP://www.customtemplate204.test/ HTTP/1.1 + diff --git a/tests/gold_tests/body_factory/data/www.default204.test_get.txt b/tests/gold_tests/body_factory/data/www.default204.test_get.txt index e77408af827..1a421ddf182 100644 --- a/tests/gold_tests/body_factory/data/www.default204.test_get.txt +++ b/tests/gold_tests/body_factory/data/www.default204.test_get.txt @@ -1,2 +1,2 @@ -GET HTTP://www.default204.test/ HTTP/1.1 - +GET HTTP://www.default204.test/ HTTP/1.1 + diff --git a/tests/gold_tests/body_factory/data/www.default304.test_get.txt b/tests/gold_tests/body_factory/data/www.default304.test_get.txt index c9064fa0ade..d251435b982 100644 --- a/tests/gold_tests/body_factory/data/www.default304.test_get.txt +++ b/tests/gold_tests/body_factory/data/www.default304.test_get.txt @@ -1,2 +1,2 @@ -GET HTTP://www.default304.test/ HTTP/1.1 - +GET HTTP://www.default304.test/ HTTP/1.1 + diff --git a/tests/gold_tests/body_factory/data/www.example.test_get_200.txt b/tests/gold_tests/body_factory/data/www.example.test_get_200.txt index c53681f3486..5994603557f 100644 --- a/tests/gold_tests/body_factory/data/www.example.test_get_200.txt +++ b/tests/gold_tests/body_factory/data/www.example.test_get_200.txt @@ -1,3 +1,3 @@ -GET /get200 HTTP/1.1 -Host: www.example.test - +GET /get200 HTTP/1.1 +Host: www.example.test + diff --git a/tests/gold_tests/body_factory/data/www.example.test_get_304.txt b/tests/gold_tests/body_factory/data/www.example.test_get_304.txt index 8d0aecf9bdb..03a8d5967eb 100644 --- a/tests/gold_tests/body_factory/data/www.example.test_get_304.txt +++ b/tests/gold_tests/body_factory/data/www.example.test_get_304.txt @@ -1,4 +1,4 @@ -GET /get304 HTTP/1.1 -Host: www.example.test -If-Modified-Since: Thu, 1 Jan 1970 00:00:00 GMT - +GET /get304 HTTP/1.1 +Host: www.example.test +If-Modified-Since: Thu, 1 Jan 1970 00:00:00 GMT + diff --git a/tests/gold_tests/body_factory/data/www.example.test_head.txt b/tests/gold_tests/body_factory/data/www.example.test_head.txt index c5a5c979e9e..514c1071065 100644 --- a/tests/gold_tests/body_factory/data/www.example.test_head.txt +++ b/tests/gold_tests/body_factory/data/www.example.test_head.txt @@ -1,3 +1,3 @@ -HEAD http://www.example.test/ HTTP/1.1 -Host: www.example.test - +HEAD http://www.example.test/ HTTP/1.1 +Host: www.example.test + diff --git a/tests/gold_tests/body_factory/data/www.example.test_head_200.txt b/tests/gold_tests/body_factory/data/www.example.test_head_200.txt index 6d214e1b365..fc0f1d536fd 100644 --- a/tests/gold_tests/body_factory/data/www.example.test_head_200.txt +++ b/tests/gold_tests/body_factory/data/www.example.test_head_200.txt @@ -1,3 +1,3 @@ -HEAD /head200 HTTP/1.1 -Host: www.example.test - +HEAD /head200 HTTP/1.1 +Host: www.example.test + diff --git a/tests/gold_tests/body_factory/http204_response.test.py b/tests/gold_tests/body_factory/http204_response.test.py index f7f21df5f28..34e531b188f 100644 --- a/tests/gold_tests/body_factory/http204_response.test.py +++ b/tests/gold_tests/body_factory/http204_response.test.py @@ -76,7 +76,7 @@ defaultTr.Processes.Default.StartBefore(Test.Processes.ts) defaultTr.StillRunningAfter = ts -defaultTr.Processes.Default.Command = "python tcp_client.py 127.0.0.1 {0} {1} | grep -v '^Date: '| grep -v '^Server: ATS/'".\ +defaultTr.Processes.Default.Command = "python3 tcp_client.py 127.0.0.1 {0} {1} | grep -v '^Date: '| grep -v '^Server: ATS/'".\ format(ts.Variables.port, 'data/{0}_get.txt'.format(DEFAULT_204_HOST)) defaultTr.Processes.Default.TimeOut = 5 # seconds defaultTr.Processes.Default.ReturnCode = 0 @@ -86,7 +86,7 @@ customTemplateTr = Test.AddTestRun("Test domain {0}".format(CUSTOM_TEMPLATE_204_HOST)) customTemplateTr.StillRunningBefore = ts customTemplateTr.StillRunningAfter = ts -customTemplateTr.Processes.Default.Command = "python tcp_client.py 127.0.0.1 {0} {1} | grep -v '^Date: '| grep -v '^Server: ATS/'".\ +customTemplateTr.Processes.Default.Command = "python3 tcp_client.py 127.0.0.1 {0} {1} | grep -v '^Date: '| grep -v '^Server: ATS/'".\ format(ts.Variables.port, 'data/{0}_get.txt'.format(CUSTOM_TEMPLATE_204_HOST)) customTemplateTr.Processes.Default.TimeOut = 5 # seconds customTemplateTr.Processes.Default.ReturnCode = 0 diff --git a/tests/gold_tests/body_factory/http204_response_plugin.test.py b/tests/gold_tests/body_factory/http204_response_plugin.test.py index ffc692db5cf..31e1b114f9d 100644 --- a/tests/gold_tests/body_factory/http204_response_plugin.test.py +++ b/tests/gold_tests/body_factory/http204_response_plugin.test.py @@ -37,7 +37,7 @@ ) ts.Disk.MakeConfigFile(regex_remap_conf_file).AddLine('//.*/ http://donotcare.test @status=204') -Test.PreparePlugin(os.path.join(Test.Variables.AtsTestToolsDir, 'plugins', 'custom204plugin.cc'), ts) +Test.PrepareTestPlugin(os.path.join(Test.Variables.AtsTestPluginsDir, 'custom204plugin.so'), ts) Test.Setup.Copy(os.path.join(os.pardir, os.pardir, 'tools', 'tcp_client.py')) Test.Setup.Copy('data') @@ -46,7 +46,7 @@ tr.Processes.Default.StartBefore(Test.Processes.ts) tr.StillRunningAfter = ts -tr.Processes.Default.Command = "python tcp_client.py 127.0.0.1 {0} {1} | grep -v '^Date: '| grep -v '^Server: ATS/'".\ +tr.Processes.Default.Command = "python3 tcp_client.py 127.0.0.1 {0} {1} | grep -v '^Date: '| grep -v '^Server: ATS/'".\ format(ts.Variables.port, 'data/{0}_get.txt'.format(CUSTOM_PLUGIN_204_HOST)) tr.Processes.Default.TimeOut = 5 # seconds tr.Processes.Default.ReturnCode = 0 diff --git a/tests/gold_tests/body_factory/http304_response.test.py b/tests/gold_tests/body_factory/http304_response.test.py index 8ce8ee78f7f..79e0e8c648b 100644 --- a/tests/gold_tests/body_factory/http304_response.test.py +++ b/tests/gold_tests/body_factory/http304_response.test.py @@ -47,7 +47,7 @@ tr.Processes.Default.StartBefore(Test.Processes.ts) tr.StillRunningAfter = ts -cmd_tpl = "python tcp_client.py 127.0.0.1 {0} {1} | grep -v '^Date: '| grep -v '^Server: ATS/' | sed 's;ApacheTrafficServer\/[^ ]*;VERSION;'" +cmd_tpl = r"python3 tcp_client.py 127.0.0.1 {0} {1} | grep -v '^Date: '| grep -v '^Server: ATS/' | sed 's;ApacheTrafficServer\/[^ ]*;VERSION;'" tr.Processes.Default.Command = cmd_tpl.format(ts.Variables.port, 'data/{0}_get.txt'.format(DEFAULT_304_HOST)) tr.Processes.Default.TimeOut = 5 # seconds tr.Processes.Default.ReturnCode = 0 diff --git a/tests/gold_tests/body_factory/http_head_no_origin.test.py b/tests/gold_tests/body_factory/http_head_no_origin.test.py index 4e918fdeb9f..92a4a6c60a8 100644 --- a/tests/gold_tests/body_factory/http_head_no_origin.test.py +++ b/tests/gold_tests/body_factory/http_head_no_origin.test.py @@ -36,7 +36,7 @@ tr.Processes.Default.StartBefore(Test.Processes.ts) tr.StillRunningAfter = ts -tr.Processes.Default.Command = "python tcp_client.py 127.0.0.1 {0} {1} | grep -v '^Date: '| grep -v '^Server: ATS/'".\ +tr.Processes.Default.Command = "python3 tcp_client.py 127.0.0.1 {0} {1} | grep -v '^Date: '| grep -v '^Server: ATS/'".\ format(ts.Variables.port, 'data/{0}_head.txt'.format(HOST)) tr.Processes.Default.TimeOut = 5 # seconds tr.Processes.Default.ReturnCode = 0 diff --git a/tests/gold_tests/body_factory/http_with_origin.test.py b/tests/gold_tests/body_factory/http_with_origin.test.py index 6de8e84743e..7d418c4f396 100644 --- a/tests/gold_tests/body_factory/http_with_origin.test.py +++ b/tests/gold_tests/body_factory/http_with_origin.test.py @@ -73,7 +73,7 @@ trhead200.StillRunningAfter = ts trhead200.StillRunningAfter = server -trhead200.Processes.Default.Command = "python tcp_client.py 127.0.0.1 {0} {1} | grep -v '^Date: '| grep -v '^Server: ATS/'".\ +trhead200.Processes.Default.Command = "python3 tcp_client.py 127.0.0.1 {0} {1} | grep -v '^Date: '| grep -v '^Server: ATS/'".\ format(ts.Variables.port, 'data/{0}_head_200.txt'.format(HOST)) trhead200.Processes.Default.TimeOut = 5 # seconds trhead200.Processes.Default.ReturnCode = 0 @@ -86,7 +86,7 @@ trget200.StillRunningAfter = ts trget200.StillRunningAfter = server -trget200.Processes.Default.Command = "python tcp_client.py 127.0.0.1 {0} {1} | grep -v '^Date: '| grep -v '^Server: ATS/'".\ +trget200.Processes.Default.Command = "python3 tcp_client.py 127.0.0.1 {0} {1} | grep -v '^Date: '| grep -v '^Server: ATS/'".\ format(ts.Variables.port, 'data/{0}_get_200.txt'.format(HOST)) trget200.Processes.Default.TimeOut = 5 # seconds trget200.Processes.Default.ReturnCode = 0 @@ -99,7 +99,7 @@ trget304.StillRunningAfter = ts trget304.StillRunningAfter = server -cmd_tpl = "python tcp_client.py 127.0.0.1 {0} {1} | grep -v '^Date: '| grep -v '^Server: ATS/' | sed 's;ApacheTrafficServer\/[^ ]*;VERSION;'" +cmd_tpl = r"python3 tcp_client.py 127.0.0.1 {0} {1} | grep -v '^Date: '| grep -v '^Server: ATS/' | sed 's;ApacheTrafficServer\/[^ ]*;VERSION;'" trget304.Processes.Default.Command = cmd_tpl.format(ts.Variables.port, 'data/{0}_get_304.txt'.format(HOST)) trget304.Processes.Default.TimeOut = 5 # seconds trget304.Processes.Default.ReturnCode = 0 diff --git a/tests/gold_tests/cache/cache-control.test.py b/tests/gold_tests/cache/cache-control.test.py index d66167e156b..3f59659282c 100644 --- a/tests/gold_tests/cache/cache-control.test.py +++ b/tests/gold_tests/cache/cache-control.test.py @@ -17,25 +17,39 @@ # See the License for the specific language governing permissions and # limitations under the License. -import os Test.Summary = ''' Test cached responses and requests with bodies ''' - + Test.ContinueOnFail = True # Define default ATS ts = Test.MakeATSProcess("ts") server = Test.MakeOriginServer("server") -#**testname is required** +# **testname is required** testName = "" request_header1 = {"headers": "GET / HTTP/1.1\r\nHost: www.example.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} -response_header1 = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\nCache-Control: max-age=300\r\n\r\n", "timestamp": "1469733493.993", "body": "xxx"} -request_header2 = {"headers": "GET /no_cache_control HTTP/1.1\r\nHost: www.example.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} -response_header2 = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": "the flinstones"} -request_header3 = {"headers": "GET /max_age_10sec HTTP/1.1\r\nHost: www.example.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} -response_header3 = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\nCache-Control: max-age=10,public\r\n\r\n", "timestamp": "1469733493.993", "body": "yabadabadoo"} +response_header1 = { + "headers": "HTTP/1.1 200 OK\r\nConnection: close\r\nCache-Control: max-age=300\r\n\r\n", + "timestamp": "1469733493.993", + "body": "xxx"} +request_header2 = { + "headers": "GET /no_cache_control HTTP/1.1\r\nHost: www.example.com\r\n\r\n", + "timestamp": "1469733493.993", + "body": ""} +response_header2 = { + "headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", + "timestamp": "1469733493.993", + "body": "the flinstones"} +request_header3 = { + "headers": "GET /max_age_10sec HTTP/1.1\r\nHost: www.example.com\r\n\r\n", + "timestamp": "1469733493.993", + "body": ""} +response_header3 = { + "headers": "HTTP/1.1 200 OK\r\nConnection: close\r\nCache-Control: max-age=10,public\r\n\r\n", + "timestamp": "1469733493.993", + "body": "yabadabadoo"} server.addResponse("sessionlog.json", request_header1, response_header1) server.addResponse("sessionlog.json", request_header2, response_header2) server.addResponse("sessionlog.json", request_header3, response_header3) @@ -59,42 +73,48 @@ tr = Test.AddTestRun() tr.Processes.Default.StartBefore(server) tr.Processes.Default.StartBefore(Test.Processes.ts, ready=1) -tr.Processes.Default.Command = 'curl -s -D - -v --ipv4 --http1.1 -H "x-debug: x-cache,via" -H "Host: www.example.com" http://localhost:{port}/max_age_10sec'.format(port=ts.Variables.port) +tr.Processes.Default.Command = 'curl -s -D - -v --ipv4 --http1.1 -H "x-debug: x-cache,via" -H "Host: www.example.com" http://localhost:{port}/max_age_10sec'.format( + port=ts.Variables.port) tr.Processes.Default.ReturnCode = 0 tr.Processes.Default.Streams.stdout = "gold/cache_and_req_body-miss.gold" tr.StillRunningAfter = ts # Test 2 - 200 cached response and using netcat tr = Test.AddTestRun() -tr.Processes.Default.Command = "printf 'GET /max_age_10sec HTTP/1.1\r\n''x-debug: x-cache,x-cache-key,via\r\n''Host: www.example.com\r\n''\r\n'|nc 127.0.0.1 -w 1 {port}".format(port=ts.Variables.port) +tr.Processes.Default.Command = "printf 'GET /max_age_10sec HTTP/1.1\r\n''x-debug: x-cache,x-cache-key,via\r\n''Host: www.example.com\r\n''\r\n'|nc 127.0.0.1 -w 1 {port}".format( + port=ts.Variables.port) tr.Processes.Default.ReturnCode = 0 tr.Processes.Default.Streams.stdout = "gold/cache_and_req_body-hit.gold" tr.StillRunningAfter = ts # Test 3 - response with no cache control, so cache-miss every time tr = Test.AddTestRun() -tr.Processes.Default.Command = "printf 'GET /no_cache_control HTTP/1.1\r\n''x-debug: x-cache,x-cache-key,via\r\n''Host: www.example.com\r\n''\r\n'|nc 127.0.0.1 -w 1 {port}".format(port=ts.Variables.port) +tr.Processes.Default.Command = "printf 'GET /no_cache_control HTTP/1.1\r\n''x-debug: x-cache,x-cache-key,via\r\n''Host: www.example.com\r\n''\r\n'|nc 127.0.0.1 -w 1 {port}".format( + port=ts.Variables.port) tr.Processes.Default.ReturnCode = 0 tr.Processes.Default.Streams.stdout = "gold/cache_no_cc.gold" tr.StillRunningAfter = ts # Test 4 - Cache-Control: no-cache (from client), so cache miss every time tr = Test.AddTestRun() -tr.Processes.Default.Command = "printf 'GET /no_cache_control HTTP/1.1\r\n''Cache-Control:no-cache\r\n''x-debug: x-cache,x-cache-key,via\r\n''Host: www.example.com\r\n''\r\n'|nc 127.0.0.1 -w 1 {port}".format(port=ts.Variables.port) +tr.Processes.Default.Command = "printf 'GET /no_cache_control HTTP/1.1\r\n''Cache-Control:no-cache\r\n''x-debug: x-cache,x-cache-key,via\r\n''Host: www.example.com\r\n''\r\n'|nc 127.0.0.1 -w 1 {port}".format( + port=ts.Variables.port) tr.Processes.Default.ReturnCode = 0 tr.Processes.Default.Streams.stdout = "gold/cache_no_cc.gold" tr.StillRunningAfter = ts # Test 5 - hit stale cache. tr = Test.AddTestRun() -tr.Processes.Default.Command = "sleep 15; printf 'GET /max_age_10sec HTTP/1.1\r\n''x-debug: x-cache,x-cache-key,via\r\n''Host: www.example.com\r\n''\r\n'|nc 127.0.0.1 -w 1 {port}".format(port=ts.Variables.port) +tr.Processes.Default.Command = "sleep 15; printf 'GET /max_age_10sec HTTP/1.1\r\n''x-debug: x-cache,x-cache-key,via\r\n''Host: www.example.com\r\n''\r\n'|nc 127.0.0.1 -w 1 {port}".format( + port=ts.Variables.port) tr.Processes.Default.ReturnCode = 0 tr.Processes.Default.Streams.stdout = "gold/cache_hit_stale.gold" tr.StillRunningAfter = ts # Test 6 - only-if-cached. 504 "Not Cached" should be returned if not in cache tr = Test.AddTestRun() -tr.Processes.Default.Command = "printf 'GET /no_cache_control HTTP/1.1\r\n''Cache-Control: only-if-cached\r\n''x-debug: x-cache,x-cache-key,via\r\n''Host: www.example.com\r\n''Cache-control: max-age=300\r\n''\r\n'|nc 127.0.0.1 -w 1 {port}".format(port=ts.Variables.port) +tr.Processes.Default.Command = "printf 'GET /no_cache_control HTTP/1.1\r\n''Cache-Control: only-if-cached\r\n''x-debug: x-cache,x-cache-key,via\r\n''Host: www.example.com\r\n''Cache-control: max-age=300\r\n''\r\n'|nc 127.0.0.1 -w 1 {port}".format( + port=ts.Variables.port) tr.Processes.Default.ReturnCode = 0 tr.Processes.Default.Streams.stdout = "gold/cache_no_cache.gold" tr.StillRunningAfter = ts diff --git a/tests/gold_tests/cache/cache-generation-disjoint.test.py b/tests/gold_tests/cache/cache-generation-disjoint.test.py index 7c6bd21f9a9..e6da37b6d3d 100644 --- a/tests/gold_tests/cache/cache-generation-disjoint.test.py +++ b/tests/gold_tests/cache/cache-generation-disjoint.test.py @@ -21,7 +21,7 @@ Test.Summary = ''' Test that the same URL path in different cache generations creates disjoint objects ''' -# need Curl + Test.ContinueOnFail = True # Define default ATS ts = Test.MakeATSProcess("ts") diff --git a/tests/gold_tests/cache/disjoint-wait-for-cache.test.py b/tests/gold_tests/cache/disjoint-wait-for-cache.test.py index 446f5421263..a0a20f4ae62 100644 --- a/tests/gold_tests/cache/disjoint-wait-for-cache.test.py +++ b/tests/gold_tests/cache/disjoint-wait-for-cache.test.py @@ -58,35 +58,35 @@ # Same URL in generation 1 is a MISS. tr = Test.AddTestRun() -tr.Processes.Default.Command = 'curl "http://127.0.0.1:{0}/generation1/cache/10/{1}" -H "x-debug: x-cache,x-cache-key,via,x-cache-generation" --verbose'.format( +tr.Processes.Default.Command = 'curl "http://127.0.0.1:{0}/generation1/cache/10/{1}" -H "x-debug: x-cache,x-cache-key,via,x-cache-generation" --verbose -o /dev/null'.format( ts.Variables.port, objectid) tr.Processes.Default.ReturnCode = 0 tr.Processes.Default.Streams.All = "gold/miss_gen1.gold" # Same URL in generation 2 is still a MISS. tr = Test.AddTestRun() -tr.Processes.Default.Command = 'curl "http://127.0.0.1:{0}/generation2/cache/10/{1}" -H "x-debug: x-cache,x-cache-key,via,x-cache-generation" --verbose'.format( +tr.Processes.Default.Command = 'curl "http://127.0.0.1:{0}/generation2/cache/10/{1}" -H "x-debug: x-cache,x-cache-key,via,x-cache-generation" --verbose -o /dev/null'.format( ts.Variables.port, objectid) tr.Processes.Default.ReturnCode = 0 tr.Processes.Default.Streams.All = "gold/miss_gen2.gold" # Second touch is a HIT for default. tr = Test.AddTestRun() -tr.Processes.Default.Command = 'curl "http://127.0.0.1:{0}/default/cache/10/{1}" -H "x-debug: x-cache,x-cache-key,via,x-cache-generation" --verbose'.format( +tr.Processes.Default.Command = 'curl "http://127.0.0.1:{0}/default/cache/10/{1}" -H "x-debug: x-cache,x-cache-key,via,x-cache-generation" --verbose -o /dev/null'.format( ts.Variables.port, objectid) tr.Processes.Default.ReturnCode = 0 tr.Processes.Default.Streams.All = "gold/hit_default-1.gold" # Second touch is a HIT for generation1. tr = Test.AddTestRun() -tr.Processes.Default.Command = 'curl "http://127.0.0.1:{0}/generation1/cache/10/{1}" -H "x-debug: x-cache,x-cache-key,via,x-cache-generation" --verbose'.format( +tr.Processes.Default.Command = 'curl "http://127.0.0.1:{0}/generation1/cache/10/{1}" -H "x-debug: x-cache,x-cache-key,via,x-cache-generation" --verbose -o /dev/null'.format( ts.Variables.port, objectid) tr.Processes.Default.ReturnCode = 0 tr.Processes.Default.Streams.All = "gold/hit_gen1.gold" # Second touch is a HIT for generation2. tr = Test.AddTestRun() -tr.Processes.Default.Command = 'curl "http://127.0.0.1:{0}/generation2/cache/10/{1}" -H "x-debug: x-cache,x-cache-key,via,x-cache-generation" --verbose'.format( +tr.Processes.Default.Command = 'curl "http://127.0.0.1:{0}/generation2/cache/10/{1}" -H "x-debug: x-cache,x-cache-key,via,x-cache-generation" --verbose -o /dev/null'.format( ts.Variables.port, objectid) tr.Processes.Default.ReturnCode = 0 tr.Processes.Default.Streams.All = "gold/hit_gen2.gold" diff --git a/tests/gold_tests/chunked_encoding/Makefile.inc b/tests/gold_tests/chunked_encoding/Makefile.inc new file mode 100644 index 00000000000..b316b4b41fd --- /dev/null +++ b/tests/gold_tests/chunked_encoding/Makefile.inc @@ -0,0 +1,19 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +noinst_PROGRAMS += gold_tests/chunked_encoding/smuggle-client +gold_tests_chunked_encoding_smuggle_client_SOURCES = gold_tests/chunked_encoding/smuggle-client.c +gold_tests_chunked_encoding_smuggle_client_LDADD = -lssl diff --git a/tests/gold_tests/chunked_encoding/case1.sh b/tests/gold_tests/chunked_encoding/case1.sh new file mode 100644 index 00000000000..42622839770 --- /dev/null +++ b/tests/gold_tests/chunked_encoding/case1.sh @@ -0,0 +1,19 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +nc -l ${2} -o outserver1 -c "sh ./delay-server.sh" & +sleep 1 +nghttp -vv https://127.0.0.1:${1}/delay-chunked-response diff --git a/tests/gold_tests/chunked_encoding/case2.sh b/tests/gold_tests/chunked_encoding/case2.sh new file mode 100644 index 00000000000..be4d76f57c7 --- /dev/null +++ b/tests/gold_tests/chunked_encoding/case2.sh @@ -0,0 +1,19 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +nc -l ${2} -o outserver2 -c "sh ./server2.sh" & +sleep 1 +curl --http2 -k https://127.0.0.1:${1}/post --verbose -H "Transfer-encoding: chunked" -d "Knock knock" diff --git a/tests/gold_tests/chunked_encoding/case3.sh b/tests/gold_tests/chunked_encoding/case3.sh new file mode 100644 index 00000000000..4af2d92b9d3 --- /dev/null +++ b/tests/gold_tests/chunked_encoding/case3.sh @@ -0,0 +1,19 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +nc -l ${2} -o outserver3 -c "sh ./server3.sh" & +sleep 1 +curl --http2 -k https://127.0.0.1:${1}/post-chunked --verbose -H "Transfer-encoding: chunked" -d "Knock knock" diff --git a/tests/gold_tests/chunked_encoding/case4.sh b/tests/gold_tests/chunked_encoding/case4.sh new file mode 100644 index 00000000000..aa513780977 --- /dev/null +++ b/tests/gold_tests/chunked_encoding/case4.sh @@ -0,0 +1,19 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +nc -l ${2} -o outserver4 -c "sh ./server4.sh" & +sleep 1 +./smuggle-client 127.0.0.1 ${1} diff --git a/tests/gold_tests/chunked_encoding/chunked_encoding.test.py b/tests/gold_tests/chunked_encoding/chunked_encoding.test.py index 8f02a1ee637..d4da19d09c6 100644 --- a/tests/gold_tests/chunked_encoding/chunked_encoding.test.py +++ b/tests/gold_tests/chunked_encoding/chunked_encoding.test.py @@ -16,16 +16,17 @@ # See the License for the specific language governing permissions and # limitations under the License. -import os Test.Summary = ''' -Test a basic remap of a http connection +Test chunked encoding processing ''' -# need Curl with HTTP/2 + Test.SkipUnless( Condition.HasCurlFeature('http2') ) Test.ContinueOnFail = True +Test.GetTcpPort("upstream_port") + # Define default ATS ts = Test.MakeATSProcess("ts", select_ports=True, enable_tls=True) server = Test.MakeOriginServer("server") @@ -41,16 +42,18 @@ "timestamp": "1469733493.993", "body": ""} -request_header2 = {"headers": "POST / HTTP/1.1\r\nHost: www.anotherexample.com\r\nContent-Type: application/x-www-form-urlencoded\r\nContent-Length: 11\r\n\r\n", - "timestamp": "1415926535.898", - "body": "knock knock"} +request_header2 = { + "headers": "POST / HTTP/1.1\r\nHost: www.anotherexample.com\r\nContent-Type: application/x-www-form-urlencoded\r\nContent-Length: 11\r\n\r\n", + "timestamp": "1415926535.898", + "body": "knock knock"} response_header2 = {"headers": "HTTP/1.1 200 OK\r\nServer: uServer\r\nConnection: close\r\nTransfer-Encoding: chunked\r\n\r\n", "timestamp": "1415926535.898", "body": ""} -request_header3 = {"headers": "POST / HTTP/1.1\r\nHost: www.yetanotherexample.com\r\nContent-Type: application/x-www-form-urlencoded\r\nContent-Length: 11\r\n\r\n", - "timestamp": "1415926535.898", - "body": "knock knock"} +request_header3 = { + "headers": "POST / HTTP/1.1\r\nHost: www.yetanotherexample.com\r\nContent-Type: application/x-www-form-urlencoded\r\nContent-Length: 11\r\n\r\n", + "timestamp": "1415926535.898", + "body": "knock knock"} response_header3 = {"headers": "HTTP/1.1 200 OK\r\nServer: uServer\r\nConnection: close\r\nTransfer-Encoding: chunked\r\n\r\n", "timestamp": "1415926535.898", "body": ""} @@ -65,10 +68,10 @@ ts.Disk.records_config.update({ 'proxy.config.diags.debug.enabled': 1, - 'proxy.config.diags.debug.tags': 'lm|ssl', + 'proxy.config.diags.debug.tags': 'http', 'proxy.config.ssl.server.cert.path': '{0}'.format(ts.Variables.SSLDir), 'proxy.config.ssl.server.private_key.path': '{0}'.format(ts.Variables.SSLDir), - 'proxy.config.ssl.client.verify.server': 0, + 'proxy.config.ssl.client.verify.server': 0, 'proxy.config.ssl.server.cipher_suite': 'ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA384:AES128-GCM-SHA256:AES256-GCM-SHA384:ECDHE-RSA-RC4-SHA:ECDHE-RSA-AES128-SHA:ECDHE-RSA-AES256-SHA:RC4-SHA:RC4-MD5:AES128-SHA:AES256-SHA:DES-CBC3-SHA!SRP:!DSS:!PSK:!aNULL:!eNULL:!SSLv2', }) @@ -81,14 +84,21 @@ ts.Disk.remap_config.AddLine( 'map https://www.anotherexample.com https://127.0.0.1:{0}'.format(server2.Variables.SSL_Port, ts.Variables.ssl_port) ) - +ts.Disk.remap_config.AddLine( + 'map / http://127.0.0.1:{0}'.format(Test.Variables.upstream_port) +) ts.Disk.ssl_multicert_config.AddLine( 'dest_ip=* ssl_cert_name=server.pem ssl_key_name=server.key' ) +# smuggle-client is built via `make`. Here we copy the built binary down to the +# test directory so that the test runs in this file can use it. +Test.Setup.Copy('smuggle-client') + # HTTP1.1 GET: www.example.com tr = Test.AddTestRun() +tr.TimeOut = 5 tr.Processes.Default.Command = 'curl --http1.1 --proxy 127.0.0.1:{0} http://www.example.com --verbose'.format( ts.Variables.port) tr.Processes.Default.ReturnCode = 0 @@ -104,6 +114,7 @@ # HTTP2 POST: www.example.com Host, chunked body tr = Test.AddTestRun() +tr.TimeOut = 5 tr.Processes.Default.Command = 'curl --http2 -k https://127.0.0.1:{0} --verbose -H "Host: www.anotherexample.com" -d "Knock knock"'.format( ts.Variables.ssl_port) tr.Processes.Default.ReturnCode = 0 @@ -111,6 +122,7 @@ # HTTP1.1 POST: www.yetanotherexample.com Host, explicit size tr = Test.AddTestRun() +tr.TimeOut = 5 tr.Processes.Default.Command = 'curl http://127.0.0.1:{0} -H "Host: www.yetanotherexample.com" --verbose -d "knock knock"'.format( ts.Variables.port) tr.Processes.Default.ReturnCode = 0 @@ -119,8 +131,24 @@ # HTTP1.1 POST: www.example.com Host, chunked body tr = Test.AddTestRun() +tr.TimeOut = 5 tr.Processes.Default.Command = 'curl http://127.0.0.1:{0} -H "Host: www.yetanotherexample.com" --verbose -H "Transfer-Encoding: chunked" -d "Knock knock"'.format( ts.Variables.port) tr.Processes.Default.ReturnCode = 0 tr.Processes.Default.Streams.stderr = "gold/chunked_POST_200.gold" tr.StillRunningAfter = server + +server4_out = Test.Disk.File("outserver4") +server4_out.Content = Testers.ExcludesExpression("sneaky", "Extra body bytes should not be delivered") +server4_out.Content += Testers.ContainsExpression("Transfer-Encoding: chunked", "Request should be chunked encoded") + +# HTTP/1.1 Try to smuggle another request to the origin +tr = Test.AddTestRun() +tr.TimeOut = 5 +tr.Setup.Copy('server4.sh') +tr.Setup.Copy('case4.sh') +tr.Processes.Default.Command = 'sh ./case4.sh {0} {1}'.format(ts.Variables.ssl_port, Test.Variables.upstream_port) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Streams.All = Testers.ExcludesExpression("content-length:", "Response should not include content length") +# Transfer encoding to origin, but no content-length +# No extra bytes in body seen by origin diff --git a/tests/gold_tests/chunked_encoding/chunked_encoding_h2.test.py b/tests/gold_tests/chunked_encoding/chunked_encoding_h2.test.py new file mode 100644 index 00000000000..d5022c03a42 --- /dev/null +++ b/tests/gold_tests/chunked_encoding/chunked_encoding_h2.test.py @@ -0,0 +1,106 @@ +''' +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +Test.Summary = ''' +Test interaction of H2 and chunked encoding +''' + +Test.SkipUnless( + Condition.HasProgram("nghttp", "Nghttp need to be installed on system for this test to work"), + Condition.HasCurlFeature('http2') +) +Test.ContinueOnFail = True + +Test.GetTcpPort("upstream_port") + +# Define default ATS +ts = Test.MakeATSProcess("ts", select_ports=True, enable_tls=True) + +# add ssl materials like key, certificates for the server +ts.addSSLfile("ssl/server.pem") +ts.addSSLfile("ssl/server.key") + +ts.Disk.records_config.update({ + 'proxy.config.http2.enabled': 1, # this option is for VZM-internal only + 'proxy.config.diags.debug.enabled': 0, + 'proxy.config.diags.debug.tags': 'http', + 'proxy.config.ssl.server.cert.path': '{0}'.format(ts.Variables.SSLDir), + 'proxy.config.ssl.server.private_key.path': '{0}'.format(ts.Variables.SSLDir), + 'proxy.config.ssl.server.cipher_suite': 'ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA384:AES128-GCM-SHA256:AES256-GCM-SHA384:ECDHE-RSA-RC4-SHA:ECDHE-RSA-AES128-SHA:ECDHE-RSA-AES256-SHA:RC4-SHA:RC4-MD5:AES128-SHA:AES256-SHA:DES-CBC3-SHA!SRP:!DSS:!PSK:!aNULL:!eNULL:!SSLv2', +}) + +ts.Disk.remap_config.AddLine( + 'map /delay-chunked-response http://127.0.0.1:{0}'.format(Test.Variables.upstream_port) +) +ts.Disk.remap_config.AddLine( + 'map / http://127.0.0.1:{0}'.format(Test.Variables.upstream_port) +) + +ts.Disk.ssl_multicert_config.AddLine( + 'dest_ip=* ssl_cert_name=server.pem ssl_key_name=server.key' +) + +# Using netcat as a cheapy origin server in case 1 so we can insert a delay in sending back the response. +# Replaced microserver for cases 2 and 3 as well because I was getting python exceptions when running +# microserver if chunked encoding headers were specified for the request headers + +# H2 GET request +# chunked response without content-length +# delay before final chunk size +server1_out = Test.Disk.File("outserver1") +tr = Test.AddTestRun() +tr.Setup.Copy('delay-server.sh') +tr.Setup.Copy('case1.sh') +tr.Processes.Default.Command = 'sh ./case1.sh {0} {1}'.format(ts.Variables.ssl_port, Test.Variables.upstream_port) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.StartBefore(Test.Processes.ts) +tr.Processes.Default.Streams.All = Testers.ExcludesExpression("RST_STREAM", "Delayed chunk close should not cause reset") +tr.Processes.Default.Streams.All += Testers.ExcludesExpression("content-length", "Should return chunked") +tr.Processes.Default.Streams.All += Testers.ContainsExpression(":status: 200", "Should get successful response") +tr.StillRunningAfter = ts +# No resets in the output +# No content lengths in the header + +# HTTP2 POST: www.example.com Host, chunked body +server2_out = Test.Disk.File("outserver2") +tr = Test.AddTestRun() +tr.Setup.Copy('server2.sh') +tr.Setup.Copy('case2.sh') +tr.Processes.Default.Command = 'sh ./case2.sh {0} {1}'.format(ts.Variables.ssl_port, Test.Variables.upstream_port) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Streams.All = Testers.ContainsExpression("HTTP/2 200", "Request should succeed") +tr.Processes.Default.Streams.All += Testers.ContainsExpression("content-length:", "Response should include content length") +server2_out = Testers.ContainsExpression("Transfer-Encoding: chunked", "Request should be chunked encoded") +# No content-length in header +# Transfer encoding to origin, but no content-length +# Content length on the way back + +# HTTP2 POST: chunked post body and chunked response +server3_out = Test.Disk.File("outserver3") +tr = Test.AddTestRun() +tr.Setup.Copy('server3.sh') +tr.Setup.Copy('case3.sh') +tr.Processes.Default.Command = 'sh ./case3.sh {0} {1}'.format(ts.Variables.ssl_port, Test.Variables.upstream_port) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Streams.All = Testers.ContainsExpression("HTTP/2 200", "Request should succeed") +tr.Processes.Default.Streams.All += Testers.ExcludesExpression("content-length:", "Response should not include content length") +server3_out = Testers.ContainsExpression("Transfer-Encoding: chunked", "Request should be chunked encoded") +# No content length in header +# Transfer encoding to origin, but no content-length +# No content length in the response diff --git a/tests/gold_tests/chunked_encoding/delay-server.sh b/tests/gold_tests/chunked_encoding/delay-server.sh new file mode 100644 index 00000000000..670c9f1e727 --- /dev/null +++ b/tests/gold_tests/chunked_encoding/delay-server.sh @@ -0,0 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +printf "HTTP/1.1 200\r\nTransfer-encoding: chunked\r\n\r\n" +printf "F\r\n123456789012345\r\n" +sleep 1 +printf "0\r\n\r\n" diff --git a/tests/gold_tests/chunked_encoding/server2.sh b/tests/gold_tests/chunked_encoding/server2.sh new file mode 100644 index 00000000000..4e48d75dd7e --- /dev/null +++ b/tests/gold_tests/chunked_encoding/server2.sh @@ -0,0 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +printf "HTTP/1.1 200\r\nContent-length: 15\r\n\r\n" +printf "123456789012345" diff --git a/tests/gold_tests/chunked_encoding/server3.sh b/tests/gold_tests/chunked_encoding/server3.sh new file mode 100644 index 00000000000..bf73609cc02 --- /dev/null +++ b/tests/gold_tests/chunked_encoding/server3.sh @@ -0,0 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +printf "HTTP/1.1 200\r\nTransfer-encoding: chunked\r\n\r\n" +printf "F\r\n123456789012345\r\n0\r\n\r\n" diff --git a/tests/gold_tests/chunked_encoding/server4.sh b/tests/gold_tests/chunked_encoding/server4.sh new file mode 100644 index 00000000000..bf73609cc02 --- /dev/null +++ b/tests/gold_tests/chunked_encoding/server4.sh @@ -0,0 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +printf "HTTP/1.1 200\r\nTransfer-encoding: chunked\r\n\r\n" +printf "F\r\n123456789012345\r\n0\r\n\r\n" diff --git a/tests/gold_tests/chunked_encoding/smuggle-client.c b/tests/gold_tests/chunked_encoding/smuggle-client.c new file mode 100644 index 00000000000..928db415b10 --- /dev/null +++ b/tests/gold_tests/chunked_encoding/smuggle-client.c @@ -0,0 +1,130 @@ +/** @file + + smuggle_client.c + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +const char *req_and_post_buf = + "GET / HTTP/1.1\r\nConnection: close\r\nHost: foo.com\r\nTransfer-Encoding: chunked\r\nContent-Length: 301\r\n\r\n0\r\n\r\nPOST " + "http://sneaky.com/ HTTP/1.1\r\nContent-Length: 10\r\nConnection: close\r\nX-Foo: Z\r\n\r\n1234567890"; + +/** + * Connect to a server. + * Handshake + * Exit immediatesly + */ +int +main(int argc, char *argv[]) +{ + struct addrinfo hints; + struct addrinfo *result, *rp; + int sfd, s; + + if (argc < 3) { + fprintf(stderr, "Usage: %s \n", argv[0]); + exit(1); + } + + const char *target = argv[1]; + const char *target_port = argv[2]; + printf("using address: %s and port: %s\n", target, target_port); + +#if OPENSSL_VERSION_NUMBER < 0x10100000L + SSL_library_init(); +#else + OPENSSL_init_ssl(0, NULL); +#endif + + /* Obtain address(es) matching host/port */ + memset(&hints, 0, sizeof(struct addrinfo)); + hints.ai_family = AF_UNSPEC; /* Allow IPv4 or IPv6 */ + hints.ai_socktype = SOCK_STREAM; /* Datagram socket */ + hints.ai_flags = 0; + hints.ai_protocol = 0; /* Any protocol */ + + s = getaddrinfo(target, target_port, &hints, &result); + if (s != 0) { + fprintf(stderr, "getaddrinfo: %s\n", gai_strerror(s)); + exit(EXIT_FAILURE); + } + + /* getaddrinfo() returns a list of address structures. + * Try each address until we successfully connect(2). + * socket(2) (or connect(2)) fails, we (close the socket + and) try the next address. */ + + for (rp = result; rp != NULL; rp = rp->ai_next) { + sfd = socket(rp->ai_family, rp->ai_socktype, rp->ai_protocol); + if (sfd == -1) + continue; + if (connect(sfd, rp->ai_addr, rp->ai_addrlen) != -1) + break; /* Success */ + + close(sfd); + } + + if (rp == NULL) { /* No address succeeded */ + fprintf(stderr, "Could not connect\n"); + exit(EXIT_FAILURE); + } + + SSL_CTX *client_ctx = SSL_CTX_new(SSLv23_client_method()); + assert(client_ctx != NULL); + SSL *ssl = SSL_new(client_ctx); + assert(ssl != NULL); + + SSL_set_fd(ssl, sfd); + int ret = SSL_connect(ssl); + assert(ret == 1); + + printf("Send request\n"); + if ((ret = SSL_write(ssl, req_and_post_buf, strlen(req_and_post_buf))) <= 0) { + int error = SSL_get_error(ssl, ret); + printf("SSL_write failed %d", error); + exit(1); + } + + int read_bytes; + do { + char input_buf[1024]; + read_bytes = SSL_read(ssl, input_buf, sizeof(input_buf) - 1); + if (read_bytes > 0) { + input_buf[read_bytes] = '\0'; + printf("Received %d bytes %s\n", read_bytes, input_buf); + } + } while (read_bytes > 0); + + close(sfd); + + exit(0); +} diff --git a/tests/gold_tests/command_argument/verify_global_plugin.test.py b/tests/gold_tests/command_argument/verify_global_plugin.test.py new file mode 100644 index 00000000000..55668912713 --- /dev/null +++ b/tests/gold_tests/command_argument/verify_global_plugin.test.py @@ -0,0 +1,173 @@ +''' +Test the verify_global_plugin TrafficServer command. +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +Test.Summary = ''' +Test that the TrafficServer verify_global_plugin command works as expected. +''' + +process_counter = 0 + + +def create_ts_process(): + """ + Create a unique ATS process with each call to this function. + """ + global process_counter + process_counter += 1 + + ts = Test.MakeATSProcess("ts{counter}".format(counter=process_counter)) + + # Ideally we would set the test run's Processes.Default to ts, but deep + # copy of processes is not currently implemented in autest. Therefore we + # replace the command which ts runs with a dummy command, and pull in + # piecemeal the values from ts that we want into the test run. + ts.Command = "sleep 100" + # sleep will return -2 when autest kills it. We set the expectation for the + # -2 return code here so the test doesn't fail because of this. + ts.ReturnCode = -2 + # Clear the ready criteria because sleep is ready as soon as it is running. + ts.Ready = None + return ts + + +""" +TEST: verify_global_plugin should complain if an argument is not passed to it. +""" +tr = Test.AddTestRun("Verify the requirement of an argument") +ts = create_ts_process() +tr.Processes.Default.Env = ts.Env +tr.Processes.Default.Command = "traffic_server -C 'verify_global_plugin'" +tr.Processes.Default.ReturnCode = 1 +tr.Processes.Default.StartBefore(ts) +tr.Processes.Default.Streams.stderr = Testers.ContainsExpression( + "ERROR: verifying a plugin requires a plugin SO file path argument", + "Should warn about the need for an SO file argument") + + +""" +TEST: verify_global_plugin should complain if the argument doesn't reference a shared +object file. +""" +tr = Test.AddTestRun("Verify the requirement of a file") +ts = create_ts_process() +tr.Processes.Default.Env = ts.Env +tr.Processes.Default.Command = \ + "traffic_server -C 'verify_global_plugin {filename}'".format( + filename="/this/file/does/not/exist.so") +tr.Processes.Default.ReturnCode = 1 +tr.Processes.Default.StartBefore(ts) +tr.Processes.Default.Streams.stderr = Testers.ContainsExpression( + "ERROR: .*No such file or directory", + "Should warn about the non-existent SO file argument") + + +""" +TEST: verify_global_plugin should complain if the shared object file doesn't +have the expected Plugin symbols. +""" +tr = Test.AddTestRun("Verify the requirement of our Plugin API.") +ts = create_ts_process() +Test.PrepareTestPlugin( + os.path.join(Test.Variables.AtsTestPluginsDir, + 'missing_ts_plugin_init.so'), + ts) +tr.Processes.Default.Env = ts.Env +tr.Processes.Default.Command = \ + "traffic_server -C 'verify_global_plugin {filename}'".format( + filename="${PROXY_CONFIG_PLUGIN_PLUGIN_DIR}/missing_ts_plugin_init.so") +tr.Processes.Default.ReturnCode = 1 +tr.Processes.Default.StartBefore(ts) +tr.Processes.Default.Streams.stderr = Testers.ContainsExpression( + "ERROR: .*unable to find TSPluginInit function in", + "Should warn about the need for the TSPluginInit symbol") +ts.Disk.diags_log.Content = Testers.ContainsExpression( + "ERROR", + "ERROR: .*unable to find TSPluginInit function in") + + +""" +TEST: Verify that passing a remap plugin produces a warning because +it doesn't have the global plugin symbols. +""" +tr = Test.AddTestRun("Verify a properly formed plugin works as expected.") +ts = create_ts_process() +Test.PrepareTestPlugin( + os.path.join(Test.Variables.AtsTestPluginsDir, + 'conf_remap_stripped.so'), + ts) +tr.Processes.Default.Env = ts.Env +tr.Processes.Default.Command = \ + "traffic_server -C 'verify_global_plugin {filename}'".format( + filename="${PROXY_CONFIG_PLUGIN_PLUGIN_DIR}/conf_remap_stripped.so") +tr.Processes.Default.ReturnCode = 1 +tr.Processes.Default.StartBefore(ts) +tr.Processes.Default.Streams.stderr = Testers.ContainsExpression( + "ERROR: .*unable to find TSPluginInit function in", + "Should warn about the need for the TSPluginInit symbol") +ts.Disk.diags_log.Content = Testers.ContainsExpression( + "ERROR", + "ERROR: .*unable to find TSPluginInit function in") + + +""" +TEST: The happy case: a global plugin shared object file is passed as an +argument that has the definition for the expected Plugin symbols. +""" +tr = Test.AddTestRun("Verify a properly formed plugin works as expected.") +ts = create_ts_process() +Test.PrepareTestPlugin( + os.path.join(Test.Variables.AtsTestPluginsDir, + 'ssl_hook_test.so'), + ts) +tr.Processes.Default.Env = ts.Env +tr.Processes.Default.Command = \ + "traffic_server -C 'verify_global_plugin {filename}'".format( + filename="${PROXY_CONFIG_PLUGIN_PLUGIN_DIR}/ssl_hook_test.so") +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.StartBefore(ts) +tr.Processes.Default.Streams.stderr = Testers.ContainsExpression( + "NOTE: verifying plugin '.*' Success", + "Verification should succeed") + + +""" +TEST: This is a regression test for a shared object file that doesn't have all +of the required symbols defined because of a malformed interaction between C +and C++ files. +""" +tr = Test.AddTestRun("Regression test for an undefined, mangled C++ symbol.") +ts = create_ts_process() +plugin_filename = 'missing_mangled_definition.so' +built_plugin_path = os.path.join(Test.Variables.AtsTestPluginsDir, plugin_filename) +ats_plugin_dir = ts.Env['PROXY_CONFIG_PLUGIN_PLUGIN_DIR'] +ts.Setup.Copy(built_plugin_path, ats_plugin_dir) +tr.Processes.Default.Env = ts.Env +tr.Processes.Default.Command = \ + "traffic_server -C 'verify_global_plugin {plugin_path}'".format( + plugin_path=os.path.join(ats_plugin_dir, plugin_filename)) +tr.Processes.Default.ReturnCode = 1 +tr.Processes.Default.StartBefore(ts) +tr.Processes.Default.Streams.stderr = Testers.ContainsExpression( + "ERROR: .*: undefined symbol: .*foo.*", + "Should warn about the need for the TSPluginInit symbol") +ts.Disk.diags_log.Content = Testers.ContainsExpression( + "ERROR", + "ERROR: .*: undefined symbol: .*foo.*") diff --git a/tests/gold_tests/command_argument/verify_remap_plugin.test.py b/tests/gold_tests/command_argument/verify_remap_plugin.test.py new file mode 100644 index 00000000000..dcd062d6c64 --- /dev/null +++ b/tests/gold_tests/command_argument/verify_remap_plugin.test.py @@ -0,0 +1,146 @@ +''' +Test the verify_remap_plugin TrafficServer command. +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +Test.Summary = ''' +Test that the TrafficServer verify_remap_plugin command works as expected. +''' + +process_counter = 0 + + +def create_ts_process(): + """ + Create a unique ATS process with each call to this function. + """ + global process_counter + process_counter += 1 + + ts = Test.MakeATSProcess("ts{counter}".format(counter=process_counter)) + + # Ideally we would set the test run's Processes.Default to ts, but deep + # copy of processes is not currently implemented in autest. Therefore we + # replace the command which ts runs with a dummy command, and pull in + # piecemeal the values from ts that we want into the test run. + ts.Command = "sleep 100" + # sleep will return -2 when autest kills it. We set the expectation for the + # -2 return code here so the test doesn't fail because of this. + ts.ReturnCode = -2 + # Clear the ready criteria because sleep is ready as soon as it is running. + ts.Ready = None + return ts + + +""" +TEST: verify_remap_plugin should complain if an argument is not passed to it. +""" +tr = Test.AddTestRun("Verify the requirement of an argument") +ts = create_ts_process() +tr.Processes.Default.Env = ts.Env +tr.Processes.Default.Command = "traffic_server -C 'verify_remap_plugin'" +tr.Processes.Default.ReturnCode = 1 +tr.Processes.Default.StartBefore(ts) +tr.Processes.Default.Streams.stderr = Testers.ContainsExpression( + "ERROR: verifying a plugin requires a plugin SO file path argument", + "Should warn about the need for an SO file argument") + + +""" +TEST: verify_remap_plugin should complain if the argument doesn't reference a shared +object file. +""" +tr = Test.AddTestRun("Verify the requirement of a file") +ts = create_ts_process() +tr.Processes.Default.Env = ts.Env +tr.Processes.Default.Command = \ + "traffic_server -C 'verify_remap_plugin {filename}'".format( + filename="/this/file/does/not/exist.so") +tr.Processes.Default.ReturnCode = 1 +tr.Processes.Default.StartBefore(ts) +tr.Processes.Default.Streams.stderr = Testers.ContainsExpression( + "ERROR: .*No such file or directory", + "Should warn about the non-existent SO file argument") + + +""" +TEST: verify_remap_plugin should complain if the shared object file doesn't +have the expected Plugin symbols. +""" +tr = Test.AddTestRun("Verify the requirement of our Plugin API.") +ts = create_ts_process() +Test.PrepareTestPlugin( + os.path.join(Test.Variables.AtsTestPluginsDir, + 'missing_ts_plugin_init.so'), + ts) +tr.Processes.Default.Env = ts.Env +tr.Processes.Default.Command = \ + "traffic_server -C 'verify_remap_plugin {filename}'".format( + filename="${PROXY_CONFIG_PLUGIN_PLUGIN_DIR}/missing_ts_plugin_init.so") +tr.Processes.Default.ReturnCode = 1 +tr.Processes.Default.StartBefore(ts) +tr.Processes.Default.Streams.stderr = Testers.ContainsExpression( + "ERROR: .*missing required function TSRemapInit", + "Should warn about the need for the TSRemapInit symbol") +ts.Disk.diags_log.Content = Testers.ContainsExpression("ERROR", + "ERROR: .*missing required function TSRemapInit") + + +""" +TEST: verify_remap_plugin should complain if the plugin has the global +plugin symbols but not the remap ones. +""" +tr = Test.AddTestRun("Verify a global plugin argument produces warning.") +ts = create_ts_process() +Test.PrepareTestPlugin( + os.path.join(Test.Variables.AtsTestPluginsDir, + 'ssl_hook_test.so'), + ts) +tr.Processes.Default.Env = ts.Env +tr.Processes.Default.Command = \ + "traffic_server -C 'verify_remap_plugin {filename}'".format( + filename="${PROXY_CONFIG_PLUGIN_PLUGIN_DIR}/ssl_hook_test.so") +tr.Processes.Default.ReturnCode = 1 +tr.Processes.Default.StartBefore(ts) +tr.Processes.Default.Streams.stderr = Testers.ContainsExpression( + "ERROR: .*missing required function TSRemapInit", + "Should warn about the need for the TSRemapInit symbol") +ts.Disk.diags_log.Content = Testers.ContainsExpression("ERROR", + "ERROR: .*missing required function TSRemapInit") + + +""" +TEST: The happy case: a remap plugin shared object file is passed as an +argument that has the definition for the expected Plugin symbols. +""" +tr = Test.AddTestRun("Verify a properly formed plugin works as expected.") +ts = create_ts_process() +Test.PrepareTestPlugin( + os.path.join(Test.Variables.AtsTestPluginsDir, + 'conf_remap_stripped.so'), + ts) +tr.Processes.Default.Env = ts.Env +tr.Processes.Default.Command = \ + "traffic_server -C 'verify_remap_plugin {filename}'".format( + filename="${PROXY_CONFIG_PLUGIN_PLUGIN_DIR}/conf_remap_stripped.so") +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.StartBefore(ts) +tr.Processes.Default.Streams.stderr = Testers.ContainsExpression( + "NOTE: verifying plugin '.*' Success", + "Verification should succeed") diff --git a/tests/gold_tests/cont_schedule/schedule.test.py b/tests/gold_tests/cont_schedule/schedule.test.py index c82db1741e5..3a2b38c539c 100644 --- a/tests/gold_tests/cont_schedule/schedule.test.py +++ b/tests/gold_tests/cont_schedule/schedule.test.py @@ -38,7 +38,7 @@ }) # Load plugin -Test.PreparePlugin(os.path.join(Test.Variables.AtsTestToolsDir, 'plugins', 'cont_schedule.cc'), ts) +Test.PrepareTestPlugin(os.path.join(Test.Variables.AtsTestPluginsDir, 'cont_schedule.so'), ts) # www.example.com Host tr = Test.AddTestRun() diff --git a/tests/gold_tests/cont_schedule/schedule_on_pool.test.py b/tests/gold_tests/cont_schedule/schedule_on_pool.test.py index 23a737391dd..176478c6e77 100644 --- a/tests/gold_tests/cont_schedule/schedule_on_pool.test.py +++ b/tests/gold_tests/cont_schedule/schedule_on_pool.test.py @@ -38,7 +38,7 @@ }) # Load plugin -Test.PreparePlugin(os.path.join(Test.Variables.AtsTestToolsDir, 'plugins', 'cont_schedule.cc'), ts, 'pool') +Test.PrepareTestPlugin(os.path.join(Test.Variables.AtsTestPluginsDir, 'cont_schedule.so'), ts, 'pool') # www.example.com Host tr = Test.AddTestRun() diff --git a/tests/gold_tests/cont_schedule/schedule_on_thread.test.py b/tests/gold_tests/cont_schedule/schedule_on_thread.test.py index 59e5a81a6fb..f54dbe2c78d 100644 --- a/tests/gold_tests/cont_schedule/schedule_on_thread.test.py +++ b/tests/gold_tests/cont_schedule/schedule_on_thread.test.py @@ -38,7 +38,7 @@ }) # Load plugin -Test.PreparePlugin(os.path.join(Test.Variables.AtsTestToolsDir, 'plugins', 'cont_schedule.cc'), ts, 'thread') +Test.PrepareTestPlugin(os.path.join(Test.Variables.AtsTestPluginsDir, 'cont_schedule.so'), ts, 'thread') # www.example.com Host tr = Test.AddTestRun() diff --git a/tests/gold_tests/cont_schedule/thread_affinity.test.py b/tests/gold_tests/cont_schedule/thread_affinity.test.py index 619a3d871fe..3562eb46b03 100644 --- a/tests/gold_tests/cont_schedule/thread_affinity.test.py +++ b/tests/gold_tests/cont_schedule/thread_affinity.test.py @@ -38,7 +38,7 @@ }) # Load plugin -Test.PreparePlugin(os.path.join(Test.Variables.AtsTestToolsDir, 'plugins', 'cont_schedule.cc'), ts, 'affinity') +Test.PrepareTestPlugin(os.path.join(Test.Variables.AtsTestPluginsDir, 'cont_schedule.so'), ts, 'affinity') # www.example.com Host tr = Test.AddTestRun() diff --git a/tests/gold_tests/continuations/double.test.py b/tests/gold_tests/continuations/double.test.py index 400ac9fe060..d81582d72a5 100644 --- a/tests/gold_tests/continuations/double.test.py +++ b/tests/gold_tests/continuations/double.test.py @@ -49,14 +49,14 @@ 'proxy.config.ssl.server.private_key.path': '{0}'.format(ts.Variables.SSLDir), 'proxy.config.http.cache.http': 0, # disable cache to simply the test. 'proxy.config.cache.enable_read_while_writer': 0, - 'proxy.config.ssl.client.verify.server': 0, + 'proxy.config.ssl.client.verify.server': 0, 'proxy.config.ssl.server.cipher_suite': 'ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA384:AES128-GCM-SHA256:AES256-GCM-SHA384:ECDHE-RSA-RC4-SHA:ECDHE-RSA-AES128-SHA:ECDHE-RSA-AES256-SHA:RC4-SHA:RC4-MD5:AES128-SHA:AES256-SHA:DES-CBC3-SHA!SRP:!DSS:!PSK:!aNULL:!eNULL:!SSLv2', 'proxy.config.http2.max_concurrent_streams_in': 65535 }) # add plugin to assist with test metrics -Test.PreparePlugin(os.path.join(Test.Variables.AtsTestToolsDir, - 'plugins', 'continuations_verify.cc'), ts) +Test.PrepareTestPlugin(os.path.join(Test.Variables.AtsTestPluginsDir, + 'continuations_verify.so'), ts) comparator_command = ''' if test "`traffic_ctl metric get continuations_verify.{0}.close.1 | cut -d ' ' -f 2`" -eq "`traffic_ctl metric get continuations_verify.{0}.close.2 | cut -d ' ' -f 2`" ; then\ @@ -75,9 +75,9 @@ # Create a bunch of curl commands to be executed in parallel. Default.Process is set in SpawnCommands. # On Fedora 28/29, it seems that curl will occaisionally timeout after a couple seconds and return exitcode 2 # Examinig the packet capture shows that Traffic Server dutifully sends the response -ps = tr.SpawnCommands(cmdstr=cmd, count=numberOfRequests, retcode=Any(0,2)) +ps = tr.SpawnCommands(cmdstr=cmd, count=numberOfRequests, retcode=Any(0, 2)) tr.Processes.Default.Env = ts.Env -tr.Processes.Default.ReturnCode = Any(0,2) +tr.Processes.Default.ReturnCode = Any(0, 2) # Execution order is: ts/server, ps(curl cmds), Default Process. tr.Processes.Default.StartBefore( @@ -98,15 +98,18 @@ # Parking this as a ready tester on a meaningless process # To stall the test runs that check for the stats until the # stats have propagated and are ready to read. -def make_done_stat_ready(tsenv): - def done_stat_ready(process, hasRunFor, **kw): - retval = subprocess.run("traffic_ctl metric get continuations_verify.test.done > done 2> /dev/null", shell=True, env=tsenv) - if retval.returncode == 0: - retval = subprocess.run("grep 1 done > /dev/null", shell = True, env=tsenv) - return retval.returncode == 0 - - return done_stat_ready - + + +def make_done_stat_ready(tsenv): + def done_stat_ready(process, hasRunFor, **kw): + retval = subprocess.run("traffic_ctl metric get continuations_verify.test.done > done 2> /dev/null", shell=True, env=tsenv) + if retval.returncode == 0: + retval = subprocess.run("grep 1 done > /dev/null", shell=True, env=tsenv) + return retval.returncode == 0 + + return done_stat_ready + + # number of sessions/transactions opened and closed are equal tr = Test.AddTestRun("Check Ssn") server2.StartupTimeout = 60 @@ -135,4 +138,3 @@ def done_stat_ready(process, hasRunFor, **kw): tr.Processes.Default.Streams.stdout = Testers.ContainsExpression("yes", 'should verify contents') tr.StillRunningAfter = ts tr.StillRunningAfter = server2 - diff --git a/tests/gold_tests/continuations/double_h2.test.py b/tests/gold_tests/continuations/double_h2.test.py index 9ab8789d543..ea76492f3dc 100644 --- a/tests/gold_tests/continuations/double_h2.test.py +++ b/tests/gold_tests/continuations/double_h2.test.py @@ -59,14 +59,14 @@ 'proxy.config.ssl.server.private_key.path': '{0}'.format(ts.Variables.SSLDir), 'proxy.config.http.cache.http': 0, # disable cache to simply the test. 'proxy.config.cache.enable_read_while_writer': 0, - 'proxy.config.ssl.client.verify.server': 0, + 'proxy.config.ssl.client.verify.server': 0, 'proxy.config.ssl.server.cipher_suite': 'ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA384:AES128-GCM-SHA256:AES256-GCM-SHA384:ECDHE-RSA-RC4-SHA:ECDHE-RSA-AES128-SHA:ECDHE-RSA-AES256-SHA:RC4-SHA:RC4-MD5:AES128-SHA:AES256-SHA:DES-CBC3-SHA!SRP:!DSS:!PSK:!aNULL:!eNULL:!SSLv2', 'proxy.config.http2.max_concurrent_streams_in': 65535 }) # add plugin to assist with test metrics -Test.PreparePlugin(os.path.join(Test.Variables.AtsTestToolsDir, - 'plugins', 'continuations_verify.cc'), ts) +Test.PrepareTestPlugin(os.path.join(Test.Variables.AtsTestPluginsDir, + 'continuations_verify.so'), ts) comparator_command = ''' if test "`traffic_ctl metric get continuations_verify.{0}.close.1 | cut -d ' ' -f 2`" -eq "`traffic_ctl metric get continuations_verify.{0}.close.2 | cut -d ' ' -f 2`" ; then\ @@ -86,9 +86,9 @@ # Create a bunch of curl commands to be executed in parallel. Default.Process is set in SpawnCommands. # On Fedora 28/29, it seems that curl will occaisionally timeout after a couple seconds and return exitcode 2 # Examinig the packet capture shows that Traffic Server dutifully sends the response -ps = tr.SpawnCommands(cmdstr=cmd, count=numberOfRequests, retcode=Any(0,2)) +ps = tr.SpawnCommands(cmdstr=cmd, count=numberOfRequests, retcode=Any(0, 2)) tr.Processes.Default.Env = ts.Env -tr.Processes.Default.ReturnCode = Any(0,2) +tr.Processes.Default.ReturnCode = Any(0, 2) # Execution order is: ts/server, ps(curl cmds), Default Process. tr.Processes.Default.StartBefore( @@ -109,15 +109,18 @@ # Parking this as a ready tester on a meaningless process # To stall the test runs that check for the stats until the # stats have propagated and are ready to read. -def make_done_stat_ready(tsenv): - def done_stat_ready(process, hasRunFor, **kw): - retval = subprocess.run("traffic_ctl metric get continuations_verify.test.done > done 2> /dev/null", shell=True, env=tsenv) - if retval.returncode == 0: - retval = subprocess.run("grep 1 done > /dev/null", shell = True, env=tsenv) - return retval.returncode == 0 - - return done_stat_ready - + + +def make_done_stat_ready(tsenv): + def done_stat_ready(process, hasRunFor, **kw): + retval = subprocess.run("traffic_ctl metric get continuations_verify.test.done > done 2> /dev/null", shell=True, env=tsenv) + if retval.returncode == 0: + retval = subprocess.run("grep 1 done > /dev/null", shell=True, env=tsenv) + return retval.returncode == 0 + + return done_stat_ready + + # number of sessions/transactions opened and closed are equal tr = Test.AddTestRun("Check Ssn") server2.StartupTimeout = 60 @@ -146,4 +149,3 @@ def done_stat_ready(process, hasRunFor, **kw): tr.Processes.Default.Streams.stdout = Testers.ContainsExpression("yes", 'should verify contents') tr.StillRunningAfter = ts tr.StillRunningAfter = server2 - diff --git a/tests/gold_tests/continuations/openclose.test.py b/tests/gold_tests/continuations/openclose.test.py index 11fd15c24b7..ee66f222aba 100644 --- a/tests/gold_tests/continuations/openclose.test.py +++ b/tests/gold_tests/continuations/openclose.test.py @@ -35,13 +35,13 @@ response_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\nContent-Length:0\r\n\r\n", "timestamp": "1469733493.993", "body": ""} -Test.PreparePlugin(os.path.join(Test.Variables.AtsTestToolsDir, - 'plugins', 'ssntxnorder_verify.cc'), ts) +Test.PrepareTestPlugin(os.path.join(Test.Variables.AtsTestPluginsDir, + 'ssntxnorder_verify.so'), ts) # add response to the server dictionary server.addResponse("sessionfile.log", request_header, response_header) ts.Disk.records_config.update({ - 'proxy.config.diags.debug.enabled': 1, + 'proxy.config.diags.debug.enabled': 0, 'proxy.config.diags.debug.tags': 'ssntxnorder_verify.*', 'proxy.config.http.cache.http': 0, # disable cache to simply the test. 'proxy.config.cache.enable_read_while_writer': 0 @@ -52,28 +52,28 @@ ts.Variables.port, server.Variables.Port) ) -cmd = 'curl -vs http://127.0.0.1:{0}'.format(ts.Variables.port) -numberOfRequests = 25 +cmd = 'curl -vs -H "host:oc.test" http://127.0.0.1:{0}'.format(ts.Variables.port) +numberOfRequests = 100 tr = Test.AddTestRun() # Create a bunch of curl commands to be executed in parallel. Default.Process is set in SpawnCommands. # On Fedora 28/29, it seems that curl will occaisionally timeout after a couple seconds and return exitcode 2 # Examinig the packet capture shows that Traffic Server dutifully sends the response -ps = tr.SpawnCommands(cmdstr=cmd, count=numberOfRequests, retcode=Any(0,2)) +ps = tr.SpawnCommands(cmdstr=cmd, count=numberOfRequests, retcode=Any(0, 2)) tr.Processes.Default.Env = ts.Env -tr.Processes.Default.ReturnCode = Any(0,2) +tr.Processes.Default.ReturnCode = Any(0, 2) # Execution order is: ts/server, ps(curl cmds), Default Process. tr.Processes.Default.StartBefore( server, ready=When.PortOpen(server.Variables.Port)) -# Adds a delay once the ts port is ready. This is because we cannot test the ts state. -tr.Processes.Default.StartBefore(ts, ready=10) +tr.Processes.Default.StartBefore(Test.Processes.ts, ready=When.PortOpen(ts.Variables.port)) ts.StartAfter(*ps) server.StartAfter(*ps) tr.StillRunningAfter = ts # Signal that all the curl processes have completed tr = Test.AddTestRun("Curl Done") +tr.DelayStart = 2 # Delaying a couple seconds to make sure the global continuation's lock contention resolves. tr.Processes.Default.Command = "traffic_ctl plugin msg done done" tr.Processes.Default.ReturnCode = 0 tr.Processes.Default.Env = ts.Env @@ -82,15 +82,18 @@ # Parking this as a ready tester on a meaningless process # To stall the test runs that check for the stats until the # stats have propagated and are ready to read. -def make_done_stat_ready(tsenv): - def done_stat_ready(process, hasRunFor, **kw): - retval = subprocess.run("traffic_ctl metric get ssntxnorder_verify.test.done > done 2> /dev/null", shell=True, env=tsenv) - if retval.returncode == 0: - retval = subprocess.run("grep 1 done > /dev/null", shell = True, env=tsenv) - return retval.returncode == 0 - - return done_stat_ready - + + +def make_done_stat_ready(tsenv): + def done_stat_ready(process, hasRunFor, **kw): + retval = subprocess.run("traffic_ctl metric get ssntxnorder_verify.test.done > done 2> /dev/null", shell=True, env=tsenv) + if retval.returncode == 0: + retval = subprocess.run("grep 1 done > /dev/null", shell=True, env=tsenv) + return retval.returncode == 0 + + return done_stat_ready + + # number of sessions/transactions opened and closed are equal tr = Test.AddTestRun("Check Ssn order errors") server2.StartupTimeout = 60 @@ -138,4 +141,3 @@ def done_stat_ready(process, hasRunFor, **kw): "ssntxnorder_verify.txn.start {}".format(numberOfRequests), 'should be the number of transactions we made') tr.StillRunningAfter = ts tr.StillRunningAfter = server - diff --git a/tests/gold_tests/continuations/openclose_h2.test.py b/tests/gold_tests/continuations/openclose_h2.test.py index 932670cfd3b..db5ba69ce0b 100644 --- a/tests/gold_tests/continuations/openclose_h2.test.py +++ b/tests/gold_tests/continuations/openclose_h2.test.py @@ -43,13 +43,14 @@ ts.addSSLfile("ssl/server.pem") ts.addSSLfile("ssl/server.key") -Test.PreparePlugin(os.path.join(Test.Variables.AtsTestToolsDir, - 'plugins', 'ssntxnorder_verify.cc'), ts) +Test.PrepareTestPlugin(os.path.join(Test.Variables.AtsTestPluginsDir, + 'ssntxnorder_verify.so'), ts) # add response to the server dictionary server.addResponse("sessionfile.log", request_header, response_header) ts.Disk.records_config.update({ - 'proxy.config.diags.debug.enabled': 1, + 'proxy.config.http2.zombie_debug_timeout_in': 10, + 'proxy.config.diags.debug.enabled': 0, 'proxy.config.diags.debug.tags': 'ssntxnorder_verify', 'proxy.config.http.cache.http': 0, # disable cache to simply the test. 'proxy.config.cache.enable_read_while_writer': 0, @@ -58,35 +59,36 @@ }) ts.Disk.remap_config.AddLine( - 'map http://oc.test:{0} http://127.0.0.1:{1}'.format( - ts.Variables.port, server.Variables.Port) + 'map https://oc.test:{0} http://127.0.0.1:{1}'.format( + ts.Variables.ssl_port, server.Variables.Port) ) ts.Disk.ssl_multicert_config.AddLine( 'dest_ip=* ssl_cert_name=server.pem ssl_key_name=server.key' ) -cmd = 'curl -k --http2 -vs https://127.0.0.1:{0}'.format(ts.Variables.ssl_port) -numberOfRequests = 25 +cmd = 'curl -k --resolve oc.test:{0}:127.0.0.1 --http2 https://oc.test:{0}'.format(ts.Variables.ssl_port) +numberOfRequests = 100 tr = Test.AddTestRun() # Create a bunch of curl commands to be executed in parallel. Default.Process is set in SpawnCommands. # On Fedora 28/29, it seems that curl will occaisionally timeout after a couple seconds and return exitcode 2 # Examinig the packet capture shows that Traffic Server dutifully sends the response -ps = tr.SpawnCommands(cmdstr=cmd, count=numberOfRequests, retcode=Any(0,2)) +ps = tr.SpawnCommands(cmdstr=cmd, count=numberOfRequests, retcode=Any(0, 2)) tr.Processes.Default.Env = ts.Env -tr.Processes.Default.ReturnCode = Any(0,2) +tr.Processes.Default.ReturnCode = Any(0, 2) # Execution order is: ts/server, ps(curl cmds), Default Process. tr.Processes.Default.StartBefore( server, ready=When.PortOpen(server.Variables.Port)) -# Adds a delay once the ts port is ready. This is because we cannot test the ts state. -tr.Processes.Default.StartBefore(ts, ready=10) +tr.Processes.Default.StartBefore(Test.Processes.ts, ready=When.PortOpen(ts.Variables.ssl_port)) +# Don't know why we need both the start before and the start after ts.StartAfter(*ps) server.StartAfter(*ps) tr.StillRunningAfter = ts # Signal that all the curl processes have completed tr = Test.AddTestRun("Curl Done") +tr.DelayStart = 2 # Delaying a couple seconds to make sure the global continuation's lock contention resolves. tr.Processes.Default.Command = "traffic_ctl plugin msg done done" tr.Processes.Default.ReturnCode = 0 tr.Processes.Default.Env = ts.Env @@ -95,14 +97,17 @@ # Parking this as a ready tester on a meaningless process # To stall the test runs that check for the stats until the # stats have propagated and are ready to read. + + def make_done_stat_ready(tsenv): - def done_stat_ready(process, hasRunFor, **kw): - retval = subprocess.run("traffic_ctl metric get ssntxnorder_verify.test.done > done 2> /dev/null", shell=True, env=tsenv) - if retval.returncode == 0: - retval = subprocess.run("grep 1 done > /dev/null", shell = True, env=tsenv) - return retval.returncode == 0 + def done_stat_ready(process, hasRunFor, **kw): + retval = subprocess.run("traffic_ctl metric get ssntxnorder_verify.test.done > done 2> /dev/null", shell=True, env=tsenv) + if retval.returncode == 0: + retval = subprocess.run("grep 1 done > /dev/null", shell=True, env=tsenv) + return retval.returncode == 0 + + return done_stat_ready - return done_stat_ready # number of sessions/transactions opened and closed are equal tr = Test.AddTestRun("Check Ssn order errors") @@ -151,4 +156,3 @@ def done_stat_ready(process, hasRunFor, **kw): "ssntxnorder_verify.txn.start {}".format(numberOfRequests), 'should be the number of transactions we made') tr.StillRunningAfter = ts tr.StillRunningAfter = server - diff --git a/tests/gold_tests/continuations/plugins/Makefile.inc b/tests/gold_tests/continuations/plugins/Makefile.inc new file mode 100644 index 00000000000..f39c9af6f39 --- /dev/null +++ b/tests/gold_tests/continuations/plugins/Makefile.inc @@ -0,0 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +noinst_LTLIBRARIES += gold_tests/continuations/plugins/session_id_verify.la +gold_tests_continuations_plugins_session_id_verify_la_SOURCES = gold_tests/continuations/plugins/session_id_verify.cc diff --git a/tests/gold_tests/h2/gold/h2spec_stdout.gold b/tests/gold_tests/h2/gold/h2spec_stdout.gold new file mode 100644 index 00000000000..02732ee469e --- /dev/null +++ b/tests/gold_tests/h2/gold/h2spec_stdout.gold @@ -0,0 +1,3 @@ +`` +Finished in `` seconds +`` tests, `` passed, 0 skipped, 0 failed diff --git a/tests/gold_tests/h2/gold/http2_9_stderr.gold b/tests/gold_tests/h2/gold/http2_9_stderr.gold new file mode 100644 index 00000000000..ea2f1297786 --- /dev/null +++ b/tests/gold_tests/h2/gold/http2_9_stderr.gold @@ -0,0 +1,10 @@ +`` +> GET /status/204 HTTP/2 +> Host: `` +> User-Agent: curl/`` +> Accept: */* +`` +< HTTP/2 204`` +< server: ATS/`` +< date: `` +`` diff --git a/tests/gold_tests/h2/gold/http2_9_stdout.gold b/tests/gold_tests/h2/gold/http2_9_stdout.gold new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/gold_tests/h2/gold/httpbin_0_stdout.gold b/tests/gold_tests/h2/gold/httpbin_0_stdout.gold index c46139dc0bf..d8e61077a93 100644 --- a/tests/gold_tests/h2/gold/httpbin_0_stdout.gold +++ b/tests/gold_tests/h2/gold/httpbin_0_stdout.gold @@ -1,10 +1,5 @@ { - "args": {}, `` - "headers": { - "Accept": "*/*", `` - "Host": `` - "User-Agent": "curl/`` - }, `` - "origin": `` + `` "url": "http://``/get" + `` } diff --git a/tests/gold_tests/h2/gold/httpbin_3_stderr.gold b/tests/gold_tests/h2/gold/httpbin_3_stderr.gold new file mode 100644 index 00000000000..8109f9de4e4 --- /dev/null +++ b/tests/gold_tests/h2/gold/httpbin_3_stderr.gold @@ -0,0 +1,9 @@ +`` +> POST /post HTTP/2 +`` +> Expect: 100-continue +`` +< HTTP/2 100`` +`` +< HTTP/2 200`` +`` diff --git a/tests/gold_tests/h2/gold/httpbin_3_stdout.gold b/tests/gold_tests/h2/gold/httpbin_3_stdout.gold new file mode 100644 index 00000000000..610580810ce --- /dev/null +++ b/tests/gold_tests/h2/gold/httpbin_3_stdout.gold @@ -0,0 +1,7 @@ +{ + `` + "form": { + "key": "value" + }, + `` +} diff --git a/tests/gold_tests/h2/gold/httpbin_access.gold b/tests/gold_tests/h2/gold/httpbin_access.gold index 3f77947edb3..d409c47d13a 100644 --- a/tests/gold_tests/h2/gold/httpbin_access.gold +++ b/tests/gold_tests/h2/gold/httpbin_access.gold @@ -1,3 +1,4 @@ [``] GET http://127.0.0.1:``/get HTTP/1.1 http/2 `` `` TCP_MISS 200 `` [``] GET http://127.0.0.1:``/bytes/0 HTTP/1.1 http/2 `` `` TCP_MISS 200 0 [``] GET http://127.0.0.1:``/stream-bytes/102400?seed=0 HTTP/1.1 http/2 `` `` TCP_MISS 200 102400 +`` diff --git a/tests/gold_tests/h2/gold/nghttp_0_stdout.gold b/tests/gold_tests/h2/gold/nghttp_0_stdout.gold new file mode 100644 index 00000000000..1487943ef0d --- /dev/null +++ b/tests/gold_tests/h2/gold/nghttp_0_stdout.gold @@ -0,0 +1,17 @@ +`` +[``] send HEADERS frame +`` +``trailer: foo +`` +[``] send DATA frame +`` +[``] send HEADERS frame +``; END_STREAM | END_HEADERS +`` +``foo: bar +`` +[``] recv (stream_id=1) :status: 200 +`` +[``] recv RST_STREAM frame +``(error_code=NO_ERROR(0x00)) +`` diff --git a/tests/gold_tests/h2/h2active_timeout.py b/tests/gold_tests/h2/h2active_timeout.py index b71608c3479..aaf2622e66b 100644 --- a/tests/gold_tests/h2/h2active_timeout.py +++ b/tests/gold_tests/h2/h2active_timeout.py @@ -16,30 +16,31 @@ # See the License for the specific language governing permissions and # limitations under the License. -import json from hyper import HTTPConnection import hyper import argparse import time -def makerequest(port): +def makerequest(port, active_timeout): hyper.tls._context = hyper.tls.init_context() hyper.tls._context.check_hostname = False hyper.tls._context.verify_mode = hyper.compat.ssl.CERT_NONE conn = HTTPConnection('localhost:{0}'.format(port), secure=True) - active_timeout = 3 - request_interval = 0.1 - loop_cnt = int((active_timeout + 2) / request_interval) - for i in range(loop_cnt): - try: - conn.request('GET', '/') - time.sleep(request_interval) - except: - print('CONNECTION_TIMEOUT') - return + try: + # delay after sending the first request + # so the H2 session active timeout triggers + # Then the next request should fail + req_id = conn.request('GET', '/') + time.sleep(active_timeout) + response = conn.get_response(req_id) + req_id = conn.request('GET', '/') + response = conn.get_response(req_id) + except Exception: + print('CONNECTION_TIMEOUT') + return print('NO_TIMEOUT') @@ -49,8 +50,11 @@ def main(): parser.add_argument("--port", "-p", type=int, help="Port to use") + parser.add_argument("--delay", "-d", + type=int, + help="Time to delay in seconds") args = parser.parse_args() - makerequest(args.port) + makerequest(args.port, args.delay) if __name__ == '__main__': diff --git a/tests/gold_tests/h2/h2bigclient.py b/tests/gold_tests/h2/h2bigclient.py index 6e0cec809e7..2e1d86d3311 100644 --- a/tests/gold_tests/h2/h2bigclient.py +++ b/tests/gold_tests/h2/h2bigclient.py @@ -16,7 +16,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import json from hyper import HTTPConnection import hyper import argparse @@ -43,7 +42,6 @@ def makerequest(port): # Fetch the object twice so we know at least one time comes from cache # Exploring timing options sites = ['/bigfile', '/bigfile'] - responses = [] request_ids = [] for site in sites: request_id = conn.request('GET', url=site) diff --git a/tests/gold_tests/h2/h2chunked.py b/tests/gold_tests/h2/h2chunked.py index 8b1d05196ab..a7fee074ace 100644 --- a/tests/gold_tests/h2/h2chunked.py +++ b/tests/gold_tests/h2/h2chunked.py @@ -16,7 +16,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import json from hyper import HTTPConnection import hyper import argparse @@ -41,9 +40,8 @@ def makerequest(port, _url): conn = HTTPConnection('localhost:{0}'.format(port), secure=True) sites = {'/'} - responses = [] request_ids = [] - for site in sites: + for _ in sites: request_id = conn.request('GET', url=_url) request_ids.append(request_id) diff --git a/tests/gold_tests/h2/h2client.py b/tests/gold_tests/h2/h2client.py index df0d19fb627..d8f3ce0a101 100644 --- a/tests/gold_tests/h2/h2client.py +++ b/tests/gold_tests/h2/h2client.py @@ -16,7 +16,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import json from hyper import HTTPConnection import hyper import argparse @@ -41,7 +40,6 @@ def makerequest(port): conn = HTTPConnection('localhost:{0}'.format(port), secure=True) sites = {'/'} - responses = [] request_ids = [] for site in sites: request_id = conn.request('GET', url=site) diff --git a/tests/gold_tests/h2/h2disable.test.py b/tests/gold_tests/h2/h2disable.test.py index f9347b669bd..5f987e32514 100644 --- a/tests/gold_tests/h2/h2disable.test.py +++ b/tests/gold_tests/h2/h2disable.test.py @@ -16,12 +16,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -import os Test.Summary = ''' Test disabling H2 on a per domain basis ''' -# need Curl Test.SkipUnless( Condition.HasCurlFeature('http2') ) @@ -58,11 +56,11 @@ }) ts.Disk.sni_yaml.AddLines([ - 'sni:', - '- fqdn: bar.com', - ' http2: off', - '- fqdn: bob.*.com', - ' http2: off', + 'sni:', + '- fqdn: bar.com', + ' http2: off', + '- fqdn: bob.*.com', + ' http2: off', ]) tr = Test.AddTestRun("Negotiate-h2") @@ -88,7 +86,8 @@ tr2.TimeOut = 5 tr2 = Test.AddTestRun("Do not negotiate h2") -tr2.Processes.Default.Command = "curl -v -k --resolve 'bob.foo.com:{0}:127.0.0.1' https://bob.foo.com:{0}".format(ts.Variables.ssl_port) +tr2.Processes.Default.Command = "curl -v -k --resolve 'bob.foo.com:{0}:127.0.0.1' https://bob.foo.com:{0}".format( + ts.Variables.ssl_port) tr2.ReturnCode = 0 tr2.StillRunningAfter = server tr2.Processes.Default.TimeOut = 5 diff --git a/tests/gold_tests/h2/h2disable_no_accept_threads.test.py b/tests/gold_tests/h2/h2disable_no_accept_threads.test.py index bbf5dd012bc..e99952ba259 100644 --- a/tests/gold_tests/h2/h2disable_no_accept_threads.test.py +++ b/tests/gold_tests/h2/h2disable_no_accept_threads.test.py @@ -16,12 +16,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -import os Test.Summary = ''' Test disabling H2 on a per domain basis ''' -# need Curl Test.SkipUnless( Condition.HasCurlFeature('http2') ) @@ -58,11 +56,11 @@ }) ts.Disk.sni_yaml.AddLines([ - 'sni:', - '- fqdn: bar.com', - ' http2: off', - '- fqdn: bob.*.com', - ' http2: off', + 'sni:', + '- fqdn: bar.com', + ' http2: off', + '- fqdn: bob.*.com', + ' http2: off', ]) tr = Test.AddTestRun("Negotiate-h2") @@ -88,7 +86,8 @@ tr2.TimeOut = 5 tr2 = Test.AddTestRun("Do not negotiate h2") -tr2.Processes.Default.Command = "curl -v -k --resolve 'bob.foo.com:{0}:127.0.0.1' https://bob.foo.com:{0}".format(ts.Variables.ssl_port) +tr2.Processes.Default.Command = "curl -v -k --resolve 'bob.foo.com:{0}:127.0.0.1' https://bob.foo.com:{0}".format( + ts.Variables.ssl_port) tr2.ReturnCode = 0 tr2.StillRunningAfter = server tr2.Processes.Default.TimeOut = 5 diff --git a/tests/gold_tests/h2/h2enable.test.py b/tests/gold_tests/h2/h2enable.test.py index 0357e93168b..6f7cf02f1eb 100644 --- a/tests/gold_tests/h2/h2enable.test.py +++ b/tests/gold_tests/h2/h2enable.test.py @@ -16,12 +16,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -import os Test.Summary = ''' Test enabling H2 on a per domain basis ''' -# need Curl Test.SkipUnless( Condition.HasCurlFeature('http2') ) @@ -58,11 +56,11 @@ }) ts.Disk.sni_yaml.AddLines([ - 'sni:', - '- fqdn: bar.com', - ' http2: on', - '- fqdn: bob.*.com', - ' http2: on', + 'sni:', + '- fqdn: bar.com', + ' http2: on', + '- fqdn: bob.*.com', + ' http2: on', ]) tr = Test.AddTestRun("Do-not-Negotiate-h2") @@ -88,7 +86,8 @@ tr2.TimeOut = 5 tr2 = Test.AddTestRun("Do negotiate h2") -tr2.Processes.Default.Command = "curl -v -k --resolve 'bob.foo.com:{0}:127.0.0.1' https://bob.foo.com:{0}".format(ts.Variables.ssl_port) +tr2.Processes.Default.Command = "curl -v -k --resolve 'bob.foo.com:{0}:127.0.0.1' https://bob.foo.com:{0}".format( + ts.Variables.ssl_port) tr2.ReturnCode = 0 tr2.StillRunningAfter = server tr2.Processes.Default.TimeOut = 5 diff --git a/tests/gold_tests/h2/h2enable_no_accept_threads.test.py b/tests/gold_tests/h2/h2enable_no_accept_threads.test.py index 0149e54fad5..0e914d0c385 100644 --- a/tests/gold_tests/h2/h2enable_no_accept_threads.test.py +++ b/tests/gold_tests/h2/h2enable_no_accept_threads.test.py @@ -16,12 +16,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -import os Test.Summary = ''' Test enabling H2 on a per domain basis ''' -# need Curl Test.SkipUnless( Condition.HasCurlFeature('http2') ) @@ -58,11 +56,11 @@ }) ts.Disk.sni_yaml.AddLines([ - 'sni:', - '- fqdn: bar.com', - ' http2: on', - '- fqdn: bob.*.com', - ' http2: on', + 'sni:', + '- fqdn: bar.com', + ' http2: on', + '- fqdn: bob.*.com', + ' http2: on', ]) tr = Test.AddTestRun("Do-not-Negotiate-h2") @@ -88,7 +86,8 @@ tr2.TimeOut = 5 tr2 = Test.AddTestRun("Do negotiate h2") -tr2.Processes.Default.Command = "curl -v -k --resolve 'bob.foo.com:{0}:127.0.0.1' https://bob.foo.com:{0}".format(ts.Variables.ssl_port) +tr2.Processes.Default.Command = "curl -v -k --resolve 'bob.foo.com:{0}:127.0.0.1' https://bob.foo.com:{0}".format( + ts.Variables.ssl_port) tr2.ReturnCode = 0 tr2.StillRunningAfter = server tr2.Processes.Default.TimeOut = 5 diff --git a/tests/gold_tests/h2/h2spec.test.py b/tests/gold_tests/h2/h2spec.test.py new file mode 100644 index 00000000000..e91fd229ff1 --- /dev/null +++ b/tests/gold_tests/h2/h2spec.test.py @@ -0,0 +1,78 @@ +''' +Test HTTP/2 with h2spec +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +Test.Summary = ''' +Test HTTP/2 with httpspec +''' + +Test.SkipUnless( + Condition.HasProgram("h2spec", "h2spec need to be installed on system for this test to work"), +) +Test.ContinueOnFail = True + +# ---- +# Setup httpbin Origin Server +# ---- +httpbin = Test.MakeHttpBinServer("httpbin") + +# ---- +# Setup ATS +# ---- +ts = Test.MakeATSProcess("ts", select_ports=False) + +# add ssl materials like key, certificates for the server +ts.addSSLfile("ssl/server.pem") +ts.addSSLfile("ssl/server.key") + +ts.Variables.ssl_port = 4443 +ts.Disk.remap_config.AddLine( + 'map / http://127.0.0.1:{0}'.format(httpbin.Variables.Port) +) +ts.Disk.ssl_multicert_config.AddLine( + 'dest_ip=* ssl_cert_name=server.pem ssl_key_name=server.key' +) +ts.Disk.records_config.update({ + 'proxy.config.http.server_ports': '{0} {1}:ssl'.format(ts.Variables.port, ts.Variables.ssl_port), + 'proxy.config.http.insert_request_via_str': 1, + 'proxy.config.http.insert_response_via_str': 1, + 'proxy.config.http.cache.http': 0, + 'proxy.config.ssl.server.cert.path': '{0}'.format(ts.Variables.SSLDir), + 'proxy.config.ssl.server.private_key.path': '{0}'.format(ts.Variables.SSLDir), + 'proxy.config.ssl.client.verify.server': 0, + 'proxy.config.diags.debug.enabled': 0, + 'proxy.config.diags.debug.tags': 'http', +}) + +# ---- +# Test Cases +# ---- + +# Known broken tests are left out (http2/6.4. and http2/6.9.) +h2spec_targets = "http2/1 http2/2 http2/3 http2/4 http2/5 http2/6.1 http2/6.2 http2/6.3 http2/6.5 http2/6.6 http2/6.7 http2/6.8 http2/7 http2/8 hpack" + +test_run = Test.AddTestRun() +test_run.Processes.Default.Command = 'h2spec {0} -t -k --timeout 10 -p {1}'.format(h2spec_targets, ts.Variables.ssl_port) +test_run.Processes.Default.ReturnCode = 0 +test_run.Processes.Default.StartBefore(httpbin, ready=When.PortOpen(httpbin.Variables.Port)) +test_run.Processes.Default.StartBefore(Test.Processes.ts, ready=When.PortOpen(ts.Variables.ssl_port)) +test_run.Processes.Default.Streams.stdout = "gold/h2spec_stdout.gold" +test_run.StillRunningAfter = httpbin + +# Over riding the built in ERROR check since we expect some error cases +ts.Disk.diags_log.Content = Testers.ContainsExpression("ERROR: HTTP/2", "h2spec tests should have error log") diff --git a/tests/gold_tests/h2/http2.test.py b/tests/gold_tests/h2/http2.test.py index 29f7a19ef75..cbced007e59 100644 --- a/tests/gold_tests/h2/http2.test.py +++ b/tests/gold_tests/h2/http2.test.py @@ -18,59 +18,94 @@ import os Test.Summary = ''' -Test a basic remap of a http connection +Test a basic remap of a http/2 connection ''' -# need Curl + Test.SkipUnless( Condition.HasCurlFeature('http2') ) Test.ContinueOnFail = True -# Define default ATS -ts = Test.MakeATSProcess("ts", select_ports=True, enable_tls=True) + +# ---- +# Setup Origin Server +# ---- server = Test.MakeOriginServer("server") -requestLocation = "test2" -reHost = "www.example.com" - -testName = "" -request_header = {"headers": "GET / HTTP/1.1\r\nHost: www.example.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} -# desired response form the origin server -response_header = {"headers": "HTTP/1.1 200 OK\r\nServer: microserver\r\nConnection: close\r\n\r\n", - "timestamp": "1469733493.993", "body": ""} -request_header2 = { - "headers": "GET /{0} HTTP/1.1\r\nHost: {1}\r\n\r\n".format(requestLocation, reHost), "timestamp": "1469733493.993", "body": ""} -# desired response form the origin server -response_header2 = {"headers": "HTTP/1.1 200 OK\r\nServer: microserver\r\nTransfer-Encoding: chunked\r\nConnection: close\r\n\r\n", - "timestamp": "1469733493.993", "body": ""} -server.addResponse("sessionlog.json", request_header, response_header) +# For Test Case 1 & 5 - / +server.addResponse("sessionlog.json", + {"headers": "GET / HTTP/1.1\r\nHost: www.example.com\r\n\r\n", + "timestamp": "1469733493.993", + "body": ""}, + {"headers": "HTTP/1.1 200 OK\r\nServer: microserver\r\nConnection: close\r\n\r\n", + "timestamp": "1469733493.993", + "body": ""}) +# For Test Case 2 - /bigfile # Add info for the large H2 download test server.addResponse("sessionlog.json", - {"headers": "GET /bigfile HTTP/1.1\r\nHost: www.example.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""}, - {"headers": "HTTP/1.1 200 OK\r\nServer: microserver\r\nConnection: close\r\nCache-Control: max-age=3600\r\nContent-Length: 191414\r\n\r\n", "timestamp": "1469733493.993", "body": ""}) + {"headers": "GET /bigfile HTTP/1.1\r\nHost: www.example.com\r\n\r\n", + "timestamp": "1469733493.993", + "body": ""}, + {"headers": "HTTP/1.1 200 OK\r\nServer: microserver\r\nConnection: close\r\nCache-Control: max-age=3600\r\nContent-Length: 191414\r\n\r\n", + "timestamp": "1469733493.993", + "body": ""}) + +# For Test Case 3 - /test2 +server.addResponse("sessionlog.json", + {"headers": "GET /test2 HTTP/1.1\r\nHost: www.example.com\r\n\r\n", + "timestamp": "1469733493.993", + "body": ""}, + {"headers": "HTTP/1.1 200 OK\r\nServer: microserver\r\nTransfer-Encoding: chunked\r\nConnection: close\r\n\r\n", + "timestamp": "1469733493.993", + "body": ""}) +# For Test Case 6 - /postchunked post_body = "12345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890" server.addResponse("sessionlog.jason", {"headers": "POST /postchunked HTTP/1.1\r\nHost: www.example.com\r\n\r\n", "timestamp": "1469733493.993", - "body": post_body}, - {"headers": "HTTP/1.1 200 OK\r\nServer: microserver\r\nConnection: close\r\nContent-Length: 10\r\n\r\n", "timestamp": "1469733493.993", "body": "0123456789"}) + "body": post_body}, + {"headers": "HTTP/1.1 200 OK\r\nServer: microserver\r\nConnection: close\r\nContent-Length: 10\r\n\r\n", + "timestamp": "1469733493.993", + "body": "0123456789"}) +# For Test Case 7 - /bigpostchunked # Make a post body that will be split across at least two frames big_post_body = "0123456789" * 131070 server.addResponse("sessionlog.jason", {"headers": "POST /bigpostchunked HTTP/1.1\r\nHost: www.example.com\r\n\r\n", "timestamp": "1469733493.993", - "body": big_post_body}, - {"headers": "HTTP/1.1 200 OK\r\nServer: microserver\r\nConnection: close\r\nContent-Length: 10\r\n\r\n", "timestamp": "1469733493.993", "body": "0123456789"}) + "body": big_post_body}, + {"headers": "HTTP/1.1 200 OK\r\nServer: microserver\r\nConnection: close\r\nContent-Length: 10\r\n\r\n", + "timestamp": "1469733493.993", + "body": "0123456789"}) +big_post_body_file = open(os.path.join(Test.RunDirectory, "big_post_body"), "w") +big_post_body_file.write(big_post_body) +big_post_body_file.close() -server.addResponse("sessionlog.json", request_header2, response_header2) +# For Test Case 8 - /huge_resp_hdrs +server.addResponse("sessionlog.json", + {"headers": "GET /huge_resp_hdrs HTTP/1.1\r\nHost: www.example.com\r\n\r\n", + "timestamp": "1469733493.993", + "body": ""}, + {"headers": "HTTP/1.1 200 OK\r\nServer: microserver\r\nConnection: close\r\nContent-Length: 6\r\n\r\n", + "timestamp": "1469733493.993", + "body": "200 OK"}) -# request/response for test case 8 -request_header = {"headers": "GET /huge_resp_hdrs HTTP/1.1\r\nHost: www.example.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} -response_header = {"headers": "HTTP/1.1 200 OK\r\nServer: microserver\r\nConnection: close\r\nContent-Length: 6\r\n\r\n", "timestamp": "1469733493.993", "body": "200 OK"} -server.addResponse("sessionlog.json", request_header, response_header) +# For Test Case 9 - /status/204 +server.addResponse("sessionlog.json", + {"headers": "GET /status/204 HTTP/1.1\r\nHost: www.example.com\r\n\r\n", + "timestamp": "1469733493.993", + "body": ""}, + {"headers": "HTTP/1.1 204 No Content\r\nServer: microserver\r\nConnection: close\r\n\r\n", + "timestamp": "1469733493.993", + "body": ""}) + +# ---- +# Setup ATS +# ---- +ts = Test.MakeATSProcess("ts", select_ports=True, enable_tls=True) # add ssl materials like key, certificates for the server ts.addSSLfile("ssl/server.pem") @@ -78,8 +113,8 @@ ts.Setup.CopyAs('rules/huge_resp_hdrs.conf', Test.RunDirectory) ts.Disk.remap_config.AddLine( - 'map /huge_resp_hdrs http://127.0.0.1:{0}/huge_resp_hdrs @plugin=header_rewrite.so @pparam={1}/huge_resp_hdrs.conf '.format(server.Variables.Port, Test.RunDirectory) -) + 'map /huge_resp_hdrs http://127.0.0.1:{0}/huge_resp_hdrs @plugin=header_rewrite.so @pparam={1}/huge_resp_hdrs.conf '.format( + server.Variables.Port, Test.RunDirectory)) ts.Disk.remap_config.AddLine( 'map / http://127.0.0.1:{0}'.format(server.Variables.Port) @@ -94,21 +129,21 @@ 'proxy.config.ssl.server.cert.path': '{0}'.format(ts.Variables.SSLDir), 'proxy.config.ssl.server.private_key.path': '{0}'.format(ts.Variables.SSLDir), 'proxy.config.http.cache.http': 0, - 'proxy.config.ssl.client.verify.server': 0, + 'proxy.config.ssl.client.verify.server': 0, 'proxy.config.ssl.server.cipher_suite': 'ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA384:AES128-GCM-SHA256:AES256-GCM-SHA384:ECDHE-RSA-RC4-SHA:ECDHE-RSA-AES128-SHA:ECDHE-RSA-AES256-SHA:RC4-SHA:RC4-MD5:AES128-SHA:AES256-SHA:DES-CBC3-SHA!SRP:!DSS:!PSK:!aNULL:!eNULL:!SSLv2', 'proxy.config.http2.active_timeout_in': 3, 'proxy.config.http2.max_concurrent_streams_in': 65535, }) -big_post_body_file = open(os.path.join(Test.RunDirectory, "big_post_body"), "w") -big_post_body_file.write(big_post_body) -big_post_body_file.close() - ts.Setup.CopyAs('h2client.py', Test.RunDirectory) ts.Setup.CopyAs('h2bigclient.py', Test.RunDirectory) ts.Setup.CopyAs('h2chunked.py', Test.RunDirectory) ts.Setup.CopyAs('h2active_timeout.py', Test.RunDirectory) +# ---- +# Test Cases +# ---- + # Test Case 1: basic H2 interaction tr = Test.AddTestRun() tr.Processes.Default.Command = 'python3 h2client.py -p {0}'.format(ts.Variables.ssl_port) @@ -128,7 +163,7 @@ # Test Case 3: Chunked content tr = Test.AddTestRun() -tr.Processes.Default.Command = 'python3 h2chunked.py -p {0} -u /{1}'.format(ts.Variables.ssl_port, requestLocation) +tr.Processes.Default.Command = 'python3 h2chunked.py -p {0} -u /test2'.format(ts.Variables.ssl_port) tr.Processes.Default.ReturnCode = 0 tr.Processes.Default.Streams.stdout = "gold/chunked.gold" tr.StillRunningAfter = server @@ -143,9 +178,9 @@ # tr.Processes.Default.Streams.stdout = "gold/replay.gold" # tr.StillRunningAfter = server -# Test Case 5:h2_active_timeout +# Test Case 5: h2_active_timeout tr = Test.AddTestRun() -tr.Processes.Default.Command = 'python3 h2active_timeout.py -p {0}'.format(ts.Variables.ssl_port) +tr.Processes.Default.Command = 'python3 h2active_timeout.py -p {0} -d 4'.format(ts.Variables.ssl_port) tr.Processes.Default.ReturnCode = 0 tr.Processes.Default.Streams.All = "gold/active_timeout.gold" tr.StillRunningAfter = server @@ -154,7 +189,8 @@ # While HTTP/2 does not support Tranfer-encoding we pass that into curl to encourage it to not set the content length # on the post body tr = Test.AddTestRun() -tr.Processes.Default.Command = 'curl -s -k -H "Transfer-Encoding: chunked" -d "{0}" https://127.0.0.1:{1}/postchunked'.format( post_body, ts.Variables.ssl_port) +tr.Processes.Default.Command = 'curl -s -k -H "Transfer-Encoding: chunked" -d "{0}" https://127.0.0.1:{1}/postchunked'.format( + post_body, ts.Variables.ssl_port) tr.Processes.Default.ReturnCode = 0 tr.Processes.Default.Streams.All = "gold/post_chunked.gold" tr.StillRunningAfter = server @@ -163,7 +199,8 @@ # While HTTP/2 does not support Tranfer-encoding we pass that into curl to encourage it to not set the content length # on the post body tr = Test.AddTestRun() -tr.Processes.Default.Command = 'curl -s -k -H "Transfer-Encoding: chunked" -d @big_post_body https://127.0.0.1:{0}/bigpostchunked'.format( ts.Variables.ssl_port) +tr.Processes.Default.Command = 'curl -s -k -H "Transfer-Encoding: chunked" -d @big_post_body https://127.0.0.1:{0}/bigpostchunked'.format( + ts.Variables.ssl_port) tr.Processes.Default.ReturnCode = 0 tr.Processes.Default.Streams.All = "gold/post_chunked.gold" tr.StillRunningAfter = server @@ -175,3 +212,11 @@ tr.Processes.Default.Streams.stdout = "gold/http2_8_stdout.gold" tr.Processes.Default.Streams.stderr = "gold/http2_8_stderr.gold" tr.StillRunningAfter = server + +# Test Case 9: Header Only Response - e.g. 204 +tr = Test.AddTestRun() +tr.Processes.Default.Command = 'curl -vs -k --http2 https://127.0.0.1:{0}/status/204'.format(ts.Variables.ssl_port) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Streams.stdout = "gold/http2_9_stdout.gold" +tr.Processes.Default.Streams.stderr = "gold/http2_9_stderr.gold" +tr.StillRunningAfter = server diff --git a/tests/gold_tests/h2/http2_priority.test.py b/tests/gold_tests/h2/http2_priority.test.py index e0911da8795..117b28610a1 100644 --- a/tests/gold_tests/h2/http2_priority.test.py +++ b/tests/gold_tests/h2/http2_priority.test.py @@ -16,15 +16,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -import os - -# ---- -# Setup Test -# ---- Test.Summary = ''' Test a basic remap of a http connection with Stream Priority Feature ''' -# need Curl + Test.SkipUnless( Condition.HasCurlFeature('http2'), Condition.HasProgram("shasum", "shasum need to be installed on system for this test to work"), @@ -38,8 +33,12 @@ # Test Case 0: server.addResponse("sessionlog.json", - {"headers": "GET /bigfile HTTP/1.1\r\nHost: www.example.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""}, - {"headers": "HTTP/1.1 200 OK\r\nServer: microserver\r\nConnection: close\r\nCache-Control: max-age=3600\r\nContent-Length: 1048576\r\n\r\n", "timestamp": "1469733493.993", "body": ""}) + {"headers": "GET /bigfile HTTP/1.1\r\nHost: www.example.com\r\n\r\n", + "timestamp": "1469733493.993", + "body": ""}, + {"headers": "HTTP/1.1 200 OK\r\nServer: microserver\r\nConnection: close\r\nCache-Control: max-age=3600\r\nContent-Length: 1048576\r\n\r\n", + "timestamp": "1469733493.993", + "body": ""}) # ---- # Setup ATS @@ -61,7 +60,7 @@ 'proxy.config.http2.no_activity_timeout_in': 3, 'proxy.config.ssl.server.cert.path': '{0}'.format(ts.Variables.SSLDir), 'proxy.config.ssl.server.private_key.path': '{0}'.format(ts.Variables.SSLDir), - 'proxy.config.ssl.client.verify.server': 0, + 'proxy.config.ssl.client.verify.server': 0, 'proxy.config.ssl.server.cipher_suite': 'ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA384:AES128-GCM-SHA256:AES256-GCM-SHA384:ECDHE-RSA-RC4-SHA:ECDHE-RSA-AES128-SHA:ECDHE-RSA-AES256-SHA:RC4-SHA:RC4-MD5:AES128-SHA:AES256-SHA:DES-CBC3-SHA!SRP:!DSS:!PSK:!aNULL:!eNULL:!SSLv2', 'proxy.config.diags.debug.enabled': 1, 'proxy.config.diags.debug.tags': 'http2', diff --git a/tests/gold_tests/h2/httpbin.test.py b/tests/gold_tests/h2/httpbin.test.py index df2698063ff..574f00ffe83 100644 --- a/tests/gold_tests/h2/httpbin.test.py +++ b/tests/gold_tests/h2/httpbin.test.py @@ -57,7 +57,7 @@ 'proxy.config.http.cache.http': 0, 'proxy.config.ssl.server.cert.path': '{0}'.format(ts.Variables.SSLDir), 'proxy.config.ssl.server.private_key.path': '{0}'.format(ts.Variables.SSLDir), - 'proxy.config.ssl.client.verify.server': 0, + 'proxy.config.ssl.client.verify.server': 0, 'proxy.config.ssl.server.cipher_suite': 'ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA384:AES128-GCM-SHA256:AES256-GCM-SHA384:ECDHE-RSA-RC4-SHA:ECDHE-RSA-AES128-SHA:ECDHE-RSA-AES256-SHA:RC4-SHA:RC4-MD5:AES128-SHA:AES256-SHA:DES-CBC3-SHA!SRP:!DSS:!PSK:!aNULL:!eNULL:!SSLv2', 'proxy.config.diags.debug.enabled': 1, 'proxy.config.diags.debug.tags': 'http2', @@ -67,27 +67,30 @@ ''' logging: formats: - # Extended Log Format. - name: access - format: |- -[%] % % % % % % % + format: '[%] % % % % % % %' logs: - filename: access format: access - } '''.split("\n") ) Test.Disk.File(os.path.join(ts.Variables.LOGDIR, 'access.log'), exists=True, content='gold/httpbin_access.gold') +# TODO: when httpbin 0.8.0 or later is released, remove below json pretty print hack +json_printer = ''' +python3 -c "import sys,json; print(json.dumps(json.load(sys.stdin), indent=2, separators=(',', ': ')))" +''' + # ---- # Test Cases # ---- # Test Case 0: Basic request and resposne test_run = Test.AddTestRun() -test_run.Processes.Default.Command = 'curl -vs -k --http2 https://127.0.0.1:{0}/get'.format(ts.Variables.ssl_port) +test_run.Processes.Default.Command = "curl -vs -k --http2 https://127.0.0.1:{0}/get | {1}".format( + ts.Variables.ssl_port, json_printer) test_run.Processes.Default.ReturnCode = 0 test_run.Processes.Default.StartBefore(httpbin, ready=When.PortOpen(httpbin.Variables.Port)) test_run.Processes.Default.StartBefore(Test.Processes.ts) @@ -112,6 +115,16 @@ test_run.Processes.Default.Streams.stderr = "gold/httpbin_2_stderr.gold" test_run.StillRunningAfter = httpbin +# Test Case 3: Expect 100-Continue +test_run = Test.AddTestRun() +test_run.Processes.Default.Command = "curl -vs -k --http2 https://127.0.0.1:{0}/post --data 'key=value' -H 'Expect: 100-continue' --expect100-timeout 1 --max-time 5 | {1}".format( + ts.Variables.ssl_port, json_printer) +test_run.Processes.Default.ReturnCode = 0 +test_run.Processes.Default.Streams.stdout = "gold/httpbin_3_stdout.gold" +test_run.Processes.Default.Streams.stderr = "gold/httpbin_3_stderr.gold" +test_run.StillRunningAfter = httpbin + + # Check Logging test_run = Test.AddTestRun() test_run.DelayStart = 10 diff --git a/tests/gold_tests/h2/nghttp.test.py b/tests/gold_tests/h2/nghttp.test.py new file mode 100644 index 00000000000..2bed24bed26 --- /dev/null +++ b/tests/gold_tests/h2/nghttp.test.py @@ -0,0 +1,87 @@ +''' +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +Test.Summary = ''' +Test with nghttp +''' + +Test.SkipUnless( + Condition.HasProgram("nghttp", "Nghttp need to be installed on system for this test to work"), +) +Test.ContinueOnFail = True + +# ---- +# Setup Origin Server +# ---- +microserver = Test.MakeOriginServer("microserver") + +# 128KB +post_body = "0123456789abcdef" * 8192 +post_body_file = open(os.path.join(Test.RunDirectory, "post_body"), "w") +post_body_file.write(post_body) +post_body_file.close() + +# For Test Case 0 +microserver.addResponse("sessionlog.json", + {"headers": "POST /post HTTP/1.1\r\nHost: www.example.com\r\nTrailer: foo\r\n\r\n", + "timestamp": "1469733493.993", + "body": post_body}, + {"headers": "HTTP/1.1 200 OK\r\nServer: microserver\r\nConnection: close\r\n\r\n", + "timestamp": "1469733493.993", + "body": ""}) + +# ---- +# Setup ATS +# ---- +ts = Test.MakeATSProcess("ts", select_ports=True, enable_tls=True) + +# add ssl materials like key, certificates for the server +ts.addSSLfile("ssl/server.pem") +ts.addSSLfile("ssl/server.key") + +ts.Disk.remap_config.AddLine( + 'map /post http://127.0.0.1:{0}/post'.format(microserver.Variables.Port) +) + +ts.Disk.ssl_multicert_config.AddLine( + 'dest_ip=* ssl_cert_name=server.pem ssl_key_name=server.key' +) + +ts.Disk.records_config.update({ + 'proxy.config.diags.debug.enabled': 1, + 'proxy.config.diags.debug.tags': 'http', + 'proxy.config.ssl.server.cert.path': '{0}'.format(ts.Variables.SSLDir), + 'proxy.config.ssl.server.private_key.path': '{0}'.format(ts.Variables.SSLDir), + 'proxy.config.http.cache.http': 0, + 'proxy.config.http2.active_timeout_in': 3, +}) + +# ---- +# Test Cases +# ---- + +# Test Case 0: Trailer +tr = Test.AddTestRun() +tr.Processes.Default.Command = "nghttp -v --no-dep 'https://127.0.0.1:{0}/post' --trailer 'foo: bar' -d 'post_body'".format( + ts.Variables.ssl_port) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.StartBefore(microserver) +tr.Processes.Default.StartBefore(Test.Processes.ts) +tr.Processes.Default.Streams.stdout = "gold/nghttp_0_stdout.gold" +tr.StillRunningAfter = microserver diff --git a/tests/gold_tests/headers/accept_webp.test.py b/tests/gold_tests/headers/accept_webp.test.py new file mode 100644 index 00000000000..26998fd62a6 --- /dev/null +++ b/tests/gold_tests/headers/accept_webp.test.py @@ -0,0 +1,79 @@ +''' +Test how we handle image/webp +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +Test.Summary = ''' +Checking that we don't serve image/webp to clients that do not support it +''' + +Test.ContinueOnFail = True + +# Define default ATS +ts = Test.MakeATSProcess("ts") +server = Test.MakeOriginServer("server") + +testName = "accept_webp" +request_header = { + "headers": "GET / HTTP/1.1\r\nHost: www.example.com\r\nAccept: image/webp,image/png,image/svg+xml,image/*;q=0.8,video/*;q=0.8,*/*;q=0.5\r\n\r\n", + "timestamp": "1469733493.993", + "body": ""} +response_header = { + "headers": "HTTP/1.1 200 OK\r\nConnection: close\r\nContent-Type: image/webp\r\nCache-Control: max-age=300\r\n", + "timestamp": "1469733493.993", + "body": "xxx"} +server.addResponse("sessionlog.json", request_header, response_header) + +# ATS Configuration +ts.Disk.records_config.update({ + 'proxy.config.diags.debug.enabled': 1, + 'proxy.config.diags.debug.tags': 'http_match', + 'proxy.config.http.cache.ignore_accept_mismatch': 0, + 'proxy.config.http.insert_response_via_str': 3, + 'proxy.config.http.cache.http': 1, + 'proxy.config.http.wait_for_cache': 1, +}) + +ts.Disk.remap_config.AddLine( + 'map http://www.example.com http://127.0.0.1:{0}'.format(server.Variables.Port) +) + +# Test 1 - Request with image/webp support from the origin +tr = Test.AddTestRun() +tr.Processes.Default.StartBefore(server, ready=When.PortOpen(server.Variables.Port)) +tr.Processes.Default.StartBefore(Test.Processes.ts) +tr.Processes.Default.Command = 'curl -s -D - -v --ipv4 --http1.1 -H "Accept: image/webp,image/png,image/svg+xml,image/*;q=0.8,video/*;q=0.8,*/*;q=0.5" -H "Host: www.example.com" http://localhost:{0}/'.format( + ts.Variables.port) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Streams.stderr = "gold/accept_webp.gold" +tr.StillRunningAfter = ts + +# Test 2 - Request with image/webp support from cache +tr = Test.AddTestRun() +tr.Processes.Default.Command = 'curl -s -D - -v --ipv4 --http1.1 -H "Accept: image/webp,image/png,image/svg+xml,image/*;q=0.8,video/*;q=0.8,*/*;q=0.5" -H "Host: www.example.com" http://localhost:{0}/'.format( + ts.Variables.port) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Streams.stderr = "gold/accept_webp_cache.gold" +tr.StillRunningAfter = ts + +# Test 3 - Request without image/webp support going to the origin - NOTE: the origin can't change the content-type :( +tr = Test.AddTestRun() +tr.Processes.Default.Command = 'curl -s -D - -v --ipv4 --http1.1 -H "Accept: image/png,image/svg+xml,image/*;q=0.8,video/*;q=0.8,*/*;q=0.5" -H "Host: www.example.com" http://localhost:{0}/'.format( + ts.Variables.port) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Streams.stderr = "gold/accept_webp_jpeg.gold" +tr.StillRunningAfter = ts diff --git a/tests/gold_tests/headers/cache_and_req_body.test.py b/tests/gold_tests/headers/cache_and_req_body.test.py index a45dc945c6b..c9aba744bd1 100644 --- a/tests/gold_tests/headers/cache_and_req_body.test.py +++ b/tests/gold_tests/headers/cache_and_req_body.test.py @@ -17,7 +17,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import os Test.Summary = ''' Test cached responses and requests with bodies using CurlHeader tester ''' @@ -28,10 +27,13 @@ ts = Test.MakeATSProcess("ts") server = Test.MakeOriginServer("server") -#**testname is required** +# **testname is required** testName = "" request_header = {"headers": "GET / HTTP/1.1\r\nHost: www.example.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} -response_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\nLast-Modified: Tue, 08 May 2018 15:49:41 GMT\r\nCache-Control: max-age=1\r\n\r\n", "timestamp": "1469733493.993", "body": "xxx"} +response_header = { + "headers": "HTTP/1.1 200 OK\r\nConnection: close\r\nLast-Modified: Tue, 08 May 2018 15:49:41 GMT\r\nCache-Control: max-age=1\r\n\r\n", + "timestamp": "1469733493.993", + "body": "xxx"} server.addResponse("sessionlog.json", request_header, response_header) # ATS Configuration @@ -49,70 +51,74 @@ ) cache_and_req_body_miss = { - 'Connection' : 'keep-alive', - 'Via' : {'equal_re' : None}, - 'Server' : {'equal_re' : '.*'}, - 'X-Cache-Key' : {'equal_re' : 'http://127.0.0.1.*'}, - 'X-Cache' : 'miss', - 'Last-Modified' : {'equal_re' : '.*'}, - 'cache-control' : 'max-age=1', - 'Content-Length' : '3', - 'Date' : {'equal_re' : '.*'}, - 'Age' : {'equal_re' : '.*'} + 'Connection': 'keep-alive', + 'Via': {'equal_re': None}, + 'Server': {'equal_re': '.*'}, + 'X-Cache-Key': {'equal_re': 'http://127.0.0.1.*'}, + 'X-Cache': 'miss', + 'Last-Modified': {'equal_re': '.*'}, + 'cache-control': 'max-age=1', + 'Content-Length': '3', + 'Date': {'equal_re': '.*'}, + 'Age': {'equal_re': '.*'} } cache_and_req_body_hit = { - 'Last-Modified' : {'equal_re' : '.*'}, - 'cache-control' : 'max-age=1', - 'Content-Length' : '3', - 'Date' : {'equal_re' : '.*'}, - 'Age' : {'equal_re' : '.*'}, - 'Connection' : 'keep-alive', - 'Via' : {'equal_re' : '.*'}, - 'Server' : {'equal_re' : '.*'}, - 'X-Cache' : 'hit-fresh', - 'HTTP/1.1 200 OK' : '' + 'Last-Modified': {'equal_re': '.*'}, + 'cache-control': 'max-age=1', + 'Content-Length': '3', + 'Date': {'equal_re': '.*'}, + 'Age': {'equal_re': '.*'}, + 'Connection': 'keep-alive', + 'Via': {'equal_re': '.*'}, + 'Server': {'equal_re': '.*'}, + 'X-Cache': 'hit-fresh', + 'HTTP/1.1 200 OK': '' } cache_and_req_body_hit_close = { - 'Last-Modified' : {'equal_re' : '.*'}, - 'cache-control' : 'max-age=1', - 'Content-Length' : '3', - 'Date' : {'equal_re' : '.*'}, - 'Age' : {'equal_re' : '.*'}, - 'Connection' : 'close', - 'Via' : {'equal_re' : '.*'}, - 'Server' : {'equal_re' : '.*'}, - 'X-Cache' : 'hit-fresh', - 'HTTP/1.1 200 OK' : '' + 'Last-Modified': {'equal_re': '.*'}, + 'cache-control': 'max-age=1', + 'Content-Length': '3', + 'Date': {'equal_re': '.*'}, + 'Age': {'equal_re': '.*'}, + 'Connection': 'close', + 'Via': {'equal_re': '.*'}, + 'Server': {'equal_re': '.*'}, + 'X-Cache': 'hit-fresh', + 'HTTP/1.1 200 OK': '' } # Test 1 - 200 response and cache fill tr = Test.AddTestRun() tr.Processes.Default.StartBefore(server) tr.Processes.Default.StartBefore(ts, ready=When.PortOpen(ts.Variables.port)) -tr.Processes.Default.Command = 'curl -s -D - -v --ipv4 --http1.1 -H "x-debug: x-cache,x-cache-key,via" -H "Host: www.example.com" http://localhost:{port}/'.format(port=ts.Variables.port) +tr.Processes.Default.Command = 'curl -s -D - -v --ipv4 --http1.1 -H "x-debug: x-cache,x-cache-key,via" -H "Host: www.example.com" http://localhost:{port}/'.format( + port=ts.Variables.port) tr.Processes.Default.ReturnCode = 0 tr.Processes.Default.Streams.stdout = Testers.CurlHeader(cache_and_req_body_miss) tr.StillRunningAfter = ts # Test 2 - 200 cached response and using netcat tr = Test.AddTestRun() -tr.Processes.Default.Command = "printf 'GET / HTTP/1.1\r\n''x-debug: x-cache,x-cache-key,via\r\n''Host: www.example.com\r\n''\r\n'|nc 127.0.0.1 -w 1 {port}".format(port=ts.Variables.port) +tr.Processes.Default.Command = "printf 'GET / HTTP/1.1\r\n''x-debug: x-cache,x-cache-key,via\r\n''Host: www.example.com\r\n''\r\n'|nc 127.0.0.1 -w 1 {port}".format( + port=ts.Variables.port) tr.Processes.Default.ReturnCode = 0 tr.Processes.Default.Streams.stdout = Testers.CurlHeader(cache_and_req_body_hit) tr.StillRunningAfter = ts # Test 3 - 200 cached response and trying to hide a request in the body tr = Test.AddTestRun() -tr.Processes.Default.Command = "printf 'GET / HTTP/1.1\r\n''x-debug: x-cache,x-cache-key,via\r\n''Host: www.example.com\r\n''Content-Length: 71\r\n''\r\n''GET /index.html?evil=zorg810 HTTP/1.1\r\n''Host: dummy-host.example.com\r\n''\r\n'|nc 127.0.0.1 -w 1 {port}".format(port=ts.Variables.port) +tr.Processes.Default.Command = "printf 'GET / HTTP/1.1\r\n''x-debug: x-cache,x-cache-key,via\r\n''Host: www.example.com\r\n''Content-Length: 71\r\n''\r\n''GET /index.html?evil=zorg810 HTTP/1.1\r\n''Host: dummy-host.example.com\r\n''\r\n'|nc 127.0.0.1 -w 1 {port}".format( + port=ts.Variables.port) tr.Processes.Default.ReturnCode = 0 tr.Processes.Default.Streams.stdout = Testers.CurlHeader(cache_and_req_body_hit) tr.StillRunningAfter = ts # Test 4 - 200 cached response and Content-Length larger than bytes sent, MUST close tr = Test.AddTestRun() -tr.Processes.Default.Command = "printf 'GET / HTTP/1.1\r\n''x-debug: x-cache,x-cache-key,via\r\n''Host: dummy-host.example.com\r\n''Cache-control: max-age=300\r\n''Content-Length: 100\r\n''\r\n''GET /index.html?evil=zorg810 HTTP/1.1\r\n''Host: dummy-host.example.com\r\n''\r\n'|nc 127.0.0.1 -w 1 {port}".format(port=ts.Variables.port) +tr.Processes.Default.Command = "printf 'GET / HTTP/1.1\r\n''x-debug: x-cache,x-cache-key,via\r\n''Host: dummy-host.example.com\r\n''Cache-control: max-age=300\r\n''Content-Length: 100\r\n''\r\n''GET /index.html?evil=zorg810 HTTP/1.1\r\n''Host: dummy-host.example.com\r\n''\r\n'|nc 127.0.0.1 -w 1 {port}".format( + port=ts.Variables.port) tr.Processes.Default.ReturnCode = 0 tr.Processes.Default.Streams.stdout = Testers.CurlHeader(cache_and_req_body_hit_close) tr.StillRunningAfter = ts diff --git a/tests/gold_tests/headers/cachedIMSRange.test.py b/tests/gold_tests/headers/cachedIMSRange.test.py index 496c945b572..a799d637bcb 100644 --- a/tests/gold_tests/headers/cachedIMSRange.test.py +++ b/tests/gold_tests/headers/cachedIMSRange.test.py @@ -17,8 +17,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import os -import time Test.Summary = ''' Test revalidating cached objects ''' @@ -29,33 +27,69 @@ # Set up Origin server # request_header is from ATS to origin; response from Origin to ATS # lookup_key is to make unique response in origin for header "UID" that will pass in ATS request -server = Test.MakeOriginServer("server",lookup_key="{%UID}") +server = Test.MakeOriginServer("server", lookup_key="{%UID}") # Initial request -request_header = {"headers": "GET / HTTP/1.1\r\nHost: www.example.com\r\nUID: Fill\r\n\r\n", "timestamp": "1469733493.993", "body": ""} -response_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\nLast-Modified: Tue, 08 May 2018 15:49:41 GMT\r\nCache-Control: max-age=1\r\n\r\n", "timestamp": "1469733493.993", "body": "xxx"} +request_header = { + "headers": "GET / HTTP/1.1\r\nHost: www.example.com\r\nUID: Fill\r\n\r\n", + "timestamp": "1469733493.993", + "body": ""} +response_header = { + "headers": "HTTP/1.1 200 OK\r\nConnection: close\r\nLast-Modified: Tue, 08 May 2018 15:49:41 GMT\r\nCache-Control: max-age=1\r\n\r\n", + "timestamp": "1469733493.993", + "body": "xxx"} server.addResponse("sessionlog.json", request_header, response_header) # IMS revalidation request -request_IMS_header = {"headers": "GET / HTTP/1.1\r\nUID: IMS\r\nIf-Modified-Since: Tue, 08 May 2018 15:49:41 GMT\r\nHost: www.example.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} -response_IMS_header = {"headers": "HTTP/1.1 304 Not Modified\r\nConnection: close\r\nCache-Control: max-age=1\r\n\r\n", "timestamp": "1469733493.993", "body": None} +request_IMS_header = { + "headers": "GET / HTTP/1.1\r\nUID: IMS\r\nIf-Modified-Since: Tue, 08 May 2018 15:49:41 GMT\r\nHost: www.example.com\r\n\r\n", + "timestamp": "1469733493.993", + "body": ""} +response_IMS_header = { + "headers": "HTTP/1.1 304 Not Modified\r\nConnection: close\r\nCache-Control: max-age=1\r\n\r\n", + "timestamp": "1469733493.993", + "body": None} server.addResponse("sessionlog.json", request_IMS_header, response_IMS_header) # EtagFill -request_etagfill_header = {"headers": "GET /etag HTTP/1.1\r\nHost: www.example.com\r\nUID: EtagFill\r\n\r\n", "timestamp": "1469733493.993", "body": None} -response_etagfill_header = {"headers": "HTTP/1.1 200 OK\r\nETag: myetag\r\nConnection: close\r\nCache-Control: max-age=1\r\n\r\n", "timestamp": "1469733493.993", "body": "xxx"} +request_etagfill_header = { + "headers": "GET /etag HTTP/1.1\r\nHost: www.example.com\r\nUID: EtagFill\r\n\r\n", + "timestamp": "1469733493.993", + "body": None} +response_etagfill_header = { + "headers": "HTTP/1.1 200 OK\r\nETag: myetag\r\nConnection: close\r\nCache-Control: max-age=1\r\n\r\n", + "timestamp": "1469733493.993", + "body": "xxx"} server.addResponse("sessionlog.json", request_etagfill_header, response_etagfill_header) # INM revalidation -request_INM_header = {"headers": "GET /etag HTTP/1.1\r\nUID: INM\r\nIf-None-Match: myetag\r\nHost: www.example.com\r\n\r\n", "timestamp": "1469733493.993", "body": None} -response_INM_header = {"headers": "HTTP/1.1 304 Not Modified\r\nConnection: close\r\nETag: myetag\r\nCache-Control: max-age=1\r\n\r\n", "timestamp": "1469733493.993", "body": None} +request_INM_header = { + "headers": "GET /etag HTTP/1.1\r\nUID: INM\r\nIf-None-Match: myetag\r\nHost: www.example.com\r\n\r\n", + "timestamp": "1469733493.993", + "body": None} +response_INM_header = { + "headers": "HTTP/1.1 304 Not Modified\r\nConnection: close\r\nETag: myetag\r\nCache-Control: max-age=1\r\n\r\n", + "timestamp": "1469733493.993", + "body": None} server.addResponse("sessionlog.json", request_INM_header, response_INM_header) # object changed to 0 byte -request_noBody_header = {"headers": "GET / HTTP/1.1\r\nUID: noBody\r\nHost: www.example.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} -response_noBody_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\nContent-Length: 0\r\nCache-Control: max-age=3\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +request_noBody_header = { + "headers": "GET / HTTP/1.1\r\nUID: noBody\r\nHost: www.example.com\r\n\r\n", + "timestamp": "1469733493.993", + "body": ""} +response_noBody_header = { + "headers": "HTTP/1.1 200 OK\r\nConnection: close\r\nContent-Length: 0\r\nCache-Control: max-age=3\r\n\r\n", + "timestamp": "1469733493.993", + "body": ""} server.addResponse("sessionlog.json", request_noBody_header, response_noBody_header) # etag object now is a 404. Yeah, 404s don't usually have Cache-Control, but, ATS's default is to cache 404s for a while. -request_etagfill_header = {"headers": "GET /etag HTTP/1.1\r\nHost: www.example.com\r\nUID: EtagError\r\n\r\n", "timestamp": "1469733493.993", "body": None} -response_etagfill_header = {"headers": "HTTP/1.1 404 Not Found\r\nConnection: close\r\nContent-Length: 0\r\nCache-Control: max-age=3\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +request_etagfill_header = { + "headers": "GET /etag HTTP/1.1\r\nHost: www.example.com\r\nUID: EtagError\r\n\r\n", + "timestamp": "1469733493.993", + "body": None} +response_etagfill_header = { + "headers": "HTTP/1.1 404 Not Found\r\nConnection: close\r\nContent-Length: 0\r\nCache-Control: max-age=3\r\n\r\n", + "timestamp": "1469733493.993", + "body": ""} server.addResponse("sessionlog.json", request_etagfill_header, response_etagfill_header) # ATS Configuration @@ -77,25 +111,32 @@ tr = Test.AddTestRun() tr.Processes.Default.StartBefore(server) tr.Processes.Default.StartBefore(ts, ready=When.PortOpen(ts.Variables.port)) -tr.Processes.Default.Command = 'curl -s -D - -v --ipv4 --http1.1 -H"UID: Fill" -H "x-debug: x-cache,x-cache-key,via" -H "Host: www.example.com" http://localhost:{0}/'.format(ts.Variables.port) +tr.Processes.Default.Command = 'curl -s -D - -v --ipv4 --http1.1 -H"UID: Fill" -H "x-debug: x-cache,x-cache-key,via" -H "Host: www.example.com" http://localhost:{0}/'.format( + ts.Variables.port) tr.Processes.Default.ReturnCode = 0 tr.Processes.Default.Streams.stdout = "cache_and_req_body-miss.gold" tr.StillRunningAfter = ts tr.StillRunningAfter = server -# Test 1 - Once it goes stale, fetch it again. We expect Origin to get IMS request, and serve a 304. We expect ATS to refresh the object, and give a 200 to user +# Test 1 - Once it goes stale, fetch it again. We expect Origin to get IMS +# request, and serve a 304. We expect ATS to refresh the object, and give +# a 200 to user tr = Test.AddTestRun() -tr.DelayStart=2 -tr.Processes.Default.Command = 'curl -s -D - -v --ipv4 --http1.1 -H"UID: IMS" -H "x-debug: x-cache,x-cache-key,via" -H "Host: www.example.com" http://localhost:{0}/'.format(ts.Variables.port) +tr.DelayStart = 2 +tr.Processes.Default.Command = 'curl -s -D - -v --ipv4 --http1.1 -H"UID: IMS" -H "x-debug: x-cache,x-cache-key,via" -H "Host: www.example.com" http://localhost:{0}/'.format( + ts.Variables.port) tr.Processes.Default.ReturnCode = 0 tr.Processes.Default.Streams.stdout = "cache_and_req_body-hit-stale.gold" tr.StillRunningAfter = ts tr.StillRunningAfter = server -# Test 2 - Once it goes stale, fetch it via a range request. We expect Origin to get IMS request, and serve a 304. We expect ATS to refresh the object, and give a 206 to user +# Test 2 - Once it goes stale, fetch it via a range request. We expect +# Origin to get IMS request, and serve a 304. We expect ATS to refresh the +# object, and give a 206 to user tr = Test.AddTestRun() -tr.DelayStart=2 -tr.Processes.Default.Command = 'curl --range 0-1 -s -D - -v --ipv4 --http1.1 -H"UID: IMS" -H "x-debug: x-cache,x-cache-key,via" -H "Host: www.example.com" http://localhost:{0}/'.format(ts.Variables.port) +tr.DelayStart = 2 +tr.Processes.Default.Command = 'curl --range 0-1 -s -D - -v --ipv4 --http1.1 -H"UID: IMS" -H "x-debug: x-cache,x-cache-key,via" -H "Host: www.example.com" http://localhost:{0}/'.format( + ts.Variables.port) tr.Processes.Default.ReturnCode = 0 tr.Processes.Default.Streams.stdout = "cache_and_req_body-hit-stale-206.gold" tr.StillRunningAfter = ts @@ -103,24 +144,31 @@ # Test 3 - Fill a new object with an Etag. Not checking the output here. tr = Test.AddTestRun() -tr.Processes.Default.Command = 'curl -s -D - -v --ipv4 --http1.1 -H"UID: EtagFill" -H "x-debug: x-cache,x-cache-key,via" -H "Host: www.example.com" http://localhost:{0}/etag'.format(ts.Variables.port) +tr.Processes.Default.Command = 'curl -s -D - -v --ipv4 --http1.1 -H"UID: EtagFill" -H "x-debug: x-cache,x-cache-key,via" -H "Host: www.example.com" http://localhost:{0}/etag'.format( + ts.Variables.port) tr.Processes.Default.ReturnCode = 0 tr.StillRunningAfter = ts tr.StillRunningAfter = server -# Test 4 - Once the etag object goes stale, fetch it again. We expect Origin to get INM request, and serve a 304. We expect ATS to refresh the object, and give a 200 to user +# Test 4 - Once the etag object goes stale, fetch it again. We expect +# Origin to get INM request, and serve a 304. We expect ATS to refresh the +# object, and give a 200 to user tr = Test.AddTestRun() -tr.DelayStart=2 -tr.Processes.Default.Command = 'curl -s -D - -v --ipv4 --http1.1 -H"UID: INM" -H "x-debug: x-cache,x-cache-key,via" -H "Host: www.example.com" http://localhost:{0}/etag'.format(ts.Variables.port) +tr.DelayStart = 2 +tr.Processes.Default.Command = 'curl -s -D - -v --ipv4 --http1.1 -H"UID: INM" -H "x-debug: x-cache,x-cache-key,via" -H "Host: www.example.com" http://localhost:{0}/etag'.format( + ts.Variables.port) tr.Processes.Default.ReturnCode = 0 tr.Processes.Default.Streams.stdout = "cache_and_req_body-hit-stale-INM.gold" tr.StillRunningAfter = ts tr.StillRunningAfter = server -# Test 5 - Once the etag object goes stale, fetch it via a range request. We expect Origin to get INM request, and serve a 304. We expect ATS to refresh the object, and give a 206 to user +# Test 5 - Once the etag object goes stale, fetch it via a range request. +# We expect Origin to get INM request, and serve a 304. We expect ATS to +# refresh the object, and give a 206 to user tr = Test.AddTestRun() -tr.DelayStart=2 -tr.Processes.Default.Command = 'curl --range 0-1 -s -D - -v --ipv4 --http1.1 -H"UID: INM" -H "x-debug: x-cache,x-cache-key,via" -H "Host: www.example.com" http://localhost:{0}/etag'.format(ts.Variables.port) +tr.DelayStart = 2 +tr.Processes.Default.Command = 'curl --range 0-1 -s -D - -v --ipv4 --http1.1 -H"UID: INM" -H "x-debug: x-cache,x-cache-key,via" -H "Host: www.example.com" http://localhost:{0}/etag'.format( + ts.Variables.port) tr.Processes.Default.ReturnCode = 0 tr.Processes.Default.Streams.stdout = "cache_and_req_body-hit-stale-206-etag.gold" tr.StillRunningAfter = ts @@ -128,8 +176,9 @@ # Test 6 - The origin changes the initial LMT object to 0 byte. We expect ATS to fetch and serve the new 0 byte object. tr = Test.AddTestRun() -tr.DelayStart=3 -tr.Processes.Default.Command = 'curl -s -D - -v --ipv4 --http1.1 -H"UID: noBody" -H "x-debug: x-cache,x-cache-key,via" -H "Host: www.example.com" http://localhost:{0}/'.format(ts.Variables.port) +tr.DelayStart = 3 +tr.Processes.Default.Command = 'curl -s -D - -v --ipv4 --http1.1 -H"UID: noBody" -H "x-debug: x-cache,x-cache-key,via" -H "Host: www.example.com" http://localhost:{0}/'.format( + ts.Variables.port) tr.Processes.Default.ReturnCode = 0 tr.Processes.Default.Streams.stdout = "cache_and_req_nobody-hit-stale.gold" tr.StillRunningAfter = ts @@ -137,8 +186,9 @@ # Test 7 - Fetch the new 0 byte object again when fresh in cache to ensure its still a 0 byte object. tr = Test.AddTestRun() -tr.DelayStart=3 -tr.Processes.Default.Command = 'curl -s -D - -v --ipv4 --http1.1 -H"UID: noBody" -H "x-debug: x-cache,x-cache-key,via" -H "Host: www.example.com" http://localhost:{0}/'.format(ts.Variables.port) +tr.DelayStart = 3 +tr.Processes.Default.Command = 'curl -s -D - -v --ipv4 --http1.1 -H"UID: noBody" -H "x-debug: x-cache,x-cache-key,via" -H "Host: www.example.com" http://localhost:{0}/'.format( + ts.Variables.port) tr.Processes.Default.ReturnCode = 0 tr.Processes.Default.Streams.stdout = "cache_and_req_nobody-hit-stale.gold" tr.StillRunningAfter = ts @@ -146,8 +196,9 @@ # Test 8 - The origin changes the etag object to 0 byte 404. We expect ATS to fetch and serve the 404 0 byte object. tr = Test.AddTestRun() -tr.DelayStart=2 -tr.Processes.Default.Command = 'curl -s -D - -v --ipv4 --http1.1 -H"UID: EtagError" -H "x-debug: x-cache,x-cache-key,via" -H "Host: www.example.com" http://localhost:{0}/etag'.format(ts.Variables.port) +tr.DelayStart = 2 +tr.Processes.Default.Command = 'curl -s -D - -v --ipv4 --http1.1 -H"UID: EtagError" -H "x-debug: x-cache,x-cache-key,via" -H "Host: www.example.com" http://localhost:{0}/etag'.format( + ts.Variables.port) tr.Processes.Default.ReturnCode = 0 tr.Processes.Default.Streams.stdout = "cache_and_error_nobody.gold" tr.StillRunningAfter = ts @@ -155,8 +206,9 @@ # Test 9 - Fetch the 0 byte etag object again when fresh in cache to ensure its still a 0 byte object tr = Test.AddTestRun() -tr.DelayStart=2 -tr.Processes.Default.Command = 'curl -s -D - -v --ipv4 --http1.1 -H"UID: EtagError" -H "x-debug: x-cache,x-cache-key,via" -H "Host: www.example.com" http://localhost:{0}/etag'.format(ts.Variables.port) +tr.DelayStart = 2 +tr.Processes.Default.Command = 'curl -s -D - -v --ipv4 --http1.1 -H"UID: EtagError" -H "x-debug: x-cache,x-cache-key,via" -H "Host: www.example.com" http://localhost:{0}/etag'.format( + ts.Variables.port) tr.Processes.Default.ReturnCode = 0 tr.Processes.Default.Streams.stdout = "cache_and_error_nobody.gold" tr.StillRunningAfter = ts diff --git a/tests/gold_tests/headers/data/www.passthrough.test_get.txt b/tests/gold_tests/headers/data/www.passthrough.test_get.txt index 0cba742da68..ffe69c2afd9 100644 --- a/tests/gold_tests/headers/data/www.passthrough.test_get.txt +++ b/tests/gold_tests/headers/data/www.passthrough.test_get.txt @@ -1,2 +1,2 @@ -GET http://www.passthrough.test/ HTTP/1.1 - +GET http://www.passthrough.test/ HTTP/1.1 + diff --git a/tests/gold_tests/headers/data/www.redirect0.test_get.txt b/tests/gold_tests/headers/data/www.redirect0.test_get.txt index 40fce98c217..3d3d219e385 100644 --- a/tests/gold_tests/headers/data/www.redirect0.test_get.txt +++ b/tests/gold_tests/headers/data/www.redirect0.test_get.txt @@ -1,2 +1,2 @@ -GET http://www.redirect0.test/ HTTP/1.1 - +GET http://www.redirect0.test/ HTTP/1.1 + diff --git a/tests/gold_tests/headers/data/www.redirect301.test_get.txt b/tests/gold_tests/headers/data/www.redirect301.test_get.txt index 48352678c17..e51c353dddc 100644 --- a/tests/gold_tests/headers/data/www.redirect301.test_get.txt +++ b/tests/gold_tests/headers/data/www.redirect301.test_get.txt @@ -1,2 +1,2 @@ -GET http://www.redirect301.test/ HTTP/1.1 - +GET http://www.redirect301.test/ HTTP/1.1 + diff --git a/tests/gold_tests/headers/data/www.redirect302.test_get.txt b/tests/gold_tests/headers/data/www.redirect302.test_get.txt index 6aae2667c6d..35f77503149 100644 --- a/tests/gold_tests/headers/data/www.redirect302.test_get.txt +++ b/tests/gold_tests/headers/data/www.redirect302.test_get.txt @@ -1,2 +1,2 @@ -GET http://www.redirect302.test/ HTTP/1.1 - +GET http://www.redirect302.test/ HTTP/1.1 + diff --git a/tests/gold_tests/headers/data/www.redirect307.test_get.txt b/tests/gold_tests/headers/data/www.redirect307.test_get.txt index b37c8ae1486..49bb1bc3072 100644 --- a/tests/gold_tests/headers/data/www.redirect307.test_get.txt +++ b/tests/gold_tests/headers/data/www.redirect307.test_get.txt @@ -1,2 +1,2 @@ -GET http://www.redirect307.test/ HTTP/1.1 - +GET http://www.redirect307.test/ HTTP/1.1 + diff --git a/tests/gold_tests/headers/data/www.redirect308.test_get.txt b/tests/gold_tests/headers/data/www.redirect308.test_get.txt index 05bcbf8ec78..097e95eb126 100644 --- a/tests/gold_tests/headers/data/www.redirect308.test_get.txt +++ b/tests/gold_tests/headers/data/www.redirect308.test_get.txt @@ -1,2 +1,2 @@ -GET http://www.redirect308.test/ HTTP/1.1 - +GET http://www.redirect308.test/ HTTP/1.1 + diff --git a/tests/gold_tests/headers/domain-blacklist-30x.test.py b/tests/gold_tests/headers/domain-blacklist-30x.test.py index dfec70547a0..275c3425890 100644 --- a/tests/gold_tests/headers/domain-blacklist-30x.test.py +++ b/tests/gold_tests/headers/domain-blacklist-30x.test.py @@ -59,7 +59,7 @@ redirect301tr = Test.AddTestRun("Test domain {0}".format(REDIRECT_301_HOST)) redirect301tr.Processes.Default.StartBefore(Test.Processes.ts) redirect301tr.StillRunningAfter = ts -redirect301tr.Processes.Default.Command = "python tcp_client.py 127.0.0.1 {0} {1} | grep -v '^Date: '| grep -v '^Server: ATS/'".\ +redirect301tr.Processes.Default.Command = "python3 tcp_client.py 127.0.0.1 {0} {1} | grep -v '^Date: '| grep -v '^Server: ATS/'".\ format(ts.Variables.port, 'data/{0}_get.txt'.format(REDIRECT_301_HOST)) redirect301tr.Processes.Default.TimeOut = 5 # seconds redirect301tr.Processes.Default.ReturnCode = 0 @@ -68,7 +68,7 @@ redirect302tr = Test.AddTestRun("Test domain {0}".format(REDIRECT_302_HOST)) redirect302tr.StillRunningBefore = ts redirect302tr.StillRunningAfter = ts -redirect302tr.Processes.Default.Command = "python tcp_client.py 127.0.0.1 {0} {1} | grep -v '^Date: '| grep -v '^Server: ATS/'".\ +redirect302tr.Processes.Default.Command = "python3 tcp_client.py 127.0.0.1 {0} {1} | grep -v '^Date: '| grep -v '^Server: ATS/'".\ format(ts.Variables.port, 'data/{0}_get.txt'.format(REDIRECT_302_HOST)) redirect302tr.Processes.Default.TimeOut = 5 # seconds redirect302tr.Processes.Default.ReturnCode = 0 @@ -78,7 +78,7 @@ redirect307tr = Test.AddTestRun("Test domain {0}".format(REDIRECT_307_HOST)) redirect302tr.StillRunningBefore = ts redirect307tr.StillRunningAfter = ts -redirect307tr.Processes.Default.Command = "python tcp_client.py 127.0.0.1 {0} {1} | grep -v '^Date: '| grep -v '^Server: ATS/'".\ +redirect307tr.Processes.Default.Command = "python3 tcp_client.py 127.0.0.1 {0} {1} | grep -v '^Date: '| grep -v '^Server: ATS/'".\ format(ts.Variables.port, 'data/{0}_get.txt'.format(REDIRECT_307_HOST)) redirect307tr.Processes.Default.TimeOut = 5 # seconds redirect307tr.Processes.Default.ReturnCode = 0 @@ -87,7 +87,7 @@ redirect308tr = Test.AddTestRun("Test domain {0}".format(REDIRECT_308_HOST)) redirect308tr.StillRunningBefore = ts redirect308tr.StillRunningAfter = ts -redirect308tr.Processes.Default.Command = "python tcp_client.py 127.0.0.1 {0} {1} | grep -v '^Date: '| grep -v '^Server: ATS/'".\ +redirect308tr.Processes.Default.Command = "python3 tcp_client.py 127.0.0.1 {0} {1} | grep -v '^Date: '| grep -v '^Server: ATS/'".\ format(ts.Variables.port, 'data/{0}_get.txt'.format(REDIRECT_308_HOST)) redirect308tr.Processes.Default.TimeOut = 5 # seconds redirect308tr.Processes.Default.ReturnCode = 0 @@ -96,7 +96,7 @@ redirect0tr = Test.AddTestRun("Test domain {0}".format(REDIRECT_0_HOST)) redirect0tr.StillRunningBefore = ts redirect0tr.StillRunningAfter = ts -redirect0tr.Processes.Default.Command = "python tcp_client.py 127.0.0.1 {0} {1} | grep -v '^Date: '| grep -v '^Server: ATS/'".\ +redirect0tr.Processes.Default.Command = "python3 tcp_client.py 127.0.0.1 {0} {1} | grep -v '^Date: '| grep -v '^Server: ATS/'".\ format(ts.Variables.port, 'data/{0}_get.txt'.format(REDIRECT_0_HOST)) redirect0tr.Processes.Default.TimeOut = 5 # seconds redirect0tr.Processes.Default.ReturnCode = 0 @@ -105,7 +105,7 @@ passthroughtr = Test.AddTestRun("Test domain {0}".format(PASSTHRU_HOST)) passthroughtr.StillRunningBefore = ts passthroughtr.StillRunningAfter = ts -passthroughtr.Processes.Default.Command = "python tcp_client.py 127.0.0.1 {0} {1} | grep -v '^Date: '| grep -v '^Server: ATS/'".\ +passthroughtr.Processes.Default.Command = "python3 tcp_client.py 127.0.0.1 {0} {1} | grep -v '^Date: '| grep -v '^Server: ATS/'".\ format(ts.Variables.port, 'data/{0}_get.txt'.format(PASSTHRU_HOST)) passthroughtr.Processes.Default.TimeOut = 5 # seconds passthroughtr.Processes.Default.ReturnCode = 0 diff --git a/tests/gold_tests/headers/field_name_space.test.py b/tests/gold_tests/headers/field_name_space.test.py new file mode 100644 index 00000000000..559ba17b6a4 --- /dev/null +++ b/tests/gold_tests/headers/field_name_space.test.py @@ -0,0 +1,53 @@ +''' +Test on handeling spaces after the field name and before the colon +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +Test.Summary = ''' +Checking on handeling spaces after the field name and before the colon +''' + +Test.ContinueOnFail = True + +# Define default ATS +ts = Test.MakeATSProcess("ts") +server = Test.MakeOriginServer("server") + +testName = "field_name_space" +request_header = { + "headers": "GET / HTTP/1.1\r\nHost: www.example.com\r\n\r\n", + "timestamp": "1469733493.993", + "body": ""} +response_header = { + "headers": "HTTP/1.1 200 OK\r\nConnection: close\r\nFoo : 123\r\nFoo: 456\r\n", + "timestamp": "1469733493.993", + "body": "xxx"} +server.addResponse("sessionlog.json", request_header, response_header) + +ts.Disk.remap_config.AddLine( + 'map http://www.example.com http://127.0.0.1:{0}'.format(server.Variables.Port) +) + +# Test spaces at the end of the field name and before the : +tr = Test.AddTestRun() +tr.Processes.Default.StartBefore(server, ready=When.PortOpen(server.Variables.Port)) +tr.Processes.Default.StartBefore(Test.Processes.ts) +tr.Processes.Default.Command = 'curl -s -D - -v --ipv4 --http1.1 -H "Host: www.example.com" http://localhost:{0}/'.format( + ts.Variables.port) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Streams.stderr = "gold/field_name_space.gold" +tr.StillRunningAfter = ts diff --git a/tests/gold_tests/headers/forwarded-observer.py b/tests/gold_tests/headers/forwarded-observer.py index f817650775a..4e5cbf992b8 100644 --- a/tests/gold_tests/headers/forwarded-observer.py +++ b/tests/gold_tests/headers/forwarded-observer.py @@ -18,7 +18,6 @@ # limitations under the License. import re -import subprocess log = open('forwarded.log', 'w') @@ -27,6 +26,7 @@ byCount = 0 byEqualUuid = "__INVALID__" + def observe(headers): global byCount diff --git a/tests/gold_tests/headers/forwarded.test.py b/tests/gold_tests/headers/forwarded.test.py index eb95e382b4b..5fed2056d67 100644 --- a/tests/gold_tests/headers/forwarded.test.py +++ b/tests/gold_tests/headers/forwarded.test.py @@ -18,7 +18,6 @@ # limitations under the License. import os -import subprocess Test.Summary = ''' Test FORWARDED header. diff --git a/tests/gold_tests/headers/general-connection-failure-502.gold b/tests/gold_tests/headers/general-connection-failure-502.gold index 4749e20f621..a5e27325fa5 100644 --- a/tests/gold_tests/headers/general-connection-failure-502.gold +++ b/tests/gold_tests/headers/general-connection-failure-502.gold @@ -1,10 +1,10 @@ -HTTP/1.1 502 connect failed -Connection: keep-alive -Cache-Control: no-store -Content-Type: text/html -Content-Language: en -Content-Length: 247 - +HTTP/1.1 502 connect failed +Connection: keep-alive +Cache-Control: no-store +Content-Type: text/html +Content-Language: en +Content-Length: 247 + Could Not Connect diff --git a/tests/gold_tests/headers/general-connection-failure-502.test.py b/tests/gold_tests/headers/general-connection-failure-502.test.py index 7054ce95ae8..4cb06c3b449 100644 --- a/tests/gold_tests/headers/general-connection-failure-502.test.py +++ b/tests/gold_tests/headers/general-connection-failure-502.test.py @@ -25,21 +25,21 @@ ts = Test.MakeATSProcess("ts") HOST = 'www.connectfail502.test' -server = Test.MakeOriginServer("server", ssl=False) # Reserves a port across autest. +server = Test.MakeOriginServer("server", ssl=False) # Reserves a port across autest. ts.Disk.remap_config.AddLine( - 'map http://{host} http://{ip}:{uport}'.format(host=HOST, ip='127.0.0.1', uport=server.Variables.Port) + 'map http://{host} http://{ip}:{uport}'.format(host=HOST, ip='127.0.0.1', uport=server.Variables.Port) ) Test.Setup.Copy(os.path.join(Test.Variables.AtsTestToolsDir, 'tcp_client.py')) -data_file=Test.Disk.File("www.connectfail502.test-get.txt", id="datafile") +data_file = Test.Disk.File("www.connectfail502.test-get.txt", id="datafile") data_file.WriteOn("GET / HTTP/1.1\r\nHost: {host}\r\n\r\n".format(host=HOST)) tr = Test.AddTestRun() tr.Processes.Default.StartBefore(Test.Processes.ts) # Do not start the origin server: We wish to simulate connection refused while hopefully no one else uses this port. -tr.Processes.Default.Command = "python tcp_client.py 127.0.0.1 {0} {1} | sed -e '/^Date: /d' -e '/^Server: ATS\//d'"\ - .format(ts.Variables.port, "www.connectfail502.test-get.txt") +tr.Processes.Default.Command = "python3 tcp_client.py 127.0.0.1 {0} {1} | sed -e '/^Date: /d' -e '/^Server: ATS\//d'"\ + .format(ts.Variables.port, "www.connectfail502.test-get.txt") tr.Processes.Default.ReturnCode = 0 tr.Processes.Default.Streams.stdout = 'general-connection-failure-502.gold' diff --git a/tests/gold_tests/headers/gold/accept_webp.gold b/tests/gold_tests/headers/gold/accept_webp.gold new file mode 100644 index 00000000000..9b6dcb47872 --- /dev/null +++ b/tests/gold_tests/headers/gold/accept_webp.gold @@ -0,0 +1,16 @@ +`` +> GET /`` +> Host: www.example.com`` +> User-Agent: curl/`` +> Accept: image/webp,image/png,image/svg+xml,image/*;q=0.8,video/*;q=0.8,*/*;q=0.5 +`` +< HTTP/1.1 200 OK +< Content-Type: image/webp +< Cache-Control: max-age=300 +< Content-Length: 3 +< Date: `` +< Age: `` +< Connection: keep-alive +< Via: http/1.1 `` (ApacheTrafficServer/`` [uScMsSfWpSeN:t cCMp sS]) +< Server: ATS/`` +`` diff --git a/tests/gold_tests/headers/gold/accept_webp_cache.gold b/tests/gold_tests/headers/gold/accept_webp_cache.gold new file mode 100644 index 00000000000..5e2cf841d0e --- /dev/null +++ b/tests/gold_tests/headers/gold/accept_webp_cache.gold @@ -0,0 +1,16 @@ +`` +> GET /`` +> Host: www.example.com`` +> User-Agent: curl/`` +> Accept: image/webp,image/png,image/svg+xml,image/*;q=0.8,video/*;q=0.8,*/*;q=0.5 +`` +< HTTP/1.1 200 OK +< Content-Type: image/webp +< Cache-Control: max-age=300 +< Content-Length: 3 +< Date: `` +< Age: `` +< Connection: keep-alive +< Via: http/1.1 `` (ApacheTrafficServer/`` [uScHs f p eN:t cCHp s ]) +< Server: ATS/`` +`` diff --git a/tests/gold_tests/headers/gold/accept_webp_jpeg.gold b/tests/gold_tests/headers/gold/accept_webp_jpeg.gold new file mode 100644 index 00000000000..1e76ccc3165 --- /dev/null +++ b/tests/gold_tests/headers/gold/accept_webp_jpeg.gold @@ -0,0 +1,16 @@ +`` +> GET /`` +> Host: www.example.com`` +> User-Agent: curl/`` +> Accept: image/png,image/svg+xml,image/*;q=0.8,video/*;q=0.8,*/*;q=0.5 +`` +< HTTP/1.1 200 OK +< Content-Type: image/webp +< Cache-Control: max-age=300 +< Content-Length: 3 +< Date: `` +< Age: `` +< Connection: keep-alive +< Via: http/1.1 `` (ApacheTrafficServer/`` [uScMsSfWpSeN:t cCMp sS]) +< Server: ATS/`` +`` diff --git a/tests/gold_tests/headers/gold/field_name_space.gold b/tests/gold_tests/headers/gold/field_name_space.gold new file mode 100644 index 00000000000..bf711789b70 --- /dev/null +++ b/tests/gold_tests/headers/gold/field_name_space.gold @@ -0,0 +1,14 @@ +`` +> GET /`` +> Host: www.example.com`` +> User-Agent: curl/`` +`` +< HTTP/1.1 200 OK +< Foo: 123 +< Foo: 456 +< Content-Length: 3 +< Date: `` +< Age: `` +< Connection: keep-alive +< Server: ATS/`` +`` diff --git a/tests/gold_tests/headers/hsts.test.py b/tests/gold_tests/headers/hsts.test.py index 434ff4b1d42..f8b7292e98e 100644 --- a/tests/gold_tests/headers/hsts.test.py +++ b/tests/gold_tests/headers/hsts.test.py @@ -17,7 +17,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import os Test.Summary = ''' heck hsts header is set correctly ''' @@ -28,7 +27,7 @@ ts = Test.MakeATSProcess("ts", select_ports=True, enable_tls=True) server = Test.MakeOriginServer("server") -#**testname is required** +# **testname is required** testName = "" request_header = {"headers": "GET / HTTP/1.1\r\nHost: www.example.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} response_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": ""} diff --git a/tests/gold_tests/headers/http408.test.py b/tests/gold_tests/headers/http408.test.py index 8e2c37b4303..e3c2d9cd697 100644 --- a/tests/gold_tests/headers/http408.test.py +++ b/tests/gold_tests/headers/http408.test.py @@ -18,7 +18,6 @@ # limitations under the License. import os -import subprocess Test.Summary = ''' Check 408 response header for protocol stack data. @@ -40,7 +39,7 @@ 'map http://{0} http://127.0.0.1:{1}'.format(HTTP_408_HOST, server.Variables.Port) ) -TIMEOUT=2 +TIMEOUT = 2 ts.Disk.records_config.update({ 'proxy.config.http.transaction_no_activity_timeout_in': TIMEOUT, }) @@ -51,8 +50,8 @@ tr = Test.AddTestRun() tr.Processes.Default.StartBefore(server) tr.Processes.Default.StartBefore(Test.Processes.ts) -tr.Processes.Default.Command = 'python tcp_client.py 127.0.0.1 {0} {1} --delay-after-send {2}'\ - .format(ts.Variables.port, 'data/{0}.txt'.format(HTTP_408_HOST), TIMEOUT + 2) +tr.Processes.Default.Command = 'python3 tcp_client.py 127.0.0.1 {0} {1} --delay-after-send {2}'\ + .format(ts.Variables.port, 'data/{0}.txt'.format(HTTP_408_HOST), TIMEOUT + 2) tr.Processes.Default.ReturnCode = 0 tr.Processes.Default.TimeOut = 10 tr.Processes.Default.Streams.stdout = "http408.gold" diff --git a/tests/gold_tests/headers/normalize_ae.test.py b/tests/gold_tests/headers/normalize_ae.test.py index 23ef42f357a..d9779a2d49a 100644 --- a/tests/gold_tests/headers/normalize_ae.test.py +++ b/tests/gold_tests/headers/normalize_ae.test.py @@ -18,7 +18,6 @@ # limitations under the License. import os -import subprocess Test.Summary = ''' Test normalizations of the Accept-Encoding header field. diff --git a/tests/gold_tests/headers/normalize_ae_observer.py b/tests/gold_tests/headers/normalize_ae_observer.py index 10de1d48e13..49947766de9 100644 --- a/tests/gold_tests/headers/normalize_ae_observer.py +++ b/tests/gold_tests/headers/normalize_ae_observer.py @@ -19,6 +19,7 @@ log = open('normalize_ae.log', 'w') + def observe(headers): seen = False @@ -36,4 +37,5 @@ def observe(headers): log.write("-\n") log.flush() + Hooks.register(Hooks.ReadRequestHook, observe) diff --git a/tests/gold_tests/headers/syntax.test.py b/tests/gold_tests/headers/syntax.test.py index c10ab02e629..262b8358561 100644 --- a/tests/gold_tests/headers/syntax.test.py +++ b/tests/gold_tests/headers/syntax.test.py @@ -17,7 +17,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import os Test.Summary = ''' Test whitespace between field name and colon in the header ''' @@ -28,7 +27,7 @@ ts = Test.MakeATSProcess("ts") server = Test.MakeOriginServer("server") -#**testname is required** +# **testname is required** testName = "" request_header = {"headers": "GET / HTTP/1.1\r\nHost: www.example.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} response_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": ""} @@ -38,46 +37,52 @@ 'map http://www.example.com http://127.0.0.1:{0}'.format(server.Variables.Port) ) -# Test 1 - 200 Response +# Test 0 - 200 Response tr = Test.AddTestRun() -tr.Processes.Default.StartBefore(server) -tr.Processes.Default.StartBefore(Test.Processes.ts, ready=When.PortOpen(server.Variables.Port)) -tr.Processes.Default.Command = 'curl -s -D - -v --ipv4 --http1.1 -H " foo: bar" -H "Host: www.example.com" http://localhost:{0}/'.format(ts.Variables.port) +tr.Processes.Default.StartBefore(server, ready=When.PortOpen(server.Variables.Port)) +tr.Processes.Default.StartBefore(Test.Processes.ts, ready=When.PortOpen(ts.Variables.port)) +tr.Processes.Default.Command = 'curl -s -D - -v --ipv4 --http1.1 -H " foo: bar" -H "Host: www.example.com" http://localhost:{0}/'.format( + ts.Variables.port) tr.Processes.Default.ReturnCode = 0 tr.Processes.Default.Streams.stdout = "syntax.200.gold" tr.StillRunningAfter = ts -# Test 2 - 400 Response - Single space after field name +# Test 1 - 400 Response - Single space after field name tr = Test.AddTestRun() -tr.Processes.Default.Command = 'curl -s -D - -v --ipv4 --http1.1 -H "foo : bar" -H "Host: www.example.com" http://localhost:{0}/'.format(ts.Variables.port) +tr.Processes.Default.Command = 'curl -s -D - -v --ipv4 --http1.1 -H "foo : bar" -H "Host: www.example.com" http://localhost:{0}/'.format( + ts.Variables.port) tr.Processes.Default.ReturnCode = 0 tr.Processes.Default.Streams.stdout = "syntax.400.gold" tr.StillRunningAfter = ts -# Test 3 - 400 Response - Double space after field name +# Test 2 - 400 Response - Double space after field name tr = Test.AddTestRun() -tr.Processes.Default.Command = 'curl -s -D - -v --ipv4 --http1.1 -H "foo : bar" -H "Host: www.example.com" http://localhost:{0}/'.format(ts.Variables.port) +tr.Processes.Default.Command = 'curl -s -D - -v --ipv4 --http1.1 -H "foo : bar" -H "Host: www.example.com" http://localhost:{0}/'.format( + ts.Variables.port) tr.Processes.Default.ReturnCode = 0 tr.Processes.Default.Streams.stdout = "syntax.400.gold" tr.StillRunningAfter = ts -# Test 4 - 400 Response - Three different Content-Length headers +# Test 3 - 400 Response - Three different Content-Length headers tr = Test.AddTestRun() -tr.Processes.Default.Command = 'curl -s -D - -v --ipv4 --http1.1 -d "hello world" -H "Content-Length: 11" -H "Content-Length: 10" -H "Content-Length: 9" -H "Host: www.example.com" http://localhost:{0}/'.format(ts.Variables.port) +tr.Processes.Default.Command = 'curl -s -D - -v --ipv4 --http1.1 -d "hello world" -H "Content-Length: 11" -H "Content-Length: 10" -H "Content-Length: 9" -H "Host: www.example.com" http://localhost:{0}/'.format( + ts.Variables.port) tr.Processes.Default.ReturnCode = 0 tr.Processes.Default.Streams.stdout = "syntax.400.gold" tr.StillRunningAfter = ts # Test 4 - 200 Response - Three same Content-Length headers tr = Test.AddTestRun() -tr.Processes.Default.Command = 'curl -s -D - -v --ipv4 --http1.1 -d "hello world" -H "Content-Length: 11" -H "Content-Length: 11" -H "Content-Length: 11" -H "Host: www.example.com" http://localhost:{0}/'.format(ts.Variables.port) +tr.Processes.Default.Command = 'curl -s -D - -v --ipv4 --http1.1 -d "hello world" -H "Content-Length: 11" -H "Content-Length: 11" -H "Content-Length: 11" -H "Host: www.example.com" http://localhost:{0}/'.format( + ts.Variables.port) tr.Processes.Default.ReturnCode = 0 tr.Processes.Default.Streams.stdout = "syntax.200.gold" tr.StillRunningAfter = ts -# Test 4 - 200 Response - Three different Content-Length headers with a Transfer ecoding header +# Test 5 - 200 Response - Three different Content-Length headers with a Transfer encoding header tr = Test.AddTestRun() -tr.Processes.Default.Command = 'curl -s -D - -v --ipv4 --http1.1 -d "hello world" -H "Transfer-Encoding: chunked" -H "Content-Length: 11" -H "Content-Length: 10" -H "Content-Length: 9" -H "Host: www.example.com" http://localhost:{0}/'.format(ts.Variables.port) +tr.Processes.Default.Command = 'curl -s -D - -v --ipv4 --http1.1 -d "hello world" -H "Transfer-Encoding: chunked" -H "Content-Length: 11" -H "Content-Length: 10" -H "Content-Length: 9" -H "Host: www.example.com" http://localhost:{0}/'.format( + ts.Variables.port) tr.Processes.Default.ReturnCode = 0 tr.Processes.Default.Streams.stdout = "syntax.200.gold" tr.StillRunningAfter = ts diff --git a/tests/gold_tests/headers/via.test.py b/tests/gold_tests/headers/via.test.py index 1244b489647..7fc07bc25e1 100644 --- a/tests/gold_tests/headers/via.test.py +++ b/tests/gold_tests/headers/via.test.py @@ -19,7 +19,6 @@ # limitations under the License. import os -import subprocess Test.Summary = ''' Check VIA header for protocol stack data. @@ -47,13 +46,18 @@ ts.addSSLfile("../remap/ssl/server.key") ts.Variables.ssl_port = 4443 -ts.Disk.records_config.update({ - 'proxy.config.http.insert_request_via_str': 4, - 'proxy.config.http.insert_response_via_str': 4, - 'proxy.config.ssl.server.cert.path': '{0}'.format(ts.Variables.SSLDir), - 'proxy.config.ssl.server.private_key.path': '{0}'.format(ts.Variables.SSLDir), - 'proxy.config.http.server_ports': 'ipv4:{0} ipv4:{1}:proto=http2;http:ssl ipv6:{0} ipv6:{1}:proto=http2;http:ssl'.format(ts.Variables.port, ts.Variables.ssl_port), -}) +ts.Disk.records_config.update( + { + 'proxy.config.http.insert_request_via_str': 4, + 'proxy.config.http.insert_response_via_str': 4, + 'proxy.config.ssl.server.cert.path': '{0}'.format( + ts.Variables.SSLDir), + 'proxy.config.ssl.server.private_key.path': '{0}'.format( + ts.Variables.SSLDir), + 'proxy.config.http.server_ports': 'ipv4:{0} ipv4:{1}:proto=http2;http:ssl ipv6:{0} ipv6:{1}:proto=http2;http:ssl'.format( + ts.Variables.port, + ts.Variables.ssl_port), + }) ts.Disk.remap_config.AddLine( 'map http://www.example.com http://127.0.0.1:{0}'.format(server.Variables.Port) diff --git a/tests/gold_tests/ip_allow/gold/200.gold b/tests/gold_tests/ip_allow/gold/200.gold new file mode 100644 index 00000000000..e6da7944111 --- /dev/null +++ b/tests/gold_tests/ip_allow/gold/200.gold @@ -0,0 +1,5 @@ +`` +> GET /get HTTP/1.1 +`` +< HTTP/1.1 200 OK +`` diff --git a/tests/gold_tests/ip_allow/gold/403.gold b/tests/gold_tests/ip_allow/gold/403.gold new file mode 100644 index 00000000000..d2164abee0e --- /dev/null +++ b/tests/gold_tests/ip_allow/gold/403.gold @@ -0,0 +1,5 @@ +`` +> CONNECT /connect HTTP/1.1 +`` +< HTTP/1.1 403 Access Denied +`` diff --git a/tests/gold_tests/ip_allow/gold/403_h2.gold b/tests/gold_tests/ip_allow/gold/403_h2.gold new file mode 100644 index 00000000000..af82fe2f7e2 --- /dev/null +++ b/tests/gold_tests/ip_allow/gold/403_h2.gold @@ -0,0 +1,5 @@ +`` +> PUSH /h2_push HTTP/2 +`` +< HTTP/2 403`` +`` diff --git a/tests/gold_tests/ip_allow/ip_allow.test.py b/tests/gold_tests/ip_allow/ip_allow.test.py new file mode 100644 index 00000000000..32c0261c3b4 --- /dev/null +++ b/tests/gold_tests/ip_allow/ip_allow.test.py @@ -0,0 +1,179 @@ +''' +Verify ip_allow filtering behavior. +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +Test.Summary = ''' +Verify ip_allow filtering behavior. +''' + +Test.ContinueOnFail = True + +# Define default ATS +ts = Test.MakeATSProcess("ts", command="traffic_manager", select_ports=True, enable_tls=True) +server = Test.MakeOriginServer("server", ssl=True) + +testName = "" +request = { + "headers": + "GET /get HTTP/1.1\r\n" + "Host: www.example.com:80\r\n\r\n", + "timestamp": "1469733493.993", + "body": ""} +response = { + "headers": + "HTTP/1.1 200 OK\r\n" + "Content-Length: 3\r\n" + "Connection: close\r\n\r\n", + "timestamp": + "1469733493.993", "body": "xxx"} +server.addResponse("sessionlog.json", request, response) + +# The following shouldn't come to the server, but in the event that there is a +# bug in ip_allow and they are sent through, have them return a 200 OK. This +# will fail the match with the gold file which expects a 403. +request = { + "headers": + "CONNECT www.example.com:80/connect HTTP/1.1\r\n" + "Host: www.example.com:80\r\n\r\n", + "timestamp": "1469733493.993", + "body": ""} +response = { + "headers": + "HTTP/1.1 200 OK\r\n" + "Content-Length: 3\r\n" + "Connection: close\r\n\r\n", + "timestamp": + "1469733493.993", "body": "xxx"} +server.addResponse("sessionlog.json", request, response) +request = { + "headers": + "PUSH www.example.com:80/h2_push HTTP/2\r\n" + "Host: www.example.com:80\r\n\r\n", + "timestamp": "1469733493.993", + "body": ""} +response = { + "headers": + "HTTP/2 200 OK\r\n" + "Content-Length: 3\r\n" + "Connection: close\r\n\r\n", + "timestamp": + "1469733493.993", "body": "xxx"} +server.addResponse("sessionlog.json", request, response) + +# Configure TLS for Traffic Server for HTTP/2. +ts.addSSLfile("ssl/server.pem") +ts.addSSLfile("ssl/server.key") + +ts.Disk.ssl_multicert_config.AddLine( + 'dest_ip=* ssl_cert_name=server.pem ssl_key_name=server.key' +) + +ts.Disk.records_config.update({ + 'proxy.config.diags.debug.enabled': 1, + 'proxy.config.diags.debug.tags': 'ip-allow', + 'proxy.config.http.connect_ports': '{0}'.format(server.Variables.SSL_Port), + 'proxy.config.ssl.server.cert.path': '{0}'.format(ts.Variables.SSLDir), + 'proxy.config.ssl.server.private_key.path': '{0}'.format(ts.Variables.SSLDir), + 'proxy.config.http.cache.http': 0, + 'proxy.config.ssl.client.verify.server': 0, + 'proxy.config.ssl.server.cipher_suite': 'ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA384:AES128-GCM-SHA256:AES256-GCM-SHA384:ECDHE-RSA-RC4-SHA:ECDHE-RSA-AES128-SHA:ECDHE-RSA-AES256-SHA:RC4-SHA:RC4-MD5:AES128-SHA:AES256-SHA:DES-CBC3-SHA!SRP:!DSS:!PSK:!aNULL:!eNULL:!SSLv2', + 'proxy.config.http2.active_timeout_in': 3, + 'proxy.config.http2.max_concurrent_streams_in': 65535, +}) + +format_string = ('%-% % % % %/% % ' + '% % %/% % %<{Y-RID}pqh> ' + '%<{Y-YPCS}pqh> %<{Host}cqh> %<{CHAD}pqh> ' + 'sftover=%<{x-safet-overlimit-rules}cqh> sftmat=%<{x-safet-matched-rules}cqh> ' + 'sftcls=%<{x-safet-classification}cqh> ' + 'sftbadclf=%<{x-safet-bad-classifiers}cqh> yra=%<{Y-RA}cqh> scheme=%') + +ts.Disk.logging_yaml.AddLines( + ''' logging: + formats: + - name: custom + format: '{}' + logs: + - filename: squid.log + format: custom +'''.format(format_string).split("\n") +) + +ts.Disk.remap_config.AddLine( + 'map / https://127.0.0.1:{0}'.format(server.Variables.SSL_Port) +) + +# Note that CONNECT is not in the allowed list. +ts.Disk.ip_allow_yaml.AddLines( + '''ip_allow: + - apply: in + ip_addrs: 0/0 + action: allow + methods: [GET, HEAD, POST ] + - apply: in + ip_addrs: ::/0 + action: allow + methods: [GET, HEAD, POST ] + +'''.split("\n") +) + +ts.Streams.stderr += Testers.ContainsExpression( + "Line 1 denial for 'CONNECT' from 127.0.0.1", + "The CONNECT request should be denied by ip_allow") +ts.Streams.stderr += Testers.ContainsExpression( + "Line 1 denial for 'PUSH' from 127.0.0.1", + "The PUSH request should be denied by ip_allow") + +# +# TEST 1: Perform a GET request. Should be allowed because GET is in the allowlist. +# +tr = Test.AddTestRun() +tr.Processes.Default.StartBefore(server, ready=When.PortOpen(server.Variables.SSL_Port)) +tr.Processes.Default.StartBefore(Test.Processes.ts, ready=When.PortOpen(ts.Variables.port)) + +tr.Processes.Default.Command = ('curl --verbose -H "Host: www.example.com" http://localhost:{ts_port}/get'. + format(ts_port=ts.Variables.port)) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Streams.stderr = 'gold/200.gold' +tr.StillRunningAfter = ts +tr.StillRunningAfter = server + +# +# TEST 2: Perform a CONNECT request. Should not be allowed because CONNECT is +# not in the allowlist. +# +tr = Test.AddTestRun() +tr.Processes.Default.Command = ('curl --verbose -X CONNECT -H "Host: localhost" http://localhost:{ts_port}/connect'. + format(ts_port=ts.Variables.port)) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Streams.stderr = 'gold/403.gold' +tr.StillRunningAfter = ts +tr.StillRunningAfter = server + +# +# TEST 3: Perform a PUSH request over HTTP/2. Should not be allowed because +# PUSH is not in the allowlist. +# +tr = Test.AddTestRun() +tr.Processes.Default.Command = ('curl --http2 --verbose -k -X PUSH -H "Host: localhost" https://localhost:{ts_port}/h2_push'. + format(ts_port=ts.Variables.ssl_port)) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Streams.stderr = 'gold/403_h2.gold' +tr.StillRunningAfter = ts +tr.StillRunningAfter = server diff --git a/tests/gold_tests/ip_allow/ssl/server.key b/tests/gold_tests/ip_allow/ssl/server.key new file mode 100644 index 00000000000..4c7a661a6bd --- /dev/null +++ b/tests/gold_tests/ip_allow/ssl/server.key @@ -0,0 +1,15 @@ +-----BEGIN RSA PRIVATE KEY----- +MIICXQIBAAKBgQDWMHOiUF+ORmZjAxI8MWE9dblb7gQSJ36WCXlPFiFx6ynF+S1E +kXAYpIip5X0pzDUaIbLukxJUAAnOtMEO0PCgxJQUrEtRWh8wiJdbdQJF0Zs/9R+u +SUgb61f+mdTQvhqefBGx+xrpfAcgtcWiZuSA9Q3fvpDj5WOWSPWXBUuxywIDAQAB +AoGBAJPxRX2gjFAGWmQbU/YVmXfNH6navh8X/nx9sLeqrpE0AFeJI/ZPiqDKzMal +B43eSfNxwVi+ZxN0L1ICUbL9KKZvHs/QBxWLA1fGVAXrz7sRplEVvakPpTfHoEnv +sKaMWVKaK/S5WGbDhElb6zb/Lwo19DsIAPjGYqFvzFJBmobJAkEA9iSeTGkR9X26 +GywZoYrIMlRh34htOIRx1UUq88rFzdrCF21kQ4lhBIkX5OZMMy652i2gyak4OZTe +YewIv8jw9QJBAN7EQNHG8jPwXfVp91/fqxVQEfumuP2i6uiWWYQgZCmla2+0xcLZ +pMQ6sQEe10hhTrVnzHgAUVp50Ntn2jwBX78CQF09veGAI9d1Cxzj9cmmAvRd1r2Q +tp8kPOLnUsALXib+6WtqewLCdcf8DtsdClyRJMIraq85tRzK8fryKNZNzkkCQEgA +yS7FDj5JgCU15hZgFk1iPx3HCt44jZM2HaL+UUHAzRQjKxTLAl3G1rWVAWLMyQML +lORoveLvotl4HOruSsMCQQCAx9dV9JUSFoyc1CWILp/FgUH/se4cjQCThGO0DoQQ +vGTYmntY7j9WRJ9esQrjdD6Clw8zM/45GIBNwnXzqo7Z +-----END RSA PRIVATE KEY----- diff --git a/tests/gold_tests/ip_allow/ssl/server.pem b/tests/gold_tests/ip_allow/ssl/server.pem new file mode 100644 index 00000000000..3584a2ec119 --- /dev/null +++ b/tests/gold_tests/ip_allow/ssl/server.pem @@ -0,0 +1,32 @@ +-----BEGIN RSA PRIVATE KEY----- +MIICXQIBAAKBgQDWMHOiUF+ORmZjAxI8MWE9dblb7gQSJ36WCXlPFiFx6ynF+S1E +kXAYpIip5X0pzDUaIbLukxJUAAnOtMEO0PCgxJQUrEtRWh8wiJdbdQJF0Zs/9R+u +SUgb61f+mdTQvhqefBGx+xrpfAcgtcWiZuSA9Q3fvpDj5WOWSPWXBUuxywIDAQAB +AoGBAJPxRX2gjFAGWmQbU/YVmXfNH6navh8X/nx9sLeqrpE0AFeJI/ZPiqDKzMal +B43eSfNxwVi+ZxN0L1ICUbL9KKZvHs/QBxWLA1fGVAXrz7sRplEVvakPpTfHoEnv +sKaMWVKaK/S5WGbDhElb6zb/Lwo19DsIAPjGYqFvzFJBmobJAkEA9iSeTGkR9X26 +GywZoYrIMlRh34htOIRx1UUq88rFzdrCF21kQ4lhBIkX5OZMMy652i2gyak4OZTe +YewIv8jw9QJBAN7EQNHG8jPwXfVp91/fqxVQEfumuP2i6uiWWYQgZCmla2+0xcLZ +pMQ6sQEe10hhTrVnzHgAUVp50Ntn2jwBX78CQF09veGAI9d1Cxzj9cmmAvRd1r2Q +tp8kPOLnUsALXib+6WtqewLCdcf8DtsdClyRJMIraq85tRzK8fryKNZNzkkCQEgA +yS7FDj5JgCU15hZgFk1iPx3HCt44jZM2HaL+UUHAzRQjKxTLAl3G1rWVAWLMyQML +lORoveLvotl4HOruSsMCQQCAx9dV9JUSFoyc1CWILp/FgUH/se4cjQCThGO0DoQQ +vGTYmntY7j9WRJ9esQrjdD6Clw8zM/45GIBNwnXzqo7Z +-----END RSA PRIVATE KEY----- +-----BEGIN CERTIFICATE----- +MIICszCCAhwCCQCl0Y79KkYjpzANBgkqhkiG9w0BAQUFADCBnTELMAkGA1UEBhMC +VVMxCzAJBgNVBAgTAklMMRIwEAYDVQQHEwlDaGFtcGFpZ24xDjAMBgNVBAoTBVlh +aG9vMQ0wCwYDVQQLEwRFZGdlMSgwJgYDVQQDEx9qdWljZXByb2R1Y2UuY29ycC5u +ZTEueWFob28uY29tMSQwIgYJKoZIhvcNAQkBFhVwZXJzaWEuYXppekB5YWhvby5j +b20wHhcNMTcwODI4MDI1MjI5WhcNMjcwODI2MDI1MjI5WjCBnTELMAkGA1UEBhMC +VVMxCzAJBgNVBAgTAklMMRIwEAYDVQQHEwlDaGFtcGFpZ24xDjAMBgNVBAoTBVlh +aG9vMQ0wCwYDVQQLEwRFZGdlMSgwJgYDVQQDEx9qdWljZXByb2R1Y2UuY29ycC5u +ZTEueWFob28uY29tMSQwIgYJKoZIhvcNAQkBFhVwZXJzaWEuYXppekB5YWhvby5j +b20wgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBANYwc6JQX45GZmMDEjwxYT11 +uVvuBBInfpYJeU8WIXHrKcX5LUSRcBikiKnlfSnMNRohsu6TElQACc60wQ7Q8KDE +lBSsS1FaHzCIl1t1AkXRmz/1H65JSBvrV/6Z1NC+Gp58EbH7Gul8ByC1xaJm5ID1 +Dd++kOPlY5ZI9ZcFS7HLAgMBAAEwDQYJKoZIhvcNAQEFBQADgYEAASZbz+d+DdI+ +ypesJrlBRosXh0w8sIjkUSSdT/OuKEVzfH/dRcb4VZDW/W2gmm0VEqSN2xYYVpW3 +hUsW2J+kByqFqX6selREwo8ui8kkyBJVo0y/MCrGM0C3qw1cSaiKoa5OqlOyO3hb +ZC9IIyWmpBxRmJFfIwS6MoTpe0/ZTJQ= +-----END CERTIFICATE----- diff --git a/tests/gold_tests/logging/all_headers.test.py b/tests/gold_tests/logging/all_headers.test.py index 6ae73d7d6ba..e5b850e88da 100644 --- a/tests/gold_tests/logging/all_headers.test.py +++ b/tests/gold_tests/logging/all_headers.test.py @@ -17,7 +17,6 @@ # limitations under the License. import os -import subprocess Test.Summary = ''' Test new "all headers" log fields @@ -33,7 +32,7 @@ request_header = {"headers": "GET / HTTP/1.1\r\nHost: does.not.matter\r\n\r\n", "timestamp": "1469733493.993", "body": ""} response_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\nCache-control: max-age=85000\r\n\r\n", - "timestamp": "1469733493.993", "body": "xxx"} + "timestamp": "1469733493.993", "body": "xxx"} server.addResponse("sessionlog.json", request_header, response_header) ts.Disk.records_config.update({ @@ -78,11 +77,12 @@ def reallyLong(): retval += ' -H "x-header{}: {}"'.format(i, value) return retval + tr = Test.AddTestRun() tr.Processes.Default.StartBefore(server) tr.Processes.Default.StartBefore(Test.Processes.ts) tr.Processes.Default.Command = ( -'curl "http://127.0.0.1:{0}" --user-agent "007" --verbose '.format(ts.Variables.port) + reallyLong() + 'curl "http://127.0.0.1:{0}" --user-agent "007" --verbose '.format(ts.Variables.port) + reallyLong() ) tr.Processes.Default.ReturnCode = 0 @@ -90,15 +90,14 @@ def reallyLong(): # tr = Test.AddTestRun() tr.Processes.Default.Command = ( -'curl "http://127.0.0.1:{0}" --user-agent "007" --verbose '.format(ts.Variables.port) + reallyLong() + 'curl "http://127.0.0.1:{0}" --user-agent "007" --verbose '.format(ts.Variables.port) + reallyLong() ) tr.Processes.Default.ReturnCode = 0 # Delay to allow TS to flush report to disk, then "sanitize" generated log. # tr = Test.AddTestRun() -tr.DelayStart = 10 -tr.Processes.Default.Command = 'python {0} {4} < {2} | sh {1} > {3}'.format( +tr.Processes.Default.Command = 'python3 {0} {2} {4} | sh {1} > {3}'.format( os.path.join(Test.TestDirectory, 'all_headers_sanitizer.py'), os.path.join(Test.TestDirectory, 'all_headers_sanitizer.sh'), os.path.join(ts.Variables.LOGDIR, 'test_all_headers.log'), diff --git a/tests/gold_tests/logging/all_headers_sanitizer.py b/tests/gold_tests/logging/all_headers_sanitizer.py index 2206b5e2acb..c7580394bd0 100644 --- a/tests/gold_tests/logging/all_headers_sanitizer.py +++ b/tests/gold_tests/logging/all_headers_sanitizer.py @@ -19,6 +19,8 @@ import sys import re +from os import path +import time rexl = [] rexl.append((re.compile(r"\{\{Date\}\:\{[^}]*\}\}"), "({__DATE__}}")) @@ -29,15 +31,27 @@ rexl.append((re.compile(r"\{\{Via\}\:\{[^}]*\}\}"), "({__VIA__}}")) rexl.append((re.compile(r"\{\{Server\}\:\{ApacheTrafficServer/[0-9.]*\}\}"), "({__ATS2_SERVER__}}")) rexl.append((re.compile(r"\{\{Age\}\:\{[0-9]*\}\}"), "({__AGE__}}")) -rexl.append((re.compile(r"\:" + sys.argv[1]), "__TS_PORT__")) # 1st and only argument is TS client port +rexl.append((re.compile(r"\:" + sys.argv[2]), "__TS_PORT__")) # 1st and only argument is TS client port # Handle inconsistencies which I think are caused by different revisions of the standard Python http.server.HTTPServer class. rexl.append((re.compile(r'\{"359670651[^"]*"\}'), '{"{359670651__WEIRD__}"}')) rexl.append((re.compile(r'\{\{Accept-Ranges\}:\{bytes\}\}'), '')) -for line in sys.stdin: - for rex, subStr in rexl: - line = rex.sub(subStr, line) - - print(line) +# Loop until the file specified in argv[1] becomes availble +filename = sys.argv[1] +processed = False +# Give up looking for file after 2 minutes +limit_count = 0 +limit_max = 120 +while not processed and limit_count < limit_max: + limit_count += 1 + if not path.exists(filename): + time.sleep(1) + else: + with open(filename, "r") as f: + processed = True + for line in f: + for rex, subStr in rexl: + line = rex.sub(subStr, line) + print(line) diff --git a/tests/gold_tests/logging/custom-log.test.py b/tests/gold_tests/logging/custom-log.test.py index 6dd132b9f0c..e8799d34eef 100644 --- a/tests/gold_tests/logging/custom-log.test.py +++ b/tests/gold_tests/logging/custom-log.test.py @@ -21,7 +21,8 @@ Test.Summary = ''' Test custom log file format ''' -# need Curl + +# this test depends on Linux specific behavior regarding loopback addresses Test.SkipUnless( Condition.IsPlatform("linux") ) diff --git a/tests/gold_tests/logging/gold/filter-test.gold b/tests/gold_tests/logging/gold/filter-test.gold new file mode 100644 index 00000000000..9dc6586efdf --- /dev/null +++ b/tests/gold_tests/logging/gold/filter-test.gold @@ -0,0 +1,5 @@ +http://test-1/test-1?name=value&email=XXXXXXXXXXXXX +http://test-2/test-2?email=XXXXXXXXXXXXX&name=password +http://test-3/test-3?trivial=password&name1=val1&email=XXXXXXXXXXXXX +http://test-4/test-4?trivial=password&email=&name=handle&session_redirect=XXXXXXXXXXXX +http://test-5/test-5?trivial=password&email=XXXXXXXXXXXXX&email=XXXXXXXXXXXXX&session_redirect=XXXXXXXXXXXX&email=XXXXXXXXXXXXX&name=value diff --git a/tests/gold_tests/logging/log-field.test.py b/tests/gold_tests/logging/log-field.test.py index b9997c7ba17..f88afc42429 100644 --- a/tests/gold_tests/logging/log-field.test.py +++ b/tests/gold_tests/logging/log-field.test.py @@ -21,28 +21,30 @@ Test.Summary = ''' Test log fields. ''' -# Only on Linux (why??) -Test.SkipUnless( - Condition.IsPlatform("linux") -) -# Define default ATS ts = Test.MakeATSProcess("ts") -# Microserver server = Test.MakeOriginServer("server") request_header = {'timestamp': 100, "headers": "GET /test-1 HTTP/1.1\r\nHost: test-1\r\n\r\n", "body": ""} -response_header = {'timestamp': 100, - "headers": "HTTP/1.1 200 OK\r\nTest: 1\r\nContent-Type: application/json\r\nConnection: close\r\nContent-Type: application/json\r\n\r\n", "body": "Test 1"} +response_header = { + 'timestamp': 100, + "headers": "HTTP/1.1 200 OK\r\nTest: 1\r\nContent-Type: application/json\r\nConnection: close\r\nContent-Type: application/json\r\n\r\n", + "body": "Test 1"} server.addResponse("sessionlog.json", request_header, response_header) server.addResponse("sessionlog.json", - {'timestamp': 101, "headers": "GET /test-2 HTTP/1.1\r\nHost: test-2\r\n\r\n", "body": ""}, - {'timestamp': 101, "headers": "HTTP/1.1 200 OK\r\nTest: 2\r\nContent-Type: application/jason\r\nConnection: close\r\nContent-Type: application/json\r\n\r\n", "body": "Test 2"} - ) + {'timestamp': 101, + "headers": "GET /test-2 HTTP/1.1\r\nHost: test-2\r\n\r\n", + "body": ""}, + {'timestamp': 101, + "headers": "HTTP/1.1 200 OK\r\nTest: 2\r\nContent-Type: application/jason\r\nConnection: close\r\nContent-Type: application/json\r\n\r\n", + "body": "Test 2"}) server.addResponse("sessionlog.json", - {'timestamp': 102, "headers": "GET /test-3 HTTP/1.1\r\nHost: test-3\r\n\r\n", "body": ""}, - {'timestamp': 102, "headers": "HTTP/1.1 200 OK\r\nTest: 3\r\nConnection: close\r\nContent-Type: application/json\r\n\r\n", "body": "Test 3"} - ) + {'timestamp': 102, + "headers": "GET /test-3 HTTP/1.1\r\nHost: test-3\r\n\r\n", + "body": ""}, + {'timestamp': 102, + "headers": "HTTP/1.1 200 OK\r\nTest: 3\r\nConnection: close\r\nContent-Type: application/json\r\n\r\n", + "body": "Test 3"}) ts.Disk.records_config.update({ 'proxy.config.net.connections_throttle': 100, diff --git a/tests/gold_tests/logging/log-filter.test.py b/tests/gold_tests/logging/log-filter.test.py new file mode 100644 index 00000000000..ea4b1621b3e --- /dev/null +++ b/tests/gold_tests/logging/log-filter.test.py @@ -0,0 +1,131 @@ +''' +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +Test.Summary = ''' +Test log filter. +''' + +ts = Test.MakeATSProcess("ts") +server = Test.MakeOriginServer("server") + +request_header = {'timestamp': 100, "headers": "GET /test-1 HTTP/1.1\r\nHost: test-1\r\n\r\n", "body": ""} +response_header = { + 'timestamp': 100, + "headers": "HTTP/1.1 200 OK\r\nTest: 1\r\nContent-Type: application/json\r\nConnection: close\r\nContent-Type: application/json\r\n\r\n", + "body": "Test 1"} +server.addResponse("sessionlog.json", request_header, response_header) +server.addResponse("sessionlog.json", + {'timestamp': 101, + "headers": "GET /test-2 HTTP/1.1\r\nHost: test-2\r\n\r\n", + "body": ""}, + {'timestamp': 101, + "headers": "HTTP/1.1 200 OK\r\nTest: 2\r\nContent-Type: application/jason\r\nConnection: close\r\nContent-Type: application/json\r\n\r\n", + "body": "Test 2"}) +server.addResponse("sessionlog.json", + {'timestamp': 102, + "headers": "GET /test-3 HTTP/1.1\r\nHost: test-3\r\n\r\n", + "body": ""}, + {'timestamp': 102, + "headers": "HTTP/1.1 200 OK\r\nTest: 3\r\nConnection: close\r\nContent-Type: application/json\r\n\r\n", + "body": "Test 3"}) +server.addResponse("sessionlog.json", + {'timestamp': 103, + "headers": "GET /test-4 HTTP/1.1\r\nHost: test-4\r\n\r\n", + "body": ""}, + {'timestamp': 103, + "headers": "HTTP/1.1 200 OK\r\nTest: 4\r\nConnection: close\r\nContent-Type: application/json\r\n\r\n", + "body": "Test 4"}) +server.addResponse("sessionlog.json", + {'timestamp': 104, + "headers": "GET /test-5 HTTP/1.1\r\nHost: test-5\r\n\r\n", + "body": ""}, + {'timestamp': 104, + "headers": "HTTP/1.1 200 OK\r\nTest: 5\r\nConnection: close\r\nContent-Type: application/json\r\n\r\n", + "body": "Test 5"}) + +ts.Disk.records_config.update({ + 'proxy.config.net.connections_throttle': 100, + 'proxy.config.http.cache.http': 0 +}) +# setup some config file for this server +ts.Disk.remap_config.AddLine( + 'map / http://localhost:{}/'.format(server.Variables.Port) +) + +ts.Disk.logging_yaml.AddLines( + ''' +logging: + filters: + - name: queryparamescaper_cquuc + action: WIPE_FIELD_VALUE + condition: cquuc CASE_INSENSITIVE_CONTAIN password,secret,access_token,session_redirect,cardNumber,code,query,search-query,prefix,keywords,email,handle + formats: + - name: custom + format: '%' + logs: + - filename: filter-test + format: custom + filters: + - queryparamescaper_cquuc +'''.split("\n") +) + +# ######################################################################### +# at the end of the different test run a custom log file should exist +# Because of this we expect the testruns to pass the real test is if the +# customlog file exists and passes the format check +Test.Disk.File(os.path.join(ts.Variables.LOGDIR, 'filter-test.log'), + exists=True, content='gold/filter-test.gold') + +# first test is a miss for default +tr = Test.AddTestRun() +# Wait for the micro server +tr.Processes.Default.StartBefore(server) +# Delay on readiness of our ssl ports +tr.Processes.Default.StartBefore(Test.Processes.ts) + +tr.Processes.Default.Command = 'curl --verbose --header "Host: test-1" "http://localhost:{0}/test-1?name=value&email=123@gmail.com"' .format( + ts.Variables.port) +tr.Processes.Default.ReturnCode = 0 + +tr = Test.AddTestRun() +tr.Processes.Default.Command = 'curl --verbose --header "Host: test-2" "http://localhost:{0}/test-2?email=123@gmail.com&name=password"' .format( + ts.Variables.port) +tr.Processes.Default.ReturnCode = 0 + +tr = Test.AddTestRun() +tr.Processes.Default.Command = 'curl --verbose --header "Host: test-3" "http://localhost:{0}/test-3?trivial=password&name1=val1&email=123@gmail.com"' .format( + ts.Variables.port) +tr.Processes.Default.ReturnCode = 0 + +tr = Test.AddTestRun() +tr.Processes.Default.Command = 'curl --verbose --header "Host: test-4" "http://localhost:{0}/test-4?trivial=password&email=&name=handle&session_redirect=wiped_string"' .format( + ts.Variables.port) +tr.Processes.Default.ReturnCode = 0 + +tr = Test.AddTestRun() +tr.Processes.Default.Command = 'curl --verbose --header "Host: test-5" "http://localhost:{0}/test-5?trivial=password&email=123@gmail.com&email=456@gmail.com&session_redirect=wiped_string&email=789@gmail.com&name=value"' .format( + ts.Variables.port) +tr.Processes.Default.ReturnCode = 0 + +tr = Test.AddTestRun() +tr.DelayStart = 10 +tr.Processes.Default.Command = 'echo "Delay for log flush"' +tr.Processes.Default.ReturnCode = 0 diff --git a/tests/gold_tests/logging/log_pipe.test.py b/tests/gold_tests/logging/log_pipe.test.py new file mode 100644 index 00000000000..7a7d1570c83 --- /dev/null +++ b/tests/gold_tests/logging/log_pipe.test.py @@ -0,0 +1,186 @@ +''' +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +Test.Summary = ''' +Test custom log file format +''' +Test.SkipUnless( + Condition.HasATSFeature('TS_HAS_PIPE_BUFFER_SIZE_CONFIG') +) + +ts_counter = 1 + + +def get_ts(logging_config): + """ + Create a Traffic Server process. + """ + global ts_counter + ts = Test.MakeATSProcess("ts{}".format(ts_counter)) + ts_counter += 1 + + ts.Disk.records_config.update({ + 'proxy.config.diags.debug.enabled': 1, + 'proxy.config.diags.debug.tags': 'log-file', + }) + + # Since we're only verifying logs and not traffic, we don't need an origin + # server. The following will simply deny the requests and emit a log + # message. + ts.Disk.remap_config.AddLine( + 'map / http://www.linkedin.com/ @action=deny' + ) + + ts.Disk.logging_yaml.AddLines(logging_config) + + return ts + + +# +# Test 1: Default configured log pipe size. +# +tr = Test.AddTestRun() +pipe_name = "default_pipe_size.pipe" +ts = get_ts( + ''' +logging: + formats: + - name: custom + format: "% %" + logs: + - filename: '{}' + mode: ascii_pipe + format: custom +'''.format(pipe_name).split("\n") +) + +pipe_path = os.path.join(ts.Variables.LOGDIR, pipe_name) + +ts.Streams.All += Testers.ContainsExpression( + "Created named pipe .*{}".format(pipe_name), + "Verify that the named pipe was created") + +ts.Streams.All += Testers.ContainsExpression( + "no readers for pipe .*{}".format(pipe_name), + "Verify that no readers for the pipe was detected.") + +ts.Streams.All += Testers.ExcludesExpression( + "New buffer size for pipe".format(pipe_name), + "Verify that the default pipe size was used.") + +curl = tr.Processes.Process("client_request", 'curl "http://127.0.0.1:{0}" --verbose'.format( + ts.Variables.port)) + +reader_output = os.path.join(ts.Variables.LOGDIR, "reader_output") +pipe_reader = tr.Processes.Process("pipe_reader", 'cat {} | tee {}'.format(pipe_path, reader_output)) +curl_ready = tr.Processes.Process("curl_ready", 'sleep 30') +# In the AuTest environment, it can take more than 10 seconds for the log file +# to be created. +curl_ready.StartupTimeout = 30 +curl_ready.Ready = When.FileContains(reader_output, '127.0.0.1') + +tr.Processes.Default.Command = "sleep 10" +tr.Processes.Default.Return = 0 + +# Process ordering. +tr.Processes.Default.StartBefore(curl_ready) +curl_ready.StartBefore(curl) +curl.StartBefore(pipe_reader) +pipe_reader.StartBefore(ts) + + +# +# Test 2: Change the log's buffer size. +# +tr = Test.AddTestRun() +pipe_name = "change_pipe_size.pipe" +# 64 KB is the default, so set the size larger than that to verify we can +# increase the size. +pipe_size = 75000 +ts = get_ts( + ''' +logging: + formats: + - name: custom + format: "% %" + logs: + - filename: '{}' + mode: ascii_pipe + format: custom + pipe_buffer_size: {} + '''.format(pipe_name, pipe_size).split("\n") +) + +pipe_path = os.path.join(ts.Variables.LOGDIR, pipe_name) + +ts.Streams.All += Testers.ContainsExpression( + "Created named pipe .*{}".format(pipe_name), + "Verify that the named pipe was created") + +ts.Streams.All += Testers.ContainsExpression( + "no readers for pipe .*{}".format(pipe_name), + "Verify that no readers for the pipe was detected.") + +ts.Streams.All += Testers.ContainsExpression( + "Previous buffer size for pipe .*{}".format(pipe_name), + "Verify that the named pipe's size was adjusted") + +# See fcntl: +# "Attempts to set the pipe capacity below the page size +# are silently rounded up to the page size." +# +# As a result of this, we cannot check that the pipe size is the exact size we +# requested, but it should be at least that big. We use the +# pipe_buffer_is_larger_than.py helper script to verify that the pipe grew in +# size. +ts.Streams.All += Testers.ContainsExpression( + "New buffer size for pipe.*{}".format(pipe_name), + "Verify that the named pipe's size was adjusted") +buffer_verifier = "pipe_buffer_is_larger_than.py" +tr.Setup.Copy(buffer_verifier) +verify_buffer_size = tr.Processes.Process( + "verify_buffer_size", + "python3 {} {} {}".format(buffer_verifier, pipe_path, pipe_size)) +verify_buffer_size.Return = 0 +verify_buffer_size.Streams.All += Testers.ContainsExpression( + "Success", + "The buffer size verifier should report success.") + +curl = tr.Processes.Process("client_request", 'curl "http://127.0.0.1:{0}" --verbose'.format( + ts.Variables.port)) + +reader_output = os.path.join(ts.Variables.LOGDIR, "reader_output") +pipe_reader = tr.Processes.Process("pipe_reader", 'cat {} | tee {}'.format(pipe_path, reader_output)) +curl_ready = tr.Processes.Process("curl_ready", 'sleep 30') +# In the AuTest environment, it can take more than 10 seconds for the log file +# to be created. +curl_ready.StartupTimeout = 30 +curl_ready.Ready = When.FileContains(reader_output, '127.0.0.1') + +tr.Processes.Default.Command = "sleep 10" +tr.Processes.Default.Return = 0 + + +# Process ordering. +tr.Processes.Default.StartBefore(verify_buffer_size) +verify_buffer_size.StartBefore(curl_ready) +curl_ready.StartBefore(curl) +curl.StartBefore(pipe_reader) +pipe_reader.StartBefore(ts) diff --git a/tests/gold_tests/logging/log_retention.test.py b/tests/gold_tests/logging/log_retention.test.py new file mode 100644 index 00000000000..ac4057a5de9 --- /dev/null +++ b/tests/gold_tests/logging/log_retention.test.py @@ -0,0 +1,473 @@ +''' +Verify correct log retention behavior. +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +Test.Summary = ''' +Test the enforcment of proxy.config.log.max_space_mb_for_logs. +''' + +# This test is sensitive to timing issues, especially in the OS CI for some +# reason. We'll leave the test here because it is helpful for when doing +# development on the log rotate code, but make it generally skipped when the +# suite of AuTests are run so it doesn't generate annoying false negatives. +Test.SkipIf(Condition.true("This test is sensitive to timing issues which makes it flaky.")) + + +class TestLogRetention: + __base_records_config = { + # Do not accept connections from clients until cache subsystem is operational. + 'proxy.config.http.wait_for_cache': 1, + 'proxy.config.diags.debug.enabled': 1, + 'proxy.config.diags.debug.tags': 'logspace', + + # Enable log rotation and auto-deletion, the subjects of this test. + 'proxy.config.log.rolling_enabled': 3, + 'proxy.config.log.auto_delete_rolled_files': 1, + + # 10 MB is the minimum rolling size. + 'proxy.config.log.rolling_size_mb': 10, + 'proxy.config.log.periodic_tasks_interval': 1, + } + + __server = None + __ts_counter = 1 + __server_is_started = False + + def __init__(self, records_config, run_description, command="traffic_manager"): + """ + Create a TestLogRetention instance. + """ + self.server = TestLogRetention.__create_server() + self.ts = self.__create_ts(records_config, command) + self.__initialize_processes() + self.tr = Test.AddTestRun(run_description) + + def __initialize_processes(self): + """ + Create a run to initialize the server and traffic_server processes so + the caller doesn't have to. + """ + tr = Test.AddTestRun("Initialize processes for ts{}".format(TestLogRetention.__ts_counter - 1)) + tr.Processes.Default.Command = self.get_curl_command() + tr.Processes.Default.ReturnCode = 0 + if not TestLogRetention.__server_is_started: + self.server.StartBefore(self.ts) + tr.Processes.Default.StartBefore(self.server) + TestLogRetention.__server_is_started = True + else: + tr.Processes.Default.StartBefore(self.ts) + + tr.StillRunningAfter = self.ts + tr.StillRunningAfter = self.server + + @classmethod + def __create_server(cls): + """ + Create and return a server process. + + There is only one server process for all the tests. This function is + re-entrant, but subsequent calls to it will return the cached version + of the single server. + """ + if cls.__server: + return cls.__server + + server = Test.MakeOriginServer("server") + request_header = {"headers": "GET / HTTP/1.1\r\n" + "Host: does.not.matter\r\n\r\n", + "timestamp": "1469733493.993", "body": ""} + response_header = {"headers": "HTTP/1.1 200 OK\r\n" + "Connection: close\r\n" + "Cache-control: max-age=85000\r\n\r\n", + "timestamp": "1469733493.993", "body": "xxx"} + server.addResponse("sessionlog.json", request_header, response_header) + cls.__server = server + return cls.__server + + def __create_ts(self, records_config, command="traffic_manager"): + """ + Create an ATS process. + + records_config: records_config values for this test. + command: The ATS process to run for the test. + """ + ts_name = "ts{counter}".format(counter=TestLogRetention.__ts_counter) + TestLogRetention.__ts_counter += 1 + self.ts = Test.MakeATSProcess(ts_name, command=command) + + combined_records_config = TestLogRetention.__base_records_config.copy() + combined_records_config.update(records_config) + self.ts.Disk.records_config.update(combined_records_config) + + self.ts.Disk.remap_config.AddLine( + 'map http://127.0.0.1:{0} http://127.0.0.1:{1}'.format( + self.ts.Variables.port, self.server.Variables.Port) + ) + return self.ts + + def get_curl_command(self): + """ + Generate the appropriate single curl command. + """ + return 'curl "http://127.0.0.1:{0}" --verbose'.format( + self.ts.Variables.port) + + def get_command_to_rotate_once(self): + """ + Generate the set of curl commands to trigger a log rotate. + """ + return 'for i in {{1..2500}}; do curl "http://127.0.0.1:{0}" --verbose; done'.format( + self.ts.Variables.port) + + def get_command_to_rotate_thrice(self): + """ + Generate the set of curl commands to trigger a log rotate. + """ + return 'for i in {{1..7500}}; do curl "http://127.0.0.1:{0}" --verbose; done'.format( + self.ts.Variables.port) + + +# +# Run 1: Verify that log deletion happens when no min_count is specified. +# +twelve_meg_log_space = { + # The following configures a 12 MB log cap with a required 2 MB head room. + # Thus the rotated log of just over 10 MB should be deleted because it + # will not leave enough head room. + 'proxy.config.log.max_space_mb_headroom': 2, + 'proxy.config.log.max_space_mb_for_logs': 12, +} +test = TestLogRetention(twelve_meg_log_space, + "Verify log rotation and deletion of the configured log file with no min_count.") + +# Configure approximately 5 KB entries for a log with no specified min_count. +test.ts.Disk.logging_yaml.AddLines( + ''' +logging: + formats: + - name: long + format: "{prefix}: %" + logs: + - filename: test_deletion + format: long +'''.format(prefix="0123456789" * 500).split("\n") +) + +# Verify that each log type was registered for auto-deletion. +test.ts.Streams.stderr = Testers.ContainsExpression( + "Registering rotated log deletion for test_deletion.log with min roll count 0", + "Verify test_deletion.log auto-delete configuration") +test.ts.Streams.stderr += Testers.ContainsExpression( + "Registering rotated log deletion for error.log with min roll count 0", + "Verify error.log auto-delete configuration") +test.ts.Streams.stderr += Testers.ContainsExpression( + "Registering rotated log deletion for traffic.out with min roll count 0", + "Verify traffic.out auto-delete configuration") +test.ts.Streams.stderr += Testers.ContainsExpression( + "Registering rotated log deletion for diags.log with min roll count 0", + "Verify diags.log auto-delete configuration") +test.ts.Streams.stderr += Testers.ContainsExpression( + "Registering rotated log deletion for manager.log with min roll count 0", + "Verify manager.log auto-delete configuration") +# Verify test_deletion was rotated and deleted. +test.ts.Streams.stderr += Testers.ContainsExpression( + "The rolled logfile.*test_deletion.log_.*was auto-deleted.*bytes were reclaimed", + "Verify that space was reclaimed") + +test.tr.Processes.Default.Command = test.get_command_to_rotate_once() +test.tr.Processes.Default.ReturnCode = 0 + +test.tr.StillRunningAfter = test.ts +test.tr.StillRunningAfter = test.server + + +# +# Test 2: Verify log deletion happens with a min_count of 1. +# +test = TestLogRetention(twelve_meg_log_space, + "Verify log rotation and deletion of the configured log file with a min_count of 1.") + +# Configure approximately 5 KB entries for a log with no specified min_count. +test.ts.Disk.logging_yaml.AddLines( + ''' +logging: + formats: + - name: long + format: "{prefix}: %" + logs: + - filename: test_deletion + rolling_min_count: 1 + format: long +'''.format(prefix="0123456789" * 500).split("\n") +) + +# Verify that each log type was registered for auto-deletion. +test.ts.Streams.stderr = Testers.ContainsExpression( + "Registering rotated log deletion for test_deletion.log with min roll count 1", + "Verify test_deletion.log auto-delete configuration") +# Only the test_deletion should have its min_count overridden. +test.ts.Streams.stderr += Testers.ContainsExpression( + "Registering rotated log deletion for error.log with min roll count 0", + "Verify error.log auto-delete configuration") +test.ts.Streams.stderr += Testers.ContainsExpression( + "Registering rotated log deletion for traffic.out with min roll count 0", + "Verify traffic.out auto-delete configuration") +test.ts.Streams.stderr += Testers.ContainsExpression( + "Registering rotated log deletion for diags.log with min roll count 0", + "Verify diags.log auto-delete configuration") +test.ts.Streams.stderr += Testers.ContainsExpression( + "Registering rotated log deletion for manager.log with min roll count 0", + "Verify manager.log auto-delete configuration") +# Verify test_deletion was rotated and deleted. +test.ts.Streams.stderr += Testers.ContainsExpression( + "The rolled logfile.*test_deletion.log_.*was auto-deleted.*bytes were reclaimed", + "Verify that space was reclaimed") + +test.tr.Processes.Default.Command = test.get_command_to_rotate_once() +test.tr.Processes.Default.ReturnCode = 0 +test.tr.StillRunningAfter = test.ts +test.tr.StillRunningAfter = test.server + + +# +# Test 3: Verify log deletion happens for a plugin's logs. +# +test = TestLogRetention(twelve_meg_log_space, + "Verify log rotation and deletion of plugin logs.") +Test.PrepareTestPlugin(os.path.join(Test.Variables.AtsTestPluginsDir, 'test_log_interface.so'), test.ts) + +# Verify that the plugin's logs and other core logs were registered for deletion. +test.ts.Streams.stderr = Testers.ContainsExpression( + "Registering rotated log deletion for test_log_interface.log with min roll count 0", + "Verify test_log_interface.log auto-delete configuration") +test.ts.Streams.stderr += Testers.ContainsExpression( + "Registering rotated log deletion for error.log with min roll count 0", + "Verify error.log auto-delete configuration") +test.ts.Streams.stderr += Testers.ContainsExpression( + "Registering rotated log deletion for traffic.out with min roll count 0", + "Verify traffic.out auto-delete configuration") +test.ts.Streams.stderr += Testers.ContainsExpression( + "Registering rotated log deletion for diags.log with min roll count 0", + "Verify diags.log auto-delete configuration") +test.ts.Streams.stderr += Testers.ContainsExpression( + "Registering rotated log deletion for manager.log with min roll count 0", + "Verify manager.log auto-delete configuration") +# Verify test_deletion was rotated and deleted. +test.ts.Streams.stderr += Testers.ContainsExpression( + "The rolled logfile.*test_log_interface.log_.*was auto-deleted.*bytes were reclaimed", + "Verify that space was reclaimed") + +test.tr.Processes.Default.Command = test.get_command_to_rotate_once() +test.tr.Processes.Default.ReturnCode = 0 +test.tr.StillRunningAfter = test.ts +test.tr.StillRunningAfter = test.server + +# +# Test 4: Verify log deletion priority behavior. +# +twenty_two_meg_log_space = { + # The following configures a 22 MB log cap with a required 2 MB head room. + # This should allow enough room for two logs being rotated. + 'proxy.config.log.max_space_mb_headroom': 2, + 'proxy.config.log.max_space_mb_for_logs': 22, +} +test = TestLogRetention(twenty_two_meg_log_space, + "Verify log deletion priority behavior.") + +# Configure approximately 5 KB entries for a log with no specified min_count. +test.ts.Disk.logging_yaml.AddLines( + ''' +logging: + formats: + - name: long + format: "{prefix}: %" + logs: + - filename: test_low_priority_deletion + rolling_min_count: 5 + format: long + + - filename: test_high_priority_deletion + rolling_min_count: 1 + format: long +'''.format(prefix="0123456789" * 500).split("\n") +) + +# Verify that each log type was registered for auto-deletion. +test.ts.Streams.stderr = Testers.ContainsExpression( + "Registering rotated log deletion for test_low_priority_deletion.log with min roll count 5", + "Verify test_low_priority_deletion.log auto-delete configuration") +test.ts.Streams.stderr += Testers.ContainsExpression( + "Registering rotated log deletion for test_high_priority_deletion.log with min roll count 1", + "Verify test_high_priority_deletion.log auto-delete configuration") +# Only the test_deletion should have its min_count overridden. +test.ts.Streams.stderr += Testers.ContainsExpression( + "Registering rotated log deletion for error.log with min roll count 0", + "Verify error.log auto-delete configuration") +test.ts.Streams.stderr += Testers.ContainsExpression( + "Registering rotated log deletion for traffic.out with min roll count 0", + "Verify traffic.out auto-delete configuration") +test.ts.Streams.stderr += Testers.ContainsExpression( + "Registering rotated log deletion for diags.log with min roll count 0", + "Verify diags.log auto-delete configuration") +test.ts.Streams.stderr += Testers.ContainsExpression( + "Registering rotated log deletion for manager.log with min roll count 0", + "Verify manager.log auto-delete configuration") +# Verify test_deletion was rotated and deleted. +test.ts.Streams.stderr += Testers.ExcludesExpression( + "The rolled logfile.*test_low_priority_deletion.log_.*was auto-deleted.*bytes were reclaimed", + "Verify that space was reclaimed from test_high_priority_deletion") +test.ts.Streams.stderr += Testers.ContainsExpression( + "The rolled logfile.*test_high_priority_deletion.log_.*was auto-deleted.*bytes were reclaimed", + "Verify that space was reclaimed from test_high_priority_deletion") + +test.tr.Processes.Default.Command = test.get_command_to_rotate_once() +test.tr.Processes.Default.ReturnCode = 0 +test.tr.StillRunningAfter = test.ts +test.tr.StillRunningAfter = test.server + +# +# Test 5: Verify min_count configuration overrides. +# +various_min_count_overrides = { + 'proxy.config.log.max_space_mb_for_logs': 22, + 'proxy.config.log.rolling_min_count': 3, + 'proxy.config.output.logfile.rolling_min_count': 4, + 'proxy.config.diags.logfile.rolling_min_count': 5, +} +test = TestLogRetention(various_min_count_overrides, + "Verify that the various min_count configurations behave as expected") + +# Only the test_deletion should have its min_count overridden. +test.ts.Streams.stderr = Testers.ContainsExpression( + "Registering rotated log deletion for error.log with min roll count 3", + "Verify error.log auto-delete configuration") +test.ts.Streams.stderr += Testers.ContainsExpression( + "Registering rotated log deletion for traffic.out with min roll count 4", + "Verify traffic.out auto-delete configuration") +test.ts.Streams.stderr += Testers.ContainsExpression( + "Registering rotated log deletion for diags.log with min roll count 5", + "Verify diags.log auto-delete configuration") +test.ts.Streams.stderr += Testers.ContainsExpression( + "Registering rotated log deletion for manager.log with min roll count 5", + "Verify manager.log auto-delete configuration") +# In case a future log is added, make sure the developer doesn't forget to +# set the min count per configuration. +test.ts.Streams.stderr += Testers.ExcludesExpression( + "Registering .* with min roll count 0", + "Verify nothing has a default min roll count of 0 per configuration") + +# This test doesn't require a log rotation. We just verify that the logs communicate +# the appropriate min_count values above. +test.tr.Processes.Default.Command = test.get_curl_command() +test.tr.Processes.Default.ReturnCode = 0 +test.tr.StillRunningAfter = test.ts +test.tr.StillRunningAfter = test.server + + +# +# Test 6: Verify log deletion does not happen when it is disabled. +# +auto_delete_disabled = twelve_meg_log_space.copy() +auto_delete_disabled.update({ + 'proxy.config.log.auto_delete_rolled_files': 0, +}) +test = TestLogRetention(auto_delete_disabled, + "Verify log deletion does not happen when auto-delet is disabled.") + +# Configure approximately 5 KB entries for a log with no specified min_count. +test.ts.Disk.logging_yaml.AddLines( + ''' +logging: + formats: + - name: long + format: "{prefix}: %" + logs: + - filename: test_deletion + rolling_min_count: 1 + format: long +'''.format(prefix="0123456789" * 500).split("\n") +) + +# Verify that each log type was registered for auto-deletion. +test.ts.Streams.stderr = Testers.ExcludesExpression( + "Registering rotated log deletion for test_deletion.log with min roll count 1", + "Verify test_deletion.log auto-delete configuration") +# Only the test_deletion should have its min_count overridden. +test.ts.Streams.stderr += Testers.ExcludesExpression( + "Registering rotated log deletion for error.log with min roll count 0", + "Verify error.log auto-delete configuration") +test.ts.Streams.stderr += Testers.ExcludesExpression( + "Registering rotated log deletion for traffic.out with min roll count 0", + "Verify traffic.out auto-delete configuration") +test.ts.Streams.stderr += Testers.ExcludesExpression( + "Registering rotated log deletion for diags.log with min roll count 0", + "Verify diags.log auto-delete configuration") +test.ts.Streams.stderr += Testers.ExcludesExpression( + "Registering rotated log deletion for manager.log with min roll count 0", + "Verify manager.log auto-delete configuration") +# Verify test_deletion was not deleted. +test.ts.Streams.stderr += Testers.ExcludesExpression( + "The rolled logfile.*test_deletion.log_.*was auto-deleted.*bytes were reclaimed", + "Verify that space was reclaimed") + +test.tr.Processes.Default.Command = test.get_command_to_rotate_once() +test.tr.Processes.Default.ReturnCode = 0 +test.tr.StillRunningAfter = test.ts +test.tr.StillRunningAfter = test.server + +# +# Test 7: Verify that max_roll_count is respected. +# +max_roll_count_of_2 = { + 'proxy.config.diags.debug.tags': 'log-file', + + # Provide plenty of max_space: we want auto-deletion to happen because of + # rolling_max_count, not max_space_mb_for_logs. + 'proxy.config.log.max_space_mb_headroom': 2, + 'proxy.config.log.max_space_mb_for_logs': 100, + + # This is the configuration under test. + 'proxy.config.log.rolling_max_count': 2, +} +test = TestLogRetention(max_roll_count_of_2, + "Verify max_roll_count is respected.") + +# Configure approximately 5 KB entries for a log with no specified min_count. +test.ts.Disk.logging_yaml.AddLines( + ''' +logging: + formats: + - name: long + format: "{prefix}: %" + logs: + - filename: test_deletion + format: long +'''.format(prefix="0123456789" * 500).split("\n") +) + +# Verify that trim happened for the rolled file. +test.ts.Streams.stderr = Testers.ContainsExpression( + "rolled logfile.*test_deletion.log.*old.* was auto-deleted", + "Verify test_deletion.log was trimmed") + +test.tr.Processes.Default.Command = test.get_command_to_rotate_thrice() +test.tr.Processes.Default.ReturnCode = 0 +test.tr.StillRunningAfter = test.ts +test.tr.StillRunningAfter = test.server diff --git a/tests/gold_tests/logging/ccid_ctid.test.py b/tests/gold_tests/logging/new_log_flds.test.py similarity index 75% rename from tests/gold_tests/logging/ccid_ctid.test.py rename to tests/gold_tests/logging/new_log_flds.test.py index 26dc65984b5..dbd192240b8 100644 --- a/tests/gold_tests/logging/ccid_ctid.test.py +++ b/tests/gold_tests/logging/new_log_flds.test.py @@ -17,12 +17,11 @@ # limitations under the License. import os -import subprocess Test.Summary = ''' -Test new ccid and ctid log fields +Test new log fields ''' -# need Curl + Test.SkipUnless( Condition.HasCurlFeature('http2') ) @@ -48,6 +47,10 @@ 'map https://127.0.0.1:{0} https://httpbin.org/ip'.format(ts.Variables.ssl_port) ) +ts.Disk.remap_config.AddLine( + 'map https://reallyreallyreallyreallylong.com https://httpbin.org/ip'.format(ts.Variables.ssl_port) +) + ts.Disk.ssl_multicert_config.AddLine( 'dest_ip=* ssl_cert_name=server.pem ssl_key_name=server.key' ) @@ -57,9 +60,9 @@ logging: formats: - name: custom - format: "% %" + format: "% % %" logs: - - filename: test_ccid_ctid + - filename: test_new_log_flds format: custom '''.split("\n") ) @@ -83,15 +86,24 @@ tr.Processes.Default.ReturnCode = 0 tr = Test.AddTestRun() -tr.Processes.Default.Command = 'curl "https://127.0.0.1:{0}" "https://127.0.0.1:{0}" --http2 --insecure --verbose'.format( - ts.Variables.ssl_port) +tr.Processes.Default.Command = ( + 'curl "https://127.0.0.1:{0}" "https://127.0.0.1:{0}" --http2 --insecure --verbose'.format( + ts.Variables.ssl_port) +) +tr.Processes.Default.ReturnCode = 0 + +tr = Test.AddTestRun() +tr.Processes.Default.Command = ( + 'curl "https://reallyreallyreallyreallylong.com:{0}" --http2 --insecure --verbose' + + ' --resolve reallyreallyreallyreallylong.com:{0}:127.0.0.1' +).format(ts.Variables.ssl_port) tr.Processes.Default.ReturnCode = 0 # Delay to allow TS to flush report to disk, then validate generated log. # tr = Test.AddTestRun() tr.DelayStart = 10 -tr.Processes.Default.Command = 'python {0} < {1}'.format( - os.path.join(Test.TestDirectory, 'ccid_ctid_observer.py'), - os.path.join(ts.Variables.LOGDIR, 'test_ccid_ctid.log')) +tr.Processes.Default.Command = 'python3 {0} < {1}'.format( + os.path.join(Test.TestDirectory, 'new_log_flds_observer.py'), + os.path.join(ts.Variables.LOGDIR, 'test_new_log_flds.log')) tr.Processes.Default.ReturnCode = 0 diff --git a/tests/gold_tests/logging/ccid_ctid_observer.py b/tests/gold_tests/logging/new_log_flds_observer.py similarity index 76% rename from tests/gold_tests/logging/ccid_ctid_observer.py rename to tests/gold_tests/logging/new_log_flds_observer.py index 1b4cee5e885..b6e4e70e9ee 100644 --- a/tests/gold_tests/logging/ccid_ctid_observer.py +++ b/tests/gold_tests/logging/new_log_flds_observer.py @@ -1,5 +1,5 @@ ''' -Examines log generated by ccid_ctid.test.py, returns 0 if valid, 1 if not. +Examines log generated by new_log_flds.test.py, returns 0 if valid, 1 if not. ''' # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file @@ -23,10 +23,12 @@ ccid = [] ctid = [] -# Read in ccid and ctid fields from each line of the generated report. +# Read in log fields from each line of the generated report. # +ln_num = 0 for ln in csv.reader(sys.stdin, delimiter=' '): - if len(ln) != 2: + ln_num += 1 + if len(ln) != 3: exit(code=1) i = int(ln[0]) if i < 0: @@ -36,6 +38,12 @@ if i < 0: exit(code=1) ctid.append(i) + if ln_num == 7: + if ln[2] != "reallyreallyreallyreallylong.com": + exit(code=1) + else: + if ln[2] != "-": + exit(code=1) # Validate contents of report. # @@ -45,7 +53,8 @@ ctid[2] != ctid[3] and ccid[3] != ccid[4] and ccid[4] == ccid[5] and - ctid[4] != ctid[5]): + ctid[4] != ctid[5] and + ccid[5] != ccid[6]): exit(code=0) # Failure exit if report was not valid. diff --git a/tests/gold_tests/logging/pipe_buffer_is_larger_than.py b/tests/gold_tests/logging/pipe_buffer_is_larger_than.py new file mode 100644 index 00000000000..e332dcbe5ef --- /dev/null +++ b/tests/gold_tests/logging/pipe_buffer_is_larger_than.py @@ -0,0 +1,69 @@ +#!/usr/bin/env python +''' +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import fcntl +import sys + +F_SETPIPE_SZ = 1031 # Linux 2.6.35+ +F_GETPIPE_SZ = 1032 # Linux 2.6.35+ + + +def parse_args(): + parser = parser = argparse.ArgumentParser( + description='Verify that a FIFO has a buffer of at least a certain size') + + parser.add_argument( + 'pipe_name', + help='The pipe name upon which to verify the size is large enough.') + + parser.add_argument( + 'minimum_buffer_size', + help='The minimu buffer size for the pipe to expect.') + + return parser.parse_args() + + +def test_fifo(fifo, minimum_buffer_size): + try: + fifo_fd = open(fifo, "rb+", buffering=0) + buffer_size = fcntl.fcntl(fifo_fd, F_GETPIPE_SZ) + + if buffer_size >= int(minimum_buffer_size): + print("Success. Size is: {} which is larger than: {}".format( + buffer_size, + minimum_buffer_size)) + return 0 + else: + print("Fail. Size is: {} which is smaller than: {}".format( + buffer_size, + minimum_buffer_size)) + return 1 + except Exception as e: + print("Unable to open fifo, error: {}".format(str(e))) + return 2 + + +def main(): + args = parse_args() + return test_fifo(args.pipe_name, args.minimum_buffer_size) + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/tests/gold_tests/null_transform/gold/null_transform-tag.gold b/tests/gold_tests/null_transform/gold/null_transform-tag.gold index 7f6084669d3..733c4d10fc2 100644 --- a/tests/gold_tests/null_transform/gold/null_transform-tag.gold +++ b/tests/gold_tests/null_transform/gold/null_transform-tag.gold @@ -1 +1 @@ -``DIAG: (null_transform)`` \ No newline at end of file +``DIAG: (null_transform)`` diff --git a/tests/gold_tests/null_transform/null_transform.test.py b/tests/gold_tests/null_transform/null_transform.test.py index 17f9bad1913..497c2c4dd11 100644 --- a/tests/gold_tests/null_transform/null_transform.test.py +++ b/tests/gold_tests/null_transform/null_transform.test.py @@ -18,11 +18,14 @@ # limitations under the License. -import os Test.Summary = ''' Test a basic null transform plugin ''' +Test.SkipUnless( + Condition.PluginExists('null_transform.so') +) + Test.ContinueOnFail = True # Define default ATS @@ -51,7 +54,7 @@ ) # Load plugin -Test.PreparePlugin(os.path.join(Test.Variables.AtsTestToolsDir, 'plugins', 'null_transform.c'), ts) +Test.PrepareInstalledPlugin('null_transform.so', ts) # www.example.com Host tr = Test.AddTestRun() diff --git a/tests/gold_tests/pluginTest/cache_range_requests/cache_range_requests.test.py b/tests/gold_tests/pluginTest/cache_range_requests/cache_range_requests.test.py new file mode 100644 index 00000000000..ba5592849b3 --- /dev/null +++ b/tests/gold_tests/pluginTest/cache_range_requests/cache_range_requests.test.py @@ -0,0 +1,366 @@ +''' +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +Test.Summary = ''' +Basic cache_range_requests plugin test +''' + +# Test description: +# Preload the cache with the entire asset to be range requested. +# Reload remap rule with cache_range_requests plugin +# Request content through the cache_range_requests plugin + +Test.SkipUnless( + Condition.PluginExists('cache_range_requests.so'), + Condition.PluginExists('xdebug.so'), +) +Test.ContinueOnFail = False +Test.testName = "cache_range_requests" + +# Define and configure ATS +ts = Test.MakeATSProcess("ts", command="traffic_server") + +# Define and configure origin server +server = Test.MakeOriginServer("server", lookup_key="{%uuid}") + +# default root +req_chk = {"headers": + "GET / HTTP/1.1\r\n" + + "Host: www.example.com\r\n" + + "uuid: none\r\n" + + "\r\n", + "timestamp": "1469733493.993", + "body": "" + } + +res_chk = {"headers": + "HTTP/1.1 200 OK\r\n" + + "Connection: close\r\n" + + "\r\n", + "timestamp": "1469733493.993", + "body": "" + } + +server.addResponse("sessionlog.json", req_chk, res_chk) + +body = "lets go surfin now" + +req_full = {"headers": + "GET /path HTTP/1.1\r\n" + + "Host: www.example.com\r\n" + + "Accept: */*\r\n" + + "uuid: full\r\n" + + "\r\n", + "timestamp": "1469733493.993", + "body": "" + } + +res_full = {"headers": + "HTTP/1.1 200 OK\r\n" + + "Cache-Control: max-age=500\r\n" + + "Connection: close\r\n" + + 'Etag: "path"\r\n' + + "\r\n", + "timestamp": "1469733493.993", + "body": body + } + +server.addResponse("sessionlog.json", req_full, res_full) + +block_bytes = 7 +bodylen = len(body) + +inner_str = "7-15" + +req_inner = {"headers": + "GET /path HTTP/1.1\r\n" + + "Host: www.example.com\r\n" + + "Accept: */*\r\n" + + "Range: bytes={}\r\n".format(inner_str) + + "uuid: inner\r\n" + + "\r\n", + "timestamp": "1469733493.993", + "body": "" + } + +res_inner = {"headers": + "HTTP/1.1 206 Partial Content\r\n" + + "Accept-Ranges: bytes\r\n" + + "Cache-Control: max-age=500\r\n" + + "Content-Range: bytes {0}/{1}\r\n".format(inner_str, bodylen) + + "Connection: close\r\n" + + 'Etag: "path"\r\n' + + "\r\n", + "timestamp": "1469733493.993", + "body": body[7:15] + } + +server.addResponse("sessionlog.json", req_inner, res_inner) + +frange_str = "0-" + +req_frange = {"headers": + "GET /path HTTP/1.1\r\n" + + "Host: www.example.com\r\n" + + "Accept: */*\r\n" + + "Range: bytes={}\r\n".format(frange_str) + + "uuid: frange\r\n" + + "\r\n", + "timestamp": "1469733493.993", + "body": "" + } + +res_frange = {"headers": + "HTTP/1.1 206 Partial Content\r\n" + + "Accept-Ranges: bytes\r\n" + + "Cache-Control: max-age=500\r\n" + + "Content-Range: bytes 0-{0}/{0}\r\n".format(bodylen) + + "Connection: close\r\n" + + 'Etag: "path"\r\n' + + "\r\n", + "timestamp": "1469733493.993", + "body": body + } + +server.addResponse("sessionlog.json", req_frange, res_frange) + +last_str = "-5" + +req_last = {"headers": + "GET /path HTTP/1.1\r\n" + + "Host: www.example.com\r\n" + + "Accept: */*\r\n" + + "Range: bytes={}\r\n".format(last_str) + + "uuid: last\r\n" + + "\r\n", + "timestamp": "1469733493.993", + "body": "" + } + +res_last = {"headers": + "HTTP/1.1 206 Partial Content\r\n" + + "Accept-Ranges: bytes\r\n" + + "Cache-Control: max-age=200\r\n" + + "Content-Range: bytes {0}-{1}/{1}\r\n".format(bodylen - 5, bodylen) + + "Connection: close\r\n" + + 'Etag: "path"\r\n' + + "\r\n", + "timestamp": "1469733493.993", + "body": body[-5:] + } + +server.addResponse("sessionlog.json", req_last, res_last) + +pselect_str = "1-10" + +req_pselect = {"headers": + "GET /path HTTP/1.1\r\n" + + "Host: parentselect\r\n" + + "Accept: */*\r\n" + + "Range: bytes={}\r\n".format(pselect_str) + + "uuid: pselect\r\n" + + "\r\n", + "timestamp": "1469733493.993", + "body": "" + } + +res_pselect = {"headers": + "HTTP/1.1 206 Partial Content\r\n" + + "Accept-Ranges: bytes\r\n" + + "Cache-Control: max-age=200\r\n" + + "Content-Range: bytes {}/19\r\n".format(pselect_str) + + "Connection: close\r\n" + + 'Etag: "path"\r\n' + + "\r\n", + "timestamp": "1469733493.993", + "body": body[1:10] + } + +server.addResponse("sessionlog.json", req_pselect, res_pselect) + +req_psd = {"headers": + "GET /path HTTP/1.1\r\n" + + "Host: psd\r\n" + + "Accept: */*\r\n" + + "Range: bytes={}\r\n".format(pselect_str) + + "uuid: pselect\r\n" + + "\r\n", + "timestamp": "1469733493.993", + "body": "" + } + +server.addResponse("sessionlog.json", req_psd, res_pselect) + +# cache range requests plugin remap +ts.Disk.remap_config.AddLines([ + 'map http://www.example.com http://127.0.0.1:{}'.format(server.Variables.Port) + + ' @plugin=cache_range_requests.so', + + # parent select cache key option + 'map http://parentselect http://127.0.0.1:{}'.format(server.Variables.Port) + + ' @plugin=cache_range_requests.so @pparam=--ps-cachekey', + + # deprecated + 'map http://psd http://127.0.0.1:{}'.format(server.Variables.Port) + + ' @plugin=cache_range_requests.so @pparam=ps_mode:cache_key_url', +]) + +# cache debug +ts.Disk.plugin_config.AddLine('xdebug.so') + +# minimal configuration +ts.Disk.records_config.update({ + 'proxy.config.diags.debug.enabled': 1, + 'proxy.config.diags.debug.tags': 'cache_range_requests', + 'proxy.config.http.cache.http': 1, + 'proxy.config.http.wait_for_cache': 1, +}) + +curl_and_args = 'curl -s -D /dev/stdout -o /dev/stderr -x localhost:{} -H "x-debug: x-cache"'.format(ts.Variables.port) + +# 0 Test - Fetch whole asset into cache +tr = Test.AddTestRun("full asset cache miss bypass") +ps = tr.Processes.Default +ps.StartBefore(server, ready=When.PortOpen(server.Variables.Port)) +ps.StartBefore(Test.Processes.ts, ready=When.PortOpen(ts.Variables.port)) +ps.Command = curl_and_args + ' http://www.example.com/path -H "uuid: full"' +ps.ReturnCode = 0 +ps.Streams.stderr = "gold/full.stderr.gold" +tr.StillRunningAfter = ts + +# test inner range +# 1 Test - Fetch range into cache +tr = Test.AddTestRun("inner range cache miss") +ps = tr.Processes.Default +ps.Command = curl_and_args + ' http://www.example.com/path -r {} -H "uuid: inner"'.format(inner_str) +ps.ReturnCode = 0 +ps.Streams.stderr = "gold/inner.stderr.gold" +ps.Streams.stdout.Content = Testers.ContainsExpression("X-Cache: miss", "expected cache miss") +ps.Streams.stdout.Content += Testers.ContainsExpression("Content-Range: bytes 7-15/18", "expected content-range header") +tr.StillRunningAfter = ts + +# 2 Test - Fetch from cache +tr = Test.AddTestRun("inner range cache hit") +ps = tr.Processes.Default +ps.Command = curl_and_args + ' http://www.example.com/path -r {}'.format(inner_str) +ps.ReturnCode = 0 +ps.Streams.stderr = "gold/inner.stderr.gold" +ps.Streams.stdout.Content = Testers.ContainsExpression("X-Cache: hit", "expected cache hit") +ps.Streams.stdout.Content += Testers.ContainsExpression("Content-Range: bytes 7-15/18", "expected content-range header") +tr.StillRunningAfter = ts + +# full range + +# 3 Test - 0- request +tr = Test.AddTestRun("0- request miss") +ps = tr.Processes.Default +ps.Command = curl_and_args + ' http://www.example.com/path -r {} -H "uuid: frange"'.format(frange_str) +ps.ReturnCode = 0 +ps.Streams.stderr = "gold/full.stderr.gold" +ps.Streams.stdout.Content = Testers.ContainsExpression("X-Cache: miss", "expected cache miss") +ps.Streams.stdout.Content += Testers.ContainsExpression("Content-Range: bytes 0-18/18", "expected content-range header") +tr.StillRunningAfter = ts + +# 4 Test - 0- request +tr = Test.AddTestRun("0- request hit") +ps = tr.Processes.Default +ps.Command = curl_and_args + ' http://www.example.com/path -r {}'.format(frange_str) +ps.ReturnCode = 0 +ps.Streams.stderr = "gold/full.stderr.gold" +ps.Streams.stdout.Content = Testers.ContainsExpression("X-Cache: hit", "expected cache hit") +ps.Streams.stdout.Content += Testers.ContainsExpression("Content-Range: bytes 0-18/18", "expected content-range header") +tr.StillRunningAfter = ts + +# end range + +# 5 Test - -5 request miss +tr = Test.AddTestRun("-5 request miss") +ps = tr.Processes.Default +ps.Command = curl_and_args + ' http://www.example.com/path -r {} -H "uuid: last"'.format(last_str) +ps.ReturnCode = 0 +ps.Streams.stderr = "gold/last.stderr.gold" +ps.Streams.stdout.Content = Testers.ContainsExpression("X-Cache: miss", "expected cache miss") +ps.Streams.stdout.Content += Testers.ContainsExpression("Content-Range: bytes 13-18/18", "expected content-range header") +tr.StillRunningAfter = ts + +# 6 Test - -5 request hit +tr = Test.AddTestRun("-5 request hit") +ps = tr.Processes.Default +ps.Command = curl_and_args + ' http://www.example.com/path -r {}'.format(last_str) +ps.ReturnCode = 0 +ps.Streams.stderr = "gold/last.stderr.gold" +ps.Streams.stdout.Content = Testers.ContainsExpression("X-Cache: hit", "expected cache hit") +ps.Streams.stdout.Content += Testers.ContainsExpression("Content-Range: bytes 13-18/18", "expected content-range header") +tr.StillRunningAfter = ts + +# Ensure 404's aren't getting cached + +# 7 Test - 404 +tr = Test.AddTestRun("404 request 1st") +ps = tr.Processes.Default +ps.Command = curl_and_args + ' http://www.example.com/404 -r 0-' +ps.Streams.stdout = "gold/404.stdout.gold" +ps.Streams.stdout.Content = Testers.ContainsExpression("X-Cache: miss", "expected cache miss") +tr.StillRunningAfter = ts + +# 8 Test - 404 +tr = Test.AddTestRun("404 request 2nd") +ps = tr.Processes.Default +ps.Command = curl_and_args + ' http://www.example.com/404 -r 0-' +ps.Streams.stdout.Content = Testers.ContainsExpression("X-Cache: miss", "expected cache miss") +ps.Streams.stdout.Content += Testers.ContainsExpression("404 Not Found", "expected 404 response") + +tr.StillRunningAfter = ts + +curl_and_args = 'curl -s -D /dev/stdout -o /dev/stderr -x localhost:{} -H "x-debug: x-parentselection-key"'.format( + ts.Variables.port) + +# 9 Test - cache_key_url request +tr = Test.AddTestRun("cache_key_url request") +ps = tr.Processes.Default +ps.Command = curl_and_args + ' http://parentselect/path -r {} -H "uuid: pselect"'.format(pselect_str) +ps.ReturnCode = 0 +ps.Streams.stdout.Content = Testers.ContainsExpression( + "X-ParentSelection-Key: .*-bytes=", + "expected bytes in parent selection key", +) +tr.StillRunningAfter = ts +tr.StillRunningAfter = server + +# 10 Test - non cache_key_url request ... no X-ParentSelection-Key +tr = Test.AddTestRun("non cache_key_url request") +ps = tr.Processes.Default +ps.Command = curl_and_args + ' http://www.example.com/path -r {} -H "uuid: inner"'.format(inner_str) +ps.ReturnCode = 0 +ps.Streams.stdout.Content = Testers.ExcludesExpression("X-ParentSelection-Key", "parent select key shouldn't show up") +tr.StillRunningAfter = ts +tr.StillRunningAfter = server + +# 11 Test - cache_key_url request -- deprecated +tr = Test.AddTestRun("cache_key_url request - dprecated") +ps = tr.Processes.Default +ps.Command = curl_and_args + ' http://psd/path -r {} -H "uuid: pselect"'.format(pselect_str) +ps.ReturnCode = 0 +ps.Streams.stdout.Content = Testers.ContainsExpression( + "X-ParentSelection-Key: .*-bytes=", + "expected bytes in parent selection key", +) +tr.StillRunningAfter = ts +tr.StillRunningAfter = server diff --git a/tests/gold_tests/pluginTest/cache_range_requests/cache_range_requests_cachekey.test.py b/tests/gold_tests/pluginTest/cache_range_requests/cache_range_requests_cachekey.test.py new file mode 100644 index 00000000000..02cf665c51a --- /dev/null +++ b/tests/gold_tests/pluginTest/cache_range_requests/cache_range_requests_cachekey.test.py @@ -0,0 +1,200 @@ +''' +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import time + +Test.Summary = ''' +cache_range_requests with cachekey +''' + +# Test description: +# Preload the cache with the entire asset to be range requested. +# Reload remap rule with cache_range_requests plugin +# Request content through the cache_range_requests plugin + +Test.SkipUnless( + Condition.PluginExists('cache_range_requests.so'), + Condition.PluginExists('cachekey.so'), + Condition.PluginExists('xdebug.so'), +) +Test.ContinueOnFail = False +Test.testName = "cache_range_requests_cachekey" + +# Define and configure ATS, enable traffic_ctl config reload +ts = Test.MakeATSProcess("ts", command="traffic_server") + +# Define and configure origin server +server = Test.MakeOriginServer("server", lookup_key="{%uuid}") + +# default root +req_chk = {"headers": + "GET / HTTP/1.1\r\n" + + "Host: www.example.com\r\n" + + "uuid: none\r\n" + + "\r\n", + "timestamp": "1469733493.993", + "body": "" + } + +res_chk = {"headers": + "HTTP/1.1 200 OK\r\n" + + "Connection: close\r\n" + + "\r\n", + "timestamp": "1469733493.993", + "body": "" + } + +server.addResponse("sessionlog.json", req_chk, res_chk) + +body = "lets go surfin now" +bodylen = len(body) + +# this request should work +req_full = {"headers": + "GET /path HTTP/1.1\r\n" + + "Host: www.example.com\r\n" + + "Accept: */*\r\n" + + "uuid: full\r\n" + + "\r\n", + "timestamp": "1469733493.993", + "body": "" + } + +res_full = {"headers": + "HTTP/1.1 206 Partial Content\r\n" + + "Accept-Ranges: bytes\r\n" + + 'Etag: "foo"\r\n' + + "Cache-Control: public, max-age=500\r\n" + + "Connection: close\r\n" + + "\r\n", + "timestamp": "1469733493.993", + "body": body + } + +server.addResponse("sessionlog.json", req_full, res_full) + +# this request should work +req_good = {"headers": + "GET /path HTTP/1.1\r\n" + + "Host: www.example.com\r\n" + + "Accept: */*\r\n" + + "Range: bytes=0-\r\n" + + "uuid: range_full\r\n" + + "\r\n", + "timestamp": "1469733493.993", + "body": "" + } + +res_good = {"headers": + "HTTP/1.1 206 Partial Content\r\n" + + "Accept-Ranges: bytes\r\n" + + 'Etag: "foo"\r\n' + + "Cache-Control: public, max-age=500\r\n" + + "Content-Range: bytes 0-{0}/{0}\r\n".format(bodylen) + + "Connection: close\r\n" + + "\r\n", + "timestamp": "1469733493.993", + "body": body + } + +server.addResponse("sessionlog.json", req_good, res_good) + +# this request should fail with a cache_range_requests asset +req_fail = {"headers": + "GET /path HTTP/1.1\r\n" + + "Host: www.fail.com\r\n" + + "Accept: */*\r\n" + + "Range: bytes=0-\r\n" + + "uuid: range_fail\r\n" + + "\r\n", + "timestamp": "1469733493.993", + "body": "" + } + +res_fail = {"headers": + "HTTP/1.1 206 Partial Content\r\n" + + "Accept-Ranges: bytes\r\n" + + 'Etag: "foo"\r\n' + + "Cache-Control: public, max-age=500\r\n" + + "Content-Range: bytes 0-{0}/{0}\r\n".format(bodylen) + + "Connection: close\r\n" + + "\r\n", + "timestamp": "1469733493.993", + "body": body + } + +server.addResponse("sessionlog.json", req_fail, res_fail) + +# cache range requests plugin remap, working config +ts.Disk.remap_config.AddLine( + 'map http://www.example.com http://127.0.0.1:{}'.format(server.Variables.Port) + + ' @plugin=cachekey.so @pparam=--include-headers=Range' + + ' @plugin=cache_range_requests.so @pparam=--no-modify-cachekey', +) + +# improperly configured cache_range_requests with cachekey +ts.Disk.remap_config.AddLine( + 'map http://www.fail.com http://127.0.0.1:{}'.format(server.Variables.Port) + + ' @plugin=cachekey.so @pparam=--static-prefix=foo' + ' @plugin=cache_range_requests.so', +) + +# cache debug +ts.Disk.plugin_config.AddLine('xdebug.so') + +# minimal configuration +ts.Disk.records_config.update({ + 'proxy.config.diags.debug.enabled': 1, + 'proxy.config.diags.debug.tags': 'cache_range_requests', + 'proxy.config.http.cache.http': 1, + 'proxy.config.http.wait_for_cache': 1, +}) + +curl_and_args = 'curl -s -D /dev/stdout -o /dev/stderr -x localhost:{} -H "x-debug: x-cache"'.format(ts.Variables.port) + +# 0 Test - Fetch full asset into cache (ensure cold) +tr = Test.AddTestRun("full asset fetch") +ps = tr.Processes.Default +ps.StartBefore(server, ready=When.PortOpen(server.Variables.Port)) +ps.StartBefore(Test.Processes.ts, ready=When.PortOpen(ts.Variables.port)) +ps.Command = curl_and_args + ' http://www.example.com/path -H "uuid: full"' +ps.ReturnCode = 0 +ps.Streams.stdout.Content = Testers.ContainsExpression("X-Cache: miss", "expected cache miss for load") +tr.StillRunningAfter = ts + +# 1 Test - Fetch whole asset into cache via range request (ensure cold) +tr = Test.AddTestRun("0- asset fetch") +ps = tr.Processes.Default +ps.Command = curl_and_args + ' http://www.example.com/path -r 0- -H "uuid: range_full"' +ps.ReturnCode = 0 +ps.Streams.stdout.Content = Testers.ContainsExpression("X-Cache: miss", "expected cache miss for load") +tr.StillRunningAfter = ts + +# 2 Test - Ensure assert happens instead of possible cache poisoning. +tr = Test.AddTestRun("Attempt poisoning") +ps = tr.Processes.Default +ps.Command = curl_and_args + ' http://www.fail.com/path -r 0- -H "uuid: range_fail"' +ps.ReturnCode = 0 +tr.StillRunningAfter = ts + +ts.Disk.diags_log.Content = Testers.ContainsExpression("ERROR", "error condition hit") +ts.Disk.diags_log.Content = Testers.ContainsExpression("failed to change the cache url", "ensure failure for misconfiguration") +ts.Disk.diags_log.Content = Testers.ContainsExpression( + "Disabling cache for this transaction to avoid cache poisoning", + "ensure transaction caching disabled") diff --git a/tests/gold_tests/pluginTest/cache_range_requests/cache_range_requests_ims.test.py b/tests/gold_tests/pluginTest/cache_range_requests/cache_range_requests_ims.test.py new file mode 100644 index 00000000000..8a6a953cf3e --- /dev/null +++ b/tests/gold_tests/pluginTest/cache_range_requests/cache_range_requests_ims.test.py @@ -0,0 +1,140 @@ +''' +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import time + +Test.Summary = ''' +cache_range_requests X-CRR-IMS plugin test +''' + +# Test description: +# Preload the cache with the entire asset to be range requested. +# Reload remap rule with cache_range_requests plugin +# Request content through the cache_range_requests plugin + +Test.SkipUnless( + Condition.PluginExists('cache_range_requests.so'), + Condition.PluginExists('xdebug.so'), +) +Test.ContinueOnFail = False +Test.testName = "cache_range_requests_ims" + +# Define and configure ATS +ts = Test.MakeATSProcess("ts", command="traffic_server") + +# Define and configure origin server +server = Test.MakeOriginServer("server") + +# default root +req_chk = {"headers": + "GET / HTTP/1.1\r\n" + + "Host: www.example.com\r\n" + + "uuid: none\r\n" + + "\r\n", + "timestamp": "1469733493.993", + "body": "" + } + +res_chk = {"headers": + "HTTP/1.1 200 OK\r\n" + + "Connection: close\r\n" + + "\r\n", + "timestamp": "1469733493.993", + "body": "" + } + +server.addResponse("sessionlog.json", req_chk, res_chk) + +body = "lets go surfin now" +bodylen = len(body) + +req_full = {"headers": + "GET /path HTTP/1.1\r\n" + + "Host: www.example.com\r\n" + + "Accept: */*\r\n" + + "Range: bytes=0-\r\n" + + "\r\n", + "timestamp": "1469733493.993", + "body": "" + } + +res_full = {"headers": + "HTTP/1.1 206 Partial Content\r\n" + + "Accept-Ranges: bytes\r\n" + + "Cache-Control: max-age=500\r\n" + + "Content-Range: bytes 0-{0}/{0}\r\n".format(bodylen) + + "Connection: close\r\n" + + 'Etag: "772102f4-56f4bc1e6d417"\r\n' + + "\r\n", + "timestamp": "1469733493.993", + "body": body + } + +server.addResponse("sessionlog.json", req_full, res_full) + +# cache range requests plugin remap +ts.Disk.remap_config.AddLine( + 'map http://www.example.com http://127.0.0.1:{}'.format(server.Variables.Port) + + ' @plugin=cache_range_requests.so @pparam=--consider-ims', +) + +# cache debug +ts.Disk.plugin_config.AddLine('xdebug.so') + +# minimal configuration +ts.Disk.records_config.update({ + 'proxy.config.diags.debug.enabled': 1, + 'proxy.config.diags.debug.tags': 'cache_range_requests', + 'proxy.config.http.cache.http': 1, + 'proxy.config.http.wait_for_cache': 1, +}) + +curl_and_args = 'curl -s -D /dev/stdout -o /dev/stderr -x localhost:{} -H "x-debug: x-cache"'.format(ts.Variables.port) + +# 0 Test - Fetch whole asset into cache +tr = Test.AddTestRun("0- range cache load") +ps = tr.Processes.Default +ps.StartBefore(server, ready=When.PortOpen(server.Variables.Port)) +ps.StartBefore(Test.Processes.ts, ready=When.PortOpen(ts.Variables.port)) +ps.Command = curl_and_args + ' http://www.example.com/path -r 0-' +ps.ReturnCode = 0 +ps.Streams.stdout.Content = Testers.ContainsExpression("X-Cache: miss", "expected cache miss for load") +tr.StillRunningAfter = ts + + +# set up the IMS date field (go in the future) RFC 2616 +futuretime = time.time() + 100 # seconds +futurestr = time.strftime("%a, %d %b %Y %H:%M:%S GMT", time.gmtime(futuretime)) + +# test inner range +# 1 Test - Fetch range into cache +tr = Test.AddTestRun("0- cache hit check") +ps = tr.Processes.Default +ps.Command = curl_and_args + ' http://www.example.com/path -r 0-'.format(futurestr) +ps.ReturnCode = 0 +ps.Streams.stdout.Content = Testers.ContainsExpression("X-Cache: hit", "expected cache hit") +tr.StillRunningAfter = ts + +# 2 Test - Ensure X-CRR-IMS header results in hit-stale +tr = Test.AddTestRun("0- range X-CRR-IMS check") +ps = tr.Processes.Default +ps.Command = curl_and_args + ' http://www.example.com/path -r 0- -H "X-CRR-IMS: {}"'.format(futurestr) +ps.ReturnCode = 0 +ps.Streams.stdout.Content = Testers.ContainsExpression("X-Cache: hit-stale", "expected cache hit-stale") +tr.StillRunningAfter = ts diff --git a/tests/gold_tests/pluginTest/cache_range_requests/gold/full.stderr.gold b/tests/gold_tests/pluginTest/cache_range_requests/gold/full.stderr.gold new file mode 100644 index 00000000000..c2475221112 --- /dev/null +++ b/tests/gold_tests/pluginTest/cache_range_requests/gold/full.stderr.gold @@ -0,0 +1 @@ +lets go surfin now \ No newline at end of file diff --git a/tests/gold_tests/pluginTest/cache_range_requests/gold/inner.stderr.gold b/tests/gold_tests/pluginTest/cache_range_requests/gold/inner.stderr.gold new file mode 100644 index 00000000000..90be4bf1a54 --- /dev/null +++ b/tests/gold_tests/pluginTest/cache_range_requests/gold/inner.stderr.gold @@ -0,0 +1 @@ + surfin \ No newline at end of file diff --git a/tests/gold_tests/pluginTest/cache_range_requests/gold/last.stderr.gold b/tests/gold_tests/pluginTest/cache_range_requests/gold/last.stderr.gold new file mode 100644 index 00000000000..836a80c7a51 --- /dev/null +++ b/tests/gold_tests/pluginTest/cache_range_requests/gold/last.stderr.gold @@ -0,0 +1 @@ +n now \ No newline at end of file diff --git a/tests/gold_tests/pluginTest/cert_update/cert_update.test.py b/tests/gold_tests/pluginTest/cert_update/cert_update.test.py index 642c328b8ab..8c545335a31 100644 --- a/tests/gold_tests/pluginTest/cert_update/cert_update.test.py +++ b/tests/gold_tests/pluginTest/cert_update/cert_update.test.py @@ -17,13 +17,17 @@ # See the License for the specific language governing permissions and # limitations under the License. +import ports + + Test.Summary = ''' Test cert_update plugin. ''' Test.SkipUnless( - Condition.HasProgram("openssl","Openssl need to be installed on system for this test to work") - ) + Condition.HasProgram("openssl", "Openssl need to be installed on system for this test to work"), + Condition.PluginExists('cert_update.so') +) # Set up origin server server = Test.MakeOriginServer("server") @@ -73,11 +77,7 @@ ]) # Set up plugin -Test.PreparePlugin(Test.Variables.AtsExampleDir + '/plugins/c-api/cert_update/cert_update.cc', ts) - -ts.Disk.plugin_config.AddLine( - 'cert_update.so' -) +Test.PrepareInstalledPlugin('cert_update.so', ts) # Server-Cert-Pre # curl should see that Traffic Server presents bar.com cert from alice @@ -112,7 +112,8 @@ # Client-Cert-Pre # s_server should see client (Traffic Server) as alice.com tr = Test.AddTestRun("Client-Cert-Pre") -s_server = tr.Processes.Process("s_server", "openssl s_server -www -key {0}/server1.pem -cert {0}/server1.pem -accept 12345 -Verify 1 -msg".format(ts.Variables.SSLDir)) +s_server = tr.Processes.Process( + "s_server", "openssl s_server -www -key {0}/server1.pem -cert {0}/server1.pem -accept 12345 -Verify 1 -msg".format(ts.Variables.SSLDir)) s_server.Ready = When.PortReady(12345) tr.Command = 'curl --verbose --insecure --header "Host: foo.com" https://localhost:{}'.format(ts.Variables.ssl_port) tr.Processes.Default.StartBefore(s_server) @@ -124,15 +125,16 @@ tr = Test.AddTestRun("Client-Cert-Update") tr.Processes.Default.Env = ts.Env tr.Processes.Default.Command = ( - 'mv {0}/client2.pem {0}/client1.pem && {1}/traffic_ctl plugin msg cert_update.client {0}/client1.pem'.format(ts.Variables.SSLDir, ts.Variables.BINDIR) -) + 'mv {0}/client2.pem {0}/client1.pem && {1}/traffic_ctl plugin msg cert_update.client {0}/client1.pem'.format( + ts.Variables.SSLDir, ts.Variables.BINDIR)) ts.Streams.all = "gold/update.gold" ts.StillRunningAfter = server # Client-Cert-After # after use traffic_ctl to update client cert, s_server should see client (Traffic Server) as bob.com tr = Test.AddTestRun("Client-Cert-After") -s_server = tr.Processes.Process("s_server", "openssl s_server -www -key {0}/server1.pem -cert {0}/server1.pem -accept 12345 -Verify 1 -msg".format(ts.Variables.SSLDir)) +s_server = tr.Processes.Process( + "s_server", "openssl s_server -www -key {0}/server1.pem -cert {0}/server1.pem -accept 12345 -Verify 1 -msg".format(ts.Variables.SSLDir)) s_server.Ready = When.PortReady(12345) tr.Processes.Default.Env = ts.Env # Move client2.pem to replace client1.pem since cert path matters in client context mapping diff --git a/tests/gold_tests/pluginTest/cert_update/ssl/client1.pem b/tests/gold_tests/pluginTest/cert_update/ssl/client1.pem index 5fda0b33090..74dc11e9bbf 100644 --- a/tests/gold_tests/pluginTest/cert_update/ssl/client1.pem +++ b/tests/gold_tests/pluginTest/cert_update/ssl/client1.pem @@ -1,83 +1,84 @@ -----BEGIN CERTIFICATE----- -MIIFXzCCA0egAwIBAgIUQW43tEf3dK2EAUi219sjNioWa8QwDQYJKoZIhvcNAQEL -BQAwPzELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAklMMQ8wDQYDVQQKDAZBcGFjaGUx -EjAQBgNVBAMMCWFsaWNlLmNvbTAeFw0xOTA1MDMyMTEzNDdaFw0yMDA1MDIyMTEz -NDdaMD8xCzAJBgNVBAYTAlVTMQswCQYDVQQIDAJJTDEPMA0GA1UECgwGQXBhY2hl -MRIwEAYDVQQDDAlhbGljZS5jb20wggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIK -AoICAQDEevfeMGJBziVeir3teXonOevM6E/4sZUxX2jDDgYvsYDkx8ConcufuO1B -svu9Pg+Odi0/zVjoJbIdF4+KqucUs+unDC2a1IHwHbUEQadlkPBmQq1Uxxc3Rb0c -RCjHzasn0p6vicq4bRmTtXSekezqTvwahDq0++Ci0fQwlbGewRBiJrrM+hbTOZdP -MR+SgIWsGRcRRTm8qtwMX/02UzE3cfY0liU6x/jMs76shhjt54hNDQJCRsNenh6m -rf5qD0i+GbQAjj6WN2LO+xuC0IoY8kbNjQTyKsOCkfhDADibyN70d/NUA3PhgqXD -4r119vhkLFR3FuVCW4mDgJ3Xo5XdiQVJ7cbrPS3Ds1lPESiJUssNPBSO3h2WM5HI -J9E9xDvQo4mrVc6phoDVFs3rtjjU9E5IMpCp8KxTQ5VpEk0/sHjijX5Q3OWWlBxX -bXrQSExj7aDvjL5SliwgobQsAX1zadtm/KBtDJTFB8emYIZfdNxSzM2a37ndZ/yZ -82NfUGY8FeY9kEMGGgLGU5KsIGgeVyFsbUHp/lGE+biup7NvlxJwV1BrujKjhpSn -RtLNeJTPwhrDgOKtdxV8zXZ3OW/nRibPciD7qno9HPmJgiknVroGtZ7ZvsNd91MD -BwyeIWC4+/39OQHHWM9tVyaKndNfKX40ewgL/FiOYkQ6m3i24wIDAQABo1MwUTAd -BgNVHQ4EFgQUJ44G/cXLEv+XvhBqRAzt4GsQMdkwHwYDVR0jBBgwFoAUJ44G/cXL -Ev+XvhBqRAzt4GsQMdkwDwYDVR0TAQH/BAUwAwEB/zANBgkqhkiG9w0BAQsFAAOC -AgEAiTnXOLgdVYKlrcmNQ6H7F6hDiZ6UiXEHyxBP+TVPF6zTIM9Zdcfpo8mfN+uC -lT9vBxzzKCiVwL6bO71jAHxukS3IIyM7FCYQYE+a3lnQBV+/LLrFhfdEgK87V5j6 -T74y++0+ssHGa96jlw4y45Rx9lvYLxpTjSEe2kYiQMhZXLKiGOw2JgRcPOtImUeQ -NHgss5d+CxGiq7qnl0OJLmInQqh3zHE3/sRu/+jJigZcn+UJHeszoJfZiKQEbgyv -w8VtwkAEBCoiTEiRDhhoovfFUWAcoaD7HkkPt8+yW9hvIgbtmMEC0tvAqr8ddPBc -UJ291FD4+03gEgk2nq3mmo0OWqzOsi8rGoJ6YvVgKLvUpirJhUhFWP/VJTVt6dA6 -C+bBd5/+J712gS0p0fkmUUPXXp5X7gTbVZxnO8WvgAPSx3lDjqF3fjzWot9ZqfOA -jskwyvTC4AoXhn9ea7ipJKDUy0LN64OHt68fYI2wq6H8Qy8dL3FhRKeCV0jCKlKW -EaYfNfK0H9EXCCNzBqUz7Y2i9dBlR6RcUR7A5L0DeOF/SVUjK6V0XOiQ32BIpOcD -82YsuACZyGvZ1xxhy25YxDoymNBXxrO+1yYxW+Ni7mmpdoUg41rzD0uIOOQcIgh1 -Cs7tSiybuRdUCA7qVThgJW+4333WHbaUo51WJD60tM1Tlms= +MIIFeTCCA2GgAwIBAgIJAISTHDcePqN7MA0GCSqGSIb3DQEBCwUAMFMxCzAJBgNV +BAYTAlVTMQswCQYDVQQIDAJJTDESMBAGA1UEBwwJQ2hhbXBhaWduMQ8wDQYDVQQK +DAZBcGFjaGUxEjAQBgNVBAMMCWFsaWNlLmNvbTAeFw0yMDA1MDQyMDMxMjRaFw0z +MDA1MDIyMDMxMjRaMFMxCzAJBgNVBAYTAlVTMQswCQYDVQQIDAJJTDESMBAGA1UE +BwwJQ2hhbXBhaWduMQ8wDQYDVQQKDAZBcGFjaGUxEjAQBgNVBAMMCWFsaWNlLmNv +bTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBANPu3ONTBTbRwc4+FfpO +Z2HufjZ21ZQiVcmndiRvoW5QzjC/eMivBzAPx9WS+8lgaUAp3G3llphv8EOuKxb0 +ka8a17dARRDmIk8oqVL3EOiCrwK+1vlaJmB5UbnadLgvBwqLHLBbZdxsD6rJK4tT +RY3w2NmJ9Rh0FGBMX+USBUmxIjEh5fowjYURqn9ND9BH9y68FfCpFGBjvlVzgKeR +78pXFwqJeM+d0tGG9k6DRdJ7qUnOWk4dWh+aaE2rHxss5LU/FZA4TicXsvrhsKs0 +ZaFk2RUCV7BOg43IHSH9nh47OtK5MtBbaW7SLkK1uMm2br5L3nI+2PTZuhkZMlfP +/XoEeoaOIBabv50iOekfHOjQ5TJ5jgUwUjRf1dhfCtfX8+2V6oMhm2lhaD5v6lxe +zwGDr9ycR9+fqmjZinLlZ0qXJ+PtAeQh7Xzoa9iEUmVJrY5llThK69zZUc27OHjy +oAiR7SeZJBQrtDXRy94IjRbhKwJJ3qCM28+yNaLjp+SF4INYJ+g7noKTI4jyAgGI +/8EicmzSbfoeR9IYEfoNTsKVfm/E7tWl9ORHufIxZyHZRg/Ri3r6tpaKSt9GZ/xv +31d7juvauGyls2LjdFKvhsKSGtJ7YxdZluNqCs5cloQmUl9TNq8eyUwueIDsjv9F +l2wkYwnTDmMEeRYy4x/f1rSDAgMBAAGjUDBOMB0GA1UdDgQWBBROGcex3gOvRDdN +zppQ2pmwL/cZJTAfBgNVHSMEGDAWgBROGcex3gOvRDdNzppQ2pmwL/cZJTAMBgNV +HRMEBTADAQH/MA0GCSqGSIb3DQEBCwUAA4ICAQA9RXNMy/iBTYE0mXdC1dhJtGhC +dWGIcx+zIR7lyKb5uzHLwqs9KPpLxzDvjTPn5+6iONjTd99m8CsB6KtdEYaEMvvL +Fk9XQLn7yUnUeHsTb0F+twBova15hlw277SrgPXlJVz9/Syrbmvb8upR1InaGO6q +2nyBMa+vEljmyfgpgGFOvfyTGczJoYWCOoxuRdkNC/jpAB8r+XcByX8Uad1BBPyk +xdVJnIh3VSX60Iu95WlO/YRhpx7R2N1HZUl32I58wYz9Fx5/0jZoD2mmI/n5NMyA +350Cumf58ESA0WDNp7tDfd4oZZwhn5/Il+iaz9TNVyujMJxMyfLwk/qIAIpFwkXZ +n0ika5T05SdPiAfSoQrx8aI48SjswDmmDrPz0BLbSdP+INDKkG/9Cpc4PG8t7T0m +KxTP1pN7Q3sj7KPu2/XbvVk1GxrDJyONSOEFOaL8x9qqMjM3gDp0wuXOw1p3vS2e +fmSkFYCzF0kiaXiXHkqYihHjqhg0nO2pjUPb/hOKmTlGwA2Aq1AxcYfGCiJf+bpV +jza4rH8vzvRYquxWnftZscpItE8eSnjhsEr56RcydItjo20Y9o4lcZFdqa3MrYe9 +g8EYVFlLczZ4JledznzffFVakvYPS7DknPJduckRfZ6C8d3tSpWidXjp6kaNHWIV +nikwHtRpVH6AO1H85Q== -----END CERTIFICATE----- -----BEGIN PRIVATE KEY----- -MIIJQwIBADANBgkqhkiG9w0BAQEFAASCCS0wggkpAgEAAoICAQDEevfeMGJBziVe -ir3teXonOevM6E/4sZUxX2jDDgYvsYDkx8ConcufuO1Bsvu9Pg+Odi0/zVjoJbId -F4+KqucUs+unDC2a1IHwHbUEQadlkPBmQq1Uxxc3Rb0cRCjHzasn0p6vicq4bRmT -tXSekezqTvwahDq0++Ci0fQwlbGewRBiJrrM+hbTOZdPMR+SgIWsGRcRRTm8qtwM -X/02UzE3cfY0liU6x/jMs76shhjt54hNDQJCRsNenh6mrf5qD0i+GbQAjj6WN2LO -+xuC0IoY8kbNjQTyKsOCkfhDADibyN70d/NUA3PhgqXD4r119vhkLFR3FuVCW4mD -gJ3Xo5XdiQVJ7cbrPS3Ds1lPESiJUssNPBSO3h2WM5HIJ9E9xDvQo4mrVc6phoDV -Fs3rtjjU9E5IMpCp8KxTQ5VpEk0/sHjijX5Q3OWWlBxXbXrQSExj7aDvjL5Sliwg -obQsAX1zadtm/KBtDJTFB8emYIZfdNxSzM2a37ndZ/yZ82NfUGY8FeY9kEMGGgLG -U5KsIGgeVyFsbUHp/lGE+biup7NvlxJwV1BrujKjhpSnRtLNeJTPwhrDgOKtdxV8 -zXZ3OW/nRibPciD7qno9HPmJgiknVroGtZ7ZvsNd91MDBwyeIWC4+/39OQHHWM9t -VyaKndNfKX40ewgL/FiOYkQ6m3i24wIDAQABAoICAQCGSinJ8jXmFjjSYm+ntOR5 -lQwGlC28o9t90GExGA/AX0jilWinlppLA8Edq3CGCrnlO/53YinHGaSgX68bLu4b -51FEbn1pGp/o9tT3IMyCDctRiXcgv6atf3veqvNYhMjbwgf1oG8vGFpn65jWnJRa -HTwP/5qEgrcpcei9oEKzZ++DtkbsvG4HVpCFbuTOZAt48fkjM7ZfrkQSLTVLARdd -dwW5MXomr/DsMFo9XrYBPMObLi4CEI2NyPLYJ4oBAYtNaxHtY0uBrj9ZiqVmBpSs -skMUULM1tWbFgnE1khwe1e8VFjr4jBeS5ZHYzcuCqhdhmC7u6nYtoZTejXTUoxlV -CsBRxGhAiM4ypIOan3V/khkQGrmTcmr35f2k3gIR6b12d36J3OeU0JbYDgXtJLuT -7QDLkupQanrsFjq6LPNZO5oZufDy30NXTTmYECe31ZGQnHUdOGM/NBVfOpc0aHPt -/5ZgoB5hEhsG2kjPSVVB57bYIVYD7GrUkTEe7nodoM/PXGIhMlTKIpCUU8v2ygNR -lQiuc4+motGRwpYMW4tEUe6mKn4KRrGnzzCed7RXgTeG5snAisHtmOTN03HC9wau -q53rVHG6rxnNqygJVKYTfRAePoquOcWP0hJ0xUGCuLzsDRJ8zO9XcI+UGpGFJnhZ -u2uqYDMZTClgGspgCMnCCQKCAQEA8p8cZFhXdmh3FENnPIJDRi7yIWesMcdlHxvO -rpmUGF/ALjRDQNq+sn7MkzgRoyTG2e7Pg47Ku8aXkzc6UDNi/xlC/RXRahtCySzS -SdFRF3WpO0WaR4POd5/Pc3ybBEAPvex0hrIkI0iufd1C4Bmnex2ohvj/BFweX22g -k+4j4HsIo73Akgr2MdqVHBeAn4b66H3+jB2ny5oi1icpXfmK2Ifb3yhbtWWTT9nF -UglqWsCAUGtPhUNApYvbIUiDW778oIgXeFOVB6G0MZrpf5dfzjDR3X7EoVDoBOJN -XCUa9n0UIDbKeHuPaQ2s0n90uv0Q4mIU/+k6YETypwTOywk7PwKCAQEAz1CFJXkt -4sszVVpGhmzXdQc39hsRQH6efF8BiiPzmp1ifoeA+Ar6yigsNMATG7W46MRLmSW7 -lcjuXspeYQC7PIq8mkkveYWHzaajcsXZkx7UuTdwGa6VEB0vBWh/7m4KQqnSIIdj -PQYm0cqmJrjKwvL3MrNyp1kZkURxF020lQ2w6nW5v5v6WleZq7/Cmflj5brTCt4Z -EaoO1t26J92o7w7CkqmNfcZMpZ/4yVdJxDoBay2BUbgfwYzLK0gevJuhGLtUQtA8 -I5bBI82Qt9AFzKbWfopSwaDWBXKHKLvcJsVHyuXrBi4ZiEiOz/sbHwm35ruGtBwD -+Gg8tzRRS3qPXQKCAQBf+csccsnegEKi1GsRR6JfMBD+X+mBI5R/8tsWvJAV+EKo -xGnaTO3k5D3++s4XUGQNL+gM7b6K+2tYhB9gPIOr0A1s2mWl6LTJqh5hrxi6BAR2 -+vil06EoNyK0V5Vm4ASaJ+CMrAmZn5XPGmjrB5r2G+xfwD35NouZSl+cRTcSBPmM -9HIqE4YCgKo9m5p5AMdekwDP4qdO5mFjf8hgcWeYcl3q8CcfIdhdXRMueaUF02Ku -7VRerhTzp9h+WRYFhA6hXlSSd9XbV/9VJCe8HmB6y1spmI0mF0BBNlhN3CvHWAFP -IP2FHbPEZfF4r4y4r4UvWIdgGJ3MGVo38bHwJW23AoIBAAmAq1m4YD4RCl1TMgBf -ZNDcb9g8DWJja2hQAoYOd9ASfr1GAMdd2XkCtmQEmdufTMZ+mOiALkUDXMnDhOf1 -XJ+9zD9WM3LiiAMJLFzKbNqtgxqqS90hf3upmsoorBSFvrqnhhYvnoDhk03yeAM2 -XTTqZiJQz2SUVPOvq29iBHEAm6djlgwOXj9d3JFezNC5+bZCBgJtg8Cnht6aczn4 -kxHCH3ERjIbDXCgLWSABfEQeVIpRH6hbRDle9sEZIS+MAqpbi9U0Lk2DT38QoR2L -z3g9/X73YCu375d4VHGtir3MNSo7t7Ykzs7MZJ9r5yZZD7Dnz5jZ3+S3AnFzWHaZ -O5ECggEBAKcRf1zVOX2XxE/hRxDon65hX3oRD7HQvBWYxBDUqnm+YSz1eeBE9Mt2 -P2mnkVNeeeN8QTjyrcjuJg+SVrjIE3Oq6sPfwlxMuQOM30CRh1BaDt2lR4VPE4fV -peuAGXOr+4SmXQu8GkcGR78kIOled7RhyrFYA0u1Sb7NoTbiwVs2MQw/5ifG86rV -PA3T8wpJ19OsHIojg4s1OyDwBoE1N2/wOLl0mdkcnFU0wqTk8ku+HUGKbflRTeQ3 -WE1Zuvx5ABbLUN7tYLwiyJHgZdrMTYmdWB5+KNm1c0hsgYsdKNtEJ5B34aHrpoWs -NjZeZp1bX/xbzu2x1h8Wjz54bWDMppY= +MIIJQwIBADANBgkqhkiG9w0BAQEFAASCCS0wggkpAgEAAoICAQDT7tzjUwU20cHO +PhX6Tmdh7n42dtWUIlXJp3Ykb6FuUM4wv3jIrwcwD8fVkvvJYGlAKdxt5ZaYb/BD +risW9JGvGte3QEUQ5iJPKKlS9xDogq8Cvtb5WiZgeVG52nS4LwcKixywW2XcbA+q +ySuLU0WN8NjZifUYdBRgTF/lEgVJsSIxIeX6MI2FEap/TQ/QR/cuvBXwqRRgY75V +c4Cnke/KVxcKiXjPndLRhvZOg0XSe6lJzlpOHVofmmhNqx8bLOS1PxWQOE4nF7L6 +4bCrNGWhZNkVAlewToONyB0h/Z4eOzrSuTLQW2lu0i5CtbjJtm6+S95yPtj02boZ +GTJXz/16BHqGjiAWm7+dIjnpHxzo0OUyeY4FMFI0X9XYXwrX1/PtleqDIZtpYWg+ +b+pcXs8Bg6/cnEffn6po2Ypy5WdKlyfj7QHkIe186GvYhFJlSa2OZZU4Suvc2VHN +uzh48qAIke0nmSQUK7Q10cveCI0W4SsCSd6gjNvPsjWi46fkheCDWCfoO56CkyOI +8gIBiP/BInJs0m36HkfSGBH6DU7ClX5vxO7VpfTkR7nyMWch2UYP0Yt6+raWikrf +Rmf8b99Xe47r2rhspbNi43RSr4bCkhrSe2MXWZbjagrOXJaEJlJfUzavHslMLniA +7I7/RZdsJGMJ0w5jBHkWMuMf39a0gwIDAQABAoICAQDQrTeOFibiLXNZMkTNq/mb +sgtQcX4nWA+EeYGYqtoXFzWPRlkVsVGwfzlTEGEfjfDUnuNOymuYOVAoNAx1k+cG +tbWF2KCo3REXRIBFEsTxTAwq9eISsDNtdseD6phW6udR+FwNQlFv0m8yyNY5X4rz +Eh1TRcWxiEqv70PU+AOA5gs2uZwK11fhmN16b1SMzu+SctxflPxdVJkK5qndVZL4 +dvxXAbmVJF96tBs+f392mxC7fZXpqHke+RiOsVn1liS0WiNPWEGLuIghA6zDRtpz +Kiudxiq1jPl313zoBSN1vPhUmMpx4jBUUFwAj5Id+6XbdjTX5Q4XOXNGV0r8r+1y +NtbiDvaDbcwNI6pD60et4zX3RACdlRegN7KgWBlOYjEoTFtqOaA4zZSTpxz3l3oy +Fb77Fv0c+esBcsawr5YhZ4GBCcS5lYtOJuBfGaT+5xNVBJXyN7Jg8wPIpFqCakVw +ZA7BxYuoi3ZZfjVUnCW9hOEFaWhnjOMdIUV7Bkpa1DVwSkjome1ppsxOEWAE6j8K +3pNBnOFasdbsO9+H/KPADaZGdt7+ZuSrJfcLBRTvfJwQeDn0UaEmBVuXZZh+9nK3 ++hTVUfHciPCYzzEeia3vZ27Bc/NLuNK9I3V1+0WY0nysC2cm/Aiw3U8nQB6usyNs +4X2qjO01PImGYJdwRjZS8QKCAQEA6PA3lw3TKg6LnDBF9T0h7DpKkN6Y08DRDEMu +KiGmRFpqO5c/9+B/yBnt4Q/iYlmuuEMpCATan0vxfDc3VNnFRN1aOLktfwAjP4q2 +Sk69n6o47QtEJWhZWoNsZxo6RTq9Yv16IfXH532Wg9pBGuVbB0DXKE3kVetozo9L +oCFilht+yc6F/lKRvz5VqCMuouP4/BbVU8VL4xobJRKiJXanUtZ/vvu8IzNiudzl +Fl9/YCVBnnK9+3KLayI21orqPkiVNMlPxb2rq42bUmMmiUqlESJfT1mc5QILevfb +Pb8VbzsaC4rW/5pYKzIHxTZwf3YlFW0xJuRqOzQLZKqdk4Sl9wKCAQEA6OpFKrio +8DjQBOjMiOeeizeKE/24dee/YgCF32eHjffXi6ltLcx0GcXEvboN3yhk7LqCgXLj +JEeaJjydOAbTA/fYM3rKEQMPRmOS1oaZsQiwbAfWBX0RhYgTZbxTHOuysC5sNGyI +vsFHOUF+Aq4A0lPP/Pkfd64mhlUFaBTH6CrgVEPvD9lU3tItrB7UIKv1m19L3VGy +2BzMZn91f/gslVR9/rUu7QYIzrvHXt5DU7AgU4nMv5RO4vJul1xj9fbFJT8N64zE +SFXm7ICOHmz0V9q3cZ4pDpeYyr+aJnvi33h0p8RFurh9l+yZatvuNVYo2VSMyVz0 ++o/8ZXsZG0zS1QKCAQBQEkMJqbbsvhM/nVhu9JS0n1UCd4IRSDaUAIPd97Mvy+S4 +l0Vl78X9AuzCcLNhG3Lz5CI2n3SHTZO1H0Xl5PXqtixNU1pQTbvQG96NrIOCXwEy +eqzyLuyIMKUSy/E72vU/EeLpyZLuAD29FUlL9YtU8vUC5cOXxVRBrQBSCyfYcZsj +w9pLkhaHpuXK5k0hsTvla/Po1+k+J7stAzq5pg/ead1/yQjjEMbfHQ+ioTEn26ay +0L59sOI78YboONhMkKa5fnntf0WDsymxKGQWzZHwUeBxZlyIXCuWGZ1DnGvB0I8j +aJZ/Ro8ISiphoFULe5FDzEUZGWtq7nj8IaoJC5ABAoIBAHSiK7Sy99YjfPeYZCQC +aIAqdbiBC0fvnwTCYkd3HohXHdbQJt4STelCAb1IV/3xymhJFWUhVOEsAeM34zrV +IAISsG9HA7z1HUcQQc2vZXMjeMUAP4pMEOQfOidt4OjQlpBnDsRJvUhYnQPs6d8J +p1uroLF7Rui1ipEmLijqjxiRr/hCdIopOLjD4x8X6P3bXohJdkID0V0rPqftF7fX +XOwCfX142WDT7sEngzTcdDaKMgXjMWaWK9K1t++P0nau9hYqtxvHR9s5OQQhdAmE +Ye9ElSwOGV03SMnunJC85OUm+rOwM4LHNA1J66F5NJX86UO7dSy4oex/AdMHi5EC +LWkCggEBAKLSqe5dXaPOp5WtQGkvAttEjZ9zlk0lfm+a1iBe9XPf1xqKy5KlFOsZ +8SCm+Tbjs1fVSgZYkAJOeechxNYKwPQhpYrTSGFLuXC8YmRLlbpmnmkOSlBP9ETC +9hId5bNcsxrkGnjK1kwa8swFt5b48wMP4PGus+v1waWsk0yoQ7h00Tp7kw0Hk9sK +QhHhRD4GDE/bcjnMnJdCjKVvMNBwdzgeb6AcV7hRJEz0ovluVw4p7f55v/LG4vIT +ZH+srAQmI9Ll+jkJr3OBmEGAJgF6AVIDlU6h2BPzE6grYOMrKW0eajroEXk6mZ/f +qeX6PmHA2jjntyyITMqOrUYUFEXYPVQ= -----END PRIVATE KEY----- diff --git a/tests/gold_tests/pluginTest/cert_update/ssl/client2.pem b/tests/gold_tests/pluginTest/cert_update/ssl/client2.pem index d57294e0bb6..c5bf6754491 100644 --- a/tests/gold_tests/pluginTest/cert_update/ssl/client2.pem +++ b/tests/gold_tests/pluginTest/cert_update/ssl/client2.pem @@ -1,84 +1,84 @@ -----BEGIN CERTIFICATE----- -MIIFkzCCA3ugAwIBAgIUCTwgIdqokJhy+6RByaaL9Qbcr0IwDQYJKoZIhvcNAQEL -BQAwWTELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAklMMQ8wDQYDVQQKDAZBcGFjaGUx -EDAOBgNVBAMMB2JvYi5jb20xGjAYBgkqhkiG9w0BCQEWC2JvYkBib2IuY29tMB4X -DTE5MDUwMzIxNDExN1oXDTIwMDUwMjIxNDExN1owWTELMAkGA1UEBhMCVVMxCzAJ -BgNVBAgMAklMMQ8wDQYDVQQKDAZBcGFjaGUxEDAOBgNVBAMMB2JvYi5jb20xGjAY -BgkqhkiG9w0BCQEWC2JvYkBib2IuY29tMIICIjANBgkqhkiG9w0BAQEFAAOCAg8A -MIICCgKCAgEAznHm20mKyPXTscvtexos8cYq0ccHKSZFKjgkWjGm7Feahv67bTuD -Sb4k4pS6inkHDkXMrcibuELBQAbSLS54X29sXwo2kkAAdAjvM2cwJWjNsTHzaRov -l5tNfQgbQv8q1zW+ZJpnllPx7O1rKc5JBFBH0/HIr8GEFAiYOTGz9GZ6zuK1zdc8 -+X9p7UGDkcBZgCYagdPKK3zrH1x3DM2LMcVn2XuZdaB2aT/dOQrXFET3czazA9pz -0rLvk/11JXPokC/6cWa0dm6Ns5hfQ6SnUmz6wBmsloloIGIGG7WLHw4GN0mfFfid -WLaA726QuCJTDDb1Ys62aXCqhzLQbRj+zMRDIFBZi9T0jri4tnTeAZbIr64tnlC6 -GhuLJ+8j3Y0r8Y2fKJ20RlPe1gbyU909bgkUEZNdvcciDsRPDQvUERa8JqB7w1D5 -xMOr+5Fh4dYGMHQsulL52KMt/vvE8rhUg4ketaU/Qwnww2TtEFqt1RCg3+H6slAZ -MiZFmdSlVcFTG6PHUf2xVelGEFf5zDhy5lcKxiEDNwAphSyaMAH/yjRRz19pE56M -WKaFvaxqdGcSmsxBLBSOVSv1zGZEGvekZTbZT8mxoqyQXR37YJIHWhpl0lUmprG9 -6MoiB4I6+qagEDU6dSMXtE1kRHfBLdG95Vg06iP2kx5MFz0YBAw0v0cCAwEAAaNT -MFEwHQYDVR0OBBYEFM7VrHywWs66Hzjb4eVFnvyimF66MB8GA1UdIwQYMBaAFM7V -rHywWs66Hzjb4eVFnvyimF66MA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZIhvcNAQEL -BQADggIBAIDsmn9uPb/cpIpludUc+9zMtvNkUjAu+Pu3N1p4Z3Ll1sWxWn9Wv3Sb -4+XNKg47G6MGrJBwXkPYtCtclIUQXBiAf0A3vTRP1vafucy4Yr2EFVEFMHUq39fb -fXuxsgyUbv6M+iFj9y8s2ebPdujCBsf68jH6tEBWgCxeocs+JG1a16r45ZgJFU71 -Anyw3DbBwLugVRUXVYp+W5vPQQFHwNSer2QvoUuESruakMWZopf0kiNrkmgRj5o/ -67KIpctsQTxawGr5db7/O2KNyXHdxVSknKrpqWwuvDZag1QE/5Le0dfT91lYC2v1 -597wqgigTb/cwVmVPMCHj0tywVGRVw9Mvq5eJKzawl5k/wvDgktTiJ5stQEA8Hoz -DiRF2VOdlZ7LI1UunaVsez4Wk3JmA7ltIfkiet47qWrxXaVx3UD0nGnaQ+M2XJEL -xlA9X9jqAJMbAcl0Jt+4jQ4ffvm+7RyegLLsVxOw/EI7jyoYb+dtV6EeEM1Hd8Yx -903Tbi5zLX0UlvVTxYPcOqYSi8nj314d8gseeC85jtixm4dVe9yfNKsa84gMPjFU -SordVG2iqSQQhWC2TZD2f8cMvZti/U/CqDzYiRPnaSPSkJcOLDGSGK1dPJDsph9c -Urw5UZOEJK31mIDbq2jQLrmcyj6ytBqoFMSidNutzA8DcENzQUL5 +MIIFdTCCA12gAwIBAgIJANvHWsQZl6X4MA0GCSqGSIb3DQEBCwUAMFExCzAJBgNV +BAYTAlVTMQswCQYDVQQIDAJJTDESMBAGA1UEBwwJQ2hhbXBhaWduMQ8wDQYDVQQK +DAZBcGFjaGUxEDAOBgNVBAMMB2JvYi5jb20wHhcNMjAwNTA0MjAzMTU3WhcNMzAw +NTAyMjAzMTU3WjBRMQswCQYDVQQGEwJVUzELMAkGA1UECAwCSUwxEjAQBgNVBAcM +CUNoYW1wYWlnbjEPMA0GA1UECgwGQXBhY2hlMRAwDgYDVQQDDAdib2IuY29tMIIC +IjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEArnASYqQnNJeuPmIDd1gpzWNq +tU7S3xqRBS0H4WpbJZwP7jqPvx9JVACKMeLWUXQwmYsAkRWp/0cBY5oTm16xJqqd +oYNy3N88VCIKzU2D2/9rgiry80bAMlTnRp8XIWh53YSPdpe7NcwQGdtkIzPelyEI +AX69utWp2tCXb/ILDo1Do/3DJvtyUdrwHxoNMQVEtQszy/AaqXMkSQPd1qiH+xAy +ksLDcTD39D7EjO0nw1aenuuM9XaCUr2CMBtmhg8X9e1/KMwWu8vGTo61CB5EZqc2 +cBLO92Qa8ed/srl3Stve5CdHrjNAdhKCV4laeDfq5rV/+H6/AIhe6V5lpLrzD0wb +5w2jyZoQ2dy1DQ6/m7i4Dsb43YHuyUIU4thAgzdLDArRMSySKMfwk6/1hlsw2IJ0 +2pZyXJUHDT10LS1HHu7Ee5rEG+Z8b38wWegP38+o9Dxl2SthiXneyVbs4dws7WY8 +R+s4f77JeLgOORD1HRsEXyZIgOwVXcQqlRz1UUZ3cta712Dxh25SAux5pBdKkvGW +NxrjPzMSeZdUHpkwU6sYwrfOAbi7LEGOCGybuR53XYF5IQD80tFg9RZVExmmo7dV +urx8AGHkRz2PUk4nu0arUYKZtUhjofptpsXe13iMX8+q5hPPiAltq6wAnvHrJd6O +fmCLHyWEp2KYjL4XnAkCAwEAAaNQME4wHQYDVR0OBBYEFFi/roUN6nTGm4x8UoFp +PQfW2+gyMB8GA1UdIwQYMBaAFFi/roUN6nTGm4x8UoFpPQfW2+gyMAwGA1UdEwQF +MAMBAf8wDQYJKoZIhvcNAQELBQADggIBAHAKIdQNwvtMqVFrMgkXQSZyLs2NcWyE +ZuG1hhGgmB7aEPIMvFXIvc2HFHQY+8kH4NJLuEnlXd+8F6ZkhTwFfQHYQDNTRE/o +6T+lOa8+xXexzjpWeLljpCsxyFvRYvMQ0e+wv/oG4vbLciSOmM+5dSOtAihwwlif +AChnK8f/UOpjqO9KIv2nO4DMCyi4BTE0pNT+HR0PjjYIB5zqJbogqSxDn3yLU+Ay +vrOAeXj4uUQaOrDF4jsAv/XvPrZ2IqrG5NhNsiCZl7c9uHwvI02F3MMq/WrZTVpC +YTnuHYL3Z77iQyhrj7mT2n9BnVEDOPGIY3r7QN5BeQfCqL8v3K4gHjY8ZEKTIGCi +zIz4V9LkPRwJ2QiocPRaApaG+NwLWPXQvvL7QY7KdqiwYMe0/zhPIWR0PvAyN2bk +0DqeKHBm2ySyI7m3m/sU4RUkgMyG6DI4mh4wp6CMpy8vOg9XUbTyPdd+7mqwNLkL +/G6FnymCbj3Ik+9303puHleKycbRbc7x0k99BRLw2NpZzgALt9XOcBQjA/sN2c5a +VYabUYaCpWk76F2s5HZtppxkyNYad7mZjiMwk82KKq0/InVFTIl479GY0ZAsv2St +DrweS6pvOXLgR1U3eyOlMF4n88DErdcREy1PRd+mSAvWZu0+kAV8+bbDRTQaU1ny +WeRCnlpbwsc1 -----END CERTIFICATE----- -----BEGIN PRIVATE KEY----- -MIIJQQIBADANBgkqhkiG9w0BAQEFAASCCSswggknAgEAAoICAQDOcebbSYrI9dOx -y+17GizxxirRxwcpJkUqOCRaMabsV5qG/rttO4NJviTilLqKeQcORcytyJu4QsFA -BtItLnhfb2xfCjaSQAB0CO8zZzAlaM2xMfNpGi+Xm019CBtC/yrXNb5kmmeWU/Hs -7WspzkkEUEfT8civwYQUCJg5MbP0ZnrO4rXN1zz5f2ntQYORwFmAJhqB08orfOsf -XHcMzYsxxWfZe5l1oHZpP905CtcURPdzNrMD2nPSsu+T/XUlc+iQL/pxZrR2bo2z -mF9DpKdSbPrAGayWiWggYgYbtYsfDgY3SZ8V+J1YtoDvbpC4IlMMNvVizrZpcKqH -MtBtGP7MxEMgUFmL1PSOuLi2dN4Blsivri2eULoaG4sn7yPdjSvxjZ8onbRGU97W -BvJT3T1uCRQRk129xyIOxE8NC9QRFrwmoHvDUPnEw6v7kWHh1gYwdCy6UvnYoy3+ -+8TyuFSDiR61pT9DCfDDZO0QWq3VEKDf4fqyUBkyJkWZ1KVVwVMbo8dR/bFV6UYQ -V/nMOHLmVwrGIQM3ACmFLJowAf/KNFHPX2kTnoxYpoW9rGp0ZxKazEEsFI5VK/XM -ZkQa96RlNtlPybGirJBdHftgkgdaGmXSVSamsb3oyiIHgjr6pqAQNTp1Ixe0TWRE -d8Et0b3lWDTqI/aTHkwXPRgEDDS/RwIDAQABAoICAA3ZzIBX7czP2XUrVnKU7gEG -p6bNcKiNHcTYYW//ttBSjFaUTqTkgkl3TWg6TE2wEw4dFr9uHyx8phSSoJVRcdgN -VLsHp2OmkaE3XD0ZpjxMTMifrlIV5K2KSOejnJihIBGyVGTRizOlzCx2PWNfqLx2 -WmtY5HsOQ7tIXFYyBH3YRlMNGN+Hmlqu9r9pTtd/jUGwzMR7HixOxEOlY4NjYvxi -3zXTOhePPYKQO1pxZL7CZcvAFsCibnTrdnF6ZtqtbSQHZAkpt9/eSPmMto+GQ4ef -Mi+jSVYMQAU0Lm67fmsF+NrKwLarqHrkO8BUrbb87o96PHp1nf1zBq3tddhr/eRG -R0+e72Dec02XMsof6d7AmBRsEJwvz7Nt7MGwszVxw3O1nzeAlpCuq/7eJrNPCfT/ -Lt16yizf5f36KGVQsNaC6saVWKUSf3I7OwZJrZpN5mbkyszWGuYEZL1fjUVYBYdM -gXOmCmHpFUeQRENJ40PuJHFgYEhJ5ZT1Mp+p3YBe4aGtix5GntP3uzpp4AUKJ3oB -f4C41a6vOieag1ivKBjusVv/j/obfn41x/BS258X4uzPCm833c6iY+jofIESyclp -kqNoAoFTGmT7sIqbQRN7Jo1H6ZbuOC1Cp1m3GmJWfcjZwif1AXr+JnzdgmrS/Rl9 -mbBX7cFA3hyoQ4GpU62pAoIBAQDrcx30eBzeEsdYbSV4F6vu11yfoVbMTVMY2fA9 -Vp5vtDG41N4epw7nbSkSuQ0ZRpMCu6aio3vEWIZfk35Xm4oP7BHZ0fglnH8NJouz -dO1QGK6t3wX6g0rB9mP4IscWIU7B7S6WNbEW32X3B5vbJl3BpMiURwdMK+O71zYB -Ip1uQeCeJrI453BzpwLCcP4So4Ib3JkdFw94BI+pHhH0npNu+5yN8S9LbsLpKBUJ -OuwarVBsaekxuNQCd8vqNQNVBI/wjRK/9kguOTqW0SiWdNmf1LyRqVIL7YqRAVKN -xOIAjkH5BfYeEfWIQtEdV3bZu/HWbN2Cj39XI3BqgkxOCVKVAoIBAQDgdrQJYXlW -eL6sBOzDBPEyZzjZRf3IyJQljl2DDkWFZjJ3Ds98T7CRm64cIW/iksVMzSvoEpl0 -j1GKQ4kNLKNz9iD+1W92Us7m1IejekKCviBqvSl+JdoiCCsnWA9+3fOCgbCKwZx1 -4Wk3aTx69KRZJtqH4o5NPENd0k5fSnbceE7sbex2Lv4ryPbQQTKgvnCTWzn6pEdi -XjNdPHhELcyWRtiUVlE6zV92GBPzl8Bz3mi1tBSVG1dY+CIdE149rSd2YsV4tQvN -A5T8kTH/bvGl8Jn1KPoQ724vizsSJgQakE0ReFc+scoz9NodQ49pqRPHdk9CFprB -5chlAJ3m9o9rAoIBAEIqUoOt8WbS3iRSX9I0zMNM0CGn5E17eVuleyaxncqEV+i6 -IUV56u1MNtulFzJAK/X7p+NSj+hofDKFr16NPiolTArrP5HKPcYDTAT9WedFWGlS -IEr69Fo3lHZZx5rHd2t17L6XjhGAbBYUlE7spDJTzW4l274jI1dZLjr5cEZYyveG -plTpbSeDCnp76FpyipCr2HddUKKInZqH8cHNgl8Q5DjbS1Amay28btTuMwV4KP6e -cMLhTur2oV5K0Ynlw1F1Q4ygeD5NJNLXKlHFupZ44RkJ/R2O/n6rYXinmF9RmuaR -L03Z/Cbzp/JX7vVXJKn+Y+1ZyA5DzkaQIUNYyVUCggEAQ4tUx9HeGmhBMDBXMqQl -FH69O5x1Lts0wUxi1VIRF4BWRT9erlComFhZfzuMmIiD+IVw5efa55lM9yc1cZJy -KS3yZdzCKr/mZM2ld0sOApvF03jSqJQpXL5Khg9YsluFsEroXgi+1TYcXEE9ot5F -KlKnxeYl3hX5S51CWihlNhi53ymA01t2vqQ9qRNFcdt8ssrr2oFevboNCMxugE2r -17i/6XtD/EbaqAW80zth/Tv7FFp5KxlMIoigc1FltXeKfXRhad5JC8s9JPdoLS4s -ZzvMiFppTXlPFd12zBJGf9vWZSBqWIJVj2bpz46J9EidnBL87K0yqpBDyijyWxLs -uwKCAQB5mpi+DSCvsVRu5vpR+curP3rJD4drIne3I3WKOKdDhGfP/eN6Zxh/F+bG -0oFPoW7bwbjmFpMXWSqt96SZY1tvfuAbFN6VtTzLhjiC06gD88KRyM8UPAjCuzl5 -hYST/Qgd7zLSjReIqY0/TCR0/tb+GXQKGSBtfX6+avWUqE4n/alq5p4hY5RPTIoz -SAnV0Jpz2IxjwYMsM9GeuD1V/7Ad9wliZ7Xh0HpBykOlZRxAC7WMKKqQJKCquGiw -U7P1zvWqTQKmxGEe6aMTL0aOv3Vqr4wda8sQGqts0Hgs6/2LB9c210MB86FE+NTm -7fEmsFnjM6wdpL55qregGgcYdPuz +MIIJQgIBADANBgkqhkiG9w0BAQEFAASCCSwwggkoAgEAAoICAQCucBJipCc0l64+ +YgN3WCnNY2q1TtLfGpEFLQfhalslnA/uOo+/H0lUAIox4tZRdDCZiwCRFan/RwFj +mhObXrEmqp2hg3Lc3zxUIgrNTYPb/2uCKvLzRsAyVOdGnxchaHndhI92l7s1zBAZ +22QjM96XIQgBfr261ana0Jdv8gsOjUOj/cMm+3JR2vAfGg0xBUS1CzPL8BqpcyRJ +A93WqIf7EDKSwsNxMPf0PsSM7SfDVp6e64z1doJSvYIwG2aGDxf17X8ozBa7y8ZO +jrUIHkRmpzZwEs73ZBrx53+yuXdK297kJ0euM0B2EoJXiVp4N+rmtX/4fr8AiF7p +XmWkuvMPTBvnDaPJmhDZ3LUNDr+buLgOxvjdge7JQhTi2ECDN0sMCtExLJIox/CT +r/WGWzDYgnTalnJclQcNPXQtLUce7sR7msQb5nxvfzBZ6A/fz6j0PGXZK2GJed7J +Vuzh3CztZjxH6zh/vsl4uA45EPUdGwRfJkiA7BVdxCqVHPVRRndy1rvXYPGHblIC +7HmkF0qS8ZY3GuM/MxJ5l1QemTBTqxjCt84BuLssQY4IbJu5HnddgXkhAPzS0WD1 +FlUTGaajt1W6vHwAYeRHPY9STie7RqtRgpm1SGOh+m2mxd7XeIxfz6rmE8+ICW2r +rACe8esl3o5+YIsfJYSnYpiMvhecCQIDAQABAoICAFeuQa+Pb52B+LWmsrNUf36b +0FFeU6SNSAPgBKwLt9EJxwAqM4MQCN3Of6hfqva6fotPUXOb/h0w5cPY+K4KROV4 +sU80MZJQK5Tmn6NeqT71/mYpjIE4OPeTscTPxuKmD0uHjE1CK3n5HsbAfm8rbWFt +2wZVXK0DJSO14ckFUwV3gF53+ZYooef/Mt+PSr4vNcAC54BFKkGufLiu1t1FTUSn +eaRHfdHxmVEOE+H6exg2+/hJ0kq3zP4l/8CKpVgRUivtuxMYsvymiwGFvf1cqWkn +97cR3TKBAQJDKO7/RrU8G6d3DxLC9E3lM9soUW9IPCzHYpHXs85ivcJ0Dodk6TMc +NR7RTiyNkSuc9FAWmpxmWyiiMW76DbmxvLyZoZVtLaOdY0lcwKYDPwTMYPaQmqhr +HSQZYO2voVvdoVav81U1FBay8VbJsDVbT8tfMsmR/aJoi0tjDucRS6v4Kfj/DDqO +pLVVgxdtc2K2nC3CfGoKWgcyi0aLCdNXxgV/cy/7qdIjN+dp/NXm49FO+U+WI/zU +i1NtbHB5+R0Ey4PKAqTeAm3aHZsOni8Fu8rQIPGadhnaI75++bRrQpsrW6uwJ07A +S7eb9rKN7p8X8xg1cAS0uHYCCRkDwl0K+oC5GYgtteShdrqb5VzVoTczJugNu3/D +HeoqIPgiX11vzUI0m/lhAoIBAQDj2YEzVPUddWn9VXfOk8K9k18ECzQONcjzGXRg +EHItPy1BWr38Xc38aw90ygqZrCP9qYA1/6yPneGCh9qB1JelGnIGSF6w8ZTSdtsQ +UzP74ACBL3nghZOVuKE0MBCmotuDGrbmKtg77t84NkhpBQG4xKx51SJBKsri75DO +eqmkfBbUN50dnbrXlXZL7IH6vpPy9kj6EHD2wa0fUcoCVW1dRn5p3bIwRXUBbRgn +gN7SGXoBL1Bi0AYRqVmTjfX9DhLeM+ELX9pC+cuAH5vs887mBZUB/+QvLuJSG0vx +gUp+325DKdWToefQtgXgEouNSUyWZn8I4j2F2VCtn0PtwonlAoIBAQDD/T3P4wNV +cTrjJpihvdCMq74wnG2KMQ2d9zOLjWAVEr/ZduUVTOfj7kW9Uwgbl9B/N9CC0ehz +lVC6pkk9r7HTBLq9l8ehUlllbB6ogJAADlyUU9NkVMFvD9fwFcnxYEd8hoPWiAtC +LOVMz4dZwgj1wwkJ52LIsVvkx3bX1o/UouSqq4+HonHmkJf7iDesVWNJOjR3tPcA +E4HNsMpD1uICWkO1qGbwMje5ph4i6dF9h6YbQ/NRkIpgA6dG2uQiZq7ZJOW9Xt5j +K1mC0wMT4JVhblZad3G5JJQI7+06Jq0Do0IIYfKngDp5DBlWIf5n4lKl8SdVNfFX +nkjVf64a51dVAoIBAQDf9K2C6BHZ98vLP1P4j9JNhmpRTjXHdeRj1nRF7ERd5wgz +gd7WDNnoLs9Naz4KGGJz9URZT0qtC3FaXPBQxsabeSf0tvOkFoDCciMWo8Gn5GB6 +mlDAeGs9MfH5mvjy4YbEEeEO0bQjDMMJvIll+I7dryscwXre9RHhB7qcyYKvc2z5 +AQUE1EGPn5BcRkboUKSZnqzSZpCyamIhM8JMku1hTmX45vk7azn7weCJC6l4d6Qt +P/VSeshMWdn7KAFOz6OKFxIvnPKq/F6jO+6GdIq8G2aARaZjHkFElSILaprvo+v8 +RkVwzCZQRmn1+iSDJjYKawqYj9ALHTSow7AAJkRlAoIBAEwIpf9NkdCHAhJ3ucUd +z5eTVVCkjzaKez7qVTvxl/H0+SrTnmIaGOhrWvDtEnsp9YA1VhY6exTEO7J9qnJd +A/7amdvN02u5d+cKAAbdAYCUEcSVlLAa1YRRwDNrMJINCQ69XM5QrxV4N2kKJfdj +eHhntsz50HANppnl7CwmGFyk6VNxBVjYY5nQRLYXjYm1ArS4+Pvr39tOtg/nY0DY +T4wcGqKR94V95fKHCLAAx/4MLxvqy628BoLjR/UMHG9gKLsjPmGuOVZ9zdccJ9iB +S1KmLKgiKKNsFmY1NnJV6M89efOolCv5ajXlxLqOq5T/z5KDMQA00jFsvdLeGtxj +HZUCggEAfAElAAZR0WHhP7tSD8IuVIQ0K/6XpTb1VwczUKtwVO4cbvm29uAFcJQL +LsFxu2n1FAH/ASlamdd5B1IcgiMYNR/akNPcxlEvP03SOti011J3h7O3ftjKG8Z7 +9kreYs30NXmEC6ONdN+o2drHb7cPCZeC32j0+dY6Hb67QSf4mzO0CoYzAJIZigxh +aO7D2XmDLLOu0b+RSMfiT7rFQSk4ZclY3rIB0qcULybdoFy6/g8k5ei6+w/hKMFK +7z+OUH8CTjWf4fYQjSYsbXEErQPZwc8nGZsBzqceeuCxyRRKtJ2IfiX4D9aMGFDL +rOw13zIpN/+aF0W+F5vJvtTwGMnq+g== -----END PRIVATE KEY----- diff --git a/tests/gold_tests/pluginTest/cert_update/ssl/server1.pem b/tests/gold_tests/pluginTest/cert_update/ssl/server1.pem index 528df5424af..2e1c907b1b6 100644 --- a/tests/gold_tests/pluginTest/cert_update/ssl/server1.pem +++ b/tests/gold_tests/pluginTest/cert_update/ssl/server1.pem @@ -1,84 +1,85 @@ -----BEGIN CERTIFICATE----- -MIIFlzCCA3+gAwIBAgIUdIFP75TySC7M5O7eQsGXeUtIceYwDQYJKoZIhvcNAQEL -BQAwWzELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAklMMQ8wDQYDVQQKDAZBcGFjaGUx -EDAOBgNVBAMMB2Jhci5jb20xHDAaBgkqhkiG9w0BCQEWDWFsaWNlQGJhci5jb20w -HhcNMTkwNTAzMjE0ODU0WhcNMjAwNTAyMjE0ODU0WjBbMQswCQYDVQQGEwJVUzEL -MAkGA1UECAwCSUwxDzANBgNVBAoMBkFwYWNoZTEQMA4GA1UEAwwHYmFyLmNvbTEc -MBoGCSqGSIb3DQEJARYNYWxpY2VAYmFyLmNvbTCCAiIwDQYJKoZIhvcNAQEBBQAD -ggIPADCCAgoCggIBAMyCp3PLf8wP3cXhvrRuuKlRod6R4n3gkoPtwfvlsf+TXwFk -mOVCkgBiXEQMOaDXHnCJdYy/I+PEjR11ipigwU13b9QnWW8DzSwaGRO9lXlA2E6J -cgUe2Wh0TD4iwpUnSl3XFObeqU5+Rp86qt1zDhw0Yb6KS3EpUXf7HZEHAmfjT5xR -Qsttk7J9F5ldi1gdxoLEh7LuLG+yGmjvdNWXEoLAloTOJdyjqmVigyzDRWXMZCsN -ykbMB5Esa4RrhCxtm1raLMDADn3qKwdlPVh2ZPlwXo/5i2KenMMBDCjYZbYkVjYT -vKQvFkFPzr1DNbaCwRuOAkPjsnKn+nz5kKROFgszIgKyw3GKWpteG2IIP09s+yHC -LIe45vkV/VyJh1/EIn8YV+ahObCTRLoY1c+PQKH5FwXe0329ooT8WDr8RwM6szXB -0FByNoLGTm+A4JA55H57g+ygbn65bYQltWPGiE+oXI0fd3DpSoxm3yjYYhD1/7Sz -Cm389gaHoTS6pm/NOXXwk+bN1Qw1DjCOSrSCjiO2rAABMvqZakd32p9FhQKr7ujd -ke1GZpRAEY+XL9RUby2Z/BhMC0oZlSGwSwKOpyGdpFBJFfE306qWsiK2ktzaQorP -R265ozBbLu+tKln62T9GnuSGt2rW+tgZ/tiSEP+WP4uZhA15JudKTL/t6/eFAgMB -AAGjUzBRMB0GA1UdDgQWBBSBfLDCTWPHivDuRZwKnvuddfAwEDAfBgNVHSMEGDAW -gBSBfLDCTWPHivDuRZwKnvuddfAwEDAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3 -DQEBCwUAA4ICAQAh+3OX/mxs0oyZhWge3IDGNoRyjU7ymYeOAnO33JKMmccp7umQ -zGrB4MrvCVnnazz3MMeEOVTei5Eg7jWtn/B2+1X0shqwq968ZE/T4I1b5Vc8xEKF -b6nRfuHlVSE0f8RQjzqMiHOelkLPcH3PPPh5shQqwSpAR4zaGt3E6f/QrlU9FPet -jORYKgIUGkQNjBZNpcm7OVlr56rvP1e9MZzrEZkPXiqwR9V3ocDu4V6T9uXsBM+8 -5ybREA1lKrDbOi13XU2+Lr4VlJg4CpEfomhhg2knpO6Iko31PT4QnIH0L7WrX+Hv -ww4ga98ansylCTFqT8Gy8ZvK5GStkh6+MkGBtGk4/5QTyc7mZcrhZmiNnaBG5BJB -SRrTWmARgUc+S1M7FODWYdH/abaHkRJJasS0wdOQXnufuuswV5DVaiT1eLpxKXYA -8N5QiLGVz5BYkWlccRu8o70I3kXzeNcaTjHXpFhtZpFAatDcD3A/7BHicgZxpXH4 -1cFX+1FCyLZEhBwkAFZ+Wy938HKaRlZKT89WGHtcJo32QNDgS8kiIWjylFMBaSR6 -MLimU0iSJKrUr8taWF+zxnq7NLDY7Nm2nFSVrlvzw8faF+cHUvxm4x8BSdnXD57c -5FNWfVi8yPZD13knRz0MH/vsoD1o1nHEKPFyw/gW+uKA5jMqng5oK+GOfA== +MIIFsTCCA5mgAwIBAgIJAIJfx7JYMfIeMA0GCSqGSIb3DQEBCwUAMG8xCzAJBgNV +BAYTAlVTMQswCQYDVQQIDAJJTDESMBAGA1UEBwwJQ2hhbXBhaWduMQ8wDQYDVQQK +DAZBcGFjaGUxEDAOBgNVBAMMB2Jhci5jb20xHDAaBgkqhkiG9w0BCQEWDWFsaWNl +QGJhci5jb20wHhcNMjAwNTA0MjAzNDIwWhcNMzAwNTAyMjAzNDIwWjBvMQswCQYD +VQQGEwJVUzELMAkGA1UECAwCSUwxEjAQBgNVBAcMCUNoYW1wYWlnbjEPMA0GA1UE +CgwGQXBhY2hlMRAwDgYDVQQDDAdiYXIuY29tMRwwGgYJKoZIhvcNAQkBFg1hbGlj +ZUBiYXIuY29tMIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEA0FB6j5Fn +CiSCxLmItA1zD+MG9YsZJC0p7Tpok4FBe1YrAIR4Zb4HjuOMpT52CPvE0PVzpGW6 +a92oIjpvJHwgJUbiKaaHgZ9axtA9fcWlomrYbW0LxuWJ0BiAFsg0qOjdEo3terGz +ydl7PXlER5Sg1LVI+j8wfzGF/AAlfuF0J4hX6fvjOliaJXHYppp5vrCOFid84oPd +n57Wi8D6HtX0/74rOEGbrQKrlZMVLZwElQ39Cjit2KSki3GRCuQf6z7S8HUo3MtH +soIQ1EMCJZnLXvLn9LSQ2f71V+FxRnqSHldE1KCIUrhlyuIAILbb1EFCWFi5CtvU +WMZkJuUOp2EzMC1ozFnTCIFQiuMMbKc7bVNE3hTx0rc1CLxcY6PDt5syV/CnRAbA +/1nKe4fksxQuSCbr+EtAlmM/ywXsqlpzpXCGXzK5RFnId50VNGmwasSzacpMtFAC +yA1Kh/5/lRveJa1hRmpiAN+wsJVWDID1ly2HJgF9untAYgwtRGENtHefc67VyzUS +QR3duDfosnH3gpWO9W1awKaCuw247PTm+gdJjy2eBAXiqZUr2Z74q6uUltkJFDWp +v3ZKQ/0KI4oPVU7ODDaTCm8zhtQKJGOM9v5lg9n3Ll+ytnr9M1c3nluTuuiqo3yb +bHuXQEIr30YvvHWBBqwoK2w3TfzPDl9ck58CAwEAAaNQME4wHQYDVR0OBBYEFAKj +IFQihP9aDp90JXHrktA0PztbMB8GA1UdIwQYMBaAFAKjIFQihP9aDp90JXHrktA0 +PztbMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQELBQADggIBABFcWR83GMIH2o37 +ifrF3m5GYC3WFL1mkVb6SctL830Vu9vUnki9fG9VD3SuXDvVzsOPVMZ0Gr5qkN2l +wJWx1A6oaKFBHxSiu0gmkAnsQ5lePZCRip70qmS58tNDAIflSf9Zd2rRK1hEdclp +FAECJXZCg6Uxn+CHIjossrEyZPFlBFrR8Y9l10a4vaJWmXCLX2gEp3YRUsRZ2SoR +sHtiykXz/DJCEKVcj5NvYsV2pKFPXHAw11DVyf630d98HW6Whe6sH8OB+jTw+5RF +fo6Ecry3mRlrYydNT7qum/aFb0+BJPbmBygiEO1PrCssFGQnJIjBg7o3y+84I7Eu +DGsL06XZ2LI+bTF13CE6YQgwvXQbjCF0cfNiL82Vp154JHe/f/W+hjM44MxZAtZG +YLjhPigiCINS51rlEySwIbRtOY7h2Jv/ArQLHe5Q1x2PycHnCgGVtK6kBxhd2t0Q +QFpC9ctU79k7wR2rn1RWmcr5g4QWGDO1YaJxojOBZkzRXcjfnIlniU+d0f5Tq2ib +HSCYoH4e/E3Kvei+PObNZyBwrCVG52sc3Xg4DGrleDd3OFu2DGIMwnhYdFAtIHjo +56I/Kz2QMRmDHxEl4gYelo7j8HHcMxOPu8s++IWdrQpCUsxSjEZjVasK7fHE9zOa +yevBkqFyV4/74/atSgC96cf95hV1 -----END CERTIFICATE----- -----BEGIN PRIVATE KEY----- -MIIJQQIBADANBgkqhkiG9w0BAQEFAASCCSswggknAgEAAoICAQDMgqdzy3/MD93F -4b60bripUaHekeJ94JKD7cH75bH/k18BZJjlQpIAYlxEDDmg1x5wiXWMvyPjxI0d -dYqYoMFNd2/UJ1lvA80sGhkTvZV5QNhOiXIFHtlodEw+IsKVJ0pd1xTm3qlOfkaf -Oqrdcw4cNGG+iktxKVF3+x2RBwJn40+cUULLbZOyfReZXYtYHcaCxIey7ixvshpo -73TVlxKCwJaEziXco6plYoMsw0VlzGQrDcpGzAeRLGuEa4QsbZta2izAwA596isH -ZT1YdmT5cF6P+YtinpzDAQwo2GW2JFY2E7ykLxZBT869QzW2gsEbjgJD47Jyp/p8 -+ZCkThYLMyICssNxilqbXhtiCD9PbPshwiyHuOb5Ff1ciYdfxCJ/GFfmoTmwk0S6 -GNXPj0Ch+RcF3tN9vaKE/Fg6/EcDOrM1wdBQcjaCxk5vgOCQOeR+e4PsoG5+uW2E -JbVjxohPqFyNH3dw6UqMZt8o2GIQ9f+0swpt/PYGh6E0uqZvzTl18JPmzdUMNQ4w -jkq0go4jtqwAATL6mWpHd9qfRYUCq+7o3ZHtRmaUQBGPly/UVG8tmfwYTAtKGZUh -sEsCjqchnaRQSRXxN9OqlrIitpLc2kKKz0duuaMwWy7vrSpZ+tk/Rp7khrdq1vrY -Gf7YkhD/lj+LmYQNeSbnSky/7ev3hQIDAQABAoICAC+gJZrjwykkcMMKZTzjpAFa -T1Xjp1klGTm7/rbIsQERsshCQxDwxcttHIuERU9diWsvt75FLPh351z66IHOvfVq -YRKI71zZB8jDcx+TwOFx5m2zuGfU3VBj9PVrZuERO1JLKkTxiYAMDCo8oVnc12Ze -FH0o+5SoyJ4mTqZdeYPz/bArhGCXbhPc6cf/btngZUBCwE89BAAm+9uAGDc9bUQh -0WuwDkUpoB7oKCAegWTJinI0TezaytBWBdvapfcqt0kbEdz5XOaZx9d7DiQxviYQ -sxTYSkt3II6RCeHhMx6Nq74ALqgVhxtCmKSeqD2OiaX/Fiv1NLNaxwyfmb2Jqbwb -K02mTx4uR9yqeY1RtagNqIoEDg5zO2gwONrv4EgpgT7F2HiaFMO5dCbjVPtPNYbP -ZD2lliQJ+kAeSUM92eRcsid8vxZXRj2ofy54Ls26LNucFF3qW4IgNsKzUpLdLiMz -rgvW34ExCb4D8JsQsJmhLQrA0S9TUzsB/PApTqPYpRneI8md5JoHOxl1gC8FCt4F -7U3PhTDRAGVvVfrz8CGkfSWhMU7vINcwqvbE4VLWF3PuejhpDN30M3KeHREkkUzY -y/ZfegmW2LG3+iGTchNS7Nc7GRRQ65Ku/4+RtgQPu7jMj57k8sDPR0LnO5kt44Sh -gYQSDBTXQX+YliYGsEvpAoIBAQD0D481MRVQOWu9fZpdxr7pWlTOqfGHQt5fO8he -QjokJ1hiA/SWYIGFXO4As5ARlD+RTAObe2R3AOQYYV24VREoCcr7gWrYJNUE4nUD -SIxZYJESPCWH0c+kr8bamWT2BTg3/BGhaWYMUhoGnMySdyz6GXP5Ll2JjeB40Ozh -YzTw0TIIhqmkIB85/KwOv4AL0pFBRCcqF1Dn9jhXlCBGhFfzbjyakFftGWdmiCrq -1+RMZzRl4EsPudvXf8cN5f2+ARNZRsLirHQVNkw0R49wktL4wc2bdj2rYFmhAtIo -cvUInVIbLYj2rHXY6Du6RMA9raoaUaC8Sl/AZqiz7H6FnFfvAoIBAQDWg8tBxwV7 -0I5K3ZcrNZuhPZKrYc1swP+Ujcu2EZWCTTQUTolusivlryneKoYBGRPbC00+jOiY -wB8WMfL8RNZrhSY45XkJVf/EOcaAudUYdmlRzDB0z1UT/P6TP5YEQkWJP4NtflMM -SkjiHFWb57K9fXO4awpv8S8g9rCc7lXyGLdBSyNNod9DPlNGLwqkqTQtW3BcI+BW -r/elvJP27tw56kcthSdwuKfXuPkOTN8qttsDF+UzX4D1rCtXBNnY06gL7NjbhOEO -2b6WSHcRM9KlJf2l334dMQgiB+h+A4z48nt5EHtju43vNcPzmGgUkbiiCWZb/49F -k78SigL+zJPLAoIBAH8zgNMbUt1uH/4x8XuAs03R+7N+lViG/HksImEmKUFglEr5 -fsfFYpwMdCs/aw4OcxcaRCMMK69ucnNWg88n3vo8KGPu7q3afH/AO3ZLoIKQtuuH -F5RzQMK3rm+OVTV4QPXE1beHxF0ViWT64hBQNsve6jfr92pS3LR7R4qs9xGwJmCV -NuNIrp29WDuTiXwf8f7PM45Xep57EKBsnmnCXkiMot937auwetjQjXW6sc00WPXj -8Zsvpinp+ef/f8FAtEHqhHY5pYLMuujghx0IGRb3g17MQJYKcIxfeQMF7znfLMTn -daQC/KThXQfW/07mLWrsMlcQeFlB6BlmYAbpFlcCggEACNjwtirOOBgW9lGDXZ3d -aF4QwY7MGTMwl2DtyPmasAAdKMVAd9dTZiq+UFJyqnLtVbh2nCDVqw8peRHgUrVI -HrEkLW6RemgYn3A+lqqTdmnT2DLSwM6YVLW3jj0uI8jT82AyPH7cUAJ0VRcUFNUO -kzAsaKvJh2psJjDmgeJ2mwCX9lJyB06o1a4pYxinmLj91O0TiklUhF7HmQdZFvMt -FBpsix0VzllfWs9fPk6/WZSnHc6Lfn3u5LMQKouhrIa2RJ+lJhCp86HZcXtVpdj8 -VCFn/8JjAjM2gajP1vqwgsgFfa3HWQqwRPBzv4VGraqA8fXvSdYVg6ofVFVq4DVx -1wKCAQBlwY0hhrcsNKtBc2G4gaAYlRV+P4gLRMzn/hCHpDNpqG4rdduLd8ftxJqL -Ky15SMEB9Pubp+ysRnkBA7kZBrWzYq9sorYy3Hv2J1usfofT7AAy4KJJkfUe49z9 -WxHVNRoVKuwAnFb8S+syg79+ep5sH7lyDTBIdjvd1tqgVnLg9xflQ8Odqm6k7BSO -M0wSV95PLzmi8TyXhYTCYT66bFIMGGeKdmc0/lpAPBIfVBqY69Pho+SMTijhgsQw -imugh187B+VObnSoXG4XrcFGexkJOQBV1G1qM+EQ611/6wEYtJj0aFLcoB0eyAv1 -JoiOSTTYKkbWxl6ZNUfJPeeFcPsy +MIIJQwIBADANBgkqhkiG9w0BAQEFAASCCS0wggkpAgEAAoICAQDQUHqPkWcKJILE +uYi0DXMP4wb1ixkkLSntOmiTgUF7VisAhHhlvgeO44ylPnYI+8TQ9XOkZbpr3agi +Om8kfCAlRuIppoeBn1rG0D19xaWiathtbQvG5YnQGIAWyDSo6N0Sje16sbPJ2Xs9 +eURHlKDUtUj6PzB/MYX8ACV+4XQniFfp++M6WJolcdimmnm+sI4WJ3zig92fntaL +wPoe1fT/vis4QZutAquVkxUtnASVDf0KOK3YpKSLcZEK5B/rPtLwdSjcy0eyghDU +QwIlmcte8uf0tJDZ/vVX4XFGepIeV0TUoIhSuGXK4gAgttvUQUJYWLkK29RYxmQm +5Q6nYTMwLWjMWdMIgVCK4wxspzttU0TeFPHStzUIvFxjo8O3mzJX8KdEBsD/Wcp7 +h+SzFC5IJuv4S0CWYz/LBeyqWnOlcIZfMrlEWch3nRU0abBqxLNpyky0UALIDUqH +/n+VG94lrWFGamIA37CwlVYMgPWXLYcmAX26e0BiDC1EYQ20d59zrtXLNRJBHd24 +N+iycfeClY71bVrApoK7Dbjs9Ob6B0mPLZ4EBeKplSvZnvirq5SW2QkUNam/dkpD +/Qojig9VTs4MNpMKbzOG1AokY4z2/mWD2fcuX7K2ev0zVzeeW5O66KqjfJtse5dA +QivfRi+8dYEGrCgrbDdN/M8OX1yTnwIDAQABAoICAFkXomNWn4gB93+ljibbfbN0 +uFw5wmXrfvv7uBAl/th+EOQmw8tXy6m7/kIKBbdv3kw0PeI4s9NAOJFzBmfHQoRO +3ZlMpLW2br+K/lGVP3LIWYnrUlcdgqRiZh3YQFVhSnOOCdwotikE/6VaAO7QTTzL +z8tfTSGJD7GoamDqkZ/6uwbSeOtSS5LqLuuKM7lvj0VAXDQQi6+h4kZhIVggz6i1 +L4ze2nLTKolNH4ijKn6JPgiqFwgJ2mndB1huGoylENhQc0qDEkXxRLtTcU6diRIx +tyy8MRta5JlSTP0EnQum7UQ26DbgsIBlygIz0q7jvjlEsNdJD3dsWhccbfj+JEKX +7x1E3TEXRCyXscXHIIbzoec/Ll0vIjlyOKnY254HYB63DwoktzNGdVZDsMMLaYSW +DnMKGdiFbfV9uO9/1z9eJJydLPeRW1ZH7rVYzGOcZsAG8ulxmumLID6q6l+ImAOS +mZ1DYglz2sD8ED0O3bZSjlGq5gf7FXHOKxaVxcQpgc8+NfMIq77sH0s1N4x8m/n5 +G+F6AypJsdD+Bx3ohQyvNf0HqDEdcdamqmy4QTdKfI/oyfHMoRykj3cOFtb7JfPD +huARcYrEIL8rdXDpgkcWW3H72g14nX6+38x/uqOfTsPNDYYksHgis5Lyji6MxGQs +/8uyrWmwTUrF3Lr9kI0BAoIBAQDvizVv+2nYSpkNqU1Fu2yjTuj/Cnm7A+EzLmt4 +k3iOmAb6wDfmurqh7hmtJrQI3wwDPSrYMHjVgKHU9U10L2u7h7bw0IxWBs4NxHoO +HovEe9pRRlXsRexpVUPBIXKqh9vKqpNexvulFn2+FDfiTR8gi1rQdQRp/qSI7a6I +yuiWLj8HwZ1SEmiT+dH4K2jHXGeh+w7ih5Wyo1YkSycX2bABUa2uVPqqWrpgaU9c +V7vuobVZpXLFXxp5WimyLGjtNuCGw3O3pjRUg0bLF15D2fojteC3tIgg7JBI7hRH +zi7hSQI5UjF5TeRB71gZ47hYJoUtgPZeAqeUfZq7pwEkC4+BAoIBAQDeoAv8S/Fw +8RXUV7hEBhSKuajnMvWsU2rrAaKhMmTHhDQWl9vk0+qFF4OvL0xpXoY0vAuK20OQ +e3uv4UtpO4v6In+5sohfNcdOCf8Sn6+qE6AwivRkTP2idk4aMdMHM7VC56fZClJO +gItasZ9J9QFOEkmIJrAoNIQhjvv1Gdobo/ZV1eMiIab7Zetgldfj55Lq+bSHXdi3 +f/r+RSzv7AFUBnlMKeTMxGyMWTgnaWaC8aKVZJM7IHtzzIgZJRZpne4flERf0gJ3 +2oRLzJog8t7RXXqt5ok/7NXBHEnMt/FkQjLv6BD1n+O7LeNMFWWV3NsFb7HhP38F +2WbMJNaMH7MfAoIBADw3t6BGCI6p+h3YEKfV/PJ01NfAb2eXCxbNtZCjKexWxXJl +vefPyzkH3QaNMzcDs0ZhKxMb5FRvuL8LR4RMo32KGXAiTRUXMtd9K92+yg4DwJcr +4k+6KDs2hwjW+rdp3dPt24w442vV4phVBtNp5chNn+/Fau2maQBPF9AliiCu0YOG +paDLJeUFKUaEyxtK3HZ44X4K7ZPxwabwCYzSWY+LU+j+TeFjW3T/JEcnewP7PNaj +Y93AEayuAMtcpKf1Yq4eFkn1izQvqC1H25edBWfSwgIMbSgxr/fVed2vNFPIAbod +Mni3HwmAPJqNh3sJe/M7dzOc4TnP9YT077F7AoECggEBANRgag6thA7ybpi7dhWu +lzogE3k7rBJRIdY/IR4WLP4IqIhFJokNgNSmsgEVZhe/DNiR65Dzy2ix5DTNAFvL +aqWc86v7HQ13CELyVls16yacwlyMsshcijnKEof6sA5WzbydsgnjRtGM32QNvp+X +Ez+fQnaYMcSpFMPO253eo6tqpz0LZjrRT7LUb88cI3BPImD3Bbl9VZj4hkC7TTs0 +MbAu3NhDvvl5CPR6yI8QhbK1QRyzbaEiQ6Mco1kZgGARQXDuyKuAdMpRs5N8BC0a +hGXGf5oWKyjIjsJCckTR1X6HiIBFKqind2DBTezvHJHytx5gh5kSMeMqml51ZhcW +xpECggEBAOaAxyTokHvedNMP4wghkXdT2iJOc4zOgeJ8dbXCb4Acw4uiQ4vE0M7Z +DBCAjb5lXD2WKfDK3iw2kXm/bF2Z+unyA5a3JBK+qduPFf1y2KIpcTKRruhoiB7V +QCTDKrAJ85brk8h0GCZGU890NR5EKhBJ4eYb6EePnZm0u/rAp+MkRErV0hECzzTF +ePyp8i4Z6Gki0iR6N6uQzGQlF6Iopwb39xVQRDxD+2PbictInDUmELNZQDJRCESV +U0KHU3wqueop58VtTHGQQaYaDjJBNut4mMpTzPO5nzJCf5CLgnggdaD/Clsts+ja +J36vL3E3Rf0HzrsfSAJFsAu+ta2gMeM= -----END PRIVATE KEY----- diff --git a/tests/gold_tests/pluginTest/cert_update/ssl/server2.pem b/tests/gold_tests/pluginTest/cert_update/ssl/server2.pem index 7f47929c916..799c0487c60 100644 --- a/tests/gold_tests/pluginTest/cert_update/ssl/server2.pem +++ b/tests/gold_tests/pluginTest/cert_update/ssl/server2.pem @@ -1,84 +1,85 @@ -----BEGIN CERTIFICATE----- -MIIFkzCCA3ugAwIBAgIUKvuCPWhK1Tgjycp6EyPW2Z04HSUwDQYJKoZIhvcNAQEL -BQAwWTELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAklMMQ8wDQYDVQQKDAZBcGFjaGUx -EDAOBgNVBAMMB2Jhci5jb20xGjAYBgkqhkiG9w0BCQEWC2JvYkBiYXIuY29tMB4X -DTE5MDUwMzIxNDgxN1oXDTIwMDUwMjIxNDgxN1owWTELMAkGA1UEBhMCVVMxCzAJ -BgNVBAgMAklMMQ8wDQYDVQQKDAZBcGFjaGUxEDAOBgNVBAMMB2Jhci5jb20xGjAY -BgkqhkiG9w0BCQEWC2JvYkBiYXIuY29tMIICIjANBgkqhkiG9w0BAQEFAAOCAg8A -MIICCgKCAgEArCvDw1lqneNGqUhsjDE2flec0PC6bzxqHcs8lgsZo2qj3NBVmCmh -PEfnYTi5pccvPejDf7IcNajPnT6kTdmi3KQyuXeSVvidr+hk0BH1D5tX1Fojbhou -i2InGudDk+tuxmd8I8eWANLoIh+/8y14b+JPe19E/vveaxlZMPw/nhTOg1Esd4bJ -N5ajKUT4bR7ROEkyHHqM0HPTKdk1EF5wNmlHzGpMvkO8XXDj3DWM2S9LFTWDA6VI -JJAMUehtY4c90PWOuzrICwZgRZkQRStg7NfUr8160lXLLOgvoLYnRCHh8DO7oDyC -KaUKRkVYlUwxW0B20byAgwfEbRXm8CXSNM7ydrQjWxGe5Lxzikabp6lJqDyGTGHm -EO7UWP1AevQz/9HQtj4zXcEBNB3XCOAocaI4+xDj4/6aK4wqqDpK0fjV63LKUqni -ymy5uibj6u9FcqewDXOvaXylDSadAz5hRUbuk3ZYJPGQkhR6kE0OoopU+uSynlGM -7EaeB4EppocFOgvOnBCOYfwjYZp8flxFKalGhMMkXGRpGcHdTo308WBRiZeHQJky -Il8cudUWfFlaiCP0Twg0f8BjpYtfE5DidMpF9YWo4U6u1ffH+EaZ1YVbZ7QN8QH5 -DZ9krRcI7WgsBNZ9kwmFUcwy4vq8xAp6leHDG2FfNiAovI9y6GepnI8CAwEAAaNT -MFEwHQYDVR0OBBYEFMNOR5+SdHCtxJVW6waYADtnKt0nMB8GA1UdIwQYMBaAFMNO -R5+SdHCtxJVW6waYADtnKt0nMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZIhvcNAQEL -BQADggIBAKuykeniFfjTxWjN03M6pIBpvO868XYhY1Q4ng1vmEJTTUa7gz0Rh6nO -h6QAFFmnMTtXBuP79a4UznHb2o+/BZKMnTDRmOv0ArgUJrYYopjTmorflWFoS/yB -n26Fhvrc9HW3469AYZ2oc0lbpMR7eshnnVhzYWTS/sLdDqk1Kys8Byf+aeZv1Zgw -sovoXb89TK/vMoVgbkU/nGNf3GyhFHPsB44z6Wod2gDNdWT7HMABFAzmnAOKb2oI -bhwA250AxgPWXPtWARoH6xFj7VKz1laRcADQAdQjKVbPsCPhp3KnMO+fLUs1vuX3 -7QLBhDGp3BQ2Ni+EKrcqcAfpZOJv2zgqYTjdCvdIq5QPd/tXJmAg886hnlfGlPuW -bpTSDbMzdT/LEoTyj5vUVp8v7bNHcLhyP5WKYc5bPdGMBc5pqo5YKGis1qBCFsld -vtl2xCRYqNfgz8fJFCbQYgdkHdIGpxtQ7X9DmM69vNUSOCMz2iYrk4L2ejUSSM7b -2qwV1vghnmoM4cx7j9T4B29k+JGT1y8xd5vd7sQCVKr2Mnj4uUzkeEjjnLnNrwdv -tj4/qV+Rc6anseARxYZvUwg2gaQfyI+qvgf8YX6ulW7KL8e9jkLEy/DlKlxnDsrx -IZrFE973EHcm/g7qZEA650OTHQbXnJYq75fYXDf97KsmDq/g6h9h +MIIFrTCCA5WgAwIBAgIJAK8YbWIVjsFGMA0GCSqGSIb3DQEBCwUAMG0xCzAJBgNV +BAYTAlVTMQswCQYDVQQIDAJJTDESMBAGA1UEBwwJQ2hhbXBhaWduMQ8wDQYDVQQK +DAZBcGFjaGUxEDAOBgNVBAMMB2Jhci5jb20xGjAYBgkqhkiG9w0BCQEWC2JvYkBi +YXIuY29tMB4XDTIwMDUwNDIwMzQ0OFoXDTMwMDUwMjIwMzQ0OFowbTELMAkGA1UE +BhMCVVMxCzAJBgNVBAgMAklMMRIwEAYDVQQHDAlDaGFtcGFpZ24xDzANBgNVBAoM +BkFwYWNoZTEQMA4GA1UEAwwHYmFyLmNvbTEaMBgGCSqGSIb3DQEJARYLYm9iQGJh +ci5jb20wggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQDTbkRxGOm7R/y4 +xyLT0mx7P2nFa90vt/+yGM5jMiBlh9gezeNF2rSG3uNV/8niUtTD3tSJcMMuRZRB +cnrj5AdgsZpZM7nvwwexCmWzOuQtWpjduHWztJqaGTu9vtNYKNh4X80vX583ZtYE +jNWAVwBaJD++jE2490bnYIXubXgv/9m4LrSbRzZS8tADrKTbJDtOLgnlK5qMYfAH +x0X6puOOTcqeULlhD6Vlh2GzkqT5FztQ9JbjDydEW9CxXkqY+skmV7b0V3kx0NCV +f4b+4WSwZxZhDZUMOZhojR1haVF0W1jUk5+CrFbdHvBZQPdVZjJX1lJFeiprbhv8 +6G26CbHAWUaZGc9hdBBWibywDh9Jk8DxhOwS5Dg9zI4pgRjcJi/19fV7ED9CqC3/ +Su6QUrjRFe3ovtl+emyt02B1OSCGLJLw0xPvIGxbyX/UMb/KRTks2vVhwYZNYvjv +wRk5rzSod+vRMye/xh9FbDB7ZoJXP/z9maPo95z2Kef0wPowJojwHjKrSgOZ9BNf +Vak7299odBv4X2YI3kTtFQZyN+LtyDGFvzdHQYcsrR8h3Jfnc4dDEdm5zHJRTAaS +MYqSF0L8IzjmwgHnqcJBub67kET6zh+dXSRHmIuVcHPtoP+ewqaUrduG1VX6EEre +EL1SiR+F7I2jFgfNTMrGCq1eemg7zQIDAQABo1AwTjAdBgNVHQ4EFgQUYbql4rSx +2PNGIoqEO5jrY4S/sVgwHwYDVR0jBBgwFoAUYbql4rSx2PNGIoqEO5jrY4S/sVgw +DAYDVR0TBAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAgEAyqW1qIbGtn2sBnAE8fBF +2y+h7QuuNukqvYwZ0BnWuryok5pLP6w3QSZ0hlJ54KsWKA7jBNrL1V2IAPIQJZlO +2ivk1WS+s+CNBjqcDZL0LydMrkcFiKeGLI12gO8jspzUPZ9DvwJnDQXnvHLB5V/6 +jnkYIwaCguAl9KVsSioxnzj4Q8ywfjZmjCt+LtXjhxSDvf/zM2wvMWmqo4JCYBP2 +upuBbO8gNqKVvgdGmQ9HF5JqGdnLilzVB/qhOcAo+k0GEYlq0SWCXUXJ88C8EKIF +A5lmsos6+lX6Q9W39J1to5Z2D0Uhw3rLzZgzp4E4L3oO0U0cWQqUA/RvJNdjnHyM +SX3UiTDYY1DJAvZP3VglYepWOlgAjUCWsZ27OaXy4MXcFufWpwOEQwSg4xS8oDg3 +HTZoWAUq5u2MTptO9cBbBLUUl6w2kKTSlk1UP+6Xe8eAibOSZJkF1VHyxTD/rRN6 +pi4LAElLVfuYK8aHE1pSn7LRR2C5RD+rLugEyiLGQfH4NNlOApjdGGHXX1DuNkmS +D6Oqf1Io9p7WKJDsM+uMuAvkZCInzBPCvM2oXp2xo5bfXBF7moz1pu4s/wkaxpHY +Te8QJ++bBRqI1ebPJ7ZtaF7HPFOnnfEAFiQw9j+iJzFnQ6CPMYfLvzmnAkWjiiCe +IpnvRp6UlnZpO64aQDaKmys= -----END CERTIFICATE----- -----BEGIN PRIVATE KEY----- -MIIJQwIBADANBgkqhkiG9w0BAQEFAASCCS0wggkpAgEAAoICAQCsK8PDWWqd40ap -SGyMMTZ+V5zQ8LpvPGodyzyWCxmjaqPc0FWYKaE8R+dhOLmlxy896MN/shw1qM+d -PqRN2aLcpDK5d5JW+J2v6GTQEfUPm1fUWiNuGi6LYica50OT627GZ3wjx5YA0ugi -H7/zLXhv4k97X0T++95rGVkw/D+eFM6DUSx3hsk3lqMpRPhtHtE4STIceozQc9Mp -2TUQXnA2aUfMaky+Q7xdcOPcNYzZL0sVNYMDpUgkkAxR6G1jhz3Q9Y67OsgLBmBF -mRBFK2Ds19SvzXrSVcss6C+gtidEIeHwM7ugPIIppQpGRViVTDFbQHbRvICDB8Rt -FebwJdI0zvJ2tCNbEZ7kvHOKRpunqUmoPIZMYeYQ7tRY/UB69DP/0dC2PjNdwQE0 -HdcI4Chxojj7EOPj/porjCqoOkrR+NXrcspSqeLKbLm6JuPq70Vyp7ANc69pfKUN -Jp0DPmFFRu6Tdlgk8ZCSFHqQTQ6iilT65LKeUYzsRp4HgSmmhwU6C86cEI5h/CNh -mnx+XEUpqUaEwyRcZGkZwd1OjfTxYFGJl4dAmTIiXxy51RZ8WVqII/RPCDR/wGOl -i18TkOJ0ykX1hajhTq7V98f4RpnVhVtntA3xAfkNn2StFwjtaCwE1n2TCYVRzDLi -+rzECnqV4cMbYV82ICi8j3LoZ6mcjwIDAQABAoICAQCX9/1QlcFk4bKCQ3oEeH/o -Hv888e8ttj7YU4cDzYJw2vUOOBoFOGGoKrOR/hbnvBv34leFhDogJwQygJNYYAzA -AnL/gAp8l+/f0BoECoWro+tvdaymR5fj0dxAg/cDPqFFoRxTHJ1L1t2tGSc9x1ny -L+kGNb5Z7wmQsQwoD887gpQSFvlP+3nqhh04lTdOYhA8RWdk+csHR+UQoDeVXjZf -4KfOR9m7a2B9feKygdXz8ims69Hbyu0V0Mv/FFBRtJMcuVKl8qmWCae6ZvOUikiA -ZbXHA3Ew7SdsWPmJOp8IJXwfoBoxwpcFmTardhRNm7ZJHGqEkIVCov1/aACy4j9a -iNy9s/D1qU3bdbz+KVgzhWCdwDjoVTeIvEIey/v7ow8W5uuJhLJcBe60fQmScgPT -NAUgvIJkSVHSQlM5KaJSuuQiMZTTNk40lQoDM3XT5klvwT7JwiXVGqz9Ot1ayOvM -8w8RQXBMBP9k4Sxe5WUTufI6XTOU1Bpc1TAOYQMQX0tj1XlXH43PLkZ+vAkuR77L -ehqa2jX7qGTu/FlvKi5QQfO1rjtQn6gBxjfyN9EcZBBWKFf0kP2gcCRqHr7r4ijR -YLxYy5ipj/G43nfBGwn7rKxIfWdLbrUE2tNnSU36DCMs0EbMu0OjnMydIIpfG1rx -ZpC7hbg/G2GTsMix6PrqEQKCAQEA0oKGjd+aKxcYVNiC8x4seQR6H/VHMJAo/yc6 -FRKx5NmDcP4AogemzNHJFxnoFZZ7Ci/BAXTTbfbbx3PayiSD5VgFZzPki+2VvL8T -cDZugzO5jSihIBHKyBhNdIqfH9cLEnoRpcYnfEy0ejw/CkIlh2ou6abUdD79O6ne -LoNVYVPqD3NeD8ysKR092oNtrBGazrKl4XcMKUUku8vDajg8RgIMH94xyx5T8CJ+ -EaRH5XbbWoS8HO5QWrIMEtMFWnO0sjiCNbdiKOh52ES/vLCd8goH9odx4DEmbKbF -3koBhq2ucglTKTuPOzR2huQLRvWoh+PSOAxneCvsw7eRp/2z9wKCAQEA0WBRb5id -FYNrFOVhnzMPCDFOBjuP4uUOil59IQ0vRTePa8PsB2sG0aeN6vf9NTtI+8aX90QL -r1rI05BVjGLqIWSNt/fko6mh8lqjd7ZF/ilIuGVy5hLsuDazUFQRCw+VwDBsn5Px -sH+GoXb1a5WBeyO+Cdzde+UGH6ONruxSXfjrnMLZPW4dRg58bNB4DwVohDW4EQxF -b4WoDySvkTv+xbjm5UU3K3g6KEA6qHwWBt6CaznagpNaw29E+YgVRVkHHabn1KXW -kwwWnamFTubEQzi1y4CANjYU3GHRTbnLj6cA2JtLndnzfg5ngjNZFz1xk80Nbu2g -6Wti1/mwzLAGKQKCAQAPShurwknYR10lDHS2Y8KnJ4QXPiFljZLstVSqoyj4jjPV -yR0Sp6udxL7uRptstflJzB4glPOmUP+1hNynQe+ygKojzMkUwLTXeKlYxRtRvDgF -4KWTRreLwPgfNtJH5b6QIP2XJMVJaejR1/5cKGHBbBzsK4nSx6Bs7PGOP0u8PfQK -Gz8BtsPqWI1diZ41mTG9QYx6y8K+XJ9GZI5U8LCwBQYMan8DWbiPAHJXpa7zI6ba -9DkVgNmlPLlTxK/m2fCN4TzT2fXvMrNm2ddzRQCzy7a4WS4UMn2v9oz4kDd8KLE6 -5yJ7JDBLBIx3T6jiBoWvGQTzvTLmr4oKzRJvOSYRAoIBAQCcKFk0gT+PXw/LavUv -VlZ8xsEpttyu3iXFH4n1z66U0kZ+moZ8Vd+lGHGpcMICJLBfUBPhUHfiljQ4Tmrv -pIp401U7g4CcbBUj+2P2EhUL5eCd3tQeMrko/2snmzuG413OFI+/SQk2mTZhUKbk -UYJbxCGlm3v5Pqwdhs9SSmF4QwH/TffWcD0XFhDI24bftSnpiWM6G8vhzG62tKbZ -DEUNVMWo88GdAzNk99qH9Nw8zVG6BVEqmBpWrrNj5JHwweCjxescV3+89oQbnOaX -HVf9xtsX50Q4qodOgwonDU58WpMv+ksgdQC84KNkoUVuJt6B3KqLNGISFYyIBmtN -jm9ZAoIBADCxJY25i2dpOUYhuM8jvFUtvTW/U7lMwsfJYnH0IR5g7vX79UeDqfw0 -FwFQWn4Kjh/A9YmvqfNac8Rz6PfNiRVTFE/PbsboKB17mB+xud2SN5fF3C+ykSrX -ootxwtRnwixiZNXcdKbvcgxYuKuFGCCTvIVkmC4qNLB7iKCNLtAjWZgDETcfGXbC -Z+swCItPKO1BH+mDJd6bTbK37sGwnMc+bD7dKXQjr/2HZ41Bk+gmpPcNJFvgjohs -o5DABidU7Zok0Of14UvbF3DuKySSvmgGMSiuqWIY38OfV2f7lNaUNmoT+38V8YN+ -qxix3I9z/SzxQmb10gFNvEzR/fQDDiA= +MIIJQQIBADANBgkqhkiG9w0BAQEFAASCCSswggknAgEAAoICAQDTbkRxGOm7R/y4 +xyLT0mx7P2nFa90vt/+yGM5jMiBlh9gezeNF2rSG3uNV/8niUtTD3tSJcMMuRZRB +cnrj5AdgsZpZM7nvwwexCmWzOuQtWpjduHWztJqaGTu9vtNYKNh4X80vX583ZtYE +jNWAVwBaJD++jE2490bnYIXubXgv/9m4LrSbRzZS8tADrKTbJDtOLgnlK5qMYfAH +x0X6puOOTcqeULlhD6Vlh2GzkqT5FztQ9JbjDydEW9CxXkqY+skmV7b0V3kx0NCV +f4b+4WSwZxZhDZUMOZhojR1haVF0W1jUk5+CrFbdHvBZQPdVZjJX1lJFeiprbhv8 +6G26CbHAWUaZGc9hdBBWibywDh9Jk8DxhOwS5Dg9zI4pgRjcJi/19fV7ED9CqC3/ +Su6QUrjRFe3ovtl+emyt02B1OSCGLJLw0xPvIGxbyX/UMb/KRTks2vVhwYZNYvjv +wRk5rzSod+vRMye/xh9FbDB7ZoJXP/z9maPo95z2Kef0wPowJojwHjKrSgOZ9BNf +Vak7299odBv4X2YI3kTtFQZyN+LtyDGFvzdHQYcsrR8h3Jfnc4dDEdm5zHJRTAaS +MYqSF0L8IzjmwgHnqcJBub67kET6zh+dXSRHmIuVcHPtoP+ewqaUrduG1VX6EEre +EL1SiR+F7I2jFgfNTMrGCq1eemg7zQIDAQABAoICACC4/TC/2VTZ9FwovdQoyg9d +JiKe6hG7mYBX5IFnvanZF3LVDKrACZaiIm8p8nXrufBCqYRIBn0/2Y0ziZ+BqD6n +3iqWHd3kwmj9IfZWVABM+EHAjb7tYcwdwahBQ3Zug+pFcX02gG7TpKX5ogWF8ZEB +8TetgU3yP3AJ+VCIcsSDBZBC8kLiB59Y4ybebnlYJzGXYIKud/HoKn0Zs1Pyxl88 +3PIZrkc+Y6vOAyoGtS+0D8Mcx8mGi95Bk5AP9t2sWrtrce5pJ2G1XsL0hNEn0yyW +F9mXf4zv0Q3FYVm1Mh0fEiV8X4Ca6aq5Yht+OYlnEr2TdxI3sR8FYKGI24qie3bw +k/WZai7CirXbeoGGXGXG1Epvp+WdIzP/qm2Nd8qspQ3CYwVaKDLeser3jk/QWz/2 +5F8xk3o4gYi53ozjcR/iU6rR4CcCVzCipCBx17MIYb289aOBC6aPTF3e8w2DoYkY +T7g/qrf+VdgJnw9pQ3mu836gpDGvx4Rs3dbzO2tLhR9Jb1oJk/Njx7bMmCZG5b3l +/mrQhh1Ypuj8dKdYQ+/Ebsdzz52Y4X+i21s9Kb7YPCrceTd4pUJ8S23UVDOzW7kM +xn1od6MUcCNZa5kNbeIQATlBT1K8B8/kB9anrkJEG4rUIvKcm1lV1WY5P5CQ86dK +iXk8T/DYsa6N2lgCxKNFAoIBAQDskPdDgjrhMrxxEL33XVR+U3lOvNXgrduzMCCH +UTM7B6ro0JKQD1kgHrI7fX0DQYaWB2xkK3YxXw5FkPoC3Wu7/h40onzbYnh5qRSb +Lp2Oy2G6MJS7LxPrE8qirah6A4JWSvTT4IJSicpJ1Z77HOcP8EI42oIJyJgXRw8G +Wcr5C3FGDozSeEt8D+DDhgZa19z3pjdEvmLVXsK04xbbNl29/34mLQqWwzVWnQl4 +x46wvyc7DmefklJK89b/ALm32J9YP6JLLwOZf88tXpPUKwK/7x6hIOLGXhmzl/XI +1Crf8P7c2JjNmX/YUK+jnAzVO6zLESkpiWR/2L71GpZVZBBLAoIBAQDkzLI7qsIN +qskcrO4N0zEyjUOdYPPYSQ+e/1uV+dTfwZXMYuBx6cWrX+Xe+cLOXdESIIiSPUTv +b3glQ2KhGUyUFCzpIFSzyF4uEzGefQ/wIZGDxawr5H9hwyTIYdAv8OWss0vk2CVs +Ku/6uiNB5/Nrjxo5HYp3YJ2qYmj4bX/7rte/XY0d66UYm3gz69zp5brKd0x5ztkf +4qqbJ/sNPVVMle3J4DqVvSnu7VvkLiKMPV35eINRuxfgPUGGomB5EqvuF8bgJTT6 +ZaLXg4DT+q9yBywQHtpTQwOndp2tZERm0MTyJ+/IcAead4RmRTEyDf3F7L7U921A +egJzt7w6msVHAoIBAG20bHZqFlQI60qjsPTd2ykaHM9e0vB2r8PRNcSK628chy2g +S9dyxqoWkiD0eNzhrkA1ARcS8KTEqCGqscZSWHu9xNQz5iL271e8CUpNu9NUjeWC +UfbcRihl7TqzvcWXiwHpkNAQ9V7Bd4X2AtwA80Z47Lzg2B0hYSaqVVVknrQPkmAi +rzuLfbXyxxDlPl0ybc6Kzmaaw8qfPzwTc64E8EQED8Q0bwyrSjj4akQQhyOAAflp +HLTrbi5EFhf7hiSz2HFcUS78hnbQxosLqRJjP8syYMnoLkI/sTuP5PNVMOzZHdfc +uj/eR2NRR6Jf47OfEOq68pPIm0qOgT+5LK480TECggEAUDZzRffXkHMPjr4Rt0cT +OGXODp6u3mS+1T8xHKM5vjhgDAeeKFGMu0gjjsiBQ1ggs/oz5xET2d16gZXePhxg +fImWNSTgjSbx3bVc7/NoYwdhvJDQi5LQFbYUbyvbkPMjkihJszET7ZggCga31aM7 +87l3uA5jPPbiT5mws2ewNsxVVwG1N4glmTroKUvHZzjjA5hPd2TNOiB9L2gU289k +2DS36M5zu4tV6C3OyXBjLEdXat6VK1IsTLEtB8ZljAy/ufkb//kLWb3IGUZBJA/+ +8jsRd5VZdVdJMKylGuTMFj+azEq0BdFA2SaNhE09sg61qW4WeRNFehVMRsVjeChK +gQKCAQAeBXVRFWQ5/11SxC+9cqY+4x89rvXJWuCzrt0FdHWa+Tdqy8sE2KeBypd+ +1qn/bevSrYvfwXfmZhAfBAUgRHRQqwIKZ4FvmlTfrNtQ00U2UjFnFOVp4GDYROpY +4yq/hKhejkL4vu8HO7tlly0TKipwbfDM0QOuryA1G4NgsRdmt/KiYNfMbcgEBGQh +emJAe3Jqu7ivSPGzLwNYOoHToeO5DAUtNRTIY5vfQLAf7IJMnYnvoVc0J7Tn/ymH +NBqVUNoMTPD0b/JOdLRRHCkUa1BUDQhCAyHzMu2VC1zVyuybkAEIl75tELkdNzHk +frK3JFYTd0nt/VBxOSGbptvq0f+V -----END PRIVATE KEY----- diff --git a/tests/gold_tests/pluginTest/client_context_dump/client_context_dump.test.py b/tests/gold_tests/pluginTest/client_context_dump/client_context_dump.test.py index 9fea371d725..40b9c86fe3e 100644 --- a/tests/gold_tests/pluginTest/client_context_dump/client_context_dump.test.py +++ b/tests/gold_tests/pluginTest/client_context_dump/client_context_dump.test.py @@ -17,12 +17,13 @@ # See the License for the specific language governing permissions and # limitations under the License. -import os -import subprocess + Test.Summary = ''' Test client_context_dump plugin ''' +Test.SkipUnless(Condition.PluginExists('client_context_dump.so')) + # Set up ATS ts = Test.MakeATSProcess("ts", command="traffic_manager", select_ports=True, enable_tls=True) @@ -52,12 +53,12 @@ ]) # Set up plugin -Test.PreparePlugin(Test.Variables.AtsExampleDir + '/plugins/c-api/client_context_dump/client_context_dump.cc', ts) +Test.PrepareInstalledPlugin('client_context_dump.so', ts) # custom log comparison. Verify the two certs we have loaded are dumped log = Test.Disk.File(ts.Variables.LOGDIR + '/client_context_dump.log', exists=True) -log.Content = Testers.ContainsExpression('two.com.pem:, Subject: C=US,ST=Illinois,L=Champaign,OU=Two,O=Apache,emailAddress=ssl@two.com,CN=two.com. SAN: . Serial: . NotAfter: Jun 21 20:39:43 2019 GMT.', "Info on two.com.pem should dump") -log.Content += Testers.ContainsExpression("one.com.pem:, Subject: CN=one.com,O=Apache,L=Champaign,ST=Illinois,C=US. SAN: . Serial: . NotAfter: May 22 18:17:04 2020 GMT.", "Info on one.com.pem should dump") +log.Content = Testers.ContainsExpression('CN=two.com', 'Info on two.com.pem should dump') +log.Content += Testers.ContainsExpression("CN=one.com", "Info on one.com.pem should dump") # traffic server test t = Test.AddTestRun("Test traffic server started properly") diff --git a/tests/gold_tests/pluginTest/client_context_dump/ssl/one.com.pem b/tests/gold_tests/pluginTest/client_context_dump/ssl/one.com.pem index 9835695fec2..1295d0e81c4 100644 --- a/tests/gold_tests/pluginTest/client_context_dump/ssl/one.com.pem +++ b/tests/gold_tests/pluginTest/client_context_dump/ssl/one.com.pem @@ -1,84 +1,84 @@ -----BEGIN CERTIFICATE----- -MIIFjzCCA3egAwIBAgIUL4Tdbs9LkVpOyO1zSKphXYcXNfwwDQYJKoZIhvcNAQEL -BQAwVzELMAkGA1UEBhMCVVMxETAPBgNVBAgMCElsbGlub2lzMRIwEAYDVQQHDAlD -aGFtcGFpZ24xDzANBgNVBAoMBkFwYWNoZTEQMA4GA1UEAwwHb25lLmNvbTAeFw0x -OTA1MjMxODE3MDRaFw0yMDA1MjIxODE3MDRaMFcxCzAJBgNVBAYTAlVTMREwDwYD -VQQIDAhJbGxpbm9pczESMBAGA1UEBwwJQ2hhbXBhaWduMQ8wDQYDVQQKDAZBcGFj -aGUxEDAOBgNVBAMMB29uZS5jb20wggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIK -AoICAQD0YSNv4UxBFFWT5SOXWtqzfNQI8RsMSQFh2qEHKyQ4oDoBv3mO1iwM143J -pCE4t/960A/XdbXsVYdHh1YRcBZu+EKmj6i0od+JRit/gBat+brNBWH11oAf9hhq -SBgl0GMD5HgcYIkRYGkhLUNllTFEGh79uPPBLLrCmdnZG9G8sNVeaPp74iyXsY57 -mw9IoD5b3t21IwMIBzTvAcsRKWWGi/s0ZSN4srERRjA3CWqF8srv28QciXXlpe2a -MRH1TC/Ec/UuT6SUtAC/G7J6oQ8KS8TwyGafZoz28z4BlfVRw9WkNv+Atxy1zvZU -3JQCPecVV3Y3mD5zyXSQjp54ukcEd3pBE7kqLVrs4IddS/bifJl564Z6WlFNblbi -8g46dQyVkcQyLb/CaKbtNVI2jT6h01LtoaXLtL8XdbKWxXIfDf03pAcAnGOUTwuP -pIMWnWypd6C6OKYJc2MO/QdP+xnyrFXPn1YlKEudEVCPKuc2Ymtd75OPit6OgZul -JPUehOSM//UxEHsCAMZ+0AAzAnEkRdqSTtVz2L8uLDSYNndoCm49+p0GCdGFSmkO -SHED1sw5Qbq0AcGmIynp+c56z0Yk7QPmLKWhmnXQPt1+TjGZQFsjoty26zlYdA7O -g9PkszUz1AiNk/qPMq2fF7pOEOtnPJbVVLdDqTOoNJsjw/RVNwIDAQABo1MwUTAd -BgNVHQ4EFgQUrFYvh5JA7/TTRZWrXqFqZZdRJuYwHwYDVR0jBBgwFoAUrFYvh5JA -7/TTRZWrXqFqZZdRJuYwDwYDVR0TAQH/BAUwAwEB/zANBgkqhkiG9w0BAQsFAAOC -AgEANXx7/cpj3ybrTq7WqPFoklnuppCZc6SYYrfGNAay6vYopYVscxwsGemkAnQY -K9ezCvMlvGw2wAMB+Ple0SiejnIXFtfv49Exti6HCnLEI+uRkuDWYR8dn3TeIj5u -OziJIMBWhzgRaM2uuQeXh/tF0RDindeKIec5gCRaUpQvyi9zVwS9hSejnnREFnbK -oDjLqsNVzXB3nORxGTDWYWSM7EvIvYpBIeUFmHZ0cdcWyVFc/umlPauCcKEsyHEm -mjmTNF3oxpsoQysjaPVsXtG0cpAw+1u8IcIoSf9R1ncXIHNAYjC4TQySofs0Pgn9 -BdKhmrFLvxhhG9tklin7z+mlhofl+kIrylGsXEkJgxY7PuHRJnHxjUVM9PdGH3Ce -6IwbaDA7OPfdQgJEWQtzO5Y3hAhLp07RXAhabxCUc8T9pZRXMed6poZFqw+YTGQ4 -ZasqYpCVx8oxuucRV8Cz9nbW0l0yyfbvWB0yCAn+Jq5gBudkxx4gFtQQHsSDQ+77 -QxrY1acflfH3E8bhs05caYLWV21YxJdNek5eHM3IPWJxi308w0oekcSLEcvP6gke -jwnbewzAwTLFft+zprt7JV7A7BxK1dleZUzk6CqNAo+D91E5NjYKwSXB/7r5JBpr -y6RRX1nxu9BL7bADWENbcLppUbPST01sgkm9lQKt7o+vSbI= +MIIFezCCA2OgAwIBAgIJAM6FWqG9EKi7MA0GCSqGSIb3DQEBCwUAMFQxCzAJBgNV +BAYTAlVTMQswCQYDVQQIDAJJTDEVMBMGA1UEBwwMRGVmYXVsdCBDaXR5MQ8wDQYD +VQQKDAZBcGFjaGUxEDAOBgNVBAMMB29uZS5jb20wHhcNMjAwNTI2MjEzMDA0WhcN +MzAwNTI0MjEzMDA0WjBUMQswCQYDVQQGEwJVUzELMAkGA1UECAwCSUwxFTATBgNV +BAcMDERlZmF1bHQgQ2l0eTEPMA0GA1UECgwGQXBhY2hlMRAwDgYDVQQDDAdvbmUu +Y29tMIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAx98t07J1TCRhrO2P +m65GhWy2iDKNuMLIo180/9CjFsbzgdglwnPjHOFAFXqyZT7Ex5KRRHyaO/zHan4n +YiuNuUHmUEmWHS2nftswyqrAqvBe+TeOeXxYHfFD76CIFhmMnnK5/+PLuh7Kg60p +9mxVC5f2GrhR3iycq7gUQAlha5Ep1iuO0EesaVwA8lcufGw474rr5FQJeUy2oVcV +pQbKhN8eaXDpxouG607ZPC2cvhk/2hcFZElCs0AoVLZ7QpnYT3mtWDu/SdYwz9b8 +rWhnonzLMRuHQFmOLSTydu9FTNYGKwsQRa0VszTVbb4vKErEQ+LhhpRgPWDYxOGw +TLT7toOZVPJJlC/U2mLKU0OSytgXN5wV9jxNs4e9GZ9/8Lq3kHLBQ1hR/SUJrnN6 +Lva9RzH0helRNEd5JT5+8QExjdRvPyZw8lpAy4D8zNvCN2FNlHoqvzo4mkESM9C2 +e9rupASZur6p1gkGz3hzBbVyQKHaTMIqj0A4P2OnOf4cB4lOz8NmUfYPeyoDnjCA +G4fJuRslzJ7ExjJ4EaRvy1DWVqgcJPw2uHgoKFoVjFkU/VtbSWaUaKoFfDSOCJB2 +udJ9bV3ZD69oPD2VFlPhOJxgJcoLniGctisnGpMwWyiuulQXtOgNMj5WGgnxiJXk +zU7pEmJsvsFCb7S4SfRgI9kir9MCAwEAAaNQME4wHQYDVR0OBBYEFO9HKOZMhO4p +P8OXpAWvutaS3aXJMB8GA1UdIwQYMBaAFO9HKOZMhO4pP8OXpAWvutaS3aXJMAwG +A1UdEwQFMAMBAf8wDQYJKoZIhvcNAQELBQADggIBACBhAlNNeX4gSpjHA1c9z6IY +rTKgzqAxZvQEU5kyNGD1S6K/LnYgG+X76j7HDuCzUced2qdiQG6Slaze4aqahIgE +YE17Y6mDnFWjyobBMjdUGWM/yb/M7pFUd8gIoaUdaEAXEGA3n9gJPDXKQgEcLNe3 +gL9v4b+6Wf97/hRGgDmAL4+zN9bz1s/Ic7gsGXEDAtz1LIxRfbio/X33Eaetc1qZ +BKlj4hLXw1VhwAb44iUDJ94Wqt29DjLNlstHE5+6R40nNQLS7p3MaLnbKbKnLkVE +ltS/vnUo9Anw0DoyBylt/OPXNdG9GBUPCCCnH4KUIQwb/lI1cQ3MCc+QLAJUslhb +nlriOkvfNS6QGr0O8A5cwFXErEw6FqLi4+Hp568Bu3U6NKnrd6HSlLKwwEe7zgW1 +SMaI65HTgrKTh59y4+JP9eq2jzWoEvUlYJRvsbT86A4/Ar8CtLxzGXBTXe4WWS94 +6v8kafE/3K4gmVWQcHIbuKfc8Uq3vVvX5HyWNgjPBjS8U6uSnzbWHhQEHBYsZHSb +5No6sh/ad+JXpQ615wTyL1rumIG9CAVXY/nj4Ptn51BPjVVX9kHFnb+Ix9mfDphn +rjV8ogWRfAqE55r7bq/Wos5aFNcNwyi+JSW1mlOpNXtrjISkJk5fOUjKNmdSBpdH +M2DWw7kZ+NhNUTGaJP+g -----END CERTIFICATE----- -----BEGIN PRIVATE KEY----- -MIIJQgIBADANBgkqhkiG9w0BAQEFAASCCSwwggkoAgEAAoICAQD0YSNv4UxBFFWT -5SOXWtqzfNQI8RsMSQFh2qEHKyQ4oDoBv3mO1iwM143JpCE4t/960A/XdbXsVYdH -h1YRcBZu+EKmj6i0od+JRit/gBat+brNBWH11oAf9hhqSBgl0GMD5HgcYIkRYGkh -LUNllTFEGh79uPPBLLrCmdnZG9G8sNVeaPp74iyXsY57mw9IoD5b3t21IwMIBzTv -AcsRKWWGi/s0ZSN4srERRjA3CWqF8srv28QciXXlpe2aMRH1TC/Ec/UuT6SUtAC/ -G7J6oQ8KS8TwyGafZoz28z4BlfVRw9WkNv+Atxy1zvZU3JQCPecVV3Y3mD5zyXSQ -jp54ukcEd3pBE7kqLVrs4IddS/bifJl564Z6WlFNblbi8g46dQyVkcQyLb/CaKbt -NVI2jT6h01LtoaXLtL8XdbKWxXIfDf03pAcAnGOUTwuPpIMWnWypd6C6OKYJc2MO -/QdP+xnyrFXPn1YlKEudEVCPKuc2Ymtd75OPit6OgZulJPUehOSM//UxEHsCAMZ+ -0AAzAnEkRdqSTtVz2L8uLDSYNndoCm49+p0GCdGFSmkOSHED1sw5Qbq0AcGmIynp -+c56z0Yk7QPmLKWhmnXQPt1+TjGZQFsjoty26zlYdA7Og9PkszUz1AiNk/qPMq2f -F7pOEOtnPJbVVLdDqTOoNJsjw/RVNwIDAQABAoICAFQWEHFwniJOctu+ni54DoRv -0S16eIoTnKwk4/9pcr8hMpRjY9ooC2Qnzxuwo9W2SeviJ3FFiWFWVIPiJ1U8shtg -xN421B/DCv1a7aCjXCpkoUfmMDy8n71fmisHv3dTap5uQH3TIZafC81km6oaWadL -LZqzxvuS/nfzJCg2EEbSubgQew+hVSKk75fMVMfyi0JlPvgSofpjMG2EunCfQ9W1 -2KvAmSHwqkh9VB272bFZR0ac4a/IBI8ONuE1PS6gFleMZrKWqNSZ0x/u0cysGla6 -l3saGWWDjcFzFMTK6mxF6/7jBasuVmygV9X/R9Y1oouPWpfqhKKmkLskt5Zm0gQl -R/pcpI8N40vtStYppX6kXFzeTt+XA46SHsGzrD+CLReH3GEH3u6Q9mNc7/A95rOg -uk6VekNNPOZ3FVXnJJeqrT+NixUlmi0bnpN1NH9G3LhJ4rzkXykT/nadu2KmsViW -iRZ+T79Vm2X4vL8g9BXGPhXE2uwV5HVkkNuzp29/HX66t8xxBwDASGFPLn19EuAi -MA8YlKuKpDLCb6eflkEVXkBY+TZJT76yF5IE3n8h2n/F6uO8qCfvP0d37WVI/Rx6 -izX/7ftFnjbshWX5davNyGu2gicPcIvHB0f8SFLK6DflWeSr2aC884nrG59YT/0f -3qGOKwlULLKimNC3D28hAoIBAQD+1LGPfPkpprRfJ1EGzMV/gBHgwW0cWdFNikVg -VRTgUQoxdN0eOoLtGbyOe+O7Q/HntiC8vj8ntezi4sYbJTerFmkV/Gu8VwYUJza1 -koz8yAgmmaJHHO/zKWkqD+q4jDYUpzE0I8PhH6nuwuWX86S+J4/fS3l6iWKWefcj -9yOmCim16NW72hdhlbshkl8Om4IVe8i5CbeZ56QI/ExO2CGOz4sZXyUYf7IMZHdE -4H7q8X8gQ+ZSiKyaj8/9mPq6teW3EwJchR1rjNdkOZVdn55huU9QuMIbf8z4thrb -JYV98CTZA/M6uh2fMGngvil+LrJVCkKvrJ5E9kciVF7RO0FfAoIBAQD1gCtbeBF4 -1erNRQj48oci+47WZFoT0CRDV2Qe1gOnnc3Yjp+8IAYQCDPcXG3dI7KslXl0kp+D -7cO+BZef5VYyDXLFl++HZLBQDoVwFjMse9yGSD+S8gyAHer7Smaqs7UxJkklK+m+ -NMlslifXKp7aMOoGjlM42umnQ177++xmr7oLA06VaZwAj1Aj3LB4J9uTS1CJeDl+ -n5+tcaBO6Lf9brnhvwigDsbkfRt4ccyY6rc4QKIM568LgXsmOe/OyUP1wda4zyCr -3K62q/bwZIKPwIhLK+hxGCp3vz/Mu7umN1XkuVXT2xqniNiApT7MXsg0lx1RgLfh -6YUilb7XhUMpAoIBAQCKla8uwp9aeG+VY/NbyFcL3OFcIrUs+uepzK2oEv00dL4f -YVezTczQFvQFZPjXab8P7WtmWexMs1JtnThxoM7ie2CQ9WK93XHP2feVzWphOoO8 -QkcPd3xC+F1Z46gZzx7GIprOqTiooKiw0Us9VOJeC3Ph0tDww/Bat1+hLpEzhkli -xYofDB81EdHgExMhBY88EcJ6Zv9zcpcxz8vMARxW5yXVmXm6FhAFT0nRqmk7ajRh -nquObQe5UsahOuX4Tl3sLylUmYwDZmfo+Kvza5Adw0KQOrpNbDZTd+2pCoHLmKLh -ZpWLdZYQcarS73fvSIPxXZAgq7ay+GB9GfcqwJfvAoIBACvFl3VumgbmdT/2MBxa -+bdGDPiy2dCwitaq7UIGPI6VN0+GVnqvZwVSwRRoMnp8U+4rlIUxY1mdegoWaytq -M40nErCiX2XPkRQlEquieatTxkT2+sbTe2EYdH4rjNSgyAykW+RRyRJNzSAcQaw+ -gCY9FGzo0XPQrFpTS8s35rWEXXJ7O3auZs8+vjY2sgwqZx8DDbAFDJNEGK9PFBsd -qTh5lpDmg74uBE5W9B/sgmM9bj+MXphYcsBlbLSrHdPL1N8rmYJIA/ZAmbIeRSAl -e5Xv6R0mDgKkIWZKZjC1xEZllV37oY7tgPogDyIY1HKR77ZYvzR5889G2KMK+gTp -UMkCggEAHIUUQ3lh2tMvLajJQPiDZmHSoNh/EVwLP/y35CjEwgrHRv3pk4j7/ZG2 -sIN4WWK2hftks4WQdk5Jc0y9++4uaWvgQQzu3Jb4sN6dx0r/C2A4sG96pgT835fQ -UwtMVZm9l/eolB7XVdcvJogJk9khmvyIlq7cZx53YMlOw0b/hG3rNdOHPYgL0GTT -v0X5v38X7GRBJQvsl2RLoolWAn8cInk7dsYG5c5Lvd7sJWJI/g/+JUZ633G7MYDK -8Ez2S4PbV9KrqB/Zyh8ZjS95HyQm4xUhbcOJRQc8qe0kZsx0JbaR3K0D8jmgnmGj -qFO5h9/cDU3/Kzwsv+Ki3hXJKN2BeA== +MIIJRAIBADANBgkqhkiG9w0BAQEFAASCCS4wggkqAgEAAoICAQDH3y3TsnVMJGGs +7Y+brkaFbLaIMo24wsijXzT/0KMWxvOB2CXCc+Mc4UAVerJlPsTHkpFEfJo7/Mdq +fidiK425QeZQSZYdLad+2zDKqsCq8F75N455fFgd8UPvoIgWGYyecrn/48u6HsqD +rSn2bFULl/YauFHeLJyruBRACWFrkSnWK47QR6xpXADyVy58bDjviuvkVAl5TLah +VxWlBsqE3x5pcOnGi4brTtk8LZy+GT/aFwVkSUKzQChUtntCmdhPea1YO79J1jDP +1vytaGeifMsxG4dAWY4tJPJ270VM1gYrCxBFrRWzNNVtvi8oSsRD4uGGlGA9YNjE +4bBMtPu2g5lU8kmUL9TaYspTQ5LK2Bc3nBX2PE2zh70Zn3/wureQcsFDWFH9JQmu +c3ou9r1HMfSF6VE0R3klPn7xATGN1G8/JnDyWkDLgPzM28I3YU2Ueiq/OjiaQRIz +0LZ72u6kBJm6vqnWCQbPeHMFtXJAodpMwiqPQDg/Y6c5/hwHiU7Pw2ZR9g97KgOe +MIAbh8m5GyXMnsTGMngRpG/LUNZWqBwk/Da4eCgoWhWMWRT9W1tJZpRoqgV8NI4I +kHa50n1tXdkPr2g8PZUWU+E4nGAlygueIZy2KycakzBbKK66VBe06A0yPlYaCfGI +leTNTukSYmy+wUJvtLhJ9GAj2SKv0wIDAQABAoICAAOsAfeFq8n+UR0QMDc3+Guz +F5dg9mGSkoUN1rdsadKdTRPxmJ/96Qo6ySab3nMluA/CjZCoiVzYxAEc2DRhgHUK +1ORctblLgWw/MqJGiMubJuOFNcTdl/DU4d6hIF7nkUHH8crVsIR21Fme8MpqKxXP +cOIEyNxADZDKDPUwNu32+MHKgxEof+5fllTdbSk0A8fC0ZqcQKBN3H0SUH97pHTU +dIaTYHgZUB06kZsda2fCpv2mtp6lCeipIRSNVa5fjS3Imnqoflco7DSlSncVF/JQ +QJtiWhXAj3Ac4ad1tdw3gl2AEoh1KZTlP5aIi+09kmV84Y/LYgk+ojzbuJVwLb+g +RnFwiBfR6posIAYfstMxMno9ubr3DRY/UXQJq6DwFn14+FjCaAaXa7Ggua4ArjfG +iGjpS8HS8PwDXnvPqWAa18P5Af9TtbhWvPh9ZIINIfUFQFTI7ZNQ2KdecmxfU+Qg +VguU8j+fiH6yFMnF1x3pcRV3xhflPJt/ahWN+L95J9rTLfaKe1Zu00ucWe98MkVK +HLiIoxJTRz2Vt+eFBa1nwrETpAwFAf+BtifFluXzKIG9poEIqzEpL/eLq6CdA8gJ +B+T85HMaLcrEOQR9iF+ifKg/KOpFDttML0cdU7cYy8P6aOy3U4fkBYCq5JZDPHUS +YKhepc+2iKUfRiCktiShAoIBAQD8yP/9PiphXEZ6kDWB2zF8Gaj4Y4zDnuv+AQb9 +kH6AAAtrAE0cvsj3qPDDEkT3ZCJQGJIxqXuHwbLyn5yJ0wXyJ4Ftk+aN7OtrcRP8 +9wCS17J1rdKv6tdOU8R9JWoqOWue52kF6KKleVmK78ad5HMK+MOjobAKsnYHxcQF +Xw+qkWbl5crwPRomszkoz/h86C+K1u+Gmf6yGqgDgHmWftdsRVzBPjqXwWctWV+e +AuiWE1d/C+5Ld3Hvso9UMnvM6fE3VEc3UEHs69Mu/RdVG3LyF55NoNF4PrhuAO02 +3nvIlI+imEkNB/otIsusksEUN9lsv9RcCibpWAbtJa3z5wePAoIBAQDKaehPuNcX +8wfYc3jRrMdIEvBnUU7wl4o3TMtghJRPQB48GQvCoyav3mRMMcCr89Z6DxmgjLFf +A1z1cPCAePImWQC/P6H5LnFsBKg1VRzxVEFxEHpORFqsKLOOuHFsnKwERyyDonYz +DI++2FPLq5+LMANonw4hAPPhEpVtngsG2y5i3N2t0i85YOL6mg8HGw4o+GaWH/GG +1f/2hL5lCZXkN4+Z/NEASpQo+svUBJI1BkPxr+F0nqk7QxQabCfFtG5eqqBUESzJ +PeYC6shIySDM1aaQjCySWWyOGXcr5WqphYhRK/QyGmkZv0hPHRI4abcUxrJpAjN1 +ZknZdgNTNJF9AoIBAQCHqsSgwI10RVbaDq147RAI36is7pisoI1dfGWpDrSls+Bs +4/N+2vH24SeeOh7je01jGVzU5HYU7SNCTeFwot3NEeDH05noT+AlgMHOgS5AoRz5 +RXoOygYV1qVc2Qi6cqjM1rup3Sn1j0Q8aeV59hK8L7ioCG33v8HS+BP1IgjBbVoB +Rqv89X4kiQnDWqKtPtvNMv/IiLuGEQJdJsWavKaXSD64w9IkgjsCvrWvkzYK3YSW +/es9e3SdxYJhGNsSpfe8zzGNgThwlDx6OoEcPygYwA5I8WealmbbOwoU6aEbjIfu +JrcGFGLzeHpBO13+oN7DG804PIaXs5O8EP3kKfkHAoIBAQCJTEleOjz8a8d4lOrx +HseWJfKXwllPbRs944NYltWa5B0eGscVYWGOjcVuwZi1ipKC/NcfRg/SKQ/XFKKu +IlvukxSkpV8TDOO2p3oJoxAcylARh2HO0SLIAFu7hXS6fZNY24ZgS8TtX6nphAdi +8ako2oqVXr5QuLq0gsyFLFzCa84NFU/w5c9Oll+gKPvkj/+M3uGHF2fXVDVpXCVh +l5zqzp1DkG+cSPoyyduUlv0tpnBT8j91wWF9Z0Stn4ti2b67y/Uw0O2WG7x3YhY0 ++OicyJdRGSPqqWmvAasKRXWSQ5dxp+TeT3fXFR7ROyDuzxZ2q0i9XdbV3WpebWRX +/iJpAoIBAQCvE1NMazv9A//Y3pXYCDLJpu/U2+py9hsOoQG9aJ/utCC7uTAXSSSm +01pOelpveDzEUTDHxXlCEA1Q3jCs8KLbKM/DyJaLGItLXZw4GUcWjlwp9gykleXz +2ydZUd97Zp+Z4EVmAlu2ZUuT59EISTE5UI/Li1QGUFf8rYjRScr1Gyx+Us+5989G +bpK4rQd66Kq01MHY+f6/LOEaTe2argXYw4OP2Ceya7BSGmxsDr2vWgTvoO3l0zgB +wCX2aylgViOFKNtywOLEj+DosiMsCEwAoMs+Y6pKwjgwzDj+aPFPaz0D/D7Lx9W7 +lxFQOtP28CoGTyQcSIkq6B5BNTSyUHa6 -----END PRIVATE KEY----- diff --git a/tests/gold_tests/pluginTest/client_context_dump/ssl/two.com.pem b/tests/gold_tests/pluginTest/client_context_dump/ssl/two.com.pem index 1d6a9174ae2..96f8ee7093f 100644 --- a/tests/gold_tests/pluginTest/client_context_dump/ssl/two.com.pem +++ b/tests/gold_tests/pluginTest/client_context_dump/ssl/two.com.pem @@ -1,83 +1,84 @@ -----BEGIN CERTIFICATE----- -MIIFWjCCA0ICFDcEKGSZ6+X7UeFhcJSF3MaAdSFaMA0GCSqGSIb3DQEBCwUAMFEx -CzAJBgNVBAYTAlVTMQswCQYDVQQIDAJJTDESMBAGA1UEBwwJQ2hhbXBhaWduMQ8w -DQYDVQQKDAZBcGFjaGUxEDAOBgNVBAMMB2Zvby5jb20wHhcNMTkwNTIyMjAzOTQz -WhcNMTkwNjIxMjAzOTQzWjCBgTEQMA4GA1UEAwwHdHdvLmNvbTEaMBgGCSqGSIb3 -DQEJARYLc3NsQHR3by5jb20xDzANBgNVBAoMBkFwYWNoZTEMMAoGA1UECwwDVHdv -MRIwEAYDVQQHDAlDaGFtcGFpZ24xETAPBgNVBAgMCElsbGlub2lzMQswCQYDVQQG -EwJVUzCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAKsQcVF2TbO+VtT+ -YbBjfwrO7+5X5hs2UY4tV0NBR3LlTmM1/qlql0ayjja2Te87ULZZ1gAyL1InnEvI -HKLPIWaUknKM2LG9y6j+ZToGpw6JdSXuDVgd7UC1SY//XIRVW0FNTWkP/aV7ScUE -4SvPuCjBCnEvddUXfc2DlskJGBDnHQ4P07yc+GnFY2chNHSngQ7ctoazvi0dEi56 -ViXgg53IUXib6veGtNfn//rk9CpQY7AM3qKJsqoKUHhBKPFXlA5z25EkBuPVMbhB -yUyrHGy377nE7FZFKSziHEA5XUS1F+ITbXxTgaPchsoU02spvlHHKSDNEFvc1GDl -n5pPQXJI34lQryjtvrneSt7mz05qKf1DNcN70ZMKkBDMmsn3wt/+TXi2w3vKkyxD -pJHT4P6P7d1kWnp3nOworl53ybcc/DGcBftu2KiMKQ0aF1zjRwwM1uCizg41IxNw -nZYsQPL47lE93kI9OZ0NjhorQI9n7Cs6AS/sRDZSJrlEFQXcidSkfGOaLj0Dhrd0 -UrahIk3r5vhtLw1BL0NQSPeJkwJQ6zkbavhKDvN4Odol2YUWdQAsPl5BJypts/CT -UOQ20FVyQQgXj9GYyJrwwrSm8YF48KrItdOGF9iHyGdeVn3BTMWaokCyWfcAeK1k -CwhCyAT/HNTorSHDSNL1TL45LsdXAgMBAAEwDQYJKoZIhvcNAQELBQADggIBAKiy -DMDfNxNmn5EnAnzwLem/nqNiPKzsjnkYChBJXVTy5uuOetjigbi7VUPPiFUHZ3+9 -kyXqRBRaGjRf4IhgepFZmpuvuxDQUuVK5XycWAme1NOerWX+y/tAxQM+exWEGAaR -KTqLmiz0fSF5Plcg71a7GS2S4drmnn6vGkDX5aJiCWJsFE694zv8F7u/VRJSF+bj -TY48tSvsGOmGzK6T9BriCaBRIC4kX/6l0zGM6CWd6pdGwT8JflK4jWoW+m8nBFqN -1QuOC3l+F4eBGCDqxyh8lZkssf2ZRxmX2W6nBFEZ7NUR+XnAUZSgpvZwULm/fMJP -gr5I8OXc8oLIIPSWRagm7O3Plyw4EeVEmrcBNoRMs/7wfQz4H+Hfct/cle8O0cWT -censViZ2JRs9BM9JHz0LnW8QhcHD8+Qu9QAJQDw3ILrhjuiMjEnT0a10ILXZftQv -QQOxsKlwxO5dcUN31JZYCx3sr83ApGEGDksIRiUDjlwO5k1jBD2Co7K4oVFgC6T1 -9nOPC36R3XcAZ7ed3XXyq5WDFC6cdjs5a+V9ScSVzaAwziHFKIeVn7Hy4pncPd5/ -yafdgQzE7o3DO94chOzb6lJn7jyERuJAUWXsRX2esT9fsoNQDJhZ9tdFNq+JVVwO -jypGgJ0QTy7rVlu+HEgMOt9WGOjYNb0BRN0at2+d +MIIFdTCCA12gAwIBAgIJAJnW+1ZJ9nHLMA0GCSqGSIb3DQEBCwUAMFExCzAJBgNV +BAYTAlVTMQswCQYDVQQIDAJJTDESMBAGA1UEBwwJQ2hhbXBhaWduMQ8wDQYDVQQK +DAZBcGFjaGUxEDAOBgNVBAMMB3R3by5jb20wHhcNMjAwNTI2MjEzMTEwWhcNMzAw +NTI0MjEzMTEwWjBRMQswCQYDVQQGEwJVUzELMAkGA1UECAwCSUwxEjAQBgNVBAcM +CUNoYW1wYWlnbjEPMA0GA1UECgwGQXBhY2hlMRAwDgYDVQQDDAd0d28uY29tMIIC +IjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEA1rNBd2hcGePkcQfuWftUi9jw +gp1mu2ICIUgRk4wnfX45doijmPVU6LfNSmR50BJp2BQU/ojvZv2aEFPoElmJlXTT +DXV0UjROvGFpDUJI024FnlBUUiCGSfHV0+iBXbi4ufWgylCvcmtXGxwKD/RIdWUN +05a1lN68PFDy6SBrPpgWgWLX+N/SlCUVDTFT6vCLEZ+S8oHy8+UfX2ppckMjqf3H +PLyRuzraBVS2HDMqaWHc2rZq6sHzLe/beDVvVFzseAMfjlHXXCHlKWOJakceQ3DP +vGLO7eruFtxAnCt1aOYgsVlzycUBmmB3yAbHaeWk0rI55FpUwVzdA4NXljZL367f +w7dO/AlM+KVjDABYaf3SWN2Oycz94jPOFAA3mgQEbmjdpibwT5p3I5QAQ01Jv72w +8/eF9tQPpSLW6etyjxn0djXC510VGsfy5dYDmeaADsGStkZEoAsgqzGvHNnH8MLp +PuIatC9rcQphcdscH9AzDPZF3tATJLyUH/52lQIE1tD36Iket1z2VDl63MMDJ6Og +Yz7xYRU5lUmUpZtPwQ9wO0ZpNosFBUQ/vEbhuuV0OFX/BkXwamjUapgQ+gsh8fox +Fhnpxnd5HwWoldFlVDhLmTxbuADyHj/DHt0g6l2+Oo+moFebdr6+i+EDjlGH3l7C +1Fpl1sQo0hMXqH8JsHUCAwEAAaNQME4wHQYDVR0OBBYEFIjAgTH38eZaHInMf6d+ +ZL/2ja7DMB8GA1UdIwQYMBaAFIjAgTH38eZaHInMf6d+ZL/2ja7DMAwGA1UdEwQF +MAMBAf8wDQYJKoZIhvcNAQELBQADggIBADFzT7m1csNNxxkssAu5KJWAfbI++47n +tUQ7o56J8GtIlh7bKgemnur3EFc551QxBXqfFpr7Cfo4p4EruzLtLqFze6LBQLqD +UOhhdCEqv8ruCJXCZzWQZHBs+F5b4BD3oARb9pOKNWJxJapd1aNllq1XSUHfmoT/ +ISqSAVrpDa+zne9Iq7UpFmjqXhlDCfOd4jwhjeQSPl/y4Jun8T+VcpTveWHQxAvt +xOlHEpUQJGwHJpXIbQ7FjtF1ICiU5j7C/T6exom/QXrysqlRH+SmAH9ItE1xFd6X +QVKKcPWqwppzpjXuY6BfUoNJGkl5CYhuoEBOmc1Rno5H2E+DmyCUelTeyDLWBBfC +OnuQd1bIipkSTTs+itjSkifkrtt4NxZ4DMZJJxc2NRet+29b6RUVV8GkWPv2X9w+ +cJZ9hnTYDFwZ4l7RbhSZgWiCXW8AXltzer8Zzu6qCjxH8fk+yoJHMJh1tafWP901 +hOOMCNH64fGc3p0BzKrkPIXrQV8MprsB58z11GmKdwBgL2Xd7pyC2omkSFaEMzU+ +KyZIurD/2Gag8c4X3Hfc/KBwUrgE/9lKSLyJe8ywJBmOtIy0MH8Co6AVeSLmZdow +87AOC9wwUrrXem9Jm6gnBalSROvfMzph11cwInKjyUpcASdG7MVzeizDE77HdhNK +NcWl2jQQJhQg -----END CERTIFICATE----- -----BEGIN PRIVATE KEY----- -MIIJQwIBADANBgkqhkiG9w0BAQEFAASCCS0wggkpAgEAAoICAQCrEHFRdk2zvlbU -/mGwY38Kzu/uV+YbNlGOLVdDQUdy5U5jNf6papdGso42tk3vO1C2WdYAMi9SJ5xL -yByizyFmlJJyjNixvcuo/mU6BqcOiXUl7g1YHe1AtUmP/1yEVVtBTU1pD/2le0nF -BOErz7gowQpxL3XVF33Ng5bJCRgQ5x0OD9O8nPhpxWNnITR0p4EO3LaGs74tHRIu -elYl4IOdyFF4m+r3hrTX5//65PQqUGOwDN6iibKqClB4QSjxV5QOc9uRJAbj1TG4 -QclMqxxst++5xOxWRSks4hxAOV1EtRfiE218U4Gj3IbKFNNrKb5RxykgzRBb3NRg -5Z+aT0FySN+JUK8o7b653kre5s9Oain9QzXDe9GTCpAQzJrJ98Lf/k14tsN7ypMs -Q6SR0+D+j+3dZFp6d5zsKK5ed8m3HPwxnAX7btiojCkNGhdc40cMDNbgos4ONSMT -cJ2WLEDy+O5RPd5CPTmdDY4aK0CPZ+wrOgEv7EQ2Uia5RBUF3InUpHxjmi49A4a3 -dFK2oSJN6+b4bS8NQS9DUEj3iZMCUOs5G2r4Sg7zeDnaJdmFFnUALD5eQScqbbPw -k1DkNtBVckEIF4/RmMia8MK0pvGBePCqyLXThhfYh8hnXlZ9wUzFmqJAsln3AHit -ZAsIQsgE/xzU6K0hw0jS9Uy+OS7HVwIDAQABAoICAQCJpLBZiOSa1XPO7GS0Zkqp -6rq6QDXh/YH/8dG+Rv9znrjFMYQY07CnbTLrKSNqdILMR1rfS4IyC7dCbuFDy4Cn -prJzw4r08a+26gOPfhzboJUHkRVhqqrlnzlyyVmrDXdhAw9fk0NX7Oz9v6Bi/T/E -YxfA3Rxl+wH55IDmgA/CQgRp9Sg9ItzrVq1WJSytFL7Os5+WoXhLmpGvnjZFQfMF -eVK8xlB5HQXUmFOrkKA6j/a2iJR3mm6NTcFUEbIdB4gVXPn2PlPg7QGVrjGIJEK5 -5ALbqm00OXAZMlLjBoVarJBsE4/MMvEkZWR7o+g92RfPe35Ha1lXYUfuM7WJl24j -c0+xPdZKQ8FzMXstcbPS7aVDdO6NTn2MttTYqpBtGpycI2yJpbKXym8vsWfhlPn+ -h4FzT7hx48nu5O89qBjFNP6IbKdyPmq8B4Sij6/TZL2PKCaZifMwAS6Z1Zh76pDo -07tfgNA2mxO2LMWM3waj1mkwsxkq+pxY8jsnmO9smJBB2RdXCTCK/EupLfFXzGXE -QN/jNeUPi9SsP4XH0kzNyZXxOmdgpt0WoL3Tla3/Y/UJDRZkro054B2wZtqf/mdj -RmIYc5IqZ3ScVFni5CQlkQmjr4IPXDzOQQnXaTtYSQFjoJlx3gvCBbDSf26O/I8O -ej3ym4vH4KkFahXp+AKPKQKCAQEA1fAbhoBiumxPvm7Tn3vB2bXfy24jU7ceBile -ePuPr9UakZ5JMzepBq+rKBqONKDz60xa40+XclDWt1yoJlANB462ns47hlrlXpm9 -1EN5XP3ZM7U1dELqgFwkAOiIERgKotTYKy0UyQb+Hj4/J2ymjLZHaKA2TeSSZdET -t8/KkWOiUdcjQv89IKW4CSdDlH2AiBlH09nt4r8kCQGEdBJM+OgQ9Ce2erBIuKg5 -Qqxgl1YEjenrSwpPsA/FrUYICPZCkq21e7NfsGLREl+ihJWs5JdTyiTSpi/E+ReN -ZCQGnIJa70gpuPYbFx6Q2vnB3rVCcdotnIfou3DMfRFapcuqNQKCAQEAzLJsNtxv -IJlU2gsIU9wY9Bbhkz9Q06IaxvaA2dj9ZhAed+F1v+1WEFR9n7k0ymU8Q8zc1qaI -kVwDbX459cTYyBvmo9WtNW//UOr1ylaUjSGXDyU+QqSgZ9e0AnsleyCUduCJWYhP -Ik7b7R02rrL3dXT+slf9HrMfedeVAzgGPzUeJS5MHsZQKddJP//rR+HbI9QKD7iv -LvVfQcEerkVDSDJ8VgoFQcLUmxu3nodhX8DLA19PXg0VoX3GR1HV8hMjBCJOrtv0 -asTfZ3J39dOtraaAOqJZhhv/0bQxs05RoBSxzLZydSRDSf1D9XE611UBm3pBJgFj -ej56MvMjE8j82wKCAQARSPAAYvkXFM8wlKW2efpEi8REHGbwSZg8aTU/0xtd2nrm -DwLdB385khHjEJoyuFpcxXOGcBTNYKioce0IA2m6FZa9p+35QfjMNuG2d6kjkULu -QZLLDOkDa+5gwGjV8LpTQ50fh+npAA6iBOd3WPjv335PfrpEetY1MbpFHJ3CN2mS -8S3hKNwYeisvWiPEqIss89Xw0Oe+bTENJTk9Y4kihyVvhJHiwcFuYfEWaPT45TND -AAZJrtCXaf7PrBvUFYl1bmF+WBTAHIvFz0JDMhCg+3UCnQ0D7lIcygFbeOmr1YBh -WtQ8JG415PtRJeK7CqwOpNEQl45/LnLnG/LV+GmNAoIBAQCcsMvjZvPuGVF5o05Z -mzbCyi0coTAHAKTMvu89UzwN/7LDA6Q2KcBiubp8JLLDZ6EtKUm3Sj1qP+wjyacm -eeuTqr/vk+aF7FidoW5K1+HY8uiGYHT7YLelJdoWuBul1/et1A1vqscgtQrmxCES -s853a/p3nKEW+fjTNPJNR3qIsemEfp8oQ5gjnzfkNMvu93XfRDX7eN0o7g1f8SvC -LSTmxDanSf5iK3jBzwLM6EbinFsLFs9TaGKxfuzjtUI3juyUAosGkTrU7CUzM82e -MM5XCghIWfR7kz1NUkllP0N+bbj4woR1JTAZGDUIUge/w8N6N6hdJlz2u5KhI7sr -LwRXAoIBAFszzSQOSuz35JucXD5IK6qw6ehF2xQPPNt4MMGoEQXacc7m3dFVXPOm -PGf4JZw/VRwXoyaD+D7yBNCmjW3wmMgAlnkvVkHJfOscpmXBCYwm7zsNGxpZLcRo -5esVqGKuomFqfjLYMuPuH1tydidDzotomcZvqCPDBywuH/aLfbO/ojpinLGyJU6t -yfNjkOS2dfjiL/2DiFzDxv3WlE1/fxBK+iBTcAiCJLjZjgQMB3u43rwXLOkWpaCd -3eIMPIjmamX2VwHyk5svQ5IYzgm8Iiy3MWOdgnIrdplWd8yn0sFnes10G4QmNn5e -k7+f38jMk3EvPFZ5xgE1uAtgf8IFEeU= +MIIJQQIBADANBgkqhkiG9w0BAQEFAASCCSswggknAgEAAoICAQDWs0F3aFwZ4+Rx +B+5Z+1SL2PCCnWa7YgIhSBGTjCd9fjl2iKOY9VTot81KZHnQEmnYFBT+iO9m/ZoQ +U+gSWYmVdNMNdXRSNE68YWkNQkjTbgWeUFRSIIZJ8dXT6IFduLi59aDKUK9ya1cb +HAoP9Eh1ZQ3TlrWU3rw8UPLpIGs+mBaBYtf439KUJRUNMVPq8IsRn5LygfLz5R9f +amlyQyOp/cc8vJG7OtoFVLYcMyppYdzatmrqwfMt79t4NW9UXOx4Ax+OUddcIeUp +Y4lqRx5DcM+8Ys7t6u4W3ECcK3Vo5iCxWXPJxQGaYHfIBsdp5aTSsjnkWlTBXN0D +g1eWNkvfrt/Dt078CUz4pWMMAFhp/dJY3Y7JzP3iM84UADeaBARuaN2mJvBPmncj +lABDTUm/vbDz94X21A+lItbp63KPGfR2NcLnXRUax/Ll1gOZ5oAOwZK2RkSgCyCr +Ma8c2cfwwuk+4hq0L2txCmFx2xwf0DMM9kXe0BMkvJQf/naVAgTW0PfoiR63XPZU +OXrcwwMno6BjPvFhFTmVSZSlm0/BD3A7Rmk2iwUFRD+8RuG65XQ4Vf8GRfBqaNRq +mBD6CyHx+jEWGenGd3kfBaiV0WVUOEuZPFu4APIeP8Me3SDqXb46j6agV5t2vr6L +4QOOUYfeXsLUWmXWxCjSExeofwmwdQIDAQABAoICAE7km7/W91EELuNh+LHT6uNg +mjBEyoU8pG+6XcsMC07P3SmzV/pS3BqPv74AYcqVmZMu/e/yPZZoxf4N9w8XG7lx +rkQZzt6OHgiW+tVGcCBRI3tjJTjwfqgZV2uA2lioEikzGxx8vZub2ew6R3BJYX7R +3VzM2niTyDOZw0jrAr2DgXGV8qqfBHAvmCa6441HSrL1Ijg+jBoXLYoQPiMgjLBg +cp/c4ugb3hVrZWfDXWhP76nmsi8FiehtZQYtDHXBl5sZ+rR/WlWelui4jLwi2foK +MCuXW1aNp2N7F7rLNV+QgJ7qseOIougpXpp/8zL0kbZii2df0hkjmIs77iliVkp1 +WXJrSXcLdiw4w3LyA2U9p5tvwB1V1OnmiT/sYRYKapRXp8RQOUaTxJWdGU7jEDv8 +8gvroQo3aVPJVHHmEJ003Q6E5dq/b2fvVlLeTc2CBCUHEhtYk2hwV+IbdzJhiVJ8 +5MsM0B/M/DTu23F6BKZLO9BWm88UCxbqJ/jI+HB+3N23XzvO9pjrFWPYu7JHWIPK +j/pCkTHQWx01DGy4nlATA8Ku+nxalZsT3wRhhjs9sneKqXrP0m2Uxhs0B4RGOLuj +3YYZLX8U968OD0qSHTwJBjrK6cZ2TgWChqVRMPOuk5d8g45ZocAlfGCM/gorKk9Q +CMEAxHp5KDnwaa8rUHkZAoIBAQDrc9clBIesXbAkOxNMeLZRHy29uQkBmxmrEAif +szReMxVjdL0q8WdMNHmZsw+28y2GGk4wNy7A8+ULt4cm/Edm1/2JJlYQdgbxJHQo +Powix2t6vfGiEekyrncA8nOz2IxxrkNHyuh8scQydWMvdc34/C6OYgQrX8EGwOYV +CzYwSo96VlZOzJfL0Y8h25uG8b6okhwmMiicVhIYSFZgUe+qPaKpWby8XoQkjE8K +FR+LRTI4avyKImaGsehEVV7rzAKiG3YLXSbykGlQamJtDBWJsAR4wrvzbd9j/Phy +WhAViIsYIITEQyCwdJzfOdrVBaNiAFjDTCL5i08hkBIOCNaPAoIBAQDpb8vHl7Ze +HMNhoUnBDUV9k8E7Tv3FTXuj29jYI+7ORTBuldWSzVyMTtpZZA6RhXYIwDP5N7cg +BLvJ5rv4zOdk7s9axnR0wowKBUthsRz9Nr7A+6ZGHIzAgM4f7VlvHhnx8pc8OHzJ +tUD6BS8T2NTvc0vUgmX8zaLiQCJ+Zr3BcuEM+VgT5f4glhRZz71dYAUhB6K5dlss +qMYZJlySMRzV3BxhIqj5LBbDHAxDADg8Vjcm5p7B2dI6EoVzcjX3gPJgxImOOV0N +pk/6cV5YS74jHQ+IS6UTD3vZf1qHXpoqMoRT7CDrrqK64KZ44k8ES7EQyVw27CuD +qTF3OIt7Rqq7AoIBABzJvBt1k1Ua22eCpZhrBfejNUeGMTi2CwxkWHmWQqfl3Q6c +/mgavgOOQvPs9dXro7xKyRaSGLBU4cNt/CNjIyPK8t/rBQjcNXsUdvNYum+iLF0E +jl+/iuC8fiZqfoUVzA42PVmw0H42igOVYLwzzixMQTD9gP8ApSw7N6aJCWYcJ7DQ +j5rw3vp3s+Vy4XA9bI9V1gVBir+asnmvlogOXJxXDuVf6HirocKKOTvFW2b+3eYS +BrCpBhof9IMuYP25ovTJIMTA7fsodwfzCffFS3jbQaC2Rp67511lV8c7hc3sNP2N +NE5+2qdYirmsbDoxh0l7O18dlwUl6FX1NI/surcCggEAc1Lo0KUx0lLRb5UX88jT +sNNjHiLLGkNKNsxw128BFmR2JdveFFViSGrYUlstG1Vc17vksVWGIcpU380exi6s +jl+wjv8GH3zRZE3zTMMMOEhoeryYC4ElRCEeNAW1LkEnvjED6BtcAuaJwDr5ZAu2 +PsmuB8GYEwmgU7gtkla9nilb52XzznjMUFr0mN4Zhlzei8/S418Gcgy7OUeLainb +Pggs+Qv/gIH4iNF9eUhNf/lz68L5YXlz836c8UjWQn0wwP+3mUzbnNeyIV8KpgJd +X8mGl+YPGliG7g+NsNzausgUOcWpCeZPvJUpiQT81nlyc4GXDgclBDl9F5IWzoM9 +6wKCAQBFErVBuC3KEkJyaA/9wphWXnJeG/8zWqGyxNkyWTeFnNpUmwDSgZ97QI9L +NFkx2FYF3EIY0XS+NMtiW6/BkTKZdAMZSBIu4wBmWJepr8vlwfU3/jLNHYZcF65h +uw82kluZcR7fHHXPKry/yjnnNcvmPP8ZIAviQjStSzSi1zf+HHSPIHIG9d3Co31s +ko2VPWcVzmJ2ZZzIIeajW7zzrIjp5LDo8gmQZ+5IEz6bW3Q57h9wIY9o+7cG36ZB +68qOHKmC/mnzjIAqG3Wtd4ZQEijexClyDEXKIva9FE+shLEijtAzF1bvE1Dm4OO2 +nJI86WZ54jWQJABr8lsOZ510mLx7 -----END PRIVATE KEY----- diff --git a/tests/gold_tests/pluginTest/combo_handler/combo_handler.test.py b/tests/gold_tests/pluginTest/combo_handler/combo_handler.test.py new file mode 100644 index 00000000000..72528e74069 --- /dev/null +++ b/tests/gold_tests/pluginTest/combo_handler/combo_handler.test.py @@ -0,0 +1,142 @@ +''' +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +Test.Summary = ''' +Test combo_handler plugin +''' + +# Skip if plugin not present. +# +Test.SkipUnless( + Condition.PluginExists('combo_handler.so'), +) + +# Function to generate a unique data file path (in the top level of the test's run directory), put data (in string 'data') into +# the file, and return the file name. +# +_data_file__file_count = 0 + + +def data_file(data): + global _data_file__file_count + file_path = Test.RunDirectory + "/tcp_client_in_{}".format(_data_file__file_count) + _data_file__file_count += 1 + with open(file_path, "x") as f: + f.write(data) + return file_path + +# Function to return command (string) to run tcp_client.py tool. 'host' 'port', and 'file_path' are the parameters to tcp_client. +# + + +def tcp_client_cmd(host, port, file_path): + return "python3 {}/tcp_client.py {} {} {}".format(Test.Variables.AtsTestToolsDir, host, port, file_path) + +# Function to return command (string) to run tcp_client.py tool. 'host' and 'port' are the first two parameters to tcp_client. +# 'data' is the data to put in the data file input to tcp_client. +# + + +def tcp_client(host, port, data): + return tcp_client_cmd(host, port, data_file(data)) + + +server = Test.MakeOriginServer("server") + + +def add_server_obj(content_type, path): + request_header = { + "headers": "GET " + path + " HTTP/1.1\r\n" + + "Host: just.any.thing\r\n\r\n", + "timestamp": "1469733493.993", + "body": "" + } + response_header = { + "headers": "HTTP/1.1 200 OK\r\n" + + "Connection: close\r\n" + + 'Etag: "359670651"\r\n' + + "Cache-Control: public, max-age=31536000\r\n" + + "Accept-Ranges: bytes\r\n" + + "Content-Type: " + content_type + "\r\n" + + "\r\n", + "timestamp": "1469733493.993", + "body": "Content for " + path + "\n" + } + server.addResponse("sessionfile.log", request_header, response_header) + + +add_server_obj("text/css ; charset=utf-8", "/obj1") +add_server_obj("text/javascript", "/sub/obj2") +add_server_obj("text/argh", "/obj3") +add_server_obj("application/javascript", "/obj4") + +ts = Test.MakeATSProcess("ts") + +ts.Disk.records_config.update({ + 'proxy.config.diags.debug.enabled': 1, + 'proxy.config.diags.debug.tags': 'http|combo_handler', +}) + +ts.Disk.plugin_config.AddLine("combo_handler.so - - - ctwl.txt") + +ts.Disk.remap_config.AddLine( + 'map http://xyz/ http://127.0.0.1/ @plugin=combo_handler.so' +) +ts.Disk.remap_config.AddLine( + 'map http://localhost/127.0.0.1/ http://127.0.0.1:{}/'.format(server.Variables.Port) +) +ts.Disk.remap_config.AddLine( + 'map http://localhost/sub/ http://127.0.0.1:{}/sub/'.format(server.Variables.Port) +) + +ts.Disk.File(ts.Variables.CONFIGDIR + "/ctwl.txt", id="ctwl_cfg", typename="ats:config") +ts.Disk.ctwl_cfg.AddLine("# test ") +ts.Disk.ctwl_cfg.AddLine("") +ts.Disk.ctwl_cfg.AddLine(" text/javascript # test side comment") +ts.Disk.ctwl_cfg.AddLine(" application/javascript") +ts.Disk.ctwl_cfg.AddLine("text/css") + +tr = Test.AddTestRun() +tr.Processes.Default.StartBefore(ts, ready=When.PortOpen(ts.Variables.port)) +tr.Processes.Default.StartBefore(server, ready=When.PortOpen(server.Variables.Port)) +tr.Processes.Default.Command = "echo start stuff" +tr.Processes.Default.ReturnCode = 0 + +tr = Test.AddTestRun() +tr.Processes.Default.Command = tcp_client("127.0.0.1", ts.Variables.port, + "GET /admin/v1/combo?obj1&sub:obj2&obj3 HTTP/1.1\n" + + "Host: xyz\n" + + "Connection: close\n" + + "\n" + ) +tr.Processes.Default.ReturnCode = 0 +f = tr.Disk.File("_output/1-tr-Default/stream.all.txt") +f.Content = "combo_handler_files/tr1.gold" + +tr = Test.AddTestRun() +tr.Processes.Default.Command = tcp_client("127.0.0.1", ts.Variables.port, + "GET /admin/v1/combo?obj1&sub:obj2&obj4 HTTP/1.1\n" + + "Host: xyz\n" + + "Connection: close\n" + + "\n" + ) +tr.Processes.Default.ReturnCode = 0 +f = tr.Disk.File("_output/2-tr-Default/stream.all.txt") +f.Content = "combo_handler_files/tr2.gold" + +ts.Disk.diags_log.Content = Testers.ContainsExpression("ERROR", "Some tests are failure tests") diff --git a/tests/gold_tests/pluginTest/combo_handler/combo_handler_files/tr1.gold b/tests/gold_tests/pluginTest/combo_handler/combo_handler_files/tr1.gold new file mode 100644 index 00000000000..133179b94b8 --- /dev/null +++ b/tests/gold_tests/pluginTest/combo_handler/combo_handler_files/tr1.gold @@ -0,0 +1,9 @@ +HTTP/1.1 403 Forbidden +Date: `` +Age: `` +Transfer-Encoding: chunked +Connection: close +Server: ATS/`` + +0 + diff --git a/tests/gold_tests/pluginTest/combo_handler/combo_handler_files/tr2.gold b/tests/gold_tests/pluginTest/combo_handler/combo_handler_files/tr2.gold new file mode 100644 index 00000000000..4d775f60930 --- /dev/null +++ b/tests/gold_tests/pluginTest/combo_handler/combo_handler_files/tr2.gold @@ -0,0 +1,18 @@ +HTTP/1.1 200 OK +Vary: Accept-Encoding +Last-Modified: `` +Content-Type: text/css ; charset=utf-8 +Cache-Control: max-age=31536000, Public +Date: `` +Age: `` +Transfer-Encoding: chunked +Connection: close +Server: ATS/`` + +3a +Content for /obj1 +Content for /sub/obj2 +Content for /obj4 + +0 + diff --git a/tests/gold_tests/pluginTest/compress/compress.gold b/tests/gold_tests/pluginTest/compress/compress.gold index af5b4f0194a..05c8eb579bc 100644 --- a/tests/gold_tests/pluginTest/compress/compress.gold +++ b/tests/gold_tests/pluginTest/compress/compress.gold @@ -1,296 +1,179 @@ > GET http://ae-0/obj0 HTTP/1.1 -> X-Ats-Compress-Test: ts/0/gzip, deflate, sdch, br +> X-Ats-Compress-Test: 0/gzip, deflate, sdch, br > Accept-Encoding: gzip, deflate, sdch, br < HTTP/1.1 200 OK < Content-Type: text/javascript < Content-Encoding: br < Vary: Accept-Encoding < Content-Length: 46 - +=== > GET http://ae-0/obj0 HTTP/1.1 -> X-Ats-Compress-Test: ts/0/gzip +> X-Ats-Compress-Test: 0/gzip > Accept-Encoding: gzip < HTTP/1.1 200 OK < Content-Type: text/javascript < Content-Encoding: gzip < Vary: Accept-Encoding < Content-Length: 71 - +=== > GET http://ae-0/obj0 HTTP/1.1 -> X-Ats-Compress-Test: ts/0/br +> X-Ats-Compress-Test: 0/br > Accept-Encoding: br < HTTP/1.1 200 OK < Content-Type: text/javascript < Content-Encoding: br < Vary: Accept-Encoding < Content-Length: 46 - +=== > GET http://ae-0/obj0 HTTP/1.1 -> X-Ats-Compress-Test: ts/0/deflate +> X-Ats-Compress-Test: 0/deflate > Accept-Encoding: deflate < HTTP/1.1 200 OK < Content-Type: text/javascript < Content-Length: 1049 - +=== > GET http://ae-1/obj1 HTTP/1.1 -> X-Ats-Compress-Test: ts/1/gzip, deflate, sdch, br +> X-Ats-Compress-Test: 1/gzip, deflate, sdch, br > Accept-Encoding: gzip, deflate, sdch, br < HTTP/1.1 200 OK < Content-Type: text/javascript < Content-Encoding: gzip < Vary: Accept-Encoding < Content-Length: 71 - +=== > GET http://ae-1/obj1 HTTP/1.1 -> X-Ats-Compress-Test: ts/1/gzip +> X-Ats-Compress-Test: 1/gzip > Accept-Encoding: gzip < HTTP/1.1 200 OK < Content-Type: text/javascript < Content-Encoding: gzip < Vary: Accept-Encoding < Content-Length: 71 - +=== > GET http://ae-1/obj1 HTTP/1.1 -> X-Ats-Compress-Test: ts/1/br +> X-Ats-Compress-Test: 1/br > Accept-Encoding: br < HTTP/1.1 200 OK < Content-Type: text/javascript < Content-Length: 1049 - +=== > GET http://ae-1/obj1 HTTP/1.1 -> X-Ats-Compress-Test: ts/1/deflate +> X-Ats-Compress-Test: 1/deflate > Accept-Encoding: deflate < HTTP/1.1 200 OK < Content-Type: text/javascript < Content-Length: 1049 - +=== > GET http://ae-2/obj2 HTTP/1.1 -> X-Ats-Compress-Test: ts/2/gzip, deflate, sdch, br +> X-Ats-Compress-Test: 2/gzip, deflate, sdch, br > Accept-Encoding: gzip, deflate, sdch, br < HTTP/1.1 200 OK < Content-Type: text/javascript < Content-Encoding: br < Vary: Accept-Encoding < Content-Length: 46 - +=== > GET http://ae-2/obj2 HTTP/1.1 -> X-Ats-Compress-Test: ts/2/gzip +> X-Ats-Compress-Test: 2/gzip > Accept-Encoding: gzip < HTTP/1.1 200 OK < Content-Type: text/javascript < Content-Encoding: gzip < Vary: Accept-Encoding < Content-Length: 71 - +=== > GET http://ae-2/obj2 HTTP/1.1 -> X-Ats-Compress-Test: ts/2/br +> X-Ats-Compress-Test: 2/br > Accept-Encoding: br < HTTP/1.1 200 OK < Content-Type: text/javascript < Content-Encoding: br < Vary: Accept-Encoding < Content-Length: 46 - +=== > GET http://ae-2/obj2 HTTP/1.1 -> X-Ats-Compress-Test: ts/2/deflate +> X-Ats-Compress-Test: 2/deflate > Accept-Encoding: deflate < HTTP/1.1 200 OK < Content-Type: text/javascript < Content-Length: 1049 - +=== > GET http://ae-0/obj0 HTTP/1.1 -> X-Ats-Compress-Test: ts2/0/gzip -> Accept-Encoding: gzip +> X-Ats-Compress-Test: 0/gzip;q=0.666 +> Accept-Encoding: gzip;q=0.666 < HTTP/1.1 200 OK < Content-Type: text/javascript < Content-Encoding: gzip < Vary: Accept-Encoding < Content-Length: 71 - +=== > GET http://ae-0/obj0 HTTP/1.1 -> X-Ats-Compress-Test: ts2/0/gzip -> Accept-Encoding: gzip +> X-Ats-Compress-Test: 0/gzip;q=0.666x +> Accept-Encoding: gzip;q=0.666x < HTTP/1.1 200 OK < Content-Type: text/javascript < Content-Encoding: gzip < Vary: Accept-Encoding < Content-Length: 71 - +=== > GET http://ae-0/obj0 HTTP/1.1 -> X-Ats-Compress-Test: ts2/0/br -> Accept-Encoding: br -< HTTP/1.1 200 OK -< Content-Type: text/javascript -< Content-Encoding: br -< Vary: Accept-Encoding -< Content-Length: 46 - -> GET http://ae-0/obj0 HTTP/1.1 -> X-Ats-Compress-Test: ts2/0/deflate -> Accept-Encoding: deflate -< HTTP/1.1 200 OK -< Content-Type: text/javascript -< Content-Length: 1049 - -> GET http://ae-1/obj1 HTTP/1.1 -> X-Ats-Compress-Test: ts2/1/gzip -> Accept-Encoding: gzip +> X-Ats-Compress-Test: 0/gzip;q=#0.666 +> Accept-Encoding: gzip;q=#0.666 < HTTP/1.1 200 OK < Content-Type: text/javascript < Content-Encoding: gzip < Vary: Accept-Encoding < Content-Length: 71 - -> GET http://ae-1/obj1 HTTP/1.1 -> X-Ats-Compress-Test: ts2/1/gzip -> Accept-Encoding: gzip -< HTTP/1.1 200 OK -< Content-Type: text/javascript -< Content-Encoding: gzip -< Vary: Accept-Encoding -< Content-Length: 71 - -> GET http://ae-1/obj1 HTTP/1.1 -> X-Ats-Compress-Test: ts2/1/br -> Accept-Encoding: br -< HTTP/1.1 200 OK -< Content-Type: text/javascript -< Content-Length: 1049 - -> GET http://ae-1/obj1 HTTP/1.1 -> X-Ats-Compress-Test: ts2/1/deflate -> Accept-Encoding: deflate -< HTTP/1.1 200 OK -< Content-Type: text/javascript -< Content-Length: 1049 - -> GET http://ae-2/obj2 HTTP/1.1 -> X-Ats-Compress-Test: ts2/2/gzip -> Accept-Encoding: gzip -< HTTP/1.1 200 OK -< Content-Type: text/javascript -< Content-Encoding: gzip -< Vary: Accept-Encoding -< Content-Length: 71 - -> GET http://ae-2/obj2 HTTP/1.1 -> X-Ats-Compress-Test: ts2/2/gzip -> Accept-Encoding: gzip +=== +> GET http://ae-0/obj0 HTTP/1.1 +> X-Ats-Compress-Test: 0/gzip; Q = 0.666 +> Accept-Encoding: gzip; Q = 0.666 < HTTP/1.1 200 OK < Content-Type: text/javascript < Content-Encoding: gzip < Vary: Accept-Encoding < Content-Length: 71 - -> GET http://ae-2/obj2 HTTP/1.1 -> X-Ats-Compress-Test: ts2/2/br -> Accept-Encoding: br -< HTTP/1.1 200 OK -< Content-Type: text/javascript -< Content-Encoding: br -< Vary: Accept-Encoding -< Content-Length: 46 - -> GET http://ae-2/obj2 HTTP/1.1 -> X-Ats-Compress-Test: ts2/2/deflate -> Accept-Encoding: deflate -< HTTP/1.1 200 OK -< Content-Type: text/javascript -< Content-Length: 1049 - +=== > GET http://ae-0/obj0 HTTP/1.1 -> X-Ats-Compress-Test: ts3/0/deflate -> Accept-Encoding: deflate +> X-Ats-Compress-Test: 0/gzip;q=0.0 +> Accept-Encoding: gzip;q=0.0 < HTTP/1.1 200 OK < Content-Type: text/javascript < Content-Length: 1049 - +=== > GET http://ae-0/obj0 HTTP/1.1 -> X-Ats-Compress-Test: ts3/0/gzip -> Accept-Encoding: gzip +> X-Ats-Compress-Test: 0/gzip;q=-0.1 +> Accept-Encoding: gzip;q=-0.1 < HTTP/1.1 200 OK < Content-Type: text/javascript < Content-Encoding: gzip < Vary: Accept-Encoding < Content-Length: 71 - +=== > GET http://ae-0/obj0 HTTP/1.1 -> X-Ats-Compress-Test: ts3/0/br -> Accept-Encoding: br -< HTTP/1.1 200 OK -< Content-Type: text/javascript -< Content-Encoding: br -< Vary: Accept-Encoding -< Content-Length: 46 - -> GET http://ae-0/obj0 HTTP/1.1 -> X-Ats-Compress-Test: ts3/0/deflate -> Accept-Encoding: deflate -< HTTP/1.1 200 OK -< Content-Type: text/javascript -< Content-Length: 1049 - -> GET http://ae-1/obj1 HTTP/1.1 -> X-Ats-Compress-Test: ts3/1/deflate -> Accept-Encoding: deflate -< HTTP/1.1 200 OK -< Content-Type: text/javascript -< Content-Length: 1049 - -> GET http://ae-1/obj1 HTTP/1.1 -> X-Ats-Compress-Test: ts3/1/gzip -> Accept-Encoding: gzip +> X-Ats-Compress-Test: 0/aaa, gzip;q=0.666, bbb +> Accept-Encoding: aaa, gzip;q=0.666, bbb < HTTP/1.1 200 OK < Content-Type: text/javascript < Content-Encoding: gzip < Vary: Accept-Encoding < Content-Length: 71 - -> GET http://ae-1/obj1 HTTP/1.1 -> X-Ats-Compress-Test: ts3/1/br -> Accept-Encoding: br +=== +> GET http://ae-0/obj0 HTTP/1.1 +> X-Ats-Compress-Test: 0/ br ; q=0.666, bbb +> Accept-Encoding: br ; q=0.666, bbb < HTTP/1.1 200 OK < Content-Type: text/javascript < Content-Encoding: br < Vary: Accept-Encoding < Content-Length: 46 - -> GET http://ae-1/obj1 HTTP/1.1 -> X-Ats-Compress-Test: ts3/1/deflate -> Accept-Encoding: deflate -< HTTP/1.1 200 OK -< Content-Type: text/javascript -< Content-Length: 1049 - -> GET http://ae-2/obj2 HTTP/1.1 -> X-Ats-Compress-Test: ts3/2/deflate -> Accept-Encoding: deflate -< HTTP/1.1 200 OK -< Content-Type: text/javascript -< Content-Length: 1049 - -> GET http://ae-2/obj2 HTTP/1.1 -> X-Ats-Compress-Test: ts3/2/gzip -> Accept-Encoding: gzip +=== +> GET http://ae-0/obj0 HTTP/1.1 +> X-Ats-Compress-Test: 0/aaa, gzip;q=0.666 , +> Accept-Encoding: aaa, gzip;q=0.666 , < HTTP/1.1 200 OK < Content-Type: text/javascript < Content-Encoding: gzip < Vary: Accept-Encoding < Content-Length: 71 - -> GET http://ae-2/obj2 HTTP/1.1 -> X-Ats-Compress-Test: ts3/2/br -> Accept-Encoding: br -< HTTP/1.1 200 OK -< Content-Type: text/javascript -< Content-Encoding: br -< Vary: Accept-Encoding -< Content-Length: 46 - -> GET http://ae-2/obj2 HTTP/1.1 -> X-Ats-Compress-Test: ts3/2/deflate -> Accept-Encoding: deflate -< HTTP/1.1 200 OK -< Content-Type: text/javascript -< Content-Length: 1049 - +=== diff --git a/tests/gold_tests/pluginTest/compress/compress.test.py b/tests/gold_tests/pluginTest/compress/compress.test.py index 65d289010c7..1cc2eb82437 100644 --- a/tests/gold_tests/pluginTest/compress/compress.test.py +++ b/tests/gold_tests/pluginTest/compress/compress.test.py @@ -16,8 +16,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import os -import subprocess Test.Summary = ''' Test compress plugin ''' @@ -32,9 +30,9 @@ Condition.HasATSFeature('TS_HAS_BROTLI') ) - server = Test.MakeOriginServer("server", options={'--load': '{}/compress_observer.py'.format(Test.TestDirectory)}) + def repeat(str, count): result = "" while count > 0: @@ -42,6 +40,7 @@ def repeat(str, count): count -= 1 return result + # Need a fairly big body, otherwise the plugin will refuse to compress body = repeat("lets go surfin now everybodys learnin how\n", 24) body = body + "lets go surfin now everybodys learnin how" @@ -64,83 +63,120 @@ def repeat(str, count): } server.addResponse("sessionfile.log", request_header, response_header) -def curl(ts, name, idx, encodingList): + +def curl(ts, idx, encodingList): return ( "curl --verbose --proxy http://127.0.0.1:{}".format(ts.Variables.port) + - " --header 'X-Ats-Compress-Test: {}/{}/{}'".format(name, idx, encodingList) + + " --header 'X-Ats-Compress-Test: {}/{}'".format(idx, encodingList) + " --header 'Accept-Encoding: {0}' 'http://ae-{1}/obj{1}'".format(encodingList, idx) + - " >> {0}/compress_long.log 2>&1 ; printf '\n\n' >> {0}/compress_long.log".format(Test.RunDirectory) + " 2>> compress_long.log ; printf '\n===\n' >> compress_long.log" ) -def oneTs(name, AeHdr1='gzip, deflate, sdch, br'): - global waitForServer - waitForTs = True +waitForServer = True - ts = Test.MakeATSProcess(name) +waitForTs = True - ts.Disk.records_config.update({ - 'proxy.config.diags.debug.enabled': 0, - 'proxy.config.diags.debug.tags': 'http|compress|cache', - 'proxy.config.http.normalize_ae': 0, - }) +ts = Test.MakeATSProcess("ts") - ts.Disk.remap_config.AddLine( - 'map http://ae-0/ http://127.0.0.1:{}/'.format(server.Variables.Port) + - ' @plugin=compress.so @pparam={}/compress.config'.format(Test.TestDirectory) - ) - ts.Disk.remap_config.AddLine( - 'map http://ae-1/ http://127.0.0.1:{}/'.format(server.Variables.Port) + - ' @plugin=conf_remap.so @pparam=proxy.config.http.normalize_ae=1' + - ' @plugin=compress.so @pparam={}/compress.config'.format(Test.TestDirectory) - ) - ts.Disk.remap_config.AddLine( - 'map http://ae-2/ http://127.0.0.1:{}/'.format(server.Variables.Port) + - ' @plugin=conf_remap.so @pparam=proxy.config.http.normalize_ae=2' + - ' @plugin=compress.so @pparam={}/compress2.config'.format(Test.TestDirectory) - ) +ts.Disk.records_config.update({ + 'proxy.config.http.cache.http': 0, + 'proxy.config.diags.debug.enabled': 1, + 'proxy.config.diags.debug.tags': 'compress', + 'proxy.config.http.normalize_ae': 0, +}) + +ts.Disk.remap_config.AddLine( + 'map http://ae-0/ http://127.0.0.1:{}/'.format(server.Variables.Port) + + ' @plugin=compress.so @pparam={}/compress.config'.format(Test.TestDirectory) +) +ts.Disk.remap_config.AddLine( + 'map http://ae-1/ http://127.0.0.1:{}/'.format(server.Variables.Port) + + ' @plugin=conf_remap.so @pparam=proxy.config.http.normalize_ae=1' + + ' @plugin=compress.so @pparam={}/compress.config'.format(Test.TestDirectory) +) +ts.Disk.remap_config.AddLine( + 'map http://ae-2/ http://127.0.0.1:{}/'.format(server.Variables.Port) + + ' @plugin=conf_remap.so @pparam=proxy.config.http.normalize_ae=2' + + ' @plugin=compress.so @pparam={}/compress2.config'.format(Test.TestDirectory) +) - for i in range(3): +for i in range(3): - tr = Test.AddTestRun() - if (waitForTs): - tr.Processes.Default.StartBefore(ts, ready=When.PortOpen(ts.Variables.port)) - waitForTs = False - if (waitForServer): - tr.Processes.Default.StartBefore(server, ready=When.PortOpen(server.Variables.Port)) - waitForServer = False - tr.Processes.Default.ReturnCode = 0 - tr.Processes.Default.Command = curl(ts, name, i, AeHdr1) + tr = Test.AddTestRun() + if (waitForTs): + tr.Processes.Default.StartBefore(ts, ready=When.PortOpen(ts.Variables.port)) + waitForTs = False + if (waitForServer): + tr.Processes.Default.StartBefore(server, ready=When.PortOpen(server.Variables.Port)) + waitForServer = False + tr.Processes.Default.ReturnCode = 0 + tr.Processes.Default.Command = curl(ts, i, 'gzip, deflate, sdch, br') - tr = Test.AddTestRun() - tr.Processes.Default.ReturnCode = 0 - tr.Processes.Default.Command = curl(ts, name, i, "gzip") + tr = Test.AddTestRun() + tr.Processes.Default.ReturnCode = 0 + tr.Processes.Default.Command = curl(ts, i, "gzip") - tr = Test.AddTestRun() - tr.Processes.Default.ReturnCode = 0 - tr.Processes.Default.Command = curl(ts, name, i, "br") + tr = Test.AddTestRun() + tr.Processes.Default.ReturnCode = 0 + tr.Processes.Default.Command = curl(ts, i, "br") - tr = Test.AddTestRun() - tr.Processes.Default.ReturnCode = 0 - tr.Processes.Default.Command = curl(ts, name, i, "deflate") + tr = Test.AddTestRun() + tr.Processes.Default.ReturnCode = 0 + tr.Processes.Default.Command = curl(ts, i, "deflate") -waitForServer = True +# Test Aceept-Encoding normalization. + +tr = Test.AddTestRun() +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Command = curl(ts, 0, "gzip;q=0.666") -oneTs("ts") -oneTs("ts2", "gzip") -oneTs("ts3", "deflate") +tr = Test.AddTestRun() +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Command = curl(ts, 0, "gzip;q=0.666x") + +tr = Test.AddTestRun() +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Command = curl(ts, 0, "gzip;q=#0.666") +tr = Test.AddTestRun() +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Command = curl(ts, 0, "gzip; Q = 0.666") + +tr = Test.AddTestRun() +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Command = curl(ts, 0, "gzip;q=0.0") + +tr = Test.AddTestRun() +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Command = curl(ts, 0, "gzip;q=-0.1") + +tr = Test.AddTestRun() +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Command = curl(ts, 0, "aaa, gzip;q=0.666, bbb") + +tr = Test.AddTestRun() +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Command = curl(ts, 0, " br ; q=0.666, bbb") + +tr = Test.AddTestRun() +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Command = curl(ts, 0, "aaa, gzip;q=0.666 , ") + +# compress_long.log contains all the output from the curl commands. The tr removes the carriage returns for easier +# readability. Curl seems to have a bug, where it will neglect to output an end of line before outputing an HTTP +# message header line. The sed command is a work-around for this problem. greplog.sh uses the grep command to +# select HTTP request/response line that should be consitent every time the test runs. +# tr = Test.AddTestRun() tr.Processes.Default.ReturnCode = 0 tr.Processes.Default.Command = ( - r"tr -d '\r' < {1}/compress_long.log | sed 's/\(..*\)\([<>]\)/\1\n\2/' | {0}/greplog.sh > {1}/compress_short.log" -).format(Test.TestDirectory, Test.RunDirectory) + r"tr -d '\r' < compress_long.log | sed 's/\(..*\)\([<>]\)/\1\n\2/' | {0}/greplog.sh > compress_short.log" +).format(Test.TestDirectory) f = tr.Disk.File("compress_short.log") f.Content = "compress.gold" -# Have to comment this out, because caching does not seem to be consistent, which is disturbing. -# -# tr = Test.AddTestRun() -# tr.Processes.Default.Command = "echo" -# f = tr.Disk.File("compress_userver.log") -# f.Content = "compress_userver.gold" +tr = Test.AddTestRun() +tr.Processes.Default.Command = "echo" +f = tr.Disk.File("compress_userver.log") +f.Content = "compress_userver.gold" diff --git a/tests/gold_tests/pluginTest/compress/compress_observer.py b/tests/gold_tests/pluginTest/compress/compress_observer.py index b959f78be62..dbdecbf7efd 100755 --- a/tests/gold_tests/pluginTest/compress/compress_observer.py +++ b/tests/gold_tests/pluginTest/compress/compress_observer.py @@ -19,8 +19,10 @@ log = open('compress_userver.log', 'w') + def observe(headers): log.write("{}\n".format(headers['X-Ats-Compress-Test'])) log.flush() + Hooks.register(Hooks.ReadRequestHook, observe) diff --git a/tests/gold_tests/pluginTest/compress/compress_userver.gold b/tests/gold_tests/pluginTest/compress/compress_userver.gold new file mode 100644 index 00000000000..f3253deb9a6 --- /dev/null +++ b/tests/gold_tests/pluginTest/compress/compress_userver.gold @@ -0,0 +1,21 @@ +0/gzip, deflate, sdch, br +0/gzip +0/br +0/deflate +1/gzip, deflate, sdch, br +1/gzip +1/br +1/deflate +2/gzip, deflate, sdch, br +2/gzip +2/br +2/deflate +0/gzip;q=0.666 +0/gzip;q=0.666x +0/gzip;q=#0.666 +0/gzip; Q = 0.666 +0/gzip;q=0.0 +0/gzip;q=-0.1 +0/aaa, gzip;q=0.666, bbb +0/ br ; q=0.666, bbb +0/aaa, gzip;q=0.666 , diff --git a/tests/gold_tests/pluginTest/compress/greplog.sh b/tests/gold_tests/pluginTest/compress/greplog.sh index b16fa48e567..5148bebe69b 100755 --- a/tests/gold_tests/pluginTest/compress/greplog.sh +++ b/tests/gold_tests/pluginTest/compress/greplog.sh @@ -20,4 +20,4 @@ grep --text \ -e '^> Accept-Encoding:' \ -e '^< Content-' \ -e '^< Vary:' \ - -e '^$' + -e '^===$' diff --git a/tests/gold_tests/pluginTest/cookie_remap/bucketcookie.test.py b/tests/gold_tests/pluginTest/cookie_remap/bucketcookie.test.py index 25254ec49ab..ff56f86f8b6 100644 --- a/tests/gold_tests/pluginTest/cookie_remap/bucketcookie.test.py +++ b/tests/gold_tests/pluginTest/cookie_remap/bucketcookie.test.py @@ -31,7 +31,8 @@ # second server is run during second test server = Test.MakeOriginServer("server", ip='127.0.0.10') -request_header = {"headers": "GET /cookiematches HTTP/1.1\r\nHost: www.example.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +request_header = {"headers": "GET /cookiematches HTTP/1.1\r\nHost: www.example.com\r\n\r\n", + "timestamp": "1469733493.993", "body": ""} # expected response from the origin server response_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": ""} @@ -39,7 +40,8 @@ server.addResponse("sessionfile.log", request_header, response_header) server2 = Test.MakeOriginServer("server2", ip='127.0.0.11') -request_header2 = {"headers": "GET /cookiedoesntmatch HTTP/1.1\r\nHost: www.example.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +request_header2 = {"headers": "GET /cookiedoesntmatch HTTP/1.1\r\nHost: www.example.com\r\n\r\n", + "timestamp": "1469733493.993", "body": ""} # expected response from the origin server response_header2 = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": ""} @@ -59,7 +61,7 @@ config1 = config1.replace("$PORT", str(server.Variables.Port)) config1 = config1.replace("$ALTPORT", str(server2.Variables.Port)) -ts.Disk.File(ts.Variables.CONFIGDIR +"/bucketconfig.txt", exists=False, id="config1") +ts.Disk.File(ts.Variables.CONFIGDIR + "/bucketconfig.txt", exists=False, id="config1") ts.Disk.config1.WriteOn(config1) ts.Disk.remap_config.AddLine( diff --git a/tests/gold_tests/pluginTest/cookie_remap/collapseslashes.test.py b/tests/gold_tests/pluginTest/cookie_remap/collapseslashes.test.py index e1df4fea8fe..c25ee53f9e2 100644 --- a/tests/gold_tests/pluginTest/cookie_remap/collapseslashes.test.py +++ b/tests/gold_tests/pluginTest/cookie_remap/collapseslashes.test.py @@ -31,7 +31,8 @@ # and verify it collapsed the double // server = Test.MakeOriginServer("server", ip='127.0.0.10') -request_header = {"headers": "GET /i/like/cheetos?.done=http://finance.yahoo.com HTTP/1.1\r\nHost: www.example.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +request_header = {"headers": "GET /i/like/cheetos?.done=http://finance.yahoo.com HTTP/1.1\r\nHost: www.example.com\r\n\r\n", + "timestamp": "1469733493.993", "body": ""} # expected response from the origin server response_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": ""} @@ -50,7 +51,7 @@ config1 = config1.replace("$PORT", str(server.Variables.Port)) -ts.Disk.File(ts.Variables.CONFIGDIR +"/collapseconfig.txt", exists=False, id="config1") +ts.Disk.File(ts.Variables.CONFIGDIR + "/collapseconfig.txt", exists=False, id="config1") ts.Disk.config1.WriteOn(config1) ts.Disk.remap_config.AddLine( diff --git a/tests/gold_tests/pluginTest/cookie_remap/connector.test.py b/tests/gold_tests/pluginTest/cookie_remap/connector.test.py index 4d595a41529..8cd16b82d54 100644 --- a/tests/gold_tests/pluginTest/cookie_remap/connector.test.py +++ b/tests/gold_tests/pluginTest/cookie_remap/connector.test.py @@ -31,7 +31,8 @@ # second server is run during second test server = Test.MakeOriginServer("server", ip='127.0.0.10') -request_header = {"headers": "GET /cookiematches HTTP/1.1\r\nHost: www.example.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +request_header = {"headers": "GET /cookiematches HTTP/1.1\r\nHost: www.example.com\r\n\r\n", + "timestamp": "1469733493.993", "body": ""} # expected response from the origin server response_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": ""} @@ -39,7 +40,8 @@ server.addResponse("sessionfile.log", request_header, response_header) server2 = Test.MakeOriginServer("server2", ip='127.0.0.11') -request_header2 = {"headers": "GET /cookiedoesntmatch HTTP/1.1\r\nHost: www.example.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +request_header2 = {"headers": "GET /cookiedoesntmatch HTTP/1.1\r\nHost: www.example.com\r\n\r\n", + "timestamp": "1469733493.993", "body": ""} # expected response from the origin server response_header2 = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": ""} @@ -59,7 +61,7 @@ config1 = config1.replace("$PORT", str(server.Variables.Port)) config1 = config1.replace("$ALTPORT", str(server2.Variables.Port)) -ts.Disk.File(ts.Variables.CONFIGDIR +"/connectorconfig.txt", exists=False, id="config1") +ts.Disk.File(ts.Variables.CONFIGDIR + "/connectorconfig.txt", exists=False, id="config1") ts.Disk.config1.WriteOn(config1) ts.Disk.remap_config.AddLine( diff --git a/tests/gold_tests/pluginTest/cookie_remap/existscookie.test.py b/tests/gold_tests/pluginTest/cookie_remap/existscookie.test.py index 6c592e820dc..dd53fbf72da 100644 --- a/tests/gold_tests/pluginTest/cookie_remap/existscookie.test.py +++ b/tests/gold_tests/pluginTest/cookie_remap/existscookie.test.py @@ -31,7 +31,8 @@ # second server is run during second test server = Test.MakeOriginServer("server", ip='127.0.0.10') -request_header = {"headers": "GET /cookieexists HTTP/1.1\r\nHost: www.example.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +request_header = {"headers": "GET /cookieexists HTTP/1.1\r\nHost: www.example.com\r\n\r\n", + "timestamp": "1469733493.993", "body": ""} # expected response from the origin server response_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": ""} @@ -39,7 +40,8 @@ server.addResponse("sessionfile.log", request_header, response_header) server2 = Test.MakeOriginServer("server2", ip='127.0.0.11') -request_header2 = {"headers": "GET /cookiedoesntexist HTTP/1.1\r\nHost: www.example.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +request_header2 = {"headers": "GET /cookiedoesntexist HTTP/1.1\r\nHost: www.example.com\r\n\r\n", + "timestamp": "1469733493.993", "body": ""} # expected response from the origin server response_header2 = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": ""} @@ -59,7 +61,7 @@ config1 = config1.replace("$PORT", str(server.Variables.Port)) config1 = config1.replace("$ALTPORT", str(server2.Variables.Port)) -ts.Disk.File(ts.Variables.CONFIGDIR +"/existsconfig.txt", exists=False, id="config1") +ts.Disk.File(ts.Variables.CONFIGDIR + "/existsconfig.txt", exists=False, id="config1") ts.Disk.config1.WriteOn(config1) ts.Disk.remap_config.AddLine( diff --git a/tests/gold_tests/pluginTest/cookie_remap/matchcookie.test.py b/tests/gold_tests/pluginTest/cookie_remap/matchcookie.test.py index c238b259e3a..39e0a75871b 100644 --- a/tests/gold_tests/pluginTest/cookie_remap/matchcookie.test.py +++ b/tests/gold_tests/pluginTest/cookie_remap/matchcookie.test.py @@ -31,7 +31,8 @@ # second server is run during second test server = Test.MakeOriginServer("server", ip='127.0.0.10') -request_header = {"headers": "GET /cookiematches?a=1&b=2&c=3 HTTP/1.1\r\nHost: www.example.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +request_header = {"headers": "GET /cookiematches?a=1&b=2&c=3 HTTP/1.1\r\nHost: www.example.com\r\n\r\n", + "timestamp": "1469733493.993", "body": ""} # expected response from the origin server response_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": ""} @@ -39,7 +40,8 @@ server.addResponse("sessionfile.log", request_header, response_header) server2 = Test.MakeOriginServer("server2", ip='127.0.0.11') -request_header2 = {"headers": "GET /cookiedoesntmatch?a=1&b=2&c=3 HTTP/1.1\r\nHost: www.example.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +request_header2 = {"headers": "GET /cookiedoesntmatch?a=1&b=2&c=3 HTTP/1.1\r\nHost: www.example.com\r\n\r\n", + "timestamp": "1469733493.993", "body": ""} # expected response from the origin server response_header2 = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": ""} @@ -59,7 +61,7 @@ config1 = config1.replace("$PORT", str(server.Variables.Port)) config1 = config1.replace("$ALTPORT", str(server2.Variables.Port)) -ts.Disk.File(ts.Variables.CONFIGDIR +"/matchconfig.txt", exists=False, id="config1") +ts.Disk.File(ts.Variables.CONFIGDIR + "/matchconfig.txt", exists=False, id="config1") ts.Disk.config1.WriteOn(config1) ts.Disk.remap_config.AddLine( diff --git a/tests/gold_tests/pluginTest/cookie_remap/matchuri.test.py b/tests/gold_tests/pluginTest/cookie_remap/matchuri.test.py index 53c894c4966..43e20f442aa 100644 --- a/tests/gold_tests/pluginTest/cookie_remap/matchuri.test.py +++ b/tests/gold_tests/pluginTest/cookie_remap/matchuri.test.py @@ -31,7 +31,8 @@ # second server is run during second test server = Test.MakeOriginServer("server", ip='127.0.0.10') -request_header = {"headers": "GET /cookiematches HTTP/1.1\r\nHost: www.example.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +request_header = {"headers": "GET /cookiematches HTTP/1.1\r\nHost: www.example.com\r\n\r\n", + "timestamp": "1469733493.993", "body": ""} # expected response from the origin server response_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": ""} @@ -39,7 +40,8 @@ server.addResponse("sessionfile.log", request_header, response_header) server2 = Test.MakeOriginServer("server2", ip='127.0.0.11') -request_header2 = {"headers": "GET /cookiedoesntmatch HTTP/1.1\r\nHost: www.example.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +request_header2 = {"headers": "GET /cookiedoesntmatch HTTP/1.1\r\nHost: www.example.com\r\n\r\n", + "timestamp": "1469733493.993", "body": ""} # expected response from the origin server response_header2 = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": ""} @@ -59,7 +61,7 @@ config1 = config1.replace("$PORT", str(server.Variables.Port)) config1 = config1.replace("$ALTPORT", str(server2.Variables.Port)) -ts.Disk.File(ts.Variables.CONFIGDIR +"/matchuriconfig.txt", exists=False, id="config1") +ts.Disk.File(ts.Variables.CONFIGDIR + "/matchuriconfig.txt", exists=False, id="config1") ts.Disk.config1.WriteOn(config1) ts.Disk.remap_config.AddLine( diff --git a/tests/gold_tests/pluginTest/cookie_remap/matrixparams.test.py b/tests/gold_tests/pluginTest/cookie_remap/matrixparams.test.py index 1da71a683eb..404f63ff3a3 100644 --- a/tests/gold_tests/pluginTest/cookie_remap/matrixparams.test.py +++ b/tests/gold_tests/pluginTest/cookie_remap/matrixparams.test.py @@ -32,27 +32,34 @@ # That's why I am not adding any canned request/response server = Test.MakeOriginServer("server", ip='127.0.0.10') -request_header = {"headers": "GET /eighth/magic;matrix=1/eighth HTTP/1.1\r\nHost: www.example.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +request_header = {"headers": "GET /eighth/magic;matrix=1/eighth HTTP/1.1\r\nHost: www.example.com\r\n\r\n", + "timestamp": "1469733493.993", "body": ""} response_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": ""} server.addResponse("sessionfile.log", request_header, response_header) -request_header_2 = {"headers": "GET /eighth/magic;matrix=1/eighth?hello=10 HTTP/1.1\r\nHost: www.example.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +request_header_2 = {"headers": "GET /eighth/magic;matrix=1/eighth?hello=10 HTTP/1.1\r\nHost: www.example.com\r\n\r\n", + "timestamp": "1469733493.993", "body": ""} response_header_2 = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": ""} server.addResponse("sessionfile.log", request_header_2, response_header_2) -request_header_3 = {"headers": "GET /tenth/magic/tenth;matrix=2 HTTP/1.1\r\nHost: www.example.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +request_header_3 = {"headers": "GET /tenth/magic/tenth;matrix=2 HTTP/1.1\r\nHost: www.example.com\r\n\r\n", + "timestamp": "1469733493.993", "body": ""} response_header_3 = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": ""} server.addResponse("sessionfile.log", request_header_3, response_header_3) -request_header_4 = {"headers": "GET /tenth/magic/tenth;matrix=2?query=10 HTTP/1.1\r\nHost: www.example.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +request_header_4 = {"headers": "GET /tenth/magic/tenth;matrix=2?query=10 HTTP/1.1\r\nHost: www.example.com\r\n\r\n", + "timestamp": "1469733493.993", "body": ""} response_header_4 = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": ""} server.addResponse("sessionfile.log", request_header_4, response_header_4) -request_header_5 = {"headers": "GET /eleventh/magic;matrix=4/eleventh;matrix=2?query=true HTTP/1.1\r\nHost: www.example.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +request_header_5 = { + "headers": "GET /eleventh/magic;matrix=4/eleventh;matrix=2?query=true HTTP/1.1\r\nHost: www.example.com\r\n\r\n", + "timestamp": "1469733493.993", + "body": ""} response_header_5 = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": ""} server.addResponse("sessionfile.log", request_header_5, response_header_5) @@ -69,7 +76,7 @@ config1 = config1.replace("$PORT", str(server.Variables.Port)) -ts.Disk.File(ts.Variables.CONFIGDIR +"/matrixconfig.txt", exists=False, id="config1") +ts.Disk.File(ts.Variables.CONFIGDIR + "/matrixconfig.txt", exists=False, id="config1") ts.Disk.config1.WriteOn(config1) ts.Disk.remap_config.AddLine( diff --git a/tests/gold_tests/pluginTest/cookie_remap/notexistscookie.test.py b/tests/gold_tests/pluginTest/cookie_remap/notexistscookie.test.py index 45199372202..e96d940a2f6 100644 --- a/tests/gold_tests/pluginTest/cookie_remap/notexistscookie.test.py +++ b/tests/gold_tests/pluginTest/cookie_remap/notexistscookie.test.py @@ -31,7 +31,8 @@ # second server is run during second test server = Test.MakeOriginServer("server", ip='127.0.0.10') -request_header = {"headers": "GET /cookiedoesntexist HTTP/1.1\r\nHost: www.example.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +request_header = {"headers": "GET /cookiedoesntexist HTTP/1.1\r\nHost: www.example.com\r\n\r\n", + "timestamp": "1469733493.993", "body": ""} # expected response from the origin server response_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": ""} @@ -39,7 +40,8 @@ server.addResponse("sessionfile.log", request_header, response_header) server2 = Test.MakeOriginServer("server2", ip='127.0.0.11') -request_header2 = {"headers": "GET /cookieexists HTTP/1.1\r\nHost: www.example.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +request_header2 = {"headers": "GET /cookieexists HTTP/1.1\r\nHost: www.example.com\r\n\r\n", + "timestamp": "1469733493.993", "body": ""} # expected response from the origin server response_header2 = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": ""} @@ -59,7 +61,7 @@ config1 = config1.replace("$PORT", str(server.Variables.Port)) config1 = config1.replace("$ALTPORT", str(server2.Variables.Port)) -ts.Disk.File(ts.Variables.CONFIGDIR +"/notexistsconfig.txt", exists=False, id="config1") +ts.Disk.File(ts.Variables.CONFIGDIR + "/notexistsconfig.txt", exists=False, id="config1") ts.Disk.config1.WriteOn(config1) ts.Disk.remap_config.AddLine( diff --git a/tests/gold_tests/pluginTest/cookie_remap/regexcookie.test.py b/tests/gold_tests/pluginTest/cookie_remap/regexcookie.test.py index 27998dfd2b8..417ac319c21 100644 --- a/tests/gold_tests/pluginTest/cookie_remap/regexcookie.test.py +++ b/tests/gold_tests/pluginTest/cookie_remap/regexcookie.test.py @@ -31,7 +31,8 @@ # second server is run during second test server = Test.MakeOriginServer("server", ip='127.0.0.10') -request_header = {"headers": "GET /regexmatches?cookies=oreos-chipsahoy-icecream HTTP/1.1\r\nHost: www.example.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +request_header = {"headers": "GET /regexmatches?cookies=oreos-chipsahoy-icecream HTTP/1.1\r\nHost: www.example.com\r\n\r\n", + "timestamp": "1469733493.993", "body": ""} # expected response from the origin server response_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": ""} @@ -39,7 +40,8 @@ server.addResponse("sessionfile.log", request_header, response_header) server2 = Test.MakeOriginServer("server2", ip='127.0.0.11') -request_header2 = {"headers": "GET /regexdoesntmatch HTTP/1.1\r\nHost: www.example.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +request_header2 = {"headers": "GET /regexdoesntmatch HTTP/1.1\r\nHost: www.example.com\r\n\r\n", + "timestamp": "1469733493.993", "body": ""} # expected response from the origin server response_header2 = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": ""} @@ -59,7 +61,7 @@ config1 = config1.replace("$PORT", str(server.Variables.Port)) config1 = config1.replace("$ALTPORT", str(server2.Variables.Port)) -ts.Disk.File(ts.Variables.CONFIGDIR +"/regexconfig.txt", exists=False, id="config1") +ts.Disk.File(ts.Variables.CONFIGDIR + "/regexconfig.txt", exists=False, id="config1") ts.Disk.config1.WriteOn(config1) ts.Disk.remap_config.AddLine( diff --git a/tests/gold_tests/pluginTest/cookie_remap/setstatus.test.py b/tests/gold_tests/pluginTest/cookie_remap/setstatus.test.py index f7a298e6184..976779a30d4 100644 --- a/tests/gold_tests/pluginTest/cookie_remap/setstatus.test.py +++ b/tests/gold_tests/pluginTest/cookie_remap/setstatus.test.py @@ -37,7 +37,7 @@ 'proxy.config.diags.debug.tags': 'cookie_remap.*|http.*|dns.*', }) -ts.Disk.File(ts.Variables.CONFIGDIR +"/statusconfig.txt", exists=False, id="config1") +ts.Disk.File(ts.Variables.CONFIGDIR + "/statusconfig.txt", exists=False, id="config1") ts.Disk.config1.WriteOn(config1) ts.Disk.remap_config.AddLine( diff --git a/tests/gold_tests/pluginTest/cookie_remap/subcookie.test.py b/tests/gold_tests/pluginTest/cookie_remap/subcookie.test.py index 9a636d75cbd..677dc7cbcf0 100644 --- a/tests/gold_tests/pluginTest/cookie_remap/subcookie.test.py +++ b/tests/gold_tests/pluginTest/cookie_remap/subcookie.test.py @@ -31,7 +31,8 @@ # second server is run during second test server = Test.MakeOriginServer("server", ip='127.0.0.10') -request_header = {"headers": "GET /cookiematches HTTP/1.1\r\nHost: www.example.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +request_header = {"headers": "GET /cookiematches HTTP/1.1\r\nHost: www.example.com\r\n\r\n", + "timestamp": "1469733493.993", "body": ""} # expected response from the origin server response_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": ""} @@ -39,7 +40,8 @@ server.addResponse("sessionfile.log", request_header, response_header) server2 = Test.MakeOriginServer("server2", ip='127.0.0.11') -request_header2 = {"headers": "GET /cookiedoesntmatch HTTP/1.1\r\nHost: www.example.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +request_header2 = {"headers": "GET /cookiedoesntmatch HTTP/1.1\r\nHost: www.example.com\r\n\r\n", + "timestamp": "1469733493.993", "body": ""} # expected response from the origin server response_header2 = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": ""} @@ -59,7 +61,7 @@ config1 = config1.replace("$PORT", str(server.Variables.Port)) config1 = config1.replace("$ALTPORT", str(server2.Variables.Port)) -ts.Disk.File(ts.Variables.CONFIGDIR +"/subcookie.txt", exists=False, id="config1") +ts.Disk.File(ts.Variables.CONFIGDIR + "/subcookie.txt", exists=False, id="config1") ts.Disk.config1.WriteOn(config1) ts.Disk.remap_config.AddLine( @@ -70,15 +72,16 @@ tr = Test.AddTestRun("cookie value matches") # Unlike in other places I am using a single line string because the & seems to # be interpreted by the autest framework or the shell (tried escaping with \) -tr.Processes.Default.Command = 'curl --proxy 127.0.0.1:{0} "http://www.example.com" -H"Cookie: fpbeta=a=1&b=2&c=3" -H "Proxy-Connection: keep-alive" --verbose '.format(ts.Variables.port) -#tr.Processes.Default.Command = ''' -#curl -#--proxy 127.0.0.1:{0} -#"http://www.example.com" -#-H\"Cookie: fpbeta=a=1&b=2&c=3\" -#-H "Proxy-Connection: keep-alive" -#--verbose -#'''.format(ts.Variables.port) +tr.Processes.Default.Command = 'curl --proxy 127.0.0.1:{0} "http://www.example.com" -H"Cookie: fpbeta=a=1&b=2&c=3" -H "Proxy-Connection: keep-alive" --verbose '.format( + ts.Variables.port) +# tr.Processes.Default.Command = ''' +# curl +# --proxy 127.0.0.1:{0} +# "http://www.example.com" +# -H\"Cookie: fpbeta=a=1&b=2&c=3\" +# -H "Proxy-Connection: keep-alive" +# --verbose +# '''.format(ts.Variables.port) tr.Processes.Default.ReturnCode = 0 # time delay as proxy.config.http.wait_for_cache could be broken tr.Processes.Default.StartBefore(server, ready=When.PortOpen(server.Variables.Port)) @@ -89,7 +92,8 @@ # Negative test case that doesn't remap because not all subops pass tr = Test.AddTestRun("cookie value doesn't match") -tr.Processes.Default.Command = 'curl --proxy 127.0.0.1:{0} "http://www.example.com" -H"Cookie: fpbeta=a=1&b=2&c=4" -H "Proxy-Connection: keep-alive" --verbose '.format(ts.Variables.port) +tr.Processes.Default.Command = 'curl --proxy 127.0.0.1:{0} "http://www.example.com" -H"Cookie: fpbeta=a=1&b=2&c=4" -H "Proxy-Connection: keep-alive" --verbose '.format( + ts.Variables.port) tr.Processes.Default.ReturnCode = 0 tr.Processes.Default.StartBefore(server2, ready=When.PortOpen(server2.Variables.Port)) tr.StillRunningAfter = ts diff --git a/tests/gold_tests/pluginTest/cookie_remap/substitute.test.py b/tests/gold_tests/pluginTest/cookie_remap/substitute.test.py index 5bb358b40fd..7ecd8e4652a 100644 --- a/tests/gold_tests/pluginTest/cookie_remap/substitute.test.py +++ b/tests/gold_tests/pluginTest/cookie_remap/substitute.test.py @@ -29,17 +29,20 @@ server = Test.MakeOriginServer("server", ip='127.0.0.10') -request_header = {"headers": "GET /photos/search?query=magic HTTP/1.1\r\nHost: www.example.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +request_header = {"headers": "GET /photos/search?query=magic HTTP/1.1\r\nHost: www.example.com\r\n\r\n", + "timestamp": "1469733493.993", "body": ""} response_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": ""} server.addResponse("sessionfile.log", request_header, response_header) -request_header_2 = {"headers": "GET /photos/search?query=/theunmatchedpath HTTP/1.1\r\nHost: www.example.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +request_header_2 = {"headers": "GET /photos/search?query=/theunmatchedpath HTTP/1.1\r\nHost: www.example.com\r\n\r\n", + "timestamp": "1469733493.993", "body": ""} response_header_2 = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": ""} server.addResponse("sessionfile.log", request_header_2, response_header_2) -request_header_3 = {"headers": "GET /photos/search/magic/foobar HTTP/1.1\r\nHost: www.example.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +request_header_3 = {"headers": "GET /photos/search/magic/foobar HTTP/1.1\r\nHost: www.example.com\r\n\r\n", + "timestamp": "1469733493.993", "body": ""} response_header_3 = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": ""} server.addResponse("sessionfile.log", request_header_3, response_header_3) @@ -56,7 +59,7 @@ config1 = config1.replace("$PORT", str(server.Variables.Port)) -ts.Disk.File(ts.Variables.CONFIGDIR +"/substituteconfig.txt", exists=False, id="config1") +ts.Disk.File(ts.Variables.CONFIGDIR + "/substituteconfig.txt", exists=False, id="config1") ts.Disk.config1.WriteOn(config1) ts.Disk.remap_config.AddLine( diff --git a/tests/gold_tests/pluginTest/cppapi/cppapi.test.py b/tests/gold_tests/pluginTest/cppapi/cppapi.test.py index 809eb2a7c40..8410d2b1c53 100644 --- a/tests/gold_tests/pluginTest/cppapi/cppapi.test.py +++ b/tests/gold_tests/pluginTest/cppapi/cppapi.test.py @@ -14,13 +14,16 @@ # See the License for the specific language governing permissions and # limitations under the License. +import os + + Test.Summary = ''' Execute plugin with cppapi tests. ''' ts = Test.MakeATSProcess("ts") -Test.PreparePlugin(Test.Variables.AtsTestToolsDir + '/plugins/test_cppapi.cc', ts, extra_build_args='-l tscppapi') +Test.PrepareTestPlugin(os.path.join(Test.Variables.AtsTestPluginsDir, 'test_cppapi.so'), ts) tr = Test.AddTestRun() tr.Processes.Default.StartBefore(Test.Processes.ts) diff --git a/tests/gold_tests/pluginTest/header_rewrite/gold/header_rewrite-client.gold b/tests/gold_tests/pluginTest/header_rewrite/gold/header_rewrite-client.gold new file mode 100644 index 00000000000..418608bcb57 --- /dev/null +++ b/tests/gold_tests/pluginTest/header_rewrite/gold/header_rewrite-client.gold @@ -0,0 +1,13 @@ +`` +> GET http://www.example.com`` +> Host: www.example.com`` +> User-Agent: curl/`` +> Accept: */* +`` +< HTTP/1.1 304 Not Modified +< Date: `` +< Proxy-Connection: keep-alive +< Server: ATS/`` +< Cache-Control: no-store +< +`` diff --git a/tests/gold_tests/pluginTest/header_rewrite/gold/header_rewrite-l_value.gold b/tests/gold_tests/pluginTest/header_rewrite/gold/header_rewrite-l_value.gold new file mode 100644 index 00000000000..8c3cf6649a8 --- /dev/null +++ b/tests/gold_tests/pluginTest/header_rewrite/gold/header_rewrite-l_value.gold @@ -0,0 +1,16 @@ +`` +> GET http://www.example.com`` +> Host: www.example.com`` +> User-Agent: curl/`` +> Accept: */* +`` +< HTTP/1.1 200 OK +< Date: `` +< Age: `` +< Transfer-Encoding: chunked +< Proxy-Connection: keep-alive +< Server: ATS/`` +< X-First: First +< X-Last: Last +< +`` diff --git a/tests/gold_tests/pluginTest/header_rewrite/gold/set-redirect.gold b/tests/gold_tests/pluginTest/header_rewrite/gold/set-redirect.gold new file mode 100644 index 00000000000..bd76250fcc0 --- /dev/null +++ b/tests/gold_tests/pluginTest/header_rewrite/gold/set-redirect.gold @@ -0,0 +1,8 @@ +`` +> HEAD / HTTP/1.1 +> Host: no_path.com +`` +< HTTP/1.1 301 Redirect +`` +< Location: http://no_path.com?name=brian/ +`` diff --git a/tests/gold_tests/pluginTest/header_rewrite/header_rewrite.test.py b/tests/gold_tests/pluginTest/header_rewrite/header_rewrite.test.py index 2341cf07764..6e850eb8573 100644 --- a/tests/gold_tests/pluginTest/header_rewrite/header_rewrite.test.py +++ b/tests/gold_tests/pluginTest/header_rewrite/header_rewrite.test.py @@ -16,7 +16,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import os Test.Summary = ''' Test a basic remap of a http connection ''' diff --git a/tests/gold_tests/pluginTest/header_rewrite/header_rewrite_l_value.test.py b/tests/gold_tests/pluginTest/header_rewrite/header_rewrite_l_value.test.py new file mode 100644 index 00000000000..28247b02276 --- /dev/null +++ b/tests/gold_tests/pluginTest/header_rewrite/header_rewrite_l_value.test.py @@ -0,0 +1,64 @@ +''' +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +Test.Summary = ''' +Test for a regression of the issue fixed in +https://github.com/apache/trafficserver/pull/5423 +Insertion of header rewrite directives in to the output +''' + +Test.ContinueOnFail = True +# Define default ATS +ts = Test.MakeATSProcess("ts") +server = Test.MakeOriginServer("server") + +Test.testName = "" +request_header = {"headers": "GET / HTTP/1.1\r\nHost: www.example.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +# expected response from the origin server +response_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": ""} + +# add response to the server dictionary +server.addResponse("sessionfile.log", request_header, response_header) +ts.Disk.records_config.update({ + 'proxy.config.diags.debug.enabled': 1, + 'proxy.config.diags.debug.tags': 'header.*', +}) + +# The following rule adds X-First and X-Last headers +ts.Setup.CopyAs('rules/rule_l_value.conf', Test.RunDirectory) + +ts.Disk.plugin_config.AddLine( + 'header_rewrite.so {0}/rule_l_value.conf'.format(Test.RunDirectory) +) +ts.Disk.remap_config.AddLine( + 'map http://www.example.com http://127.0.0.1:{0}'.format(server.Variables.Port) +) +ts.Disk.remap_config.AddLine( + 'map http://www.example.com:8080 http://127.0.0.1:{0}'.format(server.Variables.Port) +) + +# [L] test +tr = Test.AddTestRun("Header Rewrite End [L]") +tr.Processes.Default.Command = 'curl --proxy 127.0.0.1:{0} "http://www.example.com" -H "Proxy-Connection: keep-alive" --verbose'.format( + ts.Variables.port) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.StartBefore(server, ready=When.PortOpen(server.Variables.Port)) +tr.Processes.Default.StartBefore(Test.Processes.ts) +tr.Processes.Default.Streams.stderr = "gold/header_rewrite-l_value.gold" +tr.StillRunningAfter = server +ts.Streams.All = "gold/header_rewrite-tag.gold" diff --git a/tests/gold_tests/pluginTest/header_rewrite/header_rewrite_url.test.py b/tests/gold_tests/pluginTest/header_rewrite/header_rewrite_url.test.py new file mode 100644 index 00000000000..8f8427120e8 --- /dev/null +++ b/tests/gold_tests/pluginTest/header_rewrite/header_rewrite_url.test.py @@ -0,0 +1,80 @@ +''' +Test header_rewrite with URL conditions and operators. +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +Test.Summary = ''' +Test header_rewrite with URL conditions and operators. +''' + +Test.ContinueOnFail = True +ts = Test.MakeATSProcess("ts") +server = Test.MakeOriginServer("server") + +# Configure the server to return 200 responses. The rewrite rules below set a +# non-200 status, so if curl gets a 200 response something went wrong. +Test.testName = "" +request_header = {"headers": "GET / HTTP/1.1\r\nHost: www.example.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +response_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +server.addResponse("sessionfile.log", request_header, response_header) +request_header = {"headers": "GET / HTTP/1.1\r\nHost: no_path.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +server.addResponse("sessionfile.log", request_header, response_header) + +ts.Disk.records_config.update({ + 'proxy.config.diags.debug.enabled': 1, + 'proxy.config.diags.debug.tags': 'header.*', +}) +# The following rule changes the status code returned from origin server to 303 +ts.Setup.CopyAs('rules/rule_client.conf', Test.RunDirectory) +ts.Setup.CopyAs('rules/set_redirect.conf', Test.RunDirectory) + +# This configuration makes use of CLIENT-URL in conditions. +ts.Disk.remap_config.AddLine( + 'map http://www.example.com/from_path/ https://127.0.0.1:{0}/to_path/ ' + '@plugin=header_rewrite.so @pparam={1}/rule_client.conf'.format( + server.Variables.Port, Test.RunDirectory)) +ts.Disk.remap_config.AddLine( + 'map http://www.example.com:8080/from_path/ https://127.0.0.1:{0}/to_path/ ' + '@plugin=header_rewrite.so @pparam={1}/rule_client.conf'.format( + server.Variables.Port, Test.RunDirectory)) +# This configuration makes use of TO-URL in a set-redirect operator. +ts.Disk.remap_config.AddLine( + 'map http://no_path.com http://no_path.com?name=brian/ ' + '@plugin=header_rewrite.so @pparam={0}/set_redirect.conf'.format( + Test.RunDirectory)) + +# Test CLIENT-URL. +tr = Test.AddTestRun() +tr.Processes.Default.Command = ( + 'curl --proxy 127.0.0.1:{0} "http://www.example.com/from_path/hello?=foo=bar" ' + '-H "Proxy-Connection: keep-alive" --verbose'.format( + ts.Variables.port)) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.StartBefore(server, ready=When.PortOpen(server.Variables.Port)) +tr.Processes.Default.StartBefore(Test.Processes.ts) +tr.Processes.Default.Streams.stderr = "gold/header_rewrite-client.gold" +tr.StillRunningAfter = server +ts.Streams.All = "gold/header_rewrite-tag.gold" + +# Test TO-URL in a set-redirect operator. +tr = Test.AddTestRun() +tr.Processes.Default.Command = 'curl --head 127.0.0.1:{0} -H "Host: no_path.com" --verbose'.format( + ts.Variables.port) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Streams.stderr = "gold/set-redirect.gold" +tr.StillRunningAfter = server +ts.Streams.All = "gold/header_rewrite-tag.gold" diff --git a/tests/gold_tests/pluginTest/header_rewrite/rules/rule_client.conf b/tests/gold_tests/pluginTest/header_rewrite/rules/rule_client.conf new file mode 100644 index 00000000000..a481775573e --- /dev/null +++ b/tests/gold_tests/pluginTest/header_rewrite/rules/rule_client.conf @@ -0,0 +1,22 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +cond %{CLIENT-URL:PATH} /^from_path/ +cond %{CLIENT-URL:SCHEME} =http +cond %{CLIENT-URL:HOST} =www.example.com +cond %{CLIENT-URL:QUERY} /foo=bar/ +set-status 304 \ No newline at end of file diff --git a/tests/gold_tests/pluginTest/header_rewrite/rules/rule_l_value.conf b/tests/gold_tests/pluginTest/header_rewrite/rules/rule_l_value.conf new file mode 100644 index 00000000000..e80a8e4f7fa --- /dev/null +++ b/tests/gold_tests/pluginTest/header_rewrite/rules/rule_l_value.conf @@ -0,0 +1,21 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +cond %{SEND_RESPONSE_HDR_HOOK} + set-header X-First "First" + set-header X-Last "Last" [L] + diff --git a/tests/gold_tests/pluginTest/header_rewrite/rules/set_redirect.conf b/tests/gold_tests/pluginTest/header_rewrite/rules/set_redirect.conf new file mode 100644 index 00000000000..6a3230a6ada --- /dev/null +++ b/tests/gold_tests/pluginTest/header_rewrite/rules/set_redirect.conf @@ -0,0 +1,18 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set-redirect 301 %{TO-URL:URL} diff --git a/tests/gold_tests/pluginTest/lua/global.lua b/tests/gold_tests/pluginTest/lua/global.lua new file mode 100644 index 00000000000..0adea626a86 --- /dev/null +++ b/tests/gold_tests/pluginTest/lua/global.lua @@ -0,0 +1,23 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. + +function do_global_read_request() + if 'GET' == ts.client_request.get_method() then + if '/noop' == ts.client_request.get_uri() then + ts.http.set_resp(200, "noop") + end + end +end diff --git a/tests/gold_tests/pluginTest/lua/gold/lifecycle.stderr.gold b/tests/gold_tests/pluginTest/lua/gold/lifecycle.stderr.gold new file mode 100644 index 00000000000..1e4a4f87dfb --- /dev/null +++ b/tests/gold_tests/pluginTest/lua/gold/lifecycle.stderr.gold @@ -0,0 +1,6 @@ +`` +`` ts_lua (remap) id: 0 gc_kb: `` gc_kb_max: `` threads: `` threads_max: `` +`` ts_lua (remap) id: 1 gc_kb: `` gc_kb_max: `` threads: `` threads_max: `` +`` ts_lua (remap) id: 2 gc_kb: `` gc_kb_max: `` threads: `` threads_max: `` +`` ts_lua (remap) id: 3 gc_kb: `` gc_kb_max: `` threads: `` threads_max: `` +`` diff --git a/tests/gold_tests/pluginTest/lua/gold/metrics.stdout.gold b/tests/gold_tests/pluginTest/lua/gold/metrics.stdout.gold new file mode 100644 index 00000000000..4c1362bea13 --- /dev/null +++ b/tests/gold_tests/pluginTest/lua/gold/metrics.stdout.gold @@ -0,0 +1,6 @@ +plugin.lua.global.states `` +plugin.lua.global.gc_bytes `` +plugin.lua.global.threads `` +plugin.lua.remap.states `` +plugin.lua.remap.gc_bytes `` +plugin.lua.remap.threads `` diff --git a/tests/gold_tests/pluginTest/lua/hello.lua b/tests/gold_tests/pluginTest/lua/hello.lua new file mode 100644 index 00000000000..d5373f9b84a --- /dev/null +++ b/tests/gold_tests/pluginTest/lua/hello.lua @@ -0,0 +1,35 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. + +function do_remap() + if 'GET' == ts.client_request.get_method() then + if '/hello' == ts.client_request.get_uri() then + ts.http.set_resp(200, "Hello, World") + end + end +end + +function origin_intercept_handler() + local body = 'Hello, World' + local resp = 'HTTP/1.0 200 OK\r\n' .. + 'Server: Lua Black Magic\r\n' .. + 'Content-Type: text/plain\r\n' .. + 'Content-Length: ' .. string.len(body) .. '\r\n' .. + '\r\n' + ts.say(resp) + ts.say(body) + ts.flush() +end diff --git a/tests/gold_tests/pluginTest/lua/lua_states_stats.test.py b/tests/gold_tests/pluginTest/lua/lua_states_stats.test.py new file mode 100644 index 00000000000..b6a390a6640 --- /dev/null +++ b/tests/gold_tests/pluginTest/lua/lua_states_stats.test.py @@ -0,0 +1,103 @@ +''' +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +Test.Summary = ''' +Test lua states and stats functionality +''' + +Test.SkipUnless( + Condition.PluginExists('tslua.so'), +) +Test.SkipIf(Condition.true("Test cannot deterministically wait until the stats appear")) + +Test.ContinueOnFail = True +# Define default ATS +server = Test.MakeOriginServer("server") + +ts = Test.MakeATSProcess("ts", command="traffic_manager", select_ports=True) + +Test.testName = "Lua states and stats" + +# test to ensure origin server works +request_header = {"headers": "GET / HTTP/1.1\r\nHost: www.example.com\r\n\r\n", + "timestamp": "1469733493.993", "body": ""} +response_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", + "timestamp": "1469733493.993", "body": ""} + +# add response to the server dictionary +server.addResponse("sessionfile.log", request_header, response_header) + +ts.Disk.remap_config.AddLines({ + 'map / http://127.0.0.1:{}/'.format(server.Variables.Port), + 'map http://hello http://127.0.0.1:{}/'.format(server.Variables.Port) + + ' @plugin=tslua.so @pparam={}/hello.lua'.format(Test.TestDirectory) +}) + +ts.Disk.plugin_config.AddLine('tslua.so {}/global.lua'.format(Test.TestDirectory)) + +ts.Disk.records_config.update({ + 'proxy.config.diags.debug.enabled': 1, + 'proxy.config.diags.debug.tags': 'ts_lua', + 'proxy.config.plugin.lua.max_states': 4, +}) + +curl_and_args = 'curl -s -D /dev/stdout -o /dev/stderr -x localhost:{} '.format(ts.Variables.port) + +# 0 Test - Check for configured lua states +tr = Test.AddTestRun("Lua states") +ps = tr.Processes.Default # alias +ps.StartBefore(server, ready=When.PortOpen(server.Variables.Port)) +ps.StartBefore(Test.Processes.ts, ready=When.PortOpen(ts.Variables.port)) +ps.Command = "traffic_ctl config match lua" +ps.Env = ts.Env +ps.ReturnCode = 0 +ps.Streams.stdout.Content = Testers.ContainsExpression("proxy.config.plugin.lua.max_states: 4", "expected 4 states") +tr.TimeOut = 5 +tr.StillRunningAfter = ts + +# 1 Test - Exercise lua script +tr = Test.AddTestRun("Lua hello") +ps = tr.Processes.Default # alias +ps.Command = curl_and_args + ' http://hello/hello' +ps.TimeOut = 5 +ps.ReturnCode = 0 +ps.Streams.stderr.Content = Testers.ContainsExpression("Hello, World", "hello world content") +tr.TimeOut = 5 +tr.StillRunningAfter = ts + +# 2 Test - Check for metrics +tr = Test.AddTestRun("Check for metrics") +tr.DelayStart = 15 # 5s lag on metrics to update +tr.TimeOut = 5 +ps = tr.Processes.Default # alias +ps.Env = ts.Env +ps.Command = "traffic_ctl metric match lua" +ps.Env = ts.Env +ps.ReturnCode = 0 +ps.Streams.stdout = "gold/metrics.stdout.gold" +tr.StillRunningAfter = ts + +# 3 Test - Check for developer lifecycle stats +tr = Test.AddTestRun("Check for lifecycle stats") +ps = tr.Processes.Default # alias +ps.Command = "traffic_ctl plugin msg ts_lua print_stats" +ps.Env = ts.Env +ps.ReturnCode = 0 +ts.Streams.stderr = "gold/lifecycle.stderr.gold" +tr.StillRunningAfter = ts diff --git a/tests/gold_tests/pluginTest/multiplexer/multiplexer.test.py b/tests/gold_tests/pluginTest/multiplexer/multiplexer.test.py index 028ef769023..18a0ac73ce2 100644 --- a/tests/gold_tests/pluginTest/multiplexer/multiplexer.test.py +++ b/tests/gold_tests/pluginTest/multiplexer/multiplexer.test.py @@ -16,11 +16,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -import os Test.Summary = ''' Test experimental/multiplexer. ''' -# need Curl + Test.SkipUnless( Condition.PluginExists('multiplexer.so') ) @@ -44,7 +43,8 @@ # For now, just make sure the plugin loads without error. tr = Test.AddTestRun() -tr.Processes.Default.Command = 'curl --silent --proxy 127.0.0.1:{0} "http://www.example.com" -H "Proxy-Connection: close"'.format(ts.Variables.port) +tr.Processes.Default.Command = 'curl --silent --proxy 127.0.0.1:{0} "http://www.example.com" -H "Proxy-Connection: close"'.format( + ts.Variables.port) tr.Processes.Default.ReturnCode = 0 tr.Processes.Default.StartBefore(server, ready=When.PortOpen(server.Variables.Port)) tr.Processes.Default.StartBefore(Test.Processes.ts) diff --git a/tests/gold_tests/pluginTest/regex_remap/gold/regex_remap_crash.gold b/tests/gold_tests/pluginTest/regex_remap/gold/regex_remap_crash.gold index f57b30cc5e0..afa1ece8147 100644 --- a/tests/gold_tests/pluginTest/regex_remap/gold/regex_remap_crash.gold +++ b/tests/gold_tests/pluginTest/regex_remap/gold/regex_remap_crash.gold @@ -1,4 +1,4 @@ -HTTP/1.1 200 OK -`` -uuid: 180 -`` +HTTP/1.1 200 OK +`` +uuid: 180 +`` diff --git a/tests/gold_tests/pluginTest/regex_remap/gold/regex_remap_smoke.gold b/tests/gold_tests/pluginTest/regex_remap/gold/regex_remap_smoke.gold index 9d235f54c12..9994120a842 100644 --- a/tests/gold_tests/pluginTest/regex_remap/gold/regex_remap_smoke.gold +++ b/tests/gold_tests/pluginTest/regex_remap/gold/regex_remap_smoke.gold @@ -1,4 +1,4 @@ -HTTP/1.1 200 OK -`` -uuid: smoke -`` +HTTP/1.1 200 OK +`` +uuid: smoke +`` diff --git a/tests/gold_tests/pluginTest/regex_remap/regex_remap.test.py b/tests/gold_tests/pluginTest/regex_remap/regex_remap.test.py index 3d001f0a4d5..4828818e26b 100644 --- a/tests/gold_tests/pluginTest/regex_remap/regex_remap.test.py +++ b/tests/gold_tests/pluginTest/regex_remap/regex_remap.test.py @@ -21,7 +21,7 @@ Test regex_remap ''' -## Test description: +# Test description: # Load up cache, ensure fresh # Create regex reval rule, config reload: # ensure item is staled only once. @@ -76,7 +76,7 @@ tr = Test.AddTestRun("smoke test") tr.Processes.Default.StartBefore(server) tr.Processes.Default.StartBefore(Test.Processes.ts) -creq=replay_txns[0]['client-request'] +creq = replay_txns[0]['client-request'] tr.Processes.Default.Command = curl_and_args + '--header "uuid: {}" '.format(creq["headers"]["fields"][1][1]) + creq["url"] tr.Processes.Default.ReturnCode = 0 tr.Processes.Default.Streams.stdout = "gold/regex_remap_smoke.gold" @@ -84,9 +84,11 @@ # Crash test. tr = Test.AddTestRun("crash test") -creq=replay_txns[1]['client-request'] -tr.Processes.Default.Command = curl_and_args + '--header "uuid: {}" '.format(creq["headers"]["fields"][1][1]) + '"{}"'.format(creq["url"]) +creq = replay_txns[1]['client-request'] +tr.Processes.Default.Command = curl_and_args + \ + '--header "uuid: {}" '.format(creq["headers"]["fields"][1][1]) + '"{}"'.format(creq["url"]) tr.Processes.Default.ReturnCode = 0 tr.Processes.Default.Streams.stdout = "gold/regex_remap_crash.gold" -ts.Disk.diags_log.Content = Testers.ContainsExpression('ERROR: .regex_remap. Bad regular expression result -21', "Resource limit exceeded") +ts.Disk.diags_log.Content = Testers.ContainsExpression( + 'ERROR: .regex_remap. Bad regular expression result -21', "Resource limit exceeded") tr.StillRunningAfter = ts diff --git a/tests/gold_tests/pluginTest/regex_remap/replay/yts-2819.replay.json b/tests/gold_tests/pluginTest/regex_remap/replay/yts-2819.replay.json index b9dfccfd8cb..5083a134e54 100644 --- a/tests/gold_tests/pluginTest/regex_remap/replay/yts-2819.replay.json +++ b/tests/gold_tests/pluginTest/regex_remap/replay/yts-2819.replay.json @@ -62,6 +62,10 @@ [ "Content-Length", "6128" + ], + [ + "Connection", + "close" ] ] } @@ -132,6 +136,10 @@ [ "Content-Length", "6128" + ], + [ + "Connection", + "close" ] ] } diff --git a/tests/gold_tests/pluginTest/regex_revalidate/gold/regex_reval-hit.gold b/tests/gold_tests/pluginTest/regex_revalidate/gold/regex_reval-hit.gold index f98f8ac02c3..68e9596f565 100644 --- a/tests/gold_tests/pluginTest/regex_revalidate/gold/regex_reval-hit.gold +++ b/tests/gold_tests/pluginTest/regex_revalidate/gold/regex_reval-hit.gold @@ -1,10 +1,10 @@ -HTTP/1.1 200 OK -Etag: `` -Cache-Control: max-age=``,public -Content-Length: 3 -Date: `` -Connection: `` -Server: `` -X-Cache: hit-fresh -`` -`` +HTTP/1.1 200 OK +Etag: `` +Cache-Control: max-age=``,public +Content-Length: 3 +Date: `` +Connection: `` +Server: `` +X-Cache: hit-fresh +`` +`` diff --git a/tests/gold_tests/pluginTest/regex_revalidate/gold/regex_reval-miss.gold b/tests/gold_tests/pluginTest/regex_revalidate/gold/regex_reval-miss.gold index 35ec6036eff..43a2e4a6e3f 100644 --- a/tests/gold_tests/pluginTest/regex_revalidate/gold/regex_reval-miss.gold +++ b/tests/gold_tests/pluginTest/regex_revalidate/gold/regex_reval-miss.gold @@ -1,10 +1,10 @@ -HTTP/1.1 200 OK -Etag: `` -Cache-Control: max-age=``,public -Content-Length: 3 -Date: `` -Connection: `` -Server: `` -X-Cache: miss -`` -`` +HTTP/1.1 200 OK +Etag: `` +Cache-Control: max-age=``,public +Content-Length: 3 +Date: `` +Connection: `` +Server: `` +X-Cache: miss +`` +`` diff --git a/tests/gold_tests/pluginTest/regex_revalidate/gold/regex_reval-stale.gold b/tests/gold_tests/pluginTest/regex_revalidate/gold/regex_reval-stale.gold index ea29316b964..40a8044a989 100644 --- a/tests/gold_tests/pluginTest/regex_revalidate/gold/regex_reval-stale.gold +++ b/tests/gold_tests/pluginTest/regex_revalidate/gold/regex_reval-stale.gold @@ -1,10 +1,10 @@ -HTTP/1.1 200 OK -Etag: `` -Cache-Control: max-age=``,public -Content-Length: 3 -Date: `` -Connection: `` -Server: `` -X-Cache: hit-stale -`` -`` +HTTP/1.1 200 OK +Etag: `` +Cache-Control: max-age=``,public +Content-Length: 3 +Date: `` +Connection: `` +Server: `` +X-Cache: hit-stale +`` +`` diff --git a/tests/gold_tests/pluginTest/regex_revalidate/regex_revalidate.test.py b/tests/gold_tests/pluginTest/regex_revalidate/regex_revalidate.test.py index f4b79590e02..dd9babb1d1e 100644 --- a/tests/gold_tests/pluginTest/regex_revalidate/regex_revalidate.test.py +++ b/tests/gold_tests/pluginTest/regex_revalidate/regex_revalidate.test.py @@ -22,7 +22,7 @@ Test a basic regex_revalidate ''' -## Test description: +# Test description: # Load up cache, ensure fresh # Create regex reval rule, config reload: # ensure item is staled only once. @@ -44,80 +44,80 @@ # Define ATS and configure ts = Test.MakeATSProcess("ts", command="traffic_manager", select_ports=True) -#**testname is required** +# **testname is required** #testName = "regex_reval" # default root request_header_0 = {"headers": - "GET / HTTP/1.1\r\n" + - "Host: www.example.com\r\n" + - "\r\n", - "timestamp": "1469733493.993", - "body": "", -} + "GET / HTTP/1.1\r\n" + + "Host: www.example.com\r\n" + + "\r\n", + "timestamp": "1469733493.993", + "body": "", + } response_header_0 = {"headers": - "HTTP/1.1 200 OK\r\n" + - "Connection: close\r\n" + - "Cache-Control: max-age=300\r\n" + - "\r\n", - "timestamp": "1469733493.993", - "body": "xxx", -} + "HTTP/1.1 200 OK\r\n" + + "Connection: close\r\n" + + "Cache-Control: max-age=300\r\n" + + "\r\n", + "timestamp": "1469733493.993", + "body": "xxx", + } # cache item path1 request_header_1 = {"headers": - "GET /path1 HTTP/1.1\r\n" + - "Host: www.example.com\r\n" + - "\r\n", - "timestamp": "1469733493.993", - "body": "" -} + "GET /path1 HTTP/1.1\r\n" + + "Host: www.example.com\r\n" + + "\r\n", + "timestamp": "1469733493.993", + "body": "" + } response_header_1 = {"headers": - "HTTP/1.1 200 OK\r\n" + - "Connection: close\r\n" + - 'Etag: "path1"\r\n' + - "Cache-Control: max-age=600,public\r\n" + - "\r\n", - "timestamp": "1469733493.993", - "body": "abc" -} + "HTTP/1.1 200 OK\r\n" + + "Connection: close\r\n" + + 'Etag: "path1"\r\n' + + "Cache-Control: max-age=600,public\r\n" + + "\r\n", + "timestamp": "1469733493.993", + "body": "abc" + } # cache item path1a request_header_2 = {"headers": - "GET /path1a HTTP/1.1\r\n" + - "Host: www.example.com\r\n" + - "\r\n", - "timestamp": "1469733493.993", - "body": "" -} + "GET /path1a HTTP/1.1\r\n" + + "Host: www.example.com\r\n" + + "\r\n", + "timestamp": "1469733493.993", + "body": "" + } response_header_2 = {"headers": - "HTTP/1.1 200 OK\r\n" + - "Connection: close\r\n" + - 'Etag: "path1a"\r\n' + - "Cache-Control: max-age=600,public\r\n" + - "\r\n", - "timestamp": "1469733493.993", - "body": "cde" -} + "HTTP/1.1 200 OK\r\n" + + "Connection: close\r\n" + + 'Etag: "path1a"\r\n' + + "Cache-Control: max-age=600,public\r\n" + + "\r\n", + "timestamp": "1469733493.993", + "body": "cde" + } # cache item path2a request_header_3 = {"headers": - "GET /path2a HTTP/1.1\r\n" + - "Host: www.example.com\r\n" + - "\r\n", - "timestamp": "1469733493.993", - "body": "" -} + "GET /path2a HTTP/1.1\r\n" + + "Host: www.example.com\r\n" + + "\r\n", + "timestamp": "1469733493.993", + "body": "" + } response_header_3 = {"headers": - "HTTP/1.1 200 OK\r\n" + - "Connection: close\r\n" + - 'Etag: "path2a"\r\n' + - "Cache-Control: max-age=900,public\r\n" + - "\r\n", - "timestamp": "1469733493.993", - "body": "efg" -} + "HTTP/1.1 200 OK\r\n" + + "Connection: close\r\n" + + 'Etag: "path2a"\r\n' + + "Cache-Control: max-age=900,public\r\n" + + "\r\n", + "timestamp": "1469733493.993", + "body": "efg" + } server.addResponse("sessionlog.json", request_header_0, response_header_0) server.addResponse("sessionlog.json", request_header_1, response_header_1) @@ -148,7 +148,7 @@ ts.Disk.records_config.update({ 'proxy.config.diags.debug.enabled': 1, 'proxy.config.diags.debug.tags': 'regex_revalidate', -# 'proxy.config.diags.debug.enabled': 0, + # 'proxy.config.diags.debug.enabled': 0, 'proxy.config.http.cache.http': 1, 'proxy.config.http.wait_for_cache': 1, 'proxy.config.http.insert_age_in_response': 0, diff --git a/tests/gold_tests/pluginTest/server_push_preload/gold/server_push_preload_0_stdout.gold b/tests/gold_tests/pluginTest/server_push_preload/gold/server_push_preload_0_stdout.gold new file mode 100644 index 00000000000..ea3a9617b45 --- /dev/null +++ b/tests/gold_tests/pluginTest/server_push_preload/gold/server_push_preload_0_stdout.gold @@ -0,0 +1,21 @@ +`` +[``] send HEADERS frame +`` +[``] recv PUSH_PROMISE frame +`` +`` (padlen=``, promised_stream_id=2) +`` +[``] recv HEADERS frame +`` +[``] recv DATA frame +`` +[``] recv HEADERS frame +`` +[``] recv DATA frame +`` +[``] send GOAWAY frame + (last_stream_id=2, error_code=NO_ERROR(0x00), opaque_data(0)=[]) +`` +id responseEnd requestStart process code size request path + 1 `` 200 `` /index.html + 2 `` 200 `` /app/script.js diff --git a/tests/gold_tests/pluginTest/server_push_preload/server_push_preload.test.py b/tests/gold_tests/pluginTest/server_push_preload/server_push_preload.test.py new file mode 100644 index 00000000000..09e9112acc4 --- /dev/null +++ b/tests/gold_tests/pluginTest/server_push_preload/server_push_preload.test.py @@ -0,0 +1,91 @@ +''' +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +Test.Summary = ''' +Test for server_push_preload plugin +''' + +Test.SkipUnless( + Condition.PluginExists('server_push_preload.so'), + Condition.HasProgram("nghttp", + "Nghttp need to be installed on system for this test to work"), +) +Test.testName = "server_push_preload" +Test.ContinueOnFail = True + +# ---- +# Setup Origin Server +# ---- +microserver = Test.MakeOriginServer("microserver") + +# index.html +microserver.addResponse("sessionfile.log", + {"headers": "GET /index.html HTTP/1.1\r\nHost: www.example.com\r\n\r\n", "body": ""}, + {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\nLink: ; rel=preload; as=style; nopush\r\nLink: ; rel=preload; as=script\r\n\r\n", + "body": "\r\n\r\n\r\n\r\n\r\n\r\nServer Push Preload Test\r\n\r\n\r\n"}) + +# /app/style.css +microserver.addResponse("sessionfile.log", + {"headers": "GET /app/style.css HTTP/1.1\r\nHost: www.example.com\r\n\r\n", "body": ""}, + {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", + "body": "body { font-weight: bold; }\r\n"}) + +# /app/script.js +microserver.addResponse("sessionfile.log", + {"headers": "GET /app/script.js HTTP/1.1\r\nHost: www.example.com\r\n\r\n", "body": ""}, + {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", + "body": "function do_nothing() { return; }\r\n"}) + +# ---- +# Setup ATS +# ---- +ts = Test.MakeATSProcess("ts", select_ports=True, enable_tls=True) + +ts.addSSLfile("ssl/server.pem") +ts.addSSLfile("ssl/server.key") +ts.Disk.ssl_multicert_config.AddLine( + 'dest_ip=* ssl_cert_name=server.pem ssl_key_name=server.key' +) + +ts.Disk.remap_config.AddLine( + 'map / http://127.0.0.1:{0}/ @plugin=server_push_preload.so'.format( + microserver.Variables.Port) +) + +ts.Disk.records_config.update({ + 'proxy.config.diags.debug.enabled': 1, + 'proxy.config.diags.debug.tags': 'http2|server_push_preload', + 'proxy.config.ssl.server.cert.path': '{0}'.format(ts.Variables.SSLDir), + 'proxy.config.ssl.server.private_key.path': '{0}'.format(ts.Variables.SSLDir), + 'proxy.config.http2.active_timeout_in': 3, +}) + +# ---- +# Test Cases +# ---- + +# Test Case 0: Server Push by Link header +tr = Test.AddTestRun() +tr.Processes.Default.Command = "nghttp -vs --no-dep 'https://127.0.0.1:{0}/index.html'".format( + ts.Variables.ssl_port) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.StartBefore( + microserver, ready=When.PortOpen(microserver.Variables.Port)) +tr.Processes.Default.StartBefore(Test.Processes.ts) +tr.Processes.Default.Streams.stdout = "gold/server_push_preload_0_stdout.gold" +tr.StillRunningAfter = microserver diff --git a/tests/gold_tests/pluginTest/server_push_preload/ssl/server.key b/tests/gold_tests/pluginTest/server_push_preload/ssl/server.key new file mode 100644 index 00000000000..4c7a661a6bd --- /dev/null +++ b/tests/gold_tests/pluginTest/server_push_preload/ssl/server.key @@ -0,0 +1,15 @@ +-----BEGIN RSA PRIVATE KEY----- +MIICXQIBAAKBgQDWMHOiUF+ORmZjAxI8MWE9dblb7gQSJ36WCXlPFiFx6ynF+S1E +kXAYpIip5X0pzDUaIbLukxJUAAnOtMEO0PCgxJQUrEtRWh8wiJdbdQJF0Zs/9R+u +SUgb61f+mdTQvhqefBGx+xrpfAcgtcWiZuSA9Q3fvpDj5WOWSPWXBUuxywIDAQAB +AoGBAJPxRX2gjFAGWmQbU/YVmXfNH6navh8X/nx9sLeqrpE0AFeJI/ZPiqDKzMal +B43eSfNxwVi+ZxN0L1ICUbL9KKZvHs/QBxWLA1fGVAXrz7sRplEVvakPpTfHoEnv +sKaMWVKaK/S5WGbDhElb6zb/Lwo19DsIAPjGYqFvzFJBmobJAkEA9iSeTGkR9X26 +GywZoYrIMlRh34htOIRx1UUq88rFzdrCF21kQ4lhBIkX5OZMMy652i2gyak4OZTe +YewIv8jw9QJBAN7EQNHG8jPwXfVp91/fqxVQEfumuP2i6uiWWYQgZCmla2+0xcLZ +pMQ6sQEe10hhTrVnzHgAUVp50Ntn2jwBX78CQF09veGAI9d1Cxzj9cmmAvRd1r2Q +tp8kPOLnUsALXib+6WtqewLCdcf8DtsdClyRJMIraq85tRzK8fryKNZNzkkCQEgA +yS7FDj5JgCU15hZgFk1iPx3HCt44jZM2HaL+UUHAzRQjKxTLAl3G1rWVAWLMyQML +lORoveLvotl4HOruSsMCQQCAx9dV9JUSFoyc1CWILp/FgUH/se4cjQCThGO0DoQQ +vGTYmntY7j9WRJ9esQrjdD6Clw8zM/45GIBNwnXzqo7Z +-----END RSA PRIVATE KEY----- diff --git a/tests/gold_tests/pluginTest/server_push_preload/ssl/server.pem b/tests/gold_tests/pluginTest/server_push_preload/ssl/server.pem new file mode 100644 index 00000000000..3584a2ec119 --- /dev/null +++ b/tests/gold_tests/pluginTest/server_push_preload/ssl/server.pem @@ -0,0 +1,32 @@ +-----BEGIN RSA PRIVATE KEY----- +MIICXQIBAAKBgQDWMHOiUF+ORmZjAxI8MWE9dblb7gQSJ36WCXlPFiFx6ynF+S1E +kXAYpIip5X0pzDUaIbLukxJUAAnOtMEO0PCgxJQUrEtRWh8wiJdbdQJF0Zs/9R+u +SUgb61f+mdTQvhqefBGx+xrpfAcgtcWiZuSA9Q3fvpDj5WOWSPWXBUuxywIDAQAB +AoGBAJPxRX2gjFAGWmQbU/YVmXfNH6navh8X/nx9sLeqrpE0AFeJI/ZPiqDKzMal +B43eSfNxwVi+ZxN0L1ICUbL9KKZvHs/QBxWLA1fGVAXrz7sRplEVvakPpTfHoEnv +sKaMWVKaK/S5WGbDhElb6zb/Lwo19DsIAPjGYqFvzFJBmobJAkEA9iSeTGkR9X26 +GywZoYrIMlRh34htOIRx1UUq88rFzdrCF21kQ4lhBIkX5OZMMy652i2gyak4OZTe +YewIv8jw9QJBAN7EQNHG8jPwXfVp91/fqxVQEfumuP2i6uiWWYQgZCmla2+0xcLZ +pMQ6sQEe10hhTrVnzHgAUVp50Ntn2jwBX78CQF09veGAI9d1Cxzj9cmmAvRd1r2Q +tp8kPOLnUsALXib+6WtqewLCdcf8DtsdClyRJMIraq85tRzK8fryKNZNzkkCQEgA +yS7FDj5JgCU15hZgFk1iPx3HCt44jZM2HaL+UUHAzRQjKxTLAl3G1rWVAWLMyQML +lORoveLvotl4HOruSsMCQQCAx9dV9JUSFoyc1CWILp/FgUH/se4cjQCThGO0DoQQ +vGTYmntY7j9WRJ9esQrjdD6Clw8zM/45GIBNwnXzqo7Z +-----END RSA PRIVATE KEY----- +-----BEGIN CERTIFICATE----- +MIICszCCAhwCCQCl0Y79KkYjpzANBgkqhkiG9w0BAQUFADCBnTELMAkGA1UEBhMC +VVMxCzAJBgNVBAgTAklMMRIwEAYDVQQHEwlDaGFtcGFpZ24xDjAMBgNVBAoTBVlh +aG9vMQ0wCwYDVQQLEwRFZGdlMSgwJgYDVQQDEx9qdWljZXByb2R1Y2UuY29ycC5u +ZTEueWFob28uY29tMSQwIgYJKoZIhvcNAQkBFhVwZXJzaWEuYXppekB5YWhvby5j +b20wHhcNMTcwODI4MDI1MjI5WhcNMjcwODI2MDI1MjI5WjCBnTELMAkGA1UEBhMC +VVMxCzAJBgNVBAgTAklMMRIwEAYDVQQHEwlDaGFtcGFpZ24xDjAMBgNVBAoTBVlh +aG9vMQ0wCwYDVQQLEwRFZGdlMSgwJgYDVQQDEx9qdWljZXByb2R1Y2UuY29ycC5u +ZTEueWFob28uY29tMSQwIgYJKoZIhvcNAQkBFhVwZXJzaWEuYXppekB5YWhvby5j +b20wgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBANYwc6JQX45GZmMDEjwxYT11 +uVvuBBInfpYJeU8WIXHrKcX5LUSRcBikiKnlfSnMNRohsu6TElQACc60wQ7Q8KDE +lBSsS1FaHzCIl1t1AkXRmz/1H65JSBvrV/6Z1NC+Gp58EbH7Gul8ByC1xaJm5ID1 +Dd++kOPlY5ZI9ZcFS7HLAgMBAAEwDQYJKoZIhvcNAQEFBQADgYEAASZbz+d+DdI+ +ypesJrlBRosXh0w8sIjkUSSdT/OuKEVzfH/dRcb4VZDW/W2gmm0VEqSN2xYYVpW3 +hUsW2J+kByqFqX6selREwo8ui8kkyBJVo0y/MCrGM0C3qw1cSaiKoa5OqlOyO3hb +ZC9IIyWmpBxRmJFfIwS6MoTpe0/ZTJQ= +-----END CERTIFICATE----- diff --git a/tests/gold_tests/pluginTest/slice/gold/slice_200.stdout.gold b/tests/gold_tests/pluginTest/slice/gold/slice_200.stdout.gold index 1e704c37b3d..50eeb38afb7 100644 --- a/tests/gold_tests/pluginTest/slice/gold/slice_200.stdout.gold +++ b/tests/gold_tests/pluginTest/slice/gold/slice_200.stdout.gold @@ -1,8 +1,8 @@ -`` -Cache-Control: `` -Connection: `` -Content-Length: 18 -Date: `` -Etag: "path" -HTTP/1.1 200 OK -Server: `` +`` +Cache-Control: `` +Connection: `` +Content-Length: 18 +Date: `` +Etag: "path" +HTTP/1.1 200 OK +Server: `` diff --git a/tests/gold_tests/pluginTest/slice/gold/slice_206.stdout.gold b/tests/gold_tests/pluginTest/slice/gold/slice_206.stdout.gold index d4bd2de8782..0ff73438579 100644 --- a/tests/gold_tests/pluginTest/slice/gold/slice_206.stdout.gold +++ b/tests/gold_tests/pluginTest/slice/gold/slice_206.stdout.gold @@ -1,9 +1,9 @@ -`` -Cache-Control: `` -Connection: `` -Content-Length: 18 -Content-Range: bytes 0-17/18 -Date: `` -Etag: "path" -HTTP/1.1 206 Partial Content -Server: `` +`` +Cache-Control: `` +Connection: `` +Content-Length: 18 +Content-Range: bytes 0-17/18 +Date: `` +Etag: "path" +HTTP/1.1 206 Partial Content +Server: `` diff --git a/tests/gold_tests/pluginTest/slice/gold/slice_first.stdout.gold b/tests/gold_tests/pluginTest/slice/gold/slice_first.stdout.gold index 235b074c618..c12d4c194da 100644 --- a/tests/gold_tests/pluginTest/slice/gold/slice_first.stdout.gold +++ b/tests/gold_tests/pluginTest/slice/gold/slice_first.stdout.gold @@ -1,9 +1,9 @@ -`` -Cache-Control: max-age=`` -Connection: `` -Content-Length: 7 -Content-Range: bytes 0-6/18 -Date: `` -Etag: `` -HTTP/1.1 206 Partial Content -Server: `` +`` +Cache-Control: max-age=`` +Connection: `` +Content-Length: 7 +Content-Range: bytes 0-6/18 +Date: `` +Etag: `` +HTTP/1.1 206 Partial Content +Server: `` diff --git a/tests/gold_tests/pluginTest/slice/gold/slice_last.stderr.gold b/tests/gold_tests/pluginTest/slice/gold/slice_last.stderr.gold index 6c9cadd2dfd..30d9d270514 100644 --- a/tests/gold_tests/pluginTest/slice/gold/slice_last.stderr.gold +++ b/tests/gold_tests/pluginTest/slice/gold/slice_last.stderr.gold @@ -1 +1 @@ - now`` + now`` diff --git a/tests/gold_tests/pluginTest/slice/gold/slice_last.stdout.gold b/tests/gold_tests/pluginTest/slice/gold/slice_last.stdout.gold index 88fa42bcfba..8457c952f01 100644 --- a/tests/gold_tests/pluginTest/slice/gold/slice_last.stdout.gold +++ b/tests/gold_tests/pluginTest/slice/gold/slice_last.stdout.gold @@ -1,9 +1,9 @@ -`` -Cache-Control: max-age=`` -Connection: `` -Content-Length: 4 -Content-Range: bytes 14-17/18 -Date: `` -Etag: `` -HTTP/1.1 206 Partial Content -Server: `` +`` +Cache-Control: max-age=`` +Connection: `` +Content-Length: 4 +Content-Range: bytes 14-17/18 +Date: `` +Etag: `` +HTTP/1.1 206 Partial Content +Server: `` diff --git a/tests/gold_tests/pluginTest/slice/gold/slice_mid.stderr.gold b/tests/gold_tests/pluginTest/slice/gold/slice_mid.stderr.gold index a9dc13c610e..ef40e3253f5 100644 --- a/tests/gold_tests/pluginTest/slice/gold/slice_mid.stderr.gold +++ b/tests/gold_tests/pluginTest/slice/gold/slice_mid.stderr.gold @@ -1 +1 @@ -go surfin no`` +go surfin no`` diff --git a/tests/gold_tests/pluginTest/slice/gold/slice_mid.stdout.gold b/tests/gold_tests/pluginTest/slice/gold/slice_mid.stdout.gold index 5067f1cce49..e078833d768 100644 --- a/tests/gold_tests/pluginTest/slice/gold/slice_mid.stdout.gold +++ b/tests/gold_tests/pluginTest/slice/gold/slice_mid.stdout.gold @@ -1,9 +1,9 @@ -`` -Cache-Control: max-age=`` -Connection: `` -Content-Length: 12 -Content-Range: bytes 5-16/18 -Date: `` -Etag: `` -HTTP/1.1 206 Partial Content -Server: `` +`` +Cache-Control: max-age=`` +Connection: `` +Content-Length: 12 +Content-Range: bytes 5-16/18 +Date: `` +Etag: `` +HTTP/1.1 206 Partial Content +Server: `` diff --git a/tests/gold_tests/pluginTest/slice/gold_error/crr.stdout.gold b/tests/gold_tests/pluginTest/slice/gold_error/crr.stdout.gold index 955a42451c0..7109c04a13f 100644 --- a/tests/gold_tests/pluginTest/slice/gold_error/crr.stdout.gold +++ b/tests/gold_tests/pluginTest/slice/gold_error/crr.stdout.gold @@ -1,9 +1,9 @@ -`` -Cache-Control: `` -Connection: `` -Content-Length: 19 -Date: `` -Etag: `` -HTTP/1.1 200 OK -Server: `` -Via: `` +`` +Cache-Control: `` +Connection: `` +Content-Length: 19 +Date: `` +Etag: `` +HTTP/1.1 200 OK +Server: `` +Via: `` diff --git a/tests/gold_tests/pluginTest/slice/gold_error/etag.stdout.gold b/tests/gold_tests/pluginTest/slice/gold_error/etag.stdout.gold index 955a42451c0..7109c04a13f 100644 --- a/tests/gold_tests/pluginTest/slice/gold_error/etag.stdout.gold +++ b/tests/gold_tests/pluginTest/slice/gold_error/etag.stdout.gold @@ -1,9 +1,9 @@ -`` -Cache-Control: `` -Connection: `` -Content-Length: 19 -Date: `` -Etag: `` -HTTP/1.1 200 OK -Server: `` -Via: `` +`` +Cache-Control: `` +Connection: `` +Content-Length: 19 +Date: `` +Etag: `` +HTTP/1.1 200 OK +Server: `` +Via: `` diff --git a/tests/gold_tests/pluginTest/slice/gold_error/lm.stdout.gold b/tests/gold_tests/pluginTest/slice/gold_error/lm.stdout.gold index 2d83f7e9fd3..20bd1a54127 100644 --- a/tests/gold_tests/pluginTest/slice/gold_error/lm.stdout.gold +++ b/tests/gold_tests/pluginTest/slice/gold_error/lm.stdout.gold @@ -1,9 +1,9 @@ -`` -Cache-Control: `` -Connection: `` -Content-Length: 19 -Date: `` -HTTP/1.1 200 OK -Last-Modified: `` -Server: `` -Via: `` +`` +Cache-Control: `` +Connection: `` +Content-Length: 19 +Date: `` +HTTP/1.1 200 OK +Last-Modified: `` +Server: `` +Via: `` diff --git a/tests/gold_tests/pluginTest/slice/gold_error/non206.stdout.gold b/tests/gold_tests/pluginTest/slice/gold_error/non206.stdout.gold index 86ee5ddcb2a..f381661d6a6 100644 --- a/tests/gold_tests/pluginTest/slice/gold_error/non206.stdout.gold +++ b/tests/gold_tests/pluginTest/slice/gold_error/non206.stdout.gold @@ -1,10 +1,10 @@ -`` -Cache-Control: `` -Connection: `` -Content-Length: 19 -Date: `` -Etag: `` -HTTP/1.1 200 OK -Last-Modified: `` -Server: `` -Via: `` +`` +Cache-Control: `` +Connection: `` +Content-Length: 19 +Date: `` +Etag: `` +HTTP/1.1 200 OK +Last-Modified: `` +Server: `` +Via: `` diff --git a/tests/gold_tests/pluginTest/slice/slice.test.py b/tests/gold_tests/pluginTest/slice/slice.test.py index cec66ed4ff6..3c5260c2fc7 100644 --- a/tests/gold_tests/pluginTest/slice/slice.test.py +++ b/tests/gold_tests/pluginTest/slice/slice.test.py @@ -16,13 +16,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -import os -import time Test.Summary = ''' Basic slice plugin test ''' -## Test description: +# Test description: # Preload the cache with the entire asset to be range requested. # Reload remap rule with slice plugin # Request content through the slice plugin @@ -40,20 +38,20 @@ # default root request_header_chk = {"headers": - "GET / HTTP/1.1\r\n" + - "Host: www.example.com\r\n" + - "\r\n", - "timestamp": "1469733493.993", - "body": "", -} + "GET / HTTP/1.1\r\n" + + "Host: www.example.com\r\n" + + "\r\n", + "timestamp": "1469733493.993", + "body": "", + } response_header_chk = {"headers": - "HTTP/1.1 200 OK\r\n" + - "Connection: close\r\n" + - "\r\n", - "timestamp": "1469733493.993", - "body": "", -} + "HTTP/1.1 200 OK\r\n" + + "Connection: close\r\n" + + "\r\n", + "timestamp": "1469733493.993", + "body": "", + } server.addResponse("sessionlog.json", request_header_chk, response_header_chk) @@ -61,22 +59,22 @@ body = "lets go surfin now" request_header = {"headers": - "GET /path HTTP/1.1\r\n" + - "Host: www.example.com\r\n" + - "\r\n", - "timestamp": "1469733493.993", - "body": "", -} + "GET /path HTTP/1.1\r\n" + + "Host: www.example.com\r\n" + + "\r\n", + "timestamp": "1469733493.993", + "body": "", + } response_header = {"headers": - "HTTP/1.1 200 OK\r\n" + - "Connection: close\r\n" + - 'Etag: "path"\r\n' + - "Cache-Control: max-age=500\r\n" + - "\r\n", - "timestamp": "1469733493.993", - "body": body, -} + "HTTP/1.1 200 OK\r\n" + + "Connection: close\r\n" + + 'Etag: "path"\r\n' + + "Cache-Control: max-age=500\r\n" + + "\r\n", + "timestamp": "1469733493.993", + "body": body, + } server.addResponse("sessionlog.json", request_header, response_header) @@ -90,12 +88,12 @@ # minimal configuration ts.Disk.records_config.update({ - 'proxy.config.diags.debug.enabled': 1, - 'proxy.config.diags.debug.tags': 'slice', - 'proxy.config.http.cache.http': 1, - 'proxy.config.http.wait_for_cache': 1, - 'proxy.config.http.insert_age_in_response': 0, - 'proxy.config.http.response_via_str': 3, + 'proxy.config.diags.debug.enabled': 1, + 'proxy.config.diags.debug.tags': 'slice', + 'proxy.config.http.cache.http': 1, + 'proxy.config.http.wait_for_cache': 1, + 'proxy.config.http.insert_age_in_response': 0, + 'proxy.config.http.response_via_str': 3, }) # 0 Test - Prefetch entire asset into cache @@ -114,8 +112,8 @@ tr = Test.AddTestRun("Load Slice plugin") remap_config_path = ts.Disk.remap_config.Name tr.Disk.File(remap_config_path, typename="ats:config").AddLines([ - 'map / http://127.0.0.1:{}'.format(server.Variables.Port) + - ' @plugin=slice.so @pparam=--test-blockbytes={}'.format(block_bytes) + 'map / http://127.0.0.1:{}'.format(server.Variables.Port) + + ' @plugin=slice.so @pparam=--blockbytes-test={}'.format(block_bytes) ]) tr.StillRunningAfter = ts diff --git a/tests/gold_tests/pluginTest/slice/slice_error.test.py b/tests/gold_tests/pluginTest/slice/slice_error.test.py index 1c42b308609..b2032cd4d18 100644 --- a/tests/gold_tests/pluginTest/slice/slice_error.test.py +++ b/tests/gold_tests/pluginTest/slice/slice_error.test.py @@ -16,13 +16,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -import os -import time Test.Summary = ''' Slice plugin error.log test ''' -## Test description: +# Test description: # Preload the cache with the entire asset to be range requested. # Reload remap rule with slice plugin # Request content through the slice plugin @@ -38,25 +36,25 @@ # Define ATS and configure ts = Test.MakeATSProcess("ts", command="traffic_manager", select_ports=True) -body = "the quick brown fox" # len 19 +body = "the quick brown fox" # len 19 # default root request_header_chk = {"headers": - "GET / HTTP/1.1\r\n" + - "Host: www.example.com\r\n" + - "Range: bytes=0-\r\n" + - "\r\n", - "timestamp": "1469733493.993", - "body": "", -} + "GET / HTTP/1.1\r\n" + + "Host: www.example.com\r\n" + + "Range: bytes=0-\r\n" + + "\r\n", + "timestamp": "1469733493.993", + "body": "", + } response_header_chk = {"headers": - "HTTP/1.1 206 Partial Content\r\n" + - "Connection: close\r\n" + - "\r\n", - "timestamp": "1469733493.993", - "body": body, -} + "HTTP/1.1 206 Partial Content\r\n" + + "Connection: close\r\n" + + "\r\n", + "timestamp": "1469733493.993", + "body": body, + } server.addResponse("sessionlog.json", request_header_chk, response_header_chk) @@ -71,170 +69,170 @@ # Mismatch etag request_header_etag0 = {"headers": - "GET /etag HTTP/1.1\r\n" + - "Host: www.example.com\r\n" + - "Range: bytes={}\r\n".format(range0) + - "X-Slicer-Info: full content request\r\n" + - "\r\n", - "timestamp": "1469733493.993", - "body": "", -} + "GET /etag HTTP/1.1\r\n" + + "Host: www.example.com\r\n" + + "Range: bytes={}\r\n".format(range0) + + "X-Slicer-Info: full content request\r\n" + + "\r\n", + "timestamp": "1469733493.993", + "body": "", + } response_header_etag0 = {"headers": - "HTTP/1.1 206 Partial Content\r\n" + - "Connection: close\r\n" + - 'Etag: "etag0"\r\n' + - "Content-Range: bytes {}/{}\r\n".format(range0, len(body)) + - "Cache-Control: max-age=500\r\n" + - "\r\n", - "timestamp": "1469733493.993", - "body": body0, -} + "HTTP/1.1 206 Partial Content\r\n" + + "Connection: close\r\n" + + 'Etag: "etag0"\r\n' + + "Content-Range: bytes {}/{}\r\n".format(range0, len(body)) + + "Cache-Control: max-age=500\r\n" + + "\r\n", + "timestamp": "1469733493.993", + "body": body0, + } server.addResponse("sessionlog.json", request_header_etag0, response_header_etag0) request_header_etag1 = {"headers": - "GET /etag HTTP/1.1\r\n" + - "Host: www.example.com\r\n" + - "Range: bytes={}\r\n".format(range1) + - "X-Slicer-Info: full content request\r\n" + - "\r\n", - "timestamp": "1469733493.993", - "body": "", -} + "GET /etag HTTP/1.1\r\n" + + "Host: www.example.com\r\n" + + "Range: bytes={}\r\n".format(range1) + + "X-Slicer-Info: full content request\r\n" + + "\r\n", + "timestamp": "1469733493.993", + "body": "", + } response_header_etag1 = {"headers": - "HTTP/1.1 206 Partial Content\r\n" + - "Connection: close\r\n" + - 'Etag: "etag1"\r\n' + - "Content-Range: bytes {}/{}\r\n".format(range1, len(body)) + - "Cache-Control: max-age=500\r\n" + - "\r\n", - "timestamp": "1469733493.993", - "body": body1, -} + "HTTP/1.1 206 Partial Content\r\n" + + "Connection: close\r\n" + + 'Etag: "etag1"\r\n' + + "Content-Range: bytes {}/{}\r\n".format(range1, len(body)) + + "Cache-Control: max-age=500\r\n" + + "\r\n", + "timestamp": "1469733493.993", + "body": body1, + } server.addResponse("sessionlog.json", request_header_etag1, response_header_etag1) # mismatch Last-Modified request_header_lm0 = {"headers": - "GET /lastmodified HTTP/1.1\r\n" + - "Host: www.example.com\r\n" + - "Range: bytes={}\r\n".format(range0) + - "X-Slicer-Info: full content request\r\n" + - "\r\n", - "timestamp": "1469733493.993", - "body": "", -} + "GET /lastmodified HTTP/1.1\r\n" + + "Host: www.example.com\r\n" + + "Range: bytes={}\r\n".format(range0) + + "X-Slicer-Info: full content request\r\n" + + "\r\n", + "timestamp": "1469733493.993", + "body": "", + } response_header_lm0 = {"headers": - "HTTP/1.1 206 Partial Content\r\n" + - "Connection: close\r\n" + - "Last-Modified: Tue, 08 May 2018 15:49:41 GMT\r\n" + - "Content-Range: bytes {}/{}\r\n".format(range0, len(body)) + - "Cache-Control: max-age=500\r\n" + - "\r\n", - "timestamp": "1469733493.993", - "body": body0, -} + "HTTP/1.1 206 Partial Content\r\n" + + "Connection: close\r\n" + + "Last-Modified: Tue, 08 May 2018 15:49:41 GMT\r\n" + + "Content-Range: bytes {}/{}\r\n".format(range0, len(body)) + + "Cache-Control: max-age=500\r\n" + + "\r\n", + "timestamp": "1469733493.993", + "body": body0, + } server.addResponse("sessionlog.json", request_header_lm0, response_header_lm0) request_header_lm1 = {"headers": - "GET /lastmodified HTTP/1.1\r\n" + - "Host: www.example.com\r\n" + - "Range: bytes={}\r\n".format(range1) + - "X-Slicer-Info: full content request\r\n" + - "\r\n", - "timestamp": "1469733493.993", - "body": "", -} + "GET /lastmodified HTTP/1.1\r\n" + + "Host: www.example.com\r\n" + + "Range: bytes={}\r\n".format(range1) + + "X-Slicer-Info: full content request\r\n" + + "\r\n", + "timestamp": "1469733493.993", + "body": "", + } response_header_lm1 = {"headers": - "HTTP/1.1 206 Partial Content\r\n" + - "Connection: close\r\n" + - "Last-Modified: Tue, 08 Apr 2019 18:00:00 GMT\r\n" + - "Content-Range: bytes {}/{}\r\n".format(range1, len(body)) + - "Cache-Control: max-age=500\r\n" + - "\r\n", - "timestamp": "1469733493.993", - "body": body1, -} + "HTTP/1.1 206 Partial Content\r\n" + + "Connection: close\r\n" + + "Last-Modified: Tue, 08 Apr 2019 18:00:00 GMT\r\n" + + "Content-Range: bytes {}/{}\r\n".format(range1, len(body)) + + "Cache-Control: max-age=500\r\n" + + "\r\n", + "timestamp": "1469733493.993", + "body": body1, + } server.addResponse("sessionlog.json", request_header_lm1, response_header_lm1) # non 206 slice block request_header_n206_0 = {"headers": - "GET /non206 HTTP/1.1\r\n" + - "Host: www.example.com\r\n" + - "Range: bytes={}\r\n".format(range0) + - "X-Slicer-Info: full content request\r\n" + - "\r\n", - "timestamp": "1469733493.993", - "body": "", -} + "GET /non206 HTTP/1.1\r\n" + + "Host: www.example.com\r\n" + + "Range: bytes={}\r\n".format(range0) + + "X-Slicer-Info: full content request\r\n" + + "\r\n", + "timestamp": "1469733493.993", + "body": "", + } response_header_n206_0 = {"headers": - "HTTP/1.1 206 Partial Content\r\n" + - "Connection: close\r\n" + - 'Etag: "etag"\r\n' + - "Last-Modified: Tue, 08 May 2018 15:49:41 GMT\r\n" + - "Content-Range: bytes {}/{}\r\n".format(range0, len(body)) + - "Cache-Control: max-age=500\r\n" + - "\r\n", - "timestamp": "1469733493.993", - "body": body0, -} + "HTTP/1.1 206 Partial Content\r\n" + + "Connection: close\r\n" + + 'Etag: "etag"\r\n' + + "Last-Modified: Tue, 08 May 2018 15:49:41 GMT\r\n" + + "Content-Range: bytes {}/{}\r\n".format(range0, len(body)) + + "Cache-Control: max-age=500\r\n" + + "\r\n", + "timestamp": "1469733493.993", + "body": body0, + } server.addResponse("sessionlog.json", request_header_n206_0, response_header_n206_0) # mismatch content-range request_header_crr0 = {"headers": - "GET /crr HTTP/1.1\r\n" + - "Host: www.example.com\r\n" + - "Range: bytes={}\r\n".format(range0) + - "X-Slicer-Info: full content request\r\n" + - "\r\n", - "timestamp": "1469733493.993", - "body": "", -} + "GET /crr HTTP/1.1\r\n" + + "Host: www.example.com\r\n" + + "Range: bytes={}\r\n".format(range0) + + "X-Slicer-Info: full content request\r\n" + + "\r\n", + "timestamp": "1469733493.993", + "body": "", + } response_header_crr0 = {"headers": - "HTTP/1.1 206 Partial Content\r\n" + - "Connection: close\r\n" + - "Etag: crr\r\n" + - "Content-Range: bytes {}/{}\r\n".format(range0, len(body)) + - "Cache-Control: max-age=500\r\n" + - "\r\n", - "timestamp": "1469733493.993", - "body": body0, -} + "HTTP/1.1 206 Partial Content\r\n" + + "Connection: close\r\n" + + "Etag: crr\r\n" + + "Content-Range: bytes {}/{}\r\n".format(range0, len(body)) + + "Cache-Control: max-age=500\r\n" + + "\r\n", + "timestamp": "1469733493.993", + "body": body0, + } server.addResponse("sessionlog.json", request_header_crr0, response_header_crr0) request_header_crr1 = {"headers": - "GET /crr HTTP/1.1\r\n" + - "Host: www.example.com\r\n" + - "Range: bytes={}\r\n".format(range1) + - "X-Slicer-Info: full content request\r\n" + - "\r\n", - "timestamp": "1469733493.993", - "body": "", -} + "GET /crr HTTP/1.1\r\n" + + "Host: www.example.com\r\n" + + "Range: bytes={}\r\n".format(range1) + + "X-Slicer-Info: full content request\r\n" + + "\r\n", + "timestamp": "1469733493.993", + "body": "", + } response_header_crr1 = {"headers": - "HTTP/1.1 206 Partial Content\r\n" + - "Connection: close\r\n" + - "Etag: crr\r\n" + - "Content-Range: bytes {}/{}\r\n".format(range1, len(body) - 1) + - "Cache-Control: max-age=500\r\n" + - "\r\n", - "timestamp": "1469733493.993", - "body": body1, -} + "HTTP/1.1 206 Partial Content\r\n" + + "Connection: close\r\n" + + "Etag: crr\r\n" + + "Content-Range: bytes {}/{}\r\n".format(range1, len(body) - 1) + + "Cache-Control: max-age=500\r\n" + + "\r\n", + "timestamp": "1469733493.993", + "body": body1, + } server.addResponse("sessionlog.json", request_header_crr1, response_header_crr1) @@ -243,27 +241,31 @@ # set up whole asset fetch into cache ts.Disk.remap_config.AddLine( - 'map / http://127.0.0.1:{}'.format(server.Variables.Port) + - ' @plugin=slice.so @pparam=--test-blockbytes={}'.format(blockbytes) + 'map / http://127.0.0.1:{}'.format(server.Variables.Port) + + ' @plugin=slice.so @pparam=--blockbytes-test={}'.format(blockbytes) ) # minimal configuration ts.Disk.records_config.update({ - 'proxy.config.diags.debug.enabled': 0, - 'proxy.config.diags.debug.tags': 'slice', - 'proxy.config.http.cache.http': 0, - 'proxy.config.http.wait_for_cache': 0, - 'proxy.config.http.insert_age_in_response': 0, - 'proxy.config.http.insert_request_via_str': 0, - 'proxy.config.http.insert_response_via_str': 3, + 'proxy.config.diags.debug.enabled': 0, + 'proxy.config.diags.debug.tags': 'slice', + 'proxy.config.http.cache.http': 0, + 'proxy.config.http.wait_for_cache': 0, + 'proxy.config.http.insert_age_in_response': 0, + 'proxy.config.http.insert_request_via_str': 0, + 'proxy.config.http.insert_response_via_str': 3, }) # Override builtin error check as these cases will fail # taken from the slice plug code ts.Disk.diags_log.Content = Testers.ContainsExpression('reason="Mismatch block Etag"', "Mismatch block etag") ts.Disk.diags_log.Content += Testers.ContainsExpression('reason="Mismatch block Last-Modified"', "Mismatch block Last-Modified") -ts.Disk.diags_log.Content += Testers.ContainsExpression('reason="Non 206 internal block response"', "Non 206 internal block response") -ts.Disk.diags_log.Content += Testers.ContainsExpression('reason="Mismatch/Bad block Content-Range"', "Mismatch/Bad block Content-Range") +ts.Disk.diags_log.Content += Testers.ContainsExpression( + 'reason="Non 206 internal block response"', + "Non 206 internal block response") +ts.Disk.diags_log.Content += Testers.ContainsExpression( + 'reason="Mismatch/Bad block Content-Range"', + "Mismatch/Bad block Content-Range") # 0 Test - Etag mismatch test tr = Test.AddTestRun("Etag test") diff --git a/tests/gold_tests/pluginTest/slice/slice_regex.test.py b/tests/gold_tests/pluginTest/slice/slice_regex.test.py new file mode 100644 index 00000000000..366fee3a38c --- /dev/null +++ b/tests/gold_tests/pluginTest/slice/slice_regex.test.py @@ -0,0 +1,169 @@ +''' +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +Test.Summary = ''' +slice regex plugin test +''' + +# Test description: +# Preload the cache with the entire asset to be range requested. +# Reload remap rule with slice plugin +# Request content through the slice plugin + +Test.SkipUnless( + Condition.PluginExists('slice.so'), +) +Test.ContinueOnFail = False + +# configure origin server +server = Test.MakeOriginServer("server") + +# Define ATS and configure +ts = Test.MakeATSProcess("ts", command="traffic_manager", select_ports=True) + +# default root +request_header_chk = {"headers": + "GET / HTTP/1.1\r\n" + + "Host: www.example.com\r\n" + + "\r\n", + "timestamp": "1469733493.993", + "body": "", + } + +response_header_chk = {"headers": + "HTTP/1.1 200 OK\r\n" + + "Connection: close\r\n" + + "\r\n", + "timestamp": "1469733493.993", + "body": "", + } + +server.addResponse("sessionlog.json", request_header_chk, response_header_chk) + +#block_bytes = 7 +body = "lets go surfin now" + +request_header_txt = {"headers": + "GET /slice.txt HTTP/1.1\r\n" + + "Host: slice\r\n" + + "\r\n", + "timestamp": "1469733493.993", + "body": "", + } + +response_header_txt = {"headers": + "HTTP/1.1 200 OK\r\n" + + "Connection: close\r\n" + + 'Etag: "path"\r\n' + + "Cache-Control: max-age=500\r\n" + + "X-Info: notsliced\r\n" + + "\r\n", + "timestamp": "1469733493.993", + "body": body, + } + +server.addResponse("sessionlog.json", request_header_txt, response_header_txt) + +request_header_mp4 = {"headers": + "GET /slice.mp4 HTTP/1.1\r\n" + + "Host: sliced\r\n" + + "Range: bytes=0-99\r\n" + "\r\n", + "timestamp": "1469733493.993", + "body": "", + } + +response_header_mp4 = {"headers": + "HTTP/1.1 206 Partial Content\r\n" + + "Connection: close\r\n" + + 'Etag: "path"\r\n' + + "Content-Range: bytes 0-{}/{}\r\n".format(len(body) - 1, len(body)) + + "Cache-Control: max-age=500\r\n" + + "X-Info: sliced\r\n" + + "\r\n", + "timestamp": "1469733493.993", + "body": body, + } + +server.addResponse("sessionlog.json", request_header_mp4, response_header_mp4) + +curl_and_args = 'curl -s -D /dev/stdout -o /dev/stderr -x localhost:{} -H "x-debug: x-cache"'.format(ts.Variables.port) + +block_bytes = 100 + +# set up whole asset fetch into cache +ts.Disk.remap_config.AddLines([ + 'map http://exclude/ http://127.0.0.1:{}/'.format(server.Variables.Port) + + ' @plugin=slice.so' + + ' @pparam=--blockbytes-test={}'.format(block_bytes) + + ' @pparam=--exclude-regex=\\.txt' + ' @pparam=--remap-host=sliced', + 'map http://include/ http://127.0.0.1:{}/'.format(server.Variables.Port) + + ' @plugin=slice.so' + + ' @pparam=--blockbytes-test={}'.format(block_bytes) + + ' @pparam=--include-regex=\\.mp4' + ' @pparam=--remap-host=sliced', + 'map http://sliced/ http://127.0.0.1:{}/'.format(server.Variables.Port), +]) + + +# minimal configuration +ts.Disk.records_config.update({ + 'proxy.config.diags.debug.enabled': 1, + 'proxy.config.diags.debug.tags': 'slice', + 'proxy.config.http.cache.http': 0, + 'proxy.config.http.wait_for_cache': 0, + 'proxy.config.http.insert_age_in_response': 0, + 'proxy.config.http.response_via_str': 0, +}) + +# 0 Test - Exclude: ensure txt passes through +tr = Test.AddTestRun("Exclude - asset passed through") +ps = tr.Processes.Default +ps.StartBefore(server, ready=When.PortOpen(server.Variables.Port)) +ps.StartBefore(Test.Processes.ts, ready=When.PortOpen(ts.Variables.port)) +ps.Command = curl_and_args + ' http://exclude/slice.txt' +ps.ReturnCode = 0 +ps.Streams.stdout.Content = Testers.ContainsExpression("X-Info: notsliced", "expected not sliced header") +tr.StillRunningAfter = ts + +# 1 Test - Exclude mp4 gets sliced +tr = Test.AddTestRun("Exclude - asset is sliced") +ps = tr.Processes.Default +ps.Command = curl_and_args + ' http://exclude/slice.mp4' +ps.ReturnCode = 0 +ps.Streams.stdout.Content = Testers.ContainsExpression("X-Info: sliced", "expected sliced header") +tr.StillRunningAfter = ts +tr.StillRunningAfter = ts + +# 2 Test - Exclude: ensure txt passes through +tr = Test.AddTestRun("Include - asset passed through") +ps = tr.Processes.Default +ps.Command = curl_and_args + ' http://include/slice.txt' +ps.ReturnCode = 0 +ps.Streams.stdout.Content = Testers.ContainsExpression("X-Info: notsliced", "expected not sliced header") +tr.StillRunningAfter = ts + +# 3 Test - Exclude mp4 gets sliced +tr = Test.AddTestRun("Include - asset is sliced") +ps = tr.Processes.Default +ps.Command = curl_and_args + ' http://include/slice.mp4' +ps.ReturnCode = 0 +ps.Streams.stdout.Content = Testers.ContainsExpression("X-Info: sliced", "expected sliced header") +tr.StillRunningAfter = ts +tr.StillRunningAfter = ts diff --git a/tests/gold_tests/pluginTest/sslheaders/observer.py b/tests/gold_tests/pluginTest/sslheaders/observer.py index 673a56eb1a2..93fce3c7789 100644 --- a/tests/gold_tests/pluginTest/sslheaders/observer.py +++ b/tests/gold_tests/pluginTest/sslheaders/observer.py @@ -19,13 +19,15 @@ log = open('sslheaders.log', 'w') + def observe(headers): for h in headers.items(): if h[0].lower().startswith('ssl-'): - log.write(h[0] + ": " + h[1] + "\n"); + log.write(h[0] + ": " + h[1] + "\n") log.write("-\n") log.flush() + Hooks.register(Hooks.ReadRequestHook, observe) diff --git a/tests/gold_tests/pluginTest/sslheaders/sslheaders.test.py b/tests/gold_tests/pluginTest/sslheaders/sslheaders.test.py index ee8db58a265..2ad458f61e8 100644 --- a/tests/gold_tests/pluginTest/sslheaders/sslheaders.test.py +++ b/tests/gold_tests/pluginTest/sslheaders/sslheaders.test.py @@ -50,12 +50,12 @@ 'proxy.config.http.server_ports': ( 'ipv4:{0} ipv4:{1}:proto=http2;http:ssl ipv6:{0} ipv6:{1}:proto=http2;http:ssl' .format(ts.Variables.port, ts.Variables.ssl_port)), -# 'proxy.config.ssl.client.verify.server': 0, -# 'proxy.config.ssl.server.cipher_suite': 'ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA384:AES128-GCM-SHA256:AES256-GCM-SHA384:ECDHE-RSA-RC4-SHA:ECDHE-RSA-AES128-SHA:ECDHE-RSA-AES256-SHA:RC4-SHA:RC4-MD5:AES128-SHA:AES256-SHA:DES-CBC3-SHA!SRP:!DSS:!PSK:!aNULL:!eNULL:!SSLv2', -# 'proxy.config.url_remap.pristine_host_hdr' : 1, -# 'proxy.config.ssl.client.certification_level': 2, -# 'proxy.config.ssl.CA.cert.filename': '{0}/signer.pem'.format(ts.Variables.SSLDir), -# 'proxy.config.ssl.TLSv1_3': 0 + # 'proxy.config.ssl.client.verify.server': 0, + # 'proxy.config.ssl.server.cipher_suite': 'ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA384:AES128-GCM-SHA256:AES256-GCM-SHA384:ECDHE-RSA-RC4-SHA:ECDHE-RSA-AES128-SHA:ECDHE-RSA-AES256-SHA:RC4-SHA:RC4-MD5:AES128-SHA:AES256-SHA:DES-CBC3-SHA!SRP:!DSS:!PSK:!aNULL:!eNULL:!SSLv2', + # 'proxy.config.url_remap.pristine_host_hdr' : 1, + # 'proxy.config.ssl.client.certification_level': 2, + # 'proxy.config.ssl.CA.cert.filename': '{0}/signer.pem'.format(ts.Variables.SSLDir), + # 'proxy.config.ssl.TLSv1_3': 0 }) ts.Disk.ssl_multicert_config.AddLine( @@ -69,12 +69,6 @@ 'map https://bar.com http://127.0.0.1:{0}'.format(server.Variables.Port) ) -ts.Disk.sni_yaml.AddLines([ - 'sni:', - '- fqdn: "*bar.com"', - ' verify_client: STRICT', -]) - ts.Disk.plugin_config.AddLine( 'sslheaders.so SSL-Client-ID=client.subject' ) diff --git a/tests/gold_tests/pluginTest/test_hooks/hook_add.gold b/tests/gold_tests/pluginTest/test_hooks/hook_add.gold new file mode 100644 index 00000000000..9141bacdee6 --- /dev/null +++ b/tests/gold_tests/pluginTest/test_hooks/hook_add.gold @@ -0,0 +1,7 @@ +`` DIAG: (test) -- globalHandler :: TS_EVENT_HTTP_SSN_START +`` DIAG: (test) New session, cont is `` +`` DIAG: (test) -- sessionHandler :: TS_EVENT_HTTP_PRE_REMAP +`` DIAG: (test) -- transactionHandler :: TS_EVENT_HTTP_PRE_REMAP +`` DIAG: (test) -- transactionHandler :: TS_EVENT_HTTP_TXN_CLOSE +`` DIAG: (test) -- sessionHandler :: TS_EVENT_HTTP_SSN_CLOSE +`` diff --git a/tests/gold_tests/pluginTest/test_hooks/hook_add.test.py b/tests/gold_tests/pluginTest/test_hooks/hook_add.test.py new file mode 100644 index 00000000000..71532523edd --- /dev/null +++ b/tests/gold_tests/pluginTest/test_hooks/hook_add.test.py @@ -0,0 +1,60 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + + +Test.Summary = ''' +Test adding hooks +''' + +Test.ContinueOnFail = True + +server = Test.MakeOriginServer("server") + +request_header = { + "headers": "GET /argh HTTP/1.1\r\nHost: doesnotmatter\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +response_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +server.addResponse("sessionlog.json", request_header, response_header) + +ts = Test.MakeATSProcess("ts", select_ports=True, enable_tls=False) + +ts.Disk.records_config.update({ + 'proxy.config.diags.debug.tags': 'test', + 'proxy.config.diags.debug.enabled': 1, + 'proxy.config.http.cache.http': 0, + 'proxy.config.url_remap.remap_required': 0, +}) + +Test.PrepareTestPlugin(os.path.join(Test.Variables.AtsTestPluginsDir, 'hook_add_plugin.so'), ts) + +ts.Disk.remap_config.AddLine( + "map http://one http://127.0.0.1:{0}".format(server.Variables.Port) +) + +tr = Test.AddTestRun() +# Probe server port to check if ready. +tr.Processes.Default.StartBefore(server, ready=When.PortOpen(server.Variables.Port)) +# Probe TS cleartext port to check if ready (probing TLS port causes spurious VCONN hook triggers). +tr.Processes.Default.StartBefore(Test.Processes.ts, ready=When.PortOpen(ts.Variables.port)) +# +tr.Processes.Default.Command = ( + 'curl --verbose --ipv4 --header "Host: one" http://localhost:{0}/argh'.format(ts.Variables.port) +) +tr.Processes.Default.ReturnCode = 0 + +# Look at the debug output from the plugin +ts.Streams.All = "hook_add.gold" diff --git a/tests/gold_tests/pluginTest/test_hooks/log.gold b/tests/gold_tests/pluginTest/test_hooks/log.gold index b798539374a..7975eb54053 100644 --- a/tests/gold_tests/pluginTest/test_hooks/log.gold +++ b/tests/gold_tests/pluginTest/test_hooks/log.gold @@ -24,7 +24,26 @@ Transaction: event=TS_EVENT_HTTP_READ_REQUEST_HDR Global: event=TS_EVENT_HTTP_TXN_CLOSE Session: event=TS_EVENT_HTTP_TXN_CLOSE Transaction: event=TS_EVENT_HTTP_TXN_CLOSE +`` Global: event=TS_EVENT_HTTP_SSN_CLOSE Session: event=TS_EVENT_HTTP_SSN_CLOSE -Global: event=TS_EVENT_VCONN_CLOSE +`` +Global: event=TS_EVENT_VCONN_START +Global: ssl flag=1 +Global: event=TS_EVENT_SSL_SERVERNAME +Global: ssl flag=1 +Global: event=TS_EVENT_SSL_CERT Global: ssl flag=1 +Global: event=TS_EVENT_HTTP_SSN_START +Global: event=TS_EVENT_HTTP_TXN_START +Session: event=TS_EVENT_HTTP_TXN_START +Global: event=TS_EVENT_HTTP_READ_REQUEST_HDR +Session: event=TS_EVENT_HTTP_READ_REQUEST_HDR +Transaction: event=TS_EVENT_HTTP_READ_REQUEST_HDR +Global: event=TS_EVENT_HTTP_TXN_CLOSE +Session: event=TS_EVENT_HTTP_TXN_CLOSE +Transaction: event=TS_EVENT_HTTP_TXN_CLOSE +`` +Global: event=TS_EVENT_HTTP_SSN_CLOSE +Session: event=TS_EVENT_HTTP_SSN_CLOSE +`` diff --git a/tests/gold_tests/pluginTest/test_hooks/test_hooks.test.py b/tests/gold_tests/pluginTest/test_hooks/test_hooks.test.py index a35c87f16e7..17a218b2988 100644 --- a/tests/gold_tests/pluginTest/test_hooks/test_hooks.test.py +++ b/tests/gold_tests/pluginTest/test_hooks/test_hooks.test.py @@ -14,6 +14,9 @@ # See the License for the specific language governing permissions and # limitations under the License. +import os + + Test.Summary = ''' Test TS API Hooks. ''' @@ -29,8 +32,8 @@ server = Test.MakeOriginServer("server") request_header = { - "headers": "GET /argh HTTP/1.1\r\nHost: doesnotmatter\r\n\r\n", "timestamp": "1469733493.993", "body": "" } -response_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": "" } + "headers": "GET /argh HTTP/1.1\r\nHost: doesnotmatter\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +response_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": ""} server.addResponse("sessionlog.json", request_header, response_header) ts = Test.MakeATSProcess("ts", select_ports=True, enable_tls=True) @@ -52,7 +55,7 @@ 'dest_ip=* ssl_cert_name=server.pem ssl_key_name=server.key' ) -Test.PreparePlugin(Test.Variables.AtsTestToolsDir + '/plugins/test_hooks.cc', ts) +Test.PrepareTestPlugin(os.path.join(Test.Variables.AtsTestPluginsDir, 'test_hooks.so'), ts) ts.Disk.remap_config.AddLine( "map http://one http://127.0.0.1:{0}".format(server.Variables.Port) @@ -78,6 +81,12 @@ ) tr.Processes.Default.ReturnCode = 0 +tr = Test.AddTestRun() +tr.Processes.Default.Command = ( + 'curl --verbose --ipv4 --http1.1 --insecure --header "Host: one" https://localhost:{0}/argh'.format(ts.Variables.ssl_port) +) +tr.Processes.Default.ReturnCode = 0 + # The probing of the ATS port to detect when ATS is ready may be seen by ATS as a VCONN start/close, so filter out these # events from the log file. # @@ -90,3 +99,4 @@ tr.Processes.Default.ReturnCode = 0 f = tr.Disk.File("log.txt") f.Content = "log.gold" +f.Content += Testers.ContainsExpression("Global: event=TS_EVENT_VCONN_CLOSE", "VCONN_CLOSE should trigger 2 times") diff --git a/tests/gold_tests/pluginTest/traffic_dump/gold/200.gold b/tests/gold_tests/pluginTest/traffic_dump/gold/200.gold new file mode 100644 index 00000000000..1fe8660bbd7 --- /dev/null +++ b/tests/gold_tests/pluginTest/traffic_dump/gold/200.gold @@ -0,0 +1,13 @@ +`` +> GET /`` HTTP/1.1 +> Host: www.example.com`` +> User-Agent: curl/`` +> Accept: */* +`` +< HTTP/1.1 200 OK +< Content-Length: 0 +< Set-Cookie: classified_not_for_logging +< Date: `` +`` +< Server: ATS/`` +`` diff --git a/tests/gold_tests/pluginTest/traffic_dump/gold/200_bob_no_sni.gold b/tests/gold_tests/pluginTest/traffic_dump/gold/200_bob_no_sni.gold new file mode 100644 index 00000000000..9638e10bd27 --- /dev/null +++ b/tests/gold_tests/pluginTest/traffic_dump/gold/200_bob_no_sni.gold @@ -0,0 +1,7 @@ +`` +> GET / HTTP/2 +> Host: bob--cert +`` +< HTTP/2 200 +< content-length: 0 +`` diff --git a/tests/gold_tests/pluginTest/traffic_dump/gold/200_sni_bob.gold b/tests/gold_tests/pluginTest/traffic_dump/gold/200_sni_bob.gold new file mode 100644 index 00000000000..ed3fa2bbd60 --- /dev/null +++ b/tests/gold_tests/pluginTest/traffic_dump/gold/200_sni_bob.gold @@ -0,0 +1,11 @@ +`` +* Hostname bob was found in DNS cache +`` +> GET / HTTP/2 +> Host: bob +`` +< HTTP/2 200 +< content-length: 0 +`` +< server: ATS/`` +`` diff --git a/tests/gold_tests/pluginTest/traffic_dump/gold/200_sni_dave.gold b/tests/gold_tests/pluginTest/traffic_dump/gold/200_sni_dave.gold new file mode 100644 index 00000000000..0d0e91a4782 --- /dev/null +++ b/tests/gold_tests/pluginTest/traffic_dump/gold/200_sni_dave.gold @@ -0,0 +1,12 @@ +`` +* Hostname dave was found in DNS cache +`` +> GET / HTTP/2 +> Host: dave +`` +< HTTP/2 200 +< content-length: 0 +`` +< server: ATS/`` +`` + diff --git a/tests/gold_tests/pluginTest/traffic_dump/gold/4_byte_response_body.gold b/tests/gold_tests/pluginTest/traffic_dump/gold/4_byte_response_body.gold new file mode 100644 index 00000000000..81846e6191b --- /dev/null +++ b/tests/gold_tests/pluginTest/traffic_dump/gold/4_byte_response_body.gold @@ -0,0 +1,7 @@ +`` +> GET /cache_test HTTP/1.1 +`` +< HTTP/1.1 200 OK +< Cache-Control: max-age=300 +< Content-Length: 4 +`` diff --git a/tests/gold_tests/pluginTest/traffic_dump/gold/explicit_target.gold b/tests/gold_tests/pluginTest/traffic_dump/gold/explicit_target.gold new file mode 100644 index 00000000000..40e21cb7abf --- /dev/null +++ b/tests/gold_tests/pluginTest/traffic_dump/gold/explicit_target.gold @@ -0,0 +1,8 @@ +`` +> GET http://localhost:``/candy HTTP/1.1 +> Host: www.example.com`` +> User-Agent: curl/`` +> Accept: */* +`` +< Server: ATS/`` +`` diff --git a/tests/gold_tests/pluginTest/traffic_dump/gold/post_with_body.gold b/tests/gold_tests/pluginTest/traffic_dump/gold/post_with_body.gold new file mode 100644 index 00000000000..1dad3b38c75 --- /dev/null +++ b/tests/gold_tests/pluginTest/traffic_dump/gold/post_with_body.gold @@ -0,0 +1,8 @@ +`` +> POST http://localhost:``/post_with_body HTTP/1.1 +> Host: www.example.com`` +> User-Agent: curl/`` +> Accept: */* +`` +< Server: ATS/`` +`` diff --git a/tests/gold_tests/pluginTest/traffic_dump/gold/two_transactions.gold b/tests/gold_tests/pluginTest/traffic_dump/gold/two_transactions.gold new file mode 100644 index 00000000000..9ac048cb49e --- /dev/null +++ b/tests/gold_tests/pluginTest/traffic_dump/gold/two_transactions.gold @@ -0,0 +1,11 @@ +`` +> GET /first HTTP/1.1 +> Host: www.example.com +`` +< HTTP/1.1 200 OK +`` +> GET /second HTTP/1.1 +> Host: www.example.com +`` +< HTTP/1.1 200 OK +`` diff --git a/tests/gold_tests/pluginTest/traffic_dump/ssl/server.key b/tests/gold_tests/pluginTest/traffic_dump/ssl/server.key new file mode 100644 index 00000000000..9cdfc36aa5f --- /dev/null +++ b/tests/gold_tests/pluginTest/traffic_dump/ssl/server.key @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQCZkEXSlZ+ZFKFg +CPpcDG39e73BuK6E5uE38q2PHh4DV0xcsJnIUx51viqLPwYughxfP0crHyBdXoHV +dW/3WX4gpiGrdiM/dvCouheo0DPaqUUJ2nZKVYh2M57oyeiuJidlKb7BGkfw3HWP +9TV7dVyGWok/cowjopqaLHJWxg/kh2KqvUBD0CHt9Kd1XvgXVmHwE7vCv0j5owv2 +MaExTsFb16uWmVLhl1gNHI2RqCX2yLaebH1DvtbLrit1XErjtaSYeJE9clVRaqT6 +vsvLOhyB5tA9WfZqfBYr/MHDeXQfrbIf+4Cp3aTpq5grc5InIMMH0eOk6/f/4tW+ +nq1lfszZAgMBAAECggEAYvYAqRbXRRVwca0Xel5gO2yU+tSDUw5esWlow8RK3yhR +A6KjV9+Iz6P/UsEIwMwEcLUcrgNfHgybau5Fe4dmqq+lHxQA3xNNP869FIMoB4/x +98mbVYgNau8VRztnAWOBG8ZtMZA4MFZCRMVm8+rL96E8tXCiMwzEyPo/rP/ymfhN +3GRunX+GhfIA79AYNbd7HMVL+cvWWUGUF5Bc5i1wXcLy4I7b9NYtv920BeCLzSFK +BypFB7ku/vKgTcBxe4yxThxPeXPwm4WFzGYKk/Afl1j8tVXCE2U4Y3yykfC0Qk6S +ECZbCKLO2Rxi9fclIDZBHWuKejZhdjHfjeNvZ2vLoQKBgQDJzLmkVLvWAxgl1yvF +U7gwqj/TzYqtVowbjEvTNEnPU1j/hIVI343SVV/EvJmif/iRUop6sRYfLsUjpMsH +CmPysNKL3UtgSYOxLs+0xLhG4OOQRpPSf/uvl9YyWY9G3AqiC7ScthkQjEhZa4c1 +eycYy0jr42kX0OL9MuIH9q0ENQKBgQDCzvGKMs8r5E/Qd3YB9VYB60dx+6G83AHZ +YqIelykObhCdxL9n4K+p4VKKLvgTcCOLYYIkBSWRJWR+ue3s3ey9+XWd2/q4Xvfh +TCjAuO2ibMV+y5ClNlW0fQ/doIVWSDbjO2tZW1jh7YWZ4CtuVrsEisv1sk3KltMB +MguhpTUylQKBgG6TfrncMFzxrx61C+gBmvEXqQffHfkjbnx94OKnSTaQ3jiNHhez +X9v8KhD8o1bWtpay2uyl8pA9qYqBdzqxZ9kJKSW4qd/mCIJjOy87iBpWint5IPD8 +biZmldlbF9ZlJnJq5ZnlclCN/er5r8oPZHoCkj+nieOh8294nUBt25ptAoGAMnPA +EIeaKgbmONpHgLhWPwb9KOL/f1cHT5KA5CVH58nPmdyTqcaCGCAX7Vu+ueIIApgN +SWDf2thxT3S9zuOm5YiO0oRfSZKm5f2AbHE4ciFzgKQd4PvSdH0TN9XT0oW/WVhR +NAI5YcHPIQvyk4/4vXNo4Uf9Z6NqIFwisQmFXoUCgYBK/ZI/HsFsvnR5MV0tFdGM +AdNe6bsQRSZkowoaPxuWbklE4Hn6QvwEmQg3ST2O+vCQV1f1yI6YiWYoptOYscJc +MSs/HxhhaaO5ZsiuPUO6WEPzpNb2CxuIGDDtl83VtUQyjxCmOb6pqqjwzFmZ2bsw +JNMaBCzokrJTxknvauCuSQ== +-----END PRIVATE KEY----- diff --git a/tests/gold_tests/pluginTest/traffic_dump/ssl/server.pem b/tests/gold_tests/pluginTest/traffic_dump/ssl/server.pem new file mode 100644 index 00000000000..2b56cc83ea2 --- /dev/null +++ b/tests/gold_tests/pluginTest/traffic_dump/ssl/server.pem @@ -0,0 +1,21 @@ +-----BEGIN CERTIFICATE----- +MIIDZDCCAkygAwIBAgIJANod1+h9CtCaMA0GCSqGSIb3DQEBCwUAMEcxCzAJBgNV +BAYTAlVTMQswCQYDVQQIDAJJTDEPMA0GA1UECgwGQXBhY2hlMRowGAYDVQQDDBFy +YW5kb20uc2VydmVyLmNvbTAeFw0xODExMTkxNzEwMTlaFw0yODExMTYxNzEwMTla +MEcxCzAJBgNVBAYTAlVTMQswCQYDVQQIDAJJTDEPMA0GA1UECgwGQXBhY2hlMRow +GAYDVQQDDBFyYW5kb20uc2VydmVyLmNvbTCCASIwDQYJKoZIhvcNAQEBBQADggEP +ADCCAQoCggEBAJmQRdKVn5kUoWAI+lwMbf17vcG4roTm4TfyrY8eHgNXTFywmchT +HnW+Kos/Bi6CHF8/RysfIF1egdV1b/dZfiCmIat2Iz928Ki6F6jQM9qpRQnadkpV +iHYznujJ6K4mJ2UpvsEaR/DcdY/1NXt1XIZaiT9yjCOimposclbGD+SHYqq9QEPQ +Ie30p3Ve+BdWYfATu8K/SPmjC/YxoTFOwVvXq5aZUuGXWA0cjZGoJfbItp5sfUO+ +1suuK3VcSuO1pJh4kT1yVVFqpPq+y8s6HIHm0D1Z9mp8Fiv8wcN5dB+tsh/7gKnd +pOmrmCtzkicgwwfR46Tr9//i1b6erWV+zNkCAwEAAaNTMFEwHQYDVR0OBBYEFI2y +qm0+UAChDAnLrAINeFOuyUlhMB8GA1UdIwQYMBaAFI2yqm0+UAChDAnLrAINeFOu +yUlhMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZIhvcNAQELBQADggEBAA3ZNFbqxcOX +szS5A4EXCepyBJBFejEYy0CsvwQX/ai/pMrw5jqVeF0GAOTpBCVLddyY+ZV1arD2 +Pqi7Qwot9OxEZOzbCBiuMJGotruKgnWFQDHzJ9HA7KDQs270uNESAOG/xW9os9zN +MXApzqfBSR5EIQU5L3RtaiPzoKdQenGQUOj86s0Kon7snDSUzaA2VcfstMWgGvXP +JHtaVusULm0gry32cEap5G5UK+gII6DfLWgFwFGhHHmTz3mKjyGiJQ+09XBtu4lb +ENE+HGRBBA49dUKSr3kwErO4HyHnS0YrsTDnbYURCsGUDma12oijX2sCos6Q4zn8 +3svaouRrucw= +-----END CERTIFICATE----- diff --git a/tests/gold_tests/pluginTest/traffic_dump/ssl/signed-foo.key b/tests/gold_tests/pluginTest/traffic_dump/ssl/signed-foo.key new file mode 100644 index 00000000000..e2c7066f679 --- /dev/null +++ b/tests/gold_tests/pluginTest/traffic_dump/ssl/signed-foo.key @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDJheOsr0o4C0QV +elo8viYl/IjWvkj4Y6YojnO/E9T5HSfWxkRikRT8X7yqs9ClrjNcYoPNNQP5vgl1 +qAIeixUrbZZklcZ3kHF/MXdzuYV//uT3YfWrsOTdfbcu6adHtBr/Qc0w9GSbBpp4 +2ZALxzjZ2hZZq51XmKvcz55/gm/9YvlGErzvsPzkejdH6U1GKeLqGaZOqkV7N2J4 +zKmJyiPzwrmR/8y7k+9jDHQX+A2wa5gaiAMGdIM8aTV+XsNNAvespvXyX7UhwxDs +w0er0GLv4ssYCru5hF+tUcjeOCHxSnCoEC45b2zDZHATLLjr7AIJ52TcSo1kTaTi +rAOAiHRLAgMBAAECggEAB7fXBnAYOZlE3EW5WwY1U9MeMotLJCg83uTFzhWmXHwf +YHxrdhL0aM4J3cfRP+cyFGG5hox3QINkvVrX6e+NugISdnu+BCpGDocIeigq0sIi +Zs8bp524xjrgXy2XuIlPV2NfxnY1vDI+jE5Y0/qnVMCjhn+qIQa53lUdTujh/SRR +3U7di+QMK4mdGwRnInos++ENy33A+2LqtUK8i0ERkzPFa1yMQEE4DOFPzZcW+jhK +arvzBwPIn37PZmL5oyiQB1YiGPGt4XNfPBwACTMYM8LlYBfEBHG77k3bMtUf0WqE +GctoT5SIe5+YbyrWkpfHgoKPxggH3I3TrFnVvqrKQQKBgQDmcV1YbuNEQLeif521 +iGqMgPQYmnpO6k27RsZrM9ikhIgm9bVJsOqnaYzQFeSfJ3eNLXYUL6IF8g46xddw +fDBtrEjDAA9OUkNRcizbeKF+GJRMtX11d4ZNbnG1wyMZYkArZGfraZBLHPEF1pya +2iFdVfokQCBpLmX7BMQEPePyuwKBgQDf33H9njf9oO0l9GfuWDvSoaV8GwqV9x55 +sFjggQYD/xqwEprrzr524X5Y2ZiTUpBu+kqqM8GYfm3bzBKkZU1rnjwxADUwBw8U +L2U/Z7Id3om8tAdzHOSI9d7mxWA8uTsScMm0IFv2l/XBQo1+AAJSD03pcsabr4Lf +SuJGmoFTsQKBgQDVzPASEC+DL5gwh75Gop5YZXwTJ5+6f+BGlM+avquNV/kKTIU6 +LY5IbMFcfjNzBicBMOCQsfDdG0rgdJYBovc7idCoOvH4dJJIimnb5fvPBfbxhKE1 +zwMn7ARL4xQ5hNKMb8eKvpJFXkCwbgE2GpNCCXbfEy/+5jFvx2gll1ZZ6QKBgG3J +OzJ/w796irHBQLKOzI+HvAq3jCJs9KICjCNUwql1EhZkmVqooZjVDkvuMbeVlsUF +s1XyWa852RAf7Mh38VakW6pACtVJsOhaMdG9PYkOWAeVVc3qzlwoDy6mfoJo6AIs +E45lDBRLAzbKN28h/AFYBgJEygcRNCHirEKphGCRAoGAEhcaxbmMo2fHBYuvOR1Q +ZAIq1EPvysDROUBHhdTJqN1wHsuJsmVJxX42+YHcZdjtgeCdjU3HMoyCnTaRxDee +K3VeB4PobN1WpQwFklFoqcvAhW6eicdZXme7ktK120NPQsXrmjgN6Lfg3PNjosn0 +tqSxQhQ4DrSf60fxx0/M/rw= +-----END PRIVATE KEY----- diff --git a/tests/gold_tests/pluginTest/traffic_dump/ssl/signed-foo.pem b/tests/gold_tests/pluginTest/traffic_dump/ssl/signed-foo.pem new file mode 100644 index 00000000000..6f6aecf53d7 --- /dev/null +++ b/tests/gold_tests/pluginTest/traffic_dump/ssl/signed-foo.pem @@ -0,0 +1,19 @@ +-----BEGIN CERTIFICATE----- +MIIDCzCCAnQCCQC81MtBCwmQtzANBgkqhkiG9w0BAQsFADCBnTELMAkGA1UEBhMC +VVMxCzAJBgNVBAgTAklMMRIwEAYDVQQHEwlDaGFtcGFpZ24xDjAMBgNVBAoTBVlh +aG9vMQ0wCwYDVQQLEwRFZGdlMSgwJgYDVQQDEx9qdWljZXByb2R1Y2UuY29ycC5u +ZTEueWFob28uY29tMSQwIgYJKoZIhvcNAQkBFhVwZXJzaWEuYXppekB5YWhvby5j +b20wHhcNMTgxMDE1MTU1NjMzWhcNMjgxMDEyMTU1NjMzWjByMQswCQYDVQQGEwJV +UzELMAkGA1UECAwCSUwxEjAQBgNVBAcMCUNoYW1wYWlnbjEQMA4GA1UECgwHRXhh +bXBsZTEQMA4GA1UEAwwHZm9vLmNvbTEeMBwGCSqGSIb3DQEJARYPYm9iQGV4YW1w +bGUuY29tMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAyYXjrK9KOAtE +FXpaPL4mJfyI1r5I+GOmKI5zvxPU+R0n1sZEYpEU/F+8qrPQpa4zXGKDzTUD+b4J +dagCHosVK22WZJXGd5BxfzF3c7mFf/7k92H1q7Dk3X23LumnR7Qa/0HNMPRkmwaa +eNmQC8c42doWWaudV5ir3M+ef4Jv/WL5RhK877D85Ho3R+lNRini6hmmTqpFezdi +eMypicoj88K5kf/Mu5PvYwx0F/gNsGuYGogDBnSDPGk1fl7DTQL3rKb18l+1IcMQ +7MNHq9Bi7+LLGAq7uYRfrVHI3jgh8UpwqBAuOW9sw2RwEyy46+wCCedk3EqNZE2k +4qwDgIh0SwIDAQABMA0GCSqGSIb3DQEBCwUAA4GBALr2gm+KgveEcTXwURM0wxJC +m0yOR8w6MX8fxHKaekhJH1U84G64Ub0gbn2beOdLBQkG+4czLiOOOgyeukPaJJ81 +od2ooE7DrGUPGnbHYxW/70EtVF5nQEctcqpKNF/d04mVKrqI90919MJSxJ5KedHK +2H11+gUPwDWy/mAwJzEJ +-----END CERTIFICATE----- diff --git a/tests/gold_tests/pluginTest/traffic_dump/ssl/signer.pem b/tests/gold_tests/pluginTest/traffic_dump/ssl/signer.pem new file mode 100644 index 00000000000..111cd079718 --- /dev/null +++ b/tests/gold_tests/pluginTest/traffic_dump/ssl/signer.pem @@ -0,0 +1,17 @@ +-----BEGIN CERTIFICATE----- +MIICszCCAhwCCQD4jSkztmlO1TANBgkqhkiG9w0BAQsFADCBnTELMAkGA1UEBhMC +VVMxCzAJBgNVBAgTAklMMRIwEAYDVQQHEwlDaGFtcGFpZ24xDjAMBgNVBAoTBVlh +aG9vMQ0wCwYDVQQLEwRFZGdlMSgwJgYDVQQDEx9qdWljZXByb2R1Y2UuY29ycC5u +ZTEueWFob28uY29tMSQwIgYJKoZIhvcNAQkBFhVwZXJzaWEuYXppekB5YWhvby5j +b20wHhcNMTcwODI4MDM0NDQ1WhcNMjcwODI2MDM0NDQ1WjCBnTELMAkGA1UEBhMC +VVMxCzAJBgNVBAgTAklMMRIwEAYDVQQHEwlDaGFtcGFpZ24xDjAMBgNVBAoTBVlh +aG9vMQ0wCwYDVQQLEwRFZGdlMSgwJgYDVQQDEx9qdWljZXByb2R1Y2UuY29ycC5u +ZTEueWFob28uY29tMSQwIgYJKoZIhvcNAQkBFhVwZXJzaWEuYXppekB5YWhvby5j +b20wgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBANYwc6JQX45GZmMDEjwxYT11 +uVvuBBInfpYJeU8WIXHrKcX5LUSRcBikiKnlfSnMNRohsu6TElQACc60wQ7Q8KDE +lBSsS1FaHzCIl1t1AkXRmz/1H65JSBvrV/6Z1NC+Gp58EbH7Gul8ByC1xaJm5ID1 +Dd++kOPlY5ZI9ZcFS7HLAgMBAAEwDQYJKoZIhvcNAQELBQADgYEATX7975NdhIbJ +glda+sXI9a86GgOpiuKO+vKubRJQZA+UlPf2vHEONjC2+7Y1aZvZYaKYL74vxGky +zkgp6ANSPl45lqD632x0e1Z7vzW5TkqK1JB2/xH2WgDcQZmP0FuQHzVNs4GjghDr +HCp1+sQDhfPB4aLmLFeyN0TkhdH1N3M= +-----END CERTIFICATE----- diff --git a/tests/gold_tests/pluginTest/traffic_dump/traffic_dump.test.py b/tests/gold_tests/pluginTest/traffic_dump/traffic_dump.test.py new file mode 100644 index 00000000000..7ca1ef0df2a --- /dev/null +++ b/tests/gold_tests/pluginTest/traffic_dump/traffic_dump.test.py @@ -0,0 +1,323 @@ +""" +Verify traffic_dump functionality. +""" +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +Test.Summary = ''' +Verify traffic_dump functionality. +''' + +Test.SkipUnless( + Condition.PluginExists('traffic_dump.so'), +) + +# Configure the origin server. +server = Test.MakeOriginServer("server") + +request_header = {"headers": "GET /one HTTP/1.1\r\n" + "Host: www.example.com\r\nContent-Length: 0\r\n\r\n", + "timestamp": "1469733493.993", "body": ""} +response_header = {"headers": "HTTP/1.1 200 OK" + "\r\nConnection: close\r\nContent-Length: 0" + "\r\nSet-Cookie: classified_not_for_logging\r\n\r\n", + "timestamp": "1469733493.993", "body": ""} +server.addResponse("sessionfile.log", request_header, response_header) +request_header = {"headers": "GET /two HTTP/1.1\r\n" + "Host: www.example.com\r\nContent-Length: 0\r\n\r\n", + "timestamp": "1469733493.993", "body": ""} +response_header = {"headers": "HTTP/1.1 200 OK" + "\r\nConnection: close\r\nContent-Length: 0" + "\r\nSet-Cookie: classified_not_for_logging\r\n\r\n", + "timestamp": "1469733493.993", "body": ""} +server.addResponse("sessionfile.log", request_header, response_header) +request_header = {"headers": "GET /three HTTP/1.1\r\n" + "Host: www.example.com\r\nContent-Length: 0\r\n\r\n", + "timestamp": "1469733493.993", "body": ""} +response_header = {"headers": "HTTP/1.1 200 OK" + "\r\nConnection: close\r\nContent-Length: 0" + "\r\nSet-Cookie: classified_not_for_logging\r\n\r\n", + "timestamp": "1469733493.993", "body": ""} +server.addResponse("sessionfile.log", request_header, response_header) +request_header = {"headers": "GET /post_with_body HTTP/1.1\r\n" + "Host: www.example.com\r\nContent-Length: 0\r\n\r\n", + "timestamp": "1469733493.993", "body": ""} +response_header = {"headers": "HTTP/1.1 200 OK" + "\r\nConnection: close\r\nContent-Length: 0\r\n\r\n", + "timestamp": "1469733493.993", "body": ""} +server.addResponse("sessionfile.log", request_header, response_header) +request_header = {"headers": "GET /cache_test HTTP/1.1\r\n" + "Host: www.example.com\r\nContent-Length: 0\r\n\r\n", + "timestamp": "1469733493.993", "body": ""} +response_header = {"headers": "HTTP/1.1 200 OK" + "\r\nConnection: close\r\nCache-Control: max-age=300\r\n" + "Content-Length: 4\r\n\r\n", + "timestamp": "1469733493.993", "body": "1234"} +server.addResponse("sessionfile.log", request_header, response_header) +request_header = {"headers": "GET /first HTTP/1.1\r\n" + "Host: www.example.com\r\nContent-Length: 0\r\n\r\n", + "timestamp": "1469733493.993", "body": ""} +response_header = {"headers": "HTTP/1.1 200 OK" + "\r\nConnection: close\r\nContent-Length: 0\r\n\r\n", + "timestamp": "1469733493.993", "body": ""} +server.addResponse("sessionfile.log", request_header, response_header) +request_header = {"headers": "GET /second HTTP/1.1\r\n" + "Host: www.example.com\r\nContent-Length: 0\r\n\r\n", + "timestamp": "1469733493.993", "body": ""} +response_header = {"headers": "HTTP/1.1 200 OK" + "\r\nConnection: close\r\nContent-Length: 0\r\n\r\n", + "timestamp": "1469733493.993", "body": ""} +server.addResponse("sessionfile.log", request_header, response_header) + +# Define ATS and configure it. +ts = Test.MakeATSProcess("ts") +replay_dir = os.path.join(ts.RunDirectory, "ts", "log") +ts.Disk.records_config.update({ + 'proxy.config.diags.debug.enabled': 1, + 'proxy.config.diags.debug.tags': 'traffic_dump', + 'proxy.config.http.cache.http': 1, + 'proxy.config.http.wait_for_cache': 1, + 'proxy.config.http.insert_age_in_response': 0, +}) +ts.Disk.remap_config.AddLine( + 'map / http://127.0.0.1:{0}'.format(server.Variables.Port) +) +# Configure traffic_dump. +ts.Disk.plugin_config.AddLine( + 'traffic_dump.so --logdir {0} --sample 1 --limit 1000000000 ' + '--sensitive-fields "cookie,set-cookie,x-request-1,x-request-2"'.format(replay_dir) +) +# Configure logging of transactions. This is helpful for the cache test below. +ts.Disk.logging_yaml.AddLines( + ''' +logging: + formats: + - name: basic + format: "%: Read result: %:%:%, Write result: %" + logs: + - filename: transactions + format: basic +'''.split('\n')) + +# Set up trafficserver expectations. +ts.Disk.diags_log.Content = Testers.ContainsExpression( + "loading plugin.*traffic_dump.so", + "Verify the traffic_dump plugin got loaded.") +ts.Streams.stderr = Testers.ContainsExpression( + "Initialized with log directory: {0}".format(replay_dir), + "Verify traffic_dump initialized with the configured directory.") +ts.Streams.stderr += Testers.ContainsExpression( + "Initialized with sample pool size 1 bytes and disk limit 1000000000 bytes", + "Verify traffic_dump initialized with the configured disk limit.") +ts.Streams.stderr += Testers.ContainsExpression( + "Finish a session with log file of.*bytes", + "Verify traffic_dump sees the end of sessions and accounts for it.") + +# Set up the json replay file expectations. +replay_file_session_1 = os.path.join(replay_dir, "127", "0000000000000000") +ts.Disk.File(replay_file_session_1, exists=True) +replay_file_session_2 = os.path.join(replay_dir, "127", "0000000000000001") +ts.Disk.File(replay_file_session_2, exists=True) +replay_file_session_3 = os.path.join(replay_dir, "127", "0000000000000002") +ts.Disk.File(replay_file_session_3, exists=True) +replay_file_session_4 = os.path.join(replay_dir, "127", "0000000000000003") +ts.Disk.File(replay_file_session_4, exists=True) +replay_file_session_5 = os.path.join(replay_dir, "127", "0000000000000004") +ts.Disk.File(replay_file_session_5, exists=True) +replay_file_session_6 = os.path.join(replay_dir, "127", "0000000000000005") +ts.Disk.File(replay_file_session_6, exists=True) +replay_file_session_7 = os.path.join(replay_dir, "127", "0000000000000006") +ts.Disk.File(replay_file_session_7, exists=True) + +# +# Test 1: Verify the correct behavior of two transactions across two sessions. +# + +# Execute the first transaction. +tr = Test.AddTestRun("First transaction") + +tr.Processes.Default.StartBefore(server, ready=When.PortOpen(server.Variables.Port)) +tr.Processes.Default.StartBefore(Test.Processes.ts) +tr.Processes.Default.Command = \ + ('curl --http1.1 http://127.0.0.1:{0}/one -H"Cookie: donotlogthis" ' + '-H"Host: www.example.com" -H"X-Request-1: ultra_sensitive" --verbose'.format( + ts.Variables.port)) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Streams.stderr = "gold/200.gold" +tr.StillRunningAfter = server +tr.StillRunningAfter = ts +http_protocols = "tcp,ipv4" + +# Execute the second transaction. +tr = Test.AddTestRun("Second transaction") +tr.Processes.Default.Command = \ + ('curl http://127.0.0.1:{0}/two -H"Host: www.example.com" ' + '-H"X-Request-2: also_very_sensitive" --verbose'.format( + ts.Variables.port)) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Streams.stderr = "gold/200.gold" +tr.StillRunningAfter = server +tr.StillRunningAfter = ts + +# Verify the properties of the replay file for the first transaction. +tr = Test.AddTestRun("Verify the json content of the first session") +verify_replay = "verify_replay.py" +sensitive_fields_arg = ( + "--sensitive-fields cookie " + "--sensitive-fields set-cookie " + "--sensitive-fields x-request-1 " + "--sensitive-fields x-request-2 ") +tr.Setup.CopyAs(verify_replay, Test.RunDirectory) +tr.Processes.Default.Command = 'python3 {0} {1} {2} {3} --client-protocols "{4}"'.format( + verify_replay, + os.path.join(Test.Variables.AtsTestToolsDir, 'lib', 'replay_schema.json'), + replay_file_session_1, + sensitive_fields_arg, + http_protocols) +tr.Processes.Default.ReturnCode = 0 +tr.StillRunningAfter = server +tr.StillRunningAfter = ts + +# Verify the properties of the replay file for the second transaction. +tr = Test.AddTestRun("Verify the json content of the second session") +tr.Setup.CopyAs(verify_replay, Test.RunDirectory) +tr.Processes.Default.Command = "python3 {0} {1} {2} {3} --request-target '/two'".format( + verify_replay, + os.path.join(Test.Variables.AtsTestToolsDir, 'lib', 'replay_schema.json'), + replay_file_session_2, + sensitive_fields_arg) +tr.Processes.Default.ReturnCode = 0 +tr.StillRunningAfter = server +tr.StillRunningAfter = ts + +# +# Test 2: Verify the correct behavior of an explicit path in the request line. +# + +# Verify that an explicit path in the request line is recorded. +tr = Test.AddTestRun("Make a request with an explicit target.") +request_target = "http://localhost:{0}/candy".format(ts.Variables.port) +tr.Processes.Default.Command = ( + 'curl --request-target "{0}" ' + 'http://127.0.0.1:{1}/three -H"Host: www.example.com" --verbose'.format( + request_target, ts.Variables.port)) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Streams.stderr = "gold/explicit_target.gold" +tr.StillRunningAfter = server +tr.StillRunningAfter = ts + +tr = Test.AddTestRun("Verify the replay file has the explicit target.") +tr.Setup.CopyAs(verify_replay, Test.RunDirectory) + +tr.Processes.Default.Command = "python3 {0} {1} {2} {3} --request-target '{4}'".format( + verify_replay, + os.path.join(Test.Variables.AtsTestToolsDir, 'lib', 'replay_schema.json'), + replay_file_session_3, + sensitive_fields_arg, + request_target) +tr.Processes.Default.ReturnCode = 0 +tr.StillRunningAfter = server +tr.StillRunningAfter = ts + +# +# Test 3: Verify correct handling of a POST with body data. +# + +tr = Test.AddTestRun("Make a POST request with a body.") +request_target = "http://localhost:{0}/post_with_body".format(ts.Variables.port) + +# Send the replay file as the request body because it is conveniently already +# in the test run directory. +tr.Processes.Default.Command = ( + 'curl --data-binary @{0} --request-target "{1}" ' + 'http://127.0.0.1:{2} -H"Host: www.example.com" --verbose'.format( + verify_replay, request_target, ts.Variables.port)) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Streams.stderr = "gold/post_with_body.gold" +tr.StillRunningAfter = server +tr.StillRunningAfter = ts + +tr = Test.AddTestRun("Verify the client-request size node has the expected value.") +tr.Setup.CopyAs(verify_replay, Test.RunDirectory) + +size_of_verify_replay_file = os.path.getsize(os.path.join(Test.TestDirectory, verify_replay)) +tr.Processes.Default.Command = \ + "python3 {0} {1} {2} {3} --client-request-size {4}".format( + verify_replay, + os.path.join(Test.Variables.AtsTestToolsDir, 'lib', 'replay_schema.json'), + replay_file_session_4, + sensitive_fields_arg, + size_of_verify_replay_file) +tr.Processes.Default.ReturnCode = 0 +tr.StillRunningAfter = server +tr.StillRunningAfter = ts + +# +# Test 4: Verify correct handling of a response produced out of the cache. +# +tr = Test.AddTestRun("Make a request for an uncached object.") +tr.Processes.Default.Command = \ + ('curl --http1.1 http://127.0.0.1:{0}/cache_test -H"Host: www.example.com" --verbose'.format( + ts.Variables.port)) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Streams.stderr = "gold/4_byte_response_body.gold" +tr.StillRunningAfter = server +tr.StillRunningAfter = ts + +tr = Test.AddTestRun("Repeat the previous request: should be cached now.") +tr.Processes.Default.Command = \ + ('curl --http1.1 http://127.0.0.1:{0}/cache_test -H"Host: www.example.com" --verbose'.format( + ts.Variables.port)) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Streams.stderr = "gold/4_byte_response_body.gold" +tr.StillRunningAfter = server +tr.StillRunningAfter = ts + +tr = Test.AddTestRun("Verify that the cached response's replay file looks appropriate.") +tr.Setup.CopyAs(verify_replay, Test.RunDirectory) +tr.Processes.Default.Command = 'python3 {0} {1} {2} --client-protocols "{3}"'.format( + verify_replay, + os.path.join(Test.Variables.AtsTestToolsDir, 'lib', 'replay_schema.json'), + replay_file_session_6, + http_protocols) +tr.Processes.Default.ReturnCode = 0 +tr.StillRunningAfter = server +tr.StillRunningAfter = ts + +# +# Test 5: Verify correct handling of two transactions in a session. +# +tr = Test.AddTestRun("Conduct two transactions in the same session.") +tr.Processes.Default.Command = \ + ('curl --http1.1 http://127.0.0.1:{0}/first -H"Host: www.example.com" --verbose --next ' + 'curl --http1.1 http://127.0.0.1:{0}/second -H"Host: www.example.com" --verbose' + .format(ts.Variables.port)) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Streams.stderr = "gold/two_transactions.gold" +tr.StillRunningAfter = server +tr.StillRunningAfter = ts + +tr = Test.AddTestRun("Verify that the dump file can be read.") +tr.Setup.CopyAs(verify_replay, Test.RunDirectory) +tr.Processes.Default.Command = 'python3 {0} {1} {2} --client-protocols "{3}"'.format( + verify_replay, + os.path.join(Test.Variables.AtsTestToolsDir, 'lib', 'replay_schema.json'), + replay_file_session_7, + http_protocols) +tr.Processes.Default.ReturnCode = 0 +tr.StillRunningAfter = server +tr.StillRunningAfter = ts diff --git a/tests/gold_tests/pluginTest/traffic_dump/traffic_dump_sni_filter.test.py b/tests/gold_tests/pluginTest/traffic_dump/traffic_dump_sni_filter.test.py new file mode 100644 index 00000000000..5d1d01b8f76 --- /dev/null +++ b/tests/gold_tests/pluginTest/traffic_dump/traffic_dump_sni_filter.test.py @@ -0,0 +1,163 @@ +""" +Verify traffic_dump functionality. +""" +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +Test.Summary = ''' +Verify traffic_dump functionality. +''' + +Test.SkipUnless( + Condition.PluginExists('traffic_dump.so'), +) + +# Configure the origin server. +server = Test.MakeOriginServer("server") + +request_header = {"headers": "GET / HTTP/1.1\r\n" + "Host: www.example.com\r\nContent-Length: 0\r\n\r\n", + "timestamp": "1469733493.993", "body": ""} +response_header = {"headers": "HTTP/1.1 200 OK" + "\r\nConnection: close\r\nContent-Length: 0" + "\r\nSet-Cookie: classified_not_for_logging\r\n\r\n", + "timestamp": "1469733493.993", "body": ""} +server.addResponse("sessionfile.log", request_header, response_header) + +# Define ATS and configure +ts = Test.MakeATSProcess("ts", select_ports=True, enable_tls=True) +replay_dir = os.path.join(ts.RunDirectory, "ts", "log") + +ts.addSSLfile("ssl/server.pem") +ts.addSSLfile("ssl/server.key") +ts.addSSLfile("ssl/signer.pem") + +ts.Disk.records_config.update({ + 'proxy.config.diags.debug.enabled': 1, + 'proxy.config.diags.debug.tags': 'traffic_dump', + + 'proxy.config.ssl.server.cert.path': '{0}'.format(ts.Variables.SSLDir), + 'proxy.config.ssl.server.private_key.path': '{0}'.format(ts.Variables.SSLDir), + 'proxy.config.ssl.client.verify.server': 0, + 'proxy.config.url_remap.pristine_host_hdr': 1, + 'proxy.config.ssl.CA.cert.filename': '{0}/signer.pem'.format(ts.Variables.SSLDir), + 'proxy.config.exec_thread.autoconfig.scale': 1.0, + 'proxy.config.http.host_sni_policy': 2, + 'proxy.config.ssl.TLSv1_3': 0, +}) + +ts.Disk.ssl_multicert_config.AddLine( + 'dest_ip=* ssl_cert_name=server.pem ssl_key_name=server.key' +) + +ts.Disk.remap_config.AddLine( + 'map / http://127.0.0.1:{0}'.format(server.Variables.Port) +) + +ts.Disk.sni_yaml.AddLines([ + 'sni:', + '- fqdn: boblite', + ' verify_client: STRICT', + ' host_sni_policy: PERMISSIVE', + '- fqdn: bob', + ' verify_client: STRICT', +]) + +# Configure traffic_dump to filter to only dump with connections with SNI bob. +sni_filter = "bob" +ts.Disk.plugin_config.AddLine( + 'traffic_dump.so --logdir {0} --sample 1 --limit 1000000000 ' + '--sni-filter "{1}"'.format(replay_dir, sni_filter) +) + +# Set up trafficserver expectations. +ts.Streams.stderr += Testers.ContainsExpression( + "Filtering to only dump connections with SNI: {}".format(sni_filter), + "Verify filtering for the expected SNI.") + +ts.Streams.stderr += Testers.ContainsExpression( + "Ignore HTTPS session with non-filtered SNI: dave", + "Verify that the non-desired SNI session was filtered out.") + +# Set up the json replay file expectations. +replay_file_session_1 = os.path.join(replay_dir, "127", "0000000000000000") +ts.Disk.File(replay_file_session_1, exists=True) + +# The second session should be filtered out because it doesn't have the +# expected SNI (note exists is set to False). +replay_file_session_2 = os.path.join(replay_dir, "127", "0000000000000001") +ts.Disk.File(replay_file_session_2, exists=False) + +# The third session should also be filtered out because it doesn't have any +# SNI (note exists is set to False). +replay_file_session_2 = os.path.join(replay_dir, "127", "0000000000000002") +ts.Disk.File(replay_file_session_2, exists=False) + +# +# Test 1: Verify dumping a session with the desired SNI and not dumping +# the session with the other SNI. +# + +# Execute the first transaction with an SNI of bob. +tr = Test.AddTestRun("Verify dumping of a session with the filtered SNI") +tr.Setup.Copy("ssl/signed-foo.pem") +tr.Setup.Copy("ssl/signed-foo.key") +tr.Processes.Default.StartBefore(server, ready=When.PortOpen(server.Variables.Port)) +tr.Processes.Default.StartBefore(Test.Processes.ts) +tr.Processes.Default.Command = \ + ('curl --http2 --tls-max 1.2 -k -H"Host: bob" --resolve "bob:{0}:127.0.0.1" ' + '--cert ./signed-foo.pem --key ./signed-foo.key --verbose https://bob:{0}'.format(ts.Variables.ssl_port)) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Streams.stderr = "gold/200_sni_bob.gold" +tr.StillRunningAfter = server +tr.StillRunningAfter = ts +session_1_protocols = "h2,tls/1.2,tcp,ipv4" +session_1_tls_features = 'sni:bob' + +# Execute the second transaction with an SNI of dave. +tr = Test.AddTestRun("Verify that a session of a different SNI is not dumped.") +tr.Processes.Default.Command = \ + ('curl --tls-max 1.2 -k -H"Host: dave" --resolve "dave:{0}:127.0.0.1" ' + '--cert ./signed-foo.pem --key ./signed-foo.key --verbose https://dave:{0}'.format(ts.Variables.ssl_port)) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Streams.stderr = "gold/200_sni_dave.gold" +tr.StillRunningAfter = server +tr.StillRunningAfter = ts + +# Execute the third transaction without any SNI. +tr = Test.AddTestRun("Verify that a session of a non-existent SNI is not dumped.") +tr.Processes.Default.Command = \ + ('curl --tls-max 1.2 -k -H"Host: bob"' + '--cert ./signed-foo.pem --key ./signed-foo.key --verbose https://127.0.0.1:{0}'.format(ts.Variables.ssl_port)) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Streams.stderr = "gold/200_bob_no_sni.gold" +tr.StillRunningAfter = server +tr.StillRunningAfter = ts + +# Verify the properties of the replay file for the dumped transaction. +tr = Test.AddTestRun("Verify the json content of the first session") +verify_replay = "verify_replay.py" +tr.Setup.CopyAs(verify_replay, Test.RunDirectory) +tr.Processes.Default.Command = 'python3 {0} {1} {2} --client-protocols "{3}" --client-tls-features "{4}"'.format( + verify_replay, + os.path.join(Test.Variables.AtsTestToolsDir, 'lib', 'replay_schema.json'), + replay_file_session_1, + session_1_protocols, + session_1_tls_features) +tr.Processes.Default.ReturnCode = 0 +tr.StillRunningAfter = server +tr.StillRunningAfter = ts diff --git a/tests/gold_tests/pluginTest/traffic_dump/verify_replay.py b/tests/gold_tests/pluginTest/traffic_dump/verify_replay.py new file mode 100644 index 00000000000..a2260b191f0 --- /dev/null +++ b/tests/gold_tests/pluginTest/traffic_dump/verify_replay.py @@ -0,0 +1,237 @@ +""" +Verify that a given JSON replay file fulfills basic expectations. +""" +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import json +import jsonschema +import sys + + +expected_sensitive_value = \ + '''0000000 0000001 0000002 0000003 0000004 0000005 0000006 0000007 0000008 0000009 000000a 000000b 000000c 000000d ''' \ + '''000000e 000000f 0000010 0000011 0000012 0000013 0000014 0000015 0000016 0000017 0000018 0000019 000001a 000001b ''' \ + '''000001c 000001d 000001e 000001f 0000020 0000021 0000022 0000023 0000024 0000025 0000026 0000027 0000028 0000029 ''' \ + '''000002a 000002b 000002c 000002d 000002e 000002f 0000030 0000031 0000032 0000033 0000034 0000035 0000036 0000037 ''' \ + '''0000038 0000039 000003a 000003b 000003c 000003d 000003e 000003f 0000040 0000041 0000042 0000043 0000044 0000045 ''' \ + '''0000046 0000047 0000048 0000049 000004a 000004b 000004c 000004d 000004e 000004f 0000050 0000051 0000052 0000053 ''' \ + '''0000054 0000055 0000056 0000057 0000058 0000059 000005a 000005b 000005c 000005d 000005e 000005f 0000060 0000061 ''' \ + '''0000062 0000063 0000064 0000065 0000066 0000067 0000068 0000069 000006a 000006b 000006c 000006d 000006e 000006f ''' \ + '''0000070 0000071 0000072 0000073 0000074 0000075 0000076 0000077 0000078 0000079 000007a 000007b 000007c 000007d ''' \ + '''000007e 000007f 0000080 0000081 0000082 0000083 0000084 0000085 0000086 0000087 0000088 0000089 000008a 000008b ''' \ + '''000008c 000008d 000008e 000008f 0000090 0000091 0000092 0000093 0000094 0000095 0000096 0000097 0000098 0000099 ''' \ + '''000009a 000009b 000009c 000009d 000009e 000009f 00000a0 00000a1 00000a2 00000a3 00000a4 00000a5 00000a6 00000a7 ''' \ + '''00000a8 00000a9 00000aa 00000ab 00000ac 00000ad 00000ae 00000af 00000b0 00000b1 00000b2 00000b3 00000b4 00000b5 ''' \ + '''00000b6 00000b7 00000b8 00000b9 00000ba 00000bb 00000bc 00000bd 00000be 00000bf 00000c0 00000c1 00000c2 00000c3 ''' \ + '''00000c4 00000c5 00000c6 00000c7 00000c8 00000c9 00000ca 00000cb 00000cc 00000cd 00000ce 00000cf 00000d0 00000d1 ''' \ + '''00000d2 00000d3 00000d4 00000d5 00000d6 00000d7 00000d8 00000d9 00000da 00000db 00000dc 00000dd 00000de 00000df ''' \ + '''00000e0 00000e1 00000e2 00000e3 00000e4 00000e5 00000e6 00000e7 00000e8 00000e9 00000ea 00000eb 00000ec 00000ed ''' \ + '''00000ee 00000ef 00000f0 00000f1 00000f2 00000f3 00000f4 00000f5 00000f6 00000f7 00000f8 00000f9 00000fa 00000fb ''' \ + '''00000fc 00000fd 00000fe 00000ff''' + + +def validate_json(schema_json, replay_json): + """ + Validate the replay file against the provided schema. + """ + try: + jsonschema.validate(instance=replay_json, schema=schema_json) + except jsonschema.ValidationError as e: + print("The replay file does not validate against the schema: {}".format(e)) + return False + else: + return True + + +def verify_there_was_a_transaction(replay_json): + """ + Verify that the replay file has a sensible looking transaction. + """ + try: + transactions = replay_json['sessions'][0]['transactions'] + except KeyError: + print("The replay file did not have transactions in it.") + return False + + if len(transactions) < 1: + print("There are no transactions in the replay file.") + return False + transaction = transactions[0] + if not ('client-request' in transaction and 'proxy-response' in transaction): + print("There was not request and response in the transaction of the replay file.") + return False + + return True + + +def verify_request_target(replay_json, request_target): + """ + Verify that the 'url' element of the first transaction contains the request target. + """ + try: + url = replay_json['sessions'][0]['transactions'][0]['client-request']['url'] + except KeyError: + print("The replay file did not have a first transaction with a url element.") + return False + + if url != request_target: + print("Mismatched request target. Expected: {}, received: {}".format(request_target, url)) + return False + return True + + +def verify_client_request_size(replay_json, client_request_size): + """ + Verify that the 'url' element of the first transaction contains the request target. + """ + try: + size = int(replay_json['sessions'][0]['transactions'][0]['client-request']['content']['size']) + except KeyError: + print("The replay file did not have content size element in the first client-request.") + return False + + if size != client_request_size: + print("Mismatched client-request request size. Expected: {}, received: {}".format( + client_request_size, size)) + return False + return True + + +def verify_sensitive_fields_not_dumped(replay_json, sensitive_fields): + """ + Verify that all of the cookie fields have the expected value. + """ + message_types = ['client-request', 'proxy-request', 'server-response', 'proxy-response'] + try: + for session in replay_json['sessions']: + for transaction in session['transactions']: + for message_type in transaction: + if message_type not in message_types: + continue + message = transaction[message_type] + for field in message['headers']['fields']: + field_name = field[0].lower() + if field_name in sensitive_fields: + field_value = field[1] + if field_value not in expected_sensitive_value: + print("Found an unexpected cookie: {}: {}".format(field[0], field[1])) + return False + + except KeyError: + print("Could not find headers in the replay file.") + return False + + return True + + +def verify_client_protocols(replay_json, expected_protocol_features): + expected_protocols_list = sorted(expected_protocol_features.split(',')) + try: + protocol_node = replay_json['sessions'][0]['protocol'] + protocol_list = sorted(protocol_node.copy()) + if protocol_list == expected_protocols_list: + return True + else: + print('Unexpected protocol stack. Expected: "{}", found: "{}".'.format( + ','.join(expected_protocols_list), ','.join(protocol_list))) + return False + except KeyError: + print("Could not find client protocol stack node in the replay file.") + return False + + +def verify_client_tls_features(replay_json, expected_tls_features): + try: + session = replay_json['sessions'][0] + for expected_tls_feature in expected_tls_features.split(','): + expected_key, expected_value = expected_tls_feature.split(':') + tls_features = session['tls'] + try: + return tls_features[expected_key] == expected_value + except KeyError: + print("Could not find client tls feature in the replay file: {}".format(expected_key)) + return False + except KeyError: + print("Could not find client tls node in the replay file.") + return False + + +def parse_args(): + parser = argparse.ArgumentParser() + parser.add_argument("schema_file", + type=argparse.FileType('r'), + help="The schema in which to interpret validate the replay file.") + parser.add_argument("replay_file", + type=argparse.FileType('r'), + help="The replay file to validate.") + parser.add_argument("--request-target", + help="The request target ('url' element) to expect in the replay file.") + parser.add_argument("--client-request-size", + type=int, + help="The expected size value in the client-request node.") + parser.add_argument("--sensitive-fields", + action="append", + help="The fields that are considered sensitive and replaced with insensitive values.") + parser.add_argument("--client-protocols", + help="The comma-separated protocol features to expect for the client connection.") + parser.add_argument("--client-tls-features", + help="The TLS values to expect for the client connection.") + return parser.parse_args() + + +def main(): + args = parse_args() + + schema_json = json.load(args.schema_file) + replay_json = json.load(args.replay_file) + + if not validate_json(schema_json, replay_json): + return 1 + + # Verifying that there is a transaction in the replay file may seem + # unnecessary since the replay file validated against the schema. But a JSON + # file that doesn't have conflicting entry names will pass the schema. For + # instance, this passes against our replay schema: + # + # {"name": "Bob", "languages": ["English", "French"]} + # + # Thus we do the following sanity check to make sure that the replay file + # appears to have some transaction in it. + if not verify_there_was_a_transaction(replay_json): + return 1 + + if args.request_target and not verify_request_target(replay_json, args.request_target): + return 1 + + if args.client_request_size and not verify_client_request_size(replay_json, args.client_request_size): + return 1 + + if args.sensitive_fields and not verify_sensitive_fields_not_dumped(replay_json, args.sensitive_fields): + return 1 + + if args.client_protocols and not verify_client_protocols(replay_json, args.client_protocols): + return 1 + + if args.client_tls_features and not verify_client_tls_features(replay_json, args.client_tls_features): + return 1 + + return 0 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/tests/gold_tests/pluginTest/tsapi/tsapi.test.py b/tests/gold_tests/pluginTest/tsapi/tsapi.test.py index 65d07118eac..a1e82ba5b70 100644 --- a/tests/gold_tests/pluginTest/tsapi/tsapi.test.py +++ b/tests/gold_tests/pluginTest/tsapi/tsapi.test.py @@ -14,6 +14,9 @@ # See the License for the specific language governing permissions and # limitations under the License. +import os + + Test.Summary = ''' Test TS API. ''' @@ -29,8 +32,8 @@ server = Test.MakeOriginServer("server") request_header = { - "headers": "GET / HTTP/1.1\r\nHost: doesnotmatter\r\n\r\n", "timestamp": "1469733493.993", "body": "" } -response_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": "112233" } + "headers": "GET / HTTP/1.1\r\nHost: doesnotmatter\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +response_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": "112233"} server.addResponse("sessionlog.json", request_header, response_header) ts = Test.MakeATSProcess("ts", select_ports=True, enable_tls=True) @@ -59,7 +62,7 @@ "map https://myhost.test:{0} http://127.0.0.1:{0}".format(server.Variables.Port) ) -Test.PreparePlugin(Test.Variables.AtsTestToolsDir + '/plugins/test_tsapi.cc', ts) +Test.PrepareTestPlugin(os.path.join(Test.Variables.AtsTestPluginsDir, 'test_tsapi.so'), ts) tr = Test.AddTestRun() # Probe server port to check if ready. diff --git a/tests/gold_tests/pluginTest/uri_signing/config.json b/tests/gold_tests/pluginTest/uri_signing/config.json new file mode 100644 index 00000000000..e466cf740f5 --- /dev/null +++ b/tests/gold_tests/pluginTest/uri_signing/config.json @@ -0,0 +1,27 @@ +{ + "issuer": { + "id": "issuer", + "renewal_kid": "1", + "strip_token": true, + "auth_directives": [ + { + "auth": "allow", + "uri": "regex:.*crossdomain.xml" + } + ], + "keys": [ + { + "alg": "HS256", + "k": "SECRET00", + "kid": "0", + "kty": "oct" + }, + { + "alg": "HS256", + "k": "SECRET01", + "kid": "1", + "kty": "oct" + } + ] + } +} diff --git a/tests/gold_tests/pluginTest/uri_signing/gold/200.gold b/tests/gold_tests/pluginTest/uri_signing/gold/200.gold new file mode 100644 index 00000000000..afc41f9a171 --- /dev/null +++ b/tests/gold_tests/pluginTest/uri_signing/gold/200.gold @@ -0,0 +1,3 @@ +`` +< HTTP/1.1 200 OK +`` diff --git a/tests/gold_tests/pluginTest/uri_signing/gold/403.gold b/tests/gold_tests/pluginTest/uri_signing/gold/403.gold new file mode 100644 index 00000000000..1e2f71bb6f3 --- /dev/null +++ b/tests/gold_tests/pluginTest/uri_signing/gold/403.gold @@ -0,0 +1,3 @@ +`` +< HTTP/1.1 403 Forbidden +`` diff --git a/tests/gold_tests/pluginTest/uri_signing/run_sign.sh b/tests/gold_tests/pluginTest/uri_signing/run_sign.sh new file mode 100755 index 00000000000..375a83bfb7b --- /dev/null +++ b/tests/gold_tests/pluginTest/uri_signing/run_sign.sh @@ -0,0 +1,104 @@ +# Script to run sign.pl script. Single parameter is number 1 or greater selecting a set of script parameters. + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Preset arguments for generating tests +# +cmd_args () +{ +SELECT="$1" + +FUTURE="1923056084" # Monday, December 9, 2030 14:14:44 + +case "$SELECT" in +0) # future signing + echo "-c signer.json" + echo "-u http://somehost/someasset.ts" + echo "--exp=${FUTURE}" + echo "--key_index=0" + ;; +1) # expired signing (~1970) + echo "-c signer.json" + echo "-u http://somehost/someasset.ts" + echo "--key_index=0" + echo "--exp=1" + ;; +2) # future, second key + echo "-c signer.json" + echo "-u http://somehost/someasset.ts" + echo "--exp=${FUTURE}" + echo "--key_index=1" + ;; +3) + ;; +*) + echo "run_sign.sh: bad parameter" 1>&2 + exit 1 + ;; +esac +} + +# Find the path to the sign.pl script in the url_sig (source) directory. +# +find_cmd () +{ +local D T='..' +while [[ ! -d $T/.git ]] +do + if [[ ! -d $T/.. ]] ; then + echo "Working directory not in a git repo" 1>&2 + exit 1 + fi + T="$T/.." +done + +for D in $( find $T -name uri_signing -type d ) +do + if [[ -x $D/python_signer/uri_signer.py ]] ; then + echo "$D/python_signer/uri_signer.py" + return 0 + fi +done + +echo "cannot find uri_signer.py" 1>&2 +exit 1 +} + +# check for python-jose module +pip list | grep python-jose > /dev/null +status=$? +if [[ "$status" != 0 ]] ; then + echo "cannot find python-jose" 1>&2 + exit 1 +fi + +CMD=$( find_cmd ) +status=$? +if [[ "$status" != 0 ]] ; then + exit 1 +fi + + +ARGS=$( cmd_args "$1" ) +status=$? +if [[ "$status" != 0 ]] ; then + exit 1 +fi + +echo $CMD $ARGS + +$CMD $ARGS | tr ' ' '\n' | tail -1 diff --git a/tests/gold_tests/pluginTest/uri_signing/signer.json b/tests/gold_tests/pluginTest/uri_signing/signer.json new file mode 100644 index 00000000000..7c602f339ff --- /dev/null +++ b/tests/gold_tests/pluginTest/uri_signing/signer.json @@ -0,0 +1,18 @@ +{ + "iss": "issuer", + "token_lifetime": 90, + "keys": [ + { + "alg": "HS256", + "k": "SECRET00", + "kid": "0", + "kty": "oct" + }, + { + "alg": "HS256", + "k": "SECRET01", + "kid": "1", + "kty": "oct" + } + ] +} diff --git a/tests/gold_tests/pluginTest/uri_signing/uri_signing.test.py b/tests/gold_tests/pluginTest/uri_signing/uri_signing.test.py new file mode 100644 index 00000000000..3aa1aed2f70 --- /dev/null +++ b/tests/gold_tests/pluginTest/uri_signing/uri_signing.test.py @@ -0,0 +1,208 @@ +''' +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import subprocess +Test.Summary = ''' +Test uri_signing plugin +''' + +Test.ContinueOnFail = False + +# Skip if plugins not present. +Test.SkipUnless(Condition.PluginExists('uri_signing.so')) + +server = Test.MakeOriginServer("server") + +# Default origin test +req_header = {"headers": "GET / HTTP/1.1\r\nHost: www.example.com\r\n\r\n", + "timestamp": "1469733493.993", + "body": "", + } +res_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", + "timestamp": "1469733493.993", + "body": "", + } +server.addResponse("sessionfile.log", req_header, res_header) + +# Test case for normal +req_header = {"headers": + "GET /someasset.ts HTTP/1.1\r\nHost: somehost\r\n\r\n", + "timestamp": "1469733493.993", + "body": "", + } + +res_header = {"headers": + "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", + "timestamp": "1469733493.993", + "body": "somebody", + } + +server.addResponse("sessionfile.log", req_header, res_header) + +# Test case for crossdomain +req_header = {"headers": + "GET /crossdomain.xml HTTP/1.1\r\nHost: somehost\r\n\r\n", + "timestamp": "1469733493.993", + "body": "", + } + +res_header = {"headers": + "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", + "timestamp": "1469733493.993", + "body": "", + } + +server.addResponse("sessionfile.log", req_header, res_header) + +# http://user:password@host:port/path;params?query#fragment + +# Define default ATS +ts = Test.MakeATSProcess("ts") +#ts = Test.MakeATSProcess("ts", "traffic_server_valgrind.sh") + +ts.Disk.records_config.update({ + 'proxy.config.diags.debug.enabled': 1, + 'proxy.config.diags.debug.tags': 'uri_signing|http', + # 'proxy.config.diags.debug.tags': 'uri_signing', + 'proxy.config.http.cache.http': 0, # No cache +}) + +# Use unchanged incoming URL. +ts.Disk.remap_config.AddLine( + 'map http://somehost/ http://127.0.0.1:{}/'.format(server.Variables.Port) + + ' @plugin=uri_signing.so @pparam={}/config.json'.format(Test.RunDirectory) +) + +# Install configuration +ts.Setup.CopyAs('config.json', Test.RunDirectory) +ts.Setup.CopyAs('run_sign.sh', Test.RunDirectory) +ts.Setup.CopyAs('signer.json', Test.RunDirectory) +#ts.Setup.CopyAs('traffic_server_valgrind.sh', Test.RunDirectory) + +curl_and_args = 'curl -q -v -x localhost:{} '.format(ts.Variables.port) + +# 0 - reject unsigned request +tr = Test.AddTestRun("unsigned request") +ps = tr.Processes.Default +ps.StartBefore(ts) +ps.StartBefore(server, ready=When.PortOpen(server.Variables.Port)) +ps.Command = curl_and_args + 'http://somehost/someasset.ts' +ps.ReturnCode = 0 +ps.Streams.stderr = "gold/403.gold" +tr.StillRunningAfter = server +tr.StillRunningAfter = ts + +# 1 - accept a passthru request +tr = Test.AddTestRun("passthru request") +ps = tr.Processes.Default +ps.Command = curl_and_args + 'http://somehost/crossdomain.xml' +ps.ReturnCode = 0 +ps.Streams.stderr = "gold/200.gold" +tr.StillRunningAfter = server +tr.StillRunningAfter = ts + +# 2 - good token, signed "forever" (run_sign.sh 0) +tr = Test.AddTestRun("good signed") +ps = tr.Processes.Default +ps.Command = curl_and_args + '"http://somehost/someasset.ts?URISigningPackage=eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJpc3MiOiJpc3N1ZXIiLCJleHAiOjE5MjMwNTYwODR9.zw_wFQ-wvrWmfPLGj3hAUWn-GOHkiJZi2but4KV0paY"' +ps.ReturnCode = 0 +ps.Streams.stderr = "gold/200.gold" +tr.StillRunningAfter = server +tr.StillRunningAfter = ts + +# 3 - expired token (run_sign.sh 1) +tr = Test.AddTestRun("expired signed") +ps = tr.Processes.Default +ps.Command = curl_and_args + '"http://somehost/someasset.ts?URISigningPackage=eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJpc3MiOiJpc3N1ZXIiLCJleHAiOjF9.GkdlOPHQc6BqS4Q6x79GeYuVFO2zuGbaPZZsJfD6ir8"' +ps.ReturnCode = 0 +ps.Streams.stderr = "gold/403.gold" +tr.StillRunningAfter = server +tr.StillRunningAfter = ts + +# 4 - good token, different key (run_sign.sh 2) +tr = Test.AddTestRun("good token, second key") +ps = tr.Processes.Default +ps.Command = curl_and_args + '"http://somehost/someasset.ts?URISigningPackage=eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJpc3MiOiJpc3N1ZXIiLCJleHAiOjE5MjMwNTYwODR9.ozH4sNwgcOlTZT0l4RQlVCH_osxz9yI1HCBesEv-jYg"' +ps.ReturnCode = 0 +ps.Streams.stderr = "gold/200.gold" +tr.StillRunningAfter = server +tr.StillRunningAfter = ts + +# 5 - good token, inline +tr = Test.AddTestRun("good signed") +ps = tr.Processes.Default +ps.Command = curl_and_args + '"http://somehost/URISigningPackage=eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJpc3MiOiJpc3N1ZXIiLCJleHAiOjE5MjMwNTYwODR9.zw_wFQ-wvrWmfPLGj3hAUWn-GOHkiJZi2but4KV0paY/someasset.ts"' +ps.ReturnCode = 0 +ps.Streams.stderr = "gold/200.gold" +tr.StillRunningAfter = server +tr.StillRunningAfter = ts + +# 6 - expired token, inline +tr = Test.AddTestRun("expired signed") +ps = tr.Processes.Default +ps.Command = curl_and_args + '"http://somehost/URISigningPackage=eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJpc3MiOiJpc3N1ZXIiLCJleHAiOjF9.GkdlOPHQc6BqS4Q6x79GeYuVFO2zuGbaPZZsJfD6ir8/someasset.ts"' +ps.ReturnCode = 0 +ps.Streams.stderr = "gold/403.gold" +tr.StillRunningAfter = server +tr.StillRunningAfter = ts + +# 7 - good token, param +tr = Test.AddTestRun("good signed, param") +ps = tr.Processes.Default +ps.Command = curl_and_args + '"http://somehost/someasset.ts;URISigningPackage=eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJpc3MiOiJpc3N1ZXIiLCJleHAiOjE5MjMwNTYwODR9.zw_wFQ-wvrWmfPLGj3hAUWn-GOHkiJZi2but4KV0paY"' +ps.ReturnCode = 0 +ps.Streams.stderr = "gold/200.gold" +tr.StillRunningAfter = server +tr.StillRunningAfter = ts + +# 8 - expired token, param +tr = Test.AddTestRun("expired signed, param") +ps = tr.Processes.Default +ps.Command = curl_and_args + '"http://somehost/someasset.ts;URISigningPackage=eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJpc3MiOiJpc3N1ZXIiLCJleHAiOjF9.GkdlOPHQc6BqS4Q6x79GeYuVFO2zuGbaPZZsJfD6ir8"' +ps.ReturnCode = 0 +ps.Streams.stderr = "gold/403.gold" +tr.StillRunningAfter = server +tr.StillRunningAfter = ts + +# 9 - let's cookie this +tr = Test.AddTestRun("good signed cookie") +ps = tr.Processes.Default +ps.Command = curl_and_args + '"http://somehost/someasset.ts" -H "Cookie: URISigningPackage=eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJpc3MiOiJpc3N1ZXIiLCJleHAiOjE5MjMwNTYwODR9.zw_wFQ-wvrWmfPLGj3hAUWn-GOHkiJZi2but4KV0paY"' +ps.ReturnCode = 0 +ps.Streams.stderr = "gold/200.gold" +tr.StillRunningAfter = server +tr.StillRunningAfter = ts + +# 10 - expired cookie token +tr = Test.AddTestRun("expired signed cooked") +ps = tr.Processes.Default +ps.Command = curl_and_args + '"http://somehost/someasset.ts" -H "Cookie: URISigningPackage=eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJpc3MiOiJpc3N1ZXIiLCJleHAiOjF9.GkdlOPHQc6BqS4Q6x79GeYuVFO2zuGbaPZZsJfD6ir8"' +ps.ReturnCode = 0 +ps.Streams.stderr = "gold/403.gold" +tr.StillRunningAfter = server +tr.StillRunningAfter = ts + +# 9 - multiple cookies +tr = Test.AddTestRun("multiple cookies, expired then good") +ps = tr.Processes.Default +ps.Command = curl_and_args + '"http://somehost/someasset.ts" -H "Cookie: URISigningPackage=eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJpc3MiOiJpc3N1ZXIiLCJleHAiOjF9.GkdlOPHQc6BqS4Q6x79GeYuVFO2zuGbaPZZsJfD6ir8;URISigningPackage=eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJpc3MiOiJpc3N1ZXIiLCJleHAiOjE5MjMwNTYwODR9.zw_wFQ-wvrWmfPLGj3hAUWn-GOHkiJZi2but4KV0paY"' +ps.ReturnCode = 0 +ps.Streams.stderr = "gold/200.gold" +tr.StillRunningAfter = server +tr.StillRunningAfter = ts diff --git a/tests/gold_tests/pluginTest/url_sig/url_sig.gold b/tests/gold_tests/pluginTest/url_sig/url_sig.gold index 256898bea87..12043d8f3bc 100644 --- a/tests/gold_tests/pluginTest/url_sig/url_sig.gold +++ b/tests/gold_tests/pluginTest/url_sig/url_sig.gold @@ -1,15 +1,15 @@ -< HTTP/1.1 403 Forbidden -< HTTP/1.1 403 Forbidden -< HTTP/1.1 403 Forbidden -< HTTP/1.1 403 Forbidden -< HTTP/1.1 403 Forbidden -< HTTP/1.1 403 Forbidden -< HTTP/1.1 403 Forbidden -< HTTP/1.1 403 Forbidden -< HTTP/1.1 403 Forbidden -< HTTP/1.1 200 OK -< HTTP/1.1 200 OK -< HTTP/1.1 200 OK -< HTTP/1.1 200 OK -< HTTP/1.1 200 OK -< HTTP/1.1 200 OK +< HTTP/1.1 403 Forbidden +< HTTP/1.1 403 Forbidden +< HTTP/1.1 403 Forbidden +< HTTP/1.1 403 Forbidden +< HTTP/1.1 403 Forbidden +< HTTP/1.1 403 Forbidden +< HTTP/1.1 403 Forbidden +< HTTP/1.1 403 Forbidden +< HTTP/1.1 403 Forbidden +< HTTP/1.1 200 OK +< HTTP/1.1 200 OK +< HTTP/1.1 200 OK +< HTTP/1.1 200 OK +< HTTP/1.1 200 OK +< HTTP/1.1 200 OK diff --git a/tests/gold_tests/pluginTest/url_sig/url_sig.test.py b/tests/gold_tests/pluginTest/url_sig/url_sig.test.py index 3c96e898035..588f6d6b60a 100644 --- a/tests/gold_tests/pluginTest/url_sig/url_sig.test.py +++ b/tests/gold_tests/pluginTest/url_sig/url_sig.test.py @@ -18,8 +18,7 @@ import hashlib import hmac -import os -import subprocess + Test.Summary = ''' Test url_sig plugin ''' diff --git a/tests/gold_tests/pluginTest/xdebug/x_remap/four.in b/tests/gold_tests/pluginTest/xdebug/x_remap/four.in index 1982451bbe8..68420397501 100644 --- a/tests/gold_tests/pluginTest/xdebug/x_remap/four.in +++ b/tests/gold_tests/pluginTest/xdebug/x_remap/four.in @@ -1,4 +1,5 @@ GET /not_there HTTP/1.1 Host: two -X-Debug: X-Remap +X-Debug: X-Remap, Probe +Connection: close diff --git a/tests/gold_tests/pluginTest/xdebug/x_remap/fwd1.in b/tests/gold_tests/pluginTest/xdebug/x_remap/fwd1.in index c67c8ea5462..82f3fa52ecb 100644 --- a/tests/gold_tests/pluginTest/xdebug/x_remap/fwd1.in +++ b/tests/gold_tests/pluginTest/xdebug/x_remap/fwd1.in @@ -1,4 +1,5 @@ GET /argh HTTP/1.1 Host: two -X-Debug: X-Remap, fwd +X-Debug: X-Remap, fwd, Probe +Connection: close diff --git a/tests/gold_tests/pluginTest/xdebug/x_remap/fwd2.in b/tests/gold_tests/pluginTest/xdebug/x_remap/fwd2.in index 8cb9e7ce905..4f7ea91305f 100644 --- a/tests/gold_tests/pluginTest/xdebug/x_remap/fwd2.in +++ b/tests/gold_tests/pluginTest/xdebug/x_remap/fwd2.in @@ -1,4 +1,5 @@ GET /argh HTTP/1.1 Host: two -X-Debug: X-Remap, fwd=0 +X-Debug: X-Remap, fwd=0, probe +Connection: close diff --git a/tests/gold_tests/pluginTest/xdebug/x_remap/fwd3.in b/tests/gold_tests/pluginTest/xdebug/x_remap/fwd3.in index 47402f2b12a..42c187a9c9a 100644 --- a/tests/gold_tests/pluginTest/xdebug/x_remap/fwd3.in +++ b/tests/gold_tests/pluginTest/xdebug/x_remap/fwd3.in @@ -1,4 +1,5 @@ GET /argh HTTP/1.1 Host: two -X-Debug: X-Remap, fwd= 1 +X-Debug: X-Remap, fwd= 1, Probe +Connection: close diff --git a/tests/gold_tests/pluginTest/xdebug/x_remap/fwd4.in b/tests/gold_tests/pluginTest/xdebug/x_remap/fwd4.in index 75ca74b620d..db3ce259716 100644 --- a/tests/gold_tests/pluginTest/xdebug/x_remap/fwd4.in +++ b/tests/gold_tests/pluginTest/xdebug/x_remap/fwd4.in @@ -1,4 +1,5 @@ GET /argh HTTP/1.1 Host: two -X-Debug: X-Remap, fwd = 999999 +X-Debug: PROBE, X-Remap, fwd = 999999 +Connection: close diff --git a/tests/gold_tests/pluginTest/xdebug/x_remap/fwd5.in b/tests/gold_tests/pluginTest/xdebug/x_remap/fwd5.in index 3b3e9af5908..02b44e7c3de 100644 --- a/tests/gold_tests/pluginTest/xdebug/x_remap/fwd5.in +++ b/tests/gold_tests/pluginTest/xdebug/x_remap/fwd5.in @@ -1,4 +1,5 @@ GET /argh HTTP/1.1 Host: two -X-Debug: X-Remap, fwd = 999999xxx +X-Debug: X-Remap, fwd = 999999xxx, Probe +Connection: close diff --git a/tests/gold_tests/pluginTest/xdebug/x_remap/none.in b/tests/gold_tests/pluginTest/xdebug/x_remap/none.in index 6e5b7199081..a077b72790d 100644 --- a/tests/gold_tests/pluginTest/xdebug/x_remap/none.in +++ b/tests/gold_tests/pluginTest/xdebug/x_remap/none.in @@ -1,4 +1,5 @@ GET /argh HTTP/1.1 Host: none -X-Debug: X-Remap +X-Debug: X-Remap, Probe +Connection: close diff --git a/tests/gold_tests/pluginTest/xdebug/x_remap/one.in b/tests/gold_tests/pluginTest/xdebug/x_remap/one.in index 48306bed26c..6a35a87a16f 100644 --- a/tests/gold_tests/pluginTest/xdebug/x_remap/one.in +++ b/tests/gold_tests/pluginTest/xdebug/x_remap/one.in @@ -1,4 +1,5 @@ GET /argh HTTP/1.1 Host: one -X-Debug: X-Remap +X-Debug: X-Remap, probe +Connection: close diff --git a/tests/gold_tests/pluginTest/xdebug/x_remap/out.gold b/tests/gold_tests/pluginTest/xdebug/x_remap/out.gold index 0619bd2e5c8..6712754e1f8 100644 --- a/tests/gold_tests/pluginTest/xdebug/x_remap/out.gold +++ b/tests/gold_tests/pluginTest/xdebug/x_remap/out.gold @@ -1,6 +1,6 @@ HTTP/1.1 502 Cannot find server. Date: `` -Connection: keep-alive +Connection: close Server: ATS/`` Cache-Control: no-store Content-Type: text/html @@ -30,10 +30,46 @@ HTTP/1.1 200 OK Date: `` Age: `` Transfer-Encoding: chunked -Connection: keep-alive +Connection: close Server: ATS/`` X-Remap: from=http://one/, to=http://127.0.0.1:SERVER_PORT/ +`` +{'xDebugProbeAt' : '`` + 'captured':[{'type':'request', 'side':'client', 'headers': { + 'Start-Line' : 'GET http://127.0.0.1:SERVER_PORT/argh HTTP/1.1', + 'Host' : '127.0.0.1:SERVER_PORT', + 'Connection' : 'close' + }},{'type':'request', 'side':'server', 'headers': { + 'Start-Line' : 'GET /argh HTTP/1.1', + 'Host' : '127.0.0.1:SERVER_PORT', + 'Client-ip' : '127.0.0.1', + 'X-Forwarded-For' : '127.0.0.1', + 'Via' : 'http/1.1 traffic_server[`` + }} + ] +} +--- ATS xDebug Probe Injection Boundary --- + + +--- ATS xDebug Probe Injection Boundary --- + +{'xDebugProbeAt' : '`` + 'captured':[{'type':'response', 'side':'server', 'headers': { + 'Start-Line' : 'HTTP/1.1 200 OK', + 'Connection' : 'close', + 'Date' : '`` + }},{'type':'response', 'side':'client', 'headers': { + 'Start-Line' : 'HTTP/1.1 200 OK', + 'Date' : '`` + 'Age' : '`` + 'Transfer-Encoding' : 'chunked', + 'Connection' : 'close', + 'Server' : 'ATS/`` + 'X-Remap' : 'from=http://one/, to=http://127.0.0.1:SERVER_PORT/' + }} + ] +} 0 ====== @@ -41,10 +77,46 @@ HTTP/1.1 200 OK Date: `` Age: `` Transfer-Encoding: chunked -Connection: keep-alive +Connection: close Server: ATS/`` X-Remap: from=http://two/, to=http://127.0.0.1:SERVER_PORT/ +`` +{'xDebugProbeAt' : '`` + 'captured':[{'type':'request', 'side':'client', 'headers': { + 'Start-Line' : 'GET http://127.0.0.1:SERVER_PORT/argh HTTP/1.1', + 'Host' : '127.0.0.1:SERVER_PORT', + 'Connection' : 'close' + }},{'type':'request', 'side':'server', 'headers': { + 'Start-Line' : 'GET /argh HTTP/1.1', + 'Host' : '127.0.0.1:SERVER_PORT', + 'Client-ip' : '127.0.0.1', + 'X-Forwarded-For' : '127.0.0.1', + 'Via' : 'http/1.1 traffic_server[`` + }} + ] +} +--- ATS xDebug Probe Injection Boundary --- + + +--- ATS xDebug Probe Injection Boundary --- + +{'xDebugProbeAt' : '`` + 'captured':[{'type':'response', 'side':'server', 'headers': { + 'Start-Line' : 'HTTP/1.1 200 OK', + 'Connection' : 'close', + 'Date' : '`` + }},{'type':'response', 'side':'client', 'headers': { + 'Start-Line' : 'HTTP/1.1 200 OK', + 'Date' : '`` + 'Age' : '`` + 'Transfer-Encoding' : 'chunked', + 'Connection' : 'close', + 'Server' : 'ATS/`` + 'X-Remap' : 'from=http://two/, to=http://127.0.0.1:SERVER_PORT/' + }} + ] +} 0 ====== @@ -52,10 +124,46 @@ HTTP/1.1 200 OK Date: `` Age: `` Transfer-Encoding: chunked -Connection: keep-alive +Connection: close Server: ATS/`` X-Remap: from=http://three[0-9]+/, to=http://127.0.0.1:SERVER_PORT/ +`` +{'xDebugProbeAt' : '`` + 'captured':[{'type':'request', 'side':'client', 'headers': { + 'Start-Line' : 'GET http://127.0.0.1:SERVER_PORT/argh HTTP/1.1', + 'Host' : '127.0.0.1:SERVER_PORT', + 'Connection' : 'close' + }},{'type':'request', 'side':'server', 'headers': { + 'Start-Line' : 'GET /argh HTTP/1.1', + 'Host' : '127.0.0.1:SERVER_PORT', + 'Client-ip' : '127.0.0.1', + 'X-Forwarded-For' : '127.0.0.1', + 'Via' : 'http/1.1 traffic_server[`` + }} + ] +} +--- ATS xDebug Probe Injection Boundary --- + + +--- ATS xDebug Probe Injection Boundary --- + +{'xDebugProbeAt' : '`` + 'captured':[{'type':'response', 'side':'server', 'headers': { + 'Start-Line' : 'HTTP/1.1 200 OK', + 'Connection' : 'close', + 'Date' : '`` + }},{'type':'response', 'side':'client', 'headers': { + 'Start-Line' : 'HTTP/1.1 200 OK', + 'Date' : '`` + 'Age' : '`` + 'Transfer-Encoding' : 'chunked', + 'Connection' : 'close', + 'Server' : 'ATS/`` + 'X-Remap' : 'from=http://three[0-9]+/, to=http://127.0.0.1:SERVER_PORT/' + }} + ] +} 0 ====== @@ -64,9 +172,46 @@ Server: ATS/`` Date: `` Age: `` Transfer-Encoding: chunked -Connection: keep-alive +Connection: close X-Remap: from=http://two/, to=http://127.0.0.1:SERVER_PORT/ +`` +{'xDebugProbeAt' : '`` + 'captured':[{'type':'request', 'side':'client', 'headers': { + 'Start-Line' : 'GET http://127.0.0.1:SERVER_PORT/not_there HTTP/1.1', + 'Host' : '127.0.0.1:SERVER_PORT', + 'Connection' : 'close' + }},{'type':'request', 'side':'server', 'headers': { + 'Start-Line' : 'GET /not_there HTTP/1.1', + 'Host' : '127.0.0.1:SERVER_PORT', + 'Client-ip' : '127.0.0.1', + 'X-Forwarded-For' : '127.0.0.1', + 'Via' : 'http/1.1 traffic_server[`` + }} + ] +} +--- ATS xDebug Probe Injection Boundary --- + + +--- ATS xDebug Probe Injection Boundary --- + +{'xDebugProbeAt' : '`` + 'captured':[{'type':'response', 'side':'server', 'headers': { + 'Start-Line' : 'HTTP/1.1 404 Not Found', + 'Server' : 'MicroServer', + 'Connection' : 'close', + 'Date' : '`` + }},{'type':'response', 'side':'client', 'headers': { + 'Start-Line' : 'HTTP/1.1 404 Not Found', + 'Server' : 'ATS/`` + 'Date' : '`` + 'Age' : '`` + 'Transfer-Encoding' : 'chunked', + 'Connection' : 'close', + 'X-Remap' : 'from=http://two/, to=http://127.0.0.1:SERVER_PORT/' + }} + ] +} 0 ====== @@ -74,10 +219,48 @@ HTTP/1.1 200 OK Date: `` Age: `` Transfer-Encoding: chunked -Connection: keep-alive +Connection: close Server: ATS/`` X-Remap: from=http://two/, to=http://127.0.0.1:SERVER_PORT/ +`` +{'xDebugProbeAt' : '`` + 'captured':[{'type':'request', 'side':'client', 'headers': { + 'Start-Line' : 'GET http://127.0.0.1:SERVER_PORT/argh HTTP/1.1', + 'Host' : '127.0.0.1:SERVER_PORT', + 'X-Debug' : 'X-Remap, fwd, Probe', + 'Connection' : 'close' + }},{'type':'request', 'side':'server', 'headers': { + 'Start-Line' : 'GET /argh HTTP/1.1', + 'Host' : '127.0.0.1:SERVER_PORT', + 'X-Debug' : 'X-Remap, fwd, Probe', + 'Client-ip' : '127.0.0.1', + 'X-Forwarded-For' : '127.0.0.1', + 'Via' : 'http/1.1 traffic_server[`` + }} + ] +} +--- ATS xDebug Probe Injection Boundary --- + + +--- ATS xDebug Probe Injection Boundary --- + +{'xDebugProbeAt' : '`` + 'captured':[{'type':'response', 'side':'server', 'headers': { + 'Start-Line' : 'HTTP/1.1 200 OK', + 'Connection' : 'close', + 'Date' : '`` + }},{'type':'response', 'side':'client', 'headers': { + 'Start-Line' : 'HTTP/1.1 200 OK', + 'Date' : '`` + 'Age' : '`` + 'Transfer-Encoding' : 'chunked', + 'Connection' : 'close', + 'Server' : 'ATS/`` + 'X-Remap' : 'from=http://two/, to=http://127.0.0.1:SERVER_PORT/' + }} + ] +} 0 ====== @@ -85,10 +268,46 @@ HTTP/1.1 200 OK Date: `` Age: `` Transfer-Encoding: chunked -Connection: keep-alive +Connection: close Server: ATS/`` X-Remap: from=http://two/, to=http://127.0.0.1:SERVER_PORT/ +`` +{'xDebugProbeAt' : '`` + 'captured':[{'type':'request', 'side':'client', 'headers': { + 'Start-Line' : 'GET http://127.0.0.1:SERVER_PORT/argh HTTP/1.1', + 'Host' : '127.0.0.1:SERVER_PORT', + 'Connection' : 'close' + }},{'type':'request', 'side':'server', 'headers': { + 'Start-Line' : 'GET /argh HTTP/1.1', + 'Host' : '127.0.0.1:SERVER_PORT', + 'Client-ip' : '127.0.0.1', + 'X-Forwarded-For' : '127.0.0.1', + 'Via' : 'http/1.1 traffic_server[`` + }} + ] +} +--- ATS xDebug Probe Injection Boundary --- + + +--- ATS xDebug Probe Injection Boundary --- + +{'xDebugProbeAt' : '`` + 'captured':[{'type':'response', 'side':'server', 'headers': { + 'Start-Line' : 'HTTP/1.1 200 OK', + 'Connection' : 'close', + 'Date' : '`` + }},{'type':'response', 'side':'client', 'headers': { + 'Start-Line' : 'HTTP/1.1 200 OK', + 'Date' : '`` + 'Age' : '`` + 'Transfer-Encoding' : 'chunked', + 'Connection' : 'close', + 'Server' : 'ATS/`` + 'X-Remap' : 'from=http://two/, to=http://127.0.0.1:SERVER_PORT/' + }} + ] +} 0 ====== @@ -96,10 +315,48 @@ HTTP/1.1 200 OK Date: `` Age: `` Transfer-Encoding: chunked -Connection: keep-alive +Connection: close Server: ATS/`` X-Remap: from=http://two/, to=http://127.0.0.1:SERVER_PORT/ +`` +{'xDebugProbeAt' : '`` + 'captured':[{'type':'request', 'side':'client', 'headers': { + 'Start-Line' : 'GET http://127.0.0.1:SERVER_PORT/argh HTTP/1.1', + 'Host' : '127.0.0.1:SERVER_PORT', + 'X-Debug' : 'X-Remap, fwd=0, Probe', + 'Connection' : 'close' + }},{'type':'request', 'side':'server', 'headers': { + 'Start-Line' : 'GET /argh HTTP/1.1', + 'Host' : '127.0.0.1:SERVER_PORT', + 'X-Debug' : 'X-Remap, fwd=0, Probe', + 'Client-ip' : '127.0.0.1', + 'X-Forwarded-For' : '127.0.0.1', + 'Via' : 'http/1.1 traffic_server[`` + }} + ] +} +--- ATS xDebug Probe Injection Boundary --- + + +--- ATS xDebug Probe Injection Boundary --- + +{'xDebugProbeAt' : '`` + 'captured':[{'type':'response', 'side':'server', 'headers': { + 'Start-Line' : 'HTTP/1.1 200 OK', + 'Connection' : 'close', + 'Date' : '`` + }},{'type':'response', 'side':'client', 'headers': { + 'Start-Line' : 'HTTP/1.1 200 OK', + 'Date' : '`` + 'Age' : '`` + 'Transfer-Encoding' : 'chunked', + 'Connection' : 'close', + 'Server' : 'ATS/`` + 'X-Remap' : 'from=http://two/, to=http://127.0.0.1:SERVER_PORT/' + }} + ] +} 0 ====== @@ -107,10 +364,48 @@ HTTP/1.1 200 OK Date: `` Age: `` Transfer-Encoding: chunked -Connection: keep-alive +Connection: close Server: ATS/`` X-Remap: from=http://two/, to=http://127.0.0.1:SERVER_PORT/ +`` +{'xDebugProbeAt' : '`` + 'captured':[{'type':'request', 'side':'client', 'headers': { + 'Start-Line' : 'GET http://127.0.0.1:SERVER_PORT/argh HTTP/1.1', + 'Host' : '127.0.0.1:SERVER_PORT', + 'X-Debug' : 'PROBE, X-Remap, fwd=999998', + 'Connection' : 'close' + }},{'type':'request', 'side':'server', 'headers': { + 'Start-Line' : 'GET /argh HTTP/1.1', + 'Host' : '127.0.0.1:SERVER_PORT', + 'X-Debug' : 'PROBE, X-Remap, fwd=999998', + 'Client-ip' : '127.0.0.1', + 'X-Forwarded-For' : '127.0.0.1', + 'Via' : 'http/1.1 traffic_server[`` + }} + ] +} +--- ATS xDebug Probe Injection Boundary --- + + +--- ATS xDebug Probe Injection Boundary --- + +{'xDebugProbeAt' : '`` + 'captured':[{'type':'response', 'side':'server', 'headers': { + 'Start-Line' : 'HTTP/1.1 200 OK', + 'Connection' : 'close', + 'Date' : '`` + }},{'type':'response', 'side':'client', 'headers': { + 'Start-Line' : 'HTTP/1.1 200 OK', + 'Date' : '`` + 'Age' : '`` + 'Transfer-Encoding' : 'chunked', + 'Connection' : 'close', + 'Server' : 'ATS/`` + 'X-Remap' : 'from=http://two/, to=http://127.0.0.1:SERVER_PORT/' + }} + ] +} 0 ====== @@ -118,10 +413,46 @@ HTTP/1.1 200 OK Date: `` Age: `` Transfer-Encoding: chunked -Connection: keep-alive +Connection: close Server: ATS/`` X-Remap: from=http://two/, to=http://127.0.0.1:SERVER_PORT/ +`` +{'xDebugProbeAt' : '`` + 'captured':[{'type':'request', 'side':'client', 'headers': { + 'Start-Line' : 'GET http://127.0.0.1:SERVER_PORT/argh HTTP/1.1', + 'Host' : '127.0.0.1:SERVER_PORT', + 'Connection' : 'close' + }},{'type':'request', 'side':'server', 'headers': { + 'Start-Line' : 'GET /argh HTTP/1.1', + 'Host' : '127.0.0.1:SERVER_PORT', + 'Client-ip' : '127.0.0.1', + 'X-Forwarded-For' : '127.0.0.1', + 'Via' : 'http/1.1 traffic_server[`` + }} + ] +} +--- ATS xDebug Probe Injection Boundary --- + + +--- ATS xDebug Probe Injection Boundary --- + +{'xDebugProbeAt' : '`` + 'captured':[{'type':'response', 'side':'server', 'headers': { + 'Start-Line' : 'HTTP/1.1 200 OK', + 'Connection' : 'close', + 'Date' : '`` + }},{'type':'response', 'side':'client', 'headers': { + 'Start-Line' : 'HTTP/1.1 200 OK', + 'Date' : '`` + 'Age' : '`` + 'Transfer-Encoding' : 'chunked', + 'Connection' : 'close', + 'Server' : 'ATS/`` + 'X-Remap' : 'from=http://two/, to=http://127.0.0.1:SERVER_PORT/' + }} + ] +} 0 ====== diff --git a/tests/gold_tests/pluginTest/xdebug/x_remap/three.in b/tests/gold_tests/pluginTest/xdebug/x_remap/three.in index b525bf899e0..1d9612a51f9 100644 --- a/tests/gold_tests/pluginTest/xdebug/x_remap/three.in +++ b/tests/gold_tests/pluginTest/xdebug/x_remap/three.in @@ -1,4 +1,5 @@ GET /argh HTTP/1.1 Host: three123 -X-Debug: X-Remap +X-Debug: X-Remap, Probe +Connection: close diff --git a/tests/gold_tests/pluginTest/xdebug/x_remap/two.in b/tests/gold_tests/pluginTest/xdebug/x_remap/two.in index c971a91f8b8..514d173c104 100644 --- a/tests/gold_tests/pluginTest/xdebug/x_remap/two.in +++ b/tests/gold_tests/pluginTest/xdebug/x_remap/two.in @@ -1,4 +1,5 @@ GET /argh HTTP/1.1 Host: two -X-Debug: X-Remap +X-Debug: PROBE, X-Remap +Connection: close diff --git a/tests/gold_tests/pluginTest/xdebug/x_remap/x_remap-observer.py b/tests/gold_tests/pluginTest/xdebug/x_remap/x_remap-observer.py index 7de1b31be2d..7ff73b59695 100644 --- a/tests/gold_tests/pluginTest/xdebug/x_remap/x_remap-observer.py +++ b/tests/gold_tests/pluginTest/xdebug/x_remap/x_remap-observer.py @@ -19,6 +19,7 @@ log = open('x_remap.log', 'w') + def observe(headers): seen = False @@ -32,4 +33,5 @@ def observe(headers): log.write("-\n") log.flush() + Hooks.register(Hooks.ReadRequestHook, observe) diff --git a/tests/gold_tests/pluginTest/xdebug/x_remap/x_remap.gold b/tests/gold_tests/pluginTest/xdebug/x_remap/x_remap.gold index d0baa817b03..5a1e2fdebb1 100644 --- a/tests/gold_tests/pluginTest/xdebug/x_remap/x_remap.gold +++ b/tests/gold_tests/pluginTest/xdebug/x_remap/x_remap.gold @@ -6,13 +6,13 @@ X_DEBUG MISSING - X_DEBUG MISSING - -X-Remap, fwd +X-Remap, fwd, Probe - X_DEBUG MISSING - -X-Remap, fwd=0 +X-Remap, fwd=0, Probe - -X-Remap, fwd=999998 +PROBE, X-Remap, fwd=999998 - X_DEBUG MISSING - diff --git a/tests/gold_tests/pluginTest/xdebug/x_remap/x_remap.test.py b/tests/gold_tests/pluginTest/xdebug/x_remap/x_remap.test.py index fb070d0d3cb..f5ee5df0eea 100644 --- a/tests/gold_tests/pluginTest/xdebug/x_remap/x_remap.test.py +++ b/tests/gold_tests/pluginTest/xdebug/x_remap/x_remap.test.py @@ -15,14 +15,15 @@ # limitations under the License. Test.Summary = ''' -Test xdebug plugin X-Remap and fwd headers +Test xdebug plugin X-Remap, Probe and fwd headers ''' server = Test.MakeOriginServer("server", options={'--load': (Test.TestDirectory + '/x_remap-observer.py')}) request_header = { - "headers": "GET /argh HTTP/1.1\r\nHost: doesnotmatter\r\n\r\n", "timestamp": "1469733493.993", "body": "" } -response_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": "" } + "headers": "GET /argh HTTP/1.1\r\nHost: doesnotmatter\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +response_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", + "timestamp": "1469733493.993", "body": ""} server.addResponse("sessionlog.json", request_header, response_header) ts = Test.MakeATSProcess("ts") @@ -53,17 +54,19 @@ Test.Variables.AtsTestToolsDir, Test.RunDirectory) tr.Processes.Default.ReturnCode = 0 + def sendMsg(msgFile): tr = Test.AddTestRun() tr.Processes.Default.Command = ( - "( python {}/tcp_client.py 127.0.0.1 {} {}/{}.in".format( + "( python3 {}/tcp_client.py 127.0.0.1 {} {}/{}.in".format( Test.RunDirectory, ts.Variables.port, Test.TestDirectory, msgFile) + " ; echo '======' ) | sed 's/:{}/:SERVER_PORT/' >> {}/out.log 2>&1 ".format( server.Variables.Port, Test.RunDirectory) ) tr.Processes.Default.ReturnCode = 0 + sendMsg('none') sendMsg('one') sendMsg('two') diff --git a/tests/gold_tests/redirect/gold/redirect.gold b/tests/gold_tests/redirect/gold/redirect.gold index 9500bf976a0..3082e74c4ff 100644 --- a/tests/gold_tests/redirect/gold/redirect.gold +++ b/tests/gold_tests/redirect/gold/redirect.gold @@ -1,5 +1,5 @@ -HTTP/1.1 204 No Content +HTTP/1.1 204 No Content `` -Age: `` -Connection: keep-alive +Age: `` +Connection: keep-alive diff --git a/tests/gold_tests/redirect/gold/redirect_stale.gold b/tests/gold_tests/redirect/gold/redirect_stale.gold new file mode 100644 index 00000000000..fe4f3b6f71e --- /dev/null +++ b/tests/gold_tests/redirect/gold/redirect_stale.gold @@ -0,0 +1,10 @@ +HTTP/1.1 200 OK +X-Obj: obj2 +Cache-Control: max-age=2 +Content-Length: 0 + +HTTP/1.1 200 OK +X-Obj: obj2 +Cache-Control: max-age=2 +Content-Length: 0 + diff --git a/tests/gold_tests/redirect/redirect.test.py b/tests/gold_tests/redirect/redirect.test.py index f5c055643e6..06aff9fe71a 100644 --- a/tests/gold_tests/redirect/redirect.test.py +++ b/tests/gold_tests/redirect/redirect.test.py @@ -37,10 +37,10 @@ 'proxy.config.dns.nameservers': '127.0.0.1:{0}'.format(dns.Variables.Port), 'proxy.config.dns.resolv_conf': 'NULL', 'proxy.config.url_remap.remap_required': 0, # need this so the domain gets a chance to be evaluated through DNS - 'proxy.config.http.redirect.actions': 'self:follow', # redirects to self are not followed by default + 'proxy.config.http.redirect.actions': 'self:follow', # redirects to self are not followed by default }) -Test.Setup.Copy(os.path.join(Test.Variables.AtsTestToolsDir,'tcp_client.py')) +Test.Setup.Copy(os.path.join(Test.Variables.AtsTestToolsDir, 'tcp_client.py')) redirect_request_header = {"headers": "GET /redirect HTTP/1.1\r\nHost: *\r\n\r\n", "timestamp": "5678", "body": ""} redirect_response_header = {"headers": "HTTP/1.1 302 Found\r\nLocation: http://127.0.0.1:{0}/redirectDest\r\n\r\n".format( @@ -62,7 +62,8 @@ # Here and below: because autest's Copy does not behave like standard cp, it's easiest to write all of our files out and copy last. with open(os.path.join(data_path, tr.Name), 'w') as f: f.write('GET /redirect HTTP/1.1\r\nHost: iwillredirect.test:{port}\r\n\r\n'.format(port=redirect_serv.Variables.Port)) -tr.Processes.Default.Command = "python tcp_client.py 127.0.0.1 {0} {1} | egrep -v '^(Date: |Server: ATS/)'".format(ts.Variables.port, os.path.join(data_dirname, tr.Name)) +tr.Processes.Default.Command = "python3 tcp_client.py 127.0.0.1 {0} {1} | egrep -v '^(Date: |Server: ATS/)'".format( + ts.Variables.port, os.path.join(data_dirname, tr.Name)) tr.Processes.Default.StartBefore(ts) tr.Processes.Default.StartBefore(redirect_serv) tr.Processes.Default.StartBefore(dest_serv) @@ -71,7 +72,6 @@ tr.Processes.Default.ReturnCode = 0 - redirect_request_header = {"headers": "GET /redirect-relative-path HTTP/1.1\r\nHost: *\r\n\r\n", "timestamp": "5678", "body": ""} redirect_response_header = {"headers": "HTTP/1.1 302 Found\r\nLocation: /redirect\r\n\r\n", "timestamp": "5678", "body": ""} redirect_serv.addResponse("sessionfile.log", redirect_request_header, redirect_response_header) @@ -82,8 +82,10 @@ tr = Test.AddTestRun("FollowsRedirectWithRelativeLocationURI") with open(os.path.join(data_path, tr.Name), 'w') as f: - f.write('GET /redirect-relative-path HTTP/1.1\r\nHost: iwillredirect.test:{port}\r\n\r\n'.format(port=redirect_serv.Variables.Port)) -tr.Processes.Default.Command = "python tcp_client.py 127.0.0.1 {0} {1} | egrep -v '^(Date: |Server: ATS/)'".format(ts.Variables.port, os.path.join(data_dirname, tr.Name)) + f.write( + 'GET /redirect-relative-path HTTP/1.1\r\nHost: iwillredirect.test:{port}\r\n\r\n'.format(port=redirect_serv.Variables.Port)) +tr.Processes.Default.Command = "python3 tcp_client.py 127.0.0.1 {0} {1} | egrep -v '^(Date: |Server: ATS/)'".format( + ts.Variables.port, os.path.join(data_dirname, tr.Name)) tr.StillRunningAfter = ts tr.StillRunningAfter = redirect_serv tr.StillRunningAfter = dest_serv @@ -92,15 +94,17 @@ tr.Processes.Default.ReturnCode = 0 - -redirect_request_header = {"headers": "GET /redirect-relative-path-no-leading-slash HTTP/1.1\r\nHost: *\r\n\r\n", "timestamp": "5678", "body": ""} +redirect_request_header = { + "headers": "GET /redirect-relative-path-no-leading-slash HTTP/1.1\r\nHost: *\r\n\r\n", "timestamp": "5678", "body": ""} redirect_response_header = {"headers": "HTTP/1.1 302 Found\r\nLocation: redirect\r\n\r\n", "timestamp": "5678", "body": ""} redirect_serv.addResponse("sessionfile.log", redirect_request_header, redirect_response_header) tr = Test.AddTestRun("FollowsRedirectWithRelativeLocationURIMissingLeadingSlash") with open(os.path.join(data_path, tr.Name), 'w') as f: - f.write('GET /redirect-relative-path-no-leading-slash HTTP/1.1\r\nHost: iwillredirect.test:{port}\r\n\r\n'.format(port=redirect_serv.Variables.Port)) -tr.Processes.Default.Command = "python tcp_client.py 127.0.0.1 {0} {1} | egrep -v '^(Date: |Server: ATS/)'".format(ts.Variables.port, os.path.join(data_dirname, tr.Name)) + f.write( + 'GET /redirect-relative-path-no-leading-slash HTTP/1.1\r\nHost: iwillredirect.test:{port}\r\n\r\n'.format(port=redirect_serv.Variables.Port)) +tr.Processes.Default.Command = "python3 tcp_client.py 127.0.0.1 {0} {1} | egrep -v '^(Date: |Server: ATS/)'".format( + ts.Variables.port, os.path.join(data_dirname, tr.Name)) tr.StillRunningAfter = ts tr.StillRunningAfter = redirect_serv tr.StillRunningAfter = dest_serv @@ -109,37 +113,37 @@ tr.Processes.Default.ReturnCode = 0 -for status,phrase in sorted({ - 301:'Moved Permanently', - 302:'Found', - 303:'See Other', - 305:'Use Proxy', - 307:'Temporary Redirect', - 308:'Permanent Redirect', - }.items()): +for status, phrase in sorted({ + 301: 'Moved Permanently', + 302: 'Found', + 303: 'See Other', + 305: 'Use Proxy', + 307: 'Temporary Redirect', + 308: 'Permanent Redirect', +}.items()): redirect_request_header = { - "headers": ("GET /redirect{0} HTTP/1.1\r\n" - "Host: *\r\n\r\n").\ - format(status), - "timestamp": "5678", - "body": ""} + "headers": ("GET /redirect{0} HTTP/1.1\r\n" + "Host: *\r\n\r\n"). + format(status), + "timestamp": "5678", + "body": ""} redirect_response_header = { - "headers": ("HTTP/1.1 {0} {1}\r\n" - "Connection: close\r\n" - "Location: /redirect\r\n\r\n").\ - format(status, phrase), - "timestamp": "5678", - "body": ""} + "headers": ("HTTP/1.1 {0} {1}\r\n" + "Connection: close\r\n" + "Location: /redirect\r\n\r\n"). + format(status, phrase), + "timestamp": "5678", + "body": ""} redirect_serv.addResponse("sessionfile.log", redirect_request_header, redirect_response_header) tr = Test.AddTestRun("FollowsRedirect{0}".format(status)) with open(os.path.join(data_path, tr.Name), 'w') as f: f.write(('GET /redirect{0} HTTP/1.1\r\n' - 'Host: iwillredirect.test:{1}\r\n\r\n').\ - format(status, redirect_serv.Variables.Port)) - tr.Processes.Default.Command = "python tcp_client.py 127.0.0.1 {0} {1} | egrep -v '^(Date: |Server: ATS/)'".\ - format(ts.Variables.port, os.path.join(data_dirname, tr.Name)) + 'Host: iwillredirect.test:{1}\r\n\r\n'). + format(status, redirect_serv.Variables.Port)) + tr.Processes.Default.Command = "python3 tcp_client.py 127.0.0.1 {0} {1} | egrep -v '^(Date: |Server: ATS/)'".\ + format(ts.Variables.port, os.path.join(data_dirname, tr.Name)) tr.StillRunningAfter = ts tr.StillRunningAfter = redirect_serv tr.StillRunningAfter = dest_serv diff --git a/tests/gold_tests/redirect/redirect_actions.test.py b/tests/gold_tests/redirect/redirect_actions.test.py index 94ce1d5a09c..8b8b93c2432 100644 --- a/tests/gold_tests/redirect/redirect_actions.test.py +++ b/tests/gold_tests/redirect/redirect_actions.test.py @@ -27,7 +27,7 @@ Test.ContinueOnFail = False -Test.Setup.Copy(os.path.join(Test.Variables.AtsTestToolsDir,'tcp_client.py')) +Test.Setup.Copy(os.path.join(Test.Variables.AtsTestToolsDir, 'tcp_client.py')) dns = Test.MakeDNServer('dns') # This record is used in each test case to get the initial redirect response from the origin that we will handle. @@ -36,45 +36,46 @@ host = socket.gethostname() ipv4addrs = set() try: - ipv4addrs = set([ip for \ - (family,_,_,_,(ip,*_)) in \ - socket.getaddrinfo(host,port=None) if \ - socket.AF_INET == family]) + ipv4addrs = set([ip for + (family, _, _, _, (ip, *_)) in + socket.getaddrinfo(host, port=None) if + socket.AF_INET == family]) except socket.gaierror: pass ipv6addrs = set() try: - ipv6addrs = set(["[{0}]".format(ip.split('%')[0]) for \ - (family,_,_,_,(ip,*_)) in \ - socket.getaddrinfo(host,port=None) if \ - socket.AF_INET6 == family and 'fe80' != ip[0:4]]) # Skip link-local addresses. + ipv6addrs = set(["[{0}]".format(ip.split('%')[0]) for + (family, _, _, _, (ip, *_)) in + socket.getaddrinfo(host, port=None) if + socket.AF_INET6 == family and 'fe80' != ip[0:4]]) # Skip link-local addresses. except socket.gaierror: pass origin = Test.MakeOriginServer('origin', ip='0.0.0.0') -ArbitraryTimestamp='12345678' +ArbitraryTimestamp = '12345678' # This is for cases when the content is actually fetched from the invalid address. request_header = { - 'headers': ('GET / HTTP/1.1\r\n' - 'Host: *\r\n\r\n'), - 'timestamp': ArbitraryTimestamp, - 'body': ''} + 'headers': ('GET / HTTP/1.1\r\n' + 'Host: *\r\n\r\n'), + 'timestamp': ArbitraryTimestamp, + 'body': ''} response_header = { - 'headers': ('HTTP/1.1 204 No Content\r\n' - 'Connection: close\r\n\r\n'), - 'timestamp': ArbitraryTimestamp, - 'body': ''} + 'headers': ('HTTP/1.1 204 No Content\r\n' + 'Connection: close\r\n\r\n'), + 'timestamp': ArbitraryTimestamp, + 'body': ''} origin.addResponse('sessionfile.log', request_header, response_header) # Map scenarios to trafficserver processes. -trafficservers={} +trafficservers = {} data_dirname = 'generated_test_data' data_path = os.path.join(Test.TestDirectory, data_dirname) os.makedirs(data_path, exist_ok=True) + def normalizeForAutest(value): ''' autest uses "test run" names to build file and directory names, so we must transform them in case there are incompatible or @@ -85,6 +86,7 @@ def normalizeForAutest(value): return None return re.sub(r'[^a-z0-9-]', '_', value, flags=re.I) + def makeTestCase(redirectTarget, expectedAction, scenario): ''' Helper method that creates a "meta-test" from which autest generates a test case. @@ -93,7 +95,7 @@ def makeTestCase(redirectTarget, expectedAction, scenario): :param scenario: Defines the ACL to configure and the addresses to test. ''' - config = ','.join(':'.join(t) for t in sorted((addr.name.lower(),action.name.lower()) for (addr,action) in scenario.items())) + config = ','.join(':'.join(t) for t in sorted((addr.name.lower(), action.name.lower()) for (addr, action) in scenario.items())) normRedirectTarget = normalizeForAutest(redirectTarget) normConfig = normalizeForAutest(config) @@ -109,15 +111,15 @@ def makeTestCase(redirectTarget, expectedAction, scenario): if config not in trafficservers: trafficservers[config] = Test.MakeATSProcess('ts_{0}'.format(normConfig)) trafficservers[config].Disk.records_config.update({ - 'proxy.config.diags.debug.enabled': 1, - 'proxy.config.diags.debug.tags': 'http|dns|redirect', - 'proxy.config.http.number_of_redirections': 1, - 'proxy.config.http.cache.http': 0, - 'proxy.config.dns.nameservers': '127.0.0.1:{0}'.format(dns.Variables.Port), - 'proxy.config.dns.resolv_conf': 'NULL', - 'proxy.config.url_remap.remap_required': 0, - 'proxy.config.http.redirect.actions': config, - 'proxy.config.http.connect_attempts_timeout': 5, + 'proxy.config.diags.debug.enabled': 1, + 'proxy.config.diags.debug.tags': 'http|dns|redirect', + 'proxy.config.http.number_of_redirections': 1, + 'proxy.config.http.cache.http': 0, + 'proxy.config.dns.nameservers': '127.0.0.1:{0}'.format(dns.Variables.Port), + 'proxy.config.dns.resolv_conf': 'NULL', + 'proxy.config.url_remap.remap_required': 0, + 'proxy.config.http.redirect.actions': config, + 'proxy.config.http.connect_attempts_timeout': 5, 'proxy.config.http.connect_attempts_max_retries': 0, }) tr.Processes.Default.StartBefore(trafficservers[config]) @@ -132,29 +134,29 @@ def makeTestCase(redirectTarget, expectedAction, scenario): # A GET request parameterized on the config and on the target. request_header = { - 'headers': ('GET /redirect?config={0}&target={1} HTTP/1.1\r\n' - 'Host: *\r\n\r\n').\ - format(normConfig, normRedirectTarget), - 'timestamp': ArbitraryTimestamp, - 'body': ''} + 'headers': ('GET /redirect?config={0}&target={1} HTTP/1.1\r\n' + 'Host: *\r\n\r\n'). + format(normConfig, normRedirectTarget), + 'timestamp': ArbitraryTimestamp, + 'body': ''} # Returns a redirect to the test domain for the given target & the port number for the TS of the given config. response_header = { - 'headers': ('HTTP/1.1 307 Temporary Redirect\r\n' - 'Location: http://{0}:{1}/\r\n' - 'Connection: close\r\n\r\n').\ - format(testDomain, origin.Variables.Port), - 'timestamp': ArbitraryTimestamp, - 'body': ''} + 'headers': ('HTTP/1.1 307 Temporary Redirect\r\n' + 'Location: http://{0}:{1}/\r\n' + 'Connection: close\r\n\r\n'). + format(testDomain, origin.Variables.Port), + 'timestamp': ArbitraryTimestamp, + 'body': ''} origin.addResponse('sessionfile.log', request_header, response_header) # Generate the request data file. with open(os.path.join(data_path, tr.Name), 'w') as f: f.write(('GET /redirect?config={0}&target={1} HTTP/1.1\r\n' - 'Host: iwillredirect.test:{2}\r\n\r\n').\ - format(normConfig, normRedirectTarget, origin.Variables.Port)) + 'Host: iwillredirect.test:{2}\r\n\r\n'). + format(normConfig, normRedirectTarget, origin.Variables.Port)) # Set the command with the appropriate URL. - tr.Processes.Default.Command = "bash -o pipefail -c 'python tcp_client.py 127.0.0.1 {0} {1} | head -n 1'".\ - format(trafficservers[config].Variables.port, os.path.join(data_dirname, tr.Name)) + tr.Processes.Default.Command = "bash -o pipefail -c 'python3 tcp_client.py 127.0.0.1 {0} {1} | head -n 1'".\ + format(trafficservers[config].Variables.port, os.path.join(data_dirname, tr.Name)) tr.Processes.Default.ReturnCode = 0 # Generate and set the 'gold file' to check stdout goldFilePath = os.path.join(data_path, '{0}.gold'.format(tr.Name)) @@ -162,67 +164,70 @@ def makeTestCase(redirectTarget, expectedAction, scenario): f.write(expectedAction.value['expectedStatusLine']) tr.Processes.Default.Streams.stdout = goldFilePath + class AddressE(Enum): ''' Classes of addresses are mapped to example addresses. ''' - Private = ('10.0.0.1', '[fc00::1]') - Loopback = (['127.1.2.3']) # [::1] is ommitted here because it is likely overwritten by Self, and there are no others in IPv6. - Multicast = ('224.1.2.3', '[ff42::]') + Private = ('10.0.0.1', '[fc00::1]') + Loopback = (['127.1.2.3']) # [::1] is ommitted here because it is likely overwritten by Self, and there are no others in IPv6. + Multicast = ('224.1.2.3', '[ff42::]') Linklocal = ('169.254.0.1', '[fe80::]') - Routable = ('72.30.35.10', '[2001:4998:58:1836::10]') # Do not Follow redirects to these in an automated test. - Self = ipv4addrs | ipv6addrs # Addresses of this host. - Default = None # All addresses apply, nothing in particular to test. + Routable = ('72.30.35.10', '[2001:4998:58:1836::10]') # Do not Follow redirects to these in an automated test. + Self = ipv4addrs | ipv6addrs # Addresses of this host. + Default = None # All addresses apply, nothing in particular to test. + class ActionE(Enum): # Title case because 'return' is a Python keyword. - Return = {'config':'return', 'expectedStatusLine':'HTTP/1.1 307 Temporary Redirect\r\n'} - Reject = {'config':'reject', 'expectedStatusLine':'HTTP/1.1 403 Forbidden\r\n'} - Follow = {'config':'follow', 'expectedStatusLine':'HTTP/1.1 204 No Content\r\n'} + Return = {'config': 'return', 'expectedStatusLine': 'HTTP/1.1 307 Temporary Redirect\r\n'} + Reject = {'config': 'reject', 'expectedStatusLine': 'HTTP/1.1 403 Forbidden\r\n'} + Follow = {'config': 'follow', 'expectedStatusLine': 'HTTP/1.1 204 No Content\r\n'} # Added to test failure modes. - Break = {'expectedStatusLine': 'HTTP/1.1 502 Cannot find server.\r\n'} + Break = {'expectedStatusLine': 'HTTP/1.1 502 Cannot find server.\r\n'} + scenarios = [ { # Follow to loopback, but alternately reject/return others. - AddressE.Private: ActionE.Reject, - AddressE.Loopback: ActionE.Follow, + AddressE.Private: ActionE.Reject, + AddressE.Loopback: ActionE.Follow, AddressE.Multicast: ActionE.Reject, AddressE.Linklocal: ActionE.Return, - AddressE.Routable: ActionE.Reject, - AddressE.Self: ActionE.Return, - AddressE.Default: ActionE.Reject, - }, + AddressE.Routable: ActionE.Reject, + AddressE.Self: ActionE.Return, + AddressE.Default: ActionE.Reject, + }, { # Follow to loopback, but alternately reject/return others, flipped from the previous scenario. - AddressE.Private: ActionE.Return, - AddressE.Loopback: ActionE.Follow, + AddressE.Private: ActionE.Return, + AddressE.Loopback: ActionE.Follow, AddressE.Multicast: ActionE.Return, AddressE.Linklocal: ActionE.Reject, - AddressE.Routable: ActionE.Return, - AddressE.Self: ActionE.Reject, - AddressE.Default: ActionE.Return, + AddressE.Routable: ActionE.Return, + AddressE.Self: ActionE.Reject, + AddressE.Default: ActionE.Return, }, { # Return loopback, but reject everything else. - AddressE.Loopback: ActionE.Return, - AddressE.Default: ActionE.Reject, + AddressE.Loopback: ActionE.Return, + AddressE.Default: ActionE.Reject, }, { # Reject loopback, but return everything else. - AddressE.Loopback: ActionE.Reject, - AddressE.Default: ActionE.Return, + AddressE.Loopback: ActionE.Reject, + AddressE.Default: ActionE.Return, }, { # Return everything. - AddressE.Default: ActionE.Return, + AddressE.Default: ActionE.Return, }, - ] +] for scenario in scenarios: for addressClass in AddressE: diff --git a/tests/gold_tests/redirect/redirect_post.test.py b/tests/gold_tests/redirect/redirect_post.test.py index 23b7212bff1..c3073efe079 100644 --- a/tests/gold_tests/redirect/redirect_post.test.py +++ b/tests/gold_tests/redirect/redirect_post.test.py @@ -16,7 +16,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import os Test.Summary = ''' Test basic post redirection ''' @@ -37,21 +36,30 @@ ts.Disk.records_config.update({ 'proxy.config.http.number_of_redirections': MAX_REDIRECT, - 'proxy.config.http.post_copy_size' : 919430601, + 'proxy.config.http.post_copy_size': 919430601, 'proxy.config.http.cache.http': 0, - 'proxy.config.http.redirect.actions': 'self:follow', # redirects to self are not followed by default + 'proxy.config.http.redirect.actions': 'self:follow', # redirects to self are not followed by default # 'proxy.config.diags.debug.enabled': 1, }) -redirect_request_header = {"headers": "POST /redirect1 HTTP/1.1\r\nHost: *\r\nContent-Length: 52428800\r\n\r\n", "timestamp": "5678", "body": ""} +redirect_request_header = { + "headers": "POST /redirect1 HTTP/1.1\r\nHost: *\r\nContent-Length: 52428800\r\n\r\n", + "timestamp": "5678", + "body": ""} redirect_response_header = {"headers": "HTTP/1.1 302 Found\r\nLocation: http://127.0.0.1:{0}/redirect2\r\n\r\n".format( redirect_serv2.Variables.Port), "timestamp": "5678", "body": ""} -redirect_request_header2 = {"headers": "POST /redirect2 HTTP/1.1\r\nHost: *\r\nContent-Length: 52428800\r\n\r\n", "timestamp": "5678", "body": ""} +redirect_request_header2 = { + "headers": "POST /redirect2 HTTP/1.1\r\nHost: *\r\nContent-Length: 52428800\r\n\r\n", + "timestamp": "5678", + "body": ""} redirect_response_header2 = {"headers": "HTTP/1.1 302 Found\r\nLocation: http://127.0.0.1:{0}/redirectDest\r\n\r\n".format( dest_serv.Variables.Port), "timestamp": "5678", "body": ""} -dest_request_header = {"headers": "POST /redirectDest HTTP/1.1\r\nHost: *\r\nContent-Length: 52428800\r\n\r\n", "timestamp": "11", "body": ""} +dest_request_header = { + "headers": "POST /redirectDest HTTP/1.1\r\nHost: *\r\nContent-Length: 52428800\r\n\r\n", + "timestamp": "11", + "body": ""} dest_response_header = {"headers": "HTTP/1.1 204 No Content\r\n\r\n", "timestamp": "22", "body": ""} redirect_serv1.addResponse("sessionfile.log", redirect_request_header, redirect_response_header) @@ -63,7 +71,8 @@ ) tr = Test.AddTestRun() -tr.Processes.Default.Command = 'touch largefile.txt && truncate largefile.txt -s 50M && curl -i http://127.0.0.1:{0}/redirect1 -F "filename=@./largefile.txt" && rm -f largefile.txt'.format(ts.Variables.port) +tr.Processes.Default.Command = 'touch largefile.txt && truncate largefile.txt -s 50M && curl -i http://127.0.0.1:{0}/redirect1 -F "filename=@./largefile.txt" && rm -f largefile.txt'.format( + ts.Variables.port) tr.Processes.Default.StartBefore(ts) tr.Processes.Default.StartBefore(redirect_serv1) tr.Processes.Default.StartBefore(redirect_serv2) diff --git a/tests/gold_tests/redirect/redirect_stale.test.py b/tests/gold_tests/redirect/redirect_stale.test.py new file mode 100644 index 00000000000..5ca975ef39d --- /dev/null +++ b/tests/gold_tests/redirect/redirect_stale.test.py @@ -0,0 +1,95 @@ +''' +Test that redirects will be followed when refreshing stale cache objects. +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +Test.Summary = ''' +Test that redirects will be followed when refreshing stale cache objects. +''' + +server = Test.MakeOriginServer("server") + +ArbitraryTimestamp = '12345678' + +request_header = { + 'headers': ('GET /obj HTTP/1.1\r\n' + 'Host: *\r\n\r\n'), + 'timestamp': ArbitraryTimestamp, + 'body': ''} +response_header = { + 'headers': ('HTTP/1.1 302 Found\r\n' + 'Location: http://127.0.0.1:{}/obj2\r\n\r\n'.format(server.Variables.Port)), + 'timestamp': ArbitraryTimestamp, + 'body': ''} +server.addResponse('sessionfile.log', request_header, response_header) + +request_header = { + 'headers': ('GET /obj2 HTTP/1.1\r\n' + 'Host: *\r\n\r\n'), + 'timestamp': ArbitraryTimestamp, + 'body': ''} +response_header = { + 'headers': ('HTTP/1.1 200 OK\r\n' + 'X-Obj: obj2\r\n' + 'Cache-Control: max-age=2\r\n' + 'Content-Length: 0\r\n\r\n'), + 'timestamp': ArbitraryTimestamp, + 'body': ''} +server.addResponse('sessionfile.log', request_header, response_header) + +ts = Test.MakeATSProcess("ts") + +ts.Disk.records_config.update({ + 'proxy.config.diags.debug.enabled': 1, + 'proxy.config.diags.debug.tags': 'http|dns|cache|redirect', + 'proxy.config.http.cache.required_headers': 0, # Only Content-Length header required for caching. + 'proxy.config.http.push_method_enabled': 1, + 'proxy.config.url_remap.remap_required': 0, + 'proxy.config.http.redirect.actions': 'routable:follow,loopback:follow,self:follow', + 'proxy.config.http.number_of_redirections': 1 +}) + +# Set up to check the output after the tests have run. +# +log_id = Test.Disk.File("log2.txt") +log_id.Content = "gold/redirect_stale.gold" + +tr = Test.AddTestRun() +tr.Processes.Default.StartBefore(ts) +tr.Processes.Default.StartBefore(server) +tr.Processes.Default.Command = ( + r"printf 'GET /obj HTTP/1.1\r\nHost: 127.0.0.1:{}\r\n\r\n' | nc localhost {} >> log.txt".format( + server.Variables.Port, ts.Variables.port) +) +tr.Processes.Default.ReturnCode = 0 + +# Wait for the response in cache to become stale, then GET it again. +# +tr = Test.AddTestRun() +tr.Processes.Default.Command = ( + r"sleep 4 ; printf 'GET /obj HTTP/1.1\r\nHost: 127.0.0.1:{}\r\n\r\n' | nc localhost {} >> log.txt".format( + server.Variables.Port, ts.Variables.port) +) +tr.Processes.Default.ReturnCode = 0 + +# Filter out inconsistent content in test output. +# +tr = Test.AddTestRun() +tr.Processes.Default.Command = ( + r"grep -v -e '^Date: ' -e '^Age: ' -e '^Connection: ' -e '^Server: ATS/' log.txt | tr -d '\r'> log2.txt" +) +tr.Processes.Default.ReturnCode = 0 diff --git a/tests/gold_tests/remap/gold/remap-DNS-ipv6-200.gold b/tests/gold_tests/remap/gold/remap-DNS-ipv6-200.gold new file mode 100644 index 00000000000..8250185dfbe --- /dev/null +++ b/tests/gold_tests/remap/gold/remap-DNS-ipv6-200.gold @@ -0,0 +1,14 @@ +`` +> GET http://testDNS2.com`` +> Host: testDNS2.com`` +> User-Agent: curl/`` +> Accept: */* +`` +< HTTP/1.1 200 OK +< Date: `` +< Age: `` +< Transfer-Encoding: chunked +< Proxy-Connection: keep-alive +< Server: ATS/`` +< +`` diff --git a/tests/gold_tests/remap/gold/remap-ip-resolve.gold b/tests/gold_tests/remap/gold/remap-ip-resolve.gold new file mode 100644 index 00000000000..c83df3f4ca3 --- /dev/null +++ b/tests/gold_tests/remap/gold/remap-ip-resolve.gold @@ -0,0 +1,14 @@ +`` +> GET http://testDNS.com/`` +> Host: testDNS.com`` +> User-Agent: curl/`` +> Accept: */* +`` +< HTTP/1.1 200 OK +< Server: ATS/`` +< Date: `` +< Age: `` +< Transfer-Encoding: chunked +< Proxy-Connection: keep-alive +< +`` diff --git a/tests/gold_tests/remap/gold/remap-ws-upgrade-400.gold b/tests/gold_tests/remap/gold/remap-ws-upgrade-400.gold new file mode 100644 index 00000000000..a46a2490de7 --- /dev/null +++ b/tests/gold_tests/remap/gold/remap-ws-upgrade-400.gold @@ -0,0 +1,7 @@ +`` +> GET /chat HTTP/1.1 +> Host: `` +> User-Agent: curl/`` +`` +< HTTP/1.1 400 Invalid Upgrade Request +`` diff --git a/tests/gold_tests/remap/gold/remap-ws-upgrade.gold b/tests/gold_tests/remap/gold/remap-ws-upgrade.gold new file mode 100644 index 00000000000..317fee1f53c --- /dev/null +++ b/tests/gold_tests/remap/gold/remap-ws-upgrade.gold @@ -0,0 +1,11 @@ +`` +> GET /chat HTTP/1.1 +> Host: `` +> User-Agent: curl/`` +`` +< HTTP/1.1 101 Switching Protocols +< Sec-WebSocket-Accept: s3pPLMBiTxaQ9kYGzzhZRbK+xOo= +< Date: `` +< Connection: Upgrade +< Upgrade: websocket +`` diff --git a/tests/gold_tests/remap/gold/remap-zero-200.gold b/tests/gold_tests/remap/gold/remap-zero-200.gold new file mode 100644 index 00000000000..bcad7913071 --- /dev/null +++ b/tests/gold_tests/remap/gold/remap-zero-200.gold @@ -0,0 +1,7 @@ +`` +> GET / HTTP/1.1 +> Host: zero.one.two.three.com +`` +< HTTP/1.1 200 OK +< Date: `` +`` diff --git a/tests/gold_tests/remap/regex_map.test.py b/tests/gold_tests/remap/regex_map.test.py new file mode 100644 index 00000000000..54b3e7e7c25 --- /dev/null +++ b/tests/gold_tests/remap/regex_map.test.py @@ -0,0 +1,62 @@ +''' +Verify correct behavior of regex_map in remap.config. +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +Test.Summary = ''' +Verify correct behavior of regex_map in remap.config. +''' + +Test.ContinueOnFail = True +ts = Test.MakeATSProcess("ts") +server = Test.MakeOriginServer("server") +dns = Test.MakeDNServer("dns", default='127.0.0.1') + +Test.testName = "" +request_header = {"headers": "GET / HTTP/1.1\r\nHost: zero.one.two.three.com\r\n\r\n", + "timestamp": "1469733493.993", "body": ""} +response_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", + "timestamp": "1469733493.993", "body": ""} +server.addResponse("sessionfile.log", request_header, response_header) + +ts.Disk.records_config.update({ + 'proxy.config.diags.debug.enabled': 1, + 'proxy.config.diags.debug.tags': 'http.*|dns|conf_remap', + 'proxy.config.http.referer_filter': 1, + 'proxy.config.dns.nameservers': '127.0.0.1:{0}'.format(dns.Variables.Port), + 'proxy.config.dns.resolv_conf': 'NULL' +}) + +ts.Disk.remap_config.AddLine( + r'regex_map ' + r'http://(.*)?one\.two\.three\.com/ ' + r'http://$1reactivate.four.five.six.com:{}/'.format(server.Variables.Port) +) +ts.Disk.remap_config.AddLine( + r'regex_map ' + r'https://\b(?!(.*one|two|three|four|five|six)).+\b\.seven\.eight\.nine\.com/blah12345.html ' + r'https://www.example.com:{}/one/two/three/blah12345.html'.format(server.Variables.Port) +) + +tr = Test.AddTestRun() +tr.Processes.Default.Command = 'curl -H"Host: zero.one.two.three.com" http://127.0.0.1:{0}/ --verbose'.format(ts.Variables.port) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.StartBefore(server) +tr.Processes.Default.StartBefore(dns) +tr.Processes.Default.StartBefore(Test.Processes.ts) +tr.Processes.Default.Streams.stderr = "gold/remap-zero-200.gold" +tr.StillRunningAfter = server diff --git a/tests/gold_tests/remap/remap_http.test.py b/tests/gold_tests/remap/remap_http.test.py index 1417feaf57c..f46a3512f6b 100644 --- a/tests/gold_tests/remap/remap_http.test.py +++ b/tests/gold_tests/remap/remap_http.test.py @@ -16,7 +16,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import os Test.Summary = ''' Test a basic remap of a http connection ''' @@ -25,21 +24,25 @@ # Define default ATS ts = Test.MakeATSProcess("ts") server = Test.MakeOriginServer("server") -server2 = Test.MakeOriginServer("server2",lookup_key="{%Host}{PATH}") +server2 = Test.MakeOriginServer("server2", lookup_key="{%Host}{PATH}") dns = Test.MakeDNServer("dns") Test.testName = "" -request_header = {"headers": "GET / HTTP/1.1\r\nHost: www.example.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +request_header = {"headers": "GET / HTTP/1.1\r\nHost: www.example.com\r\n\r\n", + "timestamp": "1469733493.993", "body": ""} # expected response from the origin server -response_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +response_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", + "timestamp": "1469733493.993", "body": ""} -request_header2 = {"headers": "GET /test HTTP/1.1\r\nHost: www.testexample.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +request_header2 = {"headers": "GET /test HTTP/1.1\r\nHost: www.testexample.com\r\n\r\n", + "timestamp": "1469733493.993", "body": ""} # expected response from the origin server -response_header2 = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +response_header2 = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", + "timestamp": "1469733493.993", "body": ""} # add response to the server dictionary server.addResponse("sessionfile.log", request_header, response_header) -server2.addResponse("sessionfile.log",request_header2, response_header2) +server2.addResponse("sessionfile.log", request_header2, response_header2) ts.Disk.records_config.update({ 'proxy.config.diags.debug.enabled': 1, 'proxy.config.diags.debug.tags': 'http.*|dns|conf_remap', @@ -65,8 +68,8 @@ ) ts.Disk.remap_config.AddLine( - 'map http://www.testexample.com http://127.0.0.1:{0} @plugin=conf_remap.so @pparam=proxy.config.url_remap.pristine_host_hdr=1'.format(server2.Variables.Port) -) + 'map http://www.testexample.com http://127.0.0.1:{0} @plugin=conf_remap.so @pparam=proxy.config.url_remap.pristine_host_hdr=1'.format( + server2.Variables.Port)) dns.addRecords(records={"audrey.hepburn.com.": ["127.0.0.1"]}) dns.addRecords(records={"whatever.com.": ["127.0.0.1"]}) @@ -146,7 +149,8 @@ # microserver lookup test tr = Test.AddTestRun() -tr.Processes.Default.Command = 'curl --proxy 127.0.0.1:{0} "http://www.testexample.com/test" -H "Host: www.testexample.com" --verbose'.format(ts.Variables.port) +tr.Processes.Default.Command = 'curl --proxy 127.0.0.1:{0} "http://www.testexample.com/test" -H "Host: www.testexample.com" --verbose'.format( + ts.Variables.port) tr.Processes.Default.ReturnCode = 0 tr.Processes.Default.StartBefore(server2) tr.Processes.Default.Streams.stderr = "gold/lookupTest.gold" diff --git a/tests/gold_tests/remap/remap_https.test.py b/tests/gold_tests/remap/remap_https.test.py index f94768d9508..45feb1ddd83 100644 --- a/tests/gold_tests/remap/remap_https.test.py +++ b/tests/gold_tests/remap/remap_https.test.py @@ -16,7 +16,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import os Test.Summary = ''' Test a basic remap of a http connection ''' @@ -27,11 +26,13 @@ server = Test.MakeOriginServer("server") server2 = Test.MakeOriginServer("server2", ssl=True) -#**testname is required** +# **testname is required** testName = "" -request_header = {"headers": "GET / HTTP/1.1\r\nHost: www.example.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +request_header = {"headers": "GET / HTTP/1.1\r\nHost: www.example.com\r\n\r\n", + "timestamp": "1469733493.993", "body": ""} # desired response form the origin server -response_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +response_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", + "timestamp": "1469733493.993", "body": ""} server.addResponse("sessionlog.json", request_header, response_header) server2.addResponse("sessionlog.json", request_header, response_header) @@ -47,7 +48,7 @@ 'proxy.config.ssl.server.private_key.path': '{0}'.format(ts.Variables.SSLDir), # enable ssl port 'proxy.config.http.server_ports': '{0} {1}:proto=http2;http:ssl'.format(ts.Variables.port, ts.Variables.ssl_port), - 'proxy.config.ssl.client.verify.server': 0, + 'proxy.config.ssl.client.verify.server': 0, 'proxy.config.ssl.server.cipher_suite': 'ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA384:AES128-GCM-SHA256:AES256-GCM-SHA384:ECDHE-RSA-RC4-SHA:ECDHE-RSA-AES128-SHA:ECDHE-RSA-AES256-SHA:RC4-SHA:RC4-MD5:AES128-SHA:AES256-SHA:DES-CBC3-SHA!SRP:!DSS:!PSK:!aNULL:!eNULL:!SSLv2', }) @@ -58,7 +59,7 @@ 'map https://www.example.com:{1} http://127.0.0.1:{0}'.format(server.Variables.Port, ts.Variables.ssl_port) ) ts.Disk.remap_config.AddLine( - 'map https://www.anotherexample.com https://127.0.0.1:{0}'.format(server2.Variables.SSL_Port,ts.Variables.ssl_port) + 'map https://www.anotherexample.com https://127.0.0.1:{0}'.format(server2.Variables.SSL_Port, ts.Variables.ssl_port) ) diff --git a/tests/gold_tests/remap/remap_ip_resolve.test.py b/tests/gold_tests/remap/remap_ip_resolve.test.py new file mode 100644 index 00000000000..679d3c53c3e --- /dev/null +++ b/tests/gold_tests/remap/remap_ip_resolve.test.py @@ -0,0 +1,78 @@ + +''' +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +Test.Summary = ''' +Test a basic ip_resolve override using an ipv6 server +''' + +Test.ContinueOnFail = True +# Define default ATS +ts = Test.MakeATSProcess("ts") +server = Test.MakeOriginServer("server") +server_v6 = Test.MakeOriginServer("server_v6", None, None, '::1', 0) + +dns = Test.MakeDNServer("dns") + +Test.testName = "" +request_header = {"headers": "GET / HTTP/1.1\r\nHost: www.example.com\r\n\r\n", + "timestamp": "1469733493.993", "body": ""} +# expected response from the origin server +response_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", + "timestamp": "1469733493.993", "body": ""} + +# add response to the server dictionary +server.addResponse("sessionfile.log", request_header, response_header) +server_v6.addResponse("sessionfile.log", request_header, response_header) +ts.Disk.records_config.update({ + 'proxy.config.diags.debug.enabled': 1, + 'proxy.config.diags.debug.tags': 'http.*|dns|conf_remap', + 'proxy.config.http.referer_filter': 1, + 'proxy.config.dns.nameservers': '127.0.0.1:{0}'.format(dns.Variables.Port), + 'proxy.config.dns.resolv_conf': 'NULL', + 'proxy.config.hostdb.ip_resolve': 'ipv4' +}) + + +ts.Disk.remap_config.AddLine( + 'map http://testDNS.com http://test.ipv4.only.com:{0} @plugin=conf_remap.so @pparam=proxy.config.hostdb.ip_resolve=ipv6;ipv4;client'.format( + server.Variables.Port)) +ts.Disk.remap_config.AddLine( + 'map http://testDNS2.com http://test.ipv6.only.com:{0} @plugin=conf_remap.so @pparam=proxy.config.hostdb.ip_resolve=ipv6;only'.format( + server_v6.Variables.Port)) + + +dns.addRecords(records={"test.ipv4.only.com.": ["127.0.0.1"]}) +dns.addRecords(records={"test.ipv6.only.com": ["127.0.0.1", "::1"]}) + +tr = Test.AddTestRun() +tr.Processes.Default.Command = 'curl --proxy 127.0.0.1:{0} "http://testDNS.com" --verbose'.format(ts.Variables.port) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.StartBefore(server) +tr.Processes.Default.StartBefore(dns) +tr.Processes.Default.StartBefore(Test.Processes.ts) +tr.Processes.Default.Streams.stderr = "gold/remap-DNS-200.gold" +tr.StillRunningAfter = server + + +tr = Test.AddTestRun() +tr.Processes.Default.Command = 'curl --proxy 127.0.0.1:{0} "http://testDNS2.com" --verbose'.format(ts.Variables.port) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.StartBefore(server_v6) +tr.Processes.Default.Streams.stderr = "gold/remap-DNS-ipv6-200.gold" +tr.StillRunningAfter = server_v6 diff --git a/tests/gold_tests/remap/remap_ws.test.py b/tests/gold_tests/remap/remap_ws.test.py new file mode 100644 index 00000000000..21104128dc2 --- /dev/null +++ b/tests/gold_tests/remap/remap_ws.test.py @@ -0,0 +1,81 @@ +''' +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +Test.Summary = ''' +Test a basic remap of a websocket connections +''' + +Test.ContinueOnFail = True + +ts = Test.MakeATSProcess("ts", select_ports=False) +server = Test.MakeOriginServer("server") + +testName = "Test WebSocket Remaps" +request_header = {"headers": "GET /chat HTTP/1.1\r\nHost: www.example.com\r\nUpgrade: websocket\r\nConnection: Upgrade\r\n\r\n", + "body": None} +response_header = { + "headers": "HTTP/1.1 101 OK\r\nUpgrade: websocket\r\nConnection: Upgrade\r\nSec-WebSocket-Accept: s3pPLMBiTxaQ9kYGzzhZRbK+xOo=\r\n\r\n", + "body": None} +server.addResponse("sessionlog.json", request_header, response_header) + +ts.addSSLfile("ssl/server.pem") +ts.addSSLfile("ssl/server.key") + +ts.Disk.records_config.update({ + 'proxy.config.ssl.server.cert.path': '{0}'.format(ts.Variables.SSLDir), + 'proxy.config.ssl.server.private_key.path': '{0}'.format(ts.Variables.SSLDir), + 'proxy.config.http.server_ports': '{0} {1}:proto=http2;http:ssl'.format(ts.Variables.port, ts.Variables.ssl_port), +}) + +ts.Disk.remap_config.AddLines([ + 'map ws://www.example.com:{1} ws://127.0.0.1:{0}'.format(server.Variables.Port, ts.Variables.port), + 'map wss://www.example.com:{1} ws://127.0.0.1:{0}'.format(server.Variables.Port, ts.Variables.ssl_port) +]) + +ts.Disk.ssl_multicert_config.AddLine( + 'dest_ip=* ssl_cert_name=server.pem ssl_key_name=server.key' +) + +# wss mapping +tr = Test.AddTestRun() +tr.Processes.Default.StartBefore(server) +tr.Processes.Default.StartBefore(Test.Processes.ts, ready=1) +tr.Processes.Default.Command = 'curl --max-time 2 -v -s -q -H "Connection: Upgrade" -H "Upgrade: websocket" -H "Sec-WebSocket-Key: dGhlIHNhbXBsZSBub25jZQ==" -H "Sec-WebSocket-Version: 13" --http1.1 --resolve www.example.com:{0}:127.0.0.1 -k https://www.example.com:{0}/chat'.format( + ts.Variables.ssl_port) +tr.Processes.Default.ReturnCode = 28 +tr.Processes.Default.Streams.stderr = "gold/remap-ws-upgrade.gold" +tr.StillRunningAfter = server +tr.StillRunningAfter = ts + +# ws mapping +tr = Test.AddTestRun() +tr.Processes.Default.Command = 'curl --max-time 2 -v -s -q -H "Connection: Upgrade" -H "Upgrade: websocket" -H "Sec-WebSocket-Key: dGhlIHNhbXBsZSBub25jZQ==" -H "Sec-WebSocket-Version: 13" --http1.1 --resolve www.example.com:{0}:127.0.0.1 -k http://www.example.com:{0}/chat'.format( + ts.Variables.port) +tr.Processes.Default.ReturnCode = 28 +tr.Processes.Default.Streams.stderr = "gold/remap-ws-upgrade.gold" +tr.StillRunningAfter = server +tr.StillRunningAfter = ts + +# Missing required headers (should result in 400) +tr = Test.AddTestRun() +tr.Processes.Default.Command = 'curl --max-time 2 -v -s -q -H "Connection: Upgrade" -H "Upgrade: websocket" --http1.1 --resolve www.example.com:{0}:127.0.0.1 -k http://www.example.com:{0}/chat'.format( + ts.Variables.port) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Streams.stderr = "gold/remap-ws-upgrade-400.gold" +tr.StillRunningAfter = server +tr.StillRunningAfter = ts diff --git a/tests/gold_tests/runroot/runroot_error.test.py b/tests/gold_tests/runroot/runroot_error.test.py index 1fc7ca75d0a..dc0a26b2cf1 100644 --- a/tests/gold_tests/runroot/runroot_error.test.py +++ b/tests/gold_tests/runroot/runroot_error.test.py @@ -17,8 +17,6 @@ # limitations under the License. import os -import sys -import time Test.Summary = ''' Test for expected error and failure of runroot from traffic_layout. diff --git a/tests/gold_tests/runroot/runroot_init.test.py b/tests/gold_tests/runroot/runroot_init.test.py index 5e8e371a644..9c68b8e7844 100644 --- a/tests/gold_tests/runroot/runroot_init.test.py +++ b/tests/gold_tests/runroot/runroot_init.test.py @@ -17,8 +17,6 @@ # limitations under the License. import os -import sys -import time Test.Summary = ''' Test for init of runroot from traffic_layout. diff --git a/tests/gold_tests/runroot/runroot_manager.test.py b/tests/gold_tests/runroot/runroot_manager.test.py index 0954cdbcfc2..c9ce5f833cf 100644 --- a/tests/gold_tests/runroot/runroot_manager.test.py +++ b/tests/gold_tests/runroot/runroot_manager.test.py @@ -17,8 +17,6 @@ # limitations under the License. import os -import sys -import time Test.Summary = ''' Test for using of runroot of traffic_manager. diff --git a/tests/gold_tests/runroot/runroot_remove.test.py b/tests/gold_tests/runroot/runroot_remove.test.py index c2349eb95c6..987836c8f58 100644 --- a/tests/gold_tests/runroot/runroot_remove.test.py +++ b/tests/gold_tests/runroot/runroot_remove.test.py @@ -17,8 +17,6 @@ # limitations under the License. import os -import sys -import time Test.Summary = ''' Test for remove of runroot from traffic_layout. diff --git a/tests/gold_tests/runroot/runroot_use.test.py b/tests/gold_tests/runroot/runroot_use.test.py index 0ef2aefb210..9e9cbd5c593 100644 --- a/tests/gold_tests/runroot/runroot_use.test.py +++ b/tests/gold_tests/runroot/runroot_use.test.py @@ -17,8 +17,6 @@ # limitations under the License. import os -import sys -import time Test.Summary = ''' Test for using of runroot from traffic_layout. diff --git a/tests/gold_tests/runroot/runroot_verify.test.py b/tests/gold_tests/runroot/runroot_verify.test.py index 961c9706c66..f35a72a3627 100644 --- a/tests/gold_tests/runroot/runroot_verify.test.py +++ b/tests/gold_tests/runroot/runroot_verify.test.py @@ -17,8 +17,6 @@ # limitations under the License. import os -import sys -import time Test.Summary = ''' Test for verify of runroot from traffic_layout. @@ -30,14 +28,14 @@ if bindir.startswith(prefix): # get the bin directory based on removing the common prefix - binsuffix = bindir[len(prefix)+1:] + binsuffix = bindir[len(prefix) + 1:] else: # given a custom setup this might work.. or it might not binsuffix = bindir if logdir.startswith(prefix): # get the bin directory based on removing the common prefix - logsuffix = bindir[len(prefix)+1:] + logsuffix = bindir[len(prefix) + 1:] else: # given a custom setup this might work.. or it might not logsuffix = logdir diff --git a/tests/gold_tests/session_sharing/gold/200.gold b/tests/gold_tests/session_sharing/gold/200.gold new file mode 100644 index 00000000000..c1ac419ea9e --- /dev/null +++ b/tests/gold_tests/session_sharing/gold/200.gold @@ -0,0 +1,19 @@ +`` +< HTTP/1.1 200 OK +`` +< Content-Length: 0 +`` +< Connection: close +`` +< HTTP/1.1 200 OK +`` +< Content-Length: 0 +`` +< Connection: close +`` +< HTTP/1.1 200 OK +`` +< Content-Length: 0 +`` +< Connection: close +`` diff --git a/tests/gold_tests/session_sharing/session_match.test.py b/tests/gold_tests/session_sharing/session_match.test.py new file mode 100644 index 00000000000..2d95d7c3ebf --- /dev/null +++ b/tests/gold_tests/session_sharing/session_match.test.py @@ -0,0 +1,114 @@ +''' +Test that a plugin can modify server session sharing. +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +class SessionMatchTest: + TestCounter = 0 + + def __init__(self, TestSummary, sharingMatchValue): + SessionMatchTest.TestCounter += 1 + self._MyTestCount = SessionMatchTest.TestCounter + Test.Summary = TestSummary + self._tr = Test.AddTestRun() + self._sharingMatchValue = sharingMatchValue + self.setupOriginServer() + self.setupTS() + + def setupOriginServer(self): + self._server = Test.MakeOriginServer("server{counter}".format(counter=self._MyTestCount)) + request_header = {"headers": + "GET /one HTTP/1.1\r\nHost: www.example.com\r\nContent-Length: 0\r\n\r\n", + "timestamp": "1469733493.993", "body": ""} + response_header = {"headers": "HTTP/1.1 200 OK\r\nServer: microserver\r\n" + "Content-Length: 0\r\n\r\n", + "timestamp": "1469733493.993", "body": ""} + self._server.addResponse("sessionlog.json", request_header, response_header) + + request_header2 = {"headers": "GET /two HTTP/1.1\r\nContent-Length: 0\r\n" + "Host: www.example.com\r\n\r\n", + "timestamp": "1469733493.993", "body": "a\r\na\r\na\r\n\r\n"} + response_header2 = {"headers": "HTTP/1.1 200 OK\r\nServer: microserver\r\n" + "Content-Length: 0\r\n\r\n", + "timestamp": "1469733493.993", "body": ""} + self._server.addResponse("sessionlog.json", request_header2, response_header2) + + request_header3 = {"headers": "GET /three HTTP/1.1\r\nContent-Length: 0\r\n" + "Host: www.example.com\r\nConnection: close\r\n\r\n", + "timestamp": "1469733493.993", "body": "a\r\na\r\na\r\n\r\n"} + response_header3 = {"headers": "HTTP/1.1 200 OK\r\nServer: microserver\r\n" + "Connection: close\r\nContent-Length: 0\r\n\r\n", + "timestamp": "1469733493.993", "body": ""} + self._server.addResponse("sessionlog.json", request_header3, response_header3) + + def setupTS(self): + self._ts = Test.MakeATSProcess("ts{counter}".format(counter=self._MyTestCount)) + self._ts.Disk.remap_config.AddLine( + 'map / http://127.0.0.1:{0}'.format(self._server.Variables.Port) + ) + self._ts.Disk.records_config.update({ + 'proxy.config.diags.debug.enabled': 1, + 'proxy.config.diags.debug.tags': 'http', + 'proxy.config.http.auth_server_session_private': 1, + 'proxy.config.http.server_session_sharing.pool': 'global', + 'proxy.config.http.server_session_sharing.match': self._sharingMatchValue, + }) + + def _runTraffic(self): + self._tr.Processes.Default.Command = ( + 'curl -v -H\'Host: www.example.com\' -H\'Connection: close\' http://127.0.0.1:{port}/one &&' + 'curl -v -H\'Host: www.example.com\' -H\'Connection: close\' http://127.0.0.1:{port}/two &&' + 'curl -v -H\'Host: www.example.com\' -H\'Connection: close\' http://127.0.0.1:{port}/three'.format( + port=self._ts.Variables.port)) + self._tr.Processes.Default.ReturnCode = 0 + self._tr.Processes.Default.StartBefore(self._server) + self._tr.Processes.Default.StartBefore(self._ts) + self._tr.Processes.Default.Streams.stderr = "gold/200.gold" + + def runAndExpectSharing(self): + self._runTraffic() + self._ts.Streams.stderr = Testers.ContainsExpression( + "global pool search successful", + "Verify that sessions got shared") + + def runAndExpectNoSharing(self): + self._runTraffic() + self._ts.Streams.stderr = Testers.ExcludesExpression( + "global pool search successful", + "Verify that sessions did not get shared") + + +sessionMatchTest = SessionMatchTest( + TestSummary='Test that session sharing works with host matching', + sharingMatchValue='host') +sessionMatchTest.runAndExpectSharing() + +sessionMatchTest = SessionMatchTest( + TestSummary='Test that session sharing works with ip matching', + sharingMatchValue='ip') +sessionMatchTest.runAndExpectSharing() + +sessionMatchTest = SessionMatchTest( + TestSummary='Test that session sharing works with matching both ip and host', + sharingMatchValue='both') +sessionMatchTest.runAndExpectSharing() + +sessionMatchTest = SessionMatchTest( + TestSummary='Test that session sharing is disabled when matching is set to none', + sharingMatchValue='none') +sessionMatchTest.runAndExpectNoSharing() diff --git a/tests/gold_tests/shutdown/emergency.test.py b/tests/gold_tests/shutdown/emergency.test.py index ab8a6b0d557..343ca2606ff 100644 --- a/tests/gold_tests/shutdown/emergency.test.py +++ b/tests/gold_tests/shutdown/emergency.test.py @@ -38,7 +38,7 @@ }) # Load plugin -Test.PreparePlugin(os.path.join(Test.Variables.AtsTestToolsDir, 'plugins', 'emergency_shutdown.cc'), ts) +Test.PrepareTestPlugin(os.path.join(Test.Variables.AtsTestPluginsDir, 'emergency_shutdown.so'), ts) # www.example.com Host tr = Test.AddTestRun() @@ -46,6 +46,6 @@ tr.Processes.Default.ReturnCode = 0 tr.Processes.Default.StartBefore(ts) ts.ReturnCode = 33 -ts.Ready = 0 # Need this to be 0 because we are testing shutdown, this is to make autest not think ats went away for a bad reason. +ts.Ready = 0 # Need this to be 0 because we are testing shutdown, this is to make autest not think ats went away for a bad reason. ts.Streams.All = Testers.ExcludesExpression('failed to shutdown', 'should NOT contain "failed to shutdown"') ts.Disk.diags_log.Content = Testers.IncludesExpression('testing emergency shutdown', 'should contain "testing emergency shutdown"') diff --git a/tests/gold_tests/shutdown/fatal.test.py b/tests/gold_tests/shutdown/fatal.test.py index 6ba763d922f..972a73a3fe8 100644 --- a/tests/gold_tests/shutdown/fatal.test.py +++ b/tests/gold_tests/shutdown/fatal.test.py @@ -38,7 +38,7 @@ }) # Load plugin -Test.PreparePlugin(os.path.join(Test.Variables.AtsTestToolsDir, 'plugins', 'fatal_shutdown.cc'), ts) +Test.PrepareTestPlugin(os.path.join(Test.Variables.AtsTestPluginsDir, 'fatal_shutdown.so'), ts) # www.example.com Host tr = Test.AddTestRun() @@ -46,6 +46,6 @@ tr.Processes.Default.ReturnCode = 0 tr.Processes.Default.StartBefore(ts) ts.ReturnCode = 70 -ts.Ready = 0 # Need this to be 0 because we are testing shutdown, this is to make autest not think ats went away for a bad reason. +ts.Ready = 0 # Need this to be 0 because we are testing shutdown, this is to make autest not think ats went away for a bad reason. ts.Streams.All = Testers.ExcludesExpression('failed to shutdown', 'should NOT contain "failed to shutdown"') ts.Disk.diags_log.Content = Testers.IncludesExpression('testing fatal shutdown', 'should contain "testing fatal shutdown"') diff --git a/tests/gold_tests/slow_post/slow_post.test.py b/tests/gold_tests/slow_post/slow_post.test.py index c1e92f85374..2d508165c63 100644 --- a/tests/gold_tests/slow_post/slow_post.test.py +++ b/tests/gold_tests/slow_post/slow_post.test.py @@ -16,7 +16,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -import os + +Test.SkipUnless( + Condition.PluginExists('request_buffer.so') +) class SlowPostAttack: @@ -34,22 +37,21 @@ def setupOriginServer(self): response_header = {"headers": "HTTP/1.1 200 OK\r\nServer: microserver\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": ""} self._server.addResponse("sessionlog.json", request_header, response_header) - request_header2 = {"headers": "POST / HTTP/1.1\r\nTransfer-Encoding: chunked\r\nHost: www.example.com\r\nConnection: keep-alive\r\n\r\n", - "timestamp": "1469733493.993", "body": "a\r\na\r\na\r\n\r\n"} + request_header2 = { + "headers": "POST / HTTP/1.1\r\nTransfer-Encoding: chunked\r\nHost: www.example.com\r\nConnection: keep-alive\r\n\r\n", + "timestamp": "1469733493.993", + "body": "a\r\na\r\na\r\n\r\n"} response_header2 = {"headers": "HTTP/1.1 200 OK\r\nServer: microserver\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": ""} self._server.addResponse("sessionlog.json", request_header2, response_header2) def setupTS(self): - self._ts = Test.MakeATSProcess("ts", select_ports=False) + self._ts = Test.MakeATSProcess("ts", select_ports=True) self._ts.Disk.remap_config.AddLine( 'map / http://127.0.0.1:{0}'.format(self._server.Variables.Port) ) # This plugin can enable request buffer for POST. - self._ts.Disk.plugin_config.AddLine( - 'request_buffer.so' - ) - Test.PreparePlugin(os.path.join(Test.Variables.AtsTestToolsDir, 'plugins', 'request_buffer.c'), self._ts) + Test.PrepareInstalledPlugin('request_buffer.so', self._ts) self._ts.Disk.records_config.update({ 'proxy.config.diags.debug.enabled': 1, 'proxy.config.diags.debug.tags': 'http', diff --git a/tests/gold_tests/slow_post/slow_post_client.py b/tests/gold_tests/slow_post/slow_post_client.py index a132a6792fd..47fb4b4639e 100644 --- a/tests/gold_tests/slow_post/slow_post_client.py +++ b/tests/gold_tests/slow_post/slow_post_client.py @@ -34,7 +34,7 @@ def slow_post(port, slow_time): def makerequest(port, connection_limit): client_timeout = 3 - for i in range(connection_limit): + for _ in range(connection_limit): t = threading.Thread(target=slow_post, args=(port, client_timeout + 10)) t.daemon = True t.start() diff --git a/tests/gold_tests/thread_config/check_threads.py b/tests/gold_tests/thread_config/check_threads.py index 716d6bac55d..7cc46c9dd65 100755 --- a/tests/gold_tests/thread_config/check_threads.py +++ b/tests/gold_tests/thread_config/check_threads.py @@ -79,8 +79,10 @@ def count_threads(ts_path, etnet_threads, accept_threads): def main(): parser = argparse.ArgumentParser() parser.add_argument('-t', '--ts-path', type=str, dest='ts_path', help='path to traffic_server binary', required=True) - parser.add_argument('-e', '--etnet-threads', type=int, dest='etnet_threads', help='expected number of ET_NET threads', required=True) - parser.add_argument('-a', '--accept-threads', type=int, dest='accept_threads', help='expected number of accept threads', required=True) + parser.add_argument('-e', '--etnet-threads', type=int, dest='etnet_threads', + help='expected number of ET_NET threads', required=True) + parser.add_argument('-a', '--accept-threads', type=int, dest='accept_threads', + help='expected number of accept threads', required=True) args = parser.parse_args() exit(count_threads(args.ts_path, args.etnet_threads, args.accept_threads)) diff --git a/tests/gold_tests/thread_config/thread_100_0.test.py b/tests/gold_tests/thread_config/thread_100_0.test.py index b93f614200d..c4eb395e464 100644 --- a/tests/gold_tests/thread_config/thread_100_0.test.py +++ b/tests/gold_tests/thread_config/thread_100_0.test.py @@ -16,8 +16,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import sys - Test.Summary = 'Test that Trafficserver starts with different thread configurations.' Test.ContinueOnFail = True @@ -53,7 +51,8 @@ ts.Setup.CopyAs('check_threads.py', Test.RunDirectory) tr = Test.AddTestRun() -tr.Processes.Default.Command = 'curl --proxy http://127.0.0.1:{0} http://www.example.com -H "Proxy-Connection: Keep-Alive" --verbose'.format(ts.Variables.port) +tr.Processes.Default.Command = 'curl --proxy http://127.0.0.1:{0} http://www.example.com -H "Proxy-Connection: Keep-Alive" --verbose'.format( + ts.Variables.port) tr.Processes.Default.ReturnCode = 0 tr.Processes.Default.StartBefore(ts) tr.Processes.Default.StartBefore(server) diff --git a/tests/gold_tests/thread_config/thread_100_1.test.py b/tests/gold_tests/thread_config/thread_100_1.test.py index 0230b2ae1cb..d3c94ae1404 100644 --- a/tests/gold_tests/thread_config/thread_100_1.test.py +++ b/tests/gold_tests/thread_config/thread_100_1.test.py @@ -16,8 +16,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import sys - Test.Summary = 'Test that Trafficserver starts with different thread configurations.' Test.ContinueOnFail = True @@ -53,7 +51,8 @@ ts.Setup.CopyAs('check_threads.py', Test.RunDirectory) tr = Test.AddTestRun() -tr.Processes.Default.Command = 'curl --proxy http://127.0.0.1:{0} http://www.example.com -H "Proxy-Connection: Keep-Alive" --verbose'.format(ts.Variables.port) +tr.Processes.Default.Command = 'curl --proxy http://127.0.0.1:{0} http://www.example.com -H "Proxy-Connection: Keep-Alive" --verbose'.format( + ts.Variables.port) tr.Processes.Default.ReturnCode = 0 tr.Processes.Default.StartBefore(ts) tr.Processes.Default.StartBefore(server) diff --git a/tests/gold_tests/thread_config/thread_100_10.test.py b/tests/gold_tests/thread_config/thread_100_10.test.py index f324b60161b..1551c178c1f 100644 --- a/tests/gold_tests/thread_config/thread_100_10.test.py +++ b/tests/gold_tests/thread_config/thread_100_10.test.py @@ -16,8 +16,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import sys - Test.Summary = 'Test that Trafficserver starts with different thread configurations.' Test.ContinueOnFail = True @@ -53,7 +51,8 @@ ts.Setup.CopyAs('check_threads.py', Test.RunDirectory) tr = Test.AddTestRun() -tr.Processes.Default.Command = 'curl --proxy http://127.0.0.1:{0} http://www.example.com -H "Proxy-Connection: Keep-Alive" --verbose'.format(ts.Variables.port) +tr.Processes.Default.Command = 'curl --proxy http://127.0.0.1:{0} http://www.example.com -H "Proxy-Connection: Keep-Alive" --verbose'.format( + ts.Variables.port) tr.Processes.Default.ReturnCode = 0 tr.Processes.Default.StartBefore(ts) tr.Processes.Default.StartBefore(server) diff --git a/tests/gold_tests/thread_config/thread_1_0.test.py b/tests/gold_tests/thread_config/thread_1_0.test.py index a51ca5ef473..af542e34e17 100644 --- a/tests/gold_tests/thread_config/thread_1_0.test.py +++ b/tests/gold_tests/thread_config/thread_1_0.test.py @@ -16,8 +16,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import sys - Test.Summary = 'Test that Trafficserver starts with different thread configurations.' Test.ContinueOnFail = True @@ -53,7 +51,8 @@ ts.Setup.CopyAs('check_threads.py', Test.RunDirectory) tr = Test.AddTestRun() -tr.Processes.Default.Command = 'curl --proxy http://127.0.0.1:{0} http://www.example.com -H "Proxy-Connection: Keep-Alive" --verbose'.format(ts.Variables.port) +tr.Processes.Default.Command = 'curl --proxy http://127.0.0.1:{0} http://www.example.com -H "Proxy-Connection: Keep-Alive" --verbose'.format( + ts.Variables.port) tr.Processes.Default.ReturnCode = 0 tr.Processes.Default.StartBefore(ts) tr.Processes.Default.StartBefore(server) diff --git a/tests/gold_tests/thread_config/thread_1_1.test.py b/tests/gold_tests/thread_config/thread_1_1.test.py index b0b0db4899f..e11ee263673 100644 --- a/tests/gold_tests/thread_config/thread_1_1.test.py +++ b/tests/gold_tests/thread_config/thread_1_1.test.py @@ -16,8 +16,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import sys - Test.Summary = 'Test that Trafficserver starts with different thread configurations.' Test.ContinueOnFail = True @@ -53,7 +51,8 @@ ts.Setup.CopyAs('check_threads.py', Test.RunDirectory) tr = Test.AddTestRun() -tr.Processes.Default.Command = 'curl --proxy http://127.0.0.1:{0} http://www.example.com -H "Proxy-Connection: Keep-Alive" --verbose'.format(ts.Variables.port) +tr.Processes.Default.Command = 'curl --proxy http://127.0.0.1:{0} http://www.example.com -H "Proxy-Connection: Keep-Alive" --verbose'.format( + ts.Variables.port) tr.Processes.Default.ReturnCode = 0 tr.Processes.Default.StartBefore(ts) tr.Processes.Default.StartBefore(server) diff --git a/tests/gold_tests/thread_config/thread_1_10.test.py b/tests/gold_tests/thread_config/thread_1_10.test.py index 1eae1a2a012..ed2d0668a66 100644 --- a/tests/gold_tests/thread_config/thread_1_10.test.py +++ b/tests/gold_tests/thread_config/thread_1_10.test.py @@ -16,8 +16,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import sys - Test.Summary = 'Test that Trafficserver starts with different thread configurations.' Test.ContinueOnFail = True @@ -53,7 +51,8 @@ ts.Setup.CopyAs('check_threads.py', Test.RunDirectory) tr = Test.AddTestRun() -tr.Processes.Default.Command = 'curl --proxy http://127.0.0.1:{0} http://www.example.com -H "Proxy-Connection: Keep-Alive" --verbose'.format(ts.Variables.port) +tr.Processes.Default.Command = 'curl --proxy http://127.0.0.1:{0} http://www.example.com -H "Proxy-Connection: Keep-Alive" --verbose'.format( + ts.Variables.port) tr.Processes.Default.ReturnCode = 0 tr.Processes.Default.StartBefore(ts) tr.Processes.Default.StartBefore(server) diff --git a/tests/gold_tests/thread_config/thread_2_0.test.py b/tests/gold_tests/thread_config/thread_2_0.test.py index 286b376cc4d..b8891436fa8 100644 --- a/tests/gold_tests/thread_config/thread_2_0.test.py +++ b/tests/gold_tests/thread_config/thread_2_0.test.py @@ -16,8 +16,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import sys - Test.Summary = 'Test that Trafficserver starts with different thread configurations.' Test.ContinueOnFail = True @@ -53,7 +51,8 @@ ts.Setup.CopyAs('check_threads.py', Test.RunDirectory) tr = Test.AddTestRun() -tr.Processes.Default.Command = 'curl --proxy http://127.0.0.1:{0} http://www.example.com -H "Proxy-Connection: Keep-Alive" --verbose'.format(ts.Variables.port) +tr.Processes.Default.Command = 'curl --proxy http://127.0.0.1:{0} http://www.example.com -H "Proxy-Connection: Keep-Alive" --verbose'.format( + ts.Variables.port) tr.Processes.Default.ReturnCode = 0 tr.Processes.Default.StartBefore(ts) tr.Processes.Default.StartBefore(server) diff --git a/tests/gold_tests/thread_config/thread_2_1.test.py b/tests/gold_tests/thread_config/thread_2_1.test.py index 0c3fd6f7cac..701975c4468 100644 --- a/tests/gold_tests/thread_config/thread_2_1.test.py +++ b/tests/gold_tests/thread_config/thread_2_1.test.py @@ -16,8 +16,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import sys - Test.Summary = 'Test that Trafficserver starts with different thread configurations.' Test.ContinueOnFail = True @@ -53,7 +51,8 @@ ts.Setup.CopyAs('check_threads.py', Test.RunDirectory) tr = Test.AddTestRun() -tr.Processes.Default.Command = 'curl --proxy http://127.0.0.1:{0} http://www.example.com -H "Proxy-Connection: Keep-Alive" --verbose'.format(ts.Variables.port) +tr.Processes.Default.Command = 'curl --proxy http://127.0.0.1:{0} http://www.example.com -H "Proxy-Connection: Keep-Alive" --verbose'.format( + ts.Variables.port) tr.Processes.Default.ReturnCode = 0 tr.Processes.Default.StartBefore(ts) tr.Processes.Default.StartBefore(server) diff --git a/tests/gold_tests/thread_config/thread_2_10.test.py b/tests/gold_tests/thread_config/thread_2_10.test.py index aa674e3a46e..9d1799b37c2 100644 --- a/tests/gold_tests/thread_config/thread_2_10.test.py +++ b/tests/gold_tests/thread_config/thread_2_10.test.py @@ -16,8 +16,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import sys - Test.Summary = 'Test that Trafficserver starts with different thread configurations.' Test.ContinueOnFail = True @@ -53,7 +51,8 @@ ts.Setup.CopyAs('check_threads.py', Test.RunDirectory) tr = Test.AddTestRun() -tr.Processes.Default.Command = 'curl --proxy http://127.0.0.1:{0} http://www.example.com -H "Proxy-Connection: Keep-Alive" --verbose'.format(ts.Variables.port) +tr.Processes.Default.Command = 'curl --proxy http://127.0.0.1:{0} http://www.example.com -H "Proxy-Connection: Keep-Alive" --verbose'.format( + ts.Variables.port) tr.Processes.Default.ReturnCode = 0 tr.Processes.Default.StartBefore(ts) tr.Processes.Default.StartBefore(server) diff --git a/tests/gold_tests/thread_config/thread_32_0.test.py b/tests/gold_tests/thread_config/thread_32_0.test.py index 7b2c70ca902..eb42be40c8e 100644 --- a/tests/gold_tests/thread_config/thread_32_0.test.py +++ b/tests/gold_tests/thread_config/thread_32_0.test.py @@ -16,8 +16,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import sys - Test.Summary = 'Test that Trafficserver starts with different thread configurations.' Test.ContinueOnFail = True @@ -53,7 +51,8 @@ ts.Setup.CopyAs('check_threads.py', Test.RunDirectory) tr = Test.AddTestRun() -tr.Processes.Default.Command = 'curl --proxy http://127.0.0.1:{0} http://www.example.com -H "Proxy-Connection: Keep-Alive" --verbose'.format(ts.Variables.port) +tr.Processes.Default.Command = 'curl --proxy http://127.0.0.1:{0} http://www.example.com -H "Proxy-Connection: Keep-Alive" --verbose'.format( + ts.Variables.port) tr.Processes.Default.ReturnCode = 0 tr.Processes.Default.StartBefore(ts) tr.Processes.Default.StartBefore(server) diff --git a/tests/gold_tests/thread_config/thread_32_1.test.py b/tests/gold_tests/thread_config/thread_32_1.test.py index d6583609d4f..666fa155180 100644 --- a/tests/gold_tests/thread_config/thread_32_1.test.py +++ b/tests/gold_tests/thread_config/thread_32_1.test.py @@ -16,8 +16,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import sys - Test.Summary = 'Test that Trafficserver starts with different thread configurations.' Test.ContinueOnFail = True @@ -53,7 +51,8 @@ ts.Setup.CopyAs('check_threads.py', Test.RunDirectory) tr = Test.AddTestRun() -tr.Processes.Default.Command = 'curl --proxy http://127.0.0.1:{0} http://www.example.com -H "Proxy-Connection: Keep-Alive" --verbose'.format(ts.Variables.port) +tr.Processes.Default.Command = 'curl --proxy http://127.0.0.1:{0} http://www.example.com -H "Proxy-Connection: Keep-Alive" --verbose'.format( + ts.Variables.port) tr.Processes.Default.ReturnCode = 0 tr.Processes.Default.StartBefore(ts) tr.Processes.Default.StartBefore(server) diff --git a/tests/gold_tests/thread_config/thread_32_10.test.py b/tests/gold_tests/thread_config/thread_32_10.test.py index a794738171f..29c9f5eb41b 100644 --- a/tests/gold_tests/thread_config/thread_32_10.test.py +++ b/tests/gold_tests/thread_config/thread_32_10.test.py @@ -16,8 +16,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import sys - Test.Summary = 'Test that Trafficserver starts with different thread configurations.' Test.ContinueOnFail = True @@ -53,7 +51,8 @@ ts.Setup.CopyAs('check_threads.py', Test.RunDirectory) tr = Test.AddTestRun() -tr.Processes.Default.Command = 'curl --proxy http://127.0.0.1:{0} http://www.example.com -H "Proxy-Connection: Keep-Alive" --verbose'.format(ts.Variables.port) +tr.Processes.Default.Command = 'curl --proxy http://127.0.0.1:{0} http://www.example.com -H "Proxy-Connection: Keep-Alive" --verbose'.format( + ts.Variables.port) tr.Processes.Default.ReturnCode = 0 tr.Processes.Default.StartBefore(ts) tr.Processes.Default.StartBefore(server) diff --git a/tests/gold_tests/timeout/active_timeout.test.py b/tests/gold_tests/timeout/active_timeout.test.py index ef2c25f126c..d88af2de5ea 100644 --- a/tests/gold_tests/timeout/active_timeout.test.py +++ b/tests/gold_tests/timeout/active_timeout.test.py @@ -18,7 +18,6 @@ Test.Summary = 'Testing ATS active timeout' -# need Curl Test.SkipUnless( Condition.HasCurlFeature('http2') ) @@ -54,10 +53,10 @@ tr.Processes.Default.Command = 'curl -i http://127.0.0.1:{0}/file'.format(ts.Variables.port) tr.Processes.Default.Streams.stdout = Testers.ContainsExpression("Activity Timeout", "Request should fail with active timeout") -tr2= Test.AddTestRun("tr") +tr2 = Test.AddTestRun("tr") tr2.Processes.Default.Command = 'curl -k -i --http1.1 https://127.0.0.1:{0}/file'.format(ts.Variables.ssl_port) tr2.Processes.Default.Streams.stdout = Testers.ContainsExpression("Activity Timeout", "Request should fail with active timeout") -tr3= Test.AddTestRun("tr") +tr3 = Test.AddTestRun("tr") tr3.Processes.Default.Command = 'curl -k -i --http2 https://127.0.0.1:{0}/file'.format(ts.Variables.ssl_port) tr3.Processes.Default.Streams.stdout = Testers.ContainsExpression("Activity Timeout", "Request should fail with active timeout") diff --git a/tests/gold_tests/timeout/inactive_timeout.test.py b/tests/gold_tests/timeout/inactive_timeout.test.py index 307627450ad..5ddfc200de4 100644 --- a/tests/gold_tests/timeout/inactive_timeout.test.py +++ b/tests/gold_tests/timeout/inactive_timeout.test.py @@ -18,7 +18,6 @@ Test.Summary = 'Testing ATS inactivity timeout' -# need Curl Test.SkipUnless( Condition.HasCurlFeature('http2') ) @@ -52,12 +51,15 @@ tr.Processes.Default.StartBefore(server) tr.Processes.Default.StartBefore(ts, ready=When.PortOpen(ts.Variables.port)) tr.Processes.Default.Command = 'curl -i http://127.0.0.1:{0}/file'.format(ts.Variables.port) -tr.Processes.Default.Streams.stdout = Testers.ContainsExpression("Inactivity Timeout", "Request should fail with inactivity timeout") +tr.Processes.Default.Streams.stdout = Testers.ContainsExpression( + "Inactivity Timeout", "Request should fail with inactivity timeout") -tr2= Test.AddTestRun("tr") +tr2 = Test.AddTestRun("tr") tr2.Processes.Default.Command = 'curl -k -i --http1.1 https://127.0.0.1:{0}/file'.format(ts.Variables.ssl_port) -tr2.Processes.Default.Streams.stdout = Testers.ContainsExpression("Inactivity Timeout", "Request should fail with inactivity timeout") +tr2.Processes.Default.Streams.stdout = Testers.ContainsExpression( + "Inactivity Timeout", "Request should fail with inactivity timeout") -tr3= Test.AddTestRun("tr") +tr3 = Test.AddTestRun("tr") tr3.Processes.Default.Command = 'curl -k -i --http2 https://127.0.0.1:{0}/file'.format(ts.Variables.ssl_port) -tr3.Processes.Default.Streams.stdout = Testers.ContainsExpression("Inactivity Timeout", "Request should fail with inactivity timeout") +tr3.Processes.Default.Streams.stdout = Testers.ContainsExpression( + "Inactivity Timeout", "Request should fail with inactivity timeout") diff --git a/tests/gold_tests/timeout/timeout.test.py b/tests/gold_tests/timeout/timeout.test.py index 29094d73f85..fe0653a913a 100644 --- a/tests/gold_tests/timeout/timeout.test.py +++ b/tests/gold_tests/timeout/timeout.test.py @@ -38,5 +38,6 @@ tr.Processes.Default.StartBefore(server) tr.Processes.Default.StartBefore(ts, ready=When.PortOpen(ts.Variables.port)) tr.Processes.Default.StartBefore(dns) -tr.Processes.Default.Command = 'curl -i -x http://127.0.0.1:{0} http://127.0.0.1:{1}/file'.format(ts.Variables.port, server.Variables.Port) +tr.Processes.Default.Command = 'curl -i -x http://127.0.0.1:{0} http://127.0.0.1:{1}/file'.format( + ts.Variables.port, server.Variables.Port) tr.Processes.Default.Streams.stdout = "timeout.gold" diff --git a/tests/gold_tests/tls/Makefile.inc b/tests/gold_tests/tls/Makefile.inc new file mode 100644 index 00000000000..f0990ff8b0d --- /dev/null +++ b/tests/gold_tests/tls/Makefile.inc @@ -0,0 +1,19 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +noinst_PROGRAMS += gold_tests/tls/ssl-post +gold_tests_tls_ssl_post_SOURCES = gold_tests/tls/ssl-post.c +gold_tests_tls_ssl_post_LDADD = -lssl -lcrypto diff --git a/tests/gold_tests/tls/early_h1_get.txt b/tests/gold_tests/tls/early_h1_get.txt new file mode 100644 index 00000000000..93b5876dc30 --- /dev/null +++ b/tests/gold_tests/tls/early_h1_get.txt @@ -0,0 +1,3 @@ +GET /early_get HTTP/1.1 +Host: 127.0.0.1 + diff --git a/tests/gold_tests/tls/early_h1_post.txt b/tests/gold_tests/tls/early_h1_post.txt new file mode 100644 index 00000000000..117b06c080d --- /dev/null +++ b/tests/gold_tests/tls/early_h1_post.txt @@ -0,0 +1,6 @@ +POST /early_post HTTP/1.1 +Host: 127.0.0.1 +Content-Length: 11 + +knock knock + diff --git a/tests/gold_tests/tls/early_h2_get.txt b/tests/gold_tests/tls/early_h2_get.txt new file mode 100644 index 00000000000..6f535e8fc6b Binary files /dev/null and b/tests/gold_tests/tls/early_h2_get.txt differ diff --git a/tests/gold_tests/tls/early_h2_multi1.txt b/tests/gold_tests/tls/early_h2_multi1.txt new file mode 100644 index 00000000000..71c3350686a Binary files /dev/null and b/tests/gold_tests/tls/early_h2_multi1.txt differ diff --git a/tests/gold_tests/tls/early_h2_multi2.txt b/tests/gold_tests/tls/early_h2_multi2.txt new file mode 100644 index 00000000000..cdd633a7af2 Binary files /dev/null and b/tests/gold_tests/tls/early_h2_multi2.txt differ diff --git a/tests/gold_tests/tls/early_h2_post.txt b/tests/gold_tests/tls/early_h2_post.txt new file mode 100644 index 00000000000..ecee5c75455 Binary files /dev/null and b/tests/gold_tests/tls/early_h2_post.txt differ diff --git a/tests/gold_tests/tls/gold/clientcert-accesslog.gold b/tests/gold_tests/tls/gold/clientcert-accesslog.gold new file mode 100644 index 00000000000..7ac3617255c --- /dev/null +++ b/tests/gold_tests/tls/gold/clientcert-accesslog.gold @@ -0,0 +1,9 @@ +404 http://127.0.0.1:``/case3 0 1 +404 http://127.0.0.1:``/case4 0 0 +404 http://127.0.0.1:``/case5 0 0 +404 http://127.0.0.1:``/case6 0 0 +404 http://127.0.0.1:``/case7 0 0 +404 http://127.0.0.1:``/case8 0 0 +404 http://127.0.0.1:``/case9 0 0 +404 http://127.0.0.1:``/case11 0 1 +404 http://127.0.0.1:``/case14 0 0 diff --git a/tests/gold_tests/tls/gold/proxycert-accesslog.gold b/tests/gold_tests/tls/gold/proxycert-accesslog.gold new file mode 100644 index 00000000000..130f1ac7c9e --- /dev/null +++ b/tests/gold_tests/tls/gold/proxycert-accesslog.gold @@ -0,0 +1,8 @@ +200 https://127.0.0.1:``/ 2 0 +502 https://127.0.0.1:``/ 2 0 +404 https://127.0.0.1:``/ 2 0 +502 https://127.0.0.1:``/ 2 0 +200 https://127.0.0.1:``/ 2 0 +502 https://127.0.0.1:``/ 2 0 +404 https://127.0.0.1:``/ 2 0 +502 https://127.0.0.1:``/ 2 0 diff --git a/tests/gold_tests/tls/gold/proxycert2-accesslog.gold b/tests/gold_tests/tls/gold/proxycert2-accesslog.gold new file mode 100644 index 00000000000..4aaa66a8f59 --- /dev/null +++ b/tests/gold_tests/tls/gold/proxycert2-accesslog.gold @@ -0,0 +1,8 @@ +200 https://127.0.0.1:``/ 2 0 +502 https://127.0.0.1:``/ 2 0 +200 https://127.0.0.1:``/ 2 0 +502 https://127.0.0.1:``/ 2 0 +404 https://127.0.0.1:``/ 2 0 +502 https://127.0.0.1:``/ 2 0 +502 https://127.0.0.1:``/ 1 0 +502 https://127.0.0.1:``/ 1 0 diff --git a/tests/gold_tests/tls/h2_early_decode.py b/tests/gold_tests/tls/h2_early_decode.py new file mode 100755 index 00000000000..264905c3b5f --- /dev/null +++ b/tests/gold_tests/tls/h2_early_decode.py @@ -0,0 +1,264 @@ +#!/usr/bin/env python3 + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +''' +A simple tool to decode http2 frames for 0-rtt testing. +''' + +import hpack +import sys + + +class Http2FrameDefs: + + RESERVE_BIT_MASK = 0x7fffffff + + DATA_FRAME = 0x00 + HEADERS_FRAME = 0x01 + PRIORITY_FRAME = 0x02 + RST_STREAM_FRAME = 0x03 + SETTINGS_FRAME = 0x04 + PUSH_PROMISE_FRAME = 0x05 + PING_FRAME = 0x06 + GOAWAY_FRAME = 0x07 + WINDOW_UPDATE_FRAME = 0x08 + CONTINUATION_FRAME = 0x09 + + FRAME_TYPES = { + DATA_FRAME: 'DATA', + HEADERS_FRAME: 'HEADERS', + PRIORITY_FRAME: 'PRIORITY', + RST_STREAM_FRAME: 'RST_STREAM', + SETTINGS_FRAME: 'SETTINGS', + PUSH_PROMISE_FRAME: 'PUSH_PROMISE', + PING_FRAME: 'PING', + GOAWAY_FRAME: 'GOAWAY', + WINDOW_UPDATE_FRAME: 'WINDOW_UPDATE', + CONTINUATION_FRAME: 'CONTINUATION' + } + + SETTINGS_HEADER_TABLE_SIZE = 0x01 + SETTINGS_ENABLE_PUSH = 0x02 + SETTINGS_MAX_CONCURRENT_STREAMS = 0x03 + SETTINGS_INITIAL_WINDOW_SIZE = 0x04 + SETTINGS_MAX_FRAME_SIZE = 0x05 + SETTINGS_MAX_HEADER_LIST_SIZE = 0x06 + + SETTINGS_ID = { + SETTINGS_HEADER_TABLE_SIZE: 'HEADER_TABLE_SIZE', + SETTINGS_ENABLE_PUSH: 'ENABLE_PUSH', + SETTINGS_MAX_CONCURRENT_STREAMS: 'MAX_CONCURRENT_STREAMS', + SETTINGS_INITIAL_WINDOW_SIZE: 'INITIAL_WINDOW_SIZE', + SETTINGS_MAX_FRAME_SIZE: 'MAX_FRAME_SIZE', + SETTINGS_MAX_HEADER_LIST_SIZE: 'MAX_HEADER_LIST_SIZE' + } + + RST_STREAM_NO_ERROR = 0x0 + RST_STREAM_PROTOCOL_ERROR = 0x1 + RST_STREAM_INTERNAL_ERROR = 0x2 + RST_STREAM_FLOW_CONTROL_ERROR = 0x3 + RST_STREAM_SETTINGS_TIMEOUT = 0x4 + RST_STREAM_STREAM_CLOSED = 0x5 + RST_STREAM_FRAME_SIZE_ERROR = 0x6 + RST_STREAM_REFUSED_STREAM = 0x7 + RST_STREAM_CANCEL = 0x8 + RST_STREAM_COMPRESSION_ERROR = 0x9 + RST_STREAM_CONNECT_ERROR = 0xa + RST_STREAM_ENHANCE_YOUR_CALM = 0xb + RST_STREAM_INADEQUATE_SECURITY = 0xc + RST_STREAM_HTTP_1_1_REQUIRED = 0xd + + RST_STREAM_ERROR_CODES = { + RST_STREAM_NO_ERROR: 'NO_ERROR', + RST_STREAM_PROTOCOL_ERROR: 'PROTOCOL_ERROR', + RST_STREAM_INTERNAL_ERROR: 'INTERNAL_ERROR', + RST_STREAM_FLOW_CONTROL_ERROR: 'FLOW_CONTROL_ERROR', + RST_STREAM_SETTINGS_TIMEOUT: 'SETTINGS_TIMEOUT', + RST_STREAM_STREAM_CLOSED: 'STREAM_CLOSED', + RST_STREAM_FRAME_SIZE_ERROR: 'FRAME_SIZE_ERROR', + RST_STREAM_REFUSED_STREAM: 'REFUSED_STREAM', + RST_STREAM_CANCEL: 'CANCEL', + RST_STREAM_COMPRESSION_ERROR: 'COMPRESSION_ERROR', + RST_STREAM_CONNECT_ERROR: 'CONNECT_ERROR', + RST_STREAM_ENHANCE_YOUR_CALM: 'ENHANCE_YOUR_CALM', + RST_STREAM_INADEQUATE_SECURITY: 'INADEQUATE_SECURITY', + RST_STREAM_HTTP_1_1_REQUIRED: 'HTTP_1_1_REQUIRED' + } + + +class Http2Frame: + def __init__(self, length, frame_type, flags, stream_id): + self.length = length + self.frame_type = frame_type + self.flags = flags + self.stream_id = stream_id + self.payload = None + self.decode_error = None + return + + def add_payload(self, payload): + self.payload = payload + return + + def read_data(self): + if self.frame_type == Http2FrameDefs.DATA_FRAME: + return '\n' + self.payload.decode('utf-8') + else: + return '\nError: Frame type mismatch: {0}'.format(Http2FrameDefs.FRAME_TYPES[self.frame_type]) + + def read_headers(self): + if self.frame_type == Http2FrameDefs.HEADERS_FRAME: + try: + decoder = hpack.Decoder() + decoded_data = decoder.decode(self.payload) + output_str = '' + for header in decoded_data: + output_str += '\n' + for each in header: + output_str += each + ' ' + except hpack.exceptions.InvalidTableIndex: + output_str = self.payload.hex() + output_str += '\nWarning: Decode failed: Invalid table index (not too important)' + return output_str + else: + return '\nError: Frame type mismatch: {0}'.format(Http2FrameDefs.FRAME_TYPES[self.frame_type]) + + def read_rst_stream(self): + if self.frame_type == Http2FrameDefs.RST_STREAM_FRAME: + error_code = int(self.payload.hex(), 16) + return '\nError Code = {0}'.format(Http2FrameDefs.RST_STREAM_ERROR_CODES[error_code]) + else: + return '\nError: Frame type mismatch: {0}'.format(Http2FrameDefs.FRAME_TYPES[self.frame_type]) + + def read_settings(self): + if self.frame_type == Http2FrameDefs.SETTINGS_FRAME: + settings_str = '' + for i in range(0, self.length, 6): + settings_id = int(self.payload[i:i + 2].hex(), 16) + settings_val = int(self.payload[i + 2:i + 6].hex(), 16) + settings_str += '\n{0} = {1}'.format(Http2FrameDefs.SETTINGS_ID[settings_id], settings_val) + return settings_str + else: + return '\nError: Frame type mismatch: {0}'.format(Http2FrameDefs.FRAME_TYPES[self.frame_type]) + + def read_goaway(self): + if self.frame_type == Http2FrameDefs.GOAWAY_FRAME: + last_stream_id = int(self.payload[0:4].hex(), 16) & Http2FrameDefs.RESERVE_BIT_MASK + error_code = int(self.payload[4:8].hex(), 16) + debug_data = self.payload[8:].hex() + return '\nLast Stream ID = 0x{0:08x}\nError Code = 0x{1:08x}\nDebug Data = {2}'.format( + last_stream_id, error_code, debug_data + ) + else: + return '\nError: Frame type mismatch: {0}'.format(Http2FrameDefs.FRAME_TYPES[self.frame_type]) + + def read_window_update(self): + if self.frame_type == Http2FrameDefs.WINDOW_UPDATE_FRAME: + window_size_increment = int(self.payload.hex(), 16) & Http2FrameDefs.RESERVE_BIT_MASK + return '\nWindow Size Increment = {0}'.format(window_size_increment) + else: + return '\nError: Frame type mismatch: {0}'.format(Http2FrameDefs.FRAME_TYPES[self.frame_type]) + + def print_payload(self): + if self.frame_type == Http2FrameDefs.DATA_FRAME: + return self.read_data() + elif self.frame_type == Http2FrameDefs.HEADERS_FRAME: + return self.read_headers() + elif self.frame_type == Http2FrameDefs.RST_STREAM_FRAME: + return self.read_rst_stream() + elif self.frame_type == Http2FrameDefs.SETTINGS_FRAME: + return self.read_settings() + elif self.frame_type == Http2FrameDefs.GOAWAY_FRAME: + return self.read_goaway() + elif self.frame_type == Http2FrameDefs.WINDOW_UPDATE_FRAME: + return self.read_window_update() + else: + return self.payload.hex() + + def print(self): + output = 'Length: {0}\nType: {1}\nFlags: {2}\nStream ID: {3}\nPayload: {4}\n'.format( + self.length, Http2FrameDefs.FRAME_TYPES[self.frame_type], self.flags, self.stream_id, self.print_payload() + ) + if self.decode_error is not None: + output += self.decode_error + '\n' + return output + + def __str__(self): + return self.print() + + +class Decoder: + def read_frame_header(self, data): + frame = Http2Frame( + length=int(data[0:3].hex(), 16), + frame_type=int(data[3:4].hex(), 16), + flags=int(data[4:5].hex(), 16), + stream_id=int(data[5:9].hex(), 16) & Http2FrameDefs.RESERVE_BIT_MASK + ) + return frame + + def decode(self, data): + temp_data = data + frames = [] + while len(temp_data) >= 9: + frame_header = temp_data[0:9] + frame = self.read_frame_header(frame_header) + if frame.length > len(temp_data[9:]): + frame.decode_error = 'Error: Payload length greater than data: {0} > {1}'.format(frame.length, len(temp_data[9:])) + frame.add_payload(temp_data[9:]) + frames.append(frame) + else: + frame.add_payload(temp_data[9:9 + frame.length]) + frames.append(frame) + temp_data = temp_data[9 + frame.length:] + return frames + + +def main(): + # input file is output from openssl s_client. + # sample command to get this output: + # openssl s_client -bind 127.0.0.1:61991 -connect 127.0.0.1:61992 -tls1_3 + # -quiet -sess_out /home/duke/Dev/ats-test/sess.dat -sess_in + # /home/duke/Dev/ats-test/sess.dat -early_data ./gold_tests/tls/early2.txt + # >! _sandbox/tls_0rtt_server/early2_out.txt 2>&1 + + if len(sys.argv) < 2: + print('Error: No input file to decode.') + exit(1) + + lines = None + with open(sys.argv[1], 'rb') as in_file: + lines = in_file.readlines() + + data = b'' + for line in lines: + if line.startswith(bytes('SSL_connect:', 'utf-8')) or \ + line.startswith(bytes('SSL3 alert', 'utf-8')) or \ + bytes('Can\'t use SSL_get_servername', 'utf-8') in line: + continue + data += line + + d = Decoder() + frames = d.decode(data) + for frame in frames: + print(frame) + exit(0) + + +if __name__ == "__main__": + main() diff --git a/tests/gold_tests/tls/h2_early_gen.py b/tests/gold_tests/tls/h2_early_gen.py new file mode 100755 index 00000000000..aef85a9f1f9 --- /dev/null +++ b/tests/gold_tests/tls/h2_early_gen.py @@ -0,0 +1,194 @@ +#!/usr/bin/env python3 + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +''' +A simple tool to generate some raw http2 frames for 0-rtt testing. +''' + +# http2 frame format: +# +-----------------------------------------------+ +# | Length (24) | +# +---------------+---------------+---------------+ +# | Type (8) | Flags (8) | +# +-+-------------+---------------+-------------------------------+ +# |R| Stream Identifier (31) | +# +=+=============================================================+ +# | Frame Payload (0...) ... +# +---------------------------------------------------------------+ + +import hpack +import os +import sys + +H2_PREFACE = bytes.fromhex('505249202a20485454502f322e300d0a0d0a534d0d0a0d0a') + +RESERVED_BIT_MASK = 0x7FFFFFFF + +TYPE_HEADERS_FRAME = 0x01 +TYPE_SETTINGS_FRAME = 0x04 +TYPE_WINDOW_UPDATE_FRAME = 0x08 + +SETTINGS_HEADER_TABLE_SIZE = 0x01 +SETTINGS_ENABLE_PUSH = 0x02 +SETTINGS_MAX_CONCURRENT_STREAMS = 0x03 +SETTINGS_INITIAL_WINDOW_SIZE = 0x04 +SETTINGS_MAX_FRAME_SIZE = 0x05 +SETTINGS_MAX_HEADER_LIST_SIZE = 0x06 + +HEADERS_FLAG_END_STREAM = 0x01 +HEADERS_FLAG_END_HEADERS = 0x04 +HEADERS_FLAG_END_PADDED = 0x08 +HEADERS_FLAG_END_PRIORITY = 0x20 + +CURRENT_SCRIPT_PATH = os.path.dirname(os.path.abspath(__file__)) + + +def encode_payload(data): + encoder = hpack.Encoder() + data_encoded = encoder.encode(data) + return data_encoded + + +def make_frame(frame_length, frame_type, frame_flags, frame_stream_id, frame_payload): + frame_length = bytes.fromhex('{0:06x}'.format(frame_length)) + frame_type = bytes.fromhex('{0:02x}'.format(frame_type)) + frame_flags = bytes.fromhex('{0:02x}'.format(frame_flags)) + frame_stream_id = bytes.fromhex('{0:08x}'.format(RESERVED_BIT_MASK & frame_stream_id)) + + frame = frame_length + frame_type + frame_flags + frame_stream_id + + if frame_payload is not None: + frame += frame_payload + + return frame + + +def make_settins_frame(ack=False, empty=False): + payload = '' + if not ack and not empty: + payload += '{0:04x}{1:08x}'.format(SETTINGS_ENABLE_PUSH, 0) + payload += '{0:04x}{1:08x}'.format(SETTINGS_MAX_CONCURRENT_STREAMS, 100) + payload += '{0:04x}{1:08x}'.format(SETTINGS_INITIAL_WINDOW_SIZE, 1073741824) + payload = bytes.fromhex(payload) + + frame = make_frame( + frame_length=len(payload), + frame_type=TYPE_SETTINGS_FRAME, + frame_flags=1 if ack else 0, + frame_stream_id=0, + frame_payload=payload + ) + + return frame + + +def make_window_update_frame(): + payload = '{0:08x}'.format(RESERVED_BIT_MASK & 1073676289) + payload = bytes.fromhex(payload) + + frame = make_frame( + frame_length=len(payload), + frame_type=TYPE_WINDOW_UPDATE_FRAME, + frame_flags=0, + frame_stream_id=0, + frame_payload=payload + ) + return frame + + +def make_headers_frame(method, path='', stream_id=0x01): + headers = [] + if method == 'get': + headers.append((':method', 'GET')) + if path != '': + headers.append((':path', path)) + else: + headers.append((':path', '/early_get')) + elif method == 'post': + headers.append((':method', 'POST')) + if path != '': + headers.append((':path', path)) + else: + headers.append((':path', '/early_post')) + + headers.extend([ + (':scheme', 'http'), + (':authority', '127.0.0.1'), + ('host', '127.0.0.1'), + ('accept', '*/*') + ]) + + headers_encoded = encode_payload(headers) + + frame = make_frame( + frame_length=len(headers_encoded), + frame_type=TYPE_HEADERS_FRAME, + frame_flags=HEADERS_FLAG_END_STREAM | HEADERS_FLAG_END_HEADERS, + frame_stream_id=stream_id, + frame_payload=headers_encoded + ) + + return frame + + +def make_h2_req(test): + h2_req = H2_PREFACE + if test == 'get' or test == 'post': + frames = [ + make_settins_frame(ack=True), + make_headers_frame(test) + ] + for frame in frames: + h2_req += frame + elif test == 'multi1': + frames = [ + make_settins_frame(ack=True), + make_headers_frame('get', '/early_multi_1', 1), + make_headers_frame('get', '/early_multi_2', 3), + make_headers_frame('get', '/early_multi_3', 5) + ] + for frame in frames: + h2_req += frame + elif test == 'multi2': + frames = [ + make_settins_frame(ack=True), + make_headers_frame('get', '/early_multi_1', 1), + make_headers_frame('post', stream_id=3), + make_headers_frame('get', '/early_multi_3', 5) + ] + for frame in frames: + h2_req += frame + else: + pass + return h2_req + + +def write_to_file(data, file_name): + with open(file_name, 'wb') as out_file: + out_file.write(data) + return + + +def main(): + test = sys.argv[1] + write_to_file(make_h2_req(test), os.path.join(CURRENT_SCRIPT_PATH, 'early_h2_{0}.txt'.format(test))) + exit(0) + + +if __name__ == '__main__': + main() diff --git a/tests/gold_tests/tls/ssl-post.c b/tests/gold_tests/tls/ssl-post.c index 35ec0092874..58376fdb6cf 100644 --- a/tests/gold_tests/tls/ssl-post.c +++ b/tests/gold_tests/tls/ssl-post.c @@ -31,6 +31,7 @@ #include #include #include +#include #include #include @@ -58,11 +59,18 @@ SSL_locking_callback(int mode, int type, const char *file, int line) } } +// In OpenSSL 1.1, locking and thread id logic was changed and the +// CRYPTO_THREADID_set_callback function became a macro defined to be 0. In +// later versions, therefore, static analysis tools flag the use of this as a +// problem. Thus in order to see whether CRYPTO_THREADID_set_callback is a +// valid function we check that it is not a defined macro. +#if !defined(CRYPTO_THREADID_set_callback) void SSL_pthreads_thread_id(CRYPTO_THREADID *id) { CRYPTO_THREADID_set_numeric(id, (unsigned long)pthread_self()); } +#endif void * spawn_same_session_send(void *arg) @@ -134,15 +142,15 @@ spawn_same_session_send(void *arg) ret = select(sfd + 1, &reads, &writes, NULL, NULL); if (FD_ISSET(sfd, &reads) || FD_ISSET(sfd, &writes)) { ret = write_ret = SSL_write(ssl, req_buf, strlen(req_buf)); - if (write_ret >= 0) + if (write_ret > 0) post_write_ret = SSL_write(ssl, post_buf, sizeof(post_buf)); } } - while (write_ret < 0) { + while (write_ret <= 0) { write_ret = SSL_write(ssl, req_buf, strlen(req_buf)); } - while (post_write_ret < 0) { + while (post_write_ret <= 0) { post_write_ret = SSL_write(ssl, post_buf, sizeof(post_buf)); } @@ -214,9 +222,7 @@ main(int argc, char *argv[]) { struct addrinfo hints; struct addrinfo *result, *rp; - int sfd, s, j; - size_t len; - ssize_t nread; + int sfd, s; if (argc < 4) { fprintf(stderr, "Usage: %s host thread-count header-count [port]\n", argv[0]); @@ -224,7 +230,7 @@ main(int argc, char *argv[]) } char *host = argv[1]; int header_count = atoi(argv[3]); - snprintf(req_buf, sizeof(req_buf), "POST /post HTTP/1.1\r\nHost: %s\r\nConnection: close\r\nContent-length:%d\r\n", host, + snprintf(req_buf, sizeof(req_buf), "POST /post HTTP/1.1\r\nHost: %s\r\nConnection: close\r\nContent-length:%zu\r\n", host, sizeof(post_buf)); int i; for (i = 0; i < header_count; i++) { @@ -282,7 +288,12 @@ main(int argc, char *argv[]) } CRYPTO_set_locking_callback(SSL_locking_callback); + +// See the '!defined(CRYPTO_THREADID_set_callback)' comment above for why we +// test for !defined here. +#if !defined(CRYPTO_THREADID_set_callback) CRYPTO_THREADID_set_callback(SSL_pthreads_thread_id); +#endif SSL_CTX *client_ctx = SSL_CTX_new(SSLv23_client_method()); SSL *ssl = SSL_new(client_ctx); @@ -291,9 +302,13 @@ main(int argc, char *argv[]) #endif SSL_set_fd(ssl, sfd); - int ret = SSL_connect(ssl); - int read_count = 0; - int write_count = 1; + int ret = SSL_connect(ssl); + + if (ret <= 0) { + int error = SSL_get_error(ssl, ret); + printf("SSL_connect failed %d", error); + exit(1); + } printf("Sent request\n"); if ((ret = SSL_write(ssl, req_buf, strlen(req_buf))) <= 0) { @@ -325,7 +340,7 @@ main(int argc, char *argv[]) retval = NULL; pthread_join(threads[i], &retval); if (retval != NULL) { - printf("Thread %d failed 0x%x\n", i, retval); + printf("Thread %d failed %p\n", i, retval); } } diff --git a/tests/gold_tests/tls/ssl/combined-ec.pem b/tests/gold_tests/tls/ssl/combined-ec.pem new file mode 100644 index 00000000000..e9ced49ee53 --- /dev/null +++ b/tests/gold_tests/tls/ssl/combined-ec.pem @@ -0,0 +1,18 @@ +-----BEGIN CERTIFICATE----- +MIIB+jCCAZ+gAwIBAgIJAPW97i/S9OcdMAoGCCqGSM49BAMCMFkxCzAJBgNVBAYT +AlVTMQswCQYDVQQIDAJJTDEVMBMGA1UEBwwMRGVmYXVsdCBDaXR5MQ8wDQYDVQQK +DAZBcGFjaGUxFTATBgNVBAMMDGNvbWJpbmVkLmNvbTAeFw0yMDA2MjIxOTU3MDZa +Fw0zMDAzMjIxOTU3MDZaMFkxCzAJBgNVBAYTAlVTMQswCQYDVQQIDAJJTDEVMBMG +A1UEBwwMRGVmYXVsdCBDaXR5MQ8wDQYDVQQKDAZBcGFjaGUxFTATBgNVBAMMDGNv +bWJpbmVkLmNvbTBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABIBVN2VZFf4FnTKc +KF/E3MfOGw9AgggHkOBjmXTy0UBopJ2GU39kHvAk0AzuNsVhs63X3tSMjPGaoy4X +lwEfmLWjUDBOMB0GA1UdDgQWBBR0yi0/z4mhyD00kmscLF4aUlGC3zAfBgNVHSME +GDAWgBR0yi0/z4mhyD00kmscLF4aUlGC3zAMBgNVHRMEBTADAQH/MAoGCCqGSM49 +BAMCA0kAMEYCIQDE2BERQi0cN/hR+T2uPxdaLEKjhaH/FSZ8WPVQ+B60VwIhAOk2 +QwhkNb8Kj5Zr6NwMdmK2xowyP7QdeYSqQmLOSC0w +-----END CERTIFICATE----- +-----BEGIN EC PRIVATE KEY----- +MHcCAQEEIHl0cmnC4+n9FSVl/k6/kzcP3z8rxBwWxPuTlYM7LfZYoAoGCCqGSM49 +AwEHoUQDQgAEgFU3ZVkV/gWdMpwoX8Tcx84bD0CCCAeQ4GOZdPLRQGiknYZTf2Qe +8CTQDO42xWGzrdfe1IyM8ZqjLheXAR+YtQ== +-----END EC PRIVATE KEY----- diff --git a/tests/gold_tests/tls/ssl/combined.pem b/tests/gold_tests/tls/ssl/combined.pem new file mode 100644 index 00000000000..8f8c973fb4b --- /dev/null +++ b/tests/gold_tests/tls/ssl/combined.pem @@ -0,0 +1,84 @@ +-----BEGIN CERTIFICATE----- +MIIFhTCCA22gAwIBAgIJAJ+SIgl5BIzFMA0GCSqGSIb3DQEBCwUAMFkxCzAJBgNV +BAYTAlVTMQswCQYDVQQIDAJJTDEVMBMGA1UEBwwMRGVmYXVsdCBDaXR5MQ8wDQYD +VQQKDAZBcGFjaGUxFTATBgNVBAMMDGNvbWJpbmVkLmNvbTAeFw0yMDA2MjIxNzU0 +MTFaFw0zMDA2MjAxNzU0MTFaMFkxCzAJBgNVBAYTAlVTMQswCQYDVQQIDAJJTDEV +MBMGA1UEBwwMRGVmYXVsdCBDaXR5MQ8wDQYDVQQKDAZBcGFjaGUxFTATBgNVBAMM +DGNvbWJpbmVkLmNvbTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAMk+ +3gIBko7xQUrlQwaHZBtD2g4gwc70baOnJTUY8thtO4HkwDF7QtR+8kTIzc3RBxbN +0WjZidhBHUphMdopay7cO7C/NMRogNJjTQnMyzVXygwn0nDCYlqgbn8xO6Q3uJ54 +vILWZ75B+Q3uR77LBr3IBpD6OEgipIEhhCa/9F5I4OFX2T0mpjKnfka8VwjcA413 +kSoa90ilfMLfWQoX6scFHRlRYZ5rDR2/paQQL6DPhnIY2Fp0vU2H3AdiHS1uz4/5 +z1gYov7ozgWEOnxCg1fUXzAfvbrBkvtTxk8mENizRYmYZg6GON1g1ELmrcV1XDus +iTX+qEj3BzNhp3dmUuqEjb46gVfWD9BNuF2tuNG6J3cxJCYUgkI+YpXirgHCmv8d +XQ6575YFk021nn79eo6aX7K+x1yy1SA4j+ZH317ZTBr3gaCZYrkCzEbwmDo29llz +sv3D4lOH+JVWIW0sRUCHxd5oRDyHhNS62fuGXxCym9E8T+wfZ1nQTuggRkpy1MIj +nVI/tHiUXSaU/XdTk47qqRgxvVRbtgIxnvZ81JfsvX3BzGvNvUMI/UKw7bK2EkbW +yhxag1z/WAfdOnOJuW9KCe/ljsyZ18lIeL+LvxtkFnhznZbQGtUXE1FsFSaywa2q +l+z046iTcY1bTOWP+GSKby89fHoOH1ClbJBIIcQNAgMBAAGjUDBOMB0GA1UdDgQW +BBRwYTUpRi+YDV1jSAg0ccfHP4+V8zAfBgNVHSMEGDAWgBRwYTUpRi+YDV1jSAg0 +ccfHP4+V8zAMBgNVHRMEBTADAQH/MA0GCSqGSIb3DQEBCwUAA4ICAQBeNcLzVyGZ +5ZMyISTPBSWZTX0G80tT0m7JmIAvKoe2poroUYhoGr5ZycC0pRImzYj4zpNhI1sD +o5n83Z8Xvo1JNuhL8vyxc/QZEGczNSkxY5SF/gPt0D8brRryrjNSO04UOIdBTkOn +yX7cx+c7zEFiRQSmZKd7WcbEV9iOcF0+6JHVrICxCd8gp1LCj9cuH68f05BRXW9V +b0DVh7dC2BPQU2QvwVlbIX7HzDM7QyVOdnw0vcDJPZwehNquCW/wJfqrjI6OoUWJ +k/Zgb8Xj0TYqldUUkJgCEoiNL3Msznv8nyWAgo7saOMFDVTSDES3+TltZsewjgMP +b3KO1G8fUaGkMyEDgTfSl8KXbtd2FD6N3+jH6y+aeLIsVUjtHNBTjnX6ZPo4tka1 +L/9w/Op1UjqIL+vwLcXEdrcxdh3JEqb4oFYhr5g9fpkSZrzP1xHuCx2IAJlrz3GP +csI9fTzb04p4QX4aa62O70FkvVQhHIVHLzQ0lXi3+nJovLlV4Yop8mkFpvV0JefP +UDa+PcvgAq4+LsM1Y8N3IlF4VIfE2ThahEyQ0kVtNRr2wKjFxcvPWbQ0EPXcl+EY +A678CWbTxwsy+H4iIGjBmMalPjl+4W5a3UBKx6wQ1ZVPJDMpu8FIZ/JC8A1V9YcH +6GutNoBaA8oeWn8u+oYbL62dcqrQUxytaw== +-----END CERTIFICATE----- +-----BEGIN PRIVATE KEY----- +MIIJQgIBADANBgkqhkiG9w0BAQEFAASCCSwwggkoAgEAAoICAQDJPt4CAZKO8UFK +5UMGh2QbQ9oOIMHO9G2jpyU1GPLYbTuB5MAxe0LUfvJEyM3N0QcWzdFo2YnYQR1K +YTHaKWsu3DuwvzTEaIDSY00JzMs1V8oMJ9JwwmJaoG5/MTukN7ieeLyC1me+QfkN +7ke+ywa9yAaQ+jhIIqSBIYQmv/ReSODhV9k9JqYyp35GvFcI3AONd5EqGvdIpXzC +31kKF+rHBR0ZUWGeaw0dv6WkEC+gz4ZyGNhadL1Nh9wHYh0tbs+P+c9YGKL+6M4F +hDp8QoNX1F8wH726wZL7U8ZPJhDYs0WJmGYOhjjdYNRC5q3FdVw7rIk1/qhI9wcz +Yad3ZlLqhI2+OoFX1g/QTbhdrbjRuid3MSQmFIJCPmKV4q4Bwpr/HV0Oue+WBZNN +tZ5+/XqOml+yvsdcstUgOI/mR99e2Uwa94GgmWK5AsxG8Jg6NvZZc7L9w+JTh/iV +ViFtLEVAh8XeaEQ8h4TUutn7hl8QspvRPE/sH2dZ0E7oIEZKctTCI51SP7R4lF0m +lP13U5OO6qkYMb1UW7YCMZ72fNSX7L19wcxrzb1DCP1CsO2ythJG1socWoNc/1gH +3TpziblvSgnv5Y7MmdfJSHi/i78bZBZ4c52W0BrVFxNRbBUmssGtqpfs9OOok3GN +W0zlj/hkim8vPXx6Dh9QpWyQSCHEDQIDAQABAoICAB3lgQW7RhKGNLTzqilmI7EJ +O3Ot2hzsov0U76obsrgEQjrhyMuRDDSVR1iAnFJfIzZ4DQwOhTPjOH68QbOvQ4UK +orN3NFeZTsnyhpdWHToneJSltdWaixH7k27B2aJjxMGGHjsxrufM1TsEkxP9BVtc +Q911zraortegKwm2qo73NLFFY7h57WbJCzFm+WcPiFeT07T1nJbmPsqUc53UP8Sh +Ndl1BZOLQ7+PoQPmg6zYJ5j7p+iIPzctX7qZEMyXEa7J5cuonU5RN4oKQsYC+5Ao +tJtpm74K09MSdvt7JDWNLF9sjHzL49a/c8kci+QxsyoKT7f9xPJSAHfZog9S3C5H +WXXds+SFfhMQeqw6zJDoWukEeZ5Hg3oqUd2kwAbZZUobNwBaEnKG759+qlCaSNls ++C29WOLBkc4fqxR4/33A7tN9DlGycocG1yqfaQaKRt0Ny9UyhtDyi/i+mZAqe8Bp +I2bKvtkyPrXcyoIdmVP0PCfY2mrbAag2T1RrkTg24Z9YJt++c+HNBfZzrsO6oWs7 +5FHCUlWEt5baapVZxfA7ua2dwHdwQlQMV2hkh0U6qg8YA33liAdU6Os0Tjld1TJ0 +E8+TuG0NtiC9ng7O61r+43qJ6q5cMgb+Xbhv0XAKgJPwPly2W59RCd7TuThzuVIy +rGR2EFIx2vh8HmYcsNkBAoIBAQDqaAMHFHzhn5e5biUzK47OhZEwWB7xF++TylVq +Xx/+tz24CzJVhnEmUurugh7Yz8COrXd0fpt7bY6w9kovz7N0qHxhkYLykCXyyots +8Veaj1+ZJ5gLBBXbyTm6cu3jqk+rhD/h4GKawCp6OMFla2hfFo4hEK8KZA0xRLHi +1jZKLAqg7224QOdTXiCG5N6Jx1DL8ca3pVDJPG4Xb0kd1cjcWM5rc2KgG964wnkK +UEGkB63lz3h5lcXcJDhfxeJGthFvpIGpuOelyzjmYuwfoq5Ec9PIQSGjnlSt0CE6 +XgdexX6pZ1SmYCYcnk0veF0hgkQKuHhcIqCsw7nYFklNNH/lAoIBAQDbyNQLaW5g +SAdlBRnVZ0zjVAgYtTy/qWKANS0nYiPHW1q654nnztWIUa0T15v5FfzNLdkexvbK +vgghVjATVEcLHll8276NQju63HYilUT4Lz/9OsDZKDGliOG+9dblAU6MpHy08K8x +xD3UE+X8DKYTkfyBoSptqdCDfuLdKHWMvSryolN0ongyssjMif4kUyKiaLU4Wx7x +a5nwf54cb+0ZzxaiwafD/JJqbtSyMXv1PeCeCChf02DyZNHgfoPnIGNJ7dqT+Tae +j917/YTUY6+JRdcDJs9x5IrrIwkiGZJJqVhsMv0m9V82bQXBmbwMY+F/ilpCnmrj +OGYGKWnRTeEJAoIBABgf9lfPMv8hpsLt5CQ9EmiM9KFuIFkd5olmZJ4bBjb38wEz +Hc4RlSmllQpRGA2mbCIDFm1F7oiogOwTnRUIomaaRJriGAEQ9ubjE7B1sld09BjW +K17O39UMA8X9uCAbUjHL3atIpb1Zk8Wae8UNZeOLdbtPdURzgawVbt1ywImnuLxR +iBBTlbtNz7kyavjxK46h5prWB9d+QbJlwLeyXgbXmP8UFA14cNbBJSX7lpXkOHCT +hsm0sXVzwN2ShzRR3r+HxdSK9ERrAwMrITQsURU4eo58rZKiZAKzjgfsz4NgfiW/ +PcYV1TZS0IzXLXaaaphT3gdVhQXi6wijWo34nkUCggEAZCDVQbGxoFmQTNyLDWb3 +Z2WkHqWK3IJtpp0TSiryw+MBrb7IW/wl2enj4PNMUqlKt6sYjGX2jx0OFSnv0w6F +IzKbcD+oSzCOh63igBTjC/Jyw4ody5D9NT3sIpRbZ4812ushCUnRdunBhTnff/m/ +O5E5qVDkRHulzBJlhn3lN84Cn/GF9dAC1I4Q3uZLCv94+uabEOaqbTApPKDXRntT +WHu5A0MYjDgn+Ccv7VKP94VOLJDo2+cv9p4p442fyA9ATLD48IsL8Cb4r4ErH5ue +n/paX0wyG+ATzdXzVj6yH67v4rNEE5ib3O7EPZQtULO8St+cCVekuIm8KTh6xekm +CQKCAQEAw0E87tJ/H2HArgCeFWzrYhPU5eqrtNf0Ylweu5WqoaaH334WT2POszpY +k+pbeW9rPxHtUXLwfE4t3/vCv+yhpUYCTqD0jBxx6VehoGorf7d6AsJrxDaDMTR1 +7ZXUDP+hWRhLPRtGfVU6c4+aPVrOk5Dh/Ty0hX3k2C6PeI0XQubs2KCrM3BE3rqW +2QOzV215i8BsYxRO74KoPFDpd1690TA5YA4neaVXmNyojrMxsgfE3kIW+KgsFTMm +YMU8TBw96tGxHXfefAqhfAzzRDsKGFLtFGfFWnpdfLt5NYVStG/jbBEWIQ4DB+w/ +gRlOmSYtTPvP/KWvkgZ8WifJ4xMCOw== +-----END PRIVATE KEY----- diff --git a/tests/gold_tests/tls/ssl/signed-foo-ec.key b/tests/gold_tests/tls/ssl/signed-foo-ec.key new file mode 100644 index 00000000000..cbf1418298c --- /dev/null +++ b/tests/gold_tests/tls/ssl/signed-foo-ec.key @@ -0,0 +1,8 @@ +-----BEGIN EC PARAMETERS----- +BggqhkjOPQMBBw== +-----END EC PARAMETERS----- +-----BEGIN EC PRIVATE KEY----- +MHcCAQEEIG2eKyPB3C9Efe1s7MQm6wdbFpAsIbpovesRCIJnYMb4oAoGCCqGSM49 +AwEHoUQDQgAEpz5axf97f4X/kx6PN8nyjkx5ree8nPmRI/TB0rq3e0ldH32hCrVQ +chq/mZzJCD3D/uuS27xJhl6t29JyABfAlw== +-----END EC PRIVATE KEY----- diff --git a/tests/gold_tests/tls/ssl/signed-foo-ec.pem b/tests/gold_tests/tls/ssl/signed-foo-ec.pem new file mode 100644 index 00000000000..c6f362b0733 --- /dev/null +++ b/tests/gold_tests/tls/ssl/signed-foo-ec.pem @@ -0,0 +1,22 @@ +-----BEGIN CERTIFICATE----- +MIICHjCCAYcCCQC81MtBCwmQvDANBgkqhkiG9w0BAQsFADCBnTELMAkGA1UEBhMC +VVMxCzAJBgNVBAgTAklMMRIwEAYDVQQHEwlDaGFtcGFpZ24xDjAMBgNVBAoTBVlh +aG9vMQ0wCwYDVQQLEwRFZGdlMSgwJgYDVQQDEx9qdWljZXByb2R1Y2UuY29ycC5u +ZTEueWFob28uY29tMSQwIgYJKoZIhvcNAQkBFhVwZXJzaWEuYXppekB5YWhvby5j +b20wHhcNMjAwMjE3MjIzOTEyWhcNMzAwMjE0MjIzOTEyWjBQMQswCQYDVQQGEwJV +UzELMAkGA1UECAwCSUwxEjAQBgNVBAcMCUNoYW1wYWlnbjEOMAwGA1UECgwFeWFo +b28xEDAOBgNVBAMMB2Zvby5jb20wWTATBgcqhkjOPQIBBggqhkjOPQMBBwNCAASn +PlrF/3t/hf+THo83yfKOTHmt57yc+ZEj9MHSurd7SV0ffaEKtVByGr+ZnMkIPcP+ +65LbvEmGXq3b0nIAF8CXMA0GCSqGSIb3DQEBCwUAA4GBAALTvnMS/uh2BmORNzh/ +MdxfJxmgJiAPGCclJGiAdAnRAUhR0i2XlSnkFiCzxbIc8rwv84beztmeRnnLUcJK +Qc4eSdrsHyfH3g8eFmzNW0sVDaYOiXVRReif4wQzO0mf8a3m5tBWcwBt2VucO0bL +Qh8dytlcF7egrVhXMVGHVwzk +-----END CERTIFICATE----- +-----BEGIN EC PARAMETERS----- +BggqhkjOPQMBBw== +-----END EC PARAMETERS----- +-----BEGIN EC PRIVATE KEY----- +MHcCAQEEIG2eKyPB3C9Efe1s7MQm6wdbFpAsIbpovesRCIJnYMb4oAoGCCqGSM49 +AwEHoUQDQgAEpz5axf97f4X/kx6PN8nyjkx5ree8nPmRI/TB0rq3e0ldH32hCrVQ +chq/mZzJCD3D/uuS27xJhl6t29JyABfAlw== +-----END EC PRIVATE KEY----- diff --git a/tests/gold_tests/tls/ssl/signed-foo.pem b/tests/gold_tests/tls/ssl/signed-foo.pem index 6f6aecf53d7..e3bf4cd29f1 100644 --- a/tests/gold_tests/tls/ssl/signed-foo.pem +++ b/tests/gold_tests/tls/ssl/signed-foo.pem @@ -17,3 +17,31 @@ m0yOR8w6MX8fxHKaekhJH1U84G64Ub0gbn2beOdLBQkG+4czLiOOOgyeukPaJJ81 od2ooE7DrGUPGnbHYxW/70EtVF5nQEctcqpKNF/d04mVKrqI90919MJSxJ5KedHK 2H11+gUPwDWy/mAwJzEJ -----END CERTIFICATE----- +-----BEGIN PRIVATE KEY----- +MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDJheOsr0o4C0QV +elo8viYl/IjWvkj4Y6YojnO/E9T5HSfWxkRikRT8X7yqs9ClrjNcYoPNNQP5vgl1 +qAIeixUrbZZklcZ3kHF/MXdzuYV//uT3YfWrsOTdfbcu6adHtBr/Qc0w9GSbBpp4 +2ZALxzjZ2hZZq51XmKvcz55/gm/9YvlGErzvsPzkejdH6U1GKeLqGaZOqkV7N2J4 +zKmJyiPzwrmR/8y7k+9jDHQX+A2wa5gaiAMGdIM8aTV+XsNNAvespvXyX7UhwxDs +w0er0GLv4ssYCru5hF+tUcjeOCHxSnCoEC45b2zDZHATLLjr7AIJ52TcSo1kTaTi +rAOAiHRLAgMBAAECggEAB7fXBnAYOZlE3EW5WwY1U9MeMotLJCg83uTFzhWmXHwf +YHxrdhL0aM4J3cfRP+cyFGG5hox3QINkvVrX6e+NugISdnu+BCpGDocIeigq0sIi +Zs8bp524xjrgXy2XuIlPV2NfxnY1vDI+jE5Y0/qnVMCjhn+qIQa53lUdTujh/SRR +3U7di+QMK4mdGwRnInos++ENy33A+2LqtUK8i0ERkzPFa1yMQEE4DOFPzZcW+jhK +arvzBwPIn37PZmL5oyiQB1YiGPGt4XNfPBwACTMYM8LlYBfEBHG77k3bMtUf0WqE +GctoT5SIe5+YbyrWkpfHgoKPxggH3I3TrFnVvqrKQQKBgQDmcV1YbuNEQLeif521 +iGqMgPQYmnpO6k27RsZrM9ikhIgm9bVJsOqnaYzQFeSfJ3eNLXYUL6IF8g46xddw +fDBtrEjDAA9OUkNRcizbeKF+GJRMtX11d4ZNbnG1wyMZYkArZGfraZBLHPEF1pya +2iFdVfokQCBpLmX7BMQEPePyuwKBgQDf33H9njf9oO0l9GfuWDvSoaV8GwqV9x55 +sFjggQYD/xqwEprrzr524X5Y2ZiTUpBu+kqqM8GYfm3bzBKkZU1rnjwxADUwBw8U +L2U/Z7Id3om8tAdzHOSI9d7mxWA8uTsScMm0IFv2l/XBQo1+AAJSD03pcsabr4Lf +SuJGmoFTsQKBgQDVzPASEC+DL5gwh75Gop5YZXwTJ5+6f+BGlM+avquNV/kKTIU6 +LY5IbMFcfjNzBicBMOCQsfDdG0rgdJYBovc7idCoOvH4dJJIimnb5fvPBfbxhKE1 +zwMn7ARL4xQ5hNKMb8eKvpJFXkCwbgE2GpNCCXbfEy/+5jFvx2gll1ZZ6QKBgG3J +OzJ/w796irHBQLKOzI+HvAq3jCJs9KICjCNUwql1EhZkmVqooZjVDkvuMbeVlsUF +s1XyWa852RAf7Mh38VakW6pACtVJsOhaMdG9PYkOWAeVVc3qzlwoDy6mfoJo6AIs +E45lDBRLAzbKN28h/AFYBgJEygcRNCHirEKphGCRAoGAEhcaxbmMo2fHBYuvOR1Q +ZAIq1EPvysDROUBHhdTJqN1wHsuJsmVJxX42+YHcZdjtgeCdjU3HMoyCnTaRxDee +K3VeB4PobN1WpQwFklFoqcvAhW6eicdZXme7ktK120NPQsXrmjgN6Lfg3PNjosn0 +tqSxQhQ4DrSf60fxx0/M/rw= +-----END PRIVATE KEY----- diff --git a/tests/gold_tests/tls/ssl/signed-san-ec.key b/tests/gold_tests/tls/ssl/signed-san-ec.key new file mode 100644 index 00000000000..8dd236ebce9 --- /dev/null +++ b/tests/gold_tests/tls/ssl/signed-san-ec.key @@ -0,0 +1,5 @@ +-----BEGIN EC PRIVATE KEY----- +MHcCAQEEIBh9PX40P1qCTpRxqbrxRyHsy1dAlEMZmUJpOZjgS2eXoAoGCCqGSM49 +AwEHoUQDQgAEBwvhzslwxPu7OdDlJFHpFoCTfa2zxqD7MI9AI5263g5Dov8Kcs1t +Kr1GTz3sunVCEOjVv5ASNokITXFu7+Bvcw== +-----END EC PRIVATE KEY----- diff --git a/tests/gold_tests/tls/ssl/signed-san-ec.pem b/tests/gold_tests/tls/ssl/signed-san-ec.pem new file mode 100644 index 00000000000..1030f0affe6 --- /dev/null +++ b/tests/gold_tests/tls/ssl/signed-san-ec.pem @@ -0,0 +1,20 @@ +-----BEGIN CERTIFICATE----- +MIICVTCCAb6gAwIBAgIJALzUy0ELCZDBMA0GCSqGSIb3DQEBCwUAMIGdMQswCQYD +VQQGEwJVUzELMAkGA1UECBMCSUwxEjAQBgNVBAcTCUNoYW1wYWlnbjEOMAwGA1UE +ChMFWWFob28xDTALBgNVBAsTBEVkZ2UxKDAmBgNVBAMTH2p1aWNlcHJvZHVjZS5j +b3JwLm5lMS55YWhvby5jb20xJDAiBgkqhkiG9w0BCQEWFXBlcnNpYS5heml6QHlh +aG9vLmNvbTAeFw0yMDAyMTgxNjM4MTZaFw0zMDAyMTUxNjM4MTZaMFkxCzAJBgNV +BAYTAlVTMQswCQYDVQQIDAJJTDESMBAGA1UEBwwJQ2hhbXBhaWduMRUwEwYDVQQK +DAxWZXJpem9uTWVkaWExEjAQBgNVBAMMCWdyb3VwLmNvbTBZMBMGByqGSM49AgEG +CCqGSM49AwEHA0IABAcL4c7JcMT7uznQ5SRR6RaAk32ts8ag+zCPQCOdut4OQ6L/ +CnLNbSq9Rk897Lp1QhDo1b+QEjaJCE1xbu/gb3OjJzAlMCMGA1UdEQQcMBqCB29u +ZS5jb22CB3R3by5jb22CBmVjLmNvbTANBgkqhkiG9w0BAQsFAAOBgQCRgoh5YGCc +V++/kil6USZaQ0TTNhAtCeao9p5WCN1NdHNtnulacu0cYPCI0cbpy2CVZC6JMrNE +21SFssxKaeM1yoyIDEjIkr5IaCCOnr5XdOAO6/eISapkIPE/1GEbxyEgk1yqTJkr +KB22uwprz1abKaQNBfTR2bV+57JlmnNuzg== +-----END CERTIFICATE----- +-----BEGIN EC PRIVATE KEY----- +MHcCAQEEIBh9PX40P1qCTpRxqbrxRyHsy1dAlEMZmUJpOZjgS2eXoAoGCCqGSM49 +AwEHoUQDQgAEBwvhzslwxPu7OdDlJFHpFoCTfa2zxqD7MI9AI5263g5Dov8Kcs1t +Kr1GTz3sunVCEOjVv5ASNokITXFu7+Bvcw== +-----END EC PRIVATE KEY----- diff --git a/tests/gold_tests/tls/ssl/signed-san.key b/tests/gold_tests/tls/ssl/signed-san.key new file mode 100644 index 00000000000..aee6da6528a --- /dev/null +++ b/tests/gold_tests/tls/ssl/signed-san.key @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQCtQayAL7EHSxEP +uclHEa4mMh/pO5N8VULdVA6sHN9k5mUAEnL3p3nNsdVE9pLtHIxgcoXCcLAPG+eL +bSAk+C/l+pubz8GAWHdOC/3xpdrzQjOiYzi2Megl6M+VpwaMOd0s14zdFZloS37y +M7GnjLKIRO8dMz744SSAPZYIvnx1XAJ/zA9uLbYzvFWbwkuT8G2+Jo+IHl4FyMX2 +wTTXLeYf66RKUnTFcNG7ysNbUr7hDJR8E/lek00K3t8YGzYrqCtHIH3OLkpKNU3l +HhWRQ0zrNRwfF+hWfYXq3SZ5+C1eo2RZe8SHPbNPbpJxGMMkxNRkCyWuMywoc2dA +nm1HyI2nAgMBAAECggEAKkhOyvHYqEj/nvDeWEPOVnABLbBma/960/0Bn6tkMYGw +wHXALQRoS3TM8Ymjjc5by+XnEu7haK6MsZAuOhd/yQaCF2J6fNIaO6fdj63EY32S +kFzaqExBtY69qm4awPoWKi1oqUPuLm/OSVmoT5WctHjuShgJlD+N4uYkyXmDcjhd +cED+Uy6LCK0TelhkN2J3TBMD1yI2bccORTVtihXWQhr7ROzXiyV5O143LFvSTNdr +l8FlOQbJhIxThUmydecONi1PYAkP/ySYFqLjLfCmonoVJFdjdgJsiXBEtUJ/8CQY ++ar2vqRL4/4Y+9eerlZFy3e6lidofPu0YGTWdCTcUQKBgQDSvBW5+61DZ8p9f3UX +04an8CyKgHDcH/lqKQMB8GvEhWCQG2nnrs6fQieiKj4P5bwuf01oNu4txxIg1GnS +r5MTZfzCEe89X06vcwzcTOQsMfUlzx7qMpQaqzshwchFLpCD2pj2vGIlL+g6piN0 ++X7RjRykBe0Nx5wYx3RYI78DCwKBgQDSeLuz2Q3n8Oqz+m9Zl+g1jMNgY861+zv5 +fZb43xXegef82RUata9ZGaRelODYdWmNiRGGAgfe1ddOW0xc3Ve1W9lonFLE84je +Oc57S2q9LiVuKJbhDQ6br1eGLv8RywsUhNdWRBIH9Mn3+1nIXAtlHt7Jp3lDeVO/ +WJa3vvyBVQKBgFDTMsISdXHU7SUVLaPlzU+8HllAygijetXsxOqJe8v0HAUpfoUN +1tHeXbUk3ojaZEKxMM83wkJsh9dvoObd0FswUrFcj5XKaDOCvPwBwcHxp0TJG+JX +Y9aWtidMW7OtGGB6BxEbT8lTho54CkFjL/DPXpzKaRFP7d7TIRxtGWXhAoGANjTw +KvbZNQaAfFAgw5NzM++IFlg+UfJd1Pj6nChgqokMpbuHSvTGL42CHvX7HuTGhbRq +tffp7QNoS38KINTFFSmNyfqQ+ra6Znm+61RWLlknPMLpcRb6zzAOu7l46i1AMk2w +ZEBt4Gy0Y9Dxo7/JE4cq3AbtHWqvHhYD41kmEW0CgYB2gL7IbeCeQG0wNOb0xHtu +aXQzK5JUOut811QJLNPzxS+G70bbSRC9gkqFAizCSvCqzao7/3pw9unQzaR0h/3V +OKEZF8angN9ORP5mOmlMQSvUtAZYfiuCMnZ4EeAhklA9hbAqGScevrdJUxnCN2c3 +DtR0mYMkmrdwISjW9aZnrg== +-----END PRIVATE KEY----- diff --git a/tests/gold_tests/tls/ssl/signed-san.pem b/tests/gold_tests/tls/ssl/signed-san.pem new file mode 100644 index 00000000000..3ab771cb5a8 --- /dev/null +++ b/tests/gold_tests/tls/ssl/signed-san.pem @@ -0,0 +1,47 @@ +-----BEGIN CERTIFICATE----- +MIIDITCCAoqgAwIBAgIJALzUy0ELCZDAMA0GCSqGSIb3DQEBCwUAMIGdMQswCQYD +VQQGEwJVUzELMAkGA1UECBMCSUwxEjAQBgNVBAcTCUNoYW1wYWlnbjEOMAwGA1UE +ChMFWWFob28xDTALBgNVBAsTBEVkZ2UxKDAmBgNVBAMTH2p1aWNlcHJvZHVjZS5j +b3JwLm5lMS55YWhvby5jb20xJDAiBgkqhkiG9w0BCQEWFXBlcnNpYS5heml6QHlh +aG9vLmNvbTAeFw0yMDAyMTgxNjM2MTZaFw0zMDAyMTUxNjM2MTZaMFkxCzAJBgNV +BAYTAlVTMQswCQYDVQQIDAJJTDESMBAGA1UEBwwJQ2hhbXBhaWduMRUwEwYDVQQK +DAxWZXJpem9uTWVkaWExEjAQBgNVBAMMCWdyb3VwLmNvbTCCASIwDQYJKoZIhvcN +AQEBBQADggEPADCCAQoCggEBAK1BrIAvsQdLEQ+5yUcRriYyH+k7k3xVQt1UDqwc +32TmZQAScvenec2x1UT2ku0cjGByhcJwsA8b54ttICT4L+X6m5vPwYBYd04L/fGl +2vNCM6JjOLYx6CXoz5WnBow53SzXjN0VmWhLfvIzsaeMsohE7x0zPvjhJIA9lgi+ +fHVcAn/MD24ttjO8VZvCS5Pwbb4mj4geXgXIxfbBNNct5h/rpEpSdMVw0bvKw1tS +vuEMlHwT+V6TTQre3xgbNiuoK0cgfc4uSko1TeUeFZFDTOs1HB8X6FZ9herdJnn4 +LV6jZFl7xIc9s09uknEYwyTE1GQLJa4zLChzZ0CebUfIjacCAwEAAaMoMCYwJAYD +VR0RBB0wG4IHb25lLmNvbYIHdHdvLmNvbYIHcnNhLmNvbTANBgkqhkiG9w0BAQsF +AAOBgQA00nnSb9iqOa8EPJrkbEasuAqe5gw7ehDgaVHLxUrWeJUPwNJdnbYK4hLw +qWeRKM6Qgxt8rjC/vqDjAxuNjHqFbdhL3supu2bHaBH5xFRqibY5rOY6AkL9SfMU +r8Lj/NQvqtIzoFM81rhSTDRoHNazVv0TjbcZKTAT25ARX4HQIw== +-----END CERTIFICATE----- +-----BEGIN PRIVATE KEY----- +MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQCtQayAL7EHSxEP +uclHEa4mMh/pO5N8VULdVA6sHN9k5mUAEnL3p3nNsdVE9pLtHIxgcoXCcLAPG+eL +bSAk+C/l+pubz8GAWHdOC/3xpdrzQjOiYzi2Megl6M+VpwaMOd0s14zdFZloS37y +M7GnjLKIRO8dMz744SSAPZYIvnx1XAJ/zA9uLbYzvFWbwkuT8G2+Jo+IHl4FyMX2 +wTTXLeYf66RKUnTFcNG7ysNbUr7hDJR8E/lek00K3t8YGzYrqCtHIH3OLkpKNU3l +HhWRQ0zrNRwfF+hWfYXq3SZ5+C1eo2RZe8SHPbNPbpJxGMMkxNRkCyWuMywoc2dA +nm1HyI2nAgMBAAECggEAKkhOyvHYqEj/nvDeWEPOVnABLbBma/960/0Bn6tkMYGw +wHXALQRoS3TM8Ymjjc5by+XnEu7haK6MsZAuOhd/yQaCF2J6fNIaO6fdj63EY32S +kFzaqExBtY69qm4awPoWKi1oqUPuLm/OSVmoT5WctHjuShgJlD+N4uYkyXmDcjhd +cED+Uy6LCK0TelhkN2J3TBMD1yI2bccORTVtihXWQhr7ROzXiyV5O143LFvSTNdr +l8FlOQbJhIxThUmydecONi1PYAkP/ySYFqLjLfCmonoVJFdjdgJsiXBEtUJ/8CQY ++ar2vqRL4/4Y+9eerlZFy3e6lidofPu0YGTWdCTcUQKBgQDSvBW5+61DZ8p9f3UX +04an8CyKgHDcH/lqKQMB8GvEhWCQG2nnrs6fQieiKj4P5bwuf01oNu4txxIg1GnS +r5MTZfzCEe89X06vcwzcTOQsMfUlzx7qMpQaqzshwchFLpCD2pj2vGIlL+g6piN0 ++X7RjRykBe0Nx5wYx3RYI78DCwKBgQDSeLuz2Q3n8Oqz+m9Zl+g1jMNgY861+zv5 +fZb43xXegef82RUata9ZGaRelODYdWmNiRGGAgfe1ddOW0xc3Ve1W9lonFLE84je +Oc57S2q9LiVuKJbhDQ6br1eGLv8RywsUhNdWRBIH9Mn3+1nIXAtlHt7Jp3lDeVO/ +WJa3vvyBVQKBgFDTMsISdXHU7SUVLaPlzU+8HllAygijetXsxOqJe8v0HAUpfoUN +1tHeXbUk3ojaZEKxMM83wkJsh9dvoObd0FswUrFcj5XKaDOCvPwBwcHxp0TJG+JX +Y9aWtidMW7OtGGB6BxEbT8lTho54CkFjL/DPXpzKaRFP7d7TIRxtGWXhAoGANjTw +KvbZNQaAfFAgw5NzM++IFlg+UfJd1Pj6nChgqokMpbuHSvTGL42CHvX7HuTGhbRq +tffp7QNoS38KINTFFSmNyfqQ+ra6Znm+61RWLlknPMLpcRb6zzAOu7l46i1AMk2w +ZEBt4Gy0Y9Dxo7/JE4cq3AbtHWqvHhYD41kmEW0CgYB2gL7IbeCeQG0wNOb0xHtu +aXQzK5JUOut811QJLNPzxS+G70bbSRC9gkqFAizCSvCqzao7/3pw9unQzaR0h/3V +OKEZF8angN9ORP5mOmlMQSvUtAZYfiuCMnZ4EeAhklA9hbAqGScevrdJUxnCN2c3 +DtR0mYMkmrdwISjW9aZnrg== +-----END PRIVATE KEY----- diff --git a/tests/gold_tests/tls/ssl/signer.pem b/tests/gold_tests/tls/ssl/signer.pem index 58b9b9715b7..111cd079718 100644 --- a/tests/gold_tests/tls/ssl/signer.pem +++ b/tests/gold_tests/tls/ssl/signer.pem @@ -1,18 +1,3 @@ ------BEGIN RSA PRIVATE KEY----- -MIICXQIBAAKBgQDWMHOiUF+ORmZjAxI8MWE9dblb7gQSJ36WCXlPFiFx6ynF+S1E -kXAYpIip5X0pzDUaIbLukxJUAAnOtMEO0PCgxJQUrEtRWh8wiJdbdQJF0Zs/9R+u -SUgb61f+mdTQvhqefBGx+xrpfAcgtcWiZuSA9Q3fvpDj5WOWSPWXBUuxywIDAQAB -AoGBAJPxRX2gjFAGWmQbU/YVmXfNH6navh8X/nx9sLeqrpE0AFeJI/ZPiqDKzMal -B43eSfNxwVi+ZxN0L1ICUbL9KKZvHs/QBxWLA1fGVAXrz7sRplEVvakPpTfHoEnv -sKaMWVKaK/S5WGbDhElb6zb/Lwo19DsIAPjGYqFvzFJBmobJAkEA9iSeTGkR9X26 -GywZoYrIMlRh34htOIRx1UUq88rFzdrCF21kQ4lhBIkX5OZMMy652i2gyak4OZTe -YewIv8jw9QJBAN7EQNHG8jPwXfVp91/fqxVQEfumuP2i6uiWWYQgZCmla2+0xcLZ -pMQ6sQEe10hhTrVnzHgAUVp50Ntn2jwBX78CQF09veGAI9d1Cxzj9cmmAvRd1r2Q -tp8kPOLnUsALXib+6WtqewLCdcf8DtsdClyRJMIraq85tRzK8fryKNZNzkkCQEgA -yS7FDj5JgCU15hZgFk1iPx3HCt44jZM2HaL+UUHAzRQjKxTLAl3G1rWVAWLMyQML -lORoveLvotl4HOruSsMCQQCAx9dV9JUSFoyc1CWILp/FgUH/se4cjQCThGO0DoQQ -vGTYmntY7j9WRJ9esQrjdD6Clw8zM/45GIBNwnXzqo7Z ------END RSA PRIVATE KEY----- -----BEGIN CERTIFICATE----- MIICszCCAhwCCQD4jSkztmlO1TANBgkqhkiG9w0BAQsFADCBnTELMAkGA1UEBhMC VVMxCzAJBgNVBAgTAklMMRIwEAYDVQQHEwlDaGFtcGFpZ24xDjAMBgNVBAoTBVlh diff --git a/tests/gold_tests/tls/test-0rtt-s_client.py b/tests/gold_tests/tls/test-0rtt-s_client.py new file mode 100644 index 00000000000..d2a90447c87 --- /dev/null +++ b/tests/gold_tests/tls/test-0rtt-s_client.py @@ -0,0 +1,75 @@ +''' +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import subprocess +import sys +import os +import shlex +import h2_early_decode + + +def main(): + ats_port = sys.argv[1] + http_ver = sys.argv[2] + test = sys.argv[3] + sess_file_path = os.path.join(sys.argv[4], 'sess.dat') + early_data_file_path = os.path.join(sys.argv[4], 'early_{0}_{1}.txt'.format(http_ver, test)) + + s_client_cmd_1 = shlex.split( + 'openssl s_client -connect 127.0.0.1:{0} -tls1_3 -quiet -sess_out {1}'.format(ats_port, sess_file_path)) + s_client_cmd_2 = shlex.split( + 'openssl s_client -connect 127.0.0.1:{0} -tls1_3 -quiet -sess_in {1} -early_data {2}'.format(ats_port, sess_file_path, early_data_file_path)) + + create_sess_proc = subprocess.Popen(s_client_cmd_1, env=os.environ.copy( + ), stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + try: + output = create_sess_proc.communicate(timeout=1)[0] + except subprocess.TimeoutExpired: + create_sess_proc.kill() + output = create_sess_proc.communicate()[0] + + reuse_sess_proc = subprocess.Popen(s_client_cmd_2, env=os.environ.copy( + ), stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + try: + output = reuse_sess_proc.communicate(timeout=1)[0] + except subprocess.TimeoutExpired: + reuse_sess_proc.kill() + output = reuse_sess_proc.communicate()[0] + + if http_ver == 'h2': + lines = output.split(bytes('\n', 'utf-8')) + data = b'' + for line in lines: + line += b'\n' + if line.startswith(bytes('SSL_connect:', 'utf-8')) or \ + line.startswith(bytes('SSL3 alert', 'utf-8')) or \ + bytes('Can\'t use SSL_get_servername', 'utf-8') in line: + continue + data += line + d = h2_early_decode.Decoder() + frames = d.decode(data) + for frame in frames: + print(frame) + else: + print(output.decode('utf-8')) + + exit(0) + + +if __name__ == '__main__': + main() diff --git a/tests/gold_tests/tls/test-nc-s_client.sh b/tests/gold_tests/tls/test-nc-s_client.sh index a11a80c9329..8aaf1192987 100644 --- a/tests/gold_tests/tls/test-nc-s_client.sh +++ b/tests/gold_tests/tls/test-nc-s_client.sh @@ -15,5 +15,5 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -nc -l -p $1 -c 'echo -e "This is a reply"' -o test.out & -echo "This is a test" | openssl s_client -servername bar.com -connect localhost:$2 -ign_eof +nc -l -p $1 -c 'echo -e "This is a reply"' -o test.out & +echo "This is a test" | openssl s_client -servername bar.com -connect localhost:$2 -ign_eof diff --git a/tests/gold_tests/tls/tls.test.py b/tests/gold_tests/tls/tls.test.py index ad3fcb798d8..3e31200cde3 100644 --- a/tests/gold_tests/tls/tls.test.py +++ b/tests/gold_tests/tls/tls.test.py @@ -16,7 +16,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import os Test.Summary = ''' Test tls ''' @@ -25,9 +24,9 @@ ts = Test.MakeATSProcess("ts", select_ports=True, enable_tls=True) server = Test.MakeOriginServer("server") -# build test code -tr=Test.Build(target='ssl-post',sources=['ssl-post.c']) -tr.Setup.Copy('ssl-post.c') +# ssl-post is built via `make`. Here we copy the built binary down to the test +# directory so that the test runs in this file can use it. +Test.Setup.Copy('ssl-post') requestLocation = "test2" reHost = "www.example.com" @@ -49,8 +48,12 @@ # Add info the origin server responses server.addResponse("sessionlog.json", - {"headers": header_string, "timestamp": "1469733493.993", "body": post_body}, - {"headers": "HTTP/1.1 200 OK\r\nServer: microserver\r\nConnection: close\r\nCache-Control: max-age=3600\r\nContent-Length: 2\r\n\r\n", "timestamp": "1469733493.993", "body": "ok"}) + {"headers": header_string, + "timestamp": "1469733493.993", + "body": post_body}, + {"headers": "HTTP/1.1 200 OK\r\nServer: microserver\r\nConnection: close\r\nCache-Control: max-age=3600\r\nContent-Length: 2\r\n\r\n", + "timestamp": "1469733493.993", + "body": "ok"}) # add ssl materials like key, certificates for the server ts.addSSLfile("ssl/server.pem") @@ -63,13 +66,8 @@ ts.Disk.ssl_multicert_config.AddLine( 'dest_ip=* ssl_cert_name=server.pem ssl_key_name=server.key' ) -ts.Disk.records_config.update({ - 'proxy.config.ssl.server.cert.path': '{0}'.format(ts.Variables.SSLDir), - 'proxy.config.ssl.server.private_key.path': '{0}'.format(ts.Variables.SSLDir), - 'proxy.config.ssl.client.verify.server': 0, - 'proxy.config.exec_thread.autoconfig.scale': 1.0, - 'proxy.config.ssl.server.cipher_suite': 'ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA384:AES128-GCM-SHA256:AES256-GCM-SHA384:ECDHE-RSA-RC4-SHA:ECDHE-RSA-AES128-SHA:ECDHE-RSA-AES256-SHA:RC4-SHA:RC4-MD5:AES128-SHA:AES256-SHA:DES-CBC3-SHA!SRP:!DSS:!PSK:!aNULL:!eNULL:!SSLv2', -}) +ts.Disk.records_config.update({'proxy.config.ssl.server.cert.path': '{0}'.format(ts.Variables.SSLDir), 'proxy.config.ssl.server.private_key.path': '{0}'.format(ts.Variables.SSLDir), 'proxy.config.exec_thread.autoconfig.scale': 1.0, 'proxy.config.ssl.server.cipher_suite': + 'ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA384:AES128-GCM-SHA256:AES256-GCM-SHA384:ECDHE-RSA-RC4-SHA:ECDHE-RSA-AES128-SHA:ECDHE-RSA-AES256-SHA:RC4-SHA:RC4-MD5:AES128-SHA:AES256-SHA:DES-CBC3-SHA!SRP:!DSS:!PSK:!aNULL:!eNULL:!SSLv2', }) tr = Test.AddTestRun("Run-Test") tr.Command = './ssl-post 127.0.0.1 40 {0} {1}'.format(header_count, ts.Variables.ssl_port) diff --git a/tests/gold_tests/tls/tls_0rtt_server.test.py b/tests/gold_tests/tls/tls_0rtt_server.test.py new file mode 100644 index 00000000000..9b96341198c --- /dev/null +++ b/tests/gold_tests/tls/tls_0rtt_server.test.py @@ -0,0 +1,199 @@ +''' +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +Test.Summary = ''' +Test ATS TLSv1.3 0-RTT support +''' + +Test.SkipUnless(Condition.HasOpenSSLVersion('1.1.1')) + +ts = Test.MakeATSProcess('ts', select_ports=True, enable_tls=True) +server = Test.MakeOriginServer('server') + +request_header1 = { + 'headers': 'GET / HTTP/1.1\r\nHost: www.example.com\r\n\r\n', + 'timestamp': '1469733493.993', + 'body': '' +} +response_header1 = { + 'headers': 'HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n', + 'timestamp': '1469733493.993', + 'body': 'curl test' +} +request_header2 = { + 'headers': 'GET /early_get HTTP/1.1\r\nHost: www.example.com\r\n\r\n', + 'timestamp': '1469733493.993', + 'body': '' +} +response_header2 = { + 'headers': 'HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n', + 'timestamp': '1469733493.993', + 'body': 'early data accepted' +} +request_header3 = { + 'headers': 'POST /early_post HTTP/1.1\r\nHost: www.example.com\r\nContent-Length: 11\r\n\r\n', + 'timestamp': '1469733493.993', + 'body': 'knock knock' +} +response_header3 = { + 'headers': 'HTTP/1.1 200 OK\r\nServer: uServer\r\nConnection: close\r\nTransfer-Encoding: chunked\r\n\r\n', + 'timestamp': '1415926535.898', + 'body': '' +} +request_header4 = { + 'headers': 'GET /early_multi_1 HTTP/1.1\r\nHost: www.example.com\r\n\r\n', + 'timestamp': '1469733493.993', + 'body': '' +} +response_header4 = { + 'headers': 'HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n', + 'timestamp': '1469733493.993', + 'body': 'early data accepted multi_1' +} +request_header5 = { + 'headers': 'GET /early_multi_2 HTTP/1.1\r\nHost: www.example.com\r\n\r\n', + 'timestamp': '1469733493.993', + 'body': '' +} +response_header5 = { + 'headers': 'HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n', + 'timestamp': '1469733493.993', + 'body': 'early data accepted multi_2' +} +request_header6 = { + 'headers': 'GET /early_multi_3 HTTP/1.1\r\nHost: www.example.com\r\n\r\n', + 'timestamp': '1469733493.993', + 'body': '' +} +response_header6 = { + 'headers': 'HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n', + 'timestamp': '1469733493.993', + 'body': 'early data accepted multi_3' +} +server.addResponse('sessionlog.json', request_header1, response_header1) +server.addResponse('sessionlog.json', request_header2, response_header2) +server.addResponse('sessionlog.json', request_header3, response_header3) +server.addResponse('sessionlog.json', request_header4, response_header4) +server.addResponse('sessionlog.json', request_header5, response_header5) +server.addResponse('sessionlog.json', request_header6, response_header6) + +ts.addSSLfile('ssl/server.pem') +ts.addSSLfile('ssl/server.key') + +ts.Setup.Copy('test-0rtt-s_client.py') +ts.Setup.Copy('h2_early_decode.py') +ts.Setup.Copy('early_h1_get.txt') +ts.Setup.Copy('early_h1_post.txt') +ts.Setup.Copy('early_h2_get.txt') +ts.Setup.Copy('early_h2_post.txt') +ts.Setup.Copy('early_h2_multi1.txt') +ts.Setup.Copy('early_h2_multi2.txt') + +ts.Disk.records_config.update({ + 'proxy.config.diags.debug.enabled': 1, + 'proxy.config.diags.debug.tags': 'http', + 'proxy.config.exec_thread.autoconfig': 0, + 'proxy.config.exec_thread.limit': 8, + 'proxy.config.http.server_ports': '{0}:proto=http2;http:ssl'.format(ts.Variables.ssl_port), + 'proxy.config.ssl.server.cert.path': '{0}'.format(ts.Variables.SSLDir), + 'proxy.config.ssl.server.private_key.path': '{0}'.format(ts.Variables.SSLDir), + 'proxy.config.ssl.session_cache': 2, + 'proxy.config.ssl.session_cache.size': 512000, + 'proxy.config.ssl.session_cache.timeout': 7200, + 'proxy.config.ssl.session_cache.num_buckets': 32768, + 'proxy.config.ssl.server.session_ticket.enable': 1, + 'proxy.config.ssl.server.max_early_data': 16384, + 'proxy.config.ssl.server.allow_early_data_params': 0, + 'proxy.config.ssl.server.cipher_suite': 'ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:DHE-DSS-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-DSS-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES128-SHA:DHE-RSA-AES256-SHA256:DHE-DSS-AES256-SHA256:DHE-RSA-AES128-SHA256:DHE-DSS-AES128-SHA256:DHE-RSA-AES256-SHA:DHE-DSS-AES256-SHA:DHE-RSA-AES128-SHA:DHE-DSS-AES128-SHA:!aNULL:!eNULL:!EXPORT:!DES:!RC4:!MD5:!PSK:!aECDH:!EDH-DSS-DES-CBC3-SHA:!EDH-RSA-DES-CBC3-SHA:!KRB5-DES-CBC3-SHA' +}) + +ts.Disk.ssl_multicert_config.AddLine( + 'dest_ip=* ssl_cert_name=server.pem ssl_key_name=server.key' +) + +ts.Disk.remap_config.AddLine( + 'map / http://127.0.0.1:{0}'.format(server.Variables.Port) +) + +tr = Test.AddTestRun('Basic Curl Test') +tr.Processes.Default.Command = 'curl https://127.0.0.1:{0} -k'.format(ts.Variables.ssl_port) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.StartBefore(server) +tr.Processes.Default.StartBefore(Test.Processes.ts, ready=When.PortOpen(ts.Variables.ssl_port)) +tr.Processes.Default.Streams.All = Testers.ContainsExpression('curl test', 'Making sure the basics still work') +tr.Processes.Default.Streams.All += Testers.ExcludesExpression('early data accepted', '') +tr.StillRunningAfter = server +tr.StillRunningAfter += ts + +tr = Test.AddTestRun('TLSv1.3 0-RTT Support (HTTP/1.1 GET)') +tr.Processes.Default.Command = 'python3 test-0rtt-s_client.py {0} {1} {2} {3}'.format( + ts.Variables.ssl_port, 'h1', 'get', Test.RunDirectory) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Streams.All = Testers.ContainsExpression('early data accepted', '') +tr.Processes.Default.Streams.All += Testers.ExcludesExpression('curl test', '') +tr.StillRunningAfter = server +tr.StillRunningAfter += ts + +tr = Test.AddTestRun('TLSv1.3 0-RTT Support (HTTP/1.1 POST)') +tr.Processes.Default.Command = 'python3 test-0rtt-s_client.py {0} {1} {2} {3}'.format( + ts.Variables.ssl_port, 'h1', 'post', Test.RunDirectory) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Streams.All = Testers.ContainsExpression('HTTP/1.1 425 Too Early', '') +tr.Processes.Default.Streams.All += Testers.ExcludesExpression('curl test', '') +tr.Processes.Default.Streams.All += Testers.ExcludesExpression('early data accepted', '') +tr.StillRunningAfter = server +tr.StillRunningAfter += ts + +tr = Test.AddTestRun('TLSv1.3 0-RTT Support (HTTP/2 GET)') +tr.Processes.Default.Command = 'python3 test-0rtt-s_client.py {0} {1} {2} {3}'.format( + ts.Variables.ssl_port, 'h2', 'get', Test.RunDirectory) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Streams.All = Testers.ContainsExpression('early data accepted', '') +tr.Processes.Default.Streams.All += Testers.ExcludesExpression('curl test', '') +tr.StillRunningAfter = server +tr.StillRunningAfter += ts + +tr = Test.AddTestRun('TLSv1.3 0-RTT Support (HTTP/2 POST)') +tr.Processes.Default.Command = 'python3 test-0rtt-s_client.py {0} {1} {2} {3}'.format( + ts.Variables.ssl_port, 'h2', 'post', Test.RunDirectory) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Streams.All = Testers.ContainsExpression(':status 425', 'Only safe methods are allowed') +tr.Processes.Default.Streams.All += Testers.ExcludesExpression('curl test', '') +tr.Processes.Default.Streams.All += Testers.ExcludesExpression('early data accepted', '') +tr.StillRunningAfter = server +tr.StillRunningAfter += ts + +tr = Test.AddTestRun('TLSv1.3 0-RTT Support (HTTP/2 Multiplex)') +tr.Processes.Default.Command = 'python3 test-0rtt-s_client.py {0} {1} {2} {3}'.format( + ts.Variables.ssl_port, 'h2', 'multi1', Test.RunDirectory) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Streams.All = Testers.ContainsExpression('early data accepted multi_1', '') +tr.Processes.Default.Streams.All += Testers.ContainsExpression('early data accepted multi_2', '') +tr.Processes.Default.Streams.All += Testers.ContainsExpression('early data accepted multi_3', '') +tr.Processes.Default.Streams.All += Testers.ExcludesExpression('curl test', '') +tr.StillRunningAfter = server +tr.StillRunningAfter += ts + +tr = Test.AddTestRun('TLSv1.3 0-RTT Support (HTTP/2 Multiplex with POST)') +tr.Processes.Default.Command = 'python3 test-0rtt-s_client.py {0} {1} {2} {3}'.format( + ts.Variables.ssl_port, 'h2', 'multi2', Test.RunDirectory) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Streams.All = Testers.ContainsExpression('early data accepted multi_1', '') +tr.Processes.Default.Streams.All += Testers.ContainsExpression(':status 425', 'Only safe methods are allowed') +tr.Processes.Default.Streams.All += Testers.ContainsExpression('early data accepted multi_3', '') +tr.Processes.Default.Streams.All += Testers.ExcludesExpression('curl test', '') diff --git a/tests/gold_tests/tls/tls_check_cert_selection.test.py b/tests/gold_tests/tls/tls_check_cert_selection.test.py index 67e76f4d8a5..7d091b1dd07 100644 --- a/tests/gold_tests/tls/tls_check_cert_selection.test.py +++ b/tests/gold_tests/tls/tls_check_cert_selection.test.py @@ -16,7 +16,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import os Test.Summary = ''' Test ATS offering different certificates based on SNI ''' @@ -71,7 +70,8 @@ tr = Test.AddTestRun("bar.com cert") tr.Setup.Copy("ssl/signer.pem") tr.Setup.Copy("ssl/signer2.pem") -tr.Processes.Default.Command = "curl -v --cacert ./signer2.pem --resolve 'bar.com:{0}:127.0.0.1' https://bar.com:{0}".format(ts.Variables.ssl_port) +tr.Processes.Default.Command = "curl -v --cacert ./signer2.pem --resolve 'bar.com:{0}:127.0.0.1' https://bar.com:{0}".format( + ts.Variables.ssl_port) tr.ReturnCode = 0 tr.Processes.Default.StartBefore(server) tr.Processes.Default.StartBefore(dns) @@ -85,7 +85,8 @@ # Should receive a foo.com cert tr2 = Test.AddTestRun("foo.com cert") -tr2.Processes.Default.Command = "curl -v --cacert ./signer.pem --resolve 'foo.com:{0}:127.0.0.1' https://foo.com:{0}".format(ts.Variables.ssl_port) +tr2.Processes.Default.Command = "curl -v --cacert ./signer.pem --resolve 'foo.com:{0}:127.0.0.1' https://foo.com:{0}".format( + ts.Variables.ssl_port) tr2.ReturnCode = 0 tr2.StillRunningAfter = server tr2.StillRunningAfter = ts @@ -96,7 +97,8 @@ # Should receive random.server.com tr2 = Test.AddTestRun("random.server.com cert") -tr2.Processes.Default.Command = "curl -v -k --resolve 'random.server.com:{0}:127.0.0.1' https://random.server.com:{0}".format(ts.Variables.ssl_port) +tr2.Processes.Default.Command = "curl -v -k --resolve 'random.server.com:{0}:127.0.0.1' https://random.server.com:{0}".format( + ts.Variables.ssl_port) tr2.ReturnCode = 0 tr2.StillRunningAfter = server tr2.StillRunningAfter = ts @@ -109,7 +111,8 @@ # No SNI match should match specific IP address, foo.com # SNI name and returned cert name will not match, so must use -k to avoid cert verification tr2 = Test.AddTestRun("Bad SNI") -tr2.Processes.Default.Command = "curl -v -k --cacert ./signer.pem --resolve 'bad.sni.com:{0}:127.0.0.1' https://bad.sni.com:{0}".format(ts.Variables.ssl_port) +tr2.Processes.Default.Command = "curl -v -k --cacert ./signer.pem --resolve 'bad.sni.com:{0}:127.0.0.1' https://bad.sni.com:{0}".format( + ts.Variables.ssl_port) tr2.ReturnCode = 0 tr2.StillRunningAfter = server tr2.StillRunningAfter = ts diff --git a/tests/gold_tests/tls/tls_check_dual_cert_selection.test.py b/tests/gold_tests/tls/tls_check_dual_cert_selection.test.py new file mode 100644 index 00000000000..9256e2e5355 --- /dev/null +++ b/tests/gold_tests/tls/tls_check_dual_cert_selection.test.py @@ -0,0 +1,156 @@ +''' +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import re + +Test.Summary = ''' +Test ATS offering both RSA and EC certificates +''' + +# Define default ATS +ts = Test.MakeATSProcess("ts", select_ports=True, enable_tls=True) +server = Test.MakeOriginServer("server", ssl=True) +dns = Test.MakeDNServer("dns") + +request_header = {"headers": "GET / HTTP/1.1\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +response_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +server.addResponse("sessionlog.json", request_header, response_header) + +# add ssl materials like key, certificates for the server +ts.addSSLfile("ssl/signed-foo.pem") +ts.addSSLfile("ssl/signed-foo.key") +ts.addSSLfile("ssl/signed-foo-ec.pem") +ts.addSSLfile("ssl/signed-foo-ec.key") +ts.addSSLfile("ssl/signed-san.pem") +ts.addSSLfile("ssl/signed-san.key") +ts.addSSLfile("ssl/signed-san-ec.pem") +ts.addSSLfile("ssl/signed-san-ec.key") +ts.addSSLfile("ssl/signer.pem") +ts.addSSLfile("ssl/signer.key") +ts.addSSLfile("ssl/server.pem") +ts.addSSLfile("ssl/server.key") + +ts.Disk.remap_config.AddLine( + 'map / https://foo.com:{1}'.format(ts.Variables.ssl_port, server.Variables.SSL_Port)) + +ts.Disk.ssl_multicert_config.AddLines([ + 'ssl_cert_name=signed-foo-ec.pem,signed-foo.pem ssl_key_name=signed-foo-ec.key,signed-foo.key', + 'ssl_cert_name=signed-san-ec.pem,signed-san.pem ssl_key_name=signed-san-ec.key,signed-san.key', + 'dest_ip=* ssl_cert_name=server.pem ssl_key_name=server.key' +]) + +# Case 1, global config policy=permissive properties=signature +# override for foo.com policy=enforced properties=all +ts.Disk.records_config.update({ + 'proxy.config.ssl.server.cert.path': '{0}'.format(ts.Variables.SSLDir), + 'proxy.config.ssl.server.private_key.path': '{0}'.format(ts.Variables.SSLDir), + 'proxy.config.ssl.server.cipher_suite': 'ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256', + 'proxy.config.url_remap.pristine_host_hdr': 1, + 'proxy.config.dns.nameservers': '127.0.0.1:{0}'.format(dns.Variables.Port), + 'proxy.config.exec_thread.autoconfig.scale': 1.0, + 'proxy.config.dns.resolv_conf': 'NULL', + 'proxy.config.diags.debug.tags': 'ssl', + 'proxy.config.diags.debug.enabled': 1 +}) + +dns.addRecords(records={"foo.com.": ["127.0.0.1"]}) +dns.addRecords(records={"bar.com.": ["127.0.0.1"]}) + +foo_ec_string = "" +foo_rsa_string = "" +san_ec_string = "" +san_rsa_string = "" +with open(os.path.join(Test.TestDirectory, 'ssl', 'signed-foo-ec.pem'), 'r') as myfile: + file_string = myfile.read() + cert_end = file_string.find("END CERTIFICATE-----") + foo_ec_string = re.escape(file_string[0:cert_end]) +with open(os.path.join(Test.TestDirectory, 'ssl', 'signed-foo.pem'), 'r') as myfile: + file_string = myfile.read() + cert_end = file_string.find("END CERTIFICATE-----") + foo_rsa_string = re.escape(file_string[0:cert_end]) +with open(os.path.join(Test.TestDirectory, 'ssl', 'signed-san-ec.pem'), 'r') as myfile: + file_string = myfile.read() + cert_end = file_string.find("END CERTIFICATE-----") + san_ec_string = re.escape(file_string[0:cert_end]) +with open(os.path.join(Test.TestDirectory, 'ssl', 'signed-san.pem'), 'r') as myfile: + file_string = myfile.read() + cert_end = file_string.find("END CERTIFICATE-----") + san_rsa_string = re.escape(file_string[0:cert_end]) + +# Should receive a EC cert since ATS cipher list prefers EC +tr = Test.AddTestRun("Default for foo should return EC cert") +tr.Setup.Copy("ssl/signer.pem") +tr.Processes.Default.Command = "echo foo | openssl s_client -tls1_2 -servername foo.com -connect 127.0.0.1:{0}".format( + ts.Variables.ssl_port, foo_ec_string) +tr.ReturnCode = 0 +tr.Processes.Default.StartBefore(server) +tr.Processes.Default.StartBefore(dns) +tr.Processes.Default.StartBefore(Test.Processes.ts, ready=When.PortOpen(ts.Variables.ssl_port)) +tr.StillRunningAfter = server +tr.StillRunningAfter = ts +tr.Processes.Default.Streams.All += Testers.ContainsExpression(foo_ec_string, "Should select EC cert", reflags=re.S | re.M) + +# Should receive a RSA cert +tr = Test.AddTestRun("Only offer RSA ciphers, should receive RSA cert") +tr.Processes.Default.Command = "echo foo | openssl s_client -tls1_2 -servername foo.com -cipher 'ECDHE-RSA-AES128-GCM-SHA256' -connect 127.0.0.1:{0}".format( + ts.Variables.ssl_port) +tr.ReturnCode = 0 +tr.StillRunningAfter = server +tr.StillRunningAfter = ts +tr.Processes.Default.Streams.All += Testers.ContainsExpression(foo_rsa_string, "Should select RSA cert", reflags=re.S | re.M) + +# Should receive a EC cert +tr = Test.AddTestRun("Default for two.com should return EC cert") +tr.Processes.Default.Command = "echo foo | openssl s_client -tls1_2 -servername two.com -connect 127.0.0.1:{0}".format( + ts.Variables.ssl_port) +tr.ReturnCode = 0 +tr.StillRunningAfter = server +tr.StillRunningAfter = ts +tr.Processes.Default.Streams.All += Testers.ContainsExpression(san_ec_string, "Should select EC cert", reflags=re.S | re.M) +tr.Processes.Default.Streams.All += Testers.ContainsExpression("CN = group.com", "Should select a group SAN") + +# Should receive a RSA cert +tr = Test.AddTestRun("Only offer RSA ciphers, should receive RSA cert") +tr.Processes.Default.Command = "echo foo | openssl s_client -tls1_2 -servername two.com -cipher 'ECDHE-RSA-AES128-GCM-SHA256' -connect 127.0.0.1:{0}".format( + ts.Variables.ssl_port) +tr.ReturnCode = 0 +tr.StillRunningAfter = server +tr.StillRunningAfter = ts +tr.Processes.Default.Streams.All += Testers.ContainsExpression(san_rsa_string, "Should select RSA cert", reflags=re.S | re.M) +tr.Processes.Default.Streams.All += Testers.ContainsExpression("CN = group.com", "Should select a group SAN") + +# Should receive a RSA cert +tr = Test.AddTestRun("rsa.com only in rsa cert") +tr.Processes.Default.Command = "echo foo | openssl s_client -tls1_2 -servername rsa.com -connect 127.0.0.1:{0}".format( + ts.Variables.ssl_port) +tr.ReturnCode = 0 +tr.StillRunningAfter = server +tr.StillRunningAfter = ts +tr.Processes.Default.Streams.All += Testers.ContainsExpression(san_rsa_string, "Should select RSA cert", reflags=re.S | re.M) +tr.Processes.Default.Streams.All += Testers.ContainsExpression("CN = group.com", "Should select a group SAN") + +# Should receive a EC cert +tr = Test.AddTestRun("ec.com only in ec cert") +tr.Processes.Default.Command = "echo foo | openssl s_client -tls1_2 -servername ec.com -connect 127.0.0.1:{0}".format( + ts.Variables.ssl_port) +tr.ReturnCode = 0 +tr.StillRunningAfter = server +tr.StillRunningAfter = ts +tr.Processes.Default.Streams.All += Testers.ContainsExpression(san_ec_string, "Should select EC cert", reflags=re.S | re.M) +tr.Processes.Default.Streams.All += Testers.ContainsExpression("CN = group.com", "Should select a group SAN") diff --git a/tests/gold_tests/tls/tls_check_dual_cert_selection2.test.py b/tests/gold_tests/tls/tls_check_dual_cert_selection2.test.py new file mode 100644 index 00000000000..394068b7cea --- /dev/null +++ b/tests/gold_tests/tls/tls_check_dual_cert_selection2.test.py @@ -0,0 +1,187 @@ +''' +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import re + +Test.Summary = ''' +Test ATS offering both RSA and EC certificates +Combined key and cert files. Faulty key path +''' + +# Define default ATS +ts = Test.MakeATSProcess("ts", select_ports=True, enable_tls=True) +server = Test.MakeOriginServer("server", ssl=True) +dns = Test.MakeDNServer("dns") + +request_header = {"headers": "GET / HTTP/1.1\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +response_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +server.addResponse("sessionlog.json", request_header, response_header) + +# add ssl materials like key, certificates for the server +ts.addSSLfile("ssl/signed-foo.pem") +ts.addSSLfile("ssl/signed-foo.key") +ts.addSSLfile("ssl/signed-foo-ec.pem") +ts.addSSLfile("ssl/signed-foo-ec.key") +ts.addSSLfile("ssl/signed-san.pem") +ts.addSSLfile("ssl/signed-san.key") +ts.addSSLfile("ssl/signed-san-ec.pem") +ts.addSSLfile("ssl/signed-san-ec.key") +ts.addSSLfile("ssl/combined-ec.pem") +ts.addSSLfile("ssl/combined.pem") +ts.addSSLfile("ssl/signer.pem") +ts.addSSLfile("ssl/signer.key") + +ts.Disk.remap_config.AddLine( + 'map / https://foo.com:{1}'.format(ts.Variables.ssl_port, server.Variables.SSL_Port)) + +ts.Disk.ssl_multicert_config.AddLines([ + 'ssl_cert_name=combined-ec.pem,combined.pem', + 'ssl_cert_name=signed-foo-ec.pem,signed-foo.pem', + 'dest_ip=* ssl_cert_name=signed-san-ec.pem,signed-san.pem' +]) + +# Case 1, global config policy=permissive properties=signature +# override for foo.com policy=enforced properties=all +ts.Disk.records_config.update({ + 'proxy.config.ssl.server.cert.path': '{0}'.format(ts.Variables.SSLDir), + 'proxy.config.ssl.server.private_key.path': '/tmp', # Faulty key path should not matter, since there are no key files + 'proxy.config.ssl.server.cipher_suite': 'ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256', + 'proxy.config.url_remap.pristine_host_hdr': 1, + 'proxy.config.dns.nameservers': '127.0.0.1:{0}'.format(dns.Variables.Port), + 'proxy.config.exec_thread.autoconfig.scale': 1.0, + 'proxy.config.dns.resolv_conf': 'NULL', + 'proxy.config.diags.debug.tags': 'ssl', + 'proxy.config.diags.debug.enabled': 0 +}) + +dns.addRecords(records={"foo.com.": ["127.0.0.1"]}) +dns.addRecords(records={"bar.com.": ["127.0.0.1"]}) + +foo_ec_string = "" +foo_rsa_string = "" +san_ec_string = "" +san_rsa_string = "" +combo_ec_string = "" +combo_rsa_string = "" +with open(os.path.join(Test.TestDirectory, 'ssl', 'signed-foo-ec.pem'), 'r') as myfile: + file_string = myfile.read() + cert_end = file_string.find("END CERTIFICATE-----") + foo_ec_string = re.escape(file_string[0:cert_end]) +with open(os.path.join(Test.TestDirectory, 'ssl', 'signed-foo.pem'), 'r') as myfile: + file_string = myfile.read() + cert_end = file_string.find("END CERTIFICATE-----") + foo_rsa_string = re.escape(file_string[0:cert_end]) +with open(os.path.join(Test.TestDirectory, 'ssl', 'signed-san-ec.pem'), 'r') as myfile: + file_string = myfile.read() + cert_end = file_string.find("END CERTIFICATE-----") + san_ec_string = re.escape(file_string[0:cert_end]) +with open(os.path.join(Test.TestDirectory, 'ssl', 'signed-san.pem'), 'r') as myfile: + file_string = myfile.read() + cert_end = file_string.find("END CERTIFICATE-----") + san_rsa_string = re.escape(file_string[0:cert_end]) +with open(os.path.join(Test.TestDirectory, 'ssl', 'combined-ec.pem'), 'r') as myfile: + file_string = myfile.read() + cert_end = file_string.find("END CERTIFICATE-----") + combo_ec_string = re.escape(file_string[0: cert_end]) +with open(os.path.join(Test.TestDirectory, 'ssl', 'combined.pem'), 'r') as myfile: + file_string = myfile.read() + cert_end = file_string.find("END CERTIFICATE-----") + combo_rsa_string = re.escape(file_string[0: cert_end]) + +# Should receive a EC cert since ATS cipher list prefers EC +tr = Test.AddTestRun("Default for foo should return EC cert") +tr.Setup.Copy("ssl/signer.pem") +tr.Processes.Default.Command = "echo foo | openssl s_client -tls1_2 -servername foo.com -connect 127.0.0.1:{0}".format( + ts.Variables.ssl_port, foo_ec_string) +tr.ReturnCode = 0 +tr.Processes.Default.StartBefore(server) +tr.Processes.Default.StartBefore(dns) +tr.Processes.Default.StartBefore(Test.Processes.ts, ready=When.PortOpen(ts.Variables.ssl_port)) +tr.StillRunningAfter = server +tr.StillRunningAfter = ts +tr.Processes.Default.Streams.All += Testers.ContainsExpression(foo_ec_string, "Should select EC cert", reflags=re.S | re.M) + +# Should receive a RSA cert +tr = Test.AddTestRun("Only offer RSA ciphers, should receive RSA cert") +tr.Processes.Default.Command = "echo foo | openssl s_client -tls1_2 -servername foo.com -cipher 'ECDHE-RSA-AES128-GCM-SHA256' -connect 127.0.0.1:{0}".format( + ts.Variables.ssl_port) +tr.ReturnCode = 0 +tr.StillRunningAfter = server +tr.StillRunningAfter = ts +tr.Processes.Default.Streams.All += Testers.ContainsExpression(foo_rsa_string, "Should select RSA cert", reflags=re.S | re.M) + +# Should receive a EC cert +tr = Test.AddTestRun("Default for two.com should return EC cert") +tr.Processes.Default.Command = "echo foo | openssl s_client -tls1_2 -servername two.com -connect 127.0.0.1:{0}".format( + ts.Variables.ssl_port) +tr.ReturnCode = 0 +tr.StillRunningAfter = server +tr.StillRunningAfter = ts +tr.Processes.Default.Streams.All += Testers.ContainsExpression(san_ec_string, "Should select EC cert", reflags=re.S | re.M) +tr.Processes.Default.Streams.All += Testers.ContainsExpression("CN = group.com", "Should select a group SAN") + +# Should receive a RSA cert +tr = Test.AddTestRun("Only offer RSA ciphers, should receive RSA cert") +tr.Processes.Default.Command = "echo foo | openssl s_client -tls1_2 -servername two.com -cipher 'ECDHE-RSA-AES128-GCM-SHA256' -connect 127.0.0.1:{0}".format( + ts.Variables.ssl_port) +tr.ReturnCode = 0 +tr.StillRunningAfter = server +tr.StillRunningAfter = ts +tr.Processes.Default.Streams.All += Testers.ContainsExpression(san_rsa_string, "Should select RSA cert", reflags=re.S | re.M) +tr.Processes.Default.Streams.All += Testers.ContainsExpression("CN = group.com", "Should select a group SAN") + +# Should receive a RSA cert +tr = Test.AddTestRun("rsa.com only in rsa cert") +tr.Processes.Default.Command = "echo foo | openssl s_client -tls1_2 -servername rsa.com -connect 127.0.0.1:{0}".format( + ts.Variables.ssl_port) +tr.ReturnCode = 0 +tr.StillRunningAfter = server +tr.StillRunningAfter = ts +tr.Processes.Default.Streams.All += Testers.ContainsExpression(san_rsa_string, "Should select RSA cert", reflags=re.S | re.M) +tr.Processes.Default.Streams.All += Testers.ContainsExpression("CN = group.com", "Should select a group SAN") + +# Should receive a EC cert +tr = Test.AddTestRun("ec.com only in ec cert") +tr.Processes.Default.Command = "echo foo | openssl s_client -tls1_2 -servername ec.com -connect 127.0.0.1:{0}".format( + ts.Variables.ssl_port) +tr.ReturnCode = 0 +tr.StillRunningAfter = server +tr.StillRunningAfter = ts +tr.Processes.Default.Streams.All += Testers.ContainsExpression(san_ec_string, "Should select EC cert", reflags=re.S | re.M) +tr.Processes.Default.Streams.All += Testers.ContainsExpression("CN = group.com", "Should select a group SAN") + +# Should receive a EC cert +tr = Test.AddTestRun("Default for combined.com should return EC cert") +tr.Processes.Default.Command = "echo foo | openssl s_client -tls1_2 -servername combined.com -connect 127.0.0.1:{0}".format( + ts.Variables.ssl_port) +tr.ReturnCode = 0 +tr.StillRunningAfter = server +tr.StillRunningAfter = ts +tr.Processes.Default.Streams.All += Testers.ContainsExpression(combo_ec_string, "Should select EC cert", reflags=re.S | re.M) +tr.Processes.Default.Streams.All += Testers.ContainsExpression("CN = combined.com", "Should select combined pem") + +# Should receive a RSA cert +tr = Test.AddTestRun("Only offer RSA ciphers, should receive RSA cert") +tr.Processes.Default.Command = "echo foo | openssl s_client -tls1_2 -servername combined.com -cipher 'ECDHE-RSA-AES128-GCM-SHA256' -connect 127.0.0.1:{0}".format( + ts.Variables.ssl_port) +tr.ReturnCode = 0 +tr.StillRunningAfter = server +tr.StillRunningAfter = ts +tr.Processes.Default.Streams.All += Testers.ContainsExpression(combo_rsa_string, "Should select RSA cert", reflags=re.S | re.M) +tr.Processes.Default.Streams.All += Testers.ContainsExpression("CN = combined.com", "Should select combined pem") diff --git a/tests/gold_tests/tls/tls_client_cert.test.py b/tests/gold_tests/tls/tls_client_cert.test.py index d21bb6b4c83..ca9e5a24821 100644 --- a/tests/gold_tests/tls/tls_client_cert.test.py +++ b/tests/gold_tests/tls/tls_client_cert.test.py @@ -17,10 +17,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import os -import subprocess -import re - Test.Summary = ''' Test different combinations of TLS handshake hooks to ensure they are applied consistently. ''' @@ -29,9 +25,20 @@ cafile = "{0}/signer.pem".format(Test.RunDirectory) cafile2 = "{0}/signer2.pem".format(Test.RunDirectory) # --clientverify: "" empty string because microserver does store_true for argparse, but options is a dictionary -server = Test.MakeOriginServer("server", ssl=True, options = { "--clientCA": cafile, "--clientverify": ""}, clientcert="{0}/signed-foo.pem".format(Test.RunDirectory), clientkey="{0}/signed-foo.key".format(Test.RunDirectory)) -server2 = Test.MakeOriginServer("server2", ssl=True, options = { "--clientCA": cafile2, "--clientverify": ""}, clientcert="{0}/signed2-bar.pem".format(Test.RunDirectory), clientkey="{0}/signed-bar.key".format(Test.RunDirectory)) +server = Test.MakeOriginServer("server", + ssl=True, + options={"--clientCA": cafile, + "--clientverify": ""}, + clientcert="{0}/signed-foo.pem".format(Test.RunDirectory), + clientkey="{0}/signed-foo.key".format(Test.RunDirectory)) +server2 = Test.MakeOriginServer("server2", + ssl=True, + options={"--clientCA": cafile2, + "--clientverify": ""}, + clientcert="{0}/signed2-bar.pem".format(Test.RunDirectory), + clientkey="{0}/signed-bar.key".format(Test.RunDirectory)) server3 = Test.MakeOriginServer("server3") +server4 = Test.MakeOriginServer("server4") server.Setup.Copy("ssl/signer.pem") server.Setup.Copy("ssl/signer2.pem") server.Setup.Copy("ssl/signed-foo.pem") @@ -68,14 +75,13 @@ 'proxy.config.diags.debug.tags': 'ssl_verify_test', 'proxy.config.ssl.server.cert.path': '{0}'.format(ts.Variables.SSLDir), 'proxy.config.ssl.server.private_key.path': '{0}'.format(ts.Variables.SSLDir), - 'proxy.config.ssl.client.verify.server': 0, 'proxy.config.ssl.server.cipher_suite': 'ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA384:AES128-GCM-SHA256:AES256-GCM-SHA384:ECDHE-RSA-RC4-SHA:ECDHE-RSA-AES128-SHA:ECDHE-RSA-AES256-SHA:RC4-SHA:RC4-MD5:AES128-SHA:AES256-SHA:DES-CBC3-SHA!SRP:!DSS:!PSK:!aNULL:!eNULL:!SSLv2', 'proxy.config.ssl.client.cert.path': '{0}'.format(ts.Variables.SSLDir), 'proxy.config.ssl.client.cert.filename': 'signed-foo.pem', 'proxy.config.ssl.client.private_key.path': '{0}'.format(ts.Variables.SSLDir), 'proxy.config.ssl.client.private_key.filename': 'signed-foo.key', 'proxy.config.exec_thread.autoconfig.scale': 1.0, - 'proxy.config.url_remap.pristine_host_hdr' : 1, + 'proxy.config.url_remap.pristine_host_hdr': 1, }) ts.Disk.ssl_multicert_config.AddLine( @@ -98,6 +104,18 @@ ts.Disk.sni_yaml.AddLine( ' client_key: {0}/signed-bar.key'.format(ts.Variables.SSLDir)) +ts.Disk.logging_yaml.AddLines( + ''' +logging: + formats: + - name: testformat + format: '% % % %' + logs: + - mode: ascii + format: testformat + filename: squid +'''.split("\n") +) # Should succeed tr = Test.AddTestRun("Connect with first client cert to first server") @@ -111,7 +129,7 @@ tr.Processes.Default.ReturnCode = 0 tr.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could Not Connect", "Check response") -#Should fail +# Should fail trfail = Test.AddTestRun("Connect with first client cert to second server") trfail.StillRunningAfter = ts trfail.StillRunningAfter = server @@ -129,7 +147,7 @@ trbar.Processes.Default.ReturnCode = 0 trbar.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could Not Connect", "Check response") -#Should fail +# Should fail trbarfail = Test.AddTestRun("Connect with signed2 bar cert to first server") trbarfail.StillRunningAfter = ts trbarfail.StillRunningAfter = server @@ -142,7 +160,7 @@ # Update the SNI config snipath = ts.Disk.sni_yaml.AbsPath recordspath = ts.Disk.records_config.AbsPath -tr2.Disk.File(snipath, id = "sni_yaml", typename="ats:config"), +tr2.Disk.File(snipath, id="sni_yaml", typename="ats:config"), tr2.Disk.sni_yaml.AddLine( 'sni:') tr2.Disk.sni_yaml.AddLine( @@ -152,17 +170,16 @@ tr2.Disk.sni_yaml.AddLine( ' client_key: {0}/signed-bar.key'.format(ts.Variables.SSLDir)) # recreate the records.config with the cert filename changed -tr2.Disk.File(recordspath, id = "records_config", typename="ats:config:records"), +tr2.Disk.File(recordspath, id="records_config", typename="ats:config:records"), tr2.Disk.records_config.update({ 'proxy.config.ssl.server.cert.path': '{0}'.format(ts.Variables.SSLDir), 'proxy.config.ssl.server.private_key.path': '{0}'.format(ts.Variables.SSLDir), - 'proxy.config.ssl.client.verify.server': 0, 'proxy.config.ssl.server.cipher_suite': 'ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA384:AES128-GCM-SHA256:AES256-GCM-SHA384:ECDHE-RSA-RC4-SHA:ECDHE-RSA-AES128-SHA:ECDHE-RSA-AES256-SHA:RC4-SHA:RC4-MD5:AES128-SHA:AES256-SHA:DES-CBC3-SHA!SRP:!DSS:!PSK:!aNULL:!eNULL:!SSLv2', 'proxy.config.ssl.client.cert.path': '{0}'.format(ts.Variables.SSLDir), 'proxy.config.ssl.client.cert.filename': 'signed2-foo.pem', 'proxy.config.ssl.client.private_key.path': '{0}'.format(ts.Variables.SSLDir), 'proxy.config.ssl.client.private_key.filename': 'signed-foo.key', - 'proxy.config.url_remap.pristine_host_hdr' : 1, + 'proxy.config.url_remap.pristine_host_hdr': 1, }) tr2.StillRunningAfter = ts tr2.StillRunningAfter = server @@ -172,20 +189,6 @@ tr2.Processes.Default.Env = ts.Env tr2.Processes.Default.ReturnCode = 0 -# Parking this as a ready tester on a meaningless process -# Stall the test runs until the sni reload has completed -# At that point the new sni settings are ready to go -def sni_reload_done(tsenv): - def done_reload(process, hasRunFor, **kw): - cmd = "grep 'sni.yaml finished loading' {0} | wc -l > {1}/test.out".format(ts.Disk.diags_log.Name, Test.RunDirectory) - retval = subprocess.run(cmd, shell=True, env=tsenv) - if retval.returncode == 0: - cmd ="if [ -f {0}/test.out -a \"`cat {0}/test.out`\" = \"2\" ] ; then true; else false; fi".format(Test.RunDirectory) - retval = subprocess.run(cmd, shell = True, env=tsenv) - return retval.returncode == 0 - - return done_reload - tr2reload = Test.AddTestRun("Reload config") tr2reload.StillRunningAfter = ts tr2reload.StillRunningAfter = server @@ -196,10 +199,10 @@ def done_reload(process, hasRunFor, **kw): tr2reload.Processes.Default.ReturnCode = 0 -#Should succeed +# Should succeed tr3bar = Test.AddTestRun("Make request with other bar cert to first server") # Wait for the reload to complete -tr3bar.Processes.Default.StartBefore(server3, ready=sni_reload_done(ts.Env)) +tr3bar.Processes.Default.StartBefore(server3, ready=When.FileContains(ts.Disk.diags_log.Name, 'sni.yaml finished loading', 2)) tr3bar.StillRunningAfter = ts tr3bar.StillRunningAfter = server tr3bar.StillRunningAfter = server2 @@ -207,7 +210,7 @@ def done_reload(process, hasRunFor, **kw): tr3bar.Processes.Default.ReturnCode = 0 tr3bar.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could Not Connect", "Check response") -#Should fail +# Should fail tr3barfail = Test.AddTestRun("Make request with other bar cert to second server") tr3barfail.StillRunningAfter = ts tr3barfail.StillRunningAfter = server @@ -216,7 +219,7 @@ def done_reload(process, hasRunFor, **kw): tr3barfail.Processes.Default.ReturnCode = 0 tr3barfail.Processes.Default.Streams.stdout = Testers.ContainsExpression("Could Not Connect", "Check response") -#Should succeed +# Should succeed tr3 = Test.AddTestRun("Make request with other cert to second server") # Wait for the reload to complete tr3.StillRunningAfter = ts @@ -226,7 +229,7 @@ def done_reload(process, hasRunFor, **kw): tr3.Processes.Default.ReturnCode = 0 tr3.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could Not Connect", "Check response") -#Should fail +# Should fail tr3fail = Test.AddTestRun("Make request with other cert to first server") tr3fail.StillRunningAfter = ts tr3fail.StillRunningAfter = server @@ -245,7 +248,8 @@ def done_reload(process, hasRunFor, **kw): trupdate.Setup.CopyAs("ssl/signed2-bar.pem", ".", "{0}/signed-bar.pem".format(ts.Variables.SSLDir)) # in the config/ssl directory for records.config trupdate.Setup.CopyAs("ssl/signed-foo.pem", ".", "{0}/signed2-foo.pem".format(ts.Variables.SSLDir)) -trupdate.Processes.Default.Command = 'traffic_ctl config set proxy.config.ssl.client.cert.path {0}/; touch {1}'.format(ts.Variables.SSLDir,snipath) +trupdate.Processes.Default.Command = 'traffic_ctl config set proxy.config.ssl.client.cert.path {0}/; touch {1}'.format( + ts.Variables.SSLDir, snipath) # Need to copy over the environment so traffic_ctl knows where to find the unix domain socket trupdate.Processes.Default.Env = ts.Env trupdate.Processes.Default.ReturnCode = 0 @@ -258,7 +262,7 @@ def done_reload(process, hasRunFor, **kw): trreload.Processes.Default.Env = ts.Env trreload.Processes.Default.ReturnCode = 0 -#Should succeed +# Should succeed tr4bar = Test.AddTestRun("Make request with renamed bar cert to second server") # Wait for the reload to complete tr4bar.DelayStart = 10 @@ -269,7 +273,7 @@ def done_reload(process, hasRunFor, **kw): tr4bar.Processes.Default.ReturnCode = 0 tr4bar.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could Not Connect", "Check response") -#Should fail +# Should fail tr4barfail = Test.AddTestRun("Make request with renamed bar cert to first server") tr4barfail.StillRunningAfter = ts tr4barfail.StillRunningAfter = server @@ -278,7 +282,7 @@ def done_reload(process, hasRunFor, **kw): tr4barfail.Processes.Default.ReturnCode = 0 tr4barfail.Processes.Default.Streams.stdout = Testers.ContainsExpression("Could Not Connect", "Check response") -#Should succeed +# Should succeed tr4 = Test.AddTestRun("Make request with renamed foo cert to first server") tr4.StillRunningAfter = ts tr4.StillRunningAfter = server @@ -287,7 +291,7 @@ def done_reload(process, hasRunFor, **kw): tr4.Processes.Default.ReturnCode = 0 tr4.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could Not Connect", "Check response") -#Should fail +# Should fail tr4fail = Test.AddTestRun("Make request with renamed foo cert to second server") tr4fail.StillRunningAfter = ts tr4fail.StillRunningAfter = server @@ -295,3 +299,11 @@ def done_reload(process, hasRunFor, **kw): tr4fail.Processes.Default.Command = 'curl -H host:example.com http://127.0.0.1:{0}/case2'.format(ts.Variables.port) tr4fail.Processes.Default.ReturnCode = 0 tr4fail.Processes.Default.Streams.stdout = Testers.ContainsExpression("Could Not Connect", "Check response") + +tr = Test.AddTestRun("Wait for the access log to write out") +tr.Processes.Default.StartBefore(server4, ready=When.FileExists(ts.Disk.squid_log)) +tr.StillRunningAfter = ts +tr.Processes.Default.Command = 'echo "log file exists"' +tr.Processes.Default.ReturnCode = 0 + +ts.Disk.squid_log.Content = "gold/proxycert-accesslog.gold" diff --git a/tests/gold_tests/tls/tls_client_cert2.test.py b/tests/gold_tests/tls/tls_client_cert2.test.py index 5e252ed2bf3..1b2fe8478ed 100644 --- a/tests/gold_tests/tls/tls_client_cert2.test.py +++ b/tests/gold_tests/tls/tls_client_cert2.test.py @@ -17,9 +17,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import os -import re - Test.Summary = ''' Test client certs to origin selected via wildcard names in sni ''' @@ -27,8 +24,19 @@ ts = Test.MakeATSProcess("ts", command="traffic_server", select_ports=True) cafile = "{0}/signer.pem".format(Test.RunDirectory) cafile2 = "{0}/signer2.pem".format(Test.RunDirectory) -server = Test.MakeOriginServer("server", ssl=True, options = { "--clientCA": cafile, "--clientverify": ""}, clientcert="{0}/signed-foo.pem".format(Test.RunDirectory), clientkey="{0}/signed-foo.key".format(Test.RunDirectory)) -server2 = Test.MakeOriginServer("server2", ssl=True, options = { "--clientCA": cafile2, "--clientverify": ""}, clientcert="{0}/signed2-bar.pem".format(Test.RunDirectory), clientkey="{0}/signed-bar.key".format(Test.RunDirectory)) +server = Test.MakeOriginServer("server", + ssl=True, + options={"--clientCA": cafile, + "--clientverify": ""}, + clientcert="{0}/signed-foo.pem".format(Test.RunDirectory), + clientkey="{0}/signed-foo.key".format(Test.RunDirectory)) +server2 = Test.MakeOriginServer("server2", + ssl=True, + options={"--clientCA": cafile2, + "--clientverify": ""}, + clientcert="{0}/signed2-bar.pem".format(Test.RunDirectory), + clientkey="{0}/signed-bar.key".format(Test.RunDirectory)) +server4 = Test.MakeOriginServer("server4") server.Setup.Copy("ssl/signer.pem") server.Setup.Copy("ssl/signer2.pem") server.Setup.Copy("ssl/signed-foo.pem") @@ -66,10 +74,9 @@ 'proxy.config.ssl.server.private_key.path': '{0}'.format(ts.Variables.SSLDir), 'proxy.config.ssl.client.cert.path': '{0}'.format(ts.Variables.SSLDir), 'proxy.config.ssl.client.private_key.path': '{0}'.format(ts.Variables.SSLDir), - 'proxy.config.ssl.client.verify.server': 0, 'proxy.config.ssl.server.cipher_suite': 'ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA384:AES128-GCM-SHA256:AES256-GCM-SHA384:ECDHE-RSA-RC4-SHA:ECDHE-RSA-AES128-SHA:ECDHE-RSA-AES256-SHA:RC4-SHA:RC4-MD5:AES128-SHA:AES256-SHA:DES-CBC3-SHA!SRP:!DSS:!PSK:!aNULL:!eNULL:!SSLv2', 'proxy.config.exec_thread.autoconfig.scale': 1.0, - 'proxy.config.url_remap.pristine_host_hdr' : 1, + 'proxy.config.url_remap.pristine_host_hdr': 1, }) ts.Disk.ssl_multicert_config.AddLine( @@ -98,6 +105,18 @@ ' client_key: {0}/signed-foo.key'.format(ts.Variables.SSLDir), ]) +ts.Disk.logging_yaml.AddLines( + ''' +logging: + formats: + - name: testformat + format: '% % % %' + logs: + - mode: ascii + format: testformat + filename: squid +'''.split("\n") +) # Should succeed tr = Test.AddTestRun("bob.bar.com to server 1") @@ -111,7 +130,7 @@ tr.Processes.Default.ReturnCode = 0 tr.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could Not Connect", "Check response") -#Should fail +# Should fail trfail = Test.AddTestRun("bob.bar.com to server 2") trfail.StillRunningAfter = ts trfail.StillRunningAfter = server @@ -129,7 +148,7 @@ tr.Processes.Default.ReturnCode = 0 tr.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could Not Connect", "Check response") -#Should fail +# Should fail trfail = Test.AddTestRun("bob.foo.com to server 2") trfail.StillRunningAfter = ts trfail.StillRunningAfter = server @@ -147,7 +166,7 @@ tr.Processes.Default.ReturnCode = 0 tr.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could Not Connect", "Check response") -#Should fail +# Should fail trfail = Test.AddTestRun("random.bar.com to server 1") trfail.StillRunningAfter = ts trfail.StillRunningAfter = server @@ -165,7 +184,7 @@ tr.Processes.Default.ReturnCode = 0 tr.Processes.Default.Streams.stdout = Testers.ContainsExpression("Could Not Connect", "Check response") -#Should fail +# Should fail trfail = Test.AddTestRun("random.foo.com to server 1") trfail.StillRunningAfter = ts trfail.StillRunningAfter = server @@ -173,3 +192,11 @@ trfail.Processes.Default.Command = 'curl -H host:random.foo.com http://127.0.0.1:{0}/case1'.format(ts.Variables.port) trfail.Processes.Default.ReturnCode = 0 trfail.Processes.Default.Streams.stdout = Testers.ContainsExpression("Could Not Connect", "Check response") + +tr = Test.AddTestRun("Wait for the access log to write out") +tr.Processes.Default.StartBefore(server4, ready=When.FileExists(ts.Disk.squid_log)) +tr.StillRunningAfter = ts +tr.Processes.Default.Command = 'echo "Log file exists"' +tr.Processes.Default.ReturnCode = 0 + +ts.Disk.squid_log.Content = "gold/proxycert2-accesslog.gold" diff --git a/tests/gold_tests/tls/tls_client_cert_override.test.py b/tests/gold_tests/tls/tls_client_cert_override.test.py index 5bb89590eab..f04f357ae05 100644 --- a/tests/gold_tests/tls/tls_client_cert_override.test.py +++ b/tests/gold_tests/tls/tls_client_cert_override.test.py @@ -17,9 +17,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import os -import re - Test.Summary = ''' Test conf_remp to specify different client certificates to offer to the origin ''' @@ -27,8 +24,18 @@ ts = Test.MakeATSProcess("ts", command="traffic_manager", select_ports=True) cafile = "{0}/signer.pem".format(Test.RunDirectory) cafile2 = "{0}/signer2.pem".format(Test.RunDirectory) -server = Test.MakeOriginServer("server", ssl=True, options = { "--clientCA": cafile, "--clientverify": ""}, clientcert="{0}/signed-foo.pem".format(Test.RunDirectory), clientkey="{0}/signed-foo.key".format(Test.RunDirectory)) -server2 = Test.MakeOriginServer("server2", ssl=True, options = { "--clientCA": cafile2, "--clientverify": ""}, clientcert="{0}/signed2-bar.pem".format(Test.RunDirectory), clientkey="{0}/signed-bar.key".format(Test.RunDirectory)) +server = Test.MakeOriginServer("server", + ssl=True, + options={"--clientCA": cafile, + "--clientverify": ""}, + clientcert="{0}/signed-foo.pem".format(Test.RunDirectory), + clientkey="{0}/signed-foo.key".format(Test.RunDirectory)) +server2 = Test.MakeOriginServer("server2", + ssl=True, + options={"--clientCA": cafile2, + "--clientverify": ""}, + clientcert="{0}/signed2-bar.pem".format(Test.RunDirectory), + clientkey="{0}/signed-bar.key".format(Test.RunDirectory)) server.Setup.Copy("ssl/signer.pem") server.Setup.Copy("ssl/signer2.pem") server.Setup.Copy("ssl/signed-foo.pem") @@ -63,14 +70,13 @@ ts.Disk.records_config.update({ 'proxy.config.ssl.server.cert.path': '{0}'.format(ts.Variables.SSLDir), 'proxy.config.ssl.server.private_key.path': '{0}'.format(ts.Variables.SSLDir), - 'proxy.config.ssl.client.verify.server': 0, 'proxy.config.ssl.server.cipher_suite': 'ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA384:AES128-GCM-SHA256:AES256-GCM-SHA384:ECDHE-RSA-RC4-SHA:ECDHE-RSA-AES128-SHA:ECDHE-RSA-AES256-SHA:RC4-SHA:RC4-MD5:AES128-SHA:AES256-SHA:DES-CBC3-SHA!SRP:!DSS:!PSK:!aNULL:!eNULL:!SSLv2', 'proxy.config.ssl.client.cert.path': '{0}'.format(ts.Variables.SSLDir), 'proxy.config.ssl.client.cert.filename': 'signed-foo.pem', 'proxy.config.ssl.client.private_key.path': '{0}'.format(ts.Variables.SSLDir), 'proxy.config.ssl.client.private_key.filename': 'signed-foo.key', 'proxy.config.exec_thread.autoconfig.scale': 1.0, - 'proxy.config.url_remap.pristine_host_hdr' : 1, + 'proxy.config.url_remap.pristine_host_hdr': 1, }) ts.Disk.ssl_multicert_config.AddLine( @@ -78,17 +84,25 @@ ) ts.Disk.remap_config.AddLine( - 'map /case1 https://127.0.0.1:{0}/ @plugin=conf_remap.so @pparam=proxy.config.ssl.client.cert.filename={1} plugin=conf_remap.so @pparam=proxy.config.ssl.client.private_key.filename={2}'.format(server.Variables.SSL_Port, "signed-foo.pem", "signed-foo.key") -) + 'map /case1 https://127.0.0.1:{0}/ @plugin=conf_remap.so @pparam=proxy.config.ssl.client.cert.filename={1} plugin=conf_remap.so @pparam=proxy.config.ssl.client.private_key.filename={2}'.format( + server.Variables.SSL_Port, + "signed-foo.pem", + "signed-foo.key")) ts.Disk.remap_config.AddLine( - 'map /badcase1 https://127.0.0.1:{0}/ @plugin=conf_remap.so @pparam=proxy.config.ssl.client.cert.filename={1} plugin=conf_remap.so @pparam=proxy.config.ssl.client.private_key.filename={2}'.format(server.Variables.SSL_Port, "signed2-foo.pem", "signed-foo.key") -) + 'map /badcase1 https://127.0.0.1:{0}/ @plugin=conf_remap.so @pparam=proxy.config.ssl.client.cert.filename={1} plugin=conf_remap.so @pparam=proxy.config.ssl.client.private_key.filename={2}'.format( + server.Variables.SSL_Port, + "signed2-foo.pem", + "signed-foo.key")) ts.Disk.remap_config.AddLine( - 'map /case2 https://127.0.0.1:{0}/ @plugin=conf_remap.so @pparam=proxy.config.ssl.client.cert.filename={1} plugin=conf_remap.so @pparam=proxy.config.ssl.client.private_key.filename={2}'.format(server2.Variables.SSL_Port, "signed2-foo.pem", "signed-foo.key") -) + 'map /case2 https://127.0.0.1:{0}/ @plugin=conf_remap.so @pparam=proxy.config.ssl.client.cert.filename={1} plugin=conf_remap.so @pparam=proxy.config.ssl.client.private_key.filename={2}'.format( + server2.Variables.SSL_Port, + "signed2-foo.pem", + "signed-foo.key")) ts.Disk.remap_config.AddLine( - 'map /badcase2 https://127.0.0.1:{0}/ @plugin=conf_remap.so @pparam=proxy.config.ssl.client.cert.filename={1} plugin=conf_remap.so @pparam=proxy.config.ssl.client.private_key.filename={2}'.format(server2.Variables.SSL_Port, "signed-foo.pem", "signed-foo.key") -) + 'map /badcase2 https://127.0.0.1:{0}/ @plugin=conf_remap.so @pparam=proxy.config.ssl.client.cert.filename={1} plugin=conf_remap.so @pparam=proxy.config.ssl.client.private_key.filename={2}'.format( + server2.Variables.SSL_Port, + "signed-foo.pem", + "signed-foo.key")) # Should succeed tr = Test.AddTestRun("Connect with correct client cert to first server") @@ -102,7 +116,7 @@ tr.Processes.Default.ReturnCode = 0 tr.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could Not Connect", "Check response") -#Should fail +# Should fail trfail = Test.AddTestRun("Connect with bad client cert to first server") trfail.StillRunningAfter = ts trfail.StillRunningAfter = server @@ -120,7 +134,7 @@ trbar.Processes.Default.ReturnCode = 0 trbar.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could Not Connect", "Check response") -#Should fail +# Should fail trbarfail = Test.AddTestRun("Connect with bad client cert to second server") trbarfail.StillRunningAfter = ts trbarfail.StillRunningAfter = server diff --git a/tests/gold_tests/tls/tls_client_verify.test.py b/tests/gold_tests/tls/tls_client_verify.test.py index aba36829e74..e231c019d85 100644 --- a/tests/gold_tests/tls/tls_client_verify.test.py +++ b/tests/gold_tests/tls/tls_client_verify.test.py @@ -17,17 +17,16 @@ # See the License for the specific language governing permissions and # limitations under the License. -import os -import re Test.Summary = ''' Test various options for requiring certificate from client for mutual authentication TLS ''' -ts = Test.MakeATSProcess("ts", select_ports=True, enable_tls=True) +ts = Test.MakeATSProcess("ts", command="traffic_manager", select_ports=True, enable_tls=True) cafile = "{0}/signer.pem".format(Test.RunDirectory) cafile2 = "{0}/signer2.pem".format(Test.RunDirectory) server = Test.MakeOriginServer("server") +server2 = Test.MakeOriginServer("server2") request_header = {"headers": "GET / HTTP/1.1\r\nHost: example.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} response_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": ""} @@ -43,9 +42,8 @@ ts.Disk.records_config.update({ 'proxy.config.ssl.server.cert.path': '{0}'.format(ts.Variables.SSLDir), 'proxy.config.ssl.server.private_key.path': '{0}'.format(ts.Variables.SSLDir), - 'proxy.config.ssl.client.verify.server': 0, 'proxy.config.ssl.server.cipher_suite': 'ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA384:AES128-GCM-SHA256:AES256-GCM-SHA384:ECDHE-RSA-RC4-SHA:ECDHE-RSA-AES128-SHA:ECDHE-RSA-AES256-SHA:RC4-SHA:RC4-MD5:AES128-SHA:AES256-SHA:DES-CBC3-SHA!SRP:!DSS:!PSK:!aNULL:!eNULL:!SSLv2', - 'proxy.config.url_remap.pristine_host_hdr' : 1, + 'proxy.config.url_remap.pristine_host_hdr': 1, 'proxy.config.ssl.client.certification_level': 2, 'proxy.config.ssl.CA.cert.filename': '{0}/signer.pem'.format(ts.Variables.SSLDir), 'proxy.config.exec_thread.autoconfig.scale': 1.0, @@ -66,19 +64,35 @@ 'sni:', '- fqdn: bob.bar.com', ' verify_client: NONE', + '- fqdn: "bob.com"', + ' verify_client: STRICT', '- fqdn: bob.*.com', ' verify_client: NONE', '- fqdn: "*bar.com"', ' verify_client: STRICT', ]) +ts.Disk.logging_yaml.AddLines( + ''' +logging: + formats: + - name: testformat + format: '% % % %' + logs: + - mode: ascii + format: testformat + filename: squid +'''.split("\n") +) + # to foo.com w/o client cert. Should fail tr = Test.AddTestRun("Connect to foo.com without cert") tr.Processes.Default.StartBefore(Test.Processes.ts) tr.Processes.Default.StartBefore(server) tr.StillRunningAfter = ts tr.StillRunningAfter = server -tr.Processes.Default.Command = "curl --tls-max 1.2 -k --resolve 'foo.com:{0}:127.0.0.1' https://foo.com:{0}/case1".format(ts.Variables.ssl_port) +tr.Processes.Default.Command = "curl --tls-max 1.2 -k --resolve 'foo.com:{0}:127.0.0.1' https://foo.com:{0}/case1".format( + ts.Variables.ssl_port) tr.Processes.Default.ReturnCode = 35 tr = Test.AddTestRun("Connect to foo.com with bad cert") @@ -86,7 +100,8 @@ tr.Setup.Copy("ssl/server.key") tr.StillRunningAfter = ts tr.StillRunningAfter = server -tr.Processes.Default.Command = "curl --tls-max 1.2 -k --cert ./server.pem --key ./server.key --resolve 'foo.com:{0}:127.0.0.1' https://foo.com:{0}/case1".format(ts.Variables.ssl_port) +tr.Processes.Default.Command = "curl --tls-max 1.2 -k --cert ./server.pem --key ./server.key --resolve 'foo.com:{0}:127.0.0.1' https://foo.com:{0}/case2".format( + ts.Variables.ssl_port) # Should fail with badly signed certs tr.Processes.Default.ReturnCode = 35 @@ -95,14 +110,16 @@ tr.Setup.Copy("ssl/signed-foo.key") tr.StillRunningAfter = ts tr.StillRunningAfter = server -tr.Processes.Default.Command = "curl --tls-max 1.2 -k --cert ./signed-foo.pem --key ./signed-foo.key --resolve 'foo.com:{0}:127.0.0.1' https://foo.com:{0}/case1".format(ts.Variables.ssl_port) +tr.Processes.Default.Command = "curl --tls-max 1.2 -k --cert ./signed-foo.pem --key ./signed-foo.key --resolve 'foo.com:{0}:127.0.0.1' https://foo.com:{0}/case3".format( + ts.Variables.ssl_port) tr.Processes.Default.ReturnCode = 0 tr.Processes.Default.Streams.All = Testers.ExcludesExpression("error", "Check response") tr = Test.AddTestRun("Connect to bob.bar.com without cert") tr.StillRunningAfter = ts tr.StillRunningAfter = server -tr.Processes.Default.Command = "curl --tls-max 1.2 -k --resolve 'bob.bar.com:{0}:127.0.0.1' https://bob.bar.com:{0}/case1".format(ts.Variables.ssl_port) +tr.Processes.Default.Command = "curl --tls-max 1.2 -k --resolve 'bob.bar.com:{0}:127.0.0.1' https://bob.bar.com:{0}/case4".format( + ts.Variables.ssl_port) tr.Processes.Default.ReturnCode = 0 tr.Processes.Default.Streams.All = Testers.ExcludesExpression("alert", "TLS handshake should succeed") @@ -111,7 +128,8 @@ tr.Setup.Copy("ssl/signed-bar.key") tr.StillRunningAfter = ts tr.StillRunningAfter = server -tr.Processes.Default.Command = "curl --tls-max 1.2 -k --cert ./signed-bob-bar.pem --key ./signed-bar.key --resolve 'bob.bar.com:{0}:127.0.0.1' https://bob.bar.com:{0}/case1".format(ts.Variables.ssl_port) +tr.Processes.Default.Command = "curl --tls-max 1.2 -k --cert ./signed-bob-bar.pem --key ./signed-bar.key --resolve 'bob.bar.com:{0}:127.0.0.1' https://bob.bar.com:{0}/case5".format( + ts.Variables.ssl_port) tr.Processes.Default.ReturnCode = 0 tr.Processes.Default.Streams.All = Testers.ExcludesExpression("error", "Check response") @@ -120,14 +138,16 @@ tr.Setup.Copy("ssl/server.key") tr.StillRunningAfter = ts tr.StillRunningAfter = server -tr.Processes.Default.Command = "curl --tls-max 1.2 -k --cert ./server.pem --key ./server.key --resolve 'bob.bar.com:{0}:127.0.0.1' https://bob.bar.com:{0}/case1".format(ts.Variables.ssl_port) +tr.Processes.Default.Command = "curl --tls-max 1.2 -k --cert ./server.pem --key ./server.key --resolve 'bob.bar.com:{0}:127.0.0.1' https://bob.bar.com:{0}/case6".format( + ts.Variables.ssl_port) tr.Processes.Default.ReturnCode = 0 tr.Processes.Default.Streams.All = Testers.ExcludesExpression("error", "Check response") tr = Test.AddTestRun("Connect to bob.foo.com without cert") tr.StillRunningAfter = ts tr.StillRunningAfter = server -tr.Processes.Default.Command = "curl --tls-max 1.2 -k --resolve 'bob.foo.com:{0}:127.0.0.1' https://bob.foo.com:{0}/case1".format(ts.Variables.ssl_port) +tr.Processes.Default.Command = "curl --tls-max 1.2 -k --resolve 'bob.foo.com:{0}:127.0.0.1' https://bob.foo.com:{0}/case7".format( + ts.Variables.ssl_port) tr.Processes.Default.ReturnCode = 0 tr.Processes.Default.Streams.All = Testers.ExcludesExpression("alert", "TLS handshake should succeed") @@ -136,7 +156,8 @@ tr.Setup.Copy("ssl/signed-foo.key") tr.StillRunningAfter = ts tr.StillRunningAfter = server -tr.Processes.Default.Command = "curl --tls-max 1.2 -k --cert ./signed-bob-foo.pem --key ./signed-foo.key --resolve 'bob.foo.com:{0}:127.0.0.1' https://bob.foo.com:{0}/case1".format(ts.Variables.ssl_port) +tr.Processes.Default.Command = "curl --tls-max 1.2 -k --cert ./signed-bob-foo.pem --key ./signed-foo.key --resolve 'bob.foo.com:{0}:127.0.0.1' https://bob.foo.com:{0}/case8".format( + ts.Variables.ssl_port) tr.Processes.Default.ReturnCode = 0 tr.Processes.Default.Streams.All = Testers.ExcludesExpression("error", "Check response") @@ -145,14 +166,16 @@ tr.Setup.Copy("ssl/server.key") tr.StillRunningAfter = ts tr.StillRunningAfter = server -tr.Processes.Default.Command = "curl --tls-max 1.2 -k --cert ./server.pem --key ./server.key --resolve 'bob.foo.com:{0}:127.0.0.1' https://bob.foo.com:{0}/case1".format(ts.Variables.ssl_port) +tr.Processes.Default.Command = "curl --tls-max 1.2 -k --cert ./server.pem --key ./server.key --resolve 'bob.foo.com:{0}:127.0.0.1' https://bob.foo.com:{0}/case9".format( + ts.Variables.ssl_port) tr.Processes.Default.ReturnCode = 0 tr.Processes.Default.Streams.All = Testers.ExcludesExpression("error", "Check response") tr = Test.AddTestRun("Connect to bar.com without cert") tr.StillRunningAfter = ts tr.StillRunningAfter = server -tr.Processes.Default.Command = "curl --tls-max 1.2 -k --resolve 'bar.com:{0}:127.0.0.1' https://bar.com:{0}/case1".format(ts.Variables.ssl_port) +tr.Processes.Default.Command = "curl --tls-max 1.2 -k --resolve 'bar.com:{0}:127.0.0.1' https://bar.com:{0}/case10".format( + ts.Variables.ssl_port) tr.Processes.Default.ReturnCode = 35 tr = Test.AddTestRun("Connect to bar.com with cert") @@ -160,7 +183,8 @@ tr.Setup.Copy("ssl/signed-bar.key") tr.StillRunningAfter = ts tr.StillRunningAfter = server -tr.Processes.Default.Command = "curl --tls-max 1.2 -k --cert ./signed-bar.pem --key ./signed-bar.key --resolve 'bar.com:{0}:127.0.0.1' https://bar.com:{0}/case1".format(ts.Variables.ssl_port) +tr.Processes.Default.Command = "curl --tls-max 1.2 -k --cert ./signed-bar.pem --key ./signed-bar.key --resolve 'bar.com:{0}:127.0.0.1' https://bar.com:{0}/case11".format( + ts.Variables.ssl_port) tr.Processes.Default.ReturnCode = 0 tr.Processes.Default.Streams.All = Testers.ExcludesExpression("error", "TLS handshake should succeed") @@ -169,5 +193,32 @@ tr.Setup.Copy("ssl/server.key") tr.StillRunningAfter = ts tr.StillRunningAfter = server -tr.Processes.Default.Command = "curl --tls-max 1.2 -k --cert ./server.pem --key ./server.key --resolve 'bar.com:{0}:127.0.0.1' https://bar.com:{0}/case1".format(ts.Variables.ssl_port) +tr.Processes.Default.Command = "curl --tls-max 1.2 -k --cert ./server.pem --key ./server.key --resolve 'bar.com:{0}:127.0.0.1' https://bar.com:{0}/case12".format( + ts.Variables.ssl_port) tr.Processes.Default.ReturnCode = 35 + + +# Test that the fqdn's match completely. bob.com should require client certificate. bob.com.com should not +tr = Test.AddTestRun("Connect to bob.com without cert, should fail") +tr.StillRunningAfter = ts +tr.StillRunningAfter = server +tr.Processes.Default.Command = "curl --tls-max 1.2 -k --resolve 'bob.com:{0}:127.0.0.1' https://bob.com:{0}/case13".format( + ts.Variables.ssl_port) +tr.Processes.Default.ReturnCode = 35 + +tr = Test.AddTestRun("Connect to bob.com.com without cert, should succeed") +tr.StillRunningAfter = ts +tr.StillRunningAfter = server +tr.Processes.Default.Command = "curl --tls-max 1.2 -k --resolve 'bob.com.com:{0}:127.0.0.1' https://bob.com.com:{0}/case14".format( + ts.Variables.ssl_port) +tr.Processes.Default.ReturnCode = 0 + +tr = Test.AddTestRun("Wait for the access log to write out") +tr.Processes.Default.StartBefore(server2, ready=When.FileExists(ts.Disk.squid_log)) +tr.StillRunningAfter = ts +tr.StillRunningAfter = server +tr.StillRunningAfter = server2 +tr.Processes.Default.Command = 'echo "Log file exists"' +tr.Processes.Default.ReturnCode = 0 + +ts.Disk.squid_log.Content = "gold/clientcert-accesslog.gold" diff --git a/tests/gold_tests/tls/tls_client_verify2.test.py b/tests/gold_tests/tls/tls_client_verify2.test.py index 78dff33768b..a265af76044 100644 --- a/tests/gold_tests/tls/tls_client_verify2.test.py +++ b/tests/gold_tests/tls/tls_client_verify2.test.py @@ -17,9 +17,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import os -import re - Test.Summary = ''' Test various options for requiring certificate from client for mutual authentication TLS ''' @@ -43,9 +40,8 @@ ts.Disk.records_config.update({ 'proxy.config.ssl.server.cert.path': '{0}'.format(ts.Variables.SSLDir), 'proxy.config.ssl.server.private_key.path': '{0}'.format(ts.Variables.SSLDir), - 'proxy.config.ssl.client.verify.server': 0, 'proxy.config.ssl.server.cipher_suite': 'ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA384:AES128-GCM-SHA256:AES256-GCM-SHA384:ECDHE-RSA-RC4-SHA:ECDHE-RSA-AES128-SHA:ECDHE-RSA-AES256-SHA:RC4-SHA:RC4-MD5:AES128-SHA:AES256-SHA:DES-CBC3-SHA!SRP:!DSS:!PSK:!aNULL:!eNULL:!SSLv2', - 'proxy.config.url_remap.pristine_host_hdr' : 1, + 'proxy.config.url_remap.pristine_host_hdr': 1, 'proxy.config.ssl.client.certification_level': 0, 'proxy.config.ssl.CA.cert.path': '', 'proxy.config.exec_thread.autoconfig.scale': 1.0, @@ -78,7 +74,8 @@ tr.Processes.Default.StartBefore(server) tr.StillRunningAfter = ts tr.StillRunningAfter = server -tr.Processes.Default.Command = "curl --tls-max 1.2 -k --resolve 'foo.com:{0}:127.0.0.1' https://foo.com:{0}/case1".format(ts.Variables.ssl_port) +tr.Processes.Default.Command = "curl --tls-max 1.2 -k --resolve 'foo.com:{0}:127.0.0.1' https://foo.com:{0}/case1".format( + ts.Variables.ssl_port) tr.Processes.Default.ReturnCode = 0 tr.Processes.Default.Streams.All = Testers.ExcludesExpression("error", "Check response") @@ -87,14 +84,16 @@ tr.Setup.Copy("ssl/signed-foo.key") tr.StillRunningAfter = ts tr.StillRunningAfter = server -tr.Processes.Default.Command = "curl --tls-max 1.2 -k --cert ./signed-foo.pem --key ./signed-foo.key --resolve 'foo.com:{0}:127.0.0.1' https://foo.com:{0}/case1".format(ts.Variables.ssl_port) +tr.Processes.Default.Command = "curl --tls-max 1.2 -k --cert ./signed-foo.pem --key ./signed-foo.key --resolve 'foo.com:{0}:127.0.0.1' https://foo.com:{0}/case1".format( + ts.Variables.ssl_port) tr.Processes.Default.ReturnCode = 0 tr.Processes.Default.Streams.All = Testers.ExcludesExpression("error", "Check response") tr = Test.AddTestRun("Connect to bob.bar.com without cert") tr.StillRunningAfter = ts tr.StillRunningAfter = server -tr.Processes.Default.Command = "curl --tls-max 1.2 -k --resolve 'bob.bar.com:{0}:127.0.0.1' https://bob.bar.com:{0}/case1".format(ts.Variables.ssl_port) +tr.Processes.Default.Command = "curl --tls-max 1.2 -k --resolve 'bob.bar.com:{0}:127.0.0.1' https://bob.bar.com:{0}/case1".format( + ts.Variables.ssl_port) tr.Processes.Default.ReturnCode = 35 tr = Test.AddTestRun("Connect to bob.bar.com with cert") @@ -102,7 +101,8 @@ tr.Setup.Copy("ssl/signed-bar.key") tr.StillRunningAfter = ts tr.StillRunningAfter = server -tr.Processes.Default.Command = "curl --tls-max 1.2 -k --cert ./signed-bob-bar.pem --key ./signed-bar.key --resolve 'bob.bar.com:{0}:127.0.0.1' https://bob.bar.com:{0}/case1".format(ts.Variables.ssl_port) +tr.Processes.Default.Command = "curl --tls-max 1.2 -k --cert ./signed-bob-bar.pem --key ./signed-bar.key --resolve 'bob.bar.com:{0}:127.0.0.1' https://bob.bar.com:{0}/case1".format( + ts.Variables.ssl_port) tr.Processes.Default.ReturnCode = 0 tr.Processes.Default.Streams.All = Testers.ExcludesExpression("error", "TLS handshake should succeed") @@ -111,13 +111,15 @@ tr.Setup.Copy("ssl/server.key") tr.StillRunningAfter = ts tr.StillRunningAfter = server -tr.Processes.Default.Command = "curl --tls-max 1.2 -k --cert ./server.pem --key ./server.key --resolve 'bob.bar.com:{0}:127.0.0.1' https://bob.bar.com:{0}/case1".format(ts.Variables.ssl_port) +tr.Processes.Default.Command = "curl --tls-max 1.2 -k --cert ./server.pem --key ./server.key --resolve 'bob.bar.com:{0}:127.0.0.1' https://bob.bar.com:{0}/case1".format( + ts.Variables.ssl_port) tr.Processes.Default.ReturnCode = 35 tr = Test.AddTestRun("Connect to bob.foo.com without cert") tr.StillRunningAfter = ts tr.StillRunningAfter = server -tr.Processes.Default.Command = "curl --tls-max 1.2 -k --resolve 'bob.foo.com:{0}:127.0.0.1' https://bob.foo.com:{0}/case1".format(ts.Variables.ssl_port) +tr.Processes.Default.Command = "curl --tls-max 1.2 -k --resolve 'bob.foo.com:{0}:127.0.0.1' https://bob.foo.com:{0}/case1".format( + ts.Variables.ssl_port) tr.Processes.Default.ReturnCode = 35 tr = Test.AddTestRun("Connect to bob.foo.com with cert") @@ -125,7 +127,8 @@ tr.Setup.Copy("ssl/signed-foo.key") tr.StillRunningAfter = ts tr.StillRunningAfter = server -tr.Processes.Default.Command = "curl --tls-max 1.2 -k --cert ./signed-bob-foo.pem --key ./signed-foo.key --resolve 'bob.foo.com:{0}:127.0.0.1' https://bob.foo.com:{0}/case1".format(ts.Variables.ssl_port) +tr.Processes.Default.Command = "curl --tls-max 1.2 -k --cert ./signed-bob-foo.pem --key ./signed-foo.key --resolve 'bob.foo.com:{0}:127.0.0.1' https://bob.foo.com:{0}/case1".format( + ts.Variables.ssl_port) tr.Processes.Default.ReturnCode = 0 tr.Processes.Default.Streams.All = Testers.ExcludesExpression("error", "TLS handshake should succeed") @@ -134,13 +137,15 @@ tr.Setup.Copy("ssl/server.key") tr.StillRunningAfter = ts tr.StillRunningAfter = server -tr.Processes.Default.Command = "curl --tls-max 1.2 -k --cert ./server.pem --key ./server.key --resolve 'bob.foo.com:{0}:127.0.0.1' https://bob.foo.com:{0}/case1".format(ts.Variables.ssl_port) +tr.Processes.Default.Command = "curl --tls-max 1.2 -k --cert ./server.pem --key ./server.key --resolve 'bob.foo.com:{0}:127.0.0.1' https://bob.foo.com:{0}/case1".format( + ts.Variables.ssl_port) tr.Processes.Default.ReturnCode = 35 tr = Test.AddTestRun("Connect to bar.com without cert") tr.StillRunningAfter = ts tr.StillRunningAfter = server -tr.Processes.Default.Command = "curl --tls-max 1.2 -k --resolve 'bar.com:{0}:127.0.0.1' https://bar.com:{0}/case1".format(ts.Variables.ssl_port) +tr.Processes.Default.Command = "curl --tls-max 1.2 -k --resolve 'bar.com:{0}:127.0.0.1' https://bar.com:{0}/case1".format( + ts.Variables.ssl_port) tr.Processes.Default.ReturnCode = 0 tr.Processes.Default.Streams.All = Testers.ExcludesExpression("alert", "TLS handshake should succeed") @@ -149,7 +154,8 @@ tr.Setup.Copy("ssl/signed-bar.key") tr.StillRunningAfter = ts tr.StillRunningAfter = server -tr.Processes.Default.Command = "curl --tls-max 1.2 -k --cert ./signed-bar.pem --key ./signed-bar.key --resolve 'bar.com:{0}:127.0.0.1' https://bar.com:{0}/case1".format(ts.Variables.ssl_port) +tr.Processes.Default.Command = "curl --tls-max 1.2 -k --cert ./signed-bar.pem --key ./signed-bar.key --resolve 'bar.com:{0}:127.0.0.1' https://bar.com:{0}/case1".format( + ts.Variables.ssl_port) tr.Processes.Default.ReturnCode = 0 tr.Processes.Default.Streams.All = Testers.ExcludesExpression("error", "Check response") @@ -158,6 +164,7 @@ tr.Setup.Copy("ssl/server.key") tr.StillRunningAfter = ts tr.StillRunningAfter = server -tr.Processes.Default.Command = "curl --tls-max 1.2 -k --cert ./server.pem --key ./server.key --resolve 'bar.com:{0}:127.0.0.1' https://bar.com:{0}/case1".format(ts.Variables.ssl_port) +tr.Processes.Default.Command = "curl --tls-max 1.2 -k --cert ./server.pem --key ./server.key --resolve 'bar.com:{0}:127.0.0.1' https://bar.com:{0}/case1".format( + ts.Variables.ssl_port) tr.Processes.Default.ReturnCode = 0 tr.Processes.Default.Streams.All = Testers.ExcludesExpression("alert unknown ca", "TLS handshake should succeed") diff --git a/tests/gold_tests/tls/tls_client_versions.test.py b/tests/gold_tests/tls/tls_client_versions.test.py index 76cb28decd3..8f1085a51a5 100644 --- a/tests/gold_tests/tls/tls_client_versions.test.py +++ b/tests/gold_tests/tls/tls_client_versions.test.py @@ -16,7 +16,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import os Test.Summary = ''' Test TLS protocol offering based on SNI ''' @@ -24,7 +23,6 @@ # By default only offer TLSv1_2 # for special doman foo.com only offer TLSv1 and TLSv1_1 -# need Curl Test.SkipUnless( Condition.HasOpenSSLVersion("1.1.1") ) @@ -64,33 +62,37 @@ # bar.com should terminate. # empty SNI should tunnel to server_bar ts.Disk.sni_yaml.AddLines([ - 'sni:', - '- fqdn: foo.com', - ' valid_tls_versions_in: [ TLSv1, TLSv1_1 ]' + 'sni:', + '- fqdn: foo.com', + ' valid_tls_versions_in: [ TLSv1, TLSv1_1 ]' ]) # Target foo.com for TLSv1_2. Should fail tr = Test.AddTestRun("foo.com TLSv1_2") tr.Processes.Default.StartBefore(server) tr.Processes.Default.StartBefore(Test.Processes.ts) -tr.Processes.Default.Command = "curl -v --tls-max 1.2 --tlsv1.2 --resolve 'foo.com:{0}:127.0.0.1' -k https://foo.com:{0}".format(ts.Variables.ssl_port) +tr.Processes.Default.Command = "curl -v --tls-max 1.2 --tlsv1.2 --resolve 'foo.com:{0}:127.0.0.1' -k https://foo.com:{0}".format( + ts.Variables.ssl_port) tr.ReturnCode = 35 tr.StillRunningAfter = ts # Target foo.com for TLSv1. Should succeed tr = Test.AddTestRun("foo.com TLSv1") -tr.Processes.Default.Command = "curl -v --tls-max 1.0 --tlsv1 --resolve 'foo.com:{0}:127.0.0.1' -k https://foo.com:{0}".format(ts.Variables.ssl_port) +tr.Processes.Default.Command = "curl -v --tls-max 1.0 --tlsv1 --resolve 'foo.com:{0}:127.0.0.1' -k https://foo.com:{0}".format( + ts.Variables.ssl_port) tr.ReturnCode = 0 tr.StillRunningAfter = ts # Target bar.com for TLSv1. Should fail tr = Test.AddTestRun("bar.com TLSv1") -tr.Processes.Default.Command = "curl -v --tls-max 1.0 --tlsv1 --resolve 'bar.com:{0}:127.0.0.1' -k https://bar.com:{0}".format(ts.Variables.ssl_port) +tr.Processes.Default.Command = "curl -v --tls-max 1.0 --tlsv1 --resolve 'bar.com:{0}:127.0.0.1' -k https://bar.com:{0}".format( + ts.Variables.ssl_port) tr.ReturnCode = 35 tr.StillRunningAfter = ts # Target bar.com for TLSv1_2. Should succeed tr = Test.AddTestRun("bar.com TLSv1_2") -tr.Processes.Default.Command = "curl -v --tls-max 1.2 --tlsv1.2 --resolve 'bar.com:{0}:127.0.0.1' -k https://bar.com:{0}".format(ts.Variables.ssl_port) +tr.Processes.Default.Command = "curl -v --tls-max 1.2 --tlsv1.2 --resolve 'bar.com:{0}:127.0.0.1' -k https://bar.com:{0}".format( + ts.Variables.ssl_port) tr.ReturnCode = 0 tr.StillRunningAfter = ts diff --git a/tests/gold_tests/tls/tls_engine.test.py b/tests/gold_tests/tls/tls_engine.test.py new file mode 100644 index 00000000000..5be9f7c0e6f --- /dev/null +++ b/tests/gold_tests/tls/tls_engine.test.py @@ -0,0 +1,101 @@ +''' +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import os + + +# Someday should add client cert to origin to exercise the +# engine interface on the other side + +Test.Summary = ''' +Test tls via the async interface with the sample async_engine +''' + +Test.SkipUnless(Condition.HasOpenSSLVersion('1.1.1')) + +# Define default ATS +ts = Test.MakeATSProcess("ts", select_ports=True, enable_tls=True) +server = Test.MakeOriginServer("server") + +# Compile with tsxs. That should bring in the consisten versions of openssl +ts.Setup.Copy(os.path.join(Test.Variables.AtsTestToolsDir, '../../contrib/openssl', 'async_engine.c'), Test.RunDirectory) +ts.Setup.RunCommand("tsxs -o async_engine.so async_engine.c") + +# Add info the origin server responses +server.addResponse("sessionlog.json", + {"headers": "GET / HTTP/1.1\r\nuuid: basic\r\n\r\n", + "timestamp": "1469733493.993", + "body": ""}, + {"headers": "HTTP/1.1 200 OK\r\nServer: microserver\r\nConnection: close\r\nCache-Control: max-age=3600\r\nContent-Length: 2\r\n\r\n", + "timestamp": "1469733493.993", + "body": "ok"}) + +# add ssl materials like key, certificates for the server +ts.addSSLfile("ssl/server.pem") +ts.addSSLfile("ssl/server.key") + +ts.Disk.remap_config.AddLine( + 'map / http://127.0.0.1:{0}'.format(server.Variables.Port) +) + +ts.Disk.ssl_multicert_config.AddLine( + 'dest_ip=* ssl_cert_name=server.pem ssl_key_name=server.key' +) +ts.Disk.records_config.update({ + 'proxy.config.ssl.server.cert.path': '{0}'.format(ts.Variables.SSLDir), + 'proxy.config.ssl.server.private_key.path': '{0}'.format(ts.Variables.SSLDir), + 'proxy.config.exec_thread.autoconfig.scale': 1.0, + 'proxy.config.ssl.server.cipher_suite': 'ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA384:AES128-GCM-SHA256:AES256-GCM-SHA384:ECDHE-RSA-RC4-SHA:ECDHE-RSA-AES128-SHA:ECDHE-RSA-AES256-SHA:RC4-SHA:RC4-MD5:AES128-SHA:AES256-SHA:DES-CBC3-SHA!SRP:!DSS:!PSK:!aNULL:!eNULL:!SSLv2', + 'proxy.config.ssl.engine.conf_file': '{0}/ts/config/load_engine.cnf'.format(Test.RunDirectory), + 'proxy.config.ssl.async.handshake.enabled': 1, + 'proxy.config.diags.debug.enabled': 0, + 'proxy.config.diags.debug.tags': 'ssl' +}) + +ts.Disk.MakeConfigFile('load_engine.cnf').AddLines([ + 'openssl_conf = openssl_init', + '', + '[openssl_init]', + '', + 'engines = engine_section', + '', + '[engine_section]', + '', + 'async = async_section', + '', + '[async_section]', + '', + 'dynamic_path = {0}/async_engine.so'.format(Test.RunDirectory), + '', + 'engine_id = async-test', + '', + 'default_algorithms = RSA', + '', + 'init = 1']) + +# Make a basic request. Hopefully it goes through +tr = Test.AddTestRun("Run-Test") +tr.Processes.Default.Command = "curl -k -v -H uuid:basic -H host:example.com https://127.0.0.1:{0}/".format(ts.Variables.ssl_port) +tr.ReturnCode = 0 +tr.Processes.Default.StartBefore(server) +tr.Processes.Default.StartBefore(Test.Processes.ts, ready=When.PortOpen(ts.Variables.ssl_port)) +tr.Processes.Default.Streams.All = Testers.ContainsExpression(r"HTTP/(2|1\.1) 200", "Request succeeds") +tr.StillRunningAfter = server + +ts.Streams.All += Testers.ContainsExpression("Send signal to ", "The Async engine triggers") diff --git a/tests/gold_tests/tls/tls_forward_nonhttp.test.py b/tests/gold_tests/tls/tls_forward_nonhttp.test.py index bd1887bfe70..891cf1a0d25 100644 --- a/tests/gold_tests/tls/tls_forward_nonhttp.test.py +++ b/tests/gold_tests/tls/tls_forward_nonhttp.test.py @@ -49,10 +49,10 @@ # foo.com should not terminate. Just tunnel to server_foo # bar.com should terminate. Forward its tcp stream to server_bar ts.Disk.sni_yaml.AddLines([ - "sni:", - "- fqdn: bar.com", - " forward_route: localhost:4444" - ]) + "sni:", + "- fqdn: bar.com", + " forward_route: localhost:4444" +]) tr = Test.AddTestRun("forward-non-http") tr.Setup.Copy("test-nc-s_client.sh") @@ -61,5 +61,5 @@ tr.Processes.Default.StartBefore(Test.Processes.ts) tr.StillRunningAfter = ts testout_path = os.path.join(Test.RunDirectory, "test.out") -tr.Disk.File(testout_path, id = "testout") +tr.Disk.File(testout_path, id="testout") tr.Processes.Default.Streams.All += Testers.IncludesExpression("This is a reply", "s_client should get response") diff --git a/tests/gold_tests/tls/tls_hooks_client_verify.test.py b/tests/gold_tests/tls/tls_hooks_client_verify.test.py index 36092d4c0f9..97fd4dc297a 100644 --- a/tests/gold_tests/tls/tls_hooks_client_verify.test.py +++ b/tests/gold_tests/tls/tls_hooks_client_verify.test.py @@ -18,7 +18,6 @@ # limitations under the License. import os -import re Test.Summary = ''' Test different combinations of TLS handshake hooks to ensure they are applied consistently. @@ -71,7 +70,7 @@ ' verify_client: STRICT', ]) -Test.PreparePlugin(os.path.join(Test.Variables.AtsTestToolsDir, 'plugins', 'ssl_client_verify_test.cc'), ts, '-count=2 -good=foo.com') +Test.PrepareTestPlugin(os.path.join(Test.Variables.AtsTestPluginsDir, 'ssl_client_verify_test.so'), ts, '-count=2 -good=foo.com') tr = Test.AddTestRun("request good name") tr.Setup.Copy("ssl/signed-foo.pem") @@ -82,7 +81,8 @@ tr.Processes.Default.StartBefore(Test.Processes.ts) tr.StillRunningAfter = ts tr.StillRunningAfter = server -tr.Processes.Default.Command = "curl --tls-max 1.2 -k --cert ./signed-foo.pem --key ./signed-foo.key --resolve 'foo.com:{0}:127.0.0.1' https://foo.com:{0}/case1".format(ts.Variables.ssl_port) +tr.Processes.Default.Command = "curl --tls-max 1.2 -k --cert ./signed-foo.pem --key ./signed-foo.key --resolve 'foo.com:{0}:127.0.0.1' https://foo.com:{0}/case1".format( + ts.Variables.ssl_port) tr.Processes.Default.ReturnCode = 0 tr.Processes.Default.Streams.all = Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") @@ -90,7 +90,8 @@ tr2 = Test.AddTestRun("request bad name") tr2.StillRunningAfter = ts tr2.StillRunningAfter = server -tr2.Processes.Default.Command = "curl --tls-max 1.2 -k --cert ./signed-bar.pem --key ./signed-bar.key --resolve 'foo.com:{0}:127.0.0.1' https://foo.com:{0}/case1".format(ts.Variables.ssl_port) +tr2.Processes.Default.Command = "curl --tls-max 1.2 -k --cert ./signed-bar.pem --key ./signed-bar.key --resolve 'foo.com:{0}:127.0.0.1' https://foo.com:{0}/case1".format( + ts.Variables.ssl_port) tr2.Processes.Default.ReturnCode = 35 tr2.Processes.Default.Streams.all = Testers.ContainsExpression("error", "Curl attempt should have failed") @@ -99,11 +100,16 @@ tr3.Setup.Copy("ssl/server.key") tr3.StillRunningAfter = ts tr3.StillRunningAfter = server -tr3.Processes.Default.Command = "curl --tls-max 1.2 -k --cert ./server.pem --key ./server.key --resolve 'foo.com:{0}:127.0.0.1' https://foo.com:{0}/case1".format(ts.Variables.ssl_port) +tr3.Processes.Default.Command = "curl --tls-max 1.2 -k --cert ./server.pem --key ./server.key --resolve 'foo.com:{0}:127.0.0.1' https://foo.com:{0}/case1".format( + ts.Variables.ssl_port) tr3.Processes.Default.ReturnCode = 35 tr3.Processes.Default.Streams.all = Testers.ContainsExpression("error", "Curl attempt should have failed") -ts.Streams.All += Testers.ContainsExpression("Client verify callback 0 [\da-fx]+? - event is good good HS", "verify callback happens 2 times") -ts.Streams.All += Testers.ContainsExpression("Client verify callback 1 [\da-fx]+? - event is good good HS", "verify callback happens 2 times") -ts.Streams.All += Testers.ContainsExpression("Client verify callback 0 [\da-fx]+? - event is good error HS", "verify callback happens 2 times") -ts.Streams.All += Testers.ContainsExpression("Client verify callback 1 [\da-fx]+? - event is good error HS", "verify callback happens 2 times") +ts.Streams.All += Testers.ContainsExpression( + "Client verify callback 0 [\da-fx]+? - event is good good HS", "verify callback happens 2 times") +ts.Streams.All += Testers.ContainsExpression( + "Client verify callback 1 [\da-fx]+? - event is good good HS", "verify callback happens 2 times") +ts.Streams.All += Testers.ContainsExpression( + "Client verify callback 0 [\da-fx]+? - event is good error HS", "verify callback happens 2 times") +ts.Streams.All += Testers.ContainsExpression( + "Client verify callback 1 [\da-fx]+? - event is good error HS", "verify callback happens 2 times") diff --git a/tests/gold_tests/tls/tls_hooks_verify.test.py b/tests/gold_tests/tls/tls_hooks_verify.test.py index dae7e9b9e75..44dc888c707 100644 --- a/tests/gold_tests/tls/tls_hooks_verify.test.py +++ b/tests/gold_tests/tls/tls_hooks_verify.test.py @@ -18,7 +18,6 @@ # limitations under the License. import os -import re Test.Summary = ''' Test different combinations of TLS handshake hooks to ensure they are applied consistently. @@ -56,17 +55,17 @@ 'map https://bar.com:{1}/ https://127.0.0.1:{0}'.format(server.Variables.SSL_Port, ts.Variables.ssl_port) ) ts.Disk.remap_config.AddLine( - 'map https://random.com:{1}/ https://127.0.0.1:{0}'.format(server.Variables.SSL_Port,ts.Variables.ssl_port) + 'map https://random.com:{1}/ https://127.0.0.1:{0}'.format(server.Variables.SSL_Port, ts.Variables.ssl_port) ) ts.Disk.sni_yaml.AddLine( - 'sni:') + 'sni:') ts.Disk.sni_yaml.AddLine( - '- fqdn: bar.com') + '- fqdn: bar.com') ts.Disk.sni_yaml.AddLine( - ' verify_server_policy: PERMISSIVE') + ' verify_server_policy: PERMISSIVE') -Test.PreparePlugin(os.path.join(Test.Variables.AtsTestToolsDir, 'plugins', 'ssl_verify_test.cc'), ts, '-count=2 -bad=random.com -bad=bar.com') +Test.PrepareTestPlugin(os.path.join(Test.Variables.AtsTestPluginsDir, 'ssl_verify_test.so'), ts, '-count=2 -bad=random.com -bad=bar.com') tr = Test.AddTestRun("request good name") tr.Processes.Default.StartBefore(server) @@ -81,7 +80,8 @@ tr2 = Test.AddTestRun("request bad name") tr2.StillRunningAfter = ts tr2.StillRunningAfter = server -tr2.Processes.Default.Command = "curl --resolve \"random.com:{0}:127.0.0.1\" -k https://random.com:{0}".format(ts.Variables.ssl_port) +tr2.Processes.Default.Command = "curl --resolve \"random.com:{0}:127.0.0.1\" -k https://random.com:{0}".format( + ts.Variables.ssl_port) tr2.Processes.Default.ReturnCode = 0 tr2.Processes.Default.Streams.stdout = Testers.ContainsExpression("Could Not Connect", "Curl attempt should have failed") @@ -93,13 +93,29 @@ tr3.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have failed") # Over riding the built in ERROR check since we expect tr2 to fail -ts.Disk.diags_log.Content = Testers.ContainsExpression("WARNING: TS_EVENT_SSL_VERIFY_SERVER plugin failed the origin certificate check for 127.0.0.1. Action=Terminate SNI=random.com", "random.com should fail") -ts.Disk.diags_log.Content += Testers.ContainsExpression("WARNING: TS_EVENT_SSL_VERIFY_SERVER plugin failed the origin certificate check for 127.0.0.1. Action=Continue SNI=bar.com", "bar.com should fail but continue") +ts.Disk.diags_log.Content = Testers.ContainsExpression( + "WARNING: TS_EVENT_SSL_VERIFY_SERVER plugin failed the origin certificate check for 127.0.0.1. Action=Terminate SNI=random.com", + "random.com should fail") +ts.Disk.diags_log.Content += Testers.ContainsExpression( + "WARNING: TS_EVENT_SSL_VERIFY_SERVER plugin failed the origin certificate check for 127.0.0.1. Action=Continue SNI=bar.com", + "bar.com should fail but continue") ts.Disk.diags_log.Content += Testers.ExcludesExpression("SNI=foo.com", "foo.com should not fail in any way") -ts.Streams.All += Testers.ContainsExpression("Server verify callback 0 [\da-fx]+? - event is good SNI=foo.com good HS", "verify callback happens 2 times") -ts.Streams.All += Testers.ContainsExpression("Server verify callback 1 [\da-fx]+? - event is good SNI=foo.com good HS", "verify callback happens 2 times") -ts.Streams.All += Testers.ContainsExpression("Server verify callback 0 [\da-fx]+? - event is good SNI=random.com error HS", "verify callback happens 2 times") -ts.Streams.All += Testers.ContainsExpression("Server verify callback 1 [\da-fx]+? - event is good SNI=random.com error HS", "verify callback happens 2 times") -ts.Streams.All += Testers.ContainsExpression("Server verify callback 0 [\da-fx]+? - event is good SNI=bar.com error HS", "verify callback happens 2 times") -ts.Streams.All += Testers.ContainsExpression("Server verify callback 1 [\da-fx]+? - event is good SNI=bar.com error HS", "verify callback happens 2 times") +ts.Streams.All += Testers.ContainsExpression( + r"Server verify callback 0 [\da-fx]+? - event is good SNI=foo.com good HS", + "verify callback happens 2 times") +ts.Streams.All += Testers.ContainsExpression( + r"Server verify callback 1 [\da-fx]+? - event is good SNI=foo.com good HS", + "verify callback happens 2 times") +ts.Streams.All += Testers.ContainsExpression( + r"Server verify callback 0 [\da-fx]+? - event is good SNI=random.com error HS", + "verify callback happens 2 times") +ts.Streams.All += Testers.ContainsExpression( + r"Server verify callback 1 [\da-fx]+? - event is good SNI=random.com error HS", + "verify callback happens 2 times") +ts.Streams.All += Testers.ContainsExpression( + r"Server verify callback 0 [\da-fx]+? - event is good SNI=bar.com error HS", + "verify callback happens 2 times") +ts.Streams.All += Testers.ContainsExpression( + r"Server verify callback 1 [\da-fx]+? - event is good SNI=bar.com error HS", + "verify callback happens 2 times") diff --git a/tests/gold_tests/tls/tls_keepalive.test.py b/tests/gold_tests/tls/tls_keepalive.test.py index 72c32f008ce..021051f75b4 100644 --- a/tests/gold_tests/tls/tls_keepalive.test.py +++ b/tests/gold_tests/tls/tls_keepalive.test.py @@ -18,7 +18,6 @@ # limitations under the License. import os -import re Test.Summary = ''' Verify that the client-side keep alive is honored for TLS and different versions of HTTP @@ -56,7 +55,7 @@ ) ts.Disk.logging_yaml.AddLines( -''' + ''' logging: formats: - name: testformat @@ -68,32 +67,36 @@ '''.split("\n") ) -Test.PreparePlugin(os.path.join(Test.Variables.AtsTestToolsDir, 'plugins', 'ssl_hook_test.cc'), ts, '-preaccept=1') +Test.PrepareTestPlugin(os.path.join(Test.Variables.AtsTestPluginsDir, 'ssl_hook_test.so'), ts, '-preaccept=1') tr = Test.AddTestRun("Test two HTTP/1.1 requests over one TLS connection") tr.Processes.Default.StartBefore(server) tr.Processes.Default.StartBefore(Test.Processes.ts) tr.StillRunningAfter = ts tr.StillRunningAfter = server -tr.Processes.Default.Command = 'curl -k -v --http1.1 -H \'host:example.com:{0}\' https://127.0.0.1:{0} https://127.0.0.1:{0}'.format(ts.Variables.ssl_port) +tr.Processes.Default.Command = 'curl -k -v --http1.1 -H \'host:example.com:{0}\' https://127.0.0.1:{0} https://127.0.0.1:{0}'.format( + ts.Variables.ssl_port) tr.Processes.Default.ReturnCode = 0 tr = Test.AddTestRun("Test two HTTP/1.1 requests over two TLS connections") tr.StillRunningAfter = ts tr.StillRunningAfter = server -tr.Processes.Default.Command = 'curl -k -v --http1.1 -H \'host:example.com:{0}\' https://127.0.0.1:{0}; curl -k -v --http1.1 -H \'host:example.com:{0}\' https://127.0.0.1:{0}'.format(ts.Variables.ssl_port) +tr.Processes.Default.Command = 'curl -k -v --http1.1 -H \'host:example.com:{0}\' https://127.0.0.1:{0}; curl -k -v --http1.1 -H \'host:example.com:{0}\' https://127.0.0.1:{0}'.format( + ts.Variables.ssl_port) tr.Processes.Default.ReturnCode = 0 tr = Test.AddTestRun("Test two HTTP/2 requests over one TLS connection") tr.StillRunningAfter = ts tr.StillRunningAfter = server -tr.Processes.Default.Command = 'curl -k -v --http2 -H \'host:example.com:{0}\' https://127.0.0.1:{0} https://127.0.0.1:{0}'.format(ts.Variables.ssl_port) +tr.Processes.Default.Command = 'curl -k -v --http2 -H \'host:example.com:{0}\' https://127.0.0.1:{0} https://127.0.0.1:{0}'.format( + ts.Variables.ssl_port) tr.Processes.Default.ReturnCode = 0 tr = Test.AddTestRun("Test two HTTP/2 requests over two TLS connections") tr.StillRunningAfter = ts tr.StillRunningAfter = server -tr.Processes.Default.Command = 'curl -k -v --http2 -H \'host:example.com:{0}\' https://127.0.0.1:{0}; curl -k -v --http1.1 -H \'host:example.com:{0}\' https://127.0.0.1:{0}'.format(ts.Variables.ssl_port) +tr.Processes.Default.Command = 'curl -k -v --http2 -H \'host:example.com:{0}\' https://127.0.0.1:{0}; curl -k -v --http1.1 -H \'host:example.com:{0}\' https://127.0.0.1:{0}'.format( + ts.Variables.ssl_port) tr.Processes.Default.ReturnCode = 0 # Just a check to flush out the traffic log until we have a clean shutdown for traffic_server diff --git a/tests/gold_tests/tls/tls_ocsp.test.py b/tests/gold_tests/tls/tls_ocsp.test.py index 127071b1eac..12d91e4246e 100644 --- a/tests/gold_tests/tls/tls_ocsp.test.py +++ b/tests/gold_tests/tls/tls_ocsp.test.py @@ -66,5 +66,6 @@ tr.Processes.Default.StartBefore(Test.Processes.ts) tr.StillRunningAfter = server tr.StillRunningAfter = ts -tr.Processes.Default.Command = "curl -v --cacert {0} --cert-status -H \"host:example.com\" https://127.0.0.1:{1}".format(os.path.join(ts.Variables.SSLDir, "ca.ocsp.pem"), ts.Variables.ssl_port) +tr.Processes.Default.Command = "curl -v --cacert {0} --cert-status -H \"host:example.com\" https://127.0.0.1:{1}".format( + os.path.join(ts.Variables.SSLDir, "ca.ocsp.pem"), ts.Variables.ssl_port) tr.ReturnCode = 0 diff --git a/tests/gold_tests/tls/tls_partial_blind_tunnel.test.py b/tests/gold_tests/tls/tls_partial_blind_tunnel.test.py new file mode 100644 index 00000000000..44c506274c2 --- /dev/null +++ b/tests/gold_tests/tls/tls_partial_blind_tunnel.test.py @@ -0,0 +1,71 @@ +''' +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +Test.Summary = 'Testing partial_blind_tunnel' + +ts = Test.MakeATSProcess("ts", select_ports=True, enable_tls=True) +server_bar = Test.MakeOriginServer("server_bar", ssl=True) + +request_bar_header = {"headers": "GET / HTTP/1.1\r\nHost: bar.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +response_bar_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": "ok bar"} +server_bar.addResponse("sessionlog_bar.json", request_bar_header, response_bar_header) + +# add ssl materials like key, certificates for the server +ts.addSSLfile("ssl/signed-foo.pem") +ts.addSSLfile("ssl/signed-foo.key") +ts.addSSLfile("ssl/signed-bar.pem") +ts.addSSLfile("ssl/signed-bar.key") +ts.addSSLfile("ssl/signer.pem") + +# Need no remap rules. Everything should be proccessed by sni + +# Make sure the TS server certs are different from the origin certs +ts.Disk.ssl_multicert_config.AddLine( + 'dest_ip=* ssl_cert_name=signed-foo.pem ssl_key_name=signed-foo.key' +) + +# Case 1, global config policy=permissive properties=signature +# override for foo.com policy=enforced properties=all +ts.Disk.records_config.update({ + 'proxy.config.ssl.server.cert.path': '{0}'.format(ts.Variables.SSLDir), + 'proxy.config.ssl.server.private_key.path': '{0}'.format(ts.Variables.SSLDir), + 'proxy.config.http.connect_ports': '{0} {1}'.format(ts.Variables.ssl_port, server_bar.Variables.SSL_Port), + 'proxy.config.ssl.client.CA.cert.path': '{0}'.format(ts.Variables.SSLDir), + 'proxy.config.ssl.client.CA.cert.filename': 'signer.pem', +}) + +# foo.com should terminate. and reconnect via TLS upstream to bar.com +ts.Disk.sni_yaml.AddLines([ + "sni:", + "- fqdn: 'foo.com'", + " partial_blind_route: 'localhost:{0}'".format(server_bar.Variables.SSL_Port), +]) + +tr = Test.AddTestRun("Partial Blind Route") +tr.Processes.Default.Command = "curl --http1.1 -v --resolve 'foo.com:{0}:127.0.0.1' -k https://foo.com:{0}".format( + ts.Variables.ssl_port) +tr.ReturnCode = 0 +tr.Processes.Default.StartBefore(server_bar) +tr.Processes.Default.StartBefore(Test.Processes.ts) +tr.StillRunningAfter = ts + +tr.Processes.Default.Streams.All += Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") +tr.Processes.Default.Streams.All += Testers.ExcludesExpression("Not Found on Accelerato", + "Should not try to remap on Traffic Server") +tr.Processes.Default.Streams.All += Testers.ContainsExpression("HTTP/1.1 200 OK", "Should get a successful response") +tr.Processes.Default.Streams.All += Testers.ContainsExpression("ok bar", "Body is expected") diff --git a/tests/gold_tests/tls/tls_session_cache.test.py b/tests/gold_tests/tls/tls_session_cache.test.py new file mode 100644 index 00000000000..5146103705f --- /dev/null +++ b/tests/gold_tests/tls/tls_session_cache.test.py @@ -0,0 +1,94 @@ +''' +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import re +Test.Summary = ''' +Test tls session cache +''' + +# Define default ATS +ts = Test.MakeATSProcess("ts", select_ports=True, enable_tls=True) +server = Test.MakeOriginServer("server") + + +# Add info the origin server responses +request_header = {"headers": "GET / HTTP/1.1\r\nHost: www.example.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +response_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +server.addResponse("sessionlog.json", request_header, response_header) + +# add ssl materials like key, certificates for the server +ts.addSSLfile("ssl/server.pem") +ts.addSSLfile("ssl/server.key") + +ts.Disk.remap_config.AddLine( + 'map / http://127.0.0.1:{0}'.format(server.Variables.Port) +) + +ts.Disk.ssl_multicert_config.AddLine( + 'dest_ip=* ssl_cert_name=server.pem ssl_key_name=server.key' +) + +ts.Disk.records_config.update({ + 'proxy.config.ssl.server.cert.path': '{0}'.format(ts.Variables.SSLDir), + 'proxy.config.ssl.server.private_key.path': '{0}'.format(ts.Variables.SSLDir), + 'proxy.config.ssl.server.cipher_suite': 'ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA384:AES128-GCM-SHA256:AES256-GCM-SHA384:ECDHE-RSA-RC4-SHA:ECDHE-RSA-AES128-SHA:ECDHE-RSA-AES256-SHA:RC4-SHA:RC4-MD5:AES128-SHA:AES256-SHA:DES-CBC3-SHA!SRP:!DSS:!PSK:!aNULL:!eNULL:!SSLv2', + 'proxy.config.exec_thread.autoconfig.scale': 1.0, + 'proxy.config.ssl.session_cache': 2, + 'proxy.config.ssl.session_cache.size': 4096, + 'proxy.config.ssl.session_cache.num_buckets': 256, + 'proxy.config.ssl.session_cache.skip_cache_on_bucket_contention': 0, + 'proxy.config.ssl.session_cache.timeout': 0, + 'proxy.config.ssl.session_cache.auto_clear': 1, + 'proxy.config.ssl.server.session_ticket.enable': 0, +}) + +# Check that Session-ID is the same on every connection + + +def checkSession(ev): + retval = False + f = open(openssl_output, 'r') + err = "Session ids match" + if not f: + err = "Failed to open {0}".format(openssl_output) + return (retval, "Check that session ids match", err) + + content = f.read() + match = re.findall('Session-ID: ([0-9A-F]+)', content) + + if match: + if all(i == j for i, j in zip(match, match[1:])): + err = "{0} reused successfully {1} times".format(match[0], len(match)) + retval = True + else: + err = "Session is not being reused as expected" + else: + err = "Didn't find session id" + return (retval, "Check that session ids match", err) + + +tr = Test.AddTestRun("OpenSSL s_client -reconnect") +tr.Command = 'echo -e "GET / HTTP/1.0\r\n" | openssl s_client -tls1_2 -connect 127.0.0.1:{0} -reconnect'.format( + ts.Variables.ssl_port) +tr.ReturnCode = 0 +# time delay as proxy.config.http.wait_for_cache could be broken +tr.Processes.Default.StartBefore(server) +tr.Processes.Default.StartBefore(Test.Processes.ts) +openssl_output = tr.Processes.Default.Streams.stdout.AbsPath +tr.Processes.Default.Streams.All.Content = Testers.Lambda(checkSession) +tr.StillRunningAfter = server diff --git a/tests/gold_tests/tls/tls_sni_host_policy.test.py b/tests/gold_tests/tls/tls_sni_host_policy.test.py new file mode 100644 index 00000000000..70d18feeae5 --- /dev/null +++ b/tests/gold_tests/tls/tls_sni_host_policy.test.py @@ -0,0 +1,158 @@ +''' +Test exercising host and SNI mismatch controls +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +Test.Summary = ''' +Test exercising host and SNI mismatch controls +''' + +ts = Test.MakeATSProcess("ts", select_ports=True, enable_tls=True) +cafile = "{0}/signer.pem".format(Test.RunDirectory) +cafile2 = "{0}/signer2.pem".format(Test.RunDirectory) +server = Test.MakeOriginServer("server") + +request_header = {"headers": "GET /case1 HTTP/1.1\r\nHost: example.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +response_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +server.addResponse("sessionlog.json", request_header, response_header) +request_header = {"headers": "GET / HTTP/1.1\r\nHost: bar.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +response_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +server.addResponse("sessionlog.json", request_header, response_header) + +ts.addSSLfile("ssl/server.pem") +ts.addSSLfile("ssl/server.key") +ts.addSSLfile("ssl/signer.pem") + +ts.Disk.records_config.update({ + 'proxy.config.ssl.server.cert.path': '{0}'.format(ts.Variables.SSLDir), + 'proxy.config.ssl.server.private_key.path': '{0}'.format(ts.Variables.SSLDir), + 'proxy.config.ssl.client.verify.server': 0, + 'proxy.config.url_remap.pristine_host_hdr': 1, + 'proxy.config.ssl.CA.cert.filename': '{0}/signer.pem'.format(ts.Variables.SSLDir), + 'proxy.config.exec_thread.autoconfig.scale': 1.0, + 'proxy.config.http.host_sni_policy': 2, + 'proxy.config.ssl.TLSv1_3': 0 +}) + +ts.Disk.ssl_multicert_config.AddLine( + 'dest_ip=* ssl_cert_name=server.pem ssl_key_name=server.key' +) + +# Just map everything through to origin. This test is concentratign on the user-agent side +ts.Disk.remap_config.AddLine( + 'map / http://127.0.0.1:{0}/'.format(server.Variables.Port) +) + +# Scenario 1: Default no client cert required. cert required for bar.com +ts.Disk.sni_yaml.AddLines([ + 'sni:', + '- fqdn: boblite', + ' verify_client: STRICT', + ' host_sni_policy: PERMISSIVE', + '- fqdn: bob', + ' verify_client: STRICT', +]) + +# case 1 +# sni=bob and host=dave. Do not provide client cert. Should fail +tr = Test.AddTestRun("Connect to bob without cert") +tr.Processes.Default.StartBefore(Test.Processes.ts) +tr.Processes.Default.StartBefore(server) +tr.StillRunningAfter = ts +tr.StillRunningAfter = server +tr.Processes.Default.Command = "curl --tls-max 1.2 -k -H 'host:dave' --resolve 'bob:{0}:127.0.0.1' https://bob:{0}/case1".format( + ts.Variables.ssl_port) +tr.Processes.Default.ReturnCode = 35 + +# case 2 +# sni=bob and host=dave. Do provide client cert. Should succeed +tr = Test.AddTestRun("Connect to bob with good cert") +tr.Setup.Copy("ssl/signed-foo.pem") +tr.Setup.Copy("ssl/signed-foo.key") +tr.StillRunningAfter = ts +tr.StillRunningAfter = server +tr.Processes.Default.Command = "curl --tls-max 1.2 -k --cert ./signed-foo.pem --key ./signed-foo.key -H 'host:dave' --resolve 'bob:{0}:127.0.0.1' https://bob:{0}/case1".format( + ts.Variables.ssl_port) +tr.Processes.Default.ReturnCode = 0 + +# case 3 +# sni=dave and host=bob. Do not provide client cert. Should fail due to sni-host mismatch +tr = Test.AddTestRun("Connect to dave without cert") +tr.StillRunningAfter = ts +tr.StillRunningAfter = server +tr.Processes.Default.Command = "curl --tls-max 1.2 -k -H 'host:bob' --resolve 'dave:{0}:127.0.0.1' https://dave:{0}/case1".format( + ts.Variables.ssl_port) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Streams.All = Testers.ContainsExpression("Access Denied", "Check response") + +# case 4 +# sni=dave and host=bob. Do provide client cert. Should fail due to sni-host mismatch +tr = Test.AddTestRun("Connect to dave with cert") +tr.StillRunningAfter = ts +tr.StillRunningAfter = server +tr.Processes.Default.Command = "curl --tls-max 1.2 -k --cert ./signed-foo.pem --key ./signed-foo.key -H 'host:bob' --resolve 'dave:{0}:127.0.0.1' https://dave:{0}/case1".format( + ts.Variables.ssl_port) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Streams.All = Testers.ContainsExpression("Access Denied", "Check response") + +# case 5 +# sni=ellen and host=boblite. Do not provide client cert. Should warn due to sni-host mismatch +tr = Test.AddTestRun("Connect to ellen without cert") +tr.StillRunningAfter = ts +tr.StillRunningAfter = server +tr.Processes.Default.Command = "curl --tls-max 1.2 -k -H 'host:boblite' --resolve 'ellen:{0}:127.0.0.1' https://ellen:{0}/warnonly".format( + ts.Variables.ssl_port) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Streams.All = Testers.ExcludesExpression("Access Denied", "Check response") + +# case 6 +# sni=ellen and host=boblite. Do provide client cert. Should warn due to sni-host mismatch +tr = Test.AddTestRun("Connect to ellen with cert") +tr.StillRunningAfter = ts +tr.StillRunningAfter = server +tr.Processes.Default.Command = "curl --tls-max 1.2 -k --cert ./signed-foo.pem --key ./signed-foo.key -H 'host:boblite' --resolve 'ellen:{0}:127.0.0.1' https://ellen:{0}/warnonly".format( + ts.Variables.ssl_port) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Streams.All = Testers.ExcludesExpression("Access Denied", "Check response") + +# case 7 +# sni=ellen and host=fran. Do not provide client cert. No warning since neither name is mentioned in sni.yaml +tr = Test.AddTestRun("Connect to ellen without cert") +tr.StillRunningAfter = ts +tr.StillRunningAfter = server +tr.Processes.Default.Command = "curl --tls-max 1.2 -k -H 'host:fran' --resolve 'ellen:{0}:127.0.0.1' https://ellen:{0}/warnonly".format( + ts.Variables.ssl_port) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Streams.All = Testers.ExcludesExpression("Access Denied", "Check response") + +# case 8 +# sni=ellen and host=fran. Do provide client cert. No warning since neither name is mentioned in sni.yaml +tr = Test.AddTestRun("Connect to ellen with cert") +tr.StillRunningAfter = ts +tr.StillRunningAfter = server +tr.Processes.Default.Command = "curl --tls-max 1.2 -k --cert ./signed-foo.pem --key ./signed-foo.key -H 'host:fran' --resolve 'ellen:{0}:127.0.0.1' https://ellen:{0}/warnonly".format( + ts.Variables.ssl_port) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Streams.All = Testers.ExcludesExpression("Access Denied", "Check response") + + +ts.Disk.diags_log.Content += Testers.ContainsExpression( + "WARNING: SNI/hostname mismatch sni=dave host=bob action=terminate", "Should have warning on mismatch") +ts.Disk.diags_log.Content += Testers.ContainsExpression( + "WARNING: SNI/hostname mismatch sni=ellen host=boblite action=continue", "Should have warning on mismatch") +ts.Disk.diags_log.Content += Testers.ExcludesExpression("WARNING: SNI/hostname mismatch sni=ellen host=fran", + "Should not have warning on mismatch with non-policy host") diff --git a/tests/gold_tests/tls/tls_ticket.test.py b/tests/gold_tests/tls/tls_ticket.test.py index 4d34ddb679a..a8fc3b9b02d 100644 --- a/tests/gold_tests/tls/tls_ticket.test.py +++ b/tests/gold_tests/tls/tls_ticket.test.py @@ -16,7 +16,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import os import re Test.Summary = ''' Test tls tickets @@ -56,7 +55,6 @@ ts.Disk.records_config.update({ 'proxy.config.ssl.server.cert.path': '{0}'.format(ts.Variables.SSLDir), 'proxy.config.ssl.server.private_key.path': '{0}'.format(ts.Variables.SSLDir), - 'proxy.config.ssl.client.verify.server': 0, 'proxy.config.ssl.server.cipher_suite': 'ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA384:AES128-GCM-SHA256:AES256-GCM-SHA384:ECDHE-RSA-RC4-SHA:ECDHE-RSA-AES128-SHA:ECDHE-RSA-AES256-SHA:RC4-SHA:RC4-MD5:AES128-SHA:AES256-SHA:DES-CBC3-SHA!SRP:!DSS:!PSK:!aNULL:!eNULL:!SSLv2', 'proxy.config.exec_thread.autoconfig.scale': 1.0, 'proxy.config.ssl.server.session_ticket.enable': '1', @@ -65,7 +63,6 @@ ts2.Disk.records_config.update({ 'proxy.config.ssl.server.cert.path': '{0}'.format(ts2.Variables.SSLDir), 'proxy.config.ssl.server.private_key.path': '{0}'.format(ts2.Variables.SSLDir), - 'proxy.config.ssl.client.verify.server': 0, 'proxy.config.ssl.server.cipher_suite': 'ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA384:AES128-GCM-SHA256:AES256-GCM-SHA384:ECDHE-RSA-RC4-SHA:ECDHE-RSA-AES128-SHA:ECDHE-RSA-AES256-SHA:RC4-SHA:RC4-MD5:AES128-SHA:AES256-SHA:DES-CBC3-SHA!SRP:!DSS:!PSK:!aNULL:!eNULL:!SSLv2', 'proxy.config.ssl.server.session_ticket.enable': '1', 'proxy.config.exec_thread.autoconfig.scale': 1.0, @@ -75,7 +72,8 @@ tr = Test.AddTestRun("Create ticket") tr.Setup.Copy('file.ticket') -tr.Command = 'echo -e "GET / HTTP/1.0\r\n" | openssl s_client -tls1_2 -connect 127.0.0.1:{0} -sess_out ticket.out'.format(ts.Variables.ssl_port) +tr.Command = 'echo -e "GET / HTTP/1.0\r\n" | openssl s_client -tls1_2 -connect 127.0.0.1:{0} -sess_out ticket.out'.format( + ts.Variables.ssl_port) tr.ReturnCode = 0 # time delay as proxy.config.http.wait_for_cache could be broken tr.Processes.Default.StartBefore(server) @@ -84,34 +82,37 @@ tr.StillRunningAfter = server # Pull out session created in tr to test for session id in tr2 -def checkSession(ev) : - retval = False - f1 = open(path1, 'r') - f2 = open(path2, 'r') - err = "Session ids match" - if not f1 or not f2: - err = "Failed to open {0} or {1}".format(path1, path2) - return (retval, "Check that session ids match", err) - f1Content = f1.read() - f2Content = f2.read() - sessRegex = re.compile('Session-ID: ([0-9A-F]+)') - match1 = re.findall('Session-ID: ([0-9A-F]+)', f1Content) - match2 = re.findall('Session-ID: ([0-9A-F]+)', f2Content) - if match1 and match2: - if match1[0] == match2[0]: - err = "{0} and {1} do match".format(match1[0], match2[0]) - retval = True +def checkSession(ev): + retval = False + f1 = open(path1, 'r') + f2 = open(path2, 'r') + err = "Session ids match" + if not f1 or not f2: + err = "Failed to open {0} or {1}".format(path1, path2) + return (retval, "Check that session ids match", err) + + f1Content = f1.read() + f2Content = f2.read() + match1 = re.findall('Session-ID: ([0-9A-F]+)', f1Content) + match2 = re.findall('Session-ID: ([0-9A-F]+)', f2Content) + + if match1 and match2: + if match1[0] == match2[0]: + err = "{0} and {1} do match".format(match1[0], match2[0]) + retval = True + else: + err = "{0} and {1} do not match".format(match1[0], match2[0]) else: - err = "{0} and {1} do not match".format(match1[0], match2[0]) - else: - err = "Didn't find session id" - return (retval, "Check that session ids match", err) + err = "Didn't find session id" + return (retval, "Check that session ids match", err) + tr2 = Test.AddTestRun("Test ticket") tr2.Setup.Copy('file.ticket') -tr2.Command = 'echo -e "GET / HTTP/1.0\r\n" | openssl s_client -tls1_2 -connect 127.0.0.1:{0} -sess_in ticket.out'.format(ts2.Variables.ssl_port) +tr2.Command = 'echo -e "GET / HTTP/1.0\r\n" | openssl s_client -tls1_2 -connect 127.0.0.1:{0} -sess_in ticket.out'.format( + ts2.Variables.ssl_port) tr2.Processes.Default.StartBefore(Test.Processes.ts2) tr2.ReturnCode = 0 path2 = tr2.Processes.Default.Streams.stdout.AbsPath diff --git a/tests/gold_tests/tls/tls_tunnel.test.py b/tests/gold_tests/tls/tls_tunnel.test.py index c168a20bad8..9b2846400f2 100644 --- a/tests/gold_tests/tls/tls_tunnel.test.py +++ b/tests/gold_tests/tls/tls_tunnel.test.py @@ -16,8 +16,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import os -import subprocess Test.Summary = ''' Test tunneling based on SNI ''' @@ -27,6 +25,8 @@ server_foo = Test.MakeOriginServer("server_foo", ssl=True) server_bar = Test.MakeOriginServer("server_bar", ssl=True) server2 = Test.MakeOriginServer("server2") +#dns = Test.MakeDNServer("dns", default=['127.0.0.1']) +dns = Test.MakeDNServer("dns") request_foo_header = {"headers": "GET / HTTP/1.1\r\nHost: foo.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} request_bar_header = {"headers": "GET / HTTP/1.1\r\nHost: bar.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} @@ -45,6 +45,9 @@ ts.addSSLfile("ssl/signer.pem") ts.addSSLfile("ssl/signer.key") +dns.addRecords(records={"localhost": ["127.0.0.1"]}) +dns.addRecords(records={"one.testmatch": ["127.0.0.1"]}) +dns.addRecords(records={"two.example.one": ["127.0.0.1"]}) # Need no remap rules. Everything should be proccessed by sni # Make sure the TS server certs are different from the origin certs @@ -57,25 +60,31 @@ ts.Disk.records_config.update({ 'proxy.config.ssl.server.cert.path': '{0}'.format(ts.Variables.SSLDir), 'proxy.config.ssl.server.private_key.path': '{0}'.format(ts.Variables.SSLDir), - 'proxy.config.http.connect_ports': '{0} {1} {2}'.format(ts.Variables.ssl_port,server_foo.Variables.SSL_Port,server_bar.Variables.SSL_Port), + 'proxy.config.http.connect_ports': '{0} {1} {2}'.format(ts.Variables.ssl_port, server_foo.Variables.SSL_Port, server_bar.Variables.SSL_Port), 'proxy.config.ssl.server.cipher_suite': 'ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA384:AES128-GCM-SHA256:AES256-GCM-SHA384:ECDHE-RSA-RC4-SHA:ECDHE-RSA-AES128-SHA:ECDHE-RSA-AES256-SHA:RC4-SHA:RC4-MD5:AES128-SHA:AES256-SHA:DES-CBC3-SHA!SRP:!DSS:!PSK:!aNULL:!eNULL:!SSLv2', 'proxy.config.ssl.client.CA.cert.path': '{0}'.format(ts.Variables.SSLDir), 'proxy.config.ssl.client.CA.cert.filename': 'signer.pem', 'proxy.config.exec_thread.autoconfig.scale': 1.0, - 'proxy.config.url_remap.pristine_host_hdr': 1 + 'proxy.config.url_remap.pristine_host_hdr': 1, + 'proxy.config.dns.nameservers': '127.0.0.1:{0}'.format(dns.Variables.Port), + 'proxy.config.dns.resolv_conf': 'NULL' }) # foo.com should not terminate. Just tunnel to server_foo # bar.com should terminate. Forward its tcp stream to server_bar # empty SNI should tunnel to server_bar ts.Disk.sni_yaml.AddLines([ - 'sni:', - '- fqdn: foo.com', - " tunnel_route: localhost:{0}".format(server_foo.Variables.SSL_Port), - "- fqdn: bob.*.com", - " tunnel_route: localhost:{0}".format(server_foo.Variables.SSL_Port), - "- fqdn: ''", # No SNI sent - " tunnel_route: localhost:{0}".format(server_bar.Variables.SSL_Port) + 'sni:', + '- fqdn: foo.com', + " tunnel_route: localhost:{0}".format(server_foo.Variables.SSL_Port), + "- fqdn: bob.*.com", + " tunnel_route: localhost:{0}".format(server_foo.Variables.SSL_Port), + "- fqdn: '*.match.com'", + " tunnel_route: $1.testmatch:{0}".format(server_foo.Variables.SSL_Port), + "- fqdn: '*.ok.*.com'", + " tunnel_route: $2.example.$1:{0}".format(server_foo.Variables.SSL_Port), + "- fqdn: ''", # No SNI sent + " tunnel_route: localhost:{0}".format(server_bar.Variables.SSL_Port) ]) tr = Test.AddTestRun("foo.com Tunnel-test") @@ -83,21 +92,25 @@ tr.ReturnCode = 0 tr.Processes.Default.StartBefore(server_foo) tr.Processes.Default.StartBefore(server_bar) +tr.Processes.Default.StartBefore(dns) tr.Processes.Default.StartBefore(Test.Processes.ts) tr.StillRunningAfter = ts tr.Processes.Default.Streams.All += Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") -tr.Processes.Default.Streams.All += Testers.ExcludesExpression("Not Found on Accelerato", "Should not try to remap on Traffic Server") +tr.Processes.Default.Streams.All += Testers.ExcludesExpression( + "Not Found on Accelerato", "Should not try to remap on Traffic Server") tr.Processes.Default.Streams.All += Testers.ExcludesExpression("CN=foo.com", "Should not TLS terminate on Traffic Server") tr.Processes.Default.Streams.All += Testers.ContainsExpression("HTTP/1.1 200 OK", "Should get a successful response") tr.Processes.Default.Streams.All += Testers.ExcludesExpression("ATS", "Do not terminate on Traffic Server") tr.Processes.Default.Streams.All += Testers.ContainsExpression("foo ok", "Should get a response from bar") tr = Test.AddTestRun("bob.bar.com Tunnel-test") -tr.Processes.Default.Command = "curl -v --resolve 'bob.bar.com:{0}:127.0.0.1' -k https://bob.bar.com:{0}".format(ts.Variables.ssl_port) +tr.Processes.Default.Command = "curl -v --resolve 'bob.bar.com:{0}:127.0.0.1' -k https://bob.bar.com:{0}".format( + ts.Variables.ssl_port) tr.ReturnCode = 0 tr.StillRunningAfter = ts tr.Processes.Default.Streams.All += Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") -tr.Processes.Default.Streams.All += Testers.ExcludesExpression("Not Found on Accelerato", "Should not try to remap on Traffic Server") +tr.Processes.Default.Streams.All += Testers.ExcludesExpression( + "Not Found on Accelerato", "Should not try to remap on Traffic Server") tr.Processes.Default.Streams.All += Testers.ExcludesExpression("CN=foo.com", "Should not TLS terminate on Traffic Server") tr.Processes.Default.Streams.All += Testers.ContainsExpression("HTTP/1.1 200 OK", "Should get a successful response") tr.Processes.Default.Streams.All += Testers.ExcludesExpression("ATS", "Do not terminate on Traffic Server") @@ -116,21 +129,51 @@ tr.ReturnCode = 0 tr.StillRunningAfter = ts tr.Processes.Default.Streams.All += Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") -tr.Processes.Default.Streams.All += Testers.ExcludesExpression("Not Found on Accelerato", "Should not try to remap on Traffic Server") +tr.Processes.Default.Streams.All += Testers.ExcludesExpression( + "Not Found on Accelerato", "Should not try to remap on Traffic Server") tr.Processes.Default.Streams.All += Testers.ContainsExpression("HTTP/1.1 200 OK", "Should get a successful response") tr.Processes.Default.Streams.All += Testers.ExcludesExpression("ATS", "Do not terminate on Traffic Server") tr.Processes.Default.Streams.All += Testers.ContainsExpression("bar ok", "Should get a response from bar") + +tr = Test.AddTestRun("one.match.com Tunnel-test") +tr.Processes.Default.Command = "curl -vvv --resolve 'one.match.com:{0}:127.0.0.1' -k https://one.match.com:{0}".format( + ts.Variables.ssl_port) +tr.ReturnCode = 0 +tr.StillRunningAfter = ts +tr.Processes.Default.Streams.All += Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") +tr.Processes.Default.Streams.All += Testers.ExcludesExpression( + "Not Found on Accelerato", "Should not try to remap on Traffic Server") +tr.Processes.Default.Streams.All += Testers.ExcludesExpression("CN=foo.com", "Should not TLS terminate on Traffic Server") +tr.Processes.Default.Streams.All += Testers.ContainsExpression("HTTP/1.1 200 OK", "Should get a successful response") +tr.Processes.Default.Streams.All += Testers.ExcludesExpression("ATS", "Do not terminate on Traffic Server") +tr.Processes.Default.Streams.All += Testers.ContainsExpression("foo ok", "Should get a response from tm") + + +tr = Test.AddTestRun("one.ok.two.com Tunnel-test") +tr.Processes.Default.Command = "curl -vvv --resolve 'one.ok.two.com:{0}:127.0.0.1' -k https:/one.ok.two.com:{0}".format( + ts.Variables.ssl_port) +tr.ReturnCode = 0 +tr.StillRunningAfter = ts +tr.Processes.Default.Streams.All += Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") +tr.Processes.Default.Streams.All += Testers.ExcludesExpression( + "Not Found on Accelerato", "Should not try to remap on Traffic Server") +tr.Processes.Default.Streams.All += Testers.ExcludesExpression("CN=foo.com", "Should not TLS terminate on Traffic Server") +tr.Processes.Default.Streams.All += Testers.ContainsExpression("HTTP/1.1 200 OK", "Should get a successful response") +tr.Processes.Default.Streams.All += Testers.ExcludesExpression("ATS", "Do not terminate on Traffic Server") +tr.Processes.Default.Streams.All += Testers.ContainsExpression("foo ok", "Should get a response from tm") + + # Update sni file and reload tr = Test.AddTestRun("Update config files") # Update the SNI config snipath = ts.Disk.sni_yaml.AbsPath recordspath = ts.Disk.records_config.AbsPath -tr.Disk.File(snipath, id = "sni_yaml", typename="ats:config"), +tr.Disk.File(snipath, id="sni_yaml", typename="ats:config"), tr.Disk.sni_yaml.AddLines([ - 'sni:', - '- fqdn: bar.com', - ' tunnel_route: localhost:{0}'.format(server_bar.Variables.SSL_Port), + 'sni:', + '- fqdn: bar.com', + ' tunnel_route: localhost:{0}'.format(server_bar.Variables.SSL_Port), ]) tr.StillRunningAfter = ts tr.StillRunningAfter = server_foo @@ -148,25 +191,11 @@ trreload.Processes.Default.Env = ts.Env trreload.Processes.Default.ReturnCode = 0 -# Parking this as a ready tester on a meaningless process -# Stall the test runs until the sni reload has completed -# At that point the new sni settings are ready to go -def sni_reload_done(tsenv): - def done_reload(process, hasRunFor, **kw): - cmd = "grep 'sni.yaml finished loading' {0} | wc -l > {1}/test.out".format(ts.Disk.diags_log.Name, Test.RunDirectory) - retval = subprocess.run(cmd, shell=True, env=tsenv) - if retval.returncode == 0: - cmd ="if [ -f {0}/test.out -a \"`cat {0}/test.out`\" = \"2\" ] ; then true; else false; fi".format(Test.RunDirectory) - retval = subprocess.run(cmd, shell = True, env=tsenv) - return retval.returncode == 0 - - return done_reload - # Should termimate on traffic_server (not tunnel) tr = Test.AddTestRun("foo.com no Tunnel-test") tr.StillRunningAfter = ts # Wait for the reload to complete by running the sni_reload_done test -tr.Processes.Default.StartBefore(server2, ready=sni_reload_done(ts.Env)) +tr.Processes.Default.StartBefore(server2, ready=When.FileContains(ts.Disk.diags_log.Name, 'sni.yaml finished loading', 2)) tr.Processes.Default.Command = "curl -v --resolve 'foo.com:{0}:127.0.0.1' -k https://foo.com:{0}".format(ts.Variables.ssl_port) tr.Processes.Default.Streams.All += Testers.ContainsExpression("Not Found on Accelerato", "Terminates on on Traffic Server") tr.Processes.Default.Streams.All += Testers.ContainsExpression("ATS", "Terminate on Traffic Server") diff --git a/tests/gold_tests/tls/tls_tunnel_forward.test.py b/tests/gold_tests/tls/tls_tunnel_forward.test.py index 105babfef54..c604c6a46a1 100644 --- a/tests/gold_tests/tls/tls_tunnel_forward.test.py +++ b/tests/gold_tests/tls/tls_tunnel_forward.test.py @@ -16,7 +16,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import os Test.Summary = ''' Test tunneling and forwarding based on SNI ''' @@ -32,7 +31,8 @@ request_random_header = {"headers": "GET / HTTP/1.1\r\nHost: random.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} response_foo_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": "ok foo"} response_bar_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": "ok bar"} -response_random_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": "ok random"} +response_random_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", + "timestamp": "1469733493.993", "body": "ok random"} server_foo.addResponse("sessionlog_foo.json", request_foo_header, response_foo_header) server_bar.addResponse("sessionlog_bar.json", request_bar_header, response_bar_header) server_random.addResponse("sessionlog_random.json", request_random_header, response_random_header) @@ -70,14 +70,14 @@ # foo.com should not terminate. Just tunnel to server_foo # bar.com should terminate. Forward its tcp stream to server_bar ts.Disk.sni_yaml.AddLines([ - "sni:", - "- fqdn: 'foo.com'", - " tunnel_route: 'localhost:{0}'".format(server_foo.Variables.SSL_Port), - "- fqdn: 'bar.com'", - " forward_route: 'localhost:{0}'".format(server_bar.Variables.Port), - "- fqdn: ''", #default case - " forward_route: 'localhost:{0}'".format(server_random.Variables.Port), - ]) + "sni:", + "- fqdn: 'foo.com'", + " tunnel_route: 'localhost:{0}'".format(server_foo.Variables.SSL_Port), + "- fqdn: 'bar.com'", + " forward_route: 'localhost:{0}'".format(server_bar.Variables.Port), + "- fqdn: ''", # default case + " forward_route: 'localhost:{0}'".format(server_random.Variables.Port), +]) tr = Test.AddTestRun("Tunnel-test") tr.Processes.Default.Command = "curl -v --resolve 'foo.com:{0}:127.0.0.1' -k https://foo.com:{0}".format(ts.Variables.ssl_port) @@ -88,18 +88,21 @@ tr.Processes.Default.StartBefore(Test.Processes.ts) tr.StillRunningAfter = ts tr.Processes.Default.Streams.All += Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") -tr.Processes.Default.Streams.All += Testers.ExcludesExpression("Not Found on Accelerato", "Should not try to remap on Traffic Server") +tr.Processes.Default.Streams.All += Testers.ExcludesExpression( + "Not Found on Accelerato", "Should not try to remap on Traffic Server") tr.Processes.Default.Streams.All += Testers.ExcludesExpression("CN=foo.com", "Should not TLS terminate on Traffic Server") tr.Processes.Default.Streams.All += Testers.ContainsExpression("HTTP/1.1 200 OK", "Should get a successful response") tr.Processes.Default.Streams.All += Testers.ContainsExpression("ok foo", "Body is expected") tr2 = Test.AddTestRun("Forward-test") -tr2.Processes.Default.Command = "curl -v --http1.1 -H 'host:bar.com' --resolve 'bar.com:{0}:127.0.0.1' -k https://bar.com:{0}".format(ts.Variables.ssl_port) +tr2.Processes.Default.Command = "curl -v --http1.1 -H 'host:bar.com' --resolve 'bar.com:{0}:127.0.0.1' -k https://bar.com:{0}".format( + ts.Variables.ssl_port) tr2.ReturnCode = 0 tr2.StillRunningAfter = server_bar tr2.StillRunningAfter = ts tr2.Processes.Default.Streams.All += Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") -tr2.Processes.Default.Streams.All += Testers.ExcludesExpression("Not Found on Accelerato", "Should not try to remap on Traffic Server") +tr2.Processes.Default.Streams.All += Testers.ExcludesExpression( + "Not Found on Accelerato", "Should not try to remap on Traffic Server") tr2.Processes.Default.Streams.All += Testers.ContainsExpression("CN=foo.com", "Should TLS terminate on Traffic Server") tr2.Processes.Default.Streams.All += Testers.ContainsExpression("HTTP/1.1 200 OK", "Should get a successful response") tr2.Processes.Default.Streams.All += Testers.ContainsExpression("ok bar", "Body is expected") @@ -110,7 +113,8 @@ tr3.StillRunningAfter = server_random tr3.StillRunningAfter = ts tr3.Processes.Default.Streams.All += Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") -tr3.Processes.Default.Streams.All += Testers.ExcludesExpression("Not Found on Accelerato", "Should not try to remap on Traffic Server") +tr3.Processes.Default.Streams.All += Testers.ExcludesExpression( + "Not Found on Accelerato", "Should not try to remap on Traffic Server") tr3.Processes.Default.Streams.All += Testers.ContainsExpression("CN=foo.com", "Should TLS terminate on Traffic Server") tr3.Processes.Default.Streams.All += Testers.ContainsExpression("HTTP/1.1 200 OK", "Should get a successful response") tr3.Processes.Default.Streams.All += Testers.ContainsExpression("ok random", "Body is expected") diff --git a/tests/gold_tests/tls/tls_verify.test.py b/tests/gold_tests/tls/tls_verify.test.py index b8ac58f52a1..2c961b477df 100644 --- a/tests/gold_tests/tls/tls_verify.test.py +++ b/tests/gold_tests/tls/tls_verify.test.py @@ -16,16 +16,24 @@ # See the License for the specific language governing permissions and # limitations under the License. -import os Test.Summary = ''' Test tls server certificate verification options ''' # Define default ATS ts = Test.MakeATSProcess("ts", select_ports=True, enable_tls=True) -server_foo = Test.MakeOriginServer("server_foo", ssl=True, options = {"--key": "{0}/signed-foo.key".format(Test.RunDirectory), "--cert": "{0}/signed-foo.pem".format(Test.RunDirectory)}) -server_bar = Test.MakeOriginServer("server_bar", ssl=True, options = {"--key": "{0}/signed-bar.key".format(Test.RunDirectory), "--cert": "{0}/signed-bar.pem".format(Test.RunDirectory)}) -server_wild = Test.MakeOriginServer("server_wild", ssl=True, options = {"--key": "{0}/wild.key".format(Test.RunDirectory), "--cert": "{0}/wild-signed.pem".format(Test.RunDirectory)}) +server_foo = Test.MakeOriginServer("server_foo", + ssl=True, + options={"--key": "{0}/signed-foo.key".format(Test.RunDirectory), + "--cert": "{0}/signed-foo.pem".format(Test.RunDirectory)}) +server_bar = Test.MakeOriginServer("server_bar", + ssl=True, + options={"--key": "{0}/signed-bar.key".format(Test.RunDirectory), + "--cert": "{0}/signed-bar.pem".format(Test.RunDirectory)}) +server_wild = Test.MakeOriginServer("server_wild", + ssl=True, + options={"--key": "{0}/wild.key".format(Test.RunDirectory), + "--cert": "{0}/wild-signed.pem".format(Test.RunDirectory)}) server = Test.MakeOriginServer("server", ssl=True) request_foo_header = {"headers": "GET / HTTP/1.1\r\nHost: foo.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} @@ -86,16 +94,16 @@ }) ts.Disk.sni_yaml.AddLines([ - 'sni:', - '- fqdn: bar.com', - ' verify_server_policy: ENFORCED', - ' verify_server_properties: ALL', - '- fqdn: "*.wild.com"', - ' verify_server_policy: ENFORCED', - ' verify_server_properties: ALL', - '- fqdn: bad_bar.com', - ' verify_server_policy: ENFORCED', - ' verify_server_properties: ALL' + 'sni:', + '- fqdn: bar.com', + ' verify_server_policy: ENFORCED', + ' verify_server_properties: ALL', + '- fqdn: "*.wild.com"', + ' verify_server_policy: ENFORCED', + ' verify_server_properties: ALL', + '- fqdn: bad_bar.com', + ' verify_server_policy: ENFORCED', + ' verify_server_properties: ALL' ]) tr = Test.AddTestRun("Permissive-Test") @@ -147,4 +155,5 @@ # Over riding the built in ERROR check since we expect tr3 to fail ts.Disk.diags_log.Content = Testers.ExcludesExpression("verification failed", "Make sure the signatures didn't fail") -ts.Disk.diags_log.Content += Testers.ContainsExpression("WARNING: SNI \(bad_bar.com\) not in certificate", "Make sure bad_bar name checked failed.") +ts.Disk.diags_log.Content += Testers.ContainsExpression( + "WARNING: SNI \(bad_bar.com\) not in certificate", "Make sure bad_bar name checked failed.") diff --git a/tests/gold_tests/tls/tls_verify2.test.py b/tests/gold_tests/tls/tls_verify2.test.py index e141fd44d06..b218242afed 100644 --- a/tests/gold_tests/tls/tls_verify2.test.py +++ b/tests/gold_tests/tls/tls_verify2.test.py @@ -16,15 +16,20 @@ # See the License for the specific language governing permissions and # limitations under the License. -import os Test.Summary = ''' Test tls server certificate verification options ''' # Define default ATS ts = Test.MakeATSProcess("ts", select_ports=True, enable_tls=True) -server_foo = Test.MakeOriginServer("server_foo", ssl=True, options = {"--key": "{0}/signed-foo.key".format(Test.RunDirectory), "--cert": "{0}/signed-foo.pem".format(Test.RunDirectory)}) -server_bar = Test.MakeOriginServer("server_bar", ssl=True, options = {"--key": "{0}/signed-bar.key".format(Test.RunDirectory), "--cert": "{0}/signed-bar.pem".format(Test.RunDirectory)}) +server_foo = Test.MakeOriginServer("server_foo", + ssl=True, + options={"--key": "{0}/signed-foo.key".format(Test.RunDirectory), + "--cert": "{0}/signed-foo.pem".format(Test.RunDirectory)}) +server_bar = Test.MakeOriginServer("server_bar", + ssl=True, + options={"--key": "{0}/signed-bar.key".format(Test.RunDirectory), + "--cert": "{0}/signed-bar.pem".format(Test.RunDirectory)}) server = Test.MakeOriginServer("server", ssl=True) request_foo_header = {"headers": "GET / HTTP/1.1\r\nHost: foo.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} @@ -78,23 +83,23 @@ }) ts.Disk.sni_yaml.AddLine( - 'sni:') + 'sni:') ts.Disk.sni_yaml.AddLine( - '- fqdn: bar.com') + '- fqdn: bar.com') ts.Disk.sni_yaml.AddLine( - ' verify_server_policy: PERMISSIVE') + ' verify_server_policy: PERMISSIVE') ts.Disk.sni_yaml.AddLine( - ' verify_server_properties: SIGNATURE') + ' verify_server_properties: SIGNATURE') ts.Disk.sni_yaml.AddLine( - '- fqdn: bad_bar.com') + '- fqdn: bad_bar.com') ts.Disk.sni_yaml.AddLine( - ' verify_server_policy: PERMISSIVE') + ' verify_server_policy: PERMISSIVE') ts.Disk.sni_yaml.AddLine( - ' verify_server_properties: SIGNATURE') + ' verify_server_properties: SIGNATURE') ts.Disk.sni_yaml.AddLine( - '- fqdn: random.com') + '- fqdn: random.com') ts.Disk.sni_yaml.AddLine( - ' verify_server_policy: DISABLED') + ' verify_server_policy: DISABLED') tr = Test.AddTestRun("default-enforce") tr.Setup.Copy("ssl/signed-foo.key") @@ -148,9 +153,14 @@ # No name checking for the sig-only permissive override for bad_bar -ts.Disk.diags_log.Content += Testers.ExcludesExpression("WARNING: SNI \(bad_bar.com\) not in certificate", "bad_bar name checked should be skipped.") -ts.Disk.diags_log.Content = Testers.ExcludesExpression("WARNING: SNI \(foo.com\) not in certificate", "foo name checked should be skipped.") +ts.Disk.diags_log.Content += Testers.ExcludesExpression( + "WARNING: SNI \(bad_bar.com\) not in certificate", "bad_bar name checked should be skipped.") +ts.Disk.diags_log.Content = Testers.ExcludesExpression( + "WARNING: SNI \(foo.com\) not in certificate", "foo name checked should be skipped.") # No checking for the self-signed on random.com. No messages -ts.Disk.diags_log.Content += Testers.ExcludesExpression("WARNING: Core server certificate verification failed for \(random.com\)", "signature check for random.com should be skipped") -ts.Disk.diags_log.Content += Testers.ContainsExpression("WARNING: Core server certificate verification failed for \(random2.com\)", "signature check for random.com should fail'") -ts.Disk.diags_log.Content += Testers.ContainsExpression("WARNING: SNI \(bad_foo.com\) not in certificate", "bad_foo name checked should be checked.") +ts.Disk.diags_log.Content += Testers.ExcludesExpression( + "WARNING: Core server certificate verification failed for \(random.com\)", "signature check for random.com should be skipped") +ts.Disk.diags_log.Content += Testers.ContainsExpression( + "WARNING: Core server certificate verification failed for \(random2.com\)", "signature check for random.com should fail'") +ts.Disk.diags_log.Content += Testers.ContainsExpression( + "WARNING: SNI \(bad_foo.com\) not in certificate", "bad_foo name checked should be checked.") diff --git a/tests/gold_tests/tls/tls_verify3.test.py b/tests/gold_tests/tls/tls_verify3.test.py index 25d806917b3..1098870bc12 100644 --- a/tests/gold_tests/tls/tls_verify3.test.py +++ b/tests/gold_tests/tls/tls_verify3.test.py @@ -16,15 +16,20 @@ # See the License for the specific language governing permissions and # limitations under the License. -import os Test.Summary = ''' Test tls server certificate verification options ''' # Define default ATS ts = Test.MakeATSProcess("ts", select_ports=True, enable_tls=True) -server_foo = Test.MakeOriginServer("server_foo", ssl=True, options = {"--key": "{0}/signed-foo.key".format(Test.RunDirectory), "--cert": "{0}/signed-foo.pem".format(Test.RunDirectory)}) -server_bar = Test.MakeOriginServer("server_bar", ssl=True, options = {"--key": "{0}/signed-bar.key".format(Test.RunDirectory), "--cert": "{0}/signed-bar.pem".format(Test.RunDirectory)}) +server_foo = Test.MakeOriginServer("server_foo", + ssl=True, + options={"--key": "{0}/signed-foo.key".format(Test.RunDirectory), + "--cert": "{0}/signed-foo.pem".format(Test.RunDirectory)}) +server_bar = Test.MakeOriginServer("server_bar", + ssl=True, + options={"--key": "{0}/signed-bar.key".format(Test.RunDirectory), + "--cert": "{0}/signed-bar.pem".format(Test.RunDirectory)}) server = Test.MakeOriginServer("server", ssl=True) request_foo_header = {"headers": "GET / HTTP/1.1\r\nHost: foo.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} @@ -54,7 +59,7 @@ ts.Disk.remap_config.AddLine( 'map https://bar.com:{1}/ https://127.0.0.1:{0}'.format(server_bar.Variables.SSL_Port, ts.Variables.ssl_port)) ts.Disk.remap_config.AddLine( - 'map https://bob.bar.com:{1}/ https://127.0.0.1:{0}'.format(server_bar.Variables.SSL_Port,ts.Variables.ssl_port)) + 'map https://bob.bar.com:{1}/ https://127.0.0.1:{0}'.format(server_bar.Variables.SSL_Port, ts.Variables.ssl_port)) ts.Disk.remap_config.AddLine( 'map / https://127.0.0.1:{0}'.format(server.Variables.SSL_Port)) @@ -78,15 +83,15 @@ }) ts.Disk.sni_yaml.AddLines([ - 'sni:', - '- fqdn: bob.bar.com', - ' verify_server_policy: ENFORCED', - ' verify_server_properties: ALL', - '- fqdn: bob.*.com', - ' verify_server_policy: ENFORCED', - ' verify_server_properties: SIGNATURE', - "- fqdn: '*bar.com'", - ' verify_server_policy: DISABLED', + 'sni:', + '- fqdn: bob.bar.com', + ' verify_server_policy: ENFORCED', + ' verify_server_properties: ALL', + '- fqdn: bob.*.com', + ' verify_server_policy: ENFORCED', + ' verify_server_properties: SIGNATURE', + "- fqdn: '*bar.com'", + ' verify_server_policy: DISABLED', ]) tr = Test.AddTestRun("foo.com Permissive-Test") @@ -105,7 +110,8 @@ tr.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") tr = Test.AddTestRun("my.foo.com Permissive-Test log failure") -tr.Processes.Default.Command = "curl -v -k --resolve 'my.foo.com:{0}:127.0.0.1' https://my.foo.com:{0}".format(ts.Variables.ssl_port) +tr.Processes.Default.Command = "curl -v -k --resolve 'my.foo.com:{0}:127.0.0.1' https://my.foo.com:{0}".format( + ts.Variables.ssl_port) tr.ReturnCode = 0 tr.StillRunningAfter = server tr.StillRunningAfter = ts @@ -113,21 +119,24 @@ tr2 = Test.AddTestRun("bob.bar.com Override-enforcing-Test") -tr2.Processes.Default.Command = "curl -v -k --resolve 'bob.bar.com:{0}:127.0.0.1' https://bob.bar.com:{0}/".format(ts.Variables.ssl_port) +tr2.Processes.Default.Command = "curl -v -k --resolve 'bob.bar.com:{0}:127.0.0.1' https://bob.bar.com:{0}/".format( + ts.Variables.ssl_port) tr2.ReturnCode = 0 tr2.StillRunningAfter = server tr2.StillRunningAfter = ts tr2.Processes.Default.Streams.stdout = Testers.ContainsExpression("Could Not Connect", "Curl attempt should have succeeded") tr3 = Test.AddTestRun("bob.foo.com override-enforcing-name-test") -tr3.Processes.Default.Command = "curl -v -k --resolve 'bob.foo.com:{0}:127.0.0.1' https://bob.foo.com:{0}/".format(ts.Variables.ssl_port) +tr3.Processes.Default.Command = "curl -v -k --resolve 'bob.foo.com:{0}:127.0.0.1' https://bob.foo.com:{0}/".format( + ts.Variables.ssl_port) tr3.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could Not Connect", "Curl attempt should not fail") tr3.ReturnCode = 0 tr3.StillRunningAfter = server tr3.StillRunningAfter = ts tr3 = Test.AddTestRun("random.bar.com override-no-test") -tr3.Processes.Default.Command = "curl -v -k --resolve 'random.bar.com:{0}:127.0.0.1' https://random.bar.com:{0}".format(ts.Variables.ssl_port) +tr3.Processes.Default.Command = "curl -v -k --resolve 'random.bar.com:{0}:127.0.0.1' https://random.bar.com:{0}".format( + ts.Variables.ssl_port) tr3.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could Not Connect", "Curl attempt should not fail") tr3.ReturnCode = 0 tr3.StillRunningAfter = server @@ -135,5 +144,8 @@ # Over riding the built in ERROR check since we expect tr3 to fail -ts.Disk.diags_log.Content = Testers.ContainsExpression("WARNING: SNI \(bob.bar.com\) not in certificate", "Make sure bob.bar name checked failed.") -ts.Disk.diags_log.Content += Testers.ContainsExpression("WARNING: Core server certificate verification failed for \(my.foo.com\). Action=Continue", "Make sure default permissive action takes") +ts.Disk.diags_log.Content = Testers.ContainsExpression( + "WARNING: SNI \(bob.bar.com\) not in certificate", "Make sure bob.bar name checked failed.") +ts.Disk.diags_log.Content += Testers.ContainsExpression( + r"WARNING: Core server certificate verification failed for \(my.foo.com\). Action=Continue", + "Make sure default permissive action takes") diff --git a/tests/gold_tests/tls/tls_verify_base.test.py b/tests/gold_tests/tls/tls_verify_base.test.py index 7195f868f82..b1ee8529509 100644 --- a/tests/gold_tests/tls/tls_verify_base.test.py +++ b/tests/gold_tests/tls/tls_verify_base.test.py @@ -16,15 +16,20 @@ # See the License for the specific language governing permissions and # limitations under the License. -import os Test.Summary = ''' Test tls server certificate verification options ''' # Define default ATS ts = Test.MakeATSProcess("ts", select_ports=True, enable_tls=True) -server_foo = Test.MakeOriginServer("server_foo", ssl=True, options = {"--key": "{0}/signed-foo.key".format(Test.RunDirectory), "--cert": "{0}/signed-foo.pem".format(Test.RunDirectory)}) -server_bar = Test.MakeOriginServer("server_bar", ssl=True, options = {"--key": "{0}/signed-bar.key".format(Test.RunDirectory), "--cert": "{0}/signed-bar.pem".format(Test.RunDirectory)}) +server_foo = Test.MakeOriginServer("server_foo", + ssl=True, + options={"--key": "{0}/signed-foo.key".format(Test.RunDirectory), + "--cert": "{0}/signed-foo.pem".format(Test.RunDirectory)}) +server_bar = Test.MakeOriginServer("server_bar", + ssl=True, + options={"--key": "{0}/signed-bar.key".format(Test.RunDirectory), + "--cert": "{0}/signed-bar.pem".format(Test.RunDirectory)}) server = Test.MakeOriginServer("server", ssl=True) request_foo_header = {"headers": "GET / HTTP/1.1\r\nHost: foo.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} @@ -68,8 +73,6 @@ 'proxy.config.ssl.server.cert.path': '{0}'.format(ts.Variables.SSLDir), 'proxy.config.ssl.server.private_key.path': '{0}'.format(ts.Variables.SSLDir), 'proxy.config.ssl.server.cipher_suite': 'ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA384:AES128-GCM-SHA256:AES256-GCM-SHA384:ECDHE-RSA-RC4-SHA:ECDHE-RSA-AES128-SHA:ECDHE-RSA-AES256-SHA:RC4-SHA:RC4-MD5:AES128-SHA:AES256-SHA:DES-CBC3-SHA!SRP:!DSS:!PSK:!aNULL:!eNULL:!SSLv2', - # set global policy - 'proxy.config.ssl.client.verify.server': 2, 'proxy.config.ssl.client.CA.cert.path': '{0}'.format(ts.Variables.SSLDir), 'proxy.config.ssl.client.CA.cert.filename': 'signer.pem', 'proxy.config.url_remap.pristine_host_hdr': 1, @@ -78,13 +81,13 @@ }) ts.Disk.sni_yaml.AddLines([ - 'sni:', - '- fqdn: bar.com', - ' verify_server_policy: ENFORCED', - ' verify_server_properties: ALL', - '- fqdn: bad_bar.com', - ' verify_server_policy: ENFORCED', - ' verify_server_properties: ALL' + 'sni:', + '- fqdn: bar.com', + ' verify_server_policy: ENFORCED', + ' verify_server_properties: ALL', + '- fqdn: bad_bar.com', + ' verify_server_policy: ENFORCED', + ' verify_server_properties: ALL' ]) tr = Test.AddTestRun("Permissive-Test") @@ -125,5 +128,7 @@ tr3.StillRunningAfter = ts # Over riding the built in ERROR check since we expect tr3 to fail -ts.Disk.diags_log.Content = Testers.ContainsExpression("WARNING: SNI \(bad_bar.com\) not in certificate. Action=Terminate", "Make sure bad_bar name checked failed.") -ts.Disk.diags_log.Content += Testers.ContainsExpression("WARNING: SNI \(random.com\) not in certificate. Action=Continue ", "Permissive failure for random") +ts.Disk.diags_log.Content = Testers.ContainsExpression( + "WARNING: SNI \(bad_bar.com\) not in certificate. Action=Terminate", "Make sure bad_bar name checked failed.") +ts.Disk.diags_log.Content += Testers.ContainsExpression( + "WARNING: SNI \(random.com\) not in certificate. Action=Continue ", "Permissive failure for random") diff --git a/tests/gold_tests/tls/tls_verify_ca_override.test.py b/tests/gold_tests/tls/tls_verify_ca_override.test.py index 6197de7645c..f737a379bf0 100644 --- a/tests/gold_tests/tls/tls_verify_ca_override.test.py +++ b/tests/gold_tests/tls/tls_verify_ca_override.test.py @@ -16,15 +16,20 @@ # See the License for the specific language governing permissions and # limitations under the License. -import os Test.Summary = ''' Test tls server certificate verification options. Exercise conf_remap for ca bundle ''' # Define default ATS ts = Test.MakeATSProcess("ts", select_ports=True) -server1 = Test.MakeOriginServer("server1", ssl=True, options = {"--key": "{0}/signed-foo.key".format(Test.RunDirectory), "--cert": "{0}/signed-foo.pem".format(Test.RunDirectory)}) -server2 = Test.MakeOriginServer("server2", ssl=True, options = {"--key": "{0}/signed-foo.key".format(Test.RunDirectory), "--cert": "{0}/signed2-foo.pem".format(Test.RunDirectory)}) +server1 = Test.MakeOriginServer("server1", + ssl=True, + options={"--key": "{0}/signed-foo.key".format(Test.RunDirectory), + "--cert": "{0}/signed-foo.pem".format(Test.RunDirectory)}) +server2 = Test.MakeOriginServer("server2", + ssl=True, + options={"--key": "{0}/signed-foo.key".format(Test.RunDirectory), + "--cert": "{0}/signed2-foo.pem".format(Test.RunDirectory)}) request_foo_header = {"headers": "GET / HTTP/1.1\r\nHost: foo.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} request_bad_foo_header = {"headers": "GET / HTTP/1.1\r\nHost: bad_foo.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} @@ -48,16 +53,20 @@ ts.addSSLfile("ssl/signer2.key") ts.Disk.remap_config.AddLine( - 'map /case1 https://127.0.0.1:{0}/ @plugin=conf_remap.so @pparam=proxy.config.ssl.client.CA.cert.filename={1}/{2}'.format(server1.Variables.SSL_Port, ts.Variables.SSLDir, "signer.pem") + 'map /case1 https://127.0.0.1:{0}/ @plugin=conf_remap.so @pparam=proxy.config.ssl.client.CA.cert.filename={1}/{2}'.format( + server1.Variables.SSL_Port, ts.Variables.SSLDir, "signer.pem") ) ts.Disk.remap_config.AddLine( - 'map /badcase1 https://127.0.0.1:{0}/ @plugin=conf_remap.so @pparam=proxy.config.ssl.client.CA.cert.filename={1}/{2}'.format(server1.Variables.SSL_Port, ts.Variables.SSLDir, "signer2.pem") + 'map /badcase1 https://127.0.0.1:{0}/ @plugin=conf_remap.so @pparam=proxy.config.ssl.client.CA.cert.filename={1}/{2}'.format( + server1.Variables.SSL_Port, ts.Variables.SSLDir, "signer2.pem") ) ts.Disk.remap_config.AddLine( - 'map /case2 https://127.0.0.1:{0}/ @plugin=conf_remap.so @pparam=proxy.config.ssl.client.CA.cert.filename={1}/{2}'.format(server2.Variables.SSL_Port, ts.Variables.SSLDir, "signer2.pem") + 'map /case2 https://127.0.0.1:{0}/ @plugin=conf_remap.so @pparam=proxy.config.ssl.client.CA.cert.filename={1}/{2}'.format( + server2.Variables.SSL_Port, ts.Variables.SSLDir, "signer2.pem") ) ts.Disk.remap_config.AddLine( - 'map /badcase2 https://127.0.0.1:{0}/ @plugin=conf_remap.so @pparam=proxy.config.ssl.client.CA.cert.filename={1}/{2}'.format(server2.Variables.SSL_Port, ts.Variables.SSLDir, "signer.pem") + 'map /badcase2 https://127.0.0.1:{0}/ @plugin=conf_remap.so @pparam=proxy.config.ssl.client.CA.cert.filename={1}/{2}'.format( + server2.Variables.SSL_Port, ts.Variables.SSLDir, "signer.pem") ) ts.Disk.ssl_multicert_config.AddLine( diff --git a/tests/gold_tests/tls/tls_verify_not_pristine.test.py b/tests/gold_tests/tls/tls_verify_not_pristine.test.py index b04cd68a877..d5744519ce1 100644 --- a/tests/gold_tests/tls/tls_verify_not_pristine.test.py +++ b/tests/gold_tests/tls/tls_verify_not_pristine.test.py @@ -16,14 +16,16 @@ # See the License for the specific language governing permissions and # limitations under the License. -import os Test.Summary = ''' Test tls server certificate verification without a pristine host header ''' # Define default ATS ts = Test.MakeATSProcess("ts", select_ports=True, enable_tls=True) -server_foo = Test.MakeOriginServer("server_foo", ssl=True, options = {"--key": "{0}/signed-foo.key".format(Test.RunDirectory), "--cert": "{0}/signed-foo.pem".format(Test.RunDirectory)}) +server_foo = Test.MakeOriginServer("server_foo", + ssl=True, + options={"--key": "{0}/signed-foo.key".format(Test.RunDirectory), + "--cert": "{0}/signed-foo.pem".format(Test.RunDirectory)}) server = Test.MakeOriginServer("server", ssl=True) dns = Test.MakeDNServer("dns") @@ -98,4 +100,5 @@ # Over riding the built in ERROR check since we expect tr3 to fail ts.Disk.diags_log.Content = Testers.ExcludesExpression("verification failed", "Make sure the signatures didn't fail") -ts.Disk.diags_log.Content += Testers.ContainsExpression("WARNING: SNI \(bar.com\) not in certificate", "Make sure bad_bar name checked failed.") +ts.Disk.diags_log.Content += Testers.ContainsExpression("WARNING: SNI \(bar.com\) not in certificate", + "Make sure bad_bar name checked failed.") diff --git a/tests/gold_tests/tls/tls_verify_override.test.py b/tests/gold_tests/tls/tls_verify_override.test.py index 0e6a7c51cea..e4daffce8a5 100644 --- a/tests/gold_tests/tls/tls_verify_override.test.py +++ b/tests/gold_tests/tls/tls_verify_override.test.py @@ -16,15 +16,20 @@ # See the License for the specific language governing permissions and # limitations under the License. -import os Test.Summary = ''' Test tls server certificate verification options. Exercise conf_remap ''' # Define default ATS ts = Test.MakeATSProcess("ts", select_ports=True) -server_foo = Test.MakeOriginServer("server_foo", ssl=True, options = {"--key": "{0}/signed-foo.key".format(Test.RunDirectory), "--cert": "{0}/signed-foo.pem".format(Test.RunDirectory)}) -server_bar = Test.MakeOriginServer("server_bar", ssl=True, options = {"--key": "{0}/signed-bar.key".format(Test.RunDirectory), "--cert": "{0}/signed-bar.pem".format(Test.RunDirectory)}) +server_foo = Test.MakeOriginServer("server_foo", + ssl=True, + options={"--key": "{0}/signed-foo.key".format(Test.RunDirectory), + "--cert": "{0}/signed-foo.pem".format(Test.RunDirectory)}) +server_bar = Test.MakeOriginServer("server_bar", + ssl=True, + options={"--key": "{0}/signed-bar.key".format(Test.RunDirectory), + "--cert": "{0}/signed-bar.pem".format(Test.RunDirectory)}) server = Test.MakeOriginServer("server", ssl=True) dns = Test.MakeDNServer("dns") @@ -54,29 +59,39 @@ ts.Disk.remap_config.AddLine( 'map http://foo.com/basic https://foo.com:{0}'.format(server_foo.Variables.SSL_Port)) ts.Disk.remap_config.AddLine( - 'map http://foo.com/override https://foo.com:{0} @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.policy=ENFORCED'.format(server_foo.Variables.SSL_Port)) + 'map http://foo.com/override https://foo.com:{0} @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.policy=ENFORCED'.format( + server_foo.Variables.SSL_Port)) ts.Disk.remap_config.AddLine( 'map http://bar.com/basic https://bar.com:{0}'.format(server_foo.Variables.SSL_Port)) ts.Disk.remap_config.AddLine( - 'map http://bar.com/overridedisabled https://bar.com:{0} @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.policy=DISABLED'.format(server_foo.Variables.SSL_Port)) + 'map http://bar.com/overridedisabled https://bar.com:{0} @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.policy=DISABLED'.format( + server_foo.Variables.SSL_Port)) ts.Disk.remap_config.AddLine( - 'map http://bar.com/overridesignature https://bar.com:{0} @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.properties=SIGNATURE @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.policy=ENFORCED'.format(server_foo.Variables.SSL_Port)) + 'map http://bar.com/overridesignature https://bar.com:{0} @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.properties=SIGNATURE @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.policy=ENFORCED'.format( + server_foo.Variables.SSL_Port)) ts.Disk.remap_config.AddLine( - 'map http://bar.com/overrideenforced https://bar.com:{0} @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.policy=ENFORCED'.format(server_foo.Variables.SSL_Port)) + 'map http://bar.com/overrideenforced https://bar.com:{0} @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.policy=ENFORCED'.format( + server_foo.Variables.SSL_Port)) ts.Disk.remap_config.AddLine( 'map /basic https://random.com:{0}'.format(server.Variables.SSL_Port)) ts.Disk.remap_config.AddLine( - 'map /overrideenforce https://127.0.0.1:{0} @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.policy=ENFORCED'.format(server.Variables.SSL_Port)) + 'map /overrideenforce https://127.0.0.1:{0} @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.policy=ENFORCED'.format( + server.Variables.SSL_Port)) ts.Disk.remap_config.AddLine( - 'map /overridename https://127.0.0.1:{0} @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.properties=NAME'.format(server.Variables.SSL_Port)) + 'map /overridename https://127.0.0.1:{0} @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.properties=NAME'.format( + server.Variables.SSL_Port)) ts.Disk.remap_config.AddLine( - 'map /snipolicyfooremap https://foo.com:{0} @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.properties=NAME @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.policy=ENFORCED @plugin=conf_remap.so @pparam=proxy.config.ssl.client.sni_policy=remap'.format(server_bar.Variables.SSL_Port)) + 'map /snipolicyfooremap https://foo.com:{0} @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.properties=NAME @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.policy=ENFORCED @plugin=conf_remap.so @pparam=proxy.config.ssl.client.sni_policy=remap'.format( + server_bar.Variables.SSL_Port)) ts.Disk.remap_config.AddLine( - 'map /snipolicyfoohost https://foo.com:{0} @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.properties=NAME @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.policy=ENFORCED @plugin=conf_remap.so @pparam=proxy.config.ssl.client.sni_policy=host'.format(server_bar.Variables.SSL_Port)) + 'map /snipolicyfoohost https://foo.com:{0} @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.properties=NAME @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.policy=ENFORCED @plugin=conf_remap.so @pparam=proxy.config.ssl.client.sni_policy=host'.format( + server_bar.Variables.SSL_Port)) ts.Disk.remap_config.AddLine( - 'map /snipolicybarremap https://bar.com:{0} @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.properties=NAME @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.policy=ENFORCED @plugin=conf_remap.so @pparam=proxy.config.ssl.client.sni_policy=remap'.format(server_bar.Variables.SSL_Port)) + 'map /snipolicybarremap https://bar.com:{0} @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.properties=NAME @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.policy=ENFORCED @plugin=conf_remap.so @pparam=proxy.config.ssl.client.sni_policy=remap'.format( + server_bar.Variables.SSL_Port)) ts.Disk.remap_config.AddLine( - 'map /snipolicybarhost https://bar.com:{0} @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.properties=NAME @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.policy=ENFORCED @plugin=conf_remap.so @pparam=proxy.config.ssl.client.sni_policy=host'.format(server_bar.Variables.SSL_Port)) + 'map /snipolicybarhost https://bar.com:{0} @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.properties=NAME @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.policy=ENFORCED @plugin=conf_remap.so @pparam=proxy.config.ssl.client.sni_policy=host'.format( + server_bar.Variables.SSL_Port)) ts.Disk.ssl_multicert_config.AddLine( 'dest_ip=* ssl_cert_name=server.pem ssl_key_name=server.key' @@ -212,12 +227,21 @@ # Over riding the built in ERROR check since we expect some cases to fail # checks on random.com should fail with message only -ts.Disk.diags_log.Content = Testers.ContainsExpression("WARNING: Core server certificate verification failed for \(random.com\). Action=Continue Error=self signed certificate server=random.com\(127.0.0.1\) depth=0", "Warning for self signed certificate") +ts.Disk.diags_log.Content = Testers.ContainsExpression( + r"WARNING: Core server certificate verification failed for \(random.com\). Action=Continue Error=self signed certificate server=random.com\(127.0.0.1\) depth=0", + "Warning for self signed certificate") # permissive failure for bar.com -ts.Disk.diags_log.Content += Testers.ContainsExpression("WARNING: SNI \(bar.com\) not in certificate. Action=Continue server=bar.com\(127.0.0.1\)", "Warning on missing name for bar.com") +ts.Disk.diags_log.Content += Testers.ContainsExpression( + r"WARNING: SNI \(bar.com\) not in certificate. Action=Continue server=bar.com\(127.0.0.1\)", + "Warning on missing name for bar.com") # name check failure for random.com -ts.Disk.diags_log.Content += Testers.ContainsExpression("WARNING: SNI \(random.com\) not in certificate. Action=Continue server=random.com\(127.0.0.1\)", "Warning on missing name for randome.com") +ts.Disk.diags_log.Content += Testers.ContainsExpression( + r"WARNING: SNI \(random.com\) not in certificate. Action=Continue server=random.com\(127.0.0.1\)", + "Warning on missing name for randome.com") # name check failure for bar.com -ts.Disk.diags_log.Content += Testers.ContainsExpression("WARNING: SNI \(bar.com\) not in certificate. Action=Terminate server=bar.com\(127.0.0.1\)", "Failure on missing name for bar.com") +ts.Disk.diags_log.Content += Testers.ContainsExpression( + r"WARNING: SNI \(bar.com\) not in certificate. Action=Terminate server=bar.com\(127.0.0.1\)", + "Failure on missing name for bar.com") # See if the explicitly set default sni_policy of remap works. -ts.Disk.diags_log.Content += Testers.ExcludesExpression("WARNING: SNI \(foo.com\) not in certificate. Action=Continue", "Warning on missing name for foo.com") +ts.Disk.diags_log.Content += Testers.ExcludesExpression( + "WARNING: SNI \(foo.com\) not in certificate. Action=Continue", "Warning on missing name for foo.com") diff --git a/tests/gold_tests/tls/tls_verify_override_base.test.py b/tests/gold_tests/tls/tls_verify_override_base.test.py index b84a7cbc8d8..8650ad63525 100644 --- a/tests/gold_tests/tls/tls_verify_override_base.test.py +++ b/tests/gold_tests/tls/tls_verify_override_base.test.py @@ -16,15 +16,20 @@ # See the License for the specific language governing permissions and # limitations under the License. -import os Test.Summary = ''' Test tls server certificate verification options. Exercise conf_remap ''' # Define default ATS ts = Test.MakeATSProcess("ts", select_ports=True) -server_foo = Test.MakeOriginServer("server_foo", ssl=True, options = {"--key": "{0}/signed-foo.key".format(Test.RunDirectory), "--cert": "{0}/signed-foo.pem".format(Test.RunDirectory)}) -server_bar = Test.MakeOriginServer("server_bar", ssl=True, options = {"--key": "{0}/signed-bar.key".format(Test.RunDirectory), "--cert": "{0}/signed-bar.pem".format(Test.RunDirectory)}) +server_foo = Test.MakeOriginServer("server_foo", + ssl=True, + options={"--key": "{0}/signed-foo.key".format(Test.RunDirectory), + "--cert": "{0}/signed-foo.pem".format(Test.RunDirectory)}) +server_bar = Test.MakeOriginServer("server_bar", + ssl=True, + options={"--key": "{0}/signed-bar.key".format(Test.RunDirectory), + "--cert": "{0}/signed-bar.pem".format(Test.RunDirectory)}) server = Test.MakeOriginServer("server", ssl=True) dns = Test.MakeDNServer("dns") @@ -52,29 +57,39 @@ ts.Disk.remap_config.AddLine( 'map http://foo.com/basic https://foo.com:{0}'.format(server_foo.Variables.SSL_Port)) ts.Disk.remap_config.AddLine( - 'map http://foo.com/override https://foo.com:{0} @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.policy=ENFORCED'.format(server_foo.Variables.SSL_Port)) + 'map http://foo.com/override https://foo.com:{0} @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.policy=ENFORCED'.format( + server_foo.Variables.SSL_Port)) ts.Disk.remap_config.AddLine( 'map http://bar.com/basic https://bar.com:{0}'.format(server_foo.Variables.SSL_Port)) ts.Disk.remap_config.AddLine( - 'map http://bar.com/overridedisabled https://bar.com:{0} @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.policy=DISABLED'.format(server_foo.Variables.SSL_Port)) + 'map http://bar.com/overridedisabled https://bar.com:{0} @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.policy=DISABLED'.format( + server_foo.Variables.SSL_Port)) ts.Disk.remap_config.AddLine( - 'map http://bar.com/overridesignature https://bar.com:{0} @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.properties=SIGNATURE @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.policy=ENFORCED'.format(server_foo.Variables.SSL_Port)) + 'map http://bar.com/overridesignature https://bar.com:{0} @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.properties=SIGNATURE @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.policy=ENFORCED'.format( + server_foo.Variables.SSL_Port)) ts.Disk.remap_config.AddLine( - 'map http://bar.com/overrideenforced https://bar.com:{0} @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.policy=ENFORCED'.format(server_foo.Variables.SSL_Port)) + 'map http://bar.com/overrideenforced https://bar.com:{0} @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.policy=ENFORCED'.format( + server_foo.Variables.SSL_Port)) ts.Disk.remap_config.AddLine( 'map /basic https://127.0.0.1:{0}'.format(server.Variables.SSL_Port)) ts.Disk.remap_config.AddLine( - 'map /overrideenforce https://127.0.0.1:{0} @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.policy=ENFORCED'.format(server.Variables.SSL_Port)) + 'map /overrideenforce https://127.0.0.1:{0} @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.policy=ENFORCED'.format( + server.Variables.SSL_Port)) ts.Disk.remap_config.AddLine( - 'map /overridename https://127.0.0.1:{0} @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.properties=NAME'.format(server.Variables.SSL_Port)) + 'map /overridename https://127.0.0.1:{0} @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.properties=NAME'.format( + server.Variables.SSL_Port)) ts.Disk.remap_config.AddLine( - 'map /snipolicyfooremap https://foo.com:{0} @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.properties=NAME @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.policy=ENFORCED @plugin=conf_remap.so @pparam=proxy.config.ssl.client.sni_policy=remap'.format(server_bar.Variables.SSL_Port)) + 'map /snipolicyfooremap https://foo.com:{0} @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.properties=NAME @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.policy=ENFORCED @plugin=conf_remap.so @pparam=proxy.config.ssl.client.sni_policy=remap'.format( + server_bar.Variables.SSL_Port)) ts.Disk.remap_config.AddLine( - 'map /snipolicyfoohost https://foo.com:{0} @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.properties=NAME @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.policy=ENFORCED @plugin=conf_remap.so @pparam=proxy.config.ssl.client.sni_policy=host'.format(server_bar.Variables.SSL_Port)) + 'map /snipolicyfoohost https://foo.com:{0} @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.properties=NAME @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.policy=ENFORCED @plugin=conf_remap.so @pparam=proxy.config.ssl.client.sni_policy=host'.format( + server_bar.Variables.SSL_Port)) ts.Disk.remap_config.AddLine( - 'map /snipolicybarremap https://bar.com:{0} @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.properties=NAME @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.policy=ENFORCED @plugin=conf_remap.so @pparam=proxy.config.ssl.client.sni_policy=remap'.format(server_bar.Variables.SSL_Port)) + 'map /snipolicybarremap https://bar.com:{0} @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.properties=NAME @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.policy=ENFORCED @plugin=conf_remap.so @pparam=proxy.config.ssl.client.sni_policy=remap'.format( + server_bar.Variables.SSL_Port)) ts.Disk.remap_config.AddLine( - 'map /snipolicybarhost https://bar.com:{0} @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.properties=NAME @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.policy=ENFORCED @plugin=conf_remap.so @pparam=proxy.config.ssl.client.sni_policy=host'.format(server_bar.Variables.SSL_Port)) + 'map /snipolicybarhost https://bar.com:{0} @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.properties=NAME @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.policy=ENFORCED @plugin=conf_remap.so @pparam=proxy.config.ssl.client.sni_policy=host'.format( + server_bar.Variables.SSL_Port)) ts.Disk.ssl_multicert_config.AddLine( 'dest_ip=* ssl_cert_name=server.pem ssl_key_name=server.key' @@ -86,8 +101,6 @@ 'proxy.config.ssl.server.cert.path': '{0}'.format(ts.Variables.SSLDir), 'proxy.config.ssl.server.private_key.path': '{0}'.format(ts.Variables.SSLDir), 'proxy.config.ssl.server.cipher_suite': 'ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA384:AES128-GCM-SHA256:AES256-GCM-SHA384:ECDHE-RSA-RC4-SHA:ECDHE-RSA-AES128-SHA:ECDHE-RSA-AES256-SHA:RC4-SHA:RC4-MD5:AES128-SHA:AES256-SHA:DES-CBC3-SHA!SRP:!DSS:!PSK:!aNULL:!eNULL:!SSLv2', - # set global policy - 'proxy.config.ssl.client.verify.server' : 2, 'proxy.config.ssl.client.CA.cert.path': '{0}'.format(ts.Variables.SSLDir), 'proxy.config.ssl.client.CA.cert.filename': 'signer.pem', 'proxy.config.url_remap.pristine_host_hdr': 1, @@ -200,10 +213,18 @@ # Over riding the built in ERROR check since we expect some cases to fail # checks on random.com should fail with message only -ts.Disk.diags_log.Content = Testers.ContainsExpression("WARNING: Core server certificate verification failed for \(random.com\). Action=Continue Error=self signed certificate server=127.0.0.1\(127.0.0.1\) depth=0", "Warning for self signed certificate") +ts.Disk.diags_log.Content = Testers.ContainsExpression( + r"WARNING: Core server certificate verification failed for \(random.com\). Action=Continue Error=self signed certificate server=127.0.0.1\(127.0.0.1\) depth=0", + "Warning for self signed certificate") # permissive failure for bar.com -ts.Disk.diags_log.Content += Testers.ContainsExpression("WARNING: SNI \(bar.com\) not in certificate. Action=Continue server=bar.com\(127.0.0.1\)", "Warning on missing name for bar.com") +ts.Disk.diags_log.Content += Testers.ContainsExpression( + r"WARNING: SNI \(bar.com\) not in certificate. Action=Continue server=bar.com\(127.0.0.1\)", + "Warning on missing name for bar.com") # name check failure for random.com -ts.Disk.diags_log.Content += Testers.ContainsExpression("WARNING: SNI \(random.com\) not in certificate. Action=Continue server=127.0.0.1\(127.0.0.1\)", "Warning on missing name for randome.com") +ts.Disk.diags_log.Content += Testers.ContainsExpression( + r"WARNING: SNI \(random.com\) not in certificate. Action=Continue server=127.0.0.1\(127.0.0.1\)", + "Warning on missing name for randome.com") # name check failure for bar.com -ts.Disk.diags_log.Content += Testers.ContainsExpression("WARNING: SNI \(bar.com\) not in certificate. Action=Terminate server=bar.com\(127.0.0.1\)", "Failure on missing name for bar.com") +ts.Disk.diags_log.Content += Testers.ContainsExpression( + r"WARNING: SNI \(bar.com\) not in certificate. Action=Terminate server=bar.com\(127.0.0.1\)", + "Failure on missing name for bar.com") diff --git a/tests/gold_tests/tls/tls_verify_override_sni.test.py b/tests/gold_tests/tls/tls_verify_override_sni.test.py new file mode 100644 index 00000000000..be2e13b4f56 --- /dev/null +++ b/tests/gold_tests/tls/tls_verify_override_sni.test.py @@ -0,0 +1,152 @@ +''' +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +Test.Summary = ''' +Test tls server certificate verification options. Exercise conf_remap +''' + +# Define default ATS +ts = Test.MakeATSProcess("ts", select_ports=True) +cafile = "{0}/signer.pem".format(Test.RunDirectory) + +server_foo = Test.MakeOriginServer("server_foo", + ssl=True, + options={"--key": "{0}/signed-foo.key".format(Test.RunDirectory), + "--cert": "{0}/signed-foo.pem".format(Test.RunDirectory), + "--clientCA": cafile, + "--clientverify": ""}, + clientcert="{0}/signed-bar.pem".format(Test.RunDirectory), + clientkey="{0}/signed-bar.key".format(Test.RunDirectory)) +server_bar = Test.MakeOriginServer("server_bar", + ssl=True, + options={"--key": "{0}/signed-foo.key".format(Test.RunDirectory), + "--cert": "{0}/signed-foo.pem".format(Test.RunDirectory), + "--clientCA": cafile, + "--clientverify": ""}, + clientcert="{0}/signed-bar.pem".format(Test.RunDirectory), + clientkey="{0}/signed-bar.key".format(Test.RunDirectory)) + +dns = Test.MakeDNServer("dns") + +request_foo_header = {"headers": "GET / HTTP/1.1\r\nHost: foo.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +request_bad_foo_header = {"headers": "GET / HTTP/1.1\r\nHost: bad_foo.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +request_bar_header = {"headers": "GET / HTTP/1.1\r\nHost: bar.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +request_bad_bar_header = {"headers": "GET / HTTP/1.1\r\nHost: bad_bar.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +response_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +server_foo.addResponse("sessionlog.json", request_foo_header, response_header) +server_foo.addResponse("sessionlog.json", request_bad_foo_header, response_header) +server_bar.addResponse("sessionlog.json", request_bar_header, response_header) +server_bar.addResponse("sessionlog.json", request_bad_bar_header, response_header) + +server_bar.Setup.Copy("ssl/signer.pem") +server_bar.Setup.Copy("ssl/signer2.pem") +server_foo.Setup.Copy("ssl/signer.pem") +server_foo.Setup.Copy("ssl/signer2.pem") + +# add ssl materials like key, certificates for the server +ts.addSSLfile("ssl/signed-foo.pem") +ts.addSSLfile("ssl/signed-foo.key") +ts.addSSLfile("ssl/signed-bar.pem") +ts.addSSLfile("ssl/signed-bar.key") +ts.addSSLfile("ssl/server.pem") +ts.addSSLfile("ssl/server.key") +ts.addSSLfile("ssl/signer.pem") +ts.addSSLfile("ssl/signer.key") + +ts.Disk.remap_config.AddLine( + 'map http://foo.com/defaultbar https://bar.com:{0}'.format(server_bar.Variables.SSL_Port)) +ts.Disk.remap_config.AddLine( + 'map http://foo.com/default https://foo.com:{0}'.format(server_foo.Variables.SSL_Port)) +ts.Disk.remap_config.AddLine( + 'map http://foo.com/overridepolicy https://bar.com:{0} @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.policy=ENFORCED'.format( + server_foo.Variables.SSL_Port)) +ts.Disk.remap_config.AddLine( + 'map http://foo.com/overrideproperties https://bar.com:{0} @plugin=conf_remap.so @pparam=proxy.config.ssl.client.verify.server.properties=SIGNATURE'.format( + server_foo.Variables.SSL_Port)) + +ts.Disk.ssl_multicert_config.AddLine( + 'dest_ip=* ssl_cert_name=server.pem ssl_key_name=server.key' +) + +# global config policy=permissive properties=all +ts.Disk.records_config.update({ + 'proxy.config.diags.debug.enabled': 1, + 'proxy.config.diags.debug.tags': 'ssl', + 'proxy.config.ssl.server.cert.path': '{0}'.format(ts.Variables.SSLDir), + 'proxy.config.ssl.server.private_key.path': '{0}'.format(ts.Variables.SSLDir), + 'proxy.config.ssl.server.cipher_suite': 'ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA384:AES128-GCM-SHA256:AES256-GCM-SHA384:ECDHE-RSA-RC4-SHA:ECDHE-RSA-AES128-SHA:ECDHE-RSA-AES256-SHA:RC4-SHA:RC4-MD5:AES128-SHA:AES256-SHA:DES-CBC3-SHA!SRP:!DSS:!PSK:!aNULL:!eNULL:!SSLv2', + # set global policy + 'proxy.config.ssl.client.verify.server.policy': 'PERMISSIVE', + 'proxy.config.ssl.client.verify.server.properties': 'ALL', + 'proxy.config.ssl.client.CA.cert.path': '{0}'.format(ts.Variables.SSLDir), + 'proxy.config.ssl.client.CA.cert.filename': 'signer.pem', + 'proxy.config.url_remap.pristine_host_hdr': 1, + 'proxy.config.dns.nameservers': '127.0.0.1:{0}'.format(dns.Variables.Port), + 'proxy.config.dns.resolv_conf': 'NULL', + 'proxy.config.exec_thread.autoconfig.scale': 1.0, + 'proxy.config.ssl.client.sni_policy': 'remap' +}) + +ts.Disk.sni_yaml.AddLines([ + 'sni:', + '- fqdn: bar.com', + ' client_cert: "{0}/signed-foo.pem"'.format(ts.Variables.SSLDir), + ' client_key: "{0}/signed-foo.key"'.format(ts.Variables.SSLDir), +]) + +dns.addRecords(records={"foo.com.": ["127.0.0.1"]}) +dns.addRecords(records={"bar.com.": ["127.0.0.1"]}) +dns.addRecords(records={"random.com.": ["127.0.0.1"]}) + +# Should succeed with message +# exercise default settings +tr = Test.AddTestRun("default-permissive-success") +tr.Setup.Copy("ssl/signed-foo.key") +tr.Setup.Copy("ssl/signed-foo.pem") +tr.Setup.Copy("ssl/signed-bar.key") +tr.Setup.Copy("ssl/signed-bar.pem") +tr.Processes.Default.Command = 'curl -k -H \"host: foo.com\" http://127.0.0.1:{0}/defaultbar'.format(ts.Variables.port) +tr.ReturnCode = 0 +tr.Processes.Default.StartBefore(dns) +tr.Processes.Default.StartBefore(server_foo) +tr.Processes.Default.StartBefore(server_bar) +tr.Processes.Default.StartBefore(Test.Processes.ts) +tr.StillRunningAfter = ts +tr.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") + +# should fail. Exercise the override +tr2 = Test.AddTestRun("policy-override-fail") +tr2.Processes.Default.Command = "curl -k -H \"host: foo.com\" http://127.0.0.1:{0}/overridepolicy".format(ts.Variables.port) +tr2.ReturnCode = 0 +tr2.StillRunningAfter = ts +tr2.Processes.Default.Streams.stdout = Testers.ContainsExpression("Could Not Connect", "Curl attempt should fail") + +# should succeed with an error message +tr2 = Test.AddTestRun("properties-override-permissive") +tr2.Processes.Default.Command = "curl -k -H \"host: foo.com\" http://127.0.0.1:{0}/overrideproperties".format(ts.Variables.port) +tr2.ReturnCode = 0 +tr2.StillRunningAfter = ts +tr2.Processes.Default.Streams.stdout = Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded") + + +# Over riding the built in ERROR check since we expect some cases to fail +ts.Disk.diags_log.Content = Testers.ContainsExpression( + "WARNING: SNI \(bar.com\) not in certificate. Action=Continue server=bar.com", "Warning for mismatch name not enforcing") +ts.Disk.diags_log.Content += Testers.ContainsExpression( + " WARNING: SNI \(bar.com\) not in certificate. Action=Terminate server=bar.com", "Warning for enforcing mismatch") diff --git a/tests/gold_tests/tls_hooks/tls_hooks.test.py b/tests/gold_tests/tls_hooks/tls_hooks.test.py index 7383da232cb..8cd36777e2a 100644 --- a/tests/gold_tests/tls_hooks/tls_hooks.test.py +++ b/tests/gold_tests/tls_hooks/tls_hooks.test.py @@ -39,7 +39,7 @@ 'proxy.config.diags.debug.tags': 'ssl_hook_test', 'proxy.config.ssl.server.cert.path': '{0}'.format(ts.Variables.SSLDir), 'proxy.config.ssl.server.private_key.path': '{0}'.format(ts.Variables.SSLDir), - 'proxy.config.ssl.client.verify.server': 0, + 'proxy.config.ssl.client.verify.server': 0, 'proxy.config.ssl.TLSv1_3': 0, 'proxy.config.ssl.server.cipher_suite': 'ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA384:AES128-GCM-SHA256:AES256-GCM-SHA384:ECDHE-RSA-RC4-SHA:ECDHE-RSA-AES128-SHA:ECDHE-RSA-AES256-SHA:RC4-SHA:RC4-MD5:AES128-SHA:AES256-SHA:DES-CBC3-SHA!SRP:!DSS:!PSK:!aNULL:!eNULL:!SSLv2', }) @@ -52,7 +52,7 @@ 'map https://example.com:{0} http://127.0.0.1:{1}'.format(ts.Variables.ssl_port, server.Variables.Port) ) -Test.PreparePlugin(os.path.join(Test.Variables.AtsTestToolsDir, 'plugins', 'ssl_hook_test.cc'), ts, '-preaccept=1') +Test.PrepareTestPlugin(os.path.join(Test.Variables.AtsTestPluginsDir, 'ssl_hook_test.so'), ts, '-preaccept=1') tr = Test.AddTestRun("Test one preaccept hook") tr.Processes.Default.StartBefore(server) @@ -62,14 +62,17 @@ tr.Processes.Default.Command = 'curl -v -k -H \'host:example.com:{0}\' https://127.0.0.1:{0}'.format(ts.Variables.ssl_port) tr.Processes.Default.ReturnCode = 0 tr.Processes.Default.Streams.stdout = "gold/preaccept-1.gold" -tr.Processes.Default.Streams.All = Testers.ExcludesExpression("TLSv1.3 (IN), TLS handshake, Finished (20):", "Should not negotiate a TLSv1.3 connection") +tr.Processes.Default.Streams.All = Testers.ExcludesExpression( + "TLSv1.3 (IN), TLS handshake, Finished (20):", "Should not negotiate a TLSv1.3 connection") ts.Streams.stderr = "gold/ts-preaccept-1.gold" # the preaccept may get triggered twice because the test framework creates a TCP connection before handing off to traffic_server preacceptstring = "Pre accept callback 0" ts.Streams.All = Testers.ContainsExpression( - "\A(?:(?!{0}).)*{0}.*({0})?(?!.*{0}).*\Z".format(preacceptstring), "Pre accept message appears only once or twice", reflags=re.S | re.M) + r"\A(?:(?!{0}).)*{0}.*({0})?(?!.*{0}).*\Z".format(preacceptstring), + "Pre accept message appears only once or twice", + reflags=re.S | re.M) -tr.Processes.Default.TimeOut = 5 -tr.TimeOut = 5 +tr.Processes.Default.TimeOut = 15 +tr.TimeOut = 15 diff --git a/tests/gold_tests/tls_hooks/tls_hooks10.test.py b/tests/gold_tests/tls_hooks/tls_hooks10.test.py index b16fa6c4fe2..d2ec7c963e8 100644 --- a/tests/gold_tests/tls_hooks/tls_hooks10.test.py +++ b/tests/gold_tests/tls_hooks/tls_hooks10.test.py @@ -18,7 +18,6 @@ # limitations under the License. import os -import re Test.Summary = ''' Test different combinations of TLS handshake hooks to ensure they are applied consistently. @@ -39,7 +38,7 @@ 'proxy.config.diags.debug.tags': 'ssl_hook_test', 'proxy.config.ssl.server.cert.path': '{0}'.format(ts.Variables.SSLDir), 'proxy.config.ssl.server.private_key.path': '{0}'.format(ts.Variables.SSLDir), - 'proxy.config.ssl.client.verify.server': 0, + 'proxy.config.ssl.client.verify.server': 0, 'proxy.config.ssl.server.cipher_suite': 'ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA384:AES128-GCM-SHA256:AES256-GCM-SHA384:ECDHE-RSA-RC4-SHA:ECDHE-RSA-AES128-SHA:ECDHE-RSA-AES256-SHA:RC4-SHA:RC4-MD5:AES128-SHA:AES256-SHA:DES-CBC3-SHA!SRP:!DSS:!PSK:!aNULL:!eNULL:!SSLv2', }) @@ -51,7 +50,7 @@ 'map https://example.com:{0} http://127.0.0.1:{1}'.format(ts.Variables.ssl_port, server.Variables.Port) ) -Test.PreparePlugin(os.path.join(Test.Variables.AtsTestToolsDir, 'plugins', 'ssl_hook_test.cc'), ts, '-cert=1 -i=2') +Test.PrepareTestPlugin(os.path.join(Test.Variables.AtsTestPluginsDir, 'ssl_hook_test.so'), ts, '-cert=1 -i=2') tr = Test.AddTestRun("Test a combination of delayed and immediate cert hooks") tr.Processes.Default.StartBefore(server) @@ -63,5 +62,5 @@ ts.Streams.stderr = "gold/ts-cert-1-im-2.gold" -tr.Processes.Default.TimeOut = 5 -tr.TimeOut = 5 +tr.Processes.Default.TimeOut = 15 +tr.TimeOut = 15 diff --git a/tests/gold_tests/tls_hooks/tls_hooks11.test.py b/tests/gold_tests/tls_hooks/tls_hooks11.test.py index 587f630f24d..ca06718c612 100644 --- a/tests/gold_tests/tls_hooks/tls_hooks11.test.py +++ b/tests/gold_tests/tls_hooks/tls_hooks11.test.py @@ -39,7 +39,7 @@ 'proxy.config.diags.debug.tags': 'ssl_hook_test', 'proxy.config.ssl.server.cert.path': '{0}'.format(ts.Variables.SSLDir), 'proxy.config.ssl.server.private_key.path': '{0}'.format(ts.Variables.SSLDir), - 'proxy.config.ssl.client.verify.server': 0, + 'proxy.config.ssl.client.verify.server': 0, 'proxy.config.ssl.server.cipher_suite': 'ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA384:AES128-GCM-SHA256:AES256-GCM-SHA384:ECDHE-RSA-RC4-SHA:ECDHE-RSA-AES128-SHA:ECDHE-RSA-AES256-SHA:RC4-SHA:RC4-MD5:AES128-SHA:AES256-SHA:DES-CBC3-SHA!SRP:!DSS:!PSK:!aNULL:!eNULL:!SSLv2', }) @@ -51,7 +51,7 @@ 'map https://example.com:{0} http://127.0.0.1:{1}'.format(ts.Variables.ssl_port, server.Variables.Port) ) -Test.PreparePlugin(os.path.join(Test.Variables.AtsTestToolsDir, 'plugins', 'ssl_hook_test.cc'), ts, '-d=1') +Test.PrepareTestPlugin(os.path.join(Test.Variables.AtsTestPluginsDir, 'ssl_hook_test.so'), ts, '-d=1') tr = Test.AddTestRun("Test one delayed preaccept hook") tr.Processes.Default.StartBefore(server) @@ -66,6 +66,8 @@ preacceptstring = "Pre accept delay callback 0" ts.Streams.All = Testers.ContainsExpression( - "\A(?:(?!{0}).)*{0}.*({0})?(?!.*{0}).*\Z".format(preacceptstring), "Pre accept message appears only once or twice", reflags=re.S | re.M) -tr.Processes.Default.TimeOut = 5 -tr.TimeOut = 5 + r"\A(?:(?!{0}).)*{0}.*({0})?(?!.*{0}).*\Z".format(preacceptstring), + "Pre accept message appears only once or twice", + reflags=re.S | re.M) +tr.Processes.Default.TimeOut = 15 +tr.TimeOut = 15 diff --git a/tests/gold_tests/tls_hooks/tls_hooks12.test.py b/tests/gold_tests/tls_hooks/tls_hooks12.test.py index dbc76367d5b..bea831f6494 100644 --- a/tests/gold_tests/tls_hooks/tls_hooks12.test.py +++ b/tests/gold_tests/tls_hooks/tls_hooks12.test.py @@ -18,7 +18,6 @@ # limitations under the License. import os -import re Test.Summary = ''' Test different combinations of TLS handshake hooks to ensure they are applied consistently. @@ -39,7 +38,7 @@ 'proxy.config.diags.debug.tags': 'ssl_hook_test', 'proxy.config.ssl.server.cert.path': '{0}'.format(ts.Variables.SSLDir), 'proxy.config.ssl.server.private_key.path': '{0}'.format(ts.Variables.SSLDir), - 'proxy.config.ssl.client.verify.server': 0, + 'proxy.config.ssl.client.verify.server': 0, 'proxy.config.ssl.server.cipher_suite': 'ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA384:AES128-GCM-SHA256:AES256-GCM-SHA384:ECDHE-RSA-RC4-SHA:ECDHE-RSA-AES128-SHA:ECDHE-RSA-AES256-SHA:RC4-SHA:RC4-MD5:AES128-SHA:AES256-SHA:DES-CBC3-SHA!SRP:!DSS:!PSK:!aNULL:!eNULL:!SSLv2', }) @@ -51,7 +50,7 @@ 'map https://example.com:{0} http://127.0.0.1:{1}'.format(ts.Variables.ssl_port, server.Variables.Port) ) -Test.PreparePlugin(os.path.join(Test.Variables.AtsTestToolsDir, 'plugins', 'ssl_hook_test.cc'), ts, '-p=2 -d=1') +Test.PrepareTestPlugin(os.path.join(Test.Variables.AtsTestPluginsDir, 'ssl_hook_test.so'), ts, '-p=2 -d=1') tr = Test.AddTestRun("Test combination of delayed and immediate preaccept hook2") tr.Processes.Default.StartBefore(server) @@ -63,5 +62,5 @@ ts.Streams.stderr = "gold/ts-preaccept-delayed-1-immdate-2.gold" -tr.Processes.Default.TimeOut = 5 -tr.TimeOut = 5 +tr.Processes.Default.TimeOut = 15 +tr.TimeOut = 15 diff --git a/tests/gold_tests/tls_hooks/tls_hooks13.test.py b/tests/gold_tests/tls_hooks/tls_hooks13.test.py index 256d8e93d9b..65bcbd97b8e 100644 --- a/tests/gold_tests/tls_hooks/tls_hooks13.test.py +++ b/tests/gold_tests/tls_hooks/tls_hooks13.test.py @@ -18,7 +18,6 @@ # limitations under the License. import os -import re Test.Summary = ''' Test different combinations of TLS handshake hooks to ensure they are applied consistently. @@ -39,7 +38,7 @@ 'proxy.config.diags.debug.tags': 'ssl_hook_test', 'proxy.config.ssl.server.cert.path': '{0}'.format(ts.Variables.SSLDir), 'proxy.config.ssl.server.private_key.path': '{0}'.format(ts.Variables.SSLDir), - 'proxy.config.ssl.client.verify.server': 0, + 'proxy.config.ssl.client.verify.server': 0, 'proxy.config.ssl.server.cipher_suite': 'ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA384:AES128-GCM-SHA256:AES256-GCM-SHA384:ECDHE-RSA-RC4-SHA:ECDHE-RSA-AES128-SHA:ECDHE-RSA-AES256-SHA:RC4-SHA:RC4-MD5:AES128-SHA:AES256-SHA:DES-CBC3-SHA!SRP:!DSS:!PSK:!aNULL:!eNULL:!SSLv2', }) @@ -51,7 +50,7 @@ 'map https://example.com:{0} https://127.0.0.1:{1}'.format(ts.Variables.ssl_port, server.Variables.SSL_Port) ) -Test.PreparePlugin(os.path.join(Test.Variables.AtsTestToolsDir, 'plugins', 'ssl_hook_test.cc'), ts, '-out_start=1 -out_close=2') +Test.PrepareTestPlugin(os.path.join(Test.Variables.AtsTestPluginsDir, 'ssl_hook_test.so'), ts, '-out_start=1 -out_close=2') tr = Test.AddTestRun("Test outbound start and close") tr.Processes.Default.StartBefore(server) @@ -63,5 +62,5 @@ ts.Streams.stderr = "gold/ts-out-start-close-2.gold" -tr.Processes.Default.TimeOut = 5 -tr.TimeOut = 5 +tr.Processes.Default.TimeOut = 15 +tr.TimeOut = 15 diff --git a/tests/gold_tests/tls_hooks/tls_hooks14.test.py b/tests/gold_tests/tls_hooks/tls_hooks14.test.py index 1c78e8b2d34..888036cc070 100644 --- a/tests/gold_tests/tls_hooks/tls_hooks14.test.py +++ b/tests/gold_tests/tls_hooks/tls_hooks14.test.py @@ -18,7 +18,6 @@ # limitations under the License. import os -import re Test.Summary = ''' Test different combinations of TLS handshake hooks to ensure they are applied consistently. @@ -39,7 +38,7 @@ 'proxy.config.diags.debug.tags': 'ssl_hook_test', 'proxy.config.ssl.server.cert.path': '{0}'.format(ts.Variables.SSLDir), 'proxy.config.ssl.server.private_key.path': '{0}'.format(ts.Variables.SSLDir), - 'proxy.config.ssl.client.verify.server': 0, + 'proxy.config.ssl.client.verify.server': 0, 'proxy.config.ssl.server.cipher_suite': 'ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA384:AES128-GCM-SHA256:AES256-GCM-SHA384:ECDHE-RSA-RC4-SHA:ECDHE-RSA-AES128-SHA:ECDHE-RSA-AES256-SHA:RC4-SHA:RC4-MD5:AES128-SHA:AES256-SHA:DES-CBC3-SHA!SRP:!DSS:!PSK:!aNULL:!eNULL:!SSLv2', }) @@ -51,7 +50,7 @@ 'map https://example.com:{0} https://127.0.0.1:{1}'.format(ts.Variables.ssl_port, server.Variables.SSL_Port) ) -Test.PreparePlugin(os.path.join(Test.Variables.AtsTestToolsDir, 'plugins', 'ssl_hook_test.cc'), ts, '-out_start_delay=2') +Test.PrepareTestPlugin(os.path.join(Test.Variables.AtsTestPluginsDir, 'ssl_hook_test.so'), ts, '-out_start_delay=2') tr = Test.AddTestRun("Test outbound delay start") tr.Processes.Default.StartBefore(server) @@ -63,5 +62,5 @@ ts.Streams.stderr = "gold/ts-out-delay-start-2.gold" -tr.Processes.Default.TimeOut = 5 -tr.TimeOut = 5 +tr.Processes.Default.TimeOut = 15 +tr.TimeOut = 15 diff --git a/tests/gold_tests/tls_hooks/tls_hooks15.test.py b/tests/gold_tests/tls_hooks/tls_hooks15.test.py index c844dfd52fe..3cd97410b10 100644 --- a/tests/gold_tests/tls_hooks/tls_hooks15.test.py +++ b/tests/gold_tests/tls_hooks/tls_hooks15.test.py @@ -18,7 +18,6 @@ # limitations under the License. import os -import re Test.Summary = ''' Test different combinations of TLS handshake hooks to ensure they are applied consistently. @@ -39,7 +38,7 @@ 'proxy.config.diags.debug.tags': 'ssl_hook_test', 'proxy.config.ssl.server.cert.path': '{0}'.format(ts.Variables.SSLDir), 'proxy.config.ssl.server.private_key.path': '{0}'.format(ts.Variables.SSLDir), - 'proxy.config.ssl.client.verify.server': 0, + 'proxy.config.ssl.client.verify.server': 0, 'proxy.config.ssl.server.cipher_suite': 'ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA384:AES128-GCM-SHA256:AES256-GCM-SHA384:ECDHE-RSA-RC4-SHA:ECDHE-RSA-AES128-SHA:ECDHE-RSA-AES256-SHA:RC4-SHA:RC4-MD5:AES128-SHA:AES256-SHA:DES-CBC3-SHA!SRP:!DSS:!PSK:!aNULL:!eNULL:!SSLv2', }) @@ -51,7 +50,7 @@ 'map https://example.com:{0} https://127.0.0.1:{1}'.format(ts.Variables.ssl_port, server.Variables.SSL_Port) ) -Test.PreparePlugin(os.path.join(Test.Variables.AtsTestToolsDir, 'plugins', 'ssl_hook_test.cc'), ts, '-close=2 -out_close=1') +Test.PrepareTestPlugin(os.path.join(Test.Variables.AtsTestPluginsDir, 'ssl_hook_test.so'), ts, '-close=2 -out_close=1') tr = Test.AddTestRun("Test one delayed preaccept hook") tr.Processes.Default.StartBefore(server) @@ -63,5 +62,5 @@ ts.Streams.stderr = "gold/ts-close-out-close.gold" -tr.Processes.Default.TimeOut = 5 -tr.TimeOut = 5 +tr.Processes.Default.TimeOut = 15 +tr.TimeOut = 15 diff --git a/tests/gold_tests/tls_hooks/tls_hooks16.test.py b/tests/gold_tests/tls_hooks/tls_hooks16.test.py index 9f2cd21824b..fb54094d70d 100644 --- a/tests/gold_tests/tls_hooks/tls_hooks16.test.py +++ b/tests/gold_tests/tls_hooks/tls_hooks16.test.py @@ -42,7 +42,7 @@ 'proxy.config.diags.debug.tags': 'ssl_hook_test', 'proxy.config.ssl.server.cert.path': '{0}'.format(ts.Variables.SSLDir), 'proxy.config.ssl.server.private_key.path': '{0}'.format(ts.Variables.SSLDir), - 'proxy.config.ssl.client.verify.server': 0, + 'proxy.config.ssl.client.verify.server': 0, 'proxy.config.ssl.server.cipher_suite': 'ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA384:AES128-GCM-SHA256:AES256-GCM-SHA384:ECDHE-RSA-RC4-SHA:ECDHE-RSA-AES128-SHA:ECDHE-RSA-AES256-SHA:RC4-SHA:RC4-MD5:AES128-SHA:AES256-SHA:DES-CBC3-SHA!SRP:!DSS:!PSK:!aNULL:!eNULL:!SSLv2', }) @@ -54,7 +54,7 @@ 'map https://example.com:{1} http://127.0.0.1:{0}'.format(server.Variables.Port, ts.Variables.ssl_port) ) -Test.PreparePlugin(os.path.join(Test.Variables.AtsTestToolsDir, 'plugins', 'ssl_hook_test.cc'), ts, '-client_hello_imm=1') +Test.PrepareTestPlugin(os.path.join(Test.Variables.AtsTestPluginsDir, 'ssl_hook_test.so'), ts, '-client_hello_imm=1') tr = Test.AddTestRun("Test one immediate client hello hook") tr.Processes.Default.StartBefore(server) @@ -71,5 +71,5 @@ ts.Streams.All = Testers.ContainsExpression( "\A(?:(?!{0}).)*{0}(?!.*{0}).*\Z".format(snistring), "Client Hello message appears only once", reflags=re.S | re.M) -tr.Processes.Default.TimeOut = 5 -tr.TimeOut = 5 +tr.Processes.Default.TimeOut = 15 +tr.TimeOut = 15 diff --git a/tests/gold_tests/tls_hooks/tls_hooks17.test.py b/tests/gold_tests/tls_hooks/tls_hooks17.test.py index de18889d16b..43074c23318 100644 --- a/tests/gold_tests/tls_hooks/tls_hooks17.test.py +++ b/tests/gold_tests/tls_hooks/tls_hooks17.test.py @@ -42,7 +42,7 @@ 'proxy.config.diags.debug.tags': 'ssl_hook_test', 'proxy.config.ssl.server.cert.path': '{0}'.format(ts.Variables.SSLDir), 'proxy.config.ssl.server.private_key.path': '{0}'.format(ts.Variables.SSLDir), - 'proxy.config.ssl.client.verify.server': 0, + 'proxy.config.ssl.client.verify.server': 0, 'proxy.config.ssl.server.cipher_suite': 'ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA384:AES128-GCM-SHA256:AES256-GCM-SHA384:ECDHE-RSA-RC4-SHA:ECDHE-RSA-AES128-SHA:ECDHE-RSA-AES256-SHA:RC4-SHA:RC4-MD5:AES128-SHA:AES256-SHA:DES-CBC3-SHA!SRP:!DSS:!PSK:!aNULL:!eNULL:!SSLv2', }) @@ -54,7 +54,7 @@ 'map https://example.com:{1} http://127.0.0.1:{0}'.format(server.Variables.Port, ts.Variables.ssl_port) ) -Test.PreparePlugin(os.path.join(Test.Variables.AtsTestToolsDir, 'plugins', 'ssl_hook_test.cc'), ts, '-client_hello=1') +Test.PrepareTestPlugin(os.path.join(Test.Variables.AtsTestPluginsDir, 'ssl_hook_test.so'), ts, '-client_hello=1') tr = Test.AddTestRun("Test one delayed client hello hook") tr.Processes.Default.StartBefore(server) @@ -71,5 +71,5 @@ ts.Streams.All = Testers.ContainsExpression( "\A(?:(?!{0}).)*{0}(?!.*{0}).*\Z".format(snistring), "Client Hello message appears only once", reflags=re.S | re.M) -tr.Processes.Default.TimeOut = 5 -tr.TimeOut = 5 +tr.Processes.Default.TimeOut = 15 +tr.TimeOut = 15 diff --git a/tests/gold_tests/tls_hooks/tls_hooks18.test.py b/tests/gold_tests/tls_hooks/tls_hooks18.test.py index 5f0864900b4..4ce79753186 100644 --- a/tests/gold_tests/tls_hooks/tls_hooks18.test.py +++ b/tests/gold_tests/tls_hooks/tls_hooks18.test.py @@ -26,7 +26,7 @@ Test.SkipUnless( Condition.HasOpenSSLVersion("1.1.1") - ) +) ts = Test.MakeATSProcess("ts", select_ports=True, enable_tls=True) server = Test.MakeOriginServer("server") @@ -43,7 +43,7 @@ 'proxy.config.diags.debug.tags': 'ssl_hook_test', 'proxy.config.ssl.server.cert.path': '{0}'.format(ts.Variables.SSLDir), 'proxy.config.ssl.server.private_key.path': '{0}'.format(ts.Variables.SSLDir), - 'proxy.config.ssl.client.verify.server': 0, + 'proxy.config.ssl.client.verify.server': 0, 'proxy.config.ssl.server.cipher_suite': 'ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA384:AES128-GCM-SHA256:AES256-GCM-SHA384:ECDHE-RSA-RC4-SHA:ECDHE-RSA-AES128-SHA:ECDHE-RSA-AES256-SHA:RC4-SHA:RC4-MD5:AES128-SHA:AES256-SHA:DES-CBC3-SHA!SRP:!DSS:!PSK:!aNULL:!eNULL:!SSLv2', }) @@ -55,7 +55,7 @@ 'map https://example.com:{1} http://127.0.0.1:{0}'.format(server.Variables.Port, ts.Variables.ssl_port) ) -Test.PreparePlugin(os.path.join(Test.Variables.AtsTestToolsDir, 'plugins', 'ssl_hook_test.cc'), ts, '-client_hello=2') +Test.PrepareTestPlugin(os.path.join(Test.Variables.AtsTestPluginsDir, 'ssl_hook_test.so'), ts, '-client_hello=2') tr = Test.AddTestRun("Test two client hello hooks") tr.Processes.Default.StartBefore(server) @@ -75,5 +75,5 @@ ts.Streams.All = Testers.ContainsExpression( "\A(?:(?!{0}).)*{0}(?!.*{0}).*\Z".format(certstring1), "Cert message appears only once", reflags=re.S | re.M) -tr.Processes.Default.TimeOut = 5 -tr.TimeOut = 5 +tr.Processes.Default.TimeOut = 15 +tr.TimeOut = 15 diff --git a/tests/gold_tests/tls_hooks/tls_hooks2.test.py b/tests/gold_tests/tls_hooks/tls_hooks2.test.py index 61803eb9f45..fef2a489c5e 100644 --- a/tests/gold_tests/tls_hooks/tls_hooks2.test.py +++ b/tests/gold_tests/tls_hooks/tls_hooks2.test.py @@ -39,7 +39,7 @@ 'proxy.config.diags.debug.tags': 'ssl_hook_test', 'proxy.config.ssl.server.cert.path': '{0}'.format(ts.Variables.SSLDir), 'proxy.config.ssl.server.private_key.path': '{0}'.format(ts.Variables.SSLDir), - 'proxy.config.ssl.client.verify.server': 0, + 'proxy.config.ssl.client.verify.server': 0, 'proxy.config.ssl.server.cipher_suite': 'ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA384:AES128-GCM-SHA256:AES256-GCM-SHA384:ECDHE-RSA-RC4-SHA:ECDHE-RSA-AES128-SHA:ECDHE-RSA-AES256-SHA:RC4-SHA:RC4-MD5:AES128-SHA:AES256-SHA:DES-CBC3-SHA!SRP:!DSS:!PSK:!aNULL:!eNULL:!SSLv2', }) @@ -51,7 +51,7 @@ 'map https://example.com:{0} http://127.0.0.1:{1}'.format(ts.Variables.ssl_port, server.Variables.Port) ) -Test.PreparePlugin(os.path.join(Test.Variables.AtsTestToolsDir, 'plugins', 'ssl_hook_test.cc'), ts, '-sni=1') +Test.PrepareTestPlugin(os.path.join(Test.Variables.AtsTestPluginsDir, 'ssl_hook_test.so'), ts, '-sni=1') tr = Test.AddTestRun("Test one sni hook") tr.Processes.Default.StartBefore(server) @@ -68,5 +68,5 @@ ts.Streams.All = Testers.ContainsExpression( "\A(?:(?!{0}).)*{0}(?!.*{0}).*\Z".format(snistring), "SNI message appears only once", reflags=re.S | re.M) -tr.Processes.Default.TimeOut = 5 -tr.TimeOut = 5 +tr.Processes.Default.TimeOut = 15 +tr.TimeOut = 15 diff --git a/tests/gold_tests/tls_hooks/tls_hooks3.test.py b/tests/gold_tests/tls_hooks/tls_hooks3.test.py index adf76ab6323..17778fdd0a5 100644 --- a/tests/gold_tests/tls_hooks/tls_hooks3.test.py +++ b/tests/gold_tests/tls_hooks/tls_hooks3.test.py @@ -39,7 +39,7 @@ 'proxy.config.diags.debug.tags': 'ssl_hook_test', 'proxy.config.ssl.server.cert.path': '{0}'.format(ts.Variables.SSLDir), 'proxy.config.ssl.server.private_key.path': '{0}'.format(ts.Variables.SSLDir), - 'proxy.config.ssl.client.verify.server': 0, + 'proxy.config.ssl.client.verify.server': 0, 'proxy.config.ssl.server.cipher_suite': 'ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA384:AES128-GCM-SHA256:AES256-GCM-SHA384:ECDHE-RSA-RC4-SHA:ECDHE-RSA-AES128-SHA:ECDHE-RSA-AES256-SHA:RC4-SHA:RC4-MD5:AES128-SHA:AES256-SHA:DES-CBC3-SHA!SRP:!DSS:!PSK:!aNULL:!eNULL:!SSLv2', }) @@ -51,7 +51,7 @@ 'map https://example.com:{0} http://127.0.0.1:{1}'.format(ts.Variables.ssl_port, server.Variables.Port) ) -Test.PreparePlugin(os.path.join(Test.Variables.AtsTestToolsDir, 'plugins', 'ssl_hook_test.cc'), ts, '-cert=1') +Test.PrepareTestPlugin(os.path.join(Test.Variables.AtsTestPluginsDir, 'ssl_hook_test.so'), ts, '-cert=1') tr = Test.AddTestRun("Test one cert hook") tr.Processes.Default.StartBefore(server) @@ -68,5 +68,5 @@ ts.Streams.All = Testers.ContainsExpression( "\A(?:(?!{0}).)*{0}(?!.*{0}).*\Z".format(certstring), "Cert message appears only once", reflags=re.S | re.M) -tr.Processes.Default.TimeOut = 5 -tr.TimeOut = 5 +tr.Processes.Default.TimeOut = 15 +tr.TimeOut = 15 diff --git a/tests/gold_tests/tls_hooks/tls_hooks4.test.py b/tests/gold_tests/tls_hooks/tls_hooks4.test.py index 12b55ea4fe4..0ebe3dc710b 100644 --- a/tests/gold_tests/tls_hooks/tls_hooks4.test.py +++ b/tests/gold_tests/tls_hooks/tls_hooks4.test.py @@ -39,7 +39,7 @@ 'proxy.config.diags.debug.tags': 'ssl_hook_test', 'proxy.config.ssl.server.cert.path': '{0}'.format(ts.Variables.SSLDir), 'proxy.config.ssl.server.private_key.path': '{0}'.format(ts.Variables.SSLDir), - 'proxy.config.ssl.client.verify.server': 0, + 'proxy.config.ssl.client.verify.server': 0, 'proxy.config.ssl.server.cipher_suite': 'ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA384:AES128-GCM-SHA256:AES256-GCM-SHA384:ECDHE-RSA-RC4-SHA:ECDHE-RSA-AES128-SHA:ECDHE-RSA-AES256-SHA:RC4-SHA:RC4-MD5:AES128-SHA:AES256-SHA:DES-CBC3-SHA!SRP:!DSS:!PSK:!aNULL:!eNULL:!SSLv2', }) @@ -51,7 +51,7 @@ 'map https://example.com:{0} http://127.0.0.1:{1}'.format(ts.Variables.ssl_port, server.Variables.Port) ) -Test.PreparePlugin(os.path.join(Test.Variables.AtsTestToolsDir, 'plugins', 'ssl_hook_test.cc'), ts, '-cert=1 -sni=1 -preaccept=1') +Test.PrepareTestPlugin(os.path.join(Test.Variables.AtsTestPluginsDir, 'ssl_hook_test.so'), ts, '-cert=1 -sni=1 -preaccept=1') tr = Test.AddTestRun("Test one sni, one preaccept, and one cert hook") tr.Processes.Default.StartBefore(server) @@ -74,5 +74,5 @@ ts.Streams.All += Testers.ContainsExpression("\A(?:(?!{0}).)*{0}(?!.*{0}).*\Z".format(certstring), "Cert message appears only once", reflags=re.S | re.M) -tr.Processes.Default.TimeOut = 5 -tr.TimeOut = 5 +tr.Processes.Default.TimeOut = 15 +tr.TimeOut = 15 diff --git a/tests/gold_tests/tls_hooks/tls_hooks6.test.py b/tests/gold_tests/tls_hooks/tls_hooks6.test.py index 200e4bd47de..8dc7fd29c2a 100644 --- a/tests/gold_tests/tls_hooks/tls_hooks6.test.py +++ b/tests/gold_tests/tls_hooks/tls_hooks6.test.py @@ -39,7 +39,7 @@ 'proxy.config.diags.debug.tags': 'ssl_hook_test', 'proxy.config.ssl.server.cert.path': '{0}'.format(ts.Variables.SSLDir), 'proxy.config.ssl.server.private_key.path': '{0}'.format(ts.Variables.SSLDir), - 'proxy.config.ssl.client.verify.server': 0, + 'proxy.config.ssl.client.verify.server': 0, 'proxy.config.ssl.server.cipher_suite': 'ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA384:AES128-GCM-SHA256:AES256-GCM-SHA384:ECDHE-RSA-RC4-SHA:ECDHE-RSA-AES128-SHA:ECDHE-RSA-AES256-SHA:RC4-SHA:RC4-MD5:AES128-SHA:AES256-SHA:DES-CBC3-SHA!SRP:!DSS:!PSK:!aNULL:!eNULL:!SSLv2', }) @@ -51,7 +51,7 @@ 'map https://example.com:{0} http://127.0.0.1:{1}'.format(ts.Variables.ssl_port, server.Variables.Port) ) -Test.PreparePlugin(os.path.join(Test.Variables.AtsTestToolsDir, 'plugins', 'ssl_hook_test.cc'), ts, '-preaccept=2') +Test.PrepareTestPlugin(os.path.join(Test.Variables.AtsTestPluginsDir, 'ssl_hook_test.so'), ts, '-preaccept=2') tr = Test.AddTestRun("Test two preaccept hooks") tr.Processes.Default.StartBefore(server) @@ -68,9 +68,13 @@ preacceptstring0 = "Pre accept callback 0" preacceptstring1 = "Pre accept callback 1" ts.Streams.All = Testers.ContainsExpression( - "\A(?:(?!{0}).)*{0}.*({0})?(?!.*{0}).*\Z".format(preacceptstring0), "Pre accept message appears only once or twice", reflags=re.S | re.M) + r"\A(?:(?!{0}).)*{0}.*({0})?(?!.*{0}).*\Z".format(preacceptstring0), + "Pre accept message appears only once or twice", + reflags=re.S | re.M) ts.Streams.All = Testers.ContainsExpression( - "\A(?:(?!{0}).)*{0}.*({0})?(?!.*{0}).*\Z".format(preacceptstring1), "Pre accept message appears only once or twice", reflags=re.S | re.M) + r"\A(?:(?!{0}).)*{0}.*({0})?(?!.*{0}).*\Z".format(preacceptstring1), + "Pre accept message appears only once or twice", + reflags=re.S | re.M) -tr.Processes.Default.TimeOut = 5 -tr.TimeOut = 5 +tr.Processes.Default.TimeOut = 15 +tr.TimeOut = 15 diff --git a/tests/gold_tests/tls_hooks/tls_hooks7.test.py b/tests/gold_tests/tls_hooks/tls_hooks7.test.py index f488cea1f3e..45a418a166c 100644 --- a/tests/gold_tests/tls_hooks/tls_hooks7.test.py +++ b/tests/gold_tests/tls_hooks/tls_hooks7.test.py @@ -39,7 +39,7 @@ 'proxy.config.diags.debug.tags': 'ssl_hook_test', 'proxy.config.ssl.server.cert.path': '{0}'.format(ts.Variables.SSLDir), 'proxy.config.ssl.server.private_key.path': '{0}'.format(ts.Variables.SSLDir), - 'proxy.config.ssl.client.verify.server': 0, + 'proxy.config.ssl.client.verify.server': 0, 'proxy.config.ssl.server.cipher_suite': 'ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA384:AES128-GCM-SHA256:AES256-GCM-SHA384:ECDHE-RSA-RC4-SHA:ECDHE-RSA-AES128-SHA:ECDHE-RSA-AES256-SHA:RC4-SHA:RC4-MD5:AES128-SHA:AES256-SHA:DES-CBC3-SHA!SRP:!DSS:!PSK:!aNULL:!eNULL:!SSLv2', }) @@ -51,7 +51,7 @@ 'map https://example.com:{0} http://127.0.0.1:{1}'.format(ts.Variables.ssl_port, server.Variables.Port) ) -Test.PreparePlugin(os.path.join(Test.Variables.AtsTestToolsDir, 'plugins', 'ssl_hook_test.cc'), ts, '-sni=2') +Test.PrepareTestPlugin(os.path.join(Test.Variables.AtsTestPluginsDir, 'ssl_hook_test.so'), ts, '-sni=2') tr = Test.AddTestRun("Test two sni hooks") tr.Processes.Default.StartBefore(server) @@ -71,5 +71,5 @@ ts.Streams.All = Testers.ContainsExpression( "\A(?:(?!{0}).)*{0}(?!.*{0}).*\Z".format(snistring1), "SNI message appears only once", reflags=re.S | re.M) -tr.Processes.Default.TimeOut = 5 -tr.TimeOut = 5 +tr.Processes.Default.TimeOut = 15 +tr.TimeOut = 15 diff --git a/tests/gold_tests/tls_hooks/tls_hooks8.test.py b/tests/gold_tests/tls_hooks/tls_hooks8.test.py index 5af9a86efbe..6be43b9aef7 100644 --- a/tests/gold_tests/tls_hooks/tls_hooks8.test.py +++ b/tests/gold_tests/tls_hooks/tls_hooks8.test.py @@ -39,7 +39,7 @@ 'proxy.config.diags.debug.tags': 'ssl_hook_test', 'proxy.config.ssl.server.cert.path': '{0}'.format(ts.Variables.SSLDir), 'proxy.config.ssl.server.private_key.path': '{0}'.format(ts.Variables.SSLDir), - 'proxy.config.ssl.client.verify.server': 0, + 'proxy.config.ssl.client.verify.server': 0, 'proxy.config.ssl.server.cipher_suite': 'ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA384:AES128-GCM-SHA256:AES256-GCM-SHA384:ECDHE-RSA-RC4-SHA:ECDHE-RSA-AES128-SHA:ECDHE-RSA-AES256-SHA:RC4-SHA:RC4-MD5:AES128-SHA:AES256-SHA:DES-CBC3-SHA!SRP:!DSS:!PSK:!aNULL:!eNULL:!SSLv2', }) @@ -51,7 +51,7 @@ 'map https://example.com:{0} http://127.0.0.1:{1}'.format(ts.Variables.ssl_port, server.Variables.Port) ) -Test.PreparePlugin(os.path.join(Test.Variables.AtsTestToolsDir, 'plugins', 'ssl_hook_test.cc'), ts, '-cert=2') +Test.PrepareTestPlugin(os.path.join(Test.Variables.AtsTestPluginsDir, 'ssl_hook_test.so'), ts, '-cert=2') tr = Test.AddTestRun("Test two cert hooks") tr.Processes.Default.StartBefore(server) @@ -71,5 +71,5 @@ ts.Streams.All = Testers.ContainsExpression( "\A(?:(?!{0}).)*{0}(?!.*{0}).*\Z".format(certstring1), "Cert message appears only once", reflags=re.S | re.M) -tr.Processes.Default.TimeOut = 5 -tr.TimeOut = 5 +tr.Processes.Default.TimeOut = 15 +tr.TimeOut = 15 diff --git a/tests/gold_tests/tls_hooks/tls_hooks9.test.py b/tests/gold_tests/tls_hooks/tls_hooks9.test.py index 8ad5c92ee99..9cb51001a82 100644 --- a/tests/gold_tests/tls_hooks/tls_hooks9.test.py +++ b/tests/gold_tests/tls_hooks/tls_hooks9.test.py @@ -39,7 +39,7 @@ 'proxy.config.diags.debug.tags': 'ssl_hook_test', 'proxy.config.ssl.server.cert.path': '{0}'.format(ts.Variables.SSLDir), 'proxy.config.ssl.server.private_key.path': '{0}'.format(ts.Variables.SSLDir), - 'proxy.config.ssl.client.verify.server': 0, + 'proxy.config.ssl.client.verify.server': 0, 'proxy.config.ssl.server.cipher_suite': 'ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA384:AES128-GCM-SHA256:AES256-GCM-SHA384:ECDHE-RSA-RC4-SHA:ECDHE-RSA-AES128-SHA:ECDHE-RSA-AES256-SHA:RC4-SHA:RC4-MD5:AES128-SHA:AES256-SHA:DES-CBC3-SHA!SRP:!DSS:!PSK:!aNULL:!eNULL:!SSLv2', }) @@ -51,7 +51,7 @@ 'map https://example.com:{0} http://127.0.0.1:{1}'.format(ts.Variables.ssl_port, server.Variables.Port) ) -Test.PreparePlugin(os.path.join(Test.Variables.AtsTestToolsDir, 'plugins', 'ssl_hook_test.cc'), ts, '-i=1') +Test.PrepareTestPlugin(os.path.join(Test.Variables.AtsTestPluginsDir, 'ssl_hook_test.so'), ts, '-i=1') tr = Test.AddTestRun("Test one immediate cert hooks") tr.Processes.Default.StartBefore(server) @@ -67,5 +67,5 @@ certstring0 = "Cert callback 0" ts.Streams.All = Testers.ContainsExpression( "\A(?:(?!{0}).)*{0}(?!.*{0}).*\Z".format(certstring0), "Cert message appears only once", reflags=re.S | re.M) -tr.Processes.Default.TimeOut = 5 -tr.TimeOut = 5 +tr.Processes.Default.TimeOut = 15 +tr.TimeOut = 15 diff --git a/tests/include/catch.hpp b/tests/include/catch.hpp index ecd8907ea30..b4eccfc1483 100644 --- a/tests/include/catch.hpp +++ b/tests/include/catch.hpp @@ -1,9 +1,9 @@ /* - * Catch v2.2.2 - * Generated: 2018-04-06 12:05:03.186665 + * Catch v2.11.0 + * Generated: 2019-11-15 15:01:56.628356 * ---------------------------------------------------------- * This file has been merged from multiple headers. Please don't edit it directly - * Copyright (c) 2018 Two Blue Cubes Ltd. All rights reserved. + * Copyright (c) 2019 Two Blue Cubes Ltd. All rights reserved. * * Distributed under the Boost Software License, Version 1.0. (See accompanying * file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) @@ -14,8 +14,8 @@ #define CATCH_VERSION_MAJOR 2 -#define CATCH_VERSION_MINOR 2 -#define CATCH_VERSION_PATCH 2 +#define CATCH_VERSION_MINOR 11 +#define CATCH_VERSION_PATCH 0 #ifdef __clang__ # pragma clang system_header @@ -30,14 +30,17 @@ # pragma warning(push) # pragma warning(disable: 161 1682) # else // __ICC -# pragma clang diagnostic ignored "-Wunused-variable" # pragma clang diagnostic push # pragma clang diagnostic ignored "-Wpadded" # pragma clang diagnostic ignored "-Wswitch-enum" # pragma clang diagnostic ignored "-Wcovered-switch-default" # endif #elif defined __GNUC__ -# pragma GCC diagnostic ignored "-Wparentheses" + // Because REQUIREs trigger GCC's -Wparentheses, and because still + // supported version of g++ have only buggy support for _Pragmas, + // Wparentheses have to be suppressed globally. +# pragma GCC diagnostic ignored "-Wparentheses" // See #674 for details + # pragma GCC diagnostic push # pragma GCC diagnostic ignored "-Wunused-variable" # pragma GCC diagnostic ignored "-Wpadded" @@ -55,7 +58,9 @@ # if defined(CATCH_CONFIG_DISABLE_MATCHERS) # undef CATCH_CONFIG_DISABLE_MATCHERS # endif -# define CATCH_CONFIG_ENABLE_CHRONO_STRINGMAKER +# if !defined(CATCH_CONFIG_ENABLE_CHRONO_STRINGMAKER) +# define CATCH_CONFIG_ENABLE_CHRONO_STRINGMAKER +# endif #endif #if !defined(CATCH_CONFIG_IMPL_ONLY) @@ -72,7 +77,7 @@ #elif defined(linux) || defined(__linux) || defined(__linux__) # define CATCH_PLATFORM_LINUX -#elif defined(WIN32) || defined(__WIN32__) || defined(_WIN32) || defined(_MSC_VER) +#elif defined(WIN32) || defined(__WIN32__) || defined(_WIN32) || defined(_MSC_VER) || defined(__MINGW32__) # define CATCH_PLATFORM_WINDOWS #endif @@ -104,6 +109,7 @@ namespace Catch { // CATCH_CONFIG_COUNTER : is the __COUNTER__ macro supported? // CATCH_CONFIG_WINDOWS_SEH : is Windows SEH supported? // CATCH_CONFIG_POSIX_SIGNALS : are POSIX signals supported? +// CATCH_CONFIG_DISABLE_EXCEPTIONS : Are exceptions enabled? // **************** // Note to maintainers: if new toggles are added please document them // in configuration.md, too @@ -116,11 +122,11 @@ namespace Catch { #ifdef __cplusplus -# if __cplusplus >= 201402L +# if (__cplusplus >= 201402L) || (defined(_MSVC_LANG) && _MSVC_LANG >= 201402L) # define CATCH_CPP14_OR_GREATER # endif -# if __cplusplus >= 201703L +# if (__cplusplus >= 201703L) || (defined(_MSVC_LANG) && _MSVC_LANG >= 201703L) # define CATCH_CPP17_OR_GREATER # endif @@ -130,20 +136,33 @@ namespace Catch { # define CATCH_INTERNAL_CONFIG_CPP17_UNCAUGHT_EXCEPTIONS #endif -#ifdef __clang__ +// We have to avoid both ICC and Clang, because they try to mask themselves +// as gcc, and we want only GCC in this block +#if defined(__GNUC__) && !defined(__clang__) && !defined(__ICC) +# define CATCH_INTERNAL_START_WARNINGS_SUPPRESSION _Pragma( "GCC diagnostic push" ) +# define CATCH_INTERNAL_STOP_WARNINGS_SUPPRESSION _Pragma( "GCC diagnostic pop" ) +#endif + +#if defined(__clang__) + +# define CATCH_INTERNAL_START_WARNINGS_SUPPRESSION _Pragma( "clang diagnostic push" ) +# define CATCH_INTERNAL_STOP_WARNINGS_SUPPRESSION _Pragma( "clang diagnostic pop" ) -# define CATCH_INTERNAL_SUPPRESS_GLOBALS_WARNINGS \ - _Pragma( "clang diagnostic push" ) \ - _Pragma( "clang diagnostic ignored \"-Wexit-time-destructors\"" ) \ - _Pragma( "clang diagnostic ignored \"-Wglobal-constructors\"") -# define CATCH_INTERNAL_UNSUPPRESS_GLOBALS_WARNINGS \ - _Pragma( "clang diagnostic pop" ) +# define CATCH_INTERNAL_SUPPRESS_GLOBALS_WARNINGS \ + _Pragma( "clang diagnostic ignored \"-Wexit-time-destructors\"" ) \ + _Pragma( "clang diagnostic ignored \"-Wglobal-constructors\"") -# define CATCH_INTERNAL_SUPPRESS_PARENTHESES_WARNINGS \ - _Pragma( "clang diagnostic push" ) \ - _Pragma( "clang diagnostic ignored \"-Wparentheses\"" ) -# define CATCH_INTERNAL_UNSUPPRESS_PARENTHESES_WARNINGS \ - _Pragma( "clang diagnostic pop" ) +# define CATCH_INTERNAL_SUPPRESS_PARENTHESES_WARNINGS \ + _Pragma( "clang diagnostic ignored \"-Wparentheses\"" ) + +# define CATCH_INTERNAL_SUPPRESS_UNUSED_WARNINGS \ + _Pragma( "clang diagnostic ignored \"-Wunused-variable\"" ) + +# define CATCH_INTERNAL_SUPPRESS_ZERO_VARIADIC_WARNINGS \ + _Pragma( "clang diagnostic ignored \"-Wgnu-zero-variadic-macro-arguments\"" ) + +# define CATCH_INTERNAL_SUPPRESS_UNUSED_TEMPLATE_WARNINGS \ + _Pragma( "clang diagnostic ignored \"-Wunused-template\"" ) #endif // __clang__ @@ -164,6 +183,25 @@ namespace Catch { # define CATCH_CONFIG_COLOUR_NONE #endif +//////////////////////////////////////////////////////////////////////////////// +// Android somehow still does not support std::to_string +#if defined(__ANDROID__) +# define CATCH_INTERNAL_CONFIG_NO_CPP11_TO_STRING +# define CATCH_INTERNAL_CONFIG_ANDROID_LOGWRITE +#endif + +//////////////////////////////////////////////////////////////////////////////// +// Not all Windows environments support SEH properly +#if defined(__MINGW32__) +# define CATCH_INTERNAL_CONFIG_NO_WINDOWS_SEH +#endif + +//////////////////////////////////////////////////////////////////////////////// +// PS4 +#if defined(__ORBIS__) +# define CATCH_INTERNAL_CONFIG_NO_NEW_CAPTURE +#endif + //////////////////////////////////////////////////////////////////////////////// // Cygwin #ifdef __CYGWIN__ @@ -171,12 +209,22 @@ namespace Catch { // Required for some versions of Cygwin to declare gettimeofday // see: http://stackoverflow.com/questions/36901803/gettimeofday-not-declared-in-this-scope-cygwin # define _BSD_SOURCE +// some versions of cygwin (most) do not support std::to_string. Use the libstd check. +// https://gcc.gnu.org/onlinedocs/gcc-4.8.2/libstdc++/api/a01053_source.html line 2812-2813 +# if !((__cplusplus >= 201103L) && defined(_GLIBCXX_USE_C99) \ + && !defined(_GLIBCXX_HAVE_BROKEN_VSWPRINTF)) +# define CATCH_INTERNAL_CONFIG_NO_CPP11_TO_STRING + +# endif #endif // __CYGWIN__ //////////////////////////////////////////////////////////////////////////////// // Visual C++ -#ifdef _MSC_VER +#if defined(_MSC_VER) + +# define CATCH_INTERNAL_START_WARNINGS_SUPPRESSION __pragma( warning(push) ) +# define CATCH_INTERNAL_STOP_WARNINGS_SUPPRESSION __pragma( warning(pop) ) # if _MSC_VER >= 1900 // Visual Studio 2015 or newer # define CATCH_INTERNAL_CONFIG_CPP17_UNCAUGHT_EXCEPTIONS @@ -190,15 +238,37 @@ namespace Catch { # define CATCH_INTERNAL_CONFIG_WINDOWS_SEH # endif +// MSVC traditional preprocessor needs some workaround for __VA_ARGS__ +// _MSVC_TRADITIONAL == 0 means new conformant preprocessor +// _MSVC_TRADITIONAL == 1 means old traditional non-conformant preprocessor +# if !defined(_MSVC_TRADITIONAL) || (defined(_MSVC_TRADITIONAL) && _MSVC_TRADITIONAL) +# define CATCH_INTERNAL_CONFIG_TRADITIONAL_MSVC_PREPROCESSOR +# endif +#endif // _MSC_VER + +#if defined(_REENTRANT) || defined(_MSC_VER) +// Enable async processing, as -pthread is specified or no additional linking is required +# define CATCH_INTERNAL_CONFIG_USE_ASYNC #endif // _MSC_VER //////////////////////////////////////////////////////////////////////////////// +// Check if we are compiled with -fno-exceptions or equivalent +#if defined(__EXCEPTIONS) || defined(__cpp_exceptions) || defined(_CPPUNWIND) +# define CATCH_INTERNAL_CONFIG_EXCEPTIONS_ENABLED +#endif +//////////////////////////////////////////////////////////////////////////////// // DJGPP #ifdef __DJGPP__ # define CATCH_INTERNAL_CONFIG_NO_WCHAR #endif // __DJGPP__ +//////////////////////////////////////////////////////////////////////////////// +// Embarcadero C++Build +#if defined(__BORLANDC__) + #define CATCH_INTERNAL_CONFIG_POLYFILL_ISNAN +#endif + //////////////////////////////////////////////////////////////////////////////// // Use of __COUNTER__ is suppressed during code analysis in @@ -210,10 +280,59 @@ namespace Catch { #define CATCH_INTERNAL_CONFIG_COUNTER #endif +//////////////////////////////////////////////////////////////////////////////// + +// RTX is a special version of Windows that is real time. +// This means that it is detected as Windows, but does not provide +// the same set of capabilities as real Windows does. +#if defined(UNDER_RTSS) || defined(RTX64_BUILD) + #define CATCH_INTERNAL_CONFIG_NO_WINDOWS_SEH + #define CATCH_INTERNAL_CONFIG_NO_ASYNC + #define CATCH_CONFIG_COLOUR_NONE +#endif + +#if defined(__UCLIBC__) +#define CATCH_INTERNAL_CONFIG_GLOBAL_NEXTAFTER +#endif + +// Various stdlib support checks that require __has_include +#if defined(__has_include) + // Check if string_view is available and usable + #if __has_include() && defined(CATCH_CPP17_OR_GREATER) + # define CATCH_INTERNAL_CONFIG_CPP17_STRING_VIEW + #endif + + // Check if optional is available and usable + # if __has_include() && defined(CATCH_CPP17_OR_GREATER) + # define CATCH_INTERNAL_CONFIG_CPP17_OPTIONAL + # endif // __has_include() && defined(CATCH_CPP17_OR_GREATER) + + // Check if byte is available and usable + # if __has_include() && defined(CATCH_CPP17_OR_GREATER) + # define CATCH_INTERNAL_CONFIG_CPP17_BYTE + # endif // __has_include() && defined(CATCH_CPP17_OR_GREATER) + + // Check if variant is available and usable + # if __has_include() && defined(CATCH_CPP17_OR_GREATER) + # if defined(__clang__) && (__clang_major__ < 8) + // work around clang bug with libstdc++ https://bugs.llvm.org/show_bug.cgi?id=31852 + // fix should be in clang 8, workaround in libstdc++ 8.2 + # include + # if defined(__GLIBCXX__) && defined(_GLIBCXX_RELEASE) && (_GLIBCXX_RELEASE < 9) + # define CATCH_CONFIG_NO_CPP17_VARIANT + # else + # define CATCH_INTERNAL_CONFIG_CPP17_VARIANT + # endif // defined(__GLIBCXX__) && defined(_GLIBCXX_RELEASE) && (_GLIBCXX_RELEASE < 9) + # else + # define CATCH_INTERNAL_CONFIG_CPP17_VARIANT + # endif // defined(__clang__) && (__clang_major__ < 8) + # endif // __has_include() && defined(CATCH_CPP17_OR_GREATER) +#endif // defined(__has_include) + #if defined(CATCH_INTERNAL_CONFIG_COUNTER) && !defined(CATCH_CONFIG_NO_COUNTER) && !defined(CATCH_CONFIG_COUNTER) # define CATCH_CONFIG_COUNTER #endif -#if defined(CATCH_INTERNAL_CONFIG_WINDOWS_SEH) && !defined(CATCH_CONFIG_NO_WINDOWS_SEH) && !defined(CATCH_CONFIG_WINDOWS_SEH) +#if defined(CATCH_INTERNAL_CONFIG_WINDOWS_SEH) && !defined(CATCH_CONFIG_NO_WINDOWS_SEH) && !defined(CATCH_CONFIG_WINDOWS_SEH) && !defined(CATCH_INTERNAL_CONFIG_NO_WINDOWS_SEH) # define CATCH_CONFIG_WINDOWS_SEH #endif // This is set by default, because we assume that unix compilers are posix-signal-compatible by default. @@ -225,17 +344,101 @@ namespace Catch { # define CATCH_CONFIG_WCHAR #endif +#if !defined(CATCH_INTERNAL_CONFIG_NO_CPP11_TO_STRING) && !defined(CATCH_CONFIG_NO_CPP11_TO_STRING) && !defined(CATCH_CONFIG_CPP11_TO_STRING) +# define CATCH_CONFIG_CPP11_TO_STRING +#endif + +#if defined(CATCH_INTERNAL_CONFIG_CPP17_OPTIONAL) && !defined(CATCH_CONFIG_NO_CPP17_OPTIONAL) && !defined(CATCH_CONFIG_CPP17_OPTIONAL) +# define CATCH_CONFIG_CPP17_OPTIONAL +#endif + #if defined(CATCH_INTERNAL_CONFIG_CPP17_UNCAUGHT_EXCEPTIONS) && !defined(CATCH_CONFIG_NO_CPP17_UNCAUGHT_EXCEPTIONS) && !defined(CATCH_CONFIG_CPP17_UNCAUGHT_EXCEPTIONS) # define CATCH_CONFIG_CPP17_UNCAUGHT_EXCEPTIONS #endif +#if defined(CATCH_INTERNAL_CONFIG_CPP17_STRING_VIEW) && !defined(CATCH_CONFIG_NO_CPP17_STRING_VIEW) && !defined(CATCH_CONFIG_CPP17_STRING_VIEW) +# define CATCH_CONFIG_CPP17_STRING_VIEW +#endif + +#if defined(CATCH_INTERNAL_CONFIG_CPP17_VARIANT) && !defined(CATCH_CONFIG_NO_CPP17_VARIANT) && !defined(CATCH_CONFIG_CPP17_VARIANT) +# define CATCH_CONFIG_CPP17_VARIANT +#endif + +#if defined(CATCH_INTERNAL_CONFIG_CPP17_BYTE) && !defined(CATCH_CONFIG_NO_CPP17_BYTE) && !defined(CATCH_CONFIG_CPP17_BYTE) +# define CATCH_CONFIG_CPP17_BYTE +#endif + +#if defined(CATCH_CONFIG_EXPERIMENTAL_REDIRECT) +# define CATCH_INTERNAL_CONFIG_NEW_CAPTURE +#endif + +#if defined(CATCH_INTERNAL_CONFIG_NEW_CAPTURE) && !defined(CATCH_INTERNAL_CONFIG_NO_NEW_CAPTURE) && !defined(CATCH_CONFIG_NO_NEW_CAPTURE) && !defined(CATCH_CONFIG_NEW_CAPTURE) +# define CATCH_CONFIG_NEW_CAPTURE +#endif + +#if !defined(CATCH_INTERNAL_CONFIG_EXCEPTIONS_ENABLED) && !defined(CATCH_CONFIG_DISABLE_EXCEPTIONS) +# define CATCH_CONFIG_DISABLE_EXCEPTIONS +#endif + +#if defined(CATCH_INTERNAL_CONFIG_POLYFILL_ISNAN) && !defined(CATCH_CONFIG_NO_POLYFILL_ISNAN) && !defined(CATCH_CONFIG_POLYFILL_ISNAN) +# define CATCH_CONFIG_POLYFILL_ISNAN +#endif + +#if defined(CATCH_INTERNAL_CONFIG_USE_ASYNC) && !defined(CATCH_INTERNAL_CONFIG_NO_ASYNC) && !defined(CATCH_CONFIG_NO_USE_ASYNC) && !defined(CATCH_CONFIG_USE_ASYNC) +# define CATCH_CONFIG_USE_ASYNC +#endif + +#if defined(CATCH_INTERNAL_CONFIG_ANDROID_LOGWRITE) && !defined(CATCH_CONFIG_NO_ANDROID_LOGWRITE) && !defined(CATCH_CONFIG_ANDROID_LOGWRITE) +# define CATCH_CONFIG_ANDROID_LOGWRITE +#endif + +#if defined(CATCH_INTERNAL_CONFIG_GLOBAL_NEXTAFTER) && !defined(CATCH_CONFIG_NO_GLOBAL_NEXTAFTER) && !defined(CATCH_CONFIG_GLOBAL_NEXTAFTER) +# define CATCH_CONFIG_GLOBAL_NEXTAFTER +#endif + +// Even if we do not think the compiler has that warning, we still have +// to provide a macro that can be used by the code. +#if !defined(CATCH_INTERNAL_START_WARNINGS_SUPPRESSION) +# define CATCH_INTERNAL_START_WARNINGS_SUPPRESSION +#endif +#if !defined(CATCH_INTERNAL_STOP_WARNINGS_SUPPRESSION) +# define CATCH_INTERNAL_STOP_WARNINGS_SUPPRESSION +#endif #if !defined(CATCH_INTERNAL_SUPPRESS_PARENTHESES_WARNINGS) # define CATCH_INTERNAL_SUPPRESS_PARENTHESES_WARNINGS -# define CATCH_INTERNAL_UNSUPPRESS_PARENTHESES_WARNINGS #endif #if !defined(CATCH_INTERNAL_SUPPRESS_GLOBALS_WARNINGS) # define CATCH_INTERNAL_SUPPRESS_GLOBALS_WARNINGS -# define CATCH_INTERNAL_UNSUPPRESS_GLOBALS_WARNINGS +#endif +#if !defined(CATCH_INTERNAL_SUPPRESS_UNUSED_WARNINGS) +# define CATCH_INTERNAL_SUPPRESS_UNUSED_WARNINGS +#endif +#if !defined(CATCH_INTERNAL_SUPPRESS_ZERO_VARIADIC_WARNINGS) +# define CATCH_INTERNAL_SUPPRESS_ZERO_VARIADIC_WARNINGS +#endif + +#if defined(__APPLE__) && defined(__apple_build_version__) && (__clang_major__ < 10) +# undef CATCH_INTERNAL_SUPPRESS_UNUSED_TEMPLATE_WARNINGS +#elif defined(__clang__) && (__clang_major__ < 5) +# undef CATCH_INTERNAL_SUPPRESS_UNUSED_TEMPLATE_WARNINGS +#endif + +#if !defined(CATCH_INTERNAL_SUPPRESS_UNUSED_TEMPLATE_WARNINGS) +# define CATCH_INTERNAL_SUPPRESS_UNUSED_TEMPLATE_WARNINGS +#endif + +#if defined(CATCH_CONFIG_DISABLE_EXCEPTIONS) +#define CATCH_TRY if ((true)) +#define CATCH_CATCH_ALL if ((false)) +#define CATCH_CATCH_ANON(type) if ((false)) +#else +#define CATCH_TRY try +#define CATCH_CATCH_ALL catch (...) +#define CATCH_CATCH_ANON(type) catch (type) +#endif + +#if defined(CATCH_INTERNAL_CONFIG_TRADITIONAL_MSVC_PREPROCESSOR) && !defined(CATCH_CONFIG_NO_TRADITIONAL_MSVC_PREPROCESSOR) && !defined(CATCH_CONFIG_TRADITIONAL_MSVC_PREPROCESSOR) +#define CATCH_CONFIG_TRADITIONAL_MSVC_PREPROCESSOR #endif // end catch_compiler_capabilities.h @@ -251,6 +454,10 @@ namespace Catch { #include #include +// We need a dummy global operator<< so we can bring it into Catch namespace later +struct Catch_global_namespace_dummy {}; +std::ostream& operator<<(std::ostream&, Catch_global_namespace_dummy); + namespace Catch { struct CaseSensitive { enum Choice { @@ -277,12 +484,12 @@ namespace Catch { line( _line ) {} - SourceLineInfo( SourceLineInfo const& other ) = default; - SourceLineInfo( SourceLineInfo && ) = default; - SourceLineInfo& operator = ( SourceLineInfo const& ) = default; - SourceLineInfo& operator = ( SourceLineInfo && ) = default; + SourceLineInfo( SourceLineInfo const& other ) = default; + SourceLineInfo& operator = ( SourceLineInfo const& ) = default; + SourceLineInfo( SourceLineInfo&& ) noexcept = default; + SourceLineInfo& operator = ( SourceLineInfo&& ) noexcept = default; - bool empty() const noexcept; + bool empty() const noexcept { return file[0] == '\0'; } bool operator == ( SourceLineInfo const& other ) const noexcept; bool operator < ( SourceLineInfo const& other ) const noexcept; @@ -292,6 +499,11 @@ namespace Catch { std::ostream& operator << ( std::ostream& os, SourceLineInfo const& info ); + // Bring in operator<< from global namespace into Catch namespace + // This is necessary because the overload of operator<< above makes + // lookup stop at namespace Catch + using ::operator<<; + // Use this in variadic streaming macros to allow // >> +StreamEndStop // as well as @@ -318,9 +530,10 @@ namespace Catch { } // end namespace Catch #define CATCH_REGISTER_TAG_ALIAS( alias, spec ) \ + CATCH_INTERNAL_START_WARNINGS_SUPPRESSION \ CATCH_INTERNAL_SUPPRESS_GLOBALS_WARNINGS \ namespace{ Catch::RegistrarForTagAliases INTERNAL_CATCH_UNIQUE_NAME( AutoRegisterTagAlias )( alias, spec, CATCH_INTERNAL_LINEINFO ); } \ - CATCH_INTERNAL_UNSUPPRESS_GLOBALS_WARNINGS + CATCH_INTERNAL_STOP_WARNINGS_SUPPRESSION // end catch_tag_alias_autoregistrar.h // start catch_test_registry.h @@ -328,7 +541,6 @@ namespace Catch { // start catch_interfaces_testcase.h #include -#include namespace Catch { @@ -339,8 +551,6 @@ namespace Catch { virtual ~ITestInvoker(); }; - using ITestCasePtr = std::shared_ptr; - class TestCase; struct IConfig; @@ -350,6 +560,7 @@ namespace Catch { virtual std::vector const& getAllTestsSorted( IConfig const& config ) const = 0; }; + bool isThrowSafe( TestCase const& testCase, IConfig const& config ); bool matchTest( TestCase const& testCase, TestSpec const& testSpec, IConfig const& config ); std::vector filterTests( std::vector const& testCases, TestSpec const& testSpec, IConfig const& config ); std::vector const& getAllTestCasesSorted( IConfig const& config ); @@ -362,55 +573,30 @@ namespace Catch { #include #include #include +#include namespace Catch { - class StringData; - /// A non-owning string class (similar to the forthcoming std::string_view) /// Note that, because a StringRef may be a substring of another string, - /// it may not be null terminated. c_str() must return a null terminated - /// string, however, and so the StringRef will internally take ownership - /// (taking a copy), if necessary. In theory this ownership is not externally - /// visible - but it does mean (substring) StringRefs should not be shared between - /// threads. + /// it may not be null terminated. class StringRef { public: using size_type = std::size_t; + using const_iterator = const char*; private: - friend struct StringRefTestAccess; - - char const* m_start; - size_type m_size; - - char* m_data = nullptr; - - void takeOwnership(); - static constexpr char const* const s_empty = ""; - public: // construction/ assignment - StringRef() noexcept - : StringRef( s_empty, 0 ) - {} - - StringRef( StringRef const& other ) noexcept - : m_start( other.m_start ), - m_size( other.m_size ) - {} + char const* m_start = s_empty; + size_type m_size = 0; - StringRef( StringRef&& other ) noexcept - : m_start( other.m_start ), - m_size( other.m_size ), - m_data( other.m_data ) - { - other.m_data = nullptr; - } + public: // construction + constexpr StringRef() noexcept = default; StringRef( char const* rawChars ) noexcept; - StringRef( char const* rawChars, size_type size ) noexcept + constexpr StringRef( char const* rawChars, size_type size ) noexcept : m_start( rawChars ), m_size( size ) {} @@ -420,65 +606,333 @@ namespace Catch { m_size( stdString.size() ) {} - ~StringRef() noexcept { - delete[] m_data; - } - - auto operator = ( StringRef const &other ) noexcept -> StringRef& { - delete[] m_data; - m_data = nullptr; - m_start = other.m_start; - m_size = other.m_size; - return *this; + explicit operator std::string() const { + return std::string(m_start, m_size); } - operator std::string() const; - - void swap( StringRef& other ) noexcept; - public: // operators auto operator == ( StringRef const& other ) const noexcept -> bool; - auto operator != ( StringRef const& other ) const noexcept -> bool; + auto operator != (StringRef const& other) const noexcept -> bool { + return !(*this == other); + } - auto operator[] ( size_type index ) const noexcept -> char; + auto operator[] ( size_type index ) const noexcept -> char { + assert(index < m_size); + return m_start[index]; + } public: // named queries - auto empty() const noexcept -> bool { + constexpr auto empty() const noexcept -> bool { return m_size == 0; } - auto size() const noexcept -> size_type { + constexpr auto size() const noexcept -> size_type { return m_size; } - auto numberOfCharacters() const noexcept -> size_type; + // Returns the current start pointer. If the StringRef is not + // null-terminated, throws std::domain_exception auto c_str() const -> char const*; public: // substrings and searches - auto substr( size_type start, size_type size ) const noexcept -> StringRef; + // Returns a substring of [start, start + length). + // If start + length > size(), then the substring is [start, size()). + // If start > size(), then the substring is empty. + auto substr( size_type start, size_type length ) const noexcept -> StringRef; - // Returns the current start pointer. - // Note that the pointer can change when if the StringRef is a substring - auto currentData() const noexcept -> char const*; + // Returns the current start pointer. May not be null-terminated. + auto data() const noexcept -> char const*; - private: // ownership queries - may not be consistent between calls - auto isOwned() const noexcept -> bool; - auto isSubstring() const noexcept -> bool; - }; + constexpr auto isNullTerminated() const noexcept -> bool { + return m_start[m_size] == '\0'; + } - auto operator + ( StringRef const& lhs, StringRef const& rhs ) -> std::string; - auto operator + ( StringRef const& lhs, char const* rhs ) -> std::string; - auto operator + ( char const* lhs, StringRef const& rhs ) -> std::string; + public: // iterators + constexpr const_iterator begin() const { return m_start; } + constexpr const_iterator end() const { return m_start + m_size; } + }; auto operator += ( std::string& lhs, StringRef const& sr ) -> std::string&; auto operator << ( std::ostream& os, StringRef const& sr ) -> std::ostream&; - inline auto operator "" _sr( char const* rawChars, std::size_t size ) noexcept -> StringRef { + constexpr auto operator "" _sr( char const* rawChars, std::size_t size ) noexcept -> StringRef { return StringRef( rawChars, size ); } - } // namespace Catch +constexpr auto operator "" _catch_sr( char const* rawChars, std::size_t size ) noexcept -> Catch::StringRef { + return Catch::StringRef( rawChars, size ); +} + // end catch_stringref.h +// start catch_preprocessor.hpp + + +#define CATCH_RECURSION_LEVEL0(...) __VA_ARGS__ +#define CATCH_RECURSION_LEVEL1(...) CATCH_RECURSION_LEVEL0(CATCH_RECURSION_LEVEL0(CATCH_RECURSION_LEVEL0(__VA_ARGS__))) +#define CATCH_RECURSION_LEVEL2(...) CATCH_RECURSION_LEVEL1(CATCH_RECURSION_LEVEL1(CATCH_RECURSION_LEVEL1(__VA_ARGS__))) +#define CATCH_RECURSION_LEVEL3(...) CATCH_RECURSION_LEVEL2(CATCH_RECURSION_LEVEL2(CATCH_RECURSION_LEVEL2(__VA_ARGS__))) +#define CATCH_RECURSION_LEVEL4(...) CATCH_RECURSION_LEVEL3(CATCH_RECURSION_LEVEL3(CATCH_RECURSION_LEVEL3(__VA_ARGS__))) +#define CATCH_RECURSION_LEVEL5(...) CATCH_RECURSION_LEVEL4(CATCH_RECURSION_LEVEL4(CATCH_RECURSION_LEVEL4(__VA_ARGS__))) + +#ifdef CATCH_CONFIG_TRADITIONAL_MSVC_PREPROCESSOR +#define INTERNAL_CATCH_EXPAND_VARGS(...) __VA_ARGS__ +// MSVC needs more evaluations +#define CATCH_RECURSION_LEVEL6(...) CATCH_RECURSION_LEVEL5(CATCH_RECURSION_LEVEL5(CATCH_RECURSION_LEVEL5(__VA_ARGS__))) +#define CATCH_RECURSE(...) CATCH_RECURSION_LEVEL6(CATCH_RECURSION_LEVEL6(__VA_ARGS__)) +#else +#define CATCH_RECURSE(...) CATCH_RECURSION_LEVEL5(__VA_ARGS__) +#endif + +#define CATCH_REC_END(...) +#define CATCH_REC_OUT + +#define CATCH_EMPTY() +#define CATCH_DEFER(id) id CATCH_EMPTY() + +#define CATCH_REC_GET_END2() 0, CATCH_REC_END +#define CATCH_REC_GET_END1(...) CATCH_REC_GET_END2 +#define CATCH_REC_GET_END(...) CATCH_REC_GET_END1 +#define CATCH_REC_NEXT0(test, next, ...) next CATCH_REC_OUT +#define CATCH_REC_NEXT1(test, next) CATCH_DEFER ( CATCH_REC_NEXT0 ) ( test, next, 0) +#define CATCH_REC_NEXT(test, next) CATCH_REC_NEXT1(CATCH_REC_GET_END test, next) + +#define CATCH_REC_LIST0(f, x, peek, ...) , f(x) CATCH_DEFER ( CATCH_REC_NEXT(peek, CATCH_REC_LIST1) ) ( f, peek, __VA_ARGS__ ) +#define CATCH_REC_LIST1(f, x, peek, ...) , f(x) CATCH_DEFER ( CATCH_REC_NEXT(peek, CATCH_REC_LIST0) ) ( f, peek, __VA_ARGS__ ) +#define CATCH_REC_LIST2(f, x, peek, ...) f(x) CATCH_DEFER ( CATCH_REC_NEXT(peek, CATCH_REC_LIST1) ) ( f, peek, __VA_ARGS__ ) + +#define CATCH_REC_LIST0_UD(f, userdata, x, peek, ...) , f(userdata, x) CATCH_DEFER ( CATCH_REC_NEXT(peek, CATCH_REC_LIST1_UD) ) ( f, userdata, peek, __VA_ARGS__ ) +#define CATCH_REC_LIST1_UD(f, userdata, x, peek, ...) , f(userdata, x) CATCH_DEFER ( CATCH_REC_NEXT(peek, CATCH_REC_LIST0_UD) ) ( f, userdata, peek, __VA_ARGS__ ) +#define CATCH_REC_LIST2_UD(f, userdata, x, peek, ...) f(userdata, x) CATCH_DEFER ( CATCH_REC_NEXT(peek, CATCH_REC_LIST1_UD) ) ( f, userdata, peek, __VA_ARGS__ ) + +// Applies the function macro `f` to each of the remaining parameters, inserts commas between the results, +// and passes userdata as the first parameter to each invocation, +// e.g. CATCH_REC_LIST_UD(f, x, a, b, c) evaluates to f(x, a), f(x, b), f(x, c) +#define CATCH_REC_LIST_UD(f, userdata, ...) CATCH_RECURSE(CATCH_REC_LIST2_UD(f, userdata, __VA_ARGS__, ()()(), ()()(), ()()(), 0)) + +#define CATCH_REC_LIST(f, ...) CATCH_RECURSE(CATCH_REC_LIST2(f, __VA_ARGS__, ()()(), ()()(), ()()(), 0)) + +#define INTERNAL_CATCH_EXPAND1(param) INTERNAL_CATCH_EXPAND2(param) +#define INTERNAL_CATCH_EXPAND2(...) INTERNAL_CATCH_NO## __VA_ARGS__ +#define INTERNAL_CATCH_DEF(...) INTERNAL_CATCH_DEF __VA_ARGS__ +#define INTERNAL_CATCH_NOINTERNAL_CATCH_DEF +#define INTERNAL_CATCH_STRINGIZE(...) INTERNAL_CATCH_STRINGIZE2(__VA_ARGS__) +#ifndef CATCH_CONFIG_TRADITIONAL_MSVC_PREPROCESSOR +#define INTERNAL_CATCH_STRINGIZE2(...) #__VA_ARGS__ +#define INTERNAL_CATCH_STRINGIZE_WITHOUT_PARENS(param) INTERNAL_CATCH_STRINGIZE(INTERNAL_CATCH_REMOVE_PARENS(param)) +#else +// MSVC is adding extra space and needs another indirection to expand INTERNAL_CATCH_NOINTERNAL_CATCH_DEF +#define INTERNAL_CATCH_STRINGIZE2(...) INTERNAL_CATCH_STRINGIZE3(__VA_ARGS__) +#define INTERNAL_CATCH_STRINGIZE3(...) #__VA_ARGS__ +#define INTERNAL_CATCH_STRINGIZE_WITHOUT_PARENS(param) (INTERNAL_CATCH_STRINGIZE(INTERNAL_CATCH_REMOVE_PARENS(param)) + 1) +#endif + +#define INTERNAL_CATCH_MAKE_NAMESPACE2(...) ns_##__VA_ARGS__ +#define INTERNAL_CATCH_MAKE_NAMESPACE(name) INTERNAL_CATCH_MAKE_NAMESPACE2(name) + +#define INTERNAL_CATCH_REMOVE_PARENS(...) INTERNAL_CATCH_EXPAND1(INTERNAL_CATCH_DEF __VA_ARGS__) + +#ifndef CATCH_CONFIG_TRADITIONAL_MSVC_PREPROCESSOR +#define INTERNAL_CATCH_MAKE_TYPE_LIST2(...) decltype(get_wrapper()) +#define INTERNAL_CATCH_MAKE_TYPE_LIST(...) INTERNAL_CATCH_MAKE_TYPE_LIST2(INTERNAL_CATCH_REMOVE_PARENS(__VA_ARGS__)) +#else +#define INTERNAL_CATCH_MAKE_TYPE_LIST2(...) INTERNAL_CATCH_EXPAND_VARGS(decltype(get_wrapper())) +#define INTERNAL_CATCH_MAKE_TYPE_LIST(...) INTERNAL_CATCH_EXPAND_VARGS(INTERNAL_CATCH_MAKE_TYPE_LIST2(INTERNAL_CATCH_REMOVE_PARENS(__VA_ARGS__))) +#endif + +#define INTERNAL_CATCH_MAKE_TYPE_LISTS_FROM_TYPES(...)\ + CATCH_REC_LIST(INTERNAL_CATCH_MAKE_TYPE_LIST,__VA_ARGS__) + +#define INTERNAL_CATCH_REMOVE_PARENS_1_ARG(_0) INTERNAL_CATCH_REMOVE_PARENS(_0) +#define INTERNAL_CATCH_REMOVE_PARENS_2_ARG(_0, _1) INTERNAL_CATCH_REMOVE_PARENS(_0), INTERNAL_CATCH_REMOVE_PARENS_1_ARG(_1) +#define INTERNAL_CATCH_REMOVE_PARENS_3_ARG(_0, _1, _2) INTERNAL_CATCH_REMOVE_PARENS(_0), INTERNAL_CATCH_REMOVE_PARENS_2_ARG(_1, _2) +#define INTERNAL_CATCH_REMOVE_PARENS_4_ARG(_0, _1, _2, _3) INTERNAL_CATCH_REMOVE_PARENS(_0), INTERNAL_CATCH_REMOVE_PARENS_3_ARG(_1, _2, _3) +#define INTERNAL_CATCH_REMOVE_PARENS_5_ARG(_0, _1, _2, _3, _4) INTERNAL_CATCH_REMOVE_PARENS(_0), INTERNAL_CATCH_REMOVE_PARENS_4_ARG(_1, _2, _3, _4) +#define INTERNAL_CATCH_REMOVE_PARENS_6_ARG(_0, _1, _2, _3, _4, _5) INTERNAL_CATCH_REMOVE_PARENS(_0), INTERNAL_CATCH_REMOVE_PARENS_5_ARG(_1, _2, _3, _4, _5) +#define INTERNAL_CATCH_REMOVE_PARENS_7_ARG(_0, _1, _2, _3, _4, _5, _6) INTERNAL_CATCH_REMOVE_PARENS(_0), INTERNAL_CATCH_REMOVE_PARENS_6_ARG(_1, _2, _4, _5, _6) +#define INTERNAL_CATCH_REMOVE_PARENS_8_ARG(_0, _1, _2, _3, _4, _5, _6, _7) INTERNAL_CATCH_REMOVE_PARENS(_0), INTERNAL_CATCH_REMOVE_PARENS_7_ARG(_1, _2, _3, _4, _5, _6, _7) +#define INTERNAL_CATCH_REMOVE_PARENS_9_ARG(_0, _1, _2, _3, _4, _5, _6, _7, _8) INTERNAL_CATCH_REMOVE_PARENS(_0), INTERNAL_CATCH_REMOVE_PARENS_8_ARG(_1, _2, _3, _4, _5, _6, _7, _8) +#define INTERNAL_CATCH_REMOVE_PARENS_10_ARG(_0, _1, _2, _3, _4, _5, _6, _7, _8, _9) INTERNAL_CATCH_REMOVE_PARENS(_0), INTERNAL_CATCH_REMOVE_PARENS_9_ARG(_1, _2, _3, _4, _5, _6, _7, _8, _9) +#define INTERNAL_CATCH_REMOVE_PARENS_11_ARG(_0, _1, _2, _3, _4, _5, _6, _7, _8, _9, _10) INTERNAL_CATCH_REMOVE_PARENS(_0), INTERNAL_CATCH_REMOVE_PARENS_10_ARG(_1, _2, _3, _4, _5, _6, _7, _8, _9, _10) + +#define INTERNAL_CATCH_VA_NARGS_IMPL(_0, _1, _2, _3, _4, _5, _6, _7, _8, _9, _10, N, ...) N + +#define INTERNAL_CATCH_TYPE_GEN\ + template struct TypeList {};\ + template\ + constexpr auto get_wrapper() noexcept -> TypeList { return {}; }\ + template class...> struct TemplateTypeList{};\ + template class...Cs>\ + constexpr auto get_wrapper() noexcept -> TemplateTypeList { return {}; }\ + template\ + struct append;\ + template\ + struct rewrap;\ + template class, typename...>\ + struct create;\ + template class, typename>\ + struct convert;\ + \ + template \ + struct append { using type = T; };\ + template< template class L1, typename...E1, template class L2, typename...E2, typename...Rest>\ + struct append, L2, Rest...> { using type = typename append, Rest...>::type; };\ + template< template class L1, typename...E1, typename...Rest>\ + struct append, TypeList, Rest...> { using type = L1; };\ + \ + template< template class Container, template class List, typename...elems>\ + struct rewrap, List> { using type = TypeList>; };\ + template< template class Container, template class List, class...Elems, typename...Elements>\ + struct rewrap, List, Elements...> { using type = typename append>, typename rewrap, Elements...>::type>::type; };\ + \ + template
%s%d
%s%d
%s%d