From de30da422f08ed57d8a802613584d28d46730212 Mon Sep 17 00:00:00 2001 From: Sudheer Vinukonda Date: Thu, 10 Jun 2021 15:21:39 -0700 Subject: [PATCH] Add support for Remap rule hit stats This change adds stats to track hit rate for each remap rule (based on the map fromURL in the remap.config file) to allow for such visibility via a new http endpoint {remap_stats} which returns the stats in a json format More Context: Ordering of remap rules in remap.config in an efficient manner is very important as it can have a significant impact on performance and throughput that can be achieved using ATS as a proxy server. For example, ATS uses a prefix trie to match a forward map rule, but, has to run through the regex_map rules sequentially. Interleaving regex_map rules with forward map rules can thus result in unnecessary overhead when those rules are not overlapping. Further, within regex_map rules, as far as possible ordering them based on their "hit" rate (decreasing order) can result in a significant performance improvement. --- proxy/http/HttpProxyServerMain.cc | 2 ++ proxy/http/remap/Makefile.am | 2 ++ proxy/http/remap/RemapHitCount.cc | 47 +++++++++++++++++++++++++ proxy/http/remap/RemapHitCount.h | 30 ++++++++++++++++ proxy/http/remap/UrlMapping.cc | 7 ++++ proxy/http/remap/UrlMapping.h | 22 ++++++++++++ proxy/http/remap/UrlMappingPathIndex.cc | 13 +++++++ proxy/http/remap/UrlMappingPathIndex.h | 1 + proxy/http/remap/UrlRewrite.cc | 45 +++++++++++++++++++++++ proxy/http/remap/UrlRewrite.h | 2 ++ 10 files changed, 171 insertions(+) create mode 100644 proxy/http/remap/RemapHitCount.cc create mode 100644 proxy/http/remap/RemapHitCount.h diff --git a/proxy/http/HttpProxyServerMain.cc b/proxy/http/HttpProxyServerMain.cc index 0da59e095e1..1a38a8bd259 100644 --- a/proxy/http/HttpProxyServerMain.cc +++ b/proxy/http/HttpProxyServerMain.cc @@ -27,6 +27,7 @@ #include "HttpSessionAccept.h" #include "ReverseProxy.h" #include "HttpSessionManager.h" +#include "remap/RemapHitCount.h" #ifdef USE_HTTP_DEBUG_LISTS #include "Http1ClientSession.h" #endif @@ -374,6 +375,7 @@ start_HttpProxyServer() // Set up stat page for http connection count statPagesManager.register_http("connection_count", register_ShowConnectionCount); + statPagesManager.register_http("remap_hits", register_ShowRemapHitCount); // Alert plugins that connections will be accepted. APIHook *hook = lifecycle_hooks->get(TS_LIFECYCLE_PORTS_READY_HOOK); diff --git a/proxy/http/remap/Makefile.am b/proxy/http/remap/Makefile.am index 97e3e953758..dc51f477479 100644 --- a/proxy/http/remap/Makefile.am +++ b/proxy/http/remap/Makefile.am @@ -48,6 +48,8 @@ libhttp_remap_a_SOURCES = \ NextHopStrategyFactory.cc \ RemapConfig.cc \ RemapConfig.h \ + RemapHitCount.cc \ + RemapHitCount.h \ RemapPluginInfo.cc \ RemapPluginInfo.h \ PluginDso.cc \ diff --git a/proxy/http/remap/RemapHitCount.cc b/proxy/http/remap/RemapHitCount.cc new file mode 100644 index 00000000000..d72deb1cd77 --- /dev/null +++ b/proxy/http/remap/RemapHitCount.cc @@ -0,0 +1,47 @@ +/** @file + + Stats to track remap rule matches + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#include "RemapHitCount.h" +#include "UrlRewrite.h" + +extern UrlRewrite *rewrite_table; + +struct ShowRemapCount : public ShowCont { + ShowRemapCount(Continuation *c, HTTPHdr *h) : ShowCont(c, h) { SET_HANDLER(&ShowRemapCount::showHandler); } + int + showHandler(int event, Event *e) + { + auto table = rewrite_table->acquire(); + CHECK_SHOW(show(rewrite_table->PrintRemapHits().c_str())); + table->release(); + return completeJson(event, e); + } +}; + +Action * +register_ShowRemapHitCount(Continuation *c, HTTPHdr *h) +{ + ShowRemapCount *s = new ShowRemapCount(c, h); + this_ethread()->schedule_imm(s); + return &s->action; +} diff --git a/proxy/http/remap/RemapHitCount.h b/proxy/http/remap/RemapHitCount.h new file mode 100644 index 00000000000..9f006768da7 --- /dev/null +++ b/proxy/http/remap/RemapHitCount.h @@ -0,0 +1,30 @@ +/** @file + + Show endpoint for remap rule hits + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#pragma once + +#include "HTTP.h" +#include "I_EventSystem.h" +#include "Show.h" + +Action *register_ShowRemapHitCount(Continuation *, HTTPHdr *); diff --git a/proxy/http/remap/UrlMapping.cc b/proxy/http/remap/UrlMapping.cc index 4f1494b0729..9f26cd2f0f0 100644 --- a/proxy/http/remap/UrlMapping.cc +++ b/proxy/http/remap/UrlMapping.cc @@ -95,6 +95,13 @@ url_mapping::Print() const _plugin_inst_list.size()); } +std::string +url_mapping::PrintRemapHitCount() const +{ + std::string result = "{\"fromURL\": \"" + remapKey + "\", \"hit_count\": " + std::to_string(_hitCount) + "}"; + return result; +} + /** * **/ diff --git a/proxy/http/remap/UrlMapping.h b/proxy/http/remap/UrlMapping.h index a0482c14c2f..c826c4acebe 100644 --- a/proxy/http/remap/UrlMapping.h +++ b/proxy/http/remap/UrlMapping.h @@ -29,6 +29,7 @@ #include "tscore/ink_config.h" #include "AclFiltering.h" #include "URL.h" +#include "RemapHitCount.h" #include "RemapPluginInfo.h" #include "PluginFactory.h" #include "tscore/Regex.h" @@ -93,6 +94,7 @@ class url_mapping } void Print() const; + std::string PrintRemapHitCount() const; int from_path_len = 0; URL fromURL; @@ -112,6 +114,8 @@ class url_mapping acl_filter_rule *filter = nullptr; // acl filtering (list of rules) LINK(url_mapping, link); // For use with the main Queue linked list holding all the mapping std::shared_ptr strategy = nullptr; + std::string remapKey; + std::atomic _hitCount = 0; // counter can overflow int getRank() const @@ -124,6 +128,24 @@ class url_mapping _rank = rank; }; + void + setRemapKey() + { + remapKey = fromURL.string_get_ref(); + } + + const std::string & + getRemapKey() + { + return remapKey; + } + + void + incrementCount() + { + _hitCount++; + } + private: std::vector _plugin_inst_list; int _rank = 0; diff --git a/proxy/http/remap/UrlMappingPathIndex.cc b/proxy/http/remap/UrlMappingPathIndex.cc index 95a47ed66a6..ec8f454e728 100644 --- a/proxy/http/remap/UrlMappingPathIndex.cc +++ b/proxy/http/remap/UrlMappingPathIndex.cc @@ -91,3 +91,16 @@ UrlMappingPathIndex::Print() const m_trie.second->Print(); } } + +std::string +UrlMappingPathIndex::PrintUrlMappingPathIndex() const +{ + std::string result; + for (auto &m_trie : m_tries) { + for (auto const &node : *m_trie.second) { + result += node.PrintRemapHitCount(); + result += ",\n"; + } + } + return result; +} diff --git a/proxy/http/remap/UrlMappingPathIndex.h b/proxy/http/remap/UrlMappingPathIndex.h index 6e4febc3243..bb011e8aa91 100644 --- a/proxy/http/remap/UrlMappingPathIndex.h +++ b/proxy/http/remap/UrlMappingPathIndex.h @@ -38,6 +38,7 @@ class UrlMappingPathIndex bool Insert(url_mapping *mapping); url_mapping *Search(URL *request_url, int request_port, bool normal_search = true) const; void Print() const; + std::string PrintUrlMappingPathIndex() const; private: typedef Trie UrlMappingTrie; diff --git a/proxy/http/remap/UrlRewrite.cc b/proxy/http/remap/UrlRewrite.cc index bc81632d882..85dd19f5fdc 100644 --- a/proxy/http/remap/UrlRewrite.cc +++ b/proxy/http/remap/UrlRewrite.cc @@ -182,6 +182,47 @@ UrlRewrite::PrintStore(const MappingsStore &store) const } } +std::string +UrlRewrite::PrintRemapHits() +{ + std::string result; + result += PrintRemapHitsStore(forward_mappings); + result += PrintRemapHitsStore(reverse_mappings); + result += PrintRemapHitsStore(permanent_redirects); + result += PrintRemapHitsStore(temporary_redirects); + result += PrintRemapHitsStore(forward_mappings_with_recv_port); + + if (result.size() > 2) { + result.pop_back(); // remove the trailing \n + result.pop_back(); // remove the trailing , + result = "{\"list\": [\n" + result + " \n]}"; + } + + return result; +} + +/** Debugging method. */ +std::string +UrlRewrite::PrintRemapHitsStore(MappingsStore &store) +{ + std::string result; + if (store.hash_lookup) { + for (auto &it : *store.hash_lookup) { + result += it.second->PrintUrlMappingPathIndex(); + } + } + + if (!store.regex_list.empty()) { + forl_LL(RegexMapping, list_iter, store.regex_list) + { + result += list_iter->url_map->PrintRemapHitCount(); + result += ",\n"; + } + } + + return result; +} + /** If a remapping is found, returns a pointer to it otherwise NULL is returned. @@ -551,6 +592,7 @@ UrlRewrite::_addToStore(MappingsStore &store, url_mapping *new_mapping, RegexMap bool retval; new_mapping->setRank(count); // Use the mapping rules number count for rank + new_mapping->setRemapKey(); // Used for remap hit stats if (is_cur_mapping_regex) { store.regex_list.enqueue(reg_map); retval = true; @@ -766,6 +808,9 @@ UrlRewrite::_mappingLookup(MappingsStore &mappings, URL *request_url, int reques Debug("url_rewrite", "Using regex mapping with rank %d", (mapping_container.getMapping())->getRank()); retval = true; } + if (retval) { + (mapping_container.getMapping())->incrementCount(); + } return retval; } diff --git a/proxy/http/remap/UrlRewrite.h b/proxy/http/remap/UrlRewrite.h index 8567857c48f..aa5d32d25d6 100644 --- a/proxy/http/remap/UrlRewrite.h +++ b/proxy/http/remap/UrlRewrite.h @@ -146,6 +146,8 @@ class UrlRewrite : public RefCountObj void PerformACLFiltering(HttpTransact::State *s, url_mapping *mapping); void PrintStore(const MappingsStore &store) const; + std::string PrintRemapHits(); + std::string PrintRemapHitsStore(MappingsStore &store); void DestroyStore(MappingsStore &store)